From a2dd5e5ad9d647c5971c416bc9a610a8d8639c42 Mon Sep 17 00:00:00 2001
From: Alexander Chekunkov <chekunkov@gmail.com>
Date: Fri, 11 Jul 2014 16:21:57 +0300
Subject: [PATCH 0001/2974] Added referer to "Spider error processing" log
 message + fixed some pep8 issues

---
 scrapy/core/scraper.py | 47 +++++++++++++++++++++++++++---------------
 1 file changed, 30 insertions(+), 17 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index e5120ec0d..3409a0e7c 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -57,6 +57,7 @@ class Slot(object):
     def needs_backout(self):
         return self.active_size > self.max_active_size
 
+
 class Scraper(object):
 
     def __init__(self, crawler):
@@ -100,8 +101,8 @@ class Scraper(object):
             self._scrape_next(spider, slot)
             return _
         dfd.addBoth(finish_scraping)
-        dfd.addErrback(log.err, 'Scraper bug processing %s' % request, \
-            spider=spider)
+        dfd.addErrback(
+            log.err, 'Scraper bug processing %s' % request, spider=spider)
         self._scrape_next(spider, slot)
         return dfd
 
@@ -124,13 +125,13 @@ class Scraper(object):
         """Handle the different cases of request's result been a Response or a
         Failure"""
         if not isinstance(request_result, Failure):
-            return self.spidermw.scrape_response(self.call_spider, \
-                request_result, request, spider)
+            return self.spidermw.scrape_response(
+                self.call_spider, request_result, request, spider)
         else:
             # FIXME: don't ignore errors in spider middleware
             dfd = self.call_spider(request_result, request, spider)
-            return dfd.addErrback(self._log_download_errors, \
-                request_result, request, spider)
+            return dfd.addErrback(
+                self._log_download_errors, request_result, request, spider)
 
     def call_spider(self, result, request, spider):
         result.request = request
@@ -143,11 +144,21 @@ class Scraper(object):
         if isinstance(exc, CloseSpider):
             self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
             return
-        log.err(_failure, "Spider error processing %s" % request, spider=spider)
-        self.signals.send_catch_log(signal=signals.spider_error, failure=_failure, response=response, \
-            spider=spider)
-        self.crawler.stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \
-            spider=spider)
+        referer = request.headers.get('Referer')
+        log.err(
+            _failure,
+            "Spider error processing %s (referer: %s)" % (request, referer),
+            spider=spider
+        )
+        self.signals.send_catch_log(
+            signal=signals.spider_error,
+            failure=_failure, response=response,
+            spider=spider
+        )
+        self.crawler.stats.inc_value(
+            "spider_exceptions/%s" % _failure.value.__class__.__name__,
+            spider=spider
+        )
 
     def handle_spider_output(self, result, request, response, spider):
         if not result:
@@ -180,8 +191,8 @@ class Scraper(object):
         """Log and silence errors that come from the engine (typically download
         errors that got propagated thru here)
         """
-        if isinstance(download_failure, Failure) \
-                and not download_failure.check(IgnoreRequest):
+        if (isinstance(download_failure, Failure) and
+                not download_failure.check(IgnoreRequest)):
             if download_failure.frames:
                 log.err(download_failure, 'Error downloading %s' % request,
                         spider=spider)
@@ -204,13 +215,15 @@ class Scraper(object):
             if isinstance(ex, DropItem):
                 logkws = self.logformatter.dropped(item, ex, response, spider)
                 log.msg(spider=spider, **logkws)
-                return self.signals.send_catch_log_deferred(signal=signals.item_dropped, \
-                    item=item, response=response, spider=spider, exception=output.value)
+                return self.signals.send_catch_log_deferred(
+                    signal=signals.item_dropped, item=item, response=response,
+                    spider=spider, exception=output.value)
             else:
                 log.err(output, 'Error processing %s' % item, spider=spider)
         else:
             logkws = self.logformatter.scraped(output, response, spider)
             log.msg(spider=spider, **logkws)
-            return self.signals.send_catch_log_deferred(signal=signals.item_scraped, \
-                item=output, response=response, spider=spider)
+            return self.signals.send_catch_log_deferred(
+                signal=signals.item_scraped, item=output, response=response,
+                spider=spider)
 

From 84fa004793cb1be07c7a3d0ac6fd80a83b4e8487 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 30 Jun 2014 01:35:58 -0300
Subject: [PATCH 0002/2974] Add from_crawler class method to base Spider

---
 docs/topics/spiders.rst | 38 ++++++++++++++++++++++++++++++++++++++
 scrapy/spider.py        | 29 +++++++++++++++++++----------
 tests/test_spider.py    | 34 ++++++++++++++++++++++++++++++++--
 3 files changed, 89 insertions(+), 12 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 73c34e75f..de8f988c0 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -133,6 +133,44 @@ Spider
        listed here. The subsequent URLs will be generated successively from data
        contained in the start URLs.
 
+   .. attribute:: crawler
+
+      This attribute is set by the :meth:`from_crawler` class method after
+      initializating the class, and links to the
+      :class:`~scrapy.crawler.Crawler` object to which this spider instance is
+      bound.
+
+      Crawlers encapsulate a lot of components in the project for their single
+      entry access (such as extensions, middlewares, signals managers, etc).
+      See :ref:`topics-api-crawler` to know more about them.
+
+   .. attribute:: settings
+
+      Configuration on which this spider is been ran. This is a
+      :class:`~scrapy.settings.Settings` instance, see the
+      :ref:`topics-settings` topic for a detailed introduction on this subject.
+
+   .. method:: from_crawler(crawler, \*args, \**kwargs)
+
+       This is the class method used by Scrapy to create your spiders.
+
+       You probably won't need to override this directly, since the default
+       implementation acts as a proxy to the :meth:`__init__` method, calling
+       it with the given arguments `args` and named arguments `kwargs`.
+
+       Nonetheless, this method sets the :attr:`crawler` and :attr:`settings`
+       attributes in the new instance, so they can be accessed later inside the
+       spider's code.
+
+       :param crawler: crawler to which the spider will be bound
+       :type crawler: :class:`~scrapy.crawler.Crawler` instance
+
+       :param args: arguments passed to the :meth:`__init__` method
+       :type args: list
+
+       :param kwargs: keyword arguments passed to the :meth:`__init__` method
+       :type kwargs: dict
+
    .. method:: start_requests()
 
        This method must return an iterable with the first Requests to crawl for
diff --git a/scrapy/spider.py b/scrapy/spider.py
index 8ecfae2a0..89f78d6ba 100644
--- a/scrapy/spider.py
+++ b/scrapy/spider.py
@@ -3,11 +3,14 @@ Base class for Scrapy spiders
 
 See documentation in docs/topics/spiders.rst
 """
+import warnings
+
 from scrapy import log
 from scrapy.http import Request
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
 from scrapy.utils.deprecate import create_deprecated_class
+from scrapy.exceptions import ScrapyDeprecationWarning
 
 
 class Spider(object_ref):
@@ -32,18 +35,24 @@ class Spider(object_ref):
         """
         log.msg(message, spider=self, level=level, **kw)
 
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        spider = cls(*args, **kwargs)
+        spider._set_crawler(crawler)
+        return spider
+
     def set_crawler(self, crawler):
-        assert not hasattr(self, '_crawler'), "Spider already bounded to %s" % crawler
-        self._crawler = crawler
+        warnings.warn("set_crawler is deprecated, instantiate and bound the "
+                      "spider to this crawler with from_crawler method "
+                      "instead.",
+                      category=ScrapyDeprecationWarning, stacklevel=2)
+        assert not hasattr(self, 'crawler'), "Spider already bounded to a " \
+                                             "crawler"
+        self._set_crawler(crawler)
 
-    @property
-    def crawler(self):
-        assert hasattr(self, '_crawler'), "Spider not bounded to any crawler"
-        return self._crawler
-
-    @property
-    def settings(self):
-        return self.crawler.settings
+    def _set_crawler(self, crawler):
+        self.crawler = crawler
+        self.settings = crawler.settings
 
     def start_requests(self):
         for url in self.start_urls:
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 903eff7b1..53daf39fb 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -1,10 +1,12 @@
 import gzip
 import inspect
 import warnings
-from scrapy.utils.trackref import object_ref
 from io import BytesIO
-
 from twisted.trial import unittest
+try:
+    from unittest import mock
+except ImportError:
+    import mock
 
 from scrapy.spider import Spider, BaseSpider
 from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse
@@ -13,6 +15,8 @@ from scrapy.contrib.spiders import CrawlSpider, Rule, XMLFeedSpider, \
     CSVFeedSpider, SitemapSpider
 from scrapy.contrib.linkextractors import LinkExtractor
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.trackref import object_ref
+from scrapy.utils.test import get_crawler
 
 
 class SpiderTest(unittest.TestCase):
@@ -46,6 +50,32 @@ class SpiderTest(unittest.TestCase):
         self.assertRaises(ValueError, self.spider_class)
         self.assertRaises(ValueError, self.spider_class, somearg='foo')
 
+    def test_deprecated_set_crawler_method(self):
+        spider = self.spider_class('example.com')
+        crawler = get_crawler()
+        with warnings.catch_warnings(record=True) as w:
+            spider.set_crawler(crawler)
+            self.assertIn("set_crawler", str(w[0].message))
+            self.assertTrue(hasattr(spider, 'crawler'))
+            self.assertIs(spider.crawler, crawler)
+            self.assertTrue(hasattr(spider, 'settings'))
+            self.assertIs(spider.settings, crawler.settings)
+
+    def test_from_crawler_crawler_and_settings_population(self):
+        crawler = get_crawler()
+        spider = self.spider_class.from_crawler(crawler, 'example.com')
+        self.assertTrue(hasattr(spider, 'crawler'))
+        self.assertIs(spider.crawler, crawler)
+        self.assertTrue(hasattr(spider, 'settings'))
+        self.assertIs(spider.settings, crawler.settings)
+
+    def test_from_crawler_init_call(self):
+        with mock.patch.object(self.spider_class, '__init__',
+                               return_value=None) as mock_init:
+            self.spider_class.from_crawler(get_crawler(), 'example.com',
+                                           foo='bar')
+            mock_init.assert_called_once_with('example.com', foo='bar')
+
 
 class InitSpiderTest(SpiderTest):
 

From eb0253e5301ea54d4f37da8974d0dac295ebe871 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 30 Jun 2014 03:20:05 -0300
Subject: [PATCH 0003/2974] Update from_crawler method as well as set_crawler
 on CrawlSpider

---
 scrapy/contrib/spiders/crawl.py |  7 +++++++
 tests/test_spider.py            | 24 ++++++++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py
index d7e42f6f8..7dc3dacd6 100644
--- a/scrapy/contrib/spiders/crawl.py
+++ b/scrapy/contrib/spiders/crawl.py
@@ -86,6 +86,13 @@ class CrawlSpider(Spider):
             rule.process_links = get_method(rule.process_links)
             rule.process_request = get_method(rule.process_request)
 
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)
+        spider._follow_links = crawler.settings.getbool(
+            'CRAWLSPIDER_FOLLOW_LINKS', True)
+        return spider
+
     def set_crawler(self, crawler):
         super(CrawlSpider, self).set_crawler(crawler)
         self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 53daf39fb..188bef6e4 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -220,6 +220,30 @@ class CrawlSpiderTest(SpiderTest):
                            'http://example.org/about.html',
                            'http://example.org/nofollow.html'])
 
+    def test_follow_links_attribute_population(self):
+        crawler = get_crawler()
+        spider = self.spider_class.from_crawler(crawler, 'example.com')
+        self.assertTrue(hasattr(spider, '_follow_links'))
+        self.assertTrue(spider._follow_links)
+
+        crawler.settings.set('CRAWLSPIDER_FOLLOW_LINKS', False)
+        spider = self.spider_class.from_crawler(crawler, 'example.com')
+        self.assertTrue(hasattr(spider, '_follow_links'))
+        self.assertFalse(spider._follow_links)
+
+    def test_follow_links_attribute_deprecated_population(self):
+        spider = self.spider_class('example.com')
+        self.assertFalse(hasattr(spider, '_follow_links'))
+
+        spider.set_crawler(get_crawler())
+        self.assertTrue(hasattr(spider, '_follow_links'))
+        self.assertTrue(spider._follow_links)
+
+        spider = self.spider_class('example.com')
+        spider.set_crawler(get_crawler({'CRAWLSPIDER_FOLLOW_LINKS': False}))
+        self.assertTrue(hasattr(spider, '_follow_links'))
+        self.assertFalse(spider._follow_links)
+
 
 class SitemapSpiderTest(SpiderTest):
 

From a995727117d10133a20553a648e85970fc6a6543 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 17 Jul 2014 10:49:15 -0300
Subject: [PATCH 0004/2974] Connect spider_closed signal after a crawler is
 bound to a Spider

---
 scrapy/spider.py     |  8 ++++++++
 tests/test_spider.py | 16 ++++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/scrapy/spider.py b/scrapy/spider.py
index 89f78d6ba..df367b700 100644
--- a/scrapy/spider.py
+++ b/scrapy/spider.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/spiders.rst
 import warnings
 
 from scrapy import log
+from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
@@ -53,6 +54,7 @@ class Spider(object_ref):
     def _set_crawler(self, crawler):
         self.crawler = crawler
         self.settings = crawler.settings
+        crawler.signals.connect(self.close, signals.spider_closed)
 
     def start_requests(self):
         for url in self.start_urls:
@@ -68,6 +70,12 @@ class Spider(object_ref):
     def handles_request(cls, request):
         return url_is_from_spider(request.url, cls)
 
+    @staticmethod
+    def close(spider, reason):
+        closed = getattr(spider, 'closed', None)
+        if callable(closed):
+            return closed(reason)
+
     def __str__(self):
         return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self))
 
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 188bef6e4..903ea684a 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -8,6 +8,7 @@ try:
 except ImportError:
     import mock
 
+from scrapy import signals
 from scrapy.spider import Spider, BaseSpider
 from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse
 from scrapy.contrib.spiders.init import InitSpider
@@ -76,6 +77,21 @@ class SpiderTest(unittest.TestCase):
                                            foo='bar')
             mock_init.assert_called_once_with('example.com', foo='bar')
 
+    def test_closed_signal_call(self):
+        class TestSpider(self.spider_class):
+            closed_called = False
+
+            def closed(self, reason):
+                self.closed_called = True
+
+        crawler = get_crawler()
+        spider = TestSpider.from_crawler(crawler, 'example.com')
+        crawler.signals.send_catch_log(signal=signals.spider_opened,
+                                       spider=spider)
+        crawler.signals.send_catch_log(signal=signals.spider_closed,
+                                       spider=spider, reason=None)
+        self.assertTrue(spider.closed_called)
+
 
 class InitSpiderTest(SpiderTest):
 

From 3ae971468ff3a6712aa47ad3a5d2b9f0c9663b60 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 29 Jul 2014 18:47:49 -0300
Subject: [PATCH 0005/2974] Add Settings.copy, freeze and frozencopy method

---
 docs/topics/api.rst             | 24 ++++++++++++++++++++++
 scrapy/settings/__init__.py     | 16 +++++++++++++++
 tests/test_settings/__init__.py | 36 +++++++++++++++++++++++++++++++++
 3 files changed, 76 insertions(+)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 341340c2a..16bfe5f8f 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -264,6 +264,30 @@ Settings API
        :param default: the value to return if no setting is found
        :type default: any
 
+    .. method:: copy()
+
+       Make a deep copy of current settings.
+
+       This method returns a new instance of the :class:`Settings` class,
+       populated with the same values and their priorities.
+
+       Modifications to the new object won't be reflected on the original
+       settings.
+
+    .. method:: freeze()
+
+       Disable further changes to the current settings.
+
+       After calling this method, the present state of the settings will become
+       immutable. Trying to change values through the :meth:`~set` method and
+       its variants won't be possible and will be alerted.
+
+    .. method:: frozencopy()
+
+       Return an immutable copy of the current settings.
+
+       Alias for a :meth:`~freeze` call in the object returned by :meth:`copy`
+
 .. _topics-api-signals:
 
 Signals API
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 2dd6a2920..978174694 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,5 +1,6 @@
 import six
 import json
+import copy
 import warnings
 from collections import MutableMapping
 from importlib import import_module
@@ -46,6 +47,7 @@ class SettingsAttribute(object):
 class Settings(object):
 
     def __init__(self, values=None, priority='project'):
+        self.frozen = False
         self.attributes = {}
         self.setmodule(default_settings, priority='default')
         if values is not None:
@@ -93,6 +95,7 @@ class Settings(object):
         raise ValueError("Cannot convert value for setting '%s' to dict: '%s'" % (name, value))
 
     def set(self, name, value, priority='project'):
+        assert not self.frozen, "Trying to modify an immutable Settings object"
         if isinstance(priority, six.string_types):
             priority = SETTINGS_PRIORITIES[priority]
         if name not in self.attributes:
@@ -101,16 +104,29 @@ class Settings(object):
             self.attributes[name].set(value, priority)
 
     def setdict(self, values, priority='project'):
+        assert not self.frozen, "Trying to modify an immutable Settings object"
         for name, value in six.iteritems(values):
             self.set(name, value, priority)
 
     def setmodule(self, module, priority='project'):
+        assert not self.frozen, "Trying to modify an immutable Settings object"
         if isinstance(module, six.string_types):
             module = import_module(module)
         for key in dir(module):
             if key.isupper():
                 self.set(key, getattr(module, key), priority)
 
+    def copy(self):
+        return copy.deepcopy(self)
+
+    def freeze(self):
+        self.frozen = True
+
+    def frozencopy(self):
+        copy = self.copy()
+        copy.freeze()
+        return copy
+
     @property
     def overrides(self):
         warnings.warn("`Settings.overrides` attribute is deprecated and won't "
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 39e47dec6..c7e0914d6 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -190,6 +190,42 @@ class SettingsTest(unittest.TestCase):
         self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5})
         self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
 
+    def test_copy(self):
+        values = {
+            'TEST_BOOL': True,
+            'TEST_LIST': ['one', 'two'],
+            'TEST_LIST_OF_LISTS': [['first_one', 'first_two'],
+                                   ['second_one', 'second_two']]
+        }
+        self.settings.setdict(values)
+        copy = self.settings.copy()
+        self.settings.set('TEST_BOOL', False)
+        self.assertTrue(copy.get('TEST_BOOL'))
+
+        test_list = self.settings.get('TEST_LIST')
+        test_list.append('three')
+        self.assertListEqual(copy.get('TEST_LIST'), ['one', 'two'])
+
+        test_list_of_lists = self.settings.get('TEST_LIST_OF_LISTS')
+        test_list_of_lists[0].append('first_three')
+        self.assertListEqual(copy.get('TEST_LIST_OF_LISTS')[0],
+                             ['first_one', 'first_two'])
+
+    def test_freeze(self):
+        self.settings.freeze()
+        with self.assertRaises(AssertionError) as cm:
+            self.settings.set('TEST_BOOL', False)
+            self.assertEqual(str(cm.exception),
+                             "Trying to modify an immutable Settings object")
+
+    def test_frozencopy(self):
+        with mock.patch.object(self.settings, 'copy') as mock_copy:
+            with mock.patch.object(mock_copy, 'freeze') as mock_freeze:
+                mock_object = self.settings.frozencopy()
+                mock_copy.assert_call_once()
+                mock_freeze.assert_call_once()
+                self.assertEqual(mock_object, mock_copy.return_value)
+
     def test_deprecated_attribute_overrides(self):
         self.settings.set('BAR', 'fuz', priority='cmdline')
         with warnings.catch_warnings(record=True) as w:

From 39c6a80f9db6ec04cac59f116ee9620c3d540be0 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 1 Aug 2014 00:42:25 -0300
Subject: [PATCH 0006/2974] Both getdict and getlist return copies of the
 requested values

---
 docs/topics/api.rst         | 16 ++++++++++++++--
 scrapy/settings/__init__.py | 19 ++++++-------------
 2 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 16bfe5f8f..6e636e826 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -252,8 +252,8 @@ Settings API
 
     .. method:: getlist(name, default=None)
 
-       Get a setting value as a list. If the setting original type is a list it
-       will be returned verbatim. If it's a string it will be split by ",".
+       Get a setting value as a list. If the setting original type is a list, a
+       copy of it will be returned. If it's a string it will be split by ",".
 
        For example, settings populated through environment variables set to
        ``'one,two'`` will return a list ['one', 'two'] when using this method.
@@ -264,6 +264,18 @@ Settings API
        :param default: the value to return if no setting is found
        :type default: any
 
+    .. method:: getdict(name, default=None)
+
+       Get a setting value as a dictionary. If the setting original type is a
+       dictionary, a copy of it will be returned. If it's a string it will
+       evaluated as a json dictionary.
+
+       :param name: the setting name
+       :type name: string
+
+       :param default: the value to return if no setting is found
+       :type default: any
+
     .. method:: copy()
 
        Make a deep copy of current settings.
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 978174694..bbe8ef481 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -76,23 +76,16 @@ class Settings(object):
         return float(self.get(name, default))
 
     def getlist(self, name, default=None):
-        value = self.get(name)
-        if value is None:
-            return default or []
-        elif hasattr(value, '__iter__'):
-            return value
-        else:
-            return str(value).split(',')
+        value = self.get(name, default or [])
+        if isinstance(value, six.string_types):
+            value = value.split(',')
+        return list(value)
 
     def getdict(self, name, default=None):
-        value = self.get(name)
-        if value is None:
-            return default or {}
+        value = self.get(name, default or {})
         if isinstance(value, six.string_types):
             value = json.loads(value)
-        if isinstance(value, dict):
-            return value
-        raise ValueError("Cannot convert value for setting '%s' to dict: '%s'" % (name, value))
+        return dict(value)
 
     def set(self, name, value, priority='project'):
         assert not self.frozen, "Trying to modify an immutable Settings object"

From d7038b2a136a2b79df9fb16d7b3327cb29f9c46f Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 17 Jul 2014 10:25:07 -0300
Subject: [PATCH 0007/2974] SpiderManager interface cleanup

---
 docs/topics/api.rst                           | 51 ++++++++++++++++++-
 docs/topics/settings.rst                      | 10 ++++
 scrapy/interfaces.py                          | 10 ++--
 scrapy/spidermanager.py                       | 31 +++--------
 tests/test_spidermanager/__init__.py          | 27 +++++-----
 .../test_spiders/spider4.py                   | 10 ----
 6 files changed, 85 insertions(+), 54 deletions(-)
 delete mode 100644 tests/test_spidermanager/test_spiders/spider4.py

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 6e636e826..348305fb7 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -77,8 +77,7 @@ how you :ref:`configure the downloader middlewares
 
     .. attribute:: spiders
 
-        The spider manager which takes care of loading and instantiating
-        spiders.
+        The spider manager which takes care of loading spiders.
 
         Most extensions won't need to access this attribute.
 
@@ -300,6 +299,54 @@ Settings API
 
        Alias for a :meth:`~freeze` call in the object returned by :meth:`copy`
 
+.. _topics-api-spidermanager:
+
+SpiderManager API
+=================
+
+.. module:: scrapy.spidermanager
+   :synopsis: The spider manager
+
+.. class:: SpiderManager
+
+    This class is in charge of retrieving and handling the spider classes
+    defined across the project.
+
+    Custom spider managers can be employed by specifying their path in the
+    :setting:`SPIDER_MANAGER_CLASS` project setting. They must fully implement
+    the :class:`scrapy.interfaces.ISpiderManager` interface to guarantee an
+    errorless execution.
+
+    .. method:: from_settings(settings)
+
+       This class method is used by Scrapy to create an instance of the class.
+       It's called with the current project settings, and it loads the spiders
+       found in the modules of the :setting:`SPIDER_MODULES` setting.
+
+       :param settings: project settings
+       :type settings: :class:`~scrapy.settings.Settings` instance
+
+    .. method:: load(spider_name)
+
+       Get the Spider class with the given name. It'll look into the previously
+       loaded spiders for a spider class with name `spider_name` and will raise
+       a KeyError if not found.
+
+       :param spider_name: spider class name
+       :type spider_name: str
+
+    .. method:: list()
+
+       Get the names of the available spiders in the project.
+
+    .. method:: find_by_request(request)
+
+       List the spiders' names that can handle the given request. Will try to
+       match the request's url against the domains of the spiders.
+
+       :param request: queried request
+       :type request: :class:`~scrapy.http.Request` instance
+
 .. _topics-api-signals:
 
 Signals API
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 629fac2dd..8eb72eaea 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -768,6 +768,16 @@ A dict containing the scrapy contracts enabled by default in Scrapy. You should
 never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS`
 instead. For more info see :ref:`topics-contracts`.
 
+.. setting:: SPIDER_MANAGER_CLASS
+
+SPIDER_MANAGER_CLASS
+--------------------
+
+Default: ``'scrapy.spidermanager.SpiderManager'``
+
+The class that will be used for handling spiders, which must implement the
+:ref:`topics-api-spidermanager`.
+
 .. setting:: SPIDER_MIDDLEWARES
 
 SPIDER_MIDDLEWARES
diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py
index 5d8d85aca..d4596407e 100644
--- a/scrapy/interfaces.py
+++ b/scrapy/interfaces.py
@@ -2,10 +2,12 @@ from zope.interface import Interface
 
 class ISpiderManager(Interface):
 
-    def create(spider_name, **spider_args):
-        """Returns a new Spider instance for the given spider name, and using
-        the given spider arguments. If the spider name is not found, it must
-        raise a KeyError."""
+    def from_settings(settings):
+        """Returns an instance of the class for the given settings"""
+
+    def load(spider_name):
+        """Returns the Spider class for the given spider name. If the spider
+        name is not found, it must raise a KeyError."""
 
     def list():
         """Return a list with the names of all spiders available in the
diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py
index 5a0951cb4..5715b7793 100644
--- a/scrapy/spidermanager.py
+++ b/scrapy/spidermanager.py
@@ -6,7 +6,6 @@ spiders
 from zope.interface import implementer
 import six
 
-from scrapy import signals
 from scrapy.interfaces import ISpiderManager
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.spider import iter_spider_classes
@@ -15,8 +14,8 @@ from scrapy.utils.spider import iter_spider_classes
 @implementer(ISpiderManager)
 class SpiderManager(object):
 
-    def __init__(self, spider_modules):
-        self.spider_modules = spider_modules
+    def __init__(self, settings):
+        self.spider_modules = settings['SPIDER_MODULES']
         self._spiders = {}
         for name in self.spider_modules:
             for module in walk_modules(name):
@@ -28,33 +27,17 @@ class SpiderManager(object):
 
     @classmethod
     def from_settings(cls, settings):
-        return cls(settings.getlist('SPIDER_MODULES'))
+        return cls(settings)
 
-    @classmethod
-    def from_crawler(cls, crawler):
-        sm = cls.from_settings(crawler.settings)
-        sm.crawler = crawler
-        crawler.signals.connect(sm.close_spider, signals.spider_closed)
-        return sm
-
-    def create(self, spider_name, **spider_kwargs):
+    def load(self, spider_name):
         try:
-            spcls = self._spiders[spider_name]
+            return self._spiders[spider_name]
         except KeyError:
-            raise KeyError("Spider not found: %s" % spider_name)
-        if hasattr(self, 'crawler') and hasattr(spcls, 'from_crawler'):
-            return spcls.from_crawler(self.crawler, **spider_kwargs)
-        else:
-            return spcls(**spider_kwargs)
+            raise KeyError("Spider not found: {}".format(spider_name))
 
     def find_by_request(self, request):
         return [name for name, cls in six.iteritems(self._spiders)
             if cls.handles_request(request)]
 
     def list(self):
-        return self._spiders.keys()
-
-    def close_spider(self, spider, reason):
-        closed = getattr(spider, 'closed', None)
-        if callable(closed):
-            return closed(reason)
+        return list(self._spiders.keys())
diff --git a/tests/test_spidermanager/__init__.py b/tests/test_spidermanager/__init__.py
index b0dd9a851..69ab3b82a 100644
--- a/tests/test_spidermanager/__init__.py
+++ b/tests/test_spidermanager/__init__.py
@@ -10,6 +10,7 @@ from twisted.trial import unittest
 # alone
 from scrapy.interfaces import ISpiderManager
 from scrapy.spidermanager import SpiderManager
+from scrapy.settings import Settings
 from scrapy.http import Request
 
 module_dir = os.path.dirname(os.path.abspath(__file__))
@@ -23,7 +24,8 @@ class SpiderManagerTest(unittest.TestCase):
         self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
         shutil.copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(self.tmpdir)
-        self.spiderman = SpiderManager(['test_spiders_xxx'])
+        settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
+        self.spiderman = SpiderManager.from_settings(settings)
 
     def tearDown(self):
         del self.spiderman
@@ -35,14 +37,11 @@ class SpiderManagerTest(unittest.TestCase):
 
     def test_list(self):
         self.assertEqual(set(self.spiderman.list()),
-            set(['spider1', 'spider2', 'spider3', 'spider4']))
+            set(['spider1', 'spider2', 'spider3']))
 
-    def test_create(self):
-        spider1 = self.spiderman.create("spider1")
-        self.assertEqual(spider1.__class__.__name__, 'Spider1')
-        spider2 = self.spiderman.create("spider2", foo="bar")
-        self.assertEqual(spider2.__class__.__name__, 'Spider2')
-        self.assertEqual(spider2.foo, 'bar')
+    def test_load(self):
+        spider1 = self.spiderman.load("spider1")
+        self.assertEqual(spider1.__name__, 'Spider1')
 
     def test_find_by_request(self):
         self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy1.org/test')),
@@ -59,13 +58,13 @@ class SpiderManagerTest(unittest.TestCase):
             ['spider3'])
 
     def test_load_spider_module(self):
-        self.spiderman = SpiderManager(['tests.test_spidermanager.test_spiders.spider1'])
+        module = 'tests.test_spidermanager.test_spiders.spider1'
+        settings = Settings({'SPIDER_MODULES': [module]})
+        self.spiderman = SpiderManager.from_settings(settings)
         assert len(self.spiderman._spiders) == 1
 
     def test_load_base_spider(self):
-        self.spiderman = SpiderManager(['tests.test_spidermanager.test_spiders.spider0'])
+        module = 'tests.test_spidermanager.test_spiders.spider0'
+        settings = Settings({'SPIDER_MODULES': [module]})
+        self.spiderman = SpiderManager.from_settings(settings)
         assert len(self.spiderman._spiders) == 0
-
-    def test_load_from_crawler(self):
-        spider = self.spiderman.create('spider4', a='OK')
-        self.assertEqual(spider.a, 'OK')
diff --git a/tests/test_spidermanager/test_spiders/spider4.py b/tests/test_spidermanager/test_spiders/spider4.py
deleted file mode 100644
index e883e4d93..000000000
--- a/tests/test_spidermanager/test_spiders/spider4.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from scrapy.spider import Spider
-
-class Spider4(Spider):
-    name = "spider4"
-
-    @classmethod
-    def from_crawler(cls, crawler, **kwargs):
-        o = cls(**kwargs)
-        o.crawler = crawler
-        return o

From 980e30a18758f21f416036c2076bf98630c70193 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 29 Jul 2014 17:46:42 -0300
Subject: [PATCH 0008/2974] Crawler interface cleanup

---
 docs/topics/api.rst | 17 ++++++-----
 scrapy/crawler.py   | 70 ++++++++++++++++++++++-----------------------
 2 files changed, 44 insertions(+), 43 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 348305fb7..229943c55 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -28,9 +28,10 @@ contains a dictionary of all available extensions and their order similar to
 how you :ref:`configure the downloader middlewares
 <topics-downloader-middleware-setting>`.
 
-.. class:: Crawler(settings)
+.. class:: Crawler(spidercls, settings)
 
     The Crawler object must be instantiated with a
+    :class:`scrapy.spider.Spider` subclass and a
     :class:`scrapy.settings.Settings` object.
 
     .. attribute:: settings
@@ -90,16 +91,18 @@ how you :ref:`configure the downloader middlewares
         or modify the downloader and scheduler behaviour, although this is an
         advanced use and this API is not yet stable.
 
-    .. method:: configure()
+    .. attribute:: spider
 
-        Configure the crawler.
+        Spider currently being crawled. This is an instance of the spider class
+        provided while constructing the crawler, and it is created after the
+        arguments given in the :meth:`crawl` method.
 
-        This loads extensions, middlewares and spiders, leaving the crawler
-        ready to be started. It also configures the execution engine.
+    .. method:: crawl(\*args, \**kwargs)
 
-    .. method:: start()
+        Starts the crawler by instantiating its spider class with the given
+        `args` and `kwargs` arguments, while setting the execution engine in
+        motion.
 
-        Start the crawler. This calls :meth:`configure` if it hasn't been called yet.
         Returns a deferred that is fired when the crawl is finished.
 
 .. _topics-api-settings:
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index cfd6c8003..db1a083dd 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -13,16 +13,22 @@ from scrapy import log, signals
 
 class Crawler(object):
 
-    def __init__(self, settings):
-        self.configured = False
+    def __init__(self, spidercls, settings):
+        self.spidercls = spidercls
         self.settings = settings
         self.signals = SignalManager(self)
-        self.stats = load_object(settings['STATS_CLASS'])(self)
-        self._start_requests = lambda: ()
-        self._spider = None
-        # TODO: move SpiderManager to CrawlerProcess
+        self.stats = load_object(self.settings['STATS_CLASS'])(self)
+        lf_cls = load_object(self.settings['LOG_FORMATTER'])
+        self.logformatter = lf_cls.from_crawler(self)
+        self.extensions = ExtensionManager.from_crawler(self)
+
+        # Attribute kept for backward compatibility (Use CrawlerRunner.spiders)
         spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])
-        self.spiders = spman_cls.from_crawler(self)
+        self.spiders = spman_cls.from_settings(self.settings)
+
+        self.crawling = False
+        self.spider = None
+        self.engine = None
 
     def install(self):
         # TODO: remove together with scrapy.project.crawler usage
@@ -36,39 +42,31 @@ class Crawler(object):
         assert hasattr(scrapy.project, 'crawler'), "crawler not installed"
         del scrapy.project.crawler
 
-    def configure(self):
-        if self.configured:
-            return
-
-        self.configured = True
-        lf_cls = load_object(self.settings['LOG_FORMATTER'])
-        self.logformatter = lf_cls.from_crawler(self)
-        self.extensions = ExtensionManager.from_crawler(self)
-        self.engine = ExecutionEngine(self, self._spider_closed)
-
-    def crawl(self, spider, requests=None):
-        assert self._spider is None, 'Spider already attached'
-        self._spider = spider
-        spider.set_crawler(self)
-        if requests is None:
-            self._start_requests = spider.start_requests
-        else:
-            self._start_requests = lambda: requests
-
-    def _spider_closed(self, spider=None):
-        if not self.engine.open_spiders:
-            self.stop()
-
     @defer.inlineCallbacks
-    def start(self):
-        yield defer.maybeDeferred(self.configure)
-        if self._spider:
-            yield self.engine.open_spider(self._spider, self._start_requests())
-        yield defer.maybeDeferred(self.engine.start)
+    def crawl(self, *args, **kwargs):
+        assert not self.crawling, "Crawling already taking place"
+        self.crawling = True
+
+        try:
+            self.spider = self._create_spider(*args, **kwargs)
+            self.engine = self._create_engine()
+            start_requests = iter(self.spider.start_requests())
+            yield self.engine.open_spider(self.spider, start_requests)
+            yield defer.maybeDeferred(self.engine.start)
+        except Exception:
+            self.crawling = False
+            raise
+
+    def _create_spider(self, *args, **kwargs):
+        return self.spidercls.from_crawler(self, *args, **kwargs)
+
+    def _create_engine(self):
+        return ExecutionEngine(self, lambda _: self.stop())
 
     @defer.inlineCallbacks
     def stop(self):
-        if self.configured and self.engine.running:
+        if self.crawling:
+            self.crawling = False
             yield defer.maybeDeferred(self.engine.stop)
 
 

From d40273561dad76a409f847b5f8ce1daafdb1dc7c Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 30 Jul 2014 05:35:18 -0300
Subject: [PATCH 0009/2974] CrawlerProcess cleanup changes

---
 docs/topics/api.rst       | 48 +++++++++++++++++++
 docs/topics/practices.rst | 76 ++++++++++++++++++------------
 scrapy/crawler.py         | 98 ++++++++++++++++++++-------------------
 3 files changed, 144 insertions(+), 78 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 229943c55..0329e2a8f 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -105,6 +105,53 @@ how you :ref:`configure the downloader middlewares
 
         Returns a deferred that is fired when the crawl is finished.
 
+.. class:: CrawlerRunner(settings)
+
+    This is a convenient helper class that creates, configures and runs
+    crawlers inside an already setup Twisted `reactor`_.
+
+    The CrawlerRunner object must be instantiated with a
+    :class:`~scrapy.settings.Settings` object.
+
+    This class shouldn't be needed (since Scrapy is responsible of using it
+    accordingly) unless writing scripts that manually handle the crawling
+    process. See :ref:`run-from-script` for an example.
+
+    .. attribute:: crawlers
+
+       Set of :class:`crawlers <scrapy.crawler.Crawler>` created by the
+       :meth:`crawl` method.
+
+    .. attribute:: crawl_deferreds
+
+       Set of the `deferreds`_ return by the :meth:`crawl` method. This
+       collection it's useful for keeping track of current crawling state.
+
+    .. method:: crawl(spidercls, \*args, \**kwargs)
+
+       This method sets up the crawling of the given `spidercls` with the
+       provided arguments.
+
+       It takes care of loading the spider class while configuring and starting
+       a crawler for it.
+
+       Returns a deferred that is fired when the crawl is finished.
+
+       :param spidercls: spider class or spider's name inside the project
+       :type spidercls: :class:`~scrapy.spider.Spider` subclass or str
+
+       :param args: arguments to initializate the spider
+       :type args: list
+
+       :param kwargs: keyword arguments to initializate the spider
+       :type kwargs: dict
+
+    .. method:: stop()
+
+       Stops simultaneously all the crawling jobs taking place.
+
+       Returns a deferred that is fired when they all have ended.
+
 .. _topics-api-settings:
 
 Settings API
@@ -470,3 +517,4 @@ class (which they all inherit from).
 
 .. _deferreds: http://twistedmatrix.com/documents/current/core/howto/defer.html
 .. _deferred: http://twistedmatrix.com/documents/current/core/howto/defer.html
+.. _reactor: http://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 64b3a2da7..e84478d3c 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -19,8 +19,9 @@ Remember that Scrapy is built on top of the Twisted
 asynchronous networking library, so you need to run it inside the Twisted reactor.
 
 Note that you will also have to shutdown the Twisted reactor yourself after the
-spider is finished. This can be achieved by connecting a handler to the
-``signals.spider_closed`` signal.
+spider is finished. This can be achieved by adding callbacks to the deferred
+returned by the :meth:`CrawlerRunner.crawl
+<scrapy.crawler.CrawlerRunner.crawl>` method.
 
 What follows is a working example of how to do that, using the `testspiders`_
 project as example.
@@ -28,20 +29,21 @@ project as example.
 ::
 
     from twisted.internet import reactor
-    from scrapy.crawler import Crawler
-    from scrapy import log, signals
-    from testspiders.spiders.followall import FollowAllSpider
+    from scrapy.crawler import CrawlerRunner
     from scrapy.utils.project import get_project_settings
 
-    spider = FollowAllSpider(domain='scrapinghub.com')
-    settings = get_project_settings()
-    crawler = Crawler(settings)
-    crawler.signals.connect(reactor.stop, signal=signals.spider_closed)
-    crawler.configure()
-    crawler.crawl(spider)
-    crawler.start()
-    log.start()
-    reactor.run() # the script will block here until the spider_closed signal was sent
+    # If you aren't inside a Scrapy project, you could use an instance of the
+    # Settings class in scrapy.settings instead of the configuration returned
+    # by get_project_settings
+    runner = CrawlerRunner(get_project_settings())
+
+    # 'followall' is the name of one of the spiders of the project. If you
+    # aren't working in a Scrapy project, use the spider class as first
+    # argument instead of its name (or set the SPIDER_MODULES setting so Scrapy
+    # knows where to look at)
+    d = runner.crawl('followall', domain='scrapinghub.com')
+    d.addBoth(lambda _: reactor.stop())
+    reactor.run() # the script will block here until the crawling is finished
 
 .. seealso:: `Twisted Reactor Overview`_.
 
@@ -52,28 +54,42 @@ By default, Scrapy runs a single spider per process when you run ``scrapy
 crawl``. However, Scrapy supports running multiple spiders per process using
 the :ref:`internal API <topics-api>`.
 
-Here is an example, using the `testspiders`_ project:
+Here is an example that runs multiple spiders simultaneously, using the
+`testspiders`_ project:
 
 ::
 
-    from twisted.internet import reactor
-    from scrapy.crawler import Crawler
-    from scrapy import log
-    from testspiders.spiders.followall import FollowAllSpider
+    from twisted.internet import reactor, defer
+    from scrapy.crawler import CrawlerRunner
     from scrapy.utils.project import get_project_settings
 
-    def setup_crawler(domain):
-        spider = FollowAllSpider(domain=domain)
-        settings = get_project_settings()
-        crawler = Crawler(settings)
-        crawler.configure()
-        crawler.crawl(spider)
-        crawler.start()
-
+    runner = CrawlerRunner(get_project_settings())
+    dfs = set()
     for domain in ['scrapinghub.com', 'insophia.com']:
-        setup_crawler(domain)
-    log.start()
-    reactor.run()
+        d = runner.crawl('followall', domain=domain)
+        dfs.add(d)
+
+    defer.DeferredList(dfs).addBoth(lambda _: reactor.stop())
+    reactor.run() # the script will block here until all crawling jobs are finished
+
+Same example but running the spiders sequentially by chaining the deferreds:
+
+::
+
+    from twisted.internet import reactor, defer
+    from scrapy.crawler import CrawlerRunner
+    from scrapy.utils.project import get_project_settings
+
+    runner = CrawlerRunner(get_project_settings())
+
+    @defer.inlineCallbacks
+    def crawl():
+        for domain in ['scrapinghub.com', 'insophia.com']:
+            yield runner.crawl('followall', domain=domain)
+        reactor.stop()
+
+    crawl()
+    reactor.run() # the script will block here until the last crawl call is finished
 
 .. seealso:: :ref:`run-from-script`.
 
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index db1a083dd..56823166b 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,3 +1,4 @@
+import six
 import signal
 
 from twisted.internet import reactor, defer
@@ -70,31 +71,50 @@ class Crawler(object):
             yield defer.maybeDeferred(self.engine.stop)
 
 
-class CrawlerProcess(object):
-    """ A class to run multiple scrapy crawlers in a process sequentially"""
+class CrawlerRunner(object):
 
     def __init__(self, settings):
-        install_shutdown_handlers(self._signal_shutdown)
         self.settings = settings
-        self.crawlers = {}
-        self.stopping = False
-        self._started = None
+        smcls = load_object(settings['SPIDER_MANAGER_CLASS'])
+        self.spiders = smcls.from_settings(settings.frozencopy())
+        self.crawlers = set()
+        self.crawl_deferreds = set()
 
-    def create_crawler(self, name=None):
-        if name not in self.crawlers:
-            self.crawlers[name] = Crawler(self.settings)
+    def crawl(self, spidercls, *args, **kwargs):
+        crawler = self._create_logged_crawler(spidercls)
+        self.crawlers.add(crawler)
 
-        return self.crawlers[name]
+        crawler.install()
+        crawler.signals.connect(crawler.uninstall, signals.engine_stopped)
 
-    def start(self):
-        if self.start_crawling():
-            self.start_reactor()
+        d = crawler.crawl(*args, **kwargs)
+        self.crawl_deferreds.add(d)
+        return d
+
+    def _create_logged_crawler(self, spidercls):
+        crawler = self._create_crawler(spidercls)
+        log_observer = log.start_from_crawler(crawler)
+        if log_observer:
+            crawler.signals.connect(log_observer.stop, signals.engine_stopped)
+        return crawler
+
+    def _create_crawler(self, spidercls):
+        if isinstance(spidercls, six.string_types):
+            spidercls = self.spiders.load(spidercls)
+        crawler = Crawler(spidercls, self.settings.frozencopy())
+        return crawler
 
-    @defer.inlineCallbacks
     def stop(self):
-        self.stopping = True
-        if self._active_crawler:
-            yield self._active_crawler.stop()
+        return defer.DeferredList(c.stop() for c in self.crawlers)
+
+
+class CrawlerProcess(CrawlerRunner):
+    """A class to run multiple scrapy crawlers in a process simultaneously"""
+
+    def __init__(self, settings):
+        super(CrawlerProcess, self).__init__(settings)
+        install_shutdown_handlers(self._signal_shutdown)
+        self.stopping = False
 
     def _signal_shutdown(self, signum, _):
         install_shutdown_handlers(self._signal_kill)
@@ -110,43 +130,25 @@ class CrawlerProcess(object):
                 level=log.INFO, signame=signame)
         reactor.callFromThread(self._stop_reactor)
 
-    # ------------------------------------------------------------------------#
-    # The following public methods can't be considered stable and may change at
-    # any moment.
-    #
-    # start_crawling and start_reactor are called from scrapy.commands.shell
-    # They are splitted because reactor is started on a different thread than IPython shell.
-    #
-    def start_crawling(self):
-        log.scrapy_info(self.settings)
-        return self._start_crawler() is not None
+    def start(self, stop_after_crawl=True):
+        self._start_logging()
+        self._start_reactor(stop_after_crawl)
 
-    def start_reactor(self):
+    def _start_logging(self):
+        log.scrapy_info(self.settings)
+
+    def _start_reactor(self, stop_after_crawl=True):
+        if stop_after_crawl:
+            d = defer.DeferredList(self.crawl_deferreds)
+            if d.called:
+                # Don't start the reactor if the deferreds are already fired
+                return
+            d.addBoth(lambda _: self._stop_reactor())
         if self.settings.getbool('DNSCACHE_ENABLED'):
             reactor.installResolver(CachingThreadedResolver(reactor))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
         reactor.run(installSignalHandlers=False)  # blocking call
 
-    def _start_crawler(self):
-        if not self.crawlers or self.stopping:
-            return
-
-        name, crawler = self.crawlers.popitem()
-        self._active_crawler = crawler
-        log_observer = log.start_from_crawler(crawler)
-        crawler.configure()
-        crawler.install()
-        crawler.signals.connect(crawler.uninstall, signals.engine_stopped)
-        if log_observer:
-            crawler.signals.connect(log_observer.stop, signals.engine_stopped)
-        crawler.signals.connect(self._check_done, signals.engine_stopped)
-        crawler.start()
-        return name, crawler
-
-    def _check_done(self, **kwargs):
-        if not self._start_crawler():
-            self._stop_reactor()
-
     def _stop_reactor(self, _=None):
         try:
             reactor.stop()

From 870438e5f4665de836f7ff423055895b305a4e7f Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 31 Jul 2014 04:12:12 -0300
Subject: [PATCH 0010/2974] Update tests utils, fixing get_crawler and removing
 docrawl

---
 scrapy/utils/test.py                          |  15 +-
 tests/py3-ignores.txt                         |   5 +
 tests/test_closespider.py                     |  38 +++---
 tests/test_crawl.py                           | 128 +++++++++---------
 tests/test_downloader_handlers.py             |   9 +-
 tests/test_downloadermiddleware.py            |   5 +-
 ...test_downloadermiddleware_ajaxcrawlable.py |   4 +-
 ...est_downloadermiddleware_defaultheaders.py |   5 +-
 ...st_downloadermiddleware_downloadtimeout.py |   5 +-
 tests/test_downloadermiddleware_httpcache.py  |   4 +-
 tests/test_downloadermiddleware_redirect.py   |   8 +-
 tests/test_downloadermiddleware_retry.py      |   4 +-
 tests/test_downloadermiddleware_stats.py      |   4 +-
 tests/test_downloadermiddleware_useragent.py  |   5 +-
 tests/test_engine.py                          |   8 +-
 tests/test_proxy_connect.py                   |  28 ++--
 tests/test_spider.py                          |   6 +-
 tests/test_spidermiddleware_depth.py          |   5 +-
 tests/test_spidermiddleware_httperror.py      |  22 +--
 tests/test_spidermiddleware_offsite.py        |  14 +-
 tests/test_stats.py                           |   4 +-
 21 files changed, 160 insertions(+), 166 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index e6376d519..a4b769970 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -20,15 +20,17 @@ def assert_aws_environ():
     if 'AWS_ACCESS_KEY_ID' not in os.environ:
         raise SkipTest("AWS keys not found")
 
-def get_crawler(settings_dict=None):
+def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level
     priority.
     """
-    from scrapy.crawler import Crawler
+    from scrapy.crawler import CrawlerRunner
     from scrapy.settings import Settings
+    from scrapy.spider import Spider
 
-    return Crawler(Settings(settings_dict))
+    runner = CrawlerRunner(Settings(settings_dict))
+    return runner._create_crawler(spidercls or Spider)
 
 def get_pythonpath():
     """Return a PYTHONPATH suitable to use in processes so that they find this
@@ -62,10 +64,3 @@ def assert_samelines(testcase, text1, text2, msg=None):
     line endings between platforms
     """
     testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
-
-def docrawl(spider, settings=None):
-    """Configure and start Crawler; return the result of crawler.start()"""
-    crawler = get_crawler(settings)
-    crawler.configure()
-    crawler.crawl(spider)
-    return crawler.start()
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index c1619b3ae..ef88eab7e 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -14,6 +14,7 @@ tests/test_downloadermiddleware_ajaxcrawlable.py
 tests/test_downloadermiddleware_cookies.py
 tests/test_downloadermiddleware_decompression.py
 tests/test_downloadermiddleware_defaultheaders.py
+tests/test_downloadermiddleware_downloadtimeout.py
 tests/test_downloadermiddleware_httpauth.py
 tests/test_downloadermiddleware_httpcache.py
 tests/test_downloadermiddleware_httpcompression.py
@@ -22,6 +23,7 @@ tests/test_downloadermiddleware.py
 tests/test_downloadermiddleware_redirect.py
 tests/test_downloadermiddleware_retry.py
 tests/test_downloadermiddleware_robotstxt.py
+tests/test_downloadermiddleware_stats.py
 tests/test_downloadermiddleware_useragent.py
 tests/test_dupefilter.py
 tests/test_engine.py
@@ -48,9 +50,12 @@ tests/test_spidermanager/test_spiders/spider1.py
 tests/test_spidermanager/test_spiders/spider2.py
 tests/test_spidermanager/test_spiders/spider3.py
 tests/test_spidermanager/test_spiders/spider4.py
+tests/test_spidermiddleware_depth.py
 tests/test_spidermiddleware_httperror.py
+tests/test_spidermiddleware_offsite.py
 tests/test_spidermiddleware_referer.py
 tests/test_spider.py
+tests/test_stats.py
 tests/test_utils_defer.py
 tests/test_utils_iterators.py
 tests/test_utils_jsonrpc.py
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 8d30a4643..1700a861e 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -1,6 +1,6 @@
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
-from scrapy.utils.test import docrawl
+from scrapy.utils.test import get_crawler
 from tests.spiders import FollowAllSpider, ItemSpider, ErrorSpider
 from tests.mockserver import MockServer
 
@@ -16,45 +16,45 @@ class TestCloseSpider(TestCase):
 
     @defer.inlineCallbacks
     def test_closespider_itemcount(self):
-        spider = ItemSpider()
         close_on = 5
-        yield docrawl(spider, {'CLOSESPIDER_ITEMCOUNT': close_on})
-        reason = spider.meta['close_reason']
+        crawler = get_crawler(ItemSpider, {'CLOSESPIDER_ITEMCOUNT': close_on})
+        yield crawler.crawl()
+        reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_itemcount')
-        itemcount = spider.crawler.stats.get_value('item_scraped_count')
+        itemcount = crawler.stats.get_value('item_scraped_count')
         self.assertTrue(itemcount >= close_on)
 
     @defer.inlineCallbacks
     def test_closespider_pagecount(self):
-        spider = FollowAllSpider()
         close_on = 5
-        yield docrawl(spider, {'CLOSESPIDER_PAGECOUNT': close_on})
-        reason = spider.meta['close_reason']
+        crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_PAGECOUNT': close_on})
+        yield crawler.crawl()
+        reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_pagecount')
-        pagecount = spider.crawler.stats.get_value('response_received_count')
+        pagecount = crawler.stats.get_value('response_received_count')
         self.assertTrue(pagecount >= close_on)
 
     @defer.inlineCallbacks
     def test_closespider_errorcount(self):
-        spider = ErrorSpider(total=1000000)
         close_on = 5
-        yield docrawl(spider, {'CLOSESPIDER_ERRORCOUNT': close_on})
-        self.flushLoggedErrors(spider.exception_cls)
-        reason = spider.meta['close_reason']
+        crawler = get_crawler(ErrorSpider, {'CLOSESPIDER_ERRORCOUNT': close_on})
+        yield crawler.crawl(total=1000000)
+        self.flushLoggedErrors(crawler.spider.exception_cls)
+        reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_errorcount')
         key = 'spider_exceptions/{name}'\
-                .format(name=spider.exception_cls.__name__)
-        errorcount = spider.crawler.stats.get_value(key)
+                .format(name=crawler.spider.exception_cls.__name__)
+        errorcount = crawler.stats.get_value(key)
         self.assertTrue(errorcount >= close_on)
 
     @defer.inlineCallbacks
     def test_closespider_timeout(self):
-        spider = FollowAllSpider(total=1000000)
         close_on = 0.1
-        yield docrawl(spider, {'CLOSESPIDER_TIMEOUT': close_on})
-        reason = spider.meta['close_reason']
+        crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_TIMEOUT': close_on})
+        yield crawler.crawl(total=1000000)
+        reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_timeout')
-        stats = spider.crawler.stats
+        stats = crawler.stats
         start = stats.get_value('start_time')
         stop = stats.get_value('finish_time')
         diff = stop - start
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 9401bd0c9..48931d6ff 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -3,7 +3,7 @@ import socket
 import mock
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
-from scrapy.utils.test import docrawl, get_testlog
+from scrapy.utils.test import get_crawler, get_testlog
 from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
     BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
 from tests.mockserver import MockServer
@@ -21,9 +21,9 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_follow_all(self):
-        spider = FollowAllSpider()
-        yield docrawl(spider)
-        self.assertEqual(len(spider.urls_visited), 11)  # 10 + start_url
+        crawler = get_crawler(FollowAllSpider)
+        yield crawler.crawl()
+        self.assertEqual(len(crawler.spider.urls_visited), 11)  # 10 + start_url
 
     @defer.inlineCallbacks
     def test_delay(self):
@@ -37,9 +37,9 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def _test_delay(self, delay, randomize):
         settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize}
-        spider = FollowAllSpider(maxlatency=delay * 2)
-        yield docrawl(spider, settings)
-        t = spider.times
+        crawler = get_crawler(FollowAllSpider, settings)
+        yield crawler.crawl(maxlatency=delay * 2)
+        t = crawler.spider.times
         totaltime = t[-1] - t[0]
         avgd = totaltime / (len(t) - 1)
         tolerance = 0.6 if randomize else 0.2
@@ -48,85 +48,79 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_timeout_success(self):
-        spider = DelaySpider(n=0.5)
-        yield docrawl(spider)
-        self.assertTrue(spider.t1 > 0)
-        self.assertTrue(spider.t2 > 0)
-        self.assertTrue(spider.t2 > spider.t1)
+        crawler = get_crawler(DelaySpider)
+        yield crawler.crawl(n=0.5)
+        self.assertTrue(crawler.spider.t1 > 0)
+        self.assertTrue(crawler.spider.t2 > 0)
+        self.assertTrue(crawler.spider.t2 > crawler.spider.t1)
 
     @defer.inlineCallbacks
     def test_timeout_failure(self):
-        spider = DelaySpider(n=0.5)
-        yield docrawl(spider, {"DOWNLOAD_TIMEOUT": 0.35})
-        self.assertTrue(spider.t1 > 0)
-        self.assertTrue(spider.t2 == 0)
-        self.assertTrue(spider.t2_err > 0)
-        self.assertTrue(spider.t2_err > spider.t1)
+        crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
+        yield crawler.crawl(n=0.5)
+        self.assertTrue(crawler.spider.t1 > 0)
+        self.assertTrue(crawler.spider.t2 == 0)
+        self.assertTrue(crawler.spider.t2_err > 0)
+        self.assertTrue(crawler.spider.t2_err > crawler.spider.t1)
         # server hangs after receiving response headers
-        spider = DelaySpider(n=0.5, b=1)
-        yield docrawl(spider, {"DOWNLOAD_TIMEOUT": 0.35})
-        self.assertTrue(spider.t1 > 0)
-        self.assertTrue(spider.t2 == 0)
-        self.assertTrue(spider.t2_err > 0)
-        self.assertTrue(spider.t2_err > spider.t1)
+        yield crawler.crawl(n=0.5, b=1)
+        self.assertTrue(crawler.spider.t1 > 0)
+        self.assertTrue(crawler.spider.t2 == 0)
+        self.assertTrue(crawler.spider.t2_err > 0)
+        self.assertTrue(crawler.spider.t2_err > crawler.spider.t1)
 
     @defer.inlineCallbacks
     def test_retry_503(self):
-        spider = SimpleSpider("http://localhost:8998/status?n=503")
-        yield docrawl(spider)
+        crawler = get_crawler(SimpleSpider)
+        yield crawler.crawl("http://localhost:8998/status?n=503")
         self._assert_retried()
 
     @defer.inlineCallbacks
     def test_retry_conn_failed(self):
-        spider = SimpleSpider("http://localhost:65432/status?n=503")
-        yield docrawl(spider)
+        crawler = get_crawler(SimpleSpider)
+        yield crawler.crawl("http://localhost:65432/status?n=503")
         self._assert_retried()
 
     @defer.inlineCallbacks
     def test_retry_dns_error(self):
         with mock.patch('socket.gethostbyname',
                         side_effect=socket.gaierror(-5, 'No address associated with hostname')):
-            spider = SimpleSpider("http://example.com/")
-            yield docrawl(spider)
+            crawler = get_crawler(SimpleSpider)
+            yield crawler.crawl("http://example.com/")
             self._assert_retried()
 
     @defer.inlineCallbacks
     def test_start_requests_bug_before_yield(self):
-        spider = BrokenStartRequestsSpider(fail_before_yield=1)
-        yield docrawl(spider)
+        crawler = get_crawler(BrokenStartRequestsSpider)
+        yield crawler.crawl(fail_before_yield=1)
         errors = self.flushLoggedErrors(ZeroDivisionError)
         self.assertEqual(len(errors), 1)
 
     @defer.inlineCallbacks
     def test_start_requests_bug_yielding(self):
-        spider = BrokenStartRequestsSpider(fail_yielding=1)
-        yield docrawl(spider)
+        crawler = get_crawler(BrokenStartRequestsSpider)
+        yield crawler.crawl(fail_yielding=1)
         errors = self.flushLoggedErrors(ZeroDivisionError)
         self.assertEqual(len(errors), 1)
 
     @defer.inlineCallbacks
     def test_start_requests_lazyness(self):
         settings = {"CONCURRENT_REQUESTS": 1}
-        spider = BrokenStartRequestsSpider()
-        yield docrawl(spider, settings)
-        #self.assertTrue(False, spider.seedsseen)
-        #self.assertTrue(spider.seedsseen.index(None) < spider.seedsseen.index(99),
-        #                spider.seedsseen)
+        crawler = get_crawler(BrokenStartRequestsSpider, settings)
+        yield crawler.crawl()
+        #self.assertTrue(False, crawler.spider.seedsseen)
+        #self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
+        #                crawler.spider.seedsseen)
 
     @defer.inlineCallbacks
     def test_start_requests_dupes(self):
         settings = {"CONCURRENT_REQUESTS": 1}
-        spider = DuplicateStartRequestsSpider(dont_filter=True,
-                                              distinct_urls=2,
-                                              dupe_factor=3)
-        yield docrawl(spider, settings)
-        self.assertEqual(spider.visited, 6)
+        crawler = get_crawler(DuplicateStartRequestsSpider, settings)
+        yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3)
+        self.assertEqual(crawler.spider.visited, 6)
 
-        spider = DuplicateStartRequestsSpider(dont_filter=False,
-                                              distinct_urls=3,
-                                              dupe_factor=4)
-        yield docrawl(spider, settings)
-        self.assertEqual(spider.visited, 3)
+        yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4)
+        self.assertEqual(crawler.spider.visited, 3)
 
     @defer.inlineCallbacks
     def test_unbounded_response(self):
@@ -150,23 +144,23 @@ Connection: close
 foo body
 with multiples lines
 '''})
-        spider = SimpleSpider("http://localhost:8998/raw?{0}".format(query))
-        yield docrawl(spider)
+        crawler = get_crawler(SimpleSpider)
+        yield crawler.crawl("http://localhost:8998/raw?{0}".format(query))
         log = get_testlog()
         self.assertEqual(log.count("Got response 200"), 1)
 
     @defer.inlineCallbacks
     def test_retry_conn_lost(self):
         # connection lost after receiving data
-        spider = SimpleSpider("http://localhost:8998/drop?abort=0")
-        yield docrawl(spider)
+        crawler = get_crawler(SimpleSpider)
+        yield crawler.crawl("http://localhost:8998/drop?abort=0")
         self._assert_retried()
 
     @defer.inlineCallbacks
     def test_retry_conn_aborted(self):
         # connection lost before receiving data
-        spider = SimpleSpider("http://localhost:8998/drop?abort=1")
-        yield docrawl(spider)
+        crawler = get_crawler(SimpleSpider)
+        yield crawler.crawl("http://localhost:8998/drop?abort=1")
         self._assert_retried()
 
     def _assert_retried(self):
@@ -184,22 +178,22 @@ with multiples lines
         req0.meta['next'] = req1
         req1.meta['next'] = req2
         req2.meta['next'] = req3
-        spider = SingleRequestSpider(seed=req0)
-        yield docrawl(spider)
+        crawler = get_crawler(SingleRequestSpider)
+        yield crawler.crawl(seed=req0)
         # basic asserts in case of weird communication errors
-        self.assertIn('responses', spider.meta)
-        self.assertNotIn('failures', spider.meta)
+        self.assertIn('responses', crawler.spider.meta)
+        self.assertNotIn('failures', crawler.spider.meta)
         # start requests doesn't set Referer header
-        echo0 = json.loads(spider.meta['responses'][2].body)
+        echo0 = json.loads(crawler.spider.meta['responses'][2].body)
         self.assertNotIn('Referer', echo0['headers'])
         # following request sets Referer to start request url
-        echo1 = json.loads(spider.meta['responses'][1].body)
+        echo1 = json.loads(crawler.spider.meta['responses'][1].body)
         self.assertEqual(echo1['headers'].get('Referer'), [req0.url])
         # next request avoids Referer header
-        echo2 = json.loads(spider.meta['responses'][2].body)
+        echo2 = json.loads(crawler.spider.meta['responses'][2].body)
         self.assertNotIn('Referer', echo2['headers'])
         # last request explicitly sets a Referer header
-        echo3 = json.loads(spider.meta['responses'][3].body)
+        echo3 = json.loads(crawler.spider.meta['responses'][3].body)
         self.assertEqual(echo3['headers'].get('Referer'), ['http://example.com'])
 
     @defer.inlineCallbacks
@@ -208,11 +202,11 @@ with multiples lines
         est = []
 
         def cb(response):
-            est.append(get_engine_status(spider.crawler.engine))
+            est.append(get_engine_status(crawler.engine))
 
-        spider = SingleRequestSpider(seed='http://localhost:8998/', callback_func=cb)
-        yield docrawl(spider)
+        crawler = get_crawler(SingleRequestSpider)
+        yield crawler.crawl(seed='http://localhost:8998/', callback_func=cb)
         self.assertEqual(len(est), 1, est)
         s = dict(est[0])
-        self.assertEqual(s['engine.spider.name'], spider.name)
+        self.assertEqual(s['engine.spider.name'], crawler.spider.name)
         self.assertEqual(s['len(engine.scraper.slot.active)'], 1)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 6a3115004..c444d35fa 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -47,19 +47,22 @@ class LoadTestCase(unittest.TestCase):
 
     def test_enabled_handler(self):
         handlers = {'scheme': 'tests.test_downloader_handlers.DummyDH'}
-        dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers}))
+        crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
+        dh = DownloadHandlers(crawler)
         self.assertIn('scheme', dh._handlers)
         self.assertNotIn('scheme', dh._notconfigured)
 
     def test_not_configured_handler(self):
         handlers = {'scheme': 'tests.test_downloader_handlers.OffDH'}
-        dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers}))
+        crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
+        dh = DownloadHandlers(crawler)
         self.assertNotIn('scheme', dh._handlers)
         self.assertIn('scheme', dh._notconfigured)
 
     def test_disabled_handler(self):
         handlers = {'scheme': None}
-        dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers}))
+        crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
+        dh = DownloadHandlers(crawler)
         self.assertNotIn('scheme', dh._handlers)
         self.assertNotIn('scheme', dh._notconfigured)
 
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index b7d3594cd..282035f5c 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -12,9 +12,8 @@ class ManagerTestCase(TestCase):
     settings_dict = None
 
     def setUp(self):
-        self.crawler = get_crawler(self.settings_dict)
-        self.spider = Spider('foo')
-        self.spider.set_crawler(self.crawler)
+        self.crawler = get_crawler(Spider, self.settings_dict)
+        self.spider = self.crawler._create_spider('foo')
         self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler)
         # some mw depends on stats collector
         self.crawler.stats.open_spider(self.spider)
diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py
index 3e5ce6052..e73e62538 100644
--- a/tests/test_downloadermiddleware_ajaxcrawlable.py
+++ b/tests/test_downloadermiddleware_ajaxcrawlable.py
@@ -9,8 +9,8 @@ __doctests__ = ['scrapy.contrib.downloadermiddleware.ajaxcrawl']
 
 class AjaxCrawlMiddlewareTest(unittest.TestCase):
     def setUp(self):
-        self.spider = Spider('foo')
-        crawler = get_crawler({'AJAXCRAWL_ENABLED': True})
+        crawler = get_crawler(Spider, {'AJAXCRAWL_ENABLED': True})
+        self.spider = crawler._create_spider('foo')
         self.mw = AjaxCrawlMiddleware.from_crawler(crawler)
 
     def _ajaxcrawlable_body(self):
diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py
index b37a02336..09973b367 100644
--- a/tests/test_downloadermiddleware_defaultheaders.py
+++ b/tests/test_downloadermiddleware_defaultheaders.py
@@ -10,9 +10,8 @@ from scrapy.utils.test import get_crawler
 class TestDefaultHeadersMiddleware(TestCase):
 
     def get_defaults_spider_mw(self):
-        crawler = get_crawler()
-        spider = Spider('foo')
-        spider.set_crawler(crawler)
+        crawler = get_crawler(Spider)
+        spider = crawler._create_spider('foo')
         defaults = dict([(k, [v]) for k, v in \
             six.iteritems(crawler.settings.get('DEFAULT_REQUEST_HEADERS'))])
         return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler)
diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py
index 52a0cc09d..3e3ff2401 100644
--- a/tests/test_downloadermiddleware_downloadtimeout.py
+++ b/tests/test_downloadermiddleware_downloadtimeout.py
@@ -9,9 +9,8 @@ from scrapy.utils.test import get_crawler
 class DownloadTimeoutMiddlewareTest(unittest.TestCase):
 
     def get_request_spider_mw(self):
-        crawler = get_crawler()
-        spider = Spider('foo')
-        spider.set_crawler(crawler)
+        crawler = get_crawler(Spider)
+        spider = crawler._create_spider('foo')
         request = Request('http://scrapytest.org/')
         return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
 
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 0eb5e7144..1e22ae661 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -24,8 +24,8 @@ class _BaseTest(unittest.TestCase):
         self.yesterday = email.utils.formatdate(time.time() - 86400)
         self.today = email.utils.formatdate()
         self.tomorrow = email.utils.formatdate(time.time() + 86400)
-        self.crawler = get_crawler()
-        self.spider = Spider('example.com')
+        self.crawler = get_crawler(Spider)
+        self.spider = self.crawler._create_spider('example.com')
         self.tmpdir = tempfile.mkdtemp()
         self.request = Request('http://www.example.com',
                                headers={'User-Agent': 'test'})
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 8b871c7bc..beadfd362 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -10,8 +10,8 @@ from scrapy.utils.test import get_crawler
 class RedirectMiddlewareTest(unittest.TestCase):
 
     def setUp(self):
-        crawler = get_crawler()
-        self.spider = Spider('foo')
+        crawler = get_crawler(Spider)
+        self.spider = crawler._create_spider('foo')
         self.mw = RedirectMiddleware.from_crawler(crawler)
 
     def test_priority_adjust(self):
@@ -123,8 +123,8 @@ class RedirectMiddlewareTest(unittest.TestCase):
 class MetaRefreshMiddlewareTest(unittest.TestCase):
 
     def setUp(self):
-        crawler = get_crawler()
-        self.spider = Spider('foo')
+        crawler = get_crawler(Spider)
+        self.spider = crawler._create_spider('foo')
         self.mw = MetaRefreshMiddleware.from_crawler(crawler)
 
     def _body(self, interval=5, url='http://example.org/newpage'):
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 4c771f18e..e3e7c87d6 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -14,8 +14,8 @@ from scrapy.utils.test import get_crawler
 
 class RetryTest(unittest.TestCase):
     def setUp(self):
-        crawler = get_crawler()
-        self.spider = Spider('foo')
+        crawler = get_crawler(Spider)
+        self.spider = crawler._create_spider('foo')
         self.mw = RetryMiddleware.from_crawler(crawler)
         self.mw.max_retry_times = 2
 
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index edc26e543..b790ff09a 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -9,8 +9,8 @@ from scrapy.utils.test import get_crawler
 class TestDownloaderStats(TestCase):
 
     def setUp(self):
-        self.crawler = get_crawler()
-        self.spider = Spider('scrapytest.org')
+        self.crawler = get_crawler(Spider)
+        self.spider = self.crawler._create_spider('scrapytest.org')
         self.mw = DownloaderStats(self.crawler.stats)
 
         self.crawler.stats.open_spider(self.spider)
diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py
index 5fd5c24be..909d03ba5 100644
--- a/tests/test_downloadermiddleware_useragent.py
+++ b/tests/test_downloadermiddleware_useragent.py
@@ -9,9 +9,8 @@ from scrapy.utils.test import get_crawler
 class UserAgentMiddlewareTest(TestCase):
 
     def get_spider_and_mw(self, default_useragent):
-        crawler = get_crawler({'USER_AGENT': default_useragent})
-        spider = Spider('foo')
-        spider.set_crawler(crawler)
+        crawler = get_crawler(Spider, {'USER_AGENT': default_useragent})
+        spider = crawler._create_spider('foo')
         return spider, UserAgentMiddleware.from_crawler(crawler)
 
     def test_default_agent(self):
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 6a0314a02..244d339ef 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -87,20 +87,18 @@ class CrawlerRun(object):
         self.portno = self.port.getHost().port
 
         start_urls = [self.geturl("/"), self.geturl("/redirect")]
-        self.spider = TestSpider(start_urls=start_urls)
 
         for name, signal in vars(signals).items():
             if not name.startswith('_'):
                 dispatcher.connect(self.record_signal, signal)
 
-        self.crawler = get_crawler()
+        self.crawler = get_crawler(TestSpider)
         self.crawler.install()
-        self.crawler.configure()
         self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
         self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
         self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded)
-        self.crawler.crawl(self.spider)
-        self.crawler.start()
+        self.crawler.crawl(start_urls=start_urls)
+        self.spider = self.crawler.spider
 
         self.deferred = defer.Deferred()
         dispatcher.connect(self.stop, signals.engine_stopped)
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 8999e102e..5ce48ebf8 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -8,7 +8,7 @@ from netlib import http_auth
 
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
-from scrapy.utils.test import get_testlog, docrawl
+from scrapy.utils.test import get_testlog, get_crawler
 from scrapy.http import Request
 from tests.spiders import SimpleSpider, SingleRequestSpider
 from tests.mockserver import MockServer
@@ -49,29 +49,29 @@ class ProxyConnectTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_https_connect_tunnel(self):
-        spider = SimpleSpider("https://localhost:8999/status?n=200")
-        yield docrawl(spider)
+        crawler = get_crawler(SimpleSpider)
+        yield crawler.crawl("https://localhost:8999/status?n=200")
         self._assert_got_response_code(200)
 
     @defer.inlineCallbacks
     def test_https_noconnect(self):
         os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888?noconnect'
-        spider = SimpleSpider("https://localhost:8999/status?n=200")
-        yield docrawl(spider)
+        crawler = get_crawler(SimpleSpider)
+        yield crawler.crawl("https://localhost:8999/status?n=200")
         self._assert_got_response_code(200)
         os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888'
 
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
-        spider = SimpleSpider("https://localhost:99999/status?n=200")
-        yield docrawl(spider)
+        crawler = get_crawler(SimpleSpider)
+        yield crawler.crawl("https://localhost:99999/status?n=200")
         self._assert_got_tunnel_error()
 
     @defer.inlineCallbacks
     def test_https_tunnel_auth_error(self):
         os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888'
-        spider = SimpleSpider("https://localhost:8999/status?n=200")
-        yield docrawl(spider)
+        crawler = get_crawler(SimpleSpider)
+        yield crawler.crawl("https://localhost:8999/status?n=200")
         # The proxy returns a 407 error code but it does not reach the client;
         # he just sees a TunnelError.
         self._assert_got_tunnel_error()
@@ -80,17 +80,17 @@ class ProxyConnectTestCase(TestCase):
     @defer.inlineCallbacks
     def test_https_tunnel_without_leak_proxy_authorization_header(self):
         request = Request("https://localhost:8999/echo")
-        spider = SingleRequestSpider(seed=request)
-        yield docrawl(spider)
+        crawler = get_crawler(SingleRequestSpider)
+        yield crawler.crawl(seed=request)
         self._assert_got_response_code(200)
-        echo = json.loads(spider.meta['responses'][0].body)
+        echo = json.loads(crawler.spider.meta['responses'][0].body)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
     @defer.inlineCallbacks
     def test_https_noconnect_auth_error(self):
         os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888?noconnect'
-        spider = SimpleSpider("https://localhost:8999/status?n=200")
-        yield docrawl(spider)
+        crawler = get_crawler(SimpleSpider)
+        yield crawler.crawl("https://localhost:8999/status?n=200")
         self._assert_got_response_code(407)
 
     def _assert_got_response_code(self, code):
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 903ea684a..148a872dd 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -242,7 +242,8 @@ class CrawlSpiderTest(SpiderTest):
         self.assertTrue(hasattr(spider, '_follow_links'))
         self.assertTrue(spider._follow_links)
 
-        crawler.settings.set('CRAWLSPIDER_FOLLOW_LINKS', False)
+        settings_dict = {'CRAWLSPIDER_FOLLOW_LINKS': False}
+        crawler = get_crawler(settings_dict=settings_dict)
         spider = self.spider_class.from_crawler(crawler, 'example.com')
         self.assertTrue(hasattr(spider, '_follow_links'))
         self.assertFalse(spider._follow_links)
@@ -256,7 +257,8 @@ class CrawlSpiderTest(SpiderTest):
         self.assertTrue(spider._follow_links)
 
         spider = self.spider_class('example.com')
-        spider.set_crawler(get_crawler({'CRAWLSPIDER_FOLLOW_LINKS': False}))
+        settings_dict = {'CRAWLSPIDER_FOLLOW_LINKS': False}
+        spider.set_crawler(get_crawler(settings_dict=settings_dict))
         self.assertTrue(hasattr(spider, '_follow_links'))
         self.assertFalse(spider._follow_links)
 
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index 94404ff41..e7ae75ed2 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -10,9 +10,10 @@ from scrapy.utils.test import get_crawler
 class TestDepthMiddleware(TestCase):
 
     def setUp(self):
-        self.spider = Spider('scrapytest.org')
+        crawler = get_crawler(Spider)
+        self.spider = crawler._create_spider('scrapytest.org')
 
-        self.stats = StatsCollector(get_crawler())
+        self.stats = StatsCollector(crawler)
         self.stats.open_spider(self.spider)
 
         self.mw = DepthMiddleware(1, self.stats, True)
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 788a0986b..5394f0eee 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -3,7 +3,7 @@ from unittest import TestCase
 from twisted.trial.unittest import TestCase as TrialTestCase
 from twisted.internet import defer
 
-from scrapy.utils.test import docrawl, get_testlog
+from scrapy.utils.test import get_crawler, get_testlog
 from tests.mockserver import MockServer
 from scrapy.http import Response, Request
 from scrapy.spider import Spider
@@ -165,20 +165,20 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
 
     @defer.inlineCallbacks
     def test_middleware_works(self):
-        spider = _HttpErrorSpider()
-        yield docrawl(spider)
-        assert not spider.skipped, spider.skipped
-        self.assertEqual(spider.parsed, {'200'})
-        self.assertEqual(spider.failed, {'404', '402', '500'})
+        crawler = get_crawler(_HttpErrorSpider)
+        yield crawler.crawl()
+        assert not crawler.spider.skipped, crawler.spider.skipped
+        self.assertEqual(crawler.spider.parsed, {'200'})
+        self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
 
     @defer.inlineCallbacks
     def test_logging(self):
-        spider = _HttpErrorSpider(bypass_status_codes={402})
-        yield docrawl(spider)
+        crawler = get_crawler(_HttpErrorSpider)
+        yield crawler.crawl(bypass_status_codes={402})
         # print(get_testlog())
-        self.assertEqual(spider.parsed, {'200', '402'})
-        self.assertEqual(spider.skipped, {'402'})
-        self.assertEqual(spider.failed, {'404', '500'})
+        self.assertEqual(crawler.spider.parsed, {'200', '402'})
+        self.assertEqual(crawler.spider.skipped, {'402'})
+        self.assertEqual(crawler.spider.failed, {'404', '500'})
 
         log = get_testlog()
         self.assertIn('Ignoring response <404', log)
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 298cba6e4..e5e99002a 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -10,13 +10,13 @@ from scrapy.utils.test import get_crawler
 class TestOffsiteMiddleware(TestCase):
 
     def setUp(self):
-        self.spider = self._get_spider()
-        crawler = get_crawler()
+        crawler = get_crawler(Spider)
+        self.spider = crawler._create_spider(**self._get_spiderargs())
         self.mw = OffsiteMiddleware.from_crawler(crawler)
         self.mw.spider_opened(self.spider)
 
-    def _get_spider(self):
-        return Spider('foo', allowed_domains=['scrapytest.org', 'scrapy.org'])
+    def _get_spiderargs(self):
+        return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org'])
 
     def test_process_spider_output(self):
         res = Response('http://scrapytest.org')
@@ -39,8 +39,8 @@ class TestOffsiteMiddleware(TestCase):
 
 class TestOffsiteMiddleware2(TestOffsiteMiddleware):
 
-    def _get_spider(self):
-        return Spider('foo', allowed_domains=None)
+    def _get_spiderargs(self):
+        return dict(name='foo', allowed_domains=None)
 
     def test_process_spider_output(self):
         res = Response('http://scrapytest.org')
@@ -58,7 +58,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
 
     def _get_spider(self):
       bad_hostname = urlparse('http:////scrapytest.org').hostname
-      return Spider('foo', allowed_domains=['scrapytest.org', None, bad_hostname])
+      return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname])
 
     def test_process_spider_output(self):
       res = Response('http://scrapytest.org')
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 795e8e3bd..db1f50712 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -7,8 +7,8 @@ from scrapy.utils.test import get_crawler
 class StatsCollectorTest(unittest.TestCase):
 
     def setUp(self):
-        self.crawler = get_crawler()
-        self.spider = Spider('foo')
+        self.crawler = get_crawler(Spider)
+        self.spider = self.crawler._create_spider('foo')
 
     def test_collector(self):
         stats = StatsCollector(self.crawler)

From d0edad4b0bd93ed34a680ddd6563387be7797128 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 31 Jul 2014 10:10:25 -0300
Subject: [PATCH 0011/2974] Drop support for ScrapyCommand.crawler property

---
 scrapy/command.py | 28 +---------------------------
 1 file changed, 1 insertion(+), 27 deletions(-)

diff --git a/scrapy/command.py b/scrapy/command.py
index b2eb9cf8f..9ac013098 100644
--- a/scrapy/command.py
+++ b/scrapy/command.py
@@ -2,12 +2,11 @@
 Base class for Scrapy commands
 """
 import os
-import warnings
 from optparse import OptionGroup
 from twisted.python import failure
 
 from scrapy.utils.conf import arglist_to_dict
-from scrapy.exceptions import UsageError, ScrapyDeprecationWarning
+from scrapy.exceptions import UsageError
 
 
 class ScrapyCommand(object):
@@ -27,31 +26,6 @@ class ScrapyCommand(object):
         assert not hasattr(self, '_crawler'), "crawler already set"
         self._crawler = crawler
 
-    @property
-    def crawler(self):
-        warnings.warn("Command's default `crawler` is deprecated and will be removed. "
-            "Use `create_crawler` method to instatiate crawlers.",
-            ScrapyDeprecationWarning)
-
-        if not hasattr(self, '_crawler'):
-            crawler = self.crawler_process.create_crawler()
-
-            old_start = crawler.start
-            self.crawler_process.started = False
-
-            def wrapped_start():
-                if self.crawler_process.started:
-                    old_start()
-                else:
-                    self.crawler_process.started = True
-                    self.crawler_process.start()
-
-            crawler.start = wrapped_start
-
-            self.set_crawler(crawler)
-
-        return self._crawler
-
     def syntax(self):
         """
         Command syntax (preferably one-line). Do not include command name.

From 89df18bd2bc6fdc7f2084454a2c69f4db03008ad Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 31 Jul 2014 10:16:25 -0300
Subject: [PATCH 0012/2974] Fix usage of crawler_process in ScrapyCommands

---
 scrapy/commands/bench.py     |  4 +---
 scrapy/commands/check.py     | 32 +++++++++-----------------------
 scrapy/commands/crawl.py     |  4 +---
 scrapy/commands/edit.py      |  7 +++----
 scrapy/commands/genspider.py |  5 ++---
 scrapy/commands/list.py      |  3 +--
 scrapy/commands/runspider.py |  5 ++---
 scrapy/contracts/__init__.py | 17 +++++++++++++++++
 8 files changed, 36 insertions(+), 41 deletions(-)

diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 18934f976..395597546 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -22,9 +22,7 @@ class Command(ScrapyCommand):
 
     def run(self, args, opts):
         with _BenchServer():
-            spider = _BenchSpider(total=100000)
-            crawler = self.crawler_process.create_crawler()
-            crawler.crawl(spider)
+            self.crawler_process.crawl(_BenchSpider, total=100000)
             self.crawler_process.start()
 
 
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index 40ff9014b..014b00eeb 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -69,20 +69,18 @@ class Command(ScrapyCommand):
         # contract requests
         contract_reqs = defaultdict(list)
 
-        spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])
-        spiders = spman_cls.from_settings(self.settings)
+        spiders = self.crawler_process.spiders
 
-        for spider in args or spiders.list():
-            spider = spiders.create(spider)
-            requests = self.get_requests(spider, conman, result)
-            contract_reqs[spider.name] = []
+        for spidername in args or spiders.list():
+            spidercls = spiders.load(spidername)
+            spidercls.start_requests = lambda s: conman.from_spider(s, result)
 
+            tested_methods = conman.tested_methods_from_spidercls(spidercls)
             if opts.list:
-                for req in requests:
-                    contract_reqs[spider.name].append(req.callback.__name__)
-            elif requests:
-                crawler = self.crawler_process.create_crawler(spider.name)
-                crawler.crawl(spider, requests)
+                for method in tested_methods:
+                    contract_reqs[spidercls.name].append(method)
+            elif tested_methods:
+                self.crawler_process.crawl(spidercls)
 
         # start checks
         if opts.list:
@@ -101,15 +99,3 @@ class Command(ScrapyCommand):
             result.printSummary(start, stop)
             self.exitcode = int(not result.wasSuccessful())
 
-    def get_requests(self, spider, conman, result):
-        requests = []
-
-        for key, value in vars(type(spider)).items():
-            if callable(value) and value.__doc__:
-                bound_method = value.__get__(spider, type(spider))
-                request = conman.from_method(bound_method, result)
-
-                if request:
-                    requests.append(request)
-
-        return requests
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 0976de51b..b7fea7b80 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -54,7 +54,5 @@ class Command(ScrapyCommand):
             raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
         spname = args[0]
 
-        crawler = self.crawler_process.create_crawler()
-        spider = crawler.spiders.create(spname, **opts.spargs)
-        crawler.crawl(spider)
+        self.crawler_process.crawl(spname, **opts.spargs)
         self.crawler_process.start()
diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py
index b542217e1..e20e7c2e5 100644
--- a/scrapy/commands/edit.py
+++ b/scrapy/commands/edit.py
@@ -25,13 +25,12 @@ class Command(ScrapyCommand):
         if len(args) != 1:
             raise UsageError()
 
-        crawler = self.crawler_process.create_crawler()
-        editor = crawler.settings['EDITOR']
+        editor = self.settings['EDITOR']
         try:
-            spider = crawler.spiders.create(args[0])
+            spidercls = self.crawler_process.spiders.load(args[0])
         except KeyError:
             return self._err("Spider not found: %s" % args[0])
 
-        sfile = sys.modules[spider.__module__].__file__
+        sfile = sys.modules[spidercls.__module__].__file__
         sfile = sfile.replace('.pyc', '.py')
         self.exitcode = os.system('%s "%s"' % (editor, sfile))
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 3e2e24b21..52c5d9f94 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -65,15 +65,14 @@ class Command(ScrapyCommand):
             return
 
         try:
-            crawler = self.crawler_process.create_crawler()
-            spider = crawler.spiders.create(name)
+            spidercls = self.crawler_process.spiders.load(name)
         except KeyError:
             pass
         else:
             # if spider already exists and not --force then halt
             if not opts.force:
                 print("Spider %r already exists in module:" % name)
-                print("  %s" % spider.__module__)
+                print("  %s" % spidercls.__module__)
                 return
         template_file = self._find_template(opts.template)
         if template_file:
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index 0ea9c2313..2d55d59bd 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -10,6 +10,5 @@ class Command(ScrapyCommand):
         return "List available spiders"
 
     def run(self, args, opts):
-        crawler = self.crawler_process.create_crawler()
-        for s in sorted(crawler.spiders.list()):
+        for s in sorted(self.crawler_process.spiders.list()):
             print(s)
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index b1d501842..b67838619 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -83,8 +83,7 @@ class Command(ScrapyCommand):
         spclasses = list(iter_spider_classes(module))
         if not spclasses:
             raise UsageError("No spider found in file: %s\n" % filename)
-        spider = spclasses.pop()(**opts.spargs)
+        spidercls = spclasses.pop()
 
-        crawler = self.crawler_process.create_crawler()
-        crawler.crawl(spider)
+        self.crawler_process.crawl(spidercls, **opts.spargs)
         self.crawler_process.start()
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 03e6e4e0c..5eaee3d11 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -15,6 +15,15 @@ class ContractsManager(object):
         for contract in contracts:
             self.contracts[contract.name] = contract
 
+    def tested_methods_from_spidercls(self, spidercls):
+        methods = []
+        for key, value in vars(spidercls).items():
+            if (callable(value) and value.__doc__ and
+                    re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
+                methods.append(key)
+
+        return methods
+
     def extract_contracts(self, method):
         contracts = []
         for line in method.__doc__.split('\n'):
@@ -28,6 +37,14 @@ class ContractsManager(object):
 
         return contracts
 
+    def from_spider(self, spider, results):
+        requests = []
+        for method in self.tested_methods_from_spidercls(type(spider)):
+            bound_method = spider.__getattribute__(method)
+            requests.append(self.from_method(bound_method, results))
+
+        return requests
+
     def from_method(self, method, results):
         contracts = self.extract_contracts(method)
         if contracts:

From 900a487682b11696ccab5d18c9f13e0addd25f12 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 5 Aug 2014 21:01:57 -0300
Subject: [PATCH 0013/2974] Support multiple simultaneous LogObservers
 listening different crawlers

---
 docs/topics/logging.rst | 13 +++++++++---
 scrapy/crawler.py       |  5 +++++
 scrapy/log.py           | 30 +++++++++++++++------------
 tests/test_log.py       | 46 ++++++++++++++++++++++++++++++++++-------
 4 files changed, 71 insertions(+), 23 deletions(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 1a9e975d8..819884ac2 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -10,7 +10,11 @@ logging`_ but this may change in the future.
 
 .. _Twisted logging: http://twistedmatrix.com/projects/core/documentation/howto/logging.html
 
-The logging service must be explicitly started through the :func:`scrapy.log.start` function.
+The logging service must be explicitly started through the
+:func:`scrapy.log.start` function to catch the top level Scrapy's log messages.
+On top of that, each crawler has its own independent log observer
+(automatically attached when it's created) that intercepts its spider's log
+messages.
 
 .. _topics-logging-levels:
 
@@ -55,8 +59,11 @@ scrapy.log module
 
 .. function:: start(logfile=None, loglevel=None, logstdout=None)
 
-    Start the logging facility. This must be called before actually logging any
-    messages. Otherwise, messages logged before this call will get lost.
+    Start the top level Scrapy logger. This must be called before actually
+    logging any top level messages (those logged using this module's
+    :func:`~scrapy.log.msg` function instead of the :meth:`Spider.log
+    <scrapy.spider.Spider.log>` method). Otherwise, messages logged before this
+    call will get lost.
 
     :param logfile: the file path to use for logging output. If omitted, the
         :setting:`LOG_FILE` setting will be used. If both are ``None``, the log
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 56823166b..597bb2e9d 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -128,6 +128,7 @@ class CrawlerProcess(CrawlerRunner):
         signame = signal_names[signum]
         log.msg(format='Received %(signame)s twice, forcing unclean shutdown',
                 level=log.INFO, signame=signame)
+        self._stop_logging()
         reactor.callFromThread(self._stop_reactor)
 
     def start(self, stop_after_crawl=True):
@@ -135,6 +136,7 @@ class CrawlerProcess(CrawlerRunner):
         self._start_reactor(stop_after_crawl)
 
     def _start_logging(self):
+        self.log_observer = log.start_from_settings(self.settings)
         log.scrapy_info(self.settings)
 
     def _start_reactor(self, stop_after_crawl=True):
@@ -149,6 +151,9 @@ class CrawlerProcess(CrawlerRunner):
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
         reactor.run(installSignalHandlers=False)  # blocking call
 
+    def _stop_logging(self):
+        self.log_observer.stop()
+
     def _stop_reactor(self, _=None):
         try:
             reactor.stop()
diff --git a/scrapy/log.py b/scrapy/log.py
index 1f32003e7..aa53e3574 100644
--- a/scrapy/log.py
+++ b/scrapy/log.py
@@ -35,15 +35,16 @@ class ScrapyFileLogObserver(log.FileLogObserver):
     def __init__(self, f, level=INFO, encoding='utf-8', crawler=None):
         self.level = level
         self.encoding = encoding
+        self.crawler = crawler
         if crawler:
-            self.crawler = crawler
             self.emit = self._emit_with_crawler
         else:
             self.emit = self._emit
         log.FileLogObserver.__init__(self, f)
 
     def _emit(self, eventDict):
-        ev = _adapt_eventdict(eventDict, self.level, self.encoding)
+        ev = _adapt_eventdict(eventDict, self.level, self.encoding,
+                              self.crawler)
         if ev is not None:
             log.FileLogObserver.emit(self, ev)
         return ev
@@ -55,7 +56,8 @@ class ScrapyFileLogObserver(log.FileLogObserver):
             sname = 'log_count/%s' % level_names.get(level, level)
             self.crawler.stats.inc_value(sname)
 
-def _adapt_eventdict(eventDict, log_level=INFO, encoding='utf-8', prepend_level=True):
+def _adapt_eventdict(eventDict, log_level=INFO, encoding='utf-8',
+                     crawler=None, prepend_level=True):
     """Adapt Twisted log eventDict making it suitable for logging with a Scrapy
     log observer. It may return None to indicate that the event should be
     ignored by a Scrapy log observer.
@@ -78,6 +80,12 @@ def _adapt_eventdict(eventDict, log_level=INFO, encoding='utf-8', prepend_level=
     spider = ev.get('spider')
     if spider:
         ev['system'] = unicode_to_str(spider.name, encoding)
+    if crawler and (not spider or spider.crawler is not crawler):
+        # ignore events not triggered by own spiders in crawlers' observers
+        return
+    if not crawler and spider:
+        # ignore spiders' events in observers without crawler
+        return
 
     lvlname = level_names.get(level, 'NOLEVEL')
     message = ev.get('message')
@@ -140,18 +148,14 @@ def start_from_settings(settings, crawler=None):
             settings['LOG_ENCODING'], crawler)
 
 def scrapy_info(settings):
-    log_observer = start_from_settings(settings)
-    if log_observer:
-        msg("Scrapy %s started (bot: %s)" % (scrapy.__version__,
-            settings['BOT_NAME']))
+    msg("Scrapy %s started (bot: %s)" % (scrapy.__version__,
+        settings['BOT_NAME']))
 
-        msg("Optional features available: %s" % ", ".join(scrapy.optional_features),
-            level=INFO)
+    msg("Optional features available: %s" % ", ".join(scrapy.optional_features),
+        level=INFO)
 
-        d = dict(overridden_settings(settings))
-        msg(format="Overridden settings: %(settings)r", settings=d, level=INFO)
-
-        log_observer.stop()
+    d = dict(overridden_settings(settings))
+    msg(format="Overridden settings: %(settings)r", settings=d, level=INFO)
 
 def start_from_crawler(crawler):
     return start_from_settings(crawler.settings, crawler)
diff --git a/tests/test_log.py b/tests/test_log.py
index 3263cb42e..113d10004 100644
--- a/tests/test_log.py
+++ b/tests/test_log.py
@@ -6,6 +6,7 @@ from twisted.trial import unittest
 from scrapy import log
 from scrapy.spider import Spider
 from scrapy.settings import default_settings
+from scrapy.utils.test import get_crawler
 
 class LogTest(unittest.TestCase):
 
@@ -40,10 +41,10 @@ class ScrapyFileLogObserverTest(unittest.TestCase):
         log.msg("Hello")
         self.assertEqual(self.logged(), "[scrapy] INFO: Hello")
 
-    def test_msg_spider(self):
+    def test_msg_ignore_spider(self):
         spider = Spider("myspider")
         log.msg("Hello", spider=spider)
-        self.assertEqual(self.logged(), "[myspider] INFO: Hello")
+        self.failIf(self.logged())
 
     def test_msg_level1(self):
         log.msg("Hello", level=log.WARNING)
@@ -57,11 +58,6 @@ class ScrapyFileLogObserverTest(unittest.TestCase):
         log.msg("Hello", level=9999)
         self.assertEqual(self.logged(), "[scrapy] NOLEVEL: Hello")
 
-    def test_msg_level_spider(self):
-        spider = Spider("myspider")
-        log.msg("Hello", spider=spider, level=log.WARNING)
-        self.assertEqual(self.logged(), "[myspider] WARNING: Hello")
-
     def test_msg_encoding(self):
         log.msg(u"Price: \xa3100")
         self.assertEqual(self.logged(), "[scrapy] INFO: Price: \xc2\xa3100")
@@ -133,5 +129,41 @@ class Latin1ScrapyFileLogObserverTest(ScrapyFileLogObserverTest):
 #        self.assertEqual(self.first_log_line(), "[scrapy] ERROR: \xa3")
 
 
+class CrawlerScrapyFileLogObserverTest(unittest.TestCase):
+
+    def setUp(self):
+        self.f = BytesIO()
+        self.crawler = get_crawler(Spider)
+        self.spider = self.crawler.spider = self.crawler._create_spider('test')
+        self.log_observer = log.ScrapyFileLogObserver(self.f, log.INFO,
+                                                      'utf-8', self.crawler)
+        self.log_observer.start()
+
+    def tearDown(self):
+        self.flushLoggedErrors()
+        self.log_observer.stop()
+
+    def logged(self):
+        return self.f.getvalue().strip()[25:]
+
+    def test_msg_basic(self):
+        log.msg("Hello", spider=self.spider)
+        self.assertEqual(self.logged(), "[test] INFO: Hello")
+
+    def test_msg_ignore_scrapy_channel(self):
+        log.msg("Hello")
+        self.failIf(self.logged())
+
+    def test_msg_ignore_another_crawler(self):
+        crawler = get_crawler(Spider)
+        log.msg("Hello", spider=crawler._create_spider('test'))
+        self.failIf(self.logged())
+
+    def test_msg_stats_log(self):
+        assert self.crawler.stats.get_value('log_count/INFO', 0) == 0
+        log.msg("Hello", spider=self.spider)
+        self.assertEqual(self.crawler.stats.get_value('log_count/INFO'), 1)
+
+
 if __name__ == "__main__":
     unittest.main()

From 9cbbfd8b04835c40568b687ef8b13d901db988cb Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 6 Aug 2014 08:51:12 -0300
Subject: [PATCH 0014/2974] Adjust spiders' utils to new SpiderManager API

---
 scrapy/commands/fetch.py | 14 +++++-------
 scrapy/commands/parse.py | 49 +++++++++++++++++++++-------------------
 scrapy/commands/shell.py | 24 +++++++++++++++-----
 scrapy/shell.py          |  9 +++-----
 scrapy/utils/spider.py   | 18 +++++++++------
 5 files changed, 64 insertions(+), 50 deletions(-)

diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 373d323c7..ca9fd57f5 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -3,9 +3,8 @@ from w3lib.url import is_url
 
 from scrapy.command import ScrapyCommand
 from scrapy.http import Request
-from scrapy.spider import Spider
 from scrapy.exceptions import UsageError
-from scrapy.utils.spider import create_spider_for_request
+from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 
 class Command(ScrapyCommand):
 
@@ -48,12 +47,11 @@ class Command(ScrapyCommand):
         request = Request(args[0], callback=cb, dont_filter=True)
         request.meta['handle_httpstatus_all'] = True
 
-        crawler = self.crawler_process.create_crawler()
-        spider = None
+        spidercls = DefaultSpider
+        spiders = self.crawler_process.spiders
         if opts.spider:
-            spider = crawler.spiders.create(opts.spider)
+            spidercls = spiders.load(opts.spider)
         else:
-            spider = create_spider_for_request(crawler.spiders, request, \
-                default_spider=Spider('default'))
-        crawler.crawl(spider, [request])
+            spidercls = spidercls_for_request(spiders, request, spidercls)
+        self.crawler_process.crawl(spidercls, start_requests=lambda: [request])
         self.crawler_process.start()
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 0867a21a0..01c7fff0a 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -5,7 +5,7 @@ from scrapy.http import Request
 from scrapy.item import BaseItem
 from scrapy.utils import display
 from scrapy.utils.conf import arglist_to_dict
-from scrapy.utils.spider import iterate_spider_output, create_spider_for_request
+from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
 from scrapy.exceptions import UsageError
 from scrapy import log
 
@@ -113,41 +113,45 @@ class Command(ScrapyCommand):
                 requests.append(x)
         return items, requests
 
-    def get_callback_from_rules(self, response):
-        if getattr(self.spider, 'rules', None):
-            for rule in self.spider.rules:
+    def get_callback_from_rules(self, spider, response):
+        if getattr(spider, 'rules', None):
+            for rule in spider.rules:
                 if rule.link_extractor.matches(response.url) and rule.callback:
                     return rule.callback
         else:
             log.msg(format='No CrawlSpider rules found in spider %(spider)r, '
                            'please specify a callback to use for parsing',
-                    level=log.ERROR, spider=self.spider.name)
+                    level=log.ERROR, spider=spider.name)
 
-    def set_spider(self, url, opts):
+    def set_spidercls(self, url, opts):
+        spiders = self.crawler_process.spiders
         if opts.spider:
             try:
-                self.spider = self.pcrawler.spiders.create(opts.spider, **opts.spargs)
+                self.spidercls = spiders.load(opts.spider)
             except KeyError:
                 log.msg(format='Unable to find spider: %(spider)s',
                         level=log.ERROR, spider=opts.spider)
         else:
-            self.spider = create_spider_for_request(self.pcrawler.spiders, Request(url), **opts.spargs)
-            if not self.spider:
+            self.spidercls = spidercls_for_request(spiders, Request(url))
+            if not self.spidercls:
                 log.msg(format='Unable to find spider for: %(url)s',
                         level=log.ERROR, url=url)
 
-    def start_parsing(self, url, opts):
         request = Request(url, opts.callback)
-        request = self.prepare_request(request, opts)
+        _start_requests = lambda s: [self.prepare_request(s, request, opts)]
+        self.spidercls.start_requests = _start_requests
 
-        self.pcrawler.crawl(self.spider, [request])
+
+    def start_parsing(self, url, opts):
+        self.crawler_process.crawl(self.spidercls, **opts.spargs)
+        self.pcrawler = list(self.crawler_process.crawlers)[0]
         self.crawler_process.start()
 
         if not self.first_response:
-            log.msg(format='No response downloaded for: %(request)s',
-                    level=log.ERROR, request=request)
+            log.msg(format='No response downloaded for: %(url)s',
+                    level=log.ERROR, url=url)
 
-    def prepare_request(self, request, opts):
+    def prepare_request(self, spider, request, opts):
         def callback(response):
             # memorize first request
             if not self.first_response:
@@ -157,17 +161,17 @@ class Command(ScrapyCommand):
             cb = response.meta['_callback']
             if not cb:
                 if opts.rules and self.first_response == response:
-                    cb = self.get_callback_from_rules(response)
+                    cb = self.get_callback_from_rules(spider, response)
                 else:
                     cb = 'parse'
 
             if not callable(cb):
-                cb_method = getattr(self.spider, cb, None)
+                cb_method = getattr(spider, cb, None)
                 if callable(cb_method):
                     cb = cb_method
                 else:
                     log.msg(format='Cannot find callback %(callback)r in spider: %(spider)s',
-                            callback=callback, spider=self.spider.name, level=log.ERROR)
+                            callback=callback, spider=spider.name, level=log.ERROR)
                     return
 
             # parse items and requests
@@ -177,7 +181,7 @@ class Command(ScrapyCommand):
             if opts.pipelines:
                 itemproc = self.pcrawler.engine.scraper.itemproc
                 for item in items:
-                    itemproc.process_item(item, self.spider)
+                    itemproc.process_item(item, spider)
             self.add_items(depth, items)
             self.add_requests(depth, requests)
 
@@ -207,10 +211,9 @@ class Command(ScrapyCommand):
         else:
             url = args[0]
 
-        # prepare spider
-        self.pcrawler = self.crawler_process.create_crawler()
-        self.set_spider(url, opts)
+        # prepare spidercls
+        self.set_spidercls(url, opts)
 
-        if self.spider and opts.depth > 0:
+        if self.spidercls and opts.depth > 0:
             self.start_parsing(url, opts)
             self.print_results(opts)
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index ab170e665..e4d32c314 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -8,6 +8,9 @@ from threading import Thread
 
 from scrapy.command import ScrapyCommand
 from scrapy.shell import Shell
+from scrapy.http import Request
+from scrapy import log
+from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 
 
 class Command(ScrapyCommand):
@@ -38,18 +41,27 @@ class Command(ScrapyCommand):
         pass
 
     def run(self, args, opts):
-        crawler = self.crawler_process.create_crawler()
-
         url = args[0] if args else None
-        spider = crawler.spiders.create(opts.spider) if opts.spider else None
+        spiders = self.crawler_process.spiders
 
-        self.crawler_process.start_crawling()
+        spidercls = DefaultSpider
+        if opts.spider:
+            spidercls = spiders.load(opts.spider)
+        elif url:
+            spidercls = spidercls_for_request(spiders, Request(url),
+                                              spidercls, log_multiple=True)
+        crawler = self.crawler_process._create_logged_crawler(spidercls)
+        crawler.engine = crawler._create_engine()
+        crawler.engine.start()
+
+        self.crawler_process._start_logging()
         self._start_crawler_thread()
 
         shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
-        shell.start(url=url, spider=spider)
+        shell.start(url=url)
 
     def _start_crawler_thread(self):
-        t = Thread(target=self.crawler_process.start_reactor)
+        t = Thread(target=self.crawler_process._start_reactor,
+                   kwargs={'stop_after_crawl': False})
         t.daemon = True
         t.start()
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 74eaef40f..6c48ef186 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -21,7 +21,6 @@ from scrapy.spider import Spider
 from scrapy.utils.console import start_python_console
 from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
-from scrapy.utils.spider import create_spider_for_request
 
 
 class Shell(object):
@@ -67,11 +66,9 @@ class Shell(object):
             return self.spider
 
         if spider is None:
-            spider = create_spider_for_request(self.crawler.spiders,
-                                               request,
-                                               Spider('default'),
-                                               log_multiple=True)
-        spider.set_crawler(self.crawler)
+            spider = self.crawler.spider or self.crawler._create_spider()
+
+        self.crawler.spider = spider
         self.crawler.engine.open_spider(spider, close_if_idle=False)
         self.spider = spider
         return spider
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 4e43bc13f..b81cf2b9b 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -4,6 +4,7 @@ import six
 
 from scrapy import log
 from scrapy.item import BaseItem
+from scrapy.spider import Spider
 from scrapy.utils.misc import  arg_to_iter
 
 
@@ -25,21 +26,21 @@ def iter_spider_classes(module):
            getattr(obj, 'name', None):
             yield obj
 
-def create_spider_for_request(spidermanager, request, default_spider=None, \
-        log_none=False, log_multiple=False, **spider_kwargs):
-    """Create a spider to handle the given Request.
+def spidercls_for_request(spidermanager, request, default_spidercls=None,
+                          log_none=False, log_multiple=False):
+    """Return a spider class that handles the given Request.
 
     This will look for the spiders that can handle the given request (using
-    the spider manager) and return a (new) Spider if (and only if) there is
+    the spider manager) and return a Spider class if (and only if) there is
     only one Spider able to handle the Request.
 
     If multiple spiders (or no spider) are found, it will return the
-    default_spider passed. It can optionally log if multiple or no spiders
+    default_spidercls passed. It can optionally log if multiple or no spiders
     are found.
     """
     snames = spidermanager.find_by_request(request)
     if len(snames) == 1:
-        return spidermanager.create(snames[0], **spider_kwargs)
+        return spidermanager.load(snames[0])
 
     if len(snames) > 1 and log_multiple:
         log.msg(format='More than one spider can handle: %(request)s - %(snames)s',
@@ -49,5 +50,8 @@ def create_spider_for_request(spidermanager, request, default_spider=None, \
         log.msg(format='Unable to find spider that handles: %(request)s',
                 level=log.ERROR, request=request)
 
-    return default_spider
+    return default_spidercls
 
+
+class DefaultSpider(Spider):
+    name = 'default'

From c90977ca98dd51b93d91739115d843f44e6a8a94 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 8 Aug 2014 06:15:20 -0300
Subject: [PATCH 0015/2974] Drop support for scrapy.project.crawler (And
 scrapy.stats consequently)

---
 conftest.py           |  2 +-
 docs/faq.rst          | 31 -------------------------------
 docs/topics/shell.rst |  2 +-
 scrapy/crawler.py     | 15 ---------------
 scrapy/project.py     | 18 +++++++++++-------
 scrapy/shell.py       |  5 ++---
 scrapy/spider.py      |  2 +-
 scrapy/stats.py       | 13 +++++++------
 tests/test_engine.py  |  2 --
 9 files changed, 23 insertions(+), 67 deletions(-)

diff --git a/conftest.py b/conftest.py
index 9f9a5bca7..aa27ddd2b 100644
--- a/conftest.py
+++ b/conftest.py
@@ -4,7 +4,7 @@ from twisted.python import log
 
 from scrapy import optional_features
 
-collect_ignore = ["scrapy/stats.py"]
+collect_ignore = ["scrapy/stats.py", "scrapy/project.py"]
 if 'django' not in optional_features:
     collect_ignore.append("tests/test_djangoitem/models.py")
 
diff --git a/docs/faq.rst b/docs/faq.rst
index 47bfede71..1d6c56d97 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -280,37 +280,6 @@ I'm scraping a XML document and my XPath selector doesn't return any items
 
 You may need to remove namespaces. See :ref:`removing-namespaces`.
 
-
-I'm getting an error: "cannot import name crawler"
---------------------------------------------------
-
-This is caused by Scrapy changes due to the singletons removal. The error is
-most likely raised by a module (extension, middleware, pipeline or spider) in
-your Scrapy project that imports ``crawler`` from ``scrapy.project``. For
-example::
-
-    from scrapy.project import crawler
-
-    class SomeExtension(object):
-        def __init__(self):
-            self.crawler = crawler
-            # ...
-
-This way to access the crawler object is deprecated, the code should be ported
-to use ``from_crawler`` class method, for example::
-
-    class SomeExtension(object):
-
-        @classmethod
-        def from_crawler(cls, crawler):
-            o = cls()
-            o.crawler = crawler
-            return o
-
-Scrapy command line tool has some backwards compatibility in place to support
-the old import mechanism (with a deprecation warning), but this mechanism may
-not work if you use Scrapy differently (for example, as a library).
-
 .. _user agents: http://en.wikipedia.org/wiki/User_agent
 .. _LIFO: http://en.wikipedia.org/wiki/LIFO
 .. _DFO order: http://en.wikipedia.org/wiki/Depth-first_search
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 37268c3ca..5c1cfbd47 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -186,7 +186,7 @@ Here's an example of how you would call it from your spider::
             # We want to inspect one specific response.
             if ".org" in response.url:
                 from scrapy.shell import inspect_response
-                inspect_response(response)
+                inspect_response(response, self)
 
             # Rest of parsing code.
 
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 597bb2e9d..352cff6e5 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -31,18 +31,6 @@ class Crawler(object):
         self.spider = None
         self.engine = None
 
-    def install(self):
-        # TODO: remove together with scrapy.project.crawler usage
-        import scrapy.project
-        assert not hasattr(scrapy.project, 'crawler'), "crawler already installed"
-        scrapy.project.crawler = self
-
-    def uninstall(self):
-        # TODO: remove together with scrapy.project.crawler usage
-        import scrapy.project
-        assert hasattr(scrapy.project, 'crawler'), "crawler not installed"
-        del scrapy.project.crawler
-
     @defer.inlineCallbacks
     def crawl(self, *args, **kwargs):
         assert not self.crawling, "Crawling already taking place"
@@ -84,9 +72,6 @@ class CrawlerRunner(object):
         crawler = self._create_logged_crawler(spidercls)
         self.crawlers.add(crawler)
 
-        crawler.install()
-        crawler.signals.connect(crawler.uninstall, signals.engine_stopped)
-
         d = crawler.crawl(*args, **kwargs)
         self.crawl_deferreds.add(d)
         return d
diff --git a/scrapy/project.py b/scrapy/project.py
index bbe947761..d8973a6c7 100644
--- a/scrapy/project.py
+++ b/scrapy/project.py
@@ -1,13 +1,17 @@
-"""
---------- WARNING: THIS MODULE IS DEPRECATED -----------
 
-This module is deprecated. If you want to get the Scrapy crawler from your
-extension, middleware or pipeline implement the `from_crawler` class method.
+"""
+Obsolete module, kept for giving a meaningful error message when trying to
+import.
+"""
+
+raise ImportError("""scrapy.project usage has become obsolete.
+
+If you want to get the Scrapy crawler from your extension, middleware or
+pipeline implement the `from_crawler` class method (or look up for extending
+components that have already done it, such as spiders).
 
 For example:
 
     @classmethod
     def from_crawler(cls, crawler):
-        return cls(crawler)
-
-"""
+        return cls(crawler)""")
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 6c48ef186..8f87fcb41 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -123,10 +123,9 @@ class Shell(object):
         return isinstance(value, self.relevant_classes)
 
 
-def inspect_response(response, spider=None):
+def inspect_response(response, spider):
     """Open a shell to inspect the given response"""
-    from scrapy.project import crawler
-    Shell(crawler).start(response=response, spider=spider)
+    Shell(spider.crawler).start(response=response)
 
 
 def _request_deferred(request):
diff --git a/scrapy/spider.py b/scrapy/spider.py
index df367b700..943925042 100644
--- a/scrapy/spider.py
+++ b/scrapy/spider.py
@@ -93,6 +93,6 @@ class ObsoleteClass(object):
         raise AttributeError(self.message)
 
 spiders = ObsoleteClass("""
-"from scrapy.spider import spiders" no longer works - use "from scrapy.project import crawler" and then access crawler.spiders attribute"
+"from scrapy.spider import spiders" no longer works - use "from scrapy.spidermanager import SpiderManager" and instantiate it with your project settings"
 """)
 
diff --git a/scrapy/stats.py b/scrapy/stats.py
index b8128dfc2..710601430 100644
--- a/scrapy/stats.py
+++ b/scrapy/stats.py
@@ -1,7 +1,8 @@
-from scrapy.project import crawler
-stats = crawler.stats
 
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.stats` is deprecated, use `crawler.stats` attribute instead",
-    ScrapyDeprecationWarning, stacklevel=2)
+"""
+Obsolete module, kept for giving a meaningful error message when trying to
+import.
+"""
+
+raise ImportError("scrapy.stats usage has become obsolete, use "
+                  "`crawler.stats` attribute instead")
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 244d339ef..67fb8ae79 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -93,7 +93,6 @@ class CrawlerRun(object):
                 dispatcher.connect(self.record_signal, signal)
 
         self.crawler = get_crawler(TestSpider)
-        self.crawler.install()
         self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
         self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
         self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded)
@@ -109,7 +108,6 @@ class CrawlerRun(object):
         for name, signal in vars(signals).items():
             if not name.startswith('_'):
                 disconnect_all(signal)
-        self.crawler.uninstall()
         self.deferred.callback(None)
 
     def geturl(self, path):

From 419026615f040d6277e311b9307a3950f8532468 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 14 Aug 2014 09:19:41 -0300
Subject: [PATCH 0016/2974] Deprecate Crawler.spiders attribute

---
 docs/topics/api.rst   |  6 ------
 scrapy/crawler.py     | 18 ++++++++++++++----
 tests/py3-ignores.txt |  1 +
 tests/test_crawler.py | 24 ++++++++++++++++++++++++
 4 files changed, 39 insertions(+), 10 deletions(-)
 create mode 100644 tests/test_crawler.py

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 0329e2a8f..03a0b4124 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -76,12 +76,6 @@ how you :ref:`configure the downloader middlewares
         For an introduction on extensions and a list of available extensions on
         Scrapy see :ref:`topics-extensions`.
 
-    .. attribute:: spiders
-
-        The spider manager which takes care of loading spiders.
-
-        Most extensions won't need to access this attribute.
-
     .. attribute:: engine
 
         The execution engine, which coordinates the core crawling logic
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 352cff6e5..52e57fe83 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,5 +1,6 @@
 import six
 import signal
+import warnings
 
 from twisted.internet import reactor, defer
 
@@ -7,6 +8,7 @@ from scrapy.core.engine import ExecutionEngine
 from scrapy.resolver import CachingThreadedResolver
 from scrapy.extension import ExtensionManager
 from scrapy.signalmanager import SignalManager
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 from scrapy.utils.misc import load_object
 from scrapy import log, signals
@@ -23,14 +25,22 @@ class Crawler(object):
         self.logformatter = lf_cls.from_crawler(self)
         self.extensions = ExtensionManager.from_crawler(self)
 
-        # Attribute kept for backward compatibility (Use CrawlerRunner.spiders)
-        spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])
-        self.spiders = spman_cls.from_settings(self.settings)
-
         self.crawling = False
         self.spider = None
         self.engine = None
 
+    @property
+    def spiders(self):
+        if not hasattr(self, '_spiders'):
+            warnings.warn("Crawler.spiders is deprecated, use "
+                          "CrawlerRunner.spiders or instantiate "
+                          "scrapy.spidermanager.SpiderManager with your "
+                          "settings.",
+                          category=ScrapyDeprecationWarning, stacklevel=2)
+            spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])
+            self._spiders = spman_cls.from_settings(self.settings)
+        return self._spiders
+
     @defer.inlineCallbacks
     def crawl(self, *args, **kwargs):
         assert not self.crawling, "Crawling already taking place"
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index ef88eab7e..f3c667cd0 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -8,6 +8,7 @@ tests/test_contrib_exporter.py
 tests/test_contrib_linkextractors.py
 tests/test_contrib_loader.py
 tests/test_crawl.py
+tests/test_crawler.py
 tests/test_djangoitem/__init__.py
 tests/test_downloader_handlers.py
 tests/test_downloadermiddleware_ajaxcrawlable.py
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
new file mode 100644
index 000000000..55381c030
--- /dev/null
+++ b/tests/test_crawler.py
@@ -0,0 +1,24 @@
+import warnings
+import unittest
+
+from scrapy.crawler import Crawler
+from scrapy.settings import Settings
+from scrapy.utils.spider import DefaultSpider
+from scrapy.utils.misc import load_object
+
+
+class CrawlerTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.crawler = Crawler(DefaultSpider, Settings())
+
+    def test_deprecated_attribute_spiders(self):
+        with warnings.catch_warnings(record=True) as w:
+            spiders = self.crawler.spiders
+            self.assertEqual(len(w), 1)
+            self.assertIn("Crawler.spiders", str(w[0].message))
+            sm_cls = load_object(self.crawler.settings['SPIDER_MANAGER_CLASS'])
+            self.assertIsInstance(spiders, sm_cls)
+
+            self.crawler.spiders
+            self.assertEqual(len(w), 1, "Warn deprecated access only once")

From 3547ca6e618e19dda86ad1505323b24e82d317bd Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 14 Aug 2014 11:50:33 -0300
Subject: [PATCH 0017/2974] Add example on running spiders outside projects

---
 docs/topics/practices.rst | 36 +++++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index e84478d3c..b188ee562 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -32,19 +32,41 @@ project as example.
     from scrapy.crawler import CrawlerRunner
     from scrapy.utils.project import get_project_settings
 
-    # If you aren't inside a Scrapy project, you could use an instance of the
-    # Settings class in scrapy.settings instead of the configuration returned
-    # by get_project_settings
     runner = CrawlerRunner(get_project_settings())
 
-    # 'followall' is the name of one of the spiders of the project. If you
-    # aren't working in a Scrapy project, use the spider class as first
-    # argument instead of its name (or set the SPIDER_MODULES setting so Scrapy
-    # knows where to look at)
+    # 'followall' is the name of one of the spiders of the project.
     d = runner.crawl('followall', domain='scrapinghub.com')
     d.addBoth(lambda _: reactor.stop())
     reactor.run() # the script will block here until the crawling is finished
 
+Running spiders outside projects it's not much different. You have to create a
+generic :class:`~scrapy.settings.Settings` object and populate it as needed
+(See :ref:`topics-settings-ref` for the available settings), instead of using
+the configuration returned by `get_project_settings`.
+
+Spiders can still be referenced by their name if :setting:`SPIDER_MODULES` is
+set with the modules where Scrapy should look for spiders.  Otherwise, passing
+the spider class as first argument in the :meth:`CrawlerRunner.crawl
+<scrapy.crawler.CrawlerRunner.crawl>` method is enough.
+
+::
+
+    from twisted.internet import reactor
+    from scrapy.spider import Spider
+    from scrapy.crawler import CrawlerRunner
+    from scrapy.settings import Settings
+
+    class MySpider(Spider):
+        # Your spider definition
+        ...
+
+    settings = Settings({'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'})
+    runner = CrawlerRunner(settings)
+
+    d = runner.crawl(MySpider)
+    d.addBoth(lambda _: reactor.stop())
+    reactor.run() # the script will block here until the crawling is finished
+
 .. seealso:: `Twisted Reactor Overview`_.
 
 Running multiple spiders in the same process

From 70f2010db17b3796c3edce9c6ca3e00c092d6064 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 14 Aug 2014 11:59:25 -0300
Subject: [PATCH 0018/2974] Change error type when updating frozen settings

---
 scrapy/settings/__init__.py     | 10 +++++++---
 tests/test_settings/__init__.py |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index bbe8ef481..938b93564 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -88,7 +88,7 @@ class Settings(object):
         return dict(value)
 
     def set(self, name, value, priority='project'):
-        assert not self.frozen, "Trying to modify an immutable Settings object"
+        self._assert_mutability()
         if isinstance(priority, six.string_types):
             priority = SETTINGS_PRIORITIES[priority]
         if name not in self.attributes:
@@ -97,18 +97,22 @@ class Settings(object):
             self.attributes[name].set(value, priority)
 
     def setdict(self, values, priority='project'):
-        assert not self.frozen, "Trying to modify an immutable Settings object"
+        self._assert_mutability()
         for name, value in six.iteritems(values):
             self.set(name, value, priority)
 
     def setmodule(self, module, priority='project'):
-        assert not self.frozen, "Trying to modify an immutable Settings object"
+        self._assert_mutability()
         if isinstance(module, six.string_types):
             module = import_module(module)
         for key in dir(module):
             if key.isupper():
                 self.set(key, getattr(module, key), priority)
 
+    def _assert_mutability(self):
+        if self.frozen:
+            raise TypeError("Trying to modify an immutable Settings object")
+
     def copy(self):
         return copy.deepcopy(self)
 
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index c7e0914d6..38797ad45 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -213,7 +213,7 @@ class SettingsTest(unittest.TestCase):
 
     def test_freeze(self):
         self.settings.freeze()
-        with self.assertRaises(AssertionError) as cm:
+        with self.assertRaises(TypeError) as cm:
             self.settings.set('TEST_BOOL', False)
             self.assertEqual(str(cm.exception),
                              "Trying to modify an immutable Settings object")

From 6339864f95d4ecaed5e428b342277dce9457d954 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 14 Aug 2014 12:32:37 -0300
Subject: [PATCH 0019/2974] Minor refactor in the docs and functions used in
 the shell command

---
 scrapy/commands/shell.py | 6 +++++-
 scrapy/crawler.py        | 8 +++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index e4d32c314..e2ef1545e 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -50,11 +50,15 @@ class Command(ScrapyCommand):
         elif url:
             spidercls = spidercls_for_request(spiders, Request(url),
                                               spidercls, log_multiple=True)
+
+        # The crawler is created this way since the Shell manually handles the
+        # crawling engine, so the set up in the crawl method won't work
         crawler = self.crawler_process._create_logged_crawler(spidercls)
+        # The Shell class needs a persistent engine in the crawler
         crawler.engine = crawler._create_engine()
         crawler.engine.start()
 
-        self.crawler_process._start_logging()
+        self.crawler_process.start(start_reactor=False)
         self._start_crawler_thread()
 
         shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 52e57fe83..e0524021a 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -126,13 +126,11 @@ class CrawlerProcess(CrawlerRunner):
         self._stop_logging()
         reactor.callFromThread(self._stop_reactor)
 
-    def start(self, stop_after_crawl=True):
-        self._start_logging()
-        self._start_reactor(stop_after_crawl)
-
-    def _start_logging(self):
+    def start(self, stop_after_crawl=True, start_reactor=True):
         self.log_observer = log.start_from_settings(self.settings)
         log.scrapy_info(self.settings)
+        if start_reactor:
+            self._start_reactor(stop_after_crawl)
 
     def _start_reactor(self, stop_after_crawl=True):
         if stop_after_crawl:

From 6f50cf55a4ded1de1238a71b4dac7ebd85f07a75 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 13 Aug 2014 21:39:31 -0300
Subject: [PATCH 0020/2974] fix IPython shell scope issue and load IPython user
 config

---
 scrapy/utils/console.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 5dd4539ed..924501002 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -13,16 +13,16 @@ def start_python_console(namespace=None, noipython=False, banner=''):
                 raise ImportError()
 
             try:
-                try:
-                    from IPython.terminal import embed
-                except ImportError:
-                    from IPython.frontend.terminal import embed
-                sh = embed.InteractiveShellEmbed(banner1=banner)
+                from IPython.terminal.embed import InteractiveShellEmbed
+                from IPython.terminal.ipapp import load_default_config
             except ImportError:
-                from IPython.Shell import IPShellEmbed
-                sh = IPShellEmbed(banner=banner)
+                from IPython.frontend.terminal.embed import InteractiveShellEmbed
+                from IPython.frontend.terminal.ipapp import load_default_config
 
-            sh(global_ns={}, local_ns=namespace)
+            config = load_default_config()
+            shell = InteractiveShellEmbed(
+                banner1=banner, user_ns=namespace, config=config)
+            shell()
         except ImportError:
             import code
             try: # readline module is only available on unix systems

From d49766a6ac2b2685f66d69a93e4726b6a392ab2f Mon Sep 17 00:00:00 2001
From: Uyounghz <younghaizhi@gmail.com>
Date: Thu, 28 Aug 2014 19:58:58 +0800
Subject: [PATCH 0021/2974] Duplicate comma in request-response.rst

---
 docs/topics/request-response.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 192f4caee..18a0ae23f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -51,7 +51,7 @@ Request objects
 
     :param body: the request body. If a ``unicode`` is passed, then it's encoded to
       ``str`` using the `encoding` passed (which defaults to ``utf-8``). If
-      ``body`` is not given,, an empty string is stored. Regardless of the
+      ``body`` is not given, an empty string is stored. Regardless of the
       type of this argument, the final value stored will be a ``str`` (never
       ``unicode`` or ``None``).
     :type body: str or unicode

From e4689556f01fd74502584c85cba7c72c2b57640d Mon Sep 17 00:00:00 2001
From: yakxxx <yakxxx@gmail.com>
Date: Thu, 28 Aug 2014 18:47:49 +0200
Subject: [PATCH 0022/2974] SgmlLinkExtractor - fix for parsing <area> tag with
 Unicode present

---
 scrapy/contrib/linkextractors/sgml.py |  1 +
 tests/test_contrib_linkextractors.py  | 11 +++++++++++
 2 files changed, 12 insertions(+)

diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py
index 9a55c1581..3eb5fd91f 100644
--- a/scrapy/contrib/linkextractors/sgml.py
+++ b/scrapy/contrib/linkextractors/sgml.py
@@ -67,6 +67,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
         SGMLParser.reset(self)
         self.links = []
         self.base_url = None
+        self.current_link = None
 
     def unknown_starttag(self, tag, attrs):
         if tag == 'base':
diff --git a/tests/test_contrib_linkextractors.py b/tests/test_contrib_linkextractors.py
index 3617cb810..3902d4c50 100644
--- a/tests/test_contrib_linkextractors.py
+++ b/tests/test_contrib_linkextractors.py
@@ -284,6 +284,17 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
                          [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
                                fragment='', nofollow=False)])
 
+    def test_area_tag_with_unicode_present(self):
+        body = """<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>"""
+        response = HtmlResponse("http://example.org", body=body, encoding='utf-8')
+        lx = self.extractor_cls()
+        lx.extract_links(response)
+        lx.extract_links(response)
+        lx.extract_links(response)
+        self.assertEqual(lx.extract_links(response),
+                         [Link(url='http://example.org/foo', text=u'',
+                               fragment='', nofollow=False)])
+
     def test_encoded_url(self):
         body = """<html><body><div><a href="?page=2">BinB</a></body></html>"""
         response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')

From d92914d2978c0fc3acc27c832184eb98b5f6c3cc Mon Sep 17 00:00:00 2001
From: Adam Donahue <adam.donahue@gmail.com>
Date: Thu, 28 Aug 2014 20:30:50 -0400
Subject: [PATCH 0023/2974] Fix typo

---
 scrapy/utils/trackref.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index 252c40632..f250aeefa 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -2,7 +2,7 @@
 references to live object instances.
 
 If you want live objects for a particular class to be tracked, you only have to
-subclass form object_ref (instead of object).
+subclass from object_ref (instead of object).
 
 About performance: This library has a minimal performance impact when enabled,
 and no performance penalty at all when disabled (as object_ref becomes just an

From 4932ec43a76a209c6e5d32440e3bc5b41cfbfe0d Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 13 Aug 2014 01:41:16 -0300
Subject: [PATCH 0024/2974] Per-spider settings implementation

---
 scrapy/crawler.py           | 7 ++++++-
 scrapy/settings/__init__.py | 1 +
 scrapy/spider.py            | 5 +++++
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index e0524021a..eb174a7d8 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -96,7 +96,12 @@ class CrawlerRunner(object):
     def _create_crawler(self, spidercls):
         if isinstance(spidercls, six.string_types):
             spidercls = self.spiders.load(spidercls)
-        crawler = Crawler(spidercls, self.settings.frozencopy())
+
+        crawler_settings = self.settings.copy()
+        spidercls.update_settings(crawler_settings)
+        crawler_settings.freeze()
+
+        crawler = Crawler(spidercls, crawler_settings)
         return crawler
 
     def stop(self):
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 938b93564..af0d0dff1 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -15,6 +15,7 @@ SETTINGS_PRIORITIES = {
     'default': 0,
     'command': 10,
     'project': 20,
+    'spider': 30,
     'cmdline': 40,
 }
 
diff --git a/scrapy/spider.py b/scrapy/spider.py
index 943925042..0b13d2ee6 100644
--- a/scrapy/spider.py
+++ b/scrapy/spider.py
@@ -20,6 +20,7 @@ class Spider(object_ref):
     """
 
     name = None
+    custom_settings = {}
 
     def __init__(self, name=None, **kwargs):
         if name is not None:
@@ -66,6 +67,10 @@ class Spider(object_ref):
     def parse(self, response):
         raise NotImplementedError
 
+    @classmethod
+    def update_settings(cls, settings):
+        settings.setdict(cls.custom_settings, priority='spider')
+
     @classmethod
     def handles_request(cls, request):
         return url_is_from_spider(request.url, cls)

From 9ef3972cfbe5badaff52b75883c1cb889dcd501c Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 13 Aug 2014 01:41:50 -0300
Subject: [PATCH 0025/2974] Per-spider settings tests

---
 tests/test_spider.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/test_spider.py b/tests/test_spider.py
index 148a872dd..585b4b53b 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -10,6 +10,7 @@ except ImportError:
 
 from scrapy import signals
 from scrapy.spider import Spider, BaseSpider
+from scrapy.settings import Settings
 from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse
 from scrapy.contrib.spiders.init import InitSpider
 from scrapy.contrib.spiders import CrawlSpider, Rule, XMLFeedSpider, \
@@ -92,6 +93,16 @@ class SpiderTest(unittest.TestCase):
                                        spider=spider, reason=None)
         self.assertTrue(spider.closed_called)
 
+    def test_update_settings(self):
+        spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
+        project_settings = {'TEST1': 'project', 'TEST3': 'project'}
+        self.spider_class.custom_settings = spider_settings
+        settings = Settings(project_settings, priority='project')
+
+        self.spider_class.update_settings(settings)
+        self.assertEqual(settings.get('TEST1'), 'spider')
+        self.assertEqual(settings.get('TEST2'), 'spider')
+        self.assertEqual(settings.get('TEST3'), 'project')
 
 class InitSpiderTest(SpiderTest):
 

From 16e62e9c9bffc418af0b9534d5f05879281dcf9c Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 13 Aug 2014 01:42:34 -0300
Subject: [PATCH 0026/2974] Per-spider settings documentation

---
 docs/topics/api.rst      |  1 +
 docs/topics/settings.rst | 20 ++++++++++++++------
 docs/topics/spiders.rst  |  9 +++++++++
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 03a0b4124..57b8ee0cf 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -172,6 +172,7 @@ Settings API
             'default': 0,
             'command': 10,
             'project': 20,
+            'spider': 30,
             'cmdline': 40,
         }
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index e0c432cb3..89ee7605a 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -36,9 +36,10 @@ different precedence. Here is the list of them in decreasing order of
 precedence:
 
  1. Command line options (most precedence)
- 2. Project settings module
- 3. Default settings per-command
- 4. Default global settings (less precedence)
+ 2. Settings per-spider
+ 3. Project settings module
+ 4. Default settings per-command
+ 5. Default global settings (less precedence)
 
 The population of these settings sources is taken care of internally, but a
 manual handling is possible using API calls. See the
@@ -59,14 +60,21 @@ Example::
 
     scrapy crawl myspider -s LOG_FILE=scrapy.log
 
-2. Project settings module
+2. Settings per-spider
+----------------------
+
+Spiders (See the :ref:`topics-spiders` chapter for reference) can define their
+own settings that will take precedence and override the project ones. They can
+do so by setting their :attr:`scrapy.spider.Spider.custom_settings` attribute.
+
+3. Project settings module
 --------------------------
 
 The project settings module is the standard configuration file for your Scrapy
 project.  It's where most of your custom settings will be populated. For
 example:: ``myproject.settings``.
 
-3. Default settings per-command
+4. Default settings per-command
 -------------------------------
 
 Each :doc:`Scrapy tool </topics/commands>` command can have its own default
@@ -74,7 +82,7 @@ settings, which override the global default settings. Those custom command
 settings are specified in the ``default_settings`` attribute of the command
 class.
 
-4. Default global settings
+5. Default global settings
 --------------------------
 
 The global defaults are located in the ``scrapy.settings.default_settings``
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index de8f988c0..188b51836 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -133,6 +133,15 @@ Spider
        listed here. The subsequent URLs will be generated successively from data
        contained in the start URLs.
 
+   .. attribute:: custom_settings
+
+      A dictionary of settings that will be overridden from the project wide
+      configuration when running this spider. It must be defined as a class
+      attribute since the settings are updated before instantiation.
+
+      For a list of available built-in settings see:
+      :ref:`topics-settings-ref`.
+
    .. attribute:: crawler
 
       This attribute is set by the :meth:`from_crawler` class method after

From 77bd26a66de7dd80152c89d0c09abf3e7b830612 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 14 Aug 2014 14:01:12 -0300
Subject: [PATCH 0027/2974] Non mutable default in Spider.custom_settings

---
 scrapy/spider.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/spider.py b/scrapy/spider.py
index 0b13d2ee6..dada6236b 100644
--- a/scrapy/spider.py
+++ b/scrapy/spider.py
@@ -20,7 +20,7 @@ class Spider(object_ref):
     """
 
     name = None
-    custom_settings = {}
+    custom_settings = None
 
     def __init__(self, name=None, **kwargs):
         if name is not None:
@@ -69,7 +69,7 @@ class Spider(object_ref):
 
     @classmethod
     def update_settings(cls, settings):
-        settings.setdict(cls.custom_settings, priority='spider')
+        settings.setdict(cls.custom_settings or {}, priority='spider')
 
     @classmethod
     def handles_request(cls, request):

From c2592b39fde98ac8ba46c165deb4a1245954f3a1 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 14 Aug 2014 14:17:18 -0300
Subject: [PATCH 0028/2974] Test verifying that CrawlerRunner populates spider
 class settings

---
 tests/test_crawler.py | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 55381c030..0031c8215 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,7 +1,9 @@
 import warnings
 import unittest
 
-from scrapy.crawler import Crawler
+from twisted.internet import defer
+
+from scrapy.crawler import Crawler, CrawlerRunner
 from scrapy.settings import Settings
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
@@ -22,3 +24,26 @@ class CrawlerTestCase(unittest.TestCase):
 
             self.crawler.spiders
             self.assertEqual(len(w), 1, "Warn deprecated access only once")
+
+
+class CrawlerRunnerTest(unittest.TestCase):
+
+    def setUp(self):
+        self.crawler_runner = CrawlerRunner(Settings())
+
+    @defer.inlineCallbacks
+    def test_populate_spidercls_settings(self):
+        spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
+        project_settings = {'TEST1': 'project', 'TEST3': 'project'}
+
+        class CustomSettingsSpider(DefaultSpider):
+            custom_settings = spider_settings
+
+        self.crawler_runner.settings.setdict(project_settings,
+                                             priority='project')
+
+        yield self.crawler_runner.crawl(CustomSettingsSpider)
+        crawler = self.crawler_runner.crawlers.pop()
+        self.assertEqual(crawler.settings.get('TEST1'), 'spider')
+        self.assertEqual(crawler.settings.get('TEST2'), 'spider')
+        self.assertEqual(crawler.settings.get('TEST3'), 'project')

From 1dff1fbf75d8277c52cfdd003966fd7e48b972f1 Mon Sep 17 00:00:00 2001
From: eltermann <felipe.eltermann@gmail.com>
Date: Tue, 2 Sep 2014 08:33:36 -0300
Subject: [PATCH 0029/2974] Removed unused 'load=False' parameter from
 walk_modules()

---
 scrapy/utils/misc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index be394eb1d..4cfd35342 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -54,7 +54,7 @@ def load_object(path):
     return obj
 
 
-def walk_modules(path, load=False):
+def walk_modules(path):
     """Loads a module and all its submodules from a the given module path and
     returns them. If *any* module throws an exception while importing, that
     exception is thrown back.

From dfca7b3c8001d8489f8bc7c609b6da5fd072b87a Mon Sep 17 00:00:00 2001
From: andrewshir <andrewshirokoff@gmail.com>
Date: Sat, 6 Sep 2014 18:23:27 +0600
Subject: [PATCH 0030/2974] Fix bug for ".local" host name

It's necessary to put new list member in squared brackets (i.e. create new list) to merge lists properly, otherwise we will get result list with character elements instead of string element.
---
 scrapy/http/cookies.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index cc96cf8ac..3e3dff741 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -29,7 +29,7 @@ class CookieJar(object):
         if not IPV4_RE.search(req_host):
             hosts = potential_domain_matches(req_host)
             if req_host.find(".") == -1:
-                hosts += req_host + ".local"
+                hosts += [req_host + ".local"]
         else:
             hosts = [req_host]
 

From d513b5a542f5581a60a10200e07b58ec294caea5 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sun, 7 Sep 2014 13:02:39 -0300
Subject: [PATCH 0031/2974] Run root logger in CrawlerProcess creation instead
 of in its start method

---
 scrapy/crawler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index eb174a7d8..062f2f9fe 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -115,6 +115,8 @@ class CrawlerProcess(CrawlerRunner):
         super(CrawlerProcess, self).__init__(settings)
         install_shutdown_handlers(self._signal_shutdown)
         self.stopping = False
+        self.log_observer = log.start_from_settings(self.settings)
+        log.scrapy_info(settings)
 
     def _signal_shutdown(self, signum, _):
         install_shutdown_handlers(self._signal_kill)
@@ -132,8 +134,6 @@ class CrawlerProcess(CrawlerRunner):
         reactor.callFromThread(self._stop_reactor)
 
     def start(self, stop_after_crawl=True, start_reactor=True):
-        self.log_observer = log.start_from_settings(self.settings)
-        log.scrapy_info(self.settings)
         if start_reactor:
             self._start_reactor(stop_after_crawl)
 

From 51532af69a551292663fa87279eea0317b4ea3a5 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sun, 7 Sep 2014 13:03:34 -0300
Subject: [PATCH 0032/2974] Erase unneeded flag in CrawlerProcess.start

---
 scrapy/commands/shell.py | 3 +--
 scrapy/crawler.py        | 6 +-----
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index e2ef1545e..ff8c0d156 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -58,14 +58,13 @@ class Command(ScrapyCommand):
         crawler.engine = crawler._create_engine()
         crawler.engine.start()
 
-        self.crawler_process.start(start_reactor=False)
         self._start_crawler_thread()
 
         shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
         shell.start(url=url)
 
     def _start_crawler_thread(self):
-        t = Thread(target=self.crawler_process._start_reactor,
+        t = Thread(target=self.crawler_process.start,
                    kwargs={'stop_after_crawl': False})
         t.daemon = True
         t.start()
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 062f2f9fe..00de7d0c0 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -133,11 +133,7 @@ class CrawlerProcess(CrawlerRunner):
         self._stop_logging()
         reactor.callFromThread(self._stop_reactor)
 
-    def start(self, stop_after_crawl=True, start_reactor=True):
-        if start_reactor:
-            self._start_reactor(stop_after_crawl)
-
-    def _start_reactor(self, stop_after_crawl=True):
+    def start(self, stop_after_crawl=True):
         if stop_after_crawl:
             d = defer.DeferredList(self.crawl_deferreds)
             if d.called:

From 8ddf0811a886bf6a604ae5cf946a7180632b51c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Tue, 2 Sep 2014 18:07:32 -0300
Subject: [PATCH 0033/2974] Correctly detect when all managed crawlers are done
 in CrawlerRunner

---
 scrapy/commands/shell.py |  3 ++-
 scrapy/crawler.py        | 44 ++++++++++++++++++++++++----------------
 2 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index ff8c0d156..7c0706482 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -53,7 +53,8 @@ class Command(ScrapyCommand):
 
         # The crawler is created this way since the Shell manually handles the
         # crawling engine, so the set up in the crawl method won't work
-        crawler = self.crawler_process._create_logged_crawler(spidercls)
+        crawler = self.crawler_process._create_crawler(spidercls)
+        self.crawler_process._setup_crawler_logging(crawler)
         # The Shell class needs a persistent engine in the crawler
         crawler.engine = crawler._create_engine()
         crawler.engine.start()
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 00de7d0c0..f18760390 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -76,22 +76,21 @@ class CrawlerRunner(object):
         smcls = load_object(settings['SPIDER_MANAGER_CLASS'])
         self.spiders = smcls.from_settings(settings.frozencopy())
         self.crawlers = set()
-        self.crawl_deferreds = set()
+        self._active = set()
 
     def crawl(self, spidercls, *args, **kwargs):
-        crawler = self._create_logged_crawler(spidercls)
-        self.crawlers.add(crawler)
-
-        d = crawler.crawl(*args, **kwargs)
-        self.crawl_deferreds.add(d)
-        return d
-
-    def _create_logged_crawler(self, spidercls):
         crawler = self._create_crawler(spidercls)
-        log_observer = log.start_from_crawler(crawler)
-        if log_observer:
-            crawler.signals.connect(log_observer.stop, signals.engine_stopped)
-        return crawler
+        self._setup_crawler_logging(crawler)
+        self.crawlers.add(crawler)
+        d = crawler.crawl(*args, **kwargs)
+        self._active.add(d)
+
+        def _done(result):
+            self.crawlers.discard(crawler)
+            self._active.discard(d)
+            return result
+
+        return d.addBoth(_done)
 
     def _create_crawler(self, spidercls):
         if isinstance(spidercls, six.string_types):
@@ -100,13 +99,22 @@ class CrawlerRunner(object):
         crawler_settings = self.settings.copy()
         spidercls.update_settings(crawler_settings)
         crawler_settings.freeze()
+        return Crawler(spidercls, crawler_settings)
 
-        crawler = Crawler(spidercls, crawler_settings)
-        return crawler
+    def _setup_crawler_logging(self, crawler):
+        log_observer = log.start_from_crawler(crawler)
+        if log_observer:
+            crawler.signals.connect(log_observer.stop, signals.engine_stopped)
 
     def stop(self):
         return defer.DeferredList(c.stop() for c in self.crawlers)
 
+    @defer.inlineCallbacks
+    def join(self):
+        """Wait for all managed crawlers to complete"""
+        while self._active:
+            yield defer.DeferredList(self._active)
+
 
 class CrawlerProcess(CrawlerRunner):
     """A class to run multiple scrapy crawlers in a process simultaneously"""
@@ -135,13 +143,15 @@ class CrawlerProcess(CrawlerRunner):
 
     def start(self, stop_after_crawl=True):
         if stop_after_crawl:
-            d = defer.DeferredList(self.crawl_deferreds)
+            d = self.join()
+            # Don't start the reactor if the deferreds are already fired
             if d.called:
-                # Don't start the reactor if the deferreds are already fired
                 return
             d.addBoth(lambda _: self._stop_reactor())
+
         if self.settings.getbool('DNSCACHE_ENABLED'):
             reactor.installResolver(CachingThreadedResolver(reactor))
+
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
         reactor.run(installSignalHandlers=False)  # blocking call
 

From 99971dc8a883fd94909b7e1087ea3b2b450bd319 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Tue, 9 Sep 2014 20:59:07 +0000
Subject: [PATCH 0034/2974] Do not pop the crawler from the managed list

---
 tests/test_crawler.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 0031c8215..ca3af783e 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -42,8 +42,9 @@ class CrawlerRunnerTest(unittest.TestCase):
         self.crawler_runner.settings.setdict(project_settings,
                                              priority='project')
 
-        yield self.crawler_runner.crawl(CustomSettingsSpider)
-        crawler = self.crawler_runner.crawlers.pop()
+        d = self.crawler_runner.crawl(CustomSettingsSpider)
+        crawler = list(self.crawler_runner.crawlers)[0]
+        yield d
         self.assertEqual(crawler.settings.get('TEST1'), 'spider')
         self.assertEqual(crawler.settings.get('TEST2'), 'spider')
         self.assertEqual(crawler.settings.get('TEST3'), 'project')

From ce180227fa8a056a4128fd1e381c9ae5b83b1be7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 10 Sep 2014 12:04:14 -0300
Subject: [PATCH 0035/2974] Twisted 11.1.0 (precise) can not deal with
 generators in DeferredList

Also create a list of the crawlers before iterating them because crawlers are removed from the set once stopped
---
 scrapy/crawler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index f18760390..ded09c1c7 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -107,7 +107,7 @@ class CrawlerRunner(object):
             crawler.signals.connect(log_observer.stop, signals.engine_stopped)
 
     def stop(self):
-        return defer.DeferredList(c.stop() for c in self.crawlers)
+        return defer.DeferredList([c.stop() for c in list(self.crawlers)])
 
     @defer.inlineCallbacks
     def join(self):

From ec93c0fdccb09c91f1eb560a538e00565cc0f2ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 10 Sep 2014 12:05:18 -0300
Subject: [PATCH 0036/2974] Add the tests changes for previous commit

---
 tests/test_crawler.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index ca3af783e..5128ea6e9 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -31,6 +31,9 @@ class CrawlerRunnerTest(unittest.TestCase):
     def setUp(self):
         self.crawler_runner = CrawlerRunner(Settings())
 
+    def tearDown(self):
+        return self.crawler_runner.stop()
+
     @defer.inlineCallbacks
     def test_populate_spidercls_settings(self):
         spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}

From a823207f18a339ffbe66bee6d8a6ef87a447f1a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 10 Sep 2014 12:09:07 -0300
Subject: [PATCH 0037/2974] Stop logobserver only when set

---
 scrapy/crawler.py | 3 ++-
 tox.ini           | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index ded09c1c7..6866be809 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -156,7 +156,8 @@ class CrawlerProcess(CrawlerRunner):
         reactor.run(installSignalHandlers=False)  # blocking call
 
     def _stop_logging(self):
-        self.log_observer.stop()
+        if self.log_observer:
+            self.log_observer.stop()
 
     def _stop_reactor(self, _=None):
         try:
diff --git a/tox.ini b/tox.ini
index 624f550e1..8aeb1492e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -25,7 +25,6 @@ deps =
     lxml==2.3.2
     Twisted==11.1.0
     boto==2.2.2
-    Pillow<2.0
     django==1.3.1
     cssselect==0.9.1
     zope.interface==3.6.1

From c05e99a4f4bbe7a3f124a99f696d88e5a30fd066 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 10 Sep 2014 12:21:08 -0300
Subject: [PATCH 0038/2974] oops, restore Pillow from precise test requirements

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 8aeb1492e..624f550e1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -25,6 +25,7 @@ deps =
     lxml==2.3.2
     Twisted==11.1.0
     boto==2.2.2
+    Pillow<2.0
     django==1.3.1
     cssselect==0.9.1
     zope.interface==3.6.1

From 5bcabfe9c93b649c35fc4b9d1b0441db146bd406 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 10 Sep 2014 23:25:57 -0300
Subject: [PATCH 0039/2974] SPIDER_MODULES can be set as a csv string

---
 scrapy/spidermanager.py              | 2 +-
 tests/test_spidermanager/__init__.py | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py
index 5715b7793..93e91bcb4 100644
--- a/scrapy/spidermanager.py
+++ b/scrapy/spidermanager.py
@@ -15,7 +15,7 @@ from scrapy.utils.spider import iter_spider_classes
 class SpiderManager(object):
 
     def __init__(self, settings):
-        self.spider_modules = settings['SPIDER_MODULES']
+        self.spider_modules = settings.getlist('SPIDER_MODULES')
         self._spiders = {}
         for name in self.spider_modules:
             for module in walk_modules(name):
diff --git a/tests/test_spidermanager/__init__.py b/tests/test_spidermanager/__init__.py
index 69ab3b82a..94b8a1dc8 100644
--- a/tests/test_spidermanager/__init__.py
+++ b/tests/test_spidermanager/__init__.py
@@ -63,6 +63,13 @@ class SpiderManagerTest(unittest.TestCase):
         self.spiderman = SpiderManager.from_settings(settings)
         assert len(self.spiderman._spiders) == 1
 
+    def test_load_spider_module(self):
+        prefix = 'tests.test_spidermanager.test_spiders.'
+        module = ','.join(prefix + s for s in ('spider1', 'spider2'))
+        settings = Settings({'SPIDER_MODULES': module})
+        self.spiderman = SpiderManager.from_settings(settings)
+        assert len(self.spiderman._spiders) == 2
+
     def test_load_base_spider(self):
         module = 'tests.test_spidermanager.test_spiders.spider0'
         settings = Settings({'SPIDER_MODULES': [module]})

From 47b6dff9f1f47fe97de94e2419d8cff38c7bacb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=85hl=C3=A9n?= <mikael.ahlen@gmail.com>
Date: Sat, 13 Sep 2014 02:14:57 +0200
Subject: [PATCH 0040/2974] Allow to specify the quotechar in CSVFeedSpider

---
 docs/topics/spiders.rst                  |  6 ++++++
 scrapy/contrib/spiders/feed.py           |  5 +++--
 scrapy/utils/iterators.py                | 16 ++++++++++------
 tests/sample_data/feeds/feed-sample6.csv |  6 ++++++
 tests/test_utils_iterators.py            | 22 ++++++++++++++++++++++
 5 files changed, 47 insertions(+), 8 deletions(-)
 create mode 100644 tests/sample_data/feeds/feed-sample6.csv

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 188b51836..cb3f6caeb 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -523,6 +523,11 @@ CSVFeedSpider
        A string with the separator character for each field in the CSV file
        Defaults to ``','`` (comma).
 
+   .. attribute:: quotechar
+
+       A string with the enclosure character for each field in the CSV file
+       Defaults to ``'"'`` (quotation mark).
+
    .. attribute:: headers
 
        A list of the rows contained in the file CSV feed which will be used to
@@ -550,6 +555,7 @@ Let's see an example similar to the previous one, but using a
         allowed_domains = ['example.com']
         start_urls = ['http://www.example.com/feed.csv']
         delimiter = ';'
+        quotechar = "'"
         headers = ['id', 'name', 'description']
 
         def parse_row(self, response, row):
diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py
index fa538f473..1a95c5c35 100644
--- a/scrapy/contrib/spiders/feed.py
+++ b/scrapy/contrib/spiders/feed.py
@@ -97,11 +97,12 @@ class CSVFeedSpider(Spider):
     It receives a CSV file in a response; iterates through each of its rows,
     and calls parse_row with a dict containing each field's data.
 
-    You can set some options regarding the CSV file, such as the delimiter
+    You can set some options regarding the CSV file, such as the delimiter, quotechar
     and the file's headers.
     """
 
     delimiter = None # When this is None, python's csv module's default delimiter is used
+    quotechar = None # When this is None, python's csv module's default quotechar is used
     headers = None
 
     def process_results(self, response, results):
@@ -123,7 +124,7 @@ class CSVFeedSpider(Spider):
         process_results methods for pre and post-processing purposes.
         """
 
-        for row in csviter(response, self.delimiter, self.headers):
+        for row in csviter(response, self.delimiter, self.headers, self.quotechar):
             ret = self.parse_row(response, row)
             if isinstance(ret, (BaseItem, Request)):
                 ret = [ret]
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 150b077ae..78ea7114e 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -35,7 +35,7 @@ def xmliter(obj, nodename):
         yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]
 
 
-def csviter(obj, delimiter=None, headers=None, encoding=None):
+def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     """ Returns an iterator of dictionaries from the given csv object
 
     obj can be:
@@ -43,20 +43,24 @@ def csviter(obj, delimiter=None, headers=None, encoding=None):
     - a unicode string
     - a string encoded as utf-8
 
-    delimiter is the character used to separate field on the given obj.
+    delimiter is the character used to separate fields on the given obj.
 
     headers is an iterable that when provided offers the keys
     for the returned dictionaries, if not the first row is used.
+    
+    quotechar is the character used to enclosure fields on the given obj.
     """
+
     encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8'
     def _getrow(csv_r):
         return [str_to_unicode(field, encoding) for field in next(csv_r)]
 
     lines = BytesIO(_body_or_str(obj, unicode=False))
-    if delimiter:
-        csv_r = csv.reader(lines, delimiter=delimiter)
-    else:
-        csv_r = csv.reader(lines)
+
+    kwargs = {}
+    if delimiter: kwargs["delimiter"] = delimiter
+    if quotechar: kwargs["quotechar"] = quotechar
+    csv_r = csv.reader(lines, **kwargs)
 
     if not headers:
         headers = _getrow(csv_r)
diff --git a/tests/sample_data/feeds/feed-sample6.csv b/tests/sample_data/feeds/feed-sample6.csv
new file mode 100644
index 000000000..a2604653e
--- /dev/null
+++ b/tests/sample_data/feeds/feed-sample6.csv
@@ -0,0 +1,6 @@
+'id','name','value'
+1,'alpha','foobar'
+2,'unicode','únícódé‽'
+'3','multi','foo
+bar'
+4,'empty',
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index fe53f831f..544941de1 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -159,6 +159,28 @@ class UtilsCsvTestCase(unittest.TestCase):
                           {u'id': u'3', u'name': u'multi',   u'value': FOOBAR_NL},
                           {u'id': u'4', u'name': u'empty',   u'value': u''}])
 
+    def test_csviter_quotechar(self):
+        body1 = get_testdata('feeds', 'feed-sample6.csv')
+        body2 = get_testdata('feeds', 'feed-sample6.csv').replace(",", '|')
+        
+        response1 = TextResponse(url="http://example.com/", body=body1)
+        csv1 = csviter(response1, quotechar="'")
+
+        self.assertEqual([row for row in csv1],
+                         [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
+                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
+                          {u'id': u'3', u'name': u'multi',   u'value': FOOBAR_NL},
+                          {u'id': u'4', u'name': u'empty',   u'value': u''}])
+
+        response2 = TextResponse(url="http://example.com/", body=body2)
+        csv2 = csviter(response2, delimiter="|", quotechar="'")
+
+        self.assertEqual([row for row in csv2],
+                         [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
+                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
+                          {u'id': u'3', u'name': u'multi',   u'value': FOOBAR_NL},
+                          {u'id': u'4', u'name': u'empty',   u'value': u''}])
+
     def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
         body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t')
         response = Response(url="http://example.com/", body=body)

From 22da1783bdc7eac04345261b28a942419391eaab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=85hl=C3=A9n?= <mikael.ahlen@gmail.com>
Date: Sat, 13 Sep 2014 03:47:40 +0200
Subject: [PATCH 0041/2974] added a test-case for wrong quotechar

---
 tests/test_utils_iterators.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 544941de1..840f4c596 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -181,6 +181,17 @@ class UtilsCsvTestCase(unittest.TestCase):
                           {u'id': u'3', u'name': u'multi',   u'value': FOOBAR_NL},
                           {u'id': u'4', u'name': u'empty',   u'value': u''}])
 
+    def test_csviter_wrong_quotechar(self):
+        body = get_testdata('feeds', 'feed-sample6.csv')
+        response = TextResponse(url="http://example.com/", body=body)
+        csv = csviter(response)
+
+        self.assertEqual([row for row in csv],
+                         [{u"'id'": u"1",   u"'name'": u"'alpha'",   u"'value'": u"'foobar'"},
+                          {u"'id'": u"2",   u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"},
+                          {u"'id'": u"'3'", u"'name'": u"'multi'",   u"'value'": u"'foo"},
+                          {u"'id'": u"4",   u"'name'": u"'empty'",   u"'value'": u""}])
+
     def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
         body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t')
         response = Response(url="http://example.com/", body=body)

From e583c030db440c3ac0e877804f7195aa53aa460f Mon Sep 17 00:00:00 2001
From: andrewshir <andrewshirokoff@gmail.com>
Date: Sun, 14 Sep 2014 14:24:16 +0600
Subject: [PATCH 0042/2974] Test for local domains (without dots) added

---
 tests/test_downloadermiddleware_cookies.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index ffa3a550c..1ca52ef84 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -8,6 +8,12 @@ from scrapy.contrib.downloadermiddleware.cookies import CookiesMiddleware
 
 class CookiesMiddlewareTest(TestCase):
 
+    def test_local_domain(self):
+        request = Request("http://example-host/", cookies={'currencyCookie': 'USD'})
+        assert self.mw.process_request(request, self.spider) is None
+        assert 'Cookie' in request.headers
+        assert 'currencyCookie' in request.headers['Cookie']
+
     def assertCookieValEqual(self, first, second, msg=None):
         cookievaleq = lambda cv: re.split(';\s*', cv)
         return self.assertEqual(

From a312ebfb435be109dc12d65c622e3a6ad85b3f86 Mon Sep 17 00:00:00 2001
From: John-Scott Atlakson <john-scott@greenlightgo.co>
Date: Sun, 14 Sep 2014 10:54:43 -0400
Subject: [PATCH 0043/2974] Update request-response.rst

Fixed minor typo
---
 docs/topics/request-response.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 54cff9100..6438f2f2b 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -545,7 +545,7 @@ TextResponse objects
 
             unicode(response.body)
 
-        Since, in the latter case, you would be using you system default encoding
+        Since, in the latter case, you would be using the system default encoding
         (typically `ascii`) to convert the body to unicode, instead of the response
         encoding.
 

From e435b3e3a36e6742d8d756b231c868f4be7b528b Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sun, 21 Sep 2014 00:19:24 +0600
Subject: [PATCH 0044/2974] DOC simplify extension docs

---
 docs/topics/downloader-middleware.rst |  4 ++--
 docs/topics/extensions.rst            | 20 ++++++++------------
 docs/topics/item-pipeline.rst         |  3 +--
 docs/topics/spider-middleware.rst     |  4 ++--
 4 files changed, 13 insertions(+), 18 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 614e4fff6..bff0d3e1c 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -51,8 +51,8 @@ particular setting. See each middleware documentation for more info.
 Writing your own downloader middleware
 ======================================
 
-Writing your own downloader middleware is easy. Each middleware component is a
-single Python class that defines one or more of the following methods:
+Each middleware component is a Python class that defines one or
+more of the following methods:
 
 .. module:: scrapy.contrib.downloadermiddleware
 
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 593a08ddc..c23e783bf 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -5,7 +5,7 @@ Extensions
 ==========
 
 The extensions framework provides a mechanism for inserting your own
-custom functionality into Scrapy. 
+custom functionality into Scrapy.
 
 Extensions are just regular classes that are instantiated at Scrapy startup,
 when extensions are initialized.
@@ -75,14 +75,10 @@ included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to
 Writing your own extension
 ==========================
 
-Writing your own extension is easy. Each extension is a single Python class
-which doesn't need to implement any particular method. 
-
-The main entry point for a Scrapy extension (this also includes middlewares and
-pipelines) is the ``from_crawler`` class method which receives a
-``Crawler`` instance which is the main object controlling the Scrapy crawler.
-Through that object you can access settings, signals, stats, and also control
-the crawler behaviour, if your extension needs to such thing.
+Each extension is a Python class. The main entry point for a Scrapy extension
+(this also includes middlewares and pipelines) is the ``from_crawler``
+class method which receives a ``Crawler`` instance. Through the Crawler object
+you can access settings, signals, stats, and also control the crawling behaviour.
 
 Typically, extensions connect to :ref:`signals <topics-signals>` and perform
 tasks triggered by them.
@@ -133,7 +129,7 @@ Here is the code of such extension::
             crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
             crawler.signals.connect(ext.item_scraped, signal=signals.item_scraped)
 
-            # return the extension object 
+            # return the extension object
             return ext
 
         def spider_opened(self, spider):
@@ -183,12 +179,12 @@ Telnet console extension
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. module:: scrapy.telnet
-   :synopsis: Telnet console 
+   :synopsis: Telnet console
 
 .. class:: scrapy.telnet.TelnetConsole
 
 Provides a telnet console for getting into a Python interpreter inside the
-currently running Scrapy process, which can be very useful for debugging. 
+currently running Scrapy process, which can be very useful for debugging.
 
 The telnet console must be enabled by the :setting:`TELNETCONSOLE_ENABLED`
 setting, and the server will listen in the port specified in
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 146f6cbce..9cd198999 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -23,8 +23,7 @@ Typical use for item pipelines are:
 Writing your own item pipeline
 ==============================
 
-Writing your own item pipeline is easy. Each item pipeline component is a
-single Python class that must implement the following method:
+Each item pipeline component is a Python class that must implement the following method:
 
 .. method:: process_item(item, spider)
 
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 3df59998b..92dc6ac47 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -52,8 +52,8 @@ particular setting. See each middleware documentation for more info.
 Writing your own spider middleware
 ==================================
 
-Writing your own spider middleware is easy. Each middleware component is a
-single Python class that defines one or more of the following methods:
+Each middleware component is a Python class that defines one or more of the
+following methods:
 
 .. module:: scrapy.contrib.spidermiddleware
 

From 49645d4bf950cbd18f4a2c3839b93b3bf979e2f5 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sun, 21 Sep 2014 05:31:34 +0600
Subject: [PATCH 0045/2974] TST small cleanup of a cookie test

---
 tests/test_downloadermiddleware_cookies.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 1ca52ef84..3b1853c82 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -8,12 +8,6 @@ from scrapy.contrib.downloadermiddleware.cookies import CookiesMiddleware
 
 class CookiesMiddlewareTest(TestCase):
 
-    def test_local_domain(self):
-        request = Request("http://example-host/", cookies={'currencyCookie': 'USD'})
-        assert self.mw.process_request(request, self.spider) is None
-        assert 'Cookie' in request.headers
-        assert 'currencyCookie' in request.headers['Cookie']
-
     def assertCookieValEqual(self, first, second, msg=None):
         cookievaleq = lambda cv: re.split(';\s*', cv)
         return self.assertEqual(
@@ -153,3 +147,10 @@ class CookiesMiddlewareTest(TestCase):
         req6 = Request('file:///scrapy/sometempfile')
         assert self.mw.process_request(req6, self.spider) is None
         self.assertEquals(req6.headers.get('Cookie'), None)
+
+    def test_local_domain(self):
+        request = Request("http://example-host/", cookies={'currencyCookie': 'USD'})
+        assert self.mw.process_request(request, self.spider) is None
+        self.assertIn('Cookie', request.headers)
+        self.assertIn('currencyCookie', request.headers['Cookie'])
+

From 7be3479c204c2b110bc79cd8bdc9c7bdd1519163 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sun, 21 Sep 2014 06:37:32 +0600
Subject: [PATCH 0046/2974] CookieJar cleanup

---
 scrapy/http/cookies.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 3e3dff741..b1eb767cc 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -28,7 +28,7 @@ class CookieJar(object):
 
         if not IPV4_RE.search(req_host):
             hosts = potential_domain_matches(req_host)
-            if req_host.find(".") == -1:
+            if '.' not in req_host:
                 hosts += [req_host + ".local"]
         else:
             hosts = [req_host]

From a122fdbfeae42147f862e0e5a7dfe3848b8a7eac Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Jan 2014 17:39:54 +0600
Subject: [PATCH 0047/2974] Update leaks.rst: there is now only a single spider
 in a process.

---
 docs/topics/leaks.rst | 82 +++++++++++++++----------------------------
 1 file changed, 28 insertions(+), 54 deletions(-)

diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 27c50a225..c838b3c30 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -32,20 +32,13 @@ and that effectively bounds the lifetime of those referenced objects to the
 lifetime of the Request. This is, by far, the most common cause of memory leaks
 in Scrapy projects, and a quite difficult one to debug for newcomers.
 
-In big projects, the spiders are typically written by different people and some
-of those spiders could be "leaking" and thus affecting the rest of the other
-(well-written) spiders when they get to run concurrently, which, in turn,
-affects the whole crawling process. 
-
-At the same time, it's hard to avoid the reasons that cause these leaks
-without restricting the power of the framework, so we have decided not to
-restrict the functionally but provide useful tools for debugging these leaks,
-which quite often consist in an answer to the question: *which spider is leaking?*.
-
 The leak could also come from a custom middleware, pipeline or extension that
 you have written, if you are not releasing the (previously allocated) resources
-properly. For example, if you're allocating resources on
-:signal:`spider_opened` but not releasing them on :signal:`spider_closed`.
+properly.
+
+It's hard to avoid the reasons that cause these leaks
+without restricting the power of the framework, so we have decided not to
+restrict the functionally but provide useful tools for debugging these leaks.
 
 .. _topics-leaks-trackrefs:
 
@@ -54,7 +47,7 @@ Debugging memory leaks with ``trackref``
 
 ``trackref`` is a module provided by Scrapy to debug the most common cases of
 memory leaks. It basically tracks the references to all live Requests,
-Responses, Item and Selector objects. 
+Responses, Item and Selector objects.
 
 You can enter the telnet console and inspect how many objects (of the classes
 mentioned above) are currently alive using the ``prefs()`` function which is an
@@ -71,12 +64,7 @@ alias to the :func:`~scrapy.utils.trackref.print_live_refs` function::
     FormRequest                       878   oldest: 7s ago
 
 As you can see, that report also shows the "age" of the oldest object in each
-class. 
-
-If you do have leaks, chances are you can figure out which spider is leaking by
-looking at the oldest request or response. You can get the oldest object of
-each class using the :func:`~scrapy.utils.trackref.get_oldest` function like
-this (from the telnet console).
+class.
 
 Which objects are tracked?
 --------------------------
@@ -84,17 +72,16 @@ Which objects are tracked?
 The objects tracked by ``trackrefs`` are all from these classes (and all its
 subclasses):
 
-* ``scrapy.http.Request``
-* ``scrapy.http.Response``
-* ``scrapy.item.Item``
-* ``scrapy.selector.Selector``
-* ``scrapy.spider.Spider``
+* :class:`scrapy.http.Request`
+* :class:`scrapy.http.Response`
+* :class:`scrapy.item.Item`
+* :class:`scrapy.selector.Selector`
+* :class:`scrapy.spider.Spider`
 
 A real example
 --------------
 
 Let's see a concrete example of an hypothetical case of memory leaks.
-
 Suppose we have some spider with a line similar to this one::
 
     return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id,
@@ -104,7 +91,7 @@ That line is passing a response reference inside a request which effectively
 ties the response lifetime to the requests' one, and that would definitely
 cause memory leaks.
 
-Let's see how we can discover which one is the nasty spider (without knowing it
+Let's see how we can discover the cause (without knowing it
 a-priori, of course) by using the ``trackref`` tool.
 
 After the crawler is running for a few minutes and we notice its memory usage
@@ -121,20 +108,21 @@ references::
 
 The fact that there are so many live responses (and that they're so old) is
 definitely suspicious, as responses should have a relatively short lifetime
-compared to Requests. So let's check the oldest response::
+compared to Requests. The number of responses is similar to the number
+of requests, so it looks like they are tied in a some way. We can now go
+and check the code of the spider to discover the nasty line that is
+generating the leaks (passing response references inside requests).
+
+Sometimes extra information about live objects can be helpful.
+Let's check the oldest response::
 
     >>> from scrapy.utils.trackref import get_oldest
     >>> r = get_oldest('HtmlResponse')
     >>> r.url
     'http://www.somenastyspider.com/product.php?pid=123'
 
-There it is. By looking at the URL of the oldest response we can see it belongs
-to the ``somenastyspider.com`` spider. We can now go and check the code of that
-spider to discover the nasty line that is generating the leaks (passing
-response references inside requests).
-
 If you want to iterate over all objects, instead of getting the oldest one, you
-can use the :func:`iter_all` function::
+can use the :func:`scrapy.utils.trackref.iter_all` function::
 
     >>> from scrapy.utils.trackref import iter_all
     >>> [r.url for r in iter_all('HtmlResponse')]
@@ -142,19 +130,6 @@ can use the :func:`iter_all` function::
      'http://www.somenastyspider.com/product.php?pid=584',
     ...
 
-Too many spiders?
------------------
-
-If your project has too many spiders, the output of ``prefs()`` can be
-difficult to read. For this reason, that function has a ``ignore`` argument
-which can be used to ignore a particular class (and all its subclases). For
-example, using::
-
-    >>> from scrapy.spider import Spider
-    >>> prefs(ignore=Spider)
-
-Won't show any live references to spiders.
-
 .. module:: scrapy.utils.trackref
    :synopsis: Track references of live objects
 
@@ -198,15 +173,13 @@ leaks, but it only keeps track of the objects that are more likely to cause
 memory leaks (Requests, Responses, Items, and Selectors). However, there are
 other cases where the memory leaks could come from other (more or less obscure)
 objects. If this is your case, and you can't find your leaks using ``trackref``,
-you still have another resource: the `Guppy library`_. 
+you still have another resource: the `Guppy library`_.
 
 .. _Guppy library: http://pypi.python.org/pypi/guppy
 
-If you use ``setuptools``, you can install Guppy with the following command::
+If you use ``pip``, you can install Guppy with the following command::
 
-    easy_install guppy
-
-.. _setuptools: http://pypi.python.org/pypi/setuptools
+    pip install guppy
 
 The telnet console also comes with a built-in shortcut (``hpy``) for accessing
 Guppy heap objects. Here's an example to view all Python objects available in
@@ -279,7 +252,8 @@ completely. To quote the paper:
     to move to a compacting garbage collector, which is able to move objects in
     memory. This would require significant changes to the Python interpreter.*
 
-This problem will be fixed in future Scrapy releases, where we plan to adopt a
-new process model and run spiders in a pool of recyclable sub-processes.
-
 .. _this paper: http://evanjones.ca/memoryallocator/
+
+To keep memory consumption reasonable you can split the job into several
+smaller jobs or enable :ref:`persistent job queue <topics-jobs>`
+and stop/start spider from time to time.

From bc0f481a7355713978ee206d36a9356ab4be9d61 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sun, 21 Sep 2014 07:12:01 +0600
Subject: [PATCH 0048/2974] DOC bring back notes about multiple spiders per
 process because it is now documented how to do that

---
 docs/topics/leaks.rst     | 30 ++++++++++++++++++++++++------
 docs/topics/practices.rst |  2 ++
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index c838b3c30..95bb882e9 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -32,13 +32,16 @@ and that effectively bounds the lifetime of those referenced objects to the
 lifetime of the Request. This is, by far, the most common cause of memory leaks
 in Scrapy projects, and a quite difficult one to debug for newcomers.
 
+In big projects, the spiders are typically written by different people and some
+of those spiders could be "leaking" and thus affecting the rest of the other
+(well-written) spiders when they get to run concurrently, which, in turn,
+affects the whole crawling process.
+
 The leak could also come from a custom middleware, pipeline or extension that
 you have written, if you are not releasing the (previously allocated) resources
-properly.
-
-It's hard to avoid the reasons that cause these leaks
-without restricting the power of the framework, so we have decided not to
-restrict the functionally but provide useful tools for debugging these leaks.
+properly. For example, allocating resources on :signal:`spider_opened`
+but not releasing them on :signal:`spider_closed` may cause problems if
+you're running :ref:`multiple spiders per process <run-multiple-spiders>`.
 
 .. _topics-leaks-trackrefs:
 
@@ -64,7 +67,10 @@ alias to the :func:`~scrapy.utils.trackref.print_live_refs` function::
     FormRequest                       878   oldest: 7s ago
 
 As you can see, that report also shows the "age" of the oldest object in each
-class.
+class. If you're running multiple spiders per process chances are you can
+figure out which spider is leaking by looking at the oldest request or response.
+You can get the oldest object of each class using the
+:func:`~scrapy.utils.trackref.get_oldest` function (from the telnet console).
 
 Which objects are tracked?
 --------------------------
@@ -130,6 +136,18 @@ can use the :func:`scrapy.utils.trackref.iter_all` function::
      'http://www.somenastyspider.com/product.php?pid=584',
     ...
 
+Too many spiders?
+-----------------
+
+If your project has too many spiders executed in parallel,
+the output of :func:`prefs()` can be difficult to read.
+For this reason, that function has a ``ignore`` argument which can be used to
+ignore a particular class (and all its subclases). For
+example, this won't show any live references to spiders::
+
+    >>> from scrapy.spider import Spider
+    >>> prefs(ignore=Spider)
+
 .. module:: scrapy.utils.trackref
    :synopsis: Track references of live objects
 
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index b188ee562..e9c7a94bf 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -69,6 +69,8 @@ the spider class as first argument in the :meth:`CrawlerRunner.crawl
 
 .. seealso:: `Twisted Reactor Overview`_.
 
+.. _run-multiple-spiders:
+
 Running multiple spiders in the same process
 ============================================
 

From bdbca1e2d7b4bf13332eec24968d8a7b9aea2de4 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sun, 21 Sep 2014 07:30:44 +0600
Subject: [PATCH 0049/2974] DOC request queue memory usage

---
 docs/topics/leaks.rst | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 95bb882e9..d6530859b 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -43,12 +43,22 @@ properly. For example, allocating resources on :signal:`spider_opened`
 but not releasing them on :signal:`spider_closed` may cause problems if
 you're running :ref:`multiple spiders per process <run-multiple-spiders>`.
 
+Too Many Requests?
+------------------
+
+By default Scrapy keeps the request queue in memory; it includes
+:class:`~scrapy.http.Request` objects and all objects
+referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.meta`).
+While not necesserily a leak, this can take a lot of memory. Enabling
+:ref:`persistent job queue <topics-jobs>` could help keeping memory usage
+in control.
+
 .. _topics-leaks-trackrefs:
 
 Debugging memory leaks with ``trackref``
 ========================================
 
-``trackref`` is a module provided by Scrapy to debug the most common cases of
+:mod:`trackref` is a module provided by Scrapy to debug the most common cases of
 memory leaks. It basically tracks the references to all live Requests,
 Responses, Item and Selector objects.
 

From d11c8595e67680baad002bedf08dfd91f347cfb2 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 22 Sep 2014 04:29:22 +0600
Subject: [PATCH 0050/2974] drop support for CONCURRENT_REQUESTS_PER_SPIDER

---
 scrapy/core/downloader/__init__.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index a5e62a75d..ecbf33039 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -49,14 +49,6 @@ def _get_concurrency_delay(concurrency, spider, settings):
     if hasattr(spider, 'download_delay'):
         delay = spider.download_delay
 
-    # TODO: remove for Scrapy 0.15
-    c = settings.getint('CONCURRENT_REQUESTS_PER_SPIDER')
-    if c:
-        warnings.warn("CONCURRENT_REQUESTS_PER_SPIDER setting is deprecated, "
-                      "use CONCURRENT_REQUESTS_PER_DOMAIN instead", ScrapyDeprecationWarning)
-        concurrency = c
-    # ----------------------------
-
     if hasattr(spider, 'max_concurrent_requests'):
         concurrency = spider.max_concurrent_requests
 

From fe6f3efe95eb9f0a33154d239f9b1a6b1c02028d Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 22 Sep 2014 22:56:54 +0600
Subject: [PATCH 0051/2974] RobotsTxtMiddleware: remove unused attribute

---
 scrapy/contrib/downloadermiddleware/robotstxt.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py
index f1e8012e7..f9d1396fb 100644
--- a/scrapy/contrib/downloadermiddleware/robotstxt.py
+++ b/scrapy/contrib/downloadermiddleware/robotstxt.py
@@ -22,7 +22,6 @@ class RobotsTxtMiddleware(object):
         self.crawler = crawler
         self._useragent = crawler.settings.get('USER_AGENT')
         self._parsers = {}
-        self._spider_netlocs = set()
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -45,7 +44,6 @@ class RobotsTxtMiddleware(object):
             robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY)
             dfd = self.crawler.engine.download(robotsreq, spider)
             dfd.addCallback(self._parse_robots)
-            self._spider_netlocs.add(netloc)
         return self._parsers[netloc]
 
     def _parse_robots(self, response):

From 36eec8f4136a314d20c04cd8318a32e75c31a649 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 23 Sep 2014 00:10:43 +0600
Subject: [PATCH 0052/2974] dont_obey_robotstxt meta key; don't process
 requests to /robots.txt

---
 docs/topics/downloader-middleware.rst         | 10 +++-
 docs/topics/request-response.rst              |  1 +
 .../contrib/downloadermiddleware/robotstxt.py | 11 +++-
 tests/test_downloadermiddleware_robotstxt.py  | 60 ++++++++++++++-----
 4 files changed, 62 insertions(+), 20 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 614e4fff6..e4e490f61 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -785,11 +785,19 @@ RobotsTxtMiddleware
     and the :setting:`ROBOTSTXT_OBEY` setting is enabled.
 
     .. warning:: Keep in mind that, if you crawl using multiple concurrent
-       requests per domain, Scrapy could still  download some forbidden pages
+       requests per domain, Scrapy could still download some forbidden pages
        if they were requested before the robots.txt file was downloaded. This
        is a known limitation of the current robots.txt middleware and will
        be fixed in the future.
 
+.. reqmeta:: dont_obey_robotstxt
+
+If :attr:`Request.meta <scrapy.http.Request.meta>` has
+``dont_obey_robotstxt`` key set to True
+the request will be ignored by this middleware even if
+:setting:`ROBOTSTXT_OBEY` is enabled.
+
+
 DownloaderStats
 ---------------
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 6438f2f2b..b3e9d9a1d 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -228,6 +228,7 @@ Those are:
 * :reqmeta:`cookiejar`
 * :reqmeta:`redirect_urls`
 * :reqmeta:`bindaddress`
+* :reqmeta:`dont_obey_robotstxt`
 
 .. reqmeta:: bindaddress
 
diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py
index f9d1396fb..0ea402719 100644
--- a/scrapy/contrib/downloadermiddleware/robotstxt.py
+++ b/scrapy/contrib/downloadermiddleware/robotstxt.py
@@ -28,9 +28,10 @@ class RobotsTxtMiddleware(object):
         return cls(crawler)
 
     def process_request(self, request, spider):
-        useragent = self._useragent
+        if request.meta.get('dont_obey_robotstxt'):
+            return
         rp = self.robot_parser(request, spider)
-        if rp and not rp.can_fetch(useragent, request.url):
+        if rp and not rp.can_fetch(self._useragent, request.url):
             log.msg(format="Forbidden by robots.txt: %(request)s",
                     level=log.DEBUG, request=request)
             raise IgnoreRequest
@@ -41,7 +42,11 @@ class RobotsTxtMiddleware(object):
         if netloc not in self._parsers:
             self._parsers[netloc] = None
             robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc)
-            robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY)
+            robotsreq = Request(
+                robotsurl,
+                priority=self.DOWNLOAD_PRIORITY,
+                meta={'dont_obey_robotstxt': True}
+            )
             dfd = self.crawler.engine.download(robotsreq, spider)
             dfd.addCallback(self._parse_robots)
         return self._parsers[netloc]
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 31d7f0087..eba337cbd 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import re
 import mock
 from twisted.internet import reactor
@@ -11,7 +12,44 @@ from scrapy.settings import Settings
 
 class RobotsTxtMiddlewareTest(unittest.TestCase):
 
-    def test(self):
+    def test_robotstxt(self):
+        middleware = self._get_middleware()
+        # There is a bit of neglect in robotstxt.py: robots.txt is fetched asynchronously,
+        # and it is actually fetched only *after* first process_request completes.
+        # So, first process_request will always succeed.
+        # We defer test() because otherwise robots.txt download mock will be called after assertRaises failure.
+        self.assertNotIgnored(Request('http://site.local'), middleware)
+        def test(r):
+            self.assertNotIgnored(Request('http://site.local/allowed'), middleware)
+            self.assertIgnored(Request('http://site.local/admin/main'), middleware)
+            self.assertIgnored(Request('http://site.local/static/'), middleware)
+        deferred = Deferred()
+        deferred.addCallback(test)
+        reactor.callFromThread(deferred.callback, None)
+        return deferred
+
+    def test_robotstxt_meta(self):
+        meta = {'dont_obey_robotstxt': True}
+        middleware = self._get_middleware()
+        self.assertNotIgnored(Request('http://site.local', meta=meta), middleware)
+        def test(r):
+            self.assertNotIgnored(Request('http://site.local/allowed', meta=meta), middleware)
+            self.assertNotIgnored(Request('http://site.local/admin/main', meta=meta), middleware)
+            self.assertNotIgnored(Request('http://site.local/static/', meta=meta), middleware)
+        deferred = Deferred()
+        deferred.addCallback(test)
+        reactor.callFromThread(deferred.callback, None)
+        return deferred
+
+    def assertNotIgnored(self, request, middleware):
+        spider = None  # not actually used
+        self.assertIsNone(middleware.process_request(request, spider))
+
+    def assertIgnored(self, request, middleware):
+        spider = None  # not actually used
+        self.assertRaises(IgnoreRequest, middleware.process_request, request, spider)
+
+    def _get_crawler(self):
         crawler = mock.MagicMock()
         crawler.settings = Settings()
         crawler.settings.set('USER_AGENT', 'CustomAgent')
@@ -29,18 +67,8 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
             reactor.callFromThread(deferred.callback, response)
             return deferred
         crawler.engine.download.side_effect = return_response
-        middleware = RobotsTxtMiddleware(crawler)
-        spider = None  # not actually used
-        # There is a bit of neglect in robotstxt.py: robots.txt is fetched asynchronously,
-        # and it is actually fetched only *after* first process_request completes.
-        # So, first process_request will always succeed.
-        # We defer test() because otherwise robots.txt download mock will be called after assertRaises failure.
-        self.assertIsNone(middleware.process_request(Request('http://site.local'), spider))  # not affected by robots.txt
-        def test(r):
-            self.assertIsNone(middleware.process_request(Request('http://site.local/allowed'), spider))
-            self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/admin/main'), spider)
-            self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/static/'), spider)
-        deferred = Deferred()
-        deferred.addCallback(test)
-        reactor.callFromThread(deferred.callback, None)
-        return deferred
+        return crawler
+
+    def _get_middleware(self):
+        crawler = self._get_crawler()
+        return RobotsTxtMiddleware(crawler)

From 50862629138d8d6a10d3f8ceed2db01d1b58abdb Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 24 Sep 2014 13:27:14 +0600
Subject: [PATCH 0053/2974] don't hide original exception in
 scrapy.utils.misc.load_object

---
 scrapy/utils/misc.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 4cfd35342..3152db6c7 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -41,10 +41,7 @@ def load_object(path):
         raise ValueError("Error loading object '%s': not a full path" % path)
 
     module, name = path[:dot], path[dot+1:]
-    try:
-        mod = import_module(module)
-    except ImportError as e:
-        raise ImportError("Error loading object '%s': %s" % (path, e))
+    mod = import_module(module)
 
     try:
         obj = getattr(mod, name)

From 6fcf9dce50bb631ddd659217c3fa74b0e06ac809 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 25 Sep 2014 03:13:51 +0600
Subject: [PATCH 0054/2974] DOC document from_crawler method for item
 pipelines; add an example.

---
 docs/topics/item-pipeline.rst | 71 ++++++++++++++++++++++++++++++++---
 1 file changed, 65 insertions(+), 6 deletions(-)

diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 146f6cbce..cdd52f79e 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -26,7 +26,7 @@ Writing your own item pipeline
 Writing your own item pipeline is easy. Each item pipeline component is a
 single Python class that must implement the following method:
 
-.. method:: process_item(item, spider)
+.. method:: process_item(self, item, spider)
 
    This method is called for every item pipeline component and must either return
    a :class:`~scrapy.item.Item` (or any descendant class) object or raise a
@@ -41,20 +41,31 @@ single Python class that must implement the following method:
 
 Additionally, they may also implement the following methods:
 
-.. method:: open_spider(spider)
+.. method:: open_spider(self, spider)
 
    This method is called when the spider is opened.
 
    :param spider: the spider which was opened
    :type spider: :class:`~scrapy.spider.Spider` object
 
-.. method:: close_spider(spider)
+.. method:: close_spider(self, spider)
 
    This method is called when the spider is closed.
 
    :param spider: the spider which was closed
    :type spider: :class:`~scrapy.spider.Spider` object
 
+.. method:: from_crawler(cls, crawler)
+
+   If present, this classmethod is called to create a pipeline instance
+   from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
+   of the pipeline. Crawler object provides access to all Scrapy core
+   components like settings and signals; it is a way for pipeline to
+   access them and hook its functionality into Scrapy.
+
+   :param crawler: crawler that uses this pipeline
+   :type crawler: :class:`~scrapy.crawler.Crawler` object
+
 
 Item pipeline example
 =====================
@@ -62,9 +73,10 @@ Item pipeline example
 Price validation and dropping items with no prices
 --------------------------------------------------
 
-Let's take a look at the following hypothetical pipeline that adjusts the ``price``
-attribute for those items that do not include VAT (``price_excludes_vat``
-attribute), and drops those items which don't contain a price::
+Let's take a look at the following hypothetical pipeline that adjusts the
+``price`` attribute for those items that do not include VAT
+(``price_excludes_vat`` attribute), and drops those items which don't
+contain a price::
 
     from scrapy.exceptions import DropItem
 
@@ -104,6 +116,53 @@ format::
    item pipelines. If you really want to store all scraped items into a JSON
    file you should use the :ref:`Feed exports <topics-feed-exports>`.
 
+Write items to MongoDB
+----------------------
+
+In this example we'll write items to MongoDB_ using pymongo_.
+MongoDB address and database name are specified in Scrapy settings;
+MongoDB collection is named after item class.
+
+The main point of this example is to show how to use :meth:`from_crawler`
+method and how to clean up the resources properly.
+
+.. note::
+
+    Previous example (JsonWriterPipeline) doesn't clean up resources properly.
+    Fixing it is left as an exercise for the reader.
+
+::
+
+    import pymongo
+
+    class MongoPipeline(object):
+
+        def __init__(self, mongo_uri, mongo_db):
+            self.mongo_uri = mongo_uri
+            self.mongo_db = mongo_db
+
+        @classmethod
+        def from_crawler(cls, crawler):
+            return cls(
+                mongo_uri=crawler.settings.get('MONGO_URI'),
+                mongo_db=crawler.settings.get('MONGO_DATABASE', 'items')
+            )
+
+        def open_spider(self, spider):
+            self.client = pymongo.MongoClient(self.mongo_uri)
+            self.db = self.client[self.mongo_db]
+
+        def close_spider(self, spider):
+            self.client.close()
+
+        def process_item(self, item, spider):
+            collection_name = item.__class__.__name__
+            self.db[collection_name].insert(dict(item))
+            return item
+
+.. _MongoDB: http://www.mongodb.org/
+.. _pymongo: http://api.mongodb.org/python/current/
+
 Duplicates filter
 -----------------
 

From 993b543e1b895772ec8ac1aeb8db5a87d2b5f784 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Thu, 2 Oct 2014 01:17:26 -0300
Subject: [PATCH 0055/2974] mark SEP-019 as Final

---
 sep/sep-019.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sep/sep-019.rst b/sep/sep-019.rst
index 6c5e8bdd9..9fbf6a223 100644
--- a/sep/sep-019.rst
+++ b/sep/sep-019.rst
@@ -3,7 +3,7 @@ SEP      19
 Title    Per-spider settings
 Author   Pablo Hoffman, Nicolás Ramirez, Julia Medina
 Created  2013-03-07
-Status   Draft
+Status   Final (implemented with minor variations)
 =======  ===================
 
 ======================================================

From ea3b372b4fd34dd133ec433d86a7ae6aaf0105bf Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 2 Oct 2014 15:20:13 +0600
Subject: [PATCH 0056/2974] DOC typo fix in leaks.rst

---
 docs/topics/leaks.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index d6530859b..372691c8e 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -49,7 +49,7 @@ Too Many Requests?
 By default Scrapy keeps the request queue in memory; it includes
 :class:`~scrapy.http.Request` objects and all objects
 referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.meta`).
-While not necesserily a leak, this can take a lot of memory. Enabling
+While not necessarily a leak, this can take a lot of memory. Enabling
 :ref:`persistent job queue <topics-jobs>` could help keeping memory usage
 in control.
 

From 33a7c1d4380944d0a6e2efc017b85b1341fc72a3 Mon Sep 17 00:00:00 2001
From: VKen <vken85@gmail.com>
Date: Fri, 3 Oct 2014 04:16:21 +0800
Subject: [PATCH 0057/2974] updated deprecated cgi.parse_qsl to use six's
 parse_qsl

---
 scrapy/utils/url.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index ab4d75f87..36490a39d 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -6,9 +6,9 @@ Some of the functions that used to be imported from this module have been moved
 to the w3lib.url module. Always import those from there instead.
 """
 import posixpath
-from six.moves.urllib.parse import ParseResult, urlunparse, urldefrag, urlparse
+from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag,
+                                    urlparse, parse_qsl)
 import urllib
-import cgi
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this move doesn't break old code
 from w3lib.url import *
@@ -54,7 +54,7 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False,
     """
 
     scheme, netloc, path, params, query, fragment = parse_url(url)
-    keyvals = cgi.parse_qsl(query, keep_blank_values)
+    keyvals = parse_qsl(query, keep_blank_values)
     keyvals.sort()
     query = urllib.urlencode(keyvals)
     path = safe_url_string(_unquotepath(path)) or '/'

From 7db6bbce27e694243a96307e149c7092195da9b9 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Fri, 3 Oct 2014 18:41:29 +0000
Subject: [PATCH 0058/2974] Drop old engine code

  * remove Downloader import unused since 1fba64
  * remove CONCURRENT_SPIDERS deprecation warning from a1dbc6 (2011)
---
 scrapy/core/engine.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 717b34764..bd1a9f04b 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -4,16 +4,14 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spiders.
 For more information see docs/topics/architecture.rst
 
 """
-import warnings
 from time import time
 
 from twisted.internet import defer
 from twisted.python.failure import Failure
 
 from scrapy import log, signals
-from scrapy.core.downloader import Downloader
 from scrapy.core.scraper import Scraper
-from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning
+from scrapy.exceptions import DontCloseSpider
 from scrapy.http import Response, Request
 from scrapy.utils.misc import load_object
 from scrapy.utils.reactor import CallLaterOnce
@@ -63,10 +61,6 @@ class ExecutionEngine(object):
         downloader_cls = load_object(self.settings['DOWNLOADER'])
         self.downloader = downloader_cls(crawler)
         self.scraper = Scraper(crawler)
-        self._concurrent_spiders = self.settings.getint('CONCURRENT_SPIDERS', 1)
-        if self._concurrent_spiders != 1:
-            warnings.warn("CONCURRENT_SPIDERS settings is deprecated, use " \
-                "Scrapyd max_proc config instead", ScrapyDeprecationWarning)
         self._spider_closed_callback = spider_closed_callback
 
     @defer.inlineCallbacks

From 7d68b084a4a0a63d5769c51c5cde5e9c1108ddb9 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 23 Sep 2014 02:59:35 +0600
Subject: [PATCH 0059/2974] DOC document download_timeout Request.meta key and
 download_timeout spider attribute.

---
 docs/topics/downloader-middleware.rst |  9 ++++++++-
 docs/topics/request-response.rst      | 10 ++++++++++
 docs/topics/settings.rst              |  6 ++++++
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index e4e490f61..e96ec8852 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -263,7 +263,14 @@ DownloadTimeoutMiddleware
 .. class:: DownloadTimeoutMiddleware
 
     This middleware sets the download timeout for requests specified in the
-    :setting:`DOWNLOAD_TIMEOUT` setting.
+    :setting:`DOWNLOAD_TIMEOUT` setting or :attr:`download_timeout`
+    spider attribute.
+
+.. note::
+
+    You can also set download timeout per-request using
+    :reqmeta:`download_timeout` Request.meta key; this is supported
+    even when DownloadTimeoutMiddleware is disabled.
 
 HttpAuthMiddleware
 ------------------
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index b3e9d9a1d..68d87d04f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -229,6 +229,7 @@ Those are:
 * :reqmeta:`redirect_urls`
 * :reqmeta:`bindaddress`
 * :reqmeta:`dont_obey_robotstxt`
+* :reqmeta:`download_timeout`
 
 .. reqmeta:: bindaddress
 
@@ -237,6 +238,15 @@ bindaddress
 
 The IP of the outgoing IP address to use for the performing the request.
 
+.. reqmeta:: download_timeout
+
+download_timeout
+----------------
+
+The amount of time (in secs) that the downloader will wait before timing out.
+See also: :setting:`DOWNLOAD_TIMEOUT`.
+
+
 .. _topics-request-response-ref-request-subclasses:
 
 Request subclasses
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 89ee7605a..9000f0242 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -416,6 +416,12 @@ Default: ``180``
 
 The amount of time (in secs) that the downloader will wait before timing out.
 
+.. note::
+
+    This timeout can be set per spider using :attr:`download_timeout`
+    spider attribute and per-request using :reqmeta:`download_timeout`
+    Request.meta key.
+
 .. setting:: DUPEFILTER_CLASS
 
 DUPEFILTER_CLASS

From db2474f7e7ef85809528a6f92aff3127be55734c Mon Sep 17 00:00:00 2001
From: Jonas Brunsgaard <jonas.brunsgaard@gmail.com>
Date: Tue, 7 Oct 2014 13:54:04 +0200
Subject: [PATCH 0060/2974] Deleted bin folder from root, fixes #913

---
 bin/scrapy | 4 ----
 1 file changed, 4 deletions(-)
 delete mode 100755 bin/scrapy

diff --git a/bin/scrapy b/bin/scrapy
deleted file mode 100755
index 918ea7fbd..000000000
--- a/bin/scrapy
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/usr/bin/env python
-
-from scrapy.cmdline import execute
-execute()

From 22278056199244c3c87128ab9e5cf4743bb908c3 Mon Sep 17 00:00:00 2001
From: Nikolaos-Digenis Karagiannis <digenishjkl@gmail.com>
Date: Wed, 8 Oct 2014 17:46:07 +0300
Subject: [PATCH 0061/2974] Compatibility with .15 leftover

---
 scrapy/core/engine.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index bd1a9f04b..4ef1d0fc6 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -263,10 +263,8 @@ class ExecutionEngine(object):
         dfd.addBoth(lambda _: slot.scheduler.close(reason))
         dfd.addErrback(log.err, spider=spider)
 
-        # XXX: spider_stats argument was added for backwards compatibility with
-        # stats collection refactoring added in 0.15. it should be removed in 0.17.
-        dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(signal=signals.spider_closed, \
-            spider=spider, reason=reason, spider_stats=self.crawler.stats.get_stats()))
+        dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(
+            signal=signals.spider_closed, spider=spider, reason=reason))
         dfd.addErrback(log.err, spider=spider)
 
         dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason))

From 38dcf50cd6a6328f5ec293015dbe53733343991a Mon Sep 17 00:00:00 2001
From: Lazar-T <lazartelebak@gmail.com>
Date: Sat, 25 Oct 2014 00:21:16 +0200
Subject: [PATCH 0062/2974] comma instead of fullstop

---
 docs/topics/extensions.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index c23e783bf..2c2f7fb9c 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -204,7 +204,7 @@ Memory usage extension
 
 Monitors the memory used by the Scrapy process that runs the spider and:
 
-1, sends a notification e-mail when it exceeds a certain value
+1. sends a notification e-mail when it exceeds a certain value
 2. closes the spider when it exceeds a certain value
 
 The notification e-mails can be triggered when a certain warning value is

From 6cb89957312730dd4d3740c9daacb1f1a07a5f6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 5 Nov 2014 22:49:40 -0200
Subject: [PATCH 0063/2974] Update install.rst

fixes #937
---
 docs/intro/install.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index ffba0e2b3..867bc2da7 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -43,7 +43,7 @@ Windows
   the Python executable and additional scripts. The following paths need to be
   added to ``PATH``::
 
-      C:\Python2.7\;C:\Python2.7\Scripts\;
+      C:\Python27\;C:\Python27\Scripts\;
 
   To update the ``PATH`` open a Command prompt and run::
 

From 2c67bd6c57cec359b6a7894f0adeb10d9e2ab77c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 5 Nov 2014 23:05:51 -0200
Subject: [PATCH 0064/2974] pywin32 is required by Twisted. closes #937

see:
* http://twistedmatrix.com/trac/ticket/6032
* https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2028
---
 docs/intro/install.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 867bc2da7..1d786efe7 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -54,6 +54,10 @@ Windows
 
       python --version
 
+* Install `pywin32` from http://sourceforge.net/projects/pywin32/
+
+  Be sure you download the architecture (win32 or amd64) that matches your system
+
 * Install `pip`_ from https://pip.pypa.io/en/latest/installing.html
 
   Now open a Command prompt to check ``pip`` is installed correctly:: 

From b21a28cc9a4d0cfc3375974b857123b6be0386f4 Mon Sep 17 00:00:00 2001
From: HalfCrazy <hackzhuyan@gmail.com>
Date: Thu, 6 Nov 2014 00:48:11 +0800
Subject: [PATCH 0065/2974] Afterwords->Afterwards

---
 docs/topics/loaders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index d571d564d..236725309 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -61,7 +61,7 @@ In other words, data is being collected by extracting it from two XPath
 locations, using the :meth:`~ItemLoader.add_xpath` method. This is the
 data that will be assigned to the ``name`` field later.
 
-Afterwords, similar calls are used for ``price`` and ``stock`` fields
+Afterwards, similar calls are used for ``price`` and ``stock`` fields
 (the later using a CSS selector with the :meth:`~ItemLoader.add_css` method),
 and finally the ``last_update`` field is populated directly with a literal value
 (``today``) using a different method: :meth:`~ItemLoader.add_value`.

From 13f83f0da0c03dabbbdeee3410e1d5f776bd2677 Mon Sep 17 00:00:00 2001
From: Lazar-T <lazartelebak@gmail.com>
Date: Sun, 9 Nov 2014 00:04:22 +0100
Subject: [PATCH 0066/2974] typo

---
 docs/topics/autothrottle.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst
index e7b900876..475946a54 100644
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@@ -11,7 +11,7 @@ Design goals
 1. be nicer to sites instead of using default download delay of zero
 2. automatically adjust scrapy to the optimum crawling speed, so the user
    doesn't have to tune the download delays and concurrent requests to find the
-   optimum one. the user only needs to specify the maximum concurrent requests
+   optimum one. The user only needs to specify the maximum concurrent requests
    it allows, and the extension does the rest.
 
 How it works

From b422312a38b517d77f920ace68f24cbe0f60faaf Mon Sep 17 00:00:00 2001
From: Jeff Paine <JeffPaine@users.noreply.github.com>
Date: Sun, 9 Nov 2014 21:08:27 -0500
Subject: [PATCH 0067/2974] Update docs copyright year range

---
 docs/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index 7acf7c7fa..fdd3025db 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -42,7 +42,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Scrapy'
-copyright = u'2008-2013, Scrapy developers'
+copyright = u'2008-2014, Scrapy developers'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

From fa84730e702a1d94ddeee0acbb81ac7394348b98 Mon Sep 17 00:00:00 2001
From: tpeng <pengtaoo@gmail.com>
Date: Wed, 12 Nov 2014 12:28:02 +0100
Subject: [PATCH 0068/2974] avoid download large response

introduce DOWNLOAD_MAXSIZE and DOWNLOAD_WARNSIZE in settings and
download_maxsize/download_warnsize in spider/request meta, so
downloader stop downloading as soon as the received data exceed the
limit. also check the twsisted response's length in advance to stop
downloading as early as possible.
---
 docs/topics/settings.rst                  | 34 +++++++++++++
 scrapy/core/downloader/handlers/http11.py | 45 +++++++++++++++--
 scrapy/settings/default_settings.py       |  3 ++
 tests/test_downloader_handlers.py         | 60 +++++++++++++++++++++++
 4 files changed, 137 insertions(+), 5 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 9000f0242..402226743 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -422,6 +422,40 @@ The amount of time (in secs) that the downloader will wait before timing out.
     spider attribute and per-request using :reqmeta:`download_timeout`
     Request.meta key.
 
+.. setting:: DOWNLOAD_MAXSIZE
+
+DOWNLOAD_MAXSIZE
+----------------
+
+Default: `1073741824` (1024Mb)
+
+The maximum response size (in bytes) that downloader will download.
+
+If you want to disable it set to 0.
+
+.. note::
+
+    This size can be set per spider using :attr:`download_maxsize`
+    spider attribute and per-request using :reqmeta:`download_maxsize`
+    Request.meta key.
+
+.. setting:: DOWNLOAD_WARNSIZE
+
+DOWNLOAD_WARNSIZE
+----------------
+
+Default: `33554432` (32Mb)
+
+The response size (in bytes) that downloader will start to warn.
+
+If you want to disable it set to 0.
+
+.. note::
+
+    This size can be set per spider using :attr:`download_warnsize`
+    spider attribute and per-request using :reqmeta:`download_warnsize`
+    Request.meta key.
+
 .. setting:: DUPEFILTER_CLASS
 
 DUPEFILTER_CLASS
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 23cd07c51..dd3ad488b 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -9,7 +9,7 @@ from six.moves.urllib.parse import urldefrag
 from zope.interface import implements
 from twisted.internet import defer, reactor, protocol
 from twisted.web.http_headers import Headers as TxHeaders
-from twisted.web.iweb import IBodyProducer
+from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
 from twisted.internet.error import TimeoutError
 from twisted.web.http import PotentialDataLoss
 from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \
@@ -19,6 +19,7 @@ from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.core.downloader.webclient import _parse
 from scrapy.utils.misc import load_object
+from scrapy import log
 
 
 class HTTP11DownloadHandler(object):
@@ -29,10 +30,14 @@ class HTTP11DownloadHandler(object):
         self._pool._factory.noisy = False
         self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
         self._contextFactory = self._contextFactoryClass()
+        self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
+        self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
 
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
-        agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool)
+        agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool,
+            maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
+            warnsize=getattr(spider, 'download_warnsize', self._default_warnsize))
         return agent.download_request(request)
 
     def close(self):
@@ -131,11 +136,14 @@ class ScrapyAgent(object):
     _ProxyAgent = ProxyAgent
     _TunnelingAgent = TunnelingAgent
 
-    def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None):
+    def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
+                 maxsize=0, warnsize=0):
         self._contextFactory = contextFactory
         self._connectTimeout = connectTimeout
         self._bindAddress = bindAddress
         self._pool = pool
+        self._maxsize = maxsize
+        self._warnsize = warnsize
 
     def _get_agent(self, request, timeout):
         bindaddress = request.meta.get('bindaddress') or self._bindAddress
@@ -197,11 +205,25 @@ class ScrapyAgent(object):
         if txresponse.length == 0:
             return txresponse, '', None
 
+        maxsize = request.meta.get('download_maxsize', self._maxsize)
+        warnsize = request.meta.get('download_warnsize', self._warnsize)
+        expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1
+
+        if maxsize and expected_size > maxsize:
+            log.msg("Expected response size (%s) larger than download max size (%s)." % (expected_size, maxsize),
+                    logLevel=log.ERROR)
+            txresponse._transport._producer.loseConnection()
+            raise defer.CancelledError()
+
+        if warnsize and expected_size > warnsize:
+            log.msg("Expected response size (%s) larger than downlod warn size (%s)." % (expected_size, warnsize),
+                    logLevel=log.WARNING)
+
         def _cancel(_):
             txresponse._transport._producer.loseConnection()
 
         d = defer.Deferred(_cancel)
-        txresponse.deliverBody(_ResponseReader(d, txresponse, request))
+        txresponse.deliverBody(_ResponseReader(d, txresponse, request, maxsize, warnsize))
         return d
 
     def _cb_bodydone(self, result, request, url):
@@ -232,14 +254,27 @@ class _RequestBodyProducer(object):
 
 class _ResponseReader(protocol.Protocol):
 
-    def __init__(self, finished, txresponse, request):
+    def __init__(self, finished, txresponse, request, maxsize, warnsize):
         self._finished = finished
         self._txresponse = txresponse
         self._request = request
         self._bodybuf = BytesIO()
+        self._maxsize  = maxsize
+        self._warnsize  = warnsize
+        self._bytes_received = 0
 
     def dataReceived(self, bodyBytes):
         self._bodybuf.write(bodyBytes)
+        self._bytes_received += len(bodyBytes)
+
+        if self._maxsize and self._bytes_received > self._maxsize:
+            log.msg("Received (%s) bytes larger than download max size (%s)." % (self._bytes_received, self._maxsize),
+                    logLevel=log.ERROR)
+            self._finished.cancel()
+
+        if self._warnsize and self._bytes_received > self._warnsize:
+            log.msg("Received (%s) bytes larger than download warn size (%s)." % (self._bytes_received, self._warnsize),
+                    logLevel=log.WARNING)
 
     def connectionLost(self, reason):
         if self._finished.called:
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index f01203c42..1b7b3bf29 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -66,6 +66,9 @@ DOWNLOAD_HANDLERS_BASE = {
 
 DOWNLOAD_TIMEOUT = 180      # 3mins
 
+DOWNLOAD_MAXSIZE = 1073741824   # 1024m
+DOWNLOAD_WARNSIZE = 33554432    # 32m
+
 DOWNLOADER = 'scrapy.core.downloader.Downloader'
 
 DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index c444d35fa..55bb7ccf7 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -30,6 +30,8 @@ from scrapy import optional_features
 from scrapy.utils.test import get_crawler
 from scrapy.exceptions import NotConfigured
 
+from tests.mockserver import MockServer
+from tests.spiders import SingleRequestSpider
 
 class DummyDH(object):
 
@@ -211,6 +213,64 @@ class Http11TestCase(HttpTestCase):
     if 'http11' not in optional_features:
         skip = 'HTTP1.1 not supported in twisted < 11.1.0'
 
+    def test_download_without_maxsize_limit(self):
+        request = Request(self.getURL('file'))
+        d = self.download_request(request, Spider('foo'))
+        d.addCallback(lambda r: r.body)
+        d.addCallback(self.assertEquals, "0123456789")
+        return d
+
+    @defer.inlineCallbacks
+    def test_download_with_maxsize_per_req(self):
+        meta = {'download_maxsize': 2}
+        request = Request(self.getURL('file'), meta=meta)
+        d = self.download_request(request, Spider('foo'))
+        yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+
+    @defer.inlineCallbacks
+    def test_download_with_small_maxsize_per_spider(self):
+        request = Request(self.getURL('file'))
+        d = self.download_request(request, Spider('foo', download_maxsize=2))
+        yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+
+    def test_download_with_large_maxsize_per_spider(self):
+        request = Request(self.getURL('file'))
+        d = self.download_request(request, Spider('foo', download_maxsize=100))
+        d.addCallback(lambda r: r.body)
+        d.addCallback(self.assertEquals, "0123456789")
+        return d
+
+
+class Http11MockServerTestCase(unittest.TestCase):
+    """HTTP 1.1 test case with MockServer"""
+    if 'http11' not in optional_features:
+        skip = 'HTTP1.1 not supported in twisted < 11.1.0'
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_download_with_content_length(self):
+        crawler = get_crawler(SingleRequestSpider)
+        # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid
+        # download it
+        yield crawler.crawl(seed=Request(url='http://localhost:8998/partial', meta={'download_maxsize': 1000}))
+        failure = crawler.spider.meta['failure']
+        self.assertIsInstance(failure.value, defer.CancelledError)
+
+    @defer.inlineCallbacks
+    def test_download(self):
+        crawler = get_crawler(SingleRequestSpider)
+        yield crawler.crawl(seed=Request(url='http://localhost:8998'))
+        failure = crawler.spider.meta.get('failure')
+        self.assertTrue(failure == None)
+        reason = crawler.spider.meta['close_reason']
+        self.assertTrue(reason, 'finished')
+
 
 class UriResource(resource.Resource):
     """Return the full uri that was requested"""

From a69f042d1064c4beea46fa70084835dd6c91c143 Mon Sep 17 00:00:00 2001
From: tpeng <pengtaoo@gmail.com>
Date: Wed, 19 Nov 2014 11:50:07 +0100
Subject: [PATCH 0069/2974] add 2 more test cases and minor doc fixes

---
 docs/topics/settings.rst            | 10 ++++++---
 scrapy/settings/default_settings.py |  4 ++--
 tests/mockserver.py                 |  7 ++++--
 tests/test_downloader_handlers.py   | 34 +++++++++++++++++++++++++++++
 4 files changed, 48 insertions(+), 7 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 402226743..0e9e53de4 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -427,7 +427,7 @@ The amount of time (in secs) that the downloader will wait before timing out.
 DOWNLOAD_MAXSIZE
 ----------------
 
-Default: `1073741824` (1024Mb)
+Default: `1073741824` (1024MB)
 
 The maximum response size (in bytes) that downloader will download.
 
@@ -439,12 +439,14 @@ If you want to disable it set to 0.
     spider attribute and per-request using :reqmeta:`download_maxsize`
     Request.meta key.
 
+    This feature needs Twisted >= 11.1.
+
 .. setting:: DOWNLOAD_WARNSIZE
 
 DOWNLOAD_WARNSIZE
-----------------
+-----------------
 
-Default: `33554432` (32Mb)
+Default: `33554432` (32MB)
 
 The response size (in bytes) that downloader will start to warn.
 
@@ -456,6 +458,8 @@ If you want to disable it set to 0.
     spider attribute and per-request using :reqmeta:`download_warnsize`
     Request.meta key.
 
+    This feature needs Twisted >= 11.1.
+
 .. setting:: DUPEFILTER_CLASS
 
 DUPEFILTER_CLASS
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 1b7b3bf29..cf216385a 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -66,8 +66,8 @@ DOWNLOAD_HANDLERS_BASE = {
 
 DOWNLOAD_TIMEOUT = 180      # 3mins
 
-DOWNLOAD_MAXSIZE = 1073741824   # 1024m
-DOWNLOAD_WARNSIZE = 33554432    # 32m
+DOWNLOAD_MAXSIZE = 1024*1024*1024   # 1024m
+DOWNLOAD_WARNSIZE = 32*1024*1024    # 32m
 
 DOWNLOADER = 'scrapy.core.downloader.Downloader'
 
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 6910532b6..2c0ad66fb 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,9 +1,10 @@
 from __future__ import print_function
 import sys, time, random, urllib, os, json
 from subprocess import Popen, PIPE
-from twisted.web.server import Site, NOT_DONE_YET
-from twisted.web.resource import Resource
+from twisted.web.server import Site, NOT_DONE_YET, GzipEncoderFactory
+from twisted.web.resource import Resource, EncodingResourceWrapper
 from twisted.internet import reactor, defer, ssl
+from twisted.web.test.test_webclient import PayloadResource
 from scrapy import twisted_version
 
 
@@ -167,6 +168,8 @@ class Root(Resource):
         self.putChild("drop", Drop())
         self.putChild("raw", Raw())
         self.putChild("echo", Echo())
+        self.putChild('payload', PayloadResource())
+        self.putChild("xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
 
     def getChild(self, name, request):
         return self
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 55bb7ccf7..127c4a4bd 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -220,6 +220,20 @@ class Http11TestCase(HttpTestCase):
         d.addCallback(self.assertEquals, "0123456789")
         return d
 
+    @defer.inlineCallbacks
+    def test_download_with_maxsize(self):
+        request = Request(self.getURL('file'))
+
+        # 10 is minimal size for this request and the limit is only counted on
+        # response body. (regardless of headers)
+        d = self.download_request(request, Spider('foo', download_maxsize=10))
+        d.addCallback(lambda r: r.body)
+        d.addCallback(self.assertEquals, "0123456789")
+        yield d
+
+        d = self.download_request(request, Spider('foo', download_maxsize=9))
+        yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+
     @defer.inlineCallbacks
     def test_download_with_maxsize_per_req(self):
         meta = {'download_maxsize': 2}
@@ -271,6 +285,26 @@ class Http11MockServerTestCase(unittest.TestCase):
         reason = crawler.spider.meta['close_reason']
         self.assertTrue(reason, 'finished')
 
+    @defer.inlineCallbacks
+    def test_download_gzip_response(self):
+        crawler = get_crawler(SingleRequestSpider)
+        body = '1'*100 # PayloadResource requires body length to be 100
+        request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50})
+        yield crawler.crawl(seed=request)
+        failure = crawler.spider.meta['failure']
+        # download_maxsize < 100, hence the CancelledError
+        self.assertIsInstance(failure.value, defer.CancelledError)
+
+        request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
+        request = request.replace(url='http://localhost:8998/xpayload')
+        yield crawler.crawl(seed=request)
+
+        # download_maxsize = 50 is enough for the gzipped response
+        failure = crawler.spider.meta.get('failure')
+        self.assertTrue(failure == None)
+        reason = crawler.spider.meta['close_reason']
+        self.assertTrue(reason, 'finished')
+
 
 class UriResource(resource.Resource):
     """Return the full uri that was requested"""

From 7910fa017243c6f64ecb55b4272894f6eb5d35ff Mon Sep 17 00:00:00 2001
From: Martin Olveyra <molveyra@gmail.com>
Date: Fri, 21 Nov 2014 01:09:32 -0200
Subject: [PATCH 0070/2974] Force to read DOWNLOAD_TIMEOUT as int (for example
 to pass using environment variable)

---
 scrapy/contrib/downloadermiddleware/downloadtimeout.py |  2 +-
 tests/test_downloadermiddleware_downloadtimeout.py     | 10 ++++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/scrapy/contrib/downloadermiddleware/downloadtimeout.py b/scrapy/contrib/downloadermiddleware/downloadtimeout.py
index 612b081ec..18123cfce 100644
--- a/scrapy/contrib/downloadermiddleware/downloadtimeout.py
+++ b/scrapy/contrib/downloadermiddleware/downloadtimeout.py
@@ -14,7 +14,7 @@ class DownloadTimeoutMiddleware(object):
 
     @classmethod
     def from_crawler(cls, crawler):
-        o = cls(crawler.settings['DOWNLOAD_TIMEOUT'])
+        o = cls(crawler.settings.getfloat('DOWNLOAD_TIMEOUT'))
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         return o
 
diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py
index 3e3ff2401..aba06686a 100644
--- a/tests/test_downloadermiddleware_downloadtimeout.py
+++ b/tests/test_downloadermiddleware_downloadtimeout.py
@@ -8,8 +8,8 @@ from scrapy.utils.test import get_crawler
 
 class DownloadTimeoutMiddlewareTest(unittest.TestCase):
 
-    def get_request_spider_mw(self):
-        crawler = get_crawler(Spider)
+    def get_request_spider_mw(self, settings=None):
+        crawler = get_crawler(Spider, settings)
         spider = crawler._create_spider('foo')
         request = Request('http://scrapytest.org/')
         return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
@@ -20,6 +20,12 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase):
         assert mw.process_request(req, spider) is None
         self.assertEquals(req.meta.get('download_timeout'), 180)
 
+    def test_string_download_timeout(self):
+        req, spider, mw = self.get_request_spider_mw({'DOWNLOAD_TIMEOUT': '20.1'})
+        mw.spider_opened(spider)
+        assert mw.process_request(req, spider) is None
+        self.assertEquals(req.meta.get('download_timeout'), 20.1)
+
     def test_spider_has_download_timeout(self):
         req, spider, mw = self.get_request_spider_mw()
         spider.download_timeout = 2

From 314db3db8b7b84ee3541e8afdd1026681f410de6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 21 Nov 2014 10:54:43 -0200
Subject: [PATCH 0071/2974] pin mitmproxy 0.10.1 as >0.11 does not work with
 tests

---
 tests/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/requirements.txt b/tests/requirements.txt
index b7d6a0a56..cd8a4d471 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,4 +1,4 @@
 # Tests requirements
 mock
-mitmproxy >= 0.10
+mitmproxy==0.10.1
 pytest-twisted

From 8d8e1b2c0c0ba35d8496e503909d7f29dcd1906a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 21 Nov 2014 12:15:02 -0200
Subject: [PATCH 0072/2974] mitmproxy 0.10.1 needs netlib 0.10.1 too

---
 tests/requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/requirements.txt b/tests/requirements.txt
index cd8a4d471..0c1b65aa8 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,4 +1,5 @@
 # Tests requirements
 mock
 mitmproxy==0.10.1
+netlib==0.10.1
 pytest-twisted

From cd193827546d2e20029c28961622ae7def7d541d Mon Sep 17 00:00:00 2001
From: tpeng <pengtaoo@gmail.com>
Date: Tue, 25 Nov 2014 14:09:51 +0100
Subject: [PATCH 0073/2974] attemp to fix travis fails

---
 tests/mockserver.py               | 15 ++++++++-----
 tests/test_downloader_handlers.py | 36 ++++++++++++++++++-------------
 2 files changed, 31 insertions(+), 20 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 2c0ad66fb..b73208c5c 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,10 +1,10 @@
 from __future__ import print_function
 import sys, time, random, urllib, os, json
+import six
 from subprocess import Popen, PIPE
-from twisted.web.server import Site, NOT_DONE_YET, GzipEncoderFactory
-from twisted.web.resource import Resource, EncodingResourceWrapper
+from twisted.web.server import Site, NOT_DONE_YET
+from twisted.web.resource import Resource
 from twisted.internet import reactor, defer, ssl
-from twisted.web.test.test_webclient import PayloadResource
 from scrapy import twisted_version
 
 
@@ -168,8 +168,13 @@ class Root(Resource):
         self.putChild("drop", Drop())
         self.putChild("raw", Raw())
         self.putChild("echo", Echo())
-        self.putChild('payload', PayloadResource())
-        self.putChild("xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
+
+        if six.PY2 and twisted_version > (12, 3, 0):
+            from twisted.web.test.test_webclient import PayloadResource
+            from twisted.web.server import GzipEncoderFactory
+            from twisted.web.resource import EncodingResourceWrapper
+            self.putChild('payload', PayloadResource())
+            self.putChild("xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
 
     def getChild(self, name, request):
         return self
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 127c4a4bd..9021af3b4 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1,5 +1,6 @@
 import os
 import twisted
+import six
 
 from twisted.trial import unittest
 from twisted.protocols.policies import WrappingFactory
@@ -287,23 +288,28 @@ class Http11MockServerTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_download_gzip_response(self):
-        crawler = get_crawler(SingleRequestSpider)
-        body = '1'*100 # PayloadResource requires body length to be 100
-        request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50})
-        yield crawler.crawl(seed=request)
-        failure = crawler.spider.meta['failure']
-        # download_maxsize < 100, hence the CancelledError
-        self.assertIsInstance(failure.value, defer.CancelledError)
 
-        request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
-        request = request.replace(url='http://localhost:8998/xpayload')
-        yield crawler.crawl(seed=request)
+        if six.PY2 and twisted_version > (12, 3, 0):
 
-        # download_maxsize = 50 is enough for the gzipped response
-        failure = crawler.spider.meta.get('failure')
-        self.assertTrue(failure == None)
-        reason = crawler.spider.meta['close_reason']
-        self.assertTrue(reason, 'finished')
+            crawler = get_crawler(SingleRequestSpider)
+            body = '1'*100 # PayloadResource requires body length to be 100
+            request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50})
+            yield crawler.crawl(seed=request)
+            failure = crawler.spider.meta['failure']
+            # download_maxsize < 100, hence the CancelledError
+            self.assertIsInstance(failure.value, defer.CancelledError)
+
+            request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
+            request = request.replace(url='http://localhost:8998/xpayload')
+            yield crawler.crawl(seed=request)
+
+            # download_maxsize = 50 is enough for the gzipped response
+            failure = crawler.spider.meta.get('failure')
+            self.assertTrue(failure == None)
+            reason = crawler.spider.meta['close_reason']
+            self.assertTrue(reason, 'finished')
+        else:
+            raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0 and python 2.x")
 
 
 class UriResource(resource.Resource):

From e04b0aff74632702d78f7729b855c0737d50dfcf Mon Sep 17 00:00:00 2001
From: Lev Berman <ldmberman@gmail.com>
Date: Thu, 27 Nov 2014 15:10:15 +0300
Subject: [PATCH 0074/2974] An attempt to resolve #977, add signal to be sent
 when request is dropped by the scheduler

---
 docs/topics/signals.rst  | 17 +++++++++++++++++
 scrapy/core/engine.py    |  4 +++-
 scrapy/core/scheduler.py |  3 ++-
 scrapy/signals.py        |  1 +
 tests/test_engine.py     | 10 ++++++++++
 5 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 5407141db..405b131ed 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -200,6 +200,23 @@ request_scheduled
     :param spider: the spider that yielded the request
     :type spider: :class:`~scrapy.spider.Spider` object
 
+request_dropped
+-----------------
+
+.. signal:: request_dropped
+.. function:: request_dropped(request, spider)
+
+    Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be
+    downloaded later, is rejected by the scheduler.
+
+    The signal does not support returning deferreds from their handlers.
+
+    :param request: the request that reached the scheduler
+    :type request: :class:`~scrapy.http.Request` object
+
+    :param spider: the spider that yielded the request
+    :type spider: :class:`~scrapy.spider.Spider` object
+
 response_received
 -----------------
 
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 4ef1d0fc6..b009898a3 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -173,7 +173,9 @@ class ExecutionEngine(object):
     def schedule(self, request, spider):
         self.signals.send_catch_log(signal=signals.request_scheduled,
                 request=request, spider=spider)
-        return self.slot.scheduler.enqueue_request(request)
+        if not self.slot.scheduler.enqueue_request(request):
+            self.signals.send_catch_log(signal=signals.request_dropped,
+                                        request=request, spider=spider)
 
     def download(self, request, spider):
         slot = self.slot
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index ba2ca5a03..232bc6a40 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -47,7 +47,7 @@ class Scheduler(object):
     def enqueue_request(self, request):
         if not request.dont_filter and self.df.request_seen(request):
             self.df.log(request, self.spider)
-            return
+            return False
         dqok = self._dqpush(request)
         if dqok:
             self.stats.inc_value('scheduler/enqueued/disk', spider=self.spider)
@@ -55,6 +55,7 @@ class Scheduler(object):
             self._mqpush(request)
             self.stats.inc_value('scheduler/enqueued/memory', spider=self.spider)
         self.stats.inc_value('scheduler/enqueued', spider=self.spider)
+        return True
 
     def next_request(self):
         request = self.mqs.pop()
diff --git a/scrapy/signals.py b/scrapy/signals.py
index 11bbae945..de0886fb6 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -12,6 +12,7 @@ spider_idle = object()
 spider_closed = object()
 spider_error = object()
 request_scheduled = object()
+request_dropped = object()
 response_received = object()
 response_downloaded = object()
 item_scraped = object()
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 67fb8ae79..6ed700caa 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -79,6 +79,7 @@ class CrawlerRun(object):
         self.spider = None
         self.respplug = []
         self.reqplug = []
+        self.reqdropped = []
         self.itemresp = []
         self.signals_catched = {}
 
@@ -95,6 +96,7 @@ class CrawlerRun(object):
         self.crawler = get_crawler(TestSpider)
         self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
         self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
+        self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
         self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded)
         self.crawler.crawl(start_urls=start_urls)
         self.spider = self.crawler.spider
@@ -123,6 +125,9 @@ class CrawlerRun(object):
     def request_scheduled(self, request, spider):
         self.reqplug.append((request, spider))
 
+    def request_dropped(self, request, spider):
+        self.reqdropped.append((request, spider))
+
     def response_downloaded(self, response, spider):
         self.respplug.append((response, spider))
 
@@ -161,6 +166,11 @@ class EngineTest(unittest.TestCase):
         urls_requested = set([rq[0].url for rq in self.run.reqplug])
         urls_expected = set([self.run.geturl(p) for p in paths_expected])
         assert urls_expected <= urls_requested
+        scheduled_requests_count = len(self.run.reqplug)
+        dropped_requests_count = len(self.run.reqdropped)
+        responses_count = len(self.run.respplug)
+        self.assertEqual(scheduled_requests_count,
+                         dropped_requests_count + responses_count)
 
     def _assert_downloaded_responses(self):
         # response tests

From fdb6bb07c0cbe2cf664993555c376eeff750f28d Mon Sep 17 00:00:00 2001
From: Lev Berman <ldmberman@gmail.com>
Date: Fri, 28 Nov 2014 10:53:33 +0300
Subject: [PATCH 0075/2974] #977 - test dropping requests

---
 tests/test_engine.py | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index 6ed700caa..52c8e5752 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -59,6 +59,12 @@ class TestSpider(Spider):
             item['price'] = m.group(1)
         return item
 
+
+class TestDupeFilterSpider(TestSpider):
+    def make_requests_from_url(self, url):
+        return Request(url)  # dont_filter=False
+
+
 def start_test_site(debug=False):
     root_dir = os.path.join(tests_datadir, "test_site")
     r = static.File(root_dir)
@@ -75,25 +81,28 @@ def start_test_site(debug=False):
 class CrawlerRun(object):
     """A class to run the crawler and keep track of events occurred"""
 
-    def __init__(self):
+    def __init__(self, with_dupefilter=False):
         self.spider = None
         self.respplug = []
         self.reqplug = []
         self.reqdropped = []
         self.itemresp = []
         self.signals_catched = {}
+        self.spider_class = TestSpider if not with_dupefilter else \
+            TestDupeFilterSpider
 
     def run(self):
         self.port = start_test_site()
         self.portno = self.port.getHost().port
 
-        start_urls = [self.geturl("/"), self.geturl("/redirect")]
+        start_urls = [self.geturl("/"), self.geturl("/redirect"),
+                      self.geturl("/redirect")]  # a duplicate
 
         for name, signal in vars(signals).items():
             if not name.startswith('_'):
                 dispatcher.connect(self.record_signal, signal)
 
-        self.crawler = get_crawler(TestSpider)
+        self.crawler = get_crawler(self.spider_class)
         self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
         self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
         self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
@@ -146,10 +155,14 @@ class EngineTest(unittest.TestCase):
         self.run = CrawlerRun()
         yield self.run.run()
         self._assert_visited_urls()
-        self._assert_scheduled_requests()
+        self._assert_scheduled_requests(urls_to_visit=8)
         self._assert_downloaded_responses()
         self._assert_scraped_items()
         self._assert_signals_catched()
+        self.run = CrawlerRun(with_dupefilter=True)
+        yield self.run.run()
+        self._assert_scheduled_requests(urls_to_visit=7)
+        self._assert_dropped_requests()
 
     def _assert_visited_urls(self):
         must_be_visited = ["/", "/redirect", "/redirected",
@@ -158,8 +171,8 @@ class EngineTest(unittest.TestCase):
         urls_expected = set([self.run.geturl(p) for p in must_be_visited])
         assert urls_expected <= urls_visited, "URLs not visited: %s" % list(urls_expected - urls_visited)
 
-    def _assert_scheduled_requests(self):
-        self.assertEqual(6, len(self.run.reqplug))
+    def _assert_scheduled_requests(self, urls_to_visit=None):
+        self.assertEqual(urls_to_visit, len(self.run.reqplug))
 
         paths_expected = ['/item999.html', '/item2.html', '/item1.html']
 
@@ -172,9 +185,12 @@ class EngineTest(unittest.TestCase):
         self.assertEqual(scheduled_requests_count,
                          dropped_requests_count + responses_count)
 
+    def _assert_dropped_requests(self):
+        self.assertEqual(len(self.run.reqdropped), 1)
+
     def _assert_downloaded_responses(self):
         # response tests
-        self.assertEqual(6, len(self.run.respplug))
+        self.assertEqual(8, len(self.run.respplug))
 
         for response, _ in self.run.respplug:
             if self.run.getpath(response.url) == '/item999.html':

From 3602fc4fcb906a432126c87978c07bef42cea527 Mon Sep 17 00:00:00 2001
From: Stefan <misc@stefan-koch.name>
Date: Wed, 10 Dec 2014 22:48:09 +0100
Subject: [PATCH 0076/2974] fixed the variable types in mailsender
 documentation

---
 docs/topics/email.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index e73c74753..d99589413 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -63,10 +63,10 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
     :type smtpport: int
 
     :param smtptls: enforce using SMTP STARTTLS
-    :type smtpport: boolean
+    :type smtptls: boolean
 
     :param smtpssl: enforce using a secure SSL connection
-    :type smtpport: boolean
+    :type smtpssl: boolean
 
     .. classmethod:: from_settings(settings)
 

From 82b187f283b51c9c71e97d240e115b4abe09deec Mon Sep 17 00:00:00 2001
From: immerrr <immerrr@gmail.com>
Date: Thu, 11 Dec 2014 17:49:20 +0300
Subject: [PATCH 0077/2974] S3DownloadHandler: fix auth for requests with
 quoted paths/query params

---
 scrapy/core/downloader/handlers/s3.py |  6 ++++--
 tests/test_downloader_handlers.py     | 15 +++++++++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 09a76b7b7..f1e2c7767 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,3 +1,5 @@
+from urlparse import unquote
+
 from scrapy import optional_features
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
@@ -54,8 +56,8 @@ class S3DownloadHandler(object):
         signed_headers = self.conn.make_request(
                 method=request.method,
                 bucket=bucket,
-                key=p.path,
-                query_args=p.query,
+                key=unquote(p.path),
+                query_args=unquote(p.query),
                 headers=request.headers,
                 data=request.body)
         httpreq = request.replace(url=url, headers=signed_headers)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 9021af3b4..62fc280ee 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -482,6 +482,21 @@ class S3TestCase(unittest.TestCase):
         self.assertEqual(httpreq.headers['Authorization'], \
                 'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
 
+    def test_request_signing7(self):
+        # ensure that spaces are quoted properly before signing
+        req = Request(
+            ("s3://johnsmith/photos/my puppy.jpg"
+             "?response-content-disposition=my puppy.jpg"),
+            method='GET',
+            headers={
+                'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
+            })
+        httpreq = self.download_request(req, self.spider)
+        self.assertEqual(
+            httpreq.headers['Authorization'],
+            'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')
+
+
 class FTPTestCase(unittest.TestCase):
 
     username = "scrapy"

From d4cb03eded95dbd27af8dd90495fa4c277ef0b48 Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar@precifica.com.br>
Date: Thu, 11 Dec 2014 16:45:20 -0200
Subject: [PATCH 0078/2974] add CSS support for link extractors

---
 scrapy/contrib/linkextractors/lxmlhtml.py | 8 ++++----
 scrapy/contrib/linkextractors/sgml.py     | 8 ++++----
 scrapy/linkextractor.py                   | 8 +++++++-
 3 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py
index b6de74f33..f747fa99b 100644
--- a/scrapy/contrib/linkextractors/lxmlhtml.py
+++ b/scrapy/contrib/linkextractors/lxmlhtml.py
@@ -81,8 +81,8 @@ class LxmlParserLinkExtractor(object):
 class LxmlLinkExtractor(FilteringLinkExtractor):
 
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
-                 tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None,
-                 deny_extensions=None):
+                 restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True,
+                 unique=True, process_value=None, deny_extensions=None):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
         tag_func = lambda x: x in tags
         attr_func = lambda x: x in attrs
@@ -90,8 +90,8 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
             unique=unique, process=process_value)
 
         super(LxmlLinkExtractor, self).__init__(lx, allow, deny,
-            allow_domains, deny_domains, restrict_xpaths, canonicalize,
-            deny_extensions)
+            allow_domains, deny_domains, restrict_xpaths, restrict_css,
+            canonicalize, deny_extensions)
 
     def extract_links(self, response):
         html = Selector(response)
diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py
index 3eb5fd91f..3a8fdbb69 100644
--- a/scrapy/contrib/linkextractors/sgml.py
+++ b/scrapy/contrib/linkextractors/sgml.py
@@ -98,8 +98,8 @@ class BaseSgmlLinkExtractor(SGMLParser):
 class SgmlLinkExtractor(FilteringLinkExtractor):
 
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
-                 tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None,
-                 deny_extensions=None):
+                 restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True,
+                 process_value=None, deny_extensions=None):
 
         warnings.warn(
             "SgmlLinkExtractor is deprecated and will be removed in future releases. "
@@ -116,8 +116,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
                 unique=unique, process_value=process_value)
 
         super(SgmlLinkExtractor, self).__init__(lx, allow, deny,
-            allow_domains, deny_domains, restrict_xpaths, canonicalize,
-            deny_extensions)
+            allow_domains, deny_domains, restrict_xpaths, restrict_css,
+            canonicalize, deny_extensions)
 
         # FIXME: was added to fix a RegexLinkExtractor testcase
         self.base_url = None
diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py
index 5badea5e5..52271959f 100644
--- a/scrapy/linkextractor.py
+++ b/scrapy/linkextractor.py
@@ -5,6 +5,7 @@ scrapy.contrib.linkextractor).
 import re
 from six.moves.urllib.parse import urlparse
 
+from scrapy.selector.csstranslator import ScrapyHTMLTranslator
 from scrapy.utils.url import url_is_from_any_domain
 from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension
 from scrapy.utils.misc import arg_to_iter
@@ -38,8 +39,10 @@ _is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'fil
 
 class FilteringLinkExtractor(object):
 
+    _csstranslator = ScrapyHTMLTranslator()
+
     def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
-                 restrict_xpaths, canonicalize, deny_extensions):
+                 restrict_xpaths, restrict_css, canonicalize, deny_extensions):
 
         self.link_extractor = link_extractor
 
@@ -50,6 +53,9 @@ class FilteringLinkExtractor(object):
         self.deny_domains = set(arg_to_iter(deny_domains))
 
         self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths))
+        self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath,
+                                          arg_to_iter(restrict_css)))
+
         self.canonicalize = canonicalize
         if deny_extensions is None:
             deny_extensions = IGNORED_EXTENSIONS

From 403fc686b861b193a13a9abe038eda92aebf5ce8 Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar@precifica.com.br>
Date: Thu, 11 Dec 2014 18:20:30 -0200
Subject: [PATCH 0079/2974] tests for CSS support in link extractors

---
 tests/test_contrib_linkextractors.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tests/test_contrib_linkextractors.py b/tests/test_contrib_linkextractors.py
index 3902d4c50..a624f9e66 100644
--- a/tests/test_contrib_linkextractors.py
+++ b/tests/test_contrib_linkextractors.py
@@ -284,6 +284,21 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
                          [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
                                fragment='', nofollow=False)])
 
+    def test_restrict_css(self):
+        lx = self.extractor_cls(restrict_css=('#subwrapper a',))
+        self.assertEqual(lx.extract_links(self.response), [
+            Link(url='http://example.com/sample2.html', text=u'sample 2')
+        ])
+
+    def test_restrict_css_and_restrict_xpaths_together(self):
+        lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ),
+                                restrict_css=('#subwrapper + a', ))
+        self.assertEqual([link for link in lx.extract_links(self.response)], [
+            Link(url='http://example.com/sample1.html', text=u''),
+            Link(url='http://example.com/sample2.html', text=u'sample 2'),
+            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+        ])
+
     def test_area_tag_with_unicode_present(self):
         body = """<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>"""
         response = HtmlResponse("http://example.org", body=body, encoding='utf-8')

From b0730a1d16b608a34d12218ed1fd36c6720eee61 Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar@precifica.com.br>
Date: Thu, 11 Dec 2014 18:22:08 -0200
Subject: [PATCH 0080/2974] documentation for CSS support in link extractors

---
 docs/topics/link-extractors.rst | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 43740adcc..9758c2f35 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -51,7 +51,7 @@ LxmlLinkExtractor
    :synopsis: lxml's HTMLParser-based link extractors
 
 
-.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None)
+.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None)
 
     LxmlLinkExtractor is the recommended link extractor with handy filtering
     options. It is implemented using lxml's robust HTMLParser.
@@ -88,6 +88,11 @@ LxmlLinkExtractor
         links. See examples below.
     :type restrict_xpaths: str or list
 
+    :param restrict_css: a CSS selector (or list of selectors) which defines
+        regions inside the response where links should be extracted from.
+        Has the same behaviour as ``restrict_xpaths``.
+    :type restrict_css: str or list
+
     :param tags: a tag or a list of tags to consider when extracting links.
         Defaults to ``('a', 'area')``.
     :type tags: str or list

From 22247cf79152dbeb635b77f445ebd359c6d910db Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar@precifica.com.br>
Date: Mon, 15 Dec 2014 09:18:15 -0200
Subject: [PATCH 0081/2974] move restrict_css argument to end of argument list
 in link extractors for backwards compatibility, use keyword arguments in link
 extractor super().__init__() calls

---
 scrapy/contrib/linkextractors/lxmlhtml.py | 11 ++++++-----
 scrapy/contrib/linkextractors/sgml.py     | 11 ++++++-----
 scrapy/linkextractor.py                   |  2 +-
 3 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py
index f747fa99b..1ff8e4d36 100644
--- a/scrapy/contrib/linkextractors/lxmlhtml.py
+++ b/scrapy/contrib/linkextractors/lxmlhtml.py
@@ -81,17 +81,18 @@ class LxmlParserLinkExtractor(object):
 class LxmlLinkExtractor(FilteringLinkExtractor):
 
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
-                 restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True,
-                 unique=True, process_value=None, deny_extensions=None):
+                 tags=('a', 'area'), attrs=('href',), canonicalize=True,
+                 unique=True, process_value=None, deny_extensions=None, restrict_css=()):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
         tag_func = lambda x: x in tags
         attr_func = lambda x: x in attrs
         lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func,
             unique=unique, process=process_value)
 
-        super(LxmlLinkExtractor, self).__init__(lx, allow, deny,
-            allow_domains, deny_domains, restrict_xpaths, restrict_css,
-            canonicalize, deny_extensions)
+        super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
+            allow_domains=allow_domains, deny_domains=deny_domains,
+            restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
+            canonicalize=canonicalize, deny_extensions=deny_extensions)
 
     def extract_links(self, response):
         html = Selector(response)
diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py
index 3a8fdbb69..335773db1 100644
--- a/scrapy/contrib/linkextractors/sgml.py
+++ b/scrapy/contrib/linkextractors/sgml.py
@@ -98,8 +98,8 @@ class BaseSgmlLinkExtractor(SGMLParser):
 class SgmlLinkExtractor(FilteringLinkExtractor):
 
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
-                 restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True,
-                 process_value=None, deny_extensions=None):
+                 tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True,
+                 process_value=None, deny_extensions=None, restrict_css=()):
 
         warnings.warn(
             "SgmlLinkExtractor is deprecated and will be removed in future releases. "
@@ -115,9 +115,10 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
             lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
                 unique=unique, process_value=process_value)
 
-        super(SgmlLinkExtractor, self).__init__(lx, allow, deny,
-            allow_domains, deny_domains, restrict_xpaths, restrict_css,
-            canonicalize, deny_extensions)
+        super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
+            allow_domains=allow_domains, deny_domains=deny_domains,
+            restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
+            canonicalize=canonicalize, deny_extensions=deny_extensions)
 
         # FIXME: was added to fix a RegexLinkExtractor testcase
         self.base_url = None
diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py
index 52271959f..227d79b46 100644
--- a/scrapy/linkextractor.py
+++ b/scrapy/linkextractor.py
@@ -42,7 +42,7 @@ class FilteringLinkExtractor(object):
     _csstranslator = ScrapyHTMLTranslator()
 
     def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
-                 restrict_xpaths, restrict_css, canonicalize, deny_extensions):
+                 restrict_xpaths, canonicalize, deny_extensions, restrict_css):
 
         self.link_extractor = link_extractor
 

From 82d138e87e68b1e198dfdc3b0d7f62d0de2e1ceb Mon Sep 17 00:00:00 2001
From: tpeng <pengtaoo@gmail.com>
Date: Mon, 1 Dec 2014 14:15:15 +0100
Subject: [PATCH 0082/2974] support namespace prefix in xmliter_lxml

---
 scrapy/contrib_exp/iterators.py |  6 +++---
 tests/test_utils_iterators.py   | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/scrapy/contrib_exp/iterators.py b/scrapy/contrib_exp/iterators.py
index 7cf9103fd..d96105fb9 100644
--- a/scrapy/contrib_exp/iterators.py
+++ b/scrapy/contrib_exp/iterators.py
@@ -2,18 +2,18 @@ from scrapy.http import Response
 from scrapy.selector import Selector
 
 
-def xmliter_lxml(obj, nodename, namespace=None):
+def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
     from lxml import etree
     reader = _StreamReader(obj)
     tag = '{%s}%s' % (namespace, nodename) if namespace else nodename
     iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
-    selxpath = '//' + ('x:%s' % nodename if namespace else nodename)
+    selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename)
     for _, node in iterable:
         nodetext = etree.tostring(node)
         node.clear()
         xs = Selector(text=nodetext, type='xml')
         if namespace:
-            xs.register_namespace('x', namespace)
+            xs.register_namespace(prefix, namespace)
         yield xs.xpath(selxpath)[0]
 
 
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 840f4c596..d8faa810c 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -124,6 +124,38 @@ class LxmlXmliterTestCase(XmliterTestCase):
         node = next(namespace_iter)
         self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg'])
 
+    def test_xmliter_namespaces_prefix(self):
+        body = """\
+        <?xml version="1.0" encoding="UTF-8"?>
+        <root>
+            <h:table xmlns:h="http://www.w3.org/TR/html4/">
+              <h:tr>
+                <h:td>Apples</h:td>
+                <h:td>Bananas</h:td>
+              </h:tr>
+            </h:table>
+
+            <f:table xmlns:f="http://www.w3schools.com/furniture">
+              <f:name>African Coffee Table</f:name>
+              <f:width>80</f:width>
+              <f:length>120</f:length>
+            </f:table>
+
+        </root>
+        """
+        response = XmlResponse(url='http://mydummycompany.com', body=body)
+        my_iter = self.xmliter(response, 'table', 'http://www.w3.org/TR/html4/', 'h')
+
+        node = next(my_iter)
+        self.assertEqual(len(node.xpath('h:tr/h:td').extract()), 2)
+        self.assertEqual(node.xpath('h:tr/h:td[1]/text()').extract(), ['Apples'])
+        self.assertEqual(node.xpath('h:tr/h:td[2]/text()').extract(), ['Bananas'])
+
+        my_iter = self.xmliter(response, 'table', 'http://www.w3schools.com/furniture', 'f')
+
+        node = next(my_iter)
+        self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table'])
+
 
 class UtilsCsvTestCase(unittest.TestCase):
     sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')

From 880c9e52b5c0a88df39e5fda6546771f4729bc47 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Tue, 16 Dec 2014 19:28:39 +0000
Subject: [PATCH 0083/2974] t.i.b.ThreadedResolver is now a new-style class

since 2009 / twisted-9.0.0
https://github.com/twisted/twisted/commit/663d669dce6ee5009eee8c6d2a81f5199855178b
---
 scrapy/resolver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 7d9811727..e1a7aedce 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -13,7 +13,7 @@ class CachingThreadedResolver(ThreadedResolver):
     def getHostByName(self, name, timeout = (1, 3, 11, 45)):
         if name in dnscache:
             return defer.succeed(dnscache[name])
-        d = ThreadedResolver.getHostByName(self, name, timeout)
+        d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
         d.addCallback(self._cache_result, name)
         return d
 

From 09ba4ff68a7ecb00036d02ffda9ee48f48b69402 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 30 Dec 2014 19:53:50 -0300
Subject: [PATCH 0084/2974] Patches Twisted issue while closing the connection
 pool on HTTPDownloadHandler

---
 scrapy/core/downloader/handlers/http11.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index dd3ad488b..19db71bd1 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -32,6 +32,7 @@ class HTTP11DownloadHandler(object):
         self._contextFactory = self._contextFactoryClass()
         self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
         self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
+        self._disconnect_timeout = 1
 
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
@@ -41,7 +42,24 @@ class HTTP11DownloadHandler(object):
         return agent.download_request(request)
 
     def close(self):
-        return self._pool.closeCachedConnections()
+        d = self._pool.closeCachedConnections()
+        # closeCachedConnections will hang on network or server issues, so
+        # we'll manually timeout the deferred.
+        #
+        # Twisted issue addressing this problem can be found here:
+        # https://twistedmatrix.com/trac/ticket/7738.
+        #
+        # closeCachedConnections doesn't handle external errbacks, so we'll
+        # issue a callback after `_disconnect_timeout` seconds.
+        delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, [])
+
+        def cancel_delayed_call(result):
+            if delayed_call.active():
+                delayed_call.cancel()
+            return result
+
+        d.addBoth(cancel_delayed_call)
+        return d
 
 
 class TunnelError(Exception):

From d68615a5af6df2f972c7d46ab3fb8950f15747e6 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 19 Jan 2015 10:28:25 -0300
Subject: [PATCH 0085/2974] Test the parse command locally instead of against
 an external url

---
 scrapy/utils/testsite.py |  2 ++
 tests/test_commands.py   | 28 +++++++++++++++++++---------
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index 9f8419814..7d4d4b2bd 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -7,10 +7,12 @@ from twisted.web import server, resource, static, util
 class SiteTest(object):
 
     def setUp(self):
+        super(SiteTest, self).setUp()
         self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
         self.baseurl = "http://localhost:%d/" % self.site.getHost().port
 
     def tearDown(self):
+        super(SiteTest, self).tearDown()
         self.site.stopListening()
 
     def url(self, path):
diff --git a/tests/test_commands.py b/tests/test_commands.py
index eefda833e..70b4e74dc 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -8,9 +8,13 @@ from shutil import rmtree
 from tempfile import mkdtemp
 
 from twisted.trial import unittest
+from twisted.internet import defer
 
 from scrapy.utils.python import retry_on_eintr
 from scrapy.utils.test import get_testenv
+from scrapy.utils.testsite import SiteTest
+from scrapy.utils.testproc import ProcessTest
+
 
 class ProjectTest(unittest.TestCase):
     project_name = 'testproject'
@@ -177,7 +181,9 @@ from scrapy.spider import Spider
         self.assert_("Unable to load" in log)
 
 
-class ParseCommandTest(CommandTest):
+class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
+
+    command = 'parse'
 
     def setUp(self):
         super(ParseCommandTest, self).setUp()
@@ -217,17 +223,21 @@ class MyPipeline(object):
 ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
 """ % self.project_name)
 
+    @defer.inlineCallbacks
     def test_spider_arguments(self):
-        p = self.proc('parse', '--spider', self.spider_name, '-a', 'test_arg=1',
-                '-c', 'parse', 'http://scrapinghub.com')
-        log = p.stderr.read()
-        self.assert_("[parse_spider] DEBUG: It Works!" in log, log)
+        _, _, stderr = yield self.execute(['--spider', self.spider_name,
+                                           '-a', 'test_arg=1',
+                                           '-c', 'parse',
+                                           self.url('/html')])
+        self.assert_("[parse_spider] DEBUG: It Works!" in stderr, stderr)
 
+    @defer.inlineCallbacks
     def test_pipelines(self):
-        p = self.proc('parse', '--spider', self.spider_name, '--pipelines',
-                '-c', 'parse', 'http://scrapinghub.com')
-        log = p.stderr.read()
-        self.assert_("[scrapy] INFO: It Works!" in log, log)
+        _, _, stderr = yield self.execute(['--spider', self.spider_name,
+                                           '--pipelines',
+                                           '-c', 'parse',
+                                           self.url('/html')])
+        self.assert_("[scrapy] INFO: It Works!" in stderr, stderr)
 
 
 class BenchCommandTest(CommandTest):

From 73e6b35622ec98a6520aa606c610b28d1b116e82 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 19 Jan 2015 22:02:46 +0500
Subject: [PATCH 0086/2974] DOC fix a reference

---
 docs/topics/spider-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 92dc6ac47..93878943c 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -94,7 +94,7 @@ following methods:
 
         :param response: the response which generated this output from the
           spider
-        :type response: class:`~scrapy.http.Response` object
+        :type response: :class:`~scrapy.http.Response` object
 
         :param result: the result returned by the spider
         :type result: an iterable of :class:`~scrapy.http.Request` or

From 283d6a5344c97d54091a8a79ba23e7cd819e7a4e Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 19 Jan 2015 22:07:03 +0500
Subject: [PATCH 0087/2974] DOC a couple more references are fixed

---
 docs/topics/spider-middleware.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 93878943c..173c4a80c 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -101,7 +101,7 @@ following methods:
           :class:`~scrapy.item.Item` objects
 
         :param spider: the spider whose result is being processed
-        :type spider: :class:`~scrapy.item.Spider` object
+        :type spider: :class:`~scrapy.spider.Spider` object
 
 
     .. method:: process_spider_exception(response, exception, spider)
@@ -156,7 +156,7 @@ following methods:
         :type start_requests: an iterable of :class:`~scrapy.http.Request`
 
         :param spider: the spider to whom the start requests belong
-        :type spider: :class:`~scrapy.item.Spider` object
+        :type spider: :class:`~scrapy.spider.Spider` object
 
 
 .. _Exception: http://docs.python.org/library/exceptions.html#exceptions.Exception

From 4bc14da59ed9f68a66e5fcc623bfcb471d98386a Mon Sep 17 00:00:00 2001
From: Capi Etheriel <barraponto@gmail.com>
Date: Mon, 19 Jan 2015 17:21:55 -0200
Subject: [PATCH 0088/2974] Updates documentation on dynamic item classes.

Fixes #398
---
 docs/topics/practices.rst | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index e9c7a94bf..13dde52a3 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -199,8 +199,6 @@ classes instead of manually coding them.
     from scrapy.item import DictItem, Field
 
     def create_item_class(class_name, field_list):
-        field_dict = {}
-        for field_name in field_list:
-            field_dict[field_name] = Field()
+        fields = {field_name: Field() for field_name in field_list}
 
-        return type(class_name, (DictItem,), field_dict)
+        return type(class_name, (DictItem,), {'fields': fields})

From bd5d99a2d2dd61868d8c69acf410f27238443abb Mon Sep 17 00:00:00 2001
From: Jonas Tingeborn <tinjon@gmail.com>
Date: Wed, 21 Jan 2015 20:18:11 +0100
Subject: [PATCH 0089/2974] add gzip compression to filesystem http cache
 backend

---
 docs/topics/downloader-middleware.rst        | 12 +++++++++++
 scrapy/contrib/httpcache.py                  | 21 +++++++++++---------
 scrapy/settings/default_settings.py          |  1 +
 tests/test_downloadermiddleware_httpcache.py |  5 +++++
 4 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 835af2e74..e74889dd9 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -560,6 +560,18 @@ Default: ``'scrapy.contrib.httpcache.DummyPolicy'``
 
 The class which implements the cache policy.
 
+.. setting:: HTTPCACHE_GZIP
+
+HTTPCACHE_GZIP
+^^^^^^^^^^^^^^
+
+.. versionadded:: 0.25
+
+Default: ``False``
+
+If enabled, will compress all cached data with gzip.
+This setting is specific to the Filesystem backend.
+
 
 HttpCompressionMiddleware
 -------------------------
diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py
index c5cb3023f..3173656fe 100644
--- a/scrapy/contrib/httpcache.py
+++ b/scrapy/contrib/httpcache.py
@@ -1,5 +1,6 @@
 from __future__ import print_function
 import os
+import gzip
 from six.moves import cPickle as pickle
 from importlib import import_module
 from time import time
@@ -220,6 +221,8 @@ class FilesystemCacheStorage(object):
     def __init__(self, settings):
         self.cachedir = data_path(settings['HTTPCACHE_DIR'])
         self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
+        self.use_gzip = settings.getbool('HTTPCACHE_GZIP')
+        self._open = gzip.open if self.use_gzip else open
 
     def open_spider(self, spider):
         pass
@@ -233,9 +236,9 @@ class FilesystemCacheStorage(object):
         if metadata is None:
             return  # not cached
         rpath = self._get_request_path(spider, request)
-        with open(os.path.join(rpath, 'response_body'), 'rb') as f:
+        with self._open(os.path.join(rpath, 'response_body'), 'rb') as f:
             body = f.read()
-        with open(os.path.join(rpath, 'response_headers'), 'rb') as f:
+        with self._open(os.path.join(rpath, 'response_headers'), 'rb') as f:
             rawheaders = f.read()
         url = metadata.get('response_url')
         status = metadata['status']
@@ -256,17 +259,17 @@ class FilesystemCacheStorage(object):
             'response_url': response.url,
             'timestamp': time(),
         }
-        with open(os.path.join(rpath, 'meta'), 'wb') as f:
+        with self._open(os.path.join(rpath, 'meta'), 'wb') as f:
             f.write(repr(metadata))
-        with open(os.path.join(rpath, 'pickled_meta'), 'wb') as f:
+        with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f:
             pickle.dump(metadata, f, protocol=2)
-        with open(os.path.join(rpath, 'response_headers'), 'wb') as f:
+        with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f:
             f.write(headers_dict_to_raw(response.headers))
-        with open(os.path.join(rpath, 'response_body'), 'wb') as f:
+        with self._open(os.path.join(rpath, 'response_body'), 'wb') as f:
             f.write(response.body)
-        with open(os.path.join(rpath, 'request_headers'), 'wb') as f:
+        with self._open(os.path.join(rpath, 'request_headers'), 'wb') as f:
             f.write(headers_dict_to_raw(request.headers))
-        with open(os.path.join(rpath, 'request_body'), 'wb') as f:
+        with self._open(os.path.join(rpath, 'request_body'), 'wb') as f:
             f.write(request.body)
 
     def _get_request_path(self, spider, request):
@@ -281,7 +284,7 @@ class FilesystemCacheStorage(object):
         mtime = os.stat(rpath).st_mtime
         if 0 < self.expiration_secs < time() - mtime:
             return  # expired
-        with open(metapath, 'rb') as f:
+        with self._open(metapath, 'rb') as f:
             return pickle.load(f)
 
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index cf216385a..0342b1ada 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -154,6 +154,7 @@ HTTPCACHE_IGNORE_HTTP_CODES = []
 HTTPCACHE_IGNORE_SCHEMES = ['file']
 HTTPCACHE_DBM_MODULE = 'anydbm'
 HTTPCACHE_POLICY = 'scrapy.contrib.httpcache.DummyPolicy'
+HTTPCACHE_GZIP = False
 
 ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager'
 
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 1e22ae661..7ac5d5d11 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -136,6 +136,11 @@ class FilesystemStorageTest(DefaultStorageTest):
 
     storage_class = 'scrapy.contrib.httpcache.FilesystemCacheStorage'
 
+class FilesystemStorageGzipTest(FilesystemStorageTest):
+
+    def _get_settings(self, **new_settings):
+        new_settings.setdefault('HTTPCACHE_GZIP', True)
+        return super(FilesystemStorageTest, self)._get_settings(**new_settings)
 
 class LeveldbStorageTest(DefaultStorageTest):
 

From a07b4353d6038ac0782b96a88659718f5fa6d61d Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 22 Jan 2015 22:58:10 +0500
Subject: [PATCH 0090/2974] DOC reorder topics

* duplicate "topics/commands" link is removed;
* Request/Response docs are moved to "Basic Concepts";
* settings docs are moved to "Basic Concepts";
* exceptions docs are moved to "Basic Concepts";
* "signals" and "exporters" docs are moved to "Extending Scrapy";
* "Reference" section is dropped because it is empty now.
---
 docs/index.rst | 43 ++++++++++++++++++-------------------------
 1 file changed, 18 insertions(+), 25 deletions(-)

diff --git a/docs/index.rst b/docs/index.rst
index 2a1ae037b..0384dae3d 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -63,7 +63,11 @@ Basic concepts
    topics/shell
    topics/item-pipeline
    topics/feed-exports
+   topics/request-response
    topics/link-extractors
+   topics/settings
+   topics/exceptions
+
 
 :doc:`topics/commands`
     Learn about the command-line tool used to manage your Scrapy project.
@@ -89,9 +93,19 @@ Basic concepts
 :doc:`topics/feed-exports`
     Output your scraped data using different formats and storages.
 
+:doc:`topics/request-response`
+    Understand the classes used to represent HTTP requests and responses.
+
 :doc:`topics/link-extractors`
     Convenient classes to extract links to follow from pages.
 
+:doc:`topics/settings`
+    Learn how to configure Scrapy and see all :ref:`available settings <topics-settings-ref>`.
+
+:doc:`topics/exceptions`
+    See all available exceptions and their meaning.
+
+
 Built-in services
 =================
 
@@ -106,7 +120,7 @@ Built-in services
 
 :doc:`topics/logging`
     Understand the simple logging facility provided by Scrapy.
-   
+
 :doc:`topics/stats`
     Collect statistics about your scraping crawler.
 
@@ -200,6 +214,9 @@ Extending Scrapy
    topics/spider-middleware
    topics/extensions
    topics/api
+   topics/signals
+   topics/exporters
+
 
 :doc:`topics/architecture`
     Understand the Scrapy architecture.
@@ -216,33 +233,9 @@ Extending Scrapy
 :doc:`topics/api`
     Use it on extensions and middlewares to extend Scrapy functionality
 
-Reference
-=========
-
-.. toctree::
-   :hidden:
-
-   topics/request-response
-   topics/settings
-   topics/signals
-   topics/exceptions
-   topics/exporters
-
-:doc:`topics/commands`
-    Learn about the command-line tool and see all :ref:`available commands <topics-commands-ref>`.
-
-:doc:`topics/request-response`
-    Understand the classes used to represent HTTP requests and responses.
-
-:doc:`topics/settings`
-    Learn how to configure Scrapy and see all :ref:`available settings <topics-settings-ref>`.
-
 :doc:`topics/signals`
     See all available signals and how to work with them.
 
-:doc:`topics/exceptions`
-    See all available exceptions and their meaning.
-
 :doc:`topics/exporters`
     Quickly export your scraped items to a file (XML, CSV, etc).
 

From 074b4a9315620801159e5a3427d9cf77d133f501 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 29 Jan 2015 14:53:37 -0200
Subject: [PATCH 0091/2974] Contribute to master branch

ref #975
---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 6624b43b6..2309bb1f6 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,2 +1,2 @@
 The guidelines for contributing are available here:
-http://doc.scrapy.org/en/latest/contributing.html
+http://doc.scrapy.org/en/master/contributing.html

From f0bdc14522aac0b7e52a1409847e7d57530d56ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 29 Jan 2015 15:40:12 -0200
Subject: [PATCH 0092/2974] Tentative attention message about what document to
 read for contributions

---
 docs/contributing.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index d7a47a746..f8d965df8 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -4,6 +4,11 @@
 Contributing to Scrapy
 ======================
 
+.. important::
+
+    Double check you are reading the most recent version of this document at
+    http://doc.scrapy.org/en/master/contributing.html
+
 There are many ways to contribute to Scrapy. Here are some of them:
 
 * Blog about Scrapy. Tell the world how you're using Scrapy. This will help

From e42a1ac1a1ead692a25f9c12fa19614ec5c4c090 Mon Sep 17 00:00:00 2001
From: Sudhanshu Shekhar <sudshekhar02@gmail.com>
Date: Wed, 14 Jan 2015 02:31:03 +0530
Subject: [PATCH 0093/2974] Reset items_scraped instead of item_count

items_scraped is the counter that needs to be reset each time we have scraped a specific number of items in the code instead of item_count (which represents the specific number of items needed before a message is logged). Updating the source code to reflect this.
Removed some irrelevant words from the log message.
Signed-off-by: Sudhanshu Shekhar <sudshekhar02@gmail.com>
---
 docs/topics/extensions.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 2c2f7fb9c..1824bcc3a 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -140,9 +140,9 @@ Here is the code of such extension::
 
         def item_scraped(self, item, spider):
             self.items_scraped += 1
-            if self.items_scraped == self.item_count:
-                spider.log("scraped %d items, resetting counter" % self.items_scraped)
-                self.item_count = 0
+            if self.items_scraped % self.item_count == 0:
+                spider.log("scraped %d items" % self.items_scraped)
+                
 
 .. _topics-extensions-ref:
 

From 721d8d5daedf543a51a074e5d8691312e63acbd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Maik=20R=C3=B6der?= <maikroeder@gmail.com>
Date: Wed, 4 Feb 2015 11:38:37 +0100
Subject: [PATCH 0094/2974] Update form.py

Typo fixed
---
 scrapy/http/request/form.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 3aa9db960..02e77ce29 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -1,5 +1,5 @@
 """
-This module implements the FormRequest class which is a more covenient class
+This module implements the FormRequest class which is a more convenient class
 (than Request) to generate Requests based on form data.
 
 See documentation in docs/topics/request-response.rst

From 5ea4a72b8eecdae6a2ed29adbbc96df9987509ef Mon Sep 17 00:00:00 2001
From: klangner <klangner@gmail.com>
Date: Mon, 9 Feb 2015 10:28:22 +0100
Subject: [PATCH 0095/2974] py3-ignores.txt supports comments

---
 .gitignore  | 2 ++
 conftest.py | 7 ++++---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index 4eb80012f..97535ccaf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+/.vagrant
+/scrapy.iml
 *.pyc
 _trial_temp*
 dropin.cache
diff --git a/conftest.py b/conftest.py
index aa27ddd2b..1c221491e 100644
--- a/conftest.py
+++ b/conftest.py
@@ -9,9 +9,10 @@ if 'django' not in optional_features:
     collect_ignore.append("tests/test_djangoitem/models.py")
 
 if six.PY3:
-    for fn in open('tests/py3-ignores.txt'):
-        if fn.strip():
-            collect_ignore.append(fn.strip())
+    for line in open('tests/py3-ignores.txt'):
+        filePath = line.strip()
+        if len(filePath) > 0 and filePath[0] != '#':    
+            collect_ignore.append(filePath)
 
 class LogObservers:
     """Class for keeping track of log observers across test modules"""

From a0299d97f56d1d034ac5178d1c1ef9da57aa0cde Mon Sep 17 00:00:00 2001
From: "klangner@gmail.com" <klangner@gmail.com>
Date: Tue, 10 Feb 2015 10:34:44 +0100
Subject: [PATCH 0096/2974] fixed variable name

---
 conftest.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/conftest.py b/conftest.py
index 1c221491e..258a670dd 100644
--- a/conftest.py
+++ b/conftest.py
@@ -10,9 +10,9 @@ if 'django' not in optional_features:
 
 if six.PY3:
     for line in open('tests/py3-ignores.txt'):
-        filePath = line.strip()
-        if len(filePath) > 0 and filePath[0] != '#':    
-            collect_ignore.append(filePath)
+        file_path = line.strip()
+        if len(file_path) > 0 and file_path[0] != '#':
+            collect_ignore.append(file_path)
 
 class LogObservers:
     """Class for keeping track of log observers across test modules"""

From d67ca77e61020802c593c8b60a977e26bebfd7c6 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sun, 15 Feb 2015 21:14:47 -0300
Subject: [PATCH 0097/2974] Support new _getEndpoint Agent signatures on
 Twisted 15.0.0

---
 scrapy/core/downloader/handlers/http11.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 19db71bd1..634c6398b 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -19,7 +19,7 @@ from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.core.downloader.webclient import _parse
 from scrapy.utils.misc import load_object
-from scrapy import log
+from scrapy import log, twisted_version
 
 
 class HTTP11DownloadHandler(object):
@@ -142,10 +142,19 @@ class TunnelingAgent(Agent):
         self._proxyConf = proxyConf
         self._contextFactory = contextFactory
 
-    def _getEndpoint(self, scheme, host, port):
-        return TunnelingTCP4ClientEndpoint(self._reactor, host, port,
-            self._proxyConf, self._contextFactory, self._connectTimeout,
-            self._bindAddress)
+    if twisted_version >= (15, 0, 0):
+        def _getEndpoint(self, uri):
+            return TunnelingTCP4ClientEndpoint(
+                self._reactor, uri.host, uri.port, self._proxyConf,
+                self._contextFactory, self._endpointFactory._connectTimeout,
+                self._endpointFactory._bindAddress)
+    else:
+        def _getEndpoint(self, scheme, host, port):
+            return TunnelingTCP4ClientEndpoint(
+                self._reactor, host, port, self._proxyConf,
+                self._contextFactory, self._connectTimeout,
+                self._bindAddress)
+
 
 
 class ScrapyAgent(object):

From 839ffba971c29b8aa5875ca60d5ea280acd95ae3 Mon Sep 17 00:00:00 2001
From: Sudhanshu Shekhar <sudshekhar02@gmail.com>
Date: Wed, 14 Jan 2015 02:31:03 +0530
Subject: [PATCH 0098/2974] Added the first version of SelectJmes

Utilizes jmespath. Also, added tests and documentation for the same.
---
 docs/topics/loaders.rst            | 24 ++++++++++++++++++++++++
 scrapy/contrib/loader/processor.py | 21 +++++++++++++++++++++
 tests/requirements.txt             |  1 +
 tests/test_contrib_loader.py       | 28 ++++++++++++++++++++++++++--
 4 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 236725309..9df8e117d 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -675,3 +675,27 @@ Here is a list of all built-in processors:
     constructor keyword arguments are used as default context values. See
     :class:`Compose` processor for more info.
 
+.. class:: SelectJmes(json_path)
+
+    Queries the value using the json path provided to the constructor and returns the output.
+    Requires jmespath (https://github.com/jmespath/jmespath) to run.
+    This processor takes only one input at a time.
+
+    Example::
+
+        >>> from scrapy.contrib.loader.processor import SelectJmes, Compose, MapCompose
+        >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
+        >>> proc({'foo': 'bar'})
+        'bar'
+        >>> proc({'foo': {'bar': 'baz'}})
+        {'bar': 'baz'}
+
+    Working with Json::
+
+        >>> import json
+        >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
+        >>> proc_single_json_str('{"foo": "bar"}')
+        u'bar'
+        >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
+        >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
+        [u'bar']
diff --git a/scrapy/contrib/loader/processor.py b/scrapy/contrib/loader/processor.py
index 8c74c5358..3b221acaf 100644
--- a/scrapy/contrib/loader/processor.py
+++ b/scrapy/contrib/loader/processor.py
@@ -8,6 +8,7 @@ from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.datatypes import MergeDict
 from .common import wrap_loader_context
 
+
 class MapCompose(object):
 
     def __init__(self, *functions, **default_loader_context):
@@ -63,6 +64,26 @@ class Identity(object):
         return values
 
 
+class SelectJmes(object):
+    """
+        Query the input string for the jmespath (given at instantiation),
+        and return the answer
+        Requires : jmespath(https://github.com/jmespath/jmespath)
+        Note: SelectJmes accepts only one input element at a time.
+    """
+    def __init__(self, json_path):
+        self.json_path = json_path
+        import jmespath
+        self.compiled_path = jmespath.compile(self.json_path)
+
+    def __call__(self, value):
+        """Query value for the jmespath query and return answer
+        :param str value: a string with JSON data to extract from
+        :return: Element extracted according to jmespath query
+        """
+        return self.compiled_path.search(value)
+
+
 class Join(object):
 
     def __init__(self, separator=u' '):
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 0c1b65aa8..4da8e30ea 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -3,3 +3,4 @@ mock
 mitmproxy==0.10.1
 netlib==0.10.1
 pytest-twisted
+jmespath
diff --git a/tests/test_contrib_loader.py b/tests/test_contrib_loader.py
index 3330b6105..6af4cf70c 100644
--- a/tests/test_contrib_loader.py
+++ b/tests/test_contrib_loader.py
@@ -3,12 +3,11 @@ from functools import partial
 
 from scrapy.contrib.loader import ItemLoader
 from scrapy.contrib.loader.processor import Join, Identity, TakeFirst, \
-    Compose, MapCompose
+    Compose, MapCompose, SelectJmes
 from scrapy.item import Item, Field
 from scrapy.selector import Selector
 from scrapy.http import HtmlResponse
 
-
 # test items
 class NameItem(Item):
     name = Field()
@@ -579,5 +578,30 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertEqual(l.get_output_value('url'), [u'scrapy.org'])
 
 
+class SelectJmesTestCase(unittest.TestCase):
+        test_list_equals = {
+            'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+            'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
+            'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
+            'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+            'dict': (
+                'foo.bar[*].name',
+                {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
+                ['one', 'two']
+            ),
+            'list': ('[1]', [1, 2], 2)
+        }
+
+        def test_output(self):
+            for l in self.test_list_equals:
+                expr, test_list, expected = self.test_list_equals[l]
+                test = SelectJmes(expr)(test_list)
+                self.assertEqual(
+                    test,
+                    expected,
+                    msg='test "{}" got {} expected {}'.format(l, test, expected)
+                )
+
+
 if __name__ == "__main__":
     unittest.main()

From c9d7386a32aeb4bc7fe9654d194651eee1ede56c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 25 Feb 2015 18:03:53 -0200
Subject: [PATCH 0099/2974] Add 0.24.5 release notes

---
 docs/news.rst | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index d246e98bc..0f5e78e8c 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,32 @@
 Release notes
 =============
 
+0.24.5 (2015-02-25)
+-------------------
+
+- Support new _getEndpoint Agent signatures on Twisted 15.0.0 (:commit:`540b9bc`)
+- DOC a couple more references are fixed (:commit:`b4c454b`)
+- DOC fix a reference (:commit:`e3c1260`)
+- t.i.b.ThreadedResolver is now a new-style class (:commit:`9e13f42`)
+- S3DownloadHandler: fix auth for requests with quoted paths/query params (:commit:`cdb9
+- fixed the variable types in mailsender documentation (:commit:`bb3a848`)
+- Reset items_scraped instead of item_count (:commit:`edb07a4`)
+- Tentative attention message about what document to read for contributions (:commit:`7e
+- mitmproxy 0.10.1 needs netlib 0.10.1 too (:commit:`874fcdd`)
+- pin mitmproxy 0.10.1 as >0.11 does not work with tests (:commit:`c6b21f0`)
+- Test the parse command locally instead of against an external url (:commit:`c3a6628`)
+- Patches Twisted issue while closing the connection pool on HTTPDownloadHandler (:commi
+- Updates documentation on dynamic item classes. (:commit:`eeb589a`)
+- Merge pull request #943 from Lazar-T/patch-3 (:commit:`5fdab02`)
+- typo (:commit:`b0ae199`)
+- pywin32 is required by Twisted. closes #937 (:commit:`5cb0cfb`)
+- Update install.rst (:commit:`781286b`)
+- Merge pull request #928 from Lazar-T/patch-1 (:commit:`b415d04`)
+- comma instead of fullstop (:commit:`627b9ba`)
+- Merge pull request #885 from jsma/patch-1 (:commit:`de909ad`)
+- Update request-response.rst (:commit:`3f3263d`)
+- SgmlLinkExtractor - fix for parsing <area> tag with Unicode present (:commit:`49b40f0`
+
 0.24.4 (2014-08-09)
 -------------------
 

From 31e5f164d41ad4781bb610db53f7ab7bb5ddc4f9 Mon Sep 17 00:00:00 2001
From: Berker Peksag <berker.peksag@gmail.com>
Date: Fri, 6 Mar 2015 15:45:04 +0200
Subject: [PATCH 0100/2974] Import unittest.mock if available.

mock is in the stdlib since Python 3.3.
---
 tests/__init__.py                            | 5 +++++
 tests/test_crawl.py                          | 2 +-
 tests/test_downloadermiddleware_robotstxt.py | 2 +-
 tests/test_pipeline_files.py                 | 3 ++-
 tests/test_settings/__init__.py              | 5 +----
 tests/test_spider.py                         | 6 ++----
 tests/test_utils_deprecate.py                | 3 ++-
 tox.ini                                      | 1 -
 8 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/tests/__init__.py b/tests/__init__.py
index 54e79b318..c6dd45181 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -6,6 +6,11 @@ see http://doc.scrapy.org/en/latest/contributing.html#running-tests
 
 import os
 
+try:
+    import unittest.mock as mock
+except ImportError:
+    import mock
+
 tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data')
 
 def get_testdata(*paths):
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 48931d6ff..0fcc7c31e 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -1,9 +1,9 @@
 import json
 import socket
-import mock
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 from scrapy.utils.test import get_crawler, get_testlog
+from tests import mock
 from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
     BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
 from tests.mockserver import MockServer
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index eba337cbd..8b7ac7a6b 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,6 +1,5 @@
 from __future__ import absolute_import
 import re
-import mock
 from twisted.internet import reactor
 from twisted.internet.defer import Deferred
 from twisted.trial import unittest
@@ -8,6 +7,7 @@ from scrapy.contrib.downloadermiddleware.robotstxt import RobotsTxtMiddleware
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
+from tests import mock
 
 
 class RobotsTxtMiddlewareTest(unittest.TestCase):
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index d1bcf5f79..0a1737c44 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,4 +1,3 @@
-import mock
 import os
 import time
 import hashlib
@@ -14,6 +13,8 @@ from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 
+from tests import mock
+
 
 def _mocked_download_func(request, info):
     response = request.meta.get('response')
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 38797ad45..092806d20 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -1,12 +1,9 @@
 import six
 import unittest
 import warnings
-try:
-    from unittest import mock
-except ImportError:
-    import mock
 
 from scrapy.settings import Settings, SettingsAttribute, CrawlerSettings
+from tests import mock
 from . import default_settings
 
 
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 585b4b53b..56bcb2b1e 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -3,10 +3,6 @@ import inspect
 import warnings
 from io import BytesIO
 from twisted.trial import unittest
-try:
-    from unittest import mock
-except ImportError:
-    import mock
 
 from scrapy import signals
 from scrapy.spider import Spider, BaseSpider
@@ -20,6 +16,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.test import get_crawler
 
+from tests import mock
+
 
 class SpiderTest(unittest.TestCase):
 
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index d31b1d1f3..3d94bc1a0 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -3,9 +3,10 @@ from __future__ import absolute_import
 import inspect
 import unittest
 import warnings
-import mock
 from scrapy.utils.deprecate import create_deprecated_class
 
+from tests import mock
+
 
 class MyWarning(UserWarning):
     pass
diff --git a/tox.ini b/tox.ini
index 624f550e1..2dff74973 100644
--- a/tox.ini
+++ b/tox.ini
@@ -50,7 +50,6 @@ deps =
     w3lib>=1.8.0
     Pillow
     # tests requirements
-    mock
     pytest>=2.6.0
     pytest-twisted
 

From c86e1bebec9d1e994ca4afdc13610c0fb59d6c53 Mon Sep 17 00:00:00 2001
From: Berker Peksag <berker.peksag@gmail.com>
Date: Sun, 8 Mar 2015 15:18:49 +0200
Subject: [PATCH 0101/2974] assertItemsEqual was renamed to assertCountEqual in
 Python 3.

---
 tests/test_settings/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 38797ad45..127e282ea 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -33,6 +33,9 @@ class SettingsAttributeTest(unittest.TestCase):
 
 class SettingsTest(unittest.TestCase):
 
+    if six.PY3:
+        assertItemsEqual = unittest.TestCase.assertCountEqual
+
     def setUp(self):
         self.settings = Settings()
 

From f7031c08ffadf3a78182d38c46c4d4897feccaa0 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 10 Mar 2015 22:29:07 -0300
Subject: [PATCH 0102/2974] updating list of Request.meta special keys

---
 docs/topics/downloader-middleware.rst | 2 ++
 docs/topics/request-response.rst      | 3 +++
 docs/topics/settings.rst              | 2 ++
 docs/topics/spider-middleware.rst     | 5 ++++-
 4 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 835af2e74..fbc5391fa 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -603,6 +603,8 @@ HttpProxyMiddleware
 
 .. versionadded:: 0.8
 
+.. reqmeta:: proxy
+
 .. class:: HttpProxyMiddleware
 
    This middleware sets the HTTP proxy to use for requests, by setting the
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 68d87d04f..f2c37c2e6 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -224,12 +224,15 @@ Those are:
 * :reqmeta:`dont_redirect`
 * :reqmeta:`dont_retry`
 * :reqmeta:`handle_httpstatus_list`
+* :reqmeta:`handle_httpstatus_all`
 * ``dont_merge_cookies`` (see ``cookies`` parameter of :class:`Request` constructor)
 * :reqmeta:`cookiejar`
 * :reqmeta:`redirect_urls`
 * :reqmeta:`bindaddress`
 * :reqmeta:`dont_obey_robotstxt`
 * :reqmeta:`download_timeout`
+* :reqmeta:`download_maxsize`
+* :reqmeta:`proxy`
 
 .. reqmeta:: bindaddress
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0e9e53de4..ea2c208bb 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -433,6 +433,8 @@ The maximum response size (in bytes) that downloader will download.
 
 If you want to disable it set to 0.
 
+.. reqmeta:: download_maxsize
+
 .. note::
 
     This size can be set per spider using :attr:`download_maxsize`
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 92dc6ac47..fc707a87a 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -225,9 +225,12 @@ this::
 
 .. reqmeta:: handle_httpstatus_list
 
+.. reqmeta:: handle_httpstatus_all
+
 The ``handle_httpstatus_list`` key of :attr:`Request.meta
 <scrapy.http.Request.meta>` can also be used to specify which response codes to
-allow on a per-request basis.
+allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all``
+to ``True`` if you want to allow any response code for a request.
 
 Keep in mind, however, that it's usually a bad idea to handle non-200
 responses, unless you really know what you're doing.

From 57a5ee0097f5ec9ba45726d28ec943b88ec47fde Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 12 Mar 2015 23:20:44 -0300
Subject: [PATCH 0103/2974] added example value to set for proxy meta key

---
 docs/topics/downloader-middleware.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index fbc5391fa..a3647fd9b 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -608,7 +608,7 @@ HttpProxyMiddleware
 .. class:: HttpProxyMiddleware
 
    This middleware sets the HTTP proxy to use for requests, by setting the
-   ``proxy`` meta value to :class:`~scrapy.http.Request` objects.
+   ``proxy`` meta value for :class:`~scrapy.http.Request` objects.
 
    Like the Python standard library modules `urllib`_ and `urllib2`_, it obeys
    the following environment variables:
@@ -617,6 +617,9 @@ HttpProxyMiddleware
    * ``https_proxy``
    * ``no_proxy``
 
+   You can also set the meta key ``proxy`` per-request, to a value like
+   ``http://some_proxy_server:port``.
+
 .. _urllib: http://docs.python.org/library/urllib.html
 .. _urllib2: http://docs.python.org/library/urllib2.html
 

From 05a88152affd1d153c7accb37370919d1374faa5 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Sat, 14 Mar 2015 16:12:37 -0300
Subject: [PATCH 0104/2974] adding more settings to project template

---
 .../templates/project/module/settings.py.tmpl | 57 ++++++++++++++++++-
 1 file changed, 54 insertions(+), 3 deletions(-)

diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index e9f1ba0e4..572519412 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -2,16 +2,67 @@
 
 # Scrapy settings for $project_name project
 #
-# For simplicity, this file contains only the most important settings by
-# default. All the other settings are documented here:
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
 #
 #     http://doc.scrapy.org/en/latest/topics/settings.html
-#
+#     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
+#     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
 
 BOT_NAME = '$project_name'
 
 SPIDER_MODULES = ['$project_name.spiders']
 NEWSPIDER_MODULE = '$project_name.spiders'
 
+
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
 #USER_AGENT = '$project_name (+http://www.yourdomain.com)'
+
+
+# Configure a delay for requests for the same website
+# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
+#DOWNLOAD_DELAY=3
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN=16
+#CONCURRENT_REQUESTS_PER_IP=16
+
+# Disable cookies (enabled by default)
+# COOKIES_ENABLED=False
+
+# Disable Telnet Console (enabled by default)
+# TELNETCONSOLE_ENABLED=False
+
+# Override the default request headers:
+# DEFAULT_REQUEST_HEADERS = {
+#    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#    'Accept-Language': 'en',
+# }
+
+# Enable or disable spider middlewares
+# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
+# SPIDER_MIDDLEWARES = {
+#     '$project_name.middlewares.MyCustomSpiderMiddleware': 543,
+# }
+
+# Enable or disable downloader middlewares
+# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
+# DOWNLOADER_MIDDLEWARES = {
+#     '$project_name.middlewares.MyCustomDownloaderMiddleware': 543,
+# }
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See http://doc.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED=True
+# The initial download delay (default: 5)
+#AUTOTHROTTLE_START_DELAY=3
+# The maximum download delay to be set in case of high latencies (default: 60)
+#AUTOTHROTTLE_MAX_DELAY=90
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG=True
+
+# Enable and configure HTTP caching (disabled by default)
+# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED=True
+#HTTPCACHE_EXPIRATION_SECS=3600
+#HTTPCACHE_DIR='httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES=[404]

From c13e23641bbc66c5d1a760310c6eb97e35c12c11 Mon Sep 17 00:00:00 2001
From: nramirezuy <nramirez.uy@gmail.com>
Date: Mon, 28 Jul 2014 17:58:56 -0300
Subject: [PATCH 0105/2974] httpcache dont_cache meta #19 #689

---
 docs/topics/downloader-middleware.rst            |  3 +++
 docs/topics/request-response.rst                 |  1 +
 scrapy/contrib/downloadermiddleware/httpcache.py |  6 ++++++
 tests/test_downloadermiddleware_httpcache.py     | 12 ++++++++++++
 4 files changed, 22 insertions(+)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index a3647fd9b..9ef09785f 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -328,6 +328,9 @@ HttpCacheMiddleware
     You can change the HTTP cache policy with the :setting:`HTTPCACHE_POLICY`
     setting. Or you can also implement your own policy.
 
+    .. reqmeta:: dont_cache
+
+    You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals `True`.
 
 .. _httpcache-policy-dummy:
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index f2c37c2e6..b6b165d50 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -227,6 +227,7 @@ Those are:
 * :reqmeta:`handle_httpstatus_all`
 * ``dont_merge_cookies`` (see ``cookies`` parameter of :class:`Request` constructor)
 * :reqmeta:`cookiejar`
+  :reqmeta:`dont_cache`
 * :reqmeta:`redirect_urls`
 * :reqmeta:`bindaddress`
 * :reqmeta:`dont_obey_robotstxt`
diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py
index 90aa6cab7..7b4b53f7c 100644
--- a/scrapy/contrib/downloadermiddleware/httpcache.py
+++ b/scrapy/contrib/downloadermiddleware/httpcache.py
@@ -28,6 +28,9 @@ class HttpCacheMiddleware(object):
         self.storage.close_spider(spider)
 
     def process_request(self, request, spider):
+        if request.meta.get('dont_cache', False):
+            return
+
         # Skip uncacheable requests
         if not self.policy.should_cache_request(request):
             request.meta['_dont_cache'] = True  # flag as uncacheable
@@ -53,6 +56,9 @@ class HttpCacheMiddleware(object):
         request.meta['cached_response'] = cachedresponse
 
     def process_response(self, request, response, spider):
+        if request.meta.get('dont_cache', False):
+            return response
+
         # Skip cached responses and uncacheable requests
         if 'cached' in response.flags or '_dont_cache' in request.meta:
             request.meta.pop('_dont_cache', None)
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 1e22ae661..ea811cb3b 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -89,6 +89,18 @@ class _BaseTest(unittest.TestCase):
         assert any(h in request2.headers for h in ('If-None-Match', 'If-Modified-Since'))
         self.assertEqual(request1.body, request2.body)
 
+    def test_dont_cache(self):
+        with self._middleware() as mw:
+            self.request.meta['dont_cache'] = True
+            mw.process_response(self.request, self.response, self.spider)
+            self.assertEqual(mw.storage.retrieve_response(self.spider, self.request), None)
+
+        with self._middleware() as mw:
+            self.request.meta['dont_cache'] = False
+            mw.process_response(self.request, self.response, self.spider)
+            if mw.policy.should_cache_response(self.response, self.request):
+                self.assertIsInstance(mw.storage.retrieve_response(self.spider, self.request), self.response.__class__)
+
 
 class DefaultStorageTest(_BaseTest):
 

From aa56dd30ea31d6ac1785e4fb383375250b2010c9 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Mon, 16 Mar 2015 17:36:46 -0300
Subject: [PATCH 0106/2974] updating new settings to template, as pointed in PR
 review

---
 .../templates/project/module/settings.py.tmpl  | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 572519412..f4832dd27 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -18,9 +18,12 @@ NEWSPIDER_MODULE = '$project_name.spiders'
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
 #USER_AGENT = '$project_name (+http://www.yourdomain.com)'
 
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+# CONCURRENT_REQUESTS=32
 
-# Configure a delay for requests for the same website
+# Configure a delay for requests for the same website (default: 0)
 # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
 #DOWNLOAD_DELAY=3
 # The download delay setting will honor only one of:
 #CONCURRENT_REQUESTS_PER_DOMAIN=16
@@ -50,8 +53,21 @@ NEWSPIDER_MODULE = '$project_name.spiders'
 #     '$project_name.middlewares.MyCustomDownloaderMiddleware': 543,
 # }
 
+# Enable or disable extensions
+# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
+# EXTENSIONS = {
+#     'scrapy.telnet.TelnetConsole': None,
+# }
+
+# Configure item pipelines
+# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
+# ITEM_PIPELINES = {
+#     '$project_name.pipelines.SomePipeline': 300,
+# }
+
 # Enable and configure the AutoThrottle extension (disabled by default)
 # See http://doc.scrapy.org/en/latest/topics/autothrottle.html
+# NOTE: AutoThrottle will honour the standard settings for concurrency and delay
 #AUTOTHROTTLE_ENABLED=True
 # The initial download delay (default: 5)
 #AUTOTHROTTLE_START_DELAY=3

From 64082b46405c401a1e5accc7ae297c146619df3f Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 17 Mar 2015 01:46:45 +0500
Subject: [PATCH 0107/2974] DOC contribute to master branch

See GH-975 and GH-1029.
---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 6020a3670..7e5dd173f 100644
--- a/README.rst
+++ b/README.rst
@@ -58,7 +58,7 @@ See http://scrapy.org/community/
 Contributing
 ============
 
-See http://doc.scrapy.org/en/latest/contributing.html
+See http://doc.scrapy.org/en/master/contributing.html
 
 Companies using Scrapy
 ======================

From 05cb31d3060c543e4266f0e3b67eb332ac4abe0f Mon Sep 17 00:00:00 2001
From: Alexander Chekunkov <chekunkov@gmail.com>
Date: Fri, 19 Dec 2014 21:06:58 +0200
Subject: [PATCH 0108/2974] pydispatch - pep8

reformat files - proper spacing
---
 scrapy/xlib/pydispatch/dispatcher.py  | 833 +++++++++++++-------------
 scrapy/xlib/pydispatch/errors.py      |  17 +-
 scrapy/xlib/pydispatch/robust.py      |  89 ++-
 scrapy/xlib/pydispatch/robustapply.py |  19 +-
 scrapy/xlib/pydispatch/saferef.py     | 314 +++++-----
 5 files changed, 655 insertions(+), 617 deletions(-)

diff --git a/scrapy/xlib/pydispatch/dispatcher.py b/scrapy/xlib/pydispatch/dispatcher.py
index f5a9a611b..7d588dd20 100644
--- a/scrapy/xlib/pydispatch/dispatcher.py
+++ b/scrapy/xlib/pydispatch/dispatcher.py
@@ -6,24 +6,24 @@ system.
 
 Module attributes of note:
 
-	Any -- Singleton used to signal either "Any Sender" or
-		"Any Signal".  See documentation of the _Any class.
-	Anonymous -- Singleton used to signal "Anonymous Sender"
-		See documentation of the _Anonymous class.
+    Any -- Singleton used to signal either "Any Sender" or
+        "Any Signal".  See documentation of the _Any class.
+    Anonymous -- Singleton used to signal "Anonymous Sender"
+        See documentation of the _Anonymous class.
 
 Internal attributes:
-	WEAKREF_TYPES -- tuple of types/classes which represent
-		weak references to receivers, and thus must be de-
-		referenced on retrieval to retrieve the callable
-		object
-	connections -- { senderkey (id) : { signal : [receivers...]}}
-	senders -- { senderkey (id) : weakref(sender) }
-		used for cleaning up sender references on sender
-		deletion
-	sendersBack -- { receiverkey (id) : [senderkey (id)...] }
-		used for cleaning up receiver references on receiver
-		deletion, (considerably speeds up the cleanup process
-		vs. the original code.)
+    WEAKREF_TYPES -- tuple of types/classes which represent
+        weak references to receivers, and thus must be de-
+        referenced on retrieval to retrieve the callable
+        object
+    connections -- { senderkey (id) : { signal : [receivers...]}}
+    senders -- { senderkey (id) : weakref(sender) }
+        used for cleaning up sender references on sender
+        deletion
+    sendersBack -- { receiverkey (id) : [senderkey (id)...] }
+        used for cleaning up receiver references on receiver
+        deletion, (considerably speeds up the cleanup process
+        vs. the original code.)
 """
 from __future__ import generators
 import types, weakref, six
@@ -33,39 +33,47 @@ __author__ = "Patrick K. O'Brien <pobrien@orbtech.com>"
 __cvsid__ = "$Id: dispatcher.py,v 1.1.1.1 2006/07/07 15:59:38 mcfletch Exp $"
 __version__ = "$Revision: 1.1.1.1 $"[11:-2]
 
+
 class _Parameter:
-	"""Used to represent default parameter values."""
-	def __repr__(self):
-		return self.__class__.__name__
+    """Used to represent default parameter values."""
+
+    def __repr__(self):
+        return self.__class__.__name__
+
 
 class _Any(_Parameter):
-	"""Singleton used to signal either "Any Sender" or "Any Signal"
+    """Singleton used to signal either "Any Sender" or "Any Signal"
+
+    The Any object can be used with connect, disconnect,
+    send, or sendExact to signal that the parameter given
+    Any should react to all senders/signals, not just
+    a particular sender/signal.
+    """
+
 
-	The Any object can be used with connect, disconnect,
-	send, or sendExact to signal that the parameter given
-	Any should react to all senders/signals, not just
-	a particular sender/signal.
-	"""
 Any = _Any()
 
+
 class _Anonymous(_Parameter):
-	"""Singleton used to signal "Anonymous Sender"
+    """Singleton used to signal "Anonymous Sender"
+
+    The Anonymous object is used to signal that the sender
+    of a message is not specified (as distinct from being
+    "any sender").  Registering callbacks for Anonymous
+    will only receive messages sent without senders.  Sending
+    with anonymous will only send messages to those receivers
+    registered for Any or Anonymous.
+
+    Note:
+        The default sender for connect is Any, while the
+        default sender for send is Anonymous.  This has
+        the effect that if you do not specify any senders
+        in either function then all messages are routed
+        as though there was a single sender (Anonymous)
+        being used everywhere.
+    """
 
-	The Anonymous object is used to signal that the sender
-	of a message is not specified (as distinct from being
-	"any sender").  Registering callbacks for Anonymous
-	will only receive messages sent without senders.  Sending
-	with anonymous will only send messages to those receivers
-	registered for Any or Anonymous.
 
-	Note:
-		The default sender for connect is Any, while the
-		default sender for send is Anonymous.  This has
-		the effect that if you do not specify any senders
-		in either function then all messages are routed
-		as though there was a single sender (Anonymous)
-		being used everywhere.
-	"""
 Anonymous = _Anonymous()
 
 WEAKREF_TYPES = (weakref.ReferenceType, saferef.BoundMethodWeakref)
@@ -76,417 +84,428 @@ sendersBack = {}
 
 
 def connect(receiver, signal=Any, sender=Any, weak=True):
-	"""Connect receiver to sender for signal
+    """Connect receiver to sender for signal
 
-	receiver -- a callable Python object which is to receive
-		messages/signals/events.  Receivers must be hashable
-		objects.
+    receiver -- a callable Python object which is to receive
+        messages/signals/events.  Receivers must be hashable
+        objects.
 
-		if weak is True, then receiver must be weak-referencable
-		(more precisely saferef.safeRef() must be able to create
-		a reference to the receiver).
-	
-		Receivers are fairly flexible in their specification,
-		as the machinery in the robustApply module takes care
-		of most of the details regarding figuring out appropriate
-		subsets of the sent arguments to apply to a given
-		receiver.
+        if weak is True, then receiver must be weak-referencable
+        (more precisely saferef.safeRef() must be able to create
+        a reference to the receiver).
 
-		Note:
-			if receiver is itself a weak reference (a callable),
-			it will be de-referenced by the system's machinery,
-			so *generally* weak references are not suitable as
-			receivers, though some use might be found for the
-			facility whereby a higher-level library passes in
-			pre-weakrefed receiver references.
+        Receivers are fairly flexible in their specification,
+        as the machinery in the robustApply module takes care
+        of most of the details regarding figuring out appropriate
+        subsets of the sent arguments to apply to a given
+        receiver.
 
-	signal -- the signal to which the receiver should respond
-	
-		if Any, receiver will receive any signal from the
-		indicated sender (which might also be Any, but is not
-		necessarily Any).
-		
-		Otherwise must be a hashable Python object other than
-		None (DispatcherError raised on None).
-		
-	sender -- the sender to which the receiver should respond
-	
-		if Any, receiver will receive the indicated signals
-		from any sender.
-		
-		if Anonymous, receiver will only receive indicated
-		signals from send/sendExact which do not specify a
-		sender, or specify Anonymous explicitly as the sender.
+        Note:
+            if receiver is itself a weak reference (a callable),
+            it will be de-referenced by the system's machinery,
+            so *generally* weak references are not suitable as
+            receivers, though some use might be found for the
+            facility whereby a higher-level library passes in
+            pre-weakrefed receiver references.
 
-		Otherwise can be any python object.
-		
-	weak -- whether to use weak references to the receiver
-		By default, the module will attempt to use weak
-		references to the receiver objects.  If this parameter
-		is false, then strong references will be used.
+    signal -- the signal to which the receiver should respond
 
-	returns None, may raise DispatcherTypeError
-	"""
-	if signal is None:
-		raise errors.DispatcherTypeError(
-			'Signal cannot be None (receiver=%r sender=%r)'%( receiver,sender)
-		)
-	if weak:
-		receiver = saferef.safeRef(receiver, onDelete=_removeReceiver)
-	senderkey = id(sender)
-	if senderkey in connections:
-		signals = connections[senderkey]
-	else:
-		connections[senderkey] = signals = {}
-	# Keep track of senders for cleanup.
-	# Is Anonymous something we want to clean up?
-	if sender not in (None, Anonymous, Any):
-		def remove(object, senderkey=senderkey):
-			_removeSender(senderkey=senderkey)
-		# Skip objects that can not be weakly referenced, which means
-		# they won't be automatically cleaned up, but that's too bad.
-		try:
-			weakSender = weakref.ref(sender, remove)
-			senders[senderkey] = weakSender
-		except:
-			pass
-		
-	receiverID = id(receiver)
-	# get current set, remove any current references to
-	# this receiver in the set, including back-references
-	if signal in signals:
-		receivers = signals[signal]
-		_removeOldBackRefs(senderkey, signal, receiver, receivers)
-	else:
-		receivers = signals[signal] = []
-	try:
-		current = sendersBack.get( receiverID )
-		if current is None:
-			sendersBack[ receiverID ] = current = []
-		if senderkey not in current:
-			current.append(senderkey)
-	except:
-		pass
+        if Any, receiver will receive any signal from the
+        indicated sender (which might also be Any, but is not
+        necessarily Any).
 
-	receivers.append(receiver)
+        Otherwise must be a hashable Python object other than
+        None (DispatcherError raised on None).
 
+    sender -- the sender to which the receiver should respond
+
+        if Any, receiver will receive the indicated signals
+        from any sender.
+
+        if Anonymous, receiver will only receive indicated
+        signals from send/sendExact which do not specify a
+        sender, or specify Anonymous explicitly as the sender.
+
+        Otherwise can be any python object.
+
+    weak -- whether to use weak references to the receiver
+        By default, the module will attempt to use weak
+        references to the receiver objects.  If this parameter
+        is false, then strong references will be used.
+
+    returns None, may raise DispatcherTypeError
+    """
+    if signal is None:
+        raise errors.DispatcherTypeError(
+            'Signal cannot be None (receiver=%r sender=%r)' % (
+                receiver, sender)
+        )
+    if weak:
+        receiver = saferef.safeRef(receiver, onDelete=_removeReceiver)
+    senderkey = id(sender)
+    if senderkey in connections:
+        signals = connections[senderkey]
+    else:
+        connections[senderkey] = signals = {}
+    # Keep track of senders for cleanup.
+    # Is Anonymous something we want to clean up?
+    if sender not in (None, Anonymous, Any):
+        def remove(object, senderkey=senderkey):
+            _removeSender(senderkey=senderkey)
+
+        # Skip objects that can not be weakly referenced, which means
+        # they won't be automatically cleaned up, but that's too bad.
+        try:
+            weakSender = weakref.ref(sender, remove)
+            senders[senderkey] = weakSender
+        except:
+            pass
+
+    receiverID = id(receiver)
+    # get current set, remove any current references to
+    # this receiver in the set, including back-references
+    if signal in signals:
+        receivers = signals[signal]
+        _removeOldBackRefs(senderkey, signal, receiver, receivers)
+    else:
+        receivers = signals[signal] = []
+    try:
+        current = sendersBack.get(receiverID)
+        if current is None:
+            sendersBack[receiverID] = current = []
+        if senderkey not in current:
+            current.append(senderkey)
+    except:
+        pass
+
+    receivers.append(receiver)
 
 
 def disconnect(receiver, signal=Any, sender=Any, weak=True):
-	"""Disconnect receiver from sender for signal
+    """Disconnect receiver from sender for signal
 
-	receiver -- the registered receiver to disconnect
-	signal -- the registered signal to disconnect
-	sender -- the registered sender to disconnect
-	weak -- the weakref state to disconnect
+    receiver -- the registered receiver to disconnect
+    signal -- the registered signal to disconnect
+    sender -- the registered sender to disconnect
+    weak -- the weakref state to disconnect
 
-	disconnect reverses the process of connect,
-	the semantics for the individual elements are
-	logically equivalent to a tuple of
-	(receiver, signal, sender, weak) used as a key
-	to be deleted from the internal routing tables.
-	(The actual process is slightly more complex
-	but the semantics are basically the same).
+    disconnect reverses the process of connect,
+    the semantics for the individual elements are
+    logically equivalent to a tuple of
+    (receiver, signal, sender, weak) used as a key
+    to be deleted from the internal routing tables.
+    (The actual process is slightly more complex
+    but the semantics are basically the same).
 
-	Note:
-		Using disconnect is not required to cleanup
-		routing when an object is deleted, the framework
-		will remove routes for deleted objects
-		automatically.  It's only necessary to disconnect
-		if you want to stop routing to a live object.
-		
-	returns None, may raise DispatcherTypeError or
-		DispatcherKeyError
-	"""
-	if signal is None:
-		raise errors.DispatcherTypeError(
-			'Signal cannot be None (receiver=%r sender=%r)'%( receiver,sender)
-		)
-	if weak: receiver = saferef.safeRef(receiver)
-	senderkey = id(sender)
-	try:
-		signals = connections[senderkey]
-		receivers = signals[signal]
-	except KeyError:
-		raise errors.DispatcherKeyError(
-			"""No receivers found for signal %r from sender %r""" %(
-				signal,
-				sender
-			)
-		)
-	try:
-		# also removes from receivers
-		_removeOldBackRefs(senderkey, signal, receiver, receivers)
-	except ValueError:
-		raise errors.DispatcherKeyError(
-			"""No connection to receiver %s for signal %s from sender %s""" %(
-				receiver,
-				signal,
-				sender
-			)
-		)
-	_cleanupConnections(senderkey, signal)
+    Note:
+        Using disconnect is not required to cleanup
+        routing when an object is deleted, the framework
+        will remove routes for deleted objects
+        automatically.  It's only necessary to disconnect
+        if you want to stop routing to a live object.
 
-def getReceivers( sender = Any, signal = Any ):
-	"""Get list of receivers from global tables
+    returns None, may raise DispatcherTypeError or
+        DispatcherKeyError
+    """
+    if signal is None:
+        raise errors.DispatcherTypeError(
+            'Signal cannot be None (receiver=%r sender=%r)' % (
+                receiver, sender)
+        )
+    if weak: receiver = saferef.safeRef(receiver)
+    senderkey = id(sender)
+    try:
+        signals = connections[senderkey]
+        receivers = signals[signal]
+    except KeyError:
+        raise errors.DispatcherKeyError(
+            """No receivers found for signal %r from sender %r""" % (
+                signal,
+                sender
+            )
+        )
+    try:
+        # also removes from receivers
+        _removeOldBackRefs(senderkey, signal, receiver, receivers)
+    except ValueError:
+        raise errors.DispatcherKeyError(
+            """No connection to receiver %s for signal %s from sender %s""" % (
+                receiver,
+                signal,
+                sender
+            )
+        )
+    _cleanupConnections(senderkey, signal)
 
-	This utility function allows you to retrieve the
-	raw list of receivers from the connections table
-	for the given sender and signal pair.
 
-	Note:
-		there is no guarantee that this is the actual list
-		stored in the connections table, so the value
-		should be treated as a simple iterable/truth value
-		rather than, for instance a list to which you
-		might append new records.
+def getReceivers(sender=Any, signal=Any):
+    """Get list of receivers from global tables
+
+    This utility function allows you to retrieve the
+    raw list of receivers from the connections table
+    for the given sender and signal pair.
+
+    Note:
+        there is no guarantee that this is the actual list
+        stored in the connections table, so the value
+        should be treated as a simple iterable/truth value
+        rather than, for instance a list to which you
+        might append new records.
+
+    Normally you would use liveReceivers( getReceivers( ...))
+    to retrieve the actual receiver objects as an iterable
+    object.
+    """
+    try:
+        return connections[id(sender)][signal]
+    except KeyError:
+        return []
 
-	Normally you would use liveReceivers( getReceivers( ...))
-	to retrieve the actual receiver objects as an iterable
-	object.
-	"""
-	try:
-		return connections[id(sender)][signal]
-	except KeyError:
-		return []
 
 def liveReceivers(receivers):
-	"""Filter sequence of receivers to get resolved, live receivers
+    """Filter sequence of receivers to get resolved, live receivers
 
-	This is a generator which will iterate over
-	the passed sequence, checking for weak references
-	and resolving them, then returning all live
-	receivers.
-	"""
-	for receiver in receivers:
-		if isinstance( receiver, WEAKREF_TYPES):
-			# Dereference the weak reference.
-			receiver = receiver()
-			if receiver is not None:
-				yield receiver
-		else:
-			yield receiver
+    This is a generator which will iterate over
+    the passed sequence, checking for weak references
+    and resolving them, then returning all live
+    receivers.
+    """
+    for receiver in receivers:
+        if isinstance(receiver, WEAKREF_TYPES):
+            # Dereference the weak reference.
+            receiver = receiver()
+            if receiver is not None:
+                yield receiver
+        else:
+            yield receiver
 
 
+def getAllReceivers(sender=Any, signal=Any):
+    """Get list of all receivers from global tables
 
-def getAllReceivers( sender = Any, signal = Any ):
-	"""Get list of all receivers from global tables
+    This gets all receivers which should receive
+    the given signal from sender, each receiver should
+    be produced only once by the resulting generator
+    """
+    receivers = {}
+    for set in (
+            # Get receivers that receive *this* signal from *this* sender.
+            getReceivers(sender, signal),
+            # Add receivers that receive *any* signal from *this* sender.
+            getReceivers(sender, Any),
+            # Add receivers that receive *this* signal from *any* sender.
+            getReceivers(Any, signal),
+            # Add receivers that receive *any* signal from *any* sender.
+            getReceivers(Any, Any),
+    ):
+        for receiver in set:
+            if receiver:  # filter out dead instance-method weakrefs
+                try:
+                    if receiver not in receivers:
+                        receivers[receiver] = 1
+                        yield receiver
+                except TypeError:
+                    # dead weakrefs raise TypeError on hash...
+                    pass
 
-	This gets all receivers which should receive
-	the given signal from sender, each receiver should
-	be produced only once by the resulting generator
-	"""
-	receivers = {}
-	for set in (
-		# Get receivers that receive *this* signal from *this* sender.
-		getReceivers( sender, signal ),
-		# Add receivers that receive *any* signal from *this* sender.
-		getReceivers( sender, Any ),
-		# Add receivers that receive *this* signal from *any* sender.
-		getReceivers( Any, signal ),
-		# Add receivers that receive *any* signal from *any* sender.
-		getReceivers( Any, Any ),
-	):
-		for receiver in set:
-			if receiver: # filter out dead instance-method weakrefs
-				try:
-					if receiver not in receivers:
-						receivers[receiver] = 1
-						yield receiver
-				except TypeError:
-					# dead weakrefs raise TypeError on hash...
-					pass
 
 def send(signal=Any, sender=Anonymous, *arguments, **named):
-	"""Send signal from sender to all connected receivers.
-	
-	signal -- (hashable) signal value, see connect for details
+    """Send signal from sender to all connected receivers.
 
-	sender -- the sender of the signal
-	
-		if Any, only receivers registered for Any will receive
-		the message.
+    signal -- (hashable) signal value, see connect for details
 
-		if Anonymous, only receivers registered to receive
-		messages from Anonymous or Any will receive the message
+    sender -- the sender of the signal
 
-		Otherwise can be any python object (normally one
-		registered with a connect if you actually want
-		something to occur).
+        if Any, only receivers registered for Any will receive
+        the message.
 
-	arguments -- positional arguments which will be passed to
-		*all* receivers. Note that this may raise TypeErrors
-		if the receivers do not allow the particular arguments.
-		Note also that arguments are applied before named
-		arguments, so they should be used with care.
+        if Anonymous, only receivers registered to receive
+        messages from Anonymous or Any will receive the message
 
-	named -- named arguments which will be filtered according
-		to the parameters of the receivers to only provide those
-		acceptable to the receiver.
+        Otherwise can be any python object (normally one
+        registered with a connect if you actually want
+        something to occur).
 
-	Return a list of tuple pairs [(receiver, response), ... ]
+    arguments -- positional arguments which will be passed to
+        *all* receivers. Note that this may raise TypeErrors
+        if the receivers do not allow the particular arguments.
+        Note also that arguments are applied before named
+        arguments, so they should be used with care.
 
-	if any receiver raises an error, the error propagates back
-	through send, terminating the dispatch loop, so it is quite
-	possible to not have all receivers called if a raises an
-	error.
-	"""
-	# Call each receiver with whatever arguments it can accept.
-	# Return a list of tuple pairs [(receiver, response), ... ].
-	responses = []
-	for receiver in liveReceivers(getAllReceivers(sender, signal)):
-		response = robustapply.robustApply(
-			receiver,
-			signal=signal,
-			sender=sender,
-			*arguments,
-			**named
-		)
-		responses.append((receiver, response))
-	return responses
-def sendExact( signal=Any, sender=Anonymous, *arguments, **named ):
-	"""Send signal only to those receivers registered for exact message
+    named -- named arguments which will be filtered according
+        to the parameters of the receivers to only provide those
+        acceptable to the receiver.
+
+    Return a list of tuple pairs [(receiver, response), ... ]
+
+    if any receiver raises an error, the error propagates back
+    through send, terminating the dispatch loop, so it is quite
+    possible to not have all receivers called if a raises an
+    error.
+    """
+    # Call each receiver with whatever arguments it can accept.
+    # Return a list of tuple pairs [(receiver, response), ... ].
+    responses = []
+    for receiver in liveReceivers(getAllReceivers(sender, signal)):
+        response = robustapply.robustApply(
+            receiver,
+            signal=signal,
+            sender=sender,
+            *arguments,
+            **named
+        )
+        responses.append((receiver, response))
+    return responses
+
+
+def sendExact(signal=Any, sender=Anonymous, *arguments, **named):
+    """Send signal only to those receivers registered for exact message
+
+    sendExact allows for avoiding Any/Anonymous registered
+    handlers, sending only to those receivers explicitly
+    registered for a particular signal on a particular
+    sender.
+    """
+    responses = []
+    for receiver in liveReceivers(getReceivers(sender, signal)):
+        response = robustapply.robustApply(
+            receiver,
+            signal=signal,
+            sender=sender,
+            *arguments,
+            **named
+        )
+        responses.append((receiver, response))
+    return responses
 
-	sendExact allows for avoiding Any/Anonymous registered
-	handlers, sending only to those receivers explicitly
-	registered for a particular signal on a particular
-	sender.
-	"""
-	responses = []
-	for receiver in liveReceivers(getReceivers(sender, signal)):
-		response = robustapply.robustApply(
-			receiver,
-			signal=signal,
-			sender=sender,
-			*arguments,
-			**named
-		)
-		responses.append((receiver, response))
-	return responses
-	
 
 def _removeReceiver(receiver):
-	"""Remove receiver from connections."""
-	if not sendersBack:
-		# During module cleanup the mapping will be replaced with None
-		return False
-	backKey = id(receiver)
-	try:
-		backSet = sendersBack.pop(backKey)
-	except KeyError as err:
-		return False 
-	else:
-		for senderkey in backSet:
-			try:
-				signals = connections[senderkey].keys()
-			except KeyError as err:
-				pass
-			else:
-				for signal in signals:
-					try:
-						receivers = connections[senderkey][signal]
-					except KeyError:
-						pass
-					else:
-						try:
-							receivers.remove( receiver )
-						except Exception as err:
-							pass
-					_cleanupConnections(senderkey, signal)
+    """Remove receiver from connections."""
+    if not sendersBack:
+        # During module cleanup the mapping will be replaced with None
+        return False
+    backKey = id(receiver)
+    try:
+        backSet = sendersBack.pop(backKey)
+    except KeyError as err:
+        return False
+    else:
+        for senderkey in backSet:
+            try:
+                signals = connections[senderkey].keys()
+            except KeyError as err:
+                pass
+            else:
+                for signal in signals:
+                    try:
+                        receivers = connections[senderkey][signal]
+                    except KeyError:
+                        pass
+                    else:
+                        try:
+                            receivers.remove(receiver)
+                        except Exception as err:
+                            pass
+                    _cleanupConnections(senderkey, signal)
+
 
 def _cleanupConnections(senderkey, signal):
-	"""Delete any empty signals for senderkey. Delete senderkey if empty."""
-	try:
-		receivers = connections[senderkey][signal]
-	except:
-		pass
-	else:
-		if not receivers:
-			# No more connected receivers. Therefore, remove the signal.
-			try:
-				signals = connections[senderkey]
-			except KeyError:
-				pass
-			else:
-				del signals[signal]
-				if not signals:
-					# No more signal connections. Therefore, remove the sender.
-					_removeSender(senderkey)
+    """Delete any empty signals for senderkey. Delete senderkey if empty."""
+    try:
+        receivers = connections[senderkey][signal]
+    except:
+        pass
+    else:
+        if not receivers:
+            # No more connected receivers. Therefore, remove the signal.
+            try:
+                signals = connections[senderkey]
+            except KeyError:
+                pass
+            else:
+                del signals[signal]
+                if not signals:
+                    # No more signal connections. Therefore, remove the sender.
+                    _removeSender(senderkey)
+
 
 def _removeSender(senderkey):
-	"""Remove senderkey from connections."""
-	_removeBackrefs(senderkey)
-	try:
-		del connections[senderkey]
-	except KeyError:
-		pass
-	# Senderkey will only be in senders dictionary if sender 
-	# could be weakly referenced.
-	try: 
-		del senders[senderkey]
-	except: 
-		pass
+    """Remove senderkey from connections."""
+    _removeBackrefs(senderkey)
+    try:
+        del connections[senderkey]
+    except KeyError:
+        pass
+    # Senderkey will only be in senders dictionary if sender
+    # could be weakly referenced.
+    try:
+        del senders[senderkey]
+    except:
+        pass
 
 
-def _removeBackrefs( senderkey):
-	"""Remove all back-references to this senderkey"""
-	try:
-		signals = connections[senderkey]
-	except KeyError:
-		signals = None
-	else:
-		items = signals.items()
-		def allReceivers( ):
-			for signal,set in items:
-				for item in set:
-					yield item
-		for receiver in allReceivers():
-			_killBackref( receiver, senderkey )
+def _removeBackrefs(senderkey):
+    """Remove all back-references to this senderkey"""
+    try:
+        signals = connections[senderkey]
+    except KeyError:
+        signals = None
+    else:
+        items = signals.items()
+
+        def allReceivers():
+            for signal, set in items:
+                for item in set:
+                    yield item
+
+        for receiver in allReceivers():
+            _killBackref(receiver, senderkey)
+
 
 def _removeOldBackRefs(senderkey, signal, receiver, receivers):
-	"""Kill old sendersBack references from receiver
+    """Kill old sendersBack references from receiver
 
-	This guards against multiple registration of the same
-	receiver for a given signal and sender leaking memory
-	as old back reference records build up.
+    This guards against multiple registration of the same
+    receiver for a given signal and sender leaking memory
+    as old back reference records build up.
 
-	Also removes old receiver instance from receivers
-	"""
-	try:
-		index = receivers.index(receiver)
-		# need to scan back references here and remove senderkey
-	except ValueError:
-		return False
-	else:
-		oldReceiver = receivers[index]
-		del receivers[index]
-		found = 0
-		signals = connections.get(signal)
-		if signals is not None:
-			for sig, recs in six.iteritems(connections.get(signal,{})):
-				if sig != signal:
-					for rec in recs:
-						if rec is oldReceiver:
-							found = 1
-							break
-		if not found:
-			_killBackref( oldReceiver, senderkey )
-			return True
-		return False
-		
-		
-def _killBackref( receiver, senderkey ):
-	"""Do the actual removal of back reference from receiver to senderkey"""
-	receiverkey = id(receiver)
-	set = sendersBack.get( receiverkey, () )
-	while senderkey in set:
-		try:
-			set.remove( senderkey )
-		except:
-			break
-	if not set:
-		try:
-			del sendersBack[ receiverkey ]
-		except KeyError:
-			pass
-	return True
+    Also removes old receiver instance from receivers
+    """
+    try:
+        index = receivers.index(receiver)
+        # need to scan back references here and remove senderkey
+    except ValueError:
+        return False
+    else:
+        oldReceiver = receivers[index]
+        del receivers[index]
+        found = 0
+        signals = connections.get(signal)
+        if signals is not None:
+            for sig, recs in six.iteritems(connections.get(signal, {})):
+                if sig != signal:
+                    for rec in recs:
+                        if rec is oldReceiver:
+                            found = 1
+                            break
+        if not found:
+            _killBackref(oldReceiver, senderkey)
+            return True
+        return False
+
+
+def _killBackref(receiver, senderkey):
+    """Do the actual removal of back reference from receiver to senderkey"""
+    receiverkey = id(receiver)
+    set = sendersBack.get(receiverkey, ())
+    while senderkey in set:
+        try:
+            set.remove(senderkey)
+        except:
+            break
+    if not set:
+        try:
+            del sendersBack[receiverkey]
+        except KeyError:
+            pass
+    return True
diff --git a/scrapy/xlib/pydispatch/errors.py b/scrapy/xlib/pydispatch/errors.py
index a2eb32ed7..c5540d8f7 100644
--- a/scrapy/xlib/pydispatch/errors.py
+++ b/scrapy/xlib/pydispatch/errors.py
@@ -1,10 +1,15 @@
 """Error types for dispatcher mechanism
 """
 
-class DispatcherError(Exception):
-	"""Base class for all Dispatcher errors"""
-class DispatcherKeyError(KeyError, DispatcherError):
-	"""Error raised when unknown (sender,signal) set specified"""
-class DispatcherTypeError(TypeError, DispatcherError):
-	"""Error raised when inappropriate signal-type specified (None)"""
+
+class DispatcherError(Exception):
+    """Base class for all Dispatcher errors"""
+
+
+class DispatcherKeyError(KeyError, DispatcherError):
+    """Error raised when unknown (sender,signal) set specified"""
+
+
+class DispatcherTypeError(TypeError, DispatcherError):
+    """Error raised when inappropriate signal-type specified (None)"""
 
diff --git a/scrapy/xlib/pydispatch/robust.py b/scrapy/xlib/pydispatch/robust.py
index f829dbfd9..a28f127e2 100644
--- a/scrapy/xlib/pydispatch/robust.py
+++ b/scrapy/xlib/pydispatch/robust.py
@@ -2,56 +2,53 @@
 from scrapy.xlib.pydispatch.dispatcher import Any, Anonymous, liveReceivers, getAllReceivers
 from scrapy.xlib.pydispatch.robustapply import robustApply
 
-def sendRobust(
-	signal=Any, 
-	sender=Anonymous, 
-	*arguments, **named
-):
-	"""Send signal from sender to all connected receivers catching errors
-	
-	signal -- (hashable) signal value, see connect for details
 
-	sender -- the sender of the signal
-	
-		if Any, only receivers registered for Any will receive
-		the message.
+def sendRobust(signal=Any, sender=Anonymous, *arguments, **named):
+    """Send signal from sender to all connected receivers catching errors
 
-		if Anonymous, only receivers registered to receive
-		messages from Anonymous or Any will receive the message
+    signal -- (hashable) signal value, see connect for details
 
-		Otherwise can be any python object (normally one
-		registered with a connect if you actually want
-		something to occur).
+    sender -- the sender of the signal
 
-	arguments -- positional arguments which will be passed to
-		*all* receivers. Note that this may raise TypeErrors
-		if the receivers do not allow the particular arguments.
-		Note also that arguments are applied before named
-		arguments, so they should be used with care.
+        if Any, only receivers registered for Any will receive
+        the message.
 
-	named -- named arguments which will be filtered according
-		to the parameters of the receivers to only provide those
-		acceptable to the receiver.
+        if Anonymous, only receivers registered to receive
+        messages from Anonymous or Any will receive the message
 
-	Return a list of tuple pairs [(receiver, response), ... ]
+        Otherwise can be any python object (normally one
+        registered with a connect if you actually want
+        something to occur).
 
-	if any receiver raises an error (specifically any subclass of Exception),
-	the error instance is returned as the result for that receiver.
-	"""
-	# Call each receiver with whatever arguments it can accept.
-	# Return a list of tuple pairs [(receiver, response), ... ].
-	responses = []
-	for receiver in liveReceivers(getAllReceivers(sender, signal)):
-		try:
-			response = robustApply(
-				receiver,
-				signal=signal,
-				sender=sender,
-				*arguments,
-				**named
-			)
-		except Exception as err:
-			responses.append((receiver, err))
-		else:
-			responses.append((receiver, response))
-	return responses
+    arguments -- positional arguments which will be passed to
+        *all* receivers. Note that this may raise TypeErrors
+        if the receivers do not allow the particular arguments.
+        Note also that arguments are applied before named
+        arguments, so they should be used with care.
+
+    named -- named arguments which will be filtered according
+        to the parameters of the receivers to only provide those
+        acceptable to the receiver.
+
+    Return a list of tuple pairs [(receiver, response), ... ]
+
+    if any receiver raises an error (specifically any subclass of Exception),
+    the error instance is returned as the result for that receiver.
+    """
+    # Call each receiver with whatever arguments it can accept.
+    # Return a list of tuple pairs [(receiver, response), ... ].
+    responses = []
+    for receiver in liveReceivers(getAllReceivers(sender, signal)):
+        try:
+            response = robustApply(
+                receiver,
+                signal=signal,
+                sender=sender,
+                *arguments,
+                **named
+            )
+        except Exception as err:
+            responses.append((receiver, err))
+        else:
+            responses.append((receiver, response))
+    return responses
diff --git a/scrapy/xlib/pydispatch/robustapply.py b/scrapy/xlib/pydispatch/robustapply.py
index 5deda3c42..9977ac6b7 100644
--- a/scrapy/xlib/pydispatch/robustapply.py
+++ b/scrapy/xlib/pydispatch/robustapply.py
@@ -5,9 +5,9 @@ what arguments a given callable object can take,
 and subset the given arguments to match only
 those which are acceptable.
 """
-
 import inspect
 
+
 def function(receiver):
     """Get function-like callable object for given receiver
 
@@ -20,32 +20,35 @@ def function(receiver):
         # receiver is a class instance; assume it is callable.
         # Reassign receiver to the actual method that will be called.
         if hasattr(receiver.__call__, 'im_func') or \
-           hasattr(receiver.__call__, 'im_code'):
+                hasattr(receiver.__call__, 'im_code'):
             receiver = receiver.__call__
 
-    if hasattr( receiver, 'im_func' ):
+    if hasattr(receiver, 'im_func'):
         # an instance-method...
         return receiver, receiver.im_func.func_code, 1
     elif not hasattr(receiver, 'func_code'):
-        raise ValueError('unknown receiver type %s %s'%(receiver, type(receiver)))
+        raise ValueError(
+            'unknown receiver type %s %s' % (receiver, type(receiver)))
 
     return receiver, receiver.func_code, 0
 
+
 def robustApply(receiver, *arguments, **named):
     """Call receiver with arguments and an appropriate subset of named
     """
     receiver, codeObject, startIndex = function(receiver)
-    acceptable = codeObject.co_varnames[startIndex+len(arguments):codeObject.co_argcount]
-    for name in codeObject.co_varnames[startIndex:startIndex+len(arguments)]:
+    acceptable = codeObject.co_varnames[
+                 startIndex + len(arguments):codeObject.co_argcount]
+    for name in codeObject.co_varnames[startIndex:startIndex + len(arguments)]:
         if name in named:
             raise TypeError(
-                """Argument %r specified both positionally and as a keyword for calling %r"""% (
+                """Argument %r specified both positionally and as a keyword for calling %r""" % (
                     name, receiver,
                 )
             )
 
     if not (codeObject.co_flags & 8):
-        # fc does not have a **kwds type parameter, therefore 
+        # fc does not have a **kwds type parameter, therefore
         # remove unacceptable arguments.
         for arg in named.keys():
             if arg not in acceptable:
diff --git a/scrapy/xlib/pydispatch/saferef.py b/scrapy/xlib/pydispatch/saferef.py
index f1b8b1f9b..bd9659673 100644
--- a/scrapy/xlib/pydispatch/saferef.py
+++ b/scrapy/xlib/pydispatch/saferef.py
@@ -2,165 +2,179 @@
 from __future__ import print_function
 import weakref, traceback
 
-def safeRef(target, onDelete = None):
-	"""Return a *safe* weak reference to a callable target
 
-	target -- the object to be weakly referenced, if it's a
-		bound method reference, will create a BoundMethodWeakref,
-		otherwise creates a simple weakref.
-	onDelete -- if provided, will have a hard reference stored
-		to the callable to be called after the safe reference
-		goes out of scope with the reference object, (either a
-		weakref or a BoundMethodWeakref) as argument.
-	"""
-	if hasattr(target, 'im_self'):
-		if target.im_self is not None:
-			# Turn a bound method into a BoundMethodWeakref instance.
-			# Keep track of these instances for lookup by disconnect().
-			assert hasattr(target, 'im_func'), """safeRef target %r has im_self, but no im_func, don't know how to create reference"""%( target,)
-			reference = BoundMethodWeakref(
-				target=target,
-				onDelete=onDelete
-			)
-			return reference
-	if onDelete is not None:
-		return weakref.ref(target, onDelete)
-	else:
-		return weakref.ref( target )
+def safeRef(target, onDelete=None):
+    """Return a *safe* weak reference to a callable target
+
+    target -- the object to be weakly referenced, if it's a
+        bound method reference, will create a BoundMethodWeakref,
+        otherwise creates a simple weakref.
+    onDelete -- if provided, will have a hard reference stored
+        to the callable to be called after the safe reference
+        goes out of scope with the reference object, (either a
+        weakref or a BoundMethodWeakref) as argument.
+    """
+    if hasattr(target, 'im_self'):
+        if target.im_self is not None:
+            # Turn a bound method into a BoundMethodWeakref instance.
+            # Keep track of these instances for lookup by disconnect().
+            assert hasattr(target, 'im_func'), """safeRef target %r has im_self, but no im_func, don't know how to create reference"""%( target,)
+            reference = BoundMethodWeakref(
+                target=target,
+                onDelete=onDelete
+            )
+            return reference
+    if onDelete is not None:
+        return weakref.ref(target, onDelete)
+    else:
+        return weakref.ref(target)
+
 
 class BoundMethodWeakref(object):
-	"""'Safe' and reusable weak references to instance methods
+    """'Safe' and reusable weak references to instance methods
 
-	BoundMethodWeakref objects provide a mechanism for
-	referencing a bound method without requiring that the
-	method object itself (which is normally a transient
-	object) is kept alive.  Instead, the BoundMethodWeakref
-	object keeps weak references to both the object and the
-	function which together define the instance method.
+    BoundMethodWeakref objects provide a mechanism for
+    referencing a bound method without requiring that the
+    method object itself (which is normally a transient
+    object) is kept alive.  Instead, the BoundMethodWeakref
+    object keeps weak references to both the object and the
+    function which together define the instance method.
 
-	Attributes:
-		key -- the identity key for the reference, calculated
-			by the class's calculateKey method applied to the
-			target instance method
-		deletionMethods -- sequence of callable objects taking
-			single argument, a reference to this object which
-			will be called when *either* the target object or
-			target function is garbage collected (i.e. when
-			this object becomes invalid).  These are specified
-			as the onDelete parameters of safeRef calls.
-		weakSelf -- weak reference to the target object
-		weakFunc -- weak reference to the target function
+    Attributes:
+        key -- the identity key for the reference, calculated
+            by the class's calculateKey method applied to the
+            target instance method
+        deletionMethods -- sequence of callable objects taking
+            single argument, a reference to this object which
+            will be called when *either* the target object or
+            target function is garbage collected (i.e. when
+            this object becomes invalid).  These are specified
+            as the onDelete parameters of safeRef calls.
+        weakSelf -- weak reference to the target object
+        weakFunc -- weak reference to the target function
 
-	Class Attributes:
-		_allInstances -- class attribute pointing to all live
-			BoundMethodWeakref objects indexed by the class's
-			calculateKey(target) method applied to the target
-			objects.  This weak value dictionary is used to
-			short-circuit creation so that multiple references
-			to the same (object, function) pair produce the
-			same BoundMethodWeakref instance.
+    Class Attributes:
+        _allInstances -- class attribute pointing to all live
+            BoundMethodWeakref objects indexed by the class's
+            calculateKey(target) method applied to the target
+            objects.  This weak value dictionary is used to
+            short-circuit creation so that multiple references
+            to the same (object, function) pair produce the
+            same BoundMethodWeakref instance.
 
-	"""
-	_allInstances = weakref.WeakValueDictionary()
-	def __new__( cls, target, onDelete=None, *arguments,**named ):
-		"""Create new instance or return current instance
+    """
+    _allInstances = weakref.WeakValueDictionary()
 
-		Basically this method of construction allows us to
-		short-circuit creation of references to already-
-		referenced instance methods.  The key corresponding
-		to the target is calculated, and if there is already
-		an existing reference, that is returned, with its
-		deletionMethods attribute updated.  Otherwise the
-		new instance is created and registered in the table
-		of already-referenced methods.
-		"""
-		key = cls.calculateKey(target)
-		current =cls._allInstances.get(key)
-		if current is not None:
-			current.deletionMethods.append( onDelete)
-			return current
-		else:
-			base = super( BoundMethodWeakref, cls).__new__( cls )
-			cls._allInstances[key] = base
-			base.__init__( target, onDelete, *arguments,**named)
-			return base
-	def __init__(self, target, onDelete=None):
-		"""Return a weak-reference-like instance for a bound method
+    def __new__(cls, target, onDelete=None, *arguments, **named):
+        """Create new instance or return current instance
 
-		target -- the instance-method target for the weak
-			reference, must have im_self and im_func attributes
-			and be reconstructable via:
-				target.im_func.__get__( target.im_self )
-			which is true of built-in instance methods.
-		onDelete -- optional callback which will be called
-			when this weak reference ceases to be valid
-			(i.e. either the object or the function is garbage
-			collected).  Should take a single argument,
-			which will be passed a pointer to this object.
-		"""
-		def remove(weak, self=self):
-			"""Set self.isDead to true when method or instance is destroyed"""
-			methods = self.deletionMethods[:]
-			del self.deletionMethods[:]
-			try:
-				del self.__class__._allInstances[ self.key ]
-			except KeyError:
-				pass
-			for function in methods:
-				try:
-					if callable( function ):
-						function( self )
-				except Exception as e:
-					try:
-						traceback.print_exc()
-					except AttributeError as err:
-						print('''Exception during saferef %s cleanup function %s: %s'''%(
-							self, function, e
-						))
-		self.deletionMethods = [onDelete]
-		self.key = self.calculateKey( target )
-		self.weakSelf = weakref.ref(target.im_self, remove)
-		self.weakFunc = weakref.ref(target.im_func, remove)
-		self.selfName = target.im_self.__class__.__name__
-		self.funcName = str(target.im_func.__name__)
-	def calculateKey( cls, target ):
-		"""Calculate the reference key for this reference
+        Basically this method of construction allows us to
+        short-circuit creation of references to already-
+        referenced instance methods.  The key corresponding
+        to the target is calculated, and if there is already
+        an existing reference, that is returned, with its
+        deletionMethods attribute updated.  Otherwise the
+        new instance is created and registered in the table
+        of already-referenced methods.
+        """
+        key = cls.calculateKey(target)
+        current = cls._allInstances.get(key)
+        if current is not None:
+            current.deletionMethods.append(onDelete)
+            return current
+        else:
+            base = super(BoundMethodWeakref, cls).__new__(cls)
+            cls._allInstances[key] = base
+            base.__init__(target, onDelete, *arguments, **named)
+            return base
 
-		Currently this is a two-tuple of the id()'s of the
-		target object and the target function respectively.
-		"""
-		return (id(target.im_self),id(target.im_func))
-	calculateKey = classmethod( calculateKey )
-	def __str__(self):
-		"""Give a friendly representation of the object"""
-		return """%s( %s.%s )"""%(
-			self.__class__.__name__,
-			self.selfName,
-			self.funcName,
-		)
-	__repr__ = __str__
-	def __nonzero__( self ):
-		"""Whether we are still a valid reference"""
-		return self() is not None
-	def __cmp__( self, other ):
-		"""Compare with another reference"""
-		if not isinstance (other,self.__class__):
-			return cmp( self.__class__, type(other) )
-		return cmp( self.key, other.key)
-	def __call__(self):
-		"""Return a strong reference to the bound method
+    def __init__(self, target, onDelete=None):
+        """Return a weak-reference-like instance for a bound method
 
-		If the target cannot be retrieved, then will
-		return None, otherwise returns a bound instance
-		method for our object and function.
+        target -- the instance-method target for the weak
+            reference, must have im_self and im_func attributes
+            and be reconstructable via:
+                target.im_func.__get__( target.im_self )
+            which is true of built-in instance methods.
+        onDelete -- optional callback which will be called
+            when this weak reference ceases to be valid
+            (i.e. either the object or the function is garbage
+            collected).  Should take a single argument,
+            which will be passed a pointer to this object.
+        """
 
-		Note:
-			You may call this method any number of times,
-			as it does not invalidate the reference.
-		"""
-		target = self.weakSelf()
-		if target is not None:
-			function = self.weakFunc()
-			if function is not None:
-				return function.__get__(target)
-		return None
+        def remove(weak, self=self):
+            """Set self.isDead to true when method or instance is destroyed"""
+            methods = self.deletionMethods[:]
+            del self.deletionMethods[:]
+            try:
+                del self.__class__._allInstances[self.key]
+            except KeyError:
+                pass
+            for function in methods:
+                try:
+                    if callable(function):
+                        function(self)
+                except Exception as e:
+                    try:
+                        traceback.print_exc()
+                    except AttributeError as err:
+                        print(
+                            '''Exception during saferef %s cleanup function %s: %s''' % (
+                                self, function, e
+                            ))
+
+        self.deletionMethods = [onDelete]
+        self.key = self.calculateKey(target)
+        self.weakSelf = weakref.ref(target.im_self, remove)
+        self.weakFunc = weakref.ref(target.im_func, remove)
+        self.selfName = target.im_self.__class__.__name__
+        self.funcName = str(target.im_func.__name__)
+
+    def calculateKey(cls, target):
+        """Calculate the reference key for this reference
+
+        Currently this is a two-tuple of the id()'s of the
+        target object and the target function respectively.
+        """
+        return (id(target.im_self), id(target.im_func))
+
+    calculateKey = classmethod(calculateKey)
+
+    def __str__(self):
+        """Give a friendly representation of the object"""
+        return """%s( %s.%s )""" % (
+            self.__class__.__name__,
+            self.selfName,
+            self.funcName,
+        )
+
+    __repr__ = __str__
+
+    def __nonzero__(self):
+        """Whether we are still a valid reference"""
+        return self() is not None
+
+    def __cmp__(self, other):
+        """Compare with another reference"""
+        if not isinstance(other, self.__class__):
+            return cmp(self.__class__, type(other))
+        return cmp(self.key, other.key)
+
+    def __call__(self):
+        """Return a strong reference to the bound method
+
+        If the target cannot be retrieved, then will
+        return None, otherwise returns a bound instance
+        method for our object and function.
+
+        Note:
+            You may call this method any number of times,
+            as it does not invalidate the reference.
+        """
+        target = self.weakSelf()
+        if target is not None:
+            function = self.weakFunc()
+            if function is not None:
+                return function.__get__(target)
+        return None

From fd67fe273aa68acbc290d2f067230516bcebcade Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 17 Mar 2015 09:34:54 -0300
Subject: [PATCH 0109/2974] using default values for settings that are off by
 default

---
 scrapy/templates/project/module/settings.py.tmpl | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index f4832dd27..302d96b17 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -69,16 +69,17 @@ NEWSPIDER_MODULE = '$project_name.spiders'
 # See http://doc.scrapy.org/en/latest/topics/autothrottle.html
 # NOTE: AutoThrottle will honour the standard settings for concurrency and delay
 #AUTOTHROTTLE_ENABLED=True
-# The initial download delay (default: 5)
-#AUTOTHROTTLE_START_DELAY=3
-# The maximum download delay to be set in case of high latencies (default: 60)
-#AUTOTHROTTLE_MAX_DELAY=90
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY=5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY=60
 # Enable showing throttling stats for every response received:
-#AUTOTHROTTLE_DEBUG=True
+#AUTOTHROTTLE_DEBUG=False
 
 # Enable and configure HTTP caching (disabled by default)
 # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
 #HTTPCACHE_ENABLED=True
-#HTTPCACHE_EXPIRATION_SECS=3600
+#HTTPCACHE_EXPIRATION_SECS=0
 #HTTPCACHE_DIR='httpcache'
-#HTTPCACHE_IGNORE_HTTP_CODES=[404]
+#HTTPCACHE_IGNORE_HTTP_CODES=[]
+#HTTPCACHE_STORAGE='scrapy.contrib.httpcache.FilesystemCacheStorage'

From 8ce4ad06151273fe67c28a95ec59269439d95025 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 17 Mar 2015 23:07:39 +0500
Subject: [PATCH 0110/2974] remove unnecessary check from
 scrapy.utils.spider.iter_spider_output

arg_to_iter handles Items since https://github.com/scrapy/scrapy/commit/2bbd92742b796e1a565d4914a77889c884dd01ac
---
 scrapy/utils/spider.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index b81cf2b9b..40ebbefc3 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -3,13 +3,13 @@ import inspect
 import six
 
 from scrapy import log
-from scrapy.item import BaseItem
 from scrapy.spider import Spider
 from scrapy.utils.misc import  arg_to_iter
 
 
 def iterate_spider_output(result):
-    return [result] if isinstance(result, BaseItem) else arg_to_iter(result)
+    return arg_to_iter(result)
+
 
 def iter_spider_classes(module):
     """Return an iterator over all spider classes defined in the given module

From 6974902323f4b4cecbe663e6e0ff4c67c0557029 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 17 Mar 2015 23:31:29 +0500
Subject: [PATCH 0111/2974] CSVFeedSpider cleanup: use iterate_spider_output

A similar fix was made for XMLFeedSpider in https://github.com/scrapy/scrapy/commit/95fde0a4987acaa75a6749223c8b7f9bd7081c23
---
 scrapy/contrib/spiders/feed.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py
index 1a95c5c35..d83ee605e 100644
--- a/scrapy/contrib/spiders/feed.py
+++ b/scrapy/contrib/spiders/feed.py
@@ -5,8 +5,6 @@ for scraping from an XML feed.
 See documentation in docs/topics/spiders.rst
 """
 from scrapy.spider import Spider
-from scrapy.item import BaseItem
-from scrapy.http import Request
 from scrapy.utils.iterators import xmliter, csviter
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.selector import Selector
@@ -92,6 +90,7 @@ class XMLFeedSpider(Spider):
         for (prefix, uri) in self.namespaces:
             selector.register_namespace(prefix, uri)
 
+
 class CSVFeedSpider(Spider):
     """Spider for parsing CSV feeds.
     It receives a CSV file in a response; iterates through each of its rows,
@@ -125,11 +124,7 @@ class CSVFeedSpider(Spider):
         """
 
         for row in csviter(response, self.delimiter, self.headers, self.quotechar):
-            ret = self.parse_row(response, row)
-            if isinstance(ret, (BaseItem, Request)):
-                ret = [ret]
-            if not isinstance(ret, (list, tuple)):
-                raise TypeError('You cannot return an "%s" object from a spider' % type(ret).__name__)
+            ret = iterate_spider_output(self.parse_row(response, row))
             for result_item in self.process_results(response, ret):
                 yield result_item
 

From da90449edfa13b5be1550b3acc212dbf3a8c6e69 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 18 Mar 2015 07:24:15 +0500
Subject: [PATCH 0112/2974] typo fix in scrapy.contrib.pipeline.media

---
 scrapy/contrib/pipeline/media.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py
index 82270e15d..012b7979a 100644
--- a/scrapy/contrib/pipeline/media.py
+++ b/scrapy/contrib/pipeline/media.py
@@ -117,7 +117,7 @@ class MediaPipeline(object):
     def item_completed(self, results, item, info):
         """Called per item when all media requests has been processed"""
         if self.LOG_FAILED_RESULTS:
-            msg = '%s found errors proessing %s' % (self.__class__.__name__, item)
+            msg = '%s found errors processing %s' % (self.__class__.__name__, item)
             for ok, value in results:
                 if not ok:
                     log.err(value, msg, spider=info.spider)

From 39635e5f55bfbbc88d051778db6ba6f391630619 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 18 Mar 2015 07:26:56 +0500
Subject: [PATCH 0113/2974] Allow spiders to return dicts. See GH-1064.

---
 docs/topics/exporters.rst           |   5 +
 scrapy/commands/parse.py            |   2 +-
 scrapy/contracts/default.py         |   6 +-
 scrapy/contrib/exporter/__init__.py |  21 ++-
 scrapy/contrib/pipeline/files.py    |   2 +-
 scrapy/contrib/pipeline/images.py   |   2 +-
 scrapy/core/scraper.py              |   4 +-
 tests/spiders.py                    |   1 +
 tests/test_commands.py              |  20 ++-
 tests/test_contracts.py             |  51 ++++++
 tests/test_contrib_exporter.py      | 243 +++++++++++++++++-----------
 tests/test_engine.py                |  34 ++--
 tests/test_pipeline_files.py        |  43 ++---
 tests/test_pipeline_images.py       |  43 ++---
 14 files changed, 310 insertions(+), 167 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index c472f5b96..43931544f 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -197,12 +197,17 @@ BaseItemExporter
       Some exporters (like :class:`CsvItemExporter`) respect the order of the
       fields defined in this attribute.
 
+      Some exporters may require fields_to_export list in order to export the
+      data properly when spiders return dicts (not :class:`~Item` instances).
+
    .. attribute:: export_empty_fields
 
       Whether to include empty/unpopulated item fields in the exported data.
       Defaults to ``False``. Some exporters (like :class:`CsvItemExporter`)
       ignore this attribute and always export all empty fields.
 
+      This option is ignored for dict items.
+
    .. attribute:: encoding
 
       The encoding that will be used to encode unicode values. This only
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 01c7fff0a..b8cc140d4 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -107,7 +107,7 @@ class Command(ScrapyCommand):
         items, requests = [], []
 
         for x in iterate_spider_output(cb(response)):
-            if isinstance(x, BaseItem):
+            if isinstance(x, (BaseItem, dict)):
                 items.append(x)
             elif isinstance(x, Request):
                 requests.append(x)
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 1d8367f82..20582503d 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -35,8 +35,8 @@ class ReturnsContract(Contract):
     objects = {
         'request': Request,
         'requests': Request,
-        'item': BaseItem,
-        'items': BaseItem,
+        'item': (BaseItem, dict),
+        'items': (BaseItem, dict),
     }
 
     def __init__(self, *args, **kwargs):
@@ -83,7 +83,7 @@ class ScrapesContract(Contract):
 
     def post_process(self, output):
         for x in output:
-            if isinstance(x, BaseItem):
+            if isinstance(x, (BaseItem, dict)):
                 for arg in self.args:
                     if not arg in x:
                         raise ContractFail("'%s' field is missing" % arg)
diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py
index cc88f8792..7e1d01a0a 100644
--- a/scrapy/contrib/exporter/__init__.py
+++ b/scrapy/contrib/exporter/__init__.py
@@ -9,6 +9,7 @@ import marshal
 import six
 from six.moves import cPickle as pickle
 from xml.sax.saxutils import XMLGenerator
+
 from scrapy.utils.serialize import ScrapyJSONEncoder
 from scrapy.item import BaseItem
 
@@ -50,13 +51,13 @@ class BaseItemExporter(object):
         return value.encode(self.encoding) if isinstance(value, unicode) else value
 
     def _get_serialized_fields(self, item, default_value=None, include_empty=None):
-        """Return the fields to export as an iterable of tuples (name,
-        serialized_value)
+        """Return the fields to export as an iterable of tuples
+        (name, serialized_value)
         """
         if include_empty is None:
             include_empty = self.export_empty_fields
         if self.fields_to_export is None:
-            if include_empty:
+            if include_empty and not isinstance(item, dict):
                 field_iter = six.iterkeys(item.fields)
             else:
                 field_iter = six.iterkeys(item)
@@ -64,12 +65,11 @@ class BaseItemExporter(object):
             if include_empty:
                 field_iter = self.fields_to_export
             else:
-                nonempty_fields = set(item.keys())
-                field_iter = (x for x in self.fields_to_export if x in
-                              nonempty_fields)
+                field_iter = (x for x in self.fields_to_export if x in item)
+
         for field_name in field_iter:
             if field_name in item:
-                field = item.fields[field_name]
+                field = {} if isinstance(item, dict) else item.fields[field_name]
                 value = self.serialize_field(field, field_name, item[field_name])
             else:
                 value = default_value
@@ -191,7 +191,12 @@ class CsvItemExporter(BaseItemExporter):
     def _write_headers_and_set_fields_to_export(self, item):
         if self.include_headers_line:
             if not self.fields_to_export:
-                self.fields_to_export = item.fields.keys()
+                if isinstance(item, dict):
+                    # for dicts try using fields of the first item
+                    self.fields_to_export = list(item.keys())
+                else:
+                    # use fields declared in Item
+                    self.fields_to_export = list(item.fields.keys())
             self.csv_writer.writerow(self.fields_to_export)
 
 
diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py
index db8cf8b76..9e803aca0 100644
--- a/scrapy/contrib/pipeline/files.py
+++ b/scrapy/contrib/pipeline/files.py
@@ -267,7 +267,7 @@ class FilesPipeline(MediaPipeline):
         return checksum
 
     def item_completed(self, results, item, info):
-        if self.FILES_RESULT_FIELD in item.fields:
+        if isinstance(item, dict) or self.FILES_RESULT_FIELD in item.fields:
             item[self.FILES_RESULT_FIELD] = [x for ok, x in results if ok]
         return item
 
diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py
index 9c1a54455..b12995f09 100644
--- a/scrapy/contrib/pipeline/images.py
+++ b/scrapy/contrib/pipeline/images.py
@@ -109,7 +109,7 @@ class ImagesPipeline(FilesPipeline):
         return [Request(x) for x in item.get(self.IMAGES_URLS_FIELD, [])]
 
     def item_completed(self, results, item, info):
-        if self.IMAGES_RESULT_FIELD in item.fields:
+        if isinstance(item, dict) or self.IMAGES_RESULT_FIELD in item.fields:
             item[self.IMAGES_RESULT_FIELD] = [x for ok, x in results if ok]
         return item
 
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 3409a0e7c..b301aa962 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -174,7 +174,7 @@ class Scraper(object):
         """
         if isinstance(output, Request):
             self.crawler.engine.crawl(request=output, spider=spider)
-        elif isinstance(output, BaseItem):
+        elif isinstance(output, (BaseItem, dict)):
             self.slot.itemproc_size += 1
             dfd = self.itemproc.process_item(output, spider)
             dfd.addBoth(self._itemproc_finished, output, response, spider)
@@ -183,7 +183,7 @@ class Scraper(object):
             pass
         else:
             typename = type(output).__name__
-            log.msg(format='Spider must return Request, BaseItem or None, '
+            log.msg(format='Spider must return Request, BaseItem, dict or None, '
                            'got %(typename)r in %(request)s',
                     level=log.ERROR, spider=spider, request=request, typename=typename)
 
diff --git a/tests/spiders.py b/tests/spiders.py
index 83d767f5c..86ace9d6e 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -85,6 +85,7 @@ class ItemSpider(FollowAllSpider):
         for request in super(ItemSpider, self).parse(response):
             yield request
             yield Item()
+            yield {}
 
 
 class DefaultError(Exception):
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 70b4e74dc..eb3556b62 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -127,6 +127,7 @@ class MiscCommandsTest(CommandTest):
     def test_list(self):
         self.assertEqual(0, self.call('list'))
 
+
 class RunSpiderCommandTest(CommandTest):
 
     def test_runspider(self):
@@ -135,10 +136,10 @@ class RunSpiderCommandTest(CommandTest):
         fname = abspath(join(tmpdir, 'myspider.py'))
         with open(fname, 'w') as f:
             f.write("""
+import scrapy
 from scrapy import log
-from scrapy.spider import Spider
 
-class MySpider(Spider):
+class MySpider(scrapy.Spider):
     name = 'myspider'
 
     def start_requests(self):
@@ -192,16 +193,15 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
         with open(fname, 'w') as f:
             f.write("""
 from scrapy import log
-from scrapy.spider import Spider
-from scrapy.item import Item
+import scrapy
 
-class MySpider(Spider):
+class MySpider(scrapy.Spider):
     name = '{0}'
 
     def parse(self, response):
         if getattr(self, 'test_arg', None):
             self.log('It Works!')
-        return [Item()]
+        return [scrapy.Item(), dict(foo='bar')]
 """.format(self.spider_name))
 
         fname = abspath(join(self.proj_mod_path, 'pipelines.py'))
@@ -239,6 +239,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
                                            self.url('/html')])
         self.assert_("[scrapy] INFO: It Works!" in stderr, stderr)
 
+    @defer.inlineCallbacks
+    def test_parse_items(self):
+        status, out, stderr = yield self.execute(
+            ['--spider', self.spider_name, '-c', 'parse', self.url('/html')]
+        )
+        self.assertIn("""[{}, {'foo': 'bar'}]""", out)
+
+
 
 class BenchCommandTest(CommandTest):
 
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index a651576a5..d7732f55d 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -39,6 +39,13 @@ class TestSpider(Spider):
         """
         return TestItem(url=response.url)
 
+    def returns_dict_item(self, response):
+        """ method which returns item
+        @url http://scrapy.org
+        @returns items 1 1
+        """
+        return {"url": response.url}
+
     def returns_fail(self, response):
         """ method which returns item
         @url http://scrapy.org
@@ -46,6 +53,13 @@ class TestSpider(Spider):
         """
         return TestItem(url=response.url)
 
+    def returns_dict_fail(self, response):
+        """ method which returns item
+        @url http://scrapy.org
+        @returns items 0 0
+        """
+        return {'url': response.url}
+
     def scrapes_item_ok(self, response):
         """ returns item with name and url
         @url http://scrapy.org
@@ -54,6 +68,14 @@ class TestSpider(Spider):
         """
         return TestItem(name='test', url=response.url)
 
+    def scrapes_dict_item_ok(self, response):
+        """ returns item with name and url
+        @url http://scrapy.org
+        @returns items 1 1
+        @scrapes name url
+        """
+        return {'name': 'test', 'url': response.url}
+
     def scrapes_item_fail(self, response):
         """ returns item with no name
         @url http://scrapy.org
@@ -62,6 +84,14 @@ class TestSpider(Spider):
         """
         return TestItem(url=response.url)
 
+    def scrapes_dict_item_fail(self, response):
+        """ returns item with no name
+        @url http://scrapy.org
+        @returns items 1 1
+        @scrapes name url
+        """
+        return {'url': response.url}
+
     def parse_no_url(self, response):
         """ method with no url
         @returns items 1 1
@@ -110,6 +140,11 @@ class ContractsManagerTest(unittest.TestCase):
         request.callback(response)
         self.should_succeed()
 
+        # returns_dict_item
+        request = self.conman.from_method(spider.returns_dict_item, self.results)
+        request.callback(response)
+        self.should_succeed()
+
         # returns_request
         request = self.conman.from_method(spider.returns_request, self.results)
         request.callback(response)
@@ -120,6 +155,11 @@ class ContractsManagerTest(unittest.TestCase):
         request.callback(response)
         self.should_fail()
 
+        # returns_dict_fail
+        request = self.conman.from_method(spider.returns_dict_fail, self.results)
+        request.callback(response)
+        self.should_fail()
+
     def test_scrapes(self):
         spider = TestSpider()
         response = ResponseMock()
@@ -129,8 +169,19 @@ class ContractsManagerTest(unittest.TestCase):
         request.callback(response)
         self.should_succeed()
 
+        # scrapes_dict_item_ok
+        request = self.conman.from_method(spider.scrapes_dict_item_ok, self.results)
+        request.callback(response)
+        self.should_succeed()
+
         # scrapes_item_fail
         request = self.conman.from_method(spider.scrapes_item_fail,
                 self.results)
         request.callback(response)
         self.should_fail()
+
+        # scrapes_dict_item_fail
+        request = self.conman.from_method(spider.scrapes_dict_item_fail,
+                self.results)
+        request.callback(response)
+        self.should_fail()
diff --git a/tests/test_contrib_exporter.py b/tests/test_contrib_exporter.py
index 9092007e5..746aeb65b 100644
--- a/tests/test_contrib_exporter.py
+++ b/tests/test_contrib_exporter.py
@@ -1,14 +1,19 @@
-import unittest, json
+from __future__ import absolute_import
+import re
+import json
+import unittest
 from io import BytesIO
 from six.moves import cPickle as pickle
+
 import lxml.etree
-import re
 
 from scrapy.item import Item, Field
 from scrapy.utils.python import str_to_unicode
-from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \
-    PickleItemExporter, CsvItemExporter, XmlItemExporter, JsonLinesItemExporter, \
-    JsonItemExporter, PythonItemExporter
+from scrapy.contrib.exporter import (
+    BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter,
+    XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, PythonItemExporter
+)
+
 
 class TestItem(Item):
     name = Field()
@@ -33,21 +38,28 @@ class BaseItemExporterTest(unittest.TestCase):
             exported_dict[k] = str_to_unicode(v)
         self.assertEqual(self.i, exported_dict)
 
-    def test_export_item(self):
+    def assertItemExportWorks(self, item):
         self.ie.start_exporting()
         try:
-            self.ie.export_item(self.i)
+            self.ie.export_item(item)
         except NotImplementedError:
             if self.ie.__class__ is not BaseItemExporter:
                 raise
         self.ie.finish_exporting()
         self._check_output()
 
+    def test_export_item(self):
+        self.assertItemExportWorks(self.i)
+
+    def test_export_dict_item(self):
+        self.assertItemExportWorks(dict(self.i))
+
     def test_serialize_field(self):
-        self.assertEqual(self.ie.serialize_field( \
-            self.i.fields['name'], 'name', self.i['name']), 'John\xc2\xa3')
-        self.assertEqual( \
-            self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age']), '22')
+        res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name'])
+        self.assertEqual(res, 'John\xc2\xa3')
+
+        res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age'])
+        self.assertEqual(res, '22')
 
     def test_fields_to_export(self):
         ie = self._get_exporter(fields_to_export=['name'])
@@ -72,13 +84,14 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John\xc2\xa3')
         self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
 
+
 class PythonItemExporterTest(BaseItemExporterTest):
     def _get_exporter(self, **kwargs):
         return PythonItemExporter(**kwargs)
 
     def test_nested_item(self):
         i1 = TestItem(name=u'Joseph', age='22')
-        i2 = TestItem(name=u'Maria', age=i1)
+        i2 = dict(name=u'Maria', age=i1)
         i3 = TestItem(name=u'Jesus', age=i2)
         ie = self._get_exporter()
         exported = ie.export_item(i3)
@@ -107,6 +120,7 @@ class PythonItemExporterTest(BaseItemExporterTest):
         self.assertEqual(type(exported['age'][0]), dict)
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
+
 class PprintItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
@@ -115,6 +129,7 @@ class PprintItemExporterTest(BaseItemExporterTest):
     def _check_output(self):
         self._assert_expected_item(eval(self.output.getvalue()))
 
+
 class PickleItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
@@ -150,48 +165,65 @@ class CsvItemExporterTest(BaseItemExporterTest):
     def _check_output(self):
         self.assertCsvEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n')
 
-    def test_header(self):
-        output = BytesIO()
-        ie = CsvItemExporter(output, fields_to_export=self.i.fields.keys())
+    def assertExportResult(self, item, expected, **kwargs):
+        fp = BytesIO()
+        ie = CsvItemExporter(fp, **kwargs)
         ie.start_exporting()
-        ie.export_item(self.i)
+        ie.export_item(item)
         ie.finish_exporting()
-        self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n')
+        self.assertCsvEqual(fp.getvalue(), expected)
 
-        output = BytesIO()
-        ie = CsvItemExporter(output, fields_to_export=['age'])
-        ie.start_exporting()
-        ie.export_item(self.i)
-        ie.finish_exporting()
-        self.assertCsvEqual(output.getvalue(), 'age\r\n22\r\n')
+    def test_header_export_all(self):
+        self.assertExportResult(
+            item=self.i,
+            fields_to_export=self.i.fields.keys(),
+            expected='age,name\r\n22,John\xc2\xa3\r\n',
+        )
 
-        output = BytesIO()
-        ie = CsvItemExporter(output)
-        ie.start_exporting()
-        ie.export_item(self.i)
-        ie.export_item(self.i)
-        ie.finish_exporting()
-        self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
+    def test_header_export_all_dict(self):
+        self.assertExportResult(
+            item=dict(self.i),
+            expected='age,name\r\n22,John\xc2\xa3\r\n',
+        )
 
-        output = BytesIO()
-        ie = CsvItemExporter(output, include_headers_line=False)
-        ie.start_exporting()
-        ie.export_item(self.i)
-        ie.finish_exporting()
-        self.assertCsvEqual(output.getvalue(), '22,John\xc2\xa3\r\n')
+    def test_header_export_single_field(self):
+        for item in [self.i, dict(self.i)]:
+            self.assertExportResult(
+                item=item,
+                fields_to_export=['age'],
+                expected='age\r\n22\r\n',
+            )
+
+    def test_header_export_two_items(self):
+        for item in [self.i, dict(self.i)]:
+            output = BytesIO()
+            ie = CsvItemExporter(output)
+            ie.start_exporting()
+            ie.export_item(item)
+            ie.export_item(item)
+            ie.finish_exporting()
+            self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
+
+    def test_header_no_header_line(self):
+        for item in [self.i, dict(self.i)]:
+            self.assertExportResult(
+                item=item,
+                include_headers_line=False,
+                expected='22,John\xc2\xa3\r\n',
+            )
 
     def test_join_multivalue(self):
         class TestItem2(Item):
             name = Field()
             friends = Field()
 
-        i = TestItem2(name='John', friends=['Mary', 'Paul'])
-        output = BytesIO()
-        ie = CsvItemExporter(output, include_headers_line=False)
-        ie.start_exporting()
-        ie.export_item(i)
-        ie.finish_exporting()
-        self.assertCsvEqual(output.getvalue(), '"Mary,Paul",John\r\n')
+        for cls in TestItem2, dict:
+            self.assertExportResult(
+                item=cls(name='John', friends=['Mary', 'Paul']),
+                include_headers_line=False,
+                expected='"Mary,Paul",John\r\n',
+            )
+
 
 class XmlItemExporterTest(BaseItemExporterTest):
 
@@ -211,60 +243,62 @@ class XmlItemExporterTest(BaseItemExporterTest):
             return xmltuple(doc)
         return self.assertEqual(xmlsplit(first), xmlsplit(second), msg)
 
+    def assertExportResult(self, item, expected_value):
+        fp = BytesIO()
+        ie = XmlItemExporter(fp)
+        ie.start_exporting()
+        ie.export_item(item)
+        ie.finish_exporting()
+        self.assertXmlEquivalent(fp.getvalue(), expected_value)
+
     def _check_output(self):
         expected_value = '<?xml version="1.0" encoding="utf-8"?>\n<items><item><age>22</age><name>John\xc2\xa3</name></item></items>'
         self.assertXmlEquivalent(self.output.getvalue(), expected_value)
 
     def test_multivalued_fields(self):
-        output = BytesIO()
-        item = TestItem(name=[u'John\xa3', u'Doe'])
-        ie = XmlItemExporter(output)
-        ie.start_exporting()
-        ie.export_item(item)
-        ie.finish_exporting()
-        expected_value = '<?xml version="1.0" encoding="utf-8"?>\n<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
-        self.assertXmlEquivalent(output.getvalue(), expected_value)
+        self.assertExportResult(
+            TestItem(name=[u'John\xa3', u'Doe']),
+            '<?xml version="1.0" encoding="utf-8"?>\n<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
+        )
 
     def test_nested_item(self):
-        output = BytesIO()
         i1 = TestItem(name=u'foo\xa3hoo', age='22')
-        i2 = TestItem(name=u'bar', age=i1)
+        i2 = dict(name=u'bar', age=i1)
         i3 = TestItem(name=u'buz', age=i2)
-        ie = XmlItemExporter(output)
-        ie.start_exporting()
-        ie.export_item(i3)
-        ie.finish_exporting()
-        expected_value = '<?xml version="1.0" encoding="utf-8"?>\n'\
-                '<items><item>'\
-                    '<age>'\
-                        '<age>'\
-                            '<age>22</age>'\
-                            '<name>foo\xc2\xa3hoo</name>'\
-                        '</age>'\
-                        '<name>bar</name>'\
-                    '</age>'\
-                    '<name>buz</name>'\
-                '</item></items>'
-        self.assertXmlEquivalent(output.getvalue(), expected_value)
+
+        self.assertExportResult(i3,
+            '<?xml version="1.0" encoding="utf-8"?>\n'
+            '<items>'
+                '<item>'
+                    '<age>'
+                        '<age>'
+                            '<age>22</age>'
+                            '<name>foo\xc2\xa3hoo</name>'
+                        '</age>'
+                        '<name>bar</name>'
+                    '</age>'
+                    '<name>buz</name>'
+                '</item>'
+            '</items>'
+        )
 
     def test_nested_list_item(self):
-        output = BytesIO()
         i1 = TestItem(name=u'foo')
-        i2 = TestItem(name=u'bar')
+        i2 = dict(name=u'bar', v2={"egg": ["spam"]})
         i3 = TestItem(name=u'buz', age=[i1, i2])
-        ie = XmlItemExporter(output)
-        ie.start_exporting()
-        ie.export_item(i3)
-        ie.finish_exporting()
-        expected_value =  '<?xml version="1.0" encoding="utf-8"?>\n'\
-                '<items><item>'\
-                    '<age>'\
-                        '<value><name>foo</name></value>'\
-                        '<value><name>bar</name></value>'\
-                    '</age>'\
-                    '<name>buz</name>'\
-                '</item></items>'
-        self.assertXmlEquivalent(output.getvalue(), expected_value)
+
+        self.assertExportResult(i3,
+            '<?xml version="1.0" encoding="utf-8"?>\n'
+            '<items>'
+                '<item>'
+                    '<age>'
+                        '<value><name>foo</name></value>'
+                        '<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
+                    '</age>'
+                    '<name>buz</name>'
+                '</item>'
+            '</items>'
+        )
 
 
 class JsonLinesItemExporterTest(BaseItemExporterTest):
@@ -280,7 +314,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
 
     def test_nested_item(self):
         i1 = TestItem(name=u'Joseph', age='22')
-        i2 = TestItem(name=u'Maria', age=i1)
+        i2 = dict(name=u'Maria', age=i1)
         i3 = TestItem(name=u'Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
@@ -306,13 +340,19 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         exported = json.loads(self.output.getvalue().strip())
         self.assertEqual(exported, [dict(self.i)])
 
-    def test_two_items(self):
+    def assertTwoItemsExported(self, item):
         self.ie.start_exporting()
-        self.ie.export_item(self.i)
-        self.ie.export_item(self.i)
+        self.ie.export_item(item)
+        self.ie.export_item(item)
         self.ie.finish_exporting()
         exported = json.loads(self.output.getvalue())
-        self.assertEqual(exported, [dict(self.i), dict(self.i)])
+        self.assertEqual(exported, [dict(item), dict(item)])
+
+    def test_two_items(self):
+        self.assertTwoItemsExported(self.i)
+
+    def test_two_dict_items(self):
+        self.assertTwoItemsExported(dict(self.i))
 
     def test_nested_item(self):
         i1 = TestItem(name=u'Joseph\xa3', age='22')
@@ -325,6 +365,18 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}}
         self.assertEqual(exported, [expected])
 
+    def test_nested_dict_item(self):
+        i1 = dict(name=u'Joseph\xa3', age='22')
+        i2 = TestItem(name=u'Maria', age=i1)
+        i3 = dict(name=u'Jesus', age=i2)
+        self.ie.start_exporting()
+        self.ie.export_item(i3)
+        self.ie.finish_exporting()
+        exported = json.loads(self.output.getvalue())
+        expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}}
+        self.assertEqual(exported, [expected])
+
+
 class CustomItemExporterTest(unittest.TestCase):
 
     def test_exporter_custom_serializer(self):
@@ -333,16 +385,17 @@ class CustomItemExporterTest(unittest.TestCase):
                 if name == 'age':
                     return str(int(value) + 1)
                 else:
-                    return super(CustomItemExporter, self).serialize_field(field, \
-                        name, value)
+                    return super(CustomItemExporter, self).serialize_field(field, name, value)
 
         i = TestItem(name=u'John', age='22')
         ie = CustomItemExporter()
 
-        self.assertEqual( \
-            ie.serialize_field(i.fields['name'], 'name', i['name']), 'John')
-        self.assertEqual(
-            ie.serialize_field(i.fields['age'], 'age', i['age']), '23')
+        self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John')
+        self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '23')
+
+        i2 = {'name': u'John', 'age': '22'}
+        self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John')
+        self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23')
 
 
 if __name__ == '__main__':
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 52c8e5752..04fae02c0 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -28,11 +28,13 @@ from scrapy.contrib.linkextractors import LinkExtractor
 from scrapy.http import Request
 from scrapy.utils.signal import disconnect_all
 
+
 class TestItem(Item):
     name = Field()
     url = Field()
     price = Field()
 
+
 class TestSpider(Spider):
     name = "scrapytest.org"
     allowed_domains = ["scrapytest.org", "localhost"]
@@ -41,6 +43,8 @@ class TestSpider(Spider):
     name_re = re.compile("<h1>(.*?)</h1>", re.M)
     price_re = re.compile(">Price: \$(.*?)<", re.M)
 
+    item_cls = TestItem
+
     def parse(self, response):
         xlink = LinkExtractor()
         itemre = re.compile(self.itemurl_re)
@@ -49,7 +53,7 @@ class TestSpider(Spider):
                 yield Request(url=link.url, callback=self.parse_item)
 
     def parse_item(self, response):
-        item = TestItem()
+        item = self.item_cls()
         m = self.name_re.search(response.body)
         if m:
             item['name'] = m.group(1)
@@ -65,6 +69,10 @@ class TestDupeFilterSpider(TestSpider):
         return Request(url)  # dont_filter=False
 
 
+class DictItemsSpider(TestSpider):
+    item_cls = dict
+
+
 def start_test_site(debug=False):
     root_dir = os.path.join(tests_datadir, "test_site")
     r = static.File(root_dir)
@@ -81,15 +89,14 @@ def start_test_site(debug=False):
 class CrawlerRun(object):
     """A class to run the crawler and keep track of events occurred"""
 
-    def __init__(self, with_dupefilter=False):
+    def __init__(self, spider_class):
         self.spider = None
         self.respplug = []
         self.reqplug = []
         self.reqdropped = []
         self.itemresp = []
         self.signals_catched = {}
-        self.spider_class = TestSpider if not with_dupefilter else \
-            TestDupeFilterSpider
+        self.spider_class = spider_class
 
     def run(self):
         self.port = start_test_site()
@@ -152,14 +159,17 @@ class EngineTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_crawler(self):
-        self.run = CrawlerRun()
-        yield self.run.run()
-        self._assert_visited_urls()
-        self._assert_scheduled_requests(urls_to_visit=8)
-        self._assert_downloaded_responses()
-        self._assert_scraped_items()
-        self._assert_signals_catched()
-        self.run = CrawlerRun(with_dupefilter=True)
+
+        for spider in TestSpider, DictItemsSpider:
+            self.run = CrawlerRun(spider)
+            yield self.run.run()
+            self._assert_visited_urls()
+            self._assert_scheduled_requests(urls_to_visit=8)
+            self._assert_downloaded_responses()
+            self._assert_scraped_items()
+            self._assert_signals_catched()
+
+        self.run = CrawlerRun(TestDupeFilterSpider)
         yield self.run.run()
         self._assert_scheduled_requests(urls_to_visit=7)
         self._assert_dropped_requests()
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 0a1737c44..84fe4927d 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -142,35 +142,40 @@ class DeprecatedFilesPipelineTestCase(unittest.TestCase):
 class FilesPipelineTestCaseFields(unittest.TestCase):
 
     def test_item_fields_default(self):
-        from scrapy.contrib.pipeline.files import FilesPipeline
         class TestItem(Item):
             name = Field()
             file_urls = Field()
             files = Field()
-        url = 'http://www.example.com/files/1.txt'
-        item = TestItem({'name': 'item1', 'file_urls': [url]})
-        pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'}))
-        requests = list(pipeline.get_media_requests(item, None))
-        self.assertEqual(requests[0].url, url)
-        results = [(True, {'url': url})]
-        pipeline.item_completed(results, item, None)
-        self.assertEqual(item['files'], [results[0][1]])
+
+        for cls in TestItem, dict:
+            url = 'http://www.example.com/files/1.txt'
+            item = cls({'name': 'item1', 'file_urls': [url]})
+            pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'}))
+            requests = list(pipeline.get_media_requests(item, None))
+            self.assertEqual(requests[0].url, url)
+            results = [(True, {'url': url})]
+            pipeline.item_completed(results, item, None)
+            self.assertEqual(item['files'], [results[0][1]])
 
     def test_item_fields_override_settings(self):
-        from scrapy.contrib.pipeline.files import FilesPipeline
         class TestItem(Item):
             name = Field()
             files = Field()
             stored_file = Field()
-        url = 'http://www.example.com/files/1.txt'
-        item = TestItem({'name': 'item1', 'files': [url]})
-        pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/',
-                'FILES_URLS_FIELD': 'files', 'FILES_RESULT_FIELD': 'stored_file'}))
-        requests = list(pipeline.get_media_requests(item, None))
-        self.assertEqual(requests[0].url, url)
-        results = [(True, {'url': url})]
-        pipeline.item_completed(results, item, None)
-        self.assertEqual(item['stored_file'], [results[0][1]])
+
+        for cls in TestItem, dict:
+            url = 'http://www.example.com/files/1.txt'
+            item = cls({'name': 'item1', 'files': [url]})
+            pipeline = FilesPipeline.from_settings(Settings({
+                'FILES_STORE': 's3://example/files/',
+                'FILES_URLS_FIELD': 'files',
+                'FILES_RESULT_FIELD': 'stored_file'
+            }))
+            requests = list(pipeline.get_media_requests(item, None))
+            self.assertEqual(requests[0].url, url)
+            results = [(True, {'url': url})]
+            pipeline.item_completed(results, item, None)
+            self.assertEqual(item['stored_file'], [results[0][1]])
 
 
 class ItemWithFiles(Item):
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index a3b1059ef..f5750b4fc 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -168,35 +168,40 @@ class DeprecatedImagesPipelineTestCase(unittest.TestCase):
 class ImagesPipelineTestCaseFields(unittest.TestCase):
 
     def test_item_fields_default(self):
-        from scrapy.contrib.pipeline.images import ImagesPipeline
         class TestItem(Item):
             name = Field()
             image_urls = Field()
             images = Field()
-        url = 'http://www.example.com/images/1.jpg'
-        item = TestItem({'name': 'item1', 'image_urls': [url]})
-        pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': 's3://example/images/'}))
-        requests = list(pipeline.get_media_requests(item, None))
-        self.assertEqual(requests[0].url, url)
-        results = [(True, {'url': url})]
-        pipeline.item_completed(results, item, None)
-        self.assertEqual(item['images'], [results[0][1]])
+
+        for cls in TestItem, dict:
+            url = 'http://www.example.com/images/1.jpg'
+            item = cls({'name': 'item1', 'image_urls': [url]})
+            pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': 's3://example/images/'}))
+            requests = list(pipeline.get_media_requests(item, None))
+            self.assertEqual(requests[0].url, url)
+            results = [(True, {'url': url})]
+            pipeline.item_completed(results, item, None)
+            self.assertEqual(item['images'], [results[0][1]])
 
     def test_item_fields_override_settings(self):
-        from scrapy.contrib.pipeline.images import ImagesPipeline
         class TestItem(Item):
             name = Field()
             image = Field()
             stored_image = Field()
-        url = 'http://www.example.com/images/1.jpg'
-        item = TestItem({'name': 'item1', 'image': [url]})
-        pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': 's3://example/images/',
-                'IMAGES_URLS_FIELD': 'image', 'IMAGES_RESULT_FIELD': 'stored_image'}))
-        requests = list(pipeline.get_media_requests(item, None))
-        self.assertEqual(requests[0].url, url)
-        results = [(True, {'url': url})]
-        pipeline.item_completed(results, item, None)
-        self.assertEqual(item['stored_image'], [results[0][1]])
+
+        for cls in TestItem, dict:
+            url = 'http://www.example.com/images/1.jpg'
+            item = cls({'name': 'item1', 'image': [url]})
+            pipeline = ImagesPipeline.from_settings(Settings({
+                'IMAGES_STORE': 's3://example/images/',
+                'IMAGES_URLS_FIELD': 'image',
+                'IMAGES_RESULT_FIELD': 'stored_image'
+            }))
+            requests = list(pipeline.get_media_requests(item, None))
+            self.assertEqual(requests[0].url, url)
+            results = [(True, {'url': url})]
+            pipeline.item_completed(results, item, None)
+            self.assertEqual(item['stored_image'], [results[0][1]])
 
 
 def _create_image(format, *a, **kw):

From 5846d6154ca3adf87dfee7ba7294ef35949177ab Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Wed, 18 Mar 2015 00:24:16 -0300
Subject: [PATCH 0114/2974] emphasize web crawling over screen scraping on
 scrapy description. closes #586

---
 README.rst         | 2 +-
 debian/control     | 4 ++--
 scrapy/__init__.py | 2 +-
 setup.py           | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.rst b/README.rst
index 7e5dd173f..bf043e1ab 100644
--- a/README.rst
+++ b/README.rst
@@ -15,7 +15,7 @@ Scrapy
 Overview
 ========
 
-Scrapy is a fast high-level screen scraping and web crawling framework, used to
+Scrapy is a fast high-level web crawling and screen scraping framework, used to
 crawl websites and extract structured data from their pages. It can be used for
 a wide range of purposes, from data mining to monitoring and automated testing.
 
diff --git a/debian/control b/debian/control
index 4be62895f..c3dae341e 100644
--- a/debian/control
+++ b/debian/control
@@ -13,8 +13,8 @@ Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl,
 Recommends: python-setuptools
 Conflicts: python-scrapy, scrapy, scrapy-0.11
 Provides: python-scrapy, scrapy
-Description: Python web crawling and scraping framework
- Scrapy is a fast high-level screen scraping and web crawling framework, 
+Description: Python web crawling and screen scraping framework
+ Scrapy is a fast high-level web crawling and screen scraping framework, 
  used to crawl websites and extract structured data from their pages. 
  It can be used for a wide range of purposes, from data mining to 
  monitoring and automated testing.
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 10e9091b6..d60b239a0 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -1,5 +1,5 @@
 """
-Scrapy - a screen scraping framework written in Python
+Scrapy - a web crawling and screen scraping framework written for Python
 """
 
 __all__ = ['__version__', 'version_info', 'optional_features', 'twisted_version',
diff --git a/setup.py b/setup.py
index b5732cbc2..d463bccd9 100644
--- a/setup.py
+++ b/setup.py
@@ -10,7 +10,7 @@ setup(
     name='Scrapy',
     version=version,
     url='http://scrapy.org',
-    description='A high-level Python Screen Scraping framework',
+    description='A high-level Web Crawling and Screen Scraping framework',
     long_description=open('README.rst').read(),
     author='Scrapy developers',
     maintainer='Pablo Hoffman',

From 776616bdec4680e31071debbfe94cad77c3ff148 Mon Sep 17 00:00:00 2001
From: Kevin Yap <me@kevinyap.ca>
Date: Tue, 17 Mar 2015 22:20:45 -0700
Subject: [PATCH 0115/2974] Use Shields.io SVGs for README badges

- Use SVGs for badges (more friendly to retina displays).
- Add alt text to PyPI version and build status badges.
---
 README.rst | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/README.rst b/README.rst
index 7e5dd173f..396bb14f6 100644
--- a/README.rst
+++ b/README.rst
@@ -2,15 +2,17 @@
 Scrapy
 ======
 
-.. image:: https://badge.fury.io/py/Scrapy.png
-   :target: http://badge.fury.io/py/Scrapy
+.. image:: https://img.shields.io/pypi/v/Scrapy.svg
+   :target: https://pypi.python.org/pypi/Scrapy
+   :alt: PyPI Version
 
-.. image:: https://secure.travis-ci.org/scrapy/scrapy.png?branch=master
+.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
    :target: http://travis-ci.org/scrapy/scrapy
+   :alt: Build Status
 
-.. image:: https://pypip.in/wheel/Scrapy/badge.png
-    :target: https://pypi.python.org/pypi/Scrapy/
-    :alt: Wheel Status
+.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
+   :target: https://pypi.python.org/pypi/Scrapy
+   :alt: Wheel Status
 
 Overview
 ========

From 5a58d6413195d00484d5e206e006906231e5890d Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Fri, 6 Feb 2015 22:46:18 +0530
Subject: [PATCH 0116/2974] Fix some redirection links in documentation

Fixes #606
---
 docs/contributing.rst                 |  4 ++--
 docs/faq.rst                          |  8 ++++----
 docs/index.rst                        |  4 ++--
 docs/intro/examples.rst               |  2 +-
 docs/intro/install.rst                |  6 +++---
 docs/intro/overview.rst               |  2 +-
 docs/intro/tutorial.rst               |  4 ++--
 docs/news.rst                         |  8 ++++----
 docs/topics/commands.rst              |  2 +-
 docs/topics/downloader-middleware.rst |  8 ++++----
 docs/topics/email.rst                 |  2 +-
 docs/topics/exporters.rst             |  8 ++++----
 docs/topics/extensions.rst            |  2 +-
 docs/topics/firebug.rst               |  4 ++--
 docs/topics/firefox.rst               | 10 +++++-----
 docs/topics/images.rst                |  4 ++--
 docs/topics/items.rst                 | 10 +++++-----
 docs/topics/leaks.rst                 | 10 +++++-----
 docs/topics/logging.rst               |  2 +-
 docs/topics/request-response.rst      |  2 +-
 docs/topics/scrapyd.rst               |  2 +-
 docs/topics/selectors.rst             |  8 ++++----
 docs/topics/settings.rst              |  2 +-
 docs/topics/spider-middleware.rst     |  2 +-
 docs/topics/spiders.rst               |  4 ++--
 25 files changed, 60 insertions(+), 60 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index f8d965df8..f49bc536e 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -173,10 +173,10 @@ And their unit-tests are in::
     tests/test_contrib_loader.py
 
 .. _issue tracker: https://github.com/scrapy/scrapy/issues
-.. _scrapy-users: http://groups.google.com/group/scrapy-users
+.. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
 .. _Twisted unit-testing framework: http://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
 .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
 .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
 .. _open issues: https://github.com/scrapy/scrapy/issues
-.. _pull request: http://help.github.com/send-pull-requests/
+.. _pull request: https://help.github.com/send-pull-requests/
 .. _tox: https://pypi.python.org/pypi/tox
diff --git a/docs/faq.rst b/docs/faq.rst
index 1d6c56d97..71d9e4c4e 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -21,8 +21,8 @@ comparing `jinja2`_ to `Django`_.
 
 .. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/
 .. _lxml: http://lxml.de/
-.. _jinja2: http://jinja.pocoo.org/2/
-.. _Django: http://www.djangoproject.com
+.. _jinja2: http://jinja.pocoo.org/
+.. _Django: https://www.djangoproject.com/
 
 .. _faq-python-versions:
 
@@ -57,7 +57,7 @@ focus on the real problems we need to solve.
 We'd be proud if Scrapy serves as an inspiration for other projects. Feel free
 to steal from us!
 
-.. _Django: http://www.djangoproject.com
+.. _Django: https://www.djangoproject.com/
 
 Does Scrapy work with HTTP proxies?
 -----------------------------------
@@ -221,7 +221,7 @@ more info on how it works see `this page`_. Also, here's an `example spider`_
 which scrapes one of these sites.
 
 .. _this page: http://search.cpan.org/~ecarroll/HTML-TreeBuilderX-ASP_NET-0.09/lib/HTML/TreeBuilderX/ASP_NET.pm
-.. _example spider: http://github.com/AmbientLighter/rpn-fas/blob/master/fas/spiders/rnp.py
+.. _example spider: https://github.com/AmbientLighter/rpn-fas/blob/master/fas/spiders/rnp.py
 
 What's the best way to parse big XML/CSV data feeds?
 ----------------------------------------------------
diff --git a/docs/index.rst b/docs/index.rst
index 0384dae3d..507b9bea9 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -18,8 +18,8 @@ Having trouble? We'd like to help!
 * Ask a question in the `#scrapy IRC channel`_.
 * Report bugs with Scrapy in our `issue tracker`_.
 
-.. _archives of the scrapy-users mailing list: http://groups.google.com/group/scrapy-users/
-.. _post a question: http://groups.google.com/group/scrapy-users/
+.. _archives of the scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
+.. _post a question: https://groups.google.com/forum/#!forum/scrapy-users
 .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy
 .. _issue tracker: https://github.com/scrapy/scrapy/issues
 
diff --git a/docs/intro/examples.rst b/docs/intro/examples.rst
index 40a124679..c56348714 100644
--- a/docs/intro/examples.rst
+++ b/docs/intro/examples.rst
@@ -21,5 +21,5 @@ middlewares, extensions, or scripts. Feel free (and encouraged!) to share any
 code there.
 
 .. _dirbot: https://github.com/scrapy/dirbot
-.. _Downloads: https://github.com/scrapy/dirbot/archives/master
+.. _Downloads: https://github.com/scrapy/dirbot/downloads
 .. _scrapy tag on Snipplr: http://snipplr.com/all/tags/scrapy/
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 1d786efe7..fbed84055 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -37,7 +37,7 @@ Platform specific installation notes
 Windows
 -------
 
-* Install Python 2.7 from http://python.org/download/
+* Install Python 2.7 from https://www.python.org/downloads/
 
   You need to adjust ``PATH`` environment variable to include paths to
   the Python executable and additional scripts. The following paths need to be
@@ -87,8 +87,8 @@ You can follow the generic instructions or install Scrapy from `AUR Scrapy packa
     yaourt -S scrapy
 
 
-.. _Python: http://www.python.org
-.. _pip: http://www.pip-installer.org/en/latest/installing.html
+.. _Python: https://www.python.org/
+.. _pip: https://pip.pypa.io/en/latest/installing.html
 .. _easy_install: http://pypi.python.org/pypi/setuptools
 .. _Control Panel: http://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx
 .. _lxml: http://lxml.de/
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 289e975b8..c30963db8 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -258,7 +258,7 @@ interest!
 .. _the community: http://scrapy.org/community/
 .. _screen scraping: http://en.wikipedia.org/wiki/Screen_scraping
 .. _web scraping: http://en.wikipedia.org/wiki/Web_scraping
-.. _Amazon Associates Web Services: http://aws.amazon.com/associates/
+.. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html
 .. _Mininova: http://www.mininova.org
 .. _XPath: http://www.w3.org/TR/xpath
 .. _XPath reference: http://www.w3.org/TR/xpath
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index a4248d7aa..ad808316b 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -26,8 +26,8 @@ Python quickly, we recommend `Learn Python The Hard Way`_.  If you're new to pro
 and want to start with Python, take a look at `this list of Python resources
 for non-programmers`_.
 
-.. _Python: http://www.python.org
-.. _this list of Python resources for non-programmers: http://wiki.python.org/moin/BeginnersGuide/NonProgrammers
+.. _Python: https://www.python.org/
+.. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
 .. _Learn Python The Hard Way: http://learnpythonthehardway.org/book/
 
 Creating a project
diff --git a/docs/news.rst b/docs/news.rst
index 0f5e78e8c..383f59760 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -578,7 +578,7 @@ Scrapy changes:
 ------
 
 - added precise to supported ubuntu distros (:commit:`b7e46df`)
-- fixed bug in json-rpc webservice reported in https://groups.google.com/d/topic/scrapy-users/qgVBmFybNAQ/discussion. also removed no longer supported 'run' command from extras/scrapy-ws.py (:commit:`340fbdb`)
+- fixed bug in json-rpc webservice reported in https://groups.google.com/forum/#!topic/scrapy-users/qgVBmFybNAQ/discussion. also removed no longer supported 'run' command from extras/scrapy-ws.py (:commit:`340fbdb`)
 - meta tag attributes for content-type http equiv can be in any order. #123 (:commit:`0cb68af`)
 - replace "import Image" by more standard "from PIL import Image". closes #88 (:commit:`4d17048`)
 - return trial status as bin/runtests.sh exit value. #118 (:commit:`b7b2e7f`)
@@ -902,14 +902,14 @@ Backwards-incompatible changes
 First release of Scrapy.
 
 
-.. _AJAX crawleable urls: http://code.google.com/web/ajaxcrawling/docs/getting-started.html
+.. _AJAX crawleable urls: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started?csw=1
 .. _chunked transfer encoding: http://en.wikipedia.org/wiki/Chunked_transfer_encoding
 .. _w3lib: https://github.com/scrapy/w3lib
 .. _scrapely: https://github.com/scrapy/scrapely
-.. _marshal: http://docs.python.org/library/marshal.html
+.. _marshal: https://docs.python.org/2/library/marshal.html
 .. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
 .. _lxml: http://lxml.de/
 .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
-.. _resource: http://docs.python.org/library/resource.html
+.. _resource: https://docs.python.org/2/library/resource.html
 .. _queuelib: https://github.com/scrapy/queuelib
 .. _cssselect: https://github.com/SimonSapin/cssselect
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 545a2f165..5c0de0d6d 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -484,7 +484,7 @@ You can also add your custom project commands by using the
 :setting:`COMMANDS_MODULE` setting. See the Scrapy commands in
 `scrapy/commands`_ for examples on how to implement your commands.
 
-.. _scrapy/commands: https://github.com/scrapy/scrapy/blob/master/scrapy/commands
+.. _scrapy/commands: https://github.com/scrapy/scrapy/tree/master/scrapy/commands
 .. setting:: COMMANDS_MODULE
 
 COMMANDS_MODULE
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 424d52a73..5b38f048c 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -451,7 +451,7 @@ In order to use this storage backend:
 * install `LevelDB python bindings`_ like ``pip install leveldb``
 
 .. _LevelDB: http://code.google.com/p/leveldb/
-.. _leveldb python bindings: http://pypi.python.org/pypi/leveldb
+.. _leveldb python bindings: https://pypi.python.org/pypi/leveldb
 
 
 HTTPCache middleware settings
@@ -635,8 +635,8 @@ HttpProxyMiddleware
    You can also set the meta key ``proxy`` per-request, to a value like
    ``http://some_proxy_server:port``.
 
-.. _urllib: http://docs.python.org/library/urllib.html
-.. _urllib2: http://docs.python.org/library/urllib2.html
+.. _urllib: https://docs.python.org/2/library/urllib.html
+.. _urllib2: https://docs.python.org/2/library/urllib2.html
 
 RedirectMiddleware
 ------------------
@@ -890,5 +890,5 @@ enable it for :ref:`broad crawls <topics-broad-crawls>`.
 
 
 .. _DBM: http://en.wikipedia.org/wiki/Dbm
-.. _anydbm: http://docs.python.org/library/anydbm.html
+.. _anydbm: https://docs.python.org/2/library/anydbm.html
 .. _chunked transfer encoding: http://en.wikipedia.org/wiki/Chunked_transfer_encoding
diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index d99589413..789fbd4fb 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -14,7 +14,7 @@ interfering with the non-blocking IO of the crawler. It also provides a
 simple API for sending attachments and it's very easy to configure, with a few
 :ref:`settings <topics-email-settings>`.
 
-.. _smtplib: http://docs.python.org/library/smtplib.html
+.. _smtplib: https://docs.python.org/2/library/smtplib.html
 .. _Twisted non-blocking IO: http://twistedmatrix.com/documents/current/core/howto/defer-intro.html
 
 Quick example
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index c472f5b96..f7feed4af 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -297,7 +297,7 @@ CsvItemExporter
       Color TV,1200
       DVD player,200
 
-.. _csv.writer: http://docs.python.org/library/csv.html#csv.writer
+.. _csv.writer: https://docs.python.org/2/library/csv.html#csv.writer
 
 PickleItemExporter
 ------------------
@@ -318,7 +318,7 @@ PickleItemExporter
 
    Pickle isn't a human readable format, so no output examples are provided.
 
-.. _pickle module documentation: http://docs.python.org/library/pickle.html
+.. _pickle module documentation: https://docs.python.org/2/library/pickle.html
 
 PprintItemExporter
 ------------------
@@ -367,7 +367,7 @@ JsonItemExporter
       stream-friendly format, consider using :class:`JsonLinesItemExporter`
       instead, or splitting the output in multiple chunks.
 
-.. _JSONEncoder: http://docs.python.org/library/json.html#json.JSONEncoder
+.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
 
 JsonLinesItemExporter
 ---------------------
@@ -390,4 +390,4 @@ JsonLinesItemExporter
    Unlike the one produced by :class:`JsonItemExporter`, the format produced by
    this exporter is well suited for serializing large amounts of data.
 
-.. _JSONEncoder: http://docs.python.org/library/json.html#json.JSONEncoder
+.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 1824bcc3a..8cd588c4a 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -368,5 +368,5 @@ For more info see `Debugging in Python`.
 
 This extension only works on POSIX-compliant platforms (ie. not Windows).
 
-.. _Python debugger: http://docs.python.org/library/pdb.html
+.. _Python debugger: https://docs.python.org/2/library/pdb.html
 .. _Debugging in Python: http://www.ferg.org/papers/debugging_in_python.html
diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst
index 7dd5a2b76..ad3f26b50 100644
--- a/docs/topics/firebug.rst
+++ b/docs/topics/firebug.rst
@@ -32,7 +32,7 @@ you to inspect the HTML code of the different page elements just by hovering
 your mouse over them. Otherwise you would have to search for the tags manually
 through the HTML body which can be a very tedious task.
 
-.. _Inspect Element: http://www.youtube.com/watch?v=-pT_pDe54aA
+.. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA
 
 In the following screenshot you can see the `Inspect Element`_ tool in action.
 
@@ -164,4 +164,4 @@ elements.
 or tags which Therefer   in page HTML
 sources may on Firebug inspects the live DOM
 
-.. _has been shut down by Google: http://searchenginewatch.com/article/2096661/Google-Directory-Has-Been-Shut-Down
+.. _has been shut down by Google: http://searchenginewatch.com/sew/news/2096661/google-directory-shut
diff --git a/docs/topics/firefox.rst b/docs/topics/firefox.rst
index f0b8eb594..beda3b8db 100644
--- a/docs/topics/firefox.rst
+++ b/docs/topics/firefox.rst
@@ -74,9 +74,9 @@ extension to create a new cookie, delete existing cookies, see a list of cookies
 for the current site, manage cookies permissions and a lot more. 
 
 .. _Firebug: http://getfirebug.com
-.. _Inspect Element: http://www.youtube.com/watch?v=-pT_pDe54aA
-.. _XPather: https://addons.mozilla.org/firefox/addon/1192 
-.. _XPath Checker: https://addons.mozilla.org/firefox/addon/1095
-.. _Tamper Data: http://addons.mozilla.org/firefox/addon/966
-.. _Firecookie: https://addons.mozilla.org/firefox/addon/6683
+.. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA
+.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/ 
+.. _XPath Checker: https://addons.mozilla.org/en-US/firefox/addon/xpath-checker/
+.. _Tamper Data: https://addons.mozilla.org/en-US/firefox/addon/tamper-data/
+.. _Firecookie: https://addons.mozilla.org/en-US/firefox/addon/firecookie/
 
diff --git a/docs/topics/images.rst b/docs/topics/images.rst
index 91e59d2a9..4b07300eb 100644
--- a/docs/topics/images.rst
+++ b/docs/topics/images.rst
@@ -30,7 +30,7 @@ so you need to install this library in order to use the images pipeline.
 is known to cause troubles in some setups, so we recommend to use `Pillow`_
 instead of `PIL <Python Imaging Library>`_.
 
-.. _Pillow: https://github.com/python-imaging/Pillow
+.. _Pillow: https://github.com/python-pillow/Pillow
 .. _Python Imaging Library: http://www.pythonware.com/products/pil/
 
 Using the Images Pipeline
@@ -104,7 +104,7 @@ Images Storage
 File system is currently the only officially supported storage, but there is
 also (undocumented) support for `Amazon S3`_.
 
-.. _Amazon S3: https://s3.amazonaws.com/
+.. _Amazon S3: http://aws.amazon.com/s3/
 
 File system storage
 -------------------
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index ee604a7f1..17f10a88c 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -15,7 +15,7 @@ purpose.
 They provide a `dictionary-like`_ API with a convenient syntax for declaring
 their available fields.
 
-.. _dictionary-like: http://docs.python.org/library/stdtypes.html#dict
+.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
 
 .. _topics-items-declaring:
 
@@ -37,8 +37,8 @@ objects. Here is an example::
    declared similar to `Django Models`_, except that Scrapy Items are much
    simpler as there is no concept of different field types.
 
-.. _Django: http://www.djangoproject.com/
-.. _Django Models: http://docs.djangoproject.com/en/dev/topics/db/models/
+.. _Django: https://www.djangoproject.com/
+.. _Django Models: https://docs.djangoproject.com/en/dev/topics/db/models/
 
 .. _topics-items-fields:
 
@@ -214,7 +214,7 @@ Item objects
         :class:`Field` objects used in the :ref:`Item declaration
         <topics-items-declaring>`.
 
-.. _dict API: http://docs.python.org/library/stdtypes.html#dict
+.. _dict API: https://docs.python.org/2/library/stdtypes.html#dict
 
 Field objects
 =============
@@ -227,6 +227,6 @@ Field objects
     to support the :ref:`item declaration syntax <topics-items-declaring>`
     based on class attributes.
 
-.. _dict: http://docs.python.org/library/stdtypes.html#dict
+.. _dict: https://docs.python.org/2/library/stdtypes.html#dict
 
 
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 372691c8e..21cfcf9bc 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -203,7 +203,7 @@ other cases where the memory leaks could come from other (more or less obscure)
 objects. If this is your case, and you can't find your leaks using ``trackref``,
 you still have another resource: the `Guppy library`_.
 
-.. _Guppy library: http://pypi.python.org/pypi/guppy
+.. _Guppy library: https://pypi.python.org/pypi/guppy
 
 If you use ``pip``, you can install Guppy with the following command::
 
@@ -264,9 +264,9 @@ though neither Scrapy nor your project are leaking memory. This is due to a
 (not so well) known problem of Python, which may not return released memory to
 the operating system in some cases. For more information on this issue see:
 
-* `Python Memory Management <http://evanjones.ca/python-memory.html>`_
-* `Python Memory Management Part 2 <http://evanjones.ca/python-memory-part2.html>`_
-* `Python Memory Management Part 3 <http://evanjones.ca/python-memory-part3.html>`_
+* `Python Memory Management <http://www.evanjones.ca/python-memory.html>`_
+* `Python Memory Management Part 2 <http://www.evanjones.ca/python-memory-part2.html>`_
+* `Python Memory Management Part 3 <http://www.evanjones.ca/python-memory-part3.html>`_
 
 The improvements proposed by Evan Jones, which are detailed in `this paper`_,
 got merged in Python 2.5, but this only reduces the problem, it doesn't fix it
@@ -280,7 +280,7 @@ completely. To quote the paper:
     to move to a compacting garbage collector, which is able to move objects in
     memory. This would require significant changes to the Python interpreter.*
 
-.. _this paper: http://evanjones.ca/memoryallocator/
+.. _this paper: http://www.evanjones.ca/memoryallocator/
 
 To keep memory consumption reasonable you can split the job into several
 smaller jobs or enable :ref:`persistent job queue <topics-jobs>`
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 819884ac2..e9266cd6a 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -8,7 +8,7 @@ Scrapy provides a logging facility which can be used through the
 :mod:`scrapy.log` module. The current underlying implementation uses `Twisted
 logging`_ but this may change in the future.
 
-.. _Twisted logging: http://twistedmatrix.com/projects/core/documentation/howto/logging.html
+.. _Twisted logging: http://twistedmatrix.com/documents/current/core/howto/logging.html
 
 The logging service must be explicitly started through the
 :func:`scrapy.log.start` function to catch the top level Scrapy's log messages.
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index b6b165d50..302ed4f3b 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -157,7 +157,7 @@ Request objects
         ``copy()`` or ``replace()`` methods, and can also be accessed, in your
         spider, from the ``response.meta`` attribute.
 
-    .. _shallow copied: http://docs.python.org/library/copy.html
+    .. _shallow copied: https://docs.python.org/2/library/copy.html
 
     .. method:: Request.copy()
 
diff --git a/docs/topics/scrapyd.rst b/docs/topics/scrapyd.rst
index 2b7ded9dd..85d27a99e 100644
--- a/docs/topics/scrapyd.rst
+++ b/docs/topics/scrapyd.rst
@@ -8,4 +8,4 @@ Scrapyd has been moved into a separate project.
 
 Its documentation is now hosted at:
 
-    http://scrapyd.readthedocs.org/
+    http://scrapyd.readthedocs.org/en/latest/
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index d966a67d2..00ed8152c 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -38,7 +38,7 @@ For a complete reference of the selectors API see
 
 .. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/
 .. _lxml: http://lxml.de/
-.. _ElementTree: http://docs.python.org/library/xml.etree.elementtree.html
+.. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html
 .. _cssselect: https://pypi.python.org/pypi/cssselect/
 .. _XPath: http://www.w3.org/TR/xpath
 .. _CSS: http://www.w3.org/TR/selectors
@@ -403,9 +403,9 @@ Here we first iterate over ``itemscope`` elements, and for each one,
 we look for all ``itemprops`` elements and exclude those that are themselves
 inside another ``itemscope``.
 
-.. _EXSLT: http://www.exslt.org/
-.. _regular expressions: http://www.exslt.org/regexp/index.html
-.. _set manipulation: http://www.exslt.org/set/index.html
+.. _EXSLT: http://exslt.org/
+.. _regular expressions: http://exslt.org/regexp/index.html
+.. _set manipulation: http://exslt.org/set/index.html
 
 
 Some XPath tips
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index ea2c208bb..5e11e473f 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -26,7 +26,7 @@ The value of ``SCRAPY_SETTINGS_MODULE`` should be in Python path syntax, e.g.
 ``myproject.settings``. Note that the settings module should be on the
 Python `import search path`_.
 
-.. _import search path: http://docs.python.org/2/tutorial/modules.html#the-module-search-path
+.. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path
 
 Populating the settings
 =======================
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 79cfbcf5d..6f14567fc 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -159,7 +159,7 @@ following methods:
         :type spider: :class:`~scrapy.spider.Spider` object
 
 
-.. _Exception: http://docs.python.org/library/exceptions.html#exceptions.Exception
+.. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception
 
 
 .. _topics-spider-middleware-ref:
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index cb3f6caeb..a7e7d2746 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -706,7 +706,7 @@ Combine SitemapSpider with other sources of urls::
             pass # ... scrape other here ...
 
 .. _Sitemaps: http://www.sitemaps.org
-.. _Sitemap index files: http://www.sitemaps.org/protocol.php#index
+.. _Sitemap index files: http://www.sitemaps.org/protocol.html#index
 .. _robots.txt: http://www.robotstxt.org/
 .. _TLD: http://en.wikipedia.org/wiki/Top-level_domain
-.. _Scrapyd documentation: http://scrapyd.readthedocs.org/
+.. _Scrapyd documentation: http://scrapyd.readthedocs.org/en/latest/

From c05f5f175e7027752357fa4c931ce4677e7f1c6e Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Thu, 12 Mar 2015 06:57:47 +0530
Subject: [PATCH 0117/2974] Added linkfix script to docs/utils

https://github.com/scrapy/scrapy/pull/1041#issuecomment-78143576
---
 docs/utils/linkfix.py | 63 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)
 create mode 100755 docs/utils/linkfix.py

diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
new file mode 100755
index 000000000..40316968f
--- /dev/null
+++ b/docs/utils/linkfix.py
@@ -0,0 +1,63 @@
+#!/usr/bin/python
+
+"""
+
+Linkfix - a companion to sphinx's linkcheck builder.
+
+Uses the linkcheck's output file to fix links in docs.
+
+Originally created for this issue:
+https://github.com/scrapy/scrapy/issues/606
+
+Author: dufferzafar
+"""
+
+import re
+
+# Used for remembering the file (and its contents)
+# so we don't have to open the same file again.
+_filename = None
+_contents = None
+
+# A regex that matches standard linkcheck output lines
+line_re = re.compile(ur'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
+
+# Read lines from the linkcheck output file
+try:
+    with open("build/linkcheck/output.txt") as out:
+        output_lines = out.readlines()
+except IOError:
+    print("linkcheck output not found; please run linkcheck first.")
+    exit(1)
+
+# For every line, fix the respective file
+for line in output_lines:
+    match = re.match(line_re, line)
+
+    if match:
+        newfilename = match.group(1)
+        errortype = match.group(2)
+
+        # Broken links can't be fixed and
+        # I am not sure what do with the local ones.
+        if errortype.lower() in ["broken", "local"]:
+            print("Not Fixed: " + line)
+        else:
+            # If this is a new file
+            if newfilename != _filename:
+
+                # Update the previous file
+                if _filename:
+                    with open(_filename, "w") as _file:
+                        _file.write(_contents)
+
+                _filename = newfilename
+
+                # Read the new file to memory
+                with open(_filename) as _file:
+                    _contents = _file.read()
+
+            _contents = _contents.replace(match.group(3), match.group(4))
+    else:
+        # We don't understand what the current line means!
+        print("Not Understood: " + line)

From ff987fb5a571ba91cb4e8bd0472e94579acbf2a2 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 18 Mar 2015 20:00:03 -0300
Subject: [PATCH 0118/2974] Add linkfix rule to docs Makefile

---
 docs/Makefile | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/Makefile b/docs/Makefile
index c6e4dd64d..4289690f0 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -26,7 +26,7 @@ help:
 
 build: 
 	mkdir -p build/$(BUILDER) build/doctrees
-	sphinx-build $(ALLSPHINXOPTS)
+	-sphinx-build $(ALLSPHINXOPTS)
 	@echo
 
 
@@ -58,6 +58,11 @@ linkcheck: build
 	@echo "Link check complete; look for any errors in the above output " \
 	      "or in build/$(BUILDER)/output.txt"
 
+linkfix: linkcheck
+	$(PYTHON) utils/linkfix.py
+	@echo "Fixing redirecting links in docs has finished; check all " \
+	      "replacements before committing them"
+
 doctest: BUILDER = doctest
 doctest: build
 	@echo "Testing of doctests in the sources finished, look at the " \

From 4fb818a250ef48fd2e7708c569dfa6b4474a173e Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 18 Mar 2015 20:04:14 -0300
Subject: [PATCH 0119/2974] Run linkfix over current docs

---
 docs/topics/downloader-middleware.rst | 2 +-
 docs/topics/loaders.rst               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 5b38f048c..149eef59e 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -376,7 +376,7 @@ what is implemented:
 
 what is missing:
 
-* `Pragma: no-cache` support http://www.mnot.net/cache_docs/#PRAGMA
+* `Pragma: no-cache` support https://www.mnot.net/cache_docs/#PRAGMA
 * `Vary` header support http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6
 * Invalidation after updates or deletes http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10
 * ... probably others ..
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 9df8e117d..b2c8a018d 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -678,7 +678,7 @@ Here is a list of all built-in processors:
 .. class:: SelectJmes(json_path)
 
     Queries the value using the json path provided to the constructor and returns the output.
-    Requires jmespath (https://github.com/jmespath/jmespath) to run.
+    Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
     This processor takes only one input at a time.
 
     Example::

From 12eedd90d19fda91f7d2f985ee48f3e338175b06 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 18 Mar 2015 20:46:18 -0300
Subject: [PATCH 0120/2974] fix truncated 0.24.5 release notes. closes #1084

---
 docs/news.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 383f59760..bb1b374ce 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -10,14 +10,14 @@ Release notes
 - DOC a couple more references are fixed (:commit:`b4c454b`)
 - DOC fix a reference (:commit:`e3c1260`)
 - t.i.b.ThreadedResolver is now a new-style class (:commit:`9e13f42`)
-- S3DownloadHandler: fix auth for requests with quoted paths/query params (:commit:`cdb9
+- S3DownloadHandler: fix auth for requests with quoted paths/query params (:commit:`cdb9a0b`)
 - fixed the variable types in mailsender documentation (:commit:`bb3a848`)
 - Reset items_scraped instead of item_count (:commit:`edb07a4`)
-- Tentative attention message about what document to read for contributions (:commit:`7e
+- Tentative attention message about what document to read for contributions (:commit:`7ee6f7a`)
 - mitmproxy 0.10.1 needs netlib 0.10.1 too (:commit:`874fcdd`)
 - pin mitmproxy 0.10.1 as >0.11 does not work with tests (:commit:`c6b21f0`)
 - Test the parse command locally instead of against an external url (:commit:`c3a6628`)
-- Patches Twisted issue while closing the connection pool on HTTPDownloadHandler (:commi
+- Patches Twisted issue while closing the connection pool on HTTPDownloadHandler (:commit:`d0bf957`)
 - Updates documentation on dynamic item classes. (:commit:`eeb589a`)
 - Merge pull request #943 from Lazar-T/patch-3 (:commit:`5fdab02`)
 - typo (:commit:`b0ae199`)
@@ -27,7 +27,7 @@ Release notes
 - comma instead of fullstop (:commit:`627b9ba`)
 - Merge pull request #885 from jsma/patch-1 (:commit:`de909ad`)
 - Update request-response.rst (:commit:`3f3263d`)
-- SgmlLinkExtractor - fix for parsing <area> tag with Unicode present (:commit:`49b40f0`
+- SgmlLinkExtractor - fix for parsing <area> tag with Unicode present (:commit:`49b40f0`)
 
 0.24.4 (2014-08-09)
 -------------------

From 6c7bd54fc37d2f21bbbd0927eab2da827f070852 Mon Sep 17 00:00:00 2001
From: Mateusz Golewski <golewski@red-sky.pl>
Date: Thu, 30 Jan 2014 21:33:46 +0100
Subject: [PATCH 0121/2974] Add extract_first() method to SelectorList

---
 scrapy/selector/unified.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index b8a3678a8..7b877153d 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -178,6 +178,10 @@ class SelectorList(list):
     def extract(self):
         return [x.extract() for x in self]
 
+    def extract_first(self):
+        for x in self.extract():
+            return x
+
     @deprecated(use_instead='.extract()')
     def extract_unquoted(self):
         return [x.extract_unquoted() for x in self]

From bd126be3569ddd77c458c79ef9e066cacf3a3af1 Mon Sep 17 00:00:00 2001
From: Mateusz Golewski <golewski@red-sky.pl>
Date: Thu, 30 Jan 2014 21:48:50 +0100
Subject: [PATCH 0122/2974] Optimize extract_first()

---
 scrapy/selector/unified.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 7b877153d..3d9435665 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -179,8 +179,8 @@ class SelectorList(list):
         return [x.extract() for x in self]
 
     def extract_first(self):
-        for x in self.extract():
-            return x
+        for x in self:
+            return x.extract()
 
     @deprecated(use_instead='.extract()')
     def extract_unquoted(self):

From 2742b4d8c26f946c20767fd7fd3f227d00002597 Mon Sep 17 00:00:00 2001
From: Mateusz Golewski <golewski@red-sky.pl>
Date: Thu, 30 Jan 2014 23:10:53 +0100
Subject: [PATCH 0123/2974] Add tests to extract_first()

---
 tests/test_selector.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/test_selector.py b/tests/test_selector.py
index 6fbb451a6..80a9a4672 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -55,6 +55,23 @@ class SelectorTestCase(unittest.TestCase):
             ["<Selector xpath=u'//input[@value=\"\\xa9\"]/@value' data=u'\\xa9'>"]
         )
 
+    def test_extract_first(self):
+        """Test if extract_first() returns first element"""
+        body = '<ul><li id="1">1</li><li id="2">2</li></ul>'
+        response = TextResponse(url="http://example.com", body=body)
+        sel = self.sscls(response)
+
+        self.assertEqual(sel.xpath('//ul/li/text()').extract_first(),
+                         sel.xpath('//ul/li/text()').extract()[0])
+
+        self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').extract_first(),
+                         sel.xpath('//ul/li[@id="1"]/text()').extract()[0])
+
+        self.assertEqual(sel.xpath('//ul/li[2]/text()').extract_first(),
+                         sel.xpath('//ul/li/text()').extract()[1])
+
+        self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').extract_first(), None)
+
     def test_select_unicode_query(self):
         body = u"<p><input name='\xa9' value='1'/></p>"
         response = TextResponse(url="http://example.com", body=body, encoding='utf8')

From 012211accda0fb5ce3af2d4010e2b51db33bdd02 Mon Sep 17 00:00:00 2001
From: Mateusz Golewski <golewski@red-sky.pl>
Date: Thu, 30 Jan 2014 23:39:15 +0100
Subject: [PATCH 0124/2974] Add docs for extract_first()

---
 docs/topics/selectors.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 00ed8152c..0ce0f084e 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -139,6 +139,16 @@ method, as follows::
     >>> response.xpath('//title/text()').extract()
     [u'Example website']
 
+If you want to extract only first matched element, you must call the selector ``.extract_first()``
+
+    >>> sel.xpath('//ul/li').extract_first()
+    u'First list element'
+
+It returns ``None`` if no element was found:
+
+    >>> sel.xpath('//ul/li[999]').extract_first()
+    None
+
 Notice that CSS selectors can select text or attribute nodes using CSS3
 pseudo-elements::
 

From 127c6c694a75e1448ddbd3d0f699ca8074c46761 Mon Sep 17 00:00:00 2001
From: Mateusz Golewski <golewski@red-sky.pl>
Date: Sun, 2 Feb 2014 15:02:25 +0100
Subject: [PATCH 0125/2974] Fix extract_first() docs

---
 docs/topics/selectors.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 0ce0f084e..92e092246 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -139,15 +139,15 @@ method, as follows::
     >>> response.xpath('//title/text()').extract()
     [u'Example website']
 
-If you want to extract only first matched element, you must call the selector ``.extract_first()``
+If you want to extract only first matched element, you can call the selector ``.extract_first()``
 
-    >>> sel.xpath('//ul/li').extract_first()
-    u'First list element'
+    >>> sel.xpath('//div[@id="images"]/a/text()').extract_first()
+    u'Name: My image 1 '
 
 It returns ``None`` if no element was found:
 
-    >>> sel.xpath('//ul/li[999]').extract_first()
-    None
+    >>> sel.xpath('//div/[id="not-exists"]/text()').extract_first() is None
+    True
 
 Notice that CSS selectors can select text or attribute nodes using CSS3
 pseudo-elements::

From f92bc09bf433c43ab5669f7bc14108ac6fd49e5c Mon Sep 17 00:00:00 2001
From: Mateusz Golewski <golewski@red-sky.pl>
Date: Sun, 2 Feb 2014 15:45:43 +0100
Subject: [PATCH 0126/2974] Add re_first() to SelectorList and iflatten() to
 utils.python

---
 scrapy/selector/unified.py |  6 +++++-
 scrapy/utils/python.py     | 15 +++++++++++----
 tests/test_selector.py     | 18 ++++++++++++++++++
 3 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 3d9435665..889c349e3 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -6,7 +6,7 @@ from lxml import etree
 
 from scrapy.utils.misc import extract_regex
 from scrapy.utils.trackref import object_ref
-from scrapy.utils.python import unicode_to_str, flatten
+from scrapy.utils.python import unicode_to_str, flatten, iflatten
 from scrapy.utils.decorator import deprecated
 from scrapy.http import HtmlResponse, XmlResponse
 from .lxmldocument import LxmlDocument
@@ -175,6 +175,10 @@ class SelectorList(list):
     def re(self, regex):
         return flatten([x.re(regex) for x in self])
 
+    def re_first(self, regex):
+        for el in iflatten((x.re(regex) for x in self)):
+            return el
+
     def extract(self):
         return [x.extract() for x in self]
 
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 551d337eb..b6100f899 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -27,13 +27,20 @@ def flatten(x):
     >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)])
     [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]"""
 
-    result = []
+    return list(iflatten(x))
+
+
+def iflatten(x):
+    """iflatten(sequence) -> iterator
+
+    Similar to ``.flatten()``, but returns iterator instead"""
+
     for el in x:
         if hasattr(el, "__iter__"):
-            result.extend(flatten(el))
+            for el_ in flatten(el):
+                yield el_
         else:
-            result.append(el)
-    return result
+            yield el
 
 
 def unique(list_, key=lambda x: x):
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 80a9a4672..9b8613319 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -72,6 +72,24 @@ class SelectorTestCase(unittest.TestCase):
 
         self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').extract_first(), None)
 
+    def test_re_first(self):
+        """Test if re_first() returns first matched element"""
+        body = '<ul><li id="1">1</li><li id="2">2</li></ul>'
+        response = TextResponse(url="http://example.com", body=body)
+        sel = self.sscls(response)
+
+        self.assertEqual(sel.xpath('//ul/li/text()').re_first('\d'),
+                         sel.xpath('//ul/li/text()').re('\d')[0])
+
+        self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').re_first('\d'),
+                         sel.xpath('//ul/li[@id="1"]/text()').re('\d')[0])
+
+        self.assertEqual(sel.xpath('//ul/li[2]/text()').re_first('\d'),
+                         sel.xpath('//ul/li/text()').re('\d')[1])
+
+        self.assertEqual(sel.xpath('/ul/li/text()').re_first('\w+'), None)
+        self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').re_first('\d'), None)
+
     def test_select_unicode_query(self):
         body = u"<p><input name='\xa9' value='1'/></p>"
         response = TextResponse(url="http://example.com", body=body, encoding='utf8')

From 0dade7315bc59ce6b2cfacdd17895985882f5ae1 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 18 Mar 2015 20:50:17 -0300
Subject: [PATCH 0127/2974] Use generator sintax in re_first

---
 scrapy/selector/unified.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 889c349e3..c0eefb85e 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -176,7 +176,7 @@ class SelectorList(list):
         return flatten([x.re(regex) for x in self])
 
     def re_first(self, regex):
-        for el in iflatten((x.re(regex) for x in self)):
+        for el in iflatten(x.re(regex) for x in self):
             return el
 
     def extract(self):

From 959aaad20554f1ad89704229594d2efb7a835bd3 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 18 Mar 2015 21:04:15 -0300
Subject: [PATCH 0128/2974] Document `re_first`

---
 docs/topics/selectors.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 92e092246..33958cee5 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -236,6 +236,12 @@ Here's an example used to extract images names from the :ref:`HTML code
      u'My image 4',
      u'My image 5']
 
+There's an additional helper reciprocating ``.extract_first()`` for ``.re()``,
+named ``.re_first()``. Use it to extract just the first matching string::
+
+    >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
+    u'My image 1'
+
 .. _topics-selectors-relative-xpaths:
 
 Working with relative XPaths

From 817dbc6cbd04d1ee8644ccc22d1b109afcf5a892 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 19 Mar 2015 05:16:14 +0500
Subject: [PATCH 0129/2974] DOC mention dicts in documentation; explain better
 what are Items for

---
 docs/topics/architecture.rst      |  4 ++--
 docs/topics/exporters.rst         | 13 +++++++------
 docs/topics/images.rst            | 13 +++++++++----
 docs/topics/item-pipeline.rst     | 15 ++++++++-------
 docs/topics/items.rst             | 17 ++++++++++++-----
 docs/topics/practices.rst         |  7 +++----
 docs/topics/signals.rst           |  4 ++--
 docs/topics/spider-middleware.rst |  9 +++++----
 8 files changed, 48 insertions(+), 34 deletions(-)

diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index 80ccd42dd..55df1cc98 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -102,10 +102,10 @@ this:
 6. The Engine receives the Response from the Downloader and sends it to the
    Spider for processing, passing through the Spider Middleware (input direction).
 
-7. The Spider processes the Response and returns scraped Items and new Requests
+7. The Spider processes the Response and returns scraped items and new Requests
    (to follow) to the Engine.
 
-8. The Engine sends scraped Items (returned by the Spider) to the Item Pipeline
+8. The Engine sends scraped items (returned by the Spider) to the Item Pipeline
    and Requests (returned by spider) to the Scheduler
 
 9. The process repeats (from step 2) until there are no more requests from the
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 43931544f..7455b28da 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -7,7 +7,7 @@ Item Exporters
 .. module:: scrapy.contrib.exporter
    :synopsis: Item Exporters
 
-Once you have scraped your Items, you often want to persist or export those
+Once you have scraped your items, you often want to persist or export those
 items, to use the data in some other application. That is, after all, the whole
 purpose of the scraping process.
 
@@ -90,9 +90,9 @@ described next.
 1. Declaring a serializer in the field
 --------------------------------------
 
-You can declare a serializer in the :ref:`field metadata
-<topics-items-fields>`. The serializer must be a callable which receives a
-value and returns its serialized form.
+If you use :class:`~.Item` you can declare a serializer in the 
+:ref:`field metadata <topics-items-fields>`. The serializer must be 
+a callable which receives a value and returns its serialized form.
 
 Example::
 
@@ -167,8 +167,9 @@ BaseItemExporter
       value unchanged except for ``unicode`` values which are encoded to
       ``str`` using the encoding declared in the :attr:`encoding` attribute.
 
-      :param field: the field being serialized
-      :type field: :class:`~scrapy.item.Field` object
+      :param field: the field being serialized. If a raw dict is being 
+          exported (not :class:`~.Item`) *field* value is an empty dict.
+      :type field: :class:`~scrapy.item.Field` object or an empty dict 
 
       :param name: the name of the field being serialized
       :type name: str
diff --git a/docs/topics/images.rst b/docs/topics/images.rst
index 91e59d2a9..2cbff2e84 100644
--- a/docs/topics/images.rst
+++ b/docs/topics/images.rst
@@ -63,9 +63,14 @@ this:
 Usage example
 =============
 
-In order to use the image pipeline you just need to :ref:`enable it
-<topics-images-enabling>` and define an item with the ``image_urls`` and
-``images`` fields::
+In order to use the image pipeline first  
+:ref:`enable it <topics-images-enabling>`.
+
+Then, if a spider returns a dict with 'image_urls' key, 
+the pipeline will put the results under 'images' key.
+
+If you prefer to use :class:`~.Item` then define a custom 
+item with the ``image_urls`` and ``images`` fields::
 
     import scrapy
 
@@ -74,7 +79,7 @@ In order to use the image pipeline you just need to :ref:`enable it
         # ... other item fields ...
         image_urls = scrapy.Field()
         images = scrapy.Field()
-
+        
 If you need something more complex and want to override the custom images
 pipeline behaviour, see :ref:`topics-images-override`.
 
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 7b66753b8..973c77516 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -8,8 +8,8 @@ After an item has been scraped by a spider, it is sent to the Item Pipeline
 which process it through several components that are executed sequentially.
 
 Each item pipeline component (sometimes referred as just "Item Pipeline") is a
-Python class that implements a simple method. They receive an Item and perform
-an action over it, also deciding if the Item should continue through the
+Python class that implements a simple method. They receive an item and perform
+an action over it, also deciding if the item should continue through the
 pipeline or be dropped and no longer processed.
 
 Typical use for item pipelines are:
@@ -28,12 +28,12 @@ Each item pipeline component is a Python class that must implement the following
 .. method:: process_item(self, item, spider)
 
    This method is called for every item pipeline component and must either return
-   a :class:`~scrapy.item.Item` (or any descendant class) object or raise a
-   :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
+   a dict with data, :class:`~scrapy.item.Item` (or any descendant class) object 
+   or raise a :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
    processed by further pipeline components.
 
    :param item: the item scraped
-   :type item: :class:`~scrapy.item.Item` object
+   :type item: :class:`~scrapy.item.Item` object or a dict
 
    :param spider: the spider which scraped the item
    :type spider: :class:`~scrapy.spider.Spider` object
@@ -135,6 +135,8 @@ method and how to clean up the resources properly.
     import pymongo
 
     class MongoPipeline(object):
+    
+        collection_name = 'scrapy_items'
 
         def __init__(self, mongo_uri, mongo_db):
             self.mongo_uri = mongo_uri
@@ -155,8 +157,7 @@ method and how to clean up the resources properly.
             self.client.close()
 
         def process_item(self, item, spider):
-            collection_name = item.__class__.__name__
-            self.db[collection_name].insert(dict(item))
+            self.db[self.collection_name].insert(dict(item))
             return item
 
 .. _MongoDB: http://www.mongodb.org/
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index ee604a7f1..3fda20494 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -8,12 +8,21 @@ Items
    :synopsis: Item and Field classes
 
 The main goal in scraping is to extract structured data from unstructured
-sources, typically, web pages. Scrapy provides the :class:`Item` class for this
-purpose.
+sources, typically, web pages. Scrapy spiders can return the extracted data
+as Python dicts. While convenient and familiar, Python dicts lack structure:
+it is easy to make a typo in a field name or return inconsistent data,
+especially in a larger project with many spiders.
 
+To define common output data format Scrapy provides the :class:`Item` class.
 :class:`Item` objects are simple containers used to collect the scraped data.
 They provide a `dictionary-like`_ API with a convenient syntax for declaring
-their available fields.
+their available fields. 
+
+Various Scrapy components use extra information provided by Items: 
+exporters look at declared fields to figure out columns to export,
+serialization can be customized using Item fields metadata, :mod:`trackref`
+tracks Item instances to help finding memory leaks 
+(see :ref:`topics-leaks-trackrefs`_), etc.
 
 .. _dictionary-like: http://docs.python.org/library/stdtypes.html#dict
 
@@ -64,8 +73,6 @@ It's important to note that the :class:`Field` objects used to declare the item
 do not stay assigned as class attributes. Instead, they can be accessed through
 the :attr:`Item.fields` attribute.
 
-And that's all you need to know about declaring items.
-
 Working with Items
 ==================
 
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 13dde52a3..9e65c07be 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -190,11 +190,10 @@ Dynamic Creation of Item Classes
 ================================
 
 For applications in which the structure of item class is to be determined by
-user input, or other changing conditions, you can dynamically create item
-classes instead of manually coding them.
-
-::
+user input or other changing conditions you can return regular Python 
+dicts from spiders.
 
+Another option is to dynamically create Item classes::
 
     from scrapy.item import DictItem, Field
 
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 405b131ed..85cf43c76 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -71,7 +71,7 @@ item_scraped
     This signal supports returning deferreds from their handlers.
 
     :param item: the item scraped
-    :type item: :class:`~scrapy.item.Item` object
+    :type item: dict or :class:`~scrapy.item.Item` object
 
     :param spider: the spider which scraped the item
     :type spider: :class:`~scrapy.spider.Spider` object
@@ -91,7 +91,7 @@ item_dropped
     This signal supports returning deferreds from their handlers.
 
     :param item: the item dropped from the :ref:`topics-item-pipeline`
-    :type item: :class:`~scrapy.item.Item` object
+    :type item: dict or :class:`~scrapy.item.Item` object
 
     :param spider: the spider which scraped the item
     :type spider: :class:`~scrapy.spider.Spider` object
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 79cfbcf5d..0104ae552 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -90,15 +90,16 @@ following methods:
         it has processed the response.
 
         :meth:`process_spider_output` must return an iterable of
-        :class:`~scrapy.http.Request` or :class:`~scrapy.item.Item` objects.
+        :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` 
+        objects.
 
         :param response: the response which generated this output from the
           spider
         :type response: :class:`~scrapy.http.Response` object
 
         :param result: the result returned by the spider
-        :type result: an iterable of :class:`~scrapy.http.Request` or
-          :class:`~scrapy.item.Item` objects
+        :type result: an iterable of :class:`~scrapy.http.Request`, dict
+          or :class:`~scrapy.item.Item` objects
 
         :param spider: the spider whose result is being processed
         :type spider: :class:`~scrapy.spider.Spider` object
@@ -110,7 +111,7 @@ following methods:
         method (from other spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
-        iterable of :class:`~scrapy.http.Response` or
+        iterable of :class:`~scrapy.http.Response`, dict or
         :class:`~scrapy.item.Item` objects.
 
         If it returns ``None``, Scrapy will continue processing this exception,

From f16a33f34e8226a38a2990c0609a8ad15a8cf011 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 19 Mar 2015 05:25:15 +0500
Subject: [PATCH 0130/2974] DOC change structure of spider docs:

* start with scrapy.Spider, then mention spider arguments,
  then describe generic spiders;
* change wording regarding start_urls/start_requests;
* show an example of start_requests vs start_urls;
* show an example of dicts as items;
* as defining Item is an optional step now, docs for Items are
  moved below Spider docs.
---
 docs/index.rst          |   8 +-
 docs/topics/spiders.rst | 171 +++++++++++++++++++++++-----------------
 2 files changed, 102 insertions(+), 77 deletions(-)

diff --git a/docs/index.rst b/docs/index.rst
index 0384dae3d..0474cd14b 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -56,9 +56,9 @@ Basic concepts
    :hidden:
 
    topics/commands
-   topics/items
    topics/spiders
    topics/selectors
+   topics/items
    topics/loaders
    topics/shell
    topics/item-pipeline
@@ -72,9 +72,6 @@ Basic concepts
 :doc:`topics/commands`
     Learn about the command-line tool used to manage your Scrapy project.
 
-:doc:`topics/items`
-    Define the data you want to scrape.
-
 :doc:`topics/spiders`
     Write the rules to crawl your websites.
 
@@ -84,6 +81,9 @@ Basic concepts
 :doc:`topics/shell`
     Test your extraction code in an interactive environment.
 
+:doc:`topics/items`
+    Define the data you want to scrape.
+
 :doc:`topics/loaders`
     Populate your items with the extracted data.
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index cb3f6caeb..036c4e744 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -24,8 +24,9 @@ For spiders, the scraping cycle goes through something like this:
    Requests.
 
 2. In the callback function, you parse the response (web page) and return either
-   :class:`~scrapy.item.Item` objects, :class:`~scrapy.http.Request` objects,
-   or an iterable of both. Those Requests will also contain a callback (maybe
+   dicts with extracted data, :class:`~scrapy.item.Item` objects, 
+   :class:`~scrapy.http.Request` objects, or an iterable of these objects. 
+   Those Requests will also contain a callback (maybe
    the same) and will then be downloaded by Scrapy and then their
    response handled by the specified callback.
 
@@ -41,70 +42,19 @@ Even though this cycle applies (more or less) to any kind of spider, there are
 different kinds of default spiders bundled into Scrapy for different purposes.
 We will talk about those types here.
 
-.. _spiderargs:
-
-Spider arguments
-================
-
-Spiders can receive arguments that modify their behaviour. Some common uses for
-spider arguments are to define the start URLs or to restrict the crawl to
-certain sections of the site, but they can be used to configure any
-functionality of the spider.
-
-Spider arguments are passed through the :command:`crawl` command using the
-``-a`` option. For example::
-
-    scrapy crawl myspider -a category=electronics
-
-Spiders receive arguments in their constructors::
-
-    import scrapy
-
-    class MySpider(scrapy.Spider):
-        name = 'myspider'
-
-        def __init__(self, category=None, *args, **kwargs):
-            super(MySpider, self).__init__(*args, **kwargs)
-            self.start_urls = ['http://www.example.com/categories/%s' % category]
-            # ...
-
-Spider arguments can also be passed through the Scrapyd ``schedule.json`` API.
-See `Scrapyd documentation`_.
-
 .. _topics-spiders-ref:
 
-Built-in spiders reference
-==========================
-
-Scrapy comes with some useful generic spiders that you can use, to subclass
-your spiders from. Their aim is to provide convenient functionality for a few
-common scraping cases, like following all links on a site based on certain
-rules, crawling from `Sitemaps`_, or parsing a XML/CSV feed.
-
-For the examples used in the following spiders, we'll assume you have a project
-with a ``TestItem`` declared in a ``myproject.items`` module::
-
-    import scrapy
-
-    class TestItem(scrapy.Item):
-        id = scrapy.Field()
-        name = scrapy.Field()
-        description = scrapy.Field()
-
-
-.. module:: scrapy.spider
-   :synopsis: Spiders base class, spider manager and spider middleware
-
-Spider
-------
+scrapy.Spider
+=============
 
 .. class:: Spider()
 
    This is the simplest spider, and the one from which every other spider
    must inherit from (either the ones that come bundled with Scrapy, or the ones
    that you write yourself). It doesn't provide any special functionality. It just
-   requests the given ``start_urls``/``start_requests``, and calls the spider's
-   method ``parse`` for each of the resulting responses.
+   provides a default :meth:`start_requests` implementation which sends requests from
+   the :attr:`start_urls` spider attribute and calls the spider's method ``parse`` 
+   for each of the resulting responses.
 
    .. attribute:: name
 
@@ -198,15 +148,18 @@ Spider
        the method to override. For example, if you need to start by logging in using
        a POST request, you could do::
 
-           def start_requests(self):
-               return [scrapy.FormRequest("http://www.example.com/login",
-                                          formdata={'user': 'john', 'pass': 'secret'},
-                                          callback=self.logged_in)]
+           class MySpider(scrapy.Spider):
+               name = 'myspider'
+                
+               def start_requests(self):
+                   return [scrapy.FormRequest("http://www.example.com/login",
+                                              formdata={'user': 'john', 'pass': 'secret'},
+                                              callback=self.logged_in)]
 
-           def logged_in(self, response):
-               # here you would extract links to follow and return Requests for
-               # each of them, with another callback
-               pass
+               def logged_in(self, response):
+                   # here you would extract links to follow and return Requests for
+                   # each of them, with another callback
+                   pass
 
    .. method:: make_requests_from_url(url)
 
@@ -231,7 +184,7 @@ Spider
 
        This method, as well as any other Request callback, must return an
        iterable of :class:`~scrapy.http.Request` and/or
-       :class:`~scrapy.item.Item` objects.
+       dicts or :class:`~scrapy.item.Item` objects.
 
        :param response: the response to parse
        :type response: :class:~scrapy.http.Response`
@@ -247,10 +200,6 @@ Spider
        Called when the spider closes. This method provides a shortcut to
        signals.connect() for the :signal:`spider_closed` signal.
 
-
-Spider example
-~~~~~~~~~~~~~~
-
 Let's see an example::
 
     import scrapy
@@ -268,10 +217,9 @@ Let's see an example::
         def parse(self, response):
             self.log('A response from %s just arrived!' % response.url)
 
-Another example returning multiple Requests and Items from a single callback::
+Return multiple Requests and items from a single callback::
 
     import scrapy
-    from myproject.items import MyItem
 
     class MySpider(scrapy.Spider):
         name = 'example.com'
@@ -282,12 +230,89 @@ Another example returning multiple Requests and Items from a single callback::
             'http://www.example.com/3.html',
         ]
 
+        def parse(self, response):
+            for h3 in response.xpath('//h3').extract():
+                yield {"title": h3}
+
+            for url in response.xpath('//a/@href').extract():
+                yield scrapy.Request(url, callback=self.parse)
+                
+Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; 
+to give data more structure you can use :ref:`topics-items`::
+
+    import scrapy
+    from myproject.items import MyItem
+
+    class MySpider(scrapy.Spider):
+        name = 'example.com'
+        allowed_domains = ['example.com']
+        
+        def start_requests(self):
+            yield scrapy.Request('http://www.example.com/1.html', self.parse)
+            yield scrapy.Request('http://www.example.com/2.html', self.parse)
+            yield scrapy.Request('http://www.example.com/3.html', self.parse)
+
         def parse(self, response):
             for h3 in response.xpath('//h3').extract():
                 yield MyItem(title=h3)
 
             for url in response.xpath('//a/@href').extract():
                 yield scrapy.Request(url, callback=self.parse)
+    
+.. _spiderargs:
+
+Spider arguments
+================
+
+Spiders can receive arguments that modify their behaviour. Some common uses for
+spider arguments are to define the start URLs or to restrict the crawl to
+certain sections of the site, but they can be used to configure any
+functionality of the spider.
+
+Spider arguments are passed through the :command:`crawl` command using the
+``-a`` option. For example::
+
+    scrapy crawl myspider -a category=electronics
+
+Spiders receive arguments in their constructors::
+
+    import scrapy
+
+    class MySpider(scrapy.Spider):
+        name = 'myspider'
+
+        def __init__(self, category=None, *args, **kwargs):
+            super(MySpider, self).__init__(*args, **kwargs)
+            self.start_urls = ['http://www.example.com/categories/%s' % category]
+            # ...
+
+Spider arguments can also be passed through the Scrapyd ``schedule.json`` API.
+See `Scrapyd documentation`_.
+
+.. _builtin-spiders:
+                
+Generic Spiders
+===============
+
+Scrapy comes with some useful generic spiders that you can use, to subclass
+your spiders from. Their aim is to provide convenient functionality for a few
+common scraping cases, like following all links on a site based on certain
+rules, crawling from `Sitemaps`_, or parsing a XML/CSV feed.
+
+For the examples used in the following spiders, we'll assume you have a project
+with a ``TestItem`` declared in a ``myproject.items`` module::
+
+    import scrapy
+
+    class TestItem(scrapy.Item):
+        id = scrapy.Field()
+        name = scrapy.Field()
+        description = scrapy.Field()
+
+
+.. module:: scrapy.spider
+   :synopsis: Spiders base class, spider manager and spider middleware
+
 
 .. module:: scrapy.contrib.spiders
    :synopsis: Collection of generic spiders

From 643984e1b4b573db737af161f1f4975a5af712fa Mon Sep 17 00:00:00 2001
From: Faisal Anees <faisal.iiit@gmail.com>
Date: Sun, 23 Mar 2014 01:00:37 +0530
Subject: [PATCH 0131/2974] Updated architecture.rst

Added http://krondo.com/blog/?page_id=1327 as a resource
---
 docs/topics/architecture.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index 80ccd42dd..700e6d92d 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -123,8 +123,10 @@ links:
 
 * `Introduction to Deferreds in Twisted`_
 * `Twisted - hello, asynchronous programming`_
+* `Twisted Introduction - Krondo`_
 
 .. _Twisted: http://twistedmatrix.com/trac/
 .. _Introduction to Deferreds in Twisted: http://twistedmatrix.com/documents/current/core/howto/defer-intro.html
 .. _Twisted - hello, asynchronous programming: http://jessenoller.com/2009/02/11/twisted-hello-asynchronous-programming/
+.. _Twisted Introduction - Krondo: http://krondo.com/blog/?page_id=1327/
 

From 8ac397670f319c541e9bfe25db0de34215f633c7 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 19 Mar 2015 21:41:36 +0500
Subject: [PATCH 0132/2974] DOC move .. module: declaration to a proper place

---
 docs/topics/spiders.rst | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 036c4e744..2985a7a89 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -42,6 +42,9 @@ Even though this cycle applies (more or less) to any kind of spider, there are
 different kinds of default spiders bundled into Scrapy for different purposes.
 We will talk about those types here.
 
+.. module:: scrapy.spider
+   :synopsis: Spiders base class, spider manager and spider middleware
+
 .. _topics-spiders-ref:
 
 scrapy.Spider
@@ -310,10 +313,6 @@ with a ``TestItem`` declared in a ``myproject.items`` module::
         description = scrapy.Field()
 
 
-.. module:: scrapy.spider
-   :synopsis: Spiders base class, spider manager and spider middleware
-
-
 .. module:: scrapy.contrib.spiders
    :synopsis: Collection of generic spiders
 

From d7cb2b9a918dcf4cf4594a05c0c1a3b88fdca413 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 19 Mar 2015 15:41:43 -0300
Subject: [PATCH 0133/2974] making commented code indentation consistent

---
 .../templates/project/module/settings.py.tmpl | 38 +++++++++----------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 302d96b17..37cbb4d32 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -19,7 +19,7 @@ NEWSPIDER_MODULE = '$project_name.spiders'
 #USER_AGENT = '$project_name (+http://www.yourdomain.com)'
 
 # Configure maximum concurrent requests performed by Scrapy (default: 16)
-# CONCURRENT_REQUESTS=32
+#CONCURRENT_REQUESTS=32
 
 # Configure a delay for requests for the same website (default: 0)
 # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
@@ -30,40 +30,40 @@ NEWSPIDER_MODULE = '$project_name.spiders'
 #CONCURRENT_REQUESTS_PER_IP=16
 
 # Disable cookies (enabled by default)
-# COOKIES_ENABLED=False
+#COOKIES_ENABLED=False
 
 # Disable Telnet Console (enabled by default)
-# TELNETCONSOLE_ENABLED=False
+#TELNETCONSOLE_ENABLED=False
 
 # Override the default request headers:
-# DEFAULT_REQUEST_HEADERS = {
-#    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-#    'Accept-Language': 'en',
-# }
+#DEFAULT_REQUEST_HEADERS = {
+#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#   'Accept-Language': 'en',
+#}
 
 # Enable or disable spider middlewares
 # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
-# SPIDER_MIDDLEWARES = {
-#     '$project_name.middlewares.MyCustomSpiderMiddleware': 543,
-# }
+#SPIDER_MIDDLEWARES = {
+#    '$project_name.middlewares.MyCustomSpiderMiddleware': 543,
+#}
 
 # Enable or disable downloader middlewares
 # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
-# DOWNLOADER_MIDDLEWARES = {
-#     '$project_name.middlewares.MyCustomDownloaderMiddleware': 543,
-# }
+#DOWNLOADER_MIDDLEWARES = {
+#    '$project_name.middlewares.MyCustomDownloaderMiddleware': 543,
+#}
 
 # Enable or disable extensions
 # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
-# EXTENSIONS = {
-#     'scrapy.telnet.TelnetConsole': None,
-# }
+#EXTENSIONS = {
+#    'scrapy.telnet.TelnetConsole': None,
+#}
 
 # Configure item pipelines
 # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
-# ITEM_PIPELINES = {
-#     '$project_name.pipelines.SomePipeline': 300,
-# }
+#ITEM_PIPELINES = {
+#    '$project_name.pipelines.SomePipeline': 300,
+#}
 
 # Enable and configure the AutoThrottle extension (disabled by default)
 # See http://doc.scrapy.org/en/latest/topics/autothrottle.html

From c81eefaf8148af70ca9214a0a3aa776eeb9b436c Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Thu, 19 Mar 2015 17:42:48 -0300
Subject: [PATCH 0134/2974] fix doc links

---
 docs/topics/downloader-middleware.rst | 2 +-
 docs/topics/telnetconsole.rst         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 149eef59e..df679daea 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -376,7 +376,7 @@ what is implemented:
 
 what is missing:
 
-* `Pragma: no-cache` support https://www.mnot.net/cache_docs/#PRAGMA
+* `Pragma: no-cache` support http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
 * `Vary` header support http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6
 * Invalidation after updates or deletes http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10
 * ... probably others ..
diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index 267e6e1d8..b1ff88771 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -66,7 +66,7 @@ convenience:
 | ``hpy``        | for memory debugging (see :ref:`topics-leaks`)                    |
 +----------------+-------------------------------------------------------------------+
 
-.. _pprint.pprint: http://docs.python.org/library/pprint.html#pprint.pprint
+.. _pprint.pprint: https://docs.python.org/library/pprint.html#pprint.pprint
 
 Telnet console usage examples
 =============================

From cda39225073ac06ef5563b7a0fd8f8a09194ad69 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 19 Mar 2015 16:59:52 -0300
Subject: [PATCH 0135/2974] Add Response.urljoin() helper

---
 docs/topics/request-response.rst | 12 ++++++++++++
 scrapy/http/response/__init__.py |  7 +++++++
 2 files changed, 19 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 302ed4f3b..5b4ced992 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -493,6 +493,18 @@ Response objects
        given new values by whichever keyword arguments are specified. The
        attribute :attr:`Response.meta` is copied by default.
 
+    .. method:: Response.urljoin(url)
+
+        Constructs an absolute url by combining the Response's :attr:`url` with
+        a possible relative url.
+
+        This is a wrapper over `urlparse.urljoin`_, it's merely an alias for
+        making this call::
+
+            urlparse.urljoin(response.url, url)
+
+.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
+
 .. _topics-request-response-ref-response-subclasses:
 
 Response subclasses
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 7ff683eb6..e138d5cc8 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -7,6 +7,8 @@ See documentation in docs/topics/request-response.rst
 
 import copy
 
+from six.moves.urllib.parse import urljoin
+
 from scrapy.http.headers import Headers
 from scrapy.utils.trackref import object_ref
 from scrapy.http.common import obsolete_setter
@@ -75,3 +77,8 @@ class Response(object_ref):
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)
+
+    def urljoin(self, url):
+        """Join this Response's url with a possible relative url to form an
+        absolute interpretation of the latter."""
+        return urljoin(self.url, url)

From 1b6d5a011a5485f3f2c01b32fa425b512d90d7ef Mon Sep 17 00:00:00 2001
From: drack3800 <domaso@mail.ru>
Date: Sat, 21 Mar 2015 04:02:51 +0300
Subject: [PATCH 0136/2974] Added webclient test for checking Content-Length
 header in response for POST request with no given body

---
 tests/test_webclient.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index a16eb1ccf..e0b46286a 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -132,6 +132,18 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             "\r\n"
             "name=value")
 
+        # test a POST method with no body provided
+        factory = client.ScrapyHTTPClientFactory(Request(
+            method='POST',
+            url='http://foo/bar'
+        ))
+
+        self._test(factory,
+                   "POST /bar HTTP/1.0\r\n"
+                   "Host: foo\r\n"
+                   "Content-Length: 0\r\n"
+                   "\r\n")
+
         # test with single and multivalued headers
         factory = client.ScrapyHTTPClientFactory(Request(
             url='http://foo/bar',

From deb5bb530cfe14993f92d3031820153ae53e0edb Mon Sep 17 00:00:00 2001
From: drack3800 <domaso@mail.ru>
Date: Sun, 22 Mar 2015 19:25:08 +0300
Subject: [PATCH 0137/2974] Fixed bug with no specified Content-Length header
 by ScrapyHTTPClientFactory for POST request with no given body

---
 scrapy/core/downloader/webclient.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 2c6a61b8a..93ab8a391 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -112,12 +112,14 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
 
         # set Host header based on url
         self.headers.setdefault('Host', self.netloc)
-
         # set Content-Length based len of body
         if self.body is not None:
             self.headers['Content-Length'] = len(self.body)
             # just in case a broken http/1.1 decides to keep connection alive
             self.headers.setdefault("Connection", "close")
+        # Content-Length must be specified in POST method even with no body
+        elif self.method == 'POST':
+            self.headers['Content-Length'] = 0
 
     def _build_response(self, body, request):
         request.meta['download_latency'] = self.headers_time-self.start_time

From 549882590bfdb3d66b623ea7414fac5097560a96 Mon Sep 17 00:00:00 2001
From: drack3800 <domaso@mail.ru>
Date: Sun, 22 Mar 2015 19:25:08 +0300
Subject: [PATCH 0138/2974] Fixed bug with no specified Content-Length header
 by ScrapyHTTPClientFactory for POST request with no given body

---
 scrapy/core/downloader/webclient.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 2c6a61b8a..22c461b15 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -112,12 +112,15 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
 
         # set Host header based on url
         self.headers.setdefault('Host', self.netloc)
-
+        
         # set Content-Length based len of body
         if self.body is not None:
             self.headers['Content-Length'] = len(self.body)
             # just in case a broken http/1.1 decides to keep connection alive
             self.headers.setdefault("Connection", "close")
+        # Content-Length must be specified in POST method even with no body
+        elif self.method == 'POST':
+            self.headers['Content-Length'] = 0
 
     def _build_response(self, body, request):
         request.meta['download_latency'] = self.headers_time-self.start_time

From 5ac91e488339c71c9fef9cb41a320e6ee9ed8278 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 23 Mar 2015 18:11:35 +0500
Subject: [PATCH 0139/2974] DOC remove Dynamic Creation of Item Classes section

It was a hack, and dicts-as-items cover most use cases.

Dicts don't allow to attach metadata to fields,
but e.g. adding "_meta" key and removing it in a custom serializer
is no worse than creating classes dynamically.
---
 docs/topics/practices.rst | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 9e65c07be..3ec7bc29b 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -183,21 +183,3 @@ If you are still unable to prevent your bot getting banned, consider contacting
 .. _testspiders: https://github.com/scrapinghub/testspiders
 .. _Twisted Reactor Overview: http://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
 .. _Crawlera: http://crawlera.com
-
-.. _dynamic-item-classes:
-
-Dynamic Creation of Item Classes
-================================
-
-For applications in which the structure of item class is to be determined by
-user input or other changing conditions you can return regular Python 
-dicts from spiders.
-
-Another option is to dynamically create Item classes::
-
-    from scrapy.item import DictItem, Field
-
-    def create_item_class(class_name, field_list):
-        fields = {field_name: Field() for field_name in field_list}
-
-        return type(class_name, (DictItem,), {'fields': fields})

From aaeb837db409442579d260f703e2f2ca705020ca Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Tue, 24 Mar 2015 07:11:48 +0000
Subject: [PATCH 0140/2974] handle TLS SNI if we have twisted>=14.0

(closes #981, #1101)
---
 scrapy/core/downloader/contextfactory.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index e20830c71..0b39b89d8 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,11 +1,17 @@
 from OpenSSL import SSL
 from twisted.internet.ssl import ClientContextFactory
+try:
+    # available since twisted 14.0
+    from twisted.internet._sslverify import ClientTLSOptions
+except ImportError:
+    ClientTLSOptions = None
 
 
 class ScrapyClientContextFactory(ClientContextFactory):
     "A SSL context factory which is more permissive against SSL bugs."
     # see https://github.com/scrapy/scrapy/issues/82
     # and https://github.com/scrapy/scrapy/issues/26
+    # and https://github.com/scrapy/scrapy/issues/981
 
     def __init__(self):
         # see this issue on why we use TLSv1_METHOD by default
@@ -17,4 +23,6 @@ class ScrapyClientContextFactory(ClientContextFactory):
         # Enable all workarounds to SSL bugs as documented by
         # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html
         ctx.set_options(SSL.OP_ALL)
+        if hostname and ClientTLSOptions is not None: # workaround for TLS SNI
+            ClientTLSOptions(hostname, ctx)
         return ctx

From ca2575001ef29af46b337ca8576cc93a4d2f8a73 Mon Sep 17 00:00:00 2001
From: Ramiro Morales <ramiro@users.noreply.github.com>
Date: Wed, 25 Mar 2015 18:32:20 -0300
Subject: [PATCH 0141/2974] Add missing callback arg in jobs topic example.

---
 docs/topics/jobs.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 4e09000d2..38d77534a 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -90,7 +90,7 @@ But this will::
 
     def some_callback(self, response):
         somearg = 'test'
-        return scrapy.Request('http://www.example.com', meta={'somearg': somearg})
+        return scrapy.Request('http://www.example.com', , callback=self.other_callback, meta={'somearg': somearg})
 
     def other_callback(self, response):
         somearg = response.meta['somearg']

From 933dbc6be6c1dac64fe31e34c7943e050eff254a Mon Sep 17 00:00:00 2001
From: Ramiro Morales <ramiro@users.noreply.github.com>
Date: Wed, 25 Mar 2015 18:33:17 -0300
Subject: [PATCH 0142/2974] Oops

---
 docs/topics/jobs.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 38d77534a..303076015 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -90,7 +90,7 @@ But this will::
 
     def some_callback(self, response):
         somearg = 'test'
-        return scrapy.Request('http://www.example.com', , callback=self.other_callback, meta={'somearg': somearg})
+        return scrapy.Request('http://www.example.com', callback=self.other_callback, meta={'somearg': somearg})
 
     def other_callback(self, response):
         somearg = response.meta['somearg']

From 32423d4a330e8af72ae077fac6347812d57bca08 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 25 Mar 2015 19:24:36 -0300
Subject: [PATCH 0143/2974] some improvements to overview page

---
 docs/intro/overview.rst | 253 +++++++++++-----------------------------
 1 file changed, 65 insertions(+), 188 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index c30963db8..9a3015ddc 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -13,172 +13,83 @@ precisely, `web scraping`_), it can also be used to extract data using APIs
 (such as `Amazon Associates Web Services`_) or as a general purpose web
 crawler.
 
-The purpose of this document is to introduce you to the concepts behind Scrapy
-so you can get an idea of how it works and decide if Scrapy is what you need.
 
-When you're ready to start a project, you can :ref:`start with the tutorial
-<intro-tutorial>`.
+Walk-through of an example spider
+=================================
 
-Pick a website
-==============
+In order to show you what Scrapy brings to the table, we'll walk you
+through an example of a Scrapy Spider using the simplest way to run a spider.
 
-So you need to extract some information from a website, but the website doesn't
-provide any API or mechanism to access that info programmatically.  Scrapy can
-help you extract that information.
+Once you're ready to dive in more, you can :ref:`follow the tutorial
+and build a full-blown Scrapy project <intro-tutorial>`.
 
-Let's say we want to extract the URL, name, description and size of all torrent
-files added today in the `Mininova`_ site.
-
-The list of all torrents added today can be found on this page:
-
-    http://www.mininova.org/today
-
-.. _intro-overview-item:
-
-Define the data you want to scrape
-==================================
-
-The first thing is to define the data we want to scrape. In Scrapy, this is
-done through :ref:`Scrapy Items <topics-items>` (Torrent files, in this case).
-
-This would be our Item::
+So, here's the code for a spider that follows the links to the top
+voted questions on StackOverflow and scrapes some data from each page::
 
     import scrapy
 
-    class TorrentItem(scrapy.Item):
-        url = scrapy.Field()
-        name = scrapy.Field()
-        description = scrapy.Field()
-        size = scrapy.Field()
 
-Write a Spider to extract the data
-==================================
+    class StackOverflowSpider(scrapy.Spider):
+        name = 'stackoverflow'
+        start_urls = ['http://stackoverflow.com/questions?sort=votes']
 
-The next thing is to write a Spider which defines the start URL
-(http://www.mininova.org/today), the rules for following links and the rules
-for extracting the data from pages.
+        def parse(self, response):
+            for href in response.css('.question-summary h3 a::attr(href)'):
+                full_url = response.urljoin(href.extract())
+                yield scrapy.Request(full_url, callback=self.parse_question)
 
-If we take a look at that page content we'll see that all torrent URLs are like
-``http://www.mininova.org/tor/NUMBER`` where ``NUMBER`` is an integer. We'll use
-that to construct the regular expression for the links to follow: ``/tor/\d+``.
-
-We'll use `XPath`_ for selecting the data to extract from the web page HTML
-source. Let's take one of those torrent pages:
-
-    http://www.mininova.org/tor/2676093
-
-And look at the page HTML source to construct the XPath to select the data we
-want which is: torrent name, description and size.
-
-.. highlight:: html
-
-By looking at the page HTML source we can see that the file name is contained
-inside a ``<h1>`` tag::
-
-   <h1>Darwin - The Evolution Of An Exhibition</h1>
-
-.. highlight:: none
-
-An XPath expression to extract the name could be::
-
-    //h1/text()
-
-.. highlight:: html
-
-And the description is contained inside a ``<div>`` tag with ``id="description"``::
-
-   <h2>Description:</h2>
-
-   <div id="description">
-   Short documentary made for Plymouth City Museum and Art Gallery regarding the setup of an exhibit about Charles Darwin in conjunction with the 200th anniversary of his birth.
-
-   ...
-
-.. highlight:: none
-
-An XPath expression to select the description could be::
-
-    //div[@id='description']
-
-.. highlight:: html
-
-Finally, the file size is contained in the second ``<p>`` tag inside the ``<div>``
-tag with ``id=specifications``::
-
-   <div id="specifications">
-
-   <p>
-   <strong>Category:</strong>
-   <a href="/cat/4">Movies</a> &gt; <a href="/sub/35">Documentary</a>
-   </p>
-
-   <p>
-   <strong>Total size:</strong>
-   150.62&nbsp;megabyte</p>
+        def parse_question(self, response):
+            title = response.css('h1 a::text').extract_first()
+            votes = response.css('.question .vote-count-post::text').extract_first()
+            tags = response.css('.question .post-tag::text').extract()
+            body = response.css('.question .post-text').extract_first()
+            yield {
+                'title': title,
+                'votes': votes,
+                'body': body,
+                'tags': tags,
+                'link': response.url,
+            }
 
 
-.. highlight:: none
+Put this in a file, name it to something like ``stackoverflow_spider.py``
+and run the spider using the :command:`runspider` command::
 
-An XPath expression to select the file size could be::
+    scrapy runspider stackoverflow_spider.py -o top-stackoverflow-questions.json
 
-   //div[@id='specifications']/p[2]/text()[2]
 
-.. highlight:: python
+When this finishes you will have in the ``top-stackoverflow-questions.json`` file
+a list of the most upvoted questions in StackOverflow in JSON format, containing the
+title, link, number of upvotes, a list of the tags and the question content in HTML.
 
-For more information about XPath see the `XPath reference`_.
 
-Finally, here's the spider code::
+What just happened?
+-------------------
 
-    from scrapy.contrib.spiders import CrawlSpider, Rule
-    from scrapy.contrib.linkextractors import LinkExtractor
+When you ran the command ``scrapy runspider somefile.py``, Scrapy looked
+for a Spider definition inside it and ran it through its crawler engine.
 
-    class MininovaSpider(CrawlSpider):
+The crawl started by making requests to the URLs defined in the ``start_urls``
+attribute (in this case, only the URL for StackOverflow top questions page),
+and then called the default callback method ``parse`` passing the response
+object as an argument.
 
-        name = 'mininova'
-        allowed_domains = ['mininova.org']
-        start_urls = ['http://www.mininova.org/today']
-        rules = [Rule(LinkExtractor(allow=['/tor/\d+']), 'parse_torrent')]
+In the ``parse`` callback, we scrape the links to the questions and
+yield a few more requests to be processed, registering for them
+the method ``parse_question`` as the callback to be called when the
+requests are complete.
 
-        def parse_torrent(self, response):
-            torrent = TorrentItem()
-            torrent['url'] = response.url
-            torrent['name'] = response.xpath("//h1/text()").extract()
-            torrent['description'] = response.xpath("//div[@id='description']").extract()
-            torrent['size'] = response.xpath("//div[@id='info-left']/p[2]/text()[2]").extract()
-            return torrent
+Finally, the ``parse_question`` callback scrapes the question data
+for each page yielding a dict, which Scrapy then collects and
+writes to a JSON file as requested in the command line.
 
-The ``TorrentItem`` class is :ref:`defined above <intro-overview-item>`.
+.. note::
 
-Run the spider to extract the data
-==================================
+    This is using :ref:`feed exports <topics-feed-exports>` to generate the
+    JSON file, you can easily change the export format (XML or CSV, for example) or the
+    storage backend (FTP or `Amazon S3`_, for example).  You can also write an
+    :ref:`item pipeline <topics-item-pipeline>` to store the items in a database.
 
-Finally, we'll run the spider to crawl the site and output the file
-``scraped_data.json`` with the scraped data in JSON format::
-
-    scrapy crawl mininova -o scraped_data.json
-
-This uses :ref:`feed exports <topics-feed-exports>` to generate the JSON file.
-You can easily change the export format (XML or CSV, for example) or the
-storage backend (FTP or `Amazon S3`_, for example).
-
-You can also write an :ref:`item pipeline <topics-item-pipeline>` to store the
-items in a database very easily.
-
-Review scraped data
-===================
-
-If you check the ``scraped_data.json`` file after the process finishes, you'll
-see the scraped items there::
-
-    [{"url": "http://www.mininova.org/tor/2676093", "name": ["Darwin - The Evolution Of An Exhibition"], "description": ["Short documentary made for Plymouth ..."], "size": ["150.62 megabyte"]},
-    # ... other items ...
-    ]
-
-You'll notice that all field values (except for the ``url`` which was assigned
-directly) are actually lists. This is because the :ref:`selectors
-<topics-selectors>` return lists. You may want to store single values, or
-perform some additional parsing/cleansing to the values. That's what
-:ref:`Item Loaders <topics-loaders>` are for.
 
 .. _topics-whatelse:
 
@@ -189,68 +100,37 @@ You've seen how to extract and store items from a website using Scrapy, but
 this is just the surface. Scrapy provides a lot of powerful features for making
 scraping easy and efficient, such as:
 
-* Built-in support for :ref:`selecting and extracting <topics-selectors>` data
-  from HTML and XML sources
-
-* Built-in support for cleaning and sanitizing the scraped data using a
-  collection of reusable filters (called :ref:`Item Loaders <topics-loaders>`)
-  shared between all the spiders.
+* An :ref:`interactive shell console <topics-shell>` (IPython aware) for trying
+  out the CSS and XPath expressions to scrape data, very useful when writing or
+  debugging your spiders.
 
 * Built-in support for :ref:`generating feed exports <topics-feed-exports>` in
   multiple formats (JSON, CSV, XML) and storing them in multiple backends (FTP,
   S3, local filesystem)
 
-* A media pipeline for :ref:`automatically downloading images <topics-images>`
-  (or any other media) associated with the scraped items
-
-* Support for :ref:`extending Scrapy <extending-scrapy>` by plugging
-  your own functionality using :ref:`signals <topics-signals>` and a
-  well-defined API (middlewares, :ref:`extensions <topics-extensions>`, and
-  :ref:`pipelines <topics-item-pipeline>`).
-
-* Wide range of built-in middlewares and extensions for:
-
-  * cookies and session handling
-  * HTTP compression
-  * HTTP authentication
-  * HTTP cache
-  * user-agent spoofing
-  * robots.txt
-  * crawl depth restriction
-  * and more
-
 * Robust encoding support and auto-detection, for dealing with foreign,
   non-standard and broken encoding declarations.
 
-* Support for creating spiders based on pre-defined templates, to speed up
-  spider creation and make their code more consistent on large projects. See
-  :command:`genspider` command for more details.
-
-* Extensible :ref:`stats collection <topics-stats>` for multiple spider
-  metrics, useful for monitoring the performance of your spiders and detecting
-  when they get broken
-
-* An :ref:`Interactive shell console <topics-shell>` for trying XPaths, very
-  useful for writing and debugging your spiders
-
-* A :ref:`System service <topics-scrapyd>` designed to ease the deployment and
-  run of your spiders in production.
+* Strong :ref:`extensibility support <extending-scrapy>` and lots of built-in
+  extensions and middlewares to handle things like cookies, crawl throttling,
+  HTTP caching, HTTP compression, user-agent spoofing, robots.txt,
+  stats collection and many more.
 
 * A :ref:`Telnet console <topics-telnetconsole>` for hooking into a Python
   console running inside your Scrapy process, to introspect and debug your
   crawler
 
-* :ref:`Logging <topics-logging>` facility that you can hook on to for catching
-  errors during the scraping process.
+* A caching DNS resolver
 
 * Support for crawling based on URLs discovered through `Sitemaps`_
 
-* A caching DNS resolver
+* A media pipeline for :ref:`automatically downloading images <topics-images>`
+  (or any other media) associated with the scraped items
 
 What's next?
 ============
 
-The next obvious steps are for you to `download Scrapy`_, read :ref:`the
+The next obvious steps for you are to `download Scrapy`_, read :ref:`the
 tutorial <intro-tutorial>` and join `the community`_. Thanks for your
 interest!
 
@@ -258,9 +138,6 @@ interest!
 .. _the community: http://scrapy.org/community/
 .. _screen scraping: http://en.wikipedia.org/wiki/Screen_scraping
 .. _web scraping: http://en.wikipedia.org/wiki/Web_scraping
-.. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html
-.. _Mininova: http://www.mininova.org
-.. _XPath: http://www.w3.org/TR/xpath
-.. _XPath reference: http://www.w3.org/TR/xpath
+.. _Amazon Associates Web Services: http://aws.amazon.com/associates/
 .. _Amazon S3: http://aws.amazon.com/s3/
 .. _Sitemaps: http://www.sitemaps.org

From 8f4a268f3757fe663de49e80dfd9325c260fcc73 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 26 Mar 2015 12:14:56 -0300
Subject: [PATCH 0144/2974] added bit about async requests, improved phrasing

---
 docs/intro/overview.rst | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 9a3015ddc..295208bd2 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -74,10 +74,16 @@ attribute (in this case, only the URL for StackOverflow top questions page),
 and then called the default callback method ``parse`` passing the response
 object as an argument.
 
-In the ``parse`` callback, we scrape the links to the questions and
-yield a few more requests to be processed, registering for them
-the method ``parse_question`` as the callback to be called when the
-requests are complete.
+Here you notice one of the main advantages about Scrapy: requests are
+scheduled and processed asynchronously. This means that Scrapy doesn't
+need to wait for a request to be finished and processed, it can send
+another request or do other things in the meantime, which results in much
+faster crawlings.
+
+So, in the ``parse`` callback, we scrape the links to the questions and
+yield a few more requests to be done, registering for them the method
+``parse_question`` as the callback to be called for each of them as
+they finish.
 
 Finally, the ``parse_question`` callback scrapes the question data
 for each page yielding a dict, which Scrapy then collects and

From 76e3bf12508f51c8c7b532b1a55880e55e144f19 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 26 Mar 2015 14:26:20 -0300
Subject: [PATCH 0145/2974] addressing comments from the review plus further
 editing

---
 docs/intro/overview.rst      | 83 +++++++++++++++++++++++-------------
 docs/topics/autothrottle.rst |  2 +
 2 files changed, 56 insertions(+), 29 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 295208bd2..d93797cae 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -39,15 +39,11 @@ voted questions on StackOverflow and scrapes some data from each page::
                 yield scrapy.Request(full_url, callback=self.parse_question)
 
         def parse_question(self, response):
-            title = response.css('h1 a::text').extract_first()
-            votes = response.css('.question .vote-count-post::text').extract_first()
-            tags = response.css('.question .post-tag::text').extract()
-            body = response.css('.question .post-text').extract_first()
             yield {
-                'title': title,
-                'votes': votes,
-                'body': body,
-                'tags': tags,
+                'title': response.css('h1 a::text').extract()[0],
+                'votes': response.css('.question .vote-count-post::text').extract()[0],
+                'body': response.css('.question .post-text').extract()[0],
+                'tags': response.css('.question .post-tag::text').extract(),
                 'link': response.url,
             }
 
@@ -66,28 +62,36 @@ title, link, number of upvotes, a list of the tags and the question content in H
 What just happened?
 -------------------
 
-When you ran the command ``scrapy runspider somefile.py``, Scrapy looked
-for a Spider definition inside it and ran it through its crawler engine.
+When you ran the command ``scrapy runspider somefile.py``, Scrapy looked for a
+Spider definition inside it and ran it through its crawler engine.
 
 The crawl started by making requests to the URLs defined in the ``start_urls``
 attribute (in this case, only the URL for StackOverflow top questions page),
-and then called the default callback method ``parse`` passing the response
-object as an argument.
+and called the default callback method ``parse`` passing the response object as
+an argument. In the ``parse`` callback, we extract the links to the
+question pages using a CSS Selector with a custom extension that allows to get
+the value for an attribute. Then, we yield a few more requests to be sent,
+registering the method ``parse_question`` as the callback to be called for each
+of them as they finish.
 
 Here you notice one of the main advantages about Scrapy: requests are
-scheduled and processed asynchronously. This means that Scrapy doesn't
-need to wait for a request to be finished and processed, it can send
-another request or do other things in the meantime, which results in much
-faster crawlings.
+:ref:`scheduled and processed asynchronously <topics-architecture>`.  This
+means that Scrapy doesn't need to wait for a request to be finished and
+processed, it can send another request or do other things in the meantime. This
+also means that other requests can keep going even if some request fails or an
+error happens while handling it.
 
-So, in the ``parse`` callback, we scrape the links to the questions and
-yield a few more requests to be done, registering for them the method
-``parse_question`` as the callback to be called for each of them as
-they finish.
+While this enables you to do very fast crawlings sending multiple concurrent
+requests at the same time in a fault-tolerant way, Scrapy also gives you
+control over the politeness of the crawl through :ref:`a few settings
+<topics-settings-ref>`. You can do things like setting a download delay between
+each request, limit amount of concurrent requests per domain or per IP, and
+even :ref:`use an auto-throttling extension <topics-autothrottle>` that tries
+to figure out these automatically.
 
-Finally, the ``parse_question`` callback scrapes the question data
-for each page yielding a dict, which Scrapy then collects and
-writes to a JSON file as requested in the command line.
+Finally, the ``parse_question`` callback scrapes the question data for each
+page yielding a dict, which Scrapy then collects and writes to a JSON file as
+requested in the command line.
 
 .. note::
 
@@ -96,6 +100,25 @@ writes to a JSON file as requested in the command line.
     storage backend (FTP or `Amazon S3`_, for example).  You can also write an
     :ref:`item pipeline <topics-item-pipeline>` to store the items in a database.
 
+The data in the file will look like this (note: formatted for easier reading)::
+
+
+    [{
+            "body": "... LONG HTML HERE ...",
+            "link": "http://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster-than-an-unsorted-array",
+            "tags": ["java", "c++", "performance", "optimization"],
+            "title": "Why is processing a sorted array faster than an unsorted array?",
+            "votes": "9924"
+        },
+        {
+            "body": "... LONG HTML HERE ...",
+            "link": "http://stackoverflow.com/questions/1260748/how-do-i-remove-a-git-submodule",
+            "tags": ["git", "git-submodules"],
+            "title": "How do I remove a Git submodule?",
+            "votes": "1764"
+        },
+        ...]
+
 
 .. _topics-whatelse:
 
@@ -106,6 +129,10 @@ You've seen how to extract and store items from a website using Scrapy, but
 this is just the surface. Scrapy provides a lot of powerful features for making
 scraping easy and efficient, such as:
 
+* Built-in support for :ref:`selecting and extracting <topics-selectors>` data
+  from HTML/XML sources using CSS selectors extended and XPath expressions,
+  with helper methods to extract using regular expressions.
+
 * An :ref:`interactive shell console <topics-shell>` (IPython aware) for trying
   out the CSS and XPath expressions to scrape data, very useful when writing or
   debugging your spiders.
@@ -126,12 +153,10 @@ scraping easy and efficient, such as:
   console running inside your Scrapy process, to introspect and debug your
   crawler
 
-* A caching DNS resolver
-
-* Support for crawling based on URLs discovered through `Sitemaps`_
-
-* A media pipeline for :ref:`automatically downloading images <topics-images>`
-  (or any other media) associated with the scraped items
+* Plus other goodies like reusable spiders to crawl sites from `Sitemaps`_ and
+  XML/CSV feeds, a media pipeline for :ref:`automatically downloading images <topics-images>`
+  (or any other media) associated with the scraped items, a caching DNS resolver,
+  and much more!
 
 What's next?
 ============
diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst
index 475946a54..8073ec6e0 100644
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@@ -1,3 +1,5 @@
+.. _topics-autothrottle:
+
 ======================
 AutoThrottle extension
 ======================

From 13d0ecde77cc33a8f81d7165a47ed2dc9c2c5a58 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 26 Mar 2015 15:26:16 -0300
Subject: [PATCH 0146/2974] addressing more review comments, to avoid ambiguity
 on desired reading flow

---
 docs/intro/overview.rst | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index d93797cae..65b26613d 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -17,11 +17,8 @@ crawler.
 Walk-through of an example spider
 =================================
 
-In order to show you what Scrapy brings to the table, we'll walk you
-through an example of a Scrapy Spider using the simplest way to run a spider.
-
-Once you're ready to dive in more, you can :ref:`follow the tutorial
-and build a full-blown Scrapy project <intro-tutorial>`.
+In order to show you what Scrapy brings to the table, we'll walk you through an
+example of a Scrapy Spider using the simplest way to run a spider.
 
 So, here's the code for a spider that follows the links to the top
 voted questions on StackOverflow and scrapes some data from each page::
@@ -144,10 +141,18 @@ scraping easy and efficient, such as:
 * Robust encoding support and auto-detection, for dealing with foreign,
   non-standard and broken encoding declarations.
 
-* Strong :ref:`extensibility support <extending-scrapy>` and lots of built-in
-  extensions and middlewares to handle things like cookies, crawl throttling,
-  HTTP caching, HTTP compression, user-agent spoofing, robots.txt,
-  stats collection and many more.
+* :ref:`Strong extensibility support <extending-scrapy>`, allowing you to plug
+  in your own functionality using :ref:`signals <topics-signals>` and a
+  well-defined API (middlewares, :ref:`extensions <topics-extensions>`, and
+  :ref:`pipelines <topics-item-pipeline>`).
+
+* Wide range of built-in extensions and middlewares for handling:
+    * cookies and session handling
+    * HTTP features like compression, authentication, caching
+    * user-agent spoofing
+    * robots.txt
+    * crawl depth restriction
+    * and more
 
 * A :ref:`Telnet console <topics-telnetconsole>` for hooking into a Python
   console running inside your Scrapy process, to introspect and debug your
@@ -161,11 +166,11 @@ scraping easy and efficient, such as:
 What's next?
 ============
 
-The next obvious steps for you are to `download Scrapy`_, read :ref:`the
-tutorial <intro-tutorial>` and join `the community`_. Thanks for your
+The next steps for you are to :ref:`install Scrapy <intro-install>`,
+:ref:`follow through the tutorial <intro-tutorial>` to learn how to organize
+your code in Scrapy projects and `join the community`_. Thanks for your
 interest!
 
-.. _download Scrapy: http://scrapy.org/download/
 .. _the community: http://scrapy.org/community/
 .. _screen scraping: http://en.wikipedia.org/wiki/Screen_scraping
 .. _web scraping: http://en.wikipedia.org/wiki/Web_scraping

From 729861c8644b0ec3b21dd2b4d2ba15952e274197 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 26 Mar 2015 15:31:42 -0300
Subject: [PATCH 0147/2974] fixing indentation

---
 docs/intro/overview.rst | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 65b26613d..395e1227c 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -97,24 +97,24 @@ requested in the command line.
     storage backend (FTP or `Amazon S3`_, for example).  You can also write an
     :ref:`item pipeline <topics-item-pipeline>` to store the items in a database.
 
-The data in the file will look like this (note: formatted for easier reading)::
+The data in the file will look like this (note: reformatted for easier reading)::
 
 
     [{
-            "body": "... LONG HTML HERE ...",
-            "link": "http://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster-than-an-unsorted-array",
-            "tags": ["java", "c++", "performance", "optimization"],
-            "title": "Why is processing a sorted array faster than an unsorted array?",
-            "votes": "9924"
-        },
-        {
-            "body": "... LONG HTML HERE ...",
-            "link": "http://stackoverflow.com/questions/1260748/how-do-i-remove-a-git-submodule",
-            "tags": ["git", "git-submodules"],
-            "title": "How do I remove a Git submodule?",
-            "votes": "1764"
-        },
-        ...]
+        "body": "... LONG HTML HERE ...",
+        "link": "http://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster-than-an-unsorted-array",
+        "tags": ["java", "c++", "performance", "optimization"],
+        "title": "Why is processing a sorted array faster than an unsorted array?",
+        "votes": "9924"
+    },
+    {
+        "body": "... LONG HTML HERE ...",
+        "link": "http://stackoverflow.com/questions/1260748/how-do-i-remove-a-git-submodule",
+        "tags": ["git", "git-submodules"],
+        "title": "How do I remove a Git submodule?",
+        "votes": "1764"
+    },
+    ...]
 
 
 .. _topics-whatelse:

From 7402e27230958ee2c89275676969e0cc844d6e4b Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 26 Mar 2015 15:35:31 -0300
Subject: [PATCH 0148/2974] fix community link

---
 docs/intro/overview.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 395e1227c..6b0a2c2bf 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -171,7 +171,7 @@ The next steps for you are to :ref:`install Scrapy <intro-install>`,
 your code in Scrapy projects and `join the community`_. Thanks for your
 interest!
 
-.. _the community: http://scrapy.org/community/
+.. _join the community: http://scrapy.org/community/
 .. _screen scraping: http://en.wikipedia.org/wiki/Screen_scraping
 .. _web scraping: http://en.wikipedia.org/wiki/Web_scraping
 .. _Amazon Associates Web Services: http://aws.amazon.com/associates/

From 4dcecc98f919103b9886fddbcc5e6273378854cb Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 26 Mar 2015 15:45:17 -0300
Subject: [PATCH 0149/2974] moved example data to a better place

---
 docs/intro/overview.rst | 39 +++++++++++++++++++--------------------
 1 file changed, 19 insertions(+), 20 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 6b0a2c2bf..18a877cdc 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -53,7 +53,25 @@ and run the spider using the :command:`runspider` command::
 
 When this finishes you will have in the ``top-stackoverflow-questions.json`` file
 a list of the most upvoted questions in StackOverflow in JSON format, containing the
-title, link, number of upvotes, a list of the tags and the question content in HTML.
+title, link, number of upvotes, a list of the tags and the question content in HTML,
+looking like this (reformatted for easier reading)::
+
+    [{
+        "body": "... LONG HTML HERE ...",
+        "link": "http://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster-than-an-unsorted-array",
+        "tags": ["java", "c++", "performance", "optimization"],
+        "title": "Why is processing a sorted array faster than an unsorted array?",
+        "votes": "9924"
+    },
+    {
+        "body": "... LONG HTML HERE ...",
+        "link": "http://stackoverflow.com/questions/1260748/how-do-i-remove-a-git-submodule",
+        "tags": ["git", "git-submodules"],
+        "title": "How do I remove a Git submodule?",
+        "votes": "1764"
+    },
+    ...]
+
 
 
 What just happened?
@@ -97,25 +115,6 @@ requested in the command line.
     storage backend (FTP or `Amazon S3`_, for example).  You can also write an
     :ref:`item pipeline <topics-item-pipeline>` to store the items in a database.
 
-The data in the file will look like this (note: reformatted for easier reading)::
-
-
-    [{
-        "body": "... LONG HTML HERE ...",
-        "link": "http://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster-than-an-unsorted-array",
-        "tags": ["java", "c++", "performance", "optimization"],
-        "title": "Why is processing a sorted array faster than an unsorted array?",
-        "votes": "9924"
-    },
-    {
-        "body": "... LONG HTML HERE ...",
-        "link": "http://stackoverflow.com/questions/1260748/how-do-i-remove-a-git-submodule",
-        "tags": ["git", "git-submodules"],
-        "title": "How do I remove a Git submodule?",
-        "votes": "1764"
-    },
-    ...]
-
 
 .. _topics-whatelse:
 

From 475766c73a4fd6f65caa5c5c25e713eab8eac84b Mon Sep 17 00:00:00 2001
From: Peter Bronez <pbronez@deepmile.com>
Date: Thu, 26 Mar 2015 15:34:30 -0400
Subject: [PATCH 0150/2974] Converted sel.xpath() calls to response.xpath() in
 Extracting the data

---
 docs/intro/tutorial.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index ad808316b..286a591ee 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -315,19 +315,19 @@ is inside a ``<ul>`` element, in fact the *second* ``<ul>`` element.
 So we can select each ``<li>`` element belonging to the sites list with this
 code::
 
-    sel.xpath('//ul/li')
+    response.xpath('//ul/li')
 
 And from them, the sites descriptions::
 
-    sel.xpath('//ul/li/text()').extract()
+    response.xpath('//ul/li/text()').extract()
 
 The sites titles::
 
-    sel.xpath('//ul/li/a/text()').extract()
+    response.xpath('//ul/li/a/text()').extract()
 
 And the sites links::
 
-    sel.xpath('//ul/li/a/@href').extract()
+    response.xpath('//ul/li/a/@href').extract()
 
 As we've said before, each ``.xpath()`` call returns a list of selectors, so we can
 concatenate further ``.xpath()`` calls to dig deeper into a node. We are going to use

From 1134a9cab055969b3232a16037d16e5ab93759f0 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Tue, 24 Mar 2015 03:12:58 +0000
Subject: [PATCH 0151/2974] config: look in ~/.config/scrapy.cfg as well

---
 docs/topics/commands.rst | 21 +++++++++++++++++++++
 docs/topics/settings.rst |  2 ++
 scrapy/utils/conf.py     |  3 +++
 3 files changed, 26 insertions(+)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 5c0de0d6d..27d0655ef 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -13,6 +13,27 @@ just call "commands" or "Scrapy commands".
 The Scrapy tool provides several commands, for multiple purposes, and each one
 accepts a different set of arguments and options.
 
+Configuration settings
+======================
+
+Scrapy will look for configuration parameters in ini-style ``scrapy.cfg`` files
+in standard locations:
+
+1. ``/etc/scrapy.cfg`` or ``c:\scrapy\scrapy.cfg`` (system-wide),
+2. ``~/.config/scrapy.cfg`` (``$XDG_CONFIG_HOME``) and ``~/.scrapy.cfg`` (``$HOME``)
+  for global (user-wide) settings, and
+3. ``scrapy.cfg`` inside a scrapy project's root (see next section).
+
+Settings from these files are merged in the listed order of preference:
+user-defined values have higher priority than system-wide defaults
+and project-wide settings will override all others, when defined.
+
+Scrapy also understands, and can be configured through, a number of environment
+variables. Currently these are:
+
+* ``SCRAPY_SETTINGS_MODULE`` (See :ref:`topics-settings-module-envvar`)
+* ``SCRAPY_PROJECT``
+
 .. _topics-project-structure:
 
 Default structure of Scrapy projects
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 5e11e473f..0f41c427b 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -16,6 +16,8 @@ project (in case you have many).
 
 For a list of available built-in settings see: :ref:`topics-settings-ref`.
 
+.. _topics-settings-module-envvar:
+
 Designating the settings
 ========================
 
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index caa80a5a1..b883923b9 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -63,7 +63,10 @@ def get_config(use_closest=True):
 
 
 def get_sources(use_closest=True):
+    xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \
+        os.path.expanduser('~/.config')
     sources = ['/etc/scrapy.cfg', r'c:\scrapy\scrapy.cfg',
+               xdg_config_home + '/scrapy.cfg',
                os.path.expanduser('~/.scrapy.cfg')]
     if use_closest:
         sources.append(closest_scrapy_cfg())

From 84edc2ebc7afcdfd6b4789a6e1725b83eb9076ab Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Tue, 24 Mar 2015 11:42:33 +0000
Subject: [PATCH 0152/2974] Add Response.urljoin() testcase

and add evaluation of base-url for HtmlResponse.
---
 scrapy/http/response/__init__.py |  2 +-
 scrapy/http/response/text.py     |  8 ++++++++
 scrapy/utils/response.py         |  2 +-
 tests/test_http_response.py      | 23 +++++++++++++++++++++++
 4 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index e138d5cc8..5b1d34c6d 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -55,7 +55,7 @@ class Response(object_ref):
         elif body is None:
             self._body = ''
         else:
-            raise TypeError("Response body must either str or unicode. Got: '%s'" \
+            raise TypeError("Response body must either be str or unicode. Got: '%s'" \
                 % type(body).__name__)
 
     body = property(_get_body, obsolete_setter(_set_body, 'body'))
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 14030d8e5..9a435f194 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -5,9 +5,12 @@ discovering (through HTTP headers) to base Response class.
 See documentation in docs/topics/request-response.rst
 """
 
+from six.moves.urllib.parse import urljoin
+
 from w3lib.encoding import html_to_unicode, resolve_encoding, \
     html_body_declared_encoding, http_content_type_encoding
 from scrapy.http.response import Response
+from scrapy.utils.response import get_base_url
 from scrapy.utils.python import memoizemethod_noargs
 
 
@@ -63,6 +66,11 @@ class TextResponse(Response):
             self._cached_ubody = html_to_unicode(charset, self.body)[1]
         return self._cached_ubody
 
+    def urljoin(self, url):
+        """Join this Response's url with a possible relative url to form an
+        absolute interpretation of the latter."""
+        return urljoin(get_base_url(self), url)
+
     @memoizemethod_noargs
     def _headers_encoding(self):
         content_type = self.headers.get('Content-Type')
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 61f43535f..1d79ec0e3 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -13,7 +13,6 @@ from twisted.web import http
 from twisted.web.http import RESPONSES
 from w3lib import html
 
-from scrapy.http import HtmlResponse, TextResponse
 from scrapy.utils.decorator import deprecated
 
 
@@ -73,6 +72,7 @@ def open_in_browser(response, _openfunc=webbrowser.open):
     """Open the given response in a local web browser, populating the <base>
     tag for external links to work
     """
+    from scrapy.http import HtmlResponse, TextResponse
     # XXX: this implementation is a bit dirty and could be improved
     body = response.body
     if isinstance(response, HtmlResponse):
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 26a628182..b0b5b82c4 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -113,6 +113,12 @@ class BaseResponseTest(unittest.TestCase):
         self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com')
         self.assertRaises(AttributeError, setattr, r, 'body', 'xxx')
 
+    def test_urljoin(self):
+        """Test urljoin shortcut (only for existence, since behavior equals urljoin)"""
+        joined = self.response_class('http://www.example.com').urljoin('/test')
+        absolute = 'http://www.example.com/test'
+        self.assertEqual(joined, absolute)
+
 
 class ResponseText(BaseResponseTest):
 
@@ -295,6 +301,23 @@ class TextResponseTest(BaseResponseTest):
             response.selector.css("title::text").extract(),
         )
 
+    def test_urljoin_with_base_url(self):
+        """Test urljoin shortcut which also evaluates base-url through get_base_url()."""
+        body = '<html><body><base href="https://example.net"></body></html>'
+        joined = self.response_class('http://www.example.com', body=body).urljoin('/test')
+        absolute = 'https://example.net/test'
+        self.assertEqual(joined, absolute)
+
+        body = '<html><body><base href="/elsewhere"></body></html>'
+        joined = self.response_class('http://www.example.com', body=body).urljoin('test')
+        absolute = 'http://www.example.com/test'
+        self.assertEqual(joined, absolute)
+
+        body = '<html><body><base href="/elsewhere/"></body></html>'
+        joined = self.response_class('http://www.example.com', body=body).urljoin('test')
+        absolute = 'http://www.example.com/elsewhere/test'
+        self.assertEqual(joined, absolute)
+
 
 class HtmlResponseTest(TextResponseTest):
 

From 7135c4e694bc4dc970311f67386dfe8660c09012 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Fri, 27 Mar 2015 17:56:56 -0300
Subject: [PATCH 0153/2974] some minor grammar fixes

---
 docs/intro/overview.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 18a877cdc..577e769b2 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -82,7 +82,7 @@ Spider definition inside it and ran it through its crawler engine.
 
 The crawl started by making requests to the URLs defined in the ``start_urls``
 attribute (in this case, only the URL for StackOverflow top questions page),
-and called the default callback method ``parse`` passing the response object as
+and called the default callback method ``parse``, passing the response object as
 an argument. In the ``parse`` callback, we extract the links to the
 question pages using a CSS Selector with a custom extension that allows to get
 the value for an attribute. Then, we yield a few more requests to be sent,
@@ -96,12 +96,12 @@ processed, it can send another request or do other things in the meantime. This
 also means that other requests can keep going even if some request fails or an
 error happens while handling it.
 
-While this enables you to do very fast crawlings sending multiple concurrent
-requests at the same time in a fault-tolerant way, Scrapy also gives you
+While this enables you to do very fast crawlings (sending multiple concurrent
+requests at the same time, in a fault-tolerant way) Scrapy also gives you
 control over the politeness of the crawl through :ref:`a few settings
 <topics-settings-ref>`. You can do things like setting a download delay between
-each request, limit amount of concurrent requests per domain or per IP, and
-even :ref:`use an auto-throttling extension <topics-autothrottle>` that tries
+each request, limiting amount of concurrent requests per domain or per IP, and
+even :ref:`using an auto-throttling extension <topics-autothrottle>` that tries
 to figure out these automatically.
 
 Finally, the ``parse_question`` callback scrapes the question data for each
@@ -126,7 +126,7 @@ this is just the surface. Scrapy provides a lot of powerful features for making
 scraping easy and efficient, such as:
 
 * Built-in support for :ref:`selecting and extracting <topics-selectors>` data
-  from HTML/XML sources using CSS selectors extended and XPath expressions,
+  from HTML/XML sources using extended CSS selectors and XPath expressions,
   with helper methods to extract using regular expressions.
 
 * An :ref:`interactive shell console <topics-shell>` (IPython aware) for trying

From 18c5bc75c27f88f05fe7f1686dfa491b82a7f558 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Fri, 27 Mar 2015 21:00:21 -0300
Subject: [PATCH 0154/2974] some more minor grammar fixes

---
 docs/intro/overview.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 577e769b2..8777ac891 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -81,11 +81,11 @@ When you ran the command ``scrapy runspider somefile.py``, Scrapy looked for a
 Spider definition inside it and ran it through its crawler engine.
 
 The crawl started by making requests to the URLs defined in the ``start_urls``
-attribute (in this case, only the URL for StackOverflow top questions page),
+attribute (in this case, only the URL for StackOverflow top questions page)
 and called the default callback method ``parse``, passing the response object as
-an argument. In the ``parse`` callback, we extract the links to the
+an argument. In the ``parse`` callback we extract the links to the
 question pages using a CSS Selector with a custom extension that allows to get
-the value for an attribute. Then, we yield a few more requests to be sent,
+the value for an attribute. Then we yield a few more requests to be sent,
 registering the method ``parse_question`` as the callback to be called for each
 of them as they finish.
 
@@ -96,7 +96,7 @@ processed, it can send another request or do other things in the meantime. This
 also means that other requests can keep going even if some request fails or an
 error happens while handling it.
 
-While this enables you to do very fast crawlings (sending multiple concurrent
+While this enables you to do very fast crawls (sending multiple concurrent
 requests at the same time, in a fault-tolerant way) Scrapy also gives you
 control over the politeness of the crawl through :ref:`a few settings
 <topics-settings-ref>`. You can do things like setting a download delay between

From c1cd019f56b703509bff5cf97da7e2c70fb466b4 Mon Sep 17 00:00:00 2001
From: Alexander Sibiryakov <sixty-one@yandex.ru>
Date: Tue, 31 Mar 2015 10:56:31 +0200
Subject: [PATCH 0155/2974] Setting maximum thread pool size in reactor.

---
 scrapy/crawler.py                   | 3 +++
 scrapy/settings/default_settings.py | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 6866be809..41716852c 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -12,6 +12,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 from scrapy.utils.misc import load_object
 from scrapy import log, signals
+from pdb import set_trace
 
 
 class Crawler(object):
@@ -152,6 +153,8 @@ class CrawlerProcess(CrawlerRunner):
         if self.settings.getbool('DNSCACHE_ENABLED'):
             reactor.installResolver(CachingThreadedResolver(reactor))
 
+        tp = reactor.getThreadPool()
+        tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
         reactor.run(installSignalHandlers=False)  # blocking call
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 0342b1ada..5b8dc4eaa 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -194,6 +194,8 @@ NEWSPIDER_MODULE = ''
 
 RANDOMIZE_DOWNLOAD_DELAY = True
 
+REACTOR_THREADPOOL_MAXSIZE = 10
+
 REDIRECT_ENABLED = True
 REDIRECT_MAX_TIMES = 20  # uses Firefox default setting
 REDIRECT_PRIORITY_ADJUST = +2

From 5864d291d4ff96351661ce01c414f2bdae43efe0 Mon Sep 17 00:00:00 2001
From: Alexander Sibiryakov <sixty-one@yandex.ru>
Date: Tue, 31 Mar 2015 11:10:56 +0200
Subject: [PATCH 0156/2974] Setting documentation.

---
 docs/topics/settings.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 5e11e473f..4812de8b9 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -742,6 +742,17 @@ If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect.
 
 .. _wget: http://www.gnu.org/software/wget/manual/wget.html
 
+.. setting:: REACTOR_THREADPOOL_MAXSIZE
+
+REACTOR_THREADPOOL_MAXSIZE
+------------------
+
+Default: ``10``
+
+The maximum limit for Twisted Reactor thread pool size. This is common multi-purpose thread pool used by various
+Scrapy components. Threaded DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase this value if
+you're experiencing problems with insufficient blocking IO.
+
 .. setting:: REDIRECT_MAX_TIMES
 
 REDIRECT_MAX_TIMES

From 2d142d64014d7094119a4556337c90b5f4b0b900 Mon Sep 17 00:00:00 2001
From: Richard Dowinton <richard.dowinton@gmail.com>
Date: Tue, 31 Mar 2015 12:17:31 +0100
Subject: [PATCH 0157/2974] Added deployment section covering scrapyd-deploy
 and shub

---
 docs/index.rst             |  4 ++--
 docs/topics/deployment.rst | 29 +++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)
 create mode 100644 docs/topics/deployment.rst

diff --git a/docs/index.rst b/docs/index.rst
index 507b9bea9..cea652b89 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -150,7 +150,7 @@ Solving specific problems
    topics/leaks
    topics/images
    topics/ubuntu
-   topics/scrapyd
+   topics/deployment
    topics/autothrottle
    topics/benchmarking
    topics/jobs
@@ -186,7 +186,7 @@ Solving specific problems
 :doc:`topics/ubuntu`
     Install latest Scrapy packages easily on Ubuntu
 
-:doc:`topics/scrapyd`
+:doc:`topics/deployment`
     Deploying your Scrapy project in production.
 
 :doc:`topics/autothrottle`
diff --git a/docs/topics/deployment.rst b/docs/topics/deployment.rst
new file mode 100644
index 000000000..8f8c55c2f
--- /dev/null
+++ b/docs/topics/deployment.rst
@@ -0,0 +1,29 @@
+.. _topics-deployment:
+
+==========
+Deployment
+==========
+
+The recommended way to deploy Scrapy projects to a server is through `Scrapyd`_.
+
+.. _Scrapyd: https://github.com/scrapy/scrapyd
+
+Deploying to a Scrapyd Server
+=============================
+
+You can deploy to a Scrapyd server using the `Scrapyd client <https://github.com/scrapy/scrapyd-client>`_. You can add targets to your ``scrapy.cfg`` file which can be deployed to using the ``scrapyd-deploy`` command.
+
+The basic syntax is as follows:
+
+    scrapyd-deploy <target> -p <project>
+
+For more information please refer to the `Deploying your project`_ section.
+
+.. _Deploying your project: https://scrapyd.readthedocs.org/en/latest/deploy.html
+
+Deploying to Scrapinghub
+========================
+
+You can deploy to Scrapinghub using Scrapinghub's command line client, `shub`_. The configuration is read from the ``scrapy.cfg`` file just like ``scrapyd-deploy``.
+
+.. _shub: https://github.com/scrapinghub/shub

From 5916df64cd20ccc5b7fe02410b7ea485892a39da Mon Sep 17 00:00:00 2001
From: Alexander Sibiryakov <sixty-one@yandex.ru>
Date: Wed, 1 Apr 2015 11:24:12 +0200
Subject: [PATCH 0158/2974] Removing unnecessary import.

---
 scrapy/crawler.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 41716852c..52d4069ac 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -12,7 +12,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 from scrapy.utils.misc import load_object
 from scrapy import log, signals
-from pdb import set_trace
 
 
 class Crawler(object):

From 94fceb4c1500a7add5b462beab83ea0562479880 Mon Sep 17 00:00:00 2001
From: Alexander Sibiryakov <sixty-one@yandex.ru>
Date: Wed, 1 Apr 2015 11:25:10 +0200
Subject: [PATCH 0159/2974] Fixing underscore size.

---
 docs/topics/settings.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 4812de8b9..bbe822046 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -745,7 +745,7 @@ If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect.
 .. setting:: REACTOR_THREADPOOL_MAXSIZE
 
 REACTOR_THREADPOOL_MAXSIZE
-------------------
+--------------------------
 
 Default: ``10``
 

From e7b274edf3aa8320edf91a652e1c221654e26aa6 Mon Sep 17 00:00:00 2001
From: Alexander Sibiryakov <sixty-one@yandex.ru>
Date: Wed, 1 Apr 2015 11:49:55 +0200
Subject: [PATCH 0160/2974] Reformat to 80 characters per line.

---
 docs/topics/settings.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index bbe822046..3a2876a7a 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -749,9 +749,10 @@ REACTOR_THREADPOOL_MAXSIZE
 
 Default: ``10``
 
-The maximum limit for Twisted Reactor thread pool size. This is common multi-purpose thread pool used by various
-Scrapy components. Threaded DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase this value if
-you're experiencing problems with insufficient blocking IO.
+The maximum limit for Twisted Reactor thread pool size. This is common
+multi-purpose thread pool used by various Scrapy components. Threaded
+DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase
+this value if you're experiencing problems with insufficient blocking IO.
 
 .. setting:: REDIRECT_MAX_TIMES
 

From b794cdaf4bf5d43d5eb5d3802eb46976d5d55d14 Mon Sep 17 00:00:00 2001
From: Alexander Sibiryakov <sixty-one@yandex.ru>
Date: Wed, 1 Apr 2015 12:07:03 +0200
Subject: [PATCH 0161/2974] Broad crawls notes.

---
 docs/topics/broad-crawls.rst | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index b95974f5d..aaf46bc92 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -57,6 +57,27 @@ To increase the global concurrency use::
 
     CONCURRENT_REQUESTS = 100
 
+Increase Twisted IO thread pool maximum size
+============================================
+
+Currently Scrapy does DNS resolution in a blocking way with usage of thread
+pool. With higher concurrency levels the crawling could be slow or even fail
+hitting DNS resolver timeouts. Possible solution to increase the number of
+threads handling DNS queries. The DNS queue will be processed faster speeding
+up establishing of connection and crawling overall.
+
+To increase maximum thread pool size use::
+
+    REACTOR_THREADPOOL_MAXSIZE = 20
+
+Setup your own DNS
+==================
+
+If you have multiple crawling processes and single central DNS, it can act
+like DoS attack on the DNS server resulting to slow down of entire network or
+even blocking your machines. To avoid this setup your own DNS server with
+local cache and upstream to some large DNS like OpenDNS or Verizon.
+
 Reduce log level
 ================
 

From 92b574309e3102d0887fe1eba19447fbf3c57c34 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Wed, 1 Apr 2015 19:46:21 +0000
Subject: [PATCH 0162/2974] documentation build warning fixes

---
 docs/topics/commands.rst | 2 +-
 docs/topics/items.rst    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 27d0655ef..478746d3c 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -21,7 +21,7 @@ in standard locations:
 
 1. ``/etc/scrapy.cfg`` or ``c:\scrapy\scrapy.cfg`` (system-wide),
 2. ``~/.config/scrapy.cfg`` (``$XDG_CONFIG_HOME``) and ``~/.scrapy.cfg`` (``$HOME``)
-  for global (user-wide) settings, and
+   for global (user-wide) settings, and
 3. ``scrapy.cfg`` inside a scrapy project's root (see next section).
 
 Settings from these files are merged in the listed order of preference:
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index ac3eb6699..21ec0ed8c 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -22,7 +22,7 @@ Various Scrapy components use extra information provided by Items:
 exporters look at declared fields to figure out columns to export,
 serialization can be customized using Item fields metadata, :mod:`trackref`
 tracks Item instances to help finding memory leaks 
-(see :ref:`topics-leaks-trackrefs`_), etc.
+(see :ref:`topics-leaks-trackrefs`), etc.
 
 .. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
 

From dc88be75e26cf970f65cd5c3c7f5908cb4b68834 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Mon, 30 Mar 2015 14:56:02 +0000
Subject: [PATCH 0163/2974] more python3 compatibility changes for urllib

---
 scrapy/commands/deploy.py                     | 25 ++++++++++---------
 .../contrib/downloadermiddleware/httpproxy.py |  8 ++++--
 scrapy/http/request/form.py                   |  5 ++--
 scrapy/utils/url.py                           |  8 +++---
 tests/mockserver.py                           |  5 ++--
 5 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/scrapy/commands/deploy.py b/scrapy/commands/deploy.py
index 051ce661a..b7d989f69 100644
--- a/scrapy/commands/deploy.py
+++ b/scrapy/commands/deploy.py
@@ -5,10 +5,11 @@ import glob
 import tempfile
 import shutil
 import time
-import urllib2
 import netrc
 import json
+from six.moves.urllib import request
 from six.moves.urllib.parse import urlparse, urljoin
+from six.moves.urllib.error import HTTPError, URLError
 from subprocess import Popen, PIPE, check_call
 
 from w3lib.form import encode_multipart
@@ -69,7 +70,7 @@ class Command(ScrapyCommand):
         except ImportError:
             raise UsageError("setuptools not installed")
 
-        urllib2.install_opener(urllib2.build_opener(HTTPRedirectHandler))
+        request.install_opener(request.build_opener(HTTPRedirectHandler))
 
         if opts.list_targets:
             for name, target in _get_targets().items():
@@ -78,9 +79,9 @@ class Command(ScrapyCommand):
 
         if opts.list_projects:
             target = _get_target(opts.list_projects)
-            req = urllib2.Request(_url(target, 'listprojects.json'))
+            req = request.Request(_url(target, 'listprojects.json'))
             _add_auth_header(req, target)
-            f = urllib2.urlopen(req)
+            f = request.urlopen(req)
             projects = json.loads(f.read())['projects']
             print(os.linesep.join(projects))
             return
@@ -188,7 +189,7 @@ def _upload_egg(target, eggpath, project, version):
         'Content-Type': 'multipart/form-data; boundary=%s' % boundary,
         'Content-Length': str(len(body)),
     }
-    req = urllib2.Request(url, body, headers)
+    req = request.Request(url, body, headers)
     _add_auth_header(req, target)
     _log('Deploying to project "%s" in %s' % (project, url))
     return _http_post(req)
@@ -207,14 +208,14 @@ def _add_auth_header(request, target):
 
 def _http_post(request):
     try:
-        f = urllib2.urlopen(request)
+        f = request.urlopen(request)
         _log("Server response (%s):" % f.code)
         print(f.read())
         return True
-    except urllib2.HTTPError as e:
+    except HTTPError as e:
         _log("Deploy failed (%s):" % e.code)
         print(e.read())
-    except urllib2.URLError as e:
+    except URLError as e:
         _log("Deploy failed: %s" % e)
 
 def _build_egg():
@@ -237,12 +238,12 @@ def _create_default_setup_py(**kwargs):
         f.write(_SETUP_PY_TEMPLATE % kwargs)
 
 
-class HTTPRedirectHandler(urllib2.HTTPRedirectHandler):
+class HTTPRedirectHandler(request.HTTPRedirectHandler):
 
     def redirect_request(self, req, fp, code, msg, headers, newurl):
         newurl = newurl.replace(' ', '%20')
         if code in (301, 307):
-            return urllib2.Request(newurl,
+            return request.Request(newurl,
                                    data=req.get_data(),
                                    headers=req.headers,
                                    origin_req_host=req.get_origin_req_host(),
@@ -250,9 +251,9 @@ class HTTPRedirectHandler(urllib2.HTTPRedirectHandler):
         elif code in (302, 303):
             newheaders = dict((k, v) for k, v in req.headers.items()
                               if k.lower() not in ("content-length", "content-type"))
-            return urllib2.Request(newurl,
+            return request.Request(newurl,
                                    headers=newheaders,
                                    origin_req_host=req.get_origin_req_host(),
                                    unverifiable=True)
         else:
-            raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
+            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
diff --git a/scrapy/contrib/downloadermiddleware/httpproxy.py b/scrapy/contrib/downloadermiddleware/httpproxy.py
index ce09655d0..48a877c91 100644
--- a/scrapy/contrib/downloadermiddleware/httpproxy.py
+++ b/scrapy/contrib/downloadermiddleware/httpproxy.py
@@ -1,6 +1,10 @@
 import base64
-from urllib import getproxies, unquote, proxy_bypass
-from urllib2 import _parse_proxy
+from six.moves.urllib.request import getproxies, proxy_bypass
+from six.moves.urllib.parse import unquote
+try:
+    from urllib2 import _parse_proxy
+except ImportError:
+    from urllib.request import _parse_proxy
 from six.moves.urllib.parse import urlunparse
 
 from scrapy.utils.httpobj import urlparse_cached
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 02e77ce29..a4695f1a2 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -5,8 +5,7 @@ This module implements the FormRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 
-import urllib
-from six.moves.urllib.parse import urljoin
+from six.moves.urllib.parse import urljoin, urlencode
 import lxml.html
 import six
 from scrapy.http.request import Request
@@ -52,7 +51,7 @@ def _urlencode(seq, enc):
     values = [(unicode_to_str(k, enc), unicode_to_str(v, enc))
               for k, vs in seq
               for v in (vs if hasattr(vs, '__iter__') else [vs])]
-    return urllib.urlencode(values, doseq=1)
+    return urlencode(values, doseq=1)
 
 
 def _get_form(response, formname, formnumber, formxpath):
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 36490a39d..5c645aaf0 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -7,8 +7,8 @@ to the w3lib.url module. Always import those from there instead.
 """
 import posixpath
 from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag,
-                                    urlparse, parse_qsl)
-import urllib
+                                    urlparse, parse_qsl, urlencode,
+                                    unquote)
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this move doesn't break old code
 from w3lib.url import *
@@ -56,7 +56,7 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False,
     scheme, netloc, path, params, query, fragment = parse_url(url)
     keyvals = parse_qsl(query, keep_blank_values)
     keyvals.sort()
-    query = urllib.urlencode(keyvals)
+    query = urlencode(keyvals)
     path = safe_url_string(_unquotepath(path)) or '/'
     fragment = '' if not keep_fragments else fragment
     return urlunparse((scheme, netloc.lower(), path, params, query, fragment))
@@ -65,7 +65,7 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False,
 def _unquotepath(path):
     for reserved in ('2f', '2F', '3f', '3F'):
         path = path.replace('%' + reserved, '%25' + reserved.upper())
-    return urllib.unquote(path)
+    return unquote(path)
 
 
 def parse_url(url, encoding=None):
diff --git a/tests/mockserver.py b/tests/mockserver.py
index b73208c5c..d2ba92485 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,6 +1,7 @@
 from __future__ import print_function
-import sys, time, random, urllib, os, json
+import sys, time, random, os, json
 import six
+from six.moves.urllib.parse import urlencode
 from subprocess import Popen, PIPE
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
@@ -73,7 +74,7 @@ class Follow(LeafResource):
         args = request.args.copy()
         for nl in nlist:
             args["n"] = [str(nl)]
-            argstr = urllib.urlencode(args, doseq=True)
+            argstr = urlencode(args, doseq=True)
             s += "<a href='/follow?%s'>follow %d</a><br>" % (argstr, nl)
         s += """</body>"""
         request.write(s)

From 6f9265b44be6f52a2af94458a86fb4ba503fa74d Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 1 Apr 2015 23:29:10 -0300
Subject: [PATCH 0164/2974] Move spider settings population to Crawler.__init__

---
 scrapy/crawler.py     | 12 ++++++------
 tests/test_crawler.py | 21 ++++++---------------
 2 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 52d4069ac..692a896be 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -18,13 +18,17 @@ class Crawler(object):
 
     def __init__(self, spidercls, settings):
         self.spidercls = spidercls
-        self.settings = settings
+        self.settings = settings.copy()
+
         self.signals = SignalManager(self)
         self.stats = load_object(self.settings['STATS_CLASS'])(self)
         lf_cls = load_object(self.settings['LOG_FORMATTER'])
         self.logformatter = lf_cls.from_crawler(self)
         self.extensions = ExtensionManager.from_crawler(self)
 
+        self.spidercls.update_settings(self.settings)
+        self.settings.freeze()
+
         self.crawling = False
         self.spider = None
         self.engine = None
@@ -95,11 +99,7 @@ class CrawlerRunner(object):
     def _create_crawler(self, spidercls):
         if isinstance(spidercls, six.string_types):
             spidercls = self.spiders.load(spidercls)
-
-        crawler_settings = self.settings.copy()
-        spidercls.update_settings(crawler_settings)
-        crawler_settings.freeze()
-        return Crawler(spidercls, crawler_settings)
+        return Crawler(spidercls, self.settings)
 
     def _setup_crawler_logging(self, crawler):
         log_observer = log.start_from_crawler(crawler)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 5128ea6e9..34fc1a367 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -25,16 +25,6 @@ class CrawlerTestCase(unittest.TestCase):
             self.crawler.spiders
             self.assertEqual(len(w), 1, "Warn deprecated access only once")
 
-
-class CrawlerRunnerTest(unittest.TestCase):
-
-    def setUp(self):
-        self.crawler_runner = CrawlerRunner(Settings())
-
-    def tearDown(self):
-        return self.crawler_runner.stop()
-
-    @defer.inlineCallbacks
     def test_populate_spidercls_settings(self):
         spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
         project_settings = {'TEST1': 'project', 'TEST3': 'project'}
@@ -42,12 +32,13 @@ class CrawlerRunnerTest(unittest.TestCase):
         class CustomSettingsSpider(DefaultSpider):
             custom_settings = spider_settings
 
-        self.crawler_runner.settings.setdict(project_settings,
-                                             priority='project')
+        settings = Settings()
+        settings.setdict(project_settings, priority='project')
+        crawler = Crawler(CustomSettingsSpider, settings)
 
-        d = self.crawler_runner.crawl(CustomSettingsSpider)
-        crawler = list(self.crawler_runner.crawlers)[0]
-        yield d
         self.assertEqual(crawler.settings.get('TEST1'), 'spider')
         self.assertEqual(crawler.settings.get('TEST2'), 'spider')
         self.assertEqual(crawler.settings.get('TEST3'), 'project')
+
+        self.assertFalse(settings.frozen)
+        self.assertTrue(crawler.settings.frozen)

From 85aa3c7596c6e9c66daaa5503faadd03a16e1d59 Mon Sep 17 00:00:00 2001
From: Alexander Sibiryakov <sixty-one@yandex.ru>
Date: Thu, 2 Apr 2015 18:30:59 +0200
Subject: [PATCH 0165/2974] Dns cache size and timeout options

---
 docs/topics/settings.rst            | 18 ++++++++++++++++++
 scrapy/crawler.py                   |  6 +++---
 scrapy/resolver.py                  |  9 +++++++--
 scrapy/settings/default_settings.py |  2 ++
 4 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index c39601b14..1986b2f3e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -290,6 +290,24 @@ Default: ``True``
 
 Whether to enable DNS in-memory cache.
 
+.. setting:: DNSCACHE_SIZE
+
+DNSCACHE_SIZE
+----------------
+
+Default: ``10000``
+
+DNS in-memory cache size.
+
+.. setting:: DNS_TIMEOUT
+
+DNS_TIMEOUT
+----------------
+
+Default: ``60``
+
+Timeout for processing of DNS queries in seconds. Float is supported.
+
 .. setting:: DOWNLOADER
 
 DOWNLOADER
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 52d4069ac..efb181fa4 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -149,9 +149,9 @@ class CrawlerProcess(CrawlerRunner):
                 return
             d.addBoth(lambda _: self._stop_reactor())
 
-        if self.settings.getbool('DNSCACHE_ENABLED'):
-            reactor.installResolver(CachingThreadedResolver(reactor))
-
+        cache_size = self.settings.getint('DNSCACHE_SIZE') if self.settings.getbool('DNSCACHE_ENABLED') else 0
+        reactor.installResolver(CachingThreadedResolver(reactor, cache_size,
+                                                            self.settings.getfloat('DNS_TIMEOUT')))
         tp = reactor.getThreadPool()
         tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index e1a7aedce..3954fd977 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -4,15 +4,20 @@ from twisted.internet.base import ThreadedResolver
 from scrapy.utils.datatypes import LocalCache
 
 # TODO: cache misses
-# TODO: make cache size a setting
 
 dnscache = LocalCache(10000)
 
 class CachingThreadedResolver(ThreadedResolver):
+    def __init__(self, reactor, cache_size, timeout):
+        super(CachingThreadedResolver, self).__init__(reactor)
+        dnscache.limit = cache_size
+        self.timeout = timeout
 
-    def getHostByName(self, name, timeout = (1, 3, 11, 45)):
+    def getHostByName(self, name, timeout=None):
         if name in dnscache:
             return defer.succeed(dnscache[name])
+        if not timeout:
+            timeout = self.timeout
         d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
         d.addCallback(self._cache_result, name)
         return d
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 5b8dc4eaa..22005eb5d 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -52,6 +52,8 @@ DEPTH_STATS = True
 DEPTH_PRIORITY = 0
 
 DNSCACHE_ENABLED = True
+DNSCACHE_SIZE = 10000
+DNS_TIMEOUT = 60
 
 DOWNLOAD_DELAY = 0
 

From 42f7674619a779b029a3a4a9db8242d90f140453 Mon Sep 17 00:00:00 2001
From: "Victoria Terenina (torymur)" <torymur@gmail.com>
Date: Thu, 2 Apr 2015 17:48:38 +0300
Subject: [PATCH 0166/2974] fixed unhandled error in deferred
 (RobotsTxtMiddleware)

---
 scrapy/contrib/downloadermiddleware/robotstxt.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py
index 0ea402719..a58ecca8e 100644
--- a/scrapy/contrib/downloadermiddleware/robotstxt.py
+++ b/scrapy/contrib/downloadermiddleware/robotstxt.py
@@ -49,8 +49,14 @@ class RobotsTxtMiddleware(object):
             )
             dfd = self.crawler.engine.download(robotsreq, spider)
             dfd.addCallback(self._parse_robots)
+            dfd.addErrback(self._logerror, robotsreq, spider)
         return self._parsers[netloc]
 
+    def _logerror(self, failure, request, spider):
+        if failure.type is not IgnoreRequest:
+            log.msg(format="Error downloading %%(request)s: %s" % failure.value,
+                    level=log.ERROR, request=request, spider=spider)
+
     def _parse_robots(self, response):
         rp = robotparser.RobotFileParser(response.url)
         rp.parse(response.body.splitlines())

From 9d0ea5a396481e4240adeaa8d71fbc4977143fd9 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Sun, 2 Nov 2014 22:24:00 +0530
Subject: [PATCH 0167/2974] Added zsh completion for the Scrapy command-line
 tool

---
 debian/scrapy.install         |  1 +
 extras/_scrapy_zsh_completion | 25 +++++++++++++++++++++++++
 2 files changed, 26 insertions(+)
 create mode 100644 extras/_scrapy_zsh_completion

diff --git a/debian/scrapy.install b/debian/scrapy.install
index 5977d5f43..17a5861c4 100644
--- a/debian/scrapy.install
+++ b/debian/scrapy.install
@@ -1 +1,2 @@
 extras/scrapy_bash_completion etc/bash_completion.d/
+extras/_scrapy_zsh_completion /usr/local/share/zsh/site-functions/
diff --git a/extras/_scrapy_zsh_completion b/extras/_scrapy_zsh_completion
new file mode 100644
index 000000000..3d1ae0f82
--- /dev/null
+++ b/extras/_scrapy_zsh_completion
@@ -0,0 +1,25 @@
+#compdef scrapy
+
+# zsh completion for the Scrapy command-line tool
+
+_scrapy() {
+    local curcontext="$curcontext" cmd spiders
+    typeset -A opt_args
+    cmd=$words[2]
+    
+    case "$cmd" in
+        crawl|edit|check)
+            spiders=$(scrapy list 2>/dev/null) || spiders=""
+            if [[ -n "$spiders" ]]; then
+                compadd `echo $spiders`
+            fi
+        ;;
+        *)
+            if [[ CURRENT -eq 2 ]]; then
+                _arguments '*: :(check crawl deploy edit fetch genspider list parse runspider server settings shell startproject version view)'
+            fi
+        ;;
+    esac
+}
+
+_scrapy
\ No newline at end of file

From e2b4fabbf5ab118ea9bd5ad43b54c5d6e5d52aaf Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Fri, 3 Apr 2015 08:34:16 +0530
Subject: [PATCH 0168/2974] Fix zsh completion path

---
 debian/scrapy.install                                  | 2 +-
 extras/_scrapy_zsh_completion => scrapy_zsh_completion | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename extras/_scrapy_zsh_completion => scrapy_zsh_completion (100%)

diff --git a/debian/scrapy.install b/debian/scrapy.install
index 17a5861c4..c288ebed3 100644
--- a/debian/scrapy.install
+++ b/debian/scrapy.install
@@ -1,2 +1,2 @@
 extras/scrapy_bash_completion etc/bash_completion.d/
-extras/_scrapy_zsh_completion /usr/local/share/zsh/site-functions/
+extras/scrapy_zsh_completion /usr/share/zsh/vendor-completions/_scrapy
diff --git a/extras/_scrapy_zsh_completion b/scrapy_zsh_completion
similarity index 100%
rename from extras/_scrapy_zsh_completion
rename to scrapy_zsh_completion

From 7129dc36d151807e389ccbf81918f0a77a8819cd Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Fri, 3 Apr 2015 21:20:06 +0530
Subject: [PATCH 0169/2974] Move scrapy_zsh_completion inside extras directory

---
 scrapy_zsh_completion => extras/scrapy_zsh_completion | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename scrapy_zsh_completion => extras/scrapy_zsh_completion (100%)

diff --git a/scrapy_zsh_completion b/extras/scrapy_zsh_completion
similarity index 100%
rename from scrapy_zsh_completion
rename to extras/scrapy_zsh_completion

From 843a9cdbb59a35a3340f6af07db4aa61eb2b799a Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Fri, 3 Apr 2015 21:24:31 +0530
Subject: [PATCH 0170/2974] Remove deploy command from arguments

As scrapy deploy is being removed
---
 extras/scrapy_zsh_completion | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion
index 3d1ae0f82..17db1b18d 100644
--- a/extras/scrapy_zsh_completion
+++ b/extras/scrapy_zsh_completion
@@ -16,7 +16,7 @@ _scrapy() {
         ;;
         *)
             if [[ CURRENT -eq 2 ]]; then
-                _arguments '*: :(check crawl deploy edit fetch genspider list parse runspider server settings shell startproject version view)'
+                _arguments '*: :(check crawl edit fetch genspider list parse runspider server settings shell startproject version view)'
             fi
         ;;
     esac

From b1247355dc0236fb30ac0d9d08213819d70bd062 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Sun, 5 Apr 2015 17:48:54 +0530
Subject: [PATCH 0171/2974] Remove server command

---
 extras/scrapy_zsh_completion | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion
index 17db1b18d..564991aa8 100644
--- a/extras/scrapy_zsh_completion
+++ b/extras/scrapy_zsh_completion
@@ -16,7 +16,7 @@ _scrapy() {
         ;;
         *)
             if [[ CURRENT -eq 2 ]]; then
-                _arguments '*: :(check crawl edit fetch genspider list parse runspider server settings shell startproject version view)'
+                _arguments '*: :(check crawl edit fetch genspider list parse runspider settings shell startproject version view)'
             fi
         ;;
     esac

From d55ae535131a411c66fe387d965b7ac0a4a375b8 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Sun, 5 Apr 2015 17:50:12 +0530
Subject: [PATCH 0172/2974] Remove deploy and server commands from bash
 completion

---
 extras/scrapy_bash_completion | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extras/scrapy_bash_completion b/extras/scrapy_bash_completion
index 19fd81c87..07a7d88c5 100644
--- a/extras/scrapy_bash_completion
+++ b/extras/scrapy_bash_completion
@@ -11,7 +11,7 @@ _scrapy_completion() {
         ;;
         *)
             if [ $COMP_CWORD -eq 1 ]; then
-                commands="check crawl deploy edit fetch genspider list parse runspider server settings shell startproject version view"
+                commands="check crawl edit fetch genspider list parse runspider settings shell startproject version view"
                 COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W "$commands" -- "$cmd"))
             fi
         ;;

From 5a96a16914ad6e4e54a7bd313802891f82a8dc08 Mon Sep 17 00:00:00 2001
From: Dharmesh Pandav <pandav.dharmesh@gmail.com>
Date: Mon, 6 Apr 2015 14:44:47 +0530
Subject: [PATCH 0173/2974] Update form.py to improve existing capability

Add capability to search HTML Form using formid when using `FormRequest.from_response()`

refrenced issue :https://github.com/scrapy/scrapy/issues/1136
---
 scrapy/http/request/form.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index a4695f1a2..cae56f229 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -31,10 +31,10 @@ class FormRequest(Request):
                 self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr)
 
     @classmethod
-    def from_response(cls, response, formname=None, formnumber=0, formdata=None,
+    def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,
                       clickdata=None, dont_click=False, formxpath=None, **kwargs):
         kwargs.setdefault('encoding', response.encoding)
-        form = _get_form(response, formname, formnumber, formxpath)
+        form = _get_form(response, formname, formid, formnumber, formxpath)
         formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
         url = _get_form_url(form, kwargs.pop('url', None))
         method = kwargs.pop('method', form.method)
@@ -67,6 +67,11 @@ def _get_form(response, formname, formnumber, formxpath):
         if f:
             return f[0]
 
+    if formid is not None:
+        f = root.xpath('//form[@id="%s"]' % formid)
+        if f:
+            return f[0]
+            
     # Get form element from xpath, if not found, go up
     if formxpath is not None:
         nodes = root.xpath(formxpath)

From 6d48c19acd2eb9d5e9e7e8808d349b14c290bda4 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Fri, 3 Apr 2015 13:02:56 +0000
Subject: [PATCH 0174/2974] dissolve `scrapy.contrib_exp`

---
 .../downloadermiddleware/decompression.py     | 86 ++++++++++++++++++
 scrapy/contrib_exp/__init__.py                |  9 --
 scrapy/contrib_exp/djangoitem.py              |  6 --
 .../downloadermiddleware/decompression.py     | 91 ++-----------------
 scrapy/contrib_exp/iterators.py               | 46 +---------
 scrapy/utils/iterators.py                     | 40 ++++++++
 ...test_downloadermiddleware_decompression.py |  2 +-
 tests/test_utils_iterators.py                 |  3 +-
 8 files changed, 139 insertions(+), 144 deletions(-)
 create mode 100644 scrapy/contrib/downloadermiddleware/decompression.py
 delete mode 100644 scrapy/contrib_exp/djangoitem.py

diff --git a/scrapy/contrib/downloadermiddleware/decompression.py b/scrapy/contrib/downloadermiddleware/decompression.py
new file mode 100644
index 000000000..c08f50b5f
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/decompression.py
@@ -0,0 +1,86 @@
+""" This module implements the DecompressionMiddleware which tries to recognise
+and extract the potentially compressed responses that may arrive. 
+"""
+
+import bz2
+import gzip
+import zipfile
+import tarfile
+from tempfile import mktemp
+
+import six
+
+try:
+    from cStringIO import StringIO as BytesIO
+except ImportError:
+    from io import BytesIO
+
+from scrapy import log
+from scrapy.responsetypes import responsetypes
+
+
+class DecompressionMiddleware(object):
+    """ This middleware tries to recognise and extract the possibly compressed
+    responses that may arrive. """
+
+    def __init__(self):
+        self._formats = {
+            'tar': self._is_tar,
+            'zip': self._is_zip,
+            'gz': self._is_gzip,
+            'bz2': self._is_bzip2
+        }
+
+    def _is_tar(self, response):
+        archive = BytesIO(response.body)
+        try:
+            tar_file = tarfile.open(name=mktemp(), fileobj=archive)
+        except tarfile.ReadError:
+            return
+
+        body = tar_file.extractfile(tar_file.members[0]).read()
+        respcls = responsetypes.from_args(filename=tar_file.members[0].name, body=body)
+        return response.replace(body=body, cls=respcls)
+
+    def _is_zip(self, response):
+        archive = BytesIO(response.body)
+        try:
+            zip_file = zipfile.ZipFile(archive)
+        except zipfile.BadZipfile:
+            return
+
+        namelist = zip_file.namelist()
+        body = zip_file.read(namelist[0])
+        respcls = responsetypes.from_args(filename=namelist[0], body=body)
+        return response.replace(body=body, cls=respcls)
+
+    def _is_gzip(self, response):
+        archive = BytesIO(response.body)
+        try:
+            body = gzip.GzipFile(fileobj=archive).read()
+        except IOError:
+            return
+
+        respcls = responsetypes.from_args(body=body)
+        return response.replace(body=body, cls=respcls)
+
+    def _is_bzip2(self, response):
+        try:
+            body = bz2.decompress(response.body)
+        except IOError:
+            return
+
+        respcls = responsetypes.from_args(body=body)
+        return response.replace(body=body, cls=respcls)
+
+    def process_response(self, request, response, spider):
+        if not response.body:
+            return response
+
+        for fmt, func in six.iteritems(self._formats):
+            new_response = func(response)
+            if new_response:
+                log.msg(format='Decompressed response with format: %(responsefmt)s',
+                        level=log.DEBUG, spider=spider, responsefmt=fmt)
+                return new_response
+        return response
diff --git a/scrapy/contrib_exp/__init__.py b/scrapy/contrib_exp/__init__.py
index 406c7c74d..e69de29bb 100644
--- a/scrapy/contrib_exp/__init__.py
+++ b/scrapy/contrib_exp/__init__.py
@@ -1,9 +0,0 @@
-"""
-This module contains experimental code that may go into scrapy.contrib in the
-future, but it's not yet stable enough to go there (either API stable or
-functionality stable).
-
-Subscribe to Scrapy developers mailing list or join the IRC channel if you want
-to discuss about this code.
-
-"""
diff --git a/scrapy/contrib_exp/djangoitem.py b/scrapy/contrib_exp/djangoitem.py
deleted file mode 100644
index 1e855b404..000000000
--- a/scrapy/contrib_exp/djangoitem.py
+++ /dev/null
@@ -1,6 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib_exp.djangoitem` is deprecated, use `scrapy.contrib.djangoitem` instead",
-    ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.contrib.djangoitem import DjangoItem
diff --git a/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py
index c08f50b5f..8b1d61b3d 100644
--- a/scrapy/contrib_exp/downloadermiddleware/decompression.py
+++ b/scrapy/contrib_exp/downloadermiddleware/decompression.py
@@ -1,86 +1,7 @@
-""" This module implements the DecompressionMiddleware which tries to recognise
-and extract the potentially compressed responses that may arrive. 
-"""
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib_exp.downloadermiddleware.decompression` is deprecated, "
+              "use `scrapy.contrib.downloadermiddleware.decompression` instead",
+    ScrapyDeprecationWarning, stacklevel=2)
 
-import bz2
-import gzip
-import zipfile
-import tarfile
-from tempfile import mktemp
-
-import six
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
-from scrapy import log
-from scrapy.responsetypes import responsetypes
-
-
-class DecompressionMiddleware(object):
-    """ This middleware tries to recognise and extract the possibly compressed
-    responses that may arrive. """
-
-    def __init__(self):
-        self._formats = {
-            'tar': self._is_tar,
-            'zip': self._is_zip,
-            'gz': self._is_gzip,
-            'bz2': self._is_bzip2
-        }
-
-    def _is_tar(self, response):
-        archive = BytesIO(response.body)
-        try:
-            tar_file = tarfile.open(name=mktemp(), fileobj=archive)
-        except tarfile.ReadError:
-            return
-
-        body = tar_file.extractfile(tar_file.members[0]).read()
-        respcls = responsetypes.from_args(filename=tar_file.members[0].name, body=body)
-        return response.replace(body=body, cls=respcls)
-
-    def _is_zip(self, response):
-        archive = BytesIO(response.body)
-        try:
-            zip_file = zipfile.ZipFile(archive)
-        except zipfile.BadZipfile:
-            return
-
-        namelist = zip_file.namelist()
-        body = zip_file.read(namelist[0])
-        respcls = responsetypes.from_args(filename=namelist[0], body=body)
-        return response.replace(body=body, cls=respcls)
-
-    def _is_gzip(self, response):
-        archive = BytesIO(response.body)
-        try:
-            body = gzip.GzipFile(fileobj=archive).read()
-        except IOError:
-            return
-
-        respcls = responsetypes.from_args(body=body)
-        return response.replace(body=body, cls=respcls)
-
-    def _is_bzip2(self, response):
-        try:
-            body = bz2.decompress(response.body)
-        except IOError:
-            return
-
-        respcls = responsetypes.from_args(body=body)
-        return response.replace(body=body, cls=respcls)
-
-    def process_response(self, request, response, spider):
-        if not response.body:
-            return response
-
-        for fmt, func in six.iteritems(self._formats):
-            new_response = func(response)
-            if new_response:
-                log.msg(format='Decompressed response with format: %(responsefmt)s',
-                        level=log.DEBUG, spider=spider, responsefmt=fmt)
-                return new_response
-        return response
+from scrapy.contrib.downloadermiddleware.decompression import DecompressionMiddleware
diff --git a/scrapy/contrib_exp/iterators.py b/scrapy/contrib_exp/iterators.py
index d96105fb9..c59f47bcc 100644
--- a/scrapy/contrib_exp/iterators.py
+++ b/scrapy/contrib_exp/iterators.py
@@ -1,42 +1,6 @@
-from scrapy.http import Response
-from scrapy.selector import Selector
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib_exp.iterators` is deprecated, use `scrapy.utils.iterators` instead",
+    ScrapyDeprecationWarning, stacklevel=2)
 
-
-def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
-    from lxml import etree
-    reader = _StreamReader(obj)
-    tag = '{%s}%s' % (namespace, nodename) if namespace else nodename
-    iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
-    selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename)
-    for _, node in iterable:
-        nodetext = etree.tostring(node)
-        node.clear()
-        xs = Selector(text=nodetext, type='xml')
-        if namespace:
-            xs.register_namespace(prefix, namespace)
-        yield xs.xpath(selxpath)[0]
-
-
-class _StreamReader(object):
-
-    def __init__(self, obj):
-        self._ptr = 0
-        if isinstance(obj, Response):
-            self._text, self.encoding = obj.body, obj.encoding
-        else:
-            self._text, self.encoding = obj, 'utf-8'
-        self._is_unicode = isinstance(self._text, unicode)
-
-    def read(self, n=65535):
-        self.read = self._read_unicode if self._is_unicode else self._read_string
-        return self.read(n).lstrip()
-
-    def _read_string(self, n=65535):
-        s, e = self._ptr, self._ptr + n
-        self._ptr = e
-        return self._text[s:e]
-
-    def _read_unicode(self, n=65535):
-        s, e = self._ptr, self._ptr + n
-        self._ptr = e
-        return self._text[s:e].encode('utf-8')
+from scrapy.utils.iterators import xmliter_lxml
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 78ea7114e..a889114d5 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -35,6 +35,46 @@ def xmliter(obj, nodename):
         yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]
 
 
+def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
+    from lxml import etree
+    reader = _StreamReader(obj)
+    tag = '{%s}%s' % (namespace, nodename) if namespace else nodename
+    iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
+    selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename)
+    for _, node in iterable:
+        nodetext = etree.tostring(node)
+        node.clear()
+        xs = Selector(text=nodetext, type='xml')
+        if namespace:
+            xs.register_namespace(prefix, namespace)
+        yield xs.xpath(selxpath)[0]
+
+
+class _StreamReader(object):
+
+    def __init__(self, obj):
+        self._ptr = 0
+        if isinstance(obj, Response):
+            self._text, self.encoding = obj.body, obj.encoding
+        else:
+            self._text, self.encoding = obj, 'utf-8'
+        self._is_unicode = isinstance(self._text, unicode)
+
+    def read(self, n=65535):
+        self.read = self._read_unicode if self._is_unicode else self._read_string
+        return self.read(n).lstrip()
+
+    def _read_string(self, n=65535):
+        s, e = self._ptr, self._ptr + n
+        self._ptr = e
+        return self._text[s:e]
+
+    def _read_unicode(self, n=65535):
+        s, e = self._ptr, self._ptr + n
+        self._ptr = e
+        return self._text[s:e].encode('utf-8')
+
+
 def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     """ Returns an iterator of dictionaries from the given csv object
 
diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py
index 81e12b4f9..f3f862604 100644
--- a/tests/test_downloadermiddleware_decompression.py
+++ b/tests/test_downloadermiddleware_decompression.py
@@ -1,6 +1,6 @@
 from unittest import TestCase, main
 from scrapy.http import Response, XmlResponse
-from scrapy.contrib_exp.downloadermiddleware.decompression import DecompressionMiddleware
+from scrapy.contrib.downloadermiddleware.decompression import DecompressionMiddleware
 from scrapy.spider import Spider
 from tests import get_testdata
 from scrapy.utils.test import assert_samelines
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index d8faa810c..a7042a6cf 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,8 +1,7 @@
 import os
 from twisted.trial import unittest
 
-from scrapy.utils.iterators import csviter, xmliter, _body_or_str
-from scrapy.contrib_exp.iterators import xmliter_lxml
+from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
 from scrapy.http import XmlResponse, TextResponse, Response
 from tests import get_testdata
 

From 9ea309c354711e5e33864624a16b4f15b8ddd242 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Thu, 9 Apr 2015 16:56:35 -0300
Subject: [PATCH 0175/2974] rename deployment.rst -> deploy.rst (consist with
 others like debug.rst)

---
 docs/topics/{deployment.rst => deploy.rst} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename docs/topics/{deployment.rst => deploy.rst} (100%)

diff --git a/docs/topics/deployment.rst b/docs/topics/deploy.rst
similarity index 100%
rename from docs/topics/deployment.rst
rename to docs/topics/deploy.rst

From 1a12922b77fcabaea66b118980318a0f7117d2c0 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Thu, 9 Apr 2015 17:39:01 -0300
Subject: [PATCH 0176/2974] improve scrapy deploy documentation

---
 docs/faq.rst           |  2 +-
 docs/index.rst         |  6 ++--
 docs/topics/deploy.rst | 77 ++++++++++++++++++++++++++++++++----------
 3 files changed, 64 insertions(+), 21 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 71d9e4c4e..b92d173dd 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -154,7 +154,7 @@ For more info see:
 What is the recommended way to deploy a Scrapy crawler in production?
 ---------------------------------------------------------------------
 
-See :ref:`topics-scrapyd`.
+See :ref:`topics-deploy`.
 
 Can I use JSON for large exports?
 ---------------------------------
diff --git a/docs/index.rst b/docs/index.rst
index 4097aba31..b442df4ba 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -150,7 +150,7 @@ Solving specific problems
    topics/leaks
    topics/images
    topics/ubuntu
-   topics/deployment
+   topics/deploy
    topics/autothrottle
    topics/benchmarking
    topics/jobs
@@ -186,8 +186,8 @@ Solving specific problems
 :doc:`topics/ubuntu`
     Install latest Scrapy packages easily on Ubuntu
 
-:doc:`topics/deployment`
-    Deploying your Scrapy project in production.
+:doc:`topics/deploy`
+    Deploying your Scrapy spiders and run them in a remote server.
 
 :doc:`topics/autothrottle`
     Adjust crawl rate dynamically based on load.
diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst
index 8f8c55c2f..f57069921 100644
--- a/docs/topics/deploy.rst
+++ b/docs/topics/deploy.rst
@@ -1,29 +1,72 @@
-.. _topics-deployment:
+.. _topics-deploy:
 
-==========
-Deployment
-==========
+=================
+Deploying Spiders
+=================
 
-The recommended way to deploy Scrapy projects to a server is through `Scrapyd`_.
+This section describes the different options you have for deploying your Scrapy
+spiders to run them on a regular basis. Running Scrapy spiders in your local
+machine is very convenient for the (early) development stage, but not so much
+when you need to execute long-running spiders or move spiders to run in
+production continously. This is where the solutions for deploying Scrapy
+spiders come in.
 
-.. _Scrapyd: https://github.com/scrapy/scrapyd
+The most popular choices, for deploying Scrapy spiders, are:
+
+* :ref:`Scrapy Cloud <deploy-scrapy-cloud>` (open source, easier to setup)
+* :ref:`Scrapyd <deploy-scrapyd>` (open source, harder to setup)
+
+.. _deploy-scrapy-cloud:
+
+Deploying to Scrapy Cloud
+=========================
+
+`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_, the company
+behind Scrapy.
+
+Advantages:
+
+- easy to setup (no need to setup or manage servers)
+- well-designed UI to manage spiders and review scraped items, logs and stats
+- cheap pricing (cheaper than renting a server, for small workloads)
+
+Disadvantages:
+
+- it's not open source
+
+To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool.
+Please refer to the `Scrapy Cloud documentation`_ for more information.
+
+The configuration is read from the ``scrapy.cfg`` file just like
+``scrapyd-deploy``.
+
+.. _deploy-scrapyd:
 
 Deploying to a Scrapyd Server
 =============================
 
-You can deploy to a Scrapyd server using the `Scrapyd client <https://github.com/scrapy/scrapyd-client>`_. You can add targets to your ``scrapy.cfg`` file which can be deployed to using the ``scrapyd-deploy`` command.
+`Scrapyd`_ is an open source application to run Scrapy spiders. It is
+maintained by some of the Scrapy developers.
 
-The basic syntax is as follows:
+Advantages:
 
-    scrapyd-deploy <target> -p <project>
+- it's open source, so it can be installed and run anywhere
 
-For more information please refer to the `Deploying your project`_ section.
+Disadvantages:
 
+- simple UI (no analytics, graphs or rich log/items browsing)
+- requires setting up servers, installing and configuring scrapyd on them. An
+  APT repo with Ubuntu packages is provided by the Scrapyd team
+
+To deploy spiders to Scrapyd, you can use the scrapyd-deploy tool provided by
+the `scrapyd-client`_ package. Please refer to the `scrapyd-deploy
+documentation`_ for more information.
+
+.. _Scrapyd: https://github.com/scrapy/scrapyd
 .. _Deploying your project: https://scrapyd.readthedocs.org/en/latest/deploy.html
-
-Deploying to Scrapinghub
-========================
-
-You can deploy to Scrapinghub using Scrapinghub's command line client, `shub`_. The configuration is read from the ``scrapy.cfg`` file just like ``scrapyd-deploy``.
-
-.. _shub: https://github.com/scrapinghub/shub
+.. _Scrapy Cloud: http://scrapinghub.com/scrapy-cloud/
+.. _scrapyd-client: https://github.com/scrapy/scrapyd-client
+.. _shub: http://doc.scrapinghub.com/shub.html
+.. _scrapyd-deploy documentation: http://scrapyd.readthedocs.org/en/latest/deploy.html
+.. _Scrapy Cloud documentation: http://doc.scrapinghub.com/scrapy-cloud.html
+.. _Scrapinghub: http://scrapinghub.com/

From 39fd2e62ef8f67170edcd6d9b3da68ee0ba2c4e5 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Thu, 9 Apr 2015 18:01:11 -0300
Subject: [PATCH 0177/2974] fix typo in deploy.rst

---
 docs/topics/deploy.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst
index f57069921..e2e796cf7 100644
--- a/docs/topics/deploy.rst
+++ b/docs/topics/deploy.rst
@@ -13,7 +13,7 @@ spiders come in.
 
 The most popular choices, for deploying Scrapy spiders, are:
 
-* :ref:`Scrapy Cloud <deploy-scrapy-cloud>` (open source, easier to setup)
+* :ref:`Scrapy Cloud <deploy-scrapy-cloud>` (cloud-based, easier to setup)
 * :ref:`Scrapyd <deploy-scrapyd>` (open source, harder to setup)
 
 .. _deploy-scrapy-cloud:

From 543d02bc64a76d7e508b05bff31cb3a6515a2461 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 9 Apr 2015 21:36:24 -0300
Subject: [PATCH 0178/2974] Avoid name clashing in deploy.py, fixes #1143

---
 scrapy/commands/deploy.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/commands/deploy.py b/scrapy/commands/deploy.py
index b7d989f69..bd8ab3f15 100644
--- a/scrapy/commands/deploy.py
+++ b/scrapy/commands/deploy.py
@@ -194,21 +194,21 @@ def _upload_egg(target, eggpath, project, version):
     _log('Deploying to project "%s" in %s' % (project, url))
     return _http_post(req)
 
-def _add_auth_header(request, target):
+def _add_auth_header(req, target):
     if 'username' in target:
         u, p = target.get('username'), target.get('password', '')
-        request.add_header('Authorization', basic_auth_header(u, p))
+        req.add_header('Authorization', basic_auth_header(u, p))
     else: # try netrc
         try:
             host = urlparse(target['url']).hostname
             a = netrc.netrc().authenticators(host)
-            request.add_header('Authorization', basic_auth_header(a[0], a[2]))
+            req.add_header('Authorization', basic_auth_header(a[0], a[2]))
         except (netrc.NetrcParseError, IOError, TypeError):
             pass
 
-def _http_post(request):
+def _http_post(req):
     try:
-        f = request.urlopen(request)
+        f = request.urlopen(req)
         _log("Server response (%s):" % f.code)
         print(f.read())
         return True

From 6b4439eacc5612ce584de94053e725610f2e15a5 Mon Sep 17 00:00:00 2001
From: bosnj <bosnj@users.noreply.github.com>
Date: Fri, 10 Apr 2015 15:32:32 +0200
Subject: [PATCH 0179/2974] default return value for extract_first

---
 scrapy/selector/unified.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index c0eefb85e..db8b0bc2d 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -182,9 +182,9 @@ class SelectorList(list):
     def extract(self):
         return [x.extract() for x in self]
 
-    def extract_first(self):
+    def extract_first(self, default=None):
         for x in self:
-            return x.extract()
+            return x.extract() or default
 
     @deprecated(use_instead='.extract()')
     def extract_unquoted(self):

From bc705843444472dbcb8eb583b8d87e055a1b9798 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 10 Apr 2015 11:52:57 -0300
Subject: [PATCH 0180/2974] Support crawlers as first positional arg in
 CrawlerRunner.crawl()

---
 scrapy/crawler.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 692a896be..615303528 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -82,9 +82,12 @@ class CrawlerRunner(object):
         self.crawlers = set()
         self._active = set()
 
-    def crawl(self, spidercls, *args, **kwargs):
-        crawler = self._create_crawler(spidercls)
-        self._setup_crawler_logging(crawler)
+    def crawl(self, crawler_or_spidercls, *args, **kwargs):
+        crawler = crawler_or_spidercls
+        if not isinstance(crawler_or_spidercls, Crawler):
+            crawler = self._create_crawler(crawler_or_spidercls)
+            self._setup_crawler_logging(crawler)
+
         self.crawlers.add(crawler)
         d = crawler.crawl(*args, **kwargs)
         self._active.add(d)

From 86b095132770f8d1ba84f6502a5929c6ae102e04 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 10 Apr 2015 12:06:27 -0300
Subject: [PATCH 0181/2974] Delete `crawler_deferreds` doc in CrawlerRunner

This attribute is now an internal one since it's main use-case was
covered by CrawlerRunner.stop().
---
 docs/topics/api.rst | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 57b8ee0cf..70b75a868 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -116,11 +116,6 @@ how you :ref:`configure the downloader middlewares
        Set of :class:`crawlers <scrapy.crawler.Crawler>` created by the
        :meth:`crawl` method.
 
-    .. attribute:: crawl_deferreds
-
-       Set of the `deferreds`_ return by the :meth:`crawl` method. This
-       collection it's useful for keeping track of current crawling state.
-
     .. method:: crawl(spidercls, \*args, \**kwargs)
 
        This method sets up the crawling of the given `spidercls` with the

From 3dabde6706b6c2f1015167f959fefec836bab9b1 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 10 Apr 2015 12:09:07 -0300
Subject: [PATCH 0182/2974] Update docs for CrawlerRunner.crawl() new usage

---
 docs/topics/api.rst | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 70b75a868..2055682dc 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -101,7 +101,7 @@ how you :ref:`configure the downloader middlewares
 
 .. class:: CrawlerRunner(settings)
 
-    This is a convenient helper class that creates, configures and runs
+    This is a convenient helper class that keeps track of, manages and runs
     crawlers inside an already setup Twisted `reactor`_.
 
     The CrawlerRunner object must be instantiated with a
@@ -116,18 +116,23 @@ how you :ref:`configure the downloader middlewares
        Set of :class:`crawlers <scrapy.crawler.Crawler>` created by the
        :meth:`crawl` method.
 
-    .. method:: crawl(spidercls, \*args, \**kwargs)
+    .. method:: crawl(crawler_or_spidercls, \*args, \**kwargs)
 
-       This method sets up the crawling of the given `spidercls` with the
-       provided arguments.
+       This method runs a crawler with the provided arguments.
 
-       It takes care of loading the spider class while configuring and starting
-       a crawler for it.
+       It will keep track of the given crawler so it can be stopped later,
+       while calling its :meth:`Crawler.crawl` method.
+
+       If `crawler_or_spidercls` isn't a :class:`~scrapy.crawler.Crawler`
+       instance, it will try to create one using this parameter as the spider
+       class given to it.
 
        Returns a deferred that is fired when the crawl is finished.
 
-       :param spidercls: spider class or spider's name inside the project
-       :type spidercls: :class:`~scrapy.spider.Spider` subclass or str
+       :param crawler_or_spidercls: already created crawler, or a spider class
+       or spider's name inside the project to create it
+       :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance,
+        :class:`~scrapy.spider.Spider` subclass or string
 
        :param args: arguments to initializate the spider
        :type args: list

From 24a07fd892c1bc70b284f4e41780681888516336 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 10 Apr 2015 13:43:33 -0300
Subject: [PATCH 0183/2974] Verify SPIDER_MANAGER_CLASS interface in
 CrawlerRunner

---
 scrapy/crawler.py     |  3 +++
 tests/test_crawler.py | 17 +++++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 692a896be..82d94144d 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -3,9 +3,11 @@ import signal
 import warnings
 
 from twisted.internet import reactor, defer
+from zope.interface.verify import verifyClass
 
 from scrapy.core.engine import ExecutionEngine
 from scrapy.resolver import CachingThreadedResolver
+from scrapy.interfaces import ISpiderManager
 from scrapy.extension import ExtensionManager
 from scrapy.signalmanager import SignalManager
 from scrapy.exceptions import ScrapyDeprecationWarning
@@ -78,6 +80,7 @@ class CrawlerRunner(object):
     def __init__(self, settings):
         self.settings = settings
         smcls = load_object(settings['SPIDER_MANAGER_CLASS'])
+        verifyClass(ISpiderManager, smcls)
         self.spiders = smcls.from_settings(settings.frozencopy())
         self.crawlers = set()
         self._active = set()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 34fc1a367..2cf860593 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -2,6 +2,7 @@ import warnings
 import unittest
 
 from twisted.internet import defer
+from zope.interface.verify import DoesNotImplement
 
 from scrapy.crawler import Crawler, CrawlerRunner
 from scrapy.settings import Settings
@@ -42,3 +43,19 @@ class CrawlerTestCase(unittest.TestCase):
 
         self.assertFalse(settings.frozen)
         self.assertTrue(crawler.settings.frozen)
+
+
+def SpiderManagerWithWrongInterface(object):
+
+    def unneeded_method(self):
+        pass
+
+
+class CrawlerRunnerTestCase(unittest.TestCase):
+
+    def test_spider_manager_verify_interface(self):
+        settings = Settings({
+            'SPIDER_MANAGER_CLASS': 'tests.test_crawler.SpiderManagerWithWrongInterface'
+        })
+        with self.assertRaises(DoesNotImplement):
+            CrawlerRunner(settings)

From bf301b73284153f8f6de0be5eb47dbaa197a9d3e Mon Sep 17 00:00:00 2001
From: Nikolaos-Digenis Karagiannis <digenishjkl@gmail.com>
Date: Mon, 17 Nov 2014 19:30:08 +0200
Subject: [PATCH 0184/2974] encode invalid xpath with unicode_escape under PY2

The exception quotes an xpath string
which may be unicode.
---
 scrapy/selector/unified.py |  4 +++-
 tests/test_selector.py     | 19 +++++++++++--------
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index c0eefb85e..90dcfd686 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -3,6 +3,7 @@ XPath selectors based on lxml
 """
 
 from lxml import etree
+import six
 
 from scrapy.utils.misc import extract_regex
 from scrapy.utils.trackref import object_ref
@@ -95,7 +96,8 @@ class Selector(object_ref):
             result = xpathev(query, namespaces=self.namespaces,
                              smart_strings=self._lxml_smart_strings)
         except etree.XPathError:
-            raise ValueError("Invalid XPath: %s" % query)
+            msg = u"Invalid XPath: %s" % query
+            raise ValueError(msg if six.PY3 else msg.encode("unicode_escape"))
 
         if type(result) is not list:
             result = [result]
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 9b8613319..a46f7635c 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -1,6 +1,7 @@
 import re
 import warnings
 import weakref
+import six
 from twisted.trial import unittest
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import TextResponse, HtmlResponse, XmlResponse
@@ -242,17 +243,19 @@ class SelectorTestCase(unittest.TestCase):
         self.assertEqual(xs.xpath('.').extract(), [u'<root>lala</root>'])
 
     def test_invalid_xpath(self):
+        "Test invalid xpath raises ValueError with the invalid xpath"
         response = XmlResponse(url="http://example.com", body="<html></html>")
         x = self.sscls(response)
         xpath = "//test[@foo='bar]"
-        try:
-            x.xpath(xpath)
-        except ValueError as e:
-            assert xpath in str(e), "Exception message does not contain invalid xpath"
-        except Exception:
-            raise AssertionError("A invalid XPath does not raise ValueError")
-        else:
-            raise AssertionError("A invalid XPath does not raise an exception")
+        self.assertRaisesRegexp(ValueError, re.escape(xpath), x.xpath, xpath)
+
+    def test_invalid_xpath_unicode(self):
+        "Test *Unicode* invalid xpath raises ValueError with the invalid xpath"
+        response = XmlResponse(url="http://example.com", body="<html></html>")
+        x = self.sscls(response)
+        xpath = u"//test[@foo='\u0431ar]"
+        encoded = xpath if six.PY3 else xpath.encode('unicode_escape')
+        self.assertRaisesRegexp(ValueError, re.escape(encoded), x.xpath, xpath)
 
     def test_http_header_encoding_precedence(self):
         # u'\xa3'     = pound symbol in unicode

From dce48b8654977d3f7a003addd507b0aa5e71a76c Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Sat, 11 Apr 2015 13:57:55 -0300
Subject: [PATCH 0185/2974] documenting Files Pipeline together with Images
 Pipeline

---
 docs/index.rst         |   2 +-
 docs/topics/images.rst | 250 ++++++++++++++++++++++++++---------------
 2 files changed, 159 insertions(+), 93 deletions(-)

diff --git a/docs/index.rst b/docs/index.rst
index de3e015d5..f61e7f348 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -181,7 +181,7 @@ Solving specific problems
     Learn how to find and get rid of memory leaks in your crawler.
 
 :doc:`topics/images`
-    Download static images associated with your scraped items.
+    Download files and/or images associated with your scraped items.
 
 :doc:`topics/ubuntu`
     Install latest Scrapy packages easily on Ubuntu
diff --git a/docs/topics/images.rst b/docs/topics/images.rst
index 890c7fd4a..ab4a98ebd 100644
--- a/docs/topics/images.rst
+++ b/docs/topics/images.rst
@@ -1,76 +1,94 @@
 .. _topics-images:
 
-=======================
-Downloading Item Images
-=======================
+===========================================
+Downloading and processing files and images
+===========================================
 
 .. currentmodule:: scrapy.contrib.pipeline.images
 
-Scrapy provides an :doc:`item pipeline </topics/item-pipeline>` for downloading
-images attached to a particular item, for example, when you scrape products and
-also want to download their images locally.
+Scrapy provides reusable :doc:`item pipelines </topics/item-pipeline>` for
+downloading fies attached to a particular item (for example, when you scrape
+products and also want to download their images locally). These pipelines share
+a bit of functionality and structure (we refer to them as media pipelines), but
+typically you'll either use the Files Pipeline or the Images Pipeline.
 
-This pipeline, called the Images Pipeline and implemented in the
-:class:`ImagesPipeline` class, provides a convenient way for
-downloading and storing images locally with some additional features:
+Both pipelines implement these features:
+
+* Avoid re-downloading media that was downloaded recently
+* Specifying where to store the files (filesystem directory, Amazon S3 bucket)
+
+The Images Pipeline has a few extra functions for processing images:
 
 * Convert all downloaded images to a common format (JPG) and mode (RGB)
-* Avoid re-downloading images which were downloaded recently
 * Thumbnail generation
 * Check images width/height to make sure they meet a minimum constraint
 
-This pipeline also keeps an internal queue of those images which are currently
-being scheduled for download, and connects those items that arrive containing
-the same image, to that queue. This avoids downloading the same image more than
+The pipelines also keep an internal queue of those images which are currently
+being scheduled for download, and connect those items that arrive containing
+the same image to that queue. This avoids downloading the same media more than
 once when it's shared by several items.
 
-`Pillow`_ is used for thumbnailing and normalizing images to JPEG/RGB format,
-so you need to install this library in order to use the images pipeline.
-`Python Imaging Library`_ (PIL) should also work in most cases, but it
-is known to cause troubles in some setups, so we recommend to use `Pillow`_
-instead of `PIL <Python Imaging Library>`_.
+The Images Pipeline uses `Pillow`_ for thumbnailing and normalizing images to
+JPEG/RGB format, so you need to install this library in order to use it.
+`Python Imaging Library`_ (PIL) should also work in most cases, but it is known
+to cause troubles in some setups, so we recommend to use `Pillow`_ instead of
+`PIL <Python Imaging Library>`_.
 
 .. _Pillow: https://github.com/python-pillow/Pillow
 .. _Python Imaging Library: http://www.pythonware.com/products/pil/
 
-Using the Images Pipeline
+Using the Files Pipeline
 =========================
 
-The typical workflow, when using the :class:`ImagesPipeline` goes like
+The typical workflow, when using the :class:`FilesPipeline` goes like
 this:
 
-1. In a Spider, you scrape an item and put the URLs of its images into a
-   ``image_urls`` field.
+1. In a Spider, you scrape an item and put the URLs of the desired into a
+   ``file_urls`` field.
 
 2. The item is returned from the spider and goes to the item pipeline.
 
-3. When the item reaches the :class:`ImagesPipeline`, the URLs in the
-   ``image_urls`` field are scheduled for download using the standard
+3. When the item reaches the :class:`FilesPipeline`, the URLs in the
+   ``file_urls`` field are scheduled for download using the standard
    Scrapy scheduler and downloader (which means the scheduler and downloader
    middlewares are reused), but with a higher priority, processing them before other
    pages are scraped. The item remains "locked" at that particular pipeline stage
-   until the images have finish downloading (or fail for some reason).
+   until the files have finish downloading (or fail for some reason).
 
-4. When the images are downloaded another field (``images``) will be populated
+4. When the files are downloaded, another field (``files``) will be populated
    with the results. This field will contain a list of dicts with information
-   about the images downloaded, such as the downloaded path, the original
-   scraped url (taken from the ``image_urls`` field) , and the image checksum.
-   The images in the list of the ``images`` field will retain the same order of
-   the original ``image_urls`` field. If some image failed downloading, an
-   error will be logged and the image won't be present in the ``images`` field.
+   about the downloaded files, such as the downloaded path, the original
+   scraped url (taken from the ``file_urls`` field) , and the file checksum.
+   The files in the list of the ``files`` field will retain the same order of
+   the original ``file_urls`` field. If some file failed downloading, an
+   error will be logged and the file won't be present in the ``files`` field.
+
+
+Using the Images Pipeline
+=========================
+
+Using the :class:`ImagesPipeline` is a lot like using the :class:`FilesPipeline`,
+except the default field names used are different: you use ``image_urls`` for
+the image URLs of an item and it will populate an ``images`` field for the information
+about the downloaded images.
+
+The advantage of using the :class:`ImagesPipeline` for image files is that you
+can configure some extra functions like generating thumbnails and filtering
+the images based on their size.
 
 
 Usage example
 =============
 
-In order to use the image pipeline first  
-:ref:`enable it <topics-images-enabling>`.
+In order to use a media pipeline first, :ref:`enable it
+<topics-media-pipeline-enabling>`.
 
-Then, if a spider returns a dict with 'image_urls' key, 
-the pipeline will put the results under 'images' key.
+Then, if a spider returns a dict with the URLs key ('file_urls' or
+'image_urls', for the Files or Images Pipeline respectively), the pipeline will
+put the results under respective key ('files' or images').
 
-If you prefer to use :class:`~.Item` then define a custom 
-item with the ``image_urls`` and ``images`` fields::
+If you prefer to use :class:`~.Item`, then define a custom item with the
+necessary fields, like in this example for Images Pipeline::
 
     import scrapy
 
@@ -80,42 +98,57 @@ item with the ``image_urls`` and ``images`` fields::
         image_urls = scrapy.Field()
         images = scrapy.Field()
         
-If you need something more complex and want to override the custom images
-pipeline behaviour, see :ref:`topics-images-override`.
+If you need something more complex and want to override the custom pipeline
+behaviour, see :ref:`topics-media-pipeline-override`.
 
-.. _topics-images-enabling:
+.. _topics-media-pipeline-enabling:
 
-Enabling your Images Pipeline
+Enabling your Media Pipeline
 =============================
 
 .. setting:: IMAGES_STORE
+.. setting:: FILES_STORE
 
-To enable your images pipeline you must first add it to your project
-:setting:`ITEM_PIPELINES` setting::
+To enable your media pipeline you must first add it to your project
+:setting:`ITEM_PIPELINES` setting.
+
+For Images Pipeline, use::
 
     ITEM_PIPELINES = {'scrapy.contrib.pipeline.images.ImagesPipeline': 1}
 
-And set the :setting:`IMAGES_STORE` setting to a valid directory that will be
-used for storing the downloaded images. Otherwise the pipeline will remain
-disabled, even if you include it in the :setting:`ITEM_PIPELINES` setting.
+For Files Pipeline, use::
 
-For example::
+    ITEM_PIPELINES = {'scrapy.contrib.pipeline.files.FilesPipeline': 1}
+
+
+.. note::
+    You can also use both the Files and Images Pipeline at the same time.
+
+
+Then, configure the target storage setting to a valid value that will be used
+for storing the downloaded images. Otherwise the pipeline will remain disabled,
+even if you include it in the :setting:`ITEM_PIPELINES` setting.
+
+For the Files Pipeline, set the :setting:`FILES_STORE` setting::
+
+   FILES_STORE = '/path/to/valid/dir'
+
+For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
 
    IMAGES_STORE = '/path/to/valid/dir'
 
-Images Storage
-==============
+Supported Storage
+=================
 
 File system is currently the only officially supported storage, but there is
-also (undocumented) support for `Amazon S3`_.
+also (undocumented) support for storing files in `Amazon S3`_.
 
 .. _Amazon S3: http://aws.amazon.com/s3/
 
 File system storage
 -------------------
 
-The images are stored in files (one per image), using a `SHA1 hash`_ of their
-URLs for the file names.
+The files are stored using a `SHA1 hash`_ of their URLs for the file names.
 
 For example, the following image URL::
 
@@ -132,29 +165,36 @@ Will be downloaded and stored in the following file::
 Where:
 
 * ``<IMAGES_STORE>`` is the directory defined in :setting:`IMAGES_STORE` setting
+for the Images Pipeline.
 
 * ``full`` is a sub-directory to separate full images from thumbnails (if
   used). For more info see :ref:`topics-images-thumbnails`.
 
+
 Additional features
 ===================
 
-Image expiration
+File expiration
 ----------------
 
 .. setting:: IMAGES_EXPIRES
+.. setting:: FILES_EXPIRES
 
-The Image Pipeline avoids downloading images that were downloaded recently. To
-adjust this retention delay use the :setting:`IMAGES_EXPIRES` setting, which
+The Image Pipeline avoids downloading files that were downloaded recently. To
+adjust this retention delay use the :setting:`FILES_EXPIRES` setting (or
+:setting:`IMAGES_EXPIRES`, in case of Images Pipeline), which
 specifies the delay in number of days::
 
-    # 90 days of delay for image expiration
-    IMAGES_EXPIRES = 90
+    # 90 days of delay for files expiration
+    FILES_EXPIRES = 90
+
+    # 30 days of delay for images expiration
+    IMAGES_EXPIRES = 30
 
 .. _topics-images-thumbnails:
 
-Thumbnail generation
---------------------
+Thumbnail generation for images
+-------------------------------
 
 The Images Pipeline can automatically create thumbnails of the downloaded
 images.
@@ -200,8 +240,9 @@ Filtering out small images
 
 .. setting:: IMAGES_MIN_WIDTH
 
-You can drop images which are too small, by specifying the minimum allowed size
-in the :setting:`IMAGES_MIN_HEIGHT` and :setting:`IMAGES_MIN_WIDTH` settings.
+When using the Images Pipeline, you can drop images which are too small, by
+specifying the minimum allowed size in the :setting:`IMAGES_MIN_HEIGHT` and
+:setting:`IMAGES_MIN_WIDTH` settings.
 
 For example::
 
@@ -212,45 +253,45 @@ Note: these size constraints don't affect thumbnail generation at all.
 
 By default, there are no size constraints, so all images are processed.
 
-.. _topics-images-override:
+.. _topics-media-pipeline-override:
 
-Implementing your custom Images Pipeline
-========================================
+Extending the Media Pipelines
+=============================
 
-.. module:: scrapy.contrib.pipeline.images
-   :synopsis: Images Pipeline
+.. module:: scrapy.contrib.pipeline.files
+   :synopsis: Files Pipeline
 
-Here are the methods that you should override in your custom Images Pipeline:
+See here the methods that you can override in your custom Files Pipeline:
 
-.. class:: ImagesPipeline
+.. class:: FilesPipeline
 
-   .. method:: get_media_requests(item, info)
+   .. method:: FilesPipeline.get_media_requests(item, info)
 
       As seen on the workflow, the pipeline will get the URLs of the images to
-      download from the item. In order to do this, you must override the
+      download from the item. In order to do this, you can override the
       :meth:`~get_media_requests` method and return a Request for each
-      image URL::
+      file URL::
 
          def get_media_requests(self, item, info):
-             for image_url in item['image_urls']:
-                 yield scrapy.Request(image_url)
+             for file_url in item['file_urls']:
+                 yield scrapy.Request(file_url)
 
       Those requests will be processed by the pipeline and, when they have finished
       downloading, the results will be sent to the
       :meth:`~item_completed` method, as a list of 2-element tuples.
-      Each tuple will contain ``(success, image_info_or_failure)`` where:
+      Each tuple will contain ``(success, file_info_or_error)`` where:
 
       * ``success`` is a boolean which is ``True`` if the image was downloaded
         successfully or ``False`` if it failed for some reason
 
-      * ``image_info_or_error`` is a dict containing the following keys (if success
+      * ``file_info_or_error`` is a dict containing the following keys (if success
         is ``True``) or a `Twisted Failure`_ if there was a problem.
 
-        * ``url`` - the url where the image was downloaded from. This is the url of
+        * ``url`` - the url where the file was downloaded from. This is the url of
           the request returned from the :meth:`~get_media_requests`
           method.
 
-        * ``path`` - the path (relative to :setting:`IMAGES_STORE`) where the image
+        * ``path`` - the path (relative to :setting:`FILES_STORE`) where the file
           was stored
 
         * ``checksum`` - a `MD5 hash`_ of the image contents
@@ -263,21 +304,17 @@ Here are the methods that you should override in your custom Images Pipeline:
 
           [(True,
             {'checksum': '2b00042f7481c7b056c4b410d28f33cf',
-             'path': 'full/7d97e98f8af710c7e7fe703abc8f639e0ee507c4.jpg',
-             'url': 'http://www.example.com/images/product1.jpg'}),
-           (True,
-            {'checksum': 'b9628c4ab9b595f72f280b90c4fd093d',
-             'path': 'full/1ca5879492b8fd606df1964ea3c1e2f4520f076f.jpg',
-             'url': 'http://www.example.com/images/product2.jpg'}),
+             'path': 'full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg',
+             'url': 'http://www.example.com/files/product1.pdf'}),
            (False,
             Failure(...))]
 
       By default the :meth:`get_media_requests` method returns ``None`` which
-      means there are no images to download for the item.
+      means there are no files to download for the item.
 
-   .. method:: item_completed(results, items, info)
+   .. method:: FilesPipeline.item_completed(results, items, info)
 
-      The :meth:`ImagesPipeline.item_completed` method called when all image
+      The :meth:`FilesPipeline.item_completed` method called when all file
       requests for a single item have completed (either finished downloading, or
       failed for some reason).
 
@@ -286,21 +323,50 @@ Here are the methods that you should override in your custom Images Pipeline:
       return (or drop) the item, as you would in any pipeline.
 
       Here is an example of the :meth:`~item_completed` method where we
-      store the downloaded image paths (passed in results) in the ``image_paths``
-      item field, and we drop the item if it doesn't contain any images::
+      store the downloaded file paths (passed in results) in the ``file_paths``
+      item field, and we drop the item if it doesn't contain any files::
 
           from scrapy.exceptions import DropItem
 
           def item_completed(self, results, item, info):
-              image_paths = [x['path'] for ok, x in results if ok]
-              if not image_paths:
-                  raise DropItem("Item contains no images")
-              item['image_paths'] = image_paths
+              file_paths = [x['path'] for ok, x in results if ok]
+              if not file_paths:
+                  raise DropItem("Item contains no files")
+              item['file_paths'] = file_paths
               return item
 
       By default, the :meth:`item_completed` method returns the item.
 
 
+.. module:: scrapy.contrib.pipeline.images
+   :synopsis: Images Pipeline
+
+See here the methods that you can override in your custom Images Pipeline:
+
+.. class:: ImagesPipeline
+
+    The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
+    customizing the field names and adding custom behavior for images.
+
+   .. method:: ImagesPipeline.get_media_requests(item, info)
+
+      Works the same way as :meth:`FilesPipeline.get_media_requests` method,
+      but using a different field name for image urls.
+
+      Must return a Request for each image URL.
+
+   .. method:: ImagesPipeline.item_completed(results, items, info)
+
+      The :meth:`ImagesPipeline.item_completed` method is called when all image
+      requests for a single item have completed (either finished downloading, or
+      failed for some reason).
+
+      Works the same way as :meth:`FilesPipeline.item_completed` method,
+      but using a different field names for storing image downloading results.
+
+      By default, the :meth:`item_completed` method returns the item.
+
+
 Custom Images pipeline example
 ==============================
 

From 1eccd34a7032c3e50c56f61da34ef39d1aaf43ec Mon Sep 17 00:00:00 2001
From: mrpandav <pandav.dharmesh@gmail.com>
Date: Sun, 12 Apr 2015 11:11:28 +0530
Subject: [PATCH 0186/2974] adding feature improvement  for selecting form
 using form-id, in addition to formname , formnumer before we go for xpath..
 making it more idiomatic in nature

---
 scrapy/http/request/form.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index cae56f229..82cb5f5d4 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -54,7 +54,7 @@ def _urlencode(seq, enc):
     return urlencode(values, doseq=1)
 
 
-def _get_form(response, formname, formnumber, formxpath):
+def _get_form(response, formname, formid, formnumber, formxpath):
     """Find the form element """
     from scrapy.selector.lxmldocument import LxmlDocument
     root = LxmlDocument(response, lxml.html.HTMLParser)

From cf9d848f0f8169f82a455977c2df6390a50a1e59 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Sun, 12 Apr 2015 14:18:47 +0200
Subject: [PATCH 0187/2974] Fixed deprecated settings API in cmdline test

---
 tests/test_cmdline/settings.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_cmdline/settings.py b/tests/test_cmdline/settings.py
index 9aceffb0d..8a719ddf2 100644
--- a/tests/test_cmdline/settings.py
+++ b/tests/test_cmdline/settings.py
@@ -1,5 +1,5 @@
-EXTENSIONS = [
-    'tests.test_cmdline.extensions.TestExtension'
-]
+EXTENSIONS = {
+    'tests.test_cmdline.extensions.TestExtension': 0,
+}
 
 TEST1 = 'default'

From 91a60d9f3d4107df334edbbc42f7ef52c4e57706 Mon Sep 17 00:00:00 2001
From: Diana Huang <diana.k.huang@gmail.com>
Date: Sun, 12 Apr 2015 13:44:32 -0400
Subject: [PATCH 0188/2974] Remove references to the `sel` object in shell.rst

The current documentation has references to the deprecated `sel` when interacting with the shell. I've removed them and replaced uses of `sel.xpath` with `response.xpath` instead.
---
 docs/topics/shell.rst | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 5c1cfbd47..8fce0cea1 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -85,9 +85,6 @@ Those objects are:
  * ``response`` - a :class:`~scrapy.http.Response` object containing the last
    fetched page
 
- * ``sel`` - a :class:`~scrapy.selector.Selector` object constructed
-   with the last response fetched
-
  * ``settings`` - the current :ref:`Scrapy settings <topics-settings>`
 
 Example of shell session
@@ -117,7 +114,6 @@ all start with the ``[s]`` prefix)::
     [s]   item       {}
     [s]   request    <GET http://scrapy.org>
     [s]   response   <200 http://scrapy.org>
-    [s]   sel        <Selector xpath=None data=u'<html>\n  <head>\n    <meta charset="utf-8'>
     [s]   settings   <scrapy.settings.Settings object at 0x2bfd650>
     [s]   spider     <Spider 'default' at 0x20c6f50>
     [s] Useful shortcuts:
@@ -129,8 +125,8 @@ all start with the ``[s]`` prefix)::
 
 After that, we can start playing with the objects::
 
-    >>> sel.xpath("//h2/text()").extract()[0]
-    u'Welcome to Scrapy'
+    >>> response.xpath("//h1/text()").extract()[0]
+    u'Meet Scrapy'
 
     >>> fetch("http://slashdot.org")
     [s] Available Scrapy objects:
@@ -138,7 +134,6 @@ After that, we can start playing with the objects::
     [s]   item       {}
     [s]   request    <GET http://slashdot.org>
     [s]   response   <200 http://slashdot.org>
-    [s]   sel        <Selector xpath=None data=u'<html lang="en">\n<head>\n\n\n\n\n<script id="'>
     [s]   settings   <scrapy.settings.Settings object at 0x2bfd650>
     [s]   spider     <Spider 'default' at 0x20c6f50>
     [s] Useful shortcuts:
@@ -146,7 +141,7 @@ After that, we can start playing with the objects::
     [s]   fetch(req_or_url) Fetch request (or URL) and update local objects
     [s]   view(response)    View response in a browser
 
-    >>> sel.xpath('//title/text()').extract()
+    >>> response.xpath('//title/text()').extract()
     [u'Slashdot: News for nerds, stuff that matters']
 
     >>> request = request.replace(method="POST")
@@ -203,7 +198,7 @@ When you run the spider, you will get something similar to this::
 
 Then, you can check if the extraction code is working::
 
-    >>> sel.xpath('//h1[@class="fn"]')
+    >>> response.xpath('//h1[@class="fn"]')
     []
 
 Nope, it doesn't. So you can open the response in your web browser and see if

From f71175a449e2b66c54f17c1d982da5376ef96900 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Sun, 12 Apr 2015 21:56:23 -0300
Subject: [PATCH 0189/2974] More replaces of references to old `sel` shortcut

---
 docs/topics/selectors.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 33958cee5..a84ac9c99 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -141,12 +141,12 @@ method, as follows::
 
 If you want to extract only first matched element, you can call the selector ``.extract_first()``
 
-    >>> sel.xpath('//div[@id="images"]/a/text()').extract_first()
+    >>> response.xpath('//div[@id="images"]/a/text()').extract_first()
     u'Name: My image 1 '
 
 It returns ``None`` if no element was found:
 
-    >>> sel.xpath('//div/[id="not-exists"]/text()').extract_first() is None
+    >>> response.xpath('//div/[id="not-exists"]/text()').extract_first() is None
     True
 
 Notice that CSS selectors can select text or attribute nodes using CSS3
@@ -383,7 +383,7 @@ with groups of itemscopes and corresponding itemprops::
     ...   ...
     ... </div>
     ... """
-    >>>
+    >>> sel = Selector(text=doc, type="html")
     >>> for scope in sel.xpath('//div[@itemscope]'):
     ...     print "current scope:", scope.xpath('@itemtype').extract()
     ...     props = scope.xpath('''

From 973c31f73595c73fac9b825ce7587d93d7055546 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 14 Apr 2015 20:11:46 +0500
Subject: [PATCH 0190/2974] TST cleanup: use assertIn instead of assert_

---
 scrapy/utils/testproc.py |  4 +++-
 tests/test_commands.py   | 26 +++++++++++++-------------
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index cba3b4346..adddad093 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -1,13 +1,15 @@
+from __future__ import absolute_import
 import sys
 import os
 
 from twisted.internet import reactor, defer, protocol
 
+
 class ProcessTest(object):
 
     command = None
     prefix = [sys.executable, '-m', 'scrapy.cmdline']
-    cwd = os.getcwd() # trial chdirs to temp dir
+    cwd = os.getcwd()  # trial chdirs to temp dir
 
     def execute(self, args, check_code=True, settings=None):
         env = os.environ.copy()
diff --git a/tests/test_commands.py b/tests/test_commands.py
index eb3556b62..68f76d002 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -95,11 +95,11 @@ class GenspiderCommandTest(CommandTest):
         spname = 'test_spider'
         p = self.proc('genspider', spname, 'test.com', *args)
         out = retry_on_eintr(p.stdout.read)
-        self.assert_("Created spider %r using template %r in module" % (spname, tplname) in out)
-        self.assert_(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
+        self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out)
+        self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
         p = self.proc('genspider', spname, 'test.com', *args)
         out = retry_on_eintr(p.stdout.read)
-        self.assert_("Spider %r already exists in module" % spname in out)
+        self.assertIn("Spider %r already exists in module" % spname, out)
 
     def test_template_basic(self):
         self.test_template('basic')
@@ -148,10 +148,10 @@ class MySpider(scrapy.Spider):
 """)
         p = self.proc('runspider', fname)
         log = p.stderr.read()
-        self.assert_("[myspider] DEBUG: It Works!" in log, log)
-        self.assert_("[myspider] INFO: Spider opened" in log, log)
-        self.assert_("[myspider] INFO: Closing spider (finished)" in log, log)
-        self.assert_("[myspider] INFO: Spider closed (finished)" in log, log)
+        self.assertIn("[myspider] DEBUG: It Works!", log)
+        self.assertIn("[myspider] INFO: Spider opened", log)
+        self.assertIn("[myspider] INFO: Closing spider (finished)", log)
+        self.assertIn("[myspider] INFO: Spider closed (finished)", log)
 
     def test_runspider_no_spider_found(self):
         tmpdir = self.mktemp()
@@ -164,12 +164,12 @@ from scrapy.spider import Spider
 """)
         p = self.proc('runspider', fname)
         log = p.stderr.read()
-        self.assert_("No spider found in file" in log)
+        self.assertIn("No spider found in file", log)
 
     def test_runspider_file_not_found(self):
         p = self.proc('runspider', 'some_non_existent_file')
         log = p.stderr.read()
-        self.assert_("File not found: some_non_existent_file" in log)
+        self.assertIn("File not found: some_non_existent_file", log)
 
     def test_runspider_unable_to_load(self):
         tmpdir = self.mktemp()
@@ -179,7 +179,7 @@ from scrapy.spider import Spider
             f.write("")
         p = self.proc('runspider', fname)
         log = p.stderr.read()
-        self.assert_("Unable to load" in log)
+        self.assertIn("Unable to load", log)
 
 
 class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
@@ -229,7 +229,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
                                            '-a', 'test_arg=1',
                                            '-c', 'parse',
                                            self.url('/html')])
-        self.assert_("[parse_spider] DEBUG: It Works!" in stderr, stderr)
+        self.assertIn("[parse_spider] DEBUG: It Works!", stderr)
 
     @defer.inlineCallbacks
     def test_pipelines(self):
@@ -237,7 +237,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
                                            '--pipelines',
                                            '-c', 'parse',
                                            self.url('/html')])
-        self.assert_("[scrapy] INFO: It Works!" in stderr, stderr)
+        self.assertIn("[scrapy] INFO: It Works!", stderr)
 
     @defer.inlineCallbacks
     def test_parse_items(self):
@@ -254,4 +254,4 @@ class BenchCommandTest(CommandTest):
         p = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
                 '-s', 'CLOSESPIDER_TIMEOUT=0.01')
         log = p.stderr.read()
-        self.assert_('INFO: Crawled' in log, log)
+        self.assertIn('INFO: Crawled', log)

From 5436ba117b687eb3bd4e7afa69d5a67935056342 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 14 Apr 2015 20:19:28 +0500
Subject: [PATCH 0191/2974] remove deprecated code from FeedExporter

---
 scrapy/contrib/feedexport.py | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py
index 92664220c..95354d6fe 100644
--- a/scrapy/contrib/feedexport.py
+++ b/scrapy/contrib/feedexport.py
@@ -151,15 +151,7 @@ class FeedExporter(object):
 
     @classmethod
     def from_crawler(cls, crawler):
-        if len(get_func_args(cls)) < 1:
-            # FIXME: remove for scrapy 0.17
-            import warnings
-            from scrapy.exceptions import ScrapyDeprecationWarning
-            warnings.warn("%s must receive a settings object as first constructor argument." % cls.__name__,
-                ScrapyDeprecationWarning, stacklevel=2)
-            o = cls()
-        else:
-            o = cls(crawler.settings)
+        o = cls(crawler.settings)
         crawler.signals.connect(o.open_spider, signals.spider_opened)
         crawler.signals.connect(o.close_spider, signals.spider_closed)
         crawler.signals.connect(o.item_scraped, signals.item_scraped)

From 7644813286c230866c2f7287bbaead87f2be104b Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 14 Apr 2015 23:26:05 +0500
Subject: [PATCH 0192/2974] allow Crawler, CrawlerRunner and CrawlerProcess to
 accept dicts instead of Setting objects

---
 docs/topics/practices.rst | 10 +++++-----
 scrapy/crawler.py         |  6 ++++++
 tests/test_crawler.py     | 23 +++++++++++++++++++++--
 3 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 3ec7bc29b..2f848e6d9 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -52,16 +52,16 @@ the spider class as first argument in the :meth:`CrawlerRunner.crawl
 ::
 
     from twisted.internet import reactor
-    from scrapy.spider import Spider
+    import scrapy
     from scrapy.crawler import CrawlerRunner
-    from scrapy.settings import Settings
 
-    class MySpider(Spider):
+    class MySpider(scrapy.Spider):
         # Your spider definition
         ...
 
-    settings = Settings({'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'})
-    runner = CrawlerRunner(settings)
+    runner = CrawlerRunner({
+        'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
+    })
 
     d = runner.crawl(MySpider)
     d.addBoth(lambda _: reactor.stop())
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 3e4d0dc7f..1b5029219 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -9,6 +9,7 @@ from scrapy.core.engine import ExecutionEngine
 from scrapy.resolver import CachingThreadedResolver
 from scrapy.interfaces import ISpiderManager
 from scrapy.extension import ExtensionManager
+from scrapy.settings import Settings
 from scrapy.signalmanager import SignalManager
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
@@ -19,6 +20,9 @@ from scrapy import log, signals
 class Crawler(object):
 
     def __init__(self, spidercls, settings):
+        if isinstance(settings, dict):
+            settings = Settings(settings)
+
         self.spidercls = spidercls
         self.settings = settings.copy()
 
@@ -78,6 +82,8 @@ class Crawler(object):
 class CrawlerRunner(object):
 
     def __init__(self, settings):
+        if isinstance(settings, dict):
+            settings = Settings(settings)
         self.settings = settings
         smcls = load_object(settings['SPIDER_MANAGER_CLASS'])
         verifyClass(ISpiderManager, smcls)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 2cf860593..20467833f 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,11 +1,10 @@
 import warnings
 import unittest
 
-from twisted.internet import defer
 from zope.interface.verify import DoesNotImplement
 
 from scrapy.crawler import Crawler, CrawlerRunner
-from scrapy.settings import Settings
+from scrapy.settings import Settings, default_settings
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
 
@@ -44,6 +43,16 @@ class CrawlerTestCase(unittest.TestCase):
         self.assertFalse(settings.frozen)
         self.assertTrue(crawler.settings.frozen)
 
+    def test_crawler_accepts_dict(self):
+        crawler = Crawler(DefaultSpider, {'foo': 'bar'})
+        self.assertEqual(crawler.settings['foo'], 'bar')
+        self.assertEqual(
+            crawler.settings['RETRY_ENABLED'],
+            default_settings.RETRY_ENABLED
+        )
+        self.assertIsInstance(crawler.settings, Settings)
+
+
 
 def SpiderManagerWithWrongInterface(object):
 
@@ -59,3 +68,13 @@ class CrawlerRunnerTestCase(unittest.TestCase):
         })
         with self.assertRaises(DoesNotImplement):
             CrawlerRunner(settings)
+
+    def test_crawler_runner_accepts_dict(self):
+        runner = CrawlerRunner({'foo': 'bar'})
+        self.assertEqual(runner.settings['foo'], 'bar')
+        self.assertEqual(
+            runner.settings['RETRY_ENABLED'],
+            default_settings.RETRY_ENABLED
+        )
+        self.assertIsInstance(runner.settings, Settings)
+

From 1534e8540bf083c8d7beb0264cccea5488ee0250 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Apr 2015 02:48:26 +0500
Subject: [PATCH 0193/2974] FEED_EXPORT_FIELDS option

---
 docs/topics/feed-exports.rst        |  28 +++++-
 scrapy/contrib/feedexport.py        |   7 +-
 scrapy/settings/default_settings.py |   1 +
 tests/mockserver.py                 |   1 +
 tests/test_contrib_feedexport.py    | 141 ++++++++++++++++++++++++++--
 5 files changed, 165 insertions(+), 13 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index e81db6489..a2bfdad74 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -30,7 +30,7 @@ For serializing the scraped data, the feed exports use the :ref:`Item exporters
 
 But you can also extend the supported format through the
 :setting:`FEED_EXPORTERS` setting.
- 
+
 .. _topics-feed-format-json:
 
 JSON
@@ -38,7 +38,8 @@ JSON
 
  * :setting:`FEED_FORMAT`: ``json``
  * Exporter used: :class:`~scrapy.contrib.exporter.JsonItemExporter`
- * See :ref:`this warning <json-with-large-data>` if you're using JSON with large feeds
+ * See :ref:`this warning <json-with-large-data>` if you're using JSON with
+   large feeds.
 
 .. _topics-feed-format-jsonlines:
 
@@ -55,6 +56,10 @@ CSV
 
  * :setting:`FEED_FORMAT`: ``csv``
  * Exporter used: :class:`~scrapy.contrib.exporter.CsvItemExporter`
+ * To specify columns to export and their order use
+   :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
+   option, but it is important for CSV because unlike many other export
+   formats CSV uses a fixed header.
 
 .. _topics-feed-format-xml:
 
@@ -202,6 +207,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_STORAGES`
  * :setting:`FEED_EXPORTERS`
  * :setting:`FEED_STORE_EMPTY`
+ * :setting:`FEED_EXPORT_FIELDS`
 
 .. currentmodule:: scrapy.contrib.feedexport
 
@@ -225,6 +231,20 @@ FEED_FORMAT
 The serialization format to be used for the feed. See
 :ref:`topics-feed-format` for possible values.
 
+.. setting:: FEED_EXPORT_FIELDS
+
+FEED_EXPORT_FIELDS
+------------------
+
+A list of fields to export, optional.
+Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
+
+Use FEED_EXPORT_FIELDS option to define fields to export and their order.
+
+When omitted, Scrapy uses fields defined in :class:`~.Item` subclasses a spider
+is yielding. If raw dicts are used as items Scrapy tries to infer field names
+from the exported data - currently it uses field names from the first item.
+
 .. setting:: FEED_STORE_EMPTY
 
 FEED_STORE_EMPTY
@@ -249,7 +269,7 @@ The keys are URI schemes and the values are paths to storage classes.
 FEED_STORAGES_BASE
 ------------------
 
-Default:: 
+Default::
 
     {
         '': 'scrapy.contrib.feedexport.FileFeedStorage',
@@ -277,7 +297,7 @@ classes.
 FEED_EXPORTERS_BASE
 -------------------
 
-Default:: 
+Default::
 
     FEED_EXPORTERS_BASE = {
         'json': 'scrapy.contrib.exporter.JsonItemExporter',
diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py
index 92664220c..43ebbe321 100644
--- a/scrapy/contrib/feedexport.py
+++ b/scrapy/contrib/feedexport.py
@@ -146,6 +146,7 @@ class FeedExporter(object):
         if not self._exporter_supported(self.format):
             raise NotConfigured
         self.store_empty = settings.getbool('FEED_STORE_EMPTY')
+        self.export_fields = settings.getlist('FEED_EXPORT_FIELDS')
         uripar = settings['FEED_URI_PARAMS']
         self._uripar = load_object(uripar) if uripar else lambda x, y: None
 
@@ -169,7 +170,7 @@ class FeedExporter(object):
         uri = self.urifmt % self._get_uri_params(spider)
         storage = self._get_storage(uri)
         file = storage.open(spider)
-        exporter = self._get_exporter(file)
+        exporter = self._get_exporter(file, fields_to_export=self.export_fields)
         exporter.start_exporting()
         self.slot = SpiderSlot(file, exporter, storage, uri)
 
@@ -218,8 +219,8 @@ class FeedExporter(object):
         else:
             log.msg("Unknown feed storage scheme: %s" % scheme, log.ERROR)
 
-    def _get_exporter(self, *a, **kw):
-        return self.exporters[self.format](*a, **kw)
+    def _get_exporter(self, *args, **kwargs):
+        return self.exporters[self.format](*args, **kwargs)
 
     def _get_storage(self, uri):
         return self.storages[urlparse(uri).scheme](uri)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 5b8dc4eaa..57b9d843c 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -126,6 +126,7 @@ FEED_URI = None
 FEED_URI_PARAMS = None  # a function to extend uri arguments
 FEED_FORMAT = 'jsonlines'
 FEED_STORE_EMPTY = False
+FEED_EXPORT_FIELDS = None
 FEED_STORAGES = {}
 FEED_STORAGES_BASE = {
     '': 'scrapy.contrib.feedexport.FileFeedStorage',
diff --git a/tests/mockserver.py b/tests/mockserver.py
index d2ba92485..3e4f8c0e5 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -191,6 +191,7 @@ class MockServer():
         self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'],
                           stdout=PIPE, env=get_testenv())
         self.proc.stdout.readline()
+        return self
 
     def __exit__(self, exc_type, exc_value, traceback):
         self.proc.kill()
diff --git a/tests/test_contrib_feedexport.py b/tests/test_contrib_feedexport.py
index 77eb443d5..e09dacf00 100644
--- a/tests/test_contrib_feedexport.py
+++ b/tests/test_contrib_feedexport.py
@@ -1,16 +1,27 @@
+from __future__ import absolute_import
 import os
+import csv
 from io import BytesIO
+import tempfile
+import shutil
 from six.moves.urllib.parse import urlparse
 
 from zope.interface.verify import verifyObject
 from twisted.trial import unittest
 from twisted.internet import defer
+from scrapy.crawler import CrawlerRunner
+from scrapy.settings import Settings
+from tests.mockserver import MockServer
 from w3lib.url import path_to_file_uri
 
-from scrapy.spider import Spider
-from scrapy.contrib.feedexport import IFeedStorage, FileFeedStorage, FTPFeedStorage, S3FeedStorage, StdoutFeedStorage
+import scrapy
+from scrapy.contrib.feedexport import (
+    IFeedStorage, FileFeedStorage, FTPFeedStorage,
+    S3FeedStorage, StdoutFeedStorage
+)
 from scrapy.utils.test import assert_aws_environ
 
+
 class FileFeedStorageTest(unittest.TestCase):
 
     def test_store_file_uri(self):
@@ -39,7 +50,7 @@ class FileFeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def _assert_stores(self, storage, path):
-        spider = Spider("default")
+        spider = scrapy.Spider("default")
         file = storage.open(spider)
         file.write(b"content")
         yield storage.store(file)
@@ -61,7 +72,7 @@ class FTPFeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def _assert_stores(self, storage, path):
-        spider = Spider("default")
+        spider = scrapy.Spider("default")
         file = storage.open(spider)
         file.write(b"content")
         yield storage.store(file)
@@ -85,7 +96,7 @@ class S3FeedStorageTest(unittest.TestCase):
         from boto import connect_s3
         storage = S3FeedStorage(uri)
         verifyObject(IFeedStorage, storage)
-        file = storage.open(Spider("default"))
+        file = storage.open(scrapy.Spider("default"))
         file.write("content")
         yield storage.store(file)
         u = urlparse(uri)
@@ -99,7 +110,125 @@ class StdoutFeedStorageTest(unittest.TestCase):
     def test_store(self):
         out = BytesIO()
         storage = StdoutFeedStorage('stdout:', _stdout=out)
-        file = storage.open(Spider("default"))
+        file = storage.open(scrapy.Spider("default"))
         file.write(b"content")
         yield storage.store(file)
         self.assertEqual(out.getvalue(), b"content")
+
+
+class FeedExportTest(unittest.TestCase):
+
+    class MyItem(scrapy.Item):
+        foo = scrapy.Field()
+        egg = scrapy.Field()
+        baz = scrapy.Field()
+
+
+    @defer.inlineCallbacks
+    def run_and_export(self, spider_cls, settings=None):
+        """ Run spider with specified settings; return exported data. """
+        tmpdir = tempfile.mkdtemp()
+        res_name = tmpdir + '/res'
+        defaults = {
+            'FEED_URI': 'file://' + res_name,
+            'FEED_FORMAT': 'csv',
+        }
+        defaults.update(settings or {})
+        try:
+            with MockServer() as s:
+                runner = CrawlerRunner(Settings(defaults))
+                yield runner.crawl(spider_cls)
+
+            with open(res_name, 'rb') as f:
+                defer.returnValue(f.read())
+
+        finally:
+            shutil.rmtree(tmpdir)
+
+    @defer.inlineCallbacks
+    def exported_data(self, items, settings):
+        """
+        Return exported data which a spider yielding ``items`` would return.
+        """
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+            start_urls = ['http://localhost:8998/']
+
+            def parse(self, response):
+                for item in items:
+                    yield item
+
+        data = yield self.run_and_export(TestSpider, settings)
+        defer.returnValue(data)
+
+    @defer.inlineCallbacks
+    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+        settings = settings or {}
+        settings.update({'FEED_FORMAT': 'csv'})
+        data = yield self.exported_data(items, settings)
+
+        reader = csv.DictReader(data.splitlines())
+        got_rows = list(reader)
+        if ordered:
+            self.assertEqual(reader.fieldnames, header)
+        else:
+            self.assertEqual(set(reader.fieldnames), set(header))
+
+        self.assertEqual(rows, got_rows)
+
+    @defer.inlineCallbacks
+    def test_export_csv_items(self):
+        # feed exporters use field names from Item
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+            self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+        ]
+        rows = [
+            {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
+            {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
+        ]
+        header = self.MyItem.fields.keys()
+        yield self.assertExportedCsv(items, header, rows, ordered=False)
+
+    @defer.inlineCallbacks
+    def test_export_csv_dicts(self):
+        # When dicts are used, only keys from the first row are used as
+        # a header.
+        items = [
+            {'foo': 'bar', 'egg': 'spam'},
+            {'foo': 'bar', 'egg': 'spam', 'baz': 'quux'},
+        ]
+        rows = [
+            {'egg': 'spam', 'foo': 'bar'},
+            {'egg': 'spam', 'foo': 'bar'}
+        ]
+        yield self.assertExportedCsv(items, ['egg', 'foo'], rows, ordered=False)
+
+    @defer.inlineCallbacks
+    def test_export_csv_feed_export_fields(self):
+        # FEED_EXPORT_FIELDS option allows to order export fields
+        # and to select a subset of fields to export, both for Items and dicts.
+
+        for item_cls in [self.MyItem, dict]:
+            items = [
+                item_cls({'foo': 'bar1', 'egg': 'spam1'}),
+                item_cls({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+            ]
+
+            # export all columns
+            settings = {'FEED_EXPORT_FIELDS': 'foo,baz,egg'}
+            rows = [
+                {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
+                {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
+            ]
+            yield self.assertExportedCsv(items, ['foo', 'baz', 'egg'], rows,
+                                         settings=settings, ordered=True)
+
+            # export a subset of columns
+            settings = {'FEED_EXPORT_FIELDS': 'egg,baz'}
+            rows = [
+                {'egg': 'spam1', 'baz': ''},
+                {'egg': 'spam2', 'baz': 'quux2'}
+            ]
+            yield self.assertExportedCsv(items, ['egg', 'baz'], rows,
+                                         settings=settings, ordered=True)

From 647eeaea61772d80667b7715dc84cd5aff86dbfd Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Apr 2015 03:23:30 +0500
Subject: [PATCH 0194/2974] TST add a test for GH-1050.

---
 tests/test_contrib_feedexport.py | 36 ++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/tests/test_contrib_feedexport.py b/tests/test_contrib_feedexport.py
index e09dacf00..dbcfd96b1 100644
--- a/tests/test_contrib_feedexport.py
+++ b/tests/test_contrib_feedexport.py
@@ -190,6 +190,42 @@ class FeedExportTest(unittest.TestCase):
         header = self.MyItem.fields.keys()
         yield self.assertExportedCsv(items, header, rows, ordered=False)
 
+    @defer.inlineCallbacks
+    def test_export_csv_multiple_item_classes(self):
+
+        class MyItem2(scrapy.Item):
+            foo = scrapy.Field()
+            hello = scrapy.Field()
+
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+            MyItem2({'hello': 'world2', 'foo': 'bar2'}),
+            self.MyItem({'foo': 'bar3', 'egg': 'spam3', 'baz': 'quux3'}),
+            {'hello': 'world4', 'egg': 'spam4'},
+        ]
+
+        # by default, Scrapy uses fields of the first Item
+        header = self.MyItem.fields.keys()
+        rows = [
+            {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
+            {'egg': '',      'foo': 'bar2', 'baz': ''},
+            {'egg': 'spam3', 'foo': 'bar3', 'baz': 'quux3'},
+            {'egg': 'spam4', 'foo': '',     'baz': ''},
+        ]
+        yield self.assertExportedCsv(items, header, rows, ordered=False)
+
+        # but it is possible to override fields using FEED_EXPORT_FIELDS
+        header = ["foo", "baz", "hello"]
+        settings = {'FEED_EXPORT_FIELDS': header}
+        rows = [
+            {'foo': 'bar1', 'baz': '',      'hello': ''},
+            {'foo': 'bar2', 'baz': '',      'hello': 'world2'},
+            {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
+            {'foo': '',     'baz': '',      'hello': 'world4'},
+        ]
+        yield self.assertExportedCsv(items, header, rows,
+                                     settings=settings, ordered=True)
+
     @defer.inlineCallbacks
     def test_export_csv_dicts(self):
         # When dicts are used, only keys from the first row are used as

From abeb8e3a0118a3defd7cefd29c9ead930ccc5e28 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Apr 2015 03:27:11 +0500
Subject: [PATCH 0195/2974] TST skip a test in Python 3 because it imports some
 parts of Twisted which are not ported yet

---
 tests/test_contrib_feedexport.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test_contrib_feedexport.py b/tests/test_contrib_feedexport.py
index dbcfd96b1..e6fd38ee3 100644
--- a/tests/test_contrib_feedexport.py
+++ b/tests/test_contrib_feedexport.py
@@ -4,6 +4,7 @@ import csv
 from io import BytesIO
 import tempfile
 import shutil
+import six
 from six.moves.urllib.parse import urlparse
 
 from zope.interface.verify import verifyObject
@@ -118,6 +119,8 @@ class StdoutFeedStorageTest(unittest.TestCase):
 
 class FeedExportTest(unittest.TestCase):
 
+    skip = not six.PY2
+
     class MyItem(scrapy.Item):
         foo = scrapy.Field()
         egg = scrapy.Field()

From 7dcd7f3ca7ce8da2f3cc45ddc780274edb121469 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Apr 2015 03:43:30 +0500
Subject: [PATCH 0196/2974] DOC make local docs use the same theme as
 readthedocs.org

sphinx_rtd_theme is builtin in recent Sphinx versions.
---
 docs/conf.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index fdd3025db..2c1935003 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -42,7 +42,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Scrapy'
-copyright = u'2008-2014, Scrapy developers'
+copyright = u'2008-2015, Scrapy developers'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -95,10 +95,23 @@ pygments_style = 'sphinx'
 # Options for HTML output
 # -----------------------
 
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+
 # The style sheet to use for HTML and HTML Help pages. A file of that name
 # must exist either in Sphinx' static/ path, or in one of the custom paths
 # given in html_static_path.
-html_style = 'scrapydoc.css'
+# html_style = 'scrapydoc.css'
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".

From dd84f4bd9a1402505c40477749e3c820409665c2 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Apr 2015 03:44:49 +0500
Subject: [PATCH 0197/2974] DOC remove css file which isno longer needed

---
 docs/_static/scrapydoc.css | 657 -------------------------------------
 1 file changed, 657 deletions(-)
 delete mode 100644 docs/_static/scrapydoc.css

diff --git a/docs/_static/scrapydoc.css b/docs/_static/scrapydoc.css
deleted file mode 100644
index 3e58a5e70..000000000
--- a/docs/_static/scrapydoc.css
+++ /dev/null
@@ -1,657 +0,0 @@
-/**
- * Sphinx Doc Design
- */
-
-body {
-    font-family: sans-serif;
-    font-size: 100%;
-    background-color: #3d1e11;
-    color: #000;
-    margin: 0;
-    padding: 0;
-}
-
-/* :::: LAYOUT :::: */
-
-div.document {
-    background-color: #69341e;
-}
-
-div.documentwrapper {
-    float: left;
-    width: 100%;
-}
-
-div.bodywrapper {
-    margin: 0 0 0 230px;
-}
-
-div.body {
-    background-color: white;
-    padding: 0 20px 30px 20px;
-}
-
-div.sphinxsidebarwrapper {
-    padding: 10px 5px 0 10px;
-}
-
-div.sphinxsidebar {
-    float: left;
-    width: 230px;
-    margin-left: -100%;
-    font-size: 90%;
-}
-
-div.clearer {
-    clear: both;
-}
-
-div.footer {
-    color: #fff;
-    width: 100%;
-    padding: 9px 0 9px 0;
-    text-align: center;
-    font-size: 75%;
-}
-
-div.footer a {
-    color: #fff;
-    text-decoration: underline;
-}
-
-div.related {
-    background-color: #5b1616;
-    color: #fff;
-    width: 100%;
-    line-height: 30px;
-    font-size: 90%;
-}
-
-div.related h3 {
-    display: none;
-}
-
-div.related ul {
-    margin: 0;
-    padding: 0 0 0 10px;
-    list-style: none;
-}
-
-div.related li {
-    display: inline;
-}
-
-div.related li.right {
-    float: right;
-    margin-right: 5px;
-}
-
-div.related a {
-    color: white;
-}
-
-/* ::: TOC :::: */
-div.sphinxsidebar h3 {
-    font-family: 'Trebuchet MS', sans-serif;
-    color: white;
-    font-size: 1.4em;
-    font-weight: normal;
-    margin: 0;
-    padding: 0;
-}
-
-div.sphinxsidebar h3 a {
-    color: white;
-}
-
-div.sphinxsidebar h4 {
-    font-family: 'Trebuchet MS', sans-serif;
-    color: white;
-    font-size: 1.3em;
-    font-weight: normal;
-    margin: 5px 0 0 0;
-    padding: 0;
-}
-
-div.sphinxsidebar p {
-    color: white;
-}
-
-div.sphinxsidebar p.topless {
-    margin: 5px 10px 10px 10px;
-}
-
-div.sphinxsidebar ul {
-    margin: 10px;
-    padding: 0;
-    list-style: none;
-    color: white;
-}
-
-div.sphinxsidebar ul ul,
-div.sphinxsidebar ul.want-points {
-    margin-left: 20px;
-    list-style: square;
-}
-
-div.sphinxsidebar ul ul {
-    margin-top: 0;
-    margin-bottom: 0;
-}
-
-div.sphinxsidebar a {
-    color: #ffca9b;
-}
-
-div.sphinxsidebar form {
-    margin-top: 10px;
-}
-
-div.sphinxsidebar input {
-    border: 1px solid #ffca9b;
-    font-family: sans-serif;
-    font-size: 1em;
-}
-
-/* :::: MODULE CLOUD :::: */
-div.modulecloud {
-    margin: -5px 10px 5px 10px;
-    padding: 10px;
-    line-height: 160%;
-    border: 1px solid #cbe7e5;
-    background-color: #f2fbfd;
-}
-
-div.modulecloud a {
-    padding: 0 5px 0 5px;
-}
-
-/* :::: SEARCH :::: */
-ul.search {
-    margin: 10px 0 0 20px;
-    padding: 0;
-}
-
-ul.search li {
-    padding: 5px 0 5px 20px;
-    background-image: url(file.png);
-    background-repeat: no-repeat;
-    background-position: 0 7px;
-}
-
-ul.search li a {
-    font-weight: bold;
-}
-
-ul.search li div.context {
-    color: #888;
-    margin: 2px 0 0 30px;
-    text-align: left;
-}
-
-ul.keywordmatches li.goodmatch a {
-    font-weight: bold;
-}
-
-/* :::: COMMON FORM STYLES :::: */
-
-div.actions {
-    padding: 5px 10px 5px 10px;
-    border-top: 1px solid #cbe7e5;
-    border-bottom: 1px solid #cbe7e5;
-    background-color: #e0f6f4;
-}
-
-form dl {
-    color: #333;
-}
-
-form dt {
-    clear: both;
-    float: left;
-    min-width: 110px;
-    margin-right: 10px;
-    padding-top: 2px;
-}
-
-input#homepage {
-    display: none;
-}
-
-div.error {
-    margin: 5px 20px 0 0;
-    padding: 5px;
-    border: 1px solid #d00;
-    font-weight: bold;
-}
-
-/* :::: INDEX PAGE :::: */
-
-table.contentstable {
-    width: 90%;
-}
-
-table.contentstable p.biglink {
-    line-height: 150%;
-}
-
-a.biglink {
-    font-size: 1.3em;
-}
-
-span.linkdescr {
-    font-style: italic;
-    padding-top: 5px;
-    font-size: 90%;
-}
-
-/* :::: INDEX STYLES :::: */
-
-table.indextable td {
-    text-align: left;
-    vertical-align: top;
-}
-
-table.indextable dl, table.indextable dd {
-    margin-top: 0;
-    margin-bottom: 0;
-}
-
-table.indextable tr.pcap {
-    height: 10px;
-}
-
-table.indextable tr.cap {
-    margin-top: 10px;
-    background-color: #f2f2f2;
-}
-
-img.toggler {
-    margin-right: 3px;
-    margin-top: 3px;
-    cursor: pointer;
-}
-
-form.pfform {
-    margin: 10px 0 20px 0;
-}
-
-/* :::: GLOBAL STYLES :::: */
-
-.docwarning {
-    background-color: #ffe4e4;
-    padding: 10px;
-    margin: 0 -20px 0 -20px;
-    border-bottom: 1px solid #f66;
-}
-
-p.subhead {
-    font-weight: bold;
-    margin-top: 20px;
-}
-
-a {
-    color: #6e0909;
-    text-decoration: none;
-}
-
-a:hover {
-    text-decoration: underline;
-}
-
-div.body h1,
-div.body h2,
-div.body h3,
-div.body h4,
-div.body h5,
-div.body h6 {
-    font-family: 'Trebuchet MS', sans-serif;
-    background-color: #f2f2f2;
-    font-weight: normal;
-    color: #331F0A;
-    border-bottom: 1px solid #ccc;
-    margin: 20px -20px 10px -20px;
-    padding: 3px 0 3px 10px;
-}
-
-div.body h1 { margin-top: 0; font-size: 200%; }
-div.body h2 { font-size: 160%; }
-div.body h3 { font-size: 140%; }
-div.body h4 { font-size: 120%; }
-div.body h5 { font-size: 110%; }
-div.body h6 { font-size: 100%; }
-
-a.headerlink {
-    color: #c60f0f;
-    font-size: 0.8em;
-    padding: 0 4px 0 4px;
-    text-decoration: none;
-    visibility: hidden;
-}
-
-h1:hover > a.headerlink,
-h2:hover > a.headerlink,
-h3:hover > a.headerlink,
-h4:hover > a.headerlink,
-h5:hover > a.headerlink,
-h6:hover > a.headerlink,
-dt:hover > a.headerlink {
-    visibility: visible;
-}
-
-a.headerlink:hover {
-    background-color: #c60f0f;
-    color: white;
-}
-
-div.body p, div.body dd, div.body li {
-    text-align: justify;
-    line-height: 130%;
-}
-
-div.body p.caption {
-    text-align: inherit;
-}
-
-div.body td {
-    text-align: left;
-}
-
-ul.fakelist {
-    list-style: none;
-    margin: 10px 0 10px 20px;
-    padding: 0;
-}
-
-.field-list ul {
-    padding-left: 1em;
-}
-
-.first {
-    margin-top: 0 !important;
-}
-
-/* "Footnotes" heading */
-p.rubric {
-    margin-top: 30px;
-    font-weight: bold;
-}
-
-/* Sidebars */
-
-div.sidebar {
-    margin: 0 0 0.5em 1em;
-    border: 1px solid #ddb;
-    padding: 7px 7px 0 7px;
-    background-color: #ffe;
-    width: 40%;
-    float: right;
-}
-
-p.sidebar-title {
-    font-weight: bold;
-}
-
-/* "Topics" */
-
-div.topic {
-    background-color: #eee;
-    border: 1px solid #ccc;
-    padding: 7px 7px 0 7px;
-    margin: 10px 0 10px 0;
-}
-
-p.topic-title {
-    font-size: 1.1em;
-    font-weight: bold;
-    margin-top: 10px;
-}
-
-/* Admonitions */
-
-div.admonition {
-    margin-top: 10px;
-    margin-bottom: 10px;
-    padding: 7px;
-}
-
-div.admonition dt {
-    font-weight: bold;
-}
-
-div.admonition dl {
-    margin-bottom: 0;
-}
-
-div.admonition p.admonition-title + p {
-    display: inline;
-}
-
-div.seealso {
-    background-color: #ffc;
-    border: 1px solid #ff6;
-}
-
-div.warning {
-    background-color: #ffe4e4;
-    border: 1px solid #f66;
-}
-
-div.note {
-    background-color: #eee;
-    border: 1px solid #ccc;
-}
-
-p.admonition-title {
-    margin: 0px 10px 5px 0px;
-    font-weight: bold;
-    display: inline;
-}
-
-p.admonition-title:after {
-    content: ":";
-}
-
-div.body p.centered {
-    text-align: center;
-    margin-top: 25px;
-}
-
-table.docutils {
-    border: 0;
-}
-
-table.docutils td, table.docutils th {
-    padding: 1px 8px 1px 0;
-    border-top: 0;
-    border-left: 0;
-    border-right: 0;
-    border-bottom: 1px solid #aaa;
-}
-
-table.field-list td, table.field-list th {
-    border: 0 !important;
-}
-
-table.footnote td, table.footnote th {
-    border: 0 !important;
-}
-
-.field-list ul {
-    margin: 0;
-    padding-left: 1em;
-}
-
-.field-list p {
-    margin: 0;
-}
-
-dl {
-    margin-bottom: 15px;
-    clear: both;
-}
-
-dd p {
-    margin-top: 0px;
-}
-
-dd ul, dd table {
-    margin-bottom: 10px;
-}
-
-dd {
-    margin-top: 3px;
-    margin-bottom: 10px;
-    margin-left: 30px;
-}
-
-.refcount {
-    color: #060;
-}
-
-dt:target,
-.highlight {
-    background-color: #fbe54e;
-}
-
-dl.glossary dt {
-    font-weight: bold;
-    font-size: 1.1em;
-}
-
-th {
-    text-align: left;
-    padding-right: 5px;
-}
-
-pre {
-    padding: 5px;
-    background-color: #efc;
-    color: #333;
-    border: 1px solid #ac9;
-    border-left: none;
-    border-right: none;
-    overflow: auto;
-}
-
-td.linenos pre {
-    padding: 5px 0px;
-    border: 0;
-    background-color: transparent;
-    color: #aaa;
-}
-
-table.highlighttable {
-    margin-left: 0.5em;
-}
-
-table.highlighttable td {
-    padding: 0 0.5em 0 0.5em;
-}
-
-tt {
-    background-color: #ecf0f3;
-    padding: 0 1px 0 1px;
-    font-size: 0.95em;
-}
-
-tt.descname {
-    background-color: transparent;
-    font-weight: bold;
-    font-size: 1.2em;
-}
-
-tt.descclassname {
-    background-color: transparent;
-}
-
-tt.xref, a tt {
-    background-color: transparent;
-    font-weight: bold;
-}
-
-.footnote:target  { background-color: #ffa }
-
-h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt {
-    background-color: transparent;
-}
-
-.optional {
-    font-size: 1.3em;
-}
-
-.versionmodified {
-    font-style: italic;
-}
-
-form.comment {
-    margin: 0;
-    padding: 10px 30px 10px 30px;
-    background-color: #eee;
-}
-
-form.comment h3 {
-    background-color: #326591;
-    color: white;
-    margin: -10px -30px 10px -30px;
-    padding: 5px;
-    font-size: 1.4em;
-}
-
-form.comment input,
-form.comment textarea {
-    border: 1px solid #ccc;
-    padding: 2px;
-    font-family: sans-serif;
-    font-size: 100%;
-}
-
-form.comment input[type="text"] {
-    width: 240px;
-}
-
-form.comment textarea {
-    width: 100%;
-    height: 200px;
-    margin-bottom: 10px;
-}
-
-.system-message {
-    background-color: #fda;
-    padding: 5px;
-    border: 3px solid red;
-}
-
-img.math {
-    vertical-align: middle;
-}
-
-div.math p {
-    text-align: center;
-}
-
-span.eqno {
-    float: right;
-}
-
-img.logo {
-    border: 0;
-}
-
-/* :::: PRINT :::: */
-@media print {
-    div.document,
-    div.documentwrapper,
-    div.bodywrapper {
-        margin: 0;
-        width : 100%;
-    }
-
-    div.sphinxsidebar,
-    div.related,
-    div.footer,
-    div#comments div.new-comment-box,
-    #top-link {
-        display: none;
-    }
-}

From 378b6efc0e0f48f9f4eac349c5e9d2be6b3cbe0f Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Apr 2015 04:06:36 +0500
Subject: [PATCH 0198/2974] PY3 disable scrapy.telnet if twisted.conch is not
 available

---
 scrapy/telnet.py      | 10 ++++++++--
 tests/py3-ignores.txt |  1 -
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/scrapy/telnet.py b/scrapy/telnet.py
index d2617f69d..7b73a42ac 100644
--- a/scrapy/telnet.py
+++ b/scrapy/telnet.py
@@ -6,9 +6,13 @@ See documentation in docs/topics/telnetconsole.rst
 
 import pprint
 
-from twisted.conch import manhole, telnet
-from twisted.conch.insults import insults
 from twisted.internet import protocol
+try:
+    from twisted.conch import manhole, telnet
+    from twisted.conch.insults import insults
+    TWISTED_CONCH_AVAILABLE = True
+except ImportError:
+    TWISTED_CONCH_AVAILABLE = False
 
 from scrapy.exceptions import NotConfigured
 from scrapy import log, signals
@@ -32,6 +36,8 @@ class TelnetConsole(protocol.ServerFactory):
     def __init__(self, crawler):
         if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'):
             raise NotConfigured
+        if not TWISTED_CONCH_AVAILABLE:
+            raise NotConfigured
         self.crawler = crawler
         self.noisy = False
         self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index f3c667cd0..70cf22c79 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -97,5 +97,4 @@ scrapy/contrib/statsmailer.py
 scrapy/contrib/memusage.py
 scrapy/commands/deploy.py
 scrapy/commands/bench.py
-scrapy/telnet.py
 scrapy/mail.py

From 0b764876291cbba8581ce283c97f423b796f899a Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Apr 2015 04:16:26 +0500
Subject: [PATCH 0199/2974] PY3 fix Python 3 syntax errors in ajaxcrawl.py

Tests for _has_ajaxcrawlable_meta passin Python 3.
---
 scrapy/contrib/downloadermiddleware/ajaxcrawl.py | 6 ++++--
 tests/py3-ignores.txt                            | 1 -
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
index fcbfdb1e7..6c0371691 100644
--- a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
+++ b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
@@ -1,11 +1,13 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 import re
+import six
+from w3lib import html
 from scrapy import log
 from scrapy.exceptions import NotConfigured
 from scrapy.http import HtmlResponse
 from scrapy.utils.response import _noscript_re, _script_re
-from w3lib import html
+
 
 class AjaxCrawlMiddleware(object):
     """
@@ -61,7 +63,7 @@ class AjaxCrawlMiddleware(object):
 
 
 # XXX: move it to w3lib?
-_ajax_crawlable_re = re.compile(ur'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>')
+_ajax_crawlable_re = re.compile(six.u(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'))
 def _has_ajaxcrawlable_meta(text):
     """
     >>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment"  content="!"/></head><body></body></html>')
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index f3c667cd0..b584a16c4 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -92,7 +92,6 @@ scrapy/contrib/linkextractors/htmlparser.py
 scrapy/contrib/downloadermiddleware/retry.py
 scrapy/contrib/downloadermiddleware/httpproxy.py
 scrapy/contrib/downloadermiddleware/cookies.py
-scrapy/contrib/downloadermiddleware/ajaxcrawl.py
 scrapy/contrib/statsmailer.py
 scrapy/contrib/memusage.py
 scrapy/commands/deploy.py

From 020a32a3d9639bf948b8f84c8904764f99e0092b Mon Sep 17 00:00:00 2001
From: mrpandav <pandav.dharmesh@gmail.com>
Date: Wed, 15 Apr 2015 11:23:25 +0530
Subject: [PATCH 0200/2974] Adding tests for pull request #1137 - addition of
 new shortcut for html form election by formid attribute

---
 tests/test_http_request.py | 54 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index c81eebfa6..89102016b 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -532,6 +532,60 @@ class FormRequestTest(RequestTest):
         self.assertRaises(IndexError, self.request_class.from_response, \
                           response, formname="form3", formnumber=2)
 
+    def test_from_response_formid_exists(self):
+        response = _buildresponse(
+            """<form action="post.php" method="POST">
+            <input type="hidden" name="one" value="1">
+            <input type="hidden" name="two" value="2">
+            </form>
+            <form id="form2" action="post.php" method="POST">
+            <input type="hidden" name="three" value="3">
+            <input type="hidden" name="four" value="4">
+            </form>""")
+        r1 = self.request_class.from_response(response, formid="form2")
+        self.assertEqual(r1.method, 'POST')
+        fs = _qs(r1)
+        self.assertEqual(fs, {'four': ['4'], 'three': ['3']})
+
+    def test_from_response_formid_notexists_fallback_formname(self):
+        response = _buildresponse(
+            """<form action="post.php" method="POST">
+            <input type="hidden" name="one" value="1">
+            <input type="hidden" name="two" value="2">
+            </form>
+            <form id="form2" name="form2" action="post.php" method="POST">
+            <input type="hidden" name="three" value="3">
+            <input type="hidden" name="four" value="4">
+            </form>""")
+        r1 = self.request_class.from_response(response, formid="form3", formname="form2")
+        self.assertEqual(r1.method, 'POST')
+        fs = _qs(r1)
+        self.assertEqual(fs, {'four': ['4'], 'three': ['3']})
+
+    def test_from_response_formid_notexist(self):
+        response = _buildresponse(
+            """<form id="form1" action="post.php" method="POST">
+            <input type="hidden" name="one" value="1">
+            </form>
+            <form id="form2" action="post.php" method="POST">
+            <input type="hidden" name="two" value="2">
+            </form>""")
+        r1 = self.request_class.from_response(response, formid="form3")
+        self.assertEqual(r1.method, 'POST')
+        fs = _qs(r1)
+        self.assertEqual(fs, {'one': ['1']})
+
+    def test_from_response_formid_errors_formnumber(self):
+        response = _buildresponse(
+            """<form id="form1" action="post.php" method="POST">
+            <input type="hidden" name="one" value="1">
+            </form>
+            <form id="form2" name="form2" action="post.php" method="POST">
+            <input type="hidden" name="two" value="2">
+            </form>""")
+        self.assertRaises(IndexError, self.request_class.from_response, \
+                          response, formid="form3", formnumber=2)
+
     def test_from_response_select(self):
         res = _buildresponse(
             '''<form>

From a11bd3e89dd151f21b42757caba2c45c1b4f9f8a Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Apr 2015 11:50:57 +0500
Subject: [PATCH 0201/2974] DOC update docs readme file - sphinx >= 1.3 is now
 required.

---
 docs/README | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/docs/README b/docs/README
index 7fd549374..26cf25371 100644
--- a/docs/README
+++ b/docs/README
@@ -8,16 +8,12 @@ This file provides a quick guide on how to compile the Scrapy documentation.
 Setup the environment
 ---------------------
 
-To compile the documentation you need the following Python libraries:
+To compile the documentation you need Sphinx Python library. To install it
+and all its dependencies run
 
- * Sphinx
- * docutils
- * jinja
+::
 
-If you have setuptools available the following command will install all of them
-(since Sphinx requires both docutils and jinja)::
-
-    easy_install Sphinx
+    pip install 'Sphinx >= 1.3'
 
 
 Compile the documentation

From 954c8fcecbd0c5c43e371f58585a2ce18f0a6212 Mon Sep 17 00:00:00 2001
From: mrpandav <pandav.dharmesh@gmail.com>
Date: Wed, 15 Apr 2015 16:49:09 +0530
Subject: [PATCH 0202/2974] changes
 `test_from_response_formname_notexists_fallback_formid` unit test for pull
 request #1137 - addition of new shortcut for html form election by formid
 attribute

---
 tests/test_http_request.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 89102016b..01a2de118 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -547,7 +547,7 @@ class FormRequestTest(RequestTest):
         fs = _qs(r1)
         self.assertEqual(fs, {'four': ['4'], 'three': ['3']})
 
-    def test_from_response_formid_notexists_fallback_formname(self):
+    def test_from_response_formname_notexists_fallback_formid(self):
         response = _buildresponse(
             """<form action="post.php" method="POST">
             <input type="hidden" name="one" value="1">
@@ -557,7 +557,7 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="three" value="3">
             <input type="hidden" name="four" value="4">
             </form>""")
-        r1 = self.request_class.from_response(response, formid="form3", formname="form2")
+        r1 = self.request_class.from_response(response, formname="form3", formid="form2")
         self.assertEqual(r1.method, 'POST')
         fs = _qs(r1)
         self.assertEqual(fs, {'four': ['4'], 'three': ['3']})

From cb5280ab0da22a6e4ff5a78b693996b71e37214c Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Apr 2015 19:25:22 +0500
Subject: [PATCH 0203/2974] DOC tweak deployment docs

---
 docs/topics/deploy.rst | 65 +++++++++++++++++-------------------------
 1 file changed, 26 insertions(+), 39 deletions(-)

diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst
index e2e796cf7..b7394d5ce 100644
--- a/docs/topics/deploy.rst
+++ b/docs/topics/deploy.rst
@@ -11,57 +11,44 @@ when you need to execute long-running spiders or move spiders to run in
 production continously. This is where the solutions for deploying Scrapy
 spiders come in.
 
-The most popular choices, for deploying Scrapy spiders, are:
+Popular choices for deploying Scrapy spiders are:
 
-* :ref:`Scrapy Cloud <deploy-scrapy-cloud>` (cloud-based, easier to setup)
-* :ref:`Scrapyd <deploy-scrapyd>` (open source, harder to setup)
-
-.. _deploy-scrapy-cloud:
-
-Deploying to Scrapy Cloud
-=========================
-
-`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_, the company
-behind Scrapy.
-
-Advantages:
-
-- easy to setup (no need to setup or manage servers)
-- well-designed UI to manage spiders and review scraped items, logs and stats
-- cheap pricing (cheaper than renting a server, for small workloads)
-
-Disadvantages:
-
-- it's not open source
-
-To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool.
-Please refer to the `Scrapy Cloud documentation`_ for more information.
-
-The configuration is read from the ``scrapy.cfg`` file just like
-``scrapyd-deploy``.
+* :ref:`Scrapyd <deploy-scrapyd>` (open source)
+* :ref:`Scrapy Cloud <deploy-scrapy-cloud>` (cloud-based)
 
 .. _deploy-scrapyd:
 
 Deploying to a Scrapyd Server
 =============================
 
-`Scrapyd`_ is an open source application to run Scrapy spiders. It is
-maintained by some of the Scrapy developers.
-
-Advantages:
-
-- it's open source, so it can be installed and run anywhere
-
-Disadvantages:
-
-- simple UI (no analytics, graphs or rich log/items browsing)
-- requires setting up servers, installing and configuring scrapyd on them. An
-  APT repo with Ubuntu packages is provided by the Scrapyd team
+`Scrapyd`_ is an open source application to run Scrapy spiders. It provides
+a server with HTTP API, capable of running and monitoring Scrapy spiders.
 
 To deploy spiders to Scrapyd, you can use the scrapyd-deploy tool provided by
 the `scrapyd-client`_ package. Please refer to the `scrapyd-deploy
 documentation`_ for more information.
 
+Scrapyd is maintained by some of the Scrapy developers.
+
+.. _deploy-scrapy-cloud:
+
+Deploying to Scrapy Cloud
+=========================
+
+`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_,
+the company behind Scrapy.
+
+Scrapy Cloud removes the need to setup and monitor servers
+and provides a nice UI to manage spiders and review scraped items,
+logs and stats.
+
+To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool.
+Please refer to the `Scrapy Cloud documentation`_ for more information.
+
+Scrapy Cloud is compatible with Scrapyd and one can switch between
+them as needed - the configuration is read from the ``scrapy.cfg`` file
+just like ``scrapyd-deploy``.
+
 .. _Scrapyd: https://github.com/scrapy/scrapyd
 .. _Deploying your project: https://scrapyd.readthedocs.org/en/latest/deploy.html
 .. _Scrapy Cloud: http://scrapinghub.com/scrapy-cloud/

From fd1c688ad637417db539463e69ef0d59d9c2e61e Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 15 Apr 2015 14:21:48 -0300
Subject: [PATCH 0204/2974] addressing issues from review at #1150

---
 docs/topics/images.rst | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/topics/images.rst b/docs/topics/images.rst
index ab4a98ebd..903359745 100644
--- a/docs/topics/images.rst
+++ b/docs/topics/images.rst
@@ -15,7 +15,7 @@ typically you'll either use the Files Pipeline or the Images Pipeline.
 Both pipelines implement these features:
 
 * Avoid re-downloading media that was downloaded recently
-* Specifying where to store the files (filesystem directory, Amazon S3 bucket)
+* Specifying where to store the media (filesystem directory, Amazon S3 bucket)
 
 The Images Pipeline has a few extra functions for processing images:
 
@@ -23,20 +23,11 @@ The Images Pipeline has a few extra functions for processing images:
 * Thumbnail generation
 * Check images width/height to make sure they meet a minimum constraint
 
-The pipelines also keep an internal queue of those images which are currently
-being scheduled for download, and connect those items that arrive containing
-the same image to that queue. This avoids downloading the same media more than
+The pipelines also keep an internal queue of those media URLs which are currently
+being scheduled for download, and connect those responses that arrive containing
+the same media to that queue. This avoids downloading the same media more than
 once when it's shared by several items.
 
-The Images Pipeline uses `Pillow`_ for thumbnailing and normalizing images to
-JPEG/RGB format, so you need to install this library in order to use it.
-`Python Imaging Library`_ (PIL) should also work in most cases, but it is known
-to cause troubles in some setups, so we recommend to use `Pillow`_ instead of
-`PIL <Python Imaging Library>`_.
-
-.. _Pillow: https://github.com/python-pillow/Pillow
-.. _Python Imaging Library: http://www.pythonware.com/products/pil/
-
 Using the Files Pipeline
 =========================
 
@@ -76,6 +67,15 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
 can configure some extra functions like generating thumbnails and filtering
 the images based on their size.
 
+The Images Pipeline uses `Pillow`_ for thumbnailing and normalizing images to
+JPEG/RGB format, so you need to install this library in order to use it.
+`Python Imaging Library`_ (PIL) should also work in most cases, but it is known
+to cause troubles in some setups, so we recommend to use `Pillow`_ instead of
+`PIL <Python Imaging Library>`_.
+
+.. _Pillow: https://github.com/python-pillow/Pillow
+.. _Python Imaging Library: http://www.pythonware.com/products/pil/
+
 
 Usage example
 =============

From a1f3b3c7f05c505fa9f0b907681fc97f7548cfae Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 15 Apr 2015 14:26:08 -0300
Subject: [PATCH 0205/2974] images.rst -> media-pipeline.rst

---
 docs/index.rst                                 | 4 ++--
 docs/topics/{images.rst => media-pipeline.rst} | 2 +-
 scrapy/contrib/pipeline/files.py               | 2 ++
 scrapy/contrib/pipeline/images.py              | 2 +-
 4 files changed, 6 insertions(+), 4 deletions(-)
 rename docs/topics/{images.rst => media-pipeline.rst} (99%)

diff --git a/docs/index.rst b/docs/index.rst
index f61e7f348..3b245560c 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -148,7 +148,7 @@ Solving specific problems
    topics/firefox
    topics/firebug
    topics/leaks
-   topics/images
+   topics/media-pipeline
    topics/ubuntu
    topics/scrapyd
    topics/autothrottle
@@ -180,7 +180,7 @@ Solving specific problems
 :doc:`topics/leaks`
     Learn how to find and get rid of memory leaks in your crawler.
 
-:doc:`topics/images`
+:doc:`topics/media-pipeline`
     Download files and/or images associated with your scraped items.
 
 :doc:`topics/ubuntu`
diff --git a/docs/topics/images.rst b/docs/topics/media-pipeline.rst
similarity index 99%
rename from docs/topics/images.rst
rename to docs/topics/media-pipeline.rst
index 903359745..4b88c60fd 100644
--- a/docs/topics/images.rst
+++ b/docs/topics/media-pipeline.rst
@@ -1,4 +1,4 @@
-.. _topics-images:
+.. _topics-media-pipeline:
 
 ===========================================
 Downloading and processing files and images
diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py
index 9e803aca0..608614865 100644
--- a/scrapy/contrib/pipeline/files.py
+++ b/scrapy/contrib/pipeline/files.py
@@ -1,5 +1,7 @@
 """
 Files Pipeline
+
+See documentation in topics/media-pipeline.rst
 """
 
 import hashlib
diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py
index b12995f09..3cbfed49b 100644
--- a/scrapy/contrib/pipeline/images.py
+++ b/scrapy/contrib/pipeline/images.py
@@ -1,7 +1,7 @@
 """
 Images Pipeline
 
-See documentation in topics/images.rst
+See documentation in topics/media-pipeline.rst
 """
 
 import hashlib

From 403e7c7c7059486c5a4c59f900a65378de2ba3d2 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 16 Apr 2015 20:07:53 +0500
Subject: [PATCH 0206/2974] rename scrapy.spidermanager.SpiderManager to
 scrapy.spiderloader.SpiderLoader

---
 docs/topics/api.rst                           | 20 +++----
 docs/topics/settings.rst                      | 12 ++---
 scrapy/crawler.py                             | 26 ++++++---
 scrapy/interfaces.py                          | 13 +++--
 scrapy/settings/default_settings.py           |  2 +-
 scrapy/spider.py                              |  8 +--
 scrapy/spiderloader.py                        | 53 +++++++++++++++++++
 scrapy/spidermanager.py                       | 44 ++-------------
 scrapy/utils/spider.py                        |  8 +--
 tests/py3-ignores.txt                         | 14 ++---
 tests/test_crawler.py                         |  6 +--
 .../__init__.py                               | 47 ++++++++--------
 .../test_spiders/__init__.py                  |  0
 .../test_spiders/spider0.py                   |  0
 .../test_spiders/spider1.py                   |  0
 .../test_spiders/spider2.py                   |  0
 .../test_spiders/spider3.py                   |  0
 17 files changed, 145 insertions(+), 108 deletions(-)
 create mode 100644 scrapy/spiderloader.py
 rename tests/{test_spidermanager => test_spiderloader}/__init__.py (51%)
 rename tests/{test_spidermanager => test_spiderloader}/test_spiders/__init__.py (100%)
 rename tests/{test_spidermanager => test_spiderloader}/test_spiders/spider0.py (100%)
 rename tests/{test_spidermanager => test_spiderloader}/test_spiders/spider1.py (100%)
 rename tests/{test_spidermanager => test_spiderloader}/test_spiders/spider2.py (100%)
 rename tests/{test_spidermanager => test_spiderloader}/test_spiders/spider3.py (100%)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 2055682dc..48347120b 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -344,22 +344,22 @@ Settings API
 
        Alias for a :meth:`~freeze` call in the object returned by :meth:`copy`
 
-.. _topics-api-spidermanager:
+.. _topics-api-spiderloader:
 
-SpiderManager API
-=================
+SpiderLoader API
+================
 
-.. module:: scrapy.spidermanager
-   :synopsis: The spider manager
+.. module:: scrapy.loader
+   :synopsis: The spider loader
 
-.. class:: SpiderManager
+.. class:: SpiderLoader
 
     This class is in charge of retrieving and handling the spider classes
     defined across the project.
 
-    Custom spider managers can be employed by specifying their path in the
-    :setting:`SPIDER_MANAGER_CLASS` project setting. They must fully implement
-    the :class:`scrapy.interfaces.ISpiderManager` interface to guarantee an
+    Custom spider loaders can be employed by specifying their path in the
+    :setting:`SPIDER_LOADER_CLASS` project setting. They must fully implement
+    the :class:`scrapy.interfaces.ISpiderLoader` interface to guarantee an
     errorless execution.
 
     .. method:: from_settings(settings)
@@ -486,7 +486,7 @@ class (which they all inherit from).
 
         Set the given value for the given key only if current value for the
         same key is lower than value. If there is no current value for the
-        given key, the value is always set. 
+        given key, the value is always set.
 
     .. method:: min_value(key, value)
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 1986b2f3e..7deb6e0c6 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -853,15 +853,15 @@ A dict containing the scrapy contracts enabled by default in Scrapy. You should
 never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS`
 instead. For more info see :ref:`topics-contracts`.
 
-.. setting:: SPIDER_MANAGER_CLASS
+.. setting:: SPIDER_LOADER_CLASS
 
-SPIDER_MANAGER_CLASS
---------------------
+SPIDER_LOADER_CLASS
+-------------------
 
-Default: ``'scrapy.spidermanager.SpiderManager'``
+Default: ``'scrapy.spiderloader.SpiderLoader'``
 
-The class that will be used for handling spiders, which must implement the
-:ref:`topics-api-spidermanager`.
+The class that will be used for loading spiders, which must implement the
+:ref:`topics-api-spiderloader`.
 
 .. setting:: SPIDER_MIDDLEWARES
 
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 3d0576081..52956d111 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -7,7 +7,7 @@ from zope.interface.verify import verifyClass
 
 from scrapy.core.engine import ExecutionEngine
 from scrapy.resolver import CachingThreadedResolver
-from scrapy.interfaces import ISpiderManager
+from scrapy.interfaces import ISpiderLoader
 from scrapy.extension import ExtensionManager
 from scrapy.settings import Settings
 from scrapy.signalmanager import SignalManager
@@ -44,11 +44,10 @@ class Crawler(object):
         if not hasattr(self, '_spiders'):
             warnings.warn("Crawler.spiders is deprecated, use "
                           "CrawlerRunner.spiders or instantiate "
-                          "scrapy.spidermanager.SpiderManager with your "
+                          "scrapy.spiderloader.SpiderLoader with your "
                           "settings.",
                           category=ScrapyDeprecationWarning, stacklevel=2)
-            spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])
-            self._spiders = spman_cls.from_settings(self.settings)
+            self._spiders = _get_spider_loader(self.settings.frozencopy())
         return self._spiders
 
     @defer.inlineCallbacks
@@ -85,9 +84,7 @@ class CrawlerRunner(object):
         if isinstance(settings, dict):
             settings = Settings(settings)
         self.settings = settings
-        smcls = load_object(settings['SPIDER_MANAGER_CLASS'])
-        verifyClass(ISpiderManager, smcls)
-        self.spiders = smcls.from_settings(settings.frozencopy())
+        self.spiders = _get_spider_loader(settings)
         self.crawlers = set()
         self._active = set()
 
@@ -178,3 +175,18 @@ class CrawlerProcess(CrawlerRunner):
             reactor.stop()
         except RuntimeError:  # raised if already stopped or in shutdown stage
             pass
+
+
+def _get_spider_loader(settings):
+    """ Get SpiderLoader instance from settings """
+    if settings.get('SPIDER_MANAGER_CLASS'):
+        warnings.warn(
+            'SPIDER_MANAGER_CLASS option is deprecated. '
+            'Please use SPIDER_LOADER_CLASS.',
+            category=ScrapyDeprecationWarning, stacklevel=2
+        )
+    cls_path = settings.get('SPIDER_LOADER_CLASS',
+                            settings.get('SPIDER_MANAGER_CLASS'))
+    loader_cls = load_object(cls_path)
+    verifyClass(ISpiderLoader, loader_cls)
+    return loader_cls.from_settings(settings.frozencopy())
diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py
index d4596407e..eb93c6f7e 100644
--- a/scrapy/interfaces.py
+++ b/scrapy/interfaces.py
@@ -1,12 +1,12 @@
 from zope.interface import Interface
 
-class ISpiderManager(Interface):
+class ISpiderLoader(Interface):
 
     def from_settings(settings):
-        """Returns an instance of the class for the given settings"""
+        """Return an instance of the class for the given settings"""
 
     def load(spider_name):
-        """Returns the Spider class for the given spider name. If the spider
+        """Return the Spider class for the given spider name. If the spider
         name is not found, it must raise a KeyError."""
 
     def list():
@@ -14,4 +14,9 @@ class ISpiderManager(Interface):
         project"""
 
     def find_by_request(request):
-        """Returns the list of spiders names that can handle the given request"""
+        """Return the list of spiders names that can handle the given request"""
+
+
+# ISpiderManager is deprecated, don't use it!
+# An alias is kept for backwards compatibility.
+ISpiderManager = ISpiderLoader
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 22005eb5d..0a5afc7e6 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -215,7 +215,7 @@ SCHEDULER = 'scrapy.core.scheduler.Scheduler'
 SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleLifoDiskQueue'
 SCHEDULER_MEMORY_QUEUE = 'scrapy.squeue.LifoMemoryQueue'
 
-SPIDER_MANAGER_CLASS = 'scrapy.spidermanager.SpiderManager'
+SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
 
 SPIDER_MIDDLEWARES = {}
 
diff --git a/scrapy/spider.py b/scrapy/spider.py
index dada6236b..a2d091804 100644
--- a/scrapy/spider.py
+++ b/scrapy/spider.py
@@ -97,7 +97,9 @@ class ObsoleteClass(object):
     def __getattr__(self, name):
         raise AttributeError(self.message)
 
-spiders = ObsoleteClass("""
-"from scrapy.spider import spiders" no longer works - use "from scrapy.spidermanager import SpiderManager" and instantiate it with your project settings"
-""")
+spiders = ObsoleteClass(
+    '"from scrapy.spider import spiders" no longer works - use '
+    '"from scrapy.spiderloader import SpiderLoader" and instantiate '
+    'it with your project settings"'
+)
 
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
new file mode 100644
index 000000000..484fef456
--- /dev/null
+++ b/scrapy/spiderloader.py
@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+from zope.interface import implementer
+
+from scrapy.interfaces import ISpiderLoader
+from scrapy.utils.misc import walk_modules
+from scrapy.utils.spider import iter_spider_classes
+
+
+@implementer(ISpiderLoader)
+class SpiderLoader(object):
+    """
+    SpiderLoader is a class which locates and loads spiders
+    in a Scrapy project.
+    """
+    def __init__(self, settings):
+        self.spider_modules = settings.getlist('SPIDER_MODULES')
+        self._spiders = {}
+        for name in self.spider_modules:
+            for module in walk_modules(name):
+                self._load_spiders(module)
+
+    def _load_spiders(self, module):
+        for spcls in iter_spider_classes(module):
+            self._spiders[spcls.name] = spcls
+
+    @classmethod
+    def from_settings(cls, settings):
+        return cls(settings)
+
+    def load(self, spider_name):
+        """
+        Return the Spider class for the given spider name. If the spider
+        name is not found, raise a KeyError.
+        """
+        try:
+            return self._spiders[spider_name]
+        except KeyError:
+            raise KeyError("Spider not found: {}".format(spider_name))
+
+    def find_by_request(self, request):
+        """
+        Return the list of spiders names that can handle the given request.
+        """
+        return [name for name, cls in self._spiders.items()
+                if cls.handles_request(request)]
+
+    def list(self):
+        """
+        Return a list with the names of all spiders available in the project.
+        """
+        return list(self._spiders.keys())
diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py
index 93e91bcb4..220257bb1 100644
--- a/scrapy/spidermanager.py
+++ b/scrapy/spidermanager.py
@@ -1,43 +1,7 @@
 """
-SpiderManager is the class which locates and manages all website-specific
-spiders
+Backwards compatibility shim. Use scrapy.spiderloader instead.
 """
+from scrapy.spiderloader import SpiderLoader
+from scrapy.utils.deprecate import create_deprecated_class
 
-from zope.interface import implementer
-import six
-
-from scrapy.interfaces import ISpiderManager
-from scrapy.utils.misc import walk_modules
-from scrapy.utils.spider import iter_spider_classes
-
-
-@implementer(ISpiderManager)
-class SpiderManager(object):
-
-    def __init__(self, settings):
-        self.spider_modules = settings.getlist('SPIDER_MODULES')
-        self._spiders = {}
-        for name in self.spider_modules:
-            for module in walk_modules(name):
-                self._load_spiders(module)
-
-    def _load_spiders(self, module):
-        for spcls in iter_spider_classes(module):
-            self._spiders[spcls.name] = spcls
-
-    @classmethod
-    def from_settings(cls, settings):
-        return cls(settings)
-
-    def load(self, spider_name):
-        try:
-            return self._spiders[spider_name]
-        except KeyError:
-            raise KeyError("Spider not found: {}".format(spider_name))
-
-    def find_by_request(self, request):
-        return [name for name, cls in six.iteritems(self._spiders)
-            if cls.handles_request(request)]
-
-    def list(self):
-        return list(self._spiders.keys())
+SpiderManager = create_deprecated_class('SpiderManager', SpiderLoader)
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 40ebbefc3..68de76bfe 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -26,21 +26,21 @@ def iter_spider_classes(module):
            getattr(obj, 'name', None):
             yield obj
 
-def spidercls_for_request(spidermanager, request, default_spidercls=None,
+def spidercls_for_request(spiderloader, request, default_spidercls=None,
                           log_none=False, log_multiple=False):
     """Return a spider class that handles the given Request.
 
     This will look for the spiders that can handle the given request (using
-    the spider manager) and return a Spider class if (and only if) there is
+    the spider loader) and return a Spider class if (and only if) there is
     only one Spider able to handle the Request.
 
     If multiple spiders (or no spider) are found, it will return the
     default_spidercls passed. It can optionally log if multiple or no spiders
     are found.
     """
-    snames = spidermanager.find_by_request(request)
+    snames = spiderloader.find_by_request(request)
     if len(snames) == 1:
-        return spidermanager.load(snames[0])
+        return spiderloader.load(snames[0])
 
     if len(snames) > 1 and log_multiple:
         log.msg(format='More than one spider can handle: %(request)s - %(snames)s',
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index fb1daf62f..8620a433c 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -44,13 +44,13 @@ tests/test_selector_csstranslator.py
 tests/test_selector_lxmldocument.py
 tests/test_selector.py
 tests/test_settings/__init__.py
-tests/test_spidermanager/__init__.py
-tests/test_spidermanager/test_spiders/__init__.py
-tests/test_spidermanager/test_spiders/spider0.py
-tests/test_spidermanager/test_spiders/spider1.py
-tests/test_spidermanager/test_spiders/spider2.py
-tests/test_spidermanager/test_spiders/spider3.py
-tests/test_spidermanager/test_spiders/spider4.py
+tests/test_spiderloader/__init__.py
+tests/test_spiderloader/test_spiders/__init__.py
+tests/test_spiderloader/test_spiders/spider0.py
+tests/test_spiderloader/test_spiders/spider1.py
+tests/test_spiderloader/test_spiders/spider2.py
+tests/test_spiderloader/test_spiders/spider3.py
+tests/test_spiderloader/test_spiders/spider4.py
 tests/test_spidermiddleware_depth.py
 tests/test_spidermiddleware_httperror.py
 tests/test_spidermiddleware_offsite.py
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 20467833f..f0a3865cb 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -19,7 +19,7 @@ class CrawlerTestCase(unittest.TestCase):
             spiders = self.crawler.spiders
             self.assertEqual(len(w), 1)
             self.assertIn("Crawler.spiders", str(w[0].message))
-            sm_cls = load_object(self.crawler.settings['SPIDER_MANAGER_CLASS'])
+            sm_cls = load_object(self.crawler.settings['SPIDER_LOADER_CLASS'])
             self.assertIsInstance(spiders, sm_cls)
 
             self.crawler.spiders
@@ -54,7 +54,7 @@ class CrawlerTestCase(unittest.TestCase):
 
 
 
-def SpiderManagerWithWrongInterface(object):
+class SpiderLoaderWithWrongInterface(object):
 
     def unneeded_method(self):
         pass
@@ -64,7 +64,7 @@ class CrawlerRunnerTestCase(unittest.TestCase):
 
     def test_spider_manager_verify_interface(self):
         settings = Settings({
-            'SPIDER_MANAGER_CLASS': 'tests.test_crawler.SpiderManagerWithWrongInterface'
+            'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface'
         })
         with self.assertRaises(DoesNotImplement):
             CrawlerRunner(settings)
diff --git a/tests/test_spidermanager/__init__.py b/tests/test_spiderloader/__init__.py
similarity index 51%
rename from tests/test_spidermanager/__init__.py
rename to tests/test_spiderloader/__init__.py
index 94b8a1dc8..aa3aea19d 100644
--- a/tests/test_spidermanager/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -8,14 +8,15 @@ from twisted.trial import unittest
 
 # ugly hack to avoid cyclic imports of scrapy.spider when running this test
 # alone
-from scrapy.interfaces import ISpiderManager
-from scrapy.spidermanager import SpiderManager
+from scrapy.interfaces import ISpiderLoader
+from scrapy.spiderloader import SpiderLoader
 from scrapy.settings import Settings
 from scrapy.http import Request
 
 module_dir = os.path.dirname(os.path.abspath(__file__))
 
-class SpiderManagerTest(unittest.TestCase):
+
+class SpiderLoaderTest(unittest.TestCase):
 
     def setUp(self):
         orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
@@ -25,53 +26,53 @@ class SpiderManagerTest(unittest.TestCase):
         shutil.copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(self.tmpdir)
         settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
-        self.spiderman = SpiderManager.from_settings(settings)
+        self.spiderloader = SpiderLoader.from_settings(settings)
 
     def tearDown(self):
-        del self.spiderman
+        del self.spiderloader
         del sys.modules['test_spiders_xxx']
         sys.path.remove(self.tmpdir)
 
     def test_interface(self):
-        verifyObject(ISpiderManager, self.spiderman)
+        verifyObject(ISpiderLoader, self.spiderloader)
 
     def test_list(self):
-        self.assertEqual(set(self.spiderman.list()),
+        self.assertEqual(set(self.spiderloader.list()),
             set(['spider1', 'spider2', 'spider3']))
 
     def test_load(self):
-        spider1 = self.spiderman.load("spider1")
+        spider1 = self.spiderloader.load("spider1")
         self.assertEqual(spider1.__name__, 'Spider1')
 
     def test_find_by_request(self):
-        self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy1.org/test')),
+        self.assertEqual(self.spiderloader.find_by_request(Request('http://scrapy1.org/test')),
             ['spider1'])
-        self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy2.org/test')),
+        self.assertEqual(self.spiderloader.find_by_request(Request('http://scrapy2.org/test')),
             ['spider2'])
-        self.assertEqual(set(self.spiderman.find_by_request(Request('http://scrapy3.org/test'))),
+        self.assertEqual(set(self.spiderloader.find_by_request(Request('http://scrapy3.org/test'))),
             set(['spider1', 'spider2']))
-        self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy999.org/test')),
+        self.assertEqual(self.spiderloader.find_by_request(Request('http://scrapy999.org/test')),
             [])
-        self.assertEqual(self.spiderman.find_by_request(Request('http://spider3.com')),
+        self.assertEqual(self.spiderloader.find_by_request(Request('http://spider3.com')),
             [])
-        self.assertEqual(self.spiderman.find_by_request(Request('http://spider3.com/onlythis')),
+        self.assertEqual(self.spiderloader.find_by_request(Request('http://spider3.com/onlythis')),
             ['spider3'])
 
     def test_load_spider_module(self):
-        module = 'tests.test_spidermanager.test_spiders.spider1'
+        module = 'tests.test_spiderloader.test_spiders.spider1'
         settings = Settings({'SPIDER_MODULES': [module]})
-        self.spiderman = SpiderManager.from_settings(settings)
-        assert len(self.spiderman._spiders) == 1
+        self.spiderloader = SpiderLoader.from_settings(settings)
+        assert len(self.spiderloader._spiders) == 1
 
     def test_load_spider_module(self):
-        prefix = 'tests.test_spidermanager.test_spiders.'
+        prefix = 'tests.test_spiderloader.test_spiders.'
         module = ','.join(prefix + s for s in ('spider1', 'spider2'))
         settings = Settings({'SPIDER_MODULES': module})
-        self.spiderman = SpiderManager.from_settings(settings)
-        assert len(self.spiderman._spiders) == 2
+        self.spiderloader = SpiderLoader.from_settings(settings)
+        assert len(self.spiderloader._spiders) == 2
 
     def test_load_base_spider(self):
-        module = 'tests.test_spidermanager.test_spiders.spider0'
+        module = 'tests.test_spiderloader.test_spiders.spider0'
         settings = Settings({'SPIDER_MODULES': [module]})
-        self.spiderman = SpiderManager.from_settings(settings)
-        assert len(self.spiderman._spiders) == 0
+        self.spiderloader = SpiderLoader.from_settings(settings)
+        assert len(self.spiderloader._spiders) == 0
diff --git a/tests/test_spidermanager/test_spiders/__init__.py b/tests/test_spiderloader/test_spiders/__init__.py
similarity index 100%
rename from tests/test_spidermanager/test_spiders/__init__.py
rename to tests/test_spiderloader/test_spiders/__init__.py
diff --git a/tests/test_spidermanager/test_spiders/spider0.py b/tests/test_spiderloader/test_spiders/spider0.py
similarity index 100%
rename from tests/test_spidermanager/test_spiders/spider0.py
rename to tests/test_spiderloader/test_spiders/spider0.py
diff --git a/tests/test_spidermanager/test_spiders/spider1.py b/tests/test_spiderloader/test_spiders/spider1.py
similarity index 100%
rename from tests/test_spidermanager/test_spiders/spider1.py
rename to tests/test_spiderloader/test_spiders/spider1.py
diff --git a/tests/test_spidermanager/test_spiders/spider2.py b/tests/test_spiderloader/test_spiders/spider2.py
similarity index 100%
rename from tests/test_spidermanager/test_spiders/spider2.py
rename to tests/test_spiderloader/test_spiders/spider2.py
diff --git a/tests/test_spidermanager/test_spiders/spider3.py b/tests/test_spiderloader/test_spiders/spider3.py
similarity index 100%
rename from tests/test_spidermanager/test_spiders/spider3.py
rename to tests/test_spiderloader/test_spiders/spider3.py

From 271f7f54d5dab8cccb53802fad44e07afdedcbaa Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 16 Apr 2015 22:54:51 +0500
Subject: [PATCH 0207/2974] TST install service_identity package in tests to
 prevent warnings

Also, Twisted version is bumped for Python 3.x tests, just in case.

service_identity is not added to Scrapy requirements because Scrapy
supports older Twisted / PyOpenSSL versions which don't use it.
---
 tox.ini | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index 2dff74973..c000429c0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -14,6 +14,7 @@ deps =
     Pillow
     django
     leveldb
+    service_identity
     -rtests/requirements.txt
 commands =
     py.test {posargs:scrapy tests}
@@ -41,14 +42,14 @@ commands =
 [testenv:py33]
 basepython = python3.3
 deps =
-;    svn+svn://svn.twistedmatrix.com/svn/Twisted/trunk#egg=Twisted
-    Twisted >= 14.0.0
+    Twisted >= 15.1.0
     lxml>=3.2.4
     pyOpenSSL>=0.13.1
     cssselect>=0.9
     queuelib>=1.1.1
     w3lib>=1.8.0
     Pillow
+    service_identity
     # tests requirements
     pytest>=2.6.0
     pytest-twisted

From fee55657094fa8ecf871258953d461e9b1fa0e26 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 17 Apr 2015 01:25:48 +0500
Subject: [PATCH 0208/2974] don't expose deprecated crawler.spiders attribute
 in telnet console

---
 scrapy/telnet.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/telnet.py b/scrapy/telnet.py
index 7b73a42ac..d7cd601a2 100644
--- a/scrapy/telnet.py
+++ b/scrapy/telnet.py
@@ -72,7 +72,6 @@ class TelnetConsole(protocol.ServerFactory):
             'crawler': self.crawler,
             'extensions': self.crawler.extensions,
             'stats': self.crawler.stats,
-            'spiders': self.crawler.spiders,
             'settings': self.crawler.settings,
             'est': lambda: print_engine_status(self.crawler.engine),
             'p': pprint.pprint,

From ad587ea792d4f401abc69d010b350dcafe84eab6 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 17 Apr 2015 01:54:26 +0500
Subject: [PATCH 0209/2974] rename CrawlerRunner.spiders to
 CrawlerRunner.spider_loader

---
 scrapy/commands/check.py            |  6 ++---
 scrapy/commands/edit.py             |  2 +-
 scrapy/commands/fetch.py            |  6 ++---
 scrapy/commands/genspider.py        |  2 +-
 scrapy/commands/list.py             |  2 +-
 scrapy/commands/parse.py            |  6 ++---
 scrapy/commands/shell.py            |  6 ++---
 scrapy/crawler.py                   | 13 ++++++++---
 scrapy/spiderloader.py              |  2 +-
 scrapy/utils/spider.py              |  6 ++---
 tests/test_crawler.py               | 13 +++++++++--
 tests/test_spiderloader/__init__.py | 34 ++++++++++++++---------------
 12 files changed, 57 insertions(+), 41 deletions(-)

diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index 014b00eeb..bf07f360f 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -69,10 +69,10 @@ class Command(ScrapyCommand):
         # contract requests
         contract_reqs = defaultdict(list)
 
-        spiders = self.crawler_process.spiders
+        spider_loader = self.crawler_process.spider_loader
 
-        for spidername in args or spiders.list():
-            spidercls = spiders.load(spidername)
+        for spidername in args or spider_loader.list():
+            spidercls = spider_loader.load(spidername)
             spidercls.start_requests = lambda s: conman.from_spider(s, result)
 
             tested_methods = conman.tested_methods_from_spidercls(spidercls)
diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py
index e20e7c2e5..016c98d2d 100644
--- a/scrapy/commands/edit.py
+++ b/scrapy/commands/edit.py
@@ -27,7 +27,7 @@ class Command(ScrapyCommand):
 
         editor = self.settings['EDITOR']
         try:
-            spidercls = self.crawler_process.spiders.load(args[0])
+            spidercls = self.crawler_process.spider_loader.load(args[0])
         except KeyError:
             return self._err("Spider not found: %s" % args[0])
 
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index ca9fd57f5..89809a177 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -48,10 +48,10 @@ class Command(ScrapyCommand):
         request.meta['handle_httpstatus_all'] = True
 
         spidercls = DefaultSpider
-        spiders = self.crawler_process.spiders
+        spider_loader = self.crawler_process.spider_loader
         if opts.spider:
-            spidercls = spiders.load(opts.spider)
+            spidercls = spider_loader.load(opts.spider)
         else:
-            spidercls = spidercls_for_request(spiders, request, spidercls)
+            spidercls = spidercls_for_request(spider_loader, request, spidercls)
         self.crawler_process.crawl(spidercls, start_requests=lambda: [request])
         self.crawler_process.start()
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 52c5d9f94..9d68ec845 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -65,7 +65,7 @@ class Command(ScrapyCommand):
             return
 
         try:
-            spidercls = self.crawler_process.spiders.load(name)
+            spidercls = self.crawler_process.spider_loader.load(name)
         except KeyError:
             pass
         else:
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index 2d55d59bd..3b35cbb92 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -10,5 +10,5 @@ class Command(ScrapyCommand):
         return "List available spiders"
 
     def run(self, args, opts):
-        for s in sorted(self.crawler_process.spiders.list()):
+        for s in sorted(self.crawler_process.spider_loader.list()):
             print(s)
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index b8cc140d4..3e006ede3 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -124,15 +124,15 @@ class Command(ScrapyCommand):
                     level=log.ERROR, spider=spider.name)
 
     def set_spidercls(self, url, opts):
-        spiders = self.crawler_process.spiders
+        spider_loader = self.crawler_process.spider_loader
         if opts.spider:
             try:
-                self.spidercls = spiders.load(opts.spider)
+                self.spidercls = spider_loader.load(opts.spider)
             except KeyError:
                 log.msg(format='Unable to find spider: %(spider)s',
                         level=log.ERROR, spider=opts.spider)
         else:
-            self.spidercls = spidercls_for_request(spiders, Request(url))
+            self.spidercls = spidercls_for_request(spider_loader, Request(url))
             if not self.spidercls:
                 log.msg(format='Unable to find spider for: %(url)s',
                         level=log.ERROR, url=url)
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 7c0706482..f8ad8a491 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -42,13 +42,13 @@ class Command(ScrapyCommand):
 
     def run(self, args, opts):
         url = args[0] if args else None
-        spiders = self.crawler_process.spiders
+        spider_loader = self.crawler_process.spider_loader
 
         spidercls = DefaultSpider
         if opts.spider:
-            spidercls = spiders.load(opts.spider)
+            spidercls = spider_loader.load(opts.spider)
         elif url:
-            spidercls = spidercls_for_request(spiders, Request(url),
+            spidercls = spidercls_for_request(spider_loader, Request(url),
                                               spidercls, log_multiple=True)
 
         # The crawler is created this way since the Shell manually handles the
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 52956d111..dab1043ec 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -43,7 +43,7 @@ class Crawler(object):
     def spiders(self):
         if not hasattr(self, '_spiders'):
             warnings.warn("Crawler.spiders is deprecated, use "
-                          "CrawlerRunner.spiders or instantiate "
+                          "CrawlerRunner.spider_loader or instantiate "
                           "scrapy.spiderloader.SpiderLoader with your "
                           "settings.",
                           category=ScrapyDeprecationWarning, stacklevel=2)
@@ -84,10 +84,17 @@ class CrawlerRunner(object):
         if isinstance(settings, dict):
             settings = Settings(settings)
         self.settings = settings
-        self.spiders = _get_spider_loader(settings)
+        self.spider_loader = _get_spider_loader(settings)
         self.crawlers = set()
         self._active = set()
 
+    @property
+    def spiders(self):
+        warnings.warn("CrawlerRunner.spiders attribute is renamed to "
+                      "CrawlerRunner.spider_loader.",
+                      category=ScrapyDeprecationWarning, stacklevel=2)
+        return self.spider_loader
+
     def crawl(self, crawler_or_spidercls, *args, **kwargs):
         crawler = crawler_or_spidercls
         if not isinstance(crawler_or_spidercls, Crawler):
@@ -107,7 +114,7 @@ class CrawlerRunner(object):
 
     def _create_crawler(self, spidercls):
         if isinstance(spidercls, six.string_types):
-            spidercls = self.spiders.load(spidercls)
+            spidercls = self.spider_loader.load(spidercls)
         return Crawler(spidercls, self.settings)
 
     def _setup_crawler_logging(self, crawler):
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 484fef456..a88db27c8 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -41,7 +41,7 @@ class SpiderLoader(object):
 
     def find_by_request(self, request):
         """
-        Return the list of spiders names that can handle the given request.
+        Return the list of spider names that can handle the given request.
         """
         return [name for name, cls in self._spiders.items()
                 if cls.handles_request(request)]
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 68de76bfe..44f098f05 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -26,7 +26,7 @@ def iter_spider_classes(module):
            getattr(obj, 'name', None):
             yield obj
 
-def spidercls_for_request(spiderloader, request, default_spidercls=None,
+def spidercls_for_request(spider_loader, request, default_spidercls=None,
                           log_none=False, log_multiple=False):
     """Return a spider class that handles the given Request.
 
@@ -38,9 +38,9 @@ def spidercls_for_request(spiderloader, request, default_spidercls=None,
     default_spidercls passed. It can optionally log if multiple or no spiders
     are found.
     """
-    snames = spiderloader.find_by_request(request)
+    snames = spider_loader.find_by_request(request)
     if len(snames) == 1:
-        return spiderloader.load(snames[0])
+        return spider_loader.load(snames[0])
 
     if len(snames) > 1 and log_multiple:
         log.msg(format='More than one spider can handle: %(request)s - %(snames)s',
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index f0a3865cb..50da01e73 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -19,8 +19,8 @@ class CrawlerTestCase(unittest.TestCase):
             spiders = self.crawler.spiders
             self.assertEqual(len(w), 1)
             self.assertIn("Crawler.spiders", str(w[0].message))
-            sm_cls = load_object(self.crawler.settings['SPIDER_LOADER_CLASS'])
-            self.assertIsInstance(spiders, sm_cls)
+            sl_cls = load_object(self.crawler.settings['SPIDER_LOADER_CLASS'])
+            self.assertIsInstance(spiders, sl_cls)
 
             self.crawler.spiders
             self.assertEqual(len(w), 1, "Warn deprecated access only once")
@@ -78,3 +78,12 @@ class CrawlerRunnerTestCase(unittest.TestCase):
         )
         self.assertIsInstance(runner.settings, Settings)
 
+    def test_deprecated_attribute_spiders(self):
+        with warnings.catch_warnings(record=True) as w:
+            runner = CrawlerRunner(Settings())
+            spiders = runner.spiders
+            self.assertEqual(len(w), 1)
+            self.assertIn("CrawlerRunner.spiders", str(w[0].message))
+            self.assertIn("CrawlerRunner.spider_loader", str(w[0].message))
+            sl_cls = load_object(runner.settings['SPIDER_LOADER_CLASS'])
+            self.assertIsInstance(spiders, sl_cls)
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index aa3aea19d..42f2f29b3 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -26,53 +26,53 @@ class SpiderLoaderTest(unittest.TestCase):
         shutil.copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(self.tmpdir)
         settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
-        self.spiderloader = SpiderLoader.from_settings(settings)
+        self.spider_loader = SpiderLoader.from_settings(settings)
 
     def tearDown(self):
-        del self.spiderloader
+        del self.spider_loader
         del sys.modules['test_spiders_xxx']
         sys.path.remove(self.tmpdir)
 
     def test_interface(self):
-        verifyObject(ISpiderLoader, self.spiderloader)
+        verifyObject(ISpiderLoader, self.spider_loader)
 
     def test_list(self):
-        self.assertEqual(set(self.spiderloader.list()),
+        self.assertEqual(set(self.spider_loader.list()),
             set(['spider1', 'spider2', 'spider3']))
 
     def test_load(self):
-        spider1 = self.spiderloader.load("spider1")
+        spider1 = self.spider_loader.load("spider1")
         self.assertEqual(spider1.__name__, 'Spider1')
 
     def test_find_by_request(self):
-        self.assertEqual(self.spiderloader.find_by_request(Request('http://scrapy1.org/test')),
+        self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy1.org/test')),
             ['spider1'])
-        self.assertEqual(self.spiderloader.find_by_request(Request('http://scrapy2.org/test')),
+        self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy2.org/test')),
             ['spider2'])
-        self.assertEqual(set(self.spiderloader.find_by_request(Request('http://scrapy3.org/test'))),
+        self.assertEqual(set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
             set(['spider1', 'spider2']))
-        self.assertEqual(self.spiderloader.find_by_request(Request('http://scrapy999.org/test')),
+        self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
             [])
-        self.assertEqual(self.spiderloader.find_by_request(Request('http://spider3.com')),
+        self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com')),
             [])
-        self.assertEqual(self.spiderloader.find_by_request(Request('http://spider3.com/onlythis')),
+        self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')),
             ['spider3'])
 
     def test_load_spider_module(self):
         module = 'tests.test_spiderloader.test_spiders.spider1'
         settings = Settings({'SPIDER_MODULES': [module]})
-        self.spiderloader = SpiderLoader.from_settings(settings)
-        assert len(self.spiderloader._spiders) == 1
+        self.spider_loader = SpiderLoader.from_settings(settings)
+        assert len(self.spider_loader._spiders) == 1
 
     def test_load_spider_module(self):
         prefix = 'tests.test_spiderloader.test_spiders.'
         module = ','.join(prefix + s for s in ('spider1', 'spider2'))
         settings = Settings({'SPIDER_MODULES': module})
-        self.spiderloader = SpiderLoader.from_settings(settings)
-        assert len(self.spiderloader._spiders) == 2
+        self.spider_loader = SpiderLoader.from_settings(settings)
+        assert len(self.spider_loader._spiders) == 2
 
     def test_load_base_spider(self):
         module = 'tests.test_spiderloader.test_spiders.spider0'
         settings = Settings({'SPIDER_MODULES': [module]})
-        self.spiderloader = SpiderLoader.from_settings(settings)
-        assert len(self.spiderloader._spiders) == 0
+        self.spider_loader = SpiderLoader.from_settings(settings)
+        assert len(self.spider_loader._spiders) == 0

From 8d339da4e504484bc35f894ba6f7d38924812a83 Mon Sep 17 00:00:00 2001
From: bagratte <bagratte@live.com>
Date: Fri, 17 Apr 2015 20:55:02 +0400
Subject: [PATCH 0210/2974] add some minor stylistic and grammar corrections to
 tutorial.rst.

---
 docs/intro/tutorial.rst | 74 +++++++++++++++++++++--------------------
 1 file changed, 38 insertions(+), 36 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 286a591ee..b24366eb1 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -33,8 +33,8 @@ for non-programmers`_.
 Creating a project
 ==================
 
-Before you start scraping, you will have set up a new Scrapy project. Enter a
-directory where you'd like to store your code and then run::
+Before you start scraping, you will have to set up a new Scrapy project. Enter a
+directory where you'd like to store your code and run::
 
     scrapy startproject tutorial
 
@@ -69,12 +69,12 @@ like simple python dicts but provide additional protection against populating
 undeclared fields, to prevent typos.
 
 They are declared by creating a :class:`scrapy.Item <scrapy.item.Item>` class and defining
-its attributes as :class:`scrapy.Field <scrapy.item.Field>` objects, like you will in an ORM
+its attributes as :class:`scrapy.Field <scrapy.item.Field>` objects, much like in an ORM
 (don't worry if you're not familiar with ORMs, you will see that this is an
 easy task).
 
-We begin by modeling the item that we will use to hold the sites data obtained
-from dmoz.org, as we want to capture the name, url and description of the
+We begin by modeling the item that we will use to hold the site's data obtained
+from dmoz.org. As we want to capture the name, url and description of the
 sites, we define fields for each of these three attributes. To do that, we edit
 ``items.py``, found in the ``tutorial`` directory. Our Item class looks like this::
 
@@ -86,7 +86,7 @@ sites, we define fields for each of these three attributes. To do that, we edit
         desc = scrapy.Field()
 
 This may seem complicated at first, but defining the item allows you to use other handy
-components of Scrapy that need to know how your item looks.
+components of Scrapy that need to know what does your item look like.
 
 Our first Spider
 ================
@@ -95,7 +95,7 @@ Spiders are user-written classes used to scrape information from a domain (or gr
 of domains).
 
 They define an initial list of URLs to download, how to follow links, and how
-to parse the contents of those pages to extract :ref:`items <topics-items>`.
+to parse the contents of pages to extract :ref:`items <topics-items>`.
 
 To create a Spider, you must subclass :class:`scrapy.Spider <scrapy.spider.Spider>` and
 define the three main mandatory attributes:
@@ -103,12 +103,12 @@ define the three main mandatory attributes:
 * :attr:`~scrapy.spider.Spider.name`: identifies the Spider. It must be
   unique, that is, you can't set the same name for different Spiders.
 
-* :attr:`~scrapy.spider.Spider.start_urls`: is a list of URLs where the
-  Spider will begin to crawl from.  So, the first pages downloaded will be those
+* :attr:`~scrapy.spider.Spider.start_urls`: a list of URLs where the
+  Spider will begin to crawl from.  The first pages downloaded will be those
   listed here. The subsequent URLs will be generated successively from data
   contained in the start URLs.
 
-* :meth:`~scrapy.spider.Spider.parse` is a method of the spider, which will
+* :meth:`~scrapy.spider.Spider.parse` a method of the spider, which will
   be called with the downloaded :class:`~scrapy.http.Response` object of each
   start URL. The response is passed to the method as the first and only
   argument.
@@ -160,13 +160,13 @@ will get an output similar to this::
     2014-01-23 18:13:09-0400 [dmoz] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> (referer: None)
     2014-01-23 18:13:09-0400 [dmoz] INFO: Closing spider (finished)
 
-Pay attention to the lines containing ``[dmoz]``, which corresponds to our
+Pay attention to the lines containing ``[dmoz]``, which correspond to our
 spider. You can see a log line for each URL defined in ``start_urls``. Because
 these URLs are the starting ones, they have no referrers, which is shown at the
 end of the log line, where it says ``(referer: None)``.
 
-But more interesting, as our ``parse`` method instructs, two files have been
-created: *Books* and *Resources*, with the content of both URLs.
+But more interestingly, as our ``parse`` method instructs, two files have been
+created: *Books* and *Resources*, with the content of respective URLs.
 
 What just happened under the hood?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -196,7 +196,7 @@ mechanisms see the :ref:`Selectors documentation <topics-selectors>`.
 Here are some examples of XPath expressions and their meanings:
 
 * ``/html/head/title``: selects the ``<title>`` element, inside the ``<head>``
-  element of a HTML document
+  element of an HTML document
 
 * ``/html/head/title/text()``: selects the text inside the aforementioned
   ``<title>`` element.
@@ -212,7 +212,7 @@ recommend `this XPath tutorial <http://www.w3schools.com/XPath/default.asp>`_.
 
 For working with XPaths, Scrapy provides :class:`~scrapy.selector.Selector`
 class and convenient shortcuts to avoid instantiating selectors yourself
-everytime you need to select something from a response.
+every time you need to select something from a response.
 
 You can see selectors as objects that represent nodes in the document
 structure. So, the first instantiated selectors are associated with the root
@@ -222,11 +222,11 @@ Selectors have four basic methods (click on the method to see the complete API
 documentation):
 
 * :meth:`~scrapy.selector.Selector.xpath`: returns a list of selectors, each of
-  them representing the nodes selected by the xpath expression given as
+  which represents the nodes selected by the xpath expression given as
   argument.
 
 * :meth:`~scrapy.selector.Selector.css`: returns a list of selectors, each of
-  them representing the nodes selected by the CSS expression given as argument.
+  which represents the nodes selected by the CSS expression given as argument.
 
 * :meth:`~scrapy.selector.Selector.extract`: returns a unicode string with the
   selected data.
@@ -248,7 +248,7 @@ To start a shell, you must go to the project's top level directory and run::
 
 .. note::
 
-   Remember to always enclose urls with quotes when running Scrapy shell from
+   Remember to always enclose urls with quotes in running Scrapy shell from
    command-line, otherwise urls containing arguments (ie. ``&`` character)
    will not work.
 
@@ -275,10 +275,12 @@ After the shell loads, you will have the response fetched in a local
 ``response`` variable, so if you type ``response.body`` you will see the body
 of the response, or you can type ``response.headers`` to see its headers.
 
-More important, if you type ``response.selector`` you will access a selector
-object you can use to query the response, and convenient shortcuts like
-``response.xpath()`` and ``response.css()`` mapping to
-``response.selector.xpath()`` and ``response.selector.css()``
+More importantly ``response`` has a ``selector`` attribute which is an instance of
+:class:`~scrapy.selector.Selector` class, instantiated with this particular ``response``.
+You can run queries on ``response`` by calling ``response.selector.xpath()`` or
+``response.selector.css()``. There are also some convenience shortcuts like ``response.xpath()``
+or ``response.xml()`` which map directly to ``response.selector.xpath()`` and
+``response.selector.css()``.
 
 
 So let's try it::
@@ -305,27 +307,27 @@ Now, let's try to extract some real information from those pages.
 
 You could type ``response.body`` in the console, and inspect the source code to
 figure out the XPaths you need to use. However, inspecting the raw HTML code
-there could become a very tedious task. To make this an easier task, you can
-use some Firefox extensions like Firebug. For more information see
-:ref:`topics-firebug` and :ref:`topics-firefox`.
+there could become a very tedious task. To make it easier, you can
+use Firefox Developer Tools or some Firefox extensions like Firebug. For more
+information see :ref:`topics-firebug` and :ref:`topics-firefox`.
 
-After inspecting the page source, you'll find that the web sites information
+After inspecting the page source, you'll find that the web site's information
 is inside a ``<ul>`` element, in fact the *second* ``<ul>`` element.
 
-So we can select each ``<li>`` element belonging to the sites list with this
+So we can select each ``<li>`` element belonging to the site's list with this
 code::
 
     response.xpath('//ul/li')
 
-And from them, the sites descriptions::
+And from them, the site's descriptions::
 
     response.xpath('//ul/li/text()').extract()
 
-The sites titles::
+The site's titles::
 
     response.xpath('//ul/li/a/text()').extract()
 
-And the sites links::
+And the site's links::
 
     response.xpath('//ul/li/a/@href').extract()
 
@@ -365,8 +367,8 @@ Let's add this code to our spider::
                 desc = sel.xpath('text()').extract()
                 print title, link, desc
 
-Now try crawling the dmoz.org domain again and you'll see sites being printed
-in your output, run::
+Now try crawling dmoz.org again and you'll see sites being printed
+in your output. Run::
 
     scrapy crawl dmoz
 
@@ -409,7 +411,7 @@ scraped so far, the final code for our Spider would be like this::
 .. note:: You can find a fully-functional variant of this spider in the dirbot_
    project available at https://github.com/scrapy/dirbot
 
-Now doing a crawl on the dmoz.org domain yields ``DmozItem`` objects::
+Now crawling dmoz.org yields ``DmozItem`` objects::
 
    [dmoz] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
         {'desc': [u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.\n],
@@ -423,12 +425,12 @@ Now doing a crawl on the dmoz.org domain yields ``DmozItem`` objects::
 Storing the scraped data
 ========================
 
-The simplest way to store the scraped data is by using the :ref:`Feed exports
+The simplest way to store the scraped data is by using :ref:`Feed exports
 <topics-feed-exports>`, with the following command::
 
     scrapy crawl dmoz -o items.json
 
-That will generate a ``items.json`` file containing all scraped items,
+That will generate an ``items.json`` file containing all scraped items,
 serialized in `JSON`_.
 
 In small projects (like the one in this tutorial), that should be enough.
@@ -441,7 +443,7 @@ pipelines if you just want to store the scraped items.
 Next steps
 ==========
 
-This tutorial covers only the basics of Scrapy, but there's a lot of other
+This tutorial covered only the basics of Scrapy, but there's a lot of other
 features not mentioned here. Check the :ref:`topics-whatelse` section in
 :ref:`intro-overview` chapter for a quick overview of the most important ones.
 

From dffc4549d752714f9d53c64f6a3066caecb71bb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Ricardo?= <root@josericardo.eti.br>
Date: Fri, 17 Apr 2015 14:27:11 -0400
Subject: [PATCH 0211/2974] Fix small typo in the docs

---
 docs/topics/leaks.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 21cfcf9bc..9d15ebe48 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -101,7 +101,7 @@ Let's see a concrete example of an hypothetical case of memory leaks.
 Suppose we have some spider with a line similar to this one::
 
     return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id,
-        callback=self.parse, meta={referer: response}")
+        callback=self.parse, meta={referer: response})
 
 That line is passing a response reference inside a request which effectively
 ties the response lifetime to the requests' one, and that would definitely

From 02629b5f7b29d9955ea6e8468cadf30648ffbd84 Mon Sep 17 00:00:00 2001
From: rajathkumarmp <rajathkumarmp1995@gmail.com>
Date: Sat, 18 Apr 2015 13:00:34 +0530
Subject: [PATCH 0212/2974] Added link to ipython in doc.

---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index b24366eb1..c1f6d3093 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -239,7 +239,7 @@ Trying Selectors in the Shell
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 To illustrate the use of Selectors we're going to use the built-in :ref:`Scrapy
-shell <topics-shell>`, which also requires IPython (an extended Python console)
+shell <topics-shell>`, which also requires `IPython <http://ipython.org/>`_ (an extended Python console)
 installed on your system.
 
 To start a shell, you must go to the project's top level directory and run::

From beea9267a17354101cf3b4611c1e595ed2a8b45c Mon Sep 17 00:00:00 2001
From: bagratte <bagratte@live.com>
Date: Sat, 18 Apr 2015 19:48:25 +0400
Subject: [PATCH 0213/2974] minor corrections in documentation.

---
 .gitignore                |  3 +++
 docs/intro/tutorial.rst   |  4 +--
 docs/topics/commands.rst  | 12 ++++-----
 docs/topics/loaders.rst   | 12 ++++-----
 docs/topics/selectors.rst | 52 +++++++++++++++++++--------------------
 docs/topics/spiders.rst   | 10 ++++----
 6 files changed, 48 insertions(+), 45 deletions(-)

diff --git a/.gitignore b/.gitignore
index 97535ccaf..dc6e8bb67 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,6 @@ venv
 build
 dist
 .idea
+
+# Windows
+Thumbs.db
\ No newline at end of file
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index b24366eb1..a3bf7025a 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -108,7 +108,7 @@ define the three main mandatory attributes:
   listed here. The subsequent URLs will be generated successively from data
   contained in the start URLs.
 
-* :meth:`~scrapy.spider.Spider.parse` a method of the spider, which will
+* :meth:`~scrapy.spider.Spider.parse`: a method of the spider, which will
   be called with the downloaded :class:`~scrapy.http.Response` object of each
   start URL. The response is passed to the method as the first and only
   argument.
@@ -248,7 +248,7 @@ To start a shell, you must go to the project's top level directory and run::
 
 .. note::
 
-   Remember to always enclose urls with quotes in running Scrapy shell from
+   Remember to always enclose urls in quotes when running Scrapy shell from
    command-line, otherwise urls containing arguments (ie. ``&`` character)
    will not work.
 
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 478746d3c..5a223f583 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -80,8 +80,8 @@ some usage help and the available commands::
       fetch         Fetch a URL using the Scrapy downloader
     [...]
 
-The first line will print the currently active project, if you're inside a
-Scrapy project. In this, it was run from outside a project. If run from inside
+The first line will print the currently active project if you're inside a
+Scrapy project. In this example it was run from outside a project. If run from inside
 a project it would have printed something like this::
 
     Scrapy X.Y - project: myproject
@@ -135,7 +135,7 @@ Available tool commands
 =======================
 
 This section contains a list of the available built-in commands with a
-description and some usage examples. Remember you can always get more info
+description and some usage examples. Remember, you can always get more info
 about each command by running::
 
     scrapy <command> -h
@@ -196,7 +196,7 @@ genspider
 
 Create a new spider in the current project.
 
-This is just a convenient shortcut command for creating spiders based on
+This is just a convenience shortcut command for creating spiders based on
 pre-defined templates, but certainly not the only way to create spiders. You
 can just create the spider source code files yourself, instead of using this
 command.
@@ -298,7 +298,7 @@ edit
 Edit the given spider using the editor defined in the :setting:`EDITOR`
 setting.
 
-This command is provided only as a convenient shortcut for the most common
+This command is provided only as a convenience shortcut for the most common
 case, the developer is of course free to choose any tool or IDE to write and
 debug his spiders.
 
@@ -318,7 +318,7 @@ Downloads the given URL using the Scrapy downloader and writes the contents to
 standard output.
 
 The interesting thing about this command is that it fetches the page how the
-spider would download it. For example, if the spider has an ``USER_AGENT``
+spider would download it. For example, if the spider has a ``USER_AGENT``
 attribute which overrides the User Agent, it will use that one.
 
 So this command can be used to "see" how your spider would fetch a certain page.
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index b2c8a018d..603d28f78 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -9,7 +9,7 @@ Item Loaders
 
 Item Loaders provide a convenient mechanism for populating scraped :ref:`Items
 <topics-items>`. Even though Items can be populated using their own
-dictionary-like API, the Item Loaders provide a much more convenient API for
+dictionary-like API, Item Loaders provide a much more convenient API for
 populating them from a scraping process, by automating some common tasks like
 parsing the raw extracted data before assigning it.
 
@@ -25,7 +25,7 @@ Using Item Loaders to populate items
 ====================================
 
 To use an Item Loader, you must first instantiate it. You can either
-instantiate it with an dict-like object (e.g. Item or dict) or without one, in
+instantiate it with a dict-like object (e.g. Item or dict) or without one, in
 which case an Item is automatically instantiated in the Item Loader constructor
 using the Item class specified in the :attr:`ItemLoader.default_item_class`
 attribute.
@@ -67,7 +67,7 @@ and finally the ``last_update`` field is populated directly with a literal value
 (``today``) using a different method: :meth:`~ItemLoader.add_value`.
 
 Finally, when all data is collected, the :meth:`ItemLoader.load_item` method is
-called which actually populates and returns the item populated with the data
+called which actually returns the item populated with the data
 previously extracted and collected with the :meth:`~ItemLoader.add_xpath`,
 :meth:`~ItemLoader.add_css`, and :meth:`~ItemLoader.add_value` calls.
 
@@ -565,8 +565,8 @@ Here is a list of all built-in processors:
 .. class:: Identity
 
     The simplest processor, which doesn't do anything. It returns the original
-    values unchanged. It doesn't receive any constructor arguments nor accepts
-    Loader contexts.
+    values unchanged. It doesn't receive any constructor arguments, nor does it
+	accept Loader contexts.
 
     Example::
 
@@ -579,7 +579,7 @@ Here is a list of all built-in processors:
 
     Returns the first non-null/non-empty value from the values received,
     so it's typically used as an output processor to single-valued fields.
-    It doesn't receive any constructor arguments, nor accept Loader contexts.
+    It doesn't receive any constructor arguments, nor does it accept Loader contexts.
 
     Example::
 
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index a84ac9c99..ccac9ec0b 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -13,9 +13,9 @@ achieve this:
    HTML code and also deals with bad markup reasonably well, but it has one
    drawback: it's slow.
 
- * `lxml`_ is a XML parsing library (which also parses HTML) with a pythonic
-   API based on `ElementTree`_ (which is not part of the Python standard
-   library).
+ * `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
+   API based on `ElementTree`_. (lxml is not part of the Python standard
+   library.)
 
 Scrapy comes with its own mechanism for extracting data. They're called
 selectors because they "select" certain parts of the HTML document specified
@@ -72,7 +72,7 @@ Constructing from response::
     >>> Selector(response=response).xpath('//span/text()').extract()
     [u'good']
 
-For convenience, response objects exposes a selector on `.selector` attribute,
+For convenience, response objects expose a selector on `.selector` attribute,
 it's totally OK to use this shortcut when possible::
 
     >>> response.selector.xpath('//span/text()').extract()
@@ -114,17 +114,17 @@ page, let's construct an XPath for selecting the text inside the title tag::
     >>> response.selector.xpath('//title/text()')
     [<Selector (text) xpath=//title/text()>]
 
-Querying responses using XPath and CSS is so common that responses includes two
-convenient shortcuts: ``response.xpath()`` and ``response.css()``::
+Querying responses using XPath and CSS is so common that responses include two
+convenience shortcuts: ``response.xpath()`` and ``response.css()``::
 
     >>> response.xpath('//title/text()')
     [<Selector (text) xpath=//title/text()>]
     >>> response.css('title::text')
     [<Selector (text) xpath=//title/text()>]
 
-As you can see, ``.xpath()`` and ``.css()`` methods returns an
+As you can see, ``.xpath()`` and ``.css()`` methods return a
 :class:`~scrapy.selector.SelectorList` instance, which is a list of new
-selectors. This API can be used quickly for selecting nested data::
+selectors. This API can be used for quickly selecting nested data::
 
     >>> response.css('img').xpath('@src').extract()
     [u'image1_thumb.jpg',
@@ -196,7 +196,7 @@ Now we're going to get the base URL and some image links::
 Nesting selectors
 -----------------
 
-The selection methods (``.xpath()`` or ``.css()``) returns a list of selectors
+The selection methods (``.xpath()`` or ``.css()``) return a list of selectors
 of the same type, so you can call the selection methods for those selectors
 too. Here's an example::
 
@@ -221,12 +221,12 @@ too. Here's an example::
 Using selectors with regular expressions
 ----------------------------------------
 
-:class:`~scrapy.selector.Selector` also have a ``.re()`` method for extracting
+:class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting
 data using regular expressions. However, unlike using ``.xpath()`` or
-``.css()`` methods, ``.re()`` method returns a list of unicode strings. So you
+``.css()`` methods, ``.re()`` returns a list of unicode strings. So you
 can't construct nested ``.re()`` calls.
 
-Here's an example used to extract images names from the :ref:`HTML code
+Here's an example used to extract image names from the :ref:`HTML code
 <topics-selectors-htmlcode>` above::
 
     >>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
@@ -295,7 +295,7 @@ set     \http://exslt.org/sets                   `set manipulation`_
 Regular expressions
 ~~~~~~~~~~~~~~~~~~~
 
-The ``test()`` function for example can prove quite useful when XPath's
+The ``test()`` function, for example, can prove quite useful when XPath's
 ``starts-with()`` or ``contains()`` are not sufficient.
 
 Example selecting links in list item with a "class" attribute ending with a digit::
@@ -440,7 +440,7 @@ you may want to take a look first at this `XPath tutorial`_.
 Using text nodes in a condition
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-When you need to use the text content as argument to a `XPath string function`_,
+When you need to use the text content as argument to an `XPath string function`_,
 avoid using ``.//text()`` and use just ``.`` instead.
 
 This is because the expression ``.//text()`` yields a collection of text elements -- a *node-set*.
@@ -478,7 +478,7 @@ But using the ``.`` to mean the node, works::
 
 .. _`XPath string function`: http://www.w3.org/TR/xpath/#section-String-Functions
 
-Beware the difference between //node[1] and (//node)[1]
+Beware of the difference between //node[1] and (//node)[1]
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ``//node[1]`` selects all the nodes occurring first under their respective parents.
@@ -559,7 +559,7 @@ Built-in Selectors reference
   An instance of :class:`Selector` is a wrapper over response to select
   certain parts of its content.
 
-  ``response`` is a :class:`~scrapy.http.HtmlResponse` or
+  ``response`` is an :class:`~scrapy.http.HtmlResponse` or an
   :class:`~scrapy.http.XmlResponse` object that will be used for selecting and
   extracting data.
 
@@ -593,7 +593,7 @@ Built-in Selectors reference
 
       .. note::
 
-          For convenience this method can be called as ``response.xpath()``
+          For convenience, this method can be called as ``response.xpath()``
 
   .. method:: css(query)
 
@@ -644,7 +644,7 @@ SelectorList objects
 
 .. class:: SelectorList
 
-   The :class:`SelectorList` class is subclass of the builtin ``list``
+   The :class:`SelectorList` class is a subclass of the builtin ``list``
    class, which provides a few additional methods.
 
    .. method:: xpath(query)
@@ -680,17 +680,17 @@ Selector examples on HTML response
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Here's a couple of :class:`Selector` examples to illustrate several concepts.
-In all cases, we assume there is already an :class:`Selector` instantiated with
+In all cases, we assume there is already a :class:`Selector` instantiated with
 a :class:`~scrapy.http.HtmlResponse` object like this::
 
       sel = Selector(html_response)
 
-1. Select all ``<h1>`` elements from a HTML response body, returning a list of
+1. Select all ``<h1>`` elements from an HTML response body, returning a list of
    :class:`Selector` objects (ie. a :class:`SelectorList` object)::
 
       sel.xpath("//h1")
 
-2. Extract the text of all ``<h1>`` elements from a HTML response body,
+2. Extract the text of all ``<h1>`` elements from an HTML response body,
    returning a list of unicode strings::
 
       sel.xpath("//h1").extract()         # this includes the h1 tag
@@ -705,12 +705,12 @@ Selector examples on XML response
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Here's a couple of examples to illustrate several concepts. In both cases we
-assume there is already an :class:`Selector` instantiated with a
+assume there is already a :class:`Selector` instantiated with an
 :class:`~scrapy.http.XmlResponse` object like this::
 
       sel = Selector(xml_response)
 
-1. Select all ``<product>`` elements from a XML response body, returning a list
+1. Select all ``<product>`` elements from an XML response body, returning a list
    of :class:`Selector` objects (ie. a :class:`SelectorList` object)::
 
       sel.xpath("//product")
@@ -752,12 +752,12 @@ nodes can be accessed directly by their names::
      <Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>,
      ...
 
-If you wonder why the namespace removal procedure is not always called, instead
-of having to call it manually. This is because of two reasons which, in order
+If you wonder why the namespace removal procedure isn't called always by default
+instead of having to call it manually, this is because of two reasons, which, in order
 of relevance, are:
 
 1. Removing namespaces requires to iterate and modify all nodes in the
-   document, which is a reasonably expensive operation to performs for all
+   document, which is a reasonably expensive operation to perform for all
    documents crawled by Scrapy
 
 2. There could be some cases where using namespaces is actually required, in
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index e395f36d5..ddeb17d07 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -190,7 +190,7 @@ scrapy.Spider
        dicts or :class:`~scrapy.item.Item` objects.
 
        :param response: the response to parse
-       :type response: :class:~scrapy.http.Response`
+       :type response: :class:`~scrapy.http.Response`
 
    .. method:: log(message, [level, component])
 
@@ -297,10 +297,10 @@ See `Scrapyd documentation`_.
 Generic Spiders
 ===============
 
-Scrapy comes with some useful generic spiders that you can use, to subclass
+Scrapy comes with some useful generic spiders that you can use to subclass
 your spiders from. Their aim is to provide convenient functionality for a few
 common scraping cases, like following all links on a site based on certain
-rules, crawling from `Sitemaps`_, or parsing a XML/CSV feed.
+rules, crawling from `Sitemaps`_, or parsing an XML/CSV feed.
 
 For the examples used in the following spiders, we'll assume you have a project
 with a ``TestItem`` declared in a ``myproject.items`` module::
@@ -342,7 +342,7 @@ CrawlSpider
    .. method:: parse_start_url(response)
 
       This method is called for the start_urls responses. It allows to parse
-      the initial responses and must return either a
+      the initial responses and must return either an
       :class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request`
       object, or an iterable containing any of them.
 
@@ -417,7 +417,7 @@ Let's now take a look at an example CrawlSpider with rules::
 This spider would start crawling example.com's home page, collecting category
 links, and item links, parsing the latter with the ``parse_item`` method. For
 each item response, some data will be extracted from the HTML using XPath, and
-a :class:`~scrapy.item.Item` will be filled with it.
+an :class:`~scrapy.item.Item` will be filled with it.
 
 XMLFeedSpider
 -------------

From 1312bcd007cc96889143f10b6fa0629da3003833 Mon Sep 17 00:00:00 2001
From: bagratte <bagratte@live.com>
Date: Sun, 19 Apr 2015 18:58:15 +0400
Subject: [PATCH 0214/2974] minor corrections in documentation.

---
 .gitignore                      |  2 +-
 docs/topics/feed-exports.rst    |  4 ++--
 docs/topics/item-pipeline.rst   | 10 +++++-----
 docs/topics/link-extractors.rst |  2 +-
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/.gitignore b/.gitignore
index dc6e8bb67..4db22f1d8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,4 +12,4 @@ dist
 .idea
 
 # Windows
-Thumbs.db
\ No newline at end of file
+Thumbs.db
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index e81db6489..367d8de02 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -8,7 +8,7 @@ Feed exports
 
 One of the most frequently required features when implementing scrapers is
 being able to store the scraped data properly and, quite often, that means
-generating a "export file" with the scraped data (commonly called "export
+generating an "export file" with the scraped data (commonly called "export
 feed") to be consumed by other systems.
 
 Scrapy provides this functionality out of the box with the Feed Exports, which
@@ -21,7 +21,7 @@ Serialization formats
 =====================
 
 For serializing the scraped data, the feed exports use the :ref:`Item exporters
-<topics-exporters>` and these formats are supported out of the box:
+<topics-exporters>`. These formats are supported out of the box:
 
  * :ref:`topics-feed-format-json`
  * :ref:`topics-feed-format-jsonlines`
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 973c77516..dd2d79989 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -5,14 +5,14 @@ Item Pipeline
 =============
 
 After an item has been scraped by a spider, it is sent to the Item Pipeline
-which process it through several components that are executed sequentially.
+which processes it through several components that are executed sequentially.
 
 Each item pipeline component (sometimes referred as just "Item Pipeline") is a
 Python class that implements a simple method. They receive an item and perform
 an action over it, also deciding if the item should continue through the
 pipeline or be dropped and no longer processed.
 
-Typical use for item pipelines are:
+Typical uses of item pipelines are:
 
 * cleansing HTML data
 * validating scraped data (checking that the items contain certain fields)
@@ -167,7 +167,7 @@ Duplicates filter
 -----------------
 
 A filter that looks for duplicate items, and drops those items that were
-already processed. Let say that our items have an unique id, but our spider
+already processed. Let's say that our items have a unique id, but our spider
 returns multiples items with the same id::
 
 
@@ -198,6 +198,6 @@ To activate an Item Pipeline component you must add its class to the
    }
 
 The integer values you assign to classes in this setting determine the
-order they run in- items go through pipelines from order number low to
-high. It's customary to define these numbers in the 0-1000 range.
+order in which they run: items go through from lower valued to higher
+valued classes. It's customary to define these numbers in the 0-1000 range.
 
diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 9758c2f35..f2f296fba 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -82,7 +82,7 @@ LxmlLinkExtractor
         module.
     :type deny_extensions: list
 
-    :param restrict_xpaths: is a XPath (or list of XPath's) which defines
+    :param restrict_xpaths: is an XPath (or list of XPath's) which defines
         regions inside the response where links should be extracted from.
         If given, only the text selected by those XPath will be scanned for
         links. See examples below.

From ffc60910aadb8b804c440695baa4dd4e0805df4a Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sun, 19 Apr 2015 13:09:25 -0300
Subject: [PATCH 0215/2974] Remove djangoitem since we moved it to
 scrapy/scrapy-djangoitem

---
 conftest.py                       |   4 -
 docs/topics/djangoitem.rst        | 144 +-----------------------------
 scrapy/__init__.py                |   7 --
 scrapy/contrib/djangoitem.py      |  75 ----------------
 tests/py3-ignores.txt             |   1 -
 tests/test_djangoitem/__init__.py | 103 ---------------------
 tests/test_djangoitem/models.py   |  17 ----
 tests/test_djangoitem/settings.py |   8 --
 tox.ini                           |   2 -
 9 files changed, 3 insertions(+), 358 deletions(-)
 delete mode 100644 scrapy/contrib/djangoitem.py
 delete mode 100644 tests/test_djangoitem/__init__.py
 delete mode 100644 tests/test_djangoitem/models.py
 delete mode 100644 tests/test_djangoitem/settings.py

diff --git a/conftest.py b/conftest.py
index 258a670dd..21f152e45 100644
--- a/conftest.py
+++ b/conftest.py
@@ -2,11 +2,7 @@ import six
 import pytest
 from twisted.python import log
 
-from scrapy import optional_features
-
 collect_ignore = ["scrapy/stats.py", "scrapy/project.py"]
-if 'django' not in optional_features:
-    collect_ignore.append("tests/test_djangoitem/models.py")
 
 if six.PY3:
     for line in open('tests/py3-ignores.txt'):
diff --git a/docs/topics/djangoitem.rst b/docs/topics/djangoitem.rst
index 84417dfab..ae40401a7 100644
--- a/docs/topics/djangoitem.rst
+++ b/docs/topics/djangoitem.rst
@@ -4,146 +4,8 @@
 DjangoItem
 ==========
 
-:class:`DjangoItem` is a class of item that gets its fields definition from a
-Django model, you simply create a :class:`DjangoItem` and specify what Django
-model it relates to.
+DjangoItem has been moved into a separate project.
 
-Besides of getting the model fields defined on your item, :class:`DjangoItem`
-provides a method to create and populate a Django model instance with the item
-data.
+It is hosted at:
 
-Using DjangoItem
-================
-
-:class:`DjangoItem` works much like ModelForms in Django, you create a subclass
-and define its ``django_model`` attribute to be a valid Django model. With this
-you will get an item with a field for each Django model field.
-
-In addition, you can define fields that aren't present in the model and even
-override fields that are present in the model defining them in the item.
-
-Let's see some examples:
-
-Creating a Django model for the examples::
-
-    from django.db import models
-
-    class Person(models.Model):
-        name = models.CharField(max_length=255)
-        age = models.IntegerField()
-
-Defining a basic :class:`DjangoItem`::
-
-    from scrapy.contrib.djangoitem import DjangoItem
-
-    class PersonItem(DjangoItem):
-        django_model = Person
-
-:class:`DjangoItem` work just like :class:`~scrapy.item.Item`::
-
-    >>> p = PersonItem()
-    >>> p['name'] = 'John'
-    >>> p['age'] = '22'
-
-To obtain the Django model from the item, we call the extra method
-:meth:`~DjangoItem.save` of the :class:`DjangoItem`::
-
-    >>> person = p.save()
-    >>> person.name
-    'John'
-    >>> person.age
-    '22'
-    >>> person.id
-    1
-
-The model is already saved when we call :meth:`~DjangoItem.save`, we
-can prevent this by calling it with ``commit=False``. We can use
-``commit=False`` in :meth:`~DjangoItem.save` method to obtain an unsaved model::
-
-    >>> person = p.save(commit=False)
-    >>> person.name
-    'John'
-    >>> person.age
-    '22'
-    >>> person.id
-    None
-
-As said before, we can add other fields to the item::
-
-    import scrapy
-    from scrapy.contrib.djangoitem import DjangoItem
-
-    class PersonItem(DjangoItem):
-        django_model = Person
-        sex = scrapy.Field()
-
-::
-
-   >>> p = PersonItem()
-   >>> p['name'] = 'John'
-   >>> p['age'] = '22'
-   >>> p['sex'] = 'M'
-
-.. note:: fields added to the item won't be taken into account when doing a :meth:`~DjangoItem.save`
-
-And we can override the fields of the model with your own::
-
-    class PersonItem(DjangoItem):
-        django_model = Person
-        name = scrapy.Field(default='No Name')
-
-This is useful to provide properties to the field, like a default or any other
-property that your project uses.
-
-DjangoItem caveats
-==================
-
-DjangoItem is a rather convenient way to integrate Scrapy projects with Django
-models, but bear in mind that Django ORM may not scale well if you scrape a lot
-of items (ie. millions) with Scrapy. This is because a relational backend is
-often not a good choice for a write intensive application (such as a web
-crawler), specially if the database is highly normalized and with many indices.
-
-Django settings set up
-======================
-
-To use the Django models outside the Django application you need to set up the
-``DJANGO_SETTINGS_MODULE`` environment variable and --in most cases-- modify
-the ``PYTHONPATH`` environment variable to be able to import the settings
-module.
-
-There are many ways to do this depending on your use case and preferences.
-Below is detailed one of the simplest ways to do it.
-
-Suppose your Django project is named ``mysite``, is located in the path
-``/home/projects/mysite`` and you have created an app ``myapp`` with the model
-``Person``. That means your directory structure is something like this::
-
-    /home/projects/mysite
-    ├── manage.py
-    ├── myapp
-    │   ├── __init__.py
-    │   ├── models.py
-    │   ├── tests.py
-    │   └── views.py
-    └── mysite
-        ├── __init__.py
-        ├── settings.py
-        ├── urls.py
-        └── wsgi.py
-
-Then you need to add ``/home/projects/mysite`` to the ``PYTHONPATH``
-environment variable and set up the environment variable
-``DJANGO_SETTINGS_MODULE`` to ``mysite.settings``. That can be done in your
-Scrapy's settings file by adding the lines below::
-
-  import sys
-  sys.path.append('/home/projects/mysite')
-
-  import os
-  os.environ['DJANGO_SETTINGS_MODULE'] = 'mysite.settings'
-
-Notice that we modify the ``sys.path`` variable instead the ``PYTHONPATH``
-environment variable as we are already within the python runtime. If everything
-is right, you should be able to start the ``scrapy shell`` command and import
-the model ``Person`` (i.e. ``from myapp.models import Person``).
+    https://github.com/scrapy/scrapy-djangoitem
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index d60b239a0..1f38299d2 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -38,13 +38,6 @@ except ImportError:
     pass
 else:
     optional_features.add('boto')
-try:
-    import django
-    del django
-except ImportError:
-    pass
-else:
-    optional_features.add('django')
 
 from twisted import version as _txv
 twisted_version = (_txv.major, _txv.minor, _txv.micro)
diff --git a/scrapy/contrib/djangoitem.py b/scrapy/contrib/djangoitem.py
deleted file mode 100644
index 69f7ef1ac..000000000
--- a/scrapy/contrib/djangoitem.py
+++ /dev/null
@@ -1,75 +0,0 @@
-from scrapy.item import Field, Item, ItemMeta
-from scrapy import optional_features
-if 'django' in optional_features:
-    from django.core.exceptions import ValidationError
-
-
-class DjangoItemMeta(ItemMeta):
-
-    def  __new__(mcs, class_name, bases, attrs):
-        cls = super(DjangoItemMeta, mcs).__new__(mcs, class_name, bases, attrs)
-        cls.fields = cls.fields.copy()
-
-        if cls.django_model:
-            cls._model_fields = []
-            cls._model_meta = cls.django_model._meta
-            for model_field in cls._model_meta.fields:
-                if not model_field.auto_created:
-                    if model_field.name not in cls.fields:
-                        cls.fields[model_field.name] = Field()
-                    cls._model_fields.append(model_field.name)
-        return cls
-
-
-class DjangoItem(Item):
-
-    __metaclass__ = DjangoItemMeta
-
-    django_model = None
-
-    def __init__(self, *args, **kwargs):
-        super(DjangoItem, self).__init__(*args, **kwargs)
-        self._instance = None
-        self._errors = None
-
-    def save(self, commit=True):
-        if commit:
-            self.instance.save()
-        return self.instance
-
-    def is_valid(self, exclude=None):
-        self._get_errors(exclude)
-        return not bool(self._errors)
-
-    def _get_errors(self, exclude=None):
-        if self._errors is not None:
-            return self._errors
-
-        self._errors = {}
-        if exclude is None:
-            exclude = []
-
-        try:
-            self.instance.clean_fields(exclude=exclude)
-        except ValidationError as e:
-            self._errors = e.update_error_dict(self._errors)
-
-        try:
-            self.instance.clean()
-        except ValidationError as e:
-            self._errors = e.update_error_dict(self._errors)
-
-        # uniqueness is not checked, because it is faster to check it when
-        # saving object to database. Just beware, that failed save()
-        # raises IntegrityError instead of ValidationError.
-
-        return self._errors
-    errors = property(_get_errors)
-
-    @property
-    def instance(self):
-        if self._instance is None:
-            modelargs = dict((k, self.get(k)) for k in self._values
-                             if k in self._model_fields)
-            self._instance = self.django_model(**modelargs)
-        return self._instance
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index fb1daf62f..0e7b10386 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -9,7 +9,6 @@ tests/test_contrib_linkextractors.py
 tests/test_contrib_loader.py
 tests/test_crawl.py
 tests/test_crawler.py
-tests/test_djangoitem/__init__.py
 tests/test_downloader_handlers.py
 tests/test_downloadermiddleware_ajaxcrawlable.py
 tests/test_downloadermiddleware_cookies.py
diff --git a/tests/test_djangoitem/__init__.py b/tests/test_djangoitem/__init__.py
deleted file mode 100644
index 45fd7c470..000000000
--- a/tests/test_djangoitem/__init__.py
+++ /dev/null
@@ -1,103 +0,0 @@
-import os
-from twisted.trial import unittest
-
-from scrapy.contrib.djangoitem import DjangoItem, Field
-from scrapy import optional_features
-
-os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.test_djangoitem.settings'
-
-if 'django' in optional_features:
-    from .models import Person, IdentifiedPerson
-
-    class BasePersonItem(DjangoItem):
-        django_model = Person
-
-    class NewFieldPersonItem(BasePersonItem):
-        other = Field()
-
-    class OverrideFieldPersonItem(BasePersonItem):
-        age = Field()
-
-    class IdentifiedPersonItem(DjangoItem):
-        django_model = IdentifiedPerson
-
-
-class DjangoItemTest(unittest.TestCase):
-
-    def assertSortedEqual(self, first, second, msg=None):
-        return self.assertEqual(sorted(first), sorted(second), msg)
-
-    def setUp(self):
-        if 'django' not in optional_features:
-            raise unittest.SkipTest("Django is not available")
-
-    def test_base(self):
-        i = BasePersonItem()
-        self.assertSortedEqual(i.fields.keys(), ['age', 'name'])
-
-    def test_new_fields(self):
-        i = NewFieldPersonItem()
-        self.assertSortedEqual(i.fields.keys(), ['age', 'other', 'name'])
-
-    def test_override_field(self):
-        i = OverrideFieldPersonItem()
-        self.assertSortedEqual(i.fields.keys(), ['age', 'name'])
-
-    def test_custom_primary_key_field(self):
-        """
-        Test that if a custom primary key exists, it is
-        in the field list.
-        """
-        i = IdentifiedPersonItem()
-        self.assertSortedEqual(i.fields.keys(), ['age', 'identifier', 'name'])
-
-    def test_save(self):
-        i = BasePersonItem()
-        self.assertSortedEqual(i.fields.keys(), ['age', 'name'])
-
-        i['name'] = 'John'
-        i['age'] = '22'
-        person = i.save(commit=False)
-
-        self.assertEqual(person.name, 'John')
-        self.assertEqual(person.age, '22')
-
-    def test_override_save(self):
-        i = OverrideFieldPersonItem()
-
-        i['name'] = 'John'
-        # it is not obvious that "age" should be saved also, since it was
-        # redefined in child class
-        i['age'] = '22'
-        person = i.save(commit=False)
-
-        self.assertEqual(person.name, 'John')
-        self.assertEqual(person.age, '22')
-
-    def test_validation(self):
-        long_name = 'z' * 300
-        i = BasePersonItem(name=long_name)
-        self.assertFalse(i.is_valid())
-        self.assertEqual(set(i.errors), set(['age', 'name']))
-        i = BasePersonItem(name='John')
-        self.assertTrue(i.is_valid(exclude=['age']))
-        self.assertEqual({}, i.errors)
-
-        # once the item is validated, it does not validate again
-        i['name'] = long_name
-        self.assertTrue(i.is_valid())
-
-    def test_override_validation(self):
-        i = OverrideFieldPersonItem()
-        i['name'] = 'John'
-        self.assertFalse(i.is_valid())
-
-        i = i = OverrideFieldPersonItem()
-        i['name'] = 'John'
-        i['age'] = '22'
-        self.assertTrue(i.is_valid())
-
-    def test_default_field_values(self):
-        i = BasePersonItem()
-        person = i.save(commit=False)
-        self.assertEqual(person.name, 'Robot')
diff --git a/tests/test_djangoitem/models.py b/tests/test_djangoitem/models.py
deleted file mode 100644
index 2beaee752..000000000
--- a/tests/test_djangoitem/models.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from django.db import models
-
-
-class Person(models.Model):
-    name = models.CharField(max_length=255, default='Robot')
-    age = models.IntegerField()
-
-    class Meta:
-        app_label = 'test_djangoitem'
-
-class IdentifiedPerson(models.Model):
-    identifier = models.PositiveIntegerField(primary_key=True)
-    name = models.CharField(max_length=255)
-    age = models.IntegerField()
-
-    class Meta:
-        app_label = 'test_djangoitem'
diff --git a/tests/test_djangoitem/settings.py b/tests/test_djangoitem/settings.py
deleted file mode 100644
index 1bee92477..000000000
--- a/tests/test_djangoitem/settings.py
+++ /dev/null
@@ -1,8 +0,0 @@
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': ':memory:',
-    }
-}
-
-SECRET_KEY = 'top-secret'
diff --git a/tox.ini b/tox.ini
index 2dff74973..ae2ef9e07 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,7 +12,6 @@ deps =
     # Extras
     boto
     Pillow
-    django
     leveldb
     -rtests/requirements.txt
 commands =
@@ -26,7 +25,6 @@ deps =
     Twisted==11.1.0
     boto==2.2.2
     Pillow<2.0
-    django==1.3.1
     cssselect==0.9.1
     zope.interface==3.6.1
     -rtests/requirements.txt

From 16a1a938c1e5453ff167b5a3f8fc82b6d04782b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 20 Apr 2015 01:39:52 -0300
Subject: [PATCH 0216/2974] Add 0.24.6 release notes

---
 docs/news.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index bb1b374ce..adf02d9fb 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,16 @@
 Release notes
 =============
 
+0.24.6 (2015-04-20)
+-------------------
+
+- encode invalid xpath with unicode_escape under PY2 (:commit:`07cb3e5`)
+- fix IPython shell scope issue and load IPython user config (:commit:`2c8e573`)
+- Fix small typo in the docs (:commit:`d694019`)
+- Fix small typo (:commit:`f92fa83`)
+- Converted sel.xpath() calls to response.xpath() in Extracting the data (:commit:`c2c6d15`)
+
+
 0.24.5 (2015-02-25)
 -------------------
 

From 017fb25beef63639060485dae393ed6e244d3f70 Mon Sep 17 00:00:00 2001
From: Nikolaos-Digenis Karagiannis <digenishjkl@gmail.com>
Date: Mon, 20 Apr 2015 11:04:31 +0300
Subject: [PATCH 0217/2974] loader test with processors that use item's values

---
 tests/test_contrib_loader.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tests/test_contrib_loader.py b/tests/test_contrib_loader.py
index 6af4cf70c..fd65e8200 100644
--- a/tests/test_contrib_loader.py
+++ b/tests/test_contrib_loader.py
@@ -85,6 +85,27 @@ class BasicItemLoaderTest(unittest.TestCase):
         il.replace_value('sku', [valid_fragment], re=sku_re)
         self.assertEqual(il.load_item()['sku'], u'1234')
 
+    def test_self_referencing_loader(self):
+        class MyLoader(ItemLoader):
+            url_out = TakeFirst()
+
+            def img_url_out(self, values):
+                return (self.get_output_value('url') or '') + values[0]
+
+        il = MyLoader(item={})
+        il.add_value('url', 'http://example.com/')
+        il.add_value('img_url', '1234.png')
+        self.assertEqual(il.load_item(), {
+            'url': 'http://example.com/',
+            'img_url': 'http://example.com/1234.png',
+        })
+
+        il = MyLoader(item={})
+        il.add_value('img_url', '1234.png')
+        self.assertEqual(il.load_item(), {
+            'img_url': '1234.png',
+        })
+
     def test_add_value(self):
         il = TestItemLoader()
         il.add_value('name', u'marta')

From d39722f6a37bc1b1558e75b2c77542925437d3db Mon Sep 17 00:00:00 2001
From: Nikolaos-Digenis Karagiannis <digenishjkl@gmail.com>
Date: Mon, 20 Apr 2015 11:05:56 +0300
Subject: [PATCH 0218/2974] ItemLoader.load_item: iterate over copy of fields

---
 scrapy/contrib/loader/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py
index 7be2524b9..c861f8a24 100644
--- a/scrapy/contrib/loader/__init__.py
+++ b/scrapy/contrib/loader/__init__.py
@@ -80,7 +80,7 @@ class ItemLoader(object):
 
     def load_item(self):
         item = self.item
-        for field_name in self._values:
+        for field_name in tuple(self._values):
             value = self.get_output_value(field_name)
             if value is not None:
                 item[field_name] = value

From 6da091f162f8e02e95b68cf435689bac8acd2efd Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 20 Apr 2015 20:19:28 -0300
Subject: [PATCH 0219/2974] Remove djangoitem topic from index

---
 docs/index.rst | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/docs/index.rst b/docs/index.rst
index b442df4ba..55a465fad 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -154,7 +154,6 @@ Solving specific problems
    topics/autothrottle
    topics/benchmarking
    topics/jobs
-   topics/djangoitem
 
 :doc:`faq`
     Get answers to most frequently asked questions.
@@ -198,9 +197,6 @@ Solving specific problems
 :doc:`topics/jobs`
     Learn how to pause and resume crawls for large spiders.
 
-:doc:`topics/djangoitem`
-    Write scraped items using Django models.
-
 .. _extending-scrapy:
 
 Extending Scrapy

From 7ae37d61f9aa4f0244ba3d7bf1bfaca3e2ac3749 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Tue, 24 Mar 2015 07:38:09 +0000
Subject: [PATCH 0220/2974] remove scrapy deploy command

(closes #1027, #1095, #1102)
---
 docs/topics/commands.rst            |  16 +-
 extras/scrapy_bash_completion       |   2 +-
 scrapy/commands/deploy.py           | 259 ----------------------------
 scrapy/templates/project/scrapy.cfg |   2 +-
 4 files changed, 5 insertions(+), 274 deletions(-)
 delete mode 100644 scrapy/commands/deploy.py

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 5a223f583..638de5712 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -13,6 +13,9 @@ just call "commands" or "Scrapy commands".
 The Scrapy tool provides several commands, for multiple purposes, and each one
 accepts a different set of arguments and options.
 
+(The ``scrapy deploy`` command has been removed in 1.0 in favor of the
+standalone ``scrapyd-deploy``. See `Deploying your project`_.)
+
 Configuration settings
 ======================
 
@@ -168,7 +171,6 @@ Project-only commands:
 * :command:`edit`
 * :command:`parse`
 * :command:`genspider`
-* :command:`deploy`
 * :command:`bench`
 
 .. command:: startproject
@@ -474,18 +476,6 @@ version
 Prints the Scrapy version. If used with ``-v`` it also prints Python, Twisted
 and Platform info, which is useful for bug reports.
 
-.. command:: deploy
-
-deploy
-------
-
-.. versionadded:: 0.11
-
-* Syntax: ``scrapy deploy [ <target:project> | -l <target> | -L ]``
-* Requires project: *yes*
-
-Deploy the project into a Scrapyd server. See `Deploying your project`_.
-
 .. command:: bench
 
 bench
diff --git a/extras/scrapy_bash_completion b/extras/scrapy_bash_completion
index 19fd81c87..07a7d88c5 100644
--- a/extras/scrapy_bash_completion
+++ b/extras/scrapy_bash_completion
@@ -11,7 +11,7 @@ _scrapy_completion() {
         ;;
         *)
             if [ $COMP_CWORD -eq 1 ]; then
-                commands="check crawl deploy edit fetch genspider list parse runspider server settings shell startproject version view"
+                commands="check crawl edit fetch genspider list parse runspider settings shell startproject version view"
                 COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W "$commands" -- "$cmd"))
             fi
         ;;
diff --git a/scrapy/commands/deploy.py b/scrapy/commands/deploy.py
deleted file mode 100644
index bd8ab3f15..000000000
--- a/scrapy/commands/deploy.py
+++ /dev/null
@@ -1,259 +0,0 @@
-from __future__ import print_function
-import sys
-import os
-import glob
-import tempfile
-import shutil
-import time
-import netrc
-import json
-from six.moves.urllib import request
-from six.moves.urllib.parse import urlparse, urljoin
-from six.moves.urllib.error import HTTPError, URLError
-from subprocess import Popen, PIPE, check_call
-
-from w3lib.form import encode_multipart
-
-from scrapy.command import ScrapyCommand
-from scrapy.exceptions import UsageError
-from scrapy.utils.http import basic_auth_header
-from scrapy.utils.python import retry_on_eintr
-from scrapy.utils.conf import get_config, closest_scrapy_cfg
-
-_SETUP_PY_TEMPLATE = \
-"""# Automatically created by: scrapy deploy
-
-from setuptools import setup, find_packages
-
-setup(
-    name         = 'project',
-    version      = '1.0',
-    packages     = find_packages(),
-    entry_points = {'scrapy': ['settings = %(settings)s']},
-)
-"""
-
-class Command(ScrapyCommand):
-
-    requires_project = True
-
-    def syntax(self):
-        return "[options] [ [target] | -l | -L <target> ]"
-
-    def short_desc(self):
-        return "Deploy project in Scrapyd target"
-
-    def long_desc(self):
-        return "Deploy the current project into the given Scrapyd server " \
-            "(known as target)"
-
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
-        parser.add_option("-p", "--project",
-            help="the project name in the target")
-        parser.add_option("-v", "--version",
-            help="the version to deploy. Defaults to current timestamp")
-        parser.add_option("-l", "--list-targets", action="store_true", \
-            help="list available targets")
-        parser.add_option("-d", "--debug", action="store_true",
-            help="debug mode (do not remove build dir)")
-        parser.add_option("-L", "--list-projects", metavar="TARGET", \
-            help="list available projects on TARGET")
-        parser.add_option("--egg", metavar="FILE",
-            help="use the given egg, instead of building it")
-        parser.add_option("--build-egg", metavar="FILE",
-            help="only build the egg, don't deploy it")
-
-    def run(self, args, opts):
-        try:
-            import setuptools
-        except ImportError:
-            raise UsageError("setuptools not installed")
-
-        request.install_opener(request.build_opener(HTTPRedirectHandler))
-
-        if opts.list_targets:
-            for name, target in _get_targets().items():
-                print("%-20s %s" % (name, target['url']))
-            return
-
-        if opts.list_projects:
-            target = _get_target(opts.list_projects)
-            req = request.Request(_url(target, 'listprojects.json'))
-            _add_auth_header(req, target)
-            f = request.urlopen(req)
-            projects = json.loads(f.read())['projects']
-            print(os.linesep.join(projects))
-            return
-
-        tmpdir = None
-
-        if opts.build_egg: # build egg only
-            egg, tmpdir = _build_egg()
-            _log("Writing egg to %s" % opts.build_egg)
-            shutil.copyfile(egg, opts.build_egg)
-        else: # buld egg and deploy
-            target_name = _get_target_name(args)
-            target = _get_target(target_name)
-            project = _get_project(target, opts)
-            version = _get_version(target, opts)
-            if opts.egg:
-                _log("Using egg: %s" % opts.egg)
-                egg = opts.egg
-            else:
-                _log("Packing version %s" % version)
-                egg, tmpdir = _build_egg()
-            if not _upload_egg(target, egg, project, version):
-                self.exitcode = 1
-
-        if tmpdir:
-            if opts.debug:
-                _log("Output dir not removed: %s" % tmpdir)
-            else:
-                shutil.rmtree(tmpdir)
-
-def _log(message):
-    sys.stderr.write(message + os.linesep)
-
-def _get_target_name(args):
-    if len(args) > 1:
-        raise UsageError("Too many arguments: %s" % ' '.join(args))
-    elif args:
-        return args[0]
-    elif len(args) < 1:
-        return 'default'
-
-def _get_project(target, opts):
-    project = opts.project or target.get('project')
-    if not project:
-        raise UsageError("Missing project")
-    return project
-
-def _get_option(section, option, default=None):
-    cfg = get_config()
-    return cfg.get(section, option) if cfg.has_option(section, option) \
-        else default
-
-def _get_targets():
-    cfg = get_config()
-    baset = dict(cfg.items('deploy')) if cfg.has_section('deploy') else {}
-    targets = {}
-    if 'url' in baset:
-        targets['default'] = baset
-    for x in cfg.sections():
-        if x.startswith('deploy:'):
-            t = baset.copy()
-            t.update(cfg.items(x))
-            targets[x[7:]] = t
-    return targets
-
-def _get_target(name):
-    try:
-        return _get_targets()[name]
-    except KeyError:
-        raise UsageError("Unknown target: %s" % name)
-
-def _url(target, action):
-    return urljoin(target['url'], action)
-
-def _get_version(target, opts):
-    version = opts.version or target.get('version')
-    if version == 'HG':
-        p = Popen(['hg', 'tip', '--template', '{rev}'], stdout=PIPE)
-        d = 'r%s' % p.communicate()[0]
-        p = Popen(['hg', 'branch'], stdout=PIPE)
-        b = p.communicate()[0].strip('\n')
-        return '%s-%s' % (d, b)
-    elif version == 'GIT':
-        p = Popen(['git', 'describe', '--always'], stdout=PIPE)
-        d = p.communicate()[0].strip('\n')
-        p = Popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], stdout=PIPE)
-        b = p.communicate()[0].strip('\n')
-        return '%s-%s' % (d, b)
-    elif version:
-        return version
-    else:
-        return str(int(time.time()))
-
-def _upload_egg(target, eggpath, project, version):
-    with open(eggpath, 'rb') as f:
-        eggdata = f.read()
-    data = {
-        'project': project,
-        'version': version,
-        'egg': ('project.egg', eggdata),
-    }
-    body, boundary = encode_multipart(data)
-    url = _url(target, 'addversion.json')
-    headers = {
-        'Content-Type': 'multipart/form-data; boundary=%s' % boundary,
-        'Content-Length': str(len(body)),
-    }
-    req = request.Request(url, body, headers)
-    _add_auth_header(req, target)
-    _log('Deploying to project "%s" in %s' % (project, url))
-    return _http_post(req)
-
-def _add_auth_header(req, target):
-    if 'username' in target:
-        u, p = target.get('username'), target.get('password', '')
-        req.add_header('Authorization', basic_auth_header(u, p))
-    else: # try netrc
-        try:
-            host = urlparse(target['url']).hostname
-            a = netrc.netrc().authenticators(host)
-            req.add_header('Authorization', basic_auth_header(a[0], a[2]))
-        except (netrc.NetrcParseError, IOError, TypeError):
-            pass
-
-def _http_post(req):
-    try:
-        f = request.urlopen(req)
-        _log("Server response (%s):" % f.code)
-        print(f.read())
-        return True
-    except HTTPError as e:
-        _log("Deploy failed (%s):" % e.code)
-        print(e.read())
-    except URLError as e:
-        _log("Deploy failed: %s" % e)
-
-def _build_egg():
-    closest = closest_scrapy_cfg()
-    os.chdir(os.path.dirname(closest))
-    if not os.path.exists('setup.py'):
-        settings = get_config().get('settings', 'default')
-        _create_default_setup_py(settings=settings)
-    d = tempfile.mkdtemp(prefix="scrapydeploy-")
-    o = open(os.path.join(d, "stdout"), "wb")
-    e = open(os.path.join(d, "stderr"), "wb")
-    retry_on_eintr(check_call, [sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d], stdout=o, stderr=e)
-    o.close()
-    e.close()
-    egg = glob.glob(os.path.join(d, '*.egg'))[0]
-    return egg, d
-
-def _create_default_setup_py(**kwargs):
-    with open('setup.py', 'w') as f:
-        f.write(_SETUP_PY_TEMPLATE % kwargs)
-
-
-class HTTPRedirectHandler(request.HTTPRedirectHandler):
-
-    def redirect_request(self, req, fp, code, msg, headers, newurl):
-        newurl = newurl.replace(' ', '%20')
-        if code in (301, 307):
-            return request.Request(newurl,
-                                   data=req.get_data(),
-                                   headers=req.headers,
-                                   origin_req_host=req.get_origin_req_host(),
-                                   unverifiable=True)
-        elif code in (302, 303):
-            newheaders = dict((k, v) for k, v in req.headers.items()
-                              if k.lower() not in ("content-length", "content-type"))
-            return request.Request(newurl,
-                                   headers=newheaders,
-                                   origin_req_host=req.get_origin_req_host(),
-                                   unverifiable=True)
-        else:
-            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
diff --git a/scrapy/templates/project/scrapy.cfg b/scrapy/templates/project/scrapy.cfg
index 22b1100e7..d7f02e0a2 100644
--- a/scrapy/templates/project/scrapy.cfg
+++ b/scrapy/templates/project/scrapy.cfg
@@ -1,7 +1,7 @@
 # Automatically created by: scrapy startproject
 #
 # For more information about the [deploy] section see:
-# http://doc.scrapy.org/en/latest/topics/scrapyd.html
+# https://scrapyd.readthedocs.org/en/latest/deploy.html
 
 [settings]
 default = ${project_name}.settings

From 595146e158b1efec432b35128aab59878da81295 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Mon, 20 Apr 2015 21:09:03 -0300
Subject: [PATCH 0221/2974] some improvements for Scrapy tutorial

---
 docs/Makefile           |   3 +
 docs/intro/tutorial.rst | 133 ++++++++++++++++++++++++++++++----------
 2 files changed, 105 insertions(+), 31 deletions(-)

diff --git a/docs/Makefile b/docs/Makefile
index 4289690f0..3a7b8c2c3 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -8,6 +8,7 @@ PYTHON       = python
 SPHINXOPTS   =
 PAPER        =
 SOURCES      =
+SHELL        = /bin/bash
 
 ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \
                 $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES)
@@ -79,3 +80,5 @@ htmlview: html
 clean:
 	-rm -rf build/*
 
+watch: htmlview
+	watchmedo shell-command -p '*.rst' -c 'make html' -R -D
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 45e809eb9..8680c9c10 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -41,32 +41,30 @@ directory where you'd like to store your code and run::
 This will create a ``tutorial`` directory with the following contents::
 
     tutorial/
-        scrapy.cfg
-        tutorial/
+        scrapy.cfg            # deploy configuration file
+
+        tutorial/             # project's Python module, you'll import your code from here
             __init__.py
-            items.py
-            pipelines.py
-            settings.py
-            spiders/
+
+            items.py          # project items file
+
+            pipelines.py      # project pipelines file
+
+            settings.py       # project settings file
+
+            spiders/          # a directory where you'll later put your spiders
                 __init__.py
                 ...
 
-These are basically:
-
-* ``scrapy.cfg``: the project configuration file
-* ``tutorial/``: the project's python module, you'll later import your code from
-  here.
-* ``tutorial/items.py``: the project's items file.
-* ``tutorial/pipelines.py``: the project's pipelines file.
-* ``tutorial/settings.py``: the project's settings file.
-* ``tutorial/spiders/``: a directory where you'll later put your spiders.
 
 Defining our Item
 =================
 
 `Items` are containers that will be loaded with the scraped data; they work
-like simple python dicts but provide additional protection against populating
-undeclared fields, to prevent typos.
+like simple Python dicts. While you can use plain Python dicts with Scrapy,
+`Items` provide additional protection against populating undeclared fields,
+preventing typos. They can also be used with :ref:`Item Loaders
+<topics-loaders>`, a mechanism with helpers to conveniently populate `Items`.
 
 They are declared by creating a :class:`scrapy.Item <scrapy.item.Item>` class and defining
 its attributes as :class:`scrapy.Field <scrapy.item.Field>` objects, much like in an ORM
@@ -91,14 +89,14 @@ components of Scrapy that need to know what does your item look like.
 Our first Spider
 ================
 
-Spiders are user-written classes used to scrape information from a domain (or group
-of domains).
+Spiders are classes that you define and Scrapy uses to scrape information from a
+domain (or group of domains).
 
 They define an initial list of URLs to download, how to follow links, and how
 to parse the contents of pages to extract :ref:`items <topics-items>`.
 
 To create a Spider, you must subclass :class:`scrapy.Spider <scrapy.spider.Spider>` and
-define the three main mandatory attributes:
+define some attributes:
 
 * :attr:`~scrapy.spider.Spider.name`: identifies the Spider. It must be
   unique, that is, you can't set the same name for different Spiders.
@@ -134,7 +132,7 @@ This is the code for our first Spider; save it in a file named
         ]
 
         def parse(self, response):
-            filename = response.url.split("/")[-2]
+            filename = response.url.split("/")[-2] + '.html'
             with open(filename, 'wb') as f:
                 f.write(response.body)
 
@@ -145,8 +143,9 @@ To put our spider to work, go to the project's top level directory and run::
 
    scrapy crawl dmoz
 
-The ``crawl dmoz`` command runs the spider for the ``dmoz.org`` domain. You
-will get an output similar to this::
+This command runs the spider with name ``dmoz`` that we've just added, that
+will send some requests for the ``dmoz.org`` domain. You will get an output
+similar to this::
 
     2014-01-23 18:13:07-0400 [scrapy] INFO: Scrapy started (bot: tutorial)
     2014-01-23 18:13:07-0400 [scrapy] INFO: Optional features available: ...
@@ -160,13 +159,16 @@ will get an output similar to this::
     2014-01-23 18:13:09-0400 [dmoz] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> (referer: None)
     2014-01-23 18:13:09-0400 [dmoz] INFO: Closing spider (finished)
 
-Pay attention to the lines containing ``[dmoz]``, which correspond to our
-spider. You can see a log line for each URL defined in ``start_urls``. Because
-these URLs are the starting ones, they have no referrers, which is shown at the
-end of the log line, where it says ``(referer: None)``.
 
-But more interestingly, as our ``parse`` method instructs, two files have been
-created: *Books* and *Resources*, with the content of respective URLs.
+.. note::
+    The lines containing ``[dmoz]`` correspond to the log messages for
+    our spider. You can see a log line for each URL defined in ``start_urls``.
+    Because these URLs are the starting ones, they have no referrers, which is
+    shown at the end of the log line, where it says ``(referer: None)``.
+
+Now, check the files in the current directory. You should notice two new files
+have been created: *Books.html* and *Resources.html*, with the content for the respective
+URLs, as our ``parse`` method instructs.
 
 What just happened under the hood?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -207,8 +209,17 @@ Here are some examples of XPath expressions and their meanings:
   attribute ``class="mine"``
 
 These are just a couple of simple examples of what you can do with XPath, but
-XPath expressions are indeed much more powerful. To learn more about XPath we
-recommend `this XPath tutorial <http://www.w3schools.com/XPath/default.asp>`_.
+XPath expressions are indeed much more powerful. To learn more about XPath, we
+recommend `this tutorial to learn XPath through examples
+<http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this tutorial to learn "how
+to think in XPath" <http://plasmasturm.org/log/xpath101/>`_.
+
+.. note:: **CSS vs XPath:** you can go a long way extracting data from web pages
+  using only CSS selectors. However, XPath offers more power because besides
+  navigating the structure, it can also look at the content: you're
+  able to select things like: *the link that contains the text 'Next Page'*.
+  Because of this, we encourage you to learn about XPath even if you
+  already know how to construct CSS selectors.
 
 For working with XPaths, Scrapy provides :class:`~scrapy.selector.Selector`
 class and convenient shortcuts to avoid instantiating selectors yourself
@@ -422,6 +433,66 @@ Now crawling dmoz.org yields ``DmozItem`` objects::
          'link': [u'http://www.informit.com/store/product.aspx?isbn=0130211192'],
          'title': [u'XML Processing with Python']}
 
+
+Following links
+===============
+
+Let's say, instead of just scraping the stuff in *Books* and *Resources* pages,
+you want everything that is under the `Python directory
+<http://www.dmoz.org/Computers/Programming/Languages/Python/>`_.
+
+Now that you know how to extract data from a page, why not extract the links
+for the pages you are interested, follow them and then extract the data you
+want for all of them?
+
+Here is a modification to our spider that does just that::
+
+    import scrapy
+    import urlparse
+
+    from tutorial.items import DmozItem
+
+    class DmozSpider(scrapy.Spider):
+        name = "dmoz"
+        allowed_domains = ["dmoz.org"]
+        start_urls = [
+            "http://www.dmoz.org/Computers/Programming/Languages/Python/",
+        ]
+
+        def parse(self, response):
+            for href in response.css("ul.directory.dir-col > li > a::attr('href')"):
+                url = urlparse.urljoin(response.url, href.extract())
+                yield scrapy.Request(url, callback=self.parse_dir_contents)
+
+        def parse_dir_contents(self, response):
+            for sel in response.xpath('//ul/li'):
+                item = DmozItem()
+                item['title'] = sel.xpath('a/text()').extract()
+                item['link'] = sel.xpath('a/@href').extract()
+                item['desc'] = sel.xpath('text()').extract()
+                yield item
+
+Now the `parse()` method only extract the interesting links from the page, builds a
+full absolute URL (since the links can be relative) and yields new requests to
+be sent later, registering as callback the method `parse_dir_contents()` that
+will ultimately scrape the data we want.
+
+What you see here is the Scrapy's mechanism of following links: if you yield a
+Request instead of an Item in a callback method, Scrapy will schedule that
+request to be sent and register a callback method to be executed when that
+request finishes.
+
+Using this, you can build complex crawlers that follow links according to rules
+you define, and extract different kinds of data depending on the page it's
+visiting.
+
+.. note::
+    As an example spider that uses this mechanism, you can check out the
+    :class:`~scrapy.contrib.spiders.CrawlSpider` class for a generic spider
+    that implements a small rules engine that you can use to write your
+    crawlers on top of it.
+
+
 Storing the scraped data
 ========================
 

From ff007afb9df796e0df269aa760632d47b9bc037c Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 21 Apr 2015 10:57:44 -0300
Subject: [PATCH 0222/2974] expanded crawling primer with examples, and applied
 other suggestions from the review

---
 docs/intro/tutorial.rst | 52 +++++++++++++++++++++++++++++------------
 1 file changed, 37 insertions(+), 15 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 8680c9c10..13d7491aa 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -221,9 +221,10 @@ to think in XPath" <http://plasmasturm.org/log/xpath101/>`_.
   Because of this, we encourage you to learn about XPath even if you
   already know how to construct CSS selectors.
 
-For working with XPaths, Scrapy provides :class:`~scrapy.selector.Selector`
-class and convenient shortcuts to avoid instantiating selectors yourself
-every time you need to select something from a response.
+For working with CSS and XPath expressions, Scrapy provides
+:class:`~scrapy.selector.Selector` class and convenient shortcuts to avoid
+instantiating selectors yourself every time you need to select something from a
+response.
 
 You can see selectors as objects that represent nodes in the document
 structure. So, the first instantiated selectors are associated with the root
@@ -448,7 +449,6 @@ want for all of them?
 Here is a modification to our spider that does just that::
 
     import scrapy
-    import urlparse
 
     from tutorial.items import DmozItem
 
@@ -461,7 +461,7 @@ Here is a modification to our spider that does just that::
 
         def parse(self, response):
             for href in response.css("ul.directory.dir-col > li > a::attr('href')"):
-                url = urlparse.urljoin(response.url, href.extract())
+                url = response.urljoin(href.extract())
                 yield scrapy.Request(url, callback=self.parse_dir_contents)
 
         def parse_dir_contents(self, response):
@@ -472,27 +472,49 @@ Here is a modification to our spider that does just that::
                 item['desc'] = sel.xpath('text()').extract()
                 yield item
 
-Now the `parse()` method only extract the interesting links from the page, builds a
-full absolute URL (since the links can be relative) and yields new requests to
-be sent later, registering as callback the method `parse_dir_contents()` that
-will ultimately scrape the data we want.
+Now the `parse()` method only extract the interesting links from the page,
+builds a full absolute URL using the `response.urljoin` method (since the links can
+be relative) and yields new requests to be sent later, registering as callback
+the method `parse_dir_contents()` that will ultimately scrape the data we want.
 
-What you see here is the Scrapy's mechanism of following links: if you yield a
-Request instead of an Item in a callback method, Scrapy will schedule that
-request to be sent and register a callback method to be executed when that
-request finishes.
+What you see here is the Scrapy's mechanism of following links: when you yield
+a Request in a callback method, Scrapy will schedule that request to be sent
+and register a callback method to be executed when that request finishes.
 
 Using this, you can build complex crawlers that follow links according to rules
 you define, and extract different kinds of data depending on the page it's
 visiting.
 
+A common pattern is a callback method that extract some items, looks for a link
+to follow to the next page and then yields a `Request` with the same callback
+for it::
+
+    def parse_articles_follow_next_page(self, response):
+        for article in response.xpath("//article"):
+            yield {
+                ... extract article data here
+            }
+
+        next_page = response.css("ul.navigation -> li.next-page > a::attr('href')")
+        if next_page:
+            url = response.urljoin(next_page[0].extract())
+            yield Request(url, self.parse_articles_follow_next_page)
+
+This creates a sort of loop, following all the links to the next page until it
+doesn't find one -- handy for crawling blogs, forums and other sites with
+pagination.
+
+Another common pattern is to build an item with data from more than one page,
+using a `trick to pass additional data to the callbacks
+<topics-request-response-ref-request-callback-arguments>`_.
+
+
 .. note::
-    As an example spider that uses this mechanism, you can check out the
+    As an example spider that leverages this mechanism, check out the
     :class:`~scrapy.contrib.spiders.CrawlSpider` class for a generic spider
     that implements a small rules engine that you can use to write your
     crawlers on top of it.
 
-
 Storing the scraped data
 ========================
 

From f7da69d116e28752cddb65ff5aef05faeb13d4b8 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 21 Apr 2015 11:19:10 -0300
Subject: [PATCH 0223/2974] fixing example CSS expr

---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 13d7491aa..a08eb59d6 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -495,7 +495,7 @@ for it::
                 ... extract article data here
             }
 
-        next_page = response.css("ul.navigation -> li.next-page > a::attr('href')")
+        next_page = response.css("ul.navigation > li.next-page > a::attr('href')")
         if next_page:
             url = response.urljoin(next_page[0].extract())
             yield Request(url, self.parse_articles_follow_next_page)

From d63567531d36faf3718fd00fa1f754747baec387 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 21 Apr 2015 11:30:48 -0300
Subject: [PATCH 0224/2974] change data extraction in crawl example to be
 consistent with tutorial, removed statement implying mandatory usage of Item

---
 docs/intro/tutorial.rst | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index a08eb59d6..ab2302154 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -387,7 +387,7 @@ in your output. Run::
 Using our item
 --------------
 
-:class:`~scrapy.item.Item` objects are custom python dicts; you can access the
+:class:`~scrapy.item.Item` objects are custom Python dicts; you can access the
 values of their fields (attributes of the class we defined earlier) using the
 standard dict syntax like::
 
@@ -396,9 +396,8 @@ standard dict syntax like::
     >>> item['title']
     'Example title'
 
-Spiders are expected to return their scraped data inside
-:class:`~scrapy.item.Item` objects. So, in order to return the data we've
-scraped so far, the final code for our Spider would be like this::
+So, in order to return the data we've scraped so far, the final code for our
+Spider would be like this::
 
     import scrapy
 
@@ -491,9 +490,11 @@ for it::
 
     def parse_articles_follow_next_page(self, response):
         for article in response.xpath("//article"):
-            yield {
-                ... extract article data here
-            }
+            item = ArticleItem()
+
+            ... extract article data here
+
+            yield item
 
         next_page = response.css("ul.navigation > li.next-page > a::attr('href')")
         if next_page:

From c3d3a9491412d2a91b0927a05908593dcd329e4a Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Apr 2015 00:29:40 +0500
Subject: [PATCH 0225/2974] fixed backwards compatibility for
 SPIDER_MANAGER_CLASS option

---
 scrapy/crawler.py     |  4 ++--
 tests/test_crawler.py | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index dab1043ec..b4706919a 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -192,8 +192,8 @@ def _get_spider_loader(settings):
             'Please use SPIDER_LOADER_CLASS.',
             category=ScrapyDeprecationWarning, stacklevel=2
         )
-    cls_path = settings.get('SPIDER_LOADER_CLASS',
-                            settings.get('SPIDER_MANAGER_CLASS'))
+    cls_path = settings.get('SPIDER_MANAGER_CLASS',
+                            settings.get('SPIDER_LOADER_CLASS'))
     loader_cls = load_object(cls_path)
     verifyClass(ISpiderLoader, loader_cls)
     return loader_cls.from_settings(settings.frozencopy())
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 50da01e73..6a282ed84 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -5,6 +5,7 @@ from zope.interface.verify import DoesNotImplement
 
 from scrapy.crawler import Crawler, CrawlerRunner
 from scrapy.settings import Settings, default_settings
+from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
 
@@ -60,6 +61,10 @@ class SpiderLoaderWithWrongInterface(object):
         pass
 
 
+class CustomSpiderLoader(SpiderLoader):
+    pass
+
+
 class CrawlerRunnerTestCase(unittest.TestCase):
 
     def test_spider_manager_verify_interface(self):
@@ -87,3 +92,13 @@ class CrawlerRunnerTestCase(unittest.TestCase):
             self.assertIn("CrawlerRunner.spider_loader", str(w[0].message))
             sl_cls = load_object(runner.settings['SPIDER_LOADER_CLASS'])
             self.assertIsInstance(spiders, sl_cls)
+
+    def test_spidermanager_deprecation(self):
+        with warnings.catch_warnings(record=True) as w:
+            runner = CrawlerRunner({
+                'SPIDER_MANAGER_CLASS': 'tests.test_crawler.CustomSpiderLoader'
+            })
+            self.assertIsInstance(runner.spider_loader, CustomSpiderLoader)
+            self.assertEqual(len(w), 1)
+            self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message))
+

From 7871acd66553dca4d787d2d6fbff174851a10acd Mon Sep 17 00:00:00 2001
From: nramirezuy <nramirez.uy@gmail.com>
Date: Wed, 17 Jul 2013 15:22:08 -0300
Subject: [PATCH 0226/2974] Item multi inherit fixed

---
 scrapy/item.py     | 17 ++++++-----
 tests/test_item.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+), 7 deletions(-)

diff --git a/scrapy/item.py b/scrapy/item.py
index 9998010b2..8ac279642 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -24,18 +24,21 @@ class Field(dict):
 class ItemMeta(ABCMeta):
 
     def __new__(mcs, class_name, bases, attrs):
+        new_bases = tuple(base._class for base in bases if hasattr(base, '_class'))
+        _class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs)
+
         fields = {}
         new_attrs = {}
-        for n, v in six.iteritems(attrs):
+        for n in dir(_class):
+            v = getattr(_class, n)
             if isinstance(v, Field):
                 fields[n] = v
-            else:
-                new_attrs[n] = v
+            elif n in attrs:
+                new_attrs[n] = attrs[n]
 
-        cls = super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs)
-        cls.fields = cls.fields.copy()
-        cls.fields.update(fields)
-        return cls
+        new_attrs['fields'] = fields
+        new_attrs['_class'] = _class
+        return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs)
 
 
 class DictItem(MutableMapping, BaseItem):
diff --git a/tests/test_item.py b/tests/test_item.py
index 5de77d22e..76061bbd9 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -134,6 +134,76 @@ class ItemTest(unittest.TestCase):
         self.assertEqual(list(i.keys()), ['keys'])
         self.assertEqual(list(i.values()), [3])
 
+    def test_metaclass_multiple_inheritance_simple(self):
+        class A(Item):
+            save = Field(default='A')
+
+        class B(A): pass
+
+        class C(Item):
+            save = Field(default='C')
+
+        class D(B, C): pass
+
+        self.assertEqual(D(save='X')['save'], 'X')
+        self.assertEqual(D.fields, {'save': {'default': 'A'}})
+
+        # D class inverted
+        class E(C, B): pass
+
+        self.assertEqual(E(save='X')['save'], 'X')
+        self.assertEqual(E.fields, {'save': {'default': 'C'}})
+
+    def test_metaclass_multiple_inheritance_diamond(self):
+        class A(Item):
+            save = Field(default='A')
+            load = Field(default='A')
+
+        class B(A): pass
+
+        class C(A):
+            save = Field(default='C')
+
+        class D(B, C):
+            load = Field(default='D')
+
+        self.assertEqual(D(save='X')['save'], 'X')
+        self.assertEqual(D(load='X')['load'], 'X')
+        self.assertEqual(D.fields, {'save': {'default': 'C'},
+            'load': {'default': 'D'}})
+
+        # D class inverted
+        class E(C, B):
+            load = Field(default='E')
+
+        self.assertEqual(E(save='X')['save'], 'X')
+        self.assertEqual(E(load='X')['load'], 'X')
+        self.assertEqual(E.fields, {'save': {'default': 'C'},
+            'load': {'default': 'E'}})
+
+    def test_metaclass_multiple_inheritance_without_metaclass(self):
+        class A(Item):
+            save = Field(default='A')
+
+        class B(A): pass
+
+        class C(object):
+            not_allowed = Field(default='not_allowed')
+            save = Field(default='C')
+
+        class D(B, C): pass
+
+        self.assertRaises(KeyError, D, not_allowed='value')
+        self.assertEqual(D(save='X')['save'], 'X')
+        self.assertEqual(D.fields, {'save': {'default': 'A'}})
+
+        # D class inverted
+        class E(C, B): pass
+
+        self.assertRaises(KeyError, E, not_allowed='value')
+        self.assertEqual(E(save='X')['save'], 'X')
+        self.assertEqual(E.fields, {'save': {'default': 'A'}})
+
     def test_to_dict(self):
         class TestItem(Item):
             name = Field()

From 7a958f90bef3e6a1ab51bfb04260ed6186f38924 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 27 Feb 2015 23:36:30 -0300
Subject: [PATCH 0227/2974] Replace scrapy.log calls for their equivalents in
 the logging std module

Changes:
 - Each module takes 'scrapy' logger and logs through it
 - Lazy string evaluation in all log messages
 - Added missing log messages in scrapy/core/engine.py
 - Contextual data such as crawler or spider instances, and failures
---
 scrapy/commands/parse.py                      | 29 ++++---
 scrapy/commands/shell.py                      |  1 -
 scrapy/contrib/debug.py                       | 18 +++--
 .../contrib/downloadermiddleware/ajaxcrawl.py | 13 +++-
 .../contrib/downloadermiddleware/cookies.py   |  8 +-
 .../downloadermiddleware/decompression.py     | 10 ++-
 .../contrib/downloadermiddleware/redirect.py  | 14 ++--
 scrapy/contrib/downloadermiddleware/retry.py  | 14 ++--
 .../contrib/downloadermiddleware/robotstxt.py | 14 ++--
 scrapy/contrib/feedexport.py                  | 29 ++++---
 scrapy/contrib/logstats.py                    | 15 +++-
 scrapy/contrib/memusage.py                    | 14 ++--
 scrapy/contrib/pipeline/files.py              | 77 +++++++++++++------
 scrapy/contrib/pipeline/media.py              | 16 +++-
 scrapy/contrib/spidermiddleware/depth.py      | 12 ++-
 scrapy/contrib/spidermiddleware/httperror.py  | 14 ++--
 scrapy/contrib/spidermiddleware/offsite.py    |  8 +-
 scrapy/contrib/spidermiddleware/urllength.py  | 12 ++-
 scrapy/contrib/spiders/sitemap.py             |  9 ++-
 scrapy/contrib/throttle.py                    | 17 +++-
 scrapy/core/downloader/handlers/http11.py     | 28 ++++---
 scrapy/core/engine.py                         | 50 ++++++++----
 scrapy/core/scheduler.py                      | 15 ++--
 scrapy/core/scraper.py                        | 36 +++++----
 scrapy/crawler.py                             | 13 ++--
 scrapy/dupefilter.py                          | 11 +--
 scrapy/mail.py                                | 28 ++++---
 scrapy/middleware.py                          | 16 ++--
 scrapy/statscol.py                            |  8 +-
 scrapy/telnet.py                              | 10 ++-
 scrapy/utils/iterators.py                     | 15 +++-
 scrapy/utils/signal.py                        | 18 +++--
 scrapy/utils/spider.py                        | 12 +--
 tests/test_commands.py                        | 19 ++---
 34 files changed, 401 insertions(+), 222 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 3e006ede3..b28beecc0 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,5 +1,8 @@
 from __future__ import print_function
+import logging
+
 from w3lib.url import is_url
+
 from scrapy.command import ScrapyCommand
 from scrapy.http import Request
 from scrapy.item import BaseItem
@@ -7,7 +10,9 @@ from scrapy.utils import display
 from scrapy.utils.conf import arglist_to_dict
 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
 from scrapy.exceptions import UsageError
-from scrapy import log
+
+logger = logging.getLogger('scrapy')
+
 
 class Command(ScrapyCommand):
 
@@ -119,9 +124,9 @@ class Command(ScrapyCommand):
                 if rule.link_extractor.matches(response.url) and rule.callback:
                     return rule.callback
         else:
-            log.msg(format='No CrawlSpider rules found in spider %(spider)r, '
-                           'please specify a callback to use for parsing',
-                    level=log.ERROR, spider=spider.name)
+            logger.error('No CrawlSpider rules found in spider %(spider)r, '
+                         'please specify a callback to use for parsing',
+                         {'spider': spider.name})
 
     def set_spidercls(self, url, opts):
         spider_loader = self.crawler_process.spider_loader
@@ -129,13 +134,13 @@ class Command(ScrapyCommand):
             try:
                 self.spidercls = spider_loader.load(opts.spider)
             except KeyError:
-                log.msg(format='Unable to find spider: %(spider)s',
-                        level=log.ERROR, spider=opts.spider)
+                logger.error('Unable to find spider: %(spider)s',
+                             {'spider': opts.spider})
         else:
             self.spidercls = spidercls_for_request(spider_loader, Request(url))
             if not self.spidercls:
-                log.msg(format='Unable to find spider for: %(url)s',
-                        level=log.ERROR, url=url)
+                logger.error('Unable to find spider for: %(url)s',
+                             {'url': url})
 
         request = Request(url, opts.callback)
         _start_requests = lambda s: [self.prepare_request(s, request, opts)]
@@ -148,8 +153,8 @@ class Command(ScrapyCommand):
         self.crawler_process.start()
 
         if not self.first_response:
-            log.msg(format='No response downloaded for: %(url)s',
-                    level=log.ERROR, url=url)
+            logger.error('No response downloaded for: %(url)s',
+                         {'url': url})
 
     def prepare_request(self, spider, request, opts):
         def callback(response):
@@ -170,8 +175,8 @@ class Command(ScrapyCommand):
                 if callable(cb_method):
                     cb = cb_method
                 else:
-                    log.msg(format='Cannot find callback %(callback)r in spider: %(spider)s',
-                            callback=callback, spider=spider.name, level=log.ERROR)
+                    logger.error('Cannot find callback %(callback)r in spider: %(spider)s',
+                                 {'callback': callback, 'spider': spider.name})
                     return
 
             # parse items and requests
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index f8ad8a491..0b130529b 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -9,7 +9,6 @@ from threading import Thread
 from scrapy.command import ScrapyCommand
 from scrapy.shell import Shell
 from scrapy.http import Request
-from scrapy import log
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 
 
diff --git a/scrapy/contrib/debug.py b/scrapy/contrib/debug.py
index 18a746d31..f1ec67530 100644
--- a/scrapy/contrib/debug.py
+++ b/scrapy/contrib/debug.py
@@ -6,13 +6,15 @@ See documentation in docs/topics/extensions.rst
 
 import sys
 import signal
+import logging
 import traceback
 import threading
 from pdb import Pdb
 
 from scrapy.utils.engine import format_engine_status
 from scrapy.utils.trackref import format_live_refs
-from scrapy import log
+
+logger = logging.getLogger('scrapy')
 
 
 class StackTraceDump(object):
@@ -31,12 +33,14 @@ class StackTraceDump(object):
         return cls(crawler)
 
     def dump_stacktrace(self, signum, frame):
-        stackdumps = self._thread_stacks()
-        enginestatus = format_engine_status(self.crawler.engine)
-        liverefs = format_live_refs()
-        msg = "Dumping stack trace and engine status" \
-            "\n{0}\n{1}\n{2}".format(enginestatus, liverefs, stackdumps)
-        log.msg(msg)
+        log_args = {
+            'stackdumps': self._thread_stacks(),
+            'enginestatus': format_engine_status(self.crawler.engine),
+            'liverefs': format_live_refs(),
+        }
+        logger.info("Dumping stack trace and engine status\n"
+                    "%(enginestatus)s\n%(liverefs)s\n%(stackdumps)s",
+                    log_args, extra={'crawler': self.crawler})
 
     def _thread_stacks(self):
         id2name = dict((th.ident, th.name) for th in threading.enumerate())
diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
index 6c0371691..ef7f34ef9 100644
--- a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
+++ b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
@@ -1,14 +1,19 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 import re
+import logging
+
 import six
 from w3lib import html
-from scrapy import log
+
 from scrapy.exceptions import NotConfigured
 from scrapy.http import HtmlResponse
 from scrapy.utils.response import _noscript_re, _script_re
 
 
+logger = logging.getLogger('scrapy')
+
+
 class AjaxCrawlMiddleware(object):
     """
     Handle 'AJAX crawlable' pages marked as crawlable via meta tag.
@@ -46,9 +51,9 @@ class AjaxCrawlMiddleware(object):
 
         # scrapy already handles #! links properly
         ajax_crawl_request = request.replace(url=request.url+'#!')
-        log.msg(format="Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s",
-                level=log.DEBUG, spider=spider,
-                ajax_crawl_request=ajax_crawl_request, request=request)
+        logger.debug("Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s",
+                     {'ajax_crawl_request': ajax_crawl_request, 'request': request},
+                     extra={'spider': spider})
 
         ajax_crawl_request.meta['ajax_crawlable'] = True
         return ajax_crawl_request
diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/contrib/downloadermiddleware/cookies.py
index 4b63b8112..70ecc2dec 100644
--- a/scrapy/contrib/downloadermiddleware/cookies.py
+++ b/scrapy/contrib/downloadermiddleware/cookies.py
@@ -1,11 +1,13 @@
 import os
 import six
+import logging
 from collections import defaultdict
 
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
-from scrapy import log
+
+logger = logging.getLogger('scrapy')
 
 
 class CookiesMiddleware(object):
@@ -54,7 +56,7 @@ class CookiesMiddleware(object):
             if cl:
                 msg = "Sending cookies to: %s" % request + os.linesep
                 msg += os.linesep.join("Cookie: %s" % c for c in cl)
-                log.msg(msg, spider=spider, level=log.DEBUG)
+                logger.debug(msg, extra={'spider': spider})
 
     def _debug_set_cookie(self, response, spider):
         if self.debug:
@@ -62,7 +64,7 @@ class CookiesMiddleware(object):
             if cl:
                 msg = "Received cookies from: %s" % response + os.linesep
                 msg += os.linesep.join("Set-Cookie: %s" % c for c in cl)
-                log.msg(msg, spider=spider, level=log.DEBUG)
+                logger.debug(msg, extra={'spider': spider})
 
     def _format_cookie(self, cookie):
         # build cookie string
diff --git a/scrapy/contrib/downloadermiddleware/decompression.py b/scrapy/contrib/downloadermiddleware/decompression.py
index c08f50b5f..7cd506dd9 100644
--- a/scrapy/contrib/downloadermiddleware/decompression.py
+++ b/scrapy/contrib/downloadermiddleware/decompression.py
@@ -1,11 +1,12 @@
 """ This module implements the DecompressionMiddleware which tries to recognise
-and extract the potentially compressed responses that may arrive. 
+and extract the potentially compressed responses that may arrive.
 """
 
 import bz2
 import gzip
 import zipfile
 import tarfile
+import logging
 from tempfile import mktemp
 
 import six
@@ -15,9 +16,10 @@ try:
 except ImportError:
     from io import BytesIO
 
-from scrapy import log
 from scrapy.responsetypes import responsetypes
 
+logger = logging.getLogger('scrapy')
+
 
 class DecompressionMiddleware(object):
     """ This middleware tries to recognise and extract the possibly compressed
@@ -80,7 +82,7 @@ class DecompressionMiddleware(object):
         for fmt, func in six.iteritems(self._formats):
             new_response = func(response)
             if new_response:
-                log.msg(format='Decompressed response with format: %(responsefmt)s',
-                        level=log.DEBUG, spider=spider, responsefmt=fmt)
+                logger.debug('Decompressed response with format: %(responsefmt)s',
+                             {'responsefmt': fmt}, extra={'spider': spider})
                 return new_response
         return response
diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py
index cfb10d4db..68d139bc7 100644
--- a/scrapy/contrib/downloadermiddleware/redirect.py
+++ b/scrapy/contrib/downloadermiddleware/redirect.py
@@ -1,10 +1,12 @@
+import logging
 from six.moves.urllib.parse import urljoin
 
-from scrapy import log
 from scrapy.http import HtmlResponse
 from scrapy.utils.response import get_meta_refresh
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 
+logger = logging.getLogger('scrapy')
+
 
 class BaseRedirectMiddleware(object):
 
@@ -32,13 +34,13 @@ class BaseRedirectMiddleware(object):
                 [request.url]
             redirected.dont_filter = request.dont_filter
             redirected.priority = request.priority + self.priority_adjust
-            log.msg(format="Redirecting (%(reason)s) to %(redirected)s from %(request)s",
-                    level=log.DEBUG, spider=spider, request=request,
-                    redirected=redirected, reason=reason)
+            logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",
+                         {'reason': reason, 'redirected': redirected, 'request': request},
+                         extra={'spider': spider})
             return redirected
         else:
-            log.msg(format="Discarding %(request)s: max redirections reached",
-                    level=log.DEBUG, spider=spider, request=request)
+            logger.debug("Discarding %(request)s: max redirections reached",
+                         {'request': request}, extra={'spider': spider})
             raise IgnoreRequest("max redirections reached")
 
     def _redirect_request_using_get(self, request, redirect_url):
diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py
index f72f39431..749b334f1 100644
--- a/scrapy/contrib/downloadermiddleware/retry.py
+++ b/scrapy/contrib/downloadermiddleware/retry.py
@@ -17,17 +17,19 @@ About HTTP errors to consider:
   protocol. It's included by default because it's a common code used to
   indicate server overload, which would be something we want to retry
 """
+import logging
 
 from twisted.internet import defer
 from twisted.internet.error import TimeoutError, DNSLookupError, \
         ConnectionRefusedError, ConnectionDone, ConnectError, \
         ConnectionLost, TCPTimedOutError
 
-from scrapy import log
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.response import response_status_message
 from scrapy.xlib.tx import ResponseFailed
 
+logger = logging.getLogger('scrapy')
+
 
 class RetryMiddleware(object):
 
@@ -66,13 +68,15 @@ class RetryMiddleware(object):
         retries = request.meta.get('retry_times', 0) + 1
 
         if retries <= self.max_retry_times:
-            log.msg(format="Retrying %(request)s (failed %(retries)d times): %(reason)s",
-                    level=log.DEBUG, spider=spider, request=request, retries=retries, reason=reason)
+            logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",
+                         {'request': request, 'retries': retries, 'reason': reason},
+                         extra={'spider': spider})
             retryreq = request.copy()
             retryreq.meta['retry_times'] = retries
             retryreq.dont_filter = True
             retryreq.priority = request.priority + self.priority_adjust
             return retryreq
         else:
-            log.msg(format="Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
-                    level=log.DEBUG, spider=spider, request=request, retries=retries, reason=reason)
+            logger.debug("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
+                         {'request': request, 'retries': retries, 'reason': reason},
+                         extra={'spider': spider})
diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py
index a58ecca8e..12ab2dd07 100644
--- a/scrapy/contrib/downloadermiddleware/robotstxt.py
+++ b/scrapy/contrib/downloadermiddleware/robotstxt.py
@@ -4,13 +4,16 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
 
 """
 
+import logging
+
 from six.moves.urllib import robotparser
 
-from scrapy import signals, log
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 
+logger = logging.getLogger('scrapy')
+
 
 class RobotsTxtMiddleware(object):
     DOWNLOAD_PRIORITY = 1000
@@ -32,8 +35,8 @@ class RobotsTxtMiddleware(object):
             return
         rp = self.robot_parser(request, spider)
         if rp and not rp.can_fetch(self._useragent, request.url):
-            log.msg(format="Forbidden by robots.txt: %(request)s",
-                    level=log.DEBUG, request=request)
+            logger.debug("Forbidden by robots.txt: %(request)s",
+                         {'request': request}, extra={'spider': spider})
             raise IgnoreRequest
 
     def robot_parser(self, request, spider):
@@ -54,8 +57,9 @@ class RobotsTxtMiddleware(object):
 
     def _logerror(self, failure, request, spider):
         if failure.type is not IgnoreRequest:
-            log.msg(format="Error downloading %%(request)s: %s" % failure.value,
-                    level=log.ERROR, request=request, spider=spider)
+            logger.error("Error downloading %(request)s: %(f_exception)s",
+                         {'request': request, 'f_exception': failure.value},
+                         extra={'spider': spider, 'failure': failure})
 
     def _parse_robots(self, response):
         rp = robotparser.RobotFileParser(response.url)
diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py
index a8404146b..7162fbc10 100644
--- a/scrapy/contrib/feedexport.py
+++ b/scrapy/contrib/feedexport.py
@@ -4,7 +4,10 @@ Feed Exports extension
 See documentation in docs/topics/feed-exports.rst
 """
 
-import sys, os, posixpath
+import os
+import sys
+import logging
+import posixpath
 from tempfile import TemporaryFile
 from datetime import datetime
 from six.moves.urllib.parse import urlparse
@@ -14,12 +17,14 @@ from zope.interface import Interface, implementer
 from twisted.internet import defer, threads
 from w3lib.url import file_uri_to_path
 
-from scrapy import log, signals
+from scrapy import signals
 from scrapy.utils.ftp import ftp_makedirs_cwd
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import get_func_args
 
+logger = logging.getLogger('scrapy')
+
 
 class IFeedStorage(Interface):
     """Interface that all Feed Storages must implement"""
@@ -171,11 +176,15 @@ class FeedExporter(object):
         if not slot.itemcount and not self.store_empty:
             return
         slot.exporter.finish_exporting()
-        logfmt = "%%s %s feed (%d items) in: %s" % (self.format, \
-            slot.itemcount, slot.uri)
+        logfmt = "%%s %(format)s feed (%(itemcount)d items) in: %(uri)s"
+        log_args = {'format': self.format,
+                    'itemcount': slot.itemcount,
+                    'uri': slot.uri}
         d = defer.maybeDeferred(slot.storage.store, slot.file)
-        d.addCallback(lambda _: log.msg(logfmt % "Stored", spider=spider))
-        d.addErrback(log.err, logfmt % "Error storing", spider=spider)
+        d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
+                                            extra={'spider': spider}))
+        d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
+                                            extra={'spider': spider, 'failure': f}))
         return d
 
     def item_scraped(self, item, spider):
@@ -198,7 +207,7 @@ class FeedExporter(object):
     def _exporter_supported(self, format):
         if format in self.exporters:
             return True
-        log.msg("Unknown feed format: %s" % format, log.ERROR)
+        logger.error("Unknown feed format: %(format)s", {'format': format})
 
     def _storage_supported(self, uri):
         scheme = urlparse(uri).scheme
@@ -207,9 +216,11 @@ class FeedExporter(object):
                 self._get_storage(uri)
                 return True
             except NotConfigured:
-                log.msg("Disabled feed storage scheme: %s" % scheme, log.ERROR)
+                logger.error("Disabled feed storage scheme: %(scheme)s",
+                             {'scheme': scheme})
         else:
-            log.msg("Unknown feed storage scheme: %s" % scheme, log.ERROR)
+            logger.error("Unknown feed storage scheme: %(scheme)s",
+                         {'scheme': scheme})
 
     def _get_exporter(self, *args, **kwargs):
         return self.exporters[self.format](*args, **kwargs)
diff --git a/scrapy/contrib/logstats.py b/scrapy/contrib/logstats.py
index 4f2567c3f..3ea347e8d 100644
--- a/scrapy/contrib/logstats.py
+++ b/scrapy/contrib/logstats.py
@@ -1,7 +1,11 @@
+import logging
+
 from twisted.internet import task
 
 from scrapy.exceptions import NotConfigured
-from scrapy import log, signals
+from scrapy import signals
+
+logger = logging.getLogger('scrapy')
 
 
 class LogStats(object):
@@ -35,9 +39,12 @@ class LogStats(object):
         irate = (items - self.itemsprev) * self.multiplier
         prate = (pages - self.pagesprev) * self.multiplier
         self.pagesprev, self.itemsprev = pages, items
-        msg = "Crawled %d pages (at %d pages/min), scraped %d items (at %d items/min)" \
-            % (pages, prate, items, irate)
-        log.msg(msg, spider=spider)
+
+        msg = ("Crawled %(pages)d pages (at %(pagerate)d pages/min), "
+               "scraped %(items)d items (at %(itemrate)d items/min)")
+        log_args = {'pages': pages, 'pagerate': prate,
+                    'items': items, 'itemrate': irate}
+        logger.info(msg, log_args, extra={'spider': spider})
 
     def spider_closed(self, spider, reason):
         if self.task.running:
diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py
index 6bcba8e11..d1e13bfe5 100644
--- a/scrapy/contrib/memusage.py
+++ b/scrapy/contrib/memusage.py
@@ -5,16 +5,20 @@ See documentation in docs/topics/extensions.rst
 """
 import sys
 import socket
+import logging
 from pprint import pformat
 from importlib import import_module
 
 from twisted.internet import task
 
-from scrapy import signals, log
+from scrapy import signals
 from scrapy.exceptions import NotConfigured
 from scrapy.mail import MailSender
 from scrapy.utils.engine import get_engine_status
 
+logger = logging.getLogger('scrapy')
+
+
 class MemoryUsage(object):
 
     def __init__(self, crawler):
@@ -74,8 +78,8 @@ class MemoryUsage(object):
         if self.get_virtual_size() > self.limit:
             self.crawler.stats.set_value('memusage/limit_reached', 1)
             mem = self.limit/1024/1024
-            log.msg(format="Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
-                    level=log.ERROR, memusage=mem)
+            logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
+                         {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
                 subj = "%s terminated: memory usage exceeded %dM at %s" % \
                         (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
@@ -95,8 +99,8 @@ class MemoryUsage(object):
         if self.get_virtual_size() > self.warning:
             self.crawler.stats.set_value('memusage/warning_reached', 1)
             mem = self.warning/1024/1024
-            log.msg(format="Memory usage reached %(memusage)dM",
-                    level=log.WARNING, memusage=mem)
+            logger.warning("Memory usage reached %(memusage)dM",
+                           {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
                 subj = "%s warning: memory usage reached %dM at %s" % \
                         (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py
index 608614865..daedac3f7 100644
--- a/scrapy/contrib/pipeline/files.py
+++ b/scrapy/contrib/pipeline/files.py
@@ -9,6 +9,7 @@ import os
 import os.path
 import rfc822
 import time
+import logging
 from six.moves.urllib.parse import urlparse
 from collections import defaultdict
 import six
@@ -20,12 +21,13 @@ except ImportError:
 
 from twisted.internet import defer, threads
 
-from scrapy import log
 from scrapy.contrib.pipeline.media import MediaPipeline
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.misc import md5sum
 
+logger = logging.getLogger('scrapy')
+
 
 class FileException(Exception):
     """General media error exception"""
@@ -192,9 +194,13 @@ class FilesPipeline(MediaPipeline):
                 return  # returning None force download
 
             referer = request.headers.get('Referer')
-            log.msg(format='File (uptodate): Downloaded %(medianame)s from %(request)s referred in <%(referer)s>',
-                    level=log.DEBUG, spider=info.spider,
-                    medianame=self.MEDIA_NAME, request=request, referer=referer)
+            logger.debug(
+                'File (uptodate): Downloaded %(medianame)s from %(request)s '
+                'referred in <%(referer)s>',
+                {'medianame': self.MEDIA_NAME, 'request': request,
+                 'referer': referer},
+                extra={'spider': info.spider}
+            )
             self.inc_stats(info.spider, 'uptodate')
 
             checksum = result.get('checksum', None)
@@ -203,17 +209,23 @@ class FilesPipeline(MediaPipeline):
         path = self.file_path(request, info=info)
         dfd = defer.maybeDeferred(self.store.stat_file, path, info)
         dfd.addCallbacks(_onsuccess, lambda _: None)
-        dfd.addErrback(log.err, self.__class__.__name__ + '.store.stat_file')
+        dfd.addErrback(
+            lambda f:
+            logger.error(self.__class__.__name__ + '.store.stat_file',
+                         extra={'spider': info.spider, 'failure': f})
+        )
         return dfd
 
     def media_failed(self, failure, request, info):
         if not isinstance(failure.value, IgnoreRequest):
             referer = request.headers.get('Referer')
-            log.msg(format='File (unknown-error): Error downloading '
-                           '%(medianame)s from %(request)s referred in '
-                           '<%(referer)s>: %(exception)s',
-                    level=log.WARNING, spider=info.spider, exception=failure.value,
-                    medianame=self.MEDIA_NAME, request=request, referer=referer)
+            logger.warning(
+                'File (unknown-error): Error downloading %(medianame)s from '
+                '%(request)s referred in <%(referer)s>: %(exception)s',
+                {'medianame': self.MEDIA_NAME, 'request': request,
+                 'referer': referer, 'exception': failure.value},
+                extra={'spider': info.spider}
+            )
 
         raise FileException
 
@@ -221,34 +233,51 @@ class FilesPipeline(MediaPipeline):
         referer = request.headers.get('Referer')
 
         if response.status != 200:
-            log.msg(format='File (code: %(status)s): Error downloading file from %(request)s referred in <%(referer)s>',
-                    level=log.WARNING, spider=info.spider,
-                    status=response.status, request=request, referer=referer)
+            logger.warning(
+                'File (code: %(status)s): Error downloading file from '
+                '%(request)s referred in <%(referer)s>',
+                {'status': response.status,
+                 'request': request, 'referer': referer},
+                extra={'spider': info.spider}
+            )
             raise FileException('download-error')
 
         if not response.body:
-            log.msg(format='File (empty-content): Empty file from %(request)s referred in <%(referer)s>: no-content',
-                    level=log.WARNING, spider=info.spider,
-                    request=request, referer=referer)
+            logger.warning(
+                'File (empty-content): Empty file from %(request)s referred '
+                'in <%(referer)s>: no-content',
+                {'request': request, 'referer': referer},
+                extra={'spider': info.spider}
+            )
             raise FileException('empty-content')
 
         status = 'cached' if 'cached' in response.flags else 'downloaded'
-        log.msg(format='File (%(status)s): Downloaded file from %(request)s referred in <%(referer)s>',
-                level=log.DEBUG, spider=info.spider,
-                status=status, request=request, referer=referer)
+        logger.debug(
+            'File (%(status)s): Downloaded file from %(request)s referred in '
+            '<%(referer)s>',
+            {'status': status, 'request': request, 'referer': referer},
+            extra={'spider': info.spider}
+        )
         self.inc_stats(info.spider, status)
 
         try:
             path = self.file_path(request, response=response, info=info)
             checksum = self.file_downloaded(response, request, info)
         except FileException as exc:
-            whyfmt = 'File (error): Error processing file from %(request)s referred in <%(referer)s>: %(errormsg)s'
-            log.msg(format=whyfmt, level=log.WARNING, spider=info.spider,
-                    request=request, referer=referer, errormsg=str(exc))
+            logger.warning(
+                'File (error): Error processing file from %(request)s '
+                'referred in <%(referer)s>: %(errormsg)s',
+                {'request': request, 'referer': referer, 'errormsg': str(exc)},
+                extra={'spider': info.spider}, exc_info=True
+            )
             raise
         except Exception as exc:
-            whyfmt = 'File (unknown-error): Error processing file from %(request)s referred in <%(referer)s>'
-            log.err(None, whyfmt % {'request': request, 'referer': referer}, spider=info.spider)
+            logger.exception(
+                'File (unknown-error): Error processing file from %(request)s '
+                'referred in <%(referer)s>',
+                {'request': request, 'referer': referer},
+                extra={'spider': info.spider}
+            )
             raise FileException(str(exc))
 
         return {'url': request.url, 'path': path, 'checksum': checksum}
diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py
index 012b7979a..2995dded6 100644
--- a/scrapy/contrib/pipeline/media.py
+++ b/scrapy/contrib/pipeline/media.py
@@ -1,13 +1,16 @@
 from __future__ import print_function
+
+import logging
 from collections import defaultdict
 from twisted.internet.defer import Deferred, DeferredList
 from twisted.python.failure import Failure
 
 from scrapy.utils.defer import mustbe_deferred, defer_result
-from scrapy import log
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import arg_to_iter
 
+logger = logging.getLogger('scrapy')
+
 
 class MediaPipeline(object):
 
@@ -66,7 +69,9 @@ class MediaPipeline(object):
         dfd = mustbe_deferred(self.media_to_download, request, info)
         dfd.addCallback(self._check_media_to_download, request, info)
         dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
-        dfd.addErrback(log.err, spider=info.spider)
+        dfd.addErrback(lambda f: logger.error(
+            f.value, extra={'spider': info.spider, 'failure': f})
+        )
         return dfd.addBoth(lambda _: wad)  # it must return wad at last
 
     def _check_media_to_download(self, result, request, info):
@@ -117,8 +122,11 @@ class MediaPipeline(object):
     def item_completed(self, results, item, info):
         """Called per item when all media requests has been processed"""
         if self.LOG_FAILED_RESULTS:
-            msg = '%s found errors processing %s' % (self.__class__.__name__, item)
             for ok, value in results:
                 if not ok:
-                    log.err(value, msg, spider=info.spider)
+                    logger.error(
+                        '%(class)s found errors processing %(item)s',
+                        {'class': self.__class__.__name__, 'item': item},
+                        extra={'spider': info.spider, 'failure': value}
+                    )
         return item
diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py
index 5ccfc86ed..6aeb5e053 100644
--- a/scrapy/contrib/spidermiddleware/depth.py
+++ b/scrapy/contrib/spidermiddleware/depth.py
@@ -4,9 +4,13 @@ Depth Spider Middleware
 See documentation in docs/topics/spider-middleware.rst
 """
 
-from scrapy import log
+import logging
+
 from scrapy.http import Request
 
+logger = logging.getLogger('scrapy')
+
+
 class DepthMiddleware(object):
 
     def __init__(self, maxdepth, stats=None, verbose_stats=False, prio=1):
@@ -31,9 +35,9 @@ class DepthMiddleware(object):
                 if self.prio:
                     request.priority -= depth * self.prio
                 if self.maxdepth and depth > self.maxdepth:
-                    log.msg(format="Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
-                            level=log.DEBUG, spider=spider,
-                            maxdepth=self.maxdepth, requrl=request.url)
+                    logger.debug("Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
+                                 {'maxdepth': self.maxdepth, 'requrl': request.url},
+                                 extra={'spider': spider})
                     return False
                 elif self.stats:
                     if self.verbose_stats:
diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py
index 7fb7aa97c..1962eaf6c 100644
--- a/scrapy/contrib/spidermiddleware/httperror.py
+++ b/scrapy/contrib/spidermiddleware/httperror.py
@@ -3,8 +3,12 @@ HttpError Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
+import logging
+
 from scrapy.exceptions import IgnoreRequest
-from scrapy import log
+
+logger = logging.getLogger('scrapy')
+
 
 class HttpError(IgnoreRequest):
     """A non-200 response was filtered"""
@@ -42,10 +46,8 @@ class HttpErrorMiddleware(object):
 
     def process_spider_exception(self, response, exception, spider):
         if isinstance(exception, HttpError):
-            log.msg(
-                format="Ignoring response %(response)r: HTTP status code is not handled or not allowed",
-                level=log.DEBUG,
-                spider=spider,
-                response=response
+            logger.debug(
+                "Ignoring response %(response)r: HTTP status code is not handled or not allowed",
+                {'response': response}, extra={'spider': spider},
             )
             return []
diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py
index 136714508..fb69a4631 100644
--- a/scrapy/contrib/spidermiddleware/offsite.py
+++ b/scrapy/contrib/spidermiddleware/offsite.py
@@ -5,11 +5,13 @@ See documentation in docs/topics/spider-middleware.rst
 """
 
 import re
+import logging
 
 from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy import log
+
+logger = logging.getLogger('scrapy')
 
 class OffsiteMiddleware(object):
 
@@ -31,8 +33,8 @@ class OffsiteMiddleware(object):
                     domain = urlparse_cached(x).hostname
                     if domain and domain not in self.domains_seen:
                         self.domains_seen.add(domain)
-                        log.msg(format="Filtered offsite request to %(domain)r: %(request)s",
-                                level=log.DEBUG, spider=spider, domain=domain, request=x)
+                        logger.debug("Filtered offsite request to %(domain)r: %(request)s",
+                                     {'domain': domain, 'request': x}, extra={'spider': spider})
                         self.stats.inc_value('offsite/domains', spider=spider)
                     self.stats.inc_value('offsite/filtered', spider=spider)
             else:
diff --git a/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/contrib/spidermiddleware/urllength.py
index fa6f2c909..d3c716063 100644
--- a/scrapy/contrib/spidermiddleware/urllength.py
+++ b/scrapy/contrib/spidermiddleware/urllength.py
@@ -4,10 +4,14 @@ Url Length Spider Middleware
 See documentation in docs/topics/spider-middleware.rst
 """
 
-from scrapy import log
+import logging
+
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
 
+logger = logging.getLogger('scrapy')
+
+
 class UrlLengthMiddleware(object):
 
     def __init__(self, maxlength):
@@ -23,9 +27,9 @@ class UrlLengthMiddleware(object):
     def process_spider_output(self, response, result, spider):
         def _filter(request):
             if isinstance(request, Request) and len(request.url) > self.maxlength:
-                log.msg(format="Ignoring link (url length > %(maxlength)d): %(url)s ",
-                        level=log.DEBUG, spider=spider,
-                        maxlength=self.maxlength, url=request.url)
+                logger.debug("Ignoring link (url length > %(maxlength)d): %(url)s ",
+                             {'maxlength': self.maxlength, 'url': request.url},
+                             extra={'spider': spider})
                 return False
             else:
                 return True
diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py
index 84ae04d08..845e2bc18 100644
--- a/scrapy/contrib/spiders/sitemap.py
+++ b/scrapy/contrib/spiders/sitemap.py
@@ -1,10 +1,13 @@
 import re
+import logging
 
 from scrapy.spider import Spider
 from scrapy.http import Request, XmlResponse
 from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
 from scrapy.utils.gz import gunzip, is_gzipped
-from scrapy import log
+
+logger = logging.getLogger('scrapy')
+
 
 class SitemapSpider(Spider):
 
@@ -32,8 +35,8 @@ class SitemapSpider(Spider):
         else:
             body = self._get_sitemap_body(response)
             if body is None:
-                log.msg(format="Ignoring invalid sitemap: %(response)s",
-                        level=log.WARNING, spider=self, response=response)
+                logger.warning("Ignoring invalid sitemap: %(response)s",
+                               {'response': response}, extra={'spider': self})
                 return
 
             s = Sitemap(body)
diff --git a/scrapy/contrib/throttle.py b/scrapy/contrib/throttle.py
index a5601bcd0..5f72c81fc 100644
--- a/scrapy/contrib/throttle.py
+++ b/scrapy/contrib/throttle.py
@@ -1,7 +1,10 @@
 import logging
+
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
 
+logger = logging.getLogger('scrapy')
+
 
 class AutoThrottle(object):
 
@@ -47,9 +50,17 @@ class AutoThrottle(object):
             diff = slot.delay - olddelay
             size = len(response.body)
             conc = len(slot.transferring)
-            msg = "slot: %s | conc:%2d | delay:%5d ms (%+d) | latency:%5d ms | size:%6d bytes" % \
-                  (key, conc, slot.delay * 1000, diff * 1000, latency * 1000, size)
-            spider.log(msg, level=logging.INFO)
+            logger.info(
+                "slot: %(slot)s | conc:%(concurrency)2d | "
+                "delay:%(delay)5d ms (%(delaydiff)+d) | "
+                "latency:%(latency)5d ms | size:%(size)6d bytes",
+                {
+                    'slot': key, 'concurrency': conc,
+                    'delay': slot.delay * 1000, 'delaydiff': diff * 1000,
+                    'latency': latency * 1000, 'size': size
+                },
+                extra={'spider': spider}
+            )
 
     def _get_slot(self, request, spider):
         key = request.meta.get('download_slot')
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 634c6398b..11fbd35b9 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -1,7 +1,7 @@
 """Download handlers for http and https schemes"""
 
 import re
-
+import logging
 from io import BytesIO
 from time import time
 from six.moves.urllib.parse import urldefrag
@@ -19,7 +19,9 @@ from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.core.downloader.webclient import _parse
 from scrapy.utils.misc import load_object
-from scrapy import log, twisted_version
+from scrapy import twisted_version
+
+logger = logging.getLogger('scrapy')
 
 
 class HTTP11DownloadHandler(object):
@@ -237,14 +239,16 @@ class ScrapyAgent(object):
         expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1
 
         if maxsize and expected_size > maxsize:
-            log.msg("Expected response size (%s) larger than download max size (%s)." % (expected_size, maxsize),
-                    logLevel=log.ERROR)
+            logger.error("Expected response size (%(size)s) larger than "
+                         "download max size (%(maxsize)s).",
+                         {'size': expected_size, 'maxsize': maxsize})
             txresponse._transport._producer.loseConnection()
             raise defer.CancelledError()
 
         if warnsize and expected_size > warnsize:
-            log.msg("Expected response size (%s) larger than downlod warn size (%s)." % (expected_size, warnsize),
-                    logLevel=log.WARNING)
+            logger.warning("Expected response size (%(size)s) larger than "
+                           "download warn size (%(warnsize)s).",
+                           {'size': expected_size, 'warnsize': warnsize})
 
         def _cancel(_):
             txresponse._transport._producer.loseConnection()
@@ -295,13 +299,17 @@ class _ResponseReader(protocol.Protocol):
         self._bytes_received += len(bodyBytes)
 
         if self._maxsize and self._bytes_received > self._maxsize:
-            log.msg("Received (%s) bytes larger than download max size (%s)." % (self._bytes_received, self._maxsize),
-                    logLevel=log.ERROR)
+            logger.error("Received (%(bytes)s) bytes larger than download "
+                         "max size (%(maxsize)s).",
+                         {'bytes': self._bytes_received,
+                          'maxsize': self._maxsize})
             self._finished.cancel()
 
         if self._warnsize and self._bytes_received > self._warnsize:
-            log.msg("Received (%s) bytes larger than download warn size (%s)." % (self._bytes_received, self._warnsize),
-                    logLevel=log.WARNING)
+            logger.warning("Received (%(bytes)s) bytes larger than download "
+                           "warn size (%(warnsize)s).",
+                           {'bytes': self._bytes_received,
+                            'warnsize': self._warnsize})
 
     def connectionLost(self, reason):
         if self._finished.called:
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index b009898a3..7e330af1c 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -4,6 +4,7 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spiders.
 For more information see docs/topics/architecture.rst
 
 """
+import logging
 from time import time
 
 from twisted.internet import defer
@@ -16,6 +17,8 @@ from scrapy.http import Response, Request
 from scrapy.utils.misc import load_object
 from scrapy.utils.reactor import CallLaterOnce
 
+logger = logging.getLogger('scrapy')
+
 
 class Slot(object):
 
@@ -106,10 +109,10 @@ class ExecutionEngine(object):
                 request = next(slot.start_requests)
             except StopIteration:
                 slot.start_requests = None
-            except Exception as exc:
+            except Exception:
                 slot.start_requests = None
-                log.err(None, 'Obtaining request from start requests', \
-                        spider=spider)
+                logger.exception('Error while obtaining start requests',
+                                 extra={'spider': spider})
             else:
                 self.crawl(request, spider)
 
@@ -130,11 +133,14 @@ class ExecutionEngine(object):
             return
         d = self._download(request, spider)
         d.addBoth(self._handle_downloader_output, request, spider)
-        d.addErrback(log.msg, spider=spider)
+        d.addErrback(lambda f: logger.info('Error while handling downloader output',
+                                           extra={'spider': spider, 'failure': f}))
         d.addBoth(lambda _: slot.remove_request(request))
-        d.addErrback(log.msg, spider=spider)
+        d.addErrback(lambda f: logger.info('Error while removing request from slot',
+                                           extra={'spider': spider, 'failure': f}))
         d.addBoth(lambda _: slot.nextcall.schedule())
-        d.addErrback(log.msg, spider=spider)
+        d.addErrback(lambda f: logger.info('Error while scheduling new request',
+                                           extra={'spider': spider, 'failure': f}))
         return d
 
     def _handle_downloader_output(self, response, request, spider):
@@ -145,7 +151,8 @@ class ExecutionEngine(object):
             return
         # response is a Response or Failure
         d = self.scraper.enqueue_scrape(response, request, spider)
-        d.addErrback(log.err, spider=spider)
+        d.addErrback(lambda f: logger.error('Error while enqueuing downloader output',
+                                            extra={'spider': spider, 'failure': f}))
         return d
 
     def spider_is_idle(self, spider):
@@ -215,7 +222,7 @@ class ExecutionEngine(object):
     def open_spider(self, spider, start_requests=(), close_if_idle=True):
         assert self.has_capacity(), "No free spider slot when opening %r" % \
             spider.name
-        log.msg("Spider opened", spider=spider)
+        logger.info("Spider opened", extra={'spider': spider})
         nextcall = CallLaterOnce(self._next_request, spider)
         scheduler = self.scheduler_cls.from_crawler(self.crawler)
         start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider)
@@ -252,33 +259,42 @@ class ExecutionEngine(object):
         slot = self.slot
         if slot.closing:
             return slot.closing
-        log.msg(format="Closing spider (%(reason)s)", reason=reason, spider=spider)
+        logger.info("Closing spider (%(reason)s)",
+                    {'reason': reason},
+                    extra={'spider': spider})
 
         dfd = slot.close()
 
+        def log_failure(msg):
+            def errback(failure):
+                logger.error(msg, extra={'spider': spider, 'failure': failure})
+            return errback
+
         dfd.addBoth(lambda _: self.downloader.close())
-        dfd.addErrback(log.err, spider=spider)
+        dfd.addErrback(log_failure('Downloader close failure'))
 
         dfd.addBoth(lambda _: self.scraper.close_spider(spider))
-        dfd.addErrback(log.err, spider=spider)
+        dfd.addErrback(log_failure('Scraper close failure'))
 
         dfd.addBoth(lambda _: slot.scheduler.close(reason))
-        dfd.addErrback(log.err, spider=spider)
+        dfd.addErrback(log_failure('Scheduler close failure'))
 
         dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(
             signal=signals.spider_closed, spider=spider, reason=reason))
-        dfd.addErrback(log.err, spider=spider)
+        dfd.addErrback(log_failure('Error while sending spider_close signal'))
 
         dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason))
-        dfd.addErrback(log.err, spider=spider)
+        dfd.addErrback(log_failure('Stats close failure'))
 
-        dfd.addBoth(lambda _: log.msg(format="Spider closed (%(reason)s)", reason=reason, spider=spider))
+        dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)",
+                                          {'reason': reason},
+                                          extra={'spider': spider}))
 
         dfd.addBoth(lambda _: setattr(self, 'slot', None))
-        dfd.addErrback(log.err, spider=spider)
+        dfd.addErrback(log_failure('Error while unassigning slot'))
 
         dfd.addBoth(lambda _: setattr(self, 'spider', None))
-        dfd.addErrback(log.err, spider=spider)
+        dfd.addErrback(log_failure('Error while unassigning spider'))
 
         dfd.addBoth(lambda _: self._spider_closed_callback(spider))
 
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 232bc6a40..0e1acacea 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -1,12 +1,15 @@
 import os
 import json
+import logging
 from os.path import join, exists
 
 from queuelib import PriorityQueue
 from scrapy.utils.reqser import request_to_dict, request_from_dict
 from scrapy.utils.misc import load_object
 from scrapy.utils.job import job_dir
-from scrapy import log
+
+logger = logging.getLogger('scrapy')
+
 
 class Scheduler(object):
 
@@ -80,9 +83,9 @@ class Scheduler(object):
             self.dqs.push(reqd, -request.priority)
         except ValueError as e: # non serializable request
             if self.logunser:
-                log.msg(format="Unable to serialize request: %(request)s - reason: %(reason)s",
-                        level=log.ERROR, spider=self.spider,
-                        request=request, reason=e)
+                logger.exception("Unable to serialize request: %(request)s - reason: %(reason)s",
+                                 {'request': request, 'reason': e},
+                                 extra={'spider': self.spider})
             return
         else:
             return True
@@ -111,8 +114,8 @@ class Scheduler(object):
             prios = ()
         q = PriorityQueue(self._newdq, startprios=prios)
         if q:
-            log.msg(format="Resuming crawl (%(queuesize)d requests scheduled)",
-                    spider=self.spider, queuesize=len(q))
+            logger.info("Resuming crawl (%(queuesize)d requests scheduled)",
+                        {'queuesize': len(q)}, extra={'spider': self.spider})
         return q
 
     def _dqdir(self, jobdir):
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index b301aa962..4a961f8e8 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -1,6 +1,7 @@
 """This module implements the Scraper component which parses responses and
 extracts information from them"""
 
+import logging
 from collections import deque
 
 from twisted.python.failure import Failure
@@ -16,6 +17,8 @@ from scrapy.item import BaseItem
 from scrapy.core.spidermw import SpiderMiddlewareManager
 from scrapy import log
 
+logger = logging.getLogger('scrapy')
+
 
 class Slot(object):
     """Scraper slot (one per running spider)"""
@@ -102,7 +105,9 @@ class Scraper(object):
             return _
         dfd.addBoth(finish_scraping)
         dfd.addErrback(
-            log.err, 'Scraper bug processing %s' % request, spider=spider)
+            lambda f: logger.error('Scraper bug processing %(request)s',
+                                   {'request': request},
+                                   extra={'spider': spider, 'failure': f}))
         self._scrape_next(spider, slot)
         return dfd
 
@@ -145,10 +150,10 @@ class Scraper(object):
             self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
             return
         referer = request.headers.get('Referer')
-        log.err(
-            _failure,
-            "Spider error processing %s (referer: %s)" % (request, referer),
-            spider=spider
+        logger.error(
+            "Spider error processing %(request)s (referer: %(referer)s)",
+            {'request': request, 'referer': referer},
+            extra={'spider': spider, 'failure': _failure}
         )
         self.signals.send_catch_log(
             signal=signals.spider_error,
@@ -183,9 +188,10 @@ class Scraper(object):
             pass
         else:
             typename = type(output).__name__
-            log.msg(format='Spider must return Request, BaseItem, dict or None, '
-                           'got %(typename)r in %(request)s',
-                    level=log.ERROR, spider=spider, request=request, typename=typename)
+            logger.error('Spider must return Request, BaseItem, dict or None, '
+                         'got %(typename)r in %(request)s',
+                         {'request': request, 'typename': typename},
+                         extra={'spider': spider})
 
     def _log_download_errors(self, spider_failure, download_failure, request, spider):
         """Log and silence errors that come from the engine (typically download
@@ -194,14 +200,15 @@ class Scraper(object):
         if (isinstance(download_failure, Failure) and
                 not download_failure.check(IgnoreRequest)):
             if download_failure.frames:
-                log.err(download_failure, 'Error downloading %s' % request,
-                        spider=spider)
+                logger.error('Error downloading %(request)s',
+                             {'request': request},
+                             extra={'spider': spider, 'failure': download_failure})
             else:
                 errmsg = download_failure.getErrorMessage()
                 if errmsg:
-                    log.msg(format='Error downloading %(request)s: %(errmsg)s',
-                            level=log.ERROR, spider=spider, request=request,
-                            errmsg=errmsg)
+                    logger.error('Error downloading %(request)s: %(errmsg)s',
+                                 {'request': request, 'errmsg': errmsg},
+                                 extra={'spider': spider})
 
         if spider_failure is not download_failure:
             return spider_failure
@@ -219,7 +226,8 @@ class Scraper(object):
                     signal=signals.item_dropped, item=item, response=response,
                     spider=spider, exception=output.value)
             else:
-                log.err(output, 'Error processing %s' % item, spider=spider)
+                logger.error('Error processing %(item)s', {'item': item},
+                             extra={'spider': spider, 'failure': output})
         else:
             logkws = self.logformatter.scraped(output, response, spider)
             log.msg(spider=spider, **logkws)
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index b4706919a..f1ef1b524 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,5 +1,6 @@
 import six
 import signal
+import logging
 import warnings
 
 from twisted.internet import reactor, defer
@@ -14,7 +15,9 @@ from scrapy.signalmanager import SignalManager
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 from scrapy.utils.misc import load_object
-from scrapy import log, signals
+from scrapy import signals
+
+logger = logging.getLogger('scrapy')
 
 
 class Crawler(object):
@@ -145,15 +148,15 @@ class CrawlerProcess(CrawlerRunner):
     def _signal_shutdown(self, signum, _):
         install_shutdown_handlers(self._signal_kill)
         signame = signal_names[signum]
-        log.msg(format="Received %(signame)s, shutting down gracefully. Send again to force ",
-                level=log.INFO, signame=signame)
+        logger.info("Received %(signame)s, shutting down gracefully. Send again to force ",
+                    {'signame': signame})
         reactor.callFromThread(self.stop)
 
     def _signal_kill(self, signum, _):
         install_shutdown_handlers(signal.SIG_IGN)
         signame = signal_names[signum]
-        log.msg(format='Received %(signame)s twice, forcing unclean shutdown',
-                level=log.INFO, signame=signame)
+        logger.info('Received %(signame)s twice, forcing unclean shutdown',
+                    {'signame': signame})
         self._stop_logging()
         reactor.callFromThread(self._stop_reactor)
 
diff --git a/scrapy/dupefilter.py b/scrapy/dupefilter.py
index 9bd6a6e05..37376ad8a 100644
--- a/scrapy/dupefilter.py
+++ b/scrapy/dupefilter.py
@@ -1,7 +1,7 @@
 from __future__ import print_function
 import os
+import logging
 
-from scrapy import log
 from scrapy.utils.job import job_dir
 from scrapy.utils.request import request_fingerprint
 
@@ -33,6 +33,7 @@ class RFPDupeFilter(BaseDupeFilter):
         self.fingerprints = set()
         self.logdupes = True
         self.debug = debug
+        self.logger = logging.getLogger('scrapy')
         if path:
             self.file = open(os.path.join(path, 'requests.seen'), 'a+')
             self.fingerprints.update(x.rstrip() for x in self.file)
@@ -59,13 +60,13 @@ class RFPDupeFilter(BaseDupeFilter):
 
     def log(self, request, spider):
         if self.debug:
-            fmt = "Filtered duplicate request: %(request)s"
-            log.msg(format=fmt, request=request, level=log.DEBUG, spider=spider)
+            msg = "Filtered duplicate request: %(request)s"
+            self.logger.debug(msg, {'request': request}, extra={'spider': spider})
         elif self.logdupes:
-            fmt = ("Filtered duplicate request: %(request)s"
+            msg = ("Filtered duplicate request: %(request)s"
                    " - no more duplicates will be shown"
                    " (see DUPEFILTER_DEBUG to show all duplicates)")
-            log.msg(format=fmt, request=request, level=log.DEBUG, spider=spider)
+            self.logger.debug(msg, {'request': request}, extra={'spider': spider})
             self.logdupes = False
 
         spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)
diff --git a/scrapy/mail.py b/scrapy/mail.py
index e1d7c44f6..7e38663cf 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -3,6 +3,8 @@ Mail sending helpers
 
 See documentation in docs/topics/email.rst
 """
+import logging
+
 from six.moves import cStringIO as StringIO
 import six
 
@@ -20,7 +22,8 @@ else:
 from twisted.internet import defer, reactor, ssl
 from twisted.mail.smtp import ESMTPSenderFactory
 
-from scrapy import log
+logger = logging.getLogger('scrapy')
+
 
 class MailSender(object):
 
@@ -71,8 +74,10 @@ class MailSender(object):
             _callback(to=to, subject=subject, body=body, cc=cc, attach=attachs, msg=msg)
 
         if self.debug:
-            log.msg(format='Debug mail sent OK: To=%(mailto)s Cc=%(mailcc)s Subject="%(mailsubject)s" Attachs=%(mailattachs)d',
-                    level=log.DEBUG, mailto=to, mailcc=cc, mailsubject=subject, mailattachs=len(attachs))
+            logger.debug('Debug mail sent OK: To=%(mailto)s Cc=%(mailcc)s '
+                         'Subject="%(mailsubject)s" Attachs=%(mailattachs)d',
+                         {'mailto': to, 'mailcc': cc, 'mailsubject': subject,
+                          'mailattachs': len(attachs)})
             return
 
         dfd = self._sendmail(rcpts, msg.as_string())
@@ -83,17 +88,18 @@ class MailSender(object):
         return dfd
 
     def _sent_ok(self, result, to, cc, subject, nattachs):
-        log.msg(format='Mail sent OK: To=%(mailto)s Cc=%(mailcc)s '
-                       'Subject="%(mailsubject)s" Attachs=%(mailattachs)d',
-                mailto=to, mailcc=cc, mailsubject=subject, mailattachs=nattachs)
+        logger.info('Mail sent OK: To=%(mailto)s Cc=%(mailcc)s '
+                    'Subject="%(mailsubject)s" Attachs=%(mailattachs)d',
+                    {'mailto': to, 'mailcc': cc, 'mailsubject': subject,
+                     'mailattachs': nattachs})
 
     def _sent_failed(self, failure, to, cc, subject, nattachs):
         errstr = str(failure.value)
-        log.msg(format='Unable to send mail: To=%(mailto)s Cc=%(mailcc)s '
-                       'Subject="%(mailsubject)s" Attachs=%(mailattachs)d'
-                       '- %(mailerr)s',
-                level=log.ERROR, mailto=to, mailcc=cc, mailsubject=subject,
-                mailattachs=nattachs, mailerr=errstr)
+        logger.error('Unable to send mail: To=%(mailto)s Cc=%(mailcc)s '
+                     'Subject="%(mailsubject)s" Attachs=%(mailattachs)d'
+                     '- %(mailerr)s',
+                     {'mailto': to, 'mailcc': cc, 'mailsubject': subject,
+                      'mailattachs': nattachs, 'mailerr': errstr})
 
     def _sendmail(self, to_addrs, msg):
         msg = StringIO(msg)
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index b1494b137..917717de5 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,10 +1,13 @@
+import logging
 from collections import defaultdict
 
-from scrapy import log
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from scrapy.utils.defer import process_parallel, process_chain, process_chain_both
 
+logger = logging.getLogger('scrapy')
+
+
 class MiddlewareManager(object):
     """Base class for implementing middleware managers"""
 
@@ -37,12 +40,15 @@ class MiddlewareManager(object):
             except NotConfigured as e:
                 if e.args:
                     clsname = clspath.split('.')[-1]
-                    log.msg(format="Disabled %(clsname)s: %(eargs)s",
-                            level=log.WARNING, clsname=clsname, eargs=e.args[0])
+                    logger.warning("Disabled %(clsname)s: %(eargs)s",
+                                   {'clsname': clsname, 'eargs': e.args[0]},
+                                   extra={'crawler': crawler})
 
         enabled = [x.__class__.__name__ for x in middlewares]
-        log.msg(format="Enabled %(componentname)ss: %(enabledlist)s", level=log.INFO,
-                componentname=cls.component_name, enabledlist=', '.join(enabled))
+        logger.info("Enabled %(componentname)ss: %(enabledlist)s",
+                    {'componentname': cls.component_name,
+                     'enabledlist': ', '.join(enabled)},
+                    extra={'crawler': crawler})
         return cls(*middlewares)
 
     @classmethod
diff --git a/scrapy/statscol.py b/scrapy/statscol.py
index 8a7eed149..3fe32ee81 100644
--- a/scrapy/statscol.py
+++ b/scrapy/statscol.py
@@ -2,8 +2,10 @@
 Scrapy extension for collecting scraping stats
 """
 import pprint
+import logging
+
+logger = logging.getLogger('scrapy')
 
-from scrapy import log
 
 class StatsCollector(object):
 
@@ -41,8 +43,8 @@ class StatsCollector(object):
 
     def close_spider(self, spider, reason):
         if self._dump:
-            log.msg("Dumping Scrapy stats:\n" + pprint.pformat(self._stats), \
-                spider=spider)
+            logger.info("Dumping Scrapy stats:\n" + pprint.pformat(self._stats),
+                        extra={'spider': spider})
         self._persist_stats(self._stats, spider)
 
     def _persist_stats(self, stats, spider):
diff --git a/scrapy/telnet.py b/scrapy/telnet.py
index d7cd601a2..049ab32ed 100644
--- a/scrapy/telnet.py
+++ b/scrapy/telnet.py
@@ -5,6 +5,7 @@ See documentation in docs/topics/telnetconsole.rst
 """
 
 import pprint
+import logging
 
 from twisted.internet import protocol
 try:
@@ -15,7 +16,7 @@ except ImportError:
     TWISTED_CONCH_AVAILABLE = False
 
 from scrapy.exceptions import NotConfigured
-from scrapy import log, signals
+from scrapy import signals
 from scrapy.utils.trackref import print_live_refs
 from scrapy.utils.engine import print_engine_status
 from scrapy.utils.reactor import listen_tcp
@@ -26,6 +27,8 @@ try:
 except ImportError:
     hpy = None
 
+logger = logging.getLogger('scrapy')
+
 # signal to update telnet variables
 # args: telnet_vars
 update_telnet_vars = object()
@@ -52,8 +55,9 @@ class TelnetConsole(protocol.ServerFactory):
     def start_listening(self):
         self.port = listen_tcp(self.portrange, self.host, self)
         h = self.port.getHost()
-        log.msg(format="Telnet console listening on %(host)s:%(port)d",
-                level=log.DEBUG, host=h.host, port=h.port)
+        logger.debug("Telnet console listening on %(host)s:%(port)d",
+                     {'host': h.host, 'port': h.port},
+                     extra={'crawler': self.crawler})
 
     def stop_listening(self):
         self.port.stopListening()
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index a889114d5..4f81b2d9c 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -1,15 +1,20 @@
-import re, csv, six
+import re
+import csv
+import logging
 
 try:
     from cStringIO import StringIO as BytesIO
 except ImportError:
     from io import BytesIO
 
+import six
+
 from scrapy.http import TextResponse, Response
 from scrapy.selector import Selector
-from scrapy import log
 from scrapy.utils.python import re_rsearch, str_to_unicode
 
+logger = logging.getLogger('scrapy')
+
 
 def xmliter(obj, nodename):
     """Return a iterator of Selector's over all nodes of a XML document,
@@ -108,8 +113,10 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     while True:
         row = _getrow(csv_r)
         if len(row) != len(headers):
-            log.msg(format="ignoring row %(csvlnum)d (length: %(csvrow)d, should be: %(csvheader)d)",
-                    level=log.WARNING, csvlnum=csv_r.line_num, csvrow=len(row), csvheader=len(headers))
+            logger.warning("ignoring row %(csvlnum)d (length: %(csvrow)d, "
+                           "should be: %(csvheader)d)",
+                           {'csvlnum': csv_r.line_num, 'csvrow': len(row),
+                            'csvheader': len(headers)})
             continue
         else:
             yield dict(zip(headers, row))
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index 724f3a892..091955b73 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -1,5 +1,7 @@
 """Helper functions for working with signals"""
 
+import logging
+
 from twisted.internet.defer import maybeDeferred, DeferredList, Deferred
 from twisted.python.failure import Failure
 
@@ -7,7 +9,8 @@ from scrapy.xlib.pydispatch.dispatcher import Any, Anonymous, liveReceivers, \
     getAllReceivers, disconnect
 from scrapy.xlib.pydispatch.robustapply import robustApply
 
-from scrapy import log
+logger = logging.getLogger('scrapy')
+
 
 def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
     """Like pydispatcher.robust.sendRobust but it also logs errors and returns
@@ -21,14 +24,14 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
             response = robustApply(receiver, signal=signal, sender=sender,
                 *arguments, **named)
             if isinstance(response, Deferred):
-                log.msg(format="Cannot return deferreds from signal handler: %(receiver)s",
-                        level=log.ERROR, spider=spider, receiver=receiver)
+                logger.error("Cannot return deferreds from signal handler: %(receiver)s",
+                             {'receiver': receiver}, extra={'spider': spider})
         except dont_log:
             result = Failure()
         except Exception:
             result = Failure()
-            log.err(result, "Error caught on signal handler: %s" % receiver, \
-                spider=spider)
+            logger.exception("Error caught on signal handler: %(receiver)s",
+                             {'receiver': receiver}, extra={'spider': spider})
         else:
             result = response
         responses.append((receiver, result))
@@ -41,8 +44,9 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
     """
     def logerror(failure, recv):
         if dont_log is None or not isinstance(failure.value, dont_log):
-            log.err(failure, "Error caught on signal handler: %s" % recv, \
-                spider=spider)
+            logger.error("Error caught on signal handler: %(receiver)s",
+                         {'receiver': recv},
+                         extra={'spider': spider, 'failure': failure})
         return failure
 
     dont_log = named.pop('dont_log', None)
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 44f098f05..1df5e3769 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -1,11 +1,13 @@
+import logging
 import inspect
 
 import six
 
-from scrapy import log
 from scrapy.spider import Spider
 from scrapy.utils.misc import  arg_to_iter
 
+logger = logging.getLogger('scrapy')
+
 
 def iterate_spider_output(result):
     return arg_to_iter(result)
@@ -43,12 +45,12 @@ def spidercls_for_request(spider_loader, request, default_spidercls=None,
         return spider_loader.load(snames[0])
 
     if len(snames) > 1 and log_multiple:
-        log.msg(format='More than one spider can handle: %(request)s - %(snames)s',
-                level=log.ERROR, request=request, snames=', '.join(snames))
+        logger.error('More than one spider can handle: %(request)s - %(snames)s',
+                     {'request': request, 'snames': ', '.join(snames)})
 
     if len(snames) == 0 and log_none:
-        log.msg(format='Unable to find spider that handles: %(request)s',
-                level=log.ERROR, request=request)
+        logger.error('Unable to find spider that handles: %(request)s',
+                     {'request': request})
 
     return default_spidercls
 
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 68f76d002..f888c54bd 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -137,7 +137,6 @@ class RunSpiderCommandTest(CommandTest):
         with open(fname, 'w') as f:
             f.write("""
 import scrapy
-from scrapy import log
 
 class MySpider(scrapy.Spider):
     name = 'myspider'
@@ -148,10 +147,10 @@ class MySpider(scrapy.Spider):
 """)
         p = self.proc('runspider', fname)
         log = p.stderr.read()
-        self.assertIn("[myspider] DEBUG: It Works!", log)
-        self.assertIn("[myspider] INFO: Spider opened", log)
-        self.assertIn("[myspider] INFO: Closing spider (finished)", log)
-        self.assertIn("[myspider] INFO: Spider closed (finished)", log)
+        self.assertIn("DEBUG: It Works!", log)
+        self.assertIn("INFO: Spider opened", log)
+        self.assertIn("INFO: Closing spider (finished)", log)
+        self.assertIn("INFO: Spider closed (finished)", log)
 
     def test_runspider_no_spider_found(self):
         tmpdir = self.mktemp()
@@ -159,7 +158,6 @@ class MySpider(scrapy.Spider):
         fname = abspath(join(tmpdir, 'myspider.py'))
         with open(fname, 'w') as f:
             f.write("""
-from scrapy import log
 from scrapy.spider import Spider
 """)
         p = self.proc('runspider', fname)
@@ -192,7 +190,6 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
         fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
         with open(fname, 'w') as f:
             f.write("""
-from scrapy import log
 import scrapy
 
 class MySpider(scrapy.Spider):
@@ -207,13 +204,13 @@ class MySpider(scrapy.Spider):
         fname = abspath(join(self.proj_mod_path, 'pipelines.py'))
         with open(fname, 'w') as f:
             f.write("""
-from scrapy import log
+import logging
 
 class MyPipeline(object):
     component_name = 'my_pipeline'
 
     def process_item(self, item, spider):
-        log.msg('It Works!')
+        logging.info('It Works!')
         return item
 """)
 
@@ -229,7 +226,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
                                            '-a', 'test_arg=1',
                                            '-c', 'parse',
                                            self.url('/html')])
-        self.assertIn("[parse_spider] DEBUG: It Works!", stderr)
+        self.assertIn("DEBUG: It Works!", stderr)
 
     @defer.inlineCallbacks
     def test_pipelines(self):
@@ -237,7 +234,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
                                            '--pipelines',
                                            '-c', 'parse',
                                            self.url('/html')])
-        self.assertIn("[scrapy] INFO: It Works!", stderr)
+        self.assertIn("INFO: It Works!", stderr)
 
     @defer.inlineCallbacks
     def test_parse_items(self):

From c2d716807ab1d011352bc7dd81e3f213c32d0464 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 28 Feb 2015 07:23:54 -0300
Subject: [PATCH 0228/2974] Use LogCapture in testfixtures package for tests

This allows to remove `get_testlog` helper, `flushLoggedErrors` from
twisted.trial.unittest.TestCase and Twisted log observers created for
each test on conftest.py.
---
 conftest.py                              | 30 -----------
 pytest.ini                               |  2 +-
 scrapy/utils/test.py                     | 13 -----
 tests/requirements.txt                   |  1 +
 tests/test_closespider.py                |  1 -
 tests/test_crawl.py                      | 68 +++++++++++++++---------
 tests/test_pipeline_media.py             | 26 ++++-----
 tests/test_proxy_connect.py              | 43 ++++++++-------
 tests/test_spidermiddleware_httperror.py | 16 +++---
 tests/test_utils_defer.py                |  1 -
 tests/test_utils_signal.py               | 35 ++++++------
 tox.ini                                  |  1 +
 12 files changed, 102 insertions(+), 135 deletions(-)

diff --git a/conftest.py b/conftest.py
index 21f152e45..783a5a250 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,6 +1,5 @@
 import six
 import pytest
-from twisted.python import log
 
 collect_ignore = ["scrapy/stats.py", "scrapy/project.py"]
 
@@ -10,35 +9,6 @@ if six.PY3:
         if len(file_path) > 0 and file_path[0] != '#':
             collect_ignore.append(file_path)
 
-class LogObservers:
-    """Class for keeping track of log observers across test modules"""
-
-    def __init__(self):
-        self.observers = []
-
-    def add(self, logfile='test.log'):
-        fileobj = open(logfile, 'wb')
-        observer = log.FileLogObserver(fileobj)
-        log.startLoggingWithObserver(observer.emit, 0)
-        self.observers.append((fileobj, observer))
-
-    def remove(self):
-        fileobj, observer = self.observers.pop()
-        log.removeObserver(observer.emit)
-        fileobj.close()
-
-
-@pytest.fixture(scope='module')
-def log_observers():
-    return LogObservers()
-
-
-@pytest.fixture()
-def setlog(request, log_observers):
-    """Attach test.log file observer to twisted log, for trial compatibility"""
-    log_observers.add()
-    request.addfinalizer(log_observers.remove)
-
 
 @pytest.fixture()
 def chdir(tmpdir):
diff --git a/pytest.ini b/pytest.ini
index 355ee31c3..3046dfd85 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,5 +1,5 @@
 [pytest]
-usefixtures = chdir setlog
+usefixtures = chdir
 python_files=test_*.py __init__.py
 addopts = --doctest-modules --assert=plain
 twisted = 1
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index a4b769970..ad4a6aa7c 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -46,19 +46,6 @@ def get_testenv():
     env['PYTHONPATH'] = get_pythonpath()
     return env
 
-def get_testlog():
-    """Get Scrapy log of current test, ignoring the rest"""
-    with open("test.log", "rb") as fp:
-        loglines = fp.readlines()
-
-    thistest = []
-    for line in loglines[::-1]:
-        thistest.append(line)
-        if "[-] -->" in line:
-            break
-    return "".join(thistest[::-1])
-
-
 def assert_samelines(testcase, text1, text2, msg=None):
     """Asserts text1 and text2 have the same lines, ignoring differences in
     line endings between platforms
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 4da8e30ea..4f1520021 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -4,3 +4,4 @@ mitmproxy==0.10.1
 netlib==0.10.1
 pytest-twisted
 jmespath
+testfixtures
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 1700a861e..fa0b48998 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -39,7 +39,6 @@ class TestCloseSpider(TestCase):
         close_on = 5
         crawler = get_crawler(ErrorSpider, {'CLOSESPIDER_ERRORCOUNT': close_on})
         yield crawler.crawl(total=1000000)
-        self.flushLoggedErrors(crawler.spider.exception_cls)
         reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_errorcount')
         key = 'spider_exceptions/{name}'\
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 0fcc7c31e..f2ebf9c69 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -1,8 +1,12 @@
 import json
 import socket
+import logging
+
+from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
-from scrapy.utils.test import get_crawler, get_testlog
+
+from scrapy.utils.test import get_crawler
 from tests import mock
 from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
     BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
@@ -72,36 +76,47 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_retry_503(self):
         crawler = get_crawler(SimpleSpider)
-        yield crawler.crawl("http://localhost:8998/status?n=503")
-        self._assert_retried()
+        with LogCapture() as l:
+            yield crawler.crawl("http://localhost:8998/status?n=503")
+        self._assert_retried(l)
 
     @defer.inlineCallbacks
     def test_retry_conn_failed(self):
         crawler = get_crawler(SimpleSpider)
-        yield crawler.crawl("http://localhost:65432/status?n=503")
-        self._assert_retried()
+        with LogCapture() as l:
+            yield crawler.crawl("http://localhost:65432/status?n=503")
+        self._assert_retried(l)
 
     @defer.inlineCallbacks
     def test_retry_dns_error(self):
         with mock.patch('socket.gethostbyname',
                         side_effect=socket.gaierror(-5, 'No address associated with hostname')):
             crawler = get_crawler(SimpleSpider)
-            yield crawler.crawl("http://example.com/")
-            self._assert_retried()
+            with LogCapture() as l:
+                yield crawler.crawl("http://example.com/")
+            self._assert_retried(l)
 
     @defer.inlineCallbacks
     def test_start_requests_bug_before_yield(self):
-        crawler = get_crawler(BrokenStartRequestsSpider)
-        yield crawler.crawl(fail_before_yield=1)
-        errors = self.flushLoggedErrors(ZeroDivisionError)
-        self.assertEqual(len(errors), 1)
+        with LogCapture('scrapy', level=logging.ERROR) as l:
+            crawler = get_crawler(BrokenStartRequestsSpider)
+            yield crawler.crawl(fail_before_yield=1)
+
+        self.assertEqual(len(l.records), 1)
+        record = l.records[0]
+        self.assertIsNotNone(record.exc_info)
+        self.assertIs(record.exc_info[0], ZeroDivisionError)
 
     @defer.inlineCallbacks
     def test_start_requests_bug_yielding(self):
-        crawler = get_crawler(BrokenStartRequestsSpider)
-        yield crawler.crawl(fail_yielding=1)
-        errors = self.flushLoggedErrors(ZeroDivisionError)
-        self.assertEqual(len(errors), 1)
+        with LogCapture('scrapy', level=logging.ERROR) as l:
+            crawler = get_crawler(BrokenStartRequestsSpider)
+            yield crawler.crawl(fail_yielding=1)
+
+        self.assertEqual(len(l.records), 1)
+        record = l.records[0]
+        self.assertIsNotNone(record.exc_info)
+        self.assertIs(record.exc_info[0], ZeroDivisionError)
 
     @defer.inlineCallbacks
     def test_start_requests_lazyness(self):
@@ -145,28 +160,29 @@ foo body
 with multiples lines
 '''})
         crawler = get_crawler(SimpleSpider)
-        yield crawler.crawl("http://localhost:8998/raw?{0}".format(query))
-        log = get_testlog()
-        self.assertEqual(log.count("Got response 200"), 1)
+        with LogCapture() as l:
+            yield crawler.crawl("http://localhost:8998/raw?{0}".format(query))
+        self.assertEqual(str(l).count("Got response 200"), 1)
 
     @defer.inlineCallbacks
     def test_retry_conn_lost(self):
         # connection lost after receiving data
         crawler = get_crawler(SimpleSpider)
-        yield crawler.crawl("http://localhost:8998/drop?abort=0")
-        self._assert_retried()
+        with LogCapture() as l:
+            yield crawler.crawl("http://localhost:8998/drop?abort=0")
+        self._assert_retried(l)
 
     @defer.inlineCallbacks
     def test_retry_conn_aborted(self):
         # connection lost before receiving data
         crawler = get_crawler(SimpleSpider)
-        yield crawler.crawl("http://localhost:8998/drop?abort=1")
-        self._assert_retried()
+        with LogCapture() as l:
+            yield crawler.crawl("http://localhost:8998/drop?abort=1")
+        self._assert_retried(l)
 
-    def _assert_retried(self):
-        log = get_testlog()
-        self.assertEqual(log.count("Retrying"), 2)
-        self.assertEqual(log.count("Gave up retrying"), 1)
+    def _assert_retried(self, log):
+        self.assertEqual(str(log).count("Retrying"), 2)
+        self.assertEqual(str(log).count("Gave up retrying"), 1)
 
     @defer.inlineCallbacks
     def test_referer_header(self):
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index e4fae3983..0e946303b 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,9 +1,9 @@
 from __future__ import print_function
+from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.python.failure import Failure
 from twisted.internet import reactor
 from twisted.internet.defer import Deferred, inlineCallbacks
-from twisted.python import log as txlog
 
 from scrapy.http import Request, Response
 from scrapy.spider import Spider
@@ -11,7 +11,6 @@ from scrapy.utils.request import request_fingerprint
 from scrapy.contrib.pipeline.media import MediaPipeline
 from scrapy.utils.signal import disconnect_all
 from scrapy import signals
-from scrapy import log
 
 
 def _mocked_download_func(request, info):
@@ -60,26 +59,21 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         fail = Failure(Exception())
         results = [(True, 1), (False, fail)]
 
-        events = []
-        txlog.addObserver(events.append)
-        new_item = self.pipe.item_completed(results, item, self.info)
-        txlog.removeObserver(events.append)
-        self.flushLoggedErrors()
+        with LogCapture() as l:
+            new_item = self.pipe.item_completed(results, item, self.info)
 
         assert new_item is item
-        assert len(events) == 1
-        assert events[0]['logLevel'] == log.ERROR
-        assert events[0]['failure'] is fail
+        assert len(l.records) == 1
+        record = l.records[0]
+        assert record.levelname == 'ERROR'
+        assert record.failure is fail
 
         # disable failure logging and check again
         self.pipe.LOG_FAILED_RESULTS = False
-        events = []
-        txlog.addObserver(events.append)
-        new_item = self.pipe.item_completed(results, item, self.info)
-        txlog.removeObserver(events.append)
-        self.flushLoggedErrors()
+        with LogCapture() as l:
+            new_item = self.pipe.item_completed(results, item, self.info)
         assert new_item is item
-        assert len(events) == 0
+        assert len(l.records) == 0
 
     @inlineCallbacks
     def test_default_process_item(self):
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 5ce48ebf8..0f06fd53d 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -5,10 +5,11 @@ import time
 from threading import Thread
 from libmproxy import controller, proxy
 from netlib import http_auth
+from testfixtures import LogCapture
 
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
-from scrapy.utils.test import get_testlog, get_crawler
+from scrapy.utils.test import get_crawler
 from scrapy.http import Request
 from tests.spiders import SimpleSpider, SingleRequestSpider
 from tests.mockserver import MockServer
@@ -50,39 +51,44 @@ class ProxyConnectTestCase(TestCase):
     @defer.inlineCallbacks
     def test_https_connect_tunnel(self):
         crawler = get_crawler(SimpleSpider)
-        yield crawler.crawl("https://localhost:8999/status?n=200")
-        self._assert_got_response_code(200)
+        with LogCapture() as l:
+            yield crawler.crawl("https://localhost:8999/status?n=200")
+        self._assert_got_response_code(200, l)
 
     @defer.inlineCallbacks
     def test_https_noconnect(self):
         os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888?noconnect'
         crawler = get_crawler(SimpleSpider)
-        yield crawler.crawl("https://localhost:8999/status?n=200")
-        self._assert_got_response_code(200)
+        with LogCapture() as l:
+            yield crawler.crawl("https://localhost:8999/status?n=200")
+        self._assert_got_response_code(200, l)
         os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888'
 
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
         crawler = get_crawler(SimpleSpider)
-        yield crawler.crawl("https://localhost:99999/status?n=200")
-        self._assert_got_tunnel_error()
+        with LogCapture() as l:
+            yield crawler.crawl("https://localhost:99999/status?n=200")
+        self._assert_got_tunnel_error(l)
 
     @defer.inlineCallbacks
     def test_https_tunnel_auth_error(self):
         os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888'
         crawler = get_crawler(SimpleSpider)
-        yield crawler.crawl("https://localhost:8999/status?n=200")
+        with LogCapture() as l:
+            yield crawler.crawl("https://localhost:8999/status?n=200")
         # The proxy returns a 407 error code but it does not reach the client;
         # he just sees a TunnelError.
-        self._assert_got_tunnel_error()
+        self._assert_got_tunnel_error(l)
         os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888'
 
     @defer.inlineCallbacks
     def test_https_tunnel_without_leak_proxy_authorization_header(self):
         request = Request("https://localhost:8999/echo")
         crawler = get_crawler(SingleRequestSpider)
-        yield crawler.crawl(seed=request)
-        self._assert_got_response_code(200)
+        with LogCapture() as l:
+            yield crawler.crawl(seed=request)
+        self._assert_got_response_code(200, l)
         echo = json.loads(crawler.spider.meta['responses'][0].body)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
@@ -90,13 +96,12 @@ class ProxyConnectTestCase(TestCase):
     def test_https_noconnect_auth_error(self):
         os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888?noconnect'
         crawler = get_crawler(SimpleSpider)
-        yield crawler.crawl("https://localhost:8999/status?n=200")
-        self._assert_got_response_code(407)
+        with LogCapture() as l:
+            yield crawler.crawl("https://localhost:8999/status?n=200")
+        self._assert_got_response_code(407, l)
 
-    def _assert_got_response_code(self, code):
-        log = get_testlog()
-        self.assertEqual(log.count('Crawled (%d)' % code), 1)
+    def _assert_got_response_code(self, code, log):
+        self.assertEqual(str(log).count('Crawled (%d)' % code), 1)
 
-    def _assert_got_tunnel_error(self):
-        log = get_testlog()
-        self.assertEqual(log.count('TunnelError'), 1)
+    def _assert_got_tunnel_error(self, log):
+        self.assertEqual(str(log).count('TunnelError'), 1)
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 5394f0eee..503865f63 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -1,9 +1,10 @@
 from unittest import TestCase
 
+from testfixtures import LogCapture
 from twisted.trial.unittest import TestCase as TrialTestCase
 from twisted.internet import defer
 
-from scrapy.utils.test import get_crawler, get_testlog
+from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
 from scrapy.http import Response, Request
 from scrapy.spider import Spider
@@ -174,14 +175,13 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
     @defer.inlineCallbacks
     def test_logging(self):
         crawler = get_crawler(_HttpErrorSpider)
-        yield crawler.crawl(bypass_status_codes={402})
-        # print(get_testlog())
+        with LogCapture() as log:
+            yield crawler.crawl(bypass_status_codes={402})
         self.assertEqual(crawler.spider.parsed, {'200', '402'})
         self.assertEqual(crawler.spider.skipped, {'402'})
         self.assertEqual(crawler.spider.failed, {'404', '500'})
 
-        log = get_testlog()
-        self.assertIn('Ignoring response <404', log)
-        self.assertIn('Ignoring response <500', log)
-        self.assertNotIn('Ignoring response <200', log)
-        self.assertNotIn('Ignoring response <402', log)
+        self.assertIn('Ignoring response <404', str(log))
+        self.assertIn('Ignoring response <500', str(log))
+        self.assertNotIn('Ignoring response <200', str(log))
+        self.assertNotIn('Ignoring response <402', str(log))
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index b9cb3ab44..260ed6c2e 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -74,7 +74,6 @@ class DeferUtilsTest(unittest.TestCase):
     def test_process_parallel_failure(self):
         d = process_parallel([cb1, cb_fail, cb3], 'res', 'v1', 'v2')
         self.failUnlessFailure(d, TypeError)
-        self.flushLoggedErrors()
         return d
 
 
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index a9f377dab..571073911 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -1,11 +1,11 @@
+from testfixtures import LogCapture
 from twisted.trial import unittest
-from twisted.python import log as txlog
 from twisted.python.failure import Failure
 from twisted.internet import defer, reactor
 
 from scrapy.xlib.pydispatch import dispatcher
 from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
-from scrapy import log
+
 
 class SendCatchLogTest(unittest.TestCase):
 
@@ -14,26 +14,24 @@ class SendCatchLogTest(unittest.TestCase):
         test_signal = object()
         handlers_called = set()
 
-        def log_received(event):
-            handlers_called.add(log_received)
-            assert "error_handler" in event['message'][0]
-            assert event['logLevel'] == log.ERROR
-
-        txlog.addObserver(log_received)
         dispatcher.connect(self.error_handler, signal=test_signal)
         dispatcher.connect(self.ok_handler, signal=test_signal)
-        result = yield defer.maybeDeferred(self._get_result, test_signal, arg='test', \
-            handlers_called=handlers_called)
+        with LogCapture() as l:
+            result = yield defer.maybeDeferred(
+                self._get_result, test_signal, arg='test',
+                handlers_called=handlers_called
+            )
 
         assert self.error_handler in handlers_called
         assert self.ok_handler in handlers_called
-        assert log_received in handlers_called
+        self.assertEqual(len(l.records), 1)
+        record = l.records[0]
+        self.assertIn('error_handler', record.getMessage())
+        self.assertEqual(record.levelname, 'ERROR')
         self.assertEqual(result[0][0], self.error_handler)
         self.assert_(isinstance(result[0][1], Failure))
         self.assertEqual(result[1], (self.ok_handler, "OK"))
 
-        txlog.removeObserver(log_received)
-        self.flushLoggedErrors()
         dispatcher.disconnect(self.error_handler, signal=test_signal)
         dispatcher.disconnect(self.ok_handler, signal=test_signal)
 
@@ -73,12 +71,9 @@ class SendCatchLogTest2(unittest.TestCase):
     def test_error_logged_if_deferred_not_supported(self):
         test_signal = object()
         test_handler = lambda: defer.Deferred()
-        log_events = []
-        txlog.addObserver(log_events.append)
         dispatcher.connect(test_handler, test_signal)
-        send_catch_log(test_signal)
-        self.assertTrue(log_events)
-        self.assertIn("Cannot return deferreds from signal handler", str(log_events))
-        txlog.removeObserver(log_events.append)
-        self.flushLoggedErrors()
+        with LogCapture() as l:
+            send_catch_log(test_signal)
+        self.assertEqual(len(l.records), 1)
+        self.assertIn("Cannot return deferreds from signal handler", str(l))
         dispatcher.disconnect(test_handler, test_signal)
diff --git a/tox.ini b/tox.ini
index ae2ef9e07..5385cb87c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -50,6 +50,7 @@ deps =
     # tests requirements
     pytest>=2.6.0
     pytest-twisted
+    testfixtures
 
 [testenv:py34]
 basepython = python3.4

From 6f9b423215df8591b0d9e0062a40cc28678236ac Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 28 Feb 2015 03:22:06 -0300
Subject: [PATCH 0229/2974] Restructure LogFormatter to comply with std logging
 calls

---
 scrapy/core/engine.py      |  4 +--
 scrapy/core/scraper.py     |  5 ++-
 scrapy/logformatter.py     | 63 +++++++++++++++++++++++++-------------
 tests/test_logformatter.py |  8 ++---
 4 files changed, 50 insertions(+), 30 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 7e330af1c..a978d3c08 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -10,7 +10,7 @@ from time import time
 from twisted.internet import defer
 from twisted.python.failure import Failure
 
-from scrapy import log, signals
+from scrapy import signals
 from scrapy.core.scraper import Scraper
 from scrapy.exceptions import DontCloseSpider
 from scrapy.http import Response, Request
@@ -204,7 +204,7 @@ class ExecutionEngine(object):
             if isinstance(response, Response):
                 response.request = request # tie request to response received
                 logkws = self.logformatter.crawled(request, response, spider)
-                log.msg(spider=spider, **logkws)
+                logger._log(extra={'spider': spider}, **logkws)
                 self.signals.send_catch_log(signal=signals.response_received, \
                     response=response, request=request, spider=spider)
             return response
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 4a961f8e8..196f5392d 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -15,7 +15,6 @@ from scrapy import signals
 from scrapy.http import Request, Response
 from scrapy.item import BaseItem
 from scrapy.core.spidermw import SpiderMiddlewareManager
-from scrapy import log
 
 logger = logging.getLogger('scrapy')
 
@@ -221,7 +220,7 @@ class Scraper(object):
             ex = output.value
             if isinstance(ex, DropItem):
                 logkws = self.logformatter.dropped(item, ex, response, spider)
-                log.msg(spider=spider, **logkws)
+                logger._log(extra={'spider': spider}, **logkws)
                 return self.signals.send_catch_log_deferred(
                     signal=signals.item_dropped, item=item, response=response,
                     spider=spider, exception=output.value)
@@ -230,7 +229,7 @@ class Scraper(object):
                              extra={'spider': spider, 'failure': output})
         else:
             logkws = self.logformatter.scraped(output, response, spider)
-            log.msg(spider=spider, **logkws)
+            logger._log(extra={'spider': spider}, **logkws)
             return self.signals.send_catch_log_deferred(
                 signal=signals.item_scraped, item=output, response=response,
                 spider=spider)
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index d03d2d07f..a0508e0b7 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -1,46 +1,67 @@
 import os
+import logging
 
 from twisted.python.failure import Failure
 
-from scrapy import log
 
+SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s"
+DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
+CRAWLEDMSG = u"Crawled (%(status)s) %(request)s (referer: %(referer)s)%(flags)s"
 
-SCRAPEDFMT = u"Scraped from %(src)s" + os.linesep + "%(item)s"
-DROPPEDFMT = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
-CRAWLEDFMT = u"Crawled (%(status)s) %(request)s (referer: %(referer)s)%(flags)s"
 
 class LogFormatter(object):
-    """Class for generating log messages for different actions. All methods
-    must return a plain string which doesn't include the log level or the
-    timestamp
+    """Class for generating log messages for different actions.
+
+    All methods must return a dictionary listing the parameters `level`, `msg`
+    and `args` which are going to be used for constructing the log message when
+    calling logging.log.
+
+    Dictionary keys for the method outputs:
+        * `level` should be the log level for that action, you can use those
+        from the python logging library: logging.DEBUG, logging.INFO,
+        logging.WARNING, logging.ERROR and logging.CRITICAL.
+
+        * `msg` should be a string that can contain different formatting
+        placeholders. This string, formatted with the provided `args`, is going
+        to be the log message for that action.
+
+        * `args` should be a tuple or dict with the formatting placeholders for
+        `msg`.  The final log message is computed as output['msg'] %
+        output['args'].
     """
 
     def crawled(self, request, response, spider):
         flags = ' %s' % str(response.flags) if response.flags else ''
         return {
-            'level': log.DEBUG,
-            'format': CRAWLEDFMT,
-            'status': response.status,
-            'request': request,
-            'referer': request.headers.get('Referer'),
-            'flags': flags,
+            'level': logging.DEBUG,
+            'msg': CRAWLEDMSG,
+            'args': {
+                'status': response.status,
+                'request': request,
+                'referer': request.headers.get('Referer'),
+                'flags': flags,
+            }
         }
 
     def scraped(self, item, response, spider):
         src = response.getErrorMessage() if isinstance(response, Failure) else response
         return {
-            'level': log.DEBUG,
-            'format': SCRAPEDFMT,
-            'src': src,
-            'item': item,
+            'level': logging.DEBUG,
+            'msg': SCRAPEDMSG,
+            'args': {
+                'src': src,
+                'item': item,
+            }
         }
 
     def dropped(self, item, exception, response, spider):
         return {
-            'level': log.WARNING,
-            'format': DROPPEDFMT,
-            'exception': exception,
-            'item': item,
+            'level': logging.WARNING,
+            'msg': DROPPEDMSG,
+            'args': {
+                'exception': exception,
+                'item': item,
+            }
         }
 
     @classmethod
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index adec19afa..8446fd646 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -24,14 +24,14 @@ class LoggingContribTest(unittest.TestCase):
         req = Request("http://www.example.com")
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
-        logline = logkws['format'] % logkws
+        logline = logkws['msg'] % logkws['args']
         self.assertEqual(logline,
             "Crawled (200) <GET http://www.example.com> (referer: None)")
 
         req = Request("http://www.example.com", headers={'referer': 'http://example.com'})
         res = Response("http://www.example.com", flags=['cached'])
         logkws = self.formatter.crawled(req, res, self.spider)
-        logline = logkws['format'] % logkws
+        logline = logkws['msg'] % logkws['args']
         self.assertEqual(logline,
             "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
 
@@ -40,7 +40,7 @@ class LoggingContribTest(unittest.TestCase):
         exception = Exception(u"\u2018")
         response = Response("http://www.example.com")
         logkws = self.formatter.dropped(item, exception, response, self.spider)
-        logline = logkws['format'] % logkws
+        logline = logkws['msg'] % logkws['args']
         lines = logline.splitlines()
         assert all(isinstance(x, unicode) for x in lines)
         self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
@@ -50,7 +50,7 @@ class LoggingContribTest(unittest.TestCase):
         item['name'] = u'\xa3'
         response = Response("http://www.example.com")
         logkws = self.formatter.scraped(item, response, self.spider)
-        logline = logkws['format'] % logkws
+        logline = logkws['msg'] % logkws['args']
         lines = logline.splitlines()
         assert all(isinstance(x, unicode) for x in lines)
         self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])

From 8baad552676de1ea0f3789a03279a025ed26f17e Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 28 Feb 2015 06:33:03 -0300
Subject: [PATCH 0230/2974] New scrapy/utils/log.py file with basic log helpers

There are two functions, `configure_logging` and `log_scrapy_info` which
intend to replace scrapy.log.start and scrapy.log.scrapy_info
respectively.

Creating new functions makes evident the backward incompatible change of
using another logging system, and since the Python logging module is a
standard builtin, additional helpers make sense to be on a scrapy/utils
file.
---
 scrapy/crawler.py   | 10 +++-------
 scrapy/utils/log.py | 48 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 7 deletions(-)
 create mode 100644 scrapy/utils/log.py

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index f1ef1b524..f96086605 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -15,6 +15,7 @@ from scrapy.signalmanager import SignalManager
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 from scrapy.utils.misc import load_object
+from scrapy.utils.log import configure_logging, log_scrapy_info
 from scrapy import signals
 
 logger = logging.getLogger('scrapy')
@@ -142,8 +143,8 @@ class CrawlerProcess(CrawlerRunner):
         super(CrawlerProcess, self).__init__(settings)
         install_shutdown_handlers(self._signal_shutdown)
         self.stopping = False
-        self.log_observer = log.start_from_settings(self.settings)
-        log.scrapy_info(settings)
+        configure_logging(settings)
+        log_scrapy_info(settings)
 
     def _signal_shutdown(self, signum, _):
         install_shutdown_handlers(self._signal_kill)
@@ -157,7 +158,6 @@ class CrawlerProcess(CrawlerRunner):
         signame = signal_names[signum]
         logger.info('Received %(signame)s twice, forcing unclean shutdown',
                     {'signame': signame})
-        self._stop_logging()
         reactor.callFromThread(self._stop_reactor)
 
     def start(self, stop_after_crawl=True):
@@ -176,10 +176,6 @@ class CrawlerProcess(CrawlerRunner):
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
         reactor.run(installSignalHandlers=False)  # blocking call
 
-    def _stop_logging(self):
-        if self.log_observer:
-            self.log_observer.stop()
-
     def _stop_reactor(self, _=None):
         try:
             reactor.stop()
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
new file mode 100644
index 000000000..3cdb543b9
--- /dev/null
+++ b/scrapy/utils/log.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+import sys
+import logging
+from logging.config import dictConfig
+
+from twisted.python import log as twisted_log
+
+import scrapy
+from scrapy.settings import overridden_settings
+
+logger = logging.getLogger('scrapy')
+
+
+DEFAULT_LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'loggers': {
+        'scrapy': {
+            'level': 'DEBUG',
+        },
+        'twisted': {
+            'level': 'ERROR',
+        },
+    }
+}
+
+
+def configure_logging(settings=None):
+    if not sys.warnoptions:
+        # Route warnings through python logging
+        logging.captureWarnings(True)
+
+    observer = twisted_log.PythonLoggingObserver('twisted')
+    observer.start()
+
+    dictConfig(DEFAULT_LOGGING)
+
+
+def log_scrapy_info(settings):
+    logger.info("Scrapy %(version)s started (bot: %(bot)s)",
+                {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
+
+    logger.info("Optional features available: %(features)s",
+                {'features': ", ".join(scrapy.optional_features)})
+
+    d = dict(overridden_settings(settings))
+    logger.info("Overridden settings: %(settings)r", {'settings': d})

From b75556ef796efe806279778441a11cdc3c024e07 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 28 Feb 2015 08:14:53 -0300
Subject: [PATCH 0231/2974] Add a logging filter to mimic Twisted's log.err
 formating for Failures

---
 scrapy/utils/log.py     | 26 +++++++++++++++++++++++
 tests/test_utils_log.py | 46 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 tests/test_utils_log.py

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 3cdb543b9..4fd0f3afb 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -1,9 +1,11 @@
 # -*- coding: utf-8 -*-
 
+import os
 import sys
 import logging
 from logging.config import dictConfig
 
+from twisted.python.failure import Failure
 from twisted.python import log as twisted_log
 
 import scrapy
@@ -12,12 +14,36 @@ from scrapy.settings import overridden_settings
 logger = logging.getLogger('scrapy')
 
 
+class FailureFormatter(logging.Filter):
+    """Extract exc_info from Failure instances provided as contextual data
+
+    This filter mimics Twisted log.err formatting for its first `_stuff`
+    argument, which means that reprs of non Failure objects are appended to the
+    log messages.
+    """
+
+    def filter(self, record):
+        failure = record.__dict__.get('failure')
+        if failure:
+            if isinstance(failure, Failure):
+                record.exc_info = (failure.type, failure.value, failure.tb)
+            else:
+                record.msg += os.linesep + repr(failure)
+        return True
+
+
 DEFAULT_LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
+    'filters': {
+        'failure_formatter': {
+            '()': 'scrapy.utils.log.FailureFormatter',
+        },
+    },
     'loggers': {
         'scrapy': {
             'level': 'DEBUG',
+            'filters': ['failure_formatter'],
         },
         'twisted': {
             'level': 'ERROR',
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
new file mode 100644
index 000000000..f843d9797
--- /dev/null
+++ b/tests/test_utils_log.py
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+import os
+import logging
+import unittest
+
+from testfixtures import LogCapture
+from twisted.python.failure import Failure
+
+from scrapy.utils.log import FailureFormatter
+
+
+class FailureFormatterTest(unittest.TestCase):
+
+    def setUp(self):
+        self.logger = logging.getLogger('test')
+        self.filter = FailureFormatter()
+        self.logger.addFilter(self.filter)
+
+    def tearDown(self):
+        self.logger.removeFilter(self.filter)
+
+    def test_failure_format(self):
+        with LogCapture() as l:
+            try:
+                0/0
+            except ZeroDivisionError:
+                self.logger.exception('test log msg')
+                failure = Failure()
+
+            self.logger.error('test log msg', extra={'failure': failure})
+
+        self.assertEqual(len(l.records), 2)
+        exc_record, failure_record = l.records
+        self.assertTupleEqual(failure_record.exc_info, exc_record.exc_info)
+
+        formatter = logging.Formatter()
+        self.assertMultiLineEqual(formatter.format(failure_record),
+                                  formatter.format(exc_record))
+
+    def test_non_failure_format(self):
+        with LogCapture() as l:
+            self.logger.error('test log msg', extra={'failure': 3})
+
+        self.assertEqual(len(l.records), 1)
+        self.assertMultiLineEqual(l.records[0].getMessage(),
+                                  'test log msg' + os.linesep + '3')

From ac40ef611a892daa51b90f214875fa06f11806dc Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 28 Feb 2015 06:36:35 -0300
Subject: [PATCH 0232/2974] Custom handler to count log level occurrences in a
 crawler

---
 scrapy/commands/shell.py |  1 -
 scrapy/crawler.py        | 14 +++++++-------
 scrapy/utils/log.py      | 12 ++++++++++++
 tests/py3-ignores.txt    |  1 +
 tests/test_utils_log.py  | 33 ++++++++++++++++++++++++++++++++-
 5 files changed, 52 insertions(+), 9 deletions(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 0b130529b..cf99865c4 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -53,7 +53,6 @@ class Command(ScrapyCommand):
         # The crawler is created this way since the Shell manually handles the
         # crawling engine, so the set up in the crawl method won't work
         crawler = self.crawler_process._create_crawler(spidercls)
-        self.crawler_process._setup_crawler_logging(crawler)
         # The Shell class needs a persistent engine in the crawler
         crawler.engine = crawler._create_engine()
         crawler.engine.start()
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index f96086605..4eba6f83a 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -15,7 +15,7 @@ from scrapy.signalmanager import SignalManager
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 from scrapy.utils.misc import load_object
-from scrapy.utils.log import configure_logging, log_scrapy_info
+from scrapy.utils.log import LogCounterHandler, configure_logging, log_scrapy_info
 from scrapy import signals
 
 logger = logging.getLogger('scrapy')
@@ -32,6 +32,12 @@ class Crawler(object):
 
         self.signals = SignalManager(self)
         self.stats = load_object(self.settings['STATS_CLASS'])(self)
+
+        handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL'))
+        logging.root.addHandler(handler)
+        self.signals.connect(lambda: logging.root.removeHandler(handler),
+                             signals.engine_stopped)
+
         lf_cls = load_object(self.settings['LOG_FORMATTER'])
         self.logformatter = lf_cls.from_crawler(self)
         self.extensions = ExtensionManager.from_crawler(self)
@@ -103,7 +109,6 @@ class CrawlerRunner(object):
         crawler = crawler_or_spidercls
         if not isinstance(crawler_or_spidercls, Crawler):
             crawler = self._create_crawler(crawler_or_spidercls)
-            self._setup_crawler_logging(crawler)
 
         self.crawlers.add(crawler)
         d = crawler.crawl(*args, **kwargs)
@@ -121,11 +126,6 @@ class CrawlerRunner(object):
             spidercls = self.spider_loader.load(spidercls)
         return Crawler(spidercls, self.settings)
 
-    def _setup_crawler_logging(self, crawler):
-        log_observer = log.start_from_crawler(crawler)
-        if log_observer:
-            crawler.signals.connect(log_observer.stop, signals.engine_stopped)
-
     def stop(self):
         return defer.DeferredList([c.stop() for c in list(self.crawlers)])
 
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 4fd0f3afb..b3e76887d 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -72,3 +72,15 @@ def log_scrapy_info(settings):
 
     d = dict(overridden_settings(settings))
     logger.info("Overridden settings: %(settings)r", {'settings': d})
+
+
+class LogCounterHandler(logging.Handler):
+    """Record log levels count into a crawler stats"""
+
+    def __init__(self, crawler, *args, **kwargs):
+        super(LogCounterHandler, self).__init__(*args, **kwargs)
+        self.crawler = crawler
+
+    def emit(self, record):
+        sname = 'log_count/{}'.format(record.levelname)
+        self.crawler.stats.inc_value(sname)
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 7a150b281..0fc90eddb 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -59,6 +59,7 @@ tests/test_stats.py
 tests/test_utils_defer.py
 tests/test_utils_iterators.py
 tests/test_utils_jsonrpc.py
+tests/test_utils_log.py
 tests/test_utils_python.py
 tests/test_utils_reqser.py
 tests/test_utils_request.py
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index f843d9797..d98dbb574 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -6,7 +6,8 @@ import unittest
 from testfixtures import LogCapture
 from twisted.python.failure import Failure
 
-from scrapy.utils.log import FailureFormatter
+from scrapy.utils.log import FailureFormatter, LogCounterHandler
+from scrapy.utils.test import get_crawler
 
 
 class FailureFormatterTest(unittest.TestCase):
@@ -44,3 +45,33 @@ class FailureFormatterTest(unittest.TestCase):
         self.assertEqual(len(l.records), 1)
         self.assertMultiLineEqual(l.records[0].getMessage(),
                                   'test log msg' + os.linesep + '3')
+
+
+class LogCounterHandlerTest(unittest.TestCase):
+
+    def setUp(self):
+        self.logger = logging.getLogger('test')
+        self.logger.setLevel(logging.NOTSET)
+        self.logger.propagate = False
+        self.crawler = get_crawler(settings_dict={'LOG_LEVEL': 'WARNING'})
+        self.handler = LogCounterHandler(self.crawler)
+        self.logger.addHandler(self.handler)
+
+    def tearDown(self):
+        self.logger.propagate = True
+        self.logger.removeHandler(self.handler)
+
+    def test_init(self):
+        self.assertIsNone(self.crawler.stats.get_value('log_count/DEBUG'))
+        self.assertIsNone(self.crawler.stats.get_value('log_count/INFO'))
+        self.assertIsNone(self.crawler.stats.get_value('log_count/WARNING'))
+        self.assertIsNone(self.crawler.stats.get_value('log_count/ERROR'))
+        self.assertIsNone(self.crawler.stats.get_value('log_count/CRITICAL'))
+
+    def test_accepted_level(self):
+        self.logger.error('test log msg')
+        self.assertEqual(self.crawler.stats.get_value('log_count/ERROR'), 1)
+
+    def test_filtered_out_level(self):
+        self.logger.debug('test log msg')
+        self.assertIsNone(self.crawler.stats.get_value('log_count/INFO'))

From 1c8708eb829b9b57729860cecfd7cc8d1eaffaea Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 28 Feb 2015 06:56:53 -0300
Subject: [PATCH 0233/2974] Create a logger for every Spider and adapt
 Spider.log to log through it

---
 scrapy/spider.py       | 18 +++++++++++++-----
 tests/spiders.py       |  2 +-
 tests/test_commands.py |  4 ++--
 tests/test_spider.py   | 19 +++++++++++++++++++
 4 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/scrapy/spider.py b/scrapy/spider.py
index a2d091804..36623b6e2 100644
--- a/scrapy/spider.py
+++ b/scrapy/spider.py
@@ -3,9 +3,9 @@ Base class for Scrapy spiders
 
 See documentation in docs/topics/spiders.rst
 """
+import logging
 import warnings
 
-from scrapy import log
 from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.trackref import object_ref
@@ -31,11 +31,19 @@ class Spider(object_ref):
         if not hasattr(self, 'start_urls'):
             self.start_urls = []
 
-    def log(self, message, level=log.DEBUG, **kw):
-        """Log the given messages at the given log level. Always use this
-        method to send log messages from your spider
+    @property
+    def logger(self):
+        logger = logging.getLogger(self.name)
+        return logging.LoggerAdapter(logger, {'spider': self})
+
+    def log(self, message, level=logging.DEBUG, **kw):
+        """Log the given message at the given log level
+
+        This helper wraps a log call to the logger within the spider, but you
+        can use it directly (e.g. Spider.logger.info('msg')) or use any other
+        Python logger too.
         """
-        log.msg(message, spider=self, level=level, **kw)
+        self.logger.log(level, message, **kw)
 
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):
diff --git a/tests/spiders.py b/tests/spiders.py
index 86ace9d6e..5484fc5b9 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -74,7 +74,7 @@ class SimpleSpider(MetaSpider):
         self.start_urls = [url]
 
     def parse(self, response):
-        self.log("Got response %d" % response.status)
+        self.logger.info("Got response %d" % response.status)
 
 
 class ItemSpider(FollowAllSpider):
diff --git a/tests/test_commands.py b/tests/test_commands.py
index f888c54bd..b95d0b0cc 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -142,7 +142,7 @@ class MySpider(scrapy.Spider):
     name = 'myspider'
 
     def start_requests(self):
-        self.log("It Works!")
+        self.logger.debug("It Works!")
         return []
 """)
         p = self.proc('runspider', fname)
@@ -197,7 +197,7 @@ class MySpider(scrapy.Spider):
 
     def parse(self, response):
         if getattr(self, 'test_arg', None):
-            self.log('It Works!')
+            self.logger.debug('It Works!')
         return [scrapy.Item(), dict(foo='bar')]
 """.format(self.spider_name))
 
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 56bcb2b1e..517fc0995 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -2,6 +2,8 @@ import gzip
 import inspect
 import warnings
 from io import BytesIO
+
+from testfixtures import LogCapture
 from twisted.trial import unittest
 
 from scrapy import signals
@@ -102,6 +104,23 @@ class SpiderTest(unittest.TestCase):
         self.assertEqual(settings.get('TEST2'), 'spider')
         self.assertEqual(settings.get('TEST3'), 'project')
 
+    def test_logger(self):
+        spider = self.spider_class('example.com')
+        with LogCapture() as l:
+            spider.logger.info('test log msg')
+        l.check(('example.com', 'INFO', 'test log msg'))
+
+        record = l.records[0]
+        self.assertIn('spider', record.__dict__)
+        self.assertIs(record.spider, spider)
+
+    def test_log(self):
+        spider = self.spider_class('example.com')
+        with mock.patch('scrapy.spider.Spider.logger') as mock_logger:
+            spider.log('test log msg', 'INFO')
+        mock_logger.log.assert_called_once_with('INFO', 'test log msg')
+
+
 class InitSpiderTest(SpiderTest):
 
     spider_class = InitSpider

From ffd97f2f1b96296dccf65598ff3ce49e477e5fea Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 28 Feb 2015 08:16:12 -0300
Subject: [PATCH 0234/2974] Set root handlers based on settings in
 configure_logging

---
 scrapy/utils/log.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index b3e76887d..889c66015 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -53,6 +53,14 @@ DEFAULT_LOGGING = {
 
 
 def configure_logging(settings=None):
+    """Initialize and configure default loggers
+
+    This function does:
+      - Route warnings and twisted logging through Python standard logging
+      - Set FailureFormatter filter on Scrapy logger
+      - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively
+      - Create a handler for the root logger according to given settings
+    """
     if not sys.warnoptions:
         # Route warnings through python logging
         logging.captureWarnings(True)
@@ -61,6 +69,26 @@ def configure_logging(settings=None):
     observer.start()
 
     dictConfig(DEFAULT_LOGGING)
+    if settings:
+        logging.root.setLevel(logging.NOTSET)
+
+        # Set up the default log handler
+        filename = settings.get('LOG_FILE')
+        if filename:
+            encoding = settings.get('LOG_ENCODING')
+            handler = logging.FileHandler(filename, encoding=encoding)
+        elif settings.getbool('LOG_ENABLED'):
+            handler = logging.StreamHandler()
+        else:
+            handler = logging.NullHandler()
+
+        formatter = logging.Formatter(
+            fmt='%(asctime)s [%(name)s] %(levelname)s: %(message)s',
+            datefmt='%Y-%m-%d %H:%M:%S%z'
+        )
+        handler.setFormatter(formatter)
+        handler.setLevel(settings.get('LOG_LEVEL'))
+        logging.root.addHandler(handler)
 
 
 def log_scrapy_info(settings):

From 6acb3848fbc8405cb0985355561ae207b2e444a1 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 28 Feb 2015 08:17:37 -0300
Subject: [PATCH 0235/2974] Stdout redirect in configure_logging

---
 scrapy/utils/log.py     | 19 +++++++++++++++++++
 tests/test_utils_log.py | 21 ++++++++++++++++++++-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 889c66015..27cc7b227 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -72,6 +72,9 @@ def configure_logging(settings=None):
     if settings:
         logging.root.setLevel(logging.NOTSET)
 
+        if settings.getbool('LOG_STDOUT'):
+            sys.stdout = StreamLogger(logging.getLogger('stdout'))
+
         # Set up the default log handler
         filename = settings.get('LOG_FILE')
         if filename:
@@ -102,6 +105,22 @@ def log_scrapy_info(settings):
     logger.info("Overridden settings: %(settings)r", {'settings': d})
 
 
+class StreamLogger(object):
+    """Fake file-like stream object that redirects writes to a logger instance
+
+    Taken from:
+        http://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/
+    """
+    def __init__(self, logger, log_level=logging.INFO):
+        self.logger = logger
+        self.log_level = log_level
+        self.linebuf = ''
+
+    def write(self, buf):
+        for line in buf.rstrip().splitlines():
+            self.logger.log(self.log_level, line.rstrip())
+
+
 class LogCounterHandler(logging.Handler):
     """Record log levels count into a crawler stats"""
 
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index d98dbb574..42c4d6ea4 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -1,12 +1,14 @@
 # -*- coding: utf-8 -*-
+from __future__ import print_function
 import os
+import sys
 import logging
 import unittest
 
 from testfixtures import LogCapture
 from twisted.python.failure import Failure
 
-from scrapy.utils.log import FailureFormatter, LogCounterHandler
+from scrapy.utils.log import FailureFormatter, LogCounterHandler, StreamLogger
 from scrapy.utils.test import get_crawler
 
 
@@ -75,3 +77,20 @@ class LogCounterHandlerTest(unittest.TestCase):
     def test_filtered_out_level(self):
         self.logger.debug('test log msg')
         self.assertIsNone(self.crawler.stats.get_value('log_count/INFO'))
+
+
+class StreamLoggerTest(unittest.TestCase):
+
+    def setUp(self):
+        self.stdout = sys.stdout
+        logger = logging.getLogger('test')
+        logger.setLevel(logging.WARNING)
+        sys.stdout = StreamLogger(logger, logging.ERROR)
+
+    def tearDown(self):
+        sys.stdout = self.stdout
+
+    def test_redirect(self):
+        with LogCapture() as l:
+            print('test log msg')
+        l.check(('test', 'ERROR', 'test log msg'))

From c174d78f1221d65affcd94b72cfccee8d63cdf1b Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 28 Feb 2015 09:30:28 -0300
Subject: [PATCH 0236/2974] Deprecate scrapy/log.py

---
 scrapy/log.py         | 165 ++---------------------------------------
 tests/py3-ignores.txt |   1 -
 tests/test_log.py     | 169 ------------------------------------------
 3 files changed, 7 insertions(+), 328 deletions(-)
 delete mode 100644 tests/test_log.py

diff --git a/scrapy/log.py b/scrapy/log.py
index aa53e3574..f87bf6461 100644
--- a/scrapy/log.py
+++ b/scrapy/log.py
@@ -1,161 +1,10 @@
 """
-Scrapy logging facility
+This module is kept to provide a helpful warning about its removal.
+"""
 
-See documentation in docs/topics/logging.rst
-"""
-import sys
-import logging
 import warnings
-
-from twisted.python import log
-
-import scrapy
-from scrapy.utils.python import unicode_to_str
-from scrapy.settings import overridden_settings
-
-# Logging levels
-DEBUG = logging.DEBUG
-INFO = logging.INFO
-WARNING = logging.WARNING
-ERROR = logging.ERROR
-CRITICAL = logging.CRITICAL
-SILENT = CRITICAL + 1
-
-level_names = {
-    logging.DEBUG: "DEBUG",
-    logging.INFO: "INFO",
-    logging.WARNING: "WARNING",
-    logging.ERROR: "ERROR",
-    logging.CRITICAL: "CRITICAL",
-    SILENT: "SILENT",
-}
-
-class ScrapyFileLogObserver(log.FileLogObserver):
-
-    def __init__(self, f, level=INFO, encoding='utf-8', crawler=None):
-        self.level = level
-        self.encoding = encoding
-        self.crawler = crawler
-        if crawler:
-            self.emit = self._emit_with_crawler
-        else:
-            self.emit = self._emit
-        log.FileLogObserver.__init__(self, f)
-
-    def _emit(self, eventDict):
-        ev = _adapt_eventdict(eventDict, self.level, self.encoding,
-                              self.crawler)
-        if ev is not None:
-            log.FileLogObserver.emit(self, ev)
-        return ev
-
-    def _emit_with_crawler(self, eventDict):
-        ev = self._emit(eventDict)
-        if ev:
-            level = ev['logLevel']
-            sname = 'log_count/%s' % level_names.get(level, level)
-            self.crawler.stats.inc_value(sname)
-
-def _adapt_eventdict(eventDict, log_level=INFO, encoding='utf-8',
-                     crawler=None, prepend_level=True):
-    """Adapt Twisted log eventDict making it suitable for logging with a Scrapy
-    log observer. It may return None to indicate that the event should be
-    ignored by a Scrapy log observer.
-
-    `log_level` is the minimum level being logged, and `encoding` is the log
-    encoding.
-    """
-    ev = eventDict.copy()
-    if ev['isError']:
-        ev.setdefault('logLevel', ERROR)
-
-    # ignore non-error messages from outside scrapy
-    if ev.get('system') != 'scrapy' and not ev['isError']:
-        return
-
-    level = ev.get('logLevel')
-    if level < log_level:
-        return
-
-    spider = ev.get('spider')
-    if spider:
-        ev['system'] = unicode_to_str(spider.name, encoding)
-    if crawler and (not spider or spider.crawler is not crawler):
-        # ignore events not triggered by own spiders in crawlers' observers
-        return
-    if not crawler and spider:
-        # ignore spiders' events in observers without crawler
-        return
-
-    lvlname = level_names.get(level, 'NOLEVEL')
-    message = ev.get('message')
-    if message:
-        message = [unicode_to_str(x, encoding) for x in message]
-        if prepend_level:
-            message[0] = "%s: %s" % (lvlname, message[0])
-        ev['message'] = message
-
-    why = ev.get('why')
-    if why:
-        why = unicode_to_str(why, encoding)
-        if prepend_level:
-            why = "%s: %s" % (lvlname, why)
-        ev['why'] = why
-
-    fmt = ev.get('format')
-    if fmt:
-        fmt = unicode_to_str(fmt, encoding)
-        if prepend_level:
-            fmt = "%s: %s" % (lvlname, fmt)
-        ev['format'] = fmt
-
-    return ev
-
-def _get_log_level(level_name_or_id):
-    if isinstance(level_name_or_id, int):
-        return level_name_or_id
-    elif isinstance(level_name_or_id, basestring):
-        return globals()[level_name_or_id]
-    else:
-        raise ValueError("Unknown log level: %r" % level_name_or_id)
-
-def start(logfile=None, loglevel='INFO', logstdout=True, logencoding='utf-8', crawler=None):
-    loglevel = _get_log_level(loglevel)
-    file = open(logfile, 'a') if logfile else sys.stderr
-    log_observer = ScrapyFileLogObserver(file, loglevel, logencoding, crawler)
-    _oldshowwarning = warnings.showwarning
-    log.startLoggingWithObserver(log_observer.emit, setStdout=logstdout)
-    # restore warnings, wrongly silenced by Twisted
-    warnings.showwarning = _oldshowwarning
-    return log_observer
-
-def msg(message=None, _level=INFO, **kw):
-    kw['logLevel'] = kw.pop('level', _level)
-    kw.setdefault('system', 'scrapy')
-    if message is None:
-        log.msg(**kw)
-    else:
-        log.msg(message, **kw)
-
-def err(_stuff=None, _why=None, **kw):
-    kw['logLevel'] = kw.pop('level', ERROR)
-    kw.setdefault('system', 'scrapy')
-    log.err(_stuff, _why, **kw)
-
-def start_from_settings(settings, crawler=None):
-    if settings.getbool('LOG_ENABLED'):
-        return start(settings['LOG_FILE'], settings['LOG_LEVEL'], settings['LOG_STDOUT'],
-            settings['LOG_ENCODING'], crawler)
-
-def scrapy_info(settings):
-    msg("Scrapy %s started (bot: %s)" % (scrapy.__version__,
-        settings['BOT_NAME']))
-
-    msg("Optional features available: %s" % ", ".join(scrapy.optional_features),
-        level=INFO)
-
-    d = dict(overridden_settings(settings))
-    msg(format="Overridden settings: %(settings)r", settings=d, level=INFO)
-
-def start_from_crawler(crawler):
-    return start_from_settings(crawler.settings, crawler)
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.log` has been deprecated, Scrapy now relies on "
+              "the builtin Python library for logging. Read the updated "
+              "logging entry in the documentation to learn more.",
+              ScrapyDeprecationWarning, stacklevel=2)
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 0fc90eddb..d27d5dd3c 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -31,7 +31,6 @@ tests/test_http_cookies.py
 tests/test_http_request.py
 tests/test_http_response.py
 tests/test_logformatter.py
-tests/test_log.py
 tests/test_mail.py
 tests/test_middleware.py
 tests/test_pipeline_files.py
diff --git a/tests/test_log.py b/tests/test_log.py
deleted file mode 100644
index 113d10004..000000000
--- a/tests/test_log.py
+++ /dev/null
@@ -1,169 +0,0 @@
-from io import BytesIO
-
-from twisted.python import log as txlog, failure
-from twisted.trial import unittest
-
-from scrapy import log
-from scrapy.spider import Spider
-from scrapy.settings import default_settings
-from scrapy.utils.test import get_crawler
-
-class LogTest(unittest.TestCase):
-
-    def test_get_log_level(self):
-        default_log_level = getattr(log, default_settings.LOG_LEVEL)
-        self.assertEqual(log._get_log_level('WARNING'), log.WARNING)
-        self.assertEqual(log._get_log_level(log.WARNING), log.WARNING)
-        self.assertRaises(ValueError, log._get_log_level, object())
-
-class ScrapyFileLogObserverTest(unittest.TestCase):
-
-    level = log.INFO
-    encoding = 'utf-8'
-
-    def setUp(self):
-        self.f = BytesIO()
-        self.log_observer = log.ScrapyFileLogObserver(self.f, self.level, self.encoding)
-        self.log_observer.start()
-
-    def tearDown(self):
-        self.flushLoggedErrors()
-        self.log_observer.stop()
-
-    def logged(self):
-        return self.f.getvalue().strip()[25:]
-
-    def first_log_line(self):
-        logged = self.logged()
-        return logged.splitlines()[0] if logged else ''
-
-    def test_msg_basic(self):
-        log.msg("Hello")
-        self.assertEqual(self.logged(), "[scrapy] INFO: Hello")
-
-    def test_msg_ignore_spider(self):
-        spider = Spider("myspider")
-        log.msg("Hello", spider=spider)
-        self.failIf(self.logged())
-
-    def test_msg_level1(self):
-        log.msg("Hello", level=log.WARNING)
-        self.assertEqual(self.logged(), "[scrapy] WARNING: Hello")
-
-    def test_msg_level2(self):
-        log.msg("Hello", log.WARNING)
-        self.assertEqual(self.logged(), "[scrapy] WARNING: Hello")
-
-    def test_msg_wrong_level(self):
-        log.msg("Hello", level=9999)
-        self.assertEqual(self.logged(), "[scrapy] NOLEVEL: Hello")
-
-    def test_msg_encoding(self):
-        log.msg(u"Price: \xa3100")
-        self.assertEqual(self.logged(), "[scrapy] INFO: Price: \xc2\xa3100")
-
-    def test_msg_ignore_level(self):
-        log.msg("Hello", level=log.DEBUG)
-        log.msg("World", level=log.INFO)
-        self.assertEqual(self.logged(), "[scrapy] INFO: World")
-
-    def test_msg_ignore_system(self):
-        txlog.msg("Hello")
-        self.failIf(self.logged())
-
-    def test_msg_ignore_system_err(self):
-        txlog.msg("Hello")
-        self.failIf(self.logged())
-
-    def test_err_noargs(self):
-        try:
-            a = 1/0
-        except:
-            log.err()
-        self.assertIn('Traceback', self.logged())
-        self.assertIn('ZeroDivisionError', self.logged())
-
-    def test_err_why(self):
-        log.err(TypeError("bad type"), "Wrong type")
-        self.assertEqual(self.first_log_line(), "[scrapy] ERROR: Wrong type")
-        self.assertIn('TypeError', self.logged())
-        self.assertIn('bad type', self.logged())
-
-    def test_error_outside_scrapy(self):
-        """Scrapy logger should still print outside errors"""
-        txlog.err(TypeError("bad type"), "Wrong type")
-        self.assertEqual(self.first_log_line(), "[-] ERROR: Wrong type")
-        self.assertIn('TypeError', self.logged())
-        self.assertIn('bad type', self.logged())
-
-# this test fails in twisted trial observer, not in scrapy observer
-#    def test_err_why_encoding(self):
-#        log.err(TypeError("bad type"), u"\xa3")
-#        self.assertEqual(self.first_log_line(), "[scrapy] ERROR: \xc2\xa3")
-
-    def test_err_exc(self):
-        log.err(TypeError("bad type"))
-        self.assertIn('Unhandled Error', self.logged())
-        self.assertIn('TypeError', self.logged())
-        self.assertIn('bad type', self.logged())
-
-    def test_err_failure(self):
-        log.err(failure.Failure(TypeError("bad type")))
-        self.assertIn('Unhandled Error', self.logged())
-        self.assertIn('TypeError', self.logged())
-        self.assertIn('bad type', self.logged())
-
-
-class Latin1ScrapyFileLogObserverTest(ScrapyFileLogObserverTest):
-
-    encoding = 'latin-1'
-
-    def test_msg_encoding(self):
-        log.msg(u"Price: \xa3100")
-        logged = self.f.getvalue().strip()[25:]
-        self.assertEqual(self.logged(), "[scrapy] INFO: Price: \xa3100")
-
-# this test fails in twisted trial observer, not in scrapy observer
-#    def test_err_why_encoding(self):
-#        log.err(TypeError("bad type"), u"\xa3")
-#        self.assertEqual(self.first_log_line(), "[scrapy] ERROR: \xa3")
-
-
-class CrawlerScrapyFileLogObserverTest(unittest.TestCase):
-
-    def setUp(self):
-        self.f = BytesIO()
-        self.crawler = get_crawler(Spider)
-        self.spider = self.crawler.spider = self.crawler._create_spider('test')
-        self.log_observer = log.ScrapyFileLogObserver(self.f, log.INFO,
-                                                      'utf-8', self.crawler)
-        self.log_observer.start()
-
-    def tearDown(self):
-        self.flushLoggedErrors()
-        self.log_observer.stop()
-
-    def logged(self):
-        return self.f.getvalue().strip()[25:]
-
-    def test_msg_basic(self):
-        log.msg("Hello", spider=self.spider)
-        self.assertEqual(self.logged(), "[test] INFO: Hello")
-
-    def test_msg_ignore_scrapy_channel(self):
-        log.msg("Hello")
-        self.failIf(self.logged())
-
-    def test_msg_ignore_another_crawler(self):
-        crawler = get_crawler(Spider)
-        log.msg("Hello", spider=crawler._create_spider('test'))
-        self.failIf(self.logged())
-
-    def test_msg_stats_log(self):
-        assert self.crawler.stats.get_value('log_count/INFO', 0) == 0
-        log.msg("Hello", spider=self.spider)
-        self.assertEqual(self.crawler.stats.get_value('log_count/INFO'), 1)
-
-
-if __name__ == "__main__":
-    unittest.main()

From 21b9f377d6d09cfc3bc88d1c09decce9bf75c8ff Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 4 Mar 2015 15:13:57 -0300
Subject: [PATCH 0237/2974] Deprecate more frequently used functions from
 scrapy/log.py

---
 scrapy/log.py | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/scrapy/log.py b/scrapy/log.py
index f87bf6461..af0473d46 100644
--- a/scrapy/log.py
+++ b/scrapy/log.py
@@ -2,9 +2,46 @@
 This module is kept to provide a helpful warning about its removal.
 """
 
+import logging
 import warnings
+
+from twisted.python.failure import Failure
+
 from scrapy.exceptions import ScrapyDeprecationWarning
+
+logger = logging.getLogger('scrapy')
+
 warnings.warn("Module `scrapy.log` has been deprecated, Scrapy now relies on "
               "the builtin Python library for logging. Read the updated "
               "logging entry in the documentation to learn more.",
               ScrapyDeprecationWarning, stacklevel=2)
+
+
+# Imports kept for backwards-compatibility
+
+DEBUG = logging.DEBUG
+INFO = logging.INFO
+WARNING = logging.WARNING
+ERROR = logging.ERROR
+CRITICAL = logging.CRITICAL
+SILENT = CRITICAL + 1
+
+
+def msg(message, _level=logging.INFO, **kw):
+    warnings.warn('log.msg has been deprecated, create a python logger and '
+                  'log through it instead',
+                  ScrapyDeprecationWarning, stacklevel=2)
+
+    level = kw.pop('level', _level)
+    logger.log(level, message, kw)
+
+
+def err(_stuff=None, _why=None, **kw):
+    warnings.warn('log.err has been deprecated, create a python logger and '
+                  'use its error method instead',
+                  ScrapyDeprecationWarning, stacklevel=2)
+
+    level = kw.pop('level', logging.ERROR)
+    failure = kw.pop('failure', _stuff) or Failure()
+    message = kw.pop('why', _why) or failure.value
+    logger.log(level, message, kw, extra={'failure': failure})

From ccdd8bfbccddc6fcf1bbc1f28cef3266d7c12eca Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 5 Mar 2015 05:04:21 -0300
Subject: [PATCH 0238/2974] Parametrize log formatting strings

---
 docs/topics/settings.rst            | 25 +++++++++++++++++++++++++
 scrapy/settings/default_settings.py |  2 ++
 scrapy/utils/log.py                 |  4 ++--
 3 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 7deb6e0c6..3936c04b8 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -612,6 +612,31 @@ Default: ``None``
 
 File name to use for logging output. If None, standard error will be used.
 
+.. setting:: LOG_FORMAT
+
+LOG_FORMAT
+----------
+
+Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
+
+String for formatting log messsages. Refer to the `Python logging documentation`_ for the whole list of available
+placeholders.
+
+.. _Python logging documentation: https://docs.python.org/2/library/logging.html#logrecord-attributes
+
+.. setting:: LOG_DATEFORMAT
+
+LOG_DATEFORMAT
+--------------
+
+Default: ``'%Y-%m-%d %H:%M:%S%z'``
+
+String for formatting date/time, expansion of the ``%(asctime)s`` placeholder
+in :setting:`LOG_FORMAT`. Refer to the `Python datetime documentation`_ for the whole list of available
+directives.
+
+.. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior
+
 .. setting:: LOG_LEVEL
 
 LOG_LEVEL
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 49addbc81..0c3d7c5bd 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -167,6 +167,8 @@ ITEM_PIPELINES_BASE = {}
 LOG_ENABLED = True
 LOG_ENCODING = 'utf-8'
 LOG_FORMATTER = 'scrapy.logformatter.LogFormatter'
+LOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'
+LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S%z'
 LOG_STDOUT = False
 LOG_LEVEL = 'DEBUG'
 LOG_FILE = None
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 27cc7b227..e2b1edcf7 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -86,8 +86,8 @@ def configure_logging(settings=None):
             handler = logging.NullHandler()
 
         formatter = logging.Formatter(
-            fmt='%(asctime)s [%(name)s] %(levelname)s: %(message)s',
-            datefmt='%Y-%m-%d %H:%M:%S%z'
+            fmt=settings.get('LOG_FORMAT'),
+            datefmt=settings.get('LOG_DATEFORMAT')
         )
         handler.setFormatter(formatter)
         handler.setLevel(settings.get('LOG_LEVEL'))

From d47a7edc655f4113e54f256f4ceab636f530a1f1 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 10 Mar 2015 15:52:45 -0300
Subject: [PATCH 0239/2974] Update Logging topic on docs

---
 docs/index.rst          |   2 +-
 docs/topics/logging.rst | 251 +++++++++++++++++++++++++++-------------
 2 files changed, 172 insertions(+), 81 deletions(-)

diff --git a/docs/index.rst b/docs/index.rst
index 97ec9539d..6a1571a69 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -119,7 +119,7 @@ Built-in services
    topics/webservice
 
 :doc:`topics/logging`
-    Understand the simple logging facility provided by Scrapy.
+    Learn how to use Python's builtin logging on Scrapy.
 
 :doc:`topics/stats`
     Collect statistics about your scraping crawler.
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index e9266cd6a..441d6428c 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -4,126 +4,217 @@
 Logging
 =======
 
-Scrapy provides a logging facility which can be used through the
-:mod:`scrapy.log` module. The current underlying implementation uses `Twisted
-logging`_ but this may change in the future.
+.. note::
+    :mod:`scrapy.log` has been deprecated alongside its functions in favor of
+    explicit calls to the Python standard logging. Keep reading to learn more
+    about the new logging system.
 
-.. _Twisted logging: http://twistedmatrix.com/documents/current/core/howto/logging.html
+Scrapy uses `Python's builtin logging system
+<https://docs.python.org/2/library/logging.html>`_ for event logging. We'll
+provide some simple examples to get you started, but for more advanced
+use-cases it's strongly suggested to read thoroughly its documentation.
 
-The logging service must be explicitly started through the
-:func:`scrapy.log.start` function to catch the top level Scrapy's log messages.
-On top of that, each crawler has its own independent log observer
-(automatically attached when it's created) that intercepts its spider's log
-messages.
+Logging works out of the box, and can be configured to some extent with the
+Scrapy settings listed in :ref:`topics-logging-settings`.
+
+Scrapy calls :func:`scrapy.utils.log.configure_logging` to set some reasonable
+defaults and handle those settings in :ref:`topics-logging-settings` when
+running commands, so it's recommended to manually call it if you're running
+Scrapy from scripts as described in :ref:`run-from-script`.
 
 .. _topics-logging-levels:
 
 Log levels
 ==========
 
-Scrapy provides 5 logging levels:
+Python's builtin logging defines 5 different levels to indicate severity on a
+given log message. Here are the standard ones, listed in decreasing order:
 
-1. :data:`~scrapy.log.CRITICAL` - for critical errors
-2. :data:`~scrapy.log.ERROR` - for regular errors
-3. :data:`~scrapy.log.WARNING` - for warning messages
-4. :data:`~scrapy.log.INFO` - for informational messages
-5. :data:`~scrapy.log.DEBUG` - for debugging messages
-
-How to set the log level
-========================
-
-You can set the log level using the `--loglevel/-L` command line option, or
-using the :setting:`LOG_LEVEL` setting.
+1. ``logging.CRITICAL`` - for critical errors (highest severity)
+2. ``logging.ERROR`` - for regular errors
+3. ``logging.WARNING`` - for warning messages
+4. ``logging.INFO`` - for informational messages
+5. ``logging.DEBUG`` - for debugging messages (lowest severity)
 
 How to log messages
 ===================
 
-Here's a quick example of how to log a message using the ``WARNING`` level::
+Here's a quick example of how to log a message using the ``logging.WARNING``
+level::
 
-    from scrapy import log
-    log.msg("This is a warning", level=log.WARNING)
+    import logging
+    logging.warning("This is a warning")
+
+There are shortcuts for issuing log messages on any of the standard 5 levels,
+and there's also a general ``logging.log`` method which takes a given level as
+argument.  If you need so, last example could be rewrote as::
+
+    import logging
+    logging.log(logging.WARNING, "This is a warning")
+
+On top of that, you can create different "loggers" to encapsulate messages (For
+example, a common practice it's to create different loggers for every module).
+These loggers can be configured independently, and they allow hierarchical
+constructions.
+
+Last examples use the root logger behind the scenes, which is a top level
+logger where all messages are propagated to (unless otherwise specified). Using
+``logging`` helpers is merely a shortcut for getting the root logger
+explicitly, so this is also an equivalent of last snippets::
+
+    import logging
+    logger = logging.getLogger()
+    logger.warning("This is a warning")
+
+You can use a different logger just by getting its name with the
+``logging.getLogger`` function::
+
+    import logging
+    logger = logging.getLogger('mycustomlogger')
+    logger.warning("This is a warning")
+
+Finally, you can ensure having a custom logger for any module you're working on
+by using the ``__name__`` variable, which is populated with current module's
+path::
+
+    import logging
+    logger = logging.getLogger(__name__)
+    logger.warning("This is a warning")
+
+.. seealso::
+
+    Module logging, `HowTo <https://docs.python.org/2/howto/logging.html>`_
+        Basic Logging Tutorial
+
+    Module logging, `Loggers <https://docs.python.org/2/library/logging.html#logger-objects>`_
+        Further documentation on loggers
+
+.. _topics-logging-from-spiders:
 
 Logging from Spiders
 ====================
 
-The recommended way to log from spiders is by using the Spider
-:meth:`~scrapy.spider.Spider.log` method, which already populates the
-``spider`` argument of the :func:`scrapy.log.msg` function. The other arguments
-are passed directly to the :func:`~scrapy.log.msg` function.
+Scrapy provides a :data:`~scrapy.spider.Spider.logger` within each Spider
+instance, that can be accessed and used like this::
 
-scrapy.log module
-=================
+    import scrapy
 
-.. module:: scrapy.log
-   :synopsis: Logging facility
+    class MySpider(scrapy.Spider):
 
-.. function:: start(logfile=None, loglevel=None, logstdout=None)
+        name = 'myspider'
+        start_urls = ['http://scrapinghub.com']
 
-    Start the top level Scrapy logger. This must be called before actually
-    logging any top level messages (those logged using this module's
-    :func:`~scrapy.log.msg` function instead of the :meth:`Spider.log
-    <scrapy.spider.Spider.log>` method). Otherwise, messages logged before this
-    call will get lost.
+        def parse(self, response):
+            self.logger.info('Parse function called on %s', response.url)
 
-    :param logfile: the file path to use for logging output. If omitted, the
-        :setting:`LOG_FILE` setting will be used. If both are ``None``, the log
-        will be sent to standard error.
-    :type logfile: str
+That logger is created using the Spider's name, but you can use any custom
+Python logger you want. For example::
 
-    :param loglevel: the minimum logging level to log. Available values are:
-        :data:`CRITICAL`, :data:`ERROR`, :data:`WARNING`, :data:`INFO` and
-        :data:`DEBUG`.
+    import logging
+    import scrapy
 
-    :param logstdout: if ``True``, all standard output (and error) of your
-        application will be logged instead. For example if you "print 'hello'"
-        it will appear in the Scrapy log. If omitted, the :setting:`LOG_STDOUT`
-        setting will be used.
-    :type logstdout: boolean
+    logger = logging.getLogger('mycustomlogger')
 
-.. function:: msg(message, level=INFO, spider=None)
+    class MySpider(scrapy.Spider):
 
-    Log a message
+        name = 'myspider'
+        start_urls = ['http://scrapinghub.com']
 
-    :param message: the message to log
-    :type message: str
+        def parse(self, response):
+            logger.info('Parse function called on %s', response.url)
 
-    :param level: the log level for this message. See
-        :ref:`topics-logging-levels`.
+.. _topics-logging-configuration:
 
-    :param spider: the spider to use for logging this message. This parameter
-        should always be used when logging things related to a particular
-        spider.
-    :type spider: :class:`~scrapy.spider.Spider` object
+Logging configuration
+=====================
 
-.. data:: CRITICAL
+Loggers on their own don't manage how messages sent through them are displayed.
+For this task, different "handlers" can be attached to any logger instance and
+they will redirect those messages to appropriate destinations, such as the
+standard output, files, emails, etc.
 
-    Log level for critical errors
+By default, Scrapy sets and configures a handler for the root logger, based on
+the settings below.
 
-.. data:: ERROR
-
-    Log level for errors
-
-.. data:: WARNING
-
-    Log level for warnings
-
-.. data:: INFO
-
-    Log level for informational messages (recommended level for production
-    deployments)
-
-.. data:: DEBUG
-
-    Log level for debugging messages (recommended level for development)
+.. _topics-logging-settings:
 
 Logging settings
-================
+----------------
 
 These settings can be used to configure the logging:
 
+* :setting:`LOG_FILE`
 * :setting:`LOG_ENABLED`
 * :setting:`LOG_ENCODING`
-* :setting:`LOG_FILE`
 * :setting:`LOG_LEVEL`
+* :setting:`LOG_FORMAT`
+* :setting:`LOG_DATEFORMAT`
 * :setting:`LOG_STDOUT`
 
+First couple of settings define a destination for log messages. If
+:setting:`LOG_FILE` is set, messages sent through the root logger will be
+redirected to a file named :setting:`LOG_FILE` with encoding
+:setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log
+messages will be displayed on the standard error. Lastly, if
+:setting:`LOG_ENABLED` is ``False``, there won't be any visible log output.
+
+:setting:`LOG_LEVEL` determines the minimum level of severity to display, those
+messages with lower severity will be filtered out. It ranges through the
+possible levels listed in :ref:`topics-logging-levels`.
+
+:setting:`LOG_FORMAT` and :setting:`LOG_DATEFORMAT` specify formatting strings
+used as layouts for all messages. Those strings can contain any placeholders
+listed in `logging's logrecord attributes docs
+<https://docs.python.org/2/library/logging.html#logrecord-attributes>`_ and
+`datetime's strftime and strptime directives
+<https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_
+respectively.
+
+Command-line options
+--------------------
+
+There are command-line arguments, available for all commands, that you can use
+to override some of the Scrapy settings regarding logging.
+
+* ``--logfile FILE``
+    Overrides :setting:`LOG_FILE`
+* ``--loglevel/-L LEVEL``
+    Overrides :setting:`LOG_LEVEL`
+* ``--nolog``
+    Sets :setting:`LOG_ENABLED` to ``False``
+
+.. seealso::
+
+    Module `logging.handlers <https://docs.python.org/2/library/logging.handlers.html>`_
+        Further documentation on available handlers
+
+scrapy.utils.log module
+=======================
+
+.. module:: scrapy.utils.log
+   :synopsis: Logging utils
+
+.. function:: configure_logging(settings=None)
+
+    This function initializes logging defaults for Scrapy.
+
+    It's automatically called when using Scrapy commands, but needs to be
+    called explicitely when running custom scripts. In that case, its usage is
+    not required but it's recommended.
+
+    This function does:
+      - Route warnings and Twisted logging through Python standard logging
+      - Set a filter on Scrapy logger for formatting Twisted failures
+      - Assign DEBUG and ERROR levels to Scrapy and Twisted loggers
+        respectively
+
+    If `settings` is not ``None``, it will also create a root handler based on
+    the settings listed in :ref:`topics-logging-settings`.
+
+    If you plan on configuring the handlers yourself is still recommended you
+    call this function, keeping `settings` as ``None``. Bear in mind there
+    won't be any log output set by default in that case.
+
+    :param settings: settings used to create and configure a handler for the
+        root logger.
+    :type settings: :class:`~scrapy.settings.Settings` object or ``None``

From 4811d16f1d1bd2b34ef1838428e340342fdfd113 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 10 Mar 2015 15:53:38 -0300
Subject: [PATCH 0240/2974] Update `logger` attr and `log` method in the
 Spiders topic on docs

---
 docs/topics/spiders.rst | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index ddeb17d07..c64e98396 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -112,6 +112,12 @@ scrapy.Spider
       :class:`~scrapy.settings.Settings` instance, see the
       :ref:`topics-settings` topic for a detailed introduction on this subject.
 
+   .. attribute:: logger
+
+      Python logger created with the Spider's :attr:`name`. You can use it to
+      send log messages through it as described on
+      :ref:`topics-logging-from-spiders`.
+
    .. method:: from_crawler(crawler, \*args, \**kwargs)
 
        This is the class method used by Scrapy to create your spiders.
@@ -194,9 +200,9 @@ scrapy.Spider
 
    .. method:: log(message, [level, component])
 
-       Log a message using the :func:`scrapy.log.msg` function, automatically
-       populating the spider argument with the :attr:`name` of this
-       spider. For more information see :ref:`topics-logging`.
+       Wrapper that sends a log message through the Spider's :attr:`logger`,
+       kept for backwards compatibility. For more information see
+       :ref:`topics-logging-from-spiders`.
 
    .. method:: closed(reason)
 

From bd0b639b2127908816bb2b594f77f2d5210bed5b Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 10 Mar 2015 15:59:44 -0300
Subject: [PATCH 0241/2974] Fix logging usage across docs

---
 docs/topics/debug.rst            |  5 +----
 docs/topics/extensions.rst       |  9 ++++++---
 docs/topics/practices.rst        | 17 ++++++++++++++---
 docs/topics/request-response.rst |  4 ++--
 docs/topics/spiders.rst          | 10 ++++------
 5 files changed, 27 insertions(+), 18 deletions(-)

diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst
index 5739d9e1b..a3e72097c 100644
--- a/docs/topics/debug.rst
+++ b/docs/topics/debug.rst
@@ -132,16 +132,13 @@ Logging is another useful option for getting information about your spider run.
 Although not as convenient, it comes with the advantage that the logs will be
 available in all future runs should they be necessary again::
 
-    from scrapy import log
-
     def parse_details(self, response):
         item = response.meta.get('item', None)
         if item:
             # populate more `item` fields
             return item
         else:
-            self.log('No item received for %s' % response.url,
-                level=log.WARNING)
+            self.logger.warning('No item received for %s', response.url)
 
 For more information, check the :ref:`topics-logging` section.
 
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 8cd588c4a..19c296651 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -102,9 +102,12 @@ number of items will be specified through the ``MYEXT_ITEMCOUNT`` setting.
 
 Here is the code of such extension::
 
+    import logging
     from scrapy import signals
     from scrapy.exceptions import NotConfigured
 
+    logger = logging.getLogger(__name__)
+
     class SpiderOpenCloseLogging(object):
 
         def __init__(self, item_count):
@@ -133,15 +136,15 @@ Here is the code of such extension::
             return ext
 
         def spider_opened(self, spider):
-            spider.log("opened spider %s" % spider.name)
+            logger.info("opened spider %s", spider.name)
 
         def spider_closed(self, spider):
-            spider.log("closed spider %s" % spider.name)
+            logger.info("closed spider %s", spider.name)
 
         def item_scraped(self, item, spider):
             self.items_scraped += 1
             if self.items_scraped % self.item_count == 0:
-                spider.log("scraped %d items" % self.items_scraped)
+                logger.info("scraped %d items", self.items_scraped)
                 
 
 .. _topics-extensions-ref:
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 2f848e6d9..6bd74c794 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -30,9 +30,12 @@ project as example.
 
     from twisted.internet import reactor
     from scrapy.crawler import CrawlerRunner
+    from scrapy.utils.log import configure_logging
     from scrapy.utils.project import get_project_settings
 
-    runner = CrawlerRunner(get_project_settings())
+    settings = get_project_settings()
+    configure_logging(settings)
+    runner = CrawlerRunner(settings)
 
     # 'followall' is the name of one of the spiders of the project.
     d = runner.crawl('followall', domain='scrapinghub.com')
@@ -54,11 +57,13 @@ the spider class as first argument in the :meth:`CrawlerRunner.crawl
     from twisted.internet import reactor
     import scrapy
     from scrapy.crawler import CrawlerRunner
+    from scrapy.utils.log import configure_logging
 
     class MySpider(scrapy.Spider):
         # Your spider definition
         ...
 
+    configure_logging(settings)
     runner = CrawlerRunner({
         'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
     })
@@ -85,9 +90,12 @@ Here is an example that runs multiple spiders simultaneously, using the
 
     from twisted.internet import reactor, defer
     from scrapy.crawler import CrawlerRunner
+    from scrapy.utils.log import configure_logging
     from scrapy.utils.project import get_project_settings
 
-    runner = CrawlerRunner(get_project_settings())
+    settings = get_project_settings()
+    configure_logging(settings)
+    runner = CrawlerRunner(settings)
     dfs = set()
     for domain in ['scrapinghub.com', 'insophia.com']:
         d = runner.crawl('followall', domain=domain)
@@ -102,9 +110,12 @@ Same example but running the spiders sequentially by chaining the deferreds:
 
     from twisted.internet import reactor, defer
     from scrapy.crawler import CrawlerRunner
+    from scrapy.utils.log import configure_logging
     from scrapy.utils.project import get_project_settings
 
-    runner = CrawlerRunner(get_project_settings())
+    settings = get_project_settings()
+    configure_logging(settings)
+    runner = CrawlerRunner(settings)
 
     @defer.inlineCallbacks
     def crawl():
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 5b4ced992..1d695a5f2 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -189,7 +189,7 @@ Example::
 
     def parse_page2(self, response):
         # this would log http://www.example.com/some_page.html
-        self.log("Visited %s" % response.url)
+        self.logger.info("Visited %s", response.url)
 
 In some cases you may be interested in passing arguments to those callback
 functions so you can receive the arguments later, in the second callback. You
@@ -382,7 +382,7 @@ method for this job. Here's an example spider which uses it::
         def after_login(self, response):
             # check login succeed before going on
             if "authentication failed" in response.body:
-                self.log("Login failed", level=log.ERROR)
+                self.logger.error("Login failed")
                 return
 
             # continue scraping with authenticated session...
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index c64e98396..7c7d5d731 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -224,7 +224,7 @@ Let's see an example::
         ]
 
         def parse(self, response):
-            self.log('A response from %s just arrived!' % response.url)
+            self.logger.info('A response from %s just arrived!', response.url)
 
 Return multiple Requests and items from a single callback::
 
@@ -412,7 +412,7 @@ Let's now take a look at an example CrawlSpider with rules::
         )
 
         def parse_item(self, response):
-            self.log('Hi, this is an item page! %s' % response.url)
+            self.logger.info('Hi, this is an item page! %s', response.url)
             item = scrapy.Item()
             item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
             item['name'] = response.xpath('//td[@id="item_name"]/text()').extract()
@@ -515,7 +515,6 @@ XMLFeedSpider example
 
 These spiders are pretty easy to use, let's have a look at one example::
 
-    from scrapy import log
     from scrapy.contrib.spiders import XMLFeedSpider
     from myproject.items import TestItem
 
@@ -527,7 +526,7 @@ These spiders are pretty easy to use, let's have a look at one example::
         itertag = 'item'
 
         def parse_node(self, response, node):
-            log.msg('Hi, this is a <%s> node!: %s' % (self.itertag, ''.join(node.extract())))
+            self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract()))
 
             item = TestItem()
             item['id'] = node.xpath('@id').extract()
@@ -576,7 +575,6 @@ CSVFeedSpider example
 Let's see an example similar to the previous one, but using a
 :class:`CSVFeedSpider`::
 
-    from scrapy import log
     from scrapy.contrib.spiders import CSVFeedSpider
     from myproject.items import TestItem
 
@@ -589,7 +587,7 @@ Let's see an example similar to the previous one, but using a
         headers = ['id', 'name', 'description']
 
         def parse_row(self, response, row):
-            log.msg('Hi, this is a row!: %r' % row)
+            self.logger.info('Hi, this is a row!: %r', row)
 
             item = TestItem()
             item['id'] = row['id']

From 69a3d5811087093c41d7db79e39a12afc662f0f9 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 19 Mar 2015 13:12:03 -0300
Subject: [PATCH 0242/2974] Basic example on manually configuring log handlers

---
 docs/topics/logging.rst | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 441d6428c..2cb719998 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -215,6 +215,21 @@ scrapy.utils.log module
     call this function, keeping `settings` as ``None``. Bear in mind there
     won't be any log output set by default in that case.
 
+    To get you started on manually configuring logging's output, you can use
+    `logging.basicConfig()`_ to set a basic root handler. This is an example on
+    how to redirect ``INFO`` or higher messages to a file::
+
+        import logging
+        from scrapy.utils.log import configure_logging
+
+        configure_logging()  # Note we aren't providing settings in this case
+        logging.basicConfig(filename='log.txt', format='%(levelname)s: %(message)s', level=logging.INFO)
+
+    Refer to :ref:`run-from-script` for more details about using Scrapy this
+    way.
+
     :param settings: settings used to create and configure a handler for the
         root logger.
     :type settings: :class:`~scrapy.settings.Settings` object or ``None``
+
+.. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig

From 4f54ca3294600893ed5da242d07fa4d285da5adf Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 19 Mar 2015 13:24:51 -0300
Subject: [PATCH 0243/2974] Change 'scrapy' logger for '__name__' on every
 module

---
 scrapy/commands/parse.py                             | 2 +-
 scrapy/contrib/debug.py                              | 2 +-
 scrapy/contrib/downloadermiddleware/ajaxcrawl.py     | 2 +-
 scrapy/contrib/downloadermiddleware/cookies.py       | 2 +-
 scrapy/contrib/downloadermiddleware/decompression.py | 2 +-
 scrapy/contrib/downloadermiddleware/redirect.py      | 2 +-
 scrapy/contrib/downloadermiddleware/retry.py         | 2 +-
 scrapy/contrib/downloadermiddleware/robotstxt.py     | 2 +-
 scrapy/contrib/feedexport.py                         | 2 +-
 scrapy/contrib/logstats.py                           | 2 +-
 scrapy/contrib/memusage.py                           | 2 +-
 scrapy/contrib/pipeline/files.py                     | 2 +-
 scrapy/contrib/pipeline/media.py                     | 2 +-
 scrapy/contrib/spidermiddleware/depth.py             | 2 +-
 scrapy/contrib/spidermiddleware/httperror.py         | 2 +-
 scrapy/contrib/spidermiddleware/offsite.py           | 2 +-
 scrapy/contrib/spidermiddleware/urllength.py         | 2 +-
 scrapy/contrib/spiders/sitemap.py                    | 2 +-
 scrapy/contrib/throttle.py                           | 2 +-
 scrapy/core/downloader/handlers/http11.py            | 2 +-
 scrapy/core/engine.py                                | 2 +-
 scrapy/core/scheduler.py                             | 2 +-
 scrapy/core/scraper.py                               | 2 +-
 scrapy/crawler.py                                    | 2 +-
 scrapy/dupefilter.py                                 | 2 +-
 scrapy/log.py                                        | 2 +-
 scrapy/mail.py                                       | 2 +-
 scrapy/middleware.py                                 | 2 +-
 scrapy/statscol.py                                   | 2 +-
 scrapy/telnet.py                                     | 2 +-
 scrapy/utils/iterators.py                            | 2 +-
 scrapy/utils/log.py                                  | 2 +-
 scrapy/utils/signal.py                               | 2 +-
 scrapy/utils/spider.py                               | 2 +-
 34 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index b28beecc0..245b3fdf0 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -11,7 +11,7 @@ from scrapy.utils.conf import arglist_to_dict
 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
 from scrapy.exceptions import UsageError
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class Command(ScrapyCommand):
diff --git a/scrapy/contrib/debug.py b/scrapy/contrib/debug.py
index f1ec67530..625e13249 100644
--- a/scrapy/contrib/debug.py
+++ b/scrapy/contrib/debug.py
@@ -14,7 +14,7 @@ from pdb import Pdb
 from scrapy.utils.engine import format_engine_status
 from scrapy.utils.trackref import format_live_refs
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class StackTraceDump(object):
diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
index ef7f34ef9..6b543b823 100644
--- a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
+++ b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
@@ -11,7 +11,7 @@ from scrapy.http import HtmlResponse
 from scrapy.utils.response import _noscript_re, _script_re
 
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class AjaxCrawlMiddleware(object):
diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/contrib/downloadermiddleware/cookies.py
index 70ecc2dec..270d621cd 100644
--- a/scrapy/contrib/downloadermiddleware/cookies.py
+++ b/scrapy/contrib/downloadermiddleware/cookies.py
@@ -7,7 +7,7 @@ from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class CookiesMiddleware(object):
diff --git a/scrapy/contrib/downloadermiddleware/decompression.py b/scrapy/contrib/downloadermiddleware/decompression.py
index 7cd506dd9..49313cc04 100644
--- a/scrapy/contrib/downloadermiddleware/decompression.py
+++ b/scrapy/contrib/downloadermiddleware/decompression.py
@@ -18,7 +18,7 @@ except ImportError:
 
 from scrapy.responsetypes import responsetypes
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class DecompressionMiddleware(object):
diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py
index 68d139bc7..f439f43ae 100644
--- a/scrapy/contrib/downloadermiddleware/redirect.py
+++ b/scrapy/contrib/downloadermiddleware/redirect.py
@@ -5,7 +5,7 @@ from scrapy.http import HtmlResponse
 from scrapy.utils.response import get_meta_refresh
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class BaseRedirectMiddleware(object):
diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py
index 749b334f1..714103ba0 100644
--- a/scrapy/contrib/downloadermiddleware/retry.py
+++ b/scrapy/contrib/downloadermiddleware/retry.py
@@ -28,7 +28,7 @@ from scrapy.exceptions import NotConfigured
 from scrapy.utils.response import response_status_message
 from scrapy.xlib.tx import ResponseFailed
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class RetryMiddleware(object):
diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py
index 12ab2dd07..9083482f0 100644
--- a/scrapy/contrib/downloadermiddleware/robotstxt.py
+++ b/scrapy/contrib/downloadermiddleware/robotstxt.py
@@ -12,7 +12,7 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class RobotsTxtMiddleware(object):
diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py
index 7162fbc10..7c6849a73 100644
--- a/scrapy/contrib/feedexport.py
+++ b/scrapy/contrib/feedexport.py
@@ -23,7 +23,7 @@ from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import get_func_args
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class IFeedStorage(Interface):
diff --git a/scrapy/contrib/logstats.py b/scrapy/contrib/logstats.py
index 3ea347e8d..647e50f8d 100644
--- a/scrapy/contrib/logstats.py
+++ b/scrapy/contrib/logstats.py
@@ -5,7 +5,7 @@ from twisted.internet import task
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class LogStats(object):
diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py
index d1e13bfe5..402c78b68 100644
--- a/scrapy/contrib/memusage.py
+++ b/scrapy/contrib/memusage.py
@@ -16,7 +16,7 @@ from scrapy.exceptions import NotConfigured
 from scrapy.mail import MailSender
 from scrapy.utils.engine import get_engine_status
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class MemoryUsage(object):
diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py
index daedac3f7..4809005d1 100644
--- a/scrapy/contrib/pipeline/files.py
+++ b/scrapy/contrib/pipeline/files.py
@@ -26,7 +26,7 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.misc import md5sum
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class FileException(Exception):
diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py
index 2995dded6..55ef05ad4 100644
--- a/scrapy/contrib/pipeline/media.py
+++ b/scrapy/contrib/pipeline/media.py
@@ -9,7 +9,7 @@ from scrapy.utils.defer import mustbe_deferred, defer_result
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import arg_to_iter
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class MediaPipeline(object):
diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py
index 6aeb5e053..795b60eb4 100644
--- a/scrapy/contrib/spidermiddleware/depth.py
+++ b/scrapy/contrib/spidermiddleware/depth.py
@@ -8,7 +8,7 @@ import logging
 
 from scrapy.http import Request
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class DepthMiddleware(object):
diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py
index 1962eaf6c..6b7c50fce 100644
--- a/scrapy/contrib/spidermiddleware/httperror.py
+++ b/scrapy/contrib/spidermiddleware/httperror.py
@@ -7,7 +7,7 @@ import logging
 
 from scrapy.exceptions import IgnoreRequest
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class HttpError(IgnoreRequest):
diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py
index fb69a4631..a90f9f1e0 100644
--- a/scrapy/contrib/spidermiddleware/offsite.py
+++ b/scrapy/contrib/spidermiddleware/offsite.py
@@ -11,7 +11,7 @@ from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 class OffsiteMiddleware(object):
 
diff --git a/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/contrib/spidermiddleware/urllength.py
index d3c716063..a904635d8 100644
--- a/scrapy/contrib/spidermiddleware/urllength.py
+++ b/scrapy/contrib/spidermiddleware/urllength.py
@@ -9,7 +9,7 @@ import logging
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class UrlLengthMiddleware(object):
diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py
index 845e2bc18..90fb9fb4e 100644
--- a/scrapy/contrib/spiders/sitemap.py
+++ b/scrapy/contrib/spiders/sitemap.py
@@ -6,7 +6,7 @@ from scrapy.http import Request, XmlResponse
 from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
 from scrapy.utils.gz import gunzip, is_gzipped
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class SitemapSpider(Spider):
diff --git a/scrapy/contrib/throttle.py b/scrapy/contrib/throttle.py
index 5f72c81fc..a60b0cd83 100644
--- a/scrapy/contrib/throttle.py
+++ b/scrapy/contrib/throttle.py
@@ -3,7 +3,7 @@ import logging
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class AutoThrottle(object):
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 11fbd35b9..31412a0f4 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -21,7 +21,7 @@ from scrapy.core.downloader.webclient import _parse
 from scrapy.utils.misc import load_object
 from scrapy import twisted_version
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class HTTP11DownloadHandler(object):
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index a978d3c08..e912af846 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -17,7 +17,7 @@ from scrapy.http import Response, Request
 from scrapy.utils.misc import load_object
 from scrapy.utils.reactor import CallLaterOnce
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class Slot(object):
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 0e1acacea..f199711ab 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -8,7 +8,7 @@ from scrapy.utils.reqser import request_to_dict, request_from_dict
 from scrapy.utils.misc import load_object
 from scrapy.utils.job import job_dir
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class Scheduler(object):
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 196f5392d..e00be12e9 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -16,7 +16,7 @@ from scrapy.http import Request, Response
 from scrapy.item import BaseItem
 from scrapy.core.spidermw import SpiderMiddlewareManager
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class Slot(object):
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 4eba6f83a..932fbd168 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -18,7 +18,7 @@ from scrapy.utils.misc import load_object
 from scrapy.utils.log import LogCounterHandler, configure_logging, log_scrapy_info
 from scrapy import signals
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class Crawler(object):
diff --git a/scrapy/dupefilter.py b/scrapy/dupefilter.py
index 37376ad8a..739ba922f 100644
--- a/scrapy/dupefilter.py
+++ b/scrapy/dupefilter.py
@@ -33,7 +33,7 @@ class RFPDupeFilter(BaseDupeFilter):
         self.fingerprints = set()
         self.logdupes = True
         self.debug = debug
-        self.logger = logging.getLogger('scrapy')
+        self.logger = logging.getLogger(__name__)
         if path:
             self.file = open(os.path.join(path, 'requests.seen'), 'a+')
             self.fingerprints.update(x.rstrip() for x in self.file)
diff --git a/scrapy/log.py b/scrapy/log.py
index af0473d46..6f8b5ead2 100644
--- a/scrapy/log.py
+++ b/scrapy/log.py
@@ -9,7 +9,7 @@ from twisted.python.failure import Failure
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 warnings.warn("Module `scrapy.log` has been deprecated, Scrapy now relies on "
               "the builtin Python library for logging. Read the updated "
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 7e38663cf..2b4c57980 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -22,7 +22,7 @@ else:
 from twisted.internet import defer, reactor, ssl
 from twisted.mail.smtp import ESMTPSenderFactory
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class MailSender(object):
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 917717de5..a7adc39e3 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -5,7 +5,7 @@ from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from scrapy.utils.defer import process_parallel, process_chain, process_chain_both
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class MiddlewareManager(object):
diff --git a/scrapy/statscol.py b/scrapy/statscol.py
index 3fe32ee81..62b037f36 100644
--- a/scrapy/statscol.py
+++ b/scrapy/statscol.py
@@ -4,7 +4,7 @@ Scrapy extension for collecting scraping stats
 import pprint
 import logging
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class StatsCollector(object):
diff --git a/scrapy/telnet.py b/scrapy/telnet.py
index 049ab32ed..d9add1d97 100644
--- a/scrapy/telnet.py
+++ b/scrapy/telnet.py
@@ -27,7 +27,7 @@ try:
 except ImportError:
     hpy = None
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 # signal to update telnet variables
 # args: telnet_vars
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 4f81b2d9c..58ddf6215 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -13,7 +13,7 @@ from scrapy.http import TextResponse, Response
 from scrapy.selector import Selector
 from scrapy.utils.python import re_rsearch, str_to_unicode
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 def xmliter(obj, nodename):
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index e2b1edcf7..ccfeb12e1 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -11,7 +11,7 @@ from twisted.python import log as twisted_log
 import scrapy
 from scrapy.settings import overridden_settings
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 class FailureFormatter(logging.Filter):
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index 091955b73..cab172e4c 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -9,7 +9,7 @@ from scrapy.xlib.pydispatch.dispatcher import Any, Anonymous, liveReceivers, \
     getAllReceivers, disconnect
 from scrapy.xlib.pydispatch.robustapply import robustApply
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 1df5e3769..7ed2d0c3b 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -6,7 +6,7 @@ import six
 from scrapy.spider import Spider
 from scrapy.utils.misc import  arg_to_iter
 
-logger = logging.getLogger('scrapy')
+logger = logging.getLogger(__name__)
 
 
 def iterate_spider_output(result):

From 6d1205063c50563cb9867ae254c152e1e8ad5afa Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 19 Mar 2015 14:25:38 -0300
Subject: [PATCH 0244/2974] Add a filter to replace '__name__' loggers with
 'scrapy'

---
 scrapy/utils/log.py     | 22 ++++++++++++++++++++++
 tests/test_utils_log.py | 38 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index ccfeb12e1..b54c24fc4 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -32,6 +32,27 @@ class FailureFormatter(logging.Filter):
         return True
 
 
+class TopLevelFormatter(logging.Filter):
+    """Keep only top level loggers's name (direct children from root) from
+    records.
+
+    This filter will replace Scrapy loggers' names with 'scrapy'. This mimics
+    the old Scrapy log behaviour and helps shortening long names.
+
+    Since it can't be set for just one logger (it won't propagate for its
+    children), it's going to be set in the root handler, with a parametrized
+    `loggers` list where it should act.
+    """
+
+    def __init__(self, loggers=None):
+        self.loggers = loggers or []
+
+    def filter(self, record):
+        if any(record.name.startswith(l + '.') for l in self.loggers):
+            record.name = record.name.split('.', 1)[0]
+        return True
+
+
 DEFAULT_LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
@@ -91,6 +112,7 @@ def configure_logging(settings=None):
         )
         handler.setFormatter(formatter)
         handler.setLevel(settings.get('LOG_LEVEL'))
+        handler.addFilter(TopLevelFormatter(['scrapy']))
         logging.root.addHandler(handler)
 
 
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 42c4d6ea4..e6b8c77d9 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -8,7 +8,8 @@ import unittest
 from testfixtures import LogCapture
 from twisted.python.failure import Failure
 
-from scrapy.utils.log import FailureFormatter, LogCounterHandler, StreamLogger
+from scrapy.utils.log import (FailureFormatter, TopLevelFormatter,
+                              LogCounterHandler, StreamLogger)
 from scrapy.utils.test import get_crawler
 
 
@@ -49,6 +50,41 @@ class FailureFormatterTest(unittest.TestCase):
                                   'test log msg' + os.linesep + '3')
 
 
+class TopLevelFormatterTest(unittest.TestCase):
+
+    def setUp(self):
+        self.handler = LogCapture()
+        self.handler.addFilter(TopLevelFormatter(['test']))
+
+    def test_top_level_logger(self):
+        logger = logging.getLogger('test')
+        with self.handler as l:
+            logger.warning('test log msg')
+
+        l.check(('test', 'WARNING', 'test log msg'))
+
+    def test_children_logger(self):
+        logger = logging.getLogger('test.test1')
+        with self.handler as l:
+            logger.warning('test log msg')
+
+        l.check(('test', 'WARNING', 'test log msg'))
+
+    def test_overlapping_name_logger(self):
+        logger = logging.getLogger('test2')
+        with self.handler as l:
+            logger.warning('test log msg')
+
+        l.check(('test2', 'WARNING', 'test log msg'))
+
+    def test_different_name_logger(self):
+        logger = logging.getLogger('different')
+        with self.handler as l:
+            logger.warning('test log msg')
+
+        l.check(('different', 'WARNING', 'test log msg'))
+
+
 class LogCounterHandlerTest(unittest.TestCase):
 
     def setUp(self):

From 7a92dae4c8638e275a6e85d667b21e771ebb6e91 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 19 Mar 2015 15:02:54 -0300
Subject: [PATCH 0245/2974] Change Scrapy log output through docs

---
 docs/intro/tutorial.rst               | 17 +++++++--------
 docs/topics/benchmarking.rst          | 30 +++++++++++++--------------
 docs/topics/downloader-middleware.rst |  8 +++----
 docs/topics/shell.rst                 |  6 +++---
 4 files changed, 30 insertions(+), 31 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index ab2302154..479b17f99 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -154,15 +154,14 @@ similar to this::
     2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled downloader middlewares: ...
     2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled spider middlewares: ...
     2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled item pipelines: ...
-    2014-01-23 18:13:07-0400 [dmoz] INFO: Spider opened
-    2014-01-23 18:13:08-0400 [dmoz] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/> (referer: None)
-    2014-01-23 18:13:09-0400 [dmoz] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> (referer: None)
-    2014-01-23 18:13:09-0400 [dmoz] INFO: Closing spider (finished)
+    2014-01-23 18:13:07-0400 [scrapy] INFO: Spider opened
+    2014-01-23 18:13:08-0400 [scrapy] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/> (referer: None)
+    2014-01-23 18:13:09-0400 [scrapy] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> (referer: None)
+    2014-01-23 18:13:09-0400 [scrapy] INFO: Closing spider (finished)
 
 
 .. note::
-    The lines containing ``[dmoz]`` correspond to the log messages for
-    our spider. You can see a log line for each URL defined in ``start_urls``.
+    At the end you can see a log line for each URL defined in ``start_urls``.
     Because these URLs are the starting ones, they have no referrers, which is
     shown at the end of the log line, where it says ``(referer: None)``.
 
@@ -268,7 +267,7 @@ This is what the shell looks like::
 
     [ ... Scrapy log here ... ]
 
-    2014-01-23 17:11:42-0400 [default] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> (referer: None)
+    2014-01-23 17:11:42-0400 [scrapy] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> (referer: None)
     [s] Available Scrapy objects:
     [s]   crawler    <scrapy.crawler.Crawler object at 0x3636b50>
     [s]   item       {}
@@ -424,11 +423,11 @@ Spider would be like this::
 
 Now crawling dmoz.org yields ``DmozItem`` objects::
 
-   [dmoz] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
+   [scrapy] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
         {'desc': [u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.\n],
          'link': [u'http://gnosis.cx/TPiP/'],
          'title': [u'Text Processing in Python']}
-   [dmoz] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
+   [scrapy] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
         {'desc': [u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n'],
          'link': [u'http://www.informit.com/store/product.aspx?isbn=0130211192'],
          'title': [u'XML Processing with Python']}
diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst
index 9ae6dd2cb..632190067 100644
--- a/docs/topics/benchmarking.rst
+++ b/docs/topics/benchmarking.rst
@@ -19,20 +19,20 @@ To run it use::
 You should see an output like this::
 
     2013-05-16 13:08:46-0300 [scrapy] INFO: Scrapy 0.17.0 started (bot: scrapybot)
-    2013-05-16 13:08:47-0300 [follow] INFO: Spider opened
-    2013-05-16 13:08:47-0300 [follow] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:48-0300 [follow] INFO: Crawled 74 pages (at 4440 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:49-0300 [follow] INFO: Crawled 143 pages (at 4140 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:50-0300 [follow] INFO: Crawled 210 pages (at 4020 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:51-0300 [follow] INFO: Crawled 274 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:52-0300 [follow] INFO: Crawled 343 pages (at 4140 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:53-0300 [follow] INFO: Crawled 410 pages (at 4020 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:54-0300 [follow] INFO: Crawled 474 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:55-0300 [follow] INFO: Crawled 538 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:56-0300 [follow] INFO: Crawled 602 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:57-0300 [follow] INFO: Closing spider (closespider_timeout)
-    2013-05-16 13:08:57-0300 [follow] INFO: Crawled 666 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:57-0300 [follow] INFO: Dumping Scrapy stats:
+    2013-05-16 13:08:47-0300 [scrapy] INFO: Spider opened
+    2013-05-16 13:08:47-0300 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
+    2013-05-16 13:08:48-0300 [scrapy] INFO: Crawled 74 pages (at 4440 pages/min), scraped 0 items (at 0 items/min)
+    2013-05-16 13:08:49-0300 [scrapy] INFO: Crawled 143 pages (at 4140 pages/min), scraped 0 items (at 0 items/min)
+    2013-05-16 13:08:50-0300 [scrapy] INFO: Crawled 210 pages (at 4020 pages/min), scraped 0 items (at 0 items/min)
+    2013-05-16 13:08:51-0300 [scrapy] INFO: Crawled 274 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
+    2013-05-16 13:08:52-0300 [scrapy] INFO: Crawled 343 pages (at 4140 pages/min), scraped 0 items (at 0 items/min)
+    2013-05-16 13:08:53-0300 [scrapy] INFO: Crawled 410 pages (at 4020 pages/min), scraped 0 items (at 0 items/min)
+    2013-05-16 13:08:54-0300 [scrapy] INFO: Crawled 474 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
+    2013-05-16 13:08:55-0300 [scrapy] INFO: Crawled 538 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
+    2013-05-16 13:08:56-0300 [scrapy] INFO: Crawled 602 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
+    2013-05-16 13:08:57-0300 [scrapy] INFO: Closing spider (closespider_timeout)
+    2013-05-16 13:08:57-0300 [scrapy] INFO: Crawled 666 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
+    2013-05-16 13:08:57-0300 [scrapy] INFO: Dumping Scrapy stats:
         {'downloader/request_bytes': 231508,
          'downloader/request_count': 682,
          'downloader/request_method_count/GET': 682,
@@ -49,7 +49,7 @@ You should see an output like this::
          'scheduler/enqueued': 12767,
          'scheduler/enqueued/memory': 12767,
          'start_time': datetime.datetime(2013, 5, 16, 16, 8, 47, 676539)}
-    2013-05-16 13:08:57-0300 [follow] INFO: Spider closed (closespider_timeout)
+    2013-05-16 13:08:57-0300 [scrapy] INFO: Spider closed (closespider_timeout)
 
 That tells you that Scrapy is able to crawl about 3900 pages per minute in the
 hardware where you run it. Note that this is a very simple spider intended to
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index df679daea..3d24b29ad 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -232,14 +232,14 @@ header) and all cookies received in responses (ie. ``Set-Cookie`` header).
 
 Here's an example of a log with :setting:`COOKIES_DEBUG` enabled::
 
-    2011-04-06 14:35:10-0300 [diningcity] INFO: Spider opened
-    2011-04-06 14:35:10-0300 [diningcity] DEBUG: Sending cookies to: <GET http://www.diningcity.com/netherlands/index.html>
+    2011-04-06 14:35:10-0300 [scrapy] INFO: Spider opened
+    2011-04-06 14:35:10-0300 [scrapy] DEBUG: Sending cookies to: <GET http://www.diningcity.com/netherlands/index.html>
             Cookie: clientlanguage_nl=en_EN
-    2011-04-06 14:35:14-0300 [diningcity] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html>
+    2011-04-06 14:35:14-0300 [scrapy] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html>
             Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/
             Set-Cookie: ip_isocode=US
             Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/
-    2011-04-06 14:49:50-0300 [diningcity] DEBUG: Crawled (200) <GET http://www.diningcity.com/netherlands/index.html> (referer: None)
+    2011-04-06 14:49:50-0300 [scrapy] DEBUG: Crawled (200) <GET http://www.diningcity.com/netherlands/index.html> (referer: None)
     [...]
 
 
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 8fce0cea1..3b875fec5 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -187,8 +187,8 @@ Here's an example of how you would call it from your spider::
 
 When you run the spider, you will get something similar to this::
 
-    2014-01-23 17:48:31-0400 [myspider] DEBUG: Crawled (200) <GET http://example.com> (referer: None)
-    2014-01-23 17:48:31-0400 [myspider] DEBUG: Crawled (200) <GET http://example.org> (referer: None)
+    2014-01-23 17:48:31-0400 [scrapy] DEBUG: Crawled (200) <GET http://example.com> (referer: None)
+    2014-01-23 17:48:31-0400 [scrapy] DEBUG: Crawled (200) <GET http://example.org> (referer: None)
     [s] Available Scrapy objects:
     [s]   crawler    <scrapy.crawler.Crawler object at 0x1e16b50>
     ...
@@ -211,7 +211,7 @@ Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the
 crawling::
 
     >>> ^D
-    2014-01-23 17:50:03-0400 [myspider] DEBUG: Crawled (200) <GET http://example.net> (referer: None)
+    2014-01-23 17:50:03-0400 [scrapy] DEBUG: Crawled (200) <GET http://example.net> (referer: None)
     ...
 
 Note that you can't use the ``fetch`` shortcut here since the Scrapy engine is

From 4858af4e946919b80de6e67204f5a2925bc85800 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 22 Apr 2015 14:34:11 -0300
Subject: [PATCH 0246/2974] Fix backward compatible functions in scrapy.log

---
 scrapy/log.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scrapy/log.py b/scrapy/log.py
index 6f8b5ead2..c3f9c4227 100644
--- a/scrapy/log.py
+++ b/scrapy/log.py
@@ -27,13 +27,17 @@ CRITICAL = logging.CRITICAL
 SILENT = CRITICAL + 1
 
 
-def msg(message, _level=logging.INFO, **kw):
+def msg(message=None, _level=logging.INFO, **kw):
     warnings.warn('log.msg has been deprecated, create a python logger and '
                   'log through it instead',
                   ScrapyDeprecationWarning, stacklevel=2)
 
     level = kw.pop('level', _level)
-    logger.log(level, message, kw)
+    message = kw.pop('format', message)
+    # NOTE: logger.log doesn't handle well passing empty dictionaries with format
+    # arguments because of some weird use-case:
+    # https://hg.python.org/cpython/file/648dcafa7e5f/Lib/logging/__init__.py#l269
+    logger.log(level, message, *[kw] if kw else [])
 
 
 def err(_stuff=None, _why=None, **kw):
@@ -44,4 +48,4 @@ def err(_stuff=None, _why=None, **kw):
     level = kw.pop('level', logging.ERROR)
     failure = kw.pop('failure', _stuff) or Failure()
     message = kw.pop('why', _why) or failure.value
-    logger.log(level, message, kw, extra={'failure': failure})
+    logger.log(level, message, *[kw] if kw else [], extra={'failure': failure})

From 1d8f8221e6234959f0be4673a80b142451d59bc3 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 22 Apr 2015 16:50:32 -0300
Subject: [PATCH 0247/2974] Add backward compatibility to LogFormatter

---
 scrapy/core/engine.py  |  3 ++-
 scrapy/core/scraper.py |  5 +++--
 scrapy/utils/log.py    | 26 ++++++++++++++++++++++++++
 3 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index e912af846..63267f208 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -16,6 +16,7 @@ from scrapy.exceptions import DontCloseSpider
 from scrapy.http import Response, Request
 from scrapy.utils.misc import load_object
 from scrapy.utils.reactor import CallLaterOnce
+from scrapy.utils.log import logformatter_adapter
 
 logger = logging.getLogger(__name__)
 
@@ -204,7 +205,7 @@ class ExecutionEngine(object):
             if isinstance(response, Response):
                 response.request = request # tie request to response received
                 logkws = self.logformatter.crawled(request, response, spider)
-                logger._log(extra={'spider': spider}, **logkws)
+                logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
                 self.signals.send_catch_log(signal=signals.response_received, \
                     response=response, request=request, spider=spider)
             return response
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index e00be12e9..e5d8acea2 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -10,6 +10,7 @@ from twisted.internet import defer
 from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.misc import load_object
+from scrapy.utils.log import logformatter_adapter
 from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy import signals
 from scrapy.http import Request, Response
@@ -220,7 +221,7 @@ class Scraper(object):
             ex = output.value
             if isinstance(ex, DropItem):
                 logkws = self.logformatter.dropped(item, ex, response, spider)
-                logger._log(extra={'spider': spider}, **logkws)
+                logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
                 return self.signals.send_catch_log_deferred(
                     signal=signals.item_dropped, item=item, response=response,
                     spider=spider, exception=output.value)
@@ -229,7 +230,7 @@ class Scraper(object):
                              extra={'spider': spider, 'failure': output})
         else:
             logkws = self.logformatter.scraped(output, response, spider)
-            logger._log(extra={'spider': spider}, **logkws)
+            logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
             return self.signals.send_catch_log_deferred(
                 signal=signals.item_scraped, item=output, response=response,
                 spider=spider)
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index b54c24fc4..c9250d7a1 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -3,6 +3,7 @@
 import os
 import sys
 import logging
+import warnings
 from logging.config import dictConfig
 
 from twisted.python.failure import Failure
@@ -10,6 +11,7 @@ from twisted.python import log as twisted_log
 
 import scrapy
 from scrapy.settings import overridden_settings
+from scrapy.exceptions import ScrapyDeprecationWarning
 
 logger = logging.getLogger(__name__)
 
@@ -153,3 +155,27 @@ class LogCounterHandler(logging.Handler):
     def emit(self, record):
         sname = 'log_count/{}'.format(record.levelname)
         self.crawler.stats.inc_value(sname)
+
+
+def logformatter_adapter(logkws):
+    """
+    Helper that takes the dictionary output from the methods in LogFormatter
+    and adapts it into a tuple of positional arguments for logger.log calls,
+    handling backward compatibility as well.
+    """
+    if not {'level', 'msg', 'args'} <= set(logkws):
+        warnings.warn('Missing keys in LogFormatter method',
+                      ScrapyDeprecationWarning)
+
+    if 'format' in logkws:
+        warnings.warn('`format` key in LogFormatter methods has been '
+                      'deprecated, use `msg` instead',
+                      ScrapyDeprecationWarning)
+
+    level = logkws.get('level', logging.INFO)
+    message = logkws.get('format', logkws.get('msg'))
+    # NOTE: This also handles 'args' being an empty dict, that case doesn't
+    # play well in logger.log calls
+    args = logkws if not logkws.get('args') else logkws['args']
+
+    return (level, message, args)

From cc4c31e42673813be0739f3a01dfb5d7fb9417cf Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 23 Apr 2015 15:08:04 -0300
Subject: [PATCH 0248/2974] Relax SpiderLoader interface check

---
 scrapy/crawler.py     | 12 ++++++++++--
 tests/test_crawler.py |  8 +++++---
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index b4706919a..174a5ad74 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -3,7 +3,7 @@ import signal
 import warnings
 
 from twisted.internet import reactor, defer
-from zope.interface.verify import verifyClass
+from zope.interface.verify import verifyClass, DoesNotImplement
 
 from scrapy.core.engine import ExecutionEngine
 from scrapy.resolver import CachingThreadedResolver
@@ -195,5 +195,13 @@ def _get_spider_loader(settings):
     cls_path = settings.get('SPIDER_MANAGER_CLASS',
                             settings.get('SPIDER_LOADER_CLASS'))
     loader_cls = load_object(cls_path)
-    verifyClass(ISpiderLoader, loader_cls)
+    try:
+        verifyClass(ISpiderLoader, loader_cls)
+    except DoesNotImplement:
+        warnings.warn(
+            'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does '
+            'not fully implement scrapy.interfaces.ISpiderLoader interface. '
+            'Please add all missing methods to avoid unexpected runtime errors.',
+            category=ScrapyDeprecationWarning, stacklevel=2
+        )
     return loader_cls.from_settings(settings.frozencopy())
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 6a282ed84..906573887 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,8 +1,6 @@
 import warnings
 import unittest
 
-from zope.interface.verify import DoesNotImplement
-
 from scrapy.crawler import Crawler, CrawlerRunner
 from scrapy.settings import Settings, default_settings
 from scrapy.spiderloader import SpiderLoader
@@ -71,8 +69,12 @@ class CrawlerRunnerTestCase(unittest.TestCase):
         settings = Settings({
             'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface'
         })
-        with self.assertRaises(DoesNotImplement):
+        with warnings.catch_warnings(record=True) as w, \
+                self.assertRaises(AttributeError):
             CrawlerRunner(settings)
+            self.assertEqual(len(w), 1)
+            self.assertIn("SPIDER_LOADER_CLASS", str(w[0].message))
+            self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message))
 
     def test_crawler_runner_accepts_dict(self):
         runner = CrawlerRunner({'foo': 'bar'})

From 3d3633f3d2b6829f9bf90242cfc2c61539d006c4 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Sat, 25 Apr 2015 11:20:20 -0300
Subject: [PATCH 0249/2974] favoring web scraping over screen scraping in the
 descriptions

---
 README.rst                | 2 +-
 debian/control            | 4 ++--
 docs/intro/overview.rst   | 8 +++-----
 docs/topics/selectors.rst | 2 +-
 scrapy/__init__.py        | 2 +-
 setup.py                  | 2 +-
 6 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/README.rst b/README.rst
index 56bfad130..6ca5deb49 100644
--- a/README.rst
+++ b/README.rst
@@ -17,7 +17,7 @@ Scrapy
 Overview
 ========
 
-Scrapy is a fast high-level web crawling and screen scraping framework, used to
+Scrapy is a fast high-level web crawling and web scraping framework, used to
 crawl websites and extract structured data from their pages. It can be used for
 a wide range of purposes, from data mining to monitoring and automated testing.
 
diff --git a/debian/control b/debian/control
index c3dae341e..15743767b 100644
--- a/debian/control
+++ b/debian/control
@@ -13,8 +13,8 @@ Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl,
 Recommends: python-setuptools
 Conflicts: python-scrapy, scrapy, scrapy-0.11
 Provides: python-scrapy, scrapy
-Description: Python web crawling and screen scraping framework
- Scrapy is a fast high-level web crawling and screen scraping framework, 
+Description: Python web crawling and web scraping framework
+ Scrapy is a fast high-level web crawling and web scraping framework,
  used to crawl websites and extract structured data from their pages. 
  It can be used for a wide range of purposes, from data mining to 
  monitoring and automated testing.
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 8777ac891..820581457 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -8,10 +8,9 @@ Scrapy is an application framework for crawling web sites and extracting
 structured data which can be used for a wide range of useful applications, like
 data mining, information processing or historical archival.
 
-Even though Scrapy was originally designed for `screen scraping`_ (more
-precisely, `web scraping`_), it can also be used to extract data using APIs
-(such as `Amazon Associates Web Services`_) or as a general purpose web
-crawler.
+Even though Scrapy was originally designed for `web scraping`_, it can also be
+used to extract data using APIs (such as `Amazon Associates Web Services`_) or
+as a general purpose web crawler.
 
 
 Walk-through of an example spider
@@ -171,7 +170,6 @@ your code in Scrapy projects and `join the community`_. Thanks for your
 interest!
 
 .. _join the community: http://scrapy.org/community/
-.. _screen scraping: http://en.wikipedia.org/wiki/Screen_scraping
 .. _web scraping: http://en.wikipedia.org/wiki/Web_scraping
 .. _Amazon Associates Web Services: http://aws.amazon.com/associates/
 .. _Amazon S3: http://aws.amazon.com/s3/
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index ccac9ec0b..231119f3b 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -8,7 +8,7 @@ When you're scraping web pages, the most common task you need to perform is
 to extract data from the HTML source. There are several libraries available to
 achieve this:
 
- * `BeautifulSoup`_ is a very popular screen scraping library among Python
+ * `BeautifulSoup`_ is a very popular web scraping library among Python
    programmers which constructs a Python object based on the structure of the
    HTML code and also deals with bad markup reasonably well, but it has one
    drawback: it's slow.
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 1f38299d2..31ad2ff02 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -1,5 +1,5 @@
 """
-Scrapy - a web crawling and screen scraping framework written for Python
+Scrapy - a web crawling and web scraping framework written for Python
 """
 
 __all__ = ['__version__', 'version_info', 'optional_features', 'twisted_version',
diff --git a/setup.py b/setup.py
index d463bccd9..bacdca358 100644
--- a/setup.py
+++ b/setup.py
@@ -10,7 +10,7 @@ setup(
     name='Scrapy',
     version=version,
     url='http://scrapy.org',
-    description='A high-level Web Crawling and Screen Scraping framework',
+    description='A high-level Web Crawling and Web Scraping framework',
     long_description=open('README.rst').read(),
     author='Scrapy developers',
     maintainer='Pablo Hoffman',

From de6501ed1b2ec6877ed8a9ec9c0f20a079870115 Mon Sep 17 00:00:00 2001
From: Wieland Hoffmann <mineo@users.noreply.github.com>
Date: Wed, 29 Apr 2015 22:31:48 +0200
Subject: [PATCH 0250/2974] Remove a duplicate word

---
 docs/topics/api.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 48347120b..89e300b39 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -81,8 +81,8 @@ how you :ref:`configure the downloader middlewares
         The execution engine, which coordinates the core crawling logic
         between the scheduler, downloader and spiders.
 
-        Some extension may want to access the Scrapy engine, to modify inspect
-        or modify the downloader and scheduler behaviour, although this is an
+        Some extension may want to access the Scrapy engine, to inspect  or 
+        modify the downloader and scheduler behaviour, although this is an
         advanced use and this API is not yet stable.
 
     .. attribute:: spider

From d7c444fefbd9b69698618d3d18b0c3e4858ec3e5 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 20 Apr 2015 21:23:05 -0300
Subject: [PATCH 0251/2974] Move scrapy/contrib/downloadermiddleware to
 scrapy/downloadermiddlewares

---
 docs/faq.rst                                  |  4 +--
 docs/topics/downloader-middleware.rst         | 32 +++++++++----------
 docs/topics/settings.rst                      | 32 +++++++++----------
 .../downloadermiddleware/decompression.py     |  4 +--
 .../__init__.py                               |  0
 .../ajaxcrawl.py                              |  0
 .../chunked.py                                |  0
 .../cookies.py                                |  0
 .../decompression.py                          |  0
 .../defaultheaders.py                         |  0
 .../downloadtimeout.py                        |  0
 .../httpauth.py                               |  0
 .../httpcache.py                              |  0
 .../httpcompression.py                        |  0
 .../httpproxy.py                              |  0
 .../redirect.py                               |  0
 .../retry.py                                  |  0
 .../robotstxt.py                              |  0
 .../stats.py                                  |  0
 .../useragent.py                              |  0
 scrapy/settings/default_settings.py           | 30 ++++++++---------
 scrapy/utils/misc.py                          |  2 +-
 tests/py3-ignores.txt                         |  3 ++
 ...test_downloadermiddleware_ajaxcrawlable.py |  4 +--
 tests/test_downloadermiddleware_cookies.py    |  2 +-
 ...test_downloadermiddleware_decompression.py |  4 +--
 ...est_downloadermiddleware_defaultheaders.py |  2 +-
 ...st_downloadermiddleware_downloadtimeout.py |  2 +-
 tests/test_downloadermiddleware_httpauth.py   |  2 +-
 tests/test_downloadermiddleware_httpcache.py  |  2 +-
 ...st_downloadermiddleware_httpcompression.py |  2 +-
 tests/test_downloadermiddleware_httpproxy.py  |  2 +-
 tests/test_downloadermiddleware_redirect.py   |  2 +-
 tests/test_downloadermiddleware_retry.py      |  2 +-
 tests/test_downloadermiddleware_robotstxt.py  |  2 +-
 tests/test_downloadermiddleware_stats.py      |  4 +--
 tests/test_downloadermiddleware_useragent.py  |  2 +-
 37 files changed, 72 insertions(+), 69 deletions(-)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/__init__.py (100%)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/ajaxcrawl.py (100%)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/chunked.py (100%)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/cookies.py (100%)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/decompression.py (100%)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/defaultheaders.py (100%)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/downloadtimeout.py (100%)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/httpauth.py (100%)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/httpcache.py (100%)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/httpcompression.py (100%)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/httpproxy.py (100%)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/redirect.py (100%)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/retry.py (100%)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/robotstxt.py (100%)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/stats.py (100%)
 rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/useragent.py (100%)

diff --git a/docs/faq.rst b/docs/faq.rst
index b92d173dd..f593e4f16 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -64,7 +64,7 @@ Does Scrapy work with HTTP proxies?
 
 Yes. Support for HTTP proxies is provided (since Scrapy 0.8) through the HTTP
 Proxy downloader middleware. See
-:class:`~scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware`.
+:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`.
 
 How can I scrape an item with attributes in different pages?
 ------------------------------------------------------------
@@ -113,7 +113,7 @@ See previous question.
 Can I use Basic HTTP Authentication in my spiders?
 --------------------------------------------------
 
-Yes, see :class:`~scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware`.
+Yes, see :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`.
 
 Why does Scrapy download pages in English instead of my native language?
 ------------------------------------------------------------------------
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 3d24b29ad..be19bc44f 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -42,7 +42,7 @@ as its value.  For example, if you want to disable the user-agent middleware::
 
     DOWNLOADER_MIDDLEWARES = {
         'myproject.middlewares.CustomDownloaderMiddleware': 543,
-        'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': None,
+        'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
     }
 
 Finally, keep in mind that some middlewares may need to be enabled through a
@@ -54,7 +54,7 @@ Writing your own downloader middleware
 Each middleware component is a Python class that defines one or
 more of the following methods:
 
-.. module:: scrapy.contrib.downloadermiddleware
+.. module:: scrapy.downloadermiddlewares
 
 .. class:: DownloaderMiddleware
 
@@ -169,7 +169,7 @@ For a list of the components enabled by default (and their orders) see the
 CookiesMiddleware
 -----------------
 
-.. module:: scrapy.contrib.downloadermiddleware.cookies
+.. module:: scrapy.downloadermiddlewares.cookies
    :synopsis: Cookies Downloader Middleware
 
 .. class:: CookiesMiddleware
@@ -246,7 +246,7 @@ Here's an example of a log with :setting:`COOKIES_DEBUG` enabled::
 DefaultHeadersMiddleware
 ------------------------
 
-.. module:: scrapy.contrib.downloadermiddleware.defaultheaders
+.. module:: scrapy.downloadermiddlewares.defaultheaders
    :synopsis: Default Headers Downloader Middleware
 
 .. class:: DefaultHeadersMiddleware
@@ -257,7 +257,7 @@ DefaultHeadersMiddleware
 DownloadTimeoutMiddleware
 -------------------------
 
-.. module:: scrapy.contrib.downloadermiddleware.downloadtimeout
+.. module:: scrapy.downloadermiddlewares.downloadtimeout
    :synopsis: Download timeout middleware
 
 .. class:: DownloadTimeoutMiddleware
@@ -275,7 +275,7 @@ DownloadTimeoutMiddleware
 HttpAuthMiddleware
 ------------------
 
-.. module:: scrapy.contrib.downloadermiddleware.httpauth
+.. module:: scrapy.downloadermiddlewares.httpauth
    :synopsis: HTTP Auth downloader middleware
 
 .. class:: HttpAuthMiddleware
@@ -304,7 +304,7 @@ HttpAuthMiddleware
 HttpCacheMiddleware
 -------------------
 
-.. module:: scrapy.contrib.downloadermiddleware.httpcache
+.. module:: scrapy.downloadermiddlewares.httpcache
    :synopsis: HTTP Cache downloader middleware
 
 .. class:: HttpCacheMiddleware
@@ -579,7 +579,7 @@ This setting is specific to the Filesystem backend.
 HttpCompressionMiddleware
 -------------------------
 
-.. module:: scrapy.contrib.downloadermiddleware.httpcompression
+.. module:: scrapy.downloadermiddlewares.httpcompression
    :synopsis: Http Compression Middleware
 
 .. class:: HttpCompressionMiddleware
@@ -603,7 +603,7 @@ Whether the Compression middleware will be enabled.
 ChunkedTransferMiddleware
 -------------------------
 
-.. module:: scrapy.contrib.downloadermiddleware.chunked
+.. module:: scrapy.downloadermiddlewares.chunked
    :synopsis: Chunked Transfer Middleware
 
 .. class:: ChunkedTransferMiddleware
@@ -613,7 +613,7 @@ ChunkedTransferMiddleware
 HttpProxyMiddleware
 -------------------
 
-.. module:: scrapy.contrib.downloadermiddleware.httpproxy
+.. module:: scrapy.downloadermiddlewares.httpproxy
    :synopsis: Http Proxy Middleware
 
 .. versionadded:: 0.8
@@ -641,7 +641,7 @@ HttpProxyMiddleware
 RedirectMiddleware
 ------------------
 
-.. module:: scrapy.contrib.downloadermiddleware.redirect
+.. module:: scrapy.downloadermiddlewares.redirect
    :synopsis: Redirection Middleware
 
 .. class:: RedirectMiddleware
@@ -731,7 +731,7 @@ The maximum meta-refresh delay (in seconds) to follow the redirection.
 RetryMiddleware
 ---------------
 
-.. module:: scrapy.contrib.downloadermiddleware.retry
+.. module:: scrapy.downloadermiddlewares.retry
    :synopsis: Retry Middleware
 
 .. class:: RetryMiddleware
@@ -800,7 +800,7 @@ connections lost, etc) are always retried.
 RobotsTxtMiddleware
 -------------------
 
-.. module:: scrapy.contrib.downloadermiddleware.robotstxt
+.. module:: scrapy.downloadermiddlewares.robotstxt
    :synopsis: robots.txt middleware
 
 .. class:: RobotsTxtMiddleware
@@ -828,7 +828,7 @@ the request will be ignored by this middleware even if
 DownloaderStats
 ---------------
 
-.. module:: scrapy.contrib.downloadermiddleware.stats
+.. module:: scrapy.downloadermiddlewares.stats
    :synopsis: Downloader Stats Middleware
 
 .. class:: DownloaderStats
@@ -842,7 +842,7 @@ DownloaderStats
 UserAgentMiddleware
 -------------------
 
-.. module:: scrapy.contrib.downloadermiddleware.useragent
+.. module:: scrapy.downloadermiddlewares.useragent
    :synopsis: User Agent Middleware
 
 .. class:: UserAgentMiddleware
@@ -857,7 +857,7 @@ UserAgentMiddleware
 AjaxCrawlMiddleware
 -------------------
 
-.. module:: scrapy.contrib.downloadermiddleware.ajaxcrawl
+.. module:: scrapy.downloadermiddlewares.ajaxcrawl
 
 .. class:: AjaxCrawlMiddleware
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 3936c04b8..3e68ad925 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -239,7 +239,7 @@ Default::
     }
 
 The default headers used for Scrapy HTTP Requests. They're populated in the
-:class:`~scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware`.
+:class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`.
 
 .. setting:: DEPTH_LIMIT
 
@@ -335,20 +335,20 @@ DOWNLOADER_MIDDLEWARES_BASE
 Default::
 
     {
-        'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100,
-        'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300,
-        'scrapy.contrib.downloadermiddleware.downloadtimeout.DownloadTimeoutMiddleware': 350,
-        'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400,
-        'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500,
-        'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550,
-        'scrapy.contrib.downloadermiddleware.redirect.MetaRefreshMiddleware': 580,
-        'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 590,
-        'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600,
-        'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700,
-        'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750,
-        'scrapy.contrib.downloadermiddleware.chunked.ChunkedTransferMiddleware': 830,
-        'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850,
-        'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900,
+        'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
+        'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
+        'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
+        'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400,
+        'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
+        'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550,
+        'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
+        'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
+        'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
+        'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
+        'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
+        'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830,
+        'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
+        'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
     }
 
 A dict containing the downloader middlewares enabled by default in Scrapy. You
@@ -837,7 +837,7 @@ ROBOTSTXT_OBEY
 
 Default: ``False``
 
-Scope: ``scrapy.contrib.downloadermiddleware.robotstxt``
+Scope: ``scrapy.downloadermiddlewares.robotstxt``
 
 If enabled, Scrapy will respect robots.txt policies. For more information see
 :ref:`topics-dlmw-robots`
diff --git a/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py
index 8b1d61b3d..1f8490587 100644
--- a/scrapy/contrib_exp/downloadermiddleware/decompression.py
+++ b/scrapy/contrib_exp/downloadermiddleware/decompression.py
@@ -1,7 +1,7 @@
 import warnings
 from scrapy.exceptions import ScrapyDeprecationWarning
 warnings.warn("Module `scrapy.contrib_exp.downloadermiddleware.decompression` is deprecated, "
-              "use `scrapy.contrib.downloadermiddleware.decompression` instead",
+              "use `scrapy.downloadermiddlewares.decompression` instead",
     ScrapyDeprecationWarning, stacklevel=2)
 
-from scrapy.contrib.downloadermiddleware.decompression import DecompressionMiddleware
+from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware
diff --git a/scrapy/contrib/downloadermiddleware/__init__.py b/scrapy/downloadermiddlewares/__init__.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/__init__.py
rename to scrapy/downloadermiddlewares/__init__.py
diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/ajaxcrawl.py
rename to scrapy/downloadermiddlewares/ajaxcrawl.py
diff --git a/scrapy/contrib/downloadermiddleware/chunked.py b/scrapy/downloadermiddlewares/chunked.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/chunked.py
rename to scrapy/downloadermiddlewares/chunked.py
diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/downloadermiddlewares/cookies.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/cookies.py
rename to scrapy/downloadermiddlewares/cookies.py
diff --git a/scrapy/contrib/downloadermiddleware/decompression.py b/scrapy/downloadermiddlewares/decompression.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/decompression.py
rename to scrapy/downloadermiddlewares/decompression.py
diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/defaultheaders.py
rename to scrapy/downloadermiddlewares/defaultheaders.py
diff --git a/scrapy/contrib/downloadermiddleware/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/downloadtimeout.py
rename to scrapy/downloadermiddlewares/downloadtimeout.py
diff --git a/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/httpauth.py
rename to scrapy/downloadermiddlewares/httpauth.py
diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/httpcache.py
rename to scrapy/downloadermiddlewares/httpcache.py
diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/httpcompression.py
rename to scrapy/downloadermiddlewares/httpcompression.py
diff --git a/scrapy/contrib/downloadermiddleware/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/httpproxy.py
rename to scrapy/downloadermiddlewares/httpproxy.py
diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/downloadermiddlewares/redirect.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/redirect.py
rename to scrapy/downloadermiddlewares/redirect.py
diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/downloadermiddlewares/retry.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/retry.py
rename to scrapy/downloadermiddlewares/retry.py
diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/robotstxt.py
rename to scrapy/downloadermiddlewares/robotstxt.py
diff --git a/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/downloadermiddlewares/stats.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/stats.py
rename to scrapy/downloadermiddlewares/stats.py
diff --git a/scrapy/contrib/downloadermiddleware/useragent.py b/scrapy/downloadermiddlewares/useragent.py
similarity index 100%
rename from scrapy/contrib/downloadermiddleware/useragent.py
rename to scrapy/downloadermiddlewares/useragent.py
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 0c3d7c5bd..3323386aa 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -80,21 +80,21 @@ DOWNLOADER_MIDDLEWARES = {}
 
 DOWNLOADER_MIDDLEWARES_BASE = {
     # Engine side
-    'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100,
-    'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300,
-    'scrapy.contrib.downloadermiddleware.downloadtimeout.DownloadTimeoutMiddleware': 350,
-    'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400,
-    'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500,
-    'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550,
-    'scrapy.contrib.downloadermiddleware.ajaxcrawl.AjaxCrawlMiddleware': 560,
-    'scrapy.contrib.downloadermiddleware.redirect.MetaRefreshMiddleware': 580,
-    'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 590,
-    'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600,
-    'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700,
-    'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750,
-    'scrapy.contrib.downloadermiddleware.chunked.ChunkedTransferMiddleware': 830,
-    'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850,
-    'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900,
+    'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
+    'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
+    'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
+    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400,
+    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
+    'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550,
+    'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
+    'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
+    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
+    'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
+    'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
+    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
+    'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830,
+    'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
+    'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
     # Downloader side
 }
 
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 3152db6c7..f1aa4fcdb 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -32,7 +32,7 @@ def load_object(path):
     """Load an object given its absolute object path, and return it.
 
     object can be a class, function, variable o instance.
-    path ie: 'scrapy.contrib.downloadermiddelware.redirect.RedirectMiddleware'
+    path ie: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'
     """
 
     try:
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index d27d5dd3c..cf814c15c 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -91,6 +91,9 @@ scrapy/contrib/linkextractors/htmlparser.py
 scrapy/contrib/downloadermiddleware/retry.py
 scrapy/contrib/downloadermiddleware/httpproxy.py
 scrapy/contrib/downloadermiddleware/cookies.py
+scrapy/downloadermiddlewares/retry.py
+scrapy/downloadermiddlewares/httpproxy.py
+scrapy/downloadermiddlewares/cookies.py
 scrapy/contrib/statsmailer.py
 scrapy/contrib/memusage.py
 scrapy/commands/deploy.py
diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py
index e73e62538..11de6e22d 100644
--- a/tests/test_downloadermiddleware_ajaxcrawlable.py
+++ b/tests/test_downloadermiddleware_ajaxcrawlable.py
@@ -1,11 +1,11 @@
 import unittest
 
-from scrapy.contrib.downloadermiddleware.ajaxcrawl import AjaxCrawlMiddleware
+from scrapy.downloadermiddlewares.ajaxcrawl import AjaxCrawlMiddleware
 from scrapy.spider import Spider
 from scrapy.http import Request, HtmlResponse, Response
 from scrapy.utils.test import get_crawler
 
-__doctests__ = ['scrapy.contrib.downloadermiddleware.ajaxcrawl']
+__doctests__ = ['scrapy.downloadermiddlewares.ajaxcrawl']
 
 class AjaxCrawlMiddlewareTest(unittest.TestCase):
     def setUp(self):
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 3b1853c82..7f491f258 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -3,7 +3,7 @@ import re
 
 from scrapy.http import Response, Request
 from scrapy.spider import Spider
-from scrapy.contrib.downloadermiddleware.cookies import CookiesMiddleware
+from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
 
 
 class CookiesMiddlewareTest(TestCase):
diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py
index f3f862604..7aca415ef 100644
--- a/tests/test_downloadermiddleware_decompression.py
+++ b/tests/test_downloadermiddleware_decompression.py
@@ -1,6 +1,6 @@
 from unittest import TestCase, main
 from scrapy.http import Response, XmlResponse
-from scrapy.contrib.downloadermiddleware.decompression import DecompressionMiddleware
+from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware
 from scrapy.spider import Spider
 from tests import get_testdata
 from scrapy.utils.test import assert_samelines
@@ -16,7 +16,7 @@ def _test_data(formats):
 
 
 class DecompressionMiddlewareTest(TestCase):
-    
+
     test_formats = ['tar', 'xml.bz2', 'xml.gz', 'zip']
     uncompressed_body, test_responses = _test_data(test_formats)
 
diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py
index 09973b367..26520a20a 100644
--- a/tests/test_downloadermiddleware_defaultheaders.py
+++ b/tests/test_downloadermiddleware_defaultheaders.py
@@ -1,7 +1,7 @@
 from unittest import TestCase
 import six
 
-from scrapy.contrib.downloadermiddleware.defaultheaders import DefaultHeadersMiddleware
+from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
 from scrapy.http import Request
 from scrapy.spider import Spider
 from scrapy.utils.test import get_crawler
diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py
index aba06686a..282d10829 100644
--- a/tests/test_downloadermiddleware_downloadtimeout.py
+++ b/tests/test_downloadermiddleware_downloadtimeout.py
@@ -1,6 +1,6 @@
 import unittest
 
-from scrapy.contrib.downloadermiddleware.downloadtimeout import DownloadTimeoutMiddleware
+from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware
 from scrapy.spider import Spider
 from scrapy.http import Request
 from scrapy.utils.test import get_crawler
diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py
index adfcd802d..cef65b336 100644
--- a/tests/test_downloadermiddleware_httpauth.py
+++ b/tests/test_downloadermiddleware_httpauth.py
@@ -1,7 +1,7 @@
 import unittest
 
 from scrapy.http import Request
-from scrapy.contrib.downloadermiddleware.httpauth import HttpAuthMiddleware
+from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
 from scrapy.spider import Spider
 
 class TestSpider(Spider):
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 663d67be0..9c1678488 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -12,7 +12,7 @@ from scrapy.spider import Spider
 from scrapy.settings import Settings
 from scrapy.exceptions import IgnoreRequest
 from scrapy.utils.test import get_crawler
-from scrapy.contrib.downloadermiddleware.httpcache import HttpCacheMiddleware
+from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware
 
 
 class _BaseTest(unittest.TestCase):
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 1cc6f44c1..98df6d608 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -5,7 +5,7 @@ from gzip import GzipFile
 
 from scrapy.spider import Spider
 from scrapy.http import Response, Request, HtmlResponse
-from scrapy.contrib.downloadermiddleware.httpcompression import HttpCompressionMiddleware
+from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware
 from tests import tests_datadir
 from w3lib.encoding import resolve_encoding
 
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 58825c6cf..90609879c 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -2,7 +2,7 @@ import os
 import sys
 from twisted.trial.unittest import TestCase, SkipTest
 
-from scrapy.contrib.downloadermiddleware.httpproxy import HttpProxyMiddleware
+from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response, Request
 from scrapy.spider import Spider
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index fb70b13ae..3f299f258 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -1,6 +1,6 @@
 import unittest
 
-from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware, MetaRefreshMiddleware
+from scrapy.downloadermiddlewares.redirect import RedirectMiddleware, MetaRefreshMiddleware
 from scrapy.spider import Spider
 from scrapy.exceptions import IgnoreRequest
 from scrapy.http import Request, Response, HtmlResponse
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 669abea05..969452cfb 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -5,7 +5,7 @@ from twisted.internet.error import TimeoutError, DNSLookupError, \
         ConnectionLost, TCPTimedOutError
 
 from scrapy import optional_features
-from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware
+from scrapy.downloadermiddlewares.retry import RetryMiddleware
 from scrapy.xlib.tx import ResponseFailed
 from scrapy.spider import Spider
 from scrapy.http import Request, Response
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 8b7ac7a6b..bc245dcec 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -3,7 +3,7 @@ import re
 from twisted.internet import reactor
 from twisted.internet.defer import Deferred
 from twisted.trial import unittest
-from scrapy.contrib.downloadermiddleware.robotstxt import RobotsTxtMiddleware
+from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index b790ff09a..64f2d0786 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -1,6 +1,6 @@
 from unittest import TestCase
 
-from scrapy.contrib.downloadermiddleware.stats import DownloaderStats
+from scrapy.downloadermiddlewares.stats import DownloaderStats
 from scrapy.http import Request, Response
 from scrapy.spider import Spider
 from scrapy.utils.test import get_crawler
@@ -22,7 +22,7 @@ class TestDownloaderStats(TestCase):
         self.mw.process_request(self.req, self.spider)
         self.assertEqual(self.crawler.stats.get_value('downloader/request_count', \
             spider=self.spider), 1)
-        
+
     def test_process_response(self):
         self.mw.process_response(self.req, self.res, self.spider)
         self.assertEqual(self.crawler.stats.get_value('downloader/response_count', \
diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py
index 909d03ba5..ddbb8f3ac 100644
--- a/tests/test_downloadermiddleware_useragent.py
+++ b/tests/test_downloadermiddleware_useragent.py
@@ -2,7 +2,7 @@ from unittest import TestCase
 
 from scrapy.spider import Spider
 from scrapy.http import Request
-from scrapy.contrib.downloadermiddleware.useragent import UserAgentMiddleware
+from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
 from scrapy.utils.test import get_crawler
 
 

From 6b4c00cc9bfc00e715bd8fe71f1e4b214e7575da Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 20 Apr 2015 22:15:41 -0300
Subject: [PATCH 0252/2974] scrapy/contrib/downloadermiddleware shims

---
 scrapy/contrib/downloadermiddleware/__init__.py        | 0
 scrapy/contrib/downloadermiddleware/ajaxcrawl.py       | 7 +++++++
 scrapy/contrib/downloadermiddleware/chunked.py         | 7 +++++++
 scrapy/contrib/downloadermiddleware/cookies.py         | 7 +++++++
 scrapy/contrib/downloadermiddleware/decompression.py   | 7 +++++++
 scrapy/contrib/downloadermiddleware/defaultheaders.py  | 7 +++++++
 scrapy/contrib/downloadermiddleware/downloadtimeout.py | 7 +++++++
 scrapy/contrib/downloadermiddleware/httpauth.py        | 7 +++++++
 scrapy/contrib/downloadermiddleware/httpcache.py       | 7 +++++++
 scrapy/contrib/downloadermiddleware/httpcompression.py | 7 +++++++
 scrapy/contrib/downloadermiddleware/httpproxy.py       | 7 +++++++
 scrapy/contrib/downloadermiddleware/redirect.py        | 7 +++++++
 scrapy/contrib/downloadermiddleware/retry.py           | 7 +++++++
 scrapy/contrib/downloadermiddleware/robotstxt.py       | 7 +++++++
 scrapy/contrib/downloadermiddleware/stats.py           | 7 +++++++
 scrapy/contrib/downloadermiddleware/useragent.py       | 7 +++++++
 16 files changed, 105 insertions(+)
 create mode 100644 scrapy/contrib/downloadermiddleware/__init__.py
 create mode 100644 scrapy/contrib/downloadermiddleware/ajaxcrawl.py
 create mode 100644 scrapy/contrib/downloadermiddleware/chunked.py
 create mode 100644 scrapy/contrib/downloadermiddleware/cookies.py
 create mode 100644 scrapy/contrib/downloadermiddleware/decompression.py
 create mode 100644 scrapy/contrib/downloadermiddleware/defaultheaders.py
 create mode 100644 scrapy/contrib/downloadermiddleware/downloadtimeout.py
 create mode 100644 scrapy/contrib/downloadermiddleware/httpauth.py
 create mode 100644 scrapy/contrib/downloadermiddleware/httpcache.py
 create mode 100644 scrapy/contrib/downloadermiddleware/httpcompression.py
 create mode 100644 scrapy/contrib/downloadermiddleware/httpproxy.py
 create mode 100644 scrapy/contrib/downloadermiddleware/redirect.py
 create mode 100644 scrapy/contrib/downloadermiddleware/retry.py
 create mode 100644 scrapy/contrib/downloadermiddleware/robotstxt.py
 create mode 100644 scrapy/contrib/downloadermiddleware/stats.py
 create mode 100644 scrapy/contrib/downloadermiddleware/useragent.py

diff --git a/scrapy/contrib/downloadermiddleware/__init__.py b/scrapy/contrib/downloadermiddleware/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
new file mode 100644
index 000000000..90ebc46b6
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.downloadermiddleware.ajaxcrawl` is deprecated, "
+              "use `scrapy.downloadermiddlewares.ajaxcrawl` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.downloadermiddlewares.ajaxcrawl import *
diff --git a/scrapy/contrib/downloadermiddleware/chunked.py b/scrapy/contrib/downloadermiddleware/chunked.py
new file mode 100644
index 000000000..1322c9083
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/chunked.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.downloadermiddleware.chunked` is deprecated, "
+              "use `scrapy.downloadermiddlewares.chunked` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.downloadermiddlewares.chunked import *
diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/contrib/downloadermiddleware/cookies.py
new file mode 100644
index 000000000..bad970690
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/cookies.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.downloadermiddleware.cookies` is deprecated, "
+              "use `scrapy.downloadermiddlewares.cookies` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.downloadermiddlewares.cookies import *
diff --git a/scrapy/contrib/downloadermiddleware/decompression.py b/scrapy/contrib/downloadermiddleware/decompression.py
new file mode 100644
index 000000000..a541aa61e
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/decompression.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.downloadermiddleware.decompression` is deprecated, "
+              "use `scrapy.downloadermiddlewares.decompression` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.downloadermiddlewares.decompression import *
diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/contrib/downloadermiddleware/defaultheaders.py
new file mode 100644
index 000000000..cf023dc8f
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/defaultheaders.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.downloadermiddleware.defaultheaders` is deprecated, "
+              "use `scrapy.downloadermiddlewares.defaultheaders` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.downloadermiddlewares.defaultheaders import *
diff --git a/scrapy/contrib/downloadermiddleware/downloadtimeout.py b/scrapy/contrib/downloadermiddleware/downloadtimeout.py
new file mode 100644
index 000000000..84bd06acf
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/downloadtimeout.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.downloadermiddleware.downloadtimeout` is deprecated, "
+              "use `scrapy.downloadermiddlewares.downloadtimeout` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.downloadermiddlewares.downloadtimeout import *
diff --git a/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/contrib/downloadermiddleware/httpauth.py
new file mode 100644
index 000000000..a37ffa0dc
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/httpauth.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpauth` is deprecated, "
+              "use `scrapy.downloadermiddlewares.httpauth` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.downloadermiddlewares.httpauth import *
diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py
new file mode 100644
index 000000000..f5f068204
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/httpcache.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcache` is deprecated, "
+              "use `scrapy.downloadermiddlewares.httpcache` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.downloadermiddlewares.httpcache import *
diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/contrib/downloadermiddleware/httpcompression.py
new file mode 100644
index 000000000..8a52ec50b
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/httpcompression.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcompression` is deprecated, "
+              "use `scrapy.downloadermiddlewares.httpcompression` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.downloadermiddlewares.httpcompression import *
diff --git a/scrapy/contrib/downloadermiddleware/httpproxy.py b/scrapy/contrib/downloadermiddleware/httpproxy.py
new file mode 100644
index 000000000..d94d85076
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/httpproxy.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpproxy` is deprecated, "
+              "use `scrapy.downloadermiddlewares.httpproxy` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.downloadermiddlewares.httpproxy import *
diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py
new file mode 100644
index 000000000..824eee8ae
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/redirect.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.downloadermiddleware.redirect` is deprecated, "
+              "use `scrapy.downloadermiddlewares.redirect` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.downloadermiddlewares.redirect import *
diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py
new file mode 100644
index 000000000..aafe0f508
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/retry.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.downloadermiddleware.retry` is deprecated, "
+              "use `scrapy.downloadermiddlewares.retry` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.downloadermiddlewares.retry import *
diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py
new file mode 100644
index 000000000..408f760a0
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/robotstxt.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.downloadermiddleware.robotstxt` is deprecated, "
+              "use `scrapy.downloadermiddlewares.robotstxt` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.downloadermiddlewares.robotstxt import *
diff --git a/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/contrib/downloadermiddleware/stats.py
new file mode 100644
index 000000000..fa84a8206
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/stats.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.downloadermiddleware.stats` is deprecated, "
+              "use `scrapy.downloadermiddlewares.stats` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.downloadermiddlewares.stats import *
diff --git a/scrapy/contrib/downloadermiddleware/useragent.py b/scrapy/contrib/downloadermiddleware/useragent.py
new file mode 100644
index 000000000..893d5241c
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/useragent.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.downloadermiddleware.useragent` is deprecated, "
+              "use `scrapy.downloadermiddlewares.useragent` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.downloadermiddlewares.useragent import *

From 7804b3d778902d93b844f4adf59f650c6b61ef00 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 20 Apr 2015 22:41:06 -0300
Subject: [PATCH 0253/2974] Move scrapy/contrib/exporter to scrapy/exporters

---
 docs/faq.rst                                  |  2 +-
 docs/topics/exporters.rst                     |  6 ++---
 docs/topics/feed-exports.rst                  | 22 +++++++++----------
 .../exporter => exporters}/__init__.py        |  0
 scrapy/settings/default_settings.py           | 14 ++++++------
 tests/py3-ignores.txt                         |  2 +-
 ..._contrib_exporter.py => test_exporters.py} |  2 +-
 7 files changed, 24 insertions(+), 24 deletions(-)
 rename scrapy/{contrib/exporter => exporters}/__init__.py (100%)
 rename tests/{test_contrib_exporter.py => test_exporters.py} (99%)

diff --git a/docs/faq.rst b/docs/faq.rst
index f593e4f16..80d258b55 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -160,7 +160,7 @@ Can I use JSON for large exports?
 ---------------------------------
 
 It'll depend on how large your output is. See :ref:`this warning
-<json-with-large-data>` in :class:`~scrapy.contrib.exporter.JsonItemExporter`
+<json-with-large-data>` in :class:`~scrapy.exporters.JsonItemExporter`
 documentation.
 
 Can I return (Twisted) deferreds from signal handlers?
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 43846852b..af469eb7b 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -4,7 +4,7 @@
 Item Exporters
 ==============
 
-.. module:: scrapy.contrib.exporter
+.. module:: scrapy.exporters
    :synopsis: Item Exporters
 
 Once you have scraped your items, you often want to persist or export those
@@ -40,7 +40,7 @@ Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses an Item
 Exporter to export scraped items to different files, one per spider::
 
    from scrapy import signals
-   from scrapy.contrib.exporter import XmlItemExporter
+   from scrapy.exporters import XmlItemExporter
 
    class XmlExportPipeline(object):
 
@@ -117,7 +117,7 @@ after your custom code.
 
 Example::
 
-      from scrapy.contrib.exporter import XmlItemExporter
+      from scrapy.exporter import XmlItemExporter
 
       class ProductXmlExporter(XmlItemExporter):
 
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 8166a7a4e..27d601a19 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -37,7 +37,7 @@ JSON
 ----
 
  * :setting:`FEED_FORMAT`: ``json``
- * Exporter used: :class:`~scrapy.contrib.exporter.JsonItemExporter`
+ * Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
  * See :ref:`this warning <json-with-large-data>` if you're using JSON with
    large feeds.
 
@@ -47,7 +47,7 @@ JSON lines
 ----------
 
  * :setting:`FEED_FORMAT`: ``jsonlines``
- * Exporter used: :class:`~scrapy.contrib.exporter.JsonLinesItemExporter`
+ * Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
 
 .. _topics-feed-format-csv:
 
@@ -55,7 +55,7 @@ CSV
 ---
 
  * :setting:`FEED_FORMAT`: ``csv``
- * Exporter used: :class:`~scrapy.contrib.exporter.CsvItemExporter`
+ * Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
  * To specify columns to export and their order use
    :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
    option, but it is important for CSV because unlike many other export
@@ -67,7 +67,7 @@ XML
 ---
 
  * :setting:`FEED_FORMAT`: ``xml``
- * Exporter used: :class:`~scrapy.contrib.exporter.XmlItemExporter`
+ * Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
 
 .. _topics-feed-format-pickle:
 
@@ -75,7 +75,7 @@ Pickle
 ------
 
  * :setting:`FEED_FORMAT`: ``pickle``
- * Exporter used: :class:`~scrapy.contrib.exporter.PickleItemExporter`
+ * Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
 
 .. _topics-feed-format-marshal:
 
@@ -83,7 +83,7 @@ Marshal
 -------
 
  * :setting:`FEED_FORMAT`: ``marshal``
- * Exporter used: :class:`~scrapy.contrib.exporter.MarshalItemExporter`
+ * Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
 
 
 .. _topics-feed-storage:
@@ -300,11 +300,11 @@ FEED_EXPORTERS_BASE
 Default::
 
     FEED_EXPORTERS_BASE = {
-        'json': 'scrapy.contrib.exporter.JsonItemExporter',
-        'jsonlines': 'scrapy.contrib.exporter.JsonLinesItemExporter',
-        'csv': 'scrapy.contrib.exporter.CsvItemExporter',
-        'xml': 'scrapy.contrib.exporter.XmlItemExporter',
-        'marshal': 'scrapy.contrib.exporter.MarshalItemExporter',
+        'json': 'scrapy.exporters.JsonItemExporter',
+        'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
+        'csv': 'scrapy.exporters.CsvItemExporter',
+        'xml': 'scrapy.exporters.XmlItemExporter',
+        'marshal': 'scrapy.exporters.MarshalItemExporter',
     }
 
 A dict containing the built-in feed exporters supported by Scrapy.
diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/exporters/__init__.py
similarity index 100%
rename from scrapy/contrib/exporter/__init__.py
rename to scrapy/exporters/__init__.py
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 3323386aa..17db550a7 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -139,13 +139,13 @@ FEED_STORAGES_BASE = {
 }
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
-    'json': 'scrapy.contrib.exporter.JsonItemExporter',
-    'jsonlines': 'scrapy.contrib.exporter.JsonLinesItemExporter',
-    'jl': 'scrapy.contrib.exporter.JsonLinesItemExporter',
-    'csv': 'scrapy.contrib.exporter.CsvItemExporter',
-    'xml': 'scrapy.contrib.exporter.XmlItemExporter',
-    'marshal': 'scrapy.contrib.exporter.MarshalItemExporter',
-    'pickle': 'scrapy.contrib.exporter.PickleItemExporter',
+    'json': 'scrapy.exporters.JsonItemExporter',
+    'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
+    'jl': 'scrapy.exporters.JsonLinesItemExporter',
+    'csv': 'scrapy.exporters.CsvItemExporter',
+    'xml': 'scrapy.exporters.XmlItemExporter',
+    'marshal': 'scrapy.exporters.MarshalItemExporter',
+    'pickle': 'scrapy.exporters.PickleItemExporter',
 }
 
 HTTPCACHE_ENABLED = False
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index cf814c15c..8e5c5dc9b 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -4,7 +4,7 @@ tests/test_command_fetch.py
 tests/test_command_shell.py
 tests/test_commands.py
 tests/test_command_version.py
-tests/test_contrib_exporter.py
+tests/test_exporters.py
 tests/test_contrib_linkextractors.py
 tests/test_contrib_loader.py
 tests/test_crawl.py
diff --git a/tests/test_contrib_exporter.py b/tests/test_exporters.py
similarity index 99%
rename from tests/test_contrib_exporter.py
rename to tests/test_exporters.py
index 746aeb65b..df1f8f4d5 100644
--- a/tests/test_contrib_exporter.py
+++ b/tests/test_exporters.py
@@ -9,7 +9,7 @@ import lxml.etree
 
 from scrapy.item import Item, Field
 from scrapy.utils.python import str_to_unicode
-from scrapy.contrib.exporter import (
+from scrapy.exporters import (
     BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter,
     XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, PythonItemExporter
 )

From 152594ce990e0f5923d87f968e354c1a17bf463d Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 20 Apr 2015 22:46:23 -0300
Subject: [PATCH 0254/2974] scrapy/contrib/exporter shims

---
 scrapy/contrib/exporter/__init__.py | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 scrapy/contrib/exporter/__init__.py

diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py
new file mode 100644
index 000000000..d434a7bfd
--- /dev/null
+++ b/scrapy/contrib/exporter/__init__.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.exporter` is deprecated, "
+              "use `scrapy.exporters` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.exporters import *

From cf064b143747f4fd7e48670f478d40fa2101bb01 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 20 Apr 2015 22:55:33 -0300
Subject: [PATCH 0255/2974] Move scrapy/contrib/linkextractors to
 scrapy/linkextractors

---
 docs/topics/firebug.rst                                |  2 +-
 docs/topics/link-extractors.rst                        | 10 +++++-----
 docs/topics/spiders.rst                                |  2 +-
 scrapy/commands/bench.py                               |  2 +-
 scrapy/link.py                                         |  2 +-
 scrapy/linkextractor.py                                |  2 +-
 scrapy/{contrib => }/linkextractors/__init__.py        |  2 +-
 scrapy/{contrib => }/linkextractors/htmlparser.py      |  0
 scrapy/{contrib => }/linkextractors/lxmlhtml.py        |  0
 scrapy/{contrib => }/linkextractors/regex.py           |  0
 scrapy/{contrib => }/linkextractors/sgml.py            |  4 ++--
 scrapy/templates/spiders/crawl.tmpl                    |  2 +-
 tests/py3-ignores.txt                                  |  5 ++++-
 tests/spiders.py                                       |  2 +-
 tests/test_engine.py                                   |  2 +-
 ...ontrib_linkextractors.py => test_linkextractors.py} |  8 ++++----
 tests/test_spider.py                                   |  2 +-
 17 files changed, 25 insertions(+), 22 deletions(-)
 rename scrapy/{contrib => }/linkextractors/__init__.py (85%)
 rename scrapy/{contrib => }/linkextractors/htmlparser.py (100%)
 rename scrapy/{contrib => }/linkextractors/lxmlhtml.py (100%)
 rename scrapy/{contrib => }/linkextractors/regex.py (100%)
 rename scrapy/{contrib => }/linkextractors/sgml.py (97%)
 rename tests/{test_contrib_linkextractors.py => test_linkextractors.py} (98%)

diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst
index ad3f26b50..3bbb902b2 100644
--- a/docs/topics/firebug.rst
+++ b/docs/topics/firebug.rst
@@ -81,7 +81,7 @@ process and extract data from those pages.
 
 This is how the spider would look so far::
 
-   from scrapy.contrib.linkextractors import LinkExtractor
+   from scrapy.linkextractors import LinkExtractor
    from scrapy.contrib.spiders import CrawlSpider, Rule
 
    class GoogleDirectorySpider(CrawlSpider):
diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index f2f296fba..be3eb4537 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -8,7 +8,7 @@ Link extractors are objects whose only purpose is to extract links from web
 pages (:class:`scrapy.http.Response` objects) which will be eventually
 followed.
 
-There is ``scrapy.contrib.linkextractors import LinkExtractor`` available
+There is ``scrapy.linkextractors import LinkExtractor`` available
 in Scrapy, but you can create your own custom Link Extractors to suit your
 needs by implementing a simple interface.
 
@@ -30,16 +30,16 @@ extract links.
 Built-in link extractors reference
 ==================================
 
-.. module:: scrapy.contrib.linkextractors
+.. module:: scrapy.linkextractors
    :synopsis: Link extractors classes
 
 Link extractors classes bundled with Scrapy are provided in the
-:mod:`scrapy.contrib.linkextractors` module.
+:mod:`scrapy.linkextractors` module.
 
 The default link extractor is ``LinkExtractor``, which is the same as
 :class:`~.LxmlLinkExtractor`::
 
-    from scrapy.contrib.linkextractors import LinkExtractor
+    from scrapy.linkextractors import LinkExtractor
 
 There used to be other link extractor classes in previous Scrapy versions,
 but they are deprecated now.
@@ -47,7 +47,7 @@ but they are deprecated now.
 LxmlLinkExtractor
 -----------------
 
-.. module:: scrapy.contrib.linkextractors.lxmlhtml
+.. module:: scrapy.linkextractors.lxmlhtml
    :synopsis: lxml's HTMLParser-based link extractors
 
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 7c7d5d731..fdc5581a0 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -395,7 +395,7 @@ Let's now take a look at an example CrawlSpider with rules::
 
     import scrapy
     from scrapy.contrib.spiders import CrawlSpider, Rule
-    from scrapy.contrib.linkextractors import LinkExtractor
+    from scrapy.linkextractors import LinkExtractor
 
     class MySpider(CrawlSpider):
         name = 'example.com'
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 395597546..7c056a990 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -6,7 +6,7 @@ from six.moves.urllib.parse import urlencode
 
 import scrapy
 from scrapy.command import ScrapyCommand
-from scrapy.contrib.linkextractors import LinkExtractor
+from scrapy.linkextractors import LinkExtractor
 
 
 class Command(ScrapyCommand):
diff --git a/scrapy/link.py b/scrapy/link.py
index 42c0e4f48..8bdcce761 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -1,7 +1,7 @@
 """
 This module defines the Link object used in Link extractors.
 
-For actual link extractors implementation see scrapy.contrib.linkextractor, or
+For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
 """
 
diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py
index 227d79b46..2a4d18877 100644
--- a/scrapy/linkextractor.py
+++ b/scrapy/linkextractor.py
@@ -1,6 +1,6 @@
 """
 Common code and definitions used by Link extractors (located in
-scrapy.contrib.linkextractor).
+scrapy.linkextractors).
 """
 import re
 from six.moves.urllib.parse import urlparse
diff --git a/scrapy/contrib/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
similarity index 85%
rename from scrapy/contrib/linkextractors/__init__.py
rename to scrapy/linkextractors/__init__.py
index 48b9c757a..28afaa2f1 100644
--- a/scrapy/contrib/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -1,5 +1,5 @@
 """
-scrapy.contrib.linkextractors
+scrapy.linkextractors
 
 This package contains a collection of Link Extractors.
 
diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
similarity index 100%
rename from scrapy/contrib/linkextractors/htmlparser.py
rename to scrapy/linkextractors/htmlparser.py
diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
similarity index 100%
rename from scrapy/contrib/linkextractors/lxmlhtml.py
rename to scrapy/linkextractors/lxmlhtml.py
diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/linkextractors/regex.py
similarity index 100%
rename from scrapy/contrib/linkextractors/regex.py
rename to scrapy/linkextractors/regex.py
diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
similarity index 97%
rename from scrapy/contrib/linkextractors/sgml.py
rename to scrapy/linkextractors/sgml.py
index 335773db1..b1f3da416 100644
--- a/scrapy/contrib/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -20,7 +20,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
     def __init__(self, tag="a", attr="href", unique=False, process_value=None):
         warnings.warn(
             "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
-            "Please use scrapy.contrib.linkextractors.LinkExtractor",
+            "Please use scrapy.linkextractors.LinkExtractor",
             ScrapyDeprecationWarning
         )
         SGMLParser.__init__(self)
@@ -103,7 +103,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
 
         warnings.warn(
             "SgmlLinkExtractor is deprecated and will be removed in future releases. "
-            "Please use scrapy.contrib.linkextractors.LinkExtractor",
+            "Please use scrapy.linkextractors.LinkExtractor",
             ScrapyDeprecationWarning
         )
 
diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 0482a5496..b84e785fd 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 import scrapy
-from scrapy.contrib.linkextractors import LinkExtractor
+from scrapy.linkextractors import LinkExtractor
 from scrapy.contrib.spiders import CrawlSpider, Rule
 
 from $project_name.items import ${ProjectName}Item
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 8e5c5dc9b..3f1da0751 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -5,7 +5,7 @@ tests/test_command_shell.py
 tests/test_commands.py
 tests/test_command_version.py
 tests/test_exporters.py
-tests/test_contrib_linkextractors.py
+tests/test_linkextractors.py
 tests/test_contrib_loader.py
 tests/test_crawl.py
 tests/test_crawler.py
@@ -88,6 +88,9 @@ scrapy/contrib/pipeline/files.py
 scrapy/contrib/linkextractors/sgml.py
 scrapy/contrib/linkextractors/regex.py
 scrapy/contrib/linkextractors/htmlparser.py
+scrapy/linkextractors/sgml.py
+scrapy/linkextractors/regex.py
+scrapy/linkextractors/htmlparser.py
 scrapy/contrib/downloadermiddleware/retry.py
 scrapy/contrib/downloadermiddleware/httpproxy.py
 scrapy/contrib/downloadermiddleware/cookies.py
diff --git a/tests/spiders.py b/tests/spiders.py
index 5484fc5b9..c2956d741 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -8,7 +8,7 @@ from six.moves.urllib.parse import urlencode
 from scrapy.spider import Spider
 from scrapy.http import Request
 from scrapy.item import Item
-from scrapy.contrib.linkextractors import LinkExtractor
+from scrapy.linkextractors import LinkExtractor
 
 
 class MetaSpider(Spider):
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 04fae02c0..bbb94fd58 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -24,7 +24,7 @@ from scrapy.xlib.pydispatch import dispatcher
 from tests import tests_datadir
 from scrapy.spider import Spider
 from scrapy.item import Item, Field
-from scrapy.contrib.linkextractors import LinkExtractor
+from scrapy.linkextractors import LinkExtractor
 from scrapy.http import Request
 from scrapy.utils.signal import disconnect_all
 
diff --git a/tests/test_contrib_linkextractors.py b/tests/test_linkextractors.py
similarity index 98%
rename from tests/test_contrib_linkextractors.py
rename to tests/test_linkextractors.py
index a624f9e66..948289f8f 100644
--- a/tests/test_contrib_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -1,11 +1,11 @@
 import re
 import unittest
-from scrapy.contrib.linkextractors.regex import RegexLinkExtractor
+from scrapy.linkextractors.regex import RegexLinkExtractor
 from scrapy.http import HtmlResponse, XmlResponse
 from scrapy.link import Link
-from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
-from scrapy.contrib.linkextractors.lxmlhtml import LxmlLinkExtractor
+from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor
+from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
+from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
 from tests import get_testdata
 
 
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 517fc0995..02352ec29 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -13,7 +13,7 @@ from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlRespon
 from scrapy.contrib.spiders.init import InitSpider
 from scrapy.contrib.spiders import CrawlSpider, Rule, XMLFeedSpider, \
     CSVFeedSpider, SitemapSpider
-from scrapy.contrib.linkextractors import LinkExtractor
+from scrapy.linkextractors import LinkExtractor
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.test import get_crawler

From 569156be190fb7e86108c29001d6233a6698a510 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 20 Apr 2015 23:03:07 -0300
Subject: [PATCH 0256/2974] scrapy/contrib/linkextractors shims

---
 scrapy/contrib/linkextractors/__init__.py   | 7 +++++++
 scrapy/contrib/linkextractors/htmlparser.py | 7 +++++++
 scrapy/contrib/linkextractors/lxmlhtml.py   | 7 +++++++
 scrapy/contrib/linkextractors/regex.py      | 7 +++++++
 scrapy/contrib/linkextractors/sgml.py       | 7 +++++++
 5 files changed, 35 insertions(+)
 create mode 100644 scrapy/contrib/linkextractors/__init__.py
 create mode 100644 scrapy/contrib/linkextractors/htmlparser.py
 create mode 100644 scrapy/contrib/linkextractors/lxmlhtml.py
 create mode 100644 scrapy/contrib/linkextractors/regex.py
 create mode 100644 scrapy/contrib/linkextractors/sgml.py

diff --git a/scrapy/contrib/linkextractors/__init__.py b/scrapy/contrib/linkextractors/__init__.py
new file mode 100644
index 000000000..976658df3
--- /dev/null
+++ b/scrapy/contrib/linkextractors/__init__.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.linkextractors` is deprecated, "
+              "use `scrapy.linkextractors` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.linkextractors import *
diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/contrib/linkextractors/htmlparser.py
new file mode 100644
index 000000000..ff03da98f
--- /dev/null
+++ b/scrapy/contrib/linkextractors/htmlparser.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.linkextractors.htmlparser` is deprecated, "
+              "use `scrapy.linkextractors.htmlparser` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.linkextractors.htmlparser import *
diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py
new file mode 100644
index 000000000..fc2b7de3c
--- /dev/null
+++ b/scrapy/contrib/linkextractors/lxmlhtml.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.linkextractors.lxmlhtml` is deprecated, "
+              "use `scrapy.linkextractors.lxmlhtml` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.linkextractors.lxmlhtml import *
diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py
new file mode 100644
index 000000000..97bda29c1
--- /dev/null
+++ b/scrapy/contrib/linkextractors/regex.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.linkextractors.regex` is deprecated, "
+              "use `scrapy.linkextractors.regex` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.linkextractors.regex import *
diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py
new file mode 100644
index 000000000..a5a598208
--- /dev/null
+++ b/scrapy/contrib/linkextractors/sgml.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.linkextractors.sgml` is deprecated, "
+              "use `scrapy.linkextractors.sgml` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.linkextractors.sgml import *

From b47228ada8ac583f123434686e8d3493505c7127 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 20 Apr 2015 23:21:41 -0300
Subject: [PATCH 0257/2974] Move scrapy/contrib/loader to scrapy/loader

---
 docs/contributing.rst                         |  8 ++---
 docs/topics/loaders.rst                       | 32 +++++++++----------
 scrapy/{contrib => }/loader/__init__.py       |  0
 scrapy/{contrib => }/loader/common.py         |  0
 scrapy/{contrib => }/loader/processor.py      |  0
 tests/py3-ignores.txt                         |  2 +-
 ...{test_contrib_loader.py => test_loader.py} |  4 +--
 tests/test_squeue.py                          |  2 +-
 8 files changed, 24 insertions(+), 24 deletions(-)
 rename scrapy/{contrib => }/loader/__init__.py (100%)
 rename scrapy/{contrib => }/loader/common.py (100%)
 rename scrapy/{contrib => }/loader/processor.py (100%)
 rename tests/{test_contrib_loader.py => test_loader.py} (99%)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index f49bc536e..ad9a3805a 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -150,9 +150,9 @@ To run all tests go to the root directory of Scrapy source code and run:
 
     ``tox``
 
-To run a specific test (say ``tests/test_contrib_loader.py``) use:
+To run a specific test (say ``tests/test_loader.py``) use:
 
-    ``tox -- tests/test_contrib_loader.py``
+    ``tox -- tests/test_loader.py``
 
 
 Writing tests
@@ -166,11 +166,11 @@ Scrapy uses unit-tests, which are located in the `tests/`_ directory.
 Their module name typically resembles the full path of the module they're
 testing. For example, the item loaders code is in::
 
-    scrapy.contrib.loader
+    scrapy.loader
 
 And their unit-tests are in::
 
-    tests/test_contrib_loader.py
+    tests/test_loader.py
 
 .. _issue tracker: https://github.com/scrapy/scrapy/issues
 .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 603d28f78..a5efa3fb9 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -4,7 +4,7 @@
 Item Loaders
 ============
 
-.. module:: scrapy.contrib.loader
+.. module:: scrapy.loader
    :synopsis: Item Loader class
 
 Item Loaders provide a convenient mechanism for populating scraped :ref:`Items
@@ -39,7 +39,7 @@ Here is a typical Item Loader usage in a :ref:`Spider <topics-spiders>`, using
 the :ref:`Product item <topics-items-declaring>` declared in the :ref:`Items
 chapter <topics-items>`::
 
-    from scrapy.contrib.loader import ItemLoader
+    from scrapy.loader import ItemLoader
     from myproject.items import Product
 
     def parse(self, response):
@@ -150,8 +150,8 @@ Declaring Item Loaders
 Item Loaders are declared like Items, by using a class definition syntax. Here
 is an example::
 
-    from scrapy.contrib.loader import ItemLoader
-    from scrapy.contrib.loader.processor import TakeFirst, MapCompose, Join
+    from scrapy.loader import ItemLoader
+    from scrapy.loader.processor import TakeFirst, MapCompose, Join
 
     class ProductLoader(ItemLoader):
 
@@ -182,7 +182,7 @@ output processors to use: in the :ref:`Item Field <topics-items-fields>`
 metadata. Here is an example::
 
     import scrapy
-    from scrapy.contrib.loader.processor import Join, MapCompose, TakeFirst
+    from scrapy.loader.processor import Join, MapCompose, TakeFirst
     from w3lib.html import remove_tags
 
     def filter_price(value):
@@ -201,7 +201,7 @@ metadata. Here is an example::
 
 ::
 
-    >>> from scrapy.contrib.loader import ItemLoader
+    >>> from scrapy.loader import ItemLoader
     >>> il = ItemLoader(item=Product())
     >>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
     >>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
@@ -309,7 +309,7 @@ ItemLoader objects
 
         Examples::
 
-            >>> from scrapy.contrib.loader.processor import TakeFirst
+            >>> from scrapy.loader.processor import TakeFirst
             >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
             'FOO`
 
@@ -513,7 +513,7 @@ those dashes in the final product names.
 Here's how you can remove those dashes by reusing and extending the default
 Product Item Loader (``ProductLoader``)::
 
-    from scrapy.contrib.loader.processor import MapCompose
+    from scrapy.loader.processor import MapCompose
     from myproject.ItemLoaders import ProductLoader
 
     def strip_dashes(x):
@@ -526,7 +526,7 @@ Another case where extending Item Loaders can be very helpful is when you have
 multiple source formats, for example XML and HTML. In the XML version you may
 want to remove ``CDATA`` occurrences. Here's an example of how to do it::
 
-    from scrapy.contrib.loader.processor import MapCompose
+    from scrapy.loader.processor import MapCompose
     from myproject.ItemLoaders import ProductLoader
     from myproject.utils.xml import remove_cdata
 
@@ -551,7 +551,7 @@ needs.
 Available built-in processors
 =============================
 
-.. module:: scrapy.contrib.loader.processor
+.. module:: scrapy.loader.processor
    :synopsis: A collection of processors to use with Item Loaders
 
 Even though you can use any callable function as input and output processors,
@@ -570,7 +570,7 @@ Here is a list of all built-in processors:
 
     Example::
 
-        >>> from scrapy.contrib.loader.processor import Identity
+        >>> from scrapy.loader.processor import Identity
         >>> proc = Identity()
         >>> proc(['one', 'two', 'three'])
         ['one', 'two', 'three']
@@ -583,7 +583,7 @@ Here is a list of all built-in processors:
 
     Example::
 
-        >>> from scrapy.contrib.loader.processor import TakeFirst
+        >>> from scrapy.loader.processor import TakeFirst
         >>> proc = TakeFirst()
         >>> proc(['', 'one', 'two', 'three'])
         'one'
@@ -598,7 +598,7 @@ Here is a list of all built-in processors:
 
     Examples::
 
-        >>> from scrapy.contrib.loader.processor import Join
+        >>> from scrapy.loader.processor import Join
         >>> proc = Join()
         >>> proc(['one', 'two', 'three'])
         u'one two three'
@@ -619,7 +619,7 @@ Here is a list of all built-in processors:
 
     Example::
 
-        >>> from scrapy.contrib.loader.processor import Compose
+        >>> from scrapy.loader.processor import Compose
         >>> proc = Compose(lambda v: v[0], str.upper)
         >>> proc(['hello', 'world'])
         'HELLO'
@@ -666,7 +666,7 @@ Here is a list of all built-in processors:
         >>> def filter_world(x):
         ...     return None if x == 'world' else x
         ...
-        >>> from scrapy.contrib.loader.processor import MapCompose
+        >>> from scrapy.loader.processor import MapCompose
         >>> proc = MapCompose(filter_world, unicode.upper)
         >>> proc([u'hello', u'world', u'this', u'is', u'scrapy'])
         [u'HELLO, u'THIS', u'IS', u'SCRAPY']
@@ -683,7 +683,7 @@ Here is a list of all built-in processors:
 
     Example::
 
-        >>> from scrapy.contrib.loader.processor import SelectJmes, Compose, MapCompose
+        >>> from scrapy.loader.processor import SelectJmes, Compose, MapCompose
         >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
         >>> proc({'foo': 'bar'})
         'bar'
diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/loader/__init__.py
similarity index 100%
rename from scrapy/contrib/loader/__init__.py
rename to scrapy/loader/__init__.py
diff --git a/scrapy/contrib/loader/common.py b/scrapy/loader/common.py
similarity index 100%
rename from scrapy/contrib/loader/common.py
rename to scrapy/loader/common.py
diff --git a/scrapy/contrib/loader/processor.py b/scrapy/loader/processor.py
similarity index 100%
rename from scrapy/contrib/loader/processor.py
rename to scrapy/loader/processor.py
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 3f1da0751..69df2f7a2 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -6,7 +6,7 @@ tests/test_commands.py
 tests/test_command_version.py
 tests/test_exporters.py
 tests/test_linkextractors.py
-tests/test_contrib_loader.py
+tests/test_loader.py
 tests/test_crawl.py
 tests/test_crawler.py
 tests/test_downloader_handlers.py
diff --git a/tests/test_contrib_loader.py b/tests/test_loader.py
similarity index 99%
rename from tests/test_contrib_loader.py
rename to tests/test_loader.py
index fd65e8200..c51092ad0 100644
--- a/tests/test_contrib_loader.py
+++ b/tests/test_loader.py
@@ -1,8 +1,8 @@
 import unittest
 from functools import partial
 
-from scrapy.contrib.loader import ItemLoader
-from scrapy.contrib.loader.processor import Join, Identity, TakeFirst, \
+from scrapy.loader import ItemLoader
+from scrapy.loader.processor import Join, Identity, TakeFirst, \
     Compose, MapCompose, SelectJmes
 from scrapy.item import Item, Field
 from scrapy.selector import Selector
diff --git a/tests/test_squeue.py b/tests/test_squeue.py
index 83ffcc4b7..16014a897 100644
--- a/tests/test_squeue.py
+++ b/tests/test_squeue.py
@@ -2,7 +2,7 @@ from queuelib.tests import test_queue as t
 from scrapy.squeue import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue
 from scrapy.item import Item, Field
 from scrapy.http import Request
-from scrapy.contrib.loader import ItemLoader
+from scrapy.loader import ItemLoader
 
 class TestItem(Item):
     name = Field()

From d7e60f3c718bb9fbfd4151825b009a0a1d042fc4 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 20 Apr 2015 23:27:41 -0300
Subject: [PATCH 0258/2974] scrapy/contrib/loader shims

---
 scrapy/contrib/loader/__init__.py  | 7 +++++++
 scrapy/contrib/loader/common.py    | 7 +++++++
 scrapy/contrib/loader/processor.py | 7 +++++++
 3 files changed, 21 insertions(+)
 create mode 100644 scrapy/contrib/loader/__init__.py
 create mode 100644 scrapy/contrib/loader/common.py
 create mode 100644 scrapy/contrib/loader/processor.py

diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py
new file mode 100644
index 000000000..2b9453e18
--- /dev/null
+++ b/scrapy/contrib/loader/__init__.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.loader` is deprecated, "
+              "use `scrapy.loader` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.loader import *
diff --git a/scrapy/contrib/loader/common.py b/scrapy/contrib/loader/common.py
new file mode 100644
index 000000000..a59b2b7b1
--- /dev/null
+++ b/scrapy/contrib/loader/common.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.loader.common` is deprecated, "
+              "use `scrapy.loader.common` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.loader.common import *
diff --git a/scrapy/contrib/loader/processor.py b/scrapy/contrib/loader/processor.py
new file mode 100644
index 000000000..68ddff4d4
--- /dev/null
+++ b/scrapy/contrib/loader/processor.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.loader.processor` is deprecated, "
+              "use `scrapy.loader.processor` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.loader.processor import *

From 8021df18d4f1703ad6b40c6e42e08aa627d629ae Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 20 Apr 2015 23:43:38 -0300
Subject: [PATCH 0259/2974] Move scrapy/contrib/pipeline to scrapy/pipelines

---
 docs/topics/media-pipeline.rst                     | 12 ++++++------
 scrapy/{contrib/pipeline => pipelines}/__init__.py |  0
 scrapy/{contrib/pipeline => pipelines}/files.py    |  2 +-
 scrapy/{contrib/pipeline => pipelines}/images.py   |  4 ++--
 scrapy/{contrib/pipeline => pipelines}/media.py    |  0
 scrapy/settings/default_settings.py                |  2 +-
 tests/py3-ignores.txt                              |  2 ++
 tests/test_pipeline_files.py                       |  2 +-
 tests/test_pipeline_images.py                      |  2 +-
 tests/test_pipeline_media.py                       |  2 +-
 10 files changed, 15 insertions(+), 13 deletions(-)
 rename scrapy/{contrib/pipeline => pipelines}/__init__.py (100%)
 rename scrapy/{contrib/pipeline => pipelines}/files.py (99%)
 rename scrapy/{contrib/pipeline => pipelines}/images.py (97%)
 rename scrapy/{contrib/pipeline => pipelines}/media.py (100%)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 4b88c60fd..c18ed25ae 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -4,7 +4,7 @@
 Downloading and processing files and images
 ===========================================
 
-.. currentmodule:: scrapy.contrib.pipeline.images
+.. currentmodule:: scrapy.pipelines.images
 
 Scrapy provides reusable :doc:`item pipelines </topics/item-pipeline>` for
 downloading fies attached to a particular item (for example, when you scrape
@@ -114,11 +114,11 @@ To enable your media pipeline you must first add it to your project
 
 For Images Pipeline, use::
 
-    ITEM_PIPELINES = {'scrapy.contrib.pipeline.images.ImagesPipeline': 1}
+    ITEM_PIPELINES = {'scrapy.pipelines.images.ImagesPipeline': 1}
 
 For Files Pipeline, use::
 
-    ITEM_PIPELINES = {'scrapy.contrib.pipeline.files.FilesPipeline': 1}
+    ITEM_PIPELINES = {'scrapy.pipelines.files.FilesPipeline': 1}
 
 
 .. note::
@@ -258,7 +258,7 @@ By default, there are no size constraints, so all images are processed.
 Extending the Media Pipelines
 =============================
 
-.. module:: scrapy.contrib.pipeline.files
+.. module:: scrapy.pipelines.files
    :synopsis: Files Pipeline
 
 See here the methods that you can override in your custom Files Pipeline:
@@ -338,7 +338,7 @@ See here the methods that you can override in your custom Files Pipeline:
       By default, the :meth:`item_completed` method returns the item.
 
 
-.. module:: scrapy.contrib.pipeline.images
+.. module:: scrapy.pipelines.images
    :synopsis: Images Pipeline
 
 See here the methods that you can override in your custom Images Pipeline:
@@ -374,7 +374,7 @@ Here is a full example of the Images Pipeline whose methods are examplified
 above::
 
     import scrapy
-    from scrapy.contrib.pipeline.images import ImagesPipeline
+    from scrapy.pipelines.images import ImagesPipeline
     from scrapy.exceptions import DropItem
 
     class MyImagesPipeline(ImagesPipeline):
diff --git a/scrapy/contrib/pipeline/__init__.py b/scrapy/pipelines/__init__.py
similarity index 100%
rename from scrapy/contrib/pipeline/__init__.py
rename to scrapy/pipelines/__init__.py
diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/pipelines/files.py
similarity index 99%
rename from scrapy/contrib/pipeline/files.py
rename to scrapy/pipelines/files.py
index 4809005d1..2b17e0f0a 100644
--- a/scrapy/contrib/pipeline/files.py
+++ b/scrapy/pipelines/files.py
@@ -21,7 +21,7 @@ except ImportError:
 
 from twisted.internet import defer, threads
 
-from scrapy.contrib.pipeline.media import MediaPipeline
+from scrapy.pipelines.media import MediaPipeline
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.misc import md5sum
diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/pipelines/images.py
similarity index 97%
rename from scrapy/contrib/pipeline/images.py
rename to scrapy/pipelines/images.py
index 3cbfed49b..8b3bc2222 100644
--- a/scrapy/contrib/pipeline/images.py
+++ b/scrapy/pipelines/images.py
@@ -17,8 +17,8 @@ from PIL import Image
 from scrapy.utils.misc import md5sum
 from scrapy.http import Request
 from scrapy.exceptions import DropItem
-#TODO: from scrapy.contrib.pipeline.media import MediaPipeline
-from scrapy.contrib.pipeline.files import FileException, FilesPipeline
+#TODO: from scrapy.pipelines.media import MediaPipeline
+from scrapy.pipelines.files import FileException, FilesPipeline
 
 
 class NoimagesDrop(DropItem):
diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/pipelines/media.py
similarity index 100%
rename from scrapy/contrib/pipeline/media.py
rename to scrapy/pipelines/media.py
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 17db550a7..f4dcdc1a2 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -159,7 +159,7 @@ HTTPCACHE_DBM_MODULE = 'anydbm'
 HTTPCACHE_POLICY = 'scrapy.contrib.httpcache.DummyPolicy'
 HTTPCACHE_GZIP = False
 
-ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager'
+ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
 
 ITEM_PIPELINES = {}
 ITEM_PIPELINES_BASE = {}
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 69df2f7a2..a29b419b0 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -85,6 +85,8 @@ scrapy/core/downloader/handlers/ftp.py
 scrapy/core/downloader/webclient.py
 scrapy/contrib/pipeline/images.py
 scrapy/contrib/pipeline/files.py
+scrapy/pipelines/images.py
+scrapy/pipelines/files.py
 scrapy/contrib/linkextractors/sgml.py
 scrapy/contrib/linkextractors/regex.py
 scrapy/contrib/linkextractors/htmlparser.py
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 84fe4927d..b12f41174 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -8,7 +8,7 @@ from shutil import rmtree
 from twisted.trial import unittest
 from twisted.internet import defer
 
-from scrapy.contrib.pipeline.files import FilesPipeline, FSFilesStore
+from scrapy.pipelines.files import FilesPipeline, FSFilesStore
 from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index f5750b4fc..04cec4b8e 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -9,7 +9,7 @@ from twisted.trial import unittest
 from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
-from scrapy.contrib.pipeline.images import ImagesPipeline
+from scrapy.pipelines.images import ImagesPipeline
 
 skip = False
 try:
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 0e946303b..24ba9d64a 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -8,7 +8,7 @@ from twisted.internet.defer import Deferred, inlineCallbacks
 from scrapy.http import Request, Response
 from scrapy.spider import Spider
 from scrapy.utils.request import request_fingerprint
-from scrapy.contrib.pipeline.media import MediaPipeline
+from scrapy.pipelines.media import MediaPipeline
 from scrapy.utils.signal import disconnect_all
 from scrapy import signals
 

From c97a69c9071677b0ff7974e8fd1ef42acb9e207a Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 20 Apr 2015 23:47:30 -0300
Subject: [PATCH 0260/2974] scrapy/contrib/pipeline shims

---
 scrapy/contrib/pipeline/__init__.py | 7 +++++++
 scrapy/contrib/pipeline/files.py    | 7 +++++++
 scrapy/contrib/pipeline/images.py   | 7 +++++++
 scrapy/contrib/pipeline/media.py    | 7 +++++++
 4 files changed, 28 insertions(+)
 create mode 100644 scrapy/contrib/pipeline/__init__.py
 create mode 100644 scrapy/contrib/pipeline/files.py
 create mode 100644 scrapy/contrib/pipeline/images.py
 create mode 100644 scrapy/contrib/pipeline/media.py

diff --git a/scrapy/contrib/pipeline/__init__.py b/scrapy/contrib/pipeline/__init__.py
new file mode 100644
index 000000000..aedf34a3f
--- /dev/null
+++ b/scrapy/contrib/pipeline/__init__.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.pipeline` is deprecated, "
+              "use `scrapy.pipelines` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.pipelines import *
diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py
new file mode 100644
index 000000000..cd1238b5d
--- /dev/null
+++ b/scrapy/contrib/pipeline/files.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.pipeline.files` is deprecated, "
+              "use `scrapy.pipelines.files` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.pipelines.files import *
diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py
new file mode 100644
index 000000000..4f5ce4c40
--- /dev/null
+++ b/scrapy/contrib/pipeline/images.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.pipeline.images` is deprecated, "
+              "use `scrapy.pipelines.images` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.pipelines.images import *
diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py
new file mode 100644
index 000000000..4b4fea560
--- /dev/null
+++ b/scrapy/contrib/pipeline/media.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.pipeline.media` is deprecated, "
+              "use `scrapy.pipelines.media` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.pipelines.media import *

From 180272c09272c2dd47af65d5fb4cb914b051623a Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 21 Apr 2015 13:07:24 -0300
Subject: [PATCH 0261/2974] Move scrapy/contrib/spidermiddleware to
 scrapy/spidermiddlewares

---
 docs/faq.rst                                       |  2 +-
 docs/topics/settings.rst                           | 12 ++++++------
 docs/topics/spider-middleware.rst                  | 14 +++++++-------
 docs/topics/spiders.rst                            |  2 +-
 scrapy/settings/default_settings.py                | 10 +++++-----
 .../__init__.py                                    |  0
 .../depth.py                                       |  0
 .../httperror.py                                   |  0
 .../offsite.py                                     |  0
 .../referer.py                                     |  0
 .../urllength.py                                   |  0
 tests/test_spidermiddleware_depth.py               |  4 ++--
 tests/test_spidermiddleware_httperror.py           |  2 +-
 tests/test_spidermiddleware_offsite.py             |  2 +-
 tests/test_spidermiddleware_referer.py             |  2 +-
 tests/test_spidermiddleware_urllength.py           |  2 +-
 16 files changed, 26 insertions(+), 26 deletions(-)
 rename scrapy/{contrib/spidermiddleware => spidermiddlewares}/__init__.py (100%)
 rename scrapy/{contrib/spidermiddleware => spidermiddlewares}/depth.py (100%)
 rename scrapy/{contrib/spidermiddleware => spidermiddlewares}/httperror.py (100%)
 rename scrapy/{contrib/spidermiddleware => spidermiddlewares}/offsite.py (100%)
 rename scrapy/{contrib/spidermiddleware => spidermiddlewares}/referer.py (100%)
 rename scrapy/{contrib/spidermiddleware => spidermiddlewares}/urllength.py (100%)

diff --git a/docs/faq.rst b/docs/faq.rst
index 80d258b55..d98b06056 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -149,7 +149,7 @@ middleware (enabled by default) whose purpose is to filter out requests to
 domains outside the ones covered by the spider.
 
 For more info see:
-:class:`~scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware`.
+:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware`.
 
 What is the recommended way to deploy a Scrapy crawler in production?
 ---------------------------------------------------------------------
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 3e68ad925..a049f1438 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -906,11 +906,11 @@ SPIDER_MIDDLEWARES_BASE
 Default::
 
     {
-        'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50,
-        'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500,
-        'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700,
-        'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800,
-        'scrapy.contrib.spidermiddleware.depth.DepthMiddleware': 900,
+        'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
+        'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,
+        'scrapy.spidermiddlewares.referer.RefererMiddleware': 700,
+        'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800,
+        'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,
     }
 
 A dict containing the spider middlewares enabled by default in Scrapy. You
@@ -1001,7 +1001,7 @@ URLLENGTH_LIMIT
 
 Default: ``2083``
 
-Scope: ``contrib.spidermiddleware.urllength``
+Scope: ``spidermiddlewares.urllength``
 
 The maximum URL length to allow for crawled URLs. For more information about
 the default value for this setting see: http://www.boutell.com/newfaq/misc/urllength.html
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index abeae2bce..6e82333f5 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -43,7 +43,7 @@ value.  For example, if you want to disable the off-site middleware::
 
     SPIDER_MIDDLEWARES = {
         'myproject.middlewares.CustomSpiderMiddleware': 543,
-        'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': None,
+        'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,
     }
 
 Finally, keep in mind that some middlewares may need to be enabled through a
@@ -55,7 +55,7 @@ Writing your own spider middleware
 Each middleware component is a Python class that defines one or more of the
 following methods:
 
-.. module:: scrapy.contrib.spidermiddleware
+.. module:: scrapy.spidermiddlewares
 
 .. class:: SpiderMiddleware
 
@@ -178,7 +178,7 @@ For a list of the components enabled by default (and their orders) see the
 DepthMiddleware
 ---------------
 
-.. module:: scrapy.contrib.spidermiddleware.depth
+.. module:: scrapy.spidermiddlewares.depth
    :synopsis: Depth Spider Middleware
 
 .. class:: DepthMiddleware
@@ -199,7 +199,7 @@ DepthMiddleware
 HttpErrorMiddleware
 -------------------
 
-.. module:: scrapy.contrib.spidermiddleware.httperror
+.. module:: scrapy.spidermiddlewares.httperror
    :synopsis: HTTP Error Spider Middleware
 
 .. class:: HttpErrorMiddleware
@@ -264,7 +264,7 @@ Pass all responses, regardless of its status code.
 OffsiteMiddleware
 -----------------
 
-.. module:: scrapy.contrib.spidermiddleware.offsite
+.. module:: scrapy.spidermiddlewares.offsite
    :synopsis: Offsite Spider Middleware
 
 .. class:: OffsiteMiddleware
@@ -298,7 +298,7 @@ OffsiteMiddleware
 RefererMiddleware
 -----------------
 
-.. module:: scrapy.contrib.spidermiddleware.referer
+.. module:: scrapy.spidermiddlewares.referer
    :synopsis: Referer Spider Middleware
 
 .. class:: RefererMiddleware
@@ -323,7 +323,7 @@ Whether to enable referer middleware.
 UrlLengthMiddleware
 -------------------
 
-.. module:: scrapy.contrib.spidermiddleware.urllength
+.. module:: scrapy.spidermiddlewares.urllength
    :synopsis: URL Length Spider Middleware
 
 .. class:: UrlLengthMiddleware
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index fdc5581a0..df030170c 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -77,7 +77,7 @@ scrapy.Spider
        An optional list of strings containing domains that this spider is
        allowed to crawl. Requests for URLs not belonging to the domain names
        specified in this list won't be followed if
-       :class:`~scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware` is enabled.
+       :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled.
 
    .. attribute:: start_urls
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index f4dcdc1a2..a7fe36648 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -224,11 +224,11 @@ SPIDER_MIDDLEWARES = {}
 
 SPIDER_MIDDLEWARES_BASE = {
     # Engine side
-    'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50,
-    'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500,
-    'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700,
-    'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800,
-    'scrapy.contrib.spidermiddleware.depth.DepthMiddleware': 900,
+    'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
+    'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,
+    'scrapy.spidermiddlewares.referer.RefererMiddleware': 700,
+    'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800,
+    'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,
     # Spider side
 }
 
diff --git a/scrapy/contrib/spidermiddleware/__init__.py b/scrapy/spidermiddlewares/__init__.py
similarity index 100%
rename from scrapy/contrib/spidermiddleware/__init__.py
rename to scrapy/spidermiddlewares/__init__.py
diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/spidermiddlewares/depth.py
similarity index 100%
rename from scrapy/contrib/spidermiddleware/depth.py
rename to scrapy/spidermiddlewares/depth.py
diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/spidermiddlewares/httperror.py
similarity index 100%
rename from scrapy/contrib/spidermiddleware/httperror.py
rename to scrapy/spidermiddlewares/httperror.py
diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/spidermiddlewares/offsite.py
similarity index 100%
rename from scrapy/contrib/spidermiddleware/offsite.py
rename to scrapy/spidermiddlewares/offsite.py
diff --git a/scrapy/contrib/spidermiddleware/referer.py b/scrapy/spidermiddlewares/referer.py
similarity index 100%
rename from scrapy/contrib/spidermiddleware/referer.py
rename to scrapy/spidermiddlewares/referer.py
diff --git a/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/spidermiddlewares/urllength.py
similarity index 100%
rename from scrapy/contrib/spidermiddleware/urllength.py
rename to scrapy/spidermiddlewares/urllength.py
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index e7ae75ed2..8120f1a95 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -1,6 +1,6 @@
 from unittest import TestCase
 
-from scrapy.contrib.spidermiddleware.depth import DepthMiddleware
+from scrapy.spidermiddlewares.depth import DepthMiddleware
 from scrapy.http import Response, Request
 from scrapy.spider import Spider
 from scrapy.statscol import StatsCollector
@@ -37,7 +37,7 @@ class TestDepthMiddleware(TestCase):
 
         rdm = self.stats.get_value('request_depth_max', spider=self.spider)
         self.assertEquals(rdm, 1)
- 
+
     def tearDown(self):
         self.stats.close_spider(self.spider, '')
 
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 503865f63..5cd2c2566 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -8,7 +8,7 @@ from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
 from scrapy.http import Response, Request
 from scrapy.spider import Spider
-from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpError
+from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError
 from scrapy.settings import Settings
 
 
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index e5e99002a..296e8b1a3 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -4,7 +4,7 @@ from six.moves.urllib.parse import urlparse
 
 from scrapy.http import Response, Request
 from scrapy.spider import Spider
-from scrapy.contrib.spidermiddleware.offsite import OffsiteMiddleware
+from scrapy.spidermiddlewares.offsite import OffsiteMiddleware
 from scrapy.utils.test import get_crawler
 
 class TestOffsiteMiddleware(TestCase):
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index f408719d2..f2815ebd3 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -2,7 +2,7 @@ from unittest import TestCase
 
 from scrapy.http import Response, Request
 from scrapy.spider import Spider
-from scrapy.contrib.spidermiddleware.referer import RefererMiddleware
+from scrapy.spidermiddlewares.referer import RefererMiddleware
 
 
 class TestRefererMiddleware(TestCase):
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index ca2de18f2..1ef22ea07 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -1,6 +1,6 @@
 from unittest import TestCase
 
-from scrapy.contrib.spidermiddleware.urllength import UrlLengthMiddleware
+from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
 from scrapy.http import Response, Request
 from scrapy.spider import Spider
 

From b2a15ddbf3ad39464670d847a26efc613a4f8a1f Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 21 Apr 2015 13:11:56 -0300
Subject: [PATCH 0262/2974] scrapy/contrib/spidermiddleware shims

---
 scrapy/contrib/spidermiddleware/__init__.py  | 0
 scrapy/contrib/spidermiddleware/depth.py     | 7 +++++++
 scrapy/contrib/spidermiddleware/httperror.py | 7 +++++++
 scrapy/contrib/spidermiddleware/offsite.py   | 7 +++++++
 scrapy/contrib/spidermiddleware/referer.py   | 7 +++++++
 scrapy/contrib/spidermiddleware/urllength.py | 7 +++++++
 6 files changed, 35 insertions(+)
 create mode 100644 scrapy/contrib/spidermiddleware/__init__.py
 create mode 100644 scrapy/contrib/spidermiddleware/depth.py
 create mode 100644 scrapy/contrib/spidermiddleware/httperror.py
 create mode 100644 scrapy/contrib/spidermiddleware/offsite.py
 create mode 100644 scrapy/contrib/spidermiddleware/referer.py
 create mode 100644 scrapy/contrib/spidermiddleware/urllength.py

diff --git a/scrapy/contrib/spidermiddleware/__init__.py b/scrapy/contrib/spidermiddleware/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py
new file mode 100644
index 000000000..718803148
--- /dev/null
+++ b/scrapy/contrib/spidermiddleware/depth.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.spidermiddleware.depth` is deprecated, "
+              "use `scrapy.spidermiddlewares.depth` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.spidermiddlewares.depth import *
diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py
new file mode 100644
index 000000000..e39fb3f56
--- /dev/null
+++ b/scrapy/contrib/spidermiddleware/httperror.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.spidermiddleware.httperror` is deprecated, "
+              "use `scrapy.spidermiddlewares.httperror` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.spidermiddlewares.httperror import *
diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py
new file mode 100644
index 000000000..a5ed9ea7e
--- /dev/null
+++ b/scrapy/contrib/spidermiddleware/offsite.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.spidermiddleware.offsite` is deprecated, "
+              "use `scrapy.spidermiddlewares.offsite` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.spidermiddlewares.offsite import *
diff --git a/scrapy/contrib/spidermiddleware/referer.py b/scrapy/contrib/spidermiddleware/referer.py
new file mode 100644
index 000000000..fdf8d6659
--- /dev/null
+++ b/scrapy/contrib/spidermiddleware/referer.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.spidermiddleware.referer` is deprecated, "
+              "use `scrapy.spidermiddlewares.referer` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.spidermiddlewares.referer import *
diff --git a/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/contrib/spidermiddleware/urllength.py
new file mode 100644
index 000000000..5e51add59
--- /dev/null
+++ b/scrapy/contrib/spidermiddleware/urllength.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.spidermiddleware.urllength` is deprecated, "
+              "use `scrapy.spidermiddlewares.urllength` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.spidermiddlewares.urllength import *

From fe4b260ef4579219aa53b5215f80da5a5daf5c3a Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 21 Apr 2015 13:54:28 -0300
Subject: [PATCH 0263/2974] Top-level scrapy/contrib shims

---
 scrapy/contrib/closespider.py | 7 +++++++
 scrapy/contrib/corestats.py   | 7 +++++++
 scrapy/contrib/debug.py       | 7 +++++++
 scrapy/contrib/feedexport.py  | 7 +++++++
 scrapy/contrib/httpcache.py   | 7 +++++++
 scrapy/contrib/logstats.py    | 7 +++++++
 scrapy/contrib/memdebug.py    | 7 +++++++
 scrapy/contrib/memusage.py    | 7 +++++++
 scrapy/contrib/spiderstate.py | 7 +++++++
 scrapy/contrib/statsmailer.py | 7 +++++++
 scrapy/contrib/throttle.py    | 7 +++++++
 11 files changed, 77 insertions(+)
 create mode 100644 scrapy/contrib/closespider.py
 create mode 100644 scrapy/contrib/corestats.py
 create mode 100644 scrapy/contrib/debug.py
 create mode 100644 scrapy/contrib/feedexport.py
 create mode 100644 scrapy/contrib/httpcache.py
 create mode 100644 scrapy/contrib/logstats.py
 create mode 100644 scrapy/contrib/memdebug.py
 create mode 100644 scrapy/contrib/memusage.py
 create mode 100644 scrapy/contrib/spiderstate.py
 create mode 100644 scrapy/contrib/statsmailer.py
 create mode 100644 scrapy/contrib/throttle.py

diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py
new file mode 100644
index 000000000..9c52c418f
--- /dev/null
+++ b/scrapy/contrib/closespider.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.closespider` is deprecated, "
+              "use `scrapy.extensions.closespider` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.extensions.closespider import *
diff --git a/scrapy/contrib/corestats.py b/scrapy/contrib/corestats.py
new file mode 100644
index 000000000..2f5354239
--- /dev/null
+++ b/scrapy/contrib/corestats.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.corestats` is deprecated, "
+              "use `scrapy.extensions.corestats` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.extensions.corestats import *
diff --git a/scrapy/contrib/debug.py b/scrapy/contrib/debug.py
new file mode 100644
index 000000000..a38f059ce
--- /dev/null
+++ b/scrapy/contrib/debug.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.debug` is deprecated, "
+              "use `scrapy.extensions.debug` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.extensions.debug import *
diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py
new file mode 100644
index 000000000..19651998a
--- /dev/null
+++ b/scrapy/contrib/feedexport.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.feedexport` is deprecated, "
+              "use `scrapy.extensions.feedexport` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.extensions.feedexport import *
diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py
new file mode 100644
index 000000000..196372fcb
--- /dev/null
+++ b/scrapy/contrib/httpcache.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.httpcache` is deprecated, "
+              "use `scrapy.extensions.httpcache` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.extensions.httpcache import *
diff --git a/scrapy/contrib/logstats.py b/scrapy/contrib/logstats.py
new file mode 100644
index 000000000..62bc9b860
--- /dev/null
+++ b/scrapy/contrib/logstats.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.logstats` is deprecated, "
+              "use `scrapy.extensions.logstats` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.extensions.logstats import *
diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py
new file mode 100644
index 000000000..4f6e4760e
--- /dev/null
+++ b/scrapy/contrib/memdebug.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.memdebug` is deprecated, "
+              "use `scrapy.extensions.memdebug` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.extensions.memdebug import *
diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py
new file mode 100644
index 000000000..e13bd78f3
--- /dev/null
+++ b/scrapy/contrib/memusage.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.memusage` is deprecated, "
+              "use `scrapy.extensions.memusage` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.extensions.memusage import *
diff --git a/scrapy/contrib/spiderstate.py b/scrapy/contrib/spiderstate.py
new file mode 100644
index 000000000..06afc8bfc
--- /dev/null
+++ b/scrapy/contrib/spiderstate.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.spiderstate` is deprecated, "
+              "use `scrapy.extensions.spiderstate` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.extensions.spiderstate import *
diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py
new file mode 100644
index 000000000..f9c9a37f5
--- /dev/null
+++ b/scrapy/contrib/statsmailer.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.statsmailer` is deprecated, "
+              "use `scrapy.extensions.statsmailer` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.extensions.statsmailer import *
diff --git a/scrapy/contrib/throttle.py b/scrapy/contrib/throttle.py
new file mode 100644
index 000000000..d5c234871
--- /dev/null
+++ b/scrapy/contrib/throttle.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.throttle` is deprecated, "
+              "use `scrapy.extensions.throttle` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.extensions.throttle import *

From 645cdcbf9e6bbffd7fa7655091e64d67567c7f2f Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 23 Apr 2015 11:41:59 -0300
Subject: [PATCH 0264/2974] Rename scrapy/loader/processor.py to
 scrapy/loader/processors.py

---
 docs/topics/loaders.rst                       | 24 +++++++++----------
 scrapy/contrib/loader/processor.py            |  4 ++--
 scrapy/loader/__init__.py                     |  2 +-
 scrapy/loader/{processor.py => processors.py} |  0
 tests/test_loader.py                          |  2 +-
 5 files changed, 16 insertions(+), 16 deletions(-)
 rename scrapy/loader/{processor.py => processors.py} (100%)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index a5efa3fb9..4c24166e6 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -151,7 +151,7 @@ Item Loaders are declared like Items, by using a class definition syntax. Here
 is an example::
 
     from scrapy.loader import ItemLoader
-    from scrapy.loader.processor import TakeFirst, MapCompose, Join
+    from scrapy.loader.processors import TakeFirst, MapCompose, Join
 
     class ProductLoader(ItemLoader):
 
@@ -182,7 +182,7 @@ output processors to use: in the :ref:`Item Field <topics-items-fields>`
 metadata. Here is an example::
 
     import scrapy
-    from scrapy.loader.processor import Join, MapCompose, TakeFirst
+    from scrapy.loader.processors import Join, MapCompose, TakeFirst
     from w3lib.html import remove_tags
 
     def filter_price(value):
@@ -309,7 +309,7 @@ ItemLoader objects
 
         Examples::
 
-            >>> from scrapy.loader.processor import TakeFirst
+            >>> from scrapy.loader.processors import TakeFirst
             >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
             'FOO`
 
@@ -513,7 +513,7 @@ those dashes in the final product names.
 Here's how you can remove those dashes by reusing and extending the default
 Product Item Loader (``ProductLoader``)::
 
-    from scrapy.loader.processor import MapCompose
+    from scrapy.loader.processors import MapCompose
     from myproject.ItemLoaders import ProductLoader
 
     def strip_dashes(x):
@@ -526,7 +526,7 @@ Another case where extending Item Loaders can be very helpful is when you have
 multiple source formats, for example XML and HTML. In the XML version you may
 want to remove ``CDATA`` occurrences. Here's an example of how to do it::
 
-    from scrapy.loader.processor import MapCompose
+    from scrapy.loader.processors import MapCompose
     from myproject.ItemLoaders import ProductLoader
     from myproject.utils.xml import remove_cdata
 
@@ -551,7 +551,7 @@ needs.
 Available built-in processors
 =============================
 
-.. module:: scrapy.loader.processor
+.. module:: scrapy.loader.processors
    :synopsis: A collection of processors to use with Item Loaders
 
 Even though you can use any callable function as input and output processors,
@@ -570,7 +570,7 @@ Here is a list of all built-in processors:
 
     Example::
 
-        >>> from scrapy.loader.processor import Identity
+        >>> from scrapy.loader.processors import Identity
         >>> proc = Identity()
         >>> proc(['one', 'two', 'three'])
         ['one', 'two', 'three']
@@ -583,7 +583,7 @@ Here is a list of all built-in processors:
 
     Example::
 
-        >>> from scrapy.loader.processor import TakeFirst
+        >>> from scrapy.loader.processors import TakeFirst
         >>> proc = TakeFirst()
         >>> proc(['', 'one', 'two', 'three'])
         'one'
@@ -598,7 +598,7 @@ Here is a list of all built-in processors:
 
     Examples::
 
-        >>> from scrapy.loader.processor import Join
+        >>> from scrapy.loader.processors import Join
         >>> proc = Join()
         >>> proc(['one', 'two', 'three'])
         u'one two three'
@@ -619,7 +619,7 @@ Here is a list of all built-in processors:
 
     Example::
 
-        >>> from scrapy.loader.processor import Compose
+        >>> from scrapy.loader.processors import Compose
         >>> proc = Compose(lambda v: v[0], str.upper)
         >>> proc(['hello', 'world'])
         'HELLO'
@@ -666,7 +666,7 @@ Here is a list of all built-in processors:
         >>> def filter_world(x):
         ...     return None if x == 'world' else x
         ...
-        >>> from scrapy.loader.processor import MapCompose
+        >>> from scrapy.loader.processors import MapCompose
         >>> proc = MapCompose(filter_world, unicode.upper)
         >>> proc([u'hello', u'world', u'this', u'is', u'scrapy'])
         [u'HELLO, u'THIS', u'IS', u'SCRAPY']
@@ -683,7 +683,7 @@ Here is a list of all built-in processors:
 
     Example::
 
-        >>> from scrapy.loader.processor import SelectJmes, Compose, MapCompose
+        >>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
         >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
         >>> proc({'foo': 'bar'})
         'bar'
diff --git a/scrapy/contrib/loader/processor.py b/scrapy/contrib/loader/processor.py
index 68ddff4d4..da7e484a5 100644
--- a/scrapy/contrib/loader/processor.py
+++ b/scrapy/contrib/loader/processor.py
@@ -1,7 +1,7 @@
 import warnings
 from scrapy.exceptions import ScrapyDeprecationWarning
 warnings.warn("Module `scrapy.contrib.loader.processor` is deprecated, "
-              "use `scrapy.loader.processor` instead",
+              "use `scrapy.loader.processors` instead",
               ScrapyDeprecationWarning, stacklevel=2)
 
-from scrapy.loader.processor import *
+from scrapy.loader.processors import *
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index c861f8a24..b6d8506ca 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -14,7 +14,7 @@ from scrapy.utils.misc import arg_to_iter, extract_regex
 from scrapy.utils.python import flatten
 
 from .common import wrap_loader_context
-from .processor import Identity
+from .processors import Identity
 
 
 class ItemLoader(object):
diff --git a/scrapy/loader/processor.py b/scrapy/loader/processors.py
similarity index 100%
rename from scrapy/loader/processor.py
rename to scrapy/loader/processors.py
diff --git a/tests/test_loader.py b/tests/test_loader.py
index c51092ad0..6e8f7c0de 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -2,7 +2,7 @@ import unittest
 from functools import partial
 
 from scrapy.loader import ItemLoader
-from scrapy.loader.processor import Join, Identity, TakeFirst, \
+from scrapy.loader.processors import Join, Identity, TakeFirst, \
     Compose, MapCompose, SelectJmes
 from scrapy.item import Item, Field
 from scrapy.selector import Selector

From 7a7c539116df18e43b6e46b510af0d28a19f5084 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 23 Apr 2015 11:44:49 -0300
Subject: [PATCH 0265/2974] Rename scrapy/utils/decorator.py to
 scrapy/utils/decorators.py

---
 scrapy/core/downloader/handlers/file.py      | 2 +-
 scrapy/loader/__init__.py                    | 2 +-
 scrapy/selector/unified.py                   | 2 +-
 scrapy/utils/{decorator.py => decorators.py} | 0
 scrapy/utils/response.py                     | 2 +-
 5 files changed, 4 insertions(+), 4 deletions(-)
 rename scrapy/utils/{decorator.py => decorators.py} (100%)

diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py
index 85bad1583..5a63e9d08 100644
--- a/scrapy/core/downloader/handlers/file.py
+++ b/scrapy/core/downloader/handlers/file.py
@@ -1,6 +1,6 @@
 from w3lib.url import file_uri_to_path
 from scrapy.responsetypes import responsetypes
-from scrapy.utils.decorator import defers
+from scrapy.utils.decorators import defers
 
 class FileDownloadHandler(object):
 
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index b6d8506ca..84640f0b6 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -8,7 +8,7 @@ import six
 
 from scrapy.item import Item
 from scrapy.selector import Selector
-from scrapy.utils.decorator import deprecated
+from scrapy.utils.decorators import deprecated
 from scrapy.utils.deprecate import create_deprecated_class
 from scrapy.utils.misc import arg_to_iter, extract_regex
 from scrapy.utils.python import flatten
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 90dcfd686..164ded2bf 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -8,7 +8,7 @@ import six
 from scrapy.utils.misc import extract_regex
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.python import unicode_to_str, flatten, iflatten
-from scrapy.utils.decorator import deprecated
+from scrapy.utils.decorators import deprecated
 from scrapy.http import HtmlResponse, XmlResponse
 from .lxmldocument import LxmlDocument
 from .csstranslator import ScrapyHTMLTranslator, ScrapyGenericTranslator
diff --git a/scrapy/utils/decorator.py b/scrapy/utils/decorators.py
similarity index 100%
rename from scrapy/utils/decorator.py
rename to scrapy/utils/decorators.py
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 1d79ec0e3..b5d7a58ca 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -13,7 +13,7 @@ from twisted.web import http
 from twisted.web.http import RESPONSES
 from w3lib import html
 
-from scrapy.utils.decorator import deprecated
+from scrapy.utils.decorators import deprecated
 
 
 @deprecated

From 593b4ef56462b0de9224d05db6d91423e40acf85 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 23 Apr 2015 11:46:56 -0300
Subject: [PATCH 0266/2974] scrapy/utils/decorator.py shim

---
 scrapy/utils/decorator.py | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 scrapy/utils/decorator.py

diff --git a/scrapy/utils/decorator.py b/scrapy/utils/decorator.py
new file mode 100644
index 000000000..e8c8eae39
--- /dev/null
+++ b/scrapy/utils/decorator.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.utils.decorator` is deprecated, "
+              "use `scrapy.utils.decorators` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.utils.decorators import *

From f5bdf64f9cfd3802cc4d074878ce1d93f72a15db Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 23 Apr 2015 11:51:07 -0300
Subject: [PATCH 0267/2974] Rename scrapy/squeue.py to scrapy/squeues.py

---
 docs/faq.rst                              | 4 ++--
 scrapy/settings/default_settings.py       | 4 ++--
 scrapy/{squeue.py => squeues.py}          | 0
 tests/{test_squeue.py => test_squeues.py} | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)
 rename scrapy/{squeue.py => squeues.py} (100%)
 rename tests/{test_squeue.py => test_squeues.py} (97%)

diff --git a/docs/faq.rst b/docs/faq.rst
index d98b06056..212baa585 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -94,8 +94,8 @@ in most cases. If you do want to crawl in true `BFO order`_, you can do it by
 setting the following settings::
 
     DEPTH_PRIORITY = 1
-    SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleFifoDiskQueue'
-    SCHEDULER_MEMORY_QUEUE = 'scrapy.squeue.FifoMemoryQueue'
+    SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue'
+    SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue'
 
 My Scrapy crawler has memory leaks. What can I do?
 --------------------------------------------------
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 9debaabc3..f2098d626 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -215,8 +215,8 @@ RETRY_PRIORITY_ADJUST = -1
 ROBOTSTXT_OBEY = False
 
 SCHEDULER = 'scrapy.core.scheduler.Scheduler'
-SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleLifoDiskQueue'
-SCHEDULER_MEMORY_QUEUE = 'scrapy.squeue.LifoMemoryQueue'
+SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
+SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
 
 SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
 
diff --git a/scrapy/squeue.py b/scrapy/squeues.py
similarity index 100%
rename from scrapy/squeue.py
rename to scrapy/squeues.py
diff --git a/tests/test_squeue.py b/tests/test_squeues.py
similarity index 97%
rename from tests/test_squeue.py
rename to tests/test_squeues.py
index 16014a897..b3e9a2208 100644
--- a/tests/test_squeue.py
+++ b/tests/test_squeues.py
@@ -1,5 +1,5 @@
 from queuelib.tests import test_queue as t
-from scrapy.squeue import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue
+from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue
 from scrapy.item import Item, Field
 from scrapy.http import Request
 from scrapy.loader import ItemLoader

From fc346cba4d45fb5faf52983b8a25e2a3f5aa12f6 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 21 Apr 2015 13:20:08 -0300
Subject: [PATCH 0268/2974] Move scrapy/contrib/spiders to scrapy/spiders

---
 docs/intro/tutorial.rst                 |  2 +-
 docs/topics/commands.rst                |  2 +-
 docs/topics/downloader-middleware.rst   |  2 +-
 docs/topics/firebug.rst                 |  6 +++---
 docs/topics/link-extractors.rst         |  4 ++--
 docs/topics/spiders.rst                 | 16 ++++++++--------
 scrapy/contrib/spiders/__init__.py      |  3 ---
 scrapy/spiders/__init__.py              |  3 +++
 scrapy/{contrib => }/spiders/crawl.py   |  0
 scrapy/{contrib => }/spiders/feed.py    |  0
 scrapy/{contrib => }/spiders/init.py    |  0
 scrapy/{contrib => }/spiders/sitemap.py |  0
 scrapy/templates/spiders/crawl.tmpl     |  2 +-
 scrapy/templates/spiders/csvfeed.tmpl   |  2 +-
 scrapy/templates/spiders/xmlfeed.tmpl   |  2 +-
 tests/test_spider.py                    |  4 ++--
 tests/test_utils_spider.py              |  2 +-
 17 files changed, 25 insertions(+), 25 deletions(-)
 delete mode 100644 scrapy/contrib/spiders/__init__.py
 create mode 100644 scrapy/spiders/__init__.py
 rename scrapy/{contrib => }/spiders/crawl.py (100%)
 rename scrapy/{contrib => }/spiders/feed.py (100%)
 rename scrapy/{contrib => }/spiders/init.py (100%)
 rename scrapy/{contrib => }/spiders/sitemap.py (100%)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 479b17f99..219616587 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -511,7 +511,7 @@ using a `trick to pass additional data to the callbacks
 
 .. note::
     As an example spider that leverages this mechanism, check out the
-    :class:`~scrapy.contrib.spiders.CrawlSpider` class for a generic spider
+    :class:`~scrapy.spiders.CrawlSpider` class for a generic spider
     that implements a small rules engine that you can use to write your
     crawlers on top of it.
 
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 638de5712..f1f79ce70 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -399,7 +399,7 @@ Supported options:
 
 * ``--pipelines``: process items through pipelines
 
-* ``--rules`` or ``-r``: use :class:`~scrapy.contrib.spiders.CrawlSpider`
+* ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider`
   rules to discover the callback (i.e. spider method) to use for parsing the
   response
 
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index be19bc44f..bb969eca3 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -288,7 +288,7 @@ HttpAuthMiddleware
 
     Example::
 
-        from scrapy.contrib.spiders import CrawlSpider
+        from scrapy.spiders import CrawlSpider
 
         class SomeIntranetSiteSpider(CrawlSpider):
 
diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst
index 3bbb902b2..359c99450 100644
--- a/docs/topics/firebug.rst
+++ b/docs/topics/firebug.rst
@@ -74,15 +74,15 @@ So, based on that regular expression we can create the first crawling rule::
         follow=True,
     ),
 
-The :class:`~scrapy.contrib.spiders.Rule` object instructs
-:class:`~scrapy.contrib.spiders.CrawlSpider` based spiders how to follow the
+The :class:`~scrapy.spiders.Rule` object instructs
+:class:`~scrapy.spiders.CrawlSpider` based spiders how to follow the
 category links. ``parse_category`` will be a method of the spider which will
 process and extract data from those pages.
 
 This is how the spider would look so far::
 
    from scrapy.linkextractors import LinkExtractor
-   from scrapy.contrib.spiders import CrawlSpider, Rule
+   from scrapy.spiders import CrawlSpider, Rule
 
    class GoogleDirectorySpider(CrawlSpider):
        name = 'directory.google.com'
diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index be3eb4537..edb047c86 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -18,10 +18,10 @@ of :class:`scrapy.link.Link` objects. Link extractors are meant to be
 instantiated once and their ``extract_links`` method called several times
 with different responses to extract links to follow.
 
-Link extractors are used in the :class:`~scrapy.contrib.spiders.CrawlSpider`
+Link extractors are used in the :class:`~scrapy.spiders.CrawlSpider`
 class (available in Scrapy), through a set of rules, but you can also use it in
 your spiders, even if you don't subclass from
-:class:`~scrapy.contrib.spiders.CrawlSpider`, as its purpose is very simple: to
+:class:`~scrapy.spiders.CrawlSpider`, as its purpose is very simple: to
 extract links.
 
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index df030170c..d2fdd61b8 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -319,7 +319,7 @@ with a ``TestItem`` declared in a ``myproject.items`` module::
         description = scrapy.Field()
 
 
-.. module:: scrapy.contrib.spiders
+.. module:: scrapy.spiders
    :synopsis: Collection of generic spiders
 
 CrawlSpider
@@ -394,7 +394,7 @@ CrawlSpider example
 Let's now take a look at an example CrawlSpider with rules::
 
     import scrapy
-    from scrapy.contrib.spiders import CrawlSpider, Rule
+    from scrapy.spiders import CrawlSpider, Rule
     from scrapy.linkextractors import LinkExtractor
 
     class MySpider(CrawlSpider):
@@ -515,7 +515,7 @@ XMLFeedSpider example
 
 These spiders are pretty easy to use, let's have a look at one example::
 
-    from scrapy.contrib.spiders import XMLFeedSpider
+    from scrapy.spiders import XMLFeedSpider
     from myproject.items import TestItem
 
     class MySpider(XMLFeedSpider):
@@ -575,7 +575,7 @@ CSVFeedSpider example
 Let's see an example similar to the previous one, but using a
 :class:`CSVFeedSpider`::
 
-    from scrapy.contrib.spiders import CSVFeedSpider
+    from scrapy.spiders import CSVFeedSpider
     from myproject.items import TestItem
 
     class MySpider(CSVFeedSpider):
@@ -669,7 +669,7 @@ SitemapSpider examples
 Simplest example: process all urls discovered through sitemaps using the
 ``parse`` callback::
 
-    from scrapy.contrib.spiders import SitemapSpider
+    from scrapy.spiders import SitemapSpider
 
     class MySpider(SitemapSpider):
         sitemap_urls = ['http://www.example.com/sitemap.xml']
@@ -680,7 +680,7 @@ Simplest example: process all urls discovered through sitemaps using the
 Process some urls with certain callback and other urls with a different
 callback::
 
-    from scrapy.contrib.spiders import SitemapSpider
+    from scrapy.spiders import SitemapSpider
 
     class MySpider(SitemapSpider):
         sitemap_urls = ['http://www.example.com/sitemap.xml']
@@ -698,7 +698,7 @@ callback::
 Follow sitemaps defined in the `robots.txt`_ file and only follow sitemaps
 whose url contains ``/sitemap_shop``::
 
-    from scrapy.contrib.spiders import SitemapSpider
+    from scrapy.spiders import SitemapSpider
 
     class MySpider(SitemapSpider):
         sitemap_urls = ['http://www.example.com/robots.txt']
@@ -712,7 +712,7 @@ whose url contains ``/sitemap_shop``::
 
 Combine SitemapSpider with other sources of urls::
 
-    from scrapy.contrib.spiders import SitemapSpider
+    from scrapy.spiders import SitemapSpider
 
     class MySpider(SitemapSpider):
         sitemap_urls = ['http://www.example.com/robots.txt']
diff --git a/scrapy/contrib/spiders/__init__.py b/scrapy/contrib/spiders/__init__.py
deleted file mode 100644
index c16bb6c0a..000000000
--- a/scrapy/contrib/spiders/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from scrapy.contrib.spiders.crawl import CrawlSpider, Rule
-from scrapy.contrib.spiders.feed import XMLFeedSpider, CSVFeedSpider
-from scrapy.contrib.spiders.sitemap import SitemapSpider
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
new file mode 100644
index 000000000..de4f90d51
--- /dev/null
+++ b/scrapy/spiders/__init__.py
@@ -0,0 +1,3 @@
+from scrapy.spiders.crawl import CrawlSpider, Rule
+from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider
+from scrapy.spiders.sitemap import SitemapSpider
diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/spiders/crawl.py
similarity index 100%
rename from scrapy/contrib/spiders/crawl.py
rename to scrapy/spiders/crawl.py
diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/spiders/feed.py
similarity index 100%
rename from scrapy/contrib/spiders/feed.py
rename to scrapy/spiders/feed.py
diff --git a/scrapy/contrib/spiders/init.py b/scrapy/spiders/init.py
similarity index 100%
rename from scrapy/contrib/spiders/init.py
rename to scrapy/spiders/init.py
diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/spiders/sitemap.py
similarity index 100%
rename from scrapy/contrib/spiders/sitemap.py
rename to scrapy/spiders/sitemap.py
diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index b84e785fd..a179d16ff 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import scrapy
 from scrapy.linkextractors import LinkExtractor
-from scrapy.contrib.spiders import CrawlSpider, Rule
+from scrapy.spiders import CrawlSpider, Rule
 
 from $project_name.items import ${ProjectName}Item
 
diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl
index 7e10cd8a8..69c606538 100644
--- a/scrapy/templates/spiders/csvfeed.tmpl
+++ b/scrapy/templates/spiders/csvfeed.tmpl
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from scrapy.contrib.spiders import CSVFeedSpider
+from scrapy.spiders import CSVFeedSpider
 
 from $project_name.items import ${ProjectName}Item
 
diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl
index a0e26aa14..9c0910d23 100644
--- a/scrapy/templates/spiders/xmlfeed.tmpl
+++ b/scrapy/templates/spiders/xmlfeed.tmpl
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from scrapy.contrib.spiders import XMLFeedSpider
+from scrapy.spiders import XMLFeedSpider
 
 from $project_name.items import ${ProjectName}Item
 
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 02352ec29..f771399ca 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -10,8 +10,8 @@ from scrapy import signals
 from scrapy.spider import Spider, BaseSpider
 from scrapy.settings import Settings
 from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse
-from scrapy.contrib.spiders.init import InitSpider
-from scrapy.contrib.spiders import CrawlSpider, Rule, XMLFeedSpider, \
+from scrapy.spiders.init import InitSpider
+from scrapy.spiders import CrawlSpider, Rule, XMLFeedSpider, \
     CSVFeedSpider, SitemapSpider
 from scrapy.linkextractors import LinkExtractor
 from scrapy.exceptions import ScrapyDeprecationWarning
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index 334ec00f3..045e72117 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -3,7 +3,7 @@ from scrapy.http import Request
 from scrapy.item import BaseItem
 from scrapy.utils.spider import iterate_spider_output, iter_spider_classes
 
-from scrapy.contrib.spiders import CrawlSpider
+from scrapy.spiders import CrawlSpider
 
 
 class MyBaseSpider(CrawlSpider):

From 78dcd4e1e71501da3e4e13780b6c6b89738d61c5 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 23 Apr 2015 11:52:58 -0300
Subject: [PATCH 0269/2974] scrapy/squeue.py shim

---
 scrapy/squeue.py | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 scrapy/squeue.py

diff --git a/scrapy/squeue.py b/scrapy/squeue.py
new file mode 100644
index 000000000..a4a3f4238
--- /dev/null
+++ b/scrapy/squeue.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.squeue` is deprecated, "
+              "use `scrapy.squeues` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.squeues import *

From e262c5b8d5225897dda7976d2376c0ac29bf73ac Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 21 Apr 2015 13:26:53 -0300
Subject: [PATCH 0270/2974] scrapy/contrib/spiders shims

---
 scrapy/contrib/spiders/__init__.py | 7 +++++++
 scrapy/contrib/spiders/crawl.py    | 7 +++++++
 scrapy/contrib/spiders/feed.py     | 7 +++++++
 scrapy/contrib/spiders/init.py     | 7 +++++++
 scrapy/contrib/spiders/sitemap.py  | 7 +++++++
 5 files changed, 35 insertions(+)
 create mode 100644 scrapy/contrib/spiders/__init__.py
 create mode 100644 scrapy/contrib/spiders/crawl.py
 create mode 100644 scrapy/contrib/spiders/feed.py
 create mode 100644 scrapy/contrib/spiders/init.py
 create mode 100644 scrapy/contrib/spiders/sitemap.py

diff --git a/scrapy/contrib/spiders/__init__.py b/scrapy/contrib/spiders/__init__.py
new file mode 100644
index 000000000..56780533b
--- /dev/null
+++ b/scrapy/contrib/spiders/__init__.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.spiders` is deprecated, "
+              "use `scrapy.spiders` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.spiders import *
diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py
new file mode 100644
index 000000000..d20a8bb16
--- /dev/null
+++ b/scrapy/contrib/spiders/crawl.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.spiders.crawl` is deprecated, "
+              "use `scrapy.spiders.crawl` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.spiders.crawl import *
diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py
new file mode 100644
index 000000000..5eea9a062
--- /dev/null
+++ b/scrapy/contrib/spiders/feed.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.spiders.feed` is deprecated, "
+              "use `scrapy.spiders.feed` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.spiders.feed import *
diff --git a/scrapy/contrib/spiders/init.py b/scrapy/contrib/spiders/init.py
new file mode 100644
index 000000000..6d1ec0aa9
--- /dev/null
+++ b/scrapy/contrib/spiders/init.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.spiders.init` is deprecated, "
+              "use `scrapy.spiders.init` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.spiders.init import *
diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py
new file mode 100644
index 000000000..2ad231fd8
--- /dev/null
+++ b/scrapy/contrib/spiders/sitemap.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib.spiders.sitemap` is deprecated, "
+              "use `scrapy.spiders.sitemap` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.spiders.sitemap import *

From 54a4ce069c884fba8edec1fe1b850ec5634e93f9 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 23 Apr 2015 11:57:14 -0300
Subject: [PATCH 0271/2974] Rename scrapy/dupefilter.py to
 scrapy/dupefilters.py

---
 docs/topics/settings.rst                          | 2 +-
 scrapy/{dupefilter.py => dupefilters.py}          | 0
 scrapy/settings/default_settings.py               | 2 +-
 tests/py3-ignores.txt                             | 2 +-
 tests/{test_dupefilter.py => test_dupefilters.py} | 2 +-
 5 files changed, 4 insertions(+), 4 deletions(-)
 rename scrapy/{dupefilter.py => dupefilters.py} (100%)
 rename tests/{test_dupefilter.py => test_dupefilters.py} (97%)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index f331b1dba..f035bc33d 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -487,7 +487,7 @@ If you want to disable it set to 0.
 DUPEFILTER_CLASS
 ----------------
 
-Default: ``'scrapy.dupefilter.RFPDupeFilter'``
+Default: ``'scrapy.dupefilters.RFPDupeFilter'``
 
 The class used to detect and filter duplicate requests.
 
diff --git a/scrapy/dupefilter.py b/scrapy/dupefilters.py
similarity index 100%
rename from scrapy/dupefilter.py
rename to scrapy/dupefilters.py
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index f2098d626..57dfc3256 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -100,7 +100,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {
 
 DOWNLOADER_STATS = True
 
-DUPEFILTER_CLASS = 'scrapy.dupefilter.RFPDupeFilter'
+DUPEFILTER_CLASS = 'scrapy.dupefilters.RFPDupeFilter'
 
 try:
     EDITOR = os.environ['EDITOR']
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 3d87bcb9a..e0871c350 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -25,7 +25,7 @@ tests/test_downloadermiddleware_retry.py
 tests/test_downloadermiddleware_robotstxt.py
 tests/test_downloadermiddleware_stats.py
 tests/test_downloadermiddleware_useragent.py
-tests/test_dupefilter.py
+tests/test_dupefilters.py
 tests/test_engine.py
 tests/test_http_cookies.py
 tests/test_http_request.py
diff --git a/tests/test_dupefilter.py b/tests/test_dupefilters.py
similarity index 97%
rename from tests/test_dupefilter.py
rename to tests/test_dupefilters.py
index b0dd9546b..f8f800a72 100644
--- a/tests/test_dupefilter.py
+++ b/tests/test_dupefilters.py
@@ -1,7 +1,7 @@
 import hashlib
 import unittest
 
-from scrapy.dupefilter import RFPDupeFilter
+from scrapy.dupefilters import RFPDupeFilter
 from scrapy.http import Request
 
 

From 9a3e3ba505361efd12e49a42f1c77f5fa815b86a Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 21 Apr 2015 13:48:28 -0300
Subject: [PATCH 0272/2974] Move scrapy/contrib remaining top-level files to
 scrapy/extensions

---
 docs/topics/downloader-middleware.rst         | 14 ++++-----
 docs/topics/extensions.rst                    | 30 +++++++++----------
 docs/topics/feed-exports.rst                  | 12 ++++----
 docs/topics/settings.rst                      | 28 ++++++++---------
 scrapy/downloadermiddlewares/httpcache.py     | 13 --------
 scrapy/extensions/__init__.py                 |  0
 scrapy/{contrib => extensions}/closespider.py |  0
 scrapy/{contrib => extensions}/corestats.py   |  0
 scrapy/{contrib => extensions}/debug.py       |  0
 scrapy/{contrib => extensions}/feedexport.py  |  0
 scrapy/{contrib => extensions}/httpcache.py   |  0
 scrapy/{contrib => extensions}/logstats.py    |  0
 scrapy/{contrib => extensions}/memdebug.py    |  0
 scrapy/{contrib => extensions}/memusage.py    |  0
 scrapy/{contrib => extensions}/spiderstate.py |  0
 scrapy/{contrib => extensions}/statsmailer.py |  0
 scrapy/{contrib => extensions}/throttle.py    |  0
 scrapy/settings/default_settings.py           | 30 +++++++++----------
 .../templates/project/module/settings.py.tmpl |  2 +-
 tests/py3-ignores.txt                         |  2 ++
 tests/test_downloadermiddleware_httpcache.py  | 14 ++++-----
 ...ntrib_feedexport.py => test_feedexport.py} |  2 +-
 ...rib_spiderstate.py => test_spiderstate.py} |  2 +-
 23 files changed, 69 insertions(+), 80 deletions(-)
 create mode 100644 scrapy/extensions/__init__.py
 rename scrapy/{contrib => extensions}/closespider.py (100%)
 rename scrapy/{contrib => extensions}/corestats.py (100%)
 rename scrapy/{contrib => extensions}/debug.py (100%)
 rename scrapy/{contrib => extensions}/feedexport.py (100%)
 rename scrapy/{contrib => extensions}/httpcache.py (100%)
 rename scrapy/{contrib => extensions}/logstats.py (100%)
 rename scrapy/{contrib => extensions}/memdebug.py (100%)
 rename scrapy/{contrib => extensions}/memusage.py (100%)
 rename scrapy/{contrib => extensions}/spiderstate.py (100%)
 rename scrapy/{contrib => extensions}/statsmailer.py (100%)
 rename scrapy/{contrib => extensions}/throttle.py (100%)
 rename tests/{test_contrib_feedexport.py => test_feedexport.py} (99%)
 rename tests/{test_contrib_spiderstate.py => test_spiderstate.py} (94%)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index bb969eca3..5cb6c9824 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -349,7 +349,7 @@ when an Internet connection is not available. The goal is to be able to
 
 In order to use this policy, set:
 
-* :setting:`HTTPCACHE_POLICY` to ``scrapy.contrib.httpcache.DummyPolicy``
+* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.DummyPolicy``
 
 
 .. _httpcache-policy-rfc2616:
@@ -383,7 +383,7 @@ what is missing:
 
 In order to use this policy, set:
 
-* :setting:`HTTPCACHE_POLICY` to ``scrapy.contrib.httpcache.RFC2616Policy``
+* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.RFC2616Policy``
 
 
 .. _httpcache-storage-fs:
@@ -395,7 +395,7 @@ File system storage backend is available for the HTTP cache middleware.
 
 In order to use this storage backend, set:
 
-* :setting:`HTTPCACHE_STORAGE` to ``scrapy.contrib.httpcache.FilesystemCacheStorage``
+* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.FilesystemCacheStorage``
 
 Each request/response pair is stored in a different directory containing
 the following files:
@@ -430,7 +430,7 @@ By default, it uses the anydbm_ module, but you can change it with the
 
 In order to use this storage backend, set:
 
-* :setting:`HTTPCACHE_STORAGE` to ``scrapy.contrib.httpcache.DbmCacheStorage``
+* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.DbmCacheStorage``
 
 .. _httpcache-storage-leveldb:
 
@@ -447,7 +447,7 @@ the scrapy shell in parallel for the same spider.
 
 In order to use this storage backend:
 
-* set :setting:`HTTPCACHE_STORAGE` to ``scrapy.contrib.httpcache.LeveldbCacheStorage``
+* set :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.LeveldbCacheStorage``
 * install `LevelDB python bindings`_ like ``pip install leveldb``
 
 .. _LevelDB: http://code.google.com/p/leveldb/
@@ -536,7 +536,7 @@ Don't cache responses with these URI schemes.
 HTTPCACHE_STORAGE
 ^^^^^^^^^^^^^^^^^
 
-Default: ``'scrapy.contrib.httpcache.FilesystemCacheStorage'``
+Default: ``'scrapy.extensions.httpcache.FilesystemCacheStorage'``
 
 The class which implements the cache storage backend.
 
@@ -559,7 +559,7 @@ HTTPCACHE_POLICY
 
 .. versionadded:: 0.18
 
-Default: ``'scrapy.contrib.httpcache.DummyPolicy'``
+Default: ``'scrapy.extensions.httpcache.DummyPolicy'``
 
 The class which implements the cache policy.
 
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 19c296651..d5d985087 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -35,7 +35,7 @@ your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
 by a string: the full Python path to the extension's class name. For example::
 
     EXTENSIONS = {
-        'scrapy.contrib.corestats.CoreStats': 500,
+        'scrapy.extensions.corestats.CoreStats': 500,
         'scrapy.telnet.TelnetConsole': 500,
     }
 
@@ -69,7 +69,7 @@ included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to
 ``None``. For example::
 
     EXTENSIONS = {
-        'scrapy.contrib.corestats.CoreStats': None,
+        'scrapy.extensions.corestats.CoreStats': None,
     }
 
 Writing your own extension
@@ -158,7 +158,7 @@ General purpose extensions
 Log Stats extension
 ~~~~~~~~~~~~~~~~~~~
 
-.. module:: scrapy.contrib.logstats
+.. module:: scrapy.extensions.logstats
    :synopsis: Basic stats logging
 
 .. class:: LogStats
@@ -168,7 +168,7 @@ Log basic stats like crawled pages and scraped items.
 Core Stats extension
 ~~~~~~~~~~~~~~~~~~~~
 
-.. module:: scrapy.contrib.corestats
+.. module:: scrapy.extensions.corestats
    :synopsis: Core stats collection
 
 .. class:: CoreStats
@@ -198,10 +198,10 @@ setting, and the server will listen in the port specified in
 Memory usage extension
 ~~~~~~~~~~~~~~~~~~~~~~
 
-.. module:: scrapy.contrib.memusage
+.. module:: scrapy.extensions.memusage
    :synopsis: Memory usage extension
 
-.. class:: scrapy.contrib.memusage.MemoryUsage
+.. class:: scrapy.extensions.memusage.MemoryUsage
 
 .. note:: This extension does not work in Windows.
 
@@ -226,10 +226,10 @@ can be configured with the following settings:
 Memory debugger extension
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. module:: scrapy.contrib.memdebug
+.. module:: scrapy.extensions.memdebug
    :synopsis: Memory debugger extension
 
-.. class:: scrapy.contrib.memdebug.MemoryDebugger
+.. class:: scrapy.extensions.memdebug.MemoryDebugger
 
 An extension for debugging memory usage. It collects information about:
 
@@ -242,10 +242,10 @@ info will be stored in the stats.
 Close spider extension
 ~~~~~~~~~~~~~~~~~~~~~~
 
-.. module:: scrapy.contrib.closespider
+.. module:: scrapy.extensions.closespider
    :synopsis: Close spider extension
 
-.. class:: scrapy.contrib.closespider.CloseSpider
+.. class:: scrapy.extensions.closespider.CloseSpider
 
 Closes a spider automatically when some conditions are met, using a specific
 closing reason for each condition.
@@ -313,17 +313,17 @@ set), spiders won't be closed by number of errors.
 StatsMailer extension
 ~~~~~~~~~~~~~~~~~~~~~
 
-.. module:: scrapy.contrib.statsmailer
+.. module:: scrapy.extensions.statsmailer
    :synopsis: StatsMailer extension
 
-.. class:: scrapy.contrib.statsmailer.StatsMailer
+.. class:: scrapy.extensions.statsmailer.StatsMailer
 
 This simple extension can be used to send a notification e-mail every time a
 domain has finished scraping, including the Scrapy stats collected. The email
 will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS`
 setting.
 
-.. module:: scrapy.contrib.debug
+.. module:: scrapy.extensions.debug
    :synopsis: Extensions for debugging Scrapy
 
 Debugging extensions
@@ -332,7 +332,7 @@ Debugging extensions
 Stack trace dump extension
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. class:: scrapy.contrib.debug.StackTraceDump
+.. class:: scrapy.extensions.debug.StackTraceDump
 
 Dumps information about the running process when a `SIGQUIT`_ or `SIGUSR2`_
 signal is received. The information dumped is the following:
@@ -361,7 +361,7 @@ There are at least two ways to send Scrapy the `SIGQUIT`_ signal:
 Debugger extension
 ~~~~~~~~~~~~~~~~~~
 
-.. class:: scrapy.contrib.debug.Debugger
+.. class:: scrapy.extensions.debug.Debugger
 
 Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_
 signal is received. After the debugger is exited, the Scrapy process continues
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 27d601a19..faf9abc1c 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -209,7 +209,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_STORE_EMPTY`
  * :setting:`FEED_EXPORT_FIELDS`
 
-.. currentmodule:: scrapy.contrib.feedexport
+.. currentmodule:: scrapy.extensions.feedexport
 
 .. setting:: FEED_URI
 
@@ -272,11 +272,11 @@ FEED_STORAGES_BASE
 Default::
 
     {
-        '': 'scrapy.contrib.feedexport.FileFeedStorage',
-        'file': 'scrapy.contrib.feedexport.FileFeedStorage',
-        'stdout': 'scrapy.contrib.feedexport.StdoutFeedStorage',
-        's3': 'scrapy.contrib.feedexport.S3FeedStorage',
-        'ftp': 'scrapy.contrib.feedexport.FTPFeedStorage',
+        '': 'scrapy.extensions.feedexport.FileFeedStorage',
+        'file': 'scrapy.extensions.feedexport.FileFeedStorage',
+        'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
+        's3': 'scrapy.extensions.feedexport.S3FeedStorage',
+        'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
     }
 
 A dict containing the built-in feed storage backends supported by Scrapy.
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index a049f1438..f331b1dba 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -536,15 +536,15 @@ EXTENSIONS_BASE
 Default::
 
     {
-        'scrapy.contrib.corestats.CoreStats': 0,
+        'scrapy.extensions.corestats.CoreStats': 0,
         'scrapy.telnet.TelnetConsole': 0,
-        'scrapy.contrib.memusage.MemoryUsage': 0,
-        'scrapy.contrib.memdebug.MemoryDebugger': 0,
-        'scrapy.contrib.closespider.CloseSpider': 0,
-        'scrapy.contrib.feedexport.FeedExporter': 0,
-        'scrapy.contrib.logstats.LogStats': 0,
-        'scrapy.contrib.spiderstate.SpiderState': 0,
-        'scrapy.contrib.throttle.AutoThrottle': 0,
+        'scrapy.extensions.memusage.MemoryUsage': 0,
+        'scrapy.extensions.memdebug.MemoryDebugger': 0,
+        'scrapy.extensions.closespider.CloseSpider': 0,
+        'scrapy.extensions.feedexport.FeedExporter': 0,
+        'scrapy.extensions.logstats.LogStats': 0,
+        'scrapy.extensions.spiderstate.SpiderState': 0,
+        'scrapy.extensions.throttle.AutoThrottle': 0,
     }
 
 The list of available extensions. Keep in mind that some of them need to
@@ -689,7 +689,7 @@ MEMUSAGE_ENABLED
 
 Default: ``False``
 
-Scope: ``scrapy.contrib.memusage``
+Scope: ``scrapy.extensions.memusage``
 
 Whether to enable the memory usage extension that will shutdown the Scrapy
 process when it exceeds a memory limit, and also notify by email when that
@@ -704,7 +704,7 @@ MEMUSAGE_LIMIT_MB
 
 Default: ``0``
 
-Scope: ``scrapy.contrib.memusage``
+Scope: ``scrapy.extensions.memusage``
 
 The maximum amount of memory to allow (in megabytes) before shutting down
 Scrapy  (if MEMUSAGE_ENABLED is True). If zero, no check will be performed.
@@ -718,7 +718,7 @@ MEMUSAGE_NOTIFY_MAIL
 
 Default: ``False``
 
-Scope: ``scrapy.contrib.memusage``
+Scope: ``scrapy.extensions.memusage``
 
 A list of emails to notify if the memory limit has been reached.
 
@@ -735,7 +735,7 @@ MEMUSAGE_REPORT
 
 Default: ``False``
 
-Scope: ``scrapy.contrib.memusage``
+Scope: ``scrapy.extensions.memusage``
 
 Whether to send a memory usage report after each spider has been closed.
 
@@ -748,7 +748,7 @@ MEMUSAGE_WARNING_MB
 
 Default: ``0``
 
-Scope: ``scrapy.contrib.memusage``
+Scope: ``scrapy.extensions.memusage``
 
 The maximum amount of memory to allow (in megabytes) before sending a warning
 email notifying about it. If zero, no warning will be produced.
@@ -961,7 +961,7 @@ STATSMAILER_RCPTS
 Default: ``[]`` (empty list)
 
 Send Scrapy stats after spiders finish scraping. See
-:class:`~scrapy.contrib.statsmailer.StatsMailer` for more info.
+:class:`~scrapy.extensions.statsmailer.StatsMailer` for more info.
 
 .. setting:: TELNETCONSOLE_ENABLED
 
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 7b4b53f7c..bd112c48d 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -90,16 +90,3 @@ class HttpCacheMiddleware(object):
             self.storage.store_response(spider, request, response)
         else:
             self.stats.inc_value('httpcache/uncacheable', spider=spider)
-
-
-from scrapy.contrib.httpcache import FilesystemCacheStorage as _FilesystemCacheStorage
-class FilesystemCacheStorage(_FilesystemCacheStorage):
-
-    def __init__(self, *args, **kwargs):
-        import warnings
-        from scrapy.exceptions import ScrapyDeprecationWarning
-        warnings.warn('Importing FilesystemCacheStorage from '
-                      'scrapy.contrib.downloadermiddlware.httpcache is '
-                      'deprecated, use scrapy.contrib.httpcache instead.',
-                      category=ScrapyDeprecationWarning, stacklevel=1)
-        super(FilesystemCacheStorage, self).__init__(*args, **kwargs)
diff --git a/scrapy/extensions/__init__.py b/scrapy/extensions/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/scrapy/contrib/closespider.py b/scrapy/extensions/closespider.py
similarity index 100%
rename from scrapy/contrib/closespider.py
rename to scrapy/extensions/closespider.py
diff --git a/scrapy/contrib/corestats.py b/scrapy/extensions/corestats.py
similarity index 100%
rename from scrapy/contrib/corestats.py
rename to scrapy/extensions/corestats.py
diff --git a/scrapy/contrib/debug.py b/scrapy/extensions/debug.py
similarity index 100%
rename from scrapy/contrib/debug.py
rename to scrapy/extensions/debug.py
diff --git a/scrapy/contrib/feedexport.py b/scrapy/extensions/feedexport.py
similarity index 100%
rename from scrapy/contrib/feedexport.py
rename to scrapy/extensions/feedexport.py
diff --git a/scrapy/contrib/httpcache.py b/scrapy/extensions/httpcache.py
similarity index 100%
rename from scrapy/contrib/httpcache.py
rename to scrapy/extensions/httpcache.py
diff --git a/scrapy/contrib/logstats.py b/scrapy/extensions/logstats.py
similarity index 100%
rename from scrapy/contrib/logstats.py
rename to scrapy/extensions/logstats.py
diff --git a/scrapy/contrib/memdebug.py b/scrapy/extensions/memdebug.py
similarity index 100%
rename from scrapy/contrib/memdebug.py
rename to scrapy/extensions/memdebug.py
diff --git a/scrapy/contrib/memusage.py b/scrapy/extensions/memusage.py
similarity index 100%
rename from scrapy/contrib/memusage.py
rename to scrapy/extensions/memusage.py
diff --git a/scrapy/contrib/spiderstate.py b/scrapy/extensions/spiderstate.py
similarity index 100%
rename from scrapy/contrib/spiderstate.py
rename to scrapy/extensions/spiderstate.py
diff --git a/scrapy/contrib/statsmailer.py b/scrapy/extensions/statsmailer.py
similarity index 100%
rename from scrapy/contrib/statsmailer.py
rename to scrapy/extensions/statsmailer.py
diff --git a/scrapy/contrib/throttle.py b/scrapy/extensions/throttle.py
similarity index 100%
rename from scrapy/contrib/throttle.py
rename to scrapy/extensions/throttle.py
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index a7fe36648..9debaabc3 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -113,15 +113,15 @@ except KeyError:
 EXTENSIONS = {}
 
 EXTENSIONS_BASE = {
-    'scrapy.contrib.corestats.CoreStats': 0,
+    'scrapy.extensions.corestats.CoreStats': 0,
     'scrapy.telnet.TelnetConsole': 0,
-    'scrapy.contrib.memusage.MemoryUsage': 0,
-    'scrapy.contrib.memdebug.MemoryDebugger': 0,
-    'scrapy.contrib.closespider.CloseSpider': 0,
-    'scrapy.contrib.feedexport.FeedExporter': 0,
-    'scrapy.contrib.logstats.LogStats': 0,
-    'scrapy.contrib.spiderstate.SpiderState': 0,
-    'scrapy.contrib.throttle.AutoThrottle': 0,
+    'scrapy.extensions.memusage.MemoryUsage': 0,
+    'scrapy.extensions.memdebug.MemoryDebugger': 0,
+    'scrapy.extensions.closespider.CloseSpider': 0,
+    'scrapy.extensions.feedexport.FeedExporter': 0,
+    'scrapy.extensions.logstats.LogStats': 0,
+    'scrapy.extensions.spiderstate.SpiderState': 0,
+    'scrapy.extensions.throttle.AutoThrottle': 0,
 }
 
 FEED_URI = None
@@ -131,11 +131,11 @@ FEED_STORE_EMPTY = False
 FEED_EXPORT_FIELDS = None
 FEED_STORAGES = {}
 FEED_STORAGES_BASE = {
-    '': 'scrapy.contrib.feedexport.FileFeedStorage',
-    'file': 'scrapy.contrib.feedexport.FileFeedStorage',
-    'stdout': 'scrapy.contrib.feedexport.StdoutFeedStorage',
-    's3': 'scrapy.contrib.feedexport.S3FeedStorage',
-    'ftp': 'scrapy.contrib.feedexport.FTPFeedStorage',
+    '': 'scrapy.extensions.feedexport.FileFeedStorage',
+    'file': 'scrapy.extensions.feedexport.FileFeedStorage',
+    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
+    's3': 'scrapy.extensions.feedexport.S3FeedStorage',
+    'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
@@ -151,12 +151,12 @@ FEED_EXPORTERS_BASE = {
 HTTPCACHE_ENABLED = False
 HTTPCACHE_DIR = 'httpcache'
 HTTPCACHE_IGNORE_MISSING = False
-HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.FilesystemCacheStorage'
+HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
 HTTPCACHE_EXPIRATION_SECS = 0
 HTTPCACHE_IGNORE_HTTP_CODES = []
 HTTPCACHE_IGNORE_SCHEMES = ['file']
 HTTPCACHE_DBM_MODULE = 'anydbm'
-HTTPCACHE_POLICY = 'scrapy.contrib.httpcache.DummyPolicy'
+HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
 HTTPCACHE_GZIP = False
 
 ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 37cbb4d32..11ff804ba 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -82,4 +82,4 @@ NEWSPIDER_MODULE = '$project_name.spiders'
 #HTTPCACHE_EXPIRATION_SECS=0
 #HTTPCACHE_DIR='httpcache'
 #HTTPCACHE_IGNORE_HTTP_CODES=[]
-#HTTPCACHE_STORAGE='scrapy.contrib.httpcache.FilesystemCacheStorage'
+#HTTPCACHE_STORAGE='scrapy.extensions.httpcache.FilesystemCacheStorage'
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index a29b419b0..3d87bcb9a 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -100,7 +100,9 @@ scrapy/downloadermiddlewares/retry.py
 scrapy/downloadermiddlewares/httpproxy.py
 scrapy/downloadermiddlewares/cookies.py
 scrapy/contrib/statsmailer.py
+scrapy/extensions/statsmailer.py
 scrapy/contrib/memusage.py
+scrapy/extensions/memusage.py
 scrapy/commands/deploy.py
 scrapy/commands/bench.py
 scrapy/mail.py
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 9c1678488..ac954cc15 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -17,8 +17,8 @@ from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware
 
 class _BaseTest(unittest.TestCase):
 
-    storage_class = 'scrapy.contrib.httpcache.DbmCacheStorage'
-    policy_class = 'scrapy.contrib.httpcache.RFC2616Policy'
+    storage_class = 'scrapy.extensions.httpcache.DbmCacheStorage'
+    policy_class = 'scrapy.extensions.httpcache.RFC2616Policy'
 
     def setUp(self):
         self.yesterday = email.utils.formatdate(time.time() - 86400)
@@ -127,7 +127,7 @@ class DefaultStorageTest(_BaseTest):
 
 class DbmStorageTest(DefaultStorageTest):
 
-    storage_class = 'scrapy.contrib.httpcache.DbmCacheStorage'
+    storage_class = 'scrapy.extensions.httpcache.DbmCacheStorage'
 
 
 class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):
@@ -146,7 +146,7 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):
 
 class FilesystemStorageTest(DefaultStorageTest):
 
-    storage_class = 'scrapy.contrib.httpcache.FilesystemCacheStorage'
+    storage_class = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
 
 class FilesystemStorageGzipTest(FilesystemStorageTest):
 
@@ -157,12 +157,12 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
 class LeveldbStorageTest(DefaultStorageTest):
 
     pytest.importorskip('leveldb')
-    storage_class = 'scrapy.contrib.httpcache.LeveldbCacheStorage'
+    storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage'
 
 
 class DummyPolicyTest(_BaseTest):
 
-    policy_class = 'scrapy.contrib.httpcache.DummyPolicy'
+    policy_class = 'scrapy.extensions.httpcache.DummyPolicy'
 
     def test_middleware(self):
         with self._middleware() as mw:
@@ -254,7 +254,7 @@ class DummyPolicyTest(_BaseTest):
 
 class RFC2616PolicyTest(DefaultStorageTest):
 
-    policy_class = 'scrapy.contrib.httpcache.RFC2616Policy'
+    policy_class = 'scrapy.extensions.httpcache.RFC2616Policy'
 
     def _process_requestresponse(self, mw, request, response):
         try:
diff --git a/tests/test_contrib_feedexport.py b/tests/test_feedexport.py
similarity index 99%
rename from tests/test_contrib_feedexport.py
rename to tests/test_feedexport.py
index e6fd38ee3..41913e401 100644
--- a/tests/test_contrib_feedexport.py
+++ b/tests/test_feedexport.py
@@ -16,7 +16,7 @@ from tests.mockserver import MockServer
 from w3lib.url import path_to_file_uri
 
 import scrapy
-from scrapy.contrib.feedexport import (
+from scrapy.extensions.feedexport import (
     IFeedStorage, FileFeedStorage, FTPFeedStorage,
     S3FeedStorage, StdoutFeedStorage
 )
diff --git a/tests/test_contrib_spiderstate.py b/tests/test_spiderstate.py
similarity index 94%
rename from tests/test_contrib_spiderstate.py
rename to tests/test_spiderstate.py
index 8e5897db7..1ddce4b99 100644
--- a/tests/test_contrib_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -2,7 +2,7 @@ import os
 from datetime import datetime
 from twisted.trial import unittest
 
-from scrapy.contrib.spiderstate import SpiderState
+from scrapy.extensions.spiderstate import SpiderState
 from scrapy.spider import Spider
 
 

From 4ddf152be37db5e4307cb62f42389c5fc71b6814 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 23 Apr 2015 11:58:46 -0300
Subject: [PATCH 0273/2974] scrapy/dupefilter.py shim

---
 scrapy/dupefilter.py | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 scrapy/dupefilter.py

diff --git a/scrapy/dupefilter.py b/scrapy/dupefilter.py
new file mode 100644
index 000000000..232d96288
--- /dev/null
+++ b/scrapy/dupefilter.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.dupefilter` is deprecated, "
+              "use `scrapy.dupefilters` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.dupefilters import *

From b827097c91eb27c5d747935155497cb87edcb816 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 23 Apr 2015 13:07:48 -0300
Subject: [PATCH 0274/2974] Rename scrapy/statscol.py to
 scrapy/statscollectors.py

---
 docs/topics/api.rst                        | 8 ++++----
 docs/topics/settings.rst                   | 2 +-
 docs/topics/stats.rst                      | 2 +-
 scrapy/settings/default_settings.py        | 2 +-
 scrapy/{statscol.py => statscollectors.py} | 0
 tests/test_spidermiddleware_depth.py       | 2 +-
 tests/test_stats.py                        | 2 +-
 7 files changed, 9 insertions(+), 9 deletions(-)
 rename scrapy/{statscol.py => statscollectors.py} (100%)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 89e300b39..7805f63d3 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -65,7 +65,7 @@ how you :ref:`configure the downloader middlewares
 
         For an introduction on stats collection see :ref:`topics-stats`.
 
-        For the API see :class:`~scrapy.statscol.StatsCollector` class.
+        For the API see :class:`~scrapy.statscollectors.StatsCollector` class.
 
     .. attribute:: extensions
 
@@ -452,11 +452,11 @@ Stats Collector API
 ===================
 
 There are several Stats Collectors available under the
-:mod:`scrapy.statscol` module and they all implement the Stats
-Collector API defined by the :class:`~scrapy.statscol.StatsCollector`
+:mod:`scrapy.statscollectors` module and they all implement the Stats
+Collector API defined by the :class:`~scrapy.statscollectors.StatsCollector`
 class (which they all inherit from).
 
-.. module:: scrapy.statscol
+.. module:: scrapy.statscollectors
    :synopsis: Stats Collectors
 
 .. class:: StatsCollector
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index f035bc33d..26a6d762d 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -936,7 +936,7 @@ Example::
 STATS_CLASS
 -----------
 
-Default: ``'scrapy.statscol.MemoryStatsCollector'``
+Default: ``'scrapy.statscollectors.MemoryStatsCollector'``
 
 The class to use for collecting stats, who must implement the
 :ref:`topics-api-stats`.
diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst
index 5560a9446..0837610d0 100644
--- a/docs/topics/stats.rst
+++ b/docs/topics/stats.rst
@@ -75,7 +75,7 @@ available in Scrapy which extend the basic Stats Collector. You can select
 which Stats Collector to use through the :setting:`STATS_CLASS` setting. The
 default Stats Collector used is the :class:`MemoryStatsCollector`. 
 
-.. module:: scrapy.statscol
+.. module:: scrapy.statscollectors
    :synopsis: Stats Collectors
 
 MemoryStatsCollector
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 57dfc3256..7459c4d73 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -234,7 +234,7 @@ SPIDER_MIDDLEWARES_BASE = {
 
 SPIDER_MODULES = []
 
-STATS_CLASS = 'scrapy.statscol.MemoryStatsCollector'
+STATS_CLASS = 'scrapy.statscollectors.MemoryStatsCollector'
 STATS_DUMP = True
 
 STATSMAILER_RCPTS = []
diff --git a/scrapy/statscol.py b/scrapy/statscollectors.py
similarity index 100%
rename from scrapy/statscol.py
rename to scrapy/statscollectors.py
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index 8120f1a95..5317795a3 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -3,7 +3,7 @@ from unittest import TestCase
 from scrapy.spidermiddlewares.depth import DepthMiddleware
 from scrapy.http import Response, Request
 from scrapy.spider import Spider
-from scrapy.statscol import StatsCollector
+from scrapy.statscollectors import StatsCollector
 from scrapy.utils.test import get_crawler
 
 
diff --git a/tests/test_stats.py b/tests/test_stats.py
index db1f50712..34f72736c 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -1,7 +1,7 @@
 import unittest
 
 from scrapy.spider import Spider
-from scrapy.statscol import StatsCollector, DummyStatsCollector
+from scrapy.statscollectors import StatsCollector, DummyStatsCollector
 from scrapy.utils.test import get_crawler
 
 class StatsCollectorTest(unittest.TestCase):

From 62191de6eae7bbad227361dded8d4726da0e8a11 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 23 Apr 2015 13:10:15 -0300
Subject: [PATCH 0275/2974] scrapy/statscol.py shim

---
 scrapy/statscol.py | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 scrapy/statscol.py

diff --git a/scrapy/statscol.py b/scrapy/statscol.py
new file mode 100644
index 000000000..b4ddcce28
--- /dev/null
+++ b/scrapy/statscol.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.statscol` is deprecated, "
+              "use `scrapy.statscollectors` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.statscollectors import *

From 3243a13b80519650644bd5e1d500aa7dfd69bfd1 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 29 Apr 2015 23:04:34 -0300
Subject: [PATCH 0276/2974] Allow configure_logging to accept dicts as Settings
 objects

---
 scrapy/utils/log.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index c9250d7a1..23b246491 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -10,7 +10,7 @@ from twisted.python.failure import Failure
 from twisted.python import log as twisted_log
 
 import scrapy
-from scrapy.settings import overridden_settings
+from scrapy.settings import overridden_settings, Settings
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 logger = logging.getLogger(__name__)
@@ -92,6 +92,10 @@ def configure_logging(settings=None):
     observer.start()
 
     dictConfig(DEFAULT_LOGGING)
+
+    if isinstance(settings, dict):
+        settings = Settings(settings)
+
     if settings:
         logging.root.setLevel(logging.NOTSET)
 

From 353672d292492446cc0a8f95d2b06caa58965dbe Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 24 Apr 2015 18:25:47 -0300
Subject: [PATCH 0277/2974] Add autodocs Sphinx extension

---
 docs/conf.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index 2c1935003..2d22c6a64 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -26,7 +26,10 @@ sys.path.append(path.join(path.dirname(path.dirname(__file__)), "scrapy"))
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['scrapydocs']
+extensions = [
+    'scrapydocs',
+    'sphinx.ext.autodoc'
+]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

From a2ce78db63c32212512d2f9223b42cf4eb051b8e Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 24 Apr 2015 18:26:49 -0300
Subject: [PATCH 0278/2974] Use autoclass for CrawlerRunner in
 docs/topics/api.rst

---
 docs/topics/api.rst | 47 ++-------------------------------------------
 scrapy/crawler.py   | 44 +++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 45 insertions(+), 46 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 7805f63d3..5d28b3c90 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -99,52 +99,9 @@ how you :ref:`configure the downloader middlewares
 
         Returns a deferred that is fired when the crawl is finished.
 
-.. class:: CrawlerRunner(settings)
+.. autoclass:: CrawlerRunner
+   :members:
 
-    This is a convenient helper class that keeps track of, manages and runs
-    crawlers inside an already setup Twisted `reactor`_.
-
-    The CrawlerRunner object must be instantiated with a
-    :class:`~scrapy.settings.Settings` object.
-
-    This class shouldn't be needed (since Scrapy is responsible of using it
-    accordingly) unless writing scripts that manually handle the crawling
-    process. See :ref:`run-from-script` for an example.
-
-    .. attribute:: crawlers
-
-       Set of :class:`crawlers <scrapy.crawler.Crawler>` created by the
-       :meth:`crawl` method.
-
-    .. method:: crawl(crawler_or_spidercls, \*args, \**kwargs)
-
-       This method runs a crawler with the provided arguments.
-
-       It will keep track of the given crawler so it can be stopped later,
-       while calling its :meth:`Crawler.crawl` method.
-
-       If `crawler_or_spidercls` isn't a :class:`~scrapy.crawler.Crawler`
-       instance, it will try to create one using this parameter as the spider
-       class given to it.
-
-       Returns a deferred that is fired when the crawl is finished.
-
-       :param crawler_or_spidercls: already created crawler, or a spider class
-       or spider's name inside the project to create it
-       :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance,
-        :class:`~scrapy.spider.Spider` subclass or string
-
-       :param args: arguments to initializate the spider
-       :type args: list
-
-       :param kwargs: keyword arguments to initializate the spider
-       :type kwargs: dict
-
-    .. method:: stop()
-
-       Stops simultaneously all the crawling jobs taking place.
-
-       Returns a deferred that is fired when they all have ended.
 
 .. _topics-api-settings:
 
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 9883b726e..5cc2a7daa 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -89,6 +89,17 @@ class Crawler(object):
 
 
 class CrawlerRunner(object):
+    """
+    This is a convenient helper class that keeps track of, manages and runs
+    crawlers inside an already setup Twisted `reactor`_.
+
+    The CrawlerRunner object must be instantiated with a
+    :class:`~scrapy.settings.Settings` object.
+
+    This class shouldn't be needed (since Scrapy is responsible of using it
+    accordingly) unless writing scripts that manually handle the crawling
+    process. See :ref:`run-from-script` for an example.
+    """
 
     def __init__(self, settings):
         if isinstance(settings, dict):
@@ -106,6 +117,27 @@ class CrawlerRunner(object):
         return self.spider_loader
 
     def crawl(self, crawler_or_spidercls, *args, **kwargs):
+        """
+        Run a crawler with the provided arguments.
+
+        It will call the given Crawler's :meth:`~Crawler.crawl` method, while
+        keeping track of it so it can be stopped later.
+
+        If `crawler_or_spidercls` isn't a :class:`~scrapy.crawler.Crawler`
+        instance, this method will try to create one using this parameter as
+        the spider class given to it.
+
+        Returns a deferred that is fired when the crawling is finished.
+
+        :param crawler_or_spidercls: already created crawler, or a spider class
+            or spider's name inside the project to create it
+        :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance,
+            :class:`~scrapy.spider.Spider` subclass or string
+
+        :param list args: arguments to initialize the spider
+
+        :param dict kwargs: keyword arguments to initialize the spider
+        """
         crawler = crawler_or_spidercls
         if not isinstance(crawler_or_spidercls, Crawler):
             crawler = self._create_crawler(crawler_or_spidercls)
@@ -127,11 +159,21 @@ class CrawlerRunner(object):
         return Crawler(spidercls, self.settings)
 
     def stop(self):
+        """
+        Stops simultaneously all the crawling jobs taking place.
+
+        Returns a deferred that is fired when they all have ended.
+        """
         return defer.DeferredList([c.stop() for c in list(self.crawlers)])
 
     @defer.inlineCallbacks
     def join(self):
-        """Wait for all managed crawlers to complete"""
+        """
+        join()
+
+        Returns a deferred that is fired when all managed :attr:`crawlers` have
+        completed their executions.
+        """
         while self._active:
             yield defer.DeferredList(self._active)
 

From 8f276b8e9c39cf69f8b2c3e981167109b3c159a5 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 24 Apr 2015 18:28:48 -0300
Subject: [PATCH 0279/2974] Property for CrawlerRunner.crawlers so it can be
 autodocumented with Sphinx

---
 scrapy/crawler.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 5cc2a7daa..9680f6e11 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -101,12 +101,18 @@ class CrawlerRunner(object):
     process. See :ref:`run-from-script` for an example.
     """
 
+    crawlers = property(
+        lambda self: self._crawlers,
+        doc="Set of :class:`crawlers <scrapy.crawler.Crawler>` started by "
+            ":meth:`crawl` and managed by this class."
+    )
+
     def __init__(self, settings):
         if isinstance(settings, dict):
             settings = Settings(settings)
         self.settings = settings
         self.spider_loader = _get_spider_loader(settings)
-        self.crawlers = set()
+        self._crawlers = set()
         self._active = set()
 
     @property

From 3a71504d35a5e5b9eacf29c63c9a317f5167e3db Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 24 Apr 2015 18:30:00 -0300
Subject: [PATCH 0280/2974] Extend CrawlerProcess documentation

---
 docs/topics/api.rst |  4 ++++
 scrapy/crawler.py   | 30 +++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 5d28b3c90..ce28b8bc1 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -102,6 +102,10 @@ how you :ref:`configure the downloader middlewares
 .. autoclass:: CrawlerRunner
    :members:
 
+.. autoclass:: CrawlerProcess
+   :show-inheritance:
+   :members:
+   :inherited-members:
 
 .. _topics-api-settings:
 
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 9680f6e11..161ca4614 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -185,7 +185,24 @@ class CrawlerRunner(object):
 
 
 class CrawlerProcess(CrawlerRunner):
-    """A class to run multiple scrapy crawlers in a process simultaneously"""
+    """
+    A class to run multiple scrapy crawlers in a process simultaneously.
+
+    This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support
+    for starting a Twisted `reactor`_ and handling shutdown signals, like the
+    keyboard interrupt command Ctrl-C. It also configures top-level logging.
+
+    This utility should be a better fit than
+    :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another
+    Twisted `reactor`_ within your application.
+
+    The CrawlerProcess object must be instantiated with a
+    :class:`~scrapy.settings.Settings` object.
+
+    This class shouldn't be needed (since Scrapy is responsible of using it
+    accordingly) unless writing scripts that manually handle the crawling
+    process. See :ref:`run-from-script` for an example.
+    """
 
     def __init__(self, settings):
         super(CrawlerProcess, self).__init__(settings)
@@ -209,6 +226,17 @@ class CrawlerProcess(CrawlerRunner):
         reactor.callFromThread(self._stop_reactor)
 
     def start(self, stop_after_crawl=True):
+        """
+        This method starts a Twisted `reactor`_, adjusts its pool size to
+        :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based
+        on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
+
+        If `stop_after_crawl` is True, the reactor will be stopped after all
+        crawlers have finished, using :meth:`join`.
+
+        :param boolean stop_after_crawl: stop or not the reactor when all
+            crawlers have finished
+        """
         if stop_after_crawl:
             d = self.join()
             # Don't start the reactor if the deferreds are already fired

From d6a06a769fd738447b69dce9d79b74769a0f2597 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sun, 26 Apr 2015 18:04:15 -0300
Subject: [PATCH 0281/2974] Add CrawlerProcess to "Run Scrapy from a script"
 doc section

---
 docs/topics/practices.rst | 74 +++++++++++++++++++++++++++------------
 1 file changed, 52 insertions(+), 22 deletions(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 6bd74c794..1520ff738 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -18,39 +18,69 @@ the typical way of running Scrapy via ``scrapy crawl``.
 Remember that Scrapy is built on top of the Twisted
 asynchronous networking library, so you need to run it inside the Twisted reactor.
 
-Note that you will also have to shutdown the Twisted reactor yourself after the
-spider is finished. This can be achieved by adding callbacks to the deferred
-returned by the :meth:`CrawlerRunner.crawl
-<scrapy.crawler.CrawlerRunner.crawl>` method.
+First utility you can use to run your spiders is
+:class:`scrapy.crawler.CrawlerProcess`. This class will start a Twisted reactor
+for you, configuring the logging and setting shutdown handlers. This class is
+the one used by all Scrapy commands.
+
+Here's an example showing how to run a single spider with it.
+
+::
+
+    import scrapy
+    from scrapy.crawler import CrawlerProcess
+
+    class MySpider(scrapy.Spider):
+        # Your spider definition
+        ...
+
+    process = CrawlerProcess({
+        'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
+    })
+
+    process.crawl(MySpider)
+    process.start() # the script will block here until the crawling is finished
+
+Make sure to check :class:`~scrapy.crawler.CrawlerProcess` documentation to get
+acquainted with its usage details.
+
+If you are inside a Scrapy project there are some additional helpers you can
+use to import those components within the project. You can automatically import
+your spiders passing their name to :class:`~scrapy.crawler.CrawlerProcess`, and
+use ``get_project_settings`` to get a :class:`~scrapy.settings.Settings`
+instance with your project settings.
 
 What follows is a working example of how to do that, using the `testspiders`_
 project as example.
 
 ::
 
-    from twisted.internet import reactor
-    from scrapy.crawler import CrawlerRunner
-    from scrapy.utils.log import configure_logging
+    from scrapy.crawler import CrawlerProcess
     from scrapy.utils.project import get_project_settings
 
-    settings = get_project_settings()
-    configure_logging(settings)
-    runner = CrawlerRunner(settings)
+    process = CrawlerProcess(get_project_settings())
 
     # 'followall' is the name of one of the spiders of the project.
-    d = runner.crawl('followall', domain='scrapinghub.com')
-    d.addBoth(lambda _: reactor.stop())
-    reactor.run() # the script will block here until the crawling is finished
+    process.crawl('testspider', domain='scrapinghub.com')
+    process.start() # the script will block here until the crawling is finished
 
-Running spiders outside projects it's not much different. You have to create a
-generic :class:`~scrapy.settings.Settings` object and populate it as needed
-(See :ref:`topics-settings-ref` for the available settings), instead of using
-the configuration returned by `get_project_settings`.
+There's another Scrapy utility that provides more control over the crawling
+process: :class:`scrapy.crawler.CrawlerRunner`. This class is a thin wrapper
+that encapsulates some simple helpers to run multiple crawlers, but it won't
+start or interfere with existing reactors in any way.
 
-Spiders can still be referenced by their name if :setting:`SPIDER_MODULES` is
-set with the modules where Scrapy should look for spiders.  Otherwise, passing
-the spider class as first argument in the :meth:`CrawlerRunner.crawl
-<scrapy.crawler.CrawlerRunner.crawl>` method is enough.
+Using this class the reactor should be explicitly run after scheduling your
+spiders. It's recommended you use :class:`~scrapy.crawler.CrawlerRunner`
+instead of :class:`~scrapy.crawler.CrawlerProcess` if your application is
+already using Twisted and you want to run Scrapy in the same reactor.
+
+Note that you will also have to shutdown the Twisted reactor yourself after the
+spider is finished. This can be achieved by adding callbacks to the deferred
+returned by the :meth:`CrawlerRunner.crawl
+<scrapy.crawler.CrawlerRunner.crawl>` method.
+
+Here's an example of its usage, along with a callback to manually stop the
+reactor after `MySpider` has finished running.
 
 ::
 
@@ -63,7 +93,7 @@ the spider class as first argument in the :meth:`CrawlerRunner.crawl
         # Your spider definition
         ...
 
-    configure_logging(settings)
+    configure_logging({'LOG_FORMAT': '%(levelname)s: %(message)s'})
     runner = CrawlerRunner({
         'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
     })

From c1634e4914e52c5a4370bae357bb08189fbe6fed Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sun, 26 Apr 2015 18:20:23 -0300
Subject: [PATCH 0282/2974] Add CrawlerProcess to "Running multiple
 spiders[...]" doc section

---
 docs/topics/practices.rst | 67 ++++++++++++++++++++++++++++-----------
 1 file changed, 49 insertions(+), 18 deletions(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 1520ff738..3a9b21d3b 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -113,25 +113,50 @@ By default, Scrapy runs a single spider per process when you run ``scrapy
 crawl``. However, Scrapy supports running multiple spiders per process using
 the :ref:`internal API <topics-api>`.
 
-Here is an example that runs multiple spiders simultaneously, using the
-`testspiders`_ project:
+Here is an example that runs multiple spiders simultaneously:
 
 ::
 
-    from twisted.internet import reactor, defer
+    import scrapy
+    from scrapy.crawler import CrawlerProcess
+
+    class MySpider1(scrapy.Spider):
+        # Your first spider definition
+        ...
+
+    class MySpider2(scrapy.Spider):
+        # Your second spider definition
+        ...
+
+    process = CrawlerProcess({})
+    process.crawl(MySpider1)
+    process.crawl(MySpider2)
+    process.start() # the script will block here until all crawling jobs are finished
+
+Same example using :class:`~scrapy.crawler.CrawlerRunner`:
+
+::
+
+    import scrapy
+    from twisted.internet import reactor
     from scrapy.crawler import CrawlerRunner
     from scrapy.utils.log import configure_logging
-    from scrapy.utils.project import get_project_settings
 
-    settings = get_project_settings()
-    configure_logging(settings)
-    runner = CrawlerRunner(settings)
-    dfs = set()
-    for domain in ['scrapinghub.com', 'insophia.com']:
-        d = runner.crawl('followall', domain=domain)
-        dfs.add(d)
+    class MySpider1(scrapy.Spider):
+        # Your first spider definition
+        ...
+
+    class MySpider2(scrapy.Spider):
+        # Your second spider definition
+        ...
+
+    configure_logging({})
+    runner = CrawlerRunner({})
+    runner.crawl(MySpider1)
+    runner.crawl(MySpider2)
+    d = runner.join()
+    d.addBoth(lambda _: reactor.stop())
 
-    defer.DeferredList(dfs).addBoth(lambda _: reactor.stop())
     reactor.run() # the script will block here until all crawling jobs are finished
 
 Same example but running the spiders sequentially by chaining the deferreds:
@@ -141,16 +166,22 @@ Same example but running the spiders sequentially by chaining the deferreds:
     from twisted.internet import reactor, defer
     from scrapy.crawler import CrawlerRunner
     from scrapy.utils.log import configure_logging
-    from scrapy.utils.project import get_project_settings
 
-    settings = get_project_settings()
-    configure_logging(settings)
-    runner = CrawlerRunner(settings)
+    class MySpider1(scrapy.Spider):
+        # Your first spider definition
+        ...
+
+    class MySpider2(scrapy.Spider):
+        # Your second spider definition
+        ...
+
+    configure_logging({})
+    runner = CrawlerRunner({})
 
     @defer.inlineCallbacks
     def crawl():
-        for domain in ['scrapinghub.com', 'insophia.com']:
-            yield runner.crawl('followall', domain=domain)
+        yield runner.crawl(MySpider1)
+        yield runner.crawl(MySpider2)
         reactor.stop()
 
     crawl()

From 667864ae94168a616beabc3930938b28a593633d Mon Sep 17 00:00:00 2001
From: "Victoria Terenina (torymur)" <torymur@gmail.com>
Date: Thu, 30 Apr 2015 16:28:57 +0300
Subject: [PATCH 0283/2974] testing robotstxt error

---
 tests/test_downloadermiddleware_robotstxt.py | 23 +++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index bc245dcec..372a0a2c3 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,7 +1,8 @@
 from __future__ import absolute_import
 import re
-from twisted.internet import reactor
+from twisted.internet import reactor, error
 from twisted.internet.defer import Deferred
+from twisted.python import failure
 from twisted.trial import unittest
 from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
 from scrapy.exceptions import IgnoreRequest, NotConfigured
@@ -41,6 +42,26 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         reactor.callFromThread(deferred.callback, None)
         return deferred
 
+    def test_robotstxt_error(self):
+        crawler = mock.MagicMock()
+        crawler.settings = Settings()
+        crawler.settings.set('ROBOTSTXT_OBEY', True)
+        crawler.engine.download = mock.MagicMock()
+        err = error.DNSLookupError('Robotstxt address not found')
+        def return_failure(request, spider):
+            deferred = Deferred()
+            reactor.callFromThread(deferred.errback, failure.Failure(err))
+            return deferred
+        crawler.engine.download.side_effect = return_failure
+
+        middleware = RobotsTxtMiddleware(crawler)
+        middleware._logerror = mock.MagicMock()
+        middleware.process_request(Request('http://site.local'), None)
+        deferred = Deferred()
+        deferred.addErrback(lambda _: self.assertIsNone(middleware._logerror.assert_any_call()))
+        reactor.callFromThread(deferred.callback, None)
+        return deferred
+
     def assertNotIgnored(self, request, middleware):
         spider = None  # not actually used
         self.assertIsNone(middleware.process_request(request, spider))

From 9ee17fd537412d9162f55b196da5616054ec9ea5 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Fri, 1 May 2015 01:39:03 -0300
Subject: [PATCH 0284/2974] add note to doc README about using 'make watch'

---
 docs/README | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/README b/docs/README
index 26cf25371..cf04965ac 100644
--- a/docs/README
+++ b/docs/README
@@ -48,3 +48,10 @@ To cleanup all generated documentation files and start from scratch run::
 Keep in mind that this command won't touch any documentation source files.
 
 
+Recreating documentation on the fly
+-----------------------------------
+
+There is a way to recreate the doc automatically when you make changes, you
+need to install watchdog (``pip install watchdog``) and then use::
+
+    make watch

From a5a8f82b06b3c5a426169f7096ab01cea811c752 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Fri, 1 May 2015 01:51:56 -0300
Subject: [PATCH 0285/2974] move documentation about registering commands via
 setup.py to commands doc

---
 docs/conf.py                |  2 +-
 docs/experimental/index.rst | 17 -----------------
 docs/topics/commands.rst    | 21 +++++++++++++++++++++
 3 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 2d22c6a64..dfbd96675 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -100,7 +100,7 @@ pygments_style = 'sphinx'
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'sphinx_rtd_theme'
+#html_theme = 'sphinx_rtd_theme'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
diff --git a/docs/experimental/index.rst b/docs/experimental/index.rst
index 1c019c396..0621b4925 100644
--- a/docs/experimental/index.rst
+++ b/docs/experimental/index.rst
@@ -15,20 +15,3 @@ it's properly merged) . Use at your own risk.
 .. warning::
 
    This documentation is a work in progress. Use at your own risk.
-
-Add commands using external libraries
--------------------------------------
-
-You can also add Scrapy commands from an external library by adding `scrapy.commands` section into entry_points in the `setup.py`.
-
-The following example adds `my_command` command::
-
-  from setuptools import setup, find_packages
-
-  setup(name='scrapy-mymodule',
-    entry_points={
-      'scrapy.commands': [
-        'my_command=my_scrapy_module.commands:MyCommand',
-      ],
-    },
-   )
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index f1f79ce70..52c4fe446 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -511,3 +511,24 @@ Example::
     COMMANDS_MODULE = 'mybot.commands'
 
 .. _Deploying your project: http://scrapyd.readthedocs.org/en/latest/deploy.html
+
+Register commands via setup.py entry points
+-------------------------------------------
+
+.. note:: This is an experimental feature, use with caution.
+
+You can also add Scrapy commands from an external library by adding a
+``scrapy.commands`` section in the entry points of the library ``setup.py``
+file.
+
+The following example adds ``my_command`` command::
+
+  from setuptools import setup, find_packages
+
+  setup(name='scrapy-mymodule',
+    entry_points={
+      'scrapy.commands': [
+        'my_command=my_scrapy_module.commands:MyCommand',
+      ],
+    },
+   )

From b84b3585f2b925a5a88559b201d714cae81b06e9 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Fri, 1 May 2015 01:53:51 -0300
Subject: [PATCH 0286/2974] remove empty experimental section from doc

---
 docs/experimental/index.rst | 17 -----------------
 docs/index.rst              |  4 ----
 2 files changed, 21 deletions(-)
 delete mode 100644 docs/experimental/index.rst

diff --git a/docs/experimental/index.rst b/docs/experimental/index.rst
deleted file mode 100644
index 0621b4925..000000000
--- a/docs/experimental/index.rst
+++ /dev/null
@@ -1,17 +0,0 @@
-.. _experimental:
-
-Experimental features
-=====================
-
-This section documents experimental Scrapy features that may become stable in
-future releases, but whose API is not yet stable. Use them with caution, and
-subscribe to the `mailing lists <http://scrapy.org/community/>`_ to get
-notified of any changes. 
-
-Since it's not revised so frequently, this section may contain documentation
-which is outdated, incomplete or overlapping with stable documentation (until
-it's properly merged) . Use at your own risk.
-
-.. warning::
-
-   This documentation is a work in progress. Use at your own risk.
diff --git a/docs/index.rst b/docs/index.rst
index 6a1571a69..0d21f5d40 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -245,7 +245,6 @@ All the rest
    news
    contributing
    versioning
-   experimental/index
 
 :doc:`news`
     See what has changed in recent Scrapy versions.
@@ -255,6 +254,3 @@ All the rest
 
 :doc:`versioning`
     Understand Scrapy versioning and API stability.
-
-:doc:`experimental/index`
-    Learn about bleeding-edge features.

From d00e43f39c4ad82af5af9ffeb6fb35af2642d4aa Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Fri, 1 May 2015 05:24:49 -0300
Subject: [PATCH 0287/2974] restore wrongly commented line in sphinx conf.py

---
 docs/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index dfbd96675..2d22c6a64 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -100,7 +100,7 @@ pygments_style = 'sphinx'
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-#html_theme = 'sphinx_rtd_theme'
+html_theme = 'sphinx_rtd_theme'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

From 8301d00fee68e958520c0d6008d8b22420bead74 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 16 Apr 2015 18:20:50 +0500
Subject: [PATCH 0288/2974] TST don't collect tests from deprecated modules.

This removes some deprecation warnings in tests.
---
 conftest.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/conftest.py b/conftest.py
index 783a5a250..70147dcf4 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,7 +1,23 @@
+import glob
 import six
 import pytest
 
-collect_ignore = ["scrapy/stats.py", "scrapy/project.py"]
+
+def _py_files(folder):
+    return glob.glob(folder + "/*.py") + glob.glob(folder + "/*/*.py")
+
+
+collect_ignore = [
+    "scrapy/conf.py",
+    "scrapy/stats.py",
+    "scrapy/project.py",
+    "scrapy/utils/decorator.py",
+    "scrapy/statscol.py",
+    "scrapy/squeue.py",
+    "scrapy/log.py",
+    "scrapy/dupefilter.py",
+] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp")
+
 
 if six.PY3:
     for line in open('tests/py3-ignores.txt'):

From d1053d2a2fe3dba51febcb756227f623a16daf70 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Sun, 3 May 2015 18:59:17 -0300
Subject: [PATCH 0289/2974] automatic settings list for docs [WIP]

---
 docs/_ext/scrapydocs.py  | 64 ++++++++++++++++++++++++++++++++++++++++
 docs/topics/settings.rst |  2 ++
 2 files changed, 66 insertions(+)

diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index 1fa1c93d6..88e6386c9 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -1,5 +1,63 @@
 from docutils.parsers.rst.roles import set_classes
 from docutils import nodes
+from sphinx.util.compat import Directive
+from sphinx.util.nodes import make_refnode
+
+
+class settingslist_node(nodes.General, nodes.Element):
+    pass
+
+
+class SettingsListDirective(Directive):
+    def run(self):
+        return [settingslist_node('')]
+
+
+def is_setting_node(node):
+    return node.tagname == 'pending_xref' and node['reftype'] == 'setting'
+
+
+def collect_scrapy_settings_refs(app, doctree):
+    env = app.builder.env
+
+    if not hasattr(env, 'scrapy_all_settings'):
+        env.scrapy_all_settings = []
+
+    for node in doctree.traverse(is_setting_node):
+        try:
+            targetnode = node.parent[node.parent.index(node) - 1]
+            if not isinstance(targetnode, nodes.target):
+                raise IndexError
+        except IndexError:
+            targetid = "setting-%d" % env.new_serialno('setting')
+            targetnode = nodes.target('', '', ids=[targetid])
+            node.replace_self([targetnode, node])
+
+        env.scrapy_all_settings.append({
+            'docname': env.docname,
+            'lineno': node.line,
+            'node': node.deepcopy(),
+            'target': targetnode,
+        })
+
+
+def make_setting_element(setting_data, app, fromdocname):
+    text = nodes.Text(setting_data['node'].astext())
+    targetid = ''  # TODO: resolve to a proper id
+    refnode = make_refnode(app.builder, fromdocname,
+                           setting_data['docname'], targetid, text)
+
+    p = nodes.paragraph()
+    p.append(refnode)
+    return p
+
+
+def replace_settingslist_nodes(app, doctree, fromdocname):
+    env = app.builder.env
+
+    for node in doctree.traverse(settingslist_node):
+        node.replace_self([make_setting_element(d, app, fromdocname)
+                           for d in env.scrapy_all_settings])
 
 def setup(app):
     app.add_crossref_type(
@@ -27,6 +85,12 @@ def setup(app):
     app.add_role('issue', issue_role)
     app.add_role('rev', rev_role)
 
+    app.add_node(settingslist_node)
+    app.add_directive('settingslist', SettingsListDirective)
+
+    app.connect('doctree-read', collect_scrapy_settings_refs)
+    app.connect('doctree-resolved', replace_settingslist_nodes)
+
 def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
     ref = 'https://github.com/scrapy/scrapy/blob/master/' + text
     set_classes(options)
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 3936c04b8..1cf2940ff 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -133,6 +133,8 @@ to any particular component. In that case the module of that component will be
 shown, typically an extension, middleware or pipeline. It also means that the
 component must be enabled in order for the setting to have any effect.
 
+.. settingslist::
+
 .. setting:: AWS_ACCESS_KEY_ID
 
 AWS_ACCESS_KEY_ID

From 8ae05478beea4327f556f21c9c0f954826aff832 Mon Sep 17 00:00:00 2001
From: bosnj <bosnj@users.noreply.github.com>
Date: Mon, 4 May 2015 21:22:17 +0200
Subject: [PATCH 0290/2974] added docs and test case, fixed handling empty
 string vs None

---
 docs/topics/selectors.rst  | 5 +++++
 scrapy/selector/unified.py | 4 +++-
 tests/test_selector.py     | 8 ++++++++
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 33958cee5..f8a9b0410 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -149,6 +149,11 @@ It returns ``None`` if no element was found:
     >>> sel.xpath('//div/[id="not-exists"]/text()').extract_first() is None
     True
 
+A default return value can be provided as an argument, to be used instead of ``None``:
+
+    >>> sel.xpath('//div/[id="not-exists"]/text()').extract_first(default='not-found')
+    'not-found'
+
 Notice that CSS selectors can select text or attribute nodes using CSS3
 pseudo-elements::
 
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index db8b0bc2d..efb51b561 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -184,7 +184,9 @@ class SelectorList(list):
 
     def extract_first(self, default=None):
         for x in self:
-            return x.extract() or default
+            return x.extract()
+        else:
+            return default
 
     @deprecated(use_instead='.extract()')
     def extract_unquoted(self):
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 9b8613319..985424645 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -72,6 +72,14 @@ class SelectorTestCase(unittest.TestCase):
 
         self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').extract_first(), None)
 
+    def test_extract_first_default(self):
+        """Test if extract_first() returns default value when no results found"""
+        body = '<ul><li id="1">1</li><li id="2">2</li></ul>'
+        response = TextResponse(url="http://example.com", body=body)
+        sel = self.sscls(response)
+
+        self.assertEqual(sel.xpath('//div/text()').extract_first(default='missing'), 'missing')
+
     def test_re_first(self):
         """Test if re_first() returns first matched element"""
         body = '<ul><li id="1">1</li><li id="2">2</li></ul>'

From 389f6e95c5e5da3221c56ccecf77beaf6434e583 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 4 May 2015 17:11:05 -0300
Subject: [PATCH 0291/2974] Add deprecation warning to HtmlParserLinkExtractor

---
 scrapy/linkextractors/htmlparser.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index fff9eabe6..5d263ac2c 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -2,6 +2,7 @@
 HTMLParser-based link extractor
 """
 
+import warnings
 from HTMLParser import HTMLParser
 from six.moves.urllib.parse import urljoin
 
@@ -9,12 +10,20 @@ from w3lib.url import safe_url_string
 
 from scrapy.link import Link
 from scrapy.utils.python import unique as unique_list
+from scrapy.exceptions import ScrapyDeprecationWarning
+
 
 class HtmlParserLinkExtractor(HTMLParser):
 
     def __init__(self, tag="a", attr="href", process=None, unique=False):
         HTMLParser.__init__(self)
 
+        warnings.warn(
+            "HtmlParserLinkExtractor is deprecated and will be removed in "
+            "future releases. Please use scrapy.linkextractors.LinkExtractor",
+            ScrapyDeprecationWarning
+        )
+
         self.scan_tag = tag if callable(tag) else lambda t: t == tag
         self.scan_attr = attr if callable(attr) else lambda a: a == attr
         self.process_attr = process if callable(process) else lambda v: v

From 5e59f795d2fb1d3ce9da2d510b7d546b9e034f3d Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 4 May 2015 17:11:30 -0300
Subject: [PATCH 0292/2974] Increase stacklevel in deprecation warnings from
 linkextractors

---
 scrapy/linkextractors/htmlparser.py | 2 +-
 scrapy/linkextractors/sgml.py       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 5d263ac2c..14f4970b0 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -21,7 +21,7 @@ class HtmlParserLinkExtractor(HTMLParser):
         warnings.warn(
             "HtmlParserLinkExtractor is deprecated and will be removed in "
             "future releases. Please use scrapy.linkextractors.LinkExtractor",
-            ScrapyDeprecationWarning
+            ScrapyDeprecationWarning, stacklevel=2,
         )
 
         self.scan_tag = tag if callable(tag) else lambda t: t == tag
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index b1f3da416..88d2d5b91 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -21,7 +21,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
         warnings.warn(
             "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
             "Please use scrapy.linkextractors.LinkExtractor",
-            ScrapyDeprecationWarning
+            ScrapyDeprecationWarning, stacklevel=2,
         )
         SGMLParser.__init__(self)
         self.scan_tag = tag if callable(tag) else lambda t: t == tag
@@ -104,7 +104,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
         warnings.warn(
             "SgmlLinkExtractor is deprecated and will be removed in future releases. "
             "Please use scrapy.linkextractors.LinkExtractor",
-            ScrapyDeprecationWarning
+            ScrapyDeprecationWarning, stacklevel=2,
         )
 
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))

From a47859540a895dbbe4a0d7de4890f1300eb48b2e Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 4 May 2015 18:10:04 -0300
Subject: [PATCH 0293/2974] Don't collect tests by their class name

---
 pytest.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pytest.ini b/pytest.ini
index 3046dfd85..73d169601 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,5 +1,6 @@
 [pytest]
 usefixtures = chdir
 python_files=test_*.py __init__.py
+python_classes=
 addopts = --doctest-modules --assert=plain
 twisted = 1

From 3fb58a30b731b293c627f650aee3479caca1e201 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 5 May 2015 22:06:18 +0500
Subject: [PATCH 0294/2974] fixed backwards compatibility for
 scrapy.contrib.exporter.PythonItemExporter

---
 scrapy/contrib/exporter/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py
index d434a7bfd..12adaaddd 100644
--- a/scrapy/contrib/exporter/__init__.py
+++ b/scrapy/contrib/exporter/__init__.py
@@ -5,3 +5,4 @@ warnings.warn("Module `scrapy.contrib.exporter` is deprecated, "
               ScrapyDeprecationWarning, stacklevel=2)
 
 from scrapy.exporters import *
+from scrapy.exporters import PythonItemExporter

From ce735feedde39963d1086ba777d86677107110ea Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 9 May 2015 03:12:35 -0300
Subject: [PATCH 0295/2974] Delete scrapy/commands/__init__.py

---
 scrapy/commands/__init__.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 scrapy/commands/__init__.py

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
deleted file mode 100644
index e69de29bb..000000000

From 616aec92c81a4bddf7f8571ae5f95635d183d62b Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 28 Apr 2015 15:26:58 -0300
Subject: [PATCH 0296/2974] Move scrapy/command.py to
 scrapy/commands/__init__.py

---
 scrapy/cmdline.py                           | 2 +-
 scrapy/{command.py => commands/__init__.py} | 0
 scrapy/commands/bench.py                    | 2 +-
 scrapy/commands/check.py                    | 2 +-
 scrapy/commands/crawl.py                    | 2 +-
 scrapy/commands/edit.py                     | 2 +-
 scrapy/commands/fetch.py                    | 2 +-
 scrapy/commands/genspider.py                | 2 +-
 scrapy/commands/list.py                     | 2 +-
 scrapy/commands/parse.py                    | 2 +-
 scrapy/commands/runspider.py                | 2 +-
 scrapy/commands/settings.py                 | 2 +-
 scrapy/commands/shell.py                    | 2 +-
 scrapy/commands/startproject.py             | 2 +-
 scrapy/commands/version.py                  | 2 +-
 scrapy/commands/view.py                     | 3 +--
 16 files changed, 15 insertions(+), 16 deletions(-)
 rename scrapy/{command.py => commands/__init__.py} (100%)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 14787950b..a619c349a 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -8,7 +8,7 @@ import pkg_resources
 import scrapy
 from scrapy.crawler import CrawlerProcess
 from scrapy.xlib import lsprofcalltree
-from scrapy.command import ScrapyCommand
+from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import inside_project, get_project_settings
diff --git a/scrapy/command.py b/scrapy/commands/__init__.py
similarity index 100%
rename from scrapy/command.py
rename to scrapy/commands/__init__.py
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 7c056a990..90c8d56a2 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -5,7 +5,7 @@ import subprocess
 from six.moves.urllib.parse import urlencode
 
 import scrapy
-from scrapy.command import ScrapyCommand
+from scrapy.commands import ScrapyCommand
 from scrapy.linkextractors import LinkExtractor
 
 
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index bf07f360f..2917b8ba7 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -4,7 +4,7 @@ import sys
 from collections import defaultdict
 from unittest import TextTestRunner, TextTestResult as _TextTestResult
 
-from scrapy.command import ScrapyCommand
+from scrapy.commands import ScrapyCommand
 from scrapy.contracts import ContractsManager
 from scrapy.utils.misc import load_object
 from scrapy.utils.conf import build_component_list
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index b7fea7b80..72df11476 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,5 +1,5 @@
 import os
-from scrapy.command import ScrapyCommand
+from scrapy.commands import ScrapyCommand
 from scrapy.utils.conf import arglist_to_dict
 from scrapy.exceptions import UsageError
 
diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py
index 016c98d2d..2df6a730c 100644
--- a/scrapy/commands/edit.py
+++ b/scrapy/commands/edit.py
@@ -1,6 +1,6 @@
 import sys, os
 
-from scrapy.command import ScrapyCommand
+from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 
 class Command(ScrapyCommand):
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 89809a177..e61eedf50 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,7 +1,7 @@
 from __future__ import print_function
 from w3lib.url import is_url
 
-from scrapy.command import ScrapyCommand
+from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
 from scrapy.exceptions import UsageError
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 9d68ec845..2c14b3c1f 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -7,7 +7,7 @@ from importlib import import_module
 from os.path import join, dirname, abspath, exists, splitext
 
 import scrapy
-from scrapy.command import ScrapyCommand
+from scrapy.commands import ScrapyCommand
 from scrapy.utils.template import render_templatefile, string_camelcase
 from scrapy.exceptions import UsageError
 
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index 3b35cbb92..a255b3b94 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -1,5 +1,5 @@
 from __future__ import print_function
-from scrapy.command import ScrapyCommand
+from scrapy.commands import ScrapyCommand
 
 class Command(ScrapyCommand):
 
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 245b3fdf0..f90f7cdbc 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -3,7 +3,7 @@ import logging
 
 from w3lib.url import is_url
 
-from scrapy.command import ScrapyCommand
+from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
 from scrapy.item import BaseItem
 from scrapy.utils import display
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index b67838619..88f5a3015 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -3,7 +3,7 @@ import os
 from importlib import import_module
 
 from scrapy.utils.spider import iter_spider_classes
-from scrapy.command import ScrapyCommand
+from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.conf import arglist_to_dict
 
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index b8e0fe08c..65a5cd465 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -1,5 +1,5 @@
 from __future__ import print_function
-from scrapy.command import ScrapyCommand
+from scrapy.commands import ScrapyCommand
 
 class Command(ScrapyCommand):
 
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index cf99865c4..95af8586b 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -6,7 +6,7 @@ See documentation in docs/topics/shell.rst
 
 from threading import Thread
 
-from scrapy.command import ScrapyCommand
+from scrapy.commands import ScrapyCommand
 from scrapy.shell import Shell
 from scrapy.http import Request
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 5c8783ceb..0c77cad01 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -7,7 +7,7 @@ from os.path import join, exists, abspath
 from shutil import copytree, ignore_patterns
 
 import scrapy
-from scrapy.command import ScrapyCommand
+from scrapy.commands import ScrapyCommand
 from scrapy.utils.template import render_templatefile, string_camelcase
 from scrapy.exceptions import UsageError
 
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 76b97200d..a48e98f75 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -5,7 +5,7 @@ import platform
 import twisted
 
 import scrapy
-from scrapy.command import ScrapyCommand
+from scrapy.commands import ScrapyCommand
 
 
 class Command(ScrapyCommand):
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index 679c51a67..4eb44f77d 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -1,5 +1,4 @@
-from scrapy.command import ScrapyCommand
-from scrapy.commands import fetch
+from scrapy.commands import fetch, ScrapyCommand
 from scrapy.utils.response import open_in_browser
 
 class Command(fetch.Command):

From cfd40ed57f4e42c85849aaa62b0fe8b18ad69a04 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 28 Apr 2015 15:28:58 -0300
Subject: [PATCH 0297/2974] scrapy/command.py shim

---
 conftest.py       | 1 +
 scrapy/command.py | 7 +++++++
 2 files changed, 8 insertions(+)
 create mode 100644 scrapy/command.py

diff --git a/conftest.py b/conftest.py
index 70147dcf4..3a8bebd39 100644
--- a/conftest.py
+++ b/conftest.py
@@ -16,6 +16,7 @@ collect_ignore = [
     "scrapy/squeue.py",
     "scrapy/log.py",
     "scrapy/dupefilter.py",
+    "scrapy/command.py",
 ] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp")
 
 
diff --git a/scrapy/command.py b/scrapy/command.py
new file mode 100644
index 000000000..3e1219bbc
--- /dev/null
+++ b/scrapy/command.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.command` is deprecated, "
+              "use `scrapy.commands` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.commands import *

From 896b6dd4812bcb09e197312eda32b82b7460b9ed Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 9 May 2015 03:23:24 -0300
Subject: [PATCH 0298/2974] Delete scrapy/linkextractors/__init__.py

---
 scrapy/linkextractors/__init__.py | 8 --------
 1 file changed, 8 deletions(-)
 delete mode 100644 scrapy/linkextractors/__init__.py

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
deleted file mode 100644
index 28afaa2f1..000000000
--- a/scrapy/linkextractors/__init__.py
+++ /dev/null
@@ -1,8 +0,0 @@
-"""
-scrapy.linkextractors
-
-This package contains a collection of Link Extractors.
-
-For more info see docs/topics/link-extractors.rst
-"""
-from .lxmlhtml import LxmlLinkExtractor as LinkExtractor

From d72536688fa1b74cfd108202e3303a52947df96a Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 28 Apr 2015 15:42:59 -0300
Subject: [PATCH 0299/2974] Move scrapy/linkextractor.py to
 scrapy/linkextractors/__init__.py

---
 docs/topics/link-extractors.rst                        |  6 +++---
 .../{linkextractor.py => linkextractors/__init__.py}   | 10 ++++++++--
 scrapy/linkextractors/lxmlhtml.py                      |  2 +-
 scrapy/linkextractors/sgml.py                          |  2 +-
 4 files changed, 13 insertions(+), 7 deletions(-)
 rename scrapy/{linkextractor.py => linkextractors/__init__.py} (94%)

diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index edb047c86..f9b25ae63 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -78,8 +78,8 @@ LxmlLinkExtractor
     :param deny_extensions: a single value or list of strings containing
         extensions that should be ignored when extracting links.
         If not given, it will default to the
-        ``IGNORED_EXTENSIONS`` list defined in the `scrapy.linkextractor`_
-        module.
+        ``IGNORED_EXTENSIONS`` list defined in the
+        `scrapy.linkextractors`_ module.
     :type deny_extensions: list
 
     :param restrict_xpaths: is an XPath (or list of XPath's) which defines
@@ -132,4 +132,4 @@ LxmlLinkExtractor
 
     :type process_value: callable
 
-.. _scrapy.linkextractor: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractor.py
+.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py
diff --git a/scrapy/linkextractor.py b/scrapy/linkextractors/__init__.py
similarity index 94%
rename from scrapy/linkextractor.py
rename to scrapy/linkextractors/__init__.py
index 2a4d18877..8567dbb76 100644
--- a/scrapy/linkextractor.py
+++ b/scrapy/linkextractors/__init__.py
@@ -1,6 +1,9 @@
 """
-Common code and definitions used by Link extractors (located in
-scrapy.linkextractors).
+scrapy.linkextractors
+
+This package contains a collection of Link Extractors.
+
+For more info see docs/topics/link-extractors.rst
 """
 import re
 from six.moves.urllib.parse import urlparse
@@ -98,3 +101,6 @@ class FilteringLinkExtractor(object):
 
     def _extract_links(self, *args, **kwargs):
         return self.link_extractor._extract_links(*args, **kwargs)
+
+# Top-level imports
+from .lxmlhtml import LxmlLinkExtractor as LinkExtractor
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 1ff8e4d36..1c31a15b5 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -11,7 +11,7 @@ from scrapy.selector import Selector
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import unique as unique_list, str_to_unicode
-from scrapy.linkextractor import FilteringLinkExtractor
+from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.response import get_base_url
 
 
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 88d2d5b91..bae4ad5c0 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -8,7 +8,7 @@ from sgmllib import SGMLParser
 from w3lib.url import safe_url_string
 from scrapy.selector import Selector
 from scrapy.link import Link
-from scrapy.linkextractor import FilteringLinkExtractor
+from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import unique as unique_list, str_to_unicode
 from scrapy.utils.response import get_base_url

From 32945242ae230ec05f5cccc16aa33bc6c2326e4d Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 28 Apr 2015 15:44:22 -0300
Subject: [PATCH 0300/2974] scrapy/linkextractor.py shim

---
 conftest.py             | 1 +
 scrapy/linkextractor.py | 7 +++++++
 2 files changed, 8 insertions(+)
 create mode 100644 scrapy/linkextractor.py

diff --git a/conftest.py b/conftest.py
index 3a8bebd39..5e7090199 100644
--- a/conftest.py
+++ b/conftest.py
@@ -17,6 +17,7 @@ collect_ignore = [
     "scrapy/log.py",
     "scrapy/dupefilter.py",
     "scrapy/command.py",
+    "scrapy/linkextractor.py",
 ] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp")
 
 
diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py
new file mode 100644
index 000000000..b744aff8e
--- /dev/null
+++ b/scrapy/linkextractor.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.linkextractor` is deprecated, "
+              "use `scrapy.linkextractors` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.linkextractors import *

From d4926091abdb22cb52a2286dda75f281961aeca8 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 9 May 2015 03:32:01 -0300
Subject: [PATCH 0301/2974] Delete scrapy/spiders/__init__.py

---
 scrapy/spiders/__init__.py | 3 ---
 1 file changed, 3 deletions(-)
 delete mode 100644 scrapy/spiders/__init__.py

diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
deleted file mode 100644
index de4f90d51..000000000
--- a/scrapy/spiders/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from scrapy.spiders.crawl import CrawlSpider, Rule
-from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider
-from scrapy.spiders.sitemap import SitemapSpider

From d3f576a816b89260ea675aaf9d2a0140edd69949 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 9 May 2015 04:20:09 -0300
Subject: [PATCH 0302/2974] Move scrapy/spider.py to scrapy/spiders/__init__.py

---
 docs/intro/tutorial.rst                       | 14 ++++++-------
 docs/topics/api.rst                           |  2 +-
 docs/topics/downloader-middleware.rst         |  6 +++---
 docs/topics/item-pipeline.rst                 |  6 +++---
 docs/topics/leaks.rst                         |  4 ++--
 docs/topics/logging.rst                       |  2 +-
 docs/topics/request-response.rst              |  2 +-
 docs/topics/settings.rst                      |  2 +-
 docs/topics/shell.rst                         |  2 +-
 docs/topics/signals.rst                       | 20 +++++++++----------
 docs/topics/spider-middleware.rst             | 12 +++++------
 docs/topics/spiders.rst                       | 11 +++++-----
 extras/qpsclient.py                           |  2 +-
 scrapy/__init__.py                            |  2 +-
 scrapy/crawler.py                             |  2 +-
 scrapy/shell.py                               |  2 +-
 scrapy/{spider.py => spiders/__init__.py}     |  4 ++++
 scrapy/spiders/crawl.py                       |  2 +-
 scrapy/spiders/feed.py                        |  2 +-
 scrapy/spiders/init.py                        |  6 +++---
 scrapy/spiders/sitemap.py                     |  2 +-
 scrapy/utils/spider.py                        |  4 ++--
 scrapy/utils/test.py                          |  2 +-
 tests/spiders.py                              |  2 +-
 tests/test_commands.py                        |  2 +-
 tests/test_contracts.py                       |  2 +-
 tests/test_downloader_handlers.py             |  2 +-
 tests/test_downloadermiddleware.py            |  2 +-
 ...test_downloadermiddleware_ajaxcrawlable.py |  2 +-
 tests/test_downloadermiddleware_cookies.py    |  2 +-
 ...test_downloadermiddleware_decompression.py |  2 +-
 ...est_downloadermiddleware_defaultheaders.py |  2 +-
 ...st_downloadermiddleware_downloadtimeout.py |  2 +-
 tests/test_downloadermiddleware_httpauth.py   |  2 +-
 tests/test_downloadermiddleware_httpcache.py  |  2 +-
 ...st_downloadermiddleware_httpcompression.py |  2 +-
 tests/test_downloadermiddleware_httpproxy.py  |  2 +-
 tests/test_downloadermiddleware_redirect.py   |  2 +-
 tests/test_downloadermiddleware_retry.py      |  2 +-
 tests/test_downloadermiddleware_stats.py      |  2 +-
 tests/test_downloadermiddleware_useragent.py  |  2 +-
 tests/test_engine.py                          |  2 +-
 tests/test_logformatter.py                    |  2 +-
 tests/test_pipeline_media.py                  |  2 +-
 tests/test_spider.py                          |  5 ++---
 tests/test_spiderloader/__init__.py           |  2 +-
 .../test_spiderloader/test_spiders/spider0.py |  2 +-
 .../test_spiderloader/test_spiders/spider1.py |  2 +-
 .../test_spiderloader/test_spiders/spider2.py |  2 +-
 .../test_spiderloader/test_spiders/spider3.py |  2 +-
 tests/test_spidermiddleware_depth.py          |  2 +-
 tests/test_spidermiddleware_httperror.py      |  2 +-
 tests/test_spidermiddleware_offsite.py        |  2 +-
 tests/test_spidermiddleware_referer.py        |  2 +-
 tests/test_spidermiddleware_urllength.py      |  2 +-
 tests/test_spiderstate.py                     |  2 +-
 tests/test_stats.py                           |  2 +-
 tests/test_toplevel.py                        |  2 +-
 tests/test_utils_reqser.py                    |  2 +-
 tests/test_utils_url.py                       |  2 +-
 60 files changed, 96 insertions(+), 94 deletions(-)
 rename scrapy/{spider.py => spiders/__init__.py} (94%)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 219616587..0d3c49750 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -95,18 +95,18 @@ domain (or group of domains).
 They define an initial list of URLs to download, how to follow links, and how
 to parse the contents of pages to extract :ref:`items <topics-items>`.
 
-To create a Spider, you must subclass :class:`scrapy.Spider <scrapy.spider.Spider>` and
-define some attributes:
+To create a Spider, you must subclass :class:`scrapy.Spider
+<scrapy.spiders.Spider>` and define some attributes:
 
-* :attr:`~scrapy.spider.Spider.name`: identifies the Spider. It must be
+* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be
   unique, that is, you can't set the same name for different Spiders.
 
-* :attr:`~scrapy.spider.Spider.start_urls`: a list of URLs where the
+* :attr:`~scrapy.spiders.Spider.start_urls`: a list of URLs where the
   Spider will begin to crawl from.  The first pages downloaded will be those
   listed here. The subsequent URLs will be generated successively from data
   contained in the start URLs.
 
-* :meth:`~scrapy.spider.Spider.parse`: a method of the spider, which will
+* :meth:`~scrapy.spiders.Spider.parse`: a method of the spider, which will
   be called with the downloaded :class:`~scrapy.http.Response` object of each
   start URL. The response is passed to the method as the first and only
   argument.
@@ -114,7 +114,7 @@ define some attributes:
   This method is responsible for parsing the response data and extracting
   scraped data (as scraped items) and more URLs to follow.
 
-  The :meth:`~scrapy.spider.Spider.parse` method is in charge of processing
+  The :meth:`~scrapy.spiders.Spider.parse` method is in charge of processing
   the response and returning scraped data (as :class:`~scrapy.item.Item`
   objects) and more URLs to follow (as :class:`~scrapy.http.Request` objects).
 
@@ -178,7 +178,7 @@ them the ``parse`` method of the spider as their callback function.
 
 These Requests are scheduled, then executed, and :class:`scrapy.http.Response`
 objects are returned and then fed back to the spider, through the
-:meth:`~scrapy.spider.Spider.parse` method.
+:meth:`~scrapy.spiders.Spider.parse` method.
 
 Extracting Items
 ----------------
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index ce28b8bc1..e59fe9a58 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -31,7 +31,7 @@ how you :ref:`configure the downloader middlewares
 .. class:: Crawler(spidercls, settings)
 
     The Crawler object must be instantiated with a
-    :class:`scrapy.spider.Spider` subclass and a
+    :class:`scrapy.spiders.Spider` subclass and a
     :class:`scrapy.settings.Settings` object.
 
     .. attribute:: settings
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 5cb6c9824..03c5deffe 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -91,7 +91,7 @@ more of the following methods:
       :type request: :class:`~scrapy.http.Request` object
 
       :param spider: the spider for which this request is intended
-      :type spider: :class:`~scrapy.spider.Spider` object
+      :type spider: :class:`~scrapy.spiders.Spider` object
 
    .. method:: process_response(request, response, spider)
 
@@ -118,7 +118,7 @@ more of the following methods:
       :type response: :class:`~scrapy.http.Response` object
 
       :param spider: the spider for which this response is intended
-      :type spider: :class:`~scrapy.spider.Spider` object
+      :type spider: :class:`~scrapy.spiders.Spider` object
 
    .. method:: process_exception(request, exception, spider)
 
@@ -149,7 +149,7 @@ more of the following methods:
       :type exception: an ``Exception`` object
 
       :param spider: the spider for which this request is intended
-      :type spider: :class:`~scrapy.spider.Spider` object
+      :type spider: :class:`~scrapy.spiders.Spider` object
 
 .. _topics-downloader-middleware-ref:
 
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index dd2d79989..f74400b4d 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -36,7 +36,7 @@ Each item pipeline component is a Python class that must implement the following
    :type item: :class:`~scrapy.item.Item` object or a dict
 
    :param spider: the spider which scraped the item
-   :type spider: :class:`~scrapy.spider.Spider` object
+   :type spider: :class:`~scrapy.spiders.Spider` object
 
 Additionally, they may also implement the following methods:
 
@@ -45,14 +45,14 @@ Additionally, they may also implement the following methods:
    This method is called when the spider is opened.
 
    :param spider: the spider which was opened
-   :type spider: :class:`~scrapy.spider.Spider` object
+   :type spider: :class:`~scrapy.spiders.Spider` object
 
 .. method:: close_spider(self, spider)
 
    This method is called when the spider is closed.
 
    :param spider: the spider which was closed
-   :type spider: :class:`~scrapy.spider.Spider` object
+   :type spider: :class:`~scrapy.spiders.Spider` object
 
 .. method:: from_crawler(cls, crawler)
 
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 9d15ebe48..735137ea2 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -92,7 +92,7 @@ subclasses):
 * :class:`scrapy.http.Response`
 * :class:`scrapy.item.Item`
 * :class:`scrapy.selector.Selector`
-* :class:`scrapy.spider.Spider`
+* :class:`scrapy.spiders.Spider`
 
 A real example
 --------------
@@ -155,7 +155,7 @@ For this reason, that function has a ``ignore`` argument which can be used to
 ignore a particular class (and all its subclases). For
 example, this won't show any live references to spiders::
 
-    >>> from scrapy.spider import Spider
+    >>> from scrapy.spiders import Spider
     >>> prefs(ignore=Spider)
 
 .. module:: scrapy.utils.trackref
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 2cb719998..1a3f5d69f 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -94,7 +94,7 @@ path::
 Logging from Spiders
 ====================
 
-Scrapy provides a :data:`~scrapy.spider.Spider.logger` within each Spider
+Scrapy provides a :data:`~scrapy.spiders.Spider.logger` within each Spider
 instance, that can be accessed and used like this::
 
     import scrapy
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 1d695a5f2..aa601f83a 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -37,7 +37,7 @@ Request objects
        request (once its downloaded) as its first parameter. For more information
        see :ref:`topics-request-response-ref-request-callback-arguments` below.
        If a Request doesn't specify a callback, the spider's
-       :meth:`~scrapy.spider.Spider.parse` method will be used.
+       :meth:`~scrapy.spiders.Spider.parse` method will be used.
        Note that if exceptions are raised during processing, errback is called instead.
 
     :type callback: callable
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 26a6d762d..5917bea4e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -67,7 +67,7 @@ Example::
 
 Spiders (See the :ref:`topics-spiders` chapter for reference) can define their
 own settings that will take precedence and override the project ones. They can
-do so by setting their :attr:`scrapy.spider.Spider.custom_settings` attribute.
+do so by setting their :attr:`scrapy.spiders.Spider.custom_settings` attribute.
 
 3. Project settings module
 --------------------------
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 3b875fec5..9c9411d6d 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -74,7 +74,7 @@ Those objects are:
  * ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
 
  * ``spider`` - the Spider which is known to handle the URL, or a
-   :class:`~scrapy.spider.Spider` object if there is no spider found for
+   :class:`~scrapy.spiders.Spider` object if there is no spider found for
    the current URL
 
  * ``request`` - a :class:`~scrapy.http.Request` object of the last fetched
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 85cf43c76..7ea9efe49 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -74,7 +74,7 @@ item_scraped
     :type item: dict or :class:`~scrapy.item.Item` object
 
     :param spider: the spider which scraped the item
-    :type spider: :class:`~scrapy.spider.Spider` object
+    :type spider: :class:`~scrapy.spiders.Spider` object
 
     :param response: the response from where the item was scraped
     :type response: :class:`~scrapy.http.Response` object
@@ -94,7 +94,7 @@ item_dropped
     :type item: dict or :class:`~scrapy.item.Item` object
 
     :param spider: the spider which scraped the item
-    :type spider: :class:`~scrapy.spider.Spider` object
+    :type spider: :class:`~scrapy.spiders.Spider` object
 
     :param response: the response from where the item was dropped
     :type response: :class:`~scrapy.http.Response` object
@@ -116,7 +116,7 @@ spider_closed
     This signal supports returning deferreds from their handlers.
 
     :param spider: the spider which has been closed
-    :type spider: :class:`~scrapy.spider.Spider` object
+    :type spider: :class:`~scrapy.spiders.Spider` object
 
     :param reason: a string which describes the reason why the spider was closed. If
         it was closed because the spider has completed scraping, the reason
@@ -140,7 +140,7 @@ spider_opened
     This signal supports returning deferreds from their handlers.
 
     :param spider: the spider which has been opened
-    :type spider: :class:`~scrapy.spider.Spider` object
+    :type spider: :class:`~scrapy.spiders.Spider` object
 
 spider_idle
 -----------
@@ -164,7 +164,7 @@ spider_idle
     This signal does not support returning deferreds from their handlers.
 
     :param spider: the spider which has gone idle
-    :type spider: :class:`~scrapy.spider.Spider` object
+    :type spider: :class:`~scrapy.spiders.Spider` object
 
 spider_error
 ------------
@@ -181,7 +181,7 @@ spider_error
     :type response: :class:`~scrapy.http.Response` object
 
     :param spider: the spider which raised the exception
-    :type spider: :class:`~scrapy.spider.Spider` object
+    :type spider: :class:`~scrapy.spiders.Spider` object
 
 request_scheduled
 -----------------
@@ -198,7 +198,7 @@ request_scheduled
     :type request: :class:`~scrapy.http.Request` object
 
     :param spider: the spider that yielded the request
-    :type spider: :class:`~scrapy.spider.Spider` object
+    :type spider: :class:`~scrapy.spiders.Spider` object
 
 request_dropped
 -----------------
@@ -215,7 +215,7 @@ request_dropped
     :type request: :class:`~scrapy.http.Request` object
 
     :param spider: the spider that yielded the request
-    :type spider: :class:`~scrapy.spider.Spider` object
+    :type spider: :class:`~scrapy.spiders.Spider` object
 
 response_received
 -----------------
@@ -235,7 +235,7 @@ response_received
     :type request: :class:`~scrapy.http.Request` object
 
     :param spider: the spider for which the response is intended
-    :type spider: :class:`~scrapy.spider.Spider` object
+    :type spider: :class:`~scrapy.spiders.Spider` object
 
 response_downloaded
 -------------------
@@ -254,6 +254,6 @@ response_downloaded
     :type request: :class:`~scrapy.http.Request` object
 
     :param spider: the spider for which the response is intended
-    :type spider: :class:`~scrapy.spider.Spider` object
+    :type spider: :class:`~scrapy.spiders.Spider` object
 
 .. _Failure: http://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 6e82333f5..84daaaa55 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -81,7 +81,7 @@ following methods:
         :type response: :class:`~scrapy.http.Response` object
 
         :param spider: the spider for which this response is intended
-        :type spider: :class:`~scrapy.spider.Spider` object
+        :type spider: :class:`~scrapy.spiders.Spider` object
 
 
     .. method:: process_spider_output(response, result, spider)
@@ -102,7 +102,7 @@ following methods:
           or :class:`~scrapy.item.Item` objects
 
         :param spider: the spider whose result is being processed
-        :type spider: :class:`~scrapy.spider.Spider` object
+        :type spider: :class:`~scrapy.spiders.Spider` object
 
 
     .. method:: process_spider_exception(response, exception, spider)
@@ -130,7 +130,7 @@ following methods:
         :type exception: `Exception`_ object
 
         :param spider: the spider which raised the exception
-        :type spider: :class:`~scrapy.spider.Spider` object
+        :type spider: :class:`~scrapy.spiders.Spider` object
 
     .. method:: process_start_requests(start_requests, spider)
 
@@ -157,7 +157,7 @@ following methods:
         :type start_requests: an iterable of :class:`~scrapy.http.Request`
 
         :param spider: the spider to whom the start requests belong
-        :type spider: :class:`~scrapy.spider.Spider` object
+        :type spider: :class:`~scrapy.spiders.Spider` object
 
 
 .. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception
@@ -272,7 +272,7 @@ OffsiteMiddleware
    Filters out Requests for URLs outside the domains covered by the spider.
 
    This middleware filters out every request whose host names aren't in the
-   spider's :attr:`~scrapy.spider.Spider.allowed_domains` attribute.
+   spider's :attr:`~scrapy.spiders.Spider.allowed_domains` attribute.
 
    When your spider returns a request for a domain not belonging to those
    covered by the spider, this middleware will log a debug message similar to
@@ -287,7 +287,7 @@ OffsiteMiddleware
    will be printed (but only for the first request filtered).
 
    If the spider doesn't define an
-   :attr:`~scrapy.spider.Spider.allowed_domains` attribute, or the
+   :attr:`~scrapy.spiders.Spider.allowed_domains` attribute, or the
    attribute is empty, the offsite middleware will allow all requests.
 
    If the request has the :attr:`~scrapy.http.Request.dont_filter` attribute
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index d2fdd61b8..025d527a6 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -17,10 +17,10 @@ For spiders, the scraping cycle goes through something like this:
    those requests.
 
    The first requests to perform are obtained by calling the
-   :meth:`~scrapy.spider.Spider.start_requests` method which (by default)
+   :meth:`~scrapy.spiders.Spider.start_requests` method which (by default)
    generates :class:`~scrapy.http.Request` for the URLs specified in the
-   :attr:`~scrapy.spider.Spider.start_urls` and the
-   :attr:`~scrapy.spider.Spider.parse` method as callback function for the
+   :attr:`~scrapy.spiders.Spider.start_urls` and the
+   :attr:`~scrapy.spiders.Spider.parse` method as callback function for the
    Requests.
 
 2. In the callback function, you parse the response (web page) and return either
@@ -42,7 +42,7 @@ Even though this cycle applies (more or less) to any kind of spider, there are
 different kinds of default spiders bundled into Scrapy for different purposes.
 We will talk about those types here.
 
-.. module:: scrapy.spider
+.. module:: scrapy.spiders
    :synopsis: Spiders base class, spider manager and spider middleware
 
 .. _topics-spiders-ref:
@@ -319,8 +319,7 @@ with a ``TestItem`` declared in a ``myproject.items`` module::
         description = scrapy.Field()
 
 
-.. module:: scrapy.spiders
-   :synopsis: Collection of generic spiders
+.. currentmodule:: scrapy.spiders
 
 CrawlSpider
 -----------
diff --git a/extras/qpsclient.py b/extras/qpsclient.py
index 7a1baccca..bb83588dd 100644
--- a/extras/qpsclient.py
+++ b/extras/qpsclient.py
@@ -7,7 +7,7 @@ usage:
 
 """
 
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.http import Request
 
 
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 31ad2ff02..10ba9544f 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -45,7 +45,7 @@ if twisted_version >= (11, 1, 0):
     optional_features.add('http11')
 
 # Declare top-level shortcuts
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.http import Request, FormRequest
 from scrapy.selector import Selector
 from scrapy.item import Item, Field
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 161ca4614..e6f4b225f 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -138,7 +138,7 @@ class CrawlerRunner(object):
         :param crawler_or_spidercls: already created crawler, or a spider class
             or spider's name inside the project to create it
         :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance,
-            :class:`~scrapy.spider.Spider` subclass or string
+            :class:`~scrapy.spiders.Spider` subclass or string
 
         :param list args: arguments to initialize the spider
 
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 8f87fcb41..4142396ab 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -17,7 +17,7 @@ from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.item import BaseItem
 from scrapy.settings import Settings
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.utils.console import start_python_console
 from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
diff --git a/scrapy/spider.py b/scrapy/spiders/__init__.py
similarity index 94%
rename from scrapy/spider.py
rename to scrapy/spiders/__init__.py
index 36623b6e2..c08bb964a 100644
--- a/scrapy/spider.py
+++ b/scrapy/spiders/__init__.py
@@ -111,3 +111,7 @@ spiders = ObsoleteClass(
     'it with your project settings"'
 )
 
+# Top-level imports
+from scrapy.spiders.crawl import CrawlSpider, Rule
+from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider
+from scrapy.spiders.sitemap import SitemapSpider
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 7dc3dacd6..77551753e 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -9,7 +9,7 @@ import copy
 
 from scrapy.http import Request, HtmlResponse
 from scrapy.utils.spider import iterate_spider_output
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 
 def identity(x):
     return x
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index d83ee605e..06e212e1c 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -4,7 +4,7 @@ for scraping from an XML feed.
 
 See documentation in docs/topics/spiders.rst
 """
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.utils.iterators import xmliter, csviter
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.selector import Selector
diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py
index 9c94a7b33..7717c8819 100644
--- a/scrapy/spiders/init.py
+++ b/scrapy/spiders/init.py
@@ -1,4 +1,4 @@
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.utils.spider import iterate_spider_output
 
 class InitSpider(Spider):
@@ -20,8 +20,8 @@ class InitSpider(Spider):
         is called this spider is considered initialized. If you need to perform
         several requests for initializing your spider, you can do so by using
         different callbacks. The only requirement is that the final callback
-        (of the last initialization request) must be self.initialized. 
-        
+        (of the last initialization request) must be self.initialized.
+
         The default implementation calls self.initialized immediately, and
         means that no initialization is needed. This method should be
         overridden only when you need to perform requests to initialize your
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 90fb9fb4e..5aa0b944d 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -1,7 +1,7 @@
 import re
 import logging
 
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.http import Request, XmlResponse
 from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
 from scrapy.utils.gz import gunzip, is_gzipped
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 7ed2d0c3b..94b24f67e 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -3,7 +3,7 @@ import inspect
 
 import six
 
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.utils.misc import  arg_to_iter
 
 logger = logging.getLogger(__name__)
@@ -19,7 +19,7 @@ def iter_spider_classes(module):
     """
     # this needs to be imported here until get rid of the spider manager
     # singleton in scrapy.spider.spiders
-    from scrapy.spider import Spider
+    from scrapy.spiders import Spider
 
     for obj in six.itervalues(vars(module)):
         if inspect.isclass(obj) and \
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index ad4a6aa7c..bec9bdda9 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -27,7 +27,7 @@ def get_crawler(spidercls=None, settings_dict=None):
     """
     from scrapy.crawler import CrawlerRunner
     from scrapy.settings import Settings
-    from scrapy.spider import Spider
+    from scrapy.spiders import Spider
 
     runner = CrawlerRunner(Settings(settings_dict))
     return runner._create_crawler(spidercls or Spider)
diff --git a/tests/spiders.py b/tests/spiders.py
index c2956d741..516062929 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -5,7 +5,7 @@ Some spiders used for testing and benchmarking
 import time
 from six.moves.urllib.parse import urlencode
 
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.item import Item
 from scrapy.linkextractors import LinkExtractor
diff --git a/tests/test_commands.py b/tests/test_commands.py
index b95d0b0cc..7c10faf0c 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -158,7 +158,7 @@ class MySpider(scrapy.Spider):
         fname = abspath(join(tmpdir, 'myspider.py'))
         with open(fname, 'w') as f:
             f.write("""
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 """)
         p = self.proc('runspider', fname)
         log = p.stderr.read()
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index d7732f55d..1cea2afb7 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -2,7 +2,7 @@ from unittest import TextTestResult
 
 from twisted.trial import unittest
 
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.item import Item, Field
 from scrapy.contracts import ContractsManager
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 62fc280ee..df038aa88 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -24,7 +24,7 @@ from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
 
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.settings import Settings
 from scrapy import optional_features
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 282035f5c..257eab609 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -2,7 +2,7 @@ from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
 
 from scrapy.http import Request, Response
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.utils.test import get_crawler
 
diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py
index 11de6e22d..67c57778d 100644
--- a/tests/test_downloadermiddleware_ajaxcrawlable.py
+++ b/tests/test_downloadermiddleware_ajaxcrawlable.py
@@ -1,7 +1,7 @@
 import unittest
 
 from scrapy.downloadermiddlewares.ajaxcrawl import AjaxCrawlMiddleware
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.http import Request, HtmlResponse, Response
 from scrapy.utils.test import get_crawler
 
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 7f491f258..996b8c388 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -2,7 +2,7 @@ from unittest import TestCase
 import re
 
 from scrapy.http import Response, Request
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
 
 
diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py
index 7aca415ef..9143611fc 100644
--- a/tests/test_downloadermiddleware_decompression.py
+++ b/tests/test_downloadermiddleware_decompression.py
@@ -1,7 +1,7 @@
 from unittest import TestCase, main
 from scrapy.http import Response, XmlResponse
 from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from tests import get_testdata
 from scrapy.utils.test import assert_samelines
 
diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py
index 26520a20a..75d8a1921 100644
--- a/tests/test_downloadermiddleware_defaultheaders.py
+++ b/tests/test_downloadermiddleware_defaultheaders.py
@@ -3,7 +3,7 @@ import six
 
 from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
 from scrapy.http import Request
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
 
diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py
index 282d10829..446a99f36 100644
--- a/tests/test_downloadermiddleware_downloadtimeout.py
+++ b/tests/test_downloadermiddleware_downloadtimeout.py
@@ -1,7 +1,7 @@
 import unittest
 
 from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.utils.test import get_crawler
 
diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py
index cef65b336..c30fa97c6 100644
--- a/tests/test_downloadermiddleware_httpauth.py
+++ b/tests/test_downloadermiddleware_httpauth.py
@@ -2,7 +2,7 @@ import unittest
 
 from scrapy.http import Request
 from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 
 class TestSpider(Spider):
     http_user = 'foo'
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index ac954cc15..47d057e3f 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -8,7 +8,7 @@ from contextlib import contextmanager
 import pytest
 
 from scrapy.http import Response, HtmlResponse, Request
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.settings import Settings
 from scrapy.exceptions import IgnoreRequest
 from scrapy.utils.test import get_crawler
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 98df6d608..a18994ef3 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -3,7 +3,7 @@ from unittest import TestCase
 from os.path import join, abspath, dirname
 from gzip import GzipFile
 
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.http import Response, Request, HtmlResponse
 from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware
 from tests import tests_datadir
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 90609879c..191664076 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -5,7 +5,7 @@ from twisted.trial.unittest import TestCase, SkipTest
 from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response, Request
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 
 spider = Spider('foo')
 
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 3f299f258..7e88e71af 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -1,7 +1,7 @@
 import unittest
 
 from scrapy.downloadermiddlewares.redirect import RedirectMiddleware, MetaRefreshMiddleware
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.exceptions import IgnoreRequest
 from scrapy.http import Request, Response, HtmlResponse
 from scrapy.utils.test import get_crawler
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 969452cfb..c0381e144 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -7,7 +7,7 @@ from twisted.internet.error import TimeoutError, DNSLookupError, \
 from scrapy import optional_features
 from scrapy.downloadermiddlewares.retry import RetryMiddleware
 from scrapy.xlib.tx import ResponseFailed
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.http import Request, Response
 from scrapy.utils.test import get_crawler
 
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index 64f2d0786..fb46ccff6 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -2,7 +2,7 @@ from unittest import TestCase
 
 from scrapy.downloadermiddlewares.stats import DownloaderStats
 from scrapy.http import Request, Response
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
 
diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py
index ddbb8f3ac..741c8de76 100644
--- a/tests/test_downloadermiddleware_useragent.py
+++ b/tests/test_downloadermiddleware_useragent.py
@@ -1,6 +1,6 @@
 from unittest import TestCase
 
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
 from scrapy.utils.test import get_crawler
diff --git a/tests/test_engine.py b/tests/test_engine.py
index bbb94fd58..d7ad88abb 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -22,7 +22,7 @@ from scrapy import signals
 from scrapy.utils.test import get_crawler
 from scrapy.xlib.pydispatch import dispatcher
 from tests import tests_datadir
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.item import Item, Field
 from scrapy.linkextractors import LinkExtractor
 from scrapy.http import Request
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 8446fd646..ec42ef8ab 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -1,6 +1,6 @@
 import unittest
 
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.http import Request, Response
 from scrapy.item import Item, Field
 from scrapy.logformatter import LogFormatter
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 24ba9d64a..7217eee90 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -6,7 +6,7 @@ from twisted.internet import reactor
 from twisted.internet.defer import Deferred, inlineCallbacks
 
 from scrapy.http import Request, Response
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.utils.request import request_fingerprint
 from scrapy.pipelines.media import MediaPipeline
 from scrapy.utils.signal import disconnect_all
diff --git a/tests/test_spider.py b/tests/test_spider.py
index f771399ca..f2dfd2dce 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -7,11 +7,10 @@ from testfixtures import LogCapture
 from twisted.trial import unittest
 
 from scrapy import signals
-from scrapy.spider import Spider, BaseSpider
 from scrapy.settings import Settings
 from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse
 from scrapy.spiders.init import InitSpider
-from scrapy.spiders import CrawlSpider, Rule, XMLFeedSpider, \
+from scrapy.spiders import Spider, BaseSpider, CrawlSpider, Rule, XMLFeedSpider, \
     CSVFeedSpider, SitemapSpider
 from scrapy.linkextractors import LinkExtractor
 from scrapy.exceptions import ScrapyDeprecationWarning
@@ -116,7 +115,7 @@ class SpiderTest(unittest.TestCase):
 
     def test_log(self):
         spider = self.spider_class('example.com')
-        with mock.patch('scrapy.spider.Spider.logger') as mock_logger:
+        with mock.patch('scrapy.spiders.Spider.logger') as mock_logger:
             spider.log('test log msg', 'INFO')
         mock_logger.log.assert_called_once_with('INFO', 'test log msg')
 
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 42f2f29b3..7cb5e299b 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -6,7 +6,7 @@ from zope.interface.verify import verifyObject
 from twisted.trial import unittest
 
 
-# ugly hack to avoid cyclic imports of scrapy.spider when running this test
+# ugly hack to avoid cyclic imports of scrapy.spiders when running this test
 # alone
 from scrapy.interfaces import ISpiderLoader
 from scrapy.spiderloader import SpiderLoader
diff --git a/tests/test_spiderloader/test_spiders/spider0.py b/tests/test_spiderloader/test_spiders/spider0.py
index f1f19a1eb..75a90794e 100644
--- a/tests/test_spiderloader/test_spiders/spider0.py
+++ b/tests/test_spiderloader/test_spiders/spider0.py
@@ -1,4 +1,4 @@
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 
 class Spider0(Spider):
     allowed_domains = ["scrapy1.org", "scrapy3.org"]
diff --git a/tests/test_spiderloader/test_spiders/spider1.py b/tests/test_spiderloader/test_spiders/spider1.py
index 16a533ca2..76efddc7f 100644
--- a/tests/test_spiderloader/test_spiders/spider1.py
+++ b/tests/test_spiderloader/test_spiders/spider1.py
@@ -1,4 +1,4 @@
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 
 class Spider1(Spider):
     name = "spider1"
diff --git a/tests/test_spiderloader/test_spiders/spider2.py b/tests/test_spiderloader/test_spiders/spider2.py
index 4af6f7c41..0badd8437 100644
--- a/tests/test_spiderloader/test_spiders/spider2.py
+++ b/tests/test_spiderloader/test_spiders/spider2.py
@@ -1,4 +1,4 @@
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 
 class Spider2(Spider):
     name = "spider2"
diff --git a/tests/test_spiderloader/test_spiders/spider3.py b/tests/test_spiderloader/test_spiders/spider3.py
index b3e5f3da7..d406f2d4f 100644
--- a/tests/test_spiderloader/test_spiders/spider3.py
+++ b/tests/test_spiderloader/test_spiders/spider3.py
@@ -1,4 +1,4 @@
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 
 class Spider3(Spider):
     name = "spider3"
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index 5317795a3..a3cdc0114 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -2,7 +2,7 @@ from unittest import TestCase
 
 from scrapy.spidermiddlewares.depth import DepthMiddleware
 from scrapy.http import Response, Request
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.statscollectors import StatsCollector
 from scrapy.utils.test import get_crawler
 
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 5cd2c2566..a64400482 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -7,7 +7,7 @@ from twisted.internet import defer
 from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
 from scrapy.http import Response, Request
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError
 from scrapy.settings import Settings
 
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 296e8b1a3..f88c806d7 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -3,7 +3,7 @@ from unittest import TestCase
 from six.moves.urllib.parse import urlparse
 
 from scrapy.http import Response, Request
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.spidermiddlewares.offsite import OffsiteMiddleware
 from scrapy.utils.test import get_crawler
 
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index f2815ebd3..d773ea8d3 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,7 +1,7 @@
 from unittest import TestCase
 
 from scrapy.http import Response, Request
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.spidermiddlewares.referer import RefererMiddleware
 
 
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index 1ef22ea07..dca868ecf 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -2,7 +2,7 @@ from unittest import TestCase
 
 from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
 from scrapy.http import Response, Request
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 
 
 class TestUrlLengthMiddleware(TestCase):
diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index 1ddce4b99..d83015bd9 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -3,7 +3,7 @@ from datetime import datetime
 from twisted.trial import unittest
 
 from scrapy.extensions.spiderstate import SpiderState
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 
 
 class SpiderStateTest(unittest.TestCase):
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 34f72736c..5c7c0e6bb 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -1,6 +1,6 @@
 import unittest
 
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.statscollectors import StatsCollector, DummyStatsCollector
 from scrapy.utils.test import get_crawler
 
diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py
index 17cf82213..e9f220092 100644
--- a/tests/test_toplevel.py
+++ b/tests/test_toplevel.py
@@ -21,7 +21,7 @@ class ToplevelTestCase(TestCase):
         self.assertIs(scrapy.FormRequest, FormRequest)
 
     def test_spider_shortcut(self):
-        from scrapy.spider import Spider
+        from scrapy.spiders import Spider
         self.assertIs(scrapy.Spider, Spider)
 
     def test_selector_shortcut(self):
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 4ddc2f472..9139c0ad0 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -1,7 +1,7 @@
 import unittest
 
 from scrapy.http import Request
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.utils.reqser import request_to_dict, request_from_dict
 
 class RequestSerializationTest(unittest.TestCase):
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 959760068..860c76bae 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,6 +1,6 @@
 import unittest
 
-from scrapy.spider import Spider
+from scrapy.spiders import Spider
 from scrapy.utils.url import url_is_from_any_domain, url_is_from_spider, canonicalize_url
 
 __doctests__ = ['scrapy.utils.url']

From 53fdaa3f70e662b21d209e2094dffba9cd33823f Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 9 May 2015 04:21:50 -0300
Subject: [PATCH 0303/2974] scrapy/spider.py shim

---
 conftest.py      | 1 +
 scrapy/spider.py | 7 +++++++
 2 files changed, 8 insertions(+)
 create mode 100644 scrapy/spider.py

diff --git a/conftest.py b/conftest.py
index 5e7090199..5829f2952 100644
--- a/conftest.py
+++ b/conftest.py
@@ -18,6 +18,7 @@ collect_ignore = [
     "scrapy/dupefilter.py",
     "scrapy/command.py",
     "scrapy/linkextractor.py",
+    "scrapy/spider.py",
 ] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp")
 
 
diff --git a/scrapy/spider.py b/scrapy/spider.py
new file mode 100644
index 000000000..56a5a0a0b
--- /dev/null
+++ b/scrapy/spider.py
@@ -0,0 +1,7 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.spider` is deprecated, "
+              "use `scrapy.spiders` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+from scrapy.spiders import *

From 819a8eceee2646e1ca75c5c5cb099047c34c893b Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 9 May 2015 05:12:35 -0300
Subject: [PATCH 0304/2974] Mark as orphan the doc topics not listed in the
 index

---
 docs/topics/djangoitem.rst | 2 ++
 docs/topics/scrapyd.rst    | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/docs/topics/djangoitem.rst b/docs/topics/djangoitem.rst
index ae40401a7..d7b86492d 100644
--- a/docs/topics/djangoitem.rst
+++ b/docs/topics/djangoitem.rst
@@ -1,3 +1,5 @@
+:orphan:
+
 .. _topics-djangoitem:
 
 ==========
diff --git a/docs/topics/scrapyd.rst b/docs/topics/scrapyd.rst
index 85d27a99e..57921b901 100644
--- a/docs/topics/scrapyd.rst
+++ b/docs/topics/scrapyd.rst
@@ -1,3 +1,5 @@
+:orphan:
+
 .. _topics-scrapyd:
 
 =======

From acc13c98211cebf466e9ec4ed54b1c74adfd0981 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 9 May 2015 05:15:17 -0300
Subject: [PATCH 0305/2974] Delete tab used as indentation in
 docs/topics/loaders.rst

---
 docs/topics/loaders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 4c24166e6..8841711ea 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -566,7 +566,7 @@ Here is a list of all built-in processors:
 
     The simplest processor, which doesn't do anything. It returns the original
     values unchanged. It doesn't receive any constructor arguments, nor does it
-	accept Loader contexts.
+    accept Loader contexts.
 
     Example::
 

From 6fd7d8544829b8e921f65b2720f58ad21bf34a39 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 9 May 2015 05:19:15 -0300
Subject: [PATCH 0306/2974] Wrong bullet list indentation in
 docs/topics/media-pipeline.rst

---
 docs/topics/media-pipeline.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index c18ed25ae..4e8562cac 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -165,7 +165,7 @@ Will be downloaded and stored in the following file::
 Where:
 
 * ``<IMAGES_STORE>`` is the directory defined in :setting:`IMAGES_STORE` setting
-for the Images Pipeline.
+  for the Images Pipeline.
 
 * ``full`` is a sub-directory to separate full images from thumbnails (if
   used). For more info see :ref:`topics-images-thumbnails`.

From c271d8f0b1d037b021089a5c58d42609cc802c2b Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 9 May 2015 05:20:54 -0300
Subject: [PATCH 0307/2974] Title underline too short in
 docs/topics/selectors.rst

---
 docs/topics/selectors.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 231119f3b..0d82494a4 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -479,7 +479,7 @@ But using the ``.`` to mean the node, works::
 .. _`XPath string function`: http://www.w3.org/TR/xpath/#section-String-Functions
 
 Beware of the difference between //node[1] and (//node)[1]
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ``//node[1]`` selects all the nodes occurring first under their respective parents.
 

From 42c2c4b6ce80a37c3371d475439e87fc0179a94a Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Sat, 9 May 2015 05:23:12 -0300
Subject: [PATCH 0308/2974] Wrong topic link in docs/intro/overview.rst

---
 docs/intro/overview.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 820581457..595e85e28 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -157,9 +157,9 @@ scraping easy and efficient, such as:
   crawler
 
 * Plus other goodies like reusable spiders to crawl sites from `Sitemaps`_ and
-  XML/CSV feeds, a media pipeline for :ref:`automatically downloading images <topics-images>`
-  (or any other media) associated with the scraped items, a caching DNS resolver,
-  and much more!
+  XML/CSV feeds, a media pipeline for :ref:`automatically downloading images
+  <topics-media-pipeline>` (or any other media) associated with the scraped
+  items, a caching DNS resolver, and much more!
 
 What's next?
 ============

From 5753e498bf4ade06942e853279913375a0b338a3 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Sat, 9 May 2015 16:15:06 -0300
Subject: [PATCH 0309/2974] fixes referencing, and list only settings not
 documented in current document

---
 docs/_ext/scrapydocs.py  | 48 +++++++++++++++++++++++++---------------
 docs/topics/settings.rst | 12 ++++++++--
 2 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index 88e6386c9..58d9cd0c7 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -13,8 +13,24 @@ class SettingsListDirective(Directive):
         return [settingslist_node('')]
 
 
-def is_setting_node(node):
-    return node.tagname == 'pending_xref' and node['reftype'] == 'setting'
+def is_setting_index(node):
+    if node.tagname == 'index':
+        # index entries for setting directives look like:
+        # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]
+        entry_type, info, refid, _ = node['entries'][0]
+        return entry_type == 'pair' and info.endswith('; setting')
+    return False
+
+
+def get_setting_target(node):
+    # target nodes are placed next to the node in the doc tree
+    return node.parent[node.parent.index(node) + 1]
+
+
+def get_setting_name_and_refid(node):
+    """Extract setting name from directive index node"""
+    entry_type, info, refid, _ = node['entries'][0]
+    return info.replace('; setting', ''), refid
 
 
 def collect_scrapy_settings_refs(app, doctree):
@@ -23,29 +39,24 @@ def collect_scrapy_settings_refs(app, doctree):
     if not hasattr(env, 'scrapy_all_settings'):
         env.scrapy_all_settings = []
 
-    for node in doctree.traverse(is_setting_node):
-        try:
-            targetnode = node.parent[node.parent.index(node) - 1]
-            if not isinstance(targetnode, nodes.target):
-                raise IndexError
-        except IndexError:
-            targetid = "setting-%d" % env.new_serialno('setting')
-            targetnode = nodes.target('', '', ids=[targetid])
-            node.replace_self([targetnode, node])
+    for node in doctree.traverse(is_setting_index):
+        targetnode = get_setting_target(node)
+        assert isinstance(targetnode, nodes.target), "Next node is not a target"
+
+        setting_name, refid = get_setting_name_and_refid(node)
 
         env.scrapy_all_settings.append({
             'docname': env.docname,
-            'lineno': node.line,
-            'node': node.deepcopy(),
-            'target': targetnode,
+            'setting_name': setting_name,
+            'refid': refid,
         })
 
 
 def make_setting_element(setting_data, app, fromdocname):
-    text = nodes.Text(setting_data['node'].astext())
-    targetid = ''  # TODO: resolve to a proper id
     refnode = make_refnode(app.builder, fromdocname,
-                           setting_data['docname'], targetid, text)
+                           todocname=setting_data['docname'],
+                           targetid=setting_data['refid'],
+                           child=nodes.Text(setting_data['setting_name']))
 
     p = nodes.paragraph()
     p.append(refnode)
@@ -57,7 +68,8 @@ def replace_settingslist_nodes(app, doctree, fromdocname):
 
     for node in doctree.traverse(settingslist_node):
         node.replace_self([make_setting_element(d, app, fromdocname)
-                           for d in env.scrapy_all_settings])
+                           for d in env.scrapy_all_settings if fromdocname != d['docname']])
+
 
 def setup(app):
     app.add_crossref_type(
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 1cf2940ff..6cc2c0c60 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -133,8 +133,6 @@ to any particular component. In that case the module of that component will be
 shown, typically an extension, middleware or pipeline. It also means that the
 component must be enabled in order for the setting to have any effect.
 
-.. settingslist::
-
 .. setting:: AWS_ACCESS_KEY_ID
 
 AWS_ACCESS_KEY_ID
@@ -1017,6 +1015,16 @@ Default: ``"Scrapy/VERSION (+http://scrapy.org)"``
 
 The default User-Agent to use when crawling, unless overridden.
 
+
+Settings documented elsewhere:
+------------------------------
+
+The following settings are documented elsewhere, please check each specific
+case to see how to enable and use them.
+
+.. settingslist::
+
+
 .. _Amazon web services: http://aws.amazon.com/
 .. _breadth-first order: http://en.wikipedia.org/wiki/Breadth-first_search
 .. _depth-first order: http://en.wikipedia.org/wiki/Depth-first_search

From 6884b73e19050c1a6d11228886db51e1787df2bb Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Mon, 11 May 2015 22:11:30 -0300
Subject: [PATCH 0310/2974] sort settings list by name and wrap it in a bullet
 list

---
 docs/_ext/scrapydocs.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index 58d9cd0c7..f0827f2b1 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -2,6 +2,7 @@ from docutils.parsers.rst.roles import set_classes
 from docutils import nodes
 from sphinx.util.compat import Directive
 from sphinx.util.nodes import make_refnode
+from operator import itemgetter
 
 
 class settingslist_node(nodes.General, nodes.Element):
@@ -57,18 +58,24 @@ def make_setting_element(setting_data, app, fromdocname):
                            todocname=setting_data['docname'],
                            targetid=setting_data['refid'],
                            child=nodes.Text(setting_data['setting_name']))
-
     p = nodes.paragraph()
-    p.append(refnode)
-    return p
+    p += refnode
+
+    item = nodes.list_item()
+    item += p
+    return item
 
 
 def replace_settingslist_nodes(app, doctree, fromdocname):
     env = app.builder.env
 
     for node in doctree.traverse(settingslist_node):
-        node.replace_self([make_setting_element(d, app, fromdocname)
-                           for d in env.scrapy_all_settings if fromdocname != d['docname']])
+        settings_list = nodes.bullet_list()
+        settings_list.extend([make_setting_element(d, app, fromdocname)
+                              for d in sorted(env.scrapy_all_settings,
+                                              key=itemgetter('setting_name'))
+                              if fromdocname != d['docname']])
+        node.replace_self(settings_list)
 
 
 def setup(app):

From 773ea5a5ef76426dd91a8669542d2602082a5746 Mon Sep 17 00:00:00 2001
From: nramirezuy <nramirez.uy@gmail.com>
Date: Wed, 13 May 2015 13:27:42 -0300
Subject: [PATCH 0311/2974] item fields defined on fields class attribute fix

---
 scrapy/item.py     |  3 ++-
 tests/test_item.py | 36 +++++++++++++++++++++++++++++-------
 2 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/scrapy/item.py b/scrapy/item.py
index 8ac279642..138728a9a 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/item.rst
 
 from pprint import pformat
 from collections import MutableMapping
+
 from abc import ABCMeta
 import six
 
@@ -27,7 +28,7 @@ class ItemMeta(ABCMeta):
         new_bases = tuple(base._class for base in bases if hasattr(base, '_class'))
         _class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs)
 
-        fields = {}
+        fields = getattr(_class, 'fields', {})
         new_attrs = {}
         for n in dir(_class):
             v = getattr(_class, n)
diff --git a/tests/test_item.py b/tests/test_item.py
index 76061bbd9..dcb169c3a 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -120,6 +120,14 @@ class ItemTest(unittest.TestCase):
         self.assertSortedEqual(list(i.keys()), ['keys', 'values', 'name'])
         self.assertSortedEqual(list(i.values()), [u'Keys', u'Values', u'John'])
 
+    def test_metaclass_with_fields_attribute(self):
+        class TestItem(Item):
+            fields = {'new': Field(default='X')}
+
+        item = TestItem(new=u'New')
+        self.assertSortedEqual(list(item.keys()), ['new'])
+        self.assertSortedEqual(list(item.values()), [u'New'])
+
     def test_metaclass_inheritance(self):
         class BaseItem(Item):
             name = Field()
@@ -136,41 +144,51 @@ class ItemTest(unittest.TestCase):
 
     def test_metaclass_multiple_inheritance_simple(self):
         class A(Item):
+            fields = {'load': Field(default='A')}
             save = Field(default='A')
 
         class B(A): pass
 
         class C(Item):
+            fields = {'load': Field(default='C')}
             save = Field(default='C')
 
         class D(B, C): pass
 
-        self.assertEqual(D(save='X')['save'], 'X')
-        self.assertEqual(D.fields, {'save': {'default': 'A'}})
+        item = D(save='X', load='Y')
+        self.assertEqual(item['save'], 'X')
+        self.assertEqual(item['load'], 'Y')
+        self.assertEqual(D.fields, {'load': {'default': 'A'},
+            'save': {'default': 'A'}})
 
         # D class inverted
         class E(C, B): pass
 
         self.assertEqual(E(save='X')['save'], 'X')
-        self.assertEqual(E.fields, {'save': {'default': 'C'}})
+        self.assertEqual(E(load='X')['load'], 'X')
+        self.assertEqual(E.fields, {'load': {'default': 'C'},
+            'save': {'default': 'C'}})
 
     def test_metaclass_multiple_inheritance_diamond(self):
         class A(Item):
+            fields = {'update': Field(default='A')}
             save = Field(default='A')
             load = Field(default='A')
 
         class B(A): pass
 
         class C(A):
+            fields = {'update': Field(default='C')}
             save = Field(default='C')
 
         class D(B, C):
+            fields = {'update': Field(default='D')}
             load = Field(default='D')
 
         self.assertEqual(D(save='X')['save'], 'X')
         self.assertEqual(D(load='X')['load'], 'X')
         self.assertEqual(D.fields, {'save': {'default': 'C'},
-            'load': {'default': 'D'}})
+            'load': {'default': 'D'}, 'update': {'default': 'D'}})
 
         # D class inverted
         class E(C, B):
@@ -179,15 +197,17 @@ class ItemTest(unittest.TestCase):
         self.assertEqual(E(save='X')['save'], 'X')
         self.assertEqual(E(load='X')['load'], 'X')
         self.assertEqual(E.fields, {'save': {'default': 'C'},
-            'load': {'default': 'E'}})
+            'load': {'default': 'E'}, 'update': {'default': 'C'}})
 
     def test_metaclass_multiple_inheritance_without_metaclass(self):
         class A(Item):
+            fields = {'load': Field(default='A')}
             save = Field(default='A')
 
         class B(A): pass
 
         class C(object):
+            fields = {'load': Field(default='C')}
             not_allowed = Field(default='not_allowed')
             save = Field(default='C')
 
@@ -195,14 +215,16 @@ class ItemTest(unittest.TestCase):
 
         self.assertRaises(KeyError, D, not_allowed='value')
         self.assertEqual(D(save='X')['save'], 'X')
-        self.assertEqual(D.fields, {'save': {'default': 'A'}})
+        self.assertEqual(D.fields, {'save': {'default': 'A'},
+            'load': {'default': 'A'}})
 
         # D class inverted
         class E(C, B): pass
 
         self.assertRaises(KeyError, E, not_allowed='value')
         self.assertEqual(E(save='X')['save'], 'X')
-        self.assertEqual(E.fields, {'save': {'default': 'A'}})
+        self.assertEqual(E.fields, {'save': {'default': 'A'},
+            'load': {'default': 'A'}})
 
     def test_to_dict(self):
         class TestItem(Item):

From 7c61bd897cf383b88f3b2891063a82fd4a731161 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 14 May 2015 20:09:19 -0300
Subject: [PATCH 0312/2974] Fix in docs for error introduced in #1218

---
 docs/topics/link-extractors.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index f9b25ae63..8253e7650 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -79,7 +79,7 @@ LxmlLinkExtractor
         extensions that should be ignored when extracting links.
         If not given, it will default to the
         ``IGNORED_EXTENSIONS`` list defined in the
-        `scrapy.linkextractors`_ module.
+        `scrapy.linkextractors`_ package.
     :type deny_extensions: list
 
     :param restrict_xpaths: is an XPath (or list of XPath's) which defines

From a555778013d7f4520da31f2f544e94434dd4e0f8 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 14 May 2015 20:40:12 -0300
Subject: [PATCH 0313/2974] Build docs in a temporary folder to avoid caching

---
 tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index a5b9ad9e3..f6c153ae3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -62,5 +62,5 @@ changedir = docs
 deps =
     Sphinx
 commands =
-    sphinx-build -W -b html . build/html
-    sphinx-build -W -b linkcheck . build/linkcheck
+    sphinx-build -W -b html . {envtmpdir}/html
+    sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck

From ec5494292525d57f3077c122eed6a4a4cf4a0c29 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 14 May 2015 20:41:55 -0300
Subject: [PATCH 0314/2974] Create separate testenvs to build docs and check
 links

---
 tox.ini | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index f6c153ae3..5c8c8c78d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -57,10 +57,19 @@ deps =
 basepython = python3.4
 deps = {[testenv:py33]deps}
 
-[testenv:docs]
+[docs]
 changedir = docs
 deps =
     Sphinx
+
+[testenv:docs]
+changedir = {[docs]changedir}
+deps = {[docs]deps}
 commands =
     sphinx-build -W -b html . {envtmpdir}/html
+
+[testenv:docs-links]
+changedir = {[docs]changedir}
+deps = {[docs]deps}
+commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck

From cbebcbddb147cdf46dd10f8221abc31493a0c9ca Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 14 May 2015 20:42:34 -0300
Subject: [PATCH 0315/2974] Add docs toxenv to .travis.yml

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index b30d13bed..09da120bc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,6 +4,7 @@ env:
 - TOXENV=py27
 - TOXENV=precise
 - TOXENV=py33
+- TOXENV=docs
 install:
 - "./.travis-workarounds.sh"
 - pip install -U tox

From 157cec7f64f3eab47814e3865ccfc867cb0e0bd0 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 15 May 2015 00:56:47 -0300
Subject: [PATCH 0316/2974] Replace logger.exception calls for Python
 compatibility

---
 scrapy/core/engine.py     | 4 ++--
 scrapy/core/scheduler.py  | 6 +++---
 scrapy/pipelines/files.py | 4 ++--
 scrapy/utils/signal.py    | 5 +++--
 tests/test_utils_log.py   | 2 +-
 5 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 63267f208..40f19e4c3 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -112,8 +112,8 @@ class ExecutionEngine(object):
                 slot.start_requests = None
             except Exception:
                 slot.start_requests = None
-                logger.exception('Error while obtaining start requests',
-                                 extra={'spider': spider})
+                logger.error('Error while obtaining start requests',
+                             exc_info=True, extra={'spider': spider})
             else:
                 self.crawl(request, spider)
 
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index f199711ab..d132bbfd3 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -83,9 +83,9 @@ class Scheduler(object):
             self.dqs.push(reqd, -request.priority)
         except ValueError as e: # non serializable request
             if self.logunser:
-                logger.exception("Unable to serialize request: %(request)s - reason: %(reason)s",
-                                 {'request': request, 'reason': e},
-                                 extra={'spider': self.spider})
+                logger.error("Unable to serialize request: %(request)s - reason: %(reason)s",
+                             {'request': request, 'reason': e},
+                             exc_info=True, extra={'spider': self.spider})
             return
         else:
             return True
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 2b17e0f0a..c0192b867 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -272,11 +272,11 @@ class FilesPipeline(MediaPipeline):
             )
             raise
         except Exception as exc:
-            logger.exception(
+            logger.error(
                 'File (unknown-error): Error processing file from %(request)s '
                 'referred in <%(referer)s>',
                 {'request': request, 'referer': referer},
-                extra={'spider': info.spider}
+                exc_info=True, extra={'spider': info.spider}
             )
             raise FileException(str(exc))
 
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index cab172e4c..d4cc41305 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -30,8 +30,9 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
             result = Failure()
         except Exception:
             result = Failure()
-            logger.exception("Error caught on signal handler: %(receiver)s",
-                             {'receiver': receiver}, extra={'spider': spider})
+            logger.error("Error caught on signal handler: %(receiver)s",
+                         {'receiver': receiver},
+                         exc_info=True, extra={'spider': spider})
         else:
             result = response
         responses.append((receiver, result))
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index e6b8c77d9..7448dbfc0 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -28,7 +28,7 @@ class FailureFormatterTest(unittest.TestCase):
             try:
                 0/0
             except ZeroDivisionError:
-                self.logger.exception('test log msg')
+                self.logger.error('test log msg', exc_info=True)
                 failure = Failure()
 
             self.logger.error('test log msg', extra={'failure': failure})

From 6dccb3a9b320a8d0808764ac8e6e88e663e2d52c Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 15 May 2015 04:56:09 -0300
Subject: [PATCH 0317/2974] Replace FailureFormatter with direct exc_info
 conversions in log calls

---
 scrapy/core/engine.py                     | 20 +++++++---
 scrapy/core/scraper.py                    | 14 ++++---
 scrapy/downloadermiddlewares/robotstxt.py |  4 +-
 scrapy/extensions/feedexport.py           |  4 +-
 scrapy/log.py                             |  3 +-
 scrapy/pipelines/files.py                 |  4 +-
 scrapy/pipelines/media.py                 |  6 ++-
 scrapy/utils/log.py                       | 27 ++------------
 scrapy/utils/signal.py                    |  4 +-
 tests/test_pipeline_media.py              |  3 +-
 tests/test_utils_log.py                   | 45 ++++++-----------------
 11 files changed, 58 insertions(+), 76 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 40f19e4c3..992327bfe 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -16,7 +16,7 @@ from scrapy.exceptions import DontCloseSpider
 from scrapy.http import Response, Request
 from scrapy.utils.misc import load_object
 from scrapy.utils.reactor import CallLaterOnce
-from scrapy.utils.log import logformatter_adapter
+from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
 
 logger = logging.getLogger(__name__)
 
@@ -135,13 +135,16 @@ class ExecutionEngine(object):
         d = self._download(request, spider)
         d.addBoth(self._handle_downloader_output, request, spider)
         d.addErrback(lambda f: logger.info('Error while handling downloader output',
-                                           extra={'spider': spider, 'failure': f}))
+                                           exc_info=failure_to_exc_info(f),
+                                           extra={'spider': spider}))
         d.addBoth(lambda _: slot.remove_request(request))
         d.addErrback(lambda f: logger.info('Error while removing request from slot',
-                                           extra={'spider': spider, 'failure': f}))
+                                           exc_info=failure_to_exc_info(f),
+                                           extra={'spider': spider}))
         d.addBoth(lambda _: slot.nextcall.schedule())
         d.addErrback(lambda f: logger.info('Error while scheduling new request',
-                                           extra={'spider': spider, 'failure': f}))
+                                           exc_info=failure_to_exc_info(f),
+                                           extra={'spider': spider}))
         return d
 
     def _handle_downloader_output(self, response, request, spider):
@@ -153,7 +156,8 @@ class ExecutionEngine(object):
         # response is a Response or Failure
         d = self.scraper.enqueue_scrape(response, request, spider)
         d.addErrback(lambda f: logger.error('Error while enqueuing downloader output',
-                                            extra={'spider': spider, 'failure': f}))
+                                            exc_info=failure_to_exc_info(f),
+                                            extra={'spider': spider}))
         return d
 
     def spider_is_idle(self, spider):
@@ -268,7 +272,11 @@ class ExecutionEngine(object):
 
         def log_failure(msg):
             def errback(failure):
-                logger.error(msg, extra={'spider': spider, 'failure': failure})
+                logger.error(
+                    msg,
+                    exc_info=failure_to_exc_info(failure),
+                    extra={'spider': spider}
+                )
             return errback
 
         dfd.addBoth(lambda _: self.downloader.close())
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index e5d8acea2..244499be2 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -10,7 +10,7 @@ from twisted.internet import defer
 from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.misc import load_object
-from scrapy.utils.log import logformatter_adapter
+from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
 from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy import signals
 from scrapy.http import Request, Response
@@ -107,7 +107,8 @@ class Scraper(object):
         dfd.addErrback(
             lambda f: logger.error('Scraper bug processing %(request)s',
                                    {'request': request},
-                                   extra={'spider': spider, 'failure': f}))
+                                   exc_info=failure_to_exc_info(f),
+                                   extra={'spider': spider}))
         self._scrape_next(spider, slot)
         return dfd
 
@@ -153,7 +154,8 @@ class Scraper(object):
         logger.error(
             "Spider error processing %(request)s (referer: %(referer)s)",
             {'request': request, 'referer': referer},
-            extra={'spider': spider, 'failure': _failure}
+            exc_info=failure_to_exc_info(_failure),
+            extra={'spider': spider}
         )
         self.signals.send_catch_log(
             signal=signals.spider_error,
@@ -202,7 +204,8 @@ class Scraper(object):
             if download_failure.frames:
                 logger.error('Error downloading %(request)s',
                              {'request': request},
-                             extra={'spider': spider, 'failure': download_failure})
+                             exc_info=failure_to_exc_info(download_failure),
+                             extra={'spider': spider})
             else:
                 errmsg = download_failure.getErrorMessage()
                 if errmsg:
@@ -227,7 +230,8 @@ class Scraper(object):
                     spider=spider, exception=output.value)
             else:
                 logger.error('Error processing %(item)s', {'item': item},
-                             extra={'spider': spider, 'failure': output})
+                             exc_info=failure_to_exc_info(output),
+                             extra={'spider': spider})
         else:
             logkws = self.logformatter.scraped(output, response, spider)
             logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 9083482f0..77e08b7e0 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -11,6 +11,7 @@ from six.moves.urllib import robotparser
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.log import failure_to_exc_info
 
 logger = logging.getLogger(__name__)
 
@@ -59,7 +60,8 @@ class RobotsTxtMiddleware(object):
         if failure.type is not IgnoreRequest:
             logger.error("Error downloading %(request)s: %(f_exception)s",
                          {'request': request, 'f_exception': failure.value},
-                         extra={'spider': spider, 'failure': failure})
+                         exc_info=failure_to_exc_info(failure),
+                         extra={'spider': spider})
 
     def _parse_robots(self, response):
         rp = robotparser.RobotFileParser(response.url)
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 7c6849a73..3bc1c92cc 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -22,6 +22,7 @@ from scrapy.utils.ftp import ftp_makedirs_cwd
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import get_func_args
+from scrapy.utils.log import failure_to_exc_info
 
 logger = logging.getLogger(__name__)
 
@@ -184,7 +185,8 @@ class FeedExporter(object):
         d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
                                             extra={'spider': spider}))
         d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
-                                            extra={'spider': spider, 'failure': f}))
+                                            exc_info=failure_to_exc_info(f),
+                                            extra={'spider': spider}))
         return d
 
     def item_scraped(self, item, spider):
diff --git a/scrapy/log.py b/scrapy/log.py
index c3f9c4227..e1c68aa93 100644
--- a/scrapy/log.py
+++ b/scrapy/log.py
@@ -8,6 +8,7 @@ import warnings
 from twisted.python.failure import Failure
 
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.log import failure_to_exc_info
 
 logger = logging.getLogger(__name__)
 
@@ -48,4 +49,4 @@ def err(_stuff=None, _why=None, **kw):
     level = kw.pop('level', logging.ERROR)
     failure = kw.pop('failure', _stuff) or Failure()
     message = kw.pop('why', _why) or failure.value
-    logger.log(level, message, *[kw] if kw else [], extra={'failure': failure})
+    logger.log(level, message, *[kw] if kw else [], exc_info=failure_to_exc_info(failure))
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index c0192b867..250f46ad8 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -25,6 +25,7 @@ from scrapy.pipelines.media import MediaPipeline
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.misc import md5sum
+from scrapy.utils.log import failure_to_exc_info
 
 logger = logging.getLogger(__name__)
 
@@ -212,7 +213,8 @@ class FilesPipeline(MediaPipeline):
         dfd.addErrback(
             lambda f:
             logger.error(self.__class__.__name__ + '.store.stat_file',
-                         extra={'spider': info.spider, 'failure': f})
+                         exc_info=failure_to_exc_info(f),
+                         extra={'spider': info.spider})
         )
         return dfd
 
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 55ef05ad4..21b8b8986 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -8,6 +8,7 @@ from twisted.python.failure import Failure
 from scrapy.utils.defer import mustbe_deferred, defer_result
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import arg_to_iter
+from scrapy.utils.log import failure_to_exc_info
 
 logger = logging.getLogger(__name__)
 
@@ -70,7 +71,7 @@ class MediaPipeline(object):
         dfd.addCallback(self._check_media_to_download, request, info)
         dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
         dfd.addErrback(lambda f: logger.error(
-            f.value, extra={'spider': info.spider, 'failure': f})
+            f.value, exc_info=failure_to_exc_info(f), extra={'spider': info.spider})
         )
         return dfd.addBoth(lambda _: wad)  # it must return wad at last
 
@@ -127,6 +128,7 @@ class MediaPipeline(object):
                     logger.error(
                         '%(class)s found errors processing %(item)s',
                         {'class': self.__class__.__name__, 'item': item},
-                        extra={'spider': info.spider, 'failure': value}
+                        exc_info=failure_to_exc_info(value),
+                        extra={'spider': info.spider}
                     )
         return item
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 23b246491..5dabe5697 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 
-import os
 import sys
 import logging
 import warnings
@@ -16,22 +15,10 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 logger = logging.getLogger(__name__)
 
 
-class FailureFormatter(logging.Filter):
-    """Extract exc_info from Failure instances provided as contextual data
-
-    This filter mimics Twisted log.err formatting for its first `_stuff`
-    argument, which means that reprs of non Failure objects are appended to the
-    log messages.
-    """
-
-    def filter(self, record):
-        failure = record.__dict__.get('failure')
-        if failure:
-            if isinstance(failure, Failure):
-                record.exc_info = (failure.type, failure.value, failure.tb)
-            else:
-                record.msg += os.linesep + repr(failure)
-        return True
+def failure_to_exc_info(failure):
+    """Extract exc_info from Failure instances"""
+    if isinstance(failure, Failure):
+        return (failure.type, failure.value, failure.tb)
 
 
 class TopLevelFormatter(logging.Filter):
@@ -58,15 +45,9 @@ class TopLevelFormatter(logging.Filter):
 DEFAULT_LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
-    'filters': {
-        'failure_formatter': {
-            '()': 'scrapy.utils.log.FailureFormatter',
-        },
-    },
     'loggers': {
         'scrapy': {
             'level': 'DEBUG',
-            'filters': ['failure_formatter'],
         },
         'twisted': {
             'level': 'ERROR',
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index d4cc41305..d9a59e161 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -8,6 +8,7 @@ from twisted.python.failure import Failure
 from scrapy.xlib.pydispatch.dispatcher import Any, Anonymous, liveReceivers, \
     getAllReceivers, disconnect
 from scrapy.xlib.pydispatch.robustapply import robustApply
+from scrapy.utils.log import failure_to_exc_info
 
 logger = logging.getLogger(__name__)
 
@@ -47,7 +48,8 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
         if dont_log is None or not isinstance(failure.value, dont_log):
             logger.error("Error caught on signal handler: %(receiver)s",
                          {'receiver': recv},
-                         extra={'spider': spider, 'failure': failure})
+                         exc_info=failure_to_exc_info(failure),
+                         extra={'spider': spider})
         return failure
 
     dont_log = named.pop('dont_log', None)
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 7217eee90..fd8b28ce1 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -9,6 +9,7 @@ from scrapy.http import Request, Response
 from scrapy.spiders import Spider
 from scrapy.utils.request import request_fingerprint
 from scrapy.pipelines.media import MediaPipeline
+from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.signal import disconnect_all
 from scrapy import signals
 
@@ -66,7 +67,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         assert len(l.records) == 1
         record = l.records[0]
         assert record.levelname == 'ERROR'
-        assert record.failure is fail
+        self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail))
 
         # disable failure logging and check again
         self.pipe.LOG_FAILED_RESULTS = False
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 7448dbfc0..45527b03b 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 from __future__ import print_function
-import os
 import sys
 import logging
 import unittest
@@ -8,46 +7,24 @@ import unittest
 from testfixtures import LogCapture
 from twisted.python.failure import Failure
 
-from scrapy.utils.log import (FailureFormatter, TopLevelFormatter,
+from scrapy.utils.log import (failure_to_exc_info, TopLevelFormatter,
                               LogCounterHandler, StreamLogger)
 from scrapy.utils.test import get_crawler
 
 
-class FailureFormatterTest(unittest.TestCase):
+class FailureToExcInfoTest(unittest.TestCase):
 
-    def setUp(self):
-        self.logger = logging.getLogger('test')
-        self.filter = FailureFormatter()
-        self.logger.addFilter(self.filter)
+    def test_failure(self):
+        try:
+            0/0
+        except ZeroDivisionError:
+            exc_info = sys.exc_info()
+            failure = Failure()
 
-    def tearDown(self):
-        self.logger.removeFilter(self.filter)
+        self.assertTupleEqual(exc_info, failure_to_exc_info(failure))
 
-    def test_failure_format(self):
-        with LogCapture() as l:
-            try:
-                0/0
-            except ZeroDivisionError:
-                self.logger.error('test log msg', exc_info=True)
-                failure = Failure()
-
-            self.logger.error('test log msg', extra={'failure': failure})
-
-        self.assertEqual(len(l.records), 2)
-        exc_record, failure_record = l.records
-        self.assertTupleEqual(failure_record.exc_info, exc_record.exc_info)
-
-        formatter = logging.Formatter()
-        self.assertMultiLineEqual(formatter.format(failure_record),
-                                  formatter.format(exc_record))
-
-    def test_non_failure_format(self):
-        with LogCapture() as l:
-            self.logger.error('test log msg', extra={'failure': 3})
-
-        self.assertEqual(len(l.records), 1)
-        self.assertMultiLineEqual(l.records[0].getMessage(),
-                                  'test log msg' + os.linesep + '3')
+    def test_non_failure(self):
+        self.assertIsNone(failure_to_exc_info('test'))
 
 
 class TopLevelFormatterTest(unittest.TestCase):

From e7574a809e382120f424de18796d7d2e61e71625 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 15 May 2015 14:51:14 -0300
Subject: [PATCH 0318/2974] Use method to get traceback of failures in
 failure_to_exc_info

---
 scrapy/utils/log.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 5dabe5697..931e28f2d 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
 def failure_to_exc_info(failure):
     """Extract exc_info from Failure instances"""
     if isinstance(failure, Failure):
-        return (failure.type, failure.value, failure.tb)
+        return (failure.type, failure.value, failure.getTracebackObject())
 
 
 class TopLevelFormatter(logging.Filter):

From 8d9e3b7e8da201223b353f43789efc7088817c32 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 12 May 2015 18:27:57 +0500
Subject: [PATCH 0319/2974] TST add a test for JSON lines item exporter

---
 tests/test_feedexport.py | 41 ++++++++++++++++++++++++++--------------
 1 file changed, 27 insertions(+), 14 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 41913e401..44e0c1e23 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 import os
 import csv
+import json
 from io import BytesIO
 import tempfile
 import shutil
@@ -126,7 +127,6 @@ class FeedExportTest(unittest.TestCase):
         egg = scrapy.Field()
         baz = scrapy.Field()
 
-
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings=None):
         """ Run spider with specified settings; return exported data. """
@@ -180,7 +180,20 @@ class FeedExportTest(unittest.TestCase):
         self.assertEqual(rows, got_rows)
 
     @defer.inlineCallbacks
-    def test_export_csv_items(self):
+    def assertExportedJsonLines(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({'FEED_FORMAT': 'jl'})
+        data = yield self.exported_data(items, settings)
+        parsed = [json.loads(line) for line in data.splitlines()]
+        self.assertEqual(rows, parsed)
+
+    @defer.inlineCallbacks
+    def assertExported(self, items, header, rows, settings=None, ordered=True):
+        yield self.assertExportedCsv(items, header, rows, settings, ordered)
+        yield self.assertExportedJsonLines(items, rows, settings)
+
+    @defer.inlineCallbacks
+    def test_export_items(self):
         # feed exporters use field names from Item
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
@@ -191,10 +204,10 @@ class FeedExportTest(unittest.TestCase):
             {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
         ]
         header = self.MyItem.fields.keys()
-        yield self.assertExportedCsv(items, header, rows, ordered=False)
+        yield self.assertExported(items, header, rows, ordered=False)
 
     @defer.inlineCallbacks
-    def test_export_csv_multiple_item_classes(self):
+    def test_export_multiple_item_classes(self):
 
         class MyItem2(scrapy.Item):
             foo = scrapy.Field()
@@ -215,7 +228,7 @@ class FeedExportTest(unittest.TestCase):
             {'egg': 'spam3', 'foo': 'bar3', 'baz': 'quux3'},
             {'egg': 'spam4', 'foo': '',     'baz': ''},
         ]
-        yield self.assertExportedCsv(items, header, rows, ordered=False)
+        yield self.assertExported(items, header, rows, ordered=False)
 
         # but it is possible to override fields using FEED_EXPORT_FIELDS
         header = ["foo", "baz", "hello"]
@@ -226,11 +239,11 @@ class FeedExportTest(unittest.TestCase):
             {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
             {'foo': '',     'baz': '',      'hello': 'world4'},
         ]
-        yield self.assertExportedCsv(items, header, rows,
-                                     settings=settings, ordered=True)
+        yield self.assertExported(items, header, rows,
+                                  settings=settings, ordered=True)
 
     @defer.inlineCallbacks
-    def test_export_csv_dicts(self):
+    def test_export_dicts(self):
         # When dicts are used, only keys from the first row are used as
         # a header.
         items = [
@@ -241,10 +254,10 @@ class FeedExportTest(unittest.TestCase):
             {'egg': 'spam', 'foo': 'bar'},
             {'egg': 'spam', 'foo': 'bar'}
         ]
-        yield self.assertExportedCsv(items, ['egg', 'foo'], rows, ordered=False)
+        yield self.assertExported(items, ['egg', 'foo'], rows, ordered=False)
 
     @defer.inlineCallbacks
-    def test_export_csv_feed_export_fields(self):
+    def test_export_feed_export_fields(self):
         # FEED_EXPORT_FIELDS option allows to order export fields
         # and to select a subset of fields to export, both for Items and dicts.
 
@@ -260,8 +273,8 @@ class FeedExportTest(unittest.TestCase):
                 {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
                 {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
             ]
-            yield self.assertExportedCsv(items, ['foo', 'baz', 'egg'], rows,
-                                         settings=settings, ordered=True)
+            yield self.assertExported(items, ['foo', 'baz', 'egg'], rows,
+                                      settings=settings, ordered=True)
 
             # export a subset of columns
             settings = {'FEED_EXPORT_FIELDS': 'egg,baz'}
@@ -269,5 +282,5 @@ class FeedExportTest(unittest.TestCase):
                 {'egg': 'spam1', 'baz': ''},
                 {'egg': 'spam2', 'baz': 'quux2'}
             ]
-            yield self.assertExportedCsv(items, ['egg', 'baz'], rows,
-                                         settings=settings, ordered=True)
+            yield self.assertExported(items, ['egg', 'baz'], rows,
+                                      settings=settings, ordered=True)

From e1efd1917564c654627db0d3ae39c2d1f7be3e3c Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 12 May 2015 18:44:23 +0500
Subject: [PATCH 0320/2974] TST, DOC document that Scrapy only infers field
 names for CSV

---
 docs/topics/feed-exports.rst |  9 ++++++---
 tests/test_feedexport.py     | 18 ++++++++++++------
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index faf9abc1c..ea4cc2162 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -241,9 +241,12 @@ Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
 
 Use FEED_EXPORT_FIELDS option to define fields to export and their order.
 
-When omitted, Scrapy uses fields defined in :class:`~.Item` subclasses a spider
-is yielding. If raw dicts are used as items Scrapy tries to infer field names
-from the exported data - currently it uses field names from the first item.
+When omitted or empty, Scrapy uses fields defined in :class:`~.Item` subclasses
+a spider is yielding. If raw dicts are used as items, FEED_EXPORT_FIELDS
+is omitted and an exporter requires a fixed set of fields (this is the case
+for :ref:`CSV <topics-feed-format-csv>` export format) then Scrapy tries
+to infer field names from the exported data - currently it uses field names
+from the first item.
 
 .. setting:: FEED_STORE_EMPTY
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 44e0c1e23..efee9bf94 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -185,6 +185,7 @@ class FeedExportTest(unittest.TestCase):
         settings.update({'FEED_FORMAT': 'jl'})
         data = yield self.exported_data(items, settings)
         parsed = [json.loads(line) for line in data.splitlines()]
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
         self.assertEqual(rows, parsed)
 
     @defer.inlineCallbacks
@@ -220,15 +221,18 @@ class FeedExportTest(unittest.TestCase):
             {'hello': 'world4', 'egg': 'spam4'},
         ]
 
-        # by default, Scrapy uses fields of the first Item
+        # by default, Scrapy uses fields of the first Item for CSV and
+        # all fields for JSON Lines
         header = self.MyItem.fields.keys()
-        rows = [
+        rows_csv = [
             {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
             {'egg': '',      'foo': 'bar2', 'baz': ''},
             {'egg': 'spam3', 'foo': 'bar3', 'baz': 'quux3'},
             {'egg': 'spam4', 'foo': '',     'baz': ''},
         ]
-        yield self.assertExported(items, header, rows, ordered=False)
+        rows_jl = [dict(row) for row in items]
+        yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
+        yield self.assertExportedJsonLines(items, rows_jl)
 
         # but it is possible to override fields using FEED_EXPORT_FIELDS
         header = ["foo", "baz", "hello"]
@@ -245,16 +249,18 @@ class FeedExportTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_export_dicts(self):
         # When dicts are used, only keys from the first row are used as
-        # a header.
+        # a header for CSV, and all fields are used for JSON Lines.
         items = [
             {'foo': 'bar', 'egg': 'spam'},
             {'foo': 'bar', 'egg': 'spam', 'baz': 'quux'},
         ]
-        rows = [
+        rows_csv = [
             {'egg': 'spam', 'foo': 'bar'},
             {'egg': 'spam', 'foo': 'bar'}
         ]
-        yield self.assertExported(items, ['egg', 'foo'], rows, ordered=False)
+        rows_jl = items
+        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False)
+        yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
     def test_export_feed_export_fields(self):

From c41203486452203968e1879fba4d8760646ee86d Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 12 May 2015 18:45:08 +0500
Subject: [PATCH 0321/2974] Fix GH-1223

---
 scrapy/extensions/feedexport.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 3bc1c92cc..7a1a02be1 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -21,7 +21,6 @@ from scrapy import signals
 from scrapy.utils.ftp import ftp_makedirs_cwd
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
-from scrapy.utils.python import get_func_args
 from scrapy.utils.log import failure_to_exc_info
 
 logger = logging.getLogger(__name__)
@@ -152,7 +151,7 @@ class FeedExporter(object):
         if not self._exporter_supported(self.format):
             raise NotConfigured
         self.store_empty = settings.getbool('FEED_STORE_EMPTY')
-        self.export_fields = settings.getlist('FEED_EXPORT_FIELDS')
+        self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
         uripar = settings['FEED_URI_PARAMS']
         self._uripar = load_object(uripar) if uripar else lambda x, y: None
 

From 9fb318338bb499ff899ee8d1b096cc9eceb5f6cc Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 13 May 2015 00:33:48 +0500
Subject: [PATCH 0322/2974] support FEED_EXPORT_FIELDS=[]

---
 docs/topics/feed-exports.rst    | 15 +++++++++------
 scrapy/extensions/feedexport.py |  7 ++++++-
 tests/test_feedexport.py        |  4 ++++
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index ea4cc2162..f4fb07761 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -236,17 +236,20 @@ The serialization format to be used for the feed. See
 FEED_EXPORT_FIELDS
 ------------------
 
+Default: ``None``
+
 A list of fields to export, optional.
 Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
 
 Use FEED_EXPORT_FIELDS option to define fields to export and their order.
 
-When omitted or empty, Scrapy uses fields defined in :class:`~.Item` subclasses
-a spider is yielding. If raw dicts are used as items, FEED_EXPORT_FIELDS
-is omitted and an exporter requires a fixed set of fields (this is the case
-for :ref:`CSV <topics-feed-format-csv>` export format) then Scrapy tries
-to infer field names from the exported data - currently it uses field names
-from the first item.
+When FEED_EXPORT_FIELDS is None (default), Scrapy uses fields
+defined in :class:`~.Item` subclasses a spider is yielding.
+If FEED_EXPORT_FIELDS is None, raw dicts are used as items and
+an exporter requires a fixed set of fields (this is the case for
+:ref:`CSV <topics-feed-format-csv>` export format), then
+Scrapy tries to infer field names from the exported data - currently it
+uses field names from the first item.
 
 .. setting:: FEED_STORE_EMPTY
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 7a1a02be1..08e868c9f 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -151,7 +151,12 @@ class FeedExporter(object):
         if not self._exporter_supported(self.format):
             raise NotConfigured
         self.store_empty = settings.getbool('FEED_STORE_EMPTY')
-        self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
+
+        if settings.get('FEED_EXPORT_FIELDS') is None:
+            self.export_fields = None  # don't promote None to []
+        else:
+            self.export_fields = settings.getlist('FEED_EXPORT_FIELDS')
+
         uripar = settings['FEED_URI_PARAMS']
         self._uripar = load_object(uripar) if uripar else lambda x, y: None
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index efee9bf94..c3d8b64ef 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -246,6 +246,10 @@ class FeedExportTest(unittest.TestCase):
         yield self.assertExported(items, header, rows,
                                   settings=settings, ordered=True)
 
+        # edge case: FEED_EXPORT_FIELDS==[] means nothing is exported
+        settings = {'FEED_EXPORT_FIELDS': []}
+        yield self.assertExportedJsonLines(items, [{},{},{},{}], settings)
+
     @defer.inlineCallbacks
     def test_export_dicts(self):
         # When dicts are used, only keys from the first row are used as

From 9b0ca1b7a0184532a8c8909d2b88c279e3eaf6cb Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 18 May 2015 17:13:25 +0500
Subject: [PATCH 0323/2974] drop support for FEED_EXPORT_FIELD=[] meaning "no
 fields"

---
 docs/topics/feed-exports.rst    | 14 +++++++-------
 scrapy/extensions/feedexport.py |  7 +------
 tests/test_feedexport.py        | 11 ++++++-----
 3 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index f4fb07761..d9444e34a 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -243,13 +243,13 @@ Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
 
 Use FEED_EXPORT_FIELDS option to define fields to export and their order.
 
-When FEED_EXPORT_FIELDS is None (default), Scrapy uses fields
-defined in :class:`~.Item` subclasses a spider is yielding.
-If FEED_EXPORT_FIELDS is None, raw dicts are used as items and
-an exporter requires a fixed set of fields (this is the case for
-:ref:`CSV <topics-feed-format-csv>` export format), then
-Scrapy tries to infer field names from the exported data - currently it
-uses field names from the first item.
+When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses fields
+defined in dicts or :class:`~.Item` subclasses a spider is yielding.
+
+If an exporter requires a fixed set of fields (this is the case for
+:ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS
+is empty or None, then Scrapy tries to infer field names from the
+exported data - currently it uses field names from the first item.
 
 .. setting:: FEED_STORE_EMPTY
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 08e868c9f..7a1a02be1 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -151,12 +151,7 @@ class FeedExporter(object):
         if not self._exporter_supported(self.format):
             raise NotConfigured
         self.store_empty = settings.getbool('FEED_STORE_EMPTY')
-
-        if settings.get('FEED_EXPORT_FIELDS') is None:
-            self.export_fields = None  # don't promote None to []
-        else:
-            self.export_fields = settings.getlist('FEED_EXPORT_FIELDS')
-
+        self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
         uripar = settings['FEED_URI_PARAMS']
         self._uripar = load_object(uripar) if uripar else lambda x, y: None
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c3d8b64ef..d6c96ca74 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -234,7 +234,12 @@ class FeedExportTest(unittest.TestCase):
         yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
         yield self.assertExportedJsonLines(items, rows_jl)
 
-        # but it is possible to override fields using FEED_EXPORT_FIELDS
+        # edge case: FEED_EXPORT_FIELDS==[] means the same as default None
+        settings = {'FEED_EXPORT_FIELDS': []}
+        yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
+        yield self.assertExportedJsonLines(items, rows_jl, settings)
+
+        # it is possible to override fields using FEED_EXPORT_FIELDS
         header = ["foo", "baz", "hello"]
         settings = {'FEED_EXPORT_FIELDS': header}
         rows = [
@@ -246,10 +251,6 @@ class FeedExportTest(unittest.TestCase):
         yield self.assertExported(items, header, rows,
                                   settings=settings, ordered=True)
 
-        # edge case: FEED_EXPORT_FIELDS==[] means nothing is exported
-        settings = {'FEED_EXPORT_FIELDS': []}
-        yield self.assertExportedJsonLines(items, [{},{},{},{}], settings)
-
     @defer.inlineCallbacks
     def test_export_dicts(self):
         # When dicts are used, only keys from the first row are used as

From 60e79db3eebc321547f09284082516344030fad8 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 18 May 2015 19:28:37 +0500
Subject: [PATCH 0324/2974] fixed FeedExporter shutdown log messages

---
 scrapy/extensions/feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 3bc1c92cc..35ddc0fd1 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -177,7 +177,7 @@ class FeedExporter(object):
         if not slot.itemcount and not self.store_empty:
             return
         slot.exporter.finish_exporting()
-        logfmt = "%%s %(format)s feed (%(itemcount)d items) in: %(uri)s"
+        logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
         log_args = {'format': self.format,
                     'itemcount': slot.itemcount,
                     'uri': slot.uri}

From af0c8f82f4e4242079d9e4545c1000686be2d4a6 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 18 May 2015 14:46:23 -0300
Subject: [PATCH 0325/2974] Move exporters/__init__.py to exporters.py

---
 scrapy/{exporters/__init__.py => exporters.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename scrapy/{exporters/__init__.py => exporters.py} (100%)

diff --git a/scrapy/exporters/__init__.py b/scrapy/exporters.py
similarity index 100%
rename from scrapy/exporters/__init__.py
rename to scrapy/exporters.py

From 7a5b5ec4d687cf001d2fb18989e34605ac8ec5fe Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 19 May 2015 00:57:39 +0500
Subject: [PATCH 0326/2974] TST remove unnecessary lines from py3-ignores

scrapy/contrib is already skipped - see https://github.com/scrapy/scrapy/pull/1165
---
 tests/py3-ignores.txt | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index e0871c350..46cc4c06f 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -83,25 +83,15 @@ scrapy/core/downloader/handlers/http11.py
 scrapy/core/downloader/handlers/http.py
 scrapy/core/downloader/handlers/ftp.py
 scrapy/core/downloader/webclient.py
-scrapy/contrib/pipeline/images.py
-scrapy/contrib/pipeline/files.py
 scrapy/pipelines/images.py
 scrapy/pipelines/files.py
-scrapy/contrib/linkextractors/sgml.py
-scrapy/contrib/linkextractors/regex.py
-scrapy/contrib/linkextractors/htmlparser.py
 scrapy/linkextractors/sgml.py
 scrapy/linkextractors/regex.py
 scrapy/linkextractors/htmlparser.py
-scrapy/contrib/downloadermiddleware/retry.py
-scrapy/contrib/downloadermiddleware/httpproxy.py
-scrapy/contrib/downloadermiddleware/cookies.py
 scrapy/downloadermiddlewares/retry.py
 scrapy/downloadermiddlewares/httpproxy.py
 scrapy/downloadermiddlewares/cookies.py
-scrapy/contrib/statsmailer.py
 scrapy/extensions/statsmailer.py
-scrapy/contrib/memusage.py
 scrapy/extensions/memusage.py
 scrapy/commands/deploy.py
 scrapy/commands/bench.py

From 795ca3945fcc3c0394ac9f691722ddf786b5ca53 Mon Sep 17 00:00:00 2001
From: Alexander Chekunkov <chekunkov@gmail.com>
Date: Thu, 21 May 2015 15:53:05 +0700
Subject: [PATCH 0327/2974] keep level_names in scrapy.log for backwards
 compatibility

---
 scrapy/log.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/scrapy/log.py b/scrapy/log.py
index e1c68aa93..719fceaad 100644
--- a/scrapy/log.py
+++ b/scrapy/log.py
@@ -1,7 +1,6 @@
 """
 This module is kept to provide a helpful warning about its removal.
 """
-
 import logging
 import warnings
 
@@ -18,7 +17,7 @@ warnings.warn("Module `scrapy.log` has been deprecated, Scrapy now relies on "
               ScrapyDeprecationWarning, stacklevel=2)
 
 
-# Imports kept for backwards-compatibility
+# Imports and level_names variable kept for backwards-compatibility
 
 DEBUG = logging.DEBUG
 INFO = logging.INFO
@@ -27,6 +26,15 @@ ERROR = logging.ERROR
 CRITICAL = logging.CRITICAL
 SILENT = CRITICAL + 1
 
+level_names = {
+    logging.DEBUG: "DEBUG",
+    logging.INFO: "INFO",
+    logging.WARNING: "WARNING",
+    logging.ERROR: "ERROR",
+    logging.CRITICAL: "CRITICAL",
+    SILENT: "SILENT",
+}
+
 
 def msg(message=None, _level=logging.INFO, **kw):
     warnings.warn('log.msg has been deprecated, create a python logger and '

From afcf70cdc66dbdf0d511554b6c9cadc121243204 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Mon, 18 May 2015 23:00:57 -0300
Subject: [PATCH 0328/2974] Add 1.0 release notes

---
 docs/news.rst | 358 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 358 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index adf02d9fb..8c093dadc 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,364 @@
 Release notes
 =============
 
+1.0
+---
+
+You will find a lot of new features and bugfixes in this major release.  Make
+sure to check our updated :ref:`overview <intro-overview>` to get a glance of
+some of the changes, along with our brushed :ref:`tutorial <intro-tutorial>`.
+
+Support for returning dictionaries in spiders
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Declaring and returning Scrapy Items is no longer necessary to collect the
+scraped data from your spider, you can now return explicit dictionaries
+instead.
+
+*Classic version*
+
+::
+
+    class MyItem(scrapy.Item):
+        url = scrapy.Field()
+
+    class MySpider(scrapy.Spider):
+        def parse(self, response):
+            return MyItem(url=response.url)
+
+*New version*
+
+::
+
+    class MySpider(scrapy.Spider):
+        def parse(self, response):
+            return {'url': response.url}
+
+Per-spider settings (GSoC 2014)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Last Google Summer of Code project accomplished an important redesign of the
+mechanism used for populating settings, introducing explicit priorities to
+override any given setting. As an extension of that goal, we included a new
+level of priority for settings that act exclusively for a single spider,
+allowing them to redefine project settings.
+
+Start using it by defining a :attr:`~scrapy.spiders.Spider.custom_settings`
+class variable in your spider::
+
+    class MySpider(scrapy.Spider):
+        custom_settings = {
+            "DOWNLOAD_DELAY": 5.0,
+            "RETRY_ENABLED": False,
+        }
+
+Read more about settings population: :ref:`topics-settings`
+
+Python Logging
+~~~~~~~~~~~~~~
+
+Scrapy 1.0 has moved away from Twisted logging to support Python built in’s
+as default logging system. We’re maintaining backward compatibility for most
+of the old custom interface to call logging functions, but you’ll get
+warnings to switch to the Python logging API entirely.
+
+*Old version*
+
+::
+
+    from scrapy import log
+    log.msg('MESSAGE', log.INFO)
+
+*New version*
+
+::
+
+    import logging
+    logging.info('MESSAGE')
+
+Logging with spiders remains the same, but on top of the
+:meth:`~scrapy.spiders.Spider.log` method you’ll have access to a custom
+:attr:`~scrapy.spiders.Spider.logger` created for the spider to issue log
+events:
+
+::
+
+    class MySpider(scrapy.Spider):
+        def parse(self, response):
+            self.logger.info('Response received')
+
+Read more in the logging documentation: :ref:`topics-logging`
+
+Crawler API refactoring (GSoC 2014)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Another milestone for last Google Summer of Code was a refactoring of the
+internal API, seeking a simpler and easier usage. Check new core interface
+in: :ref:`topics-api`
+
+A common situation where you will face these changes is while running Scrapy
+from scripts. Here’s a quick example of how to run a Spider manually with the
+new API:
+
+::
+
+    from scrapy.crawler import CrawlerProcess
+
+    process = CrawlerProcess({
+        'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
+    })
+    process.crawl(MySpider)
+    process.start()
+
+Bear in mind this feature is still under development and its API may change
+until it reaches a stable status.
+
+See more examples for scripts running Scrapy: :ref:`topics-practices`
+
+Module Relocations
+~~~~~~~~~~~~~~~~~~
+
+There’s been a large rearrangement of modules trying to improve the general
+structure of Scrapy. Main changes were separating various subpackages into
+new projects and dissolving both `scrapy.contrib` and `scrapy.contrib_exp`
+into top level packages. Backward compatibility was kept among internal
+relocations, while importing deprecated modules expect warnings indicating
+their new place.
+
+Full list of relocations
+************************
+
+Outsourced packages
+
+.. note::
+    These extensions went through some minor changes, e.g. some setting names
+    were changed. Please check the documentation in each new repository to
+    get familiar with the new usage.
+
++-------------------------------------+-------------------------------------+
+| Old location                        | New location                        |
++=====================================+=====================================+
+| scrapy.commands.deploy              | `scrapyd-client <https://github.com |
+|                                     | /scrapy/scrapyd-client>`_           |
+|                                     | (See other alternatives here:       |
+|                                     | :ref:`topics-deploy`)               |
++-------------------------------------+-------------------------------------+
+| scrapy.contrib.djangoitem           | `scrapy-djangoitem <https://github. |
+|                                     | com/scrapy/scrapy-djangoitem>`_     |
++-------------------------------------+-------------------------------------+
+| scrapy.webservice                   | `scrapy-jsonrpc <https://github.com |
+|                                     | /scrapy/scrapy-jsonrpc>`_           |
++-------------------------------------+-------------------------------------+
+
+`scrapy.contrib_exp` and `scrapy.contrib` dissolutions
+
++-------------------------------------+-------------------------------------+
+| Old location                        | New location                        |
++=====================================+=====================================+
+| scrapy.contrib\_exp.downloadermidd\ | scrapy.downloadermiddlewares.decom\ |
+| leware.decompression                | pression                            |
++-------------------------------------+-------------------------------------+
+| scrapy.contrib\_exp.iterators       | scrapy.utils.iterators              |
++-------------------------------------+-------------------------------------+
+| scrapy.contrib.downloadermiddleware | scrapy.downloadermiddlewares        |
++-------------------------------------+-------------------------------------+
+| scrapy.contrib.exporter             | scrapy.exporters                    |
++-------------------------------------+-------------------------------------+
+| scrapy.contrib.linkextractors       | scrapy.linkextractors               |
++-------------------------------------+-------------------------------------+
+| scrapy.contrib.loader               | scrapy.loader                       |
++-------------------------------------+-------------------------------------+
+| scrapy.contrib.loader.processor     | scrapy.loader.processors            |
++-------------------------------------+-------------------------------------+
+| scrapy.contrib.pipeline             | scrapy.pipelines                    |
++-------------------------------------+-------------------------------------+
+| scrapy.contrib.spidermiddleware     | scrapy.spidermiddlewares            |
++-------------------------------------+-------------------------------------+
+| scrapy.contrib.spiders              | scrapy.spiders                      |
++-------------------------------------+-------------------------------------+
+| * scrapy.contrib.closespider        | scrapy.extensions.\*                |
+| * scrapy.contrib.corestats          |                                     |
+| * scrapy.contrib.debug              |                                     |
+| * scrapy.contrib.feedexport         |                                     |
+| * scrapy.contrib.httpcache          |                                     |
+| * scrapy.contrib.logstats           |                                     |
+| * scrapy.contrib.memdebug           |                                     |
+| * scrapy.contrib.memusage           |                                     |
+| * scrapy.contrib.spiderstate        |                                     |
+| * scrapy.contrib.statsmailer        |                                     |
+| * scrapy.contrib.throttle           |                                     |
++-------------------------------------+-------------------------------------+
+
+Plural renames and Modules unification
+
++-------------------------------------+-------------------------------------+
+| Old location                        | New location                        |
++=====================================+=====================================+
+| scrapy.command                      | scrapy.commands                     |
++-------------------------------------+-------------------------------------+
+| scrapy.dupefilter                   | scrapy.dupefilters                  |
++-------------------------------------+-------------------------------------+
+| scrapy.linkextractor                | scrapy.linkextractors               |
++-------------------------------------+-------------------------------------+
+| scrapy.spider                       | scrapy.spiders                      |
++-------------------------------------+-------------------------------------+
+| scrapy.squeue                       | scrapy.squeues                      |
++-------------------------------------+-------------------------------------+
+| scrapy.statscol                     | scrapy.statscollectors              |
++-------------------------------------+-------------------------------------+
+| scrapy.utils.decorator              | scrapy.utils.decorators             |
++-------------------------------------+-------------------------------------+
+
+Class renames
+
++-------------------------------------+-------------------------------------+
+| Old location                        | New location                        |
++=====================================+=====================================+
+| scrapy.spidermanager.SpiderManager  | scrapy.spiderloader.SpiderLoader    |
++-------------------------------------+-------------------------------------+
+
+Settings renames
+
++-------------------------------------+-------------------------------------+
+| Old location                        | New location                        |
++=====================================+=====================================+
+| SPIDER\_MANAGER\_CLASS              | SPIDER\_LOADER\_CLASS               |
++-------------------------------------+-------------------------------------+
+
+Changelog
+~~~~~~~~~
+
+New Features and Enhancements
+
+- Python logging (:issue:`1060`, :issue:`1235`, :issue:`1236`, :issue:`1240`)
+- FEED_EXPORT_FIELDS option (:issue:`1159`, :issue:`1224`)
+- Dns cache size and timeout options (:issue:`1132`)
+- support namespace prefix in xmliter_lxml (:issue:`963`)
+- Reactor threadpool max size setting (:issue:`1123`)
+- Allow spiders to return dicts. (:issue:`1081`)
+- Add Response.urljoin() helper (:issue:`1086`)
+- look in ~/.config/scrapy.cfg for user config (:issue:`1098`)
+- handle TLS SNI (:issue:`1101`)
+- Selectorlist extract first (:issue:`624`, :issue:`1145`)
+- Added JmesSelect (:issue:`1016`)
+- add gzip compression to filesystem http cache backend (:issue:`1020`)
+- CSS support in link extractors (:issue:`983`)
+- httpcache dont_cache meta #19 #689 (:issue:`821`)
+- add signal to be sent when request is dropped by the scheduler
+  (:issue:`961`)
+- avoid download large response (:issue:`946`)
+- Allow to specify the quotechar in CSVFeedSpider (:issue:`882`)
+- Add referer to "Spider error processing" log message (:issue:`795`)
+- process robots.txt once (:issue:`896`)
+- GSoC Per-spider settings (:issue:`854`)
+- Add project name validation (:issue:`817`)
+- GSoC API cleanup (:issue:`816`, :issue:`1128`, :issue:`1147`,
+  :issue:`1148`, :issue:`1156`, :issue:`1185`, :issue:`1187`)
+
+Deprecations and Removals
+
+- Deprecate htmlparser link extractor (:issue:`1205`)
+- remove deprecated code from FeedExporter (:issue:`1155`)
+- a leftover for.15 compatibility (:issue:`925`)
+- drop support for CONCURRENT_REQUESTS_PER_SPIDER (:issue:`895`)
+- Drop old engine code (:issue:`911`)
+- Deprecate SgmlLinkExtractor (:issue:`777`)
+
+Relocations
+
+- Move exporters/__init__.py to exporters.py (:issue:`1242`)
+- Move base classes to their packages (:issue:`1218`, :issue:`1233`)
+- Module relocation (:issue:`1181`, :issue:`1210`)
+- rename SpiderManager to SpiderLoader (:issue:`1166`)
+- Remove djangoitem (:issue:`1177`)
+- remove scrapy deploy command (:issue:`1102`)
+- dissolve contrib_exp (:issue:`1134`)
+- Deleted bin folder from root, fixes #913 (:issue:`914`)
+- Remove jsonrpc based webservice (:issue:`859`)
+- Move Test cases under project root dir (:issue:`827`, :issue:`841`)
+
+Documentation
+
+- CrawlerProcess documentation (:issue:`1190`)
+- Favoring web scraping over screen scraping in the descriptions
+  (:issue:`1188`)
+- Some improvements for Scrapy tutorial (:issue:`1180`)
+- Documenting Files Pipeline together with Images Pipeline (:issue:`1150`)
+- deployment docs tweaks (:issue:`1164`)
+- Added deployment section covering scrapyd-deploy and shub (:issue:`1124`)
+- Adding more settings to project template (:issue:`1073`)
+- some improvements to overview page (:issue:`1106`)
+- Updated link in docs/topics/architecture.rst (:issue:`647`)
+- DOC reorder topics (:issue:`1022`)
+- updating list of Request.meta special keys (:issue:`1071`)
+- DOC document download_timeout (:issue:`898`)
+- DOC simplify extension docs (:issue:`893`)
+- Leaks docs (:issue:`894`)
+- DOC document from_crawler method for item pipelines (:issue:`904`)
+- Corrections & Sphinx related fixes (:issue:`1220`, :issue:`1219`,
+  :issue:`1196`, :issue:`1172`, :issue:`1171`, :issue:`1169`, :issue:`1160`,
+  :issue:`1154`, :issue:`1127`, :issue:`1112`, :issue:`1105`, :issue:`1041`,
+  :issue:`1082`, :issue:`1033`, :issue:`944`, :issue:`866`, :issue:`864`,
+  :issue:`796`)
+
+Bugfixes
+
+- Item multi inheritance fix (:issue:`353`, :issue:`1228`)
+- ItemLoader.load_item: iterate over copy of fields (:issue:`722`)
+- Fix Unhandled error in Deferred (RobotsTxtMiddleware) (:issue:`1131`,
+  :issue:`1197`)
+- Force to read DOWNLOAD_TIMEOUT as int (:issue:`954`)
+- scrapy.utils.misc.load_object should print full traceback (:issue:`902`)
+- Fix bug for ".local" host name (:issue:`878`)
+- Fix for Enabled extensions, middlewares, pipelines info not printed
+  anymore (:issue:`879`)
+- fix dont_merge_cookies bad behaviour when set to false on meta
+  (:issue:`846`)
+
+Python 3 In Progress Support
+
+- disable scrapy.telnet if twisted.conch is not available (:issue:`1161`)
+- fix Python 3 syntax errors in ajaxcrawl.py (:issue:`1162`)
+- more python3 compatibility changes for urllib (:issue:`1121`)
+- assertItemsEqual was renamed to assertCountEqual in Python 3.
+  (:issue:`1070`)
+- Import unittest.mock if available. (:issue:`1066`)
+- updated deprecated cgi.parse_qsl to use six's parse_qsl (:issue:`909`)
+- Prevent Python 3 port regressions (:issue:`830`)
+- PY3: use MutableMapping for python 3 (:issue:`810`)
+- PY3: use six.BytesIO and six.moves.cStringIO (:issue:`803`)
+- PY3: fix xmlrpclib and email imports (:issue:`801`)
+- PY3: use six for robotparser and urlparse (:issue:`800`)
+- PY3: use six.iterkeys, six.iteritems, and tempfile (:issue:`799`)
+- PY3: fix has_key and use six.moves.configparser (:issue:`798`)
+- PY3: use six.moves.cPickle (:issue:`797`)
+- PY3 make it possible to run some tests in Python3 (:issue:`776`)
+
+Tests
+
+- remove unnecessary lines from py3-ignores (:issue:`1243`)
+- Fix remaining warnings from pytest while collecting tests (:issue:`1206`)
+- Add docs build to travis (:issue:`1234`)
+- TST don't collect tests from deprecated modules. (:issue:`1165`)
+- install service_identity package in tests to prevent warnings
+  (:issue:`1168`)
+- Fix deprecated settings API in tests (:issue:`1152`)
+- Add test for webclient with POST method and no body given (:issue:`1089`)
+- py3-ignores.txt supports comments (:issue:`1044`)
+- modernize some of the asserts (:issue:`835`)
+- selector.__repr__ test (:issue:`779`)
+
+Code refactoring
+
+- CSVFeedSpider cleanup: use iterate_spider_output (:issue:`1079`)
+- remove unnecessary check from scrapy.utils.spider.iter_spider_output
+  (:issue:`1078`)
+- Pydispatch pep8 (:issue:`992`)
+- Removed unused 'load=False' parameter from walk_modules() (:issue:`871`)
+- For consistency, use `job_dir` helper in `SpiderState` extension.
+  (:issue:`805`)
+- rename "sflo" local variables to less cryptic "log_observer" (:issue:`775`)
+
 0.24.6 (2015-04-20)
 -------------------
 

From 600164594c1bf1c8a4930634ccf6ab2a0da5d7d8 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 22 May 2015 01:04:57 -0300
Subject: [PATCH 0329/2974] New release cycle in .bumpversion.cfg

1.0.0dev1 -> 1.0.0rc1 -> 1.0.0 -> 1.1.0dev1 -> ...
---
 .bumpversion.cfg | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index d373d676a..5d31da1fb 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -3,6 +3,25 @@ current_version = 0.25.1
 commit = True
 tag = True
 tag_name = {new_version}
+parse = ^
+	(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)
+	(?:(?P<prerel>[abc]|rc|dev)(?P<prerelversion>\d+))?
+serialize = 
+	{major}.{minor}.{patch}{prerel}{prerelversion}
+	{major}.{minor}.{patch}
 
 [bumpversion:file:scrapy/VERSION]
 
+[bumpversion:part:prerel]
+optional_value = gamma
+values =
+	dev
+	rc
+	gamma
+
+[bumpversion:part:prerelversion]
+values = 
+	1
+	2
+	3
+

From de6d232a02f46e9f843a1f3576c8a9b871b444af Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 22 May 2015 13:24:27 -0300
Subject: [PATCH 0330/2974] =?UTF-8?q?Bump=20version:=200.25.1=20=E2=86=92?=
 =?UTF-8?q?=201.0.0rc1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 4 ++--
 scrapy/VERSION   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 5d31da1fb..e3f1414b2 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.25.1
+current_version = 1.0.0rc1
 commit = True
 tag = True
 tag_name = {new_version}
@@ -14,7 +14,7 @@ serialize =
 
 [bumpversion:part:prerel]
 optional_value = gamma
-values =
+values = 
 	dev
 	rc
 	gamma
diff --git a/scrapy/VERSION b/scrapy/VERSION
index af2dabf3f..6a056a8b1 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-0.25.1
+1.0.0rc1

From 4b2763c6f99cb25c8f36912ab3aba40a01cba8c3 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 22 May 2015 13:24:50 -0300
Subject: [PATCH 0331/2974] =?UTF-8?q?Bump=20version:=201.0.0rc1=20?=
 =?UTF-8?q?=E2=86=92=201.1.0dev1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index e3f1414b2..f64f87e85 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.0.0rc1
+current_version = 1.1.0dev1
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 6a056a8b1..3ac32d3a6 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.0.0rc1
+1.1.0dev1

From 27ce3225bdf843567e037c2cdba387d0b75e24ef Mon Sep 17 00:00:00 2001
From: Alexey Vishnevsky <alexey@senexx.com>
Date: Mon, 16 Mar 2015 11:00:27 +0200
Subject: [PATCH 0332/2974] Makes scrapy more async by letting to reactor spend
 another couple of cycles to accomplish its needs.

---
 scrapy/utils/defer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 45a451d08..5df93748d 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -12,7 +12,7 @@ def defer_fail(_failure):
     next reactor loop
     """
     d = defer.Deferred()
-    reactor.callLater(0, d.errback, _failure)
+    reactor.callLater(0.1, d.errback, _failure)
     return d
 
 def defer_succeed(result):
@@ -20,7 +20,7 @@ def defer_succeed(result):
     next reactor loop
     """
     d = defer.Deferred()
-    reactor.callLater(0, d.callback, result)
+    reactor.callLater(0.1, d.callback, result)
     return d
 
 def defer_result(result):

From d439c26d76468d1732128f037a646f72409ea135 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 22 May 2015 17:12:43 -0300
Subject: [PATCH 0333/2974] update docstring and release notes

---
 docs/news.rst         |  1 +
 scrapy/utils/defer.py | 10 ++++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 8c093dadc..ebc5304d1 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -256,6 +256,7 @@ New Features and Enhancements
 - Add project name validation (:issue:`817`)
 - GSoC API cleanup (:issue:`816`, :issue:`1128`, :issue:`1147`,
   :issue:`1148`, :issue:`1156`, :issue:`1185`, :issue:`1187`)
+- Be more responsive with IO operations (:issue:`1074` and :issue:`1075`)
 
 Deprecations and Removals
 
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 5df93748d..97c41af5e 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -8,16 +8,22 @@ from twisted.python import failure
 from scrapy.exceptions import IgnoreRequest
 
 def defer_fail(_failure):
-    """Same as twisted.internet.defer.fail, but delay calling errback until
+    """Same as twisted.internet.defer.fail but delay calling errback until
     next reactor loop
+
+    It delays by 100ms so reactor has a chance to go trough readers and writers
+    before attending pending delayed calls, so do not set delay to zero.
     """
     d = defer.Deferred()
     reactor.callLater(0.1, d.errback, _failure)
     return d
 
 def defer_succeed(result):
-    """Same as twsited.internet.defer.succed, but delay calling callback until
+    """Same as twisted.internet.defer.succeed but delay calling callback until
     next reactor loop
+
+    It delays by 100ms so reactor has a chance to go trough readers and writers
+    before attending pending delayed calls, so do not set delay to zero.
     """
     d = defer.Deferred()
     reactor.callLater(0.1, d.callback, result)

From ebe889a6635ae4ee33186155912a8931bac1692d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Sat, 23 May 2015 20:50:30 -0300
Subject: [PATCH 0334/2974] Unquote request path before passing to FTPClient,
 it already escape paths

---
 scrapy/core/downloader/handlers/ftp.py |  4 ++--
 tests/test_downloader_handlers.py      | 17 ++++++++++++++++-
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index d96e37fef..93fa0a3a7 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
 
 import re
 from io import BytesIO
-from six.moves.urllib.parse import urlparse
+from six.moves.urllib.parse import urlparse, unquote
 
 from twisted.internet import reactor
 from twisted.protocols.ftp import FTPClient, CommandFailed
@@ -73,7 +73,7 @@ class FTPDownloadHandler(object):
                                     request.meta["ftp_password"],
                                     passive=request.meta.get("ftp_passive", 1))
         return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient,
-                                request, parsed_url.path)
+                                request, unquote(parsed_url.path))
 
     def gotClient(self, client, request, filepath):
         self.client = client
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index df038aa88..c814de307 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -511,7 +511,9 @@ class FTPTestCase(unittest.TestCase):
         os.mkdir(self.directory)
         userdir = os.path.join(self.directory, self.username)
         os.mkdir(userdir)
-        FilePath(userdir).child('file.txt').setContent("I have the power!")
+        fp = FilePath(userdir)
+        fp.child('file.txt').setContent("I have the power!")
+        fp.child('file with spaces.txt').setContent("Moooooooooo power!")
 
         # setup server
         realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory)
@@ -547,6 +549,19 @@ class FTPTestCase(unittest.TestCase):
             self.assertEqual(r.headers, {'Local Filename': [''], 'Size': ['17']})
         return self._add_test_callbacks(d, _test)
 
+    def test_ftp_download_path_with_spaces(self):
+        request = Request(
+            url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum,
+            meta={"ftp_user": self.username, "ftp_password": self.password}
+        )
+        d = self.download_handler.download_request(request, None)
+
+        def _test(r):
+            self.assertEqual(r.status, 200)
+            self.assertEqual(r.body, 'Moooooooooo power!')
+            self.assertEqual(r.headers, {'Local Filename': [''], 'Size': ['18']})
+        return self._add_test_callbacks(d, _test)
+
     def test_ftp_download_notexist(self):
         request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum,
                 meta={"ftp_user": self.username, "ftp_password": self.password})

From 545c4224f9d7ee077fec458c0ee4b3fc0afc0f9c Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Mon, 25 May 2015 16:01:54 -0300
Subject: [PATCH 0335/2974] update old crawlera link

---
 docs/topics/practices.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 3a9b21d3b..7bdd28573 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -254,4 +254,4 @@ If you are still unable to prevent your bot getting banned, consider contacting
 .. _Google cache: http://www.googleguide.com/cached_pages.html
 .. _testspiders: https://github.com/scrapinghub/testspiders
 .. _Twisted Reactor Overview: http://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
-.. _Crawlera: http://crawlera.com
+.. _Crawlera: http://scrapinghub.com/crawlera

From b0ea3e38d1a045561409ea76c5d267edb24e6f33 Mon Sep 17 00:00:00 2001
From: Alexander Chekunkov <chekunkov@gmail.com>
Date: Tue, 26 May 2015 17:37:16 +0700
Subject: [PATCH 0336/2974] remove CrawlerProcess.stopping as it isn't used any
 more

---
 scrapy/crawler.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index e6f4b225f..dadc1b0be 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -207,7 +207,6 @@ class CrawlerProcess(CrawlerRunner):
     def __init__(self, settings):
         super(CrawlerProcess, self).__init__(settings)
         install_shutdown_handlers(self._signal_shutdown)
-        self.stopping = False
         configure_logging(settings)
         log_scrapy_info(settings)
 

From b2765aabd817f26ea40b4373aae1fc84ac86cc2b Mon Sep 17 00:00:00 2001
From: Alexander Chekunkov <chekunkov@gmail.com>
Date: Wed, 27 May 2015 13:52:47 +0700
Subject: [PATCH 0337/2974] LogCounterHandler is never removed from root
 handlers list, fix that

lambda is garbage collected and because receiver is added as weak reference by default - when signals.engine_stopped is fired logging.root.removeHandler is not executed. Fixed that by assigning lambda to a private argument and not by using connect(..., weak=False) because I belive this lambda function should be collected with crawler object
---
 scrapy/crawler.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index e6f4b225f..8ec253ed9 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -35,8 +35,10 @@ class Crawler(object):
 
         handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL'))
         logging.root.addHandler(handler)
-        self.signals.connect(lambda: logging.root.removeHandler(handler),
-                             signals.engine_stopped)
+        # lambda is assigned to Crawler attribute because this way it is not
+        # garbage collected after leaving __init__ scope
+        self.__remove_handler = lambda: logging.root.removeHandler(handler)
+        self.signals.connect(self.__remove_handler, signals.engine_stopped)
 
         lf_cls = load_object(self.settings['LOG_FORMATTER'])
         self.logformatter = lf_cls.from_crawler(self)

From 26f50d3f43bd0a4efdd13d566123090b6efa7847 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 27 May 2015 09:17:18 -0300
Subject: [PATCH 0338/2974] Extend regex for tags that deploy to PyPI to
 support new release cycle

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 09da120bc..4a0ef834c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,4 +25,4 @@ deploy:
     tags: true
     all_branches: true
     repo: scrapy/scrapy
-    condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9][.][0-9]*[02468][.]"
+    condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+)?$"

From 343d20d7911a0fe51a6ed04eaafa679e91a5b567 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 27 May 2015 11:53:54 -0300
Subject: [PATCH 0339/2974] Update 1.0 release notes

---
 docs/news.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index ebc5304d1..8686398eb 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -232,7 +232,8 @@ Changelog
 
 New Features and Enhancements
 
-- Python logging (:issue:`1060`, :issue:`1235`, :issue:`1236`, :issue:`1240`)
+- Python logging (:issue:`1060`, :issue:`1235`, :issue:`1236`, :issue:`1240`,
+  :issue:`1259`)
 - FEED_EXPORT_FIELDS option (:issue:`1159`, :issue:`1224`)
 - Dns cache size and timeout options (:issue:`1132`)
 - support namespace prefix in xmliter_lxml (:issue:`963`)
@@ -255,7 +256,7 @@ New Features and Enhancements
 - GSoC Per-spider settings (:issue:`854`)
 - Add project name validation (:issue:`817`)
 - GSoC API cleanup (:issue:`816`, :issue:`1128`, :issue:`1147`,
-  :issue:`1148`, :issue:`1156`, :issue:`1185`, :issue:`1187`)
+  :issue:`1148`, :issue:`1156`, :issue:`1185`, :issue:`1187`, :issue:`1258`)
 - Be more responsive with IO operations (:issue:`1074` and :issue:`1075`)
 
 Deprecations and Removals

From 342cb622f1ea93268477da557099010bbd72529a Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 27 May 2015 23:04:58 +0500
Subject: [PATCH 0340/2974] DOC fix non-working link (by removing it).

See https://github.com/scrapy/scrapy/pull/1260
---
 docs/topics/media-pipeline.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 4e8562cac..4ee4f1758 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -71,7 +71,7 @@ The Images Pipeline uses `Pillow`_ for thumbnailing and normalizing images to
 JPEG/RGB format, so you need to install this library in order to use it.
 `Python Imaging Library`_ (PIL) should also work in most cases, but it is known
 to cause troubles in some setups, so we recommend to use `Pillow`_ instead of
-`PIL <Python Imaging Library>`_.
+PIL.
 
 .. _Pillow: https://github.com/python-pillow/Pillow
 .. _Python Imaging Library: http://www.pythonware.com/products/pil/

From aa6a72707daabfb6217f52e4774f2ff038f83dcc Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 30 May 2015 06:59:15 +0500
Subject: [PATCH 0341/2974] fixed CrawlerProcess when settings are passed as
 dicts

See https://github.com/scrapy/scrapy/pull/1156
---
 scrapy/crawler.py     |  4 ++--
 tests/test_crawler.py | 12 +++++++++++-
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 9b35b3642..2cd658276 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -209,8 +209,8 @@ class CrawlerProcess(CrawlerRunner):
     def __init__(self, settings):
         super(CrawlerProcess, self).__init__(settings)
         install_shutdown_handlers(self._signal_shutdown)
-        configure_logging(settings)
-        log_scrapy_info(settings)
+        configure_logging(self.settings)
+        log_scrapy_info(self.settings)
 
     def _signal_shutdown(self, signum, _):
         install_shutdown_handlers(self._signal_kill)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 906573887..53f517c27 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,7 +1,7 @@
 import warnings
 import unittest
 
-from scrapy.crawler import Crawler, CrawlerRunner
+from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
 from scrapy.settings import Settings, default_settings
 from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.spider import DefaultSpider
@@ -104,3 +104,13 @@ class CrawlerRunnerTestCase(unittest.TestCase):
             self.assertEqual(len(w), 1)
             self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message))
 
+
+class CrawlerProcessTest(unittest.TestCase):
+    def test_crawler_process_accepts_dict(self):
+        runner = CrawlerProcess({'foo': 'bar'})
+        self.assertEqual(runner.settings['foo'], 'bar')
+        self.assertEqual(
+            runner.settings['RETRY_ENABLED'],
+            default_settings.RETRY_ENABLED
+        )
+        self.assertIsInstance(runner.settings, Settings)

From 4446baae33b8654ec505df06cf95528eff5ccaf1 Mon Sep 17 00:00:00 2001
From: Jamey Sharp <jamey@minilop.net>
Date: Sun, 28 Dec 2014 19:16:31 -0800
Subject: [PATCH 0342/2974] Use cached responses if revalidation errors out.

---
 scrapy/downloadermiddlewares/httpcache.py | 16 ++++++++++++++++
 scrapy/extensions/httpcache.py            |  8 ++++++++
 2 files changed, 24 insertions(+)

diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index bd112c48d..521327bfe 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -1,11 +1,21 @@
 from email.utils import formatdate
+from twisted.internet import defer
+from twisted.internet.error import TimeoutError, DNSLookupError, \
+        ConnectionRefusedError, ConnectionDone, ConnectError, \
+        ConnectionLost, TCPTimedOutError
 from scrapy import signals
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.utils.misc import load_object
+from scrapy.xlib.tx import ResponseFailed
 
 
 class HttpCacheMiddleware(object):
 
+    DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
+                           ConnectionRefusedError, ConnectionDone, ConnectError,
+                           ConnectionLost, TCPTimedOutError, ResponseFailed,
+                           IOError)
+
     def __init__(self, settings, stats):
         if not settings.getbool('HTTPCACHE_ENABLED'):
             raise NotConfigured
@@ -84,6 +94,12 @@ class HttpCacheMiddleware(object):
         self._cache_response(spider, response, request, cachedresponse)
         return response
 
+    def process_exception(self, request, exception, spider):
+        cachedresponse = request.meta.pop('cached_response', None)
+        if cachedresponse is not None and isinstance(exception, self.DOWNLOAD_EXCEPTIONS):
+            self.stats.inc_value('httpcache/errorrecovery', spider=spider)
+            return cachedresponse
+
     def _cache_response(self, spider, response, request, cachedresponse):
         if self.policy.should_cache_response(response, request):
             self.stats.inc_value('httpcache/store', spider=spider)
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 3173656fe..8011581ac 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -99,6 +99,14 @@ class RFC2616Policy(object):
         return False
 
     def is_cached_response_valid(self, cachedresponse, response, request):
+        # Use the cached response if the new response is a server error,
+        # as long as the old response didn't specify must-revalidate.
+        if response.status >= 500:
+            cc = self._parse_cachecontrol(cachedresponse)
+            if 'must-revalidate' not in cc:
+                return True
+
+        # Use the cached response if the server says it hasn't changed.
         return response.status == 304
 
     def _set_conditional_validators(self, request, cachedresponse):

From dd3a46295c069561b0c278a8af0db784b57a6416 Mon Sep 17 00:00:00 2001
From: Jamey Sharp <jamey@minilop.net>
Date: Sun, 28 Dec 2014 19:21:45 -0800
Subject: [PATCH 0343/2974] Support "Cache-Control: max-stale" in requests.

This allows spiders to be configured with the full RFC2616 cache policy,
but avoid revalidation on a request-by-request basis, while remaining
conformant with the HTTP spec.
---
 scrapy/extensions/httpcache.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 8011581ac..665ad3439 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -94,6 +94,25 @@ class RFC2616Policy(object):
         currentage = self._compute_current_age(cachedresponse, request, now)
         if currentage < freshnesslifetime:
             return True
+
+        if 'max-stale' in ccreq and 'must-revalidate' not in cc:
+            # From RFC2616: "Indicates that the client is willing to
+            # accept a response that has exceeded its expiration time.
+            # If max-stale is assigned a value, then the client is
+            # willing to accept a response that has exceeded its
+            # expiration time by no more than the specified number of
+            # seconds. If no value is assigned to max-stale, then the
+            # client is willing to accept a stale response of any age."
+            staleage = ccreq['max-stale']
+            if staleage is None:
+                return True
+
+            try:
+                if currentage < freshnesslifetime + max(0, int(staleage)):
+                    return True
+            except ValueError:
+                pass
+
         # Cached response is stale, try to set validators if any
         self._set_conditional_validators(request, cachedresponse)
         return False

From e23a38133726b716f5931e59e163cfe70169d17c Mon Sep 17 00:00:00 2001
From: Jamey Sharp <jamey@minilop.net>
Date: Sun, 28 Dec 2014 19:43:16 -0800
Subject: [PATCH 0344/2974] Let spiders ignore bogus Cache-Control headers.

Sites often set "no-store", "no-cache", "must-revalidate", etc., but get
upset at the traffic a spider can generate if it respects those
directives.

Allow the spider's author to selectively ignore Cache-Control directives
that are known to be unimportant for the sites being crawled.

We assume that the spider will not issue Cache-Control directives in
requests unless it actually needs them, so directives in requests are
not filtered.
---
 scrapy/extensions/httpcache.py      | 9 +++++++--
 scrapy/settings/default_settings.py | 1 +
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 665ad3439..c0efb8996 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -7,7 +7,7 @@ from time import time
 from weakref import WeakKeyDictionary
 from email.utils import mktime_tz, parsedate_tz
 from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
-from scrapy.http import Headers
+from scrapy.http import Headers, Response
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.project import data_path
@@ -39,12 +39,17 @@ class RFC2616Policy(object):
 
     def __init__(self, settings):
         self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
+        self.ignore_response_cache_controls = settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')
         self._cc_parsed = WeakKeyDictionary()
 
     def _parse_cachecontrol(self, r):
         if r not in self._cc_parsed:
             cch = r.headers.get('Cache-Control', '')
-            self._cc_parsed[r] = parse_cachecontrol(cch)
+            parsed = parse_cachecontrol(cch)
+            if isinstance(r, Response):
+                for key in self.ignore_response_cache_controls:
+                    parsed.pop(key, None)
+            self._cc_parsed[r] = parsed
         return self._cc_parsed[r]
 
     def should_cache_request(self, request):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 9debaabc3..bd1bb0936 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -155,6 +155,7 @@ HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
 HTTPCACHE_EXPIRATION_SECS = 0
 HTTPCACHE_IGNORE_HTTP_CODES = []
 HTTPCACHE_IGNORE_SCHEMES = ['file']
+HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
 HTTPCACHE_DBM_MODULE = 'anydbm'
 HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
 HTTPCACHE_GZIP = False

From c3b2cabf6c6600a5a2c6bbef2035ac7616ef6a06 Mon Sep 17 00:00:00 2001
From: Jamey Sharp <jamey@minilop.net>
Date: Sun, 28 Dec 2014 20:04:36 -0800
Subject: [PATCH 0345/2974] Allow setting RFC2616Policy to cache
 unconditionally.

A spider may wish to have all responses available in the cache, for
future use with "Cache-Control: max-stale", for instance. The
DummyPolicy caches all responses but never revalidates them, and
sometimes a more nuanced policy is desirable.

This setting still respects "Cache-Control: no-store" directives in
responses. If you don't want that, filter "no-store" out of the
Cache-Control headers in responses you feed to the cache middleware.
---
 scrapy/extensions/httpcache.py      | 4 ++++
 scrapy/settings/default_settings.py | 1 +
 2 files changed, 5 insertions(+)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index c0efb8996..4276ec928 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -38,6 +38,7 @@ class RFC2616Policy(object):
     MAXAGE = 3600 * 24 * 365  # one year
 
     def __init__(self, settings):
+        self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE')
         self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
         self.ignore_response_cache_controls = settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')
         self._cc_parsed = WeakKeyDictionary()
@@ -73,6 +74,9 @@ class RFC2616Policy(object):
         # Never cache 304 (Not Modified) responses
         elif response.status == 304:
             return False
+        # Cache unconditionally if configured to do so
+        elif self.always_store:
+            return True
         # Any hint on response expiration is good
         elif 'max-age' in cc or 'Expires' in response.headers:
             return True
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index bd1bb0936..5f9f4b98e 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -153,6 +153,7 @@ HTTPCACHE_DIR = 'httpcache'
 HTTPCACHE_IGNORE_MISSING = False
 HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
 HTTPCACHE_EXPIRATION_SECS = 0
+HTTPCACHE_ALWAYS_STORE = False
 HTTPCACHE_IGNORE_HTTP_CODES = []
 HTTPCACHE_IGNORE_SCHEMES = ['file']
 HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []

From 19915504422ccf6735b11f176ae8170c43562c29 Mon Sep 17 00:00:00 2001
From: Jamey Sharp <jamey@minilop.net>
Date: Mon, 29 Dec 2014 14:06:04 -0800
Subject: [PATCH 0346/2974] Allow client to bound max-age for revalidation.

Unlike specifying "Cache-Control: no-cache", if the request specifies
"max-age=0", then the cached validators will be used if possible to
avoid re-fetching unchanged pages.

That said, it's still useful to be able to specify "no-cache" on the
request, in cases where the origin server may have changed page contents
without changing validators.
---
 scrapy/extensions/httpcache.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 4276ec928..f33fcf819 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -101,6 +101,11 @@ class RFC2616Policy(object):
         now = time()
         freshnesslifetime = self._compute_freshness_lifetime(cachedresponse, request, now)
         currentage = self._compute_current_age(cachedresponse, request, now)
+
+        reqmaxage = self._get_max_age(ccreq)
+        if reqmaxage is not None:
+            freshnesslifetime = min(freshnesslifetime, reqmaxage)
+
         if currentage < freshnesslifetime:
             return True
 
@@ -144,15 +149,19 @@ class RFC2616Policy(object):
         if 'ETag' in cachedresponse.headers:
             request.headers['If-None-Match'] = cachedresponse.headers['ETag']
 
+    def _get_max_age(self, cc):
+        try:
+            return max(0, int(cc['max-age']))
+        except (KeyError, ValueError):
+            return None
+
     def _compute_freshness_lifetime(self, response, request, now):
         # Reference nsHttpResponseHead::ComputeFreshnessLifetime
         # http://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#410
         cc = self._parse_cachecontrol(response)
-        if 'max-age' in cc:
-            try:
-                return max(0, int(cc['max-age']))
-            except ValueError:
-                pass
+        maxage = self._get_max_age(cc)
+        if maxage is not None:
+            return maxage
 
         # Parse date header or synthesize it if none exists
         date = rfc1123_to_epoch(response.headers.get('Date')) or now

From bb3ebf13f97a06e8fce0e3dd6a734ab2f4a91fbd Mon Sep 17 00:00:00 2001
From: Marven Sanchez <marven.sanchez@gmail.com>
Date: Mon, 1 Jun 2015 18:20:12 +0800
Subject: [PATCH 0347/2974] Add tests for RFC2616 policy enhancements Add
 `scrapy/downloadermiddlewares/httpcache.py` to `tests/py3-ignores.txt

---
 tests/py3-ignores.txt                        |  1 +
 tests/test_downloadermiddleware_httpcache.py | 85 ++++++++++++++++++++
 2 files changed, 86 insertions(+)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 3d87bcb9a..141c44909 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -97,6 +97,7 @@ scrapy/contrib/downloadermiddleware/retry.py
 scrapy/contrib/downloadermiddleware/httpproxy.py
 scrapy/contrib/downloadermiddleware/cookies.py
 scrapy/downloadermiddlewares/retry.py
+scrapy/downloadermiddlewares/httpcache.py
 scrapy/downloadermiddlewares/httpproxy.py
 scrapy/downloadermiddlewares/cookies.py
 scrapy/contrib/statsmailer.py
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index ac954cc15..6c95e7b3a 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -343,6 +343,25 @@ class RFC2616PolicyTest(DefaultStorageTest):
                     self.assertFalse(resc)
                     assert 'cached' not in res2.flags
 
+        # cache unconditionally unless response contains no-store or is a 304
+        with self._middleware(HTTPCACHE_ALWAYS_STORE=True) as mw:
+            for idx, (_, status, headers) in enumerate(responses):
+                shouldcache = 'no-store' not in headers.get('Cache-Control', '') and status != 304
+                req0 = Request('http://example2-%d.com' % idx)
+                res0 = Response(req0.url, status=status, headers=headers)
+                res1 = self._process_requestresponse(mw, req0, res0)
+                res304 = res0.replace(status=304)
+                res2 = self._process_requestresponse(mw, req0, res304 if shouldcache else res0)
+                self.assertEqualResponse(res1, res0)
+                self.assertEqualResponse(res2, res0)
+                resc = mw.storage.retrieve_response(self.spider, req0)
+                if shouldcache:
+                    self.assertEqualResponse(resc, res1)
+                    assert 'cached' in res2.flags and res2.status != 304
+                else:
+                    self.assertFalse(resc)
+                    assert 'cached' not in res2.flags
+
     def test_cached_and_fresh(self):
         sampledata = [
             (200, {'Date': self.yesterday, 'Expires': self.tomorrow}),
@@ -381,6 +400,13 @@ class RFC2616PolicyTest(DefaultStorageTest):
                 res2 = self._process_requestresponse(mw, req0, None)
                 self.assertEqualResponse(res1, res2)
                 assert 'cached' in res2.flags
+                # validate cached response if request max-age set as 0
+                req1 = req0.replace(headers={'Cache-Control': 'max-age=0'})
+                res304 = res0.replace(status=304)
+                assert mw.process_request(req1, self.spider) is None
+                res3 = self._process_requestresponse(mw, req1, res304)
+                self.assertEqualResponse(res1, res3)
+                assert 'cached' in res3.flags
 
     def test_cached_and_stale(self):
         sampledata = [
@@ -395,6 +421,9 @@ class RFC2616PolicyTest(DefaultStorageTest):
             (200, {'Cache-Control': 'no-cache'}),
             (200, {'Cache-Control': 'no-cache', 'ETag': 'foo'}),
             (200, {'Cache-Control': 'no-cache', 'Last-Modified': self.yesterday}),
+            (200, {'Cache-Control': 'no-cache,must-revalidate', 'Last-Modified': self.yesterday}),
+            (200, {'Cache-Control': 'must-revalidate', 'Expires': self.yesterday, 'Last-Modified': self.yesterday}),
+            (200, {'Cache-Control': 'max-age=86400,must-revalidate', 'Age': '86405'}),
         ]
         with self._middleware() as mw:
             for idx, (status, headers) in enumerate(sampledata):
@@ -410,6 +439,7 @@ class RFC2616PolicyTest(DefaultStorageTest):
                 res2 = self._process_requestresponse(mw, req0, res0b)
                 self.assertEqualResponse(res2, res0b)
                 assert 'cached' not in res2.flags
+                cc = headers.get('Cache-Control', '')
                 # Previous response expired too, subsequent request to same
                 # resource must revalidate and succeed on 304 if validators
                 # are present
@@ -418,7 +448,62 @@ class RFC2616PolicyTest(DefaultStorageTest):
                     res3 = self._process_requestresponse(mw, req0, res0c)
                     self.assertEqualResponse(res3, res0b)
                     assert 'cached' in res3.flags
+                    # get cached response on server errors unless must-revalidate
+                    # in cached response
+                    res0d = res0b.replace(status=500)
+                    res4 = self._process_requestresponse(mw, req0, res0d)
+                    if 'must-revalidate' in cc:
+                        assert 'cached' not in res4.flags
+                        self.assertEqualResponse(res4, res0d)
+                    else:
+                        assert 'cached' in res4.flags
+                        self.assertEqualResponse(res4, res0b)
+                # Requests with max-stale can fetch expired cached responses
+                # unless cached response has must-revalidate
+                req1 = req0.replace(headers={'Cache-Control': 'max-stale'})
+                res5 = self._process_requestresponse(mw, req1, res0b)
+                self.assertEqualResponse(res5, res0b)
+                if 'no-cache' in cc or 'must-revalidate' in cc:
+                    assert 'cached' not in res5.flags
+                else:
+                    assert 'cached' in res5.flags
 
+    def test_process_exception(self):
+        with self._middleware() as mw:
+            res0 = Response(self.request.url, headers={'Expires': self.yesterday})
+            req0 = Request(self.request.url)
+            self._process_requestresponse(mw, req0, res0)
+            for e in mw.DOWNLOAD_EXCEPTIONS:
+                # Simulate encountering an error on download attempts
+                assert mw.process_request(req0, self.spider) is None
+                res1 = mw.process_exception(req0, e('foo'), self.spider)
+                # Use cached response as recovery
+                assert 'cached' in res1.flags
+                self.assertEqualResponse(res0, res1)
+            # Do not use cached response for unhandled exceptions
+            mw.process_request(req0, self.spider)
+            assert mw.process_exception(req0, Exception('foo'), self.spider) is None
+
+    def test_ignore_response_cache_controls(self):
+        sampledata = [
+            (200, {'Date': self.yesterday, 'Expires': self.tomorrow}),
+            (200, {'Date': self.yesterday, 'Cache-Control': 'no-store,max-age=86405'}),
+            (200, {'Age': '299', 'Cache-Control': 'max-age=300,no-cache'}),
+            (300, {'Cache-Control': 'no-cache'}),
+            (200, {'Expires': self.tomorrow, 'Cache-Control': 'no-store'}),
+        ]
+        with self._middleware(HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS=['no-cache', 'no-store']) as mw:
+            for idx, (status, headers) in enumerate(sampledata):
+                req0 = Request('http://example-%d.com' % idx)
+                res0 = Response(req0.url, status=status, headers=headers)
+                # cache fresh response
+                res1 = self._process_requestresponse(mw, req0, res0)
+                self.assertEqualResponse(res1, res0)
+                assert 'cached' not in res1.flags
+                # return fresh cached response without network interaction
+                res2 = self._process_requestresponse(mw, req0, None)
+                self.assertEqualResponse(res1, res2)
+                assert 'cached' in res2.flags
 
 if __name__ == '__main__':
     unittest.main()

From 8771d1f79bccd8163e08185e591fd702e9f0b715 Mon Sep 17 00:00:00 2001
From: Marven Sanchez <marven.sanchez@gmail.com>
Date: Mon, 1 Jun 2015 18:20:59 +0800
Subject: [PATCH 0348/2974] Update HTTPCache middleware docs

---
 docs/topics/downloader-middleware.rst | 51 +++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 5cb6c9824..38533c47a 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -373,6 +373,18 @@ what is implemented:
 * Revalidate stale responses based on `Last-Modified` response header
 * Revalidate stale responses based on `ETag` response header
 * Set `Date` header for any received response missing it
+* Support `max-stale` cache-control directive in requests
+
+  This allows spiders to be configured with the full RFC2616 cache policy,
+  but avoid revalidation on a request-by-request basis, while remaining
+  conformant with the HTTP spec.
+
+  Example:
+
+  Add `Cache-Control: max-stale=600` to Request headers to accept responses that
+  have exceeded their expiration time by no more than 600 seconds.
+
+  See also: RFC2616, 14.9.3
 
 what is missing:
 
@@ -575,6 +587,45 @@ Default: ``False``
 If enabled, will compress all cached data with gzip.
 This setting is specific to the Filesystem backend.
 
+.. setting:: HTTPCACHE_ALWAYS_STORE
+
+HTTPCACHE_ALWAYS_STORE
+^^^^^^^^^^^^^^^^^^^^^^
+
+.. versionadded:: 0.25
+
+Default: ``False``
+
+If enabled, will cache pages unconditionally.
+
+A spider may wish to have all responses available in the cache, for
+future use with `Cache-Control: max-stale`, for instance. The
+DummyPolicy caches all responses but never revalidates them, and
+sometimes a more nuanced policy is desirable.
+
+This setting still respects `Cache-Control: no-store` directives in responses.
+If you don't want that, filter `no-store` out of the Cache-Control headers in
+responses you feedto the cache middleware.
+
+.. setting:: HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS
+
+HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. versionadded:: 0.25
+
+Default: ``[]``
+
+List of Cache-Control directives in responses to be ignored.
+
+Sites often set "no-store", "no-cache", "must-revalidate", etc., but get
+upset at the traffic a spider can generate if it respects those
+directives. This allows to selectively ignore Cache-Control directives
+that are known to be unimportant for the sites being crawled.
+
+We assume that the spider will not issue Cache-Control directives
+in requests unless it actually needs them, so directives in requests are
+not filtered.
 
 HttpCompressionMiddleware
 -------------------------

From bd2fe996aabe5467a89d9eff6831e6b0bf731be4 Mon Sep 17 00:00:00 2001
From: Ally Weir <ally.pcgf@gmail.com>
Date: Mon, 1 Jun 2015 16:42:51 +0100
Subject: [PATCH 0349/2974] Spelling correction

incorrect use of "too" instead of "to"
---
 docs/faq.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 212baa585..2e61f44ee 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -190,7 +190,7 @@ Or by setting a global download delay in your project with the
 Can I call ``pdb.set_trace()`` from my spiders to debug them?
 -------------------------------------------------------------
 
-Yes, but you can also use the Scrapy shell which allows you too quickly analyze
+Yes, but you can also use the Scrapy shell which allows you to quickly analyze
 (and even modify) the response being processed by your spider, which is, quite
 often, more useful than plain old ``pdb.set_trace()``.
 

From ffc7b7fd6c22264b5cd6f25c5165e04f0f070cff Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 29 May 2015 15:04:29 -0300
Subject: [PATCH 0350/2974] Add helper to update deprecated class paths

---
 scrapy/utils/conf.py          | 22 ++++++++++++++++++----
 scrapy/utils/deprecate.py     | 34 ++++++++++++++++++++++++++++++++++
 tests/test_utils_conf.py      | 33 +++++++++++++++++++++++++++++----
 tests/test_utils_deprecate.py | 27 ++++++++++++++++++++++++++-
 4 files changed, 107 insertions(+), 9 deletions(-)

diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index b883923b9..423ca43e2 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -5,16 +5,30 @@ from operator import itemgetter
 import six
 from six.moves.configparser import SafeConfigParser
 
+from scrapy.utils.deprecate import update_classpath
 
-def build_component_list(base, custom):
+
+def build_component_list(base, custom, convert=update_classpath):
     """Compose a component list based on a custom and base dict of components
     (typically middlewares or extensions), unless custom is already a list, in
     which case it's returned.
     """
+
+    def _check_components(complist):
+        if len({convert(c) for c in complist}) != len(complist):
+            raise ValueError('Some paths in {!r} convert to the same object, '
+                             'please update your settings'.format(complist))
+
     if isinstance(custom, (list, tuple)):
-        return custom
-    compdict = base.copy()
-    compdict.update(custom)
+        _check_components(custom)
+        return type(custom)(convert(c) for c in custom)
+
+    def _map_keys(compdict):
+        _check_components(compdict)
+        return {convert(k): v for k, v in six.iteritems(compdict)}
+
+    compdict = _map_keys(base)
+    compdict.update(_map_keys(custom))
     items = (x for x in six.iteritems(compdict) if x[1] is not None)
     return [x[0] for x in sorted(items, key=itemgetter(1))]
 
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index 6f1940611..37e94fae9 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -121,3 +121,37 @@ def _clspath(cls, forced=None):
     if forced is not None:
         return forced
     return '{}.{}'.format(cls.__module__, cls.__name__)
+
+
+DEPRECATION_RULES = [
+    ('scrapy.contrib_exp.downloadermiddleware.decompression.', 'scrapy.downloadermiddlewares.decompression.'),
+    ('scrapy.contrib_exp.iterators.', 'scrapy.utils.iterators.'),
+    ('scrapy.contrib.downloadermiddleware.', 'scrapy.downloadermiddlewares.'),
+    ('scrapy.contrib.exporter.', 'scrapy.exporters.'),
+    ('scrapy.contrib.linkextractors.', 'scrapy.linkextractors.'),
+    ('scrapy.contrib.loader.processor.', 'scrapy.loader.processors.'),
+    ('scrapy.contrib.loader.', 'scrapy.loader.'),
+    ('scrapy.contrib.pipeline.', 'scrapy.pipelines.'),
+    ('scrapy.contrib.spidermiddleware.', 'scrapy.spidermiddlewares.'),
+    ('scrapy.contrib.spiders.', 'scrapy.spiders.'),
+    ('scrapy.contrib.', 'scrapy.extensions.'),
+    ('scrapy.command.', 'scrapy.commands.'),
+    ('scrapy.dupefilter.', 'scrapy.dupefilters.'),
+    ('scrapy.linkextractor.', 'scrapy.linkextractors.'),
+    ('scrapy.spider.', 'scrapy.spiders.'),
+    ('scrapy.squeue.', 'scrapy.squeues.'),
+    ('scrapy.statscol.', 'scrapy.statscollectors.'),
+    ('scrapy.utils.decorator.', 'scrapy.utils.decorators.'),
+    ('scrapy.spidermanager.SpiderManager', 'scrapy.spiderloader.SpiderLoader'),
+]
+
+
+def update_classpath(path):
+    """Update a deprecated path from an object with its new location"""
+    for prefix, replacement in DEPRECATION_RULES:
+        if path.startswith(prefix):
+            new_path = path.replace(prefix, replacement, 1)
+            warnings.warn("`{}` class is deprecated, use `{}` instead".format(path, new_path),
+                          ScrapyDeprecationWarning)
+            return new_path
+    return path
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 35277a423..177d024fb 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -2,16 +2,41 @@ import unittest
 
 from scrapy.utils.conf import build_component_list, arglist_to_dict
 
-class UtilsConfTestCase(unittest.TestCase):
 
-    def test_build_component_list(self):
+class BuildComponentListTest(unittest.TestCase):
+
+    def test_build_dict(self):
         base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None}
         custom = {'two': None, 'three': 8, 'four': 4}
-        self.assertEqual(build_component_list(base, custom),
+        self.assertEqual(build_component_list(base, custom, lambda x: x),
                          ['one', 'four', 'five', 'three'])
 
+    def test_return_list(self):
         custom = ['a', 'b', 'c']
-        self.assertEqual(build_component_list(base, custom), custom)
+        self.assertEqual(build_component_list(None, custom, lambda x: x),
+                         custom)
+
+    def test_map_dict(self):
+        custom = {'one': 1, 'two': 2, 'three': 3}
+        self.assertEqual(build_component_list({}, custom, lambda x: x.upper()),
+                         ['ONE', 'TWO', 'THREE'])
+
+    def test_map_list(self):
+        custom = ['a', 'b', 'c']
+        self.assertEqual(build_component_list(None, custom, lambda x: x.upper()),
+                         ['A', 'B', 'C'])
+
+    def test_duplicate_components_in_dict(self):
+        duplicate_dict = {'one': 1, 'two': 2, 'ONE': 4}
+        self.assertRaises(ValueError,
+                          build_component_list, {}, duplicate_dict, lambda x: x.lower())
+
+    def test_duplicate_components_in_list(self):
+        duplicate_list = ['a', 'b', 'a']
+        self.assertRaises(ValueError,
+                          build_component_list, None, duplicate_list, lambda x: x)
+
+class UtilsConfTestCase(unittest.TestCase):
 
     def test_arglist_to_dict(self):
         self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']),
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 3d94bc1a0..41b8100d7 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -3,7 +3,7 @@ from __future__ import absolute_import
 import inspect
 import unittest
 import warnings
-from scrapy.utils.deprecate import create_deprecated_class
+from scrapy.utils.deprecate import create_deprecated_class, update_classpath
 
 from tests import mock
 
@@ -248,3 +248,28 @@ class WarnWhenSubclassedTest(unittest.TestCase):
                     pass
 
         self.assertIn("Error detecting parent module", str(w[0].message))
+
+
+@mock.patch('scrapy.utils.deprecate.DEPRECATION_RULES',
+            [('scrapy.contrib.pipeline.', 'scrapy.pipelines.'),
+             ('scrapy.contrib.', 'scrapy.extensions.')])
+class UpdateClassPathTest(unittest.TestCase):
+
+    def test_old_path_gets_fixed(self):
+        with warnings.catch_warnings(record=True) as w:
+            output = update_classpath('scrapy.contrib.debug.Debug')
+        self.assertEqual(output, 'scrapy.extensions.debug.Debug')
+        self.assertEqual(len(w), 1)
+        self.assertIn("scrapy.contrib.debug.Debug", str(w[0].message))
+        self.assertIn("scrapy.extensions.debug.Debug", str(w[0].message))
+
+    def test_sorted_replacement(self):
+        with warnings.catch_warnings(record=True):
+            output = update_classpath('scrapy.contrib.pipeline.Pipeline')
+        self.assertEqual(output, 'scrapy.pipelines.Pipeline')
+
+    def test_unmatched_path_stays_the_same(self):
+        with warnings.catch_warnings(record=True) as w:
+            output = update_classpath('scrapy.unmatched.Path')
+        self.assertEqual(output, 'scrapy.unmatched.Path')
+        self.assertEqual(len(w), 0)

From cc2f3e1b46b6036fa78bdcc7686bd4ffa3774dbf Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 3 Jun 2015 04:26:20 +0500
Subject: [PATCH 0351/2974] TST a test case to show custom_settings doesn't
 always work

---
 tests/test_crawler.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 53f517c27..1c0804675 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,11 +1,13 @@
 import warnings
 import unittest
 
+import scrapy
 from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
 from scrapy.settings import Settings, default_settings
 from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
+from scrapy.extensions.throttle import AutoThrottle
 
 
 class CrawlerTestCase(unittest.TestCase):
@@ -52,6 +54,18 @@ class CrawlerTestCase(unittest.TestCase):
         self.assertIsInstance(crawler.settings, Settings)
 
 
+class SpiderSettingsTestCase(unittest.TestCase):
+    def test_spider_custom_settings(self):
+        class MySpider(scrapy.Spider):
+            name = 'spider'
+            custom_settings = {
+                'AUTOTHROTTLE_ENABLED': True
+            }
+
+        crawler = Crawler(MySpider, {})
+        enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
+        self.assertIn(AutoThrottle, enabled_exts)
+
 
 class SpiderLoaderWithWrongInterface(object):
 

From d42c420a6d45a818f59db71a3f9b30d88625598b Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 3 Jun 2015 04:29:10 +0500
Subject: [PATCH 0352/2974] fixed spider custom_settings

https://github.com/scrapy/scrapy/pull/1128 moved spidercls.update_settings
call to a later stage; this commit moves it back.
---
 scrapy/crawler.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 2cd658276..204395c77 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -29,6 +29,7 @@ class Crawler(object):
 
         self.spidercls = spidercls
         self.settings = settings.copy()
+        self.spidercls.update_settings(self.settings)
 
         self.signals = SignalManager(self)
         self.stats = load_object(self.settings['STATS_CLASS'])(self)
@@ -44,9 +45,7 @@ class Crawler(object):
         self.logformatter = lf_cls.from_crawler(self)
         self.extensions = ExtensionManager.from_crawler(self)
 
-        self.spidercls.update_settings(self.settings)
         self.settings.freeze()
-
         self.crawling = False
         self.spider = None
         self.engine = None

From 367ea81e716c5e08d6f47ed84d5b0bfb3211cf0c Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Thu, 4 Jun 2015 03:51:48 +0800
Subject: [PATCH 0353/2974] Remove deprecated %z formatting from the default
 LOG_DATEFORMAT

---
 docs/topics/settings.rst            | 2 +-
 scrapy/settings/default_settings.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index ac4f8d6d5..a9eba41ce 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -629,7 +629,7 @@ placeholders.
 LOG_DATEFORMAT
 --------------
 
-Default: ``'%Y-%m-%d %H:%M:%S%z'``
+Default: ``'%Y-%m-%d %H:%M:%S'``
 
 String for formatting date/time, expansion of the ``%(asctime)s`` placeholder
 in :setting:`LOG_FORMAT`. Refer to the `Python datetime documentation`_ for the whole list of available
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 7459c4d73..0ea014007 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -168,7 +168,7 @@ LOG_ENABLED = True
 LOG_ENCODING = 'utf-8'
 LOG_FORMATTER = 'scrapy.logformatter.LogFormatter'
 LOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'
-LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S%z'
+LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'
 LOG_STDOUT = False
 LOG_LEVEL = 'DEBUG'
 LOG_FILE = None

From eae25a04d96877a6d74627492307619d38c96e5c Mon Sep 17 00:00:00 2001
From: Chris Nilsson <christopher@otherchirps.net>
Date: Sat, 6 Jun 2015 00:39:14 +1000
Subject: [PATCH 0354/2974] Added MEMUSAGE_CHECK_INTERVAL_SECONDS to Memory
 usage extension options.

Kept the default as it was, at 60.0 seconds. But added a setting to
allow this to be changed as desired.
---
 docs/topics/extensions.rst    |  1 +
 docs/topics/settings.rst      | 14 ++++++++++++++
 scrapy/extensions/memusage.py |  7 ++++---
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index d5d985087..fb5220e9d 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -222,6 +222,7 @@ can be configured with the following settings:
 * :setting:`MEMUSAGE_WARNING_MB`
 * :setting:`MEMUSAGE_NOTIFY_MAIL`
 * :setting:`MEMUSAGE_REPORT`
+* :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS`
 
 Memory debugger extension
 ~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index a9eba41ce..b1dee667e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -711,6 +711,20 @@ Scrapy  (if MEMUSAGE_ENABLED is True). If zero, no check will be performed.
 
 See :ref:`topics-extensions-ref-memusage`.
 
+.. setting:: MEMUSAGE_CHECK_INTERVAL_SECONDS
+
+MEMUSAGE_CHECK_INTERVAL_SECONDS
+-------------------------------
+
+Default: ``60.0``
+
+Scope: ``scrapy.extensions.memusage``
+
+The frequence which the current memory usage will be checked against the 
+limits set by :setting:`MEMUSAGE_LIMIT_MB` and :setting:`MEMUSAGE_WARNING_MB`.
+
+See :ref:`topics-extensions-ref-memusage`.
+
 .. setting:: MEMUSAGE_NOTIFY_MAIL
 
 MEMUSAGE_NOTIFY_MAIL
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 402c78b68..34bad8246 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -36,6 +36,7 @@ class MemoryUsage(object):
         self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
         self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
         self.report = crawler.settings.getbool('MEMUSAGE_REPORT')
+        self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS', 60.0)
         self.mail = MailSender.from_settings(crawler.settings)
         crawler.signals.connect(self.engine_started, signal=signals.engine_started)
         crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)
@@ -56,15 +57,15 @@ class MemoryUsage(object):
         self.tasks = []
         tsk = task.LoopingCall(self.update)
         self.tasks.append(tsk)
-        tsk.start(60.0, now=True)
+        tsk.start(self.check_interval, now=True)
         if self.limit:
             tsk = task.LoopingCall(self._check_limit)
             self.tasks.append(tsk)
-            tsk.start(60.0, now=True)
+            tsk.start(self.check_interval, now=True)
         if self.warning:
             tsk = task.LoopingCall(self._check_warning)
             self.tasks.append(tsk)
-            tsk.start(60.0, now=True)
+            tsk.start(self.check_interval, now=True)
 
     def engine_stopped(self):
         for tsk in self.tasks:

From 24d8a852697cd8cf4fe7b940e6f224e9438639f2 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 5 Jun 2015 17:10:18 -0300
Subject: [PATCH 0355/2974] Update release notes for 1.0.0rc2

(cherry picked from commit 6e61d54168cf471363be3e7e54d75ad544b9f6e1)
---
 docs/news.rst | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 8686398eb..da7e64e15 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -233,7 +233,7 @@ Changelog
 New Features and Enhancements
 
 - Python logging (:issue:`1060`, :issue:`1235`, :issue:`1236`, :issue:`1240`,
-  :issue:`1259`)
+  :issue:`1259`, :issue:`1278`)
 - FEED_EXPORT_FIELDS option (:issue:`1159`, :issue:`1224`)
 - Dns cache size and timeout options (:issue:`1132`)
 - support namespace prefix in xmliter_lxml (:issue:`963`)
@@ -256,7 +256,8 @@ New Features and Enhancements
 - GSoC Per-spider settings (:issue:`854`)
 - Add project name validation (:issue:`817`)
 - GSoC API cleanup (:issue:`816`, :issue:`1128`, :issue:`1147`,
-  :issue:`1148`, :issue:`1156`, :issue:`1185`, :issue:`1187`, :issue:`1258`)
+  :issue:`1148`, :issue:`1156`, :issue:`1185`, :issue:`1187`, :issue:`1258`,
+  :issue:`1268`, :issue:`1276`)
 - Be more responsive with IO operations (:issue:`1074` and :issue:`1075`)
 
 Deprecations and Removals
@@ -280,6 +281,8 @@ Relocations
 - Deleted bin folder from root, fixes #913 (:issue:`914`)
 - Remove jsonrpc based webservice (:issue:`859`)
 - Move Test cases under project root dir (:issue:`827`, :issue:`841`)
+- Fix backward incompatibility for relocated paths in settings
+  (:issue:`1267`)
 
 Documentation
 
@@ -303,7 +306,7 @@ Documentation
   :issue:`1196`, :issue:`1172`, :issue:`1171`, :issue:`1169`, :issue:`1160`,
   :issue:`1154`, :issue:`1127`, :issue:`1112`, :issue:`1105`, :issue:`1041`,
   :issue:`1082`, :issue:`1033`, :issue:`944`, :issue:`866`, :issue:`864`,
-  :issue:`796`)
+  :issue:`796`, :issue:`1260`, :issue:`1271`)
 
 Bugfixes
 

From 33d145e2f5a1f470398d96184b4da158e3ef7240 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 6 Jun 2015 02:49:39 +0500
Subject: [PATCH 0356/2974] CrawlerProcess cleanup

* remove unneeded lambda;
* extract _get_dns_resolver method and format code to pep8.
---
 scrapy/crawler.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 204395c77..9ab233573 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -242,16 +242,25 @@ class CrawlerProcess(CrawlerRunner):
             # Don't start the reactor if the deferreds are already fired
             if d.called:
                 return
-            d.addBoth(lambda _: self._stop_reactor())
+            d.addBoth(self._stop_reactor)
 
-        cache_size = self.settings.getint('DNSCACHE_SIZE') if self.settings.getbool('DNSCACHE_ENABLED') else 0
-        reactor.installResolver(CachingThreadedResolver(reactor, cache_size,
-                                                            self.settings.getfloat('DNS_TIMEOUT')))
+        reactor.installResolver(self._get_dns_resolver())
         tp = reactor.getThreadPool()
         tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
         reactor.run(installSignalHandlers=False)  # blocking call
 
+    def _get_dns_resolver(self):
+        if self.settings.getbool('DNSCACHE_ENABLED'):
+            cache_size = self.settings.getint('DNSCACHE_SIZE')
+        else:
+            cache_size = 0
+        return CachingThreadedResolver(
+            reactor=reactor,
+            cache_size=cache_size,
+            timeout=self.settings.getfloat('DNS_TIMEOUT')
+        )
+
     def _stop_reactor(self, _=None):
         try:
             reactor.stop()

From 64399d18d81bda9fdb1771627949d36f9f412981 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 6 Jun 2015 02:53:36 +0500
Subject: [PATCH 0357/2974] Stop reactor on Ctrl-C regardless of
 'stop_after_crawl'. Fixes GH-1279.

---
 scrapy/crawler.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 9ab233573..21b3c748f 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -216,7 +216,7 @@ class CrawlerProcess(CrawlerRunner):
         signame = signal_names[signum]
         logger.info("Received %(signame)s, shutting down gracefully. Send again to force ",
                     {'signame': signame})
-        reactor.callFromThread(self.stop)
+        reactor.callFromThread(self._graceful_stop_reactor)
 
     def _signal_kill(self, signum, _):
         install_shutdown_handlers(signal.SIG_IGN)
@@ -261,6 +261,11 @@ class CrawlerProcess(CrawlerRunner):
             timeout=self.settings.getfloat('DNS_TIMEOUT')
         )
 
+    def _graceful_stop_reactor(self):
+        d = self.stop()
+        d.addBoth(self._stop_reactor)
+        return d
+
     def _stop_reactor(self, _=None):
         try:
             reactor.stop()

From d047665c027d0a88dedc515098d276eed481c134 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 6 Jun 2015 03:23:13 +0500
Subject: [PATCH 0358/2974] make "settings" argument optional for Crawler,
 CrawlerRunner and CrawlerProcess

---
 docs/topics/practices.rst | 10 ++++-----
 scrapy/crawler.py         | 10 ++++-----
 tests/test_crawler.py     | 43 +++++++++++++++++++++++----------------
 3 files changed, 34 insertions(+), 29 deletions(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 7bdd28573..4c8c83924 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -94,9 +94,7 @@ reactor after `MySpider` has finished running.
         ...
 
     configure_logging({'LOG_FORMAT': '%(levelname)s: %(message)s'})
-    runner = CrawlerRunner({
-        'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
-    })
+    runner = CrawlerRunner()
 
     d = runner.crawl(MySpider)
     d.addBoth(lambda _: reactor.stop())
@@ -128,7 +126,7 @@ Here is an example that runs multiple spiders simultaneously:
         # Your second spider definition
         ...
 
-    process = CrawlerProcess({})
+    process = CrawlerProcess()
     process.crawl(MySpider1)
     process.crawl(MySpider2)
     process.start() # the script will block here until all crawling jobs are finished
@@ -151,7 +149,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
         ...
 
     configure_logging({})
-    runner = CrawlerRunner({})
+    runner = CrawlerRunner()
     runner.crawl(MySpider1)
     runner.crawl(MySpider2)
     d = runner.join()
@@ -176,7 +174,7 @@ Same example but running the spiders sequentially by chaining the deferreds:
         ...
 
     configure_logging({})
-    runner = CrawlerRunner({})
+    runner = CrawlerRunner()
 
     @defer.inlineCallbacks
     def crawl():
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 204395c77..3e18182c0 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -23,8 +23,8 @@ logger = logging.getLogger(__name__)
 
 class Crawler(object):
 
-    def __init__(self, spidercls, settings):
-        if isinstance(settings, dict):
+    def __init__(self, spidercls, settings=None):
+        if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
 
         self.spidercls = spidercls
@@ -108,8 +108,8 @@ class CrawlerRunner(object):
             ":meth:`crawl` and managed by this class."
     )
 
-    def __init__(self, settings):
-        if isinstance(settings, dict):
+    def __init__(self, settings=None):
+        if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
         self.settings = settings
         self.spider_loader = _get_spider_loader(settings)
@@ -205,7 +205,7 @@ class CrawlerProcess(CrawlerRunner):
     process. See :ref:`run-from-script` for an example.
     """
 
-    def __init__(self, settings):
+    def __init__(self, settings=None):
         super(CrawlerProcess, self).__init__(settings)
         install_shutdown_handlers(self._signal_shutdown)
         configure_logging(self.settings)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 1c0804675..53a1202e3 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -10,7 +10,14 @@ from scrapy.utils.misc import load_object
 from scrapy.extensions.throttle import AutoThrottle
 
 
-class CrawlerTestCase(unittest.TestCase):
+class BaseCrawlerTest(unittest.TestCase):
+
+    def assertOptionIsDefault(self, settings, key):
+        self.assertIsInstance(settings, Settings)
+        self.assertEqual(settings[key], getattr(default_settings, key))
+
+
+class CrawlerTestCase(BaseCrawlerTest):
 
     def setUp(self):
         self.crawler = Crawler(DefaultSpider, Settings())
@@ -47,11 +54,11 @@ class CrawlerTestCase(unittest.TestCase):
     def test_crawler_accepts_dict(self):
         crawler = Crawler(DefaultSpider, {'foo': 'bar'})
         self.assertEqual(crawler.settings['foo'], 'bar')
-        self.assertEqual(
-            crawler.settings['RETRY_ENABLED'],
-            default_settings.RETRY_ENABLED
-        )
-        self.assertIsInstance(crawler.settings, Settings)
+        self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
+
+    def test_crawler_accepts_None(self):
+        crawler = Crawler(DefaultSpider)
+        self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
 
 
 class SpiderSettingsTestCase(unittest.TestCase):
@@ -77,7 +84,7 @@ class CustomSpiderLoader(SpiderLoader):
     pass
 
 
-class CrawlerRunnerTestCase(unittest.TestCase):
+class CrawlerRunnerTestCase(BaseCrawlerTest):
 
     def test_spider_manager_verify_interface(self):
         settings = Settings({
@@ -93,11 +100,11 @@ class CrawlerRunnerTestCase(unittest.TestCase):
     def test_crawler_runner_accepts_dict(self):
         runner = CrawlerRunner({'foo': 'bar'})
         self.assertEqual(runner.settings['foo'], 'bar')
-        self.assertEqual(
-            runner.settings['RETRY_ENABLED'],
-            default_settings.RETRY_ENABLED
-        )
-        self.assertIsInstance(runner.settings, Settings)
+        self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED')
+
+    def test_crawler_runner_accepts_None(self):
+        runner = CrawlerRunner()
+        self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED')
 
     def test_deprecated_attribute_spiders(self):
         with warnings.catch_warnings(record=True) as w:
@@ -119,12 +126,12 @@ class CrawlerRunnerTestCase(unittest.TestCase):
             self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message))
 
 
-class CrawlerProcessTest(unittest.TestCase):
+class CrawlerProcessTest(BaseCrawlerTest):
     def test_crawler_process_accepts_dict(self):
         runner = CrawlerProcess({'foo': 'bar'})
         self.assertEqual(runner.settings['foo'], 'bar')
-        self.assertEqual(
-            runner.settings['RETRY_ENABLED'],
-            default_settings.RETRY_ENABLED
-        )
-        self.assertIsInstance(runner.settings, Settings)
+        self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED')
+
+    def test_crawler_process_accepts_None(self):
+        runner = CrawlerProcess()
+        self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED')

From 0c532baf4c76bc902ead3edd2ca0645e1bc66676 Mon Sep 17 00:00:00 2001
From: Chris Nilsson <christopher@otherchirps.net>
Date: Sat, 6 Jun 2015 11:18:13 +1000
Subject: [PATCH 0359/2974] Removed typo, and clarified time unit of setting

---
 docs/topics/settings.rst | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index b1dee667e..f89aa30c8 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -720,8 +720,12 @@ Default: ``60.0``
 
 Scope: ``scrapy.extensions.memusage``
 
-The frequence which the current memory usage will be checked against the 
-limits set by :setting:`MEMUSAGE_LIMIT_MB` and :setting:`MEMUSAGE_WARNING_MB`.
+The :ref:`Memory usage extension <topics-extensions-ref-memusage>`
+checks the current memory usage, versus the limits set by 
+:setting:`MEMUSAGE_LIMIT_MB` and :setting:`MEMUSAGE_WARNING_MB`, 
+at fixed time intervals.
+
+This sets the length of these intervals, in seconds.
 
 See :ref:`topics-extensions-ref-memusage`.
 

From 61dec83f706665e5c5238a17f2efdca3d642d9db Mon Sep 17 00:00:00 2001
From: Chris Nilsson <christopher@otherchirps.net>
Date: Sat, 6 Jun 2015 11:19:29 +1000
Subject: [PATCH 0360/2974] Moved default value of
 MEMUSAGE_CHECK_INTERVAL_SECONDS to default_settings

---
 scrapy/extensions/memusage.py       | 2 +-
 scrapy/settings/default_settings.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 34bad8246..322213cf0 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -36,7 +36,7 @@ class MemoryUsage(object):
         self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
         self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
         self.report = crawler.settings.getbool('MEMUSAGE_REPORT')
-        self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS', 60.0)
+        self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
         self.mail = MailSender.from_settings(crawler.settings)
         crawler.signals.connect(self.engine_started, signal=signals.engine_started)
         crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 0ea014007..79b6e44f0 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -186,6 +186,7 @@ MAIL_USER = None
 MEMDEBUG_ENABLED = False        # enable memory debugging
 MEMDEBUG_NOTIFY = []            # send memory debugging report by mail at engine shutdown
 
+MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
 MEMUSAGE_ENABLED = False
 MEMUSAGE_LIMIT_MB = 0
 MEMUSAGE_NOTIFY_MAIL = []

From e575f4444657261cfec39b977f3efbdf0e34961c Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Mon, 8 Jun 2015 10:52:42 +0200
Subject: [PATCH 0361/2974] [settings/default_settings.py] dont retry 400

As in HTTP specs:

"10.4.1 400 Bad Request

The request could not be understood by the server due to malformed
syntax. The client SHOULD NOT repeat the request without
modifications."

Scrapy should not retry 400 by default.
---
 scrapy/settings/default_settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 0ea014007..62c3e4c94 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -209,7 +209,7 @@ REFERER_ENABLED = True
 
 RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
-RETRY_HTTP_CODES = [500, 502, 503, 504, 400, 408]
+RETRY_HTTP_CODES = [500, 502, 503, 504, 408]
 RETRY_PRIORITY_ADJUST = -1
 
 ROBOTSTXT_OBEY = False

From 3cbf8a0b2ba58afc83ac30a012ed23852cc60ad4 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 8 Jun 2015 18:35:44 +0500
Subject: [PATCH 0362/2974] extract CrawlerRunner._crawl method which always
 expects Crawler

It provides an extension point where crawler instance is available;
it should make it easier to write alternative CrawlerRunner.crawl
implementations.

See also: https://github.com/scrapy/scrapy/pull/1256
---
 scrapy/crawler.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 3e18182c0..1740e8ba6 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -148,7 +148,9 @@ class CrawlerRunner(object):
         crawler = crawler_or_spidercls
         if not isinstance(crawler_or_spidercls, Crawler):
             crawler = self._create_crawler(crawler_or_spidercls)
+        return self._crawl(crawler, *args, **kwargs)
 
+    def _crawl(self, crawler, *args, **kwargs):
         self.crawlers.add(crawler)
         d = crawler.crawl(*args, **kwargs)
         self._active.add(d)

From 9a787893e3dccd8bcbfa6821806fde2e2ef4515e Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 6 Jun 2015 04:05:54 +0500
Subject: [PATCH 0363/2974] (backwards-incompatible) allow to pass
 settings=None to configure_logging

* use explicit argument for disabling root handler;
* handle LOG_STDOUT even if install_root_handler is False
---
 docs/topics/logging.rst   | 23 +++++++++------
 docs/topics/practices.rst |  4 +--
 scrapy/utils/log.py       | 59 ++++++++++++++++++++++-----------------
 3 files changed, 49 insertions(+), 37 deletions(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 1a3f5d69f..469f0c81d 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -194,7 +194,7 @@ scrapy.utils.log module
 .. module:: scrapy.utils.log
    :synopsis: Logging utils
 
-.. function:: configure_logging(settings=None)
+.. function:: configure_logging(settings=None, install_root_handler=True)
 
     This function initializes logging defaults for Scrapy.
 
@@ -203,16 +203,17 @@ scrapy.utils.log module
     not required but it's recommended.
 
     This function does:
-      - Route warnings and Twisted logging through Python standard logging
-      - Set a filter on Scrapy logger for formatting Twisted failures
-      - Assign DEBUG and ERROR levels to Scrapy and Twisted loggers
-        respectively
+      - Route warnings and twisted logging through Python standard logging
+      - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively
+      - Route stdout to log if LOG_STDOUT setting is True
 
-    If `settings` is not ``None``, it will also create a root handler based on
-    the settings listed in :ref:`topics-logging-settings`.
+    When ``install_root_handler`` is True (default), this function also
+      - Sets FailureFormatter filter on Scrapy logger
+      - Creates a root handler based on the settings listed
+        in :ref:`topics-logging-settings`.
 
     If you plan on configuring the handlers yourself is still recommended you
-    call this function, keeping `settings` as ``None``. Bear in mind there
+    call this function, passing `install_root_handler=False`. Bear in mind there
     won't be any log output set by default in that case.
 
     To get you started on manually configuring logging's output, you can use
@@ -222,7 +223,7 @@ scrapy.utils.log module
         import logging
         from scrapy.utils.log import configure_logging
 
-        configure_logging()  # Note we aren't providing settings in this case
+        configure_logging(install_root_handler=False)
         logging.basicConfig(filename='log.txt', format='%(levelname)s: %(message)s', level=logging.INFO)
 
     Refer to :ref:`run-from-script` for more details about using Scrapy this
@@ -232,4 +233,8 @@ scrapy.utils.log module
         root logger.
     :type settings: :class:`~scrapy.settings.Settings` object or ``None``
 
+    :param install_root_handler: whether to install root logging handler
+        (default: True)
+    :type install_root_handler: bool
+
 .. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 4c8c83924..7666a84cf 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -148,7 +148,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
         # Your second spider definition
         ...
 
-    configure_logging({})
+    configure_logging()
     runner = CrawlerRunner()
     runner.crawl(MySpider1)
     runner.crawl(MySpider2)
@@ -173,7 +173,7 @@ Same example but running the spiders sequentially by chaining the deferreds:
         # Your second spider definition
         ...
 
-    configure_logging({})
+    configure_logging()
     runner = CrawlerRunner()
 
     @defer.inlineCallbacks
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 931e28f2d..0102d1740 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -56,14 +56,17 @@ DEFAULT_LOGGING = {
 }
 
 
-def configure_logging(settings=None):
+def configure_logging(settings=None, install_root_handler=True):
     """Initialize and configure default loggers
 
     This function does:
       - Route warnings and twisted logging through Python standard logging
-      - Set FailureFormatter filter on Scrapy logger
       - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively
-      - Create a handler for the root logger according to given settings
+      - Route stdout to log if LOG_STDOUT setting is True
+
+    When ``install_root_handler`` is True (default), this function also
+      - Sets FailureFormatter filter on Scrapy logger
+      - Creates a handler for the root logger according to given settings
     """
     if not sys.warnoptions:
         # Route warnings through python logging
@@ -74,35 +77,39 @@ def configure_logging(settings=None):
 
     dictConfig(DEFAULT_LOGGING)
 
-    if isinstance(settings, dict):
+    if isinstance(settings, dict) or settings is None:
         settings = Settings(settings)
 
-    if settings:
+    if settings.getbool('LOG_STDOUT'):
+        sys.stdout = StreamLogger(logging.getLogger('stdout'))
+
+    if install_root_handler:
         logging.root.setLevel(logging.NOTSET)
-
-        if settings.getbool('LOG_STDOUT'):
-            sys.stdout = StreamLogger(logging.getLogger('stdout'))
-
-        # Set up the default log handler
-        filename = settings.get('LOG_FILE')
-        if filename:
-            encoding = settings.get('LOG_ENCODING')
-            handler = logging.FileHandler(filename, encoding=encoding)
-        elif settings.getbool('LOG_ENABLED'):
-            handler = logging.StreamHandler()
-        else:
-            handler = logging.NullHandler()
-
-        formatter = logging.Formatter(
-            fmt=settings.get('LOG_FORMAT'),
-            datefmt=settings.get('LOG_DATEFORMAT')
-        )
-        handler.setFormatter(formatter)
-        handler.setLevel(settings.get('LOG_LEVEL'))
-        handler.addFilter(TopLevelFormatter(['scrapy']))
+        handler = _get_handler(settings)
         logging.root.addHandler(handler)
 
 
+def _get_handler(settings):
+    """ Return a log handler object according to settings """
+    filename = settings.get('LOG_FILE')
+    if filename:
+        encoding = settings.get('LOG_ENCODING')
+        handler = logging.FileHandler(filename, encoding=encoding)
+    elif settings.getbool('LOG_ENABLED'):
+        handler = logging.StreamHandler()
+    else:
+        handler = logging.NullHandler()
+
+    formatter = logging.Formatter(
+        fmt=settings.get('LOG_FORMAT'),
+        datefmt=settings.get('LOG_DATEFORMAT')
+    )
+    handler.setFormatter(formatter)
+    handler.setLevel(settings.get('LOG_LEVEL'))
+    handler.addFilter(TopLevelFormatter(['scrapy']))
+    return handler
+
+
 def log_scrapy_info(settings):
     logger.info("Scrapy %(version)s started (bot: %(bot)s)",
                 {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})

From 1740fcf1a6236ca7b847be2206b40b088dceec8c Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 8 Jun 2015 21:05:58 +0500
Subject: [PATCH 0364/2974] DOC SignalManager docstrings. See GH-713.

This change is not 100% backwards compatible because of *args changes.
Their usage was not documented, so we're not breaking public interface.
---
 docs/topics/api.rst     | 52 +++-------------------------
 scrapy/signalmanager.py | 76 ++++++++++++++++++++++++++++++++---------
 2 files changed, 64 insertions(+), 64 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index e59fe9a58..f3a03bb74 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -358,54 +358,10 @@ SpiderLoader API
 Signals API
 ===========
 
-.. module:: scrapy.signalmanager
-   :synopsis: The signal manager
-
-.. class:: SignalManager
-
-    .. method:: connect(receiver, signal)
-
-        Connect a receiver function to a signal.
-
-        The signal can be any object, although Scrapy comes with some
-        predefined signals that are documented in the :ref:`topics-signals`
-        section.
-
-        :param receiver: the function to be connected
-        :type receiver: callable
-
-        :param signal: the signal to connect to
-        :type signal: object
-
-    .. method:: send_catch_log(signal, \*\*kwargs)
-
-        Send a signal, catch exceptions and log them.
-
-        The keyword arguments are passed to the signal handlers (connected
-        through the :meth:`connect` method).
-
-    .. method:: send_catch_log_deferred(signal, \*\*kwargs)
-
-        Like :meth:`send_catch_log` but supports returning `deferreds`_ from
-        signal handlers.
-
-        Returns a `deferred`_ that gets fired once all signal handlers
-        deferreds were fired. Send a signal, catch exceptions and log them.
-
-        The keyword arguments are passed to the signal handlers (connected
-        through the :meth:`connect` method).
-
-    .. method:: disconnect(receiver, signal)
-
-        Disconnect a receiver function from a signal. This has the opposite
-        effect of the :meth:`connect` method, and the arguments are the same.
-
-    .. method:: disconnect_all(signal)
-
-        Disconnect all receivers from the given signal.
-
-        :param signal: the signal to disconnect from
-        :type signal: object
+.. automodule:: scrapy.signalmanager
+    :synopsis: The signal manager
+    :members:
+    :undoc-members:
 
 .. _topics-api-stats:
 
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index 4a3e3d92d..90a14bd63 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -1,27 +1,71 @@
+from __future__ import absolute_import
 from scrapy.xlib.pydispatch import dispatcher
-from scrapy.utils import signal
+from scrapy.utils import signal as _signal
+
 
 class SignalManager(object):
 
     def __init__(self, sender=dispatcher.Anonymous):
         self.sender = sender
 
-    def connect(self, *a, **kw):
-        kw.setdefault('sender', self.sender)
-        return dispatcher.connect(*a, **kw)
+    def connect(self, receiver, signal, **kwargs):
+        """
+        Connect a receiver function to a signal.
 
-    def disconnect(self, *a, **kw):
-        kw.setdefault('sender', self.sender)
-        return dispatcher.disconnect(*a, **kw)
+        The signal can be any object, although Scrapy comes with some
+        predefined signals that are documented in the :ref:`topics-signals`
+        section.
 
-    def send_catch_log(self, *a, **kw):
-        kw.setdefault('sender', self.sender)
-        return signal.send_catch_log(*a, **kw)
+        :param receiver: the function to be connected
+        :type receiver: callable
 
-    def send_catch_log_deferred(self, *a, **kw):
-        kw.setdefault('sender', self.sender)
-        return signal.send_catch_log_deferred(*a, **kw)
+        :param signal: the signal to connect to
+        :type signal: object
+        """
+        kwargs.setdefault('sender', self.sender)
+        return dispatcher.connect(receiver, signal, **kwargs)
 
-    def disconnect_all(self, *a, **kw):
-        kw.setdefault('sender', self.sender)
-        return signal.disconnect_all(*a, **kw)
+    def disconnect(self, receiver, signal, **kwargs):
+        """
+        Disconnect a receiver function from a signal. This has the
+        opposite effect of the :meth:`connect` method, and the arguments
+        are the same.
+        """
+        kwargs.setdefault('sender', self.sender)
+        return dispatcher.disconnect(receiver, signal, **kwargs)
+
+    def send_catch_log(self, signal, **kwargs):
+        """
+        Send a signal, catch exceptions and log them.
+
+        The keyword arguments are passed to the signal handlers (connected
+        through the :meth:`connect` method).
+        """
+        kwargs.setdefault('sender', self.sender)
+        return _signal.send_catch_log(signal, **kwargs)
+
+    def send_catch_log_deferred(self, signal, **kwargs):
+        """
+        Like :meth:`send_catch_log` but supports returning `deferred`_ from
+        signal handlers.
+
+        Returns a `deferred`_ that gets fired once all signal handlers
+        deferreds were fired. Send a signal, catch exceptions and log them.
+
+        The keyword arguments are passed to the signal handlers (connected
+        through the :meth:`connect` method).
+
+        .. _deferreds: http://twistedmatrix.com/documents/current/core/howto/defer.html
+        """
+        kwargs.setdefault('sender', self.sender)
+        return _signal.send_catch_log_deferred(signal, **kwargs)
+
+    def disconnect_all(self, signal, **kwargs):
+        """
+        Disconnect all receivers from the given signal.
+
+        :param signal: the signal to disconnect from
+        :type signal: object
+        """
+        kwargs.setdefault('sender', self.sender)
+        return _signal.disconnect_all(signal, **kwargs)

From 790c67b643666b34376c29985a4d2976a351456d Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 9 Jun 2015 02:20:10 +0500
Subject: [PATCH 0365/2974] DOC spider_error doesn't support deferreds

---
 docs/topics/signals.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 7ea9efe49..5dd3b9ef5 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -174,6 +174,8 @@ spider_error
 
     Sent when a spider callback generates an error (ie. raises an exception).
 
+    This signal does not support returning deferreds from their handlers.
+
     :param failure: the exception raised as a Twisted `Failure`_ object
     :type failure: `Failure`_ object
 

From 66e5591eb9e459f36cec43d4c9e1d8dba82d6cff Mon Sep 17 00:00:00 2001
From: Berker Peksag <berker.peksag@gmail.com>
Date: Tue, 9 Jun 2015 04:26:42 +0300
Subject: [PATCH 0366/2974] Fix typos in scrapy/commands/setting.py

* intepreted -> interpreted
* "a list" instead of "an float" in --getlist help
* "an float"-> "a float"

Also, backslashes were redundant. So I removed them.
---
 scrapy/commands/settings.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index 65a5cd465..0e73f4f58 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -14,16 +14,16 @@ class Command(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("--get", dest="get", metavar="SETTING", \
+        parser.add_option("--get", dest="get", metavar="SETTING",
             help="print raw setting value")
-        parser.add_option("--getbool", dest="getbool", metavar="SETTING", \
-            help="print setting value, intepreted as a boolean")
-        parser.add_option("--getint", dest="getint", metavar="SETTING", \
-            help="print setting value, intepreted as an integer")
-        parser.add_option("--getfloat", dest="getfloat", metavar="SETTING", \
-            help="print setting value, intepreted as an float")
-        parser.add_option("--getlist", dest="getlist", metavar="SETTING", \
-            help="print setting value, intepreted as an float")
+        parser.add_option("--getbool", dest="getbool", metavar="SETTING",
+            help="print setting value, interpreted as a boolean")
+        parser.add_option("--getint", dest="getint", metavar="SETTING",
+            help="print setting value, interpreted as an integer")
+        parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
+            help="print setting value, interpreted as a float")
+        parser.add_option("--getlist", dest="getlist", metavar="SETTING",
+            help="print setting value, interpreted as a list")
 
     def run(self, args, opts):
         settings = self.crawler_process.settings

From 8a48d9c6a87211b3424ac7f2714c50915849fac0 Mon Sep 17 00:00:00 2001
From: Berker Peksag <berker.peksag@gmail.com>
Date: Tue, 9 Jun 2015 04:48:33 +0300
Subject: [PATCH 0367/2974] Ignore ScrapyDeprecationWarning warnings properly.

Conflicts:

	tests/test_utils_deprecate.py
---
 scrapy/linkextractors/sgml.py |  3 ++-
 tests/test_selector.py        |  9 ++++++---
 tests/test_utils_deprecate.py | 16 +++++++++++-----
 3 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index bae4ad5c0..a9b8fe9de 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -111,7 +111,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
         tag_func = lambda x: x in tags
         attr_func = lambda x: x in attrs
 
-        with warnings.catch_warnings(record=True):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
             lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
                 unique=unique, process_value=process_value)
 
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 09dbe6086..ad6a0e21c 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -490,21 +490,24 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
             self.assertTrue(isinstance(usel, XPathSelector))
 
     def test_xpathselector(self):
-        with warnings.catch_warnings(record=True):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
             hs = XPathSelector(text=self.text)
             self.assertEqual(hs.select("//div").extract(),
                              [u'<div><img src="a.jpg"><p>Hello</p></div>'])
             self.assertRaises(RuntimeError, hs.css, 'div')
 
     def test_htmlxpathselector(self):
-        with warnings.catch_warnings(record=True):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
             hs = HtmlXPathSelector(text=self.text)
             self.assertEqual(hs.select("//div").extract(),
                              [u'<div><img src="a.jpg"><p>Hello</p></div>'])
             self.assertRaises(RuntimeError, hs.css, 'div')
 
     def test_xmlxpathselector(self):
-        with warnings.catch_warnings(record=True):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
             xs = XmlXPathSelector(text=self.text)
             self.assertEqual(xs.select("//div").extract(),
                              [u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 41b8100d7..3e7236fb1 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -3,6 +3,7 @@ from __future__ import absolute_import
 import inspect
 import unittest
 import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.deprecate import create_deprecated_class, update_classpath
 
 from tests import mock
@@ -109,7 +110,8 @@ class WarnWhenSubclassedTest(unittest.TestCase):
                                              warn_category=MyWarning)
 
         # ignore subclassing warnings
-        with warnings.catch_warnings(record=True):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
             class UserClass(Deprecated):
                 pass
 
@@ -138,7 +140,8 @@ class WarnWhenSubclassedTest(unittest.TestCase):
         self.assertIn("tests.test_utils_deprecate.Deprecated", msg)
 
     def test_issubclass(self):
-        with warnings.catch_warnings(record=True):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
             DeprecatedName = create_deprecated_class('DeprecatedName', NewName)
 
             class UpdatedUserClass1(NewName):
@@ -173,7 +176,8 @@ class WarnWhenSubclassedTest(unittest.TestCase):
         self.assertRaises(TypeError, issubclass, object(), DeprecatedName)
 
     def test_isinstance(self):
-        with warnings.catch_warnings(record=True):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
             DeprecatedName = create_deprecated_class('DeprecatedName', NewName)
 
             class UpdatedUserClass2(NewName):
@@ -206,7 +210,8 @@ class WarnWhenSubclassedTest(unittest.TestCase):
         assert not isinstance(OldStyleClass(), DeprecatedName)
 
     def test_clsdict(self):
-        with warnings.catch_warnings(record=True):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
             Deprecated = create_deprecated_class('Deprecated', NewName, {'foo': 'bar'})
 
         self.assertEqual(Deprecated.foo, 'bar')
@@ -264,7 +269,8 @@ class UpdateClassPathTest(unittest.TestCase):
         self.assertIn("scrapy.extensions.debug.Debug", str(w[0].message))
 
     def test_sorted_replacement(self):
-        with warnings.catch_warnings(record=True):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
             output = update_classpath('scrapy.contrib.pipeline.Pipeline')
         self.assertEqual(output, 'scrapy.pipelines.Pipeline')
 

From a611f8dd2dd9fb9f7fe81245a488c4493744014d Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 9 Jun 2015 22:57:18 +0500
Subject: [PATCH 0368/2974] DOC remove FailureFormatter mentions, stop
 copy-pasting configure_logging docstring

---
 docs/topics/logging.rst | 58 ++++++++++++++++-------------------------
 scrapy/utils/log.py     | 24 ++++++++++++-----
 2 files changed, 40 insertions(+), 42 deletions(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 469f0c81d..3d7ce958c 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -194,47 +194,33 @@ scrapy.utils.log module
 .. module:: scrapy.utils.log
    :synopsis: Logging utils
 
-.. function:: configure_logging(settings=None, install_root_handler=True)
+.. autofunction:: configure_logging
 
-    This function initializes logging defaults for Scrapy.
+``configure_logging`` is automatically called when using Scrapy commands,
+but needs to be called explicitly when running custom scripts. In that
+case, its usage is not required but it's recommended.
 
-    It's automatically called when using Scrapy commands, but needs to be
-    called explicitely when running custom scripts. In that case, its usage is
-    not required but it's recommended.
+If you plan on configuring the handlers yourself is still recommended you
+call this function, passing `install_root_handler=False`. Bear in mind
+there won't be any log output set by default in that case.
 
-    This function does:
-      - Route warnings and twisted logging through Python standard logging
-      - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively
-      - Route stdout to log if LOG_STDOUT setting is True
+To get you started on manually configuring logging's output, you can use
+`logging.basicConfig()`_ to set a basic root handler. This is an example on
+how to redirect ``INFO`` or higher messages to a file::
 
-    When ``install_root_handler`` is True (default), this function also
-      - Sets FailureFormatter filter on Scrapy logger
-      - Creates a root handler based on the settings listed
-        in :ref:`topics-logging-settings`.
+    import logging
+    from scrapy.utils.log import configure_logging
 
-    If you plan on configuring the handlers yourself is still recommended you
-    call this function, passing `install_root_handler=False`. Bear in mind there
-    won't be any log output set by default in that case.
+    configure_logging(install_root_handler=False)
+    logging.basicConfig(
+        filename='log.txt',
+        format='%(levelname)s: %(message)s',
+        level=logging.INFO
+    )
 
-    To get you started on manually configuring logging's output, you can use
-    `logging.basicConfig()`_ to set a basic root handler. This is an example on
-    how to redirect ``INFO`` or higher messages to a file::
-
-        import logging
-        from scrapy.utils.log import configure_logging
-
-        configure_logging(install_root_handler=False)
-        logging.basicConfig(filename='log.txt', format='%(levelname)s: %(message)s', level=logging.INFO)
-
-    Refer to :ref:`run-from-script` for more details about using Scrapy this
-    way.
-
-    :param settings: settings used to create and configure a handler for the
-        root logger.
-    :type settings: :class:`~scrapy.settings.Settings` object or ``None``
-
-    :param install_root_handler: whether to install root logging handler
-        (default: True)
-    :type install_root_handler: bool
+Refer to :ref:`run-from-script` for more details about using Scrapy this
+way.
 
 .. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig
+
+
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 0102d1740..d40202953 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -57,16 +57,28 @@ DEFAULT_LOGGING = {
 
 
 def configure_logging(settings=None, install_root_handler=True):
-    """Initialize and configure default loggers
+    """
+    Initialize logging defaults for Scrapy.
+
+    :param settings: settings used to create and configure a handler for the
+        root logger (default: None).
+    :type settings: dict, :class:`~scrapy.settings.Settings` object or ``None``
+
+    :param install_root_handler: whether to install root logging handler
+        (default: True)
+    :type install_root_handler: bool
 
     This function does:
-      - Route warnings and twisted logging through Python standard logging
-      - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively
-      - Route stdout to log if LOG_STDOUT setting is True
+
+    - Route warnings and twisted logging through Python standard logging
+    - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively
+    - Route stdout to log if LOG_STDOUT setting is True
 
     When ``install_root_handler`` is True (default), this function also
-      - Sets FailureFormatter filter on Scrapy logger
-      - Creates a handler for the root logger according to given settings
+    creates a handler for the root logger according to given settings
+    (see :ref:`topics-logging-settings`). You can override default options
+    using ``settings`` argument. When ``settings`` is empty or None, defaults
+    are used.
     """
     if not sys.warnoptions:
         # Route warnings through python logging

From 6c9daf3a952abbd53b653037b22a5e15b0b9e4cc Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 10 Jun 2015 01:44:19 +0500
Subject: [PATCH 0369/2974] DOC remove unnecessary links; fix references in
 send_catch_log_deferred docstring

---
 docs/topics/api.rst     | 2 --
 scrapy/signalmanager.py | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index f3a03bb74..f54341eb8 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -427,6 +427,4 @@ class (which they all inherit from).
         Close the given spider. After this is called, no more specific stats
         can be accessed or collected.
 
-.. _deferreds: http://twistedmatrix.com/documents/current/core/howto/defer.html
-.. _deferred: http://twistedmatrix.com/documents/current/core/howto/defer.html
 .. _reactor: http://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index 90a14bd63..bcc0df2c6 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -46,10 +46,10 @@ class SignalManager(object):
 
     def send_catch_log_deferred(self, signal, **kwargs):
         """
-        Like :meth:`send_catch_log` but supports returning `deferred`_ from
+        Like :meth:`send_catch_log` but supports returning `deferreds`_ from
         signal handlers.
 
-        Returns a `deferred`_ that gets fired once all signal handlers
+        Returns a Deferred that gets fired once all signal handlers
         deferreds were fired. Send a signal, catch exceptions and log them.
 
         The keyword arguments are passed to the signal handlers (connected

From 877c25793a25bdf372a84d2c34bbed2754cd58f9 Mon Sep 17 00:00:00 2001
From: Rolando Espinoza <darkrho@gmail.com>
Date: Thu, 11 Jun 2015 13:27:49 -0400
Subject: [PATCH 0370/2974] Do leveldb compactation for the httpcache on
 closing.

---
 scrapy/extensions/httpcache.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 3173656fe..f31cb3278 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -302,6 +302,9 @@ class LeveldbCacheStorage(object):
         self.db = self._leveldb.LevelDB(dbpath)
 
     def close_spider(self, spider):
+        # Do compactation each time to save space and also recreate files to
+        # avoid them being removed in storages with timestamp-based autoremoval.
+        self.db.CompactRange()
         del self.db
 
     def retrieve_response(self, spider, request):

From 6a4c475e870e9c529395c5ac0ef0729304f069a2 Mon Sep 17 00:00:00 2001
From: Bryan Crowe <bryan@bryan-crowe.com>
Date: Thu, 11 Jun 2015 19:47:30 -0400
Subject: [PATCH 0371/2974] Fix a couple typos

---
 docs/topics/selectors.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index bd4135b34..688c2b715 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -668,12 +668,12 @@ SelectorList objects
 
    .. method:: extract()
 
-       Call the ``.extract()`` method for each element is this list and return
+       Call the ``.extract()`` method for each element in this list and return
        their results flattened, as a list of unicode strings.
 
    .. method:: re()
 
-       Call the ``.re()`` method for each element is this list and return
+       Call the ``.re()`` method for each element in this list and return
        their results flattened, as a list of unicode strings.
 
    .. method:: __nonzero__()

From 36bc912cdd061f6fd7845254133e582c33cbab3d Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 12 Jun 2015 12:56:57 -0300
Subject: [PATCH 0372/2974] DOC indent additional docs for configure_logging

---
 docs/topics/logging.rst | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 3d7ce958c..062c6e518 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -196,30 +196,30 @@ scrapy.utils.log module
 
 .. autofunction:: configure_logging
 
-``configure_logging`` is automatically called when using Scrapy commands,
-but needs to be called explicitly when running custom scripts. In that
-case, its usage is not required but it's recommended.
+    ``configure_logging`` is automatically called when using Scrapy commands,
+    but needs to be called explicitly when running custom scripts. In that
+    case, its usage is not required but it's recommended.
 
-If you plan on configuring the handlers yourself is still recommended you
-call this function, passing `install_root_handler=False`. Bear in mind
-there won't be any log output set by default in that case.
+    If you plan on configuring the handlers yourself is still recommended you
+    call this function, passing `install_root_handler=False`. Bear in mind
+    there won't be any log output set by default in that case.
 
-To get you started on manually configuring logging's output, you can use
-`logging.basicConfig()`_ to set a basic root handler. This is an example on
-how to redirect ``INFO`` or higher messages to a file::
+    To get you started on manually configuring logging's output, you can use
+    `logging.basicConfig()`_ to set a basic root handler. This is an example
+    on how to redirect ``INFO`` or higher messages to a file::
 
-    import logging
-    from scrapy.utils.log import configure_logging
+        import logging
+        from scrapy.utils.log import configure_logging
 
-    configure_logging(install_root_handler=False)
-    logging.basicConfig(
-        filename='log.txt',
-        format='%(levelname)s: %(message)s',
-        level=logging.INFO
-    )
+        configure_logging(install_root_handler=False)
+        logging.basicConfig(
+            filename='log.txt',
+            format='%(levelname)s: %(message)s',
+            level=logging.INFO
+        )
 
-Refer to :ref:`run-from-script` for more details about using Scrapy this
-way.
+    Refer to :ref:`run-from-script` for more details about using Scrapy this
+    way.
 
 .. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig
 

From f958fb9d62d71c7b2bb2cd4822b6559e8e6f8087 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 12 Jun 2015 16:39:02 -0300
Subject: [PATCH 0373/2974] Update release notes for 1.0.0rc3

---
 docs/news.rst | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index da7e64e15..9095d9a1e 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -233,7 +233,7 @@ Changelog
 New Features and Enhancements
 
 - Python logging (:issue:`1060`, :issue:`1235`, :issue:`1236`, :issue:`1240`,
-  :issue:`1259`, :issue:`1278`)
+  :issue:`1259`, :issue:`1278`, :issue:`1286`)
 - FEED_EXPORT_FIELDS option (:issue:`1159`, :issue:`1224`)
 - Dns cache size and timeout options (:issue:`1132`)
 - support namespace prefix in xmliter_lxml (:issue:`963`)
@@ -257,8 +257,9 @@ New Features and Enhancements
 - Add project name validation (:issue:`817`)
 - GSoC API cleanup (:issue:`816`, :issue:`1128`, :issue:`1147`,
   :issue:`1148`, :issue:`1156`, :issue:`1185`, :issue:`1187`, :issue:`1258`,
-  :issue:`1268`, :issue:`1276`)
+  :issue:`1268`, :issue:`1276`, :issue:`1285`, :issue:`1284`)
 - Be more responsive with IO operations (:issue:`1074` and :issue:`1075`)
+- Do leveldb compaction for httpcache on closing (:issue:`1297`)
 
 Deprecations and Removals
 
@@ -302,11 +303,12 @@ Documentation
 - DOC simplify extension docs (:issue:`893`)
 - Leaks docs (:issue:`894`)
 - DOC document from_crawler method for item pipelines (:issue:`904`)
+- Spider_error doesn't support deferreds (:issue:`1292`)
 - Corrections & Sphinx related fixes (:issue:`1220`, :issue:`1219`,
   :issue:`1196`, :issue:`1172`, :issue:`1171`, :issue:`1169`, :issue:`1160`,
   :issue:`1154`, :issue:`1127`, :issue:`1112`, :issue:`1105`, :issue:`1041`,
   :issue:`1082`, :issue:`1033`, :issue:`944`, :issue:`866`, :issue:`864`,
-  :issue:`796`, :issue:`1260`, :issue:`1271`)
+  :issue:`796`, :issue:`1260`, :issue:`1271`, :issue:`1293`, :issue:`1298`)
 
 Bugfixes
 

From 2de5c660589ce0a8f8a0d501349327e5a060d222 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Sun, 14 Jun 2015 12:39:29 -0300
Subject: [PATCH 0374/2974] improvements for docs on how to access settings

---
 docs/Makefile            |  1 +
 docs/topics/settings.rst | 45 +++++++++++++++++++++++++++++++---------
 2 files changed, 36 insertions(+), 10 deletions(-)

diff --git a/docs/Makefile b/docs/Makefile
index 3a7b8c2c3..a51ce8e17 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -23,6 +23,7 @@ help:
 	@echo "  text      to make plain text files"
 	@echo "  changes   to make an overview over all changed/added/deprecated items"
 	@echo "  linkcheck to check all external links for integrity"
+	@echo "  watch     build HTML docs, open in browser and watch for changes"
 
 
 build: 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index a9eba41ce..0e2a621f4 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -67,14 +67,22 @@ Example::
 
 Spiders (See the :ref:`topics-spiders` chapter for reference) can define their
 own settings that will take precedence and override the project ones. They can
-do so by setting their :attr:`scrapy.spiders.Spider.custom_settings` attribute.
+do so by setting their :attr:`~scrapy.spiders.Spider.custom_settings` attribute::
+
+    class MySpider(scrapy.Spider):
+        name = 'myspider'
+
+        custom_settings = {
+            'SOME_SETTING': 'some value',
+        }
 
 3. Project settings module
 --------------------------
 
 The project settings module is the standard configuration file for your Scrapy
-project.  It's where most of your custom settings will be populated. For
-example:: ``myproject.settings``.
+project, it's where most of your custom settings will be populated. For a
+standard Scrapy project, this means you'll be adding or changing the settings
+in the ``settings.py`` file created for your project.
 
 4. Default settings per-command
 -------------------------------
@@ -95,22 +103,39 @@ How to access settings
 
 .. highlight:: python
 
+In a spider, the settings are available through ``self.settings``::
+
+    class MySpider(scrapy.Spider):
+        name = 'myspider'
+        start_urls = ['http://example.com']
+
+        def parse(self, response):
+            print("Existing settings: %s" % self.settings.attributes.keys())
+
+.. note::
+    The ``settings`` attribute is set in the base Spider class after the spider
+    is initialized.  If you want to use the settings before the initialization
+    (e.g., in your spider's ``__init__()`` method), you'll need to override the
+    :meth:`~scrapy.spiders.Spider.from_crawler` method.
+
 Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings`
 attribute of the Crawler that is passed to ``from_crawler`` method in
-extensions and middlewares::
+extensions, middlewares and item pipelines::
 
     class MyExtension(object):
+        def __init__(self, log_is_enabled=False):
+            if log_is_enabled:
+                print("log is enabled!")
 
         @classmethod
         def from_crawler(cls, crawler):
             settings = crawler.settings
-            if settings['LOG_ENABLED']:
-                print "log is enabled!"
+            return cls(settings.getbool('LOG_ENABLED'))
 
-In other words, settings can be accessed like a dict, but it's usually preferred
-to extract the setting in the format you need it to avoid type errors. In order
-to do that you'll have to use one of the methods provided the
-:class:`~scrapy.settings.Settings` API.
+The settings object can be used like a dict (e.g.,
+``settings['LOG_ENABLED']``), but it's usually preferred to extract the setting
+in the format you need it to avoid type errors, using one of the methods
+provided by the :class:`~scrapy.settings.Settings` API.
 
 Rationale for setting names
 ===========================

From 140f58251bf2292891e6bc6233589d3d0e944bb8 Mon Sep 17 00:00:00 2001
From: agusc <agustin@scrapinghub.com>
Date: Mon, 15 Jun 2015 14:39:10 -0300
Subject: [PATCH 0375/2974] removed SUFFIX from scrapy name package

---
 Makefile.buildbot |  5 +++--
 debian/changelog  |  2 +-
 debian/control    |  4 ++--
 extras/makedeb.py | 40 ----------------------------------------
 4 files changed, 6 insertions(+), 45 deletions(-)
 delete mode 100644 extras/makedeb.py

diff --git a/Makefile.buildbot b/Makefile.buildbot
index 68c8bdc54..a6c4bbbc3 100644
--- a/Makefile.buildbot
+++ b/Makefile.buildbot
@@ -8,8 +8,9 @@ test:
 	-s3cmd sync -P htmlcov/ s3://static.scrapy.org/coverage-scrapy-$(BRANCH)/
 
 build:
-	test $(BRANCH) != master || git describe >scrapy/VERSION
-	python extras/makedeb.py build
+	test $(BRANCH) != master || git describe --tags | sed -r 's/([0-9]+.[0-9]*.[0-9]*)(rc|dev)(.*)/\1~\2\3/' >scrapy/VERSION
+	debchange -m -D unstable --force-distribution -v `python setup.py --version`+`date +%s` "Automatic build"
+	debuild -us -uc -b
 
 clean:
 	git checkout debian scrapy/VERSION
diff --git a/debian/changelog b/debian/changelog
index f4f5b9d9c..dde97f9e3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-scrapy-SUFFIX (0.11) unstable; urgency=low
+scrapy (0.11) unstable; urgency=low
 
   * Initial release.
 
diff --git a/debian/control b/debian/control
index 15743767b..75225f30d 100644
--- a/debian/control
+++ b/debian/control
@@ -1,4 +1,4 @@
-Source: scrapy-SUFFIX
+Source: scrapy
 Section: python
 Priority: optional
 Maintainer: Scrapinghub Team <info@scrapinghub.com>
@@ -6,7 +6,7 @@ Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3l
 Standards-Version: 3.8.4
 Homepage: http://scrapy.org/
 
-Package: scrapy-SUFFIX
+Package: scrapy
 Architecture: all
 Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl,
  python-w3lib (>= 1.8.0), python-queuelib, python-cssselect (>= 0.9), python-six (>=1.5.2)
diff --git a/extras/makedeb.py b/extras/makedeb.py
deleted file mode 100644
index 04dea2cb0..000000000
--- a/extras/makedeb.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import sys, os, glob, shutil
-from subprocess import check_call
-from scrapy import version_info
-
-def build(suffix):
-    for ifn in glob.glob("debian/scrapy.*"):
-        s = open(ifn).read()
-        s = s.replace('SUFFIX', suffix)
-        pre, suf = ifn.split('.', 1)
-        ofn = "%s-%s.%s" % (pre, suffix, suf)
-        with open(ofn, 'w') as of:
-            of.write(s)
-
-    for ifn in ['debian/control', 'debian/changelog']:
-        s = open(ifn).read()
-        s = s.replace('SUFFIX', suffix)
-        with open(ifn, 'w') as of:
-            of.write(s)
-
-    check_call('debchange -m -D unstable --force-distribution -v $(python setup.py --version)+$(date +%s) "Automatic build"', \
-        shell=True)
-    check_call('debuild -us -uc -b', shell=True)
-
-def clean(suffix):
-    for f in glob.glob("debian/python-scrapy%s*" % suffix):
-        if os.path.isdir(f):
-            shutil.rmtree(f)
-        else:
-            os.remove(f)
-
-def main():
-    cmd = sys.argv[1]
-    suffix = '%s.%s' % version_info[:2]
-    if cmd == 'build':
-        build(suffix)
-    elif cmd == 'clean':
-        clean(suffix)
-
-if __name__ == '__main__':
-    main()

From 47b1b748c25f2a5b13f0a9ffab09823f7e43e8b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 15 Jun 2015 20:51:37 -0300
Subject: [PATCH 0376/2974] Generated version as pep440 and dpkg compatible

---
 Makefile.buildbot | 6 ++++--
 debian/control    | 4 ++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/Makefile.buildbot b/Makefile.buildbot
index a6c4bbbc3..775538259 100644
--- a/Makefile.buildbot
+++ b/Makefile.buildbot
@@ -8,8 +8,10 @@ test:
 	-s3cmd sync -P htmlcov/ s3://static.scrapy.org/coverage-scrapy-$(BRANCH)/
 
 build:
-	test $(BRANCH) != master || git describe --tags | sed -r 's/([0-9]+.[0-9]*.[0-9]*)(rc|dev)(.*)/\1~\2\3/' >scrapy/VERSION
-	debchange -m -D unstable --force-distribution -v `python setup.py --version`+`date +%s` "Automatic build"
+	git describe --tags --match '[0-9]*' |sed 's/-/.post/;s/-g/+g/' >scrapy/VERSION
+	debchange -m -D unstable --force-distribution -v \
+	  $$(python setup.py --version |sed -r 's/([0-9]+.[0-9]+.[0-9]+)(a|b|rc|dev)([0-9]*)/\1~\2\3/')-$$(date +%s) \
+	  "Automatic build"
 	debuild -us -uc -b
 
 clean:
diff --git a/debian/control b/debian/control
index 75225f30d..f3a31753b 100644
--- a/debian/control
+++ b/debian/control
@@ -11,8 +11,8 @@ Architecture: all
 Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl,
  python-w3lib (>= 1.8.0), python-queuelib, python-cssselect (>= 0.9), python-six (>=1.5.2)
 Recommends: python-setuptools
-Conflicts: python-scrapy, scrapy, scrapy-0.11
-Provides: python-scrapy, scrapy
+Conflicts: python-scrapy, scrapy-0.25
+Provides: python-scrapy, scrapy-0.25
 Description: Python web crawling and web scraping framework
  Scrapy is a fast high-level web crawling and web scraping framework,
  used to crawl websites and extract structured data from their pages. 

From f4dd8bcdc29cb5411d0fe2ed8f53e75883402ede Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Tue, 16 Jun 2015 17:31:37 +0300
Subject: [PATCH 0377/2974] Disable dupefilter in shell

---
 scrapy/commands/shell.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 92ebbe605..77ae1358b 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -16,7 +16,11 @@ from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 class Command(ScrapyCommand):
 
     requires_project = False
-    default_settings = {'KEEP_ALIVE': True, 'LOGSTATS_INTERVAL': 0}
+    default_settings = {
+        'KEEP_ALIVE': True,
+        'LOGSTATS_INTERVAL': 0,
+        'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter',
+    }
 
     def syntax(self):
         return "[url|file]"

From af97c373eac5f6c137f632e2db844bde3abb2f49 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 19 Jun 2015 18:49:13 -0300
Subject: [PATCH 0378/2974] Update release date for 1.0

(cherry picked from commit c89fa29c415f669fd42ad4bae88e754ed0194a4f)
---
 docs/news.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 9095d9a1e..2cf1365e5 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,8 +3,8 @@
 Release notes
 =============
 
-1.0
----
+1.0 (2015-06-19)
+----------------
 
 You will find a lot of new features and bugfixes in this major release.  Make
 sure to check our updated :ref:`overview <intro-overview>` to get a glance of

From 38e5bfb61c3448c4d839b671e29b09b92e016477 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Mon, 22 Jun 2015 10:57:24 -0300
Subject: [PATCH 0379/2974] remove version suffix from ubuntu package

---
 docs/topics/ubuntu.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst
index cfe49d722..dbbcaacfc 100644
--- a/docs/topics/ubuntu.rst
+++ b/docs/topics/ubuntu.rst
@@ -21,11 +21,11 @@ To use the packages:
 
     echo 'deb http://archive.scrapy.org/ubuntu scrapy main' | sudo tee /etc/apt/sources.list.d/scrapy.list
 
-3. Update package lists and install the scrapy-|version| package:
+3. Update package lists and install the scrapy package:
 
    .. parsed-literal::
 
-      sudo apt-get update && sudo apt-get install scrapy-|version|
+      sudo apt-get update && sudo apt-get install scrapy
 
 .. note:: Repeat step 3 if you are trying to upgrade Scrapy.
 

From eb8fed4ecd8fbb51822553534b8b2f3cc459de60 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 23 Jun 2015 00:43:15 -0300
Subject: [PATCH 0380/2974] support empty password for http_proxy config

---
 scrapy/downloadermiddlewares/httpproxy.py    | 2 +-
 tests/test_downloadermiddleware_httpproxy.py | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 48a877c91..dda6a3d2a 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -25,7 +25,7 @@ class HttpProxyMiddleware(object):
         proxy_type, user, password, hostport = _parse_proxy(url)
         proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
 
-        if user and password:
+        if user:
             user_pass = '%s:%s' % (unquote(user), unquote(password))
             creds = base64.b64encode(user_pass).strip()
         else:
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 191664076..5b9717a89 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -54,6 +54,14 @@ class TestDefaultHeadersMiddleware(TestCase):
         self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
         self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjpwYXNz')
 
+    def test_proxy_auth_empty_passwd(self):
+        os.environ['http_proxy'] = 'https://user:@proxy:3128'
+        mw = HttpProxyMiddleware()
+        req = Request('http://scrapytest.org')
+        assert mw.process_request(req, spider) is None
+        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjo=')
+
     def test_proxy_already_seted(self):
         os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
         mw = HttpProxyMiddleware()

From 8da9836db3d6071eff6d0267ed47a7af81b830d3 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 24 Jun 2015 03:20:46 +0500
Subject: [PATCH 0381/2974] don't access settings from downloader.Slot

---
 scrapy/core/downloader/__init__.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index ecbf33039..682c95a15 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -17,10 +17,11 @@ from .handlers import DownloadHandlers
 class Slot(object):
     """Downloader slot"""
 
-    def __init__(self, concurrency, delay, settings):
+    def __init__(self, concurrency, delay, randomize_delay):
         self.concurrency = concurrency
         self.delay = delay
-        self.randomize_delay = settings.getbool('RANDOMIZE_DOWNLOAD_DELAY')
+        self.randomize_delay = randomize_delay
+
         self.active = set()
         self.queue = deque()
         self.transferring = set()
@@ -66,6 +67,7 @@ class Downloader(object):
         self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS')
         self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP')
+        self.randomize_delay = self.settings.getbool('RANDOMIZE_DOWNLOAD_DELAY')
         self.middleware = DownloaderMiddlewareManager.from_crawler(crawler)
         self._slot_gc_loop = task.LoopingCall(self._slot_gc)
         self._slot_gc_loop.start(60)
@@ -87,7 +89,7 @@ class Downloader(object):
         if key not in self.slots:
             conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
             conc, delay = _get_concurrency_delay(conc, spider, self.settings)
-            self.slots[key] = Slot(conc, delay, self.settings)
+            self.slots[key] = Slot(conc, delay, self.randomize_delay)
 
         return key, self.slots[key]
 

From cb0445e8b884d350786fb6df829c9e00ade5036f Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 24 Jun 2015 03:22:23 +0500
Subject: [PATCH 0382/2974] Slot.__str__ and __repr__ methods; remove unused
 import

---
 scrapy/core/downloader/__init__.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 682c95a15..965471851 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -1,6 +1,8 @@
+from __future__ import absolute_import
 import random
 import warnings
 from time import time
+from datetime import datetime
 from collections import deque
 
 from twisted.internet import reactor, defer, task
@@ -8,7 +10,6 @@ from twisted.internet import reactor, defer, task
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.resolver import dnscache
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy import signals
 from .middleware import DownloaderMiddlewareManager
 from .handlers import DownloadHandlers
@@ -40,6 +41,21 @@ class Slot(object):
         if self.latercall and self.latercall.active():
             self.latercall.cancel()
 
+    def __repr__(self):
+        cls_name = self.__class__.__name__
+        return "%s(concurrency=%r, delay=%0.2f, randomize_delay=%r)" % (
+            cls_name, self.concurrency, self.delay, self.randomize_delay)
+
+    def __str__(self):
+        return (
+            "<downloader.Slot concurrency=%r delay=%0.2f randomize_delay=%r "
+            "len(active)=%d len(queue)=%d len(transferring)=%d lastseen=%s>" % (
+                self.concurrency, self.delay, self.randomize_delay,
+                len(self.active), len(self.queue), len(self.transferring),
+                datetime.fromtimestamp(self.lastseen).isoformat()
+            )
+        )
+
 
 def _get_concurrency_delay(concurrency, spider, settings):
     delay = settings.getfloat('DOWNLOAD_DELAY')

From 44c8ef2d4728b3caa8293dbf5660fb91019a81c6 Mon Sep 17 00:00:00 2001
From: Marco DallaG <marco.dallagiacoma@gmail.com>
Date: Wed, 24 Jun 2015 11:44:48 +0200
Subject: [PATCH 0383/2974] DOC Bring Ubuntu and Archlinux outside of Windows
 subsection

In the installation guide, in the "Platform specific installation notes" section, Ubuntu and ArchLinux are currently subsections of Windows, which does not make sense imho.
This commit changes the section tree from:

Platform specific installation notes
- Windows
  -- Archlinux
  -- Ubuntu

To:

Platform specific installation notes
- Windows
- Ubuntu
- Archlinux
---
 docs/intro/install.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index fbed84055..67507016d 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -70,7 +70,7 @@ Windows
       pip install Scrapy
 
 Ubuntu 9.10 or above
-~~~~~~~~~~~~~~~~~~~~
+--------------------
 
 **Don't** use the ``python-scrapy`` package provided by Ubuntu, they are
 typically too old and slow to catch up with latest Scrapy.
@@ -80,7 +80,7 @@ solve all dependencies for you and are continuously updated with the latest bug
 fixes.
 
 Archlinux
-~~~~~~~~~
+---------
 
 You can follow the generic instructions or install Scrapy from `AUR Scrapy package`::
 

From 49fe9157987d0d3fefb812d7af4f560566eaa42e Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 25 Jun 2015 10:12:56 -0400
Subject: [PATCH 0384/2974] BF: fail if docs failed to build

---
 docs/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/Makefile b/docs/Makefile
index 3a7b8c2c3..373a1ae82 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -27,7 +27,7 @@ help:
 
 build: 
 	mkdir -p build/$(BUILDER) build/doctrees
-	-sphinx-build $(ALLSPHINXOPTS)
+	sphinx-build $(ALLSPHINXOPTS)
 	@echo
 
 

From 48582be9a6caefa1cdf9e6c852f17fce903b553b Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 25 Jun 2015 10:46:22 -0400
Subject: [PATCH 0385/2974] DOC(ENH): specify path to rtd theme explicitly

---
 docs/conf.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index 2d22c6a64..a593bbd65 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -108,7 +108,10 @@ html_theme = 'sphinx_rtd_theme'
 #html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+# Add path to the RTD explicitly to robustify builds (otherwise might
+# fail in a clean Debian build env)
+import sphinx_rtd_theme
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 
 # The style sheet to use for HTML and HTML Help pages. A file of that name

From 4cea1c1231da1583fb177e976f473fa52b9ec450 Mon Sep 17 00:00:00 2001
From: Capi Etheriel <barraponto@gmail.com>
Date: Thu, 25 Jun 2015 15:11:56 -0300
Subject: [PATCH 0386/2974] Fix SelectJmes documentation

---
 scrapy/loader/processors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index 3b221acaf..bf7c74bfe 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -78,7 +78,7 @@ class SelectJmes(object):
 
     def __call__(self, value):
         """Query value for the jmespath query and return answer
-        :param str value: a string with JSON data to extract from
+        :param value: a data structure (dict, list) to extract from
         :return: Element extracted according to jmespath query
         """
         return self.compiled_path.search(value)

From 5f3e1e9a169c6920f5d1edfb20dd77339c32d81e Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 25 Jun 2015 21:30:35 -0400
Subject: [PATCH 0387/2974] ENH: make explicit build-ignore-errors to be used
 by linkfix

---
 docs/Makefile | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/docs/Makefile b/docs/Makefile
index 373a1ae82..26b2b52b1 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -24,12 +24,17 @@ help:
 	@echo "  changes   to make an overview over all changed/added/deprecated items"
 	@echo "  linkcheck to check all external links for integrity"
 
-
-build: 
+build-dirs:
 	mkdir -p build/$(BUILDER) build/doctrees
+
+build: build-dirs
 	sphinx-build $(ALLSPHINXOPTS)
 	@echo
 
+build-ignore-errors: build-dirs
+	-sphinx-build $(ALLSPHINXOPTS)
+	@echo
+
 
 html: BUILDER = html
 html: build
@@ -59,7 +64,8 @@ linkcheck: build
 	@echo "Link check complete; look for any errors in the above output " \
 	      "or in build/$(BUILDER)/output.txt"
 
-linkfix: linkcheck
+linkfix: BUILDER = linkcheck
+linkfix: build-ignore-errors
 	$(PYTHON) utils/linkfix.py
 	@echo "Fixing redirecting links in docs has finished; check all " \
 	      "replacements before committing them"

From 5207d440555139d750d6d99d006404c472199632 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 25 Jun 2015 23:00:00 -0400
Subject: [PATCH 0388/2974] ENH: include tests/ to source distribution in
 MANIFEST.in

---
 MANIFEST.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index 0561cc74c..04b3e1fb9 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -11,3 +11,4 @@ recursive-include docs *
 prune docs/build
 recursive-include extras *
 recursive-include bin *
+recursive-include tests *

From e6272e5209de38e2632d63ccb39e645b048c97b3 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 26 Jun 2015 18:44:23 +0500
Subject: [PATCH 0389/2974] make AutoThrottle._adjust_delay easier to
 understand

---
 scrapy/extensions/throttle.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index a60b0cd83..8c85d13f9 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -68,13 +68,22 @@ class AutoThrottle(object):
 
     def _adjust_delay(self, slot, latency, response):
         """Define delay adjustment policy"""
+
+        # Adjust the delay to be closer to latency.
+        new_delay = (slot.delay + latency) / 2.0
+
         # If latency is bigger than old delay, then use latency instead of mean.
-        # It works better with problematic sites
-        new_delay = min(max(self.mindelay, latency, (slot.delay + latency) / 2.0), self.maxdelay)
+        # It works better with problematic sites.
+        new_delay = max(latency, new_delay)
+
+        # Make sure self.mindelay <= new_delay <= self.max_delay
+        new_delay = min(max(self.mindelay, new_delay), self.maxdelay)
 
         # Dont adjust delay if response status != 200 and new delay is smaller
         # than old one, as error pages (and redirections) are usually small and
         # so tend to reduce latency, thus provoking a positive feedback by
         # reducing delay instead of increase.
-        if response.status == 200 or new_delay > slot.delay:
-            slot.delay = new_delay
+        if response.status != 200 and new_delay <= slot.delay:
+            return
+
+        slot.delay = new_delay

From 584252e8f2f8577d37d648042d0a78a41af8e913 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 26 Jun 2015 18:58:29 +0500
Subject: [PATCH 0390/2974] move AutoThrottle default options to
 default_settings.py

---
 scrapy/extensions/throttle.py       | 4 ++--
 scrapy/settings/default_settings.py | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index 8c85d13f9..9dceaf00d 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -33,10 +33,10 @@ class AutoThrottle(object):
             s.getfloat('DOWNLOAD_DELAY')
 
     def _max_delay(self, spider):
-        return self.crawler.settings.getfloat('AUTOTHROTTLE_MAX_DELAY', 60.0)
+        return self.crawler.settings.getfloat('AUTOTHROTTLE_MAX_DELAY')
 
     def _start_delay(self, spider):
-        return max(self.mindelay, self.crawler.settings.getfloat('AUTOTHROTTLE_START_DELAY', 5.0))
+        return max(self.mindelay, self.crawler.settings.getfloat('AUTOTHROTTLE_START_DELAY'))
 
     def _response_downloaded(self, response, request, spider):
         key, slot = self._get_slot(request, spider)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 0ea014007..6df3925e5 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -20,6 +20,11 @@ from os.path import join, abspath, dirname
 
 AJAXCRAWL_ENABLED = False
 
+AUTOTHROTTLE_ENABLED = False
+AUTOTHROTTLE_DEBUG = False
+AUTOTHROTTLE_MAX_DELAY = 60.0
+AUTOTHROTTLE_START_DELAY = 5.0
+
 BOT_NAME = 'scrapybot'
 
 CLOSESPIDER_TIMEOUT = 0

From 17cb51315b875157d397719c2834d1d02f2ca49f Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 26 Jun 2015 19:07:54 +0500
Subject: [PATCH 0391/2974] drop support for AUTOTHROTTLE_MIN_DOWNLOAD_DELAY

it was deprecated for years
---
 scrapy/extensions/throttle.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index 9dceaf00d..46273353b 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -28,9 +28,7 @@ class AutoThrottle(object):
 
     def _min_delay(self, spider):
         s = self.crawler.settings
-        return getattr(spider, 'download_delay', 0.0) or \
-            s.getfloat('AUTOTHROTTLE_MIN_DOWNLOAD_DELAY') or \
-            s.getfloat('DOWNLOAD_DELAY')
+        return getattr(spider, 'download_delay', 0.0) or s.getfloat('DOWNLOAD_DELAY')
 
     def _max_delay(self, spider):
         return self.crawler.settings.getfloat('AUTOTHROTTLE_MAX_DELAY')

From 1fef9f113237630664ef05645127ca50662d71b6 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 26 Jun 2015 19:09:26 +0500
Subject: [PATCH 0392/2974] AutoThrottle: respect download_delay=0 spider
 attribute

---
 scrapy/extensions/throttle.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index 46273353b..d2ff8665e 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -28,7 +28,7 @@ class AutoThrottle(object):
 
     def _min_delay(self, spider):
         s = self.crawler.settings
-        return getattr(spider, 'download_delay', 0.0) or s.getfloat('DOWNLOAD_DELAY')
+        return getattr(spider, 'download_delay', s.getfloat('DOWNLOAD_DELAY'))
 
     def _max_delay(self, spider):
         return self.crawler.settings.getfloat('AUTOTHROTTLE_MAX_DELAY')

From 63317531f958cc36000f527d352ee0b7453653cb Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 26 Jun 2015 20:47:58 +0500
Subject: [PATCH 0393/2974] DOC fix authrottle docs

see https://github.com/scrapy/scrapy/pull/502/files#r8574692
---
 docs/topics/autothrottle.rst | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst
index 8073ec6e0..c1a7be6ff 100644
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@@ -33,18 +33,18 @@ server) is, and this extension builds on that premise.
 Throttling algorithm
 ====================
 
-This adjusts download delays and concurrency based on the following rules:
+This adjusts download delays based on the following rules:
 
-1. spiders always start with one concurrent request and a download delay of
-   :setting:`AUTOTHROTTLE_START_DELAY`
+1. spiders always start with a download delay of
+   :setting:`AUTOTHROTTLE_START_DELAY`;
 2. when a response is received, the download delay is adjusted to the
    average of previous download delay and the latency of the response.
 
 .. note:: The AutoThrottle extension honours the standard Scrapy settings for
-   concurrency and delay. This means that it will never set a download delay
-   lower than :setting:`DOWNLOAD_DELAY` or a concurrency higher than
-   :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
-   (or :setting:`CONCURRENT_REQUESTS_PER_IP`, depending on which one you use).
+   concurrency and delay. This means that it will respect
+   :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
+   :setting:`CONCURRENT_REQUESTS_PER_IP` options and
+   never set a download delay lower than :setting:`DOWNLOAD_DELAY`.
 
 Settings
 ========

From d850238c2220e2d082ea9805fa356b62d975697f Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 27 Jun 2015 04:59:42 +0500
Subject: [PATCH 0394/2974] add AUTOTHROTTLE_TARGET_CONCURRENCY option and
 expand AutoThrottle docs

---
 docs/topics/autothrottle.rst        | 102 ++++++++++++++++++++++------
 docs/topics/settings.rst            |  11 +--
 scrapy/extensions/throttle.py       |  14 ++--
 scrapy/settings/default_settings.py |   1 +
 4 files changed, 100 insertions(+), 28 deletions(-)

diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst
index c1a7be6ff..0d664cf67 100644
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@@ -12,13 +12,65 @@ Design goals
 
 1. be nicer to sites instead of using default download delay of zero
 2. automatically adjust scrapy to the optimum crawling speed, so the user
-   doesn't have to tune the download delays and concurrent requests to find the
-   optimum one. The user only needs to specify the maximum concurrent requests
+   doesn't have to tune the download delays to find the optimum one.
+   The user only needs to specify the maximum concurrent requests
    it allows, and the extension does the rest.
 
+.. _autothrottle-algorithm:
+
 How it works
 ============
 
+AutoThrottle extension adjusts download delays dynamically to make spider send
+:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent requests on average
+to each remote website.
+
+It uses download latency to compute the delays. The main idea is the
+following: if a server needs ``latency`` seconds to respond, a client
+should send a request each ``latency/N`` seconds to have ``N`` requests
+processed in parallel.
+
+Instead of adjusting the delays one can just set a small fixed
+download delay and impose hard limits on concurrency using
+:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
+:setting:`CONCURRENT_REQUESTS_PER_IP` options. It will provide a similar
+effect, but there are some important differences:
+
+* because the download delay is small there will be occasional bursts
+  of requests;
+* often non-200 (error) responses can be returned faster than regular
+  responses, so with a small download delay and a hard concurrency limit
+  crawler will be sending requests to server faster when server starts to
+  return errors. But this is an opposite of what crawler should do - in case
+  of errors it makes more sense to slow down: these errors may be caused by
+  the high request rate.
+
+AutoThrottle doesn't have these issues.
+
+Throttling algorithm
+====================
+
+AutoThrottle algorithm adjusts download delays based on the following rules:
+
+1. spiders always start with a download delay of
+   :setting:`AUTOTHROTTLE_START_DELAY`;
+2. when a response is received, the target download delay is calculated as
+   ``latency / N`` where ``latency`` is a latency of the response,
+   and ``N`` is :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY`.
+3. download delay for next requests is set to the average of previous
+   download delay and the target download delay;
+4. latencies of non-200 responses are not allowed to decrease the delay;
+5. download delay can't become less than :setting:`DOWNLOAD_DELAY` or greater
+   than :setting:`AUTOTHROTTLE_MAX_DELAY`
+
+.. note:: The AutoThrottle extension honours the standard Scrapy settings for
+   concurrency and delay. This means that it will respect
+   :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
+   :setting:`CONCURRENT_REQUESTS_PER_IP` options and
+   never set a download delay lower than :setting:`DOWNLOAD_DELAY`.
+
+.. _download-latency:
+
 In Scrapy, the download latency is measured as the time elapsed between
 establishing the TCP connection and receiving the HTTP headers.
 
@@ -28,24 +80,6 @@ callback, for example, and unable to attend downloads. However, these latencies
 should still give a reasonable estimate of how busy Scrapy (and ultimately, the
 server) is, and this extension builds on that premise.
 
-.. _autothrottle-algorithm:
-
-Throttling algorithm
-====================
-
-This adjusts download delays based on the following rules:
-
-1. spiders always start with a download delay of
-   :setting:`AUTOTHROTTLE_START_DELAY`;
-2. when a response is received, the download delay is adjusted to the
-   average of previous download delay and the latency of the response.
-
-.. note:: The AutoThrottle extension honours the standard Scrapy settings for
-   concurrency and delay. This means that it will respect
-   :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
-   :setting:`CONCURRENT_REQUESTS_PER_IP` options and
-   never set a download delay lower than :setting:`DOWNLOAD_DELAY`.
-
 Settings
 ========
 
@@ -88,6 +122,34 @@ Default: ``60.0``
 
 The maximum download delay (in seconds) to be set in case of high latencies.
 
+.. setting:: AUTOTHROTTLE_TARGET_CONCURRENCY
+
+AUTOTHROTTLE_TARGET_CONCURRENCY
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Default: ``1.0``
+
+Average number of requests Scrapy should be sending in parallel to remote
+websites.
+
+By default, AutoThrottle adjusts the delay to send a single
+concurrent request to each of the remote websites. Set this option to
+a higher value (e.g. ``2.0``) to increase the throughput and the load on remote
+servers. A lower ``AUTOTHROTTLE_TARGET_CONCURRENCY`` value
+(e.g. ``0.5``) makes the crawler more conservative and polite.
+
+Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
+and :setting:`CONCURRENT_REQUESTS_PER_IP` options are still respected
+when AutoThrottle extension is enabled. This means that if
+``AUTOTHROTTLE_TARGET_CONCURRENCY`` is set to a value higher than
+:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
+:setting:`CONCURRENT_REQUESTS_PER_IP`, the crawler won't reach this number
+of concurrent requests.
+
+At every given time point Scrapy can be sending more or less concurrent
+requests than ``AUTOTHROTTLE_TARGET_CONCURRENCY``; it is a suggested
+value the crawler tries to approach, not a hard limit.
+
 .. setting:: AUTOTHROTTLE_DEBUG
 
 AUTOTHROTTLE_DEBUG
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index a9eba41ce..2b983f9d9 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -187,7 +187,6 @@ Default: ``16``
 The maximum number of concurrent (ie. simultaneous) requests that will be
 performed by the Scrapy downloader.
 
-
 .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
 
 CONCURRENT_REQUESTS_PER_DOMAIN
@@ -198,6 +197,10 @@ Default: ``8``
 The maximum number of concurrent (ie. simultaneous) requests that will be
 performed to any single domain.
 
+See also: :ref:`topics-autothrottle` and its
+:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` option.
+
+
 .. setting:: CONCURRENT_REQUESTS_PER_IP
 
 CONCURRENT_REQUESTS_PER_IP
@@ -211,9 +214,9 @@ performed to any single IP. If non-zero, the
 used instead. In other words, concurrency limits will be applied per IP, not
 per domain.
 
-This setting also affects :setting:`DOWNLOAD_DELAY`:
-if :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, download delay is
-enforced per IP, not per domain.
+This setting also affects :setting:`DOWNLOAD_DELAY` and
+:ref:`topics-autothrottle`: if :setting:`CONCURRENT_REQUESTS_PER_IP`
+is non-zero, download delay is enforced per IP, not per domain.
 
 
 .. setting:: DEFAULT_ITEM_CLASS
diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index d2ff8665e..198d4bbb0 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -14,6 +14,7 @@ class AutoThrottle(object):
             raise NotConfigured
 
         self.debug = crawler.settings.getbool("AUTOTHROTTLE_DEBUG")
+        self.target_concurrency = crawler.settings.getfloat("AUTOTHROTTLE_TARGET_CONCURRENCY")
         crawler.signals.connect(self._spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(self._response_downloaded, signal=signals.response_downloaded)
 
@@ -67,12 +68,17 @@ class AutoThrottle(object):
     def _adjust_delay(self, slot, latency, response):
         """Define delay adjustment policy"""
 
-        # Adjust the delay to be closer to latency.
-        new_delay = (slot.delay + latency) / 2.0
+        # If a server needs `latency` seconds to respond then
+        # we should send a request each `latency/N` seconds
+        # to have N requests processed in parallel
+        target_delay = latency / self.target_concurrency
 
-        # If latency is bigger than old delay, then use latency instead of mean.
+        # Adjust the delay to make it closer to target_delay
+        new_delay = (slot.delay + target_delay) / 2.0
+
+        # If target delay is bigger than old delay, then use it instead of mean.
         # It works better with problematic sites.
-        new_delay = max(latency, new_delay)
+        new_delay = max(target_delay, new_delay)
 
         # Make sure self.mindelay <= new_delay <= self.max_delay
         new_delay = min(max(self.mindelay, new_delay), self.maxdelay)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 6df3925e5..62bb1ccd3 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -24,6 +24,7 @@ AUTOTHROTTLE_ENABLED = False
 AUTOTHROTTLE_DEBUG = False
 AUTOTHROTTLE_MAX_DELAY = 60.0
 AUTOTHROTTLE_START_DELAY = 5.0
+AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
 
 BOT_NAME = 'scrapybot'
 

From 6fa3f2421cc620f88d764628385f3be7bb52aa86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 1 Jul 2015 01:49:05 -0300
Subject: [PATCH 0395/2974] Add 1.0.1 release notes

---
 docs/news.rst | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 2cf1365e5..c775603c9 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,8 +3,18 @@
 Release notes
 =============
 
-1.0 (2015-06-19)
-----------------
+1.0.1 (2015-07-01)
+------------------
+
+- Unquote request path before passing to FTPClient, it already escape paths (:commit:`cc00ad2`)
+- include tests/ to source distribution in MANIFEST.in (:commit:`eca227e`)
+- DOC Fix SelectJmes documentation (:commit:`b8567bc`)
+- DOC Bring Ubuntu and Archlinux outside of Windows subsection (:commit:`392233f`)
+- DOC remove version suffix from ubuntu package (:commit:`5303c66`)
+- DOC Update release date for 1.0 (:commit:`c89fa29`)
+
+1.0.0 (2015-06-19)
+------------------
 
 You will find a lot of new features and bugfixes in this major release.  Make
 sure to check our updated :ref:`overview <intro-overview>` to get a glance of

From 9a15fcf89a151811de8ac783419df0512c863d5e Mon Sep 17 00:00:00 2001
From: Demelziraptor <demelza@circularvale.com>
Date: Thu, 2 Jul 2015 19:51:49 +0200
Subject: [PATCH 0396/2974] interpreting application/x-json as TextResponse

---
 scrapy/responsetypes.py     | 1 +
 tests/test_responsetypes.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 460385444..7c017feef 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -24,6 +24,7 @@ class ResponseTypes(object):
         'application/vnd.wap.xhtml+xml': 'scrapy.http.HtmlResponse',
         'application/xml': 'scrapy.http.XmlResponse',
         'application/json': 'scrapy.http.TextResponse',
+        'application/x-json': 'scrapy.http.TextResponse',
         'application/javascript': 'scrapy.http.TextResponse',
         'application/x-javascript': 'scrapy.http.TextResponse',
         'text/xml': 'scrapy.http.XmlResponse',
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 942d22b99..b4083403a 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -35,6 +35,7 @@ class ResponseTypesTest(unittest.TestCase):
             ('application/vnd.wap.xhtml+xml; charset=utf-8', HtmlResponse),
             ('application/xml; charset=UTF-8', XmlResponse),
             ('application/octet-stream', Response),
+            ('application/x-json; encoding=UTF8;charset=UTF-8', TextResponse),
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_type(source)

From d438075a47fe928ee9bbec7ca2defe79646899b4 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Fri, 3 Jul 2015 16:51:53 +0200
Subject: [PATCH 0397/2974] Make Sphinx autodoc use local, not system-wide
 Scrapy

---
 docs/conf.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/conf.py b/docs/conf.py
index 2d22c6a64..628596b71 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -19,6 +19,7 @@ from os import path
 # absolute, like shown here.
 sys.path.append(path.join(path.dirname(__file__), "_ext"))
 sys.path.append(path.join(path.dirname(path.dirname(__file__)), "scrapy"))
+sys.path.insert(0, path.dirname(path.dirname(__file__)))
 
 
 # General configuration

From 7bef61dbbdd46e77d0287847b17f388f407258f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nikola=20Pavlovi=C4=87?= <nzp@riseup.net>
Date: Fri, 10 Jul 2015 02:35:27 +0200
Subject: [PATCH 0398/2974] Fix reference.

---
 docs/intro/tutorial.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 0d3c49750..73b4cd034 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -505,8 +505,8 @@ doesn't find one -- handy for crawling blogs, forums and other sites with
 pagination.
 
 Another common pattern is to build an item with data from more than one page,
-using a `trick to pass additional data to the callbacks
-<topics-request-response-ref-request-callback-arguments>`_.
+using a :ref:`trick to pass additional data to the callbacks
+<topics-request-response-ref-request-callback-arguments>`.
 
 
 .. note::

From d3d61fde870bb8a1364579959458ad9eacc1a5de Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 10 Jul 2015 19:58:38 -0300
Subject: [PATCH 0399/2974] Fix a recent mock regression by not using it in a
 test

---
 tests/test_settings/__init__.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 5fa22ca77..54b834aa0 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -219,12 +219,9 @@ class SettingsTest(unittest.TestCase):
                              "Trying to modify an immutable Settings object")
 
     def test_frozencopy(self):
-        with mock.patch.object(self.settings, 'copy') as mock_copy:
-            with mock.patch.object(mock_copy, 'freeze') as mock_freeze:
-                mock_object = self.settings.frozencopy()
-                mock_copy.assert_call_once()
-                mock_freeze.assert_call_once()
-                self.assertEqual(mock_object, mock_copy.return_value)
+        frozencopy = self.settings.frozencopy()
+        self.assertTrue(frozencopy.frozen)
+        self.assertIsNot(frozencopy, self.settings)
 
     def test_deprecated_attribute_overrides(self):
         self.settings.set('BAR', 'fuz', priority='cmdline')

From 290ebee090a5d886d825f1373efb7d861c4f2124 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 10 Jul 2015 17:14:50 -0300
Subject: [PATCH 0400/2974] Add OS X installation section to docs

Closes #1342 and (possibly) #1126
---
 docs/intro/install.rst | 63 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 67507016d..26631ac1f 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -86,6 +86,67 @@ You can follow the generic instructions or install Scrapy from `AUR Scrapy packa
 
     yaourt -S scrapy
 
+Mac OS X
+--------
+
+Building Scrapy's dependencies requires the presence of a C compiler and
+development headers. On OS X this is typically provided by Apple’s Xcode
+development tools. To install the Xcode command line tools open a terminal
+window and run::
+
+    xcode-select --install
+
+There's a `known issue <https://github.com/pypa/pip/issues/2468>`_ that
+prevents ``pip`` from updating system packages. This has to be addressed to
+successfully install Scrapy and its dependencies. Here are some proposed
+solutions:
+
+* *(Recommended)* **Don't** use system python, install a new, updated version
+  that doesn't conflict with the rest of your system. Here's how to do it using
+  the `homebrew`_ package manager:
+
+  * Install `homebrew`_::
+
+      ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+
+  * Update your ``PATH`` variable to state that homebrew packages should be
+    used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly
+    if you're using `zsh`_ as default shell)::
+
+      echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc
+
+  * Reload ``.bashrc`` to ensure the changes have taken place::
+
+      source ~/.bashrc
+
+  * Install python::
+
+      brew install python
+
+  * Latest versions of python have ``pip`` bundled with them so you won't need
+    to install it separately. If this is not the case, upgrade python::
+
+      brew update; brew upgrade python
+
+* *(Alternative)* Force system python to load the user installed packages
+  first:
+
+  * Update your ``PYTHONPATH`` variable (Change ``.bashrc`` to ``.zshrc``
+    accordantly if you're using `zsh`_ as default shell)::
+
+      echo "export PYTHONPATH=/Library/Python/2.7/site-packages:$PYTHONPATH" >> ~/.bashrc
+
+  * Reload ``.bashrc`` to ensure the changes have taken place::
+
+      source ~/.bashrc
+
+  * Install ``pip``::
+
+     sudo easy_install pip
+
+After any of these workarounds you should be able to install Scrapy::
+
+  pip install Scrapy
 
 .. _Python: https://www.python.org/
 .. _pip: https://pip.pypa.io/en/latest/installing.html
@@ -95,3 +156,5 @@ You can follow the generic instructions or install Scrapy from `AUR Scrapy packa
 .. _OpenSSL: https://pypi.python.org/pypi/pyOpenSSL
 .. _setuptools: https://pypi.python.org/pypi/setuptools
 .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
+.. _homebrew: http://brew.sh/
+.. _zsh: http://www.zsh.org/

From 5723e6b457312465a136a94ba915fb2ffed005ce Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 10 Jul 2015 18:52:01 -0300
Subject: [PATCH 0401/2974] Add non-python dependencies to Ubuntu install
 section in the docs

Closes #1314 and closes #1198.
---
 docs/intro/install.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 26631ac1f..f1ab90db4 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -79,6 +79,16 @@ Instead, use the official :ref:`Ubuntu Packages <topics-ubuntu>`, which already
 solve all dependencies for you and are continuously updated with the latest bug
 fixes.
 
+If you prefer to build the python dependencies locally instead of relying on
+system packages you'll need to install their required non-python dependencies
+first::
+
+    sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
+
+You can install Scrapy with ``pip`` after that::
+
+    pip install Scrapy
+
 Archlinux
 ---------
 

From 439b376d5bb506ab0e5a0cdecd8af6927a6e2fbf Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 10 Jul 2015 19:06:23 -0300
Subject: [PATCH 0402/2974] Note in install docs about pip being already
 included in python>=2.7.9

---
 docs/intro/install.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index f1ab90db4..246964d3f 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -14,7 +14,8 @@ The installation steps assume that you have the following things installed:
 * `Python`_ 2.7
 
 * `pip`_ and `setuptools`_ Python packages. Nowadays `pip`_ requires and
-  installs `setuptools`_ if not installed.
+  installs `setuptools`_ if not installed. Python 2.7.9 and later include
+  `pip`_ by default, so you may have it already.
 
 * `lxml`_. Most Linux distributions ships prepackaged versions of lxml.
   Otherwise refer to http://lxml.de/installation.html
@@ -58,7 +59,8 @@ Windows
 
   Be sure you download the architecture (win32 or amd64) that matches your system
 
-* Install `pip`_ from https://pip.pypa.io/en/latest/installing.html
+* *(Only required for Python<2.7.9)* Install `pip`_ from
+  https://pip.pypa.io/en/latest/installing.html
 
   Now open a Command prompt to check ``pip`` is installed correctly:: 
 

From 320d3a59b9d89b8f3e7c13add04e28e7453a3f96 Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Fri, 10 Jul 2015 19:15:14 -0300
Subject: [PATCH 0403/2974] Add oldest supported tox version to contributing
 docs

Better fix for #1337
---
 docs/contributing.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index ad9a3805a..87cea2a5f 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -146,6 +146,14 @@ tests requires `tox`_.
 Running tests
 -------------
 
+Make sure you have a recent enough `tox`_ installation:
+
+    ``tox --version``
+
+If your version is older than 1.7.0, please update it first:
+
+    ``pip install -U tox``
+
 To run all tests go to the root directory of Scrapy source code and run:
 
     ``tox``

From 4f56c6c79bd29dbd2e139533179b2039d7054aa4 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sat, 11 Jul 2015 11:27:33 +0000
Subject: [PATCH 0404/2974] Unset environment proxies for tests

because urllib doesn't handle $no_proxy correctly
and the unittest webserver is always local.
---
 tests/__init__.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/__init__.py b/tests/__init__.py
index c6dd45181..d017afad4 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -6,6 +6,13 @@ see http://doc.scrapy.org/en/latest/contributing.html#running-tests
 
 import os
 
+# ignore system-wide proxies for tests
+# which would send requests to a totally unsuspecting server
+# (e.g. because urllib does not fully understand the proxy spec)
+os.environ['http_proxy'] = ''
+os.environ['https_proxy'] = ''
+os.environ['ftp_proxy'] = ''
+
 try:
     import unittest.mock as mock
 except ImportError:

From c908d3166015a9014b3b4ece54f313614cc43032 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Thu, 16 Jul 2015 12:50:26 +0200
Subject: [PATCH 0405/2974] Make RedirectMiddleware respect
 Spider.handle_httpstatus_list

---
 docs/topics/downloader-middleware.rst       |  9 +++++++++
 scrapy/downloadermiddlewares/redirect.py    |  3 ++-
 tests/test_downloadermiddleware_redirect.py | 17 ++++++++++++++---
 3 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index a6a2f7d62..6d986bbf7 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -715,6 +715,15 @@ settings (see the settings documentation for more info):
 If :attr:`Request.meta <scrapy.http.Request.meta>` has ``dont_redirect``
 key set to True, the request will be ignored by this middleware.
 
+If you want to handle some redirect status codes in your spider, you can
+specify these in the ``handle_httpstatus_list`` spider attribute.
+
+For example, if you want the redirect middleware to ignore 301 and 302
+responses (and pass them through to your spider) you can do this::
+
+    class MySpider(CrawlSpider):
+        handle_httpstatus_list = [301, 302]
+
 
 RedirectMiddleware settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index f439f43ae..363e56cb8 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -54,7 +54,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):
     """Handle redirection of requests based on response status and meta-refresh html tag"""
 
     def process_response(self, request, response, spider):
-        if request.meta.get('dont_redirect', False):
+        if (request.meta.get('dont_redirect', False) or
+               response.status in getattr(spider, 'handle_httpstatus_list', [])):
             return response
 
         if request.method == 'HEAD':
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 7e88e71af..be5bfcc6b 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -10,9 +10,9 @@ from scrapy.utils.test import get_crawler
 class RedirectMiddlewareTest(unittest.TestCase):
 
     def setUp(self):
-        crawler = get_crawler(Spider)
-        self.spider = crawler._create_spider('foo')
-        self.mw = RedirectMiddleware.from_crawler(crawler)
+        self.crawler = get_crawler(Spider)
+        self.spider = self.crawler._create_spider('foo')
+        self.mw = RedirectMiddleware.from_crawler(self.crawler)
 
     def test_priority_adjust(self):
         req = Request('http://a.com')
@@ -129,6 +129,17 @@ class RedirectMiddlewareTest(unittest.TestCase):
         self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
         self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])
 
+    def test_spider_handling(self):
+        smartspider = self.crawler._create_spider('smarty')
+        smartspider.handle_httpstatus_list = [404, 301, 302]
+        url = 'http://www.example.com/301'
+        url2 = 'http://www.example.com/redirected'
+        req = Request(url, meta={'dont_redirect': True})
+        rsp = Response(url, headers={'Location': url2}, status=301)
+        r = self.mw.process_response(req, rsp, smartspider)
+        self.assertIs(r, rsp)
+
+
 class MetaRefreshMiddlewareTest(unittest.TestCase):
 
     def setUp(self):

From 198be7d679b3b3e778806cc6e3f50767bd34c5cd Mon Sep 17 00:00:00 2001
From: Andrew Murray <radarhere@gmail.com>
Date: Thu, 16 Jul 2015 23:59:02 +1000
Subject: [PATCH 0406/2974] Fixed typos

---
 docs/news.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index c775603c9..1a2f8dc98 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -938,7 +938,7 @@ Scrapy changes:
 - StackTraceDump extension: also dump trackref live references (:commit:`fe2ce93`)
 - nested items now fully supported in JSON and JSONLines exporters
 - added :reqmeta:`cookiejar` Request meta key to support multiple cookie sessions per spider
-- decoupled encoding detection code to `w3lib.encoding`_, and ported Scrapy code to use that mdule
+- decoupled encoding detection code to `w3lib.encoding`_, and ported Scrapy code to use that module
 - dropped support for Python 2.5. See http://blog.scrapinghub.com/2012/02/27/scrapy-0-15-dropping-support-for-python-2-5/
 - dropped support for Twisted 2.5
 - added :setting:`REFERER_ENABLED` setting, to control referer middleware
@@ -1247,7 +1247,7 @@ New features
 - Added ``dont_click`` argument to ``FormRequest.from_response()`` method (:rev:`1813`, :rev:`1816`)
 - Added ``clickdata`` argument to ``FormRequest.from_response()`` method (:rev:`1802`, :rev:`1803`)
 - Added support for HTTP proxies (``HttpProxyMiddleware``) (:rev:`1781`, :rev:`1785`)
-- Offiste spider middleware now logs messages when filtering out requests (:rev:`1841`)
+- Offsite spider middleware now logs messages when filtering out requests (:rev:`1841`)
 
 Backwards-incompatible changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

From c153217bfbd9c4166bfec8c7eb3b57cf25b41c55 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Fri, 17 Jul 2015 16:11:53 +0200
Subject: [PATCH 0407/2974] Fix RedirectMiddleware test

---
 tests/test_downloadermiddleware_redirect.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index be5bfcc6b..9b00caa51 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -134,7 +134,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
         smartspider.handle_httpstatus_list = [404, 301, 302]
         url = 'http://www.example.com/301'
         url2 = 'http://www.example.com/redirected'
-        req = Request(url, meta={'dont_redirect': True})
+        req = Request(url)
         rsp = Response(url, headers={'Location': url2}, status=301)
         r = self.mw.process_response(req, rsp, smartspider)
         self.assertIs(r, rsp)

From 61cd27e5c7b777a54bd5baebc22ff46a1512298c Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 23 Jul 2015 15:00:06 +0200
Subject: [PATCH 0408/2974] Rename unicode_to_str and str_to_unicode method.
 Fixes GH-778.

---
 scrapy/http/request/form.py       |  4 +-
 scrapy/linkextractors/lxmlhtml.py |  2 +-
 scrapy/linkextractors/sgml.py     |  4 +-
 scrapy/selector/unified.py        |  4 +-
 scrapy/utils/iterators.py         |  4 +-
 scrapy/utils/python.py            | 66 ++++++++++++++++------------
 scrapy/utils/url.py               |  7 +--
 tests/test_exporters.py           |  4 +-
 tests/test_utils_python.py        | 73 ++++++++++++++++---------------
 9 files changed, 92 insertions(+), 76 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index a4695f1a2..e083ce6e9 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -9,7 +9,7 @@ from six.moves.urllib.parse import urljoin, urlencode
 import lxml.html
 import six
 from scrapy.http.request import Request
-from scrapy.utils.python import unicode_to_str
+from scrapy.utils.python import to_bytes
 
 
 class FormRequest(Request):
@@ -48,7 +48,7 @@ def _get_form_url(form, url):
 
 
 def _urlencode(seq, enc):
-    values = [(unicode_to_str(k, enc), unicode_to_str(v, enc))
+    values = [(to_bytes(k, enc), to_bytes(v, enc))
               for k, vs in seq
               for v in (vs if hasattr(vs, '__iter__') else [vs])]
     return urlencode(values, doseq=1)
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 1c31a15b5..c952a5f83 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -10,7 +10,7 @@ import lxml.etree as etree
 from scrapy.selector import Selector
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.python import unique as unique_list, str_to_unicode
+from scrapy.utils.python import unique as unique_list
 from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.response import get_base_url
 
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index a9b8fe9de..e4c2c274f 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -10,7 +10,7 @@ from scrapy.selector import Selector
 from scrapy.link import Link
 from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.python import unique as unique_list, str_to_unicode
+from scrapy.utils.python import unique as unique_list, to_unicode
 from scrapy.utils.response import get_base_url
 from scrapy.exceptions import ScrapyDeprecationWarning
 
@@ -44,7 +44,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
                 link.url = link.url.encode(response_encoding)
             link.url = urljoin(base_url, link.url)
             link.url = safe_url_string(link.url, response_encoding)
-            link.text = str_to_unicode(link.text, response_encoding, errors='replace').strip()
+            link.text = to_unicode(link.text, response_encoding, errors='replace').strip()
             ret.append(link)
 
         return ret
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 0a2f40c6a..69ea9a03d 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -7,7 +7,7 @@ import six
 
 from scrapy.utils.misc import extract_regex
 from scrapy.utils.trackref import object_ref
-from scrapy.utils.python import unicode_to_str, flatten, iflatten
+from scrapy.utils.python import to_bytes, flatten, iflatten
 from scrapy.utils.decorators import deprecated
 from scrapy.http import HtmlResponse, XmlResponse
 from .lxmldocument import LxmlDocument
@@ -44,7 +44,7 @@ def _st(response, st):
 def _response_from_text(text, st):
     rt = XmlResponse if st == 'xml' else HtmlResponse
     return rt(url='about:blank', encoding='utf-8',
-              body=unicode_to_str(text, 'utf-8'))
+              body=to_bytes(text, 'utf-8'))
 
 
 class Selector(object_ref):
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 58ddf6215..b9f682431 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -11,7 +11,7 @@ import six
 
 from scrapy.http import TextResponse, Response
 from scrapy.selector import Selector
-from scrapy.utils.python import re_rsearch, str_to_unicode
+from scrapy.utils.python import re_rsearch, to_unicode
 
 logger = logging.getLogger(__name__)
 
@@ -98,7 +98,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
     encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8'
     def _getrow(csv_r):
-        return [str_to_unicode(field, encoding) for field in next(csv_r)]
+        return [to_unicode(field, encoding) for field in next(csv_r)]
 
     lines = BytesIO(_body_or_str(obj, unicode=False))
 
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index b6100f899..5d5d5b004 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -1,9 +1,5 @@
 """
 This module contains essential stuff that should've come with Python itself ;)
-
-It also contains functions (or functionality) which is in Python versions
-higher than 2.5 which used to be the lowest version supported by Scrapy.
-
 """
 import os
 import re
@@ -13,6 +9,8 @@ import errno
 import six
 from functools import partial, wraps
 
+from scrapy.utils.decorators import deprecated
+
 
 def flatten(x):
     """flatten(sequence) -> list
@@ -56,37 +54,44 @@ def unique(list_, key=lambda x: x):
     return result
 
 
+@deprecated("scrapy.utils.python.to_unicode")
 def str_to_unicode(text, encoding=None, errors='strict'):
-    """Return the unicode representation of text in the given encoding. Unlike
-    .encode(encoding) this function can be applied directly to a unicode
-    object without the risk of double-decoding problems (which can happen if
-    you don't use the default 'ascii' encoding)
-    """
+    """ This function is deprecated.
+    Please use scrapy.utils.python.to_unicode. """
+    return to_unicode(text, encoding, errors)
 
-    if encoding is None:
-        encoding = 'utf-8'
-    if isinstance(text, str):
-        return text.decode(encoding, errors)
-    elif isinstance(text, unicode):
-        return text
-    else:
-        raise TypeError('str_to_unicode must receive a str or unicode object, got %s' % type(text).__name__)
 
+@deprecated("scrapy.utils.python.to_bytes")
 def unicode_to_str(text, encoding=None, errors='strict'):
-    """Return the str representation of text in the given encoding. Unlike
-    .encode(encoding) this function can be applied directly to a str
-    object without the risk of double-decoding problems (which can happen if
-    you don't use the default 'ascii' encoding)
-    """
+    """ This function is deprecated. Please use scrapy.utils.python.to_bytes """
+    return to_bytes(text, encoding, errors)
 
+
+def to_unicode(text, encoding=None, errors='strict'):
+    """Return the unicode representation of a bytes object `text`. If `text`
+    is already an unicode object, return it as-is."""
+    if isinstance(text, six.text_type):
+        return text
+    if not isinstance(text, (bytes, six.text_type)):
+        raise TypeError('to_unicode must receive a bytes, str or unicode '
+                        'object, got %s' % type(text).__name__)
     if encoding is None:
         encoding = 'utf-8'
-    if isinstance(text, unicode):
-        return text.encode(encoding, errors)
-    elif isinstance(text, str):
+    return text.decode(encoding, errors)
+
+
+def to_bytes(text, encoding=None, errors='strict'):
+    """Return the binary representation of `text`. If `text`
+    is already a bytes object, return it as-is."""
+    if isinstance(text, bytes):
         return text
-    else:
-        raise TypeError('unicode_to_str must receive a unicode or str object, got %s' % type(text).__name__)
+    if not isinstance(text, six.string_types):
+        raise TypeError('to_bytes must receive a unicode, str or bytes '
+                        'object, got %s' % type(text).__name__)
+    if encoding is None:
+        encoding = 'utf-8'
+    return text.encode(encoding, errors)
+
 
 def re_rsearch(pattern, text, chunk_size=1024):
     """
@@ -117,6 +122,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
             return (offset + matches[-1].span()[0], offset + matches[-1].span()[1])
     return None
 
+
 def memoizemethod_noargs(method):
     """Decorator to cache the result of a method (without arguments) using a
     weak reference to its object
@@ -131,6 +137,7 @@ def memoizemethod_noargs(method):
 
 _BINARYCHARS = set(map(chr, range(32))) - set(["\0", "\t", "\n", "\r"])
 
+
 def isbinarytext(text):
     """Return True if the given text is considered binary, or false
     otherwise, by looking for binary bytes at their chars
@@ -138,6 +145,7 @@ def isbinarytext(text):
     assert isinstance(text, str), "text must be str, got '%s'" % type(text).__name__
     return any(c in _BINARYCHARS for c in text)
 
+
 def get_func_args(func, stripself=False):
     """Return the argument name list of a callable"""
     if inspect.isfunction(func):
@@ -164,6 +172,7 @@ def get_func_args(func, stripself=False):
         func_args.pop(0)
     return func_args
 
+
 def get_spec(func):
     """Returns (args, kwargs) tuple for a function
     >>> import re
@@ -200,6 +209,7 @@ def get_spec(func):
     kwargs = dict(zip(spec.args[firstdefault:], defaults))
     return args, kwargs
 
+
 def equal_attributes(obj1, obj2, attributes):
     """Compare two objects attributes"""
     # not attributes given return False by default
@@ -249,6 +259,7 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
         d[k] = v
     return d
 
+
 def is_writable(path):
     """Return True if the given path can be written (if it exists) or created
     (if it doesn't exist)
@@ -258,6 +269,7 @@ def is_writable(path):
     else:
         return os.access(os.path.dirname(path), os.W_OK)
 
+
 def setattr_default(obj, name, value):
     """Set attribute value, but only if it's not already set. Similar to
     setdefault() for dicts.
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 5c645aaf0..8a8c56814 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -12,7 +12,7 @@ from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag,
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this move doesn't break old code
 from w3lib.url import *
-from scrapy.utils.python import unicode_to_str
+from scrapy.utils.python import to_bytes
 
 
 def url_is_from_any_domain(url, domains):
@@ -72,8 +72,9 @@ def parse_url(url, encoding=None):
     """Return urlparsed url from the given argument (which could be an already
     parsed url)
     """
-    return url if isinstance(url, ParseResult) else \
-        urlparse(unicode_to_str(url, encoding))
+    if isinstance(url, ParseResult):
+        return url
+    return urlparse(to_bytes(url, encoding))
 
 
 def escape_ajax(url):
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index df1f8f4d5..b24633959 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -8,7 +8,7 @@ from six.moves import cPickle as pickle
 import lxml.etree
 
 from scrapy.item import Item, Field
-from scrapy.utils.python import str_to_unicode
+from scrapy.utils.python import to_unicode
 from scrapy.exporters import (
     BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter,
     XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, PythonItemExporter
@@ -35,7 +35,7 @@ class BaseItemExporterTest(unittest.TestCase):
 
     def _assert_expected_item(self, exported_dict):
         for k, v in exported_dict.items():
-            exported_dict[k] = str_to_unicode(v)
+            exported_dict[k] = to_unicode(v)
         self.assertEqual(self.i, exported_dict)
 
     def assertItemExportWorks(self, item):
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index badfefe04..fa77356de 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -3,45 +3,48 @@ import operator
 import unittest
 from itertools import count
 
-from scrapy.utils.python import str_to_unicode, unicode_to_str, \
-    memoizemethod_noargs, isbinarytext, equal_attributes, \
-    WeakKeyCache, stringify_dict, get_func_args
+from scrapy.utils.python import (
+    memoizemethod_noargs, isbinarytext, equal_attributes,
+    WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode)
 
 __doctests__ = ['scrapy.utils.python']
 
+
+class ToUnicodeTest(unittest.TestCase):
+    def test_converting_an_utf8_encoded_string_to_unicode(self):
+        self.assertEqual(to_unicode('lel\xc3\xb1e'), u'lel\xf1e')
+
+    def test_converting_a_latin_1_encoded_string_to_unicode(self):
+        self.assertEqual(to_unicode('lel\xf1e', 'latin-1'), u'lel\xf1e')
+
+    def test_converting_a_unicode_to_unicode_should_return_the_same_object(self):
+        self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
+
+    def test_converting_a_strange_object_should_raise_TypeError(self):
+        self.assertRaises(TypeError, to_unicode, 423)
+
+    def test_check_errors_argument_works(self):
+        self.assertIn(u'\ufffd', to_unicode('a\xedb', 'utf-8', errors='replace'))
+
+
+class ToBytesTest(unittest.TestCase):
+    def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self):
+        self.assertEqual(to_bytes(u'\xa3 49'), '\xc2\xa3 49')
+
+    def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):
+        self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), '\xa3 49')
+
+    def test_converting_a_regular_string_to_string_should_return_the_same_object(self):
+        self.assertEqual(to_bytes('lel\xf1e'), 'lel\xf1e')
+
+    def test_converting_a_strange_object_should_raise_TypeError(self):
+        self.assertRaises(TypeError, to_bytes, unittest)
+
+    def test_check_errors_argument_works(self):
+        self.assertIn('?', to_bytes(u'a\ufffdb', 'latin-1', errors='replace'))
+
+
 class UtilsPythonTestCase(unittest.TestCase):
-    def test_str_to_unicode(self):
-        # converting an utf-8 encoded string to unicode
-        self.assertEqual(str_to_unicode('lel\xc3\xb1e'), u'lel\xf1e')
-
-        # converting a latin-1 encoded string to unicode
-        self.assertEqual(str_to_unicode('lel\xf1e', 'latin-1'), u'lel\xf1e')
-
-        # converting a unicode to unicode should return the same object
-        self.assertEqual(str_to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
-
-        # converting a strange object should raise TypeError
-        self.assertRaises(TypeError, str_to_unicode, 423)
-
-        # check errors argument works
-        assert u'\ufffd' in str_to_unicode('a\xedb', 'utf-8', errors='replace')
-
-    def test_unicode_to_str(self):
-        # converting a unicode object to an utf-8 encoded string
-        self.assertEqual(unicode_to_str(u'\xa3 49'), '\xc2\xa3 49')
-
-        # converting a unicode object to a latin-1 encoded string
-        self.assertEqual(unicode_to_str(u'\xa3 49', 'latin-1'), '\xa3 49')
-
-        # converting a regular string to string should return the same object
-        self.assertEqual(unicode_to_str('lel\xf1e'), 'lel\xf1e')
-
-        # converting a strange object should raise TypeError
-        self.assertRaises(TypeError, unicode_to_str, unittest)
-
-        # check errors argument works
-        assert '?' in unicode_to_str(u'a\ufffdb', 'latin-1', errors='replace')
-
     def test_memoizemethod_noargs(self):
         class A(object):
 

From 887936ebf6d39a977a3558e249fdb875fc3f21ce Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 23 Jul 2015 17:26:20 +0200
Subject: [PATCH 0409/2974] PY3 port flatten and iflatten

---
 scrapy/utils/python.py | 35 +++++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 5d5d5b004..f691a302f 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -23,8 +23,12 @@ def flatten(x):
     >>> [1, 2, [3,4], (5,6)]
     [1, 2, [3, 4], (5, 6)]
     >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)])
-    [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]"""
-
+    [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]
+    >>> flatten(["foo", "bar"])
+    ['foo', 'bar']
+    >>> flatten(["foo", ["baz", 42], "bar"])
+    ['foo', 'baz', 42, 'bar']
+    """
     return list(iflatten(x))
 
 
@@ -32,15 +36,38 @@ def iflatten(x):
     """iflatten(sequence) -> iterator
 
     Similar to ``.flatten()``, but returns iterator instead"""
-
     for el in x:
-        if hasattr(el, "__iter__"):
+        if is_listlike(el):
             for el_ in flatten(el):
                 yield el_
         else:
             yield el
 
 
+def is_listlike(x):
+    """
+    >>> is_listlike("foo")
+    False
+    >>> is_listlike(5)
+    False
+    >>> is_listlike(b"foo")
+    False
+    >>> is_listlike([b"foo"])
+    True
+    >>> is_listlike((b"foo",))
+    True
+    >>> is_listlike({})
+    True
+    >>> is_listlike(set())
+    True
+    >>> is_listlike((x for x in range(3)))
+    True
+    >>> is_listlike(six.moves.xrange(5))
+    True
+    """
+    return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))
+
+
 def unique(list_, key=lambda x: x):
     """efficient function to uniquify a list preserving item order"""
     seen = set()

From 41bcae5d00a3207fa15c7e2150a92cd103c7552b Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 23 Jul 2015 17:35:23 +0200
Subject: [PATCH 0410/2974] TST fix to_bytes and to_unicode tests in Python 3.x

---
 tests/test_utils_python.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index fa77356de..a5f183e6e 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -12,10 +12,10 @@ __doctests__ = ['scrapy.utils.python']
 
 class ToUnicodeTest(unittest.TestCase):
     def test_converting_an_utf8_encoded_string_to_unicode(self):
-        self.assertEqual(to_unicode('lel\xc3\xb1e'), u'lel\xf1e')
+        self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e')
 
     def test_converting_a_latin_1_encoded_string_to_unicode(self):
-        self.assertEqual(to_unicode('lel\xf1e', 'latin-1'), u'lel\xf1e')
+        self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e')
 
     def test_converting_a_unicode_to_unicode_should_return_the_same_object(self):
         self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
@@ -24,24 +24,24 @@ class ToUnicodeTest(unittest.TestCase):
         self.assertRaises(TypeError, to_unicode, 423)
 
     def test_check_errors_argument_works(self):
-        self.assertIn(u'\ufffd', to_unicode('a\xedb', 'utf-8', errors='replace'))
+        self.assertIn(u'\ufffd', to_unicode(b'a\xedb', 'utf-8', errors='replace'))
 
 
 class ToBytesTest(unittest.TestCase):
     def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self):
-        self.assertEqual(to_bytes(u'\xa3 49'), '\xc2\xa3 49')
+        self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49')
 
     def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):
-        self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), '\xa3 49')
+        self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49')
 
     def test_converting_a_regular_string_to_string_should_return_the_same_object(self):
-        self.assertEqual(to_bytes('lel\xf1e'), 'lel\xf1e')
+        self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e')
 
     def test_converting_a_strange_object_should_raise_TypeError(self):
         self.assertRaises(TypeError, to_bytes, unittest)
 
     def test_check_errors_argument_works(self):
-        self.assertIn('?', to_bytes(u'a\ufffdb', 'latin-1', errors='replace'))
+        self.assertIn(b'?', to_bytes(u'a\ufffdb', 'latin-1', errors='replace'))
 
 
 class UtilsPythonTestCase(unittest.TestCase):

From 407349865279795387b9ff84afb9011e2303bb9d Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 23 Jul 2015 18:33:56 +0200
Subject: [PATCH 0411/2974] port utils.python

* stringify_dict is deprecated
* is_writable is deprecated
* setattr_default is deprecated
* get_spec is untested
* re_rsearch is untested
* retry_on_eintr is untested
---
 scrapy/utils/python.py     | 24 +++++++++++++-------
 tests/test_utils_python.py | 46 ++++++++++++++++++++++----------------
 2 files changed, 43 insertions(+), 27 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index f691a302f..57016811f 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -142,11 +142,14 @@ def re_rsearch(pattern, text, chunk_size=1024):
             yield (text[offset:], offset)
         yield (text, 0)
 
-    pattern = re.compile(pattern) if isinstance(pattern, basestring) else pattern
+    if isinstance(pattern, six.string_types):
+        pattern = re.compile(pattern)
+
     for chunk, offset in _chunk_iter():
         matches = [match for match in pattern.finditer(chunk)]
         if matches:
-            return (offset + matches[-1].span()[0], offset + matches[-1].span()[1])
+            start, end = matches[-1].span()
+            return offset + start, offset + end
     return None
 
 
@@ -162,14 +165,16 @@ def memoizemethod_noargs(method):
         return cache[self]
     return new_method
 
-_BINARYCHARS = set(map(chr, range(32))) - set(["\0", "\t", "\n", "\r"])
+_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
+_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
 
 
 def isbinarytext(text):
-    """Return True if the given text is considered binary, or false
+    """Return True if the given text is considered binary, or False
     otherwise, by looking for binary bytes at their chars
     """
-    assert isinstance(text, str), "text must be str, got '%s'" % type(text).__name__
+    if not isinstance(text, bytes):
+        raise TypeError("text must be bytes, got '%s'" % type(text).__name__)
     return any(c in _BINARYCHARS for c in text)
 
 
@@ -273,20 +278,22 @@ class WeakKeyCache(object):
         return self._weakdict[key]
 
 
+@deprecated
 def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
-    """Return a (new) dict with the unicode keys (and values if, keys_only is
+    """Return a (new) dict with unicode keys (and values when "keys_only" is
     False) of the given dict converted to strings. `dct_or_tuples` can be a
     dict or a list of tuples, like any dict constructor supports.
     """
     d = {}
     for k, v in six.iteritems(dict(dct_or_tuples)):
-        k = k.encode(encoding) if isinstance(k, unicode) else k
+        k = k.encode(encoding) if isinstance(k, six.text_type) else k
         if not keys_only:
-            v = v.encode(encoding) if isinstance(v, unicode) else v
+            v = v.encode(encoding) if isinstance(v, six.text_type) else v
         d[k] = v
     return d
 
 
+@deprecated
 def is_writable(path):
     """Return True if the given path can be written (if it exists) or created
     (if it doesn't exist)
@@ -297,6 +304,7 @@ def is_writable(path):
         return os.access(os.path.dirname(path), os.W_OK)
 
 
+@deprecated
 def setattr_default(obj, name, value):
     """Set attribute value, but only if it's not already set. Similar to
     setdefault() for dicts.
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index a5f183e6e..3b99fec5b 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -2,6 +2,7 @@ import functools
 import operator
 import unittest
 from itertools import count
+import six
 
 from scrapy.utils.python import (
     memoizemethod_noargs, isbinarytext, equal_attributes,
@@ -44,7 +45,7 @@ class ToBytesTest(unittest.TestCase):
         self.assertIn(b'?', to_bytes(u'a\ufffdb', 'latin-1', errors='replace'))
 
 
-class UtilsPythonTestCase(unittest.TestCase):
+class MemoizedMethodTest(unittest.TestCase):
     def test_memoizemethod_noargs(self):
         class A(object):
 
@@ -62,19 +63,23 @@ class UtilsPythonTestCase(unittest.TestCase):
         assert one is two
         assert one is not three
 
+
+class IsBinaryTextTest(unittest.TestCase):
     def test_isbinarytext(self):
+        assert not isbinarytext(b"hello")
 
-        # basic tests
-        assert not isbinarytext("hello")
-
-        # utf-16 strings contain null bytes
+    def utf_16_strings_contain_null_bytes(self):
         assert not isbinarytext(u"hello".encode('utf-16'))
 
-        # one with encoding
-        assert not isbinarytext("<div>Price \xa3</div>")
+    def test_one_with_encoding(self):
+        assert not isbinarytext(b"<div>Price \xa3</div>")
 
-        # finally some real binary bytes
-        assert isbinarytext("\x02\xa3")
+    def test_finally_some_real_binary_bytes(self):
+        assert isbinarytext(b"\x02\xa3")
+
+
+
+class UtilsPythonTestCase(unittest.TestCase):
 
     def test_equal_attributes(self):
         class Obj:
@@ -134,29 +139,32 @@ class UtilsPythonTestCase(unittest.TestCase):
         del k
         self.assertFalse(len(wk._weakdict))
 
+    @unittest.skipUnless(six.PY2, "deprecated function")
     def test_stringify_dict(self):
-        d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
+        d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'}
         d2 = stringify_dict(d, keys_only=False)
         self.assertEqual(d, d2)
-        self.failIf(d is d2) # shouldn't modify in place
-        self.failIf(any(isinstance(x, unicode) for x in d2.keys()))
-        self.failIf(any(isinstance(x, unicode) for x in d2.values()))
+        self.failIf(d is d2)  # shouldn't modify in place
+        self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()))
+        self.failIf(any(isinstance(x, six.text_type) for x in d2.values()))
 
+    @unittest.skipUnless(six.PY2, "deprecated function")
     def test_stringify_dict_tuples(self):
         tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')]
         d = dict(tuples)
         d2 = stringify_dict(tuples, keys_only=False)
         self.assertEqual(d, d2)
-        self.failIf(d is d2) # shouldn't modify in place
-        self.failIf(any(isinstance(x, unicode) for x in d2.keys()), d2.keys())
-        self.failIf(any(isinstance(x, unicode) for x in d2.values()))
+        self.failIf(d is d2)  # shouldn't modify in place
+        self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())
+        self.failIf(any(isinstance(x, six.text_type) for x in d2.values()))
 
+    @unittest.skipUnless(six.PY2, "deprecated function")
     def test_stringify_dict_keys_only(self):
         d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
         d2 = stringify_dict(d)
         self.assertEqual(d, d2)
-        self.failIf(d is d2) # shouldn't modify in place
-        self.failIf(any(isinstance(x, unicode) for x in d2.keys()))
+        self.failIf(d is d2)  # shouldn't modify in place
+        self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()))
 
     def test_get_func_args(self):
         def f1(a, b, c):
@@ -194,7 +202,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         self.assertEqual(get_func_args(object), [])
 
         # TODO: how do we fix this to return the actual argument names?
-        self.assertEqual(get_func_args(unicode.split), [])
+        self.assertEqual(get_func_args(six.text_type.split), [])
         self.assertEqual(get_func_args(" ".join), [])
         self.assertEqual(get_func_args(operator.itemgetter(2)), [])
 

From a7b4a3e7f95fed8f1d4b9bddf79f7e9a91555fdb Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 23 Jul 2015 20:03:33 +0200
Subject: [PATCH 0412/2974] cleanup

* run test_utils_python in Python 3;
* make tests for 'errors' argument more explicit
* add missing test_ prefix utf_16_strings_contain_null_bytes;
* cleanup test names.
---
 tests/py3-ignores.txt      |  1 -
 tests/test_utils_python.py | 20 +++++++++++++-------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 842217dd0..aa091835c 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -59,7 +59,6 @@ tests/test_utils_defer.py
 tests/test_utils_iterators.py
 tests/test_utils_jsonrpc.py
 tests/test_utils_log.py
-tests/test_utils_python.py
 tests/test_utils_reqser.py
 tests/test_utils_request.py
 tests/test_utils_response.py
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3b99fec5b..ca394ebf5 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -24,8 +24,11 @@ class ToUnicodeTest(unittest.TestCase):
     def test_converting_a_strange_object_should_raise_TypeError(self):
         self.assertRaises(TypeError, to_unicode, 423)
 
-    def test_check_errors_argument_works(self):
-        self.assertIn(u'\ufffd', to_unicode(b'a\xedb', 'utf-8', errors='replace'))
+    def test_errors_argument(self):
+        self.assertEqual(
+            to_unicode(b'a\xedb', 'utf-8', errors='replace'),
+            u'a\ufffdb'
+        )
 
 
 class ToBytesTest(unittest.TestCase):
@@ -35,14 +38,17 @@ class ToBytesTest(unittest.TestCase):
     def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):
         self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49')
 
-    def test_converting_a_regular_string_to_string_should_return_the_same_object(self):
+    def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self):
         self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e')
 
     def test_converting_a_strange_object_should_raise_TypeError(self):
         self.assertRaises(TypeError, to_bytes, unittest)
 
-    def test_check_errors_argument_works(self):
-        self.assertIn(b'?', to_bytes(u'a\ufffdb', 'latin-1', errors='replace'))
+    def test_errors_argument(self):
+        self.assertEqual(
+            to_bytes(u'a\ufffdb', 'latin-1', errors='replace'),
+            b'a?b'
+        )
 
 
 class MemoizedMethodTest(unittest.TestCase):
@@ -68,13 +74,13 @@ class IsBinaryTextTest(unittest.TestCase):
     def test_isbinarytext(self):
         assert not isbinarytext(b"hello")
 
-    def utf_16_strings_contain_null_bytes(self):
+    def test_utf_16_strings_contain_null_bytes(self):
         assert not isbinarytext(u"hello".encode('utf-16'))
 
     def test_one_with_encoding(self):
         assert not isbinarytext(b"<div>Price \xa3</div>")
 
-    def test_finally_some_real_binary_bytes(self):
+    def test_real_binary_bytes(self):
         assert isbinarytext(b"\x02\xa3")
 
 

From b3dda1e9c577b2387520fb163166d7632e3fd78a Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Fri, 24 Jul 2015 01:48:43 +0000
Subject: [PATCH 0413/2974] Put a blurb about support channels in CONTRIBUTING

---
 CONTRIBUTING.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 2309bb1f6..88c472f6f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,2 +1,6 @@
 The guidelines for contributing are available here:
 http://doc.scrapy.org/en/master/contributing.html
+
+Please do not abuse the issue tracker for support questions.
+If your issue topic can be rephrased to "How to ...?", please use the
+support channels to get it answered: http://scrapy.org/community/

From f576b3ffeeb86ceb88631e14dd56ea5b96f53e35 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 25 Jul 2015 00:40:45 +0200
Subject: [PATCH 0414/2974] [tmp] improve python 3 support for scrapy.utils.url

---
 scrapy/utils/python.py  |  9 +++++++
 scrapy/utils/url.py     | 22 ++++++++++-------
 tests/test_utils_url.py | 53 +++++++++++++++++++++++++++--------------
 3 files changed, 58 insertions(+), 26 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 57016811f..94ee8a557 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -120,6 +120,15 @@ def to_bytes(text, encoding=None, errors='strict'):
     return text.encode(encoding, errors)
 
 
+def to_native_str(text, encoding=None, errors='strict'):
+    """ Return str representation of `text`
+    (bytes in Python 2.x and unicode in Python 3.x). """
+    if six.PY2:
+        return to_bytes(text, encoding, errors)
+    else:
+        return to_unicode(text, encoding, errors)
+
+
 def re_rsearch(pattern, text, chunk_size=1024):
     """
     This function does a reverse search in a text using a regular expression
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 8a8c56814..99f350361 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -10,19 +10,20 @@ from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag,
                                     urlparse, parse_qsl, urlencode,
                                     unquote)
 
-# scrapy.utils.url was moved to w3lib.url and import * ensures this move doesn't break old code
+# scrapy.utils.url was moved to w3lib.url and import * ensures this
+# move doesn't break old code
 from w3lib.url import *
-from scrapy.utils.python import to_bytes
+from w3lib.url import _safe_chars
+from scrapy.utils.python import to_native_str
 
 
 def url_is_from_any_domain(url, domains):
     """Return True if the url belongs to any of the given domains"""
     host = parse_url(url).netloc.lower()
-
-    if host:
-        return any(((host == d.lower()) or (host.endswith('.%s' % d.lower())) for d in domains))
-    else:
+    if not host:
         return False
+    domains = [d.lower() for d in domains]
+    return any((host == d) or (host.endswith('.%s' % d)) for d in domains)
 
 
 def url_is_from_spider(url, spider):
@@ -36,7 +37,7 @@ def url_has_any_extension(url, extensions):
 
 
 def canonicalize_url(url, keep_blank_values=True, keep_fragments=False,
-        encoding=None):
+                     encoding=None):
     """Canonicalize the given url by applying the following procedures:
 
     - sort query arguments, first by key, then by value
@@ -57,6 +58,11 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False,
     keyvals = parse_qsl(query, keep_blank_values)
     keyvals.sort()
     query = urlencode(keyvals)
+
+    # XXX: copied from w3lib.url.safe_url_string to add encoding argument
+    # path = to_native_str(path, encoding)
+    # path = moves.urllib.parse.quote(path, _safe_chars, encoding='latin1') or '/'
+
     path = safe_url_string(_unquotepath(path)) or '/'
     fragment = '' if not keep_fragments else fragment
     return urlunparse((scheme, netloc.lower(), path, params, query, fragment))
@@ -74,7 +80,7 @@ def parse_url(url, encoding=None):
     """
     if isinstance(url, ParseResult):
         return url
-    return urlparse(to_bytes(url, encoding))
+    return urlparse(to_native_str(url, encoding))
 
 
 def escape_ajax(url):
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 860c76bae..7bf0e5b4a 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,7 +1,10 @@
+# -*- coding: utf-8 -*-
 import unittest
 
+import six
 from scrapy.spiders import Spider
-from scrapy.utils.url import url_is_from_any_domain, url_is_from_spider, canonicalize_url
+from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
+                              canonicalize_url)
 
 __doctests__ = ['scrapy.utils.url']
 
@@ -70,18 +73,23 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider))
         self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider))
 
+
+class CanonicalizeUrlTest(unittest.TestCase):
+
     def test_canonicalize_url(self):
         # simplest case
         self.assertEqual(canonicalize_url("http://www.example.com/"),
                                           "http://www.example.com/")
 
-        # always return a str
+    def test_return_str(self):
         assert isinstance(canonicalize_url(u"http://www.example.com"), str)
+        assert isinstance(canonicalize_url(b"http://www.example.com"), str)
 
-        # append missing path
+    def test_append_missing_path(self):
         self.assertEqual(canonicalize_url("http://www.example.com"),
                                           "http://www.example.com/")
-        # typical usage
+
+    def test_typical_usage(self):
         self.assertEqual(canonicalize_url("http://www.example.com/do?a=1&b=2&c=3"),
                                           "http://www.example.com/do?a=1&b=2&c=3")
         self.assertEqual(canonicalize_url("http://www.example.com/do?c=1&b=2&a=3"),
@@ -89,11 +97,11 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertEqual(canonicalize_url("http://www.example.com/do?&a=1"),
                                           "http://www.example.com/do?a=1")
 
-        # sorting by argument values
+    def test_sorting(self):
         self.assertEqual(canonicalize_url("http://www.example.com/do?c=3&b=5&b=2&a=50"),
                                           "http://www.example.com/do?a=50&b=2&b=5&c=3")
 
-        # using keep_blank_values
+    def test_keep_blank_values(self):
         self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2", keep_blank_values=False),
                                           "http://www.example.com/do?a=2")
         self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2"),
@@ -106,7 +114,7 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertEqual(canonicalize_url(u'http://www.example.com/do?1750,4'),
                                            'http://www.example.com/do?1750%2C4=')
 
-        # spaces
+    def test_spaces(self):
         self.assertEqual(canonicalize_url("http://www.example.com/do?q=a space&a=1"),
                                           "http://www.example.com/do?a=1&q=a+space")
         self.assertEqual(canonicalize_url("http://www.example.com/do?q=a+space&a=1"),
@@ -114,43 +122,52 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertEqual(canonicalize_url("http://www.example.com/do?q=a%20space&a=1"),
                                           "http://www.example.com/do?a=1&q=a+space")
 
-        # normalize percent-encoding case (in paths)
+    @unittest.skipUnless(six.PY2, "TODO")
+    def test_normalize_percent_encoding_in_paths(self):
         self.assertEqual(canonicalize_url("http://www.example.com/a%a3do"),
                                           "http://www.example.com/a%A3do"),
-        # normalize percent-encoding case (in query arguments)
+
+    @unittest.skipUnless(six.PY2, "TODO")
+    def test_normalize_percent_encoding_in_query_arguments(self):
         self.assertEqual(canonicalize_url("http://www.example.com/do?k=b%a3"),
                                           "http://www.example.com/do?k=b%A3")
 
-        # non-ASCII percent-encoding in paths
+    def test_non_ascii_percent_encoding_in_paths(self):
         self.assertEqual(canonicalize_url("http://www.example.com/a do?a=1"),
                                           "http://www.example.com/a%20do?a=1"),
         self.assertEqual(canonicalize_url("http://www.example.com/a %20do?a=1"),
                                           "http://www.example.com/a%20%20do?a=1"),
-        self.assertEqual(canonicalize_url("http://www.example.com/a do\xc2\xa3.html?a=1"),
+        self.assertEqual(canonicalize_url(u"http://www.example.com/a do£.html?a=1"),
                                           "http://www.example.com/a%20do%C2%A3.html?a=1")
-        # non-ASCII percent-encoding in query arguments
+        self.assertEqual(canonicalize_url(b"http://www.example.com/a do\xc2\xa3.html?a=1"),
+                                          "http://www.example.com/a%20do%C2%A3.html?a=1")
+
+    def test_non_ascii_percent_encoding_in_query_arguments(self):
         self.assertEqual(canonicalize_url(u"http://www.example.com/do?price=\xa3500&a=5&z=3"),
                                           u"http://www.example.com/do?a=5&price=%C2%A3500&z=3")
-        self.assertEqual(canonicalize_url("http://www.example.com/do?price=\xc2\xa3500&a=5&z=3"),
+        self.assertEqual(canonicalize_url(b"http://www.example.com/do?price=\xc2\xa3500&a=5&z=3"),
                                           "http://www.example.com/do?a=5&price=%C2%A3500&z=3")
-        self.assertEqual(canonicalize_url("http://www.example.com/do?price(\xc2\xa3)=500&a=1"),
+        self.assertEqual(canonicalize_url(b"http://www.example.com/do?price(\xc2\xa3)=500&a=1"),
                                           "http://www.example.com/do?a=1&price%28%C2%A3%29=500")
 
-        # urls containing auth and ports
+    def test_urls_with_auth_and_ports(self):
         self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com:81/do?now=1"),
                                           u"http://user:pass@www.example.com:81/do?now=1")
 
-        # remove fragments
+    def test_remove_fragments(self):
         self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com/do?a=1#frag"),
                                           u"http://user:pass@www.example.com/do?a=1")
         self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com/do?a=1#frag", keep_fragments=True),
                                           u"http://user:pass@www.example.com/do?a=1#frag")
 
+    def test_dont_convert_safe_characters(self):
         # dont convert safe characters to percent encoding representation
         self.assertEqual(canonicalize_url(
             "http://www.simplybedrooms.com/White-Bedroom-Furniture/Bedroom-Mirror:-Josephine-Cheval-Mirror.html"),
             "http://www.simplybedrooms.com/White-Bedroom-Furniture/Bedroom-Mirror:-Josephine-Cheval-Mirror.html")
 
+    @unittest.skipUnless(six.PY2, "TODO")
+    def test_safe_characters_unicode(self):
         # urllib.quote uses a mapping cache of encoded characters. when parsing
         # an already percent-encoded url, it will fail if that url was not
         # percent-encoded as utf-8, that's why canonicalize_url must always
@@ -159,11 +176,11 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertEqual(canonicalize_url(u'http://www.example.com/caf%E9-con-leche.htm'),
                                            'http://www.example.com/caf%E9-con-leche.htm')
 
-        # domains are case insensitive
+    def test_domains_are_case_insensitive(self):
         self.assertEqual(canonicalize_url("http://www.EXAMPLE.com/"),
                                           "http://www.example.com/")
 
-        # quoted slash and question sign
+    def test_quoted_slash_and_question_sign(self):
         self.assertEqual(canonicalize_url("http://foo.com/AC%2FDC+rocks%3f/?yeah=1"),
                          "http://foo.com/AC%2FDC+rocks%3F/?yeah=1")
         self.assertEqual(canonicalize_url("http://foo.com/AC%2FDC/"),

From 9aaa8f81251e3859ee52eef2aaed60ce9f28ecb0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Zolt=C3=A1n=20Szeredi?= <zoli.szeredi@gmail.com>
Date: Sat, 25 Jul 2015 12:58:39 +0200
Subject: [PATCH 0415/2974] Improve python3 support for test_utils_defer.

---
 tests/test_utils_defer.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 260ed6c2e..f49bbfafe 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -5,6 +5,8 @@ from twisted.python.failure import Failure
 from scrapy.utils.defer import mustbe_deferred, process_chain, \
     process_chain_both, process_parallel, iter_errback
 
+from six.moves import xrange
+
 
 class MustbeDeferredTest(unittest.TestCase):
     def test_success_function(self):
@@ -86,7 +88,7 @@ class IterErrbackTest(unittest.TestCase):
 
         errors = []
         out = list(iter_errback(itergood(), errors.append))
-        self.assertEqual(out, range(10))
+        self.assertEqual(out, list(range(10)))
         self.failIf(errors)
 
     def test_iter_errback_bad(self):

From 1bf7a98000be2d055729d771bb30d220d5c91672 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Zolt=C3=A1n=20Szeredi?= <zoli.szeredi@gmail.com>
Date: Sat, 25 Jul 2015 13:05:50 +0200
Subject: [PATCH 0416/2974] Remove test_utils_defer from ignores.

---
 tests/py3-ignores.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index aa091835c..efb13bcd2 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -55,7 +55,6 @@ tests/test_spidermiddleware_offsite.py
 tests/test_spidermiddleware_referer.py
 tests/test_spider.py
 tests/test_stats.py
-tests/test_utils_defer.py
 tests/test_utils_iterators.py
 tests/test_utils_jsonrpc.py
 tests/test_utils_log.py

From e853d9e910151a83dcb944fa3c7ad82a19075edd Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 25 Jul 2015 13:08:44 +0200
Subject: [PATCH 0417/2974] partial port of Request and Response

---
 scrapy/http/request/__init__.py  |  27 +++----
 scrapy/http/request/form.py      |   6 +-
 scrapy/http/response/__init__.py |  30 ++++----
 scrapy/http/response/text.py     |  29 ++++----
 scrapy/selector/unified.py       |   4 +-
 scrapy/utils/misc.py             |   9 +--
 tests/test_http_request.py       | 105 ++++++++++++++-------------
 tests/test_http_response.py      | 120 +++++++++++++++----------------
 8 files changed, 161 insertions(+), 169 deletions(-)

diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 89d9ee584..6b802b36e 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -8,6 +8,7 @@ import six
 from w3lib.url import safe_url_string
 
 from scrapy.http.headers import Headers
+from scrapy.utils.python import to_native_str, to_bytes
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import escape_ajax
 from scrapy.http.common import obsolete_setter
@@ -46,15 +47,12 @@ class Request(object_ref):
         return self._url
 
     def _set_url(self, url):
-        if isinstance(url, str):
-            self._url = escape_ajax(safe_url_string(url))
-        elif isinstance(url, six.text_type):
-            if self.encoding is None:
-                raise TypeError('Cannot convert unicode url - %s has no encoding' %
-                                type(self).__name__)
-            self._set_url(url.encode(self.encoding))
-        else:
+        if not isinstance(url, six.string_types):
             raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
+
+        url = to_native_str(url, self.encoding)
+        self._url = escape_ajax(safe_url_string(url))
+
         if ':' not in self._url:
             raise ValueError('Missing scheme in request url: %s' % self._url)
 
@@ -64,17 +62,10 @@ class Request(object_ref):
         return self._body
 
     def _set_body(self, body):
-        if isinstance(body, str):
-            self._body = body
-        elif isinstance(body, six.text_type):
-            if self.encoding is None:
-                raise TypeError('Cannot convert unicode body - %s has no encoding' %
-                                type(self).__name__)
-            self._body = body.encode(self.encoding)
-        elif body is None:
-            self._body = ''
+        if body is None:
+            self._body = b''
         else:
-            raise TypeError("Request body must either str or unicode. Got: '%s'" % type(body).__name__)
+            self._body = to_bytes(body, self.encoding)
 
     body = property(_get_body, obsolete_setter(_set_body, 'body'))
 
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index e083ce6e9..65c6895f0 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -9,7 +9,7 @@ from six.moves.urllib.parse import urljoin, urlencode
 import lxml.html
 import six
 from scrapy.http.request import Request
-from scrapy.utils.python import to_bytes
+from scrapy.utils.python import to_bytes, is_listlike
 
 
 class FormRequest(Request):
@@ -25,7 +25,7 @@ class FormRequest(Request):
             items = formdata.items() if isinstance(formdata, dict) else formdata
             querystr = _urlencode(items, self.encoding)
             if self.method == 'POST':
-                self.headers.setdefault('Content-Type', 'application/x-www-form-urlencoded')
+                self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded')
                 self._set_body(querystr)
             else:
                 self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr)
@@ -50,7 +50,7 @@ def _get_form_url(form, url):
 def _urlencode(seq, enc):
     values = [(to_bytes(k, enc), to_bytes(v, enc))
               for k, vs in seq
-              for v in (vs if hasattr(vs, '__iter__') else [vs])]
+              for v in (vs if is_listlike(vs) else [vs])]
     return urlencode(values, doseq=1)
 
 
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 5b1d34c6d..983154001 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -4,9 +4,6 @@ responses in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
-
-import copy
-
 from six.moves.urllib.parse import urljoin
 
 from scrapy.http.headers import Headers
@@ -15,7 +12,7 @@ from scrapy.http.common import obsolete_setter
 
 class Response(object_ref):
 
-    def __init__(self, url, status=200, headers=None, body='', flags=None, request=None):
+    def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None):
         self.headers = Headers(headers or {})
         self.status = int(status)
         self._set_body(body)
@@ -28,8 +25,10 @@ class Response(object_ref):
         try:
             return self.request.meta
         except AttributeError:
-            raise AttributeError("Response.meta not available, this response " \
-                "is not tied to any request")
+            raise AttributeError(
+                "Response.meta not available, this response "
+                "is not tied to any request"
+            )
 
     def _get_url(self):
         return self._url
@@ -38,7 +37,7 @@ class Response(object_ref):
         if isinstance(url, str):
             self._url = url
         else:
-            raise TypeError('%s url must be str, got %s:' % (type(self).__name__, \
+            raise TypeError('%s url must be str, got %s:' % (type(self).__name__,
                 type(url).__name__))
 
     url = property(_get_url, obsolete_setter(_set_url, 'url'))
@@ -47,16 +46,15 @@ class Response(object_ref):
         return self._body
 
     def _set_body(self, body):
-        if isinstance(body, str):
-            self._body = body
-        elif isinstance(body, unicode):
-            raise TypeError("Cannot assign a unicode body to a raw Response. " \
-                "Use TextResponse, HtmlResponse, etc")
-        elif body is None:
-            self._body = ''
+        if body is None:
+            self._body = b''
+        elif not isinstance(body, bytes):
+            raise TypeError(
+                "Response body must be bytes. "
+                "If you want to pass unicode body use TextResponse "
+                "or HtmlResponse.")
         else:
-            raise TypeError("Response body must either be str or unicode. Got: '%s'" \
-                % type(body).__name__)
+            self._body = body
 
     body = property(_get_body, obsolete_setter(_set_body, 'body'))
 
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 9a435f194..1c416bf82 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -5,13 +5,14 @@ discovering (through HTTP headers) to base Response class.
 See documentation in docs/topics/request-response.rst
 """
 
+import six
 from six.moves.urllib.parse import urljoin
 
 from w3lib.encoding import html_to_unicode, resolve_encoding, \
     html_body_declared_encoding, http_content_type_encoding
 from scrapy.http.response import Response
 from scrapy.utils.response import get_base_url
-from scrapy.utils.python import memoizemethod_noargs
+from scrapy.utils.python import memoizemethod_noargs, to_native_str
 
 
 class TextResponse(Response):
@@ -26,18 +27,18 @@ class TextResponse(Response):
         super(TextResponse, self).__init__(*args, **kwargs)
 
     def _set_url(self, url):
-        if isinstance(url, unicode):
-            if self.encoding is None:
-                raise TypeError('Cannot convert unicode url - %s has no encoding' %
-                    type(self).__name__)
-            self._url = url.encode(self.encoding)
+        if isinstance(url, six.text_type):
+            if six.PY2 and self.encoding is None:
+                raise TypeError("Cannot convert unicode url - %s "
+                                "has no encoding" % type(self).__name__)
+            self._url = to_native_str(url, self.encoding)
         else:
             super(TextResponse, self)._set_url(url)
 
     def _set_body(self, body):
-        self._body = ''
-        if isinstance(body, unicode):
-            if self.encoding is None:
+        self._body = b''  # used by encoding detection
+        if isinstance(body, six.text_type):
+            if self._encoding is None:
                 raise TypeError('Cannot convert unicode body - %s has no encoding' %
                     type(self).__name__)
             self._body = body.encode(self._encoding)
@@ -73,14 +74,14 @@ class TextResponse(Response):
 
     @memoizemethod_noargs
     def _headers_encoding(self):
-        content_type = self.headers.get('Content-Type')
-        return http_content_type_encoding(content_type)
+        content_type = self.headers.get(b'Content-Type', b'')
+        return http_content_type_encoding(to_native_str(content_type))
 
     def _body_inferred_encoding(self):
         if self._cached_benc is None:
-            content_type = self.headers.get('Content-Type')
-            benc, ubody = html_to_unicode(content_type, self.body, \
-                    auto_detect_fun=self._auto_detect_fun, \
+            content_type = to_native_str(self.headers.get(b'Content-Type', b''))
+            benc, ubody = html_to_unicode(content_type, self.body,
+                    auto_detect_fun=self._auto_detect_fun,
                     default_encoding=self._DEFAULT_ENCODING)
             self._cached_benc = benc
             self._cached_ubody = ubody
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 69ea9a03d..eed8f94f7 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -121,7 +121,7 @@ class Selector(object_ref):
         try:
             return etree.tostring(self._root,
                                   method=self._tostring_method,
-                                  encoding=unicode,
+                                  encoding="unicode",
                                   with_tail=False)
         except (AttributeError, TypeError):
             if self._root is True:
@@ -129,7 +129,7 @@ class Selector(object_ref):
             elif self._root is False:
                 return u'0'
             else:
-                return unicode(self._root)
+                return six.text_type(self._root)
 
     def register_namespace(self, prefix, uri):
         if self.namespaces is None:
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index f1aa4fcdb..c269b7f74 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -7,7 +7,7 @@ from pkgutil import iter_modules
 import six
 from w3lib.html import replace_entities
 
-from scrapy.utils.python import flatten
+from scrapy.utils.python import flatten, to_unicode
 from scrapy.item import BaseItem
 
 
@@ -81,7 +81,7 @@ def extract_regex(regex, text, encoding='utf-8'):
     * if the regex doesn't contain any group the entire regex matching is returned
     """
 
-    if isinstance(regex, basestring):
+    if isinstance(regex, six.string_types):
         regex = re.compile(regex, re.UNICODE)
 
     try:
@@ -90,10 +90,11 @@ def extract_regex(regex, text, encoding='utf-8'):
         strings = regex.findall(text)    # full regex or numbered groups
     strings = flatten(strings)
 
-    if isinstance(text, unicode):
+    if isinstance(text, six.text_type):
         return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
     else:
-        return [replace_entities(unicode(s, encoding), keep=['lt', 'amp']) for s in strings]
+        return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])
+                for s in strings]
 
 
 def md5sum(file):
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index c81eebfa6..805a22a1f 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1,9 +1,12 @@
 import cgi
 import unittest
+
+import six
 from six.moves import xmlrpc_client as xmlrpclib
 from six.moves.urllib.parse import urlparse
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, HtmlResponse
+from scrapy.utils.python import to_bytes, to_native_str
 
 
 class RequestTest(unittest.TestCase):
@@ -31,13 +34,13 @@ class RequestTest(unittest.TestCase):
         self.assertEqual(r.meta, self.default_meta)
 
         meta = {"lala": "lolo"}
-        headers = {"caca": "coco"}
+        headers = {b"caca": b"coco"}
         r = self.request_class("http://www.example.com", meta=meta, headers=headers, body="a body")
 
         assert r.meta is not meta
         self.assertEqual(r.meta, meta)
         assert r.headers is not headers
-        self.assertEqual(r.headers["caca"], "coco")
+        self.assertEqual(r.headers[b"caca"], b"coco")
 
     def test_url_no_scheme(self):
         self.assertRaises(ValueError, self.request_class, 'foo')
@@ -45,7 +48,7 @@ class RequestTest(unittest.TestCase):
     def test_headers(self):
         # Different ways of setting headers attribute
         url = 'http://www.scrapy.org'
-        headers = {'Accept':'gzip', 'Custom-Header':'nothing to tell you'}
+        headers = {b'Accept':'gzip', b'Custom-Header':'nothing to tell you'}
         r = self.request_class(url=url, headers=headers)
         p = self.request_class(url=url, headers=r.headers)
 
@@ -57,9 +60,9 @@ class RequestTest(unittest.TestCase):
         h = Headers({'key1': u'val1', u'key2': 'val2'})
         h[u'newkey'] = u'newval'
         for k, v in h.iteritems():
-            self.assert_(isinstance(k, str))
+            self.assert_(isinstance(k, bytes))
             for s in v:
-                self.assert_(isinstance(s, str))
+                self.assert_(isinstance(s, bytes))
 
     def test_eq(self):
         url = 'http://www.scrapy.org'
@@ -73,17 +76,17 @@ class RequestTest(unittest.TestCase):
         self.assertEqual(len(set_), 2)
 
     def test_url(self):
-        """Request url tests"""
         r = self.request_class(url="http://www.scrapy.org/path")
         self.assertEqual(r.url, "http://www.scrapy.org/path")
 
-        # url quoting on creation
+    def test_url_quoting(self):
         r = self.request_class(url="http://www.scrapy.org/blank%20space")
         self.assertEqual(r.url, "http://www.scrapy.org/blank%20space")
         r = self.request_class(url="http://www.scrapy.org/blank space")
         self.assertEqual(r.url, "http://www.scrapy.org/blank%20space")
 
-        # url encoding
+    @unittest.skipUnless(six.PY2, "TODO")
+    def test_url_encoding(self):
         r1 = self.request_class(url=u"http://www.scrapy.org/price/\xa3", encoding="utf-8")
         r2 = self.request_class(url=u"http://www.scrapy.org/price/\xa3", encoding="latin1")
         self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3")
@@ -91,19 +94,19 @@ class RequestTest(unittest.TestCase):
 
     def test_body(self):
         r1 = self.request_class(url="http://www.example.com/")
-        assert r1.body == ''
+        assert r1.body == b''
 
-        r2 = self.request_class(url="http://www.example.com/", body="")
-        assert isinstance(r2.body, str)
+        r2 = self.request_class(url="http://www.example.com/", body=b"")
+        assert isinstance(r2.body, bytes)
         self.assertEqual(r2.encoding, 'utf-8') # default encoding
 
         r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8')
-        assert isinstance(r3.body, str)
-        self.assertEqual(r3.body, "Price: \xc2\xa3100")
+        assert isinstance(r3.body, bytes)
+        self.assertEqual(r3.body, b"Price: \xc2\xa3100")
 
         r4 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='latin1')
-        assert isinstance(r4.body, str)
-        self.assertEqual(r4.body, "Price: \xa3100")
+        assert isinstance(r4.body, bytes)
+        self.assertEqual(r4.body, b"Price: \xa3100")
 
     def test_ajax_url(self):
         # ascii url
@@ -155,18 +158,19 @@ class RequestTest(unittest.TestCase):
     def test_replace(self):
         """Test Request.replace() method"""
         r1 = self.request_class("http://www.example.com", method='GET')
-        hdrs = Headers(dict(r1.headers, key='value'))
+        hdrs = Headers(r1.headers)
+        hdrs[b'key'] = b'value'
         r2 = r1.replace(method="POST", body="New body", headers=hdrs)
         self.assertEqual(r1.url, r2.url)
         self.assertEqual((r1.method, r2.method), ("GET", "POST"))
-        self.assertEqual((r1.body, r2.body), ('', "New body"))
+        self.assertEqual((r1.body, r2.body), (b'', b"New body"))
         self.assertEqual((r1.headers, r2.headers), (self.default_headers, hdrs))
 
         # Empty attributes (which may fail if not compared properly)
         r3 = self.request_class("http://www.example.com", meta={'a': 1}, dont_filter=True)
-        r4 = r3.replace(url="http://www.example.com/2", body='', meta={}, dont_filter=False)
+        r4 = r3.replace(url="http://www.example.com/2", body=b'', meta={}, dont_filter=False)
         self.assertEqual(r4.url, "http://www.example.com/2")
-        self.assertEqual(r4.body, '')
+        self.assertEqual(r4.body, b'')
         self.assertEqual(r4.meta, {})
         assert r4.dont_filter is False
 
@@ -184,39 +188,41 @@ class FormRequestTest(RequestTest):
 
     request_class = FormRequest
 
-    def assertSortedEqual(self, first, second, msg=None):
+    def assertQueryEqual(self, first, second, msg=None):
+        first = to_native_str(first).split("&")
+        second = to_native_str(second).split("&")
         return self.assertEqual(sorted(first), sorted(second), msg)
 
     def test_empty_formdata(self):
         r1 = self.request_class("http://www.example.com", formdata={})
-        self.assertEqual(r1.body, '')
+        self.assertEqual(r1.body, b'')
 
+    @unittest.skipUnless(six.PY2, "TODO")
     def test_default_encoding(self):
         # using default encoding (utf-8)
         data = {'one': 'two', 'price': '\xc2\xa3 100'}
         r2 = self.request_class("http://www.example.com", formdata=data)
         self.assertEqual(r2.method, 'POST')
         self.assertEqual(r2.encoding, 'utf-8')
-        self.assertSortedEqual(r2.body.split('&'),
-                               'price=%C2%A3+100&one=two'.split('&'))
-        self.assertEqual(r2.headers['Content-Type'], 'application/x-www-form-urlencoded')
+        self.assertQueryEqual(r2.body, b'price=%C2%A3+100&one=two')
+        self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')
 
     def test_custom_encoding(self):
         data = {'price': u'\xa3 100'}
         r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
         self.assertEqual(r3.encoding, 'latin1')
-        self.assertEqual(r3.body, 'price=%A3+100')
+        self.assertEqual(r3.body, b'price=%A3+100')
 
     def test_multi_key_values(self):
         # using multiples values for a single key
         data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']}
         r3 = self.request_class("http://www.example.com", formdata=data)
-        self.assertSortedEqual(r3.body.split('&'),
-            'colours=red&colours=blue&colours=green&price=%C2%A3+100'.split('&'))
+        self.assertQueryEqual(r3.body,
+            b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
 
     def test_from_response_post(self):
         response = _buildresponse(
-            """<form action="post.php" method="POST">
+            b"""<form action="post.php" method="POST">
             <input type="hidden" name="test" value="val1">
             <input type="hidden" name="test" value="val2">
             <input type="hidden" name="test2" value="xxx">
@@ -225,13 +231,13 @@ class FormRequestTest(RequestTest):
         req = self.request_class.from_response(response,
                 formdata={'one': ['two', 'three'], 'six': 'seven'})
         self.assertEqual(req.method, 'POST')
-        self.assertEqual(req.headers['Content-type'], 'application/x-www-form-urlencoded')
+        self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req)
-        self.assertEqual(set(fs["test"]), set(["val1", "val2"]))
-        self.assertEqual(set(fs["one"]), set(["two", "three"]))
-        self.assertEqual(fs['test2'], ['xxx'])
-        self.assertEqual(fs['six'], ['seven'])
+        self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"})
+        self.assertEqual(set(fs[b"one"]), {b"two", b"three"})
+        self.assertEqual(fs[b'test2'], [b'xxx'])
+        self.assertEqual(fs[b'six'], [b'seven'])
 
     def test_from_response_extra_headers(self):
         response = _buildresponse(
@@ -244,8 +250,8 @@ class FormRequestTest(RequestTest):
                 formdata={'one': ['two', 'three'], 'six': 'seven'},
                 headers={"Accept-Encoding": "gzip,deflate"})
         self.assertEqual(req.method, 'POST')
-        self.assertEqual(req.headers['Content-type'], 'application/x-www-form-urlencoded')
-        self.assertEqual(req.headers['Accept-Encoding'], 'gzip,deflate')
+        self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.headers['Accept-Encoding'], b'gzip,deflate')
 
     def test_from_response_get(self):
         response = _buildresponse(
@@ -274,8 +280,8 @@ class FormRequestTest(RequestTest):
             </form>""")
         req = self.request_class.from_response(response, formdata={'two': '2'})
         fs = _qs(req)
-        self.assertEqual(fs['one'], ['1'])
-        self.assertEqual(fs['two'], ['2'])
+        self.assertEqual(fs[b'one'], [b'1'])
+        self.assertEqual(fs[b'two'], [b'2'])
 
     def test_from_response_override_method(self):
         response = _buildresponse(
@@ -379,7 +385,7 @@ class FormRequestTest(RequestTest):
         req = self.request_class.from_response(response, \
                 clickdata={'name': u'price in \u00a3'})
         fs = _qs(req)
-        self.assertTrue(fs[u'price in \u00a3'.encode('utf-8')])
+        self.assertTrue(fs[to_native_str(u'price in \u00a3')])
 
     def test_from_response_multiple_forms_clickdata(self):
         response = _buildresponse(
@@ -489,9 +495,9 @@ class FormRequestTest(RequestTest):
             </form>""")
         r1 = self.request_class.from_response(response, formdata={'two':'3'})
         self.assertEqual(r1.method, 'POST')
-        self.assertEqual(r1.headers['Content-type'], 'application/x-www-form-urlencoded')
+        self.assertEqual(r1.headers['Content-type'], b'application/x-www-form-urlencoded')
         fs = _qs(r1)
-        self.assertEqual(fs, {'one': ['1'], 'two': ['3']})
+        self.assertEqual(fs, {b'one': [b'1'], b'two': [b'3']})
 
     def test_from_response_formname_exists(self):
         response = _buildresponse(
@@ -506,7 +512,7 @@ class FormRequestTest(RequestTest):
         r1 = self.request_class.from_response(response, formname="form2")
         self.assertEqual(r1.method, 'POST')
         fs = _qs(r1)
-        self.assertEqual(fs, {'four': ['4'], 'three': ['3']})
+        self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']})
 
     def test_from_response_formname_notexist(self):
         response = _buildresponse(
@@ -519,7 +525,7 @@ class FormRequestTest(RequestTest):
         r1 = self.request_class.from_response(response, formname="form3")
         self.assertEqual(r1.method, 'POST')
         fs = _qs(r1)
-        self.assertEqual(fs, {'one': ['1']})
+        self.assertEqual(fs, {b'one': [b'1']})
 
     def test_from_response_formname_errors_formnumber(self):
         response = _buildresponse(
@@ -664,11 +670,11 @@ class FormRequestTest(RequestTest):
             </form>""")
         r1 = self.request_class.from_response(response, formxpath="//form[@action='post.php']")
         fs = _qs(r1)
-        self.assertEqual(fs['one'], ['1'])
+        self.assertEqual(fs[b'one'], [b'1'])
 
         r1 = self.request_class.from_response(response, formxpath="//form/input[@name='four']")
         fs = _qs(r1)
-        self.assertEqual(fs['three'], ['3'])
+        self.assertEqual(fs[b'three'], [b'3'])
 
         self.assertRaises(ValueError, self.request_class.from_response,
                           response, formxpath="//form/input[@name='abc']")
@@ -691,12 +697,12 @@ class XmlRpcRequestTest(RequestTest):
 
     request_class = XmlRpcRequest
     default_method = 'POST'
-    default_headers = {'Content-Type': ['text/xml']}
+    default_headers = {b'Content-Type': [b'text/xml']}
 
     def _test_request(self, **kwargs):
         r = self.request_class('http://scrapytest.org/rpc2', **kwargs)
-        self.assertEqual(r.headers['Content-Type'], 'text/xml')
-        self.assertEqual(r.body, xmlrpclib.dumps(**kwargs))
+        self.assertEqual(r.headers[b'Content-Type'], b'text/xml')
+        self.assertEqual(r.body, to_bytes(xmlrpclib.dumps(**kwargs)))
         self.assertEqual(r.method, 'POST')
         self.assertEqual(r.encoding, kwargs.get('encoding', 'utf-8'))
         self.assertTrue(r.dont_filter, True)
@@ -706,11 +712,14 @@ class XmlRpcRequestTest(RequestTest):
         self._test_request(params=('username', 'password'), methodname='login')
         self._test_request(params=('response', ), methodresponse='login')
         self._test_request(params=(u'pas\xa3',), encoding='utf-8')
-        self._test_request(params=(u'pas\xa3',), encoding='latin')
         self._test_request(params=(None,), allow_none=1)
         self.assertRaises(TypeError, self._test_request)
         self.assertRaises(TypeError, self._test_request, params=(None,))
 
+    @unittest.skipUnless(six.PY2, "TODO")
+    def test_latin1(self):
+        self._test_request(params=(u'pas\xa3',), encoding='latin')
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index b0b5b82c4..b49d46ea8 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,8 +1,12 @@
 import unittest
 
+import six
 from w3lib.encoding import resolve_encoding
-from scrapy.http import Request, Response, TextResponse, HtmlResponse, XmlResponse, Headers
+
+from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
+                         XmlResponse, Headers)
 from scrapy.selector import Selector
+from scrapy.utils.python import to_native_str
 
 
 class BaseResponseTest(unittest.TestCase):
@@ -14,10 +18,10 @@ class BaseResponseTest(unittest.TestCase):
         self.assertRaises(Exception, self.response_class)
         self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class))
         # body can be str or None
-        self.assertTrue(isinstance(self.response_class('http://example.com/', body=''), self.response_class))
-        self.assertTrue(isinstance(self.response_class('http://example.com/', body='body'), self.response_class))
+        self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class))
+        self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class))
         # test presence of all optional parameters
-        self.assertTrue(isinstance(self.response_class('http://example.com/', headers={}, status=200, body=''), self.response_class))
+        self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class))
 
         r = self.response_class("http://www.example.com")
         assert isinstance(r.url, str)
@@ -27,12 +31,12 @@ class BaseResponseTest(unittest.TestCase):
         assert isinstance(r.headers, Headers)
         self.assertEqual(r.headers, {})
 
-        headers = {"caca": "coco"}
-        body = "a body"
+        headers = {"foo": "bar"}
+        body = b"a body"
         r = self.response_class("http://www.example.com", headers=headers, body=body)
 
         assert r.headers is not headers
-        self.assertEqual(r.headers["caca"], "coco")
+        self.assertEqual(r.headers[b"foo"], b"bar")
 
         r = self.response_class("http://www.example.com", status=301)
         self.assertEqual(r.status, 301)
@@ -43,7 +47,7 @@ class BaseResponseTest(unittest.TestCase):
     def test_copy(self):
         """Test Response copy"""
 
-        r1 = self.response_class("http://www.example.com", body="Some body")
+        r1 = self.response_class("http://www.example.com", body=b"Some body")
         r1.flags.append('cached')
         r2 = r1.copy()
 
@@ -61,7 +65,7 @@ class BaseResponseTest(unittest.TestCase):
     def test_copy_meta(self):
         req = Request("http://www.example.com")
         req.meta['foo'] = 'bar'
-        r1 = self.response_class("http://www.example.com", body="Some body", request=req)
+        r1 = self.response_class("http://www.example.com", body=b"Some body", request=req)
         assert r1.meta is req.meta
 
     def test_copy_inherited_classes(self):
@@ -79,30 +83,30 @@ class BaseResponseTest(unittest.TestCase):
         """Test Response.replace() method"""
         hdrs = Headers({"key": "value"})
         r1 = self.response_class("http://www.example.com")
-        r2 = r1.replace(status=301, body="New body", headers=hdrs)
-        assert r1.body == ''
+        r2 = r1.replace(status=301, body=b"New body", headers=hdrs)
+        assert r1.body == b''
         self.assertEqual(r1.url, r2.url)
         self.assertEqual((r1.status, r2.status), (200, 301))
-        self.assertEqual((r1.body, r2.body), ('', "New body"))
+        self.assertEqual((r1.body, r2.body), (b'', b"New body"))
         self.assertEqual((r1.headers, r2.headers), ({}, hdrs))
 
         # Empty attributes (which may fail if not compared properly)
         r3 = self.response_class("http://www.example.com", flags=['cached'])
-        r4 = r3.replace(body='', flags=[])
-        self.assertEqual(r4.body, '')
+        r4 = r3.replace(body=b'', flags=[])
+        self.assertEqual(r4.body, b'')
         self.assertEqual(r4.flags, [])
 
     def _assert_response_values(self, response, encoding, body):
-        if isinstance(body, unicode):
+        if isinstance(body, six.text_type):
             body_unicode = body
-            body_str = body.encode(encoding)
+            body_bytes = body.encode(encoding)
         else:
             body_unicode = body.decode(encoding)
-            body_str = body
+            body_bytes = body
 
-        assert isinstance(response.body, str)
+        assert isinstance(response.body, bytes)
         self._assert_response_encoding(response, encoding)
-        self.assertEqual(response.body, body_str)
+        self.assertEqual(response.body, body_bytes)
         self.assertEqual(response.body_as_unicode(), body_unicode)
 
     def _assert_response_encoding(self, response, encoding):
@@ -120,12 +124,6 @@ class BaseResponseTest(unittest.TestCase):
         self.assertEqual(joined, absolute)
 
 
-class ResponseText(BaseResponseTest):
-
-    def test_no_unicode_url(self):
-        self.assertRaises(TypeError, self.response_class, u'http://www.example.com')
-
-
 class TextResponseTest(BaseResponseTest):
 
     response_class = TextResponse
@@ -152,11 +150,11 @@ class TextResponseTest(BaseResponseTest):
         assert isinstance(resp.url, str)
 
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')
-        self.assertEqual(resp.url, 'http://www.example.com/price/\xc2\xa3')
+        self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
         resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]})
-        self.assertEqual(resp.url, 'http://www.example.com/price/\xc2\xa3')
+        self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))
         resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
 
@@ -168,17 +166,17 @@ class TextResponseTest(BaseResponseTest):
         r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
 
         # check body_as_unicode
-        self.assertTrue(isinstance(r1.body_as_unicode(), unicode))
+        self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type))
         self.assertEqual(r1.body_as_unicode(), unicode_string)
 
     def test_encoding(self):
-        r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body="\xc2\xa3")
+        r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xc2\xa3")
         r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
-        r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body="\xa3")
-        r4 = self.response_class("http://www.example.com", body="\xa2\xa3")
-        r5 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=None"]}, body="\xc2\xa3")
-        r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gb2312"]}, body="\xa8D")
-        r7 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gbk"]}, body="\xa8D")
+        r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=b"\xa3")
+        r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3")
+        r5 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=None"]}, body=b"\xc2\xa3")
+        r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gb2312"]}, body=b"\xa8D")
+        r7 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gbk"]}, body=b"\xa8D")
 
         self.assertEqual(r1._headers_encoding(), "utf-8")
         self.assertEqual(r2._headers_encoding(), None)
@@ -203,21 +201,21 @@ class TextResponseTest(BaseResponseTest):
         """Check that unknown declared encodings are ignored"""
         r = self.response_class("http://www.example.com",
                                 headers={"Content-type": ["text/html; charset=UKNOWN"]},
-                                body="\xc2\xa3")
+                                body=b"\xc2\xa3")
         self.assertEqual(r._declared_encoding(), None)
         self._assert_response_values(r, 'utf-8', u"\xa3")
 
     def test_utf16(self):
         """Test utf-16 because UnicodeDammit is known to have problems with"""
         r = self.response_class("http://www.example.com",
-                                body='\xff\xfeh\x00i\x00',
+                                body=b'\xff\xfeh\x00i\x00',
                                 encoding='utf-16')
         self._assert_response_values(r, 'utf-16', u"hi")
 
     def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self):
         r6 = self.response_class("http://www.example.com",
                                  headers={"Content-type": ["text/html; charset=utf-8"]},
-                                 body="\xef\xbb\xbfWORD\xe3\xab")
+                                 body=b"\xef\xbb\xbfWORD\xe3\xab")
         self.assertEqual(r6.encoding, 'utf-8')
         self.assertEqual(r6.body_as_unicode(), u'WORD\ufffd\ufffd')
 
@@ -227,7 +225,7 @@ class TextResponseTest(BaseResponseTest):
         # response.body_as_unicode() in indistint order doesn't affect final
         # values for encoding and decoded body.
         url = 'http://example.com'
-        body = "\xef\xbb\xbfWORD"
+        body = b"\xef\xbb\xbfWORD"
         headers = {"Content-type": ["text/html; charset=utf-8"]}
 
         # Test response without content-type and BOM encoding
@@ -250,7 +248,7 @@ class TextResponseTest(BaseResponseTest):
 
     def test_replace_wrong_encoding(self):
         """Test invalid chars are replaced properly"""
-        r = self.response_class("http://www.example.com", encoding='utf-8', body='PREFIX\xe3\xabSUFFIX')
+        r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX')
         # XXX: Policy for replacing invalid chars may suffer minor variations
         # but it should always contain the unicode replacement char (u'\ufffd')
         assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode())
@@ -259,7 +257,7 @@ class TextResponseTest(BaseResponseTest):
 
         # Do not destroy html tags due to encoding bugs
         r = self.response_class("http://example.com", encoding='utf-8', \
-                body='\xf0<span>value</span>')
+                body=b'\xf0<span>value</span>')
         assert u'<span>value</span>' in r.body_as_unicode(), repr(r.body_as_unicode())
 
         # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
@@ -267,7 +265,7 @@ class TextResponseTest(BaseResponseTest):
         #assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode())
 
     def test_selector(self):
-        body = "<html><head><title>Some page</title><body></body></html>"
+        body = b"<html><head><title>Some page</title><body></body></html>"
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertIsInstance(response.selector, Selector)
@@ -289,7 +287,7 @@ class TextResponseTest(BaseResponseTest):
         )
 
     def test_selector_shortcuts(self):
-        body = "<html><head><title>Some page</title><body></body></html>"
+        body = b"<html><head><title>Some page</title><body></body></html>"
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
@@ -303,17 +301,17 @@ class TextResponseTest(BaseResponseTest):
 
     def test_urljoin_with_base_url(self):
         """Test urljoin shortcut which also evaluates base-url through get_base_url()."""
-        body = '<html><body><base href="https://example.net"></body></html>'
+        body = b'<html><body><base href="https://example.net"></body></html>'
         joined = self.response_class('http://www.example.com', body=body).urljoin('/test')
         absolute = 'https://example.net/test'
         self.assertEqual(joined, absolute)
 
-        body = '<html><body><base href="/elsewhere"></body></html>'
+        body = b'<html><body><base href="/elsewhere"></body></html>'
         joined = self.response_class('http://www.example.com', body=body).urljoin('test')
         absolute = 'http://www.example.com/test'
         self.assertEqual(joined, absolute)
 
-        body = '<html><body><base href="/elsewhere/"></body></html>'
+        body = b'<html><body><base href="/elsewhere/"></body></html>'
         joined = self.response_class('http://www.example.com', body=body).urljoin('test')
         absolute = 'http://www.example.com/elsewhere/test'
         self.assertEqual(joined, absolute)
@@ -325,13 +323,13 @@ class HtmlResponseTest(TextResponseTest):
 
     def test_html_encoding(self):
 
-        body = """<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+        body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
         </head><body>Price: \xa3100</body></html>'
         """
         r1 = self.response_class("http://www.example.com", body=body)
         self._assert_response_values(r1, 'iso-8859-1', body)
 
-        body = """<?xml version="1.0" encoding="iso-8859-1"?>
+        body = b"""<?xml version="1.0" encoding="iso-8859-1"?>
         <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
         Price: \xa3100
         """
@@ -339,19 +337,19 @@ class HtmlResponseTest(TextResponseTest):
         self._assert_response_values(r2, 'iso-8859-1', body)
 
         # for conflicting declarations headers must take precedence
-        body = """<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+        body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=utf-8">
         </head><body>Price: \xa3100</body></html>'
         """
         r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=body)
         self._assert_response_values(r3, 'iso-8859-1', body)
 
         # make sure replace() preserves the encoding of the original response
-        body = "New body \xa3"
+        body = b"New body \xa3"
         r4 = r3.replace(body=body)
         self._assert_response_values(r4, 'iso-8859-1', body)
 
     def test_html5_meta_charset(self):
-        body = """<html><head><meta charset="gb2312" /><title>Some page</title><body>bla bla</body>"""
+        body = b"""<html><head><meta charset="gb2312" /><title>Some page</title><body>bla bla</body>"""
         r1 = self.response_class("http://www.example.com", body=body)
         self._assert_response_values(r1, 'gb2312', body)
 
@@ -361,26 +359,25 @@ class XmlResponseTest(TextResponseTest):
     response_class = XmlResponse
 
     def test_xml_encoding(self):
-
-        body = "<xml></xml>"
+        body = b"<xml></xml>"
         r1 = self.response_class("http://www.example.com", body=body)
         self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body)
 
-        body = """<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
+        body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
         r2 = self.response_class("http://www.example.com", body=body)
         self._assert_response_values(r2, 'iso-8859-1', body)
 
         # make sure replace() preserves the explicit encoding passed in the constructor
-        body = """<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
+        body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
         r3 = self.response_class("http://www.example.com", body=body, encoding='utf-8')
-        body2 = "New body"
+        body2 = b"New body"
         r4 = r3.replace(body=body2)
         self._assert_response_values(r4, 'utf-8', body2)
 
     def test_replace_encoding(self):
         # make sure replace() keeps the previous encoding unless overridden explicitly
-        body = """<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
-        body2 = """<?xml version="1.0" encoding="utf-8"?><xml></xml>"""
+        body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
+        body2 = b"""<?xml version="1.0" encoding="utf-8"?><xml></xml>"""
         r5 = self.response_class("http://www.example.com", body=body)
         r6 = r5.replace(body=body2)
         r7 = r5.replace(body=body2, encoding='utf-8')
@@ -389,7 +386,7 @@ class XmlResponseTest(TextResponseTest):
         self._assert_response_values(r7, 'utf-8', body2)
 
     def test_selector(self):
-        body = '<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'
+        body = b'<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertIsInstance(response.selector, Selector)
@@ -403,15 +400,10 @@ class XmlResponseTest(TextResponseTest):
         )
 
     def test_selector_shortcuts(self):
-        body = '<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'
+        body = b'<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
             response.xpath("//elem/text()").extract(),
             response.selector.xpath("//elem/text()").extract(),
         )
-
-
-
-if __name__ == "__main__":
-    unittest.main()

From e438df7028bda5d2ff6fa163524424fa50eab0d7 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 25 Jul 2015 13:10:13 +0200
Subject: [PATCH 0418/2974] PY3 enable Request and Response tests

---
 tests/py3-ignores.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index aa091835c..9aa308243 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -28,8 +28,6 @@ tests/test_downloadermiddleware_useragent.py
 tests/test_dupefilters.py
 tests/test_engine.py
 tests/test_http_cookies.py
-tests/test_http_request.py
-tests/test_http_response.py
 tests/test_logformatter.py
 tests/test_mail.py
 tests/test_middleware.py

From ea56067e40593efceb68bbc605d32bb57d42f4ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Sat, 25 Jul 2015 16:39:02 +0200
Subject: [PATCH 0419/2974] ignore moved files from py3 tests

---
 tests/py3-ignores.txt | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 9aa308243..06e3a9839 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -46,7 +46,6 @@ tests/test_spiderloader/test_spiders/spider0.py
 tests/test_spiderloader/test_spiders/spider1.py
 tests/test_spiderloader/test_spiders/spider2.py
 tests/test_spiderloader/test_spiders/spider3.py
-tests/test_spiderloader/test_spiders/spider4.py
 tests/test_spidermiddleware_depth.py
 tests/test_spidermiddleware_httperror.py
 tests/test_spidermiddleware_offsite.py
@@ -55,7 +54,6 @@ tests/test_spider.py
 tests/test_stats.py
 tests/test_utils_defer.py
 tests/test_utils_iterators.py
-tests/test_utils_jsonrpc.py
 tests/test_utils_log.py
 tests/test_utils_reqser.py
 tests/test_utils_request.py
@@ -91,6 +89,5 @@ scrapy/downloadermiddlewares/httpproxy.py
 scrapy/downloadermiddlewares/cookies.py
 scrapy/extensions/statsmailer.py
 scrapy/extensions/memusage.py
-scrapy/commands/deploy.py
 scrapy/commands/bench.py
 scrapy/mail.py

From f750ee4c006859391957d45d14f620d1c407576e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Sat, 25 Jul 2015 16:50:54 +0200
Subject: [PATCH 0420/2974] Remove ignored passing tests from py3 branch

---
 tests/py3-ignores.txt | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 06e3a9839..1f80f97f8 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -39,13 +39,6 @@ tests/test_responsetypes.py
 tests/test_selector_csstranslator.py
 tests/test_selector_lxmldocument.py
 tests/test_selector.py
-tests/test_settings/__init__.py
-tests/test_spiderloader/__init__.py
-tests/test_spiderloader/test_spiders/__init__.py
-tests/test_spiderloader/test_spiders/spider0.py
-tests/test_spiderloader/test_spiders/spider1.py
-tests/test_spiderloader/test_spiders/spider2.py
-tests/test_spiderloader/test_spiders/spider3.py
 tests/test_spidermiddleware_depth.py
 tests/test_spidermiddleware_httperror.py
 tests/test_spidermiddleware_offsite.py
@@ -58,10 +51,8 @@ tests/test_utils_log.py
 tests/test_utils_reqser.py
 tests/test_utils_request.py
 tests/test_utils_response.py
-tests/test_utils_serialize.py
 tests/test_utils_signal.py
 tests/test_utils_template.py
-tests/test_utils_url.py
 tests/test_webclient.py
 
 scrapy/xlib/tx/iweb.py

From 7874bb9f13cc4ec797925adc99bf58742071ed6c Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 25 Jul 2015 17:14:56 +0200
Subject: [PATCH 0421/2974] PY3 port utils.request

---
 scrapy/utils/request.py     | 20 ++++++++++++--------
 tests/py3-ignores.txt       |  1 -
 tests/test_utils_request.py | 22 +++++++++++-----------
 3 files changed, 23 insertions(+), 20 deletions(-)

diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 0e6e6675d..4871681a9 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -10,6 +10,7 @@ from six.moves.urllib.parse import urlunparse
 
 from twisted.internet.defer import Deferred
 from w3lib.http import basic_auth_header
+from scrapy.utils.python import to_bytes, to_native_str
 
 from scrapy.utils.url import canonicalize_url
 from scrapy.utils.httpobj import urlparse_cached
@@ -44,13 +45,14 @@ def request_fingerprint(request, include_headers=None):
 
     """
     if include_headers:
-        include_headers = tuple([h.lower() for h in sorted(include_headers)])
+        include_headers = tuple([to_bytes(h.lower())
+                                 for h in sorted(include_headers)])
     cache = _fingerprint_cache.setdefault(request, {})
     if include_headers not in cache:
         fp = hashlib.sha1()
-        fp.update(request.method)
-        fp.update(canonicalize_url(request.url))
-        fp.update(request.body or '')
+        fp.update(to_bytes(request.method))
+        fp.update(to_bytes(canonicalize_url(request.url)))
+        fp.update(request.body or b'')
         if include_headers:
             for hdr in include_headers:
                 if hdr in request.headers:
@@ -60,12 +62,14 @@ def request_fingerprint(request, include_headers=None):
         cache[include_headers] = fp.hexdigest()
     return cache[include_headers]
 
+
 def request_authenticate(request, username, password):
     """Autenticate the given request (in place) using the HTTP basic access
     authentication mechanism (RFC 2617) and the given username and password
     """
     request.headers['Authorization'] = basic_auth_header(username, password)
 
+
 def request_httprepr(request):
     """Return the raw HTTP representation (as string) of the given request.
     This is provided only for reference since it's not the actual stream of
@@ -74,11 +78,11 @@ def request_httprepr(request):
     """
     parsed = urlparse_cached(request)
     path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
-    s  = "%s %s HTTP/1.1\r\n" % (request.method, path)
-    s += "Host: %s\r\n" % parsed.hostname
+    s = to_bytes(request.method) + b" " + to_bytes(path) + b" HTTP/1.1\r\n"
+    s += b"Host: " + to_bytes(parsed.hostname) + b"\r\n"
     if request.headers:
-        s += request.headers.to_string() + "\r\n"
-    s += "\r\n"
+        s += request.headers.to_string() + b"\r\n"
+    s += b"\r\n"
     s += request.body
     return s
 
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 1f80f97f8..78ac2df7d 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -49,7 +49,6 @@ tests/test_utils_defer.py
 tests/test_utils_iterators.py
 tests/test_utils_log.py
 tests/test_utils_reqser.py
-tests/test_utils_request.py
 tests/test_utils_response.py
 tests/test_utils_signal.py
 tests/test_utils_template.py
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index c4a54c0ff..d058bba96 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -21,15 +21,15 @@ class UtilsRequestTest(unittest.TestCase):
 
         r1 = Request("http://www.example.com/members/offers.html")
         r2 = Request("http://www.example.com/members/offers.html")
-        r2.headers['SESSIONID'] = "somehash"
+        r2.headers['SESSIONID'] = b"somehash"
         self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
 
         r1 = Request("http://www.example.com/")
         r2 = Request("http://www.example.com/")
-        r2.headers['Accept-Language'] = 'en'
+        r2.headers['Accept-Language'] = b'en'
         r3 = Request("http://www.example.com/")
-        r3.headers['Accept-Language'] = 'en'
-        r3.headers['SESSIONID'] = "somehash"
+        r3.headers['Accept-Language'] = b'en'
+        r3.headers['SESSIONID'] = b"somehash"
 
         self.assertEqual(request_fingerprint(r1), request_fingerprint(r2), request_fingerprint(r3))
 
@@ -44,7 +44,7 @@ class UtilsRequestTest(unittest.TestCase):
 
         r1 = Request("http://www.example.com")
         r2 = Request("http://www.example.com", method='POST')
-        r3 = Request("http://www.example.com", method='POST', body='request body')
+        r3 = Request("http://www.example.com", method='POST', body=b'request body')
 
         self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
         self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r3))
@@ -52,24 +52,24 @@ class UtilsRequestTest(unittest.TestCase):
         # cached fingerprint must be cleared on request copy
         r1 = Request("http://www.example.com")
         fp1 = request_fingerprint(r1)
-        r2 = r1.replace(url = "http://www.example.com/other")
+        r2 = r1.replace(url="http://www.example.com/other")
         fp2 = request_fingerprint(r2)
         self.assertNotEqual(fp1, fp2)
 
     def test_request_authenticate(self):
         r = Request("http://www.example.com")
         request_authenticate(r, 'someuser', 'somepass')
-        self.assertEqual(r.headers['Authorization'], 'Basic c29tZXVzZXI6c29tZXBhc3M=')
+        self.assertEqual(r.headers['Authorization'], b'Basic c29tZXVzZXI6c29tZXBhc3M=')
 
     def test_request_httprepr(self):
         r1 = Request("http://www.example.com")
-        self.assertEqual(request_httprepr(r1), 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
+        self.assertEqual(request_httprepr(r1), b'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
 
         r1 = Request("http://www.example.com/some/page.html?arg=1")
-        self.assertEqual(request_httprepr(r1), 'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
+        self.assertEqual(request_httprepr(r1), b'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
 
-        r1 = Request("http://www.example.com", method='POST', headers={"Content-type": "text/html"}, body="Some body")
-        self.assertEqual(request_httprepr(r1), 'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body')
+        r1 = Request("http://www.example.com", method='POST', headers={"Content-type": b"text/html"}, body=b"Some body")
+        self.assertEqual(request_httprepr(r1), b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body')
 
 if __name__ == "__main__":
     unittest.main()

From 82304c4faeaf7820058f92d74c86411d5f193d5e Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 25 Jul 2015 17:48:34 +0200
Subject: [PATCH 0422/2974] PY3: fixed scrapy.utils.defer

---
 scrapy/utils/defer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 97c41af5e..7a3f20476 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -61,7 +61,7 @@ def parallel(iterable, count, callable, *args, **named):
     """
     coop = task.Cooperator()
     work = (callable(elem, *args, **named) for elem in iterable)
-    return defer.DeferredList([coop.coiterate(work) for i in xrange(count)])
+    return defer.DeferredList([coop.coiterate(work) for i in range(count)])
 
 def process_chain(callbacks, input, *a, **kw):
     """Return a Deferred built by chaining the given callbacks"""

From 776064aee19b545f80c37c45ba64a6332199f583 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Sat, 25 Jul 2015 17:38:07 +0200
Subject: [PATCH 0423/2974] PY3 port signals

---
 requirements.txt                      |   1 +
 scrapy/signalmanager.py               |   2 +-
 scrapy/utils/signal.py                |  12 +-
 scrapy/xlib/pydispatch/__init__.py    |   6 -
 scrapy/xlib/pydispatch/dispatcher.py  | 511 --------------------------
 scrapy/xlib/pydispatch/errors.py      |  15 -
 scrapy/xlib/pydispatch/license.txt    |  34 --
 scrapy/xlib/pydispatch/robust.py      |  54 ---
 scrapy/xlib/pydispatch/robustapply.py |  58 ---
 scrapy/xlib/pydispatch/saferef.py     | 180 ---------
 setup.py                              |   1 +
 tests/py3-ignores.txt                 |   1 -
 tests/test_engine.py                  |   2 +-
 tests/test_utils_signal.py            |   2 +-
 14 files changed, 14 insertions(+), 865 deletions(-)
 delete mode 100644 scrapy/xlib/pydispatch/__init__.py
 delete mode 100644 scrapy/xlib/pydispatch/dispatcher.py
 delete mode 100644 scrapy/xlib/pydispatch/errors.py
 delete mode 100644 scrapy/xlib/pydispatch/license.txt
 delete mode 100644 scrapy/xlib/pydispatch/robust.py
 delete mode 100644 scrapy/xlib/pydispatch/robustapply.py
 delete mode 100644 scrapy/xlib/pydispatch/saferef.py

diff --git a/requirements.txt b/requirements.txt
index 005b8f4f5..03e5329c0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,3 +5,4 @@ cssselect>=0.9
 w3lib>=1.8.0
 queuelib
 six>=1.5.2
+PyDispatcher>=2.0.5
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index bcc0df2c6..fd79905e9 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -1,5 +1,5 @@
 from __future__ import absolute_import
-from scrapy.xlib.pydispatch import dispatcher
+from pydispatch import dispatcher
 from scrapy.utils import signal as _signal
 
 
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index d9a59e161..de00bac49 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -5,19 +5,23 @@ import logging
 from twisted.internet.defer import maybeDeferred, DeferredList, Deferred
 from twisted.python.failure import Failure
 
-from scrapy.xlib.pydispatch.dispatcher import Any, Anonymous, liveReceivers, \
+from pydispatch.dispatcher import Any, Anonymous, liveReceivers, \
     getAllReceivers, disconnect
-from scrapy.xlib.pydispatch.robustapply import robustApply
+from pydispatch.robustapply import robustApply
 from scrapy.utils.log import failure_to_exc_info
 
 logger = logging.getLogger(__name__)
 
 
+class _IgnoredException(Exception):
+    pass
+
+
 def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
     """Like pydispatcher.robust.sendRobust but it also logs errors and returns
     Failures instead of exceptions.
     """
-    dont_log = named.pop('dont_log', None)
+    dont_log = named.pop('dont_log', _IgnoredException)
     spider = named.get('spider', None)
     responses = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
@@ -39,6 +43,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
         responses.append((receiver, result))
     return responses
 
+
 def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
     """Like send_catch_log but supports returning deferreds on signal handlers.
     Returns a deferred that gets fired once all signal handlers deferreds were
@@ -65,6 +70,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
     d.addCallback(lambda out: [x[1] for x in out])
     return d
 
+
 def disconnect_all(signal=Any, sender=Any):
     """Disconnect all signal handlers. Useful for cleaning up after running
     tests
diff --git a/scrapy/xlib/pydispatch/__init__.py b/scrapy/xlib/pydispatch/__init__.py
deleted file mode 100644
index 78511a0d8..000000000
--- a/scrapy/xlib/pydispatch/__init__.py
+++ /dev/null
@@ -1,6 +0,0 @@
-"""Multi-consumer multi-producer dispatching mechanism
-"""
-__version__ = "2.0.0"
-__author__ = "Patrick K. O'Brien"
-__license__ = "BSD-style, see license.txt for details"
-
diff --git a/scrapy/xlib/pydispatch/dispatcher.py b/scrapy/xlib/pydispatch/dispatcher.py
deleted file mode 100644
index 7d588dd20..000000000
--- a/scrapy/xlib/pydispatch/dispatcher.py
+++ /dev/null
@@ -1,511 +0,0 @@
-"""Multiple-producer-multiple-consumer signal-dispatching
-
-dispatcher is the core of the PyDispatcher system,
-providing the primary API and the core logic for the
-system.
-
-Module attributes of note:
-
-    Any -- Singleton used to signal either "Any Sender" or
-        "Any Signal".  See documentation of the _Any class.
-    Anonymous -- Singleton used to signal "Anonymous Sender"
-        See documentation of the _Anonymous class.
-
-Internal attributes:
-    WEAKREF_TYPES -- tuple of types/classes which represent
-        weak references to receivers, and thus must be de-
-        referenced on retrieval to retrieve the callable
-        object
-    connections -- { senderkey (id) : { signal : [receivers...]}}
-    senders -- { senderkey (id) : weakref(sender) }
-        used for cleaning up sender references on sender
-        deletion
-    sendersBack -- { receiverkey (id) : [senderkey (id)...] }
-        used for cleaning up receiver references on receiver
-        deletion, (considerably speeds up the cleanup process
-        vs. the original code.)
-"""
-from __future__ import generators
-import types, weakref, six
-from scrapy.xlib.pydispatch import saferef, robustapply, errors
-
-__author__ = "Patrick K. O'Brien <pobrien@orbtech.com>"
-__cvsid__ = "$Id: dispatcher.py,v 1.1.1.1 2006/07/07 15:59:38 mcfletch Exp $"
-__version__ = "$Revision: 1.1.1.1 $"[11:-2]
-
-
-class _Parameter:
-    """Used to represent default parameter values."""
-
-    def __repr__(self):
-        return self.__class__.__name__
-
-
-class _Any(_Parameter):
-    """Singleton used to signal either "Any Sender" or "Any Signal"
-
-    The Any object can be used with connect, disconnect,
-    send, or sendExact to signal that the parameter given
-    Any should react to all senders/signals, not just
-    a particular sender/signal.
-    """
-
-
-Any = _Any()
-
-
-class _Anonymous(_Parameter):
-    """Singleton used to signal "Anonymous Sender"
-
-    The Anonymous object is used to signal that the sender
-    of a message is not specified (as distinct from being
-    "any sender").  Registering callbacks for Anonymous
-    will only receive messages sent without senders.  Sending
-    with anonymous will only send messages to those receivers
-    registered for Any or Anonymous.
-
-    Note:
-        The default sender for connect is Any, while the
-        default sender for send is Anonymous.  This has
-        the effect that if you do not specify any senders
-        in either function then all messages are routed
-        as though there was a single sender (Anonymous)
-        being used everywhere.
-    """
-
-
-Anonymous = _Anonymous()
-
-WEAKREF_TYPES = (weakref.ReferenceType, saferef.BoundMethodWeakref)
-
-connections = {}
-senders = {}
-sendersBack = {}
-
-
-def connect(receiver, signal=Any, sender=Any, weak=True):
-    """Connect receiver to sender for signal
-
-    receiver -- a callable Python object which is to receive
-        messages/signals/events.  Receivers must be hashable
-        objects.
-
-        if weak is True, then receiver must be weak-referencable
-        (more precisely saferef.safeRef() must be able to create
-        a reference to the receiver).
-
-        Receivers are fairly flexible in their specification,
-        as the machinery in the robustApply module takes care
-        of most of the details regarding figuring out appropriate
-        subsets of the sent arguments to apply to a given
-        receiver.
-
-        Note:
-            if receiver is itself a weak reference (a callable),
-            it will be de-referenced by the system's machinery,
-            so *generally* weak references are not suitable as
-            receivers, though some use might be found for the
-            facility whereby a higher-level library passes in
-            pre-weakrefed receiver references.
-
-    signal -- the signal to which the receiver should respond
-
-        if Any, receiver will receive any signal from the
-        indicated sender (which might also be Any, but is not
-        necessarily Any).
-
-        Otherwise must be a hashable Python object other than
-        None (DispatcherError raised on None).
-
-    sender -- the sender to which the receiver should respond
-
-        if Any, receiver will receive the indicated signals
-        from any sender.
-
-        if Anonymous, receiver will only receive indicated
-        signals from send/sendExact which do not specify a
-        sender, or specify Anonymous explicitly as the sender.
-
-        Otherwise can be any python object.
-
-    weak -- whether to use weak references to the receiver
-        By default, the module will attempt to use weak
-        references to the receiver objects.  If this parameter
-        is false, then strong references will be used.
-
-    returns None, may raise DispatcherTypeError
-    """
-    if signal is None:
-        raise errors.DispatcherTypeError(
-            'Signal cannot be None (receiver=%r sender=%r)' % (
-                receiver, sender)
-        )
-    if weak:
-        receiver = saferef.safeRef(receiver, onDelete=_removeReceiver)
-    senderkey = id(sender)
-    if senderkey in connections:
-        signals = connections[senderkey]
-    else:
-        connections[senderkey] = signals = {}
-    # Keep track of senders for cleanup.
-    # Is Anonymous something we want to clean up?
-    if sender not in (None, Anonymous, Any):
-        def remove(object, senderkey=senderkey):
-            _removeSender(senderkey=senderkey)
-
-        # Skip objects that can not be weakly referenced, which means
-        # they won't be automatically cleaned up, but that's too bad.
-        try:
-            weakSender = weakref.ref(sender, remove)
-            senders[senderkey] = weakSender
-        except:
-            pass
-
-    receiverID = id(receiver)
-    # get current set, remove any current references to
-    # this receiver in the set, including back-references
-    if signal in signals:
-        receivers = signals[signal]
-        _removeOldBackRefs(senderkey, signal, receiver, receivers)
-    else:
-        receivers = signals[signal] = []
-    try:
-        current = sendersBack.get(receiverID)
-        if current is None:
-            sendersBack[receiverID] = current = []
-        if senderkey not in current:
-            current.append(senderkey)
-    except:
-        pass
-
-    receivers.append(receiver)
-
-
-def disconnect(receiver, signal=Any, sender=Any, weak=True):
-    """Disconnect receiver from sender for signal
-
-    receiver -- the registered receiver to disconnect
-    signal -- the registered signal to disconnect
-    sender -- the registered sender to disconnect
-    weak -- the weakref state to disconnect
-
-    disconnect reverses the process of connect,
-    the semantics for the individual elements are
-    logically equivalent to a tuple of
-    (receiver, signal, sender, weak) used as a key
-    to be deleted from the internal routing tables.
-    (The actual process is slightly more complex
-    but the semantics are basically the same).
-
-    Note:
-        Using disconnect is not required to cleanup
-        routing when an object is deleted, the framework
-        will remove routes for deleted objects
-        automatically.  It's only necessary to disconnect
-        if you want to stop routing to a live object.
-
-    returns None, may raise DispatcherTypeError or
-        DispatcherKeyError
-    """
-    if signal is None:
-        raise errors.DispatcherTypeError(
-            'Signal cannot be None (receiver=%r sender=%r)' % (
-                receiver, sender)
-        )
-    if weak: receiver = saferef.safeRef(receiver)
-    senderkey = id(sender)
-    try:
-        signals = connections[senderkey]
-        receivers = signals[signal]
-    except KeyError:
-        raise errors.DispatcherKeyError(
-            """No receivers found for signal %r from sender %r""" % (
-                signal,
-                sender
-            )
-        )
-    try:
-        # also removes from receivers
-        _removeOldBackRefs(senderkey, signal, receiver, receivers)
-    except ValueError:
-        raise errors.DispatcherKeyError(
-            """No connection to receiver %s for signal %s from sender %s""" % (
-                receiver,
-                signal,
-                sender
-            )
-        )
-    _cleanupConnections(senderkey, signal)
-
-
-def getReceivers(sender=Any, signal=Any):
-    """Get list of receivers from global tables
-
-    This utility function allows you to retrieve the
-    raw list of receivers from the connections table
-    for the given sender and signal pair.
-
-    Note:
-        there is no guarantee that this is the actual list
-        stored in the connections table, so the value
-        should be treated as a simple iterable/truth value
-        rather than, for instance a list to which you
-        might append new records.
-
-    Normally you would use liveReceivers( getReceivers( ...))
-    to retrieve the actual receiver objects as an iterable
-    object.
-    """
-    try:
-        return connections[id(sender)][signal]
-    except KeyError:
-        return []
-
-
-def liveReceivers(receivers):
-    """Filter sequence of receivers to get resolved, live receivers
-
-    This is a generator which will iterate over
-    the passed sequence, checking for weak references
-    and resolving them, then returning all live
-    receivers.
-    """
-    for receiver in receivers:
-        if isinstance(receiver, WEAKREF_TYPES):
-            # Dereference the weak reference.
-            receiver = receiver()
-            if receiver is not None:
-                yield receiver
-        else:
-            yield receiver
-
-
-def getAllReceivers(sender=Any, signal=Any):
-    """Get list of all receivers from global tables
-
-    This gets all receivers which should receive
-    the given signal from sender, each receiver should
-    be produced only once by the resulting generator
-    """
-    receivers = {}
-    for set in (
-            # Get receivers that receive *this* signal from *this* sender.
-            getReceivers(sender, signal),
-            # Add receivers that receive *any* signal from *this* sender.
-            getReceivers(sender, Any),
-            # Add receivers that receive *this* signal from *any* sender.
-            getReceivers(Any, signal),
-            # Add receivers that receive *any* signal from *any* sender.
-            getReceivers(Any, Any),
-    ):
-        for receiver in set:
-            if receiver:  # filter out dead instance-method weakrefs
-                try:
-                    if receiver not in receivers:
-                        receivers[receiver] = 1
-                        yield receiver
-                except TypeError:
-                    # dead weakrefs raise TypeError on hash...
-                    pass
-
-
-def send(signal=Any, sender=Anonymous, *arguments, **named):
-    """Send signal from sender to all connected receivers.
-
-    signal -- (hashable) signal value, see connect for details
-
-    sender -- the sender of the signal
-
-        if Any, only receivers registered for Any will receive
-        the message.
-
-        if Anonymous, only receivers registered to receive
-        messages from Anonymous or Any will receive the message
-
-        Otherwise can be any python object (normally one
-        registered with a connect if you actually want
-        something to occur).
-
-    arguments -- positional arguments which will be passed to
-        *all* receivers. Note that this may raise TypeErrors
-        if the receivers do not allow the particular arguments.
-        Note also that arguments are applied before named
-        arguments, so they should be used with care.
-
-    named -- named arguments which will be filtered according
-        to the parameters of the receivers to only provide those
-        acceptable to the receiver.
-
-    Return a list of tuple pairs [(receiver, response), ... ]
-
-    if any receiver raises an error, the error propagates back
-    through send, terminating the dispatch loop, so it is quite
-    possible to not have all receivers called if a raises an
-    error.
-    """
-    # Call each receiver with whatever arguments it can accept.
-    # Return a list of tuple pairs [(receiver, response), ... ].
-    responses = []
-    for receiver in liveReceivers(getAllReceivers(sender, signal)):
-        response = robustapply.robustApply(
-            receiver,
-            signal=signal,
-            sender=sender,
-            *arguments,
-            **named
-        )
-        responses.append((receiver, response))
-    return responses
-
-
-def sendExact(signal=Any, sender=Anonymous, *arguments, **named):
-    """Send signal only to those receivers registered for exact message
-
-    sendExact allows for avoiding Any/Anonymous registered
-    handlers, sending only to those receivers explicitly
-    registered for a particular signal on a particular
-    sender.
-    """
-    responses = []
-    for receiver in liveReceivers(getReceivers(sender, signal)):
-        response = robustapply.robustApply(
-            receiver,
-            signal=signal,
-            sender=sender,
-            *arguments,
-            **named
-        )
-        responses.append((receiver, response))
-    return responses
-
-
-def _removeReceiver(receiver):
-    """Remove receiver from connections."""
-    if not sendersBack:
-        # During module cleanup the mapping will be replaced with None
-        return False
-    backKey = id(receiver)
-    try:
-        backSet = sendersBack.pop(backKey)
-    except KeyError as err:
-        return False
-    else:
-        for senderkey in backSet:
-            try:
-                signals = connections[senderkey].keys()
-            except KeyError as err:
-                pass
-            else:
-                for signal in signals:
-                    try:
-                        receivers = connections[senderkey][signal]
-                    except KeyError:
-                        pass
-                    else:
-                        try:
-                            receivers.remove(receiver)
-                        except Exception as err:
-                            pass
-                    _cleanupConnections(senderkey, signal)
-
-
-def _cleanupConnections(senderkey, signal):
-    """Delete any empty signals for senderkey. Delete senderkey if empty."""
-    try:
-        receivers = connections[senderkey][signal]
-    except:
-        pass
-    else:
-        if not receivers:
-            # No more connected receivers. Therefore, remove the signal.
-            try:
-                signals = connections[senderkey]
-            except KeyError:
-                pass
-            else:
-                del signals[signal]
-                if not signals:
-                    # No more signal connections. Therefore, remove the sender.
-                    _removeSender(senderkey)
-
-
-def _removeSender(senderkey):
-    """Remove senderkey from connections."""
-    _removeBackrefs(senderkey)
-    try:
-        del connections[senderkey]
-    except KeyError:
-        pass
-    # Senderkey will only be in senders dictionary if sender
-    # could be weakly referenced.
-    try:
-        del senders[senderkey]
-    except:
-        pass
-
-
-def _removeBackrefs(senderkey):
-    """Remove all back-references to this senderkey"""
-    try:
-        signals = connections[senderkey]
-    except KeyError:
-        signals = None
-    else:
-        items = signals.items()
-
-        def allReceivers():
-            for signal, set in items:
-                for item in set:
-                    yield item
-
-        for receiver in allReceivers():
-            _killBackref(receiver, senderkey)
-
-
-def _removeOldBackRefs(senderkey, signal, receiver, receivers):
-    """Kill old sendersBack references from receiver
-
-    This guards against multiple registration of the same
-    receiver for a given signal and sender leaking memory
-    as old back reference records build up.
-
-    Also removes old receiver instance from receivers
-    """
-    try:
-        index = receivers.index(receiver)
-        # need to scan back references here and remove senderkey
-    except ValueError:
-        return False
-    else:
-        oldReceiver = receivers[index]
-        del receivers[index]
-        found = 0
-        signals = connections.get(signal)
-        if signals is not None:
-            for sig, recs in six.iteritems(connections.get(signal, {})):
-                if sig != signal:
-                    for rec in recs:
-                        if rec is oldReceiver:
-                            found = 1
-                            break
-        if not found:
-            _killBackref(oldReceiver, senderkey)
-            return True
-        return False
-
-
-def _killBackref(receiver, senderkey):
-    """Do the actual removal of back reference from receiver to senderkey"""
-    receiverkey = id(receiver)
-    set = sendersBack.get(receiverkey, ())
-    while senderkey in set:
-        try:
-            set.remove(senderkey)
-        except:
-            break
-    if not set:
-        try:
-            del sendersBack[receiverkey]
-        except KeyError:
-            pass
-    return True
diff --git a/scrapy/xlib/pydispatch/errors.py b/scrapy/xlib/pydispatch/errors.py
deleted file mode 100644
index c5540d8f7..000000000
--- a/scrapy/xlib/pydispatch/errors.py
+++ /dev/null
@@ -1,15 +0,0 @@
-"""Error types for dispatcher mechanism
-"""
-
-
-class DispatcherError(Exception):
-    """Base class for all Dispatcher errors"""
-
-
-class DispatcherKeyError(KeyError, DispatcherError):
-    """Error raised when unknown (sender,signal) set specified"""
-
-
-class DispatcherTypeError(TypeError, DispatcherError):
-    """Error raised when inappropriate signal-type specified (None)"""
-
diff --git a/scrapy/xlib/pydispatch/license.txt b/scrapy/xlib/pydispatch/license.txt
deleted file mode 100644
index 0960a6073..000000000
--- a/scrapy/xlib/pydispatch/license.txt
+++ /dev/null
@@ -1,34 +0,0 @@
-PyDispatcher License
-
-	Copyright (c) 2001-2006, Patrick K. O'Brien and Contributors
-	All rights reserved.
-	
-	Redistribution and use in source and binary forms, with or without
-	modification, are permitted provided that the following conditions
-	are met:
-	
-		Redistributions of source code must retain the above copyright
-		notice, this list of conditions and the following disclaimer.
-	
-		Redistributions in binary form must reproduce the above
-		copyright notice, this list of conditions and the following
-		disclaimer in the documentation and/or other materials
-		provided with the distribution.
-	
-		The name of Patrick K. O'Brien, or the name of any Contributor,
-		may not be used to endorse or promote products derived from this 
-		software without specific prior written permission.
-	
-	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-	``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-	LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-	FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-	COPYRIGHT HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
-	INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-	(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-	SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-	HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-	STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-	ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
-	OF THE POSSIBILITY OF SUCH DAMAGE. 
-
diff --git a/scrapy/xlib/pydispatch/robust.py b/scrapy/xlib/pydispatch/robust.py
deleted file mode 100644
index a28f127e2..000000000
--- a/scrapy/xlib/pydispatch/robust.py
+++ /dev/null
@@ -1,54 +0,0 @@
-"""Module implementing error-catching version of send (sendRobust)"""
-from scrapy.xlib.pydispatch.dispatcher import Any, Anonymous, liveReceivers, getAllReceivers
-from scrapy.xlib.pydispatch.robustapply import robustApply
-
-
-def sendRobust(signal=Any, sender=Anonymous, *arguments, **named):
-    """Send signal from sender to all connected receivers catching errors
-
-    signal -- (hashable) signal value, see connect for details
-
-    sender -- the sender of the signal
-
-        if Any, only receivers registered for Any will receive
-        the message.
-
-        if Anonymous, only receivers registered to receive
-        messages from Anonymous or Any will receive the message
-
-        Otherwise can be any python object (normally one
-        registered with a connect if you actually want
-        something to occur).
-
-    arguments -- positional arguments which will be passed to
-        *all* receivers. Note that this may raise TypeErrors
-        if the receivers do not allow the particular arguments.
-        Note also that arguments are applied before named
-        arguments, so they should be used with care.
-
-    named -- named arguments which will be filtered according
-        to the parameters of the receivers to only provide those
-        acceptable to the receiver.
-
-    Return a list of tuple pairs [(receiver, response), ... ]
-
-    if any receiver raises an error (specifically any subclass of Exception),
-    the error instance is returned as the result for that receiver.
-    """
-    # Call each receiver with whatever arguments it can accept.
-    # Return a list of tuple pairs [(receiver, response), ... ].
-    responses = []
-    for receiver in liveReceivers(getAllReceivers(sender, signal)):
-        try:
-            response = robustApply(
-                receiver,
-                signal=signal,
-                sender=sender,
-                *arguments,
-                **named
-            )
-        except Exception as err:
-            responses.append((receiver, err))
-        else:
-            responses.append((receiver, response))
-    return responses
diff --git a/scrapy/xlib/pydispatch/robustapply.py b/scrapy/xlib/pydispatch/robustapply.py
deleted file mode 100644
index 9977ac6b7..000000000
--- a/scrapy/xlib/pydispatch/robustapply.py
+++ /dev/null
@@ -1,58 +0,0 @@
-"""Robust apply mechanism
-
-Provides a function "call", which can sort out
-what arguments a given callable object can take,
-and subset the given arguments to match only
-those which are acceptable.
-"""
-import inspect
-
-
-def function(receiver):
-    """Get function-like callable object for given receiver
-
-    returns (function_or_method, codeObject, fromMethod)
-
-    If fromMethod is true, then the callable already
-    has its first argument bound
-    """
-    if inspect.isclass(receiver) and hasattr(receiver, '__call__'):
-        # receiver is a class instance; assume it is callable.
-        # Reassign receiver to the actual method that will be called.
-        if hasattr(receiver.__call__, 'im_func') or \
-                hasattr(receiver.__call__, 'im_code'):
-            receiver = receiver.__call__
-
-    if hasattr(receiver, 'im_func'):
-        # an instance-method...
-        return receiver, receiver.im_func.func_code, 1
-    elif not hasattr(receiver, 'func_code'):
-        raise ValueError(
-            'unknown receiver type %s %s' % (receiver, type(receiver)))
-
-    return receiver, receiver.func_code, 0
-
-
-def robustApply(receiver, *arguments, **named):
-    """Call receiver with arguments and an appropriate subset of named
-    """
-    receiver, codeObject, startIndex = function(receiver)
-    acceptable = codeObject.co_varnames[
-                 startIndex + len(arguments):codeObject.co_argcount]
-    for name in codeObject.co_varnames[startIndex:startIndex + len(arguments)]:
-        if name in named:
-            raise TypeError(
-                """Argument %r specified both positionally and as a keyword for calling %r""" % (
-                    name, receiver,
-                )
-            )
-
-    if not (codeObject.co_flags & 8):
-        # fc does not have a **kwds type parameter, therefore
-        # remove unacceptable arguments.
-        for arg in named.keys():
-            if arg not in acceptable:
-                del named[arg]
-
-    return receiver(*arguments, **named)
-
diff --git a/scrapy/xlib/pydispatch/saferef.py b/scrapy/xlib/pydispatch/saferef.py
deleted file mode 100644
index bd9659673..000000000
--- a/scrapy/xlib/pydispatch/saferef.py
+++ /dev/null
@@ -1,180 +0,0 @@
-"""Refactored "safe reference" from dispatcher.py"""
-from __future__ import print_function
-import weakref, traceback
-
-
-def safeRef(target, onDelete=None):
-    """Return a *safe* weak reference to a callable target
-
-    target -- the object to be weakly referenced, if it's a
-        bound method reference, will create a BoundMethodWeakref,
-        otherwise creates a simple weakref.
-    onDelete -- if provided, will have a hard reference stored
-        to the callable to be called after the safe reference
-        goes out of scope with the reference object, (either a
-        weakref or a BoundMethodWeakref) as argument.
-    """
-    if hasattr(target, 'im_self'):
-        if target.im_self is not None:
-            # Turn a bound method into a BoundMethodWeakref instance.
-            # Keep track of these instances for lookup by disconnect().
-            assert hasattr(target, 'im_func'), """safeRef target %r has im_self, but no im_func, don't know how to create reference"""%( target,)
-            reference = BoundMethodWeakref(
-                target=target,
-                onDelete=onDelete
-            )
-            return reference
-    if onDelete is not None:
-        return weakref.ref(target, onDelete)
-    else:
-        return weakref.ref(target)
-
-
-class BoundMethodWeakref(object):
-    """'Safe' and reusable weak references to instance methods
-
-    BoundMethodWeakref objects provide a mechanism for
-    referencing a bound method without requiring that the
-    method object itself (which is normally a transient
-    object) is kept alive.  Instead, the BoundMethodWeakref
-    object keeps weak references to both the object and the
-    function which together define the instance method.
-
-    Attributes:
-        key -- the identity key for the reference, calculated
-            by the class's calculateKey method applied to the
-            target instance method
-        deletionMethods -- sequence of callable objects taking
-            single argument, a reference to this object which
-            will be called when *either* the target object or
-            target function is garbage collected (i.e. when
-            this object becomes invalid).  These are specified
-            as the onDelete parameters of safeRef calls.
-        weakSelf -- weak reference to the target object
-        weakFunc -- weak reference to the target function
-
-    Class Attributes:
-        _allInstances -- class attribute pointing to all live
-            BoundMethodWeakref objects indexed by the class's
-            calculateKey(target) method applied to the target
-            objects.  This weak value dictionary is used to
-            short-circuit creation so that multiple references
-            to the same (object, function) pair produce the
-            same BoundMethodWeakref instance.
-
-    """
-    _allInstances = weakref.WeakValueDictionary()
-
-    def __new__(cls, target, onDelete=None, *arguments, **named):
-        """Create new instance or return current instance
-
-        Basically this method of construction allows us to
-        short-circuit creation of references to already-
-        referenced instance methods.  The key corresponding
-        to the target is calculated, and if there is already
-        an existing reference, that is returned, with its
-        deletionMethods attribute updated.  Otherwise the
-        new instance is created and registered in the table
-        of already-referenced methods.
-        """
-        key = cls.calculateKey(target)
-        current = cls._allInstances.get(key)
-        if current is not None:
-            current.deletionMethods.append(onDelete)
-            return current
-        else:
-            base = super(BoundMethodWeakref, cls).__new__(cls)
-            cls._allInstances[key] = base
-            base.__init__(target, onDelete, *arguments, **named)
-            return base
-
-    def __init__(self, target, onDelete=None):
-        """Return a weak-reference-like instance for a bound method
-
-        target -- the instance-method target for the weak
-            reference, must have im_self and im_func attributes
-            and be reconstructable via:
-                target.im_func.__get__( target.im_self )
-            which is true of built-in instance methods.
-        onDelete -- optional callback which will be called
-            when this weak reference ceases to be valid
-            (i.e. either the object or the function is garbage
-            collected).  Should take a single argument,
-            which will be passed a pointer to this object.
-        """
-
-        def remove(weak, self=self):
-            """Set self.isDead to true when method or instance is destroyed"""
-            methods = self.deletionMethods[:]
-            del self.deletionMethods[:]
-            try:
-                del self.__class__._allInstances[self.key]
-            except KeyError:
-                pass
-            for function in methods:
-                try:
-                    if callable(function):
-                        function(self)
-                except Exception as e:
-                    try:
-                        traceback.print_exc()
-                    except AttributeError as err:
-                        print(
-                            '''Exception during saferef %s cleanup function %s: %s''' % (
-                                self, function, e
-                            ))
-
-        self.deletionMethods = [onDelete]
-        self.key = self.calculateKey(target)
-        self.weakSelf = weakref.ref(target.im_self, remove)
-        self.weakFunc = weakref.ref(target.im_func, remove)
-        self.selfName = target.im_self.__class__.__name__
-        self.funcName = str(target.im_func.__name__)
-
-    def calculateKey(cls, target):
-        """Calculate the reference key for this reference
-
-        Currently this is a two-tuple of the id()'s of the
-        target object and the target function respectively.
-        """
-        return (id(target.im_self), id(target.im_func))
-
-    calculateKey = classmethod(calculateKey)
-
-    def __str__(self):
-        """Give a friendly representation of the object"""
-        return """%s( %s.%s )""" % (
-            self.__class__.__name__,
-            self.selfName,
-            self.funcName,
-        )
-
-    __repr__ = __str__
-
-    def __nonzero__(self):
-        """Whether we are still a valid reference"""
-        return self() is not None
-
-    def __cmp__(self, other):
-        """Compare with another reference"""
-        if not isinstance(other, self.__class__):
-            return cmp(self.__class__, type(other))
-        return cmp(self.key, other.key)
-
-    def __call__(self):
-        """Return a strong reference to the bound method
-
-        If the target cannot be retrieved, then will
-        return None, otherwise returns a bound instance
-        method for our object and function.
-
-        Note:
-            You may call this method any number of times,
-            as it does not invalidate the reference.
-        """
-        target = self.weakSelf()
-        if target is not None:
-            function = self.weakFunc()
-            if function is not None:
-                return function.__get__(target)
-        return None
diff --git a/setup.py b/setup.py
index bacdca358..6dc1cfd94 100644
--- a/setup.py
+++ b/setup.py
@@ -44,5 +44,6 @@ setup(
         'pyOpenSSL',
         'cssselect>=0.9',
         'six>=1.5.2',
+        'PyDispatcher>=2.0.5',
     ],
 )
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 6c14adf43..9d84e7634 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -49,7 +49,6 @@ tests/test_utils_iterators.py
 tests/test_utils_log.py
 tests/test_utils_reqser.py
 tests/test_utils_response.py
-tests/test_utils_signal.py
 tests/test_utils_template.py
 tests/test_webclient.py
 
diff --git a/tests/test_engine.py b/tests/test_engine.py
index d7ad88abb..e14957eae 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -20,7 +20,7 @@ from twisted.trial import unittest
 
 from scrapy import signals
 from scrapy.utils.test import get_crawler
-from scrapy.xlib.pydispatch import dispatcher
+from pydispatch import dispatcher
 from tests import tests_datadir
 from scrapy.spiders import Spider
 from scrapy.item import Item, Field
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index 571073911..b7de85049 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -2,8 +2,8 @@ from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.python.failure import Failure
 from twisted.internet import defer, reactor
+from pydispatch import dispatcher
 
-from scrapy.xlib.pydispatch import dispatcher
 from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
 
 

From 2c937f35544de1492c627d8bcad9747653d4c813 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Sat, 25 Jul 2015 18:10:29 +0200
Subject: [PATCH 0424/2974] PY3 port scrapy.dupefilters

---
 tests/py3-ignores.txt     | 1 -
 tests/test_dupefilters.py | 3 ++-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 9d84e7634..0265f2985 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -25,7 +25,6 @@ tests/test_downloadermiddleware_retry.py
 tests/test_downloadermiddleware_robotstxt.py
 tests/test_downloadermiddleware_stats.py
 tests/test_downloadermiddleware_useragent.py
-tests/test_dupefilters.py
 tests/test_engine.py
 tests/test_http_cookies.py
 tests/test_logformatter.py
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index f8f800a72..8de50f004 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -3,6 +3,7 @@ import unittest
 
 from scrapy.dupefilters import RFPDupeFilter
 from scrapy.http import Request
+from scrapy.utils.python import to_bytes
 
 
 class RFPDupeFilterTest(unittest.TestCase):
@@ -43,7 +44,7 @@ class RFPDupeFilterTest(unittest.TestCase):
 
             def request_fingerprint(self, request):
                 fp = hashlib.sha1()
-                fp.update(request.url.lower())
+                fp.update(to_bytes(request.url.lower()))
                 return fp.hexdigest()
 
         case_insensitive_dupefilter = CaseInsensitiveRFPDupeFilter()

From c78e2636f944c291c875108c32bfe229112ad9fa Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 28 Jul 2015 14:31:42 +0500
Subject: [PATCH 0425/2974] TST a test to show that dupefilter persistence is
 not working

---
 tests/test_dupefilters.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index f8f800a72..a9d92a1a2 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -1,5 +1,7 @@
 import hashlib
+import tempfile
 import unittest
+import shutil
 
 from scrapy.dupefilters import RFPDupeFilter
 from scrapy.http import Request
@@ -23,6 +25,27 @@ class RFPDupeFilterTest(unittest.TestCase):
 
         dupefilter.close('finished')
 
+    def test_dupefilter_path(self):
+        r1 = Request('http://scrapytest.org/1')
+        r2 = Request('http://scrapytest.org/2')
+
+        path = tempfile.mkdtemp()
+        try:
+            df = RFPDupeFilter(path)
+            df.open()
+            assert not df.request_seen(r1)
+            assert df.request_seen(r1)
+            df.close('finished')
+
+            df2 = RFPDupeFilter(path)
+            df2.open()
+            assert df2.request_seen(r1)
+            assert not df2.request_seen(r2)
+            assert df2.request_seen(r2)
+            df2.close('finished')
+        finally:
+            shutil.rmtree(path)
+
     def test_request_fingerprint(self):
         """Test if customization of request_fingerprint method will change
         output of request_seen.

From 836cb84b7a78f7d8ea439362234387d799c06b5b Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 28 Jul 2015 14:32:14 +0500
Subject: [PATCH 0426/2974] fixed RFPDupeFilter persistence

---
 scrapy/dupefilters.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 739ba922f..9d8966b9c 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -36,6 +36,7 @@ class RFPDupeFilter(BaseDupeFilter):
         self.logger = logging.getLogger(__name__)
         if path:
             self.file = open(os.path.join(path, 'requests.seen'), 'a+')
+            self.file.seek(0)
             self.fingerprints.update(x.rstrip() for x in self.file)
 
     @classmethod

From 1a1bc2c6b5230fdff516b38c2bee247baeb7c5b0 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sat, 25 Jul 2015 17:57:46 +0000
Subject: [PATCH 0427/2974] PY3 port
 scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware

---
 scrapy/downloadermiddlewares/robotstxt.py    |  14 ++-
 tests/test_downloadermiddleware_robotstxt.py | 119 ++++++++++++++-----
 2 files changed, 99 insertions(+), 34 deletions(-)

diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 77e08b7e0..457620d85 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -65,5 +65,17 @@ class RobotsTxtMiddleware(object):
 
     def _parse_robots(self, response):
         rp = robotparser.RobotFileParser(response.url)
-        rp.parse(response.body.splitlines())
+        body = ''
+        if hasattr(response, 'body_as_unicode'):
+            body = response.body_as_unicode()
+        else: # last effort try
+            try:
+                body = response.body.decode('utf-8')
+            except UnicodeDecodeError:
+                # If we found garbage, disregard it:,
+                # but keep the lookup cached (in self._parsers)
+                # Running rp.parse() will set rp state from
+                # 'disallow all' to 'allow any'.
+                pass
+        rp.parse(body.splitlines())
         self._parsers[urlparse_cached(response).netloc] = rp
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 372a0a2c3..b9c002f85 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -6,15 +6,44 @@ from twisted.python import failure
 from twisted.trial import unittest
 from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
 from scrapy.exceptions import IgnoreRequest, NotConfigured
-from scrapy.http import Request, Response
+from scrapy.http import Request, Response, TextResponse
 from scrapy.settings import Settings
 from tests import mock
 
 
 class RobotsTxtMiddlewareTest(unittest.TestCase):
 
+    def setUp(self):
+        self.crawler = mock.MagicMock()
+        self.crawler.settings = Settings()
+        self.crawler.engine.download = mock.MagicMock()
+
+    def tearDown(self):
+        del self.crawler
+
+    def test_robotstxt_settings(self):
+        self.crawler.settings = Settings()
+        self.crawler.settings.set('USER_AGENT', 'CustomAgent')
+        self.assertRaises(NotConfigured, RobotsTxtMiddleware, self.crawler)
+
+    def _get_successful_crawler(self):
+        crawler = self.crawler
+        crawler.settings.set('ROBOTSTXT_OBEY', True)
+        ROBOTS = re.sub(b'^\s+(?m)', b'', b'''
+        User-Agent: *
+        Disallow: /admin/
+        Disallow: /static/
+        ''')
+        response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
+        def return_response(request, spider):
+            deferred = Deferred()
+            reactor.callFromThread(deferred.callback, response)
+            return deferred
+        crawler.engine.download.side_effect = return_response
+        return crawler
+
     def test_robotstxt(self):
-        middleware = self._get_middleware()
+        middleware = RobotsTxtMiddleware(self._get_successful_crawler())
         # There is a bit of neglect in robotstxt.py: robots.txt is fetched asynchronously,
         # and it is actually fetched only *after* first process_request completes.
         # So, first process_request will always succeed.
@@ -30,8 +59,8 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         return deferred
 
     def test_robotstxt_meta(self):
+        middleware = RobotsTxtMiddleware(self._get_successful_crawler())
         meta = {'dont_obey_robotstxt': True}
-        middleware = self._get_middleware()
         self.assertNotIgnored(Request('http://site.local', meta=meta), middleware)
         def test(r):
             self.assertNotIgnored(Request('http://site.local/allowed', meta=meta), middleware)
@@ -42,19 +71,67 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         reactor.callFromThread(deferred.callback, None)
         return deferred
 
-    def test_robotstxt_error(self):
-        crawler = mock.MagicMock()
-        crawler.settings = Settings()
+    def _get_garbage_crawler(self):
+        crawler = self.crawler
         crawler.settings.set('ROBOTSTXT_OBEY', True)
-        crawler.engine.download = mock.MagicMock()
+        response = Response('http://site.local/robots.txt', body=b'GIF89a\xd3\x00\xfe\x00\xa2')
+        def return_response(request, spider):
+            deferred = Deferred()
+            reactor.callFromThread(deferred.callback, response)
+            return deferred
+        crawler.engine.download.side_effect = return_response
+        return crawler
+
+    def test_robotstxt_garbage(self):
+        # garbage response should be discarded, equal 'allow all'
+        middleware = RobotsTxtMiddleware(self._get_garbage_crawler())
+        middleware._logerror = mock.MagicMock()
+        middleware.process_request(Request('http://site.local'), None)
+        self.assertNotIgnored(Request('http://site.local'), middleware)
+        def test(r):
+            self.assertNotIgnored(Request('http://site.local/allowed'), middleware)
+            self.assertNotIgnored(Request('http://site.local/admin/main'), middleware)
+            self.assertNotIgnored(Request('http://site.local/static/'), middleware)
+        deferred = Deferred()
+        deferred.addCallback(test)
+        deferred.addErrback(lambda _: self.assertIsNone(middleware._logerror.assert_any_call()))
+        reactor.callFromThread(deferred.callback, None)
+        return deferred
+
+    def _get_emptybody_crawler(self):
+        crawler = self.crawler
+        crawler.settings.set('ROBOTSTXT_OBEY', True)
+        response = Response('http://site.local/robots.txt')
+        def return_response(request, spider):
+            deferred = Deferred()
+            reactor.callFromThread(deferred.callback, response)
+            return deferred
+        crawler.engine.download.side_effect = return_response
+        return crawler
+
+    def test_robotstxt_empty_response(self):
+        # empty response should equal 'allow all'
+        middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
+        self.assertNotIgnored(Request('http://site.local'), middleware)
+        def test(r):
+            self.assertNotIgnored(Request('http://site.local/allowed'), middleware)
+            self.assertNotIgnored(Request('http://site.local/admin/main'), middleware)
+            self.assertNotIgnored(Request('http://site.local/static/'), middleware)
+        deferred = Deferred()
+        deferred.addCallback(test)
+        reactor.callFromThread(deferred.callback, None)
+        return deferred
+
+    def test_robotstxt_error(self):
+        self.crawler.settings.set('ROBOTSTXT_OBEY', True)
         err = error.DNSLookupError('Robotstxt address not found')
         def return_failure(request, spider):
             deferred = Deferred()
             reactor.callFromThread(deferred.errback, failure.Failure(err))
             return deferred
-        crawler.engine.download.side_effect = return_failure
+        self.crawler.engine.download.side_effect = return_failure
 
-        middleware = RobotsTxtMiddleware(crawler)
+        middleware = RobotsTxtMiddleware(self.crawler)
         middleware._logerror = mock.MagicMock()
         middleware.process_request(Request('http://site.local'), None)
         deferred = Deferred()
@@ -69,27 +146,3 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
     def assertIgnored(self, request, middleware):
         spider = None  # not actually used
         self.assertRaises(IgnoreRequest, middleware.process_request, request, spider)
-
-    def _get_crawler(self):
-        crawler = mock.MagicMock()
-        crawler.settings = Settings()
-        crawler.settings.set('USER_AGENT', 'CustomAgent')
-        self.assertRaises(NotConfigured, RobotsTxtMiddleware, crawler)
-        crawler.settings.set('ROBOTSTXT_OBEY', True)
-        crawler.engine.download = mock.MagicMock()
-        ROBOTS = re.sub(r'^\s+(?m)', '', '''
-        User-Agent: *
-        Disallow: /admin/
-        Disallow: /static/
-        ''')
-        response = Response('http://site.local/robots.txt', body=ROBOTS)
-        def return_response(request, spider):
-            deferred = Deferred()
-            reactor.callFromThread(deferred.callback, response)
-            return deferred
-        crawler.engine.download.side_effect = return_response
-        return crawler
-
-    def _get_middleware(self):
-        crawler = self._get_crawler()
-        return RobotsTxtMiddleware(crawler)

From 757f983a97332340faed38dacbd2c384af52f15b Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 29 Jul 2015 18:37:04 +0500
Subject: [PATCH 0428/2974] PY3: enable RobotsTxtMiddleware tests

---
 tests/py3-ignores.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 0265f2985..4dfab4a9a 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -22,7 +22,6 @@ tests/test_downloadermiddleware_httpproxy.py
 tests/test_downloadermiddleware.py
 tests/test_downloadermiddleware_redirect.py
 tests/test_downloadermiddleware_retry.py
-tests/test_downloadermiddleware_robotstxt.py
 tests/test_downloadermiddleware_stats.py
 tests/test_downloadermiddleware_useragent.py
 tests/test_engine.py

From 683ef2a8d9666bbf20462a35d01d517ad1eca73e Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sat, 25 Jul 2015 16:44:28 +0000
Subject: [PATCH 0429/2974] replace rfc822 with email.utils

---
 scrapy/pipelines/files.py | 6 +++---
 scrapy/utils/defer.py     | 2 +-
 scrapy/utils/misc.py      | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 250f46ad8..a449793c9 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -7,9 +7,9 @@ See documentation in topics/media-pipeline.rst
 import hashlib
 import os
 import os.path
-import rfc822
 import time
 import logging
+from email.utils import parsedate_tz, mktime_tz
 from six.moves.urllib.parse import urlparse
 from collections import defaultdict
 import six
@@ -91,8 +91,8 @@ class S3FilesStore(object):
         def _onsuccess(boto_key):
             checksum = boto_key.etag.strip('"')
             last_modified = boto_key.last_modified
-            modified_tuple = rfc822.parsedate_tz(last_modified)
-            modified_stamp = int(rfc822.mktime_tz(modified_tuple))
+            modified_tuple = parsedate_tz(last_modified)
+            modified_stamp = int(mktime_tz(modified_tuple))
             return {'checksum': checksum, 'last_modified': modified_stamp}
 
         return self._get_boto_key(path).addCallback(_onsuccess)
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 7a3f20476..8f3824abf 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -97,7 +97,7 @@ def iter_errback(iterable, errback, *a, **kw):
     iterating it.
     """
     it = iter(iterable)
-    while 1:
+    while True:
         try:
             yield next(it)
         except StopIteration:
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index c269b7f74..4215e41d2 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -106,7 +106,7 @@ def md5sum(file):
     '784406af91dd5a54fbb9c84c2236595a'
     """
     m = hashlib.md5()
-    while 1:
+    while True:
         d = file.read(8096)
         if not d:
             break

From 8ab1648a363090d8fc22547cb3b763d79ed30c7d Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sat, 25 Jul 2015 16:58:30 +0000
Subject: [PATCH 0430/2974] PY3 fix test middleware

---
 tests/py3-ignores.txt    |  1 -
 tests/test_middleware.py | 21 +++++++++++++++------
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 4dfab4a9a..a6c92d508 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -28,7 +28,6 @@ tests/test_engine.py
 tests/test_http_cookies.py
 tests/test_logformatter.py
 tests/test_mail.py
-tests/test_middleware.py
 tests/test_pipeline_files.py
 tests/test_pipeline_images.py
 tests/test_pipeline_media.py
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index 48131462c..b6d885330 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -3,6 +3,7 @@ from twisted.trial import unittest
 from scrapy.settings import Settings
 from scrapy.exceptions import NotConfigured
 from scrapy.middleware import MiddlewareManager
+import six
 
 class M1(object):
 
@@ -65,12 +66,20 @@ class MiddlewareManagerTest(unittest.TestCase):
 
     def test_methods(self):
         mwman = TestMiddlewareManager(M1(), M2(), M3())
-        self.assertEqual([x.im_class for x in mwman.methods['open_spider']],
-            [M1, M2])
-        self.assertEqual([x.im_class for x in mwman.methods['close_spider']],
-            [M2, M1])
-        self.assertEqual([x.im_class for x in mwman.methods['process']],
-            [M1, M3])
+        if six.PY2:
+            self.assertEqual([x.im_class for x in mwman.methods['open_spider']],
+                [M1, M2])
+            self.assertEqual([x.im_class for x in mwman.methods['close_spider']],
+                [M2, M1])
+            self.assertEqual([x.im_class for x in mwman.methods['process']],
+                [M1, M3])
+        else:
+            self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
+                [M1, M2])
+            self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
+                [M2, M1])
+            self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
+                [M1, M3])
 
     def test_enabled(self):
         m1, m2, m3 = M1(), M2(), M3()

From ec8afbc060fdd4b5159470c3964f4482ccf4dbe3 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sat, 25 Jul 2015 17:56:46 +0000
Subject: [PATCH 0431/2974] PY3 fix test pipeline media

---
 tests/py3-ignores.txt        | 1 -
 tests/test_pipeline_files.py | 2 +-
 tests/test_pipeline_media.py | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index a6c92d508..96b07ae28 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -30,7 +30,6 @@ tests/test_logformatter.py
 tests/test_mail.py
 tests/test_pipeline_files.py
 tests/test_pipeline_images.py
-tests/test_pipeline_media.py
 tests/test_proxy_connect.py
 tests/test_responsetypes.py
 tests/test_selector_csstranslator.py
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index b12f41174..ac0438eba 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -192,7 +192,7 @@ def _create_item_with_files(*files):
 def _prepare_request_object(item_url):
     return Request(
         item_url,
-        meta={'response': Response(item_url, status=200, body='data')})
+        meta={'response': Response(item_url, status=200, body=b'data')})
 
 
 if __name__ == "__main__":
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index fd8b28ce1..f30b4fea3 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -44,7 +44,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 
     def test_default_media_downloaded(self):
         request = Request('http://url')
-        response = Response('http://url', body='')
+        response = Response('http://url', body=b'')
         assert self.pipe.media_downloaded(response, request, self.info) is response
 
     def test_default_media_failed(self):

From e044bfa60f072aca25e5561136dc0b9e89b2a1ed Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sat, 25 Jul 2015 20:51:27 +0000
Subject: [PATCH 0432/2974] PY3 fix test downloadermiddleware decompression

---
 scrapy/responsetypes.py                          | 4 ++--
 tests/py3-ignores.txt                            | 1 -
 tests/test_downloadermiddleware_decompression.py | 2 +-
 tests/test_responsetypes.py                      | 2 +-
 4 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 7c017feef..c212f5706 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -92,9 +92,9 @@ class ResponseTypes(object):
         chunk = body[:5000]
         if isbinarytext(chunk):
             return self.from_mimetype('application/octet-stream')
-        elif "<html>" in chunk.lower():
+        elif b"<html>" in chunk.lower():
             return self.from_mimetype('text/html')
-        elif "<?xml" in chunk.lower():
+        elif b"<?xml" in chunk.lower():
             return self.from_mimetype('text/xml')
         else:
             return self.from_mimetype('text')
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 96b07ae28..84bd01c0e 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -12,7 +12,6 @@ tests/test_crawler.py
 tests/test_downloader_handlers.py
 tests/test_downloadermiddleware_ajaxcrawlable.py
 tests/test_downloadermiddleware_cookies.py
-tests/test_downloadermiddleware_decompression.py
 tests/test_downloadermiddleware_defaultheaders.py
 tests/test_downloadermiddleware_downloadtimeout.py
 tests/test_downloadermiddleware_httpauth.py
diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py
index 9143611fc..77b35a8c3 100644
--- a/tests/test_downloadermiddleware_decompression.py
+++ b/tests/test_downloadermiddleware_decompression.py
@@ -39,7 +39,7 @@ class DecompressionMiddlewareTest(TestCase):
         assert_samelines(self, new.body, rsp.body)
 
     def test_empty_response(self):
-        rsp = Response(url='http://test.com', body='')
+        rsp = Response(url='http://test.com', body=b'')
         new = self.mw.process_response(None, rsp, self.spider)
         assert new is rsp
         assert not rsp.body
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index b4083403a..1d78d0976 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -51,7 +51,7 @@ class ResponseTypesTest(unittest.TestCase):
         for source, cls in mappings:
             retcls = responsetypes.from_body(source)
             assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls)
-        
+
     def test_from_headers(self):
         mappings = [
             ({'Content-Type': ['text/html; charset=utf-8']}, HtmlResponse),

From 9d17d594712fdfd0da9e240c79901f7cb8d1441f Mon Sep 17 00:00:00 2001
From: Gregory Vigo Torres <gregory.vigo@gmail.com>
Date: Wed, 29 Jul 2015 20:05:45 +0200
Subject: [PATCH 0433/2974] from_content_disposition

---
 scrapy/responsetypes.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index c212f5706..5b0bc50a5 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -11,7 +11,7 @@ import six
 
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
-from scrapy.utils.python import isbinarytext
+from scrapy.utils.python import isbinarytext, to_bytes, to_native_str
 
 class ResponseTypes(object):
 
@@ -59,7 +59,7 @@ class ResponseTypes(object):
 
     def from_content_disposition(self, content_disposition):
         try:
-            filename = content_disposition.split(';')[1].split('=')[1]
+            filename = to_native_str(content_disposition).split(';')[1].split('=')[1]
             filename = filename.strip('"\'')
             return self.from_filename(filename)
         except IndexError:

From 36ae635fbe47d68025b475762cece2a6c9e8257f Mon Sep 17 00:00:00 2001
From: Gregory Vigo Torres <gregory.vigo@gmail.com>
Date: Wed, 29 Jul 2015 20:07:26 +0200
Subject: [PATCH 0434/2974] from_content_type

---
 scrapy/responsetypes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 5b0bc50a5..3e3518e79 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -54,7 +54,7 @@ class ResponseTypes(object):
         header """
         if content_encoding:
             return Response
-        mimetype = content_type.split(';')[0].strip().lower()
+        mimetype = to_native_str(content_type).split(';')[0].strip().lower()
         return self.from_mimetype(mimetype)
 
     def from_content_disposition(self, content_disposition):

From 42b8988eb2ca7bc31c0a3ab2346b584bed162e18 Mon Sep 17 00:00:00 2001
From: Gregory Vigo Torres <gregory.vigo@gmail.com>
Date: Wed, 29 Jul 2015 20:09:06 +0200
Subject: [PATCH 0435/2974] PY3 port responsetypes from_body

---
 scrapy/responsetypes.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 3e3518e79..b519c56d3 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -90,6 +90,7 @@ class ResponseTypes(object):
         it's not meant to be used except for special cases where response types
         cannot be guess using more straightforward methods."""
         chunk = body[:5000]
+        chunk = to_bytes(chunk)
         if isbinarytext(chunk):
             return self.from_mimetype('application/octet-stream')
         elif b"<html>" in chunk.lower():

From 9c12a3f23adeea263da870010d6e5ec207e98c8e Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Thu, 30 Jul 2015 12:12:08 +0200
Subject: [PATCH 0436/2974] Remove incorrect path for Sphinx autodoc

---
 docs/conf.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index 628596b71..dd9b8e227 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -18,7 +18,6 @@ from os import path
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
 sys.path.append(path.join(path.dirname(__file__), "_ext"))
-sys.path.append(path.join(path.dirname(path.dirname(__file__)), "scrapy"))
 sys.path.insert(0, path.dirname(path.dirname(__file__)))
 
 

From 5d75d44f2c1779d7176ea3e122478cb55cd98ee1 Mon Sep 17 00:00:00 2001
From: Gregory Vigo Torres <gregory.vigo@gmail.com>
Date: Thu, 30 Jul 2015 13:07:42 +0200
Subject: [PATCH 0437/2974] removed test_responsetypes from py3ignores

---
 tests/py3-ignores.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 84bd01c0e..47abd8004 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -30,7 +30,7 @@ tests/test_mail.py
 tests/test_pipeline_files.py
 tests/test_pipeline_images.py
 tests/test_proxy_connect.py
-tests/test_responsetypes.py
+
 tests/test_selector_csstranslator.py
 tests/test_selector_lxmldocument.py
 tests/test_selector.py

From 06b91da943c1355ddc8a1018b67ebd5f35bc833e Mon Sep 17 00:00:00 2001
From: Gregory Vigo Torres <gregory.vigo@gmail.com>
Date: Fri, 31 Jul 2015 14:31:11 +0200
Subject: [PATCH 0438/2974] using bytes for body constant

---
 tests/test_responsetypes.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 1d78d0976..2374d518f 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -43,10 +43,10 @@ class ResponseTypesTest(unittest.TestCase):
 
     def test_from_body(self):
         mappings = [
-            ('\x03\x02\xdf\xdd\x23', Response),
-            ('Some plain text\ndata with tabs\t and null bytes\0', TextResponse),
-            ('<html><head><title>Hello</title></head>', HtmlResponse),
-            ('<?xml version="1.0" encoding="utf-8"', XmlResponse),
+            (b'\x03\x02\xdf\xdd\x23', Response),
+            (b'Some plain text\ndata with tabs\t and null bytes\0', TextResponse),
+            (b'<html><head><title>Hello</title></head>', HtmlResponse),
+            (b'<?xml version="1.0" encoding="utf-8"', XmlResponse),
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_body(source)

From de6e013b9a8080cf759096e793272f6814e3617d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 31 Jul 2015 14:54:07 -0300
Subject: [PATCH 0439/2974] fix form requests tests on py3

---
 tests/test_http_request.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 5709b6b31..02111f075 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -551,7 +551,7 @@ class FormRequestTest(RequestTest):
         r1 = self.request_class.from_response(response, formid="form2")
         self.assertEqual(r1.method, 'POST')
         fs = _qs(r1)
-        self.assertEqual(fs, {'four': ['4'], 'three': ['3']})
+        self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']})
 
     def test_from_response_formname_notexists_fallback_formid(self):
         response = _buildresponse(
@@ -566,7 +566,7 @@ class FormRequestTest(RequestTest):
         r1 = self.request_class.from_response(response, formname="form3", formid="form2")
         self.assertEqual(r1.method, 'POST')
         fs = _qs(r1)
-        self.assertEqual(fs, {'four': ['4'], 'three': ['3']})
+        self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']})
 
     def test_from_response_formid_notexist(self):
         response = _buildresponse(
@@ -579,7 +579,7 @@ class FormRequestTest(RequestTest):
         r1 = self.request_class.from_response(response, formid="form3")
         self.assertEqual(r1.method, 'POST')
         fs = _qs(r1)
-        self.assertEqual(fs, {'one': ['1']})
+        self.assertEqual(fs, {b'one': [b'1']})
 
     def test_from_response_formid_errors_formnumber(self):
         response = _buildresponse(

From 02b5182608bb09dbf14486c939d70b641ad50660 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 31 Jul 2015 23:48:49 +0500
Subject: [PATCH 0440/2974] small ResponseTypes cleanup

---
 scrapy/responsetypes.py | 19 +++++++++++--------
 tests/py3-ignores.txt   |  1 -
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index b519c56d3..4880cc7b9 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -1,9 +1,8 @@
 """
 This module implements a class which returns the appropriate Response class
 based on different criteria.
-
 """
-
+from __future__ import absolute_import
 from mimetypes import MimeTypes
 from pkgutil import get_data
 from io import StringIO
@@ -13,6 +12,7 @@ from scrapy.http import Response
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import isbinarytext, to_bytes, to_native_str
 
+
 class ResponseTypes(object):
 
     CLASSES = {
@@ -69,11 +69,13 @@ class ResponseTypes(object):
         """Return the most appropriate Response class by looking at the HTTP
         headers"""
         cls = Response
-        if 'Content-Type' in headers:
-            cls = self.from_content_type(headers['Content-type'], \
-                headers.get('Content-Encoding'))
-        if cls is Response and 'Content-Disposition' in headers:
-            cls = self.from_content_disposition(headers['Content-Disposition'])
+        if b'Content-Type' in headers:
+            cls = self.from_content_type(
+                content_type=headers[b'Content-type'],
+                content_encoding=headers.get(b'Content-Encoding')
+            )
+        if cls is Response and b'Content-Disposition' in headers:
+            cls = self.from_content_disposition(headers[b'Content-Disposition'])
         return cls
 
     def from_filename(self, filename):
@@ -101,7 +103,8 @@ class ResponseTypes(object):
             return self.from_mimetype('text')
 
     def from_args(self, headers=None, url=None, filename=None, body=None):
-        """Guess the most appropriate Response class based on the given arguments"""
+        """Guess the most appropriate Response class based on
+        the given arguments."""
         cls = Response
         if headers is not None:
             cls = self.from_headers(headers)
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 47abd8004..c51bc5981 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -30,7 +30,6 @@ tests/test_mail.py
 tests/test_pipeline_files.py
 tests/test_pipeline_images.py
 tests/test_proxy_connect.py
-
 tests/test_selector_csstranslator.py
 tests/test_selector_lxmldocument.py
 tests/test_selector.py

From 5ae94ae42d656445b97e72ba859aae35cb9c4921 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 31 Jul 2015 16:33:37 -0300
Subject: [PATCH 0441/2974] Use new travis-ci container based builds

---
 .travis-workarounds.sh | 15 ---------------
 .travis.yml            |  2 +-
 2 files changed, 1 insertion(+), 16 deletions(-)
 delete mode 100755 .travis-workarounds.sh

diff --git a/.travis-workarounds.sh b/.travis-workarounds.sh
deleted file mode 100755
index 5c34e54f7..000000000
--- a/.travis-workarounds.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-set -e
-set -x
-
-if [[ "${TOXENV}" == "pypy" ]]; then
-    sudo add-apt-repository -y ppa:pypy/ppa
-    sudo apt-get -qy update
-    sudo apt-get install -y pypy pypy-dev
-    # This is required because we need to get rid of the Travis installed PyPy
-    # or it'll take precedence over the PPA installed one.
-    sudo rm -rf /usr/local/pypy/bin
-fi
-
-# Workaround travis-ci/travis-ci#2065
-pip install -U wheel
diff --git a/.travis.yml b/.travis.yml
index 4a0ef834c..419f52c96 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,12 +1,12 @@
 language: python
 python: 2.7
+sudo: false
 env:
 - TOXENV=py27
 - TOXENV=precise
 - TOXENV=py33
 - TOXENV=docs
 install:
-- "./.travis-workarounds.sh"
 - pip install -U tox
 script: tox
 notifications:

From 7f927f68e1d3eae71550a48c5484099ca7dd9ce1 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 1 Aug 2015 00:35:43 +0500
Subject: [PATCH 0442/2974] PY3 port scrapy.utils.response

---
 scrapy/utils/response.py     | 29 ++++++++++++---------
 tests/py3-ignores.txt        |  1 -
 tests/test_utils_response.py | 49 +++++++++++++++++++++++-------------
 3 files changed, 49 insertions(+), 30 deletions(-)

diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index b5d7a58ca..3d1af7e51 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -2,7 +2,6 @@
 This module provides some useful functions for working with
 scrapy.http.Response objects
 """
-
 import os
 import re
 import weakref
@@ -11,6 +10,7 @@ import tempfile
 
 from twisted.web import http
 from twisted.web.http import RESPONSES
+from scrapy.utils.python import to_bytes
 from w3lib import html
 
 from scrapy.utils.decorators import deprecated
@@ -27,10 +27,11 @@ def get_base_url(response):
     """Return the base url of the given response, joined with the response url"""
     if response not in _baseurl_cache:
         text = response.body_as_unicode()[0:4096]
-        _baseurl_cache[response] = html.get_base_url(text, response.url, \
+        _baseurl_cache[response] = html.get_base_url(text, response.url,
             response.encoding)
     return _baseurl_cache[response]
 
+
 _noscript_re = re.compile(u'<noscript>.*?</noscript>', re.IGNORECASE | re.DOTALL)
 _script_re = re.compile(u'<script.*?>.*?</script>', re.IGNORECASE | re.DOTALL)
 _metaref_cache = weakref.WeakKeyDictionary()
@@ -40,10 +41,11 @@ def get_meta_refresh(response):
         text = response.body_as_unicode()[0:4096]
         text = _noscript_re.sub(u'', text)
         text = _script_re.sub(u'', text)
-        _metaref_cache[response] = html.get_meta_refresh(text, response.url, \
+        _metaref_cache[response] = html.get_meta_refresh(text, response.url,
             response.encoding)
     return _metaref_cache[response]
 
+
 def response_status_message(status):
     """Return status code plus status text descriptive message
 
@@ -55,19 +57,21 @@ def response_status_message(status):
     """
     return '%s %s' % (status, http.responses.get(int(status)))
 
+
 def response_httprepr(response):
-    """Return raw HTTP representation (as string) of the given response. This
+    """Return raw HTTP representation (as bytes) of the given response. This
     is provided only for reference, since it's not the exact stream of bytes
     that was received (that's not exposed by Twisted).
     """
-
-    s = "HTTP/1.1 %d %s\r\n" % (response.status, RESPONSES.get(response.status, ''))
+    s = b"HTTP/1.1 " + to_bytes(str(response.status)) + b" " + \
+        to_bytes(RESPONSES.get(response.status, b'')) + b"\r\n"
     if response.headers:
-        s += response.headers.to_string() + "\r\n"
-    s += "\r\n"
+        s += response.headers.to_string() + b"\r\n"
+    s += b"\r\n"
     s += response.body
     return s
 
+
 def open_in_browser(response, _openfunc=webbrowser.open):
     """Open the given response in a local web browser, populating the <base>
     tag for external links to work
@@ -76,14 +80,15 @@ def open_in_browser(response, _openfunc=webbrowser.open):
     # XXX: this implementation is a bit dirty and could be improved
     body = response.body
     if isinstance(response, HtmlResponse):
-        if '<base' not in body:
-            body = body.replace('<head>', '<head><base href="%s">' % response.url)
+        if b'<base' not in body:
+            repl = '<head><base href="%s">' % response.url
+            body = body.replace(b'<head>', to_bytes(repl))
         ext = '.html'
     elif isinstance(response, TextResponse):
         ext = '.txt'
     else:
-        raise TypeError("Unsupported response type: %s" % \
-            response.__class__.__name__)
+        raise TypeError("Unsupported response type: %s" %
+                        response.__class__.__name__)
     fd, fname = tempfile.mkstemp(ext)
     os.write(fd, body)
     os.close(fd)
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index c51bc5981..432b58cac 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -42,7 +42,6 @@ tests/test_stats.py
 tests/test_utils_iterators.py
 tests/test_utils_log.py
 tests/test_utils_reqser.py
-tests/test_utils_response.py
 tests/test_utils_template.py
 tests/test_webclient.py
 
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 92c92c057..1d1638e06 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -3,53 +3,59 @@ import unittest
 from six.moves.urllib.parse import urlparse
 
 from scrapy.http import Response, TextResponse, HtmlResponse
-from scrapy.utils.response import response_httprepr, open_in_browser, get_meta_refresh
+from scrapy.utils.python import to_bytes
+from scrapy.utils.response import (response_httprepr, open_in_browser,
+                                   get_meta_refresh, get_base_url)
 
 __doctests__ = ['scrapy.utils.response']
 
+
 class ResponseUtilsTest(unittest.TestCase):
-    dummy_response = TextResponse(url='http://example.org/', body='dummy_response')
+    dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response')
 
     def test_response_httprepr(self):
         r1 = Response("http://www.example.com")
-        self.assertEqual(response_httprepr(r1), 'HTTP/1.1 200 OK\r\n\r\n')
+        self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n')
 
-        r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body="Some body")
-        self.assertEqual(response_httprepr(r1), 'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')
+        r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body=b"Some body")
+        self.assertEqual(response_httprepr(r1), b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')
 
-        r1 = Response("http://www.example.com", status=6666, headers={"Content-type": "text/html"}, body="Some body")
-        self.assertEqual(response_httprepr(r1), 'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')
+        r1 = Response("http://www.example.com", status=6666, headers={"Content-type": "text/html"}, body=b"Some body")
+        self.assertEqual(response_httprepr(r1), b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')
 
     def test_open_in_browser(self):
         url = "http:///www.example.com/some/page.html"
-        body = "<html> <head> <title>test page</title> </head> <body>test body</body> </html>"
+        body = b"<html> <head> <title>test page</title> </head> <body>test body</body> </html>"
+
         def browser_open(burl):
             path = urlparse(burl).path
             if not os.path.exists(path):
                 path = burl.replace('file://', '')
-            bbody = open(path).read()
-            assert '<base href="%s">' % url in bbody, "<base> tag not added"
+            with open(path, "rb") as f:
+                bbody = f.read()
+            self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody)
             return True
         response = HtmlResponse(url, body=body)
         assert open_in_browser(response, _openfunc=browser_open), \
             "Browser not called"
-        self.assertRaises(TypeError, open_in_browser, Response(url, body=body), \
-            debug=True)
+
+        resp = Response(url, body=body)
+        self.assertRaises(TypeError, open_in_browser, resp, debug=True)
 
     def test_get_meta_refresh(self):
-        r1 = HtmlResponse("http://www.example.com", body="""
+        r1 = HtmlResponse("http://www.example.com", body=b"""
         <html>
         <head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
         <body>blahablsdfsal&amp;</body>
         </html>""")
-        r2 = HtmlResponse("http://www.example.com", body="""
+        r2 = HtmlResponse("http://www.example.com", body=b"""
         <html>
         <head><title>Dummy</title><noScript>
         <meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
         </noSCRIPT>
         <body>blahablsdfsal&amp;</body>
         </html>""")
-        r3 = HtmlResponse("http://www.example.com", body="""
+        r3 = HtmlResponse("http://www.example.com", body=b"""
     <noscript><meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage</noscript>
     <script type="text/javascript">
     if(!checkCookies()){
@@ -61,5 +67,14 @@ class ResponseUtilsTest(unittest.TestCase):
         self.assertEqual(get_meta_refresh(r2), (None, None))
         self.assertEqual(get_meta_refresh(r3), (None, None))
 
-if __name__ == "__main__":
-    unittest.main()
+    def test_get_base_url(self):
+        resp = HtmlResponse("http://www.example.com", body=b"""
+        <html>
+        <head><base href="http://www.example.com/img/" target="_blank"></head>
+        <body>blahablsdfsal&amp;</body>
+        </html>""")
+        self.assertEqual(get_base_url(resp), "http://www.example.com/img/")
+
+        resp2 = HtmlResponse("http://www.example.com", body=b"""
+        <html><body>blahablsdfsal&amp;</body></html>""")
+        self.assertEqual(get_base_url(resp2), "http://www.example.com")

From b01292df02a379a5d587ddfd4149914f83a20204 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 1 Aug 2015 00:36:16 +0500
Subject: [PATCH 0443/2974] fix request_httprepr docstring

---
 scrapy/utils/request.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 4871681a9..ac415e508 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -71,7 +71,7 @@ def request_authenticate(request, username, password):
 
 
 def request_httprepr(request):
-    """Return the raw HTTP representation (as string) of the given request.
+    """Return the raw HTTP representation (as bytes) of the given request.
     This is provided only for reference since it's not the actual stream of
     bytes that will be send when performing the request (that's controlled
     by Twisted).

From 5d4cdb59a90cd2c36636f323713e28ed35579feb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 31 Jul 2015 16:39:12 -0300
Subject: [PATCH 0444/2974] cache pip cache

---
 .travis.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 419f52c96..18ef5a57f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,6 +15,9 @@ notifications:
     skip_join: true
     channels:
     - irc.freenode.org#scrapy
+cache:
+  directories:
+    - $HOME/.cache/pip
 deploy:
   provider: pypi
   distributions: "sdist bdist_wheel"

From 9316b2317bdf9b66ef90abccfcee16169bc537f1 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 1 Aug 2015 00:43:13 +0500
Subject: [PATCH 0445/2974] remove scrapy.utils.testsite from PY3 ignores

---
 conftest.py              | 5 +++++
 scrapy/utils/testsite.py | 2 ++
 tests/py3-ignores.txt    | 2 --
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/conftest.py b/conftest.py
index 5829f2952..f9ca3ab93 100644
--- a/conftest.py
+++ b/conftest.py
@@ -8,6 +8,7 @@ def _py_files(folder):
 
 
 collect_ignore = [
+    # deprecated or moved modules
     "scrapy/conf.py",
     "scrapy/stats.py",
     "scrapy/project.py",
@@ -19,6 +20,10 @@ collect_ignore = [
     "scrapy/command.py",
     "scrapy/linkextractor.py",
     "scrapy/spider.py",
+
+    # not a test, but looks like a test
+    "scrapy/utils/testsite.py",
+
 ] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp")
 
 
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index 7d4d4b2bd..01508bdb4 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -4,6 +4,7 @@ from six.moves.urllib.parse import urljoin
 from twisted.internet import reactor
 from twisted.web import server, resource, static, util
 
+
 class SiteTest(object):
 
     def setUp(self):
@@ -18,6 +19,7 @@ class SiteTest(object):
     def url(self, path):
         return urljoin(self.baseurl, path)
 
+
 def test_site():
     r = resource.Resource()
     r.putChild("text", static.Data("Works", "text/plain"))
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index c51bc5981..afebb0b31 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -52,8 +52,6 @@ scrapy/xlib/tx/endpoints.py
 scrapy/xlib/tx/client.py
 scrapy/xlib/tx/_newclient.py
 scrapy/xlib/tx/__init__.py
-scrapy/xlib/tx/__init__.py
-scrapy/utils/testsite.py
 scrapy/http/cookies.py
 scrapy/core/downloader/handlers/s3.py
 scrapy/core/downloader/handlers/http11.py

From 49fd77d693bfe17c162480d7fc86f5bed946f2c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 31 Jul 2015 16:45:43 -0300
Subject: [PATCH 0446/2974] indent travis yaml

---
 .travis.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 18ef5a57f..d0758fdaf 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,12 +2,12 @@ language: python
 python: 2.7
 sudo: false
 env:
-- TOXENV=py27
-- TOXENV=precise
-- TOXENV=py33
-- TOXENV=docs
+ - TOXENV=py27
+ - TOXENV=precise
+ - TOXENV=py33
+ - TOXENV=docs
 install:
-- pip install -U tox
+ - pip install -U tox
 script: tox
 notifications:
   irc:

From 08123207c56821173acb92a991462e684ce5062a Mon Sep 17 00:00:00 2001
From: David Tagatac <david@tagatac.net>
Date: Fri, 31 Jul 2015 17:01:59 -0400
Subject: [PATCH 0447/2974] minor: scrapy.Spider grammar and clarity

---
 docs/topics/spiders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 025d527a6..025232fab 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -53,7 +53,7 @@ scrapy.Spider
 .. class:: Spider()
 
    This is the simplest spider, and the one from which every other spider
-   must inherit from (either the ones that come bundled with Scrapy, or the ones
+   must inherit (including spiders that come bundled with Scrapy, as well as spiders
    that you write yourself). It doesn't provide any special functionality. It just
    provides a default :meth:`start_requests` implementation which sends requests from
    the :attr:`start_urls` spider attribute and calls the spider's method ``parse`` 

From b2fd9bdb63c45024872813587f61007c5ae06396 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sun, 12 Jul 2015 16:38:18 +0000
Subject: [PATCH 0448/2974] lazy-load s3 boto

---
 scrapy/__init__.py                    |  7 ----
 scrapy/core/downloader/handlers/s3.py | 47 +++++++++++++++------------
 scrapy/pipelines/files.py             |  8 +++--
 tests/test_downloader_handlers.py     |  8 ++++-
 4 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 10ba9544f..c0477f509 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -31,13 +31,6 @@ del _monkeypatches
 optional_features = set()
 # TODO: backwards compatibility, remove for Scrapy 0.20
 optional_features.add('ssl')
-try:
-    import boto
-    del boto
-except ImportError:
-    pass
-else:
-    optional_features.add('boto')
 
 from twisted import version as _txv
 twisted_version = (_txv.major, _txv.minor, _txv.micro)
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index f1e2c7767..f890300c4 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,39 +1,44 @@
 from urlparse import unquote
 
-from scrapy import optional_features
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
 from .http import HTTPDownloadHandler
 
-try:
-    from boto.s3.connection import S3Connection
-except ImportError:
-    S3Connection = object
 
-class _v19_S3Connection(S3Connection):
-    """A dummy S3Connection wrapper that doesn't do any syncronous download"""
-    def _mexe(self, method, bucket, key, headers, *args, **kwargs):
-        return headers
+def get_s3_connection():
+    try:
+        from boto.s3.connection import S3Connection
+    except ImportError:
+        return None
 
-class _v20_S3Connection(S3Connection):
-    """A dummy S3Connection wrapper that doesn't do any syncronous download"""
-    def _mexe(self, http_request, *args, **kwargs):
-        http_request.authorize(connection=self)
-        return http_request.headers
+    class _v19_S3Connection(S3Connection):
+        """A dummy S3Connection wrapper that doesn't do any synchronous download"""
+        def _mexe(self, method, bucket, key, headers, *args, **kwargs):
+            return headers
 
-try:
-    import boto.auth
-except ImportError:
-    _S3Connection = _v19_S3Connection
-else:
-    _S3Connection = _v20_S3Connection
+    class _v20_S3Connection(S3Connection):
+        """A dummy S3Connection wrapper that doesn't do any synchronous download"""
+        def _mexe(self, http_request, *args, **kwargs):
+            http_request.authorize(connection=self)
+            return http_request.headers
+
+    try:
+        import boto.auth
+    except ImportError:
+        _S3Connection = _v19_S3Connection
+    else:
+        _S3Connection = _v20_S3Connection
+
+    return _S3Connection
 
 
 class S3DownloadHandler(object):
 
     def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
             httpdownloadhandler=HTTPDownloadHandler):
-        if 'boto' not in optional_features:
+
+        _S3Connection = get_s3_connection()
+        if _S3Connection is None:
             raise NotConfigured("missing boto library")
 
         if not aws_access_key_id:
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index a449793c9..308d2f3c1 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -84,6 +84,11 @@ class S3FilesStore(object):
     }
 
     def __init__(self, uri):
+        try:
+            from boto.s3.connection import S3Connection
+            self.S3Connection = S3Connection
+        except ImportError:
+            raise NotConfigured("missing boto library")
         assert uri.startswith('s3://')
         self.bucket, self.prefix = uri[5:].split('/', 1)
 
@@ -98,10 +103,9 @@ class S3FilesStore(object):
         return self._get_boto_key(path).addCallback(_onsuccess)
 
     def _get_boto_bucket(self):
-        from boto.s3.connection import S3Connection
         # disable ssl (is_secure=False) because of this python bug:
         # http://bugs.python.org/issue5103
-        c = S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False)
+        c = self.S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False)
         return c.get_bucket(self.bucket, validate=False)
 
     def _get_boto_key(self, path):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index c814de307..131f6edb7 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -395,7 +395,13 @@ class HttpDownloadHandlerMock(object):
         return request
 
 class S3TestCase(unittest.TestCase):
-    skip = 'boto' not in optional_features and 'missing boto library'
+    download_handler_cls = S3DownloadHandler
+    try:
+        # can't instance without settings, but ignore that
+        download_handler_cls({})
+    except NotConfigured:
+        skip = 'missing boto library'
+    except KeyError: pass
 
     # test use same example keys than amazon developer guide
     # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf

From 5f02ef82e8560242eb34b336f385addfdef3211d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 31 Jul 2015 17:27:53 -0300
Subject: [PATCH 0449/2974] PY3 port http cookies handling

---
 scrapy/http/cookies.py                     | 26 +++++++++++++-----
 tests/py3-ignores.txt                      |  3 ---
 tests/test_downloadermiddleware_cookies.py | 31 +++++++++++-----------
 tests/test_http_cookies.py                 | 20 +++++++++-----
 4 files changed, 49 insertions(+), 31 deletions(-)

diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index b1eb767cc..740f21d24 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -1,6 +1,9 @@
 import time
-from cookielib import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
+from six.moves.http_cookiejar import (
+    CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
+)
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_native_str
 
 
 class CookieJar(object):
@@ -97,6 +100,7 @@ def potential_domain_matches(domain):
         pass
     return matches + ['.' + d for d in matches]
 
+
 class _DummyLock(object):
     def acquire(self):
         pass
@@ -133,6 +137,11 @@ class WrappedRequest(object):
         """
         return self.request.meta.get('is_unverifiable', False)
 
+    # python3 uses request.unverifiable
+    @property
+    def unverifiable(self):
+        return self.is_unverifiable()
+
     def get_origin_req_host(self):
         return urlparse_cached(self.request).hostname
 
@@ -140,14 +149,16 @@ class WrappedRequest(object):
         return name in self.request.headers
 
     def get_header(self, name, default=None):
-        return self.request.headers.get(name, default)
+        return to_native_str(self.request.headers.get(name, default))
 
     def header_items(self):
-        return self.request.headers.items()
+        return [
+            (to_native_str(k), [to_native_str(x) for x in v])
+            for k, v in self.request.headers.items()
+        ]
 
     def add_unredirected_header(self, name, value):
         self.request.headers.appendlist(name, value)
-        #print 'add_unredirected_header', self.request.headers
 
 
 class WrappedResponse(object):
@@ -158,5 +169,8 @@ class WrappedResponse(object):
     def info(self):
         return self
 
-    def getheaders(self, name):
-        return self.response.headers.getlist(name)
+    # python3 cookiejars calls get_all
+    def get_all(self, name, default=None):
+        return [to_native_str(v) for v in self.response.headers.getlist(name)]
+    # python2 cookiejars calls getheaders
+    getheaders = get_all
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 0d4d397a3..469d2c5e1 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -11,7 +11,6 @@ tests/test_crawl.py
 tests/test_crawler.py
 tests/test_downloader_handlers.py
 tests/test_downloadermiddleware_ajaxcrawlable.py
-tests/test_downloadermiddleware_cookies.py
 tests/test_downloadermiddleware_defaultheaders.py
 tests/test_downloadermiddleware_downloadtimeout.py
 tests/test_downloadermiddleware_httpauth.py
@@ -24,7 +23,6 @@ tests/test_downloadermiddleware_retry.py
 tests/test_downloadermiddleware_stats.py
 tests/test_downloadermiddleware_useragent.py
 tests/test_engine.py
-tests/test_http_cookies.py
 tests/test_logformatter.py
 tests/test_mail.py
 tests/test_pipeline_files.py
@@ -51,7 +49,6 @@ scrapy/xlib/tx/endpoints.py
 scrapy/xlib/tx/client.py
 scrapy/xlib/tx/_newclient.py
 scrapy/xlib/tx/__init__.py
-scrapy/http/cookies.py
 scrapy/core/downloader/handlers/s3.py
 scrapy/core/downloader/handlers/http11.py
 scrapy/core/downloader/handlers/http.py
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 996b8c388..6174f8c3f 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -9,7 +9,7 @@ from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
 class CookiesMiddlewareTest(TestCase):
 
     def assertCookieValEqual(self, first, second, msg=None):
-        cookievaleq = lambda cv: re.split(';\s*', cv)
+        cookievaleq = lambda cv: re.split(';\s*', cv.decode('latin1'))
         return self.assertEqual(
             sorted(cookievaleq(first)),
             sorted(cookievaleq(second)), msg)
@@ -34,7 +34,7 @@ class CookiesMiddlewareTest(TestCase):
 
         req2 = Request('http://scrapytest.org/sub1/')
         assert self.mw.process_request(req2, self.spider) is None
-        self.assertEquals(req2.headers.get('Cookie'), "C1=value1")
+        self.assertEquals(req2.headers.get('Cookie'), b"C1=value1")
 
     def test_dont_merge_cookies(self):
         # merge some cookies into jar
@@ -55,12 +55,12 @@ class CookiesMiddlewareTest(TestCase):
         # check that cookies are merged back
         req = Request('http://scrapytest.org/mergeme')
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEquals(req.headers.get('Cookie'), 'C1=value1')
+        self.assertEquals(req.headers.get('Cookie'), b'C1=value1')
 
         # check that cookies are merged when dont_merge_cookies is passed as 0
         req = Request('http://scrapytest.org/mergeme', meta={'dont_merge_cookies': 0})
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEquals(req.headers.get('Cookie'), 'C1=value1')
+        self.assertEquals(req.headers.get('Cookie'), b'C1=value1')
 
     def test_complex_cookies(self):
         # merge some cookies into jar
@@ -76,12 +76,12 @@ class CookiesMiddlewareTest(TestCase):
         # embed C1 and C3 for scrapytest.org/foo
         req = Request('http://scrapytest.org/foo')
         self.mw.process_request(req, self.spider)
-        assert req.headers.get('Cookie') in ('C1=value1; C3=value3', 'C3=value3; C1=value1')
+        assert req.headers.get('Cookie') in (b'C1=value1; C3=value3', b'C3=value3; C1=value1')
 
         # embed C2 for scrapytest.org/bar
         req = Request('http://scrapytest.org/bar')
         self.mw.process_request(req, self.spider)
-        self.assertEquals(req.headers.get('Cookie'), 'C2=value2')
+        self.assertEquals(req.headers.get('Cookie'), b'C2=value2')
 
         # embed nothing for scrapytest.org/baz
         req = Request('http://scrapytest.org/baz')
@@ -91,7 +91,7 @@ class CookiesMiddlewareTest(TestCase):
     def test_merge_request_cookies(self):
         req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'})
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEquals(req.headers.get('Cookie'), 'galleta=salada')
+        self.assertEquals(req.headers.get('Cookie'), b'galleta=salada')
 
         headers = {'Set-Cookie': 'C1=value1; path=/'}
         res = Response('http://scrapytest.org/', headers=headers)
@@ -100,12 +100,12 @@ class CookiesMiddlewareTest(TestCase):
         req2 = Request('http://scrapytest.org/sub1/')
         assert self.mw.process_request(req2, self.spider) is None
 
-        self.assertCookieValEqual(req2.headers.get('Cookie'), "C1=value1; galleta=salada")
+        self.assertCookieValEqual(req2.headers.get('Cookie'), b"C1=value1; galleta=salada")
 
     def test_cookiejar_key(self):
         req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}, meta={'cookiejar': "store1"})
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEquals(req.headers.get('Cookie'), 'galleta=salada')
+        self.assertEquals(req.headers.get('Cookie'), b'galleta=salada')
 
         headers = {'Set-Cookie': 'C1=value1; path=/'}
         res = Response('http://scrapytest.org/', headers=headers, request=req)
@@ -113,11 +113,11 @@ class CookiesMiddlewareTest(TestCase):
 
         req2 = Request('http://scrapytest.org/', meta=res.meta)
         assert self.mw.process_request(req2, self.spider) is None
-        self.assertCookieValEqual(req2.headers.get('Cookie'),'C1=value1; galleta=salada')
+        self.assertCookieValEqual(req2.headers.get('Cookie'), b'C1=value1; galleta=salada')
 
         req3 = Request('http://scrapytest.org/', cookies={'galleta': 'dulce'}, meta={'cookiejar': "store2"})
         assert self.mw.process_request(req3, self.spider) is None
-        self.assertEquals(req3.headers.get('Cookie'), 'galleta=dulce')
+        self.assertEquals(req3.headers.get('Cookie'), b'galleta=dulce')
 
         headers = {'Set-Cookie': 'C2=value2; path=/'}
         res2 = Response('http://scrapytest.org/', headers=headers, request=req3)
@@ -125,7 +125,7 @@ class CookiesMiddlewareTest(TestCase):
 
         req4 = Request('http://scrapytest.org/', meta=res2.meta)
         assert self.mw.process_request(req4, self.spider) is None
-        self.assertCookieValEqual(req4.headers.get('Cookie'), 'C2=value2; galleta=dulce')
+        self.assertCookieValEqual(req4.headers.get('Cookie'), b'C2=value2; galleta=dulce')
 
         #cookies from hosts with port
         req5_1 = Request('http://scrapytest.org:1104/')
@@ -137,11 +137,11 @@ class CookiesMiddlewareTest(TestCase):
 
         req5_2 = Request('http://scrapytest.org:1104/some-redirected-path')
         assert self.mw.process_request(req5_2, self.spider) is None
-        self.assertEquals(req5_2.headers.get('Cookie'), 'C1=value1')
+        self.assertEquals(req5_2.headers.get('Cookie'), b'C1=value1')
 
         req5_3 = Request('http://scrapytest.org/some-redirected-path')
         assert self.mw.process_request(req5_3, self.spider) is None
-        self.assertEquals(req5_3.headers.get('Cookie'), 'C1=value1')
+        self.assertEquals(req5_3.headers.get('Cookie'), b'C1=value1')
 
         #skip cookie retrieval for not http request
         req6 = Request('file:///scrapy/sometempfile')
@@ -152,5 +152,4 @@ class CookiesMiddlewareTest(TestCase):
         request = Request("http://example-host/", cookies={'currencyCookie': 'USD'})
         assert self.mw.process_request(request, self.spider) is None
         self.assertIn('Cookie', request.headers)
-        self.assertIn('currencyCookie', request.headers['Cookie'])
-
+        self.assertEqual(b'currencyCookie=USD', request.headers['Cookie'])
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index 3d6993491..d529f609b 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -8,8 +8,8 @@ from scrapy.http.cookies import WrappedRequest, WrappedResponse
 class WrappedRequestTest(TestCase):
 
     def setUp(self):
-        self.request = Request("http://www.example.com/page.html", \
-            headers={"Content-Type": "text/html"})
+        self.request = Request("http://www.example.com/page.html",
+                               headers={"Content-Type": "text/html"})
         self.wrapped = WrappedRequest(self.request)
 
     def test_get_full_url(self):
@@ -23,10 +23,12 @@ class WrappedRequestTest(TestCase):
 
     def test_is_unverifiable(self):
         self.assertFalse(self.wrapped.is_unverifiable())
+        self.assertFalse(self.wrapped.unverifiable)
 
     def test_is_unverifiable2(self):
         self.request.meta['is_unverifiable'] = True
         self.assertTrue(self.wrapped.is_unverifiable())
+        self.assertTrue(self.wrapped.unverifiable)
 
     def test_get_origin_req_host(self):
         self.assertEqual(self.wrapped.get_origin_req_host(), 'www.example.com')
@@ -40,17 +42,19 @@ class WrappedRequestTest(TestCase):
         self.assertEqual(self.wrapped.get_header('xxxxx', 'def'), 'def')
 
     def test_header_items(self):
-        self.assertEqual(self.wrapped.header_items(), [('Content-Type', ['text/html'])])
+        self.assertEqual(self.wrapped.header_items(),
+                         [('Content-Type', ['text/html'])])
 
     def test_add_unredirected_header(self):
         self.wrapped.add_unredirected_header('hello', 'world')
-        self.assertEqual(self.request.headers['hello'], 'world')
+        self.assertEqual(self.request.headers['hello'], b'world')
+
 
 class WrappedResponseTest(TestCase):
 
     def setUp(self):
-        self.response = Response("http://www.example.com/page.html", 
-            headers={"Content-TYpe": "text/html"})
+        self.response = Response("http://www.example.com/page.html",
+                                 headers={"Content-TYpe": "text/html"})
         self.wrapped = WrappedResponse(self.response)
 
     def test_info(self):
@@ -58,3 +62,7 @@ class WrappedResponseTest(TestCase):
 
     def test_getheaders(self):
         self.assertEqual(self.wrapped.getheaders('content-type'), ['text/html'])
+
+    def test_get_all(self):
+        # get_all result must be native string
+        self.assertEqual(self.wrapped.get_all('content-type'), ['text/html'])

From dba7e39f61cbe2c22d3c9064f32f6e36d74f14b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 3 Aug 2015 10:53:40 -0300
Subject: [PATCH 0450/2974] Do not break cookie parsing on non-utf8 headers

---
 scrapy/http/cookies.py                     |  9 ++++++---
 tests/test_downloadermiddleware_cookies.py | 18 +++++++++++++++---
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 740f21d24..e92c3fe73 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -149,11 +149,13 @@ class WrappedRequest(object):
         return name in self.request.headers
 
     def get_header(self, name, default=None):
-        return to_native_str(self.request.headers.get(name, default))
+        return to_native_str(self.request.headers.get(name, default),
+                             errors='replace')
 
     def header_items(self):
         return [
-            (to_native_str(k), [to_native_str(x) for x in v])
+            (to_native_str(k, errors='replace'),
+             [to_native_str(x, errors='replace') for x in v])
             for k, v in self.request.headers.items()
         ]
 
@@ -171,6 +173,7 @@ class WrappedResponse(object):
 
     # python3 cookiejars calls get_all
     def get_all(self, name, default=None):
-        return [to_native_str(v) for v in self.response.headers.getlist(name)]
+        return [to_native_str(v, errors='replace')
+                for v in self.response.headers.getlist(name)]
     # python2 cookiejars calls getheaders
     getheaders = get_all
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 6174f8c3f..63be0beb8 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -22,20 +22,32 @@ class CookiesMiddlewareTest(TestCase):
         del self.mw
 
     def test_basic(self):
-        headers = {'Set-Cookie': 'C1=value1; path=/'}
         req = Request('http://scrapytest.org/')
         assert self.mw.process_request(req, self.spider) is None
         assert 'Cookie' not in req.headers
 
+        headers = {'Set-Cookie': 'C1=value1; path=/'}
         res = Response('http://scrapytest.org/', headers=headers)
         assert self.mw.process_response(req, res, self.spider) is res
 
-        #assert res.cookies
-
         req2 = Request('http://scrapytest.org/sub1/')
         assert self.mw.process_request(req2, self.spider) is None
         self.assertEquals(req2.headers.get('Cookie'), b"C1=value1")
 
+    def test_do_not_break_on_non_utf8_header(self):
+        req = Request('http://scrapytest.org/')
+        assert self.mw.process_request(req, self.spider) is None
+        assert 'Cookie' not in req.headers
+
+        headers = {'Set-Cookie': b'C1=in\xa3valid; path=/',
+                   'Other': b'ignore\xa3me'}
+        res = Response('http://scrapytest.org/', headers=headers)
+        assert self.mw.process_response(req, res, self.spider) is res
+
+        req2 = Request('http://scrapytest.org/sub1/')
+        assert self.mw.process_request(req2, self.spider) is None
+        self.assertIn('Cookie', req2.headers)
+
     def test_dont_merge_cookies(self):
         # merge some cookies into jar
         headers = {'Set-Cookie': 'C1=value1; path=/'}

From c6adf648dcfe59b52c0e4663e701a232e78d7bf2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 3 Aug 2015 16:28:29 -0300
Subject: [PATCH 0451/2974] PY3 port COOKIES_DEBUG and add tests

---
 scrapy/downloadermiddlewares/cookies.py    | 15 +++--
 scrapy/mail.py                             |  3 +-
 tests/test_downloadermiddleware_cookies.py | 64 +++++++++++++++++++++-
 3 files changed, 74 insertions(+), 8 deletions(-)

diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 270d621cd..321c0171b 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -6,6 +6,7 @@ from collections import defaultdict
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
+from scrapy.utils.python import to_native_str
 
 logger = logging.getLogger(__name__)
 
@@ -52,18 +53,20 @@ class CookiesMiddleware(object):
 
     def _debug_cookie(self, request, spider):
         if self.debug:
-            cl = request.headers.getlist('Cookie')
+            cl = [to_native_str(c, errors='replace')
+                  for c in request.headers.getlist('Cookie')]
             if cl:
-                msg = "Sending cookies to: %s" % request + os.linesep
-                msg += os.linesep.join("Cookie: %s" % c for c in cl)
+                cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)
+                msg = "Sending cookies to: {}\n{}".format(request, cookies)
                 logger.debug(msg, extra={'spider': spider})
 
     def _debug_set_cookie(self, response, spider):
         if self.debug:
-            cl = response.headers.getlist('Set-Cookie')
+            cl = [to_native_str(c, errors='replace')
+                  for c in response.headers.getlist('Set-Cookie')]
             if cl:
-                msg = "Received cookies from: %s" % response + os.linesep
-                msg += os.linesep.join("Set-Cookie: %s" % c for c in cl)
+                cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)
+                msg = "Received cookies from: {}\n{}".format(response, cookies)
                 logger.debug(msg, extra={'spider': spider})
 
     def _format_cookie(self, cookie):
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 2b4c57980..ad8ecbe13 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -20,7 +20,6 @@ else:
     from email import encoders as Encoders
 
 from twisted.internet import defer, reactor, ssl
-from twisted.mail.smtp import ESMTPSenderFactory
 
 logger = logging.getLogger(__name__)
 
@@ -102,6 +101,8 @@ class MailSender(object):
                       'mailattachs': nattachs, 'mailerr': errstr})
 
     def _sendmail(self, to_addrs, msg):
+        # Import twisted.mail here because it is not available in python3
+        from twisted.mail.smtp import ESMTPSenderFactory
         msg = StringIO(msg)
         d = defer.Deferred()
         factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 63be0beb8..66d9faa79 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -1,8 +1,12 @@
-from unittest import TestCase
 import re
+import logging
+from unittest import TestCase
+from testfixtures import LogCapture
 
 from scrapy.http import Response, Request
 from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
+from scrapy.exceptions import NotConfigured
 from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
 
 
@@ -34,6 +38,64 @@ class CookiesMiddlewareTest(TestCase):
         assert self.mw.process_request(req2, self.spider) is None
         self.assertEquals(req2.headers.get('Cookie'), b"C1=value1")
 
+    def test_setting_false_cookies_enabled(self):
+        self.assertRaises(
+            NotConfigured,
+            CookiesMiddleware.from_crawler,
+            get_crawler(settings_dict={'COOKIES_ENABLED': False})
+        )
+
+    def test_setting_default_cookies_enabled(self):
+        self.assertIsInstance(
+            CookiesMiddleware.from_crawler(get_crawler()),
+            CookiesMiddleware
+        )
+
+    def test_setting_true_cookies_enabled(self):
+        self.assertIsInstance(
+            CookiesMiddleware.from_crawler(
+                get_crawler(settings_dict={'COOKIES_ENABLED': True})
+            ),
+            CookiesMiddleware
+        )
+
+    def test_setting_enabled_cookies_debug(self):
+        crawler = get_crawler(settings_dict={'COOKIES_DEBUG': True})
+        mw = CookiesMiddleware.from_crawler(crawler)
+        with LogCapture('scrapy.downloadermiddlewares.cookies',
+                        level=logging.DEBUG) as l:
+            req = Request('http://scrapytest.org/')
+            res = Response('http://scrapytest.org/',
+                           headers={'Set-Cookie': 'C1=value1; path=/'})
+            mw.process_response(req, res, crawler.spider)
+            req2 = Request('http://scrapytest.org/sub1/')
+            mw.process_request(req2, crawler.spider)
+
+            l.check(
+                ('scrapy.downloadermiddlewares.cookies',
+                 'DEBUG',
+                 'Received cookies from: <200 http://scrapytest.org/>\n'
+                 'Set-Cookie: C1=value1; path=/\n'),
+                ('scrapy.downloadermiddlewares.cookies',
+                 'DEBUG',
+                 'Sending cookies to: <GET http://scrapytest.org/sub1/>\n'
+                 'Cookie: C1=value1\n'),
+            )
+
+    def test_setting_disabled_cookies_debug(self):
+        crawler = get_crawler(settings_dict={'COOKIES_DEBUG': False})
+        mw = CookiesMiddleware.from_crawler(crawler)
+        with LogCapture('scrapy.downloadermiddlewares.cookies',
+                        level=logging.DEBUG) as l:
+            req = Request('http://scrapytest.org/')
+            res = Response('http://scrapytest.org/',
+                           headers={'Set-Cookie': 'C1=value1; path=/'})
+            mw.process_response(req, res, crawler.spider)
+            req2 = Request('http://scrapytest.org/sub1/')
+            mw.process_request(req2, crawler.spider)
+
+            l.check()
+
     def test_do_not_break_on_non_utf8_header(self):
         req = Request('http://scrapytest.org/')
         assert self.mw.process_request(req, self.spider) is None

From f4fc05c2a1a676f711650d1661f7f5f78a27b501 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 3 Aug 2015 17:18:08 -0300
Subject: [PATCH 0452/2974] Do not propagate cookie log messages in tests so
 TopLevelFormatter does not rewrite them

---
 tests/test_downloadermiddleware_cookies.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 66d9faa79..26d9794b6 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -63,6 +63,7 @@ class CookiesMiddlewareTest(TestCase):
         crawler = get_crawler(settings_dict={'COOKIES_DEBUG': True})
         mw = CookiesMiddleware.from_crawler(crawler)
         with LogCapture('scrapy.downloadermiddlewares.cookies',
+                        propagate=False,
                         level=logging.DEBUG) as l:
             req = Request('http://scrapytest.org/')
             res = Response('http://scrapytest.org/',
@@ -86,6 +87,7 @@ class CookiesMiddlewareTest(TestCase):
         crawler = get_crawler(settings_dict={'COOKIES_DEBUG': False})
         mw = CookiesMiddleware.from_crawler(crawler)
         with LogCapture('scrapy.downloadermiddlewares.cookies',
+                        propagate=False,
                         level=logging.DEBUG) as l:
             req = Request('http://scrapytest.org/')
             res = Response('http://scrapytest.org/',

From 91991e0bdcb9342813523bf393c3ed61c3a12def Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 4 Aug 2015 02:17:46 +0500
Subject: [PATCH 0453/2974] PY port scrapy.utils.reqser

---
 scrapy/utils/reqser.py     | 21 +++++++++++++++------
 tests/py3-ignores.txt      |  1 -
 tests/test_utils_reqser.py | 14 +++++++++-----
 3 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index d02585a32..7e1e99e48 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -1,8 +1,11 @@
 """
 Helper functions for serializing (and deserializing) requests.
 """
+import six
 
 from scrapy.http import Request
+from scrapy.utils.python import to_unicode, to_native_str
+
 
 def request_to_dict(request, spider=None):
     """Convert Request object to a dict.
@@ -17,7 +20,7 @@ def request_to_dict(request, spider=None):
     if callable(eb):
         eb = _find_method(spider, eb)
     d = {
-        'url': request.url.decode('ascii'), # urls should be safe (safe_string_url)
+        'url': to_unicode(request.url),  # urls should be safe (safe_string_url)
         'callback': cb,
         'errback': eb,
         'method': request.method,
@@ -45,7 +48,7 @@ def request_from_dict(d, spider=None):
     if eb and spider:
         eb = _get_method(spider, eb)
     return Request(
-        url=d['url'].encode('ascii'),
+        url=to_native_str(d['url']),
         callback=cb,
         errback=eb,
         method=d['method'],
@@ -59,10 +62,16 @@ def request_from_dict(d, spider=None):
 
 
 def _find_method(obj, func):
-    if obj and hasattr(func, 'im_self') and func.im_self is obj:
-        return func.im_func.__name__
-    else:
-        raise ValueError("Function %s is not a method of: %s" % (func, obj))
+    if obj:
+        try:
+            func_self = six.get_method_self(func)
+        except AttributeError:  # func has no __self__
+            pass
+        else:
+            if func_self is obj:
+                return six.get_method_function(func).__name__
+    raise ValueError("Function %s is not a method of: %s" % (func, obj))
+
 
 def _get_method(obj, name):
     name = str(name)
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 469d2c5e1..038f715a6 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -39,7 +39,6 @@ tests/test_spider.py
 tests/test_stats.py
 tests/test_utils_iterators.py
 tests/test_utils_log.py
-tests/test_utils_reqser.py
 tests/test_utils_template.py
 tests/test_webclient.py
 
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 9139c0ad0..40c44f7d9 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -1,9 +1,11 @@
+# -*- coding: utf-8 -*-
 import unittest
 
 from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.utils.reqser import request_to_dict, request_from_dict
 
+
 class RequestSerializationTest(unittest.TestCase):
 
     def setUp(self):
@@ -20,18 +22,18 @@ class RequestSerializationTest(unittest.TestCase):
             method="POST",
             body="some body",
             headers={'content-encoding': 'text/html; charset=latin-1'},
-            cookies={'currency': 'usd'},
+            cookies={'currency': u'руб'},
             encoding='latin-1',
             priority=20,
             meta={'a': 'b'})
         self._assert_serializes_ok(r)
 
     def test_latin1_body(self):
-        r = Request("http://www.example.com", body="\xa3")
+        r = Request("http://www.example.com", body=b"\xa3")
         self._assert_serializes_ok(r)
 
     def test_utf8_body(self):
-        r = Request("http://www.example.com", body="\xc2\xa3")
+        r = Request("http://www.example.com", body=b"\xc2\xa3")
         self._assert_serializes_ok(r)
 
     def _assert_serializes_ok(self, request, spider=None):
@@ -53,8 +55,8 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertEqual(r1.dont_filter, r2.dont_filter)
 
     def test_callback_serialization(self):
-        r = Request("http://www.example.com", callback=self.spider.parse_item, \
-            errback=self.spider.handle_error)
+        r = Request("http://www.example.com", callback=self.spider.parse_item,
+                    errback=self.spider.handle_error)
         self._assert_serializes_ok(r, spider=self.spider)
 
     def test_unserializable_callback1(self):
@@ -69,7 +71,9 @@ class RequestSerializationTest(unittest.TestCase):
 
 class TestSpider(Spider):
     name = 'test'
+
     def parse_item(self, response):
         pass
+
     def handle_error(self, failure):
         pass

From ea4199f8e17a6b019e8b9da45791d1af33f978df Mon Sep 17 00:00:00 2001
From: Mikhail Lyundin <mlyundin@gmail.com>
Date: Wed, 5 Aug 2015 22:42:53 +0300
Subject: [PATCH 0454/2974] Minor method name fix

---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 73b4cd034..e9df9c2d2 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -290,7 +290,7 @@ More importantly ``response`` has a ``selector`` attribute which is an instance
 :class:`~scrapy.selector.Selector` class, instantiated with this particular ``response``.
 You can run queries on ``response`` by calling ``response.selector.xpath()`` or
 ``response.selector.css()``. There are also some convenience shortcuts like ``response.xpath()``
-or ``response.xml()`` which map directly to ``response.selector.xpath()`` and
+or ``response.css()`` which map directly to ``response.selector.xpath()`` and
 ``response.selector.css()``.
 
 

From c5ffa1a59f87a806319f40a436010428812bbf49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 5 Aug 2015 20:33:25 -0300
Subject: [PATCH 0455/2974] Twisted 15.3.0 does not raises expected exception
 serializing lambda functions

---
 scrapy/_monkeypatches.py | 11 +++++++++++
 tests/test_squeues.py    |  4 ++++
 2 files changed, 15 insertions(+)

diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index e0ae45b64..782891326 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -1,4 +1,5 @@
 import sys
+from six.moves import copyreg
 
 if sys.version_info[0] == 2:
     from urlparse import urlparse
@@ -12,3 +13,13 @@ if sys.version_info[0] == 2:
     if urlparse('s3://bucket/key?key=value').query != 'key=value':
         from urlparse import uses_query
         uses_query.append('s3')
+
+
+# Undo what Twisted's perspective broker adds to pickle register
+# to prevent bugs like Twisted#7989 while serializing requests
+import twisted.persisted.styles  # NOQA
+# Remove only entries with twisted serializers for non-twisted types.
+for k, v in frozenset(copyreg.dispatch_table.items()):
+    if not getattr(k, '__module__', '').startswith('twisted') \
+            and getattr(v, '__module__', '').startswith('twisted'):
+        copyreg.dispatch_table.pop(k)
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index b3e9a2208..f2d7be67d 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -31,6 +31,8 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
         self.assertEqual(q.pop(), {'a': 'dict'})
 
     def test_nonserializable_object(self):
+        # Trigger Twisted bug #7989
+        import twisted.persisted.styles  # NOQA
         q = self.queue()
         self.assertRaises(ValueError, q.push, lambda x: x)
 
@@ -109,6 +111,8 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
         self.assertEqual(q.pop(), 'a')
 
     def test_nonserializable_object(self):
+        # Trigger Twisted bug #7989
+        import twisted.persisted.styles  # NOQA
         q = self.queue()
         self.assertRaises(ValueError, q.push, lambda x: x)
 

From 57fafc702dec66544d3d96b9d3d26c0ded725e77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 6 Aug 2015 18:01:51 -0300
Subject: [PATCH 0456/2974] Add 1.0.2 release notes

---
 docs/news.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 1a2f8dc98..21612fcc7 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,16 @@
 Release notes
 =============
 
+1.0.2 (2015-08-06)
+------------------
+
+- Twisted 15.3.0 does not raises PicklingError serializing lambda functions (:commit:`b04dd7d`)
+- Minor method name fix (:commit:`6f85c7f`)
+- minor: scrapy.Spider grammar and clarity (:commit:`9c9d2e0`)
+- Put a blurb about support channels in CONTRIBUTING (:commit:`c63882b`)
+- Fixed typos (:commit:`a9ae7b0`)
+- Fix doc reference. (:commit:`7c8a4fe`)
+
 1.0.1 (2015-07-01)
 ------------------
 

From 4d41cc0dc4821da07d467b368528c61ce48a0df2 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Wed, 29 Jul 2015 16:32:11 +0000
Subject: [PATCH 0457/2974] PY3 split requirements into files

---
 requirements-py3.txt       |  6 ++++++
 tests/requirements-py3.txt |  3 +++
 tox.ini                    | 13 +++----------
 3 files changed, 12 insertions(+), 10 deletions(-)
 create mode 100644 requirements-py3.txt
 create mode 100644 tests/requirements-py3.txt

diff --git a/requirements-py3.txt b/requirements-py3.txt
new file mode 100644
index 000000000..81669da39
--- /dev/null
+++ b/requirements-py3.txt
@@ -0,0 +1,6 @@
+Twisted >= 15.1.0
+lxml>=3.2.4
+pyOpenSSL>=0.13.1
+cssselect>=0.9
+queuelib>=1.1.1
+w3lib>=1.8.0
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
new file mode 100644
index 000000000..a92fdd4a8
--- /dev/null
+++ b/tests/requirements-py3.txt
@@ -0,0 +1,3 @@
+pytest>=2.6.0
+pytest-twisted
+testfixtures
diff --git a/tox.ini b/tox.ini
index 5c8c8c78d..fe6b5de38 100644
--- a/tox.ini
+++ b/tox.ini
@@ -40,18 +40,11 @@ commands =
 [testenv:py33]
 basepython = python3.3
 deps =
-    Twisted >= 15.1.0
-    lxml>=3.2.4
-    pyOpenSSL>=0.13.1
-    cssselect>=0.9
-    queuelib>=1.1.1
-    w3lib>=1.8.0
+    -rrequirements-py3.txt
+    # Extras
     Pillow
     service_identity
-    # tests requirements
-    pytest>=2.6.0
-    pytest-twisted
-    testfixtures
+    -rtests/requirements-py3.txt
 
 [testenv:py34]
 basepython = python3.4

From 3e6d6c43ac0763adf2cd92efdb4a1dc2ba165440 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Wed, 29 Jul 2015 15:33:52 +0000
Subject: [PATCH 0458/2974] PY3 fix test cmdline

---
 scrapy/cmdline.py              | 6 +++---
 scrapy/utils/testproc.py       | 4 ++--
 tests/py3-ignores.txt          | 2 --
 tests/test_cmdline/__init__.py | 5 +++--
 tests/test_command_version.py  | 4 +++-
 5 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index a619c349a..35050c13d 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -18,10 +18,10 @@ def _iter_command_classes(module_name):
     # TODO: add `name` attribute to commands and and merge this function with
     # scrapy.utils.spider.iter_spider_classes
     for module in walk_modules(module_name):
-        for obj in vars(module).itervalues():
+        for obj in vars(module).values():
             if inspect.isclass(obj) and \
-               issubclass(obj, ScrapyCommand) and \
-               obj.__module__ == module.__name__:
+                    issubclass(obj, ScrapyCommand) and \
+                    obj.__module__ == module.__name__:
                 yield obj
 
 def _get_commands_from_module(module, inproject):
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index adddad093..f268e91ff 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -35,8 +35,8 @@ class TestProcessProtocol(protocol.ProcessProtocol):
 
     def __init__(self):
         self.deferred = defer.Deferred()
-        self.out = ''
-        self.err = ''
+        self.out = b''
+        self.err = b''
         self.exitcode = None
 
     def outReceived(self, data):
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 038f715a6..d0f9e9e91 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,9 +1,7 @@
 tests/test_closespider.py
-tests/test_cmdline/__init__.py
 tests/test_command_fetch.py
 tests/test_command_shell.py
 tests/test_commands.py
-tests/test_command_version.py
 tests/test_exporters.py
 tests/test_linkextractors.py
 tests/test_loader.py
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 00fce2fbc..28ba76827 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -11,10 +11,11 @@ class CmdlineTest(unittest.TestCase):
         self.env['SCRAPY_SETTINGS_MODULE'] = 'tests.test_cmdline.settings'
 
     def _execute(self, *new_args, **kwargs):
+        encoding = getattr(sys.stdout, 'encoding') or 'utf-8'
         args = (sys.executable, '-m', 'scrapy.cmdline') + new_args
         proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs)
-        comm = proc.communicate()
-        return comm[0].strip()
+        comm = proc.communicate()[0].strip()
+        return comm.decode(encoding)
 
     def test_default_settings(self):
         self.assertEqual(self._execute('settings', '--get', 'TEST1'), \
diff --git a/tests/test_command_version.py b/tests/test_command_version.py
index 6f0380d77..420713d87 100644
--- a/tests/test_command_version.py
+++ b/tests/test_command_version.py
@@ -1,3 +1,4 @@
+import sys
 from twisted.trial import unittest
 from twisted.internet import defer
 
@@ -11,5 +12,6 @@ class VersionTest(ProcessTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_output(self):
+        encoding = getattr(sys.stdout, 'encoding') or 'utf-8'
         _, out, _ = yield self.execute([])
-        self.assertEqual(out.strip(), "Scrapy %s" % scrapy.__version__)
+        self.assertEqual(out.strip().decode(encoding), "Scrapy %s" % scrapy.__version__)

From 6e762ce25cb15ed16f10bc218f38133801548604 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Wed, 29 Jul 2015 15:34:27 +0000
Subject: [PATCH 0459/2974] PY3 renames (six types)

---
 scrapy/core/downloader/handlers/s3.py | 2 +-
 scrapy/core/downloader/middleware.py  | 8 ++++----
 scrapy/core/spidermw.py               | 6 ++++--
 scrapy/linkextractors/htmlparser.py   | 2 +-
 tests/test_crawl.py                   | 2 +-
 5 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index f890300c4..311815b70 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,4 +1,4 @@
-from urlparse import unquote
+from six.moves.urllib.parse import unquote
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index dcc588ef2..413a05dd1 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -3,7 +3,7 @@ Downloader Middleware manager
 
 See documentation in docs/topics/downloader-middleware.rst
 """
-
+import six
 from scrapy.http import Request, Response
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.defer import mustbe_deferred
@@ -32,7 +32,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 response = method(request=request, spider=spider)
                 assert response is None or isinstance(response, (Response, Request)), \
                         'Middleware %s.process_request must return None, Response or Request, got %s' % \
-                        (method.im_self.__class__.__name__, response.__class__.__name__)
+                        (six.get_method_self(method).__class__.__name__, response.__class__.__name__)
                 if response:
                     return response
             return download_func(request=request, spider=spider)
@@ -46,7 +46,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 response = method(request=request, response=response, spider=spider)
                 assert isinstance(response, (Response, Request)), \
                     'Middleware %s.process_response must return Response or Request, got %s' % \
-                    (method.im_self.__class__.__name__, type(response))
+                    (six.get_method_self(method).__class__.__name__, type(response))
                 if isinstance(response, Request):
                     return response
             return response
@@ -57,7 +57,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 response = method(request=request, exception=exception, spider=spider)
                 assert response is None or isinstance(response, (Response, Request)), \
                     'Middleware %s.process_exception must return None, Response or Request, got %s' % \
-                    (method.im_self.__class__.__name__, type(response))
+                    (six.get_method_self(method).__class__.__name__, type(response))
                 if response:
                     return response
             return _failure
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index f6bb62afb..c1c5b10fc 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,7 +3,7 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
-
+import six
 from twisted.python.failure import Failure
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.defer import mustbe_deferred
@@ -33,7 +33,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
             self.methods['process_start_requests'].insert(0, mw.process_start_requests)
 
     def scrape_response(self, scrape_func, response, request, spider):
-        fname = lambda f:'%s.%s' % (f.im_self.__class__.__name__, f.im_func.__name__)
+        fname = lambda f:'%s.%s' % (
+                six.get_method_self(f).__class__.__name__,
+                six.get_method_function(f).__name__)
 
         def process_spider_input(response):
             for method in self.methods['process_spider_input']:
diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 14f4970b0..202340f53 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -3,7 +3,7 @@ HTMLParser-based link extractor
 """
 
 import warnings
-from HTMLParser import HTMLParser
+from six.moves.html_parser import HTMLParser
 from six.moves.urllib.parse import urljoin
 
 from w3lib.url import safe_url_string
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index f2ebf9c69..6d21acab0 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -141,7 +141,7 @@ class CrawlTestCase(TestCase):
     def test_unbounded_response(self):
         # Completeness of responses without Content-Length or Transfer-Encoding
         # can not be determined, we treat them as valid but flagged as "partial"
-        from urllib import urlencode
+        from six.moves.urllib.parse import urlencode
         query = urlencode({'raw': '''\
 HTTP/1.1 200 OK
 Server: Apache-Coyote/1.1

From 991197003bdf8e908aebe5ce39ceff353af7e016 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Wed, 29 Jul 2015 17:38:13 +0000
Subject: [PATCH 0460/2974] PY3 fix tests pipelines files

---
 scrapy/pipelines/files.py    | 3 ++-
 tests/test_pipeline_files.py | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 308d2f3c1..a85aad4e7 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -26,6 +26,7 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.misc import md5sum
 from scrapy.utils.log import failure_to_exc_info
+from scrapy.utils.python import to_bytes
 
 logger = logging.getLogger(__name__)
 
@@ -330,7 +331,7 @@ class FilesPipeline(MediaPipeline):
             return self.file_key(url)
         ## end of deprecation warning block
 
-        media_guid = hashlib.sha1(url).hexdigest()  # change to request.url after deprecation
+        media_guid = hashlib.sha1(to_bytes(url)).hexdigest()  # change to request.url after deprecation
         media_ext = os.path.splitext(url)[1]  # change to request.url after deprecation
         return 'full/%s%s' % (media_guid, media_ext)
 
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index ac0438eba..c9977f5ca 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -12,6 +12,7 @@ from scrapy.pipelines.files import FilesPipeline, FSFilesStore
 from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
+from scrapy.utils.python import to_bytes
 
 from tests import mock
 
@@ -103,7 +104,7 @@ class FilesPipelineTestCase(unittest.TestCase):
 
 class DeprecatedFilesPipeline(FilesPipeline):
     def file_key(self, url):
-        media_guid = hashlib.sha1(url).hexdigest()
+        media_guid = hashlib.sha1(to_bytes(url)).hexdigest()
         media_ext = os.path.splitext(url)[1]
         return 'empty/%s%s' % (media_guid, media_ext)
 

From 34eced0ee822ac395fde457984faf4afcc77f713 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Wed, 29 Jul 2015 17:48:27 +0000
Subject: [PATCH 0461/2974] PY3 fix tests pipelines images

---
 scrapy/pipelines/images.py    | 5 +++--
 tests/test_pipeline_images.py | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 8b3bc2222..ff73b44b7 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -15,6 +15,7 @@ except ImportError:
 from PIL import Image
 
 from scrapy.utils.misc import md5sum
+from scrapy.utils.python import to_bytes
 from scrapy.http import Request
 from scrapy.exceptions import DropItem
 #TODO: from scrapy.pipelines.media import MediaPipeline
@@ -138,7 +139,7 @@ class ImagesPipeline(FilesPipeline):
             return self.image_key(url)
         ## end of deprecation warning block
 
-        image_guid = hashlib.sha1(url).hexdigest()  # change to request.url after deprecation
+        image_guid = hashlib.sha1(to_bytes(url)).hexdigest()  # change to request.url after deprecation
         return 'full/%s.jpg' % (image_guid)
 
     def thumb_path(self, request, thumb_id, response=None, info=None):
@@ -163,7 +164,7 @@ class ImagesPipeline(FilesPipeline):
             return self.thumb_key(url, thumb_id)
         ## end of deprecation warning block
 
-        thumb_guid = hashlib.sha1(url).hexdigest()  # change to request.url after deprecation
+        thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest()  # change to request.url after deprecation
         return 'thumbs/%s/%s.jpg' % (thumb_id, thumb_guid)
 
     # deprecated
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 04cec4b8e..f52fb4d3d 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -10,6 +10,7 @@ from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.pipelines.images import ImagesPipeline
+from scrapy.utils.python import to_bytes
 
 skip = False
 try:
@@ -100,11 +101,11 @@ class DeprecatedImagesPipeline(ImagesPipeline):
         return self.image_key(url)
 
     def image_key(self, url):
-        image_guid = hashlib.sha1(url).hexdigest()
+        image_guid = hashlib.sha1(to_bytes(url)).hexdigest()
         return 'empty/%s.jpg' % (image_guid)
 
     def thumb_key(self, url, thumb_id):
-        thumb_guid = hashlib.sha1(url).hexdigest()
+        thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest()
         return 'thumbsup/%s/%s.jpg' % (thumb_id, thumb_guid)
 
 

From 45d441d444ed1d1e2f94739e574ff9f1290cf3dd Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Wed, 29 Jul 2015 16:16:51 +0000
Subject: [PATCH 0462/2974] PY3 fix test loader

---
 tests/py3-ignores.txt      |  1 -
 tests/requirements-py3.txt |  1 +
 tests/test_loader.py       | 13 +++++++------
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index d0f9e9e91..9be3a99a8 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -4,7 +4,6 @@ tests/test_command_shell.py
 tests/test_commands.py
 tests/test_exporters.py
 tests/test_linkextractors.py
-tests/test_loader.py
 tests/test_crawl.py
 tests/test_crawler.py
 tests/test_downloader_handlers.py
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index a92fdd4a8..8f9e22f0b 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,3 +1,4 @@
 pytest>=2.6.0
 pytest-twisted
 testfixtures
+jmespath
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 6e8f7c0de..8cf5e484a 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -1,4 +1,5 @@
 import unittest
+import six
 from functools import partial
 
 from scrapy.loader import ItemLoader
@@ -141,7 +142,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_get_value(self):
         il = NameItemLoader()
-        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), unicode.upper))
+        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), six.text_type.upper))
         self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
         self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
 
@@ -242,7 +243,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_extend_custom_input_processors(self):
         class ChildItemLoader(TestItemLoader):
-            name_in = MapCompose(TestItemLoader.name_in, unicode.swapcase)
+            name_in = MapCompose(TestItemLoader.name_in, six.text_type.swapcase)
 
         il = ChildItemLoader()
         il.add_value('name', u'marta')
@@ -250,7 +251,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_extend_default_input_processors(self):
         class ChildDefaultedItemLoader(DefaultedItemLoader):
-            name_in = MapCompose(DefaultedItemLoader.default_input_processor, unicode.swapcase)
+            name_in = MapCompose(DefaultedItemLoader.default_input_processor, six.text_type.swapcase)
 
         il = ChildDefaultedItemLoader()
         il.add_value('name', u'marta')
@@ -423,7 +424,7 @@ class ProcessorsTest(unittest.TestCase):
         self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
         self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
         self.assertEqual(proc(['hello', 'world']), u'hello world')
-        self.assert_(isinstance(proc(['hello', 'world']), unicode))
+        self.assert_(isinstance(proc(['hello', 'world']), six.text_type))
 
     def test_compose(self):
         proc = Compose(lambda v: v[0], str.upper)
@@ -435,13 +436,13 @@ class ProcessorsTest(unittest.TestCase):
 
     def test_mapcompose(self):
         filter_world = lambda x: None if x == 'world' else x
-        proc = MapCompose(filter_world, unicode.upper)
+        proc = MapCompose(filter_world, six.text_type.upper)
         self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
                          [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
 
 
 class SelectortemLoaderTest(unittest.TestCase):
-    response = HtmlResponse(url="", body="""
+    response = HtmlResponse(url="", encoding='utf-8', body=b"""
     <html>
     <body>
     <div id="id">marta</div>

From 17b5e9fb86b3969884213cde9e5a44647396f560 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Wed, 29 Jul 2015 20:52:25 +0000
Subject: [PATCH 0463/2974] PY3 response bodies as bytes

---
 tests/test_selector_csstranslator.py |  2 +-
 tests/test_selector_lxmldocument.py  |  4 ++--
 tests/test_utils_iterators.py        | 10 +++++-----
 tests/test_utils_reqser.py           |  2 +-
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/test_selector_csstranslator.py b/tests/test_selector_csstranslator.py
index 7ef9003aa..1bc8882f8 100644
--- a/tests/test_selector_csstranslator.py
+++ b/tests/test_selector_csstranslator.py
@@ -9,7 +9,7 @@ from cssselect.parser import SelectorSyntaxError
 from cssselect.xpath import ExpressionError
 
 
-HTMLBODY = '''
+HTMLBODY = b'''
 <html>
 <body>
 <div>
diff --git a/tests/test_selector_lxmldocument.py b/tests/test_selector_lxmldocument.py
index 7dab1d4b1..090cc21bc 100644
--- a/tests/test_selector_lxmldocument.py
+++ b/tests/test_selector_lxmldocument.py
@@ -6,7 +6,7 @@ from scrapy.http import TextResponse, HtmlResponse
 class LxmlDocumentTest(unittest.TestCase):
 
     def test_caching(self):
-        r1 = HtmlResponse('http://www.example.com', body='<html><head></head><body></body></html>')
+        r1 = HtmlResponse('http://www.example.com', body=b'<html><head></head><body></body></html>')
         r2 = r1.copy()
 
         doc1 = LxmlDocument(r1)
@@ -19,7 +19,7 @@ class LxmlDocumentTest(unittest.TestCase):
 
     def test_null_char(self):
         # make sure bodies with null char ('\x00') don't raise a TypeError exception
-        body = 'test problematic \x00 body'
+        body = b'test problematic \x00 body'
         response = TextResponse('http://example.com/catalog/product/blabla-123',
                                 headers={'Content-Type': 'text/plain; charset=utf-8'},
                                 body=body)
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index a7042a6cf..f2780dcf1 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -13,7 +13,7 @@ class XmliterTestCase(unittest.TestCase):
     xmliter = staticmethod(xmliter)
 
     def test_xmliter(self):
-        body = """<?xml version="1.0" encoding="UTF-8"?>\
+        body = b"""<?xml version="1.0" encoding="UTF-8"?>\
             <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="someschmea.xsd">\
               <product id="001">\
                 <type>Type 1</type>\
@@ -40,7 +40,7 @@ class XmliterTestCase(unittest.TestCase):
                          [[u'one'], [u'two']])
 
     def test_xmliter_namespaces(self):
-        body = """\
+        body = b"""\
             <?xml version="1.0" encoding="UTF-8"?>
             <rss version="2.0" xmlns:g="http://base.google.com/ns/1.0">
                 <channel>
@@ -83,7 +83,7 @@ class XmliterTestCase(unittest.TestCase):
         self.assertRaises(StopIteration, next, iter)
 
     def test_xmliter_encoding(self):
-        body = '<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
+        body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
         response = XmlResponse('http://www.example.com', body=body)
         self.assertEqual(
             self.xmliter(response, 'item').next().extract(),
@@ -95,7 +95,7 @@ class LxmlXmliterTestCase(XmliterTestCase):
     xmliter = staticmethod(xmliter_lxml)
 
     def test_xmliter_iterate_namespace(self):
-        body = """\
+        body = b"""\
             <?xml version="1.0" encoding="UTF-8"?>
             <rss version="2.0" xmlns="http://base.google.com/ns/1.0">
                 <channel>
@@ -124,7 +124,7 @@ class LxmlXmliterTestCase(XmliterTestCase):
         self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg'])
 
     def test_xmliter_namespaces_prefix(self):
-        body = """\
+        body = b"""\
         <?xml version="1.0" encoding="UTF-8"?>
         <root>
             <h:table xmlns:h="http://www.w3.org/TR/html4/">
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 40c44f7d9..a62f13e21 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -20,7 +20,7 @@ class RequestSerializationTest(unittest.TestCase):
             callback='parse_item',
             errback='handle_error',
             method="POST",
-            body="some body",
+            body=b"some body",
             headers={'content-encoding': 'text/html; charset=latin-1'},
             cookies={'currency': u'руб'},
             encoding='latin-1',

From 56be610e6e26ec7a17ec16dccc09fa832facb0fa Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 8 Aug 2015 04:54:47 +0500
Subject: [PATCH 0464/2974] TST a test for --profile option

---
 tests/test_cmdline/__init__.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 28ba76827..1e2905e95 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -1,9 +1,18 @@
+import os
 import sys
+import shutil
+import pstats
+import tempfile
 from subprocess import Popen, PIPE
 import unittest
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from io import StringIO
 
 from scrapy.utils.test import get_testenv
 
+
 class CmdlineTest(unittest.TestCase):
 
     def setUp(self):
@@ -30,3 +39,18 @@ class CmdlineTest(unittest.TestCase):
         self.assertEqual(self._execute('settings', '--get', 'TEST1'), \
                          'override')
 
+    def test_profiling(self):
+        path = tempfile.mkdtemp()
+        filename = os.path.join(path, 'res.prof')
+        try:
+            self._execute('version', '--profile', filename)
+            self.assertTrue(os.path.exists(filename))
+            out = StringIO()
+            stats = pstats.Stats(filename, stream=out)
+            stats.print_stats()
+            out.seek(0)
+            stats = out.read()
+            self.assertIn('scrapy/commands/version.py', stats)
+            self.assertIn('tottime', stats)
+        finally:
+            shutil.rmtree(path)

From 93accb7fb346e47feda24b70d4af35ae0ae4f069 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 8 Aug 2015 05:20:48 +0500
Subject: [PATCH 0465/2974] PY3 nicer log messages in FilesPipeline

---
 scrapy/pipelines/files.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index a85aad4e7..db49aff65 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -26,7 +26,7 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.misc import md5sum
 from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.python import to_bytes
+from scrapy.utils.python import to_bytes, to_native_str
 
 logger = logging.getLogger(__name__)
 
@@ -199,7 +199,7 @@ class FilesPipeline(MediaPipeline):
             if age_days > self.EXPIRES:
                 return  # returning None force download
 
-            referer = request.headers.get('Referer')
+            referer = _get_referer(request)
             logger.debug(
                 'File (uptodate): Downloaded %(medianame)s from %(request)s '
                 'referred in <%(referer)s>',
@@ -225,7 +225,7 @@ class FilesPipeline(MediaPipeline):
 
     def media_failed(self, failure, request, info):
         if not isinstance(failure.value, IgnoreRequest):
-            referer = request.headers.get('Referer')
+            referer = _get_referer(request)
             logger.warning(
                 'File (unknown-error): Error downloading %(medianame)s from '
                 '%(request)s referred in <%(referer)s>: %(exception)s',
@@ -237,7 +237,7 @@ class FilesPipeline(MediaPipeline):
         raise FileException
 
     def media_downloaded(self, response, request, info):
-        referer = request.headers.get('Referer')
+        referer = _get_referer(request)
 
         if response.status != 200:
             logger.warning(
@@ -339,3 +339,11 @@ class FilesPipeline(MediaPipeline):
     def file_key(self, url):
         return self.file_path(url)
     file_key._base = True
+
+
+def _get_referer(request):
+    """ Return Referer HTTP header suitable for logging """
+    referrer = request.headers.get('Referer')
+    if referrer is None:
+        return referrer
+    return to_native_str(referrer, errors='replace')

From 5c4666a3d489bd3efa2e188de58721a125a5bfad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 10 Aug 2015 11:17:20 -0300
Subject: [PATCH 0466/2974] Workaround for travis#253

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index d0758fdaf..d714c0ff8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,7 +7,7 @@ env:
  - TOXENV=py33
  - TOXENV=docs
 install:
- - pip install -U tox
+ - pip install -U tox twine wheel
 script: tox
 notifications:
   irc:

From 8d45b3c4810cb5304ba1193b45697a0df1157326 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 10 Aug 2015 14:10:58 -0300
Subject: [PATCH 0467/2974] Do not fail representing non-http requests

---
 scrapy/utils/request.py     | 2 +-
 tests/test_utils_request.py | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index ac415e508..0487d1e1b 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -79,7 +79,7 @@ def request_httprepr(request):
     parsed = urlparse_cached(request)
     path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
     s = to_bytes(request.method) + b" " + to_bytes(path) + b" HTTP/1.1\r\n"
-    s += b"Host: " + to_bytes(parsed.hostname) + b"\r\n"
+    s += b"Host: " + to_bytes(parsed.hostname or b'') + b"\r\n"
     if request.headers:
         s += request.headers.to_string() + b"\r\n"
     s += b"\r\n"
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index d058bba96..e8a4eb3ea 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -71,5 +71,10 @@ class UtilsRequestTest(unittest.TestCase):
         r1 = Request("http://www.example.com", method='POST', headers={"Content-type": b"text/html"}, body=b"Some body")
         self.assertEqual(request_httprepr(r1), b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body')
 
+    def test_request_httprepr_for_non_http_request(self):
+        # the representation is not important but it must not fail.
+        request_httprepr(Request("file:///tmp/foo.txt"))
+        request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
+
 if __name__ == "__main__":
     unittest.main()

From accd28cc71e20a194c7e2497b395cd3b0910c7e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 10 Aug 2015 15:13:25 -0300
Subject: [PATCH 0468/2974] PY3: port scrapy.utils.trackref

---
 scrapy/utils/trackref.py     | 29 +++++++++------
 tests/test_utils_trackref.py | 70 ++++++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+), 10 deletions(-)
 create mode 100644 tests/test_utils_trackref.py

diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index f250aeefa..283b0739e 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -10,15 +10,17 @@ alias to object in that case).
 """
 
 from __future__ import print_function
-import weakref, os, six
-from collections import defaultdict
+import weakref
 from time import time
 from operator import itemgetter
+from collections import defaultdict
+import six
+
 
 NoneType = type(None)
-
 live_refs = defaultdict(weakref.WeakKeyDictionary)
 
+
 class object_ref(object):
     """Inherit from this class (instead of object) to a keep a record of live
     instances"""
@@ -30,27 +32,34 @@ class object_ref(object):
         live_refs[cls][obj] = time()
         return obj
 
+
 def format_live_refs(ignore=NoneType):
-    s = "Live References" + os.linesep + os.linesep
+    s = "Live References\n\n"
     now = time()
-    for cls, wdict in six.iteritems(live_refs):
+    for cls, wdict in sorted(six.iteritems(live_refs),
+                             key=lambda x: x[0].__name__):
         if not wdict:
             continue
         if issubclass(cls, ignore):
             continue
-        oldest = min(wdict.itervalues())
-        s += "%-30s %6d   oldest: %ds ago" % (cls.__name__, len(wdict), \
-            now-oldest) + os.linesep
+        oldest = min(six.itervalues(wdict))
+        s += "%-30s %6d   oldest: %ds ago\n" % (
+            cls.__name__, len(wdict), now - oldest
+        )
     return s
 
+
 def print_live_refs(*a, **kw):
     print(format_live_refs(*a, **kw))
 
+
 def get_oldest(class_name):
     for cls, wdict in six.iteritems(live_refs):
         if cls.__name__ == class_name:
-            if wdict:
-                return min(six.iteritems(wdict), key=itemgetter(1))[0]
+            if not wdict:
+                break
+            return min(six.iteritems(wdict), key=itemgetter(1))[0]
+
 
 def iter_all(class_name):
     for cls, wdict in six.iteritems(live_refs):
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
new file mode 100644
index 000000000..c6072fc0d
--- /dev/null
+++ b/tests/test_utils_trackref.py
@@ -0,0 +1,70 @@
+import six
+import unittest
+from scrapy.utils import trackref
+from tests import mock
+
+
+class Foo(trackref.object_ref):
+    pass
+
+
+class Bar(trackref.object_ref):
+    pass
+
+
+class TrackrefTestCase(unittest.TestCase):
+
+    def setUp(self):
+        trackref.live_refs.clear()
+
+    def test_format_live_refs(self):
+        o1 = Foo()  # NOQA
+        o2 = Bar()  # NOQA
+        o3 = Foo()  # NOQA
+        self.assertEqual(
+            trackref.format_live_refs(),
+            '''\
+Live References
+
+Bar                                 1   oldest: 0s ago
+Foo                                 2   oldest: 0s ago
+''')
+
+        self.assertEqual(
+            trackref.format_live_refs(ignore=Foo),
+            '''\
+Live References
+
+Bar                                 1   oldest: 0s ago
+''')
+
+    @mock.patch('sys.stdout', new_callable=six.StringIO)
+    def test_print_live_refs_empty(self, stdout):
+        trackref.print_live_refs()
+        self.assertEqual(stdout.getvalue(), 'Live References\n\n\n')
+
+    @mock.patch('sys.stdout', new_callable=six.StringIO)
+    def test_print_live_refs_with_objects(self, stdout):
+        o1 = Foo()  # NOQA
+        trackref.print_live_refs()
+        self.assertEqual(stdout.getvalue(), '''\
+Live References
+
+Foo                                 1   oldest: 0s ago\n\n''')
+
+    def test_get_oldest(self):
+        o1 = Foo()  # NOQA
+        o2 = Bar()  # NOQA
+        o3 = Foo()  # NOQA
+        self.assertIs(trackref.get_oldest('Foo'), o1)
+        self.assertIs(trackref.get_oldest('Bar'), o2)
+        self.assertIsNone(trackref.get_oldest('XXX'))
+
+    def test_iter_all(self):
+        o1 = Foo()  # NOQA
+        o2 = Bar()  # NOQA
+        o3 = Foo()  # NOQA
+        self.assertEqual(
+            set(trackref.iter_all('Foo')),
+            {o1, o3},
+        )

From 666ebfa1d97264bc4e6adb78fe4ce1a9ea15cc1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 10 Aug 2015 16:45:27 -0300
Subject: [PATCH 0469/2974] py3: port dictionary itervalues call

---
 scrapy/core/downloader/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 965471851..8beb08159 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -5,6 +5,7 @@ from time import time
 from datetime import datetime
 from collections import deque
 
+import six
 from twisted.internet import reactor, defer, task
 
 from scrapy.utils.defer import mustbe_deferred
@@ -188,7 +189,7 @@ class Downloader(object):
 
     def close(self):
         self._slot_gc_loop.stop()
-        for slot in self.slots.itervalues():
+        for slot in six.itervalues(self.slots):
             slot.close()
 
     def _slot_gc(self, age=60):

From 8c7997083fc1455daeda8d13d8037325f4b74909 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sun, 12 Jul 2015 16:40:51 +0000
Subject: [PATCH 0470/2974] lazy-loading for DownloadHandlers

---
 scrapy/core/downloader/handlers/__init__.py | 49 ++++++++++++++-------
 tests/test_downloader_handlers.py           | 11 ++++-
 2 files changed, 43 insertions(+), 17 deletions(-)

diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index ea0842e62..abf01c905 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -11,8 +11,10 @@ from scrapy import signals
 class DownloadHandlers(object):
 
     def __init__(self, crawler):
-        self._handlers = {}
-        self._notconfigured = {}
+        self._crawler_settings = crawler.settings
+        self._schemes = {} # stores acceptable schemes on instancing
+        self._handlers = {} # stores instanced handlers for schemes
+        self._notconfigured = {} # remembers failed handlers
         handlers = crawler.settings.get('DOWNLOAD_HANDLERS_BASE')
         handlers.update(crawler.settings.get('DOWNLOAD_HANDLERS', {}))
         for scheme, clspath in six.iteritems(handlers):
@@ -20,25 +22,40 @@ class DownloadHandlers(object):
             # component (extension, middleware, etc).
             if clspath is None:
                 continue
-            cls = load_object(clspath)
-            try:
-                dh = cls(crawler.settings)
-            except NotConfigured as ex:
-                self._notconfigured[scheme] = str(ex)
-            else:
-                self._handlers[scheme] = dh
+            self._schemes[scheme] = clspath
 
         crawler.signals.connect(self._close, signals.engine_stopped)
 
+    def _get_handler(self, scheme):
+        """Lazy-load the downloadhandler for a scheme
+        only on the first request for that scheme.
+        """
+        if scheme in self._handlers:
+            return self._handlers[scheme]
+        if scheme in self._notconfigured:
+            return None
+        if scheme not in self._schemes:
+            self._notconfigured[scheme] = \
+                    'no handler available for that scheme'
+            return None
+
+        dhcls = load_object(self._schemes[scheme])
+        try:
+            dh = dhcls(self._crawler_settings)
+        except NotConfigured as ex:
+            self._notconfigured[scheme] = str(ex)
+            return None
+        else:
+            self._handlers[scheme] = dh
+        return self._handlers[scheme]
+
     def download_request(self, request, spider):
         scheme = urlparse_cached(request).scheme
-        try:
-            handler = self._handlers[scheme].download_request
-        except KeyError:
-            msg = self._notconfigured.get(scheme, \
-                    'no handler available for that scheme')
-            raise NotSupported("Unsupported URL scheme '%s': %s" % (scheme, msg))
-        return handler(request, spider)
+        handler = self._get_handler(scheme)
+        if not handler:
+            raise NotSupported("Unsupported URL scheme '%s': %s" %
+                    (scheme, self._notconfigured[scheme]))
+        return handler.download_request(request, spider)
 
     @defer.inlineCallbacks
     def _close(self, *_a, **_kw):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 131f6edb7..e4d957d8e 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -52,6 +52,9 @@ class LoadTestCase(unittest.TestCase):
         handlers = {'scheme': 'tests.test_downloader_handlers.DummyDH'}
         crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
         dh = DownloadHandlers(crawler)
+        self.assertIn('scheme', dh._schemes)
+        for scheme in handlers: # force load handlers
+            dh._get_handler(scheme)
         self.assertIn('scheme', dh._handlers)
         self.assertNotIn('scheme', dh._notconfigured)
 
@@ -59,6 +62,9 @@ class LoadTestCase(unittest.TestCase):
         handlers = {'scheme': 'tests.test_downloader_handlers.OffDH'}
         crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
         dh = DownloadHandlers(crawler)
+        self.assertIn('scheme', dh._schemes)
+        for scheme in handlers: # force load handlers
+            dh._get_handler(scheme)
         self.assertNotIn('scheme', dh._handlers)
         self.assertIn('scheme', dh._notconfigured)
 
@@ -66,8 +72,11 @@ class LoadTestCase(unittest.TestCase):
         handlers = {'scheme': None}
         crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
         dh = DownloadHandlers(crawler)
+        self.assertNotIn('scheme', dh._schemes)
+        for scheme in handlers: # force load handlers
+            dh._get_handler(scheme)
         self.assertNotIn('scheme', dh._handlers)
-        self.assertNotIn('scheme', dh._notconfigured)
+        self.assertIn('scheme', dh._notconfigured)
 
 
 class FileTestCase(unittest.TestCase):

From d3804b3439d72e718cf68ff9fc1798bb9232625c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 10 Aug 2015 16:52:36 -0300
Subject: [PATCH 0471/2974] log errors importing or instanciating handlers

---
 scrapy/core/downloader/handlers/__init__.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index abf01c905..062f674b2 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -1,5 +1,6 @@
 """Download handlers for different schemes"""
 
+import logging
 from twisted.internet import defer
 import six
 from scrapy.exceptions import NotSupported, NotConfigured
@@ -8,6 +9,9 @@ from scrapy.utils.misc import load_object
 from scrapy import signals
 
 
+logger = logging.getLogger(__name__)
+
+
 class DownloadHandlers(object):
 
     def __init__(self, crawler):
@@ -39,12 +43,16 @@ class DownloadHandlers(object):
                     'no handler available for that scheme'
             return None
 
-        dhcls = load_object(self._schemes[scheme])
         try:
+            dhcls = load_object(self._schemes[scheme])
             dh = dhcls(self._crawler_settings)
         except NotConfigured as ex:
             self._notconfigured[scheme] = str(ex)
             return None
+        except Exception as ex:
+            logger.exception()
+            self._notconfigured[scheme] = str(ex)
+            return None
         else:
             self._handlers[scheme] = dh
         return self._handlers[scheme]

From 15ccf79cad46385041fb8c3a7bf697d9d7ee7c55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 10 Aug 2015 18:18:13 -0300
Subject: [PATCH 0472/2974] Log errors importing or initializing download
 handlers

---
 scrapy/core/downloader/handlers/__init__.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 062f674b2..65e5bc21f 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -43,14 +43,16 @@ class DownloadHandlers(object):
                     'no handler available for that scheme'
             return None
 
+        path = self._schemes[scheme]
         try:
-            dhcls = load_object(self._schemes[scheme])
+            dhcls = load_object(path)
             dh = dhcls(self._crawler_settings)
         except NotConfigured as ex:
             self._notconfigured[scheme] = str(ex)
             return None
         except Exception as ex:
-            logger.exception()
+            logger.exception('Loading "{}" for scheme "{}" handler'\
+                             .format(path, scheme))
             self._notconfigured[scheme] = str(ex)
             return None
         else:

From eb44152a585a54c5c2ee27c9eaed32aa2f9b15a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 10 Aug 2015 18:25:11 -0300
Subject: [PATCH 0473/2974] lints

---
 scrapy/core/downloader/handlers/__init__.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 65e5bc21f..0e732cfe7 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -16,9 +16,9 @@ class DownloadHandlers(object):
 
     def __init__(self, crawler):
         self._crawler_settings = crawler.settings
-        self._schemes = {} # stores acceptable schemes on instancing
-        self._handlers = {} # stores instanced handlers for schemes
-        self._notconfigured = {} # remembers failed handlers
+        self._schemes = {}  # stores acceptable schemes on instancing
+        self._handlers = {}  # stores instanced handlers for schemes
+        self._notconfigured = {}  # remembers failed handlers
         handlers = crawler.settings.get('DOWNLOAD_HANDLERS_BASE')
         handlers.update(crawler.settings.get('DOWNLOAD_HANDLERS', {}))
         for scheme, clspath in six.iteritems(handlers):
@@ -39,8 +39,7 @@ class DownloadHandlers(object):
         if scheme in self._notconfigured:
             return None
         if scheme not in self._schemes:
-            self._notconfigured[scheme] = \
-                    'no handler available for that scheme'
+            self._notconfigured[scheme] = 'no handler available for that scheme'
             return None
 
         path = self._schemes[scheme]
@@ -51,7 +50,7 @@ class DownloadHandlers(object):
             self._notconfigured[scheme] = str(ex)
             return None
         except Exception as ex:
-            logger.exception('Loading "{}" for scheme "{}" handler'\
+            logger.exception('Loading "{}" for scheme "{}" handler'
                              .format(path, scheme))
             self._notconfigured[scheme] = str(ex)
             return None
@@ -64,7 +63,7 @@ class DownloadHandlers(object):
         handler = self._get_handler(scheme)
         if not handler:
             raise NotSupported("Unsupported URL scheme '%s': %s" %
-                    (scheme, self._notconfigured[scheme]))
+                               (scheme, self._notconfigured[scheme]))
         return handler.download_request(request, spider)
 
     @defer.inlineCallbacks

From 8738521d0a221e8800909121660746373dc5f823 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 10 Aug 2015 19:35:57 -0300
Subject: [PATCH 0474/2974] Add docstrings

---
 scrapy/utils/trackref.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index 283b0739e..eed14c5a1 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -34,6 +34,7 @@ class object_ref(object):
 
 
 def format_live_refs(ignore=NoneType):
+    """Return a tabular representation of tracked objects"""
     s = "Live References\n\n"
     now = time()
     for cls, wdict in sorted(six.iteritems(live_refs),
@@ -50,10 +51,12 @@ def format_live_refs(ignore=NoneType):
 
 
 def print_live_refs(*a, **kw):
+    """Print tracked objects"""
     print(format_live_refs(*a, **kw))
 
 
 def get_oldest(class_name):
+    """Get the oldest object for a specific class name"""
     for cls, wdict in six.iteritems(live_refs):
         if cls.__name__ == class_name:
             if not wdict:
@@ -62,6 +65,7 @@ def get_oldest(class_name):
 
 
 def iter_all(class_name):
+    """Iterate over all objects of the same class by its class name"""
     for cls, wdict in six.iteritems(live_refs):
         if cls.__name__ == class_name:
             return six.iterkeys(wdict)

From 7717501ab28fa2760da6b1b9be998eb589988821 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Tue, 11 Aug 2015 10:38:31 -0300
Subject: [PATCH 0475/2974] Use log formatting and pass crawler reference

---
 scrapy/core/downloader/handlers/__init__.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 0e732cfe7..6c9514af6 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
 class DownloadHandlers(object):
 
     def __init__(self, crawler):
-        self._crawler_settings = crawler.settings
+        self._crawler = crawler
         self._schemes = {}  # stores acceptable schemes on instancing
         self._handlers = {}  # stores instanced handlers for schemes
         self._notconfigured = {}  # remembers failed handlers
@@ -45,13 +45,14 @@ class DownloadHandlers(object):
         path = self._schemes[scheme]
         try:
             dhcls = load_object(path)
-            dh = dhcls(self._crawler_settings)
+            dh = dhcls(self._crawler.settings)
         except NotConfigured as ex:
             self._notconfigured[scheme] = str(ex)
             return None
         except Exception as ex:
-            logger.exception('Loading "{}" for scheme "{}" handler'
-                             .format(path, scheme))
+            logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"',
+                         {"clspath": path, "scheme": scheme},
+                         exc_info=True,  extra={'crawler': self._crawler})
             self._notconfigured[scheme] = str(ex)
             return None
         else:

From 75d3d033827ef7d0611ff1a0dc8d953ffe6e22c1 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 11 Aug 2015 20:14:22 +0500
Subject: [PATCH 0476/2974] add service_identity to scrapy install_requires

---
 requirements-py3.txt | 1 +
 requirements.txt     | 1 +
 setup.py             | 1 +
 tox.ini              | 2 --
 4 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/requirements-py3.txt b/requirements-py3.txt
index 81669da39..a9a2e3be0 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -4,3 +4,4 @@ pyOpenSSL>=0.13.1
 cssselect>=0.9
 queuelib>=1.1.1
 w3lib>=1.8.0
+service_identity
diff --git a/requirements.txt b/requirements.txt
index 03e5329c0..a05cd3680 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,3 +6,4 @@ w3lib>=1.8.0
 queuelib
 six>=1.5.2
 PyDispatcher>=2.0.5
+service_identity
diff --git a/setup.py b/setup.py
index 6dc1cfd94..7214f0dc6 100644
--- a/setup.py
+++ b/setup.py
@@ -45,5 +45,6 @@ setup(
         'cssselect>=0.9',
         'six>=1.5.2',
         'PyDispatcher>=2.0.5',
+        'service_identity',
     ],
 )
diff --git a/tox.ini b/tox.ini
index fe6b5de38..9e5b2fa13 100644
--- a/tox.ini
+++ b/tox.ini
@@ -13,7 +13,6 @@ deps =
     boto
     Pillow
     leveldb
-    service_identity
     -rtests/requirements.txt
 commands =
     py.test {posargs:scrapy tests}
@@ -43,7 +42,6 @@ deps =
     -rrequirements-py3.txt
     # Extras
     Pillow
-    service_identity
     -rtests/requirements-py3.txt
 
 [testenv:py34]

From f67a808d9e53beb3ad6fed32575b1e554b969849 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Tue, 11 Aug 2015 13:47:04 -0300
Subject: [PATCH 0477/2974] Add 1.0.3 release notes

---
 docs/news.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 21612fcc7..5df1b1a6a 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,12 @@
 Release notes
 =============
 
+1.0.3 (2015-08-11)
+------------------
+
+- add service_identity to scrapy install_requires (:commit:`cbc2501`)
+- Workaround for travis#296 (:commit:`66af9cd`)
+
 1.0.2 (2015-08-06)
 ------------------
 

From ce21884a976a35d28fd0d1a3b8efd73c178a7595 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Mon, 3 Aug 2015 20:15:21 -0300
Subject: [PATCH 0478/2974] migrating scrapy Selector to use Parsel

---
 requirements.txt           |   1 +
 scrapy/selector/unified.py | 167 +++----------------------------------
 setup.py                   |   1 +
 tests/test_selector.py     |   2 +-
 4 files changed, 15 insertions(+), 156 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index a05cd3680..23be40daa 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,3 +7,4 @@ queuelib
 six>=1.5.2
 PyDispatcher>=2.0.5
 service_identity
+parsel>=0.9.0
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index eed8f94f7..d00c1cd41 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -2,36 +2,18 @@
 XPath selectors based on lxml
 """
 
-from lxml import etree
-import six
-
-from scrapy.utils.misc import extract_regex
 from scrapy.utils.trackref import object_ref
-from scrapy.utils.python import to_bytes, flatten, iflatten
-from scrapy.utils.decorators import deprecated
+from scrapy.utils.python import to_bytes
 from scrapy.http import HtmlResponse, XmlResponse
+from scrapy.utils.decorators import deprecated
+from parsel import Selector as ParselSelector, SelectorList
+from parsel.unified import _ctgroup
 from .lxmldocument import LxmlDocument
-from .csstranslator import ScrapyHTMLTranslator, ScrapyGenericTranslator
 
 
 __all__ = ['Selector', 'SelectorList']
 
 
-class SafeXMLParser(etree.XMLParser):
-    def __init__(self, *args, **kwargs):
-        kwargs.setdefault('resolve_entities', False)
-        super(SafeXMLParser, self).__init__(*args, **kwargs)
-
-_ctgroup = {
-    'html': {'_parser': etree.HTMLParser,
-             '_csstranslator': ScrapyHTMLTranslator(),
-             '_tostring_method': 'html'},
-    'xml': {'_parser': SafeXMLParser,
-            '_csstranslator': ScrapyGenericTranslator(),
-            '_tostring_method': 'xml'},
-}
-
-
 def _st(response, st):
     if st is None:
         return 'xml' if isinstance(response, XmlResponse) else 'html'
@@ -47,111 +29,25 @@ def _response_from_text(text, st):
               body=to_bytes(text, 'utf-8'))
 
 
-class Selector(object_ref):
+class Selector(ParselSelector, object_ref):
 
-    __slots__ = ['response', 'text', 'namespaces', 'type', '_expr', '_root',
-                 '__weakref__', '_parser', '_csstranslator', '_tostring_method']
+    __slots__ = ['response']
 
-    _default_type = None
-    _default_namespaces = {
-        "re": "http://exslt.org/regular-expressions",
+    def __init__(self, response=None, text=None, type=None, root=None, **kwargs):
+        st = _st(response, type or self._default_type)
+        root = kwargs.get('root', root)
 
-        # supported in libxslt:
-        # set:difference
-        # set:has-same-node
-        # set:intersection
-        # set:leading
-        # set:trailing
-        "set": "http://exslt.org/sets"
-    }
-    _lxml_smart_strings = False
-
-    def __init__(self, response=None, text=None, type=None, namespaces=None,
-                 _root=None, _expr=None):
-        self.type = st = _st(response, type or self._default_type)
         self._parser = _ctgroup[st]['_parser']
-        self._csstranslator = _ctgroup[st]['_csstranslator']
-        self._tostring_method = _ctgroup[st]['_tostring_method']
 
         if text is not None:
             response = _response_from_text(text, st)
 
         if response is not None:
-            _root = LxmlDocument(response, self._parser)
+            root = LxmlDocument(response, self._parser)
 
         self.response = response
-        self.namespaces = dict(self._default_namespaces)
-        if namespaces is not None:
-            self.namespaces.update(namespaces)
-        self._root = _root
-        self._expr = _expr
-
-    def xpath(self, query):
-        try:
-            xpathev = self._root.xpath
-        except AttributeError:
-            return SelectorList([])
-
-        try:
-            result = xpathev(query, namespaces=self.namespaces,
-                             smart_strings=self._lxml_smart_strings)
-        except etree.XPathError:
-            msg = u"Invalid XPath: %s" % query
-            raise ValueError(msg if six.PY3 else msg.encode("unicode_escape"))
-
-        if type(result) is not list:
-            result = [result]
-
-        result = [self.__class__(_root=x, _expr=query,
-                                 namespaces=self.namespaces,
-                                 type=self.type)
-                  for x in result]
-        return SelectorList(result)
-
-    def css(self, query):
-        return self.xpath(self._css2xpath(query))
-
-    def _css2xpath(self, query):
-        return self._csstranslator.css_to_xpath(query)
-
-    def re(self, regex):
-        return extract_regex(regex, self.extract())
-
-    def extract(self):
-        try:
-            return etree.tostring(self._root,
-                                  method=self._tostring_method,
-                                  encoding="unicode",
-                                  with_tail=False)
-        except (AttributeError, TypeError):
-            if self._root is True:
-                return u'1'
-            elif self._root is False:
-                return u'0'
-            else:
-                return six.text_type(self._root)
-
-    def register_namespace(self, prefix, uri):
-        if self.namespaces is None:
-            self.namespaces = {}
-        self.namespaces[prefix] = uri
-
-    def remove_namespaces(self):
-        for el in self._root.iter('*'):
-            if el.tag.startswith('{'):
-                el.tag = el.tag.split('}', 1)[1]
-            # loop on element attributes also
-            for an in el.attrib.keys():
-                if an.startswith('{'):
-                    el.attrib[an.split('}', 1)[1]] = el.attrib.pop(an)
-
-    def __nonzero__(self):
-        return bool(self.extract())
-
-    def __str__(self):
-        data = repr(self.extract()[:40])
-        return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data)
-    __repr__ = __str__
+        text = response.body_as_unicode() if response else None
+        super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
 
     # Deprecated api
     @deprecated(use_instead='.xpath()')
@@ -162,42 +58,3 @@ class Selector(object_ref):
     def extract_unquoted(self):
         return self.extract()
 
-
-class SelectorList(list):
-
-    def __getslice__(self, i, j):
-        return self.__class__(list.__getslice__(self, i, j))
-
-    def xpath(self, xpath):
-        return self.__class__(flatten([x.xpath(xpath) for x in self]))
-
-    def css(self, xpath):
-        return self.__class__(flatten([x.css(xpath) for x in self]))
-
-    def re(self, regex):
-        return flatten([x.re(regex) for x in self])
-
-    def re_first(self, regex):
-        for el in iflatten(x.re(regex) for x in self):
-            return el
-
-    def extract(self):
-        return [x.extract() for x in self]
-
-    def extract_first(self, default=None):
-        for x in self:
-            return x.extract()
-        else:
-            return default
-
-    @deprecated(use_instead='.extract()')
-    def extract_unquoted(self):
-        return [x.extract_unquoted() for x in self]
-
-    @deprecated(use_instead='.xpath()')
-    def x(self, xpath):
-        return self.select(xpath)
-
-    @deprecated(use_instead='.xpath()')
-    def select(self, xpath):
-        return self.xpath(xpath)
diff --git a/setup.py b/setup.py
index 7214f0dc6..959a73517 100644
--- a/setup.py
+++ b/setup.py
@@ -44,6 +44,7 @@ setup(
         'pyOpenSSL',
         'cssselect>=0.9',
         'six>=1.5.2',
+        'parsel>=0.9.0',
         'PyDispatcher>=2.0.5',
         'service_identity',
     ],
diff --git a/tests/test_selector.py b/tests/test_selector.py
index ad6a0e21c..02f82298c 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -332,7 +332,7 @@ class SelectorTestCase(unittest.TestCase):
 
     def test_weakref_slots(self):
         """Check that classes are using slots and are weak-referenceable"""
-        x = self.sscls()
+        x = self.sscls(text='')
         weakref.ref(x)
         assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
             x.__class__.__name__

From c7b29d118d97b6fb510ba5b6ae3fb8eea41196c0 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Mon, 3 Aug 2015 20:37:42 -0300
Subject: [PATCH 0479/2974] fix support to legacy _root argument

---
 scrapy/selector/unified.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index d00c1cd41..9e0716671 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -35,7 +35,9 @@ class Selector(ParselSelector, object_ref):
 
     def __init__(self, response=None, text=None, type=None, root=None, **kwargs):
         st = _st(response, type or self._default_type)
-        root = kwargs.get('root', root)
+
+        # supporting legacy _root argument
+        root = kwargs.get('_root', root)
 
         self._parser = _ctgroup[st]['_parser']
 

From 3a572e2f3b494327c918c2ac4a7150727c448f30 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Mon, 3 Aug 2015 20:38:15 -0300
Subject: [PATCH 0480/2974] cleanup csstranslator module, keeping only imports

---
 scrapy/selector/csstranslator.py | 94 ++------------------------------
 1 file changed, 6 insertions(+), 88 deletions(-)

diff --git a/scrapy/selector/csstranslator.py b/scrapy/selector/csstranslator.py
index 7482837a0..dfeb00be0 100644
--- a/scrapy/selector/csstranslator.py
+++ b/scrapy/selector/csstranslator.py
@@ -1,88 +1,6 @@
-from cssselect import GenericTranslator, HTMLTranslator
-from cssselect.xpath import _unicode_safe_getattr, XPathExpr, ExpressionError
-from cssselect.parser import FunctionalPseudoElement
-
-
-class ScrapyXPathExpr(XPathExpr):
-
-    textnode = False
-    attribute = None
-
-    @classmethod
-    def from_xpath(cls, xpath, textnode=False, attribute=None):
-        x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition)
-        x.textnode = textnode
-        x.attribute = attribute
-        return x
-
-    def __str__(self):
-        path = super(ScrapyXPathExpr, self).__str__()
-        if self.textnode:
-            if path == '*':
-                path = 'text()'
-            elif path.endswith('::*/*'):
-                path = path[:-3] + 'text()'
-            else:
-                path += '/text()'
-
-        if self.attribute is not None:
-            if path.endswith('::*/*'):
-                path = path[:-2]
-            path += '/@%s' % self.attribute
-
-        return path
-
-    def join(self, combiner, other):
-        super(ScrapyXPathExpr, self).join(combiner, other)
-        self.textnode = other.textnode
-        self.attribute = other.attribute
-        return self
-
-
-class TranslatorMixin(object):
-
-    def xpath_element(self, selector):
-        xpath = super(TranslatorMixin, self).xpath_element(selector)
-        return ScrapyXPathExpr.from_xpath(xpath)
-
-    def xpath_pseudo_element(self, xpath, pseudo_element):
-        if isinstance(pseudo_element, FunctionalPseudoElement):
-            method = 'xpath_%s_functional_pseudo_element' % (
-                pseudo_element.name.replace('-', '_'))
-            method = _unicode_safe_getattr(self, method, None)
-            if not method:
-                raise ExpressionError(
-                    "The functional pseudo-element ::%s() is unknown"
-                % pseudo_element.name)
-            xpath = method(xpath, pseudo_element)
-        else:
-            method = 'xpath_%s_simple_pseudo_element' % (
-                pseudo_element.replace('-', '_'))
-            method = _unicode_safe_getattr(self, method, None)
-            if not method:
-                raise ExpressionError(
-                    "The pseudo-element ::%s is unknown"
-                    % pseudo_element)
-            xpath = method(xpath)
-        return xpath
-
-    def xpath_attr_functional_pseudo_element(self, xpath, function):
-        if function.argument_types() not in (['STRING'], ['IDENT']):
-            raise ExpressionError(
-                "Expected a single string or ident for ::attr(), got %r"
-                % function.arguments)
-        return ScrapyXPathExpr.from_xpath(xpath,
-            attribute=function.arguments[0].value)
-
-    def xpath_text_simple_pseudo_element(self, xpath):
-        """Support selecting text nodes using ::text pseudo-element"""
-        return ScrapyXPathExpr.from_xpath(xpath, textnode=True)
-
-
-class ScrapyGenericTranslator(TranslatorMixin, GenericTranslator):
-    pass
-
-
-class ScrapyHTMLTranslator(TranslatorMixin, HTMLTranslator):
-    pass
-
+from parsel.csstranslator import (
+    ScrapyXPathExpr,
+    TranslatorMixin,
+    ScrapyGenericTranslator,
+    ScrapyHTMLTranslator
+)

From 01d948f0fd792c3700306efe1bd38ae2163cb042 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 4 Aug 2015 16:28:02 -0300
Subject: [PATCH 0481/2974] remove selector support for LxmlDocument DOM cache
 and add deprecation warning for _root argument

---
 scrapy/selector/unified.py | 20 +++++++++-----------
 tests/test_selector.py     | 11 +++++++++++
 2 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 9e0716671..db2a8510a 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -2,13 +2,13 @@
 XPath selectors based on lxml
 """
 
+import warnings
+from parsel import Selector as ParselSelector, SelectorList
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.python import to_bytes
 from scrapy.http import HtmlResponse, XmlResponse
 from scrapy.utils.decorators import deprecated
-from parsel import Selector as ParselSelector, SelectorList
-from parsel.unified import _ctgroup
-from .lxmldocument import LxmlDocument
+from scrapy.exceptions import ScrapyDeprecationWarning
 
 
 __all__ = ['Selector', 'SelectorList']
@@ -33,22 +33,21 @@ class Selector(ParselSelector, object_ref):
 
     __slots__ = ['response']
 
-    def __init__(self, response=None, text=None, type=None, root=None, **kwargs):
+    def __init__(self, response=None, text=None, type=None, root=None, _root=None, **kwargs):
         st = _st(response, type or self._default_type)
 
-        # supporting legacy _root argument
-        root = kwargs.get('_root', root)
-
-        self._parser = _ctgroup[st]['_parser']
+        if root is None and _root is not None:
+            warnings.warn("Argument `_root` is deprecated, use `root` instead",
+                          ScrapyDeprecationWarning, stacklevel=2)
+            root = _root
 
         if text is not None:
             response = _response_from_text(text, st)
 
         if response is not None:
-            root = LxmlDocument(response, self._parser)
+            text = response.body_as_unicode()
 
         self.response = response
-        text = response.body_as_unicode() if response else None
         super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
 
     # Deprecated api
@@ -59,4 +58,3 @@ class Selector(ParselSelector, object_ref):
     @deprecated(use_instead='.extract()')
     def extract_unquoted(self):
         return self.extract()
-
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 02f82298c..dd27ae7da 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -7,6 +7,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import TextResponse, HtmlResponse, XmlResponse
 from scrapy.selector import Selector
 from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector
+from lxml import etree
+from tests import mock
 
 
 class SelectorTestCase(unittest.TestCase):
@@ -37,6 +39,15 @@ class SelectorTestCase(unittest.TestCase):
         self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
                          [u'12'])
 
+    @mock.patch('scrapy.selector.unified.warnings')
+    def test_deprecated_root_argument(self, warnings):
+        root = etree.fromstring(u'<html/>')
+        sel = self.sscls(_root=root)
+        self.assertEqual(root, sel._root)
+        warnings.warn.assert_called_once_with(
+            'Argument `_root` is deprecated, use `root` instead',
+            mock.ANY, stacklevel=2)
+
     def test_representation_slice(self):
         body = u"<p><input name='{}' value='\xa9'/></p>".format(50 * 'b')
         response = TextResponse(url="http://example.com", body=body, encoding='utf8')

From 17d7347a3698ac7e3650f539578777d63c933262 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 5 Aug 2015 16:57:41 -0300
Subject: [PATCH 0482/2974] update minimal parsel version, add deprecated
 classes for csstranslator modules, fix test

---
 requirements.txt                 |  2 +-
 scrapy/selector/csstranslator.py | 21 +++++++++++++++------
 setup.py                         |  2 +-
 tests/test_selector.py           |  2 +-
 4 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 23be40daa..ffa5b3025 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,4 +7,4 @@ queuelib
 six>=1.5.2
 PyDispatcher>=2.0.5
 service_identity
-parsel>=0.9.0
+parsel>=0.9.1
diff --git a/scrapy/selector/csstranslator.py b/scrapy/selector/csstranslator.py
index dfeb00be0..8d7f034af 100644
--- a/scrapy/selector/csstranslator.py
+++ b/scrapy/selector/csstranslator.py
@@ -1,6 +1,15 @@
-from parsel.csstranslator import (
-    ScrapyXPathExpr,
-    TranslatorMixin,
-    ScrapyGenericTranslator,
-    ScrapyHTMLTranslator
-)
+from parsel.csstranslator import XPathExpr, GenericTranslator, HTMLTranslator
+from scrapy.utils.deprecate import create_deprecated_class
+
+
+ScrapyXPathExpr = create_deprecated_class(
+    'ScrapyXPathExpr', XPathExpr,
+    new_class_path='parsel.csstranslator.XPathExpr')
+
+ScrapyGenericTranslator = create_deprecated_class(
+    'ScrapyGenericTranslator', GenericTranslator,
+    new_class_path='parsel.csstranslator.GenericTranslator')
+
+ScrapyHTMLTranslator = create_deprecated_class(
+    'ScrapyHTMLTranslator', HTMLTranslator,
+    new_class_path='parsel.csstranslator.HTMLTranslator')
diff --git a/setup.py b/setup.py
index 959a73517..6fe303f15 100644
--- a/setup.py
+++ b/setup.py
@@ -44,7 +44,7 @@ setup(
         'pyOpenSSL',
         'cssselect>=0.9',
         'six>=1.5.2',
-        'parsel>=0.9.0',
+        'parsel>=0.9.1',
         'PyDispatcher>=2.0.5',
         'service_identity',
     ],
diff --git a/tests/test_selector.py b/tests/test_selector.py
index dd27ae7da..8dc194e19 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -43,7 +43,7 @@ class SelectorTestCase(unittest.TestCase):
     def test_deprecated_root_argument(self, warnings):
         root = etree.fromstring(u'<html/>')
         sel = self.sscls(_root=root)
-        self.assertEqual(root, sel._root)
+        self.assertIs(root, sel._root)
         warnings.warn.assert_called_once_with(
             'Argument `_root` is deprecated, use `root` instead',
             mock.ANY, stacklevel=2)

From 35c1dcdbc2eeaace7fce5e58e22f395625b87422 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 5 Aug 2015 19:47:16 -0300
Subject: [PATCH 0483/2974] use response.selector in link extractors instead of
 instantiating new Selector

---
 scrapy/linkextractors/lxmlhtml.py | 8 +++-----
 scrapy/linkextractors/sgml.py     | 2 +-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index c952a5f83..7c41a88ff 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -65,9 +65,8 @@ class LxmlParserLinkExtractor(object):
                 if self.unique else links
 
     def extract_links(self, response):
-        html = Selector(response)
         base_url = get_base_url(response)
-        return self._extract_links(html, response.url, response.encoding, base_url)
+        return self._extract_links(response.selector, response.url, response.encoding, base_url)
 
     def _process_links(self, links):
         """ Normalize and filter extracted links
@@ -95,14 +94,13 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
             canonicalize=canonicalize, deny_extensions=deny_extensions)
 
     def extract_links(self, response):
-        html = Selector(response)
         base_url = get_base_url(response)
         if self.restrict_xpaths:
             docs = [subdoc
                     for x in self.restrict_xpaths
-                    for subdoc in html.xpath(x)]
+                    for subdoc in response.xpath(x)]
         else:
-            docs = [html]
+            docs = [response.selector]
         all_links = []
         for doc in docs:
             links = self._extract_links(doc, response.url, response.encoding, base_url)
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index e4c2c274f..d045baa24 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -127,7 +127,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
     def extract_links(self, response):
         base_url = None
         if self.restrict_xpaths:
-            sel = Selector(response)
+            sel = response.selector
             base_url = get_base_url(response)
             body = u''.join(f
                             for x in self.restrict_xpaths

From 6287fc310948d15869c14fff9781800fd3ccc3c7 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 6 Aug 2015 21:55:05 -0300
Subject: [PATCH 0484/2974] remove lxmldocument dependency from
 http.request.form

---
 scrapy/http/request/form.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 0d37004fb..ad3f0571e 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -7,6 +7,7 @@ See documentation in docs/topics/request-response.rst
 
 from six.moves.urllib.parse import urljoin, urlencode
 import lxml.html
+from lxml import etree
 import six
 from scrapy.http.request import Request
 from scrapy.utils.python import to_bytes, is_listlike
@@ -54,10 +55,15 @@ def _urlencode(seq, enc):
     return urlencode(values, doseq=1)
 
 
+def _create_parser_from_response(response, parser_cls):
+    body = response.body_as_unicode().strip().encode('utf8') or b'<html/>'
+    parser = parser_cls(recover=True, encoding='utf8')
+    return etree.fromstring(body, parser=parser, base_url=response.url)
+
+
 def _get_form(response, formname, formid, formnumber, formxpath):
     """Find the form element """
-    from scrapy.selector.lxmldocument import LxmlDocument
-    root = LxmlDocument(response, lxml.html.HTMLParser)
+    root = _create_parser_from_response(response, lxml.html.HTMLParser)
     forms = root.xpath('//form')
     if not forms:
         raise ValueError("No <form> element found in %s" % response)

From 94c3a345b75db3e28dc0cca565650008d69e8992 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 6 Aug 2015 22:35:43 -0300
Subject: [PATCH 0485/2974] remove deprecated module lxmldocument

---
 scrapy/selector/lxmldocument.py     | 31 -----------------------------
 tests/py3-ignores.txt               |  1 -
 tests/test_selector_lxmldocument.py | 26 ------------------------
 3 files changed, 58 deletions(-)
 delete mode 100644 scrapy/selector/lxmldocument.py
 delete mode 100644 tests/test_selector_lxmldocument.py

diff --git a/scrapy/selector/lxmldocument.py b/scrapy/selector/lxmldocument.py
deleted file mode 100644
index 817349b58..000000000
--- a/scrapy/selector/lxmldocument.py
+++ /dev/null
@@ -1,31 +0,0 @@
-"""
-This module contains a simple class (LxmlDocument) which provides cache and
-garbage collection to lxml element tree documents.
-"""
-
-import weakref
-from lxml import etree
-from scrapy.utils.trackref import object_ref
-
-
-def _factory(response, parser_cls):
-    url = response.url
-    body = response.body_as_unicode().strip().encode('utf8') or '<html/>'
-    parser = parser_cls(recover=True, encoding='utf8')
-    return etree.fromstring(body, parser=parser, base_url=url)
-
-
-class LxmlDocument(object_ref):
-
-    cache = weakref.WeakKeyDictionary()
-    __slots__ = ['__weakref__']
-
-    def __new__(cls, response, parser=etree.HTMLParser):
-        cache = cls.cache.setdefault(response, {})
-        if parser not in cache:
-            obj = object_ref.__new__(cls)
-            cache[parser] = _factory(response, parser)
-        return cache[parser]
-
-    def __str__(self):
-        return "<LxmlDocument %s>" % self.root.tag
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 9be3a99a8..8f5c0de48 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -26,7 +26,6 @@ tests/test_pipeline_files.py
 tests/test_pipeline_images.py
 tests/test_proxy_connect.py
 tests/test_selector_csstranslator.py
-tests/test_selector_lxmldocument.py
 tests/test_selector.py
 tests/test_spidermiddleware_depth.py
 tests/test_spidermiddleware_httperror.py
diff --git a/tests/test_selector_lxmldocument.py b/tests/test_selector_lxmldocument.py
deleted file mode 100644
index 090cc21bc..000000000
--- a/tests/test_selector_lxmldocument.py
+++ /dev/null
@@ -1,26 +0,0 @@
-import unittest
-from scrapy.selector.lxmldocument import LxmlDocument
-from scrapy.http import TextResponse, HtmlResponse
-
-
-class LxmlDocumentTest(unittest.TestCase):
-
-    def test_caching(self):
-        r1 = HtmlResponse('http://www.example.com', body=b'<html><head></head><body></body></html>')
-        r2 = r1.copy()
-
-        doc1 = LxmlDocument(r1)
-        doc2 = LxmlDocument(r1)
-        doc3 = LxmlDocument(r2)
-
-        # make sure it's cached
-        assert doc1 is doc2
-        assert doc1 is not doc3
-
-    def test_null_char(self):
-        # make sure bodies with null char ('\x00') don't raise a TypeError exception
-        body = b'test problematic \x00 body'
-        response = TextResponse('http://example.com/catalog/product/blabla-123',
-                                headers={'Content-Type': 'text/plain; charset=utf-8'},
-                                body=body)
-        LxmlDocument(response)

From 67c98b185b4ee2a55c71dd1c0266b200fd3bcc04 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 6 Aug 2015 23:31:06 -0300
Subject: [PATCH 0486/2974] avoid harcoded check for selector type

---
 scrapy/selector/unified.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index db2a8510a..3c75a92d2 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -17,10 +17,7 @@ __all__ = ['Selector', 'SelectorList']
 def _st(response, st):
     if st is None:
         return 'xml' if isinstance(response, XmlResponse) else 'html'
-    elif st in ('xml', 'html'):
-        return st
-    else:
-        raise ValueError('Invalid type: %s' % st)
+    return st
 
 
 def _response_from_text(text, st):

From 2fe6d128f51f94f55ac9d83ac05eb9c763c478b8 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Fri, 7 Aug 2015 15:26:54 -0300
Subject: [PATCH 0487/2974] upgrade parsel and using promoted root attribute

---
 requirements.txt                  | 2 +-
 scrapy/linkextractors/lxmlhtml.py | 2 +-
 scrapy/selector/unified.py        | 6 ++++++
 setup.py                          | 2 +-
 tests/test_selector.py            | 2 +-
 5 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index ffa5b3025..f8a0f7ad3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,4 +7,4 @@ queuelib
 six>=1.5.2
 PyDispatcher>=2.0.5
 service_identity
-parsel>=0.9.1
+parsel>=0.9.2
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 7c41a88ff..606a45212 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -47,7 +47,7 @@ class LxmlParserLinkExtractor(object):
     def _extract_links(self, selector, response_url, response_encoding, base_url):
         links = []
         # hacky way to get the underlying lxml parsed document
-        for el, attr, attr_val in self._iter_links(selector._root):
+        for el, attr, attr_val in self._iter_links(selector.root):
             # pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
             attr_val = urljoin(base_url, attr_val)
             url = self.process_attr(attr_val)
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 3c75a92d2..e229b10b8 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -48,6 +48,12 @@ class Selector(ParselSelector, object_ref):
         super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
 
     # Deprecated api
+    @property
+    def _root(self):
+        warnings.warn("Attribute `_root` is deprecated, use `root` instead",
+                      ScrapyDeprecationWarning, stacklevel=2)
+        return self.root
+
     @deprecated(use_instead='.xpath()')
     def select(self, xpath):
         return self.xpath(xpath)
diff --git a/setup.py b/setup.py
index 6fe303f15..2d2d8660f 100644
--- a/setup.py
+++ b/setup.py
@@ -44,7 +44,7 @@ setup(
         'pyOpenSSL',
         'cssselect>=0.9',
         'six>=1.5.2',
-        'parsel>=0.9.1',
+        'parsel>=0.9.2',
         'PyDispatcher>=2.0.5',
         'service_identity',
     ],
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 8dc194e19..d9660c674 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -43,7 +43,7 @@ class SelectorTestCase(unittest.TestCase):
     def test_deprecated_root_argument(self, warnings):
         root = etree.fromstring(u'<html/>')
         sel = self.sscls(_root=root)
-        self.assertIs(root, sel._root)
+        self.assertIs(root, sel.root)
         warnings.warn.assert_called_once_with(
             'Argument `_root` is deprecated, use `root` instead',
             mock.ANY, stacklevel=2)

From 26ebccd37a22e15eb522d6e7d979fd816ebbcdd8 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Fri, 7 Aug 2015 17:33:59 -0300
Subject: [PATCH 0488/2974] upgrade parsel and use its function to instantiate
 root for finding form

---
 requirements.txt            |  2 +-
 scrapy/http/request/form.py | 11 +++--------
 setup.py                    |  2 +-
 3 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index f8a0f7ad3..afefb9d33 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,4 +7,4 @@ queuelib
 six>=1.5.2
 PyDispatcher>=2.0.5
 service_identity
-parsel>=0.9.2
+parsel>=0.9.3
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index ad3f0571e..0b1d3b926 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst
 
 from six.moves.urllib.parse import urljoin, urlencode
 import lxml.html
-from lxml import etree
+from parsel.selector import create_root_node
 import six
 from scrapy.http.request import Request
 from scrapy.utils.python import to_bytes, is_listlike
@@ -55,15 +55,10 @@ def _urlencode(seq, enc):
     return urlencode(values, doseq=1)
 
 
-def _create_parser_from_response(response, parser_cls):
-    body = response.body_as_unicode().strip().encode('utf8') or b'<html/>'
-    parser = parser_cls(recover=True, encoding='utf8')
-    return etree.fromstring(body, parser=parser, base_url=response.url)
-
-
 def _get_form(response, formname, formid, formnumber, formxpath):
     """Find the form element """
-    root = _create_parser_from_response(response, lxml.html.HTMLParser)
+    text = response.body_as_unicode()
+    root = create_root_node(text, lxml.html.HTMLParser, base_url=response.url)
     forms = root.xpath('//form')
     if not forms:
         raise ValueError("No <form> element found in %s" % response)
diff --git a/setup.py b/setup.py
index 2d2d8660f..4eb8d2318 100644
--- a/setup.py
+++ b/setup.py
@@ -44,7 +44,7 @@ setup(
         'pyOpenSSL',
         'cssselect>=0.9',
         'six>=1.5.2',
-        'parsel>=0.9.2',
+        'parsel>=0.9.3',
         'PyDispatcher>=2.0.5',
         'service_identity',
     ],

From 12579b9afa20612734355c6d0d10481857dbd184 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Sun, 9 Aug 2015 01:21:39 -0300
Subject: [PATCH 0489/2974] warning when ambiguous root arguments and minor
 cleanups

---
 scrapy/linkextractors/sgml.py |  3 +--
 scrapy/selector/unified.py    |  7 +++++--
 tests/test_selector.py        | 24 +++++++++++++++---------
 3 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index d045baa24..4a6a24254 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -127,11 +127,10 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
     def extract_links(self, response):
         base_url = None
         if self.restrict_xpaths:
-            sel = response.selector
             base_url = get_base_url(response)
             body = u''.join(f
                             for x in self.restrict_xpaths
-                            for f in sel.xpath(x).extract()
+                            for f in response.xpath(x).extract()
                             ).encode(response.encoding, errors='xmlcharrefreplace')
         else:
             body = response.body
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index e229b10b8..a8f80c84d 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -33,10 +33,13 @@ class Selector(ParselSelector, object_ref):
     def __init__(self, response=None, text=None, type=None, root=None, _root=None, **kwargs):
         st = _st(response, type or self._default_type)
 
-        if root is None and _root is not None:
+        if _root is not None:
             warnings.warn("Argument `_root` is deprecated, use `root` instead",
                           ScrapyDeprecationWarning, stacklevel=2)
-            root = _root
+            if root is None:
+                root = _root
+            else:
+                warnings.warn("Ignoring deprecated `_root` argument, using provided `root`")
 
         if text is not None:
             response = _response_from_text(text, st)
diff --git a/tests/test_selector.py b/tests/test_selector.py
index d9660c674..dc37da86b 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -8,7 +8,6 @@ from scrapy.http import TextResponse, HtmlResponse, XmlResponse
 from scrapy.selector import Selector
 from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector
 from lxml import etree
-from tests import mock
 
 
 class SelectorTestCase(unittest.TestCase):
@@ -39,14 +38,21 @@ class SelectorTestCase(unittest.TestCase):
         self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
                          [u'12'])
 
-    @mock.patch('scrapy.selector.unified.warnings')
-    def test_deprecated_root_argument(self, warnings):
-        root = etree.fromstring(u'<html/>')
-        sel = self.sscls(_root=root)
-        self.assertIs(root, sel.root)
-        warnings.warn.assert_called_once_with(
-            'Argument `_root` is deprecated, use `root` instead',
-            mock.ANY, stacklevel=2)
+    def test_deprecated_root_argument(self):
+        with warnings.catch_warnings(record=True) as w:
+            root = etree.fromstring(u'<html/>')
+            sel = self.sscls(_root=root)
+            self.assertIs(root, sel.root)
+            self.assertEqual(str(w[-1].message),
+                             'Argument `_root` is deprecated, use `root` instead')
+
+    def test_deprecated_root_argument_ambiguous(self):
+        with warnings.catch_warnings(record=True) as w:
+            _root = etree.fromstring(u'<xml/>')
+            root = etree.fromstring(u'<html/>')
+            sel = self.sscls(_root=_root, root=root)
+            self.assertIs(root, sel.root)
+            self.assertIn('Ignoring deprecated `_root` argument', str(w[-1].message))
 
     def test_representation_slice(self):
         body = u"<p><input name='{}' value='\xa9'/></p>".format(50 * 'b')

From 3a03ef7c08f0b4b454d13933ea2239b053824675 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Sun, 9 Aug 2015 15:23:43 -0300
Subject: [PATCH 0490/2974] cleanup tests for selectors and translators

---
 tests/py3-ignores.txt                |   2 -
 tests/test_selector.py               | 499 +--------------------------
 tests/test_selector_csstranslator.py | 159 +--------
 3 files changed, 29 insertions(+), 631 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 8f5c0de48..5a009db36 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -25,8 +25,6 @@ tests/test_mail.py
 tests/test_pipeline_files.py
 tests/test_pipeline_images.py
 tests/test_proxy_connect.py
-tests/test_selector_csstranslator.py
-tests/test_selector.py
 tests/test_spidermiddleware_depth.py
 tests/test_spidermiddleware_httperror.py
 tests/test_spidermiddleware_offsite.py
diff --git a/tests/test_selector.py b/tests/test_selector.py
index dc37da86b..4806bb90b 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -12,18 +12,16 @@ from lxml import etree
 
 class SelectorTestCase(unittest.TestCase):
 
-    sscls = Selector
-
     def test_simple_selection(self):
         """Simple selector tests"""
-        body = "<p><input name='a'value='1'/><input name='b'value='2'/></p>"
-        response = TextResponse(url="http://example.com", body=body)
-        sel = self.sscls(response)
+        body = u"<p><input name='a'value='1'/><input name='b'value='2'/></p>"
+        response = TextResponse(url="http://example.com", body=body, encoding='utf-8')
+        sel = Selector(response)
 
         xl = sel.xpath('//input')
         self.assertEqual(2, len(xl))
         for x in xl:
-            assert isinstance(x, self.sscls)
+            assert isinstance(x, Selector)
 
         self.assertEqual(sel.xpath('//input').extract(),
                          [x.extract() for x in sel.xpath('//input')])
@@ -41,7 +39,7 @@ class SelectorTestCase(unittest.TestCase):
     def test_deprecated_root_argument(self):
         with warnings.catch_warnings(record=True) as w:
             root = etree.fromstring(u'<html/>')
-            sel = self.sscls(_root=root)
+            sel = Selector(_root=root)
             self.assertIs(root, sel.root)
             self.assertEqual(str(w[-1].message),
                              'Argument `_root` is deprecated, use `root` instead')
@@ -50,238 +48,22 @@ class SelectorTestCase(unittest.TestCase):
         with warnings.catch_warnings(record=True) as w:
             _root = etree.fromstring(u'<xml/>')
             root = etree.fromstring(u'<html/>')
-            sel = self.sscls(_root=_root, root=root)
+            sel = Selector(_root=_root, root=root)
             self.assertIs(root, sel.root)
             self.assertIn('Ignoring deprecated `_root` argument', str(w[-1].message))
 
-    def test_representation_slice(self):
-        body = u"<p><input name='{}' value='\xa9'/></p>".format(50 * 'b')
-        response = TextResponse(url="http://example.com", body=body, encoding='utf8')
-        sel = self.sscls(response)
-
-        self.assertEqual(
-            map(repr, sel.xpath('//input/@name')),
-            ["<Selector xpath='//input/@name' data=u'{}'>".format(40 * 'b')]
-        )
-
-    def test_representation_unicode_query(self):
-        body = u"<p><input name='{}' value='\xa9'/></p>".format(50 * 'b')
-        response = TextResponse(url="http://example.com", body=body, encoding='utf8')
-        sel = self.sscls(response)
-        self.assertEqual(
-            map(repr, sel.xpath(u'//input[@value="\xa9"]/@value')),
-            ["<Selector xpath=u'//input[@value=\"\\xa9\"]/@value' data=u'\\xa9'>"]
-        )
-
-    def test_extract_first(self):
-        """Test if extract_first() returns first element"""
-        body = '<ul><li id="1">1</li><li id="2">2</li></ul>'
-        response = TextResponse(url="http://example.com", body=body)
-        sel = self.sscls(response)
-
-        self.assertEqual(sel.xpath('//ul/li/text()').extract_first(),
-                         sel.xpath('//ul/li/text()').extract()[0])
-
-        self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').extract_first(),
-                         sel.xpath('//ul/li[@id="1"]/text()').extract()[0])
-
-        self.assertEqual(sel.xpath('//ul/li[2]/text()').extract_first(),
-                         sel.xpath('//ul/li/text()').extract()[1])
-
-        self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').extract_first(), None)
-
-    def test_extract_first_default(self):
-        """Test if extract_first() returns default value when no results found"""
-        body = '<ul><li id="1">1</li><li id="2">2</li></ul>'
-        response = TextResponse(url="http://example.com", body=body)
-        sel = self.sscls(response)
-
-        self.assertEqual(sel.xpath('//div/text()').extract_first(default='missing'), 'missing')
-
-    def test_re_first(self):
-        """Test if re_first() returns first matched element"""
-        body = '<ul><li id="1">1</li><li id="2">2</li></ul>'
-        response = TextResponse(url="http://example.com", body=body)
-        sel = self.sscls(response)
-
-        self.assertEqual(sel.xpath('//ul/li/text()').re_first('\d'),
-                         sel.xpath('//ul/li/text()').re('\d')[0])
-
-        self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').re_first('\d'),
-                         sel.xpath('//ul/li[@id="1"]/text()').re('\d')[0])
-
-        self.assertEqual(sel.xpath('//ul/li[2]/text()').re_first('\d'),
-                         sel.xpath('//ul/li/text()').re('\d')[1])
-
-        self.assertEqual(sel.xpath('/ul/li/text()').re_first('\w+'), None)
-        self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').re_first('\d'), None)
-
-    def test_select_unicode_query(self):
-        body = u"<p><input name='\xa9' value='1'/></p>"
-        response = TextResponse(url="http://example.com", body=body, encoding='utf8')
-        sel = self.sscls(response)
-        self.assertEqual(sel.xpath(u'//input[@name="\xa9"]/@value').extract(), [u'1'])
-
-    def test_list_elements_type(self):
-        """Test Selector returning the same type in selection methods"""
-        text = '<p>test<p>'
-        assert isinstance(self.sscls(text=text).xpath("//p")[0], self.sscls)
-        assert isinstance(self.sscls(text=text).css("p")[0], self.sscls)
-
-    def test_boolean_result(self):
-        body = "<p><input name='a'value='1'/><input name='b'value='2'/></p>"
-        response = TextResponse(url="http://example.com", body=body)
-        xs = self.sscls(response)
-        self.assertEquals(xs.xpath("//input[@name='a']/@name='a'").extract(), [u'1'])
-        self.assertEquals(xs.xpath("//input[@name='a']/@name='n'").extract(), [u'0'])
-
-    def test_differences_parsing_xml_vs_html(self):
-        """Test that XML and HTML Selector's behave differently"""
-        # some text which is parsed differently by XML and HTML flavors
-        text = '<div><img src="a.jpg"><p>Hello</div>'
-        hs = self.sscls(text=text, type='html')
-        self.assertEqual(hs.xpath("//div").extract(),
-                         [u'<div><img src="a.jpg"><p>Hello</p></div>'])
-
-        xs = self.sscls(text=text, type='xml')
-        self.assertEqual(xs.xpath("//div").extract(),
-                         [u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
-
     def test_flavor_detection(self):
-        text = '<div><img src="a.jpg"><p>Hello</div>'
-        sel = self.sscls(XmlResponse('http://example.com', body=text))
+        text = u'<div><img src="a.jpg"><p>Hello</div>'
+        sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
         self.assertEqual(sel.type, 'xml')
         self.assertEqual(sel.xpath("//div").extract(),
                          [u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
 
-        sel = self.sscls(HtmlResponse('http://example.com', body=text))
+        sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
         self.assertEqual(sel.type, 'html')
         self.assertEqual(sel.xpath("//div").extract(),
                          [u'<div><img src="a.jpg"><p>Hello</p></div>'])
 
-    def test_nested_selectors(self):
-        """Nested selector tests"""
-        body = """<body>
-                    <div class='one'>
-                      <ul>
-                        <li>one</li><li>two</li>
-                      </ul>
-                    </div>
-                    <div class='two'>
-                      <ul>
-                        <li>four</li><li>five</li><li>six</li>
-                      </ul>
-                    </div>
-                  </body>"""
-
-        response = HtmlResponse(url="http://example.com", body=body)
-        x = self.sscls(response)
-        divtwo = x.xpath('//div[@class="two"]')
-        self.assertEqual(divtwo.xpath("//li").extract(),
-                         ["<li>one</li>", "<li>two</li>", "<li>four</li>", "<li>five</li>", "<li>six</li>"])
-        self.assertEqual(divtwo.xpath("./ul/li").extract(),
-                         ["<li>four</li>", "<li>five</li>", "<li>six</li>"])
-        self.assertEqual(divtwo.xpath(".//li").extract(),
-                         ["<li>four</li>", "<li>five</li>", "<li>six</li>"])
-        self.assertEqual(divtwo.xpath("./li").extract(), [])
-
-    def test_mixed_nested_selectors(self):
-        body = '''<body>
-                    <div id=1>not<span>me</span></div>
-                    <div class="dos"><p>text</p><a href='#'>foo</a></div>
-               </body>'''
-        sel = self.sscls(text=body)
-        self.assertEqual(sel.xpath('//div[@id="1"]').css('span::text').extract(), [u'me'])
-        self.assertEqual(sel.css('#1').xpath('./span/text()').extract(), [u'me'])
-
-    def test_dont_strip(self):
-        sel = self.sscls(text='<div>fff: <a href="#">zzz</a></div>')
-        self.assertEqual(sel.xpath("//text()").extract(), [u'fff: ', u'zzz'])
-
-    def test_namespaces_simple(self):
-        body = """
-        <test xmlns:somens="http://scrapy.org">
-           <somens:a id="foo">take this</a>
-           <a id="bar">found</a>
-        </test>
-        """
-
-        response = XmlResponse(url="http://example.com", body=body)
-        x = self.sscls(response)
-
-        x.register_namespace("somens", "http://scrapy.org")
-        self.assertEqual(x.xpath("//somens:a/text()").extract(),
-                         [u'take this'])
-
-    def test_namespaces_multiple(self):
-        body = """<?xml version="1.0" encoding="UTF-8"?>
-<BrowseNode xmlns="http://webservices.amazon.com/AWSECommerceService/2005-10-05"
-            xmlns:b="http://somens.com"
-            xmlns:p="http://www.scrapy.org/product" >
-    <b:Operation>hello</b:Operation>
-    <TestTag b:att="value"><Other>value</Other></TestTag>
-    <p:SecondTestTag><material>iron</material><price>90</price><p:name>Dried Rose</p:name></p:SecondTestTag>
-</BrowseNode>
-        """
-        response = XmlResponse(url="http://example.com", body=body)
-        x = self.sscls(response)
-        x.register_namespace("xmlns", "http://webservices.amazon.com/AWSECommerceService/2005-10-05")
-        x.register_namespace("p", "http://www.scrapy.org/product")
-        x.register_namespace("b", "http://somens.com")
-        self.assertEqual(len(x.xpath("//xmlns:TestTag")), 1)
-        self.assertEqual(x.xpath("//b:Operation/text()").extract()[0], 'hello')
-        self.assertEqual(x.xpath("//xmlns:TestTag/@b:att").extract()[0], 'value')
-        self.assertEqual(x.xpath("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90')
-        self.assertEqual(x.xpath("//p:SecondTestTag").xpath("./xmlns:price/text()")[0].extract(), '90')
-        self.assertEqual(x.xpath("//p:SecondTestTag/xmlns:material/text()").extract()[0], 'iron')
-
-    def test_re(self):
-        body = """<div>Name: Mary
-                    <ul>
-                      <li>Name: John</li>
-                      <li>Age: 10</li>
-                      <li>Name: Paul</li>
-                      <li>Age: 20</li>
-                    </ul>
-                    Age: 20
-                  </div>"""
-        response = HtmlResponse(url="http://example.com", body=body)
-        x = self.sscls(response)
-
-        name_re = re.compile("Name: (\w+)")
-        self.assertEqual(x.xpath("//ul/li").re(name_re),
-                         ["John", "Paul"])
-        self.assertEqual(x.xpath("//ul/li").re("Age: (\d+)"),
-                         ["10", "20"])
-
-    def test_re_intl(self):
-        body = """<div>Evento: cumplea\xc3\xb1os</div>"""
-        response = HtmlResponse(url="http://example.com", body=body, encoding='utf-8')
-        x = self.sscls(response)
-        self.assertEqual(x.xpath("//div").re("Evento: (\w+)"), [u'cumplea\xf1os'])
-
-    def test_selector_over_text(self):
-        hs = self.sscls(text='<root>lala</root>')
-        self.assertEqual(hs.extract(), u'<html><body><root>lala</root></body></html>')
-        xs = self.sscls(text='<root>lala</root>', type='xml')
-        self.assertEqual(xs.extract(), u'<root>lala</root>')
-        self.assertEqual(xs.xpath('.').extract(), [u'<root>lala</root>'])
-
-    def test_invalid_xpath(self):
-        "Test invalid xpath raises ValueError with the invalid xpath"
-        response = XmlResponse(url="http://example.com", body="<html></html>")
-        x = self.sscls(response)
-        xpath = "//test[@foo='bar]"
-        self.assertRaisesRegexp(ValueError, re.escape(xpath), x.xpath, xpath)
-
-    def test_invalid_xpath_unicode(self):
-        "Test *Unicode* invalid xpath raises ValueError with the invalid xpath"
-        response = XmlResponse(url="http://example.com", body="<html></html>")
-        x = self.sscls(response)
-        xpath = u"//test[@foo='\u0431ar]"
-        encoded = xpath if six.PY3 else xpath.encode('unicode_escape')
-        self.assertRaisesRegexp(ValueError, re.escape(encoded), x.xpath, xpath)
-
     def test_http_header_encoding_precedence(self):
         # u'\xa3'     = pound symbol in unicode
         # u'\xc2\xa3' = pound symbol in utf-8
@@ -297,136 +79,28 @@ class SelectorTestCase(unittest.TestCase):
 
         headers = {'Content-Type': ['text/html; charset=utf-8']}
         response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
-        x = self.sscls(response)
+        x = Selector(response)
         self.assertEquals(x.xpath("//span[@id='blank']/text()").extract(),
                           [u'\xa3'])
 
-    def test_empty_bodies(self):
-        # shouldn't raise errors
-        r1 = TextResponse('http://www.example.com', body='')
-        self.sscls(r1).xpath('//text()').extract()
-
-    def test_null_bytes(self):
-        # shouldn't raise errors
-        r1 = TextResponse('http://www.example.com', \
-                          body='<root>pre\x00post</root>', \
-                          encoding='utf-8')
-        self.sscls(r1).xpath('//text()').extract()
-
     def test_badly_encoded_body(self):
         # \xe9 alone isn't valid utf8 sequence
         r1 = TextResponse('http://www.example.com', \
-                          body='<html><p>an Jos\xe9 de</p><html>', \
+                          body=u'<html><p>an Jos\xe9 de</p><html>', \
                           encoding='utf-8')
-        self.sscls(r1).xpath('//text()').extract()
-
-    def test_select_on_unevaluable_nodes(self):
-        r = self.sscls(text=u'<span class="big">some text</span>')
-        # Text node
-        x1 = r.xpath('//text()')
-        self.assertEquals(x1.extract(), [u'some text'])
-        self.assertEquals(x1.xpath('.//b').extract(), [])
-        # Tag attribute
-        x1 = r.xpath('//span/@class')
-        self.assertEquals(x1.extract(), [u'big'])
-        self.assertEquals(x1.xpath('.//text()').extract(), [])
-
-    def test_select_on_text_nodes(self):
-        r = self.sscls(text=u'<div><b>Options:</b>opt1</div><div><b>Other</b>opt2</div>')
-        x1 = r.xpath("//div/descendant::text()[preceding-sibling::b[contains(text(), 'Options')]]")
-        self.assertEquals(x1.extract(), [u'opt1'])
-
-        x1 = r.xpath("//div/descendant::text()/preceding-sibling::b[contains(text(), 'Options')]")
-        self.assertEquals(x1.extract(), [u'<b>Options:</b>'])
-
-    def test_nested_select_on_text_nodes(self):
-        # FIXME: does not work with lxml backend [upstream]
-        r = self.sscls(text=u'<div><b>Options:</b>opt1</div><div><b>Other</b>opt2</div>')
-        x1 = r.xpath("//div/descendant::text()")
-        x2 = x1.xpath("./preceding-sibling::b[contains(text(), 'Options')]")
-        self.assertEquals(x2.extract(), [u'<b>Options:</b>'])
-    test_nested_select_on_text_nodes.skip = "Text nodes lost parent node reference in lxml"
+        Selector(r1).xpath('//text()').extract()
 
     def test_weakref_slots(self):
         """Check that classes are using slots and are weak-referenceable"""
-        x = self.sscls(text='')
+        x = Selector(text='')
         weakref.ref(x)
         assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
             x.__class__.__name__
 
-    def test_remove_namespaces(self):
-        xml = """<?xml version="1.0" encoding="UTF-8"?>
-<feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en-US" xmlns:media="http://search.yahoo.com/mrss/">
-  <link type="text/html">
-  <link type="application/atom+xml">
-</feed>
-"""
-        sel = self.sscls(XmlResponse("http://example.com/feed.atom", body=xml))
-        self.assertEqual(len(sel.xpath("//link")), 0)
-        sel.remove_namespaces()
-        self.assertEqual(len(sel.xpath("//link")), 2)
-
-    def test_remove_attributes_namespaces(self):
-        xml = """<?xml version="1.0" encoding="UTF-8"?>
-<feed xmlns:atom="http://www.w3.org/2005/Atom" xml:lang="en-US" xmlns:media="http://search.yahoo.com/mrss/">
-  <link atom:type="text/html">
-  <link atom:type="application/atom+xml">
-</feed>
-"""
-        sel = self.sscls(XmlResponse("http://example.com/feed.atom", body=xml))
-        self.assertEqual(len(sel.xpath("//link/@type")), 0)
-        sel.remove_namespaces()
-        self.assertEqual(len(sel.xpath("//link/@type")), 2)
-
-    def test_smart_strings(self):
-        """Lxml smart strings return values"""
-
-        class SmartStringsSelector(Selector):
-            _lxml_smart_strings = True
-
-        body = """<body>
-                    <div class='one'>
-                      <ul>
-                        <li>one</li><li>two</li>
-                      </ul>
-                    </div>
-                    <div class='two'>
-                      <ul>
-                        <li>four</li><li>five</li><li>six</li>
-                      </ul>
-                    </div>
-                  </body>"""
-
-        response = HtmlResponse(url="http://example.com", body=body)
-
-        # .getparent() is available for text nodes and attributes
-        # only when smart_strings are on
-        x = self.sscls(response)
-        li_text = x.xpath('//li/text()')
-        self.assertFalse(any(map(lambda e: hasattr(e._root, 'getparent'), li_text)))
-        div_class = x.xpath('//div/@class')
-        self.assertFalse(any(map(lambda e: hasattr(e._root, 'getparent'), div_class)))
-
-        x = SmartStringsSelector(response)
-        li_text = x.xpath('//li/text()')
-        self.assertTrue(all(map(lambda e: hasattr(e._root, 'getparent'), li_text)))
-        div_class = x.xpath('//div/@class')
-        self.assertTrue(all(map(lambda e: hasattr(e._root, 'getparent'), div_class)))
-
-    def test_xml_entity_expansion(self):
-        malicious_xml = '<?xml version="1.0" encoding="ISO-8859-1"?>'\
-            '<!DOCTYPE foo [ <!ELEMENT foo ANY > <!ENTITY xxe SYSTEM '\
-            '"file:///etc/passwd" >]><foo>&xxe;</foo>'
-
-        response = XmlResponse('http://example.com', body=malicious_xml)
-        sel = self.sscls(response=response)
-
-        self.assertEqual(sel.extract(), '<foo>&xxe;</foo>')
-
 
 class DeprecatedXpathSelectorTest(unittest.TestCase):
 
-    text = '<div><img src="a.jpg"><p>Hello</div>'
+    text = u'<div><img src="a.jpg"><p>Hello</div>'
 
     def test_warnings_xpathselector(self):
         cls = XPathSelector
@@ -504,147 +178,4 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
             self.assertTrue(isinstance(sel, Selector))
             self.assertTrue(isinstance(usel, Selector))
             self.assertTrue(isinstance(sel, XPathSelector))
-            self.assertTrue(isinstance(usel, XPathSelector))
-
-    def test_xpathselector(self):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
-            hs = XPathSelector(text=self.text)
-            self.assertEqual(hs.select("//div").extract(),
-                             [u'<div><img src="a.jpg"><p>Hello</p></div>'])
-            self.assertRaises(RuntimeError, hs.css, 'div')
-
-    def test_htmlxpathselector(self):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
-            hs = HtmlXPathSelector(text=self.text)
-            self.assertEqual(hs.select("//div").extract(),
-                             [u'<div><img src="a.jpg"><p>Hello</p></div>'])
-            self.assertRaises(RuntimeError, hs.css, 'div')
-
-    def test_xmlxpathselector(self):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
-            xs = XmlXPathSelector(text=self.text)
-            self.assertEqual(xs.select("//div").extract(),
-                             [u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
-            self.assertRaises(RuntimeError, xs.css, 'div')
-
-
-class ExsltTestCase(unittest.TestCase):
-
-    sscls = Selector
-
-    def test_regexp(self):
-        """EXSLT regular expression tests"""
-        body = """
-        <p><input name='a' value='1'/><input name='b' value='2'/></p>
-        <div class="links">
-        <a href="/first.html">first link</a>
-        <a href="/second.html">second link</a>
-        <a href="http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml">EXSLT match example</a>
-        </div>
-        """
-        response = TextResponse(url="http://example.com", body=body)
-        sel = self.sscls(response)
-
-        # re:test()
-        self.assertEqual(
-            sel.xpath(
-                '//input[re:test(@name, "[A-Z]+", "i")]').extract(),
-            [x.extract() for x in sel.xpath('//input[re:test(@name, "[A-Z]+", "i")]')])
-        self.assertEqual(
-            [x.extract()
-             for x in sel.xpath(
-                 '//a[re:test(@href, "\.html$")]/text()')],
-            [u'first link', u'second link'])
-        self.assertEqual(
-            [x.extract()
-             for x in sel.xpath(
-                 '//a[re:test(@href, "first")]/text()')],
-            [u'first link'])
-        self.assertEqual(
-            [x.extract()
-             for x in sel.xpath(
-                 '//a[re:test(@href, "second")]/text()')],
-            [u'second link'])
-
-
-        # re:match() is rather special: it returns a node-set of <match> nodes
-        #[u'<match>http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml</match>',
-        #u'<match>http</match>',
-        #u'<match>www.bayes.co.uk</match>',
-        #u'<match></match>',
-        #u'<match>/xml/index.xml?/xml/utils/rechecker.xml</match>']
-        self.assertEqual(
-            sel.xpath('re:match(//a[re:test(@href, "\.xml$")]/@href,'
-                      '"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(),
-            [u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml',
-             u'http',
-             u'www.bayes.co.uk',
-             u'',
-             u'/xml/index.xml?/xml/utils/rechecker.xml'])
-
-
-
-        # re:replace()
-        self.assertEqual(
-            sel.xpath('re:replace(//a[re:test(@href, "\.xml$")]/@href,'
-                      '"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(),
-            [u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html'])
-
-    def test_set(self):
-        """EXSLT set manipulation tests"""
-        # microdata example from http://schema.org/Event
-        body="""
-        <div itemscope itemtype="http://schema.org/Event">
-          <a itemprop="url" href="nba-miami-philidelphia-game3.html">
-          NBA Eastern Conference First Round Playoff Tickets:
-          <span itemprop="name"> Miami Heat at Philadelphia 76ers - Game 3 (Home Game 1) </span>
-          </a>
-
-          <meta itemprop="startDate" content="2016-04-21T20:00">
-            Thu, 04/21/16
-            8:00 p.m.
-
-          <div itemprop="location" itemscope itemtype="http://schema.org/Place">
-            <a itemprop="url" href="wells-fargo-center.html">
-            Wells Fargo Center
-            </a>
-            <div itemprop="address" itemscope itemtype="http://schema.org/PostalAddress">
-              <span itemprop="addressLocality">Philadelphia</span>,
-              <span itemprop="addressRegion">PA</span>
-            </div>
-          </div>
-
-          <div itemprop="offers" itemscope itemtype="http://schema.org/AggregateOffer">
-            Priced from: <span itemprop="lowPrice">$35</span>
-            <span itemprop="offerCount">1938</span> tickets left
-          </div>
-        </div>
-        """
-        response = TextResponse(url="http://example.com", body=body)
-        sel = self.sscls(response)
-
-        self.assertEqual(
-            sel.xpath('''//div[@itemtype="http://schema.org/Event"]
-                            //@itemprop''').extract(),
-            [u'url',
-             u'name',
-             u'startDate',
-             u'location',
-             u'url',
-             u'address',
-             u'addressLocality',
-             u'addressRegion',
-             u'offers',
-             u'lowPrice',
-             u'offerCount']
-        )
-
-        self.assertEqual(sel.xpath('''
-                set:difference(//div[@itemtype="http://schema.org/Event"]
-                                    //@itemprop,
-                               //div[@itemtype="http://schema.org/Event"]
-                                    //*[@itemscope]/*/@itemprop)''').extract(),
-                         [u'url', u'name', u'startDate', u'location', u'offers'])
+            self.assertTrue(isinstance(usel, XPathSelector))
\ No newline at end of file
diff --git a/tests/test_selector_csstranslator.py b/tests/test_selector_csstranslator.py
index 1bc8882f8..2d82fcba7 100644
--- a/tests/test_selector_csstranslator.py
+++ b/tests/test_selector_csstranslator.py
@@ -1,153 +1,22 @@
 """
 Selector tests for cssselect backend
 """
+import warnings
 from twisted.trial import unittest
-from scrapy.http import HtmlResponse
-from scrapy.selector.csstranslator import ScrapyHTMLTranslator
-from scrapy.selector import Selector
-from cssselect.parser import SelectorSyntaxError
-from cssselect.xpath import ExpressionError
+from scrapy.selector.csstranslator import (
+    ScrapyHTMLTranslator,
+    ScrapyGenericTranslator,
+    ScrapyXPathExpr
+)
 
 
-HTMLBODY = b'''
-<html>
-<body>
-<div>
- <a id="name-anchor" name="foo"></a>
- <a id="tag-anchor" rel="tag" href="http://localhost/foo">link</a>
- <a id="nofollow-anchor" rel="nofollow" href="https://example.org"> link</a>
- <p id="paragraph">
-   lorem ipsum text
-   <b id="p-b">hi</b> <em id="p-em">there</em>
-   <b id="p-b2">guy</b>
-   <input type="checkbox" id="checkbox-unchecked" />
-   <input type="checkbox" id="checkbox-disabled" disabled="" />
-   <input type="text" id="text-checked" checked="checked" />
-   <input type="hidden" />
-   <input type="hidden" disabled="disabled" />
-   <input type="checkbox" id="checkbox-checked" checked="checked" />
-   <input type="checkbox" id="checkbox-disabled-checked"
-          disabled="disabled" checked="checked" />
-   <fieldset id="fieldset" disabled="disabled">
-     <input type="checkbox" id="checkbox-fieldset-disabled" />
-     <input type="hidden" />
-   </fieldset>
- </p>
- <map name="dummymap">
-   <area shape="circle" coords="200,250,25" href="foo.html" id="area-href" />
-   <area shape="default" id="area-nohref" />
- </map>
-</div>
-<div class="cool-footer" id="foobar-div" foobar="ab bc cde">
-    <span id="foobar-span">foo ter</span>
-</div>
-</body></html>
-'''
+class DeprecatedClassesTest(unittest.TestCase):
+
+    def test_deprecated_warnings(self):
+        for cls in [ScrapyHTMLTranslator, ScrapyGenericTranslator, ScrapyXPathExpr]:
+            with warnings.catch_warnings(record=True) as w:
+                obj = cls()
+                self.assertIn('%s is deprecated' % cls.__name__, str(w[-1].message),
+                              'Missing deprecate warning for %s' % cls.__name__)
 
 
-class TranslatorMixinTest(unittest.TestCase):
-
-    tr_cls = ScrapyHTMLTranslator
-
-    def setUp(self):
-        self.tr = self.tr_cls()
-        self.c2x = self.tr.css_to_xpath
-
-    def test_attr_function(self):
-        cases = [
-            ('::attr(name)', u'descendant-or-self::*/@name'),
-            ('a::attr(href)', u'descendant-or-self::a/@href'),
-            ('a ::attr(img)', u'descendant-or-self::a/descendant-or-self::*/@img'),
-            ('a > ::attr(class)', u'descendant-or-self::a/*/@class'),
-        ]
-        for css, xpath in cases:
-            self.assertEqual(self.c2x(css), xpath, css)
-
-    def test_attr_function_exception(self):
-        cases = [
-            ('::attr(12)', ExpressionError),
-            ('::attr(34test)', ExpressionError),
-            ('::attr(@href)', SelectorSyntaxError),
-        ]
-        for css, exc in cases:
-            self.assertRaises(exc, self.c2x, css)
-
-    def test_text_pseudo_element(self):
-        cases = [
-            ('::text', u'descendant-or-self::text()'),
-            ('p::text', u'descendant-or-self::p/text()'),
-            ('p ::text', u'descendant-or-self::p/descendant-or-self::text()'),
-            ('#id::text', u"descendant-or-self::*[@id = 'id']/text()"),
-            ('p#id::text', u"descendant-or-self::p[@id = 'id']/text()"),
-            ('p#id ::text', u"descendant-or-self::p[@id = 'id']/descendant-or-self::text()"),
-            ('p#id > ::text', u"descendant-or-self::p[@id = 'id']/*/text()"),
-            ('p#id ~ ::text', u"descendant-or-self::p[@id = 'id']/following-sibling::*/text()"),
-            ('a[href]::text', u'descendant-or-self::a[@href]/text()'),
-            ('a[href] ::text', u'descendant-or-self::a[@href]/descendant-or-self::text()'),
-            ('p::text, a::text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"),
-        ]
-        for css, xpath in cases:
-            self.assertEqual(self.c2x(css), xpath, css)
-
-    def test_pseudo_function_exception(self):
-        cases = [
-            ('::attribute(12)', ExpressionError),
-            ('::text()', ExpressionError),
-            ('::attr(@href)', SelectorSyntaxError),
-        ]
-        for css, exc in cases:
-            self.assertRaises(exc, self.c2x, css)
-
-    def test_unknown_pseudo_element(self):
-        cases = [
-            ('::text-node', ExpressionError),
-        ]
-        for css, exc in cases:
-            self.assertRaises(exc, self.c2x, css)
-
-    def test_unknown_pseudo_class(self):
-        cases = [
-            (':text', ExpressionError),
-            (':attribute(name)', ExpressionError),
-        ]
-        for css, exc in cases:
-            self.assertRaises(exc, self.c2x, css)
-
-
-class CSSSelectorTest(unittest.TestCase):
-
-    sscls = Selector
-
-    def setUp(self):
-        self.htmlresponse = HtmlResponse('http://example.com', body=HTMLBODY)
-        self.sel = self.sscls(self.htmlresponse)
-
-    def x(self, *a, **kw):
-        return [v.strip() for v in self.sel.css(*a, **kw).extract() if v.strip()]
-
-    def test_selector_simple(self):
-        for x in self.sel.css('input'):
-            self.assertTrue(isinstance(x, self.sel.__class__), x)
-        self.assertEqual(self.sel.css('input').extract(),
-                         [x.extract() for x in self.sel.css('input')])
-
-    def test_text_pseudo_element(self):
-        self.assertEqual(self.x('#p-b2'), [u'<b id="p-b2">guy</b>'])
-        self.assertEqual(self.x('#p-b2::text'), [u'guy'])
-        self.assertEqual(self.x('#p-b2 ::text'), [u'guy'])
-        self.assertEqual(self.x('#paragraph::text'), [u'lorem ipsum text'])
-        self.assertEqual(self.x('#paragraph ::text'), [u'lorem ipsum text', u'hi', u'there', u'guy'])
-        self.assertEqual(self.x('p::text'), [u'lorem ipsum text'])
-        self.assertEqual(self.x('p ::text'), [u'lorem ipsum text', u'hi', u'there', u'guy'])
-
-    def test_attribute_function(self):
-        self.assertEqual(self.x('#p-b2::attr(id)'), [u'p-b2'])
-        self.assertEqual(self.x('.cool-footer::attr(class)'), [u'cool-footer'])
-        self.assertEqual(self.x('.cool-footer ::attr(id)'), [u'foobar-div', u'foobar-span'])
-        self.assertEqual(self.x('map[name="dummymap"] ::attr(shape)'), [u'circle', u'default'])
-
-    def test_nested_selector(self):
-        self.assertEqual(self.sel.css('p').css('b::text').extract(),
-                         [u'hi', u'guy'])
-        self.assertEqual(self.sel.css('div').css('area:last-child').extract(),
-                         [u'<area shape="default" id="area-nohref">'])

From 8ef5aa2ffce4b2b8955b58b532920098f6fe8ea4 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 11 Aug 2015 13:58:53 -0300
Subject: [PATCH 0491/2974] using bytes for response body in tests

---
 tests/test_selector.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/tests/test_selector.py b/tests/test_selector.py
index 4806bb90b..2d6d8c439 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -1,9 +1,6 @@
-import re
 import warnings
 import weakref
-import six
 from twisted.trial import unittest
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import TextResponse, HtmlResponse, XmlResponse
 from scrapy.selector import Selector
 from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector
@@ -14,7 +11,7 @@ class SelectorTestCase(unittest.TestCase):
 
     def test_simple_selection(self):
         """Simple selector tests"""
-        body = u"<p><input name='a'value='1'/><input name='b'value='2'/></p>"
+        body = b"<p><input name='a'value='1'/><input name='b'value='2'/></p>"
         response = TextResponse(url="http://example.com", body=body, encoding='utf-8')
         sel = Selector(response)
 
@@ -53,7 +50,7 @@ class SelectorTestCase(unittest.TestCase):
             self.assertIn('Ignoring deprecated `_root` argument', str(w[-1].message))
 
     def test_flavor_detection(self):
-        text = u'<div><img src="a.jpg"><p>Hello</div>'
+        text = b'<div><img src="a.jpg"><p>Hello</div>'
         sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
         self.assertEqual(sel.type, 'xml')
         self.assertEqual(sel.xpath("//div").extract(),
@@ -86,7 +83,7 @@ class SelectorTestCase(unittest.TestCase):
     def test_badly_encoded_body(self):
         # \xe9 alone isn't valid utf8 sequence
         r1 = TextResponse('http://www.example.com', \
-                          body=u'<html><p>an Jos\xe9 de</p><html>', \
+                          body=b'<html><p>an Jos\xe9 de</p><html>', \
                           encoding='utf-8')
         Selector(r1).xpath('//text()').extract()
 
@@ -100,7 +97,7 @@ class SelectorTestCase(unittest.TestCase):
 
 class DeprecatedXpathSelectorTest(unittest.TestCase):
 
-    text = u'<div><img src="a.jpg"><p>Hello</div>'
+    text = '<div><img src="a.jpg"><p>Hello</div>'
 
     def test_warnings_xpathselector(self):
         cls = XPathSelector
@@ -178,4 +175,4 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
             self.assertTrue(isinstance(sel, Selector))
             self.assertTrue(isinstance(usel, Selector))
             self.assertTrue(isinstance(sel, XPathSelector))
-            self.assertTrue(isinstance(usel, XPathSelector))
\ No newline at end of file
+            self.assertTrue(isinstance(usel, XPathSelector))

From e50610bd3a8c5a5bad3f19353ce5214f11333e74 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 11 Aug 2015 14:06:24 -0300
Subject: [PATCH 0492/2974] set base_url in kwargs to be fully backward
 compatible

---
 scrapy/selector/unified.py | 1 +
 tests/test_selector.py     | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index a8f80c84d..25cf6c98d 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -46,6 +46,7 @@ class Selector(ParselSelector, object_ref):
 
         if response is not None:
             text = response.body_as_unicode()
+            kwargs.setdefault('base_url', response.url)
 
         self.response = response
         super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 2d6d8c439..19b807a3f 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -33,6 +33,13 @@ class SelectorTestCase(unittest.TestCase):
         self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
                          [u'12'])
 
+    def test_root_base_url(self):
+        body = b'<html><form action="/path"><input name="a" /></form></html>'
+        url = "http://example.com"
+        response = TextResponse(url=url, body=body, encoding='utf-8')
+        sel = Selector(response)
+        self.assertEqual(url, sel.root.base)
+
     def test_deprecated_root_argument(self):
         with warnings.catch_warnings(record=True) as w:
             root = etree.fromstring(u'<html/>')

From e2f31f3018c0037f65982209c22f93b80a5d6e7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Tue, 11 Aug 2015 14:09:49 -0300
Subject: [PATCH 0493/2974] explicit close file on file:// scheme handler

---
 scrapy/core/downloader/handlers/file.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py
index 5a63e9d08..9346ce08d 100644
--- a/scrapy/core/downloader/handlers/file.py
+++ b/scrapy/core/downloader/handlers/file.py
@@ -10,6 +10,7 @@ class FileDownloadHandler(object):
     @defers
     def download_request(self, request, spider):
         filepath = file_uri_to_path(request.url)
-        body = open(filepath, 'rb').read()
+        with open(filepath, 'rb') as fo:
+            body = fo.read()
         respcls = responsetypes.from_args(filename=filepath, body=body)
         return respcls(url=request.url, body=body)

From 766c2551527574936d3210188c9b95b3f8ea726a Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 11 Aug 2015 15:20:33 -0300
Subject: [PATCH 0494/2974] upgrade parsel and add shim for deprecated
 selectorlist methods

---
 requirements.txt           |  2 +-
 scrapy/selector/unified.py | 17 ++++++++++++++++-
 tests/test_selector.py     | 22 ++++++++++++++++++++++
 3 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index afefb9d33..368e9340b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,4 +7,4 @@ queuelib
 six>=1.5.2
 PyDispatcher>=2.0.5
 service_identity
-parsel>=0.9.3
+parsel>=0.9.5
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 25cf6c98d..dddf80c06 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -3,7 +3,7 @@ XPath selectors based on lxml
 """
 
 import warnings
-from parsel import Selector as ParselSelector, SelectorList
+from parsel import Selector as ParselSelector, SelectorList as ParselSelectorList
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.python import to_bytes
 from scrapy.http import HtmlResponse, XmlResponse
@@ -26,9 +26,24 @@ def _response_from_text(text, st):
               body=to_bytes(text, 'utf-8'))
 
 
+class SelectorList(ParselSelectorList, object_ref):
+    @deprecated(use_instead='.extract()')
+    def extract_unquoted(self):
+        return [x.extract_unquoted() for x in self]
+
+    @deprecated(use_instead='.xpath()')
+    def x(self, xpath):
+        return self.select(xpath)
+
+    @deprecated(use_instead='.xpath()')
+    def select(self, xpath):
+        return self.xpath(xpath)
+
+
 class Selector(ParselSelector, object_ref):
 
     __slots__ = ['response']
+    selectorlist_cls = SelectorList
 
     def __init__(self, response=None, text=None, type=None, root=None, _root=None, **kwargs):
         st = _st(response, type or self._default_type)
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 19b807a3f..141455b66 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -101,6 +101,28 @@ class SelectorTestCase(unittest.TestCase):
         assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
             x.__class__.__name__
 
+    def test_deprecated_selector_methods(self):
+        sel = Selector(TextResponse(url="http://example.com", body=b'<p>some text</p>'))
+
+        with warnings.catch_warnings(record=True) as w:
+            sel.select('//p')
+            self.assertSubstring('Use .xpath() instead', str(w[-1].message))
+
+        with warnings.catch_warnings(record=True) as w:
+            sel.extract_unquoted()
+            self.assertSubstring('Use .extract() instead', str(w[-1].message))
+
+    def test_deprecated_selectorlist_methods(self):
+        sel = Selector(TextResponse(url="http://example.com", body=b'<p>some text</p>'))
+
+        with warnings.catch_warnings(record=True) as w:
+            sel.xpath('//p').select('.')
+            self.assertSubstring('Use .xpath() instead', str(w[-1].message))
+
+        with warnings.catch_warnings(record=True) as w:
+            sel.xpath('//p').extract_unquoted()
+            self.assertSubstring('Use .extract() instead', str(w[-1].message))
+
 
 class DeprecatedXpathSelectorTest(unittest.TestCase):
 

From a5abd19e846521b19886b4e7e2ac82f0c401b088 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 11 Aug 2015 15:58:29 -0300
Subject: [PATCH 0495/2974] make Parsel's Selector more private, remove direct
 dependency of ParselSelectorList

---
 scrapy/selector/unified.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index dddf80c06..5d77f7624 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -3,7 +3,7 @@ XPath selectors based on lxml
 """
 
 import warnings
-from parsel import Selector as ParselSelector, SelectorList as ParselSelectorList
+from parsel import Selector as _ParselSelector
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.python import to_bytes
 from scrapy.http import HtmlResponse, XmlResponse
@@ -26,7 +26,7 @@ def _response_from_text(text, st):
               body=to_bytes(text, 'utf-8'))
 
 
-class SelectorList(ParselSelectorList, object_ref):
+class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
     @deprecated(use_instead='.extract()')
     def extract_unquoted(self):
         return [x.extract_unquoted() for x in self]
@@ -40,7 +40,7 @@ class SelectorList(ParselSelectorList, object_ref):
         return self.xpath(xpath)
 
 
-class Selector(ParselSelector, object_ref):
+class Selector(_ParselSelector, object_ref):
 
     __slots__ = ['response']
     selectorlist_cls = SelectorList

From 1ed6a01c4b4764ce55ca199049ee69338d4a808a Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 12 Aug 2015 19:13:54 +0500
Subject: [PATCH 0496/2974] Add Python 3 porting status badge to the README

---
 README.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/README.rst b/README.rst
index 6ca5deb49..7006925ae 100644
--- a/README.rst
+++ b/README.rst
@@ -13,6 +13,11 @@ Scrapy
 .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
    :target: https://pypi.python.org/pypi/Scrapy
    :alt: Wheel Status
+   
+.. image:: http://static.scrapy.org/py3progress/badge.svg
+   :target: https://github.com/scrapy/scrapy/wiki/Python-3-Porting
+   :alt: Python 3 Porting Status
+
 
 Overview
 ========

From 4ce0f53f7edc558ce70ef7b08cdcfd57fcd8b9fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 12 Aug 2015 14:21:55 -0300
Subject: [PATCH 0497/2974] Replace usage of deprecated class by its parsel\'s
 counterpart

---
 scrapy/linkextractors/__init__.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 8567dbb76..bb799e572 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -6,12 +6,14 @@ This package contains a collection of Link Extractors.
 For more info see docs/topics/link-extractors.rst
 """
 import re
-from six.moves.urllib.parse import urlparse
 
-from scrapy.selector.csstranslator import ScrapyHTMLTranslator
-from scrapy.utils.url import url_is_from_any_domain
-from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension
+from six.moves.urllib.parse import urlparse
+from parsel.csstranslator import HTMLTranslator
+
 from scrapy.utils.misc import arg_to_iter
+from scrapy.utils.url import (
+    canonicalize_url, url_is_from_any_domain, url_has_any_extension,
+)
 
 
 # common file extensions that are not followed if they occur in links
@@ -42,7 +44,7 @@ _is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'fil
 
 class FilteringLinkExtractor(object):
 
-    _csstranslator = ScrapyHTMLTranslator()
+    _csstranslator = HTMLTranslator()
 
     def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
                  restrict_xpaths, canonicalize, deny_extensions, restrict_css):

From 7f634598b6df1e96ceba41a69f8b8cd8cf6453bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 12 Aug 2015 13:13:07 -0300
Subject: [PATCH 0498/2974] Add openssl version to version command

---
 scrapy/commands/version.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index a48e98f75..81433a657 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -3,6 +3,7 @@ import sys
 import platform
 
 import twisted
+import OpenSSL
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -26,11 +27,22 @@ class Command(ScrapyCommand):
             import lxml.etree
             lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
             libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
-            print("Scrapy  : %s" % scrapy.__version__)
-            print("lxml    : %s" % lxml_version)
-            print("libxml2 : %s" % libxml2_version)
-            print("Twisted : %s" % twisted.version.short())
-            print("Python  : %s" % sys.version.replace("\n", "- "))
-            print("Platform: %s" % platform.platform())
+            print("Scrapy    : %s" % scrapy.__version__)
+            print("lxml      : %s" % lxml_version)
+            print("libxml2   : %s" % libxml2_version)
+            print("Twisted   : %s" % twisted.version.short())
+            print("Python    : %s" % sys.version.replace("\n", "- "))
+            print("pyOpenSSL : %s" % self._get_openssl_version())
+            print("Platform  : %s" % platform.platform())
         else:
             print("Scrapy %s" % scrapy.__version__)
+
+    def _get_openssl_version(self):
+        try:
+            openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\
+                .decode('ascii', errors='replace')
+        # pyOpenSSL 0.12 does not expose openssl version
+        except AttributeError:
+            openssl = 'Unknown OpenSSL version'
+
+        return '{} ({})'.format(OpenSSL.version.__version__, openssl)

From 108c3e63a5ec1b251134dc4742f87d9933e9c619 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 12 Aug 2015 16:40:04 -0300
Subject: [PATCH 0499/2974] Add coverage report trough codecov.io

---
 .coveragerc                | 18 +++++++++++++++++-
 .travis.yml                |  5 ++++-
 README.rst                 |  4 ++++
 tests/requirements-py3.txt |  1 +
 tests/requirements.txt     |  1 +
 tox.ini                    |  4 ++--
 6 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/.coveragerc b/.coveragerc
index 3baaf659a..3105409ba 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -1,3 +1,19 @@
 [run]
+branch = true
 include = scrapy/*
-omit = scrapy/xlib*,scrapy/tests*
+omit =
+  tests/*
+  scrapy/xlib/*
+  scrapy/conf.py
+  scrapy/stats.py
+  scrapy/project.py
+  scrapy/utils/decorator.py
+  scrapy/statscol.py
+  scrapy/squeue.py
+  scrapy/log.py
+  scrapy/dupefilter.py
+  scrapy/command.py
+  scrapy/linkextractor.py
+  scrapy/spider.py
+  scrapy/contrib/*
+  scrapy/contrib_exp/*
diff --git a/.travis.yml b/.travis.yml
index d714c0ff8..aee08919e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,8 +7,11 @@ env:
  - TOXENV=py33
  - TOXENV=docs
 install:
- - pip install -U tox twine wheel
+ - pip install -U tox twine wheel codecov coveralls
 script: tox
+after_success:
+  - codecov
+  - coveralls
 notifications:
   irc:
     use_notice: true
diff --git a/README.rst b/README.rst
index 7006925ae..ec3604875 100644
--- a/README.rst
+++ b/README.rst
@@ -18,6 +18,10 @@ Scrapy
    :target: https://github.com/scrapy/scrapy/wiki/Python-3-Porting
    :alt: Python 3 Porting Status
 
+.. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg
+   :target: http://codecov.io/github/scrapy/scrapy?branch=master
+   :alt: Coverage report
+
 
 Overview
 ========
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 8f9e22f0b..e37a2e68d 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,4 +1,5 @@
 pytest>=2.6.0
 pytest-twisted
+pytest-cov
 testfixtures
 jmespath
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 4f1520021..452a884e0 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -3,5 +3,6 @@ mock
 mitmproxy==0.10.1
 netlib==0.10.1
 pytest-twisted
+pytest-cov
 jmespath
 testfixtures
diff --git a/tox.ini b/tox.ini
index 9e5b2fa13..806219898 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,7 +15,7 @@ deps =
     leveldb
     -rtests/requirements.txt
 commands =
-    py.test {posargs:scrapy tests}
+    py.test --cov=scrapy {posargs:scrapy tests}
 
 [testenv:precise]
 basepython = python2.7
@@ -34,7 +34,7 @@ basepython = python2.7
 commands =
     pip install -U https://github.com/scrapy/w3lib/archive/master.zip#egg=w3lib
     pip install -U https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib
-    py.test {posargs:scrapy tests}
+    py.test --cov=scrapy {posargs:scrapy tests}
 
 [testenv:py33]
 basepython = python3.3

From cb9577a6d4e67fd361ec18d6ff49cc85001d1350 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 13 Aug 2015 19:02:36 -0300
Subject: [PATCH 0500/2974] Do not be verbose with coverage report by default

---
 tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index 806219898..8b4c9e875 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,7 +15,7 @@ deps =
     leveldb
     -rtests/requirements.txt
 commands =
-    py.test --cov=scrapy {posargs:scrapy tests}
+    py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
 
 [testenv:precise]
 basepython = python2.7
@@ -34,7 +34,7 @@ basepython = python2.7
 commands =
     pip install -U https://github.com/scrapy/w3lib/archive/master.zip#egg=w3lib
     pip install -U https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib
-    py.test --cov=scrapy {posargs:scrapy tests}
+    py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
 
 [testenv:py33]
 basepython = python3.3

From 54f98378042455f1c21848b19d68e65dd07f0ec1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 13 Aug 2015 19:02:51 -0300
Subject: [PATCH 0501/2974] Document how to get coverage report

---
 docs/contributing.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index ad9a3805a..be34bf98c 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -154,6 +154,13 @@ To run a specific test (say ``tests/test_loader.py``) use:
 
     ``tox -- tests/test_loader.py``
 
+To see coverage report install `coverage`_ (``pip install coverage``) and run:
+
+    ``coverage report``
+
+see output of ``coverage --help`` for more options like html or xml report.
+
+.. _coverage: https://pypi.python.org/pypi/coverage
 
 Writing tests
 -------------

From 43fc2f23b80083f89ae1e982bfdee5d4e0322556 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 13 Aug 2015 19:30:06 -0300
Subject: [PATCH 0502/2974] Increase coverage of version command

---
 tests/test_command_version.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/test_command_version.py b/tests/test_command_version.py
index 420713d87..37e1f2543 100644
--- a/tests/test_command_version.py
+++ b/tests/test_command_version.py
@@ -14,4 +14,16 @@ class VersionTest(ProcessTest, unittest.TestCase):
     def test_output(self):
         encoding = getattr(sys.stdout, 'encoding') or 'utf-8'
         _, out, _ = yield self.execute([])
-        self.assertEqual(out.strip().decode(encoding), "Scrapy %s" % scrapy.__version__)
+        self.assertEqual(
+            out.strip().decode(encoding),
+            "Scrapy %s" % scrapy.__version__,
+        )
+
+    @defer.inlineCallbacks
+    def test_verbose_output(self):
+        encoding = getattr(sys.stdout, 'encoding') or 'utf-8'
+        _, out, _ = yield self.execute(['-v'])
+        headers = [l.partition(":")[0].strip()
+                   for l in out.strip().decode(encoding).splitlines()]
+        self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', 'Twisted',
+                                   'Python', 'pyOpenSSL', 'Platform'])

From 27077d2b6cb398cfb54d58c061e233b929868ab7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 13 Aug 2015 20:52:38 -0300
Subject: [PATCH 0503/2974] Forward coverage envvars needed to cover subprocess

---
 tests/__init__.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/__init__.py b/tests/__init__.py
index d017afad4..d940f28ea 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -13,6 +13,14 @@ os.environ['http_proxy'] = ''
 os.environ['https_proxy'] = ''
 os.environ['ftp_proxy'] = ''
 
+# Absolutize paths to coverage config and output file because tests that
+# spawn subprocesses also changes current working directory.
+_sourceroot = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if 'COV_CORE_CONFIG' in os.environ:
+    os.environ['COVERAGE_FILE'] = os.path.join(_sourceroot, '.coverage')
+    os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot,
+                                                 os.environ['COV_CORE_CONFIG'])
+
 try:
     import unittest.mock as mock
 except ImportError:

From 1b4fd3a8dffbbfc0207d707cd11ca84d11e76064 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sun, 12 Jul 2015 18:27:41 +0000
Subject: [PATCH 0504/2974] Support anonymous connections in S3DownloadHandler

Also consider any unknown keyword args for S3DownloadHandler as
arguments to pass on to S3Connection (e.g. proxy settings).
---
 scrapy/core/downloader/handlers/s3.py | 11 +++++++++--
 tests/test_downloader_handlers.py     | 21 +++++++++++++++++++--
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 311815b70..38cfd1e10 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -35,7 +35,7 @@ def get_s3_connection():
 class S3DownloadHandler(object):
 
     def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
-            httpdownloadhandler=HTTPDownloadHandler):
+            httpdownloadhandler=HTTPDownloadHandler, **kw):
 
         _S3Connection = get_s3_connection()
         if _S3Connection is None:
@@ -46,8 +46,15 @@ class S3DownloadHandler(object):
         if not aws_secret_access_key:
             aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
 
+        # If no credentials could be found anywhere,
+        # consider this an anonymous connection request by default;
+        # unless 'anon' was set explicitly (True/False).
+        anon = kw.get('anon', None)
+        if anon is None and not aws_access_key_id and not aws_secret_access_key:
+            kw['anon'] = True
+
         try:
-            self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key)
+            self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key, **kw)
         except Exception as ex:
             raise NotConfigured(str(ex))
         self._download_http = httpdownloadhandler(settings).download_request
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index e4d957d8e..8280b21aa 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -403,6 +403,23 @@ class HttpDownloadHandlerMock(object):
     def download_request(self, request, spider):
         return request
 
+class S3AnonTestCase(unittest.TestCase):
+    skip = 'boto' not in optional_features and 'missing boto library'
+
+    def setUp(self):
+        self.s3reqh = S3DownloadHandler(Settings(),
+                httpdownloadhandler=HttpDownloadHandlerMock,
+                #anon=True, # is implicit
+        )
+        self.download_request = self.s3reqh.download_request
+        self.spider = Spider('foo')
+
+    def test_anon_request(self):
+        req = Request('s3://aws-publicdatasets/')
+        httpreq = self.download_request(req, self.spider)
+        self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True)
+        self.assertEqual(self.s3reqh.conn.anon, True)
+
 class S3TestCase(unittest.TestCase):
     download_handler_cls = S3DownloadHandler
     try:
@@ -420,8 +437,8 @@ class S3TestCase(unittest.TestCase):
     AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'
 
     def setUp(self):
-        s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID, \
-                self.AWS_SECRET_ACCESS_KEY, \
+        s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID,
+                self.AWS_SECRET_ACCESS_KEY,
                 httpdownloadhandler=HttpDownloadHandlerMock)
         self.download_request = s3reqh.download_request
         self.spider = Spider('foo')

From de15fcdf333397d0f1fb1830d221c48e2df2580e Mon Sep 17 00:00:00 2001
From: Andrew Scorpil <webdev@scorpil.com>
Date: Fri, 24 Oct 2014 15:58:12 +0300
Subject: [PATCH 0505/2974] [LinkExtractors] Ignore bogus links

(rebased the code for scrapy 1.0 and made a few code improvements --nyov)
---
 scrapy/linkextractors/htmlparser.py |  5 ++-
 scrapy/linkextractors/lxmlhtml.py   | 12 ++++---
 scrapy/linkextractors/regex.py      | 15 +++++++--
 scrapy/linkextractors/sgml.py       |  5 ++-
 tests/test_linkextractors.py        | 52 +++++++++++++++++++++++++++++
 5 files changed, 80 insertions(+), 9 deletions(-)

diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 202340f53..dcc261b31 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -41,7 +41,10 @@ class HtmlParserLinkExtractor(HTMLParser):
         for link in links:
             if isinstance(link.url, unicode):
                 link.url = link.url.encode(response_encoding)
-            link.url = urljoin(base_url, link.url)
+            try:
+                link.url = urljoin(base_url, link.url)
+            except ValueError:
+                continue
             link.url = safe_url_string(link.url, response_encoding)
             link.text = link.text.decode(response_encoding)
             ret.append(link)
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 606a45212..e9fa521f3 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -49,10 +49,14 @@ class LxmlParserLinkExtractor(object):
         # hacky way to get the underlying lxml parsed document
         for el, attr, attr_val in self._iter_links(selector.root):
             # pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
-            attr_val = urljoin(base_url, attr_val)
-            url = self.process_attr(attr_val)
-            if url is None:
-                continue
+            try:
+                attr_val = urljoin(base_url, attr_val)
+            except ValueError:
+                continue # skipping bogus links
+            else:
+                url = self.process_attr(attr_val)
+                if url is None:
+                    continue
             if isinstance(url, unicode):
                 url = url.encode(response_encoding)
             # to fix relative links after process_value
diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py
index 905eb8969..b6f8d5d30 100644
--- a/scrapy/linkextractors/regex.py
+++ b/scrapy/linkextractors/regex.py
@@ -14,16 +14,25 @@ def clean_link(link_text):
     """Remove leading and trailing whitespace and punctuation"""
     return link_text.strip("\t\r\n '\"")
 
+
 class RegexLinkExtractor(SgmlLinkExtractor):
     """High performant link extractor"""
 
     def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
+        def clean_text(text):
+            return replace_escape_chars(remove_tags(text.decode(response_encoding))).strip()
+
+        def clean_url(url):
+            clean_url = ''
+            try:
+                clean_url = urljoin(base_url, replace_entities(clean_link(url.decode(response_encoding))))
+            except ValueError:
+                pass
+            return clean_url
+
         if base_url is None:
             base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
 
-        clean_url = lambda u: urljoin(base_url, replace_entities(clean_link(u.decode(response_encoding))))
-        clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip()
-
         links_text = linkre.findall(response_text)
         return [Link(clean_url(url).encode(response_encoding),
                      clean_text(text))
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 4a6a24254..acecd9c0d 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -42,7 +42,10 @@ class BaseSgmlLinkExtractor(SGMLParser):
         for link in self.links:
             if isinstance(link.url, unicode):
                 link.url = link.url.encode(response_encoding)
-            link.url = urljoin(base_url, link.url)
+            try:
+                link.url = urljoin(base_url, link.url)
+            except ValueError:
+                continue
             link.url = safe_url_string(link.url, response_encoding)
             link.text = to_unicode(link.text, response_encoding, errors='replace').strip()
             ret.append(link)
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 948289f8f..d78b25f25 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -491,10 +491,36 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
                           Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)]
                         )
 
+    def test_link_wrong_href(self):
+        html = """
+        <a href="http://example.org/item1.html">Item 1</a>
+        <a href="http://[example.org/item2.html">Item 2</a>
+        <a href="http://example.org/item3.html">Item 3</a>
+        """
+        response = HtmlResponse("http://example.org/index.html", body=html)
+        lx = self.extractor_cls()
+        self.assertEqual([link for link in lx.extract_links(response)], [
+            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
+            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
+        ])
+
 
 class LxmlLinkExtractorTestCase(SgmlLinkExtractorTestCase):
     extractor_cls = LxmlLinkExtractor
 
+    def test_link_wrong_href(self):
+        html = """
+        <a href="http://example.org/item1.html">Item 1</a>
+        <a href="http://[example.org/item2.html">Item 2</a>
+        <a href="http://example.org/item3.html">Item 3</a>
+        """
+        response = HtmlResponse("http://example.org/index.html", body=html)
+        lx = self.extractor_cls()
+        self.assertEqual([link for link in lx.extract_links(response)], [
+            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
+            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
+        ])
+
 
 class HtmlParserLinkExtractorTestCase(unittest.TestCase):
 
@@ -512,6 +538,19 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase):
                           Link(url='http://www.google.com/something', text=u''),
                           Link(url='http://example.com/innertag.html', text=u'inner tag'),])
 
+    def test_link_wrong_href(self):
+        html = """
+        <a href="http://example.org/item1.html">Item 1</a>
+        <a href="http://[example.org/item2.html">Item 2</a>
+        <a href="http://example.org/item3.html">Item 3</a>
+        """
+        response = HtmlResponse("http://example.org/index.html", body=html)
+        lx = HtmlParserLinkExtractor()
+        self.assertEqual([link for link in lx.extract_links(response)], [
+            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
+            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
+        ])
+
 
 class RegexLinkExtractorTestCase(unittest.TestCase):
 
@@ -528,6 +567,19 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
                           Link(url='http://www.google.com/something', text=u''),
                           Link(url='http://example.com/innertag.html', text=u'inner tag'),])
 
+    def test_link_wrong_href(self):
+        html = """
+        <a href="http://example.org/item1.html">Item 1</a>
+        <a href="http://[example.org/item2.html">Item 2</a>
+        <a href="http://example.org/item3.html">Item 3</a>
+        """
+        response = HtmlResponse("http://example.org/index.html", body=html)
+        lx = RegexLinkExtractor()
+        self.assertEqual([link for link in lx.extract_links(response)], [
+            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
+            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
+        ])
+
 
 if __name__ == "__main__":
     unittest.main()

From 46ee1c6ee2084caa797da4b2cd441d7551be2dbe Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 19 Aug 2015 02:21:08 -0300
Subject: [PATCH 0506/2974] Reference Homebrew's homepage for installation
 instructions

---
 docs/intro/install.rst | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 246964d3f..918197e43 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -117,9 +117,7 @@ solutions:
   that doesn't conflict with the rest of your system. Here's how to do it using
   the `homebrew`_ package manager:
 
-  * Install `homebrew`_::
-
-      ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+  * Install `homebrew`_ following the instructions in http://brew.sh/
 
   * Update your ``PATH`` variable to state that homebrew packages should be
     used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly

From d3c3408cd46fd735ea6b2770f7d30c9e2b1ab04b Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Wed, 19 Aug 2015 03:19:27 -0300
Subject: [PATCH 0507/2974] Replace alternative OSX install workaround with
 virtualenv

---
 docs/intro/install.rst | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 918197e43..978d89920 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -138,21 +138,14 @@ solutions:
 
       brew update; brew upgrade python
 
-* *(Alternative)* Force system python to load the user installed packages
-  first:
+* *(Optional)* Install Scrapy inside an isolated python environment.
 
-  * Update your ``PYTHONPATH`` variable (Change ``.bashrc`` to ``.zshrc``
-    accordantly if you're using `zsh`_ as default shell)::
+  This method is a workaround for the above OS X issue, but it's an overall
+  good practice for managing dependencies and can complement the first method.
 
-      echo "export PYTHONPATH=/Library/Python/2.7/site-packages:$PYTHONPATH" >> ~/.bashrc
-
-  * Reload ``.bashrc`` to ensure the changes have taken place::
-
-      source ~/.bashrc
-
-  * Install ``pip``::
-
-     sudo easy_install pip
+  `virtualenv`_ is a tool you can use to create virtual environments in python.
+  We recommended reading a tutorial like
+  http://docs.python-guide.org/en/latest/dev/virtualenvs/ to get started.
 
 After any of these workarounds you should be able to install Scrapy::
 
@@ -168,3 +161,4 @@ After any of these workarounds you should be able to install Scrapy::
 .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
 .. _homebrew: http://brew.sh/
 .. _zsh: http://www.zsh.org/
+.. _virtualenv: https://virtualenv.pypa.io/en/latest/

From b8b1e8e5448a05415bba544c6725bc0eaf184165 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aivars=20Kalv=C4=81ns?= <aivars.kalvans@gmail.com>
Date: Wed, 19 Aug 2015 16:54:10 +0300
Subject: [PATCH 0508/2974] Make common practices sample code match the
 comments

---
 docs/topics/practices.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 7666a84cf..9ae34f423 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -61,7 +61,7 @@ project as example.
     process = CrawlerProcess(get_project_settings())
 
     # 'followall' is the name of one of the spiders of the project.
-    process.crawl('testspider', domain='scrapinghub.com')
+    process.crawl('followall', domain='scrapinghub.com')
     process.start() # the script will block here until the crawling is finished
 
 There's another Scrapy utility that provides more control over the crawling

From 509cc8d41e362e995cfcfaef43891490d0bdbf44 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Wed, 25 Mar 2015 10:24:07 +0000
Subject: [PATCH 0509/2974] Add support for bpython console.

Adds support for configuration of shells from scrapy.cfg
and SCRAPY_PYTHON_SHELL.

config snippet:

cat <<EOF >> ~/.scrapy.cfg
[settings]
# shell can be one of ipython, bpython or python;
# to be tried as the interactive python console
# (in above order, unless set here).
shell = python
EOF

(closes #270, #1100, #1301)
---
 docs/topics/commands.rst    |   5 +-
 docs/topics/shell.rst       |  15 ++++++
 scrapy/shell.py             |  26 +++++++++-
 scrapy/utils/console.py     | 100 +++++++++++++++++++++++++-----------
 tests/requirements.txt      |   3 ++
 tests/test_utils_console.py |  44 ++++++++++++++++
 6 files changed, 162 insertions(+), 31 deletions(-)
 create mode 100644 tests/test_utils_console.py

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 52c4fe446..16af52eea 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -16,6 +16,8 @@ accepts a different set of arguments and options.
 (The ``scrapy deploy`` command has been removed in 1.0 in favor of the
 standalone ``scrapyd-deploy``. See `Deploying your project`_.)
 
+.. _topics-config-settings:
+
 Configuration settings
 ======================
 
@@ -34,8 +36,9 @@ and project-wide settings will override all others, when defined.
 Scrapy also understands, and can be configured through, a number of environment
 variables. Currently these are:
 
-* ``SCRAPY_SETTINGS_MODULE`` (See :ref:`topics-settings-module-envvar`)
+* ``SCRAPY_SETTINGS_MODULE`` (see :ref:`topics-settings-module-envvar`)
 * ``SCRAPY_PROJECT``
+* ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`)
 
 .. _topics-project-structure:
 
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 9c9411d6d..327e22e90 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -17,6 +17,9 @@ spider, without having to run the spider to test every change.
 Once you get familiarized with the Scrapy shell, you'll see that it's an
 invaluable tool for developing and debugging your spiders.
 
+Configuring the shell
+=====================
+
 If you have `IPython`_ installed, the Scrapy shell will use it (instead of the
 standard Python console). The `IPython`_ console is much more powerful and
 provides smart auto-completion and colorized output, among other things.
@@ -25,8 +28,20 @@ We highly recommend you install `IPython`_, specially if you're working on
 Unix systems (where `IPython`_ excels). See the `IPython installation guide`_
 for more info.
 
+Scrapy also has support for `bpython`_, and will try to use it where `IPython`_
+is unavailable.
+
+Through scrapy's settings you can configure it to use any one of
+``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which
+are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment
+variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
+
+    [settings]
+    shell = bpython
+
 .. _IPython: http://ipython.org/
 .. _IPython installation guide: http://ipython.org/install.html
+.. _bpython: http://www.bpython-interpreter.org/
 
 Launch the shell
 ================
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 4142396ab..a60b45dbb 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -5,6 +5,7 @@ See documentation in docs/topics/shell.rst
 """
 from __future__ import print_function
 
+import os
 import signal
 import warnings
 
@@ -21,6 +22,8 @@ from scrapy.spiders import Spider
 from scrapy.utils.console import start_python_console
 from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
+from scrapy.utils.conf import get_config
+from scrapy.utils.console import DEFAULT_PYTHON_SHELLS
 
 
 class Shell(object):
@@ -52,7 +55,28 @@ class Shell(object):
         if self.code:
             print(eval(self.code, globals(), self.vars))
         else:
-            start_python_console(self.vars)
+            """
+            Detect interactive shell setting in scrapy.cfg
+            e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg
+            [settings]
+            # shell can be one of ipython, bpython or python;
+            # to be used as the interactive python console, if available.
+            # (default is ipython, fallbacks in the order listed above)
+            shell = python
+            """
+            cfg = get_config()
+            section, option = 'settings', 'shell'
+            env = os.environ.get('SCRAPY_PYTHON_SHELL')
+            shells = []
+            if env:
+                shells += env.strip().lower().split(',')
+            elif cfg.has_option(section, option):
+                shells += [cfg.get(section, option).strip().lower()]
+            else: # try all by default
+                shells += DEFAULT_PYTHON_SHELLS.keys()
+            # always add standard shell as fallback
+            shells += ['python']
+            start_python_console(self.vars, shells=shells)
 
     def _schedule(self, request, spider):
         spider = self._open_spider(request, spider)
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 924501002..a712df30e 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -1,37 +1,79 @@
+from functools import wraps
+from collections import OrderedDict
 
-def start_python_console(namespace=None, noipython=False, banner=''):
-    """Start Python console binded to the given namespace. If IPython is
-    available, an IPython console will be started instead, unless `noipython`
-    is True. Also, tab completion will be used on Unix systems.
+def _embed_ipython_shell(namespace={}, banner=''):
+    """Start an IPython Shell"""
+    try:
+        from IPython.terminal.embed import InteractiveShellEmbed
+        from IPython.terminal.ipapp import load_default_config
+    except ImportError:
+        from IPython.frontend.terminal.embed import InteractiveShellEmbed
+        from IPython.frontend.terminal.ipapp import load_default_config
+
+    @wraps(_embed_ipython_shell)
+    def wrapper(namespace=namespace, banner=''):
+        config = load_default_config()
+        shell = InteractiveShellEmbed(
+            banner1=banner, user_ns=namespace, config=config)
+        shell()
+    return wrapper
+
+def _embed_bpython_shell(namespace={}, banner=''):
+    """Start a bpython shell"""
+    import bpython
+    @wraps(_embed_bpython_shell)
+    def wrapper(namespace=namespace, banner=''):
+        bpython.embed(locals_=namespace, banner=banner)
+    return wrapper
+
+def _embed_standard_shell(namespace={}, banner=''):
+    """Start a standard python shell"""
+    import code
+    try: # readline module is only available on unix systems
+        import readline
+    except ImportError:
+        pass
+    else:
+        import rlcompleter
+        readline.parse_and_bind("tab:complete")
+    @wraps(_embed_standard_shell)
+    def wrapper(namespace=namespace, banner=''):
+        code.interact(banner=banner, local=namespace)
+    return wrapper
+
+DEFAULT_PYTHON_SHELLS = OrderedDict([
+    ('ipython', _embed_ipython_shell),
+    ('bpython', _embed_bpython_shell),
+    ( 'python', _embed_standard_shell),
+])
+
+def get_shell_embed_func(shells=None, known_shells=None):
+    """Return the first acceptable shell-embed function
+    from a given list of shell names.
+    """
+    if shells is None: # list, preference order of shells
+        shells = DEFAULT_PYTHON_SHELLS.keys()
+    if known_shells is None: # available embeddable shells
+        known_shells = DEFAULT_PYTHON_SHELLS.copy()
+    for shell in shells:
+        if shell in known_shells:
+            try:
+                # function test: run all setup code (imports),
+                # but dont fall into the shell
+                return known_shells[shell]()
+            except ImportError:
+                continue
+
+def start_python_console(namespace=None, banner='', shells=None):
+    """Start Python console bound to the given namespace.
+    Readline support and tab completion will be used on Unix, if available.
     """
     if namespace is None:
         namespace = {}
 
     try:
-        try: # use IPython if available
-            if noipython:
-                raise ImportError()
-
-            try:
-                from IPython.terminal.embed import InteractiveShellEmbed
-                from IPython.terminal.ipapp import load_default_config
-            except ImportError:
-                from IPython.frontend.terminal.embed import InteractiveShellEmbed
-                from IPython.frontend.terminal.ipapp import load_default_config
-
-            config = load_default_config()
-            shell = InteractiveShellEmbed(
-                banner1=banner, user_ns=namespace, config=config)
-            shell()
-        except ImportError:
-            import code
-            try: # readline module is only available on unix systems
-                import readline
-            except ImportError:
-                pass
-            else:
-                import rlcompleter
-                readline.parse_and_bind("tab:complete")
-            code.interact(banner=banner, local=namespace)
+        shell = get_shell_embed_func(shells)
+        if shell is not None:
+            shell(namespace=namespace, banner=banner)
     except SystemExit: # raised when using exit() in python code.interact
         pass
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 452a884e0..f88d17538 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -6,3 +6,6 @@ pytest-twisted
 pytest-cov
 jmespath
 testfixtures
+# optional for shell wrapper tests
+bpython
+ipython
diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py
new file mode 100644
index 000000000..65782747b
--- /dev/null
+++ b/tests/test_utils_console.py
@@ -0,0 +1,44 @@
+import unittest
+
+from scrapy.utils.console import get_shell_embed_func
+try:
+    import bpython
+    bpy = True
+    del bpython
+except ImportError:
+    bpy = False
+try:
+    import IPython
+    ipy = True
+    del IPython
+except ImportError:
+    ipy = False
+
+class UtilsConsoleTestCase(unittest.TestCase):
+
+    def test_get_shell_embed_func(self):
+
+        shell = get_shell_embed_func(['invalid'])
+        self.assertEqual(shell, None)
+
+        shell = get_shell_embed_func(['invalid','python'])
+        self.assertTrue(callable(shell))
+        self.assertEqual(shell.__name__, '_embed_standard_shell')
+
+    @unittest.skipIf(not bpy, 'bpython not available in testenv')
+    def test_get_shell_embed_func2(self):
+
+        shell = get_shell_embed_func(['bpython'])
+        self.assertTrue(callable(shell))
+        self.assertEqual(shell.__name__, '_embed_bpython_shell')
+
+    @unittest.skipIf(not ipy, 'IPython not available in testenv')
+    def test_get_shell_embed_func3(self):
+
+        # default shell should be 'ipython'
+        shell = get_shell_embed_func()
+        self.assertEqual(shell.__name__, '_embed_ipython_shell')
+
+
+if __name__ == "__main__":
+    unittest.main()

From ee66382f29385bfeabb9b8e036cc56b274c8303a Mon Sep 17 00:00:00 2001
From: Luar Roji <luar@scrapinghub.com>
Date: Fri, 21 Aug 2015 01:04:19 +0100
Subject: [PATCH 0510/2974] Fixed banner display in all 3 python shells

---
 scrapy/shell.py | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/scrapy/shell.py b/scrapy/shell.py
index a60b45dbb..f008ce39a 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -72,11 +72,12 @@ class Shell(object):
                 shells += env.strip().lower().split(',')
             elif cfg.has_option(section, option):
                 shells += [cfg.get(section, option).strip().lower()]
-            else: # try all by default
+            else:  # try all by default
                 shells += DEFAULT_PYTHON_SHELLS.keys()
             # always add standard shell as fallback
             shells += ['python']
-            start_python_console(self.vars, shells=shells)
+            start_python_console(self.vars, shells=shells,
+                                 banner=self.vars.pop('banner', ''))
 
     def _schedule(self, request, spider):
         spider = self._open_spider(request, spider)
@@ -127,21 +128,25 @@ class Shell(object):
         self.vars['shelp'] = self.print_help
         self.update_vars(self.vars)
         if not self.code:
-            self.print_help()
+            self.vars['banner'] = self.get_help()
 
     def print_help(self):
-        self.p("Available Scrapy objects:")
+        print(self.get_help())
+
+    def get_help(self):
+        b = []
+        b.append("Available Scrapy objects:")
         for k, v in sorted(self.vars.items()):
             if self._is_relevant(v):
-                self.p("  %-10s %s" % (k, v))
-        self.p("Useful shortcuts:")
-        self.p("  shelp()           Shell help (print this help)")
+                b.append("  %-10s %s" % (k, v))
+        b.append("Useful shortcuts:")
+        b.append("  shelp()           Shell help (print this help)")
         if self.inthread:
-            self.p("  fetch(req_or_url) Fetch request (or URL) and update local objects")
-        self.p("  view(response)    View response in a browser")
+            b.append("  fetch(req_or_url) Fetch request (or URL) and "
+                     "update local objects")
+        b.append("  view(response)    View response in a browser")
 
-    def p(self, line=''):
-        print("[s] %s" % line)
+        return "\n".join(["[s] %s" % l for l in b])
 
     def _is_relevant(self, value):
         return isinstance(value, self.relevant_classes)
@@ -165,6 +170,7 @@ def _request_deferred(request):
     """
     request_callback = request.callback
     request_errback = request.errback
+
     def _restore_callbacks(result):
         request.callback = request_callback
         request.errback = request_errback

From d164398a27736f75286cc435eca69b06ff7c1c06 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Fri, 21 Aug 2015 13:22:42 +0200
Subject: [PATCH 0511/2974] Fix RedirectMiddleware not honouring meta
 handle_httpstatus keys

---
 docs/topics/downloader-middleware.rst       |  6 ++++++
 scrapy/downloadermiddlewares/redirect.py    |  4 +++-
 tests/test_downloadermiddleware_redirect.py | 11 +++++++++++
 3 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 6d986bbf7..73cc67423 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -724,6 +724,12 @@ responses (and pass them through to your spider) you can do this::
     class MySpider(CrawlSpider):
         handle_httpstatus_list = [301, 302]
 
+The ``handle_httpstatus_list`` key of :attr:`Request.meta
+<scrapy.http.Request.meta>` can also be used to specify which response codes to
+allow on a per-request basis. You can also set the meta key
+``handle_httpstatus_all`` to ``True`` if you want to allow any response code
+for a request.
+
 
 RedirectMiddleware settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 363e56cb8..3cf8d2bee 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -55,7 +55,9 @@ class RedirectMiddleware(BaseRedirectMiddleware):
 
     def process_response(self, request, response, spider):
         if (request.meta.get('dont_redirect', False) or
-               response.status in getattr(spider, 'handle_httpstatus_list', [])):
+               response.status in getattr(spider, 'handle_httpstatus_list', []) or
+               response.status in request.meta.get('handle_httpstatus_list', []) or
+               request.meta.get('handle_httpstatus_all', False)):
             return response
 
         if request.method == 'HEAD':
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 9b00caa51..b3db7c42b 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -139,6 +139,17 @@ class RedirectMiddlewareTest(unittest.TestCase):
         r = self.mw.process_response(req, rsp, smartspider)
         self.assertIs(r, rsp)
 
+    def test_request_meta_handling(self):
+        url = 'http://www.example.com/301'
+        url2 = 'http://www.example.com/redirected'
+        def _test_passthrough(req):
+            rsp = Response(url, headers={'Location': url2}, status=301, request=req)
+            r = self.mw.process_response(req, rsp, self.spider)
+            self.assertIs(r, rsp)
+        _test_passthrough(Request(url, meta={'handle_httpstatus_list':
+                                                           [404, 301, 302]}))
+        _test_passthrough(Request(url, meta={'handle_httpstatus_all': True}))
+
 
 class MetaRefreshMiddlewareTest(unittest.TestCase):
 

From 787b5af30e01040251efff091a19691d4f5d1f09 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 26 Aug 2015 01:58:33 +0500
Subject: [PATCH 0512/2974] add coverage files to gitignore

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index 4db22f1d8..b116640b4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,6 +10,8 @@ venv
 build
 dist
 .idea
+htmlcov/
+.coverage
 
 # Windows
 Thumbs.db

From 7da769feb24e03629d7b7acb94a631459a4a333c Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 26 Aug 2015 01:58:59 +0500
Subject: [PATCH 0513/2974] enable test_stats and test_utils_log tests in
 Python 3

---
 scrapy/statscollectors.py | 1 +
 tests/py3-ignores.txt     | 2 --
 tests/test_stats.py       | 4 +---
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py
index 62b037f36..6da9ddcd2 100644
--- a/scrapy/statscollectors.py
+++ b/scrapy/statscollectors.py
@@ -50,6 +50,7 @@ class StatsCollector(object):
     def _persist_stats(self, stats, spider):
         pass
 
+
 class MemoryStatsCollector(StatsCollector):
 
     def __init__(self, crawler):
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 5a009db36..4d7fdc430 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -30,9 +30,7 @@ tests/test_spidermiddleware_httperror.py
 tests/test_spidermiddleware_offsite.py
 tests/test_spidermiddleware_referer.py
 tests/test_spider.py
-tests/test_stats.py
 tests/test_utils_iterators.py
-tests/test_utils_log.py
 tests/test_utils_template.py
 tests/test_webclient.py
 
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 5c7c0e6bb..9f950ebc9 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -4,6 +4,7 @@ from scrapy.spiders import Spider
 from scrapy.statscollectors import StatsCollector, DummyStatsCollector
 from scrapy.utils.test import get_crawler
 
+
 class StatsCollectorTest(unittest.TestCase):
 
     def setUp(self):
@@ -50,6 +51,3 @@ class StatsCollectorTest(unittest.TestCase):
         stats.set_value('test', 'value', spider=self.spider)
         self.assertEqual(stats.get_stats(), {})
         self.assertEqual(stats.get_stats('a'), {})
-
-if __name__ == "__main__":
-    unittest.main()

From 642af00bb7a676470155a907080839ba410a9adc Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 26 Aug 2015 02:19:33 +0500
Subject: [PATCH 0514/2974] fix Referer logging

---
 scrapy/core/scraper.py    |  4 ++--
 scrapy/logformatter.py    |  8 ++++++--
 scrapy/pipelines/files.py | 17 +++++------------
 scrapy/utils/request.py   |  8 +++++++-
 4 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 244499be2..67198179d 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -16,6 +16,7 @@ from scrapy import signals
 from scrapy.http import Request, Response
 from scrapy.item import BaseItem
 from scrapy.core.spidermw import SpiderMiddlewareManager
+from scrapy.utils.request import referer_str
 
 logger = logging.getLogger(__name__)
 
@@ -150,10 +151,9 @@ class Scraper(object):
         if isinstance(exc, CloseSpider):
             self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
             return
-        referer = request.headers.get('Referer')
         logger.error(
             "Spider error processing %(request)s (referer: %(referer)s)",
-            {'request': request, 'referer': referer},
+            {'request': request, 'referer': referer_str(request)},
             exc_info=failure_to_exc_info(_failure),
             extra={'spider': spider}
         )
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index a0508e0b7..2160d9ab0 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -3,6 +3,7 @@ import logging
 
 from twisted.python.failure import Failure
 
+from scrapy.utils.request import referer_str
 
 SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s"
 DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
@@ -38,13 +39,16 @@ class LogFormatter(object):
             'args': {
                 'status': response.status,
                 'request': request,
-                'referer': request.headers.get('Referer'),
+                'referer': referer_str(request),
                 'flags': flags,
             }
         }
 
     def scraped(self, item, response, spider):
-        src = response.getErrorMessage() if isinstance(response, Failure) else response
+        if isinstance(response, Failure):
+            src = response.getErrorMessage()
+        else:
+            src = response
         return {
             'level': logging.DEBUG,
             'msg': SCRAPEDMSG,
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index db49aff65..e4011d31d 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -26,7 +26,8 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.misc import md5sum
 from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes
+from scrapy.utils.request import referer_str
 
 logger = logging.getLogger(__name__)
 
@@ -199,7 +200,7 @@ class FilesPipeline(MediaPipeline):
             if age_days > self.EXPIRES:
                 return  # returning None force download
 
-            referer = _get_referer(request)
+            referer = referer_str(request)
             logger.debug(
                 'File (uptodate): Downloaded %(medianame)s from %(request)s '
                 'referred in <%(referer)s>',
@@ -225,7 +226,7 @@ class FilesPipeline(MediaPipeline):
 
     def media_failed(self, failure, request, info):
         if not isinstance(failure.value, IgnoreRequest):
-            referer = _get_referer(request)
+            referer = referer_str(request)
             logger.warning(
                 'File (unknown-error): Error downloading %(medianame)s from '
                 '%(request)s referred in <%(referer)s>: %(exception)s',
@@ -237,7 +238,7 @@ class FilesPipeline(MediaPipeline):
         raise FileException
 
     def media_downloaded(self, response, request, info):
-        referer = _get_referer(request)
+        referer = referer_str(request)
 
         if response.status != 200:
             logger.warning(
@@ -339,11 +340,3 @@ class FilesPipeline(MediaPipeline):
     def file_key(self, url):
         return self.file_path(url)
     file_key._base = True
-
-
-def _get_referer(request):
-    """ Return Referer HTTP header suitable for logging """
-    referrer = request.headers.get('Referer')
-    if referrer is None:
-        return referrer
-    return to_native_str(referrer, errors='replace')
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 0487d1e1b..e361b7433 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -8,7 +8,6 @@ import hashlib
 import weakref
 from six.moves.urllib.parse import urlunparse
 
-from twisted.internet.defer import Deferred
 from w3lib.http import basic_auth_header
 from scrapy.utils.python import to_bytes, to_native_str
 
@@ -86,3 +85,10 @@ def request_httprepr(request):
     s += request.body
     return s
 
+
+def referer_str(request):
+    """ Return Referer HTTP header suitable for logging. """
+    referrer = request.headers.get('Referer')
+    if referrer is None:
+        return referrer
+    return to_native_str(referrer, errors='replace')

From 68a47ade46e6f6fb4235e1355b6a535e9c422f29 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 26 Aug 2015 02:34:21 +0500
Subject: [PATCH 0515/2974] PY3 port test_logformatter

---
 tests/py3-ignores.txt      | 1 -
 tests/test_logformatter.py | 5 +++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 4d7fdc430..f380e6679 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -20,7 +20,6 @@ tests/test_downloadermiddleware_retry.py
 tests/test_downloadermiddleware_stats.py
 tests/test_downloadermiddleware_useragent.py
 tests/test_engine.py
-tests/test_logformatter.py
 tests/test_mail.py
 tests/test_pipeline_files.py
 tests/test_pipeline_images.py
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index ec42ef8ab..50e9662c6 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -1,4 +1,5 @@
 import unittest
+import six
 
 from scrapy.spiders import Spider
 from scrapy.http import Request, Response
@@ -42,7 +43,7 @@ class LoggingContribTest(unittest.TestCase):
         logkws = self.formatter.dropped(item, exception, response, self.spider)
         logline = logkws['msg'] % logkws['args']
         lines = logline.splitlines()
-        assert all(isinstance(x, unicode) for x in lines)
+        assert all(isinstance(x, six.text_type) for x in lines)
         self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
 
     def test_scraped(self):
@@ -52,7 +53,7 @@ class LoggingContribTest(unittest.TestCase):
         logkws = self.formatter.scraped(item, response, self.spider)
         logline = logkws['msg'] % logkws['args']
         lines = logline.splitlines()
-        assert all(isinstance(x, unicode) for x in lines)
+        assert all(isinstance(x, six.text_type) for x in lines)
         self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])
 
 if __name__ == "__main__":

From f5c54072ecec1b19390e9a624d0f1fe0882d2791 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Wed, 6 May 2015 21:21:20 +0300
Subject: [PATCH 0516/2974] Test utils.template.render_templatefile()

---
 tests/test_utils_template.py | 41 ++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index e690a8537..40b733233 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -1 +1,42 @@
+import os
+from shutil import rmtree
+from tempfile import mkdtemp
+import unittest
+from scrapy.utils.template import render_templatefile
+
+
 __doctests__ = ['scrapy.utils.template']
+
+
+class UtilsRenderTemplateFileTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.tmp_path = mkdtemp()
+
+    def tearDown(self):
+        rmtree(self.tmp_path)
+
+    def test_simple_render(self):
+
+        context = dict(project_name='proj', name='spi', classname='TheSpider')
+        template = u'from ${project_name}.spiders.${name} import ${classname}'
+        rendered = u'from proj.spiders.spi import TheSpider'
+
+        template_path = os.path.join(self.tmp_path, 'templ.py.tmpl')
+        render_path = os.path.join(self.tmp_path, 'templ.py')
+
+        with open(template_path, 'wb') as tmpl_file:
+            tmpl_file.write(template.encode('utf8'))
+        assert os.path.isfile(template_path)  # Failure of test itself
+
+        render_templatefile(template_path, **context)
+
+        self.assertFalse(os.path.exists(template_path))
+        with open(render_path, 'rb') as result:
+            self.assertEqual(result.read().decode('utf8'), rendered)
+
+        os.remove(render_path)
+        assert not os.path.exists(render_path)  # Failure of test iself
+
+if '__main__' == __name__:
+    unittest.main()

From 56b3cf07fff4d3794dcdbf99f6d7faa629fa243e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Wed, 6 May 2015 22:36:14 +0300
Subject: [PATCH 0517/2974] fix string manipulation in render_templatefile()

---
 scrapy/utils/template.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py
index 1cfcd82ee..90aa8f705 100644
--- a/scrapy/utils/template.py
+++ b/scrapy/utils/template.py
@@ -10,7 +10,8 @@ def render_templatefile(path, **kwargs):
 
     content = string.Template(raw).substitute(**kwargs)
 
-    with open(path.rstrip('.tmpl'), 'wb') as file:
+    render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path
+    with open(render_path, 'wb') as file:
         file.write(content)
     if path.endswith('.tmpl'):
         os.remove(path)

From aa88fd797adc11d87d886e62ef55449de6032e3a Mon Sep 17 00:00:00 2001
From: Rick <rickh@xs4all.nl>
Date: Sun, 23 Aug 2015 15:58:53 +0200
Subject: [PATCH 0518/2974] Small grammatical change

---
 docs/intro/tutorial.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index e9df9c2d2..45d1f9b29 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -83,8 +83,8 @@ sites, we define fields for each of these three attributes. To do that, we edit
         link = scrapy.Field()
         desc = scrapy.Field()
 
-This may seem complicated at first, but defining the item allows you to use other handy
-components of Scrapy that need to know what does your item look like.
+This may seem complicated at first, but defining an item class allows you to use other handy
+components and helpers within Scrapy.
 
 Our first Spider
 ================

From 8ecc4544b3747eb9be33153483b62c6441bd7c56 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 27 Aug 2015 21:37:24 +0500
Subject: [PATCH 0519/2974] enable console tests in PY3

---
 tests/requirements-py3.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index e37a2e68d..cac8da9b5 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -3,3 +3,6 @@ pytest-twisted
 pytest-cov
 testfixtures
 jmespath
+# optional for shell wrapper tests
+bpython
+ipython

From cb3007c06647867ac53072c01fc7fabe778a8fcc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rafa=C5=82=20Gutkowski?= <goodkowski@ralph-mbp.local>
Date: Thu, 7 May 2015 15:30:12 +0200
Subject: [PATCH 0520/2974] support link rel attribute with multiple values

---
 scrapy/linkextractors/lxmlhtml.py |  4 ++--
 scrapy/linkextractors/sgml.py     |  4 ++--
 scrapy/utils/misc.py              |  5 +++++
 tests/test_linkextractors.py      | 15 +++++++++++++--
 4 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index e9fa521f3..7064e886d 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -9,7 +9,7 @@ import lxml.etree as etree
 
 from scrapy.selector import Selector
 from scrapy.link import Link
-from scrapy.utils.misc import arg_to_iter
+from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list
 from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.response import get_base_url
@@ -62,7 +62,7 @@ class LxmlParserLinkExtractor(object):
             # to fix relative links after process_value
             url = urljoin(response_url, url)
             link = Link(url, _collect_string_content(el) or u'',
-                nofollow=True if el.get('rel') == 'nofollow' else False)
+                        nofollow=rel_has_nofollow(el.get('rel')))
             links.append(link)
 
         return unique_list(links, key=lambda link: link.url) \
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index acecd9c0d..7084d0180 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -9,7 +9,7 @@ from w3lib.url import safe_url_string
 from scrapy.selector import Selector
 from scrapy.link import Link
 from scrapy.linkextractors import FilteringLinkExtractor
-from scrapy.utils.misc import arg_to_iter
+from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list, to_unicode
 from scrapy.utils.response import get_base_url
 from scrapy.exceptions import ScrapyDeprecationWarning
@@ -80,7 +80,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
                 if self.scan_attr(attr):
                     url = self.process_value(value)
                     if url is not None:
-                        link = Link(url=url, nofollow=True if dict(attrs).get('rel') == 'nofollow' else False)
+                        link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
                         self.links.append(link)
                         self.current_link = link
 
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 4215e41d2..303a413d8 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -112,3 +112,8 @@ def md5sum(file):
             break
         m.update(d)
     return m.hexdigest()
+
+def rel_has_nofollow(rel):
+    """Return True if link rel attribute has nofollow type"""
+    return True if rel is not None and 'nofollow' in rel.split() else False
+    
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index d78b25f25..d32ff2d55 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -96,12 +96,14 @@ class LinkExtractorTestCase(unittest.TestCase):
         html = """
         <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
         <a href="about.html">About us</a>
+        <a href="http://google.com/something" rel="external nofollow">Something</a>
         """
         response = HtmlResponse("http://example.org/page.html", body=html)
         lx = SgmlLinkExtractor()
         self.assertEqual([link for link in lx.extract_links(response)], [
             Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
             Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
+            Link(url='http://google.com/something', text=u'Something', nofollow=True),
         ])
 
 
@@ -205,6 +207,9 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
         <div>
         <p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p>
         </div>
+        <div>
+        <p><a href="http://google.com/something" rel="external nofollow">External link not to follow</a></p>
+        </div>
         </body></html>"""
         response = HtmlResponse("http://example.org/somepage/index.html", body=html)
 
@@ -214,6 +219,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
             Link(url='http://example.org/follow.html', text=u'Follow this link'),
             Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True),
             Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'),
+            Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
         ])
 
     def test_matches(self):
@@ -467,6 +473,9 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
     <div>
     <p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p>
     </div>
+    <div>
+    <p><a href="http://google.com/something" rel="external nofollow">External link not to follow</a></p>
+    </div>
 </body>
 </html>
         """
@@ -478,7 +487,8 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
                          [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
                           Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
                           Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                          Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)]
+                          Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
+                          Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
                         )
 
         response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
@@ -488,7 +498,8 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
                          [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
                           Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
                           Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                          Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)]
+                          Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
+                          Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
                         )
 
     def test_link_wrong_href(self):

From 08162a15d8a5d5123fb241c08a0ea20595a2a051 Mon Sep 17 00:00:00 2001
From: David Tagatac <dtagatac@cs.columbia.edu>
Date: Thu, 27 Aug 2015 17:37:16 -0400
Subject: [PATCH 0521/2974] minor: scrapy.Spider docs grammar

---
 docs/topics/spiders.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 025232fab..6a8655fe7 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -108,7 +108,7 @@ scrapy.Spider
 
    .. attribute:: settings
 
-      Configuration on which this spider is been ran. This is a
+      Configuration for running this spider. This is a
       :class:`~scrapy.settings.Settings` instance, see the
       :ref:`topics-settings` topic for a detailed introduction on this subject.
 
@@ -122,12 +122,12 @@ scrapy.Spider
 
        This is the class method used by Scrapy to create your spiders.
 
-       You probably won't need to override this directly, since the default
+       You probably won't need to override this directly because the default
        implementation acts as a proxy to the :meth:`__init__` method, calling
        it with the given arguments `args` and named arguments `kwargs`.
 
        Nonetheless, this method sets the :attr:`crawler` and :attr:`settings`
-       attributes in the new instance, so they can be accessed later inside the
+       attributes in the new instance so they can be accessed later inside the
        spider's code.
 
        :param crawler: crawler to which the spider will be bound

From f46a4500803dac7bd64284862c3b5937bb525a64 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 27 Aug 2015 16:44:34 +0500
Subject: [PATCH 0522/2974] refactor test_linkextractors

* rename LinkExtractorTestCase to BaseSgmlLinkExtractorTestCase
* add BaseLinkExtractorTestCase link extractor tests can inherit from
  and decouple it from SgmlLinkExtractor
* add an extra check for deny_extensions
* xfail test_restrict_xpaths_with_html_entities for LxmlLinkExtractor explicitly
---
 tests/test_linkextractors.py | 88 ++++++++++++++++++++++--------------
 1 file changed, 55 insertions(+), 33 deletions(-)

diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index d78b25f25..3e202bf02 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -1,5 +1,8 @@
 import re
 import unittest
+
+import pytest
+
 from scrapy.linkextractors.regex import RegexLinkExtractor
 from scrapy.http import HtmlResponse, XmlResponse
 from scrapy.link import Link
@@ -9,7 +12,7 @@ from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
 from tests import get_testdata
 
 
-class LinkExtractorTestCase(unittest.TestCase):
+class BaseSgmlLinkExtractorTestCase(unittest.TestCase):
     def test_basic(self):
         html = """<html><head><title>Page title<title>
         <body><p><a href="item/12.html">Item 12</a></p>
@@ -92,30 +95,21 @@ class LinkExtractorTestCase(unittest.TestCase):
         self.assertEqual(lx.matches(url1), True)
         self.assertEqual(lx.matches(url2), True)
 
-    def test_link_nofollow(self):
-        html = """
-        <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
-        <a href="about.html">About us</a>
-        """
-        response = HtmlResponse("http://example.org/page.html", body=html)
-        lx = SgmlLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
-            Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
-        ])
 
-
-class SgmlLinkExtractorTestCase(unittest.TestCase):
-    extractor_cls = SgmlLinkExtractor
+class BaseLinkExtractorTestCase(unittest.TestCase):
+    extractor_cls = None
 
     def setUp(self):
+        if self.extractor_cls is None:
+            raise unittest.SkipTest()
         body = get_testdata('link_extractor', 'sgml_linkextractor.html')
         self.response = HtmlResponse(url='http://example.com/index', body=body)
 
     def test_urls_type(self):
-        '''Test that the resulting urls are regular strings and not a unicode objects'''
+        ''' Test that the resulting urls are str objects '''
         lx = self.extractor_cls()
-        self.assertTrue(all(isinstance(link.url, str) for link in lx.extract_links(self.response)))
+        self.assertTrue(all(isinstance(link.url, str)
+                            for link in lx.extract_links(self.response)))
 
     def test_extraction(self):
         '''Test the extractor's behaviour among different situations'''
@@ -271,7 +265,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
     def test_restrict_xpaths_with_html_entities(self):
         html = '<html><body><p><a href="/&hearts;/you?c=&euro;">text</a></p></body></html>'
         response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
-        links = SgmlLinkExtractor(restrict_xpaths='//p').extract_links(response)
+        links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
         self.assertEqual(links,
                          [Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')])
 
@@ -326,7 +320,8 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
             Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
         ])
 
-    def test_deny_extensions(self):
+    def test_ignored_extensions(self):
+        # jpg is ignored by default
         html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
         response = HtmlResponse("http://example.org/", body=html)
         lx = self.extractor_cls()
@@ -334,9 +329,10 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
             Link(url='http://example.org/page.html', text=u'asd'),
         ])
 
-        lx = SgmlLinkExtractor(deny_extensions="jpg")
+        # override denied extensions
+        lx = self.extractor_cls(deny_extensions=['html'])
         self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.org/page.html', text=u'asd'),
+            Link(url='http://example.org/photo.jpg'),
         ])
 
     def test_process_value(self):
@@ -388,13 +384,6 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
         lx = self.extractor_cls(attrs=None)
         self.assertEqual(lx.extract_links(self.response), [])
 
-        html = """<html><area href="sample1.html"></area><a ref="sample2.html">sample text 2</a></html>"""
-        response = HtmlResponse("http://example.com/index.html", body=html)
-        lx = SgmlLinkExtractor(attrs=("href"))
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.com/sample1.html', text=u''),
-        ])
-
     def test_tags(self):
         html = """<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>"""
         response = HtmlResponse("http://example.com/index.html", body=html)
@@ -505,7 +494,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
         ])
 
 
-class LxmlLinkExtractorTestCase(SgmlLinkExtractorTestCase):
+class LxmlLinkExtractorTestCase(BaseLinkExtractorTestCase):
     extractor_cls = LxmlLinkExtractor
 
     def test_link_wrong_href(self):
@@ -521,6 +510,10 @@ class LxmlLinkExtractorTestCase(SgmlLinkExtractorTestCase):
             Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
         ])
 
+    @pytest.mark.xfail
+    def test_restrict_xpaths_with_html_entities(self):
+        super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()
+
 
 class HtmlParserLinkExtractorTestCase(unittest.TestCase):
 
@@ -552,6 +545,39 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase):
         ])
 
 
+class SgmlLinkExtractorTestCase(BaseLinkExtractorTestCase):
+    extractor_cls = SgmlLinkExtractor
+
+    def test_deny_extensions(self):
+        html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
+        response = HtmlResponse("http://example.org/", body=html)
+        lx = SgmlLinkExtractor(deny_extensions="jpg")
+        self.assertEqual(lx.extract_links(response), [
+            Link(url='http://example.org/page.html', text=u'asd'),
+        ])
+
+    def test_attrs_sgml(self):
+        html = """<html><area href="sample1.html"></area>
+        <a ref="sample2.html">sample text 2</a></html>"""
+        response = HtmlResponse("http://example.com/index.html", body=html)
+        lx = SgmlLinkExtractor(attrs="href")
+        self.assertEqual(lx.extract_links(response), [
+            Link(url='http://example.com/sample1.html', text=u''),
+        ])
+
+    def test_link_nofollow(self):
+        html = """
+        <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
+        <a href="about.html">About us</a>
+        """
+        response = HtmlResponse("http://example.org/page.html", body=html)
+        lx = SgmlLinkExtractor()
+        self.assertEqual([link for link in lx.extract_links(response)], [
+            Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
+            Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
+        ])
+
+
 class RegexLinkExtractorTestCase(unittest.TestCase):
 
     def setUp(self):
@@ -579,7 +605,3 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
             Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
             Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
         ])
-
-
-if __name__ == "__main__":
-    unittest.main()

From f2edbd05deda6d22649bdc753e9dd48995ee6aba Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 27 Aug 2015 17:35:29 +0500
Subject: [PATCH 0523/2974] PY3 port LinkExtractor

* tests for other link extractors are moved to test_linkextractors_deprecated.py
* in Python 3 Link is converted to use native strings for urls
* minor cleanups
---
 scrapy/link.py                          |   9 +-
 scrapy/linkextractors/__init__.py       |  11 +-
 scrapy/linkextractors/lxmlhtml.py       |  23 +-
 tests/py3-ignores.txt                   |   2 +-
 tests/test_link.py                      |  18 +-
 tests/test_linkextractors.py            | 880 ++++++++++--------------
 tests/test_linkextractors_deprecated.py | 190 +++++
 7 files changed, 577 insertions(+), 556 deletions(-)
 create mode 100644 tests/test_linkextractors_deprecated.py

diff --git a/scrapy/link.py b/scrapy/link.py
index 8bdcce761..dc6e64adc 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -4,8 +4,8 @@ This module defines the Link object used in Link extractors.
 For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
 """
+from scrapy.utils.python import to_native_str
 
-import six
 
 class Link(object):
     """Link objects represent an extracted link by the LinkExtractor."""
@@ -13,11 +13,10 @@ class Link(object):
     __slots__ = ['url', 'text', 'fragment', 'nofollow']
 
     def __init__(self, url, text='', fragment='', nofollow=False):
-        if isinstance(url, six.text_type):
+        if not isinstance(url, str):
             import warnings
-            warnings.warn("Do not instantiate Link objects with unicode urls. "
-                "Assuming utf-8 encoding (which could be wrong)")
-            url = url.encode('utf-8')
+            warnings.warn("Link urls must be str objects.")
+            url = to_native_str(url)
         self.url = url
         self.text = text
         self.fragment = fragment
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index bb799e572..64efa0c55 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -39,7 +39,7 @@ IGNORED_EXTENSIONS = [
 
 _re_type = type(re.compile("", 0))
 _matches = lambda url, regexs: any((r.search(url) for r in regexs))
-_is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'file'])
+_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'}
 
 
 class FilteringLinkExtractor(object):
@@ -51,8 +51,10 @@ class FilteringLinkExtractor(object):
 
         self.link_extractor = link_extractor
 
-        self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)]
-        self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)]
+        self.allow_res = [x if isinstance(x, _re_type) else re.compile(x)
+                          for x in arg_to_iter(allow)]
+        self.deny_res = [x if isinstance(x, _re_type) else re.compile(x)
+                         for x in arg_to_iter(deny)]
 
         self.allow_domains = set(arg_to_iter(allow_domains))
         self.deny_domains = set(arg_to_iter(deny_domains))
@@ -64,7 +66,7 @@ class FilteringLinkExtractor(object):
         self.canonicalize = canonicalize
         if deny_extensions is None:
             deny_extensions = IGNORED_EXTENSIONS
-        self.deny_extensions = set(['.' + e for e in arg_to_iter(deny_extensions)])
+        self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)}
 
     def _link_allowed(self, link):
         if not _is_valid_url(link.url):
@@ -104,5 +106,6 @@ class FilteringLinkExtractor(object):
     def _extract_links(self, *args, **kwargs):
         return self.link_extractor._extract_links(*args, **kwargs)
 
+
 # Top-level imports
 from .lxmlhtml import LxmlLinkExtractor as LinkExtractor
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index e9fa521f3..e39c9950e 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -1,16 +1,14 @@
 """
 Link extractor based on lxml.html
 """
-
-import re
+import six
 from six.moves.urllib.parse import urlparse, urljoin
 
 import lxml.etree as etree
 
-from scrapy.selector import Selector
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.python import unique as unique_list
+from scrapy.utils.python import unique as unique_list, to_native_str
 from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.response import get_base_url
 
@@ -20,8 +18,9 @@ XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
 
 _collect_string_content = etree.XPath("string()")
 
+
 def _nons(tag):
-    if isinstance(tag, basestring):
+    if isinstance(tag, six.string_types):
         if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
             return tag.split('}')[-1]
     return tag
@@ -57,16 +56,13 @@ class LxmlParserLinkExtractor(object):
                 url = self.process_attr(attr_val)
                 if url is None:
                     continue
-            if isinstance(url, unicode):
-                url = url.encode(response_encoding)
+            url = to_native_str(url, encoding=response_encoding)
             # to fix relative links after process_value
             url = urljoin(response_url, url)
             link = Link(url, _collect_string_content(el) or u'',
                 nofollow=True if el.get('rel') == 'nofollow' else False)
             links.append(link)
-
-        return unique_list(links, key=lambda link: link.url) \
-                if self.unique else links
+        return self._deduplicate_if_needed(links)
 
     def extract_links(self, response):
         base_url = get_base_url(response)
@@ -77,7 +73,11 @@ class LxmlParserLinkExtractor(object):
 
         The subclass should override it if neccessary
         """
-        links = unique_list(links, key=lambda link: link.url) if self.unique else links
+        return self._deduplicate_if_needed(links)
+
+    def _deduplicate_if_needed(self, links):
+        if self.unique:
+            return unique_list(links, key=lambda link: link.url)
         return links
 
 
@@ -110,4 +110,3 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
             links = self._extract_links(doc, response.url, response.encoding, base_url)
             all_links.extend(self._process_links(links))
         return unique_list(all_links)
-
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index f380e6679..b40293f57 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -3,7 +3,7 @@ tests/test_command_fetch.py
 tests/test_command_shell.py
 tests/test_commands.py
 tests/test_exporters.py
-tests/test_linkextractors.py
+tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
 tests/test_crawler.py
 tests/test_downloader_handlers.py
diff --git a/tests/test_link.py b/tests/test_link.py
index 0b79e47cd..c8487698f 100644
--- a/tests/test_link.py
+++ b/tests/test_link.py
@@ -1,8 +1,10 @@
 import unittest
 import warnings
+import six
 
 from scrapy.link import Link
 
+
 class LinkTest(unittest.TestCase):
 
     def _assert_same_links(self, link1, link2):
@@ -43,9 +45,15 @@ class LinkTest(unittest.TestCase):
         l2 = eval(repr(l1))
         self._assert_same_links(l1, l2)
 
-    def test_unicode_url(self):
+    def test_non_str_url(self):
         with warnings.catch_warnings(record=True) as w:
-            link = Link(u"http://www.example.com/\xa3")
-            self.assertIsInstance(link.url, bytes)
-            self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3')
-            assert len(w) == 1, "warning not issued"
+            if six.PY2:
+                link = Link(u"http://www.example.com/\xa3")
+                self.assertIsInstance(link.url, str)
+                self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3')
+            else:
+                link = Link(b"http://www.example.com/\xc2\xa3")
+                self.assertIsInstance(link.url, str)
+                self.assertEqual(link.url, u'http://www.example.com/\xa3')
+
+        assert len(w) == 1, "warning not issued"
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 3e202bf02..5966a3caf 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -3,190 +3,360 @@ import unittest
 
 import pytest
 
-from scrapy.linkextractors.regex import RegexLinkExtractor
 from scrapy.http import HtmlResponse, XmlResponse
 from scrapy.link import Link
-from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor
-from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
 from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
 from tests import get_testdata
 
 
-class BaseSgmlLinkExtractorTestCase(unittest.TestCase):
-    def test_basic(self):
-        html = """<html><head><title>Page title<title>
-        <body><p><a href="item/12.html">Item 12</a></p>
-        <p><a href="/about.html">About us</a></p>
-        <img src="/logo.png" alt="Company logo (not a link)" />
-        <p><a href="../othercat.html">Other category</a></p>
-        <p><a href="/">&gt;&gt;</a></p>
-        <p><a href="/" /></p>
-        </body></html>"""
-        response = HtmlResponse("http://example.org/somepage/index.html", body=html)
+# a hack to skip base class tests in pytest
+class Base:
+    class LinkExtractorTestCase(unittest.TestCase):
+        extractor_cls = None
 
-        lx = BaseSgmlLinkExtractor()  # default: tag=a, attr=href
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://example.org/somepage/item/12.html', text='Item 12'),
-                          Link(url='http://example.org/about.html', text='About us'),
-                          Link(url='http://example.org/othercat.html', text='Other category'),
-                          Link(url='http://example.org/', text='>>'),
-                          Link(url='http://example.org/', text='')])
+        def setUp(self):
+            body = get_testdata('link_extractor', 'sgml_linkextractor.html')
+            self.response = HtmlResponse(url='http://example.com/index', body=body)
 
-    def test_base_url(self):
-        html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
-        <body><p><a href="item/12.html">Item 12</a></p>
-        </body></html>"""
-        response = HtmlResponse("http://example.org/somepage/index.html", body=html)
+        def test_urls_type(self):
+            ''' Test that the resulting urls are str objects '''
+            lx = self.extractor_cls()
+            self.assertTrue(all(isinstance(link.url, str)
+                                for link in lx.extract_links(self.response)))
 
-        lx = BaseSgmlLinkExtractor()  # default: tag=a, attr=href
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
+        def test_extraction(self):
+            '''Test the extractor's behaviour among different situations'''
 
-        # base url is an absolute path and relative to host
-        html = """<html><head><title>Page title<title><base href="/" />
-        <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
-        response = HtmlResponse("https://example.org/somepage/index.html", body=html)
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='https://example.org/item/12.html', text='Item 12')])
+            lx = self.extractor_cls()
+            self.assertEqual([link for link in lx.extract_links(self.response)], [
+                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://www.google.com/something', text=u''),
+                Link(url='http://example.com/innertag.html', text=u'inner tag'),
+            ])
 
-        # base url has no scheme
-        html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" />
-        <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
-        response = HtmlResponse("https://example.org/somepage/index.html", body=html)
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')])
+            lx = self.extractor_cls(allow=('sample', ))
+            self.assertEqual([link for link in lx.extract_links(self.response)], [
+                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+            ])
 
-    def test_link_text_wrong_encoding(self):
-        html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>"""
-        response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8')
-        lx = BaseSgmlLinkExtractor()
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'),
-        ])
+            lx = self.extractor_cls(allow=('sample', ), unique=False)
+            self.assertEqual([link for link in lx.extract_links(self.response)], [
+                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+            ])
 
-    def test_extraction_encoding(self):
-        body = get_testdata('link_extractor', 'linkextractor_noenc.html')
-        response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})
-        response_noenc = HtmlResponse(url='http://example.com/noenc', body=body)
-        body = get_testdata('link_extractor', 'linkextractor_latin1.html')
-        response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body)
+            lx = self.extractor_cls(allow=('sample', ))
+            self.assertEqual([link for link in lx.extract_links(self.response)], [
+                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+            ])
 
-        lx = BaseSgmlLinkExtractor()
-        self.assertEqual(lx.extract_links(response_utf8), [
-            Link(url='http://example.com/sample_%C3%B1.html', text=''),
-            Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
-        ])
+            lx = self.extractor_cls(allow=('sample', ), deny=('3', ))
+            self.assertEqual([link for link in lx.extract_links(self.response)], [
+                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+            ])
 
-        self.assertEqual(lx.extract_links(response_noenc), [
-            Link(url='http://example.com/sample_%C3%B1.html', text=''),
-            Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
-        ])
+            lx = self.extractor_cls(allow_domains=('google.com', ))
+            self.assertEqual([link for link in lx.extract_links(self.response)], [
+                Link(url='http://www.google.com/something', text=u''),
+            ])
 
-        self.assertEqual(lx.extract_links(response_latin1), [
-            Link(url='http://example.com/sample_%F1.html', text=''),
-            Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')),
-        ])
+        def test_extraction_using_single_values(self):
+            '''Test the extractor's behaviour among different situations'''
 
-    def test_matches(self):
-        url1 = 'http://lotsofstuff.com/stuff1/index'
-        url2 = 'http://evenmorestuff.com/uglystuff/index'
+            lx = self.extractor_cls(allow='sample')
+            self.assertEqual([link for link in lx.extract_links(self.response)], [
+                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+            ])
 
-        lx = BaseSgmlLinkExtractor()
-        self.assertEqual(lx.matches(url1), True)
-        self.assertEqual(lx.matches(url2), True)
+            lx = self.extractor_cls(allow='sample', deny='3')
+            self.assertEqual([link for link in lx.extract_links(self.response)], [
+                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+            ])
 
+            lx = self.extractor_cls(allow_domains='google.com')
+            self.assertEqual([link for link in lx.extract_links(self.response)], [
+                Link(url='http://www.google.com/something', text=u''),
+            ])
 
-class BaseLinkExtractorTestCase(unittest.TestCase):
-    extractor_cls = None
+            lx = self.extractor_cls(deny_domains='example.com')
+            self.assertEqual([link for link in lx.extract_links(self.response)], [
+                Link(url='http://www.google.com/something', text=u''),
+            ])
 
-    def setUp(self):
-        if self.extractor_cls is None:
-            raise unittest.SkipTest()
-        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
-        self.response = HtmlResponse(url='http://example.com/index', body=body)
+        def test_nofollow(self):
+            '''Test the extractor's behaviour for links with rel="nofollow"'''
 
-    def test_urls_type(self):
-        ''' Test that the resulting urls are str objects '''
-        lx = self.extractor_cls()
-        self.assertTrue(all(isinstance(link.url, str)
-                            for link in lx.extract_links(self.response)))
+            html = b"""<html><head><title>Page title<title>
+            <body>
+            <div class='links'>
+            <p><a href="/about.html">About us</a></p>
+            </div>
+            <div>
+            <p><a href="/follow.html">Follow this link</a></p>
+            </div>
+            <div>
+            <p><a href="/nofollow.html" rel="nofollow">Dont follow this one</a></p>
+            </div>
+            <div>
+            <p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p>
+            </div>
+            </body></html>"""
+            response = HtmlResponse("http://example.org/somepage/index.html", body=html)
 
-    def test_extraction(self):
-        '''Test the extractor's behaviour among different situations'''
+            lx = self.extractor_cls()
+            self.assertEqual(lx.extract_links(response), [
+                Link(url='http://example.org/about.html', text=u'About us'),
+                Link(url='http://example.org/follow.html', text=u'Follow this link'),
+                Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True),
+                Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'),
+            ])
 
-        lx = self.extractor_cls()
-        self.assertEqual([link for link in lx.extract_links(self.response)], [
-            Link(url='http://example.com/sample1.html', text=u''),
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-            Link(url='http://www.google.com/something', text=u''),
-            Link(url='http://example.com/innertag.html', text=u'inner tag'),
-        ])
+        def test_matches(self):
+            url1 = 'http://lotsofstuff.com/stuff1/index'
+            url2 = 'http://evenmorestuff.com/uglystuff/index'
 
-        lx = self.extractor_cls(allow=('sample', ))
-        self.assertEqual([link for link in lx.extract_links(self.response)], [
-            Link(url='http://example.com/sample1.html', text=u''),
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-        ])
+            lx = self.extractor_cls(allow=(r'stuff1', ))
+            self.assertEqual(lx.matches(url1), True)
+            self.assertEqual(lx.matches(url2), False)
 
-        lx = self.extractor_cls(allow=('sample', ), unique=False)
-        self.assertEqual([link for link in lx.extract_links(self.response)], [
-            Link(url='http://example.com/sample1.html', text=u''),
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
-        ])
+            lx = self.extractor_cls(deny=(r'uglystuff', ))
+            self.assertEqual(lx.matches(url1), True)
+            self.assertEqual(lx.matches(url2), False)
 
-        lx = self.extractor_cls(allow=('sample', ))
-        self.assertEqual([link for link in lx.extract_links(self.response)], [
-            Link(url='http://example.com/sample1.html', text=u''),
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-        ])
+            lx = self.extractor_cls(allow_domains=('evenmorestuff.com', ))
+            self.assertEqual(lx.matches(url1), False)
+            self.assertEqual(lx.matches(url2), True)
 
-        lx = self.extractor_cls(allow=('sample', ), deny=('3', ))
-        self.assertEqual([link for link in lx.extract_links(self.response)], [
-            Link(url='http://example.com/sample1.html', text=u''),
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-        ])
+            lx = self.extractor_cls(deny_domains=('lotsofstuff.com', ))
+            self.assertEqual(lx.matches(url1), False)
+            self.assertEqual(lx.matches(url2), True)
 
-        lx = self.extractor_cls(allow_domains=('google.com', ))
-        self.assertEqual([link for link in lx.extract_links(self.response)], [
-            Link(url='http://www.google.com/something', text=u''),
-        ])
+            lx = self.extractor_cls(allow=('blah1',), deny=('blah2',),
+                                   allow_domains=('blah1.com',),
+                                   deny_domains=('blah2.com',))
+            self.assertEqual(lx.matches('http://blah1.com/blah1'), True)
+            self.assertEqual(lx.matches('http://blah1.com/blah2'), False)
+            self.assertEqual(lx.matches('http://blah2.com/blah1'), False)
+            self.assertEqual(lx.matches('http://blah2.com/blah2'), False)
 
-    def test_extraction_using_single_values(self):
-        '''Test the extractor's behaviour among different situations'''
+        def test_restrict_xpaths(self):
+            lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ))
+            self.assertEqual([link for link in lx.extract_links(self.response)], [
+                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+            ])
 
-        lx = self.extractor_cls(allow='sample')
-        self.assertEqual([link for link in lx.extract_links(self.response)], [
-            Link(url='http://example.com/sample1.html', text=u''),
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-        ])
+        def test_restrict_xpaths_encoding(self):
+            """Test restrict_xpaths with encodings"""
+            html = b"""<html><head><title>Page title<title>
+            <body><p><a href="item/12.html">Item 12</a></p>
+            <div class='links'>
+            <p><a href="/about.html">About us\xa3</a></p>
+            </div>
+            <div>
+            <p><a href="/nofollow.html">This shouldn't be followed</a></p>
+            </div>
+            </body></html>"""
+            response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
 
-        lx = self.extractor_cls(allow='sample', deny='3')
-        self.assertEqual([link for link in lx.extract_links(self.response)], [
-            Link(url='http://example.com/sample1.html', text=u''),
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-        ])
+            lx = self.extractor_cls(restrict_xpaths="//div[@class='links']")
+            self.assertEqual(lx.extract_links(response),
+                             [Link(url='http://example.org/about.html', text=u'About us\xa3')])
 
-        lx = self.extractor_cls(allow_domains='google.com')
-        self.assertEqual([link for link in lx.extract_links(self.response)], [
-            Link(url='http://www.google.com/something', text=u''),
-        ])
+        def test_restrict_xpaths_with_html_entities(self):
+            html = b'<html><body><p><a href="/&hearts;/you?c=&euro;">text</a></p></body></html>'
+            response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
+            links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
+            self.assertEqual(links,
+                             [Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')])
 
-        lx = self.extractor_cls(deny_domains='example.com')
-        self.assertEqual([link for link in lx.extract_links(self.response)], [
-            Link(url='http://www.google.com/something', text=u''),
-        ])
+        def test_restrict_xpaths_concat_in_handle_data(self):
+            """html entities cause SGMLParser to call handle_data hook twice"""
+            body = b"""<html><body><div><a href="/foo">&gt;\xbe\xa9&lt;\xb6\xab</a></body></html>"""
+            response = HtmlResponse("http://example.org", body=body, encoding='gb18030')
+            lx = self.extractor_cls(restrict_xpaths="//div")
+            self.assertEqual(lx.extract_links(response),
+                             [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
+                                   fragment='', nofollow=False)])
 
-    def test_nofollow(self):
-        '''Test the extractor's behaviour for links with rel="nofollow"'''
+        def test_restrict_css(self):
+            lx = self.extractor_cls(restrict_css=('#subwrapper a',))
+            self.assertEqual(lx.extract_links(self.response), [
+                Link(url='http://example.com/sample2.html', text=u'sample 2')
+            ])
 
-        html = """<html><head><title>Page title<title>
-        <body>
+        def test_restrict_css_and_restrict_xpaths_together(self):
+            lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ),
+                                    restrict_css=('#subwrapper + a', ))
+            self.assertEqual([link for link in lx.extract_links(self.response)], [
+                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+            ])
+
+        def test_area_tag_with_unicode_present(self):
+            body = b"""<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>"""
+            response = HtmlResponse("http://example.org", body=body, encoding='utf-8')
+            lx = self.extractor_cls()
+            lx.extract_links(response)
+            lx.extract_links(response)
+            lx.extract_links(response)
+            self.assertEqual(lx.extract_links(response),
+                             [Link(url='http://example.org/foo', text=u'',
+                                   fragment='', nofollow=False)])
+
+        def test_encoded_url(self):
+            body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>"""
+            response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
+            lx = self.extractor_cls()
+            self.assertEqual(lx.extract_links(response), [
+                Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
+            ])
+
+        def test_encoded_url_in_restricted_xpath(self):
+            body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>"""
+            response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
+            lx = self.extractor_cls(restrict_xpaths="//div")
+            self.assertEqual(lx.extract_links(response), [
+                Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
+            ])
+
+        def test_ignored_extensions(self):
+            # jpg is ignored by default
+            html = b"""<a href="page.html">asd</a> and <a href="photo.jpg">"""
+            response = HtmlResponse("http://example.org/", body=html)
+            lx = self.extractor_cls()
+            self.assertEqual(lx.extract_links(response), [
+                Link(url='http://example.org/page.html', text=u'asd'),
+            ])
+
+            # override denied extensions
+            lx = self.extractor_cls(deny_extensions=['html'])
+            self.assertEqual(lx.extract_links(response), [
+                Link(url='http://example.org/photo.jpg'),
+            ])
+
+        def test_process_value(self):
+            """Test restrict_xpaths with encodings"""
+            html = b"""
+            <a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Link text</a>
+            <a href="/about.html">About us</a>
+            """
+            response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
+
+            def process_value(value):
+                m = re.search("javascript:goToPage\('(.*?)'", value)
+                if m:
+                    return m.group(1)
+
+            lx = self.extractor_cls(process_value=process_value)
+            self.assertEqual(lx.extract_links(response),
+                             [Link(url='http://example.org/other/page.html', text='Link text')])
+
+        def test_base_url_with_restrict_xpaths(self):
+            html = b"""<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
+            <body><p><a href="item/12.html">Item 12</a></p>
+            </body></html>"""
+            response = HtmlResponse("http://example.org/somepage/index.html", body=html)
+            lx = self.extractor_cls(restrict_xpaths="//p")
+            self.assertEqual(lx.extract_links(response),
+                             [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
+
+        def test_attrs(self):
+            lx = self.extractor_cls(attrs="href")
+            self.assertEqual(lx.extract_links(self.response), [
+                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://www.google.com/something', text=u''),
+                Link(url='http://example.com/innertag.html', text=u'inner tag'),
+            ])
+
+            lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())
+            self.assertEqual(lx.extract_links(self.response), [
+                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample2.jpg', text=u''),
+                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://www.google.com/something', text=u''),
+                Link(url='http://example.com/innertag.html', text=u'inner tag'),
+            ])
+
+            lx = self.extractor_cls(attrs=None)
+            self.assertEqual(lx.extract_links(self.response), [])
+
+        def test_tags(self):
+            html = b"""<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>"""
+            response = HtmlResponse("http://example.com/index.html", body=html)
+
+            lx = self.extractor_cls(tags=None)
+            self.assertEqual(lx.extract_links(response), [])
+
+            lx = self.extractor_cls()
+            self.assertEqual(lx.extract_links(response), [
+                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+            ])
+
+            lx = self.extractor_cls(tags="area")
+            self.assertEqual(lx.extract_links(response), [
+                Link(url='http://example.com/sample1.html', text=u''),
+            ])
+
+            lx = self.extractor_cls(tags="a")
+            self.assertEqual(lx.extract_links(response), [
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+            ])
+
+            lx = self.extractor_cls(tags=("a","img"), attrs=("href", "src"), deny_extensions=())
+            self.assertEqual(lx.extract_links(response), [
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample2.jpg', text=u''),
+            ])
+
+        def test_tags_attrs(self):
+            html = b"""
+            <html><body>
+            <div id="item1" data-url="get?id=1"><a href="#">Item 1</a></div>
+            <div id="item2" data-url="get?id=2"><a href="#">Item 2</a></div>
+            </body></html>
+            """
+            response = HtmlResponse("http://example.com/index.html", body=html)
+
+            lx = self.extractor_cls(tags='div', attrs='data-url')
+            self.assertEqual(lx.extract_links(response), [
+                Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
+                Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
+            ])
+
+            lx = self.extractor_cls(tags=('div',), attrs=('data-url',))
+            self.assertEqual(lx.extract_links(response), [
+                Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
+                Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
+            ])
+
+        def test_xhtml(self):
+            xhtml = b"""
+    <?xml version="1.0"?>
+    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+    <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+    <head>
+        <title>XHTML document title</title>
+    </head>
+    <body>
         <div class='links'>
         <p><a href="/about.html">About us</a></p>
         </div>
@@ -199,306 +369,49 @@ class BaseLinkExtractorTestCase(unittest.TestCase):
         <div>
         <p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p>
         </div>
-        </body></html>"""
-        response = HtmlResponse("http://example.org/somepage/index.html", body=html)
+    </body>
+    </html>
+            """
 
-        lx = self.extractor_cls()
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.org/about.html', text=u'About us'),
-            Link(url='http://example.org/follow.html', text=u'Follow this link'),
-            Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True),
-            Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'),
-        ])
+            response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)
 
-    def test_matches(self):
-        url1 = 'http://lotsofstuff.com/stuff1/index'
-        url2 = 'http://evenmorestuff.com/uglystuff/index'
+            lx = self.extractor_cls()
+            self.assertEqual(lx.extract_links(response),
+                             [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+                              Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+                              Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
+                              Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)]
+                            )
 
-        lx = self.extractor_cls(allow=(r'stuff1', ))
-        self.assertEqual(lx.matches(url1), True)
-        self.assertEqual(lx.matches(url2), False)
+            response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
 
-        lx = self.extractor_cls(deny=(r'uglystuff', ))
-        self.assertEqual(lx.matches(url1), True)
-        self.assertEqual(lx.matches(url2), False)
+            lx = self.extractor_cls()
+            self.assertEqual(lx.extract_links(response),
+                             [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+                              Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+                              Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
+                              Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)]
+                            )
 
-        lx = self.extractor_cls(allow_domains=('evenmorestuff.com', ))
-        self.assertEqual(lx.matches(url1), False)
-        self.assertEqual(lx.matches(url2), True)
-
-        lx = self.extractor_cls(deny_domains=('lotsofstuff.com', ))
-        self.assertEqual(lx.matches(url1), False)
-        self.assertEqual(lx.matches(url2), True)
-
-        lx = self.extractor_cls(allow=('blah1',), deny=('blah2',),
-                               allow_domains=('blah1.com',),
-                               deny_domains=('blah2.com',))
-        self.assertEqual(lx.matches('http://blah1.com/blah1'), True)
-        self.assertEqual(lx.matches('http://blah1.com/blah2'), False)
-        self.assertEqual(lx.matches('http://blah2.com/blah1'), False)
-        self.assertEqual(lx.matches('http://blah2.com/blah2'), False)
-
-    def test_restrict_xpaths(self):
-        lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ))
-        self.assertEqual([link for link in lx.extract_links(self.response)], [
-            Link(url='http://example.com/sample1.html', text=u''),
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-        ])
-
-    def test_restrict_xpaths_encoding(self):
-        """Test restrict_xpaths with encodings"""
-        html = """<html><head><title>Page title<title>
-        <body><p><a href="item/12.html">Item 12</a></p>
-        <div class='links'>
-        <p><a href="/about.html">About us\xa3</a></p>
-        </div>
-        <div>
-        <p><a href="/nofollow.html">This shouldn't be followed</a></p>
-        </div>
-        </body></html>"""
-        response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
-
-        lx = self.extractor_cls(restrict_xpaths="//div[@class='links']")
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://example.org/about.html', text=u'About us\xa3')])
-
-    def test_restrict_xpaths_with_html_entities(self):
-        html = '<html><body><p><a href="/&hearts;/you?c=&euro;">text</a></p></body></html>'
-        response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
-        links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
-        self.assertEqual(links,
-                         [Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')])
-
-    def test_restrict_xpaths_concat_in_handle_data(self):
-        """html entities cause SGMLParser to call handle_data hook twice"""
-        body = """<html><body><div><a href="/foo">&gt;\xbe\xa9&lt;\xb6\xab</a></body></html>"""
-        response = HtmlResponse("http://example.org", body=body, encoding='gb18030')
-        lx = self.extractor_cls(restrict_xpaths="//div")
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
-                               fragment='', nofollow=False)])
-
-    def test_restrict_css(self):
-        lx = self.extractor_cls(restrict_css=('#subwrapper a',))
-        self.assertEqual(lx.extract_links(self.response), [
-            Link(url='http://example.com/sample2.html', text=u'sample 2')
-        ])
-
-    def test_restrict_css_and_restrict_xpaths_together(self):
-        lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ),
-                                restrict_css=('#subwrapper + a', ))
-        self.assertEqual([link for link in lx.extract_links(self.response)], [
-            Link(url='http://example.com/sample1.html', text=u''),
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-        ])
-
-    def test_area_tag_with_unicode_present(self):
-        body = """<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>"""
-        response = HtmlResponse("http://example.org", body=body, encoding='utf-8')
-        lx = self.extractor_cls()
-        lx.extract_links(response)
-        lx.extract_links(response)
-        lx.extract_links(response)
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://example.org/foo', text=u'',
-                               fragment='', nofollow=False)])
-
-    def test_encoded_url(self):
-        body = """<html><body><div><a href="?page=2">BinB</a></body></html>"""
-        response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
-        lx = self.extractor_cls()
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
-        ])
-
-    def test_encoded_url_in_restricted_xpath(self):
-        body = """<html><body><div><a href="?page=2">BinB</a></body></html>"""
-        response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
-        lx = self.extractor_cls(restrict_xpaths="//div")
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
-        ])
-
-    def test_ignored_extensions(self):
-        # jpg is ignored by default
-        html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
-        response = HtmlResponse("http://example.org/", body=html)
-        lx = self.extractor_cls()
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.org/page.html', text=u'asd'),
-        ])
-
-        # override denied extensions
-        lx = self.extractor_cls(deny_extensions=['html'])
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.org/photo.jpg'),
-        ])
-
-    def test_process_value(self):
-        """Test restrict_xpaths with encodings"""
-        html = """
-        <a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Link text</a>
-        <a href="/about.html">About us</a>
-        """
-        response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
-
-        def process_value(value):
-            m = re.search("javascript:goToPage\('(.*?)'", value)
-            if m:
-                return m.group(1)
-
-        lx = self.extractor_cls(process_value=process_value)
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://example.org/other/page.html', text='Link text')])
-
-    def test_base_url_with_restrict_xpaths(self):
-        html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
-        <body><p><a href="item/12.html">Item 12</a></p>
-        </body></html>"""
-        response = HtmlResponse("http://example.org/somepage/index.html", body=html)
-        lx = self.extractor_cls(restrict_xpaths="//p")
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
-
-    def test_attrs(self):
-        lx = self.extractor_cls(attrs="href")
-        self.assertEqual(lx.extract_links(self.response), [
-            Link(url='http://example.com/sample1.html', text=u''),
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-            Link(url='http://www.google.com/something', text=u''),
-            Link(url='http://example.com/innertag.html', text=u'inner tag'),
-        ])
-
-        lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())
-        self.assertEqual(lx.extract_links(self.response), [
-            Link(url='http://example.com/sample1.html', text=u''),
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-            Link(url='http://example.com/sample2.jpg', text=u''),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-            Link(url='http://www.google.com/something', text=u''),
-            Link(url='http://example.com/innertag.html', text=u'inner tag'),
-        ])
-
-        lx = self.extractor_cls(attrs=None)
-        self.assertEqual(lx.extract_links(self.response), [])
-
-    def test_tags(self):
-        html = """<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>"""
-        response = HtmlResponse("http://example.com/index.html", body=html)
-
-        lx = self.extractor_cls(tags=None)
-        self.assertEqual(lx.extract_links(response), [])
-
-        lx = self.extractor_cls()
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.com/sample1.html', text=u''),
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-        ])
-
-        lx = self.extractor_cls(tags="area")
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.com/sample1.html', text=u''),
-        ])
-
-        lx = self.extractor_cls(tags="a")
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-        ])
-
-        lx = self.extractor_cls(tags=("a","img"), attrs=("href", "src"), deny_extensions=())
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-            Link(url='http://example.com/sample2.jpg', text=u''),
-        ])
-
-    def test_tags_attrs(self):
-        html = """
-        <html><body>
-        <div id="item1" data-url="get?id=1"><a href="#">Item 1</a></div>
-        <div id="item2" data-url="get?id=2"><a href="#">Item 2</a></div>
-        </body></html>
-        """
-        response = HtmlResponse("http://example.com/index.html", body=html)
-
-        lx = self.extractor_cls(tags='div', attrs='data-url')
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
-            Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
-        ])
-
-        lx = self.extractor_cls(tags=('div',), attrs=('data-url',))
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
-            Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
-        ])
-
-    def test_xhtml(self):
-        xhtml = """
-<?xml version="1.0"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
-    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-    <title>XHTML document title</title>
-</head>
-<body>
-    <div class='links'>
-    <p><a href="/about.html">About us</a></p>
-    </div>
-    <div>
-    <p><a href="/follow.html">Follow this link</a></p>
-    </div>
-    <div>
-    <p><a href="/nofollow.html" rel="nofollow">Dont follow this one</a></p>
-    </div>
-    <div>
-    <p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p>
-    </div>
-</body>
-</html>
-        """
-
-        response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)
-
-        lx = self.extractor_cls()
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                          Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                          Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                          Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)]
-                        )
-
-        response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
-
-        lx = self.extractor_cls()
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                          Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                          Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                          Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)]
-                        )
-
-    def test_link_wrong_href(self):
-        html = """
-        <a href="http://example.org/item1.html">Item 1</a>
-        <a href="http://[example.org/item2.html">Item 2</a>
-        <a href="http://example.org/item3.html">Item 3</a>
-        """
-        response = HtmlResponse("http://example.org/index.html", body=html)
-        lx = self.extractor_cls()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
-        ])
+        def test_link_wrong_href(self):
+            html = b"""
+            <a href="http://example.org/item1.html">Item 1</a>
+            <a href="http://[example.org/item2.html">Item 2</a>
+            <a href="http://example.org/item3.html">Item 3</a>
+            """
+            response = HtmlResponse("http://example.org/index.html", body=html)
+            lx = self.extractor_cls()
+            self.assertEqual([link for link in lx.extract_links(response)], [
+                Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
+                Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
+            ])
 
 
-class LxmlLinkExtractorTestCase(BaseLinkExtractorTestCase):
+class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
     extractor_cls = LxmlLinkExtractor
 
     def test_link_wrong_href(self):
-        html = """
+        html = b"""
         <a href="http://example.org/item1.html">Item 1</a>
         <a href="http://[example.org/item2.html">Item 2</a>
         <a href="http://example.org/item3.html">Item 3</a>
@@ -514,94 +427,3 @@ class LxmlLinkExtractorTestCase(BaseLinkExtractorTestCase):
     def test_restrict_xpaths_with_html_entities(self):
         super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()
 
-
-class HtmlParserLinkExtractorTestCase(unittest.TestCase):
-
-    def setUp(self):
-        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
-        self.response = HtmlResponse(url='http://example.com/index', body=body)
-
-    def test_extraction(self):
-        # Default arguments
-        lx = HtmlParserLinkExtractor()
-        self.assertEqual(lx.extract_links(self.response),
-                         [Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                          Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-                          Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
-                          Link(url='http://www.google.com/something', text=u''),
-                          Link(url='http://example.com/innertag.html', text=u'inner tag'),])
-
-    def test_link_wrong_href(self):
-        html = """
-        <a href="http://example.org/item1.html">Item 1</a>
-        <a href="http://[example.org/item2.html">Item 2</a>
-        <a href="http://example.org/item3.html">Item 3</a>
-        """
-        response = HtmlResponse("http://example.org/index.html", body=html)
-        lx = HtmlParserLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
-        ])
-
-
-class SgmlLinkExtractorTestCase(BaseLinkExtractorTestCase):
-    extractor_cls = SgmlLinkExtractor
-
-    def test_deny_extensions(self):
-        html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
-        response = HtmlResponse("http://example.org/", body=html)
-        lx = SgmlLinkExtractor(deny_extensions="jpg")
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.org/page.html', text=u'asd'),
-        ])
-
-    def test_attrs_sgml(self):
-        html = """<html><area href="sample1.html"></area>
-        <a ref="sample2.html">sample text 2</a></html>"""
-        response = HtmlResponse("http://example.com/index.html", body=html)
-        lx = SgmlLinkExtractor(attrs="href")
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.com/sample1.html', text=u''),
-        ])
-
-    def test_link_nofollow(self):
-        html = """
-        <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
-        <a href="about.html">About us</a>
-        """
-        response = HtmlResponse("http://example.org/page.html", body=html)
-        lx = SgmlLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
-            Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
-        ])
-
-
-class RegexLinkExtractorTestCase(unittest.TestCase):
-
-    def setUp(self):
-        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
-        self.response = HtmlResponse(url='http://example.com/index', body=body)
-
-    def test_extraction(self):
-        # Default arguments
-        lx = RegexLinkExtractor()
-        self.assertEqual(lx.extract_links(self.response),
-                         [Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                          Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-                          Link(url='http://www.google.com/something', text=u''),
-                          Link(url='http://example.com/innertag.html', text=u'inner tag'),])
-
-    def test_link_wrong_href(self):
-        html = """
-        <a href="http://example.org/item1.html">Item 1</a>
-        <a href="http://[example.org/item2.html">Item 2</a>
-        <a href="http://example.org/item3.html">Item 3</a>
-        """
-        response = HtmlResponse("http://example.org/index.html", body=html)
-        lx = RegexLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
-        ])
diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py
new file mode 100644
index 000000000..fd5a78879
--- /dev/null
+++ b/tests/test_linkextractors_deprecated.py
@@ -0,0 +1,190 @@
+import unittest
+from scrapy.linkextractors.regex import RegexLinkExtractor
+from scrapy.http import HtmlResponse
+from scrapy.link import Link
+from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor
+from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
+from tests import get_testdata
+
+from tests.test_linkextractors import Base
+
+
+class BaseSgmlLinkExtractorTestCase(unittest.TestCase):
+    # XXX: should we move some of these tests to base link extractor tests?
+
+    def test_basic(self):
+        html = """<html><head><title>Page title<title>
+        <body><p><a href="item/12.html">Item 12</a></p>
+        <p><a href="/about.html">About us</a></p>
+        <img src="/logo.png" alt="Company logo (not a link)" />
+        <p><a href="../othercat.html">Other category</a></p>
+        <p><a href="/">&gt;&gt;</a></p>
+        <p><a href="/" /></p>
+        </body></html>"""
+        response = HtmlResponse("http://example.org/somepage/index.html", body=html)
+
+        lx = BaseSgmlLinkExtractor()  # default: tag=a, attr=href
+        self.assertEqual(lx.extract_links(response),
+                         [Link(url='http://example.org/somepage/item/12.html', text='Item 12'),
+                          Link(url='http://example.org/about.html', text='About us'),
+                          Link(url='http://example.org/othercat.html', text='Other category'),
+                          Link(url='http://example.org/', text='>>'),
+                          Link(url='http://example.org/', text='')])
+
+    def test_base_url(self):
+        html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
+        <body><p><a href="item/12.html">Item 12</a></p>
+        </body></html>"""
+        response = HtmlResponse("http://example.org/somepage/index.html", body=html)
+
+        lx = BaseSgmlLinkExtractor()  # default: tag=a, attr=href
+        self.assertEqual(lx.extract_links(response),
+                         [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
+
+        # base url is an absolute path and relative to host
+        html = """<html><head><title>Page title<title><base href="/" />
+        <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
+        response = HtmlResponse("https://example.org/somepage/index.html", body=html)
+        self.assertEqual(lx.extract_links(response),
+                         [Link(url='https://example.org/item/12.html', text='Item 12')])
+
+        # base url has no scheme
+        html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" />
+        <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
+        response = HtmlResponse("https://example.org/somepage/index.html", body=html)
+        self.assertEqual(lx.extract_links(response),
+                         [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')])
+
+    def test_link_text_wrong_encoding(self):
+        html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>"""
+        response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8')
+        lx = BaseSgmlLinkExtractor()
+        self.assertEqual(lx.extract_links(response), [
+            Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'),
+        ])
+
+    def test_extraction_encoding(self):
+        body = get_testdata('link_extractor', 'linkextractor_noenc.html')
+        response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})
+        response_noenc = HtmlResponse(url='http://example.com/noenc', body=body)
+        body = get_testdata('link_extractor', 'linkextractor_latin1.html')
+        response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body)
+
+        lx = BaseSgmlLinkExtractor()
+        self.assertEqual(lx.extract_links(response_utf8), [
+            Link(url='http://example.com/sample_%C3%B1.html', text=''),
+            Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
+        ])
+
+        self.assertEqual(lx.extract_links(response_noenc), [
+            Link(url='http://example.com/sample_%C3%B1.html', text=''),
+            Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
+        ])
+
+        self.assertEqual(lx.extract_links(response_latin1), [
+            Link(url='http://example.com/sample_%F1.html', text=''),
+            Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')),
+        ])
+
+    def test_matches(self):
+        url1 = 'http://lotsofstuff.com/stuff1/index'
+        url2 = 'http://evenmorestuff.com/uglystuff/index'
+
+        lx = BaseSgmlLinkExtractor()
+        self.assertEqual(lx.matches(url1), True)
+        self.assertEqual(lx.matches(url2), True)
+
+
+class HtmlParserLinkExtractorTestCase(unittest.TestCase):
+
+    def setUp(self):
+        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
+        self.response = HtmlResponse(url='http://example.com/index', body=body)
+
+    def test_extraction(self):
+        # Default arguments
+        lx = HtmlParserLinkExtractor()
+        self.assertEqual(lx.extract_links(self.response),
+                         [Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                          Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                          Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+                          Link(url='http://www.google.com/something', text=u''),
+                          Link(url='http://example.com/innertag.html', text=u'inner tag'),])
+
+    def test_link_wrong_href(self):
+        html = """
+        <a href="http://example.org/item1.html">Item 1</a>
+        <a href="http://[example.org/item2.html">Item 2</a>
+        <a href="http://example.org/item3.html">Item 3</a>
+        """
+        response = HtmlResponse("http://example.org/index.html", body=html)
+        lx = HtmlParserLinkExtractor()
+        self.assertEqual([link for link in lx.extract_links(response)], [
+            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
+            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
+        ])
+
+
+class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
+    extractor_cls = SgmlLinkExtractor
+
+    def test_deny_extensions(self):
+        html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
+        response = HtmlResponse("http://example.org/", body=html)
+        lx = SgmlLinkExtractor(deny_extensions="jpg")
+        self.assertEqual(lx.extract_links(response), [
+            Link(url='http://example.org/page.html', text=u'asd'),
+        ])
+
+    def test_attrs_sgml(self):
+        html = """<html><area href="sample1.html"></area>
+        <a ref="sample2.html">sample text 2</a></html>"""
+        response = HtmlResponse("http://example.com/index.html", body=html)
+        lx = SgmlLinkExtractor(attrs="href")
+        self.assertEqual(lx.extract_links(response), [
+            Link(url='http://example.com/sample1.html', text=u''),
+        ])
+
+    def test_link_nofollow(self):
+        html = """
+        <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
+        <a href="about.html">About us</a>
+        """
+        response = HtmlResponse("http://example.org/page.html", body=html)
+        lx = SgmlLinkExtractor()
+        self.assertEqual([link for link in lx.extract_links(response)], [
+            Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
+            Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
+        ])
+
+
+class RegexLinkExtractorTestCase(unittest.TestCase):
+    # XXX: RegexLinkExtractor is not deprecated yet, but it must be rewritten
+    # not to depend on SgmlLinkExractor. Its speed is also much worse
+    # than it should be.
+
+    def setUp(self):
+        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
+        self.response = HtmlResponse(url='http://example.com/index', body=body)
+
+    def test_extraction(self):
+        # Default arguments
+        lx = RegexLinkExtractor()
+        self.assertEqual(lx.extract_links(self.response),
+                         [Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                          Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                          Link(url='http://www.google.com/something', text=u''),
+                          Link(url='http://example.com/innertag.html', text=u'inner tag'),])
+
+    def test_link_wrong_href(self):
+        html = """
+        <a href="http://example.org/item1.html">Item 1</a>
+        <a href="http://[example.org/item2.html">Item 2</a>
+        <a href="http://example.org/item3.html">Item 3</a>
+        """
+        response = HtmlResponse("http://example.org/index.html", body=html)
+        lx = RegexLinkExtractor()
+        self.assertEqual([link for link in lx.extract_links(response)], [
+            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
+            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
+        ])

From d5984bbea99f81765596e1aa57d03aff51612576 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 28 Aug 2015 02:12:36 +0500
Subject: [PATCH 0524/2974] PY3 port scrapy.spiders

---
 scrapy/spiders/crawl.py   |  9 +++++++--
 scrapy/spiders/init.py    |  1 +
 scrapy/spiders/sitemap.py | 14 +++++++++-----
 scrapy/utils/gz.py        |  6 ++++--
 tests/py3-ignores.txt     |  1 -
 tests/test_spider.py      | 22 ++++++++++++++--------
 tests/test_utils_gz.py    |  2 +-
 7 files changed, 36 insertions(+), 19 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 77551753e..031f649d6 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -6,14 +6,17 @@ See documentation in docs/topics/spiders.rst
 """
 
 import copy
+import six
 
 from scrapy.http import Request, HtmlResponse
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.spiders import Spider
 
+
 def identity(x):
     return x
 
+
 class Rule(object):
 
     def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity):
@@ -27,6 +30,7 @@ class Rule(object):
         else:
             self.follow = follow
 
+
 class CrawlSpider(Spider):
 
     rules = ()
@@ -49,7 +53,8 @@ class CrawlSpider(Spider):
             return
         seen = set()
         for n, rule in enumerate(self._rules):
-            links = [l for l in rule.link_extractor.extract_links(response) if l not in seen]
+            links = [lnk for lnk in rule.link_extractor.extract_links(response)
+                     if lnk not in seen]
             if links and rule.process_links:
                 links = rule.process_links(links)
             for link in links:
@@ -77,7 +82,7 @@ class CrawlSpider(Spider):
         def get_method(method):
             if callable(method):
                 return method
-            elif isinstance(method, basestring):
+            elif isinstance(method, six.string_types):
                 return getattr(self, method, None)
 
         self._rules = [copy.copy(r) for r in self.rules]
diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py
index 7717c8819..2efb1a869 100644
--- a/scrapy/spiders/init.py
+++ b/scrapy/spiders/init.py
@@ -1,6 +1,7 @@
 from scrapy.spiders import Spider
 from scrapy.utils.spider import iterate_spider_output
 
+
 class InitSpider(Spider):
     """Base Spider with initialization facilities"""
 
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 5aa0b944d..eede467a8 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -1,5 +1,6 @@
 import re
 import logging
+import six
 
 from scrapy.spiders import Spider
 from scrapy.http import Request, XmlResponse
@@ -20,13 +21,14 @@ class SitemapSpider(Spider):
         super(SitemapSpider, self).__init__(*a, **kw)
         self._cbs = []
         for r, c in self.sitemap_rules:
-            if isinstance(c, basestring):
+            if isinstance(c, six.string_types):
                 c = getattr(self, c)
             self._cbs.append((regex(r), c))
         self._follow = [regex(x) for x in self.sitemap_follow]
 
     def start_requests(self):
-        return (Request(x, callback=self._parse_sitemap) for x in self.sitemap_urls)
+        for url in self.sitemap_urls:
+            yield Request(url, self._parse_sitemap)
 
     def _parse_sitemap(self, response):
         if response.url.endswith('/robots.txt'):
@@ -52,8 +54,8 @@ class SitemapSpider(Spider):
                             break
 
     def _get_sitemap_body(self, response):
-        """Return the sitemap body contained in the given response, or None if the
-        response is not a sitemap.
+        """Return the sitemap body contained in the given response,
+        or None if the response is not a sitemap.
         """
         if isinstance(response, XmlResponse):
             return response.body
@@ -64,11 +66,13 @@ class SitemapSpider(Spider):
         elif response.url.endswith('.xml.gz'):
             return gunzip(response.body)
 
+
 def regex(x):
-    if isinstance(x, basestring):
+    if isinstance(x, six.string_types):
         return re.compile(x)
     return x
 
+
 def iterloc(it, alt=False):
     for d in it:
         yield d['loc']
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 741948359..7fa4bba57 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -7,6 +7,7 @@ except ImportError:
 
 from gzip import GzipFile
 
+
 def gunzip(data):
     """Gunzip the given data and return as much data as possible.
 
@@ -31,7 +32,8 @@ def gunzip(data):
                 raise
     return output
 
+
 def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
-    ctype = response.headers.get('Content-Type', '')
-    return ctype in ('application/x-gzip', 'application/gzip')
+    ctype = response.headers.get('Content-Type', b'')
+    return ctype in (b'application/x-gzip', b'application/gzip')
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index b40293f57..2eb22f149 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -28,7 +28,6 @@ tests/test_spidermiddleware_depth.py
 tests/test_spidermiddleware_httperror.py
 tests/test_spidermiddleware_offsite.py
 tests/test_spidermiddleware_referer.py
-tests/test_spider.py
 tests/test_utils_iterators.py
 tests/test_utils_template.py
 tests/test_webclient.py
diff --git a/tests/test_spider.py b/tests/test_spider.py
index f2dfd2dce..4d5d4b07e 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -301,26 +301,32 @@ class SitemapSpiderTest(SpiderTest):
     g.close()
     GZBODY = f.getvalue()
 
-    def test_get_sitemap_body(self):
+    def assertSitemapBody(self, response, body):
         spider = self.spider_class("example.com")
+        self.assertEqual(spider._get_sitemap_body(response), body)
 
+    def test_get_sitemap_body(self):
         r = XmlResponse(url="http://www.example.com/", body=self.BODY)
-        self.assertEqual(spider._get_sitemap_body(r), self.BODY)
+        self.assertSitemapBody(r, self.BODY)
 
         r = HtmlResponse(url="http://www.example.com/", body=self.BODY)
-        self.assertEqual(spider._get_sitemap_body(r), None)
+        self.assertSitemapBody(r, None)
 
         r = Response(url="http://www.example.com/favicon.ico", body=self.BODY)
-        self.assertEqual(spider._get_sitemap_body(r), None)
+        self.assertSitemapBody(r, None)
 
-        r = Response(url="http://www.example.com/sitemap", body=self.GZBODY, headers={"content-type": "application/gzip"})
-        self.assertEqual(spider._get_sitemap_body(r), self.BODY)
+    def test_get_sitemap_body_gzip_headers(self):
+        r = Response(url="http://www.example.com/sitemap", body=self.GZBODY,
+                     headers={"content-type": "application/gzip"})
+        self.assertSitemapBody(r, self.BODY)
 
+    def test_get_sitemap_body_xml_url(self):
         r = TextResponse(url="http://www.example.com/sitemap.xml", body=self.BODY)
-        self.assertEqual(spider._get_sitemap_body(r), self.BODY)
+        self.assertSitemapBody(r, self.BODY)
 
+    def test_get_sitemap_body_xml_url_compressed(self):
         r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY)
-        self.assertEqual(spider._get_sitemap_body(r), self.BODY)
+        self.assertSitemapBody(r, self.BODY)
 
 
 class BaseSpiderDeprecationTest(unittest.TestCase):
diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index 94e7b71be..8fb1e414d 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -7,7 +7,7 @@ from tests import tests_datadir
 SAMPLEDIR = join(tests_datadir, 'compressed')
 
 
-class GzTest(unittest.TestCase):
+class GunzipTest(unittest.TestCase):
 
     def test_gunzip_basic(self):
         with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f:

From ff24cbbc477e8bd5459034e702bafd0c5ea1fc43 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 28 Aug 2015 02:52:17 +0500
Subject: [PATCH 0525/2974] PY3 depth, offsite and referer spider middlewares;
 Crawler

---
 scrapy/spidermiddlewares/depth.py      | 14 +++++++++-----
 scrapy/spidermiddlewares/offsite.py    |  1 +
 tests/py3-ignores.txt                  |  4 ----
 tests/test_spidermiddleware_referer.py |  2 +-
 4 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 795b60eb4..e2f039146 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -35,14 +35,18 @@ class DepthMiddleware(object):
                 if self.prio:
                     request.priority -= depth * self.prio
                 if self.maxdepth and depth > self.maxdepth:
-                    logger.debug("Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
-                                 {'maxdepth': self.maxdepth, 'requrl': request.url},
-                                 extra={'spider': spider})
+                    logger.debug(
+                        "Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
+                        {'maxdepth': self.maxdepth, 'requrl': request.url},
+                        extra={'spider': spider}
+                    )
                     return False
                 elif self.stats:
                     if self.verbose_stats:
-                        self.stats.inc_value('request_depth_count/%s' % depth, spider=spider)
-                    self.stats.max_value('request_depth_max', depth, spider=spider)
+                        self.stats.inc_value('request_depth_count/%s' % depth,
+                                             spider=spider)
+                    self.stats.max_value('request_depth_max', depth,
+                                         spider=spider)
             return True
 
         # base case (depth=0)
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index a90f9f1e0..ea1c9270f 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -13,6 +13,7 @@ from scrapy.utils.httpobj import urlparse_cached
 
 logger = logging.getLogger(__name__)
 
+
 class OffsiteMiddleware(object):
 
     def __init__(self, stats):
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 2eb22f149..759eeffff 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -5,7 +5,6 @@ tests/test_commands.py
 tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
-tests/test_crawler.py
 tests/test_downloader_handlers.py
 tests/test_downloadermiddleware_ajaxcrawlable.py
 tests/test_downloadermiddleware_defaultheaders.py
@@ -24,10 +23,7 @@ tests/test_mail.py
 tests/test_pipeline_files.py
 tests/test_pipeline_images.py
 tests/test_proxy_connect.py
-tests/test_spidermiddleware_depth.py
 tests/test_spidermiddleware_httperror.py
-tests/test_spidermiddleware_offsite.py
-tests/test_spidermiddleware_referer.py
 tests/test_utils_iterators.py
 tests/test_utils_template.py
 tests/test_webclient.py
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index d773ea8d3..bd7673efb 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -17,5 +17,5 @@ class TestRefererMiddleware(TestCase):
 
         out = list(self.mw.process_spider_output(res, reqs, self.spider))
         self.assertEquals(out[0].headers.get('Referer'),
-                          'http://scrapytest.org')
+                          b'http://scrapytest.org')
 

From f7052413e092346b7d460d589d23fecaa4351930 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 28 Aug 2015 23:04:02 +0500
Subject: [PATCH 0526/2974] PY3 raise an exception if bytes are passed as url
 to Link constructor

---
 scrapy/link.py     | 15 +++++++++++----
 tests/test_link.py | 38 ++++++++++++++++++--------------------
 2 files changed, 29 insertions(+), 24 deletions(-)

diff --git a/scrapy/link.py b/scrapy/link.py
index dc6e64adc..2c8301680 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -4,7 +4,10 @@ This module defines the Link object used in Link extractors.
 For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
 """
-from scrapy.utils.python import to_native_str
+import warnings
+import six
+
+from scrapy.utils.python import to_bytes
 
 
 class Link(object):
@@ -14,9 +17,13 @@ class Link(object):
 
     def __init__(self, url, text='', fragment='', nofollow=False):
         if not isinstance(url, str):
-            import warnings
-            warnings.warn("Link urls must be str objects.")
-            url = to_native_str(url)
+            if six.PY2:
+                warnings.warn("Link urls must be str objects. "
+                              "Assuming utf-8 encoding (which could be wrong)")
+                url = to_bytes(url, encoding='utf8')
+            else:
+                got = url.__class__.__name__
+                raise TypeError("Link urls must be str objects, got %s" % got)
         self.url = url
         self.text = text
         self.fragment = fragment
diff --git a/tests/test_link.py b/tests/test_link.py
index c8487698f..955430b37 100644
--- a/tests/test_link.py
+++ b/tests/test_link.py
@@ -16,44 +16,42 @@ class LinkTest(unittest.TestCase):
         self.assertNotEqual(hash(link1), hash(link2))
 
     def test_eq_and_hash(self):
-        l1 = Link(b"http://www.example.com")
-        l2 = Link(b"http://www.example.com/other")
-        l3 = Link(b"http://www.example.com")
+        l1 = Link("http://www.example.com")
+        l2 = Link("http://www.example.com/other")
+        l3 = Link("http://www.example.com")
 
         self._assert_same_links(l1, l1)
         self._assert_different_links(l1, l2)
         self._assert_same_links(l1, l3)
 
-        l4 = Link(b"http://www.example.com", text="test")
-        l5 = Link(b"http://www.example.com", text="test2")
-        l6 = Link(b"http://www.example.com", text="test")
+        l4 = Link("http://www.example.com", text="test")
+        l5 = Link("http://www.example.com", text="test2")
+        l6 = Link("http://www.example.com", text="test")
 
         self._assert_same_links(l4, l4)
         self._assert_different_links(l4, l5)
         self._assert_same_links(l4, l6)
 
-        l7 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=False)
-        l8 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=False)
-        l9 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=True)
-        l10 = Link(b"http://www.example.com", text="test", fragment='other', nofollow=False)
+        l7 = Link("http://www.example.com", text="test", fragment='something', nofollow=False)
+        l8 = Link("http://www.example.com", text="test", fragment='something', nofollow=False)
+        l9 = Link("http://www.example.com", text="test", fragment='something', nofollow=True)
+        l10 = Link("http://www.example.com", text="test", fragment='other', nofollow=False)
         self._assert_same_links(l7, l8)
         self._assert_different_links(l7, l9)
         self._assert_different_links(l7, l10)
 
     def test_repr(self):
-        l1 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=True)
+        l1 = Link("http://www.example.com", text="test", fragment='something', nofollow=True)
         l2 = eval(repr(l1))
         self._assert_same_links(l1, l2)
 
-    def test_non_str_url(self):
-        with warnings.catch_warnings(record=True) as w:
-            if six.PY2:
+    def test_non_str_url_py2(self):
+        if six.PY2:
+            with warnings.catch_warnings(record=True) as w:
                 link = Link(u"http://www.example.com/\xa3")
                 self.assertIsInstance(link.url, str)
                 self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3')
-            else:
-                link = Link(b"http://www.example.com/\xc2\xa3")
-                self.assertIsInstance(link.url, str)
-                self.assertEqual(link.url, u'http://www.example.com/\xa3')
-
-        assert len(w) == 1, "warning not issued"
+            assert len(w) == 1, "warning not issued"
+        else:
+            with self.assertRaises(TypeError):
+                Link(b"http://www.example.com/\xc2\xa3")

From 44bfcbcf0f26a469c96feed35c40715b8b58c6a2 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 31 Aug 2015 00:49:38 +0500
Subject: [PATCH 0527/2974] TST split LinkExtractorTestCase.test_extraction
 into several methods; remove duplicated test

---
 tests/test_linkextractors.py | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index e6db6a400..129336d14 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -24,9 +24,7 @@ class Base:
             self.assertTrue(all(isinstance(link.url, str)
                                 for link in lx.extract_links(self.response)))
 
-        def test_extraction(self):
-            '''Test the extractor's behaviour among different situations'''
-
+        def test_extract_all_links(self):
             lx = self.extractor_cls()
             self.assertEqual([link for link in lx.extract_links(self.response)], [
                 Link(url='http://example.com/sample1.html', text=u''),
@@ -36,6 +34,7 @@ class Base:
                 Link(url='http://example.com/innertag.html', text=u'inner tag'),
             ])
 
+        def test_extract_filter_allow(self):
             lx = self.extractor_cls(allow=('sample', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
                 Link(url='http://example.com/sample1.html', text=u''),
@@ -43,6 +42,7 @@ class Base:
                 Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
             ])
 
+        def test_extract_filter_allow_with_duplicates(self):
             lx = self.extractor_cls(allow=('sample', ), unique=False)
             self.assertEqual([link for link in lx.extract_links(self.response)], [
                 Link(url='http://example.com/sample1.html', text=u''),
@@ -51,19 +51,14 @@ class Base:
                 Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
             ])
 
-            lx = self.extractor_cls(allow=('sample', ))
-            self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-            ])
-
+        def test_extract_filter_allow_and_deny(self):
             lx = self.extractor_cls(allow=('sample', ), deny=('3', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
                 Link(url='http://example.com/sample1.html', text=u''),
                 Link(url='http://example.com/sample2.html', text=u'sample 2'),
             ])
 
+        def test_extract_filter_allowed_domains(self):
             lx = self.extractor_cls(allow_domains=('google.com', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
                 Link(url='http://www.google.com/something', text=u''),

From e5f26078fa6c49c37aa0523f98b39977989454f1 Mon Sep 17 00:00:00 2001
From: Olaf Dietsche <olafdietsche@users.noreply.github.com>
Date: Sat, 11 Jul 2015 13:53:21 +0200
Subject: [PATCH 0528/2974] Add test: submit form with button control

---
 tests/test_http_request.py | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 02111f075..b87045325 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -733,6 +733,40 @@ class FormRequestTest(RequestTest):
         self.assertRaises(ValueError, self.request_class.from_response,
                           response, formxpath="//form/input[@name='abc']")
 
+    def test_from_response_button_submit(self):
+        response = _buildresponse(
+            """<form action="post.php" method="POST">
+            <input type="hidden" name="test1" value="val1">
+            <input type="hidden" name="test2" value="val2">
+            <button type="submit" name="button1" value="submit1">Submit</button>
+            </form>""",
+            url="http://www.example.com/this/list.html")
+        req = self.request_class.from_response(response)
+        self.assertEqual(req.method, 'POST')
+        self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.url, "http://www.example.com/this/post.php")
+        fs = _qs(req)
+        self.assertEqual(fs[b'test1'], [b'val1'])
+        self.assertEqual(fs[b'test2'], [b'val2'])
+        self.assertEqual(fs[b'button1'], [b'submit1'])
+
+    def test_from_response_button_notype(self):
+        response = _buildresponse(
+            """<form action="post.php" method="POST">
+            <input type="hidden" name="test1" value="val1">
+            <input type="hidden" name="test2" value="val2">
+            <button name="button1" value="submit1">Submit</button>
+            </form>""",
+            url="http://www.example.com/this/list.html")
+        req = self.request_class.from_response(response)
+        self.assertEqual(req.method, 'POST')
+        self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.url, "http://www.example.com/this/post.php")
+        fs = _qs(req)
+        self.assertEqual(fs[b'test1'], [b'val1'])
+        self.assertEqual(fs[b'test2'], [b'val2'])
+        self.assertEqual(fs[b'button1'], [b'submit1'])
+
 def _buildresponse(body, **kwargs):
     kwargs.setdefault('body', body)
     kwargs.setdefault('url', 'http://example.com')

From 121d7535beb327d1d63a92ae39b9974d8eec7bb3 Mon Sep 17 00:00:00 2001
From: Olaf Dietsche <olafdietsche@users.noreply.github.com>
Date: Sat, 11 Jul 2015 16:11:31 +0200
Subject: [PATCH 0529/2974] Allow button cotrols to submit form

---
 scrapy/http/request/form.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 0b1d3b926..1920cefe1 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -150,14 +150,16 @@ def _get_clickable(clickdata, form):
     if the latter is given. If not, it returns the first
     clickable element found
     """
-    clickables = [el for el in form.xpath('.//input[@type="submit"]')]
+    clickables = [el for el in form.xpath('descendant::input[@type="submit"]'
+                                          '|descendant::button[@type="submit"]'
+                                          '|descendant::button[not(@type)]')]
     if not clickables:
         return
 
     # If we don't have clickdata, we just use the first clickable element
     if clickdata is None:
         el = clickables[0]
-        return (el.name, el.value)
+        return (el.get('name'), el.get('value'))
 
     # If clickdata is given, we compare it to the clickable elements to find a
     # match. We first look to see if the number is specified in clickdata,
@@ -169,7 +171,7 @@ def _get_clickable(clickdata, form):
         except IndexError:
             pass
         else:
-            return (el.name, el.value)
+            return (el.get('name'), el.get('value'))
 
     # We didn't find it, so now we build an XPath expression out of the other
     # arguments, because they can be used as such
@@ -177,7 +179,7 @@ def _get_clickable(clickdata, form):
             u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))
     el = form.xpath(xpath)
     if len(el) == 1:
-        return (el[0].name, el[0].value)
+        return (el[0].get('name'), el[0].get('value'))
     elif len(el) > 1:
         raise ValueError("Multiple elements found (%r) matching the criteria "
                          "in clickdata: %r" % (el, clickdata))

From 45101829a52d9681b59fc401d659c0ee34792e2f Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 1 Sep 2015 03:49:52 +0500
Subject: [PATCH 0530/2974] PY3 fix AjacCrawlable middleware tests

---
 tests/py3-ignores.txt                            | 1 -
 tests/test_downloadermiddleware_ajaxcrawlable.py | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 759eeffff..7d077adcb 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -6,7 +6,6 @@ tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
 tests/test_downloader_handlers.py
-tests/test_downloadermiddleware_ajaxcrawlable.py
 tests/test_downloadermiddleware_defaultheaders.py
 tests/test_downloadermiddleware_downloadtimeout.py
 tests/test_downloadermiddleware_httpauth.py
diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py
index 67c57778d..493691ea4 100644
--- a/tests/test_downloadermiddleware_ajaxcrawlable.py
+++ b/tests/test_downloadermiddleware_ajaxcrawlable.py
@@ -14,7 +14,7 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase):
         self.mw = AjaxCrawlMiddleware.from_crawler(crawler)
 
     def _ajaxcrawlable_body(self):
-        return '<html><head><meta name="fragment" content="!"/></head><body></body></html>'
+        return b'<html><head><meta name="fragment" content="!"/></head><body></body></html>'
 
     def _req_resp(self, url, req_kwargs=None, resp_kwargs=None):
         req = Request(url, **(req_kwargs or {}))
@@ -53,6 +53,6 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase):
         assert resp3 is resp2
 
     def test_noncrawlable_body(self):
-        req, resp = self._req_resp('http://example.com/', {}, {'body': '<html></html>'})
+        req, resp = self._req_resp('http://example.com/', {}, {'body': b'<html></html>'})
         resp2 = self.mw.process_response(req, resp, self.spider)
         self.assertIs(resp, resp2)

From 3cf1911a9237c8cf23f6bbb7cba35335a75623e2 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 1 Sep 2015 03:57:05 +0500
Subject: [PATCH 0531/2974] PY3 fix DefaultHeadersMiddleware tests

---
 tests/py3-ignores.txt                             |  1 -
 tests/test_downloadermiddleware_defaultheaders.py | 13 ++++++++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 7d077adcb..145e98cd7 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -6,7 +6,6 @@ tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
 tests/test_downloader_handlers.py
-tests/test_downloadermiddleware_defaultheaders.py
 tests/test_downloadermiddleware_downloadtimeout.py
 tests/test_downloadermiddleware_httpauth.py
 tests/test_downloadermiddleware_httpcache.py
diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py
index 75d8a1921..80efa83f9 100644
--- a/tests/test_downloadermiddleware_defaultheaders.py
+++ b/tests/test_downloadermiddleware_defaultheaders.py
@@ -1,10 +1,10 @@
 from unittest import TestCase
-import six
 
 from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
 from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
+from scrapy.utils.python import to_bytes
 
 
 class TestDefaultHeadersMiddleware(TestCase):
@@ -12,8 +12,10 @@ class TestDefaultHeadersMiddleware(TestCase):
     def get_defaults_spider_mw(self):
         crawler = get_crawler(Spider)
         spider = crawler._create_spider('foo')
-        defaults = dict([(k, [v]) for k, v in \
-            six.iteritems(crawler.settings.get('DEFAULT_REQUEST_HEADERS'))])
+        defaults = {
+            to_bytes(k): [to_bytes(v)]
+            for k, v in crawler.settings.get('DEFAULT_REQUEST_HEADERS').items()
+        }
         return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler)
 
     def test_process_request(self):
@@ -25,9 +27,10 @@ class TestDefaultHeadersMiddleware(TestCase):
     def test_update_headers(self):
         defaults, spider, mw = self.get_defaults_spider_mw()
         headers = {'Accept-Language': ['es'], 'Test-Header': ['test']}
+        bytes_headers = {b'Accept-Language': [b'es'], b'Test-Header': [b'test']}
         req = Request('http://www.scrapytest.org', headers=headers)
-        self.assertEquals(req.headers, headers)
+        self.assertEquals(req.headers, bytes_headers)
 
         mw.process_request(req, spider)
-        defaults.update(headers)
+        defaults.update(bytes_headers)
         self.assertEquals(req.headers, defaults)

From 179a4409fba0e38df13899300dca41d53d84ad92 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 1 Sep 2015 03:58:25 +0500
Subject: [PATCH 0532/2974] PY3 enable DownloadTimeoutMiddleware tests

---
 tests/py3-ignores.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 145e98cd7..4fedc48e6 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -6,7 +6,6 @@ tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
 tests/test_downloader_handlers.py
-tests/test_downloadermiddleware_downloadtimeout.py
 tests/test_downloadermiddleware_httpauth.py
 tests/test_downloadermiddleware_httpcache.py
 tests/test_downloadermiddleware_httpcompression.py

From 78a4cd0f1c32477d42f14e4c48daedb24be44ebf Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 1 Sep 2015 04:00:26 +0500
Subject: [PATCH 0533/2974] PY3 fix HttpAuthMiddleware tests

---
 scrapy/downloadermiddlewares/httpauth.py    |  4 ++--
 tests/py3-ignores.txt                       |  1 -
 tests/test_downloadermiddleware_httpauth.py | 13 ++++++-------
 3 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py
index 7854e899a..7aa7a62bc 100644
--- a/scrapy/downloadermiddlewares/httpauth.py
+++ b/scrapy/downloadermiddlewares/httpauth.py
@@ -27,5 +27,5 @@ class HttpAuthMiddleware(object):
 
     def process_request(self, request, spider):
         auth = getattr(self, 'auth', None)
-        if auth and 'Authorization' not in request.headers:
-            request.headers['Authorization'] = auth
+        if auth and b'Authorization' not in request.headers:
+            request.headers[b'Authorization'] = auth
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 4fedc48e6..793f0c6cd 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -6,7 +6,6 @@ tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
 tests/test_downloader_handlers.py
-tests/test_downloadermiddleware_httpauth.py
 tests/test_downloadermiddleware_httpcache.py
 tests/test_downloadermiddleware_httpcompression.py
 tests/test_downloadermiddleware_httpproxy.py
diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py
index c30fa97c6..425a5cc79 100644
--- a/tests/test_downloadermiddleware_httpauth.py
+++ b/tests/test_downloadermiddleware_httpauth.py
@@ -4,10 +4,12 @@ from scrapy.http import Request
 from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
 from scrapy.spiders import Spider
 
+
 class TestSpider(Spider):
     http_user = 'foo'
     http_pass = 'bar'
 
+
 class HttpAuthMiddlewareTest(unittest.TestCase):
 
     def setUp(self):
@@ -21,13 +23,10 @@ class HttpAuthMiddlewareTest(unittest.TestCase):
     def test_auth(self):
         req = Request('http://scrapytest.org/')
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEquals(req.headers['Authorization'], 'Basic Zm9vOmJhcg==')
+        self.assertEquals(req.headers['Authorization'], b'Basic Zm9vOmJhcg==')
 
     def test_auth_already_set(self):
-        req = Request('http://scrapytest.org/', headers=dict(Authorization='Digest 123'))
+        req = Request('http://scrapytest.org/',
+                      headers=dict(Authorization='Digest 123'))
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEquals(req.headers['Authorization'], 'Digest 123')
-
-
-if __name__ == '__main__':
-    unittest.main()
+        self.assertEquals(req.headers['Authorization'], b'Digest 123')

From 3a9c73bc5d95fa59543522d0e997ce339c3a6c99 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 1 Sep 2015 04:09:15 +0500
Subject: [PATCH 0534/2974] PY3 fix DownloaderStats middleware tests

---
 tests/py3-ignores.txt                    |  1 -
 tests/test_downloadermiddleware_stats.py | 27 +++++++++++++++++-------
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 793f0c6cd..d405950f9 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -12,7 +12,6 @@ tests/test_downloadermiddleware_httpproxy.py
 tests/test_downloadermiddleware.py
 tests/test_downloadermiddleware_redirect.py
 tests/test_downloadermiddleware_retry.py
-tests/test_downloadermiddleware_stats.py
 tests/test_downloadermiddleware_useragent.py
 tests/test_engine.py
 tests/test_mail.py
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index fb46ccff6..1f2616e35 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -6,6 +6,10 @@ from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
 
+class MyException(Exception):
+    pass
+
+
 class TestDownloaderStats(TestCase):
 
     def setUp(self):
@@ -18,21 +22,28 @@ class TestDownloaderStats(TestCase):
         self.req = Request('http://scrapytest.org')
         self.res = Response('scrapytest.org', status=400)
 
+    def assertStatsEqual(self, key, value):
+        self.assertEqual(
+            self.crawler.stats.get_value(key, spider=self.spider),
+            value,
+            str(self.crawler.stats.get_stats(self.spider))
+        )
+
     def test_process_request(self):
         self.mw.process_request(self.req, self.spider)
-        self.assertEqual(self.crawler.stats.get_value('downloader/request_count', \
-            spider=self.spider), 1)
+        self.assertStatsEqual('downloader/request_count', 1)
 
     def test_process_response(self):
         self.mw.process_response(self.req, self.res, self.spider)
-        self.assertEqual(self.crawler.stats.get_value('downloader/response_count', \
-            spider=self.spider), 1)
+        self.assertStatsEqual('downloader/response_count', 1)
 
     def test_process_exception(self):
-        self.mw.process_exception(self.req, Exception(), self.spider)
-        self.assertEqual(self.crawler.stats.get_value('downloader/exception_count', \
-            spider=self.spider), 1)
+        self.mw.process_exception(self.req, MyException(), self.spider)
+        self.assertStatsEqual('downloader/exception_count', 1)
+        self.assertStatsEqual(
+            'downloader/exception_type_count/tests.test_downloadermiddleware_stats.MyException',
+            1
+        )
 
     def tearDown(self):
         self.crawler.stats.close_spider(self.spider, '')
-

From c44cafe4f58d6c5e5a3a8142f81ee0201fb8268a Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 1 Sep 2015 04:11:33 +0500
Subject: [PATCH 0535/2974] PY3 fix UserAgentMiddleware tests

---
 scrapy/downloadermiddlewares/useragent.py    | 2 +-
 tests/py3-ignores.txt                        | 1 -
 tests/test_downloadermiddleware_useragent.py | 9 +++++----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py
index 17793cad7..d24750c69 100644
--- a/scrapy/downloadermiddlewares/useragent.py
+++ b/scrapy/downloadermiddlewares/useragent.py
@@ -20,4 +20,4 @@ class UserAgentMiddleware(object):
 
     def process_request(self, request, spider):
         if self.user_agent:
-            request.headers.setdefault('User-Agent', self.user_agent)
+            request.headers.setdefault(b'User-Agent', self.user_agent)
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index d405950f9..95e2181e3 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -12,7 +12,6 @@ tests/test_downloadermiddleware_httpproxy.py
 tests/test_downloadermiddleware.py
 tests/test_downloadermiddleware_redirect.py
 tests/test_downloadermiddleware_retry.py
-tests/test_downloadermiddleware_useragent.py
 tests/test_engine.py
 tests/test_mail.py
 tests/test_pipeline_files.py
diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py
index 741c8de76..1e41fdace 100644
--- a/tests/test_downloadermiddleware_useragent.py
+++ b/tests/test_downloadermiddleware_useragent.py
@@ -17,7 +17,7 @@ class UserAgentMiddlewareTest(TestCase):
         spider, mw = self.get_spider_and_mw('default_useragent')
         req = Request('http://scrapytest.org/')
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.headers['User-Agent'], 'default_useragent')
+        self.assertEquals(req.headers['User-Agent'], b'default_useragent')
 
     def test_remove_agent(self):
         # settings UESR_AGENT to None should remove the user agent
@@ -34,15 +34,16 @@ class UserAgentMiddlewareTest(TestCase):
         mw.spider_opened(spider)
         req = Request('http://scrapytest.org/')
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.headers['User-Agent'], 'spider_useragent')
+        self.assertEquals(req.headers['User-Agent'], b'spider_useragent')
 
     def test_header_agent(self):
         spider, mw = self.get_spider_and_mw('default_useragent')
         spider.user_agent = 'spider_useragent'
         mw.spider_opened(spider)
-        req = Request('http://scrapytest.org/', headers={'User-Agent': 'header_useragent'})
+        req = Request('http://scrapytest.org/',
+                      headers={'User-Agent': 'header_useragent'})
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.headers['User-Agent'], 'header_useragent')
+        self.assertEquals(req.headers['User-Agent'], b'header_useragent')
 
     def test_no_agent(self):
         spider, mw = self.get_spider_and_mw(None)

From 2c28b53cc08bae41c0a00ffeae3c8d277a1d3b84 Mon Sep 17 00:00:00 2001
From: Olaf Dietsche <olafdietsche@users.noreply.github.com>
Date: Sun, 12 Jul 2015 13:28:06 +0200
Subject: [PATCH 0536/2974] Add tests with incomplete buttons

---
 tests/test_http_request.py | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index b87045325..ff0941961 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -767,6 +767,40 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs[b'test2'], [b'val2'])
         self.assertEqual(fs[b'button1'], [b'submit1'])
 
+    def test_from_response_submit_novalue(self):
+        response = _buildresponse(
+            """<form action="post.php" method="POST">
+            <input type="hidden" name="test1" value="val1">
+            <input type="hidden" name="test2" value="val2">
+            <input type="submit" name="button1">Submit</button>
+            </form>""",
+            url="http://www.example.com/this/list.html")
+        req = self.request_class.from_response(response)
+        self.assertEqual(req.method, 'POST')
+        self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.url, "http://www.example.com/this/post.php")
+        fs = _qs(req)
+        self.assertEqual(fs[b'test1'], [b'val1'])
+        self.assertEqual(fs[b'test2'], [b'val2'])
+        self.assertEqual(fs[b'button1'], [b''])
+
+    def test_from_response_button_novalue(self):
+        response = _buildresponse(
+            """<form action="post.php" method="POST">
+            <input type="hidden" name="test1" value="val1">
+            <input type="hidden" name="test2" value="val2">
+            <button type="submit" name="button1">Submit</button>
+            </form>""",
+            url="http://www.example.com/this/list.html")
+        req = self.request_class.from_response(response)
+        self.assertEqual(req.method, 'POST')
+        self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.url, "http://www.example.com/this/post.php")
+        fs = _qs(req)
+        self.assertEqual(fs[b'test1'], [b'val1'])
+        self.assertEqual(fs[b'test2'], [b'val2'])
+        self.assertEqual(fs[b'button1'], [b''])
+
 def _buildresponse(body, **kwargs):
     kwargs.setdefault('body', body)
     kwargs.setdefault('url', 'http://example.com')

From 937277e8598bd3f62642d4d8267cec2defcead7c Mon Sep 17 00:00:00 2001
From: Olaf Dietsche <olafdietsche@users.noreply.github.com>
Date: Sun, 12 Jul 2015 13:37:28 +0200
Subject: [PATCH 0537/2974] Make sure value attribute values are set.

---
 scrapy/http/request/form.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 1920cefe1..a12a2fd07 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -159,7 +159,7 @@ def _get_clickable(clickdata, form):
     # If we don't have clickdata, we just use the first clickable element
     if clickdata is None:
         el = clickables[0]
-        return (el.get('name'), el.get('value'))
+        return (el.get('name'), el.get('value') or '')
 
     # If clickdata is given, we compare it to the clickable elements to find a
     # match. We first look to see if the number is specified in clickdata,
@@ -171,7 +171,7 @@ def _get_clickable(clickdata, form):
         except IndexError:
             pass
         else:
-            return (el.get('name'), el.get('value'))
+            return (el.get('name'), el.get('value') or '')
 
     # We didn't find it, so now we build an XPath expression out of the other
     # arguments, because they can be used as such
@@ -179,7 +179,7 @@ def _get_clickable(clickdata, form):
             u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))
     el = form.xpath(xpath)
     if len(el) == 1:
-        return (el[0].get('name'), el[0].get('value'))
+        return (el[0].get('name'), el[0].get('value') or '')
     elif len(el) > 1:
         raise ValueError("Multiple elements found (%r) matching the criteria "
                          "in clickdata: %r" % (el, clickdata))

From ca83a0b02880aad8c34f48ac81c7005880f5140e Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Tue, 1 Sep 2015 13:22:43 -0300
Subject: [PATCH 0538/2974] Support for returning deferreds in downloader
 middleware methods.

---
 docs/topics/downloader-middleware.rst |  2 ++
 scrapy/core/downloader/middleware.py  | 29 ++++++++++++++++++---------
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 73cc67423..4603c555b 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -58,6 +58,8 @@ more of the following methods:
 
 .. class:: DownloaderMiddleware
 
+   .. note::  Any of the downloader middleware methods may also return a deferred.
+
    .. method:: process_request(request, spider)
 
       This method is called for each request that goes through the download
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 413a05dd1..9cd30c144 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -4,11 +4,15 @@ Downloader Middleware manager
 See documentation in docs/topics/downloader-middleware.rst
 """
 import six
+
+from twisted.internet import defer
+
 from scrapy.http import Request, Response
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.conf import build_component_list
 
+
 class DownloaderMiddlewareManager(MiddlewareManager):
 
     component_name = 'downloader middleware'
@@ -27,40 +31,45 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             self.methods['process_exception'].insert(0, mw.process_exception)
 
     def download(self, download_func, request, spider):
+        @defer.inlineCallbacks
         def process_request(request):
             for method in self.methods['process_request']:
-                response = method(request=request, spider=spider)
+                response = yield method(request=request, spider=spider)
                 assert response is None or isinstance(response, (Response, Request)), \
                         'Middleware %s.process_request must return None, Response or Request, got %s' % \
                         (six.get_method_self(method).__class__.__name__, response.__class__.__name__)
                 if response:
-                    return response
-            return download_func(request=request, spider=spider)
+                    defer.returnValue(response)
+            defer.returnValue((yield download_func(request=request,spider=spider)))
 
+        @defer.inlineCallbacks
         def process_response(response):
             assert response is not None, 'Received None in process_response'
             if isinstance(response, Request):
-                return response
+                defer.returnValue(response)
 
             for method in self.methods['process_response']:
-                response = method(request=request, response=response, spider=spider)
+                response = yield method(request=request, response=response,
+                                        spider=spider)
                 assert isinstance(response, (Response, Request)), \
                     'Middleware %s.process_response must return Response or Request, got %s' % \
                     (six.get_method_self(method).__class__.__name__, type(response))
                 if isinstance(response, Request):
-                    return response
-            return response
+                    defer.returnValue(response)
+            defer.returnValue(response)
 
+        @defer.inlineCallbacks
         def process_exception(_failure):
             exception = _failure.value
             for method in self.methods['process_exception']:
-                response = method(request=request, exception=exception, spider=spider)
+                response = yield method(request=request, exception=exception,
+                                        spider=spider)
                 assert response is None or isinstance(response, (Response, Request)), \
                     'Middleware %s.process_exception must return None, Response or Request, got %s' % \
                     (six.get_method_self(method).__class__.__name__, type(response))
                 if response:
-                    return response
-            return _failure
+                    defer.returnValue(response)
+            defer.returnValue(_failure)
 
         deferred = mustbe_deferred(process_request, request)
         deferred.addErrback(process_exception)

From 9ce9a293a6e0eefcb43f61d38d5f7ccc655d7889 Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Tue, 1 Sep 2015 15:24:55 -0300
Subject: [PATCH 0539/2974] Always check robots.txt before making another
 request in RobotsTxtMiddleware.

---
 docs/topics/downloader-middleware.rst        |  6 --
 scrapy/downloadermiddlewares/robotstxt.py    | 39 ++++++++---
 tests/test_downloadermiddleware_robotstxt.py | 72 +++++++-------------
 3 files changed, 57 insertions(+), 60 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 4603c555b..38c9456db 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -879,12 +879,6 @@ RobotsTxtMiddleware
     To make sure Scrapy respects robots.txt make sure the middleware is enabled
     and the :setting:`ROBOTSTXT_OBEY` setting is enabled.
 
-    .. warning:: Keep in mind that, if you crawl using multiple concurrent
-       requests per domain, Scrapy could still download some forbidden pages
-       if they were requested before the robots.txt file was downloaded. This
-       is a known limitation of the current robots.txt middleware and will
-       be fixed in the future.
-
 .. reqmeta:: dont_obey_robotstxt
 
 If :attr:`Request.meta <scrapy.http.Request.meta>` has
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 457620d85..c061c2407 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -8,6 +8,7 @@ import logging
 
 from six.moves.urllib import robotparser
 
+from twisted.internet.defer import Deferred, maybeDeferred
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
@@ -34,17 +35,22 @@ class RobotsTxtMiddleware(object):
     def process_request(self, request, spider):
         if request.meta.get('dont_obey_robotstxt'):
             return
-        rp = self.robot_parser(request, spider)
-        if rp and not rp.can_fetch(self._useragent, request.url):
+        d = maybeDeferred(self.robot_parser, request, spider)
+        d.addCallback(self.process_request_2, request, spider)
+        return d
+
+    def process_request_2(self, rp, request, spider):
+        if rp is not None and not rp.can_fetch(self._useragent, request.url):
             logger.debug("Forbidden by robots.txt: %(request)s",
                          {'request': request}, extra={'spider': spider})
-            raise IgnoreRequest
+            raise IgnoreRequest()
 
     def robot_parser(self, request, spider):
         url = urlparse_cached(request)
         netloc = url.netloc
+
         if netloc not in self._parsers:
-            self._parsers[netloc] = None
+            self._parsers[netloc] = Deferred()
             robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc)
             robotsreq = Request(
                 robotsurl,
@@ -52,9 +58,19 @@ class RobotsTxtMiddleware(object):
                 meta={'dont_obey_robotstxt': True}
             )
             dfd = self.crawler.engine.download(robotsreq, spider)
-            dfd.addCallback(self._parse_robots)
+            dfd.addCallback(self._parse_robots, netloc)
             dfd.addErrback(self._logerror, robotsreq, spider)
-        return self._parsers[netloc]
+            dfd.addErrback(self._robots_error, netloc)
+
+        if isinstance(self._parsers[netloc], Deferred):
+            d = Deferred()
+            def cb(result):
+                d.callback(result)
+                return result
+            self._parsers[netloc].addCallback(cb)
+            return d
+        else:
+            return self._parsers[netloc]
 
     def _logerror(self, failure, request, spider):
         if failure.type is not IgnoreRequest:
@@ -62,8 +78,9 @@ class RobotsTxtMiddleware(object):
                          {'request': request, 'f_exception': failure.value},
                          exc_info=failure_to_exc_info(failure),
                          extra={'spider': spider})
+        return failure
 
-    def _parse_robots(self, response):
+    def _parse_robots(self, response, netloc):
         rp = robotparser.RobotFileParser(response.url)
         body = ''
         if hasattr(response, 'body_as_unicode'):
@@ -78,4 +95,10 @@ class RobotsTxtMiddleware(object):
                 # 'disallow all' to 'allow any'.
                 pass
         rp.parse(body.splitlines())
-        self._parsers[urlparse_cached(response).netloc] = rp
+
+        rp_dfd = self._parsers[netloc]
+        self._parsers[netloc] = rp
+        rp_dfd.callback(rp)
+
+    def _robots_error(self, failure, netloc):
+        self._parsers.pop(netloc).callback(None)
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index b9c002f85..8a7238dd1 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 import re
 from twisted.internet import reactor, error
-from twisted.internet.defer import Deferred
+from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
 from twisted.python import failure
 from twisted.trial import unittest
 from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
@@ -44,32 +44,20 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
 
     def test_robotstxt(self):
         middleware = RobotsTxtMiddleware(self._get_successful_crawler())
-        # There is a bit of neglect in robotstxt.py: robots.txt is fetched asynchronously,
-        # and it is actually fetched only *after* first process_request completes.
-        # So, first process_request will always succeed.
-        # We defer test() because otherwise robots.txt download mock will be called after assertRaises failure.
-        self.assertNotIgnored(Request('http://site.local'), middleware)
-        def test(r):
-            self.assertNotIgnored(Request('http://site.local/allowed'), middleware)
-            self.assertIgnored(Request('http://site.local/admin/main'), middleware)
+        return DeferredList([
+            self.assertNotIgnored(Request('http://site.local/allowed'), middleware),
+            self.assertIgnored(Request('http://site.local/admin/main'), middleware),
             self.assertIgnored(Request('http://site.local/static/'), middleware)
-        deferred = Deferred()
-        deferred.addCallback(test)
-        reactor.callFromThread(deferred.callback, None)
-        return deferred
+        ], fireOnOneErrback=True)
 
     def test_robotstxt_meta(self):
         middleware = RobotsTxtMiddleware(self._get_successful_crawler())
         meta = {'dont_obey_robotstxt': True}
-        self.assertNotIgnored(Request('http://site.local', meta=meta), middleware)
-        def test(r):
-            self.assertNotIgnored(Request('http://site.local/allowed', meta=meta), middleware)
-            self.assertNotIgnored(Request('http://site.local/admin/main', meta=meta), middleware)
+        return DeferredList([
+            self.assertNotIgnored(Request('http://site.local/allowed', meta=meta), middleware),
+            self.assertNotIgnored(Request('http://site.local/admin/main', meta=meta), middleware),
             self.assertNotIgnored(Request('http://site.local/static/', meta=meta), middleware)
-        deferred = Deferred()
-        deferred.addCallback(test)
-        reactor.callFromThread(deferred.callback, None)
-        return deferred
+        ], fireOnOneErrback=True)
 
     def _get_garbage_crawler(self):
         crawler = self.crawler
@@ -85,17 +73,12 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
     def test_robotstxt_garbage(self):
         # garbage response should be discarded, equal 'allow all'
         middleware = RobotsTxtMiddleware(self._get_garbage_crawler())
-        middleware._logerror = mock.MagicMock()
-        middleware.process_request(Request('http://site.local'), None)
-        self.assertNotIgnored(Request('http://site.local'), middleware)
-        def test(r):
-            self.assertNotIgnored(Request('http://site.local/allowed'), middleware)
-            self.assertNotIgnored(Request('http://site.local/admin/main'), middleware)
+        deferred = DeferredList([
+            self.assertNotIgnored(Request('http://site.local'), middleware),
+            self.assertNotIgnored(Request('http://site.local/allowed'), middleware),
+            self.assertNotIgnored(Request('http://site.local/admin/main'), middleware),
             self.assertNotIgnored(Request('http://site.local/static/'), middleware)
-        deferred = Deferred()
-        deferred.addCallback(test)
-        deferred.addErrback(lambda _: self.assertIsNone(middleware._logerror.assert_any_call()))
-        reactor.callFromThread(deferred.callback, None)
+        ], fireOnOneErrback=True)
         return deferred
 
     def _get_emptybody_crawler(self):
@@ -112,15 +95,11 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
     def test_robotstxt_empty_response(self):
         # empty response should equal 'allow all'
         middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
-        self.assertNotIgnored(Request('http://site.local'), middleware)
-        def test(r):
-            self.assertNotIgnored(Request('http://site.local/allowed'), middleware)
-            self.assertNotIgnored(Request('http://site.local/admin/main'), middleware)
+        return DeferredList([
+            self.assertNotIgnored(Request('http://site.local/allowed'), middleware),
+            self.assertNotIgnored(Request('http://site.local/admin/main'), middleware),
             self.assertNotIgnored(Request('http://site.local/static/'), middleware)
-        deferred = Deferred()
-        deferred.addCallback(test)
-        reactor.callFromThread(deferred.callback, None)
-        return deferred
+        ], fireOnOneErrback=True)
 
     def test_robotstxt_error(self):
         self.crawler.settings.set('ROBOTSTXT_OBEY', True)
@@ -132,17 +111,18 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         self.crawler.engine.download.side_effect = return_failure
 
         middleware = RobotsTxtMiddleware(self.crawler)
-        middleware._logerror = mock.MagicMock()
-        middleware.process_request(Request('http://site.local'), None)
-        deferred = Deferred()
-        deferred.addErrback(lambda _: self.assertIsNone(middleware._logerror.assert_any_call()))
-        reactor.callFromThread(deferred.callback, None)
+        middleware._logerror = mock.MagicMock(side_effect=lambda fail, req, spider: fail)
+        deferred = middleware.process_request(Request('http://site.local'), None)
+        deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called))
         return deferred
 
     def assertNotIgnored(self, request, middleware):
         spider = None  # not actually used
-        self.assertIsNone(middleware.process_request(request, spider))
+        dfd = maybeDeferred(middleware.process_request, request, spider)
+        dfd.addCallback(self.assertIsNone)
+        return dfd
 
     def assertIgnored(self, request, middleware):
         spider = None  # not actually used
-        self.assertRaises(IgnoreRequest, middleware.process_request, request, spider)
+        return self.assertFailure(maybeDeferred(middleware.process_request, request, spider),
+                                  IgnoreRequest)

From a6a629e707c5bdaa0601fad6edf9074fe3b2533b Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Wed, 2 Sep 2015 01:39:04 -0300
Subject: [PATCH 0540/2974] Call actual error logger in robots.txt middleware
 tests.

---
 tests/test_downloadermiddleware_robotstxt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 8a7238dd1..e0e8ade47 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -111,7 +111,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         self.crawler.engine.download.side_effect = return_failure
 
         middleware = RobotsTxtMiddleware(self.crawler)
-        middleware._logerror = mock.MagicMock(side_effect=lambda fail, req, spider: fail)
+        middleware._logerror = mock.MagicMock(side_effect=middleware._logerror)
         deferred = middleware.process_request(Request('http://site.local'), None)
         deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called))
         return deferred

From 668e5fd25706171fe227f2f161aa331435560849 Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Wed, 2 Sep 2015 01:43:22 -0300
Subject: [PATCH 0541/2974] Test for robots.txt middleware for processing a
 request for which the robots.txt parser is ready.

---
 tests/test_downloadermiddleware_robotstxt.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index e0e8ade47..0c03f96a9 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -50,6 +50,12 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
             self.assertIgnored(Request('http://site.local/static/'), middleware)
         ], fireOnOneErrback=True)
 
+    def test_robotstxt_ready_parser(self):
+        middleware = RobotsTxtMiddleware(self._get_successful_crawler())
+        d = self.assertNotIgnored(Request('http://site.local/allowed'), middleware)
+        d.addCallback(lambda _: self.assertNotIgnored(Request('http://site.local/allowed'), middleware))
+        return d
+
     def test_robotstxt_meta(self):
         middleware = RobotsTxtMiddleware(self._get_successful_crawler())
         meta = {'dont_obey_robotstxt': True}

From 1f4af9d81eeb1c54288f647b6dfb6de8bccffa3f Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Wed, 2 Sep 2015 10:33:53 -0300
Subject: [PATCH 0542/2974] Test for robots.txt middleware for logger not being
 called when the request for robots.txt is ignored.

---
 tests/test_downloadermiddleware_robotstxt.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 0c03f96a9..5f45dcb82 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -4,7 +4,8 @@ from twisted.internet import reactor, error
 from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
 from twisted.python import failure
 from twisted.trial import unittest
-from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
+from scrapy.downloadermiddlewares.robotstxt import (RobotsTxtMiddleware,
+                                                    logger as mw_module_logger)
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response, TextResponse
 from scrapy.settings import Settings
@@ -122,6 +123,21 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called))
         return deferred
 
+    def test_ignore_robotstxt_request(self):
+        self.crawler.settings.set('ROBOTSTXT_OBEY', True)
+        def ignore_request(request, spider):
+            deferred = Deferred()
+            reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest()))
+            return deferred
+        self.crawler.engine.download.side_effect = ignore_request
+
+        middleware = RobotsTxtMiddleware(self.crawler)
+        mw_module_logger.error = mock.MagicMock()
+
+        d = self.assertNotIgnored(Request('http://site.local/allowed'), middleware)
+        d.addCallback(lambda _: self.assertFalse(mw_module_logger.error.called))
+        return d
+
     def assertNotIgnored(self, request, middleware):
         spider = None  # not actually used
         dfd = maybeDeferred(middleware.process_request, request, spider)

From 2748b38592b7f66634b47a782ba571769de8b048 Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Wed, 2 Sep 2015 11:15:37 -0300
Subject: [PATCH 0543/2974] Test for not calling the download function when
 downloader middleware returns a response in process_request.

---
 tests/test_downloadermiddleware.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 257eab609..a5e765d9d 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -5,6 +5,7 @@ from scrapy.http import Request, Response
 from scrapy.spiders import Spider
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.utils.test import get_crawler
+from tests import mock
 
 
 class ManagerTestCase(TestCase):
@@ -90,3 +91,24 @@ class DefaultsTest(ManagerTestCase):
             'Location': 'http://example.com/login',
         })
         self.assertRaises(IOError, self._download, request=req, response=resp)
+
+
+class ResponseFromProcessRequestTest(ManagerTestCase):
+    """Tests middleware returning a response from process_request."""
+
+    def test_download_func_not_called(self):
+        class ResponseMiddleware(object):
+            def process_request(self, request, spider):
+                return Response(request.url)
+
+        self.mwman._add_middleware(ResponseMiddleware())
+
+        req = Request('http://example.com/index.html')
+        download_func = mock.MagicMock()
+        dfd = self.mwman.download(download_func, req, self.spider)
+        results = []
+        dfd.addBoth(results.append)
+        self._wait(dfd)
+
+        self.assertIsInstance(results[0], Response)
+        self.assertFalse(download_func.called)

From dd83f6123d54493661ae00ef668a5c2ee7d4e199 Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Wed, 2 Sep 2015 11:36:15 -0300
Subject: [PATCH 0544/2974] Test for same response object in downloader
 middleware test.

---
 tests/test_downloadermiddleware.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index a5e765d9d..13f35b92a 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -97,9 +97,11 @@ class ResponseFromProcessRequestTest(ManagerTestCase):
     """Tests middleware returning a response from process_request."""
 
     def test_download_func_not_called(self):
+        resp = Response('http://example.com/index.html')
+
         class ResponseMiddleware(object):
             def process_request(self, request, spider):
-                return Response(request.url)
+                return resp
 
         self.mwman._add_middleware(ResponseMiddleware())
 
@@ -110,5 +112,5 @@ class ResponseFromProcessRequestTest(ManagerTestCase):
         dfd.addBoth(results.append)
         self._wait(dfd)
 
-        self.assertIsInstance(results[0], Response)
+        self.assertIs(results[0], resp)
         self.assertFalse(download_func.called)

From 833efbfcbdeea43f73d59ce222a12384ba8fff82 Mon Sep 17 00:00:00 2001
From: Robert Weindl <robert.weindl@blackstack.net>
Date: Wed, 2 Sep 2015 17:26:14 +0200
Subject: [PATCH 0545/2974] Update tutorial.rst

Add missing "scrapy." prefix.
---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 45d1f9b29..dce165cf4 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -498,7 +498,7 @@ for it::
         next_page = response.css("ul.navigation > li.next-page > a::attr('href')")
         if next_page:
             url = response.urljoin(next_page[0].extract())
-            yield Request(url, self.parse_articles_follow_next_page)
+            yield scrapy.Request(url, self.parse_articles_follow_next_page)
 
 This creates a sort of loop, following all the links to the next page until it
 doesn't find one -- handy for crawling blogs, forums and other sites with

From d9fddabe959ae93e4f4a50c04f6306a4e79a2af8 Mon Sep 17 00:00:00 2001
From: Mikhail Lyundin <mlyundin@gmail.com>
Date: Wed, 2 Sep 2015 23:12:36 +0300
Subject: [PATCH 0546/2974] Avoid creation of temporary list object in iflatten

---
 scrapy/utils/python.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 94ee8a557..2dafb3710 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -38,7 +38,7 @@ def iflatten(x):
     Similar to ``.flatten()``, but returns iterator instead"""
     for el in x:
         if is_listlike(el):
-            for el_ in flatten(el):
+            for el_ in iflatten(el):
                 yield el_
         else:
             yield el

From d022d3cb9e05ce66d602e0ee0120f2c25f1533ae Mon Sep 17 00:00:00 2001
From: Mikhail Lyundin <mlyundin@gmail.com>
Date: Thu, 3 Sep 2015 22:49:56 +0300
Subject: [PATCH 0547/2974] equal_attributes function optimization

---
 scrapy/utils/python.py | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 2dafb3710..d566783b2 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -257,20 +257,14 @@ def equal_attributes(obj1, obj2, attributes):
     if not attributes:
         return False
 
+    temp1, temp2 = object(), object()
     for attr in attributes:
         # support callables like itemgetter
         if callable(attr):
-            if not attr(obj1) == attr(obj2):
-                return False
-        else:
-            # check that objects has attribute
-            if not hasattr(obj1, attr):
-                return False
-            if not hasattr(obj2, attr):
-                return False
-            # compare object attributes
-            if not getattr(obj1, attr) == getattr(obj2, attr):
+            if attr(obj1) != attr(obj2):
                 return False
+        elif getattr(obj1, attr, temp1) != getattr(obj2, attr, temp2):
+            return False
     # all attributes equal
     return True
 

From faf9265c91e35c2e14c4fb78731ec694cce63d56 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 4 Sep 2015 20:50:48 +0500
Subject: [PATCH 0548/2974] fixed compatibility with Twisted 15.4.0

---
 scrapy/utils/response.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 3d1af7e51..c4ad52f14 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -9,8 +9,7 @@ import webbrowser
 import tempfile
 
 from twisted.web import http
-from twisted.web.http import RESPONSES
-from scrapy.utils.python import to_bytes
+from scrapy.utils.python import to_bytes, to_native_str
 from w3lib import html
 
 from scrapy.utils.decorators import deprecated
@@ -55,7 +54,7 @@ def response_status_message(status):
     >>> response_status_message(404)
     '404 Not Found'
     """
-    return '%s %s' % (status, http.responses.get(int(status)))
+    return '%s %s' % (status, to_native_str(http.RESPONSES.get(int(status))))
 
 
 def response_httprepr(response):
@@ -64,7 +63,7 @@ def response_httprepr(response):
     that was received (that's not exposed by Twisted).
     """
     s = b"HTTP/1.1 " + to_bytes(str(response.status)) + b" " + \
-        to_bytes(RESPONSES.get(response.status, b'')) + b"\r\n"
+        to_bytes(http.RESPONSES.get(response.status, b'')) + b"\r\n"
     if response.headers:
         s += response.headers.to_string() + b"\r\n"
     s += b"\r\n"

From d30e539049190d869f4730d3d79cb16af7c7ac8a Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 4 Sep 2015 20:55:41 +0500
Subject: [PATCH 0549/2974] don't run tests twice on Travis if a PR is made
 from a scrapy/scrapy branch

---
 .travis.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index aee08919e..7b24e051e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,10 @@
 language: python
 python: 2.7
 sudo: false
+branches:
+  only:
+    - master
+    - /^\d\.\d+$/
 env:
  - TOXENV=py27
  - TOXENV=precise

From 1f95af3c07da64621a86de643ffd993423baacd3 Mon Sep 17 00:00:00 2001
From: Daniel Collins <dacjames@gmail.com>
Date: Sat, 29 Aug 2015 12:04:00 -0700
Subject: [PATCH 0550/2974] add support for a nested loaders

---
 scrapy/loader/__init__.py |  41 +++++++++++++--
 tests/test_loader.py      | 108 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 145 insertions(+), 4 deletions(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 84640f0b6..102ffc351 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -24,16 +24,46 @@ class ItemLoader(object):
     default_output_processor = Identity()
     default_selector_class = Selector
 
-    def __init__(self, item=None, selector=None, response=None, **context):
+    def __init__(self, item=None, selector=None, response=None, parent=None, **context):
         if selector is None and response is not None:
             selector = self.default_selector_class(response)
         self.selector = selector
         context.update(selector=selector, response=response)
         if item is None:
             item = self.default_item_class()
-        self.item = context['item'] = item
         self.context = context
-        self._values = defaultdict(list)
+        self.parent = parent
+        self._local_item = context['item'] = item
+        self._local_values = defaultdict(list)
+
+    @property
+    def _values(self):
+        if self.parent is not None:
+            return self.parent._values
+        else:
+            return self._local_values
+
+    @property
+    def item(self):
+        if self.parent is not None:
+            return self.parent.item
+        else:
+            return self._local_item
+
+    def nested_loader(self, xpath=None, css=None):
+        if xpath is not None and css is not None:
+            raise ValueError("Cannot nest a loader with both a xpath selector and a css selector")
+
+        if xpath is not None:
+            selector = self.selector.xpath(xpath)
+
+        if css is not None:
+            selector = self.selector.css(css)
+
+        subloader = self.__class__(
+            item=self.item, selector=selector, parent=self
+        )
+        return subloader
 
     def add_value(self, field_name, value, *processors, **kw):
         value = self.get_value(value, *processors, **kw)
@@ -84,6 +114,10 @@ class ItemLoader(object):
             value = self.get_output_value(field_name)
             if value is not None:
                 item[field_name] = value
+
+        # for loader in self._subloaders:
+        #     loader.load_item()
+
         return item
 
     def get_output_value(self, field_name):
@@ -168,5 +202,4 @@ class ItemLoader(object):
         csss = arg_to_iter(csss)
         return flatten([self.selector.css(css).extract() for css in csss])
 
-
 XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 8cf5e484a..2d34da810 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -19,11 +19,24 @@ class TestItem(NameItem):
     summary = Field()
 
 
+class TestNestedItem(Item):
+    name = Field()
+    name_div = Field()
+    name_value = Field()
+
+    url = Field()
+    image = Field()
+
+
 # test item loaders
 class NameItemLoader(ItemLoader):
     default_item_class = TestItem
 
 
+class NestedItemLoader(ItemLoader):
+    default_item_class = TestNestedItem
+
+
 class TestItemLoader(NameItemLoader):
     name_in = MapCompose(lambda v: v.title())
 
@@ -600,6 +613,101 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertEqual(l.get_output_value('url'), [u'scrapy.org'])
 
 
+class SubselectorLoaderTest(unittest.TestCase):
+    response = HtmlResponse(url="", encoding='utf-8', body=b"""
+    <html>
+    <body>
+    <header>
+      <div id="id">marta</div>
+      <p>paragraph</p>
+    </header>
+    <footer class="footer">
+      <a href="http://www.scrapy.org">homepage</a>
+      <img src="/images/logo.png" width="244" height="65" alt="Scrapy">
+    </footer>
+    </body>
+    </html>
+    """)
+
+    def test_nested_xpath(self):
+        l = NestedItemLoader(response=self.response)
+        nl = l.nested_loader(xpath="//header")
+        nl.add_xpath('name', 'div/text()')
+        nl.add_css('name_div', '#id')
+        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
+
+        self.assertEqual(l.get_output_value('name'), [u'marta'])
+        self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
+        self.assertEqual(l.get_output_value('name_value'),  [u'marta'])
+
+        self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
+        self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
+        self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value'))
+
+    def test_nested_css(self):
+        l = NestedItemLoader(response=self.response)
+        nl = l.nested_loader(css="header")
+        nl.add_xpath('name', 'div/text()')
+        nl.add_css('name_div', '#id')
+        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
+
+        self.assertEqual(l.get_output_value('name'), [u'marta'])
+        self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
+        self.assertEqual(l.get_output_value('name_value'),  [u'marta'])
+
+        self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
+        self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
+        self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value'))
+
+    def test_nested_replace(self):
+        l = NestedItemLoader(response=self.response)
+        nl1 = l.nested_loader(xpath='//footer')
+        nl2 = nl1.nested_loader(xpath='a')
+
+        l.add_xpath('url', '//footer/a/@href')
+        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        nl1.replace_xpath('url', 'img/@src')
+        self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
+        nl2.replace_xpath('url', '@href')
+        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+
+    def test_nested_ordering(self):
+        l = NestedItemLoader(response=self.response)
+        nl1 = l.nested_loader(xpath='//footer')
+        nl2 = nl1.nested_loader(xpath='a')
+
+        nl1.add_xpath('url', 'img/@src')
+        l.add_xpath('url', '//footer/a/@href')
+        nl2.add_xpath('url', 'text()')
+        l.add_xpath('url', '//footer/a/@href')
+
+        self.assertEqual(l.get_output_value('url'), [
+            u'/images/logo.png',
+            u'http://www.scrapy.org',
+            u'homepage',
+            u'http://www.scrapy.org',
+        ])
+
+    def test_nested_load_item(self):
+        l = NestedItemLoader(response=self.response)
+        nl1 = l.nested_loader(xpath='//footer')
+        nl2 = nl1.nested_loader(xpath='img')
+
+        l.add_xpath('name', '//header/div/text()')
+        nl1.add_xpath('url', 'a/@href')
+        nl2.add_xpath('image', '@src')
+
+        item = l.load_item()
+
+        assert item is l.item
+        assert item is nl1.item
+        assert item is nl2.item
+
+        self.assertEqual(item['name'], [u'marta'])
+        self.assertEqual(item['url'], [u'http://www.scrapy.org'])
+        self.assertEqual(item['image'], [u'/images/logo.png'])
+
+
 class SelectJmesTestCase(unittest.TestCase):
         test_list_equals = {
             'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),

From 425e35ee9023f5a973869d7e0d926297d9f75c35 Mon Sep 17 00:00:00 2001
From: Daniel Collins <dacjames@gmail.com>
Date: Sat, 29 Aug 2015 12:29:01 -0700
Subject: [PATCH 0551/2974] removed commented code and add test for error
 handling

---
 scrapy/loader/__init__.py | 3 ---
 tests/test_loader.py      | 5 +++++
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 102ffc351..f2c43ab4c 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -115,9 +115,6 @@ class ItemLoader(object):
             if value is not None:
                 item[field_name] = value
 
-        # for loader in self._subloaders:
-        #     loader.load_item()
-
         return item
 
     def get_output_value(self, field_name):
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 2d34da810..90273c2c8 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -707,6 +707,11 @@ class SubselectorLoaderTest(unittest.TestCase):
         self.assertEqual(item['url'], [u'http://www.scrapy.org'])
         self.assertEqual(item['image'], [u'/images/logo.png'])
 
+    def test_nested_bad_arguments(self):
+        l = NestedItemLoader(response=self.response)
+        with self.assertRaises(ValueError):
+            l.nested_loader(css="#id", xpath="//footer")
+
 
 class SelectJmesTestCase(unittest.TestCase):
         test_list_equals = {

From 88c92cb68b5a6f9b0c50c801f92dacc4fc0fa52a Mon Sep 17 00:00:00 2001
From: Daniel Collins <dacjames@gmail.com>
Date: Sat, 29 Aug 2015 14:23:25 -0700
Subject: [PATCH 0552/2974] provide documentation for nested loaders

---
 docs/topics/loaders.rst | 54 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 8841711ea..0c20f32ef 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -432,6 +432,14 @@ ItemLoader objects
         <topics-loaders-processors>` to get the final value to assign to each
         item field.
 
+    .. method:: nested_loader(xpath=selector, css=selector)
+
+        Create a nested loader with either an xpath selector or css selector.
+        The supplied selector is applied relative to selector associated
+        with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
+        with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
+        :meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
+
     .. method:: get_collected_values(field_name)
 
         Return the collected values for the given field.
@@ -490,6 +498,52 @@ ItemLoader objects
         :attr:`default_selector_class`. This attribute is meant to be
         read-only.
 
+.. _topics-loaders-nested:
+
+Nested Loaders
+==============
+
+When parsing related values from a subsection of a document, it can be
+useful to create nested loaders.  Imagine you're extracting details from
+a footer of a page that looks something like:
+
+Example::
+
+    <footer>
+        <a class="social" href="http://facebook.com/whatever">Like Us</a>
+        <a class="social" href="http://twitter.com/whatever">Follow Us</a>
+        <a class="email" href="mailto:whatever@example.com">Email Us</a>
+    </footer>
+
+Without nested loaders, you need to specify the full xpath (or css) for each value
+that you wish to extract.
+
+Example::
+
+    loader = ItemLoader(item=Item())
+    # load stuff not in the footer
+    loader.add_xpath('social', '//footer/a[@class = "social"]/@href')
+    loader.add_xpath('email', '//footer/a[@class = "email"]/@href')
+    loader.load_item()
+
+Instead, you can create a nested loader with the footer selector and add values
+relative to the footer.  The functionality is the same but you avoid repeating
+the footer selector.
+
+Example::
+
+    loader = ItemLoader(item=Item())
+    # load stuff not in the footer
+    footer_loader = loader.nested_loader(xpath='//footer')
+    footer_loader.add_xpath('social', 'a[@class = "social"]/@href')
+    footer_loader.add_xpath('email', 'a[@class = "email"]/@href')
+    # no need to call footer_loader.load_item()
+    loader.load_item()
+
+You can nest loaders arbitrarilly and they work with either xpath or css selectors.
+As a general guideline, use nested loaders when they make your code simpler but do
+not go overboard with nesting or your parser can become difficult to read.
+
 .. _topics-loaders-extending:
 
 Reusing and extending Item Loaders

From 311d5cd495c7bb82c39a85dd85438871518ea165 Mon Sep 17 00:00:00 2001
From: Daniel Collins <daniel.collins@viasat.com>
Date: Fri, 4 Sep 2015 12:19:10 -0700
Subject: [PATCH 0553/2974] split nested_loader into seperate methods

---
 scrapy/loader/__init__.py | 23 ++++++++++++-----------
 tests/test_loader.py      | 21 ++++++++-------------
 2 files changed, 20 insertions(+), 24 deletions(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index f2c43ab4c..431f4d44b 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -50,18 +50,19 @@ class ItemLoader(object):
         else:
             return self._local_item
 
-    def nested_loader(self, xpath=None, css=None):
-        if xpath is not None and css is not None:
-            raise ValueError("Cannot nest a loader with both a xpath selector and a css selector")
-
-        if xpath is not None:
-            selector = self.selector.xpath(xpath)
-
-        if css is not None:
-            selector = self.selector.css(css)
-
+    def nested_xpath(self, xpath, **context):
+        selector = self.selector.xpath(xpath)
+        context.update(selector=selector)
         subloader = self.__class__(
-            item=self.item, selector=selector, parent=self
+            item=self.item, parent=self, **context
+        )
+        return subloader
+
+    def nested_css(self, css, **context):
+        selector = self.selector.css(css)
+        context.update(selector=selector)
+        subloader = self.__class__(
+            item=self.item, parent=self, **context
         )
         return subloader
 
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 90273c2c8..2693a18d9 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -631,7 +631,7 @@ class SubselectorLoaderTest(unittest.TestCase):
 
     def test_nested_xpath(self):
         l = NestedItemLoader(response=self.response)
-        nl = l.nested_loader(xpath="//header")
+        nl = l.nested_xpath("//header")
         nl.add_xpath('name', 'div/text()')
         nl.add_css('name_div', '#id')
         nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
@@ -646,7 +646,7 @@ class SubselectorLoaderTest(unittest.TestCase):
 
     def test_nested_css(self):
         l = NestedItemLoader(response=self.response)
-        nl = l.nested_loader(css="header")
+        nl = l.nested_css("header")
         nl.add_xpath('name', 'div/text()')
         nl.add_css('name_div', '#id')
         nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
@@ -661,8 +661,8 @@ class SubselectorLoaderTest(unittest.TestCase):
 
     def test_nested_replace(self):
         l = NestedItemLoader(response=self.response)
-        nl1 = l.nested_loader(xpath='//footer')
-        nl2 = nl1.nested_loader(xpath='a')
+        nl1 = l.nested_xpath('//footer')
+        nl2 = nl1.nested_xpath('a')
 
         l.add_xpath('url', '//footer/a/@href')
         self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
@@ -673,8 +673,8 @@ class SubselectorLoaderTest(unittest.TestCase):
 
     def test_nested_ordering(self):
         l = NestedItemLoader(response=self.response)
-        nl1 = l.nested_loader(xpath='//footer')
-        nl2 = nl1.nested_loader(xpath='a')
+        nl1 = l.nested_xpath('//footer')
+        nl2 = nl1.nested_xpath('a')
 
         nl1.add_xpath('url', 'img/@src')
         l.add_xpath('url', '//footer/a/@href')
@@ -690,8 +690,8 @@ class SubselectorLoaderTest(unittest.TestCase):
 
     def test_nested_load_item(self):
         l = NestedItemLoader(response=self.response)
-        nl1 = l.nested_loader(xpath='//footer')
-        nl2 = nl1.nested_loader(xpath='img')
+        nl1 = l.nested_xpath('//footer')
+        nl2 = nl1.nested_xpath('img')
 
         l.add_xpath('name', '//header/div/text()')
         nl1.add_xpath('url', 'a/@href')
@@ -707,11 +707,6 @@ class SubselectorLoaderTest(unittest.TestCase):
         self.assertEqual(item['url'], [u'http://www.scrapy.org'])
         self.assertEqual(item['image'], [u'/images/logo.png'])
 
-    def test_nested_bad_arguments(self):
-        l = NestedItemLoader(response=self.response)
-        with self.assertRaises(ValueError):
-            l.nested_loader(css="#id", xpath="//footer")
-
 
 class SelectJmesTestCase(unittest.TestCase):
         test_list_equals = {

From be7821a48200e784e26f7b6160d44a854953b12e Mon Sep 17 00:00:00 2001
From: Mikhail Lyundin <mlyundin@gmail.com>
Date: Fri, 4 Sep 2015 20:35:27 +0300
Subject: [PATCH 0554/2974] Optimization - avoid temporary list objects,
 unnecessary function call

---
 scrapy/commands/check.py     | 2 +-
 scrapy/commands/genspider.py | 4 ++--
 scrapy/crawler.py            | 2 +-
 scrapy/loader/__init__.py    | 6 +++---
 scrapy/shell.py              | 2 +-
 scrapy/utils/defer.py        | 2 +-
 scrapy/utils/request.py      | 4 ++--
 7 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index 2917b8ba7..017595f04 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -62,7 +62,7 @@ class Command(ScrapyCommand):
             self.settings['SPIDER_CONTRACTS_BASE'],
             self.settings['SPIDER_CONTRACTS'],
         )
-        conman = ContractsManager([load_object(c) for c in contracts])
+        conman = ContractsManager(load_object(c) for c in contracts)
         runner = TextTestRunner(verbosity=2 if opts.verbose else 1)
         result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)
 
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 2c14b3c1f..f0d435379 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -88,8 +88,8 @@ class Command(ScrapyCommand):
             'module': module,
             'name': name,
             'domain': domain,
-            'classname': '%sSpider' % ''.join([s.capitalize() \
-                for s in module.split('_')])
+            'classname': '%sSpider' % ''.join(s.capitalize() \
+                for s in module.split('_'))
         }
         spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
         spiders_dir = abspath(dirname(spiders_module.__file__))
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 2f1a92d31..c7e3bb528 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -173,7 +173,7 @@ class CrawlerRunner(object):
 
         Returns a deferred that is fired when they all have ended.
         """
-        return defer.DeferredList([c.stop() for c in list(self.crawlers)])
+        return defer.DeferredList([c.stop() for c in self.crawlers])
 
     @defer.inlineCallbacks
     def join(self):
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 84640f0b6..6c2ff968e 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -69,7 +69,7 @@ class ItemLoader(object):
         regex = kw.get('re', None)
         if regex:
             value = arg_to_iter(value)
-            value = flatten([extract_regex(regex, x) for x in value])
+            value = flatten(extract_regex(regex, x) for x in value)
 
         for proc in processors:
             if value is None:
@@ -149,7 +149,7 @@ class ItemLoader(object):
     def _get_xpathvalues(self, xpaths, **kw):
         self._check_selector_method()
         xpaths = arg_to_iter(xpaths)
-        return flatten([self.selector.xpath(xpath).extract() for xpath in xpaths])
+        return flatten(self.selector.xpath(xpath).extract() for xpath in xpaths)
 
     def add_css(self, field_name, css, *processors, **kw):
         values = self._get_cssvalues(css, **kw)
@@ -166,7 +166,7 @@ class ItemLoader(object):
     def _get_cssvalues(self, csss, **kw):
         self._check_selector_method()
         csss = arg_to_iter(csss)
-        return flatten([self.selector.css(css).extract() for css in csss])
+        return flatten(self.selector.css(css).extract() for css in csss)
 
 
 XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)
diff --git a/scrapy/shell.py b/scrapy/shell.py
index f008ce39a..099e1af0a 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -146,7 +146,7 @@ class Shell(object):
                      "update local objects")
         b.append("  view(response)    View response in a browser")
 
-        return "\n".join(["[s] %s" % l for l in b])
+        return "\n".join("[s] %s" % l for l in b)
 
     def _is_relevant(self, value):
         return isinstance(value, self.relevant_classes)
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 8f3824abf..bb4c74a6e 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -61,7 +61,7 @@ def parallel(iterable, count, callable, *args, **named):
     """
     coop = task.Cooperator()
     work = (callable(elem, *args, **named) for elem in iterable)
-    return defer.DeferredList([coop.coiterate(work) for i in range(count)])
+    return defer.DeferredList([coop.coiterate(work) for _ in range(count)])
 
 def process_chain(callbacks, input, *a, **kw):
     """Return a Deferred built by chaining the given callbacks"""
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index e361b7433..a98ca61ce 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -44,8 +44,8 @@ def request_fingerprint(request, include_headers=None):
 
     """
     if include_headers:
-        include_headers = tuple([to_bytes(h.lower())
-                                 for h in sorted(include_headers)])
+        include_headers = tuple(to_bytes(h.lower())
+                                 for h in sorted(include_headers))
     cache = _fingerprint_cache.setdefault(request, {})
     if include_headers not in cache:
         fp = hashlib.sha1()

From eb7b9d506f599f821842448a1b1dc7b3928decde Mon Sep 17 00:00:00 2001
From: Julia Medina <who.knows.shia@gmail.com>
Date: Tue, 8 Sep 2015 18:49:44 -0300
Subject: [PATCH 0555/2974] Add note to ubuntu install section about debian
 compatibility

---
 docs/intro/install.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 978d89920..3adb4e6b0 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -91,6 +91,11 @@ You can install Scrapy with ``pip`` after that::
 
     pip install Scrapy
 
+.. note::
+
+    The same non-python dependencies can be used to install Scrapy in Debian
+    Wheezy (7.0) and above.
+
 Archlinux
 ---------
 

From 9a64d8ff97f2b188d2be8a1d7944a47790adda2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 9 Sep 2015 15:55:55 -0300
Subject: [PATCH 0556/2974] fix scrapy squeue tests after recent changes to
 queuelib

---
 tests/test_squeues.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index f2d7be67d..48871ceeb 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -19,7 +19,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
     chunksize = 100000
 
     def queue(self):
-        return MarshalFifoDiskQueue(self.qdir, chunksize=self.chunksize)
+        return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize)
 
     def test_serialize(self):
         q = self.queue()
@@ -54,7 +54,7 @@ class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest):
     chunksize = 100000
 
     def queue(self):
-        return PickleFifoDiskQueue(self.qdir, chunksize=self.chunksize)
+        return PickleFifoDiskQueue(self.qpath, chunksize=self.chunksize)
 
     def test_serialize_item(self):
         q = self.queue()
@@ -99,7 +99,7 @@ class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
 class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
 
     def queue(self):
-        return MarshalLifoDiskQueue(self.path)
+        return MarshalLifoDiskQueue(self.qpath)
 
     def test_serialize(self):
         q = self.queue()
@@ -120,7 +120,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
 class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest):
 
     def queue(self):
-        return PickleLifoDiskQueue(self.path)
+        return PickleLifoDiskQueue(self.qpath)
 
     def test_serialize_item(self):
         q = self.queue()

From 563b1500bf384e104169ee214bed328833658843 Mon Sep 17 00:00:00 2001
From: Alexander Chekunkov <chekunkov@gmail.com>
Date: Fri, 29 May 2015 17:45:45 +0700
Subject: [PATCH 0557/2974] Make list of enabled middlewares more readable

---
 scrapy/middleware.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index a7adc39e3..2ef5f30e2 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,5 +1,6 @@
-import logging
 from collections import defaultdict
+import logging
+import pprint
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
@@ -43,11 +44,9 @@ class MiddlewareManager(object):
                     logger.warning("Disabled %(clsname)s: %(eargs)s",
                                    {'clsname': clsname, 'eargs': e.args[0]},
                                    extra={'crawler': crawler})
-
-        enabled = [x.__class__.__name__ for x in middlewares]
-        logger.info("Enabled %(componentname)ss: %(enabledlist)s",
+        logger.info("Enabled %(componentname)ss:\n%(enabledlist)s",
                     {'componentname': cls.component_name,
-                     'enabledlist': ', '.join(enabled)},
+                     'enabledlist': pprint.pformat(mwlist)},
                     extra={'crawler': crawler})
         return cls(*middlewares)
 

From 44bd01d51cc0d2008a73e67845922a23d7aee344 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 10 Sep 2015 16:19:56 -0300
Subject: [PATCH 0558/2974] preparatory refactor in redirect mware

---
 scrapy/downloadermiddlewares/redirect.py | 31 +++++++++++-------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 3cf8d2bee..c14340b27 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -55,28 +55,25 @@ class RedirectMiddleware(BaseRedirectMiddleware):
 
     def process_response(self, request, response, spider):
         if (request.meta.get('dont_redirect', False) or
-               response.status in getattr(spider, 'handle_httpstatus_list', []) or
-               response.status in request.meta.get('handle_httpstatus_list', []) or
-               request.meta.get('handle_httpstatus_all', False)):
+                response.status in getattr(spider, 'handle_httpstatus_list', []) or
+                response.status in request.meta.get('handle_httpstatus_list', []) or
+                request.meta.get('handle_httpstatus_all', False)):
             return response
 
-        if request.method == 'HEAD':
-            if response.status in [301, 302, 303, 307] and 'Location' in response.headers:
-                redirected_url = urljoin(request.url, response.headers['location'])
+        location = None
+        if 'Location' in response.headers:
+            location = response.headers['location']
+
+        if location is not None and response.status in [301, 302, 303, 307]:
+            redirected_url = urljoin(request.url, location)
+
+            if response.status in [301, 307] or request.method == 'HEAD':
                 redirected = request.replace(url=redirected_url)
                 return self._redirect(redirected, request, spider, response.status)
-            else:
-                return response
 
-        if response.status in [302, 303] and 'Location' in response.headers:
-            redirected_url = urljoin(request.url, response.headers['location'])
-            redirected = self._redirect_request_using_get(request, redirected_url)
-            return self._redirect(redirected, request, spider, response.status)
-
-        if response.status in [301, 307] and 'Location' in response.headers:
-            redirected_url = urljoin(request.url, response.headers['location'])
-            redirected = request.replace(url=redirected_url)
-            return self._redirect(redirected, request, spider, response.status)
+            if response.status in [302, 303]:
+                redirected = self._redirect_request_using_get(request, redirected_url)
+                return self._redirect(redirected, request, spider, response.status)
 
         return response
 

From defa89913559b543cf7336653e44350481c1c38a Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 10 Sep 2015 16:31:59 -0300
Subject: [PATCH 0559/2974] PY3 port redirect middleware

---
 scrapy/downloadermiddlewares/redirect.py    |  2 +-
 tests/py3-ignores.txt                       |  1 -
 tests/test_downloadermiddleware_redirect.py | 19 +++++++++++--------
 3 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index c14340b27..ceb0a55a3 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -62,7 +62,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
 
         location = None
         if 'Location' in response.headers:
-            location = response.headers['location']
+            location = response.headers['location'].decode('latin1')
 
         if location is not None and response.status in [301, 302, 303, 307]:
             redirected_url = urljoin(request.url, location)
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 95e2181e3..e40b4c73c 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -10,7 +10,6 @@ tests/test_downloadermiddleware_httpcache.py
 tests/test_downloadermiddleware_httpcompression.py
 tests/test_downloadermiddleware_httpproxy.py
 tests/test_downloadermiddleware.py
-tests/test_downloadermiddleware_redirect.py
 tests/test_downloadermiddleware_retry.py
 tests/test_engine.py
 tests/test_mail.py
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index b3db7c42b..15e1dff37 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -164,13 +164,13 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
 
     def test_priority_adjust(self):
         req = Request('http://a.com')
-        rsp = HtmlResponse(req.url, body=self._body())
+        rsp = HtmlResponse(req.url, body=self._body(), encoding='utf-8')
         req2 = self.mw.process_response(req, rsp, self.spider)
         assert req2.priority > req.priority
 
     def test_meta_refresh(self):
         req = Request(url='http://example.org')
-        rsp = HtmlResponse(req.url, body=self._body())
+        rsp = HtmlResponse(req.url, body=self._body(), encoding='utf-8')
         req2 = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req2, Request)
         self.assertEqual(req2.url, 'http://example.org/newpage')
@@ -178,14 +178,16 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
     def test_meta_refresh_with_high_interval(self):
         # meta-refresh with high intervals don't trigger redirects
         req = Request(url='http://example.org')
-        rsp = HtmlResponse(url='http://example.org', body=self._body(interval=1000))
+        rsp = HtmlResponse(url='http://example.org',
+                           body=self._body(interval=1000),
+                           encoding='utf-8')
         rsp2 = self.mw.process_response(req, rsp, self.spider)
         assert rsp is rsp2
 
     def test_meta_refresh_trough_posted_request(self):
         req = Request(url='http://example.org', method='POST', body='test',
                       headers={'Content-Type': 'text/plain', 'Content-length': '4'})
-        rsp = HtmlResponse(req.url, body=self._body())
+        rsp = HtmlResponse(req.url, body=self._body(), encoding='utf-8')
         req2 = self.mw.process_response(req, rsp, self.spider)
 
         assert isinstance(req2, Request)
@@ -201,7 +203,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
     def test_max_redirect_times(self):
         self.mw.max_redirect_times = 1
         req = Request('http://scrapytest.org/max')
-        rsp = HtmlResponse(req.url, body=self._body())
+        rsp = HtmlResponse(req.url, body=self._body(), encoding='utf-8')
 
         req = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req, Request)
@@ -212,7 +214,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
     def test_ttl(self):
         self.mw.max_redirect_times = 100
         req = Request('http://scrapytest.org/302', meta={'redirect_ttl': 1})
-        rsp = HtmlResponse(req.url, body=self._body())
+        rsp = HtmlResponse(req.url, body=self._body(), encoding='utf-8')
 
         req = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req, Request)
@@ -220,10 +222,10 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
 
     def test_redirect_urls(self):
         req1 = Request('http://scrapytest.org/first')
-        rsp1 = HtmlResponse(req1.url, body=self._body(url='/redirected'))
+        rsp1 = HtmlResponse(req1.url, body=self._body(url='/redirected'), encoding='utf-8')
         req2 = self.mw.process_response(req1, rsp1, self.spider)
         assert isinstance(req2, Request), req2
-        rsp2 = HtmlResponse(req2.url, body=self._body(url='/redirected2'))
+        rsp2 = HtmlResponse(req2.url, body=self._body(url='/redirected2'), encoding='utf-8')
         req3 = self.mw.process_response(req2, rsp2, self.spider)
         assert isinstance(req3, Request), req3
         self.assertEqual(req2.url, 'http://scrapytest.org/redirected')
@@ -231,5 +233,6 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
         self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
         self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])
 
+
 if __name__ == "__main__":
     unittest.main()

From 179c4588ca12d363ab8ce9d2badad6c456d353a9 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 10 Sep 2015 19:36:47 -0300
Subject: [PATCH 0560/2974] adding test for latin1 location

---
 tests/test_downloadermiddleware_redirect.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 15e1dff37..11d9fd152 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import unittest
 
 from scrapy.downloadermiddlewares.redirect import RedirectMiddleware, MetaRefreshMiddleware
@@ -150,6 +152,14 @@ class RedirectMiddlewareTest(unittest.TestCase):
                                                            [404, 301, 302]}))
         _test_passthrough(Request(url, meta={'handle_httpstatus_all': True}))
 
+    def test_latin1_location(self):
+        req = Request('http://scrapytest.org/first')
+        latin1_path = u'/ação'.encode('latin1')
+        resp = Response('http://scrapytest.org/first', headers={'Location': latin1_path}, status=302)
+        req_result = self.mw.process_response(req, resp, self.spider)
+        perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
+        self.assertEquals(perc_encoded_utf8_url, req_result.url)
+
 
 class MetaRefreshMiddlewareTest(unittest.TestCase):
 

From 81950f773d588000ef2bbe720caa0c2fc4d3f0df Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Fri, 11 Sep 2015 18:51:48 -0300
Subject: [PATCH 0561/2974] made encoding conversion more explicit, added test
 for header with utf-8 encoding replicating what browsers do

---
 scrapy/downloadermiddlewares/redirect.py    |  4 +++-
 tests/test_downloadermiddleware_redirect.py | 12 ++++++++++--
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index ceb0a55a3..3a6da85e8 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -3,6 +3,7 @@ from six.moves.urllib.parse import urljoin
 
 from scrapy.http import HtmlResponse
 from scrapy.utils.response import get_meta_refresh
+from scrapy.utils.python import to_native_str
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 
 logger = logging.getLogger(__name__)
@@ -62,7 +63,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):
 
         location = None
         if 'Location' in response.headers:
-            location = response.headers['location'].decode('latin1')
+            # HTTP header is ascii or latin1, redirected url will be percent-encoded utf-8
+            location = to_native_str(response.headers['location'].decode('latin1'))
 
         if location is not None and response.status in [301, 302, 303, 307]:
             redirected_url = urljoin(request.url, location)
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 11d9fd152..1f4c2d67a 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -154,12 +154,20 @@ class RedirectMiddlewareTest(unittest.TestCase):
 
     def test_latin1_location(self):
         req = Request('http://scrapytest.org/first')
-        latin1_path = u'/ação'.encode('latin1')
-        resp = Response('http://scrapytest.org/first', headers={'Location': latin1_path}, status=302)
+        latin1_location = u'/ação'.encode('latin1')  # HTTP historically supports latin1
+        resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302)
         req_result = self.mw.process_response(req, resp, self.spider)
         perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
         self.assertEquals(perc_encoded_utf8_url, req_result.url)
 
+    def test_location_with_wrong_encoding(self):
+        req = Request('http://scrapytest.org/first')
+        utf8_location = u'/ação'  # header with wrong encoding (utf-8)
+        resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302)
+        req_result = self.mw.process_response(req, resp, self.spider)
+        perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%83%C2%A7%C3%83%C2%A3o'
+        self.assertEquals(perc_encoded_utf8_url, req_result.url)
+
 
 class MetaRefreshMiddlewareTest(unittest.TestCase):
 

From b06d0706e6644187b4c4a6f7ac80e8adbf376907 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Sun, 13 Sep 2015 11:49:59 -0300
Subject: [PATCH 0562/2974] refactoring redirect logic

---
 scrapy/downloadermiddlewares/redirect.py | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 3a6da85e8..4ed7e4c24 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -61,23 +61,21 @@ class RedirectMiddleware(BaseRedirectMiddleware):
                 request.meta.get('handle_httpstatus_all', False)):
             return response
 
-        location = None
-        if 'Location' in response.headers:
-            # HTTP header is ascii or latin1, redirected url will be percent-encoded utf-8
-            location = to_native_str(response.headers['location'].decode('latin1'))
+        allowed_status = (301, 302, 303, 307)
+        if 'Location' not in response.headers or response.status not in allowed_status:
+            return response
 
-        if location is not None and response.status in [301, 302, 303, 307]:
-            redirected_url = urljoin(request.url, location)
+        # HTTP header is ascii or latin1, redirected url will be percent-encoded utf-8
+        location = to_native_str(response.headers['location'].decode('latin1'))
 
-            if response.status in [301, 307] or request.method == 'HEAD':
-                redirected = request.replace(url=redirected_url)
-                return self._redirect(redirected, request, spider, response.status)
+        redirected_url = urljoin(request.url, location)
 
-            if response.status in [302, 303]:
-                redirected = self._redirect_request_using_get(request, redirected_url)
-                return self._redirect(redirected, request, spider, response.status)
+        if response.status in (301, 307) or request.method == 'HEAD':
+            redirected = request.replace(url=redirected_url)
+            return self._redirect(redirected, request, spider, response.status)
 
-        return response
+        redirected = self._redirect_request_using_get(request, redirected_url)
+        return self._redirect(redirected, request, spider, response.status)
 
 
 class MetaRefreshMiddleware(BaseRedirectMiddleware):

From 430e90f4b209af51deec0a85d214dfa3171eaf0e Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Sun, 13 Sep 2015 12:29:41 -0300
Subject: [PATCH 0563/2974] minor refactor on metarefresh redirect mware test

---
 tests/test_downloadermiddleware_redirect.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 1f4c2d67a..9db073cc5 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -177,18 +177,18 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
         self.mw = MetaRefreshMiddleware.from_crawler(crawler)
 
     def _body(self, interval=5, url='http://example.org/newpage'):
-        return """<html><head><meta http-equiv="refresh" content="{0};url={1}"/></head></html>"""\
-                .format(interval, url)
+        html = u"""<html><head><meta http-equiv="refresh" content="{0};url={1}"/></head></html>"""
+        return html.format(interval, url).encode('utf-8')
 
     def test_priority_adjust(self):
         req = Request('http://a.com')
-        rsp = HtmlResponse(req.url, body=self._body(), encoding='utf-8')
+        rsp = HtmlResponse(req.url, body=self._body())
         req2 = self.mw.process_response(req, rsp, self.spider)
         assert req2.priority > req.priority
 
     def test_meta_refresh(self):
         req = Request(url='http://example.org')
-        rsp = HtmlResponse(req.url, body=self._body(), encoding='utf-8')
+        rsp = HtmlResponse(req.url, body=self._body())
         req2 = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req2, Request)
         self.assertEqual(req2.url, 'http://example.org/newpage')
@@ -205,7 +205,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
     def test_meta_refresh_trough_posted_request(self):
         req = Request(url='http://example.org', method='POST', body='test',
                       headers={'Content-Type': 'text/plain', 'Content-length': '4'})
-        rsp = HtmlResponse(req.url, body=self._body(), encoding='utf-8')
+        rsp = HtmlResponse(req.url, body=self._body())
         req2 = self.mw.process_response(req, rsp, self.spider)
 
         assert isinstance(req2, Request)
@@ -221,7 +221,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
     def test_max_redirect_times(self):
         self.mw.max_redirect_times = 1
         req = Request('http://scrapytest.org/max')
-        rsp = HtmlResponse(req.url, body=self._body(), encoding='utf-8')
+        rsp = HtmlResponse(req.url, body=self._body())
 
         req = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req, Request)
@@ -232,7 +232,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
     def test_ttl(self):
         self.mw.max_redirect_times = 100
         req = Request('http://scrapytest.org/302', meta={'redirect_ttl': 1})
-        rsp = HtmlResponse(req.url, body=self._body(), encoding='utf-8')
+        rsp = HtmlResponse(req.url, body=self._body())
 
         req = self.mw.process_response(req, rsp, self.spider)
         assert isinstance(req, Request)
@@ -240,10 +240,10 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
 
     def test_redirect_urls(self):
         req1 = Request('http://scrapytest.org/first')
-        rsp1 = HtmlResponse(req1.url, body=self._body(url='/redirected'), encoding='utf-8')
+        rsp1 = HtmlResponse(req1.url, body=self._body(url='/redirected'))
         req2 = self.mw.process_response(req1, rsp1, self.spider)
         assert isinstance(req2, Request), req2
-        rsp2 = HtmlResponse(req2.url, body=self._body(url='/redirected2'), encoding='utf-8')
+        rsp2 = HtmlResponse(req2.url, body=self._body(url='/redirected2'))
         req3 = self.mw.process_response(req2, rsp2, self.spider)
         assert isinstance(req3, Request), req3
         self.assertEqual(req2.url, 'http://scrapytest.org/redirected')

From 5a13022713d6dc6a88ad914798b7d62e2b3deffb Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sun, 13 Sep 2015 21:46:20 +0500
Subject: [PATCH 0564/2974] style fixes for settings.py created by scrapy
 startproject

---
 .../templates/project/module/settings.py.tmpl | 34 ++++++++++---------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 11ff804ba..2e660d7cf 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -19,21 +19,21 @@ NEWSPIDER_MODULE = '$project_name.spiders'
 #USER_AGENT = '$project_name (+http://www.yourdomain.com)'
 
 # Configure maximum concurrent requests performed by Scrapy (default: 16)
-#CONCURRENT_REQUESTS=32
+#CONCURRENT_REQUESTS = 32
 
 # Configure a delay for requests for the same website (default: 0)
 # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
 # See also autothrottle settings and docs
-#DOWNLOAD_DELAY=3
+#DOWNLOAD_DELAY = 3
 # The download delay setting will honor only one of:
-#CONCURRENT_REQUESTS_PER_DOMAIN=16
-#CONCURRENT_REQUESTS_PER_IP=16
+#CONCURRENT_REQUESTS_PER_DOMAIN = 16
+#CONCURRENT_REQUESTS_PER_IP = 16
 
 # Disable cookies (enabled by default)
-#COOKIES_ENABLED=False
+#COOKIES_ENABLED = False
 
 # Disable Telnet Console (enabled by default)
-#TELNETCONSOLE_ENABLED=False
+#TELNETCONSOLE_ENABLED = False
 
 # Override the default request headers:
 #DEFAULT_REQUEST_HEADERS = {
@@ -67,19 +67,21 @@ NEWSPIDER_MODULE = '$project_name.spiders'
 
 # Enable and configure the AutoThrottle extension (disabled by default)
 # See http://doc.scrapy.org/en/latest/topics/autothrottle.html
-# NOTE: AutoThrottle will honour the standard settings for concurrency and delay
-#AUTOTHROTTLE_ENABLED=True
+#AUTOTHROTTLE_ENABLED = True
 # The initial download delay
-#AUTOTHROTTLE_START_DELAY=5
+#AUTOTHROTTLE_START_DELAY = 5
 # The maximum download delay to be set in case of high latencies
-#AUTOTHROTTLE_MAX_DELAY=60
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
 # Enable showing throttling stats for every response received:
-#AUTOTHROTTLE_DEBUG=False
+#AUTOTHROTTLE_DEBUG = False
 
 # Enable and configure HTTP caching (disabled by default)
 # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
-#HTTPCACHE_ENABLED=True
-#HTTPCACHE_EXPIRATION_SECS=0
-#HTTPCACHE_DIR='httpcache'
-#HTTPCACHE_IGNORE_HTTP_CODES=[]
-#HTTPCACHE_STORAGE='scrapy.extensions.httpcache.FilesystemCacheStorage'
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

From ecbfe4bd6661acd165407e7f1e5abf1f0a2fa31c Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sun, 12 Jul 2015 16:53:54 +0000
Subject: [PATCH 0565/2974] drop deprecated "optional_features" set

---
 scrapy/__init__.py                       | 9 +--------
 scrapy/core/downloader/handlers/http.py  | 4 ++--
 scrapy/utils/log.py                      | 3 ---
 tests/test_downloader_handlers.py        | 7 +++----
 tests/test_downloadermiddleware_retry.py | 4 ++--
 tests/test_toplevel.py                   | 4 ----
 6 files changed, 8 insertions(+), 23 deletions(-)

diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index c0477f509..03ec6c667 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -2,7 +2,7 @@
 Scrapy - a web crawling and web scraping framework written for Python
 """
 
-__all__ = ['__version__', 'version_info', 'optional_features', 'twisted_version',
+__all__ = ['__version__', 'version_info', 'twisted_version',
            'Spider', 'Request', 'FormRequest', 'Selector', 'Item', 'Field']
 
 # Scrapy version
@@ -27,15 +27,8 @@ del warnings
 from . import _monkeypatches
 del _monkeypatches
 
-# WARNING: optional_features set is deprecated and will be removed soon. Do not use.
-optional_features = set()
-# TODO: backwards compatibility, remove for Scrapy 0.20
-optional_features.add('ssl')
-
 from twisted import version as _txv
 twisted_version = (_txv.major, _txv.minor, _txv.micro)
-if twisted_version >= (11, 1, 0):
-    optional_features.add('http11')
 
 # Declare top-level shortcuts
 from scrapy.spiders import Spider
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index 1efebb939..81da2615a 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -1,7 +1,7 @@
-from scrapy import optional_features
+from scrapy import twisted_version
 from .http10 import HTTP10DownloadHandler
 
-if 'http11' in optional_features:
+if twisted_version >= (11, 1, 0):
     from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
 else:
     HTTPDownloadHandler = HTTP10DownloadHandler
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index d40202953..cc2f0b164 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -126,9 +126,6 @@ def log_scrapy_info(settings):
     logger.info("Scrapy %(version)s started (bot: %(bot)s)",
                 {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
 
-    logger.info("Optional features available: %(features)s",
-                {'features': ", ".join(scrapy.optional_features)})
-
     d = dict(overridden_settings(settings))
     logger.info("Overridden settings: %(settings)r", {'settings': d})
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index e4d957d8e..d2a349b40 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -27,7 +27,6 @@ from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
 from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.settings import Settings
-from scrapy import optional_features
 from scrapy.utils.test import get_crawler
 from scrapy.exceptions import NotConfigured
 
@@ -220,7 +219,7 @@ class Http10TestCase(HttpTestCase):
 class Http11TestCase(HttpTestCase):
     """HTTP 1.1 test case"""
     download_handler_cls = HTTP11DownloadHandler
-    if 'http11' not in optional_features:
+    if twisted_version < (11, 1, 0):
         skip = 'HTTP1.1 not supported in twisted < 11.1.0'
 
     def test_download_without_maxsize_limit(self):
@@ -267,7 +266,7 @@ class Http11TestCase(HttpTestCase):
 
 class Http11MockServerTestCase(unittest.TestCase):
     """HTTP 1.1 test case with MockServer"""
-    if 'http11' not in optional_features:
+    if twisted_version < (11, 1, 0):
         skip = 'HTTP1.1 not supported in twisted < 11.1.0'
 
     def setUp(self):
@@ -392,7 +391,7 @@ class Http10ProxyTestCase(HttpProxyTestCase):
 
 class Http11ProxyTestCase(HttpProxyTestCase):
     download_handler_cls = HTTP11DownloadHandler
-    if 'http11' not in optional_features:
+    if twisted_version < (11, 1, 0):
         skip = 'HTTP1.1 not supported in twisted < 11.1.0'
 
 
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index c0381e144..20561e771 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -4,7 +4,7 @@ from twisted.internet.error import TimeoutError, DNSLookupError, \
         ConnectionRefusedError, ConnectionDone, ConnectError, \
         ConnectionLost, TCPTimedOutError
 
-from scrapy import optional_features
+from scrapy import twisted_version
 from scrapy.downloadermiddlewares.retry import RetryMiddleware
 from scrapy.xlib.tx import ResponseFailed
 from scrapy.spiders import Spider
@@ -75,7 +75,7 @@ class RetryTest(unittest.TestCase):
         exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError,
                 DNSLookupError, ConnectionRefusedError, ConnectionDone,
                 ConnectError, ConnectionLost]
-        if 'http11' in optional_features:
+        if twisted_version >= (11, 1, 0): # http11 available
             exceptions.append(ResponseFailed)
 
         for exc in exceptions:
diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py
index e9f220092..91bbe43bc 100644
--- a/tests/test_toplevel.py
+++ b/tests/test_toplevel.py
@@ -11,10 +11,6 @@ class ToplevelTestCase(TestCase):
     def test_version_info(self):
         self.assertIs(type(scrapy.version_info), tuple)
 
-    def test_optional_features(self):
-        self.assertIs(type(scrapy.optional_features), set)
-        self.assertIn('ssl', scrapy.optional_features)
-
     def test_request_shortcut(self):
         from scrapy.http import Request, FormRequest
         self.assertIs(scrapy.Request, Request)

From 036109e7ded7e186594c5c1ae642400e9e7d4b25 Mon Sep 17 00:00:00 2001
From: Daniel Collins <dacjames@gmail.com>
Date: Tue, 15 Sep 2015 23:49:35 -0700
Subject: [PATCH 0566/2974] updte nested loader documentation

---
 docs/topics/loaders.rst | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 0c20f32ef..b188a0b81 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -432,9 +432,17 @@ ItemLoader objects
         <topics-loaders-processors>` to get the final value to assign to each
         item field.
 
-    .. method:: nested_loader(xpath=selector, css=selector)
+    .. method:: nested_xpath(xpath)
 
-        Create a nested loader with either an xpath selector or css selector.
+        Create a nested loader with an xpath selector.
+        The supplied selector is applied relative to selector associated
+        with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
+        with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
+        :meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
+
+    .. method:: nested_css(css)
+
+        Create a nested loader with a css selector.
         The supplied selector is applied relative to selector associated
         with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
         with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
@@ -534,7 +542,7 @@ Example::
 
     loader = ItemLoader(item=Item())
     # load stuff not in the footer
-    footer_loader = loader.nested_loader(xpath='//footer')
+    footer_loader = loader.nested_xpath('//footer')
     footer_loader.add_xpath('social', 'a[@class = "social"]/@href')
     footer_loader.add_xpath('email', 'a[@class = "email"]/@href')
     # no need to call footer_loader.load_item()

From 14f7f22555421381c9f9470180a0e0b0fbdf66aa Mon Sep 17 00:00:00 2001
From: hy <jiang.haiyun@gmail.com>
Date: Wed, 16 Sep 2015 16:59:23 +0800
Subject: [PATCH 0567/2974] fix typos in downloader-middleware.rst and
 exceptions.rst, middlware -> middleware

---
 docs/topics/downloader-middleware.rst | 2 +-
 docs/topics/exceptions.rst            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 38c9456db..b79da1b95 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -804,7 +804,7 @@ RetryMiddleware
 
 .. class:: RetryMiddleware
 
-   A middlware to retry failed requests that are potentially caused by
+   A middleware to retry failed requests that are potentially caused by
    temporary problems such as a connection timeout or HTTP 500 error.
 
 Failed pages are collected on the scraping process and rescheduled at the
diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index 8a10ee796..9f8d16d84 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -57,7 +57,7 @@ remain disabled. Those components include:
 
  * Extensions
  * Item pipelines
- * Downloader middlwares
+ * Downloader middlewares
  * Spider middlewares
 
 The exception must be raised in the component constructor.

From 22327ef47bd4b8098baf43d334d7852ae0c42690 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Thu, 17 Sep 2015 01:05:52 -0300
Subject: [PATCH 0568/2974] disable log on startproject command

---
 scrapy/commands/startproject.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 0c77cad01..4375b6d4c 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -27,6 +27,7 @@ IGNORE = ignore_patterns('*.pyc', '.svn')
 class Command(ScrapyCommand):
 
     requires_project = False
+    default_settings = {'LOG_ENABLED': False}
 
     def syntax(self):
         return "<project_name>"

From a57efdb0083a19d504bf2f27ddaad2ce7bdc0987 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 17 Sep 2015 14:07:47 -0300
Subject: [PATCH 0569/2974] Add PyPI download stats badge

---
 README.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.rst b/README.rst
index ec3604875..6cbed75ee 100644
--- a/README.rst
+++ b/README.rst
@@ -6,6 +6,10 @@ Scrapy
    :target: https://pypi.python.org/pypi/Scrapy
    :alt: PyPI Version
 
+.. image:: https://img.shields.io/pypi/dm/Scrapy.svg
+   :target: https://pypi.python.org/pypi/Scrapy
+   :alt: PyPI Monthly downloads
+
 .. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
    :target: http://travis-ci.org/scrapy/scrapy
    :alt: Build Status

From a3390afc66134e77f98ae3bfae7bc23479ed8566 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 17 Sep 2015 19:42:29 -0300
Subject: [PATCH 0570/2974] test suite requires recent queuelib to pass because
 it depends on queuelib test suite

---
 tests/requirements-py3.txt | 2 ++
 tests/requirements.txt     | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index cac8da9b5..53cb49f9b 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -6,3 +6,5 @@ jmespath
 # optional for shell wrapper tests
 bpython
 ipython
+# required because squeue tests extend from queuelib testsuite
+queuelib>=1.4.2
diff --git a/tests/requirements.txt b/tests/requirements.txt
index f88d17538..a5e734417 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -9,3 +9,5 @@ testfixtures
 # optional for shell wrapper tests
 bpython
 ipython
+# required because squeue tests extend from queuelib testsuite
+queuelib>=1.4.2

From 51b124903302ed52f3752715b308def6927c01e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 18 Sep 2015 02:27:07 -0300
Subject: [PATCH 0571/2974] Revert "test suite requires recent queuelib to pass
 because it depends on queuelib test suite"

This reverts commit a3390afc66134e77f98ae3bfae7bc23479ed8566.
---
 tests/requirements-py3.txt | 2 --
 tests/requirements.txt     | 2 --
 2 files changed, 4 deletions(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 53cb49f9b..cac8da9b5 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -6,5 +6,3 @@ jmespath
 # optional for shell wrapper tests
 bpython
 ipython
-# required because squeue tests extend from queuelib testsuite
-queuelib>=1.4.2
diff --git a/tests/requirements.txt b/tests/requirements.txt
index a5e734417..f88d17538 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -9,5 +9,3 @@ testfixtures
 # optional for shell wrapper tests
 bpython
 ipython
-# required because squeue tests extend from queuelib testsuite
-queuelib>=1.4.2

From 97a52665a0b96c8e2b93303dd5b10276ffac1ed2 Mon Sep 17 00:00:00 2001
From: Demelziraptor <demelza@circularvale.com>
Date: Fri, 18 Sep 2015 17:16:43 +0900
Subject: [PATCH 0572/2974] interpreting json-amazonui-streaming as
 TextResponse

---
 scrapy/responsetypes.py     | 1 +
 tests/test_responsetypes.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 4880cc7b9..319656648 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -25,6 +25,7 @@ class ResponseTypes(object):
         'application/xml': 'scrapy.http.XmlResponse',
         'application/json': 'scrapy.http.TextResponse',
         'application/x-json': 'scrapy.http.TextResponse',
+        'application/json-amazonui-streaming': 'scrapy.http.TextResponse',
         'application/javascript': 'scrapy.http.TextResponse',
         'application/x-javascript': 'scrapy.http.TextResponse',
         'text/xml': 'scrapy.http.XmlResponse',
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 2374d518f..b34147b74 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -36,6 +36,7 @@ class ResponseTypesTest(unittest.TestCase):
             ('application/xml; charset=UTF-8', XmlResponse),
             ('application/octet-stream', Response),
             ('application/x-json; encoding=UTF8;charset=UTF-8', TextResponse),
+            ('application/json-amazonui-streaming;charset=UTF-8', TextResponse),
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_type(source)

From 916141a08f37e2a1dc816ce44bfc3e636770b7c9 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 21 Sep 2015 21:12:52 +0500
Subject: [PATCH 0573/2974] TST pin pytest to 2.7.3

---
 tests/requirements-py3.txt | 2 +-
 tests/requirements.txt     | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index cac8da9b5..a709a734e 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,4 +1,4 @@
-pytest>=2.6.0
+pytest==2.7.3
 pytest-twisted
 pytest-cov
 testfixtures
diff --git a/tests/requirements.txt b/tests/requirements.txt
index f88d17538..8901fe16b 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -2,6 +2,7 @@
 mock
 mitmproxy==0.10.1
 netlib==0.10.1
+pytest==2.7.3
 pytest-twisted
 pytest-cov
 jmespath

From c517951a484c25346e96651c34e88105dc0908ef Mon Sep 17 00:00:00 2001
From: preetwinder <preetwinder14496@gmail.com>
Date: Wed, 16 Sep 2015 14:05:05 +0530
Subject: [PATCH 0574/2974] add_scheme_if_missing for scrapy shell command

---
 scrapy/commands/shell.py |  3 +++
 scrapy/utils/url.py      |  8 ++++++++
 tests/test_utils_url.py  | 18 +++++++++++++++++-
 3 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 95af8586b..e94e339de 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -9,6 +9,7 @@ from threading import Thread
 from scrapy.commands import ScrapyCommand
 from scrapy.shell import Shell
 from scrapy.http import Request
+from scrapy.utils.url import add_scheme_if_missing
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 
 
@@ -41,6 +42,8 @@ class Command(ScrapyCommand):
 
     def run(self, args, opts):
         url = args[0] if args else None
+        if url:
+            url = add_scheme_if_missing(url)
         spider_loader = self.crawler_process.spider_loader
 
         spidercls = DefaultSpider
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 99f350361..94ec4de1b 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -110,3 +110,11 @@ def escape_ajax(url):
     if not frag.startswith('!'):
         return url
     return add_or_replace_parameter(defrag, '_escaped_fragment_', frag[1:])
+
+def add_scheme_if_missing(url):
+    parser = parse_url(url)
+    if not parser.scheme:
+        if not parser.netloc:
+            parser = parser._replace(netloc=parser.path, path='')
+        parser = parser._replace(scheme='http')
+    return parser.geturl() 
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 7bf0e5b4a..fae4c988b 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -4,7 +4,7 @@ import unittest
 import six
 from scrapy.spiders import Spider
 from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
-                              canonicalize_url)
+                              canonicalize_url, add_scheme_if_missing)
 
 __doctests__ = ['scrapy.utils.url']
 
@@ -73,6 +73,22 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider))
         self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider))
 
+    def test_add_scheme_if_missing(self):
+        self.assertEqual(add_scheme_if_missing('http://www.example.com'),
+                                               'http://www.example.com')
+        self.assertEqual(add_scheme_if_missing('http://www.example.com/some/page.html'),
+                                               'http://www.example.com/some/page.html')
+        self.assertEqual(add_scheme_if_missing('http://example.com'),
+                                               'http://example.com')
+        self.assertEqual(add_scheme_if_missing('www.example.com'),
+                                               'http://www.example.com')
+        self.assertEqual(add_scheme_if_missing('example.com'),
+                                               'http://example.com')
+        self.assertEqual(add_scheme_if_missing('//example.com'),
+                                               'http://example.com')
+        self.assertEqual(add_scheme_if_missing('https://www.example.com'),
+                                               'https://www.example.com')
+
 
 class CanonicalizeUrlTest(unittest.TestCase):
 

From 8c629eee3e41a4d40f620e3a3f594391735b5a9f Mon Sep 17 00:00:00 2001
From: preetwinder <preetwinder14496@gmail.com>
Date: Fri, 18 Sep 2015 16:31:37 +0530
Subject: [PATCH 0575/2974] adds docstring, tests and correction

---
 scrapy/commands/shell.py |  4 ++--
 scrapy/utils/url.py      | 13 ++++++------
 tests/test_utils_url.py  | 44 ++++++++++++++++++++++++++++++++--------
 3 files changed, 44 insertions(+), 17 deletions(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index e94e339de..92ebbe605 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -9,7 +9,7 @@ from threading import Thread
 from scrapy.commands import ScrapyCommand
 from scrapy.shell import Shell
 from scrapy.http import Request
-from scrapy.utils.url import add_scheme_if_missing
+from scrapy.utils.url import add_http_if_no_scheme
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 
 
@@ -43,7 +43,7 @@ class Command(ScrapyCommand):
     def run(self, args, opts):
         url = args[0] if args else None
         if url:
-            url = add_scheme_if_missing(url)
+            url = add_http_if_no_scheme(url)
         spider_loader = self.crawler_process.spider_loader
 
         spidercls = DefaultSpider
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 94ec4de1b..c0934ddcf 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -111,10 +111,11 @@ def escape_ajax(url):
         return url
     return add_or_replace_parameter(defrag, '_escaped_fragment_', frag[1:])
 
-def add_scheme_if_missing(url):
+def add_http_if_no_scheme(url):
+    """Adds http as the default scheme if it is missing from the url"""
     parser = parse_url(url)
-    if not parser.scheme:
-        if not parser.netloc:
-            parser = parser._replace(netloc=parser.path, path='')
-        parser = parser._replace(scheme='http')
-    return parser.geturl() 
+    if url.startswith('//'):
+        url = 'http:' + url
+    elif not parser.scheme or not parser.netloc:
+        url = 'http://' + url
+    return url
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index fae4c988b..7ccf68c7a 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -4,7 +4,7 @@ import unittest
 import six
 from scrapy.spiders import Spider
 from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
-                              canonicalize_url, add_scheme_if_missing)
+                              canonicalize_url, add_http_if_no_scheme)
 
 __doctests__ = ['scrapy.utils.url']
 
@@ -73,21 +73,47 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider))
         self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider))
 
-    def test_add_scheme_if_missing(self):
-        self.assertEqual(add_scheme_if_missing('http://www.example.com'),
+    def test_add_http_if_no_scheme(self):
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
                                                'http://www.example.com')
-        self.assertEqual(add_scheme_if_missing('http://www.example.com/some/page.html'),
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'),
                                                'http://www.example.com/some/page.html')
-        self.assertEqual(add_scheme_if_missing('http://example.com'),
+        self.assertEqual(add_http_if_no_scheme('http://example.com'),
                                                'http://example.com')
-        self.assertEqual(add_scheme_if_missing('www.example.com'),
+        self.assertEqual(add_http_if_no_scheme('www.example.com'),
                                                'http://www.example.com')
-        self.assertEqual(add_scheme_if_missing('example.com'),
+        self.assertEqual(add_http_if_no_scheme('example.com'),
                                                'http://example.com')
-        self.assertEqual(add_scheme_if_missing('//example.com'),
+        self.assertEqual(add_http_if_no_scheme('//example.com'),
                                                'http://example.com')
-        self.assertEqual(add_scheme_if_missing('https://www.example.com'),
+        self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'),
+                                               'http://www.example.com/some/page.html')
+        self.assertEqual(add_http_if_no_scheme('www.example.com:80'),
+                                               'http://www.example.com:80')
+        self.assertEqual(add_http_if_no_scheme('www.example.com:80/some/page.html'),
+                                               'http://www.example.com:80/some/page.html')
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com:80/some/page.html'),
+                                               'http://www.example.com:80/some/page.html')
+        self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'),
+                                               'http://www.example.com/some/page#frag')
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'),
+                                               'http://www.example.com/some/page#frag')
+        self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
+                                               'http://www.example.com/do?a=1&b=2&c=3')
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
+                                               'http://www.example.com/do?a=1&b=2&c=3')
+        self.assertEqual(add_http_if_no_scheme('username:password@example.com/some/page.html'),
+                                               'http://username:password@example.com/some/page.html')
+        self.assertEqual(add_http_if_no_scheme('http://username:password@example.com/some/page.html'),
+                                               'http://username:password@example.com/some/page.html')
+        self.assertEqual(add_http_if_no_scheme('username:password@example.com:80/some/part?a=1&b=2&c=3#frag'),
+                                               'http://username:password@example.com:80/some/part?a=1&b=2&c=3#frag')
+        self.assertEqual(add_http_if_no_scheme('http://username:password@example.com:80/some/part?a=1&b=2&c=3#frag'),
+                                               'http://username:password@example.com:80/some/part?a=1&b=2&c=3#frag')
+        self.assertEqual(add_http_if_no_scheme('https://www.example.com'),
                                                'https://www.example.com')
+        self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'),
+                                               'ftp://www.example.com')
 
 
 class CanonicalizeUrlTest(unittest.TestCase):

From 9d96e767a1baf3b7737440eae0f1d2c5d433f798 Mon Sep 17 00:00:00 2001
From: preetwinder <preetwinder14496@gmail.com>
Date: Thu, 24 Sep 2015 17:31:30 +0000
Subject: [PATCH 0576/2974] Minor changes to add_http_if_no_scheme

---
 scrapy/utils/url.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index c0934ddcf..398407a64 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -111,11 +111,13 @@ def escape_ajax(url):
         return url
     return add_or_replace_parameter(defrag, '_escaped_fragment_', frag[1:])
 
+
 def add_http_if_no_scheme(url):
-    """Adds http as the default scheme if it is missing from the url"""
-    parser = parse_url(url)
+    """Add http as the default scheme if it is missing from the url."""
     if url.startswith('//'):
         url = 'http:' + url
-    elif not parser.scheme or not parser.netloc:
+        return url
+    parser = parse_url(url)
+    if not parser.scheme or not parser.netloc:
         url = 'http://' + url
     return url

From 47c8e2ba781e351868305f65b989d4b18f54279f Mon Sep 17 00:00:00 2001
From: preetwinder <preetwinder14496@gmail.com>
Date: Thu, 24 Sep 2015 17:57:25 +0000
Subject: [PATCH 0577/2974] Restructure tests for add_http_if_no_scheme
 function

---
 tests/test_utils_url.py | 149 +++++++++++++++++++++++++++++-----------
 1 file changed, 107 insertions(+), 42 deletions(-)

diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 7ccf68c7a..314ccd30f 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -73,48 +73,6 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider))
         self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider))
 
-    def test_add_http_if_no_scheme(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
-                                               'http://www.example.com')
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'),
-                                               'http://www.example.com/some/page.html')
-        self.assertEqual(add_http_if_no_scheme('http://example.com'),
-                                               'http://example.com')
-        self.assertEqual(add_http_if_no_scheme('www.example.com'),
-                                               'http://www.example.com')
-        self.assertEqual(add_http_if_no_scheme('example.com'),
-                                               'http://example.com')
-        self.assertEqual(add_http_if_no_scheme('//example.com'),
-                                               'http://example.com')
-        self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'),
-                                               'http://www.example.com/some/page.html')
-        self.assertEqual(add_http_if_no_scheme('www.example.com:80'),
-                                               'http://www.example.com:80')
-        self.assertEqual(add_http_if_no_scheme('www.example.com:80/some/page.html'),
-                                               'http://www.example.com:80/some/page.html')
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com:80/some/page.html'),
-                                               'http://www.example.com:80/some/page.html')
-        self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'),
-                                               'http://www.example.com/some/page#frag')
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'),
-                                               'http://www.example.com/some/page#frag')
-        self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
-                                               'http://www.example.com/do?a=1&b=2&c=3')
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
-                                               'http://www.example.com/do?a=1&b=2&c=3')
-        self.assertEqual(add_http_if_no_scheme('username:password@example.com/some/page.html'),
-                                               'http://username:password@example.com/some/page.html')
-        self.assertEqual(add_http_if_no_scheme('http://username:password@example.com/some/page.html'),
-                                               'http://username:password@example.com/some/page.html')
-        self.assertEqual(add_http_if_no_scheme('username:password@example.com:80/some/part?a=1&b=2&c=3#frag'),
-                                               'http://username:password@example.com:80/some/part?a=1&b=2&c=3#frag')
-        self.assertEqual(add_http_if_no_scheme('http://username:password@example.com:80/some/part?a=1&b=2&c=3#frag'),
-                                               'http://username:password@example.com:80/some/part?a=1&b=2&c=3#frag')
-        self.assertEqual(add_http_if_no_scheme('https://www.example.com'),
-                                               'https://www.example.com')
-        self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'),
-                                               'ftp://www.example.com')
-
 
 class CanonicalizeUrlTest(unittest.TestCase):
 
@@ -229,5 +187,112 @@ class CanonicalizeUrlTest(unittest.TestCase):
                          "http://foo.com/AC%2FDC/")
 
 
+class AddHttpIfNoScheme(unittest.TestCase):
+    
+    def test_add_scheme(self):
+        self.assertEqual(add_http_if_no_scheme('www.example.com'),
+                                               'http://www.example.com')
+
+    def test_without_subdomain(self):
+        self.assertEqual(add_http_if_no_scheme('example.com'),
+                                               'http://example.com')
+
+    def test_path(self):
+        self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'),
+                                               'http://www.example.com/some/page.html')
+
+    def test_port(self):
+        self.assertEqual(add_http_if_no_scheme('www.example.com:80'),
+                                               'http://www.example.com:80')
+
+    def test_fragment(self):
+        self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'),
+                                               'http://www.example.com/some/page#frag')
+
+    def test_query(self):
+        self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
+                                               'http://www.example.com/do?a=1&b=2&c=3')
+
+    def test_username_password(self):
+        self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'),
+                                               'http://username:password@www.example.com')
+    
+    def test_complete_url(self):
+        self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+
+    def test_preserve_http(self):
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
+                                               'http://www.example.com')
+
+    def test_preserve_http_without_subdomain(self):
+        self.assertEqual(add_http_if_no_scheme('http://example.com'),
+                                               'http://example.com')
+
+    def test_preserve_http_path(self):
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'),
+                                               'http://www.example.com/some/page.html')
+
+    def test_preserve_http_port(self):
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'),
+                                               'http://www.example.com:80')
+
+    def test_preserve_http_fragment(self):
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'),
+                                               'http://www.example.com/some/page#frag')
+
+    def test_preserve_http_query(self):
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
+                                               'http://www.example.com/do?a=1&b=2&c=3')
+
+    def test_preserve_http_username_password(self):
+        self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'),
+                                               'http://username:password@www.example.com')
+
+    def test_preserve_http_complete_url(self):
+        self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+
+    def test_protocol_relative(self):
+        self.assertEqual(add_http_if_no_scheme('//www.example.com'),
+                                               'http://www.example.com')
+
+    def test_protocol_relative_without_subdomain(self):
+        self.assertEqual(add_http_if_no_scheme('//example.com'),
+                                               'http://example.com')
+
+    def test_protocol_relative_path(self):
+        self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'),
+                                               'http://www.example.com/some/page.html')
+
+    def test_protocol_relative_port(self):
+        self.assertEqual(add_http_if_no_scheme('//www.example.com:80'),
+                                               'http://www.example.com:80')
+
+    def test_protocol_relative_fragment(self):
+        self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'),
+                                               'http://www.example.com/some/page#frag')
+
+    def test_protocol_relative_query(self):
+        self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
+                                               'http://www.example.com/do?a=1&b=2&c=3')
+
+    def test_protocol_relative_username_password(self):
+        self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'),
+                                               'http://username:password@www.example.com')
+
+    def test_protocol_relative_complete_url(self):
+        self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+
+    def test_preserve_https(self):
+        self.assertEqual(add_http_if_no_scheme('https://www.example.com'),
+                                               'https://www.example.com')
+
+    def test_preserve_ftp(self):
+        self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'),
+                                               'ftp://www.example.com')
+
+
 if __name__ == "__main__":
     unittest.main()

From 75cd056223a5a8da87a361aee42a541afcf27553 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Fri, 25 Sep 2015 16:39:21 -0300
Subject: [PATCH 0578/2974] disable log on version command. closes #1426

---
 scrapy/commands/version.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 81433a657..4bf085c9e 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -11,6 +11,8 @@ from scrapy.commands import ScrapyCommand
 
 class Command(ScrapyCommand):
 
+    default_settings = {'LOG_ENABLED': False}
+
     def syntax(self):
         return "[-v]"
 

From eaad10facfacb151d7a886bd3c67adc9c10a930a Mon Sep 17 00:00:00 2001
From: Marius Gedminas <marius@gedmin.as>
Date: Mon, 28 Sep 2015 12:36:34 +0300
Subject: [PATCH 0579/2974] Typo

---
 docs/topics/spiders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 6a8655fe7..5fd187e4e 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -375,7 +375,7 @@ Crawling rules
 
    ``follow`` is a boolean which specifies if links should be followed from each
    response extracted with this rule. If ``callback`` is None ``follow`` defaults
-   to ``True``, otherwise it default to ``False``.
+   to ``True``, otherwise it defaults to ``False``.
 
    ``process_links`` is a callable, or a string (in which case a method from the
    spider object with that name will be used) which will be called for each list

From 0620e7643351290654feea3bc8f45a4e77636c8f Mon Sep 17 00:00:00 2001
From: Marius Gedminas <marius@gedmin.as>
Date: Mon, 28 Sep 2015 12:29:12 +0300
Subject: [PATCH 0580/2974] Fix list formatting

---
 docs/topics/request-response.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index aa601f83a..0abec1f96 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -227,7 +227,7 @@ Those are:
 * :reqmeta:`handle_httpstatus_all`
 * ``dont_merge_cookies`` (see ``cookies`` parameter of :class:`Request` constructor)
 * :reqmeta:`cookiejar`
-  :reqmeta:`dont_cache`
+* :reqmeta:`dont_cache`
 * :reqmeta:`redirect_urls`
 * :reqmeta:`bindaddress`
 * :reqmeta:`dont_obey_robotstxt`

From 8379bea7ed17af093213f6bd884bac9b5b33bb90 Mon Sep 17 00:00:00 2001
From: smirecki <steven.mirecki@gmail.com>
Date: Fri, 2 Oct 2015 23:48:27 -0400
Subject: [PATCH 0581/2974] Typos corrections

I've made a few small corrections, some spelling changes and typo fixes.
I've tried to respect regional spelling differences and avoided proposing hyphenating compound words.

 Please enter the commit message for your changes. Lines starting
---
 docs/topics/deploy.rst  | 2 +-
 docs/topics/firebug.rst | 2 +-
 docs/topics/loaders.rst | 2 +-
 docs/topics/shell.rst   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst
index b7394d5ce..bc48ddce7 100644
--- a/docs/topics/deploy.rst
+++ b/docs/topics/deploy.rst
@@ -8,7 +8,7 @@ This section describes the different options you have for deploying your Scrapy
 spiders to run them on a regular basis. Running Scrapy spiders in your local
 machine is very convenient for the (early) development stage, but not so much
 when you need to execute long-running spiders or move spiders to run in
-production continously. This is where the solutions for deploying Scrapy
+production continuously. This is where the solutions for deploying Scrapy
 spiders come in.
 
 Popular choices for deploying Scrapy spiders are:
diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst
index 359c99450..b93ee4ff9 100644
--- a/docs/topics/firebug.rst
+++ b/docs/topics/firebug.rst
@@ -118,7 +118,7 @@ they work as we expect.
 
 As you can see, the page markup is not very descriptive: the elements don't
 contain ``id``, ``class`` or any attribute that clearly identifies them, so
-we''ll use the ranking bars as a reference point to select the data to extract
+we'll use the ranking bars as a reference point to select the data to extract
 when we construct our XPaths.
 
 After using FireBug, we can see that each link is inside a ``td`` tag, which is
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index b188a0b81..01de528f0 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -548,7 +548,7 @@ Example::
     # no need to call footer_loader.load_item()
     loader.load_item()
 
-You can nest loaders arbitrarilly and they work with either xpath or css selectors.
+You can nest loaders arbitrarily and they work with either xpath or css selectors.
 As a general guideline, use nested loaders when they make your code simpler but do
 not go overboard with nesting or your parser can become difficult to read.
 
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 327e22e90..2b118bfbd 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -71,7 +71,7 @@ Available Shortcuts
  * ``view(response)`` - open the given response in your local web browser, for
    inspection. This will add a `\<base\> tag`_ to the response body in order
    for external links (such as images and style sheets) to display properly.
-   Note, however,that this will create a temporary file in your computer,
+   Note, however, that this will create a temporary file in your computer,
    which won't be removed automatically.
 
 .. _<base> tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base

From d523c75f5e354f6e4c28c52517c2e0f392fcb96b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Mon, 5 Oct 2015 13:26:40 +0300
Subject: [PATCH 0582/2974] Relocate telnetconsole to extensions/

---
 docs/topics/extensions.rst                    |  6 +-
 docs/topics/settings.rst                      |  2 +-
 docs/topics/telnetconsole.rst                 |  2 +-
 scrapy/extensions/telnet.py                   | 88 ++++++++++++++++++
 scrapy/settings/default_settings.py           |  2 +-
 scrapy/telnet.py                              | 93 ++-----------------
 .../templates/project/module/settings.py.tmpl |  2 +-
 scrapy/utils/deprecate.py                     |  1 +
 8 files changed, 102 insertions(+), 94 deletions(-)
 create mode 100644 scrapy/extensions/telnet.py

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index fb5220e9d..598826b4c 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -36,7 +36,7 @@ by a string: the full Python path to the extension's class name. For example::
 
     EXTENSIONS = {
         'scrapy.extensions.corestats.CoreStats': 500,
-        'scrapy.telnet.TelnetConsole': 500,
+        'scrapy.extensions.telnet.TelnetConsole': 500,
     }
 
 
@@ -181,10 +181,10 @@ enabled (see :ref:`topics-stats`).
 Telnet console extension
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. module:: scrapy.telnet
+.. module:: scrapy.extensions.telnet
    :synopsis: Telnet console
 
-.. class:: scrapy.telnet.TelnetConsole
+.. class:: scrapy.extensions.telnet.TelnetConsole
 
 Provides a telnet console for getting into a Python interpreter inside the
 currently running Scrapy process, which can be very useful for debugging.
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 484065406..9f36ec34a 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -565,7 +565,7 @@ Default::
 
     {
         'scrapy.extensions.corestats.CoreStats': 0,
-        'scrapy.telnet.TelnetConsole': 0,
+        'scrapy.extensions.telnet.TelnetConsole': 0,
         'scrapy.extensions.memusage.MemoryUsage': 0,
         'scrapy.extensions.memdebug.MemoryDebugger': 0,
         'scrapy.extensions.closespider.CloseSpider': 0,
diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index b1ff88771..ce79c9f35 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -4,7 +4,7 @@
 Telnet Console
 ==============
 
-.. module:: scrapy.telnet
+.. module:: scrapy.extensions.telnet
    :synopsis: The Telnet Console
 
 Scrapy comes with a built-in telnet console for inspecting and controlling a
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
new file mode 100644
index 000000000..d9add1d97
--- /dev/null
+++ b/scrapy/extensions/telnet.py
@@ -0,0 +1,88 @@
+"""
+Scrapy Telnet Console extension
+
+See documentation in docs/topics/telnetconsole.rst
+"""
+
+import pprint
+import logging
+
+from twisted.internet import protocol
+try:
+    from twisted.conch import manhole, telnet
+    from twisted.conch.insults import insults
+    TWISTED_CONCH_AVAILABLE = True
+except ImportError:
+    TWISTED_CONCH_AVAILABLE = False
+
+from scrapy.exceptions import NotConfigured
+from scrapy import signals
+from scrapy.utils.trackref import print_live_refs
+from scrapy.utils.engine import print_engine_status
+from scrapy.utils.reactor import listen_tcp
+
+try:
+    import guppy
+    hpy = guppy.hpy()
+except ImportError:
+    hpy = None
+
+logger = logging.getLogger(__name__)
+
+# signal to update telnet variables
+# args: telnet_vars
+update_telnet_vars = object()
+
+
+class TelnetConsole(protocol.ServerFactory):
+
+    def __init__(self, crawler):
+        if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'):
+            raise NotConfigured
+        if not TWISTED_CONCH_AVAILABLE:
+            raise NotConfigured
+        self.crawler = crawler
+        self.noisy = False
+        self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]
+        self.host = crawler.settings['TELNETCONSOLE_HOST']
+        self.crawler.signals.connect(self.start_listening, signals.engine_started)
+        self.crawler.signals.connect(self.stop_listening, signals.engine_stopped)
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    def start_listening(self):
+        self.port = listen_tcp(self.portrange, self.host, self)
+        h = self.port.getHost()
+        logger.debug("Telnet console listening on %(host)s:%(port)d",
+                     {'host': h.host, 'port': h.port},
+                     extra={'crawler': self.crawler})
+
+    def stop_listening(self):
+        self.port.stopListening()
+
+    def protocol(self):
+        telnet_vars = self._get_telnet_vars()
+        return telnet.TelnetTransport(telnet.TelnetBootstrapProtocol,
+            insults.ServerProtocol, manhole.Manhole, telnet_vars)
+
+    def _get_telnet_vars(self):
+        # Note: if you add entries here also update topics/telnetconsole.rst
+        telnet_vars = {
+            'engine': self.crawler.engine,
+            'spider': self.crawler.engine.spider,
+            'slot': self.crawler.engine.slot,
+            'crawler': self.crawler,
+            'extensions': self.crawler.extensions,
+            'stats': self.crawler.stats,
+            'settings': self.crawler.settings,
+            'est': lambda: print_engine_status(self.crawler.engine),
+            'p': pprint.pprint,
+            'prefs': print_live_refs,
+            'hpy': hpy,
+            'help': "This is Scrapy telnet console. For more info see: " \
+                "http://doc.scrapy.org/en/latest/topics/telnetconsole.html",
+        }
+        self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars)
+        return telnet_vars
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 5d48ac1a7..8435b0354 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -120,7 +120,7 @@ EXTENSIONS = {}
 
 EXTENSIONS_BASE = {
     'scrapy.extensions.corestats.CoreStats': 0,
-    'scrapy.telnet.TelnetConsole': 0,
+    'scrapy.extensions.telnet.TelnetConsole': 0,
     'scrapy.extensions.memusage.MemoryUsage': 0,
     'scrapy.extensions.memdebug.MemoryDebugger': 0,
     'scrapy.extensions.closespider.CloseSpider': 0,
diff --git a/scrapy/telnet.py b/scrapy/telnet.py
index d9add1d97..70915f486 100644
--- a/scrapy/telnet.py
+++ b/scrapy/telnet.py
@@ -1,88 +1,7 @@
-"""
-Scrapy Telnet Console extension
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.telnet` is deprecated, "
+              "use `scrapy.extensions.telnet` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
 
-See documentation in docs/topics/telnetconsole.rst
-"""
-
-import pprint
-import logging
-
-from twisted.internet import protocol
-try:
-    from twisted.conch import manhole, telnet
-    from twisted.conch.insults import insults
-    TWISTED_CONCH_AVAILABLE = True
-except ImportError:
-    TWISTED_CONCH_AVAILABLE = False
-
-from scrapy.exceptions import NotConfigured
-from scrapy import signals
-from scrapy.utils.trackref import print_live_refs
-from scrapy.utils.engine import print_engine_status
-from scrapy.utils.reactor import listen_tcp
-
-try:
-    import guppy
-    hpy = guppy.hpy()
-except ImportError:
-    hpy = None
-
-logger = logging.getLogger(__name__)
-
-# signal to update telnet variables
-# args: telnet_vars
-update_telnet_vars = object()
-
-
-class TelnetConsole(protocol.ServerFactory):
-
-    def __init__(self, crawler):
-        if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'):
-            raise NotConfigured
-        if not TWISTED_CONCH_AVAILABLE:
-            raise NotConfigured
-        self.crawler = crawler
-        self.noisy = False
-        self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]
-        self.host = crawler.settings['TELNETCONSOLE_HOST']
-        self.crawler.signals.connect(self.start_listening, signals.engine_started)
-        self.crawler.signals.connect(self.stop_listening, signals.engine_stopped)
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler)
-
-    def start_listening(self):
-        self.port = listen_tcp(self.portrange, self.host, self)
-        h = self.port.getHost()
-        logger.debug("Telnet console listening on %(host)s:%(port)d",
-                     {'host': h.host, 'port': h.port},
-                     extra={'crawler': self.crawler})
-
-    def stop_listening(self):
-        self.port.stopListening()
-
-    def protocol(self):
-        telnet_vars = self._get_telnet_vars()
-        return telnet.TelnetTransport(telnet.TelnetBootstrapProtocol,
-            insults.ServerProtocol, manhole.Manhole, telnet_vars)
-
-    def _get_telnet_vars(self):
-        # Note: if you add entries here also update topics/telnetconsole.rst
-        telnet_vars = {
-            'engine': self.crawler.engine,
-            'spider': self.crawler.engine.spider,
-            'slot': self.crawler.engine.slot,
-            'crawler': self.crawler,
-            'extensions': self.crawler.extensions,
-            'stats': self.crawler.stats,
-            'settings': self.crawler.settings,
-            'est': lambda: print_engine_status(self.crawler.engine),
-            'p': pprint.pprint,
-            'prefs': print_live_refs,
-            'hpy': hpy,
-            'help': "This is Scrapy telnet console. For more info see: " \
-                "http://doc.scrapy.org/en/latest/topics/telnetconsole.html",
-        }
-        self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars)
-        return telnet_vars
+from scrapy.extensions.telnet import *
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 2e660d7cf..822812c9a 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -56,7 +56,7 @@ NEWSPIDER_MODULE = '$project_name.spiders'
 # Enable or disable extensions
 # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
 #EXTENSIONS = {
-#    'scrapy.telnet.TelnetConsole': None,
+#    'scrapy.extensions.telnet.TelnetConsole': None,
 #}
 
 # Configure item pipelines
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index 37e94fae9..0fc33e0c4 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -138,6 +138,7 @@ DEPRECATION_RULES = [
     ('scrapy.command.', 'scrapy.commands.'),
     ('scrapy.dupefilter.', 'scrapy.dupefilters.'),
     ('scrapy.linkextractor.', 'scrapy.linkextractors.'),
+    ('scrapy.telnet.', 'scrapy.extensions.telnet.'),
     ('scrapy.spider.', 'scrapy.spiders.'),
     ('scrapy.squeue.', 'scrapy.squeues.'),
     ('scrapy.statscol.', 'scrapy.statscollectors.'),

From 669be0a2b0a6781b1ac1b623eda3e37775b218cf Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 6 Oct 2015 17:24:52 +0500
Subject: [PATCH 0583/2974] TST don't use broken Pillow version in tests

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 8b4c9e875..eae7e8e47 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,7 +11,7 @@ deps =
     -rrequirements.txt
     # Extras
     boto
-    Pillow
+    Pillow != 3.0.0
     leveldb
     -rtests/requirements.txt
 commands =

From 4e66955411a546f9b9a621ce06a709e3b2fbc4fc Mon Sep 17 00:00:00 2001
From: Hoat Le <hoatlevan@gmail.com>
Date: Wed, 7 Oct 2015 15:43:02 +0700
Subject: [PATCH 0584/2974] fix ValueError: Invalid XPath:
 //div/[id="not-exists"]/text() on selectors.rst

>>> response.xpath('//div/[id="not-exists"]/text()').extract_first() is None
Traceback (most recent call last):
  File "<console>", line 1, in <module>
  File "/home/vagrant/.virtualenvs/scrapy/lib/python2.7/site-packages/scrapy/http/response/text.py", line 109, in xpath
    return self.selector.xpath(query)
  File "/home/vagrant/.virtualenvs/scrapy/lib/python2.7/site-packages/scrapy/selector/unified.py", line 100, in xpath
    raise ValueError(msg if six.PY3 else msg.encode("unicode_escape"))
ValueError: Invalid XPath: //div/[id="not-exists"]/text()
---
 docs/topics/selectors.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 688c2b715..9d19506d2 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -146,7 +146,7 @@ If you want to extract only first matched element, you can call the selector ``.
 
 It returns ``None`` if no element was found:
 
-    >>> response.xpath('//div/[id="not-exists"]/text()').extract_first() is None
+    >>> response.xpath('//div[@id="not-exists"]/text()').extract_first() is None
     True
 
 A default return value can be provided as an argument, to be used instead of ``None``:

From 2869cf8ddea73761cab148283ce07ab14baa57b4 Mon Sep 17 00:00:00 2001
From: hoatle <hoatle@teracy.com>
Date: Wed, 7 Oct 2015 15:51:12 +0700
Subject: [PATCH 0585/2974] fix another invalid xpath error

---
 docs/topics/selectors.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 9d19506d2..273cae0f8 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -151,7 +151,7 @@ It returns ``None`` if no element was found:
 
 A default return value can be provided as an argument, to be used instead of ``None``:
 
-    >>> sel.xpath('//div/[id="not-exists"]/text()').extract_first(default='not-found')
+    >>> sel.xpath('//div[@id="not-exists"]/text()').extract_first(default='not-found')
     'not-found'
 
 Notice that CSS selectors can select text or attribute nodes using CSS3

From d66efb13badcaf7939b1779d02b28db3f5ab65a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Wed, 7 Oct 2015 14:43:47 +0300
Subject: [PATCH 0586/2974] test xml nodename with dots

---
 tests/test_utils_iterators.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index f2780dcf1..590c53302 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -33,6 +33,19 @@ class XmliterTestCase(unittest.TestCase):
         self.assertEqual(attrs,
                          [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])])
 
+    def test_xmliter_unusual_node(self):
+        body = b"""<?xml version="1.0" encoding="UTF-8"?>
+            <root>
+                <matchme...></matchme...>
+                <matchmenot></matchmenot>
+            </root>
+        """
+        response = XmlResponse(url="http://example.com", body=body)
+        nodenames = [e.xpath('name()').extract()
+                 for e in self.xmliter(response, 'matchme...')]
+        self.assertEqual(nodenames, [['matchme...']])
+
+
     def test_xmliter_text(self):
         body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
 

From f56062d04549e9906b9f824a7dbfa063df2b8abf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Wed, 7 Oct 2015 14:47:23 +0300
Subject: [PATCH 0587/2974] escape nodename in xmliter regex

---
 scrapy/utils/iterators.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index b9f682431..c0d93f7a9 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -25,8 +25,10 @@ def xmliter(obj, nodename):
     - a unicode string
     - a string encoded as utf-8
     """
-    HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S)
-    HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S)
+    nodename_patt = re.escape(nodename)
+
+    HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename_patt, re.S)
+    HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename_patt, re.S)
     text = _body_or_str(obj)
 
     header_start = re.search(HEADER_START_RE, text)
@@ -34,7 +36,7 @@ def xmliter(obj, nodename):
     header_end = re_rsearch(HEADER_END_RE, text)
     header_end = text[header_end[1]:].strip() if header_end else ''
 
-    r = re.compile(r"<%s[\s>].*?</%s>" % (nodename, nodename), re.DOTALL)
+    r = re.compile(r"<{0}[\s>].*?</{0}>".format(nodename_patt), re.DOTALL)
     for match in r.finditer(text):
         nodetext = header_start + match.group() + header_end
         yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]

From 1b6d60c2514989402cd1c94d5c0ea8594f5aef16 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 6 Oct 2015 23:56:28 +0500
Subject: [PATCH 0588/2974] DOC fix docs after GH-1289.

---
 docs/topics/downloader-middleware.rst | 13 ++++++-------
 scrapy/downloadermiddlewares/retry.py |  6 ------
 2 files changed, 6 insertions(+), 13 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index b79da1b95..9122e5cb5 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -819,12 +819,6 @@ settings (see the settings documentation for more info):
 * :setting:`RETRY_TIMES`
 * :setting:`RETRY_HTTP_CODES`
 
-About HTTP errors to consider:
-
-You may want to remove 400 from :setting:`RETRY_HTTP_CODES`, if you stick to the
-HTTP protocol. It's included by default because it's a common code used
-to indicate server overload, which would be something we want to retry.
-
 .. reqmeta:: dont_retry
 
 If :attr:`Request.meta <scrapy.http.Request.meta>` has ``dont_retry`` key
@@ -858,11 +852,16 @@ Maximum number of times to retry, in addition to the first download.
 RETRY_HTTP_CODES
 ^^^^^^^^^^^^^^^^
 
-Default: ``[500, 502, 503, 504, 400, 408]``
+Default: ``[500, 502, 503, 504, 408]``
 
 Which HTTP response codes to retry. Other errors (DNS lookup issues,
 connections lost, etc) are always retried.
 
+In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because
+it is a common code used to indicate server overload. It is not included by
+default because HTTP specs say so.
+
+
 .. _topics-dlmw-robots:
 
 RobotsTxtMiddleware
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 714103ba0..803ed5fc0 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -10,12 +10,6 @@ Failed pages are collected on the scraping process and rescheduled at the end,
 once the spider has finished crawling all regular (non failed) pages. Once
 there is no more failed pages to retry this middleware sends a signal
 (retry_complete), so other extensions could connect to that signal.
-
-About HTTP errors to consider:
-
-- You may want to remove 400 from RETRY_HTTP_CODES, if you stick to the HTTP
-  protocol. It's included by default because it's a common code used to
-  indicate server overload, which would be something we want to retry
 """
 import logging
 

From 215905bdb65cb0b97938c4d2acd13c8337a586da Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 12 Oct 2015 17:59:42 +0500
Subject: [PATCH 0589/2974] PY3 port bench, startproject, genspider, list and
 runspider commands

---
 scrapy/commands/genspider.py |  5 ++++-
 scrapy/commands/parse.py     | 33 ++++++++++++++++-----------------
 scrapy/utils/template.py     | 10 ++++++----
 tests/py3-ignores.txt        |  2 --
 tests/test_commands.py       | 23 +++++++++++++----------
 5 files changed, 39 insertions(+), 34 deletions(-)

diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index f0d435379..58bdb9156 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -11,6 +11,7 @@ from scrapy.commands import ScrapyCommand
 from scrapy.utils.template import render_templatefile, string_camelcase
 from scrapy.exceptions import UsageError
 
+
 def sanitize_module_name(module_name):
     """Sanitize the given module name, by replacing dashes and points
     with underscores and prefixing it with a letter if it doesn't start
@@ -21,6 +22,7 @@ def sanitize_module_name(module_name):
         module_name = "a" + module_name
     return module_name
 
+
 class Command(ScrapyCommand):
 
     requires_project = True
@@ -52,7 +54,8 @@ class Command(ScrapyCommand):
         if opts.dump:
             template_file = self._find_template(opts.dump)
             if template_file:
-                print(open(template_file, 'r').read())
+                with open(template_file, "r") as f:
+                    print(f.read())
             return
         if len(args) != 2:
             raise UsageError()
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index f90f7cdbc..0185bcabd 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -32,33 +32,34 @@ class Command(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("--spider", dest="spider", default=None, \
+        parser.add_option("--spider", dest="spider", default=None,
             help="use this spider without looking for one")
-        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \
+        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
             help="set spider argument (may be repeated)")
-        parser.add_option("--pipelines", action="store_true", \
+        parser.add_option("--pipelines", action="store_true",
             help="process items through pipelines")
-        parser.add_option("--nolinks", dest="nolinks", action="store_true", \
+        parser.add_option("--nolinks", dest="nolinks", action="store_true",
             help="don't show links to follow (extracted requests)")
-        parser.add_option("--noitems", dest="noitems", action="store_true", \
+        parser.add_option("--noitems", dest="noitems", action="store_true",
             help="don't show scraped items")
-        parser.add_option("--nocolour", dest="nocolour", action="store_true", \
+        parser.add_option("--nocolour", dest="nocolour", action="store_true",
             help="avoid using pygments to colorize the output")
-        parser.add_option("-r", "--rules", dest="rules", action="store_true", \
+        parser.add_option("-r", "--rules", dest="rules", action="store_true",
             help="use CrawlSpider rules to discover the callback")
-        parser.add_option("-c", "--callback", dest="callback", \
+        parser.add_option("-c", "--callback", dest="callback",
             help="use this callback for parsing, instead looking for a callback")
-        parser.add_option("-d", "--depth", dest="depth", type="int", default=1, \
+        parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
             help="maximum depth for parsing requests [default: %default]")
-        parser.add_option("-v", "--verbose", dest="verbose", action="store_true", \
+        parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
             help="print each depth level one by one")
 
 
     @property
     def max_level(self):
-        levels = self.items.keys() + self.requests.keys()
-        if levels: return max(levels)
-        else: return 0
+        levels = list(self.items.keys()) + list(self.requests.keys())
+        if not levels:
+            return 0
+        return max(levels)
 
     def add_items(self, lvl, new_items):
         old_items = self.items.get(lvl, [])
@@ -79,7 +80,7 @@ class Command(ScrapyCommand):
 
     def print_requests(self, lvl=None, colour=True):
         if lvl is None:
-            levels = self.requests.keys()
+            levels = list(self.requests.keys())
             if levels:
                 requests = self.requests[max(levels)]
             else:
@@ -94,7 +95,7 @@ class Command(ScrapyCommand):
         colour = not opts.nocolour
 
         if opts.verbose:
-            for level in xrange(1, self.max_level+1):
+            for level in range(1, self.max_level+1):
                 print('\n>>> DEPTH LEVEL: %s <<<' % level)
                 if not opts.noitems:
                     self.print_items(level, colour)
@@ -107,7 +108,6 @@ class Command(ScrapyCommand):
             if not opts.nolinks:
                 self.print_requests(colour=colour)
 
-
     def run_callback(self, response, cb):
         items, requests = [], []
 
@@ -146,7 +146,6 @@ class Command(ScrapyCommand):
         _start_requests = lambda s: [self.prepare_request(s, request, opts)]
         self.spidercls.start_requests = _start_requests
 
-
     def start_parsing(self, url, opts):
         self.crawler_process.crawl(self.spidercls, **opts.spargs)
         self.pcrawler = list(self.crawler_process.crawlers)[0]
diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py
index 90aa8f705..1d7bd006c 100644
--- a/scrapy/utils/template.py
+++ b/scrapy/utils/template.py
@@ -4,18 +4,20 @@ import os
 import re
 import string
 
+
 def render_templatefile(path, **kwargs):
-    with open(path, 'rb') as file:
-        raw = file.read()
+    with open(path, 'rb') as fp:
+        raw = fp.read().decode('utf8')
 
     content = string.Template(raw).substitute(**kwargs)
 
     render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path
-    with open(render_path, 'wb') as file:
-        file.write(content)
+    with open(render_path, 'wb') as fp:
+        fp.write(content.encode('utf8'))
     if path.endswith('.tmpl'):
         os.remove(path)
 
+
 CAMELCASE_INVALID_CHARS = re.compile('[^a-zA-Z\d]')
 def string_camelcase(string):
     """ Convert a word  to its CamelCase version and remove invalid chars
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index e40b4c73c..55ed75c92 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,7 +1,6 @@
 tests/test_closespider.py
 tests/test_command_fetch.py
 tests/test_command_shell.py
-tests/test_commands.py
 tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
@@ -43,5 +42,4 @@ scrapy/downloadermiddlewares/httpproxy.py
 scrapy/downloadermiddlewares/cookies.py
 scrapy/extensions/statsmailer.py
 scrapy/extensions/memusage.py
-scrapy/commands/bench.py
 scrapy/mail.py
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 7c10faf0c..e0c0648ca 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -6,10 +6,12 @@ from time import sleep
 from os.path import exists, join, abspath
 from shutil import rmtree
 from tempfile import mkdtemp
+import six
 
 from twisted.trial import unittest
 from twisted.internet import defer
 
+from scrapy.utils.python import to_native_str
 from scrapy.utils.python import retry_on_eintr
 from scrapy.utils.test import get_testenv
 from scrapy.utils.testsite import SiteTest
@@ -94,11 +96,11 @@ class GenspiderCommandTest(CommandTest):
         args = ['--template=%s' % tplname] if tplname else []
         spname = 'test_spider'
         p = self.proc('genspider', spname, 'test.com', *args)
-        out = retry_on_eintr(p.stdout.read)
+        out = to_native_str(retry_on_eintr(p.stdout.read))
         self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out)
         self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
         p = self.proc('genspider', spname, 'test.com', *args)
-        out = retry_on_eintr(p.stdout.read)
+        out = to_native_str(retry_on_eintr(p.stdout.read))
         self.assertIn("Spider %r already exists in module" % spname, out)
 
     def test_template_basic(self):
@@ -146,7 +148,7 @@ class MySpider(scrapy.Spider):
         return []
 """)
         p = self.proc('runspider', fname)
-        log = p.stderr.read()
+        log = to_native_str(p.stderr.read())
         self.assertIn("DEBUG: It Works!", log)
         self.assertIn("INFO: Spider opened", log)
         self.assertIn("INFO: Closing spider (finished)", log)
@@ -161,12 +163,12 @@ class MySpider(scrapy.Spider):
 from scrapy.spiders import Spider
 """)
         p = self.proc('runspider', fname)
-        log = p.stderr.read()
+        log = to_native_str(p.stderr.read())
         self.assertIn("No spider found in file", log)
 
     def test_runspider_file_not_found(self):
         p = self.proc('runspider', 'some_non_existent_file')
-        log = p.stderr.read()
+        log = to_native_str(p.stderr.read())
         self.assertIn("File not found: some_non_existent_file", log)
 
     def test_runspider_unable_to_load(self):
@@ -176,11 +178,12 @@ from scrapy.spiders import Spider
         with open(fname, 'w') as f:
             f.write("")
         p = self.proc('runspider', fname)
-        log = p.stderr.read()
+        log = to_native_str(p.stderr.read())
         self.assertIn("Unable to load", log)
 
 
 class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
+    skip = not six.PY2
 
     command = 'parse'
 
@@ -226,7 +229,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
                                            '-a', 'test_arg=1',
                                            '-c', 'parse',
                                            self.url('/html')])
-        self.assertIn("DEBUG: It Works!", stderr)
+        self.assertIn("DEBUG: It Works!", to_native_str(stderr))
 
     @defer.inlineCallbacks
     def test_pipelines(self):
@@ -234,14 +237,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
                                            '--pipelines',
                                            '-c', 'parse',
                                            self.url('/html')])
-        self.assertIn("INFO: It Works!", stderr)
+        self.assertIn("INFO: It Works!", to_native_str(stderr))
 
     @defer.inlineCallbacks
     def test_parse_items(self):
         status, out, stderr = yield self.execute(
             ['--spider', self.spider_name, '-c', 'parse', self.url('/html')]
         )
-        self.assertIn("""[{}, {'foo': 'bar'}]""", out)
+        self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
 
 
 
@@ -250,5 +253,5 @@ class BenchCommandTest(CommandTest):
     def test_run(self):
         p = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
                 '-s', 'CLOSESPIDER_TIMEOUT=0.01')
-        log = p.stderr.read()
+        log = to_native_str(p.stderr.read())
         self.assertIn('INFO: Crawled', log)

From 09fc9b48b610b84cd0faee31482145c41523c5e1 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 12 Oct 2015 18:12:56 +0500
Subject: [PATCH 0590/2974] drop coveralls support

---
 .travis.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7b24e051e..e857abbd8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,11 +11,10 @@ env:
  - TOXENV=py33
  - TOXENV=docs
 install:
- - pip install -U tox twine wheel codecov coveralls
+ - pip install -U tox twine wheel codecov
 script: tox
 after_success:
   - codecov
-  - coveralls
 notifications:
   irc:
     use_notice: true

From cc76fb3dff13fd18327a850e09be62b6b79e8c28 Mon Sep 17 00:00:00 2001
From: Christian Pedersen <chripede@gmail.com>
Date: Wed, 14 Oct 2015 14:07:34 +0200
Subject: [PATCH 0591/2974] Version 1 now exists

---
 docs/versioning.rst | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/docs/versioning.rst b/docs/versioning.rst
index fffa68b33..6ae9ce981 100644
--- a/docs/versioning.rst
+++ b/docs/versioning.rst
@@ -12,8 +12,7 @@ Scrapy uses the `odd-numbered versions for development releases`_.
 There are 3 numbers in a Scrapy version: *A.B.C*
 
 * *A* is the major version. This will rarely change and will signify very
-  large changes. So far, only zero is available for *A* as Scrapy hasn't yet
-  reached 1.0.
+  large changes.
 * *B* is the release number. This will include many changes including features
   and things that possibly break backwards compatibility. Even Bs will be
   stable branches, and odd Bs will be development.
@@ -27,12 +26,10 @@ For example:
 API Stability
 =============
 
-API stability is one of Scrapy major goals for the *1.0* release, which doesn't
-have a due date scheduled yet.
+API stability was one of the major goals for the *1.0* release.
 
 Methods or functions that start with a single dash (``_``) are private and
-should never be relied as stable. Besides those, the plan is to stabilize and
-document the entire API, as we approach the 1.0 release. 
+should never be relied as stable.
 
 Also, keep in mind that stable doesn't mean complete: stable APIs could grow
 new methods or functionality but the existing methods should keep working the

From d577c4702dc903e8dd2ae86058d4c779cfb9a32b Mon Sep 17 00:00:00 2001
From: Valdir Stumm Jr <valdir@scrapinghub.com>
Date: Mon, 26 Oct 2015 00:00:20 -0200
Subject: [PATCH 0592/2974] fixed a typo in the documentation.

---
 docs/topics/media-pipeline.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 4ee4f1758..5ed6ce97d 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -7,7 +7,7 @@ Downloading and processing files and images
 .. currentmodule:: scrapy.pipelines.images
 
 Scrapy provides reusable :doc:`item pipelines </topics/item-pipeline>` for
-downloading fies attached to a particular item (for example, when you scrape
+downloading files attached to a particular item (for example, when you scrape
 products and also want to download their images locally). These pipelines share
 a bit of functionality and structure (we refer to them as media pipelines), but
 typically you'll either use the Files Pipeline or the Images Pipeline.

From a769a1ef784a4383bc2f740d3a74b1e6cc6aeff9 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Fri, 19 Jun 2015 15:01:24 +0200
Subject: [PATCH 0593/2974] Introduce BaseSettings with full dictionary
 interface

---
 docs/topics/api.rst                     |  75 ++++++++---
 scrapy/settings/__init__.py             |  89 +++++++++++--
 tests/test_cmdline/__init__.py          |  14 +++
 tests/test_cmdline/extensions.py        |   5 +
 tests/test_settings/__init__.py         | 159 ++++++++++++++++++------
 tests/test_settings/default_settings.py |   3 +
 6 files changed, 280 insertions(+), 65 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index f54341eb8..923bd80b0 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -140,26 +140,41 @@ Settings API
     For a detailed explanation on each settings sources, see:
     :ref:`topics-settings`.
 
+.. function:: get_settings_priority(priority)
+
+    Small helper function that looks up a given string priority in the
+    :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its
+    numerical value, or directly returns a given numerical priority.
+
 .. class:: Settings(values={}, priority='project')
 
     This object stores Scrapy settings for the configuration of internal
     components, and can be used for any further customization.
 
-    After instantiation of this class, the new object will have the global
-    default settings described on :ref:`topics-settings-ref` already
-    populated.
+    It is a direct subclass and supports all methods of
+    :class:`~scrapy.settings.BaseSettings`. Additionally, after instantiation
+    of this class, the new object will have the global default settings
+    described on :ref:`topics-settings-ref` already populated.
 
-    Additional values can be passed on initialization with the ``values``
-    argument, and they would take the ``priority`` level.  If the latter
+.. class:: BaseSettings(values={}, priority='project')
+
+    Instances of this class behave like dictionaries, but store priorities
+    along with their ``(key, value)`` pairs, and can be frozen (i.e. marked
+    immutable).
+
+    Key-value entries can be passed on initialization with the ``values``
+    argument, and they would take the ``priority`` level (unless ``values`` is
+    already an instance of :class:`~scrapy.settings.BaseSettings`, in which
+    case the existing priority levels will be kept).  If the ``priority``
     argument is a string, the priority name will be looked up in
-    :attr:`~scrapy.settings.SETTINGS_PRIORITIES`. Otherwise, a expecific
-    integer should be provided.
+    :attr:`~scrapy.settings.SETTINGS_PRIORITIES`. Otherwise, a specific integer
+    should be provided.
 
     Once the object is created, new settings can be loaded or updated with the
-    :meth:`~scrapy.settings.Settings.set` method, and can be accessed with the
-    square bracket notation of dictionaries, or with the
-    :meth:`~scrapy.settings.Settings.get` method of the instance and its value
-    conversion variants.  When requesting a stored key, the value with the
+    :meth:`~scrapy.settings.BaseSettings.set` method, and can be accessed with
+    the square bracket notation of dictionaries, or with the
+    :meth:`~scrapy.settings.BaseSettings.get` method of the instance and its
+    value conversion variants. When requesting a stored key, the value with the
     highest priority will be retrieved.
 
     .. method:: set(name, value, priority='project')
@@ -180,16 +195,23 @@ Settings API
            :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
        :type priority: string or int
 
-    .. method:: setdict(values, priority='project')
+    .. method:: update(values, priority='project')
 
        Store key/value pairs with a given priority.
 
        This is a helper function that calls
-       :meth:`~scrapy.settings.Settings.set` for every item of ``values``
+       :meth:`~scrapy.settings.BaseSettings.set` for every item of ``values``
        with the provided ``priority``.
 
+       If ``values`` is a string, it is assumed to be JSON-encoded and parsed
+       into a dict with ``json.loads()`` first. If it is a
+       :class:`~scrapy.settings.BaseSettings` instance, the per-key priorities
+       will be used and the ``priority`` parameter ignored. This allows
+       inserting/updating settings with different priorities with a single
+       command.
+
        :param values: the settings names and values
-       :type values: dict
+       :type values: dict or string or :class:`~scrapy.settings.BaseSettings`
 
        :param priority: the priority of the settings. Should be a key of
            :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
@@ -200,7 +222,7 @@ Settings API
        Store settings from a module with a given priority.
 
        This is a helper function that calls
-       :meth:`~scrapy.settings.Settings.set` for every globally declared
+       :meth:`~scrapy.settings.BaseSettings.set` for every globally declared
        uppercase variable of ``module`` with the provided ``priority``.
 
        :param module: the module or the path of the module
@@ -272,8 +294,12 @@ Settings API
     .. method:: getdict(name, default=None)
 
        Get a setting value as a dictionary. If the setting original type is a
-       dictionary, a copy of it will be returned. If it's a string it will
-       evaluated as a json dictionary.
+       dictionary, a copy of it will be returned. If it is a string it will be
+       evaluated as a JSON dictionary. In the case that it is a
+       :class:`~scrapy.settings.BaseSettings` instance itself, it will be
+       converted to a dictionary, containing all its current settings values
+       as they would be returned by :meth:`~scrapy.settings.BaseSettings.get`,
+       and losing all information about priority and mutability.
 
        :param name: the setting name
        :type name: string
@@ -305,6 +331,21 @@ Settings API
 
        Alias for a :meth:`~freeze` call in the object returned by :meth:`copy`
 
+    .. method:: getpriority(name)
+
+       Return the current numerical priority value of a setting, or ``None`` if
+       the given ``name`` does not exist.
+
+       :param name: the setting name
+       :type name: string
+
+    .. method:: maxpriority()
+
+       Return the numerical value of the highest priority present throughout
+       all settings, or the numerical value for ``default`` from
+       :attr:`~scrapy.settings.SETTINGS_PRIORITIES` if there are no settings
+       stored.
+
 .. _topics-api-spiderloader:
 
 SpiderLoader API
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index af0d0dff1..fa7fa3178 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -2,7 +2,7 @@ import six
 import json
 import copy
 import warnings
-from collections import MutableMapping
+from collections import Mapping, MutableMapping
 from importlib import import_module
 
 from scrapy.utils.deprecate import create_deprecated_class
@@ -19,6 +19,12 @@ SETTINGS_PRIORITIES = {
     'cmdline': 40,
 }
 
+def get_settings_priority(priority):
+    if isinstance(priority, six.string_types):
+        return SETTINGS_PRIORITIES[priority]
+    else:
+        return priority
+
 
 class SettingsAttribute(object):
 
@@ -45,21 +51,22 @@ class SettingsAttribute(object):
     __repr__ = __str__
 
 
-class Settings(object):
+class BaseSettings(MutableMapping):
 
     def __init__(self, values=None, priority='project'):
         self.frozen = False
         self.attributes = {}
-        self.setmodule(default_settings, priority='default')
-        if values is not None:
-            self.setdict(values, priority)
+        self.update(values, priority)
 
     def __getitem__(self, opt_name):
         value = None
-        if opt_name in self.attributes:
+        if opt_name in self:
             value = self.attributes[opt_name].value
         return value
 
+    def __contains__(self, name):
+        return name in self.attributes
+
     def get(self, name, default=None):
         return self[name] if self[name] is not None else default
 
@@ -88,19 +95,34 @@ class Settings(object):
             value = json.loads(value)
         return dict(value)
 
+    def getpriority(self, name):
+        prio = None
+        if name in self:
+            prio = self.attributes[name].priority
+        return prio
+
+    def maxpriority(self):
+        if len(self) > 0:
+            return max(self.getpriority(name) for name in self)
+        else:
+            return get_settings_priority('default')
+
+    def __setitem__(self, name, value):
+        self.set(name, value)
+
     def set(self, name, value, priority='project'):
         self._assert_mutability()
-        if isinstance(priority, six.string_types):
-            priority = SETTINGS_PRIORITIES[priority]
-        if name not in self.attributes:
-            self.attributes[name] = SettingsAttribute(value, priority)
+        priority = get_settings_priority(priority)
+        if name not in self:
+            if isinstance(value, SettingsAttribute):
+                self.attributes[name] = value
+            else:
+                self.attributes[name] = SettingsAttribute(value, priority)
         else:
             self.attributes[name].set(value, priority)
 
     def setdict(self, values, priority='project'):
-        self._assert_mutability()
-        for name, value in six.iteritems(values):
-            self.set(name, value, priority)
+        self.update(values, priority)
 
     def setmodule(self, module, priority='project'):
         self._assert_mutability()
@@ -110,6 +132,28 @@ class Settings(object):
             if key.isupper():
                 self.set(key, getattr(module, key), priority)
 
+    def update(self, values, priority='project'):
+        self._assert_mutability()
+        if isinstance(values, six.string_types):
+            values = json.loads(values)
+        if values is not None:
+            if isinstance(values, BaseSettings):
+                for name, value in six.iteritems(values):
+                    self.set(name, value, values.getpriority(name))
+            else:
+                for name, value in six.iteritems(values):
+                    self.set(name, value, priority)
+
+    def delete(self, name, priority='project'):
+        self._assert_mutability()
+        priority = get_settings_priority(priority)
+        if priority >= self.getpriority(name):
+            del self.attributes[name]
+
+    def __delitem__(self, name):
+        self._assert_mutability()
+        del self.attributes[name]
+
     def _assert_mutability(self):
         if self.frozen:
             raise TypeError("Trying to modify an immutable Settings object")
@@ -125,6 +169,17 @@ class Settings(object):
         copy.freeze()
         return copy
 
+    def __iter__(self):
+        return iter(self.attributes)
+
+    def __len__(self):
+        return len(self.attributes)
+
+    def __str__(self):
+        return str(self.attributes)
+
+    __repr__ = __str__
+
     @property
     def overrides(self):
         warnings.warn("`Settings.overrides` attribute is deprecated and won't "
@@ -174,6 +229,14 @@ class _DictProxy(MutableMapping):
         return iter(self.o)
 
 
+class Settings(BaseSettings):
+
+    def __init__(self, values=None, priority='project'):
+        super(Settings, self).__init__()
+        self.setmodule(default_settings, 'default')
+        self.update(values, priority)
+
+
 class CrawlerSettings(Settings):
 
     def __init__(self, settings_module=None, **kw):
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 1e2905e95..5192fb0fa 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -1,4 +1,5 @@
 import os
+import json
 import sys
 import shutil
 import pstats
@@ -54,3 +55,16 @@ class CmdlineTest(unittest.TestCase):
             self.assertIn('tottime', stats)
         finally:
             shutil.rmtree(path)
+
+    def test_override_dict_settings(self):
+        settingsstr = self._execute('settings', '--get', 'EXTENSIONS', '-s',
+                                    ('EXTENSIONS={"tests.test_cmdline.extensions.TestExtension": '
+                                     '100, "tests.test_cmdline.extensions.DummyExtension": 200}'))
+        # XXX: There's gotta be a smarter way to do this...
+        self.assertNotIn("...", settingsstr)
+        for char in ("'", "<", ">", 'u"'):
+            settingsstr = settingsstr.replace(char, '"')
+        settingsdict = json.loads(settingsstr)
+        self.assertIn('tests.test_cmdline.extensions.DummyExtension', settingsdict)
+        self.assertIn('value=200', settingsdict['tests.test_cmdline.extensions.DummyExtension'])
+        self.assertIn('value=100', settingsdict['tests.test_cmdline.extensions.TestExtension'])
diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py
index 4d347966a..72867eb56 100644
--- a/tests/test_cmdline/extensions.py
+++ b/tests/test_cmdline/extensions.py
@@ -8,3 +8,8 @@ class TestExtension(object):
     @classmethod
     def from_crawler(cls, crawler):
         return cls(crawler.settings)
+
+
+class DummyExtension(object):
+    pass
+
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 54b834aa0..a473f3c3f 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -2,7 +2,8 @@ import six
 import unittest
 import warnings
 
-from scrapy.settings import Settings, SettingsAttribute, CrawlerSettings
+from scrapy.settings import (BaseSettings, Settings, SettingsAttribute,
+                             CrawlerSettings)
 from tests import mock
 from . import default_settings
 
@@ -33,35 +34,16 @@ class SettingsTest(unittest.TestCase):
     if six.PY3:
         assertItemsEqual = unittest.TestCase.assertCountEqual
 
+
+class BaseSettingsTest(unittest.TestCase):
+
+    if six.PY3:
+        assertItemsEqual = unittest.TestCase.assertCountEqual
+
     def setUp(self):
-        self.settings = Settings()
-
-    @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {'default': 10})
-    @mock.patch('scrapy.settings.default_settings', default_settings)
-    def test_initial_defaults(self):
-        settings = Settings()
-        self.assertEqual(len(settings.attributes), 1)
-        self.assertIn('TEST_DEFAULT', settings.attributes)
-
-        attr = settings.attributes['TEST_DEFAULT']
-        self.assertIsInstance(attr, SettingsAttribute)
-        self.assertEqual(attr.value, 'defvalue')
-        self.assertEqual(attr.priority, 10)
-
-    @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {})
-    @mock.patch('scrapy.settings.default_settings', {})
-    def test_initial_values(self):
-        settings = Settings({'TEST_OPTION': 'value'}, 10)
-        self.assertEqual(len(settings.attributes), 1)
-        self.assertIn('TEST_OPTION', settings.attributes)
-
-        attr = settings.attributes['TEST_OPTION']
-        self.assertIsInstance(attr, SettingsAttribute)
-        self.assertEqual(attr.value, 'value')
-        self.assertEqual(attr.priority, 10)
+        self.settings = BaseSettings()
 
     def test_set_new_attribute(self):
-        self.settings.attributes = {}
         self.settings.set('TEST_OPTION', 'value', 0)
         self.assertIn('TEST_OPTION', self.settings.attributes)
 
@@ -70,6 +52,12 @@ class SettingsTest(unittest.TestCase):
         self.assertEqual(attr.value, 'value')
         self.assertEqual(attr.priority, 0)
 
+    def test_set_settingsattribute(self):
+        myattr = SettingsAttribute(0, 30) # Note priority 30
+        self.settings.set('TEST_ATTR', myattr, 10)
+        self.assertEqual(self.settings.get('TEST_ATTR'), 0)
+        self.assertEqual(self.settings.getpriority('TEST_ATTR'), 30)
+
     def test_set_instance_identity_on_update(self):
         attr = SettingsAttribute('value', 0)
         self.settings.attributes = {'TEST_OPTION': attr}
@@ -79,13 +67,11 @@ class SettingsTest(unittest.TestCase):
         self.assertIs(attr, self.settings.attributes['TEST_OPTION'])
 
     def test_set_calls_settings_attributes_methods_on_update(self):
-        with mock.patch.object(SettingsAttribute, '__setattr__') as mock_setattr, \
-                mock.patch.object(SettingsAttribute, 'set') as mock_set:
+        attr = SettingsAttribute('value', 10)
+        with mock.patch.object(attr, '__setattr__') as mock_setattr, \
+                mock.patch.object(attr, 'set') as mock_set:
 
-            attr = SettingsAttribute('value', 10)
             self.settings.attributes = {'TEST_OPTION': attr}
-            mock_set.reset_mock()
-            mock_setattr.reset_mock()
 
             for priority in (0, 10, 20):
                 self.settings.set('TEST_OPTION', 'othervalue', priority)
@@ -94,6 +80,19 @@ class SettingsTest(unittest.TestCase):
                 mock_set.reset_mock()
                 mock_setattr.reset_mock()
 
+    def test_setitem(self):
+        settings = BaseSettings()
+        settings.set('key', 'a', 'default')
+        settings['key'] = 'b'
+        self.assertEqual(settings['key'], 'b')
+        self.assertEqual(settings.getpriority('key'), 20)
+        settings['key'] = 'c'
+        self.assertEqual(settings['key'], 'c')
+        settings['key2'] = 'x'
+        self.assertIn('key2', settings)
+        self.assertEqual(settings['key2'], 'x')
+        self.assertEqual(settings.getpriority('key2'), 20)
+
     def test_setdict_alias(self):
         with mock.patch.object(self.settings, 'set') as mock_set:
             self.settings.setdict({'TEST_1': 'value1', 'TEST_2': 'value2'}, 10)
@@ -118,7 +117,8 @@ class SettingsTest(unittest.TestCase):
     def test_setmodule_alias(self):
         with mock.patch.object(self.settings, 'set') as mock_set:
             self.settings.setmodule(default_settings, 10)
-            mock_set.assert_called_with('TEST_DEFAULT', 'defvalue', 10)
+            mock_set.assert_any_call('TEST_DEFAULT', 'defvalue', 10)
+            mock_set.assert_any_call('TEST_DICT', {'key': 'val'}, 10)
 
     def test_setmodule_by_path(self):
         self.settings.attributes = {}
@@ -132,11 +132,55 @@ class SettingsTest(unittest.TestCase):
         self.assertItemsEqual(six.iterkeys(self.settings.attributes),
                               six.iterkeys(ctrl_attributes))
 
-        for attr, ctrl_attr in zip(six.itervalues(self.settings.attributes),
-                                   six.itervalues(ctrl_attributes)):
+        for key in six.iterkeys(ctrl_attributes):
+            attr = self.settings.attributes[key]
+            ctrl_attr = ctrl_attributes[key]
             self.assertEqual(attr.value, ctrl_attr.value)
             self.assertEqual(attr.priority, ctrl_attr.priority)
 
+    def test_update(self):
+        settings = BaseSettings({'key_lowprio': 0}, priority=0)
+        settings.set('key_highprio', 10, priority=50)
+        custom_settings = BaseSettings({'key_lowprio': 1, 'key_highprio': 11}, priority=30)
+        custom_settings.set('newkey_one', None, priority=50)
+        custom_dict = {'key_lowprio': 2, 'key_highprio': 12, 'newkey_two': None}
+
+        settings.update(custom_dict, priority=20)
+        self.assertEqual(settings['key_lowprio'], 2)
+        self.assertEqual(settings.getpriority('key_lowprio'), 20)
+        self.assertEqual(settings['key_highprio'], 10)
+        self.assertIn('newkey_two', settings)
+        self.assertEqual(settings.getpriority('newkey_two'), 20)
+
+        settings.update(custom_settings)
+        self.assertEqual(settings['key_lowprio'], 1)
+        self.assertEqual(settings.getpriority('key_lowprio'), 30)
+        self.assertEqual(settings['key_highprio'], 10)
+        self.assertIn('newkey_one', settings)
+        self.assertEqual(settings.getpriority('newkey_one'), 50)
+
+        settings.update({'key_lowprio': 3}, priority=20)
+        self.assertEqual(settings['key_lowprio'], 1)
+
+    def test_update_jsonstring(self):
+        settings = BaseSettings({'number': 0, 'dict': BaseSettings({'key': 'val'})})
+        settings.update('{"number": 1, "newnumber": 2}')
+        self.assertEqual(settings['number'], 1)
+        self.assertEqual(settings['newnumber'], 2)
+        settings.set("dict", '{"key": "newval", "newkey": "newval2"}')
+        self.assertEqual(settings['dict']['key'], "newval")
+        self.assertEqual(settings['dict']['newkey'], "newval2")
+
+    def test_delete(self):
+        settings = BaseSettings({'key': None})
+        settings.set('key_highprio', None, priority=50)
+        settings.delete('key')
+        settings.delete('key_highprio')
+        self.assertNotIn('key', settings)
+        self.assertIn('key_highprio', settings)
+        del settings['key_highprio']
+        self.assertNotIn('key_highprio', settings)
+
     def test_get(self):
         test_configuration = {
             'TEST_ENABLED1': '1',
@@ -190,6 +234,18 @@ class SettingsTest(unittest.TestCase):
         self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5})
         self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
 
+    def test_getpriority(self):
+        settings = BaseSettings({'key': 'value'}, priority=99)
+        self.assertEqual(settings.getpriority('key'), 99)
+        self.assertEqual(settings.getpriority('nonexistentkey'), None)
+
+    def test_maxpriority(self):
+        # Empty settings should return 'default'
+        self.assertEqual(self.settings.maxpriority(), 0)
+        self.settings.set('A', 0, 10)
+        self.settings.set('B', 0, 30)
+        self.assertEqual(self.settings.maxpriority(), 30)
+
     def test_copy(self):
         values = {
             'TEST_BOOL': True,
@@ -254,6 +310,39 @@ class SettingsTest(unittest.TestCase):
             self.assertIn('BAR', self.settings.defaults)
 
 
+class SettingsTest(unittest.TestCase):
+
+    if six.PY3:
+        assertItemsEqual = unittest.TestCase.assertCountEqual
+
+    def setUp(self):
+        self.settings = Settings()
+
+    @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {'default': 10})
+    @mock.patch('scrapy.settings.default_settings', default_settings)
+    def test_initial_defaults(self):
+        settings = Settings()
+        self.assertEqual(len(settings.attributes), 2)
+        self.assertIn('TEST_DEFAULT', settings.attributes)
+
+        attr = settings.attributes['TEST_DEFAULT']
+        self.assertIsInstance(attr, SettingsAttribute)
+        self.assertEqual(attr.value, 'defvalue')
+        self.assertEqual(attr.priority, 10)
+
+    @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {})
+    @mock.patch('scrapy.settings.default_settings', {})
+    def test_initial_values(self):
+        settings = Settings({'TEST_OPTION': 'value'}, 10)
+        self.assertEqual(len(settings.attributes), 1)
+        self.assertIn('TEST_OPTION', settings.attributes)
+
+        attr = settings.attributes['TEST_OPTION']
+        self.assertIsInstance(attr, SettingsAttribute)
+        self.assertEqual(attr.value, 'value')
+        self.assertEqual(attr.priority, 10)
+
+
 class CrawlerSettingsTest(unittest.TestCase):
 
     def test_deprecated_crawlersettings(self):
diff --git a/tests/test_settings/default_settings.py b/tests/test_settings/default_settings.py
index 23005d4c6..c24b5a9b9 100644
--- a/tests/test_settings/default_settings.py
+++ b/tests/test_settings/default_settings.py
@@ -1,2 +1,5 @@
 
 TEST_DEFAULT = 'defvalue'
+
+TEST_DICT = {'key': 'val'}
+

From 26586ef5a605c6f6a23143b22812779857cd4e3e Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Fri, 19 Jun 2015 15:09:36 +0200
Subject: [PATCH 0594/2974] Deprecate _BASE settings, unify _BASE
 backwards-compatibility

---
 docs/topics/downloader-middleware.rst         |  22 ++-
 docs/topics/extensions.rst                    |  20 ++-
 docs/topics/feed-exports.rst                  |  41 +++--
 docs/topics/settings.rst                      | 142 +++++++-----------
 docs/topics/spider-middleware.rst             |  22 ++-
 scrapy/commands/check.py                      |   5 +-
 scrapy/commands/crawl.py                      |   8 +-
 scrapy/commands/runspider.py                  |   8 +-
 scrapy/core/downloader/handlers/__init__.py   |   8 +-
 scrapy/core/downloader/middleware.py          |   3 +-
 scrapy/core/spidermw.py                       |   3 +-
 .../downloadermiddlewares/defaultheaders.py   |   5 +-
 scrapy/extension.py                           |   3 +-
 scrapy/extensions/feedexport.py               |   4 +-
 scrapy/pipelines/__init__.py                  |  10 +-
 scrapy/settings/__init__.py                   |  41 ++++-
 scrapy/settings/default_settings.py           |  25 +--
 scrapy/utils/conf.py                          |  43 ++++--
 tests/test_settings/__init__.py               |  49 +++++-
 tests/test_utils_conf.py                      |  43 ++++--
 20 files changed, 267 insertions(+), 238 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 9122e5cb5..08d8f3edf 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -23,22 +23,20 @@ Here's an example::
         'myproject.middlewares.CustomDownloaderMiddleware': 543,
     }
 
-The :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the
-:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant to
-be overridden) and then sorted by order to get the final sorted list of enabled
-middlewares: the first middleware is the one closer to the engine and the last
-is the one closer to the downloader.
+The specified :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the
+default one (i.e. it does not overwrite it) and then sorted by order to get the
+final sorted list of enabled middlewares: the first middleware is the one
+closer to the engine and the last is the one closer to the downloader.
 
-To decide which order to assign to your middleware see the
-:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting and pick a value according to
+To decide which order to assign to your middleware see the default
+:setting:`DOWNLOADER_MIDDLEWARES` setting and pick a value according to
 where you want to insert the middleware. The order does matter because each
 middleware performs a different action and your middleware could depend on some
 previous (or subsequent) middleware being applied.
 
-If you want to disable a built-in middleware (the ones defined in
-:setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it
-in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign `None`
-as its value.  For example, if you want to disable the user-agent middleware::
+If you want to disable a built-in middleware you must define it in your
+project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None`` as its
+value. For example, if you want to disable the user-agent middleware::
 
     DOWNLOADER_MIDDLEWARES = {
         'myproject.middlewares.CustomDownloaderMiddleware': 543,
@@ -164,7 +162,7 @@ middleware, see the :ref:`downloader middleware usage guide
 <topics-downloader-middleware>`.
 
 For a list of the components enabled by default (and their orders) see the
-:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting.
+:setting:`DOWNLOADER_MIDDLEWARES` setting.
 
 .. _cookies-mw:
 
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index fb5220e9d..a71b8bcee 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -42,17 +42,15 @@ by a string: the full Python path to the extension's class name. For example::
 
 As you can see, the :setting:`EXTENSIONS` setting is a dict where the keys are
 the extension paths, and their values are the orders, which define the
-extension *loading* order. Extensions orders are not as important as middleware
-orders though, and they are typically irrelevant, ie. it doesn't matter in
-which order the extensions are loaded because they don't depend on each other
-[1].
+extension *loading* order. The specified :setting:`EXTENSIONS` setting is merged
+with the default one (i.e. it does not overwrite it) and then sorted by order
+to get the final sorted list of enabled extensions.
 
-However, this feature can be exploited if you need to add an extension which
-depends on other extensions already loaded.
-
-[1] This is is why the :setting:`EXTENSIONS_BASE` setting in Scrapy (which
-contains all built-in extensions enabled by default) defines all the extensions
-with the same order (``500``).
+As extensions typically do not depend on each other, their loading order is
+irrelevant in most cases. This is why the default :setting:`EXTENSIONS` setting
+defines all extensions with the same order (``500``). However, this feature can
+be exploited if you need to add an extension which depends on other extensions
+already loaded.
 
 Available, enabled and disabled extensions
 ==========================================
@@ -65,7 +63,7 @@ Disabling an extension
 ======================
 
 In order to disable an extension that comes enabled by default (ie. those
-included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to
+included in the default :setting:`EXTENSIONS` setting) you must set its order to
 ``None``. For example::
 
     EXTENSIONS = {
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index d9444e34a..d8b8da166 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -265,16 +265,6 @@ Whether to export empty feeds (ie. feeds with no items).
 FEED_STORAGES
 -------------
 
-Default:: ``{}``
-
-A dict containing additional feed storage backends supported by your project.
-The keys are URI schemes and the values are paths to storage classes.
-
-.. setting:: FEED_STORAGES_BASE
-
-FEED_STORAGES_BASE
-------------------
-
 Default::
 
     {
@@ -285,36 +275,39 @@ Default::
         'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
     }
 
-A dict containing the built-in feed storage backends supported by Scrapy.
+A dict containing all feed storage backends supported by your project. The keys
+are URI schemes and the values are paths to storage classes.
+
+When you set :setting:`FEED_STORAGES` manually, e.g. in your project's settings
+module, it will be merged with the default, not overwrite it. If you want to
+disable any of the default feed storage backends, you must assign ``None`` as
+their value.
 
 .. setting:: FEED_EXPORTERS
 
 FEED_EXPORTERS
 --------------
 
-Default:: ``{}``
-
-A dict containing additional exporters supported by your project. The keys are
-URI schemes and the values are paths to :ref:`Item exporter <topics-exporters>`
-classes.
-
-.. setting:: FEED_EXPORTERS_BASE
-
-FEED_EXPORTERS_BASE
--------------------
-
 Default::
 
-    FEED_EXPORTERS_BASE = {
+    {
         'json': 'scrapy.exporters.JsonItemExporter',
         'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
+        'jl': 'scrapy.exporters.JsonLinesItemExporter',
         'csv': 'scrapy.exporters.CsvItemExporter',
         'xml': 'scrapy.exporters.XmlItemExporter',
         'marshal': 'scrapy.exporters.MarshalItemExporter',
+        'pickle': 'scrapy.exporters.PickleItemExporter',
     }
 
-A dict containing the built-in feed exporters supported by Scrapy.
+A dict containing all feed exporters supported by your project. The keys are
+URI schemes and the values are paths to :ref:`Item exporter <topics-exporters>`
+classes.
 
+When you set :setting:`FEED_EXPORTERS` manually, e.g. in your project's settings
+module, it will be merged with the default, not overwrite it. If you want to
+disable any of the default feed exporters, you must assign ``None`` as their
+value.
 
 .. _URI: http://en.wikipedia.org/wiki/Uniform_Resource_Identifier
 .. _Amazon S3: http://aws.amazon.com/s3/
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 484065406..642f4eb84 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -269,6 +269,11 @@ Default::
 The default headers used for Scrapy HTTP Requests. They're populated in the
 :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`.
 
+When you set :setting:`DEFAULT_REQUEST_HEADERS` manually, e.g. in your
+project's settings module, it will be merged with the default, not overwrite it.
+If you want to disable any of the default request headers (and not replace them)
+you must assign ``None`` as their value.
+
 .. setting:: DEPTH_LIMIT
 
 DEPTH_LIMIT
@@ -350,16 +355,6 @@ The downloader to use for crawling.
 DOWNLOADER_MIDDLEWARES
 ----------------------
 
-Default:: ``{}``
-
-A dict containing the downloader middlewares enabled in your project, and their
-orders. For more info see :ref:`topics-downloader-middleware-setting`.
-
-.. setting:: DOWNLOADER_MIDDLEWARES_BASE
-
-DOWNLOADER_MIDDLEWARES_BASE
----------------------------
-
 Default::
 
     {
@@ -369,6 +364,7 @@ Default::
         'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400,
         'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
         'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550,
+        'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
         'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
         'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
         'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
@@ -379,10 +375,16 @@ Default::
         'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
     }
 
-A dict containing the downloader middlewares enabled by default in Scrapy. You
-should never modify this setting in your project, modify
-:setting:`DOWNLOADER_MIDDLEWARES` instead.  For more info see
-:ref:`topics-downloader-middleware-setting`.
+A dict containing the downloader middlewares enabled in your project, and their
+orders. Low orders are closer to the engine, high orders are closer to the
+downloader.
+
+When you set :setting:`DOWNLOADER_MIDDLEWARES` manually, e.g. in your project's
+settings module, it will be merged with the default, not overwrite it. If you
+want to disable any of the default downloader middlewares you must assign
+``None`` as their value.
+
+For more info see :ref:`topics-downloader-middleware-setting`.
 
 .. setting:: DOWNLOADER_STATS
 
@@ -423,33 +425,23 @@ spider attribute.
 DOWNLOAD_HANDLERS
 -----------------
 
-Default: ``{}``
-
-A dict containing the request downloader handlers enabled in your project.
-See `DOWNLOAD_HANDLERS_BASE` for example format.
-
-.. setting:: DOWNLOAD_HANDLERS_BASE
-
-DOWNLOAD_HANDLERS_BASE
-----------------------
-
 Default::
 
     {
         'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
-        'http': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler',
-        'https': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler',
+        'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
+        'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
         's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',
+        'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler',
     }
 
-A dict containing the request download handlers enabled by default in Scrapy.
-You should never modify this setting in your project, modify
-:setting:`DOWNLOAD_HANDLERS` instead.
 
-If you want to disable any of the above download handlers you must define them
-in your project's :setting:`DOWNLOAD_HANDLERS` setting and assign `None`
-as their value.  For example, if you want to disable the file download
-handler::
+A dict containing the request downloader handlers enabled in your project.
+
+When you set :setting:`DOWNLOAD_HANDLERS` manually, e.g. in your project's
+settings module, it will be merged with the default, not overwrite it. If you
+want to disable any of the default download handlers you must assign ``None``
+as their value. For example, if you want to disable the file download handler::
 
     DOWNLOAD_HANDLERS = {
         'file': None,
@@ -552,15 +544,6 @@ to ``vi`` (on Unix systems) or the IDLE editor (on Windows).
 EXTENSIONS
 ----------
 
-Default:: ``{}``
-
-A dict containing the extensions enabled in your project, and their orders.
-
-.. setting:: EXTENSIONS_BASE
-
-EXTENSIONS_BASE
----------------
-
 Default::
 
     {
@@ -575,13 +558,19 @@ Default::
         'scrapy.extensions.throttle.AutoThrottle': 0,
     }
 
-The list of available extensions. Keep in mind that some of them need to
-be enabled through a setting. By default, this setting contains all stable
-built-in extensions.
+A dict containing the extensions enabled in your project, and their orders. By
+default, this setting contains all stable built-in extensions. Keep in mind that
+some of them need to be enabled through a setting.
+
+When you set :setting:`EXTENSIONS` manually, e.g. in your project's settings
+module, it will be merged with the default, not overwrite it. If you want to
+disable any of the default enabled extensions you must assign ``None`` as their
+value.
 
 For more information See the :ref:`extensions user guide  <topics-extensions>`
 and the :ref:`list of available extensions <topics-extensions-ref>`.
 
+
 .. setting:: ITEM_PIPELINES
 
 ITEM_PIPELINES
@@ -589,12 +578,9 @@ ITEM_PIPELINES
 
 Default: ``{}``
 
-A dict containing the item pipelines to use, and their orders. The dict is
-empty by default order values are arbitrary but it's customary to define them
-in the 0-1000 range.
-
-Lists are supported in :setting:`ITEM_PIPELINES` for backwards compatibility,
-but they are deprecated.
+A dict containing the item pipelines to use, and their orders. Order values are
+arbitrary, but it is customary to define them in the 0-1000 range. Lower orders
+process before higher orders.
 
 Example::
 
@@ -603,16 +589,6 @@ Example::
        'mybot.pipelines.validate.StoreMyItem': 800,
    }
 
-.. setting:: ITEM_PIPELINES_BASE
-
-ITEM_PIPELINES_BASE
--------------------
-
-Default: ``{}``
-
-A dict containing the pipelines enabled by default in Scrapy. You should never
-modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
-
 .. setting:: LOG_ENABLED
 
 LOG_ENABLED
@@ -638,7 +614,7 @@ LOG_FILE
 
 Default: ``None``
 
-File name to use for logging output. If None, standard error will be used.
+File name to use for logging output. If ``None``, standard error will be used.
 
 .. setting:: LOG_FORMAT
 
@@ -902,16 +878,6 @@ The scheduler to use for crawling.
 SPIDER_CONTRACTS
 ----------------
 
-Default:: ``{}``
-
-A dict containing the scrapy contracts enabled in your project, used for
-testing spiders. For more info see :ref:`topics-contracts`.
-
-.. setting:: SPIDER_CONTRACTS_BASE
-
-SPIDER_CONTRACTS_BASE
----------------------
-
 Default::
 
     {
@@ -920,9 +886,13 @@ Default::
         'scrapy.contracts.default.ScrapesContract': 3,
     }
 
-A dict containing the scrapy contracts enabled by default in Scrapy. You should
-never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS`
-instead. For more info see :ref:`topics-contracts`.
+A dict containing the scrapy contracts enabled in your project, used for
+testing spiders. For more info see :ref:`topics-contracts`.
+
+When you set :setting:`SPIDER_CONTRACTS` manually, e.g. in your project's
+settings module, it will be merged with the default, not overwrite it. If you
+want to disable any of the default contracts you must assign ``None`` as their
+value.
 
 .. setting:: SPIDER_LOADER_CLASS
 
@@ -939,16 +909,6 @@ The class that will be used for loading spiders, which must implement the
 SPIDER_MIDDLEWARES
 ------------------
 
-Default:: ``{}``
-
-A dict containing the spider middlewares enabled in your project, and their
-orders. For more info see :ref:`topics-spider-middleware-setting`.
-
-.. setting:: SPIDER_MIDDLEWARES_BASE
-
-SPIDER_MIDDLEWARES_BASE
------------------------
-
 Default::
 
     {
@@ -959,10 +919,14 @@ Default::
         'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,
     }
 
-A dict containing the spider middlewares enabled by default in Scrapy. You
-should never modify this setting in your project, modify
-:setting:`SPIDER_MIDDLEWARES` instead. For more info see
-:ref:`topics-spider-middleware-setting`.
+A dict containing the spider middlewares enabled in your project, and their
+orders. Low orders are closer to the engine, high orders are closer to the
+spider. For more info see :ref:`topics-spider-middleware-setting`.
+
+When you set :setting:`SPIDER_MIDDLEWARES` manually, e.g. in your project's
+settings module, it will be merged with the default, not overwrite it. If you
+want to disable any of the default spider middlewares you must assign ``None``
+as their value.
 
 .. setting:: SPIDER_MODULES
 
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 84daaaa55..d448801d3 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -24,22 +24,20 @@ Here's an example::
         'myproject.middlewares.CustomSpiderMiddleware': 543,
     }
 
-The :setting:`SPIDER_MIDDLEWARES` setting is merged with the
-:setting:`SPIDER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant to
-be overridden) and then sorted by order to get the final sorted list of enabled
-middlewares: the first middleware is the one closer to the engine and the last
-is the one closer to the spider.
+The specified :setting:`SPIDER_MIDDLEWARES` setting is merged with the default
+one (i.e. it does not overwrite it) and then sorted by order to get the final
+sorted list of enabled middlewares: the first middleware is the one closer to
+the engine and the last is the one closer to the spider.
 
-To decide which order to assign to your middleware see the
-:setting:`SPIDER_MIDDLEWARES_BASE` setting and pick a value according to where
+To decide which order to assign to your middleware see the default
+:setting:`SPIDER_MIDDLEWARES` setting and pick a value according to where
 you want to insert the middleware. The order does matter because each
 middleware performs a different action and your middleware could depend on some
 previous (or subsequent) middleware being applied.
 
-If you want to disable a builtin middleware (the ones defined in
-:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
-in your project :setting:`SPIDER_MIDDLEWARES` setting and assign `None` as its
-value.  For example, if you want to disable the off-site middleware::
+If you want to disable a builtin middleware you must define it in your project's
+:setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its value. For
+example, if you want to disable the off-site middleware::
 
     SPIDER_MIDDLEWARES = {
         'myproject.middlewares.CustomSpiderMiddleware': 543,
@@ -173,7 +171,7 @@ information on how to use them and how to write your own spider middleware, see
 the :ref:`spider middleware usage guide <topics-spider-middleware>`.
 
 For a list of the components enabled by default (and their orders) see the
-:setting:`SPIDER_MIDDLEWARES_BASE` setting.
+:setting:`SPIDER_MIDDLEWARES` setting.
 
 DepthMiddleware
 ---------------
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index 017595f04..a423ba2c9 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -58,10 +58,7 @@ class Command(ScrapyCommand):
 
     def run(self, args, opts):
         # load contracts
-        contracts = build_component_list(
-            self.settings['SPIDER_CONTRACTS_BASE'],
-            self.settings['SPIDER_CONTRACTS'],
-        )
+        contracts = build_component_list(self.settings._getcomposite('SPIDER_CONTRACTS'))
         conman = ContractsManager(load_object(c) for c in contracts)
         runner = TextTestRunner(verbosity=2 if opts.verbose else 1)
         result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 72df11476..9c8a3d4ce 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,6 +1,6 @@
 import os
 from scrapy.commands import ScrapyCommand
-from scrapy.utils.conf import arglist_to_dict
+from scrapy.utils.conf import arglist_to_dict, remove_none_values
 from scrapy.exceptions import UsageError
 
 
@@ -34,10 +34,8 @@ class Command(ScrapyCommand):
                 self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
                 self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            valid_output_formats = (
-                list(self.settings.getdict('FEED_EXPORTERS').keys()) +
-                list(self.settings.getdict('FEED_EXPORTERS_BASE').keys())
-            )
+            feed_exporters = remove_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
+            valid_output_formats = feed_exporters.keys()
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
             if opts.output_format not in valid_output_formats:
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 88f5a3015..7d85984c3 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -5,7 +5,7 @@ from importlib import import_module
 from scrapy.utils.spider import iter_spider_classes
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
-from scrapy.utils.conf import arglist_to_dict
+from scrapy.utils.conf import arglist_to_dict, remove_none_values
 
 
 def _import_file(filepath):
@@ -57,10 +57,8 @@ class Command(ScrapyCommand):
                 self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
                 self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            valid_output_formats = (
-                list(self.settings.getdict('FEED_EXPORTERS').keys()) +
-                list(self.settings.getdict('FEED_EXPORTERS_BASE').keys())
-            )
+            feed_exporters = remove_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
+            valid_output_formats = feed_exporters.keys()
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
             if opts.output_format not in valid_output_formats:
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 6c9514af6..9b118c39b 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -4,6 +4,7 @@ import logging
 from twisted.internet import defer
 import six
 from scrapy.exceptions import NotSupported, NotConfigured
+from scrapy.utils.conf import remove_none_values
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
 from scrapy import signals
@@ -19,13 +20,8 @@ class DownloadHandlers(object):
         self._schemes = {}  # stores acceptable schemes on instancing
         self._handlers = {}  # stores instanced handlers for schemes
         self._notconfigured = {}  # remembers failed handlers
-        handlers = crawler.settings.get('DOWNLOAD_HANDLERS_BASE')
-        handlers.update(crawler.settings.get('DOWNLOAD_HANDLERS', {}))
+        handlers = remove_none_values(crawler.settings._getcomposite('DOWNLOAD_HANDLERS'))
         for scheme, clspath in six.iteritems(handlers):
-            # Allow to disable a handler just like any other
-            # component (extension, middleware, etc).
-            if clspath is None:
-                continue
             self._schemes[scheme] = clspath
 
         crawler.signals.connect(self._close, signals.engine_stopped)
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 9cd30c144..958113fc3 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -19,8 +19,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(settings['DOWNLOADER_MIDDLEWARES_BASE'], \
-            settings['DOWNLOADER_MIDDLEWARES'])
+        return build_component_list(settings._getcomposite('DOWNLOADER_MIDDLEWARES'))
 
     def _add_middleware(self, mw):
         if hasattr(mw, 'process_request'):
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index c1c5b10fc..b5c80c350 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -18,8 +18,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(settings['SPIDER_MIDDLEWARES_BASE'], \
-            settings['SPIDER_MIDDLEWARES'])
+        return build_component_list(settings._getcomposite('SPIDER_MIDDLEWARES'))
 
     def _add_middleware(self, mw):
         super(SpiderMiddlewareManager, self)._add_middleware(mw)
diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py
index f1d2bd631..c8924c04a 100644
--- a/scrapy/downloadermiddlewares/defaultheaders.py
+++ b/scrapy/downloadermiddlewares/defaultheaders.py
@@ -4,6 +4,8 @@ DefaultHeaders downloader middleware
 See documentation in docs/topics/downloader-middleware.rst
 """
 
+from scrapy.utils.conf import remove_none_values
+
 
 class DefaultHeadersMiddleware(object):
 
@@ -12,7 +14,8 @@ class DefaultHeadersMiddleware(object):
 
     @classmethod
     def from_crawler(cls, crawler):
-        return cls(crawler.settings.get('DEFAULT_REQUEST_HEADERS').items())
+        headers = remove_none_values(crawler.settings['DEFAULT_REQUEST_HEADERS'])
+        return cls(headers.items())
 
     def process_request(self, request, spider):
         for k, v in self._headers:
diff --git a/scrapy/extension.py b/scrapy/extension.py
index f68b1ba68..4ceb32c68 100644
--- a/scrapy/extension.py
+++ b/scrapy/extension.py
@@ -12,5 +12,4 @@ class ExtensionManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(settings['EXTENSIONS_BASE'], \
-            settings['EXTENSIONS'])
+        return build_component_list(settings._getcomposite('EXTENSIONS'))
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 7560e89d3..fb07657d6 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -18,6 +18,7 @@ from twisted.internet import defer, threads
 from w3lib.url import file_uri_to_path
 
 from scrapy import signals
+from scrapy.utils.conf import remove_none_values
 from scrapy.utils.ftp import ftp_makedirs_cwd
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
@@ -195,8 +196,7 @@ class FeedExporter(object):
         return item
 
     def _load_components(self, setting_prefix):
-        conf = dict(self.settings['%s_BASE' % setting_prefix])
-        conf.update(self.settings[setting_prefix])
+        conf = remove_none_values(self.settings._getcomposite(setting_prefix))
         d = {}
         for k, v in conf.items():
             try:
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index d433498f5..8df0d3154 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -13,15 +13,7 @@ class ItemPipelineManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        item_pipelines = settings['ITEM_PIPELINES']
-        if isinstance(item_pipelines, (tuple, list, set, frozenset)):
-            from scrapy.exceptions import ScrapyDeprecationWarning
-            import warnings
-            warnings.warn('ITEM_PIPELINES defined as a list or a set is deprecated, switch to a dict',
-                category=ScrapyDeprecationWarning, stacklevel=1)
-            # convert old ITEM_PIPELINE list to a dict with order 500
-            item_pipelines = dict(zip(item_pipelines, range(500, 500+len(item_pipelines))))
-        return build_component_list(settings['ITEM_PIPELINES_BASE'], item_pipelines)
+        return build_component_list(settings._getcomposite('ITEM_PIPELINES'))
 
     def _add_middleware(self, pipe):
         super(ItemPipelineManager, self)._add_middleware(pipe)
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index fa7fa3178..7eea562e1 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -36,13 +36,21 @@ class SettingsAttribute(object):
 
     def __init__(self, value, priority):
         self.value = value
-        self.priority = priority
+        if isinstance(self.value, BaseSettings):
+            self.priority = max(self.value.maxpriority(), priority)
+        else:
+            self.priority = priority
 
     def set(self, value, priority):
         """Sets value if priority is higher or equal than current priority."""
-        if priority >= self.priority:
-            self.value = value
-            self.priority = priority
+        if isinstance(self.value, BaseSettings):
+            # Ignore self.priority if self.value has per-key priorities
+            self.value.update(value, priority)
+            self.priority = max(self.value.maxpriority(), priority)
+        else:
+            if priority >= self.priority:
+                self.value = value
+                self.priority = priority
 
     def __str__(self):
         return "<SettingsAttribute value={self.value!r} " \
@@ -95,6 +103,20 @@ class BaseSettings(MutableMapping):
             value = json.loads(value)
         return dict(value)
 
+    def _getcomposite(self, name):
+        # DO NOT USE THIS FUNCTION IN YOUR CUSTOM PROJECTS
+        # It's for internal use in the transition away from the _BASE settings and
+        # will be removed along with _BASE support in a future release
+        basename = name + "_BASE"
+        if basename in self:
+            warnings.warn('_BASE settings are deprecated.',
+                          category=ScrapyDeprecationWarning)
+            compsett = BaseSettings(self[name + "_BASE"], priority='default')
+            compsett.update(self[name])
+            return compsett
+        else:
+            return self[name]
+
     def getpriority(self, name):
         prio = None
         if name in self:
@@ -232,16 +254,25 @@ class _DictProxy(MutableMapping):
 class Settings(BaseSettings):
 
     def __init__(self, values=None, priority='project'):
+        # Do not pass kwarg values here. We don't want to promote user-defined
+        # dicts, and we want to update, not replace, default dicts with the
+        # values given by the user
         super(Settings, self).__init__()
         self.setmodule(default_settings, 'default')
+        # Promote default dictionaries to BaseSettings instances for per-key
+        # priorities
+        for name in self:
+            val = self[name]
+            if isinstance(val, dict):
+                self.set(name, BaseSettings(val, 'default'), 'default')
         self.update(values, priority)
 
 
 class CrawlerSettings(Settings):
 
     def __init__(self, settings_module=None, **kw):
-        Settings.__init__(self, **kw)
         self.settings_module = settings_module
+        Settings.__init__(self, **kw)
 
     def __getitem__(self, opt_name):
         if opt_name in self.overrides:
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 5d48ac1a7..ba7cece19 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -63,8 +63,7 @@ DNS_TIMEOUT = 60
 
 DOWNLOAD_DELAY = 0
 
-DOWNLOAD_HANDLERS = {}
-DOWNLOAD_HANDLERS_BASE = {
+DOWNLOAD_HANDLERS = {
     'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
     'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
     'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
@@ -82,9 +81,7 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader'
 DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
 DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
 
-DOWNLOADER_MIDDLEWARES = {}
-
-DOWNLOADER_MIDDLEWARES_BASE = {
+DOWNLOADER_MIDDLEWARES = {
     # Engine side
     'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
     'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
@@ -116,9 +113,7 @@ except KeyError:
     else:
         EDITOR = 'vi'
 
-EXTENSIONS = {}
-
-EXTENSIONS_BASE = {
+EXTENSIONS = {
     'scrapy.extensions.corestats.CoreStats': 0,
     'scrapy.telnet.TelnetConsole': 0,
     'scrapy.extensions.memusage.MemoryUsage': 0,
@@ -135,16 +130,14 @@ FEED_URI_PARAMS = None  # a function to extend uri arguments
 FEED_FORMAT = 'jsonlines'
 FEED_STORE_EMPTY = False
 FEED_EXPORT_FIELDS = None
-FEED_STORAGES = {}
-FEED_STORAGES_BASE = {
+FEED_STORAGES = {
     '': 'scrapy.extensions.feedexport.FileFeedStorage',
     'file': 'scrapy.extensions.feedexport.FileFeedStorage',
     'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_EXPORTERS = {}
-FEED_EXPORTERS_BASE = {
+FEED_EXPORTERS = {
     'json': 'scrapy.exporters.JsonItemExporter',
     'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
     'jl': 'scrapy.exporters.JsonLinesItemExporter',
@@ -170,7 +163,6 @@ HTTPCACHE_GZIP = False
 ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
 
 ITEM_PIPELINES = {}
-ITEM_PIPELINES_BASE = {}
 
 LOG_ENABLED = True
 LOG_ENCODING = 'utf-8'
@@ -229,9 +221,7 @@ SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
 
 SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
 
-SPIDER_MIDDLEWARES = {}
-
-SPIDER_MIDDLEWARES_BASE = {
+SPIDER_MIDDLEWARES = {
     # Engine side
     'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
     'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,
@@ -258,8 +248,7 @@ TELNETCONSOLE_ENABLED = 1
 TELNETCONSOLE_PORT = [6023, 6073]
 TELNETCONSOLE_HOST = '127.0.0.1'
 
-SPIDER_CONTRACTS = {}
-SPIDER_CONTRACTS_BASE = {
+SPIDER_CONTRACTS = {
     'scrapy.contracts.default.UrlContract': 1,
     'scrapy.contracts.default.ReturnsContract': 2,
     'scrapy.contracts.default.ScrapesContract': 3,
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 423ca43e2..80c644657 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -1,36 +1,49 @@
 import os
 import sys
+import warnings
 from operator import itemgetter
 
 import six
 from six.moves.configparser import SafeConfigParser
 
+from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
 
 
-def build_component_list(base, custom, convert=update_classpath):
-    """Compose a component list based on a custom and base dict of components
-    (typically middlewares or extensions), unless custom is already a list, in
-    which case it's returned.
-    """
+def build_component_list(compdict, convert=update_classpath):
+    """Compose a component list from a { class: order } dictionary."""
 
     def _check_components(complist):
         if len({convert(c) for c in complist}) != len(complist):
             raise ValueError('Some paths in {!r} convert to the same object, '
                              'please update your settings'.format(complist))
 
-    if isinstance(custom, (list, tuple)):
-        _check_components(custom)
-        return type(custom)(convert(c) for c in custom)
-
     def _map_keys(compdict):
-        _check_components(compdict)
-        return {convert(k): v for k, v in six.iteritems(compdict)}
+        if isinstance(compdict, BaseSettings):
+            compbs = BaseSettings()
+            for k, v in six.iteritems(compdict):
+                prio = compdict.getpriority(k)
+                if compbs.getpriority(convert(k)) == prio:
+                    raise ValueError('Some paths in {!r} convert to the same '
+                                     'object, please update your settings'
+                                     ''.format(list(compdict.keys())))
+                else:
+                    compbs.set(convert(k), v, priority=prio)
+            return compbs
+        else:
+            _check_components(compdict)
+            return {convert(k): v for k, v in six.iteritems(compdict)}
 
-    compdict = _map_keys(base)
-    compdict.update(_map_keys(custom))
-    items = (x for x in six.iteritems(compdict) if x[1] is not None)
-    return [x[0] for x in sorted(items, key=itemgetter(1))]
+    if isinstance(compdict, (list, tuple)):
+        _check_components(compdict)
+        return type(compdict)(convert(c) for c in compdict)
+    compdict = remove_none_values(_map_keys(compdict))
+    return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
+
+
+def remove_none_values(compdict):
+    """Return dict with all pairs that have value 'None' removed"""
+    return {k: v for k, v in six.iteritems(compdict) if v is not None}
 
 
 def arglist_to_dict(arglist):
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index a473f3c3f..bb38964ef 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -28,11 +28,22 @@ class SettingsAttributeTest(unittest.TestCase):
         self.assertEqual(self.attribute.value, 'value')
         self.assertEqual(self.attribute.priority, 10)
 
+    def test_set_per_key_priorities(self):
+        attribute = SettingsAttribute(
+                        BaseSettings({'one': 10, 'two': 20}, 0),
+                        0)
 
-class SettingsTest(unittest.TestCase):
+        new_dict = {'one': 11, 'two': 21}
+        attribute.set(new_dict, 10)
+        self.assertEqual(attribute.value['one'], 11)
+        self.assertEqual(attribute.value['two'], 21)
 
-    if six.PY3:
-        assertItemsEqual = unittest.TestCase.assertCountEqual
+        new_settings = BaseSettings()
+        new_settings.set('one', 12, 20)
+        new_settings.set('two', 12, 0)
+        attribute.set(new_settings, 0)
+        self.assertEqual(attribute.value['one'], 12)
+        self.assertEqual(attribute.value['two'], 21)
 
 
 class BaseSettingsTest(unittest.TestCase):
@@ -239,6 +250,20 @@ class BaseSettingsTest(unittest.TestCase):
         self.assertEqual(settings.getpriority('key'), 99)
         self.assertEqual(settings.getpriority('nonexistentkey'), None)
 
+    def test_getcomposite(self):
+        s = BaseSettings({'TEST_BASE': {1: 1, 2: 2},
+                          'TEST': BaseSettings({1: 10}),
+                          'HASNOBASE': BaseSettings({1: 1})})
+        cs = s._getcomposite('TEST')
+        self.assertEqual(len(cs), 2)
+        self.assertEqual(cs[1], 10)
+        self.assertEqual(cs[2], 2)
+        cs = s._getcomposite('HASNOBASE')
+        self.assertEqual(len(cs), 1)
+        self.assertEqual(cs[1], 1)
+        cs = s._getcomposite('NONEXISTENT')
+        self.assertIsNone(cs)
+
     def test_maxpriority(self):
         # Empty settings should return 'default'
         self.assertEqual(self.settings.maxpriority(), 0)
@@ -342,6 +367,24 @@ class SettingsTest(unittest.TestCase):
         self.assertEqual(attr.value, 'value')
         self.assertEqual(attr.priority, 10)
 
+    @mock.patch('scrapy.settings.default_settings', default_settings)
+    def test_autopromote_dicts(self):
+        settings = Settings()
+        mydict = settings.get('TEST_DICT')
+        self.assertIsInstance(mydict, BaseSettings)
+        self.assertIn('key', mydict)
+        self.assertEqual(mydict['key'], 'val')
+        self.assertEqual(mydict.getpriority('key'), 0)
+
+    @mock.patch('scrapy.settings.default_settings', default_settings)
+    def test_getdict_autodegrade_basesettings(self):
+        settings = Settings()
+        mydict = settings.getdict('TEST_DICT')
+        self.assertIsInstance(mydict, dict)
+        self.assertEqual(len(mydict), 1)
+        self.assertIn('key', mydict)
+        self.assertEqual(mydict['key'], 'val')
+
 
 class CrawlerSettingsTest(unittest.TestCase):
 
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 177d024fb..e94ccc49b 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -1,43 +1,64 @@
 import unittest
 
-from scrapy.utils.conf import build_component_list, arglist_to_dict
+from scrapy.settings import BaseSettings
+from scrapy.utils.conf import (build_component_list, arglist_to_dict,
+                               remove_none_values)
 
 
 class BuildComponentListTest(unittest.TestCase):
 
     def test_build_dict(self):
-        base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None}
-        custom = {'two': None, 'three': 8, 'four': 4}
-        self.assertEqual(build_component_list(base, custom, lambda x: x),
-                         ['one', 'four', 'five', 'three'])
+        d = {'one': 1, 'two': None, 'three': 8, 'four': 4}
+        self.assertEqual(build_component_list(d, lambda x: x),
+                         ['one', 'four', 'three'])
 
     def test_return_list(self):
         custom = ['a', 'b', 'c']
-        self.assertEqual(build_component_list(None, custom, lambda x: x),
-                         custom)
+        self.assertEqual(build_component_list(custom, lambda x: x), custom)
 
     def test_map_dict(self):
         custom = {'one': 1, 'two': 2, 'three': 3}
-        self.assertEqual(build_component_list({}, custom, lambda x: x.upper()),
+        self.assertEqual(build_component_list(custom, lambda x: x.upper()),
                          ['ONE', 'TWO', 'THREE'])
 
     def test_map_list(self):
         custom = ['a', 'b', 'c']
-        self.assertEqual(build_component_list(None, custom, lambda x: x.upper()),
+        self.assertEqual(build_component_list(custom, lambda x: x.upper()),
                          ['A', 'B', 'C'])
 
     def test_duplicate_components_in_dict(self):
         duplicate_dict = {'one': 1, 'two': 2, 'ONE': 4}
         self.assertRaises(ValueError,
-                          build_component_list, {}, duplicate_dict, lambda x: x.lower())
+                          build_component_list, duplicate_dict, lambda x: x.lower())
 
     def test_duplicate_components_in_list(self):
         duplicate_list = ['a', 'b', 'a']
         self.assertRaises(ValueError,
-                          build_component_list, None, duplicate_list, lambda x: x)
+                          build_component_list, duplicate_list, lambda x: x)
+
+    def test_duplicate_components_in_basesettings(self):
+        # Higher priority takes precedence
+        duplicate_bs = BaseSettings({'one': 1, 'two': 2}, priority=0)
+        duplicate_bs.set('ONE', 4, priority=10)
+        self.assertEqual(build_component_list(duplicate_bs, convert=lambda x: x.lower()),
+                         ['two', 'one'])
+        duplicate_bs.set('one', duplicate_bs['one'], priority=20)
+        self.assertEqual(build_component_list(duplicate_bs, convert=lambda x: x.lower()),
+                         ['one', 'two'])
+        # Same priority raises ValueError
+        duplicate_bs.set('ONE', duplicate_bs['ONE'], priority=20)
+        self.assertRaises(ValueError,
+                          build_component_list, duplicate_bs, convert=lambda x: x.lower())
+
 
 class UtilsConfTestCase(unittest.TestCase):
 
+    def test_remove_none_values(self):
+        comps = {'one': 1, 'none': None, 'three': 3, 'four': 4}
+        compscopy = dict(comps)
+        del compscopy['none']
+        self.assertEqual(remove_none_values(comps), compscopy)
+
     def test_arglist_to_dict(self):
         self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']),
             {'arg1': 'val1', 'arg2': 'val2'})

From bb6dee611ca72cda61d0daa12880c5d4c5aba9c8 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Thu, 2 Jul 2015 16:51:15 +0200
Subject: [PATCH 0595/2974] Move Settings documentation to docstrings

---
 docs/topics/api.rst         | 209 ++----------------------------------
 scrapy/settings/__init__.py | 193 ++++++++++++++++++++++++++++++++-
 2 files changed, 195 insertions(+), 207 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 923bd80b0..42c0133c1 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -140,211 +140,14 @@ Settings API
     For a detailed explanation on each settings sources, see:
     :ref:`topics-settings`.
 
-.. function:: get_settings_priority(priority)
+.. autofunction:: get_settings_priority
 
-    Small helper function that looks up a given string priority in the
-    :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its
-    numerical value, or directly returns a given numerical priority.
+.. autoclass:: Settings
+   :show-inheritance:
+   :members:
 
-.. class:: Settings(values={}, priority='project')
-
-    This object stores Scrapy settings for the configuration of internal
-    components, and can be used for any further customization.
-
-    It is a direct subclass and supports all methods of
-    :class:`~scrapy.settings.BaseSettings`. Additionally, after instantiation
-    of this class, the new object will have the global default settings
-    described on :ref:`topics-settings-ref` already populated.
-
-.. class:: BaseSettings(values={}, priority='project')
-
-    Instances of this class behave like dictionaries, but store priorities
-    along with their ``(key, value)`` pairs, and can be frozen (i.e. marked
-    immutable).
-
-    Key-value entries can be passed on initialization with the ``values``
-    argument, and they would take the ``priority`` level (unless ``values`` is
-    already an instance of :class:`~scrapy.settings.BaseSettings`, in which
-    case the existing priority levels will be kept).  If the ``priority``
-    argument is a string, the priority name will be looked up in
-    :attr:`~scrapy.settings.SETTINGS_PRIORITIES`. Otherwise, a specific integer
-    should be provided.
-
-    Once the object is created, new settings can be loaded or updated with the
-    :meth:`~scrapy.settings.BaseSettings.set` method, and can be accessed with
-    the square bracket notation of dictionaries, or with the
-    :meth:`~scrapy.settings.BaseSettings.get` method of the instance and its
-    value conversion variants. When requesting a stored key, the value with the
-    highest priority will be retrieved.
-
-    .. method:: set(name, value, priority='project')
-
-       Store a key/value attribute with a given priority.
-
-       Settings should be populated *before* configuring the Crawler object
-       (through the :meth:`~scrapy.crawler.Crawler.configure` method),
-       otherwise they won't have any effect.
-
-       :param name: the setting name
-       :type name: string
-
-       :param value: the value to associate with the setting
-       :type value: any
-
-       :param priority: the priority of the setting. Should be a key of
-           :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
-       :type priority: string or int
-
-    .. method:: update(values, priority='project')
-
-       Store key/value pairs with a given priority.
-
-       This is a helper function that calls
-       :meth:`~scrapy.settings.BaseSettings.set` for every item of ``values``
-       with the provided ``priority``.
-
-       If ``values`` is a string, it is assumed to be JSON-encoded and parsed
-       into a dict with ``json.loads()`` first. If it is a
-       :class:`~scrapy.settings.BaseSettings` instance, the per-key priorities
-       will be used and the ``priority`` parameter ignored. This allows
-       inserting/updating settings with different priorities with a single
-       command.
-
-       :param values: the settings names and values
-       :type values: dict or string or :class:`~scrapy.settings.BaseSettings`
-
-       :param priority: the priority of the settings. Should be a key of
-           :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
-       :type priority: string or int
-
-    .. method:: setmodule(module, priority='project')
-
-       Store settings from a module with a given priority.
-
-       This is a helper function that calls
-       :meth:`~scrapy.settings.BaseSettings.set` for every globally declared
-       uppercase variable of ``module`` with the provided ``priority``.
-
-       :param module: the module or the path of the module
-       :type module: module object or string
-
-       :param priority: the priority of the settings. Should be a key of
-           :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
-       :type priority: string or int
-
-    .. method:: get(name, default=None)
-
-       Get a setting value without affecting its original type.
-
-       :param name: the setting name
-       :type name: string
-
-       :param default: the value to return if no setting is found
-       :type default: any
-
-    .. method:: getbool(name, default=False)
-
-       Get a setting value as a boolean. For example, both ``1`` and ``'1'``, and
-       ``True`` return ``True``, while ``0``, ``'0'``, ``False`` and ``None``
-       return ``False````
-
-       For example, settings populated through environment variables set to ``'0'``
-       will return ``False`` when using this method.
-
-       :param name: the setting name
-       :type name: string
-
-       :param default: the value to return if no setting is found
-       :type default: any
-
-    .. method:: getint(name, default=0)
-
-       Get a setting value as an int
-
-       :param name: the setting name
-       :type name: string
-
-       :param default: the value to return if no setting is found
-       :type default: any
-
-    .. method:: getfloat(name, default=0.0)
-
-       Get a setting value as a float
-
-       :param name: the setting name
-       :type name: string
-
-       :param default: the value to return if no setting is found
-       :type default: any
-
-    .. method:: getlist(name, default=None)
-
-       Get a setting value as a list. If the setting original type is a list, a
-       copy of it will be returned. If it's a string it will be split by ",".
-
-       For example, settings populated through environment variables set to
-       ``'one,two'`` will return a list ['one', 'two'] when using this method.
-
-       :param name: the setting name
-       :type name: string
-
-       :param default: the value to return if no setting is found
-       :type default: any
-
-    .. method:: getdict(name, default=None)
-
-       Get a setting value as a dictionary. If the setting original type is a
-       dictionary, a copy of it will be returned. If it is a string it will be
-       evaluated as a JSON dictionary. In the case that it is a
-       :class:`~scrapy.settings.BaseSettings` instance itself, it will be
-       converted to a dictionary, containing all its current settings values
-       as they would be returned by :meth:`~scrapy.settings.BaseSettings.get`,
-       and losing all information about priority and mutability.
-
-       :param name: the setting name
-       :type name: string
-
-       :param default: the value to return if no setting is found
-       :type default: any
-
-    .. method:: copy()
-
-       Make a deep copy of current settings.
-
-       This method returns a new instance of the :class:`Settings` class,
-       populated with the same values and their priorities.
-
-       Modifications to the new object won't be reflected on the original
-       settings.
-
-    .. method:: freeze()
-
-       Disable further changes to the current settings.
-
-       After calling this method, the present state of the settings will become
-       immutable. Trying to change values through the :meth:`~set` method and
-       its variants won't be possible and will be alerted.
-
-    .. method:: frozencopy()
-
-       Return an immutable copy of the current settings.
-
-       Alias for a :meth:`~freeze` call in the object returned by :meth:`copy`
-
-    .. method:: getpriority(name)
-
-       Return the current numerical priority value of a setting, or ``None`` if
-       the given ``name`` does not exist.
-
-       :param name: the setting name
-       :type name: string
-
-    .. method:: maxpriority()
-
-       Return the numerical value of the highest priority present throughout
-       all settings, or the numerical value for ``default`` from
-       :attr:`~scrapy.settings.SETTINGS_PRIORITIES` if there are no settings
-       stored.
+.. autoclass:: BaseSettings
+   :members:
 
 .. _topics-api-spiderloader:
 
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 7eea562e1..1216aabcb 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -20,6 +20,11 @@ SETTINGS_PRIORITIES = {
 }
 
 def get_settings_priority(priority):
+    """
+    Small helper function that looks up a given string priority in the
+    :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its
+    numerical value, or directly returns a given numerical priority.
+    """
     if isinstance(priority, six.string_types):
         return SETTINGS_PRIORITIES[priority]
     else:
@@ -60,6 +65,26 @@ class SettingsAttribute(object):
 
 
 class BaseSettings(MutableMapping):
+    """
+    Instances of this class behave like dictionaries, but store priorities
+    along with their ``(key, value)`` pairs, and can be frozen (i.e. marked
+    immutable).
+
+    Key-value entries can be passed on initialization with the ``values``
+    argument, and they would take the ``priority`` level (unless ``values`` is
+    already an instance of :class:`~scrapy.settings.BaseSettings`, in which
+    case the existing priority levels will be kept).  If the ``priority``
+    argument is a string, the priority name will be looked up in
+    :attr:`~scrapy.settings.SETTINGS_PRIORITIES`. Otherwise, a specific integer
+    should be provided.
+
+    Once the object is created, new settings can be loaded or updated with the
+    :meth:`~scrapy.settings.BaseSettings.set` method, and can be accessed with
+    the square bracket notation of dictionaries, or with the
+    :meth:`~scrapy.settings.BaseSettings.get` method of the instance and its
+    value conversion variants. When requesting a stored key, the value with the
+    highest priority will be retrieved.
+    """
 
     def __init__(self, values=None, priority='project'):
         self.frozen = False
@@ -76,28 +101,94 @@ class BaseSettings(MutableMapping):
         return name in self.attributes
 
     def get(self, name, default=None):
+        """
+        Get a setting value without affecting its original type.
+
+        :param name: the setting name
+        :type name: string
+
+        :param default: the value to return if no setting is found
+        :type default: any
+        """
         return self[name] if self[name] is not None else default
 
     def getbool(self, name, default=False):
         """
-        True is: 1, '1', True
-        False is: 0, '0', False, None
+        Get a setting value as a boolean.
+        
+        ``1``, ``'1'``, and ``True`` return ``True``, while ``0``, ``'0'``,
+        ``False`` and ``None`` return ``False``. 
+
+        For example, settings populated through environment variables set to
+        ``'0'`` will return ``False`` when using this method.
+
+        :param name: the setting name
+        :type name: string
+
+        :param default: the value to return if no setting is found
+        :type default: any
         """
         return bool(int(self.get(name, default)))
 
     def getint(self, name, default=0):
+        """
+        Get a setting value as an int.
+
+        :param name: the setting name
+        :type name: string
+
+        :param default: the value to return if no setting is found
+        :type default: any
+        """
         return int(self.get(name, default))
 
     def getfloat(self, name, default=0.0):
+        """
+        Get a setting value as a float.
+
+        :param name: the setting name
+        :type name: string
+
+        :param default: the value to return if no setting is found
+        :type default: any
+        """
         return float(self.get(name, default))
 
     def getlist(self, name, default=None):
+        """
+        Get a setting value as a list. If the setting original type is a list, a
+        copy of it will be returned. If it's a string it will be split by ",".
+
+        For example, settings populated through environment variables set to
+        ``'one,two'`` will return a list ['one', 'two'] when using this method.
+
+        :param name: the setting name
+        :type name: string
+
+        :param default: the value to return if no setting is found
+        :type default: any
+        """
         value = self.get(name, default or [])
         if isinstance(value, six.string_types):
             value = value.split(',')
         return list(value)
 
     def getdict(self, name, default=None):
+        """
+        Get a setting value as a dictionary. If the setting original type is a
+        dictionary, a copy of it will be returned. If it is a string it will be
+        evaluated as a JSON dictionary. In the case that it is a
+        :class:`~scrapy.settings.BaseSettings` instance itself, it will be
+        converted to a dictionary, containing all its current settings values
+        as they would be returned by :meth:`~scrapy.settings.BaseSettings.get`,
+        and losing all information about priority and mutability.
+
+        :param name: the setting name
+        :type name: string
+
+        :param default: the value to return if no setting is found
+        :type default: any
+        """
         value = self.get(name, default or {})
         if isinstance(value, six.string_types):
             value = json.loads(value)
@@ -118,12 +209,25 @@ class BaseSettings(MutableMapping):
             return self[name]
 
     def getpriority(self, name):
+        """
+        Return the current numerical priority value of a setting, or ``None`` if
+        the given ``name`` does not exist.
+
+        :param name: the setting name
+        :type name: string
+        """
         prio = None
         if name in self:
             prio = self.attributes[name].priority
         return prio
 
     def maxpriority(self):
+        """
+        Return the numerical value of the highest priority present throughout
+        all settings, or the numerical value for ``default`` from
+        :attr:`~scrapy.settings.SETTINGS_PRIORITIES` if there are no settings
+        stored.
+        """
         if len(self) > 0:
             return max(self.getpriority(name) for name in self)
         else:
@@ -133,6 +237,23 @@ class BaseSettings(MutableMapping):
         self.set(name, value)
 
     def set(self, name, value, priority='project'):
+        """
+        Store a key/value attribute with a given priority.
+
+        Settings should be populated *before* configuring the Crawler object
+        (through the :meth:`~scrapy.crawler.Crawler.configure` method),
+        otherwise they won't have any effect.
+
+        :param name: the setting name
+        :type name: string
+
+        :param value: the value to associate with the setting
+        :type value: any
+
+        :param priority: the priority of the setting. Should be a key of
+            :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
+        :type priority: string or int
+        """
         self._assert_mutability()
         priority = get_settings_priority(priority)
         if name not in self:
@@ -147,6 +268,20 @@ class BaseSettings(MutableMapping):
         self.update(values, priority)
 
     def setmodule(self, module, priority='project'):
+        """
+        Store settings from a module with a given priority.
+
+        This is a helper function that calls
+        :meth:`~scrapy.settings.BaseSettings.set` for every globally declared
+        uppercase variable of ``module`` with the provided ``priority``.
+
+        :param module: the module or the path of the module
+        :type module: module object or string
+
+        :param priority: the priority of the settings. Should be a key of
+            :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
+        :type priority: string or int
+        """
         self._assert_mutability()
         if isinstance(module, six.string_types):
             module = import_module(module)
@@ -155,6 +290,27 @@ class BaseSettings(MutableMapping):
                 self.set(key, getattr(module, key), priority)
 
     def update(self, values, priority='project'):
+        """
+        Store key/value pairs with a given priority.
+
+        This is a helper function that calls
+        :meth:`~scrapy.settings.BaseSettings.set` for every item of ``values``
+        with the provided ``priority``.
+
+        If ``values`` is a string, it is assumed to be JSON-encoded and parsed
+        into a dict with ``json.loads()`` first. If it is a
+        :class:`~scrapy.settings.BaseSettings` instance, the per-key priorities
+        will be used and the ``priority`` parameter ignored. This allows
+        inserting/updating settings with different priorities with a single
+        command.
+
+        :param values: the settings names and values
+        :type values: dict or string or :class:`~scrapy.settings.BaseSettings`
+
+        :param priority: the priority of the settings. Should be a key of
+            :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
+        :type priority: string or int
+        """
         self._assert_mutability()
         if isinstance(values, six.string_types):
             values = json.loads(values)
@@ -181,12 +337,33 @@ class BaseSettings(MutableMapping):
             raise TypeError("Trying to modify an immutable Settings object")
 
     def copy(self):
+        """
+        Make a deep copy of current settings.
+
+        This method returns a new instance of the :class:`Settings` class,
+        populated with the same values and their priorities.
+
+        Modifications to the new object won't be reflected on the original
+        settings.
+        """
         return copy.deepcopy(self)
 
     def freeze(self):
+        """
+        Disable further changes to the current settings.
+
+        After calling this method, the present state of the settings will become
+        immutable. Trying to change values through the :meth:`~set` method and
+        its variants won't be possible and will be alerted.
+        """
         self.frozen = True
 
     def frozencopy(self):
+        """
+        Return an immutable copy of the current settings.
+
+        Alias for a :meth:`~freeze` call in the object returned by :meth:`copy`.
+        """
         copy = self.copy()
         copy.freeze()
         return copy
@@ -252,6 +429,15 @@ class _DictProxy(MutableMapping):
 
 
 class Settings(BaseSettings):
+    """
+    This object stores Scrapy settings for the configuration of internal
+    components, and can be used for any further customization.
+
+    It is a direct subclass and supports all methods of
+    :class:`~scrapy.settings.BaseSettings`. Additionally, after instantiation
+    of this class, the new object will have the global default settings
+    described on :ref:`topics-settings-ref` already populated.
+    """
 
     def __init__(self, values=None, priority='project'):
         # Do not pass kwarg values here. We don't want to promote user-defined
@@ -261,8 +447,7 @@ class Settings(BaseSettings):
         self.setmodule(default_settings, 'default')
         # Promote default dictionaries to BaseSettings instances for per-key
         # priorities
-        for name in self:
-            val = self[name]
+        for name, val in six.iteritems(self):
             if isinstance(val, dict):
                 self.set(name, BaseSettings(val, 'default'), 'default')
         self.update(values, priority)

From 9bd7af8a625d63a3372346bf6c69c80a2a9832a8 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Tue, 25 Aug 2015 23:41:34 +0200
Subject: [PATCH 0596/2974] Remove unused import in scrapy.settings

---
 scrapy/settings/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 1216aabcb..ed201e980 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -2,7 +2,7 @@ import six
 import json
 import copy
 import warnings
-from collections import Mapping, MutableMapping
+from collections import MutableMapping
 from importlib import import_module
 
 from scrapy.utils.deprecate import create_deprecated_class

From 9eb3597d159a8556259946b7acf39d3c368113dc Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Tue, 25 Aug 2015 23:43:54 +0200
Subject: [PATCH 0597/2974] PEP8ify settings module

---
 scrapy/settings/__init__.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index ed201e980..6c922a709 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -19,6 +19,7 @@ SETTINGS_PRIORITIES = {
     'cmdline': 40,
 }
 
+
 def get_settings_priority(priority):
     """
     Small helper function that looks up a given string priority in the
@@ -196,8 +197,8 @@ class BaseSettings(MutableMapping):
 
     def _getcomposite(self, name):
         # DO NOT USE THIS FUNCTION IN YOUR CUSTOM PROJECTS
-        # It's for internal use in the transition away from the _BASE settings and
-        # will be removed along with _BASE support in a future release
+        # It's for internal use in the transition away from the _BASE settings
+        # and will be removed along with _BASE support in a future release
         basename = name + "_BASE"
         if basename in self:
             warnings.warn('_BASE settings are deprecated.',
@@ -482,6 +483,7 @@ def iter_default_settings():
         if name.isupper():
             yield name, getattr(default_settings, name)
 
+
 def overridden_settings(settings):
     """Return a dict of the settings that have been overridden"""
     for name, defvalue in iter_default_settings():

From 90198e5324f3172ec83c457049a7a66a2805d875 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Tue, 25 Aug 2015 23:44:37 +0200
Subject: [PATCH 0598/2974] Add __repr__ method for BaseSettings

---
 scrapy/settings/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 6c922a709..3ae2187ae 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -378,7 +378,8 @@ class BaseSettings(MutableMapping):
     def __str__(self):
         return str(self.attributes)
 
-    __repr__ = __str__
+    def __repr__(self):
+        return "<%s %s>" % (self.__class__.__name__, self.attributes)
 
     @property
     def overrides(self):

From f249b309ab779b5ab518f54f309d7a4ac6661ec7 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Wed, 26 Aug 2015 00:26:06 +0200
Subject: [PATCH 0599/2974] Move scrapy.utils.conf.remove_none_values to
 s.u.python.without_none_values

---
 scrapy/commands/crawl.py                       |  5 +++--
 scrapy/commands/runspider.py                   |  5 +++--
 scrapy/core/downloader/handlers/__init__.py    |  4 ++--
 scrapy/downloadermiddlewares/defaultheaders.py |  4 ++--
 scrapy/extensions/feedexport.py                |  4 ++--
 scrapy/utils/conf.py                           |  9 ++-------
 scrapy/utils/python.py                         | 12 ++++++++++++
 tests/test_utils_conf.py                       |  9 +--------
 tests/test_utils_python.py                     | 10 +++++++++-
 9 files changed, 36 insertions(+), 26 deletions(-)

diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 9c8a3d4ce..7f5c64c20 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,6 +1,7 @@
 import os
 from scrapy.commands import ScrapyCommand
-from scrapy.utils.conf import arglist_to_dict, remove_none_values
+from scrapy.utils.conf import arglist_to_dict
+from scrapy.utils.python import without_none_values
 from scrapy.exceptions import UsageError
 
 
@@ -34,7 +35,7 @@ class Command(ScrapyCommand):
                 self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
                 self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            feed_exporters = remove_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
+            feed_exporters = without_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
             valid_output_formats = feed_exporters.keys()
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 7d85984c3..72229bcf5 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -5,7 +5,8 @@ from importlib import import_module
 from scrapy.utils.spider import iter_spider_classes
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
-from scrapy.utils.conf import arglist_to_dict, remove_none_values
+from scrapy.utils.conf import arglist_to_dict
+from scrapy.utils.python import without_none_values
 
 
 def _import_file(filepath):
@@ -57,7 +58,7 @@ class Command(ScrapyCommand):
                 self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
                 self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            feed_exporters = remove_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
+            feed_exporters = without_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
             valid_output_formats = feed_exporters.keys()
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 9b118c39b..0e78e04f4 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -4,9 +4,9 @@ import logging
 from twisted.internet import defer
 import six
 from scrapy.exceptions import NotSupported, NotConfigured
-from scrapy.utils.conf import remove_none_values
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
+from scrapy.utils.python import without_none_values
 from scrapy import signals
 
 
@@ -20,7 +20,7 @@ class DownloadHandlers(object):
         self._schemes = {}  # stores acceptable schemes on instancing
         self._handlers = {}  # stores instanced handlers for schemes
         self._notconfigured = {}  # remembers failed handlers
-        handlers = remove_none_values(crawler.settings._getcomposite('DOWNLOAD_HANDLERS'))
+        handlers = without_none_values(crawler.settings._getcomposite('DOWNLOAD_HANDLERS'))
         for scheme, clspath in six.iteritems(handlers):
             self._schemes[scheme] = clspath
 
diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py
index c8924c04a..93fe97673 100644
--- a/scrapy/downloadermiddlewares/defaultheaders.py
+++ b/scrapy/downloadermiddlewares/defaultheaders.py
@@ -4,7 +4,7 @@ DefaultHeaders downloader middleware
 See documentation in docs/topics/downloader-middleware.rst
 """
 
-from scrapy.utils.conf import remove_none_values
+from scrapy.utils.python import without_none_values
 
 
 class DefaultHeadersMiddleware(object):
@@ -14,7 +14,7 @@ class DefaultHeadersMiddleware(object):
 
     @classmethod
     def from_crawler(cls, crawler):
-        headers = remove_none_values(crawler.settings['DEFAULT_REQUEST_HEADERS'])
+        headers = without_none_values(crawler.settings['DEFAULT_REQUEST_HEADERS'])
         return cls(headers.items())
 
     def process_request(self, request, spider):
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index fb07657d6..1e27a1e7e 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -18,11 +18,11 @@ from twisted.internet import defer, threads
 from w3lib.url import file_uri_to_path
 
 from scrapy import signals
-from scrapy.utils.conf import remove_none_values
 from scrapy.utils.ftp import ftp_makedirs_cwd
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from scrapy.utils.log import failure_to_exc_info
+from scrapy.utils.python import without_none_values
 
 logger = logging.getLogger(__name__)
 
@@ -196,7 +196,7 @@ class FeedExporter(object):
         return item
 
     def _load_components(self, setting_prefix):
-        conf = remove_none_values(self.settings._getcomposite(setting_prefix))
+        conf = without_none_values(self.settings._getcomposite(setting_prefix))
         d = {}
         for k, v in conf.items():
             try:
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 80c644657..57f2b6322 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -1,6 +1,5 @@
 import os
 import sys
-import warnings
 from operator import itemgetter
 
 import six
@@ -8,6 +7,7 @@ from six.moves.configparser import SafeConfigParser
 
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
+from scrapy.utils.python import without_none_values
 
 
 def build_component_list(compdict, convert=update_classpath):
@@ -37,15 +37,10 @@ def build_component_list(compdict, convert=update_classpath):
     if isinstance(compdict, (list, tuple)):
         _check_components(compdict)
         return type(compdict)(convert(c) for c in compdict)
-    compdict = remove_none_values(_map_keys(compdict))
+    compdict = without_none_values(_map_keys(compdict))
     return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
 
 
-def remove_none_values(compdict):
-    """Return dict with all pairs that have value 'None' removed"""
-    return {k: v for k, v in six.iteritems(compdict) if v is not None}
-
-
 def arglist_to_dict(arglist):
     """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a
     dict
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index d566783b2..1f9d02df5 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -324,3 +324,15 @@ def retry_on_eintr(function, *args, **kw):
         except IOError as e:
             if e.errno != errno.EINTR:
                 raise
+
+
+def without_none_values(iterable):
+    """Return a copy of `iterable` with all `None` entries removed.
+
+    If `iterable` is a mapping, return a dictionary where all pairs that have
+    value `None` have been removed.
+    """
+    try:
+        return {k: v for k, v in six.iteritems(iterable) if v is not None}
+    except AttributeError:
+        return type(iterable)((v for v in iterable if v is not None))
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index e94ccc49b..af15d3184 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -1,8 +1,7 @@
 import unittest
 
 from scrapy.settings import BaseSettings
-from scrapy.utils.conf import (build_component_list, arglist_to_dict,
-                               remove_none_values)
+from scrapy.utils.conf import build_component_list, arglist_to_dict
 
 
 class BuildComponentListTest(unittest.TestCase):
@@ -53,12 +52,6 @@ class BuildComponentListTest(unittest.TestCase):
 
 class UtilsConfTestCase(unittest.TestCase):
 
-    def test_remove_none_values(self):
-        comps = {'one': 1, 'none': None, 'three': 3, 'four': 4}
-        compscopy = dict(comps)
-        del compscopy['none']
-        self.assertEqual(remove_none_values(comps), compscopy)
-
     def test_arglist_to_dict(self):
         self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']),
             {'arg1': 'val1', 'arg2': 'val2'})
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index ca394ebf5..4f0834902 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -6,7 +6,8 @@ import six
 
 from scrapy.utils.python import (
     memoizemethod_noargs, isbinarytext, equal_attributes,
-    WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode)
+    WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode,
+    without_none_values)
 
 __doctests__ = ['scrapy.utils.python']
 
@@ -212,5 +213,12 @@ class UtilsPythonTestCase(unittest.TestCase):
         self.assertEqual(get_func_args(" ".join), [])
         self.assertEqual(get_func_args(operator.itemgetter(2)), [])
 
+    def test_without_none_values(self):
+        self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
+        self.assertEqual(without_none_values((1, None, 3, 4)), (1, 3, 4))
+        self.assertEqual(
+            without_none_values({'one': 1, 'none': None, 'three': 3, 'four': 4}),
+            {'one': 1, 'three': 3, 'four': 4})
+
 if __name__ == "__main__":
     unittest.main()

From 03f1720afb4a437314659a306286f440df664a0b Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Tue, 27 Oct 2015 13:56:14 +0100
Subject: [PATCH 0600/2974] Fix backwards-compatibility for users who
 explicitly set _BASE settings

---
 scrapy/settings/__init__.py     | 18 ++++++++++++------
 tests/test_settings/__init__.py | 13 +++++++++----
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 3ae2187ae..13656298b 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -116,9 +116,9 @@ class BaseSettings(MutableMapping):
     def getbool(self, name, default=False):
         """
         Get a setting value as a boolean.
-        
+
         ``1``, ``'1'``, and ``True`` return ``True``, while ``0``, ``'0'``,
-        ``False`` and ``None`` return ``False``. 
+        ``False`` and ``None`` return ``False``.
 
         For example, settings populated through environment variables set to
         ``'0'`` will return ``False`` when using this method.
@@ -203,11 +203,17 @@ class BaseSettings(MutableMapping):
         if basename in self:
             warnings.warn('_BASE settings are deprecated.',
                           category=ScrapyDeprecationWarning)
-            compsett = BaseSettings(self[name + "_BASE"], priority='default')
-            compsett.update(self[name])
+            # When users defined a _BASE setting, they explicitly don't want to
+            # use any of Scrapy's defaults. Therefore, we only use these entries
+            # from self[name] (where the defaults now live) that have a priority
+            # higher than 'default'
+            compsett = BaseSettings(self[basename], priority='default')
+            for k in self[name]:
+                prio = self[name].getpriority(k)
+                if prio > get_settings_priority('default'):
+                    compsett.set(k, self[name][k], prio)
             return compsett
-        else:
-            return self[name]
+        return self[name]
 
     def getpriority(self, name):
         """
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index bb38964ef..03e7d8686 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -252,12 +252,17 @@ class BaseSettingsTest(unittest.TestCase):
 
     def test_getcomposite(self):
         s = BaseSettings({'TEST_BASE': {1: 1, 2: 2},
-                          'TEST': BaseSettings({1: 10}),
-                          'HASNOBASE': BaseSettings({1: 1})})
+                          'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
+                          'HASNOBASE': BaseSettings({1: 1}, 'default')})
+        s['TEST'].set(4, 4, priority='project')
+        # When users specify a _BASE setting they explicitly don't want to use
+        # Scrapy's defaults, so we don't want to see anything that has a
+        # 'default' priority from TEST
         cs = s._getcomposite('TEST')
-        self.assertEqual(len(cs), 2)
-        self.assertEqual(cs[1], 10)
+        self.assertEqual(len(cs), 3)
+        self.assertEqual(cs[1], 1)
         self.assertEqual(cs[2], 2)
+        self.assertEqual(cs[4], 4)
         cs = s._getcomposite('HASNOBASE')
         self.assertEqual(len(cs), 1)
         self.assertEqual(cs[1], 1)

From c34dbe955d1a72ddef97afa1f0fb92becd9f4ca3 Mon Sep 17 00:00:00 2001
From: Pengyu CHEN <pengyu@libstarrify.so>
Date: Thu, 29 Oct 2015 14:18:59 +0800
Subject: [PATCH 0601/2974] fixed: Issue #1562 (Incorrectly picked URL in
 `scrapy.http.FormRequest.from_response` when there is a `<base>` tag)

---
 scrapy/http/request/form.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index a12a2fd07..4a9bd732e 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -11,6 +11,7 @@ from parsel.selector import create_root_node
 import six
 from scrapy.http.request import Request
 from scrapy.utils.python import to_bytes, is_listlike
+from scrapy.utils.response import get_base_url
 
 
 class FormRequest(Request):
@@ -44,7 +45,7 @@ class FormRequest(Request):
 
 def _get_form_url(form, url):
     if url is None:
-        return form.action or form.base_url
+        return urljoin(form.base_url, form.action)
     return urljoin(form.base_url, url)
 
 
@@ -58,7 +59,7 @@ def _urlencode(seq, enc):
 def _get_form(response, formname, formid, formnumber, formxpath):
     """Find the form element """
     text = response.body_as_unicode()
-    root = create_root_node(text, lxml.html.HTMLParser, base_url=response.url)
+    root = create_root_node(text, lxml.html.HTMLParser, base_url=get_base_url(response))
     forms = root.xpath('//form')
     if not forms:
         raise ValueError("No <form> element found in %s" % response)

From e379f58cad0289d725a5241606542d0e20ecd73d Mon Sep 17 00:00:00 2001
From: Pengyu CHEN <pengyu@libstarrify.so>
Date: Thu, 29 Oct 2015 14:52:31 +0800
Subject: [PATCH 0602/2974] fixed: Issue #1564 (Incorrectly picked URL in
 `scrapy.linkextractors.regex.RegexLinkExtractor` when there is a `<base>`
 tag. )

---
 scrapy/linkextractors/regex.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py
index b6f8d5d30..0fc7b079f 100644
--- a/scrapy/linkextractors/regex.py
+++ b/scrapy/linkextractors/regex.py
@@ -1,7 +1,7 @@
 import re
 from six.moves.urllib.parse import urljoin
 
-from w3lib.html import remove_tags, replace_entities, replace_escape_chars
+from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
 
 from scrapy.link import Link
 from .sgml import SgmlLinkExtractor
@@ -31,7 +31,7 @@ class RegexLinkExtractor(SgmlLinkExtractor):
             return clean_url
 
         if base_url is None:
-            base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
+            base_url = get_base_url(response_text, response_url, response_encoding)
 
         links_text = linkre.findall(response_text)
         return [Link(clean_url(url).encode(response_encoding),

From 8307c1212f48c83faf1f331f0bcafb439dbeb89c Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Mon, 10 Aug 2015 23:43:06 +0200
Subject: [PATCH 0603/2974] Add ExecutionEngine.close() method

---
 scrapy/core/engine.py | 15 +++++++++++++++
 scrapy/crawler.py     |  5 ++++-
 tests/test_crawl.py   | 21 +++++++++++++++++++++
 tests/test_engine.py  | 24 ++++++++++++++++++++++++
 4 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 992327bfe..eb2779b12 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -84,6 +84,21 @@ class ExecutionEngine(object):
         dfd = self._close_all_spiders()
         return dfd.addBoth(lambda _: self._finish_stopping_engine())
 
+    def close(self):
+        """Close the execution engine gracefully.
+
+        If it has already been started, stop it. In all cases, close all spiders
+        and the downloader.
+        """
+        if self.running:
+            # Will also close spiders and downloader
+            return self.stop()
+        elif self.open_spiders:
+            # Will also close downloader
+            return self._close_all_spiders()
+        else:
+            return defer.succeed(self.downloader.close())
+
     def pause(self):
         """Pause the execution engine"""
         self.paused = True
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index c7e3bb528..a1937f2bd 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -73,8 +73,11 @@ class Crawler(object):
             yield self.engine.open_spider(self.spider, start_requests)
             yield defer.maybeDeferred(self.engine.start)
         except Exception:
+            exc = defer.fail()
             self.crawling = False
-            raise
+            if self.engine is not None:
+                yield self.engine.close()
+            yield exc
 
     def _create_spider(self, *args, **kwargs):
         return self.spidercls.from_crawler(self, *args, **kwargs)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 6d21acab0..b2105dcfa 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -226,3 +226,24 @@ with multiples lines
         s = dict(est[0])
         self.assertEqual(s['engine.spider.name'], crawler.spider.name)
         self.assertEqual(s['len(engine.scraper.slot.active)'], 1)
+
+    @defer.inlineCallbacks
+    def test_graceful_crawl_error_handling(self):
+        """
+        Test whether errors happening anywhere in Crawler.crawl() are properly
+        reported (and not somehow swallowed) after a graceful engine shutdown.
+        The errors should not come from within Scrapy's core but from within
+        spiders/middlewares/etc., e.g. raised in Spider.start_requests(),
+        SpiderMiddleware.process_start_requests(), etc.
+        """
+
+        class TestError(Exception):
+            pass
+
+        class FaultySpider(SimpleSpider):
+            def start_requests(self):
+                raise TestError
+
+        crawler = get_crawler(FaultySpider)
+        yield self.assertFailure(crawler.crawl(), TestError)
+        self.assertFalse(crawler.crawling)
diff --git a/tests/test_engine.py b/tests/test_engine.py
index e14957eae..dad921a60 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -19,6 +19,7 @@ from twisted.web import server, static, util
 from twisted.trial import unittest
 
 from scrapy import signals
+from scrapy.core.engine import ExecutionEngine
 from scrapy.utils.test import get_crawler
 from pydispatch import dispatcher
 from tests import tests_datadir
@@ -234,6 +235,29 @@ class EngineTest(unittest.TestCase):
         self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},
                          self.run.signals_catched[signals.spider_closed])
 
+    @defer.inlineCallbacks
+    def test_close_downloader(self):
+        e = ExecutionEngine(get_crawler(TestSpider), lambda: None)
+        yield e.close()
+
+    @defer.inlineCallbacks
+    def test_close_spiders_downloader(self):
+        e = ExecutionEngine(get_crawler(TestSpider), lambda: None)
+        yield e.open_spider(TestSpider(), [])
+        self.assertEqual(len(e.open_spiders), 1)
+        yield e.close()
+        self.assertEqual(len(e.open_spiders), 0)
+
+    @defer.inlineCallbacks
+    def test_close_engine_spiders_downloader(self):
+        e = ExecutionEngine(get_crawler(TestSpider), lambda: None)
+        yield e.open_spider(TestSpider(), [])
+        e.start()
+        self.assertTrue(e.running)
+        yield e.close()
+        self.assertFalse(e.running)
+        self.assertEqual(len(e.open_spiders), 0)
+
 
 if __name__ == "__main__":
     if len(sys.argv) > 1 and sys.argv[1] == 'runserver':

From 51ca84c9b4278e92c9e08e38adfd8819c964e3be Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Thu, 29 Oct 2015 09:49:12 +0100
Subject: [PATCH 0604/2974] Add missing tests for scrapy.settings module

---
 tests/test_settings/__init__.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 03e7d8686..66b611e83 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -3,11 +3,20 @@ import unittest
 import warnings
 
 from scrapy.settings import (BaseSettings, Settings, SettingsAttribute,
-                             CrawlerSettings)
+                             CrawlerSettings, SETTINGS_PRIORITIES,
+                             get_settings_priority)
 from tests import mock
 from . import default_settings
 
 
+class SettingsGlobalFuncsTest(unittest.TestCase):
+
+    def test_get_settings_priority(self):
+        for prio_str, prio_num in six.iteritems(SETTINGS_PRIORITIES):
+            self.assertEqual(get_settings_priority(prio_str), prio_num)
+        self.assertEqual(get_settings_priority(99), 99)
+
+
 class SettingsAttributeTest(unittest.TestCase):
 
     def setUp(self):
@@ -45,6 +54,10 @@ class SettingsAttributeTest(unittest.TestCase):
         self.assertEqual(attribute.value['one'], 12)
         self.assertEqual(attribute.value['two'], 21)
 
+    def test_repr(self):
+        self.assertEqual(repr(self.attribute),
+                         "<SettingsAttribute value='value' priority=10>")
+
 
 class BaseSettingsTest(unittest.TestCase):
 
@@ -329,7 +342,6 @@ class BaseSettingsTest(unittest.TestCase):
             self.assertEqual(self.settings.get('FOO'), 'fez')
             self.assertEqual(self.settings.overrides.get('FOO'), 'fez')
 
-
     def test_deprecated_attribute_defaults(self):
         self.settings.set('BAR', 'fuz', priority='default')
         with warnings.catch_warnings(record=True) as w:
@@ -339,6 +351,14 @@ class BaseSettingsTest(unittest.TestCase):
             self.assertEqual(self.settings.defaults.get('BAR'), 'foo')
             self.assertIn('BAR', self.settings.defaults)
 
+    def test_repr(self):
+        settings = BaseSettings()
+        self.assertEqual(repr(settings), "<BaseSettings {}>")
+        attr = SettingsAttribute('testval', 15)
+        settings['testkey'] = attr
+        self.assertEqual(repr(settings),
+                         "<BaseSettings {'testkey': %s}>" % repr(attr))
+
 
 class SettingsTest(unittest.TestCase):
 

From 32ff4cc1d7e2c4fcb3427bd37ffbbe74ec95e174 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Thu, 29 Oct 2015 17:36:02 +0100
Subject: [PATCH 0605/2974] PEP8ify scrapy.settings tests

---
 tests/test_settings/__init__.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 66b611e83..4ef08bb0b 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -39,8 +39,7 @@ class SettingsAttributeTest(unittest.TestCase):
 
     def test_set_per_key_priorities(self):
         attribute = SettingsAttribute(
-                        BaseSettings({'one': 10, 'two': 20}, 0),
-                        0)
+            BaseSettings({'one': 10, 'two': 20}, 0), 0)
 
         new_dict = {'one': 11, 'two': 21}
         attribute.set(new_dict, 10)
@@ -77,7 +76,7 @@ class BaseSettingsTest(unittest.TestCase):
         self.assertEqual(attr.priority, 0)
 
     def test_set_settingsattribute(self):
-        myattr = SettingsAttribute(0, 30) # Note priority 30
+        myattr = SettingsAttribute(0, 30)  # Note priority 30
         self.settings.set('TEST_ATTR', myattr, 10)
         self.assertEqual(self.settings.get('TEST_ATTR'), 0)
         self.assertEqual(self.settings.getpriority('TEST_ATTR'), 30)
@@ -165,7 +164,8 @@ class BaseSettingsTest(unittest.TestCase):
     def test_update(self):
         settings = BaseSettings({'key_lowprio': 0}, priority=0)
         settings.set('key_highprio', 10, priority=50)
-        custom_settings = BaseSettings({'key_lowprio': 1, 'key_highprio': 11}, priority=30)
+        custom_settings = BaseSettings({'key_lowprio': 1, 'key_highprio': 11},
+                                       priority=30)
         custom_settings.set('newkey_one', None, priority=50)
         custom_dict = {'key_lowprio': 2, 'key_highprio': 12, 'newkey_two': None}
 

From 11b11c9803fa7f1ed41c0c34076336d701676235 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 6 Oct 2015 02:20:04 +0500
Subject: [PATCH 0606/2974] CrawlerRunner.create_crawler method

---
 scrapy/crawler.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index a1937f2bd..bdcfa9d0c 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -148,9 +148,7 @@ class CrawlerRunner(object):
 
         :param dict kwargs: keyword arguments to initialize the spider
         """
-        crawler = crawler_or_spidercls
-        if not isinstance(crawler_or_spidercls, Crawler):
-            crawler = self._create_crawler(crawler_or_spidercls)
+        crawler = self.create_crawler(crawler_or_spidercls)
         return self._crawl(crawler, *args, **kwargs)
 
     def _crawl(self, crawler, *args, **kwargs):
@@ -165,6 +163,21 @@ class CrawlerRunner(object):
 
         return d.addBoth(_done)
 
+    def create_crawler(self, crawler_or_spidercls):
+        """
+        Return a :class:`~scrapy.crawler.Crawler` object.
+
+        * If `crawler_or_spidercls` is a Crawler, it is returned as-is.
+        * If `crawler_or_spidercls` is a Spider subclass, a new Crawler
+          is constructed for it.
+        * If `crawler_or_spidercls` is a string, this function finds
+          a spider with this name in a Scrapy project (using spider loader),
+          then creates a Crawler instance for it.
+        """
+        if isinstance(crawler_or_spidercls, Crawler):
+            return crawler_or_spidercls
+        return self._create_crawler(crawler_or_spidercls)
+
     def _create_crawler(self, spidercls):
         if isinstance(spidercls, six.string_types):
             spidercls = self.spider_loader.load(spidercls)

From a49c82ad625ee96b50489d0b05c323b7a8107830 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 6 Oct 2015 17:23:47 +0500
Subject: [PATCH 0607/2974] TST improve CrawlerRunner tests

* use CrawlerRunner.create_crawler instead of get_crawler helper in test_crawl;
* add a test for loading spiders by name;
* add a test for passing Crawler objects instead of Spider objects;
* add a test for CrawlerRunner.join
---
 scrapy/utils/test.py                |  2 +-
 tests/test_crawl.py                 | 57 +++++++++++++++++++----------
 tests/test_spiderloader/__init__.py | 13 +++++++
 3 files changed, 52 insertions(+), 20 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index bec9bdda9..b1110cbd7 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -30,7 +30,7 @@ def get_crawler(spidercls=None, settings_dict=None):
     from scrapy.spiders import Spider
 
     runner = CrawlerRunner(Settings(settings_dict))
-    return runner._create_crawler(spidercls or Spider)
+    return runner.create_crawler(spidercls or Spider)
 
 def get_pythonpath():
     """Return a PYTHONPATH suitable to use in processes so that they find this
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index b2105dcfa..814eb30d2 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -6,12 +6,12 @@ from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 
-from scrapy.utils.test import get_crawler
+from scrapy.http import Request
+from scrapy.crawler import CrawlerRunner
 from tests import mock
 from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
     BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
 from tests.mockserver import MockServer
-from scrapy.http import Request
 
 
 class CrawlTestCase(TestCase):
@@ -19,13 +19,14 @@ class CrawlTestCase(TestCase):
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
+        self.runner = CrawlerRunner()
 
     def tearDown(self):
         self.mockserver.__exit__(None, None, None)
 
     @defer.inlineCallbacks
     def test_follow_all(self):
-        crawler = get_crawler(FollowAllSpider)
+        crawler = self.runner.create_crawler(FollowAllSpider)
         yield crawler.crawl()
         self.assertEqual(len(crawler.spider.urls_visited), 11)  # 10 + start_url
 
@@ -41,7 +42,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def _test_delay(self, delay, randomize):
         settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize}
-        crawler = get_crawler(FollowAllSpider, settings)
+        crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
         yield crawler.crawl(maxlatency=delay * 2)
         t = crawler.spider.times
         totaltime = t[-1] - t[0]
@@ -52,7 +53,7 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_timeout_success(self):
-        crawler = get_crawler(DelaySpider)
+        crawler = self.runner.create_crawler(DelaySpider)
         yield crawler.crawl(n=0.5)
         self.assertTrue(crawler.spider.t1 > 0)
         self.assertTrue(crawler.spider.t2 > 0)
@@ -60,7 +61,7 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_timeout_failure(self):
-        crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
+        crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider)
         yield crawler.crawl(n=0.5)
         self.assertTrue(crawler.spider.t1 > 0)
         self.assertTrue(crawler.spider.t2 == 0)
@@ -75,14 +76,14 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_retry_503(self):
-        crawler = get_crawler(SimpleSpider)
+        crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
             yield crawler.crawl("http://localhost:8998/status?n=503")
         self._assert_retried(l)
 
     @defer.inlineCallbacks
     def test_retry_conn_failed(self):
-        crawler = get_crawler(SimpleSpider)
+        crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
             yield crawler.crawl("http://localhost:65432/status?n=503")
         self._assert_retried(l)
@@ -91,7 +92,7 @@ class CrawlTestCase(TestCase):
     def test_retry_dns_error(self):
         with mock.patch('socket.gethostbyname',
                         side_effect=socket.gaierror(-5, 'No address associated with hostname')):
-            crawler = get_crawler(SimpleSpider)
+            crawler = self.runner.create_crawler(SimpleSpider)
             with LogCapture() as l:
                 yield crawler.crawl("http://example.com/")
             self._assert_retried(l)
@@ -99,7 +100,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_bug_before_yield(self):
         with LogCapture('scrapy', level=logging.ERROR) as l:
-            crawler = get_crawler(BrokenStartRequestsSpider)
+            crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_before_yield=1)
 
         self.assertEqual(len(l.records), 1)
@@ -110,7 +111,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_bug_yielding(self):
         with LogCapture('scrapy', level=logging.ERROR) as l:
-            crawler = get_crawler(BrokenStartRequestsSpider)
+            crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_yielding=1)
 
         self.assertEqual(len(l.records), 1)
@@ -121,7 +122,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_lazyness(self):
         settings = {"CONCURRENT_REQUESTS": 1}
-        crawler = get_crawler(BrokenStartRequestsSpider, settings)
+        crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider)
         yield crawler.crawl()
         #self.assertTrue(False, crawler.spider.seedsseen)
         #self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
@@ -130,7 +131,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_dupes(self):
         settings = {"CONCURRENT_REQUESTS": 1}
-        crawler = get_crawler(DuplicateStartRequestsSpider, settings)
+        crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider)
         yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3)
         self.assertEqual(crawler.spider.visited, 6)
 
@@ -159,7 +160,7 @@ Connection: close
 foo body
 with multiples lines
 '''})
-        crawler = get_crawler(SimpleSpider)
+        crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
             yield crawler.crawl("http://localhost:8998/raw?{0}".format(query))
         self.assertEqual(str(l).count("Got response 200"), 1)
@@ -167,7 +168,7 @@ with multiples lines
     @defer.inlineCallbacks
     def test_retry_conn_lost(self):
         # connection lost after receiving data
-        crawler = get_crawler(SimpleSpider)
+        crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
             yield crawler.crawl("http://localhost:8998/drop?abort=0")
         self._assert_retried(l)
@@ -175,7 +176,7 @@ with multiples lines
     @defer.inlineCallbacks
     def test_retry_conn_aborted(self):
         # connection lost before receiving data
-        crawler = get_crawler(SimpleSpider)
+        crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
             yield crawler.crawl("http://localhost:8998/drop?abort=1")
         self._assert_retried(l)
@@ -194,7 +195,7 @@ with multiples lines
         req0.meta['next'] = req1
         req1.meta['next'] = req2
         req2.meta['next'] = req3
-        crawler = get_crawler(SingleRequestSpider)
+        crawler = self.runner.create_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=req0)
         # basic asserts in case of weird communication errors
         self.assertIn('responses', crawler.spider.meta)
@@ -220,7 +221,7 @@ with multiples lines
         def cb(response):
             est.append(get_engine_status(crawler.engine))
 
-        crawler = get_crawler(SingleRequestSpider)
+        crawler = self.runner.create_crawler(SingleRequestSpider)
         yield crawler.crawl(seed='http://localhost:8998/', callback_func=cb)
         self.assertEqual(len(est), 1, est)
         s = dict(est[0])
@@ -244,6 +245,24 @@ with multiples lines
             def start_requests(self):
                 raise TestError
 
-        crawler = get_crawler(FaultySpider)
+        crawler = self.runner.create_crawler(FaultySpider)
         yield self.assertFailure(crawler.crawl(), TestError)
         self.assertFalse(crawler.crawling)
+
+    @defer.inlineCallbacks
+    def test_crawlerrunner_accepts_crawler(self):
+        crawler = self.runner.create_crawler(SimpleSpider)
+        with LogCapture() as log:
+            yield self.runner.crawl(crawler, "http://localhost:8998/status?n=200")
+        self.assertIn("Got response 200", str(log))
+
+    @defer.inlineCallbacks
+    def test_crawl_multiple(self):
+        self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=200")
+        self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=503")
+
+        with LogCapture() as log:
+            yield self.runner.join()
+
+        self._assert_retried(log)
+        self.assertIn("Got response 200", str(log))
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 7cb5e299b..68dca2e98 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -8,10 +8,12 @@ from twisted.trial import unittest
 
 # ugly hack to avoid cyclic imports of scrapy.spiders when running this test
 # alone
+import scrapy
 from scrapy.interfaces import ISpiderLoader
 from scrapy.spiderloader import SpiderLoader
 from scrapy.settings import Settings
 from scrapy.http import Request
+from scrapy.crawler import CrawlerRunner
 
 module_dir = os.path.dirname(os.path.abspath(__file__))
 
@@ -76,3 +78,14 @@ class SpiderLoaderTest(unittest.TestCase):
         settings = Settings({'SPIDER_MODULES': [module]})
         self.spider_loader = SpiderLoader.from_settings(settings)
         assert len(self.spider_loader._spiders) == 0
+
+    def test_crawler_runner_loading(self):
+        module = 'tests.test_spiderloader.test_spiders.spider1'
+        runner = CrawlerRunner({'SPIDER_MODULES': [module]})
+
+        self.assertRaisesRegexp(KeyError, 'Spider not found',
+                                runner.create_crawler, 'spider2')
+
+        crawler = runner.create_crawler('spider1')
+        self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
+        self.assertEqual(crawler.spidercls.name, 'spider1')

From 0000b6e9efc45d90b4ecd6be8c41d39a7f8b0578 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 30 Oct 2015 20:27:03 +0500
Subject: [PATCH 0608/2974] TST cleanup scrapy.utils.test.get_crawler

---
 scrapy/utils/test.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index b1110cbd7..51edfd353 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -26,10 +26,9 @@ def get_crawler(spidercls=None, settings_dict=None):
     priority.
     """
     from scrapy.crawler import CrawlerRunner
-    from scrapy.settings import Settings
     from scrapy.spiders import Spider
 
-    runner = CrawlerRunner(Settings(settings_dict))
+    runner = CrawlerRunner(settings_dict)
     return runner.create_crawler(spidercls or Spider)
 
 def get_pythonpath():

From 98a2e77a75f0b203ce9841801e342a11f15ad419 Mon Sep 17 00:00:00 2001
From: Leonid Amirov <leonid.amirov@gmail.com>
Date: Mon, 2 Nov 2015 15:30:49 +0300
Subject: [PATCH 0609/2974] issue GH #1550 - fixed error: shell command wasn't
 accepting files URIs

---
 scrapy/commands/shell.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 92ebbe605..f10da4370 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -5,11 +5,11 @@ See documentation in docs/topics/shell.rst
 """
 
 from threading import Thread
+from w3lib.url import any_to_uri
 
 from scrapy.commands import ScrapyCommand
 from scrapy.shell import Shell
 from scrapy.http import Request
-from scrapy.utils.url import add_http_if_no_scheme
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 
 
@@ -43,7 +43,8 @@ class Command(ScrapyCommand):
     def run(self, args, opts):
         url = args[0] if args else None
         if url:
-            url = add_http_if_no_scheme(url)
+            url = any_to_uri(url)
+
         spider_loader = self.crawler_process.spider_loader
 
         spidercls = DefaultSpider

From a41c64bfb97a7bba8ec138345d48b94d8734d27e Mon Sep 17 00:00:00 2001
From: Leonid Amirov <leonid.amirov@gmail.com>
Date: Mon, 2 Nov 2015 16:06:21 +0300
Subject: [PATCH 0610/2974] issue GH #1550 - fixed bugs in
 scrapy.utils.url.add_http_if_no_scheme(): when given URI where scheme is
 present, but not 'http' the function gave bad result

---
 scrapy/utils/url.py | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 398407a64..3eac5fb3d 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -6,6 +6,7 @@ Some of the functions that used to be imported from this module have been moved
 to the w3lib.url module. Always import those from there instead.
 """
 import posixpath
+from urlparse import urlsplit, urlunsplit
 from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag,
                                     urlparse, parse_qsl, urlencode,
                                     unquote)
@@ -114,10 +115,16 @@ def escape_ajax(url):
 
 def add_http_if_no_scheme(url):
     """Add http as the default scheme if it is missing from the url."""
-    if url.startswith('//'):
-        url = 'http:' + url
-        return url
-    parser = parse_url(url)
-    if not parser.scheme or not parser.netloc:
-        url = 'http://' + url
-    return url
+    parts = urlsplit(url)
+    scheme = parts.scheme or "http"
+    if parts.netloc:
+        netloc = parts.netloc
+        path = parts.path
+    else:
+        path_parts = url.split("/", 1)
+        netloc = path_parts[0]
+        path = path_parts[1] if len(path_parts) > 1 else "/"
+
+    return urlunsplit((
+        scheme, netloc, path, parts.query, parts.fragment
+    ))

From bc9db65358bcae3fc228d4b8099d319cba479ff2 Mon Sep 17 00:00:00 2001
From: Leonid Amirov <leonid.amirov@gmail.com>
Date: Mon, 2 Nov 2015 16:08:19 +0300
Subject: [PATCH 0611/2974] issue GH #1550 - scrapy shell argument fixes:
 "example.com" requests "http://example.com"; "example" requests
 "file://example"; "./example.com" requests "file://example.com"

---
 scrapy/commands/shell.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index f10da4370..cb441bc9d 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -5,11 +5,13 @@ See documentation in docs/topics/shell.rst
 """
 
 from threading import Thread
+import urlparse
 from w3lib.url import any_to_uri
 
 from scrapy.commands import ScrapyCommand
 from scrapy.shell import Shell
 from scrapy.http import Request
+from scrapy.utils.url import add_http_if_no_scheme
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 
 
@@ -43,7 +45,16 @@ class Command(ScrapyCommand):
     def run(self, args, opts):
         url = args[0] if args else None
         if url:
-            url = any_to_uri(url)
+            parts = urlparse.urlsplit(url)
+            if not parts.scheme:
+                if "." not in parts.path.split("/", 1)[0]:
+                    url = any_to_uri(url)
+
+                for pattern in ["/", "./", "../"]:
+                    if url.startswith(pattern):
+                        url = any_to_uri(url)
+                        break
+                url = add_http_if_no_scheme(url)
 
         spider_loader = self.crawler_process.spider_loader
 

From e19bf4aecc9027fd4023282a371c13f77fadd510 Mon Sep 17 00:00:00 2001
From: Pengyu CHEN <pengyu@libstarrify.so>
Date: Mon, 2 Nov 2015 22:52:41 +0800
Subject: [PATCH 0612/2974] added: Test case for the fix

---
 tests/test_http_request.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index ff0941961..60fd855dd 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -801,6 +801,25 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs[b'test2'], [b'val2'])
         self.assertEqual(fs[b'button1'], [b''])
 
+    def test_html_base_form_action(self):
+        response = _buildresponse(
+            """
+            <html>
+                <head>
+                    <base href="http://b.com/">
+                </head>
+                <body>
+                    <form action="test_form">
+                    </form>
+                </body>
+            </html>
+            """,
+            url='http://a.com/'
+        )
+        req = self.request_class.from_response(response)
+        self.assertEqual(req.url, 'http://b.com/test_form')
+
+
 def _buildresponse(body, **kwargs):
     kwargs.setdefault('body', body)
     kwargs.setdefault('url', 'http://example.com')

From 94486bb294a6e765efe14affddb1df355a6c298b Mon Sep 17 00:00:00 2001
From: Pengyu CHEN <pengyu@libstarrify.so>
Date: Mon, 2 Nov 2015 23:00:42 +0800
Subject: [PATCH 0613/2974] added: Test case for the fix.

---
 tests/test_linkextractors_deprecated.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py
index e3664f8d8..89dcb75c2 100644
--- a/tests/test_linkextractors_deprecated.py
+++ b/tests/test_linkextractors_deprecated.py
@@ -190,3 +190,20 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
             Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
             Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
         ])
+
+    def test_html_base_href(self):
+        html = """
+        <html>
+            <head>
+                <base href="http://b.com/">
+            </head>
+            <body>
+                <a href="test.html"></a>
+            </body>
+        </html>
+        """
+        response = HtmlResponse("http://a.com/", body=html)
+        lx = RegexLinkExtractor()
+        self.assertEqual([link for link in lx.extract_links(response)], [
+            Link(url='http://b.com/test.html', text=u'', nofollow=False),
+        ])

From dd45b31fe42635ff43df62c667f1ace5f9169737 Mon Sep 17 00:00:00 2001
From: Leonid Amirov <leonid.amirov@gmail.com>
Date: Tue, 3 Nov 2015 14:32:30 +0300
Subject: [PATCH 0614/2974] issue GH #1550 - rewritten add_http_if_no_scheme()

---
 scrapy/utils/url.py | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 3eac5fb3d..0e36003ad 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -6,7 +6,7 @@ Some of the functions that used to be imported from this module have been moved
 to the w3lib.url module. Always import those from there instead.
 """
 import posixpath
-from urlparse import urlsplit, urlunsplit
+import re
 from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag,
                                     urlparse, parse_qsl, urlencode,
                                     unquote)
@@ -115,16 +115,10 @@ def escape_ajax(url):
 
 def add_http_if_no_scheme(url):
     """Add http as the default scheme if it is missing from the url."""
-    parts = urlsplit(url)
-    scheme = parts.scheme or "http"
-    if parts.netloc:
-        netloc = parts.netloc
-        path = parts.path
-    else:
-        path_parts = url.split("/", 1)
-        netloc = path_parts[0]
-        path = path_parts[1] if len(path_parts) > 1 else "/"
+    match = re.match(r"^\w+://", url, flags=re.I)
+    parts = urlparse(url)
+    if not match:
+        scheme = "http:" if parts.netloc else "http://"
+        url = scheme + url
 
-    return urlunsplit((
-        scheme, netloc, path, parts.query, parts.fragment
-    ))
+    return url

From 97b51ea33b6b96241ef87861619e6820ee9e765c Mon Sep 17 00:00:00 2001
From: Leonid Amirov <leonid.amirov@gmail.com>
Date: Tue, 3 Nov 2015 14:57:37 +0300
Subject: [PATCH 0615/2974] issue GH #1550 - six library is used instead of
 urlparse for python3 compatibility

---
 scrapy/commands/shell.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index cb441bc9d..6f58a2300 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -4,8 +4,9 @@ Scrapy Shell
 See documentation in docs/topics/shell.rst
 """
 
+import re
+from six.moves.urllib.parse import urlparse, urlunparse
 from threading import Thread
-import urlparse
 from w3lib.url import any_to_uri
 
 from scrapy.commands import ScrapyCommand
@@ -45,7 +46,7 @@ class Command(ScrapyCommand):
     def run(self, args, opts):
         url = args[0] if args else None
         if url:
-            parts = urlparse.urlsplit(url)
+            parts = urlparse(url)
             if not parts.scheme:
                 if "." not in parts.path.split("/", 1)[0]:
                     url = any_to_uri(url)

From e66f64989409ddf9117f854934d52b2772aace74 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Fri, 6 Nov 2015 01:14:49 +0100
Subject: [PATCH 0616/2974] Bring back _BASE settings

---
 docs/topics/downloader-middleware.rst |  22 +++--
 docs/topics/extensions.rst            |  13 +--
 docs/topics/feed-exports.rst          |  47 +++++++---
 docs/topics/settings.rst              | 128 +++++++++++++++++---------
 docs/topics/spider-middleware.rst     |  22 +++--
 scrapy/settings/default_settings.py   |  25 +++--
 6 files changed, 169 insertions(+), 88 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 08d8f3edf..cc0254d29 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -23,20 +23,22 @@ Here's an example::
         'myproject.middlewares.CustomDownloaderMiddleware': 543,
     }
 
-The specified :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the
-default one (i.e. it does not overwrite it) and then sorted by order to get the
-final sorted list of enabled middlewares: the first middleware is the one
-closer to the engine and the last is the one closer to the downloader.
+The :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the
+:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant
+to be overridden) and then sorted by order to get the final sorted list of
+enabled middlewares: the first middleware is the one closer to the engine and
+the last is the one closer to the downloader.
 
-To decide which order to assign to your middleware see the default
-:setting:`DOWNLOADER_MIDDLEWARES` setting and pick a value according to
+To decide which order to assign to your middleware see the
+:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting and pick a value according to
 where you want to insert the middleware. The order does matter because each
 middleware performs a different action and your middleware could depend on some
 previous (or subsequent) middleware being applied.
 
-If you want to disable a built-in middleware you must define it in your
-project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None`` as its
-value. For example, if you want to disable the user-agent middleware::
+If you want to disable a built-in middleware (the ones defined in
+:setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it
+in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign `None`
+as its value.  For example, if you want to disable the user-agent middleware::
 
     DOWNLOADER_MIDDLEWARES = {
         'myproject.middlewares.CustomDownloaderMiddleware': 543,
@@ -162,7 +164,7 @@ middleware, see the :ref:`downloader middleware usage guide
 <topics-downloader-middleware>`.
 
 For a list of the components enabled by default (and their orders) see the
-:setting:`DOWNLOADER_MIDDLEWARES` setting.
+:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting.
 
 .. _cookies-mw:
 
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 11c0aadb6..847353868 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -42,13 +42,14 @@ by a string: the full Python path to the extension's class name. For example::
 
 As you can see, the :setting:`EXTENSIONS` setting is a dict where the keys are
 the extension paths, and their values are the orders, which define the
-extension *loading* order. The specified :setting:`EXTENSIONS` setting is merged
-with the default one (i.e. it does not overwrite it) and then sorted by order
-to get the final sorted list of enabled extensions.
+extension *loading* order. The :setting:`EXTENSIONS` setting is merged with the
+:setting:`EXTENSIONS_BASE` setting defined in Scrapy (and not meant to be
+overridden) and then sorted by order to get the final sorted list of enabled
+extensions.
 
 As extensions typically do not depend on each other, their loading order is
-irrelevant in most cases. This is why the default :setting:`EXTENSIONS` setting
-defines all extensions with the same order (``500``). However, this feature can
+irrelevant in most cases. This is why the :setting:`EXTENSIONS_BASE` setting
+defines all extensions with the same order (``0``). However, this feature can
 be exploited if you need to add an extension which depends on other extensions
 already loaded.
 
@@ -63,7 +64,7 @@ Disabling an extension
 ======================
 
 In order to disable an extension that comes enabled by default (ie. those
-included in the default :setting:`EXTENSIONS` setting) you must set its order to
+included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to
 ``None``. For example::
 
     EXTENSIONS = {
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index d8b8da166..03c6fb3fb 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -265,6 +265,16 @@ Whether to export empty feeds (ie. feeds with no items).
 FEED_STORAGES
 -------------
 
+Default:: ``{}``
+
+A dict containing additional feed storage backends supported by your project.
+The keys are URI schemes and the values are paths to storage classes.
+
+.. setting:: FEED_STORAGES_BASE
+
+FEED_STORAGES_BASE
+------------------
+
 Default::
 
     {
@@ -275,19 +285,30 @@ Default::
         'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
     }
 
-A dict containing all feed storage backends supported by your project. The keys
-are URI schemes and the values are paths to storage classes.
+A dict containing the built-in feed storage backends supported by Scrapy. You
+can disable any of these backends by assigning ``None`` to their URI scheme in
+:setting:`FEED_STORAGES`. E.g., to disable the built-in FTP storage backend
+(without replacement), place this in your ``settings.py``::
 
-When you set :setting:`FEED_STORAGES` manually, e.g. in your project's settings
-module, it will be merged with the default, not overwrite it. If you want to
-disable any of the default feed storage backends, you must assign ``None`` as
-their value.
+    FEED_STORAGES = {
+        'ftp': None,
+    }
 
 .. setting:: FEED_EXPORTERS
 
 FEED_EXPORTERS
 --------------
 
+Default:: ``{}``
+
+A dict containing additional exporters supported by your project. The keys are
+serialization formats and the values are paths to :ref:`Item exporter
+<topics-exporters>` classes.
+
+.. setting:: FEED_EXPORTERS_BASE
+
+FEED_EXPORTERS_BASE
+-------------------
 Default::
 
     {
@@ -300,14 +321,14 @@ Default::
         'pickle': 'scrapy.exporters.PickleItemExporter',
     }
 
-A dict containing all feed exporters supported by your project. The keys are
-URI schemes and the values are paths to :ref:`Item exporter <topics-exporters>`
-classes.
+A dict containing the built-in feed exporters supported by Scrapy. You can
+disable any of these exporters by assigning ``None`` to their serialization
+format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
+(without replacement), place this in your ``settings.py``::
 
-When you set :setting:`FEED_EXPORTERS` manually, e.g. in your project's settings
-module, it will be merged with the default, not overwrite it. If you want to
-disable any of the default feed exporters, you must assign ``None`` as their
-value.
+    FEED_EXPORTERS = {
+        'csv': None,
+    }
 
 .. _URI: http://en.wikipedia.org/wiki/Uniform_Resource_Identifier
 .. _Amazon S3: http://aws.amazon.com/s3/
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 8908fae7e..aa0417e1a 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -269,11 +269,6 @@ Default::
 The default headers used for Scrapy HTTP Requests. They're populated in the
 :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`.
 
-When you set :setting:`DEFAULT_REQUEST_HEADERS` manually, e.g. in your
-project's settings module, it will be merged with the default, not overwrite it.
-If you want to disable any of the default request headers (and not replace them)
-you must assign ``None`` as their value.
-
 .. setting:: DEPTH_LIMIT
 
 DEPTH_LIMIT
@@ -355,6 +350,16 @@ The downloader to use for crawling.
 DOWNLOADER_MIDDLEWARES
 ----------------------
 
+Default:: ``{}``
+
+A dict containing the downloader middlewares enabled in your project, and their
+orders. For more info see :ref:`topics-downloader-middleware-setting`.
+
+.. setting:: DOWNLOADER_MIDDLEWARES_BASE
+
+DOWNLOADER_MIDDLEWARES_BASE
+---------------------------
+
 Default::
 
     {
@@ -375,16 +380,11 @@ Default::
         'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
     }
 
-A dict containing the downloader middlewares enabled in your project, and their
-orders. Low orders are closer to the engine, high orders are closer to the
-downloader.
-
-When you set :setting:`DOWNLOADER_MIDDLEWARES` manually, e.g. in your project's
-settings module, it will be merged with the default, not overwrite it. If you
-want to disable any of the default downloader middlewares you must assign
-``None`` as their value.
-
-For more info see :ref:`topics-downloader-middleware-setting`.
+A dict containing the downloader middlewares enabled by default in Scrapy. Low
+orders are closer to the engine, high orders are closer to the downloader. You
+should never modify this setting in your project, modify
+:setting:`DOWNLOADER_MIDDLEWARES` instead.  For more info see
+:ref:`topics-downloader-middleware-setting`.
 
 .. setting:: DOWNLOADER_STATS
 
@@ -425,6 +425,16 @@ spider attribute.
 DOWNLOAD_HANDLERS
 -----------------
 
+Default: ``{}``
+
+A dict containing the request downloader handlers enabled in your project.
+See :setting:`DOWNLOAD_HANDLERS_BASE` for example format.
+
+.. setting:: DOWNLOAD_HANDLERS_BASE
+
+DOWNLOAD_HANDLERS_BASE
+----------------------
+
 Default::
 
     {
@@ -436,15 +446,16 @@ Default::
     }
 
 
-A dict containing the request downloader handlers enabled in your project.
+A dict containing the request download handlers enabled by default in Scrapy.
+You should never modify this setting in your project, modify
+:setting:`DOWNLOAD_HANDLERS` instead.
 
-When you set :setting:`DOWNLOAD_HANDLERS` manually, e.g. in your project's
-settings module, it will be merged with the default, not overwrite it. If you
-want to disable any of the default download handlers you must assign ``None``
-as their value. For example, if you want to disable the file download handler::
+You can disable any of these download handlers by assigning ``None`` to their
+URI scheme in :setting:`DOWNLOAD_HANDLERS`. E.g., to disable the built-in FTP
+handler (without replacement), place this in your ``settings.py``::
 
     DOWNLOAD_HANDLERS = {
-        'file': None,
+        'ftp': None,
     }
 
 .. setting:: DOWNLOAD_TIMEOUT
@@ -544,6 +555,15 @@ to ``vi`` (on Unix systems) or the IDLE editor (on Windows).
 EXTENSIONS
 ----------
 
+Default:: ``{}``
+
+A dict containing the extensions enabled in your project, and their orders.
+
+.. setting:: EXTENSIONS_BASE
+
+EXTENSIONS_BASE
+---------------
+
 Default::
 
     {
@@ -558,15 +578,10 @@ Default::
         'scrapy.extensions.throttle.AutoThrottle': 0,
     }
 
-A dict containing the extensions enabled in your project, and their orders. By
-default, this setting contains all stable built-in extensions. Keep in mind that
+A dict containing the extensions available by default in Scrapy, and their
+orders. This setting contains all stable built-in extensions. Keep in mind that
 some of them need to be enabled through a setting.
 
-When you set :setting:`EXTENSIONS` manually, e.g. in your project's settings
-module, it will be merged with the default, not overwrite it. If you want to
-disable any of the default enabled extensions you must assign ``None`` as their
-value.
-
 For more information See the :ref:`extensions user guide  <topics-extensions>`
 and the :ref:`list of available extensions <topics-extensions-ref>`.
 
@@ -589,6 +604,16 @@ Example::
        'mybot.pipelines.validate.StoreMyItem': 800,
    }
 
+.. setting:: ITEM_PIPELINES_BASE
+
+ITEM_PIPELINES_BASE
+-------------------
+
+Default: ``{}``
+
+A dict containing the pipelines enabled by default in Scrapy. You should never
+modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
+
 .. setting:: LOG_ENABLED
 
 LOG_ENABLED
@@ -878,6 +903,16 @@ The scheduler to use for crawling.
 SPIDER_CONTRACTS
 ----------------
 
+Default:: ``{}``
+
+A dict containing the spider contracts enabled in your project, used for
+testing spiders. For more info see :ref:`topics-contracts`.
+
+.. setting:: SPIDER_CONTRACTS_BASE
+
+SPIDER_CONTRACTS_BASE
+---------------------
+
 Default::
 
     {
@@ -886,13 +921,17 @@ Default::
         'scrapy.contracts.default.ScrapesContract': 3,
     }
 
-A dict containing the scrapy contracts enabled in your project, used for
-testing spiders. For more info see :ref:`topics-contracts`.
+A dict containing the scrapy contracts enabled by default in Scrapy. You should
+never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS`
+instead. For more info see :ref:`topics-contracts`.
 
-When you set :setting:`SPIDER_CONTRACTS` manually, e.g. in your project's
-settings module, it will be merged with the default, not overwrite it. If you
-want to disable any of the default contracts you must assign ``None`` as their
-value.
+You can disable any of these contracts by assigning ``None`` to their class
+path in :setting:`SPIDER_CONTRACTS`. E.g., to disable the built-in
+``ScrapesContract``, place this in your ``settings.py``::
+
+    SPIDER_CONTRACTS = {
+        'scrapy.contracts.default.ScrapesContract': None,
+    }
 
 .. setting:: SPIDER_LOADER_CLASS
 
@@ -909,6 +948,16 @@ The class that will be used for loading spiders, which must implement the
 SPIDER_MIDDLEWARES
 ------------------
 
+Default:: ``{}``
+
+A dict containing the spider middlewares enabled in your project, and their
+orders. For more info see :ref:`topics-spider-middleware-setting`.
+
+.. setting:: SPIDER_MIDDLEWARES_BASE
+
+SPIDER_MIDDLEWARES_BASE
+-----------------------
+
 Default::
 
     {
@@ -919,14 +968,9 @@ Default::
         'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,
     }
 
-A dict containing the spider middlewares enabled in your project, and their
-orders. Low orders are closer to the engine, high orders are closer to the
-spider. For more info see :ref:`topics-spider-middleware-setting`.
-
-When you set :setting:`SPIDER_MIDDLEWARES` manually, e.g. in your project's
-settings module, it will be merged with the default, not overwrite it. If you
-want to disable any of the default spider middlewares you must assign ``None``
-as their value.
+A dict containing the spider middlewares enabled by default in Scrapy, and
+their orders. Low orders are closer to the engine, high orders are closer to
+the spider. For more info see :ref:`topics-spider-middleware-setting`.
 
 .. setting:: SPIDER_MODULES
 
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index d448801d3..84daaaa55 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -24,20 +24,22 @@ Here's an example::
         'myproject.middlewares.CustomSpiderMiddleware': 543,
     }
 
-The specified :setting:`SPIDER_MIDDLEWARES` setting is merged with the default
-one (i.e. it does not overwrite it) and then sorted by order to get the final
-sorted list of enabled middlewares: the first middleware is the one closer to
-the engine and the last is the one closer to the spider.
+The :setting:`SPIDER_MIDDLEWARES` setting is merged with the
+:setting:`SPIDER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant to
+be overridden) and then sorted by order to get the final sorted list of enabled
+middlewares: the first middleware is the one closer to the engine and the last
+is the one closer to the spider.
 
-To decide which order to assign to your middleware see the default
-:setting:`SPIDER_MIDDLEWARES` setting and pick a value according to where
+To decide which order to assign to your middleware see the
+:setting:`SPIDER_MIDDLEWARES_BASE` setting and pick a value according to where
 you want to insert the middleware. The order does matter because each
 middleware performs a different action and your middleware could depend on some
 previous (or subsequent) middleware being applied.
 
-If you want to disable a builtin middleware you must define it in your project's
-:setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its value. For
-example, if you want to disable the off-site middleware::
+If you want to disable a builtin middleware (the ones defined in
+:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
+in your project :setting:`SPIDER_MIDDLEWARES` setting and assign `None` as its
+value.  For example, if you want to disable the off-site middleware::
 
     SPIDER_MIDDLEWARES = {
         'myproject.middlewares.CustomSpiderMiddleware': 543,
@@ -171,7 +173,7 @@ information on how to use them and how to write your own spider middleware, see
 the :ref:`spider middleware usage guide <topics-spider-middleware>`.
 
 For a list of the components enabled by default (and their orders) see the
-:setting:`SPIDER_MIDDLEWARES` setting.
+:setting:`SPIDER_MIDDLEWARES_BASE` setting.
 
 DepthMiddleware
 ---------------
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 375efcdbb..8435b0354 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -63,7 +63,8 @@ DNS_TIMEOUT = 60
 
 DOWNLOAD_DELAY = 0
 
-DOWNLOAD_HANDLERS = {
+DOWNLOAD_HANDLERS = {}
+DOWNLOAD_HANDLERS_BASE = {
     'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
     'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
     'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
@@ -81,7 +82,9 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader'
 DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
 DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
 
-DOWNLOADER_MIDDLEWARES = {
+DOWNLOADER_MIDDLEWARES = {}
+
+DOWNLOADER_MIDDLEWARES_BASE = {
     # Engine side
     'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
     'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
@@ -113,7 +116,9 @@ except KeyError:
     else:
         EDITOR = 'vi'
 
-EXTENSIONS = {
+EXTENSIONS = {}
+
+EXTENSIONS_BASE = {
     'scrapy.extensions.corestats.CoreStats': 0,
     'scrapy.extensions.telnet.TelnetConsole': 0,
     'scrapy.extensions.memusage.MemoryUsage': 0,
@@ -130,14 +135,16 @@ FEED_URI_PARAMS = None  # a function to extend uri arguments
 FEED_FORMAT = 'jsonlines'
 FEED_STORE_EMPTY = False
 FEED_EXPORT_FIELDS = None
-FEED_STORAGES = {
+FEED_STORAGES = {}
+FEED_STORAGES_BASE = {
     '': 'scrapy.extensions.feedexport.FileFeedStorage',
     'file': 'scrapy.extensions.feedexport.FileFeedStorage',
     'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_EXPORTERS = {
+FEED_EXPORTERS = {}
+FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',
     'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
     'jl': 'scrapy.exporters.JsonLinesItemExporter',
@@ -163,6 +170,7 @@ HTTPCACHE_GZIP = False
 ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
 
 ITEM_PIPELINES = {}
+ITEM_PIPELINES_BASE = {}
 
 LOG_ENABLED = True
 LOG_ENCODING = 'utf-8'
@@ -221,7 +229,9 @@ SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
 
 SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
 
-SPIDER_MIDDLEWARES = {
+SPIDER_MIDDLEWARES = {}
+
+SPIDER_MIDDLEWARES_BASE = {
     # Engine side
     'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
     'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,
@@ -248,7 +258,8 @@ TELNETCONSOLE_ENABLED = 1
 TELNETCONSOLE_PORT = [6023, 6073]
 TELNETCONSOLE_HOST = '127.0.0.1'
 
-SPIDER_CONTRACTS = {
+SPIDER_CONTRACTS = {}
+SPIDER_CONTRACTS_BASE = {
     'scrapy.contracts.default.UrlContract': 1,
     'scrapy.contracts.default.ReturnsContract': 2,
     'scrapy.contracts.default.ScrapesContract': 3,

From b6a023ce987a064b222b1aa2de03a50991f387fe Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Fri, 6 Nov 2015 11:35:59 +0100
Subject: [PATCH 0617/2974] Add backwards compatibility for
 build_component_list

---
 scrapy/utils/conf.py     | 14 ++++++++++----
 tests/test_utils_conf.py | 37 +++++++++++++++++++++++++------------
 2 files changed, 35 insertions(+), 16 deletions(-)

diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 57f2b6322..e8af90f11 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -10,7 +10,7 @@ from scrapy.utils.deprecate import update_classpath
 from scrapy.utils.python import without_none_values
 
 
-def build_component_list(compdict, convert=update_classpath):
+def build_component_list(compdict, custom=None, convert=update_classpath):
     """Compose a component list from a { class: order } dictionary."""
 
     def _check_components(complist):
@@ -34,9 +34,15 @@ def build_component_list(compdict, convert=update_classpath):
             _check_components(compdict)
             return {convert(k): v for k, v in six.iteritems(compdict)}
 
-    if isinstance(compdict, (list, tuple)):
-        _check_components(compdict)
-        return type(compdict)(convert(c) for c in compdict)
+    # BEGIN Backwards compatibility for old (base, custom) call signature
+    if isinstance(custom, (list, tuple)):
+        _check_components(custom)
+        return type(custom)(convert(c) for c in custom)
+
+    if custom is not None:
+        compdict.update(custom)
+    # END Backwards compatibility
+
     compdict = without_none_values(_map_keys(compdict))
     return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
 
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index af15d3184..dab41ac8d 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -8,46 +8,59 @@ class BuildComponentListTest(unittest.TestCase):
 
     def test_build_dict(self):
         d = {'one': 1, 'two': None, 'three': 8, 'four': 4}
-        self.assertEqual(build_component_list(d, lambda x: x),
+        self.assertEqual(build_component_list(d, convert=lambda x: x),
                          ['one', 'four', 'three'])
 
+    def test_backwards_compatible_build_dict(self):
+        base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None}
+        custom = {'two': None, 'three': 8, 'four': 4}
+        self.assertEqual(build_component_list(base, custom,
+                                              convert=lambda x: x),
+                         ['one', 'four', 'five', 'three'])
+
     def test_return_list(self):
         custom = ['a', 'b', 'c']
-        self.assertEqual(build_component_list(custom, lambda x: x), custom)
+        self.assertEqual(build_component_list(None, custom,
+                                              convert=lambda x: x),
+                         custom)
 
     def test_map_dict(self):
         custom = {'one': 1, 'two': 2, 'three': 3}
-        self.assertEqual(build_component_list(custom, lambda x: x.upper()),
+        self.assertEqual(build_component_list({}, custom,
+                                              convert=lambda x: x.upper()),
                          ['ONE', 'TWO', 'THREE'])
 
     def test_map_list(self):
         custom = ['a', 'b', 'c']
-        self.assertEqual(build_component_list(custom, lambda x: x.upper()),
+        self.assertEqual(build_component_list(None, custom,
+                                              lambda x: x.upper()),
                          ['A', 'B', 'C'])
 
     def test_duplicate_components_in_dict(self):
         duplicate_dict = {'one': 1, 'two': 2, 'ONE': 4}
-        self.assertRaises(ValueError,
-                          build_component_list, duplicate_dict, lambda x: x.lower())
+        self.assertRaises(ValueError, build_component_list, {}, duplicate_dict,
+                          convert=lambda x: x.lower())
 
     def test_duplicate_components_in_list(self):
         duplicate_list = ['a', 'b', 'a']
-        self.assertRaises(ValueError,
-                          build_component_list, duplicate_list, lambda x: x)
+        self.assertRaises(ValueError, build_component_list, None,
+                          duplicate_list, convert=lambda x: x)
 
     def test_duplicate_components_in_basesettings(self):
         # Higher priority takes precedence
         duplicate_bs = BaseSettings({'one': 1, 'two': 2}, priority=0)
         duplicate_bs.set('ONE', 4, priority=10)
-        self.assertEqual(build_component_list(duplicate_bs, convert=lambda x: x.lower()),
+        self.assertEqual(build_component_list(duplicate_bs,
+                                              convert=lambda x: x.lower()),
                          ['two', 'one'])
         duplicate_bs.set('one', duplicate_bs['one'], priority=20)
-        self.assertEqual(build_component_list(duplicate_bs, convert=lambda x: x.lower()),
+        self.assertEqual(build_component_list(duplicate_bs,
+                                              convert=lambda x: x.lower()),
                          ['one', 'two'])
         # Same priority raises ValueError
         duplicate_bs.set('ONE', duplicate_bs['ONE'], priority=20)
-        self.assertRaises(ValueError,
-                          build_component_list, duplicate_bs, convert=lambda x: x.lower())
+        self.assertRaises(ValueError, build_component_list, duplicate_bs,
+                          convert=lambda x: x.lower())
 
 
 class UtilsConfTestCase(unittest.TestCase):

From 52ecee6a6271ef2d981f7f1993311abd8ed31bd9 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Fri, 6 Nov 2015 13:15:32 +0100
Subject: [PATCH 0618/2974] Replace BaseSettings._getcomposite() with public
 .getwithbase() method

---
 scrapy/commands/check.py                    |  2 +-
 scrapy/commands/crawl.py                    |  3 ++-
 scrapy/commands/runspider.py                |  2 +-
 scrapy/core/downloader/handlers/__init__.py |  3 ++-
 scrapy/core/downloader/middleware.py        |  3 ++-
 scrapy/core/spidermw.py                     |  2 +-
 scrapy/extension.py                         |  2 +-
 scrapy/extensions/feedexport.py             |  2 +-
 scrapy/pipelines/__init__.py                |  2 +-
 scrapy/settings/__init__.py                 | 30 ++++++++-------------
 tests/test_settings/__init__.py             | 25 ++++++-----------
 11 files changed, 31 insertions(+), 45 deletions(-)

diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index a423ba2c9..b8a9ef989 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -58,7 +58,7 @@ class Command(ScrapyCommand):
 
     def run(self, args, opts):
         # load contracts
-        contracts = build_component_list(self.settings._getcomposite('SPIDER_CONTRACTS'))
+        contracts = build_component_list(self.settings.getwithbase('SPIDER_CONTRACTS'))
         conman = ContractsManager(load_object(c) for c in contracts)
         runner = TextTestRunner(verbosity=2 if opts.verbose else 1)
         result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 7f5c64c20..4b986bf9d 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -35,7 +35,8 @@ class Command(ScrapyCommand):
                 self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
                 self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            feed_exporters = without_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
+            feed_exporters = without_none_values(
+                self.settings.getwithbase('FEED_EXPORTERS'))
             valid_output_formats = feed_exporters.keys()
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 72229bcf5..1da09e4da 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -58,7 +58,7 @@ class Command(ScrapyCommand):
                 self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
             else:
                 self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            feed_exporters = without_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
+            feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))
             valid_output_formats = feed_exporters.keys()
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 0e78e04f4..bc5cd742e 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -20,7 +20,8 @@ class DownloadHandlers(object):
         self._schemes = {}  # stores acceptable schemes on instancing
         self._handlers = {}  # stores instanced handlers for schemes
         self._notconfigured = {}  # remembers failed handlers
-        handlers = without_none_values(crawler.settings._getcomposite('DOWNLOAD_HANDLERS'))
+        handlers = without_none_values(
+            crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
         for scheme, clspath in six.iteritems(handlers):
             self._schemes[scheme] = clspath
 
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 958113fc3..c3b23e284 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -19,7 +19,8 @@ class DownloaderMiddlewareManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(settings._getcomposite('DOWNLOADER_MIDDLEWARES'))
+        return build_component_list(
+            settings.getwithbase('DOWNLOADER_MIDDLEWARES'))
 
     def _add_middleware(self, mw):
         if hasattr(mw, 'process_request'):
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index b5c80c350..a206e4b0c 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -18,7 +18,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(settings._getcomposite('SPIDER_MIDDLEWARES'))
+        return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
 
     def _add_middleware(self, mw):
         super(SpiderMiddlewareManager, self)._add_middleware(mw)
diff --git a/scrapy/extension.py b/scrapy/extension.py
index 4ceb32c68..e39e456fa 100644
--- a/scrapy/extension.py
+++ b/scrapy/extension.py
@@ -12,4 +12,4 @@ class ExtensionManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(settings._getcomposite('EXTENSIONS'))
+        return build_component_list(settings.getwithbase('EXTENSIONS'))
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 1e27a1e7e..daea551cb 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -196,7 +196,7 @@ class FeedExporter(object):
         return item
 
     def _load_components(self, setting_prefix):
-        conf = without_none_values(self.settings._getcomposite(setting_prefix))
+        conf = without_none_values(self.settings.getwithbase(setting_prefix))
         d = {}
         for k, v in conf.items():
             try:
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index 8df0d3154..2ef8786d0 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -13,7 +13,7 @@ class ItemPipelineManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return build_component_list(settings._getcomposite('ITEM_PIPELINES'))
+        return build_component_list(settings.getwithbase('ITEM_PIPELINES'))
 
     def _add_middleware(self, pipe):
         super(ItemPipelineManager, self)._add_middleware(pipe)
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 13656298b..b0f59ccc7 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -195,25 +195,17 @@ class BaseSettings(MutableMapping):
             value = json.loads(value)
         return dict(value)
 
-    def _getcomposite(self, name):
-        # DO NOT USE THIS FUNCTION IN YOUR CUSTOM PROJECTS
-        # It's for internal use in the transition away from the _BASE settings
-        # and will be removed along with _BASE support in a future release
-        basename = name + "_BASE"
-        if basename in self:
-            warnings.warn('_BASE settings are deprecated.',
-                          category=ScrapyDeprecationWarning)
-            # When users defined a _BASE setting, they explicitly don't want to
-            # use any of Scrapy's defaults. Therefore, we only use these entries
-            # from self[name] (where the defaults now live) that have a priority
-            # higher than 'default'
-            compsett = BaseSettings(self[basename], priority='default')
-            for k in self[name]:
-                prio = self[name].getpriority(k)
-                if prio > get_settings_priority('default'):
-                    compsett.set(k, self[name][k], prio)
-            return compsett
-        return self[name]
+    def getwithbase(self, name):
+        """Get a composition of a dictionary-like setting and its `_BASE`
+        counterpart.
+
+        :param name: name of the dictionary-like setting
+        :type name: string
+        """
+        compbs = BaseSettings()
+        compbs.update(self[name + '_BASE'])
+        compbs.update(self[name])
+        return compbs
 
     def getpriority(self, name):
         """
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 4ef08bb0b..8d98d2cec 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -263,24 +263,15 @@ class BaseSettingsTest(unittest.TestCase):
         self.assertEqual(settings.getpriority('key'), 99)
         self.assertEqual(settings.getpriority('nonexistentkey'), None)
 
-    def test_getcomposite(self):
-        s = BaseSettings({'TEST_BASE': {1: 1, 2: 2},
+    def test_getwithbase(self):
+        s = BaseSettings({'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
                           'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
-                          'HASNOBASE': BaseSettings({1: 1}, 'default')})
-        s['TEST'].set(4, 4, priority='project')
-        # When users specify a _BASE setting they explicitly don't want to use
-        # Scrapy's defaults, so we don't want to see anything that has a
-        # 'default' priority from TEST
-        cs = s._getcomposite('TEST')
-        self.assertEqual(len(cs), 3)
-        self.assertEqual(cs[1], 1)
-        self.assertEqual(cs[2], 2)
-        self.assertEqual(cs[4], 4)
-        cs = s._getcomposite('HASNOBASE')
-        self.assertEqual(len(cs), 1)
-        self.assertEqual(cs[1], 1)
-        cs = s._getcomposite('NONEXISTENT')
-        self.assertIsNone(cs)
+                          'HASNOBASE': BaseSettings({3: 3000}, 'default')})
+        s['TEST'].set(2, 200, 'cmdline')
+        six.assertCountEqual(self, s.getwithbase('TEST'),
+                             {1: 1, 2: 200, 3: 30})
+        six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE'])
+        self.assertEqual(s.getwithbase('NONEXISTENT'), {})
 
     def test_maxpriority(self):
         # Empty settings should return 'default'

From 44f6ada034f9d75e0089f45d3ad9f3c6fa5777f1 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Fri, 6 Nov 2015 15:54:50 +0100
Subject: [PATCH 0619/2974] Overwrite, not update, dictionary-like settings

---
 scrapy/settings/__init__.py     | 13 +++++--------
 tests/test_cmdline/__init__.py  | 19 ++++++++++---------
 tests/test_settings/__init__.py | 25 +++++++++++++------------
 3 files changed, 28 insertions(+), 29 deletions(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index b0f59ccc7..e62bdd08e 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -49,14 +49,11 @@ class SettingsAttribute(object):
 
     def set(self, value, priority):
         """Sets value if priority is higher or equal than current priority."""
-        if isinstance(self.value, BaseSettings):
-            # Ignore self.priority if self.value has per-key priorities
-            self.value.update(value, priority)
-            self.priority = max(self.value.maxpriority(), priority)
-        else:
-            if priority >= self.priority:
-                self.value = value
-                self.priority = priority
+        if priority >= self.priority:
+            if isinstance(self.value, BaseSettings):
+                value = BaseSettings(value, priority=priority)
+            self.value = value
+            self.priority = priority
 
     def __str__(self):
         return "<SettingsAttribute value={self.value!r} " \
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 5192fb0fa..c2de4fbc8 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -1,10 +1,11 @@
-import os
 import json
-import sys
-import shutil
+import os
 import pstats
-import tempfile
+import shutil
+import six
 from subprocess import Popen, PIPE
+import sys
+import tempfile
 import unittest
 try:
     from cStringIO import StringIO
@@ -57,14 +58,14 @@ class CmdlineTest(unittest.TestCase):
             shutil.rmtree(path)
 
     def test_override_dict_settings(self):
+        EXT_PATH = "tests.test_cmdline.extensions.DummyExtension"
+        EXTENSIONS = {EXT_PATH: 200}
         settingsstr = self._execute('settings', '--get', 'EXTENSIONS', '-s',
-                                    ('EXTENSIONS={"tests.test_cmdline.extensions.TestExtension": '
-                                     '100, "tests.test_cmdline.extensions.DummyExtension": 200}'))
+                                    'EXTENSIONS=' + json.dumps(EXTENSIONS))
         # XXX: There's gotta be a smarter way to do this...
         self.assertNotIn("...", settingsstr)
         for char in ("'", "<", ">", 'u"'):
             settingsstr = settingsstr.replace(char, '"')
         settingsdict = json.loads(settingsstr)
-        self.assertIn('tests.test_cmdline.extensions.DummyExtension', settingsdict)
-        self.assertIn('value=200', settingsdict['tests.test_cmdline.extensions.DummyExtension'])
-        self.assertIn('value=100', settingsdict['tests.test_cmdline.extensions.TestExtension'])
+        six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys())
+        self.assertIn('value=200', settingsdict[EXT_PATH])
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 8d98d2cec..44b9b6df3 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -37,21 +37,22 @@ class SettingsAttributeTest(unittest.TestCase):
         self.assertEqual(self.attribute.value, 'value')
         self.assertEqual(self.attribute.priority, 10)
 
-    def test_set_per_key_priorities(self):
-        attribute = SettingsAttribute(
-            BaseSettings({'one': 10, 'two': 20}, 0), 0)
+    def test_overwrite_basesettings(self):
+        original_dict = {'one': 10, 'two': 20}
+        original_settings = BaseSettings(original_dict, 0)
+        attribute = SettingsAttribute(original_settings, 0)
 
-        new_dict = {'one': 11, 'two': 21}
+        new_dict = {'three': 11, 'four': 21}
         attribute.set(new_dict, 10)
-        self.assertEqual(attribute.value['one'], 11)
-        self.assertEqual(attribute.value['two'], 21)
+        self.assertIsInstance(attribute.value, BaseSettings)
+        six.assertCountEqual(self, attribute.value, new_dict)
+        six.assertCountEqual(self, original_settings, original_dict)
 
-        new_settings = BaseSettings()
-        new_settings.set('one', 12, 20)
-        new_settings.set('two', 12, 0)
-        attribute.set(new_settings, 0)
-        self.assertEqual(attribute.value['one'], 12)
-        self.assertEqual(attribute.value['two'], 21)
+        new_settings = BaseSettings({'five': 12}, 0)
+        attribute.set(new_settings, 0)  # Insufficient priority
+        six.assertCountEqual(self, attribute.value, new_dict)
+        attribute.set(new_settings, 10)
+        six.assertCountEqual(self, attribute.value, new_settings)
 
     def test_repr(self):
         self.assertEqual(repr(self.attribute),

From 4f364764aea624a3db9a6033b5dba8a70f93535d Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Tue, 10 Nov 2015 22:48:50 +0100
Subject: [PATCH 0620/2974] Simplify BaseSettings.__get__(), .getpriority()

---
 scrapy/settings/__init__.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index e62bdd08e..342d2585e 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -90,10 +90,9 @@ class BaseSettings(MutableMapping):
         self.update(values, priority)
 
     def __getitem__(self, opt_name):
-        value = None
-        if opt_name in self:
-            value = self.attributes[opt_name].value
-        return value
+        if opt_name not in self:
+            return None
+        return self.attributes[opt_name].value
 
     def __contains__(self, name):
         return name in self.attributes
@@ -212,10 +211,9 @@ class BaseSettings(MutableMapping):
         :param name: the setting name
         :type name: string
         """
-        prio = None
-        if name in self:
-            prio = self.attributes[name].priority
-        return prio
+        if name not in self:
+            return None
+        return self.attributes[name].priority
 
     def maxpriority(self):
         """

From 881bf19e6804c3b82543f4f23d6ed60e1a4bb651 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Fri, 13 Nov 2015 17:08:13 +0200
Subject: [PATCH 0621/2974] FormRequest: test case-insensitive type attribute

---
 tests/test_http_request.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 60fd855dd..593698dd2 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -305,6 +305,19 @@ class FormRequestTest(RequestTest):
         request = FormRequest.from_response(response, url='/relative')
         self.assertEqual(request.url, 'http://example.com/relative')
 
+    def test_from_response_case_insensitive(self):
+        response = _buildresponse(
+            """<form action="get.php" method="GET">
+            <input type="SuBmIt" name="clickable1" value="clicked1">
+            <input type="iMaGe" name="i1" src="http://my.image.org/1.jpg">
+            <input type="submit" name="clickable2" value="clicked2">
+            </form>""")
+        req = self.request_class.from_response(response)
+        fs = _qs(req)
+        self.assertEqual(fs['clickable1'], ['clicked1'])
+        self.assertFalse('i1' in fs, fs)  # xpath in _get_inputs()
+        self.assertFalse('clickable2' in fs, fs)  # xpath in _get_clickable()
+
     def test_from_response_submit_first_clickable(self):
         response = _buildresponse(
             """<form action="get.php" method="GET">

From 650acad2b73c1abafe8ff95808b4d77b65ef67cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Fri, 13 Nov 2015 17:57:46 +0200
Subject: [PATCH 0622/2974] FormRequest: fix case-insensitive type attributes

---
 scrapy/http/request/form.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 4a9bd732e..ea4bfd564 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -107,8 +107,13 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
 
     inputs = form.xpath('descendant::textarea'
                         '|descendant::select'
-                        '|descendant::input[@type!="submit" and @type!="image" and @type!="reset"'
-                        'and ((@type!="checkbox" and @type!="radio") or @checked)]')
+                        '|descendant::input[@type['
+                        ' translate(., "SUBMIT", "submit") != "submit"'
+                        ' and translate(., "IMAGE", "image") !="image"'
+                        ' and translate(., "RESET", "reset") != "reset"'
+                        ' and (../@checked or ('
+                        '  translate(., "CHECKBOX", "checkbox") != "checkbox"'
+                        '  and translate(., "RADIO", "radio") != "radio"))]]')
     values = [(k, u'' if v is None else v)
               for k, v in (_value(e) for e in inputs)
               if k and k not in formdata]
@@ -151,9 +156,11 @@ def _get_clickable(clickdata, form):
     if the latter is given. If not, it returns the first
     clickable element found
     """
-    clickables = [el for el in form.xpath('descendant::input[@type="submit"]'
-                                          '|descendant::button[@type="submit"]'
-                                          '|descendant::button[not(@type)]')]
+    clickables = [
+        el for el in form.xpath(
+            'descendant::*[(self::input or self::button)'
+            ' and translate(@type, "SUBMIT", "submit") = "submit"]'
+            '|descendant::button[not(@type)]')]
     if not clickables:
         return
 

From 7a438a51b783f099894212abdefc95ad591fce3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Fri, 13 Nov 2015 18:03:54 +0200
Subject: [PATCH 0623/2974] FormRequest: test default <input> type (is text)

---
 tests/test_http_request.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 593698dd2..7964a6591 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -675,10 +675,11 @@ class FormRequestTest(RequestTest):
             <input type="text" name="i2">
             <input type="text" value="i3v1">
             <input type="text">
+            <input name="i4" value="i4v1">
             </form>''')
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(fs, {'i1': ['i1v1'], 'i2': ['']})
+        self.assertEqual(fs, {'i1': ['i1v1'], 'i2': [''], 'i4': ['i4v1']})
 
     def test_from_response_input_hidden(self):
         res = _buildresponse(

From 2d25eab0df4c5c8bb1894cbc45cc15059d726c8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Fri, 13 Nov 2015 18:05:07 +0200
Subject: [PATCH 0624/2974] FormRequest: <input>'s default type must be text

---
 scrapy/http/request/form.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index ea4bfd564..8ff394602 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -107,7 +107,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
 
     inputs = form.xpath('descendant::textarea'
                         '|descendant::select'
-                        '|descendant::input[@type['
+                        '|descendant::input[not(@type) or @type['
                         ' translate(., "SUBMIT", "submit") != "submit"'
                         ' and translate(., "IMAGE", "image") !="image"'
                         ' and translate(., "RESET", "reset") != "reset"'

From 395ef805eb74c1c575d32ab0c52d406f55c7d276 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Fri, 13 Nov 2015 18:54:02 +0200
Subject: [PATCH 0625/2974] FormRequest: test unicode xpath expr & exception

---
 tests/test_http_request.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 7964a6591..f82b2de8d 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1,5 +1,6 @@
 import cgi
 import unittest
+import re
 
 import six
 from six.moves import xmlrpc_client as xmlrpclib
@@ -747,6 +748,18 @@ class FormRequestTest(RequestTest):
         self.assertRaises(ValueError, self.request_class.from_response,
                           response, formxpath="//form/input[@name='abc']")
 
+    def test_from_response_unicode_xpath(self):
+        response = _buildresponse(b'<form name="\xd1\x8a"></form>')
+        r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']")
+        fs = _qs(r)
+        self.assertEqual(fs, {})
+
+        xpath = u"//form[@name='\u03b1']"
+        encoded = xpath if six.PY3 else xpath.encode('unicode_escape')
+        self.assertRaisesRegexp(ValueError, re.escape(encoded),
+                                self.request_class.from_response,
+                                response, formxpath=xpath)
+
     def test_from_response_button_submit(self):
         response = _buildresponse(
             """<form action="post.php" method="POST">

From 4f98be60be700e0c0c359f3cbaa1535ae9ece7ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Fri, 13 Nov 2015 18:56:05 +0200
Subject: [PATCH 0626/2974] FormRequest: fix unicode xpath exception

---
 scrapy/http/request/form.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 8ff394602..948ad05c9 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -85,7 +85,8 @@ def _get_form(response, formname, formid, formnumber, formxpath):
                 el = el.getparent()
                 if el is None:
                     break
-        raise ValueError('No <form> element found with %s' % formxpath)
+        encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape')
+        raise ValueError('No <form> element found with %s' % encoded)
 
     # If we get here, it means that either formname was None
     # or invalid

From ebfdb9bb03fd876bf519535dd57ec0816e0c3c2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Sat, 14 Nov 2015 23:24:07 +0200
Subject: [PATCH 0627/2974] readable xpath with exslt

---
 scrapy/http/request/form.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 948ad05c9..f623a5aa3 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -109,12 +109,11 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
     inputs = form.xpath('descendant::textarea'
                         '|descendant::select'
                         '|descendant::input[not(@type) or @type['
-                        ' translate(., "SUBMIT", "submit") != "submit"'
-                        ' and translate(., "IMAGE", "image") !="image"'
-                        ' and translate(., "RESET", "reset") != "reset"'
-                        ' and (../@checked or ('
-                        '  translate(., "CHECKBOX", "checkbox") != "checkbox"'
-                        '  and translate(., "RADIO", "radio") != "radio"))]]')
+                        ' not(re:test(., "^(?:submit|image|reset)$", "i"))'
+                        ' and (../@checked or'
+                        '  not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
+                        namespaces={
+                            "re": "http://exslt.org/regular-expressions"})
     values = [(k, u'' if v is None else v)
               for k, v in (_value(e) for e in inputs)
               if k and k not in formdata]
@@ -160,8 +159,10 @@ def _get_clickable(clickdata, form):
     clickables = [
         el for el in form.xpath(
             'descendant::*[(self::input or self::button)'
-            ' and translate(@type, "SUBMIT", "submit") = "submit"]'
-            '|descendant::button[not(@type)]')]
+            ' and re:test(@type, "^submit$", "i")]'
+            '|descendant::button[not(@type)]',
+            namespaces={"re": "http://exslt.org/regular-expressions"})
+        ]
     if not clickables:
         return
 

From 0025d5a9439a1f737bccb8ac1dfc6ff85daabb85 Mon Sep 17 00:00:00 2001
From: David Chen <mvjome@gmail.com>
Date: Mon, 16 Nov 2015 07:30:17 +0800
Subject: [PATCH 0628/2974] Fixed minor grammar issues.

---
 docs/faq.rst                  | 2 +-
 docs/topics/broad-crawls.rst  | 4 ++--
 docs/topics/extensions.rst    | 4 ++--
 docs/topics/item-pipeline.rst | 2 +-
 docs/topics/items.rst         | 2 +-
 docs/topics/leaks.rst         | 2 +-
 docs/topics/practices.rst     | 2 +-
 docs/topics/selectors.rst     | 4 ++--
 8 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 2e61f44ee..3d2bd8d4d 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -144,7 +144,7 @@ I get "Filtered offsite request" messages. How can I fix them?
 Those messages (logged with ``DEBUG`` level) don't necessarily mean there is a
 problem, so you may not need to fix them.
 
-Those message are thrown by the Offsite Spider Middleware, which is a spider
+Those messages are thrown by the Offsite Spider Middleware, which is a spider
 middleware (enabled by default) whose purpose is to filter out requests to
 domains outside the ones covered by the spider.
 
diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index aaf46bc92..79f0b3b53 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -34,7 +34,7 @@ These are some common properties often found in broad crawls:
 
 As said above, Scrapy default settings are optimized for focused crawls, not
 broad crawls. However, due to its asynchronous architecture, Scrapy is very
-well suited for performing fast broad crawls. This page summarize some things
+well suited for performing fast broad crawls. This page summarizes some things
 you need to keep in mind when using Scrapy for doing broad crawls, along with
 concrete suggestions of Scrapy settings to tune in order to achieve an
 efficient broad crawl.
@@ -46,7 +46,7 @@ Concurrency is the number of requests that are processed in parallel. There is
 a global limit and a per-domain limit.
 
 The default global concurrency limit in Scrapy is not suitable for crawling
-many different  domains in parallel, so you will want to increase it. How much
+many different domains in parallel, so you will want to increase it. How much
 to increase it will depend on how much CPU you crawler will have available. A
 good starting point is ``100``, but the best way to find out is by doing some
 trials and identifying at what concurrency your Scrapy process gets CPU
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 11c0aadb6..f9e709514 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -17,7 +17,7 @@ Extensions use the :ref:`Scrapy settings <topics-settings>` to manage their
 settings, just like any other Scrapy code.
 
 It is customary for extensions to prefix their settings with their own name, to
-avoid collision with existing (and future) extensions. For example, an
+avoid collision with existing (and future) extensions. For example, a
 hypothetic extension to handle `Google Sitemaps`_ would use settings like
 `GOOGLESITEMAP_ENABLED`, `GOOGLESITEMAP_DEPTH`, and so on.
 
@@ -143,7 +143,7 @@ Here is the code of such extension::
             self.items_scraped += 1
             if self.items_scraped % self.item_count == 0:
                 logger.info("scraped %d items", self.items_scraped)
-                
+
 
 .. _topics-extensions-ref:
 
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index f74400b4d..28969be61 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -95,7 +95,7 @@ contain a price::
 Write items to a JSON file
 --------------------------
 
-The following pipeline stores all scraped items (from all spiders) into a a
+The following pipeline stores all scraped items (from all spiders) into a
 single ``items.jl`` file, containing one item per line serialized in JSON
 format::
 
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 21ec0ed8c..4a8f47e93 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -61,7 +61,7 @@ the example above.
 You can specify any kind of metadata for each field. There is no restriction on
 the values accepted by :class:`Field` objects. For this same
 reason, there is no reference list of all available metadata keys. Each key
-defined in :class:`Field` objects could be used by a different components, and
+defined in :class:`Field` objects could be used by a different component, and
 only those components know about it. You can also define and use any other
 :class:`Field` key in your project too, for your own needs. The main goal of
 :class:`Field` objects is to provide a way to define all field metadata in one
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 735137ea2..92590c180 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -97,7 +97,7 @@ subclasses):
 A real example
 --------------
 
-Let's see a concrete example of an hypothetical case of memory leaks.
+Let's see a concrete example of a hypothetical case of memory leaks.
 Suppose we have some spider with a line similar to this one::
 
     return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id,
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 9ae34f423..60fe2267c 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -228,7 +228,7 @@ with varying degrees of sophistication. Getting around those measures can be
 difficult and tricky, and may sometimes require special infrastructure. Please
 consider contacting `commercial support`_ if in doubt.
 
-Here are some tips to keep in mind when dealing with these kind of sites:
+Here are some tips to keep in mind when dealing with these kinds of sites:
 
 * rotate your user agent from a pool of well-known ones from browsers (google
   around to get a list of them)
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 273cae0f8..8dc82dfe5 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -579,7 +579,7 @@ Built-in Selectors reference
     is used together with ``text``.
 
     If ``type`` is ``None`` and a ``response`` is passed, the selector type is
-    inferred from the response type as follow:
+    inferred from the response type as follows:
 
         * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
         * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
@@ -757,7 +757,7 @@ nodes can be accessed directly by their names::
      <Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>,
      ...
 
-If you wonder why the namespace removal procedure isn't called always by default
+If you wonder why the namespace removal procedure isn't always called by default
 instead of having to call it manually, this is because of two reasons, which, in order
 of relevance, are:
 

From 751155eadeb3b053666804061608b0ceb41f15f6 Mon Sep 17 00:00:00 2001
From: Pengyu CHEN <pengyu@libstarrify.so>
Date: Thu, 19 Nov 2015 00:38:05 +0800
Subject: [PATCH 0629/2974] removed: Unused code

---
 scrapy/linkextractors/sgml.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 7084d0180..9938e071f 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -124,9 +124,6 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
             restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
             canonicalize=canonicalize, deny_extensions=deny_extensions)
 
-        # FIXME: was added to fix a RegexLinkExtractor testcase
-        self.base_url = None
-
     def extract_links(self, response):
         base_url = None
         if self.restrict_xpaths:

From cc8ddb68b0787f24f4dc928e7ebe7575e1b740d8 Mon Sep 17 00:00:00 2001
From: Rolando Espinoza <darkrho@gmail.com>
Date: Tue, 29 Sep 2015 20:05:50 -0400
Subject: [PATCH 0630/2974] Added installation notes about using Conda for
 Windows and other OSes.

---
 docs/intro/install.rst | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 3adb4e6b0..abb94c006 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -24,9 +24,7 @@ The installation steps assume that you have the following things installed:
   where the Python installer ships it bundled.
 
 You can install Scrapy using pip (which is the canonical way to install Python
-packages).
-
-To install using pip::
+packages). To install using ``pip`` run::
 
    pip install Scrapy
 
@@ -35,6 +33,25 @@ To install using pip::
 Platform specific installation notes
 ====================================
 
+Anaconda
+--------
+
+.. note::
+
+  For Windows users, or if you have issues installing through `pip`, this is
+  the recommended way to install Scrapy.
+
+If you already have installed `Anaconda`_ or `Miniconda`_, the company
+`Scrapinghub`_ maintains official conda packages for Linux, Windows and OS X.
+
+To install Scrapy in Linux or OS X, use:
+
+  conda install -c scrapinghub scrapy 
+
+To install Scrapy in Windows, use:
+
+  conda install -c scrapinghub scrapy pywin32
+
 Windows
 -------
 
@@ -167,3 +184,6 @@ After any of these workarounds you should be able to install Scrapy::
 .. _homebrew: http://brew.sh/
 .. _zsh: http://www.zsh.org/
 .. _virtualenv: https://virtualenv.pypa.io/en/latest/
+.. _Scrapinghub: http://scrapinghub.com
+.. _Anaconda: http://docs.continuum.io/anaconda/index
+.. _Miniconda: http://conda.pydata.org/docs/install/quick.html

From 2c251000d3f53f34e2d3610199f2fee9c6ecf8b0 Mon Sep 17 00:00:00 2001
From: palego <palego@users.noreply.github.com>
Date: Sat, 31 Oct 2015 16:19:11 +0100
Subject: [PATCH 0631/2974] custom project templates

allow override of TEMPLATES_DIR for startproject
copy full TEMPLATES_DIR/project tree
doc update
---
 docs/topics/settings.rst        |  7 ++++++-
 scrapy/commands/startproject.py | 20 ++++++++++++--------
 tests/test_commands.py          | 23 ++++++++++++++++++++++-
 3 files changed, 40 insertions(+), 10 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index aa0417e1a..60b5f4585 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1046,7 +1046,12 @@ TEMPLATES_DIR
 Default: ``templates`` dir inside scrapy module
 
 The directory where to look for templates when creating new projects with
-:command:`startproject` command.
+:command:`startproject` command and new spiders with :command:`genspider` 
+command.
+
+The project name must not conflict with the name of custom files or directories
+in the ``project`` subdirectory.
+
 
 .. setting:: URLLENGTH_LIMIT
 
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 4375b6d4c..b3c5011df 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,10 +1,9 @@
 from __future__ import print_function
 import re
-import shutil
 import string
 from importlib import import_module
 from os.path import join, exists, abspath
-from shutil import copytree, ignore_patterns
+from shutil import copytree, ignore_patterns, move
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -12,8 +11,6 @@ from scrapy.utils.template import render_templatefile, string_camelcase
 from scrapy.exceptions import UsageError
 
 
-TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'project')
-
 TEMPLATES_TO_RENDER = (
     ('scrapy.cfg',),
     ('${project_name}', 'settings.py.tmpl'),
@@ -63,17 +60,24 @@ class Command(ScrapyCommand):
             self.exitcode = 1
             return
 
-        moduletpl = join(TEMPLATES_PATH, 'module')
-        copytree(moduletpl, join(project_name, project_name), ignore=IGNORE)
-        shutil.copy(join(TEMPLATES_PATH, 'scrapy.cfg'), project_name)
+        copytree(self.templates_dir, project_name, ignore=IGNORE)
+        move(join(project_name, 'module'), join(project_name, project_name))
         for paths in TEMPLATES_TO_RENDER:
             path = join(*paths)
             tplfile = join(project_name,
                 string.Template(path).substitute(project_name=project_name))
             render_templatefile(tplfile, project_name=project_name,
                 ProjectName=string_camelcase(project_name))
-        print("New Scrapy project %r created in:" % project_name)
+        print("New Scrapy project %r, using template directory %r, created in:" % \
+              (project_name, self.templates_dir))
         print("    %s\n" % abspath(project_name))
         print("You can start your first spider with:")
         print("    cd %s" % project_name)
         print("    scrapy genspider example example.com")
+
+    @property
+    def templates_dir(self):
+        _templates_base_dir = self.settings['TEMPLATES_DIR'] or \
+            join(scrapy.__path__[0], 'templates')
+        return join(_templates_base_dir, 'project')
+    
\ No newline at end of file
diff --git a/tests/test_commands.py b/tests/test_commands.py
index e0c0648ca..5755b3881 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -4,13 +4,14 @@ import subprocess
 import tempfile
 from time import sleep
 from os.path import exists, join, abspath
-from shutil import rmtree
+from shutil import rmtree, copytree
 from tempfile import mkdtemp
 import six
 
 from twisted.trial import unittest
 from twisted.internet import defer
 
+import scrapy
 from scrapy.utils.python import to_native_str
 from scrapy.utils.python import retry_on_eintr
 from scrapy.utils.test import get_testenv
@@ -71,6 +72,26 @@ class StartprojectTest(ProjectTest):
         self.assertEqual(1, self.call('startproject', self.project_name))
         self.assertEqual(1, self.call('startproject', 'wrong---project---name'))
         self.assertEqual(1, self.call('startproject', 'sys'))
+    
+
+class StartprojectTemplatesTest(ProjectTest):
+
+    def setUp(self):
+        super(StartprojectTemplatesTest, self).setUp()
+        self.tmpl = join(self.temp_path, 'templates')
+        self.tmpl_proj = join(self.tmpl, 'project')
+        
+    def test_startproject_template_override(self):
+        copytree(join(scrapy.__path__[0], 'templates'), self.tmpl)
+        os.mknod(join(self.tmpl_proj, 'root_template'))
+        assert exists(join(self.tmpl_proj, 'root_template'))
+
+        args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl]
+        p = self.proc('startproject', self.project_name, *args)
+        out = to_native_str(retry_on_eintr(p.stdout.read))
+        self.assertIn("New Scrapy project %r, using template directory %r, created in:" % \
+                      (self.project_name, join(self.tmpl, 'project')), out)
+        assert exists(join(self.proj_path, 'root_template'))
 
 
 class CommandTest(ProjectTest):

From 5750ce92c0c9298aaf5df77825fb0d4f99d53392 Mon Sep 17 00:00:00 2001
From: Alexander Sibiryakov <sixty-one@yandex.ru>
Date: Thu, 26 Nov 2015 17:19:19 +0100
Subject: [PATCH 0632/2974] nextcall repetitive calls (heartbeats).

---
 scrapy/core/engine.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index eb2779b12..ef4403106 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -7,7 +7,7 @@ For more information see docs/topics/architecture.rst
 import logging
 from time import time
 
-from twisted.internet import defer
+from twisted.internet import defer, task
 from twisted.python.failure import Failure
 
 from scrapy import signals
@@ -30,6 +30,7 @@ class Slot(object):
         self.close_if_idle = close_if_idle
         self.nextcall = nextcall
         self.scheduler = scheduler
+        self.heartbeat = task.LoopingCall(nextcall.schedule)
 
     def add_request(self, request):
         self.inprogress.add(request)
@@ -47,6 +48,7 @@ class Slot(object):
         if self.closing and not self.inprogress:
             if self.nextcall:
                 self.nextcall.cancel()
+                self.heartbeat.stop()
             self.closing.callback(None)
 
 
@@ -113,7 +115,6 @@ class ExecutionEngine(object):
             return
 
         if self.paused:
-            slot.nextcall.schedule(5)
             return
 
         while not self._needs_backout(spider):
@@ -254,6 +255,7 @@ class ExecutionEngine(object):
         self.crawler.stats.open_spider(spider)
         yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider)
         slot.nextcall.schedule()
+        slot.heartbeat.start(5)
 
     def _spider_idle(self, spider):
         """Called when a spider gets idle. This function is called when there
@@ -267,7 +269,6 @@ class ExecutionEngine(object):
             spider=spider, dont_log=DontCloseSpider)
         if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \
                 for _, x in res):
-            self.slot.nextcall.schedule(5)
             return
 
         if self.spider_is_idle(spider):

From c4d29ecaef534cdab31fb73ec83545010f7a5ad8 Mon Sep 17 00:00:00 2001
From: Alexander Sibiryakov <sixty-one@yandex.ru>
Date: Wed, 2 Dec 2015 17:05:44 +0100
Subject: [PATCH 0633/2974] Ignoring xlib/tx folder, depending on Twisted
 version.

---
 conftest.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/conftest.py b/conftest.py
index f9ca3ab93..b0ac1badd 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,6 +1,7 @@
 import glob
 import six
 import pytest
+from twisted import version as twisted_version
 
 
 def _py_files(folder):
@@ -26,6 +27,9 @@ collect_ignore = [
 
 ] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp")
 
+if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, 5, 0):
+    collect_ignore += _py_files("scrapy/xlib/tx")
+
 
 if six.PY3:
     for line in open('tests/py3-ignores.txt'):

From 016875fd513c66b7ee1bcc80a1f2ae0d6ffda2d2 Mon Sep 17 00:00:00 2001
From: Leonid Amirov <leonid.amirov@gmail.com>
Date: Thu, 3 Dec 2015 15:30:06 +0300
Subject: [PATCH 0634/2974] added more verbosity for log and for exception when
 download is cancelled because of a size limit

---
 scrapy/core/downloader/handlers/http11.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 31412a0f4..78f3f74fa 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -239,11 +239,16 @@ class ScrapyAgent(object):
         expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1
 
         if maxsize and expected_size > maxsize:
-            logger.error("Expected response size (%(size)s) larger than "
-                         "download max size (%(maxsize)s).",
-                         {'size': expected_size, 'maxsize': maxsize})
+            error_message = (
+                "Cancelling download of {url}: expected response "
+                "size ({size}) larger than "
+                "download max size ({maxsize}).".format(
+                    url=request.url, size=expected_size, maxsize=maxsize
+                )
+            )
+            logger.error(error_message)
             txresponse._transport._producer.loseConnection()
-            raise defer.CancelledError()
+            raise defer.CancelledError(error_message)
 
         if warnsize and expected_size > warnsize:
             logger.warning("Expected response size (%(size)s) larger than "

From 37289815ac98404458c43159abfd46dd54ed4b7b Mon Sep 17 00:00:00 2001
From: Leonid Amirov <leonid.amirov@gmail.com>
Date: Thu, 3 Dec 2015 16:15:50 +0300
Subject: [PATCH 0635/2974] code formatting fix

---
 scrapy/core/downloader/handlers/http11.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 78f3f74fa..9d075e877 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -239,13 +239,11 @@ class ScrapyAgent(object):
         expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1
 
         if maxsize and expected_size > maxsize:
-            error_message = (
-                "Cancelling download of {url}: expected response "
-                "size ({size}) larger than "
-                "download max size ({maxsize}).".format(
-                    url=request.url, size=expected_size, maxsize=maxsize
-                )
-            )
+            error_message = ("Cancelling download of {url}: expected response "
+                             "size ({size}) larger than "
+                             "download max size ({maxsize})."
+            ).format(url=request.url, size=expected_size, maxsize=maxsize)
+
             logger.error(error_message)
             txresponse._transport._producer.loseConnection()
             raise defer.CancelledError(error_message)

From f8ae99d18fb0365c340152e880245bb4340d7a1f Mon Sep 17 00:00:00 2001
From: Rolando Espinoza <darkrho@gmail.com>
Date: Sat, 5 Dec 2015 09:48:17 -0400
Subject: [PATCH 0636/2974] DOC Removed pywin32 from install instructions as
 it's already declared as dependency.

---
 docs/intro/install.rst | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index abb94c006..122de47f6 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -44,13 +44,10 @@ Anaconda
 If you already have installed `Anaconda`_ or `Miniconda`_, the company
 `Scrapinghub`_ maintains official conda packages for Linux, Windows and OS X.
 
-To install Scrapy in Linux or OS X, use:
+To install Scrapy using ``conda``, run::
 
   conda install -c scrapinghub scrapy 
 
-To install Scrapy in Windows, use:
-
-  conda install -c scrapinghub scrapy pywin32
 
 Windows
 -------

From 4be4ef038ea8e206bdd3682dab3cb7b5184994eb Mon Sep 17 00:00:00 2001
From: orangain <orangain@gmail.com>
Date: Sat, 12 Dec 2015 16:21:00 +0900
Subject: [PATCH 0637/2974] DOC: Add captions to toctrees which appear in
 sidebar

---
 docs/index.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/index.rst b/docs/index.rst
index 0d21f5d40..4cb3eb741 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -28,6 +28,7 @@ First steps
 ===========
 
 .. toctree::
+   :caption: First steps
    :hidden:
 
    intro/overview
@@ -53,6 +54,7 @@ Basic concepts
 ==============
 
 .. toctree::
+   :caption: Basic concepts
    :hidden:
 
    topics/commands
@@ -110,6 +112,7 @@ Built-in services
 =================
 
 .. toctree::
+   :caption: Built-in services
    :hidden:
 
    topics/logging
@@ -138,6 +141,7 @@ Solving specific problems
 =========================
 
 .. toctree::
+   :caption: Solving specific problems
    :hidden:
 
    faq
@@ -203,6 +207,7 @@ Extending Scrapy
 ================
 
 .. toctree::
+   :caption: Extending Scrapy
    :hidden:
 
    topics/architecture
@@ -240,6 +245,7 @@ All the rest
 ============
 
 .. toctree::
+   :caption: All the rest
    :hidden:
 
    news

From 719b1353a7a49d13f1f54e37a85b8b692915c0c7 Mon Sep 17 00:00:00 2001
From: seales <cseales6@gmail.com>
Date: Sun, 13 Dec 2015 19:39:48 -0800
Subject: [PATCH 0638/2974] Spelling fixes

---
 sep/sep-003.rst | 2 +-
 sep/sep-014.rst | 2 +-
 sep/sep-018.rst | 2 +-
 sep/sep-020.rst | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/sep/sep-003.rst b/sep/sep-003.rst
index 282574968..184839525 100644
--- a/sep/sep-003.rst
+++ b/sep/sep-003.rst
@@ -146,7 +146,7 @@ Default values
 
    p['numbers'] # returns []
 
-Accesing and changing nested item values
+Accessing and changing nested item values
 ----------------------------------------
 
 ::
diff --git a/sep/sep-014.rst b/sep/sep-014.rst
index 98a31b1aa..8ca81824d 100644
--- a/sep/sep-014.rst
+++ b/sep/sep-014.rst
@@ -54,7 +54,7 @@ Request Extractors
 
 Request Extractors takes response object and determines which requests follow.
 
-This is an enhancemente to ``LinkExtractors`` which returns urls (links),
+This is an enhancement to ``LinkExtractors`` which returns urls (links),
 Request Extractors return Request objects. 
 
 Request Processors
diff --git a/sep/sep-018.rst b/sep/sep-018.rst
index e30821917..aca7ac342 100644
--- a/sep/sep-018.rst
+++ b/sep/sep-018.rst
@@ -477,7 +477,7 @@ This is a port of the Offsite middleware to the new spider middleware API:
     
        def should_follow(self, request, spider):
            info = self.spiders[spider]
-           # hostanme can be None for wrong urls (like javascript links)
+           # hostname can be None for wrong urls (like javascript links)
            host = urlparse_cached(request).hostname or ''
            return bool(info.regex.search(host))
 
diff --git a/sep/sep-020.rst b/sep/sep-020.rst
index 7b2c043b7..49d068479 100644
--- a/sep/sep-020.rst
+++ b/sep/sep-020.rst
@@ -23,9 +23,9 @@ Rationale
 =========
 
 There are certain markup patterns that lend themselves quite nicely to
-automated parsing, for example the ``<table>`` tag outlilnes such a pattern
+automated parsing, for example the ``<table>`` tag outlines such a pattern
 for populating a database table with the embedded ``<tr>`` elements denoting
-the rows and the furthur embedded ``<td>`` elements denoting the individual
+the rows and the further embedded ``<td>`` elements denoting the individual
 fields.
 
 One pattern that is particularly well suited for auto-populating an Item Loader

From bcce8d3d80462946483a403a1fb25b3e12229136 Mon Sep 17 00:00:00 2001
From: orangain <orangain@gmail.com>
Date: Thu, 17 Dec 2015 14:48:37 +0900
Subject: [PATCH 0639/2974] DOC: Update MetaRefreshMiddlware's setting
 variables

* `REDIRECT_MAX_METAREFRESH_DELAY` has been deprecated and was
  renamed to `METAREFRESH_MAXDELAY`.
* Merge duplicate documents about `METAREFRESH_MAXDELAY` appeared both
  in the settings page and the downloader-middlewares page.
---
 docs/topics/downloader-middleware.rst |  8 +++++---
 docs/topics/settings.rst              | 10 ----------
 2 files changed, 5 insertions(+), 13 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index cc0254d29..3641da231 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -787,14 +787,16 @@ Default: ``True``
 
 Whether the Meta Refresh middleware will be enabled.
 
-.. setting:: REDIRECT_MAX_METAREFRESH_DELAY
+.. setting:: METAREFRESH_MAXDELAY
 
-REDIRECT_MAX_METAREFRESH_DELAY
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+METAREFRESH_MAXDELAY
+^^^^^^^^^^^^^^^^^^^^
 
 Default: ``100``
 
 The maximum meta-refresh delay (in seconds) to follow the redirection.
+Some sites use meta-refresh for redirecting to a session expired page, so we
+restrict automatic redirection to the maximum delay.
 
 RetryMiddleware
 ---------------
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 60b5f4585..cc070d8c0 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -857,16 +857,6 @@ Defines the maximum times a request can be redirected. After this maximum the
 request's response is returned as is. We used Firefox default value for the
 same task.
 
-.. setting:: REDIRECT_MAX_METAREFRESH_DELAY
-
-REDIRECT_MAX_METAREFRESH_DELAY
-------------------------------
-
-Default: ``100``
-
-Some sites use meta-refresh for redirecting to a session expired page, so we
-restrict automatic redirection to a maximum delay (in seconds)
-
 .. setting:: REDIRECT_PRIORITY_ADJUST
 
 REDIRECT_PRIORITY_ADJUST

From 4f49aab7c068c553c5dcb2935ee3e4df6c0b71aa Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Fri, 18 Dec 2015 16:16:05 -0500
Subject: [PATCH 0640/2974] BF: robustify _monkeypatches check for twisted -
 str() name first (Closes: #1634)

In my case, while running datalad tests using nose, scrapy was failing
since v was None
---
 scrapy/_monkeypatches.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index 782891326..60e0de1f2 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -20,6 +20,6 @@ if sys.version_info[0] == 2:
 import twisted.persisted.styles  # NOQA
 # Remove only entries with twisted serializers for non-twisted types.
 for k, v in frozenset(copyreg.dispatch_table.items()):
-    if not getattr(k, '__module__', '').startswith('twisted') \
-            and getattr(v, '__module__', '').startswith('twisted'):
+    if not str(getattr(k, '__module__', '')).startswith('twisted') \
+            and str(getattr(v, '__module__', '')).startswith('twisted'):
         copyreg.dispatch_table.pop(k)

From f57121c77be82ad7521c3f4ed5cda17ada118ef4 Mon Sep 17 00:00:00 2001
From: Aron Bordin <aron.bordin@gmail.com>
Date: Wed, 30 Dec 2015 13:10:13 -0200
Subject: [PATCH 0641/2974] show download warnsize once

---
 scrapy/core/downloader/handlers/http11.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 31412a0f4..403accef8 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -292,6 +292,7 @@ class _ResponseReader(protocol.Protocol):
         self._bodybuf = BytesIO()
         self._maxsize  = maxsize
         self._warnsize  = warnsize
+        self._reached_warnsize = False
         self._bytes_received = 0
 
     def dataReceived(self, bodyBytes):
@@ -305,11 +306,12 @@ class _ResponseReader(protocol.Protocol):
                           'maxsize': self._maxsize})
             self._finished.cancel()
 
-        if self._warnsize and self._bytes_received > self._warnsize:
-            logger.warning("Received (%(bytes)s) bytes larger than download "
-                           "warn size (%(warnsize)s).",
-                           {'bytes': self._bytes_received,
-                            'warnsize': self._warnsize})
+        if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize:
+            self._reached_warnsize = True
+            logger.warning("Received more bytes than download "
+                           "warn size (%(warnsize)s) in request %(request)s.",
+                           {'warnsize': self._warnsize,
+                            'request': self._request})
 
     def connectionLost(self, reason):
         if self._finished.called:

From 1b435b2887ac4e19159cd4416f9396e022d0382a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 30 Dec 2015 15:43:04 -0300
Subject: [PATCH 0642/2974] Add 1.0.4 release notes

---
 docs/news.rst | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 5df1b1a6a..4d7dc4d41 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,58 @@
 Release notes
 =============
 
+1.0.4 (2015-12-30)
+------------------
+
+- Ignoring xlib/tx folder, depending on Twisted version. (:commit:`7dfa979`)
+- Run on new travis-ci infra (:commit:`6e42f0b`)
+- Spelling fixes (:commit:`823a1cc`)
+- escape nodename in xmliter regex (:commit:`da3c155`)
+- test xml nodename with dots (:commit:`4418fc3`)
+- TST don't use broken Pillow version in tests (:commit:`a55078c`)
+- disable log on version command. closes #1426 (:commit:`86fc330`)
+- disable log on startproject command (:commit:`db4c9fe`)
+- Add PyPI download stats badge (:commit:`df2b944`)
+- don't run tests twice on Travis if a PR is made from a scrapy/scrapy branch (:commit:`a83ab41`)
+- Add Python 3 porting status badge to the README (:commit:`73ac80d`)
+- fixed RFPDupeFilter persistence (:commit:`97d080e`)
+- TST a test to show that dupefilter persistence is not working (:commit:`97f2fb3`)
+- explicit close file on file:// scheme handler (:commit:`d9b4850`)
+- Disable dupefilter in shell (:commit:`c0d0734`)
+- DOC: Add captions to toctrees which appear in sidebar (:commit:`aa239ad`)
+- DOC Removed pywin32 from install instructions as it's already declared as dependency. (:commit:`10eb400`)
+- Added installation notes about using Conda for Windows and other OSes. (:commit:`1c3600a`)
+- Fixed minor grammar issues. (:commit:`7f4ddd5`)
+- fixed a typo in the documentation. (:commit:`b71f677`)
+- Version 1 now exists (:commit:`5456c0e`)
+- fix another invalid xpath error (:commit:`0a1366e`)
+- fix ValueError: Invalid XPath: //div/[id="not-exists"]/text() on selectors.rst (:commit:`ca8d60f`)
+- Typos corrections (:commit:`7067117`)
+- fix typos in downloader-middleware.rst and exceptions.rst, middlware -> middleware (:commit:`32f115c`)
+- Add note to ubuntu install section about debian compatibility (:commit:`23fda69`)
+- Replace alternative OSX install workaround with virtualenv (:commit:`98b63ee`)
+- Reference Homebrew's homepage for installation instructions (:commit:`1925db1`)
+- Add oldest supported tox version to contributing docs (:commit:`5d10d6d`)
+- Note in install docs about pip being already included in python>=2.7.9 (:commit:`85c980e`)
+- Add non-python dependencies to Ubuntu install section in the docs (:commit:`fbd010d`)
+- Add OS X installation section to docs (:commit:`d8f4cba`)
+- DOC(ENH): specify path to rtd theme explicitly (:commit:`de73b1a`)
+- minor: scrapy.Spider docs grammar (:commit:`1ddcc7b`)
+- Make common practices sample code match the comments (:commit:`1b85bcf`)
+- nextcall repetitive calls (heartbeats). (:commit:`55f7104`)
+- Backport fix compatibility with Twisted 15.4.0 (:commit:`b262411`)
+- pin pytest to 2.7.3 (:commit:`a6535c2`)
+- Merge pull request #1512 from mgedmin/patch-1 (:commit:`8876111`)
+- Merge pull request #1513 from mgedmin/patch-2 (:commit:`5d4daf8`)
+- Typo (:commit:`f8d0682`)
+- Fix list formatting (:commit:`5f83a93`)
+- fix scrapy squeue tests after recent changes to queuelib (:commit:`3365c01`)
+- Merge pull request #1475 from rweindl/patch-1 (:commit:`2d688cd`)
+- Update tutorial.rst (:commit:`fbc1f25`)
+- Merge pull request #1449 from rhoekman/patch-1 (:commit:`7d6538c`)
+- Small grammatical change (:commit:`8752294`)
+- Add openssl version to version command (:commit:`13c45ac`)
+
 1.0.3 (2015-08-11)
 ------------------
 

From c702c5301523e57ce276d80c8106f9a52853948d Mon Sep 17 00:00:00 2001
From: palego <palego@users.noreply.github.com>
Date: Sun, 3 Jan 2016 14:33:42 +0100
Subject: [PATCH 0643/2974] change os.mknod() for open()

os.mknod() is a privileged command on OS X, making the test fail
---
 tests/test_commands.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 5755b3881..8edccd4bd 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -83,7 +83,8 @@ class StartprojectTemplatesTest(ProjectTest):
         
     def test_startproject_template_override(self):
         copytree(join(scrapy.__path__[0], 'templates'), self.tmpl)
-        os.mknod(join(self.tmpl_proj, 'root_template'))
+        with open(join(self.tmpl_proj, 'root_template'), 'w'):
+            pass
         assert exists(join(self.tmpl_proj, 'root_template'))
 
         args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl]

From d03d262f5290d7a439ac797947413894f158f425 Mon Sep 17 00:00:00 2001
From: palego <palego@users.noreply.github.com>
Date: Mon, 4 Jan 2016 10:00:13 +0100
Subject: [PATCH 0644/2974] indentation

---
 scrapy/downloadermiddlewares/retry.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 803ed5fc0..3324aa21a 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -56,7 +56,7 @@ class RetryMiddleware(object):
     def process_exception(self, request, exception, spider):
         if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \
                 and not request.meta.get('dont_retry', False):
-             return self._retry(request, exception, spider)
+            return self._retry(request, exception, spider)
 
     def _retry(self, request, reason, spider):
         retries = request.meta.get('retry_times', 0) + 1

From 2abc9bc901491b24ea1a35058ae2b86e44492c88 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Jr <valdir@scrapinghub.com>
Date: Wed, 6 Jan 2016 10:29:45 -0200
Subject: [PATCH 0645/2974] Update deprecated examples

* update the scrapy.org example to deal with the new layout.
* replaced slashdot.org by reddit.com, because it seems that slashdot is blocking requests.
---
 docs/topics/shell.rst | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 2b118bfbd..3569cbf37 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -106,10 +106,10 @@ Example of shell session
 ========================
 
 Here's an example of a typical shell session where we start by scraping the
-http://scrapy.org page, and then proceed to scrape the http://slashdot.org
-page. Finally, we modify the (Slashdot) request method to POST and re-fetch it
-getting a HTTP 405 (method not allowed) error. We end the session by typing
-Ctrl-D (in Unix systems) or Ctrl-Z in Windows.
+http://scrapy.org page, and then proceed to scrape the http://reddit.com
+page. Finally, we modify the (Reddit) request method to POST and re-fetch it
+getting an error. We end the session by typing Ctrl-D (in Unix systems) or
+Ctrl-Z in Windows.
 
 Keep in mind that the data extracted here may not be the same when you try it,
 as those pages are not static and could have changed by the time you test this.
@@ -140,24 +140,24 @@ all start with the ``[s]`` prefix)::
 
 After that, we can start playing with the objects::
 
-    >>> response.xpath("//h1/text()").extract()[0]
-    u'Meet Scrapy'
+    >>> response.xpath('//title/text()').extract_first()
+    u'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
 
-    >>> fetch("http://slashdot.org")
+    >>> fetch("http://reddit.com")
     [s] Available Scrapy objects:
-    [s]   crawler    <scrapy.crawler.Crawler object at 0x1a13b50>
+    [s]   crawler    <scrapy.crawler.Crawler object at 0x7fb3ed9c9c90>
     [s]   item       {}
-    [s]   request    <GET http://slashdot.org>
-    [s]   response   <200 http://slashdot.org>
-    [s]   settings   <scrapy.settings.Settings object at 0x2bfd650>
-    [s]   spider     <Spider 'default' at 0x20c6f50>
+    [s]   request    <GET http://reddit.com>
+    [s]   response   <200 https://www.reddit.com/>
+    [s]   settings   <scrapy.settings.Settings object at 0x7fb3ed9c9c10>
+    [s]   spider     <DefaultSpider 'default' at 0x7fb3ecdd3390>
     [s] Useful shortcuts:
     [s]   shelp()           Shell help (print this help)
     [s]   fetch(req_or_url) Fetch request (or URL) and update local objects
     [s]   view(response)    View response in a browser
 
     >>> response.xpath('//title/text()').extract()
-    [u'Slashdot: News for nerds, stuff that matters']
+    [u'reddit: the front page of the internet']
 
     >>> request = request.replace(method="POST")
 

From d4872940dbb450fcd0fa688766af0aa6dfccc861 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 6 Jan 2016 21:21:21 +0100
Subject: [PATCH 0646/2974] PY3: port utils/iterators

---
 scrapy/utils/iterators.py     | 14 ++++++++++----
 tests/py3-ignores.txt         |  1 -
 tests/test_utils_iterators.py | 21 +++++++++++----------
 3 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index c0d93f7a9..ed286f5c5 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -8,6 +8,8 @@ except ImportError:
     from io import BytesIO
 
 import six
+if six.PY3:
+    from io import StringIO
 
 from scrapy.http import TextResponse, Response
 from scrapy.selector import Selector
@@ -65,7 +67,7 @@ class _StreamReader(object):
             self._text, self.encoding = obj.body, obj.encoding
         else:
             self._text, self.encoding = obj, 'utf-8'
-        self._is_unicode = isinstance(self._text, unicode)
+        self._is_unicode = isinstance(self._text, six.text_type)
 
     def read(self, n=65535):
         self.read = self._read_unicode if self._is_unicode else self._read_string
@@ -94,7 +96,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
     headers is an iterable that when provided offers the keys
     for the returned dictionaries, if not the first row is used.
-    
+
     quotechar is the character used to enclosure fields on the given obj.
     """
 
@@ -102,7 +104,11 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     def _getrow(csv_r):
         return [to_unicode(field, encoding) for field in next(csv_r)]
 
-    lines = BytesIO(_body_or_str(obj, unicode=False))
+    # Python 3 csv reader input object needs to return strings
+    if six.PY3:
+        lines = StringIO(_body_or_str(obj, unicode=True))
+    else:
+        lines = BytesIO(_body_or_str(obj, unicode=False))
 
     kwargs = {}
     if delimiter: kwargs["delimiter"] = delimiter
@@ -125,7 +131,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
 
 def _body_or_str(obj, unicode=True):
-    assert isinstance(obj, (Response, six.string_types)), \
+    assert isinstance(obj, (Response, six.string_types, six.binary_type)), \
         "obj must be Response or basestring, not %s" % type(obj).__name__
     if isinstance(obj, Response):
         if not unicode:
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 55ed75c92..015578e1e 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -16,7 +16,6 @@ tests/test_pipeline_files.py
 tests/test_pipeline_images.py
 tests/test_proxy_connect.py
 tests/test_spidermiddleware_httperror.py
-tests/test_utils_iterators.py
 tests/test_utils_template.py
 tests/test_webclient.py
 
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 590c53302..d42ed2c91 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,4 +1,5 @@
 import os
+import six
 from twisted.trial import unittest
 
 from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
@@ -99,7 +100,7 @@ class XmliterTestCase(unittest.TestCase):
         body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
         response = XmlResponse('http://www.example.com', body=body)
         self.assertEqual(
-            self.xmliter(response, 'item').next().extract(),
+            next(self.xmliter(response, 'item')).extract(),
             u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
         )
 
@@ -189,11 +190,11 @@ class UtilsCsvTestCase(unittest.TestCase):
 
         # explicit type check cuz' we no like stinkin' autocasting! yarrr
         for result_row in result:
-            self.assert_(all((isinstance(k, unicode) for k in result_row.keys())))
-            self.assert_(all((isinstance(v, unicode) for v in result_row.values())))
+            self.assert_(all((isinstance(k, six.text_type) for k in result_row.keys())))
+            self.assert_(all((isinstance(v, six.text_type) for v in result_row.values())))
 
     def test_csviter_delimiter(self):
-        body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t')
+        body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
         response = TextResponse(url="http://example.com/", body=body)
         csv = csviter(response, delimiter='\t')
 
@@ -205,8 +206,8 @@ class UtilsCsvTestCase(unittest.TestCase):
 
     def test_csviter_quotechar(self):
         body1 = get_testdata('feeds', 'feed-sample6.csv')
-        body2 = get_testdata('feeds', 'feed-sample6.csv').replace(",", '|')
-        
+        body2 = get_testdata('feeds', 'feed-sample6.csv').replace(b',', b'|')
+
         response1 = TextResponse(url="http://example.com/", body=body1)
         csv1 = csviter(response1, quotechar="'")
 
@@ -237,7 +238,7 @@ class UtilsCsvTestCase(unittest.TestCase):
                           {u"'id'": u"4",   u"'name'": u"'empty'",   u"'value'": u""}])
 
     def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
-        body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t')
+        body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
         response = Response(url="http://example.com/", body=body)
         csv = csviter(response, delimiter='\t')
 
@@ -249,10 +250,10 @@ class UtilsCsvTestCase(unittest.TestCase):
 
     def test_csviter_headers(self):
         sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()
-        headers, body = sample[0].split(','), '\n'.join(sample[1:])
+        headers, body = sample[0].split(b','), b'\n'.join(sample[1:])
 
         response = TextResponse(url="http://example.com/", body=body)
-        csv = csviter(response, headers=headers)
+        csv = csviter(response, headers=[h.decode('utf-8') for h in headers])
 
         self.assertEqual([row for row in csv],
                          [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
@@ -262,7 +263,7 @@ class UtilsCsvTestCase(unittest.TestCase):
 
     def test_csviter_falserow(self):
         body = get_testdata('feeds', 'feed-sample3.csv')
-        body = '\n'.join((body, 'a,b', 'a,b,c,d'))
+        body = b'\n'.join((body, b'a,b', b'a,b,c,d'))
 
         response = TextResponse(url="http://example.com/", body=body)
         csv = csviter(response)

From 6ddd8147382348f4796ac905f0357925dfa81da1 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 12 Jan 2016 10:48:45 +0100
Subject: [PATCH 0647/2974] Support unicode tags in xml iterators (fixes #1665)

---
 scrapy/utils/iterators.py     | 10 +++----
 tests/test_utils_iterators.py | 51 ++++++++++++++++++++++++++++++++++-
 2 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index c0d93f7a9..bec985062 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
 
 def xmliter(obj, nodename):
     """Return a iterator of Selector's over all nodes of a XML document,
-       given tha name of the node to iterate. Useful for parsing XML feeds.
+       given the name of the node to iterate. Useful for parsing XML feeds.
 
     obj can be:
     - a Response object
@@ -36,7 +36,7 @@ def xmliter(obj, nodename):
     header_end = re_rsearch(HEADER_END_RE, text)
     header_end = text[header_end[1]:].strip() if header_end else ''
 
-    r = re.compile(r"<{0}[\s>].*?</{0}>".format(nodename_patt), re.DOTALL)
+    r = re.compile(r'<%(np)s[\s>].*?</%(np)s>' % {'np': nodename_patt}, re.DOTALL)
     for match in r.finditer(text):
         nodetext = header_start + match.group() + header_end
         yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]
@@ -49,7 +49,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
     iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
     selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename)
     for _, node in iterable:
-        nodetext = etree.tostring(node)
+        nodetext = etree.tostring(node, encoding='unicode')
         node.clear()
         xs = Selector(text=nodetext, type='xml')
         if namespace:
@@ -94,7 +94,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
     headers is an iterable that when provided offers the keys
     for the returned dictionaries, if not the first row is used.
-    
+
     quotechar is the character used to enclosure fields on the given obj.
     """
 
@@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
 
 def _body_or_str(obj, unicode=True):
-    assert isinstance(obj, (Response, six.string_types)), \
+    assert isinstance(obj, (Response, six.string_types, bytes)), \
         "obj must be Response or basestring, not %s" % type(obj).__name__
     if isinstance(obj, Response):
         if not unicode:
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 590c53302..8dceed7cc 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 import os
 from twisted.trial import unittest
 
@@ -45,6 +46,54 @@ class XmliterTestCase(unittest.TestCase):
                  for e in self.xmliter(response, 'matchme...')]
         self.assertEqual(nodenames, [['matchme...']])
 
+    def test_xmliter_unicode(self):
+        # example taken from https://github.com/scrapy/scrapy/issues/1665
+        body = """<?xml version="1.0" encoding="UTF-8"?>
+            <þingflokkar>
+               <þingflokkur id="26">
+                  <heiti />
+                  <skammstafanir>
+                     <stuttskammstöfun>-</stuttskammstöfun>
+                     <löngskammstöfun />
+                  </skammstafanir>
+                  <tímabil>
+                     <fyrstaþing>80</fyrstaþing>
+                  </tímabil>
+               </þingflokkur>
+               <þingflokkur id="21">
+                  <heiti>Alþýðubandalag</heiti>
+                  <skammstafanir>
+                     <stuttskammstöfun>Ab</stuttskammstöfun>
+                     <löngskammstöfun>Alþb.</löngskammstöfun>
+                  </skammstafanir>
+                  <tímabil>
+                     <fyrstaþing>76</fyrstaþing>
+                     <síðastaþing>123</síðastaþing>
+                  </tímabil>
+               </þingflokkur>
+               <þingflokkur id="27">
+                  <heiti>Alþýðuflokkur</heiti>
+                  <skammstafanir>
+                     <stuttskammstöfun>A</stuttskammstöfun>
+                     <löngskammstöfun>Alþfl.</löngskammstöfun>
+                  </skammstafanir>
+                  <tímabil>
+                     <fyrstaþing>27</fyrstaþing>
+                     <síðastaþing>120</síðastaþing>
+                  </tímabil>
+               </þingflokkur>
+            </þingflokkar>"""
+        response = XmlResponse(url="http://example.com", body=body)
+        attrs = []
+        for x in self.xmliter(response, u'þingflokkur'):
+            attrs.append((x.xpath('@id').extract(),
+                          x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),
+                          x.xpath(u'./tímabil/fyrstaþing/text()').extract()))
+
+        self.assertEqual(attrs,
+                         [([u'26'], [u'-'], [u'80']),
+                          ([u'21'], [u'Ab'], [u'76']),
+                          ([u'27'], [u'A'], [u'27'])])
 
     def test_xmliter_text(self):
         body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
@@ -206,7 +255,7 @@ class UtilsCsvTestCase(unittest.TestCase):
     def test_csviter_quotechar(self):
         body1 = get_testdata('feeds', 'feed-sample6.csv')
         body2 = get_testdata('feeds', 'feed-sample6.csv').replace(",", '|')
-        
+
         response1 = TextResponse(url="http://example.com/", body=body1)
         csv1 = csviter(response1, quotechar="'")
 

From d7d4ef67a697243143df969e32b8ed956394f4fb Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 12 Jan 2016 11:08:49 +0100
Subject: [PATCH 0648/2974] Changes following comments

---
 scrapy/utils/iterators.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index ed286f5c5..ce59c9719 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -1,15 +1,12 @@
 import re
 import csv
 import logging
-
 try:
     from cStringIO import StringIO as BytesIO
 except ImportError:
     from io import BytesIO
-
+from io import StringIO
 import six
-if six.PY3:
-    from io import StringIO
 
 from scrapy.http import TextResponse, Response
 from scrapy.selector import Selector
@@ -131,7 +128,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
 
 def _body_or_str(obj, unicode=True):
-    assert isinstance(obj, (Response, six.string_types, six.binary_type)), \
+    assert isinstance(obj, (Response, six.string_types, bytes)), \
         "obj must be Response or basestring, not %s" % type(obj).__name__
     if isinstance(obj, Response):
         if not unicode:

From 9fad25f3d14091d250cc4b1d668befca00c30ef0 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 13 Jan 2016 11:42:41 +0100
Subject: [PATCH 0649/2974] Use explicit Unicode and bytes for XML body in
 tests

---
 scrapy/utils/iterators.py     |  9 ++++++---
 tests/test_utils_iterators.py | 19 +++++++++++++++++--
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index c215a0bdd..69c7f2c23 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -48,7 +48,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
     iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
     selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename)
     for _, node in iterable:
-        nodetext = etree.tostring(node, encoding='unicode')
+        nodetext = etree.tostring(node, encoding=six.text_type)
         node.clear()
         xs = Selector(text=nodetext, type='xml')
         if namespace:
@@ -128,8 +128,11 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
 
 def _body_or_str(obj, unicode=True):
-    assert isinstance(obj, (Response, six.string_types, bytes)), \
-        "obj must be Response or basestring, not %s" % type(obj).__name__
+    expected_types = (Response, six.text_type, six.binary_type)
+    assert isinstance(obj, expected_types), \
+        "obj must be %s, not %s" % (
+            " or ".join(t.__name__ for t in expected_types),
+            type(obj).__name__)
     if isinstance(obj, Response):
         if not unicode:
             return obj.body
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index de103fea5..74c22d420 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -49,7 +49,7 @@ class XmliterTestCase(unittest.TestCase):
 
     def test_xmliter_unicode(self):
         # example taken from https://github.com/scrapy/scrapy/issues/1665
-        body = """<?xml version="1.0" encoding="UTF-8"?>
+        body = u"""<?xml version="1.0" encoding="UTF-8"?>
             <þingflokkar>
                <þingflokkur id="26">
                   <heiti />
@@ -84,7 +84,22 @@ class XmliterTestCase(unittest.TestCase):
                   </tímabil>
                </þingflokkur>
             </þingflokkar>"""
-        response = XmlResponse(url="http://example.com", body=body)
+
+        # with bytes
+        response = XmlResponse(url="http://example.com", body=body.encode('utf-8'))
+        attrs = []
+        for x in self.xmliter(response, u'þingflokkur'):
+            attrs.append((x.xpath('@id').extract(),
+                          x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),
+                          x.xpath(u'./tímabil/fyrstaþing/text()').extract()))
+
+        self.assertEqual(attrs,
+                         [([u'26'], [u'-'], [u'80']),
+                          ([u'21'], [u'Ab'], [u'76']),
+                          ([u'27'], [u'A'], [u'27'])])
+
+        # Unicode body needs encoding information
+        response = XmlResponse(url="http://example.com", body=body, encoding='utf-8')
         attrs = []
         for x in self.xmliter(response, u'þingflokkur'):
             attrs.append((x.xpath('@id').extract(),

From d4c7d72b2b6fc20c1df0f697dd60801b93848628 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 13 Jan 2016 12:13:47 +0100
Subject: [PATCH 0650/2974] Add tests for input type in xmliter calls

---
 tests/test_utils_iterators.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 74c22d420..8c4d6cf59 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -160,6 +160,10 @@ class XmliterTestCase(unittest.TestCase):
 
         self.assertRaises(StopIteration, next, iter)
 
+    def test_xmliter_objtype_exception(self):
+        i = self.xmliter(42, 'product')
+        self.assertRaises(AssertionError, next, i)
+
     def test_xmliter_encoding(self):
         body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
         response = XmlResponse('http://www.example.com', body=body)
@@ -233,6 +237,9 @@ class LxmlXmliterTestCase(XmliterTestCase):
         node = next(my_iter)
         self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table'])
 
+    def test_xmliter_objtype_exception(self):
+        i = self.xmliter(42, 'product')
+        self.assertRaises(TypeError, next, i)
 
 class UtilsCsvTestCase(unittest.TestCase):
     sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')

From 1347015a80b9d8dafe0e1ac067d65b7e0e7c3f84 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 13 Jan 2016 12:32:28 +0100
Subject: [PATCH 0651/2974] Refactored test code

---
 tests/test_utils_iterators.py | 37 +++++++++++++----------------------
 1 file changed, 14 insertions(+), 23 deletions(-)

diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 8c4d6cf59..b2e3889a4 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -85,31 +85,22 @@ class XmliterTestCase(unittest.TestCase):
                </þingflokkur>
             </þingflokkar>"""
 
-        # with bytes
-        response = XmlResponse(url="http://example.com", body=body.encode('utf-8'))
-        attrs = []
-        for x in self.xmliter(response, u'þingflokkur'):
-            attrs.append((x.xpath('@id').extract(),
-                          x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),
-                          x.xpath(u'./tímabil/fyrstaþing/text()').extract()))
+        for r in (
+            # with bytes
+            XmlResponse(url="http://example.com", body=body.encode('utf-8')),
+            # Unicode body needs encoding information
+            XmlResponse(url="http://example.com", body=body, encoding='utf-8')):
 
-        self.assertEqual(attrs,
-                         [([u'26'], [u'-'], [u'80']),
-                          ([u'21'], [u'Ab'], [u'76']),
-                          ([u'27'], [u'A'], [u'27'])])
+            attrs = []
+            for x in self.xmliter(r, u'þingflokkur'):
+                attrs.append((x.xpath('@id').extract(),
+                              x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),
+                              x.xpath(u'./tímabil/fyrstaþing/text()').extract()))
 
-        # Unicode body needs encoding information
-        response = XmlResponse(url="http://example.com", body=body, encoding='utf-8')
-        attrs = []
-        for x in self.xmliter(response, u'þingflokkur'):
-            attrs.append((x.xpath('@id').extract(),
-                          x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),
-                          x.xpath(u'./tímabil/fyrstaþing/text()').extract()))
-
-        self.assertEqual(attrs,
-                         [([u'26'], [u'-'], [u'80']),
-                          ([u'21'], [u'Ab'], [u'76']),
-                          ([u'27'], [u'A'], [u'27'])])
+            self.assertEqual(attrs,
+                             [([u'26'], [u'-'], [u'80']),
+                              ([u'21'], [u'Ab'], [u'76']),
+                              ([u'27'], [u'A'], [u'27'])])
 
     def test_xmliter_text(self):
         body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""

From a93d49a64ca170d98de98ee44a181ced04a23bea Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 13 Jan 2016 12:47:42 +0100
Subject: [PATCH 0652/2974] Add Python 3.5 tox env and Python 3.4 tests in
 Travis CI

---
 .travis.yml | 1 +
 tox.ini     | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index e857abbd8..65cfaad03 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,6 +9,7 @@ env:
  - TOXENV=py27
  - TOXENV=precise
  - TOXENV=py33
+ - TOXENV=py34
  - TOXENV=docs
 install:
  - pip install -U tox twine wheel codecov
diff --git a/tox.ini b/tox.ini
index eae7e8e47..b8d45d5b9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -48,6 +48,10 @@ deps =
 basepython = python3.4
 deps = {[testenv:py33]deps}
 
+[testenv:py35]
+basepython = python3.5
+deps = {[testenv:py33]deps}
+
 [docs]
 changedir = docs
 deps =

From f3889b0bce84cbd46852799a7bc95128c4c3b2d5 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 11:41:49 +0300
Subject: [PATCH 0653/2974] py3 compat: encode delimiter, method and path in
 ScrapyHTTPPageGetter

---
 scrapy/core/downloader/webclient.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index add5576ef..c335939d0 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -7,6 +7,7 @@ from twisted.internet import defer
 
 from scrapy.http import Headers
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_bytes
 from scrapy.responsetypes import responsetypes
 
 
@@ -29,13 +30,17 @@ def _parse(url):
 
 class ScrapyHTTPPageGetter(HTTPClient):
 
-    delimiter = '\n'
+    delimiter = b'\n'
 
     def connectionMade(self):
         self.headers = Headers() # bucket for response headers
 
         # Method command
-        self.sendCommand(self.factory.method, self.factory.path)
+        self.sendCommand(
+            to_bytes(self.factory.method, encoding='ascii'),
+            # XXX - do we need to percent-encode path somewhere?
+            # https://en.wikipedia.org/wiki/Percent-encoding#Character_data
+            to_bytes(self.factory.path))
         # Headers
         for key, values in self.factory.headers.items():
             for value in values:

From 9f2be23a3957ff17b252c47958affb9f5fcd6e72 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 11:42:23 +0300
Subject: [PATCH 0654/2974] webclient tests, py3: fix setUp, pass test_getPage

---
 tests/test_webclient.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index e0b46286a..d56c9f683 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -14,18 +14,21 @@ from twisted.protocols.policies import WrappingFactory
 
 from scrapy.core.downloader import webclient as client
 from scrapy.http import Request, Headers
+from scrapy.utils.python import to_bytes, to_unicode
 
 
 def getPage(url, contextFactory=None, *args, **kwargs):
     """Adapted version of twisted.web.client.getPage"""
-    def _clientfactory(*args, **kwargs):
+    def _clientfactory(url, *args, **kwargs):
+        url = to_unicode(url)
         timeout = kwargs.pop('timeout', 0)
-        f = client.ScrapyHTTPClientFactory(Request(*args, **kwargs), timeout=timeout)
+        f = client.ScrapyHTTPClientFactory(
+            Request(url, *args, **kwargs), timeout=timeout)
         f.deferred.addCallback(lambda r: r.body)
         return f
 
     from twisted.web.client import _makeGetterFactory
-    return _makeGetterFactory(url, _clientfactory,
+    return _makeGetterFactory(to_bytes(url), _clientfactory,
         contextFactory=contextFactory, *args, **kwargs).deferred
 
 
@@ -212,7 +215,7 @@ class WebClientTestCase(unittest.TestCase):
     def setUp(self):
         name = self.mktemp()
         os.mkdir(name)
-        FilePath(name).child("file").setContent("0123456789")
+        FilePath(name).child("file").setContent(b"0123456789")
         r = static.File(name)
         r.putChild("redirect", util.Redirect("/file"))
         r.putChild("wait", ForeverTakingResource())
@@ -250,7 +253,7 @@ class WebClientTestCase(unittest.TestCase):
         the body of the response if the default method B{GET} is used.
         """
         d = getPage(self.getURL("file"))
-        d.addCallback(self.assertEquals, "0123456789")
+        d.addCallback(self.assertEquals, b"0123456789")
         return d
 
 

From 73ff87c1dc05e98104f854d16792e302625c5e98 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 12:03:08 +0300
Subject: [PATCH 0655/2974] decode body from utf-8, as scrapy stores body as
 bytes, and twisted has already converted  to unicode

---
 scrapy/core/downloader/webclient.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index c335939d0..ba7bd798c 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -132,6 +132,9 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
         status = int(self.status)
         headers = Headers(self.response_headers)
         respcls = responsetypes.from_args(headers=headers, url=self.url)
+        # XXX - scrapy response stores body as bytes,
+        # but maybe it makes sense to be able to store unicode?
+        body = to_bytes(body)
         return respcls(url=self.url, status=status, headers=headers, body=body)
 
     def _set_connection_attributes(self, request):

From 1d5ab671833b1e054bf15a1b562a33b5ac5f1af7 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 12:04:26 +0300
Subject: [PATCH 0656/2974] pass test_getPageHead on py3

---
 tests/test_webclient.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index d56c9f683..3784aa401 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -266,8 +266,8 @@ class WebClientTestCase(unittest.TestCase):
         def _getPage(method):
             return getPage(self.getURL("file"), method=method)
         return defer.gatherResults([
-            _getPage("head").addCallback(self.assertEqual, ""),
-            _getPage("HEAD").addCallback(self.assertEqual, "")])
+            _getPage("head").addCallback(self.assertEqual, b""),
+            _getPage("HEAD").addCallback(self.assertEqual, b"")])
 
 
     def test_timeoutNotTriggering(self):

From 945674eb8f351e71aea090f1e94c3fa337c52cf8 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 12:25:54 +0300
Subject: [PATCH 0657/2974] pass test_externalUnicodeInterference - the logic
 for py3 is clearly inverse of what was expected in this test, as scrapy
 Request url must be unicode

---
 tests/test_webclient.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 3784aa401..84717e5eb 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -3,6 +3,7 @@ from twisted.internet import defer
 Tests borrowed from the twisted.web.client tests.
 """
 import os
+import six
 from six.moves.urllib.parse import urlparse
 
 from twisted.trial import unittest
@@ -75,8 +76,10 @@ class ParseUrlTestCase(unittest.TestCase):
         elements of its return tuple, even when passed an URL which has
         previously been passed to L{urlparse} as a C{unicode} string.
         """
-        badInput = u'http://example.com/path'
-        goodInput = badInput.encode('ascii')
+        goodInput = u'http://example.com/path'
+        badInput = goodInput.encode('ascii')
+        if six.PY2:
+            goodInput, badInput = badInput, goodInput
         urlparse(badInput)
         scheme, netloc, host, port, path = self._parse(goodInput)
         self.assertTrue(isinstance(scheme, str))

From 325b6af6c2c55bcf47fe339cfe139b357eb1673e Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 12:29:59 +0300
Subject: [PATCH 0658/2974] fix ScrapyHTTPPageGetterTests for py3 - we expect
 bytes here

---
 tests/test_webclient.py | 78 ++++++++++++++++++++---------------------
 1 file changed, 39 insertions(+), 39 deletions(-)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 84717e5eb..84b1011b8 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -105,22 +105,22 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
                 'Useful': 'value'}))
 
         self._test(factory,
-            "GET /bar HTTP/1.0\r\n"
-            "Content-Length: 9\r\n"
-            "Useful: value\r\n"
-            "Connection: close\r\n"
-            "User-Agent: fooble\r\n"
-            "Host: example.net\r\n"
-            "Cookie: blah blah\r\n"
-            "\r\n"
-            "some data")
+            b"GET /bar HTTP/1.0\r\n"
+            b"Content-Length: 9\r\n"
+            b"Useful: value\r\n"
+            b"Connection: close\r\n"
+            b"User-Agent: fooble\r\n"
+            b"Host: example.net\r\n"
+            b"Cookie: blah blah\r\n"
+            b"\r\n"
+            b"some data")
 
         # test minimal sent headers
         factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar'))
         self._test(factory,
-            "GET /bar HTTP/1.0\r\n"
-            "Host: foo\r\n"
-            "\r\n")
+            b"GET /bar HTTP/1.0\r\n"
+            b"Host: foo\r\n"
+            b"\r\n")
 
         # test a simple POST with body and content-type
         factory = client.ScrapyHTTPClientFactory(Request(
@@ -130,13 +130,13 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             headers={'Content-Type': 'application/x-www-form-urlencoded'}))
 
         self._test(factory,
-            "POST /bar HTTP/1.0\r\n"
-            "Host: foo\r\n"
-            "Connection: close\r\n"
-            "Content-Type: application/x-www-form-urlencoded\r\n"
-            "Content-Length: 10\r\n"
-            "\r\n"
-            "name=value")
+            b"POST /bar HTTP/1.0\r\n"
+            b"Host: foo\r\n"
+            b"Connection: close\r\n"
+            b"Content-Type: application/x-www-form-urlencoded\r\n"
+            b"Content-Length: 10\r\n"
+            b"\r\n"
+            b"name=value")
 
         # test a POST method with no body provided
         factory = client.ScrapyHTTPClientFactory(Request(
@@ -145,10 +145,10 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
         ))
 
         self._test(factory,
-                   "POST /bar HTTP/1.0\r\n"
-                   "Host: foo\r\n"
-                   "Content-Length: 0\r\n"
-                   "\r\n")
+            b"POST /bar HTTP/1.0\r\n"
+            b"Host: foo\r\n"
+            b"Content-Length: 0\r\n"
+            b"\r\n")
 
         # test with single and multivalued headers
         factory = client.ScrapyHTTPClientFactory(Request(
@@ -159,12 +159,12 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
                 }))
 
         self._test(factory,
-            "GET /bar HTTP/1.0\r\n"
-            "Host: foo\r\n"
-            "X-Meta-Multivalued: value1\r\n"
-            "X-Meta-Multivalued: value2\r\n"
-            "X-Meta-Single: single\r\n"
-            "\r\n")
+            b"GET /bar HTTP/1.0\r\n"
+            b"Host: foo\r\n"
+            b"X-Meta-Multivalued: value1\r\n"
+            b"X-Meta-Multivalued: value2\r\n"
+            b"X-Meta-Single: single\r\n"
+            b"\r\n")
 
         # same test with single and multivalued headers but using Headers class
         factory = client.ScrapyHTTPClientFactory(Request(
@@ -175,12 +175,12 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
                 })))
 
         self._test(factory,
-            "GET /bar HTTP/1.0\r\n"
-            "Host: foo\r\n"
-            "X-Meta-Multivalued: value1\r\n"
-            "X-Meta-Multivalued: value2\r\n"
-            "X-Meta-Single: single\r\n"
-            "\r\n")
+            b"GET /bar HTTP/1.0\r\n"
+            b"Host: foo\r\n"
+            b"X-Meta-Multivalued: value1\r\n"
+            b"X-Meta-Multivalued: value2\r\n"
+            b"X-Meta-Single: single\r\n"
+            b"\r\n")
 
     def _test(self, factory, testvalue):
         transport = StringTransport()
@@ -199,10 +199,10 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
         protocol = client.ScrapyHTTPPageGetter()
         protocol.factory = factory
         protocol.headers = Headers()
-        protocol.dataReceived("HTTP/1.0 200 OK\n")
-        protocol.dataReceived("Hello: World\n")
-        protocol.dataReceived("Foo: Bar\n")
-        protocol.dataReceived("\n")
+        protocol.dataReceived(b"HTTP/1.0 200 OK\n")
+        protocol.dataReceived(b"Hello: World\n")
+        protocol.dataReceived(b"Foo: Bar\n")
+        protocol.dataReceived(b"\n")
         self.assertEqual(protocol.headers,
             Headers({'Hello': ['World'], 'Foo': ['Bar']}))
 

From 85b0e6c9c766218bd48268f0c115c77b3886e539 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 14 Jan 2016 10:50:51 +0100
Subject: [PATCH 0659/2974] Travis: run tox with Python 3.5 + add Python 3.5
 tests

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 65cfaad03..ae9c745ac 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,5 @@
 language: python
-python: 2.7
+python: 3.5
 sudo: false
 branches:
   only:
@@ -9,7 +9,7 @@ env:
  - TOXENV=py27
  - TOXENV=precise
  - TOXENV=py33
- - TOXENV=py34
+ - TOXENV=py35
  - TOXENV=docs
 install:
  - pip install -U tox twine wheel codecov

From ae4aa2c3b24a967556f602a031520c535ebeb77d Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 13:04:10 +0300
Subject: [PATCH 0660/2974] py3 test fix: putChild expects bytes as path

---
 tests/test_webclient.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 84b1011b8..02e24de05 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -220,13 +220,13 @@ class WebClientTestCase(unittest.TestCase):
         os.mkdir(name)
         FilePath(name).child("file").setContent(b"0123456789")
         r = static.File(name)
-        r.putChild("redirect", util.Redirect("/file"))
-        r.putChild("wait", ForeverTakingResource())
-        r.putChild("error", ErrorResource())
-        r.putChild("nolength", NoLengthResource())
-        r.putChild("host", HostHeaderResource())
-        r.putChild("payload", PayloadResource())
-        r.putChild("broken", BrokenDownloadResource())
+        r.putChild(b"redirect", util.Redirect("/file"))
+        r.putChild(b"wait", ForeverTakingResource())
+        r.putChild(b"error", ErrorResource())
+        r.putChild(b"nolength", NoLengthResource())
+        r.putChild(b"host", HostHeaderResource())
+        r.putChild(b"payload", PayloadResource())
+        r.putChild(b"broken", BrokenDownloadResource())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
         self.port = self._listen(self.wrapper)

From b5f9bc8499293aced54f54250a18e148af356267 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 13:04:45 +0300
Subject: [PATCH 0661/2974] py3 test fixes in test_webclient - expect bytes as
 page body

---
 tests/test_webclient.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 02e24de05..05038c407 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -240,14 +240,17 @@ class WebClientTestCase(unittest.TestCase):
 
     def testPayload(self):
         s = "0123456789" * 10
-        return getPage(self.getURL("payload"), body=s).addCallback(self.assertEquals, s)
+        return getPage(self.getURL("payload"), body=s).addCallback(
+            self.assertEquals, to_bytes(s))
 
     def testHostHeader(self):
         # if we pass Host header explicitly, it should be used, otherwise
         # it should extract from url
         return defer.gatherResults([
-            getPage(self.getURL("host")).addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno),
-            getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(self.assertEquals, "www.example.com")])
+            getPage(self.getURL("host")).addCallback(
+                self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno)),
+            getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(
+                self.assertEquals, to_bytes("www.example.com"))])
 
 
     def test_getPage(self):
@@ -280,7 +283,8 @@ class WebClientTestCase(unittest.TestCase):
         called back with the contents of the page.
         """
         d = getPage(self.getURL("host"), timeout=100)
-        d.addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno)
+        d.addCallback(
+            self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno))
         return d
 
 
@@ -309,7 +313,7 @@ class WebClientTestCase(unittest.TestCase):
         return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile)
 
     def _cbNoSuchFile(self, pageData):
-        self.assert_('404 - No Such Resource' in pageData)
+        self.assert_(b'404 - No Such Resource' in pageData)
 
     def testFactoryInfo(self):
         url = self.getURL('file')
@@ -329,6 +333,6 @@ class WebClientTestCase(unittest.TestCase):
 
     def _cbRedirect(self, pageData):
         self.assertEquals(pageData,
-                '\n<html>\n    <head>\n        <meta http-equiv="refresh" content="0;URL=/file">\n'
-                '    </head>\n    <body bgcolor="#FFFFFF" text="#000000">\n    '
-                '<a href="/file">click here</a>\n    </body>\n</html>\n')
+                b'\n<html>\n    <head>\n        <meta http-equiv="refresh" content="0;URL=/file">\n'
+                b'    </head>\n    <body bgcolor="#FFFFFF" text="#000000">\n    '
+                b'<a href="/file">click here</a>\n    </body>\n</html>\n')

From 88f55312af0d9551f00686d79eaaf1c122740aa1 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 13:43:37 +0300
Subject: [PATCH 0662/2974] py3 fix in testFactoryInfo - factory attirbutes are
 bytes in twisted

---
 tests/test_webclient.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 05038c407..0a31d8b5d 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -323,10 +323,10 @@ class WebClientTestCase(unittest.TestCase):
         return factory.deferred.addCallback(self._cbFactoryInfo, factory)
 
     def _cbFactoryInfo(self, ignoredResult, factory):
-        self.assertEquals(factory.status, '200')
-        self.assert_(factory.version.startswith('HTTP/'))
-        self.assertEquals(factory.message, 'OK')
-        self.assertEquals(factory.response_headers['content-length'], '10')
+        self.assertEquals(factory.status, b'200')
+        self.assert_(factory.version.startswith(b'HTTP/'))
+        self.assertEquals(factory.message, b'OK')
+        self.assertEquals(factory.response_headers[b'content-length'], b'10')
 
     def testRedirect(self):
         return getPage(self.getURL("redirect")).addCallback(self._cbRedirect)

From 30c7b4e4cc5411af1015aa292e20e4c453607428 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 13:44:14 +0300
Subject: [PATCH 0663/2974] py3 compat in test_timeoutTriggering cleanup

---
 tests/test_webclient.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 0a31d8b5d..fa0083d44 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -300,7 +300,7 @@ class WebClientTestCase(unittest.TestCase):
         def cleanup(passthrough):
             # Clean up the server which is hanging around not doing
             # anything.
-            connected = self.wrapper.protocols.keys()
+            connected = list(six.iterkeys(self.wrapper.protocols))
             # There might be nothing here if the server managed to already see
             # that the connection was lost.
             if connected:

From 01783561781e1620580e0f1b66f32a514ac2b960 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 13:46:46 +0300
Subject: [PATCH 0664/2974] py3 fix testRedirect: url is bytes here

---
 tests/test_webclient.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index fa0083d44..66f8ed4cf 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -220,7 +220,7 @@ class WebClientTestCase(unittest.TestCase):
         os.mkdir(name)
         FilePath(name).child("file").setContent(b"0123456789")
         r = static.File(name)
-        r.putChild(b"redirect", util.Redirect("/file"))
+        r.putChild(b"redirect", util.Redirect(b"/file"))
         r.putChild(b"wait", ForeverTakingResource())
         r.putChild(b"error", ErrorResource())
         r.putChild(b"nolength", NoLengthResource())

From 6a412d25037d40dfbcc6392b6a862488c153e46f Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 13:48:48 +0300
Subject: [PATCH 0665/2974] all tests pass in test_webclient.py on py3 -
 removing from py3-ignores

---
 tests/py3-ignores.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 015578e1e..9e75ecf92 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -17,7 +17,6 @@ tests/test_pipeline_images.py
 tests/test_proxy_connect.py
 tests/test_spidermiddleware_httperror.py
 tests/test_utils_template.py
-tests/test_webclient.py
 
 scrapy/xlib/tx/iweb.py
 scrapy/xlib/tx/interfaces.py
@@ -29,7 +28,6 @@ scrapy/core/downloader/handlers/s3.py
 scrapy/core/downloader/handlers/http11.py
 scrapy/core/downloader/handlers/http.py
 scrapy/core/downloader/handlers/ftp.py
-scrapy/core/downloader/webclient.py
 scrapy/pipelines/images.py
 scrapy/pipelines/files.py
 scrapy/linkextractors/sgml.py

From e5fb6094384f8d96e8a677b5fd9f2e6bfe2b97d3 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 14:45:02 +0300
Subject: [PATCH 0666/2974] make ScrapyHTTPClientFactory comply to twisted
 HTTPClientFactory protocol - use bytes (encoding are likely wrong at this
 stage)

---
 scrapy/core/downloader/webclient.py | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index ba7bd798c..841322bdd 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -12,13 +12,15 @@ from scrapy.responsetypes import responsetypes
 
 
 def _parsed_url_args(parsed):
+    b = lambda x: to_bytes(x, encoding='ascii')
     path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
-    host = parsed.hostname
+    path = to_bytes(path)  # FIXME
+    host = b(parsed.hostname)  # FIXME
     port = parsed.port
-    scheme = parsed.scheme
-    netloc = parsed.netloc
+    scheme = b(parsed.scheme)
+    netloc = b(parsed.netloc)  # FIXME - host + port
     if port is None:
-        port = 443 if scheme == 'https' else 80
+        port = 443 if scheme == b'https' else 80
     return scheme, netloc, host, port, path
 
 
@@ -36,11 +38,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
         self.headers = Headers() # bucket for response headers
 
         # Method command
-        self.sendCommand(
-            to_bytes(self.factory.method, encoding='ascii'),
-            # XXX - do we need to percent-encode path somewhere?
-            # https://en.wikipedia.org/wiki/Percent-encoding#Character_data
-            to_bytes(self.factory.path))
+        self.sendCommand(self.factory.method, self.factory.path)
         # Headers
         for key, values in self.factory.headers.items():
             for value in values:
@@ -96,8 +94,10 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
     afterFoundGet = False
 
     def __init__(self, request, timeout=180):
-        self.url = urldefrag(request.url)[0]
-        self.method = request.method
+        self._url = urldefrag(request.url)[0]
+        # converting to bytes to comply to Twisted interface
+        self.url = to_bytes(self._url)  # FIXME
+        self.method = to_bytes(request.method, encoding='ascii')
         self.body = request.body or None
         self.headers = Headers(request.headers)
         self.response_headers = None
@@ -131,11 +131,11 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
         request.meta['download_latency'] = self.headers_time-self.start_time
         status = int(self.status)
         headers = Headers(self.response_headers)
-        respcls = responsetypes.from_args(headers=headers, url=self.url)
+        respcls = responsetypes.from_args(headers=headers, url=self._url)
         # XXX - scrapy response stores body as bytes,
         # but maybe it makes sense to be able to store unicode?
         body = to_bytes(body)
-        return respcls(url=self.url, status=status, headers=headers, body=body)
+        return respcls(url=self._url, status=status, headers=headers, body=body)
 
     def _set_connection_attributes(self, request):
         parsed = urlparse_cached(request)

From ac2cf191d1868f19897a108a861e3170485a676a Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 14:50:58 +0300
Subject: [PATCH 0667/2974] py3: remove comments, utf-8 is fine here: as
 twisted ultimately uses urllib.parse.quote that assepts bytes and assumes
 utf-8

---
 scrapy/core/downloader/webclient.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 841322bdd..2d848aee0 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -12,13 +12,12 @@ from scrapy.responsetypes import responsetypes
 
 
 def _parsed_url_args(parsed):
-    b = lambda x: to_bytes(x, encoding='ascii')
     path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
-    path = to_bytes(path)  # FIXME
-    host = b(parsed.hostname)  # FIXME
+    path = to_bytes(path)
+    host = to_bytes(parsed.hostname)
     port = parsed.port
-    scheme = b(parsed.scheme)
-    netloc = b(parsed.netloc)  # FIXME - host + port
+    scheme = to_bytes(parsed.scheme, encoding='ascii')
+    netloc = to_bytes(parsed.netloc)
     if port is None:
         port = 443 if scheme == b'https' else 80
     return scheme, netloc, host, port, path
@@ -96,7 +95,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
     def __init__(self, request, timeout=180):
         self._url = urldefrag(request.url)[0]
         # converting to bytes to comply to Twisted interface
-        self.url = to_bytes(self._url)  # FIXME
+        self.url = to_bytes(self._url)
         self.method = to_bytes(request.method, encoding='ascii')
         self.body = request.body or None
         self.headers = Headers(request.headers)

From 8df35bcac6190ab92635e46009c2e099c8e6010d Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 14:52:44 +0300
Subject: [PATCH 0668/2974] rm note to self: to be discussed in PR

---
 scrapy/core/downloader/webclient.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 2d848aee0..15d14ae49 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -131,8 +131,6 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
         status = int(self.status)
         headers = Headers(self.response_headers)
         respcls = responsetypes.from_args(headers=headers, url=self._url)
-        # XXX - scrapy response stores body as bytes,
-        # but maybe it makes sense to be able to store unicode?
         body = to_bytes(body)
         return respcls(url=self._url, status=status, headers=headers, body=body)
 

From 5c2241ccc7a886ea48ce12c2fb67cee5db24ce4e Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 15:30:28 +0300
Subject: [PATCH 0669/2974] py3: fix webclient tests after making
 ScrapyHTTPClientFactory use bytes as in twisted

---
 scrapy/core/downloader/webclient.py |  4 ++--
 tests/test_webclient.py             | 14 ++++++++------
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 15d14ae49..d2cdd6f98 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -65,7 +65,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
         self.factory.noPage(reason)
 
     def handleResponse(self, response):
-        if self.factory.method.upper() == 'HEAD':
+        if self.factory.method.upper() == b'HEAD':
             self.factory.page('')
         elif self.length is not None and self.length > 0:
             self.factory.noPage(self._connection_lost_reason)
@@ -123,7 +123,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
             # just in case a broken http/1.1 decides to keep connection alive
             self.headers.setdefault("Connection", "close")
         # Content-Length must be specified in POST method even with no body
-        elif self.method == 'POST':
+        elif self.method == b'POST':
             self.headers['Content-Length'] = 0
 
     def _build_response(self, body, request):
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 66f8ed4cf..412e10c89 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -68,6 +68,8 @@ class ParseUrlTestCase(unittest.TestCase):
     )
 
         for url, test in tests:
+            test = tuple(
+                to_bytes(x) if not isinstance(x, int) else x for x in test)
             self.assertEquals(client._parse(url), test, url)
 
     def test_externalUnicodeInterference(self):
@@ -82,10 +84,10 @@ class ParseUrlTestCase(unittest.TestCase):
             goodInput, badInput = badInput, goodInput
         urlparse(badInput)
         scheme, netloc, host, port, path = self._parse(goodInput)
-        self.assertTrue(isinstance(scheme, str))
-        self.assertTrue(isinstance(netloc, str))
-        self.assertTrue(isinstance(host, str))
-        self.assertTrue(isinstance(path, str))
+        self.assertTrue(isinstance(scheme, bytes))
+        self.assertTrue(isinstance(netloc, bytes))
+        self.assertTrue(isinstance(host, bytes))
+        self.assertTrue(isinstance(path, bytes))
         self.assertTrue(isinstance(port, int))
 
 
@@ -317,9 +319,9 @@ class WebClientTestCase(unittest.TestCase):
 
     def testFactoryInfo(self):
         url = self.getURL('file')
-        scheme, netloc, host, port, path = client._parse(url)
+        _, _, host, port, _ = client._parse(url)
         factory = client.ScrapyHTTPClientFactory(Request(url))
-        reactor.connectTCP(host, port, factory)
+        reactor.connectTCP(to_unicode(host), port, factory)
         return factory.deferred.addCallback(self._cbFactoryInfo, factory)
 
     def _cbFactoryInfo(self, ignoredResult, factory):

From 32bb5b682a7cf4f2baca78914f18297c120eef2d Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 16:11:16 +0300
Subject: [PATCH 0670/2974] fix import of test_downloader_handlers.py: use
 @implementer, move failing on py3 imports into corresponding tests

---
 scrapy/core/downloader/handlers/http11.py | 4 ++--
 tests/test_downloader_handlers.py         | 8 +++++---
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 31412a0f4..7c937a036 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -6,7 +6,7 @@ from io import BytesIO
 from time import time
 from six.moves.urllib.parse import urldefrag
 
-from zope.interface import implements
+from zope.interface import implementer
 from twisted.internet import defer, reactor, protocol
 from twisted.web.http_headers import Headers as TxHeaders
 from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
@@ -265,8 +265,8 @@ class ScrapyAgent(object):
         return respcls(url=url, status=status, headers=headers, body=body, flags=flags)
 
 
+@implementer(IBodyProducer)
 class _RequestBodyProducer(object):
-    implements(IBodyProducer)
 
     def __init__(self, body):
         self.body = body
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index d2a349b40..5f1703c5c 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -10,9 +10,7 @@ from twisted.web import server, static, util, resource
 from twisted.web.test.test_webclient import ForeverTakingResource, \
         NoLengthResource, HostHeaderResource, \
         PayloadResource, BrokenDownloadResource
-from twisted.protocols.ftp import FTPRealm, FTPFactory
 from twisted.cred import portal, checkers, credentials
-from twisted.protocols.ftp import FTPClient, ConnectionLost
 from w3lib.url import path_to_file_uri
 
 from scrapy import twisted_version
@@ -22,7 +20,6 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownlo
 from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
 from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
-from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
 
 from scrapy.spiders import Spider
 from scrapy.http import Request
@@ -520,6 +517,9 @@ class FTPTestCase(unittest.TestCase):
         skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home"
 
     def setUp(self):
+        from twisted.protocols.ftp import FTPRealm, FTPFactory
+        from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
+
         # setup dirs and test file
         self.directory = self.mktemp()
         os.mkdir(self.directory)
@@ -601,6 +601,8 @@ class FTPTestCase(unittest.TestCase):
         return self._add_test_callbacks(d, _test)
 
     def test_invalid_credentials(self):
+        from twisted.protocols.ftp import ConnectionLost
+
         request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
                 meta={"ftp_user": self.username, "ftp_password": 'invalid'})
         d = self.download_handler.download_request(request, None)

From 3509378b8be0df7cb38d3823068288b5daa37612 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 16:29:19 +0300
Subject: [PATCH 0671/2974] py3: pass first http downloader test, simple
 crawler works now, yay!

---
 scrapy/core/downloader/handlers/http11.py |  9 ++++++---
 scrapy/http/response/__init__.py          |  3 ++-
 tests/test_downloader_handlers.py         | 20 ++++++++++----------
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 7c937a036..34070ebc6 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -19,6 +19,7 @@ from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.core.downloader.webclient import _parse
 from scrapy.utils.misc import load_object
+from scrapy.utils.python import to_bytes, to_unicode
 from scrapy import twisted_version
 
 logger = logging.getLogger(__name__)
@@ -200,8 +201,8 @@ class ScrapyAgent(object):
         agent = self._get_agent(request, timeout)
 
         # request details
-        url = urldefrag(request.url)[0]
-        method = request.method
+        url = to_bytes(urldefrag(request.url)[0])
+        method = to_bytes(request.method)
         headers = TxHeaders(request.headers)
         if isinstance(agent, self._TunnelingAgent):
             headers.removeHeader('Proxy-Authorization')
@@ -261,8 +262,10 @@ class ScrapyAgent(object):
         txresponse, body, flags = result
         status = int(txresponse.code)
         headers = Headers(txresponse.headers.getAllRawHeaders())
+        url = to_unicode(url)
         respcls = responsetypes.from_args(headers=headers, url=url)
-        return respcls(url=url, status=status, headers=headers, body=body, flags=flags)
+        return respcls(
+            url=url, status=status, headers=headers, body=body, flags=flags)
 
 
 @implementer(IBodyProducer)
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 983154001..59ef15682 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -4,6 +4,7 @@ responses in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
+import six
 from six.moves.urllib.parse import urljoin
 
 from scrapy.http.headers import Headers
@@ -34,7 +35,7 @@ class Response(object_ref):
         return self._url
 
     def _set_url(self, url):
-        if isinstance(url, str):
+        if isinstance(url, six.string_types):
             self._url = url
         else:
             raise TypeError('%s url must be str, got %s:' % (type(self).__name__,
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 5f1703c5c..cdb1ad02d 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -88,7 +88,7 @@ class FileTestCase(unittest.TestCase):
         def _test(response):
             self.assertEquals(response.url, request.url)
             self.assertEquals(response.status, 200)
-            self.assertEquals(response.body, '0123456789')
+            self.assertEquals(response.body, b'0123456789')
 
         request = Request(path_to_file_uri(self.tmpname + '^'))
         assert request.url.upper().endswith('%5E')
@@ -107,15 +107,15 @@ class HttpTestCase(unittest.TestCase):
     def setUp(self):
         name = self.mktemp()
         os.mkdir(name)
-        FilePath(name).child("file").setContent("0123456789")
+        FilePath(name).child("file").setContent(b"0123456789")
         r = static.File(name)
-        r.putChild("redirect", util.Redirect("/file"))
-        r.putChild("wait", ForeverTakingResource())
-        r.putChild("hang-after-headers", ForeverTakingResource(write=True))
-        r.putChild("nolength", NoLengthResource())
-        r.putChild("host", HostHeaderResource())
-        r.putChild("payload", PayloadResource())
-        r.putChild("broken", BrokenDownloadResource())
+        r.putChild(b"redirect", util.Redirect(b"/file"))
+        r.putChild(b"wait", ForeverTakingResource())
+        r.putChild(b"hang-after-headers", ForeverTakingResource(write=True))
+        r.putChild(b"nolength", NoLengthResource())
+        r.putChild(b"host", HostHeaderResource())
+        r.putChild(b"payload", PayloadResource())
+        r.putChild(b"broken", BrokenDownloadResource())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
         self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1')
@@ -136,7 +136,7 @@ class HttpTestCase(unittest.TestCase):
         request = Request(self.getURL('file'))
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, "0123456789")
+        d.addCallback(self.assertEquals, b"0123456789")
         return d
 
     def test_download_head(self):

From 6b79fffa9a4c53cb6a6af2e9d9251b95c28496b4 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 16:37:18 +0300
Subject: [PATCH 0672/2974] py3: pass all of HttpTestCase

---
 scrapy/core/downloader/handlers/http11.py |  4 ++--
 tests/test_downloader_handlers.py         | 14 +++++++-------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 34070ebc6..dbb002710 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -182,7 +182,7 @@ class ScrapyAgent(object):
             _, _, proxyHost, proxyPort, proxyParams = _parse(proxy)
             scheme = _parse(request.url)[0]
             omitConnectTunnel = proxyParams.find('noconnect') >= 0
-            if  scheme == 'https' and not omitConnectTunnel:
+            if  scheme == b'https' and not omitConnectTunnel:
                 proxyConf = (proxyHost, proxyPort,
                              request.headers.get('Proxy-Authorization', None))
                 return self._TunnelingAgent(reactor, proxyConf,
@@ -233,7 +233,7 @@ class ScrapyAgent(object):
     def _cb_bodyready(self, txresponse, request):
         # deliverBody hangs for responses without body
         if txresponse.length == 0:
-            return txresponse, '', None
+            return txresponse, b'', None
 
         maxsize = request.meta.get('download_maxsize', self._maxsize)
         warnsize = request.meta.get('download_warnsize', self._warnsize)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index cdb1ad02d..c017a9eb2 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1,5 +1,4 @@
 import os
-import twisted
 import six
 
 from twisted.trial import unittest
@@ -25,6 +24,7 @@ from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.settings import Settings
 from scrapy.utils.test import get_crawler
+from scrapy.utils.python import to_bytes
 from scrapy.exceptions import NotConfigured
 
 from tests.mockserver import MockServer
@@ -143,7 +143,7 @@ class HttpTestCase(unittest.TestCase):
         request = Request(self.getURL('file'), method='HEAD')
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, '')
+        d.addCallback(self.assertEquals, b'')
         return d
 
     def test_redirect_status(self):
@@ -175,7 +175,7 @@ class HttpTestCase(unittest.TestCase):
 
     def test_host_header_not_in_request_headers(self):
         def _test(response):
-            self.assertEquals(response.body, '127.0.0.1:%d' % self.portno)
+            self.assertEquals(response.body, to_bytes('127.0.0.1:%d' % self.portno))
             self.assertEquals(request.headers, {})
 
         request = Request(self.getURL('host'))
@@ -183,19 +183,19 @@ class HttpTestCase(unittest.TestCase):
 
     def test_host_header_seted_in_request_headers(self):
         def _test(response):
-            self.assertEquals(response.body, 'example.com')
-            self.assertEquals(request.headers.get('Host'), 'example.com')
+            self.assertEquals(response.body, b'example.com')
+            self.assertEquals(request.headers.get('Host'), b'example.com')
 
         request = Request(self.getURL('host'), headers={'Host': 'example.com'})
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, 'example.com')
+        d.addCallback(self.assertEquals, b'example.com')
         return d
 
     def test_payload(self):
-        body = '1'*100 # PayloadResource requires body length to be 100
+        body = b'1'*100 # PayloadResource requires body length to be 100
         request = Request(self.getURL('payload'), method='POST', body=body)
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.body)

From c6f14a39de14b15b78d3fd4a098ec246536e12c4 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 16:50:16 +0300
Subject: [PATCH 0673/2974] py3: fix http10 downloader - unicode host expected
 here

---
 scrapy/core/downloader/handlers/http10.py | 5 +++--
 tests/test_downloader_handlers.py         | 6 +++---
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index 11b2acdae..0322bbe49 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -2,6 +2,7 @@
 """
 from twisted.internet import reactor
 from scrapy.utils.misc import load_object
+from scrapy.utils.python import to_unicode
 
 
 class HTTP10DownloadHandler(object):
@@ -17,8 +18,8 @@ class HTTP10DownloadHandler(object):
         return factory.deferred
 
     def _connect(self, factory):
-        host, port = factory.host, factory.port
-        if factory.scheme == 'https':
+        host, port = to_unicode(factory.host), factory.port
+        if factory.scheme == b'https':
             return reactor.connectSSL(host, port, factory,
                                       self.ClientContextFactory())
         else:
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index c017a9eb2..780f08806 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -223,7 +223,7 @@ class Http11TestCase(HttpTestCase):
         request = Request(self.getURL('file'))
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, "0123456789")
+        d.addCallback(self.assertEquals, b"0123456789")
         return d
 
     @defer.inlineCallbacks
@@ -234,7 +234,7 @@ class Http11TestCase(HttpTestCase):
         # response body. (regardless of headers)
         d = self.download_request(request, Spider('foo', download_maxsize=10))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, "0123456789")
+        d.addCallback(self.assertEquals, b"0123456789")
         yield d
 
         d = self.download_request(request, Spider('foo', download_maxsize=9))
@@ -257,7 +257,7 @@ class Http11TestCase(HttpTestCase):
         request = Request(self.getURL('file'))
         d = self.download_request(request, Spider('foo', download_maxsize=100))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, "0123456789")
+        d.addCallback(self.assertEquals, b"0123456789")
         return d
 
 

From 4950f5988ef1df5bc6b6ec2c4a70a7956f64b539 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 17:24:08 +0300
Subject: [PATCH 0674/2974] py3: pass http proxy tests

---
 scrapy/core/downloader/handlers/http11.py | 7 ++++---
 tests/test_downloader_handlers.py         | 6 +++---
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index dbb002710..d81093a9f 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -181,10 +181,11 @@ class ScrapyAgent(object):
         if proxy:
             _, _, proxyHost, proxyPort, proxyParams = _parse(proxy)
             scheme = _parse(request.url)[0]
-            omitConnectTunnel = proxyParams.find('noconnect') >= 0
+            proxyHost = to_unicode(proxyHost)
+            omitConnectTunnel = proxyParams.find(b'noconnect') >= 0
             if  scheme == b'https' and not omitConnectTunnel:
                 proxyConf = (proxyHost, proxyPort,
-                             request.headers.get('Proxy-Authorization', None))
+                             request.headers.get(b'Proxy-Authorization', None))
                 return self._TunnelingAgent(reactor, proxyConf,
                     contextFactory=self._contextFactory, connectTimeout=timeout,
                     bindAddress=bindaddress, pool=self._pool)
@@ -205,7 +206,7 @@ class ScrapyAgent(object):
         method = to_bytes(request.method)
         headers = TxHeaders(request.headers)
         if isinstance(agent, self._TunnelingAgent):
-            headers.removeHeader('Proxy-Authorization')
+            headers.removeHeader(b'Proxy-Authorization')
         bodyproducer = _RequestBodyProducer(request.body) if request.body else None
 
         start_time = time()
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 780f08806..ebf1d2f9c 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -351,7 +351,7 @@ class HttpProxyTestCase(unittest.TestCase):
         def _test(response):
             self.assertEquals(response.status, 200)
             self.assertEquals(response.url, request.url)
-            self.assertEquals(response.body, 'http://example.com')
+            self.assertEquals(response.body, b'http://example.com')
 
         http_proxy = self.getURL('')
         request = Request('http://example.com', meta={'proxy': http_proxy})
@@ -361,7 +361,7 @@ class HttpProxyTestCase(unittest.TestCase):
         def _test(response):
             self.assertEquals(response.status, 200)
             self.assertEquals(response.url, request.url)
-            self.assertEquals(response.body, 'https://example.com')
+            self.assertEquals(response.body, b'https://example.com')
 
         http_proxy = '%s?noconnect' % self.getURL('')
         request = Request('https://example.com', meta={'proxy': http_proxy})
@@ -371,7 +371,7 @@ class HttpProxyTestCase(unittest.TestCase):
         def _test(response):
             self.assertEquals(response.status, 200)
             self.assertEquals(response.url, request.url)
-            self.assertEquals(response.body, '/path/to/resource')
+            self.assertEquals(response.body, b'/path/to/resource')
 
         request = Request(self.getURL('path/to/resource'))
         return self.download_request(request, Spider('foo')).addCallback(_test)

From f46a9d595dee801d0ea13d7cdaab8b8de952929f Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 17:31:58 +0300
Subject: [PATCH 0675/2974] skip ftp tests on py3 for now

---
 tests/test_downloader_handlers.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index ebf1d2f9c..b3c1c565f 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -515,6 +515,8 @@ class FTPTestCase(unittest.TestCase):
 
     if twisted_version < (10, 2, 0):
         skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home"
+    if six.PY3:
+        skip = "Twisted missing ftp support for PY3"
 
     def setUp(self):
         from twisted.protocols.ftp import FTPRealm, FTPFactory

From 2aa6c92ffca50c8f6e5d057ac2808a99785eb88f Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 17:52:50 +0300
Subject: [PATCH 0676/2974] py3 fixes in tests.mockserver

---
 tests/mockserver.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 3e4f8c0e5..1ab8e4b8d 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -134,12 +134,12 @@ class Echo(LeafResource):
 class Partial(LeafResource):
 
     def render_GET(self, request):
-        request.setHeader("Content-Length", "1024")
+        request.setHeader(b"Content-Length", b"1024")
         self.deferRequest(request, 0, self._delayedRender, request)
         return NOT_DONE_YET
 
     def _delayedRender(self, request):
-        request.write("partial content\n")
+        request.write(b"partial content\n")
         request.finish()
 
 
@@ -147,7 +147,7 @@ class Drop(Partial):
 
     def _delayedRender(self, request):
         abort = getarg(request, "abort", 0, type=int)
-        request.write("this connection will be dropped\n")
+        request.write(b"this connection will be dropped\n")
         tr = request.channel.transport
         try:
             if abort and hasattr(tr, 'abortConnection'):
@@ -162,13 +162,13 @@ class Root(Resource):
 
     def __init__(self):
         Resource.__init__(self)
-        self.putChild("status", Status())
-        self.putChild("follow", Follow())
-        self.putChild("delay", Delay())
-        self.putChild("partial", Partial())
-        self.putChild("drop", Drop())
-        self.putChild("raw", Raw())
-        self.putChild("echo", Echo())
+        self.putChild(b"status", Status())
+        self.putChild(b"follow", Follow())
+        self.putChild(b"delay", Delay())
+        self.putChild(b"partial", Partial())
+        self.putChild(b"drop", Drop())
+        self.putChild(b"raw", Raw())
+        self.putChild(b"echo", Echo())
 
         if six.PY2 and twisted_version > (12, 3, 0):
             from twisted.web.test.test_webclient import PayloadResource
@@ -181,7 +181,7 @@ class Root(Resource):
         return self
 
     def render(self, request):
-        return 'Scrapy mock HTTP server\n'
+        return b'Scrapy mock HTTP server\n'
 
 
 class MockServer():

From 81a90c3af65ce863c073e2d83a6b149a03e7d4cb Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 18:47:06 +0300
Subject: [PATCH 0677/2974] unskip part of test_download_gzip_response on py3,
 file a twisted issue for the remaining part

---
 tests/mockserver.py               |  6 +++---
 tests/test_downloader_handlers.py | 21 ++++++++++++---------
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 1ab8e4b8d..02bab0efd 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -170,12 +170,12 @@ class Root(Resource):
         self.putChild(b"raw", Raw())
         self.putChild(b"echo", Echo())
 
-        if six.PY2 and twisted_version > (12, 3, 0):
+        if twisted_version > (12, 3, 0):
             from twisted.web.test.test_webclient import PayloadResource
             from twisted.web.server import GzipEncoderFactory
             from twisted.web.resource import EncodingResourceWrapper
-            self.putChild('payload', PayloadResource())
-            self.putChild("xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
+            self.putChild(b"payload", PayloadResource())
+            self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
 
     def getChild(self, name, request):
         return self
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index b3c1c565f..a8de28d4b 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -294,27 +294,30 @@ class Http11MockServerTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def test_download_gzip_response(self):
 
-        if six.PY2 and twisted_version > (12, 3, 0):
+        if twisted_version > (12, 3, 0):
 
             crawler = get_crawler(SingleRequestSpider)
-            body = '1'*100 # PayloadResource requires body length to be 100
+            body = b'1'*100 # PayloadResource requires body length to be 100
             request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50})
             yield crawler.crawl(seed=request)
             failure = crawler.spider.meta['failure']
             # download_maxsize < 100, hence the CancelledError
             self.assertIsInstance(failure.value, defer.CancelledError)
 
-            request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
+            request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
             request = request.replace(url='http://localhost:8998/xpayload')
             yield crawler.crawl(seed=request)
 
-            # download_maxsize = 50 is enough for the gzipped response
-            failure = crawler.spider.meta.get('failure')
-            self.assertTrue(failure == None)
-            reason = crawler.spider.meta['close_reason']
-            self.assertTrue(reason, 'finished')
+            if six.PY2:
+                # download_maxsize = 50 is enough for the gzipped response
+                # See issue https://twistedmatrix.com/trac/ticket/8175
+                raise unittest.SkipTest("xpayload only enabled for PY2")
+                failure = crawler.spider.meta.get('failure')
+                self.assertTrue(failure == None)
+                reason = crawler.spider.meta['close_reason']
+                self.assertTrue(reason, 'finished')
         else:
-            raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0 and python 2.x")
+            raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0")
 
 
 class UriResource(resource.Resource):

From 99f1f2ad1dbff9ae2e97755b2d61e03ab2339a6d Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 19:00:48 +0300
Subject: [PATCH 0678/2974] unskip tests and modules ported to py3

---
 tests/py3-ignores.txt | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 9e75ecf92..57e80f590 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -4,7 +4,6 @@ tests/test_command_shell.py
 tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
-tests/test_downloader_handlers.py
 tests/test_downloadermiddleware_httpcache.py
 tests/test_downloadermiddleware_httpcompression.py
 tests/test_downloadermiddleware_httpproxy.py
@@ -25,8 +24,6 @@ scrapy/xlib/tx/client.py
 scrapy/xlib/tx/_newclient.py
 scrapy/xlib/tx/__init__.py
 scrapy/core/downloader/handlers/s3.py
-scrapy/core/downloader/handlers/http11.py
-scrapy/core/downloader/handlers/http.py
 scrapy/core/downloader/handlers/ftp.py
 scrapy/pipelines/images.py
 scrapy/pipelines/files.py

From 94ab7bee6c5630bcb36e7b758daf31e45d493613 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 15 Jan 2016 11:26:01 +0300
Subject: [PATCH 0679/2974] py3: body to bytes in tests, unskip test file

---
 tests/py3-ignores.txt              | 1 -
 tests/test_downloadermiddleware.py | 7 ++++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 57e80f590..0da1b6089 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -7,7 +7,6 @@ tests/test_crawl.py
 tests/test_downloadermiddleware_httpcache.py
 tests/test_downloadermiddleware_httpcompression.py
 tests/test_downloadermiddleware_httpproxy.py
-tests/test_downloadermiddleware.py
 tests/test_downloadermiddleware_retry.py
 tests/test_engine.py
 tests/test_mail.py
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 13f35b92a..fb51392b2 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -5,6 +5,7 @@ from scrapy.http import Request, Response
 from scrapy.spiders import Spider
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.utils.test import get_crawler
+from scrapy.utils.python import to_bytes
 from tests import mock
 
 
@@ -68,7 +69,7 @@ class DefaultsTest(ManagerTestCase):
 
         """
         req = Request('http://example.com')
-        body = '<p>You are being redirected</p>'
+        body = b'<p>You are being redirected</p>'
         resp = Response(req.url, status=302, body=body, headers={
             'Content-Length': str(len(body)),
             'Content-Type': 'text/html',
@@ -78,12 +79,12 @@ class DefaultsTest(ManagerTestCase):
         ret = self._download(request=req, response=resp)
         self.assertTrue(isinstance(ret, Request),
                         "Not redirected: {0!r}".format(ret))
-        self.assertEqual(ret.url, resp.headers['Location'],
+        self.assertEqual(to_bytes(ret.url), resp.headers['Location'],
                          "Not redirected to location header")
 
     def test_200_and_invalid_gzipped_body_must_fail(self):
         req = Request('http://example.com')
-        body = '<p>You are being redirected</p>'
+        body = b'<p>You are being redirected</p>'
         resp = Response(req.url, status=200, body=body, headers={
             'Content-Length': str(len(body)),
             'Content-Type': 'text/html',

From dbf6cc73d96f5015e14d4075e5a0d50db2a96453 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 15 Jan 2016 11:46:56 +0300
Subject: [PATCH 0680/2974] py3: add leveldb to py33 test env, fix anydbm
 module name on py3

---
 scrapy/settings/default_settings.py | 4 +++-
 tox.ini                             | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8435b0354..b151933b6 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -18,6 +18,8 @@ import sys
 from importlib import import_module
 from os.path import join, abspath, dirname
 
+import six
+
 AJAXCRAWL_ENABLED = False
 
 AUTOTHROTTLE_ENABLED = False
@@ -163,7 +165,7 @@ HTTPCACHE_ALWAYS_STORE = False
 HTTPCACHE_IGNORE_HTTP_CODES = []
 HTTPCACHE_IGNORE_SCHEMES = ['file']
 HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
-HTTPCACHE_DBM_MODULE = 'anydbm'
+HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
 HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
 HTTPCACHE_GZIP = False
 
diff --git a/tox.ini b/tox.ini
index eae7e8e47..874a22ee2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -42,6 +42,7 @@ deps =
     -rrequirements-py3.txt
     # Extras
     Pillow
+    leveldb
     -rtests/requirements-py3.txt
 
 [testenv:py34]

From e7ed1fd70df28d529b5f1df04bb850bb260dfc01 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 15 Jan 2016 11:48:07 +0300
Subject: [PATCH 0681/2974] py3 compat in httpcache - headers are bytes

---
 scrapy/extensions/httpcache.py | 52 +++++++++++++++++-----------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 2911dd6bc..80f615818 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -45,7 +45,7 @@ class RFC2616Policy(object):
 
     def _parse_cachecontrol(self, r):
         if r not in self._cc_parsed:
-            cch = r.headers.get('Cache-Control', '')
+            cch = r.headers.get(b'Cache-Control', b'')
             parsed = parse_cachecontrol(cch)
             if isinstance(r, Response):
                 for key in self.ignore_response_cache_controls:
@@ -58,7 +58,7 @@ class RFC2616Policy(object):
             return False
         cc = self._parse_cachecontrol(request)
         # obey user-agent directive "Cache-Control: no-store"
-        if 'no-store' in cc:
+        if b'no-store' in cc:
             return False
         # Any other is eligible for caching
         return True
@@ -69,7 +69,7 @@ class RFC2616Policy(object):
         # Status code 206 is not included because cache can not deal with partial contents
         cc = self._parse_cachecontrol(response)
         # obey directive "Cache-Control: no-store"
-        if 'no-store' in cc:
+        if b'no-store' in cc:
             return False
         # Never cache 304 (Not Modified) responses
         elif response.status == 304:
@@ -78,14 +78,14 @@ class RFC2616Policy(object):
         elif self.always_store:
             return True
         # Any hint on response expiration is good
-        elif 'max-age' in cc or 'Expires' in response.headers:
+        elif b'max-age' in cc or b'Expires' in response.headers:
             return True
         # Firefox fallbacks this statuses to one year expiration if none is set
         elif response.status in (300, 301, 308):
             return True
         # Other statuses without expiration requires at least one validator
         elif response.status in (200, 203, 401):
-            return 'Last-Modified' in response.headers or 'ETag' in response.headers
+            return b'Last-Modified' in response.headers or b'ETag' in response.headers
         # Any other is probably not eligible for caching
         # Makes no sense to cache responses that does not contain expiration
         # info and can not be revalidated
@@ -95,7 +95,7 @@ class RFC2616Policy(object):
     def is_cached_response_fresh(self, cachedresponse, request):
         cc = self._parse_cachecontrol(cachedresponse)
         ccreq = self._parse_cachecontrol(request)
-        if 'no-cache' in cc or 'no-cache' in ccreq:
+        if b'no-cache' in cc or b'no-cache' in ccreq:
             return False
 
         now = time()
@@ -109,7 +109,7 @@ class RFC2616Policy(object):
         if currentage < freshnesslifetime:
             return True
 
-        if 'max-stale' in ccreq and 'must-revalidate' not in cc:
+        if b'max-stale' in ccreq and b'must-revalidate' not in cc:
             # From RFC2616: "Indicates that the client is willing to
             # accept a response that has exceeded its expiration time.
             # If max-stale is assigned a value, then the client is
@@ -117,7 +117,7 @@ class RFC2616Policy(object):
             # expiration time by no more than the specified number of
             # seconds. If no value is assigned to max-stale, then the
             # client is willing to accept a stale response of any age."
-            staleage = ccreq['max-stale']
+            staleage = ccreq[b'max-stale']
             if staleage is None:
                 return True
 
@@ -136,22 +136,22 @@ class RFC2616Policy(object):
         # as long as the old response didn't specify must-revalidate.
         if response.status >= 500:
             cc = self._parse_cachecontrol(cachedresponse)
-            if 'must-revalidate' not in cc:
+            if b'must-revalidate' not in cc:
                 return True
 
         # Use the cached response if the server says it hasn't changed.
         return response.status == 304
 
     def _set_conditional_validators(self, request, cachedresponse):
-        if 'Last-Modified' in cachedresponse.headers:
-            request.headers['If-Modified-Since'] = cachedresponse.headers['Last-Modified']
+        if b'Last-Modified' in cachedresponse.headers:
+            request.headers[b'If-Modified-Since'] = cachedresponse.headers[b'Last-Modified']
 
-        if 'ETag' in cachedresponse.headers:
-            request.headers['If-None-Match'] = cachedresponse.headers['ETag']
+        if b'ETag' in cachedresponse.headers:
+            request.headers[b'If-None-Match'] = cachedresponse.headers[b'ETag']
 
     def _get_max_age(self, cc):
         try:
-            return max(0, int(cc['max-age']))
+            return max(0, int(cc[b'max-age']))
         except (KeyError, ValueError):
             return None
 
@@ -164,18 +164,18 @@ class RFC2616Policy(object):
             return maxage
 
         # Parse date header or synthesize it if none exists
-        date = rfc1123_to_epoch(response.headers.get('Date')) or now
+        date = rfc1123_to_epoch(response.headers.get(b'Date')) or now
 
         # Try HTTP/1.0 Expires header
-        if 'Expires' in response.headers:
-            expires = rfc1123_to_epoch(response.headers['Expires'])
+        if b'Expires' in response.headers:
+            expires = rfc1123_to_epoch(response.headers[b'Expires'])
             # When parsing Expires header fails RFC 2616 section 14.21 says we
             # should treat this as an expiration time in the past.
             return max(0, expires - date) if expires else 0
 
         # Fallback to heuristic using last-modified header
         # This is not in RFC but on Firefox caching implementation
-        lastmodified = rfc1123_to_epoch(response.headers.get('Last-Modified'))
+        lastmodified = rfc1123_to_epoch(response.headers.get(b'Last-Modified'))
         if lastmodified and lastmodified <= date:
             return (date - lastmodified) / 10
 
@@ -192,13 +192,13 @@ class RFC2616Policy(object):
         currentage = 0
         # If Date header is not set we assume it is a fast connection, and
         # clock is in sync with the server
-        date = rfc1123_to_epoch(response.headers.get('Date')) or now
+        date = rfc1123_to_epoch(response.headers.get(b'Date')) or now
         if now > date:
             currentage = now - date
 
-        if 'Age' in response.headers:
+        if b'Age' in response.headers:
             try:
-                age = int(response.headers['Age'])
+                age = int(response.headers[b'Age'])
                 currentage = max(currentage, age)
             except ValueError:
                 pass
@@ -404,16 +404,16 @@ def parse_cachecontrol(header):
 
     http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
 
-    >>> parse_cachecontrol('public, max-age=3600') == {'public': None,
-    ...                                                'max-age': '3600'}
+    >>> parse_cachecontrol(b'public, max-age=3600') == {b'public': None,
+    ...                                                 b'max-age': b'3600'}
     True
-    >>> parse_cachecontrol('') == {}
+    >>> parse_cachecontrol(b'') == {}
     True
 
     """
     directives = {}
-    for directive in header.split(','):
-        key, sep, val = directive.strip().partition('=')
+    for directive in header.split(b','):
+        key, sep, val = directive.strip().partition(b'=')
         if key:
             directives[key.lower()] = val if sep else None
     return directives

From ddc91dda270e44d80dbe08cc79d4249f82cde093 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 15 Jan 2016 11:49:28 +0300
Subject: [PATCH 0682/2974] py3: fix _BaseTest in httpcache

---
 tests/test_downloadermiddleware_httpcache.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 53389ae3b..5a636cc53 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -31,7 +31,7 @@ class _BaseTest(unittest.TestCase):
                                headers={'User-Agent': 'test'})
         self.response = Response('http://www.example.com',
                                  headers={'Content-Type': 'text/html'},
-                                 body='test body',
+                                 body=b'test body',
                                  status=202)
         self.crawler.stats.open_spider(self.spider)
 
@@ -84,9 +84,9 @@ class _BaseTest(unittest.TestCase):
 
     def assertEqualRequestButWithCacheValidators(self, request1, request2):
         self.assertEqual(request1.url, request2.url)
-        assert not 'If-None-Match' in request1.headers
-        assert not 'If-Modified-Since' in request1.headers
-        assert any(h in request2.headers for h in ('If-None-Match', 'If-Modified-Since'))
+        assert not b'If-None-Match' in request1.headers
+        assert not b'If-Modified-Since' in request1.headers
+        assert any(h in request2.headers for h in (b'If-None-Match', b'If-Modified-Since'))
         self.assertEqual(request1.body, request2.body)
 
     def test_dont_cache(self):

From ea0471e33a769f98f853b5ba145b75a9227111a1 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 15 Jan 2016 12:22:41 +0300
Subject: [PATCH 0683/2974] py3: fix LeveldbCacheStorage - using bytes as keys
 and values in leveldb

---
 scrapy/extensions/httpcache.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 80f615818..02f9fcee6 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -12,6 +12,7 @@ from scrapy.responsetypes import responsetypes
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.project import data_path
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_bytes
 
 
 class DummyPolicy(object):
@@ -305,7 +306,7 @@ class FilesystemCacheStorage(object):
             'timestamp': time(),
         }
         with self._open(os.path.join(rpath, 'meta'), 'wb') as f:
-            f.write(repr(metadata))
+            f.write(to_bytes(repr(metadata)))
         with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f:
             pickle.dump(metadata, f, protocol=2)
         with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f:
@@ -373,14 +374,14 @@ class LeveldbCacheStorage(object):
             'body': response.body,
         }
         batch = self._leveldb.WriteBatch()
-        batch.Put('%s_data' % key, pickle.dumps(data, protocol=2))
-        batch.Put('%s_time' % key, str(time()))
+        batch.Put(key + b'_data', pickle.dumps(data, protocol=2))
+        batch.Put(key + b'_time', to_bytes(str(time())))
         self.db.Write(batch)
 
     def _read_data(self, spider, request):
         key = self._request_key(request)
         try:
-            ts = self.db.Get('%s_time' % key)
+            ts = self.db.Get(key + b'_time')
         except KeyError:
             return  # not found or invalid entry
 
@@ -388,14 +389,14 @@ class LeveldbCacheStorage(object):
             return  # expired
 
         try:
-            data = self.db.Get('%s_data' % key)
+            data = self.db.Get(key + b'_data')
         except KeyError:
             return  # invalid entry
         else:
             return pickle.loads(data)
 
     def _request_key(self, request):
-        return request_fingerprint(request)
+        return to_bytes(request_fingerprint(request))
 
 
 

From 87849780bcf77aa47ac42cc1c7b24e1185563c7d Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 15 Jan 2016 13:13:10 +0300
Subject: [PATCH 0684/2974] some py3 fixes for RFC2616Policy

---
 scrapy/extensions/httpcache.py               | 3 ++-
 tests/test_downloadermiddleware_httpcache.py | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 02f9fcee6..91b3ef262 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -41,7 +41,8 @@ class RFC2616Policy(object):
     def __init__(self, settings):
         self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE')
         self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
-        self.ignore_response_cache_controls = settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')
+        self.ignore_response_cache_controls = map(
+            to_bytes, settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS'))
         self._cc_parsed = WeakKeyDictionary()
 
     def _parse_cachecontrol(self, r):
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 5a636cc53..4e0c72304 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -291,7 +291,7 @@ class RFC2616PolicyTest(DefaultStorageTest):
             self.assertEqualResponse(res2, res3)
             # request with no-cache directive must not return cached response
             # but it allows new response to be stored
-            res0b = res0.replace(body='foo')
+            res0b = res0.replace(body=b'foo')
             res4 = self._process_requestresponse(mw, req2, res0b)
             self.assertEqualResponse(res4, res0b)
             assert 'cached' not in res4.flags
@@ -435,7 +435,7 @@ class RFC2616PolicyTest(DefaultStorageTest):
                 assert 'cached' not in res1.flags
                 # Same request but as cached response is stale a new response must
                 # be returned
-                res0b = res0a.replace(body='bar')
+                res0b = res0a.replace(body=b'bar')
                 res2 = self._process_requestresponse(mw, req0, res0b)
                 self.assertEqualResponse(res2, res0b)
                 assert 'cached' not in res2.flags

From 131f4632472922234a7abde116eb566329a952a2 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 15 Jan 2016 11:17:52 +0100
Subject: [PATCH 0685/2974] Allow failures for Python 3.5 for now

---
 .travis.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index ae9c745ac..ac93e337d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,6 +11,9 @@ env:
  - TOXENV=py33
  - TOXENV=py35
  - TOXENV=docs
+matrix:
+  allow_failures:
+    - env: TOXENV=py35
 install:
  - pip install -U tox twine wheel codecov
 script: tox

From 96fcf4cea41a067b4feb4f8adaa5b9ae1d5d38dd Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 15 Jan 2016 13:27:28 +0300
Subject: [PATCH 0686/2974] add a check that byte url is not accepted in
 http.Response on py3

---
 tests/test_http_response.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index b49d46ea8..710a5b29d 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -17,6 +17,8 @@ class BaseResponseTest(unittest.TestCase):
         # Response requires url in the consturctor
         self.assertRaises(Exception, self.response_class)
         self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class))
+        if not six.PY2:
+            self.assertRaises(TypeError, self.response_class, b"http://example.com")
         # body can be str or None
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class))
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class))

From a4ca1668d894920e4a74c8f4204fa3ee53039a1d Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 15 Jan 2016 14:20:19 +0300
Subject: [PATCH 0687/2974] add https test for http10 handler (no luck with
 testing https with http11 so far)

---
 tests/mockserver.py               | 12 ++++++++----
 tests/test_downloader_handlers.py | 18 +++++++++++++++---
 2 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 02bab0efd..e7953c4d4 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -199,14 +199,18 @@ class MockServer():
         time.sleep(0.2)
 
 
+def ssl_context_factory():
+    return ssl.DefaultOpenSSLContextFactory(
+         os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
+         os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
+         )
+
+
 if __name__ == "__main__":
     root = Root()
     factory = Site(root)
     httpPort = reactor.listenTCP(8998, factory)
-    contextFactory = ssl.DefaultOpenSSLContextFactory(
-         os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
-         os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
-         )
+    contextFactory = ssl_context_factory()
     httpsPort = reactor.listenSSL(8999, factory, contextFactory)
 
     def print_listening():
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index a8de28d4b..84d1aa191 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -27,7 +27,7 @@ from scrapy.utils.test import get_crawler
 from scrapy.utils.python import to_bytes
 from scrapy.exceptions import NotConfigured
 
-from tests.mockserver import MockServer
+from tests.mockserver import MockServer, ssl_context_factory
 from tests.spiders import SingleRequestSpider
 
 class DummyDH(object):
@@ -102,6 +102,7 @@ class FileTestCase(unittest.TestCase):
 
 class HttpTestCase(unittest.TestCase):
 
+    scheme = 'http'
     download_handler_cls = HTTPDownloadHandler
 
     def setUp(self):
@@ -118,7 +119,12 @@ class HttpTestCase(unittest.TestCase):
         r.putChild(b"broken", BrokenDownloadResource())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
-        self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1')
+        self.host = '127.0.0.1'
+        if self.scheme == 'https':
+            self.port = reactor.listenSSL(
+                0, self.wrapper, ssl_context_factory(), interface=self.host)
+        else:
+            self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
         self.portno = self.port.getHost().port
         self.download_handler = self.download_handler_cls(Settings())
         self.download_request = self.download_handler.download_request
@@ -130,7 +136,7 @@ class HttpTestCase(unittest.TestCase):
             yield self.download_handler.close()
 
     def getURL(self, path):
-        return "http://127.0.0.1:%d/%s" % (self.portno, path)
+        return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path)
 
     def test_download(self):
         request = Request(self.getURL('file'))
@@ -213,6 +219,12 @@ class Http10TestCase(HttpTestCase):
     download_handler_cls = HTTP10DownloadHandler
 
 
+class Https10TestCase(Http10TestCase):
+    scheme = 'https'
+    def test_timeout_download_from_spider(self):
+        raise unittest.SkipTest("test_timeout_download_from_spider skipped under https")
+
+
 class Http11TestCase(HttpTestCase):
     """HTTP 1.1 test case"""
     download_handler_cls = HTTP11DownloadHandler

From 4398d95a02659c90df496a9156c7b3297cc42f50 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 15 Jan 2016 14:54:12 +0300
Subject: [PATCH 0688/2974] skip this file on py3 again - it has one
 compression test, sould be done separately

---
 tests/py3-ignores.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 0da1b6089..57e80f590 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -7,6 +7,7 @@ tests/test_crawl.py
 tests/test_downloadermiddleware_httpcache.py
 tests/test_downloadermiddleware_httpcompression.py
 tests/test_downloadermiddleware_httpproxy.py
+tests/test_downloadermiddleware.py
 tests/test_downloadermiddleware_retry.py
 tests/test_engine.py
 tests/test_mail.py

From 8330776c2193c3954cba2277525401de35672e00 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 15 Jan 2016 15:16:12 +0300
Subject: [PATCH 0689/2974] fix error reporting in test: we can fail in
 process_request too, so result should always be defined

---
 tests/test_downloadermiddleware_httpcache.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 4e0c72304..12b69860a 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -257,6 +257,7 @@ class RFC2616PolicyTest(DefaultStorageTest):
     policy_class = 'scrapy.extensions.httpcache.RFC2616Policy'
 
     def _process_requestresponse(self, mw, request, response):
+        result = None
         try:
             result = mw.process_request(request, self.spider)
             if result:

From b0648271d69fc3e5b50c430ad50f1afe44acf0d4 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 15 Jan 2016 15:17:30 +0300
Subject: [PATCH 0690/2974] py3 fix for rfc1123_to_epoch - "except Exception"
 was hiding bytes/str error

---
 scrapy/extensions/httpcache.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 91b3ef262..03ea88a10 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -12,7 +12,7 @@ from scrapy.responsetypes import responsetypes
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.project import data_path
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_bytes
+from scrapy.utils.python import to_bytes, to_unicode
 
 
 class DummyPolicy(object):
@@ -423,6 +423,7 @@ def parse_cachecontrol(header):
 
 def rfc1123_to_epoch(date_str):
     try:
+        date_str = to_unicode(date_str, encoding='ascii')
         return mktime_tz(parsedate_tz(date_str))
     except Exception:
         return None

From 085fdd628314d6d3268c6ebbd6e9388fbbb2d37f Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 15 Jan 2016 15:40:45 +0300
Subject: [PATCH 0691/2974] py3 fix for ignoring cache controls - map is not a
 list

---
 scrapy/extensions/httpcache.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 03ea88a10..a871cc895 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -41,8 +41,8 @@ class RFC2616Policy(object):
     def __init__(self, settings):
         self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE')
         self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
-        self.ignore_response_cache_controls = map(
-            to_bytes, settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS'))
+        self.ignore_response_cache_controls = [to_bytes(cc) for cc in
+            settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')]
         self._cc_parsed = WeakKeyDictionary()
 
     def _parse_cachecontrol(self, r):

From 7d44c5dcea6c523b4cac19eefa5eb22966c5a90f Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 15 Jan 2016 15:41:31 +0300
Subject: [PATCH 0692/2974] py3: unskip
 tests/test_downloadermiddleware_httpcache.py and
 scrapy/downloadermiddlewares/httpcache.py

---
 tests/py3-ignores.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 57e80f590..2a9f06c8c 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -4,7 +4,6 @@ tests/test_command_shell.py
 tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
-tests/test_downloadermiddleware_httpcache.py
 tests/test_downloadermiddleware_httpcompression.py
 tests/test_downloadermiddleware_httpproxy.py
 tests/test_downloadermiddleware.py
@@ -31,7 +30,6 @@ scrapy/linkextractors/sgml.py
 scrapy/linkextractors/regex.py
 scrapy/linkextractors/htmlparser.py
 scrapy/downloadermiddlewares/retry.py
-scrapy/downloadermiddlewares/httpcache.py
 scrapy/downloadermiddlewares/httpproxy.py
 scrapy/downloadermiddlewares/cookies.py
 scrapy/extensions/statsmailer.py

From ee4fadc00724f02f9098625ac4d72fb29eac4dcf Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 15 Jan 2016 14:57:15 +0100
Subject: [PATCH 0693/2974] Use .read1() if available when using GzipFile

---
 .travis.yml           |  3 ---
 scrapy/utils/gz.py    | 20 ++++++++++++++------
 tests/test_squeues.py |  4 ++--
 3 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index ac93e337d..ae9c745ac 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,9 +11,6 @@ env:
  - TOXENV=py33
  - TOXENV=py35
  - TOXENV=docs
-matrix:
-  allow_failures:
-    - env: TOXENV=py35
 install:
  - pip install -U tox twine wheel codecov
 script: tox
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 7fa4bba57..df1d29698 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -4,30 +4,38 @@ try:
     from cStringIO import StringIO as BytesIO
 except ImportError:
     from io import BytesIO
-
+from io import UnsupportedOperation
 from gzip import GzipFile
 
+class ReadOneGzipFile(GzipFile):
+    def readone(self, size=-1):
+        try:
+            return self.read1(size)
+        except UnsupportedOperation:
+            return self.read(size)
 
 def gunzip(data):
     """Gunzip the given data and return as much data as possible.
 
     This is resilient to CRC checksum errors.
     """
-    f = GzipFile(fileobj=BytesIO(data))
+    f = ReadOneGzipFile(fileobj=BytesIO(data))
     output = b''
     chunk = b'.'
     while chunk:
         try:
-            chunk = f.read(8196)
+            chunk = f.readone(8196)
             output += chunk
         except (IOError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise
             # see issue 87 about catching struct.error
             # some pages are quite small so output is '' and f.extrabuf
             # contains the whole page content
-            if output or f.extrabuf:
-                output += f.extrabuf
-                break
+            if output or getattr(f, 'extrabuf', None):
+                try:
+                    output += f.extrabuf
+                finally:
+                    break
             else:
                 raise
     return output
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 48871ceeb..232f539e6 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -34,7 +34,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
         # Trigger Twisted bug #7989
         import twisted.persisted.styles  # NOQA
         q = self.queue()
-        self.assertRaises(ValueError, q.push, lambda x: x)
+        self.assertRaises((ValueError, AttributeError), q.push, lambda x: x)
 
 class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
     chunksize = 1
@@ -114,7 +114,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
         # Trigger Twisted bug #7989
         import twisted.persisted.styles  # NOQA
         q = self.queue()
-        self.assertRaises(ValueError, q.push, lambda x: x)
+        self.assertRaises((ValueError, AttributeError), q.push, lambda x: x)
 
 
 class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest):

From 73d78ec99fc622fe9a552c78193c0b4bf652533b Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 15 Jan 2016 17:59:20 +0100
Subject: [PATCH 0694/2974] Add Code of Conduct Version 1.3.0 from
 http://contributor-covenant.org/

Closes #1645
---
 CODE_OF_CONDUCT.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++
 README.rst         |  6 ++++++
 2 files changed, 56 insertions(+)
 create mode 100644 CODE_OF_CONDUCT.md

diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
new file mode 100644
index 000000000..95b4a7e3c
--- /dev/null
+++ b/CODE_OF_CONDUCT.md
@@ -0,0 +1,50 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, and in the interest of
+fostering an open and welcoming community, we pledge to respect all people who
+contribute through reporting issues, posting feature requests, updating
+documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free
+experience for everyone, regardless of level of experience, gender, gender
+identity and expression, sexual orientation, disability, personal appearance,
+body size, race, ethnicity, age, religion, or nationality.
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery
+* Personal attacks
+* Trolling or insulting/derogatory comments
+* Public or private harassment
+* Publishing other's private information, such as physical or electronic
+  addresses, without explicit permission
+* Other unethical or unprofessional conduct
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+By adopting this Code of Conduct, project maintainers commit themselves to
+fairly and consistently applying these principles to every aspect of managing
+this project. Project maintainers who do not follow or enforce the Code of
+Conduct may be permanently removed from the project team.
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting a project maintainer at opensource@scrapinghub.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. Maintainers are
+obligated to maintain confidentiality with regard to the reporter of an
+incident.
+
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 1.3.0, available at
+[http://contributor-covenant.org/version/1/3/0/][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/3/0/
diff --git a/README.rst b/README.rst
index 6cbed75ee..8a7d2c71d 100644
--- a/README.rst
+++ b/README.rst
@@ -73,6 +73,12 @@ See http://scrapy.org/community/
 Contributing
 ============
 
+Please note that this project is released with a Contributor Code of Conduct
+(see CODE_OF_CONDUCT.md).
+
+By participating in this project you agree to abide by its terms.
+Please report unacceptable behavior to opensource@scrapinghub.com.
+
 See http://doc.scrapy.org/en/master/contributing.html
 
 Companies using Scrapy

From bb38400db560a4d814ea6c12364404d6daa75da5 Mon Sep 17 00:00:00 2001
From: Ralph Gutkowski <goodkowski@gmail.com>
Date: Fri, 15 Jan 2016 19:00:58 +0100
Subject: [PATCH 0695/2974] Update Stats Collection documentation

`pages_crawled` value doesn't exist. Replace it with `downloader/response_count` in order to avoid confusion.
---
 docs/topics/stats.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst
index 0837610d0..290fc065c 100644
--- a/docs/topics/stats.rst
+++ b/docs/topics/stats.rst
@@ -47,7 +47,7 @@ Set stat value::
 
 Increment stat value::
 
-    stats.inc_value('pages_crawled')
+    stats.inc_value('downloader/response_count')
 
 Set stat value only if greater than previous::
 
@@ -59,13 +59,13 @@ Set stat value only if lower than previous::
 
 Get stat value::
 
-    >>> stats.get_value('pages_crawled')
+    >>> stats.get_value('downloader/response_count')
     8
 
 Get all stats::
 
     >>> stats.get_stats()
-    {'pages_crawled': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
+    {'downloader/response_count': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
 
 Available Stats Collectors
 ==========================

From 79147a61a797f81fee44b000eb5ac9591593ff88 Mon Sep 17 00:00:00 2001
From: Ralph Gutkowski <goodkowski@gmail.com>
Date: Fri, 15 Jan 2016 19:25:56 +0100
Subject: [PATCH 0696/2974] Update stats.rst

---
 docs/topics/stats.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst
index 290fc065c..dd0c6216b 100644
--- a/docs/topics/stats.rst
+++ b/docs/topics/stats.rst
@@ -47,7 +47,7 @@ Set stat value::
 
 Increment stat value::
 
-    stats.inc_value('downloader/response_count')
+    stats.inc_value('custom_count')
 
 Set stat value only if greater than previous::
 
@@ -59,13 +59,13 @@ Set stat value only if lower than previous::
 
 Get stat value::
 
-    >>> stats.get_value('downloader/response_count')
-    8
+    >>> stats.get_value('custom_count')
+    1
 
 Get all stats::
 
     >>> stats.get_stats()
-    {'downloader/response_count': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
+    {'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
 
 Available Stats Collectors
 ==========================

From 4e44766653c294c54a3bb960b3734ee70bc663a4 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 15 Jan 2016 19:51:21 +0100
Subject: [PATCH 0697/2974] Use "unicode" string for lxml.etree.tostring()
 serialization

---
 scrapy/utils/iterators.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 69c7f2c23..b0688791e 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -48,7 +48,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
     iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
     selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename)
     for _, node in iterable:
-        nodetext = etree.tostring(node, encoding=six.text_type)
+        nodetext = etree.tostring(node, encoding='unicode')
         node.clear()
         xs = Selector(text=nodetext, type='xml')
         if namespace:

From cd735e377c2a26bc8ebb925bf6031429549faadf Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@bluewin.ch>
Date: Mon, 18 Jan 2016 07:45:36 +0100
Subject: [PATCH 0698/2974] Simplify if statement

---
 conftest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/conftest.py b/conftest.py
index b0ac1badd..8b4faf8fc 100644
--- a/conftest.py
+++ b/conftest.py
@@ -34,7 +34,7 @@ if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15,
 if six.PY3:
     for line in open('tests/py3-ignores.txt'):
         file_path = line.strip()
-        if len(file_path) > 0 and file_path[0] != '#':
+        if file_path and file_path[0] != '#':
             collect_ignore.append(file_path)
 
 

From d3c2b0cf7ec185d18464e85974ee7499acecad79 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 10:06:14 +0300
Subject: [PATCH 0699/2974] py3 webclient: I was mistaken about unicode body,
 revert conversion to bytes and fix HEAD response

---
 scrapy/core/downloader/webclient.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index d2cdd6f98..bbbb98f60 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -66,7 +66,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
 
     def handleResponse(self, response):
         if self.factory.method.upper() == b'HEAD':
-            self.factory.page('')
+            self.factory.page(b'')
         elif self.length is not None and self.length > 0:
             self.factory.noPage(self._connection_lost_reason)
         else:
@@ -131,7 +131,6 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
         status = int(self.status)
         headers = Headers(self.response_headers)
         respcls = responsetypes.from_args(headers=headers, url=self._url)
-        body = to_bytes(body)
         return respcls(url=self._url, status=status, headers=headers, body=body)
 
     def _set_connection_attributes(self, request):

From 673df5e4161337136154eeccfc4e327d053dbd12 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 11:44:49 +0300
Subject: [PATCH 0700/2974] add webclient test - check that non-standart body
 encoding matches Content-Encoding header

---
 tests/test_webclient.py | 31 ++++++++++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 412e10c89..3ee6c24c2 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -7,7 +7,7 @@ import six
 from six.moves.urllib.parse import urlparse
 
 from twisted.trial import unittest
-from twisted.web import server, static, error, util
+from twisted.web import server, static, util, resource
 from twisted.internet import reactor, defer
 from twisted.test.proto_helpers import StringTransport
 from twisted.python.filepath import FilePath
@@ -18,14 +18,14 @@ from scrapy.http import Request, Headers
 from scrapy.utils.python import to_bytes, to_unicode
 
 
-def getPage(url, contextFactory=None, *args, **kwargs):
+def getPage(url, contextFactory=None, r_transform=None, *args, **kwargs):
     """Adapted version of twisted.web.client.getPage"""
     def _clientfactory(url, *args, **kwargs):
         url = to_unicode(url)
         timeout = kwargs.pop('timeout', 0)
         f = client.ScrapyHTTPClientFactory(
             Request(url, *args, **kwargs), timeout=timeout)
-        f.deferred.addCallback(lambda r: r.body)
+        f.deferred.addCallback(r_transform or (lambda r: r.body))
         return f
 
     from twisted.web.client import _makeGetterFactory
@@ -213,6 +213,16 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \
         ErrorResource, NoLengthResource, HostHeaderResource, \
         PayloadResource, BrokenDownloadResource
 
+
+class EncodingResource(resource.Resource):
+    out_encoding = 'cp1251'
+
+    def render(self, request):
+        body = to_unicode(request.content.read())
+        request.setHeader(b'content-encoding', self.out_encoding)
+        return body.encode(self.out_encoding)
+
+
 class WebClientTestCase(unittest.TestCase):
     def _listen(self, site):
         return reactor.listenTCP(0, site, interface="127.0.0.1")
@@ -229,6 +239,7 @@ class WebClientTestCase(unittest.TestCase):
         r.putChild(b"host", HostHeaderResource())
         r.putChild(b"payload", PayloadResource())
         r.putChild(b"broken", BrokenDownloadResource())
+        r.putChild(b"encoding", EncodingResource())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
         self.port = self._listen(self.wrapper)
@@ -338,3 +349,17 @@ class WebClientTestCase(unittest.TestCase):
                 b'\n<html>\n    <head>\n        <meta http-equiv="refresh" content="0;URL=/file">\n'
                 b'    </head>\n    <body bgcolor="#FFFFFF" text="#000000">\n    '
                 b'<a href="/file">click here</a>\n    </body>\n</html>\n')
+
+    def test_Encoding(self):
+        """ Test that non-standart body encoding matches
+        Content-Encoding header """
+        body = b'\xd0\x81\xd1\x8e\xd0\xaf'
+        return getPage(
+            self.getURL('encoding'), body=body, r_transform=lambda r: r)\
+            .addCallback(self._check_Encoding, body)
+
+    def _check_Encoding(self, response, original_body):
+        content_encoding = to_unicode(response.headers[b'Content-Encoding'])
+        self.assertEquals(content_encoding, EncodingResource.out_encoding)
+        self.assertEquals(
+            response.body.decode(content_encoding), to_unicode(original_body))

From 6b905a9aecb0e0e22339353a983257f3707d98b8 Mon Sep 17 00:00:00 2001
From: palego <palego@users.noreply.github.com>
Date: Sat, 16 Jan 2016 14:23:58 +0100
Subject: [PATCH 0701/2974] split-up the assertIn

to deal with OS X intricacies (directories prefixed with /private)
---
 tests/test_commands.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 8edccd4bd..aa1b7cc7a 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -90,8 +90,8 @@ class StartprojectTemplatesTest(ProjectTest):
         args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl]
         p = self.proc('startproject', self.project_name, *args)
         out = to_native_str(retry_on_eintr(p.stdout.read))
-        self.assertIn("New Scrapy project %r, using template directory %r, created in:" % \
-                      (self.project_name, join(self.tmpl, 'project')), out)
+        self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out)
+        self.assertIn(self.tmpl_proj, out)
         assert exists(join(self.proj_path, 'root_template'))
 
 

From f2e2ff5e1f2eb1d11e18be248e27bef7fc9b4ef7 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 12:20:01 +0300
Subject: [PATCH 0702/2974] py3: webclient assumes that urls come from
 Request.url and are ascii-only

---
 scrapy/core/downloader/webclient.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index bbbb98f60..9bcc51943 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -12,18 +12,26 @@ from scrapy.responsetypes import responsetypes
 
 
 def _parsed_url_args(parsed):
+    # Assume parsed is urlparse-d from Request.url,
+    # which was passed via safe_url_string and is ascii-only.
+    b = lambda s: to_bytes(s, encoding='ascii')
     path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
-    path = to_bytes(path)
-    host = to_bytes(parsed.hostname)
+    path = b(path)
+    host = b(parsed.hostname)
     port = parsed.port
-    scheme = to_bytes(parsed.scheme, encoding='ascii')
-    netloc = to_bytes(parsed.netloc)
+    scheme = b(parsed.scheme)
+    netloc = b(parsed.netloc)
     if port is None:
         port = 443 if scheme == b'https' else 80
     return scheme, netloc, host, port, path
 
 
 def _parse(url):
+    """ Return tuple of (scheme, netloc, host, port, path),
+    all in bytes except for port which is int.
+    Assume url is from Request.url, which was passed via safe_url_string
+    and is ascii-only.
+    """
     url = url.strip()
     parsed = urlparse(url)
     return _parsed_url_args(parsed)
@@ -95,7 +103,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
     def __init__(self, request, timeout=180):
         self._url = urldefrag(request.url)[0]
         # converting to bytes to comply to Twisted interface
-        self.url = to_bytes(self._url)
+        self.url = to_bytes(self._url, encoding='ascii')
         self.method = to_bytes(request.method, encoding='ascii')
         self.body = request.body or None
         self.headers = Headers(request.headers)

From 04f69fd18406a9361f3c13df32b5f6ea295ecdbf Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 12:37:46 +0300
Subject: [PATCH 0703/2974] add https 1.1 downloader test - localhost is a
 valid DNS-ID

---
 tests/test_downloader_handlers.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 84d1aa191..80eed86f2 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -119,7 +119,7 @@ class HttpTestCase(unittest.TestCase):
         r.putChild(b"broken", BrokenDownloadResource())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
-        self.host = '127.0.0.1'
+        self.host = 'localhost'
         if self.scheme == 'https':
             self.port = reactor.listenSSL(
                 0, self.wrapper, ssl_context_factory(), interface=self.host)
@@ -273,6 +273,10 @@ class Http11TestCase(HttpTestCase):
         return d
 
 
+class Https11TestCase(Http11TestCase):
+    scheme = 'https'
+
+
 class Http11MockServerTestCase(unittest.TestCase):
     """HTTP 1.1 test case with MockServer"""
     if twisted_version < (11, 1, 0):

From 98c060d0b2cc76934e16abc03a033f21850fd565 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 12:42:21 +0300
Subject: [PATCH 0704/2974] py3: fix http 1.1 test with https, and use
 self.host everywhere

---
 tests/test_downloader_handlers.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 80eed86f2..999fa4c0a 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -168,6 +168,9 @@ class HttpTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_timeout_download_from_spider(self):
+        if self.scheme == 'https':
+            raise unittest.SkipTest(
+                'test_timeout_download_from_spider skipped under https')
         spider = Spider('foo')
         meta = {'download_timeout': 0.2}
         # client connects but no data is received
@@ -181,7 +184,8 @@ class HttpTestCase(unittest.TestCase):
 
     def test_host_header_not_in_request_headers(self):
         def _test(response):
-            self.assertEquals(response.body, to_bytes('127.0.0.1:%d' % self.portno))
+            self.assertEquals(
+                response.body, to_bytes('%s:%d' % (self.host, self.portno)))
             self.assertEquals(request.headers, {})
 
         request = Request(self.getURL('host'))
@@ -221,8 +225,6 @@ class Http10TestCase(HttpTestCase):
 
 class Https10TestCase(Http10TestCase):
     scheme = 'https'
-    def test_timeout_download_from_spider(self):
-        raise unittest.SkipTest("test_timeout_download_from_spider skipped under https")
 
 
 class Http11TestCase(HttpTestCase):

From 0f527849f2e8eddaf5d756b061699f2eca522a18 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 14:44:04 +0300
Subject: [PATCH 0705/2974] https proxy tunneling - add a test (not perfect,
 but covers all impl) and fix for py3

---
 scrapy/core/downloader/handlers/http11.py | 14 +++++++++-----
 tests/test_downloader_handlers.py         | 10 ++++++++++
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index d81093a9f..729b80b05 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -78,7 +78,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
     for it.
     """
 
-    _responseMatcher = re.compile('HTTP/1\.. 200')
+    _responseMatcher = re.compile(b'HTTP/1\.. 200')
 
     def __init__(self, reactor, host, port, proxyConf, contextFactory,
                  timeout=30, bindAddress=None):
@@ -92,11 +92,15 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
 
     def requestTunnel(self, protocol):
         """Asks the proxy to open a tunnel."""
-        tunnelReq = 'CONNECT %s:%s HTTP/1.1\r\n' % (self._tunneledHost,
-                                                  self._tunneledPort)
+        tunnelReq = (
+            b'CONNECT ' +
+            to_bytes(self._tunneledHost, encoding='ascii') + b':' +
+            to_bytes(str(self._tunneledPort)) +
+            b' HTTP/1.1\r\n')
         if self._proxyAuthHeader:
-            tunnelReq += 'Proxy-Authorization: %s\r\n' % self._proxyAuthHeader
-        tunnelReq += '\r\n'
+            tunnelReq += \
+                b'Proxy-Authorization: ' + self._proxyAuthHeader + b'\r\n'
+        tunnelReq += b'\r\n'
         protocol.transport.write(tunnelReq)
         self._protocolDataReceived = protocol.dataReceived
         protocol.dataReceived = self.processProxyResponse
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 999fa4c0a..2d6c05741 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -388,6 +388,16 @@ class HttpProxyTestCase(unittest.TestCase):
         request = Request('https://example.com', meta={'proxy': http_proxy})
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
+    @defer.inlineCallbacks
+    def test_download_with_proxy_https_timeout(self):
+        http_proxy = self.getURL('')
+        domain = 'https://no-such-domain.nosuch'
+        request = Request(
+            domain, meta={'proxy': http_proxy, 'download_timeout': 0.2})
+        d = self.download_request(request, Spider('foo'))
+        timeout = yield self.assertFailure(d, error.TimeoutError)
+        self.assertIn(domain, timeout.osError)
+
     def test_download_without_proxy(self):
         def _test(response):
             self.assertEquals(response.status, 200)

From 7af64e8fd2a90102d85ff0f453cd6ba1dde71caa Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 15:00:43 +0300
Subject: [PATCH 0706/2974] py3: remove extra encoding/decoding of url: pass it
 as bytes only when required

---
 scrapy/core/downloader/handlers/http11.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 729b80b05..82cf507f7 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -206,7 +206,7 @@ class ScrapyAgent(object):
         agent = self._get_agent(request, timeout)
 
         # request details
-        url = to_bytes(urldefrag(request.url)[0])
+        url = urldefrag(request.url)[0]
         method = to_bytes(request.method)
         headers = TxHeaders(request.headers)
         if isinstance(agent, self._TunnelingAgent):
@@ -214,7 +214,8 @@ class ScrapyAgent(object):
         bodyproducer = _RequestBodyProducer(request.body) if request.body else None
 
         start_time = time()
-        d = agent.request(method, url, headers, bodyproducer)
+        d = agent.request(
+            method, to_bytes(url, encoding='ascii'), headers, bodyproducer)
         # set download latency
         d.addCallback(self._cb_latency, request, start_time)
         # response body is ready to be consumed
@@ -267,10 +268,8 @@ class ScrapyAgent(object):
         txresponse, body, flags = result
         status = int(txresponse.code)
         headers = Headers(txresponse.headers.getAllRawHeaders())
-        url = to_unicode(url)
         respcls = responsetypes.from_args(headers=headers, url=url)
-        return respcls(
-            url=url, status=status, headers=headers, body=body, flags=flags)
+        return respcls(url=url, status=status, headers=headers, body=body, flags=flags)
 
 
 @implementer(IBodyProducer)

From b940606b7e3dacbf5d639965c54ccbb87a214c95 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 15:06:15 +0300
Subject: [PATCH 0707/2974] this is a test for TunnelingTCP4ClientEndpoint -
 move into Http11ProxyTestCase

---
 tests/test_downloader_handlers.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 2d6c05741..59320597e 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -388,16 +388,6 @@ class HttpProxyTestCase(unittest.TestCase):
         request = Request('https://example.com', meta={'proxy': http_proxy})
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
-    @defer.inlineCallbacks
-    def test_download_with_proxy_https_timeout(self):
-        http_proxy = self.getURL('')
-        domain = 'https://no-such-domain.nosuch'
-        request = Request(
-            domain, meta={'proxy': http_proxy, 'download_timeout': 0.2})
-        d = self.download_request(request, Spider('foo'))
-        timeout = yield self.assertFailure(d, error.TimeoutError)
-        self.assertIn(domain, timeout.osError)
-
     def test_download_without_proxy(self):
         def _test(response):
             self.assertEquals(response.status, 200)
@@ -422,6 +412,17 @@ class Http11ProxyTestCase(HttpProxyTestCase):
     if twisted_version < (11, 1, 0):
         skip = 'HTTP1.1 not supported in twisted < 11.1.0'
 
+    @defer.inlineCallbacks
+    def test_download_with_proxy_https_timeout(self):
+        """ Test TunnelingTCP4ClientEndpoint """
+        http_proxy = self.getURL('')
+        domain = 'https://no-such-domain.nosuch'
+        request = Request(
+            domain, meta={'proxy': http_proxy, 'download_timeout': 0.2})
+        d = self.download_request(request, Spider('foo'))
+        timeout = yield self.assertFailure(d, error.TimeoutError)
+        self.assertIn(domain, timeout.osError)
+
 
 class HttpDownloadHandlerMock(object):
     def __init__(self, settings):

From a2efd389b09984de15ee0653d169c0bbbfd61a05 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 15:09:54 +0300
Subject: [PATCH 0708/2974] clarify: rename r_transform to response_transform

---
 tests/test_webclient.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 3ee6c24c2..dbe659d5c 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -18,14 +18,14 @@ from scrapy.http import Request, Headers
 from scrapy.utils.python import to_bytes, to_unicode
 
 
-def getPage(url, contextFactory=None, r_transform=None, *args, **kwargs):
+def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
     """Adapted version of twisted.web.client.getPage"""
     def _clientfactory(url, *args, **kwargs):
         url = to_unicode(url)
         timeout = kwargs.pop('timeout', 0)
         f = client.ScrapyHTTPClientFactory(
             Request(url, *args, **kwargs), timeout=timeout)
-        f.deferred.addCallback(r_transform or (lambda r: r.body))
+        f.deferred.addCallback(response_transform or (lambda r: r.body))
         return f
 
     from twisted.web.client import _makeGetterFactory
@@ -355,7 +355,7 @@ class WebClientTestCase(unittest.TestCase):
         Content-Encoding header """
         body = b'\xd0\x81\xd1\x8e\xd0\xaf'
         return getPage(
-            self.getURL('encoding'), body=body, r_transform=lambda r: r)\
+            self.getURL('encoding'), body=body, response_transform=lambda r: r)\
             .addCallback(self._check_Encoding, body)
 
     def _check_Encoding(self, response, original_body):

From 494643458270311341c509f5476c61737aa27a70 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 15:23:01 +0300
Subject: [PATCH 0709/2974] revert most changes to this test, and clarify - it
 is valid only on py2, because urls are strictly unicode on py3

---
 tests/test_webclient.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index dbe659d5c..9b5beda4c 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -78,16 +78,17 @@ class ParseUrlTestCase(unittest.TestCase):
         elements of its return tuple, even when passed an URL which has
         previously been passed to L{urlparse} as a C{unicode} string.
         """
-        goodInput = u'http://example.com/path'
-        badInput = goodInput.encode('ascii')
-        if six.PY2:
-            goodInput, badInput = badInput, goodInput
-        urlparse(badInput)
+        if not six.PY2:
+            raise unittest.SkipTest(
+                "Applies only to Py2, as urls can be ONLY unicode on Py3")
+        badInput = u'http://example.com/path'
+        goodInput = badInput.encode('ascii')
+        self._parse(badInput)  # cache badInput in urlparse_cached
         scheme, netloc, host, port, path = self._parse(goodInput)
-        self.assertTrue(isinstance(scheme, bytes))
-        self.assertTrue(isinstance(netloc, bytes))
-        self.assertTrue(isinstance(host, bytes))
-        self.assertTrue(isinstance(path, bytes))
+        self.assertTrue(isinstance(scheme, str))
+        self.assertTrue(isinstance(netloc, str))
+        self.assertTrue(isinstance(host, str))
+        self.assertTrue(isinstance(path, str))
         self.assertTrue(isinstance(port, int))
 
 

From 0b9336418ef40ca95052ebbaa02f12953e165115 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 16:43:58 +0300
Subject: [PATCH 0710/2974] py3: port compression downloader middleware and
 tests

---
 .../downloadermiddlewares/httpcompression.py  |  8 +++----
 tests/py3-ignores.txt                         |  1 -
 ...st_downloadermiddleware_httpcompression.py | 24 +++++++++----------
 3 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 719507396..7ab304c17 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -9,13 +9,13 @@ from scrapy.exceptions import NotConfigured
 class HttpCompressionMiddleware(object):
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
-    
+
     @classmethod
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('COMPRESSION_ENABLED'):
             raise NotConfigured
         return cls()
-    
+
     def process_request(self, request, spider):
         request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
 
@@ -39,10 +39,10 @@ class HttpCompressionMiddleware(object):
         return response
 
     def _decode(self, body, encoding):
-        if encoding == 'gzip' or encoding == 'x-gzip':
+        if encoding == b'gzip' or encoding == b'x-gzip':
             body = gunzip(body)
 
-        if encoding == 'deflate':
+        if encoding == b'deflate':
             try:
                 body = zlib.decompress(body)
             except zlib.error:
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 2a9f06c8c..dbf63f0f5 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -4,7 +4,6 @@ tests/test_command_shell.py
 tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
-tests/test_downloadermiddleware_httpcompression.py
 tests/test_downloadermiddleware_httpproxy.py
 tests/test_downloadermiddleware.py
 tests/test_downloadermiddleware_retry.py
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index a18994ef3..2e6e47fef 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -50,46 +50,46 @@ class HttpCompressionTest(TestCase):
         request = Request('http://scrapytest.org')
         assert 'Accept-Encoding' not in request.headers
         self.mw.process_request(request, self.spider)
-        self.assertEqual(request.headers.get('Accept-Encoding'), 'gzip,deflate')
+        self.assertEqual(request.headers.get('Accept-Encoding'), b'gzip,deflate')
 
     def test_process_response_gzip(self):
         response = self._getresponse('gzip')
         request = response.request
 
-        self.assertEqual(response.headers['Content-Encoding'], 'gzip')
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is not response
-        assert newresponse.body.startswith('<!DOCTYPE')
+        assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
 
     def test_process_response_rawdeflate(self):
         response = self._getresponse('rawdeflate')
         request = response.request
 
-        self.assertEqual(response.headers['Content-Encoding'], 'deflate')
+        self.assertEqual(response.headers['Content-Encoding'], b'deflate')
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is not response
-        assert newresponse.body.startswith('<!DOCTYPE')
+        assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
 
     def test_process_response_zlibdelate(self):
         response = self._getresponse('zlibdeflate')
         request = response.request
 
-        self.assertEqual(response.headers['Content-Encoding'], 'deflate')
+        self.assertEqual(response.headers['Content-Encoding'], b'deflate')
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is not response
-        assert newresponse.body.startswith('<!DOCTYPE')
+        assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
 
     def test_process_response_plain(self):
-        response = Response('http://scrapytest.org', body='<!DOCTYPE...')
+        response = Response('http://scrapytest.org', body=b'<!DOCTYPE...')
         request = Request('http://scrapytest.org')
 
         assert not response.headers.get('Content-Encoding')
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is response
-        assert newresponse.body.startswith('<!DOCTYPE')
+        assert newresponse.body.startswith(b'<!DOCTYPE')
 
     def test_multipleencodings(self):
         response = self._getresponse('gzip')
@@ -97,7 +97,7 @@ class HttpCompressionTest(TestCase):
         request = response.request
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is not response
-        self.assertEqual(newresponse.headers.getlist('Content-Encoding'), ['uuencode'])
+        self.assertEqual(newresponse.headers.getlist('Content-Encoding'), [b'uuencode'])
 
     def test_process_response_encoding_inside_body(self):
         headers = {
@@ -142,5 +142,5 @@ class HttpCompressionTest(TestCase):
 
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIs(newresponse, response)
-        self.assertEqual(response.headers['Content-Encoding'], 'gzip')
-        self.assertEqual(response.headers['Content-Type'], 'application/gzip')
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        self.assertEqual(response.headers['Content-Type'], b'application/gzip')

From 324f2c1a6fb00b0ad4ea870ed61613bb288f929f Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 16:45:22 +0300
Subject: [PATCH 0711/2974] common test_downloadermiddleware.py also passes now
 due to fixes in compression middleware

---
 tests/py3-ignores.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index dbf63f0f5..1f7d85ef7 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -5,7 +5,6 @@ tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
 tests/test_downloadermiddleware_httpproxy.py
-tests/test_downloadermiddleware.py
 tests/test_downloadermiddleware_retry.py
 tests/test_engine.py
 tests/test_mail.py

From bcbad2905d3ff375e85ede723d13560d765ee729 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 18 Jan 2016 15:22:29 +0100
Subject: [PATCH 0712/2974] Stick with ValueError for queue/serialization
 exception tests

---
 tests/test_squeues.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 232f539e6..48871ceeb 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -34,7 +34,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
         # Trigger Twisted bug #7989
         import twisted.persisted.styles  # NOQA
         q = self.queue()
-        self.assertRaises((ValueError, AttributeError), q.push, lambda x: x)
+        self.assertRaises(ValueError, q.push, lambda x: x)
 
 class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
     chunksize = 1
@@ -114,7 +114,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
         # Trigger Twisted bug #7989
         import twisted.persisted.styles  # NOQA
         q = self.queue()
-        self.assertRaises((ValueError, AttributeError), q.push, lambda x: x)
+        self.assertRaises(ValueError, q.push, lambda x: x)
 
 
 class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest):

From 120fb4adeb957a348f01bf60e821b2dcab2f9de1 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 19:00:40 +0300
Subject: [PATCH 0713/2974] revert bogus change

---
 scrapy/http/response/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 59ef15682..09c4e725a 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -35,7 +35,7 @@ class Response(object_ref):
         return self._url
 
     def _set_url(self, url):
-        if isinstance(url, six.string_types):
+        if isinstance(url, str):
             self._url = url
         else:
             raise TypeError('%s url must be str, got %s:' % (type(self).__name__,

From 7fdd3225b293d4951ba079e1141683e7fc55f905 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 19:09:09 +0300
Subject: [PATCH 0714/2974] fix test skipping logic - this is (temporary)
 py2-only part

---
 tests/test_downloader_handlers.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 59320597e..e6d219168 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -322,18 +322,18 @@ class Http11MockServerTestCase(unittest.TestCase):
             # download_maxsize < 100, hence the CancelledError
             self.assertIsInstance(failure.value, defer.CancelledError)
 
-            request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
-            request = request.replace(url='http://localhost:8998/xpayload')
-            yield crawler.crawl(seed=request)
-
             if six.PY2:
+                request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
+                request = request.replace(url='http://localhost:8998/xpayload')
+                yield crawler.crawl(seed=request)
                 # download_maxsize = 50 is enough for the gzipped response
                 # See issue https://twistedmatrix.com/trac/ticket/8175
-                raise unittest.SkipTest("xpayload only enabled for PY2")
                 failure = crawler.spider.meta.get('failure')
                 self.assertTrue(failure == None)
                 reason = crawler.spider.meta['close_reason']
                 self.assertTrue(reason, 'finished')
+            else:
+                raise unittest.SkipTest("xpayload only enabled for PY2")
         else:
             raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0")
 

From de98d8d00658181ed46a96835e2ab95f2f6cd457 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 19:27:31 +0300
Subject: [PATCH 0715/2974] move comment about test skipped on py3 into a
 proper place

---
 tests/test_downloader_handlers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index e6d219168..c936b72ed 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -327,12 +327,12 @@ class Http11MockServerTestCase(unittest.TestCase):
                 request = request.replace(url='http://localhost:8998/xpayload')
                 yield crawler.crawl(seed=request)
                 # download_maxsize = 50 is enough for the gzipped response
-                # See issue https://twistedmatrix.com/trac/ticket/8175
                 failure = crawler.spider.meta.get('failure')
                 self.assertTrue(failure == None)
                 reason = crawler.spider.meta['close_reason']
                 self.assertTrue(reason, 'finished')
             else:
+                # See issue https://twistedmatrix.com/trac/ticket/8175
                 raise unittest.SkipTest("xpayload only enabled for PY2")
         else:
             raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0")

From fd99ef86dfca50dbd36b2c1a022cf30a0720dbea Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 18 Jan 2016 17:57:55 +0100
Subject: [PATCH 0716/2974] Test for AttributeError when pickling objects
 (Python>=3.5)

Same "fix" as in e.g. https://github.com/joblib/joblib/pull/246
---
 scrapy/squeues.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 6e2a60fd2..21520f454 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -25,7 +25,9 @@ def _serializable_queue(queue_class, serialize, deserialize):
 def _pickle_serialize(obj):
     try:
         return pickle.dumps(obj, protocol=2)
-    except pickle.PicklingError as e:
+    # Python>=3.5 raises AttributeError here while
+    # Python<=3.4 raises pickle.PicklingError
+    except (pickle.PicklingError, AttributeError) as e:
         raise ValueError(str(e))
 
 PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \

From ff235fa19ad30f34be7ff435497e19d8e97a8970 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 19 Jan 2016 00:00:31 +0500
Subject: [PATCH 0717/2974] Remove --lsprof command-line option. Fixes GH-1531

---
 scrapy/cmdline.py             |   9 +--
 scrapy/commands/__init__.py   |   2 -
 scrapy/xlib/lsprofcalltree.py | 120 ----------------------------------
 3 files changed, 1 insertion(+), 130 deletions(-)
 delete mode 100644 scrapy/xlib/lsprofcalltree.py

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 35050c13d..cb7bbd64d 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -7,7 +7,6 @@ import pkg_resources
 
 import scrapy
 from scrapy.crawler import CrawlerProcess
-from scrapy.xlib import lsprofcalltree
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.misc import walk_modules
@@ -144,7 +143,7 @@ def execute(argv=None, settings=None):
     sys.exit(cmd.exitcode)
 
 def _run_command(cmd, args, opts):
-    if opts.profile or opts.lsprof:
+    if opts.profile:
         _run_command_profiled(cmd, args, opts)
     else:
         cmd.run(args, opts)
@@ -152,17 +151,11 @@ def _run_command(cmd, args, opts):
 def _run_command_profiled(cmd, args, opts):
     if opts.profile:
         sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile)
-    if opts.lsprof:
-        sys.stderr.write("scrapy: writing lsprof stats to %r\n" % opts.lsprof)
     loc = locals()
     p = cProfile.Profile()
     p.runctx('cmd.run(args, opts)', globals(), loc)
     if opts.profile:
         p.dump_stats(opts.profile)
-    k = lsprofcalltree.KCacheGrind(p)
-    if opts.lsprof:
-        with open(opts.lsprof, 'w') as f:
-            k.output(f)
 
 if __name__ == '__main__':
     execute()
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 9ac013098..43b420821 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -65,8 +65,6 @@ class ScrapyCommand(object):
             help="disable logging completely")
         group.add_option("--profile", metavar="FILE", default=None,
             help="write python cProfile stats to FILE")
-        group.add_option("--lsprof", metavar="FILE", default=None,
-            help="write lsprof profiling stats to FILE")
         group.add_option("--pidfile", metavar="FILE",
             help="write process ID to FILE")
         group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
diff --git a/scrapy/xlib/lsprofcalltree.py b/scrapy/xlib/lsprofcalltree.py
deleted file mode 100644
index a604016cc..000000000
--- a/scrapy/xlib/lsprofcalltree.py
+++ /dev/null
@@ -1,120 +0,0 @@
-# lsprofcalltree.py: lsprof output which is readable by kcachegrind
-# David Allouche
-# Jp Calderone & Itamar Shtull-Trauring
-# Johan Dahlin
-
-from __future__ import print_function
-import optparse
-import os
-import sys
-
-try:
-    import cProfile
-except ImportError:
-    raise SystemExit("This script requires cProfile from Python 2.5")
-
-def label(code):
-    if isinstance(code, str):
-        return ('~', 0, code)    # built-in functions ('~' sorts at the end)
-    else:
-        return '%s %s:%d' % (code.co_name,
-                             code.co_filename,
-                             code.co_firstlineno)
-
-class KCacheGrind(object):
-    def __init__(self, profiler):
-        self.data = profiler.getstats()
-        self.out_file = None
-
-    def output(self, out_file):
-        self.out_file = out_file
-        print('events: Ticks', file=out_file)
-        self._print_summary()
-        for entry in self.data:
-            self._entry(entry)
-
-    def _print_summary(self):
-        max_cost = 0
-        for entry in self.data:
-            totaltime = int(entry.totaltime * 1000)
-            max_cost = max(max_cost, totaltime)
-        print('summary: %d' % (max_cost,), file=self.out_file)
-
-    def _entry(self, entry):
-        out_file = self.out_file
-
-        code = entry.code
-        #print >> out_file, 'ob=%s' % (code.co_filename,)
-        if isinstance(code, str):
-            print('fi=~', file=out_file)
-        else:
-            print('fi=%s' % (code.co_filename,), file=out_file)
-        print('fn=%s' % (label(code),), file=out_file)
-
-        inlinetime = int(entry.inlinetime * 1000)
-        if isinstance(code, str):
-            print('0 ', inlinetime, file=out_file)
-        else:
-            print('%d %d' % (code.co_firstlineno, inlinetime), file=out_file)
-
-        # recursive calls are counted in entry.calls
-        if entry.calls:
-            calls = entry.calls
-        else:
-            calls = []
-
-        if isinstance(code, str):
-            lineno = 0
-        else:
-            lineno = code.co_firstlineno
-
-        for subentry in calls:
-            self._subentry(lineno, subentry)
-        print(file=out_file)
-
-    def _subentry(self, lineno, subentry):
-        out_file = self.out_file
-        code = subentry.code
-        #print >> out_file, 'cob=%s' % (code.co_filename,)
-        print('cfn=%s' % (label(code),), file=out_file)
-        if isinstance(code, str):
-            print('cfi=~', file=out_file)
-            print('calls=%d 0' % (subentry.callcount,), file=out_file)
-        else:
-            print('cfi=%s' % (code.co_filename,), file=out_file)
-            print('calls=%d %d' % (
-                subentry.callcount, code.co_firstlineno), file=out_file)
-
-        totaltime = int(subentry.totaltime * 1000)
-        print('%d %d' % (lineno, totaltime), file=out_file)
-
-def main(args):
-    usage = "%s [-o output_file_path] scriptfile [arg] ..."
-    parser = optparse.OptionParser(usage=usage % sys.argv[0])
-    parser.allow_interspersed_args = False
-    parser.add_option('-o', '--outfile', dest="outfile",
-                      help="Save stats to <outfile>", default=None)
-
-    if not sys.argv[1:]:
-        parser.print_usage()
-        sys.exit(2)
-
-    options, args = parser.parse_args()
-
-    if not options.outfile:
-        options.outfile = '%s.log' % os.path.basename(args[0])
-
-    sys.argv[:] = args
-
-    prof = cProfile.Profile()
-    try:
-        try:
-            prof = prof.run('execfile(%r)' % (sys.argv[0],))
-        except SystemExit:
-            pass
-    finally:
-        kg = KCacheGrind(prof)
-        kg.output(file(options.outfile, 'w'))
-
-if __name__ == '__main__':
-    sys.exit(main(sys.argv))

From 5ac54ed3391b027cccb140731f82f1f026ea2f7c Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 19 Jan 2016 17:25:50 +0300
Subject: [PATCH 0718/2974] raise minimal twisted version for py3

---
 requirements-py3.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-py3.txt b/requirements-py3.txt
index a9a2e3be0..065095101 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -1,4 +1,4 @@
-Twisted >= 15.1.0
+Twisted >= 15.5.0
 lxml>=3.2.4
 pyOpenSSL>=0.13.1
 cssselect>=0.9

From bb50c0be2fd7e737f2e2bd772f333e68cd02db06 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 19 Jan 2016 17:30:59 +0300
Subject: [PATCH 0719/2974] remove unused import

---
 scrapy/http/response/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 09c4e725a..983154001 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -4,7 +4,6 @@ responses in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
-import six
 from six.moves.urllib.parse import urljoin
 
 from scrapy.http.headers import Headers

From 9c3117a914c27b3acbb2b0300d8938d6e3b49b9e Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 19 Jan 2016 17:32:53 +0300
Subject: [PATCH 0720/2974] more pythonic check of noconnect in proxy params

---
 scrapy/core/downloader/handlers/http11.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 82cf507f7..bda72f5e6 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -186,7 +186,7 @@ class ScrapyAgent(object):
             _, _, proxyHost, proxyPort, proxyParams = _parse(proxy)
             scheme = _parse(request.url)[0]
             proxyHost = to_unicode(proxyHost)
-            omitConnectTunnel = proxyParams.find(b'noconnect') >= 0
+            omitConnectTunnel = b'noconnect' in proxyParams
             if  scheme == b'https' and not omitConnectTunnel:
                 proxyConf = (proxyHost, proxyPort,
                              request.headers.get(b'Proxy-Authorization', None))

From 4c8417284062af7fe1c0107a0fcfdd377424dc50 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 19 Jan 2016 18:24:58 +0300
Subject: [PATCH 0721/2974] move leveldb to tests/requirements-py3.txt

---
 tests/requirements-py3.txt | 1 +
 tox.ini                    | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index a709a734e..5cf786a89 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -3,6 +3,7 @@ pytest-twisted
 pytest-cov
 testfixtures
 jmespath
+leveldb
 # optional for shell wrapper tests
 bpython
 ipython
diff --git a/tox.ini b/tox.ini
index 874a22ee2..eae7e8e47 100644
--- a/tox.ini
+++ b/tox.ini
@@ -42,7 +42,6 @@ deps =
     -rrequirements-py3.txt
     # Extras
     Pillow
-    leveldb
     -rtests/requirements-py3.txt
 
 [testenv:py34]

From 1f2233837a4219d02be73dc0836dfc885d47fffb Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 19 Jan 2016 16:58:24 +0100
Subject: [PATCH 0722/2974] Use if Py2/Py3 function instead of custom GzipFile
 class method

---
 scrapy/utils/gz.py | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index df1d29698..3e6596b0b 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -7,24 +7,35 @@ except ImportError:
 from io import UnsupportedOperation
 from gzip import GzipFile
 
-class ReadOneGzipFile(GzipFile):
-    def readone(self, size=-1):
-        try:
-            return self.read1(size)
-        except UnsupportedOperation:
-            return self.read(size)
+import six
+
+
+# - Python>=3.5 GzipFile's read() has issues returning leftover
+#   uncompressed data when input is corrupted
+#   (regression or bug-fix compared to Python 3.4)
+# - read1(), which fetches data before raising EOFError on next call
+#   works here but is only available from Python>=3.3
+# - scrapy does not support Python 3.2
+# - Python 2.7 GzipFile works fine with standard read() + extrabuf
+if six.PY3:
+    def read1(gzf, size=-1):
+        return gzf.read1(size)
+else:
+    def read1(gzf, size=-1):
+        return gzf.read(size)
+
 
 def gunzip(data):
     """Gunzip the given data and return as much data as possible.
 
     This is resilient to CRC checksum errors.
     """
-    f = ReadOneGzipFile(fileobj=BytesIO(data))
+    f = GzipFile(fileobj=BytesIO(data))
     output = b''
     chunk = b'.'
     while chunk:
         try:
-            chunk = f.readone(8196)
+            chunk = read1(f, 8196)
             output += chunk
         except (IOError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise

From 2b5245839ceaaa256c7edccee351ea79129fa3e4 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 19 Jan 2016 17:04:57 +0100
Subject: [PATCH 0723/2974] Remove unused import statement

---
 scrapy/utils/gz.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 3e6596b0b..d69fb598d 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -4,7 +4,6 @@ try:
     from cStringIO import StringIO as BytesIO
 except ImportError:
     from io import BytesIO
-from io import UnsupportedOperation
 from gzip import GzipFile
 
 import six

From e15f361b0580eb85a13f79b2c96417d17b9a0de6 Mon Sep 17 00:00:00 2001
From: carlosp420 <mycalesis@gmail.com>
Date: Tue, 19 Jan 2016 11:12:43 -0500
Subject: [PATCH 0724/2974] fixed typo You -> you

---
 docs/topics/broad-crawls.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index 79f0b3b53..28ed7c064 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -50,7 +50,7 @@ many different domains in parallel, so you will want to increase it. How much
 to increase it will depend on how much CPU you crawler will have available. A
 good starting point is ``100``, but the best way to find out is by doing some
 trials and identifying at what concurrency your Scrapy process gets CPU
-bounded. For optimum performance, You should pick a concurrency where CPU usage
+bounded. For optimum performance, you should pick a concurrency where CPU usage
 is at 80-90%.
 
 To increase the global concurrency use::

From 176610f91068cb9663be184094e450de0aac77bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Tue, 19 Jan 2016 15:34:26 -0300
Subject: [PATCH 0725/2974] optional_features has been removed

---
 tests/test_downloader_handlers.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index a474b75d2..1eb6192ce 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -431,8 +431,12 @@ class HttpDownloadHandlerMock(object):
     def download_request(self, request, spider):
         return request
 
+
 class S3AnonTestCase(unittest.TestCase):
-    skip = 'boto' not in optional_features and 'missing boto library'
+    try:
+        import boto
+    except ImportError:
+        skip = 'missing boto library'
 
     def setUp(self):
         self.s3reqh = S3DownloadHandler(Settings(),
@@ -448,14 +452,13 @@ class S3AnonTestCase(unittest.TestCase):
         self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True)
         self.assertEqual(self.s3reqh.conn.anon, True)
 
+
 class S3TestCase(unittest.TestCase):
     download_handler_cls = S3DownloadHandler
     try:
-        # can't instance without settings, but ignore that
-        download_handler_cls({})
-    except NotConfigured:
+        import boto
+    except ImportError:
         skip = 'missing boto library'
-    except KeyError: pass
 
     # test use same example keys than amazon developer guide
     # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf

From f0cf5463c85bd38243ac1539487d7118003a790e Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Wed, 20 Jan 2016 12:23:48 +0300
Subject: [PATCH 0726/2974] cleanup http11 tunneling connection after #1678 -
 construct string and then convert to bytes

---
 scrapy/core/downloader/handlers/http11.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index a4d5a28c8..ad3285a32 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -92,11 +92,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
 
     def requestTunnel(self, protocol):
         """Asks the proxy to open a tunnel."""
-        tunnelReq = (
-            b'CONNECT ' +
-            to_bytes(self._tunneledHost, encoding='ascii') + b':' +
-            to_bytes(str(self._tunneledPort)) +
-            b' HTTP/1.1\r\n')
+        tunnelReq = to_bytes(
+            'CONNECT %s:%s HTTP/1.1\r\n' % (
+                self._tunneledHost, self._tunneledPort), encoding='ascii')
         if self._proxyAuthHeader:
             tunnelReq += \
                 b'Proxy-Authorization: ' + self._proxyAuthHeader + b'\r\n'

From 29ff84a7920dd27a7b723f21b68ccc6e2076c08a Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 20 Jan 2016 12:03:38 +0100
Subject: [PATCH 0727/2974] Invert PY2/PY3 test for conditional read1()
 definition

---
 scrapy/utils/gz.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index d69fb598d..d035f9fdf 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -16,12 +16,12 @@ import six
 #   works here but is only available from Python>=3.3
 # - scrapy does not support Python 3.2
 # - Python 2.7 GzipFile works fine with standard read() + extrabuf
-if six.PY3:
-    def read1(gzf, size=-1):
-        return gzf.read1(size)
-else:
+if six.PY2:
     def read1(gzf, size=-1):
         return gzf.read(size)
+else:
+    def read1(gzf, size=-1):
+        return gzf.read1(size)
 
 
 def gunzip(data):

From a32b59ac7507b9b909edfde4cfa9a9effae27641 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 18:54:32 +0300
Subject: [PATCH 0728/2974] py3: fix EngineTest.test_crawler

---
 tests/test_engine.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index dad921a60..df68e5281 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -55,11 +55,12 @@ class TestSpider(Spider):
 
     def parse_item(self, response):
         item = self.item_cls()
-        m = self.name_re.search(response.body)
+        body = response.body_as_unicode()
+        m = self.name_re.search(body)
         if m:
             item['name'] = m.group(1)
         item['url'] = response.url
-        m = self.price_re.search(response.body)
+        m = self.price_re.search(body)
         if m:
             item['price'] = m.group(1)
         return item
@@ -77,8 +78,8 @@ class DictItemsSpider(TestSpider):
 def start_test_site(debug=False):
     root_dir = os.path.join(tests_datadir, "test_site")
     r = static.File(root_dir)
-    r.putChild("redirect", util.Redirect("/redirected"))
-    r.putChild("redirected", static.Data("Redirected here", "text/plain"))
+    r.putChild(b"redirect", util.Redirect(b"/redirected"))
+    r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
 
     port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1")
     if debug:

From 0b08b4bfcfb629aefd4f83cf85e64c74bc1077c3 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 19 Jan 2016 10:07:00 +0300
Subject: [PATCH 0729/2974] fix engine tests - this callback
 (spider_closed_callback) should accept one argument, but the error was hidden
 on py2

---
 tests/test_engine.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index df68e5281..9f2c02bff 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -238,12 +238,12 @@ class EngineTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_close_downloader(self):
-        e = ExecutionEngine(get_crawler(TestSpider), lambda: None)
+        e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
         yield e.close()
 
     @defer.inlineCallbacks
     def test_close_spiders_downloader(self):
-        e = ExecutionEngine(get_crawler(TestSpider), lambda: None)
+        e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
         yield e.open_spider(TestSpider(), [])
         self.assertEqual(len(e.open_spiders), 1)
         yield e.close()
@@ -251,7 +251,7 @@ class EngineTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_close_engine_spiders_downloader(self):
-        e = ExecutionEngine(get_crawler(TestSpider), lambda: None)
+        e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
         yield e.open_spider(TestSpider(), [])
         e.start()
         self.assertTrue(e.running)

From c1db60188aaf05d3a7f055d9b736669e679aed52 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 19 Jan 2016 10:07:12 +0300
Subject: [PATCH 0730/2974] py3: unskip passing test_engine

---
 tests/py3-ignores.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 1f7d85ef7..3e147e9e9 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -6,7 +6,6 @@ tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
 tests/test_downloadermiddleware_httpproxy.py
 tests/test_downloadermiddleware_retry.py
-tests/test_engine.py
 tests/test_mail.py
 tests/test_pipeline_files.py
 tests/test_pipeline_images.py

From 47d3c63338c4550341cc7518bcb48d0fc606b67d Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 19 Jan 2016 16:28:22 +0300
Subject: [PATCH 0731/2974] py3: port fetch and shell commands, and review +
 enable already passing test_closespider.py and tests/test_utils_template.py

---
 scrapy/commands/fetch.py    |  6 ++++--
 scrapy/utils/testsite.py    | 12 ++++++------
 tests/py3-ignores.txt       |  4 ----
 tests/test_command_fetch.py |  8 ++++----
 tests/test_command_shell.py | 14 +++++++-------
 5 files changed, 21 insertions(+), 23 deletions(-)

diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index e61eedf50..3888da210 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -5,6 +5,7 @@ from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
 from scrapy.exceptions import UsageError
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
+from scrapy.utils.python import to_unicode
 
 class Command(ScrapyCommand):
 
@@ -30,7 +31,8 @@ class Command(ScrapyCommand):
     def _print_headers(self, headers, prefix):
         for key, values in headers.items():
             for value in values:
-                print('%s %s: %s' % (prefix, key, value))
+                print('%s %s: %s' % (
+                      prefix, to_unicode(key), to_unicode(value)))
 
     def _print_response(self, response, opts):
         if opts.headers:
@@ -38,7 +40,7 @@ class Command(ScrapyCommand):
             print('>')
             self._print_headers(response.headers, '<')
         else:
-            print(response.body)
+            print(to_unicode(response.body))
 
     def run(self, args, opts):
         if len(args) != 1 or not is_url(args[0]):
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index 01508bdb4..ad0375443 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -22,13 +22,13 @@ class SiteTest(object):
 
 def test_site():
     r = resource.Resource()
-    r.putChild("text", static.Data("Works", "text/plain"))
-    r.putChild("html", static.Data("<body><p class='one'>Works</p><p class='two'>World</p></body>", "text/html"))
-    r.putChild("enc-gb18030", static.Data("<p>gb18030 encoding</p>", "text/html; charset=gb18030"))
-    r.putChild("redirect", util.Redirect("/redirected"))
-    r.putChild("redirected", static.Data("Redirected here", "text/plain"))
+    r.putChild(b"text", static.Data(b"Works", "text/plain"))
+    r.putChild(b"html", static.Data(b"<body><p class='one'>Works</p><p class='two'>World</p></body>", "text/html"))
+    r.putChild(b"enc-gb18030", static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"))
+    r.putChild(b"redirect", util.Redirect(b"/redirected"))
+    r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
     return server.Site(r)
-    
+
 
 if __name__ == '__main__':
     port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 1f7d85ef7..1f0f34c49 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,6 +1,3 @@
-tests/test_closespider.py
-tests/test_command_fetch.py
-tests/test_command_shell.py
 tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
@@ -12,7 +9,6 @@ tests/test_pipeline_files.py
 tests/test_pipeline_images.py
 tests/test_proxy_connect.py
 tests/test_spidermiddleware_httperror.py
-tests/test_utils_template.py
 
 scrapy/xlib/tx/iweb.py
 scrapy/xlib/tx/interfaces.py
diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py
index 5283852b7..4843a9a2f 100644
--- a/tests/test_command_fetch.py
+++ b/tests/test_command_fetch.py
@@ -12,11 +12,11 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
     @defer.inlineCallbacks
     def test_output(self):
         _, out, _ = yield self.execute([self.url('/text')])
-        self.assertEqual(out.strip(), 'Works')
+        self.assertEqual(out.strip(), b'Works')
 
     @defer.inlineCallbacks
     def test_headers(self):
         _, out, _ = yield self.execute([self.url('/text'), '--headers'])
-        out = out.replace('\r', '') # required on win32
-        assert 'Server: TwistedWeb' in out
-        assert 'Content-Type: text/plain' in out
+        out = out.replace(b'\r', b'') # required on win32
+        assert b'Server: TwistedWeb' in out, out
+        assert b'Content-Type: text/plain' in out
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index a56236d54..105202754 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -12,38 +12,38 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
     @defer.inlineCallbacks
     def test_empty(self):
         _, out, _ = yield self.execute(['-c', 'item'])
-        assert '{}' in out
+        assert b'{}' in out
 
     @defer.inlineCallbacks
     def test_response_body(self):
         _, out, _ = yield self.execute([self.url('/text'), '-c', 'response.body'])
-        assert 'Works' in out
+        assert b'Works' in out
 
     @defer.inlineCallbacks
     def test_response_type_text(self):
         _, out, _ = yield self.execute([self.url('/text'), '-c', 'type(response)'])
-        assert 'TextResponse' in out
+        assert b'TextResponse' in out
 
     @defer.inlineCallbacks
     def test_response_type_html(self):
         _, out, _ = yield self.execute([self.url('/html'), '-c', 'type(response)'])
-        assert 'HtmlResponse' in out
+        assert b'HtmlResponse' in out
 
     @defer.inlineCallbacks
     def test_response_selector_html(self):
         xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]'
         _, out, _ = yield self.execute([self.url('/html'), '-c', xpath])
-        self.assertEqual(out.strip(), 'Works')
+        self.assertEqual(out.strip(), b'Works')
 
     @defer.inlineCallbacks
     def test_response_encoding_gb18030(self):
         _, out, _ = yield self.execute([self.url('/enc-gb18030'), '-c', 'response.encoding'])
-        self.assertEqual(out.strip(), 'gb18030')
+        self.assertEqual(out.strip(), b'gb18030')
 
     @defer.inlineCallbacks
     def test_redirect(self):
         _, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url'])
-        assert out.strip().endswith('/redirected')
+        assert out.strip().endswith(b'/redirected')
 
     @defer.inlineCallbacks
     def test_request_replace(self):

From c6d013ec85747fe7a0ec57852667bc4e29ec3e9c Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 19 Jan 2016 16:42:34 +0300
Subject: [PATCH 0732/2974] py3 fetch command: it may actually be better to try
 to print bytes as-is

---
 scrapy/commands/fetch.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 3888da210..49fa18ab2 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,11 +1,11 @@
 from __future__ import print_function
+import sys, six
 from w3lib.url import is_url
 
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
 from scrapy.exceptions import UsageError
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
-from scrapy.utils.python import to_unicode
 
 class Command(ScrapyCommand):
 
@@ -29,10 +29,10 @@ class Command(ScrapyCommand):
             help="print response HTTP headers instead of body")
 
     def _print_headers(self, headers, prefix):
+        prefix = prefix.encode()
         for key, values in headers.items():
             for value in values:
-                print('%s %s: %s' % (
-                      prefix, to_unicode(key), to_unicode(value)))
+                self._print_bytes(prefix + b' ' + key + b': ' + value)
 
     def _print_response(self, response, opts):
         if opts.headers:
@@ -40,7 +40,11 @@ class Command(ScrapyCommand):
             print('>')
             self._print_headers(response.headers, '<')
         else:
-            print(to_unicode(response.body))
+            self._print_bytes(response.body)
+
+    def _print_bytes(self, bytes_):
+        bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer
+        bytes_writer.write(bytes_ + b'\n')
 
     def run(self, args, opts):
         if len(args) != 1 or not is_url(args[0]):

From a5da7531c42bb26af49787aa901c3d19b80abf2f Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 19 Jan 2016 18:06:20 +0300
Subject: [PATCH 0733/2974] py3 backout skipping test_closespider - it was
 fixed on another branch

---
 tests/py3-ignores.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 1f0f34c49..f8c631827 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,3 +1,4 @@
+tests/test_closespider.py
 tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_crawl.py

From fd24e22442d900170e7a5888b27698c9414b2983 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Wed, 20 Jan 2016 23:30:58 +0300
Subject: [PATCH 0734/2974] use byte constants for prefix instead of encoding
 it

---
 scrapy/commands/fetch.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 49fa18ab2..f09a873c1 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -29,16 +29,15 @@ class Command(ScrapyCommand):
             help="print response HTTP headers instead of body")
 
     def _print_headers(self, headers, prefix):
-        prefix = prefix.encode()
         for key, values in headers.items():
             for value in values:
                 self._print_bytes(prefix + b' ' + key + b': ' + value)
 
     def _print_response(self, response, opts):
         if opts.headers:
-            self._print_headers(response.request.headers, '>')
+            self._print_headers(response.request.headers, b'>')
             print('>')
-            self._print_headers(response.headers, '<')
+            self._print_headers(response.headers, b'<')
         else:
             self._print_bytes(response.body)
 

From 659715ecd92c3f39ed0b52509adefb73c49fa56c Mon Sep 17 00:00:00 2001
From: Capi Etheriel <barraponto@gmail.com>
Date: Fri, 24 Jul 2015 12:07:59 -0300
Subject: [PATCH 0735/2974] implements FormRequest.from_response CSS support

---
 docs/topics/request-response.rst |  8 +++++++-
 scrapy/http/request/form.py      | 10 ++++++++--
 tests/test_http_request.py       | 20 ++++++++++++++++++++
 3 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 0abec1f96..8f519b459 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -282,7 +282,7 @@ fields with form data from :class:`Response` objects.
     The :class:`FormRequest` objects support the following class method in
     addition to the standard :class:`Request` methods:
 
-    .. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, clickdata=None, dont_click=False, ...])
+    .. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...])
 
        Returns a new :class:`FormRequest` object with its form field values
        pre-populated with those found in the HTML ``<form>`` element contained
@@ -310,6 +310,9 @@ fields with form data from :class:`Response` objects.
        :param formxpath: if given, the first form that matches the xpath will be used.
        :type formxpath: string
 
+       :param formcss: if given, the first form that matches the css selector will be used.
+       :type formcss: string
+
        :param formnumber: the number of form to use, when the response contains
           multiple forms. The first one (and also the default) is ``0``.
        :type formnumber: integer
@@ -339,6 +342,9 @@ fields with form data from :class:`Response` objects.
        .. versionadded:: 0.17
           The ``formxpath`` parameter.
 
+       .. versionadded:: 1.1.5
+          The ``formcss`` parameter.
+
 Request usage examples
 ----------------------
 
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index f623a5aa3..5501634d3 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -34,8 +34,14 @@ class FormRequest(Request):
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,
-                      clickdata=None, dont_click=False, formxpath=None, **kwargs):
+                      clickdata=None, dont_click=False, formxpath=None, formcss=None, **kwargs):
+
         kwargs.setdefault('encoding', response.encoding)
+
+        if formcss is not None:
+            from parsel.csstranslator import HTMLTranslator
+            formxpath = HTMLTranslator().css_to_xpath(formcss)
+
         form = _get_form(response, formname, formid, formnumber, formxpath)
         formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
         url = _get_form_url(form, kwargs.pop('url', None))
@@ -73,7 +79,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
         f = root.xpath('//form[@id="%s"]' % formid)
         if f:
             return f[0]
-            
+
     # Get form element from xpath, if not found, go up
     if formxpath is not None:
         nodes = root.xpath(formxpath)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index f82b2de8d..b81d43c41 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -846,6 +846,26 @@ class FormRequestTest(RequestTest):
         req = self.request_class.from_response(response)
         self.assertEqual(req.url, 'http://b.com/test_form')
 
+    def test_from_response_css(self):
+        response = _buildresponse(
+            """<form action="post.php" method="POST">
+            <input type="hidden" name="one" value="1">
+            <input type="hidden" name="two" value="2">
+            </form>
+            <form action="post2.php" method="POST">
+            <input type="hidden" name="three" value="3">
+            <input type="hidden" name="four" value="4">
+            </form>""")
+        r1 = self.request_class.from_response(response, formcss="form[action='post.php']")
+        fs = _qs(r1)
+        self.assertEqual(fs[b'one'], [b'1'])
+
+        r1 = self.request_class.from_response(response, formcss="input[name='four']")
+        fs = _qs(r1)
+        self.assertEqual(fs[b'three'], [b'3'])
+
+        self.assertRaises(ValueError, self.request_class.from_response,
+                          response, formcss="input[name='abc']")
 
 def _buildresponse(body, **kwargs):
     kwargs.setdefault('body', body)

From d4c4ca80624c2540d5dedd218695d38542d92a01 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 21 Jan 2016 09:42:15 -0200
Subject: [PATCH 0736/2974] fix version number to appear new feature

---
 docs/topics/request-response.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 8f519b459..ea64d1599 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -342,7 +342,7 @@ fields with form data from :class:`Response` objects.
        .. versionadded:: 0.17
           The ``formxpath`` parameter.
 
-       .. versionadded:: 1.1.5
+       .. versionadded:: 1.1.0
           The ``formcss`` parameter.
 
 Request usage examples

From 80c55f19a143d8938ced81a599100259509567a1 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 21 Jan 2016 18:31:58 +0500
Subject: [PATCH 0737/2974] PY3 fixed scrapy bench command

---
 scrapy/utils/benchserver.py | 8 ++++----
 tests/test_commands.py      | 5 +++--
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index 4385d72a9..a9a2c938e 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -16,15 +16,15 @@ class Root(Resource):
         total = _getarg(request, 'total', 100, int)
         show = _getarg(request, 'show', 10, int)
         nlist = [random.randint(1, total) for _ in range(show)]
-        request.write("<html><head></head><body>")
+        request.write(b"<html><head></head><body>")
         args = request.args.copy()
         for nl in nlist:
             args['n'] = nl
             argstr = urlencode(args, doseq=True)
             request.write("<a href='/follow?{0}'>follow {1}</a><br>"
-                          .format(argstr, nl))
-        request.write("</body></html>")
-        return ''
+                          .format(argstr, nl).encode('utf8'))
+        request.write(b"</body></html>")
+        return b''
 
 
 def _getarg(request, name, default=None, type=str):
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 5755b3881..057112d12 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -72,7 +72,7 @@ class StartprojectTest(ProjectTest):
         self.assertEqual(1, self.call('startproject', self.project_name))
         self.assertEqual(1, self.call('startproject', 'wrong---project---name'))
         self.assertEqual(1, self.call('startproject', 'sys'))
-    
+
 
 class StartprojectTemplatesTest(ProjectTest):
 
@@ -80,7 +80,7 @@ class StartprojectTemplatesTest(ProjectTest):
         super(StartprojectTemplatesTest, self).setUp()
         self.tmpl = join(self.temp_path, 'templates')
         self.tmpl_proj = join(self.tmpl, 'project')
-        
+
     def test_startproject_template_override(self):
         copytree(join(scrapy.__path__[0], 'templates'), self.tmpl)
         os.mknod(join(self.tmpl_proj, 'root_template'))
@@ -276,3 +276,4 @@ class BenchCommandTest(CommandTest):
                 '-s', 'CLOSESPIDER_TIMEOUT=0.01')
         log = to_native_str(p.stderr.read())
         self.assertIn('INFO: Crawled', log)
+        self.assertNotIn('Unhandled Error', log)

From a18dc24471f121ed85d1bee4281d43c3a3728162 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 21 Jan 2016 18:44:37 +0500
Subject: [PATCH 0738/2974] correctly process arguments for bench server

---
 scrapy/utils/benchserver.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index a9a2c938e..5bbda6e27 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -13,8 +13,8 @@ class Root(Resource):
         return self
 
     def render(self, request):
-        total = _getarg(request, 'total', 100, int)
-        show = _getarg(request, 'show', 10, int)
+        total = _getarg(request, b'total', 100, int)
+        show = _getarg(request, b'show', 10, int)
         nlist = [random.randint(1, total) for _ in range(show)]
         request.write(b"<html><head></head><body>")
         args = request.args.copy()

From f042ad0f39594d59a1a2032e6294ff1890638138 Mon Sep 17 00:00:00 2001
From: Raul Gallegos <raul.ogh@gmail.com>
Date: Thu, 10 Dec 2015 23:04:25 -0500
Subject: [PATCH 0739/2974] py3 fix HttpProxy and Retry Middlewares

---
 scrapy/downloadermiddlewares/httpproxy.py    | 6 +++---
 tests/py3-ignores.txt                        | 4 ----
 tests/test_downloadermiddleware_httpproxy.py | 4 ++--
 tests/test_downloadermiddleware_retry.py     | 8 ++++----
 4 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index dda6a3d2a..8c3514fd0 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -9,7 +9,7 @@ from six.moves.urllib.parse import urlunparse
 
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.exceptions import NotConfigured
-
+from scrapy.utils.python import to_bytes
 
 class HttpProxyMiddleware(object):
 
@@ -26,7 +26,7 @@ class HttpProxyMiddleware(object):
         proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
 
         if user:
-            user_pass = '%s:%s' % (unquote(user), unquote(password))
+            user_pass = to_bytes('%s:%s' % (unquote(user), unquote(password)))
             creds = base64.b64encode(user_pass).strip()
         else:
             creds = None
@@ -52,4 +52,4 @@ class HttpProxyMiddleware(object):
         creds, proxy = self.proxies[scheme]
         request.meta['proxy'] = proxy
         if creds:
-            request.headers['Proxy-Authorization'] = 'Basic ' + creds
+            request.headers['Proxy-Authorization'] = b'Basic ' + creds
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 8c883ee3c..185a278fb 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -2,8 +2,6 @@ tests/test_closespider.py
 tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
-tests/test_downloadermiddleware_httpproxy.py
-tests/test_downloadermiddleware_retry.py
 tests/test_mail.py
 tests/test_pipeline_files.py
 tests/test_pipeline_images.py
@@ -23,8 +21,6 @@ scrapy/pipelines/files.py
 scrapy/linkextractors/sgml.py
 scrapy/linkextractors/regex.py
 scrapy/linkextractors/htmlparser.py
-scrapy/downloadermiddlewares/retry.py
-scrapy/downloadermiddlewares/httpproxy.py
 scrapy/downloadermiddlewares/cookies.py
 scrapy/extensions/statsmailer.py
 scrapy/extensions/memusage.py
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 5b9717a89..7676b2a00 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -52,7 +52,7 @@ class TestDefaultHeadersMiddleware(TestCase):
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
         self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
-        self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjpwYXNz')
+        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz')
 
     def test_proxy_auth_empty_passwd(self):
         os.environ['http_proxy'] = 'https://user:@proxy:3128'
@@ -60,7 +60,7 @@ class TestDefaultHeadersMiddleware(TestCase):
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
         self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
-        self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjo=')
+        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=')
 
     def test_proxy_already_seted(self):
         os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 20561e771..3de9399cf 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -21,20 +21,20 @@ class RetryTest(unittest.TestCase):
 
     def test_priority_adjust(self):
         req = Request('http://www.scrapytest.org/503')
-        rsp = Response('http://www.scrapytest.org/503', body='', status=503)
+        rsp = Response('http://www.scrapytest.org/503', body=b'', status=503)
         req2 = self.mw.process_response(req, rsp, self.spider)
         assert req2.priority < req.priority
 
     def test_404(self):
         req = Request('http://www.scrapytest.org/404')
-        rsp = Response('http://www.scrapytest.org/404', body='', status=404)
+        rsp = Response('http://www.scrapytest.org/404', body=b'', status=404)
 
         # dont retry 404s
         assert self.mw.process_response(req, rsp, self.spider) is rsp
 
     def test_dont_retry(self):
         req = Request('http://www.scrapytest.org/503', meta={'dont_retry': True})
-        rsp = Response('http://www.scrapytest.org/503', body='', status=503)
+        rsp = Response('http://www.scrapytest.org/503', body=b'', status=503)
 
         # first retry
         r = self.mw.process_response(req, rsp, self.spider)
@@ -56,7 +56,7 @@ class RetryTest(unittest.TestCase):
 
     def test_503(self):
         req = Request('http://www.scrapytest.org/503')
-        rsp = Response('http://www.scrapytest.org/503', body='', status=503)
+        rsp = Response('http://www.scrapytest.org/503', body=b'', status=503)
 
         # first retry
         req = self.mw.process_response(req, rsp, self.spider)

From a06a5f00f4a62028416ada4919994efd7d439eb2 Mon Sep 17 00:00:00 2001
From: Raul Gallegos <raul.ogh@gmail.com>
Date: Wed, 20 Jan 2016 13:52:52 -0500
Subject: [PATCH 0740/2974] adding configurable encoding for httpproxy
 authentication

---
 docs/topics/downloader-middleware.rst        | 12 ++++++++++++
 scrapy/downloadermiddlewares/httpproxy.py    | 13 +++++++++++--
 scrapy/settings/default_settings.py          |  2 ++
 tests/test_downloadermiddleware_httpproxy.py | 19 +++++++++++++++++--
 4 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 3641da231..a97d5a696 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -951,6 +951,18 @@ Default: ``False``
 Whether the AjaxCrawlMiddleware will be enabled. You may want to
 enable it for :ref:`broad crawls <topics-broad-crawls>`.
 
+HttpProxyMiddleware settings
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. setting:: HTTPPROXY_AUTH_ENCODING
+
+HTTPPROXY_AUTH_ENCODING
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Default: ``"latin-1"``
+
+The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
+
 
 .. _DBM: http://en.wikipedia.org/wiki/Dbm
 .. _anydbm: https://docs.python.org/2/library/anydbm.html
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 8c3514fd0..b01bab76d 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -11,9 +11,11 @@ from scrapy.utils.httpobj import urlparse_cached
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.python import to_bytes
 
+
 class HttpProxyMiddleware(object):
 
-    def __init__(self):
+    def __init__(self, auth_encoding='latin-1'):
+        self.auth_encoding = auth_encoding
         self.proxies = {}
         for type, url in getproxies().items():
             self.proxies[type] = self._get_proxy(url, type)
@@ -21,12 +23,19 @@ class HttpProxyMiddleware(object):
         if not self.proxies:
             raise NotConfigured
 
+    @classmethod
+    def from_crawler(cls, crawler):
+        auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')
+        return cls(auth_encoding)
+
     def _get_proxy(self, url, orig_type):
         proxy_type, user, password, hostport = _parse_proxy(url)
         proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
 
         if user:
-            user_pass = to_bytes('%s:%s' % (unquote(user), unquote(password)))
+            user_pass = to_bytes(
+                '%s:%s' % (unquote(user), unquote(password)),
+                encoding=self.auth_encoding)
             creds = base64.b64encode(user_pass).strip()
         else:
             creds = None
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index b151933b6..44e74dc61 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -169,6 +169,8 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
 HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
 HTTPCACHE_GZIP = False
 
+HTTPPROXY_AUTH_ENCODING = 'latin-1'
+
 ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
 
 ITEM_PIPELINES = {}
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 7676b2a00..2b26431a4 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -9,6 +9,7 @@ from scrapy.spiders import Spider
 
 spider = Spider('foo')
 
+
 class TestDefaultHeadersMiddleware(TestCase):
 
     failureException = AssertionError
@@ -62,6 +63,22 @@ class TestDefaultHeadersMiddleware(TestCase):
         self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
         self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=')
 
+    def test_proxy_auth_encoding(self):
+        # utf-8 encoding
+        os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128'
+        mw = HttpProxyMiddleware(auth_encoding='utf-8')
+        req = Request('http://scrapytest.org')
+        assert mw.process_request(req, spider) is None
+        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
+
+        # default latin-1 encoding
+        mw = HttpProxyMiddleware(auth_encoding='latin-1')
+        req = Request('http://scrapytest.org')
+        assert mw.process_request(req, spider) is None
+        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
+
     def test_proxy_already_seted(self):
         os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
         mw = HttpProxyMiddleware()
@@ -69,7 +86,6 @@ class TestDefaultHeadersMiddleware(TestCase):
         assert mw.process_request(req, spider) is None
         assert 'proxy' in req.meta and req.meta['proxy'] is None
 
-
     def test_no_proxy(self):
         os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
         mw = HttpProxyMiddleware()
@@ -88,4 +104,3 @@ class TestDefaultHeadersMiddleware(TestCase):
         req = Request('http://noproxy.com')
         assert mw.process_request(req, spider) is None
         assert 'proxy' not in req.meta
-

From 20b839b44ba5795aa0a6cf96ac71d4524072fcab Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 19 Jan 2016 12:42:45 +0300
Subject: [PATCH 0741/2974] py3: pass first crawl test (test_follow_all): fix
 mock server

---
 tests/mockserver.py | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index e7953c4d4..336633f4b 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -7,6 +7,7 @@ from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
 from twisted.internet import reactor, defer, ssl
 from scrapy import twisted_version
+from scrapy.utils.python import to_bytes
 
 
 if twisted_version < (11, 0, 0):
@@ -55,12 +56,12 @@ class LeafResource(Resource):
 class Follow(LeafResource):
 
     def render(self, request):
-        total = getarg(request, "total", 100, type=int)
-        show = getarg(request, "show", 1, type=int)
-        order = getarg(request, "order", "desc")
-        maxlatency = getarg(request, "maxlatency", 0, type=float)
-        n = getarg(request, "n", total, type=int)
-        if order == "rand":
+        total = getarg(request, b"total", 100, type=int)
+        show = getarg(request, b"show", 1, type=int)
+        order = getarg(request, b"order", b"desc")
+        maxlatency = getarg(request, b"maxlatency", 0, type=float)
+        n = getarg(request, b"n", total, type=int)
+        if order == b"rand":
             nlist = [random.randint(1, total) for _ in range(show)]
         else:  # order == "desc"
             nlist = range(n, max(n - show, 0), -1)
@@ -73,19 +74,19 @@ class Follow(LeafResource):
         s = """<html> <head></head> <body>"""
         args = request.args.copy()
         for nl in nlist:
-            args["n"] = [str(nl)]
+            args[b"n"] = [to_bytes(str(nl))]
             argstr = urlencode(args, doseq=True)
             s += "<a href='/follow?%s'>follow %d</a><br>" % (argstr, nl)
         s += """</body>"""
-        request.write(s)
+        request.write(to_bytes(s))
         request.finish()
 
 
 class Delay(LeafResource):
 
     def render_GET(self, request):
-        n = getarg(request, "n", 1, type=float)
-        b = getarg(request, "b", 1, type=int)
+        n = getarg(request, b"n", 1, type=float)
+        b = getarg(request, b"b", 1, type=int)
         if b:
             # send headers now and delay body
             request.write('')
@@ -93,16 +94,16 @@ class Delay(LeafResource):
         return NOT_DONE_YET
 
     def _delayedRender(self, request, n):
-        request.write("Response delayed for %0.3f seconds\n" % n)
+        request.write(to_bytes("Response delayed for %0.3f seconds\n" % n))
         request.finish()
 
 
 class Status(LeafResource):
 
     def render_GET(self, request):
-        n = getarg(request, "n", 200, type=int)
+        n = getarg(request, b"n", 200, type=int)
         request.setResponseCode(n)
-        return ""
+        return b""
 
 
 class Raw(LeafResource):
@@ -114,7 +115,7 @@ class Raw(LeafResource):
     render_POST = render_GET
 
     def _delayedRender(self, request):
-        raw = getarg(request, 'raw', 'HTTP 1.1 200 OK\n')
+        raw = getarg(request, b'raw', b'HTTP 1.1 200 OK\n')
         request.startedWriting = 1
         request.write(raw)
         request.channel.transport.loseConnection()
@@ -128,7 +129,7 @@ class Echo(LeafResource):
             'headers': dict(request.requestHeaders.getAllRawHeaders()),
             'body': request.content.read(),
         }
-        return json.dumps(output)
+        return to_bytes(json.dumps(output))
 
 
 class Partial(LeafResource):
@@ -146,7 +147,7 @@ class Partial(LeafResource):
 class Drop(Partial):
 
     def _delayedRender(self, request):
-        abort = getarg(request, "abort", 0, type=int)
+        abort = getarg(request, b"abort", 0, type=int)
         request.write(b"this connection will be dropped\n")
         tr = request.channel.transport
         try:

From 0680950b9898aa77e4c494ab8a318d791ef0d55f Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 19 Jan 2016 13:06:31 +0300
Subject: [PATCH 0742/2974] py3: pass CrawlTestCase.test_referer_header, fixing
 Echo resource in mockserver and json decoding in test

---
 tests/mockserver.py | 9 +++++----
 tests/test_crawl.py | 9 +++++----
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 336633f4b..6877c786c 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,13 +1,12 @@
 from __future__ import print_function
 import sys, time, random, os, json
-import six
 from six.moves.urllib.parse import urlencode
 from subprocess import Popen, PIPE
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
 from twisted.internet import reactor, defer, ssl
 from scrapy import twisted_version
-from scrapy.utils.python import to_bytes
+from scrapy.utils.python import to_bytes, to_unicode
 
 
 if twisted_version < (11, 0, 0):
@@ -126,8 +125,10 @@ class Echo(LeafResource):
 
     def render_GET(self, request):
         output = {
-            'headers': dict(request.requestHeaders.getAllRawHeaders()),
-            'body': request.content.read(),
+            'headers': dict(
+                (to_unicode(k), [to_unicode(v) for v in vs])
+                for k, vs in request.requestHeaders.getAllRawHeaders()),
+            'body': to_unicode(request.content.read()),
         }
         return to_bytes(json.dumps(output))
 
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 814eb30d2..90fd921c8 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -8,6 +8,7 @@ from twisted.trial.unittest import TestCase
 
 from scrapy.http import Request
 from scrapy.crawler import CrawlerRunner
+from scrapy.utils.python import to_unicode
 from tests import mock
 from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
     BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
@@ -201,16 +202,16 @@ with multiples lines
         self.assertIn('responses', crawler.spider.meta)
         self.assertNotIn('failures', crawler.spider.meta)
         # start requests doesn't set Referer header
-        echo0 = json.loads(crawler.spider.meta['responses'][2].body)
+        echo0 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body))
         self.assertNotIn('Referer', echo0['headers'])
         # following request sets Referer to start request url
-        echo1 = json.loads(crawler.spider.meta['responses'][1].body)
+        echo1 = json.loads(to_unicode(crawler.spider.meta['responses'][1].body))
         self.assertEqual(echo1['headers'].get('Referer'), [req0.url])
         # next request avoids Referer header
-        echo2 = json.loads(crawler.spider.meta['responses'][2].body)
+        echo2 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body))
         self.assertNotIn('Referer', echo2['headers'])
         # last request explicitly sets a Referer header
-        echo3 = json.loads(crawler.spider.meta['responses'][3].body)
+        echo3 = json.loads(to_unicode(crawler.spider.meta['responses'][3].body))
         self.assertEqual(echo3['headers'].get('Referer'), ['http://example.com'])
 
     @defer.inlineCallbacks

From ad2b3321b90245e59b2e4b99eb7a4296f6c7e768 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 19 Jan 2016 13:13:41 +0300
Subject: [PATCH 0743/2974] py3 compat: use range, fixes
 CrawlTestCase.test_start_requests_bug_yielding

---
 tests/spiders.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index 516062929..711d80cac 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -119,7 +119,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
         if self.fail_before_yield:
             1 / 0
 
-        for s in xrange(100):
+        for s in range(100):
             qargs = {'total': 10, 'seed': s}
             url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1)
             yield Request(url, meta={'seed': s})

From bf5f54fa339b44fec3451c88a78e4620f56c3bc8 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 19 Jan 2016 16:01:30 +0300
Subject: [PATCH 0744/2974] py3: fix getarg

---
 tests/mockserver.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 6877c786c..365ec81fd 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -30,9 +30,12 @@ else:
     from twisted.internet.task import deferLater
 
 
-def getarg(request, name, default=None, type=str):
+def getarg(request, name, default=None, type=None):
     if name in request.args:
-        return type(request.args[name][0])
+        value = request.args[name][0]
+        if type is not None:
+            value = type(value)
+        return value
     else:
         return default
 

From 4607f2843e25b559f7483186916a1d91329c9dd8 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 19 Jan 2016 16:01:43 +0300
Subject: [PATCH 0745/2974] py3: unskip test_crawl

---
 tests/py3-ignores.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 185a278fb..c8beea8a3 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,7 +1,6 @@
 tests/test_closespider.py
 tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
-tests/test_crawl.py
 tests/test_mail.py
 tests/test_pipeline_files.py
 tests/test_pipeline_images.py

From 5813de883888a69cf015ad506d6052e6191feb6e Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 19 Jan 2016 18:08:05 +0300
Subject: [PATCH 0746/2974] py3: unskip test_closespider - it passes after
 fixing mockserver.Follow resouce on py3

---
 tests/py3-ignores.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index c8beea8a3..f189a4c86 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,4 +1,3 @@
-tests/test_closespider.py
 tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_mail.py

From a76ecd4ef0bd7fa2dbe2e02d5b5721b39ead18c0 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 8 Oct 2015 22:18:14 -0300
Subject: [PATCH 0747/2974] remove test_exporters from py3 ignores

---
 tests/py3-ignores.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index f189a4c86..570287d9d 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,4 +1,3 @@
-tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_mail.py
 tests/test_pipeline_files.py

From b6ef1f19fd768243407206a882d689764624b42c Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Fri, 9 Oct 2015 00:19:05 -0300
Subject: [PATCH 0748/2974] make BaseItemExporter export unicode, pushed down
 previous behavior for classes that need it

---
 scrapy/exporters.py     | 15 +++++++------
 tests/test_exporters.py | 48 ++++++++++++++++++++++++++++++++++++++---
 2 files changed, 53 insertions(+), 10 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 7e1d01a0a..6f679480d 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -38,7 +38,7 @@ class BaseItemExporter(object):
         raise NotImplementedError
 
     def serialize_field(self, field, name, value):
-        serializer = field.get('serializer', self._to_str_if_unicode)
+        serializer = field.get('serializer', lambda x: x)
         return serializer(value)
 
     def start_exporting(self):
@@ -47,9 +47,6 @@ class BaseItemExporter(object):
     def finish_exporting(self):
         pass
 
-    def _to_str_if_unicode(self, value):
-        return value.encode(self.encoding) if isinstance(value, unicode) else value
-
     def _get_serialized_fields(self, item, default_value=None, include_empty=None):
         """Return the fields to export as an iterable of tuples
         (name, serialized_value)
@@ -89,7 +86,7 @@ class JsonLinesItemExporter(BaseItemExporter):
         self.file.write(self.encoder.encode(itemdict) + '\n')
 
 
-class JsonItemExporter(JsonLinesItemExporter):
+class JsonItemExporter(BaseItemExporter):
 
     def __init__(self, file, **kwargs):
         self._configure(kwargs, dont_fail=True)
@@ -170,13 +167,17 @@ class CsvItemExporter(BaseItemExporter):
         self._headers_not_written = True
         self._join_multivalued = join_multivalued
 
+    def serialize_field(self, field, name, value):
+        serializer = field.get('serializer', self._to_str_if_unicode)
+        return serializer(value)
+
     def _to_str_if_unicode(self, value):
         if isinstance(value, (list, tuple)):
             try:
                 value = self._join_multivalued.join(value)
             except TypeError:  # list in value may not contain strings
                 pass
-        return super(CsvItemExporter, self)._to_str_if_unicode(value)
+        return value.encode(self.encoding) if isinstance(value, unicode) else value
 
     def export_item(self, item):
         if self._headers_not_written:
@@ -251,7 +252,7 @@ class PythonItemExporter(BaseItemExporter):
             return dict(self._serialize_dict(value))
         if hasattr(value, '__iter__'):
             return [self._serialize_value(v) for v in value]
-        return self._to_str_if_unicode(value)
+        return value.encode(self.encoding) if isinstance(value, unicode) else value
 
     def _serialize_dict(self, value):
         for key, val in six.iteritems(value):
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index b24633959..c84fb978a 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -23,7 +23,7 @@ class TestItem(Item):
 class BaseItemExporterTest(unittest.TestCase):
 
     def setUp(self):
-        self.i = TestItem(name=u'John\xa3', age='22')
+        self.i = TestItem(name=u'John\xa3', age=u'22')
         self.output = BytesIO()
         self.ie = self._get_exporter()
 
@@ -55,6 +55,42 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertItemExportWorks(dict(self.i))
 
     def test_serialize_field(self):
+        res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name'])
+        self.assertEqual(res, u'John\xa3')
+
+        res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age'])
+        self.assertEqual(res, u'22')
+
+    def test_fields_to_export(self):
+        ie = self._get_exporter(fields_to_export=['name'])
+        self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')])
+
+    def test_field_custom_serializer(self):
+        def custom_serializer(value):
+            return str(int(value) + 2)
+
+        class CustomFieldItem(Item):
+            name = Field()
+            age = Field(serializer=custom_serializer)
+
+        i = CustomFieldItem(name=u'John\xa3', age=u'22')
+
+        ie = self._get_exporter()
+        self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3')
+        self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
+
+
+class MidRefactoringBaseItemExporterTest(BaseItemExporterTest):
+    """Class introduced just to keep old behavior of BaseItemExporterTest for the
+    test cases that inherit from it while we make changes to exporters one by
+    one -- a needed refactoring trick because the test cases are quite coupled.
+
+    When we're done with the changes, we'll have ditched this class.
+    """
+    def test_serialize_field(self):
+        if self.ie.__class__ is BaseItemExporter:
+            return
+
         res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name'])
         self.assertEqual(res, 'John\xc2\xa3')
 
@@ -62,6 +98,9 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertEqual(res, '22')
 
     def test_fields_to_export(self):
+        if self.ie.__class__ is BaseItemExporter:
+            return
+
         ie = self._get_exporter(fields_to_export=['name'])
         self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xc2\xa3')])
 
@@ -71,6 +110,9 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertEqual(name, 'John\xa3')
 
     def test_field_custom_serializer(self):
+        if self.ie.__class__ is BaseItemExporter:
+            return
+
         def custom_serializer(value):
             return str(int(value) + 2)
 
@@ -85,7 +127,7 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
 
 
-class PythonItemExporterTest(BaseItemExporterTest):
+class PythonItemExporterTest(MidRefactoringBaseItemExporterTest):
     def _get_exporter(self, **kwargs):
         return PythonItemExporter(**kwargs)
 
@@ -152,7 +194,7 @@ class PickleItemExporterTest(BaseItemExporterTest):
         self.assertEqual(pickle.load(f), i2)
 
 
-class CsvItemExporterTest(BaseItemExporterTest):
+class CsvItemExporterTest(MidRefactoringBaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
         return CsvItemExporter(self.output, **kwargs)

From c76190d491fca9f35b6758bdc06c34d77f5d9be9 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 21 Jan 2016 18:24:06 -0200
Subject: [PATCH 0749/2974] PY3: ported json(lines), xml exporters

---
 scrapy/exporters.py     | 39 +++++++++++++++-------
 tests/test_exporters.py | 72 +++++++++++++++++++++--------------------
 2 files changed, 64 insertions(+), 47 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 6f679480d..4138f6192 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -11,7 +11,10 @@ from six.moves import cPickle as pickle
 from xml.sax.saxutils import XMLGenerator
 
 from scrapy.utils.serialize import ScrapyJSONEncoder
+from scrapy.utils.python import to_bytes, to_unicode
 from scrapy.item import BaseItem
+import warnings
+
 
 __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
            'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter',
@@ -83,7 +86,7 @@ class JsonLinesItemExporter(BaseItemExporter):
 
     def export_item(self, item):
         itemdict = dict(self._get_serialized_fields(item))
-        self.file.write(self.encoder.encode(itemdict) + '\n')
+        self.file.write(to_bytes(self.encoder.encode(itemdict) + '\n'))
 
 
 class JsonItemExporter(BaseItemExporter):
@@ -95,18 +98,18 @@ class JsonItemExporter(BaseItemExporter):
         self.first_item = True
 
     def start_exporting(self):
-        self.file.write("[")
+        self.file.write(b"[")
 
     def finish_exporting(self):
-        self.file.write("]")
+        self.file.write(b"]")
 
     def export_item(self, item):
         if self.first_item:
             self.first_item = False
         else:
-            self.file.write(',\n')
+            self.file.write(b',\n')
         itemdict = dict(self._get_serialized_fields(item))
-        self.file.write(self.encoder.encode(itemdict))
+        self.file.write(to_bytes(self.encoder.encode(itemdict)))
 
 
 class XmlItemExporter(BaseItemExporter):
@@ -136,8 +139,9 @@ class XmlItemExporter(BaseItemExporter):
         if hasattr(serialized_value, 'items'):
             for subname, value in serialized_value.items():
                 self._export_xml_field(subname, value)
-        elif hasattr(serialized_value, '__iter__'):
-            for value in serialized_value:
+        elif (hasattr(serialized_value, '__iter__')
+              and not isinstance(serialized_value, six.string_types)):
+              for value in serialized_value:
                 self._export_xml_field('value', value)
         else:
             self._xg_characters(serialized_value)
@@ -150,7 +154,7 @@ class XmlItemExporter(BaseItemExporter):
     # and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
     if sys.version_info[:3] >= (2, 7, 4):
         def _xg_characters(self, serialized_value):
-            if not isinstance(serialized_value, unicode):
+            if not isinstance(serialized_value, six.text_type):
                 serialized_value = serialized_value.decode(self.encoding)
             return self.xg.characters(serialized_value)
     else:
@@ -177,7 +181,7 @@ class CsvItemExporter(BaseItemExporter):
                 value = self._join_multivalued.join(value)
             except TypeError:  # list in value may not contain strings
                 pass
-        return value.encode(self.encoding) if isinstance(value, unicode) else value
+        return value.encode(self.encoding) if isinstance(value, six.text_type) else value
 
     def export_item(self, item):
         if self._headers_not_written:
@@ -231,7 +235,7 @@ class PprintItemExporter(BaseItemExporter):
 
     def export_item(self, item):
         itemdict = dict(self._get_serialized_fields(item))
-        self.file.write(pprint.pformat(itemdict) + '\n')
+        self.file.write(to_bytes(pprint.pformat(itemdict) + '\n'))
 
 
 class PythonItemExporter(BaseItemExporter):
@@ -240,6 +244,13 @@ class PythonItemExporter(BaseItemExporter):
     json, msgpack, binc, etc) can be used on top of it. Its main goal is to
     seamless support what BaseItemExporter does plus nested items.
     """
+    def _configure(self, options, dont_fail=False):
+        self.binary = options.pop('binary', True)
+        super(PythonItemExporter, self)._configure(options, dont_fail)
+        if self.binary:
+            warnings.warn(
+                "PythonItemExporter will drop support for binary export in the future",
+                PendingDeprecationWarning)
 
     def serialize_field(self, field, name, value):
         serializer = field.get('serializer', self._serialize_value)
@@ -250,9 +261,13 @@ class PythonItemExporter(BaseItemExporter):
             return self.export_item(value)
         if isinstance(value, dict):
             return dict(self._serialize_dict(value))
-        if hasattr(value, '__iter__'):
+        if hasattr(value, '__iter__') \
+                and not isinstance(value, six.string_types):
             return [self._serialize_value(v) for v in value]
-        return value.encode(self.encoding) if isinstance(value, unicode) else value
+        if self.binary:
+            return to_bytes(value, encoding=self.encoding)
+        else:
+            return to_unicode(value, encoding=self.encoding)
 
     def _serialize_dict(self, value):
         for key, val in six.iteritems(value):
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index c84fb978a..05374e617 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -3,6 +3,7 @@ import re
 import json
 import unittest
 from io import BytesIO
+import six
 from six.moves import cPickle as pickle
 
 import lxml.etree
@@ -80,7 +81,7 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
 
 
-class MidRefactoringBaseItemExporterTest(BaseItemExporterTest):
+class IntermediateRefactoringBaseItemExporterTest(BaseItemExporterTest):
     """Class introduced just to keep old behavior of BaseItemExporterTest for the
     test cases that inherit from it while we make changes to exporters one by
     one -- a needed refactoring trick because the test cases are quite coupled.
@@ -127,9 +128,9 @@ class MidRefactoringBaseItemExporterTest(BaseItemExporterTest):
         self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
 
 
-class PythonItemExporterTest(MidRefactoringBaseItemExporterTest):
+class PythonItemExporterTest(BaseItemExporterTest):
     def _get_exporter(self, **kwargs):
-        return PythonItemExporter(**kwargs)
+        return PythonItemExporter(binary=False, **kwargs)
 
     def test_nested_item(self):
         i1 = TestItem(name=u'Joseph', age='22')
@@ -194,7 +195,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
         self.assertEqual(pickle.load(f), i2)
 
 
-class CsvItemExporterTest(MidRefactoringBaseItemExporterTest):
+@unittest.skipUnless(six.PY2, "TODO")
+class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
         return CsvItemExporter(self.output, **kwargs)
@@ -294,13 +296,13 @@ class XmlItemExporterTest(BaseItemExporterTest):
         self.assertXmlEquivalent(fp.getvalue(), expected_value)
 
     def _check_output(self):
-        expected_value = '<?xml version="1.0" encoding="utf-8"?>\n<items><item><age>22</age><name>John\xc2\xa3</name></item></items>'
+        expected_value = u'<?xml version="1.0" ?>\n<items><item><age>22</age><name>John\xa3</name></item></items>'
         self.assertXmlEquivalent(self.output.getvalue(), expected_value)
 
     def test_multivalued_fields(self):
         self.assertExportResult(
             TestItem(name=[u'John\xa3', u'Doe']),
-            '<?xml version="1.0" encoding="utf-8"?>\n<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
+            u'<?xml version="1.0" ?>\n<items><item><name><value>John\xa3</value><value>Doe</value></name></item></items>'
         )
 
     def test_nested_item(self):
@@ -309,19 +311,19 @@ class XmlItemExporterTest(BaseItemExporterTest):
         i3 = TestItem(name=u'buz', age=i2)
 
         self.assertExportResult(i3,
-            '<?xml version="1.0" encoding="utf-8"?>\n'
-            '<items>'
-                '<item>'
-                    '<age>'
-                        '<age>'
-                            '<age>22</age>'
-                            '<name>foo\xc2\xa3hoo</name>'
-                        '</age>'
-                        '<name>bar</name>'
-                    '</age>'
-                    '<name>buz</name>'
-                '</item>'
-            '</items>'
+            u'<?xml version="1.0" ?>\n'
+            u'<items>'
+                u'<item>'
+                    u'<age>'
+                        u'<age>'
+                            u'<age>22</age>'
+                            u'<name>foo\xa3hoo</name>'
+                        u'</age>'
+                        u'<name>bar</name>'
+                    u'</age>'
+                    u'<name>buz</name>'
+                u'</item>'
+            u'</items>'
         )
 
     def test_nested_list_item(self):
@@ -330,16 +332,16 @@ class XmlItemExporterTest(BaseItemExporterTest):
         i3 = TestItem(name=u'buz', age=[i1, i2])
 
         self.assertExportResult(i3,
-            '<?xml version="1.0" encoding="utf-8"?>\n'
-            '<items>'
-                '<item>'
-                    '<age>'
-                        '<value><name>foo</name></value>'
-                        '<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
-                    '</age>'
-                    '<name>buz</name>'
-                '</item>'
-            '</items>'
+            u'<?xml version="1.0" ?>\n'
+            u'<items>'
+                u'<item>'
+                    u'<age>'
+                        u'<value><name>foo</name></value>'
+                        u'<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
+                    u'</age>'
+                    u'<name>buz</name>'
+                u'</item>'
+            u'</items>'
         )
 
 
@@ -351,7 +353,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         return JsonLinesItemExporter(self.output, **kwargs)
 
     def _check_output(self):
-        exported = json.loads(self.output.getvalue().strip())
+        exported = json.loads(to_unicode(self.output.getvalue().strip()))
         self.assertEqual(exported, dict(self.i))
 
     def test_nested_item(self):
@@ -361,7 +363,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
-        exported = json.loads(self.output.getvalue())
+        exported = json.loads(to_unicode(self.output.getvalue()))
         self.assertEqual(exported, self._expected_nested)
 
     def test_extra_keywords(self):
@@ -379,7 +381,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         return JsonItemExporter(self.output, **kwargs)
 
     def _check_output(self):
-        exported = json.loads(self.output.getvalue().strip())
+        exported = json.loads(to_unicode(self.output.getvalue().strip()))
         self.assertEqual(exported, [dict(self.i)])
 
     def assertTwoItemsExported(self, item):
@@ -387,7 +389,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.export_item(item)
         self.ie.export_item(item)
         self.ie.finish_exporting()
-        exported = json.loads(self.output.getvalue())
+        exported = json.loads(to_unicode(self.output.getvalue()))
         self.assertEqual(exported, [dict(item), dict(item)])
 
     def test_two_items(self):
@@ -403,7 +405,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
-        exported = json.loads(self.output.getvalue())
+        exported = json.loads(to_unicode(self.output.getvalue()))
         expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}}
         self.assertEqual(exported, [expected])
 
@@ -414,7 +416,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
-        exported = json.loads(self.output.getvalue())
+        exported = json.loads(to_unicode(self.output.getvalue()))
         expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}}
         self.assertEqual(exported, [expected])
 

From fed7c8b4fca3bb2722eebca97b298b0316ebfbc2 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 21 Jan 2016 18:39:59 -0200
Subject: [PATCH 0750/2974] fix: use is_listlike

---
 scrapy/exporters.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 4138f6192..ad14f38b3 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -11,7 +11,7 @@ from six.moves import cPickle as pickle
 from xml.sax.saxutils import XMLGenerator
 
 from scrapy.utils.serialize import ScrapyJSONEncoder
-from scrapy.utils.python import to_bytes, to_unicode
+from scrapy.utils.python import to_bytes, to_unicode, is_listlike
 from scrapy.item import BaseItem
 import warnings
 
@@ -139,8 +139,7 @@ class XmlItemExporter(BaseItemExporter):
         if hasattr(serialized_value, 'items'):
             for subname, value in serialized_value.items():
                 self._export_xml_field(subname, value)
-        elif (hasattr(serialized_value, '__iter__')
-              and not isinstance(serialized_value, six.string_types)):
+        elif is_listlike(serialized_value):
               for value in serialized_value:
                 self._export_xml_field('value', value)
         else:
@@ -261,8 +260,7 @@ class PythonItemExporter(BaseItemExporter):
             return self.export_item(value)
         if isinstance(value, dict):
             return dict(self._serialize_dict(value))
-        if hasattr(value, '__iter__') \
-                and not isinstance(value, six.string_types):
+        if is_listlike(value):
             return [self._serialize_value(v) for v in value]
         if self.binary:
             return to_bytes(value, encoding=self.encoding)

From 9f35c286431584e07f9a53b1b2e7b7822a18f7e1 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 21 Jan 2016 18:43:36 -0200
Subject: [PATCH 0751/2974] fix indentation

---
 scrapy/exporters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index ad14f38b3..c029ac473 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -140,7 +140,7 @@ class XmlItemExporter(BaseItemExporter):
             for subname, value in serialized_value.items():
                 self._export_xml_field(subname, value)
         elif is_listlike(serialized_value):
-              for value in serialized_value:
+            for value in serialized_value:
                 self._export_xml_field('value', value)
         else:
             self._xg_characters(serialized_value)

From 240ecbf32378a8be9d87c98bec348301fdd05dc4 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 21 Jan 2016 22:59:48 +0100
Subject: [PATCH 0752/2974] Add local file tests for scrapy shell command

Continuation of #1579
---
 tests/test_command_shell.py | 41 +++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 105202754..7ae685c64 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -1,9 +1,14 @@
+from os.path import join, abspath, dirname, relpath, commonprefix
+import os
+
 from twisted.trial import unittest
 from twisted.internet import defer
 
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
 
+from tests import tests_datadir
+
 
 class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
@@ -51,3 +56,39 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
         code = "fetch('{0}') or fetch(response.request.replace(method='POST'))"
         errcode, out, _ = yield self.execute(['-c', code.format(url)])
         self.assertEqual(errcode, 0, out)
+
+    @defer.inlineCallbacks
+    def test_local_files(self):
+        test_file_path = join(tests_datadir, 'test_site/index.html')
+        valid_paths = [
+            test_file_path,
+            relpath(test_file_path),
+            'file://'+test_file_path,
+            './tests/sample_data/test_site/index.html',
+            'tests/sample_data/test_site/index.html',
+        ]
+        for filepath in valid_paths:
+            _, out, _ = yield self.execute([filepath, '-c', 'item'])
+            assert b'{}' in out
+
+    @defer.inlineCallbacks
+    def test_local_files_invalid(self):
+        invalid_filepaths = [
+            '../nothinghere.html',
+            './tests/sample_data/test_site/nothinghere.html'
+        ]
+        for filepath in invalid_filepaths:
+            errcode, out, err = yield self.execute([filepath, '-c', 'item'],
+                                           check_code=False)
+            self.assertEqual(errcode, 1, out or err)
+            self.assertIn(b'No such file or directory', err)
+
+        # currently, this will try to find a host...
+        invalid_paths = [
+            'nothinghere.html',
+        ]
+        for filepath in invalid_paths:
+            errcode, out, err = yield self.execute([filepath, '-c', 'item'],
+                                           check_code=False)
+            self.assertEqual(errcode, 1, out or err)
+            self.assertIn(b'DNS lookup failed', err)

From 8bd5b60889bef67cb32879562d3cc0e751431ed2 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 21 Jan 2016 23:23:50 +0100
Subject: [PATCH 0753/2974] Remove relpath filepath

---
 tests/test_command_shell.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 7ae685c64..dd201cff3 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -1,5 +1,4 @@
-from os.path import join, abspath, dirname, relpath, commonprefix
-import os
+from os.path import join
 
 from twisted.trial import unittest
 from twisted.internet import defer
@@ -62,7 +61,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
         test_file_path = join(tests_datadir, 'test_site/index.html')
         valid_paths = [
             test_file_path,
-            relpath(test_file_path),
+            # relpath(test_file_path),
             'file://'+test_file_path,
             './tests/sample_data/test_site/index.html',
             'tests/sample_data/test_site/index.html',

From b746d85f4ca11f7f0149d06f7a4b58501ad3ee23 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 21 Jan 2016 21:12:43 -0200
Subject: [PATCH 0754/2974] PY3 port csv exporter

---
 scrapy/exporters.py     | 17 ++++++-----
 tests/test_exporters.py | 67 ++++++-----------------------------------
 2 files changed, 20 insertions(+), 64 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index c029ac473..8d7ffbc71 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -3,6 +3,7 @@ Item Exporters are used to export/serialize items into different formats.
 """
 
 import csv
+import io
 import sys
 import pprint
 import marshal
@@ -11,7 +12,7 @@ from six.moves import cPickle as pickle
 from xml.sax.saxutils import XMLGenerator
 
 from scrapy.utils.serialize import ScrapyJSONEncoder
-from scrapy.utils.python import to_bytes, to_unicode, is_listlike
+from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike
 from scrapy.item import BaseItem
 import warnings
 
@@ -166,21 +167,22 @@ class CsvItemExporter(BaseItemExporter):
     def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs):
         self._configure(kwargs, dont_fail=True)
         self.include_headers_line = include_headers_line
+        file = file if six.PY2 else io.TextIOWrapper(file, line_buffering=True)
         self.csv_writer = csv.writer(file, **kwargs)
         self._headers_not_written = True
         self._join_multivalued = join_multivalued
 
     def serialize_field(self, field, name, value):
-        serializer = field.get('serializer', self._to_str_if_unicode)
+        serializer = field.get('serializer', self._join_if_needed)
         return serializer(value)
 
-    def _to_str_if_unicode(self, value):
+    def _join_if_needed(self, value):
         if isinstance(value, (list, tuple)):
             try:
-                value = self._join_multivalued.join(value)
+                return self._join_multivalued.join(value)
             except TypeError:  # list in value may not contain strings
                 pass
-        return value.encode(self.encoding) if isinstance(value, six.text_type) else value
+        return value
 
     def export_item(self, item):
         if self._headers_not_written:
@@ -189,7 +191,7 @@ class CsvItemExporter(BaseItemExporter):
 
         fields = self._get_serialized_fields(item, default_value='',
                                              include_empty=True)
-        values = [x[1] for x in fields]
+        values = [to_native_str(x) for _, x in fields]
         self.csv_writer.writerow(values)
 
     def _write_headers_and_set_fields_to_export(self, item):
@@ -201,7 +203,8 @@ class CsvItemExporter(BaseItemExporter):
                 else:
                     # use fields declared in Item
                     self.fields_to_export = list(item.fields.keys())
-            self.csv_writer.writerow(self.fields_to_export)
+            row = [to_native_str(s) for s in self.fields_to_export]
+            self.csv_writer.writerow(row)
 
 
 class PickleItemExporter(BaseItemExporter):
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 05374e617..39e996062 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -3,7 +3,6 @@ import re
 import json
 import unittest
 from io import BytesIO
-import six
 from six.moves import cPickle as pickle
 
 import lxml.etree
@@ -81,53 +80,6 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
 
 
-class IntermediateRefactoringBaseItemExporterTest(BaseItemExporterTest):
-    """Class introduced just to keep old behavior of BaseItemExporterTest for the
-    test cases that inherit from it while we make changes to exporters one by
-    one -- a needed refactoring trick because the test cases are quite coupled.
-
-    When we're done with the changes, we'll have ditched this class.
-    """
-    def test_serialize_field(self):
-        if self.ie.__class__ is BaseItemExporter:
-            return
-
-        res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name'])
-        self.assertEqual(res, 'John\xc2\xa3')
-
-        res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age'])
-        self.assertEqual(res, '22')
-
-    def test_fields_to_export(self):
-        if self.ie.__class__ is BaseItemExporter:
-            return
-
-        ie = self._get_exporter(fields_to_export=['name'])
-        self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xc2\xa3')])
-
-        ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
-        name = list(ie._get_serialized_fields(self.i))[0][1]
-        assert isinstance(name, str)
-        self.assertEqual(name, 'John\xa3')
-
-    def test_field_custom_serializer(self):
-        if self.ie.__class__ is BaseItemExporter:
-            return
-
-        def custom_serializer(value):
-            return str(int(value) + 2)
-
-        class CustomFieldItem(Item):
-            name = Field()
-            age = Field(serializer=custom_serializer)
-
-        i = CustomFieldItem(name=u'John\xa3', age='22')
-
-        ie = self._get_exporter()
-        self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John\xc2\xa3')
-        self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
-
-
 class PythonItemExporterTest(BaseItemExporterTest):
     def _get_exporter(self, **kwargs):
         return PythonItemExporter(binary=False, **kwargs)
@@ -195,19 +147,19 @@ class PickleItemExporterTest(BaseItemExporterTest):
         self.assertEqual(pickle.load(f), i2)
 
 
-@unittest.skipUnless(six.PY2, "TODO")
-class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest):
-
+class CsvItemExporterTest(BaseItemExporterTest):
     def _get_exporter(self, **kwargs):
         return CsvItemExporter(self.output, **kwargs)
 
     def assertCsvEqual(self, first, second, msg=None):
+        first = to_unicode(first)
+        second = to_unicode(second)
         csvsplit = lambda csv: [sorted(re.split(r'(,|\s+)', line))
                                 for line in csv.splitlines(True)]
         return self.assertEqual(csvsplit(first), csvsplit(second), msg)
 
     def _check_output(self):
-        self.assertCsvEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n')
+        self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')
 
     def assertExportResult(self, item, expected, **kwargs):
         fp = BytesIO()
@@ -221,13 +173,13 @@ class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest):
         self.assertExportResult(
             item=self.i,
             fields_to_export=self.i.fields.keys(),
-            expected='age,name\r\n22,John\xc2\xa3\r\n',
+            expected=b'age,name\r\n22,John\xc2\xa3\r\n',
         )
 
     def test_header_export_all_dict(self):
         self.assertExportResult(
             item=dict(self.i),
-            expected='age,name\r\n22,John\xc2\xa3\r\n',
+            expected=b'age,name\r\n22,John\xc2\xa3\r\n',
         )
 
     def test_header_export_single_field(self):
@@ -235,7 +187,7 @@ class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest):
             self.assertExportResult(
                 item=item,
                 fields_to_export=['age'],
-                expected='age\r\n22\r\n',
+                expected=b'age\r\n22\r\n',
             )
 
     def test_header_export_two_items(self):
@@ -246,14 +198,15 @@ class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest):
             ie.export_item(item)
             ie.export_item(item)
             ie.finish_exporting()
-            self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
+            self.assertCsvEqual(output.getvalue(),
+                                b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
 
     def test_header_no_header_line(self):
         for item in [self.i, dict(self.i)]:
             self.assertExportResult(
                 item=item,
                 include_headers_line=False,
-                expected='22,John\xc2\xa3\r\n',
+                expected=b'22,John\xc2\xa3\r\n',
             )
 
     def test_join_multivalue(self):

From 2514973242e35831fd90493b3db17227c3c0195e Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 21 Jan 2016 21:22:12 -0200
Subject: [PATCH 0755/2974] re-enable skipped feed export tests

---
 tests/test_feedexport.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index d6c96ca74..8e1cadc74 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -22,6 +22,7 @@ from scrapy.extensions.feedexport import (
     S3FeedStorage, StdoutFeedStorage
 )
 from scrapy.utils.test import assert_aws_environ
+from scrapy.utils.python import to_native_str
 
 
 class FileFeedStorageTest(unittest.TestCase):
@@ -120,8 +121,6 @@ class StdoutFeedStorageTest(unittest.TestCase):
 
 class FeedExportTest(unittest.TestCase):
 
-    skip = not six.PY2
-
     class MyItem(scrapy.Item):
         foo = scrapy.Field()
         egg = scrapy.Field()
@@ -170,7 +169,7 @@ class FeedExportTest(unittest.TestCase):
         settings.update({'FEED_FORMAT': 'csv'})
         data = yield self.exported_data(items, settings)
 
-        reader = csv.DictReader(data.splitlines())
+        reader = csv.DictReader(to_native_str(data).splitlines())
         got_rows = list(reader)
         if ordered:
             self.assertEqual(reader.fieldnames, header)
@@ -184,7 +183,7 @@ class FeedExportTest(unittest.TestCase):
         settings = settings or {}
         settings.update({'FEED_FORMAT': 'jl'})
         data = yield self.exported_data(items, settings)
-        parsed = [json.loads(line) for line in data.splitlines()]
+        parsed = [json.loads(to_native_str(line)) for line in data.splitlines()]
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         self.assertEqual(rows, parsed)
 

From e938752973b4fc53e0fa0c0bc68a431613b987e4 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 21 Jan 2016 21:51:59 -0200
Subject: [PATCH 0756/2974] add test for PythonItemExporter binary mode

---
 scrapy/exporters.py     | 6 +++++-
 tests/test_exporters.py | 7 +++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 8d7ffbc71..118df34a7 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -272,7 +272,11 @@ class PythonItemExporter(BaseItemExporter):
 
     def _serialize_dict(self, value):
         for key, val in six.iteritems(value):
+            key = to_bytes(key) if self.binary else key
             yield key, self._serialize_value(val)
 
     def export_item(self, item):
-        return dict(self._get_serialized_fields(item))
+        result = dict(self._get_serialized_fields(item))
+        if self.binary:
+            result = dict(self._serialize_dict(result))
+        return result
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 39e996062..9e57745dc 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -2,6 +2,7 @@ from __future__ import absolute_import
 import re
 import json
 import unittest
+import warnings
 from io import BytesIO
 from six.moves import cPickle as pickle
 
@@ -115,6 +116,12 @@ class PythonItemExporterTest(BaseItemExporterTest):
         self.assertEqual(type(exported['age'][0]), dict)
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
+    def test_export_binary(self):
+        exporter = PythonItemExporter(binary=True)
+        value = TestItem(name=u'John\xa3', age=u'22')
+        expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
+        self.assertEqual(expected, exporter.export_item(value))
+
 
 class PprintItemExporterTest(BaseItemExporterTest):
 

From 35ada107297fc900e48824f9a94da478a5ae1c7a Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 22 Jan 2016 13:39:27 +0500
Subject: [PATCH 0757/2974] PY3 enable tests for scrapy parse command

scrapy parse command is already ported
---
 tests/test_commands.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index a5230eb13..1a30368ba 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -205,8 +205,6 @@ from scrapy.spiders import Spider
 
 
 class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
-    skip = not six.PY2
-
     command = 'parse'
 
     def setUp(self):

From 6d73e057b500f5063df63abd856ccd77cbafa1f8 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 22 Jan 2016 13:07:42 +0100
Subject: [PATCH 0758/2974] Extract guess_scheme function and refactor tests

---
 scrapy/commands/shell.py    |  30 ++++++----
 tests/test_command_shell.py | 113 ++++++++++++++++++++++++++----------
 2 files changed, 102 insertions(+), 41 deletions(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 14dd0a41a..c975387be 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -16,6 +16,24 @@ from scrapy.utils.url import add_http_if_no_scheme
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 
 
+def guess_scheme(url):
+    """Given an URL as string,
+    returns a FileURI if it looks like a file path,
+    otherwise returns an HTTP URL
+    """
+    parts = urlparse(url)
+    if not parts.scheme:
+        if "." not in parts.path.split("/", 1)[0]:
+            url = any_to_uri(url)
+
+        for pattern in ["/", "./", "../"]:
+            if url.startswith(pattern):
+                url = any_to_uri(url)
+                break
+        url = add_http_if_no_scheme(url)
+    return url
+
+
 class Command(ScrapyCommand):
 
     requires_project = False
@@ -50,16 +68,8 @@ class Command(ScrapyCommand):
     def run(self, args, opts):
         url = args[0] if args else None
         if url:
-            parts = urlparse(url)
-            if not parts.scheme:
-                if "." not in parts.path.split("/", 1)[0]:
-                    url = any_to_uri(url)
-
-                for pattern in ["/", "./", "../"]:
-                    if url.startswith(pattern):
-                        url = any_to_uri(url)
-                        break
-                url = add_http_if_no_scheme(url)
+            # first argument may be a local file
+            url = guess_scheme(url)
 
         spider_loader = self.crawler_process.spider_loader
 
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index dd201cff3..37c6b8c90 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -3,12 +3,77 @@ from os.path import join
 from twisted.trial import unittest
 from twisted.internet import defer
 
+from scrapy.commands.shell import guess_scheme
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
 
 from tests import tests_datadir
 
 
+class ShellURLTest(unittest.TestCase):
+
+    def test_file_uri_relative001(self):
+        # FIXME: 'index.html' is interpreted as a domain name
+        #        is this correct?
+        url = guess_scheme('index.html')
+        assert url.startswith('http://')
+
+    def test_file_uri_relative002(self):
+        url = guess_scheme('./index.html')
+        assert url.startswith('file://')
+
+    def test_file_uri_relative003(self):
+        url = guess_scheme('../data/index.html')
+        assert url.startswith('file://')
+
+    def test_file_uri_relative004(self):
+        url = guess_scheme('subdir/index.html')
+        assert url.startswith('file://')
+
+    def test_file_uri_absolute001(self):
+        """Absolute file paths get prepended with "file://" scheme"""
+        iurl = '/home/user/www/index.html'
+        url = guess_scheme(iurl)
+        self.assertEquals(url, 'file://'+iurl)
+
+    def test_file_uri_scheme(self):
+        """Output File URI does not change if "file://" scheme is set"""
+        iurl = 'file:///home/user/www/index.html'
+        url = guess_scheme(iurl)
+        self.assertEquals(url, iurl)
+
+    def test_file_uri_windows(self):
+        raise unittest.SkipTest("Windows filepath are not supported for scrapy shell")
+        url = guess_scheme('C:\absolute\path\to\a\file.html')
+        assert url.startswith('file://')
+
+    def test_http_url_001(self):
+        url = guess_scheme('index.html')
+        assert url.startswith('http://')
+
+    def test_http_url_002(self):
+        url = guess_scheme('example.com')
+        assert url.startswith('http://')
+
+    def test_http_url_003(self):
+        url = guess_scheme('www.example.com')
+        assert url.startswith('http://')
+
+    def test_http_url_004(self):
+        url = guess_scheme('www.example.com/index')
+        assert url.startswith('http://')
+
+    def test_http_url_005(self):
+        url = guess_scheme('www.example.com/index.html')
+        assert url.startswith('http://')
+
+    def test_http_url_scheme(self):
+        """An full HTTP URL is unaltered"""
+        iurl = 'http://www.example.com/index.html'
+        url = guess_scheme(iurl)
+        self.assertEquals(url, iurl)
+
+
 class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     command = 'shell'
@@ -57,37 +122,23 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
         self.assertEqual(errcode, 0, out)
 
     @defer.inlineCallbacks
-    def test_local_files(self):
-        test_file_path = join(tests_datadir, 'test_site/index.html')
-        valid_paths = [
-            test_file_path,
-            # relpath(test_file_path),
-            'file://'+test_file_path,
-            './tests/sample_data/test_site/index.html',
-            'tests/sample_data/test_site/index.html',
-        ]
-        for filepath in valid_paths:
-            _, out, _ = yield self.execute([filepath, '-c', 'item'])
-            assert b'{}' in out
+    def test_local_file(self):
+        filepath = join(tests_datadir, 'test_site/index.html')
+        _, out, _ = yield self.execute([filepath, '-c', 'item'])
+        assert b'{}' in out
 
     @defer.inlineCallbacks
-    def test_local_files_invalid(self):
-        invalid_filepaths = [
-            '../nothinghere.html',
-            './tests/sample_data/test_site/nothinghere.html'
-        ]
-        for filepath in invalid_filepaths:
-            errcode, out, err = yield self.execute([filepath, '-c', 'item'],
-                                           check_code=False)
-            self.assertEqual(errcode, 1, out or err)
-            self.assertIn(b'No such file or directory', err)
+    def test_local_nofile(self):
+        filepath = 'file:///tests/sample_data/test_site/nothinghere.html'
+        errcode, out, err = yield self.execute([filepath, '-c', 'item'],
+                                       check_code=False)
+        self.assertEqual(errcode, 1, out or err)
+        self.assertIn(b'No such file or directory', err)
 
-        # currently, this will try to find a host...
-        invalid_paths = [
-            'nothinghere.html',
-        ]
-        for filepath in invalid_paths:
-            errcode, out, err = yield self.execute([filepath, '-c', 'item'],
-                                           check_code=False)
-            self.assertEqual(errcode, 1, out or err)
-            self.assertIn(b'DNS lookup failed', err)
+    @defer.inlineCallbacks
+    def test_dns_failures(self):
+        url = 'www.somedomainthatdoesntexi.st'
+        errcode, out, err = yield self.execute([url, '-c', 'item'],
+                                       check_code=False)
+        self.assertEqual(errcode, 1, out or err)
+        self.assertIn(b'DNS lookup failed', err)

From d0955fd08320f8402303bb4424e7dfab384068f4 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Fri, 22 Jan 2016 10:07:55 -0200
Subject: [PATCH 0759/2974] add back test for latin-1 encoding

---
 tests/test_exporters.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 9e57745dc..070624830 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -2,11 +2,11 @@ from __future__ import absolute_import
 import re
 import json
 import unittest
-import warnings
 from io import BytesIO
 from six.moves import cPickle as pickle
 
 import lxml.etree
+import six
 
 from scrapy.item import Item, Field
 from scrapy.utils.python import to_unicode
@@ -66,6 +66,11 @@ class BaseItemExporterTest(unittest.TestCase):
         ie = self._get_exporter(fields_to_export=['name'])
         self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')])
 
+        ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
+        _, name = list(ie._get_serialized_fields(self.i))[0]
+        assert isinstance(name, six.text_type)
+        self.assertEqual(name, u'John\xa3')
+
     def test_field_custom_serializer(self):
         def custom_serializer(value):
             return str(int(value) + 2)

From be239f339c4d4d43ff4f7bd5d3248bb4abd32834 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 22 Jan 2016 13:13:46 +0100
Subject: [PATCH 0760/2974] Remove unused import

---
 scrapy/commands/shell.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index c975387be..1e8427753 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -5,7 +5,7 @@ See documentation in docs/topics/shell.rst
 """
 
 import re
-from six.moves.urllib.parse import urlparse, urlunparse
+from six.moves.urllib.parse import urlparse
 from threading import Thread
 from w3lib.url import any_to_uri
 

From 60052b3c68267eca6c0ed2a178378a2a777b3898 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 22 Jan 2016 13:18:08 +0100
Subject: [PATCH 0761/2974] Remove unused re import

---
 scrapy/commands/shell.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 1e8427753..25ceb5f99 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -4,7 +4,6 @@ Scrapy Shell
 See documentation in docs/topics/shell.rst
 """
 
-import re
 from six.moves.urllib.parse import urlparse
 from threading import Thread
 from w3lib.url import any_to_uri

From 7a51d370f3a59bcfe118f3ba079c5ea6eda5af75 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 22 Jan 2016 17:16:27 +0100
Subject: [PATCH 0762/2974] Regex-based guess_scheme() + refactor tests

---
 scrapy/commands/shell.py    |  28 ++++++----
 tests/test_command_shell.py | 102 +++++++++++++++++-------------------
 2 files changed, 65 insertions(+), 65 deletions(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 25ceb5f99..5201feb42 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -3,7 +3,7 @@ Scrapy Shell
 
 See documentation in docs/topics/shell.rst
 """
-
+import re
 from six.moves.urllib.parse import urlparse
 from threading import Thread
 from w3lib.url import any_to_uri
@@ -21,16 +21,22 @@ def guess_scheme(url):
     otherwise returns an HTTP URL
     """
     parts = urlparse(url)
-    if not parts.scheme:
-        if "." not in parts.path.split("/", 1)[0]:
-            url = any_to_uri(url)
-
-        for pattern in ["/", "./", "../"]:
-            if url.startswith(pattern):
-                url = any_to_uri(url)
-                break
-        url = add_http_if_no_scheme(url)
-    return url
+    if parts.scheme:
+        return url
+    # Note: this does not match Windows filepath
+    if re.match(r'''^                   # start with...
+                    (
+                        \.              # ...a single dot,
+                        (
+                            \. | [^/\.]+  # optionally followed by
+                        )?                # either a second dot or some characters
+                    )?      # optional match of ".", ".." or ".blabla"
+                    /       # at least one "/" for a file path,
+                    .       # and something after the "/"
+                    ''', parts.path, flags=re.VERBOSE):
+        return any_to_uri(url)
+    else:
+        return add_http_if_no_scheme(url)
 
 
 class Command(ScrapyCommand):
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 37c6b8c90..a61d520fa 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -11,67 +11,61 @@ from tests import tests_datadir
 
 
 class ShellURLTest(unittest.TestCase):
+    pass
 
-    def test_file_uri_relative001(self):
-        # FIXME: 'index.html' is interpreted as a domain name
-        #        is this correct?
-        url = guess_scheme('index.html')
-        assert url.startswith('http://')
+def create_guess_scheme_t(args):
+    def do_expected(self):
+        url = guess_scheme(args[0])
+        assert url.startswith(args[1]), \
+            'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (
+                args[0], url, args[1])
+    return do_expected
 
-    def test_file_uri_relative002(self):
-        url = guess_scheme('./index.html')
-        assert url.startswith('file://')
+def create_skipped_scheme_t(args):
+    def do_expected(self):
+        raise unittest.SkipTest(args[2])
+        url = guess_scheme(args[0])
+        assert url.startswith(args[1])
+    return do_expected
 
-    def test_file_uri_relative003(self):
-        url = guess_scheme('../data/index.html')
-        assert url.startswith('file://')
+for k, args in enumerate ([
+            ('/index',                              'file://'),
+            ('/index.html',                         'file://'),
+            ('./index.html',                        'file://'),
+            ('../index.html',                       'file://'),
+            ('../../index.html',                    'file://'),
+            ('./data/index.html',                   'file://'),
+            ('.hidden/data/index.html',             'file://'),
+            ('/home/user/www/index.html',           'file://'),
+            ('//home/user/www/index.html',          'file://'),
+            ('file:///home/user/www/index.html',    'file://'),
 
-    def test_file_uri_relative004(self):
-        url = guess_scheme('subdir/index.html')
-        assert url.startswith('file://')
+            ('index.html',                          'http://'),
+            ('example.com',                         'http://'),
+            ('www.example.com',                     'http://'),
+            ('www.example.com/index.html',          'http://'),
+            ('http://example.com',                  'http://'),
+            ('http://example.com/index.html',       'http://'),
+            ('localhost',                           'http://'),
+            ('localhost/index.html',                'http://'),
 
-    def test_file_uri_absolute001(self):
-        """Absolute file paths get prepended with "file://" scheme"""
-        iurl = '/home/user/www/index.html'
-        url = guess_scheme(iurl)
-        self.assertEquals(url, 'file://'+iurl)
+            # some corner cases (default to http://)
+            ('/',                                   'http://'),
+            ('.../test',                            'http://'),
 
-    def test_file_uri_scheme(self):
-        """Output File URI does not change if "file://" scheme is set"""
-        iurl = 'file:///home/user/www/index.html'
-        url = guess_scheme(iurl)
-        self.assertEquals(url, iurl)
+        ], start=1):
+    t_method = create_guess_scheme_t(args)
+    t_method.__name__ = 'test_uri_%03d' % k
+    setattr (ShellURLTest, t_method.__name__, t_method)
 
-    def test_file_uri_windows(self):
-        raise unittest.SkipTest("Windows filepath are not supported for scrapy shell")
-        url = guess_scheme('C:\absolute\path\to\a\file.html')
-        assert url.startswith('file://')
-
-    def test_http_url_001(self):
-        url = guess_scheme('index.html')
-        assert url.startswith('http://')
-
-    def test_http_url_002(self):
-        url = guess_scheme('example.com')
-        assert url.startswith('http://')
-
-    def test_http_url_003(self):
-        url = guess_scheme('www.example.com')
-        assert url.startswith('http://')
-
-    def test_http_url_004(self):
-        url = guess_scheme('www.example.com/index')
-        assert url.startswith('http://')
-
-    def test_http_url_005(self):
-        url = guess_scheme('www.example.com/index.html')
-        assert url.startswith('http://')
-
-    def test_http_url_scheme(self):
-        """An full HTTP URL is unaltered"""
-        iurl = 'http://www.example.com/index.html'
-        url = guess_scheme(iurl)
-        self.assertEquals(url, iurl)
+# TODO: the following tests do not pass with current implementation
+for k, args in enumerate ([
+            ('C:\absolute\path\to\a\file.html',     'file://',
+             'Windows filepath are not supported for scrapy shell'),
+        ], start=1):
+    t_method = create_skipped_scheme_t(args)
+    t_method.__name__ = 'test_uri_skipped_%03d' % k
+    setattr (ShellURLTest, t_method.__name__, t_method)
 
 
 class ShellTest(ProcessTest, SiteTest, unittest.TestCase):

From 1a30a7774b7d530394fdd761f2a59403b778fe10 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 22 Jan 2016 18:22:19 +0100
Subject: [PATCH 0763/2974] Use pytest.mark.parametrize decorator

---
 tests/test_command_shell.py | 96 +++++++++++++++----------------------
 1 file changed, 39 insertions(+), 57 deletions(-)

diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index a61d520fa..9032e4124 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -1,5 +1,7 @@
 from os.path import join
 
+import pytest
+
 from twisted.trial import unittest
 from twisted.internet import defer
 
@@ -10,63 +12,6 @@ from scrapy.utils.testproc import ProcessTest
 from tests import tests_datadir
 
 
-class ShellURLTest(unittest.TestCase):
-    pass
-
-def create_guess_scheme_t(args):
-    def do_expected(self):
-        url = guess_scheme(args[0])
-        assert url.startswith(args[1]), \
-            'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (
-                args[0], url, args[1])
-    return do_expected
-
-def create_skipped_scheme_t(args):
-    def do_expected(self):
-        raise unittest.SkipTest(args[2])
-        url = guess_scheme(args[0])
-        assert url.startswith(args[1])
-    return do_expected
-
-for k, args in enumerate ([
-            ('/index',                              'file://'),
-            ('/index.html',                         'file://'),
-            ('./index.html',                        'file://'),
-            ('../index.html',                       'file://'),
-            ('../../index.html',                    'file://'),
-            ('./data/index.html',                   'file://'),
-            ('.hidden/data/index.html',             'file://'),
-            ('/home/user/www/index.html',           'file://'),
-            ('//home/user/www/index.html',          'file://'),
-            ('file:///home/user/www/index.html',    'file://'),
-
-            ('index.html',                          'http://'),
-            ('example.com',                         'http://'),
-            ('www.example.com',                     'http://'),
-            ('www.example.com/index.html',          'http://'),
-            ('http://example.com',                  'http://'),
-            ('http://example.com/index.html',       'http://'),
-            ('localhost',                           'http://'),
-            ('localhost/index.html',                'http://'),
-
-            # some corner cases (default to http://)
-            ('/',                                   'http://'),
-            ('.../test',                            'http://'),
-
-        ], start=1):
-    t_method = create_guess_scheme_t(args)
-    t_method.__name__ = 'test_uri_%03d' % k
-    setattr (ShellURLTest, t_method.__name__, t_method)
-
-# TODO: the following tests do not pass with current implementation
-for k, args in enumerate ([
-            ('C:\absolute\path\to\a\file.html',     'file://',
-             'Windows filepath are not supported for scrapy shell'),
-        ], start=1):
-    t_method = create_skipped_scheme_t(args)
-    t_method.__name__ = 'test_uri_skipped_%03d' % k
-    setattr (ShellURLTest, t_method.__name__, t_method)
-
 
 class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
@@ -136,3 +81,40 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
                                        check_code=False)
         self.assertEqual(errcode, 1, out or err)
         self.assertIn(b'DNS lookup failed', err)
+
+
+@pytest.mark.parametrize("url, scheme", [
+    ('/index',                              'file://'),
+    ('/index.html',                         'file://'),
+    ('./index.html',                        'file://'),
+    ('../index.html',                       'file://'),
+    ('../../index.html',                    'file://'),
+    ('./data/index.html',                   'file://'),
+    ('.hidden/data/index.html',             'file://'),
+    ('/home/user/www/index.html',           'file://'),
+    ('//home/user/www/index.html',          'file://'),
+    ('file:///home/user/www/index.html',    'file://'),
+
+    ('index.html',                          'http://'),
+    ('example.com',                         'http://'),
+    ('www.example.com',                     'http://'),
+    ('www.example.com/index.html',          'http://'),
+    ('http://example.com',                  'http://'),
+    ('http://example.com/index.html',       'http://'),
+    ('localhost',                           'http://'),
+    ('localhost/index.html',                'http://'),
+
+    # some corner cases (default to http://)
+    ('/',                                   'http://'),
+    ('.../test',                            'http://'),
+
+    pytest.mark.xfail(
+        (r'C:\absolute\path\to\a\file.html', 'file://'),
+         reason = 'Windows filepath are not supported for scrapy shell'
+    ),
+])
+def test_guess_scheme(url, scheme):
+    guessed_url = guess_scheme(url)
+    assert guessed_url.startswith(scheme), \
+        'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (
+            url, guessed_url, scheme)

From 5f09da60c1a360cf0bd55cfcc892e44a13d5c583 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 22 Jan 2016 23:48:58 +0100
Subject: [PATCH 0764/2974] Revert "Use pytest.mark.parametrize decorator"

This reverts commit 1a30a7774b7d530394fdd761f2a59403b778fe10.
---
 tests/test_command_shell.py | 96 ++++++++++++++++++++++---------------
 1 file changed, 57 insertions(+), 39 deletions(-)

diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 9032e4124..a61d520fa 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -1,7 +1,5 @@
 from os.path import join
 
-import pytest
-
 from twisted.trial import unittest
 from twisted.internet import defer
 
@@ -12,6 +10,63 @@ from scrapy.utils.testproc import ProcessTest
 from tests import tests_datadir
 
 
+class ShellURLTest(unittest.TestCase):
+    pass
+
+def create_guess_scheme_t(args):
+    def do_expected(self):
+        url = guess_scheme(args[0])
+        assert url.startswith(args[1]), \
+            'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (
+                args[0], url, args[1])
+    return do_expected
+
+def create_skipped_scheme_t(args):
+    def do_expected(self):
+        raise unittest.SkipTest(args[2])
+        url = guess_scheme(args[0])
+        assert url.startswith(args[1])
+    return do_expected
+
+for k, args in enumerate ([
+            ('/index',                              'file://'),
+            ('/index.html',                         'file://'),
+            ('./index.html',                        'file://'),
+            ('../index.html',                       'file://'),
+            ('../../index.html',                    'file://'),
+            ('./data/index.html',                   'file://'),
+            ('.hidden/data/index.html',             'file://'),
+            ('/home/user/www/index.html',           'file://'),
+            ('//home/user/www/index.html',          'file://'),
+            ('file:///home/user/www/index.html',    'file://'),
+
+            ('index.html',                          'http://'),
+            ('example.com',                         'http://'),
+            ('www.example.com',                     'http://'),
+            ('www.example.com/index.html',          'http://'),
+            ('http://example.com',                  'http://'),
+            ('http://example.com/index.html',       'http://'),
+            ('localhost',                           'http://'),
+            ('localhost/index.html',                'http://'),
+
+            # some corner cases (default to http://)
+            ('/',                                   'http://'),
+            ('.../test',                            'http://'),
+
+        ], start=1):
+    t_method = create_guess_scheme_t(args)
+    t_method.__name__ = 'test_uri_%03d' % k
+    setattr (ShellURLTest, t_method.__name__, t_method)
+
+# TODO: the following tests do not pass with current implementation
+for k, args in enumerate ([
+            ('C:\absolute\path\to\a\file.html',     'file://',
+             'Windows filepath are not supported for scrapy shell'),
+        ], start=1):
+    t_method = create_skipped_scheme_t(args)
+    t_method.__name__ = 'test_uri_skipped_%03d' % k
+    setattr (ShellURLTest, t_method.__name__, t_method)
+
 
 class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
@@ -81,40 +136,3 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
                                        check_code=False)
         self.assertEqual(errcode, 1, out or err)
         self.assertIn(b'DNS lookup failed', err)
-
-
-@pytest.mark.parametrize("url, scheme", [
-    ('/index',                              'file://'),
-    ('/index.html',                         'file://'),
-    ('./index.html',                        'file://'),
-    ('../index.html',                       'file://'),
-    ('../../index.html',                    'file://'),
-    ('./data/index.html',                   'file://'),
-    ('.hidden/data/index.html',             'file://'),
-    ('/home/user/www/index.html',           'file://'),
-    ('//home/user/www/index.html',          'file://'),
-    ('file:///home/user/www/index.html',    'file://'),
-
-    ('index.html',                          'http://'),
-    ('example.com',                         'http://'),
-    ('www.example.com',                     'http://'),
-    ('www.example.com/index.html',          'http://'),
-    ('http://example.com',                  'http://'),
-    ('http://example.com/index.html',       'http://'),
-    ('localhost',                           'http://'),
-    ('localhost/index.html',                'http://'),
-
-    # some corner cases (default to http://)
-    ('/',                                   'http://'),
-    ('.../test',                            'http://'),
-
-    pytest.mark.xfail(
-        (r'C:\absolute\path\to\a\file.html', 'file://'),
-         reason = 'Windows filepath are not supported for scrapy shell'
-    ),
-])
-def test_guess_scheme(url, scheme):
-    guessed_url = guess_scheme(url)
-    assert guessed_url.startswith(scheme), \
-        'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (
-            url, guessed_url, scheme)

From c75f1fe46a8a2a3c471eeef2c023754ee5e6c2f1 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Sat, 23 Jan 2016 16:09:57 -0200
Subject: [PATCH 0765/2974] restore bytes instead of text, for easier reviewing

---
 tests/test_exporters.py | 50 ++++++++++++++++++++---------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 070624830..00352f61e 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -261,13 +261,13 @@ class XmlItemExporterTest(BaseItemExporterTest):
         self.assertXmlEquivalent(fp.getvalue(), expected_value)
 
     def _check_output(self):
-        expected_value = u'<?xml version="1.0" ?>\n<items><item><age>22</age><name>John\xa3</name></item></items>'
+        expected_value = b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><age>22</age><name>John\xc2\xa3</name></item></items>'
         self.assertXmlEquivalent(self.output.getvalue(), expected_value)
 
     def test_multivalued_fields(self):
         self.assertExportResult(
             TestItem(name=[u'John\xa3', u'Doe']),
-            u'<?xml version="1.0" ?>\n<items><item><name><value>John\xa3</value><value>Doe</value></name></item></items>'
+            b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
         )
 
     def test_nested_item(self):
@@ -276,19 +276,19 @@ class XmlItemExporterTest(BaseItemExporterTest):
         i3 = TestItem(name=u'buz', age=i2)
 
         self.assertExportResult(i3,
-            u'<?xml version="1.0" ?>\n'
-            u'<items>'
-                u'<item>'
-                    u'<age>'
-                        u'<age>'
-                            u'<age>22</age>'
-                            u'<name>foo\xa3hoo</name>'
-                        u'</age>'
-                        u'<name>bar</name>'
-                    u'</age>'
-                    u'<name>buz</name>'
-                u'</item>'
-            u'</items>'
+            b'<?xml version="1.0" encoding="utf-8"?>\n'
+            b'<items>'
+                b'<item>'
+                    b'<age>'
+                        b'<age>'
+                            b'<age>22</age>'
+                            b'<name>foo\xc2\xa3hoo</name>'
+                        b'</age>'
+                        b'<name>bar</name>'
+                    b'</age>'
+                    b'<name>buz</name>'
+                b'</item>'
+            b'</items>'
         )
 
     def test_nested_list_item(self):
@@ -297,16 +297,16 @@ class XmlItemExporterTest(BaseItemExporterTest):
         i3 = TestItem(name=u'buz', age=[i1, i2])
 
         self.assertExportResult(i3,
-            u'<?xml version="1.0" ?>\n'
-            u'<items>'
-                u'<item>'
-                    u'<age>'
-                        u'<value><name>foo</name></value>'
-                        u'<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
-                    u'</age>'
-                    u'<name>buz</name>'
-                u'</item>'
-            u'</items>'
+            b'<?xml version="1.0" encoding="utf-8"?>\n'
+            b'<items>'
+                b'<item>'
+                    b'<age>'
+                        b'<value><name>foo</name></value>'
+                        b'<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
+                    b'</age>'
+                    b'<name>buz</name>'
+                b'</item>'
+            b'</items>'
         )
 
 

From 935b1da8c2d1d801f4d7d9ce8f498c1d7a527644 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Sat, 23 Jan 2016 16:13:42 -0200
Subject: [PATCH 0766/2974] uses ScrapyDeprecationWarning instead of silenced
 PendingDeprecationWarning

---
 scrapy/exporters.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 118df34a7..fa6663ed4 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -14,6 +14,7 @@ from xml.sax.saxutils import XMLGenerator
 from scrapy.utils.serialize import ScrapyJSONEncoder
 from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike
 from scrapy.item import BaseItem
+from scrapy.exceptions import ScrapyDeprecationWarning
 import warnings
 
 
@@ -252,7 +253,7 @@ class PythonItemExporter(BaseItemExporter):
         if self.binary:
             warnings.warn(
                 "PythonItemExporter will drop support for binary export in the future",
-                PendingDeprecationWarning)
+                ScrapyDeprecationWarning)
 
     def serialize_field(self, field, name, value):
         serializer = field.get('serializer', self._serialize_value)

From 9fbe6f3e814578f90f206031ae99a344b842e400 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Sat, 23 Jan 2016 17:17:40 -0200
Subject: [PATCH 0767/2974] added feedexport test for xml output

---
 tests/test_feedexport.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 8e1cadc74..8db9d589e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -5,7 +5,6 @@ import json
 from io import BytesIO
 import tempfile
 import shutil
-import six
 from six.moves.urllib.parse import urlparse
 
 from zope.interface.verify import verifyObject
@@ -187,10 +186,22 @@ class FeedExportTest(unittest.TestCase):
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         self.assertEqual(rows, parsed)
 
+    @defer.inlineCallbacks
+    def assertExportedXml(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({'FEED_FORMAT': 'xml'})
+        data = yield self.exported_data(items, settings)
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        import lxml.etree
+        root = lxml.etree.fromstring(data)
+        got_rows = [{e.tag: e.text for e in it} for it in root.findall('item')]
+        self.assertEqual(rows, got_rows)
+
     @defer.inlineCallbacks
     def assertExported(self, items, header, rows, settings=None, ordered=True):
         yield self.assertExportedCsv(items, header, rows, settings, ordered)
         yield self.assertExportedJsonLines(items, rows, settings)
+        yield self.assertExportedXml(items, rows, settings)
 
     @defer.inlineCallbacks
     def test_export_items(self):

From 9704226ee4c933e1214e029d66005f1bee2fb766 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Sun, 24 Jan 2016 13:25:14 +0300
Subject: [PATCH 0768/2974] py3: fix test_mail - get_payload returns bytes when
 decode is True

---
 tests/test_mail.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_mail.py b/tests/test_mail.py
index 58d44bdb3..25dd35099 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -53,8 +53,8 @@ class MailSenderTest(unittest.TestCase):
         self.assertEqual(len(payload), 2)
 
         text, attach = payload
-        self.assertEqual(text.get_payload(decode=True), 'body')
-        self.assertEqual(attach.get_payload(decode=True), 'content')
+        self.assertEqual(text.get_payload(decode=True), b'body')
+        self.assertEqual(attach.get_payload(decode=True), b'content')
 
     def _catch_mail_sent(self, **kwargs):
         self.catched_msg = dict(**kwargs)

From 860353b0c03fffb2fed44942b4389f92bd7f7c09 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Sun, 24 Jan 2016 13:27:41 +0300
Subject: [PATCH 0769/2974] py3: unskip test_mail and scrapy/mail.py

---
 tests/py3-ignores.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index f189a4c86..70d3fb905 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,6 +1,5 @@
 tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
-tests/test_mail.py
 tests/test_pipeline_files.py
 tests/test_pipeline_images.py
 tests/test_proxy_connect.py
@@ -22,4 +21,3 @@ scrapy/linkextractors/htmlparser.py
 scrapy/downloadermiddlewares/cookies.py
 scrapy/extensions/statsmailer.py
 scrapy/extensions/memusage.py
-scrapy/mail.py

From 333d4c91fb998b4f7f8a9e184e73715c33a38ce9 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Sun, 24 Jan 2016 22:52:50 +0300
Subject: [PATCH 0770/2974] py3: add boto to py3 test requirements,
 test_pipeline_files and test_pipeline_images passing now

---
 tests/py3-ignores.txt      | 2 --
 tests/requirements-py3.txt | 1 +
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 70d3fb905..212f40f23 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,7 +1,5 @@
 tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
-tests/test_pipeline_files.py
-tests/test_pipeline_images.py
 tests/test_proxy_connect.py
 tests/test_spidermiddleware_httperror.py
 
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 5cf786a89..73e73e651 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -4,6 +4,7 @@ pytest-cov
 testfixtures
 jmespath
 leveldb
+boto
 # optional for shell wrapper tests
 bpython
 ipython

From 097082cffa0a9f11f72a3cee4d8941b7c2566538 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Sun, 24 Jan 2016 23:05:23 +0300
Subject: [PATCH 0771/2974] reviewed py3 compat in pipelines/images.py and
 pipelines/files.py

---
 tests/py3-ignores.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 212f40f23..eb2cc4f5a 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -11,8 +11,6 @@ scrapy/xlib/tx/_newclient.py
 scrapy/xlib/tx/__init__.py
 scrapy/core/downloader/handlers/s3.py
 scrapy/core/downloader/handlers/ftp.py
-scrapy/pipelines/images.py
-scrapy/pipelines/files.py
 scrapy/linkextractors/sgml.py
 scrapy/linkextractors/regex.py
 scrapy/linkextractors/htmlparser.py

From 4233b3cda4514a364511bc6f35a495cfbe50c4a6 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Sun, 24 Jan 2016 23:10:03 +0300
Subject: [PATCH 0772/2974] py3: reviewed passing
 test_spidermiddleware_httperror.py

---
 tests/py3-ignores.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 70d3fb905..e753b993e 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -3,7 +3,6 @@ tests/test_linkextractors_deprecated.py
 tests/test_pipeline_files.py
 tests/test_pipeline_images.py
 tests/test_proxy_connect.py
-tests/test_spidermiddleware_httperror.py
 
 scrapy/xlib/tx/iweb.py
 scrapy/xlib/tx/interfaces.py

From 1be90323c27bfda7588d4437a1996c5e92eb452d Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Sun, 24 Jan 2016 23:44:56 +0300
Subject: [PATCH 0773/2974] py3: properly skip s3 tests on py3

---
 tests/test_downloader_handlers.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 1eb6192ce..56608bfc6 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -437,6 +437,8 @@ class S3AnonTestCase(unittest.TestCase):
         import boto
     except ImportError:
         skip = 'missing boto library'
+    if six.PY3:
+        skip = 'S3 not supported on Py3'
 
     def setUp(self):
         self.s3reqh = S3DownloadHandler(Settings(),
@@ -459,6 +461,8 @@ class S3TestCase(unittest.TestCase):
         import boto
     except ImportError:
         skip = 'missing boto library'
+    if six.PY3:
+        skip = 'S3 not supported on Py3'
 
     # test use same example keys than amazon developer guide
     # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf

From 0c44fac2b54e72202787fda9d221f62a234151d4 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Sun, 24 Jan 2016 19:17:42 -0200
Subject: [PATCH 0774/2974] added tests for feed export marshal and pickle

---
 tests/test_feedexport.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 8db9d589e..176fd93e3 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -197,11 +197,42 @@ class FeedExportTest(unittest.TestCase):
         got_rows = [{e.tag: e.text for e in it} for it in root.findall('item')]
         self.assertEqual(rows, got_rows)
 
+    def _load_until_eof(self, data, load_func):
+        bytes_output = BytesIO(data)
+        result = []
+        while True:
+            try:
+                result.append(load_func(bytes_output))
+            except EOFError:
+                break
+        return result
+
+    @defer.inlineCallbacks
+    def assertExportedPickle(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({'FEED_FORMAT': 'pickle'})
+        data = yield self.exported_data(items, settings)
+        expected = [{k: v for k, v in row.items() if v} for row in rows]
+        import pickle
+        result = self._load_until_eof(data, load_func=pickle.load)
+        self.assertEqual(expected, result)
+
+    @defer.inlineCallbacks
+    def assertExportedMarshal(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({'FEED_FORMAT': 'marshal'})
+        data = yield self.exported_data(items, settings)
+        expected = [{k: v for k, v in row.items() if v} for row in rows]
+        import marshal
+        result = self._load_until_eof(data, load_func=marshal.load)
+        self.assertEqual(expected, result)
+
     @defer.inlineCallbacks
     def assertExported(self, items, header, rows, settings=None, ordered=True):
         yield self.assertExportedCsv(items, header, rows, settings, ordered)
         yield self.assertExportedJsonLines(items, rows, settings)
         yield self.assertExportedXml(items, rows, settings)
+        yield self.assertExportedPickle(items, rows, settings)
 
     @defer.inlineCallbacks
     def test_export_items(self):

From fb8ab2427bff636375b64fdc459562432c8ac420 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 25 Jan 2016 13:13:35 +0100
Subject: [PATCH 0775/2974] Move urlparsing statement in
 add_http_if_no_scheme()

---
 scrapy/utils/url.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 0e36003ad..0acbbb6ab 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -116,8 +116,8 @@ def escape_ajax(url):
 def add_http_if_no_scheme(url):
     """Add http as the default scheme if it is missing from the url."""
     match = re.match(r"^\w+://", url, flags=re.I)
-    parts = urlparse(url)
     if not match:
+        parts = urlparse(url)
         scheme = "http:" if parts.netloc else "http://"
         url = scheme + url
 

From 23b3336c1feb3acb603a935f11e442557d6434f4 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Mon, 25 Jan 2016 22:11:04 -0200
Subject: [PATCH 0776/2974] add test for invalid option

---
 tests/test_exporters.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 00352f61e..61a0229a4 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -90,6 +90,10 @@ class PythonItemExporterTest(BaseItemExporterTest):
     def _get_exporter(self, **kwargs):
         return PythonItemExporter(binary=False, **kwargs)
 
+    def test_invalid_option(self):
+        with self.assertRaisesRegexp(TypeError, "Unexpected options: invalid_option"):
+            PythonItemExporter(invalid_option='something')
+
     def test_nested_item(self):
         i1 = TestItem(name=u'Joseph', age='22')
         i2 = dict(name=u'Maria', age=i1)

From 2dfdde3c79a5be468302a1e825cc5ad77444a8ac Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Mon, 25 Jan 2016 22:24:35 -0200
Subject: [PATCH 0777/2974] fallback to repr when can't convert to native
 string

---
 scrapy/exporters.py     | 11 +++++++++--
 tests/test_exporters.py |  7 +++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index fa6663ed4..69c180ea4 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -192,9 +192,16 @@ class CsvItemExporter(BaseItemExporter):
 
         fields = self._get_serialized_fields(item, default_value='',
                                              include_empty=True)
-        values = [to_native_str(x) for _, x in fields]
+        values = list(self._build_row(x for _, x in fields))
         self.csv_writer.writerow(values)
 
+    def _build_row(self, values):
+        for s in values:
+            try:
+                yield to_native_str(s)
+            except TypeError:
+                yield to_native_str(repr(s))
+
     def _write_headers_and_set_fields_to_export(self, item):
         if self.include_headers_line:
             if not self.fields_to_export:
@@ -204,7 +211,7 @@ class CsvItemExporter(BaseItemExporter):
                 else:
                     # use fields declared in Item
                     self.fields_to_export = list(item.fields.keys())
-            row = [to_native_str(s) for s in self.fields_to_export]
+            row = list(self._build_row(self.fields_to_export))
             self.csv_writer.writerow(row)
 
 
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 61a0229a4..8930545a6 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -237,6 +237,13 @@ class CsvItemExporterTest(BaseItemExporterTest):
                 expected='"Mary,Paul",John\r\n',
             )
 
+    def test_join_multivalue_not_strings(self):
+        self.assertExportResult(
+            item=dict(name='John', friends=[4, 8]),
+            include_headers_line=False,
+            expected='"[4, 8]",John\r\n',
+        )
+
 
 class XmlItemExporterTest(BaseItemExporterTest):
 

From d0eacfe0f90263035f98beb8fe6b5a8f182d5a1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Tue, 26 Jan 2016 00:26:27 -0300
Subject: [PATCH 0778/2974] Add test case for marshal item exporter

---
 scrapy/exporters.py     |  2 +-
 tests/test_exporters.py | 16 +++++++++++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 69c180ea4..145468dbe 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -158,7 +158,7 @@ class XmlItemExporter(BaseItemExporter):
             if not isinstance(serialized_value, six.text_type):
                 serialized_value = serialized_value.decode(self.encoding)
             return self.xg.characters(serialized_value)
-    else:
+    else:  # pragma: no cover
         def _xg_characters(self, serialized_value):
             return self.xg.characters(serialized_value)
 
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 8930545a6..1633e1039 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -1,6 +1,8 @@
 from __future__ import absolute_import
 import re
 import json
+import marshal
+import tempfile
 import unittest
 from io import BytesIO
 from six.moves import cPickle as pickle
@@ -12,7 +14,8 @@ from scrapy.item import Item, Field
 from scrapy.utils.python import to_unicode
 from scrapy.exporters import (
     BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter,
-    XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, PythonItemExporter
+    XmlItemExporter, JsonLinesItemExporter, JsonItemExporter,
+    PythonItemExporter, MarshalItemExporter
 )
 
 
@@ -163,6 +166,17 @@ class PickleItemExporterTest(BaseItemExporterTest):
         self.assertEqual(pickle.load(f), i2)
 
 
+class MarshalItemExporterTest(BaseItemExporterTest):
+
+    def _get_exporter(self, **kwargs):
+        self.output = tempfile.TemporaryFile()
+        return MarshalItemExporter(self.output, **kwargs)
+
+    def _check_output(self):
+        self.output.seek(0)
+        self._assert_expected_item(marshal.load(self.output))
+
+
 class CsvItemExporterTest(BaseItemExporterTest):
     def _get_exporter(self, **kwargs):
         return CsvItemExporter(self.output, **kwargs)

From 7070dae48da02fa29aa1650af4df28561e343436 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 26 Jan 2016 13:56:16 +0500
Subject: [PATCH 0779/2974] deprecate unused and untested
 scrapy.utils.datatypes.SiteNode

---
 scrapy/utils/datatypes.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 097bd1ac9..2b54982b8 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -137,10 +137,18 @@ class MultiValueDict(dict):
         for key, value in six.iteritems(kwargs):
             self.setlistdefault(key, []).append(value)
 
+
 class SiteNode(object):
     """Class to represent a site node (page, image or any other file)"""
 
     def __init__(self, url):
+        warnings.warn(
+            "scrapy.utils.datatypes.SiteNode is deprecated "
+            "and will be removed in future releases.",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2
+        )
+
         self.url = url
         self.itemnames = []
         self.children = []

From 9c2aa50ea20d2333eb131c6aae7b9842d646e32e Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 26 Jan 2016 13:58:20 +0500
Subject: [PATCH 0780/2974] deprecate unused and untested
 scrapy.utils.datatypes.MultiValueDict

---
 scrapy/utils/datatypes.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 2b54982b8..d04b43176 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -7,11 +7,22 @@ This module must not depend on any module outside the Standard Library.
 
 import copy
 import six
+import warnings
 from collections import OrderedDict
 
+from scrapy.exceptions import ScrapyDeprecationWarning
+
 
 class MultiValueDictKeyError(KeyError):
-    pass
+    def __init__(self, *args, **kwargs):
+        warnings.warn(
+            "scrapy.utils.datatypes.MultiValueDictKeyError is deprecated "
+            "and will be removed in future releases.",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2
+        )
+        super(MultiValueDictKeyError, self).__init__(*args, **kwargs)
+
 
 class MultiValueDict(dict):
     """
@@ -31,6 +42,10 @@ class MultiValueDict(dict):
     single name-value pairs.
     """
     def __init__(self, key_to_list_mapping=()):
+        warnings.warn("scrapy.utils.datatypes.MultiValueDict is deprecated "
+                      "and will be removed in future releases.",
+                      category=ScrapyDeprecationWarning,
+                      stacklevel=2)
         dict.__init__(self, key_to_list_mapping)
 
     def __repr__(self):

From 713e1eee9b14ef95515b20baacb83daba9c1277a Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 26 Jan 2016 10:44:38 +0100
Subject: [PATCH 0781/2974] Update docs about local files support for "scrapy
 shell"

---
 docs/topics/commands.rst |  4 +++-
 docs/topics/shell.rst    | 30 ++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 16af52eea..9a40a2c29 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -373,7 +373,9 @@ shell
 * Requires project: *no*
 
 Starts the Scrapy shell for the given URL (if given) or empty if no URL is
-given. See :ref:`topics-shell` for more info.
+given. Also supports UNIX-style local file paths, either relative with
+``./`` or ``../`` prefixes or absolute file paths.
+See :ref:`topics-shell` for more info.
 
 Usage example::
 
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 3569cbf37..4af11fbb6 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -53,6 +53,36 @@ this::
 
 Where the ``<url>`` is the URL you want to scrape.
 
+:command:`shell` also works for local files. This can be handy if you want
+to play around with a local copy of a web page. :command:`shell` understands
+the following syntaxes for local files::
+
+    # UNIX-style
+    scrapy shell ./path/to/file.html
+    scrapy shell ../other/path/to/file.html
+    scrapy shell /absolute/path/to/file.html
+
+    # File URI
+    scrapy shell file:///absolute/path/to/file.html
+
+.. warning:: :command:`shell` will interpret ``index.html`` as a domain name,
+   not as a relative path to a local file, and will trigger a DNS lookup error::
+
+    $ scrapy shell index.html
+    [ ... scrapy shell starts ... ]
+    2016-01-26 10:29:51 [scrapy] DEBUG: Gave up retrying <GET http://index.html>
+    (failed 3 times): DNS lookup failed:
+    address 'index.html' not found: [Errno -5] No address associated with hostname.
+    [ ... traceback ... ]
+    twisted.internet.error.DNSLookupError: DNS lookup failed:
+    address 'index.html' not found: [Errno -5] No address associated with hostname.
+
+   Use ``./`` prefix instead::
+
+    $ scrapy shell ./index.html
+    [ ... scrapy shell starts ... ]
+
+
 Using the shell
 ===============
 

From 1cffa99e0d524ad6a3f989893de1693042e92f92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Tue, 26 Jan 2016 12:35:40 +0200
Subject: [PATCH 0782/2974] tests+doc for subdomains in offsite middleware

---
 docs/topics/spider-middleware.rst      | 3 +++
 docs/topics/spiders.rst                | 2 +-
 tests/test_spidermiddleware_offsite.py | 9 ++++++---
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 84daaaa55..ced481c71 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -273,6 +273,9 @@ OffsiteMiddleware
 
    This middleware filters out every request whose host names aren't in the
    spider's :attr:`~scrapy.spiders.Spider.allowed_domains` attribute.
+   All subdomains of any domain in the list are also allowed.
+   E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org``
+   but not ``www2.example.com`` nor ``example.com``.
 
    When your spider returns a request for a domain not belonging to those
    covered by the spider, this middleware will log a debug message similar to
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 5fd187e4e..b700ea0ef 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -76,7 +76,7 @@ scrapy.Spider
 
        An optional list of strings containing domains that this spider is
        allowed to crawl. Requests for URLs not belonging to the domain names
-       specified in this list won't be followed if
+       specified in this list (or their subdomains) won't be followed if
        :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled.
 
    .. attribute:: start_urls
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index f88c806d7..37c3a450b 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -16,7 +16,7 @@ class TestOffsiteMiddleware(TestCase):
         self.mw.spider_opened(self.spider)
 
     def _get_spiderargs(self):
-        return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org'])
+        return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org', 'scrapy.test.org'])
 
     def test_process_spider_output(self):
         res = Response('http://scrapytest.org')
@@ -24,13 +24,16 @@ class TestOffsiteMiddleware(TestCase):
         onsite_reqs = [Request('http://scrapytest.org/1'),
                        Request('http://scrapy.org/1'),
                        Request('http://sub.scrapy.org/1'),
-                       Request('http://offsite.tld/letmepass', dont_filter=True)]
+                       Request('http://offsite.tld/letmepass', dont_filter=True),
+                       Request('http://scrapy.test.org/')]
         offsite_reqs = [Request('http://scrapy2.org'),
                        Request('http://offsite.tld/'),
                        Request('http://offsite.tld/scrapytest.org'),
                        Request('http://offsite.tld/rogue.scrapytest.org'),
                        Request('http://rogue.scrapytest.org.haha.com'),
-                       Request('http://roguescrapytest.org')]
+                       Request('http://roguescrapytest.org'),
+                       Request('http://test.org/'),
+                       Request('http://notscrapy.test.org/')]
         reqs = onsite_reqs + offsite_reqs
 
         out = list(self.mw.process_spider_output(res, reqs, self.spider))

From 7608da8868baba981fa1d84f7691caaf1e0bd5d8 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 26 Jan 2016 13:01:12 +0100
Subject: [PATCH 0783/2974] Fix logging of enabled middlewares

Wrong middlewares list was being pretty-printed
(introduced in #1263)
---
 scrapy/middleware.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 2ef5f30e2..6120488e2 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -44,9 +44,11 @@ class MiddlewareManager(object):
                     logger.warning("Disabled %(clsname)s: %(eargs)s",
                                    {'clsname': clsname, 'eargs': e.args[0]},
                                    extra={'crawler': crawler})
+
+        enabled = [x.__class__.__name__ for x in middlewares]
         logger.info("Enabled %(componentname)ss:\n%(enabledlist)s",
                     {'componentname': cls.component_name,
-                     'enabledlist': pprint.pformat(mwlist)},
+                     'enabledlist': pprint.pformat(enabled)},
                     extra={'crawler': crawler})
         return cls(*middlewares)
 

From 6ee8d8650a5d9041d6eeddddabcd16c0e0e8d9c9 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 26 Jan 2016 13:08:42 +0100
Subject: [PATCH 0784/2974] Disable CloseSpider extension if no CLOSPIDER_*
 setting set

---
 scrapy/extensions/closespider.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index a5df5e8a7..9ccf356ec 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -9,6 +9,7 @@ from collections import defaultdict
 from twisted.internet import reactor
 
 from scrapy import signals
+from scrapy.exceptions import NotConfigured
 
 
 class CloseSpider(object):
@@ -23,6 +24,9 @@ class CloseSpider(object):
             'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'),
             }
 
+        if not any(self.close_on.values()):
+            raise NotConfigured
+
         self.counter = defaultdict(int)
 
         if self.close_on.get('errorcount'):

From f30758c246ef10dc5ddb2316b747b28e109c9327 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 26 Jan 2016 17:47:46 +0500
Subject: [PATCH 0785/2974] Enable robots.txt handling by default for new
 projects. Fixes GH-1668.

For backwards compatibility reasons the default value is not changed.
---
 docs/topics/settings.rst                         | 14 ++++++++++----
 scrapy/templates/project/module/settings.py.tmpl |  3 +++
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index cc070d8c0..0959a87a7 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -750,8 +750,8 @@ Default: ``60.0``
 Scope: ``scrapy.extensions.memusage``
 
 The :ref:`Memory usage extension <topics-extensions-ref-memusage>`
-checks the current memory usage, versus the limits set by 
-:setting:`MEMUSAGE_LIMIT_MB` and :setting:`MEMUSAGE_WARNING_MB`, 
+checks the current memory usage, versus the limits set by
+:setting:`MEMUSAGE_LIMIT_MB` and :setting:`MEMUSAGE_WARNING_MB`,
 at fixed time intervals.
 
 This sets the length of these intervals, in seconds.
@@ -877,7 +877,13 @@ Default: ``False``
 Scope: ``scrapy.downloadermiddlewares.robotstxt``
 
 If enabled, Scrapy will respect robots.txt policies. For more information see
-:ref:`topics-dlmw-robots`
+:ref:`topics-dlmw-robots`.
+
+.. note::
+
+    While the default value is ``False`` for historical reasons,
+    this option is enabled by default in settings.py file generated
+    by ``scrapy startproject`` command.
 
 .. setting:: SCHEDULER
 
@@ -1036,7 +1042,7 @@ TEMPLATES_DIR
 Default: ``templates`` dir inside scrapy module
 
 The directory where to look for templates when creating new projects with
-:command:`startproject` command and new spiders with :command:`genspider` 
+:command:`startproject` command and new spiders with :command:`genspider`
 command.
 
 The project name must not conflict with the name of custom files or directories
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 822812c9a..f13e85871 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -18,6 +18,9 @@ NEWSPIDER_MODULE = '$project_name.spiders'
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
 #USER_AGENT = '$project_name (+http://www.yourdomain.com)'
 
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = True
+
 # Configure maximum concurrent requests performed by Scrapy (default: 16)
 #CONCURRENT_REQUESTS = 32
 

From 0349bbf9d3691ad89a5e265db4220fb3e64324ff Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 26 Jan 2016 15:25:15 +0100
Subject: [PATCH 0786/2974] Disable SpiderState extension if no JOBDIR set

---
 scrapy/extensions/spiderstate.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 3799c7c66..2220cbd8f 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -2,6 +2,7 @@ import os
 from six.moves import cPickle as pickle
 
 from scrapy import signals
+from scrapy.exceptions import NotConfigured
 from scrapy.utils.job import job_dir
 
 class SpiderState(object):
@@ -12,7 +13,11 @@ class SpiderState(object):
 
     @classmethod
     def from_crawler(cls, crawler):
-        obj = cls(job_dir(crawler.settings))
+        jobdir = job_dir(crawler.settings)
+        if not jobdir:
+            raise NotConfigured
+
+        obj = cls(jobdir)
         crawler.signals.connect(obj.spider_closed, signal=signals.spider_closed)
         crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened)
         return obj

From 29695375d16ae20e3a97dc78bc12662996a9319b Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 26 Jan 2016 16:33:24 +0100
Subject: [PATCH 0787/2974] Add test for raised exception with SpiderState
 extension when no JOBDIR used

---
 tests/test_spiderstate.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index d83015bd9..d1d6debec 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -4,6 +4,8 @@ from twisted.trial import unittest
 
 from scrapy.extensions.spiderstate import SpiderState
 from scrapy.spiders import Spider
+from scrapy.exceptions import NotConfigured
+from scrapy.utils.test import get_crawler
 
 
 class SpiderStateTest(unittest.TestCase):
@@ -34,3 +36,7 @@ class SpiderStateTest(unittest.TestCase):
         ss.spider_opened(spider)
         self.assertEqual(spider.state, {})
         ss.spider_closed(spider)
+
+    def test_not_configured(self):
+        crawler = get_crawler(Spider)
+        self.assertRaises(NotConfigured, SpiderState.from_crawler, crawler)

From c22a4e3bb84448f778613717c3749f226df7880f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 26 Jan 2016 16:41:16 +0100
Subject: [PATCH 0788/2974] Use long classes names for enabled middlewares in
 startup logs

---
 scrapy/middleware.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 6120488e2..be36f977e 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -28,6 +28,7 @@ class MiddlewareManager(object):
     def from_settings(cls, settings, crawler=None):
         mwlist = cls._get_mwlist_from_settings(settings)
         middlewares = []
+        enabled = []
         for clspath in mwlist:
             try:
                 mwcls = load_object(clspath)
@@ -38,6 +39,7 @@ class MiddlewareManager(object):
                 else:
                     mw = mwcls()
                 middlewares.append(mw)
+                enabled.append(clspath)
             except NotConfigured as e:
                 if e.args:
                     clsname = clspath.split('.')[-1]
@@ -45,7 +47,6 @@ class MiddlewareManager(object):
                                    {'clsname': clsname, 'eargs': e.args[0]},
                                    extra={'crawler': crawler})
 
-        enabled = [x.__class__.__name__ for x in middlewares]
         logger.info("Enabled %(componentname)ss:\n%(enabledlist)s",
                     {'componentname': cls.component_name,
                      'enabledlist': pprint.pformat(enabled)},

From bb1f4013a3883d72bcaa14b06a86ab428b99adfa Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 26 Jan 2016 17:23:28 +0100
Subject: [PATCH 0789/2974] Rewrite warning about shell with local files as
 note

---
 docs/topics/shell.rst | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 4af11fbb6..a6ca036d2 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -65,22 +65,24 @@ the following syntaxes for local files::
     # File URI
     scrapy shell file:///absolute/path/to/file.html
 
-.. warning:: :command:`shell` will interpret ``index.html`` as a domain name,
-   not as a relative path to a local file, and will trigger a DNS lookup error::
+.. note:: When using relative file paths, be explicit and prepend them
+    with ``./`` (or ``../`` when relevant).
+    ``scrapy shell index.html`` will not work as one might expect (and
+    this is by design, not a bug).
 
-    $ scrapy shell index.html
-    [ ... scrapy shell starts ... ]
-    2016-01-26 10:29:51 [scrapy] DEBUG: Gave up retrying <GET http://index.html>
-    (failed 3 times): DNS lookup failed:
-    address 'index.html' not found: [Errno -5] No address associated with hostname.
-    [ ... traceback ... ]
-    twisted.internet.error.DNSLookupError: DNS lookup failed:
-    address 'index.html' not found: [Errno -5] No address associated with hostname.
+    Because :command:`shell` favors HTTP URLs over File URIs,
+    and ``index.html`` being syntactically similar to ``example.com``,
+    :command:`shell` will treat ``index.html`` as a domain name and trigger
+    a DNS lookup error::
 
-   Use ``./`` prefix instead::
+        $ scrapy shell index.html
+        [ ... scrapy shell starts ... ]
+        [ ... traceback ... ]
+        twisted.internet.error.DNSLookupError: DNS lookup failed:
+        address 'index.html' not found: [Errno -5] No address associated with hostname.
 
-    $ scrapy shell ./index.html
-    [ ... scrapy shell starts ... ]
+    :command:`shell` will not test beforehand if a file called ``index.html``
+    exists in the current directory. Again, be explicit.
 
 
 Using the shell

From 1c83108893cd3cc05e5b8b16e9c03d4a4786fdd6 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 26 Jan 2016 19:24:11 +0100
Subject: [PATCH 0790/2974] Clarify priority adjust settings docs

Fixes #1593
---
 docs/topics/settings.rst | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0959a87a7..116a10f83 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -864,8 +864,26 @@ REDIRECT_PRIORITY_ADJUST
 
 Default: ``+2``
 
-Adjust redirect request priority relative to original request.
-A negative priority adjust means more priority.
+Scope: ``scrapy.downloadermiddlewares.redirect.RedirectMiddleware``
+
+Adjust redirect request priority relative to original request:
+
+- **a positive priority adjust (default) means higher priority.**
+- a negative priority adjust means lower priority.
+
+.. setting:: RETRY_PRIORITY_ADJUST
+
+RETRY_PRIORITY_ADJUST
+---------------------
+
+Default: ``-1``
+
+Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware``
+
+Adjust retry request priority relative to original request:
+
+- a positive priority adjust means higher priority.
+- **a negative priority adjust (default) means lower priority.**
 
 .. setting:: ROBOTSTXT_OBEY
 

From 4bcbb77bcc7d7668340baa064db15f29617cf0cb Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 27 Jan 2016 01:28:11 +0500
Subject: [PATCH 0791/2974] response.text. Fixes GH-1729.

---
 docs/topics/request-response.rst          | 42 +++++++++++++----------
 scrapy/downloadermiddlewares/ajaxcrawl.py |  2 +-
 scrapy/downloadermiddlewares/robotstxt.py |  4 +--
 scrapy/http/request/form.py               |  4 +--
 scrapy/http/response/text.py              |  5 +++
 scrapy/selector/unified.py                |  2 +-
 scrapy/utils/iterators.py                 |  2 +-
 scrapy/utils/response.py                  |  4 +--
 tests/test_engine.py                      |  5 ++-
 tests/test_http_response.py               | 30 +++++++++-------
 10 files changed, 58 insertions(+), 42 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index ea64d1599..2e92961a9 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -445,10 +445,10 @@ Response objects
 
     .. attribute:: Response.body
 
-        A str containing the body of this Response. Keep in mind that Response.body
-        is always a str. If you want the unicode version use
-        :meth:`TextResponse.body_as_unicode` (only available in
-        :class:`TextResponse` and subclasses).
+        The body of this Response. Keep in mind that Response.body
+        is always a bytes object. If you want the unicode version use
+        :attr:`TextResponse.txt` (only available in :class:`TextResponse`
+        and subclasses).
 
         This attribute is read-only. To change the body of a Response use
         :meth:`replace`.
@@ -542,6 +542,21 @@ TextResponse objects
     :class:`TextResponse` objects support the following attributes in addition
     to the standard :class:`Response` ones:
 
+    .. attribute:: TextResponse.text
+
+       Response body, as unicode.
+
+       The same as ``response.body.decode(response.encoding)``, but the
+       result is cached after the first call, so you can access
+       ``response.text`` multiple times without extra overhead.
+
+       .. note::
+
+            ``unicode(response.body)`` is not a correct way to convert response
+            body to unicode: you would be using the system default encoding
+            (typically `ascii`) instead of the response encoding.
+
+
     .. attribute:: TextResponse.encoding
 
        A string with the encoding of this response. The encoding is resolved by
@@ -568,20 +583,6 @@ TextResponse objects
     :class:`TextResponse` objects support the following methods in addition to
     the standard :class:`Response` ones:
 
-    .. method:: TextResponse.body_as_unicode()
-
-        Returns the body of the response as unicode. This is equivalent to::
-
-            response.body.decode(response.encoding)
-
-        But **not** equivalent to::
-
-            unicode(response.body)
-
-        Since, in the latter case, you would be using the system default encoding
-        (typically `ascii`) to convert the body to unicode, instead of the response
-        encoding.
-
     .. method:: TextResponse.xpath(query)
 
         A shortcut to ``TextResponse.selector.xpath(query)``::
@@ -594,6 +595,11 @@ TextResponse objects
 
             response.css('p')
 
+    .. method:: TextResponse.body_as_unicode()
+
+        The same as :attr:`text`, but available as a method. This method is
+        kept for backwards compatibility; please prefer ``response.text``.
+
 
 HtmlResponse objects
 --------------------
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 6b543b823..da373eca2 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -63,7 +63,7 @@ class AjaxCrawlMiddleware(object):
         Return True if a page without hash fragment could be "AJAX crawlable"
         according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.
         """
-        body = response.body_as_unicode()[:self.lookup_bytes]
+        body = response.text[:self.lookup_bytes]
         return _has_ajaxcrawlable_meta(body)
 
 
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index c061c2407..d4a33dc36 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -83,8 +83,8 @@ class RobotsTxtMiddleware(object):
     def _parse_robots(self, response, netloc):
         rp = robotparser.RobotFileParser(response.url)
         body = ''
-        if hasattr(response, 'body_as_unicode'):
-            body = response.body_as_unicode()
+        if hasattr(response, 'text'):
+            body = response.text
         else: # last effort try
             try:
                 body = response.body.decode('utf-8')
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 5501634d3..2862dc096 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -64,8 +64,8 @@ def _urlencode(seq, enc):
 
 def _get_form(response, formname, formid, formnumber, formxpath):
     """Find the form element """
-    text = response.body_as_unicode()
-    root = create_root_node(text, lxml.html.HTMLParser, base_url=get_base_url(response))
+    root = create_root_node(response.text, lxml.html.HTMLParser,
+                            base_url=get_base_url(response))
     forms = root.xpath('//form')
     if not forms:
         raise ValueError("No <form> element found in %s" % response)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 1c416bf82..9c667ab7e 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -67,6 +67,11 @@ class TextResponse(Response):
             self._cached_ubody = html_to_unicode(charset, self.body)[1]
         return self._cached_ubody
 
+    @property
+    def text(self):
+        """ Body as unicode """
+        return self.body_as_unicode()
+
     def urljoin(self, url):
         """Join this Response's url with a possible relative url to form an
         absolute interpretation of the latter."""
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 5d77f7624..15f3d26df 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -60,7 +60,7 @@ class Selector(_ParselSelector, object_ref):
             response = _response_from_text(text, st)
 
         if response is not None:
-            text = response.body_as_unicode()
+            text = response.text
             kwargs.setdefault('base_url', response.url)
 
         self.response = response
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index b0688791e..73857b410 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True):
         if not unicode:
             return obj.body
         elif isinstance(obj, TextResponse):
-            return obj.body_as_unicode()
+            return obj.text
         else:
             return obj.body.decode('utf-8')
     elif isinstance(obj, six.text_type):
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index c4ad52f14..73db2641e 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -25,7 +25,7 @@ _baseurl_cache = weakref.WeakKeyDictionary()
 def get_base_url(response):
     """Return the base url of the given response, joined with the response url"""
     if response not in _baseurl_cache:
-        text = response.body_as_unicode()[0:4096]
+        text = response.text[0:4096]
         _baseurl_cache[response] = html.get_base_url(text, response.url,
             response.encoding)
     return _baseurl_cache[response]
@@ -37,7 +37,7 @@ _metaref_cache = weakref.WeakKeyDictionary()
 def get_meta_refresh(response):
     """Parse the http-equiv refrsh parameter from the given response"""
     if response not in _metaref_cache:
-        text = response.body_as_unicode()[0:4096]
+        text = response.text[0:4096]
         text = _noscript_re.sub(u'', text)
         text = _script_re.sub(u'', text)
         _metaref_cache[response] = html.get_meta_refresh(text, response.url,
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 9f2c02bff..baf6ef1bf 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -55,12 +55,11 @@ class TestSpider(Spider):
 
     def parse_item(self, response):
         item = self.item_cls()
-        body = response.body_as_unicode()
-        m = self.name_re.search(body)
+        m = self.name_re.search(response.text)
         if m:
             item['name'] = m.group(1)
         item['url'] = response.url
-        m = self.price_re.search(body)
+        m = self.price_re.search(response.text)
         if m:
             item['price'] = m.group(1)
         return item
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 710a5b29d..c7f36687a 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -107,9 +107,11 @@ class BaseResponseTest(unittest.TestCase):
             body_bytes = body
 
         assert isinstance(response.body, bytes)
+        assert isinstance(response.text, six.text_type)
         self._assert_response_encoding(response, encoding)
         self.assertEqual(response.body, body_bytes)
         self.assertEqual(response.body_as_unicode(), body_unicode)
+        self.assertEqual(response.text, body_unicode)
 
     def _assert_response_encoding(self, response, encoding):
         self.assertEqual(response.encoding, resolve_encoding(encoding))
@@ -171,6 +173,10 @@ class TextResponseTest(BaseResponseTest):
         self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type))
         self.assertEqual(r1.body_as_unicode(), unicode_string)
 
+        # check response.text
+        self.assertTrue(isinstance(r1.text, six.text_type))
+        self.assertEqual(r1.text, unicode_string)
+
     def test_encoding(self):
         r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xc2\xa3")
         r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
@@ -219,12 +225,12 @@ class TextResponseTest(BaseResponseTest):
                                  headers={"Content-type": ["text/html; charset=utf-8"]},
                                  body=b"\xef\xbb\xbfWORD\xe3\xab")
         self.assertEqual(r6.encoding, 'utf-8')
-        self.assertEqual(r6.body_as_unicode(), u'WORD\ufffd\ufffd')
+        self.assertEqual(r6.text, u'WORD\ufffd\ufffd')
 
     def test_bom_is_removed_from_body(self):
         # Inferring encoding from body also cache decoded body as sideeffect,
         # this test tries to ensure that calling response.encoding and
-        # response.body_as_unicode() in indistint order doesn't affect final
+        # response.text in indistint order doesn't affect final
         # values for encoding and decoded body.
         url = 'http://example.com'
         body = b"\xef\xbb\xbfWORD"
@@ -233,9 +239,9 @@ class TextResponseTest(BaseResponseTest):
         # Test response without content-type and BOM encoding
         response = self.response_class(url, body=body)
         self.assertEqual(response.encoding, 'utf-8')
-        self.assertEqual(response.body_as_unicode(), u'WORD')
+        self.assertEqual(response.text, u'WORD')
         response = self.response_class(url, body=body)
-        self.assertEqual(response.body_as_unicode(), u'WORD')
+        self.assertEqual(response.text, u'WORD')
         self.assertEqual(response.encoding, 'utf-8')
 
         # Body caching sideeffect isn't triggered when encoding is declared in
@@ -243,9 +249,9 @@ class TextResponseTest(BaseResponseTest):
         # body
         response = self.response_class(url, headers=headers, body=body)
         self.assertEqual(response.encoding, 'utf-8')
-        self.assertEqual(response.body_as_unicode(), u'WORD')
+        self.assertEqual(response.text, u'WORD')
         response = self.response_class(url, headers=headers, body=body)
-        self.assertEqual(response.body_as_unicode(), u'WORD')
+        self.assertEqual(response.text, u'WORD')
         self.assertEqual(response.encoding, 'utf-8')
 
     def test_replace_wrong_encoding(self):
@@ -253,18 +259,18 @@ class TextResponseTest(BaseResponseTest):
         r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX')
         # XXX: Policy for replacing invalid chars may suffer minor variations
         # but it should always contain the unicode replacement char (u'\ufffd')
-        assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode())
-        assert u'PREFIX' in r.body_as_unicode(), repr(r.body_as_unicode())
-        assert u'SUFFIX' in r.body_as_unicode(), repr(r.body_as_unicode())
+        assert u'\ufffd' in r.text, repr(r.text)
+        assert u'PREFIX' in r.text, repr(r.text)
+        assert u'SUFFIX' in r.text, repr(r.text)
 
         # Do not destroy html tags due to encoding bugs
         r = self.response_class("http://example.com", encoding='utf-8', \
                 body=b'\xf0<span>value</span>')
-        assert u'<span>value</span>' in r.body_as_unicode(), repr(r.body_as_unicode())
+        assert u'<span>value</span>' in r.text, repr(r.text)
 
         # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
-        #r = self.response_class("http://www.example.com", body='PREFIX\xe3\xabSUFFIX')
-        #assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode())
+        #r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
+        #assert u'\ufffd' in r.text, repr(r.text)
 
     def test_selector(self):
         body = b"<html><head><title>Some page</title><body></body></html>"

From 6ed08d23329bc33142804ea78320993d0c680175 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 27 Jan 2016 11:53:29 +0100
Subject: [PATCH 0792/2974] Add note for DEPTH_PRIORITY

---
 docs/topics/settings.rst | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 116a10f83..052be4429 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -276,6 +276,8 @@ DEPTH_LIMIT
 
 Default: ``0``
 
+Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
+
 The maximum depth that will be allowed to crawl for any site. If zero, no limit
 will be imposed.
 
@@ -286,9 +288,20 @@ DEPTH_PRIORITY
 
 Default: ``0``
 
-An integer that is used to adjust the request priority based on its depth.
+Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
 
-If zero, no priority adjustment is made from depth.
+An integer that is used to adjust the request priority based on its depth:
+
+- **a positive value will decrease the priority**
+- a negative value will increase priority
+
+If zero (default), no priority adjustment is made from depth.
+
+.. note::
+
+    This setting adjusts priority **in the opposite way** compared to
+    other priority settings :setting:`REDIRECT_PRIORITY_ADJUST`
+    and :setting:`RETRY_PRIORITY_ADJUST`.
 
 .. setting:: DEPTH_STATS
 
@@ -297,6 +310,8 @@ DEPTH_STATS
 
 Default: ``True``
 
+Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
+
 Whether to collect maximum depth stats.
 
 .. setting:: DEPTH_STATS_VERBOSE
@@ -306,6 +321,8 @@ DEPTH_STATS_VERBOSE
 
 Default: ``False``
 
+Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
+
 Whether to collect verbose depth stats. If this is enabled, the number of
 requests for each depth is collected in the stats.
 

From d999e3f7a704c5009999573d9514a5a51bd8be13 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 27 Jan 2016 12:57:03 +0100
Subject: [PATCH 0793/2974] More explicit description of DEPTH_PRIORITY

---
 docs/faq.rst             |  4 +++-
 docs/topics/settings.rst | 10 +++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 3d2bd8d4d..b3412211a 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -45,7 +45,7 @@ Did Scrapy "steal" X from Django?
 
 Probably, but we don't like that word. We think Django_ is a great open source
 project and an example to follow, so we've used it as an inspiration for
-Scrapy. 
+Scrapy.
 
 We believe that, if something is already done well, there's no need to reinvent
 it. This concept, besides being one of the foundations for open source and free
@@ -85,6 +85,8 @@ How can I simulate a user login in my spider?
 
 See :ref:`topics-request-response-ref-request-userlogin`.
 
+.. _faq-bfo-dfo:
+
 Does Scrapy crawl in breadth-first or depth-first order?
 --------------------------------------------------------
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 052be4429..725345f2a 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -292,10 +292,14 @@ Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
 
 An integer that is used to adjust the request priority based on its depth:
 
-- **a positive value will decrease the priority**
-- a negative value will increase priority
+- if zero (default), no priority adjustment is made from depth
+- **a positive value will decrease the priority, i.e. higher depth
+  requests will be processed later** ; this is commonly used when doing
+  breadth-first crawls (BFO)
+- a negative value will increase priority, i.e., higher depth requests
+  will be processed sooner (DFO)
 
-If zero (default), no priority adjustment is made from depth.
+See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO.
 
 .. note::
 

From e0f48c486e4e658e80a4fea22f0fee58c3f01a0c Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 27 Jan 2016 13:04:08 +0100
Subject: [PATCH 0794/2974] Add link to CoC mardown file on Github

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 8a7d2c71d..3e050bb1e 100644
--- a/README.rst
+++ b/README.rst
@@ -74,7 +74,7 @@ Contributing
 ============
 
 Please note that this project is released with a Contributor Code of Conduct
-(see CODE_OF_CONDUCT.md).
+(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
 
 By participating in this project you agree to abide by its terms.
 Please report unacceptable behavior to opensource@scrapinghub.com.

From 7ca9ae19765d2c49c0e838ebbfc1596d0fbcd7d9 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 27 Jan 2016 17:54:28 +0500
Subject: [PATCH 0795/2974] DOC typo fix

---
 docs/topics/request-response.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 2e92961a9..82e674cee 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -447,7 +447,7 @@ Response objects
 
         The body of this Response. Keep in mind that Response.body
         is always a bytes object. If you want the unicode version use
-        :attr:`TextResponse.txt` (only available in :class:`TextResponse`
+        :attr:`TextResponse.text` (only available in :class:`TextResponse`
         and subclasses).
 
         This attribute is read-only. To change the body of a Response use

From dc8701ea429d4ded2f66d6b7c8fbce0bbcd0041a Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Wed, 27 Jan 2016 12:56:42 -0200
Subject: [PATCH 0796/2974] Add test for already failed deferreds when
 downloading page in robots.txt middleware.

---
 tests/test_downloadermiddleware_robotstxt.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 5f45dcb82..f2e94e171 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -123,6 +123,18 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called))
         return deferred
 
+    def test_robotstxt_immediate_error(self):
+        self.crawler.settings.set('ROBOTSTXT_OBEY', True)
+        err = error.DNSLookupError('Robotstxt address not found')
+        def immediate_failure(request, spider):
+            deferred = Deferred()
+            deferred.errback(failure.Failure(err))
+            return deferred
+        self.crawler.engine.download.side_effect = immediate_failure
+
+        middleware = RobotsTxtMiddleware(self.crawler)
+        return self.assertNotIgnored(Request('http://site.local'), middleware)
+
     def test_ignore_robotstxt_request(self):
         self.crawler.settings.set('ROBOTSTXT_OBEY', True)
         def ignore_request(request, spider):

From b2beb3e85d2e82977d259eea71402809d00d197e Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Wed, 27 Jan 2016 13:09:08 -0200
Subject: [PATCH 0797/2974] Fix handling of already failed deferreds when
 downloading page in robots.txt middleware.

---
 scrapy/downloadermiddlewares/robotstxt.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index c061c2407..7f6f0d012 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -8,7 +8,9 @@ import logging
 
 from six.moves.urllib import robotparser
 
+from twisted.internet import reactor
 from twisted.internet.defer import Deferred, maybeDeferred
+from twisted.internet.task import deferLater
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
@@ -57,7 +59,13 @@ class RobotsTxtMiddleware(object):
                 priority=self.DOWNLOAD_PRIORITY,
                 meta={'dont_obey_robotstxt': True}
             )
-            dfd = self.crawler.engine.download(robotsreq, spider)
+            # engine.download() can return an already-called deferred, e.g. if a
+            # middleware returns a response in process_request(). Using
+            # deferLater() ensures that the error callback isn't called
+            # immediately upon being added, so that it doesn't remove the key
+            # before we check for it.
+            dfd = deferLater(reactor, 0, self.crawler.engine.download,
+                             robotsreq, spider)
             dfd.addCallback(self._parse_robots, netloc)
             dfd.addErrback(self._logerror, robotsreq, spider)
             dfd.addErrback(self._robots_error, netloc)

From f1d971a5c0cdfe0f4fe5619146cd6818324fc98e Mon Sep 17 00:00:00 2001
From: stummjr <stummjr@gmail.com>
Date: Wed, 27 Jan 2016 14:34:46 -0200
Subject: [PATCH 0798/2974] fix PythonItemExporter for non-string types

---
 scrapy/exporters.py     |  8 ++++----
 tests/test_exporters.py | 13 +++++++++++++
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 145468dbe..c7c78d054 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -273,10 +273,10 @@ class PythonItemExporter(BaseItemExporter):
             return dict(self._serialize_dict(value))
         if is_listlike(value):
             return [self._serialize_value(v) for v in value]
-        if self.binary:
-            return to_bytes(value, encoding=self.encoding)
-        else:
-            return to_unicode(value, encoding=self.encoding)
+        encode_func = to_bytes if self.binary else to_unicode
+        if isinstance(value, (six.text_type, bytes)):
+            return encode_func(value, encoding=self.encoding)
+        return value
 
     def _serialize_dict(self, value):
         for key, val in six.iteritems(value):
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 1633e1039..662f8ec5c 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -134,6 +134,19 @@ class PythonItemExporterTest(BaseItemExporterTest):
         expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
         self.assertEqual(expected, exporter.export_item(value))
 
+    def test_other_python_types_item(self):
+        from datetime import datetime
+        now = datetime.now()
+        item = {
+            'boolean': False,
+            'number': 22,
+            'time': now,
+            'float': 3.14,
+        }
+        ie = self._get_exporter()
+        exported = ie.export_item(item)
+        self.assertEqual(exported, item)
+
 
 class PprintItemExporterTest(BaseItemExporterTest):
 

From c55ff110a34d39be27bbd3d03fbf52caa271b4c9 Mon Sep 17 00:00:00 2001
From: stummjr <stummjr@gmail.com>
Date: Wed, 27 Jan 2016 15:43:17 -0200
Subject: [PATCH 0799/2974] Fix CSV exporter for non string Python types.

---
 scrapy/exporters.py     |  2 +-
 tests/test_exporters.py | 15 +++++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index c7c78d054..55d74332b 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -200,7 +200,7 @@ class CsvItemExporter(BaseItemExporter):
             try:
                 yield to_native_str(s)
             except TypeError:
-                yield to_native_str(repr(s))
+                yield to_native_str(str(s))
 
     def _write_headers_and_set_fields_to_export(self, item):
         if self.include_headers_line:
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 662f8ec5c..97c09a495 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -271,6 +271,21 @@ class CsvItemExporterTest(BaseItemExporterTest):
             expected='"[4, 8]",John\r\n',
         )
 
+    def test_other_python_types_item(self):
+        from datetime import datetime
+        now = datetime(2015, 1, 1, 1, 1, 1)
+        item = {
+            'boolean': False,
+            'number': 22,
+            'time': now,
+            'float': 3.14,
+        }
+        self.assertExportResult(
+            item=item,
+            include_headers_line=False,
+            expected='22,False,3.14,2015-01-01 01:01:01\r\n'
+        )
+
 
 class XmlItemExporterTest(BaseItemExporterTest):
 

From 27758f60ada4791c044bfe8bc86d267aa930c744 Mon Sep 17 00:00:00 2001
From: stummjr <stummjr@gmail.com>
Date: Wed, 27 Jan 2016 16:28:01 -0200
Subject: [PATCH 0800/2974] Changes fallback for CSVItemExporter, avoiding to
 call to_native_str(str()).

---
 scrapy/exporters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 55d74332b..35f50838b 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -200,7 +200,7 @@ class CsvItemExporter(BaseItemExporter):
             try:
                 yield to_native_str(s)
             except TypeError:
-                yield to_native_str(str(s))
+                yield s
 
     def _write_headers_and_set_fields_to_export(self, item):
         if self.include_headers_line:

From 3e080c3c52720535519ba1be7dee472258b19647 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 28 Jan 2016 00:59:27 +0500
Subject: [PATCH 0801/2974] call .text from .body_as_unicode() and not the
 other way around

---
 scrapy/http/response/text.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 9c667ab7e..afa430329 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -59,7 +59,12 @@ class TextResponse(Response):
 
     def body_as_unicode(self):
         """Return body as unicode"""
-        # check for self.encoding before _cached_ubody just in
+        return self.text
+
+    @property
+    def text(self):
+        """ Body as unicode """
+        # access self.encoding before _cached_ubody to make sure
         # _body_inferred_encoding is called
         benc = self.encoding
         if self._cached_ubody is None:
@@ -67,11 +72,6 @@ class TextResponse(Response):
             self._cached_ubody = html_to_unicode(charset, self.body)[1]
         return self._cached_ubody
 
-    @property
-    def text(self):
-        """ Body as unicode """
-        return self.body_as_unicode()
-
     def urljoin(self, url):
         """Join this Response's url with a possible relative url to form an
         absolute interpretation of the latter."""

From 90e3ae1c580875e4e68c9d7238d0fb4642306bf9 Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Wed, 27 Jan 2016 21:00:35 -0200
Subject: [PATCH 0802/2974] Do not forget failed requests in robots.txt
 middleware.

---
 scrapy/downloadermiddlewares/robotstxt.py | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 7f6f0d012..6fdba90cc 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -8,9 +8,7 @@ import logging
 
 from six.moves.urllib import robotparser
 
-from twisted.internet import reactor
 from twisted.internet.defer import Deferred, maybeDeferred
-from twisted.internet.task import deferLater
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
@@ -59,13 +57,7 @@ class RobotsTxtMiddleware(object):
                 priority=self.DOWNLOAD_PRIORITY,
                 meta={'dont_obey_robotstxt': True}
             )
-            # engine.download() can return an already-called deferred, e.g. if a
-            # middleware returns a response in process_request(). Using
-            # deferLater() ensures that the error callback isn't called
-            # immediately upon being added, so that it doesn't remove the key
-            # before we check for it.
-            dfd = deferLater(reactor, 0, self.crawler.engine.download,
-                             robotsreq, spider)
+            dfd = self.crawler.engine.download(robotsreq, spider)
             dfd.addCallback(self._parse_robots, netloc)
             dfd.addErrback(self._logerror, robotsreq, spider)
             dfd.addErrback(self._robots_error, netloc)
@@ -109,4 +101,6 @@ class RobotsTxtMiddleware(object):
         rp_dfd.callback(rp)
 
     def _robots_error(self, failure, netloc):
-        self._parsers.pop(netloc).callback(None)
+        rp_dfd = self._parsers[netloc]
+        self._parsers[netloc] = None
+        rp_dfd.callback(None)

From cae268402d13a6d419c56024b3051b1cad3d82e1 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 28 Jan 2016 13:42:04 +0100
Subject: [PATCH 0803/2974] Move guess_scheme() to scrapy.utils.url

---
 scrapy/commands/shell.py    | 29 +----------------------------
 scrapy/utils/url.py         | 24 ++++++++++++++++++++++++
 tests/test_command_shell.py |  2 +-
 3 files changed, 26 insertions(+), 29 deletions(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 5201feb42..7be7f7256 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -3,40 +3,13 @@ Scrapy Shell
 
 See documentation in docs/topics/shell.rst
 """
-import re
-from six.moves.urllib.parse import urlparse
 from threading import Thread
-from w3lib.url import any_to_uri
 
 from scrapy.commands import ScrapyCommand
 from scrapy.shell import Shell
 from scrapy.http import Request
-from scrapy.utils.url import add_http_if_no_scheme
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
-
-
-def guess_scheme(url):
-    """Given an URL as string,
-    returns a FileURI if it looks like a file path,
-    otherwise returns an HTTP URL
-    """
-    parts = urlparse(url)
-    if parts.scheme:
-        return url
-    # Note: this does not match Windows filepath
-    if re.match(r'''^                   # start with...
-                    (
-                        \.              # ...a single dot,
-                        (
-                            \. | [^/\.]+  # optionally followed by
-                        )?                # either a second dot or some characters
-                    )?      # optional match of ".", ".." or ".blabla"
-                    /       # at least one "/" for a file path,
-                    .       # and something after the "/"
-                    ''', parts.path, flags=re.VERBOSE):
-        return any_to_uri(url)
-    else:
-        return add_http_if_no_scheme(url)
+from scrapy.utils.url import guess_scheme
 
 
 class Command(ScrapyCommand):
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 0acbbb6ab..4b47566e5 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -122,3 +122,27 @@ def add_http_if_no_scheme(url):
         url = scheme + url
 
     return url
+
+
+def guess_scheme(url):
+    """Given an URL as string,
+    returns a FileURI if it looks like a file path,
+    otherwise returns an HTTP URL
+    """
+    parts = urlparse(url)
+    if parts.scheme:
+        return url
+    # Note: this does not match Windows filepath
+    if re.match(r'''^                   # start with...
+                    (
+                        \.              # ...a single dot,
+                        (
+                            \. | [^/\.]+  # optionally followed by
+                        )?                # either a second dot or some characters
+                    )?      # optional match of ".", ".." or ".blabla"
+                    /       # at least one "/" for a file path,
+                    .       # and something after the "/"
+                    ''', parts.path, flags=re.VERBOSE):
+        return any_to_uri(url)
+    else:
+        return add_http_if_no_scheme(url)
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index a61d520fa..35a5fa21a 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -3,9 +3,9 @@ from os.path import join
 from twisted.trial import unittest
 from twisted.internet import defer
 
-from scrapy.commands.shell import guess_scheme
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
+from scrapy.utils.url import guess_scheme
 
 from tests import tests_datadir
 

From 481e251775a089a8e82c480a83181146d1bb6847 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 28 Jan 2016 13:51:50 +0100
Subject: [PATCH 0804/2974] Move guess_scheme() tests to relevant test module

---
 tests/test_command_shell.py | 58 ---------------------------------
 tests/test_utils_url.py     | 65 +++++++++++++++++++++++++++++++++++--
 2 files changed, 62 insertions(+), 61 deletions(-)

diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 35a5fa21a..9d0965902 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -10,64 +10,6 @@ from scrapy.utils.url import guess_scheme
 from tests import tests_datadir
 
 
-class ShellURLTest(unittest.TestCase):
-    pass
-
-def create_guess_scheme_t(args):
-    def do_expected(self):
-        url = guess_scheme(args[0])
-        assert url.startswith(args[1]), \
-            'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (
-                args[0], url, args[1])
-    return do_expected
-
-def create_skipped_scheme_t(args):
-    def do_expected(self):
-        raise unittest.SkipTest(args[2])
-        url = guess_scheme(args[0])
-        assert url.startswith(args[1])
-    return do_expected
-
-for k, args in enumerate ([
-            ('/index',                              'file://'),
-            ('/index.html',                         'file://'),
-            ('./index.html',                        'file://'),
-            ('../index.html',                       'file://'),
-            ('../../index.html',                    'file://'),
-            ('./data/index.html',                   'file://'),
-            ('.hidden/data/index.html',             'file://'),
-            ('/home/user/www/index.html',           'file://'),
-            ('//home/user/www/index.html',          'file://'),
-            ('file:///home/user/www/index.html',    'file://'),
-
-            ('index.html',                          'http://'),
-            ('example.com',                         'http://'),
-            ('www.example.com',                     'http://'),
-            ('www.example.com/index.html',          'http://'),
-            ('http://example.com',                  'http://'),
-            ('http://example.com/index.html',       'http://'),
-            ('localhost',                           'http://'),
-            ('localhost/index.html',                'http://'),
-
-            # some corner cases (default to http://)
-            ('/',                                   'http://'),
-            ('.../test',                            'http://'),
-
-        ], start=1):
-    t_method = create_guess_scheme_t(args)
-    t_method.__name__ = 'test_uri_%03d' % k
-    setattr (ShellURLTest, t_method.__name__, t_method)
-
-# TODO: the following tests do not pass with current implementation
-for k, args in enumerate ([
-            ('C:\absolute\path\to\a\file.html',     'file://',
-             'Windows filepath are not supported for scrapy shell'),
-        ], start=1):
-    t_method = create_skipped_scheme_t(args)
-    t_method.__name__ = 'test_uri_skipped_%03d' % k
-    setattr (ShellURLTest, t_method.__name__, t_method)
-
-
 class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     command = 'shell'
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 314ccd30f..73ad11f8a 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -4,7 +4,8 @@ import unittest
 import six
 from scrapy.spiders import Spider
 from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
-                              canonicalize_url, add_http_if_no_scheme)
+                              canonicalize_url, add_http_if_no_scheme,
+                              guess_scheme)
 
 __doctests__ = ['scrapy.utils.url']
 
@@ -188,7 +189,7 @@ class CanonicalizeUrlTest(unittest.TestCase):
 
 
 class AddHttpIfNoScheme(unittest.TestCase):
-    
+
     def test_add_scheme(self):
         self.assertEqual(add_http_if_no_scheme('www.example.com'),
                                                'http://www.example.com')
@@ -216,7 +217,7 @@ class AddHttpIfNoScheme(unittest.TestCase):
     def test_username_password(self):
         self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'),
                                                'http://username:password@www.example.com')
-    
+
     def test_complete_url(self):
         self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
                                                'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
@@ -294,5 +295,63 @@ class AddHttpIfNoScheme(unittest.TestCase):
                                                'ftp://www.example.com')
 
 
+class GuessSchemeTest(unittest.TestCase):
+    pass
+
+def create_guess_scheme_t(args):
+    def do_expected(self):
+        url = guess_scheme(args[0])
+        assert url.startswith(args[1]), \
+            'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (
+                args[0], url, args[1])
+    return do_expected
+
+def create_skipped_scheme_t(args):
+    def do_expected(self):
+        raise unittest.SkipTest(args[2])
+        url = guess_scheme(args[0])
+        assert url.startswith(args[1])
+    return do_expected
+
+for k, args in enumerate ([
+            ('/index',                              'file://'),
+            ('/index.html',                         'file://'),
+            ('./index.html',                        'file://'),
+            ('../index.html',                       'file://'),
+            ('../../index.html',                    'file://'),
+            ('./data/index.html',                   'file://'),
+            ('.hidden/data/index.html',             'file://'),
+            ('/home/user/www/index.html',           'file://'),
+            ('//home/user/www/index.html',          'file://'),
+            ('file:///home/user/www/index.html',    'file://'),
+
+            ('index.html',                          'http://'),
+            ('example.com',                         'http://'),
+            ('www.example.com',                     'http://'),
+            ('www.example.com/index.html',          'http://'),
+            ('http://example.com',                  'http://'),
+            ('http://example.com/index.html',       'http://'),
+            ('localhost',                           'http://'),
+            ('localhost/index.html',                'http://'),
+
+            # some corner cases (default to http://)
+            ('/',                                   'http://'),
+            ('.../test',                            'http://'),
+
+        ], start=1):
+    t_method = create_guess_scheme_t(args)
+    t_method.__name__ = 'test_uri_%03d' % k
+    setattr (GuessSchemeTest, t_method.__name__, t_method)
+
+# TODO: the following tests do not pass with current implementation
+for k, args in enumerate ([
+            ('C:\absolute\path\to\a\file.html',     'file://',
+             'Windows filepath are not supported for scrapy shell'),
+        ], start=1):
+    t_method = create_skipped_scheme_t(args)
+    t_method.__name__ = 'test_uri_skipped_%03d' % k
+    setattr (GuessSchemeTest, t_method.__name__, t_method)
+
+
 if __name__ == "__main__":
     unittest.main()

From e9f6b98816c220129fb02e6c09f488ecc8d686bd Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 28 Jan 2016 14:39:19 +0100
Subject: [PATCH 0805/2974] Amend guess_scheme() docstring

---
 scrapy/utils/url.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 4b47566e5..adef4a800 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -125,10 +125,7 @@ def add_http_if_no_scheme(url):
 
 
 def guess_scheme(url):
-    """Given an URL as string,
-    returns a FileURI if it looks like a file path,
-    otherwise returns an HTTP URL
-    """
+    """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise."""
     parts = urlparse(url)
     if parts.scheme:
         return url

From cf2ebb0687b9198f017cab836806bae5a5f3009c Mon Sep 17 00:00:00 2001
From: stummjr <stummjr@gmail.com>
Date: Wed, 27 Jan 2016 19:36:16 -0200
Subject: [PATCH 0806/2974] Include tests for exporters: JSON, JSON-Lines,
 Pickle and Marshal.

---
 tests/test_exporters.py | 70 ++++++++++++++++++++++++++++++-----------
 1 file changed, 51 insertions(+), 19 deletions(-)

diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 97c09a495..7ba5a0af6 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -5,6 +5,7 @@ import marshal
 import tempfile
 import unittest
 from io import BytesIO
+from datetime import datetime
 from six.moves import cPickle as pickle
 
 import lxml.etree
@@ -42,6 +43,14 @@ class BaseItemExporterTest(unittest.TestCase):
             exported_dict[k] = to_unicode(v)
         self.assertEqual(self.i, exported_dict)
 
+    def _get_nonstring_types_item(self):
+        return {
+            'boolean': False,
+            'number': 22,
+            'time': datetime(2015, 1, 1, 1, 1, 1),
+            'float': 3.14,
+        }
+
     def assertItemExportWorks(self, item):
         self.ie.start_exporting()
         try:
@@ -134,15 +143,8 @@ class PythonItemExporterTest(BaseItemExporterTest):
         expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
         self.assertEqual(expected, exporter.export_item(value))
 
-    def test_other_python_types_item(self):
-        from datetime import datetime
-        now = datetime.now()
-        item = {
-            'boolean': False,
-            'number': 22,
-            'time': now,
-            'float': 3.14,
-        }
+    def test_nonstring_types_item(self):
+        item = self._get_nonstring_types_item()
         ie = self._get_exporter()
         exported = ie.export_item(item)
         self.assertEqual(exported, item)
@@ -178,6 +180,15 @@ class PickleItemExporterTest(BaseItemExporterTest):
         self.assertEqual(pickle.load(f), i1)
         self.assertEqual(pickle.load(f), i2)
 
+    def test_nonstring_types_item(self):
+        item = self._get_nonstring_types_item()
+        fp = BytesIO()
+        ie = PickleItemExporter(fp)
+        ie.start_exporting()
+        ie.export_item(item)
+        ie.finish_exporting()
+        self.assertEqual(pickle.loads(fp.getvalue()), item)
+
 
 class MarshalItemExporterTest(BaseItemExporterTest):
 
@@ -189,6 +200,17 @@ class MarshalItemExporterTest(BaseItemExporterTest):
         self.output.seek(0)
         self._assert_expected_item(marshal.load(self.output))
 
+    def test_nonstring_types_item(self):
+        item = self._get_nonstring_types_item()
+        item.pop('time')  # datetime is not marshallable
+        fp = tempfile.TemporaryFile()
+        ie = MarshalItemExporter(fp)
+        ie.start_exporting()
+        ie.export_item(item)
+        ie.finish_exporting()
+        fp.seek(0)
+        self.assertEqual(marshal.load(fp), item)
+
 
 class CsvItemExporterTest(BaseItemExporterTest):
     def _get_exporter(self, **kwargs):
@@ -271,17 +293,9 @@ class CsvItemExporterTest(BaseItemExporterTest):
             expected='"[4, 8]",John\r\n',
         )
 
-    def test_other_python_types_item(self):
-        from datetime import datetime
-        now = datetime(2015, 1, 1, 1, 1, 1)
-        item = {
-            'boolean': False,
-            'number': 22,
-            'time': now,
-            'float': 3.14,
-        }
+    def test_nonstring_types_item(self):
         self.assertExportResult(
-            item=item,
+            item=self._get_nonstring_types_item(),
             include_headers_line=False,
             expected='22,False,3.14,2015-01-01 01:01:01\r\n'
         )
@@ -390,6 +404,15 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         self._check_output()
         self.assertRaises(TypeError, self._get_exporter, foo_unknown_keyword_bar=True)
 
+    def test_nonstring_types_item(self):
+        item = self._get_nonstring_types_item()
+        self.ie.start_exporting()
+        self.ie.export_item(item)
+        self.ie.finish_exporting()
+        exported = json.loads(to_unicode(self.output.getvalue()))
+        item['time'] = str(item['time'])
+        self.assertEqual(exported, item)
+
 
 class JsonItemExporterTest(JsonLinesItemExporterTest):
 
@@ -438,6 +461,15 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}}
         self.assertEqual(exported, [expected])
 
+    def test_nonstring_types_item(self):
+        item = self._get_nonstring_types_item()
+        self.ie.start_exporting()
+        self.ie.export_item(item)
+        self.ie.finish_exporting()
+        exported = json.loads(to_unicode(self.output.getvalue()))
+        item['time'] = str(item['time'])
+        self.assertEqual(exported, [item])
+
 
 class CustomItemExporterTest(unittest.TestCase):
 

From 78f00401cd284fd7bdcbb525ee94ea0bbedaa7cd Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 29 Jan 2016 16:56:05 +0100
Subject: [PATCH 0807/2974] Remove unused import in tests

---
 tests/test_command_shell.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 9d0965902..c532fc0d8 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -5,7 +5,6 @@ from twisted.internet import defer
 
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
-from scrapy.utils.url import guess_scheme
 
 from tests import tests_datadir
 

From a1ebff83d39e65cf4ed34281a1e81ea6cd108fe0 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 29 Jan 2016 18:39:34 +0100
Subject: [PATCH 0808/2974] Remove __str__ and __repr__ from settings,
 introduce copy_to_dict() instead

Settings instances as dict's are easier to print or pretty print in the shell

Fixes #1732
---
 scrapy/settings/__init__.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 342d2585e..918bfc1e5 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -368,11 +368,25 @@ class BaseSettings(MutableMapping):
     def __len__(self):
         return len(self.attributes)
 
-    def __str__(self):
-        return str(self.attributes)
+    def _to_dict(self):
+        return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
+                for k, v in six.iteritems(self)}
 
-    def __repr__(self):
-        return "<%s %s>" % (self.__class__.__name__, self.attributes)
+    def copy_to_dict(self):
+        """
+        Make a copy of current settings and convert to a dict.
+
+        This method returns a new dict populated with the same values
+        and their priorities as the current settings.
+
+        Modifications to the returned dict won't be reflected on the original
+        settings.
+
+        This method can be useful for example for printing settings
+        in Scrapy shell.
+        """
+        settings = self.copy()
+        return settings._to_dict()
 
     @property
     def overrides(self):

From aa78758bc744b3264c330cc18e58a8b15315517d Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 29 Jan 2016 18:59:12 +0100
Subject: [PATCH 0809/2974] Update tests for settings copy_to_dict()

---
 tests/test_settings/__init__.py | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 44b9b6df3..4acf22cba 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -302,6 +302,21 @@ class BaseSettingsTest(unittest.TestCase):
         self.assertListEqual(copy.get('TEST_LIST_OF_LISTS')[0],
                              ['first_one', 'first_two'])
 
+    def test_copy_to_dict(self):
+        s = BaseSettings({'TEST_STRING': 'a string',
+                          'TEST_LIST': [1, 2],
+                          'TEST_BOOLEAN': False,
+                          'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
+                          'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
+                          'HASNOBASE': BaseSettings({3: 3000}, 'default')})
+        self.assertDictEqual(s.copy_to_dict(),
+                            {'HASNOBASE': {3: 3000},
+                             'TEST': {1: 10, 3: 30},
+                             'TEST_BASE': {1: 1, 2: 2},
+                             'TEST_BOOLEAN': False,
+                             'TEST_LIST': [1, 2],
+                             'TEST_STRING': 'a string'})
+
     def test_freeze(self):
         self.settings.freeze()
         with self.assertRaises(TypeError) as cm:
@@ -343,14 +358,6 @@ class BaseSettingsTest(unittest.TestCase):
             self.assertEqual(self.settings.defaults.get('BAR'), 'foo')
             self.assertIn('BAR', self.settings.defaults)
 
-    def test_repr(self):
-        settings = BaseSettings()
-        self.assertEqual(repr(settings), "<BaseSettings {}>")
-        attr = SettingsAttribute('testval', 15)
-        settings['testkey'] = attr
-        self.assertEqual(repr(settings),
-                         "<BaseSettings {'testkey': %s}>" % repr(attr))
-
 
 class SettingsTest(unittest.TestCase):
 

From d843a0aae862a84b54f18f9d43ae957c182197b6 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 29 Jan 2016 21:12:03 +0100
Subject: [PATCH 0810/2974] Amend "settings" command to output JSON for dict
 settings

---
 scrapy/commands/settings.py    | 9 ++++++++-
 tests/test_cmdline/__init__.py | 2 +-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index 0e73f4f58..bce4e6086 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -1,5 +1,8 @@
 from __future__ import print_function
+import json
+
 from scrapy.commands import ScrapyCommand
+from scrapy.settings import BaseSettings
 
 class Command(ScrapyCommand):
 
@@ -28,7 +31,11 @@ class Command(ScrapyCommand):
     def run(self, args, opts):
         settings = self.crawler_process.settings
         if opts.get:
-            print(settings.get(opts.get))
+            s = settings.get(opts.get)
+            if isinstance(s, BaseSettings):
+                print(json.dumps(s.copy_to_dict()))
+            else:
+                print(s)
         elif opts.getbool:
             print(settings.getbool(opts.getbool))
         elif opts.getint:
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index c2de4fbc8..7733e7180 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -68,4 +68,4 @@ class CmdlineTest(unittest.TestCase):
             settingsstr = settingsstr.replace(char, '"')
         settingsdict = json.loads(settingsstr)
         six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys())
-        self.assertIn('value=200', settingsdict[EXT_PATH])
+        self.assertEquals(200, settingsdict[EXT_PATH])

From f9dc02e23ae35e5ba9dce74dcc2571d02e90eb22 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 28 Jan 2016 02:59:23 +0500
Subject: [PATCH 0811/2974] PY3 fix downloader slots GC

---
 scrapy/core/downloader/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 8beb08159..d835e65f7 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -194,6 +194,6 @@ class Downloader(object):
 
     def _slot_gc(self, age=60):
         mintime = time() - age
-        for key, slot in self.slots.items():
+        for key, slot in list(self.slots.items()):
             if not slot.active and slot.lastseen + slot.delay < mintime:
                 self.slots.pop(key).close()

From bb2cf7c0d7199fffe0aa100e5c8a51c6b4b82fc2 Mon Sep 17 00:00:00 2001
From: stummjr <stummjr@gmail.com>
Date: Fri, 29 Jan 2016 19:23:26 -0200
Subject: [PATCH 0812/2974] Fixed bug on XMLItemExporter with non-string fields
 in items

---
 scrapy/exporters.py     |  4 +++-
 tests/test_exporters.py | 14 ++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 35f50838b..360007c0f 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -144,8 +144,10 @@ class XmlItemExporter(BaseItemExporter):
         elif is_listlike(serialized_value):
             for value in serialized_value:
                 self._export_xml_field('value', value)
-        else:
+        elif isinstance(serialized_value, six.text_type):
             self._xg_characters(serialized_value)
+        else:
+            self._xg_characters(str(serialized_value))
         self.xg.endElement(name)
 
     # Workaround for http://bugs.python.org/issue17606
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 7ba5a0af6..cd72c661a 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -376,6 +376,20 @@ class XmlItemExporterTest(BaseItemExporterTest):
             b'</items>'
         )
 
+    def test_nonstring_types_item(self):
+        item = self._get_nonstring_types_item()
+        self.assertExportResult(item,
+            b'<?xml version="1.0" encoding="utf-8"?>\n'
+            b'<items>'
+               b'<item>'
+                   b'<float>3.14</float>'
+                   b'<boolean>False</boolean>'
+                   b'<number>22</number>'
+                   b'<time>2015-01-01 01:01:01</time>'
+               b'</item>'
+            b'</items>'
+        )
+
 
 class JsonLinesItemExporterTest(BaseItemExporterTest):
 

From 268e912273dcb7bacc5a7102d5a3f90a868f035f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 1 Feb 2016 12:43:27 +0100
Subject: [PATCH 0813/2974] Add pretty-printting of settings as dict if using
 IPython shell

Suggested by @digenis
see http://ipython.readthedocs.org/en/stable/api/generated/IPython.lib.pretty.html?#extending
---
 scrapy/settings/__init__.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 918bfc1e5..7b7808959 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -4,6 +4,7 @@ import copy
 import warnings
 from collections import MutableMapping
 from importlib import import_module
+from pprint import pformat
 
 from scrapy.utils.deprecate import create_deprecated_class
 from scrapy.exceptions import ScrapyDeprecationWarning
@@ -388,6 +389,12 @@ class BaseSettings(MutableMapping):
         settings = self.copy()
         return settings._to_dict()
 
+    def _repr_pretty_(self, p, cycle):
+        if cycle:
+            p.text(repr(self))
+        else:
+            p.text(pformat(self.copy_to_dict()))
+
     @property
     def overrides(self):
         warnings.warn("`Settings.overrides` attribute is deprecated and won't "

From 65fb67f2dbdb36763832b7b332bdbff0fd2bd9db Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 3 Feb 2016 01:01:16 +0500
Subject: [PATCH 0814/2974] PY3 fixed CrawlerRunner.stop

---
 scrapy/crawler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index bdcfa9d0c..ef99c243a 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -189,7 +189,7 @@ class CrawlerRunner(object):
 
         Returns a deferred that is fired when they all have ended.
         """
-        return defer.DeferredList([c.stop() for c in self.crawlers])
+        return defer.DeferredList([c.stop() for c in list(self.crawlers)])
 
     @defer.inlineCallbacks
     def join(self):

From c6591b5c9f46271731ee711a23da952037a15388 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 3 Feb 2016 05:37:40 +0500
Subject: [PATCH 0815/2974] more efficient ExecutionEngine.spider_is_idle

---
 scrapy/core/engine.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index ef4403106..3c4bc662c 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -177,12 +177,23 @@ class ExecutionEngine(object):
         return d
 
     def spider_is_idle(self, spider):
-        scraper_idle = self.scraper.slot.is_idle()
-        pending = self.slot.scheduler.has_pending_requests()
-        downloading = bool(self.downloader.active)
-        pending_start_requests = self.slot.start_requests is not None
-        idle = scraper_idle and not (pending or downloading or pending_start_requests)
-        return idle
+        if not self.scraper.slot.is_idle():
+            # scraper is not idle
+            return False
+
+        if self.downloader.active:
+            # downloader has pending requests
+            return False
+
+        if self.slot.start_requests is not None:
+            # not all start requests are handled
+            return False
+
+        if self.slot.scheduler.has_pending_requests():
+            # scheduler has pending requests
+            return False
+
+        return True
 
     @property
     def open_spiders(self):

From db0697bc0654e490a2cc8773b8c65ad70f88a019 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 3 Feb 2016 12:32:40 +0100
Subject: [PATCH 0816/2974] Add 1.1 release notes (draft)

---
 docs/news.rst | 332 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 331 insertions(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 4d7dc4d41..6a83a6163 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,336 @@
 Release notes
 =============
 
+1.1.0 (unreleased)
+------------------
+
+Python 3 Support (basic)
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+We have been hard at work to make Scrapy work on Python 3. Some features
+are still missing (and may never be ported to Python 3, see below),
+but you can now run spiders on Python 3.3, 3.4 and 3.5.
+
+Almost all of addons/middleware should work, but here are the current
+limitations we know of:
+
+- s3 downloads are not supported (see :issue:`1718`)
+- sending emails is not supported
+- FTP download handler is not supported (non-Python 3 ported Twisted dependency)
+- telnet is not supported (non-Python 3 ported Twisted dependency)
+- there are problems with non-ASCII URLs in Python 3
+- reported problems with HTTP cache created in Python 2.x which can't be used in 3.x (to be checked)
+- there is also a nasty issue with cryptography library:
+  recent versions don't work well on OS X  + Python 3.5 (see https://github.com/pyca/cryptography/issues/2690),
+  downgrading to an older version helps
+
+
+New Features and Enhancements
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Command line tool has completion for zsh (:issue:`934`)
+- ``scrapy shell`` works with local files again ; this was a regression
+  identified in 1.0+ releases (:issue:`1710`, :issue:`1550`)
+- ``scrapy shell`` now also checks a  new ``SCRAPY_PYTHON_SHELL`` environment
+  variable to launch the interactive shell of your choice ;
+  ``bpython`` is a newly supported option too (:issue:`1444`)
+- Autothrottle has gotten a code cleanup and better docs ;
+  there's also a new ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting which
+  allows to send more than 1 concurrent request on average (:issue:`1324`)
+- Memory usage extension has a new ``MEMUSAGE_CHECK_INTERVAL_SECONDS``
+  setting to change default check interval (:issue:`1282`)
+- HTTP caching follows RFC2616 more closely (TODO: link to docs);
+  2 new settings can be used to control level of compliancy:
+  ``HTTPCACHE_ALWAYS_STORE`` and ``HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS``
+  (:issue:`1151`)
+- Scheme Download handlers are now lazy-loaded on first request using
+  that scheme (``http(s)://``, ``ftp://``, ``file://``, ``s3://``)
+  (:issue:`1390`, :issue:`1421`)
+- RedirectMiddleware now skips status codes in ``handle_httpstatus_list``,
+  set either as spider  attribute or ``Request``'s ``meta`` key
+  (:issue:`1334`, :issue:`1364`, :issue:`1447`)
+
+
+- Form submit button plain #1469 (https://github.com/scrapy/scrapy/commit/b876755f1cee619d8c421357777d223037d5289c)
+    Fixes: Form submit button (https://github.com/scrapy/scrapy/issues/1354)
+- Implement FormRequest.from_response CSS support #1382 (https://github.com/scrapy/scrapy/commit/a6e5c848feb672c117f3380976077b6d0f42e3a6)
+ + Fix version number to appear new feature #1706
+
+- Incomplete submit button #1472 (https://github.com/scrapy/scrapy/commit/bc499cb552dad362494b86082e47d1f732095874)
+
+- dont retry 400 #1289 (https://github.com/scrapy/scrapy/milestones/Scrapy%201.1)
+    + DOC fix docs after GH-1289. #1530 (https://github.com/scrapy/scrapy/commit/451318ef7a4e8ee7837b83e73b158da98f579980)
+    WARNING: BACKWARDS INCOMPATIBLE!
+- DOC fix docs after GH-1289. #1530 (https://github.com/scrapy/scrapy/commit/451318ef7a4e8ee7837b83e73b158da98f579980)
+
+- Support for returning deferreds in middlewares #1473 (https://github.com/scrapy/scrapy/commit/dd473145f2e1ae2d3c9462c489f3289a96e447f4)
+    Adds support for returning deferreds in middlewares, and makes use of this to fix a limitation in RobotsTxtMiddleware.
+    Fixes #1471
+- add support for a nested loaders #1467 (https://github.com/scrapy/scrapy/commit/3c596dcf4606315e4eb88608e3ecde430fe18c29)
+    Closes: https://github.com/scrapy/scrapy/pull/818
+    Adds a nested_xpath()/nested_css() methods to ItemLoader. (TODO: add links to docs)
+
+- add_scheme_if_missing for `scrapy shell` command #1498 (https://github.com/scrapy/scrapy/commit/fe15f93e533be36e81e0385691fe5571c88b0b31)
+    Fixes: #1487
+    Warning: backward incompatible
+
+    + see: https://github.com/scrapy/scrapy/issues/1550, https://github.com/scrapy/scrapy/pull/1710
+- Per-key priorities for dict-like settings by promoting dicts to Settings instances #1149 (https://github.com/scrapy/scrapy/commit/dd9f777ba725d7a7dbb192302cc52a120005ad64)
+  + Backwards compatible per key priorities #1586 (https://github.com/scrapy/scrapy/commit/54216d7afe9d545031c57b5821f2c821faa2ccc3)
+
+    Fixes: Per-key priorities for dictionary-like settings #1135
+    Obsoletes: Settings.updatedict() method to update dictionary-like settings #1110
+- Support anonymous S3DownloadHandler (boto) connections #1358 (https://github.com/scrapy/scrapy/commit/5ec4319885e4be87b0248cb80b5213f68829129e)
+    + optional_features has been removed #1699
+
+- Enable robots.txt handling by default for new projects. #1724 (https://github.com/scrapy/scrapy/commit/0d368c5d6fd468aed301ed5967f8bfe9d5e86101)
+    WARNING: backwards incompatible
+
+- Disable CloseSpider extension if no CLOSPIDER_* setting set #1723 (https://github.com/scrapy/scrapy/commit/2246280bb6f71d7d52e24aca5b4ce955b3aa1363)
+- Disable SpiderState extension if no JOBDIR set #1725
+
+- Add Code of Conduct Version 1.3.0 from http://contributor-covenant.org/ #1681
+
+
+API changes
+~~~~~~~~~~~
+
+- Update form.py to improve existing capability PR #1137 (https://github.com/scrapy/scrapy/commit/786f62664b41f264bf4213a1ee3805774d82ed69)
+    Adds "formid" parameter for Form from_response()
+
+- Add ExecutionEngine.close() method #1423 (https://github.com/scrapy/scrapy/commit/caf2080b8095acd11de6018911025076ead23585)
+    Adds a new method as a single entry point for shutting down the engine
+    and integrates it into Crawler.crawl() for graceful error handling during the crawling process.
+
+    TODO: explain what this does
+- public Crawler.create_crawler method #1528 (https://github.com/scrapy/scrapy/commit/57f87b95d4d705f8afdd8fb9f7551033a7d88ee2)
+    Note: this is a Core API change
+    Note: this is CrawlerRunner.create_crawler(), not Crawler.create_crawler
+    http://doc.scrapy.org/en/master/topics/api.html?#scrapy.crawler.CrawlerRunner.create_crawler
+
+        Return a Crawler object.
+
+        If crawler_or_spidercls is a Crawler, it is returned as-is.
+        If crawler_or_spidercls is a Spider subclass, a new Crawler is constructed for it.
+        If crawler_or_spidercls is a string, this function finds a spider with this name in a Scrapy project (using spider loader), then creates a Crawler instance for it.
+
+- API CHANGE: response.text #1730 + micro-optimize response.text #1740
+    New `.text` attribute on TextResponses
+    Response body, as unicode.
+
+
+Deprecations and Removals
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- drop deprecated "optional_features" set #1359 (https://github.com/scrapy/scrapy/commit/7d187735ffecb0f49cffce1a9058961146212f59)
+- Remove --lsprof command-line option. #1689 (https://github.com/scrapy/scrapy/commit/56b69d2ea85ccdebfa5ec7945f1ed1df54b4b87f)
+    WARNING: backward incompatible, but doesnt break user code
+
+- deprecated unused and untested code in scrapy.utils.datatypes #1720
+    DEPRECATION: these will be removed in next releases
+        scrapy.utils.datatypes.MultiValueDictKeyError
+        scrapy.utils.datatypes.MultiValueDict
+        scrapy.utils.datatypes.SiteNode
+
+
+Relocations
+~~~~~~~~~~~
+
+- Migrating selectors to use parsel #1409 (https://github.com/scrapy/scrapy/commit/15c1300d35e4764ea343d98c133bc83f7c90c2d6)
+ + Replace usage of deprecated class by its parsel\'s counterpart #1431 (https://github.com/scrapy/scrapy/commit/12bebb61725272cdd977ce914d18a4b18ec0cb77)
+    closes Scrapy.selector Enhancement Proposal (https://github.com/scrapy/scrapy/issues/906)
+- Relocate telnetconsole to extensions/ #1524 (https://github.com/scrapy/scrapy/commit/72eeead6db7a5fdbce49a59102bb6a7125d56bc1)
+    Fixes: Move scrapy.telnet to scrapy.extensions.telnet #1520
+
+    See discussion on disabling telnet by default: (still open) https://github.com/scrapy/scrapy/issues/1572
+    Note that telnet is not enabled on Python 3 (https://github.com/scrapy/scrapy/pull/1524#issuecomment-146985595)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+- DOC SignalManager docstrings. See GH-713. #1291 (https://github.com/scrapy/scrapy/commit/5bd0395be4dc6d8315ad2726f1dbbd9c0b57b143)
+- Improvements for docs on how to access settings #1302 (https://github.com/scrapy/scrapy/commit/8b3ca4f250b4d831403c7fcfa72efe7ecdfa5247)
+    (closes: https://github.com/scrapy/scrapy/issues/1300)
+- Make Sphinx autodoc use local, not system-wide Scrapy PR #1335 (https://github.com/scrapy/scrapy/commit/b6eb3404a287508949ddb215e3f553a10fe43b8c)
+- DOCS: Update deprecated examples #1660 (https://github.com/scrapy/scrapy/commit/95e8ff8ba1dff3ec045dce931b6ea4314e887399)
+- DOCS: Update Stats Collection documentation for @master #1683 (https://github.com/scrapy/scrapy/commit/3f1f15bc4d3ee81612bce00fa0106ed16a7f72e5)
+- DOCS: DOC: Update MetaRefreshMiddlware's setting variables #1642 (https://github.com/scrapy/scrapy/commit/b1e44436bc4629773388d25ad9ab7b8ecf43d15e)
+
+    REDIRECT_MAX_METAREFRESH_DELAY has been deprecated and was renamed to METAREFRESH_MAXDELAY.
+    Merge duplicate documents about METAREFRESH_MAXDELAY appeared both in the settings page and the downloader-middlewares page.
+
+    Leftover from https://github.com/scrapy/scrapy/commit/defc4f89b542b756276f0920921dc00fe3ec4675
+- DOCS;TESTS: tests+doc for subdomains in offsite middleware #1721
+- DOCS: Clarify priority adjust settings docs #1727
+
+
+Bugfixes
+~~~~~~~~
+
+- Support empty password for http_proxy config #1313 (https://github.com/scrapy/scrapy/commit/07f4f12e8b5417fe3e9f70560f7b60bc488570e8)
+    Fixes #1274 HTTP_PROXY variable with username and empty password not supported
+- interpreting application/x-json as TextResponse #1333 (https://github.com/scrapy/scrapy/commit/2a7dc31f4cab7b13aacb632bdc78c50af754e76f)
+- Support link rel attribute with multiple values #1214 (https://github.com/scrapy/scrapy/commit/aa31811cfdc85eda07ddab25178d5003155523ec)
+    Fixes: nofollow doesnt work correcly when there multiple values in rel attribute #1201
+- BUG FIX: for Incorrectly picked URL in `scrapy.http.FormRequest.from_response` when there is a `<base>` tag #1562
+    PR #1563 (https://github.com/scrapy/scrapy/commit/9548691fdd47077a53f85daace091ef4af599cb9)
+- Startproject templates override #1575 (https://github.com/scrapy/scrapy/commit/3881eaff456d0d2704aa126f7c389080580d8f6c)
+    Closes: Override of TEMPLATES_DIR does not work for "startproject" command (https://github.com/scrapy/scrapy/issues/671)
+- BUG FIX: Various FormRequest tests+fixes #1597 (https://github.com/scrapy/scrapy/commit/dc6502639556efbd06d45319efa8320e84e88fde)
+    Fixes: FormRequest should consider input type values case-insensitive #1595
+    Fixes: FormRequest doesn't handle input elements without type attribute #1596
+- BUG FIX: for Incorrectly picked URL in `scrapy.linkextractors.regex.RegexLinkExtractor` when there is a `<base>` tag. #1564
+    PR #1565 (https://github.com/scrapy/scrapy/commit/17aba44f169fc3a86b6a1f46f30cf5fe29500db1)
+- BUG FIX: BF: robustify _monkeypatches check for twisted - str() name first (Closes #1634) #1644 (https://github.com/scrapy/scrapy/commit/57f99fc34ebc7cb8a2a84371b89552e6623c9e9d)
+    Fixes: https://github.com/scrapy/scrapy/issues/1634
+- Fix bug on XMLItemExporter with non-string fields in items #1747
+    Fixes: AttributeError when exporting non-string types through XMLFeedExporter #1738
+- change os.mknod() for open() #1657
+    Fixes: Test for startproject command fails in OS X #1635
+- BUG FIX: Fix PythonItemExporter and CSVExporter for non-string item types #1737
+
+
+Python 3 porting effort
+~~~~~~~~~~~~~~~~~~~~~~~
+
+- Python 3: PY3 port scrapy.utils.python PR #1379
+- Python 3: In-progress Python 3 port PR #1384
+    TODO: worth describing?
+- Python 3: fix form requests tests on py3 (https://github.com/scrapy/scrapy/commit/de6e013b9a8080cf759096e793272f6814e3617d)
+- Python 3: Port scrapy/responsetypes.py https://github.com/scrapy/scrapy/commit/d05cf6e0af8c26863cbb1edc7a8199165eaeeb5d
+- Python 3: remove scrapy.utils.testsite from PY3 ignores #1397
+- Python 3: PY3 port scrapy.utils.response #1396
+- Python 3: PY3 port http cookies handling #1398 (https://github.com/scrapy/scrapy/commit/95e6bd2f8da9c0ed79c3667ae0619d35541de346)
+
+- Python 3: PY3 port scrapy.utils.reqser #1408 (https://github.com/scrapy/scrapy/commit/311293ffdc63892bd5ab8494310529a6da0f5b62)
+
+- Python 3: nyov's PY3 changes #1415
+    Various files:
+        requirements-py3.txt
+        scrapy/cmdline.py
+        scrapy/core/downloader/handlers/s3.py
+        scrapy/core/downloader/middleware.py
+        scrapy/core/spidermw.py
+        scrapy/linkextractors/htmlparser.py
+        scrapy/pipelines/files.py
+        scrapy/pipelines/images.py
+        scrapy/utils/testproc.py
+        tests/py3-ignores.txt
+        tests/requirements-py3.txt
+        tests/test_cmdline/__init__.py
+        tests/test_command_version.py
+        tests/test_crawl.py
+        tests/test_loader.py
+        tests/test_pipeline_files.py
+        tests/test_pipeline_images.py
+        tests/test_selector_csstranslator.py
+        tests/test_selector_lxmldocument.py
+        tests/test_utils_iterators.py
+        tests/test_utils_reqser.py
+        tox.ini
+- Python 3: py3: port dictionary itervalues call (666ebfa1d97264bc4e6adb78fe4ce1a9ea15cc1f)
+- Python 3: PY3: port scrapy.utils.trackref #1420 (https://github.com/scrapy/scrapy/commit/fa3d84b0504e25f7478f7fac723a45)
+- Python 3: Small Python 3 fixes #1456 (https://github.com/scrapy/scrapy/commit/026a1caffb9f0bafbefba4f56af61a7347750f20)
+- Python 3: enable console tests in PY3 (8ecc4544b3747eb9be33153483b62c6441bd7c56)
+- Python 3: assorted Python 3 porting #1461 (https://github.com/scrapy/scrapy/commit/0018caf0b61e4f10857e61cddb347c3854bacc4b)
+    Port LxmlLinkExtractor and leave other link extractors Python 2.x - only.
+
+    refactor test_linkextractors
+    move tests for deprecated link extractors to another file and ignore it in Python 3
+    port LxmlLinkExtractor to Python 3
+    + scrapy.spiders and a couple more things
+
+- port some downloader middlewares to Python 3 #1470 (https://github.com/scrapy/scrapy/commit/3919ad64c5873d360aa1a412bee5270aad121760)
+    scrapy/downloadermiddlewares/httpauth.py
+    scrapy/downloadermiddlewares/useragent.py
+- Python 3: PY3 redirect downloader mware #1488 (https://github.com/scrapy/scrapy/commit/4d1c5c3d32591c37e37f879f0e77e50db7124603)
+- PY3 port bench, startproject, genspider, list and runspider commands #1535 (https://github.com/scrapy/scrapy/commit/411174cf38ebda00422529637b427a591c114eff)
+    Fixes: PY3 enable test_commands.ParseCommandTest #1536
+- Python 3:
+    - py3: fix webclient #1676 (https://github.com/scrapy/scrapy/commit/49fe631d8946f87e783c59e44a498f3d43083e2e)
+    - Py3: port http downloaders #1678 (https://github.com/scrapy/scrapy/commit/b4fb9d35342bc41a0149b74ecca38c056beaa220)
+    - Raise minimal twisted version for py3 #1694 (https://github.com/scrapy/scrapy/commit/d59d3f1e296795116704baa01780ff11870257f1)
+    - Cleanup http11 tunneling connection after #1678 #1701
+    - Py3: port downloader cache and compression middlewares #1680
+    - Add Python 3.5 tox env + Python 3.5 tests in Travis #1674 (https://github.com/scrapy/scrapy/commit/8fb9a6f8191dc0bf2dfb39ef01b1eb63e49bc23b)
+    - Py3: port test_engine #1691
+    - Py3: port commands fetch and shell #1693
+    - py3 fix HttpProxy and Retry Middlewares #1637
+    - PY3 fixed scrapy bench command #1708
+    - Py3: port test crawl #1692
+    - PY3 enable tests for scrapy parse command #1711
+    - py3: fix test_mail #1715
+    - py3: reviewed passing test_spidermiddleware_httperror.py #1717
+    - py3: test_pipeline_files and test_pipeline_images #1716
+    - PY3 exporters #1499
+    - PY3 fix downloader slots GC #1741
+- Python 3: PY3: port utils/iterators #1661 (https://github.com/scrapy/scrapy/commit/f01fd076420f0e58a1a165be31ec505eeb561ef4)
+
+
+Tests, CI and Deploys
+~~~~~~~~~~~~~~~~~~~~~
+
+- BF: fail if docs failed to build #1319
+- Run on new travis-ci infra (https://github.com/scrapy/scrapy/commit/805a491647fabfed58acb9d2)
+    no more travis workarounds (removed .travis-workarounds.sh)
+- Unset environment proxies for tests #1353 (https://github.com/scrapy/scrapy/commit/cbfb24dbeb82c791e82f1d9249685aa4d75fed3e)
+- Coverage and reports at codecov.io and coveralls.io #1433 (https://github.com/scrapy/scrapy/commit/9adb5c31c06bc22d1b5243a04633a)
+- drop coveralls support #1537 (https://github.com/scrapy/scrapy/commit/65f4ba349cb341736b67c0307074cef2cf0bd12e)
+- Add some missing tests for scrapy.settings #1570 (https://github.com/scrapy/scrapy/commit/9424ca0fdbdd492f3049fe08be8848f92e84fde3)
+- DOCS;TESTS: tests+doc for subdomains in offsite middleware #1721
+- TESTS: Include tests for non-string items to Exporters #1742
+
+
+Logging
+~~~~~~~
+
+- Ignore ScrapyDeprecationWarning warnings properly. #1294 (https://github.com/scrapy/scrapy/commit/64466526350820bdb424dc70968b4e015fd13641)
+- Do not fail representing non-http requests #1419 (https://github.com/scrapy/scrapy/commit/bdcc78b4ddf47b6161b962b9d9fc8851b11f0117)
+- Make list of enabled middlewares more readable #1263 (https://github.com/scrapy/scrapy/commit/a7787628ff53322e295be315e5595c555eb8e057)
+- added more verbosity for log and for exception when download is cancelled because of a size limit #1624 (https://github.com/scrapy/scrapy/commit/fdc3c9d561ad87e417447fcee9adcc8cd6dbc594)
+- LOGGING: show download warnsize once #1654 (https://github.com/scrapy/scrapy/commit/6827eab2c59e93d8ec46ef308bc751c6c00f32fd)
+- LOGGING: Fix logging of enabled middlewares #1722 + Use long classes names for enabled middlewares in startup logs #1726
+
+
+Code refactoring
+~~~~~~~~~~~~~~~~
+
+- Avoid creation of temporary list object in iflatten #1476 (https://github.com/scrapy/scrapy/commit/6ae8963256f52bcc26ea8b4edc938743b07b6b2c)
+- equal_attributes function optimization #1477 (https://github.com/scrapy/scrapy/commit/6490cb534e8e9a9068a8e298a8c6edb6be9725c5)
+- Optimization - avoid temporary list objects, unnecessary function call #1481 (https://github.com/scrapy/scrapy/commit/3e13740a5765152e1b8241ad4db91efac5c746d7)
+- Small downloader slots cleanup #1315 (https://github.com/scrapy/scrapy/commit/8a140b6ba1cf89e4a3bb74f8afb6e81c283e298b)
+    downloader.Slot becomes unaware of Scrapy settings;
+    it got __str__ and __repr__ methods useful in manhole;
+    unused import is dropped;
+    absolute_imports future import is added (I like adding it everywhere).
+- extract CrawlerRunner._crawl method which always expects Crawler #1290 (https://github.com/scrapy/scrapy/commit/5bcda9b7d13b9c3b486c2b247fd6d87a7b59df1a)
+    Provides an extension point where crawler instance is available;
+    makes it easier to write alternative CrawlerRunner.crawl implementations.
+    User can override CrawlerRunner._crawl method and connect signals there.
+
+
+Other changes
+~~~~~~~~~~~~~
+
+
+- Extend regex for tags that deploy to PyPI to support new release cycle (:commit:`26f50d3`)
+- rename str_to_unicode and unicode_to_str functions (ISSUE #778) (https://github.com/scrapy/scrapy/commit/61cd27e5c7b777a54)
+
+- fix utils.template.render_templatefile() bug +test #1212 (https://github.com/scrapy/scrapy/commit/71bd79e70fb10ed4899b15ca3ffa9aaa16567727)
+
+- style fixes for settings.py created by `scrapy startproject` #1496 (https://github.com/scrapy/scrapy/commit/5279da9916c00c7a6679cfc555f9a2b1863b4821)
+    Adds AUTOTHROTTLE_TARGET_CONCURRENCY to settings.py
+
+- (MINOR) Simplify if statement #1686 (https://github.com/scrapy/scrapy/commit/9ef25d7b68fe90c5e6b94bd3e81755089e743080)
+    Note: in conftest.py
+
+- (MINOR) fix indentation #1687 (https://github.com/scrapy/scrapy/commit/66f41aba3cbfa642b37354e8419e3d1437b88348)
+    Note: in scrapy/downloadermiddlewares/retry.py
+- (MINOR) fixed typo You -> you #1698 (https://github.com/scrapy/scrapy/commit/e8b26e2ab25ac7ec15c03d3c0b766c7aa8f48cce)
+    Fixes DOWNLOAD_WARNSIZE is too verbose #1303
+
+
 1.0.4 (2015-12-30)
 ------------------
 
@@ -590,7 +920,7 @@ Enhancements
 - Document `request_scheduled` signal (:issue:`746`)
 - Add a note about reporting security issues (:issue:`697`)
 - Add LevelDB http cache storage backend (:issue:`626`, :issue:`500`)
-- Sort spider list output of `scrapy list` command (:issue:`742`) 
+- Sort spider list output of `scrapy list` command (:issue:`742`)
 - Multiple documentation enhancemens and fixes
   (:issue:`575`, :issue:`587`, :issue:`590`, :issue:`596`, :issue:`610`,
   :issue:`617`, :issue:`618`, :issue:`627`, :issue:`613`, :issue:`643`,

From 142aa21737647864d8c9cdfff7e086be71041834 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 3 Feb 2016 12:33:10 +0100
Subject: [PATCH 0817/2974] Add AUTOTHROTTLE_TARGET_CONCURRENCY versionadded
 note

---
 docs/topics/autothrottle.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst
index 0d664cf67..b83946a58 100644
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@@ -127,6 +127,8 @@ The maximum download delay (in seconds) to be set in case of high latencies.
 AUTOTHROTTLE_TARGET_CONCURRENCY
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+.. versionadded:: 1.1
+
 Default: ``1.0``
 
 Average number of requests Scrapy should be sending in parallel to remote

From 2b033eebcee5feb18bd96de19a843edb67fd7803 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 3 Feb 2016 12:34:44 +0100
Subject: [PATCH 0818/2974] Fix recently added HTTPCACHE_ settings versionadded
 notes

---
 docs/topics/downloader-middleware.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index a97d5a696..4215cf69c 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -582,7 +582,7 @@ The class which implements the cache policy.
 HTTPCACHE_GZIP
 ^^^^^^^^^^^^^^
 
-.. versionadded:: 0.25
+.. versionadded:: 1.0
 
 Default: ``False``
 
@@ -594,7 +594,7 @@ This setting is specific to the Filesystem backend.
 HTTPCACHE_ALWAYS_STORE
 ^^^^^^^^^^^^^^^^^^^^^^
 
-.. versionadded:: 0.25
+.. versionadded:: 1.1
 
 Default: ``False``
 
@@ -614,7 +614,7 @@ responses you feedto the cache middleware.
 HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-.. versionadded:: 0.25
+.. versionadded:: 1.1
 
 Default: ``[]``
 

From 44d8df2060dade244f40c70f1417236ef6e3cc50 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 3 Feb 2016 12:35:26 +0100
Subject: [PATCH 0819/2974] Add versionadded note for
 MEMUSAGE_CHECK_INTERVAL_SECONDS

---
 docs/topics/settings.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 725345f2a..f8f35b5e3 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -766,6 +766,8 @@ See :ref:`topics-extensions-ref-memusage`.
 MEMUSAGE_CHECK_INTERVAL_SECONDS
 -------------------------------
 
+.. versionadded:: 1.1
+
 Default: ``60.0``
 
 Scope: ``scrapy.extensions.memusage``

From a21c90bee7658f3889a5093cda1693217c9b2e14 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Jr <stummjr@gmail.com>
Date: Wed, 3 Feb 2016 11:54:46 -0200
Subject: [PATCH 0820/2974] edits on Py3 Support and New features sections

---
 docs/news.rst | 76 +++++++++++++++++----------------------------------
 1 file changed, 25 insertions(+), 51 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 6a83a6163..f710192e8 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -9,34 +9,31 @@ Release notes
 Python 3 Support (basic)
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-We have been hard at work to make Scrapy work on Python 3. Some features
-are still missing (and may never be ported to Python 3, see below),
-but you can now run spiders on Python 3.3, 3.4 and 3.5.
+We have been hard at work to make Scrapy run on Python 3. As a result, now you can run spiders on Python 3.3, 3.4 and 3.5, although some features are still missing (some of them may never be ported to Python 3).
 
-Almost all of addons/middleware should work, but here are the current
-limitations we know of:
+Almost all addons/middlewares are expected to work. However, we are aware of some limitations:
 
 - s3 downloads are not supported (see :issue:`1718`)
 - sending emails is not supported
 - FTP download handler is not supported (non-Python 3 ported Twisted dependency)
 - telnet is not supported (non-Python 3 ported Twisted dependency)
 - there are problems with non-ASCII URLs in Python 3
-- reported problems with HTTP cache created in Python 2.x which can't be used in 3.x (to be checked)
-- there is also a nasty issue with cryptography library:
-  recent versions don't work well on OS X  + Python 3.5 (see https://github.com/pyca/cryptography/issues/2690),
-  downgrading to an older version helps
-
+- reported problems with HTTP caches created by Scrapy in Python 2.x which can't be reused in Scrapy in Python 3.x (to be checked)
+- there is also a nasty issue with `cryptography` library: recent versions don't work well on OS X + Python 3.5 (see https://github.com/pyca/cryptography/issues/2690). As a workaround, you can downgrade the library to an older version.
 
 New Features and Enhancements
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- Command line tool has completion for zsh (:issue:`934`)
-- ``scrapy shell`` works with local files again ; this was a regression
+- Command line tool completion for zsh (:issue:`934`)
+- ``scrapy shell`` works with local files again; this was a regression
   identified in 1.0+ releases (:issue:`1710`, :issue:`1550`)
 - ``scrapy shell`` now also checks a  new ``SCRAPY_PYTHON_SHELL`` environment
-  variable to launch the interactive shell of your choice ;
+  variable to launch the interactive shell of your choice;
   ``bpython`` is a newly supported option too (:issue:`1444`)
-- Autothrottle has gotten a code cleanup and better docs ;
+- Scrapy shell now have `http` as the default schema for URLs. Now, you can
+  start it by: `scrapy shell scrapy.org` #1498 (https://github.com/scrapy/scrapy/commit/fe15f93e533be36e81e0385691fe5571c88b0b31). Fixes: #1487 **Warning: backwards incompatible!**
+  + see: https://github.com/scrapy/scrapy/issues/1550, https://github.com/scrapy/scrapy/pull/1710
+- Autothrottle code has been cleaned up and its docs have been improved;
   there's also a new ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting which
   allows to send more than 1 concurrent request on average (:issue:`1324`)
 - Memory usage extension has a new ``MEMUSAGE_CHECK_INTERVAL_SECONDS``
@@ -48,50 +45,27 @@ New Features and Enhancements
 - Scheme Download handlers are now lazy-loaded on first request using
   that scheme (``http(s)://``, ``ftp://``, ``file://``, ``s3://``)
   (:issue:`1390`, :issue:`1421`)
-- RedirectMiddleware now skips status codes in ``handle_httpstatus_list``,
-  set either as spider  attribute or ``Request``'s ``meta`` key
+- RedirectMiddleware now skips the status codes from ``handle_httpstatus_list``. You can set it either as spider attribute or ``Request``'s ``meta`` key
   (:issue:`1334`, :issue:`1364`, :issue:`1447`)
-
-
-- Form submit button plain #1469 (https://github.com/scrapy/scrapy/commit/b876755f1cee619d8c421357777d223037d5289c)
-    Fixes: Form submit button (https://github.com/scrapy/scrapy/issues/1354)
-- Implement FormRequest.from_response CSS support #1382 (https://github.com/scrapy/scrapy/commit/a6e5c848feb672c117f3380976077b6d0f42e3a6)
- + Fix version number to appear new feature #1706
-
+- Form submission now works with `<button>` elements too (https://github.com/scrapy/scrapy/commit/b876755f1cee619d8c421357777d223037d5289c).
 - Incomplete submit button #1472 (https://github.com/scrapy/scrapy/commit/bc499cb552dad362494b86082e47d1f732095874)
-
-- dont retry 400 #1289 (https://github.com/scrapy/scrapy/milestones/Scrapy%201.1)
-    + DOC fix docs after GH-1289. #1530 (https://github.com/scrapy/scrapy/commit/451318ef7a4e8ee7837b83e73b158da98f579980)
-    WARNING: BACKWARDS INCOMPATIBLE!
-- DOC fix docs after GH-1289. #1530 (https://github.com/scrapy/scrapy/commit/451318ef7a4e8ee7837b83e73b158da98f579980)
-
-- Support for returning deferreds in middlewares #1473 (https://github.com/scrapy/scrapy/commit/dd473145f2e1ae2d3c9462c489f3289a96e447f4)
-    Adds support for returning deferreds in middlewares, and makes use of this to fix a limitation in RobotsTxtMiddleware.
-    Fixes #1471
-- add support for a nested loaders #1467 (https://github.com/scrapy/scrapy/commit/3c596dcf4606315e4eb88608e3ecde430fe18c29)
-    Closes: https://github.com/scrapy/scrapy/pull/818
-    Adds a nested_xpath()/nested_css() methods to ItemLoader. (TODO: add links to docs)
-
-- add_scheme_if_missing for `scrapy shell` command #1498 (https://github.com/scrapy/scrapy/commit/fe15f93e533be36e81e0385691fe5571c88b0b31)
-    Fixes: #1487
-    Warning: backward incompatible
-
-    + see: https://github.com/scrapy/scrapy/issues/1550, https://github.com/scrapy/scrapy/pull/1710
+- `FormRequest.from_response` now allows to define through CSS selectors which form from the response should be used. It previously supported only XPath. #1382 (https://github.com/scrapy/scrapy/commit/a6e5c848feb672c117f3380976077b6d0f42e3a6)
+- Scrapy doesn't retry requests that got `400 Bad Request` reponse anymore. #1289 (https://github.com/scrapy/scrapy/milestones/Scrapy%201.1) **Warning: backwards incompatible!**
+    + Fix docs for #1289 in #1530 (https://github.com/scrapy/scrapy/commit/451318ef7a4e8ee7837b83e73b158da98f579980)
+- Middlewares now can return deferreds. (https://github.com/scrapy/scrapy/commit/dd473145f2e1ae2d3c9462c489f3289a96e447f4)
+    + As a consequence, `RobotsTxtMiddleware` now fully respects `robots.txt`. (:issue:`#1471` #1473).
+- Item Loaders now support nested loaders #1467 (https://github.com/scrapy/scrapy/commit/3c596dcf4606315e4eb88608e3ecde430fe18c29)
 - Per-key priorities for dict-like settings by promoting dicts to Settings instances #1149 (https://github.com/scrapy/scrapy/commit/dd9f777ba725d7a7dbb192302cc52a120005ad64)
   + Backwards compatible per key priorities #1586 (https://github.com/scrapy/scrapy/commit/54216d7afe9d545031c57b5821f2c821faa2ccc3)
-
     Fixes: Per-key priorities for dictionary-like settings #1135
     Obsoletes: Settings.updatedict() method to update dictionary-like settings #1110
-- Support anonymous S3DownloadHandler (boto) connections #1358 (https://github.com/scrapy/scrapy/commit/5ec4319885e4be87b0248cb80b5213f68829129e)
-    + optional_features has been removed #1699
-
+- Anonymous S3DownloadHandler (boto) connections are supported now #1358 (https://github.com/scrapy/scrapy/commit/5ec4319885e4be87b0248cb80b5213f68829129e)
+    + `optional_features` has been removed #1699
 - Enable robots.txt handling by default for new projects. #1724 (https://github.com/scrapy/scrapy/commit/0d368c5d6fd468aed301ed5967f8bfe9d5e86101)
-    WARNING: backwards incompatible
-
-- Disable CloseSpider extension if no CLOSPIDER_* setting set #1723 (https://github.com/scrapy/scrapy/commit/2246280bb6f71d7d52e24aca5b4ce955b3aa1363)
-- Disable SpiderState extension if no JOBDIR set #1725
-
-- Add Code of Conduct Version 1.3.0 from http://contributor-covenant.org/ #1681
+    **Warning: backwards incompatible**
+- CloseSpider extension is disabled if no `CLOSEPIDER_*` setting is defined #1723 (https://github.com/scrapy/scrapy/commit/2246280bb6f71d7d52e24aca5b4ce955b3aa1363)
+- SpiderState extension is disabled if no `JOBDIR` is set #1725
+- Scrapy now has a Code of Conduct: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md #1681
 
 
 API changes

From 1fd95c9c9f44d92f4cbdcf2a442f684c3d26533a Mon Sep 17 00:00:00 2001
From: Valdir Stumm Jr <stummjr@gmail.com>
Date: Wed, 3 Feb 2016 12:45:48 -0200
Subject: [PATCH 0821/2974] fixed references to issues and fit text to 80 cols
 in the first two sections

---
 docs/news.rst | 87 +++++++++++++++++++++++++++++++--------------------
 1 file changed, 53 insertions(+), 34 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index f710192e8..300707229 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -9,63 +9,82 @@ Release notes
 Python 3 Support (basic)
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-We have been hard at work to make Scrapy run on Python 3. As a result, now you can run spiders on Python 3.3, 3.4 and 3.5, although some features are still missing (some of them may never be ported to Python 3).
+We have been hard at work to make Scrapy run on Python 3. As a result, now you
+can run spiders on Python 3.3, 3.4 and 3.5, although some features are still
+missing (some of them may never be ported to Python 3).
 
-Almost all addons/middlewares are expected to work. However, we are aware of some limitations:
+Almost all addons/middlewares are expected to work. However, we are aware of
+some limitations:
 
 - s3 downloads are not supported (see :issue:`1718`)
 - sending emails is not supported
-- FTP download handler is not supported (non-Python 3 ported Twisted dependency)
+- FTP download handler is not supported (non-Python 3 ported Twisted
+  dependency)
 - telnet is not supported (non-Python 3 ported Twisted dependency)
 - there are problems with non-ASCII URLs in Python 3
-- reported problems with HTTP caches created by Scrapy in Python 2.x which can't be reused in Scrapy in Python 3.x (to be checked)
-- there is also a nasty issue with `cryptography` library: recent versions don't work well on OS X + Python 3.5 (see https://github.com/pyca/cryptography/issues/2690). As a workaround, you can downgrade the library to an older version.
+- reported problems with HTTP caches created by Scrapy in Python 2.x which
+  can't be reused in Scrapy in Python 3.x (to be checked)
+- there is also a nasty issue with `cryptography` library: recent versions
+  don't work well on OS X + Python 3.5
+  (see https://github.com/pyca/cryptography/issues/2690). As a workaround, you
+  can downgrade the library to an older version.
 
 New Features and Enhancements
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- Command line tool completion for zsh (:issue:`934`)
+- Command line tool completion for zsh (:issue:`934`).
 - ``scrapy shell`` works with local files again; this was a regression
-  identified in 1.0+ releases (:issue:`1710`, :issue:`1550`)
+  identified in 1.0+ releases (:issue:`1710`, :issue:`1550`).
 - ``scrapy shell`` now also checks a  new ``SCRAPY_PYTHON_SHELL`` environment
   variable to launch the interactive shell of your choice;
-  ``bpython`` is a newly supported option too (:issue:`1444`)
+  ``bpython`` is a newly supported option too (:issue:`1444`).
 - Scrapy shell now have `http` as the default schema for URLs. Now, you can
-  start it by: `scrapy shell scrapy.org` #1498 (https://github.com/scrapy/scrapy/commit/fe15f93e533be36e81e0385691fe5571c88b0b31). Fixes: #1487 **Warning: backwards incompatible!**
-  + see: https://github.com/scrapy/scrapy/issues/1550, https://github.com/scrapy/scrapy/pull/1710
+  start it by: `scrapy shell scrapy.org` (:issue:`1498`).
+  **Warning: backwards incompatible!**
+  + see: :issue:`1550`, :issue:`1710`.
 - Autothrottle code has been cleaned up and its docs have been improved;
   there's also a new ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting which
-  allows to send more than 1 concurrent request on average (:issue:`1324`)
+  allows to send more than 1 concurrent request on average (:issue:`1324`).
 - Memory usage extension has a new ``MEMUSAGE_CHECK_INTERVAL_SECONDS``
-  setting to change default check interval (:issue:`1282`)
+  setting to change default check interval (:issue:`1282`).
 - HTTP caching follows RFC2616 more closely (TODO: link to docs);
   2 new settings can be used to control level of compliancy:
   ``HTTPCACHE_ALWAYS_STORE`` and ``HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS``
-  (:issue:`1151`)
+  (:issue:`1151`).
 - Scheme Download handlers are now lazy-loaded on first request using
   that scheme (``http(s)://``, ``ftp://``, ``file://``, ``s3://``)
-  (:issue:`1390`, :issue:`1421`)
-- RedirectMiddleware now skips the status codes from ``handle_httpstatus_list``. You can set it either as spider attribute or ``Request``'s ``meta`` key
-  (:issue:`1334`, :issue:`1364`, :issue:`1447`)
-- Form submission now works with `<button>` elements too (https://github.com/scrapy/scrapy/commit/b876755f1cee619d8c421357777d223037d5289c).
-- Incomplete submit button #1472 (https://github.com/scrapy/scrapy/commit/bc499cb552dad362494b86082e47d1f732095874)
-- `FormRequest.from_response` now allows to define through CSS selectors which form from the response should be used. It previously supported only XPath. #1382 (https://github.com/scrapy/scrapy/commit/a6e5c848feb672c117f3380976077b6d0f42e3a6)
-- Scrapy doesn't retry requests that got `400 Bad Request` reponse anymore. #1289 (https://github.com/scrapy/scrapy/milestones/Scrapy%201.1) **Warning: backwards incompatible!**
-    + Fix docs for #1289 in #1530 (https://github.com/scrapy/scrapy/commit/451318ef7a4e8ee7837b83e73b158da98f579980)
-- Middlewares now can return deferreds. (https://github.com/scrapy/scrapy/commit/dd473145f2e1ae2d3c9462c489f3289a96e447f4)
-    + As a consequence, `RobotsTxtMiddleware` now fully respects `robots.txt`. (:issue:`#1471` #1473).
-- Item Loaders now support nested loaders #1467 (https://github.com/scrapy/scrapy/commit/3c596dcf4606315e4eb88608e3ecde430fe18c29)
-- Per-key priorities for dict-like settings by promoting dicts to Settings instances #1149 (https://github.com/scrapy/scrapy/commit/dd9f777ba725d7a7dbb192302cc52a120005ad64)
-  + Backwards compatible per key priorities #1586 (https://github.com/scrapy/scrapy/commit/54216d7afe9d545031c57b5821f2c821faa2ccc3)
-    Fixes: Per-key priorities for dictionary-like settings #1135
-    Obsoletes: Settings.updatedict() method to update dictionary-like settings #1110
-- Anonymous S3DownloadHandler (boto) connections are supported now #1358 (https://github.com/scrapy/scrapy/commit/5ec4319885e4be87b0248cb80b5213f68829129e)
-    + `optional_features` has been removed #1699
-- Enable robots.txt handling by default for new projects. #1724 (https://github.com/scrapy/scrapy/commit/0d368c5d6fd468aed301ed5967f8bfe9d5e86101)
+  (:issue:`1390`, :issue:`1421`).
+- RedirectMiddleware now skips the status codes from
+  ``handle_httpstatus_list``. You can set it either as spider attribute or
+  ``Request``'s ``meta`` key (:issue:`1334`, :issue:`1364`, :issue:`1447`).
+- Form submission now works with `<button>` elements too (:issue:`1469`).
+- Incomplete submit button support (:issue:`1472`).
+- `FormRequest.from_response` now allows to define through CSS selectors which
+  form from the response should be used. It previously supported only XPath
+  (:issue:`1382`).
+- Scrapy doesn't retry requests that got `400 Bad Request` reponse anymore
+  (:issue:`1289`). **Warning: backwards incompatible!**
+- Middlewares now can return deferreds (:issue:`1473`).
+    + As a consequence, `RobotsTxtMiddleware` now fully respects `robots.txt`
+      (:issue:`1471`).
+- Item Loaders now support nested loaders (:issue:`1467`).
+- Per-key priorities for dict-like settings by promoting dicts to Settings
+  instances (:issue:`1149`).
+  + Backwards compatible per key priorities (:issue:`1586`).
+  + Fixes: Per-key priorities for dictionary-like settings (:issue:`1135`).
+  + Obsoletes: `Settings.updatedict()` method to update dictionary-like
+    settings (:issue:`1110`).
+- Anonymous `S3DownloadHandler` (boto) connections are supported now
+  (:issue:`1358`).
+    + `optional_features` has been removed (:issue:`1699`).
+- Enable robots.txt handling by default for new projects (:issue:`1724`).
     **Warning: backwards incompatible**
-- CloseSpider extension is disabled if no `CLOSEPIDER_*` setting is defined #1723 (https://github.com/scrapy/scrapy/commit/2246280bb6f71d7d52e24aca5b4ce955b3aa1363)
-- SpiderState extension is disabled if no `JOBDIR` is set #1725
-- Scrapy now has a Code of Conduct: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md #1681
+- CloseSpider extension is disabled if no `CLOSESPIDER_*` setting is defined
+  (:issue:`1723`).
+- SpiderState extension is disabled if no `JOBDIR` is set (:issue:`1725`)
+- Scrapy now has a Code of Conduct:
+  https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
+  (:issue:`1681`)
 
 
 API changes

From ed02455010635680acd57a6181cbcb2fe2790aea Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 3 Feb 2016 17:27:43 +0100
Subject: [PATCH 0822/2974] Format "Logging" subsection

---
 docs/news.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 300707229..e9294985b 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -280,12 +280,12 @@ Tests, CI and Deploys
 Logging
 ~~~~~~~
 
-- Ignore ScrapyDeprecationWarning warnings properly. #1294 (https://github.com/scrapy/scrapy/commit/64466526350820bdb424dc70968b4e015fd13641)
-- Do not fail representing non-http requests #1419 (https://github.com/scrapy/scrapy/commit/bdcc78b4ddf47b6161b962b9d9fc8851b11f0117)
-- Make list of enabled middlewares more readable #1263 (https://github.com/scrapy/scrapy/commit/a7787628ff53322e295be315e5595c555eb8e057)
-- added more verbosity for log and for exception when download is cancelled because of a size limit #1624 (https://github.com/scrapy/scrapy/commit/fdc3c9d561ad87e417447fcee9adcc8cd6dbc594)
-- LOGGING: show download warnsize once #1654 (https://github.com/scrapy/scrapy/commit/6827eab2c59e93d8ec46ef308bc751c6c00f32fd)
-- LOGGING: Fix logging of enabled middlewares #1722 + Use long classes names for enabled middlewares in startup logs #1726
+- Ignore ScrapyDeprecationWarning warnings properly (:issue:`1294`)
+- Do not fail representing non-HTTP requests (:issue:`1419`)
+- Make list of enabled middlewares more readable (:issue:`1263`)
+- Be more verbose when download is cancelled because of size limit (:issue:`1624`)
+- Show warning on download size only once (:issue:`1654`)
+- Fix logging of enabled middlewares in startup logs (:issue:`1722` and :issue:`1726`)
 
 
 Code refactoring

From 3b2a6abc0ba6a33efe106d2212b31af791f75ff2 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 3 Feb 2016 17:37:58 +0100
Subject: [PATCH 0823/2974] Format "Code refactoring" subsection

---
 docs/news.rst | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index e9294985b..a4bab7556 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -291,18 +291,9 @@ Logging
 Code refactoring
 ~~~~~~~~~~~~~~~~
 
-- Avoid creation of temporary list object in iflatten #1476 (https://github.com/scrapy/scrapy/commit/6ae8963256f52bcc26ea8b4edc938743b07b6b2c)
-- equal_attributes function optimization #1477 (https://github.com/scrapy/scrapy/commit/6490cb534e8e9a9068a8e298a8c6edb6be9725c5)
-- Optimization - avoid temporary list objects, unnecessary function call #1481 (https://github.com/scrapy/scrapy/commit/3e13740a5765152e1b8241ad4db91efac5c746d7)
-- Small downloader slots cleanup #1315 (https://github.com/scrapy/scrapy/commit/8a140b6ba1cf89e4a3bb74f8afb6e81c283e298b)
-    downloader.Slot becomes unaware of Scrapy settings;
-    it got __str__ and __repr__ methods useful in manhole;
-    unused import is dropped;
-    absolute_imports future import is added (I like adding it everywhere).
-- extract CrawlerRunner._crawl method which always expects Crawler #1290 (https://github.com/scrapy/scrapy/commit/5bcda9b7d13b9c3b486c2b247fd6d87a7b59df1a)
-    Provides an extension point where crawler instance is available;
-    makes it easier to write alternative CrawlerRunner.crawl implementations.
-    User can override CrawlerRunner._crawl method and connect signals there.
+- Optimization on lists (:issue:`1476` and :issue:`1481`)
+- equal_attributes function optimization (:issue:`1477`)
+- Downloader slots cleanup (:issue:`1315`)
 
 
 Other changes
@@ -325,6 +316,11 @@ Other changes
 - (MINOR) fixed typo You -> you #1698 (https://github.com/scrapy/scrapy/commit/e8b26e2ab25ac7ec15c03d3c0b766c7aa8f48cce)
     Fixes DOWNLOAD_WARNSIZE is too verbose #1303
 
+- extract CrawlerRunner._crawl method which always expects Crawler #1290 (https://github.com/scrapy/scrapy/commit/5bcda9b7d13b9c3b486c2b247fd6d87a7b59df1a)
+    Provides an extension point where crawler instance is available;
+    makes it easier to write alternative CrawlerRunner.crawl implementations.
+    User can override CrawlerRunner._crawl method and connect signals there.
+
 
 1.0.4 (2015-12-30)
 ------------------

From fd3193da86fc9689697bf6905f986f485e50fb04 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 3 Feb 2016 18:19:38 +0100
Subject: [PATCH 0824/2974] Reword some of "New Features and Enhancements"

---
 docs/news.rst | 88 ++++++++++++++++++++++++++-------------------------
 1 file changed, 45 insertions(+), 43 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index a4bab7556..50440c63c 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -32,16 +32,24 @@ some limitations:
 New Features and Enhancements
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+- Scrapy now has a Code of Conduct:
+  https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
+  (:issue:`1681`)
 - Command line tool completion for zsh (:issue:`934`).
-- ``scrapy shell`` works with local files again; this was a regression
-  identified in 1.0+ releases (:issue:`1710`, :issue:`1550`).
-- ``scrapy shell`` now also checks a  new ``SCRAPY_PYTHON_SHELL`` environment
-  variable to launch the interactive shell of your choice;
-  ``bpython`` is a newly supported option too (:issue:`1444`).
-- Scrapy shell now have `http` as the default schema for URLs. Now, you can
-  start it by: `scrapy shell scrapy.org` (:issue:`1498`).
-  **Warning: backwards incompatible!**
-  + see: :issue:`1550`, :issue:`1710`.
+- ``scrapy shell`` got a few changes of its own:
+
+  - it now checks a new ``SCRAPY_PYTHON_SHELL`` environment
+    variable to launch the interactive shell of your choice
+    (one of ``ipython``, ``bpython`` or ``python`` that is);
+  - it will try ``bpython`` if ``ipython`` is not available
+    (:issue:`1444`).
+  - it uses ``http://`` as the default scheme for URLs (:issue:`1498`)
+    (try ``scrapy shell scrapy.org``)
+    **Warning: backwards incompatible!**
+  - unless argument looks like a relative file path, which works again;
+    this was a regression identified in 1.0+ releases
+    (:issue:`1710`, :issue:`1550`).
+
 - Autothrottle code has been cleaned up and its docs have been improved;
   there's also a new ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting which
   allows to send more than 1 concurrent request on average (:issue:`1324`).
@@ -51,40 +59,32 @@ New Features and Enhancements
   2 new settings can be used to control level of compliancy:
   ``HTTPCACHE_ALWAYS_STORE`` and ``HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS``
   (:issue:`1151`).
-- Scheme Download handlers are now lazy-loaded on first request using
-  that scheme (``http(s)://``, ``ftp://``, ``file://``, ``s3://``)
-  (:issue:`1390`, :issue:`1421`).
+- Download handlers are now lazy-loaded on first request using their
+  scheme (:issue:`1390`, :issue:`1421`).
 - RedirectMiddleware now skips the status codes from
-  ``handle_httpstatus_list``. You can set it either as spider attribute or
-  ``Request``'s ``meta`` key (:issue:`1334`, :issue:`1364`, :issue:`1447`).
-- Form submission now works with `<button>` elements too (:issue:`1469`).
-- Incomplete submit button support (:issue:`1472`).
-- `FormRequest.from_response` now allows to define through CSS selectors which
-  form from the response should be used. It previously supported only XPath
-  (:issue:`1382`).
-- Scrapy doesn't retry requests that got `400 Bad Request` reponse anymore
-  (:issue:`1289`). **Warning: backwards incompatible!**
-- Middlewares now can return deferreds (:issue:`1473`).
-    + As a consequence, `RobotsTxtMiddleware` now fully respects `robots.txt`
-      (:issue:`1471`).
-- Item Loaders now support nested loaders (:issue:`1467`).
-- Per-key priorities for dict-like settings by promoting dicts to Settings
-  instances (:issue:`1149`).
-  + Backwards compatible per key priorities (:issue:`1586`).
-  + Fixes: Per-key priorities for dictionary-like settings (:issue:`1135`).
-  + Obsoletes: `Settings.updatedict()` method to update dictionary-like
-    settings (:issue:`1110`).
-- Anonymous `S3DownloadHandler` (boto) connections are supported now
-  (:issue:`1358`).
-    + `optional_features` has been removed (:issue:`1699`).
-- Enable robots.txt handling by default for new projects (:issue:`1724`).
-    **Warning: backwards incompatible**
-- CloseSpider extension is disabled if no `CLOSESPIDER_*` setting is defined
-  (:issue:`1723`).
-- SpiderState extension is disabled if no `JOBDIR` is set (:issue:`1725`)
-- Scrapy now has a Code of Conduct:
-  https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
-  (:issue:`1681`)
+  ``handle_httpstatus_list``. You can set it either as a spider attribute
+  or in ``Request``'s ``meta`` key (:issue:`1334`, :issue:`1364`, :issue:`1447`).
+- Form submission:
+
+  - now works with ``<button>`` elements too (:issue:`1469`).
+  - an empty string is used for submit buttons without a ``value``
+
+- Scrapy does not retry requests that got a ``HTTP 400 Bad Request``
+  response anymore (:issue:`1289`).
+  **Warning: backwards incompatible!**
+- Middlewares now can return deferreds (:issue:`1473`);
+
+  - as a consequence, ``RobotsTxtMiddleware`` now fully respects `robots.txt`
+    (:issue:`1471`).
+
+- ItemLoaders now support nested loaders (:issue:`1467`).
+- dict-like settings now have per-key priorities
+  (:issue:`1135`, :issue:`1149` and :issue:`1586`).
+- Anonymous S3 connections are now supported (:issue:`1358`).
+- ``/robots.txt`` compliance is enabled by default for new projects (:issue:`1724`).
+  **Warning: backwards incompatible**
+- ``CloseSpider`` and ``SpiderState`` extensions get disabled if no relevant
+  setting is set (:issue:`1723`, :issue:`1725`).
 
 
 API changes
@@ -92,7 +92,9 @@ API changes
 
 - Update form.py to improve existing capability PR #1137 (https://github.com/scrapy/scrapy/commit/786f62664b41f264bf4213a1ee3805774d82ed69)
     Adds "formid" parameter for Form from_response()
-
+- `FormRequest.from_response` now allows to define through CSS selectors which
+  form from the response should be used. It previously supported only XPath
+  (:issue:`1382`).
 - Add ExecutionEngine.close() method #1423 (https://github.com/scrapy/scrapy/commit/caf2080b8095acd11de6018911025076ead23585)
     Adds a new method as a single entry point for shutting down the engine
     and integrates it into Crawler.crawl() for graceful error handling during the crawling process.

From 0f816524e86461f5cfd28aaec2a9f971e571b226 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Jr <stummjr@gmail.com>
Date: Wed, 3 Feb 2016 15:07:12 -0200
Subject: [PATCH 0825/2974] edit API changes, Deprecations, and Relocations
 sections.

---
 docs/news.rst | 75 +++++++++++++++++++++------------------------------
 1 file changed, 31 insertions(+), 44 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 50440c63c..900c6d1c5 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -32,10 +32,8 @@ some limitations:
 New Features and Enhancements
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- Scrapy now has a Code of Conduct:
-  https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
-  (:issue:`1681`)
-- Command line tool completion for zsh (:issue:`934`).
+- Scrapy now has a `Code of Conduct`_ (:issue:`1681`).
+- Command line tool now has completion for zsh (:issue:`934`).
 - ``scrapy shell`` got a few changes of its own:
 
   - it now checks a new ``SCRAPY_PYTHON_SHELL`` environment
@@ -53,9 +51,9 @@ New Features and Enhancements
 - Autothrottle code has been cleaned up and its docs have been improved;
   there's also a new ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting which
   allows to send more than 1 concurrent request on average (:issue:`1324`).
-- Memory usage extension has a new ``MEMUSAGE_CHECK_INTERVAL_SECONDS``
+- Memory usage extension got a new ``MEMUSAGE_CHECK_INTERVAL_SECONDS``
   setting to change default check interval (:issue:`1282`).
-- HTTP caching follows RFC2616 more closely (TODO: link to docs);
+- HTTP caching now follows RFC2616 more closely (TODO: link to docs);
   2 new settings can be used to control level of compliancy:
   ``HTTPCACHE_ALWAYS_STORE`` and ``HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS``
   (:issue:`1151`).
@@ -67,7 +65,7 @@ New Features and Enhancements
 - Form submission:
 
   - now works with ``<button>`` elements too (:issue:`1469`).
-  - an empty string is used for submit buttons without a ``value``
+  - an empty string is now used for submit buttons without a ``value``
 
 - Scrapy does not retry requests that got a ``HTTP 400 Bad Request``
   response anymore (:issue:`1289`).
@@ -81,67 +79,56 @@ New Features and Enhancements
 - dict-like settings now have per-key priorities
   (:issue:`1135`, :issue:`1149` and :issue:`1586`).
 - Anonymous S3 connections are now supported (:issue:`1358`).
-- ``/robots.txt`` compliance is enabled by default for new projects (:issue:`1724`).
+- ``/robots.txt`` compliance is now enabled by default for new projects (:issue:`1724`).
   **Warning: backwards incompatible**
-- ``CloseSpider`` and ``SpiderState`` extensions get disabled if no relevant
+- ``CloseSpider`` and ``SpiderState`` extensions now get disabled if no relevant
   setting is set (:issue:`1723`, :issue:`1725`).
 
+.. _`Code of Conduct`: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
 
 API changes
 ~~~~~~~~~~~
 
-- Update form.py to improve existing capability PR #1137 (https://github.com/scrapy/scrapy/commit/786f62664b41f264bf4213a1ee3805774d82ed69)
-    Adds "formid" parameter for Form from_response()
-- `FormRequest.from_response` now allows to define through CSS selectors which
-  form from the response should be used. It previously supported only XPath
-  (:issue:`1382`).
-- Add ExecutionEngine.close() method #1423 (https://github.com/scrapy/scrapy/commit/caf2080b8095acd11de6018911025076ead23585)
-    Adds a new method as a single entry point for shutting down the engine
-    and integrates it into Crawler.crawl() for graceful error handling during the crawling process.
+- ``FormRequest.from_response`` now allows: 
+  
+  - to set which Form you want to get through a new ``formid`` parameter (:issue:`1137`).
 
-    TODO: explain what this does
-- public Crawler.create_crawler method #1528 (https://github.com/scrapy/scrapy/commit/57f87b95d4d705f8afdd8fb9f7551033a7d88ee2)
-    Note: this is a Core API change
-    Note: this is CrawlerRunner.create_crawler(), not Crawler.create_crawler
-    http://doc.scrapy.org/en/master/topics/api.html?#scrapy.crawler.CrawlerRunner.create_crawler
+  - to define through CSS selectors which form from the response should be used. 
+  It previously supported only XPath  (:issue:`1382`).
 
-        Return a Crawler object.
+- New methods:
 
-        If crawler_or_spidercls is a Crawler, it is returned as-is.
-        If crawler_or_spidercls is a Spider subclass, a new Crawler is constructed for it.
-        If crawler_or_spidercls is a string, this function finds a spider with this name in a Scrapy project (using spider loader), then creates a Crawler instance for it.
+  - ``ExecutionEngine.close`` (:issue:`1423`).
+  - ``CrawlerRunner.create_crawler`` (:issue:`1528`).
 
-- API CHANGE: response.text #1730 + micro-optimize response.text #1740
-    New `.text` attribute on TextResponses
-    Response body, as unicode.
+- ``.text`` is a new attribute on ``TextResponse``. It's a shortcut to the 
+  ``.body_as_unicode()`` method (:issue:`1730`).
 
 
 Deprecations and Removals
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- drop deprecated "optional_features" set #1359 (https://github.com/scrapy/scrapy/commit/7d187735ffecb0f49cffce1a9058961146212f59)
-- Remove --lsprof command-line option. #1689 (https://github.com/scrapy/scrapy/commit/56b69d2ea85ccdebfa5ec7945f1ed1df54b4b87f)
-    WARNING: backward incompatible, but doesnt break user code
+- The ``optional_features`` set has been removed (:issue:`1359`).
+- The ``--lsprof`` command line option has been removed (:issue:`1689`).
+  **Warning: backward incompatible**, but doesn't break user code.
+- The following datatypes were deprecated, because they were not used anywhere
+  (:issue:`1720`):
 
-- deprecated unused and untested code in scrapy.utils.datatypes #1720
-    DEPRECATION: these will be removed in next releases
-        scrapy.utils.datatypes.MultiValueDictKeyError
-        scrapy.utils.datatypes.MultiValueDict
-        scrapy.utils.datatypes.SiteNode
+  + ``scrapy.utils.datatypes.MultiValueDictKeyError``
+  + ``scrapy.utils.datatypes.MultiValueDict``
+  + ``scrapy.utils.datatypes.SiteNode``
 
 
 Relocations
 ~~~~~~~~~~~
 
-- Migrating selectors to use parsel #1409 (https://github.com/scrapy/scrapy/commit/15c1300d35e4764ea343d98c133bc83f7c90c2d6)
- + Replace usage of deprecated class by its parsel\'s counterpart #1431 (https://github.com/scrapy/scrapy/commit/12bebb61725272cdd977ce914d18a4b18ec0cb77)
-    closes Scrapy.selector Enhancement Proposal (https://github.com/scrapy/scrapy/issues/906)
-- Relocate telnetconsole to extensions/ #1524 (https://github.com/scrapy/scrapy/commit/72eeead6db7a5fdbce49a59102bb6a7125d56bc1)
-    Fixes: Move scrapy.telnet to scrapy.extensions.telnet #1520
+- Selectors were ported to use parsel_ (:issue:`1409`).
+- ``telnetconsole`` was relocated to ``extensions/`` (:issue:`1524`).\
 
-    See discussion on disabling telnet by default: (still open) https://github.com/scrapy/scrapy/issues/1572
-    Note that telnet is not enabled on Python 3 (https://github.com/scrapy/scrapy/pull/1524#issuecomment-146985595)
+  + Note: telnet is not enabled on Python 3
+    (https://github.com/scrapy/scrapy/pull/1524#issuecomment-146985595)
 
+.. _parsel: https://github.com/scrapy/parsel
 
 Documentation
 ~~~~~~~~~~~~~

From 51dc741fa2958700b3ab24c8abecde0bf91c7292 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 3 Feb 2016 16:08:46 +0100
Subject: [PATCH 0826/2974] Add 1.0.5 release notes

---
 docs/news.rst | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 4d7dc4d41..3e60b2b4f 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,14 @@
 Release notes
 =============
 
+1.0.5 (2016-02-04)
+------------------
+
+- FIX: [Backport] Ignore bogus links in LinkExtractors (fixes :issue:`907`, :commit:`108195e`)
+- TST: Changed buildbot makefile to use 'pytest' (:commit:`1f3d90a`)
+- DOC: Fixed typos in tutorial and media-pipeline (:commit:`808a9ea` and :commit:`803bd87`)
+- DOC: Add AjaxCrawlMiddleware to DOWNLOADER_MIDDLEWARES_BASE in settings docs (:commit:`aa94121`)
+
 1.0.4 (2015-12-30)
 ------------------
 
@@ -590,7 +598,7 @@ Enhancements
 - Document `request_scheduled` signal (:issue:`746`)
 - Add a note about reporting security issues (:issue:`697`)
 - Add LevelDB http cache storage backend (:issue:`626`, :issue:`500`)
-- Sort spider list output of `scrapy list` command (:issue:`742`) 
+- Sort spider list output of `scrapy list` command (:issue:`742`)
 - Multiple documentation enhancemens and fixes
   (:issue:`575`, :issue:`587`, :issue:`590`, :issue:`596`, :issue:`610`,
   :issue:`617`, :issue:`618`, :issue:`627`, :issue:`613`, :issue:`643`,

From dd7b0eb359dc030f66a1e6ec1bdbbbd15afbd8e1 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Jr <stummjr@gmail.com>
Date: Wed, 3 Feb 2016 19:23:02 -0200
Subject: [PATCH 0827/2974] Updates to release notes

---
 docs/news.rst | 340 ++++++++++++++------------------------------------
 1 file changed, 92 insertions(+), 248 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 900c6d1c5..25f459a49 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,107 +3,109 @@
 Release notes
 =============
 
-1.1.0 (unreleased)
-------------------
+1.1.0
+-----
 
-Python 3 Support (basic)
-~~~~~~~~~~~~~~~~~~~~~~~~
+This 1.1 release brings a lot of interesting features and bug fixes:
+
+- Scrapy 1.1 has basic Python 3 support (requires Twisted >= 15.5). See 
+  :ref:`news_basicpy3` for more details and some limitations.
+- Hot new features:
+
+  - ItemLoaders now support nested loaders (:issue:`1467`).
+  - ``FormRequest.from_response`` improvements (:issue:`1382`, :issue:`1137`).
+  - Added setting :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` and improved
+    AutoThrottle docs (:issue:`1324`).
+  - Added ``response.text`` to get body as unicode (:issue:`1730`).
+  - Anonymous S3 connections (:issue:`1358`).
+  - Deferreds in downloader middlewares (:issue:`1473`). This enables better 
+    robots.txt handling (:issue:`1471`).
+  - HTTP cache improvements (:issue:`1151`).
+
+- These bug fixes may require your attention:
+
+  - Don't retry bad requests (HTTP 400) (:issue:`1289`).
+  - Fix shell files argument handling (:issue:`1710`, :issue:`1550`).
+  - Fixes on robots.txt handling (:issue:`1783`).
+  - Exporters work on unicode (:issue:`1080`).
+  - Fix xmliter to accept nodenames with dots (:issue:`1533`).
+
+Keep reading for more details on other improvements and bug fixes.
+
+.. _news_basicpy3:
+
+Basic Python 3 Support
+~~~~~~~~~~~~~~~~~~~~~~
 
 We have been hard at work to make Scrapy run on Python 3. As a result, now you
-can run spiders on Python 3.3, 3.4 and 3.5, although some features are still
-missing (some of them may never be ported to Python 3).
+can run spiders on Python 3.3, 3.4 and 3.5 (Twisted >= 15.5 required). Some
+features are still missing (and some may never be ported).
 
 Almost all addons/middlewares are expected to work. However, we are aware of
 some limitations:
 
-- s3 downloads are not supported (see :issue:`1718`)
-- sending emails is not supported
+- S3 downloads are not supported (see :issue:`1718`)
+- Sending emails is not supported
 - FTP download handler is not supported (non-Python 3 ported Twisted
   dependency)
-- telnet is not supported (non-Python 3 ported Twisted dependency)
-- there are problems with non-ASCII URLs in Python 3
-- reported problems with HTTP caches created by Scrapy in Python 2.x which
+- Telnet is not supported (non-Python 3 ported Twisted dependency)
+- There are problems with non-ASCII URLs in Python 3
+- Reported problems with HTTP caches created by Scrapy in Python 2.x which
   can't be reused in Scrapy in Python 3.x (to be checked)
-- there is also a nasty issue with `cryptography` library: recent versions
+- There is also a nasty issue with `cryptography` library: recent versions
   don't work well on OS X + Python 3.5
   (see https://github.com/pyca/cryptography/issues/2690). As a workaround, you
   can downgrade the library to an older version.
 
-New Features and Enhancements
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Additional New Features and Enhancements
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 - Scrapy now has a `Code of Conduct`_ (:issue:`1681`).
 - Command line tool now has completion for zsh (:issue:`934`).
-- ``scrapy shell`` got a few changes of its own:
+- Improvements to ``scrapy shell``:
 
-  - it now checks a new ``SCRAPY_PYTHON_SHELL`` environment
-    variable to launch the interactive shell of your choice
-    (one of ``ipython``, ``bpython`` or ``python`` that is);
-  - it will try ``bpython`` if ``ipython`` is not available
-    (:issue:`1444`).
-  - it uses ``http://`` as the default scheme for URLs (:issue:`1498`)
-    (try ``scrapy shell scrapy.org``)
+  - Support for bpython and configure preferred Python shell via
+    ``SCRAPY_PYTHON_SHELL`` (:issue:`1100`, :issue:`1444`).
+  - Support URLs without scheme (:issue:`1498`)
     **Warning: backwards incompatible!**
-  - unless argument looks like a relative file path, which works again;
-    this was a regression identified in 1.0+ releases
-    (:issue:`1710`, :issue:`1550`).
+  - Bring back support for relative file path (:issue:`1710`, :issue:`1550`).
 
-- Autothrottle code has been cleaned up and its docs have been improved;
-  there's also a new ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting which
-  allows to send more than 1 concurrent request on average (:issue:`1324`).
-- Memory usage extension got a new ``MEMUSAGE_CHECK_INTERVAL_SECONDS``
-  setting to change default check interval (:issue:`1282`).
-- HTTP caching now follows RFC2616 more closely (TODO: link to docs);
-  2 new settings can be used to control level of compliancy:
-  ``HTTPCACHE_ALWAYS_STORE`` and ``HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS``
-  (:issue:`1151`).
+- Added :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` setting to change default check 
+  interval (:issue:`1282`).
+- HTTP caching now follows RFC2616 more closely, added settings
+  :setting:`HTTPCACHE_ALWAYS_STORE` and
+  :setting:`HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS` (:issue:`1151`).
 - Download handlers are now lazy-loaded on first request using their
   scheme (:issue:`1390`, :issue:`1421`).
-- RedirectMiddleware now skips the status codes from
-  ``handle_httpstatus_list``. You can set it either as a spider attribute
-  or in ``Request``'s ``meta`` key (:issue:`1334`, :issue:`1364`, :issue:`1447`).
+- ``RedirectMiddleware`` now skips the status codes from
+  ``handle_httpstatus_list`` on spider attribute
+  or in ``Request``'s ``meta`` key (:issue:`1334`, :issue:`1364`,
+  :issue:`1447`).
 - Form submission:
 
   - now works with ``<button>`` elements too (:issue:`1469`).
-  - an empty string is now used for submit buttons without a ``value``
+  - an empty string is now used for submit buttons without a value
+    (:issue:`1472`)
 
-- Scrapy does not retry requests that got a ``HTTP 400 Bad Request``
-  response anymore (:issue:`1289`).
-  **Warning: backwards incompatible!**
-- Middlewares now can return deferreds (:issue:`1473`);
-
-  - as a consequence, ``RobotsTxtMiddleware`` now fully respects `robots.txt`
-    (:issue:`1471`).
-
-- ItemLoaders now support nested loaders (:issue:`1467`).
-- dict-like settings now have per-key priorities
+- Item Loaders now support nested loaders (:issue:`1467`).
+- Dict-like settings now have per-key priorities
   (:issue:`1135`, :issue:`1149` and :issue:`1586`).
-- Anonymous S3 connections are now supported (:issue:`1358`).
-- ``/robots.txt`` compliance is now enabled by default for new projects (:issue:`1724`).
-  **Warning: backwards incompatible**
+- robots.txt compliance now enabled by default for new projects (:issue:`1724`).
 - ``CloseSpider`` and ``SpiderState`` extensions now get disabled if no relevant
   setting is set (:issue:`1723`, :issue:`1725`).
-
+- Added method ``ExecutionEngine.close`` (:issue:`1423`). 
+- Added method ``CrawlerRunner.create_crawler`` (:issue:`1528`).
+- Tons of documentation updates and related fixes (:issue:`1291`, :issue:`1302`,
+  :issue:`1335`, :issue:`1683`, :issue:`1660`, :issue:`1642`, :issue:`1721`,
+  :issue:`1727`).
+- Other refactoring, optimizations and cleanup (:issue:`1476`, :issue:`1481`,
+  :issue:`1477` and :issue:`1315`).
+- Added ``to_bytes`` and ``to_unicode``, deprecated ``str_to_unicode`` and
+  ``unicode_to_str`` functions (:issue:`778`).
+- Extracted ``CrawlerRunner._crawl`` method which always expects ``Crawler``
+  instance (:issue:`1290`).
 .. _`Code of Conduct`: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
 
-API changes
-~~~~~~~~~~~
-
-- ``FormRequest.from_response`` now allows: 
-  
-  - to set which Form you want to get through a new ``formid`` parameter (:issue:`1137`).
-
-  - to define through CSS selectors which form from the response should be used. 
-  It previously supported only XPath  (:issue:`1382`).
-
-- New methods:
-
-  - ``ExecutionEngine.close`` (:issue:`1423`).
-  - ``CrawlerRunner.create_crawler`` (:issue:`1528`).
-
-- ``.text`` is a new attribute on ``TextResponse``. It's a shortcut to the 
-  ``.body_as_unicode()`` method (:issue:`1730`).
-
 
 Deprecations and Removals
 ~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -111,8 +113,7 @@ Deprecations and Removals
 - The ``optional_features`` set has been removed (:issue:`1359`).
 - The ``--lsprof`` command line option has been removed (:issue:`1689`).
   **Warning: backward incompatible**, but doesn't break user code.
-- The following datatypes were deprecated, because they were not used anywhere
-  (:issue:`1720`):
+- The following datatypes were deprecated (:issue:`1720`):
 
   + ``scrapy.utils.datatypes.MultiValueDictKeyError``
   + ``scrapy.utils.datatypes.MultiValueDict``
@@ -123,192 +124,35 @@ Relocations
 ~~~~~~~~~~~
 
 - Selectors were ported to use parsel_ (:issue:`1409`).
-- ``telnetconsole`` was relocated to ``extensions/`` (:issue:`1524`).\
+- ``telnetconsole`` was relocated to ``extensions/`` (:issue:`1524`).
 
   + Note: telnet is not enabled on Python 3
     (https://github.com/scrapy/scrapy/pull/1524#issuecomment-146985595)
 
 .. _parsel: https://github.com/scrapy/parsel
 
-Documentation
-~~~~~~~~~~~~~
-
-- DOC SignalManager docstrings. See GH-713. #1291 (https://github.com/scrapy/scrapy/commit/5bd0395be4dc6d8315ad2726f1dbbd9c0b57b143)
-- Improvements for docs on how to access settings #1302 (https://github.com/scrapy/scrapy/commit/8b3ca4f250b4d831403c7fcfa72efe7ecdfa5247)
-    (closes: https://github.com/scrapy/scrapy/issues/1300)
-- Make Sphinx autodoc use local, not system-wide Scrapy PR #1335 (https://github.com/scrapy/scrapy/commit/b6eb3404a287508949ddb215e3f553a10fe43b8c)
-- DOCS: Update deprecated examples #1660 (https://github.com/scrapy/scrapy/commit/95e8ff8ba1dff3ec045dce931b6ea4314e887399)
-- DOCS: Update Stats Collection documentation for @master #1683 (https://github.com/scrapy/scrapy/commit/3f1f15bc4d3ee81612bce00fa0106ed16a7f72e5)
-- DOCS: DOC: Update MetaRefreshMiddlware's setting variables #1642 (https://github.com/scrapy/scrapy/commit/b1e44436bc4629773388d25ad9ab7b8ecf43d15e)
-
-    REDIRECT_MAX_METAREFRESH_DELAY has been deprecated and was renamed to METAREFRESH_MAXDELAY.
-    Merge duplicate documents about METAREFRESH_MAXDELAY appeared both in the settings page and the downloader-middlewares page.
-
-    Leftover from https://github.com/scrapy/scrapy/commit/defc4f89b542b756276f0920921dc00fe3ec4675
-- DOCS;TESTS: tests+doc for subdomains in offsite middleware #1721
-- DOCS: Clarify priority adjust settings docs #1727
-
 
 Bugfixes
 ~~~~~~~~
 
-- Support empty password for http_proxy config #1313 (https://github.com/scrapy/scrapy/commit/07f4f12e8b5417fe3e9f70560f7b60bc488570e8)
-    Fixes #1274 HTTP_PROXY variable with username and empty password not supported
-- interpreting application/x-json as TextResponse #1333 (https://github.com/scrapy/scrapy/commit/2a7dc31f4cab7b13aacb632bdc78c50af754e76f)
-- Support link rel attribute with multiple values #1214 (https://github.com/scrapy/scrapy/commit/aa31811cfdc85eda07ddab25178d5003155523ec)
-    Fixes: nofollow doesnt work correcly when there multiple values in rel attribute #1201
-- BUG FIX: for Incorrectly picked URL in `scrapy.http.FormRequest.from_response` when there is a `<base>` tag #1562
-    PR #1563 (https://github.com/scrapy/scrapy/commit/9548691fdd47077a53f85daace091ef4af599cb9)
-- Startproject templates override #1575 (https://github.com/scrapy/scrapy/commit/3881eaff456d0d2704aa126f7c389080580d8f6c)
-    Closes: Override of TEMPLATES_DIR does not work for "startproject" command (https://github.com/scrapy/scrapy/issues/671)
-- BUG FIX: Various FormRequest tests+fixes #1597 (https://github.com/scrapy/scrapy/commit/dc6502639556efbd06d45319efa8320e84e88fde)
-    Fixes: FormRequest should consider input type values case-insensitive #1595
-    Fixes: FormRequest doesn't handle input elements without type attribute #1596
-- BUG FIX: for Incorrectly picked URL in `scrapy.linkextractors.regex.RegexLinkExtractor` when there is a `<base>` tag. #1564
-    PR #1565 (https://github.com/scrapy/scrapy/commit/17aba44f169fc3a86b6a1f46f30cf5fe29500db1)
-- BUG FIX: BF: robustify _monkeypatches check for twisted - str() name first (Closes #1634) #1644 (https://github.com/scrapy/scrapy/commit/57f99fc34ebc7cb8a2a84371b89552e6623c9e9d)
-    Fixes: https://github.com/scrapy/scrapy/issues/1634
-- Fix bug on XMLItemExporter with non-string fields in items #1747
-    Fixes: AttributeError when exporting non-string types through XMLFeedExporter #1738
-- change os.mknod() for open() #1657
-    Fixes: Test for startproject command fails in OS X #1635
-- BUG FIX: Fix PythonItemExporter and CSVExporter for non-string item types #1737
-
-
-Python 3 porting effort
-~~~~~~~~~~~~~~~~~~~~~~~
-
-- Python 3: PY3 port scrapy.utils.python PR #1379
-- Python 3: In-progress Python 3 port PR #1384
-    TODO: worth describing?
-- Python 3: fix form requests tests on py3 (https://github.com/scrapy/scrapy/commit/de6e013b9a8080cf759096e793272f6814e3617d)
-- Python 3: Port scrapy/responsetypes.py https://github.com/scrapy/scrapy/commit/d05cf6e0af8c26863cbb1edc7a8199165eaeeb5d
-- Python 3: remove scrapy.utils.testsite from PY3 ignores #1397
-- Python 3: PY3 port scrapy.utils.response #1396
-- Python 3: PY3 port http cookies handling #1398 (https://github.com/scrapy/scrapy/commit/95e6bd2f8da9c0ed79c3667ae0619d35541de346)
-
-- Python 3: PY3 port scrapy.utils.reqser #1408 (https://github.com/scrapy/scrapy/commit/311293ffdc63892bd5ab8494310529a6da0f5b62)
-
-- Python 3: nyov's PY3 changes #1415
-    Various files:
-        requirements-py3.txt
-        scrapy/cmdline.py
-        scrapy/core/downloader/handlers/s3.py
-        scrapy/core/downloader/middleware.py
-        scrapy/core/spidermw.py
-        scrapy/linkextractors/htmlparser.py
-        scrapy/pipelines/files.py
-        scrapy/pipelines/images.py
-        scrapy/utils/testproc.py
-        tests/py3-ignores.txt
-        tests/requirements-py3.txt
-        tests/test_cmdline/__init__.py
-        tests/test_command_version.py
-        tests/test_crawl.py
-        tests/test_loader.py
-        tests/test_pipeline_files.py
-        tests/test_pipeline_images.py
-        tests/test_selector_csstranslator.py
-        tests/test_selector_lxmldocument.py
-        tests/test_utils_iterators.py
-        tests/test_utils_reqser.py
-        tox.ini
-- Python 3: py3: port dictionary itervalues call (666ebfa1d97264bc4e6adb78fe4ce1a9ea15cc1f)
-- Python 3: PY3: port scrapy.utils.trackref #1420 (https://github.com/scrapy/scrapy/commit/fa3d84b0504e25f7478f7fac723a45)
-- Python 3: Small Python 3 fixes #1456 (https://github.com/scrapy/scrapy/commit/026a1caffb9f0bafbefba4f56af61a7347750f20)
-- Python 3: enable console tests in PY3 (8ecc4544b3747eb9be33153483b62c6441bd7c56)
-- Python 3: assorted Python 3 porting #1461 (https://github.com/scrapy/scrapy/commit/0018caf0b61e4f10857e61cddb347c3854bacc4b)
-    Port LxmlLinkExtractor and leave other link extractors Python 2.x - only.
-
-    refactor test_linkextractors
-    move tests for deprecated link extractors to another file and ignore it in Python 3
-    port LxmlLinkExtractor to Python 3
-    + scrapy.spiders and a couple more things
-
-- port some downloader middlewares to Python 3 #1470 (https://github.com/scrapy/scrapy/commit/3919ad64c5873d360aa1a412bee5270aad121760)
-    scrapy/downloadermiddlewares/httpauth.py
-    scrapy/downloadermiddlewares/useragent.py
-- Python 3: PY3 redirect downloader mware #1488 (https://github.com/scrapy/scrapy/commit/4d1c5c3d32591c37e37f879f0e77e50db7124603)
-- PY3 port bench, startproject, genspider, list and runspider commands #1535 (https://github.com/scrapy/scrapy/commit/411174cf38ebda00422529637b427a591c114eff)
-    Fixes: PY3 enable test_commands.ParseCommandTest #1536
-- Python 3:
-    - py3: fix webclient #1676 (https://github.com/scrapy/scrapy/commit/49fe631d8946f87e783c59e44a498f3d43083e2e)
-    - Py3: port http downloaders #1678 (https://github.com/scrapy/scrapy/commit/b4fb9d35342bc41a0149b74ecca38c056beaa220)
-    - Raise minimal twisted version for py3 #1694 (https://github.com/scrapy/scrapy/commit/d59d3f1e296795116704baa01780ff11870257f1)
-    - Cleanup http11 tunneling connection after #1678 #1701
-    - Py3: port downloader cache and compression middlewares #1680
-    - Add Python 3.5 tox env + Python 3.5 tests in Travis #1674 (https://github.com/scrapy/scrapy/commit/8fb9a6f8191dc0bf2dfb39ef01b1eb63e49bc23b)
-    - Py3: port test_engine #1691
-    - Py3: port commands fetch and shell #1693
-    - py3 fix HttpProxy and Retry Middlewares #1637
-    - PY3 fixed scrapy bench command #1708
-    - Py3: port test crawl #1692
-    - PY3 enable tests for scrapy parse command #1711
-    - py3: fix test_mail #1715
-    - py3: reviewed passing test_spidermiddleware_httperror.py #1717
-    - py3: test_pipeline_files and test_pipeline_images #1716
-    - PY3 exporters #1499
-    - PY3 fix downloader slots GC #1741
-- Python 3: PY3: port utils/iterators #1661 (https://github.com/scrapy/scrapy/commit/f01fd076420f0e58a1a165be31ec505eeb561ef4)
-
-
-Tests, CI and Deploys
-~~~~~~~~~~~~~~~~~~~~~
-
-- BF: fail if docs failed to build #1319
-- Run on new travis-ci infra (https://github.com/scrapy/scrapy/commit/805a491647fabfed58acb9d2)
-    no more travis workarounds (removed .travis-workarounds.sh)
-- Unset environment proxies for tests #1353 (https://github.com/scrapy/scrapy/commit/cbfb24dbeb82c791e82f1d9249685aa4d75fed3e)
-- Coverage and reports at codecov.io and coveralls.io #1433 (https://github.com/scrapy/scrapy/commit/9adb5c31c06bc22d1b5243a04633a)
-- drop coveralls support #1537 (https://github.com/scrapy/scrapy/commit/65f4ba349cb341736b67c0307074cef2cf0bd12e)
-- Add some missing tests for scrapy.settings #1570 (https://github.com/scrapy/scrapy/commit/9424ca0fdbdd492f3049fe08be8848f92e84fde3)
-- DOCS;TESTS: tests+doc for subdomains in offsite middleware #1721
-- TESTS: Include tests for non-string items to Exporters #1742
-
-
-Logging
-~~~~~~~
-
-- Ignore ScrapyDeprecationWarning warnings properly (:issue:`1294`)
-- Do not fail representing non-HTTP requests (:issue:`1419`)
-- Make list of enabled middlewares more readable (:issue:`1263`)
-- Be more verbose when download is cancelled because of size limit (:issue:`1624`)
-- Show warning on download size only once (:issue:`1654`)
-- Fix logging of enabled middlewares in startup logs (:issue:`1722` and :issue:`1726`)
-
-
-Code refactoring
-~~~~~~~~~~~~~~~~
-
-- Optimization on lists (:issue:`1476` and :issue:`1481`)
-- equal_attributes function optimization (:issue:`1477`)
-- Downloader slots cleanup (:issue:`1315`)
-
-
-Other changes
-~~~~~~~~~~~~~
-
-
-- Extend regex for tags that deploy to PyPI to support new release cycle (:commit:`26f50d3`)
-- rename str_to_unicode and unicode_to_str functions (ISSUE #778) (https://github.com/scrapy/scrapy/commit/61cd27e5c7b777a54)
-
-- fix utils.template.render_templatefile() bug +test #1212 (https://github.com/scrapy/scrapy/commit/71bd79e70fb10ed4899b15ca3ffa9aaa16567727)
-
-- style fixes for settings.py created by `scrapy startproject` #1496 (https://github.com/scrapy/scrapy/commit/5279da9916c00c7a6679cfc555f9a2b1863b4821)
-    Adds AUTOTHROTTLE_TARGET_CONCURRENCY to settings.py
-
-- (MINOR) Simplify if statement #1686 (https://github.com/scrapy/scrapy/commit/9ef25d7b68fe90c5e6b94bd3e81755089e743080)
-    Note: in conftest.py
-
-- (MINOR) fix indentation #1687 (https://github.com/scrapy/scrapy/commit/66f41aba3cbfa642b37354e8419e3d1437b88348)
-    Note: in scrapy/downloadermiddlewares/retry.py
-- (MINOR) fixed typo You -> you #1698 (https://github.com/scrapy/scrapy/commit/e8b26e2ab25ac7ec15c03d3c0b766c7aa8f48cce)
-    Fixes DOWNLOAD_WARNSIZE is too verbose #1303
-
-- extract CrawlerRunner._crawl method which always expects Crawler #1290 (https://github.com/scrapy/scrapy/commit/5bcda9b7d13b9c3b486c2b247fd6d87a7b59df1a)
-    Provides an extension point where crawler instance is available;
-    makes it easier to write alternative CrawlerRunner.crawl implementations.
-    User can override CrawlerRunner._crawl method and connect signals there.
+- Scrapy does not retry requests that got a ``HTTP 400 Bad Request``
+  response anymore (:issue:`1289`). **Warning: backwards incompatible!**
+- Support empty password for http_proxy config (:issue:`1274`).
+- Interpret ``application/x-json`` as ``TextResponse`` (:issue:`1333`).
+- Support link rel attribute with multiple values (:issue:`1201`).
+- Fixed ``scrapy.http.FormRequest.from_response`` when there is a ``<base>``
+  tag (:issue:`1564`).
+- Fixed :setting:`TEMPLATES_DIR` handling (:issue:`1575`).
+- Various ``FormRequest`` fixes (:issue:`1595`, :issue:`1596`, :issue:`1597`).
+- Makes ``_monkeypatches`` more robust (:issue:`1634`).
+- Fixed bug on ``XMLItemExporter`` with non-string fields in 
+  items (:issue:`1738`).
+- Fixed startproject command in OS X (:issue:`1635`).
+- Fixed PythonItemExporter and CSVExporter for non-string item
+  types (:issue:`1737`).
+- Various logging related fixes (:issue:`1294`, :issue:`1419`, :issue:`1263`,
+  :issue:`1624`, :issue:`1654`, :issue:`1722`, :issue:`1726` and :issue:`1303`).
+- Fixed bug in ``utils.template.render_templatefile()`` (:issue:`1212`).
 
 
 1.0.4 (2015-12-30)

From 972f1925684f16a512205dccdd5cff9478007760 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 3 Feb 2016 22:58:49 +0100
Subject: [PATCH 0828/2974] Blank line before CoC

---
 docs/news.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/news.rst b/docs/news.rst
index 3f118f218..775513a98 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -104,6 +104,7 @@ Additional New Features and Enhancements
   ``unicode_to_str`` functions (:issue:`778`).
 - Extracted ``CrawlerRunner._crawl`` method which always expects ``Crawler``
   instance (:issue:`1290`).
+
 .. _`Code of Conduct`: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
 
 

From 89a088b3c349af11e16e66089ae4991bcd196113 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 3 Feb 2016 20:45:55 -0200
Subject: [PATCH 0829/2974] applying review comments

---
 docs/news.rst | 44 ++++++++++++++++++++++++--------------------
 1 file changed, 24 insertions(+), 20 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 775513a98..0c5228ca2 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -12,7 +12,7 @@ This 1.1 release brings a lot of interesting features and bug fixes:
   :ref:`news_basicpy3` for more details and some limitations.
 - Hot new features:
 
-  - ItemLoaders now support nested loaders (:issue:`1467`).
+  - Item loaders now support nested loaders (:issue:`1467`).
   - ``FormRequest.from_response`` improvements (:issue:`1382`, :issue:`1137`).
   - Added setting :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` and improved
     AutoThrottle docs (:issue:`1324`).
@@ -20,38 +20,47 @@ This 1.1 release brings a lot of interesting features and bug fixes:
   - Anonymous S3 connections (:issue:`1358`).
   - Deferreds in downloader middlewares (:issue:`1473`). This enables better
     robots.txt handling (:issue:`1471`).
-  - HTTP cache improvements (:issue:`1151`).
+  - HTTP caching now follows RFC2616 more closely, added settings
+    :setting:`HTTPCACHE_ALWAYS_STORE` and
+    :setting:`HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS` (:issue:`1151`).
 
 - These bug fixes may require your attention:
 
-  - Don't retry bad requests (HTTP 400) (:issue:`1289`).
+  - Don't retry bad requests (HTTP 400) by default (:issue:`1289`).
+    If you need the old behavior, add ``400`` to :setting:`RETRY_HTTP_CODES`.
   - Fix shell files argument handling (:issue:`1710`, :issue:`1550`).
-  - Fixes on robots.txt handling (:issue:`1783`).
-  - Exporters work on unicode (:issue:`1080`).
-  - Fix xmliter to accept nodenames with dots (:issue:`1533`).
+    If you try ``scrapy shell index.html`` it will try to load the URL http://index.html,
+    use ``scrapy shell ./index.html`` to load a local file.
+  - Robots.txt compliance is now enabled by default for newly-created projects
+    (:issue:`1724`).
+  - Exporters now work on unicode, instead of bytes by default (:issue:`1080`).
+    If you use ``PythonItemExporter``, you may want to update your code to
+    disable binary mode which is now deprecated.
+  - Accept XML node names containing dots as valid (:issue:`1533`).
 
 Keep reading for more details on other improvements and bug fixes.
 
 .. _news_basicpy3:
 
-Basic Python 3 Support
-~~~~~~~~~~~~~~~~~~~~~~
+Beta Python 3 Support
+~~~~~~~~~~~~~~~~~~~~~
 
-We have been hard at work to make Scrapy run on Python 3. As a result, now you
-can run spiders on Python 3.3, 3.4 and 3.5 (Twisted >= 15.5 required). Some
+We have been `hard at work to make Scrapy run on Python 3
+<https://github.com/scrapy/scrapy/wiki/Python-3-Porting>`_. As a result, now
+you can run spiders on Python 3.3, 3.4 and 3.5 (Twisted >= 15.5 required). Some
 features are still missing (and some may never be ported).
 
-Almost all addons/middlewares are expected to work. However, we are aware of
-some limitations:
 
+Almost all addons/middlewares are expected to work. However, we are aware of
+some limitations in Python 3:
+
+- Doesn't work in Windows yet (non-Python 3 ported Twisted dependency)
 - S3 downloads are not supported (see :issue:`1718`)
-- Sending emails is not supported
+- Sending emails is not supported (non-Python 3 ported Twisted dependency)
 - FTP download handler is not supported (non-Python 3 ported Twisted
   dependency)
 - Telnet is not supported (non-Python 3 ported Twisted dependency)
 - There are problems with non-ASCII URLs in Python 3
-- Reported problems with HTTP caches created by Scrapy in Python 2.x which
-  can't be reused in Scrapy in Python 3.x (to be checked)
 - There is also a nasty issue with `cryptography` library: recent versions
   don't work well on OS X + Python 3.5
   (see https://github.com/pyca/cryptography/issues/2690). As a workaround, you
@@ -72,9 +81,6 @@ Additional New Features and Enhancements
 
 - Added :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` setting to change default check
   interval (:issue:`1282`).
-- HTTP caching now follows RFC2616 more closely, added settings
-  :setting:`HTTPCACHE_ALWAYS_STORE` and
-  :setting:`HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS` (:issue:`1151`).
 - Download handlers are now lazy-loaded on first request using their
   scheme (:issue:`1390`, :issue:`1421`).
 - ``RedirectMiddleware`` now skips the status codes from
@@ -87,10 +93,8 @@ Additional New Features and Enhancements
   - an empty string is now used for submit buttons without a value
     (:issue:`1472`)
 
-- Item Loaders now support nested loaders (:issue:`1467`).
 - Dict-like settings now have per-key priorities
   (:issue:`1135`, :issue:`1149` and :issue:`1586`).
-- robots.txt compliance now enabled by default for new projects (:issue:`1724`).
 - ``CloseSpider`` and ``SpiderState`` extensions now get disabled if no relevant
   setting is set (:issue:`1723`, :issue:`1725`).
 - Added method ``ExecutionEngine.close`` (:issue:`1423`).

From 2f697ce59f8517217da960d7313ce4bab21c2b17 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 3 Feb 2016 20:51:26 -0200
Subject: [PATCH 0830/2974] added note about how to revert to old behavior for
 robotstxt

---
 docs/news.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 0c5228ca2..ea797c1ee 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -32,7 +32,8 @@ This 1.1 release brings a lot of interesting features and bug fixes:
     If you try ``scrapy shell index.html`` it will try to load the URL http://index.html,
     use ``scrapy shell ./index.html`` to load a local file.
   - Robots.txt compliance is now enabled by default for newly-created projects
-    (:issue:`1724`).
+    (:issue:`1724`). If you need old behavior, update :setting:`ROBOTSTXT_OBEY`
+    in ``settings.py`` file when creating a new project.
   - Exporters now work on unicode, instead of bytes by default (:issue:`1080`).
     If you use ``PythonItemExporter``, you may want to update your code to
     disable binary mode which is now deprecated.

From 9cfefd52773e87fd1532ea9686537f3947755e6f Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 3 Feb 2016 20:59:56 -0200
Subject: [PATCH 0831/2974] favor beta over basic to describe PY3 support

---
 docs/news.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index ea797c1ee..78655cad7 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -8,8 +8,8 @@ Release notes
 
 This 1.1 release brings a lot of interesting features and bug fixes:
 
-- Scrapy 1.1 has basic Python 3 support (requires Twisted >= 15.5). See
-  :ref:`news_basicpy3` for more details and some limitations.
+- Scrapy 1.1 has beta Python 3 support (requires Twisted >= 15.5). See
+  :ref:`news_betapy3` for more details and some limitations.
 - Hot new features:
 
   - Item loaders now support nested loaders (:issue:`1467`).
@@ -41,7 +41,7 @@ This 1.1 release brings a lot of interesting features and bug fixes:
 
 Keep reading for more details on other improvements and bug fixes.
 
-.. _news_basicpy3:
+.. _news_betapy3:
 
 Beta Python 3 Support
 ~~~~~~~~~~~~~~~~~~~~~

From d40add7b276a0377ba575a5b5a63a8840b8255b1 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 3 Feb 2016 21:18:25 -0200
Subject: [PATCH 0832/2974] add note about robots.txt waiting and make it
 explicit builtin extensions only are ported

---
 docs/news.rst | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 78655cad7..93b8f477e 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -32,8 +32,10 @@ This 1.1 release brings a lot of interesting features and bug fixes:
     If you try ``scrapy shell index.html`` it will try to load the URL http://index.html,
     use ``scrapy shell ./index.html`` to load a local file.
   - Robots.txt compliance is now enabled by default for newly-created projects
-    (:issue:`1724`). If you need old behavior, update :setting:`ROBOTSTXT_OBEY`
-    in ``settings.py`` file when creating a new project.
+    (:issue:`1724`). Scrapy will also wait for robots.txt to be downloaded
+    before proceeding with the crawl. (:issue:`1735`). If you need the old
+    behavior, update :setting:`ROBOTSTXT_OBEY` in ``settings.py`` file when
+    creating a new project.
   - Exporters now work on unicode, instead of bytes by default (:issue:`1080`).
     If you use ``PythonItemExporter``, you may want to update your code to
     disable binary mode which is now deprecated.
@@ -52,7 +54,7 @@ you can run spiders on Python 3.3, 3.4 and 3.5 (Twisted >= 15.5 required). Some
 features are still missing (and some may never be ported).
 
 
-Almost all addons/middlewares are expected to work. However, we are aware of
+Almost all builtin extensions/middlewares are expected to work. However, we are aware of
 some limitations in Python 3:
 
 - Doesn't work in Windows yet (non-Python 3 ported Twisted dependency)

From 3180abc8002525fc5af1a918d8e9e409c3fdc0cd Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 3 Feb 2016 21:28:05 -0200
Subject: [PATCH 0833/2974] applying more review comments

---
 docs/news.rst | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 93b8f477e..dd1ab451a 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -23,6 +23,9 @@ This 1.1 release brings a lot of interesting features and bug fixes:
   - HTTP caching now follows RFC2616 more closely, added settings
     :setting:`HTTPCACHE_ALWAYS_STORE` and
     :setting:`HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS` (:issue:`1151`).
+  - Selectors were extracted to the parsel_ library (:issue:`1409`). This means
+    you can use Scrapy Selectors without Scrapy and also upgrade the
+    selectors engine without needing to upgrade Scrapy.
 
 - These bug fixes may require your attention:
 
@@ -64,10 +67,6 @@ some limitations in Python 3:
   dependency)
 - Telnet is not supported (non-Python 3 ported Twisted dependency)
 - There are problems with non-ASCII URLs in Python 3
-- There is also a nasty issue with `cryptography` library: recent versions
-  don't work well on OS X + Python 3.5
-  (see https://github.com/pyca/cryptography/issues/2690). As a workaround, you
-  can downgrade the library to an older version.
 
 Additional New Features and Enhancements
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -106,11 +105,7 @@ Additional New Features and Enhancements
   :issue:`1335`, :issue:`1683`, :issue:`1660`, :issue:`1642`, :issue:`1721`,
   :issue:`1727`).
 - Other refactoring, optimizations and cleanup (:issue:`1476`, :issue:`1481`,
-  :issue:`1477` and :issue:`1315`).
-- Added ``to_bytes`` and ``to_unicode``, deprecated ``str_to_unicode`` and
-  ``unicode_to_str`` functions (:issue:`778`).
-- Extracted ``CrawlerRunner._crawl`` method which always expects ``Crawler``
-  instance (:issue:`1290`).
+  :issue:`1477`, :issue:`1315` and :issue:`1290`).
 
 .. _`Code of Conduct`: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
 
@@ -118,6 +113,8 @@ Additional New Features and Enhancements
 Deprecations and Removals
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
+- Added ``to_bytes`` and ``to_unicode``, deprecated ``str_to_unicode`` and
+  ``unicode_to_str`` functions (:issue:`778`).
 - The ``optional_features`` set has been removed (:issue:`1359`).
 - The ``--lsprof`` command line option has been removed (:issue:`1689`).
   **Warning: backward incompatible**, but doesn't break user code.
@@ -131,7 +128,6 @@ Deprecations and Removals
 Relocations
 ~~~~~~~~~~~
 
-- Selectors were ported to use parsel_ (:issue:`1409`).
 - ``telnetconsole`` was relocated to ``extensions/`` (:issue:`1524`).
 
   + Note: telnet is not enabled on Python 3

From 2167573b32766ce693eb5516ecb2892146e00805 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 3 Feb 2016 21:31:27 -0200
Subject: [PATCH 0834/2974] fix comment about disabling robots.txt

---
 docs/news.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index dd1ab451a..5a4049e26 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -36,9 +36,9 @@ This 1.1 release brings a lot of interesting features and bug fixes:
     use ``scrapy shell ./index.html`` to load a local file.
   - Robots.txt compliance is now enabled by default for newly-created projects
     (:issue:`1724`). Scrapy will also wait for robots.txt to be downloaded
-    before proceeding with the crawl. (:issue:`1735`). If you need the old
-    behavior, update :setting:`ROBOTSTXT_OBEY` in ``settings.py`` file when
-    creating a new project.
+    before proceeding with the crawl. (:issue:`1735`). If you want to disable
+    this behavior, update :setting:`ROBOTSTXT_OBEY` in ``settings.py`` file
+    after creating a new project.
   - Exporters now work on unicode, instead of bytes by default (:issue:`1080`).
     If you use ``PythonItemExporter``, you may want to update your code to
     disable binary mode which is now deprecated.

From ca083146543d68774ca003e78a7cd9f63b849788 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 3 Feb 2016 21:38:25 -0200
Subject: [PATCH 0835/2974] makes more explicit source of problems in Python 3
 limitations

---
 docs/news.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 5a4049e26..d38fe34ef 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -36,7 +36,7 @@ This 1.1 release brings a lot of interesting features and bug fixes:
     use ``scrapy shell ./index.html`` to load a local file.
   - Robots.txt compliance is now enabled by default for newly-created projects
     (:issue:`1724`). Scrapy will also wait for robots.txt to be downloaded
-    before proceeding with the crawl. (:issue:`1735`). If you want to disable
+    before proceeding with the crawl (:issue:`1735`). If you want to disable
     this behavior, update :setting:`ROBOTSTXT_OBEY` in ``settings.py`` file
     after creating a new project.
   - Exporters now work on unicode, instead of bytes by default (:issue:`1080`).
@@ -60,13 +60,13 @@ features are still missing (and some may never be ported).
 Almost all builtin extensions/middlewares are expected to work. However, we are aware of
 some limitations in Python 3:
 
-- Doesn't work in Windows yet (non-Python 3 ported Twisted dependency)
+- Scrapy doesn't work yet in Windows with Python 3 (non-Python 3 ported Twisted dependency)
 - S3 downloads are not supported (see :issue:`1718`)
 - Sending emails is not supported (non-Python 3 ported Twisted dependency)
 - FTP download handler is not supported (non-Python 3 ported Twisted
   dependency)
 - Telnet is not supported (non-Python 3 ported Twisted dependency)
-- There are problems with non-ASCII URLs in Python 3
+- Scrapy has problems handling non-ASCII URLs in Python 3
 
 Additional New Features and Enhancements
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

From a0b63a7707930069a1f4234156c9b92c76a2cde8 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 3 Feb 2016 21:52:47 -0200
Subject: [PATCH 0836/2974] =?UTF-8?q?Bump=20version:=201.1.0dev1=20?=
 =?UTF-8?q?=E2=86=92=201.1.0rc1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index f64f87e85..4517467d5 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.1.0dev1
+current_version = 1.1.0rc1
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 3ac32d3a6..686366e4b 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.1.0dev1
+1.1.0rc1

From aaccdbb49b6d59e92a73b1d38a086a404cef5fa7 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 3 Feb 2016 21:54:39 -0200
Subject: [PATCH 0837/2974] =?UTF-8?q?Bump=20version:=201.1.0rc1=20?=
 =?UTF-8?q?=E2=86=92=201.2.0dev1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 4517467d5..84180e35b 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.1.0rc1
+current_version = 1.2.0dev1
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 686366e4b..2fd4c7182 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.1.0rc1
+1.2.0dev1

From 6cf4fdbae6cac9b18b257b6004d746f1431788b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 3 Feb 2016 22:53:37 -0300
Subject: [PATCH 0838/2974] Enable travis builds on tag patterns

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index ae9c745ac..c58ab39a5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,6 +5,7 @@ branches:
   only:
     - master
     - /^\d\.\d+$/
+    - /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/
 env:
  - TOXENV=py27
  - TOXENV=precise
@@ -33,6 +34,5 @@ deploy:
     secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA=
   on:
     tags: true
-    all_branches: true
     repo: scrapy/scrapy
-    condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+)?$"
+    condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|dev[0-9]+)?$"

From 8f269558f164b0ad0e18545654ff4c2ff89d437b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 3 Feb 2016 22:58:21 -0300
Subject: [PATCH 0839/2974] =?UTF-8?q?Bump=20version:=201.2.0dev1=20?=
 =?UTF-8?q?=E2=86=92=201.2.0dev2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 84180e35b..39a1c6963 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.2.0dev1
+current_version = 1.2.0dev2
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 2fd4c7182..a25fc280b 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.2.0dev1
+1.2.0dev2

From 061c63592a3678e059fdeb964e6878663698237b Mon Sep 17 00:00:00 2001
From: Nicolas Pennequin <nicolas.pennequin@free.fr>
Date: Mon, 4 Jan 2016 21:26:49 +0100
Subject: [PATCH 0840/2974] MailSender.send: allow passing a charset.

Resolves Issue #348
---
 scrapy/mail.py     |  7 +++++--
 tests/test_mail.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+), 2 deletions(-)

diff --git a/scrapy/mail.py b/scrapy/mail.py
index ad8ecbe13..c6339f25b 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -43,7 +43,7 @@ class MailSender(object):
             settings['MAIL_PASS'], settings.getint('MAIL_PORT'),
             settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL'))
 
-    def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', _callback=None):
+    def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None):
         if attachs:
             msg = MIMEMultipart()
         else:
@@ -57,8 +57,11 @@ class MailSender(object):
             rcpts.extend(cc)
             msg['Cc'] = COMMASPACE.join(cc)
 
+        if charset:
+            msg.set_charset(charset)
+
         if attachs:
-            msg.attach(MIMEText(body))
+            msg.attach(MIMEText(body, 'plain', charset or 'us-ascii'))
             for attach_name, mimetype, f in attachs:
                 part = MIMEBase(*mimetype.split('/'))
                 part.set_payload(f.read())
diff --git a/tests/test_mail.py b/tests/test_mail.py
index 25dd35099..bd7e49621 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -1,5 +1,8 @@
+# coding=utf-8
+
 import unittest
 from io import BytesIO
+from email.charset import Charset
 
 from scrapy.mail import MailSender
 
@@ -54,11 +57,58 @@ class MailSenderTest(unittest.TestCase):
 
         text, attach = payload
         self.assertEqual(text.get_payload(decode=True), b'body')
+        self.assertEqual(text.get_charset(), Charset('us-ascii'))
         self.assertEqual(attach.get_payload(decode=True), b'content')
 
     def _catch_mail_sent(self, **kwargs):
         self.catched_msg = dict(**kwargs)
 
+    def test_send_utf8(self):
+        subject = u'sübjèçt'
+        body = u'bödÿ-àéïöñß'
+        mailsender = MailSender(debug=True)
+        mailsender.send(to=['test@scrapy.org'], subject=subject, body=body,
+                        charset='utf-8', _callback=self._catch_mail_sent)
+
+        assert self.catched_msg
+        self.assertEqual(self.catched_msg['subject'], subject)
+        self.assertEqual(self.catched_msg['body'], body)
+
+        msg = self.catched_msg['msg']
+        self.assertEqual(msg['subject'], subject)
+        self.assertEqual(msg.get_payload(), body)
+        self.assertEqual(msg.get_charset(), Charset('utf-8'))
+        self.assertEqual(msg.get('Content-Type'), 'text/plain; charset="utf-8"')
+
+    def test_send_attach_utf8(self):
+        subject = u'sübjèçt'
+        body = u'bödÿ-àéïöñß'
+        attach = BytesIO()
+        attach.write(body.encode('utf-8'))
+        attach.seek(0)
+        attachs = [('attachment', 'text/plain', attach)]
+
+        mailsender = MailSender(debug=True)
+        mailsender.send(to=['test@scrapy.org'], subject=subject, body=body,
+                        attachs=attachs, charset='utf-8', _callback=self._catch_mail_sent)
+
+        assert self.catched_msg
+        self.assertEqual(self.catched_msg['subject'], subject)
+        self.assertEqual(self.catched_msg['body'], body)
+
+        msg = self.catched_msg['msg']
+        self.assertEqual(msg['subject'], subject)
+        self.assertEqual(msg.get_charset(), Charset('utf-8'))
+        self.assertEqual(msg.get('Content-Type'), 'multipart/mixed; charset="utf-8"')
+
+        payload = msg.get_payload()
+        assert isinstance(payload, list)
+        self.assertEqual(len(payload), 2)
+
+        text, attach = payload
+        self.assertEqual(text.get_payload(decode=True).decode('utf-8'), body)
+        self.assertEqual(text.get_charset(), Charset('utf-8'))
+        self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body)
 
 if __name__ == "__main__":
     unittest.main()

From 6efc7a9de0563cfccc79b27327788e81e5b21200 Mon Sep 17 00:00:00 2001
From: Nicolas Pennequin <nicolas.pennequin@free.fr>
Date: Wed, 6 Jan 2016 22:32:58 +0100
Subject: [PATCH 0841/2974] Update the email doc for the charset argument to
 send().

---
 docs/topics/email.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index 789fbd4fb..96487d865 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -76,7 +76,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
         :param settings: the e-mail recipients
         :type settings: :class:`scrapy.settings.Settings` object
 
-    .. method:: send(to, subject, body, cc=None, attachs=(), mimetype='text/plain')
+    .. method:: send(to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None)
 
         Send email to the given recipients.
 
@@ -102,6 +102,9 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
         :param mimetype: the MIME type of the e-mail
         :type mimetype: str
 
+        :param charset: the character encoding to use for the e-mail contents
+        :type charset: str
+
 
 .. _topics-email-settings:
 

From 25c56159b86288311630cc0cf6db9d755aeeff1e Mon Sep 17 00:00:00 2001
From: orangain <orangain@gmail.com>
Date: Sat, 6 Feb 2016 22:26:46 +0900
Subject: [PATCH 0842/2974] Fix SitemapSpider to extract sitemap urls from
 robots.txt properly

This will fix #1766.
---
 scrapy/spiders/sitemap.py |  2 +-
 tests/test_spider.py      | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index eede467a8..89d96c330 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -32,7 +32,7 @@ class SitemapSpider(Spider):
 
     def _parse_sitemap(self, response):
         if response.url.endswith('/robots.txt'):
-            for url in sitemap_urls_from_robots(response.body):
+            for url in sitemap_urls_from_robots(response.text):
                 yield Request(url, callback=self._parse_sitemap)
         else:
             body = self._get_sitemap_body(response)
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 4d5d4b07e..1d22c1212 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -328,6 +328,18 @@ class SitemapSpiderTest(SpiderTest):
         r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY)
         self.assertSitemapBody(r, self.BODY)
 
+    def test_get_sitemap_urls_from_robotstxt(self):
+        robots = b"""# Sitemap files
+Sitemap: http://example.com/sitemap.xml
+Sitemap: http://example.com/sitemap-product-index.xml
+"""
+
+        r = TextResponse(url="http://www.example.com/robots.txt", body=robots)
+        spider = self.spider_class("example.com")
+        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
+                         ['http://example.com/sitemap.xml',
+                          'http://example.com/sitemap-product-index.xml'])
+
 
 class BaseSpiderDeprecationTest(unittest.TestCase):
 

From 1cb841bfb3c51ed0f6d5408b79af2086c771da86 Mon Sep 17 00:00:00 2001
From: orangain <orangain@gmail.com>
Date: Sun, 7 Feb 2016 09:41:16 +0900
Subject: [PATCH 0843/2974] PY3: Fix TypeError when outputting to stdout

This will fix #1768.
---
 scrapy/extensions/feedexport.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index daea551cb..fce5e251b 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -10,6 +10,7 @@ import logging
 import posixpath
 from tempfile import TemporaryFile
 from datetime import datetime
+import six
 from six.moves.urllib.parse import urlparse
 from ftplib import FTP
 
@@ -57,7 +58,9 @@ class BlockingFeedStorage(object):
 @implementer(IFeedStorage)
 class StdoutFeedStorage(object):
 
-    def __init__(self, uri, _stdout=sys.stdout):
+    def __init__(self, uri, _stdout=None):
+        if not _stdout:
+            _stdout = sys.stdout if six.PY2 else sys.stdout.buffer
         self._stdout = _stdout
 
     def open(self, spider):

From 1f743996ff00a7b728d59b93d0967e1eb50072f0 Mon Sep 17 00:00:00 2001
From: orangain <orangain@gmail.com>
Date: Sun, 7 Feb 2016 14:19:27 +0900
Subject: [PATCH 0844/2974] PY3: Implement some attributes of WrappedRequest
 required in Python 3

This will fix #1770.
---
 scrapy/http/cookies.py     | 22 +++++++++++++++++++---
 tests/test_http_cookies.py |  4 ++++
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index e92c3fe73..a1e95102e 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -137,13 +137,29 @@ class WrappedRequest(object):
         """
         return self.request.meta.get('is_unverifiable', False)
 
-    # python3 uses request.unverifiable
+    def get_origin_req_host(self):
+        return urlparse_cached(self.request).hostname
+
+    # python3 uses attributes instead of methods
+    @property
+    def full_url(self):
+        return self.get_full_url()
+
+    @property
+    def host(self):
+        return self.get_host()
+
+    @property
+    def type(self):
+        return self.get_type()
+
     @property
     def unverifiable(self):
         return self.is_unverifiable()
 
-    def get_origin_req_host(self):
-        return urlparse_cached(self.request).hostname
+    @property
+    def origin_req_host(self):
+        return self.get_origin_req_host()
 
     def has_header(self, name):
         return name in self.request.headers
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index d529f609b..549f779d8 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -14,12 +14,15 @@ class WrappedRequestTest(TestCase):
 
     def test_get_full_url(self):
         self.assertEqual(self.wrapped.get_full_url(), self.request.url)
+        self.assertEqual(self.wrapped.full_url, self.request.url)
 
     def test_get_host(self):
         self.assertEqual(self.wrapped.get_host(), urlparse(self.request.url).netloc)
+        self.assertEqual(self.wrapped.host, urlparse(self.request.url).netloc)
 
     def test_get_type(self):
         self.assertEqual(self.wrapped.get_type(), urlparse(self.request.url).scheme)
+        self.assertEqual(self.wrapped.type, urlparse(self.request.url).scheme)
 
     def test_is_unverifiable(self):
         self.assertFalse(self.wrapped.is_unverifiable())
@@ -32,6 +35,7 @@ class WrappedRequestTest(TestCase):
 
     def test_get_origin_req_host(self):
         self.assertEqual(self.wrapped.get_origin_req_host(), 'www.example.com')
+        self.assertEqual(self.wrapped.origin_req_host, 'www.example.com')
 
     def test_has_header(self):
         self.assertTrue(self.wrapped.has_header('content-type'))

From 164493df2e9f63fd7895ca50677c9f805c89201f Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Sat, 6 Feb 2016 00:08:59 -0200
Subject: [PATCH 0845/2974] add deprecation for pydispatch (thanks for the help
 @redapple)

---
 docs/news.rst                       |  3 +++
 scrapy/xlib/pydispatch.py           | 19 +++++++++++++++++++
 tests/test_pydispatch_deprecated.py | 12 ++++++++++++
 3 files changed, 34 insertions(+)
 create mode 100644 scrapy/xlib/pydispatch.py
 create mode 100644 tests/test_pydispatch_deprecated.py

diff --git a/docs/news.rst b/docs/news.rst
index d38fe34ef..423ec2a40 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -124,6 +124,9 @@ Deprecations and Removals
   + ``scrapy.utils.datatypes.MultiValueDict``
   + ``scrapy.utils.datatypes.SiteNode``
 
+- The previously bundled ``scrapy.xlib.pydispatch`` library was deprecated and
+  replaced by `pydispatcher <https://pypi.python.org/pypi/PyDispatcher>`_.
+
 
 Relocations
 ~~~~~~~~~~~
diff --git a/scrapy/xlib/pydispatch.py b/scrapy/xlib/pydispatch.py
new file mode 100644
index 000000000..5ffeaf579
--- /dev/null
+++ b/scrapy/xlib/pydispatch.py
@@ -0,0 +1,19 @@
+from __future__ import absolute_import
+
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+
+from pydispatch import (
+    dispatcher,
+    errors,
+    robust,
+    robustapply,
+    saferef,
+)
+
+warnings.warn("Importing from scrapy.xlib.pydispatch is deprecated and will"
+              " no longer be supported in future Scrapy versions."
+              " If you just want to connect signals use the from_crawler class method,"
+              " otherwise import pydispatch directly if needed."
+              " See: https://github.com/scrapy/scrapy/issues/1762",
+              ScrapyDeprecationWarning, stacklevel=2)
diff --git a/tests/test_pydispatch_deprecated.py b/tests/test_pydispatch_deprecated.py
new file mode 100644
index 000000000..6d3237fe1
--- /dev/null
+++ b/tests/test_pydispatch_deprecated.py
@@ -0,0 +1,12 @@
+import unittest
+import warnings
+from six.moves import reload_module
+
+
+class DeprecatedPydispatchTest(unittest.TestCase):
+    def test_import_xlib_pydispatch_show_warning(self):
+        with warnings.catch_warnings(record=True) as w:
+            from scrapy.xlib import pydispatch
+            reload_module(pydispatch)
+        self.assertIn('Importing from scrapy.xlib.pydispatch is deprecated',
+                      str(w[0].message))

From eaf3a239e48a503e4e5e4e81d3daf4a0f1f97efe Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 5 Feb 2016 12:52:59 +0300
Subject: [PATCH 0846/2974] using botocore for s3 request signing: proof of
 concept

---
 scrapy/core/downloader/handlers/s3.py | 59 ++++++++++++++++++++-------
 tests/test_downloader_handlers.py     | 39 +++++++++---------
 2 files changed, 65 insertions(+), 33 deletions(-)

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 38cfd1e10..d3feb9815 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,7 +1,9 @@
+import six
 from six.moves.urllib.parse import unquote
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_unicode
 from .http import HTTPDownloadHandler
 
 
@@ -37,10 +39,6 @@ class S3DownloadHandler(object):
     def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
             httpdownloadhandler=HTTPDownloadHandler, **kw):
 
-        _S3Connection = get_s3_connection()
-        if _S3Connection is None:
-            raise NotConfigured("missing boto library")
-
         if not aws_access_key_id:
             aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
         if not aws_secret_access_key:
@@ -53,10 +51,27 @@ class S3DownloadHandler(object):
         if anon is None and not aws_access_key_id and not aws_secret_access_key:
             kw['anon'] = True
 
+        self._signer = None
         try:
-            self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key, **kw)
-        except Exception as ex:
-            raise NotConfigured(str(ex))
+            import botocore.auth
+            import botocore.credentials
+        except ImportError:
+            if six.PY3:
+                raise NotConfigured("missing botocore library")
+            _S3Connection = get_s3_connection()
+            if _S3Connection is None:
+                raise NotConfigured("missing botocore or boto library")
+            try:
+                self.conn = _S3Connection(
+                    aws_access_key_id, aws_secret_access_key, **kw)
+            except Exception as ex:
+                raise NotConfigured(str(ex))
+        else:
+            SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
+            # TODO - anon
+            self._signer = SignerCls(botocore.credentials.Credentials(
+                aws_access_key_id, aws_secret_access_key))
+
         self._download_http = httpdownloadhandler(settings).download_request
 
     def download_request(self, request, spider):
@@ -65,12 +80,28 @@ class S3DownloadHandler(object):
         bucket = p.hostname
         path = p.path + '?' + p.query if p.query else p.path
         url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path)
-        signed_headers = self.conn.make_request(
+        if self._signer is not None:
+            import botocore.awsrequest
+            from botocore.vendored.requests.structures import CaseInsensitiveDict
+            print(url, request.headers)
+            awsrequest = botocore.awsrequest.AWSRequest(
                 method=request.method,
-                bucket=bucket,
-                key=unquote(p.path),
-                query_args=unquote(p.query),
-                headers=request.headers,
+                url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path),
+                # TODO - move to a header method
+                headers=CaseInsensitiveDict(
+                    (to_unicode(key), to_unicode(b','.join(value)))
+                    for key, value in request.headers.items()),
                 data=request.body)
-        httpreq = request.replace(url=url, headers=signed_headers)
-        return self._download_http(httpreq, spider)
+            self._signer.add_auth(awsrequest)
+            request = request.replace(
+                url=url, headers=awsrequest.headers.items())
+        else:
+            signed_headers = self.conn.make_request(
+                    method=request.method,
+                    bucket=bucket,
+                    key=unquote(p.path),
+                    query_args=unquote(p.query),
+                    headers=request.headers,
+                    data=request.body)
+            request = request.replace(url=url, headers=signed_headers)
+        return self._download_http(request, spider)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 56608bfc6..06e232503 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -432,13 +432,20 @@ class HttpDownloadHandlerMock(object):
         return request
 
 
-class S3AnonTestCase(unittest.TestCase):
-    try:
-        import boto
-    except ImportError:
-        skip = 'missing boto library'
+class BaseS3TestCase(unittest.TestCase):
     if six.PY3:
-        skip = 'S3 not supported on Py3'
+        try:
+            import botocore
+        except ImportError:
+            skip = 'missing botocore library'
+    else:
+        try:
+            import boto
+        except ImportError:
+            skip = 'missing boto library'
+
+
+class S3AnonTestCase(BaseS3TestCase):
 
     def setUp(self):
         self.s3reqh = S3DownloadHandler(Settings(),
@@ -457,12 +464,6 @@ class S3AnonTestCase(unittest.TestCase):
 
 class S3TestCase(unittest.TestCase):
     download_handler_cls = S3DownloadHandler
-    try:
-        import boto
-    except ImportError:
-        skip = 'missing boto library'
-    if six.PY3:
-        skip = 'S3 not supported on Py3'
 
     # test use same example keys than amazon developer guide
     # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf
@@ -484,7 +485,7 @@ class S3TestCase(unittest.TestCase):
                 headers={'Date': 'Tue, 27 Mar 2007 19:36:42 +0000'})
         httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
-                'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
+                b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
 
     def test_request_signing2(self):
         # puts an object into the johnsmith bucket.
@@ -495,7 +496,7 @@ class S3TestCase(unittest.TestCase):
             })
         httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
-                'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
+                b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
 
     def test_request_signing3(self):
         # lists the content of the johnsmith bucket.
@@ -506,7 +507,7 @@ class S3TestCase(unittest.TestCase):
                     })
         httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
-                'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
+                b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
 
     def test_request_signing4(self):
         # fetches the access control policy sub-resource for the 'johnsmith' bucket.
@@ -514,7 +515,7 @@ class S3TestCase(unittest.TestCase):
                 method='GET', headers={'Date': 'Tue, 27 Mar 2007 19:44:46 +0000'})
         httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
-                'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
+                b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
 
     def test_request_signing5(self):
         # deletes an object from the 'johnsmith' bucket using the
@@ -526,7 +527,7 @@ class S3TestCase(unittest.TestCase):
                     })
         httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
-                'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
+                b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
 
     def test_request_signing6(self):
         # uploads an object to a CNAME style virtual hosted bucket with metadata.
@@ -547,7 +548,7 @@ class S3TestCase(unittest.TestCase):
                     })
         httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
-                'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
+                b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
 
     def test_request_signing7(self):
         # ensure that spaces are quoted properly before signing
@@ -561,7 +562,7 @@ class S3TestCase(unittest.TestCase):
         httpreq = self.download_request(req, self.spider)
         self.assertEqual(
             httpreq.headers['Authorization'],
-            'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')
+            b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')
 
 
 class FTPTestCase(unittest.TestCase):

From 467553cc2922d1d844fa136a036716eef3158eb9 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 5 Feb 2016 13:30:22 +0300
Subject: [PATCH 0847/2974] fix anon test: in this case we do no signing, just
 change the url

---
 scrapy/core/downloader/handlers/s3.py | 7 +++++--
 tests/test_downloader_handlers.py     | 6 ++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index d3feb9815..6d28f866e 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -47,9 +47,10 @@ class S3DownloadHandler(object):
         # If no credentials could be found anywhere,
         # consider this an anonymous connection request by default;
         # unless 'anon' was set explicitly (True/False).
-        anon = kw.get('anon', None)
+        anon = kw.get('anon')
         if anon is None and not aws_access_key_id and not aws_secret_access_key:
             kw['anon'] = True
+        self.anon = kw.get('anon')
 
         self._signer = None
         try:
@@ -80,7 +81,9 @@ class S3DownloadHandler(object):
         bucket = p.hostname
         path = p.path + '?' + p.query if p.query else p.path
         url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path)
-        if self._signer is not None:
+        if self.anon:
+            request = request.replace(url=url)
+        elif self._signer is not None:
             import botocore.awsrequest
             from botocore.vendored.requests.structures import CaseInsensitiveDict
             print(url, request.headers)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 06e232503..0f79a208d 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -458,8 +458,10 @@ class S3AnonTestCase(BaseS3TestCase):
     def test_anon_request(self):
         req = Request('s3://aws-publicdatasets/')
         httpreq = self.download_request(req, self.spider)
-        self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True)
-        self.assertEqual(self.s3reqh.conn.anon, True)
+        self.assertEqual(hasattr(self.s3reqh, 'anon'), True)
+        self.assertEqual(self.s3reqh.anon, True)
+        self.assertEqual(
+            httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/')
 
 
 class S3TestCase(unittest.TestCase):

From 1b1092b7d073320531986b35461ca5633dc2829a Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 5 Feb 2016 14:19:04 +0300
Subject: [PATCH 0848/2974] add Headers.to_native_string_dict - useful when
 interfacing with other libraries

---
 scrapy/core/downloader/handlers/s3.py | 7 +------
 scrapy/http/headers.py                | 7 +++++++
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 6d28f866e..0903b84ad 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -85,15 +85,10 @@ class S3DownloadHandler(object):
             request = request.replace(url=url)
         elif self._signer is not None:
             import botocore.awsrequest
-            from botocore.vendored.requests.structures import CaseInsensitiveDict
-            print(url, request.headers)
             awsrequest = botocore.awsrequest.AWSRequest(
                 method=request.method,
                 url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path),
-                # TODO - move to a header method
-                headers=CaseInsensitiveDict(
-                    (to_unicode(key), to_unicode(b','.join(value)))
-                    for key, value in request.headers.items()),
+                headers=request.headers.to_native_string_dict(),
                 data=request.body)
             self._signer.add_auth(awsrequest)
             request = request.replace(
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 13f0f0383..d0c4cd0fb 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,6 +1,7 @@
 import six
 from w3lib.http import headers_dict_to_raw
 from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.python import to_unicode
 
 
 class Headers(CaselessDict):
@@ -78,6 +79,12 @@ class Headers(CaselessDict):
     def to_string(self):
         return headers_dict_to_raw(self)
 
+    def to_native_string_dict(self):
+        return CaselessDict(
+            (to_unicode(key, encoding=self.encoding),
+             to_unicode(b','.join(value), encoding=self.encoding))
+            for key, value in self.items())
+
     def __copy__(self):
         return self.__class__(self)
     copy = __copy__

From c3fec83e7eaeaa1d59a479b70857ccaf98a54e35 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 5 Feb 2016 14:46:56 +0300
Subject: [PATCH 0849/2974] use botocore by default, boto is still used in
 "precise" env

---
 tests/requirements-py3.txt | 2 +-
 tox.ini                    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 73e73e651..2a89763a5 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -4,7 +4,7 @@ pytest-cov
 testfixtures
 jmespath
 leveldb
-boto
+botocore
 # optional for shell wrapper tests
 bpython
 ipython
diff --git a/tox.ini b/tox.ini
index b8d45d5b9..fb31762d8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -10,7 +10,7 @@ envlist = py27
 deps =
     -rrequirements.txt
     # Extras
-    boto
+    botocore
     Pillow != 3.0.0
     leveldb
     -rtests/requirements.txt

From 7748ee6bba8eacb889ce68cd6ced273255b8b9a9 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 5 Feb 2016 14:52:03 +0300
Subject: [PATCH 0850/2974] mock date in s3 tests when using botocore

---
 scrapy/core/downloader/handlers/s3.py |  8 +--
 tests/test_downloader_handlers.py     | 93 ++++++++++++++++++---------
 2 files changed, 67 insertions(+), 34 deletions(-)

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 0903b84ad..cb2bb46b1 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -68,10 +68,10 @@ class S3DownloadHandler(object):
             except Exception as ex:
                 raise NotConfigured(str(ex))
         else:
-            SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
-            # TODO - anon
-            self._signer = SignerCls(botocore.credentials.Credentials(
-                aws_access_key_id, aws_secret_access_key))
+            if not self.anon:
+                SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
+                self._signer = SignerCls(botocore.credentials.Credentials(
+                    aws_access_key_id, aws_secret_access_key))
 
         self._download_http = httpdownloadhandler(settings).download_request
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 0f79a208d..6c4d2e0db 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1,5 +1,10 @@
 import os
 import six
+import contextlib
+try:
+    from unittest import mock
+except ImportError:
+    import mock
 
 from twisted.trial import unittest
 from twisted.protocols.policies import WrappingFactory
@@ -433,16 +438,16 @@ class HttpDownloadHandlerMock(object):
 
 
 class BaseS3TestCase(unittest.TestCase):
-    if six.PY3:
-        try:
-            import botocore
-        except ImportError:
+    try:
+        import botocore
+    except ImportError:
+        if six.PY2:
+            try:
+                import boto
+            except ImportError:
+                skip = 'missing botocore or boto library'
+        else:
             skip = 'missing botocore library'
-    else:
-        try:
-            import boto
-        except ImportError:
-            skip = 'missing boto library'
 
 
 class S3AnonTestCase(BaseS3TestCase):
@@ -464,7 +469,7 @@ class S3AnonTestCase(BaseS3TestCase):
             httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/')
 
 
-class S3TestCase(unittest.TestCase):
+class S3TestCase(BaseS3TestCase):
     download_handler_cls = S3DownloadHandler
 
     # test use same example keys than amazon developer guide
@@ -481,63 +486,89 @@ class S3TestCase(unittest.TestCase):
         self.download_request = s3reqh.download_request
         self.spider = Spider('foo')
 
+    @contextlib.contextmanager
+    def _mocked_date(self, date):
+        try:
+            import botocore.auth
+        except ImportError:
+            yield
+        else:
+            # We need to mock botocore.auth.formatdate, because otherwise
+            # botocore overrides Date header with current date and time
+            # and Authorization header is different each time
+            with mock.patch('botocore.auth.formatdate') as mock_formatdate:
+                mock_formatdate.return_value = date
+                yield
+
     def test_request_signing1(self):
         # gets an object from the johnsmith bucket.
-        req = Request('s3://johnsmith/photos/puppy.jpg',
-                headers={'Date': 'Tue, 27 Mar 2007 19:36:42 +0000'})
-        httpreq = self.download_request(req, self.spider)
+        date ='Tue, 27 Mar 2007 19:36:42 +0000'
+        req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date})
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
                 b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
 
     def test_request_signing2(self):
         # puts an object into the johnsmith bucket.
+        date = 'Tue, 27 Mar 2007 21:15:45 +0000'
         req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={
             'Content-Type': 'image/jpeg',
-            'Date': 'Tue, 27 Mar 2007 21:15:45 +0000',
+            'Date': date,
             'Content-Length': '94328',
             })
-        httpreq = self.download_request(req, self.spider)
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
                 b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
 
     def test_request_signing3(self):
         # lists the content of the johnsmith bucket.
+        date = 'Tue, 27 Mar 2007 19:42:41 +0000'
         req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \
                 method='GET', headers={
                     'User-Agent': 'Mozilla/5.0',
-                    'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
+                    'Date': date,
                     })
-        httpreq = self.download_request(req, self.spider)
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
                 b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
 
     def test_request_signing4(self):
         # fetches the access control policy sub-resource for the 'johnsmith' bucket.
-        req = Request('s3://johnsmith/?acl', \
-                method='GET', headers={'Date': 'Tue, 27 Mar 2007 19:44:46 +0000'})
-        httpreq = self.download_request(req, self.spider)
+        date = 'Tue, 27 Mar 2007 19:44:46 +0000'
+        req = Request('s3://johnsmith/?acl',
+            method='GET', headers={'Date': date})
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
                 b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
 
     def test_request_signing5(self):
         # deletes an object from the 'johnsmith' bucket using the
         # path-style and Date alternative.
+        date = 'Tue, 27 Mar 2007 21:20:27 +0000'
         req = Request('s3://johnsmith/photos/puppy.jpg', \
                 method='DELETE', headers={
-                    'Date': 'Tue, 27 Mar 2007 21:20:27 +0000',
+                    'Date': date,
                     'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
                     })
-        httpreq = self.download_request(req, self.spider)
-        self.assertEqual(httpreq.headers['Authorization'], \
-                b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
+        # botocore does not override Date with x-amz-date
+        self.assertIn(httpreq.headers['Authorization'], [
+                b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=',
+                b'AWS 0PN5J17HBGZHT7JJ3X82:otYM2krxnuHhAofO4oqIV7wcfdU='])
 
     def test_request_signing6(self):
         # uploads an object to a CNAME style virtual hosted bucket with metadata.
+        date = 'Tue, 27 Mar 2007 21:06:08 +0000'
         req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \
                 method='PUT', headers={
                     'User-Agent': 'curl/7.15.5',
                     'Host': 'static.johnsmith.net:8080',
-                    'Date': 'Tue, 27 Mar 2007 21:06:08 +0000',
+                    'Date': date,
                     'x-amz-acl': 'public-read',
                     'content-type': 'application/x-download',
                     'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',
@@ -548,20 +579,22 @@ class S3TestCase(unittest.TestCase):
                     'Content-Encoding': 'gzip',
                     'Content-Length': '5913339',
                     })
-        httpreq = self.download_request(req, self.spider)
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
                 b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
 
     def test_request_signing7(self):
         # ensure that spaces are quoted properly before signing
+        date = 'Tue, 27 Mar 2007 19:42:41 +0000'
         req = Request(
             ("s3://johnsmith/photos/my puppy.jpg"
              "?response-content-disposition=my puppy.jpg"),
             method='GET',
-            headers={
-                'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
-            })
-        httpreq = self.download_request(req, self.spider)
+            headers={'Date': date},
+            )
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(
             httpreq.headers['Authorization'],
             b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')

From d6bea3bf2eb4793555366a3341fe41456704b860 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 5 Feb 2016 15:17:59 +0300
Subject: [PATCH 0851/2974] botocore not only does not allow passing our own
 Date header, but does not handle x-amz-date according to the spec

---
 tests/test_downloader_handlers.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 6c4d2e0db..57225ee3d 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -546,6 +546,11 @@ class S3TestCase(BaseS3TestCase):
                 b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
 
     def test_request_signing5(self):
+        try: import botocore
+        except ImportError: pass
+        else:
+            raise unittest.SkipTest(
+                'botocore does not support overriding date with x-amz-date')
         # deletes an object from the 'johnsmith' bucket using the
         # path-style and Date alternative.
         date = 'Tue, 27 Mar 2007 21:20:27 +0000'
@@ -557,9 +562,8 @@ class S3TestCase(BaseS3TestCase):
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
         # botocore does not override Date with x-amz-date
-        self.assertIn(httpreq.headers['Authorization'], [
-                b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=',
-                b'AWS 0PN5J17HBGZHT7JJ3X82:otYM2krxnuHhAofO4oqIV7wcfdU='])
+        self.assertEqual(httpreq.headers['Authorization'],
+                b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
 
     def test_request_signing6(self):
         # uploads an object to a CNAME style virtual hosted bucket with metadata.

From bcb92b50dc1d1106ee418d6d7a701d87cde4010c Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 5 Feb 2016 17:38:47 +0300
Subject: [PATCH 0852/2974] check that no extra kwargs are silently discarded

---
 scrapy/core/downloader/handlers/s3.py | 3 +++
 tests/test_downloader_handlers.py     | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index cb2bb46b1..e218a8741 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -68,6 +68,9 @@ class S3DownloadHandler(object):
             except Exception as ex:
                 raise NotConfigured(str(ex))
         else:
+            kw.pop('anon', None)
+            if kw:
+                raise TypeError('Unexpected keyword arguments: %s' % kw)
             if not self.anon:
                 SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
                 self._signer = SignerCls(botocore.credentials.Credentials(
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 57225ee3d..c0342b806 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -500,6 +500,10 @@ class S3TestCase(BaseS3TestCase):
                 mock_formatdate.return_value = date
                 yield
 
+    def test_extra_kw(self):
+        with self.assertRaises(TypeError):
+            S3DownloadHandler(Settings(), extra_kw=True)
+
     def test_request_signing1(self):
         # gets an object from the johnsmith bucket.
         date ='Tue, 27 Mar 2007 19:36:42 +0000'

From 408bc1580b73c958b2a83785817ef72a1d642198 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 15 Feb 2016 16:20:13 +0300
Subject: [PATCH 0853/2974] Pass env variables required for running tests
 against real s3 via tox.

---
 tox.ini | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tox.ini b/tox.ini
index fb31762d8..4d8236b4e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -14,6 +14,10 @@ deps =
     Pillow != 3.0.0
     leveldb
     -rtests/requirements.txt
+passenv =
+    FEEDTEST_S3_URI
+    AWS_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY
 commands =
     py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
 

From 19b2910ad145cb4b86ed621e7045f0afdf810d7f Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 15 Feb 2016 16:25:29 +0300
Subject: [PATCH 0854/2974] Fix assert_aws_environ: check for botocore with
 boto fallback on PY2

---
 scrapy/utils/test.py              | 19 ++++++++++++++-----
 tests/test_downloader_handlers.py | 21 +++++----------------
 2 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 51edfd353..1ac2e575f 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -5,6 +5,7 @@ This module contains some assorted functions used in tests
 import os
 
 from importlib import import_module
+import six
 from twisted.trial.unittest import SkipTest
 
 
@@ -12,14 +13,22 @@ def assert_aws_environ():
     """Asserts the current environment is suitable for running AWS testsi.
     Raises SkipTest with the reason if it's not.
     """
-    try:
-        import boto
-    except ImportError as e:
-        raise SkipTest(str(e))
-
+    skip_if_no_boto()
     if 'AWS_ACCESS_KEY_ID' not in os.environ:
         raise SkipTest("AWS keys not found")
 
+def skip_if_no_boto():
+    try:
+        import botocore
+    except ImportError:
+        if six.PY2:
+            try:
+                import boto
+            except ImportError:
+                raise SkipTest('missing botocore or boto library')
+        else:
+            raise SkipTest('missing botocore library')
+
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index c0342b806..f34a286c2 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -28,7 +28,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.settings import Settings
-from scrapy.utils.test import get_crawler
+from scrapy.utils.test import get_crawler, skip_if_no_boto
 from scrapy.utils.python import to_bytes
 from scrapy.exceptions import NotConfigured
 
@@ -437,22 +437,10 @@ class HttpDownloadHandlerMock(object):
         return request
 
 
-class BaseS3TestCase(unittest.TestCase):
-    try:
-        import botocore
-    except ImportError:
-        if six.PY2:
-            try:
-                import boto
-            except ImportError:
-                skip = 'missing botocore or boto library'
-        else:
-            skip = 'missing botocore library'
-
-
-class S3AnonTestCase(BaseS3TestCase):
+class S3AnonTestCase(unittest.TestCase):
 
     def setUp(self):
+        skip_if_no_boto()
         self.s3reqh = S3DownloadHandler(Settings(),
                 httpdownloadhandler=HttpDownloadHandlerMock,
                 #anon=True, # is implicit
@@ -469,7 +457,7 @@ class S3AnonTestCase(BaseS3TestCase):
             httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/')
 
 
-class S3TestCase(BaseS3TestCase):
+class S3TestCase(unittest.TestCase):
     download_handler_cls = S3DownloadHandler
 
     # test use same example keys than amazon developer guide
@@ -480,6 +468,7 @@ class S3TestCase(BaseS3TestCase):
     AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'
 
     def setUp(self):
+        skip_if_no_boto()
         s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID,
                 self.AWS_SECRET_ACCESS_KEY,
                 httpdownloadhandler=HttpDownloadHandlerMock)

From 5d2f067458ce8a3e39d717afae2034dace5db54c Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 15 Feb 2016 16:36:15 +0300
Subject: [PATCH 0855/2974] S3FeedStorageTest: delete key after test

---
 tests/test_feedexport.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 176fd93e3..8015b0320 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -102,8 +102,10 @@ class S3FeedStorageTest(unittest.TestCase):
         file.write("content")
         yield storage.store(file)
         u = urlparse(uri)
-        key = connect_s3().get_bucket(u.hostname, validate=False).get_key(u.path)
+        bucket = connect_s3().get_bucket(u.hostname, validate=False)
+        key = bucket.get_key(u.path)
         self.assertEqual(key.get_contents_as_string(), "content")
+        bucket.delete_key(u.path)
 
 
 class StdoutFeedStorageTest(unittest.TestCase):

From 3ada45a9bb6adcfe5546a515a0574e7efe94c720 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 15 Feb 2016 17:27:34 +0300
Subject: [PATCH 0856/2974] S3FeedStorageTest: add botocore support, and
 organize boto/botocore checks

---
 scrapy/core/downloader/handlers/s3.py | 31 +++++++++-----------------
 scrapy/extensions/feedexport.py       | 32 ++++++++++++++++++---------
 scrapy/utils/boto.py                  | 20 +++++++++++++++++
 scrapy/utils/test.py                  | 16 +++++---------
 tests/test_feedexport.py              | 24 +++++++++++++++-----
 5 files changed, 77 insertions(+), 46 deletions(-)
 create mode 100644 scrapy/utils/boto.py

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index e218a8741..dd7bce2be 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,17 +1,13 @@
-import six
 from six.moves.urllib.parse import unquote
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_unicode
+from scrapy.utils.boto import is_botocore
 from .http import HTTPDownloadHandler
 
 
-def get_s3_connection():
-    try:
-        from boto.s3.connection import S3Connection
-    except ImportError:
-        return None
+def _get_boto_connection():
+    from boto.s3.connection import S3Connection
 
     class _v19_S3Connection(S3Connection):
         """A dummy S3Connection wrapper that doesn't do any synchronous download"""
@@ -53,21 +49,9 @@ class S3DownloadHandler(object):
         self.anon = kw.get('anon')
 
         self._signer = None
-        try:
+        if is_botocore():
             import botocore.auth
             import botocore.credentials
-        except ImportError:
-            if six.PY3:
-                raise NotConfigured("missing botocore library")
-            _S3Connection = get_s3_connection()
-            if _S3Connection is None:
-                raise NotConfigured("missing botocore or boto library")
-            try:
-                self.conn = _S3Connection(
-                    aws_access_key_id, aws_secret_access_key, **kw)
-            except Exception as ex:
-                raise NotConfigured(str(ex))
-        else:
             kw.pop('anon', None)
             if kw:
                 raise TypeError('Unexpected keyword arguments: %s' % kw)
@@ -75,6 +59,13 @@ class S3DownloadHandler(object):
                 SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
                 self._signer = SignerCls(botocore.credentials.Credentials(
                     aws_access_key_id, aws_secret_access_key))
+        else:
+            _S3Connection = _get_boto_connection()
+            try:
+                self.conn = _S3Connection(
+                    aws_access_key_id, aws_secret_access_key, **kw)
+            except Exception as ex:
+                raise NotConfigured(str(ex))
 
         self._download_http = httpdownloadhandler(settings).download_request
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index fce5e251b..3dab2d77e 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -24,6 +24,7 @@ from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.python import without_none_values
+from scrapy.utils.boto import is_botocore
 
 logger = logging.getLogger(__name__)
 
@@ -90,24 +91,33 @@ class S3FeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri):
         from scrapy.conf import settings
-        try:
-            import boto
-        except ImportError:
-            raise NotConfigured
-        self.connect_s3 = boto.connect_s3
         u = urlparse(uri)
         self.bucketname = u.hostname
         self.access_key = u.username or settings['AWS_ACCESS_KEY_ID']
         self.secret_key = u.password or settings['AWS_SECRET_ACCESS_KEY']
-        self.keyname = u.path
+        self.is_botocore = is_botocore()
+        self.keyname = u.path[1:]  # remove first "/"
+        if self.is_botocore:
+            import botocore.session
+            session = botocore.session.get_session()
+            self.s3_client = session.create_client(
+                's3', aws_access_key_id=self.access_key,
+                aws_secret_access_key=self.secret_key)
+        else:
+            import boto
+            self.connect_s3 = boto.connect_s3
 
     def _store_in_thread(self, file):
         file.seek(0)
-        conn = self.connect_s3(self.access_key, self.secret_key)
-        bucket = conn.get_bucket(self.bucketname, validate=False)
-        key = bucket.new_key(self.keyname)
-        key.set_contents_from_file(file)
-        key.close()
+        if self.is_botocore:
+            self.s3_client.put_object(
+                Bucket=self.bucketname, Key=self.keyname, Body=file)
+        else:
+            conn = self.connect_s3(self.access_key, self.secret_key)
+            bucket = conn.get_bucket(self.bucketname, validate=False)
+            key = bucket.new_key(self.keyname)
+            key.set_contents_from_file(file)
+            key.close()
 
 
 class FTPFeedStorage(BlockingFeedStorage):
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
new file mode 100644
index 000000000..fd6b22b88
--- /dev/null
+++ b/scrapy/utils/boto.py
@@ -0,0 +1,20 @@
+"""Boto/botocore helpers"""
+
+import six
+
+from scrapy.exceptions import NotConfigured
+
+
+def is_botocore():
+    try:
+        import botocore
+        return True
+    except ImportError:
+        if six.PY2:
+            try:
+                import boto
+                return False
+            except ImportError:
+                raise NotConfigured('missing botocore or boto library')
+        else:
+            raise NotConfigured('missing botocore library')
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 1ac2e575f..d2f7c0ae4 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -5,9 +5,11 @@ This module contains some assorted functions used in tests
 import os
 
 from importlib import import_module
-import six
 from twisted.trial.unittest import SkipTest
 
+from scrapy.exceptions import NotConfigured
+from scrapy.utils.boto import is_botocore
+
 
 def assert_aws_environ():
     """Asserts the current environment is suitable for running AWS testsi.
@@ -19,15 +21,9 @@ def assert_aws_environ():
 
 def skip_if_no_boto():
     try:
-        import botocore
-    except ImportError:
-        if six.PY2:
-            try:
-                import boto
-            except ImportError:
-                raise SkipTest('missing botocore or boto library')
-        else:
-            raise SkipTest('missing botocore library')
+        is_botocore()
+    except NotConfigured as e:
+        raise SkipTest(e.message)
 
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 8015b0320..beb800fb5 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -22,6 +22,7 @@ from scrapy.extensions.feedexport import (
 )
 from scrapy.utils.test import assert_aws_environ
 from scrapy.utils.python import to_native_str
+from scrapy.utils.boto import is_botocore
 
 
 class FileFeedStorageTest(unittest.TestCase):
@@ -95,17 +96,30 @@ class S3FeedStorageTest(unittest.TestCase):
         uri = os.environ.get('FEEDTEST_S3_URI')
         if not uri:
             raise unittest.SkipTest("No S3 URI available for testing")
-        from boto import connect_s3
         storage = S3FeedStorage(uri)
         verifyObject(IFeedStorage, storage)
         file = storage.open(scrapy.Spider("default"))
         file.write("content")
         yield storage.store(file)
         u = urlparse(uri)
-        bucket = connect_s3().get_bucket(u.hostname, validate=False)
-        key = bucket.get_key(u.path)
-        self.assertEqual(key.get_contents_as_string(), "content")
-        bucket.delete_key(u.path)
+        content = self._get_content_and_delete(u.hostname, u.path[1:])
+        self.assertEqual(content, "content")
+
+    def _get_content_and_delete(self, bucket, path):
+        if is_botocore():
+            import botocore.session
+            session = botocore.session.get_session()
+            client = session.create_client('s3')
+            key = client.get_object(Bucket=bucket, Key=path)
+            content = key['Body'].read()
+            client.delete_object(Bucket=bucket, Key=path)
+        else:
+            from boto import connect_s3
+            bucket = connect_s3().get_bucket(bucket, validate=False)
+            key = bucket.get_key(path)
+            content = key.get_contents_as_string()
+            bucket.delete_key(path)
+        return content
 
 
 class StdoutFeedStorageTest(unittest.TestCase):

From d1470e85a2987ad53c3b3a89d801a846a053e133 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 15 Feb 2016 17:28:50 +0300
Subject: [PATCH 0857/2974] S3FeedStorageTest: pass on py3, add some non-ascii
 content to be sure

---
 tests/test_feedexport.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index beb800fb5..f3cf1c2cb 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -99,11 +99,12 @@ class S3FeedStorageTest(unittest.TestCase):
         storage = S3FeedStorage(uri)
         verifyObject(IFeedStorage, storage)
         file = storage.open(scrapy.Spider("default"))
-        file.write("content")
+        expected_content = b"content: \xe2\x98\x83"
+        file.write(expected_content)
         yield storage.store(file)
         u = urlparse(uri)
         content = self._get_content_and_delete(u.hostname, u.path[1:])
-        self.assertEqual(content, "content")
+        self.assertEqual(content, expected_content)
 
     def _get_content_and_delete(self, bucket, path):
         if is_botocore():

From 32cd8c91654a8442495cc00aebf5322d1fbc644b Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 15 Feb 2016 17:50:47 +0300
Subject: [PATCH 0858/2974] add direct test for S3FilesStore

---
 scrapy/utils/test.py         | 20 ++++++++++++++++++++
 tests/test_feedexport.py     | 22 +++-------------------
 tests/test_pipeline_files.py | 27 ++++++++++++++++++++++++++-
 tox.ini                      |  2 +-
 4 files changed, 50 insertions(+), 21 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index d2f7c0ae4..43abd64a0 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -2,6 +2,7 @@
 This module contains some assorted functions used in tests
 """
 
+from __future__ import absolute_import
 import os
 
 from importlib import import_module
@@ -25,6 +26,25 @@ def skip_if_no_boto():
     except NotConfigured as e:
         raise SkipTest(e.message)
 
+def get_s3_content_and_delete(bucket, path):
+    """ Get content from s3 key, and delete key afterwards.
+    """
+    if is_botocore():
+        import botocore.session
+        session = botocore.session.get_session()
+        client = session.create_client('s3')
+        key = client.get_object(Bucket=bucket, Key=path)
+        content = key['Body'].read()
+        client.delete_object(Bucket=bucket, Key=path)
+    else:
+        import boto
+        # assuming boto=2.2.2
+        bucket = boto.connect_s3().get_bucket(bucket, validate=False)
+        key = bucket.get_key(path)
+        content = key.get_contents_as_string()
+        bucket.delete_key(path)
+    return content
+
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index f3cf1c2cb..fd2f5a2ba 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -20,7 +20,7 @@ from scrapy.extensions.feedexport import (
     IFeedStorage, FileFeedStorage, FTPFeedStorage,
     S3FeedStorage, StdoutFeedStorage
 )
-from scrapy.utils.test import assert_aws_environ
+from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
 from scrapy.utils.python import to_native_str
 from scrapy.utils.boto import is_botocore
 
@@ -93,7 +93,7 @@ class S3FeedStorageTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_store(self):
         assert_aws_environ()
-        uri = os.environ.get('FEEDTEST_S3_URI')
+        uri = os.environ.get('S3_TEST_FILE_URI')
         if not uri:
             raise unittest.SkipTest("No S3 URI available for testing")
         storage = S3FeedStorage(uri)
@@ -103,25 +103,9 @@ class S3FeedStorageTest(unittest.TestCase):
         file.write(expected_content)
         yield storage.store(file)
         u = urlparse(uri)
-        content = self._get_content_and_delete(u.hostname, u.path[1:])
+        content = get_s3_content_and_delete(u.hostname, u.path[1:])
         self.assertEqual(content, expected_content)
 
-    def _get_content_and_delete(self, bucket, path):
-        if is_botocore():
-            import botocore.session
-            session = botocore.session.get_session()
-            client = session.create_client('s3')
-            key = client.get_object(Bucket=bucket, Key=path)
-            content = key['Body'].read()
-            client.delete_object(Bucket=bucket, Key=path)
-        else:
-            from boto import connect_s3
-            bucket = connect_s3().get_bucket(bucket, validate=False)
-            key = bucket.get_key(path)
-            content = key.get_contents_as_string()
-            bucket.delete_key(path)
-        return content
-
 
 class StdoutFeedStorageTest(unittest.TestCase):
 
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index c9977f5ca..6ea47086f 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -4,15 +4,18 @@ import hashlib
 import warnings
 from tempfile import mkdtemp
 from shutil import rmtree
+from six.moves.urllib.parse import urlparse
+from six import BytesIO
 
 from twisted.trial import unittest
 from twisted.internet import defer
 
-from scrapy.pipelines.files import FilesPipeline, FSFilesStore
+from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore
 from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes
+from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
 
 from tests import mock
 
@@ -179,6 +182,28 @@ class FilesPipelineTestCaseFields(unittest.TestCase):
             self.assertEqual(item['stored_file'], [results[0][1]])
 
 
+class TestS3FilesStore(unittest.TestCase):
+    @defer.inlineCallbacks
+    def test_persist(self):
+        assert_aws_environ()
+        uri = os.environ.get('S3_TEST_FILE_URI')
+        if not uri:
+            raise unittest.SkipTest("No S3 URI available for testing")
+        data = b"TestS3FilesStore: \xe2\x98\x83"
+        buf = BytesIO(data)
+        meta = {'foo': 'bar'}
+        path = ''
+        store = S3FilesStore(uri)
+        yield store.persist_file(path, buf, info=None, meta=meta)
+        s = yield store.stat_file(path, info=None)
+        self.assertIn('last_modified', s)
+        self.assertIn('checksum', s)
+        self.assertEqual(s['checksum'], b'3187896a9657a28163abb31667df64c8')
+        u = urlparse(uri)
+        content = get_s3_content_and_delete(u.hostname, u.path[1:])
+        self.assertEqual(content, data)
+
+
 class ItemWithFiles(Item):
     file_urls = Field()
     files = Field()
diff --git a/tox.ini b/tox.ini
index 4d8236b4e..2a8067618 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,7 +15,7 @@ deps =
     leveldb
     -rtests/requirements.txt
 passenv =
-    FEEDTEST_S3_URI
+    S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY
 commands =

From cfc567f48e934dc5b148c681c11a73f974e0d37c Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 15 Feb 2016 19:12:55 +0300
Subject: [PATCH 0859/2974] botocore support for S3FilesStore

---
 scrapy/pipelines/files.py | 65 +++++++++++++++++++++++++++------------
 tests/test_feedexport.py  |  1 -
 2 files changed, 46 insertions(+), 20 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index e4011d31d..c757b0a3f 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -28,6 +28,7 @@ from scrapy.utils.misc import md5sum
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.python import to_bytes
 from scrapy.utils.request import referer_str
+from scrapy.utils.boto import is_botocore
 
 logger = logging.getLogger(__name__)
 
@@ -86,20 +87,30 @@ class S3FilesStore(object):
     }
 
     def __init__(self, uri):
-        try:
+        self.is_botocore = is_botocore()
+        if self.is_botocore:
+            import botocore.session
+            session = botocore.session.get_session()
+            self.s3_client = session.create_client(
+                's3', aws_access_key_id=self.AWS_ACCESS_KEY_ID,
+                aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY)
+        else:
             from boto.s3.connection import S3Connection
             self.S3Connection = S3Connection
-        except ImportError:
-            raise NotConfigured("missing boto library")
         assert uri.startswith('s3://')
         self.bucket, self.prefix = uri[5:].split('/', 1)
 
     def stat_file(self, path, info):
         def _onsuccess(boto_key):
-            checksum = boto_key.etag.strip('"')
-            last_modified = boto_key.last_modified
-            modified_tuple = parsedate_tz(last_modified)
-            modified_stamp = int(mktime_tz(modified_tuple))
+            if self.is_botocore:
+                checksum = boto_key['ETag'].strip('"')
+                last_modified = boto_key['LastModified']
+                modified_stamp = time.mktime(last_modified.timetuple())
+            else:
+                checksum = boto_key.etag.strip('"')
+                last_modified = boto_key.last_modified
+                modified_tuple = parsedate_tz(last_modified)
+                modified_stamp = int(mktime_tz(modified_tuple))
             return {'checksum': checksum, 'last_modified': modified_stamp}
 
         return self._get_boto_key(path).addCallback(_onsuccess)
@@ -111,24 +122,40 @@ class S3FilesStore(object):
         return c.get_bucket(self.bucket, validate=False)
 
     def _get_boto_key(self, path):
-        b = self._get_boto_bucket()
         key_name = '%s%s' % (self.prefix, path)
-        return threads.deferToThread(b.get_key, key_name)
+        if self.is_botocore:
+            return threads.deferToThread(
+                self.s3_client.head_object,
+                Bucket=self.bucket,
+                Key=key_name)
+        else:
+            b = self._get_boto_bucket()
+            return threads.deferToThread(b.get_key, key_name)
 
     def persist_file(self, path, buf, info, meta=None, headers=None):
         """Upload file to S3 storage"""
-        b = self._get_boto_bucket()
         key_name = '%s%s' % (self.prefix, path)
-        k = b.new_key(key_name)
-        if meta:
-            for metakey, metavalue in six.iteritems(meta):
-                k.set_metadata(metakey, str(metavalue))
-        h = self.HEADERS.copy()
-        if headers:
-            h.update(headers)
         buf.seek(0)
-        return threads.deferToThread(k.set_contents_from_string, buf.getvalue(),
-                                     headers=h, policy=self.POLICY)
+        if self.is_botocore:
+            return threads.deferToThread(
+                self.s3_client.put_object,
+                Bucket=self.bucket,
+                Key=key_name,
+                Body=buf,
+                Metadata={k: str(v) for k, v in six.iteritems(meta)},
+                ACL=self.POLICY)
+        else:
+            b = self._get_boto_bucket()
+            k = b.new_key(key_name)
+            if meta:
+                for metakey, metavalue in six.iteritems(meta):
+                    k.set_metadata(metakey, str(metavalue))
+            h = self.HEADERS.copy()
+            if headers:
+                h.update(headers)
+            return threads.deferToThread(
+                k.set_contents_from_string, buf.getvalue(),
+                headers=h, policy=self.POLICY)
 
 
 class FilesPipeline(MediaPipeline):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index fd2f5a2ba..c76d26b57 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -22,7 +22,6 @@ from scrapy.extensions.feedexport import (
 )
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
 from scrapy.utils.python import to_native_str
-from scrapy.utils.boto import is_botocore
 
 
 class FileFeedStorageTest(unittest.TestCase):

From 3cb7a567ea0f0623fb77bd19b652d89625189f67 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 15 Feb 2016 19:14:41 +0300
Subject: [PATCH 0860/2974] py3 fix for TestS3FilesStore: checksum is a native
 string

---
 tests/test_pipeline_files.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 6ea47086f..e445d9989 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -198,7 +198,7 @@ class TestS3FilesStore(unittest.TestCase):
         s = yield store.stat_file(path, info=None)
         self.assertIn('last_modified', s)
         self.assertIn('checksum', s)
-        self.assertEqual(s['checksum'], b'3187896a9657a28163abb31667df64c8')
+        self.assertEqual(s['checksum'], '3187896a9657a28163abb31667df64c8')
         u = urlparse(uri)
         content = get_s3_content_and_delete(u.hostname, u.path[1:])
         self.assertEqual(content, data)

From 08bc41cc685ae2a8282caf20147bd0963932de24 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 15 Feb 2016 19:15:55 +0300
Subject: [PATCH 0861/2974] py3: reviewed s3 downloader handlers

---
 tests/py3-ignores.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 6385ad3b1..ec2947003 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -7,7 +7,6 @@ scrapy/xlib/tx/endpoints.py
 scrapy/xlib/tx/client.py
 scrapy/xlib/tx/_newclient.py
 scrapy/xlib/tx/__init__.py
-scrapy/core/downloader/handlers/s3.py
 scrapy/core/downloader/handlers/ftp.py
 scrapy/linkextractors/sgml.py
 scrapy/linkextractors/regex.py

From 77ebb136840acaa700b6c0ada5e028217a503cfe Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 15 Feb 2016 19:26:15 +0300
Subject: [PATCH 0862/2974] fix assertRaises for precise env

---
 tests/test_downloader_handlers.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index f34a286c2..e08d2e4a4 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -490,8 +490,8 @@ class S3TestCase(unittest.TestCase):
                 yield
 
     def test_extra_kw(self):
-        with self.assertRaises(TypeError):
-            S3DownloadHandler(Settings(), extra_kw=True)
+        self.assertRaises(
+            TypeError, S3DownloadHandler, Settings(), extra_kw=True)
 
     def test_request_signing1(self):
         # gets an object from the johnsmith bucket.

From e7c4806c5ea7092ecf41724bcd1a4179f9685324 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 15 Feb 2016 19:33:18 +0300
Subject: [PATCH 0863/2974] Update feedstorage docs: add botocore, mention that
 boto is supported only on Python 2

---
 docs/topics/feed-exports.rst | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 03c6fb3fb..e5037129c 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -99,12 +99,12 @@ The storages backends supported out of the box are:
 
  * :ref:`topics-feed-storage-fs`
  * :ref:`topics-feed-storage-ftp`
- * :ref:`topics-feed-storage-s3` (requires boto_)
+ * :ref:`topics-feed-storage-s3` (requires botocore_ or boto_)
  * :ref:`topics-feed-storage-stdout`
 
 Some storage backends may be unavailable if the required external libraries are
-not available. For example, the S3 backend is only available if the boto_
-library is installed.
+not available. For example, the S3 backend is only available if the botocore_
+or boto_ library is installed (Scrapy supports boto_ only on Python 2).
 
 
 .. _topics-feed-uri-params:
@@ -177,7 +177,7 @@ The feeds are stored on `Amazon S3`_.
    * ``s3://mybucket/path/to/export.csv``
    * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
 
- * Required external libraries: `boto`_
+ * Required external libraries: `botocore`_ or `boto`_
 
 The AWS credentials can be passed as user/password in the URI, or they can be
 passed through the following settings:
@@ -332,4 +332,5 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 
 .. _URI: http://en.wikipedia.org/wiki/Uniform_Resource_Identifier
 .. _Amazon S3: http://aws.amazon.com/s3/
-.. _boto: http://code.google.com/p/boto/
+.. _boto: https://github.com/boto/boto
+.. _botocore: https://github.com/boto/botocore

From d1ecb8cd38bd8922780731682b696adc97974240 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 15 Feb 2016 19:48:28 +0300
Subject: [PATCH 0864/2974] Fix S3TestCase for precise env: we reraise
 TypeError as NotConfigured in this case

---
 tests/test_downloader_handlers.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index e08d2e4a4..1885a53a0 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -490,8 +490,12 @@ class S3TestCase(unittest.TestCase):
                 yield
 
     def test_extra_kw(self):
-        self.assertRaises(
-            TypeError, S3DownloadHandler, Settings(), extra_kw=True)
+        try:
+            S3DownloadHandler(Settings(), extra_kw=True)
+        except Exception as e:
+            self.assertIsInstance(e, (TypeError, NotConfigured))
+        else:
+            assert False
 
     def test_request_signing1(self):
         # gets an object from the johnsmith bucket.

From 49313a6988bb9161f9036098c454a180d48ca4b4 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 15 Feb 2016 20:16:40 +0300
Subject: [PATCH 0865/2974] use absolute_import to import external boto package

---
 scrapy/utils/boto.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index fd6b22b88..421ab2f7e 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -1,5 +1,6 @@
 """Boto/botocore helpers"""
 
+from __future__ import absolute_import
 import six
 
 from scrapy.exceptions import NotConfigured

From cabed6f183cfb2ab778c57be8c75802fec5e54d4 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 17 Feb 2016 16:55:28 +0100
Subject: [PATCH 0866/2974] More liberal Content-Disposition header parsing

Fixes #1782
---
 scrapy/responsetypes.py     |  3 ++-
 tests/test_responsetypes.py | 11 +++++++++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 4880cc7b9..c667b141d 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -59,7 +59,8 @@ class ResponseTypes(object):
 
     def from_content_disposition(self, content_disposition):
         try:
-            filename = to_native_str(content_disposition).split(';')[1].split('=')[1]
+            filename = to_native_str(content_disposition,
+                encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
             filename = filename.strip('"\'')
             return self.from_filename(filename)
         except IndexError:
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 2374d518f..118136ac4 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 import unittest
 from scrapy.responsetypes import responsetypes
 
@@ -20,8 +21,14 @@ class ResponseTypesTest(unittest.TestCase):
 
     def test_from_content_disposition(self):
         mappings = [
-            ('attachment; filename="data.xml"', XmlResponse),
-            ('attachment; filename=data.xml', XmlResponse),
+            (b'attachment; filename="data.xml"', XmlResponse),
+            (b'attachment; filename=data.xml', XmlResponse),
+            (u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
+            (u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
+            (u'attachment;filename=data高.doc'.encode('gbk'), Response),
+            (u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
+            (u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
+
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_disposition(source)

From 06da7af9e293d00087799ac819f9ac4caa4901e7 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 17 Feb 2016 23:03:12 +0500
Subject: [PATCH 0867/2974] TST clean up RunSpiderCommandTest

---
 tests/test_commands.py | 41 +++++++++++++++++++++--------------------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 1a30368ba..5d45cd62d 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -6,7 +6,7 @@ from time import sleep
 from os.path import exists, join, abspath
 from shutil import rmtree, copytree
 from tempfile import mkdtemp
-import six
+from contextlib import contextmanager
 
 from twisted.trial import unittest
 from twisted.internet import defer
@@ -154,12 +154,24 @@ class MiscCommandsTest(CommandTest):
 
 class RunSpiderCommandTest(CommandTest):
 
-    def test_runspider(self):
+    @contextmanager
+    def _create_file(self, content, name):
         tmpdir = self.mktemp()
         os.mkdir(tmpdir)
-        fname = abspath(join(tmpdir, 'myspider.py'))
+        fname = abspath(join(tmpdir, name))
         with open(fname, 'w') as f:
-            f.write("""
+            f.write(content)
+        try:
+            yield fname
+        finally:
+            rmtree(tmpdir)
+
+    def runspider(self, code, name='myspider.py'):
+        with self._create_file(code, name) as fname:
+            return self.proc('runspider', fname)
+
+    def test_runspider(self):
+        spider = """
 import scrapy
 
 class MySpider(scrapy.Spider):
@@ -168,23 +180,17 @@ class MySpider(scrapy.Spider):
     def start_requests(self):
         self.logger.debug("It Works!")
         return []
-""")
-        p = self.proc('runspider', fname)
+"""
+        p = self.runspider(spider)
         log = to_native_str(p.stderr.read())
+
         self.assertIn("DEBUG: It Works!", log)
         self.assertIn("INFO: Spider opened", log)
         self.assertIn("INFO: Closing spider (finished)", log)
         self.assertIn("INFO: Spider closed (finished)", log)
 
     def test_runspider_no_spider_found(self):
-        tmpdir = self.mktemp()
-        os.mkdir(tmpdir)
-        fname = abspath(join(tmpdir, 'myspider.py'))
-        with open(fname, 'w') as f:
-            f.write("""
-from scrapy.spiders import Spider
-""")
-        p = self.proc('runspider', fname)
+        p = self.runspider("from scrapy.spiders import Spider\n")
         log = to_native_str(p.stderr.read())
         self.assertIn("No spider found in file", log)
 
@@ -194,12 +200,7 @@ from scrapy.spiders import Spider
         self.assertIn("File not found: some_non_existent_file", log)
 
     def test_runspider_unable_to_load(self):
-        tmpdir = self.mktemp()
-        os.mkdir(tmpdir)
-        fname = abspath(join(tmpdir, 'myspider.txt'))
-        with open(fname, 'w') as f:
-            f.write("")
-        p = self.proc('runspider', fname)
+        p = self.runspider("", "myspider.txt")
         log = to_native_str(p.stderr.read())
         self.assertIn("Unable to load", log)
 

From f766dd0ba8afce0ec9890106fe0fd477aaf04d29 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 17 Feb 2016 23:07:03 +0500
Subject: [PATCH 0868/2974] Preserve tracebacks better. Fixes GH-1760.

---
 scrapy/crawler.py      | 15 +++++++++++++--
 tests/test_commands.py | 14 ++++++++++++++
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index ef99c243a..3e695f793 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -3,6 +3,7 @@ import signal
 import logging
 import warnings
 
+import sys
 from twisted.internet import reactor, defer
 from zope.interface.verify import verifyClass, DoesNotImplement
 
@@ -73,11 +74,21 @@ class Crawler(object):
             yield self.engine.open_spider(self.spider, start_requests)
             yield defer.maybeDeferred(self.engine.start)
         except Exception:
-            exc = defer.fail()
+            # In Python 2 reraising an exception after yield discards
+            # the original traceback (see http://bugs.python.org/issue7563),
+            # so sys.exc_info() workaround is used.
+            # This workaround also works in Python 3, but it is not needed,
+            # and it is slower, so in Python 3 we use native `raise`.
+            if six.PY2:
+                exc_info = sys.exc_info()
+
             self.crawling = False
             if self.engine is not None:
                 yield self.engine.close()
-            yield exc
+
+            if six.PY2:
+                raise six.reraise(*exc_info)
+            raise
 
     def _create_spider(self, *args, **kwargs):
         return self.spidercls.from_crawler(self, *args, **kwargs)
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 5d45cd62d..93b53dbeb 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -204,6 +204,20 @@ class MySpider(scrapy.Spider):
         log = to_native_str(p.stderr.read())
         self.assertIn("Unable to load", log)
 
+    def test_start_requests_errors(self):
+        p = self.runspider("""
+import scrapy
+
+class BadSpider(scrapy.Spider):
+    name = "bad"
+    def start_requests(self):
+        raise Exception("oops!")
+        """, name="badspider.py")
+        log = to_native_str(p.stderr.read())
+        print(log)
+        self.assertIn("start_requests", log)
+        self.assertIn("badspider.py", log)
+
 
 class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
     command = 'parse'

From 617631f2646d349c4bdc288b3d7b41ad483f5ec6 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 18 Feb 2016 10:10:16 +0300
Subject: [PATCH 0869/2974] Fix method name: this always returns unicode keys
 and values

---
 scrapy/core/downloader/handlers/s3.py | 2 +-
 scrapy/http/headers.py                | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index dd7bce2be..d8bbdd326 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -82,7 +82,7 @@ class S3DownloadHandler(object):
             awsrequest = botocore.awsrequest.AWSRequest(
                 method=request.method,
                 url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path),
-                headers=request.headers.to_native_string_dict(),
+                headers=request.headers.to_unicode_dict(),
                 data=request.body)
             self._signer.add_auth(awsrequest)
             request = request.replace(
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index d0c4cd0fb..62507eb19 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -79,7 +79,10 @@ class Headers(CaselessDict):
     def to_string(self):
         return headers_dict_to_raw(self)
 
-    def to_native_string_dict(self):
+    def to_unicode_dict(self):
+        """ Return headers as a CaselessDict with unicode keys
+        and unicode values. Multiple values are joined with ','.
+        """
         return CaselessDict(
             (to_unicode(key, encoding=self.encoding),
              to_unicode(b','.join(value), encoding=self.encoding))

From d61fbcc8b5fee1c2407d42fb72bd72bb18d40e25 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 18 Feb 2016 10:57:02 +0300
Subject: [PATCH 0870/2974] Support headers in S3FilesStore.persist_file for
 botocore

---
 scrapy/pipelines/files.py    | 36 +++++++++++++++++++++++++++++++++++-
 scrapy/utils/test.py         |  4 ++--
 tests/test_pipeline_files.py | 18 ++++++++++++++++--
 3 files changed, 53 insertions(+), 5 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index c757b0a3f..45ceddcbb 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -29,6 +29,7 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.python import to_bytes
 from scrapy.utils.request import referer_str
 from scrapy.utils.boto import is_botocore
+from scrapy.utils.datatypes import CaselessDict
 
 logger = logging.getLogger(__name__)
 
@@ -137,13 +138,17 @@ class S3FilesStore(object):
         key_name = '%s%s' % (self.prefix, path)
         buf.seek(0)
         if self.is_botocore:
+            extra = self._headers_to_botocore_kwargs(self.HEADERS)
+            if headers:
+                extra.update(self._headers_to_botocore_kwargs(headers))
             return threads.deferToThread(
                 self.s3_client.put_object,
                 Bucket=self.bucket,
                 Key=key_name,
                 Body=buf,
                 Metadata={k: str(v) for k, v in six.iteritems(meta)},
-                ACL=self.POLICY)
+                ACL=self.POLICY,
+                **extra)
         else:
             b = self._get_boto_bucket()
             k = b.new_key(key_name)
@@ -157,6 +162,35 @@ class S3FilesStore(object):
                 k.set_contents_from_string, buf.getvalue(),
                 headers=h, policy=self.POLICY)
 
+    def _headers_to_botocore_kwargs(self, headers):
+        """ Convert headers to botocore keyword agruments.
+        """
+        # This is required while we need to support both boto and botocore.
+        mapping = CaselessDict({
+            'Content-Type': 'ContentType',
+            'Cache-Control': 'CacheControl',
+            'Content-Disposition': 'ContentDisposition',
+            'Content-Encoding': 'ContentEncoding',
+            'Content-Language': 'ContentLanguage',
+            'Content-Length': 'ContentLength',
+            'Content-MD5': 'ContentMD5',
+            'Expires': 'Expires',
+            'X-Amz-Grant-Full-Control': 'GrantFullControl',
+            'X-Amz-Grant-Read': 'GrantRead',
+            'X-Amz-Grant-Read-ACP': 'GrantReadACP',
+            'X-Amz-Grant-Write-ACP': 'GrantWriteACP',
+            })
+        extra = {}
+        for key, value in six.iteritems(headers):
+            try:
+                kwarg = mapping[key]
+            except KeyError:
+                raise TypeError(
+                    'Header "%s" is not supported by botocore' % key)
+            else:
+                extra[kwarg] = value
+        return extra
+
 
 class FilesPipeline(MediaPipeline):
     """Abstract pipeline that implement the file downloading
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 43abd64a0..bf66a8cbe 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -26,7 +26,7 @@ def skip_if_no_boto():
     except NotConfigured as e:
         raise SkipTest(e.message)
 
-def get_s3_content_and_delete(bucket, path):
+def get_s3_content_and_delete(bucket, path, with_key=False):
     """ Get content from s3 key, and delete key afterwards.
     """
     if is_botocore():
@@ -43,7 +43,7 @@ def get_s3_content_and_delete(bucket, path):
         key = bucket.get_key(path)
         content = key.get_contents_as_string()
         bucket.delete_key(path)
-    return content
+    return (content, key) if with_key else content
 
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index e445d9989..77e75d5ac 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -16,6 +16,7 @@ from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
+from scrapy.utils.boto import is_botocore
 
 from tests import mock
 
@@ -194,14 +195,27 @@ class TestS3FilesStore(unittest.TestCase):
         meta = {'foo': 'bar'}
         path = ''
         store = S3FilesStore(uri)
-        yield store.persist_file(path, buf, info=None, meta=meta)
+        yield store.persist_file(
+            path, buf, info=None, meta=meta,
+            headers={'Content-Type': 'image/png'})
         s = yield store.stat_file(path, info=None)
         self.assertIn('last_modified', s)
         self.assertIn('checksum', s)
         self.assertEqual(s['checksum'], '3187896a9657a28163abb31667df64c8')
         u = urlparse(uri)
-        content = get_s3_content_and_delete(u.hostname, u.path[1:])
+        content, key = get_s3_content_and_delete(
+            u.hostname, u.path[1:], with_key=True)
         self.assertEqual(content, data)
+        if is_botocore():
+            self.assertEqual(key['Metadata'], {'foo': 'bar'})
+            self.assertEqual(
+                key['CacheControl'], S3FilesStore.HEADERS['Cache-Control'])
+            self.assertEqual(key['ContentType'], 'image/png')
+        else:
+            self.assertEqual(key.metadata, {'foo': 'bar'})
+            self.assertEqual(
+                key.cache_control, S3FilesStore.HEADERS['Cache-Control'])
+            self.assertEqual(key.content_type, 'image/png')
 
 
 class ItemWithFiles(Item):

From 104027d78d10592a395ffeb8f34ac9fc0ae1702f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 18 Feb 2016 11:45:03 +0100
Subject: [PATCH 0871/2974] Minor change on quotes

Trying to force Travis CI to build
---
 tests/test_commands.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 93b53dbeb..2e47160d7 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -200,9 +200,9 @@ class MySpider(scrapy.Spider):
         self.assertIn("File not found: some_non_existent_file", log)
 
     def test_runspider_unable_to_load(self):
-        p = self.runspider("", "myspider.txt")
+        p = self.runspider('', 'myspider.txt')
         log = to_native_str(p.stderr.read())
-        self.assertIn("Unable to load", log)
+        self.assertIn('Unable to load', log)
 
     def test_start_requests_errors(self):
         p = self.runspider("""

From 30a27effa9b20eb5cf410e00840a56286a2e8d92 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 19 Feb 2016 01:54:23 +0100
Subject: [PATCH 0872/2974] Use best practices for TLS connections when using
 Twisted>=14.0

---
 scrapy/core/downloader/contextfactory.py | 88 ++++++++++++++++++------
 1 file changed, 66 insertions(+), 22 deletions(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 0b39b89d8..347b74648 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,28 +1,72 @@
 from OpenSSL import SSL
 from twisted.internet.ssl import ClientContextFactory
-try:
-    # available since twisted 14.0
-    from twisted.internet._sslverify import ClientTLSOptions
-except ImportError:
-    ClientTLSOptions = None
+
+from scrapy import twisted_version
+
+if twisted_version >= (14, 0, 0):
+
+    from twisted.web.client import BrowserLikePolicyForHTTPS
+    from twisted.internet.ssl import optionsForClientTLS
+
+    class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
+        """
+        Using Twisted recommended context factory for twisted.web.client.Agent
+
+        Quoting:
+        "The default is to use a BrowserLikePolicyForHTTPS,
+        so unless you have special requirements you can leave this as-is."
+
+        See http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html
+        """
 
 
-class ScrapyClientContextFactory(ClientContextFactory):
-    "A SSL context factory which is more permissive against SSL bugs."
-    # see https://github.com/scrapy/scrapy/issues/82
-    # and https://github.com/scrapy/scrapy/issues/26
-    # and https://github.com/scrapy/scrapy/issues/981
+    class OpenSSLMethodContextFactory(ScrapyClientContextFactory):
 
-    def __init__(self):
-        # see this issue on why we use TLSv1_METHOD by default
-        # https://github.com/scrapy/scrapy/issues/194
-        self.method = SSL.TLSv1_METHOD
+        openssl_method = SSL.SSLv23_METHOD
 
-    def getContext(self, hostname=None, port=None):
-        ctx = ClientContextFactory.getContext(self)
-        # Enable all workarounds to SSL bugs as documented by
-        # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html
-        ctx.set_options(SSL.OP_ALL)
-        if hostname and ClientTLSOptions is not None: # workaround for TLS SNI
-            ClientTLSOptions(hostname, ctx)
-        return ctx
+        def creatorForNetloc(self, hostname, port):
+            return optionsForClientTLS(hostname.decode("ascii"),
+                                       trustRoot=self._trustRoot,
+                                       extraCertificateOptions={
+                                            'method': self.openssl_method
+                                       })
+
+
+else:
+
+    class OpenSSLMethodContextFactory(ClientContextFactory):
+        "A SSL context factory which is more permissive against SSL bugs."
+        # see https://github.com/scrapy/scrapy/issues/82
+        # and https://github.com/scrapy/scrapy/issues/26
+        # and https://github.com/scrapy/scrapy/issues/981
+        openssl_method = SSL.SSLv23_METHOD
+
+        def __init__(self):
+            self.method = self.openssl_method
+
+        def getContext(self, hostname=None, port=None):
+            ctx = ClientContextFactory.getContext(self)
+            # Enable all workarounds to SSL bugs as documented by
+            # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html
+            ctx.set_options(SSL.OP_ALL)
+            if hostname and ClientTLSOptions is not None: # workaround for TLS SNI
+                ClientTLSOptions(hostname, ctx)
+            return ctx
+
+    ScrapyClientContextFactory = OpenSSLMethodContextFactory
+
+
+class SSLv3ContextFactory(OpenSSLMethodContextFactory):
+    openssl_method = SSL.SSLv3_METHOD
+
+
+class TLSv1ContextFactory(OpenSSLMethodContextFactory):
+    openssl_method = SSL.TLSv1_METHOD
+
+
+class TLSv11ContextFactory(OpenSSLMethodContextFactory):
+    openssl_method = SSL.TLSv1_1_METHOD
+
+
+class TLSv12ContextFactory(OpenSSLMethodContextFactory):
+    openssl_method = SSL.TLSv1_2_METHOD

From 45f972cb3f664705fefc932f34a1e88134819fb4 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 19 Feb 2016 02:31:57 +0100
Subject: [PATCH 0873/2974] Implement IPolicyForHTTPS

---
 scrapy/core/downloader/contextfactory.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 347b74648..4d90d17f8 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -5,9 +5,11 @@ from scrapy import twisted_version
 
 if twisted_version >= (14, 0, 0):
 
-    from twisted.web.client import BrowserLikePolicyForHTTPS
     from twisted.internet.ssl import optionsForClientTLS
+    from twisted.web.client import BrowserLikePolicyForHTTPS
+    from twisted.web.iweb import IPolicyForHTTPS
 
+    @implementer(IPolicyForHTTPS)
     class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         """
         Using Twisted recommended context factory for twisted.web.client.Agent
@@ -20,6 +22,7 @@ if twisted_version >= (14, 0, 0):
         """
 
 
+    @implementer(IPolicyForHTTPS)
     class OpenSSLMethodContextFactory(ScrapyClientContextFactory):
 
         openssl_method = SSL.SSLv23_METHOD

From 406b9a06fe67db09b53041724a126d59f11cb519 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 19 Feb 2016 02:56:26 +0100
Subject: [PATCH 0874/2974] Add missing import for implementer

Also remove TLSv1.1 and TLSv1.2 method: these are available only
from pyOpenSSL 0.14
https://github.com/pyca/pyopenssl/releases/tag/v0.14a1
---
 scrapy/core/downloader/contextfactory.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 4d90d17f8..28e8a0e53 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -5,6 +5,8 @@ from scrapy import twisted_version
 
 if twisted_version >= (14, 0, 0):
 
+    from zope.interface.declarations import implementer
+
     from twisted.internet.ssl import optionsForClientTLS
     from twisted.web.client import BrowserLikePolicyForHTTPS
     from twisted.web.iweb import IPolicyForHTTPS
@@ -65,11 +67,3 @@ class SSLv3ContextFactory(OpenSSLMethodContextFactory):
 
 class TLSv1ContextFactory(OpenSSLMethodContextFactory):
     openssl_method = SSL.TLSv1_METHOD
-
-
-class TLSv11ContextFactory(OpenSSLMethodContextFactory):
-    openssl_method = SSL.TLSv1_1_METHOD
-
-
-class TLSv12ContextFactory(OpenSSLMethodContextFactory):
-    openssl_method = SSL.TLSv1_2_METHOD

From 18a381816d1e2e4a08e8c8655fa38f9bb0af312f Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 19 Feb 2016 18:31:49 +0500
Subject: [PATCH 0875/2974] Remove unneeded `raise` (thanks @lopuhin).

---
 scrapy/crawler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 3e695f793..443a9aa2f 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -87,7 +87,7 @@ class Crawler(object):
                 yield self.engine.close()
 
             if six.PY2:
-                raise six.reraise(*exc_info)
+                six.reraise(*exc_info)
             raise
 
     def _create_spider(self, *args, **kwargs):

From 5f5374209f520edf80c5a414cc45033c3ed75a0f Mon Sep 17 00:00:00 2001
From: Victor Mireyev <victor@opennodecloud.com>
Date: Fri, 19 Feb 2016 17:20:34 +0300
Subject: [PATCH 0876/2974] Fix typo in docstring

---
 scrapy/utils/misc.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 303a413d8..741d378d6 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -31,7 +31,7 @@ def arg_to_iter(arg):
 def load_object(path):
     """Load an object given its absolute object path, and return it.
 
-    object can be a class, function, variable o instance.
+    object can be a class, function, variable or an instance.
     path ie: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'
     """
 
@@ -52,7 +52,7 @@ def load_object(path):
 
 
 def walk_modules(path):
-    """Loads a module and all its submodules from a the given module path and
+    """Loads a module and all its submodules from the given module path and
     returns them. If *any* module throws an exception while importing, that
     exception is thrown back.
 

From 523e98dabe01842b9103fdb750237f5a6275963b Mon Sep 17 00:00:00 2001
From: Victor Mireyev <victor@opennodecloud.com>
Date: Fri, 19 Feb 2016 17:32:01 +0300
Subject: [PATCH 0877/2974] Fix typo in docstring

---
 scrapy/utils/reactor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index a99063a61..83186a372 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -19,7 +19,7 @@ def listen_tcp(portrange, host, factory):
 
 class CallLaterOnce(object):
     """Schedule a function to be called in the next reactor loop, but only if
-    it hasn't been already scheduled since the last time it run.
+    it hasn't been already scheduled since the last time it ran.
     """
 
     def __init__(self, func, *a, **kw):

From 57990fba7a12a83517456487979df6461a347328 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Sat, 20 Feb 2016 01:32:21 +0100
Subject: [PATCH 0878/2974] Backward compatibility for HTTP/10 context factory

New DOWNLOADER_CLIENT_TLS_METHOD setting to configure TLS method
---
 scrapy/core/downloader/contextfactory.py  | 70 +++++++++++++++--------
 scrapy/core/downloader/handlers/http11.py | 11 +++-
 scrapy/core/downloader/tls.py             | 16 ++++++
 scrapy/settings/default_settings.py       |  2 +
 tests/keys/server.pem                     | 52 +++++++++++++++++
 tests/mockserver.py                       |  4 +-
 6 files changed, 128 insertions(+), 27 deletions(-)
 create mode 100644 scrapy/core/downloader/tls.py
 create mode 100644 tests/keys/server.pem

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 28e8a0e53..da4dd8f35 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -7,47 +7,79 @@ if twisted_version >= (14, 0, 0):
 
     from zope.interface.declarations import implementer
 
-    from twisted.internet.ssl import optionsForClientTLS
+    from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust
+    from twisted.internet._sslverify import ClientTLSOptions
     from twisted.web.client import BrowserLikePolicyForHTTPS
     from twisted.web.iweb import IPolicyForHTTPS
 
     @implementer(IPolicyForHTTPS)
     class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         """
-        Using Twisted recommended context factory for twisted.web.client.Agent
+        Non-peer-certificate verifying HTTPS context factory
 
-        Quoting:
-        "The default is to use a BrowserLikePolicyForHTTPS,
-        so unless you have special requirements you can leave this as-is."
+        Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
+        which allows TLS protocol negotiation
 
-        See http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html
+        'A TLS/SSL connection established with [this method] may
+         understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
         """
 
+        def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs):
+            super(BrowserLikePolicyForHTTPS, self).__init__(*args, **kwargs)
+            self._ssl_method = method
+
+        def getCertificateOptions(self):
+            # setting verify=True will require you to provide CAs
+            # to verify against; in other words: it's not that simple
+            return CertificateOptions(verify=False, method=self._ssl_method)
+
+        # kept for old-style HTTP/1.0 downloader context twisted calls,
+        # e.g. connectSSL()
+        def getContext(self, hostname=None, port=None):
+            return self.getCertificateOptions().getContext()
+
+        def creatorForNetloc(self, hostname, port):
+            return ClientTLSOptions(hostname.decode("ascii"), self.getContext())
+
 
     @implementer(IPolicyForHTTPS)
-    class OpenSSLMethodContextFactory(ScrapyClientContextFactory):
+    class BrowserLikeContextFactory(ScrapyClientContextFactory):
+        """
+        Twisted-recommended context factory for web clients.
 
-        openssl_method = SSL.SSLv23_METHOD
+        Quoting http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
+        "The default is to use a BrowserLikePolicyForHTTPS,
+        so unless you have special requirements you can leave this as-is."
 
+        creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
+        except this context factory allows setting the TLS/SSL method to use.
+
+        Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
+        which allows TLS protocol negotiation.
+        """
         def creatorForNetloc(self, hostname, port):
+
+            # trustRoot set to platformTrust() will use the platform's root CAs.
+            #
+            # This means that a website like https://www.cacert.org will be rejected
+            # by default, since CAcert.org CA certificate is seldom shipped.
             return optionsForClientTLS(hostname.decode("ascii"),
-                                       trustRoot=self._trustRoot,
+                                       trustRoot=platformTrust(),
                                        extraCertificateOptions={
-                                            'method': self.openssl_method
+                                            'method': self._ssl_method,
                                        })
 
 
 else:
 
-    class OpenSSLMethodContextFactory(ClientContextFactory):
+    class ScrapyClientContextFactory(ClientContextFactory):
         "A SSL context factory which is more permissive against SSL bugs."
         # see https://github.com/scrapy/scrapy/issues/82
         # and https://github.com/scrapy/scrapy/issues/26
         # and https://github.com/scrapy/scrapy/issues/981
-        openssl_method = SSL.SSLv23_METHOD
 
-        def __init__(self):
-            self.method = self.openssl_method
+        def __init__(self, method=SSL.SSLv23_METHOD):
+            self.method = method
 
         def getContext(self, hostname=None, port=None):
             ctx = ClientContextFactory.getContext(self)
@@ -57,13 +89,3 @@ else:
             if hostname and ClientTLSOptions is not None: # workaround for TLS SNI
                 ClientTLSOptions(hostname, ctx)
             return ctx
-
-    ScrapyClientContextFactory = OpenSSLMethodContextFactory
-
-
-class SSLv3ContextFactory(OpenSSLMethodContextFactory):
-    openssl_method = SSL.SSLv3_METHOD
-
-
-class TLSv1ContextFactory(OpenSSLMethodContextFactory):
-    openssl_method = SSL.TLSv1_METHOD
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index ad3285a32..522bac16f 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -18,6 +18,7 @@ from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \
 from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.core.downloader.webclient import _parse
+from scrapy.core.downloader.tls import openssl_methods, METHOD_TLS
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_bytes, to_unicode
 from scrapy import twisted_version
@@ -31,8 +32,16 @@ class HTTP11DownloadHandler(object):
         self._pool = HTTPConnectionPool(reactor, persistent=True)
         self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self._pool._factory.noisy = False
+
+        self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
         self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
-        self._contextFactory = self._contextFactoryClass()
+        # try method-aware context factory
+        try:
+            self._contextFactory = self._contextFactoryClass(method=self._sslMethod)
+        except TypeError:
+            # use defaults
+            self._contextFactory = self._contextFactoryClass()
+
         self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
         self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
         self._disconnect_timeout = 1
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
new file mode 100644
index 000000000..121ee83ac
--- /dev/null
+++ b/scrapy/core/downloader/tls.py
@@ -0,0 +1,16 @@
+from OpenSSL import SSL
+
+
+METHOD_SSLv3 = 'SSLv3'
+METHOD_TLS = 'TLS'
+METHOD_TLSv10 = 'TLSv1.0'
+METHOD_TLSv11 = 'TLSv1.1'
+METHOD_TLSv12 = 'TLSv1.2'
+
+openssl_methods = {
+    METHOD_TLS: SSL.SSLv23_METHOD,                      # protocol negotiation (recommended)
+    METHOD_SSLv3: SSL.SSLv3_METHOD,                     # SSL 3 (NOT recommended)
+    METHOD_TLSv10: SSL.TLSv1_METHOD,                    # TLS 1.0 only
+    METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only
+    METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
+}
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 44e74dc61..7a4f640ce 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -83,6 +83,8 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader'
 
 DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
 DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
+DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
+                                     # also allowing negotiation
 
 DOWNLOADER_MIDDLEWARES = {}
 
diff --git a/tests/keys/server.pem b/tests/keys/server.pem
new file mode 100644
index 000000000..8356f4a49
--- /dev/null
+++ b/tests/keys/server.pem
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDGnXh/GMCLpNNI
+AIvfBWlPrRCLFWxd2ICLYSUq3/jwh31CppGKlfaSmUYnrMxnT4hg2f6gBqlmq1gK
+jQqDkqQQtHsSljRQF58NRFtz99w45jRmrGs+F5zoggJuyv7/lKXy/BXOc40NodIl
+qRuo/uhoPjeXaUNziwpRj4rByRdLwQ6MfRcfdZ0TpVJ4J7apJ66pZt85L63u8TZi
+AZdBgcowX6giola7kUUMG66bSi2X6sIFxdXHwzrWlFNnSbKqOqKlGrjmBHUYqRFr
+gLALKkaMpF30olHn7QLqJ2592hMFVkOQAzNr8Xb15mF27BuWFYd7P6TirsxCEd+9
+BFFbhl0vAgMBAAECggEAf1ndN3GBlIi9SL/A7+GiYwpPPz8fWxVFZxmFIXa3QlM+
+CAyR6dC6Z8mL6EiuT9f5VFCzKZzb5g8bxrgk87SFKojvGT3ikTB0NaeNFFDrjjhd
+hTAtG4U8gQFL2gqjcvG3bpQgz13cJc+K1ccXC0dXce/i6Vz/eQjANwfZKuRr4Y/p
+Ml3ZtkarT7cZ2TZRDYd643U2nfrNmeLA9wwwO5mgNYu7HSpI4idpVM+rzoS9ZbU1
+Jxhrdq7GELvnd+Ko3WHTSrkQyb53GtJaLFOSAMxQlZAbRGJMv2VtXSPe+NyA0kxO
+V4O9CJ2QL1Bxqk4MQicKGtGoIsFbfR/qYQZgNEuHoQKBgQDlqEYSGpEcyKNvCjZ+
+Dv5hFUT7GZ24MNRvMBdMHPJEkmxvNpwaNkOGXL6zmhDk5Y0HOrzNv64Llmg623y0
+pn9Vh3CvMKDlq5t8910BJhXzZAM9/E9ui/YuQsFwCdWJQFYVMZUIAdylxbDkg6Mf
+WzHkx82edwWgb7hOFJhEW3h51wKBgQDdZaofTKeSLUmxIbF1yqm4iX67wyhHFFZL
+RZLEcu35ZTJqz3TRpr9KpHCtq9J2gZbqo2Dvqznwfk+yUqT2gkibtk3qvmv/qHQt
+FGX5joWLD2E22UgGc4bzonTSQcxPfPYjNZa6Iv7koPgJqO1k3aYLG3mDMgI/Yurz
+3j8udCI8aQKBgQCe8uNkfky3PkqXfNsQTnJTYTQ4EpettuYg+oj1Xgdz3F/+bS6H
+mc0Mfr5ichyFQDdmrImmYaclT/ld2zGpmp8A5FfieOazbx2T1WTieaixpyPzT+Du
+IyLFC+D0GWpYr1WlH4cFXryHY5c14cIR3r1emcc/iSM00n4eVHS6wxBUSwKBgQCQ
+fTAI20slgD7uxjF90XTwhXNbcONReDlVSKOfZ+5dbCj6QkhYJv4C5czP2yowwyRO
+H7A82j+m75htpbgZVS+zx8eUIxByumqPnRdsRhJrje8BD88MvsrdKDIbomuHmOM+
+WVP0SLCXX2JhU4kS4gDikNy9vZ5v5cu2ul37oDlTuQKBgBl4Qs6Q0xUotlvy/fMf
+L3c6GUIeRsLyQCKFfKYiUNLHqqutBSYmsAlx3XOhcK4DZ7/gFAXO6mpYJlwXOdrI
+IMcSXuJvZDPAYMXOyZUTgNjPcbQ8t0by+EKPwnTReIBf1Y17vPDfisf5IEKZEWvM
+YDXg6cfx9R5QePjvZohlItOu
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIID8DCCAtgCAws5AzANBgkqhkiG9w0BAQUFADCBuzELMAkGA1UEBhMCVFIxDjAM
+BgNVBAgUBcdvcnVtMRkwFwYDVQQHHhAAQgBhAV8AbQBhAGsA5wExMRIwEAYDVQQD
+Ewlsb2NhbGhvc3QxHDAaBgNVBAoTE1R3aXN0ZWQgTWF0cml4IExhYnMxJDAiBgNV
+BAsTG0F1dG9tYXRlZCBUZXN0aW5nIEF1dGhvcml0eTEpMCcGCSqGSIb3DQEJARYa
+c2VjdXJpdHlAdHdpc3RlZG1hdHJpeC5jb20wIBcNMTQwOTE2MDE0MjQ2WhgPMjEx
+NDA4MjMwMTQyNDZaMIG7MQswCQYDVQQGEwJUUjEOMAwGA1UECBQFx29ydW0xGTAX
+BgNVBAceEABCAGEBXwBtAGEAawDnATExEjAQBgNVBAMTCWxvY2FsaG9zdDEcMBoG
+A1UEChMTVHdpc3RlZCBNYXRyaXggTGFiczEkMCIGA1UECxMbQXV0b21hdGVkIFRl
+c3RpbmcgQXV0aG9yaXR5MSkwJwYJKoZIhvcNAQkBFhpzZWN1cml0eUB0d2lzdGVk
+bWF0cml4LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMadeH8Y
+wIuk00gAi98FaU+tEIsVbF3YgIthJSrf+PCHfUKmkYqV9pKZRieszGdPiGDZ/qAG
+qWarWAqNCoOSpBC0exKWNFAXnw1EW3P33DjmNGasaz4XnOiCAm7K/v+UpfL8Fc5z
+jQ2h0iWpG6j+6Gg+N5dpQ3OLClGPisHJF0vBDox9Fx91nROlUngntqknrqlm3zkv
+re7xNmIBl0GByjBfqCKiVruRRQwbrptKLZfqwgXF1cfDOtaUU2dJsqo6oqUauOYE
+dRipEWuAsAsqRoykXfSiUeftAuonbn3aEwVWQ5ADM2vxdvXmYXbsG5YVh3s/pOKu
+zEIR370EUVuGXS8CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAK+HqbIUN6qHYYQZw
+7qqsTJWni8NOfK3aguyKAPcdCPP2DCZ6zlxxkUWL57gvsohyDu8Nr9iSI6wePjmI
+cN9eCZdc6mD9kYW4qBYhh2T48TOhEEW7zO6bWQUqWohHW+bG+GfrHnvxIx56OC2B
+eDS2djvvScYm45etlHprfrVEDIssh956O6qJCySax3D2w+i8YanXji1EbS61XTDw
+CMxDdWYmd2MDARRwlMcfcUIfKZUGl5NmqpnOx+H5MyAGwt86s647GMYZborQh+Mj
+tNHVpyKf/a8/HjqP1sCOrjCPZIjP6Qp5j4gifAjUStNmCgaBe7CpFtBSLnHqb4o/
+gU7u1w==
+-----END CERTIFICATE-----
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 365ec81fd..087ad958f 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -206,8 +206,8 @@ class MockServer():
 
 def ssl_context_factory():
     return ssl.DefaultOpenSSLContextFactory(
-         os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
-         os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
+         os.path.join(os.path.dirname(__file__), 'keys/server.pem'),
+         os.path.join(os.path.dirname(__file__), 'keys/server.pem'),
          )
 
 

From 49c757f208eb41e637171543b5b337f4a280387a Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Sat, 20 Feb 2016 01:58:59 +0100
Subject: [PATCH 0879/2974] Fix import for Ubuntu 12.04 precise (Twisted
 11.1.0)

Revert test server certificate change
---
 scrapy/core/downloader/contextfactory.py | 10 ++---
 tests/keys/server.pem                    | 52 ------------------------
 tests/mockserver.py                      |  4 +-
 3 files changed, 5 insertions(+), 61 deletions(-)
 delete mode 100644 tests/keys/server.pem

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index da4dd8f35..5e6ad57a1 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,12 +1,11 @@
 from OpenSSL import SSL
 from twisted.internet.ssl import ClientContextFactory
 
-from scrapy import twisted_version
-
-if twisted_version >= (14, 0, 0):
+try:
 
     from zope.interface.declarations import implementer
 
+    # the following should be available from Twisted 14.0.0
     from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust
     from twisted.internet._sslverify import ClientTLSOptions
     from twisted.web.client import BrowserLikePolicyForHTTPS
@@ -69,8 +68,7 @@ if twisted_version >= (14, 0, 0):
                                             'method': self._ssl_method,
                                        })
 
-
-else:
+except ImportError:
 
     class ScrapyClientContextFactory(ClientContextFactory):
         "A SSL context factory which is more permissive against SSL bugs."
@@ -86,6 +84,4 @@ else:
             # Enable all workarounds to SSL bugs as documented by
             # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html
             ctx.set_options(SSL.OP_ALL)
-            if hostname and ClientTLSOptions is not None: # workaround for TLS SNI
-                ClientTLSOptions(hostname, ctx)
             return ctx
diff --git a/tests/keys/server.pem b/tests/keys/server.pem
deleted file mode 100644
index 8356f4a49..000000000
--- a/tests/keys/server.pem
+++ /dev/null
@@ -1,52 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDGnXh/GMCLpNNI
-AIvfBWlPrRCLFWxd2ICLYSUq3/jwh31CppGKlfaSmUYnrMxnT4hg2f6gBqlmq1gK
-jQqDkqQQtHsSljRQF58NRFtz99w45jRmrGs+F5zoggJuyv7/lKXy/BXOc40NodIl
-qRuo/uhoPjeXaUNziwpRj4rByRdLwQ6MfRcfdZ0TpVJ4J7apJ66pZt85L63u8TZi
-AZdBgcowX6giola7kUUMG66bSi2X6sIFxdXHwzrWlFNnSbKqOqKlGrjmBHUYqRFr
-gLALKkaMpF30olHn7QLqJ2592hMFVkOQAzNr8Xb15mF27BuWFYd7P6TirsxCEd+9
-BFFbhl0vAgMBAAECggEAf1ndN3GBlIi9SL/A7+GiYwpPPz8fWxVFZxmFIXa3QlM+
-CAyR6dC6Z8mL6EiuT9f5VFCzKZzb5g8bxrgk87SFKojvGT3ikTB0NaeNFFDrjjhd
-hTAtG4U8gQFL2gqjcvG3bpQgz13cJc+K1ccXC0dXce/i6Vz/eQjANwfZKuRr4Y/p
-Ml3ZtkarT7cZ2TZRDYd643U2nfrNmeLA9wwwO5mgNYu7HSpI4idpVM+rzoS9ZbU1
-Jxhrdq7GELvnd+Ko3WHTSrkQyb53GtJaLFOSAMxQlZAbRGJMv2VtXSPe+NyA0kxO
-V4O9CJ2QL1Bxqk4MQicKGtGoIsFbfR/qYQZgNEuHoQKBgQDlqEYSGpEcyKNvCjZ+
-Dv5hFUT7GZ24MNRvMBdMHPJEkmxvNpwaNkOGXL6zmhDk5Y0HOrzNv64Llmg623y0
-pn9Vh3CvMKDlq5t8910BJhXzZAM9/E9ui/YuQsFwCdWJQFYVMZUIAdylxbDkg6Mf
-WzHkx82edwWgb7hOFJhEW3h51wKBgQDdZaofTKeSLUmxIbF1yqm4iX67wyhHFFZL
-RZLEcu35ZTJqz3TRpr9KpHCtq9J2gZbqo2Dvqznwfk+yUqT2gkibtk3qvmv/qHQt
-FGX5joWLD2E22UgGc4bzonTSQcxPfPYjNZa6Iv7koPgJqO1k3aYLG3mDMgI/Yurz
-3j8udCI8aQKBgQCe8uNkfky3PkqXfNsQTnJTYTQ4EpettuYg+oj1Xgdz3F/+bS6H
-mc0Mfr5ichyFQDdmrImmYaclT/ld2zGpmp8A5FfieOazbx2T1WTieaixpyPzT+Du
-IyLFC+D0GWpYr1WlH4cFXryHY5c14cIR3r1emcc/iSM00n4eVHS6wxBUSwKBgQCQ
-fTAI20slgD7uxjF90XTwhXNbcONReDlVSKOfZ+5dbCj6QkhYJv4C5czP2yowwyRO
-H7A82j+m75htpbgZVS+zx8eUIxByumqPnRdsRhJrje8BD88MvsrdKDIbomuHmOM+
-WVP0SLCXX2JhU4kS4gDikNy9vZ5v5cu2ul37oDlTuQKBgBl4Qs6Q0xUotlvy/fMf
-L3c6GUIeRsLyQCKFfKYiUNLHqqutBSYmsAlx3XOhcK4DZ7/gFAXO6mpYJlwXOdrI
-IMcSXuJvZDPAYMXOyZUTgNjPcbQ8t0by+EKPwnTReIBf1Y17vPDfisf5IEKZEWvM
-YDXg6cfx9R5QePjvZohlItOu
------END PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-MIID8DCCAtgCAws5AzANBgkqhkiG9w0BAQUFADCBuzELMAkGA1UEBhMCVFIxDjAM
-BgNVBAgUBcdvcnVtMRkwFwYDVQQHHhAAQgBhAV8AbQBhAGsA5wExMRIwEAYDVQQD
-Ewlsb2NhbGhvc3QxHDAaBgNVBAoTE1R3aXN0ZWQgTWF0cml4IExhYnMxJDAiBgNV
-BAsTG0F1dG9tYXRlZCBUZXN0aW5nIEF1dGhvcml0eTEpMCcGCSqGSIb3DQEJARYa
-c2VjdXJpdHlAdHdpc3RlZG1hdHJpeC5jb20wIBcNMTQwOTE2MDE0MjQ2WhgPMjEx
-NDA4MjMwMTQyNDZaMIG7MQswCQYDVQQGEwJUUjEOMAwGA1UECBQFx29ydW0xGTAX
-BgNVBAceEABCAGEBXwBtAGEAawDnATExEjAQBgNVBAMTCWxvY2FsaG9zdDEcMBoG
-A1UEChMTVHdpc3RlZCBNYXRyaXggTGFiczEkMCIGA1UECxMbQXV0b21hdGVkIFRl
-c3RpbmcgQXV0aG9yaXR5MSkwJwYJKoZIhvcNAQkBFhpzZWN1cml0eUB0d2lzdGVk
-bWF0cml4LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMadeH8Y
-wIuk00gAi98FaU+tEIsVbF3YgIthJSrf+PCHfUKmkYqV9pKZRieszGdPiGDZ/qAG
-qWarWAqNCoOSpBC0exKWNFAXnw1EW3P33DjmNGasaz4XnOiCAm7K/v+UpfL8Fc5z
-jQ2h0iWpG6j+6Gg+N5dpQ3OLClGPisHJF0vBDox9Fx91nROlUngntqknrqlm3zkv
-re7xNmIBl0GByjBfqCKiVruRRQwbrptKLZfqwgXF1cfDOtaUU2dJsqo6oqUauOYE
-dRipEWuAsAsqRoykXfSiUeftAuonbn3aEwVWQ5ADM2vxdvXmYXbsG5YVh3s/pOKu
-zEIR370EUVuGXS8CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAK+HqbIUN6qHYYQZw
-7qqsTJWni8NOfK3aguyKAPcdCPP2DCZ6zlxxkUWL57gvsohyDu8Nr9iSI6wePjmI
-cN9eCZdc6mD9kYW4qBYhh2T48TOhEEW7zO6bWQUqWohHW+bG+GfrHnvxIx56OC2B
-eDS2djvvScYm45etlHprfrVEDIssh956O6qJCySax3D2w+i8YanXji1EbS61XTDw
-CMxDdWYmd2MDARRwlMcfcUIfKZUGl5NmqpnOx+H5MyAGwt86s647GMYZborQh+Mj
-tNHVpyKf/a8/HjqP1sCOrjCPZIjP6Qp5j4gifAjUStNmCgaBe7CpFtBSLnHqb4o/
-gU7u1w==
------END CERTIFICATE-----
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 087ad958f..365ec81fd 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -206,8 +206,8 @@ class MockServer():
 
 def ssl_context_factory():
     return ssl.DefaultOpenSSLContextFactory(
-         os.path.join(os.path.dirname(__file__), 'keys/server.pem'),
-         os.path.join(os.path.dirname(__file__), 'keys/server.pem'),
+         os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
+         os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
          )
 
 

From 3f946b5278cb964ce1dab1b6e8a3a74e16d00fca Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Sat, 20 Feb 2016 02:07:45 +0100
Subject: [PATCH 0880/2974] Fix super() call

---
 scrapy/core/downloader/contextfactory.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 5e6ad57a1..afc52c354 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -24,7 +24,7 @@ try:
         """
 
         def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs):
-            super(BrowserLikePolicyForHTTPS, self).__init__(*args, **kwargs)
+            super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
             self._ssl_method = method
 
         def getCertificateOptions(self):

From 6174192564250d715ffa69b492bc58a901fba07f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Sat, 20 Feb 2016 23:27:04 +0100
Subject: [PATCH 0881/2974] Add "Content-Length: 0" for body-less HTTP/1.1 POST
 requests

GH-823 was fixed only for HTTP/1.0 (in GH-1089)
---
 scrapy/core/downloader/handlers/http11.py | 16 +++++++++++-
 tests/test_downloader_handlers.py         | 32 +++++++++++++++++++++++
 2 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index ad3285a32..ca13c609d 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -209,7 +209,21 @@ class ScrapyAgent(object):
         headers = TxHeaders(request.headers)
         if isinstance(agent, self._TunnelingAgent):
             headers.removeHeader(b'Proxy-Authorization')
-        bodyproducer = _RequestBodyProducer(request.body) if request.body else None
+        if request.body:
+            bodyproducer = _RequestBodyProducer(request.body)
+        else:
+            bodyproducer = None
+            # Setting Content-Length: 0 even for POST requests is not a
+            # MUST per HTTP RFCs, but it's common behavior, and some
+            # server require this otherwise returing HTTP 411
+            #
+            # RFC 7230#section-3.3.2:
+            # "a Content-Length header field is normally sent in a POST
+            # request even when the value is 0 (indicating an empty payload body)."
+            #
+            # Twisted Agent will not add "Content-Length: 0" by itself
+            if method == b'POST':
+                headers.addRawHeader(b'Content-Length', b'0')
 
         start_time = time()
         d = agent.request(
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 1885a53a0..88b60e4ff 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -105,6 +105,15 @@ class FileTestCase(unittest.TestCase):
         return self.assertFailure(d, IOError)
 
 
+class ContentLengthHeaderResource(resource.Resource):
+    """
+    A testing resource which renders itself as the value of the Content-Length
+    header from the request.
+    """
+    def render(self, request):
+        return request.requestHeaders.getRawHeaders(b"content-length")[0]
+
+
 class HttpTestCase(unittest.TestCase):
 
     scheme = 'http'
@@ -122,6 +131,7 @@ class HttpTestCase(unittest.TestCase):
         r.putChild(b"host", HostHeaderResource())
         r.putChild(b"payload", PayloadResource())
         r.putChild(b"broken", BrokenDownloadResource())
+        r.putChild(b"contentlength", ContentLengthHeaderResource())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
         self.host = 'localhost'
@@ -209,6 +219,28 @@ class HttpTestCase(unittest.TestCase):
         d.addCallback(self.assertEquals, b'example.com')
         return d
 
+    def test_content_length_zero_bodyless_post_request_headers(self):
+        """Tests if "Content-Length: 0" is sent for bodyless POST requests.
+
+        This is not strictly required by HTTP RFCs but can cause trouble
+        for some web servers.
+        See:
+        https://github.com/scrapy/scrapy/issues/823
+        https://issues.apache.org/jira/browse/TS-2902
+        https://github.com/kennethreitz/requests/issues/405
+        https://bugs.python.org/issue14721
+        """
+        def _test(response):
+            self.assertEquals(response.body, b'0')
+
+        request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'})
+        return self.download_request(request, Spider('foo')).addCallback(_test)
+
+        d = self.download_request(request, Spider('foo'))
+        d.addCallback(lambda r: r.body)
+        d.addCallback(self.assertEquals, b'0')
+        return d
+
     def test_payload(self):
         body = b'1'*100 # PayloadResource requires body length to be 100
         request = Request(self.getURL('payload'), method='POST', body=body)

From ac8f97c33bccc7647d80d9c3ccdf1eafc22636e3 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Sun, 21 Feb 2016 00:49:41 +0100
Subject: [PATCH 0882/2974] Fix typo in comment

---
 scrapy/core/downloader/handlers/http11.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index ca13c609d..b831d1ae8 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -215,7 +215,7 @@ class ScrapyAgent(object):
             bodyproducer = None
             # Setting Content-Length: 0 even for POST requests is not a
             # MUST per HTTP RFCs, but it's common behavior, and some
-            # server require this otherwise returing HTTP 411
+            # servers require this, otherwise returning HTTP 411 Length required
             #
             # RFC 7230#section-3.3.2:
             # "a Content-Length header field is normally sent in a POST

From e9bd328936e1d0172fd89033f0b847c63045423a Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Sun, 21 Feb 2016 01:08:50 +0100
Subject: [PATCH 0883/2974] Fix SkipTest() message for botocore import test

---
 scrapy/utils/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index bf66a8cbe..2aa530902 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -24,7 +24,7 @@ def skip_if_no_boto():
     try:
         is_botocore()
     except NotConfigured as e:
-        raise SkipTest(e.message)
+        raise SkipTest(str(e))
 
 def get_s3_content_and_delete(bucket, path, with_key=False):
     """ Get content from s3 key, and delete key afterwards.

From ad4c1169c49c0ac6becb070ef4fab1439a9eda6b Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Sun, 21 Feb 2016 01:16:46 +0100
Subject: [PATCH 0884/2974] Pass exception directly to SkipTest()

It prints the same as passing `str(e)`
---
 scrapy/utils/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 2aa530902..d2ef68912 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -24,7 +24,7 @@ def skip_if_no_boto():
     try:
         is_botocore()
     except NotConfigured as e:
-        raise SkipTest(str(e))
+        raise SkipTest(e)
 
 def get_s3_content_and_delete(bucket, path, with_key=False):
     """ Get content from s3 key, and delete key afterwards.

From 35fb630c4ec6c996a90cc4f50085275525ec9105 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 23 Feb 2016 16:28:48 +0100
Subject: [PATCH 0885/2974] Revert "[MRG+1] Cleanup http11 tunneling connection
 after #1678"

---
 scrapy/core/downloader/handlers/http11.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index ad3285a32..a4d5a28c8 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -92,9 +92,11 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
 
     def requestTunnel(self, protocol):
         """Asks the proxy to open a tunnel."""
-        tunnelReq = to_bytes(
-            'CONNECT %s:%s HTTP/1.1\r\n' % (
-                self._tunneledHost, self._tunneledPort), encoding='ascii')
+        tunnelReq = (
+            b'CONNECT ' +
+            to_bytes(self._tunneledHost, encoding='ascii') + b':' +
+            to_bytes(str(self._tunneledPort)) +
+            b' HTTP/1.1\r\n')
         if self._proxyAuthHeader:
             tunnelReq += \
                 b'Proxy-Authorization: ' + self._proxyAuthHeader + b'\r\n'

From 095495e99ce327d0fc6cf2064f8a6d871d2f7dd1 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 24 Feb 2016 01:24:58 +0100
Subject: [PATCH 0886/2974] Backward-compatibility for common Scrapy context
 factory patterns

---
 scrapy/core/downloader/contextfactory.py | 13 ++++++++++++-
 scrapy/core/downloader/tls.py            |  4 ++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index afc52c354..09fcd491a 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -30,7 +30,18 @@ try:
         def getCertificateOptions(self):
             # setting verify=True will require you to provide CAs
             # to verify against; in other words: it's not that simple
-            return CertificateOptions(verify=False, method=self._ssl_method)
+
+            # backward-compatible SSL/TLS method:
+            #
+            # * this will respect `method` attribute in often recommended
+            #   `ScrapyClientContextFactory` subclass
+            #   (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
+            #
+            # * getattr() for `_ssl_method` attribute for context factories
+            #   not calling super(..., self).__init__
+            return CertificateOptions(verify=False,
+                        method=getattr(self, 'method',
+                                       getattr(self, '_ssl_method', None)))
 
         # kept for old-style HTTP/1.0 downloader context twisted calls,
         # e.g. connectSSL()
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 121ee83ac..64ebb0714 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -8,8 +8,8 @@ METHOD_TLSv11 = 'TLSv1.1'
 METHOD_TLSv12 = 'TLSv1.2'
 
 openssl_methods = {
-    METHOD_TLS: SSL.SSLv23_METHOD,                      # protocol negotiation (recommended)
-    METHOD_SSLv3: SSL.SSLv3_METHOD,                     # SSL 3 (NOT recommended)
+    METHOD_TLS:    SSL.SSLv23_METHOD,                   # protocol negotiation (recommended)
+    METHOD_SSLv3:  SSL.SSLv3_METHOD,                    # SSL 3 (NOT recommended)
     METHOD_TLSv10: SSL.TLSv1_METHOD,                    # TLS 1.0 only
     METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only
     METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only

From c9890d5f7662dc2673696ceb80870c5318746495 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 24 Feb 2016 01:26:04 +0100
Subject: [PATCH 0887/2974] Add warning for context factories not accepting
 `method` param

---
 scrapy/core/downloader/handlers/http11.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 522bac16f..1fe31b206 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -4,6 +4,7 @@ import re
 import logging
 from io import BytesIO
 from time import time
+import warnings
 from six.moves.urllib.parse import urldefrag
 
 from zope.interface import implementer
@@ -18,7 +19,7 @@ from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \
 from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.core.downloader.webclient import _parse
-from scrapy.core.downloader.tls import openssl_methods, METHOD_TLS
+from scrapy.core.downloader.tls import openssl_methods
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_bytes, to_unicode
 from scrapy import twisted_version
@@ -39,9 +40,12 @@ class HTTP11DownloadHandler(object):
         try:
             self._contextFactory = self._contextFactoryClass(method=self._sslMethod)
         except TypeError:
-            # use defaults
+            # use context factory defaults
             self._contextFactory = self._contextFactoryClass()
-
+            warnings.warn("""
+    You are using a context factory class that does not accept the `method` argument
+    (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD).
+    Please upgrade your context factory class to handle or ignore it.""")
         self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
         self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
         self._disconnect_timeout = 1

From 164f300762297bb6069a25cfc1cbb819e1bff9ec Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 19 Feb 2016 18:22:56 +0300
Subject: [PATCH 0888/2974] See #1778 - change default S3 ACL to "private" and
 allow customization via settings

---
 docs/topics/feed-exports.rst        | 3 +++
 docs/topics/settings.rst            | 9 +++++++++
 scrapy/pipelines/files.py           | 4 +++-
 scrapy/settings/default_settings.py | 2 ++
 4 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index e5037129c..1f17ccd51 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -185,6 +185,9 @@ passed through the following settings:
  * :setting:`AWS_ACCESS_KEY_ID`
  * :setting:`AWS_SECRET_ACCESS_KEY`
 
+Default access policy for uploaded files is ``private``, it can be changed
+(for example, to ``public-read``) via :setting:`S3_STORE_ACL`.
+
 .. _topics-feed-storage-stdout:
 
 Standard output
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index f8f35b5e3..b7d6f2d9a 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -926,6 +926,15 @@ If enabled, Scrapy will respect robots.txt policies. For more information see
     this option is enabled by default in settings.py file generated
     by ``scrapy startproject`` command.
 
+.. setting:: S3_STORE_ACL
+
+S3_STORE_ACL
+------------
+
+Default: ``'private'``
+
+S3-specific access control policy (ACL) for uploaded files.
+
 .. setting:: SCHEDULER
 
 SCHEDULER
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 45ceddcbb..b48f43094 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -82,7 +82,8 @@ class S3FilesStore(object):
     AWS_ACCESS_KEY_ID = None
     AWS_SECRET_ACCESS_KEY = None
 
-    POLICY = 'public-read'
+    POLICY = 'private'  # Overriden from settings.S3_STORE_ACL in
+                        # FilesPipeline.from_settings.
     HEADERS = {
         'Cache-Control': 'max-age=172800',
     }
@@ -232,6 +233,7 @@ class FilesPipeline(MediaPipeline):
         s3store = cls.STORE_SCHEMES['s3']
         s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
+        s3store.POLICY = settings['S3_STORE_ACL']
 
         cls.FILES_URLS_FIELD = settings.get('FILES_URLS_FIELD', cls.DEFAULT_FILES_URLS_FIELD)
         cls.FILES_RESULT_FIELD = settings.get('FILES_RESULT_FIELD', cls.DEFAULT_FILES_RESULT_FIELD)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 44e74dc61..e31dc6aef 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -231,6 +231,8 @@ SCHEDULER = 'scrapy.core.scheduler.Scheduler'
 SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
 SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
 
+S3_STORE_ACL = 'private'
+
 SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
 
 SPIDER_MIDDLEWARES = {}

From 6137dd96d9b52e1a4e4e28dd64924f877a63329f Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Wed, 24 Feb 2016 10:16:10 +0300
Subject: [PATCH 0889/2974] Fix documentation for S3_STORE_ACL (now
 settings.FILES_STORE_S3_ACL) settings: it has nothing to do with feed
 exporters.

---
 docs/topics/feed-exports.rst        |  3 ---
 docs/topics/settings.rst            | 18 +++++++++---------
 scrapy/pipelines/files.py           |  4 ++--
 scrapy/settings/default_settings.py |  4 ++--
 4 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 1f17ccd51..e5037129c 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -185,9 +185,6 @@ passed through the following settings:
  * :setting:`AWS_ACCESS_KEY_ID`
  * :setting:`AWS_SECRET_ACCESS_KEY`
 
-Default access policy for uploaded files is ``private``, it can be changed
-(for example, to ``public-read``) via :setting:`S3_STORE_ACL`.
-
 .. _topics-feed-storage-stdout:
 
 Standard output
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index b7d6f2d9a..0b1d99dba 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -607,6 +607,15 @@ For more information See the :ref:`extensions user guide  <topics-extensions>`
 and the :ref:`list of available extensions <topics-extensions-ref>`.
 
 
+.. setting:: FILES_STORE_S3_ACL
+
+FILES_STORE_S3_ACL
+------------------
+
+Default: ``'private'``
+
+S3-specific access control policy (ACL) for S3 files store.
+
 .. setting:: ITEM_PIPELINES
 
 ITEM_PIPELINES
@@ -926,15 +935,6 @@ If enabled, Scrapy will respect robots.txt policies. For more information see
     this option is enabled by default in settings.py file generated
     by ``scrapy startproject`` command.
 
-.. setting:: S3_STORE_ACL
-
-S3_STORE_ACL
-------------
-
-Default: ``'private'``
-
-S3-specific access control policy (ACL) for uploaded files.
-
 .. setting:: SCHEDULER
 
 SCHEDULER
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index b48f43094..1fd2571e2 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -82,7 +82,7 @@ class S3FilesStore(object):
     AWS_ACCESS_KEY_ID = None
     AWS_SECRET_ACCESS_KEY = None
 
-    POLICY = 'private'  # Overriden from settings.S3_STORE_ACL in
+    POLICY = 'private'  # Overriden from settings.FILES_STORE_S3_ACL in
                         # FilesPipeline.from_settings.
     HEADERS = {
         'Cache-Control': 'max-age=172800',
@@ -233,7 +233,7 @@ class FilesPipeline(MediaPipeline):
         s3store = cls.STORE_SCHEMES['s3']
         s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
-        s3store.POLICY = settings['S3_STORE_ACL']
+        s3store.POLICY = settings['FILES_STORE_S3_ACL']
 
         cls.FILES_URLS_FIELD = settings.get('FILES_URLS_FIELD', cls.DEFAULT_FILES_URLS_FIELD)
         cls.FILES_RESULT_FIELD = settings.get('FILES_RESULT_FIELD', cls.DEFAULT_FILES_RESULT_FIELD)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index e31dc6aef..6a989d048 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -156,6 +156,8 @@ FEED_EXPORTERS_BASE = {
     'pickle': 'scrapy.exporters.PickleItemExporter',
 }
 
+FILES_STORE_S3_ACL = 'private'
+
 HTTPCACHE_ENABLED = False
 HTTPCACHE_DIR = 'httpcache'
 HTTPCACHE_IGNORE_MISSING = False
@@ -231,8 +233,6 @@ SCHEDULER = 'scrapy.core.scheduler.Scheduler'
 SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
 SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
 
-S3_STORE_ACL = 'private'
-
 SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
 
 SPIDER_MIDDLEWARES = {}

From c29a1b98b3b49a961c67ac64fa303e0265dab2d1 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 24 Feb 2016 16:01:18 +0100
Subject: [PATCH 0890/2974] Make warning message a bit nicer in logs

---
 scrapy/core/downloader/handlers/http11.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 1fe31b206..40ad2d814 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -42,10 +42,11 @@ class HTTP11DownloadHandler(object):
         except TypeError:
             # use context factory defaults
             self._contextFactory = self._contextFactoryClass()
-            warnings.warn("""
+            msg = """
     You are using a context factory class that does not accept the `method` argument
     (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD).
-    Please upgrade your context factory class to handle or ignore it.""")
+    Please upgrade your context factory class to handle or ignore it."""
+            warnings.warn(msg)
         self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
         self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
         self._disconnect_timeout = 1

From 0336c2504af45a87d2aae4c1ed45039c0e0ea183 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 24 Feb 2016 16:42:25 +0100
Subject: [PATCH 0891/2974] Use context factory class name in warning message

---
 scrapy/core/downloader/handlers/http11.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 40ad2d814..00c09c75c 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -43,9 +43,10 @@ class HTTP11DownloadHandler(object):
             # use context factory defaults
             self._contextFactory = self._contextFactoryClass()
             msg = """
-    You are using a context factory class that does not accept the `method` argument
-    (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD).
-    Please upgrade your context factory class to handle or ignore it."""
+ '%s' does not accept `method` argument (type OpenSSL.SSL method,\
+ e.g. OpenSSL.SSL.SSLv23_METHOD).\
+ Please upgrade your context factory class to handle it or ignore it.""" % (
+                settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
             warnings.warn(msg)
         self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
         self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')

From ecddc093a4b9cb6057f229df82c5c5693fca9612 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 22 Feb 2016 17:52:26 +0100
Subject: [PATCH 0892/2974] Explicitly call Twisted transport stopProducing()
 on HTTP/1.1 timeouts

---
 scrapy/core/downloader/handlers/http11.py | 10 ++++++++++
 tests/test_downloader_handlers.py         | 12 +++++++-----
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 5261f542b..6fc2bd2bd 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -193,6 +193,7 @@ class ScrapyAgent(object):
         self._pool = pool
         self._maxsize = maxsize
         self._warnsize = warnsize
+        self._txresponse = None
 
     def _get_agent(self, request, timeout):
         bindaddress = request.meta.get('bindaddress') or self._bindAddress
@@ -259,6 +260,11 @@ class ScrapyAgent(object):
         if self._timeout_cl.active():
             self._timeout_cl.cancel()
             return result
+        # needed for HTTPS requests, otherwise _ResponseReader doesn't
+        # receive connectionLost()
+        if self._txresponse:
+            self._txresponse._transport.stopProducing()
+
         raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout))
 
     def _cb_latency(self, result, request, start_time):
@@ -294,6 +300,10 @@ class ScrapyAgent(object):
 
         d = defer.Deferred(_cancel)
         txresponse.deliverBody(_ResponseReader(d, txresponse, request, maxsize, warnsize))
+
+        # save response for timeouts
+        self._txresponse = txresponse
+
         return d
 
     def _cb_bodydone(self, result, request, url):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 88b60e4ff..e13becf1b 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -182,17 +182,19 @@ class HttpTestCase(unittest.TestCase):
         return d
 
     @defer.inlineCallbacks
-    def test_timeout_download_from_spider(self):
-        if self.scheme == 'https':
-            raise unittest.SkipTest(
-                'test_timeout_download_from_spider skipped under https')
+    def test_timeout_download_from_spider_nodata_rcvd(self):
+        # client connects but no data is received
         spider = Spider('foo')
         meta = {'download_timeout': 0.2}
-        # client connects but no data is received
         request = Request(self.getURL('wait'), meta=meta)
         d = self.download_request(request, spider)
         yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)
+
+    @defer.inlineCallbacks
+    def test_timeout_download_from_spider_server_hangs(self):
         # client connects, server send headers and some body bytes but hangs
+        spider = Spider('foo')
+        meta = {'download_timeout': 0.2}
         request = Request(self.getURL('hang-after-headers'), meta=meta)
         d = self.download_request(request, spider)
         yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)

From c9e781357da20d8dbb9302e3c983a5dbcd2afb84 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 22 Feb 2016 18:44:22 +0100
Subject: [PATCH 0893/2974] Explicitly call Twisted transport stopProducing()
 on HTTP/1.0 timeouts

---
 scrapy/core/downloader/webclient.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 9bcc51943..1c89a0f9e 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -83,6 +83,11 @@ class ScrapyHTTPPageGetter(HTTPClient):
 
     def timeout(self):
         self.transport.loseConnection()
+
+        # transport cleanup needed for HTTPS connections
+        if self.factory.url.startswith(b'https'):
+            self.transport.stopProducing()
+
         self.factory.noPage(\
                 defer.TimeoutError("Getting %s took longer than %s seconds." % \
                 (self.factory.url, self.factory.timeout)))

From 84dea1945569dd8003ccda97ab068e536e5f5120 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 25 Feb 2016 00:59:15 +0100
Subject: [PATCH 0894/2974] Update release notes

---
 docs/news.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 423ec2a40..da29ed58b 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -97,6 +97,7 @@ Additional New Features and Enhancements
 
 - Dict-like settings now have per-key priorities
   (:issue:`1135`, :issue:`1149` and :issue:`1586`).
+- Sending non-ASCII emails (:issue:`1662`)
 - ``CloseSpider`` and ``SpiderState`` extensions now get disabled if no relevant
   setting is set (:issue:`1723`, :issue:`1725`).
 - Added method ``ExecutionEngine.close`` (:issue:`1423`).
@@ -105,7 +106,7 @@ Additional New Features and Enhancements
   :issue:`1335`, :issue:`1683`, :issue:`1660`, :issue:`1642`, :issue:`1721`,
   :issue:`1727`).
 - Other refactoring, optimizations and cleanup (:issue:`1476`, :issue:`1481`,
-  :issue:`1477`, :issue:`1315` and :issue:`1290`).
+  :issue:`1477`, :issue:`1315`, :issue:`1290` and :issue:`1750`).
 
 .. _`Code of Conduct`: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
 

From 1ce4c86c2b6677133d6bcafc9ed36ebbddfc2d2e Mon Sep 17 00:00:00 2001
From: Lucas Moauro <lagenar@gmail.com>
Date: Thu, 25 Feb 2016 21:08:50 -0300
Subject: [PATCH 0895/2974] Refactored SpiderLoader class constructor for
 easier subclassing

---
 scrapy/spiderloader.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index a88db27c8..fbf68cec4 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -17,14 +17,17 @@ class SpiderLoader(object):
     def __init__(self, settings):
         self.spider_modules = settings.getlist('SPIDER_MODULES')
         self._spiders = {}
-        for name in self.spider_modules:
-            for module in walk_modules(name):
-                self._load_spiders(module)
-
+        self._load_all_spiders()
+            
     def _load_spiders(self, module):
         for spcls in iter_spider_classes(module):
             self._spiders[spcls.name] = spcls
 
+    def _load_all_spiders(self):
+        for name in self.spider_modules:
+            for module in walk_modules(name):
+                self._load_spiders(module)
+
     @classmethod
     def from_settings(cls, settings):
         return cls(settings)

From 174f526709deaabedf6960a64caab416b192f962 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 26 Feb 2016 18:19:52 +0100
Subject: [PATCH 0896/2974] Document DOWNLOADER_* settings for HTTP/1.0 and TLS

---
 docs/topics/settings.rst | 72 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 72 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0b1d99dba..3bd0f9df5 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -366,6 +366,78 @@ Default: ``'scrapy.core.downloader.Downloader'``
 
 The downloader to use for crawling.
 
+.. setting:: DOWNLOADER_HTTPCLIENTFACTORY
+
+DOWNLOADER_HTTPCLIENTFACTORY
+----------------------------
+
+Default: ``'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'``
+
+Defines a Twisted ``protocol.ClientFactory``  class to use for HTTP/1.0
+connections (for ``HTTP10DownloadHandler``).
+
+.. note::
+
+    HTTP/1.0 is rarely used nowadays so you can safely ignore this setting,
+    unless you use Twisted<11.1, or if you really want to use HTTP/1.0
+    and override :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme
+    accordingly, i.e. to
+    ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
+
+.. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY
+
+DOWNLOADER_CLIENTCONTEXTFACTORY
+-------------------------------
+
+Default: ``'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'``
+
+Represents the classpath to the ContextFactory to use.
+
+Here, "ContextFactory" is a Twisted term for SSL/TLS contexts, defining
+the TLS/SSL protocol version to use, whether to do certificate verification,
+or even enable client-side authentication (and various other things).
+
+.. note::
+
+    Scrapy default context factory **does NOT perform remote server
+    certificate verification**. This is usually fine for web scraping.
+
+    If you do need remote server certificate verification enabled,
+    Scrapy also has another context factory class that you can set,
+    ``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``,
+    which uses the platform's certificates to validate remote endpoints.
+    **This is only available if you use Twisted>=14.0.**
+
+If you do use a custom ContextFactory, make sure it accepts a ``method``
+parameter at init (this is the ``OpenSSL.SSL`` method mapping
+:setting:`DOWNLOADER_CLIENT_TLS_METHOD`).
+
+.. setting:: DOWNLOADER_CLIENT_TLS_METHOD
+
+DOWNLOADER_CLIENT_TLS_METHOD
+----------------------------
+
+Default: ``'TLS'``
+
+Use this setting to customize the TLS/SSL method used by the default
+HTTP/1.1 downloader.
+
+This setting must be one of these string values:
+
+- ``'TLS'``: maps to OpenSSL's ``TLS_method()`` (a.k.a ``SSLv23_method()``),
+  which allows protocol negotiation, starting from the highest supported
+  by the platform; **default, recommended**
+- ``'TLSv1.0'``: this value forces HTTPS connections to use TLS version 1.0 ;
+  set this if you want the behavior of Scrapy<1.1
+- ``'TLSv1.1'``: forces TLS version 1.1
+- ``'TLSv1.2'``: forces TLS version 1.2
+- ``'SSLv3'``: forces SSL version 3 (**not recommended**)
+
+.. note::
+
+    We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
+    or above (Twisted>=14.0 if you can).
+
 .. setting:: DOWNLOADER_MIDDLEWARES
 
 DOWNLOADER_MIDDLEWARES

From 709b4fa8845ec9e5b15fa60fe20237a327cd79bd Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 26 Feb 2016 18:35:29 +0100
Subject: [PATCH 0897/2974] Update release notes about HTTPS downloader

---
 docs/news.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index da29ed58b..d3457a775 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -26,6 +26,9 @@ This 1.1 release brings a lot of interesting features and bug fixes:
   - Selectors were extracted to the parsel_ library (:issue:`1409`). This means
     you can use Scrapy Selectors without Scrapy and also upgrade the
     selectors engine without needing to upgrade Scrapy.
+  - HTTPS downloader now does TLS protocol negotiation by default,
+    instead of forcing TLS 1.0. You can also set the SSL/TLS method
+    using the new :setting:`DOWNLOADER_CLIENT_TLS_METHOD`.
 
 - These bug fixes may require your attention:
 
@@ -85,6 +88,10 @@ Additional New Features and Enhancements
   interval (:issue:`1282`).
 - Download handlers are now lazy-loaded on first request using their
   scheme (:issue:`1390`, :issue:`1421`).
+- HTTPS download handlers do not force TLS 1.0 anymore; instead,
+  OpenSSL's ``SSLv23_method()/TLS_method()`` is used allowing to try
+  negotiating with the remote hosts the highest TLS protocol version
+  it can (:issue:`1794`, :issue:`1629`).
 - ``RedirectMiddleware`` now skips the status codes from
   ``handle_httpstatus_list`` on spider attribute
   or in ``Request``'s ``meta`` key (:issue:`1334`, :issue:`1364`,

From 90c64a6d136bc28a8deb4cfecced61d1fe136491 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 29 Feb 2016 12:27:25 +0100
Subject: [PATCH 0898/2974] Update release notes about change of default S3 ACL
 policy to "private"

---
 docs/news.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index d3457a775..e6625c99f 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -46,6 +46,10 @@ This 1.1 release brings a lot of interesting features and bug fixes:
     If you use ``PythonItemExporter``, you may want to update your code to
     disable binary mode which is now deprecated.
   - Accept XML node names containing dots as valid (:issue:`1533`).
+  - When uploading files or images to S3 (with ``FilesPipeline`` or
+    ``ImagesPipeline``), the default ACL policy is now "private" instead
+    of "public" **Warning: backwards incompatible!**.
+    You can use :setting:`FILES_STORE_S3_ACL` to change it.
 
 Keep reading for more details on other improvements and bug fixes.
 
@@ -64,7 +68,6 @@ Almost all builtin extensions/middlewares are expected to work. However, we are
 some limitations in Python 3:
 
 - Scrapy doesn't work yet in Windows with Python 3 (non-Python 3 ported Twisted dependency)
-- S3 downloads are not supported (see :issue:`1718`)
 - Sending emails is not supported (non-Python 3 ported Twisted dependency)
 - FTP download handler is not supported (non-Python 3 ported Twisted
   dependency)

From f3c7a509b96a387b06e0f5adab740d9e39b1d516 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 1 Mar 2016 11:00:20 +0100
Subject: [PATCH 0899/2974] Build universal wheels

---
 setup.cfg | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/setup.cfg b/setup.cfg
index 5e219a08d..2296a1052 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,2 +1,5 @@
 [bdist_rpm]
 doc_files = docs AUTHORS INSTALL LICENSE README.rst
+
+[bdist_wheel]
+universal=1

From 94e28adfb7049be8ec0e9ce00bab1ab3a0b25301 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 1 Mar 2016 16:29:12 +0500
Subject: [PATCH 0900/2974] Extract a function to build CONNECT request; add
 tests for it. See GH-1701 and GH-1808.

---
 scrapy/core/downloader/handlers/http11.py | 34 +++++++++++++++++------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 5261f542b..c93a48597 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -107,15 +107,8 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
 
     def requestTunnel(self, protocol):
         """Asks the proxy to open a tunnel."""
-        tunnelReq = (
-            b'CONNECT ' +
-            to_bytes(self._tunneledHost, encoding='ascii') + b':' +
-            to_bytes(str(self._tunneledPort)) +
-            b' HTTP/1.1\r\n')
-        if self._proxyAuthHeader:
-            tunnelReq += \
-                b'Proxy-Authorization: ' + self._proxyAuthHeader + b'\r\n'
-        tunnelReq += b'\r\n'
+        tunnelReq = tunnel_request_data(self._tunneledHost, self._tunneledPort,
+                                        self._proxyAuthHeader)
         protocol.transport.write(tunnelReq)
         self._protocolDataReceived = protocol.dataReceived
         protocol.dataReceived = self.processProxyResponse
@@ -149,6 +142,29 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         return self._tunnelReadyDeferred
 
 
+def tunnel_request_data(host, port, proxy_auth_header=None):
+    r"""
+    Return binary content of a CONNECT request.
+
+    >>> from scrapy.utils.python import to_native_str as s
+    >>> s(tunnel_request_data("example.com", 8080))
+    'CONNECT example.com:8080 HTTP/1.1\r\n\r\n'
+    >>> s(tunnel_request_data("example.com", 8080, b"123"))
+    'CONNECT example.com:8080 HTTP/1.1\r\nProxy-Authorization: 123\r\n\r\n'
+    >>> s(tunnel_request_data(b"example.com", "8090"))
+    'CONNECT example.com:8090 HTTP/1.1\r\n\r\n'
+    """
+    tunnel_req = (
+        b'CONNECT ' +
+        to_bytes(host, encoding='ascii') + b':' +
+        to_bytes(str(port)) +
+        b' HTTP/1.1\r\n')
+    if proxy_auth_header:
+        tunnel_req += b'Proxy-Authorization: ' + proxy_auth_header + b'\r\n'
+    tunnel_req += b'\r\n'
+    return tunnel_req
+
+
 class TunnelingAgent(Agent):
     """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS
     downloads. It may look strange that we have chosen to subclass Agent and not

From 17d3bec699b0b851c02fdad40ab6056ef96c5877 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 1 Mar 2016 16:34:13 +0500
Subject: [PATCH 0901/2974] declare Python 3 support in setup.py

---
 setup.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/setup.py b/setup.py
index 4eb8d2318..d91bb6328 100644
--- a/setup.py
+++ b/setup.py
@@ -32,6 +32,10 @@ setup(
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',

From 2a6524ee3a21283db01dae8bf3a11832eca79614 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sat, 27 Feb 2016 15:18:31 +0000
Subject: [PATCH 0902/2974] Allow core Scheduler priority queue customization

---
 scrapy/core/scheduler.py            | 13 ++++++++-----
 scrapy/settings/default_settings.py |  1 +
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index d132bbfd3..8be9a0019 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -3,7 +3,6 @@ import json
 import logging
 from os.path import join, exists
 
-from queuelib import PriorityQueue
 from scrapy.utils.reqser import request_to_dict, request_from_dict
 from scrapy.utils.misc import load_object
 from scrapy.utils.job import job_dir
@@ -13,9 +12,11 @@ logger = logging.getLogger(__name__)
 
 class Scheduler(object):
 
-    def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, logunser=False, stats=None):
+    def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None,
+                 logunser=False, stats=None, pqclass=None):
         self.df = dupefilter
         self.dqdir = self._dqdir(jobdir)
+        self.pqclass = pqclass
         self.dqclass = dqclass
         self.mqclass = mqclass
         self.logunser = logunser
@@ -26,17 +27,19 @@ class Scheduler(object):
         settings = crawler.settings
         dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
         dupefilter = dupefilter_cls.from_settings(settings)
+        pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
         dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
         mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
         logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS')
-        return cls(dupefilter, job_dir(settings), dqclass, mqclass, logunser, crawler.stats)
+        return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
+                   stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass)
 
     def has_pending_requests(self):
         return len(self) > 0
 
     def open(self, spider):
         self.spider = spider
-        self.mqs = PriorityQueue(self._newmq)
+        self.mqs = self.pqclass(self._newmq)
         self.dqs = self._dq() if self.dqdir else None
         return self.df.open()
 
@@ -112,7 +115,7 @@ class Scheduler(object):
                 prios = json.load(f)
         else:
             prios = ()
-        q = PriorityQueue(self._newdq, startprios=prios)
+        q = self.pqclass(self._newdq, startprios=prios)
         if q:
             logger.info("Resuming crawl (%(queuesize)d requests scheduled)",
                         {'queuesize': len(q)}, extra={'spider': self.spider})
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8e6d29045..d449c4891 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -234,6 +234,7 @@ ROBOTSTXT_OBEY = False
 SCHEDULER = 'scrapy.core.scheduler.Scheduler'
 SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
 SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
+SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'
 
 SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
 

From 0e288d4a711f061a01a951d2e1d9892f737211e0 Mon Sep 17 00:00:00 2001
From: djunzu <djunzu@users.noreply.github.com>
Date: Tue, 1 Mar 2016 21:02:13 -0300
Subject: [PATCH 0903/2974] Add pps extension to IGNORED_EXTENSIONS

	modified:   scrapy/linkextractors/__init__.py
---
 scrapy/linkextractors/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 64efa0c55..efb11e43c 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -30,7 +30,8 @@ IGNORED_EXTENSIONS = [
     'm4a',
 
     # office suites
-    'xls', 'xlsx', 'ppt', 'pptx', 'doc', 'docx', 'odt', 'ods', 'odg', 'odp',
+    'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg',
+    'odp',
 
     # other
     'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar',

From 5876b9aa30c199b75f9f85c0cc17234e4dbb23ac Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Wed, 2 Mar 2016 01:13:02 +0000
Subject: [PATCH 0904/2974] Update documentation links

---
 docs/contributing.rst                 |  4 ++--
 docs/faq.rst                          | 14 +++++++-------
 docs/intro/install.rst                |  8 ++++----
 docs/intro/overview.rst               |  6 +++---
 docs/intro/tutorial.rst               |  8 ++++----
 docs/news.rst                         |  9 +++++----
 docs/topics/api.rst                   |  2 +-
 docs/topics/architecture.rst          |  6 +++---
 docs/topics/djangoitem.rst            |  2 +-
 docs/topics/downloader-middleware.rst | 14 +++++++-------
 docs/topics/email.rst                 |  2 +-
 docs/topics/extensions.rst            |  6 +++---
 docs/topics/feed-exports.rst          |  4 ++--
 docs/topics/firebug.rst               |  2 +-
 docs/topics/item-pipeline.rst         |  4 ++--
 docs/topics/media-pipeline.rst        |  8 ++++----
 docs/topics/practices.rst             |  2 +-
 docs/topics/request-response.rst      |  2 +-
 docs/topics/selectors.rst             | 10 +++++-----
 docs/topics/settings.rst              |  6 +++---
 docs/topics/shell.rst                 |  2 +-
 docs/topics/signals.rst               |  4 ++--
 docs/topics/spider-middleware.rst     |  4 ++--
 docs/topics/spiders.rst               |  2 +-
 docs/topics/webservice.rst            |  2 +-
 docs/versioning.rst                   |  2 +-
 26 files changed, 68 insertions(+), 67 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 548d3d18d..a21af3ff4 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -120,7 +120,7 @@ Scrapy Contrib
 ==============
 
 Scrapy contrib shares a similar rationale as Django contrib, which is explained
-in `this post <http://jacobian.org/writing/what-is-django-contrib/>`_. If you
+in `this post <https://jacobian.org/writing/what-is-django-contrib/>`_. If you
 are working on a new functionality, please follow that rationale to decide
 whether it should be a Scrapy contrib. If unsure, you can ask in
 `scrapy-users`_.
@@ -189,7 +189,7 @@ And their unit-tests are in::
 
 .. _issue tracker: https://github.com/scrapy/scrapy/issues
 .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
-.. _Twisted unit-testing framework: http://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
+.. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
 .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
 .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
 .. _open issues: https://github.com/scrapy/scrapy/issues
diff --git a/docs/faq.rst b/docs/faq.rst
index b3412211a..b132a9c2e 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -77,8 +77,8 @@ Scrapy crashes with: ImportError: No module named win32api
 
 You need to install `pywin32`_ because of `this Twisted bug`_.
 
-.. _pywin32: http://sourceforge.net/projects/pywin32/
-.. _this Twisted bug: http://twistedmatrix.com/trac/ticket/3707
+.. _pywin32: https://sourceforge.net/projects/pywin32/
+.. _this Twisted bug: https://twistedmatrix.com/trac/ticket/3707
 
 How can I simulate a user login in my spider?
 ---------------------------------------------
@@ -123,7 +123,7 @@ Why does Scrapy download pages in English instead of my native language?
 Try changing the default `Accept-Language`_ request header by overriding the
 :setting:`DEFAULT_REQUEST_HEADERS` setting.
 
-.. _Accept-Language: http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
+.. _Accept-Language: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
 
 Where can I find some example Scrapy projects?
 ----------------------------------------------
@@ -282,7 +282,7 @@ I'm scraping a XML document and my XPath selector doesn't return any items
 
 You may need to remove namespaces. See :ref:`removing-namespaces`.
 
-.. _user agents: http://en.wikipedia.org/wiki/User_agent
-.. _LIFO: http://en.wikipedia.org/wiki/LIFO
-.. _DFO order: http://en.wikipedia.org/wiki/Depth-first_search
-.. _BFO order: http://en.wikipedia.org/wiki/Breadth-first_search
+.. _user agents: https://en.wikipedia.org/wiki/User_agent
+.. _LIFO: https://en.wikipedia.org/wiki/LIFO
+.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
+.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 122de47f6..25520b4b9 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -74,7 +74,7 @@ Windows
   Be sure you download the architecture (win32 or amd64) that matches your system
 
 * *(Only required for Python<2.7.9)* Install `pip`_ from
-  https://pip.pypa.io/en/latest/installing.html
+  https://pip.pypa.io/en/latest/installing/
 
   Now open a Command prompt to check ``pip`` is installed correctly:: 
 
@@ -171,9 +171,9 @@ After any of these workarounds you should be able to install Scrapy::
   pip install Scrapy
 
 .. _Python: https://www.python.org/
-.. _pip: https://pip.pypa.io/en/latest/installing.html
-.. _easy_install: http://pypi.python.org/pypi/setuptools
-.. _Control Panel: http://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx
+.. _pip: https://pip.pypa.io/en/latest/installing/
+.. _easy_install: https://pypi.python.org/pypi/setuptools
+.. _Control Panel: https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx
 .. _lxml: http://lxml.de/
 .. _OpenSSL: https://pypi.python.org/pypi/pyOpenSSL
 .. _setuptools: https://pypi.python.org/pypi/setuptools
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 595e85e28..0673aed93 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -170,7 +170,7 @@ your code in Scrapy projects and `join the community`_. Thanks for your
 interest!
 
 .. _join the community: http://scrapy.org/community/
-.. _web scraping: http://en.wikipedia.org/wiki/Web_scraping
-.. _Amazon Associates Web Services: http://aws.amazon.com/associates/
-.. _Amazon S3: http://aws.amazon.com/s3/
+.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
+.. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html
+.. _Amazon S3: https://aws.amazon.com/s3/
 .. _Sitemaps: http://www.sitemaps.org
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index dce165cf4..f2af6e17c 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -7,7 +7,7 @@ Scrapy Tutorial
 In this tutorial, we'll assume that Scrapy is already installed on your system.
 If that's not the case, see :ref:`intro-install`.
 
-We are going to use `Open directory project (dmoz) <http://www.dmoz.org/>`_ as
+We are going to use `Open directory project (dmoz) <https://www.dmoz.org/>`_ as
 our example domain to scrape.
 
 This tutorial will walk you through these tasks:
@@ -191,8 +191,8 @@ based on `XPath`_ or `CSS`_ expressions called :ref:`Scrapy Selectors
 <topics-selectors>`.  For more information about selectors and other extraction
 mechanisms see the :ref:`Selectors documentation <topics-selectors>`.
 
-.. _XPath: http://www.w3.org/TR/xpath
-.. _CSS: http://www.w3.org/TR/selectors
+.. _XPath: https://www.w3.org/TR/xpath
+.. _CSS: https://www.w3.org/TR/selectors
 
 Here are some examples of XPath expressions and their meanings:
 
@@ -544,5 +544,5 @@ Then, we recommend you continue by playing with an example project (see
 :ref:`intro-examples`), and then continue with the section
 :ref:`section-basics`.
 
-.. _JSON: http://en.wikipedia.org/wiki/JSON
+.. _JSON: https://en.wikipedia.org/wiki/JSON
 .. _dirbot: https://github.com/scrapy/dirbot
diff --git a/docs/news.rst b/docs/news.rst
index e6625c99f..f515e5bb1 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -403,10 +403,11 @@ Outsourced packages
 |                                     | :ref:`topics-deploy`)               |
 +-------------------------------------+-------------------------------------+
 | scrapy.contrib.djangoitem           | `scrapy-djangoitem <https://github. |
-|                                     | com/scrapy/scrapy-djangoitem>`_     |
+|                                     | com/scrapy-plugins/scrapy-djangoite |
+|                                     | m>`_                                |
 +-------------------------------------+-------------------------------------+
 | scrapy.webservice                   | `scrapy-jsonrpc <https://github.com |
-|                                     | /scrapy/scrapy-jsonrpc>`_           |
+|                                     | /scrapy-plugins/scrapy-jsonrpc>`_   |
 +-------------------------------------+-------------------------------------+
 
 `scrapy.contrib_exp` and `scrapy.contrib` dissolutions
@@ -1186,7 +1187,7 @@ Scrapy changes:
 - nested items now fully supported in JSON and JSONLines exporters
 - added :reqmeta:`cookiejar` Request meta key to support multiple cookie sessions per spider
 - decoupled encoding detection code to `w3lib.encoding`_, and ported Scrapy code to use that module
-- dropped support for Python 2.5. See http://blog.scrapinghub.com/2012/02/27/scrapy-0-15-dropping-support-for-python-2-5/
+- dropped support for Python 2.5. See https://blog.scrapinghub.com/2012/02/27/scrapy-0-15-dropping-support-for-python-2-5/
 - dropped support for Twisted 2.5
 - added :setting:`REFERER_ENABLED` setting, to control referer middleware
 - changed default user agent to: ``Scrapy/VERSION (+http://scrapy.org)``
@@ -1535,7 +1536,7 @@ First release of Scrapy.
 
 
 .. _AJAX crawleable urls: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started?csw=1
-.. _chunked transfer encoding: http://en.wikipedia.org/wiki/Chunked_transfer_encoding
+.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
 .. _w3lib: https://github.com/scrapy/w3lib
 .. _scrapely: https://github.com/scrapy/scrapely
 .. _marshal: https://docs.python.org/2/library/marshal.html
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 42c0133c1..d470a0d41 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -271,4 +271,4 @@ class (which they all inherit from).
         Close the given spider. After this is called, no more specific stats
         can be accessed or collected.
 
-.. _reactor: http://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
+.. _reactor: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index dad171a07..aef033dc3 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -125,8 +125,8 @@ links:
 * `Twisted - hello, asynchronous programming`_
 * `Twisted Introduction - Krondo`_
 
-.. _Twisted: http://twistedmatrix.com/trac/
-.. _Introduction to Deferreds in Twisted: http://twistedmatrix.com/documents/current/core/howto/defer-intro.html
+.. _Twisted: https://twistedmatrix.com/trac/
+.. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
 .. _Twisted - hello, asynchronous programming: http://jessenoller.com/2009/02/11/twisted-hello-asynchronous-programming/
-.. _Twisted Introduction - Krondo: http://krondo.com/blog/?page_id=1327/
+.. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/
 
diff --git a/docs/topics/djangoitem.rst b/docs/topics/djangoitem.rst
index d7b86492d..6a3621192 100644
--- a/docs/topics/djangoitem.rst
+++ b/docs/topics/djangoitem.rst
@@ -10,4 +10,4 @@ DjangoItem has been moved into a separate project.
 
 It is hosted at:
 
-    https://github.com/scrapy/scrapy-djangoitem
+    https://github.com/scrapy-plugins/scrapy-djangoitem
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 4215cf69c..59a73493d 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -300,7 +300,7 @@ HttpAuthMiddleware
 
             # .. rest of the spider code omitted ...
 
-.. _Basic access authentication: http://en.wikipedia.org/wiki/Basic_access_authentication
+.. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication
 
 
 HttpCacheMiddleware
@@ -390,9 +390,9 @@ what is implemented:
 
 what is missing:
 
-* `Pragma: no-cache` support http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
-* `Vary` header support http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6
-* Invalidation after updates or deletes http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10
+* `Pragma: no-cache` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
+* `Vary` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6
+* Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10
 * ... probably others ..
 
 In order to use this policy, set:
@@ -464,7 +464,7 @@ In order to use this storage backend:
 * set :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.LeveldbCacheStorage``
 * install `LevelDB python bindings`_ like ``pip install leveldb``
 
-.. _LevelDB: http://code.google.com/p/leveldb/
+.. _LevelDB: https://github.com/google/leveldb
 .. _leveldb python bindings: https://pypi.python.org/pypi/leveldb
 
 
@@ -964,6 +964,6 @@ Default: ``"latin-1"``
 The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
 
 
-.. _DBM: http://en.wikipedia.org/wiki/Dbm
+.. _DBM: https://en.wikipedia.org/wiki/Dbm
 .. _anydbm: https://docs.python.org/2/library/anydbm.html
-.. _chunked transfer encoding: http://en.wikipedia.org/wiki/Chunked_transfer_encoding
+.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index 96487d865..2b5812c8d 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -15,7 +15,7 @@ simple API for sending attachments and it's very easy to configure, with a few
 :ref:`settings <topics-email-settings>`.
 
 .. _smtplib: https://docs.python.org/2/library/smtplib.html
-.. _Twisted non-blocking IO: http://twistedmatrix.com/documents/current/core/howto/defer-intro.html
+.. _Twisted non-blocking IO: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
 
 Quick example
 =============
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 14528ba9a..7f2952f4c 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -21,7 +21,7 @@ avoid collision with existing (and future) extensions. For example, a
 hypothetic extension to handle `Google Sitemaps`_ would use settings like
 `GOOGLESITEMAP_ENABLED`, `GOOGLESITEMAP_DEPTH`, and so on.
 
-.. _Google Sitemaps: http://en.wikipedia.org/wiki/Sitemaps
+.. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps
 
 Loading & activating extensions
 ===============================
@@ -355,8 +355,8 @@ There are at least two ways to send Scrapy the `SIGQUIT`_ signal:
 
     kill -QUIT <pid>
 
-.. _SIGUSR2: http://en.wikipedia.org/wiki/SIGUSR1_and_SIGUSR2
-.. _SIGQUIT: http://en.wikipedia.org/wiki/SIGQUIT
+.. _SIGUSR2: https://en.wikipedia.org/wiki/SIGUSR1_and_SIGUSR2
+.. _SIGQUIT: https://en.wikipedia.org/wiki/SIGQUIT
 
 Debugger extension
 ~~~~~~~~~~~~~~~~~~
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index e5037129c..dc5912fb9 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -330,7 +330,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
         'csv': None,
     }
 
-.. _URI: http://en.wikipedia.org/wiki/Uniform_Resource_Identifier
-.. _Amazon S3: http://aws.amazon.com/s3/
+.. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
+.. _Amazon S3: https://aws.amazon.com/s3/
 .. _boto: https://github.com/boto/boto
 .. _botocore: https://github.com/boto/botocore
diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst
index b93ee4ff9..8f0a5767b 100644
--- a/docs/topics/firebug.rst
+++ b/docs/topics/firebug.rst
@@ -164,4 +164,4 @@ elements.
 or tags which Therefer   in page HTML
 sources may on Firebug inspects the live DOM
 
-.. _has been shut down by Google: http://searchenginewatch.com/sew/news/2096661/google-directory-shut
+.. _has been shut down by Google: https://searchenginewatch.com/sew/news/2096661/google-directory-shut
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 28969be61..b9b4c2058 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -160,8 +160,8 @@ method and how to clean up the resources properly.
             self.db[self.collection_name].insert(dict(item))
             return item
 
-.. _MongoDB: http://www.mongodb.org/
-.. _pymongo: http://api.mongodb.org/python/current/
+.. _MongoDB: https://www.mongodb.org/
+.. _pymongo: https://api.mongodb.org/python/current/
 
 Duplicates filter
 -----------------
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 5ed6ce97d..bf5d08d54 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -143,7 +143,7 @@ Supported Storage
 File system is currently the only officially supported storage, but there is
 also (undocumented) support for storing files in `Amazon S3`_.
 
-.. _Amazon S3: http://aws.amazon.com/s3/
+.. _Amazon S3: https://aws.amazon.com/s3/
 
 File system storage
 -------------------
@@ -223,7 +223,7 @@ Where:
 
 * ``<image_id>`` is the `SHA1 hash`_ of the image url
 
-.. _SHA1 hash: http://en.wikipedia.org/wiki/SHA_hash_functions
+.. _SHA1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions
 
 Example of image files stored using ``small`` and ``big`` thumbnail names::
 
@@ -390,5 +390,5 @@ above::
             item['image_paths'] = image_paths
             return item
 
-.. _Twisted Failure: http://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
-.. _MD5 hash: http://en.wikipedia.org/wiki/MD5
+.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
+.. _MD5 hash: https://en.wikipedia.org/wiki/MD5
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 60fe2267c..5d40c745d 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -251,5 +251,5 @@ If you are still unable to prevent your bot getting banned, consider contacting
 .. _ProxyMesh: http://proxymesh.com/
 .. _Google cache: http://www.googleguide.com/cached_pages.html
 .. _testspiders: https://github.com/scrapinghub/testspiders
-.. _Twisted Reactor Overview: http://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
+.. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
 .. _Crawlera: http://scrapinghub.com/crawlera
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 82e674cee..10affc21f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -621,4 +621,4 @@ XmlResponse objects
     adds encoding auto-discovering support by looking into the XML declaration
     line.  See :attr:`TextResponse.encoding`.
 
-.. _Twisted Failure: http://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
+.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 8dc82dfe5..3c5c170a3 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -40,8 +40,8 @@ For a complete reference of the selectors API see
 .. _lxml: http://lxml.de/
 .. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html
 .. _cssselect: https://pypi.python.org/pypi/cssselect/
-.. _XPath: http://www.w3.org/TR/xpath
-.. _CSS: http://www.w3.org/TR/selectors
+.. _XPath: https://www.w3.org/TR/xpath
+.. _CSS: https://www.w3.org/TR/selectors
 
 
 Using selectors
@@ -281,7 +281,7 @@ Another common case would be to extract all direct ``<p>`` children::
 For more details about relative XPaths see the `Location Paths`_ section in the
 XPath specification.
 
-.. _Location Paths: http://www.w3.org/TR/xpath#location-paths
+.. _Location Paths: https://www.w3.org/TR/xpath#location-paths
 
 Using EXSLT extensions
 ----------------------
@@ -439,7 +439,7 @@ you may want to take a look first at this `XPath tutorial`_.
 
 
 .. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
-.. _`this post from ScrapingHub's blog`: http://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
+.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
 
 
 Using text nodes in a condition
@@ -481,7 +481,7 @@ But using the ``.`` to mean the node, works::
     >>> sel.xpath("//a[contains(., 'Next Page')]").extract()
     [u'<a href="#">Click here to go to the <strong>Next Page</strong></a>']
 
-.. _`XPath string function`: http://www.w3.org/TR/xpath/#section-String-Functions
+.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
 
 Beware of the difference between //node[1] and (//node)[1]
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 3bd0f9df5..e42dc6370 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1202,6 +1202,6 @@ case to see how to enable and use them.
 .. settingslist::
 
 
-.. _Amazon web services: http://aws.amazon.com/
-.. _breadth-first order: http://en.wikipedia.org/wiki/Breadth-first_search
-.. _depth-first order: http://en.wikipedia.org/wiki/Depth-first_search
+.. _Amazon web services: https://aws.amazon.com/
+.. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
+.. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index a6ca036d2..322c3ddfa 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -138,7 +138,7 @@ Example of shell session
 ========================
 
 Here's an example of a typical shell session where we start by scraping the
-http://scrapy.org page, and then proceed to scrape the http://reddit.com
+http://scrapy.org page, and then proceed to scrape the https://reddit.com
 page. Finally, we modify the (Reddit) request method to POST and re-fetch it
 getting an error. We end the session by typing Ctrl-D (in Unix systems) or
 Ctrl-Z in Windows.
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 5dd3b9ef5..410717f7d 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -22,7 +22,7 @@ Deferred signal handlers
 Some signals support returning `Twisted deferreds`_ from their handlers, see
 the :ref:`topics-signals-ref` below to know which ones.
 
-.. _Twisted deferreds: http://twistedmatrix.com/documents/current/core/howto/defer.html
+.. _Twisted deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html
 
 .. _topics-signals-ref:
 
@@ -258,4 +258,4 @@ response_downloaded
     :param spider: the spider for which the response is intended
     :type spider: :class:`~scrapy.spiders.Spider` object
 
-.. _Failure: http://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
+.. _Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index ced481c71..a38c1ab65 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -211,7 +211,7 @@ HttpErrorMiddleware
 According to the `HTTP standard`_, successful responses are those whose
 status codes are in the 200-300 range.
 
-.. _HTTP standard: http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+.. _HTTP standard: https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
 
 If you still want to process response codes outside that range, you can
 specify which response codes the spider is able to handle using the
@@ -238,7 +238,7 @@ responses, unless you really know what you're doing.
 
 For more information see: `HTTP Status Code Definitions`_.
 
-.. _HTTP Status Code Definitions: http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+.. _HTTP Status Code Definitions: https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
 
 HttpErrorMiddleware settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index b700ea0ef..0e473709a 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -735,5 +735,5 @@ Combine SitemapSpider with other sources of urls::
 .. _Sitemaps: http://www.sitemaps.org
 .. _Sitemap index files: http://www.sitemaps.org/protocol.html#index
 .. _robots.txt: http://www.robotstxt.org/
-.. _TLD: http://en.wikipedia.org/wiki/Top-level_domain
+.. _TLD: https://en.wikipedia.org/wiki/Top-level_domain
 .. _Scrapyd documentation: http://scrapyd.readthedocs.org/en/latest/
diff --git a/docs/topics/webservice.rst b/docs/topics/webservice.rst
index a064acb16..2c4052c04 100644
--- a/docs/topics/webservice.rst
+++ b/docs/topics/webservice.rst
@@ -8,4 +8,4 @@ webservice has been moved into a separate project.
 
 It is hosted at:
 
-    https://github.com/scrapy/scrapy-jsonrpc
+    https://github.com/scrapy-plugins/scrapy-jsonrpc
diff --git a/docs/versioning.rst b/docs/versioning.rst
index 6ae9ce981..8e7908762 100644
--- a/docs/versioning.rst
+++ b/docs/versioning.rst
@@ -36,5 +36,5 @@ new methods or functionality but the existing methods should keep working the
 same way.
 
 
-.. _odd-numbered versions for development releases: http://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases
+.. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases
 

From 2cfe9e424ddf30044c49aba50d73c7db635bc9ed Mon Sep 17 00:00:00 2001
From: Aron Bordin <aron.bordin@gmail.com>
Date: Sat, 5 Mar 2016 19:54:06 -0300
Subject: [PATCH 0905/2974] small doc style fixes

---
 docs/faq.rst                     | 4 +---
 docs/topics/email.rst            | 4 ++--
 docs/topics/media-pipeline.rst   | 6 +++---
 docs/topics/request-response.rst | 1 +
 docs/topics/settings.rst         | 4 ++--
 docs/topics/signals.rst          | 2 +-
 6 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index b132a9c2e..411e043b7 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -33,7 +33,7 @@ Scrapy is supported under Python 2.7 only.
 Python 2.6 support was dropped starting at Scrapy 0.20.
 
 Does Scrapy work with Python 3?
----------------------------------
+-------------------------------
 
 No, but there are plans to support Python 3.3+.
 At the moment, Scrapy works with Python 2.7.
@@ -57,8 +57,6 @@ focus on the real problems we need to solve.
 We'd be proud if Scrapy serves as an inspiration for other projects. Feel free
 to steal from us!
 
-.. _Django: https://www.djangoproject.com/
-
 Does Scrapy work with HTTP proxies?
 -----------------------------------
 
diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index 2b5812c8d..62ebc4c08 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -164,7 +164,7 @@ Password to use for SMTP authentication, along with :setting:`MAIL_USER`.
 .. setting:: MAIL_TLS
 
 MAIL_TLS
----------
+--------
 
 Default: ``False``
 
@@ -173,7 +173,7 @@ Enforce using STARTTLS. STARTTLS is a way to take an existing insecure connectio
 .. setting:: MAIL_SSL
 
 MAIL_SSL
----------
+--------
 
 Default: ``False``
 
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index bf5d08d54..a34f4c053 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -29,7 +29,7 @@ the same media to that queue. This avoids downloading the same media more than
 once when it's shared by several items.
 
 Using the Files Pipeline
-=========================
+========================
 
 The typical workflow, when using the :class:`FilesPipeline` goes like
 this:
@@ -104,7 +104,7 @@ behaviour, see :ref:`topics-media-pipeline-override`.
 .. _topics-media-pipeline-enabling:
 
 Enabling your Media Pipeline
-=============================
+============================
 
 .. setting:: IMAGES_STORE
 .. setting:: FILES_STORE
@@ -175,7 +175,7 @@ Additional features
 ===================
 
 File expiration
-----------------
+---------------
 
 .. setting:: IMAGES_EXPIRES
 .. setting:: FILES_EXPIRES
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 10affc21f..7b48c4d1c 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -67,6 +67,7 @@ Request objects
 
             request_with_cookies = Request(url="http://www.example.com",
                                            cookies={'currency': 'USD', 'country': 'UY'})
+
         2. Using a list of dicts::
 
             request_with_cookies = Request(url="http://www.example.com",
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index e42dc6370..0716f1bd4 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -342,7 +342,7 @@ Whether to enable DNS in-memory cache.
 .. setting:: DNSCACHE_SIZE
 
 DNSCACHE_SIZE
-----------------
+-------------
 
 Default: ``10000``
 
@@ -351,7 +351,7 @@ DNS in-memory cache size.
 .. setting:: DNS_TIMEOUT
 
 DNS_TIMEOUT
-----------------
+-----------
 
 Default: ``60``
 
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 410717f7d..6434c98c5 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -203,7 +203,7 @@ request_scheduled
     :type spider: :class:`~scrapy.spiders.Spider` object
 
 request_dropped
------------------
+---------------
 
 .. signal:: request_dropped
 .. function:: request_dropped(request, spider)

From 65c7c05060fd2d1fc161d4904243d5e0b31e202b Mon Sep 17 00:00:00 2001
From: pawelmhm <pawelmhm@gmail.com>
Date: Sat, 12 Mar 2016 14:07:20 +0100
Subject: [PATCH 0906/2974] response_status_message should not fail on
 non-standard HTTP codes

utility is used in retry middleware and it was failing to handle non-standard HTTP codes.
Instead of raising exceptions when passing through to_native_str it should return
"Unknown status" message.
---
 scrapy/utils/response.py     | 8 +-------
 tests/test_utils_response.py | 7 ++++++-
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 73db2641e..d9e951463 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -47,14 +47,8 @@ def get_meta_refresh(response):
 
 def response_status_message(status):
     """Return status code plus status text descriptive message
-
-    >>> response_status_message(200)
-    '200 OK'
-
-    >>> response_status_message(404)
-    '404 Not Found'
     """
-    return '%s %s' % (status, to_native_str(http.RESPONSES.get(int(status))))
+    return '%s %s' % (status, to_native_str(http.RESPONSES.get(int(status), "Unknown Status")))
 
 
 def response_httprepr(response):
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 1d1638e06..bea4dade3 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -5,7 +5,7 @@ from six.moves.urllib.parse import urlparse
 from scrapy.http import Response, TextResponse, HtmlResponse
 from scrapy.utils.python import to_bytes
 from scrapy.utils.response import (response_httprepr, open_in_browser,
-                                   get_meta_refresh, get_base_url)
+                                   get_meta_refresh, get_base_url, response_status_message)
 
 __doctests__ = ['scrapy.utils.response']
 
@@ -78,3 +78,8 @@ class ResponseUtilsTest(unittest.TestCase):
         resp2 = HtmlResponse("http://www.example.com", body=b"""
         <html><body>blahablsdfsal&amp;</body></html>""")
         self.assertEqual(get_base_url(resp2), "http://www.example.com")
+
+    def test_response_status_message(self):
+        self.assertEqual(response_status_message(200), '200 OK')
+        self.assertEqual(response_status_message(404), '404 Not Found')
+        self.assertEqual(response_status_message(573), "573 Unknown Status")

From 3ca977a8cb3bf1ac80c1850203b8ef819e087e45 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 25 Mar 2016 07:11:33 +0500
Subject: [PATCH 0907/2974] DOC improved Architecture overview

* spiders don't have to work on specific domains;
* explain what to use Downloader middleware for
  and what to use Spider middleware for;
* Engine no longer locates spiders based on domains;
* "Spider middleware output direction" step was missing.

See also: GH-1569.
---
 docs/topics/architecture.rst | 54 +++++++++++++++++++++++-------------
 1 file changed, 34 insertions(+), 20 deletions(-)

diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index aef033dc3..dc03d47e8 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -48,8 +48,7 @@ Spiders
 
 Spiders are custom classes written by Scrapy users to parse responses and
 extract items (aka scraped items) from them or additional URLs (requests) to
-follow. Each spider is able to handle a specific domain (or group of domains).
-For more information see :ref:`topics-spiders`.
+follow. For more information see :ref:`topics-spiders`.
 
 Item Pipeline
 -------------
@@ -64,18 +63,35 @@ Downloader middlewares
 
 Downloader middlewares are specific hooks that sit between the Engine and the
 Downloader and process requests when they pass from the Engine to the
-Downloader, and responses that pass from Downloader to the Engine. They provide
-a convenient mechanism for extending Scrapy functionality by plugging custom
-code. For more information see :ref:`topics-downloader-middleware`.
+Downloader, and responses that pass from Downloader to the Engine.
+
+Use a Downloader middleware if you need to do one of the following:
+
+* process a request just before it is sent to the Downloader
+  (i.e. right before Scrapy sends the request to the website);
+* change received response before passing it to a spider;
+* send a new Request instead of passing received response to a spider;
+* pass response to a spider without fetching a web page;
+* silently drop some requests.
+
+For more information see :ref:`topics-downloader-middleware`.
 
 Spider middlewares
 ------------------
 
 Spider middlewares are specific hooks that sit between the Engine and the
 Spiders and are able to process spider input (responses) and output (items and
-requests). They provide a convenient mechanism for extending Scrapy
-functionality by plugging custom code. For more information see
-:ref:`topics-spider-middleware`.
+requests).
+
+Use a Spider middleware if you need to
+
+* post-process output of spider callbacks - change/add/remove requests or items;
+* post-process start_requests;
+* handle spider exceptions;
+* call errback instead of callback for some of the requests based on response
+  content.
+
+For more information see :ref:`topics-spider-middleware`.
 
 Data flow
 =========
@@ -83,32 +99,30 @@ Data flow
 The data flow in Scrapy is controlled by the execution engine, and goes like
 this:
 
-1. The Engine opens a domain, locates the Spider that handles that domain, and
-   asks the spider for the first URLs to crawl.
-
-2. The Engine gets the first URLs to crawl from the Spider and schedules them
+1. The Engine gets the first URLs to crawl from the Spider and schedules them
    in the Scheduler, as Requests.
 
-3. The Engine asks the Scheduler for the next URLs to crawl.
+2. The Engine asks the Scheduler for the next URLs to crawl.
 
-4. The Scheduler returns the next URLs to crawl to the Engine and the Engine
+3. The Scheduler returns the next URLs to crawl to the Engine and the Engine
    sends them to the Downloader, passing through the Downloader Middleware
    (request direction).
 
-5. Once the page finishes downloading the Downloader generates a Response (with
+4. Once the page finishes downloading the Downloader generates a Response (with
    that page) and sends it to the Engine, passing through the Downloader
    Middleware (response direction).
 
-6. The Engine receives the Response from the Downloader and sends it to the
+5. The Engine receives the Response from the Downloader and sends it to the
    Spider for processing, passing through the Spider Middleware (input direction).
 
-7. The Spider processes the Response and returns scraped items and new Requests
+6. The Spider processes the Response and returns scraped items and new Requests
    (to follow) to the Engine.
 
-8. The Engine sends scraped items (returned by the Spider) to the Item Pipeline
-   and Requests (returned by spider) to the Scheduler
+7. The Engine passes scraped items and new Requests returned by a spider
+   through Spider Middleware (output direction), and then sends processed
+   items to Item Pipelines and processed Requests to the Scheduler.
 
-9. The process repeats (from step 2) until there are no more requests from the
+8. The process repeats (from step 2) until there are no more requests from the
    Scheduler, and the Engine closes the domain.
 
 Event-driven networking

From 4f335b5a01af275f413021c8670132d32d2daf6f Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 25 Mar 2016 17:03:41 +0500
Subject: [PATCH 0908/2974] DOC clarify Architecture docs

---
 docs/topics/architecture.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index dc03d47e8..b4588652c 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -122,8 +122,8 @@ this:
    through Spider Middleware (output direction), and then sends processed
    items to Item Pipelines and processed Requests to the Scheduler.
 
-8. The process repeats (from step 2) until there are no more requests from the
-   Scheduler, and the Engine closes the domain.
+8. The process repeats (from step 1) until there are no more requests from the
+   Scheduler.
 
 Event-driven networking
 =======================

From 5045a4f1681b8c4a152210a9245e59af30744422 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 25 Mar 2016 18:35:55 +0300
Subject: [PATCH 0909/2974] Fix handling of meta=None in
 S3FilesStore.persist_file

---
 scrapy/pipelines/files.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 1fd2571e2..c5dbd307b 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -147,7 +147,7 @@ class S3FilesStore(object):
                 Bucket=self.bucket,
                 Key=key_name,
                 Body=buf,
-                Metadata={k: str(v) for k, v in six.iteritems(meta)},
+                Metadata={k: str(v) for k, v in six.iteritems(meta or {})},
                 ACL=self.POLICY,
                 **extra)
         else:

From fc8cd45a48fb21d33e7f6edaec2aa8275c6ea942 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Sun, 27 Mar 2016 21:29:21 +0200
Subject: [PATCH 0910/2974] Fix a race condition in the FilesPipeline

Checksum calculation could happen simultaniously with
persisting the file in the store (which is done in a thread):
they operated on the same buf object.
Concretely this lead to a bug with S3FilesStore
when using botocore: the signature did not match because
the position in the buf was already at the end.
The fix is to move checksum calculation before passing buf
to the store.
---
 scrapy/pipelines/files.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index c5dbd307b..b1b8404bb 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -364,8 +364,9 @@ class FilesPipeline(MediaPipeline):
     def file_downloaded(self, response, request, info):
         path = self.file_path(request, response=response, info=info)
         buf = BytesIO(response.body)
-        self.store.persist_file(path, buf, info)
         checksum = md5sum(buf)
+        buf.seek(0)
+        self.store.persist_file(path, buf, info)
         return checksum
 
     def item_completed(self, results, item, info):

From 7082454f2af7b7a3c881c5fdc8ac9c03b595e18d Mon Sep 17 00:00:00 2001
From: Lele <lele@sshadm.in>
Date: Fri, 18 Mar 2016 03:30:31 +0100
Subject: [PATCH 0911/2974] Changed sel. to response. for clarity

Changed sel. to response. to comply with the rest of the examples in the same section, to avoid confusion.
---
 docs/topics/selectors.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 3c5c170a3..3a0695568 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -151,7 +151,7 @@ It returns ``None`` if no element was found:
 
 A default return value can be provided as an argument, to be used instead of ``None``:
 
-    >>> sel.xpath('//div[@id="not-exists"]/text()').extract_first(default='not-found')
+    >>> response.xpath('//div[@id="not-exists"]/text()').extract_first(default='not-found')
     'not-found'
 
 Notice that CSS selectors can select text or attribute nodes using CSS3

From a583e4d531306b7628b42f8a32c7db892ad86cf1 Mon Sep 17 00:00:00 2001
From: nanolab <alexey.ryabukhin@gmail.com>
Date: Wed, 23 Mar 2016 11:37:01 +0200
Subject: [PATCH 0912/2974] Update httpcache.py

It checks cache directory modification time, but have to check file modification time.
---
 scrapy/extensions/httpcache.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index a871cc895..247cac64e 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -328,7 +328,7 @@ class FilesystemCacheStorage(object):
         metapath = os.path.join(rpath, 'pickled_meta')
         if not os.path.exists(metapath):
             return  # not found
-        mtime = os.stat(rpath).st_mtime
+        mtime = os.stat(metapath).st_mtime
         if 0 < self.expiration_secs < time() - mtime:
             return  # expired
         with self._open(metapath, 'rb') as f:

From 1075587dbd15e5ccb9a83c4ca14086c1e135fe12 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 30 Mar 2016 14:31:10 +0200
Subject: [PATCH 0913/2974] Add support for Sphinx 1.4

See http://www.sphinx-doc.org/en/stable/changes.html#release-1-4-released-mar-28-2016

sphinx_rtd_theme has become optional, needs to be added to reqs

https://github.com/sphinx-doc/sphinx/pull/2320 changes node entries tuples
to 5 values instead of 4

`sh` syntax highlighting added very locally in selectors.rst
because of this warning/error with Sphinx 1.4:

```
Warning, treated as error:
/home/paul/src/scrapy/docs/topics/selectors.rst:743:
WARNING: Could not lex literal_block as "python". Highlighting skipped.
```
---
 docs/_ext/scrapydocs.py   | 4 ++--
 docs/topics/selectors.rst | 4 ++++
 tox.ini                   | 1 +
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index f0827f2b1..83b0d2cc6 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -18,7 +18,7 @@ def is_setting_index(node):
     if node.tagname == 'index':
         # index entries for setting directives look like:
         # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]
-        entry_type, info, refid, _ = node['entries'][0]
+        entry_type, info, refid = node['entries'][0][:3]
         return entry_type == 'pair' and info.endswith('; setting')
     return False
 
@@ -30,7 +30,7 @@ def get_setting_target(node):
 
 def get_setting_name_and_refid(node):
     """Extract setting name from directive index node"""
-    entry_type, info, refid, _ = node['entries'][0]
+    entry_type, info, refid = node['entries'][0][:3]
     return info.replace('; setting', ''), refid
 
 
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 3a0695568..d1845b20c 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -738,10 +738,14 @@ simple/convenient XPaths. You can use the
 
 Let's show an example that illustrates this with Github blog atom feed.
 
+.. highlight:: sh
+
 First, we open the shell with the url we want to scrape::
 
     $ scrapy shell https://github.com/blog.atom
 
+.. highlight:: python
+
 Once in the shell we can try selecting all ``<link>`` objects and see that it
 doesn't work (because the Atom XML namespace is obfuscating those nodes)::
 
diff --git a/tox.ini b/tox.ini
index 2a8067618..f6de64b27 100644
--- a/tox.ini
+++ b/tox.ini
@@ -60,6 +60,7 @@ deps = {[testenv:py33]deps}
 changedir = docs
 deps =
     Sphinx
+    sphinx_rtd_theme
 
 [testenv:docs]
 changedir = {[docs]changedir}

From 3787fec460f134a91228d01a8fc0277354e4ee0c Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sat, 26 Mar 2016 22:22:01 +0000
Subject: [PATCH 0914/2974] Remove duplicate code now handled by newer w3lib

see https://github.com/scrapy/w3lib/commit/f3029a6a1071ba390472d4dd244fb3280d585c0a
---
 requirements.txt                          | 2 +-
 scrapy/downloadermiddlewares/ajaxcrawl.py | 8 +++-----
 scrapy/utils/response.py                  | 7 +------
 setup.py                                  | 2 +-
 4 files changed, 6 insertions(+), 13 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 368e9340b..db042cfcf 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@ Twisted>=10.0.0
 lxml
 pyOpenSSL
 cssselect>=0.9
-w3lib>=1.8.0
+w3lib>=1.13.0
 queuelib
 six>=1.5.2
 PyDispatcher>=2.0.5
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index da373eca2..72715dba7 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -8,7 +8,6 @@ from w3lib import html
 
 from scrapy.exceptions import NotConfigured
 from scrapy.http import HtmlResponse
-from scrapy.utils.response import _noscript_re, _script_re
 
 
 logger = logging.getLogger(__name__)
@@ -89,8 +88,7 @@ def _has_ajaxcrawlable_meta(text):
     if 'content' not in text:
         return False
 
-    text = _script_re.sub(u'', text)
-    text = _noscript_re.sub(u'', text)
-    text = html.remove_comments(html.replace_entities(text))
+    text = html.remove_tags_with_content(text, ('script', 'noscript'))
+    text = html.replace_entities(text)
+    text = html.remove_comments(text)
     return _ajax_crawlable_re.search(text) is not None
-
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 73db2641e..27ea5593e 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -3,7 +3,6 @@ This module provides some useful functions for working with
 scrapy.http.Response objects
 """
 import os
-import re
 import weakref
 import webbrowser
 import tempfile
@@ -31,17 +30,13 @@ def get_base_url(response):
     return _baseurl_cache[response]
 
 
-_noscript_re = re.compile(u'<noscript>.*?</noscript>', re.IGNORECASE | re.DOTALL)
-_script_re = re.compile(u'<script.*?>.*?</script>', re.IGNORECASE | re.DOTALL)
 _metaref_cache = weakref.WeakKeyDictionary()
 def get_meta_refresh(response):
     """Parse the http-equiv refrsh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.text[0:4096]
-        text = _noscript_re.sub(u'', text)
-        text = _script_re.sub(u'', text)
         _metaref_cache[response] = html.get_meta_refresh(text, response.url,
-            response.encoding)
+            response.encoding, ignore_tags=('script', 'noscript'))
     return _metaref_cache[response]
 
 
diff --git a/setup.py b/setup.py
index d91bb6328..8387f81f4 100644
--- a/setup.py
+++ b/setup.py
@@ -42,7 +42,7 @@ setup(
     ],
     install_requires=[
         'Twisted>=10.0.0',
-        'w3lib>=1.8.0',
+        'w3lib>=1.13.0',
         'queuelib',
         'lxml',
         'pyOpenSSL',

From e8ca4675720b2393fdbd2305c03d94c58752c55a Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sun, 6 Mar 2016 16:45:06 +0000
Subject: [PATCH 0915/2974] Rename isbinarytext function to binary_is_text for
 clarity

Closes #1389
---
 scrapy/responsetypes.py    |  4 ++--
 scrapy/utils/python.py     | 20 ++++++++++++++------
 tests/test_utils_python.py | 14 +++++++-------
 3 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index c667b141d..075f757af 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -10,7 +10,7 @@ import six
 
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
-from scrapy.utils.python import isbinarytext, to_bytes, to_native_str
+from scrapy.utils.python import binary_is_text, to_bytes, to_native_str
 
 
 class ResponseTypes(object):
@@ -94,7 +94,7 @@ class ResponseTypes(object):
         cannot be guess using more straightforward methods."""
         chunk = body[:5000]
         chunk = to_bytes(chunk)
-        if isbinarytext(chunk):
+        if not binary_is_text(chunk):
             return self.from_mimetype('application/octet-stream')
         elif b"<html>" in chunk.lower():
             return self.from_mimetype('text/html')
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 1f9d02df5..42fbbda7f 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -174,17 +174,25 @@ def memoizemethod_noargs(method):
         return cache[self]
     return new_method
 
+
 _BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
 _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
 
-
+@deprecated("scrapy.utils.python.binary_is_text")
 def isbinarytext(text):
-    """Return True if the given text is considered binary, or False
-    otherwise, by looking for binary bytes at their chars
+    """ This function is deprecated.
+    Please use scrapy.utils.python.binary_is_text, which was created to be more
+    clear about the functions behavior: it is behaving inverted to this one. """
+    return not binary_is_text(text)
+
+
+def binary_is_text(data):
+    """ Returns `True` if the given ``data`` argument (a ``bytes`` object)
+    does not contain unprintable control characters.
     """
-    if not isinstance(text, bytes):
-        raise TypeError("text must be bytes, got '%s'" % type(text).__name__)
-    return any(c in _BINARYCHARS for c in text)
+    if not isinstance(data, bytes):
+        raise TypeError("data must be bytes, got '%s'" % type(data).__name__)
+    return all(c not in _BINARYCHARS for c in data)
 
 
 def get_func_args(func, stripself=False):
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 4f0834902..9a0cc975d 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -5,7 +5,7 @@ from itertools import count
 import six
 
 from scrapy.utils.python import (
-    memoizemethod_noargs, isbinarytext, equal_attributes,
+    memoizemethod_noargs, binary_is_text, equal_attributes,
     WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode,
     without_none_values)
 
@@ -71,18 +71,18 @@ class MemoizedMethodTest(unittest.TestCase):
         assert one is not three
 
 
-class IsBinaryTextTest(unittest.TestCase):
-    def test_isbinarytext(self):
-        assert not isbinarytext(b"hello")
+class BinaryIsTextTest(unittest.TestCase):
+    def test_binaryistext(self):
+        assert binary_is_text(b"hello")
 
     def test_utf_16_strings_contain_null_bytes(self):
-        assert not isbinarytext(u"hello".encode('utf-16'))
+        assert binary_is_text(u"hello".encode('utf-16'))
 
     def test_one_with_encoding(self):
-        assert not isbinarytext(b"<div>Price \xa3</div>")
+        assert binary_is_text(b"<div>Price \xa3</div>")
 
     def test_real_binary_bytes(self):
-        assert isbinarytext(b"\x02\xa3")
+        assert not binary_is_text(b"\x02\xa3")
 
 
 

From e9d48f8a8ed79be0f7ee16caa24dea3b8ae9e54d Mon Sep 17 00:00:00 2001
From: djunzu <djunzu@users.noreply.github.com>
Date: Thu, 31 Mar 2016 19:19:49 -0300
Subject: [PATCH 0916/2974] Add tests.

	modified:   tests/test_pipeline_files.py
	modified:   tests/test_pipeline_images.py
---
 tests/test_pipeline_files.py  | 29 +++++++++++++++++++++
 tests/test_pipeline_images.py | 48 +++++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+)

diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 77e75d5ac..f480b0c18 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -183,6 +183,35 @@ class FilesPipelineTestCaseFields(unittest.TestCase):
             self.assertEqual(item['stored_file'], [results[0][1]])
 
 
+class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
+
+    def setUp(self):
+        self.tempdir = mkdtemp()
+        self.pipeline = FilesPipeline(self.tempdir)
+        self.default_settings = Settings()
+
+    def tearDown(self):
+        rmtree(self.tempdir)
+
+    def test_expires(self):
+        another_pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir,
+                                                                'FILES_EXPIRES': 42}))
+        self.assertEqual(self.pipeline.expires, self.default_settings.getint('FILES_EXPIRES'))
+        self.assertEqual(another_pipeline.expires, 42)
+
+    def test_files_urls_field(self):
+        another_pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir,
+                                                                'FILES_URLS_FIELD': 'funny_field'}))
+        self.assertEqual(self.pipeline.files_urls_field, self.default_settings.get('FILES_URLS_FIELD'))
+        self.assertEqual(another_pipeline.files_urls_field, 'funny_field')
+
+    def test_files_result_field(self):
+        another_pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir,
+                                                                'FILES_RESULT_FIELD': 'funny_field'}))
+        self.assertEqual(self.pipeline.files_result_field, self.default_settings.get('FILES_RESULT_FIELD'))
+        self.assertEqual(another_pipeline.files_result_field, 'funny_field')
+
+
 class TestS3FilesStore(unittest.TestCase):
     @defer.inlineCallbacks
     def test_persist(self):
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index f52fb4d3d..f48547b0f 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -205,6 +205,54 @@ class ImagesPipelineTestCaseFields(unittest.TestCase):
             self.assertEqual(item['stored_image'], [results[0][1]])
 
 
+class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
+
+    def setUp(self):
+        self.tempdir = mkdtemp()
+        self.pipeline = ImagesPipeline(self.tempdir)
+        self.default_settings = Settings()
+
+    def tearDown(self):
+        rmtree(self.tempdir)
+
+    def test_expires(self):
+        another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
+                                                                'IMAGES_EXPIRES': 42}))
+        self.assertEqual(self.pipeline.expires, self.default_settings.getint('IMAGES_EXPIRES'))
+        self.assertEqual(another_pipeline.expires, 42)
+
+    def test_images_urls_field(self):
+        another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
+                                                                'IMAGES_URLS_FIELD': 'funny_field'}))
+        self.assertEqual(self.pipeline.images_urls_field, self.default_settings.get('IMAGES_URLS_FIELD'))
+        self.assertEqual(another_pipeline.images_urls_field, 'funny_field')
+
+    def test_images_result_field(self):
+        another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
+                                                                'IMAGES_RESULT_FIELD': 'funny_field'}))
+        self.assertEqual(self.pipeline.images_result_field, self.default_settings.get('IMAGES_RESULT_FIELD'))
+        self.assertEqual(another_pipeline.images_result_field, 'funny_field')
+
+    def test_min_width(self):
+        another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
+                                                                'IMAGES_MIN_WIDTH': 42}))
+        self.assertEqual(self.pipeline.min_width, self.default_settings.getint('IMAGES_MIN_WIDTH'))
+        self.assertEqual(another_pipeline.min_width, 42)
+
+    def test_min_height(self):
+        another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
+                                                                'IMAGES_MIN_HEIGHT': 42}))
+        self.assertEqual(self.pipeline.min_height, self.default_settings.getint('IMAGES_MIN_HEIGHT'))
+        self.assertEqual(another_pipeline.min_height, 42)
+
+    def test_thumbs(self):
+        custom_thumbs = {'small': (50, 50), 'big': (270, 270)}
+        another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
+                                                                'IMAGES_THUMBS': custom_thumbs}))
+        self.assertEqual(self.pipeline.thumbs, self.default_settings.get('IMAGES_THUMBS'))
+        self.assertEqual(another_pipeline.thumbs, custom_thumbs)
+
+
 def _create_image(format, *a, **kw):
     buf = TemporaryFile()
     Image.new(*a, **kw).save(buf, format)

From c7fc17866feb6c858673eefc690f5b10048a8b9c Mon Sep 17 00:00:00 2001
From: djunzu <djunzu@users.noreply.github.com>
Date: Thu, 31 Mar 2016 19:20:33 -0300
Subject: [PATCH 0917/2974] Move default settings to
 settings/default_settings.py.

	modified:   scrapy/pipelines/files.py
	modified:   scrapy/pipelines/images.py
	modified:   scrapy/settings/default_settings.py
---
 scrapy/pipelines/files.py           |  9 +++------
 scrapy/pipelines/images.py          | 17 ++++++-----------
 scrapy/settings/default_settings.py | 10 ++++++++++
 3 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index b1b8404bb..338286092 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -213,14 +213,11 @@ class FilesPipeline(MediaPipeline):
     """
 
     MEDIA_NAME = "file"
-    EXPIRES = 90
     STORE_SCHEMES = {
         '': FSFilesStore,
         'file': FSFilesStore,
         's3': S3FilesStore,
     }
-    DEFAULT_FILES_URLS_FIELD = 'file_urls'
-    DEFAULT_FILES_RESULT_FIELD = 'files'
 
     def __init__(self, store_uri, download_func=None):
         if not store_uri:
@@ -235,9 +232,9 @@ class FilesPipeline(MediaPipeline):
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
         s3store.POLICY = settings['FILES_STORE_S3_ACL']
 
-        cls.FILES_URLS_FIELD = settings.get('FILES_URLS_FIELD', cls.DEFAULT_FILES_URLS_FIELD)
-        cls.FILES_RESULT_FIELD = settings.get('FILES_RESULT_FIELD', cls.DEFAULT_FILES_RESULT_FIELD)
-        cls.EXPIRES = settings.getint('FILES_EXPIRES', 90)
+        cls.FILES_URLS_FIELD = settings.get('FILES_URLS_FIELD')
+        cls.FILES_RESULT_FIELD = settings.get('FILES_RESULT_FIELD')
+        cls.EXPIRES = settings.getint('FILES_EXPIRES')
         store_uri = settings['FILES_STORE']
         return cls(store_uri)
 
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index ff73b44b7..1f6b0c7e7 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -36,24 +36,19 @@ class ImagesPipeline(FilesPipeline):
     """
 
     MEDIA_NAME = 'image'
-    MIN_WIDTH = 0
-    MIN_HEIGHT = 0
-    THUMBS = {}
-    DEFAULT_IMAGES_URLS_FIELD = 'image_urls'
-    DEFAULT_IMAGES_RESULT_FIELD = 'images'
 
     @classmethod
     def from_settings(cls, settings):
-        cls.MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH', 0)
-        cls.MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT', 0)
-        cls.EXPIRES = settings.getint('IMAGES_EXPIRES', 90)
-        cls.THUMBS = settings.get('IMAGES_THUMBS', {})
+        cls.MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH')
+        cls.MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT')
+        cls.EXPIRES = settings.getint('IMAGES_EXPIRES')
+        cls.THUMBS = settings.get('IMAGES_THUMBS')
         s3store = cls.STORE_SCHEMES['s3']
         s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
 
-        cls.IMAGES_URLS_FIELD = settings.get('IMAGES_URLS_FIELD', cls.DEFAULT_IMAGES_URLS_FIELD)
-        cls.IMAGES_RESULT_FIELD = settings.get('IMAGES_RESULT_FIELD', cls.DEFAULT_IMAGES_RESULT_FIELD)
+        cls.IMAGES_URLS_FIELD = settings.get('IMAGES_URLS_FIELD')
+        cls.IMAGES_RESULT_FIELD = settings.get('IMAGES_RESULT_FIELD')
         store_uri = settings['IMAGES_STORE']
         return cls(store_uri)
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index d449c4891..843506741 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -159,6 +159,9 @@ FEED_EXPORTERS_BASE = {
 }
 
 FILES_STORE_S3_ACL = 'private'
+FILES_EXPIRES = 90
+FILES_URLS_FIELD = 'file_urls'
+FILES_RESULT_FIELD = 'files'
 
 HTTPCACHE_ENABLED = False
 HTTPCACHE_DIR = 'httpcache'
@@ -175,6 +178,13 @@ HTTPCACHE_GZIP = False
 
 HTTPPROXY_AUTH_ENCODING = 'latin-1'
 
+IMAGES_MIN_WIDTH = 0
+IMAGES_MIN_HEIGHT = 0
+IMAGES_EXPIRES = 90
+IMAGES_THUMBS = {}
+IMAGES_URLS_FIELD = 'image_urls'
+IMAGES_RESULT_FIELD = 'images'
+
 ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
 
 ITEM_PIPELINES = {}

From 8228a0c49113a7cd4ab1099fae746268183655fa Mon Sep 17 00:00:00 2001
From: djunzu <djunzu@users.noreply.github.com>
Date: Thu, 31 Mar 2016 19:20:39 -0300
Subject: [PATCH 0918/2974] Change FilesPipeline class attributes to instance
 attributes.

	modified:   scrapy/pipelines/files.py
	modified:   tests/test_pipeline_files.py
---
 scrapy/pipelines/files.py    | 24 +++++++++++++++---------
 tests/test_pipeline_files.py |  2 +-
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 338286092..c94794173 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -22,6 +22,7 @@ except ImportError:
 from twisted.internet import defer, threads
 
 from scrapy.pipelines.media import MediaPipeline
+from scrapy.settings import Settings
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.misc import md5sum
@@ -219,10 +220,18 @@ class FilesPipeline(MediaPipeline):
         's3': S3FilesStore,
     }
 
-    def __init__(self, store_uri, download_func=None):
+    def __init__(self, store_uri, download_func=None, settings=None):
         if not store_uri:
             raise NotConfigured
+        
+        if isinstance(settings, dict) or settings is None:
+            settings = Settings(settings)
+        
         self.store = self._get_store(store_uri)
+        self.expires = settings.getint('FILES_EXPIRES')
+        self.files_urls_field = settings.get('FILES_URLS_FIELD')
+        self.files_result_field = settings.get('FILES_RESULT_FIELD')
+
         super(FilesPipeline, self).__init__(download_func=download_func)
 
     @classmethod
@@ -232,11 +241,8 @@ class FilesPipeline(MediaPipeline):
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
         s3store.POLICY = settings['FILES_STORE_S3_ACL']
 
-        cls.FILES_URLS_FIELD = settings.get('FILES_URLS_FIELD')
-        cls.FILES_RESULT_FIELD = settings.get('FILES_RESULT_FIELD')
-        cls.EXPIRES = settings.getint('FILES_EXPIRES')
         store_uri = settings['FILES_STORE']
-        return cls(store_uri)
+        return cls(store_uri, settings=settings)
 
     def _get_store(self, uri):
         if os.path.isabs(uri):  # to support win32 paths like: C:\\some\dir
@@ -257,7 +263,7 @@ class FilesPipeline(MediaPipeline):
 
             age_seconds = time.time() - last_modified
             age_days = age_seconds / 60 / 60 / 24
-            if age_days > self.EXPIRES:
+            if age_days > self.expires:
                 return  # returning None force download
 
             referer = referer_str(request)
@@ -356,7 +362,7 @@ class FilesPipeline(MediaPipeline):
 
     ### Overridable Interface
     def get_media_requests(self, item, info):
-        return [Request(x) for x in item.get(self.FILES_URLS_FIELD, [])]
+        return [Request(x) for x in item.get(self.files_urls_field, [])]
 
     def file_downloaded(self, response, request, info):
         path = self.file_path(request, response=response, info=info)
@@ -367,8 +373,8 @@ class FilesPipeline(MediaPipeline):
         return checksum
 
     def item_completed(self, results, item, info):
-        if isinstance(item, dict) or self.FILES_RESULT_FIELD in item.fields:
-            item[self.FILES_RESULT_FIELD] = [x for ok, x in results if ok]
+        if isinstance(item, dict) or self.files_result_field in item.fields:
+            item[self.files_result_field] = [x for ok, x in results if ok]
         return item
 
     def file_path(self, request, response=None, info=None):
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index f480b0c18..391538562 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -91,7 +91,7 @@ class FilesPipelineTestCase(unittest.TestCase):
         patchers = [
             mock.patch.object(FSFilesStore, 'stat_file', return_value={
                 'checksum': 'abc',
-                'last_modified': time.time() - (FilesPipeline.EXPIRES * 60 * 60 * 24 * 2)}),
+                'last_modified': time.time() - (self.pipeline.expires * 60 * 60 * 24 * 2)}),
             mock.patch.object(FilesPipeline, 'get_media_requests',
                               return_value=[_prepare_request_object(item_url)]),
             mock.patch.object(FilesPipeline, 'inc_stats', return_value=True)

From 537083524e2a9ccf545776b23b3248220dc9f16a Mon Sep 17 00:00:00 2001
From: djunzu <djunzu@users.noreply.github.com>
Date: Thu, 31 Mar 2016 19:20:43 -0300
Subject: [PATCH 0919/2974] Change ImagesPipeline class attributes to instance
 attributes.

	modified:   scrapy/pipelines/images.py
---
 scrapy/pipelines/images.py | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 1f6b0c7e7..c597b6cca 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -17,6 +17,7 @@ from PIL import Image
 from scrapy.utils.misc import md5sum
 from scrapy.utils.python import to_bytes
 from scrapy.http import Request
+from scrapy.settings import Settings
 from scrapy.exceptions import DropItem
 #TODO: from scrapy.pipelines.media import MediaPipeline
 from scrapy.pipelines.files import FileException, FilesPipeline
@@ -37,20 +38,27 @@ class ImagesPipeline(FilesPipeline):
 
     MEDIA_NAME = 'image'
 
+    def __init__(self, store_uri, download_func=None, settings=None):
+        super(ImagesPipeline, self).__init__(store_uri, settings=settings, download_func=download_func)
+        
+        if isinstance(settings, dict) or settings is None:
+            settings = Settings(settings)
+
+        self.expires = settings.getint('IMAGES_EXPIRES')
+        self.images_urls_field = settings.get('IMAGES_URLS_FIELD')
+        self.images_result_field = settings.get('IMAGES_RESULT_FIELD')
+        self.min_width = settings.getint('IMAGES_MIN_WIDTH')
+        self.min_height = settings.getint('IMAGES_MIN_HEIGHT')
+        self.thumbs = settings.get('IMAGES_THUMBS')
+
     @classmethod
     def from_settings(cls, settings):
-        cls.MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH')
-        cls.MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT')
-        cls.EXPIRES = settings.getint('IMAGES_EXPIRES')
-        cls.THUMBS = settings.get('IMAGES_THUMBS')
         s3store = cls.STORE_SCHEMES['s3']
         s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
 
-        cls.IMAGES_URLS_FIELD = settings.get('IMAGES_URLS_FIELD')
-        cls.IMAGES_RESULT_FIELD = settings.get('IMAGES_RESULT_FIELD')
         store_uri = settings['IMAGES_STORE']
-        return cls(store_uri)
+        return cls(store_uri, settings=settings)
 
     def file_downloaded(self, response, request, info):
         return self.image_downloaded(response, request, info)
@@ -73,14 +81,14 @@ class ImagesPipeline(FilesPipeline):
         orig_image = Image.open(BytesIO(response.body))
 
         width, height = orig_image.size
-        if width < self.MIN_WIDTH or height < self.MIN_HEIGHT:
+        if width < self.min_width or height < self.min_height:
             raise ImageException("Image too small (%dx%d < %dx%d)" %
-                                 (width, height, self.MIN_WIDTH, self.MIN_HEIGHT))
+                                 (width, height, self.min_width, self.min_height))
 
         image, buf = self.convert_image(orig_image)
         yield path, image, buf
 
-        for thumb_id, size in six.iteritems(self.THUMBS):
+        for thumb_id, size in six.iteritems(self.thumbs):
             thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
             thumb_image, thumb_buf = self.convert_image(image, size)
             yield thumb_path, thumb_image, thumb_buf
@@ -102,11 +110,11 @@ class ImagesPipeline(FilesPipeline):
         return image, buf
 
     def get_media_requests(self, item, info):
-        return [Request(x) for x in item.get(self.IMAGES_URLS_FIELD, [])]
+        return [Request(x) for x in item.get(self.images_urls_field, [])]
 
     def item_completed(self, results, item, info):
-        if isinstance(item, dict) or self.IMAGES_RESULT_FIELD in item.fields:
-            item[self.IMAGES_RESULT_FIELD] = [x for ok, x in results if ok]
+        if isinstance(item, dict) or self.images_result_field in item.fields:
+            item[self.images_result_field] = [x for ok, x in results if ok]
         return item
 
     def file_path(self, request, response=None, info=None):

From 9250a5bffa91c24dbea5c5d64c3c7cd9992a6ee7 Mon Sep 17 00:00:00 2001
From: Aron Bordin <aron.bordin@gmail.com>
Date: Sat, 5 Mar 2016 19:36:02 -0300
Subject: [PATCH 0920/2974] added FEED_TEMPDIR to settings

---
 docs/topics/settings.rst            |  9 +++++++
 scrapy/extensions/feedexport.py     |  8 ++++--
 scrapy/settings/default_settings.py |  1 +
 tests/test_feedexport.py            | 41 ++++++++++++++++++++++++++---
 4 files changed, 54 insertions(+), 5 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index e42dc6370..1bf428edb 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -178,6 +178,15 @@ Default: ``None``
 The AWS secret key used by code that requires access to `Amazon Web services`_,
 such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
 
+.. setting:: FEED_TEMPDIR
+
+FEED_TEMPDIR
+------------
+
+The Feed Temp dir allows you to set a custom folder to save crawler
+temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
+:ref:`Amazon S3 <topics-feed-storage-s3>`.
+
 .. setting:: BOT_NAME
 
 BOT_NAME
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 3dab2d77e..5d90ef738 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -8,7 +8,7 @@ import os
 import sys
 import logging
 import posixpath
-from tempfile import TemporaryFile
+from tempfile import NamedTemporaryFile
 from datetime import datetime
 import six
 from six.moves.urllib.parse import urlparse
@@ -47,7 +47,11 @@ class IFeedStorage(Interface):
 class BlockingFeedStorage(object):
 
     def open(self, spider):
-        return TemporaryFile(prefix='feed-')
+        path = spider.crawler.settings['FEED_TEMPDIR']
+        if path and not os.path.isdir(path):
+            raise OSError('Not a Directory: ' + str(path))
+
+        return NamedTemporaryFile(prefix='feed-', dir=path)
 
     def store(self, file):
         return threads.deferToThread(self._store_in_thread, file)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index d449c4891..b9d01e155 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -134,6 +134,7 @@ EXTENSIONS_BASE = {
     'scrapy.extensions.throttle.AutoThrottle': 0,
 }
 
+FEED_TEMPDIR = None
 FEED_URI = None
 FEED_URI_PARAMS = None  # a function to extend uri arguments
 FEED_FORMAT = 'jsonlines'
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c76d26b57..05f19d33e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -18,9 +18,9 @@ from w3lib.url import path_to_file_uri
 import scrapy
 from scrapy.extensions.feedexport import (
     IFeedStorage, FileFeedStorage, FTPFeedStorage,
-    S3FeedStorage, StdoutFeedStorage
-)
-from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
+    S3FeedStorage, StdoutFeedStorage,
+    BlockingFeedStorage)
+from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
 from scrapy.utils.python import to_native_str
 
 
@@ -87,6 +87,41 @@ class FTPFeedStorageTest(unittest.TestCase):
             self.assertEqual(fp.read(), b"new content")
 
 
+class BlockingFeedStorageTest(unittest.TestCase):
+
+    def get_test_spider(self, settings=None):
+        class TestSpider(scrapy.Spider):
+            name = 'test_spider'
+        crawler = get_crawler(settings_dict=settings)
+        spider = TestSpider.from_crawler(crawler)
+        return spider
+
+    def test_default_temp_dir(self):
+        b = BlockingFeedStorage()
+
+        tmp = b.open(self.get_test_spider())
+        tmp_path = os.path.dirname(tmp.name)
+        self.assertEqual(tmp_path, tempfile.gettempdir())
+
+    def test_temp_file(self):
+        b = BlockingFeedStorage()
+
+        tests_path = os.path.dirname(os.path.abspath(__file__))
+        spider = self.get_test_spider({'FEED_TEMPDIR': tests_path})
+        tmp = b.open(spider)
+        tmp_path = os.path.dirname(tmp.name)
+        self.assertEqual(tmp_path, tests_path)
+
+    def test_invalid_folder(self):
+        b = BlockingFeedStorage()
+
+        tests_path = os.path.dirname(os.path.abspath(__file__))
+        invalid_path = os.path.join(tests_path, 'invalid_path')
+        spider = self.get_test_spider({'FEED_TEMPDIR': invalid_path})
+
+        self.assertRaises(OSError, b.open, spider=spider)
+
+
 class S3FeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks

From 103f6eaa88f2fe618b58525ac04278bf5e4bb831 Mon Sep 17 00:00:00 2001
From: Pengyu CHEN <pengyu@libstarrify.so>
Date: Sat, 2 Apr 2016 02:04:50 +0800
Subject: [PATCH 0921/2974] Added: Making it case-insensitive when extracting
 sitemap URLs from a robots.txt

---
 scrapy/utils/sitemap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index bbf37bc28..008196435 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -39,5 +39,5 @@ def sitemap_urls_from_robots(robots_text):
     robots.txt file
     """
     for line in robots_text.splitlines():
-        if line.lstrip().startswith('Sitemap:'):
+        if line.lstrip().lower().startswith('sitemap:'):
             yield line.split(':', 1)[1].strip()

From 6988e9cd4ba4734b5061e0fb1e56b6ca061ea4d6 Mon Sep 17 00:00:00 2001
From: djunzu <djunzu@users.noreply.github.com>
Date: Thu, 31 Mar 2016 19:20:48 -0300
Subject: [PATCH 0922/2974] Update docs.

	modified:   docs/topics/media-pipeline.rst
---
 docs/topics/media-pipeline.rst | 83 +++++++++++++++++++++++-----------
 1 file changed, 56 insertions(+), 27 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index a34f4c053..3da243d29 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -77,30 +77,6 @@ PIL.
 .. _Python Imaging Library: http://www.pythonware.com/products/pil/
 
 
-Usage example
-=============
-
-In order to use a media pipeline first, :ref:`enable it
-<topics-media-pipeline-enabling>`.
-
-Then, if a spider returns a dict with the URLs key ('file_urls' or
-'image_urls', for the Files or Images Pipeline respectively), the pipeline will
-put the results under respective key ('files' or images').
-
-If you prefer to use :class:`~.Item`, then define a custom item with the
-necessary fields, like in this example for Images Pipeline::
-
-    import scrapy
-
-    class MyItem(scrapy.Item):
-
-        # ... other item fields ...
-        image_urls = scrapy.Field()
-        images = scrapy.Field()
-        
-If you need something more complex and want to override the custom pipeline
-behaviour, see :ref:`topics-media-pipeline-override`.
-
 .. _topics-media-pipeline-enabling:
 
 Enabling your Media Pipeline
@@ -171,6 +147,51 @@ Where:
   used). For more info see :ref:`topics-images-thumbnails`.
 
 
+Usage example
+=============
+
+.. setting:: FILES_URLS_FIELD
+.. setting:: FILES_RESULT_FIELD
+.. setting:: IMAGES_URLS_FIELD
+.. setting:: IMAGES_RESULT_FIELD
+
+In order to use a media pipeline first, :ref:`enable it
+<topics-media-pipeline-enabling>`.
+
+Then, if a spider returns a dict with the URLs key (``file_urls`` or
+``image_urls``, for the Files or Images Pipeline respectively), the pipeline will
+put the results under respective key (``files`` or ``images``).
+
+If you prefer to use :class:`~.Item`, then define a custom item with the
+necessary fields, like in this example for Images Pipeline::
+
+    import scrapy
+
+    class MyItem(scrapy.Item):
+
+        # ... other item fields ...
+        image_urls = scrapy.Field()
+        images = scrapy.Field()
+
+If you want to use another field name for the URLs key or for the results key,
+it is also possible to override it.
+
+For the Files Pipeline, set :setting:`FILES_URLS_FIELD` and/or
+:setting:`FILES_RESULT_FIELD` settings::
+
+    FILES_URLS_FIELD = 'field_name_for_your_files_urls'
+    FILES_RESULT_FIELD = 'field_name_for_your_processed_files'
+
+For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/or
+:setting:`IMAGES_RESULT_FIELD` settings::
+
+    IMAGES_URLS_FIELD = 'field_name_for_your_images_urls'
+    IMAGES_RESULT_FIELD = 'field_name_for_your_processed_images'
+
+If you need something more complex and want to override the custom pipeline
+behaviour, see :ref:`topics-media-pipeline-override`.
+
+
 Additional features
 ===================
 
@@ -185,12 +206,14 @@ adjust this retention delay use the :setting:`FILES_EXPIRES` setting (or
 :setting:`IMAGES_EXPIRES`, in case of Images Pipeline), which
 specifies the delay in number of days::
 
-    # 90 days of delay for files expiration
-    FILES_EXPIRES = 90
+    # 120 days of delay for files expiration
+    FILES_EXPIRES = 120
 
     # 30 days of delay for images expiration
     IMAGES_EXPIRES = 30
 
+The default value for both settings is 90 days.
+
 .. _topics-images-thumbnails:
 
 Thumbnail generation for images
@@ -249,7 +272,13 @@ For example::
    IMAGES_MIN_HEIGHT = 110
    IMAGES_MIN_WIDTH = 110
 
-Note: these size constraints don't affect thumbnail generation at all.
+.. note::
+    The size constraints don't affect thumbnail generation at all.
+
+It is possible to set just one size constraint or both. When setting both of
+them, only images that satisfy both minimum sizes will be saved. For the
+above example, images of sizes (105 x 105) or (105 x 200) or (200 x 105) will
+all be dropped because at least one dimension is shorter than the constraint.
 
 By default, there are no size constraints, so all images are processed.
 

From 1656fbcffa28f3b69862109631741a9c0118ac80 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 8 Apr 2016 23:25:50 +0200
Subject: [PATCH 0923/2974] Fix link extractor tests for non-ASCII characters
 from latin1 document

URL path component should use UTF-8 before percent-encoding (that's what
browsers do when you open scrapy/tests/sample_data/link_extractor/linkextractor_latin1.html
and follow the links)
This matches current w3lib v1.14.1
---
 tests/test_linkextractors_deprecated.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py
index 89dcb75c2..7759575f3 100644
--- a/tests/test_linkextractors_deprecated.py
+++ b/tests/test_linkextractors_deprecated.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 import unittest
 from scrapy.linkextractors.regex import RegexLinkExtractor
 from scrapy.http import HtmlResponse
@@ -81,9 +82,14 @@ class BaseSgmlLinkExtractorTestCase(unittest.TestCase):
             Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
         ])
 
+        # document encoding does not affect URL path component, only query part
+        # >>> u'sample_ñ.html'.encode('utf8')
+        # 'sample_\xc3\xb1.html'
+        # >>> u"sample_á.html".encode('utf8')
+        # 'sample_\xc3\xa1.html'
         self.assertEqual(lx.extract_links(response_latin1), [
-            Link(url='http://example.com/sample_%F1.html', text=''),
-            Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')),
+            Link(url='http://example.com/sample_%C3%B1.html', text=''),
+            Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')),
         ])
 
     def test_matches(self):

From 7b5243a2636f520b0d00d90e47256960fe8f1805 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Sat, 9 Apr 2016 15:15:01 +0200
Subject: [PATCH 0924/2974] Add link extractor test for non-ASCII characters in
 query part of URL

---
 .../link_extractor/linkextractor_latin1.html  | 25 +++++++++++--------
 tests/test_linkextractors_deprecated.py       | 11 ++++++--
 2 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/tests/sample_data/link_extractor/linkextractor_latin1.html b/tests/sample_data/link_extractor/linkextractor_latin1.html
index 68609d8d3..fc31d7e5d 100644
--- a/tests/sample_data/link_extractor/linkextractor_latin1.html
+++ b/tests/sample_data/link_extractor/linkextractor_latin1.html
@@ -1,15 +1,18 @@
 <html>
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=latin-1">
-<base href='http://example.com' />
-<title>Sample page with links for testing RegexLinkExtractor</title>
-</head>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=latin-1">
+    <base href='http://example.com' />
+    <title>Sample page with links for testing RegexLinkExtractor</title>
+  </head>
 <body>
-<div id='wrapper'>
-<div id='subwrapper'>
-<a href='sample_�.html'><img src='sample2.jpg'/></a>
-</div>
-<a href='sample_�.html' title='sample �'>sample � text</a>
-</div>
+  <div id='wrapper'>
+    <div id='subwrapper'>
+      <a href='sample_�.html'><img src='sample2.jpg'/></a>
+    </div>
+    <a href='sample_�.html' title='sample �'>sample � text</a>
+    <div id='subwrapper2'>
+      <a href='sample_�.html?price=�32&�=unit'><img src='sample3.jpg'/></a>
+    </div>
+  </div>
 </body>
 </html>
diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py
index 7759575f3..36dfe174f 100644
--- a/tests/test_linkextractors_deprecated.py
+++ b/tests/test_linkextractors_deprecated.py
@@ -84,12 +84,19 @@ class BaseSgmlLinkExtractorTestCase(unittest.TestCase):
 
         # document encoding does not affect URL path component, only query part
         # >>> u'sample_ñ.html'.encode('utf8')
-        # 'sample_\xc3\xb1.html'
+        # b'sample_\xc3\xb1.html'
         # >>> u"sample_á.html".encode('utf8')
-        # 'sample_\xc3\xa1.html'
+        # b'sample_\xc3\xa1.html'
+        # >>> u"sample_ö.html".encode('utf8')
+        # b'sample_\xc3\xb6.html'
+        # >>> u"£32".encode('latin1')
+        # b'\xa332'
+        # >>> u"µ".encode('latin1')
+        # b'\xb5'
         self.assertEqual(lx.extract_links(response_latin1), [
             Link(url='http://example.com/sample_%C3%B1.html', text=''),
             Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')),
+            Link(url='http://example.com/sample_%C3%B6.html?price=%A332&%B5=unit', text=''),
         ])
 
     def test_matches(self):

From a3557dd34d82bd541ba1a9a5d60b6c573a61e11d Mon Sep 17 00:00:00 2001
From: nblock <nblock@users.noreply.github.com>
Date: Mon, 11 Apr 2016 14:06:57 +0200
Subject: [PATCH 0925/2974] Fix spelling mistake

---
 docs/topics/loaders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 01de528f0..ad86dba63 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -62,7 +62,7 @@ locations, using the :meth:`~ItemLoader.add_xpath` method. This is the
 data that will be assigned to the ``name`` field later.
 
 Afterwards, similar calls are used for ``price`` and ``stock`` fields
-(the later using a CSS selector with the :meth:`~ItemLoader.add_css` method),
+(the latter using a CSS selector with the :meth:`~ItemLoader.add_css` method),
 and finally the ``last_update`` field is populated directly with a literal value
 (``today``) using a different method: :meth:`~ItemLoader.add_value`.
 

From 86e4442d177ea781f53f0f105c5ad814560a0bc4 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 6 Apr 2016 20:02:50 +0200
Subject: [PATCH 0926/2974] Fix HTTP Pool key for HTTPS proxy tunneled
 connections (CONNECT method)

---
 scrapy/core/downloader/handlers/http11.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 9cb659e67..157432ee1 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -127,7 +127,8 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
             self._tunnelReadyDeferred.callback(self._protocol)
         else:
             self._tunnelReadyDeferred.errback(
-                TunnelError('Could not open CONNECT tunnel.'))
+                TunnelError('Could not open CONNECT tunnel with proxy %s:%s' % (
+                    self._host, self._port)))
 
     def connectFailed(self, reason):
         """Propagates the errback to the appropriate deferred."""
@@ -193,6 +194,14 @@ class TunnelingAgent(Agent):
                 self._contextFactory, self._connectTimeout,
                 self._bindAddress)
 
+    def _requestWithEndpoint(self, key, endpoint, method, parsedURI,
+            headers, bodyProducer, requestPath):
+        # proxy host and port are required for HTTP pool `key`
+        # otherwise, same remote host connection request could reuse
+        # a cached tunneled connection to a different proxy
+        key = key + self._proxyConf
+        return super(TunnelingAgent, self)._requestWithEndpoint(key, endpoint, method, parsedURI,
+            headers, bodyProducer, requestPath)
 
 
 class ScrapyAgent(object):

From d42a98d3b590515bae30fb698e7aba2d7511608e Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 12 Apr 2016 00:33:25 +0200
Subject: [PATCH 0927/2974] Use newer w3lib.url.safe_url_string() and re-enable
 HTTP request tests

---
 requirements.txt                |   2 +-
 scrapy/http/request/__init__.py |   6 +-
 tests/test_http_request.py      | 285 +++++++++++++++++++++++++-------
 3 files changed, 228 insertions(+), 65 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index db042cfcf..451ab5d5d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@ Twisted>=10.0.0
 lxml
 pyOpenSSL
 cssselect>=0.9
-w3lib>=1.13.0
+w3lib>=1.14.1
 queuelib
 six>=1.5.2
 PyDispatcher>=2.0.5
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 6b802b36e..f48325a0f 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -8,7 +8,7 @@ import six
 from w3lib.url import safe_url_string
 
 from scrapy.http.headers import Headers
-from scrapy.utils.python import to_native_str, to_bytes
+from scrapy.utils.python import to_bytes
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import escape_ajax
 from scrapy.http.common import obsolete_setter
@@ -50,8 +50,8 @@ class Request(object_ref):
         if not isinstance(url, six.string_types):
             raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
 
-        url = to_native_str(url, self.encoding)
-        self._url = escape_ajax(safe_url_string(url))
+        s = safe_url_string(url, self.encoding)
+        self._url = escape_ajax(s)
 
         if ':' not in self._url:
             raise ValueError('Missing scheme in request url: %s' % self._url)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index b81d43c41..d7216e1d2 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1,10 +1,13 @@
+# -*- coding: utf-8 -*-
 import cgi
 import unittest
 import re
 
 import six
 from six.moves import xmlrpc_client as xmlrpclib
-from six.moves.urllib.parse import urlparse
+from six.moves.urllib.parse import urlparse, parse_qs, unquote
+if six.PY3:
+    from urllib.parse import unquote_to_bytes
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_native_str
@@ -86,12 +89,60 @@ class RequestTest(unittest.TestCase):
         r = self.request_class(url="http://www.scrapy.org/blank space")
         self.assertEqual(r.url, "http://www.scrapy.org/blank%20space")
 
-    @unittest.skipUnless(six.PY2, "TODO")
     def test_url_encoding(self):
-        r1 = self.request_class(url=u"http://www.scrapy.org/price/\xa3", encoding="utf-8")
-        r2 = self.request_class(url=u"http://www.scrapy.org/price/\xa3", encoding="latin1")
-        self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3")
-        self.assertEqual(r2.url, "http://www.scrapy.org/price/%A3")
+        r = self.request_class(url=u"http://www.scrapy.org/price/£")
+        self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
+
+    def test_url_encoding_other(self):
+        # encoding affects only query part of URI, not path
+        # path part should always be UTF-8 encoded before percent-escaping
+        r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="utf-8")
+        self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
+
+        r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="latin1")
+        self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
+
+    def test_url_encoding_query(self):
+        r1 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ")
+        self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
+
+        # should be same as above
+        r2 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")
+        self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
+
+    def test_url_encoding_query_latin1(self):
+        # encoding is used for encoding query-string before percent-escaping;
+        # path is still UTF-8 encoded before percent-escaping
+        r3 = self.request_class(url=u"http://www.scrapy.org/price/µ?currency=£", encoding="latin1")
+        self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3")
+
+    def test_url_encoding_nonutf8_untouched(self):
+        # percent-escaping sequences that do not match valid UTF-8 sequences
+        # should be kept untouched (just upper-cased perhaps)
+        #
+        # See https://tools.ietf.org/html/rfc3987#section-3.2
+        #
+        # "Conversions from URIs to IRIs MUST NOT use any character encoding
+        # other than UTF-8 in steps 3 and 4, even if it might be possible to
+        # guess from the context that another character encoding than UTF-8 was
+        # used in the URI.  For example, the URI
+        # "http://www.example.org/r%E9sum%E9.html" might with some guessing be
+        # interpreted to contain two e-acute characters encoded as iso-8859-1.
+        # It must not be converted to an IRI containing these e-acute
+        # characters.  Otherwise, in the future the IRI will be mapped to
+        # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different
+        # URI from "http://www.example.org/r%E9sum%E9.html".
+        r1 = self.request_class(url=u"http://www.scrapy.org/price/%a3")
+        self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3")
+
+        r2 = self.request_class(url=u"http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
+        self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
+
+        r3 = self.request_class(url=u"http://www.scrapy.org/résumé/%a3")
+        self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
+
+        r4 = self.request_class(url=u"http://www.example.org/r%E9sum%E9.html")
+        self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html")
 
     def test_body(self):
         r1 = self.request_class(url="http://www.example.com/")
@@ -198,19 +249,46 @@ class FormRequestTest(RequestTest):
         r1 = self.request_class("http://www.example.com", formdata={})
         self.assertEqual(r1.body, b'')
 
-    @unittest.skipUnless(six.PY2, "TODO")
-    def test_default_encoding(self):
+    def test_default_encoding_bytes(self):
         # using default encoding (utf-8)
-        data = {'one': 'two', 'price': '\xc2\xa3 100'}
+        data = {b'one': b'two', b'price': b'\xc2\xa3 100'}
         r2 = self.request_class("http://www.example.com", formdata=data)
         self.assertEqual(r2.method, 'POST')
         self.assertEqual(r2.encoding, 'utf-8')
         self.assertQueryEqual(r2.body, b'price=%C2%A3+100&one=two')
         self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')
 
-    def test_custom_encoding(self):
-        data = {'price': u'\xa3 100'}
-        r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
+    def test_default_encoding_textual_data(self):
+        # using default encoding (utf-8)
+        data = {u'µ one': u'two', u'price': u'£ 100'}
+        r2 = self.request_class("http://www.example.com", formdata=data)
+        self.assertEqual(r2.method, 'POST')
+        self.assertEqual(r2.encoding, 'utf-8')
+        self.assertQueryEqual(r2.body, b'price=%C2%A3+100&%C2%B5+one=two')
+        self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')
+
+    def test_default_encoding_mixed_data(self):
+        # using default encoding (utf-8)
+        data = {u'\u00b5one': b'two', b'price\xc2\xa3': u'\u00a3 100'}
+        r2 = self.request_class("http://www.example.com", formdata=data)
+        self.assertEqual(r2.method, 'POST')
+        self.assertEqual(r2.encoding, 'utf-8')
+        self.assertQueryEqual(r2.body, b'%C2%B5one=two&price%C2%A3=%C2%A3+100')
+        self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')
+
+    def test_custom_encoding_bytes(self):
+        data = {b'\xb5 one': b'two', b'price': b'\xa3 100'}
+        r2 = self.request_class("http://www.example.com", formdata=data,
+                                    encoding='latin1')
+        self.assertEqual(r2.method, 'POST')
+        self.assertEqual(r2.encoding, 'latin1')
+        self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two')
+        self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')
+
+    def test_custom_encoding_textual_data(self):
+        data = {'price': u'£ 100'}
+        r3 = self.request_class("http://www.example.com", formdata=data,
+                                    encoding='latin1')
         self.assertEqual(r3.encoding, 'latin1')
         self.assertEqual(r3.body, b'price=%A3+100')
 
@@ -231,15 +309,78 @@ class FormRequestTest(RequestTest):
             url="http://www.example.com/this/list.html")
         req = self.request_class.from_response(response,
                 formdata={'one': ['two', 'three'], 'six': 'seven'})
+
         self.assertEqual(req.method, 'POST')
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req)
-        self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"})
-        self.assertEqual(set(fs[b"one"]), {b"two", b"three"})
+        self.assertEqual(set(fs[b'test']), {b'val1', b'val2'})
+        self.assertEqual(set(fs[b'one']), {b'two', b'three'})
         self.assertEqual(fs[b'test2'], [b'xxx'])
         self.assertEqual(fs[b'six'], [b'seven'])
 
+    def test_from_response_post_nonascii_bytes_utf8(self):
+        response = _buildresponse(
+            b"""<form action="post.php" method="POST">
+            <input type="hidden" name="test \xc2\xa3" value="val1">
+            <input type="hidden" name="test \xc2\xa3" value="val2">
+            <input type="hidden" name="test2" value="xxx \xc2\xb5">
+            </form>""",
+            url="http://www.example.com/this/list.html")
+        req = self.request_class.from_response(response,
+                formdata={'one': ['two', 'three'], 'six': 'seven'})
+
+        self.assertEqual(req.method, 'POST')
+        self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.url, "http://www.example.com/this/post.php")
+        fs = _qs(req, to_unicode=True)
+        self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
+        self.assertEqual(set(fs[u'one']), {u'two', u'three'})
+        self.assertEqual(fs[u'test2'], [u'xxx µ'])
+        self.assertEqual(fs[u'six'], [u'seven'])
+
+    def test_from_response_post_nonascii_bytes_latin1(self):
+        response = _buildresponse(
+            b"""<form action="post.php" method="POST">
+            <input type="hidden" name="test \xa3" value="val1">
+            <input type="hidden" name="test \xa3" value="val2">
+            <input type="hidden" name="test2" value="xxx \xb5">
+            </form>""",
+            url="http://www.example.com/this/list.html",
+            encoding='latin1',
+            )
+        req = self.request_class.from_response(response,
+                formdata={'one': ['two', 'three'], 'six': 'seven'})
+
+        self.assertEqual(req.method, 'POST')
+        self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.url, "http://www.example.com/this/post.php")
+        fs = _qs(req, to_unicode=True, encoding='latin1')
+        self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
+        self.assertEqual(set(fs[u'one']), {u'two', u'three'})
+        self.assertEqual(fs[u'test2'], [u'xxx µ'])
+        self.assertEqual(fs[u'six'], [u'seven'])
+
+    def test_from_response_post_nonascii_unicode(self):
+        response = _buildresponse(
+            u"""<form action="post.php" method="POST">
+            <input type="hidden" name="test £" value="val1">
+            <input type="hidden" name="test £" value="val2">
+            <input type="hidden" name="test2" value="xxx µ">
+            </form>""",
+            url="http://www.example.com/this/list.html")
+        req = self.request_class.from_response(response,
+                formdata={'one': ['two', 'three'], 'six': 'seven'})
+
+        self.assertEqual(req.method, 'POST')
+        self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
+        self.assertEqual(req.url, "http://www.example.com/this/post.php")
+        fs = _qs(req, to_unicode=True)
+        self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
+        self.assertEqual(set(fs[u'one']), {u'two', u'three'})
+        self.assertEqual(fs[u'test2'], [u'xxx µ'])
+        self.assertEqual(fs[u'six'], [u'seven'])
+
     def test_from_response_extra_headers(self):
         response = _buildresponse(
             """<form action="post.php" method="POST">
@@ -268,10 +409,10 @@ class FormRequestTest(RequestTest):
         self.assertEqual(urlparse(r1.url).hostname, "www.example.com")
         self.assertEqual(urlparse(r1.url).path, "/this/get.php")
         fs = _qs(r1)
-        self.assertEqual(set(fs['test']), set(['val1', 'val2']))
-        self.assertEqual(set(fs['one']), set(['two', 'three']))
-        self.assertEqual(fs['test2'], ['xxx'])
-        self.assertEqual(fs['six'], ['seven'])
+        self.assertEqual(set(fs[b'test']), set([b'val1', b'val2']))
+        self.assertEqual(set(fs[b'one']), set([b'two', b'three']))
+        self.assertEqual(fs[b'test2'], [b'xxx'])
+        self.assertEqual(fs[b'six'], [b'seven'])
 
     def test_from_response_override_params(self):
         response = _buildresponse(
@@ -315,9 +456,9 @@ class FormRequestTest(RequestTest):
             </form>""")
         req = self.request_class.from_response(response)
         fs = _qs(req)
-        self.assertEqual(fs['clickable1'], ['clicked1'])
-        self.assertFalse('i1' in fs, fs)  # xpath in _get_inputs()
-        self.assertFalse('clickable2' in fs, fs)  # xpath in _get_clickable()
+        self.assertEqual(fs[b'clickable1'], [b'clicked1'])
+        self.assertFalse(b'i1' in fs, fs)  # xpath in _get_inputs()
+        self.assertFalse(b'clickable2' in fs, fs)  # xpath in _get_clickable()
 
     def test_from_response_submit_first_clickable(self):
         response = _buildresponse(
@@ -329,10 +470,10 @@ class FormRequestTest(RequestTest):
             </form>""")
         req = self.request_class.from_response(response, formdata={'two': '2'})
         fs = _qs(req)
-        self.assertEqual(fs['clickable1'], ['clicked1'])
-        self.assertFalse('clickable2' in fs, fs)
-        self.assertEqual(fs['one'], ['1'])
-        self.assertEqual(fs['two'], ['2'])
+        self.assertEqual(fs[b'clickable1'], [b'clicked1'])
+        self.assertFalse(b'clickable2' in fs, fs)
+        self.assertEqual(fs[b'one'], [b'1'])
+        self.assertEqual(fs[b'two'], [b'2'])
 
     def test_from_response_submit_not_first_clickable(self):
         response = _buildresponse(
@@ -345,10 +486,10 @@ class FormRequestTest(RequestTest):
         req = self.request_class.from_response(response, formdata={'two': '2'}, \
                                               clickdata={'name': 'clickable2'})
         fs = _qs(req)
-        self.assertEqual(fs['clickable2'], ['clicked2'])
-        self.assertFalse('clickable1' in fs, fs)
-        self.assertEqual(fs['one'], ['1'])
-        self.assertEqual(fs['two'], ['2'])
+        self.assertEqual(fs[b'clickable2'], [b'clicked2'])
+        self.assertFalse(b'clickable1' in fs, fs)
+        self.assertEqual(fs[b'one'], [b'1'])
+        self.assertEqual(fs[b'two'], [b'2'])
 
     def test_from_response_dont_submit_image_as_input(self):
         response = _buildresponse(
@@ -359,7 +500,7 @@ class FormRequestTest(RequestTest):
             </form>""")
         req = self.request_class.from_response(response, dont_click=True)
         fs = _qs(req)
-        self.assertEqual(fs, {'i1': ['i1v']})
+        self.assertEqual(fs, {b'i1': [b'i1v']})
 
     def test_from_response_dont_submit_reset_as_input(self):
         response = _buildresponse(
@@ -371,7 +512,7 @@ class FormRequestTest(RequestTest):
             </form>""")
         req = self.request_class.from_response(response, dont_click=True)
         fs = _qs(req)
-        self.assertEqual(fs, {'i1': ['i1v'], 'i2': ['i2v']})
+        self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']})
 
     def test_from_response_multiple_clickdata(self):
         response = _buildresponse(
@@ -382,11 +523,11 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="two" value="clicked2">
             </form>""")
         req = self.request_class.from_response(response, \
-                clickdata={'name': 'clickable', 'value': 'clicked2'})
+                clickdata={u'name': u'clickable', u'value': u'clicked2'})
         fs = _qs(req)
-        self.assertEqual(fs['clickable'], ['clicked2'])
-        self.assertEqual(fs['one'], ['clicked1'])
-        self.assertEqual(fs['two'], ['clicked2'])
+        self.assertEqual(fs[b'clickable'], [b'clicked2'])
+        self.assertEqual(fs[b'one'], [b'clicked1'])
+        self.assertEqual(fs[b'two'], [b'clicked2'])
 
     def test_from_response_unicode_clickdata(self):
         response = _buildresponse(
@@ -397,9 +538,24 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="eurosign" value="\u20ac">
             </form>""")
         req = self.request_class.from_response(response, \
-                clickdata={'name': u'price in \u00a3'})
-        fs = _qs(req)
-        self.assertTrue(fs[to_native_str(u'price in \u00a3')])
+                clickdata={u'name': u'price in \u00a3'})
+        fs = _qs(req, to_unicode=True)
+        self.assertTrue(fs[u'price in \u00a3'])
+
+    def test_from_response_unicode_clickdata_latin1(self):
+        response = _buildresponse(
+            u"""<form action="get.php" method="GET">
+            <input type="submit" name="price in \u00a3" value="\u00a3 1000">
+            <input type="submit" name="price in \u00a5" value="\u00a5 2000">
+            <input type="hidden" name="poundsign" value="\u00a3">
+            <input type="hidden" name="yensign" value="\u00a5">
+            </form>""",
+            encoding='latin1')
+        req = self.request_class.from_response(response, \
+                clickdata={u'name': u'price in \u00a5'})
+        fs = _qs(req, to_unicode=True, encoding='latin1')
+        self.assertTrue(fs[u'price in \u00a5'])
+
 
     def test_from_response_multiple_forms_clickdata(self):
         response = _buildresponse(
@@ -413,18 +569,18 @@ class FormRequestTest(RequestTest):
             </form>
             """)
         req = self.request_class.from_response(response, formname='form2', \
-                clickdata={'name': 'clickable'})
+                clickdata={u'name': u'clickable'})
         fs = _qs(req)
-        self.assertEqual(fs['clickable'], ['clicked2'])
-        self.assertEqual(fs['field2'], ['value2'])
-        self.assertFalse('field1' in fs, fs)
+        self.assertEqual(fs[b'clickable'], [b'clicked2'])
+        self.assertEqual(fs[b'field2'], [b'value2'])
+        self.assertFalse(b'field1' in fs, fs)
 
     def test_from_response_override_clickable(self):
         response = _buildresponse('''<form><input type="submit" name="clickme" value="one"> </form>''')
         req = self.request_class.from_response(response, \
                 formdata={'clickme': 'two'}, clickdata={'name': 'clickme'})
         fs = _qs(req)
-        self.assertEqual(fs['clickme'], ['two'])
+        self.assertEqual(fs[b'clickme'], [b'two'])
 
     def test_from_response_dont_click(self):
         response = _buildresponse(
@@ -436,8 +592,8 @@ class FormRequestTest(RequestTest):
             </form>""")
         r1 = self.request_class.from_response(response, dont_click=True)
         fs = _qs(r1)
-        self.assertFalse('clickable1' in fs, fs)
-        self.assertFalse('clickable2' in fs, fs)
+        self.assertFalse(b'clickable1' in fs, fs)
+        self.assertFalse(b'clickable2' in fs, fs)
 
     def test_from_response_ambiguous_clickdata(self):
         response = _buildresponse(
@@ -468,8 +624,8 @@ class FormRequestTest(RequestTest):
             """)
         req = self.request_class.from_response(response, clickdata={'nr': 1})
         fs = _qs(req)
-        self.assertIn('clickable2', fs)
-        self.assertNotIn('clickable1', fs)
+        self.assertIn(b'clickable2', fs)
+        self.assertNotIn(b'clickable1', fs)
 
     def test_from_response_invalid_nr_index_clickdata(self):
         response = _buildresponse(
@@ -490,7 +646,7 @@ class FormRequestTest(RequestTest):
                                   """</form></body></html>""")
         req = self.request_class.from_response(response, formdata={'bar': 'buz'})
         fs = _qs(req)
-        self.assertEqual(fs, {'foo': ['xxx'], 'bar': ['buz']})
+        self.assertEqual(fs, {b'foo': [b'xxx'], b'bar': [b'buz']})
 
     def test_from_response_errors_formnumber(self):
         response = _buildresponse(
@@ -634,7 +790,7 @@ class FormRequestTest(RequestTest):
             <select name="i7"/>
             </form>''')
         req = self.request_class.from_response(res)
-        fs = _qs(req)
+        fs = _qs(req, to_unicode=True)
         self.assertEqual(fs, {'i1': ['i1v2'], 'i2': ['i2v1'], 'i4': ['i4v2', 'i4v3']})
 
     def test_from_response_radio(self):
@@ -651,7 +807,7 @@ class FormRequestTest(RequestTest):
             </form>''')
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(fs, {'i1': ['iv2'], 'i2': ['on']})
+        self.assertEqual(fs, {b'i1': [b'iv2'], b'i2': [b'on']})
 
     def test_from_response_checkbox(self):
         res = _buildresponse(
@@ -667,7 +823,7 @@ class FormRequestTest(RequestTest):
             </form>''')
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(fs, {'i1': ['iv2'], 'i2': ['on']})
+        self.assertEqual(fs, {b'i1': [b'iv2'], b'i2': [b'on']})
 
     def test_from_response_input_text(self):
         res = _buildresponse(
@@ -680,7 +836,7 @@ class FormRequestTest(RequestTest):
             </form>''')
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(fs, {'i1': ['i1v1'], 'i2': [''], 'i4': ['i4v1']})
+        self.assertEqual(fs, {b'i1': [b'i1v1'], b'i2': [b''], b'i4': [b'i4v1']})
 
     def test_from_response_input_hidden(self):
         res = _buildresponse(
@@ -692,7 +848,7 @@ class FormRequestTest(RequestTest):
             </form>''')
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(fs, {'i1': ['i1v1'], 'i2': ['']})
+        self.assertEqual(fs, {b'i1': [b'i1v1'], b'i2': [b'']})
 
     def test_from_response_input_textarea(self):
         res = _buildresponse(
@@ -704,7 +860,7 @@ class FormRequestTest(RequestTest):
             </form>''')
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(fs, {'i1': ['i1v'], 'i2': [''], 'i3': ['']})
+        self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b''], b'i3': [b'']})
 
     def test_from_response_descendants(self):
         res = _buildresponse(
@@ -725,7 +881,7 @@ class FormRequestTest(RequestTest):
             </form>''')
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(set(fs), set(['h2', 'i2', 'i1', 'i3', 'h1', 'i5', 'i4']))
+        self.assertEqual(set(fs), set([b'h2', b'i2', b'i1', b'i3', b'h1', b'i5', b'i4']))
 
     def test_from_response_xpath(self):
         response = _buildresponse(
@@ -873,12 +1029,18 @@ def _buildresponse(body, **kwargs):
     kwargs.setdefault('encoding', 'utf-8')
     return HtmlResponse(**kwargs)
 
-def _qs(req):
+def _qs(req, encoding='utf-8', to_unicode=False):
     if req.method == 'POST':
         qs = req.body
     else:
         qs = req.url.partition('?')[2]
-    return cgi.parse_qs(qs, True)
+    if six.PY2:
+        uqs = unquote(to_native_str(qs, encoding))
+    elif six.PY3:
+        uqs = unquote_to_bytes(qs)
+    if to_unicode:
+        uqs = uqs.decode(encoding)
+    return parse_qs(uqs, True)
 
 
 class XmlRpcRequestTest(RequestTest):
@@ -890,7 +1052,9 @@ class XmlRpcRequestTest(RequestTest):
     def _test_request(self, **kwargs):
         r = self.request_class('http://scrapytest.org/rpc2', **kwargs)
         self.assertEqual(r.headers[b'Content-Type'], b'text/xml')
-        self.assertEqual(r.body, to_bytes(xmlrpclib.dumps(**kwargs)))
+        self.assertEqual(r.body,
+                         to_bytes(xmlrpclib.dumps(**kwargs),
+                                  encoding=kwargs.get('encoding', 'utf-8')))
         self.assertEqual(r.method, 'POST')
         self.assertEqual(r.encoding, kwargs.get('encoding', 'utf-8'))
         self.assertTrue(r.dont_filter, True)
@@ -899,14 +1063,13 @@ class XmlRpcRequestTest(RequestTest):
         self._test_request(params=('value',))
         self._test_request(params=('username', 'password'), methodname='login')
         self._test_request(params=('response', ), methodresponse='login')
-        self._test_request(params=(u'pas\xa3',), encoding='utf-8')
+        self._test_request(params=(u'pas£',), encoding='utf-8')
         self._test_request(params=(None,), allow_none=1)
         self.assertRaises(TypeError, self._test_request)
         self.assertRaises(TypeError, self._test_request, params=(None,))
 
-    @unittest.skipUnless(six.PY2, "TODO")
     def test_latin1(self):
-        self._test_request(params=(u'pas\xa3',), encoding='latin')
+        self._test_request(params=(u'pas£',), encoding='latin1')
 
 
 if __name__ == "__main__":

From 1ec49c2adaaa29d1590b4146a9cd4581dd4d65b2 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 12 Apr 2016 11:48:57 +0300
Subject: [PATCH 0928/2974] Fix FAQ entry about python versions support

---
 docs/faq.rst | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 411e043b7..5cd62710a 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -29,16 +29,9 @@ comparing `jinja2`_ to `Django`_.
 What Python versions does Scrapy support?
 -----------------------------------------
 
-Scrapy is supported under Python 2.7 only.
+Scrapy is supported under Python 2.7 and Python 3.3+.
 Python 2.6 support was dropped starting at Scrapy 0.20.
-
-Does Scrapy work with Python 3?
--------------------------------
-
-No, but there are plans to support Python 3.3+.
-At the moment, Scrapy works with Python 2.7.
-
-.. seealso:: :ref:`faq-python-versions`.
+Python 3 support was added in Scrapy 1.1.
 
 Did Scrapy "steal" X from Django?
 ---------------------------------

From 2849ebf4c6e9ecfa95dfc5543158f5c1a46256e3 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 12 Apr 2016 14:07:33 +0200
Subject: [PATCH 0929/2974] Reference StackOverflow's "minimal, complete, and
 verifiable example" guide

---
 docs/contributing.rst | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index a21af3ff4..f1f4c63ae 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -48,15 +48,18 @@ guidelines when reporting a new bug.
   if you're not sure if what you're seeing is a bug. You can also ask in the
   `#scrapy` IRC channel.
 
-* write complete, reproducible, specific bug reports. The smaller the test
+* write **complete, reproducible, specific bug reports**. The smaller the test
   case, the better. Remember that other developers won't have your project to
   reproduce the bug, so please include all relevant files required to reproduce
-  it.
+  it. See for example StackOverflow's guide on creating a
+  `Minimal, Complete, and Verifiable example`_ exhibiting the issue.
 
 * include the output of ``scrapy version -v`` so developers working on your bug
   know exactly which version and platform it occurred on, which is often very
   helpful for reproducing it, or knowing if it was already fixed.
 
+.. _Minimal, Complete, and Verifiable example: https://stackoverflow.com/help/mcve
+
 Writing patches
 ===============
 

From a087d2593a36dcacfeb9353d9ca1e6737478ddd9 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 14 Apr 2016 00:12:25 +0200
Subject: [PATCH 0930/2974] Ignore HTTPS certificate verification failures

Fixes #1930
---
 scrapy/core/downloader/contextfactory.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 09fcd491a..3e147ae36 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,6 +1,7 @@
 from OpenSSL import SSL
 from twisted.internet.ssl import ClientContextFactory
 
+
 try:
 
     from zope.interface.declarations import implementer
@@ -11,6 +12,12 @@ try:
     from twisted.web.client import BrowserLikePolicyForHTTPS
     from twisted.web.iweb import IPolicyForHTTPS
 
+
+    class ScrapyClientTLSOptions(ClientTLSOptions):
+        def _identityVerifyingInfoCallback(self, connection, where, ret):
+            pass
+
+
     @implementer(IPolicyForHTTPS)
     class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         """
@@ -49,7 +56,7 @@ try:
             return self.getCertificateOptions().getContext()
 
         def creatorForNetloc(self, hostname, port):
-            return ClientTLSOptions(hostname.decode("ascii"), self.getContext())
+            return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext())
 
 
     @implementer(IPolicyForHTTPS)

From 25ee023561a59401ec932d2afae8ee62e7b5e7d5 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 14 Apr 2016 17:19:55 +0200
Subject: [PATCH 0931/2974] Catch VerificationError but keep the rest of
 ClientTLSOptions

---
 scrapy/core/downloader/contextfactory.py | 12 ++++----
 scrapy/core/downloader/tls.py            | 36 ++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 3e147ae36..b643d935b 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,21 +1,19 @@
 from OpenSSL import SSL
 from twisted.internet.ssl import ClientContextFactory
 
-
 try:
 
     from zope.interface.declarations import implementer
 
     # the following should be available from Twisted 14.0.0
-    from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust
-    from twisted.internet._sslverify import ClientTLSOptions
+    from twisted.internet.ssl import (optionsForClientTLS,
+                                      CertificateOptions,
+                                      platformTrust)
+
     from twisted.web.client import BrowserLikePolicyForHTTPS
     from twisted.web.iweb import IPolicyForHTTPS
 
-
-    class ScrapyClientTLSOptions(ClientTLSOptions):
-        def _identityVerifyingInfoCallback(self, connection, where, ret):
-            pass
+    from scrapy.core.downloader.tls import ScrapyClientTLSOptions
 
 
     @implementer(IPolicyForHTTPS)
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 64ebb0714..2cf8ba869 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -1,6 +1,9 @@
+import logging
 from OpenSSL import SSL
 
 
+logger = logging.getLogger(__name__)
+
 METHOD_SSLv3 = 'SSLv3'
 METHOD_TLS = 'TLS'
 METHOD_TLSv10 = 'TLSv1.0'
@@ -14,3 +17,36 @@ openssl_methods = {
     METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only
     METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
 }
+
+# ClientTLSOptions requires a recent-enough version of Twisted
+try:
+
+    # taken from twisted/twisted/internet/_sslverify.py
+    try:
+        from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START
+    except ImportError:
+        SSL_CB_HANDSHAKE_START = 0x10
+        SSL_CB_HANDSHAKE_DONE = 0x20
+
+    from twisted.internet._sslverify import (ClientTLSOptions,
+                                             _maybeSetHostNameIndication,
+                                             verifyHostname,
+                                             VerificationError)
+
+    class ScrapyClientTLSOptions(ClientTLSOptions):
+        # same as Twisted's ClientTLSOptions,
+        # except that VerificationError is caught
+        # and doesn't close the connection
+        def _identityVerifyingInfoCallback(self, connection, where, ret):
+            if where & SSL_CB_HANDSHAKE_START:
+                _maybeSetHostNameIndication(connection, self._hostnameBytes)
+            elif where & SSL_CB_HANDSHAKE_DONE:
+                try:
+                    verifyHostname(connection, self._hostnameASCII)
+                except VerificationError as e:
+                    logger.warning(e)
+
+except ImportError:
+    # ImportError should not matter for older Twisted versions
+    # as the above is not used in the fallback ScrapyClientContextFactory
+    pass

From d6760dbaac0f1de1938abec7207ba9b52f12c8cd Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 18 Apr 2016 18:30:01 +0200
Subject: [PATCH 0932/2974] Set SNI properly when using CONNECT

---
 scrapy/core/downloader/handlers/http11.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 9cb659e67..4a0612cdf 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -122,7 +122,10 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         """
         self._protocol.dataReceived = self._protocolDataReceived
         if  TunnelingTCP4ClientEndpoint._responseMatcher.match(bytes):
-            self._protocol.transport.startTLS(self._contextFactory,
+            # this set proper Server Name Indication extension
+            sslOptions = self._contextFactory.creatorForNetloc(
+                self._tunneledHost, self._tunneledPort)
+            self._protocol.transport.startTLS(sslOptions,
                                               self._protocolFactory)
             self._tunnelReadyDeferred.callback(self._protocol)
         else:

From dcea11a70c52e3fb03e185cda7065267c2e4b869 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 19 Apr 2016 10:41:13 +0200
Subject: [PATCH 0933/2974] Fall back to no-SNi context factory is Twisted<14
 is used

---
 scrapy/core/downloader/handlers/http11.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 4a0612cdf..1634a2a4b 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -122,9 +122,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         """
         self._protocol.dataReceived = self._protocolDataReceived
         if  TunnelingTCP4ClientEndpoint._responseMatcher.match(bytes):
-            # this set proper Server Name Indication extension
-            sslOptions = self._contextFactory.creatorForNetloc(
-                self._tunneledHost, self._tunneledPort)
+            try:
+                # this sets proper Server Name Indication extension
+                # but is only available for Twisted>=14.0
+                sslOptions = self._contextFactory.creatorForNetloc(
+                    self._tunneledHost, self._tunneledPort)
+            except AttributeError:
+                # fall back to non-SNI SSL context factory
+                sslOptions = self._contextFactory
             self._protocol.transport.startTLS(sslOptions,
                                               self._protocolFactory)
             self._tunnelReadyDeferred.callback(self._protocol)

From cd979ace40f26e8d921aaa4cf2d603b434fa2064 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 20 Apr 2016 14:42:03 +0200
Subject: [PATCH 0934/2974] Add HTTPS tests with non-hostname-maching server
 certificate

---
 tests/keys/example-com.cert.pem   | 26 ++++++++++
 tests/keys/example-com.conf       | 84 +++++++++++++++++++++++++++++++
 tests/keys/example-com.gen.README | 24 +++++++++
 tests/keys/example-com.key.pem    | 28 +++++++++++
 tests/mockserver.py               |  6 +--
 tests/test_downloader_handlers.py | 19 ++++++-
 6 files changed, 183 insertions(+), 4 deletions(-)
 create mode 100644 tests/keys/example-com.cert.pem
 create mode 100644 tests/keys/example-com.conf
 create mode 100644 tests/keys/example-com.gen.README
 create mode 100644 tests/keys/example-com.key.pem

diff --git a/tests/keys/example-com.cert.pem b/tests/keys/example-com.cert.pem
new file mode 100644
index 000000000..af87198aa
--- /dev/null
+++ b/tests/keys/example-com.cert.pem
@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE-----
+MIIEVTCCAz2gAwIBAgIJANuZ/6fbAJNcMA0GCSqGSIb3DQEBCwUAMH0xCzAJBgNV
+BAYTAlhXMQswCQYDVQQIDAJYVzEVMBMGA1UEBwwMVGhlIEludGVybmV0MQ8wDQYD
+VQQKDAZTY3JhcHkxGDAWBgNVBAMMD3d3dy5leGFtcGxlLmNvbTEfMB0GCSqGSIb3
+DQEJARYQdGVzdEBleGFtcGxlLmNvbTAgFw0xNjA0MjAxMjExNTZaGA8yMTE2MDMy
+NzEyMTE1NlowfTELMAkGA1UEBhMCWFcxCzAJBgNVBAgMAlhXMRUwEwYDVQQHDAxU
+aGUgSW50ZXJuZXQxDzANBgNVBAoMBlNjcmFweTEYMBYGA1UEAwwPd3d3LmV4YW1w
+bGUuY29tMR8wHQYJKoZIhvcNAQkBFhB0ZXN0QGV4YW1wbGUuY29tMIIBIjANBgkq
+hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5r2BzxXivtQXvIwrTGug8l6vjuDhf0WD
+HBU5yIHWvX1rT2MQFuZpu120iFGOK7MBYBBdmAIGsR5cHZ03cqQkAXOGQ5ug5O/u
+d/GZUbcgyv8WCwW71MWLb5CNiDWj/vZq6CUqRg2QkUpkjr/DcQrKsm67yadKLgyd
+G85OyZO6NBuAukQcKrNhspk/Ms55X7RbgYPUbZ0bBee4b3GRnE7PLltIsHo/tloV
+ynC0Sd3T1taYyyG7IJd2LWJELzK0Ww+QUV2qoOdZjl8db1x5c99OR6xY0+Mjf14r
+6kkXOBpPkrJ990qU40+z406u2HPf2abR4D/DUoe9qw+fElCeiuXFFQIDAQABo4HV
+MIHSMB0GA1UdDgQWBBTY3DPInWZxrmQfPHA5w2R+AsbnOjAfBgNVHSMEGDAWgBTY
+3DPInWZxrmQfPHA5w2R+AsbnOjAJBgNVHRMEAjAAMAsGA1UdDwQEAwIFoDBKBgNV
+HREEQzBBggtleGFtcGxlLmNvbYIPd3d3LmV4YW1wbGUuY29tghBtYWlsLmV4YW1w
+bGUuY29tgg9mdHAuZXhhbXBsZS5jb20wLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wg
+R2VuZXJhdGVkIENlcnRpZmljYXRlMA0GCSqGSIb3DQEBCwUAA4IBAQDmnzdIu9XV
+/Tnn5/mt9E98YEsF/eaXBSKG+f2oZc3n2errKwY5qYqULlu8mhajGWTd5EfFCjbD
+lH7nmBdRUAhjzKWntc1G84eaWwHyv+N/5WJrtUfa2A1Ps3Mu9Vz4k2M9HGi/s6KX
+IshezlHe3/TyhIT/WC+MZhpFTL73dpuIgHmp2NjjlJqtG25eC6zmonsc2RApJPa8
+6J0WY/ISH9OwhDSbI9+TIE8QwdC7draiCKK/oid3Jg9fzaEQW+Pr3/4AmYWH1j8s
+iaOVYIXYtt3urQ2Q+qfh34kfRfX5IqAdd1r/fnUjaOLhPNJxGP2KuFaYbdSC9p+n
+bfExzQHUvH2n
+-----END CERTIFICATE-----
diff --git a/tests/keys/example-com.conf b/tests/keys/example-com.conf
new file mode 100644
index 000000000..8aa338cd5
--- /dev/null
+++ b/tests/keys/example-com.conf
@@ -0,0 +1,84 @@
+# this is copied from http://stackoverflow.com/a/27931596
+[ req ]
+default_bits        = 2048
+default_keyfile     = server-key.pem
+distinguished_name  = subject
+req_extensions      = req_ext
+x509_extensions     = x509_ext
+string_mask         = utf8only
+
+# The Subject DN can be formed using X501 or RFC 4514 (see RFC 4519 for a description).
+#   Its sort of a mashup. For example, RFC 4514 does not provide emailAddress.
+[ subject ]
+countryName         = Country Name (2 letter code)
+countryName_default     = US
+
+stateOrProvinceName     = State or Province Name (full name)
+stateOrProvinceName_default = NY
+
+localityName            = Locality Name (eg, city)
+localityName_default        = New York
+
+organizationName         = Organization Name (eg, company)
+organizationName_default    = Example, LLC
+
+# Use a friendly name here because its presented to the user. The server's DNS
+#   names are placed in Subject Alternate Names. Plus, DNS names here is deprecated
+#   by both IETF and CA/Browser Forums. If you place a DNS name here, then you 
+#   must include the DNS name in the SAN too (otherwise, Chrome and others that
+#   strictly follow the CA/Browser Baseline Requirements will fail).
+commonName          = Common Name (e.g. server FQDN or YOUR name)
+commonName_default      = Example Company
+
+emailAddress            = Email Address
+emailAddress_default        = test@example.com
+
+# Section x509_ext is used when generating a self-signed certificate. I.e., openssl req -x509 ...
+[ x509_ext ]
+
+subjectKeyIdentifier        = hash
+authorityKeyIdentifier  = keyid,issuer
+
+# You only need digitalSignature below. *If* you don't allow
+#   RSA Key transport (i.e., you use ephemeral cipher suites), then
+#   omit keyEncipherment because that's key transport.
+basicConstraints        = CA:FALSE
+keyUsage            = digitalSignature, keyEncipherment
+subjectAltName          = @alternate_names
+nsComment           = "OpenSSL Generated Certificate"
+
+# RFC 5280, Section 4.2.1.12 makes EKU optional
+#   CA/Browser Baseline Requirements, Appendix (B)(3)(G) makes me confused
+#   In either case, you probably only need serverAuth.
+# extendedKeyUsage  = serverAuth, clientAuth
+
+# Section req_ext is used when generating a certificate signing request. I.e., openssl req ...
+[ req_ext ]
+
+subjectKeyIdentifier        = hash
+
+basicConstraints        = CA:FALSE
+keyUsage            = digitalSignature, keyEncipherment
+subjectAltName          = @alternate_names
+nsComment           = "OpenSSL Generated Certificate"
+
+# RFC 5280, Section 4.2.1.12 makes EKU optional
+#   CA/Browser Baseline Requirements, Appendix (B)(3)(G) makes me confused
+#   In either case, you probably only need serverAuth.
+# extendedKeyUsage  = serverAuth, clientAuth
+
+[ alternate_names ]
+
+DNS.1       = example.com
+DNS.2       = www.example.com
+DNS.3       = mail.example.com
+DNS.4       = ftp.example.com
+
+# Add these if you need them. But usually you don't want them or
+#   need them in production. You may need them for development.
+# DNS.5       = localhost
+# DNS.6       = localhost.localdomain
+# DNS.7       = 127.0.0.1
+
+# IPv6 localhost
+# DNS.8     = ::1
diff --git a/tests/keys/example-com.gen.README b/tests/keys/example-com.gen.README
new file mode 100644
index 000000000..955e7b057
--- /dev/null
+++ b/tests/keys/example-com.gen.README
@@ -0,0 +1,24 @@
+$ openssl req -config example-com.conf \
+    -new -x509 -sha256 -newkey rsa:2048 -nodes \
+    -keyout example-com.key.pem \
+    -days 36500 \
+    -out example-com.cert.pem
+Generating a 2048 bit RSA private key
+....+++
+.....................+++
+writing new private key to 'example-com.key.pem'
+-----
+You are about to be asked to enter information that will be incorporated
+into your certificate request.
+What you are about to enter is what is called a Distinguished Name or a DN.
+There are quite a few fields but you can leave some blank
+For some fields there will be a default value,
+If you enter '.', the field will be left blank.
+-----
+Country Name (2 letter code) [US]:XW
+State or Province Name (full name) [NY]:XW
+Locality Name (eg, city) [New York]:The Internet
+Organization Name (eg, company) [Example, LLC]:Scrapy
+Common Name (e.g. server FQDN or YOUR name) [Example Company]:www.example.com
+Email Address [test@example.com]:
+
diff --git a/tests/keys/example-com.key.pem b/tests/keys/example-com.key.pem
new file mode 100644
index 000000000..56e805df8
--- /dev/null
+++ b/tests/keys/example-com.key.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDmvYHPFeK+1Be8
+jCtMa6DyXq+O4OF/RYMcFTnIgda9fWtPYxAW5mm7XbSIUY4rswFgEF2YAgaxHlwd
+nTdypCQBc4ZDm6Dk7+538ZlRtyDK/xYLBbvUxYtvkI2INaP+9mroJSpGDZCRSmSO
+v8NxCsqybrvJp0ouDJ0bzk7Jk7o0G4C6RBwqs2GymT8yznlftFuBg9RtnRsF57hv
+cZGcTs8uW0iwej+2WhXKcLRJ3dPW1pjLIbsgl3YtYkQvMrRbD5BRXaqg51mOXx1v
+XHlz305HrFjT4yN/XivqSRc4Gk+Ssn33SpTjT7PjTq7Yc9/ZptHgP8NSh72rD58S
+UJ6K5cUVAgMBAAECggEAEVxi3vTzmY4Vtx8Ixvg2JOZQ6TwsW0ocoklKjraONLWy
+FEgM1txBSlKzmaohO1J9oP+6Owyz+jGhlqzUljCZcO6DRKT6Bx+yXp3z/jz8H6nv
+u6aTyl+OrTdAHuaCT1W1F4BsXNb6cKQbSs5M4z1/oMtKH9MRdsOGMqhkLzCJSxA+
+E+rRomrP6E9XooLxqpSJooxmX772XPHE7+ZILzRF0viXJ6z0Jd1cOuAQqyIYvmHM
+4313kYJdAKYgJNxe8M8mYLeZcGwImAz/pNQ5R+uVyZlW3kXVzvS7B2m+KcW7Olu8
+r4Ocpdyh20GStpw1f+tk7PLl+SkwFslK+uI7Wl/ygQKBgQD+QsREu1sv507CnvYC
+FZnFryhHUzxMWIX5bvH3YoPbVptqwG5Nj05zIQPCZAEanW4HENCujO9oWbZqLANO
+Th5sNOkBJiC4X6+1NIzQIszaZs4nKIyWNLIOcP4p20k3cR0sS8wLSLwcuW2is27N
+ACKa8u93X1Gb27V0qUhmEqP3NQKBgQDoUY3HRtOVQnpoi5zbTiSKwlBKEkMAawaW
+Q8VSZmrNQZXpcwa2JYN0IeiHnVjctdLul1u9qj5goghTV4XMQ8LSZs0emhvgJxMa
+QpsDLTRr0mBtmduOwZW9a8EcbI2NCth/Irsdl892+y8UVoAO2G6Fgr0DhgXWOJEo
+RcUUkGHyYQKBgHCkT6NLhYhhZykdl0sxGqDTinqey3XfOetZVWUNhfDkG5JdkgBW
+XqDunWW/PCX4XMhQkMLjuSR3qjK7MPO99AhoIFnb9F76NTOIBmInKK4RIX+DnVTm
+H4P6Vv9gIL2pJQ18vva40G3BUGrmJ042ox4WRjSSS+tKmMcIQATIeU+JAoGBAIpr
+TRZW1Oox2c1Aogzo2kzyyfPYPaOaISqW5pLaAviZA0E9D9qnL1OagHmM/s1CaJNQ
+C5FORiw9XsiJdWbnWMUUC1MYb1N29KLI3KNf48P6bQngijjcjuN1uHG/G/fVZnkZ
+sHNJaItzzfFIOLSfr/pMk2HuELw6qAJez7YY8MMBAoGAf5sC2S7duDH2KliMSYhd
+yByHEVTbf7JdJVd/7kq5MPZauJtYztGTSwyyM+gBv+lxdY1jGu/iNu9xAD2DJlE3
+RTPgTIID/PaSmUVotHLq86hbazKPBorx6UWkbGsthEaSF/cTY2eFFFsK/Awoj0yU
+ZhraPWu8S27Pcr9HvZvh5tg=
+-----END PRIVATE KEY-----
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 365ec81fd..a40e2e501 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -204,10 +204,10 @@ class MockServer():
         time.sleep(0.2)
 
 
-def ssl_context_factory():
+def ssl_context_factory(keyfile='keys/cert.pem', certfile='keys/cert.pem'):
     return ssl.DefaultOpenSSLContextFactory(
-         os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
-         os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
+         os.path.join(os.path.dirname(__file__), keyfile),
+         os.path.join(os.path.dirname(__file__), certfile),
          )
 
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index e13becf1b..45a806f2e 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -119,6 +119,10 @@ class HttpTestCase(unittest.TestCase):
     scheme = 'http'
     download_handler_cls = HTTPDownloadHandler
 
+    # only used for HTTPS tests
+    keyfile = 'keys/cert.pem'
+    certfile = 'keys/cert.pem'
+
     def setUp(self):
         name = self.mktemp()
         os.mkdir(name)
@@ -137,7 +141,8 @@ class HttpTestCase(unittest.TestCase):
         self.host = 'localhost'
         if self.scheme == 'https':
             self.port = reactor.listenSSL(
-                0, self.wrapper, ssl_context_factory(), interface=self.host)
+                0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile),
+                interface=self.host)
         else:
             self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
         self.portno = self.port.getHost().port
@@ -318,6 +323,18 @@ class Https11TestCase(Http11TestCase):
     scheme = 'https'
 
 
+class Https11WrongHostnameTestCase(Http11TestCase):
+    scheme = 'https'
+
+    # above tests use a server certificate for "localhost",
+    # client connection to "localhost" too.
+    # here we test that even if the server certificate is for another domain,
+    # "www.example.com" in this case,
+    # the tests still pass
+    keyfile = 'keys/example-com.key.pem'
+    certfile = 'keys/example-com.cert.pem'
+
+
 class Http11MockServerTestCase(unittest.TestCase):
     """HTTP 1.1 test case with MockServer"""
     if twisted_version < (11, 1, 0):

From 417279ccebfca7d26ccc91e253654d86c030c514 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 20 Apr 2016 15:12:19 +0200
Subject: [PATCH 0935/2974] Bump up w3lib requirement to v1.14.2

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 451ab5d5d..ad9226e87 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@ Twisted>=10.0.0
 lxml
 pyOpenSSL
 cssselect>=0.9
-w3lib>=1.14.1
+w3lib>=1.14.2
 queuelib
 six>=1.5.2
 PyDispatcher>=2.0.5

From 68dedf54cb27847f6d035099b61aa06226549fad Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 21 Apr 2016 14:38:18 +0200
Subject: [PATCH 0936/2974] Fix canonicalize_url() on Python 3 and re-enable
 tests

---
 scrapy/utils/url.py     | 160 ++++++++++++++++++++++++++++++++++++----
 tests/test_utils_url.py |  81 ++++++++++++++++++--
 2 files changed, 217 insertions(+), 24 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index adef4a800..716513648 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -7,15 +7,18 @@ to the w3lib.url module. Always import those from there instead.
 """
 import posixpath
 import re
+import six
 from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag,
                                     urlparse, parse_qsl, urlencode,
-                                    unquote)
+                                    quote, unquote)
+if six.PY3:
+    from urllib.parse import unquote_to_bytes
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this
 # move doesn't break old code
 from w3lib.url import *
 from w3lib.url import _safe_chars
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_bytes, to_native_str, to_unicode
 
 
 def url_is_from_any_domain(url, domains):
@@ -37,42 +40,114 @@ def url_has_any_extension(url, extensions):
     return posixpath.splitext(parse_url(url).path)[1].lower() in extensions
 
 
+def _safe_ParseResult(parts, encoding='utf8', path_encoding='utf8'):
+    return (
+        to_native_str(parts.scheme),
+        to_native_str(parts.netloc.encode('idna')),
+
+        # default encoding for path component SHOULD be UTF-8
+        quote(to_bytes(parts.path, path_encoding), _safe_chars),
+        quote(to_bytes(parts.params, path_encoding), _safe_chars),
+
+        # encoding of query and fragment follows page encoding
+        # or form-charset (if known and passed)
+        quote(to_bytes(parts.query, encoding), _safe_chars),
+        quote(to_bytes(parts.fragment, encoding), _safe_chars)
+    )
+
+
 def canonicalize_url(url, keep_blank_values=True, keep_fragments=False,
                      encoding=None):
     """Canonicalize the given url by applying the following procedures:
 
     - sort query arguments, first by key, then by value
-    - percent encode paths and query arguments. non-ASCII characters are
-      percent-encoded using UTF-8 (RFC-3986)
+    - percent encode paths ; non-ASCII characters are percent-encoded
+      using UTF-8 (RFC-3986)
+    - percent encode query arguments ; non-ASCII characters are percent-encoded
+      using passed `encoding` (UTF-8 by default)
     - normalize all spaces (in query arguments) '+' (plus symbol)
     - normalize percent encodings case (%2f -> %2F)
-    - remove query arguments with blank values (unless keep_blank_values is True)
-    - remove fragments (unless keep_fragments is True)
+    - remove query arguments with blank values (unless `keep_blank_values` is True)
+    - remove fragments (unless `keep_fragments` is True)
 
-    The url passed can be a str or unicode, while the url returned is always a
-    str.
+    The url passed can be bytes or unicode, while the url returned is
+    always a native str (bytes in Python 2, unicode in Python 3).
 
     For examples see the tests in tests/test_utils_url.py
     """
+    # If supplied `encoding` is not compatible with all characters in `url`,
+    # fallback to UTF-8 as safety net.
+    # UTF-8 can handle all Unicode characters,
+    # so we should be covered regarding URL normalization,
+    # if not for proper URL expected by remote website.
+    try:
+        scheme, netloc, path, params, query, fragment = _safe_ParseResult(
+            parse_url(url), encoding=encoding)
+    except UnicodeError as e:
+        if encoding != 'utf8':
+            scheme, netloc, path, params, query, fragment = _safe_ParseResult(
+                parse_url(url), encoding='utf8')
+        else:
+            raise
 
-    scheme, netloc, path, params, query, fragment = parse_url(url)
-    keyvals = parse_qsl(query, keep_blank_values)
+    # 1. decode query-string as UTF-8 (or keep raw bytes),
+    #    sort values,
+    #    and percent-encode them back
+    if not six.PY2:
+        # Python3's urllib.parse.parse_qsl does not work as wanted
+        # for percent-encoded characters that do not match passed encoding,
+        # they get lost.
+        #
+        # e.g., 'q=b%a3' becomes [('q', 'b\ufffd')]
+        # (ie. with 'REPLACEMENT CHARACTER' (U+FFFD),
+        #      instead of \xa3 that you get with Python2's parse_qsl)
+        #
+        # what we want here is to keep raw bytes, and percent encode them
+        # so as to preserve whatever encoding what originally used.
+        #
+        # See https://tools.ietf.org/html/rfc3987#section-6.4:
+        #
+        # For example, it is possible to have a URI reference of
+        # "http://www.example.org/r%E9sum%E9.xml#r%C3%A9sum%C3%A9", where the
+        # document name is encoded in iso-8859-1 based on server settings, but
+        # where the fragment identifier is encoded in UTF-8 according to
+        # [XPointer]. The IRI corresponding to the above URI would be (in XML
+        # notation)
+        # "http://www.example.org/r%E9sum%E9.xml#r&#xE9;sum&#xE9;".
+        # Similar considerations apply to query parts.  The functionality of
+        # IRIs (namely, to be able to include non-ASCII characters) can only be
+        # used if the query part is encoded in UTF-8.
+        keyvals = parse_qsl_to_bytes(query, keep_blank_values)
+    else:
+        keyvals = parse_qsl(query, keep_blank_values)
     keyvals.sort()
     query = urlencode(keyvals)
 
-    # XXX: copied from w3lib.url.safe_url_string to add encoding argument
-    # path = to_native_str(path, encoding)
-    # path = moves.urllib.parse.quote(path, _safe_chars, encoding='latin1') or '/'
+    # 2. decode percent-encoded sequences in path as UTF-8 (or keep raw bytes)
+    #    and percent-encode path again (this normalizes to upper-case %XX)
+    uqp = _unquotepath(path)
+    path = quote(uqp, _safe_chars) or '/'
 
-    path = safe_url_string(_unquotepath(path)) or '/'
     fragment = '' if not keep_fragments else fragment
+
+    # every part should be safe already
     return urlunparse((scheme, netloc.lower(), path, params, query, fragment))
 
 
 def _unquotepath(path):
     for reserved in ('2f', '2F', '3f', '3F'):
         path = path.replace('%' + reserved, '%25' + reserved.upper())
-    return unquote(path)
+
+    if six.PY3:
+        # standard lib's unquote() does not work in Python 3
+        # for non-UTF-8 percent-escaped characters, they get lost.
+        # e.g., '%a3' becomes 'REPLACEMENT CHARACTER' (U+FFFD)
+        #
+        # unquote_to_bytes() returns raw bytes instead
+        return unquote_to_bytes(path)
+    else:
+        # in Python 2, '%a3' becomes '\xa3', which is what we want
+        return unquote(path)
 
 
 def parse_url(url, encoding=None):
@@ -81,7 +156,60 @@ def parse_url(url, encoding=None):
     """
     if isinstance(url, ParseResult):
         return url
-    return urlparse(to_native_str(url, encoding))
+    return urlparse(to_unicode(url, encoding))
+
+
+if six.PY3:
+    from urllib.parse import _coerce_args, unquote_to_bytes
+
+    def parse_qsl_to_bytes(qs, keep_blank_values=False, strict_parsing=False):
+        """Parse a query given as a string argument.
+
+        Data are returned as a list of name, value pairs as bytes.
+
+        Arguments:
+
+        qs: percent-encoded query string to be parsed
+
+        keep_blank_values: flag indicating whether blank values in
+            percent-encoded queries should be treated as blank strings.  A
+            true value indicates that blanks should be retained as blank
+            strings.  The default false value indicates that blank values
+            are to be ignored and treated as if they were  not included.
+
+        strict_parsing: flag indicating what to do with parsing errors. If
+            false (the default), errors are silently ignored. If true,
+            errors raise a ValueError exception.
+
+        """
+        # This code is the same as Python3's parse_qsl()
+        # (at https://hg.python.org/cpython/rev/c38ac7ab8d9a)
+        # except for the unquote(s, encoding, errors) calls replaced
+        # with unquote_to_bytes(s)
+        qs, _coerce_result = _coerce_args(qs)
+        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
+        r = []
+        for name_value in pairs:
+            if not name_value and not strict_parsing:
+                continue
+            nv = name_value.split('=', 1)
+            if len(nv) != 2:
+                if strict_parsing:
+                    raise ValueError("bad query field: %r" % (name_value,))
+                # Handle case of a control-name with no equal sign
+                if keep_blank_values:
+                    nv.append('')
+                else:
+                    continue
+            if len(nv[1]) or keep_blank_values:
+                name = nv[0].replace('+', ' ')
+                name = unquote_to_bytes(name)
+                name = _coerce_result(name)
+                value = nv[1].replace('+', ' ')
+                value = unquote_to_bytes(value)
+                value = _coerce_result(value)
+                r.append((name, value))
+        return r
 
 
 def escape_ajax(url):
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 73ad11f8a..88237d47a 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -2,10 +2,12 @@
 import unittest
 
 import six
+from six.moves.urllib.parse import urlparse
+
 from scrapy.spiders import Spider
 from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
                               canonicalize_url, add_http_if_no_scheme,
-                              guess_scheme)
+                              guess_scheme, parse_url)
 
 __doctests__ = ['scrapy.utils.url']
 
@@ -123,16 +125,55 @@ class CanonicalizeUrlTest(unittest.TestCase):
         self.assertEqual(canonicalize_url("http://www.example.com/do?q=a%20space&a=1"),
                                           "http://www.example.com/do?a=1&q=a+space")
 
-    @unittest.skipUnless(six.PY2, "TODO")
-    def test_normalize_percent_encoding_in_paths(self):
-        self.assertEqual(canonicalize_url("http://www.example.com/a%a3do"),
-                                          "http://www.example.com/a%A3do"),
+    def test_canonicalize_url_unicode_path(self):
+        self.assertEqual(canonicalize_url(u"http://www.example.com/résumé"),
+                                          "http://www.example.com/r%C3%A9sum%C3%A9")
+
+    def test_canonicalize_url_unicode_query_string(self):
+        # default encoding for path and query is UTF-8
+        self.assertEqual(canonicalize_url(u"http://www.example.com/résumé?q=résumé"),
+                                          "http://www.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9")
+
+        # passed encoding will affect query string
+        self.assertEqual(canonicalize_url(u"http://www.example.com/résumé?q=résumé", encoding='latin1'),
+                                          "http://www.example.com/r%C3%A9sum%C3%A9?q=r%E9sum%E9")
+
+        self.assertEqual(canonicalize_url(u"http://www.example.com/résumé?country=Россия", encoding='cp1251'),
+                                          "http://www.example.com/r%C3%A9sum%C3%A9?country=%D0%EE%F1%F1%E8%FF")
+
+    def test_canonicalize_url_unicode_query_string_wrong_encoding(self):
+        # trying to encode with wrong encoding
+        # fallback to UTF-8
+        self.assertEqual(canonicalize_url(u"http://www.example.com/résumé?currency=€", encoding='latin1'),
+                                          "http://www.example.com/r%C3%A9sum%C3%A9?currency=%E2%82%AC")
+
+        self.assertEqual(canonicalize_url(u"http://www.example.com/résumé?country=Россия", encoding='latin1'),
+                                          "http://www.example.com/r%C3%A9sum%C3%A9?country=%D0%A0%D0%BE%D1%81%D1%81%D0%B8%D1%8F")
+
+    def test_normalize_percent_encoding_in_paths(self):
+        self.assertEqual(canonicalize_url("http://www.example.com/r%c3%a9sum%c3%a9"),
+                                          "http://www.example.com/r%C3%A9sum%C3%A9")
+
+        # non-UTF8 encoded sequences: they should be kept untouched, only upper-cased
+        # 'latin1'-encoded sequence in path
+        self.assertEqual(canonicalize_url("http://www.example.com/a%a3do"),
+                                          "http://www.example.com/a%A3do")
+
+        # 'latin1'-encoded path, UTF-8 encoded query string
+        self.assertEqual(canonicalize_url("http://www.example.com/a%a3do?q=r%c3%a9sum%c3%a9"),
+                                          "http://www.example.com/a%A3do?q=r%C3%A9sum%C3%A9")
+
+        # 'latin1'-encoded path and query string
+        self.assertEqual(canonicalize_url("http://www.example.com/a%a3do?q=r%e9sum%e9"),
+                                          "http://www.example.com/a%A3do?q=r%E9sum%E9")
 
-    @unittest.skipUnless(six.PY2, "TODO")
     def test_normalize_percent_encoding_in_query_arguments(self):
         self.assertEqual(canonicalize_url("http://www.example.com/do?k=b%a3"),
                                           "http://www.example.com/do?k=b%A3")
 
+        self.assertEqual(canonicalize_url("http://www.example.com/do?k=r%c3%a9sum%c3%a9"),
+                                          "http://www.example.com/do?k=r%C3%A9sum%C3%A9")
+
     def test_non_ascii_percent_encoding_in_paths(self):
         self.assertEqual(canonicalize_url("http://www.example.com/a do?a=1"),
                                           "http://www.example.com/a%20do?a=1"),
@@ -144,7 +185,7 @@ class CanonicalizeUrlTest(unittest.TestCase):
                                           "http://www.example.com/a%20do%C2%A3.html?a=1")
 
     def test_non_ascii_percent_encoding_in_query_arguments(self):
-        self.assertEqual(canonicalize_url(u"http://www.example.com/do?price=\xa3500&a=5&z=3"),
+        self.assertEqual(canonicalize_url(u"http://www.example.com/do?price=£500&a=5&z=3"),
                                           u"http://www.example.com/do?a=5&price=%C2%A3500&z=3")
         self.assertEqual(canonicalize_url(b"http://www.example.com/do?price=\xc2\xa3500&a=5&z=3"),
                                           "http://www.example.com/do?a=5&price=%C2%A3500&z=3")
@@ -167,7 +208,6 @@ class CanonicalizeUrlTest(unittest.TestCase):
             "http://www.simplybedrooms.com/White-Bedroom-Furniture/Bedroom-Mirror:-Josephine-Cheval-Mirror.html"),
             "http://www.simplybedrooms.com/White-Bedroom-Furniture/Bedroom-Mirror:-Josephine-Cheval-Mirror.html")
 
-    @unittest.skipUnless(six.PY2, "TODO")
     def test_safe_characters_unicode(self):
         # urllib.quote uses a mapping cache of encoded characters. when parsing
         # an already percent-encoded url, it will fail if that url was not
@@ -181,12 +221,37 @@ class CanonicalizeUrlTest(unittest.TestCase):
         self.assertEqual(canonicalize_url("http://www.EXAMPLE.com/"),
                                           "http://www.example.com/")
 
+    def test_canonicalize_idns(self):
+        self.assertEqual(canonicalize_url(u'http://www.bücher.de?q=bücher'),
+                                           'http://www.xn--bcher-kva.de/?q=b%C3%BCcher')
+        # Japanese (+ reordering query parameters)
+        self.assertEqual(canonicalize_url(u'http://はじめよう.みんな/?query=サ&maxResults=5'),
+                                           'http://xn--p8j9a0d9c9a.xn--q9jyb4c/?maxResults=5&query=%E3%82%B5')
+
     def test_quoted_slash_and_question_sign(self):
         self.assertEqual(canonicalize_url("http://foo.com/AC%2FDC+rocks%3f/?yeah=1"),
                          "http://foo.com/AC%2FDC+rocks%3F/?yeah=1")
         self.assertEqual(canonicalize_url("http://foo.com/AC%2FDC/"),
                          "http://foo.com/AC%2FDC/")
 
+    def test_canonicalize_urlparsed(self):
+        # canonicalize_url() can be passed an already urlparse'd URL
+        self.assertEqual(canonicalize_url(urlparse(u"http://www.example.com/résumé?q=résumé")),
+                                          "http://www.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9")
+        self.assertEqual(canonicalize_url(urlparse('http://www.example.com/caf%e9-con-leche.htm')),
+                                          'http://www.example.com/caf%E9-con-leche.htm')
+        self.assertEqual(canonicalize_url(urlparse("http://www.example.com/a%a3do?q=r%c3%a9sum%c3%a9")),
+                                          "http://www.example.com/a%A3do?q=r%C3%A9sum%C3%A9")
+
+    def test_canonicalize_parse_url(self):
+        # parse_url() wraps urlparse and is used in link extractors
+        self.assertEqual(canonicalize_url(parse_url(u"http://www.example.com/résumé?q=résumé")),
+                                          "http://www.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9")
+        self.assertEqual(canonicalize_url(parse_url('http://www.example.com/caf%e9-con-leche.htm')),
+                                          'http://www.example.com/caf%E9-con-leche.htm')
+        self.assertEqual(canonicalize_url(parse_url("http://www.example.com/a%a3do?q=r%c3%a9sum%c3%a9")),
+                                          "http://www.example.com/a%A3do?q=r%C3%A9sum%C3%A9")
+
 
 class AddHttpIfNoScheme(unittest.TestCase):
 

From 8efa98794dd0343ebb605b715c320b0a1523a7fa Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 21 Apr 2016 16:51:17 +0200
Subject: [PATCH 0937/2974] Allow more pre-releases with bumpversion

---
 .bumpversion.cfg | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 39a1c6963..49a7e239f 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -24,4 +24,6 @@ values =
 	1
 	2
 	3
+	4
+	5
 

From 19a4a0adf953d2a3d428411b6b0306997276238c Mon Sep 17 00:00:00 2001
From: Patrick Connolly <patrick.c.connolly@gmail.com>
Date: Sun, 24 Apr 2016 14:55:08 -0400
Subject: [PATCH 0938/2974] Place brackets on own lines with JsonItemExporter.

Placing the opening and closing brackets on their own lines makes it slightly easier to sort lines after the `spider_closed` signal is fired.
---
 scrapy/exporters.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 360007c0f..d507bcf31 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -100,10 +100,10 @@ class JsonItemExporter(BaseItemExporter):
         self.first_item = True
 
     def start_exporting(self):
-        self.file.write(b"[")
+        self.file.write(b"[\n")
 
     def finish_exporting(self):
-        self.file.write(b"]")
+        self.file.write(b"\n]")
 
     def export_item(self, item):
         if self.first_item:

From 25401fd322b6c8f74d618e998fa675ea5eb494b8 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 26 Apr 2016 15:12:00 +0200
Subject: [PATCH 0939/2974] Use six.PY2 instead of six.PY3 for Python version
 variations

Also don't test passed encoding against 'utf8';
Just consider that if encoding failed, it must have been another encoding.
---
 scrapy/utils/url.py | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 716513648..069666dbc 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -83,17 +83,16 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False,
     try:
         scheme, netloc, path, params, query, fragment = _safe_ParseResult(
             parse_url(url), encoding=encoding)
-    except UnicodeError as e:
-        if encoding != 'utf8':
-            scheme, netloc, path, params, query, fragment = _safe_ParseResult(
-                parse_url(url), encoding='utf8')
-        else:
-            raise
+    except UnicodeEncodeError as e:
+        scheme, netloc, path, params, query, fragment = _safe_ParseResult(
+            parse_url(url), encoding='utf8')
 
     # 1. decode query-string as UTF-8 (or keep raw bytes),
     #    sort values,
     #    and percent-encode them back
-    if not six.PY2:
+    if six.PY2:
+        keyvals = parse_qsl(query, keep_blank_values)
+    else:
         # Python3's urllib.parse.parse_qsl does not work as wanted
         # for percent-encoded characters that do not match passed encoding,
         # they get lost.
@@ -118,8 +117,6 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False,
         # IRIs (namely, to be able to include non-ASCII characters) can only be
         # used if the query part is encoded in UTF-8.
         keyvals = parse_qsl_to_bytes(query, keep_blank_values)
-    else:
-        keyvals = parse_qsl(query, keep_blank_values)
     keyvals.sort()
     query = urlencode(keyvals)
 
@@ -138,16 +135,17 @@ def _unquotepath(path):
     for reserved in ('2f', '2F', '3f', '3F'):
         path = path.replace('%' + reserved, '%25' + reserved.upper())
 
-    if six.PY3:
-        # standard lib's unquote() does not work in Python 3
-        # for non-UTF-8 percent-escaped characters, they get lost.
+    if six.PY2:
+        # in Python 2, '%a3' becomes '\xa3', which is what we want
+        return unquote(path)
+    else:
+        # in Python 3,
+        # standard lib's unquote() does not work for non-UTF-8
+        # percent-escaped characters, they get lost.
         # e.g., '%a3' becomes 'REPLACEMENT CHARACTER' (U+FFFD)
         #
         # unquote_to_bytes() returns raw bytes instead
         return unquote_to_bytes(path)
-    else:
-        # in Python 2, '%a3' becomes '\xa3', which is what we want
-        return unquote(path)
 
 
 def parse_url(url, encoding=None):

From efbe75ea5ef862e44ccee6b9410a8317f9fad329 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 26 Apr 2016 16:14:15 +0200
Subject: [PATCH 0940/2974] Use six.PY2 also for conditional imports

---
 scrapy/utils/url.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 069666dbc..c80fc6e70 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -11,7 +11,7 @@ import six
 from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag,
                                     urlparse, parse_qsl, urlencode,
                                     quote, unquote)
-if six.PY3:
+if not six.PY2:
     from urllib.parse import unquote_to_bytes
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this
@@ -157,7 +157,7 @@ def parse_url(url, encoding=None):
     return urlparse(to_unicode(url, encoding))
 
 
-if six.PY3:
+if not six.PY2:
     from urllib.parse import _coerce_args, unquote_to_bytes
 
     def parse_qsl_to_bytes(qs, keep_blank_values=False, strict_parsing=False):

From 0e11b3e6f021b96dd97f84b465583a32a8663420 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 26 Apr 2016 20:03:17 +0200
Subject: [PATCH 0941/2974] Add idempotence tests for canonicalize_url

---
 tests/test_utils_url.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 88237d47a..1fc3a3510 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -252,6 +252,19 @@ class CanonicalizeUrlTest(unittest.TestCase):
         self.assertEqual(canonicalize_url(parse_url("http://www.example.com/a%a3do?q=r%c3%a9sum%c3%a9")),
                                           "http://www.example.com/a%A3do?q=r%C3%A9sum%C3%A9")
 
+    def test_canonicalize_url_idempotence(self):
+        for url, enc in [(u'http://www.bücher.de/résumé?q=résumé', 'utf8'),
+                         (u'http://www.example.com/résumé?q=résumé', 'latin1'),
+                         (u'http://www.example.com/résumé?country=Россия', 'cp1251'),
+                         (u'http://はじめよう.みんな/?query=サ&maxResults=5', 'iso2022jp')]:
+            canonicalized = canonicalize_url(url, encoding=enc)
+
+            # if we canonicalize again, we ge the same result
+            self.assertEqual(canonicalize_url(canonicalized, encoding=enc), canonicalized)
+
+            # without encoding, already canonicalized URL is canonicalized identically
+            self.assertEqual(canonicalize_url(canonicalized), canonicalized)
+
 
 class AddHttpIfNoScheme(unittest.TestCase):
 

From dc9de7524fceacb7fe1b46c48012fea6a7459e55 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 28 Apr 2016 15:28:28 +0200
Subject: [PATCH 0942/2974] Update release notes with recent features and bug
 fixes.

---
 docs/news.rst | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index f515e5bb1..2cc190781 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -50,6 +50,11 @@ This 1.1 release brings a lot of interesting features and bug fixes:
     ``ImagesPipeline``), the default ACL policy is now "private" instead
     of "public" **Warning: backwards incompatible!**.
     You can use :setting:`FILES_STORE_S3_ACL` to change it.
+  - We've reimplemented ``canonicalize_url()`` for more correct output,
+    especially for URLs with non-ASCII characters (:issue:`1947`).
+    This could change link extractors output compared to previous scrapy versions.
+    This may also invalidate some cache entries you could still have from pre-1.1 runs.
+    **Warning: backwards incompatible!**.
 
 Keep reading for more details on other improvements and bug fixes.
 
@@ -72,7 +77,6 @@ some limitations in Python 3:
 - FTP download handler is not supported (non-Python 3 ported Twisted
   dependency)
 - Telnet is not supported (non-Python 3 ported Twisted dependency)
-- Scrapy has problems handling non-ASCII URLs in Python 3
 
 Additional New Features and Enhancements
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -112,11 +116,18 @@ Additional New Features and Enhancements
   setting is set (:issue:`1723`, :issue:`1725`).
 - Added method ``ExecutionEngine.close`` (:issue:`1423`).
 - Added method ``CrawlerRunner.create_crawler`` (:issue:`1528`).
+- Scheduler priority queue can now be customized via
+  :setting:`SCHEDULER_PRIORITY_QUEUE` (:issue:`1822`).
+- ``.pps`` links are now ignored by default in link extractors (:issue:`1835`).
+- temporary data folder for FTP and S3 feed storages can be customized
+  using a new :setting:`FEED_TEMPDIR` setting (:issue:`1847`).
+- ``FilesPipeline`` and ``ImagesPipeline`` settings are now instance attributes
+  instead of class attributes, enabling spider-specific behaviors (:issue:`1891`).
 - Tons of documentation updates and related fixes (:issue:`1291`, :issue:`1302`,
   :issue:`1335`, :issue:`1683`, :issue:`1660`, :issue:`1642`, :issue:`1721`,
-  :issue:`1727`).
+  :issue:`1727`, :issue:`1879`).
 - Other refactoring, optimizations and cleanup (:issue:`1476`, :issue:`1481`,
-  :issue:`1477`, :issue:`1315`, :issue:`1290` and :issue:`1750`).
+  :issue:`1477`, :issue:`1315`, :issue:`1290`, :issue:`1750`, :issue:`1881`).
 
 .. _`Code of Conduct`: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
 
@@ -126,6 +137,8 @@ Deprecations and Removals
 
 - Added ``to_bytes`` and ``to_unicode``, deprecated ``str_to_unicode`` and
   ``unicode_to_str`` functions (:issue:`778`).
+- ``binary_is_text`` is introduced, to replace use of ``isbinarytext``
+  (but with inverse return value) (:issue:`1851`)
 - The ``optional_features`` set has been removed (:issue:`1359`).
 - The ``--lsprof`` command line option has been removed (:issue:`1689`).
   **Warning: backward incompatible**, but doesn't break user code.
@@ -171,6 +184,12 @@ Bugfixes
 - Various logging related fixes (:issue:`1294`, :issue:`1419`, :issue:`1263`,
   :issue:`1624`, :issue:`1654`, :issue:`1722`, :issue:`1726` and :issue:`1303`).
 - Fixed bug in ``utils.template.render_templatefile()`` (:issue:`1212`).
+- Fixed bug with filestorage HTTP cache checking wrong modified time (:issue:`1875`).
+- ``RetryMiddleware`` is now robust to non-standard HTTP status codes
+  (:issue:`1857`).
+- sitemaps extraction from ``robots.txt`` is now case-insensitive (:issue:`1902`).
+- HTTPS+CONNECT tunnels could get mixed up when using multiple proxies
+  to same remote host (:issue:`1912`).
 
 
 1.0.5 (2016-02-04)

From cb38f3896b4886dbeb1a74ab457196a77e388790 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 28 Apr 2016 17:54:37 +0200
Subject: [PATCH 0943/2974] Add notes on botocore support and JsonItemExporter
 opening/closing brackets

---
 docs/news.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 2cc190781..0482796b4 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -123,6 +123,10 @@ Additional New Features and Enhancements
   using a new :setting:`FEED_TEMPDIR` setting (:issue:`1847`).
 - ``FilesPipeline`` and ``ImagesPipeline`` settings are now instance attributes
   instead of class attributes, enabling spider-specific behaviors (:issue:`1891`).
+- ``JsonItemExporter`` now formats opening and closing square brackets
+  on their own line (first and last lines of output file) (:issue:`1950`).
+- If available, ``botocore`` is used for ``S3FeedStorage``, ``S3DownloadHandler``
+  and ``S3FilesStore`` (:issue:`1761`, :issue:`1883`).
 - Tons of documentation updates and related fixes (:issue:`1291`, :issue:`1302`,
   :issue:`1335`, :issue:`1683`, :issue:`1660`, :issue:`1642`, :issue:`1721`,
   :issue:`1727`, :issue:`1879`).

From c8bff2375161c4934212b6cedbfb866c5d909fc8 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 29 Apr 2016 10:29:37 +0200
Subject: [PATCH 0944/2974] Bump w3lib version dependency in setup.py

---
 requirements-py3.txt | 2 +-
 setup.py             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements-py3.txt b/requirements-py3.txt
index 065095101..08ccf1958 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -3,5 +3,5 @@ lxml>=3.2.4
 pyOpenSSL>=0.13.1
 cssselect>=0.9
 queuelib>=1.1.1
-w3lib>=1.8.0
+w3lib>=1.14.2
 service_identity
diff --git a/setup.py b/setup.py
index 8387f81f4..000d84614 100644
--- a/setup.py
+++ b/setup.py
@@ -42,7 +42,7 @@ setup(
     ],
     install_requires=[
         'Twisted>=10.0.0',
-        'w3lib>=1.13.0',
+        'w3lib>=1.14.2',
         'queuelib',
         'lxml',
         'pyOpenSSL',

From 334ee40e3de7f93b9e178ecc4310224d0f4fe022 Mon Sep 17 00:00:00 2001
From: Pengyu CHEN <pengyu@libstarrify.so>
Date: Mon, 9 May 2016 20:43:41 +0800
Subject: [PATCH 0945/2974] Added: Retrying
 scrapy.core.downloader.handlers.http11.TunnelError

---
 scrapy/downloadermiddlewares/retry.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 3324aa21a..74938067f 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -21,6 +21,7 @@ from twisted.internet.error import TimeoutError, DNSLookupError, \
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.response import response_status_message
 from scrapy.xlib.tx import ResponseFailed
+from scrapy.core.downloader.handlers.http11 import TunnelError
 
 logger = logging.getLogger(__name__)
 
@@ -32,7 +33,7 @@ class RetryMiddleware(object):
     EXCEPTIONS_TO_RETRY = (defer.TimeoutError, TimeoutError, DNSLookupError,
                            ConnectionRefusedError, ConnectionDone, ConnectError,
                            ConnectionLost, TCPTimedOutError, ResponseFailed,
-                           IOError)
+                           IOError, TunnelError)
 
     def __init__(self, settings):
         if not settings.getbool('RETRY_ENABLED'):

From 3f8be374c3bc1630b355bfc9a37452f2fd4ffd40 Mon Sep 17 00:00:00 2001
From: Panayiotis Lipiridis <lipiridis@gmail.com>
Date: Mon, 9 May 2016 16:15:18 +0200
Subject: [PATCH 0946/2974] Github >> GitHub

---
 docs/contributing.rst     | 4 ++--
 docs/topics/selectors.rst | 2 +-
 docs/topics/ubuntu.rst    | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index f1f4c63ae..4e8330b3c 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -86,7 +86,7 @@ Well-written patches should:
 Submitting patches
 ==================
 
-The best way to submit a patch is to issue a `pull request`_ on Github,
+The best way to submit a patch is to issue a `pull request`_ on GitHub,
 optionally creating a new issue first.
 
 Remember to explain what was fixed or the new functionality (what it is, why
@@ -96,7 +96,7 @@ developers to understand and accept your patch.
 You can also discuss the new functionality (or bug fix) before creating the
 patch, but it's always good to have a patch ready to illustrate your arguments
 and show that you have put some additional thought into the subject. A good
-starting point is to send a pull request on Github. It can be simple enough to
+starting point is to send a pull request on GitHub. It can be simple enough to
 illustrate your idea, and leave documentation/tests for later, after the idea
 has been validated and proven useful. Alternatively, you can send an email to
 `scrapy-users`_ to discuss your idea first.
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index d1845b20c..39ec9b73c 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -736,7 +736,7 @@ namespaces altogether and just work with element names, to write more
 simple/convenient XPaths. You can use the
 :meth:`Selector.remove_namespaces` method for that.
 
-Let's show an example that illustrates this with Github blog atom feed.
+Let's show an example that illustrates this with GitHub blog atom feed.
 
 .. highlight:: sh
 
diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst
index dbbcaacfc..102df581c 100644
--- a/docs/topics/ubuntu.rst
+++ b/docs/topics/ubuntu.rst
@@ -8,7 +8,7 @@ Ubuntu packages
 
 `Scrapinghub`_ publishes apt-gettable packages which are generally fresher than
 those in Ubuntu, and more stable too since they're continuously built from
-`Github repo`_ (master & stable branches) and so they contain the latest bug
+`GitHub repo`_ (master & stable branches) and so they contain the latest bug
 fixes.
 
 To use the packages:
@@ -33,4 +33,4 @@ To use the packages:
    repositories, it's very outdated and it isn't supported by Scrapy team.
 
 .. _Scrapinghub: http://scrapinghub.com/
-.. _Github repo: https://github.com/scrapy/scrapy
+.. _GitHub repo: https://github.com/scrapy/scrapy

From 137197b98b1d3e94a1be88db48abbf537d90f6e4 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 11 May 2016 13:12:35 +0200
Subject: [PATCH 0947/2974] Prevent empty warnings in case of certificate
 verification failures

Fixes #1976
---
 scrapy/core/downloader/tls.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 2cf8ba869..5fa8e2723 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -44,7 +44,9 @@ try:
                 try:
                     verifyHostname(connection, self._hostnameASCII)
                 except VerificationError as e:
-                    logger.warning(e)
+                    logger.warning(
+                        'Remote certificate is not valid for hostname "{}"; {}'.format(
+                            self._hostnameASCII, e))
 
 except ImportError:
     # ImportError should not matter for older Twisted versions

From d62654dfda6d0c45fe39dfe5a73c3f0b3371c02d Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 11 May 2016 14:39:55 +0200
Subject: [PATCH 0948/2974] Merge 1.0.6 release notes

---
 docs/news.rst | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 0482796b4..eb506efce 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -188,14 +188,20 @@ Bugfixes
 - Various logging related fixes (:issue:`1294`, :issue:`1419`, :issue:`1263`,
   :issue:`1624`, :issue:`1654`, :issue:`1722`, :issue:`1726` and :issue:`1303`).
 - Fixed bug in ``utils.template.render_templatefile()`` (:issue:`1212`).
-- Fixed bug with filestorage HTTP cache checking wrong modified time (:issue:`1875`).
-- ``RetryMiddleware`` is now robust to non-standard HTTP status codes
-  (:issue:`1857`).
 - sitemaps extraction from ``robots.txt`` is now case-insensitive (:issue:`1902`).
 - HTTPS+CONNECT tunnels could get mixed up when using multiple proxies
   to same remote host (:issue:`1912`).
 
 
+1.0.6 (2016-05-04)
+------------------
+
+- FIX: RetryMiddleware is now robust to non-standard HTTP status codes (:issue:`1857`)
+- FIX: Filestorage HTTP cache was checking wrong modified time (:issue:`1875`)
+- DOC: Support for Sphinx 1.4+ (:issue:`1893`)
+- DOC: Consistency in selectors examples (:issue:`1869`)
+
+
 1.0.5 (2016-02-04)
 ------------------
 

From aecc23d24a801f8b7c50497bc37a0c0a2e37e173 Mon Sep 17 00:00:00 2001
From: mgachhui <mgachhui@gmail.com>
Date: Sun, 8 May 2016 18:39:56 +0530
Subject: [PATCH 0949/2974] Added Python 3.3+ to readme

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 3e050bb1e..a0138970a 100644
--- a/README.rst
+++ b/README.rst
@@ -40,7 +40,7 @@ http://scrapy.org
 Requirements
 ============
 
-* Python 2.7
+* Python 2.7 or Python 3.3+
 * Works on Linux, Windows, Mac OSX, BSD
 
 Install

From 0700e437f227a3b060980d99da1c039895482c1f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 11 May 2016 17:31:50 +0200
Subject: [PATCH 0950/2974] Update release notes and date for 1.1.0

---
 docs/news.rst | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index eb506efce..cbcd4d613 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,8 +3,8 @@
 Release notes
 =============
 
-1.1.0
------
+1.1.0 (2016-05-11)
+------------------
 
 This 1.1 release brings a lot of interesting features and bug fixes:
 
@@ -69,14 +69,13 @@ you can run spiders on Python 3.3, 3.4 and 3.5 (Twisted >= 15.5 required). Some
 features are still missing (and some may never be ported).
 
 
-Almost all builtin extensions/middlewares are expected to work. However, we are aware of
-some limitations in Python 3:
+Almost all builtin extensions/middlewares are expected to work.
+However, we are aware of some limitations in Python 3:
 
-- Scrapy doesn't work yet in Windows with Python 3 (non-Python 3 ported Twisted dependency)
-- Sending emails is not supported (non-Python 3 ported Twisted dependency)
-- FTP download handler is not supported (non-Python 3 ported Twisted
-  dependency)
-- Telnet is not supported (non-Python 3 ported Twisted dependency)
+- Scrapy has not been tested on Windows with Python 3
+- Sending emails is not supported
+- FTP download handler is not supported
+- Telnet console is not supported
 
 Additional New Features and Enhancements
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

From 4cef1a1d0060148716ffec4ad6ce9bf709ad1ea2 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Fri, 13 May 2016 12:35:35 +0200
Subject: [PATCH 0951/2974] [image_pipeline] bring back uppercase pipeline
 attributes

allow users to have class attributes on image pipelines. This assumes
that class attributes are useful if users want to have different pipeline
classes inhriting from ImagePipeline.
---
 scrapy/pipelines/files.py           |  9 ++++++---
 scrapy/pipelines/images.py          | 21 +++++++++++++--------
 scrapy/settings/default_settings.py |  7 -------
 tests/test_pipeline_images.py       | 12 ++++++++----
 4 files changed, 27 insertions(+), 22 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index c94794173..3e6ad554d 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -214,11 +214,14 @@ class FilesPipeline(MediaPipeline):
     """
 
     MEDIA_NAME = "file"
+    EXPIRES = 90
     STORE_SCHEMES = {
         '': FSFilesStore,
         'file': FSFilesStore,
         's3': S3FilesStore,
     }
+    DEFAULT_FILES_URLS_FIELD = 'file_urls'
+    DEFAULT_FILES_RESULT_FIELD = 'files'
 
     def __init__(self, store_uri, download_func=None, settings=None):
         if not store_uri:
@@ -228,9 +231,9 @@ class FilesPipeline(MediaPipeline):
             settings = Settings(settings)
         
         self.store = self._get_store(store_uri)
-        self.expires = settings.getint('FILES_EXPIRES')
-        self.files_urls_field = settings.get('FILES_URLS_FIELD')
-        self.files_result_field = settings.get('FILES_RESULT_FIELD')
+        self.expires = settings.getint('FILES_EXPIRES', self.EXPIRES)
+        self.files_urls_field = settings.get('FILES_URLS_FIELD', self.DEFAULT_FILES_URLS_FIELD)
+        self.files_result_field = settings.get('FILES_RESULT_FIELD', self.DEFAULT_FILES_RESULT_FIELD)
 
         super(FilesPipeline, self).__init__(download_func=download_func)
 
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index c597b6cca..ac78ee6c0 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -37,26 +37,31 @@ class ImagesPipeline(FilesPipeline):
     """
 
     MEDIA_NAME = 'image'
+    MIN_WIDTH = 0
+    MIN_HEIGHT = 0
+    EXPIRES = 0
+    THUMBS = {}
+    IMAGES_URLS_FIELD = 'image_urls'
+    IMAGES_RESULT_FIELD = 'images'
 
     def __init__(self, store_uri, download_func=None, settings=None):
         super(ImagesPipeline, self).__init__(store_uri, settings=settings, download_func=download_func)
-        
+
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
 
-        self.expires = settings.getint('IMAGES_EXPIRES')
-        self.images_urls_field = settings.get('IMAGES_URLS_FIELD')
-        self.images_result_field = settings.get('IMAGES_RESULT_FIELD')
-        self.min_width = settings.getint('IMAGES_MIN_WIDTH')
-        self.min_height = settings.getint('IMAGES_MIN_HEIGHT')
-        self.thumbs = settings.get('IMAGES_THUMBS')
+        self.expires = settings.getint('IMAGES_EXPIRES', self.EXPIRES)
+        self.images_urls_field = settings.get('IMAGES_URLS_FIELD', self.IMAGES_URLS_FIELD)
+        self.images_result_field = settings.get('IMAGES_RESULT_FIELD', self.IMAGES_RESULT_FIELD)
+        self.min_width = settings.getint('IMAGES_MIN_WIDTH', self.MIN_WIDTH)
+        self.min_height = settings.getint('IMAGES_MIN_HEIGHT', self.MIN_HEIGHT)
+        self.thumbs = settings.get('IMAGES_THUMBS', self.THUMBS)
 
     @classmethod
     def from_settings(cls, settings):
         s3store = cls.STORE_SCHEMES['s3']
         s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
-
         store_uri = settings['IMAGES_STORE']
         return cls(store_uri, settings=settings)
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8f064f81e..1e447e9e2 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -179,13 +179,6 @@ HTTPCACHE_GZIP = False
 
 HTTPPROXY_AUTH_ENCODING = 'latin-1'
 
-IMAGES_MIN_WIDTH = 0
-IMAGES_MIN_HEIGHT = 0
-IMAGES_EXPIRES = 90
-IMAGES_THUMBS = {}
-IMAGES_URLS_FIELD = 'image_urls'
-IMAGES_RESULT_FIELD = 'images'
-
 ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
 
 ITEM_PIPELINES = {}
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index f48547b0f..3ce138000 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -224,19 +224,22 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
     def test_images_urls_field(self):
         another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
                                                                 'IMAGES_URLS_FIELD': 'funny_field'}))
-        self.assertEqual(self.pipeline.images_urls_field, self.default_settings.get('IMAGES_URLS_FIELD'))
+        default = self.pipeline.IMAGES_URLS_FIELD
+        self.assertEqual(self.pipeline.images_urls_field, self.default_settings.get('IMAGES_URLS_FIELD', default))
         self.assertEqual(another_pipeline.images_urls_field, 'funny_field')
 
     def test_images_result_field(self):
         another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
                                                                 'IMAGES_RESULT_FIELD': 'funny_field'}))
-        self.assertEqual(self.pipeline.images_result_field, self.default_settings.get('IMAGES_RESULT_FIELD'))
+        default = self.pipeline.IMAGES_RESULT_FIELD
+        self.assertEqual(self.pipeline.images_result_field, self.default_settings.get('IMAGES_RESULT_FIELD', default))
         self.assertEqual(another_pipeline.images_result_field, 'funny_field')
 
     def test_min_width(self):
         another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
                                                                 'IMAGES_MIN_WIDTH': 42}))
-        self.assertEqual(self.pipeline.min_width, self.default_settings.getint('IMAGES_MIN_WIDTH'))
+        default = self.pipeline.MIN_WIDTH
+        self.assertEqual(self.pipeline.min_width, self.default_settings.getint('IMAGES_MIN_WIDTH', default))
         self.assertEqual(another_pipeline.min_width, 42)
 
     def test_min_height(self):
@@ -249,7 +252,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
         custom_thumbs = {'small': (50, 50), 'big': (270, 270)}
         another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
                                                                 'IMAGES_THUMBS': custom_thumbs}))
-        self.assertEqual(self.pipeline.thumbs, self.default_settings.get('IMAGES_THUMBS'))
+        default = self.pipeline.THUMBS
+        self.assertEqual(self.pipeline.thumbs, self.default_settings.get('IMAGES_THUMBS', default))
         self.assertEqual(another_pipeline.thumbs, custom_thumbs)
 
 

From 9b3c72cb378bc0979f3acb6593f5bb4a39aa6a8a Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 17 May 2016 16:24:05 +0200
Subject: [PATCH 0952/2974] DOC Place FEED_TEMPDIR setting at lexicographical
 position

---
 docs/topics/settings.rst | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index f5a0b9637..7f49aacdb 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -178,15 +178,6 @@ Default: ``None``
 The AWS secret key used by code that requires access to `Amazon Web services`_,
 such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
 
-.. setting:: FEED_TEMPDIR
-
-FEED_TEMPDIR
-------------
-
-The Feed Temp dir allows you to set a custom folder to save crawler
-temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
-:ref:`Amazon S3 <topics-feed-storage-s3>`.
-
 .. setting:: BOT_NAME
 
 BOT_NAME
@@ -688,6 +679,16 @@ For more information See the :ref:`extensions user guide  <topics-extensions>`
 and the :ref:`list of available extensions <topics-extensions-ref>`.
 
 
+.. setting:: FEED_TEMPDIR
+
+FEED_TEMPDIR
+------------
+
+The Feed Temp dir allows you to set a custom folder to save crawler
+temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
+:ref:`Amazon S3 <topics-feed-storage-s3>`.
+
+
 .. setting:: FILES_STORE_S3_ACL
 
 FILES_STORE_S3_ACL

From 149c4cd41e28447d4ff83a291ea6ac4c73dedc60 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 17 May 2016 17:53:40 +0200
Subject: [PATCH 0953/2974] [DOC][Overview] Use idiomatic .extract_first()

---
 docs/intro/overview.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 0673aed93..6c1291c1f 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -36,9 +36,9 @@ voted questions on StackOverflow and scrapes some data from each page::
 
         def parse_question(self, response):
             yield {
-                'title': response.css('h1 a::text').extract()[0],
-                'votes': response.css('.question .vote-count-post::text').extract()[0],
-                'body': response.css('.question .post-text').extract()[0],
+                'title': response.css('h1 a::text').extract_first(),
+                'votes': response.css('.question .vote-count-post::text').extract_first(),
+                'body': response.css('.question .post-text').extract_first(),
                 'tags': response.css('.question .post-tag::text').extract(),
                 'link': response.url,
             }

From 6c67db3917dd07761b113f3e3b37ebf5a9558f00 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Wed, 18 May 2016 12:04:52 +0200
Subject: [PATCH 0954/2974] [image_pipeline] tests for class attrs backward
 compatibility

and docs about image pipeline settings.
---
 docs/topics/media-pipeline.rst |  4 ++++
 tests/test_pipeline_images.py  | 23 +++++++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 3da243d29..2b41a4f43 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -191,6 +191,10 @@ For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/or
 If you need something more complex and want to override the custom pipeline
 behaviour, see :ref:`topics-media-pipeline-override`.
 
+.. note:: If you have multiple image pipelines and you want to have different settings in different pipelines
+   you can either set class attributes in pipeline object or add custom settings keys and load them at pipeline
+   initialization.
+
 
 Additional features
 ===================
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 3ce138000..49257e175 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -256,6 +256,29 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
         self.assertEqual(self.pipeline.thumbs, self.default_settings.get('IMAGES_THUMBS', default))
         self.assertEqual(another_pipeline.thumbs, custom_thumbs)
 
+    def test_class_attrs_preserved(self):
+
+        class UserDefinedImagePipeline(ImagesPipeline):
+            MIN_WIDTH = 1000
+
+        # If image settings are not defined values are taken from class attributes.
+        pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir}))
+        self.assertEqual(pipeline.min_width, 1000)
+
+    def test_class_attrs_not_preserved_if_settings_defined(self):
+
+        class UserDefinedImagePipeline(ImagesPipeline):
+            MIN_WIDTH = 1000
+
+        settings = {
+            "IMAGES_STORE": self.tempdir,
+            "IMAGES_MIN_WIDTH": 90
+        }
+
+        # If image settings are defined they override class attributes.
+        pipeline = UserDefinedImagePipeline.from_settings(Settings(settings))
+        self.assertEqual(pipeline.min_width, 90)
+
 
 def _create_image(format, *a, **kw):
     buf = TemporaryFile()

From c2c8036a97f0c3b18432638f50538019888ec1be Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 18 May 2016 16:52:09 +0200
Subject: [PATCH 0955/2974] DOC Update copyright notice

---
 docs/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index 2f9acc30a..640dcd7cb 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -45,7 +45,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Scrapy'
-copyright = u'2008-2015, Scrapy developers'
+copyright = u'2008-2016, Scrapy developers'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

From b3367c7acd8d004e1fdeda0c8258fd38299e7b73 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 18 May 2016 18:00:09 +0200
Subject: [PATCH 0956/2974] DOC Add info and example on errbacks

---
 docs/topics/request-response.rst | 65 ++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 7b48c4d1c..c54685d9e 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -117,6 +117,8 @@ Request objects
        raised while processing the request. This includes pages that failed
        with 404 HTTP errors and such. It receives a `Twisted Failure`_ instance
        as first parameter.
+       For more information,
+       see :ref:`topics-request-response-ref-errbacks` below.
     :type errback: callable
 
     .. attribute:: Request.url
@@ -212,6 +214,69 @@ different fields from different pages::
         item['other_url'] = response.url
         return item
 
+
+.. _topics-request-response-ref-errbacks:
+
+Using errbacks to catch exceptions in request processing
+--------------------------------------------------------
+
+The errback of a request is a function that will be called when an exception
+is raise while processing it.
+
+It receives a `Twisted Failure`_ instance as first parameter and can be
+used to track connection establishment timeouts, DNS errors etc.
+
+Here's an example spider logging all errors and catching some specific
+errors if needed::
+
+    import scrapy
+
+    from scrapy.spidermiddlewares.httperror import HttpError
+    from twisted.internet.error import DNSLookupError
+    from twisted.internet.error import TimeoutError, TCPTimedOutError
+
+    class ErrbackSpider(scrapy.Spider):
+        name = "errback_example"
+        start_urls = [
+            "http://www.httpbin.org/",              # HTTP 200 expected
+            "http://www.httpbin.org/status/404",    # Not found error
+            "http://www.httpbin.org/status/500",    # server issue
+            "http://www.httpbin.org:12345/",        # non-responding host, timeout expected
+            "http://www.httphttpbinbin.org/",       # DNS error expected
+        ]
+
+        def start_requests(self):
+            for u in self.start_urls:
+                yield scrapy.Request(u, callback=self.parse_httpbin,
+                                        errback=self.errback_httpbin,
+                                        dont_filter=True)
+
+        def parse_httpbin(self, response):
+            self.logger.info('Got successful response from {}'.format(response.url))
+            # do something useful here...
+
+        def errback_httpbin(self, failure):
+            # log all failures
+            self.logger.error(repr(failure))
+
+            # in case you want to do something special for some errors,
+            # you may need the failure's type:
+
+            if failure.check(HttpError):
+                # these exceptions come from HttpError spider middleware
+                # you can get the non-200 response
+                response = failure.value.response
+                self.logger.error('HttpError on %s', response.url)
+
+            elif failure.check(DNSLookupError):
+                # this is the original request
+                request = failure.request
+                self.logger.error('DNSLookupError on %s', request.url)
+
+            elif failure.check(TimeoutError, TCPTimedOutError):
+                request = failure.request
+                self.logger.error('TimeoutError on %s', request.url)
+
 .. _topics-request-meta:
 
 Request.meta special keys

From 1bc9d35a878a4028b4dd802707c7b513d4c3ef86 Mon Sep 17 00:00:00 2001
From: Matvei Nazaruk <matvei.nazaruk@gmail.com>
Date: Thu, 19 May 2016 22:24:37 +0300
Subject: [PATCH 0957/2974] Fixed choosing of response class based on body.

---
 scrapy/core/downloader/handlers/http11.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 88c6b9480..348b39e28 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -343,7 +343,7 @@ class ScrapyAgent(object):
         txresponse, body, flags = result
         status = int(txresponse.code)
         headers = Headers(txresponse.headers.getAllRawHeaders())
-        respcls = responsetypes.from_args(headers=headers, url=url)
+        respcls = responsetypes.from_args(headers=headers, url=url, body=body)
         return respcls(url=url, status=status, headers=headers, body=body, flags=flags)
 
 

From ce48bae533cfaf093778481a5a599943afa15846 Mon Sep 17 00:00:00 2001
From: omer <omer.schliefer@picscout.com>
Date: Mon, 23 May 2016 19:52:38 +0300
Subject: [PATCH 0958/2974] Do not decode of head response

---
 tests/test_downloadermiddleware_httpcompression.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 2e6e47fef..9ef66ba1c 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -144,3 +144,13 @@ class HttpCompressionTest(TestCase):
         self.assertIs(newresponse, response)
         self.assertEqual(response.headers['Content-Encoding'], b'gzip')
         self.assertEqual(response.headers['Content-Type'], b'application/gzip')
+
+    def test_process_response_head_request_no_decode_required(self):
+        response = self._getresponse('gzip')
+        response.headers['Content-Type'] = 'application/gzip'
+        request = response.request
+        request.method = 'HEAD'
+        response = response.replace(body = None)
+        newresponse = self.mw.process_response(request, response, self.spider)
+        self.assertIs(newresponse, response)
+        self.assertEquals(response.body, '')

From 6beb4f0119619170edfb4579f28da22c9250e84b Mon Sep 17 00:00:00 2001
From: Felipe Ruhland <felipe.ruhland@gmail.com>
Date: Mon, 23 May 2016 23:14:33 -0300
Subject: [PATCH 0959/2974] Created project_dir optional parameter in
 startproject command line

---
 scrapy/commands/startproject.py | 66 +++++++++++++++++++++++++++++----
 1 file changed, 59 insertions(+), 7 deletions(-)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index b3c5011df..8f3243a2a 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,9 +1,10 @@
 from __future__ import print_function
 import re
+import os
 import string
 from importlib import import_module
 from os.path import join, exists, abspath
-from shutil import copytree, ignore_patterns, move
+from shutil import ignore_patterns, move, copy2, copystat
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -27,7 +28,7 @@ class Command(ScrapyCommand):
     default_settings = {'LOG_ENABLED': False}
 
     def syntax(self):
-        return "<project_name>"
+        return "<project_name> [project_dir]"
 
     def short_desc(self):
         return "Create new project"
@@ -51,20 +52,71 @@ class Command(ScrapyCommand):
             return True
         return False
 
+    def _copytree(self, src, dst, symlinks=False, ignore=None):
+        names = os.listdir(src)
+        if ignore is not None:
+            ignored_names = ignore(src, names)
+        else:
+            ignored_names = set()
+
+        if not os.path.exists(dst):
+            os.makedirs(dst)
+
+        errors = []
+        for name in names:
+            if name in ignored_names:
+                continue
+            srcname = os.path.join(src, name)
+            dstname = os.path.join(dst, name)
+            try:
+                if symlinks and os.path.islink(srcname):
+                    linkto = os.readlink(srcname)
+                    os.symlink(linkto, dstname)
+                elif os.path.isdir(srcname):
+                    self._copytree(srcname, dstname, symlinks, ignore)
+                else:
+                    # Will raise a SpecialFileError for unsupported file types
+                    copy2(srcname, dstname)
+            # catch the Error from the recursive copytree so that we can
+            # continue with other files
+            except EnvironmentError, err:
+                errors.extend(err.args[0])
+            except EnvironmentError, why:
+                errors.append((srcname, dstname, str(why)))
+        try:
+            copystat(src, dst)
+        except OSError, why:
+            if WindowsError is not None and isinstance(why, WindowsError):
+                # Copying file access times may fail on Windows
+                pass
+            else:
+                errors.append((src, dst, str(why)))
+        if errors:
+            raise EnvironmentError(errors)
+
     def run(self, args, opts):
-        if len(args) != 1:
+        if len(args) not in (1, 2):
             raise UsageError()
+
         project_name = args[0]
+        project_dir = args[0]
+
+        if len(args) == 2:
+            project_dir = args[1]
+            if exists(join(project_dir, 'scrapy.cfg')):
+                self.exitcode = 1
+                print('Error: scrapy.cfg already exists in %s' % abspath(project_dir))
+                return
 
         if not self._is_valid_name(project_name):
             self.exitcode = 1
             return
 
-        copytree(self.templates_dir, project_name, ignore=IGNORE)
-        move(join(project_name, 'module'), join(project_name, project_name))
+        self._copytree(self.templates_dir, abspath(project_dir), ignore=IGNORE)
+        move(join(project_dir, 'module'), join(project_dir, project_name))
         for paths in TEMPLATES_TO_RENDER:
             path = join(*paths)
-            tplfile = join(project_name,
+            tplfile = join(project_dir,
                 string.Template(path).substitute(project_name=project_name))
             render_templatefile(tplfile, project_name=project_name,
                 ProjectName=string_camelcase(project_name))
@@ -80,4 +132,4 @@ class Command(ScrapyCommand):
         _templates_base_dir = self.settings['TEMPLATES_DIR'] or \
             join(scrapy.__path__[0], 'templates')
         return join(_templates_base_dir, 'project')
-    
\ No newline at end of file
+    

From 2521f031d6db5c986ab88dc37c793513f771c3b1 Mon Sep 17 00:00:00 2001
From: Felipe Ruhland <felipe.ruhland@gmail.com>
Date: Mon, 23 May 2016 23:15:53 -0300
Subject: [PATCH 0960/2974] Created new tests for implementation

---
 tests/test_commands.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 2e47160d7..42cade849 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -73,6 +73,23 @@ class StartprojectTest(ProjectTest):
         self.assertEqual(1, self.call('startproject', 'wrong---project---name'))
         self.assertEqual(1, self.call('startproject', 'sys'))
 
+    def test_startproject_with_project_dir(self):
+        project_dir = mkdtemp()
+        self.assertEqual(0, self.call('startproject', self.project_name, project_dir))
+
+        assert exists(join(abspath(project_dir), 'scrapy.cfg'))
+        assert exists(join(abspath(project_dir), 'testproject'))
+        assert exists(join(join(abspath(project_dir), self.project_name), '__init__.py'))
+        assert exists(join(join(abspath(project_dir), self.project_name), 'items.py'))
+        assert exists(join(join(abspath(project_dir), self.project_name), 'pipelines.py'))
+        assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py'))
+        assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py'))
+
+        self.assertEqual(1, self.call('startproject', self.project_name, project_dir))
+        self.assertEqual(1, self.call('startproject', self.project_name + '2', project_dir))
+        self.assertEqual(1, self.call('startproject', 'wrong---project---name'))
+        self.assertEqual(1, self.call('startproject', 'sys'))
+
 
 class StartprojectTemplatesTest(ProjectTest):
 

From 089483aece181913bf19548725653462a52febdf Mon Sep 17 00:00:00 2001
From: Felipe Ruhland <felipe.ruhland@gmail.com>
Date: Mon, 23 May 2016 23:16:15 -0300
Subject: [PATCH 0961/2974] Updated docs for new option in command line

---
 docs/topics/commands.rst | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 9a40a2c29..8e1a0ad0e 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -103,13 +103,14 @@ Creating projects
 The first thing you typically do with the ``scrapy`` tool is create your Scrapy
 project::
 
-    scrapy startproject myproject
+    scrapy startproject myproject [project_dir]
 
-That will create a Scrapy project under the ``myproject`` directory.
+That will create a Scrapy project under the ``project_dir`` directory.
+If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``myproject``.
 
 Next, you go inside the new project directory::
 
-    cd myproject
+    cd project_dir
 
 And you're ready to use the ``scrapy`` command to manage and control your
 project from there.
@@ -181,11 +182,12 @@ Project-only commands:
 startproject
 ------------
 
-* Syntax: ``scrapy startproject <project_name>``
+* Syntax: ``scrapy startproject <project_name> [project_dir]``
 * Requires project: *no*
 
-Creates a new Scrapy project named ``project_name``, under the ``project_name``
+Creates a new Scrapy project named ``project_name``, under the ``project_dir``
 directory.
+If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``myproject``.
 
 Usage example::
 

From ffa77e1ac60fb3924ee133ce3358447389827b1d Mon Sep 17 00:00:00 2001
From: omer <omer.schliefer@picscout.com>
Date: Tue, 24 May 2016 14:24:29 +0300
Subject: [PATCH 0962/2974] Do not decode of head response (2)

---
 scrapy/downloadermiddlewares/httpcompression.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 7ab304c17..bcf20f10c 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -20,6 +20,9 @@ class HttpCompressionMiddleware(object):
         request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
 
     def process_response(self, request, response, spider):
+
+        if request.method == 'HEAD':
+            return response
         if isinstance(response, Response):
             content_encoding = response.headers.getlist('Content-Encoding')
             if content_encoding and not is_gzipped(response):

From 24a45cc6e6d15eab7aafebd4cde5364201d2b133 Mon Sep 17 00:00:00 2001
From: Felipe Ruhland <felipe.ruhland@gmail.com>
Date: Tue, 24 May 2016 10:58:50 -0300
Subject: [PATCH 0963/2974] Fix py35 compatibility tests

---
 scrapy/commands/startproject.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 8f3243a2a..4da129865 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -79,13 +79,13 @@ class Command(ScrapyCommand):
                     copy2(srcname, dstname)
             # catch the Error from the recursive copytree so that we can
             # continue with other files
-            except EnvironmentError, err:
+            except EnvironmentError as err:
                 errors.extend(err.args[0])
-            except EnvironmentError, why:
+            except EnvironmentError as why:
                 errors.append((srcname, dstname, str(why)))
         try:
             copystat(src, dst)
-        except OSError, why:
+        except OSError as why:
             if WindowsError is not None and isinstance(why, WindowsError):
                 # Copying file access times may fail on Windows
                 pass

From fc9a45ee9129912e4ee138b0a17d8ed8a27a0c9e Mon Sep 17 00:00:00 2001
From: Felipe Ruhland <felipe.ruhland@gmail.com>
Date: Tue, 24 May 2016 11:57:56 -0300
Subject: [PATCH 0964/2974] Simplified copytree function

---
 scrapy/commands/startproject.py | 40 ++++++++-------------------------
 1 file changed, 9 insertions(+), 31 deletions(-)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 4da129865..c0e7af88a 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -52,47 +52,25 @@ class Command(ScrapyCommand):
             return True
         return False
 
-    def _copytree(self, src, dst, symlinks=False, ignore=None):
+    def _copytree(self, src, dst):
+        ignore = IGNORE
         names = os.listdir(src)
-        if ignore is not None:
-            ignored_names = ignore(src, names)
-        else:
-            ignored_names = set()
+        ignored_names = ignore(src, names)
 
         if not os.path.exists(dst):
             os.makedirs(dst)
 
-        errors = []
         for name in names:
             if name in ignored_names:
                 continue
+
             srcname = os.path.join(src, name)
             dstname = os.path.join(dst, name)
-            try:
-                if symlinks and os.path.islink(srcname):
-                    linkto = os.readlink(srcname)
-                    os.symlink(linkto, dstname)
-                elif os.path.isdir(srcname):
-                    self._copytree(srcname, dstname, symlinks, ignore)
-                else:
-                    # Will raise a SpecialFileError for unsupported file types
-                    copy2(srcname, dstname)
-            # catch the Error from the recursive copytree so that we can
-            # continue with other files
-            except EnvironmentError as err:
-                errors.extend(err.args[0])
-            except EnvironmentError as why:
-                errors.append((srcname, dstname, str(why)))
-        try:
-            copystat(src, dst)
-        except OSError as why:
-            if WindowsError is not None and isinstance(why, WindowsError):
-                # Copying file access times may fail on Windows
-                pass
+            if os.path.isdir(srcname):
+                self._copytree(srcname, dstname)
             else:
-                errors.append((src, dst, str(why)))
-        if errors:
-            raise EnvironmentError(errors)
+                copy2(srcname, dstname)
+        copystat(src, dst)
 
     def run(self, args, opts):
         if len(args) not in (1, 2):
@@ -112,7 +90,7 @@ class Command(ScrapyCommand):
             self.exitcode = 1
             return
 
-        self._copytree(self.templates_dir, abspath(project_dir), ignore=IGNORE)
+        self._copytree(self.templates_dir, abspath(project_dir))
         move(join(project_dir, 'module'), join(project_dir, project_name))
         for paths in TEMPLATES_TO_RENDER:
             path = join(*paths)

From b8a09d7ab7baad9e8ea2e7a8dd1ed0fe4e4fedc7 Mon Sep 17 00:00:00 2001
From: Felipe Ruhland <felipe.ruhland@gmail.com>
Date: Tue, 24 May 2016 11:58:52 -0300
Subject: [PATCH 0965/2974] Added tests for more or less parameters

---
 tests/test_commands.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 42cade849..a487d14f7 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -89,6 +89,8 @@ class StartprojectTest(ProjectTest):
         self.assertEqual(1, self.call('startproject', self.project_name + '2', project_dir))
         self.assertEqual(1, self.call('startproject', 'wrong---project---name'))
         self.assertEqual(1, self.call('startproject', 'sys'))
+        self.assertEqual(2, self.call('startproject'))
+        self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params'))
 
 
 class StartprojectTemplatesTest(ProjectTest):

From 85c4ecb92856f7d50f8dbf7c92d629bd5db83f02 Mon Sep 17 00:00:00 2001
From: Felipe Ruhland <felipe.ruhland@gmail.com>
Date: Tue, 24 May 2016 13:00:41 -0300
Subject: [PATCH 0966/2974] Removed validation of project_name dir exists

---
 scrapy/commands/startproject.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index c0e7af88a..a51a586ab 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -44,8 +44,6 @@ class Command(ScrapyCommand):
         if not re.search(r'^[_a-zA-Z]\w*$', project_name):
             print('Error: Project names must begin with a letter and contain'\
                     ' only\nletters, numbers and underscores')
-        elif exists(project_name):
-            print('Error: Directory %r already exists' % project_name)
         elif _module_exists(project_name):
             print('Error: Module %r already exists' % project_name)
         else:

From 9ad54b381535eec583e387cb75d7205ccc8d7d3a Mon Sep 17 00:00:00 2001
From: Felipe Ruhland <felipe.ruhland@gmail.com>
Date: Tue, 24 May 2016 13:03:33 -0300
Subject: [PATCH 0967/2974] Fix template description after create project

---
 scrapy/commands/startproject.py | 4 ++--
 tests/test_commands.py          | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index a51a586ab..0940d67f9 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -98,9 +98,9 @@ class Command(ScrapyCommand):
                 ProjectName=string_camelcase(project_name))
         print("New Scrapy project %r, using template directory %r, created in:" % \
               (project_name, self.templates_dir))
-        print("    %s\n" % abspath(project_name))
+        print("    %s\n" % abspath(project_dir))
         print("You can start your first spider with:")
-        print("    cd %s" % project_name)
+        print("    cd %s" % project_dir)
         print("    scrapy genspider example example.com")
 
     @property
diff --git a/tests/test_commands.py b/tests/test_commands.py
index a487d14f7..ca7a19eaa 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -85,6 +85,8 @@ class StartprojectTest(ProjectTest):
         assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py'))
         assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py'))
 
+        self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2'))
+
         self.assertEqual(1, self.call('startproject', self.project_name, project_dir))
         self.assertEqual(1, self.call('startproject', self.project_name + '2', project_dir))
         self.assertEqual(1, self.call('startproject', 'wrong---project---name'))

From c8ec79d95922b4435af47b5ca03e08eccc990432 Mon Sep 17 00:00:00 2001
From: omer <omer.schliefer@picscout.com>
Date: Wed, 25 May 2016 06:56:06 +0300
Subject: [PATCH 0968/2974] fix issue with '' in python 3

---
 tests/test_downloadermiddleware_httpcompression.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 9ef66ba1c..24955a515 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -153,4 +153,4 @@ class HttpCompressionTest(TestCase):
         response = response.replace(body = None)
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIs(newresponse, response)
-        self.assertEquals(response.body, '')
+        self.assertEquals(response.body, b'')

From da8d0eadd6ad2d7d03195c32234dae0aa3888160 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 27 May 2016 10:47:57 +0300
Subject: [PATCH 0969/2974] Use "url" variable in the example

Instead of hardcoded http://www.example.com: without it url variable is unused and only one request will make it past dupefilter.
---
 docs/topics/downloader-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 59a73493d..31545d548 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -200,7 +200,7 @@ There is support for keeping multiple cookie sessions per spider by using the
 For example::
 
     for i, url in enumerate(urls):
-        yield scrapy.Request("http://www.example.com", meta={'cookiejar': i},
+        yield scrapy.Request(url, meta={'cookiejar': i},
             callback=self.parse_page)
 
 Keep in mind that the :reqmeta:`cookiejar` meta key is not "sticky". You need to keep

From 1aec5200bc81493623f2a4e077b4e80e104e47d5 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 8 Jun 2016 16:49:33 +0200
Subject: [PATCH 0970/2974] Do not fail on canonicalizing URLs with wrong
 netlocs

Fixes #2010
---
 scrapy/utils/url.py     |  9 ++++++++-
 tests/test_utils_url.py | 14 ++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index c80fc6e70..406eb5843 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -41,9 +41,16 @@ def url_has_any_extension(url, extensions):
 
 
 def _safe_ParseResult(parts, encoding='utf8', path_encoding='utf8'):
+    # IDNA encoding can fail for too long labels (>63 characters)
+    # or missing labels (e.g. http://.example.com)
+    try:
+        netloc = parts.netloc.encode('idna')
+    except UnicodeError:
+        netloc = parts.netloc
+
     return (
         to_native_str(parts.scheme),
-        to_native_str(parts.netloc.encode('idna')),
+        to_native_str(netloc),
 
         # default encoding for path component SHOULD be UTF-8
         quote(to_bytes(parts.path, path_encoding), _safe_chars),
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 1fc3a3510..b4819874d 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -265,6 +265,20 @@ class CanonicalizeUrlTest(unittest.TestCase):
             # without encoding, already canonicalized URL is canonicalized identically
             self.assertEqual(canonicalize_url(canonicalized), canonicalized)
 
+    def test_canonicalize_url_idna_exceptions(self):
+        # missing DNS label
+        self.assertEqual(
+            canonicalize_url(u"http://.example.com/résumé?q=résumé"),
+            "http://.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9")
+
+        # DNS label too long
+        self.assertEqual(
+            canonicalize_url(
+                u"http://www.{label}.com/résumé?q=résumé".format(
+                    label=u"example"*11)),
+            "http://www.{label}.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9".format(
+                    label=u"example"*11))
+
 
 class AddHttpIfNoScheme(unittest.TestCase):
 

From a62d4b081c8eef1e54c6f7399e7a4154e3f330f3 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Fri, 10 Jun 2016 12:48:02 +0200
Subject: [PATCH 0971/2974] [image-pipeline] image settings with class name

allow to have image settings with class name, so that settings for user defined ImagePipeline
subclasses can be defined easily.
---
 docs/topics/media-pipeline.rst |  6 +++---
 scrapy/pipelines/images.py     | 27 +++++++++++++++++++++------
 tests/test_pipeline_images.py  | 18 ++++++++++++++++--
 3 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 2b41a4f43..96339d03d 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -191,9 +191,9 @@ For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/or
 If you need something more complex and want to override the custom pipeline
 behaviour, see :ref:`topics-media-pipeline-override`.
 
-.. note:: If you have multiple image pipelines and you want to have different settings in different pipelines
-   you can either set class attributes in pipeline object or add custom settings keys and load them at pipeline
-   initialization.
+.. note:: If you have multiple image pipelines inheriting from ImagePipeline and you want to have different settings in different pipelines
+   you can set setting keys preceded with uppercase name of your pipeline class. E.g. if your pipeline is called
+   MyPipeline and you want to have custom IMAGES_URLS_FIELD you define setting MYPIPELINE_IMAGES_URLS_FIELD.
 
 
 Additional features
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index ac78ee6c0..9ba04750a 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -50,12 +50,27 @@ class ImagesPipeline(FilesPipeline):
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
 
-        self.expires = settings.getint('IMAGES_EXPIRES', self.EXPIRES)
-        self.images_urls_field = settings.get('IMAGES_URLS_FIELD', self.IMAGES_URLS_FIELD)
-        self.images_result_field = settings.get('IMAGES_RESULT_FIELD', self.IMAGES_RESULT_FIELD)
-        self.min_width = settings.getint('IMAGES_MIN_WIDTH', self.MIN_WIDTH)
-        self.min_height = settings.getint('IMAGES_MIN_HEIGHT', self.MIN_HEIGHT)
-        self.thumbs = settings.get('IMAGES_THUMBS', self.THUMBS)
+        def key_for_pipe(key):
+            """
+            Allow setting settings for user defined ImagePipelines that inherit from base.
+
+            User can define setting key:
+
+            MYPIPELINENAME_IMAGE_SETTING_NAME = <some value>
+
+            and it will override default settings and class attributes.
+            """
+            class_name = self.__class__.__name__
+            if class_name == "ImagesPipeline":
+                return key
+            return "{}_{}".format(class_name.upper(), key)
+
+        self.expires = settings.getint(key_for_pipe('IMAGES_EXPIRES'), self.EXPIRES)
+        self.images_urls_field = settings.get(key_for_pipe('IMAGES_URLS_FIELD'), self.IMAGES_URLS_FIELD)
+        self.images_result_field = settings.get(key_for_pipe('IMAGES_RESULT_FIELD'), self.IMAGES_RESULT_FIELD)
+        self.min_width = settings.getint(key_for_pipe('IMAGES_MIN_WIDTH'), self.MIN_WIDTH)
+        self.min_height = settings.getint(key_for_pipe('IMAGES_MIN_HEIGHT'), self.MIN_HEIGHT)
+        self.thumbs = settings.get(key_for_pipe('IMAGES_THUMBS'), self.THUMBS)
 
     @classmethod
     def from_settings(cls, settings):
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 49257e175..3b68faed0 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -265,7 +265,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
         pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir}))
         self.assertEqual(pipeline.min_width, 1000)
 
-    def test_class_attrs_not_preserved_if_settings_defined(self):
+    def test_class_attrs_preserved_if_only_global_settings_defined(self):
 
         class UserDefinedImagePipeline(ImagesPipeline):
             MIN_WIDTH = 1000
@@ -277,7 +277,21 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
 
         # If image settings are defined they override class attributes.
         pipeline = UserDefinedImagePipeline.from_settings(Settings(settings))
-        self.assertEqual(pipeline.min_width, 90)
+        self.assertEqual(pipeline.min_width, 1000)
+
+    def test_settings_multiple_pipelilines(self):
+        # If user has multiple pipelines he can define setting keys preceded with
+        # pipeline class name.
+        class UserDefinedPipeline(ImagesPipeline):
+            pass
+
+        settings = {
+            "IMAGES_MIN_WIDTH": 10,
+            "USERDEFINEDPIPELINE_IMAGES_MIN_WIDTH": 1999,
+            "IMAGES_STORE": self.tempdir
+        }
+        user_pipeline = UserDefinedPipeline.from_settings(Settings(settings))
+        self.assertEqual(user_pipeline.min_width, 1999)
 
 
 def _create_image(format, *a, **kw):

From 7978237ed57eddf0cbb50c236c58243e5ed94899 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 10 Jun 2016 17:42:18 +0200
Subject: [PATCH 0972/2974] Add FAQ entry on using BeautifulSoup in spider
 callbacks

---
 docs/faq.rst | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/docs/faq.rst b/docs/faq.rst
index 5cd62710a..712f3b585 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -3,6 +3,8 @@
 Frequently Asked Questions
 ==========================
 
+.. _faq-scrapy-bs-cmp:
+
 How does Scrapy compare to BeautifulSoup or lxml?
 -------------------------------------------------
 
@@ -24,6 +26,36 @@ comparing `jinja2`_ to `Django`_.
 .. _jinja2: http://jinja.pocoo.org/
 .. _Django: https://www.djangoproject.com/
 
+How can I use Scrapy with BeautifulSoup?
+----------------------------------------
+
+As mentioned :ref:`above <faq-scrapy-bs-cmp>`, BeautifulSoup can be used
+for parsing HTML responses in Scrapy callbacks.
+You just have to feed the response's body into a ``BeautifulSoup`` object
+and extract whatever data you need from it.
+
+Here's an example spider using ``lxml`` parser with BeautifulSoup API::
+
+
+    from bs4 import BeautifulSoup
+    import scrapy
+
+
+    class ExampleSpider(scrapy.Spider):
+        name = "example"
+        allowed_domains = ["example.com"]
+        start_urls = (
+            'http://www.example.com/',
+        )
+
+        def parse(self, response):
+            soup = BeautifulSoup(response.text, 'lxml')
+            yield {
+                "url": response.url,
+                "title": soup.h1.string
+            }
+
+
 .. _faq-python-versions:
 
 What Python versions does Scrapy support?

From 989f6b8843c949aa2ce3839a37399fcada442ac7 Mon Sep 17 00:00:00 2001
From: Joakim Uddholm <joakim@uddholm.com>
Date: Sun, 12 Jun 2016 01:38:01 +0200
Subject: [PATCH 0973/2974] Test to show bug with is_gzipped and Content-Type:
 application/gzip;charset.

---
 tests/test_utils_gz.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index 8fb1e414d..e107615f3 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -1,7 +1,8 @@
 import unittest
 from os.path import join
 
-from scrapy.utils.gz import gunzip
+from scrapy.utils.gz import gunzip, is_gzipped
+from scrapy.http import Response, Headers
 from tests import tests_datadir
 
 SAMPLEDIR = join(tests_datadir, 'compressed')
@@ -27,3 +28,16 @@ class GunzipTest(unittest.TestCase):
         with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f:
             text = gunzip(f.read())
             assert text.endswith(b'</html>')
+
+    def test_is_gzipped(self):
+        hdrs = Headers({"Content-Type": "application/x-gzip"})
+        r1 = Response("http://www.example.com", headers=hdrs)
+        self.assertTrue(is_gzipped(r1))
+        r2 = Response("http://www.example.com")
+        self.assertTrue(not is_gzipped(r2))
+        hdrs = Headers({"Content-Type": "application/javascript"})
+        r3 = Response("http://www.example.com", headers=hdrs)
+        self.assertTrue(not is_gzipped(r3))
+        hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"})
+        r1 = Response("http://www.example.com", headers=hdrs)
+        self.assertTrue(is_gzipped(r1))

From db729f5b304212518e3208995f44e66e55af420e Mon Sep 17 00:00:00 2001
From: Joakim Uddholm <joakim@uddholm.com>
Date: Sun, 12 Jun 2016 02:26:16 +0200
Subject: [PATCH 0974/2974] Suggested fix for is_gzipped

---
 scrapy/utils/gz.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index d035f9fdf..f174950a4 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -54,4 +54,4 @@ def gunzip(data):
 def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
     ctype = response.headers.get('Content-Type', b'')
-    return ctype in (b'application/x-gzip', b'application/gzip')
+    return b'application/x-gzip' in ctype or b'application/gzip' in ctype

From 2c98a88a0e584fff64eb70f48b869494eca1d7ae Mon Sep 17 00:00:00 2001
From: Joakim Uddholm <joakim@uddholm.com>
Date: Sun, 12 Jun 2016 10:49:34 +0200
Subject: [PATCH 0975/2974] Separated tests based on case

---
 tests/test_utils_gz.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index e107615f3..3648d5c43 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -29,15 +29,24 @@ class GunzipTest(unittest.TestCase):
             text = gunzip(f.read())
             assert text.endswith(b'</html>')
 
-    def test_is_gzipped(self):
+    def test_is_gzipped_right(self):
         hdrs = Headers({"Content-Type": "application/x-gzip"})
         r1 = Response("http://www.example.com", headers=hdrs)
         self.assertTrue(is_gzipped(r1))
-        r2 = Response("http://www.example.com")
-        self.assertTrue(not is_gzipped(r2))
+        hdrs = Headers({"Content-Type": "application/gzip"})
+        r1 = Response("http://www.example.com", headers=hdrs)
+        self.assertTrue(is_gzipped(r1))
+
+    def test_is_gzipped_empty(self):
+        r1 = Response("http://www.example.com")
+        self.assertTrue(not is_gzipped(r1))
+
+    def test_is_gzipped_wrong(self):
         hdrs = Headers({"Content-Type": "application/javascript"})
-        r3 = Response("http://www.example.com", headers=hdrs)
-        self.assertTrue(not is_gzipped(r3))
+        r1 = Response("http://www.example.com", headers=hdrs)
+        self.assertTrue(not is_gzipped(r1))
+
+    def test_is_gzipped_with_charset(self):
         hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"})
         r1 = Response("http://www.example.com", headers=hdrs)
         self.assertTrue(is_gzipped(r1))

From 0bb12889695064665edd82eceaff27ea910d70db Mon Sep 17 00:00:00 2001
From: Matt O'Connell <mattoconnell408@gmail.com>
Date: Mon, 13 Jun 2016 16:21:02 -0400
Subject: [PATCH 0976/2974] Typo fixes

---
 docs/intro/tutorial.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index f2af6e17c..6ecd637c3 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -470,12 +470,12 @@ Here is a modification to our spider that does just that::
                 item['desc'] = sel.xpath('text()').extract()
                 yield item
 
-Now the `parse()` method only extract the interesting links from the page,
+Now the `parse()` method only extracts the interesting links from the page,
 builds a full absolute URL using the `response.urljoin` method (since the links can
 be relative) and yields new requests to be sent later, registering as callback
 the method `parse_dir_contents()` that will ultimately scrape the data we want.
 
-What you see here is the Scrapy's mechanism of following links: when you yield
+What you see here is Scrapy's mechanism of following links: when you yield
 a Request in a callback method, Scrapy will schedule that request to be sent
 and register a callback method to be executed when that request finishes.
 
@@ -483,7 +483,7 @@ Using this, you can build complex crawlers that follow links according to rules
 you define, and extract different kinds of data depending on the page it's
 visiting.
 
-A common pattern is a callback method that extract some items, looks for a link
+A common pattern is a callback method that extracts some items, looks for a link
 to follow to the next page and then yields a `Request` with the same callback
 for it::
 

From b76b374648bc7736e37574c6e800f0428f014dfa Mon Sep 17 00:00:00 2001
From: Matvei Nazaruk <matvei.nazaruk@gmail.com>
Date: Mon, 13 Jun 2016 22:36:13 +0300
Subject: [PATCH 0977/2974] Added test for http11 choosing response type
 without content-type header.

---
 tests/test_downloader_handlers.py | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 45a806f2e..09a0950e8 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -27,6 +27,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 
 from scrapy.spiders import Spider
 from scrapy.http import Request
+from scrapy.http.response.text import TextResponse
 from scrapy.settings import Settings
 from scrapy.utils.test import get_crawler, skip_if_no_boto
 from scrapy.utils.python import to_bytes
@@ -114,6 +115,16 @@ class ContentLengthHeaderResource(resource.Resource):
         return request.requestHeaders.getRawHeaders(b"content-length")[0]
 
 
+class EmptyContentTypeHeaderResource(resource.Resource):
+    """
+    A testing resource which renders itself as the value of request body
+    without content-type header in response.
+    """
+    def render(self, request):
+        request.setHeader("content-type", "")
+        return request.content.read()
+
+
 class HttpTestCase(unittest.TestCase):
 
     scheme = 'http'
@@ -136,6 +147,7 @@ class HttpTestCase(unittest.TestCase):
         r.putChild(b"payload", PayloadResource())
         r.putChild(b"broken", BrokenDownloadResource())
         r.putChild(b"contentlength", ContentLengthHeaderResource())
+        r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
         self.host = 'localhost'
@@ -243,11 +255,6 @@ class HttpTestCase(unittest.TestCase):
         request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'})
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
-        d = self.download_request(request, Spider('foo'))
-        d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, b'0')
-        return d
-
     def test_payload(self):
         body = b'1'*100 # PayloadResource requires body length to be 100
         request = Request(self.getURL('payload'), method='POST', body=body)
@@ -284,6 +291,20 @@ class Http11TestCase(HttpTestCase):
         d.addCallback(self.assertEquals, b"0123456789")
         return d
 
+    def test_response_class_choosing_request(self):
+        """Tests choosing of correct response type
+         in case of Content-Type is empty but body contains text.
+        """
+        body = b'Some plain text\ndata with tabs\t and null bytes\0'
+
+        def _test_type(response):
+            self.assertEquals(type(response), TextResponse)
+
+        request = Request(self.getURL('nocontenttype'), body=body)
+        d = self.download_request(request, Spider('foo'))
+        d.addCallback(_test_type)
+        return d
+
     @defer.inlineCallbacks
     def test_download_with_maxsize(self):
         request = Request(self.getURL('file'))

From 124e218a3b3d8c5c4f924da5eb8399b205349d19 Mon Sep 17 00:00:00 2001
From: Joakim Uddholm <joakim@uddholm.com>
Date: Tue, 14 Jun 2016 14:22:18 +0200
Subject: [PATCH 0978/2974] Added new testcases suggested by @redapple.

---
 tests/test_utils_gz.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index 3648d5c43..a9bd29bae 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -37,14 +37,28 @@ class GunzipTest(unittest.TestCase):
         r1 = Response("http://www.example.com", headers=hdrs)
         self.assertTrue(is_gzipped(r1))
 
+    def test_is_gzipped_not_quite(self):
+        hdrs = Headers({"Content-Type": "application/gzippppp"})
+        r1 = Response("http://www.example.com", headers=hdrs)
+        self.assertFalse(is_gzipped(r1))
+
+    def test_is_gzipped_case_insensitive(self):
+        hdrs = Headers({"Content-Type": "Application/X-Gzip"})
+        r1 = Response("http://www.example.com", headers=hdrs)
+        self.assertTrue(is_gzipped(r1))
+
+        hdrs = Headers({"Content-Type": "application/X-GZIP ; charset=utf-8"})
+        r1 = Response("http://www.example.com", headers=hdrs)
+        self.assertTrue(is_gzipped(r1))
+
     def test_is_gzipped_empty(self):
         r1 = Response("http://www.example.com")
-        self.assertTrue(not is_gzipped(r1))
+        self.assertFalse(is_gzipped(r1))
 
     def test_is_gzipped_wrong(self):
         hdrs = Headers({"Content-Type": "application/javascript"})
         r1 = Response("http://www.example.com", headers=hdrs)
-        self.assertTrue(not is_gzipped(r1))
+        self.assertFalse(is_gzipped(r1))
 
     def test_is_gzipped_with_charset(self):
         hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"})

From 259426ec9995da9a5415de9d851febf788160cf3 Mon Sep 17 00:00:00 2001
From: Joakim Uddholm <joakim@uddholm.com>
Date: Tue, 14 Jun 2016 14:39:16 +0200
Subject: [PATCH 0979/2974] is_gzipped: Changed to regex to check the
 content-type header. Also suggested by @redapple.

---
 scrapy/utils/gz.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index f174950a4..f2a9555b1 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -7,7 +7,7 @@ except ImportError:
 from gzip import GzipFile
 
 import six
-
+import re
 
 # - Python>=3.5 GzipFile's read() has issues returning leftover
 #   uncompressed data when input is corrupted
@@ -50,8 +50,9 @@ def gunzip(data):
                 raise
     return output
 
+_is_gzipped_re = re.compile(br'^application/(x-)?gzip\b', re.I)
 
 def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
     ctype = response.headers.get('Content-Type', b'')
-    return b'application/x-gzip' in ctype or b'application/gzip' in ctype
+    return not _is_gzipped_re.search(ctype) is None

From 6cbd92fac159672977239b4928c1cce15ba8b37a Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 14 Jun 2016 15:33:34 +0200
Subject: [PATCH 0980/2974] Add note on how to choose parser with BeautifulSoup

---
 docs/faq.rst | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 712f3b585..af6103828 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -26,15 +26,17 @@ comparing `jinja2`_ to `Django`_.
 .. _jinja2: http://jinja.pocoo.org/
 .. _Django: https://www.djangoproject.com/
 
-How can I use Scrapy with BeautifulSoup?
-----------------------------------------
+Can I use Scrapy with BeautifulSoup?
+------------------------------------
 
-As mentioned :ref:`above <faq-scrapy-bs-cmp>`, BeautifulSoup can be used
+Yes, you can.
+As mentioned :ref:`above <faq-scrapy-bs-cmp>`, `BeautifulSoup`_ can be used
 for parsing HTML responses in Scrapy callbacks.
 You just have to feed the response's body into a ``BeautifulSoup`` object
 and extract whatever data you need from it.
 
-Here's an example spider using ``lxml`` parser with BeautifulSoup API::
+Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser
+(so you get the same parsing speed as with scrapy/parsel selectors)::
 
 
     from bs4 import BeautifulSoup
@@ -49,12 +51,19 @@ Here's an example spider using ``lxml`` parser with BeautifulSoup API::
         )
 
         def parse(self, response):
+            # use lxml to get decent HTML parsing speed
             soup = BeautifulSoup(response.text, 'lxml')
             yield {
                 "url": response.url,
                 "title": soup.h1.string
             }
 
+.. note::
+
+    ``BeautifulSoup`` supports several HTML/XML parsers.
+    See `BeautifulSoup's official documentation`_ on which ones are available.
+
+.. _BeautifulSoup's official documentation: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#specifying-the-parser-to-use
 
 .. _faq-python-versions:
 

From 36928d897ca44b0a62ebcb1c3fb358cbfd07440f Mon Sep 17 00:00:00 2001
From: Joakim Uddholm <joakim@uddholm.com>
Date: Tue, 14 Jun 2016 15:40:20 +0200
Subject: [PATCH 0981/2974] is_gzipped: improved readability

---
 scrapy/utils/gz.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index f2a9555b1..cfb652143 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -55,4 +55,4 @@ _is_gzipped_re = re.compile(br'^application/(x-)?gzip\b', re.I)
 def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
     ctype = response.headers.get('Content-Type', b'')
-    return not _is_gzipped_re.search(ctype) is None
+    return _is_gzipped_re.search(ctype) is not None

From d715172528925ec0242d29122a0cad6a6f4fd4e2 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Tue, 14 Jun 2016 19:09:56 +0200
Subject: [PATCH 0982/2974] [image_pipeline] unify and simplify tests for
 setting loading

there was identical test for different setting keys. I unified it into
one unit test.

Fixes comments for tests, adds comments about intention of uppercase attrs.

Adds another test for user defined setting keys and uppercase attrs.
---
 scrapy/pipelines/images.py    |  3 ++
 tests/test_pipeline_images.py | 77 ++++++++++++++++-------------------
 2 files changed, 39 insertions(+), 41 deletions(-)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 9ba04750a..b9a594e23 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -37,6 +37,9 @@ class ImagesPipeline(FilesPipeline):
     """
 
     MEDIA_NAME = 'image'
+
+    # Uppercase attributes kept for backward compatibility with code that subclasses
+    # ImagesPipeline. They may be overriden by settings.
     MIN_WIDTH = 0
     MIN_HEIGHT = 0
     EXPIRES = 0
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 3b68faed0..69e2eb82f 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -207,6 +207,7 @@ class ImagesPipelineTestCaseFields(unittest.TestCase):
 
 class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
 
+
     def setUp(self):
         self.tempdir = mkdtemp()
         self.pipeline = ImagesPipeline(self.tempdir)
@@ -215,46 +216,27 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
     def tearDown(self):
         rmtree(self.tempdir)
 
-    def test_expires(self):
-        another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
-                                                                'IMAGES_EXPIRES': 42}))
-        self.assertEqual(self.pipeline.expires, self.default_settings.getint('IMAGES_EXPIRES'))
-        self.assertEqual(another_pipeline.expires, 42)
-
-    def test_images_urls_field(self):
-        another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
-                                                                'IMAGES_URLS_FIELD': 'funny_field'}))
-        default = self.pipeline.IMAGES_URLS_FIELD
-        self.assertEqual(self.pipeline.images_urls_field, self.default_settings.get('IMAGES_URLS_FIELD', default))
-        self.assertEqual(another_pipeline.images_urls_field, 'funny_field')
-
-    def test_images_result_field(self):
-        another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
-                                                                'IMAGES_RESULT_FIELD': 'funny_field'}))
-        default = self.pipeline.IMAGES_RESULT_FIELD
-        self.assertEqual(self.pipeline.images_result_field, self.default_settings.get('IMAGES_RESULT_FIELD', default))
-        self.assertEqual(another_pipeline.images_result_field, 'funny_field')
-
-    def test_min_width(self):
-        another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
-                                                                'IMAGES_MIN_WIDTH': 42}))
-        default = self.pipeline.MIN_WIDTH
-        self.assertEqual(self.pipeline.min_width, self.default_settings.getint('IMAGES_MIN_WIDTH', default))
-        self.assertEqual(another_pipeline.min_width, 42)
-
-    def test_min_height(self):
-        another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
-                                                                'IMAGES_MIN_HEIGHT': 42}))
-        self.assertEqual(self.pipeline.min_height, self.default_settings.getint('IMAGES_MIN_HEIGHT'))
-        self.assertEqual(another_pipeline.min_height, 42)
-
-    def test_thumbs(self):
-        custom_thumbs = {'small': (50, 50), 'big': (270, 270)}
-        another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
-                                                                'IMAGES_THUMBS': custom_thumbs}))
-        default = self.pipeline.THUMBS
-        self.assertEqual(self.pipeline.thumbs, self.default_settings.get('IMAGES_THUMBS', default))
-        self.assertEqual(another_pipeline.thumbs, custom_thumbs)
+    def test_different_settings_for_different_instances(self):
+        custom_settings = [
+            # Order is: key name in settings.py, value, name of pipeline attribute.
+            ("IMAGES_EXPIRES", 42, "EXPIRES"),
+            ("IMAGES_STORE", self.tempdir, "IMAGES_STORE"),
+            ("IMAGES_RESULT_FIELD", "funny_field", "IMAGES_RESULT_FIELD"),
+            ("IMAGES_URLS_FIELD", "other_field", "IMAGES_URLS_FIELD"),
+            ("IMAGES_MIN_WIDTH", 99, "MIN_WIDTH"),
+            ("IMAGES_MIN_HEIGHT", 112, "MIN_HEIGHT"),
+            ("IMAGES_THUMBS", {'small': (50, 50), 'big': (270, 270)}, "THUMBS")
+        ]
+        default_settings = Settings()
+        default_sts_pipe = ImagesPipeline(self.tempdir, settings=default_settings)
+        user_sts_pipe = ImagesPipeline.from_settings(Settings({k: v for k, v, _ in custom_settings}))
+        for key, custom_value, attr_name in custom_settings:
+            if attr_name == "IMAGES_STORE":
+                # this is not set as pipeline attribute
+                continue
+            expected_default_value = getattr(default_sts_pipe, attr_name)
+            self.assertEqual(getattr(default_sts_pipe, attr_name), expected_default_value, key)
+            self.assertEqual(getattr(user_sts_pipe, attr_name.lower()), custom_value, key)
 
     def test_class_attrs_preserved(self):
 
@@ -275,7 +257,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
             "IMAGES_MIN_WIDTH": 90
         }
 
-        # If image settings are defined they override class attributes.
+        # Class attributes for subclass of ImagePipeline override default setting keys.
         pipeline = UserDefinedImagePipeline.from_settings(Settings(settings))
         self.assertEqual(pipeline.min_width, 1000)
 
@@ -293,6 +275,19 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
         user_pipeline = UserDefinedPipeline.from_settings(Settings(settings))
         self.assertEqual(user_pipeline.min_width, 1999)
 
+    def test_settings_multiple_pipelilines_and_class_attrs(self):
+        # Setting keys for user defined pipeline override class attributes.
+        class UserDefinedPipeline(ImagesPipeline):
+            MIN_WIDTH = 200
+
+        settings = {
+            "IMAGES_MIN_WIDTH": 10,
+            "USERDEFINEDPIPELINE_IMAGES_MIN_WIDTH": 1999,
+            "IMAGES_STORE": self.tempdir
+        }
+        user_pipeline = UserDefinedPipeline.from_settings(Settings(settings))
+        self.assertEqual(user_pipeline.min_width, 1999)
+
 
 def _create_image(format, *a, **kw):
     buf = TemporaryFile()

From 1ff9a4828ba4c43343e36b2f1935363c067e5c84 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 14 Jun 2016 19:33:56 +0200
Subject: [PATCH 0983/2974] Do not commit on any HTML parsing speed assumption
 with BS4+lxml

---
 docs/faq.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index af6103828..82e1f3422 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -35,8 +35,7 @@ for parsing HTML responses in Scrapy callbacks.
 You just have to feed the response's body into a ``BeautifulSoup`` object
 and extract whatever data you need from it.
 
-Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser
-(so you get the same parsing speed as with scrapy/parsel selectors)::
+Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser::
 
 
     from bs4 import BeautifulSoup

From 23f99e98c4cf97891202847f3384a35b6fc57e7c Mon Sep 17 00:00:00 2001
From: Joakim Uddholm <joakim@uddholm.com>
Date: Tue, 14 Jun 2016 21:33:51 +0200
Subject: [PATCH 0984/2974] is_gzipped: Separated tests again.

---
 tests/test_utils_gz.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index a9bd29bae..2b47bf8da 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -29,10 +29,12 @@ class GunzipTest(unittest.TestCase):
             text = gunzip(f.read())
             assert text.endswith(b'</html>')
 
-    def test_is_gzipped_right(self):
+    def test_is_x_gzipped_right(self):
         hdrs = Headers({"Content-Type": "application/x-gzip"})
         r1 = Response("http://www.example.com", headers=hdrs)
         self.assertTrue(is_gzipped(r1))
+
+    def test_is_gzipped_right(self):
         hdrs = Headers({"Content-Type": "application/gzip"})
         r1 = Response("http://www.example.com", headers=hdrs)
         self.assertTrue(is_gzipped(r1))

From ee39d11e45a6ca20a24d28cd08c4b20ea1eae40e Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Wed, 15 Jun 2016 11:25:38 +0200
Subject: [PATCH 0985/2974] [image_pipeline] refactor and simplify tests for
 image settings

unify tests that test same thing for different attribute values into one. Add
better docstrings for tests.
---
 scrapy/pipelines/images.py    |   2 +-
 tests/test_pipeline_images.py | 188 ++++++++++++++++++++++------------
 2 files changed, 125 insertions(+), 65 deletions(-)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index b9a594e23..465d7c492 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -39,7 +39,7 @@ class ImagesPipeline(FilesPipeline):
     MEDIA_NAME = 'image'
 
     # Uppercase attributes kept for backward compatibility with code that subclasses
-    # ImagesPipeline. They may be overriden by settings.
+    # ImagesPipeline. They may be overridden by settings.
     MIN_WIDTH = 0
     MIN_HEIGHT = 0
     EXPIRES = 0
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 69e2eb82f..177a887d0 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -1,5 +1,6 @@
 import os
 import hashlib
+import random
 import warnings
 from tempfile import mkdtemp, TemporaryFile
 from shutil import rmtree
@@ -206,87 +207,146 @@ class ImagesPipelineTestCaseFields(unittest.TestCase):
 
 
 class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
+    img_cls_attribute_names = [
+        # Pipeline attribute names with corresponding setting names.
+        ("EXPIRES", "IMAGES_EXPIRES"),
+        ("MIN_WIDTH", "IMAGES_MIN_WIDTH"),
+        ("MIN_HEIGHT", "IMAGES_MIN_HEIGHT"),
+        ("IMAGES_URLS_FIELD", "IMAGES_URLS_FIELD"),
+        ("IMAGES_RESULT_FIELD", "IMAGES_RESULT_FIELD"),
+        ("THUMBS", "IMAGES_THUMBS")
+    ]
+
+    # This should match what is defined in ImagesPipeline.
+    default_pipeline_settings = dict(
+        MIN_WIDTH=0,
+        MIN_HEIGHT=0,
+        EXPIRES=0,
+        THUMBS={},
+        IMAGES_URLS_FIELD='image_urls',
+        IMAGES_RESULT_FIELD='images'
+    )
 
 
     def setUp(self):
         self.tempdir = mkdtemp()
-        self.pipeline = ImagesPipeline(self.tempdir)
-        self.default_settings = Settings()
 
     def tearDown(self):
         rmtree(self.tempdir)
 
+    def _generate_fake_settings(self, prefix=None):
+        """
+        :param prefix: string for setting keys
+        :return: dictionary of image pipeline settings
+        """
+
+        def random_string():
+            return "".join([chr(random.randint(97, 123)) for _ in range(10)])
+
+        settings = {
+            "IMAGES_EXPIRES": random.randint(1, 1000),
+            "IMAGES_STORE": self.tempdir,
+            "IMAGES_RESULT_FIELD": random_string(),
+            "IMAGES_URLS_FIELD": random_string(),
+            "IMAGES_MIN_WIDTH": random.randint(1, 1000),
+            "IMAGES_MIN_HEIGHT": random.randint(1, 1000),
+            "IMAGES_THUMBS": {
+                'small': (random.randint(1, 1000), random.randint(1, 1000)),
+                'big': (random.randint(1, 1000), random.randint(1, 1000))
+            }
+        }
+        if not prefix:
+            return settings
+
+        return {prefix.upper() + "_" + k if k != "IMAGES_STORE" else k: v for k, v in settings.items()}
+
+    def _generate_fake_pipeline_subclass(self):
+        """
+        :return: ImagePipeline class will all uppercase attributes set.
+        """
+        class UserDefinedImagePipeline(ImagesPipeline):
+            # Values should be in different range than fake_settings.
+            MIN_WIDTH = random.randint(1000, 2000)
+            MIN_HEIGHT = random.randint(1000, 2000)
+            THUMBS = {
+                'small': (random.randint(1000, 2000), random.randint(1000, 2000)),
+                'big': (random.randint(1000, 2000), random.randint(1000, 2000))
+            }
+            EXPIRES = random.randint(1000, 2000)
+            IMAGES_URLS_FIELD = "field_one"
+            IMAGES_RESULT_FIELD = "field_two"
+
+        return UserDefinedImagePipeline
+
     def test_different_settings_for_different_instances(self):
-        custom_settings = [
-            # Order is: key name in settings.py, value, name of pipeline attribute.
-            ("IMAGES_EXPIRES", 42, "EXPIRES"),
-            ("IMAGES_STORE", self.tempdir, "IMAGES_STORE"),
-            ("IMAGES_RESULT_FIELD", "funny_field", "IMAGES_RESULT_FIELD"),
-            ("IMAGES_URLS_FIELD", "other_field", "IMAGES_URLS_FIELD"),
-            ("IMAGES_MIN_WIDTH", 99, "MIN_WIDTH"),
-            ("IMAGES_MIN_HEIGHT", 112, "MIN_HEIGHT"),
-            ("IMAGES_THUMBS", {'small': (50, 50), 'big': (270, 270)}, "THUMBS")
-        ]
+        """
+        If there are two instances of ImagesPipeline class with different settings, they should
+        have different settings.
+        """
+        custom_settings = self._generate_fake_settings()
         default_settings = Settings()
         default_sts_pipe = ImagesPipeline(self.tempdir, settings=default_settings)
-        user_sts_pipe = ImagesPipeline.from_settings(Settings({k: v for k, v, _ in custom_settings}))
-        for key, custom_value, attr_name in custom_settings:
-            if attr_name == "IMAGES_STORE":
-                # this is not set as pipeline attribute
-                continue
-            expected_default_value = getattr(default_sts_pipe, attr_name)
-            self.assertEqual(getattr(default_sts_pipe, attr_name), expected_default_value, key)
-            self.assertEqual(getattr(user_sts_pipe, attr_name.lower()), custom_value, key)
+        user_sts_pipe = ImagesPipeline.from_settings(Settings(custom_settings))
+        for pipe_attr, settings_attr in self.img_cls_attribute_names:
+            expected_default_value = self.default_pipeline_settings.get(pipe_attr)
+            custom_value = custom_settings.get(settings_attr)
+            self.assertNotEqual(expected_default_value, custom_value)
+            self.assertEqual(getattr(default_sts_pipe, pipe_attr.lower()), expected_default_value)
+            self.assertEqual(getattr(user_sts_pipe, pipe_attr.lower()), custom_value)
 
-    def test_class_attrs_preserved(self):
+    def test_subclass_attrs_preserved_default_settings(self):
+        """
+        If image settings are not defined at all subclass of ImagePipeline takes values
+        from class attributes.
+        """
+        pipeline_cls = self._generate_fake_pipeline_subclass()
+        pipeline = pipeline_cls.from_settings(Settings({"IMAGES_STORE": self.tempdir}))
+        for pipe_attr, settings_attr in self.img_cls_attribute_names:
+            # Instance attribute (lowercase) must be equal to class attribute (uppercase).
+            attr_value = getattr(pipeline, pipe_attr.lower())
+            self.assertEqual(attr_value, getattr(pipeline, pipe_attr))
 
+    def test_subclass_attrs_preserved_custom_settings(self):
+        """
+        If image settings are defined but they are not defined for subclass class attributes
+        should be preserved.
+        """
+        pipeline_cls = self._generate_fake_pipeline_subclass()
+        settings = self._generate_fake_settings()
+        pipeline = pipeline_cls.from_settings(Settings(settings))
+        for pipe_attr, settings_attr in self.img_cls_attribute_names:
+            # Instance attribute (lowercase) must be equal to class attribute (uppercase).
+            value = getattr(pipeline, pipe_attr.lower())
+            self.assertEqual(value, getattr(pipeline, pipe_attr))
+
+    def test_custom_settings_for_subclasses(self):
+        """
+        If there are custom settings for subclass and NO class attributes, pipeline should use custom
+        settings.
+        """
         class UserDefinedImagePipeline(ImagesPipeline):
-            MIN_WIDTH = 1000
-
-        # If image settings are not defined values are taken from class attributes.
-        pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir}))
-        self.assertEqual(pipeline.min_width, 1000)
-
-    def test_class_attrs_preserved_if_only_global_settings_defined(self):
-
-        class UserDefinedImagePipeline(ImagesPipeline):
-            MIN_WIDTH = 1000
-
-        settings = {
-            "IMAGES_STORE": self.tempdir,
-            "IMAGES_MIN_WIDTH": 90
-        }
-
-        # Class attributes for subclass of ImagePipeline override default setting keys.
-        pipeline = UserDefinedImagePipeline.from_settings(Settings(settings))
-        self.assertEqual(pipeline.min_width, 1000)
-
-    def test_settings_multiple_pipelilines(self):
-        # If user has multiple pipelines he can define setting keys preceded with
-        # pipeline class name.
-        class UserDefinedPipeline(ImagesPipeline):
             pass
 
-        settings = {
-            "IMAGES_MIN_WIDTH": 10,
-            "USERDEFINEDPIPELINE_IMAGES_MIN_WIDTH": 1999,
-            "IMAGES_STORE": self.tempdir
-        }
-        user_pipeline = UserDefinedPipeline.from_settings(Settings(settings))
-        self.assertEqual(user_pipeline.min_width, 1999)
+        prefix = UserDefinedImagePipeline.__name__.upper()
+        settings = self._generate_fake_settings(prefix=prefix)
+        user_pipeline = UserDefinedImagePipeline.from_settings(Settings(settings))
+        for pipe_attr, settings_attr in self.img_cls_attribute_names:
+            # Values from settings for custom pipeline should be set on pipeline instance.
+            custom_value = settings.get(prefix + "_" + settings_attr)
+            self.assertEqual(getattr(user_pipeline, pipe_attr.lower()), custom_value)
 
-    def test_settings_multiple_pipelilines_and_class_attrs(self):
-        # Setting keys for user defined pipeline override class attributes.
-        class UserDefinedPipeline(ImagesPipeline):
-            MIN_WIDTH = 200
-
-        settings = {
-            "IMAGES_MIN_WIDTH": 10,
-            "USERDEFINEDPIPELINE_IMAGES_MIN_WIDTH": 1999,
-            "IMAGES_STORE": self.tempdir
-        }
-        user_pipeline = UserDefinedPipeline.from_settings(Settings(settings))
-        self.assertEqual(user_pipeline.min_width, 1999)
+    def test_custom_settings_and_class_attrs_for_subclasses(self):
+        """
+        If there are custom settings for subclass AND class attributes
+        setting keys are preferred and override attributes.
+        """
+        pipeline_cls = self._generate_fake_pipeline_subclass()
+        prefix = pipeline_cls.__name__.upper()
+        settings = self._generate_fake_settings(prefix=prefix)
+        user_pipeline = pipeline_cls.from_settings(Settings(settings))
+        for pipe_attr, settings_attr in self.img_cls_attribute_names:
+            custom_value = settings.get(prefix + "_" + settings_attr)
+            self.assertEqual(getattr(user_pipeline, pipe_attr.lower()), custom_value)
 
 
 def _create_image(format, *a, **kw):

From 72e4d5f33ef8130acf833031b018192c39b60152 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Wed, 15 Jun 2016 14:07:17 +0200
Subject: [PATCH 0986/2974] [image_pipeline] another test for subclass
 inheritance

test case when subclass inherits from base class and has no attributes nor
settings defined.
---
 tests/test_pipeline_images.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 177a887d0..a2dd5aa28 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -319,6 +319,20 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
             value = getattr(pipeline, pipe_attr.lower())
             self.assertEqual(value, getattr(pipeline, pipe_attr))
 
+    def test_no_custom_settings_for_subclasses(self):
+        """
+        If there are no settings for subclass and no subclass attributes, pipeline should use
+        attributes of base class.
+        """
+        class UserDefinedImagePipeline(ImagesPipeline):
+            pass
+
+        user_pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir}))
+        for pipe_attr, settings_attr in self.img_cls_attribute_names:
+            # Values from settings for custom pipeline should be set on pipeline instance.
+            custom_value = self.default_pipeline_settings.get(pipe_attr.upper())
+            self.assertEqual(getattr(user_pipeline, pipe_attr.lower()), custom_value)
+
     def test_custom_settings_for_subclasses(self):
         """
         If there are custom settings for subclass and NO class attributes, pipeline should use custom

From c6d1686d98f640be87df4337015e83fba2964f53 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Wed, 15 Jun 2016 14:48:25 +0200
Subject: [PATCH 0987/2974] [files_pipeline] unify tests for files pipeline

if test tests same thing but for different field it can be unified into one.
---
 tests/test_pipeline_files.py | 52 ++++++++++++++++++++++++------------
 1 file changed, 35 insertions(+), 17 deletions(-)

diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 391538562..760684c2b 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,4 +1,5 @@
 import os
+import random
 import time
 import hashlib
 import warnings
@@ -184,32 +185,49 @@ class FilesPipelineTestCaseFields(unittest.TestCase):
 
 
 class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
+    default_cls_settings = {
+        "EXPIRES": 90,
+        "DEFAULT_FILES_URLS_FIELD": "file_urls",
+        "DEFAULT_FILES_RESULT_FIELD": "files"
+    }
+    file_cls_attr_settings_map = {
+        ("EXPIRES", "FILES_EXPIRES"),
+        ("DEFAULT_FILES_URLS_FIELD", "FILES_URLS_FIELD"),
+        ("DEFAULT_FILES_RESULT_FIELD", "FILES_RESULT_FIELD")
+    }
 
     def setUp(self):
         self.tempdir = mkdtemp()
-        self.pipeline = FilesPipeline(self.tempdir)
-        self.default_settings = Settings()
 
     def tearDown(self):
         rmtree(self.tempdir)
 
-    def test_expires(self):
-        another_pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir,
-                                                                'FILES_EXPIRES': 42}))
-        self.assertEqual(self.pipeline.expires, self.default_settings.getint('FILES_EXPIRES'))
-        self.assertEqual(another_pipeline.expires, 42)
+    def _generate_fake_settings(self, prefix=None):
 
-    def test_files_urls_field(self):
-        another_pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir,
-                                                                'FILES_URLS_FIELD': 'funny_field'}))
-        self.assertEqual(self.pipeline.files_urls_field, self.default_settings.get('FILES_URLS_FIELD'))
-        self.assertEqual(another_pipeline.files_urls_field, 'funny_field')
+        def random_string():
+            return "".join([chr(random.randint(97, 123)) for _ in range(10)])
 
-    def test_files_result_field(self):
-        another_pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir,
-                                                                'FILES_RESULT_FIELD': 'funny_field'}))
-        self.assertEqual(self.pipeline.files_result_field, self.default_settings.get('FILES_RESULT_FIELD'))
-        self.assertEqual(another_pipeline.files_result_field, 'funny_field')
+        settings = {
+            "FILES_EXPIRES": random.randint(1, 1000),
+            "FILES_URLS_FIELD": random_string(),
+            "FILES_RESULT_FIELD": random_string(),
+            "FILES_STORE": self.tempdir
+        }
+        if not prefix:
+            return settings
+
+        return {prefix.upper() + "_" + k: v for k, v in settings.items()}
+
+    def test_different_settings_for_different_instances(self):
+        custom_settings = self._generate_fake_settings()
+        another_pipeline = FilesPipeline.from_settings(Settings(custom_settings))
+        one_pipeline = FilesPipeline(self.tempdir)
+        for pipe_attr, settings_attr in self.file_cls_attr_settings_map:
+            default_value = self.default_cls_settings[pipe_attr]
+            self.assertEqual(getattr(one_pipeline, pipe_attr), default_value)
+            custom_value = custom_settings[settings_attr]
+            pipe_attr_lower = pipe_attr.lower().replace("default_", "")
+            self.assertEqual(getattr(another_pipeline, pipe_attr_lower), custom_value)
 
 
 class TestS3FilesStore(unittest.TestCase):

From acbfdc618496f304c1a81f5f07ff0cacabfe8394 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Wed, 15 Jun 2016 15:12:18 +0200
Subject: [PATCH 0988/2974] [files_pipeline] ensure class attributes are
 preserved

dont override class attributes with default settings (same as in image pipeline).
---
 scrapy/settings/default_settings.py |  3 ---
 tests/test_pipeline_files.py        | 23 +++++++++++++++++++++++
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 1e447e9e2..b9d01e155 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -160,9 +160,6 @@ FEED_EXPORTERS_BASE = {
 }
 
 FILES_STORE_S3_ACL = 'private'
-FILES_EXPIRES = 90
-FILES_URLS_FIELD = 'file_urls'
-FILES_RESULT_FIELD = 'files'
 
 HTTPCACHE_ENABLED = False
 HTTPCACHE_DIR = 'httpcache'
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 760684c2b..4c64f6f3e 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -218,7 +218,20 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
 
         return {prefix.upper() + "_" + k: v for k, v in settings.items()}
 
+    def _generate_fake_pipeline(self):
+
+        class UserDefinedFilePipeline(FilesPipeline):
+            FILES_EXPIRES = random.randint(1001, 2000)
+            DEFAULT_FILES_URLS_FIELD = "alfa"
+            DEFAULT_FILES_RESULT_FIELD = "beta"
+
+        return UserDefinedFilePipeline
+
     def test_different_settings_for_different_instances(self):
+        """
+        If there are different instances with different settings they should keep
+        different settings.
+        """
         custom_settings = self._generate_fake_settings()
         another_pipeline = FilesPipeline.from_settings(Settings(custom_settings))
         one_pipeline = FilesPipeline(self.tempdir)
@@ -229,6 +242,16 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
             pipe_attr_lower = pipe_attr.lower().replace("default_", "")
             self.assertEqual(getattr(another_pipeline, pipe_attr_lower), custom_value)
 
+    def test_subclass_attributes_preserved_if_no_settings(self):
+        """
+        If subclasses override class attributes and there are no special settings those values should be kept.
+        """
+        pipe_cls = self._generate_fake_pipeline()
+        pipe = pipe_cls.from_settings(Settings({"FILES_STORE": self.tempdir}))
+        for pipe_attr, settings_attr in self.file_cls_attr_settings_map:
+            attr_lower = pipe_attr.lower().replace("default_", "")
+            self.assertEqual(getattr(pipe, attr_lower), getattr(pipe, pipe_attr))
+
 
 class TestS3FilesStore(unittest.TestCase):
     @defer.inlineCallbacks

From 539d34bce08c6c0bf19f8ae31e9d77271c3e22b7 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Wed, 15 Jun 2016 15:39:11 +0200
Subject: [PATCH 0989/2974] [media-pipeline, file-pipeline] allow setting
 custom settings for subclasses

* move key_for_pipe function to media pipeline so that file pipeline can use it
* use key_for_pipe in file pipeline so that users can define custom settings for subclasses easily
* add tests for file pipelines attributes and settings
---
 scrapy/pipelines/files.py    | 15 ++++++--
 scrapy/pipelines/images.py   | 40 ++++++++++----------
 scrapy/pipelines/media.py    | 16 ++++++++
 tests/test_pipeline_files.py | 73 +++++++++++++++++++++++++++++++-----
 4 files changed, 109 insertions(+), 35 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 3e6ad554d..b9c43dc3b 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -229,11 +229,18 @@ class FilesPipeline(MediaPipeline):
         
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
-        
+
+        cls_name = "FilesPipeline"
         self.store = self._get_store(store_uri)
-        self.expires = settings.getint('FILES_EXPIRES', self.EXPIRES)
-        self.files_urls_field = settings.get('FILES_URLS_FIELD', self.DEFAULT_FILES_URLS_FIELD)
-        self.files_result_field = settings.get('FILES_RESULT_FIELD', self.DEFAULT_FILES_RESULT_FIELD)
+        self.expires = settings.getint(
+            self._key_for_pipe('FILES_EXPIRES', cls_name), self.EXPIRES
+        )
+        self.files_urls_field = settings.get(
+            self._key_for_pipe('FILES_URLS_FIELD', cls_name), self.DEFAULT_FILES_URLS_FIELD
+        )
+        self.files_result_field = settings.get(
+            self._key_for_pipe('FILES_RESULT_FIELD', cls_name), self.DEFAULT_FILES_RESULT_FIELD
+        )
 
         super(FilesPipeline, self).__init__(download_func=download_func)
 
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 465d7c492..de616211e 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -53,27 +53,25 @@ class ImagesPipeline(FilesPipeline):
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
 
-        def key_for_pipe(key):
-            """
-            Allow setting settings for user defined ImagePipelines that inherit from base.
-
-            User can define setting key:
-
-            MYPIPELINENAME_IMAGE_SETTING_NAME = <some value>
-
-            and it will override default settings and class attributes.
-            """
-            class_name = self.__class__.__name__
-            if class_name == "ImagesPipeline":
-                return key
-            return "{}_{}".format(class_name.upper(), key)
-
-        self.expires = settings.getint(key_for_pipe('IMAGES_EXPIRES'), self.EXPIRES)
-        self.images_urls_field = settings.get(key_for_pipe('IMAGES_URLS_FIELD'), self.IMAGES_URLS_FIELD)
-        self.images_result_field = settings.get(key_for_pipe('IMAGES_RESULT_FIELD'), self.IMAGES_RESULT_FIELD)
-        self.min_width = settings.getint(key_for_pipe('IMAGES_MIN_WIDTH'), self.MIN_WIDTH)
-        self.min_height = settings.getint(key_for_pipe('IMAGES_MIN_HEIGHT'), self.MIN_HEIGHT)
-        self.thumbs = settings.get(key_for_pipe('IMAGES_THUMBS'), self.THUMBS)
+        cls_name = "ImagesPipeline"
+        self.expires = settings.getint(
+            self._key_for_pipe('IMAGES_EXPIRES', cls_name), self.EXPIRES
+        )
+        self.images_urls_field = settings.get(
+            self._key_for_pipe('IMAGES_URLS_FIELD', cls_name), self.IMAGES_URLS_FIELD
+        )
+        self.images_result_field = settings.get(
+            self._key_for_pipe('IMAGES_RESULT_FIELD', cls_name), self.IMAGES_RESULT_FIELD
+        )
+        self.min_width = settings.getint(
+            self._key_for_pipe('IMAGES_MIN_WIDTH', cls_name), self.MIN_WIDTH
+        )
+        self.min_height = settings.getint(
+            self._key_for_pipe('IMAGES_MIN_HEIGHT', cls_name), self.MIN_HEIGHT
+        )
+        self.thumbs = settings.get(
+            self._key_for_pipe('IMAGES_THUMBS', cls_name), self.THUMBS
+        )
 
     @classmethod
     def from_settings(cls, settings):
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 21b8b8986..740312f8f 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -27,6 +27,22 @@ class MediaPipeline(object):
     def __init__(self, download_func=None):
         self.download_func = download_func
 
+
+    def _key_for_pipe(self, key, base_class_name):
+        """
+        Allow setting settings for user defined MediaPipelines that inherit from base.
+
+        User can define setting key:
+
+        MYPIPELINENAME_IMAGE_SETTING_NAME = <some value>
+
+        and it will override default settings and class attributes.
+        """
+        class_name = self.__class__.__name__
+        if class_name == base_class_name:
+            return key
+        return "{}_{}".format(class_name.upper(), key)
+
     @classmethod
     def from_crawler(cls, crawler):
         try:
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 4c64f6f3e..fd54b7229 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -191,9 +191,9 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
         "DEFAULT_FILES_RESULT_FIELD": "files"
     }
     file_cls_attr_settings_map = {
-        ("EXPIRES", "FILES_EXPIRES"),
-        ("DEFAULT_FILES_URLS_FIELD", "FILES_URLS_FIELD"),
-        ("DEFAULT_FILES_RESULT_FIELD", "FILES_RESULT_FIELD")
+        ("EXPIRES", "FILES_EXPIRES", "expires"),
+        ("DEFAULT_FILES_URLS_FIELD", "FILES_URLS_FIELD", "files_urls_field"),
+        ("DEFAULT_FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field")
     }
 
     def setUp(self):
@@ -216,7 +216,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
         if not prefix:
             return settings
 
-        return {prefix.upper() + "_" + k: v for k, v in settings.items()}
+        return {prefix.upper() + "_" + k if k != "FILES_STORE" else k: v for k, v in settings.items()}
 
     def _generate_fake_pipeline(self):
 
@@ -235,12 +235,11 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
         custom_settings = self._generate_fake_settings()
         another_pipeline = FilesPipeline.from_settings(Settings(custom_settings))
         one_pipeline = FilesPipeline(self.tempdir)
-        for pipe_attr, settings_attr in self.file_cls_attr_settings_map:
+        for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
             default_value = self.default_cls_settings[pipe_attr]
             self.assertEqual(getattr(one_pipeline, pipe_attr), default_value)
             custom_value = custom_settings[settings_attr]
-            pipe_attr_lower = pipe_attr.lower().replace("default_", "")
-            self.assertEqual(getattr(another_pipeline, pipe_attr_lower), custom_value)
+            self.assertEqual(getattr(another_pipeline, pipe_ins_attr), custom_value)
 
     def test_subclass_attributes_preserved_if_no_settings(self):
         """
@@ -248,9 +247,63 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
         """
         pipe_cls = self._generate_fake_pipeline()
         pipe = pipe_cls.from_settings(Settings({"FILES_STORE": self.tempdir}))
-        for pipe_attr, settings_attr in self.file_cls_attr_settings_map:
-            attr_lower = pipe_attr.lower().replace("default_", "")
-            self.assertEqual(getattr(pipe, attr_lower), getattr(pipe, pipe_attr))
+        for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
+            self.assertEqual(getattr(pipe, pipe_ins_attr), getattr(pipe, pipe_attr))
+
+    def test_subclass_attrs_preserved_custom_settings(self):
+        """
+        If file settings are defined but they are not defined for subclass class attributes
+        should be preserved.
+        """
+        pipeline_cls = self._generate_fake_pipeline()
+        settings = self._generate_fake_settings()
+        pipeline = pipeline_cls.from_settings(Settings(settings))
+        for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
+            value = getattr(pipeline, pipe_ins_attr)
+            self.assertEqual(value, getattr(pipeline, pipe_attr))
+
+    def test_no_custom_settings_for_subclasses(self):
+        """
+        If there are no settings for subclass and no subclass attributes, pipeline should use
+        attributes of base class.
+        """
+        class UserDefinedFilesPipeline(FilesPipeline):
+            pass
+
+        user_pipeline = UserDefinedFilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir}))
+        for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
+            # Values from settings for custom pipeline should be set on pipeline instance.
+            custom_value = self.default_cls_settings.get(pipe_attr.upper())
+            self.assertEqual(getattr(user_pipeline, pipe_ins_attr), custom_value)
+
+    def test_custom_settings_for_subclasses(self):
+        """
+        If there are custom settings for subclass and NO class attributes, pipeline should use custom
+        settings.
+        """
+        class UserDefinedFilesPipeline(FilesPipeline):
+            pass
+
+        prefix = UserDefinedFilesPipeline.__name__.upper()
+        settings = self._generate_fake_settings(prefix=prefix)
+        user_pipeline = UserDefinedFilesPipeline.from_settings(Settings(settings))
+        for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
+            # Values from settings for custom pipeline should be set on pipeline instance.
+            custom_value = settings.get(prefix + "_" + settings_attr)
+            self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value)
+
+    def test_custom_settings_and_class_attrs_for_subclasses(self):
+        """
+        If there are custom settings for subclass AND class attributes
+        setting keys are preferred and override attributes.
+        """
+        pipeline_cls = self._generate_fake_pipeline()
+        prefix = pipeline_cls.__name__.upper()
+        settings = self._generate_fake_settings(prefix=prefix)
+        user_pipeline = pipeline_cls.from_settings(Settings(settings))
+        for pipe_cls_attr, settings_attr, pipe_inst_attr  in self.file_cls_attr_settings_map:
+            custom_value = settings.get(prefix + "_" + settings_attr)
+            self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value)
 
 
 class TestS3FilesStore(unittest.TestCase):

From 10b79c9b3ec438653b25d531c30888849e080aa5 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Wed, 15 Jun 2016 15:49:11 +0200
Subject: [PATCH 0990/2974] [files-pipeline] update docs with note about
 settings

for subclasses.
---
 docs/topics/media-pipeline.rst | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 96339d03d..6cbac913c 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -191,9 +191,10 @@ For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/or
 If you need something more complex and want to override the custom pipeline
 behaviour, see :ref:`topics-media-pipeline-override`.
 
-.. note:: If you have multiple image pipelines inheriting from ImagePipeline and you want to have different settings in different pipelines
-   you can set setting keys preceded with uppercase name of your pipeline class. E.g. if your pipeline is called
-   MyPipeline and you want to have custom IMAGES_URLS_FIELD you define setting MYPIPELINE_IMAGES_URLS_FIELD.
+If you have multiple image pipelines inheriting from ImagePipeline and you want to have different settings in different pipelines
+you can set setting keys preceded with uppercase name of your pipeline class. E.g. if your pipeline is called
+MyPipeline and you want to have custom IMAGES_URLS_FIELD you define setting MYPIPELINE_IMAGES_URLS_FIELD and your custom
+settings will be used.
 
 
 Additional features
@@ -218,6 +219,14 @@ specifies the delay in number of days::
 
 The default value for both settings is 90 days.
 
+If you have pipeline that subclasses FilesPipeline and you'd like to have different setting
+for it you can set setting keys preceded by uppercase class name. E.g. given pipeline class
+called MyPipeline you can set setting key:
+
+    MYPIPELINE_FILES_EXPIRES = 180
+
+and pipeline class MyPipeline will have expiration time set to 180.
+
 .. _topics-images-thumbnails:
 
 Thumbnail generation for images

From 73cc066caad75e3dc7aa3e8343d0bc89baab4cae Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Fri, 17 Jun 2016 09:21:58 +0200
Subject: [PATCH 0991/2974] [docs] add note about windows + python3

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index cbcd4d613..ac87b449d 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -72,7 +72,7 @@ features are still missing (and some may never be ported).
 Almost all builtin extensions/middlewares are expected to work.
 However, we are aware of some limitations in Python 3:
 
-- Scrapy has not been tested on Windows with Python 3
+- Scrapy does not work on Windows with Python 3
 - Sending emails is not supported
 - FTP download handler is not supported
 - Telnet console is not supported

From 07d1605586d36a65538a4e9a8b3432fb9ffc0a97 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Fri, 17 Jun 2016 13:28:51 +0200
Subject: [PATCH 0992/2974] [docs] warnings about windows + python 3 in faq and
 install

---
 docs/faq.rst           | 2 +-
 docs/intro/install.rst | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 82e1f3422..35551d3cc 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -71,7 +71,7 @@ What Python versions does Scrapy support?
 
 Scrapy is supported under Python 2.7 and Python 3.3+.
 Python 2.6 support was dropped starting at Scrapy 0.20.
-Python 3 support was added in Scrapy 1.1.
+Python 3 support was added in Scrapy 1.1. Python 3 is not yet supported on Windows.
 
 Did Scrapy "steal" X from Django?
 ---------------------------------
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 25520b4b9..16b8761c2 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -11,7 +11,7 @@ Installing Scrapy
 
 The installation steps assume that you have the following things installed:
 
-* `Python`_ 2.7
+* `Python`_ 2.7 or above 3.3
 
 * `pip`_ and `setuptools`_ Python packages. Nowadays `pip`_ requires and
   installs `setuptools`_ if not installed. Python 2.7.9 and later include
@@ -85,6 +85,10 @@ Windows
 
       pip install Scrapy
 
+.. note::
+     Python 3 is not supported on Windows. Installation of Scrapy on Windows
+     with Python 3 will fail.
+
 Ubuntu 9.10 or above
 --------------------
 

From fa4d0cdfe5df7165c220e2c17c2de14262fc713e Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Mon, 20 Jun 2016 12:39:09 +0200
Subject: [PATCH 0993/2974] [FilesPipeline, ImagesPipeline] fix for cls attrs
 with DEFAULT prefix

some class attributes for ImagePipeline and FilesPipeline had DEFAULT prefix. These
attributes should be preserved as well, if users subclasses define values for
DEFAULT_<CLS_ATTRIBUTE_NAME> attribute this value should be preserved.
---
 scrapy/pipelines/files.py     |  8 ++++++--
 scrapy/pipelines/images.py    | 15 +++++++++++----
 tests/test_pipeline_files.py  | 29 ++++++++++++++++++++++-------
 tests/test_pipeline_images.py | 13 +++++++++++++
 4 files changed, 52 insertions(+), 13 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index b9c43dc3b..73eda5f34 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -235,11 +235,15 @@ class FilesPipeline(MediaPipeline):
         self.expires = settings.getint(
             self._key_for_pipe('FILES_EXPIRES', cls_name), self.EXPIRES
         )
+        if not hasattr(self, "FILES_URLS_FIELD"):
+            self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD
+        if not hasattr(self, "FILES_RESULT_FIELD"):
+            self.FILES_RESULT_FIELD = self.DEFAULT_FILES_RESULT_FIELD
         self.files_urls_field = settings.get(
-            self._key_for_pipe('FILES_URLS_FIELD', cls_name), self.DEFAULT_FILES_URLS_FIELD
+            self._key_for_pipe('FILES_URLS_FIELD', cls_name), self.FILES_URLS_FIELD
         )
         self.files_result_field = settings.get(
-            self._key_for_pipe('FILES_RESULT_FIELD', cls_name), self.DEFAULT_FILES_RESULT_FIELD
+            self._key_for_pipe('FILES_RESULT_FIELD', cls_name), self.FILES_RESULT_FIELD
         )
 
         super(FilesPipeline, self).__init__(download_func=download_func)
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index de616211e..73377e2c2 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -44,8 +44,8 @@ class ImagesPipeline(FilesPipeline):
     MIN_HEIGHT = 0
     EXPIRES = 0
     THUMBS = {}
-    IMAGES_URLS_FIELD = 'image_urls'
-    IMAGES_RESULT_FIELD = 'images'
+    DEFAULT_IMAGES_URLS_FIELD = 'image_urls'
+    DEFAULT_IMAGES_RESULT_FIELD = 'images'
 
     def __init__(self, store_uri, download_func=None, settings=None):
         super(ImagesPipeline, self).__init__(store_uri, settings=settings, download_func=download_func)
@@ -57,11 +57,18 @@ class ImagesPipeline(FilesPipeline):
         self.expires = settings.getint(
             self._key_for_pipe('IMAGES_EXPIRES', cls_name), self.EXPIRES
         )
+        if not hasattr(self, "IMAGES_RESULT_FIELD"):
+            self.IMAGES_RESULT_FIELD = self.DEFAULT_IMAGES_RESULT_FIELD
+        if not hasattr(self, "IMAGES_URLS_FIELD"):
+            self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD
+
+        default_images_urls_field = getattr(self, "IMAGES_URLS_FIELD", "DEFAULT_IMAGES_URLS_FIELD")
         self.images_urls_field = settings.get(
-            self._key_for_pipe('IMAGES_URLS_FIELD', cls_name), self.IMAGES_URLS_FIELD
+            self._key_for_pipe('IMAGES_URLS_FIELD', cls_name), default_images_urls_field
         )
+        default_images_result_field = getattr(self, "IMAGES_RESULT_FIELD", "DEFAULT_IMAGES_RESULT_FIELD")
         self.images_result_field = settings.get(
-            self._key_for_pipe('IMAGES_RESULT_FIELD', cls_name), self.IMAGES_RESULT_FIELD
+            self._key_for_pipe('IMAGES_RESULT_FIELD', cls_name), default_images_result_field
         )
         self.min_width = settings.getint(
             self._key_for_pipe('IMAGES_MIN_WIDTH', cls_name), self.MIN_WIDTH
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index fd54b7229..bda2a2199 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -187,13 +187,13 @@ class FilesPipelineTestCaseFields(unittest.TestCase):
 class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
     default_cls_settings = {
         "EXPIRES": 90,
-        "DEFAULT_FILES_URLS_FIELD": "file_urls",
-        "DEFAULT_FILES_RESULT_FIELD": "files"
+        "FILES_URLS_FIELD": "file_urls",
+        "FILES_RESULT_FIELD": "files"
     }
     file_cls_attr_settings_map = {
         ("EXPIRES", "FILES_EXPIRES", "expires"),
-        ("DEFAULT_FILES_URLS_FIELD", "FILES_URLS_FIELD", "files_urls_field"),
-        ("DEFAULT_FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field")
+        ("FILES_URLS_FIELD", "FILES_URLS_FIELD", "files_urls_field"),
+        ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field")
     }
 
     def setUp(self):
@@ -221,9 +221,9 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
     def _generate_fake_pipeline(self):
 
         class UserDefinedFilePipeline(FilesPipeline):
-            FILES_EXPIRES = random.randint(1001, 2000)
-            DEFAULT_FILES_URLS_FIELD = "alfa"
-            DEFAULT_FILES_RESULT_FIELD = "beta"
+            EXPIRES = 1001
+            FILES_URLS_FIELD = "alfa"
+            FILES_RESULT_FIELD = "beta"
 
         return UserDefinedFilePipeline
 
@@ -239,6 +239,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
             default_value = self.default_cls_settings[pipe_attr]
             self.assertEqual(getattr(one_pipeline, pipe_attr), default_value)
             custom_value = custom_settings[settings_attr]
+            self.assertNotEqual(default_value, custom_value)
             self.assertEqual(getattr(another_pipeline, pipe_ins_attr), custom_value)
 
     def test_subclass_attributes_preserved_if_no_settings(self):
@@ -248,6 +249,8 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
         pipe_cls = self._generate_fake_pipeline()
         pipe = pipe_cls.from_settings(Settings({"FILES_STORE": self.tempdir}))
         for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
+            custom_value = getattr(pipe, pipe_ins_attr)
+            self.assertNotEqual(custom_value, self.default_cls_settings[pipe_attr])
             self.assertEqual(getattr(pipe, pipe_ins_attr), getattr(pipe, pipe_attr))
 
     def test_subclass_attrs_preserved_custom_settings(self):
@@ -260,6 +263,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
         pipeline = pipeline_cls.from_settings(Settings(settings))
         for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
             value = getattr(pipeline, pipe_ins_attr)
+            self.assertNotEqual(value, self.default_cls_settings[pipe_attr])
             self.assertEqual(value, getattr(pipeline, pipe_attr))
 
     def test_no_custom_settings_for_subclasses(self):
@@ -290,6 +294,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
         for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
             # Values from settings for custom pipeline should be set on pipeline instance.
             custom_value = settings.get(prefix + "_" + settings_attr)
+            self.assertNotEqual(custom_value, self.default_cls_settings[pipe_attr])
             self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value)
 
     def test_custom_settings_and_class_attrs_for_subclasses(self):
@@ -303,8 +308,18 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
         user_pipeline = pipeline_cls.from_settings(Settings(settings))
         for pipe_cls_attr, settings_attr, pipe_inst_attr  in self.file_cls_attr_settings_map:
             custom_value = settings.get(prefix + "_" + settings_attr)
+            self.assertNotEqual(custom_value, self.default_cls_settings[pipe_cls_attr])
             self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value)
 
+    def test_cls_attrs_with_DEFAULT_prefix(self):
+        class UserDefinedFilesPipeline(FilesPipeline):
+            DEFAULT_FILES_RESULT_FIELD = "this"
+            DEFAULT_FILES_URLS_FIELD = "that"
+
+        pipeline = UserDefinedFilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir}))
+        self.assertEqual(pipeline.files_result_field, "this")
+        self.assertEqual(pipeline.files_urls_field, "that")
+
 
 class TestS3FilesStore(unittest.TestCase):
     @defer.inlineCallbacks
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index a2dd5aa28..6ccd9791e 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -304,6 +304,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
         for pipe_attr, settings_attr in self.img_cls_attribute_names:
             # Instance attribute (lowercase) must be equal to class attribute (uppercase).
             attr_value = getattr(pipeline, pipe_attr.lower())
+            self.assertNotEqual(attr_value, self.default_pipeline_settings[pipe_attr])
             self.assertEqual(attr_value, getattr(pipeline, pipe_attr))
 
     def test_subclass_attrs_preserved_custom_settings(self):
@@ -317,6 +318,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
         for pipe_attr, settings_attr in self.img_cls_attribute_names:
             # Instance attribute (lowercase) must be equal to class attribute (uppercase).
             value = getattr(pipeline, pipe_attr.lower())
+            self.assertNotEqual(value, self.default_pipeline_settings[pipe_attr])
             self.assertEqual(value, getattr(pipeline, pipe_attr))
 
     def test_no_custom_settings_for_subclasses(self):
@@ -347,6 +349,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
         for pipe_attr, settings_attr in self.img_cls_attribute_names:
             # Values from settings for custom pipeline should be set on pipeline instance.
             custom_value = settings.get(prefix + "_" + settings_attr)
+            self.assertNotEqual(custom_value, self.default_pipeline_settings[pipe_attr])
             self.assertEqual(getattr(user_pipeline, pipe_attr.lower()), custom_value)
 
     def test_custom_settings_and_class_attrs_for_subclasses(self):
@@ -360,8 +363,18 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
         user_pipeline = pipeline_cls.from_settings(Settings(settings))
         for pipe_attr, settings_attr in self.img_cls_attribute_names:
             custom_value = settings.get(prefix + "_" + settings_attr)
+            self.assertNotEqual(custom_value, self.default_pipeline_settings[pipe_attr])
             self.assertEqual(getattr(user_pipeline, pipe_attr.lower()), custom_value)
 
+    def test_cls_attrs_with_DEFAULT_prefix(self):
+        class UserDefinedImagePipeline(ImagesPipeline):
+            DEFAULT_IMAGES_URLS_FIELD = "something"
+            DEFAULT_IMAGES_RESULT_FIELD = "something_else"
+
+        pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir}))
+        self.assertEqual(pipeline.images_result_field, "something_else")
+        self.assertEqual(pipeline.images_urls_field, "something")
+
 
 def _create_image(format, *a, **kw):
     buf = TemporaryFile()

From 10a2c46e12e54e666de743b9ceca09c18370ccec Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 20 Jun 2016 13:39:37 +0200
Subject: [PATCH 0994/2974] [HttpCompressionMiddleware] Do not decompress
 binary/octet-stream responses

---
 scrapy/utils/gz.py                            |  9 +++++++--
 ...st_downloadermiddleware_httpcompression.py | 20 +++++++++++++++++++
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index d035f9fdf..b2f737193 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -53,5 +53,10 @@ def gunzip(data):
 
 def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
-    ctype = response.headers.get('Content-Type', b'')
-    return ctype in (b'application/x-gzip', b'application/gzip')
+    ctype = response.headers.get('Content-Type', b'').lower()
+    cenc = response.headers.get('Content-Encoding', b'').lower()
+    return (
+        ctype in (b'application/x-gzip', b'application/gzip') or
+        (ctype in (b'application/octet-stream', b'binary/octet-stream') and
+         cenc in (b'gzip', b'x-gzip'))
+    )
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 24955a515..b2426946d 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -145,6 +145,26 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(response.headers['Content-Encoding'], b'gzip')
         self.assertEqual(response.headers['Content-Type'], b'application/gzip')
 
+    def test_process_response_gzip_app_octetstream_contenttype(self):
+        response = self._getresponse('gzip')
+        response.headers['Content-Type'] = 'application/octet-stream'
+        request = response.request
+
+        newresponse = self.mw.process_response(request, response, self.spider)
+        self.assertIs(newresponse, response)
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        self.assertEqual(response.headers['Content-Type'], b'application/octet-stream')
+
+    def test_process_response_gzip_binary_octetstream_contenttype(self):
+        response = self._getresponse('x-gzip')
+        response.headers['Content-Type'] = 'binary/octet-stream'
+        request = response.request
+
+        newresponse = self.mw.process_response(request, response, self.spider)
+        self.assertIs(newresponse, response)
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        self.assertEqual(response.headers['Content-Type'], b'binary/octet-stream')
+
     def test_process_response_head_request_no_decode_required(self):
         response = self._getresponse('gzip')
         response.headers['Content-Type'] = 'application/gzip'

From d9343463cb69d01a43ed12d8931bbcd86d2aae09 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 21 Jun 2016 13:26:32 +0200
Subject: [PATCH 0995/2974] Add "Host" header in CONNECT requests to HTTPS
 proxies

---
 scrapy/core/downloader/handlers/http11.py | 33 +++++++++++++----------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 88c6b9480..d02027dd1 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -93,7 +93,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
     for it.
     """
 
-    _responseMatcher = re.compile(b'HTTP/1\.. 200')
+    _responseMatcher = re.compile(b'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
 
     def __init__(self, reactor, host, port, proxyConf, contextFactory,
                  timeout=30, bindAddress=None):
@@ -115,13 +115,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         self._protocol = protocol
         return protocol
 
-    def processProxyResponse(self, bytes):
+    def processProxyResponse(self, rcvd_bytes):
         """Processes the response from the proxy. If the tunnel is successfully
         created, notifies the client that we are ready to send requests. If not
         raises a TunnelError.
         """
         self._protocol.dataReceived = self._protocolDataReceived
-        if  TunnelingTCP4ClientEndpoint._responseMatcher.match(bytes):
+        respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(rcvd_bytes)
+        if respm and int(respm.group('status')) == 200:
             try:
                 # this sets proper Server Name Indication extension
                 # but is only available for Twisted>=14.0
@@ -134,9 +135,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
                                               self._protocolFactory)
             self._tunnelReadyDeferred.callback(self._protocol)
         else:
+            if respm:
+                extra = {'status': int(respm.group('status')),
+                         'reason': respm.group('reason').strip()}
+            else:
+                extra = rcvd_bytes[:32]
             self._tunnelReadyDeferred.errback(
-                TunnelError('Could not open CONNECT tunnel with proxy %s:%s' % (
-                    self._host, self._port)))
+                TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % (
+                    self._host, self._port, extra)))
 
     def connectFailed(self, reason):
         """Propagates the errback to the appropriate deferred."""
@@ -151,23 +157,22 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         return self._tunnelReadyDeferred
 
 
-def tunnel_request_data(host, port, proxy_auth_header=None):
+def tunnel_request_data(host, port, proxy_auth_header=None, host_header=True):
     r"""
     Return binary content of a CONNECT request.
 
     >>> from scrapy.utils.python import to_native_str as s
     >>> s(tunnel_request_data("example.com", 8080))
-    'CONNECT example.com:8080 HTTP/1.1\r\n\r\n'
+    'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
     >>> s(tunnel_request_data("example.com", 8080, b"123"))
-    'CONNECT example.com:8080 HTTP/1.1\r\nProxy-Authorization: 123\r\n\r\n'
+    'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n'
     >>> s(tunnel_request_data(b"example.com", "8090"))
-    'CONNECT example.com:8090 HTTP/1.1\r\n\r\n'
+    'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n'
     """
-    tunnel_req = (
-        b'CONNECT ' +
-        to_bytes(host, encoding='ascii') + b':' +
-        to_bytes(str(port)) +
-        b' HTTP/1.1\r\n')
+    host_value = to_bytes(host, encoding='ascii') + b':' + to_bytes(str(port))
+    tunnel_req = b'CONNECT ' + host_value + b' HTTP/1.1\r\n'
+    if host_header:
+        tunnel_req += b'Host: ' + host_value + b'\r\n'
     if proxy_auth_header:
         tunnel_req += b'Proxy-Authorization: ' + proxy_auth_header + b'\r\n'
     tunnel_req += b'\r\n'

From 6539277f995bcfc9310ea92dce180cfa11d982c9 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 21 Jun 2016 17:14:41 +0200
Subject: [PATCH 0996/2974] Fix CONNECT request timeout (with an ugly hack)

---
 tests/test_downloader_handlers.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 45a806f2e..c63c64d86 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -401,7 +401,13 @@ class UriResource(resource.Resource):
         return self
 
     def render(self, request):
-        return request.uri
+        # Note: this is an ugly hack for CONNECT request timeout test.
+        #       Returning some data here fail SSL/TLS handshake
+        # ToDo: implement proper HTTPS proxy tests, not faking them.
+        if request.method != b'CONNECT':
+            return request.uri
+        else:
+            return b''
 
 
 class HttpProxyTestCase(unittest.TestCase):

From b67440dec01f27e5a2cb5984d809c840b53afcf6 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Mon, 4 Jul 2016 16:35:59 +0200
Subject: [PATCH 0997/2974] docs on Scrapy on Windows + Python 3

---
 docs/faq.rst           | 5 ++++-
 docs/intro/install.rst | 4 ++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 35551d3cc..415331515 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -71,7 +71,10 @@ What Python versions does Scrapy support?
 
 Scrapy is supported under Python 2.7 and Python 3.3+.
 Python 2.6 support was dropped starting at Scrapy 0.20.
-Python 3 support was added in Scrapy 1.1. Python 3 is not yet supported on Windows.
+Python 3 support was added in Scrapy 1.1.
+
+.. note::
+    Python 3 is not yet supported on Windows.
 
 Did Scrapy "steal" X from Django?
 ---------------------------------
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 16b8761c2..3364c3b31 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -86,8 +86,8 @@ Windows
       pip install Scrapy
 
 .. note::
-     Python 3 is not supported on Windows. Installation of Scrapy on Windows
-     with Python 3 will fail.
+     Python 3 is not supported on Windows. This is because Scrapy core requirement Twisted does not support
+     Python 3 on Windows.
 
 Ubuntu 9.10 or above
 --------------------

From 2a92ffb409b5f58659fce796a5b051a011bba984 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Sat, 4 Jun 2016 10:32:29 +0300
Subject: [PATCH 0998/2974] Encourage descriptive PR titles

---
 docs/contributing.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 4e8330b3c..b0a435ad2 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -100,6 +100,11 @@ starting point is to send a pull request on GitHub. It can be simple enough to
 illustrate your idea, and leave documentation/tests for later, after the idea
 has been validated and proven useful. Alternatively, you can send an email to
 `scrapy-users`_ to discuss your idea first.
+When writing GitHub pull requests, try to keep titles short but descriptive.
+E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests"
+prefer "Fix hanging when exception occurs in start_requests (#411)"
+instead of "Fix for #411".
+Complete titles make it easy to skim through the issue tracker.
 
 Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports
 removal, etc) in separate commits than functional changes. This will make pull

From 49ac7de23162bd92dc0f2ba88967aa53bbc4af2b Mon Sep 17 00:00:00 2001
From: Raul Gallegos <raul.ogh@gmail.com>
Date: Tue, 5 Jul 2016 15:38:17 -0500
Subject: [PATCH 0999/2974] prioritize default headers over user agent

---
 docs/topics/settings.rst            | 6 +++---
 scrapy/settings/default_settings.py | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 7f49aacdb..c845c59b9 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -459,9 +459,9 @@ Default::
         'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
         'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
         'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
-        'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400,
-        'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
-        'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550,
+        'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
+        'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,
+        'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
         'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
         'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
         'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8f064f81e..e563e56aa 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -93,9 +93,9 @@ DOWNLOADER_MIDDLEWARES_BASE = {
     'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
     'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
     'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
-    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400,
-    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
-    'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550,
+    'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
+    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,
+    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
     'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
     'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
     'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,

From 37efdde3e3dd4efecf7dccd361fceab6c851d0ff Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 6 Jul 2016 14:18:59 +0200
Subject: [PATCH 1000/2974] Catch and ignore TLS verification exception for
 IP-address hosts

Fixes GH-2092
---
 scrapy/core/downloader/tls.py     | 5 +++++
 tests/test_downloader_handlers.py | 8 ++++++++
 2 files changed, 13 insertions(+)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 5fa8e2723..9bf6a575b 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -48,6 +48,11 @@ try:
                         'Remote certificate is not valid for hostname "{}"; {}'.format(
                             self._hostnameASCII, e))
 
+                except ValueError as e:
+                    logger.warning(
+                        'SSL/TLS verification failed for hostname "{}"; {}'.format(
+                            self._hostnameASCII, e))
+
 except ImportError:
     # ImportError should not matter for older Twisted versions
     # as the above is not used in the fallback ScrapyClientContextFactory
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 45a806f2e..fb73c43ee 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -335,6 +335,14 @@ class Https11WrongHostnameTestCase(Http11TestCase):
     certfile = 'keys/example-com.cert.pem'
 
 
+class Https11InvalidDNSId(Https11TestCase):
+    """Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
+
+    def setUp(self):
+        super(Https11InvalidDNSId, self).setUp()
+        self.host = '127.0.0.1'
+
+
 class Http11MockServerTestCase(unittest.TestCase):
     """HTTP 1.1 test case with MockServer"""
     if twisted_version < (11, 1, 0):

From 4273734744bc8083713b63c3aaff6ef6368c1727 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 6 Jul 2016 18:29:49 +0500
Subject: [PATCH 1001/2974] TST pin pytest-cov to 2.2.1; upgrade pytest

---
 tests/requirements-py3.txt | 4 ++--
 tests/requirements.txt     | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 2a89763a5..ed189c66c 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,6 +1,6 @@
-pytest==2.7.3
+pytest==2.9.2
 pytest-twisted
-pytest-cov
+pytest-cov==2.2.1
 testfixtures
 jmespath
 leveldb
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 8901fe16b..9d0c3c996 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -2,9 +2,9 @@
 mock
 mitmproxy==0.10.1
 netlib==0.10.1
-pytest==2.7.3
+pytest==2.9.2
 pytest-twisted
-pytest-cov
+pytest-cov==2.2.1
 jmespath
 testfixtures
 # optional for shell wrapper tests

From 15d0c89159851ae9916e49a27cbcb36335b1bc2e Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 6 Jul 2016 17:15:21 +0200
Subject: [PATCH 1002/2974] Cleanup unused argument

---
 scrapy/core/downloader/handlers/http11.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index d02027dd1..f07397a4f 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -157,7 +157,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         return self._tunnelReadyDeferred
 
 
-def tunnel_request_data(host, port, proxy_auth_header=None, host_header=True):
+def tunnel_request_data(host, port, proxy_auth_header=None):
     r"""
     Return binary content of a CONNECT request.
 
@@ -171,8 +171,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None, host_header=True):
     """
     host_value = to_bytes(host, encoding='ascii') + b':' + to_bytes(str(port))
     tunnel_req = b'CONNECT ' + host_value + b' HTTP/1.1\r\n'
-    if host_header:
-        tunnel_req += b'Host: ' + host_value + b'\r\n'
+    tunnel_req += b'Host: ' + host_value + b'\r\n'
     if proxy_auth_header:
         tunnel_req += b'Proxy-Authorization: ' + proxy_auth_header + b'\r\n'
     tunnel_req += b'\r\n'

From 1779f5fecacb49d59461799115fb02b6f80e974f Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Sat, 11 Jun 2016 20:44:08 -0300
Subject: [PATCH 1003/2974] enable genspider command outside projects

---
 scrapy/commands/genspider.py | 17 +++++++++++------
 tests/test_commands.py       |  7 +++++++
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 58bdb9156..d5498bb5c 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -25,7 +25,7 @@ def sanitize_module_name(module_name):
 
 class Command(ScrapyCommand):
 
-    requires_project = True
+    requires_project = False
     default_settings = {'LOG_ENABLED': False}
 
     def syntax(self):
@@ -94,14 +94,19 @@ class Command(ScrapyCommand):
             'classname': '%sSpider' % ''.join(s.capitalize() \
                 for s in module.split('_'))
         }
-        spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
-        spiders_dir = abspath(dirname(spiders_module.__file__))
+        if self.settings.get('NEWSPIDER_MODULE'):
+            spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
+            spiders_dir = abspath(dirname(spiders_module.__file__))
+        else:
+            spiders_module = None
+            spiders_dir = "."
         spider_file = "%s.py" % join(spiders_dir, module)
         shutil.copyfile(template_file, spider_file)
         render_templatefile(spider_file, **tvars)
-        print("Created spider %r using template %r in module:" % (name, \
-            template_name))
-        print("  %s.%s" % (spiders_module.__name__, module))
+        print("Created spider %r using template %r " % (name, \
+            template_name), end=('' if spiders_module else '\n'))
+        if spiders_module:
+            print("in module:\n  %s.%s" % (spiders_module.__name__, module))
 
     def _find_template(self, template):
         template_file = join(self.templates_dir, '%s.tmpl' % template)
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 2e47160d7..cf415a388 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -146,6 +146,13 @@ class GenspiderCommandTest(CommandTest):
         assert not exists(join(self.proj_mod_path, 'spiders', '%s.py' % self.project_name))
 
 
+class GenspiderStandaloneCommandTest(ProjectTest):
+
+    def test_generate_standalone_spider(self):
+        self.call('genspider', 'example', 'example.com')
+        assert exists(join(self.temp_path, 'example.py'))
+
+
 class MiscCommandsTest(CommandTest):
 
     def test_list(self):

From 8987b17730622708a5103b5e4cc024904870ed32 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Mon, 4 Jul 2016 18:59:53 -0300
Subject: [PATCH 1004/2974] remove references to Item classes in templates

---
 scrapy/templates/spiders/crawl.tmpl   | 4 +---
 scrapy/templates/spiders/csvfeed.tmpl | 4 +---
 scrapy/templates/spiders/xmlfeed.tmpl | 4 +---
 3 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index a179d16ff..154237d9c 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -3,8 +3,6 @@ import scrapy
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule
 
-from $project_name.items import ${ProjectName}Item
-
 
 class $classname(CrawlSpider):
     name = '$name'
@@ -16,7 +14,7 @@ class $classname(CrawlSpider):
     )
 
     def parse_item(self, response):
-        i = ${ProjectName}Item()
+        i = {}
         #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
         #i['name'] = response.xpath('//div[@id="name"]').extract()
         #i['description'] = response.xpath('//div[@id="description"]').extract()
diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl
index 69c606538..0544e0ae7 100644
--- a/scrapy/templates/spiders/csvfeed.tmpl
+++ b/scrapy/templates/spiders/csvfeed.tmpl
@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 from scrapy.spiders import CSVFeedSpider
 
-from $project_name.items import ${ProjectName}Item
-
 
 class $classname(CSVFeedSpider):
     name = '$name'
@@ -16,7 +14,7 @@ class $classname(CSVFeedSpider):
     #    return response
 
     def parse_row(self, response, row):
-        i = ${ProjectName}Item()
+        i = {}
         #i['url'] = row['url']
         #i['name'] = row['name']
         #i['description'] = row['description']
diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl
index 9c0910d23..d8ff61f6e 100644
--- a/scrapy/templates/spiders/xmlfeed.tmpl
+++ b/scrapy/templates/spiders/xmlfeed.tmpl
@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 from scrapy.spiders import XMLFeedSpider
 
-from $project_name.items import ${ProjectName}Item
-
 
 class $classname(XMLFeedSpider):
     name = '$name'
@@ -12,7 +10,7 @@ class $classname(XMLFeedSpider):
     itertag = 'item' # change it accordingly
 
     def parse_node(self, response, selector):
-        i = ${ProjectName}Item()
+        i = {}
         #i['url'] = selector.select('url').extract()
         #i['name'] = selector.select('name').extract()
         #i['description'] = selector.select('description').extract()

From 081595a2e49d17abf9c9d4d3763fa239f79a83e6 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Tue, 5 Jul 2016 22:48:18 -0300
Subject: [PATCH 1005/2974] document new genspider behavior

---
 docs/topics/commands.rst | 33 +++++++++++----------------------
 1 file changed, 11 insertions(+), 22 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 9a40a2c29..d7999900b 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -159,6 +159,7 @@ settings).
 Global commands:
 
 * :command:`startproject`
+* :command:`genspider`
 * :command:`settings`
 * :command:`runspider`
 * :command:`shell`
@@ -173,7 +174,6 @@ Project-only commands:
 * :command:`list`
 * :command:`edit`
 * :command:`parse`
-* :command:`genspider`
 * :command:`bench`
 
 .. command:: startproject
@@ -197,14 +197,9 @@ genspider
 ---------
 
 * Syntax: ``scrapy genspider [-t template] <name> <domain>``
-* Requires project: *yes*
+* Requires project: *no*
 
-Create a new spider in the current project.
-
-This is just a convenience shortcut command for creating spiders based on
-pre-defined templates, but certainly not the only way to create spiders. You
-can just create the spider source code files yourself, instead of using this
-command.
+Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
 
 Usage example::
 
@@ -215,22 +210,16 @@ Usage example::
       csvfeed
       xmlfeed
 
-    $ scrapy genspider -d basic
-    import scrapy
+    $ scrapy genspider example example.com
+    Created spider 'example' using template 'basic'
 
-    class $classname(scrapy.Spider):
-        name = "$name"
-        allowed_domains = ["$domain"]
-        start_urls = (
-            'http://www.$domain/',
-            )
+    $ scrapy genspider -t crawl scrapyorg scrapy.org
+    Created spider 'scrapyorg' using template 'crawl'
 
-        def parse(self, response):
-            pass
-
-    $ scrapy genspider -t basic example example.com
-    Created spider 'example' using template 'basic' in module:
-      mybot.spiders.example
+This is just a convenience shortcut command for creating spiders based on
+pre-defined templates, but certainly not the only way to create spiders. You
+can just create the spider source code files yourself, instead of using this
+command.
 
 .. command:: crawl
 

From 962eb11c73bdb56b81e96eddccde189a22f43963 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 11 Jul 2016 11:30:05 +0200
Subject: [PATCH 1006/2974] Simplify regex conditions

---
 scrapy/utils/gz.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 9fe88d108..afc7ed128 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -50,13 +50,12 @@ def gunzip(data):
                 raise
     return output
 
-_is_gzipped_re = re.compile(br'^application/(x-)?gzip\b', re.I)
-_is_octetstream_re = re.compile(br'^(application|binary)/octet-stream\b', re.I)
+_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
+_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search
 
 def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
     ctype = response.headers.get('Content-Type', b'')
     cenc = response.headers.get('Content-Encoding', b'').lower()
-    return (_is_gzipped_re.search(ctype) is not None or
-            (_is_octetstream_re.search(ctype) is not None and
-             cenc in (b'gzip', b'x-gzip')))
+    return (_is_gzipped(ctype) or
+            (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')))

From c22cc1096be5aaa3f381976ba0b70a014405dc4f Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Tue, 12 Jul 2016 13:58:36 +0200
Subject: [PATCH 1007/2974] [image_pipeline] style edits

* 80 characters line limit
* shortening some code
* removed dead code
* add doctest for _key_for_pipe function
---
 docs/topics/media-pipeline.rst | 15 ++++++++-------
 scrapy/pipelines/files.py      | 10 ++++++----
 scrapy/pipelines/images.py     | 18 ++++++++++--------
 scrapy/pipelines/media.py      | 17 ++++++++---------
 4 files changed, 32 insertions(+), 28 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 6cbac913c..f18789ab0 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -191,10 +191,11 @@ For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/or
 If you need something more complex and want to override the custom pipeline
 behaviour, see :ref:`topics-media-pipeline-override`.
 
-If you have multiple image pipelines inheriting from ImagePipeline and you want to have different settings in different pipelines
-you can set setting keys preceded with uppercase name of your pipeline class. E.g. if your pipeline is called
-MyPipeline and you want to have custom IMAGES_URLS_FIELD you define setting MYPIPELINE_IMAGES_URLS_FIELD and your custom
-settings will be used.
+If you have multiple image pipelines inheriting from ImagePipeline and you want
+to have different settings in different pipelines you can set setting keys
+preceded with uppercase name of your pipeline class. E.g. if your pipeline is
+called MyPipeline and you want to have custom IMAGES_URLS_FIELD you define
+setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used.
 
 
 Additional features
@@ -219,9 +220,9 @@ specifies the delay in number of days::
 
 The default value for both settings is 90 days.
 
-If you have pipeline that subclasses FilesPipeline and you'd like to have different setting
-for it you can set setting keys preceded by uppercase class name. E.g. given pipeline class
-called MyPipeline you can set setting key:
+If you have pipeline that subclasses FilesPipeline and you'd like to have
+different setting for it you can set setting keys preceded by uppercase
+class name. E.g. given pipeline class called MyPipeline you can set setting key:
 
     MYPIPELINE_FILES_EXPIRES = 180
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 73eda5f34..8cdc548f6 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -3,7 +3,7 @@ Files Pipeline
 
 See documentation in topics/media-pipeline.rst
 """
-
+import functools
 import hashlib
 import os
 import os.path
@@ -232,18 +232,20 @@ class FilesPipeline(MediaPipeline):
 
         cls_name = "FilesPipeline"
         self.store = self._get_store(store_uri)
+        resolve = functools.partial(self._key_for_pipe,
+                                    base_class_name=cls_name)
         self.expires = settings.getint(
-            self._key_for_pipe('FILES_EXPIRES', cls_name), self.EXPIRES
+            resolve('FILES_EXPIRES'), self.EXPIRES
         )
         if not hasattr(self, "FILES_URLS_FIELD"):
             self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD
         if not hasattr(self, "FILES_RESULT_FIELD"):
             self.FILES_RESULT_FIELD = self.DEFAULT_FILES_RESULT_FIELD
         self.files_urls_field = settings.get(
-            self._key_for_pipe('FILES_URLS_FIELD', cls_name), self.FILES_URLS_FIELD
+            resolve('FILES_URLS_FIELD'), self.FILES_URLS_FIELD
         )
         self.files_result_field = settings.get(
-            self._key_for_pipe('FILES_RESULT_FIELD', cls_name), self.FILES_RESULT_FIELD
+            resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD
         )
 
         super(FilesPipeline, self).__init__(download_func=download_func)
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 73377e2c2..0278ec32f 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -3,7 +3,7 @@ Images Pipeline
 
 See documentation in topics/media-pipeline.rst
 """
-
+import functools
 import hashlib
 import six
 
@@ -62,22 +62,24 @@ class ImagesPipeline(FilesPipeline):
         if not hasattr(self, "IMAGES_URLS_FIELD"):
             self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD
 
-        default_images_urls_field = getattr(self, "IMAGES_URLS_FIELD", "DEFAULT_IMAGES_URLS_FIELD")
+        resolve = functools.partial(self._key_for_pipe, base_class_name=cls_name)
+
         self.images_urls_field = settings.get(
-            self._key_for_pipe('IMAGES_URLS_FIELD', cls_name), default_images_urls_field
+            resolve('IMAGES_URLS_FIELD'),
+            self.IMAGES_URLS_FIELD
         )
-        default_images_result_field = getattr(self, "IMAGES_RESULT_FIELD", "DEFAULT_IMAGES_RESULT_FIELD")
         self.images_result_field = settings.get(
-            self._key_for_pipe('IMAGES_RESULT_FIELD', cls_name), default_images_result_field
+            resolve('IMAGES_RESULT_FIELD'),
+            self.IMAGES_RESULT_FIELD
         )
         self.min_width = settings.getint(
-            self._key_for_pipe('IMAGES_MIN_WIDTH', cls_name), self.MIN_WIDTH
+            resolve('IMAGES_MIN_WIDTH'), self.MIN_WIDTH
         )
         self.min_height = settings.getint(
-            self._key_for_pipe('IMAGES_MIN_HEIGHT', cls_name), self.MIN_HEIGHT
+            resolve('IMAGES_MIN_HEIGHT'), self.MIN_HEIGHT
         )
         self.thumbs = settings.get(
-            self._key_for_pipe('IMAGES_THUMBS', cls_name), self.THUMBS
+            resolve('IMAGES_THUMBS'), self.THUMBS
         )
 
     @classmethod
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 740312f8f..fe59b9f7c 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -28,18 +28,17 @@ class MediaPipeline(object):
         self.download_func = download_func
 
 
-    def _key_for_pipe(self, key, base_class_name):
+    def _key_for_pipe(self, key, base_class_name=None):
         """
-        Allow setting settings for user defined MediaPipelines that inherit from base.
-
-        User can define setting key:
-
-        MYPIPELINENAME_IMAGE_SETTING_NAME = <some value>
-
-        and it will override default settings and class attributes.
+        >>> result = MediaPipeline()._key_for_pipe("IMAGES")
+        >>> assert result == "IMAGES"
+        >>> class MyPipe(MediaPipeline):
+        ...     pass
+        >>> other_key = MyPipe()._key_for_pipe("IMAGES", base_class_name="MediaPipeline")
+        >>> assert other_key == "MYPIPE_IMAGES"
         """
         class_name = self.__class__.__name__
-        if class_name == base_class_name:
+        if class_name == base_class_name or not base_class_name:
             return key
         return "{}_{}".format(class_name.upper(), key)
 

From 9818c97f6e9ac721bb136e4d2a4bfe1ffc8cc047 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Tue, 12 Jul 2016 14:15:41 +0200
Subject: [PATCH 1008/2974] [image_pipeline] dont use assert in doctest

---
 scrapy/pipelines/media.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index fe59b9f7c..82b4b462e 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -30,12 +30,12 @@ class MediaPipeline(object):
 
     def _key_for_pipe(self, key, base_class_name=None):
         """
-        >>> result = MediaPipeline()._key_for_pipe("IMAGES")
-        >>> assert result == "IMAGES"
+        >>> MediaPipeline()._key_for_pipe("IMAGES")
+        'IMAGES'
         >>> class MyPipe(MediaPipeline):
         ...     pass
-        >>> other_key = MyPipe()._key_for_pipe("IMAGES", base_class_name="MediaPipeline")
-        >>> assert other_key == "MYPIPE_IMAGES"
+        >>> MyPipe()._key_for_pipe("IMAGES", base_class_name="MediaPipeline")
+        'MYPIPE_IMAGES'
         """
         class_name = self.__class__.__name__
         if class_name == base_class_name or not base_class_name:

From 33a39b368ffab6641997e7611d588487176716de Mon Sep 17 00:00:00 2001
From: Dracony <draconyster@gmail.com>
Date: Wed, 8 Jun 2016 17:24:08 +0200
Subject: [PATCH 1009/2974] added FEED_EXPORT_ENCODING setting to allow
 encoding specification

---
 docs/topics/feed-exports.rst        | 15 +++++++++++++++
 scrapy/exporters.py                 | 27 +++++++++++++++++++++------
 scrapy/extensions/feedexport.py     |  4 +++-
 scrapy/settings/default_settings.py |  1 +
 tests/test_feedexport.py            | 29 +++++++++++++++++++++++++++++
 5 files changed, 69 insertions(+), 7 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index dc5912fb9..f0d3e660d 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -207,6 +207,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_STORAGES`
  * :setting:`FEED_EXPORTERS`
  * :setting:`FEED_STORE_EMPTY`
+ * :setting:`FEED_EXPORT_ENCODING`
  * :setting:`FEED_EXPORT_FIELDS`
 
 .. currentmodule:: scrapy.extensions.feedexport
@@ -231,6 +232,20 @@ FEED_FORMAT
 The serialization format to be used for the feed. See
 :ref:`topics-feed-format` for possible values.
 
+.. setting:: FEED_EXPORT_ENCODING
+
+FEED_EXPORT_ENCODING
+--------------------
+
+Default: ``None``
+
+The encoding to be used for the feed.
+
+If unset or set to ``None`` (default) it uses UTF-8 for everything except JSON output,
+which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons.
+
+Use ``utf-8`` if you want UTF-8 for JSON too.
+
 .. setting:: FEED_EXPORT_FIELDS
 
 FEED_EXPORT_FIELDS
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index d507bcf31..c4b1b3476 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -33,9 +33,9 @@ class BaseItemExporter(object):
         If dont_fail is set, it won't raise an exception on unexpected options
         (useful for using with keyword arguments in subclasses constructors)
         """
+        self.encoding = options.pop('encoding', None)
         self.fields_to_export = options.pop('fields_to_export', None)
         self.export_empty_fields = options.pop('export_empty_fields', False)
-        self.encoding = options.pop('encoding', 'utf-8')
         if not dont_fail and options:
             raise TypeError("Unexpected options: %s" % ', '.join(options.keys()))
 
@@ -84,11 +84,13 @@ class JsonLinesItemExporter(BaseItemExporter):
     def __init__(self, file, **kwargs):
         self._configure(kwargs, dont_fail=True)
         self.file = file
+        kwargs.setdefault('ensure_ascii', not self.encoding)
         self.encoder = ScrapyJSONEncoder(**kwargs)
 
     def export_item(self, item):
         itemdict = dict(self._get_serialized_fields(item))
-        self.file.write(to_bytes(self.encoder.encode(itemdict) + '\n'))
+        data = self.encoder.encode(itemdict) + '\n'
+        self.file.write(to_bytes(data, self.encoding))
 
 
 class JsonItemExporter(BaseItemExporter):
@@ -96,6 +98,7 @@ class JsonItemExporter(BaseItemExporter):
     def __init__(self, file, **kwargs):
         self._configure(kwargs, dont_fail=True)
         self.file = file
+        kwargs.setdefault('ensure_ascii', not self.encoding)
         self.encoder = ScrapyJSONEncoder(**kwargs)
         self.first_item = True
 
@@ -111,7 +114,8 @@ class JsonItemExporter(BaseItemExporter):
         else:
             self.file.write(b',\n')
         itemdict = dict(self._get_serialized_fields(item))
-        self.file.write(to_bytes(self.encoder.encode(itemdict)))
+        data = self.encoder.encode(itemdict)
+        self.file.write(to_bytes(data, self.encoding))
 
 
 class XmlItemExporter(BaseItemExporter):
@@ -120,6 +124,8 @@ class XmlItemExporter(BaseItemExporter):
         self.item_element = kwargs.pop('item_element', 'item')
         self.root_element = kwargs.pop('root_element', 'items')
         self._configure(kwargs)
+        if not self.encoding:
+            self.encoding = 'utf-8'
         self.xg = XMLGenerator(file, encoding=self.encoding)
 
     def start_exporting(self):
@@ -169,9 +175,16 @@ class CsvItemExporter(BaseItemExporter):
 
     def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs):
         self._configure(kwargs, dont_fail=True)
+        if not self.encoding:
+            self.encoding = 'utf-8'
         self.include_headers_line = include_headers_line
-        file = file if six.PY2 else io.TextIOWrapper(file, line_buffering=True)
-        self.csv_writer = csv.writer(file, **kwargs)
+        self.stream = io.TextIOWrapper(
+            file,
+            line_buffering=False,
+            write_through=True,
+            encoding=self.encoding
+        ) if six.PY3 else file
+        self.csv_writer = csv.writer(self.stream, **kwargs)
         self._headers_not_written = True
         self._join_multivalued = join_multivalued
 
@@ -200,7 +213,7 @@ class CsvItemExporter(BaseItemExporter):
     def _build_row(self, values):
         for s in values:
             try:
-                yield to_native_str(s)
+                yield to_native_str(s, self.encoding)
             except TypeError:
                 yield s
 
@@ -263,6 +276,8 @@ class PythonItemExporter(BaseItemExporter):
             warnings.warn(
                 "PythonItemExporter will drop support for binary export in the future",
                 ScrapyDeprecationWarning)
+        if not self.encoding:
+            self.encoding = 'utf-8'
 
     def serialize_field(self, field, name, value):
         serializer = field.get('serializer', self._serialize_value)
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 5d90ef738..c3fc66de5 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -162,6 +162,7 @@ class FeedExporter(object):
         if not self.urifmt:
             raise NotConfigured
         self.format = settings['FEED_FORMAT'].lower()
+        self.export_encoding = settings['FEED_EXPORT_ENCODING']
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
         if not self._storage_supported(self.urifmt):
@@ -185,7 +186,8 @@ class FeedExporter(object):
         uri = self.urifmt % self._get_uri_params(spider)
         storage = self._get_storage(uri)
         file = storage.open(spider)
-        exporter = self._get_exporter(file, fields_to_export=self.export_fields)
+        exporter = self._get_exporter(file, fields_to_export=self.export_fields,
+            encoding=self.export_encoding)
         exporter.start_exporting()
         self.slot = SpiderSlot(file, exporter, storage, uri)
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8f064f81e..a31e216c4 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -139,6 +139,7 @@ FEED_URI = None
 FEED_URI_PARAMS = None  # a function to extend uri arguments
 FEED_FORMAT = 'jsonlines'
 FEED_STORE_EMPTY = False
+FEED_EXPORT_ENCODING = None
 FEED_EXPORT_FIELDS = None
 FEED_STORAGES = {}
 FEED_STORAGES_BASE = {
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 05f19d33e..353b21927 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -371,3 +371,32 @@ class FeedExportTest(unittest.TestCase):
             ]
             yield self.assertExported(items, ['egg', 'baz'], rows,
                                       settings=settings, ordered=True)
+
+    @defer.inlineCallbacks
+    def test_export_encoding(self):
+        items = [dict({'foo': u'Test\xd6'})]
+        header = ['foo']
+        
+        formats = {
+            'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'),
+            'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
+            'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
+            'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
+        }
+        
+        for format in formats:
+            settings = {'FEED_FORMAT': format}
+            data = yield self.exported_data(items, settings)
+            self.assertEqual(formats[format], data)
+            
+        formats = {
+            'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'),
+            'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),
+            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),
+            'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
+        }
+        
+        for format in formats:
+            settings = {'FEED_FORMAT': format, 'FEED_EXPORT_ENCODING': 'latin-1'}
+            data = yield self.exported_data(items, settings)
+            self.assertEqual(formats[format], data)

From 859bcf48206369d4ff40f85bd55bbfdcadf65e25 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 12 Jul 2016 17:53:19 +0200
Subject: [PATCH 1010/2974] Rephrase warning

---
 scrapy/core/downloader/tls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 9bf6a575b..5d2d68d82 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -50,7 +50,7 @@ try:
 
                 except ValueError as e:
                     logger.warning(
-                        'SSL/TLS verification failed for hostname "{}"; {}'.format(
+                        'Ignoring remote certificate verification failure for hostname "{}"; {}'.format(
                             self._hostnameASCII, e))
 
 except ImportError:

From 2489f84d3927fc294fa60f0efa6f2b9c886543b7 Mon Sep 17 00:00:00 2001
From: Kenneth Schnall <kas@users.noreply.github.com>
Date: Tue, 12 Jul 2016 15:34:50 -0400
Subject: [PATCH 1011/2974] Update practices.rst

---
 docs/topics/practices.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 5d40c745d..7dae68470 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -18,7 +18,7 @@ the typical way of running Scrapy via ``scrapy crawl``.
 Remember that Scrapy is built on top of the Twisted
 asynchronous networking library, so you need to run it inside the Twisted reactor.
 
-First utility you can use to run your spiders is
+The first utility you can use to run your spiders is
 :class:`scrapy.crawler.CrawlerProcess`. This class will start a Twisted reactor
 for you, configuring the logging and setting shutdown handlers. This class is
 the one used by all Scrapy commands.

From 005cf949b8618259427c811da35e89f29c73feed Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 13 Jul 2016 11:01:30 +0200
Subject: [PATCH 1012/2974] Change wording of warning + docstring for
 ScrapyClientTLSOptions

---
 scrapy/core/downloader/tls.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 5d2d68d82..00c94ee2e 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -34,9 +34,15 @@ try:
                                              VerificationError)
 
     class ScrapyClientTLSOptions(ClientTLSOptions):
-        # same as Twisted's ClientTLSOptions,
-        # except that VerificationError is caught
-        # and doesn't close the connection
+        """
+        SSL Client connection creator ignoring certificate verification errors
+        (for genuinely invalid certificates or bugs in verification code).
+
+        Same as Twisted's private _sslverify.ClientTLSOptions,
+        except that VerificationError and ValueError exceptions are caught,
+        so that the connection is not closed, only logging warnings.
+        """
+
         def _identityVerifyingInfoCallback(self, connection, where, ret):
             if where & SSL_CB_HANDSHAKE_START:
                 _maybeSetHostNameIndication(connection, self._hostnameBytes)
@@ -50,8 +56,9 @@ try:
 
                 except ValueError as e:
                     logger.warning(
-                        'Ignoring remote certificate verification failure for hostname "{}"; {}'.format(
-                            self._hostnameASCII, e))
+                        'Ignoring error while verifying certificate '
+                        'from host "{}" (exception: {})'.format(
+                            self._hostnameASCII, repr(e)))
 
 except ImportError:
     # ImportError should not matter for older Twisted versions

From ceecf3b26c542ffcd74d6a81233d58ed54ea3268 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Wed, 13 Jul 2016 16:17:34 +0200
Subject: [PATCH 1013/2974] [image_pipeline] minor style tweaks

---
 scrapy/pipelines/images.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 0278ec32f..a511887b6 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -48,22 +48,23 @@ class ImagesPipeline(FilesPipeline):
     DEFAULT_IMAGES_RESULT_FIELD = 'images'
 
     def __init__(self, store_uri, download_func=None, settings=None):
-        super(ImagesPipeline, self).__init__(store_uri, settings=settings, download_func=download_func)
+        super(ImagesPipeline, self).__init__(store_uri, settings=settings,
+                                             download_func=download_func)
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
 
-        cls_name = "ImagesPipeline"
+        resolve = functools.partial(self._key_for_pipe,
+                                    base_class_name="ImagesPipeline")
         self.expires = settings.getint(
-            self._key_for_pipe('IMAGES_EXPIRES', cls_name), self.EXPIRES
+            resolve("IMAGES_EXPIRES"), self.EXPIRES
         )
+
         if not hasattr(self, "IMAGES_RESULT_FIELD"):
             self.IMAGES_RESULT_FIELD = self.DEFAULT_IMAGES_RESULT_FIELD
         if not hasattr(self, "IMAGES_URLS_FIELD"):
             self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD
 
-        resolve = functools.partial(self._key_for_pipe, base_class_name=cls_name)
-
         self.images_urls_field = settings.get(
             resolve('IMAGES_URLS_FIELD'),
             self.IMAGES_URLS_FIELD

From 5fabed514e6edea5594b632ba105831abccc72e5 Mon Sep 17 00:00:00 2001
From: Kenneth Schnall <kas@users.noreply.github.com>
Date: Fri, 15 Jul 2016 23:14:11 -0400
Subject: [PATCH 1014/2974] Update logging.rst

---
 docs/topics/logging.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 062c6e518..b7aa6d985 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -151,7 +151,7 @@ These settings can be used to configure the logging:
 * :setting:`LOG_DATEFORMAT`
 * :setting:`LOG_STDOUT`
 
-First couple of settings define a destination for log messages. If
+The first couple of settings define a destination for log messages. If
 :setting:`LOG_FILE` is set, messages sent through the root logger will be
 redirected to a file named :setting:`LOG_FILE` with encoding
 :setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log

From de64a1f68afe24a246152f778b7f3b236b9ae439 Mon Sep 17 00:00:00 2001
From: Felipe Ruhland <felipe.ruhland@gmail.com>
Date: Tue, 19 Jul 2016 00:04:45 -0300
Subject: [PATCH 1015/2974] Fix scrapy.cfg validation

Signed-off-by: Felipe Ruhland <felipe.ruhland@gmail.com>
---
 scrapy/commands/startproject.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 0940d67f9..2218bd134 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -79,10 +79,11 @@ class Command(ScrapyCommand):
 
         if len(args) == 2:
             project_dir = args[1]
-            if exists(join(project_dir, 'scrapy.cfg')):
-                self.exitcode = 1
-                print('Error: scrapy.cfg already exists in %s' % abspath(project_dir))
-                return
+
+        if exists(join(project_dir, 'scrapy.cfg')):
+            self.exitcode = 1
+            print('Error: scrapy.cfg already exists in %s' % abspath(project_dir))
+            return
 
         if not self._is_valid_name(project_name):
             self.exitcode = 1

From fe088925a3eff9d1de5682efaa33a391c6dc7744 Mon Sep 17 00:00:00 2001
From: Felipe Ruhland <felipe.ruhland@gmail.com>
Date: Tue, 19 Jul 2016 00:12:39 -0300
Subject: [PATCH 1016/2974] Included implementation notes in docstring

Signed-off-by: Felipe Ruhland <felipe.ruhland@gmail.com>
---
 scrapy/commands/startproject.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 2218bd134..e3989baaf 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -51,6 +51,14 @@ class Command(ScrapyCommand):
         return False
 
     def _copytree(self, src, dst):
+        """
+        Since the original function always creates the directory, to resolve
+        the issue a new function had to be created. It's a simple copy and
+        was reduced for this case.
+
+        More info at:
+        https://github.com/scrapy/scrapy/pull/2005
+        """
         ignore = IGNORE
         names = os.listdir(src)
         ignored_names = ignore(src, names)

From f72991a96b9b61d07c76fb690b901c396ae8e048 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jes=C3=BAs=20Losada?= <dev@jesuslosada.com>
Date: Thu, 21 Jul 2016 21:18:22 +0200
Subject: [PATCH 1017/2974] Add the IMAGES_STORE_S3_ACL setting

---
 scrapy/pipelines/images.py          | 2 ++
 scrapy/settings/default_settings.py | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index a511887b6..964541d6a 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -88,6 +88,8 @@ class ImagesPipeline(FilesPipeline):
         s3store = cls.STORE_SCHEMES['s3']
         s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
+        s3store.POLICY = settings['IMAGES_STORE_S3_ACL']
+
         store_uri = settings['IMAGES_STORE']
         return cls(store_uri, settings=settings)
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 2c267b4cc..36879b74a 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -177,6 +177,8 @@ HTTPCACHE_GZIP = False
 
 HTTPPROXY_AUTH_ENCODING = 'latin-1'
 
+IMAGES_STORE_S3_ACL = 'private'
+
 ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
 
 ITEM_PIPELINES = {}

From c6a2ca4e933a43e5862c247ee71040e48943db91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jes=C3=BAs=20Losada?= <dev@jesuslosada.com>
Date: Fri, 22 Jul 2016 19:32:29 +0200
Subject: [PATCH 1018/2974] Document S3 capabilities in FilesPipeline and
 ImagesPipeline

---
 docs/topics/media-pipeline.rst | 19 ++++++++++++++++++-
 docs/topics/settings.rst       |  9 ---------
 2 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index f18789ab0..7f0cb1ba6 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -117,7 +117,7 @@ Supported Storage
 =================
 
 File system is currently the only officially supported storage, but there is
-also (undocumented) support for storing files in `Amazon S3`_.
+also support for storing files in `Amazon S3`_.
 
 .. _Amazon S3: https://aws.amazon.com/s3/
 
@@ -146,6 +146,23 @@ Where:
 * ``full`` is a sub-directory to separate full images from thumbnails (if
   used). For more info see :ref:`topics-images-thumbnails`.
 
+Amazon S3 storage
+-----------------
+
+.. setting:: FILES_STORE_S3_ACL
+.. setting:: IMAGES_STORE_S3_ACL
+
+:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3
+bucket. Scrapy will automatically upload the files to the bucket.
+
+For example, this is a valid :setting:`IMAGES_STORE` value::
+
+    IMAGES_STORE = 's3://bucket/images'
+
+You can modify the Access Control List (ACL) policy used for the stored files,
+which is defined by the :setting:`FILES_STORE_S3_ACL` and
+:setting:`IMAGES_STORE_S3_ACL` settings. By default, the ACL is set to
+``private``.
 
 Usage example
 =============
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index c845c59b9..b8d87bbe4 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -689,15 +689,6 @@ temporary files before uploading with :ref:`FTP feed storage <topics-feed-storag
 :ref:`Amazon S3 <topics-feed-storage-s3>`.
 
 
-.. setting:: FILES_STORE_S3_ACL
-
-FILES_STORE_S3_ACL
-------------------
-
-Default: ``'private'``
-
-S3-specific access control policy (ACL) for S3 files store.
-
 .. setting:: ITEM_PIPELINES
 
 ITEM_PIPELINES

From f193c52acd23a8a6539da9873ccbda27ee6475a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jes=C3=BAs=20Losada?= <dev@jesuslosada.com>
Date: Fri, 22 Jul 2016 19:47:29 +0200
Subject: [PATCH 1019/2974] Fix IMAGES_EXPIRES default value

The default value should be 90.
---
 scrapy/pipelines/images.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index a511887b6..f008a5420 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -42,7 +42,7 @@ class ImagesPipeline(FilesPipeline):
     # ImagesPipeline. They may be overridden by settings.
     MIN_WIDTH = 0
     MIN_HEIGHT = 0
-    EXPIRES = 0
+    EXPIRES = 90
     THUMBS = {}
     DEFAULT_IMAGES_URLS_FIELD = 'image_urls'
     DEFAULT_IMAGES_RESULT_FIELD = 'images'

From 7c3e3b484edfb30240b980a09a3719165f4f4a15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jes=C3=BAs=20Losada?= <dev@jesuslosada.com>
Date: Fri, 22 Jul 2016 20:03:49 +0200
Subject: [PATCH 1020/2974] Fix ImagesPipeline test settings

---
 tests/test_pipeline_images.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 6ccd9791e..8286582de 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -221,7 +221,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
     default_pipeline_settings = dict(
         MIN_WIDTH=0,
         MIN_HEIGHT=0,
-        EXPIRES=0,
+        EXPIRES=90,
         THUMBS={},
         IMAGES_URLS_FIELD='image_urls',
         IMAGES_RESULT_FIELD='images'

From e17fdd72766eea6bc81fcfb9e3c24a9929eeeba5 Mon Sep 17 00:00:00 2001
From: Diego Allen <diegoallen@gmail.com>
Date: Thu, 16 Jun 2016 21:56:24 -0400
Subject: [PATCH 1021/2974] Add set serialization to ScrapyJSONEncoder

---
 scrapy/utils/serialize.py     | 4 +++-
 tests/test_utils_serialize.py | 7 ++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py
index 8320be095..9dd72ea71 100644
--- a/scrapy/utils/serialize.py
+++ b/scrapy/utils/serialize.py
@@ -14,7 +14,9 @@ class ScrapyJSONEncoder(json.JSONEncoder):
     TIME_FORMAT = "%H:%M:%S"
 
     def default(self, o):
-        if isinstance(o, datetime.datetime):
+        if isinstance(o, set):
+            return list(o)
+        elif isinstance(o, datetime.datetime):
             return o.strftime("%s %s" % (self.DATE_FORMAT, self.TIME_FORMAT))
         elif isinstance(o, datetime.date):
             return o.strftime(self.DATE_FORMAT)
diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py
index 278cf91e3..6dc117779 100644
--- a/tests/test_utils_serialize.py
+++ b/tests/test_utils_serialize.py
@@ -23,9 +23,14 @@ class JsonEncoderTestCase(unittest.TestCase):
         ts = "10:11:12"
         dec = Decimal("1000.12")
         decs = "1000.12"
+        s = {'foo'}
+        ss = ['foo']
+        dt_set = {dt}
+        dt_sets = [dts]
 
         for input, output in [('foo', 'foo'), (d, ds), (t, ts), (dt, dts),
-                              (dec, decs), (['foo', d], ['foo', ds])]:
+                              (dec, decs), (['foo', d], ['foo', ds]), (s, ss),
+                              (dt_set, dt_sets)]:
             self.assertEqual(self.encoder.encode(input), json.dumps(output))
 
     def test_encode_deferred(self):

From 82215560891bd593fb4bdff7e77614cbb4ff4c6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jes=C3=BAs=20Losada?= <dev@jesuslosada.com>
Date: Fri, 22 Jul 2016 23:22:18 +0200
Subject: [PATCH 1022/2974] Add documentation on Amazon S3 ACLs

---
 docs/topics/media-pipeline.rst | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 7f0cb1ba6..f13a72a0b 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -162,7 +162,14 @@ For example, this is a valid :setting:`IMAGES_STORE` value::
 You can modify the Access Control List (ACL) policy used for the stored files,
 which is defined by the :setting:`FILES_STORE_S3_ACL` and
 :setting:`IMAGES_STORE_S3_ACL` settings. By default, the ACL is set to
-``private``.
+``private``. To make the files publicly available use the ``public-read``
+policy::
+
+    IMAGES_STORE_S3_ACL = 'public-read'
+
+For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
+
+.. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
 Usage example
 =============

From 472a8a47d0b213b45c9d3fc390628c8516650e81 Mon Sep 17 00:00:00 2001
From: Darshan Chaudhary <deathbullet@gmail.com>
Date: Sat, 21 Nov 2015 00:40:01 +0530
Subject: [PATCH 1023/2974] Change name, log once

---
 docs/topics/jobs.rst                |  2 ++
 docs/topics/settings.rst            | 10 ++++++++++
 scrapy/core/scheduler.py            | 10 +++++++---
 scrapy/settings/default_settings.py |  2 +-
 4 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 303076015..093cf9bb5 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -96,4 +96,6 @@ But this will::
         somearg = response.meta['somearg']
         print "the argument passed is:", somearg
 
+If you wish to log the requests that couldn't be serialized, you can set the ``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page. It is ``False`` by default.
+
 .. _pickle: http://docs.python.org/library/pickle.html
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index c845c59b9..d03da44c6 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1026,6 +1026,16 @@ Default: ``'scrapy.core.scheduler.Scheduler'``
 
 The scheduler to use for crawling.
 
+.. setting:: SCHEDULER_DEBUG
+
+SCHEDULER_DEBUG
+---------------
+
+Default: ``False``
+
+Setting to ``True`` will log the first unserializable request encountered.
+Stats collected.
+
 .. setting:: SPIDER_CONTRACTS
 
 SPIDER_CONTRACTS
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 8be9a0019..e0788a8c5 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -30,7 +30,7 @@ class Scheduler(object):
         pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
         dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
         mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
-        logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS')
+        logunser = settings.getbool('SCHEDULER_DEBUG')
         return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
                    stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass)
 
@@ -86,9 +86,13 @@ class Scheduler(object):
             self.dqs.push(reqd, -request.priority)
         except ValueError as e: # non serializable request
             if self.logunser:
-                logger.error("Unable to serialize request: %(request)s - reason: %(reason)s",
-                             {'request': request, 'reason': e},
+                msg = ("Unable to serialize request: %(request)s - reason: %(reason)s"
+                       " - no more unserializable requests will be logged"
+                       " (stats being collected)")
+                logger.error(msg, {'request': request, 'reason': e},
                              exc_info=True, extra={'spider': self.spider})
+                self.logunser = False
+            self.stats.inc_value('scheduler/unserializable', spider=self.spider)
             return
         else:
             return True
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 2c267b4cc..673df5129 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -191,7 +191,7 @@ LOG_STDOUT = False
 LOG_LEVEL = 'DEBUG'
 LOG_FILE = None
 
-LOG_UNSERIALIZABLE_REQUESTS = False
+SCHEDULER_DEBUG = False
 
 LOGSTATS_INTERVAL = 60.0
 

From 2d9e5937c6cd40e7fdafeddd5e54242815763b9a Mon Sep 17 00:00:00 2001
From: Darshan Chaudhary <deathbullet@gmail.com>
Date: Wed, 25 Nov 2015 12:34:11 +0530
Subject: [PATCH 1024/2974] Include deprecated warning

---
 scrapy/core/scheduler.py      | 2 +-
 scrapy/settings/deprecated.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index e0788a8c5..a3cb1bcff 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -30,7 +30,7 @@ class Scheduler(object):
         pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
         dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
         mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
-        logunser = settings.getbool('SCHEDULER_DEBUG')
+        logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS', settings.getbool('SCHEDULER_DEBUG'))
         return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
                    stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass)
 
diff --git a/scrapy/settings/deprecated.py b/scrapy/settings/deprecated.py
index c20c35c9c..91ed689e8 100644
--- a/scrapy/settings/deprecated.py
+++ b/scrapy/settings/deprecated.py
@@ -14,6 +14,7 @@ DEPRECATED_SETTINGS = [
     ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'),
     ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'),
     ('REDIRECT_MAX_METAREFRESH_DELAY', 'use METAREFRESH_MAXDELAY instead'),
+    ('LOG_UNSERIALIZABLE_REQUESTS', 'use SCHEDULER_DEBUG instead'),
 ]
 
 

From 0c77b6d033a01a6279734e1626ca7e49dc8d0784 Mon Sep 17 00:00:00 2001
From: darshanime <deathbullet@gmail.com>
Date: Mon, 25 Jul 2016 17:55:05 +0530
Subject: [PATCH 1025/2974] update docs for settings

---
 docs/topics/jobs.rst     |  4 +++-
 docs/topics/settings.rst | 11 +++++++++--
 scrapy/core/scheduler.py | 11 ++++++-----
 3 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 093cf9bb5..55105dcfd 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -96,6 +96,8 @@ But this will::
         somearg = response.meta['somearg']
         print "the argument passed is:", somearg
 
-If you wish to log the requests that couldn't be serialized, you can set the ``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page. It is ``False`` by default.
+If you wish to log the requests that couldn't be serialized, you can set the
+``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page.
+It is ``False`` by default.
 
 .. _pickle: http://docs.python.org/library/pickle.html
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index d03da44c6..d14f230e4 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1033,8 +1033,15 @@ SCHEDULER_DEBUG
 
 Default: ``False``
 
-Setting to ``True`` will log the first unserializable request encountered.
-Stats collected.
+Setting to ``True`` will log debug information about the requests scheduler.
+This currently logs(only once) if the requests cannot be serialized to disk.
+Stats counter (``scheduler/unserializable``) tracks the number of times this happens.
+
+Example entry in logs::
+
+    1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request: <request>
+    - reason: <exception> - no more unserializable requests will be logged (stats being
+    collected)
 
 .. setting:: SPIDER_CONTRACTS
 
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index a3cb1bcff..dcd6fb989 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -84,15 +84,16 @@ class Scheduler(object):
         try:
             reqd = request_to_dict(request, self.spider)
             self.dqs.push(reqd, -request.priority)
-        except ValueError as e: # non serializable request
+        except ValueError as e:  # non serializable request
             if self.logunser:
-                msg = ("Unable to serialize request: %(request)s - reason: %(reason)s"
-                       " - no more unserializable requests will be logged"
-                       " (stats being collected)")
+                msg = ("Unable to serialize request: %(request)s - reason:"
+                       " %(reason)s - no more unserializable requests will be"
+                       " logged (stats being collected)")
                 logger.error(msg, {'request': request, 'reason': e},
                              exc_info=True, extra={'spider': self.spider})
                 self.logunser = False
-            self.stats.inc_value('scheduler/unserializable', spider=self.spider)
+            self.stats.inc_value('scheduler/unserializable',
+                                 spider=self.spider)
             return
         else:
             return True

From a2e64525544c8fa5ba0d9ad85aedac0801b7e3de Mon Sep 17 00:00:00 2001
From: Darshan Chaudhary <deathbullet@gmail.com>
Date: Thu, 29 Oct 2015 15:40:07 +0530
Subject: [PATCH 1026/2974] Include signal example

---
 docs/topics/media-pipeline.rst |  2 +-
 docs/topics/signals.rst        | 31 +++++++++++++++++++++++++++++++
 scrapy/utils/misc.py           |  2 +-
 3 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 4ee4f1758..5ed6ce97d 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -7,7 +7,7 @@ Downloading and processing files and images
 .. currentmodule:: scrapy.pipelines.images
 
 Scrapy provides reusable :doc:`item pipelines </topics/item-pipeline>` for
-downloading fies attached to a particular item (for example, when you scrape
+downloading files attached to a particular item (for example, when you scrape
 products and also want to download their images locally). These pipelines share
 a bit of functionality and structure (we refer to them as media pipelines), but
 typically you'll either use the Files Pipeline or the Images Pipeline.
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 5dd3b9ef5..19d5e8df9 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -16,6 +16,37 @@ deliver the arguments that the handler receives.
 You can connect to signals (or send your own) through the
 :ref:`topics-api-signals`.
 
+Here is a simple example showing how you can catch signals and perform some action:
+::
+
+    from scrapy import signals
+    from scrapy import Spider
+
+
+    class DmozSpider(Spider):
+        name = "dmoz"
+        allowed_domains = ["dmoz.org"]
+        start_urls = [
+            "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
+            "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/",
+        ]
+
+
+        @classmethod
+        def from_crawler(cls, crawler, *args, **kwargs):
+            spider = super(DmozSpider, cls).from_crawler(crawler, *args, **kwargs)
+            crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
+            return spider
+
+
+        def spider_closed(self, spider):
+            spider.logger.info('Spider closed: %s', spider.name)
+
+
+        def parse(self, response):
+            pass
+
+
 Deferred signal handlers
 ========================
 
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 303a413d8..f20070b5d 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -1,4 +1,4 @@
-"""Helper functions which doesn't fit anywhere else"""
+"""Helper functions which don't fit anywhere else"""
 import re
 import hashlib
 from importlib import import_module

From d8e62e660db231446a430a2e59c660bced8e4a3e Mon Sep 17 00:00:00 2001
From: darshanime <deathbullet@gmail.com>
Date: Tue, 26 Jul 2016 20:46:12 +0530
Subject: [PATCH 1027/2974] update log demo print

---
 docs/topics/jobs.rst     | 2 +-
 docs/topics/settings.rst | 9 +++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 55105dcfd..4f9e38086 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -97,7 +97,7 @@ But this will::
         print "the argument passed is:", somearg
 
 If you wish to log the requests that couldn't be serialized, you can set the
-``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page.
+:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
 It is ``False`` by default.
 
 .. _pickle: http://docs.python.org/library/pickle.html
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index d14f230e4..e55c8bb4b 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1034,14 +1034,15 @@ SCHEDULER_DEBUG
 Default: ``False``
 
 Setting to ``True`` will log debug information about the requests scheduler.
-This currently logs(only once) if the requests cannot be serialized to disk.
+This currently logs (only once) if the requests cannot be serialized to disk.
 Stats counter (``scheduler/unserializable``) tracks the number of times this happens.
 
 Example entry in logs::
 
-    1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request: <request>
-    - reason: <exception> - no more unserializable requests will be logged (stats being
-    collected)
+    1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request:
+    <GET http://example.com> - reason: cannot serialize <Request at 0x9a7c7ec>
+    (type Request)> - no more unserializable requests will be logged
+    (see 'scheduler/unserializable' stats counter)
 
 .. setting:: SPIDER_CONTRACTS
 

From 643dbeffcf6951991fdeed0069690c55684e66ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Fri, 29 Jul 2016 17:13:59 +0300
Subject: [PATCH 1028/2974] Selector should not receive both response and text

---
 scrapy/selector/unified.py | 4 ++++
 tests/test_selector.py     | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 15f3d26df..64cb0232c 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -46,6 +46,10 @@ class Selector(_ParselSelector, object_ref):
     selectorlist_cls = SelectorList
 
     def __init__(self, response=None, text=None, type=None, root=None, _root=None, **kwargs):
+        if not(response is None or text is None):
+           raise ValueError('%s.__init__() received both response and text'
+                            % self.__class__.__name__)
+
         st = _st(response, type or self._default_type)
 
         if _root is not None:
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 141455b66..af0cc4de2 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -123,6 +123,10 @@ class SelectorTestCase(unittest.TestCase):
             sel.xpath('//p').extract_unquoted()
             self.assertSubstring('Use .extract() instead', str(w[-1].message))
 
+    def test_selector_bad_args(self):
+        with self.assertRaisesRegexp(ValueError, 'received both response and text'):
+            Selector(TextResponse(url='http://example.com', body=b''), text=u'')
+
 
 class DeprecatedXpathSelectorTest(unittest.TestCase):
 

From 4eec05351fd505dcd5e1060d25fb268675b13901 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Mon, 1 Aug 2016 11:37:51 -0300
Subject: [PATCH 1029/2974] remove bumpversion prerelease configuration I
 propose to remove the prerelease configuration from bumpversion, because I
 think its behavior is just too confusing.

The rational for this is that making the release procedure predictable
is more important than facilitating making pre-releases, which are sort
of the exception in the workflow.

The current configuration makes most common cases confusing:

* bug fix releases require you have to remember to use `--serialize "{major}.{minor}.{patch}"`
* to start a pre-release cycle, you actually use `minor` or `patch`
* to do the actual minor or patch release, you use `prerel`

Also, `prerel` breaks if you run it on a branch with a final release,
because it can't parse the prerelease information.

Therefore, I propose keeping the bumpversion defaults, and do the
prereleases (dev1, dev2, rc1, etc) manually (with `--new-version`),
which makes for a more predictable and intuitive behavior.

* `bumpversion minor` and `bumpversion patch` will work as expected
* pre-releases will be manually handled, but this seems a small overhead
  than remembering the details I mention above.

If you're happy with this, I'll also update [the wiki][1] with new
instructions.

[1]: https://github.com/scrapy/scrapy/wiki/Scrapy-release-procedure
---
 .bumpversion.cfg | 22 ----------------------
 1 file changed, 22 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 49a7e239f..12f5cb16c 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -3,27 +3,5 @@ current_version = 1.2.0dev2
 commit = True
 tag = True
 tag_name = {new_version}
-parse = ^
-	(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)
-	(?:(?P<prerel>[abc]|rc|dev)(?P<prerelversion>\d+))?
-serialize = 
-	{major}.{minor}.{patch}{prerel}{prerelversion}
-	{major}.{minor}.{patch}
 
 [bumpversion:file:scrapy/VERSION]
-
-[bumpversion:part:prerel]
-optional_value = gamma
-values = 
-	dev
-	rc
-	gamma
-
-[bumpversion:part:prerelversion]
-values = 
-	1
-	2
-	3
-	4
-	5
-

From 63876fc690ac500f353e8f9f7260d84e28c6eaf7 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Jr <valdir@scrapinghub.com>
Date: Mon, 1 Aug 2016 12:16:50 -0300
Subject: [PATCH 1030/2974] Remove download stats badge

---
 README.rst | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/README.rst b/README.rst
index a0138970a..74ae1f30d 100644
--- a/README.rst
+++ b/README.rst
@@ -6,10 +6,6 @@ Scrapy
    :target: https://pypi.python.org/pypi/Scrapy
    :alt: PyPI Version
 
-.. image:: https://img.shields.io/pypi/dm/Scrapy.svg
-   :target: https://pypi.python.org/pypi/Scrapy
-   :alt: PyPI Monthly downloads
-
 .. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
    :target: http://travis-ci.org/scrapy/scrapy
    :alt: Build Status

From e1d118d5ca3e0f4c3bc968fb16348bcbce322625 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 11 Jul 2016 16:26:42 +0200
Subject: [PATCH 1031/2974] Update release notes for upcoming 1.1.1 release

---
 docs/news.rst | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index ac87b449d..92481a741 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,46 @@
 Release notes
 =============
 
+1.1.1 (YYYY-MM-DD)
+------------------
+
+Bug fixes
+~~~~~~~~~
+
+- Add "Host" header in CONNECT requests to HTTPS proxies (:issue:`2069`)
+- Use response ``body`` when choosing response class (:issue:`2001`, fixes :issue:`2000`)
+- Do not fail on canonicalizing URLs with wrong netlocs (:issue:`2038`, fixes :issue:`2010`)
+- a few fixes for ``HttpCompressionMiddleware`` (and ``SitemapSpider``):
+  - Do not decode HEAD responses (:issue:`2008`, fixes :issue:`1899`)
+  - Handle charset parameter in gzip Content-Type header (:issue:`2050`, fixes :issue:`2049`)
+  - Do not decompress gzip octet-stream responses (:issue:`2065`, fixes :issue:`2063`)
+
+New features
+~~~~~~~~~~~~
+
+- Enable genspider command outside project folder (:issue:`2052`)
+- Retry HTTPS CONNECT ``TunnelError`` by default (:issue:`1974`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+- ``FEED_TEMPDIR`` setting at lexicographical position (:commit:`9b3c72c`)
+- Use idiomatic ``.extract_first()`` in overview (:issue:`1994`)
+- Update years in copyright notice (:commit:`c2c8036`)
+- Add information and example on errbacks (:issue:`1995`)
+- Use "url" variable in downloader middleware example (:issue:`2015`)
+- Grammar fixes (:issue:`2054`)
+- New FAQ entry on using BeautifulSoup in spider callbacks (:issue:`2048`)
+- Add notes about scrapy not working on Windows with Python 3 (:issue:`2060`)
+- Encourage complete titles in pull requests (:issue:`2026`)
+
+Tests
+~~~~~
+
+- Upgrade py.test requirement on Travis CI and Pin pytest-cov to 2.2.1 (:issue:`2095`)
+
+
 1.1.0 (2016-05-11)
 ------------------
 

From 928e93f8f337f9f247ccd4ed031dcfe9d4af1cb0 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 13 Jul 2016 17:36:13 +0200
Subject: [PATCH 1032/2974] Update notes with latest 1.1 commits

---
 docs/news.rst | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 92481a741..0a6a40587 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -10,12 +10,24 @@ Bug fixes
 ~~~~~~~~~
 
 - Add "Host" header in CONNECT requests to HTTPS proxies (:issue:`2069`)
-- Use response ``body`` when choosing response class (:issue:`2001`, fixes :issue:`2000`)
-- Do not fail on canonicalizing URLs with wrong netlocs (:issue:`2038`, fixes :issue:`2010`)
+- Use response ``body`` when choosing response class
+  (:issue:`2001`, fixes :issue:`2000`)
+- Do not fail on canonicalizing URLs with wrong netlocs
+  (:issue:`2038`, fixes :issue:`2010`)
 - a few fixes for ``HttpCompressionMiddleware`` (and ``SitemapSpider``):
+
   - Do not decode HEAD responses (:issue:`2008`, fixes :issue:`1899`)
-  - Handle charset parameter in gzip Content-Type header (:issue:`2050`, fixes :issue:`2049`)
-  - Do not decompress gzip octet-stream responses (:issue:`2065`, fixes :issue:`2063`)
+  - Handle charset parameter in gzip Content-Type header
+    (:issue:`2050`, fixes :issue:`2049`)
+  - Do not decompress gzip octet-stream responses
+    (:issue:`2065`, fixes :issue:`2063`)
+
+- Catch (and ignore with a warning) exception when verifying certificate
+  against IP-address hosts (:issue:`2094`, fixes :issue:`2092`)
+- Make ``FilesPipeline`` and ``ImagesPipeline`` backward compatible again
+  regarding the use of legacy class attributes for customization
+  (:issue:`1989`, fixes :issue:`1985`)
+
 
 New features
 ~~~~~~~~~~~~
@@ -32,7 +44,7 @@ Documentation
 - Update years in copyright notice (:commit:`c2c8036`)
 - Add information and example on errbacks (:issue:`1995`)
 - Use "url" variable in downloader middleware example (:issue:`2015`)
-- Grammar fixes (:issue:`2054`)
+- Grammar fixes (:issue:`2054`, :issue:`2120`)
 - New FAQ entry on using BeautifulSoup in spider callbacks (:issue:`2048`)
 - Add notes about scrapy not working on Windows with Python 3 (:issue:`2060`)
 - Encourage complete titles in pull requests (:issue:`2026`)

From 5b1d98b8c81418b8e7d802bd7f2c3f37475c3ed8 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 13 Jul 2016 17:59:11 +0200
Subject: [PATCH 1033/2974] Update 1.1.1 release date

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 0a6a40587..82b59d5a8 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,7 +3,7 @@
 Release notes
 =============
 
-1.1.1 (YYYY-MM-DD)
+1.1.1 (2016-07-13)
 ------------------
 
 Bug fixes

From 04f93e096cd57b0a08187c7e28a11eb10de17d27 Mon Sep 17 00:00:00 2001
From: Lorena <lguerra@cid-inc.com>
Date: Thu, 4 Aug 2016 10:04:47 -0700
Subject: [PATCH 1034/2974] updated graph for organization/clarity

---
 docs/topics/_images/scrapy_architecture_02.png | Bin 0 -> 34097 bytes
 docs/topics/architecture.rst                   |   5 ++---
 2 files changed, 2 insertions(+), 3 deletions(-)
 create mode 100644 docs/topics/_images/scrapy_architecture_02.png

diff --git a/docs/topics/_images/scrapy_architecture_02.png b/docs/topics/_images/scrapy_architecture_02.png
new file mode 100644
index 0000000000000000000000000000000000000000..9178b35414eb46bccda1b075d3cb8ac5bbfa0405
GIT binary patch
literal 34097
zcmbTeby!s0+crFkN=Zmcr-U?0rywOIA{~Q-ba$(ibSWKz2-3~apwtl3ok~jg49&NO
z`+I-S{d~`Re1AOq;2!24XRp2XT34Lcd0v~)mnw2NSmam`2n0t#{`o5i1g#GOxgCjd
z8+_s=<oX=^!gP?=afU$fj8OmG^74?0gg|H^3eTUt_L$z9bJW(GNME~VXqBQ3tTnRf
zic-c;W-Kv?&zBcP)R->4m~+Wh5ts^F^{h~FXKD#kNX;2hIr!2id@i??{DnEA|EcTD
zv#GwP$1WK}w($yo?J8wlJFDVHacu&u1@iT<P0&T|na~B^f#3G?r#wH?#h{aX$2eaq
z;gM5V=v^Accr-MG%=ns`l0pDx3K?dOW<veOm_ohMQGgd(Q?&oQ2HJqP?_zNN^NL?_
z3%r-&2QUA;O2Y4fxBoi*--Fzo4*Az0|D67>LH;@Ye+=@!pToaq^3Ndu>pA@UMv{wZ
zBWK7dDSghjVV3IbiD6-3R3dJJxL~#Nq-tE2A_YiDNk=NQjm-x#zCD0IB>RK6A}Itk
zU%ws%9|i;jJO$<f#~-~}+W*CR{Ch6{I{lxW`PXv3^JLXtyqwy-vYUoYU9B?IRecTg
z20JE6lEwUAd;9-40z?Xd_1mq%RoKAOv#!2<Y5%W_aJmX}>4u9iY^Jz`RjX!Z+ph)M
z5VYP~jf|X0PZ(HZ<Kl9!(|5X?*sGzg&cJo<BMwdwV2Lg$DN$)~{5)s3yRq@20z1P3
z%md!}3j5A&D?t@%1wCEel$4YUKYs%Q16uqf*Ha}5HJxJ3!m)z6hK1cky%=l`l5qVJ
zmavqZM<H!P!^6X~vy>Qjw%ojD_buFis7K8HKK7y@Ctrqj)z=GeH}LH=m>C$P_pS^#
zvyR&5U`~|g=Q}pcjZCPXd>*rdF)wJ*;>SRfRwj09^=?7(S-xO8y~}11&m35H!J@0L
zuP2g8UMEy5S#l6umUD7;c0O29;LonmX1DT6AO-(y!=Jjat99t8rIk3c?$R&^PJCP0
z9i&;P(EyF^8W|n^<UlIf&uDXO1->z7XK!z>weUa}+#5K=GfSJA4Aeg92<#xw4LB1g
zHn+F7(l*~KFC-$mzDd%81#hUYpPrfVB)CQEdStqGJM|Zn!~?TLx1R%Bwc0k7Q@Kdb
zfhtiUusCn6ZC7_THsa<cH`FYbJZAko31SaT_djQ|6c-nBaBw(fbuEQ@3sl2kh20H>
zg}s}{O#aC3V=CMbeg9mX7mh91j~YuSHb3vG=Z)l3hYqYMo~+)5^aW;Y=^It+uH=0!
z>XCm`S>UeV=y?F=4BR`v<kb+N#65kpg*;ygfhlcQl$oBbRc$fHLy0{e$F?npx8&z5
zG;>|}wM=YAo)0Og5uM7GX|o?3-8+^m2pl~ITN<(CXwTysl-!)z>ub4NsOjHre_OM#
zy0KB;FFwx=8}dp{)h#N}MhBBa;;#Csqi1eh9OE~p$JL+I{T}uX7@USz@+%as&~KpZ
z!&acKcO{NE(+(z5f)KVNmBfv)S2}()H#_UIF`A!j#XVZu!%8AuxO?W=X!dqGarRR-
zi@Lh{_VzZB<E_@T!T5CSs+Kj*XIk-Rim(eCO(GdJqLziy@xYTyKeF7hO6nE0DI5O8
z-W71WP9be?DyL@cU>(Mx=NC~kcVj4a)xqNJdn~g_oc1VsX1|$j9yPJHvB|-ufUHJl
zY*EskM_1<&_n(!gl(l*1uITm`R(U6`&{r_)fpyueoQ^N1`l-6zm%uD8Oo=PRq;OyW
znNkreg=J;E8P8^S4Z544cK7zJ1&t?nk+r_ycB=>3>?UbIX!h<XpRzfK=k3U_IWB4^
zN*o~HH*?LOZD@Q9V^EY!Nljhx@EIBK{iGm#I%Obf=R=qvF?8ql`_9hJKq0*-&eD<f
z<;laaY<Q!Hz0-X{%QLbEPqq~~7U<2<Aatn%_Jw`5tWk*1WdmT+mB8xXOST%9!<!1x
z8A=d(g<>OM=qhdY4Iw>AJ7?i^E7nmZA-(flVCK-2Fjg5gVG)rrU)Hc}Tp4;3VIi;<
zVJTabSy%n3K77$#gcx^3siU)5HlimK60GI!sJFrR$>8)h)o6M+=!;fDuCsMXR+%uw
zcS_ul8cmTuoE~AZPlRTys(U7<Cns6DevRI<7JA_u22D~w9f-*(sRz!K5UM&b$m`Z?
ze;fE04i3&J>OQ<5#W5g%O=Hf1v*b=KEG>up0tc#QH;)T^`+#?B^lV^(UnJo&!A9Hc
zPo&giiX*Q!ey-EafNz}oo!wm5l(bq@=Eu^$#X?QD+^k*%*IHQi_cjyj79&OAU0_)@
zk2$A2#~?9JT!2}vPmllp{d>l~rOBi#3S@=k@$!YsxhjcZgaA$VH>yXo*l&*!GxpeY
zZd;Sj)M?T3zZw^g<QJ^aA7qxP<Pj=}WGrbWHD496>7SRH!Fq)`>oyM#4nRZzQHa#C
zbb`%a^HtYj0)v}?fPkv%AIQ7pm+?4|@czx?dVNk1kfTcC=NHUJy`nvBG=nXxqYBH*
zhgO{OAL(60-9aT@_{)t}stVqorG0p)3aOySM`5W<-8aNqLs?Wm?Q`>o#v0($+^!em
zr=P!MWj&kkNh^YIGlMkMnq8^QuB@zV2h+=V{+$#0+i1_}FP^P_)Qr(A7G|Lw*r~q1
zPFTp?T&Af)ju3?UuKY88NFS!y6r_v@7$Z2$UVjdF<?iN|Tj)J$T>ex*V89-N?>Ss~
zf69ryzrSB4q%A87F0{*m*7`zLl^?<$o12@P+)tHuOnFpGyjtHpe$+lYHAO}`H8vCo
zfgDPM??50iZx!d_n@pERR_3En|I6{W`~QPK{s(lR)#4!o0H!sJFf|3spt;CU<H&0|
z5QS;p2|Y|t!g&5QOPif2Jctt-^^2Reh)$w~MPlLgOQXjuFOSZ$FMLv>Zoa!2Xk706
z_3P!*k_jPUevt$}4-Y@jgO9Q+8f4W1#H8^xm?#LvW?*Ag;BT|_S}@JHMkQ`Yx_mNO
zzWw#fviw>PU$qCeiYU0G9lBpHKq*cb+V>itcel#d*yH52puWjdg6e<9kiR}Tv6s%e
zi!fvc*P#1~7$1J6@`(`z9+qn-M@RRK9E^N?FcEzGJVRs{Hjg<=SXo&I*DZViE;G$;
z^ZoN^eo6UTVS$p%vs-lx_>s*bPe09{#ptAmK0oW~vgcxZLX6Lp_}Da4%-hYuAts@N
zApALfM@L5$+Hk77xAf5bpWcxPALOF8gHI=osYrj2bFMOfvgX`VXZq6Or=QjdBZOYI
z{CEh<DdKK%tA0Q6u<YR*BrI#NV#~|Mcrh^`K<|ZO{;*+o%;gVsn!zG%l0X}{w$GUr
zJ8qP!3zRaImCPMVi7m@)QLZE2BAqx8$T@Jj*)CFP?>V!vdD}~4-PO{NVq`akWa^eY
zb8yH&WEAH)PlSrn=%-J6F6~3g9NW>@5N?S%(yri6=90jreN0;m>M#|`kwDAk$CX#Q
z?zPjQhOX6T-`uLRiej>Sp8YhW-V6_``IK;w1Hh6g{S;=gn?&zo<Vco9T*hmo`m<wM
z$7<_YpK7I`9fz1Q(wW@FMLlJv400*{cPj%URK9<*{ys;xK2#uAOlbKPbP6lwhhN#p
z2~>>c$3E~EBpkT>BAyz9W0a2*sp@hRgqMl!ju){q?D-z<ma0@}rA0bd{X@H9T5?rZ
zQ&2$I%h*HE1g}&@?V%LvB-V;!<O0pv)%h)}+}P8;vkPyI+z>l;`5CzOS3<%TO3DZ`
z2M@Nvb#XPF?vp9t)8XIG7n8QfsAcyg$Md1~M{`=rBD83iP7d5e;bte#G)YM24K+LC
zKg$t9v>8Lre=^#bGCWR+N$!76pPX41gpangbg9UTWTc~L#X9z|4Z=5$w<K16&tB1x
z@&+CgD$C5I5*cCIJCuQO&xEa6^FuQCE%uTH2^VW*NBP8M8yL|ApZ-br2#tg_59-Fu
zhBsS$)A*m|?uWGL5b2qyt2h1G!^+GG1~)e}R>D78!Y|4Zh@XAFsEzCx+6gy_m<=(j
zvaQbQ$!YPe&H|fMGyKxbnEQ@t)lg`t5(Tlhra67=<FAg7GH)rs@0Hzuuh;NgWo2@R
zlthd9%bTChIXp^<>xV1tPn~SGQcL>rF->FpD2YXcY|Y(h$<v!ZSdJCUM*RjW)l1XR
zF@|}XG0vVyIF3PQdX+FTj=`iB5-dDapJps;@m9zDq4;vd<ml^bZOy9@5yH^4(C2Yl
zbu3;@6;>K`hMC@9tqFm_e19D8H(tf)-E2m`OV#|!(@EDk`FTd<a^qjE<^T=yo}*K(
z1gP($L5fJ?Z~ivyimLQVD`jXy7PREYA#Hbs=g0%zs5_SD)5;CA(osl~j4bbds^>7X
z4D-^U`1Q@5E9!375}Sq~^oc957P}M@|LSK#oSaadH^6+kGsn>ggb7270z=4BQ-p!_
z*txtO+O&@7r2IR{WQwN!Y_SJm;Z;^ygy4_;(49K>>_QEm@Iwtojn$tQmbR``XL0Wz
zg=gsS@Vk-U%^vEv0B76tC0EpI0po+Lhi&ceRC6!#zPsB!Y_(POd>e|3s~gd|MTH(?
zIR+{R-qFxMe~Ks?xtf&j-mNw>lR2HzQtOO}2C<Ipv5V(7RVMtf4nHBmq)5IesJh}^
zUu4o#DG&q?hrbnBW1%WVNv0QqA=pBw%RwMTgK5gB7zlqp8m2TbwTp{m);lVCbEcAu
zr&FBl$0z(ej-n4iw9{TcS`b%ZcsIcjQ~#3Pjfue1-Qq8ZE?3+mznHkW#WRR2jd*WD
zJMSh94b4NET^fpTp7#n^z<i6i9zPyjzbOVk3JCK!+PNGlF-p~B#Dnme9~%pt`=*91
z37@!f@CX`Mt;Gi>r`*wzEJ_Ic0s=8oNGB)DLJD&TiNuRk3hjfM&L|YF@bmHyH>UR=
z-V1k#SH&$O1(d%-DSKlASlRkY4Qzia+)C(7mS2Wt;cv%qmgnb#gaHB%m_|&Nm-Ns`
zTjCOl+tH(kqMLK!CSbP$DTJuPGB6GL$CwJB_G#<6(=Vix#hclO(^#4ps(--4aoG5A
zulR|wZ;Nb$$PKbX+25-4_8*%f5lcTEyM$yj(^DBa_;)*JO?6%46-o_@A2+%2aej^O
zkKM);X;?)Ypnq@i)Fe7G28R~=yV$R$EQ{SeGrIQmLQ?s@sZFK72d=)?YpV&pW6K8q
z$G3GT8MlZh8{eZ$<jLNoi~8ss8vH)+cd3uSYk%xx@<|hZ;nA|fPfTubtNx*%<gdP7
z@HjrB^A@HEa@LC1_Aw{0$agUc_3nbxzK%+oj$m<6ah`--`KqM&>Fea9>OvgwV-|G!
z6q`iVoHnNW_<r3RU$vox;a4Em5*G*%iOrIV&ywym3%aqJbdEznJe(d2Zs32oDey3#
z|C!e%KN0YTLELcVq0LRAAP{2APPqxZ|D1eFoX1<<E2DrvYA93$kx9%YRZ89HJy^De
zkYJ0DM<Ab{d<_Ez`Hpzm0eZ6Q=emD=#LL27(f75Qc;Vy(rwVPAgP-sC@cpgaR2^PK
zwe^#|?@9X)T8-D61xccmhjHP;S9D<(<+NP^&j|P*{={oF_H3+K<o{M?F8HDtqXM2W
z({tt3B|LrWkmRE*dS{m<I!HF`*#lMSQGTAN;0ZD_h@TPkANoM_dzNwKVee|G^QQS~
zrt2$o{zavcUy>jd@C3!@QJ)d0xSTCOv`zgmJ^*<foE`D>!#8F*qBpb>?7_gQBEp*5
z@1K&LT|EMR@=Alr_p^e3>B@9ct`onU(C4pd0dHA^pPk6o|E#JNW{XUF80UCaR!ip{
zac!Vj{1G^!f-wvqGdnsB?Nz@f!&CMAOkA}!9J;{oQ2)8Wjn$uQ`6;JV<?K@BlLSg)
zhxx9X)bkC<!)Wm{@HP<N8|snehqQC8!(Z(1Jzv~87{QbH#8xP-30;_-K5IimAN_3q
zz2Iy~%v=2W@7E2(Tzmbwd^Zt)ZD^)Lbf68`g1&EV!dBkR)xjmJ_H@s8$1pepV+a*`
zdKnMcfBlmBh;oCgp02z!Dgz8PH$DZl<hMGx8S5UGuE_yoqt;$KLUip55~rq@J99LJ
zrCh9R4F{{!P3I=Z#|H=X1&@z8{r6{`r9Q@YMK#shFTe+5lr}I0G8{y5D=Lx^wNLMR
zk_l^4qaraITi?XRS>`J;Nw7Ux`_|0Rdwe{XFC3y(T=3eje=FYYEgGGU*d!rYT@bf@
zb9sE1;?r{m^CK?LV)@Xi*`aZMg8K<cznr>hY-SMXGd11tbM)$J|3>rZdZ$^9&-y1_
zQW$hs$R7b#@QmjD<eDcV)QPfn;?n*#^kCa`a?yVz*LSNEB~It1W-L7sV96U)P-41A
zz`V7;|BZ&C;cDLddf-O@RcT8`f`v72s@#VkGSerEY9OAJNxeFE!S36K@GcO%Hc@q#
z))?#B*c6$bCdml!ci+;M(@U)pWQ?SMihd3XBJ;NCJHY^G`7@m!#(tMK;-V!A*B0+c
z$#537tG+qzYc*Hj;(ws9?Nqndn|by+bn1fi963-AW?acWZt4KfWu|DQH<rcQtM9-n
z*KafFb;iF2cU4)EpHX%K?s$3ywcu;s-u@oHTid*mou8r1vnNV3I>AJRBgH(XJC=ht
z5&o`DiJcAI^Ux{%7H?#!u<%=hC*7gZ3_pJ3;ZgpzuvQPjQK;BA6Kdee;=3pPUL~=-
z6st!DBn=W%7h_{GBxkrVNyey%p}#K=f3pXBXGDfx7%Hj6bkL#p-qDQu@TLE56T=T9
zIlUQU8y_B!1$rheEf{YGsYs=?<2n^?+1$~bU}a?sPc_T+L+%iO;J)=_!d~>$ErF$F
z!-Z@<@+LoNe_s*aR6Az&M)Z+afOQ>>+Mp;XAP8O$BxZ1=$aED^{F0l)*w#=UgkPWk
zh(?eKP=Sdq_ip!k>@2<b`uEs1Ebg>RL7D%ZOwKjJk^EMp-K`wO_bTq*Iy#z^k8g<9
zso1j3p57;(lj1j7ayki1u-ldu#8ERfb$~j+Z8PefX;w}r3=bd6$V?p-{1yTgfFzTD
zNL^Er($Hq*R+;B2#0V1vSqkz@_%cJ0fh`ML-&*qR)hzu|WJp_8n=v;rkqu93WjSp~
z49>d~3L5`^;JssEx=Pi{IVo=8a)N2A87q)=9PwMM19;vz2ta9OLl58z!9Ip#;l}EY
zj>-xO37ShFlr4YE0stf2<iP2rvkr!1?QT7FZj`f&;I9!L<-*QnXkp6nDG2TxV+F$+
z0?8#8^fsK0Zt6OJTnld{=dwt&Nals}Amw$<3?A?U-$TT*>QJPQ&887I!Fd_R?YXh&
zlxT2+$2H&#yHrtb1C)ANAB<cU>s|cp5Dnj()&ZKC(W0nwyvQBa&|nhVC1uB!DFsPC
zk81dQ>``HKCdhFNa6bz^+1~rU!bD<(^=VCb1x4J~JZ9N{kS%QfNSh;V)gl<wyU<Us
zy17Q!r#E+E=VJ13E})4nH(W8aMxd$KTu=c&@p94Xj;i+H?VR3}pHfn8UdfupJsV5r
zx~`Dk-wsf1*$oiKv;MZ1r)~b?VC%b1&Htf#y%06YrP5dLlrg1UVV-DjJ_q&XRv0yW
zT_G$JybBUo!7DKh7WqVgZ=w2)@*+2;fK1^7UXdj8Pi6pv?u38UnG;G6OYczUa=lQx
zSWXw3V;it}Dp7O337ZgG5QAgCWgROO7}q4<(X}sHFj~%0VsGa{N0%;txE(_hW(dt=
z0VB$9waIVQT?_1`_zSKDR2SrPZ~5$WbpI9|D1JI>FLty{+B{6vx3_=gG*&Yf+-1#1
zU9X(Q%8j=sE2nxNZv@x!4G+2Xix()=(z<!|$DWEJf?_HlP>k)eW+VK1k8e}8f1f~?
z)4yKFa;o~hTdRIYZmyp=ck4q2%Y8Xw;wf<*v5^8aZT39rM6J<5GH4Grhv*w@y6i45
z&FfPE!;9s}#=wN$f&QpLjHHm;>2~UlJlin|34Mq_;CW>qsY{DbG8_19^vIYKY&nuH
zdi|{oJbr3ABh0TRD557c_a5sSzxBNW+1Ag^pHKPyX+AG`rOWE4h>Fe_Y#Pc>;Sg<?
zrmVl#S1;`{k0)IbTHhAD8JcF;PZM(`&RSbN7!)PNko$&;ti|u|3iI*)B-+IOgjP+D
zO1#QZA#+2d36-DLIs=z8*(m+pb_L$nYfPO>9yFjMn$jHVvivqxTdv@366fy{mA!0v
zi|e~{fZvoCxM*x|(97;idx6E6r>3M#{XM(l$-BJfZ+yH#=|Kxd@7;BP3fhsUY+)_5
z2=DJcDs|~|u^tPNkoQB*yW;h`j7*Q`iF@g)nj<chGUI|->B^t}AMkS8jhGtM05RO?
zv^)P~nd?~9Iswl>dERN2oUXk=i_266tCV+L(SMvz=n!kjMC{D=uxK9#Ms>Pnew`Fs
z8E<Mq7Z@=F^6J<<=QqtpPIBxpzMvq0O#36FhKZcb-~$7ZfhZ~^X8Q)FYHc)*v6n<H
zNqv+_e|>NYN6);P2~f}+v@h&u3m@?5oSRWZY!OJcd!3)x_`BwwuO>&n_xEq!32A}S
z4ZOWPv&PZT^L!+2J@Vuh`4l}r-<an*f%8b8M!$;*Ho(oM4UgjuIUw>pewNO4qdd0U
z_O`NG>9{g^nYE`h_t4{ZIlYR^y<AFs{y2IHd3@pM_GJ$EGt-~W#nv~lS7%xE=VV&l
zxjI0nfw4q4pglrg<I9LGPpSx)f$`+lXfhU-6Vn?IKZ}m^oA<uEXJy%zq%W^_xveEf
zN`2I}zlQ&it~5Q`$+)Ds*52KtPcYN^rkSrqdvBGT3{WGR_wQJQ5rN^HH^NB<SRm}*
zL@U)u!ah#bmw)RvI?0;pu@Io8VD-8Lh;>z0HK@a)@BN$el5_ikcq|Au&$y$H%4Xr*
zWc&|@<zt4Yv@tW|q{OzYW*5ggNmy7`FJ3rpJ$nEu2h8g=tNL;|T<RAS_hTWvqUIB<
zSWlky=Sb!ivT+Uk@IMi)hLqtulV_;TL{eR7QWT>X;RFBHGHi}m!A}hfzy6+0j~39Z
zG%0k7gW|HV&joZ7HrC&{n4qyG=YRQ9(&)1;H^4HuZq?gk$CnIhH!_d6e@H5Yx?Yrn
zG9rdLh!tr%P)gxFPdEjEKLs@^$THIvywZMG!&~JQWqX95KU-sAuZ#gZNmi%j)RPvg
zq4cTu>_BUkDtf_jXP|*%=4H!g*?UgiDL;QYH8tPxS?u#H(_oe+PamJ13<*t;5`)mG
zi@4Qj_F29Jf!Y&VoUm8eKMprLIp>?wN$xpu`>nh1J$j@eyJY;_R;Cnc&&`;x7#qaK
z{Ncw+O3W*gl4ZA_B*3FvrLH3r$P}+O=PHYyEZA+)*O$N7kDku_UNrkkw|=i(=gUYj
z*W<IVVPV2D$<+tmby)2|l-7?X&dVOn5mox$Zv7eIO4%~FXLAr!Tw2Uou#jTTZ#aE^
znZMu5>T9Z5Jms*wyXwNoAhE?Vsckt~m&|+@E)3Y0_ZC(DKBD-Mx$`hv6d&j%Q<@2N
zbO!ufsF8!^3%TXjheG<Ou=VbU!a+O}k!<!D>fuweLhtJeWu4TnLyR?vGDy~-@05pz
zDh_?e2fpxa+?D9BO9fsn$cQWw{$Ab3qV2i4UTtsuJsi74Djr6H#Rwy3PfG!T9BrjV
zg4^xHgIsQfZNAC31k>B%I@*{<Kf>36=ODg+QFUTJ2QtMLN#=!M|2{JI=@S4v2O(`B
zkdHK90?a`2wcH~A-s^74BU3a<ujkWI_kH$Hk9<}&itGO5oiB&Rv4&CairJsDc?y+Q
zodjoID*xzwkwFA;Cb*#*rYS-rl$F>W)PpZB3Kkc^fNX9BiU6$S`~prCpxRv98o8Vt
zX|0?2xG24C4;a1t0u?lwGU~hHksg1s@5;KUGmk3PJ5P)b<4Y40z&klpy$59IOC9an
zP9ts1JbnUH5$)_|w9tE4LNR1t$mzfHr;ce08!163=E&ryzH9?o$Nwk`Sk#s7?$(ud
zW?QyGkd1re^Nfb|K}^+V2hlD-weg|Y_>&2d1o8Tri@)XvJug<vN|%}fpr`1`W!QH5
z>E@gnVah5&U0~@LQ91zy#7X(@?}1>v3&HW3%$}a$v*J}b_W@U@uD|>9*vQ=x`+VK%
z>Yg%Z=l#4*&RLi$MmgBY*i4tGtDGlp2;;4-)@d!|TR&gKq~5MUt^dNb7J~PR_T6mj
zUwTLxi;}mv<@8}Ud{a)=-_NIL$>V2L@q(u9_XS<7#g_oW{nHE3=zX5HAzd=(YdzK(
z&&Al7e=_2FpZoR=Yp>jawBu@aZ@@R^eHi6-9yA!~;ZKdYS`5zI8L=<S^}W&LlUhV8
zhhAQ*(X~$wEnv~fPeNjrt`U}~MudL}!!c7{6JqiVHzM)7zK+mnNLlTcyT6wA@%;NI
zNEN*#PE|w(p_C}BEE3joiZMVAVaEq7J~;l>4FHxJ95=7*k|1Y&d2mCm{QE2aQrrL4
zFz{~bxWqd~_W%Bmkbfqn|5}m|A}K3$8C&4FfQ0ft48+g-Wlib?8KB%&JrD3Y>*o`#
zM70>aGaEUNRX%JfovhkJ^H`;l;x8l6@I65ze-9#H-PXFu__ESt-&Koi;~X~#+~NPK
zwv#>2djp#RYHKc6qCocwP-b&;-{yYq?d_ejgVj4`4rpm<<&CUwY;5?5XX^jGQNop#
zBM}9SKzk+!YO|X@XQJC(4p((Z8L#6a1H|;!jYxR={kw_H>Y434mGoCcG6tLz<=X60
z7%dC-u$gp~yfLd|577}V1Q1gx1^D<*FRy}w?-h<s4-YTiXxKnLTX9>?1u}DyW+6fz
zTw-9&+R_s9&TY;_MYT^rW${#3R1B}zoo#}cHUqf8+lm3UN0X#*3<&dPFH7?C_dP^;
zrwe*D3`@0sHSnhv=jL{<9STw6ZubDW-VO*_7<52j4#HQ-t88qXUD^kNOs_I)7`Xn@
zV#UO!t8luL2$0e?V&NR7QhFU;=hy`gdGJv?9K&0k>d90wqKlB!V^0_#A76Jb*DdA^
z2SR@}gMP4+ZE{H4pFOW8t5r?dE>KO)rbwH&Vugp@OdA`Ry^5H}z2~&wpV^L`O>g|P
z9ed>Mw%>CLbbP!^8wMQ;(}mLK{zS9>Om=@<hxTT=%<Eb4G2l*Cw!Be5iJ!G|mzI{c
zwk{mA`s_RD2-atGnqWJZokUR|j_?q592--~8(DFxo7(|PR5(^2VMb>{)fIp~-6{Om
zrsQvV2KaeEcl+FsbE>kp%A8be7LstES^Tyni9?<KhEO<A^sK5r_kwN*6O*s{*(_n~
z*WU{Z<Hu;HZMOG4z&5a-#^f6yd<u~`cW!TJr2cwau5FqLE67J3-78L0+lX2_pvQJ~
z?96A}4@Pf3xd;k2k7wMfJr2REO1~bR|AS`Rm;hnN0kDS_8}3X3tgOR$C_%iCS=`F|
z69<W+xw*Ls>i7z}3pT|X4S~F@LG-PzPKP5W!?lGvnz>}MzOHV{s+ymle|&7rQuWho
zRiCu#Tr(7)<`m3GzaPBz;1}AtA@#PP;pxvM&$kR*k0Ie<89@BIeUK&t_U7=k>ayLb
zj#BY?L-S3W09dqbx5BR{d_YuGRyL@2QBa<AG?;LtnvufD8zIBEGs<Dn=5B7Dhc89d
zp;}UuL6~fbsmBMv&3nC@zMo~HWa!d{z+&y3U3Hi);texd#k6s`1p{c@h~5mW(Pjs2
z5py2~*3`Qu6MQ!mg{Tof32WR$8q?khMl8sIy(fMlJHJdfv$Ws-*_io(K$`rl5knke
zR6&RzQ4zsHzgadihDIeePB@9=EoFg(hk$GdXTlAU?$n=JVIMI2BmgWINTh#hA|oSt
zqu9dYa{SuXU3xIBmo0qhhqNc00<2hI+U!6Wu83eP(buEcSxK?<`TfJbx+8KBCfSBQ
z9ku}@-+&!t-$t3@mPyNb!*n4MH?7Wn`sb$44q;`m4NzY<3s&kP4w%9_P#1H8%>bh)
z(zN#W_SR3XPN0<6!&$*Z9|J7}+j|MpiE#DDW<C>9*~L)~^nvoV`JUm&h&*0y`DbI~
zhhH031bzS1FfgFw{UVsfs{$C1onUTsl&Ba2Q6#V6$~e8|_xH&?#`E7HR9z@vT^DgS
zeZFQMg^)N@@jvRVG@UAk%{Mq)H&>T#@omkDUXK4@hAL$w;T-GziryJ#S_O8NPZU^W
z>l>DaW5hEEL}Iv0>_jwUUG>q>T+clEDs07J<^iDII$A*rc-N;gX)E_ibiwhgZv#7G
z7s73if(V_S>zA#6nkmWPr!Fk@tsD{V$i9?ffKsujM2W=4D{GGPg?^=rliin#KK={S
zI4nQ|xdqPR2NDBZhd5D+@ax^YRlb<cl)p*%*-(3J_^r{=B9*1KI#48a=Rn~k*<Eij
zr+|m5r$NQ`5e)!|qN>cz8fq{1d8BUzazg6kT7h_d44Q@iVJz|3@4hPxBOa9E4Y$Bv
z|90=cTF#92)wpp`c}B8c;(Jd7Xs<$XDC<>|VpB(-&<K{0-$=iP1NB1p%{Dt-t?59C
zhE`XnCVKa{ePP0gq~XqXrt1qgaxzqSE34=u{}9;#Q-yZDY|$`CVl*Y)omas|)4mUY
zCxrJrpvQ)Ns?wTHPo5Xu^ugRDgajSr$9GERvrS?O_Aa&C@S;koexhd5hYPn*GlFXZ
zIMLAnlbeoWY~=Iob|t8=pn#B|t<Xc>v47d}r5#A?FTV(Lx6h-cxVe5d@$2IJ$!mwV
zb~tdHrcLS`RC0Ysoacc=gslz7L>%^Y_K;Gm+P4To+WnLBFd>jBsf@gblgI;8ZqXJF
zun}5Y{z;e?5O#hLWgt2Ul@nf>DbN?3zAnhdU2KT66p+qod%^b>_C|bx1>VK00`XnF
zE<DSx@N{TSX$RwGC3!(;{=79sY$?J!?f7rlV%6v_(ll|?<d^e;D%2i@Nua55Or?GD
zW%p3Mdq&R=`eX~9y$kw&wuk;yO8A;}=+1u-1l$4z;1{j@4<BrgWnD^O?MM*9xD9S#
z)zH-j!&4CmI~Qu=|E;4?lgJ8uq`2V#^*H<a7ykN-RXi{6H&vHSsZYeK6D9nETdhU^
zF+Rzfife1tMG9)Kmlt-gE=ri28VXUuXjjqG8{D7|TdS!6nrecPod0R~VHH2hT9#?g
z=N85co7{i*HI0zu_Qp4@1>z5I*er!VZTvE>%yel$5urGEVs_?Z_#QP%#Q<^v&*!>{
zZkjGs-Tzfy)0uJ6I7N5{A7fOK$=CAYkoj*zDBxuzvq{xF>&U6!GB2;*my`R}^+QnT
zxF}T^ZZPMFF=&Ci2cZl6-Cj}U0g%d*-J{a)AzN+@05Ob?jvf_)W;=slZ!1A$)+^Sm
zYpkoQ%g<l0*ZMLrF**4cd0uUOyOq~C%?2VF>?BAD($}wFzd%ol9Sr^U`@q7}OTUIW
zyUHnDUER1+Oj^v&T?WuJpn2y8C4iPsT|GUt_wTxv6lzl@Hi@OGs;UIjRqApF*1jI!
zU@`el<8sD24K*UV=b(vAzp}cT@AM2c;}~Di1XYwfro80Vuu`mBF`{NT?xvBDnAqEZ
zL3jJTq_lK^O5Q(xfYWoXuCD7Y^^&w-=5puk5@Qv`GS<C`q@MEggYSbwfS<pB;<Y=u
zyKC<DiBB`=-Ah*ipo`gE*jIUqc4XrDV}PZ6hHd%veiDN=lMZ7`3yYZI4F|#R-S^72
z1W}5Nntv_Qs|nn_6h`6L<AD6e$CigN18bl$2?e=IISO6t^u;q(HD^JSjEvgo;XFvI
zzuw0CCgX;*y{!cORizWwuo=~K;YW$gv#8wLwPa~$mryQfm!8~iE2oJD8s0#6+|Fwf
z^j-tF^eduAe^NI7(9tFKe)G$-vb0QQ6=n+?o}aga&1`OLm=0kmql1m7H3x7D5G{>X
zJF#>kZ|(yexV^Bwu>pSnnsUNEFa+?5I$DmRVyw8Ez_4yJC53$3%F+AN-Dc2evw_8#
zSXfu*IX+4OqnbLNpPqgkST<$D$Hxb5WLv@3#^yDi!s^Mo^?|U$6+y+64Y)SgtmuA{
z@NbosmF4AXQ9Sf5BDY$3v-R&t=7LlGS{8PZz3e5$I5A>B4xH50<H@rCrd`<0q}DL!
z=+@Z88Z{`yUSmAAGTI1Ud*&zOC`fq_nsHobVQUM}pdBpLiV_9+YZD^I`nO6-IBjj;
zv6siB{{Xoirwg<{T2*&`8`Dwr2*T$r&v*Ys`Zm&WL?~S)yu%naV}|D&XZ3q@qWvhB
zP%8vq0S@qnE8+nAA&AO{gAF^+P?(*zAwLhz%DFh31vCQ|7P8@Ev<KPfO#E@G0}g@U
z1W0x!5@-_3-La@<zd!-I=n8xp4%}GMS7iF0XR)BSsOM;Qax%#WJHNEf37((@nnsYS
zrIP@9YZme;pb+Ad$Q^V@ZxDbFy^><S+$Ds%8Dy;9UzX0YR-XC+>;P<)w^*f=SmD)A
zkr8TAzvZxO7cgu-majO}+<Z+bXbv^BASG^M@1p^H4eS+j-Ctl;-zK`A9x$c;T6Ga#
zku@dJA_{V*xdp13qUTS{Xvt#D9?1X9F{{=E#-Y!tWo^BAr29bU=O*$Tpg(dr!H(K?
z{W#)H9)YL1@Bsj>09L|g3^=t8)Ly)TIzNg*3xE@ntbF0Y9MJiM>P_px*g5tZi`pI^
z*Q^`dcJmgc2BpLNd_Rm7(i`RU07CZst#x$Q*X>Tksn161SVg$5=4|I!So_Lv#<9DL
zXY?o}BOn;9Vz*u1{ZV4XPaW!fSA0D^JsmW#C2|@JGi~MPvsP;Xd-|@GSHcWM1NI0$
z2i>_FTU$AkRH!Uf!q8g*Vh2BSHzC<l|BEfq(koc<Q9dhBOZm`=s%2qeaPSM~Xp|{E
z7vnlfq(}|p=kfefObGlIijTcFJu+&;uczNF2f|7XvtG&I;NWjiw@j66-OCU!`S=)Y
z?jPzEZTlV#U|hkLb^5rj^w>VtYT|Yxj$OG_RU#@I*0)hXn44zWvTsQ?qGrkl_{EGT
zrB=<Ri!kUmv$nE|3mmN_m=E;+bUauQ2>jf$X&!VxCM1x!;>u5DQEUAMo)yx@b2&*s
z{pzP&xiJ(1r0qqEYl5x6E-YU03Iigz@-jEY?XWy3(qz~mNwEJQFulsl_wYBrL7|q|
z#It(_mB>VrWNckl<1f|~L~|~4PcQhGs?p>EEjO`3V{?MS7pZZmdAF<`)t^Qe_%hG7
z%UJQ`v&~j9l5@7jew)o3M?_%cIm8J9su7U+EI|w0yQkWVsQ2JDopdhmEsYrCY{#@C
zq4g#G=i~Bw;OBTO)q)Jwpf73cZp%Gi{X<@aacF3?u<0!GUjMB3eqZ@7nmjWIZHO9x
zOc2P>|LbSq?LU<^aR6`xZGA`QhWp#sY7W=QGAVm2KYh;w@wZFGr7CZj5r9Xn0Lye=
zMB-JMqw9~(o^_aPG_tcFR|!|{g*a4I4?tHSxe>JWN8i7I*@I89?DTDYzBC_&XOc*@
z_v7BlJXc$eEb0F9(XThRXV*6}7>J#K^f-=b@&xzIHQU04_|ftEN=qt3AQ0cz{UWGQ
zqs44wS}UXf3F*ko!%l!tK_gwPpY<4^RaQTfVAC;$Fia%Gm)&&7$t(6JfGaBh1g!>;
zMde6n?gL04t9feX3D?G0*xGekU^)q?{3S^M`UVgTK-lNe$AGC`krJ{^y-|MZ9rRqg
zGYp;*y<>clj*Pg<ao*ZSDnT2&%AB9zNP%kLvo+vs_9SPtyY`bcKT>aoQg4zypfB59
zi4$v3@^%AoEo%<K_<G-G=~VSfBRN-*0)$Cz9j1UkKbQ?8s;CrvF993}Kr5MM#^CI)
z9XY<_hv&8BPT=lYi4+(Bu%S%_lRBC^9UCRA;kN?Pf=||;scxgBw|BVH{j7;2Q8fv1
zN*R|R$)~t3;V3Q#>>F?tQ$}bXg1Cc1FGVi#KlWNlGgY<lT!5|x7__R#s9iRa!S*Xu
zrpn^t2#$|Zx((5$l_kCPPKk7Ln?`&vVX+5bJYqXQ<3GU#RG`j>^Y{GHmzpW0D5EPQ
zxT`Kd>G}5SY55%Z+DdXHtDRfX(g|3Gm7b}grKU<;vzQ)XNw1bc-ZW9v39p_TW}3B%
z16?TW5^ocx0S(XV@;aI+_}b+ax4<*&W?CnOfa8fs)d&DSa2@uVjHY&TI^a@a?XRjZ
zgS`tT-6wMdent8BSg*6d{0;9gO!acfdJv0rZ$U@~elK0q25MIyGqv^qIxE(=p$V>E
z0RN16&+C=#ezCCdMEoW6;Pk8=vte8Q;3poKM)iE2DLMkcVyC^Ae}88YVb=oi9NBWr
zuGJfHQD(Bo#-T;sk)S+8Xnw%<ixBM@-E%|BePCsAW##g3)44Uyso(s34OWqsQ28DM
z>5e~>*J1>I_V3QIpz+kU(#zqC_Di|}iukq%bb+U#=kZErl#@uOxDQ*Rm5#+o)~OIs
zvOd}CSX=Y*TgH)?%iF#U?M(CCeimC+{&BafPC2Qdc_@jS*tr8m6T+{}e|1<xJ#|~m
z;_>N$OYB*<jIL|lMYhlrx^)Op#53BErN_@&`PslC;N|mI*iq&tO_!c;m(rx1s~XJ7
zDDsvMkBJv-BJ|{ccG81+vwxE*>ZYy}#c%eBW$IV34i^}~Xw`SFna&g=ZemYlgDyN#
z|12K{Dvw0c59rf<(c6H9H@H;u2G$Pjr!v8e%iDtG)aUY~<@#;T@tRd?3|JOS5e~xK
zFn_<J4|KBLt2r}@_DW0RB<1=!=ZkDz_n?0;dlkIxF3rhrB^7oXEnhHahWR>tLuTdt
zAtWBSJ}p9S7$AQu2>_o#aqP=0fnVvB?zNFp4loT&fwDB=QqG~;?7j$psjn*+7qO|)
zPt73yQqL{>6Y8YY`PqF^%UN<zIF&s==<Bd5;H-75KIoIip8dkFLHfElw2CUW#iEdJ
zwj$;!o35<q+i3jk#Zy^BX6-<G$MrPsu_MElrQhgPkKbOCDicCl<9N1aZO#_OvPkD&
zW=zGKbE>Rx=0~(M2MS*2H*Hvfx|BKiCzw=VZU$o;v46eF=~>2y$j^=PBm+<6&NDsp
zB)X_ypZk|ITeq>BwB`B!9F_1~xv;PKobjDwMdmK>O4G0k=BwV7rKXO|5=F<$#)?l&
ziJ`st`O61N>KlzAiCqL9XHk@7VLKq<>|JKPo+s~SR?<L-bC1;hR5)w7J{da<v_aWl
zz;bdpTMM#!bYjB3FvIdGbo*e@8x~C|nlly7bG&_+vkU>%OS;%+znm97rXwTntMYKr
zHYI*?HAo2cEw-UeGOZ5jzZO63?ojAg7@X)o*_jy@Yy9S&lon}vy$wiW?X$G>v!0;p
zZmBZzqR&lu-<<_EU>b$rCa0Iv2c3SMV6aEl4_=@PD_#*FMZq|S8s$HrC?$(-X$N2|
zXVX@|-GJHKZmNSowER+AjP4cgPU`Z`)T?rwz9(<+pzIm)mbFdgQA?!c8R;XPyB=(N
zO>%bGu{?w*F0AreIW-bQnO`oOrrbVPSH8Zb)YRnUVbAG70CG#)I?S9-^?4oygdQ#a
z^x&Ikh~iKb?T#17`xt=I0t~H-FhJA>KR<7~0p3XqwlF!VZHPMNCnq1mz;hafhOP~B
zQXfGjQnKq*M?jr3E3<)DW@Tf#uR9<}(6CPAJYpn!!+V6veqCBx%6e(x;_{RrRB2Ar
zPNf0?*WP+C@cWlF*Fv-9uqg{WyFMTJNRzYGfq=RBDdf9uJY%y-kb@v4e<}w)hDVcO
z!<^s7D0miPS($@l;%y^Lw-`WhfLu|vgwRM2b#xwhAd1u8#^%VY2_P)qk6pHdo5$cW
zknPx7eNI<jP+hQuH5O6lG}@GF4?nCJgN6Z$4^-B^w$-&)HX+M)!o)%bCd(RkP?q*x
zo{?8rSXf+KoR^m%^f?m~Q%S=qLV1a%<=Y-9bNr>i^<~h91!zoSUZA{L3Xf1Hji%r~
zY;jES4O@&NJ~co+f-&^pY7LwoC$l>JdlulCrrFsdyI(6#pcd_Y37*56*aRe~m26`9
z3}D)hFa25&@}mLzqTc}jlyOUF+)>oyi|u%oE#;dHxxSA8mTKS2dilMj<@#uSgjqEM
zAV#m<CZtNHxbM})=}Q)(%rY~q>k(C@&ku(^AcZ@pNhiy4p7dW*szbEz8S6YUF4qR-
zYTkkjCt$&y_^Qu!3}3P-nLgEGlO5eIWD`v9UtL`dlK37S7_#avCw)0yU97sVjz@yB
zQf(K<6`COnmlC}sZF6eRzPDb8E=!?|P}kKT5|>{Ocbk|4xs#P-WMqPZg1{?Nd?>TE
z8P=WK71`2}9jF=xHe(B$(bLldMW}vx2qSg{u}m!KC=n;u_mydq@DA|cSBdp8s%!%#
z1UNtu4xU@Hs^<KG7JZCBd1+~Jlp7LG_^`_S8H+c~$3{O!19h1XfF08QuG1s2YId;%
z`%7K<*{2uQx(!x9O4V0(n-nidD(j_ulLkd?czF2X;h`tLb87F(s{vjc@Fa_@9(byb
zD^1w-MeZ_qdIHq>pvqFMteKQ1iT4ntmQgb;{#ZMGT5O1M=k_#wesB;U_8emfkOqgR
zfDWi~go??7l9ppDL!&f{(No8*2<E@nO}eyT-R1Q-!(02_4x_Xg8Vbj_OdDF+zEv(Q
z_pZUDi?}|*o`X+b)ZjOGZo^G#CVyF*>%MWhIV>#xUYs6EeZ081SX^9;i;G)aG;T74
z>g)G8rvq}@`&h4#q?yW7C2M9IFv5<m16FAoB00n|>swpk`8m$S-nBzlH@5;F-6$I4
z%Bd>s_!u(Xw~n^`0NVrJ)~%uG+cz08>f`pVIyd`}!?k#4SzPKkItl=PTZdOu?<V@Q
zGn#kV8R{KyWO}8TY{*Ku%e6aprCBvFGchgJ*RyhU-zl#|Bi&hm?Jr|vFMJZs8{9>L
zbwe=(l^N^$ZU0qzC6c%2<(7!dXx<cN)@`t@y#xj{D;@R(u+%yNV0z2@edq06hSCZL
z6{u|Q#rM(8;jBB$w3_}t#|P6=&J;-N=jiQuKk_$BvGS%44tuk8&cNz@kaO<-mz&T`
zQO^UgSU}8BOfD-GRTn6wI$I53h@N5^PgZa2U6~QMdwu5o$0bdSRdq_#8&M=`b|}zt
z`YT^OuRk4B3aqK}4c$hjzgv8zTfRMxocZ*Ce@CfZ5X8c9+roT-?lVW3NykUi8!FXw
zGoaMgTG5H_j%MzNN%si?(5#n`5)j2J+=n3GYBcZFo<VtlF4H}itVyK+sN$koc`88q
z&Cd}6#+|hEc&6aEdMs$BK<gm+SW*3|9Nz?KL(yQa_lHZR<ZLWiV;8aN|4`{+7b-Ab
z5W*Pq@pNc=6cfbXKx?m)|As!v<(iN0{H4nd)Tsho_4Jpt_&{v|<UN%XWA{BXgAkw#
zaDBAwpyp3iRwHs1p^omds%8>8bfWe%VNZQ@Poo$p!5Nl@W9uu^y1QHKYj-h0z*81X
zwY`}5ACgX@aHvs<Zs$StUGnP_4iNohFtRAf*iJ61xfInO1%;Cq#eZoub%cKJ@<wkt
zzyruH5I;CQb)>-&2qR8)Rm2R_A1uMsmf-?G+)Go*OKW6OzzoS|ae7MNIkxR4lHR{w
zJ-xvihZ06uRUN=HaDbMudp18hHukNAX;bNy&&$}04^b7a$I>+YUnz`%PsL{*$P4fT
zd|Z+QB%w?Xo?$n}W0W2gNhxaTw^eUR?(~lGQu;C#%@tSuQE)HJzNZz<<S5RC4AJ;t
z_WCQRoNh2Dzz}*7F~!?`<NZN^4+0Dr5b(^ZtCU*1Ln;zhrg_apD5C<uegGUEpID)@
z^u61+Zg_!`U?-6K5~8%(6Go+L@UtsjT;pPVuhhNGRNefN)mn;$$7dwS?neW1|IFXX
z$o5Z*Z$Vt*qqxD2^!ymvIZ~(){|T0IT0CO*Y~ZQXIH-Q=etWphqcsD)M6-F%BSVO7
z;2=3E>E&TnW9ufJ^h?9IVfUYzF2SPq5lxpF^`>LR0;`EoWTm7#vQ7tf$R)CSl1SzL
zyrq#(AZ5h?I(bzpHc-=y7^?K`p3s+|=26yH*`05>G{sq=uUq>?<saKwJQR-;Q>M;4
z?s)NGX$qkF3VL(7uWEd%=31v+FQh#c9)pq)ICoWBBhFu(4wLeWqvOZTm6d6MI6&ff
z%4oCgW_PmQeBURcPV1`NSj~{vpnGjoQ5;&zL7I#eT`aaWUUE1K0F<i;keZ!K9Ssfj
zE@kJpJYjYxi<!u#%mlG0TW<f|*vIn)Al|Uefsz8?`Vl@3hkuGK6m&&CQ@bj6;`RzH
z81;;Pv$p#F8WA~(PHF=BGcIzR{miC9&Yeu-_IARZlj8C#U5=5=_I`gac1deTL@W0_
z4j;a5hW(#j0I@#=61&Ry74JWDo)p&{;l2~|`BMDdZ&QiEkb|H{;)`EPilHI>h#Mjy
z5tOp(l&x|t)f>T?Vr(IM6zTTxnrrKU9-n=ePsQgO6FXR2-k$U6xIl3wKBvKDC2dDX
zVaoDF-|Nd(<N}3mC))Sorx)8@_S&6zyP!~2(1bss01aJ_3l<7Wi~FBYfHDJM;Wk^j
zI6Qq~(ZQNYy3_-_9Qk`YKYsjR<B=z<#m2@y_VT?ayAgC>Ez_LS@VsZ_rdY~$FKIBG
z@XoR^!o9{rUeB55arrD3mVRqeCaL%umnFow3M%NM>M_u<k>nGW9QcA}&eqSyJ!6Wc
z-|)9Dx)eXHR|uDT8ummfG{gCL^~ieF7^6R-49FS7_Jz#|nk*}2P`HC220ZDLDM>H@
zmtdkJNtLYElJ(EO4p}@EZfbl^;>Km~sZbyMJyED(H>$Mn7;^s)kF*Ew^{1WbnBNlS
zQ<B`<9F?XbaSNs>BZ&<N`GEfF^7FvUma&V!JCCkTaRawxkb|}o_)<1gb#}c`6uQ?4
ztr4xs3TJ~63BOk*BzfyJ5_M*4^)xMe6vt6cq4<mKdNlRdTjox0+Mri|%$rYQ1J5;z
zK`*FTSG~mPVgesUsnX!aRb^(n$EGx0J95GQrMHH+k)GSE19HutKVgHg<jan*ow}~0
zKz{Vfro`EivR+TM;#=W}Wa{4)N%v(B`g#vB*BVN_bP(3wuF3uhk&PKP-zQM9J&<a1
z8xQDno7-Fk3y&KHUm&(#%x6;BiB!ZyQ}#%dJ>|`5WZae>T}uw1Xs$?HbDuZ0m&)0O
zTXgM%^}`&gp_G{M*mbj2ZC^Zvr9;OIIdq|pZT|D+SCt}IN)zG&XuD~m$mtohka=RM
z*E{vTm*(y)?z8lPCmsH0yX7x)oF=yQ5WdeE_<9q?E7x26Ud@-Evn>#V#X|=57{Ge<
z%ynD92RMz+uQ2v$7S0-K10sUxuXZm4_xihsoCEIW8m~m@@n23D<j(LHum#Rb%p999
z#3YV2G5fsx++a5`uQivSP=9yZmvARBrs={<ML&+NVXgg}&|LZTmi+b90*&vI(ujob
ztK1nKu)uAE_P!oRk@Kb3e@86+Y3;A}-2Kmvx2MKV;(x$@Yh7g>cte*m&xapv)i87`
z;)T0IJ&6r0JT(0}L5I}xXR?)p?6YD8DL2r-M?4@V8WNzhzd{2&8%~U(79)r4m3L8}
z|0Hdu4FsJ6gi@0>{H|_zb<Yj5N=tXYB^n-_`n0vRQSjT;USFN>HtaTozgtin38-qT
zUGb57@ia<ydS?f5o_M=Y$nFE)M|9dz^bh{=S@C^`@|Qp~?DcmCt%FtPx|#BZ$&QvE
zTnWCe-Md^k-EYmZ+m!HAR-WW#Q3N|dC3s}FEkL8TQ`_M(6Zzg<B`Tfbb^Oj4HjRGW
z-VB@K{Y|ziy@z7RWcpum%f?x7E(zpB`OD``kE4H_MMKpOOT6w&eVlZtSYSlQZKCj4
zGu;-r_9@X3`s0T=h3&K^mRbP&giYsn0&5ZWU5vxe1_sX>$emJpDcxwPc)E6dfLCfT
zN1hps-XFZk$(U!i_iKEC3haW1ps;oQXSuQKAMjL(i@9~eM3v0iqqm#8yLPogc0p{=
zr4mH=z}S#Le~!~b>{&%GLPXGcChH`8SzFPnQDn^TNI_Z6B&;P<<z5?T?Rw&it9HdB
z0lIWbOYPf&aU6R?LPC&RQ%^)i*-H_}$G)3-3&KvzaQEiJ^ix~>34%TP#KfX1x{b=$
z8B<AgW3<4cbg54x@TCyiA@dU6JJqlWf42~Q`ia=N0cJ!qBk5!J+3Z<I#oy`iRyCUO
zCHw~kGtr9m)BUT<{Y{7(rO8c+hOfZDvlcRr{nxsrK0dkjk>7qmGq$Csj2NN$(rC|4
zBy>GJ;~4V#{FmHlZ!iw=lQ{vqmcXsMI+f+m6RaLyuYG>aah_8h@%Hj4NY@J8Elg}`
z;~s2F9~pRAeO1X&nLER_iCqo?1@l%ZCY?&7t06^2Jk6W1pKU|6TN!)Yyn#xS0Jek;
zi(zuVy>!X^Ch>bC=^hB})IN(H-9~}~Hm`x$9G`K{68O;#iaqtoKSw1sBCiXYIUvv^
zayq63{fv2z4L$jZ*q&)YHtkZ0c^&;7`g-&CD}JR^#viFNOC%AqG_5>hd-KsNM<^jV
z%=GTZ^%@E@9XR~lyu;ibF3BGFa;NqiSO+PLl_(ep2$X_HCHT;n-e0A6Gx_Ff99?u1
zYl_-q^u~p+OYQThE5@9n^is%|$8YbZy^em?5Vc+tnS~rs;ktfF^Tnx)11+SUu;j@l
zt$K@nr?01;5Z7#K&8N)flw#G)+E2z~AukZ<AtCCKxK_t*8}7;Rg!btJVQ-_;sbVno
zFw}eo4nl0wRo2{4HIN;-XJNve9jO!?^l+6n8G%1E;6+MiY-MG&d<Mt?ouk~BT?2sU
z@@*SxDfHO+1F!^E!8o9xK>MN~zaAVnokGwXvDF>Vyx@zRgI=FafDSHU|Emj7Kw*m1
ziSXsFO=}r?nw=R$QP#ccJ)bm*p7UHwj}}8DHBL`V2yE^U*l~9RyvL^}!otDD?Md_k
z5{FNIrieQy5!NIBe6CLXV^JzLR$J&Pv6+jm{gI2+gO6MFnuX6tM=7u)vFLKOf+I30
zUI%#SMjYdSXCPqnf}1}`lfqv4xICHD;?VD&Wp-R?G%SRD$`p505aTM-jjvOGfZNX@
z!2YU66XkQ)Q?d8$58<l9=q<zaYs@3|Q=f#013?QIsGn{sKnFqa$4jzV4wyVO3_&F}
zaix0qHfRdt+y`=z(|>DGb8EO9%~#(TPnalT)p3jN&dgfT_S4F!1&t&UCI>H4@86X@
zKfgO%0AgJea(<@K(>cyHZsGqa?W@D0?z(m{K|w-ML_it=$w4V8NeM~mZs~5AK><Oe
zW9Sx8Qc{6|5ftg}&Y?p(hdFzAzW1E>yWZ=Z>s;r|A9Fpz!*B23j<xS~uY0X67<tEp
z(3lUk8>2vxeeri$gnOiJ1(~)2du2VrLV-8@SmY1Bh1Hk)WRQx8x6XSjH7`-t3GR9a
zLC@-qOs7q<LWKOW=fpfb`lu(*U@t9TCeC@aF)%>oxx`Fml^)@GR>9=kyUF4tUeBdO
zp>uqY=zOgYB#$Giw~&y}ZE&iQ@aUq)#;BpIiC$6lyD~HUJOj6_#yZ|lTUp{>RiX|<
zI2G$^dM#yjdVSTx7MNj`s2;KgA<prXO3hL#WcIPIh?g3ewru8iVq$o62i_o}Sw3M~
z4Fa?oJ<E>#<@4-n2p^y3i(tSe0}tfyy3Ang3P4Q#b`4H{8<+Fmle}X*S}QE=PX1_~
zBope_^VODkk4Mex<D9CQ&?-)Hflx)+ZcximA)vxum+xK;{sN@V3jA6I<MsPcBTs+L
zmY7-{IQreADt5!uFw6_l$Ln>&ns&8rrh3na>S5Iw<?n15E&eLpDPZiK7xq@ZXnYbQ
zpkTdw)3r)j65d5zp3n+R?clqU!M@Wsy9n8XxRy22b}2=~f$^jk)PwK*<!;^YVh6UN
z8Lu9u&6q_A+`N@;p8~9Hb|^-kMe<N8=^U05H!edzzaA9~fPSe_Psl*-%Ro$OUTfLw
z9=n4D#C9WEe;g~pvKh0Q8X5xOO`u)_(!6SnE*^<+pU>Qxs21Eq(%_@9v)LbRC*Vb*
zWX{aE*gB+v>$6%tBXlUb6}hPNdWDIAXp`*(9vN9usTO=N`psBEPi`9Fwdb>4zr1&H
zu~vFv4ToLzm`-YMu$VH631}2@k!EHHWIloSM2(-ua|)$<+Z)UhAAk^;h~IKD<J-Vr
ze#U}j35l!2;m-af<YJ$O4Q0Ooi>+Rm`uh8n=ah|u=H}ZOBQ{bMrLTG`pD^DogNSeB
z&0ke3Gxn2OL0617#gl%e|MEK=hk5V2I~($Ooi)$7toF<2V{5a1!Y-?^4TpO6Y+n1@
zq9h54mQ$txYB+NX#%y1lgIq;m!t<2~EG%p^g)qw$(<eT^BA%vMs+sm_DY+{}gHbY%
z4*?D@sT9Z{4gLiRVLW@*inry+Z$|AU{pz&XLj6~_%W`*`h~Mepg-tAh64gn)=b<6$
z*c&+A3obU?zN@TuQ_-jTC6$$~o}T<(v<ImTluR?&aS<oIpvzS&$D-&jGlKcGn~v=P
zcBWF%xir?xPHlD+=2`A5Xrjt7zAXdi$r#tPm!7#aXVnx#NO~*w;|30s&9ke*WAaov
zRT3kN1nuQ(^v$z{gmVereL>zhJVD2+Jl&lE8#JP)f0B||rTfbhf*PuMHA?*4w!bZh
z|HSY!2PQtmE5S5h<Yt8BExtv1S`wN^16l{=e2W%%(ri~%B&uh&i2$uOpi^9{L7H1%
z4-EFm&`?!nr2^;kJ|MELI}+vSP$R|$Q3Qs1R{wBjh;jxphw2Y&XrRJB3)1RsELHh_
z%bfzI`KVu>^ELHv9me1~_!eTXTU%x)#VtT9EU;L9&X2Uts~9r?GZOUbKOe8N$xt{r
zI;|m8iA-W!IDh-;WpPndKve7dJG7E)q|sAtPE(_*sFHCW%sG!&LQw39!Q+nv`+Lk}
zNg~%G<Ot`U)YWnZmm8UlBJLkVsLGJaI=v+k2%LeZt*dT_!`C?*@w|22r=#xM8LT1l
z=TpMHa+}3NS}D?!U_S|ib8>Udz4hRh5~sc*<mFX-|4>MveibN5bIm)?w||5wMZx&s
zM@e>elNls@YEQ*Z?=j|gJoSS;(NuHA1;7T}O|0uKyc<!J53plZ-6Dkolq=Da?ANyH
z@m%+t&%Q_UDSQsqr0{U`rj5(T4|Ccw%c=2{&sgS$evrTiNn#g!6`Us8rpw|3iGOjP
zn=J-Y3G5Z9*@zE1lelFG9vU`{CjYs24^`4re|U)8JYSRbD0DaWp4Qe3{wu!?<aw*N
z$L>3km5-zy*@nKT;4zK&Z7@*8ytvp#ww>q1KM`r~6VICg^hdm-nTeDN;Anxqd1A9W
zRVjHfC`FBvYWnk=Y3B}h>biB&NuW2A!t8b2m~o3ho*$z`YalslJECZzwkZtea7(nH
z_^f*mgNsYDyerZAz}^4)dq?XHes{jBJl#%dEO1DhM?=ZfXXmpi!l`{TI-T#b9!sBb
z%bk`oqHYOxZQ1t!o)K5-7Ia%qWgGvjDD2b!C@EFh0m9q#!d<9r@*(r*pX06cW@3L@
zo6lNWCiT`^q^|Cqo$w0?h}YKM)zm!BjhxOnS{^^-jLx+H;F4nBCdZsfagH$hG|JC-
zEvo)>`vP5XHo_9>eA42;REOCan5sBEM`-~TPlAT{+r*}GjK{MvU0Gh6{uyUuos-!=
zKk<YA^c=D!-BX%iU?|+McUo6yS<xY3f%R0!33($l{J9f8C!Mt15<h&Oqv*~iTP}Ne
zl~g#_nYLr}4dQetc|GHd?120Gm@PIwU{|OK^h<7G$|Z*Nj@G(W`X|}?+cmCzPA<>g
zq*R8_kwg!m=qygO7b*dHjyelvyHN`JV)*9Kg$IFY54`2EkL|l{;!}J1jBHY#!dkfo
zzET9#tX(6ePj$#g7gm&2U0q#MBLLLaNT0sJX*tDwHT#+$tY-t;yh?hv^nHt<#!I$U
zw_NAxyR1E|4-CfB{dNvx3)(H*+}hYnp@tV86$Xsm^J_ag9#8(-OtS^Zs>gS*rJKsH
zZYNg<Wf0R1B$B8+$mZY@iF_JvlackK%2n4#%@x1t?U8UI=H*_H8MS$XnV(BA+aqlR
zP}R~9_dYPgrv<8$B8~O9!mX}};4=Mq`e2=fDM4snUFoia#@47SD3=I{Q@Sownzlvw
z`gz5glE<T@2LgrE%h1z-j5XWLlG5}7Dii5w?{f8Oz~b4pKHi4)oq2rs47b934<t6V
zvm`q{g^<+8&?$`RS~Ui5AolAzM=&eMV=eTWrcGQ>8Pm?eT%mPea~DkbNlX4-L4(lf
zDQE0cw$K$`ck9m@PbDfzc&Ft;JxGh-n}^vB{Sgl=6Q&sJuHUAGZd=1U3T!e0bfCo|
z%BP&$#47Y-yJVCim>S>TE0*`UC8_G)RdZRq>+#boXFr!L@`|$A>W*o1>pBmz-Q*$t
z-mN@1O*-|SE4k?5$C|b9?wn3~b8fc6>At=!!ihu$Cc74!q|fy2uTm8)L8ZHm?;!B?
znn|sbcVf>R74f>gL=N4{mhzslN_XG_)8pAAJW`@hK~8=vNW7fl)?HF=T<%(OLu2DC
z+xmP1HEnH{?Sms2<_LtZ#&()ksF*Iyi0R`doq{_D#>U1%vnSzZF8zoMQF_r{#T;cb
z4j`M;4y1A*Zk%)cp;uDxbHt0)C=NH5i}-6)@Adthp-TR7_tE*5sF~=&tF4O2p0hWt
z#xu*-L@#J1-N&ZNuK-rl=!loR6yt``UAwtfEUreGZDI3zPoy4Yv0G66n_z6*vK%$E
zO|~&T?DIvv`h370Jy6ZuCla;6`j8~K3TfnB9fR~q=?)0x-=i^~R5A{B5O=qptgsqV
zJ#}k1?Q+BH?<w6-ZP|41*cL-EM;vC{8LQ!(pGg@X*P#srRpTVafo2fjtVZTF&Bd_R
z#26_QS+r5`Y^ru~;`sQu%4Xz4aB!(X{iEAKS{nK@4Sr{pqJP-Ffq2BY0;s_^?Y+)E
z_ECv<AvXNvF040%is`^CDlM(}&aj$&Mp+HR^rYm#g+m)-28ZZbUdVGNaRVM`t?sh0
zPu1qeZ>`0;4V!@a)1I|ofiT0WHsDj2m2n&Bs-mam3V)5{`N|b!X-<D^fb7&g;h2AC
z9~Pv!$n2~=KbL`7T01jXh@|d0U5dD|qg`U;c9Cv3PGDDX+z;18&S0inAlA$3el=ih
zH^oZFMJ06^>14iYu4$N)f`N3K&B%)g8*9%%M=sY9o~vb|k2KFX^DUp|g>v3UtH%=z
zm%}pj{+=%#3h;3nZXzl5AH--Lm>)FAV(K(@a>dTJONoN7ZXzT3#F50Gr+zq@!rlGG
zF<VI;+qc=IhSp&<TIp+_o^aNWLaokmQ<hMDcF45{;+_o#5F{?G>*DWOwbI-r7q6Y<
z^PGdHVQOrT^6yNJH~)ouRY#1;C@txXC2p_gXTbGEO7v<})zx$Bmq6siq5c3)`3nrZ
zk|=s#p0l2TDFOP%qa@D^O|>SPWX?fPzW}PLC7`GwT2JbNKlvC&9BuZFFD>8ImXo$5
z8gSpn@2OWVlje|KaIOJ?;z68DG&j)7&N%|QCZWR|gcfyl?EeH+R)RDffuhX&c-Mi$
z2$D_g?21zp=rH@?e!g4O%T`n+<miue9f;Ga?P6bLhoHfL49uP`?A&siMZO<*;=+_w
z=?Nk8w?&pZR!z)eTM~?qeFT;c0m;tg6%|$2pjpZ*YeR({2!tdu%T2PjtF7(h59~=}
z{r=;$@qYJ#2fw+;rn&hC6g9WW=~7Ib%pyx{ExHf`D>&qQBI51_WUM9N!}7G^AB7t>
z_2_xY%+o)}>2Jb~GPWnPXMLJ^fB8a>#7wEv>VOIBLOKY9?u@m#q(=UY0J=S~yJk-V
z#LyE1B@}I=dgZ6_Pf&bQviZJJDsh~wR4km0=W^BO^xA^#Yn)qt0!s(Ypd3u5Lr<ET
zQdZJo`I>CBuq=w$t@93O8I;(}%E|(1FAF<tnT!tixK0nd=#s<ELU#!->`te=5HEmg
zDTW}#AvLWucv>Z@MvPEg!%h7UJf@?kK_EPy%^CWm<9NSB^3S(>-U1K^T~j#0B(P9*
z-UqBG##cKNus<m3w&WpVuq9JZ6c-0c+~R71D%2bDvDqYguVx`=<Hocu3rW_DD;{l}
zOi`gBdRkQnO%iBpqzJJ$Ucxu&5)yegPx<Sr97T%!Zsc$zrS+W#sHA(VA_K=m*bkd(
zneOZk`9Cc!5?RDk@?3XVI5Dr_wko9q!Y<NCeb(tP;M`P80kLQKJ@-xqc0RR=AJr!@
zT^72t_3?Snpw*W1a<9C8Cwu&L@EQSfUxv-Brq1BoJOYF|&a`2@(f2ebx+AL6!a<o2
z-h`sbGL}7?_<=&ABwB&_#FUh$u=9b83)0&`l{r(0kdWgj-a-`hK;}%ghz<f0(+aEj
zG_q2}70*BR@H41`2a>nUAXdw4(!sWFT)Gp!Ux6}s?ntBG1HIP>MqC4VDm?uX1SVct
zMNO%*AU)FYA}%H_48v+8OYNYFcx7~aE(Y(bA|~wX*iF8q?8cyMw>m`gXrmt-=-g6_
ziz^)c95`ehKBgc&;nE3IpQ3*@q%?GODM}N~RJ6{v@ZWsI>e#D`xGeI>N=aOvr?4_I
zb82fW)t{SsTjWUj$=xn@+VkZ2aL|Vps(GPXVs>M4&9A^=3d=Ji^Un(ny&ysCYnsSi
zhGlGifWMc7SKE9!bGvBX$&N#}?%*|S40kdMVfIq{GZ+OPB>NgYVidaCd?@d-_i5QV
zJ0Ply7U$cfvH|3X+8Hi((xTveDE?N(@Z8Fw+~cGh7NSyh{^>j7<D`fA+b|w#Y}k63
znNzlyQSX8qZCZ|_GdQznrBv?WWxS5L=Vz{KN5sjD?Z|2-f`O%@;M|ny0rZnMR^6QV
zA}-<Wl>3BTXg1sOZe&urV)BE7g?uD4jkZ7g8dlS5q^P!3&dS-=+N!Mf97WAi*_2ol
zl00zv4tlyKC`RjaI2kkMQlQs~Y*shOX2(QfmhWH6^w?bSixGjlL|D=0&%4iBP>i3l
z&rLUST*#;k4bQeN;)(2M39#94IGt}0yLXcivFw~#=hv`uJpYF*0<{RqJCl7V#yHqz
zI%d=}@*kR|2JEo7cRs$psaw>_Rla}y9^+|o(AEte$G^BQw~j;841qu%mfT$Gpts?L
zj(L)`T-hTpuORT!{vbri4T=v!wh+S0-4ibSgo*vP39)Gx*yjJgYdB+jEM)=SY($iL
zk^i6WLK-s=_s-j1P~u)zl~%>RxG8l8h1JE>sLV3|qsx{%2PL^&Pm(g)IubHY#w59Z
zb6>#<9F3$oFe{b-nI$BYV)|ewGNKEKmrvyi2t;@Gh$B~f4ep)34i@}&=knn(ilg*h
z$k+sj4!$Y4Lrj89GEAoXGGmBF;9g^IbLmLrGYkis!{tydlC&D5`AgX2(N;~xH#E`$
z;FVMTrRb#@{c~^O1E6aZ`|N^;V?+|T@igZTwP?DB;m++G=ML(fx;>40Ep~&Vbi5CL
zFjbcG?QegBkprEj>84(<+w0%%9vCwUQqew{y-7$4@@TOMME{jG3pNz^Pq3W+=QsE>
zoQgT`9!A2e{$4Wpq98{mVla#TNlDt)W6=RBl2Tws%xqa|{9I%;-0tJv#;UOWzlK+(
z0i$MCv}udD^QXr(GGp~I37s;=9avg-H+AowM-ddDL;eEK_uF5{&wkfBnUx+k)81uz
zpOcW`$K$#vIX_|Og>G%l(pVU%0~YzyHEi;a(bjcCFAtXUM?JU25Bqtcd1u)RsE@NN
z&$7KTFE$J9Ecc)nowrSyWbT4q{M=~wJKUMHq&B>YeASaSvoyqDyiQ^B!l!R%SLeuO
z(H5s7bPFUqp<=#Ooy~b(1xp_XUI(5Apg@<9aiPEqWel&(eeC;uE4l&mW~jRJt#rkm
z>54yjHWT<+FV@RJ9ncF>00U&9t~pg^s^-0uyg&bZ!LqhR$%Ba6no4w_m~7HewA{Ic
ziIDShxd_}|W3mPKOX}HD$EaQmz+;8AYrBy%lmdf~wj%)UZyE27skqX46I%=jUw>u9
zo4pK-@b5aS1Eusg?&@ok9?#}1CN7sGTUMaoV&H{u%ertv$&udeLFAO?37@bBv~IcE
z!SG@yWlbM~)DYM_Cd3(#t^rHsWu1GiE-YY9qWInE_o|n^i^_#rp5aBbeTH&=bo{`5
z(Y0iwyxi1`uVA|-h&zK-7EB-VR!;e#r=Ozyeja`8hM%?b?mAyCWjKq#x{5)Apn8VL
zCh_r=838b%ahcem$z+`95;S#b1H-(#dz@sCo`zj^m?kTb-9`#@^^xz|^N3s60m0#R
zc6qN0h7UzuoN&wMT;b^FuQRo}R}44|5kLk0^L;fPA5U~wrF_4h7y1>L<=@mM{FEZR
zy)#?XgEO6El&m~UyP=o;c~BqL#dvBy_B1T1(6FXvrBl*zf49QnLyWe`67@D7(rq6d
z?daLgj@)T1yDBxN_C`<N+o8Jgr#z?y2NX|5_A{#48Q(@~Gh!JT?=`B7ap~5np8oFr
z0=@T>Ph*BSIk0~EE!!JNN0>D2?|8>iGJ?Zk=`#shFB=%M>ram@vNW=_t(o%tFNvDC
zmy3qk>Yhtw7(1hN-{Ec3rL_zp^0G#N=CLr)<7i=6+{@TQ#Yr97*nR#!G36jgeDk95
zSG}+SCX?8NfpQww?(^gOa(?{8!+z}H6h%ByT03znLxxg=%?Z#3deTc<z3fPj0Vlhc
z6IssvKb=Ku1<?}4714bdkFjJwbBa7R4D*|~rtEoPsb=tKKHp`t!NXJBO_>?dAAYz&
z<Jft=s~Tr_Ok|#yyHj^=?0FvL@ODlp1FWWfvQz2GPW|{D@x$X)`?Pn~ehDLHm;<BH
z4#$6K0cOx{sw6c{sUD6#(`D~2mX*9@n&zS%k3TNhfvp9OtXnltUMvI^%QN6XNBIOJ
zY0tcSasm7pQS<=TAmZ(l1hDo^<=Zxst8S6>v{R+T>Bn>D^C(`IiPbNbRzK|bh_?z_
zNISV<ZdfJkGM}^8NCOHsgUz_q4F+*`UB-2H<~V6<hi8gj{QBBVcR@*eNr_3IZZL45
z|BQEACx~Uxf|@C|uOPxfq0WwH5=xX9<)sbKPa=C*$f+$3>w}yt{-h~*5riTK<IVM)
zPxzYzNhs@gPov)jA64M|hyXhoKm_OR!|K()FK~S`*0vcYjQD?`v#v^!h^G?WoJq9y
zxy(G0O2#TbS5~4%>hww~b-jchd#SyC-Bq9tia_XqV)HpuksYYe3+`#C&|@VpFE4=V
zLDhQS-b!X?A|SV{tb8`V+Vu&%iJvZp>8nVW^&sIO($8tTgN3q4-vo?HUzbFliwFe|
zFmGXc51l`h-}x}Y`n|0Kxq;w9IrHMc`+9MLam46A9$J?LC|nOTclO^wFgIC&&r%vB
zWEZ4>65UDPKx!RO51IK#j54K%pk3YE#x~u7yvS8>2-nWup78orAaMdt>E;moWi0?Z
zYwJ)?Z<o|>oPriW-#^}M;EF&DqxIzIX8Y;aN6dLPRyU338?u{Vj!X@2&f?m`YJhQ|
z{F`=995MF-tc%B-_pnA@5;P7}NdxXyR#qUD2RJVXS%XrbW8Bqn3jn$DA3l6wQFHhW
z1(Zr4G0-TA<E3<I=;*BNc!Kxw#RKJ|?Mu6MNOU<$0*j}VF8aK?Ydr;CfTe@7oEbDU
zVakI!z1b*$v7)Gr3gowiCnpA|Z+oEw*y;(zut{dlY!5zN+$`6T==uSaT=ftZNL<GY
z9X2@7WqJ~&9m6YRhK=(J`c2OjJTfor6stWUya|ic#fIqEC-*W~PdO=ug)ljrD2`=Y
zU#TcNt~UwOPdTaA5I^y+1G7OR@J=if=w-vZxj~MWRmBekO{qfYz$y@6fn>>HAbld#
z{&W|3bU@_<xQK!ik!g6vlj8w;5~@>+l~8-jyLhza1%<bj&LyUMGTK6C5*dNr?DasN
z=3w3&oJUrd7Mwa~Wq0<+sw8NM^7i(~UlUX2xJvU9n-OM!Py2|1qKbu=o&B}J%Ekt0
z6hwv;A~*`k7UNIp4(S6yxdUuBbk|8!6BY76j#1sze+iFO+#$HY5cfL}#26r$>S|d{
zU1a|`y+(XE!ZTJI#@#hw_kV$Xwy*g#u-A1SY?O$Mi)?>0Df>J&1av)ifJ~4RkG5c*
zEh+u(*i8v_=*x96%qxC{X1vQu2Nm-G$!=06`B}@YX`bNIv+N^NND|0!>_8P17Vdcp
z8I<xFbb~mbLFxBPS0b;lP`<k+mFMMDpxrp8IF))Uv~_Z|*#@ltu_1Lq^T~f5j`skx
zCEwFJdH>QYt4XL%?e5b~ui|TH@ep?;sIKM@B)zwy?@@!B1i%dgU(k+<s<CCg1o|=r
zw(B9Ikp<P&!=Q|MJgZYhQQP3;<OWb@bBs`W?BJtM1EI&UWQ*g0<8)ErB*{b1^w(^^
z-7Ur;xnCX}NUVMH*TzujD|6`_L@!zPEqd%*nze$99_rkx`yLpBD6h7PaQI7mkmANW
z1Sb)lUI4UcmFRcMUmCE~PycmN19;JzH4F?4tYO*<>~ON9hVv8e{P9=1u-9(r;|R&H
zz+d7aaRlsqzRo(JRxa%e56CdCzy{V>#~77q6=@VgTY;#~IR!Y}*afATtia?(s%Owk
zJ_nZ=SvwR|p8}+8`OFOBhr4&Ev~y#|<C^mY-P5ku*JXm;3d^c=6?0UmEPn0fBY^dA
z{PF$C^v$BLgkqn-VcP~$D<NkE0R!XyHWg!LebpT($m?=YLqnls9b9tJX69Ts#S|+G
z&UO<~XHB1@EfAa}2`{4uB4@CTfpIyx^3x2`g_>+(k#OASKe}z2r^5(=sE<Duol0wp
zSL3<^3UGji!DQ8a3SHltX}W^-DY$em#h^A_Z^;xBgbPrl(%KuB_ZfJAHvw6%wKvTG
zjjdA*NU`zgjSw_HWdvBs0tqT$0}qg+c${kV!2j4+r%wZR5<#;~|EH0+%gX?Prj=9k
z78dpRG(qQ&gO>vuk*xYMO*f-{R8sKSJCfqO9fJ<B2mbXHHqXf~PyMxi(A`nGXFLOz
zjK}4{YH6OeoBjfs)@sowEL{S$(b4xUv$vT6`mQH7*(^Qe46m^NZ!XfS&2&C0s0?1&
z^jqKM!s^dkRi(BPZmabW;}KEi68ENv7Bb4yKe#P{#OfzEO4aQbIB$LH1UYa<!vbmD
zCvv0sEmZ?kB%eAK&mA>^nUi~IG2xw~g69O%BSIRCdZGL1>-T_Al9M!sY>=1!5~o?o
zaWMu*r2hDEgaTcia#{Y4`*@o3X$^pki|BLmh`A!w6lmocZftFitv7n6dixcFvCR=O
zooNgQ<S7#m^d4u3Z0OSE{>(?4R@7qd*T`ySg@N6_;(4;Q_gXvzoPke0Y)x()jHwxS
z`;>l{!E>_B6#aVeww&NYz=Q|G^jWE!^UH(c3dGtS?1pGw3xjn+6T6*m0Uieu9+K+$
z%%NfGnd7eN{sBE4LMy3Jl#(|WRBQuxY}Jr(oaZ6KCq-4Y+kJep;f^taf%vJG!?XSl
zB5=@)5nxW9o=A*bR6fi&X>M8LLDtZ(r@Y8^7&kExkSFp!gR`_t4fBfffkS)T-TmD4
z2q4Wd{olv?WG>__ZZGV%DSGa2mc>-&D&UcvF^N}8XJx-ZkyR4z9|emC`}o`~LVIFo
zseG5B!DUw*%+vWD>~f@8f<$&#r>7e%NHTl4*M|JgIb8U3=27kW8n`Xyn&zFlAV49R
zFPl+b=nSYti-QxS=WhQe7P&Qi4YyT6?GC4(N2>tSN%p&_-m04tT{$4SO$&w#KpD^g
zO(H09%7f;y?i@6HBy7OlCh6$f&H?q<82wF>UUS>UphiFmuaEjR)!I2*)r$oAq1KKW
z7JZpwJe-fwfCl&qK$##up8){(0K#i-jWXcNYdqw6@;+xST|Ub&qu1~)Q@v+6)K_L6
zs$4x+*Mb{FjO)A1+Ua%wYf8%K-!?1laQ#eXa85ga!LkN4&q1`}^F{#yl>muDz{lCQ
zE3LQE?wF`17_@Y%C+=MoxK44t0A@a2WXDO>PQsOPYcsl7K9HDGSh?HjW+pgaaa4a+
zVsxK|eFZ)e%rr^oPZ}J~H1Z|#nEUlF076_{TLCMl2_VmJLg9ZyKeecGRS(RA8m_Ka
z{O7-Scnuxjet01?=w*yC%X+nm)@5Cp#3rlbe2q6iSR6c6%CmN$_b7qRaZ<lFlfR5p
zjsoe+<vlwz#341>Ko<d`wpwf;%>kGw%Z}X5*~8#UfwRAhG_K`lRMp_T+HIMVY97j6
z_awu*3u0@RbST3O0iPi5y<g)`-aoy!wqsJIlcs2CXJ-d))}bMO3lLCE(!}NiaZrW`
zXskh=F1Yhz#}hth6KBWu*5&X*ZQ4@Nw}>M+J<}aCyFl$Z$QlhAA-!}+ezLU}l;{8$
zVX4Xo96ht*vzm@v4@MuEPtDmbg>7bc#amfPJ~e~;f{Kvnn4U59*ziLg+*8c_%2`jw
zE|aRTGjIR^oSqS$9u`gNu-TpwZ@Ni*?;iCPn&yAuA4<{K{g@;H(MfY(?-5{cUpZ5?
zh|$m#R)Um}&O0Ey-GM5f*aV=8HwAzYw&fA#2_w&9=ZUOte1~HZs$0Q9#M=jlh>;1k
zOlDw|%>PV^OR*?J@b5R1Q9dFJ_G`@K<af2rG_+?c=6hu5ohigv@dUeWaMZBgE36W_
zsBH81fPTd1ZfFNE6p#zYk@m$p6A(QIpSBJ#8z9Af8V<?@0Zh8-o<A_w+e_0@olvpX
z@8%O4N+BXp`~bQiEMATd5g9AsY>cPBeEQcob^UmC{riH>^VSn%Wz`kj1MYv8V&Y%$
znnQ~r03q!^ZvI~^Ie5)(I8$sJ_O_7{P^clYW#PcPZEWo3FL|Q4ktrm~(V=H<S`<L`
zgYTv7oL|~GX$&rTF(EZ~^>#(ydyRcN1BRqgMXg<_?%#%E?oornTVT;@YL3Lk4_8%H
zfx){}1U%>E-M!lOMPI+Rq}FxHk$(f9u<8u#um+>Qa{PK<n1e!N>{^+59<BCXu3h|Y
z;P<s_>(cAP_3Ea!#>dAKDLjSJBO(aB3OiBY%nIB9b<)S1X|AxXi=I(}WVcxbCal5-
z!Nw{1##;-UnZSCs-%N^k9UC1;W9tECn=9dA+fDx<MZ!0_Y!71F7B6FKu0F)Uzi3S>
zb-1$+J9(27K!;7rHm2%ERza=bD*!<o{=B50e%>7c;~xm}96&V=>#FL?N=a<;zOF8}
zD)vq+08+t~CvZ3%Kyh%K$F?z0ggF{rmW$~Swv=g^2M6bu1ONq2O~Dp%b*ctcFRZNQ
zUS0h;iS_$|Yluyu#=)n+=0b{r<5d_payNL+iSYSl&hP5eZte}{*1;Kzb3>m%V)COd
z*Mml%a-OZW?%Bl4tZw*^?YN`{HQwgrWir@Kq<Bcx87B4D4IF&p=l_RVZ&*9=pQP1)
z@rD1v8vfJAf4+g6`mZW<|2%~4`YM=*Pj6QIwo%m2EhaT>LBxBv@lJaVYWMV`(c4Aj
zcm1=KfwIW%+!;Fqz(GF^yB&;=ZOZ>EOC&shCe34~?6fSh=MLtaZ`C6yh<M~;q^2Qn
zMnUh^E!y%!sQH1-%!1ezoTbNLj2@JQy>7OdnL^UZkcjJHat!gf4|fFa9%geL_SLLs
z5Un|IL92tW-7tzaVW9Y_+Byd4*n}}1I%7^>Kwlz2ASIO292G#Z<yk3&-V&ny9o|>=
z^fi(ky-gi~vDpr#WZa&ean8hHQ?=r=pPXgJ(b^E-1_>r-M0}=4eo`G-glF?3>p{de
zuqv1e(mDY)$Sg{gJ5zCZO3jhS@_hzRKbCVI5abDEUlUzGZEdL<JpWuVuS%h_uQH3M
zp;BmMQp6{e`dRk)G7%Y2qZ#MVm}3Wp;gSGIgw^zjG|<Vkji23b8l#IT`#QRBa=?yZ
zLos{dV&IXoSOoKp>mrpr-gcQDFqqw0T6*OTHNTEGOxGd{4U6YY?*dGL!$t$1!?E0B
z+*j{NesmwCW6Km-BV8^R8y9^Z8InFQgqiA!fAgmYP*B#89Io-YxgD3U2wL@llMaWC
z98}>0v#c5h9D{#N_CEwCmVWwg)qFsIz~}P&r>{}73{Ik2!)&i3e-ZMvA@d;hqqGxK
z^~8Jwv9iv@6g#fJwK!tdWuUem0|VXh>HEgHzdsI#8<ba;s5NK<iD3lcgbD@~55HbP
z{~>dlOw9SXupVABPJQpz{^fEp&zcei;o{Q3UkygeArw}0>^qsM=3H^)E>$Co4++er
zP50q9<SeqXSGL(-SUU`GQ4*fLm-tA1rMNA?5#OPR-R<gWxGvHANA33nu3kRzh8&fT
z#<hJ}q+#B&`xHV;M1YxsB(5r;ia~0V8*!H~BZFXn+<%(-(Pkuuh4JasH(I`<55z&C
zbVPZNev^~oVi)lVgM6xaI(&zNy?dpI*rdKM(XN#^xgGa|=CVj!VTPA)<7~X6RznDV
z+1lXV{aj@gX9DuwoCt+!(rBzj6$|}UNnrg1ec>sqLnvNIp96t+Ir&HM*>Oj&hH<Yy
zv!F8=k^|gTi;GY;o1vPU-udSl&Uu31j&uDlW)E29-YsDeYFJxAW1qf-AJmN-d@7#R
z(EG)Knco==H|~DlCYdW;7)Q!oNLP?VzeYzV?ubwswR);#RWigmnt1bI!atGLAjQ>k
zRVtb@nDR!X$nm<1a|I89Zr75ngmhBht6eOa_DunpZ4~jgEE36Soox&t*PkCrks8)H
z71geD?PhZFjwXhZcp5TY>SM8}J0z+Uf@;juq_TFoRdNv=PblWA?nw;kP#p}V&P9Jb
zhMV#WK`(}h#MiVdrxSI*Fyrs3U(UzsI)0FEW7b|{*2=4YsKX(lG~3nUWKxH;CFX4U
z1UMyWBpz&j7;-DrKenrG&88UM!f<RX-8E05Mw=hqgv$92x|u*kVmggl67q_OCl5z7
zV3+3u^8))LqSN;h4YN_2Ja`4tYoBkDna4zY84K6^mi2(w_IV0@HcyT9OGfAC5rvz7
z4h+@E8!H^rh?8d97VvMLiRX}%y)abaedy8=_UE?C<7I3B!v99Wl<4Rdwa_Z{{*3_Z
z2o}EOHkDuEQb~~eR$x{QxI6BeEvk0BN7<9vs^84>;ILBr(cQwT{fAyu89>Fm0(Taz
zhPSBWl5~X|hnE-OsP><Bi+b3)bzBY~e$B;RGy;TwmWer!r+t1Uf!yVuEbHzI5GbV8
zenYIUNF1GU-7k(i2~jmgks}{qCZIf49RKR0*;eeu%cO-0f)B+@2!M`*#uR3;Gss0Y
zjt*$Kc{+g(ur3EZwR@g&fOJVP9P3DmAXykFXV83bt<^-Y=~*6GgZUD?nYM3_1|^UO
z%Wju@a*`n=Z$4WBfB@HWU*&bItHF9F_!oGZF7D}2R>Og`X^?<}CG;gitO!cAjZbK`
z5J71^Y0cZQBena?;)PdBDlUuYIB0T&!Ubh1l=jolHWRaCl@Y5OZ8@#>rt>ehaF;)+
z+4}H1Xcl?@M3Nz~F!2{f_d!DEPYfO9?`l3yk>pHSFO_gDHw(%8H3CB5!t@$?m*YmL
zYP_M?_$A+)3Vs5h*_SbIOX$1KVeSC8bupy-)g*<XU3sqtgO;mLu6;lEJ#J=rm7|ik
znYio=*cSkDzF7GvsqAcJ`|uB8k<{<!l!Vk<n*xI;NeUDI-~qoo$K?a%bOf_babo8%
zr8L|09%j+(Wwm=dmWN&;kAm3Q%&-BJ%#XPujxB(1hTtDO8RO*8dsy?BLpg4(-{*+*
zzRX}C@B~!kWn7zZzr|hk09^~~+mVw5lL9=e>Ge$#@wk*OQRazPgE!&D_rZ~>6D-I%
z3Ol>FGE>u}kbCiNH-|`6N559_^(e5gfD?cnio)tKaYvtFjvgvIh4B+27agTklMOil
zXqC&2Ub1XJTl$Qh-EOEr9k5@3>8h{F$uZrYt_L)$oV@%~c6N|4-Zz~L@CzIcN)@3Z
z0rzEaZzl&1F58^etn%2N2GGl`hor85%<Fe)>~TPeP8I9?%LIgNfH9cvDYR~c$oUH9
zc&&T<?7JBS0U${Jh7&~yU%XpZUcTWL-ErGx8X!u{;hfA-yxrd$0RWjF{@$rhlpnsP
zwc9Un9jEfjsV|pSCWu0J(6dUIt%jUW3mL4omV;7LLz$ppB><H&GBTh<s=_D=g<69H
z=|6yhyBe`=apU=O?sdRJ0>2!xPODHNy9CTW_0s}Ay+JvhiMke@#T@$i@Q@!O0_YFh
z1HhdGUR=BNhPc$2W_4NZ{<K6|2G!rRGe9sL#@mf*R!56MewH`A;`L7<KLu_CIwuQ6
z1;87A;S~dRg<9nF0=O@8j-G~NA0jckHh_@OUAhH8dPm@<t01`c6o{#bo4}2HPlv9|
z0*)4NE&yZ3dH@PnHL^dO)*~BGLx7V3Nzjl3rn==M(JzN1dg-_C=YlEg3#LXpOn7W`
za(b42U*?Qxz?`#`3Kpc<acw-EIGzZmX6IW5o%K1t*5g<dG=GInJ<RMHF<3i?8M`Hy
zo`#<8E;l8iD6D-#us6e;Yfx0V0fbge8Nt-I;RC+{;s<80-J-(H)fK4azfZ7Z>yQDg
zzN_K&x}5O=q^`vbk_27>6jXDXS=jl!rkRp}t3PPA9Suf-=vc%L^lMZM1}1%wPd+P~
za<X9oO!CN+{ETD$iN?~6kne|=j0PiWR1B#wiu+ykGkV#u`0+_$zZ-U8LqZa&FG;cf
zB*_wZ$lH!`HK{GVJ7^aPkrmr>!B3i5*@%6yll1I)5OO%m*sVUpHT0hS6q5x!A9Lvi
zpz7VZhqUp@w|)|guSLjAb_HJdK}k{OiLJl~Kaqz;uuR0(1v|qA-kPHWSR!;29c%bi
zy;tScg9NtIO2-R!@xd+=m_2(opEus6+igf5`s49W)#g{c--@WVHp%gVE}%Y(nxoDU
z^o(>nV1U5XJ)s)kJX~6`K$C}a+i-#Ebwxiy_!x?YO@X8FsTsnD9FHR^cn~W2##W-8
z_n|@IBzA3IYj7FLJ+hwB2HqMlkFY+kcd}7Goqu8*0*5HRNwFi;HWOCg_o`_aD<1Z-
zX-yRlAO8q;THRF1rs~~B5m~rv&z&?}P+Ax`3)Tyy;?)tgXM7z;D!&haIg)8IPCmT?
zfVjM#A@iF|vp<M*bMyTq1*R&e^stVO0%=fNFe$1yN4bqud_P#4zNCj6R9>zmzXHxd
z_>W?XiV@L=Xr&7c7Z8PlX8(*60F}r<sXp$SQw3EIAW;Rt_lMJhG~&Z!KBhJfMi+_Y
zm*)7QUP;=Kr-Vm`fWXcc)IWXylJ<Ir5iofEjP@RV!r~!dFG=Yc>et#cP-4l$#SIqV
z`?YiHS1Gwa)Am=<)cigw`lj^(R9vk#{4KCx6b(uv6yYSs-u#r%Uq@CoW1R{nz@sqw
z`1DO2AOrvfv-mTmpeglxsqL>5qrV>)s1Jk0DZk)>O%CqCRPV5!i1O?k0M{5x{%sD5
zD&AIJ4s!5yRKVNOK?RWT3B%f7+$98_Y*5a1VAR?C+>RWyGh*v7aeQ=^+fOI7d@%66
zMlApER`5X<VADZFLf_J<TL&f?YfjOpB*wn{%1b|(O(Nk_p$pSn5NlMTba0v4_*f&s
zQN{tzKJZT)s+CftTm2Ip`G@T1{wNL*>N<|6!UoMsGP5>Au|98;t^M(080FfuX0?+g
z^vaAYJ}o7o45--GvH&<(P!jF+gmawc=GGGjbzDEU2Wnv~CY4$~9%C1!IXMy$5rur<
z##*mZ;Gly$HSLLoWmW22l2Y=0dqZr59uv7x4|%dDfC`MjKoIxUlun5rpR+i?m?f1I
zjI?D3v~vl5VhsY|#ZT*})7b)E)-hMQ0)oVUd*DKuxx;8w?F}OTi)T{gA7CNk6B1+y
z?oQb93UGb{ZEw2dLy{CgCAso6cW|AvyAP=AkW+jfP;~L|<uvOyMVVz)k1P!_Gbd-H
z0Zu=tWOv783z3`KjB<W`l1ZX8Y6d}1Akh;L*aA591STgIu#}!V*?lnd54rvQ_Uz*A
zJ==5|c&r~Y_%mYt3~{_@J1*?@2M3um!3T6cA`&*+SA9IC_QrRAPR6i!vU(`>sG}Qy
z=6H4#OdFw_fmZcp&5p}TE9A+=8aAL<<+#@`>ndx9DlImkdVW3n73Ii37|8zYj*{7x
zkARKmr?hTTyf2Vg%#l|?Kt-_vyNEmLxmYd*)&^iha5|iWlM0YFdap+T@qT3{p9ph^
zKJxYf#zW^~Uga+Dw@I#;JF^eKFSELPpiXm5tp|?pvcUvA<Sv*ZyIn=NPrY_tBuaW$
zyom1+h_bxc!_U^NyaFH9T>z#N01jEq>??k@K)|Ss>c63kPB&kDWdp<fJ#efUI}a!i
zM?A;+Z@)vv0W<(e*d2*++@|iv4G<4u3jhL((xy~RV_u#reEFqt$9^pKv^^rS?%68<
zK;-1scz?9B%QR6tFL5c@m^i=P!DK5hTS5##Uyewp)Zz^@iv{n;()&U}TIV*|0<t<2
z>xOIrriv$bNqYKG>NgqX?u`I|kRKJH+BlIVAG^DpJ<mq5F)H9^fs^SAhknbl0ycuH
z29P?T8+2H(w-g?-!BYEVDZ~vpd?Et7Bcj@6fG-8S7$hgUGdw7%;7lBg=wqQsPX1?*
z8XUD60f)jd#58C-mCUc-x}u6Wn-xj3bcq3R7)WJ+k@;d(C#C`h9k8(6Y9Z$Z!h&_A
z%yREyYKCk%uv}YE=^%H)hhN5SMYVw2AEJuU%TYm#J=`0xSQmrU!M?dU>u`-d!s67>
z0Z>B-W}Dfwe$-*J_|33$dtk_a9-lBRqmvJv+|6LU(amcXM;;FGb&2A30a5>_bAL}}
zl?|CmNxq_FbuFGfU`^UltqM6nDKuoamTx3xuw&UBni~y?<h(1QF!NU_`esfcih97k
z00s)3)fWrJIC|<8!{Uc2!pA;@0eed<ev{@8f<$Xg`UVcp^$J<Zmv174q6aY@O0H`O
zPfVQC#j>(nIhFQBTibe~%Gq?&EPf&ku%o<3{-XysSn(8hh-B;Y?Q(#7L0B|G1bEIX
zi|qFh#Nr>e5B^M22GC9z(HybG-UH$eK#{sgd|18ceTykZkDI4=b3Yt(kh&|dvq;7U
z+-9oHZq2wmfArpC*O`A%?v$t}&kuqdLoe;CAZGR4nt2&4M>$lV2XD<-U%*%8!e_<t
zEOa>@?5b;+Ruazx*E29gvEMpTL)3}!!Yx<_K1KFuD9@T!vSti1l=^)-u@J*Zfb~Y%
z@W#h1#yi;qwM4Z^Q#?0@mpq?GwVksEhJ3jfqBtiW0?AQXa?rjZ;>D=#Tiv?NTv?jz
zJ696;YX?&4?QKQ@p%dX}RIj2Y`C64P18-E$OWJ64yzH}-l5P}el`13|m&L^qxE<p|
zG?30VFNfXt4$kK<Os?u7Y@cv;h#Pdh=EH4$lCim)oHqtt&r)kir+2P^x(+w=itnKU
zPLnLXWz?6f(vl*8zl+avUmNlJTn9Vo9Z%iQ-ZwV%MY&h%-Z6u0x8t+ilI51ss&dj>
zAq;O(dtCVTWq6Dl>P%PsoIomYHjxf$_}U^|A0GS4KH}J~ic>?bfx8#_kyc)gZr4?#
zHj8aUi+_wM=GA3c>+jbk$v-a=TkYdjD!5sv$1>JbB!8j%;5@W8H`ZF@k|(6dQo-gi
zd_urgSfPl@8ip@?(wY%pd@47$s7bxM+7|+<PEz3HCDlKh^FeTuJ_=D!qN&Ky*)C(w
z6R%cMZ;y!!5_(?<4UcMlCUbIY?T?wT-)&^qC1<np@$C*%i^xDAKi{)w&cg2X!8$(0
z><EpPmsW)Qv6?R@b&8yw%L{W9Pr6Q1CpW@eY5N!PYCC%ow=Zd@%&!5z*Mh({Cg$sx
zOjab;w6bBPvyiZ^i(`A|Uvj5CMiBNU_k%_BPf8i`Ocw|N-?L!*4+E5!T<9l=p*60_
zQ1a|E+Q6-qA0nHBVpM(2y=|eIj~+L^m|7!!{!B*HxZ(C?KR=7P-i!3K6Rv!{6-ew%
zDGZ^G45m*mRFKf9HI}ds^QYL6F|;FpOzXWxQ*rC^@i;i0DOi{3|DLSE4&uMa*Z&|k
z!TJTi{O9|U@dEBM_`yHl|1e&cfBx?e0SgCwF2DbsT>U>0X8-=@|Ivs4c*y^IAO7P-
n{&|Iedy#)y|KIeXvmApj-V$pc6+;`1eMea-MadFzqxb&<wnwrM

literal 0
HcmV?d00001

diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index b4588652c..675c1b51a 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -16,9 +16,9 @@ components and an outline of the data flow that takes place inside the system
 below with links for more detailed information about them. The data flow is
 also described below.
 
-.. image:: _images/scrapy_architecture.png
+.. image:: _images/scrapy_architecture_02.png
    :width: 700
-   :height: 494
+   :height: 470
    :alt: Scrapy architecture
 
 Components
@@ -143,4 +143,3 @@ links:
 .. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
 .. _Twisted - hello, asynchronous programming: http://jessenoller.com/2009/02/11/twisted-hello-asynchronous-programming/
 .. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/
-

From 7d432872bf4a9fe0c287e796aa1d9464b2f55e66 Mon Sep 17 00:00:00 2001
From: Lorena <lguerra@cid-inc.com>
Date: Thu, 4 Aug 2016 11:01:14 -0700
Subject: [PATCH 1035/2974] text updates to match graphic

---
 .../topics/_images/scrapy_architecture_02.png | Bin 34097 -> 34433 bytes
 docs/topics/architecture.rst                  |  29 +++++++++---------
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/docs/topics/_images/scrapy_architecture_02.png b/docs/topics/_images/scrapy_architecture_02.png
index 9178b35414eb46bccda1b075d3cb8ac5bbfa0405..80be02078a101e8fbf58ed605e2e365bd43ed346 100644
GIT binary patch
literal 34433
zcmb?@1yq$?w=RMrh=fwoDWH_njglfI(v5U?H&W7FQX<{C=@4l)A>G}v0qN$h&G(;k
z{(H_C_l|MLeFttC8}@qVJJ*b7&Sx%yKFNrqKOuaAfPjE5@j>J>0s>MW0s>+f3L<#N
zmD@oC{D*4&LDdcc0aFM5?}4kcXcz(lIf8`9dj;p|-8nCn<TH=`y9fj{AFO3$Jzhui
z&<{tMr&3B$&ZjjLv?Zf4mJY7biqsQv-29p{9m6k8CfM<N!z=e4*yq2NIK$%WVdS(s
zn57M$?Y}c*@b=dzr6f7pz<yLNPD^i(|DJ3OZ&g8|cu?WJPeyiiYC++LK;h>saC`7_
zuKuLX%7gkql5GBLBQrTA1qaBAFiac41pgmp3jRt#1YXGXk^b}5&m4T~9f|&*uUHii
z!1qEN;N_p!ciqU~+kgA`zb|tCafE+c<e!iK+amvb{C_O+|1N_cmJp8qp-$9?RbNbj
zA0y=|!NI}frP{KB;5kyBOi@u$(aNS}f%jj(ehrtJd_t(Y%3`KEIQU6DDS_kp!51Fz
zg!;(r|6L*fzn?`%=1B$2Z}-fd7G^=}yi>krO%4>Pk1F<G8vb{+{F{dVLpj1IH8aH{
zTfI=|7-Mm9F;A*Y?nqAqPwD{7vCyqI73SEmu&<o{v!Ng-M{JMyY__(|#HDHe@X$5^
z`qIg|PJ^AD{jhpsXKc&<@ZDk2_V)I3j%<F*-@*rCNC@6VpWop#Job6_YyV800gu2U
zW_N0M_&Lf$HFfo!?QNM<o@~0{**$A6bB^S}jToD{{QOrgRNEGXW2O+bvF^PS*BehS
za;#4{qLvDSALVe;2NTL(xy-&J|1&$Aa)8y?0XM5KDS4cZof9B9feNlD>O(}-v)C&x
zVhnEQq5!qRvDakJg=#gJa!2yl_qPb;R2lHt`JqGt>4S!q7dM{lWLNG2xg%SS4NY35
z{f3g03T9@7t^ziJK#Se4V?!9wY_ZfdNQTFACjA)jLcniutrydrj7C@j0nLm1X9dz_
zG4u+Jvcp#7TWF=ufJAVSM0Fr)2D<Tz1T517=QtLs)f`@E6wd`>sVktl-3l;kF8%26
zLY*l8m#D(kgBwTAd>DbgHO~<cuqvWNDb$sOfKY{F-?N5=+2^{;-7OE_8=ORoBy|`_
ze*<a~SB|MtWBQpEcWFRgl3!oXBPv(rF!?W$NA;P}Dg@ehPI|c-{shX8lo97C;fQLj
z=Z*WF-*^(_j76EdC~+kBui55P%~rf>Szs<~gDjvgV*q=7*WF1OxzeRoS_AeWF4&a&
z!H3_l<!$Fv`y8z}y1-!%0d0Bt7~-l^XTIgj_j}{xN_Ca4VL9fnu~2H~EF-#D=N?6Y
z=bs9&javI{SlNS<y!@fsRZ*81*63Cld`=oci-ppi32mr&mUNz+oUE<}zjCy$^5}xw
zmLfBV?^Va33sl!BolUMDZ%3Wtr>`WTX;7|$=;Pg?8zrtRE2h!3J>;VJ+l-6^L!2Co
z+z41%_E&I2A)0l9HXE>e8lhrxIIv*WiWK47LdJ;kc$d5kjX6qAH3mEf1a;}_<!x){
zSxDk`qLW5<JTA@lLn?X9R`sXw@YBu4ZMP80s?W*6(W@!|gsak<Cyfxtb7te@G;6Dw
zWwNS)K=Q-#c&i`~U`PaC$1=@Wo#2ag)?$YgzDROKnb$=?h~&4AiHRZLo&5Fd*HT+x
zjm3<ToZQ<vfm65UnA4Sixhb_1h!Pa-)fO|GAQ6=P$?N%VArS7M!-Fe-+1S`N?SXYG
zD=X7(l*Zam6c!fR)oDzdIIcQ2%%xf1eg95aGsEw2$94G4v4Q`h9Ow$VR*;`Rv}M1x
zwq_H7i14{}ml7ee&aZg><ig#t;pUeQ66(fv6!eE{(>&5+j4ZC2Cg|FsGhb?IY9)Vp
zepkf|Z2PI8ZI5F5pwr^72lFBbA4sUC5F$}6Yt&IP=sv2et4m6T%;u&3)kOIMQNtwW
z`axp_oYNhh#=(o*ztM62vX&&PHt7&sU}EKOw;#-&XN>i)Ym+?f_(KHh)(;#A`d^>s
zOG`_$j9%W{=I=1Q3Z@h;?$FK8&yU&~-?EpP$f<A9=#Y>~($kuwpg2*P-nhK+yvVzz
z0e%IALM=GdXJbzhykFu~uyP>T?~$Oq<B-RBiG`B)thx?<xzwkQoy2`+2vCyzIC?72
zIahw|+0vmahx0N9`U>IQzz>aw2rTb0h-#{<O-r3pWN4V#LV4qP4vRIFae*5kdz+7r
zj#66z)we6k$Y$?FW=&FcAi$bT>laC#7(h@#Q)|>JowGF3dWMilWLoin9J@%#Oj1%(
zPfw2=E7G@JKBO1~6q#K4{nx>iM|&wf+)}HseAXl>r=s|HJm7V?#8L7;vRK+s5#EJF
znIm*#fdDl?KRh{^=+37u^Yka46f>Qu91kz=nBJ@4pE9D9dORyr_o5XIOo4r})u}Bn
ze_JA;o@%o{oGl67IZE$uGpv->Vuu{Af(^E1^DqsFFIN}XKDwx4bw`3{*GLEb==hoy
z<jS>BH;!aR)-?i-WNAiK2`&mJu%Bw_{e1u2Vt4uMYR<9-y7t+wTB@dJ;(L$Oa)OUm
z{~ywbE|3;za|T!wd$Z{7led0&*NRP@2C$5<f-S~sr5*nDKSjiaO3Ixk=H~Ww8rxWI
zhQYrdz|(&syTdX@0(9-KGl>6duXm18_(W1OWK&0YN<!6n{TJ+Cl5NYcZbH<^C|jQA
zjZGe<NMRww6KT>U91^~G!R3B_0K5fc+)Y>gN#IxhSTofYne=6Js~;R^#T?Z5xs%iS
z4dtPb6)#oo7GK9xehZDg-QCeWGI0U;(>!pvxDv!diA_j=I?s)dub*9k9F$Y?iFGdM
z0m5@#Y9e3~C7DctVQX2VK%ZNLq}3`6{r&xR(AMCe-IdX3<0=gC0|PRFZ4_pZVx^rF
zb!6|g&?s{T{7~Sk-l1Q>3jYCori#gr0%N@(k69vkUV@0=OVN;*x4E8dmGcjn*n9}y
zd_d52@C<xJ1>=7>zlze6O@Icy*JwDn3F_Tdn+i8ij#2NCRplEN0n%?`!sRM-cxal<
z#L0&9Xu_VJa-PlkKYl5E`muQ;dU0Vtm-4m&1%!0VCmp&QVq%w_onl>G8wd6T5KbKY
zm}*owFo9~C+g4cFA1iS2)tINgv9O|J(t_Gz!!e5Y8bPVs?<Ll#59SXL(<lU6I*qAK
zvEiU1KB~FAe8Ep>i?B)NIfOGI-Zq08E*E)M2Muk+W8WJ+Jp$h4faT@y(b0&h;RCe#
zNbfj!Mu`Rvo^i&P<gvVc_-Hl!9oyyJq7K7K&_HCmmz!!*zIXmMu+KNp#u@D3)2Etg
z^&1WN;=Xz#x9_W0G`Nf|^?gk`I=8T0EV47C(sc%WE*&~+84dg|AoS?&o!(Wv*g@Ac
zIBM<KlL~4d<2WdY-tFz6ippw-b+nfMEeXjSdAR~!TQ|A*t5vFt8kv45$7v<3L*Cx(
zy^137w%0Y@obB%ZnI6z#T=eSI5)6c1mFRk=aE$OUZPYHkmFYixFGvJ)P)E8I4R0Vz
z)(%)r+omUyu#dv36!I5e{hGMO@o+6(Gp<Qf=p%KD0mwk&8Q(#aD?G%S0ky~YtlTnA
zSAr3iTpFMMh8W}dsnE<P4fl|qc6H%y<g#;@`b<u+{{A-EO9?8yiy4hXud9!~#CJ3<
zdO<wwe7l%ND-g0#BcOf#>BUKBajWw`b=Jyv#!Q;{-?A-zO+;b6KN}_p%#;bcXH}jL
zgoUQOAnctUDU`>_`9lNb5vjgyz48@w7JS#wR*|ln?v>tg++F;{gh<s)-~BoP38|a`
z!|D;woMv2HU%E7q$QwNRv=+<|8Iytg#4K$7s&MPEeug&2Q$O8u2Em@*-@j*|z0Q?~
zz(UbhKUpREg=K#WrqRy@)^bW~P9btPsiKWMkwy))=$KYij3s3LruYYancr@aAt8%?
zX4K!m@@;LBYv!^Keaj8B>03W0BH)ehlh$SLEN|}IW(YTZK}Ly6U0sVvFHhXlGAd%(
zI$7Y6NhhB<Z(DkaqplYd!@qREqMZ>CiLTQ+R>C$~!p2AChxK!3*?6KO3T$-WY@leB
zL3T!OR*Oe=hGTX{_3$SHU6zOX5O8nvWL!5fBg&}PnKs<o4<x|zqWGvch%&8B4iVug
z<Nx^5YM1p^Dq-tr?KhjP<!+d=^0$Zak-I6lyxcGyM{>f{dSHvev*C-tPeRBVI>sJd
zqz#!TppBzY=-<Xe$5HU)0t4#*)NAO987rw85elq^POj=NY(`&aJJCGn#txz`wUs+|
zHRuSoZ4x*$oPcl9XGNNuP!ox8cc{obt9|z2YSlVNko&9XouAql7)mB|BOBUNo+mNC
zjM?Jjnums7x<32zI+x^|7}8LlATiUQEhU1dM8P4_kGP{{fM>_Qk-n8Nnfx4|ljj5(
z(r`UnHnwJH{_MZQE??7HrMnV)UWndnV{_4qL5~i6=$JI}Ptpf{?#8~!)z!MFC_5^u
zU@dJuw)J1XfG7Q8d9I9+rpqS#wxSC)Jj|)z6HBu=ST(VfGM@#1U3RN*=S}=meFj&i
zpt|xAI;=29vCLpCuaKUqEO7{sC9<TObYB3Cl?f>^;iXzwYJA=e)aOCSP_6yO1ASEo
z5NQ}3bd8OBq~{Z~a=w0@uBiBVovkd_85{uul<I4HmpWz|tl$n$?l)yPW`_6m%M{Vu
zEkB^rjciHD68eAMqFW+lS~b6~D9fbmfY=})5y{cBup71Ta@8utuLH*fbx|f18{1we
zW$5qUl~!`_Y__>~<2F1G+%0`Z{dLP(qq-Y>eOriM90aOba`@bqm%l83LK89sBw?rk
zQ4hQUl>Y)PF>7)3;9&aaPY~bWRjs!()yfJ8vI8QN`a7yY(lTj;q|F`rl<-?F);q&6
z0~0Px!;freo=<(!Na?44{~R7jS+++H-egXg^h{0hCHDKnpO=6VAN4G-je7S2df0;W
zCK?@qfSrqD-ob`VQYNS9O^ai;59g!qP#~w35XB{hrfy6OxGji1&j13EJn{RP8_3jU
zvAZe##%26Iey*Sragzk6p=u6{F%@W=rR-%)e-us<Xl58rp=(}f8t4v0hjL)t(%}3=
z<b{D80hC#`SmoQ^y7M@6<+n{2w_$o}Dm6dH#=xwOTKQ0x^1f1>Rs&JOki>?{HmaLR
z@#AK{hd+mk_&J3*ykEW}G1)^EbqioZqma>VdatWhKj6ot&3Wo&=~$;BJ%6HX71{Cr
zw^qO$(Kj*NCIPfTX-<hbe(zoDJWd9`&3Z0EVj^W$wg-fj*~m6Ybs(Pg`Ckx8Dk~%%
z+hcr~tlPkgF%h<T3-;DeYM5Vm2snC0a3*m2H#{;m<g6DBa4#R8xQ_?1xa%W~DqN{P
z?>Ht{R|6oTqRAg6;jWRN_UZVO)%ho@S`ci?i;~ol-s~KKTfyUuwo%sf!>yC#Pn%NI
zLV^rzqNSzsIN-t7*4jSocB~{c-+7V;=FYuino`xgBna7xQpc@FJR0TX7-#4DAgJ%1
z|6>vJz8?@DCET=Rb-mGXH`10>rlLz)T6})lW%YyXjZt||Px{$e<*Zcaq;l!N?@HYc
zaRf9F_>8$Y-V5sV{Kfn%65_&5`ZT(%6~_|+uQvvQTl3m~PFT)V6a>(E$gWj&aRyk&
z$GFPNJk%5PNq!+-+FxyLrd#dRbGhbmUe|klQx%C!tuA8ghLY_UGBMKkO?d4%9}jYl
zFeVb=$vg(xF`&C{_&zcp`~B3s$+SIePWu#_Bo^snEB1`$uuX`IljV2OB!OY~;BdM&
z&_HMb%T}S(<rTxkI$K;JcW*ZF8X=qSkj-z-VceXKF-4Qr-;sSIf&S=Inb&*0Rg}OH
zse|T(sJ{u=qJOAU3;V;6?a3pO$_awBC(Hq@AQ`PR%XHkS{U=I?E`f^cdW_jPU%lIz
zZ-n4&54)KkR)e;Cx#xzALj%@wHox0)+F>Q8U@UK;fRgsY?D%P>$JJ=6RbIhmn`Q&4
z`?bvpojj2;W`V`V+0G6&X;dW0VUbOv@5N!=HRFq8*NlFfb%d?{Shf@rF3BRzB6gKn
zKY^*_v>YzvYLIB6$T_l0AWa#{#iwmK-#8@sj*XhR-nvk<F9@C}+zh0W6_zr;qHj2C
zgV=3!ZSSnX*cBDG<(sdskA%L(bcHun84VsR{*c;6C6b*C&o45sr!ownV3yqIr2_fy
zckh!UcCQ=R{HH&Fy)8~cLYRHOCwjiD74K@eT)A76d;|?0oetgMz*}DsFwna>c|;T|
zLS=Mp(5RHvZoX%1MTUVa*=Ja{Mx|cut`Ef{a1s2heihi@n9tiar`QwbMdydLyl}d&
zEPOiJnQLYQZZ1|QAbDpl3i4W`6#CZ@^q5>aCWtIMdzX>bCz&siiBeCgWhV9XlG2G9
zZkw-e2XAjkN?X$6jm=n+1ASW`nYu60fy|~tlkZXBvhIv9#f&sIDL?l~@ud9nGW_)P
zVq<ZEly`h+zJw7&LT?2vx1#Kg*<cth85|qxcjF0NFO_KC)np3uBJ!suobFz%erHeD
zs9fJqcm3F)X?cAl>sirmetS6I#L6L{$s3}ph!<!M+EonXO2-K4>I~Q+?EKY@$bZ2@
zrQ67!eC>!B9XGA<y13XFJUhNNUT79~+7G91k(oTVhcO5RkAd7*amHc{5B<Ly8OGmK
zWBqJ5FIlKN66EDEoMr9tk7#RMj5vku7+WdlqDu^)|J0aE{PWs3<6SOvTK3TzC5XF6
zEGz=e-^3F{NseT<fi{}4!kCFe-WE>8-^Xx<rN|rb@U<)v+<7hg%}Y9EnN1!PgxV-(
zA;A@n1cX7h0L6P}OUu+GxTh~TI^<u)#>RCNv&gsD83j$HIG-f#5XsKV1RNI1B&@Yf
z-s9^FwW2~&MXxHeiBO7$%qC5eyylF8*IyE4RNg+HI1h#Czmt}bh{x1Jqp*)5NarE}
z`7FU688O5ewql#PdEyWpeX}7U&Efqh>t7k{L6)SCjFX${R5X|3eIAS#`&~lMJh*Oh
zb)O<l^j<Gg)=1TM=&K270uMnzNKD9PsjklgxK&eKp(<7Yx6ugbqI-S+O-(}FE7zOn
zycYA^2;JTw&id-DOt}XDunpisqx>)8*qxJG>qJi#6p*r|4UaxEbc6g#e%sl#koKv{
zXw!DqG8fxIK^;Y`D?wP>LKOfQT8w3<QdXL$d+R&CymF-6J&}XCuJ6t1m`Yx~QvEr`
z1(d?AYi-`Rvz*sRGCGSVM7-T(>6gDkj3i@cFEvtVvp!yE=-~<b=<8ZjVBaA&64!zP
zq6oi&lJlJ9yXsfHgUo-nbc^_-l_X10k^wN>ih89=lsY!sNG%p+AO>PPj!>ciF-mcM
zq|rPp4(z)?HPhqP&2xvX>)Ae8Iq{Kx|BISZHsh7^&S6|d8h$}$6`WwL1wjrm!K|}F
zXtaPA2!loBT^VC97+H_EJ{?PK9u9lR0e#5Qt)RDDSzI(2Wp>r;@q0{*pQsm{K*pAm
zIU5}}y0kV4H&M$qG(J{kZo%e+CBD`Y<114+|Ne?cuBn8J_q<{RTBX|o){j?wT!w;i
z5@3)eFiJ!k8cqhS8=9^7ech5ud_H<N@$9N2h`?!pl}OSNc3@e0RftLRZ5r*PQf~Kw
zX2b9&#e&=?kG6mv--d+XZw1=F`m?pC3JC`AgcW2~kQdzZIA<Mf^d&H5a&_JQuAcGE
z;|O{fDGm0r67|dOUQfdunC;OIe0s7jA)V77+vd!|PIf{yVuLQu*M7aD!^<Qa78=^w
z!SM$x26?;);=l8{(mujmb1ze3#|=mwY+-wrvI2<<xa@zNemR>^|5a6BLAzwSV)#Jc
zqJygDwf&>@MNne}WdPj5$RMUCEdj4)pF7V99{1}Z*|54rmKkL&eD?TD)}ni}#VsHd
zvUXJSiV*}F%*0}VO9!v6lJuw<Z-4yM&1Y5`tk?#*aqG{T0lPyd{*lm;?4;1Q&GNlC
zuhWwR0UIi^jE=+gDbR$s`}AYPf{rPNY<M_ElK%^+p%{M9n$h1*ggdCmcljG6qtV<R
zHduhqn-1I_-+8X|Og|A2B`_S{*ax|`|8c?AlRoLVp;{4TnV#2$PwNiI9)pO1ry)ys
z`CTw}Mw$O|;MQpCZ2p5)^2@v7<Huw#60N9_M$O3fD`SU=5_3VJ!%!|D%l(O72qYyF
zTY=s1`7(`cK2AL9JM*PUyM`8lxy;_Zz-U4KvW+#r8T6B6uTw}DsmBqL0LK;tp!G}%
zuQMJ!#jW7!IMKf4z~f+P8B)?0FGx0~*SJ|$Hx?ka1{jT4z&Jac@;HfWp7swvS-5jq
z7W}BSmAE%lfa#TZy~S@vnzolQW+8hwZ^G|8Wyk6uK^X5RVt5=D=75edjA>Unsw}Tu
zzD(uaL7T7~<CX6vR_SqIdKgbXfkH9g<VxlC**Q)^`!dH$_mbpiSN)0ly2=WXrQl)G
z;57xS5pB;OAryj4e=HCn)_=RVE{Cg_7d}Coz{5=?BFLaOghr;P6QQaDJgburFUv7C
z^N*A&YdONvDDdp%dhjX9+_(2nX{X+|Wb>3V2pkbOSu##Rgi)UDZ0*cl;*aoi4w^r`
z?m}}Ayqsfk;oO<f+R2;lr6*8JPDA~p@*1Gv)C8t-j(+pCzBko6)I`5ND;o?KK0<ia
z$!x)tz>_1jUfjP~Fp{x3ipF`D?sK~u7(%*}Mwm*k?f!WlW6w<Xa5hNieB5pW`sAq6
z-Gb19tNC<XgG6WCGN^Ez{4JF-?xuZm-}-=LLLvvR0|<F367V1n<xdzC+}1QUPKS}#
zT?D1N7-nKoj*oCQ-Yq^QXexBu?-Cqmt@1XQQp{I;js4AMN~{yq%{)jC>M2FRaw(TT
zJ9wWUB*hwW^a?$DJMyNQ?HOpgni1R}Gl&Ljwd7aDLdNC4yd|{LnQq?koH@!t{N*0%
zNSr=)yZz==<PEXQ4GIa$YV|#~P4e<U27-Z}3$dFVkga=hU!(uc>A|h<mKw$Lckvs$
zJfdG-c5mTq+B>YRai625*We+n>nV*KnFN;mE|}+@c^km}yBCUIPOK|z^OsCPLI2C2
z1XNzy79YU0>3Ft?K|ePUM}9Ha;bz6f8I4OY@;nViOf9J{+Qp0MQTP<;uZ(BOl5AIm
z1EMd&mR#cDhtt`B@{PtwbZ>Kc;DxUliy7%zS2v8-zs33cJ1Huayk>}XQpUwIl=Z3I
zML(IfJ%FuaV<H@tQ$n!#Holk(-x%@`biV|a1U<P-vp$!iuCT5&rg!xS8_L^UG1|~j
zSI}20*Bzogh*zYn;|Tys&6|zxBjdwCix&;f8%nM(x?9h)N5*1f6$rx6o<VS%Jd0Z+
zvzSEjN>Wge@y*_URIQDkK~?|6goto-!>(q?a1rBXIB8VUHYxg452=n6vR_a%tDwen
z*RJ|wq?qybrEhRBk7!c)p<DTrcK;N!mlIuOFXyNWJrI9ggN<qVbD(h;SzKDoRIrd@
z#Gy5P<J(gXeRY$d3Nf9Fvg+HG3h)}0D{CGuyWyC}(di)|Aj1^N^w87uF=jg{w~&AD
z^K*jU^=39W`Kq&1=Am!WE0QS5{tfGA@N#yzbm2m1S~))7{cq1ye8};*N4}27+P1RO
zQ8<DmzjZ`-gm$o)KUR5<1f(SMU2crtFVF=9U-FVA8{{0^Q`i_+ZD_m;QJ}-cLYOe7
zzcdQ^a)fkuM;SxHT_ard+J6U%E2SrW-Mome7Ik-hL!#0h@w^JcIUlLi4<}|A4SeO%
zl$Pk3{z>V@<!-4@4TF_3zOHx?CYtp9(@Qsask-&dL~!`dAAk7BOKYrv$g^5pi!SNs
z=HIlV>Xzu5qmLn0&PP^g<I0CQ2#In3bQ;n==-a!yr%4kW9u)?MfOx?;lg$Kg8gR5-
zVPf!|4D%87uo5Xl_%&5{L$=CkzedgTHnY9Ok*nF{v{1R}Wp!ECoyS420}X;pG4ZRe
zPD5U#p!^>{D$1<mTFZ!;EI<qDotT}Y&LGsTglNbrpULZXXo_YR4U><E--f37l`V(2
zJ&H(XhRW_?yU3#La!7R?F1^y~T94$LEhd1X7-B(E@hNS_;UDLpp^X=49=SfxJ?i~v
zXj8PpkEDGmo>YcrsgY#Flol#26(Fuhra=t|DX?9((#Chm!dITpceW%AjDRH{t?(K(
z)ao5O5Bfho-Q-j(p42dy^i=<wyUjG)B92miIW`A09)4ZL>UgTVYkmiI*2v4_baQ53
z(^+S5TiKEIj@*0p7vOQ&{JFCM8oZwcC?l6@5co2E64bpTF6Qa!UbW)<yQ+BMvqj#5
zMx)(M(YA02zlYt}WehWod)~RcgVY;DIFa5h-En5Y*lbmZrAzrD-0{Ny54xKt>FM=y
zKspR)DaY%ctxs&=s~q}t=O*-RD6{36P-NKy@-8pf$Rhv}|Bap}Z#XG(NtF4APM!)u
zBYU9pWorK-{B+Ob#%M3O;;m-dHaD%Gv!`bmne)!ud3J<E+W!+fofrsm)(6`EJG=jn
z46Hk<M{iR9o#O`D|K9{esLS(a0#vl^Wu~H?=Uz(@D++~7vhv`@!!-Wsgj4*JzSKfz
zqX+OChU9Ey60P5Alz+LUJ=o>-1F^2)@*vP25x%2StWHI}NeY*{L(TplU<i?G|D$OS
z*FnWMnKsLsEBEI7nKfbChKZfx{QM-#4ZxQJnw<kM+JJMg=h4*EOr6+?BLcL9Ol~2d
z>gMBWkbK?=3;UXPt=XD5{rQ}X=)D#MS0<`87ct;!uuz=&NE_xnw?|)z%8kNo$wE3P
z{SK~f00p>ZuU0$`u6*7r!0?7AfB$~%=>?V=Gqo@=`RmMwj*brKj0%#e*ujlTwc=@3
zK-U0{H#-{;W`G$NmHYnv`>HS_BjdKE%^+grjq5mH-$N>3CMsG*N`)g^vG410a&jOL
zMKN`UhTfIGc`g%%m1Co$&P`f@Z5eY@`)5!*K<M~mJ;or$65>i7I6OQA;;gS5PR0JD
z1FSC-lN%pt?#Qc|zZdS?N!@rA8Kt`X9*-3JUZB8fp$|zseS9R`#*E@W81W?4)n`U?
zRf?68Isk#jgT*9VSKZgPWDTC4-L4P@p3p@-+@IMsE4+$~U}rm=bSL~ogH!p3rzE7g
zy53sSp{Inz-4+n(56E+gfxW@zTAKth_V}@D{?;3x7q5))<Dfj$=Z0_)blGqbTh=sW
zWMmW<_nSg`-|lh&GQXH{hy(D=%R-K3W@Z`=kdZCGVS!)E&+jpXFzKpx8+ZL(Y(5<d
zH))`EE#h@8a#Q*CD=h(l@;9*QRuoR}(52tv2r7sgObM?2x`0(9v{-XYm6@2Fq^qF|
zri2QD2?>4nZyf4?daT!AQaz-QZy>%<N+&{jINAVsdEN5vw^oYs@GAzo28dXjw#jY9
z2sHji{%23myq5gY&<9^iOxj#w!psEkdgg?$rY{&CbUy_yrGpwd!B9VE8Pf5S(L;&Z
zeKed-9ZpycbiyyOmW9%BH=xFdu1va*c6OC&ANW^n0F%2CTnxzgVkI-cT>`T_6_Pml
z-9=!gwoc_*boF(mE=b22V~7u+)2Q&3IA(z)BaXui#IheRdXPcKPpfj%nhQ{&jwvdZ
z33froXUaI)K%HCmKn@4Imi6`Zo}M1b1g3mcUdv4$OC5ETC3_wuAMfd}9PUnzj!!5?
zRj*eEw%Mbm;>#2>NxC+s5dYb`?rTEjmS=FbQE}AvDQ{2g7+#1d(G-{mnI*64&@ERh
zHbv7aRZi7Y56?FOu=;}5+=ACV#ha^dK^))!gO6hKs}v0o2zv<*Z&2-)$TAI4R8@jZ
zZ(TXQf!H(Ez|@jwn;^D>t-BC;puz>R;jhojU@oICPI%x@P3$7_Cmlwh@WAa5*tnJ2
zpL7!2oGCK78f9Vw<FwEF9T<regzGwVO^l5r4XAC_rl+l$&!kq+Fg+{zEd!ce2&X5s
z6k4V}wIpHU!Y7;<_yPBy+2p2umvoh5-Z)^i|NX5#a0*pr#!`b=OfX9TNX43lIrvOP
z;h5{<FP~Z#Ge+*HYQkP@nSx@vV4g+BT&Ac5h2Ha$R8jTRB@vXUvV884ljCfbw;$e*
zy$0!Jjn8B0lnNxy0j$|G2)xq4np<6^ZUSv(^eOzo>xdZvfw|j20k;Vd$XwS4h?9mc
zrZPPSn@8|(jtDAt|0~rHRWERy4Ef}0eN0OP1xbIz{wU3Qh6gMk$FIoJ?RyHk4h1~n
zcRNN+L{Pg_9sf-Q70mj$ZS4&TXh#Yy0ta~KNV|JV^-61>%faBprPB!^krh}d`HuVD
z&997k;-u~idpe#$yw3W1qrPz<bHFKCDCv&Y-05-AxJ)`q04SwF6@4-K0F*9<c!EHM
zt3|<*8Goa`%G-Vapr>fBfHIWJ*|+h~)Kn?w@0vN=7U%clBNGZ&^MFYHX&b(dlQHa?
z1<(Y8lldAo7mE!b^vm<NJQw2AK5;wZ_Cd1ZuP)+A86$hZ@DT9#2&xtWnWJ!i9pnyp
zzl4GBNzQ#v_;BW>U?2v@xj+u^wpZJXG8gN85Oi@h95N*e7>eRD#{evi3bU@@LOZ>}
z82z0Z1*icz^VxBKEj4fIQ+Rw+(K3A6+L@}JyQi18P-H3fyduQOGCL65xEHK2bwd_6
zr2-akSNl+btpBWS5&;hx3|n}c8_m|S_ZZvP-gJi+3d!I%ZxAfp)95M%PfwD;5@A(j
z66eN;(L2=oL%_uDSCjOvxXR$O<JmZ*pRbGiY7Wo#zbceIK?-#DGpD8qBZ)wCm%7ic
zP#FOEa0hcK2sV<!;Ep|SV}tPRjztz9H1eLfr0N3Hgxz{DtM<n#zhv@?4($(uiZPt&
zj#nAbg1NX_zN!N|Z|BK`(Dwu`?@O_k>`MS32<E-s)Ryv(k0kWU13`iH=-Y_H<v@sj
zH3iAkp9ueO6$N4m@OJDRi<@{;fR0u?)Crp4dxb!*xGm+5-D+L6Gab$5Vj%XF{eD95
zDB6hQGdg0Q_*kG|7WVLkWYB*N9KKV^1mhfd<32T{Bl9G)S1XJ2tTsd=Yrk*Y{IDKt
zd4yU%_U`BaLnrVFr~lYrpcNr$4g~T{&~eGN48@Q|F5sYRi@6ku7!X(e0{Toya~OSj
zM*VGl%OIX0FesjS!DOy~QDKHG$b4IH%ssQdXGT@ykl}_Ax#KvecG_M15gFq+p@mD<
z3JG9=cNi55M<lShtINSzP6NGqzA-x)l)u;4gxWv09zi(1AA^RN;g2Ke@r64g+|(HO
z0Y`h_dxpFfNG)?CmwLJ<S*|9_;5r`3r5hgJ&Ozp$AcF+7;cZY1T5Mf));DTYnr~CT
ze~&}upqV}b4uw@O6>)E<epp}8pu@}mxOYvF7RV!jf&oKG^7{aNIB+KSkI&AUL)6wz
zL4*d$eMi7Y%Sul#b76mC4;3Ngn+x0{dY7!Ls;a=~(__R34RfR@51;RWnF%lu*l%cN
zVj|H@>rI{55&<`GynCErZfa_3W`_0<5sVR{I)R}!p>JT;1?2s+W7no*())ev0V5RD
z5nv|9@8H7ScCfm-di4qpn12ngI5rrt7h685nPDUwi}npatai++Xl&FF5Jh>0@{o{_
z5TGU(mpbU$FzLX?5lAra$bVRjfU%D5?6kpAswWhLL_}aT2wX%&L}SQfzMXTaB_yR2
z>dMMsz7J&G)%Ur3uXy#h(>{oT5gn^~Fw0l#+qP7u!T<nLg)#}oV=%3WM}jYas6w|*
zw#QE@3$i!XZ-5MRUhjXL=36~+1+$S@zxU6~pr)Ek2>`$0)-WYV%Hf>Ub4GOORZgv{
z<bd_2%)z|l%HK)YseG<X4-7S85?s^M)3dV-dN#8HBSB{3N=UrK7+~OsQ~~cXRJdFI
zGtSGUwCybliUer3XRfJ<NfHweeXuPUy=s`#0%KnxVw*g058{R{Sp$}NRnacu=@hCS
z$WhzpC4(FCIHLVi6^||$(g!D|rwh=hOwB6%lu7Hs#Abu~>@x!&D*z_yeW7cnmX>L6
zPdXt~f%7xd(|%YS$;CN2ojWH5RQX|HVcV~Ihy@*F1!#LfyiM+hIRf;sIy9wQZtv*G
z0Vd+O#a5kY5mZECLlC<0z%T5d0W_A>0f9WQrB<1@G&TK96+5wWa&;r0xgGkl&Q#OK
z)!1RBC`$`nYVLrzyA+;Kn}=AN@hvPYz>q1-Q9cX_9v>&7%Xt!EU`h${prTy}16jRT
ziJjQ3Nvm*-FhR+PVF_2LyFEuhZa(pCr}UaV4<la4oMi&lu|$E7`~tRogN#P15H$rr
z*W*)AW}QAJ#J=}~-vhTE0!PV~ZX*7aY=$ce;DSIpGZsp_zq#tfl5O3tW5ekyFxkf&
z3bU<S?TE6h;Vtpw-1}C6CCZ2q#rZi&!(}@O;FI3tSul~+=s$X<A)r54n7Va#m8-O?
z!hi>+GpY3fgp=WHAVXll0~v$?@#L_z_Pw!-K^C!3I8k6c^qif8V|;9^r-}!nL@j~3
zZ!ey5mqZ^tJT>(y;Wcmx0Ln*j9dnyE4wtOC0A6eqQecctx8wxVVGWw&YtTS26${XJ
zST{on-IECcMEF!ncn<U(3ni2?(oX_v3IVIvX_Npb%7;)^_)YAeHOyJEN516y$zxwk
z(kAorP{wH;Oxc0G0ti;4WVJFjt9iqgoEA9>pP^Sm@3~ebop5#6O4PGifa;NEz`odK
z&cG}!EFfyd0L(AWpm~dm*P(bbP*ysC$x6o;Gr(4`m7x@-sy0N*^v@G~V-)be?Sz!^
zlSY_ZWBATa+r>Qds$a=z-e7Sb;pOF}rKM%otk0631fgd;uNq9??aOG%dLo00gGsZV
zjhlOFYD%J<7akfk#N!P>ps|pN#%Jl&jN3fAcs@W+`I&A>ainp!YyXUsjV+G|!kW}g
zBoQFGGQcR2?Xc~Jh+vBV@5lO_val!j<0&650W+eSvJDIfKn8&a>bUmiVAMra!RBQn
zY6RVet9(g?S}}_h6lPn1t-4qkc4z}@Pfd0l-s8&OM}^O<|I#o{cpH)YcG|GgrAZ66
zXrnPzGO}fEX<3q=FY)~I%138xQn_(q=yQzZ{`J4kAm)H6)=_ryHE&l9A1e)Bj6l8Z
zx|e*}ex<roo(LefbPP2=CEm2Bs~^hRpvCwg`cMA)z{$-m!HketX%unFe<h<0$XVtS
ze9By(o!woH<AKI2uRHevM!vBldW!ZV#TL%=5w3%AJ+w;}FbI>@qgAYJc08}$AMx7z
zsU8?Q1Jml7)g)+)OUe~PH46;~ztYA^cR9>k!k7>vk!tTp%s#;3SJwLgG>`<rV|E*W
z=KkO}#be>;g6Fpm4)Zl4U%^*t>Fm3~`{Mq2gIns~JqNK%ilFcuQrO(xwAXrbh*9|5
zsf{Z#g6)g@!FDVF&T_Et*`<GqaTN|w6MOsP)>K0m=gA(QrK$dSq<!Dtuvr}I+I0Nq
zQ~-G<^4aZV*-hr79c8?t(v(Z%V`o{=@@W5*wc6@>3Fj;jyklQfdsW@FczNj3eYRft
zxFUR#t$iMgMd?^Aczzv0Wpz#yKzS)m^t$B?m*<~Pj0&Gx6lza-Ecx)4V=^J1H9t4?
z5Jnkt>q=P51iz`&awwHnml*z?mB}3p->HNjCl@NURPdHosi3@li&Iyf>UYnIy;HNT
zRseT=4)3~QS=sKgq$21janD?j2`cR?H69pI?^gYwAwq~RQ<2rqy6IL?b~;vZD$13$
zjD=Qj)hO(k6{6KBB<xPV7k}{K=cvnmw=AzXW{c6~@1_7)%@75=u5K3N4S#$Fi2%)9
z%Ro?tfUhqUh;CX;Q;@bQGjV|DFVWu#;WD$O-L-4SnFLh52Pk&Ufbs?D`L<-#DOLWy
zU{$Gy^WFO+Y+gyS1&POqA0kmEluJAfIQ=P2L;$W~csZmu*W`37x{az%1VM7sYW{OD
zEAmE)6EX2)OI<L$UGs}YdJ>=b=wr{n$}V^)rrvFb{Yk$cGhMT!e6e@;8*Piw2Nc;D
zyDmvukKlgFVAg6?UC4Tz-SKHSNdj=E?eN)R7%1pLDJnl(2k_l*#xWOz&RgcjH^SBn
z6P9@kQfG*81BuVnd4kG2XwD{BUI~7Uukvk&p*U|iP=j_+3JL!_>xu$${w`A185-kA
zk1r4hD!O_?K(jk?4?y!gB~KvEu0`sAB><K81$G-u9FJZf^Dd7{-%Ys1#Me_k?r%mz
zY18CCeYC>>S(FUEhLFgxM`5G#_)>~qIVx{(kFvm_l`MP&ou#LI7ZF0=aFaO+s~2(>
zG}vp;x>}n`6+F&`3FtCuT{~}A_<_~E#2lxUnS_Z)<9lw_D3qS^F+MJ%{kpVt^H}%_
zGe_lDsu4%Yo%D|GN#8*P4|4OU?V6(bG!|$swhZ&lli>)h#^IHMZ9yeM)9-vF4-uqd
zNPGgcsGk|6v-%rpP)s)EnGnuwZN(o~`E!W&_m<0dc_(2(8Q-veUynVGW-uV0JhF}c
zy31E-1KBX)4j|zlp94KsLpP6AbitWS5=8Cx6c3naR9U&VU__<*8^xRVdaVk5c$Fat
zzYy=z*zrp}*9dNM(1pHDKE`h=z4@H0{cQFsSZWgY?oT517wBfnsvIyP1Cj8PV@+#5
z{GBZ*E0GcI_e{;XkV$Uv@XFW1db48BIq*=>awuc)%4poqZN1J9MeDg}i<z3kWDpFK
zE64us_G}u|X<kg1Y!!%nR%o;<Y<xC5Vd<%B1WJr;G6>k@$ES^ada9>C0|TNT?{wX+
zOte`DDqs2eg+2N>X`=c@@%m^5|LNOHR@yl&S3N>?^<TDNkgt@O$l0HhffiQ3mcI2+
zr<&$#)KR)K&uMGiP+)XG!)vYcN`myU%Jro!(${%vkiBlsRhVBTrPRYVeB!~EgHxa^
z#k+}EG#z|VJy#7ntnLbt{-|JR$6^fI3wriz_ISLSIk%i0lwst;hkYHU1x&Rz*@wGS
zN9LcB3(yLbM5iOVTZ{|YXDcG@C5}T6cbO<eI?%%0Wy%m1mWS(f3bvXAx^6Ky75lE~
zzk?oQ=v(ua{$qbPVdiFFdGv!CDZTQalY(SXY;I#3#^!+6M_>lT&0(TS^Gn3j-zuaY
zLNp)k&|gF{r2H!pgg50O-4$Q%Ik5Jjr=LEOzuSyEYMKA+`m;I~6Bk6<NXFjg%dUio
zGaRo1rjDDV-p^_pG)%%-(a=ii{39&^Qi6j%5^fgUxvfw3cUQjo*T&S^Zxp9){@71E
zp(VJxZBrh^u7F1qhG-+ElIwZp&kNcu*{=0d?vR;~{TZIdyC3Uxoue}w(mfAA%YxPJ
z^^I}d`DTF8USCwCM;_p|ZdyxkJIoAjTB#fOZEu}=n3-g3rN1^iPNz3ZYYHdV2Xs$;
zIGsrY+UVQZ9d%+VlE(Wwrc=D@`9ZTmt|eQIv16QO-w)5ML!6v-TgWbbli@=75AS@l
z$QdR4)Dn5&OUzauer6CxK?C5ZuM@DAvJcy^eZQQ1{lHyk6`ji*7i^&uj_S6l``7v7
z2rTTLc$qCCY*~T*IwPI=K&!_Cs?>Lmci%bSTsa^?0l9aL<mRf)&&I{|B1q)0PYlL4
zwINu?dlC8sFg?sxJm{eHa&)ok2vB50+Qbg7kaN>KI9PLk=pw<|2>kTyYU~sg7XMjc
ztYB>abslYiABX_u$>@zHlk)GF*uW1rkCehNm^JnZ0`-Vw%Q?_<;Dx7tsxcc62M690
ze=#vJ%Bb$u;<a^ZH_V7mQZtfTlasX>PN~GF{{H^-V!HKCGSb(VyHD>PC=`EuzECa&
zPBH;xgYpnG?&d579kxMB^)Mqo-i&@8;N+GCtC|@SjKGnt6w3|Ja|JTjHI<hSfm0*w
z>^<b96F<OkJ{R$Q%Xj<szF2Wl=a+Dq(Jjw<t6%8#DwuQ!j1+)rh2_galIR)d*S7VY
z-MJ_rNp*jY`k)Ukb8~ZNXRhZ#p%D?<X9qB%P61E5C1x3#D~sC(v_~xs<W{mzqIy~M
zY{LZ^>_GEs%f6RfU0!|!6e42kb(&+-)35!E&CIZcCe5qCNtM!xLS%!w%2_5Pd(dD7
zZOG~G_jO5phK3wYL>D&DWhj$g4Zyuupt^u1<<Ztze<WMTZ$h*M<c78ORjA1fSKn|%
zA|AkfWhg=!pR~0XwnlPg)A+WIF6Q0#o51(!f?fhRPdaGLlYR?&P3Q+vgwd8Q&qoBw
zUXxlqRA3Fqg1%!Y6k+&Rv$>nm=iR%#91KAyk*R-B=(=ggx9R43-+S1Xb{BGK>}+i*
z%Qk8Sx>Iak+pX^K-WM8h9xB4HfolK1?CVRk<~bXl(eXzF+Ipnt*JLSP`}`DsJ^J9N
z!fOTwhS#rIlB60{q4J>Y#3eChNb9|Ck2cKBkB=KxPAv!XQ>#XT9$#Zdj<t(0;{-T$
z0FGr~Xci7yfQlOEA6%A})oR5`h3rVoeD@6VkLGHg=`luToj+QSGIl-b4lkqZ8F5iT
zX3=Rjq`OOuDUVnva{H}xD80`&lf>}WFYUI9hl%uwzfeZxWgdq$K0f}&#>SE5ItM5@
zwG1YQhku4u1ACw60v0CK14<J>EP&!V%Fv=bkb%a;!a@;8bUgT5q3^<W7^u9%c0ti>
zqwa?V>c)F6L6qohECw7?n*O~<aPs0~bb>*b>m_oU>#JKMn2J-!QAF>rRQ&>TILJVg
z;r&?LU$n*Ux{pWU_qU<t4|>EvvHzrWVPJY$hou_-90oXG2`*n>-;Kd^e4*8i4Ipnq
zY%C5w{?5te^0Myy4#PA4`Am?t?;Kaz{Vi*NLAQ_b7uqP1L@MEAS5|`RvO$BXbYf;|
zD!<d1+v{7u;fSS_kfAZCSGi{lE7{rDK!Fb06DRJYXk<pbjU)r<k^`W1^(o+9#-x7E
zBc{7V*@5u6s`}MrtrM26(J=SZ@NX;Os$-535fNSJ4)R*j=(<K%%zR}=H$C_p3$cwk
zf9^{#&}nC^Oxsgy%gLch?l-KN0o?+^c3RrfhG`{|`tk5D;4B&OK|Cvxsp|cK5>-xJ
z&^);4yZGHjDlBnW0nhfP?B5)1FN0e3A(T>#U-i?O>Jfg$-Br6?Zu)JBP)7Oq_}Fa>
zr1H5O&N(gIjFoyi9Io~pZ46>!U>LG|(RRrNYCOlp#1#BOq^&3O_f>+`0@ii%V8DpY
zNmir&(e6~`VRXOt-Q`3sIE3l6H*23@P6-3)_4e-8;U=emt@;&}`E1-b8_?P>no*Ls
zioRJ1E?fCnHRfw7Ts;F*+V3qSjY*dcrr^E6z;z>w!{;L0=7gH8gMgdo{#W9pMi+qM
z=-kR8o@g+Tq0$`1Gwi8fR0#MIUM4NjBmD;@GjGuEu*%=QCCfasBxrJy1aYP0e2|rX
z9@caBCh<+UcC$y@q$VeAK4ubauOv%)88hF3`EfhJ2>zb5UL{^ghbg3BM1dACI-q=R
zTl$oXB{BS?Xy?GSX&e1*_9QqC1<p=YgHQyHlg4Pca6fM0i0{YpEl>hDUACrVd(_sV
z*r__<$mbyikvC0BKii!d5c^o=tPBgJAb{b&Ohbe<Mcybg>Rxp5g?U`B1Gi%E1NMq{
zy*HhZ-4p%WA8lOXlU79;+2}nX+_f3RDumK@Ix{>|3s@^~OfUGEMQc^RVN3@R@EC9)
zj7cjd%THJihnpy<amTSCRc2#U&En*_1ce`ndH7u5lww2|m3k!Rg}Z<Qr>ar(a!Bwh
zzI|+QKH0T3MjIBAY*XxYw0J-{NaEHVB5(K(#u6xD;-441jJ2=8xW_teiFyvj<L)WJ
zQ7^k6179lItuIc|Rh@PFw4#}Wi>)4>-8$>iQTox<+86-x8IMxm>~ST;ECU>D<R%UW
zhXK1yO_IE0WDOXh5Q70nE@D)pM>agErmjj80Z$(yDixl9vt_1qXU=@4Ti-f#CHVEx
zD0n25X;9$_RXtpPBY^<qZ7>E0TE{BNXQ3^$SIYReSG4pODwL?2vvE&MMjigf=^>T=
z>EJe*V7!x}W#9&-3czdqF29w9Z`OP9)B9MNdkWn<ySm2s7_AOmx}V1;WF4m-<fdkf
zbBbZM<XWoQK*KK9CT>)J6z<H1!mgpGytf83BtwD=DMWL&d7S2V6MGxeT31b#O+9BC
zym)WJv}N{(T22(u*C^{YiY>Hd^vnQ8px%vzk8LbETS7LrUPe`B4&{B%O;Eb%vK4lb
zcnu27^2(dzwv>|{^p4E(>*KK3&6Q0le(0dF#6}I+CeCYfS-j@XS^7JbYC7hmg4Nbb
zfNddblrc*~V}Y~N9%(HbysXuB(~Uf5JR56cV^*vEDNZ{9Ot-50S0>!cn><Hnq=n1k
zxPWJ_OGXK}YB><9kAL(}wWSI$+j28quV6f;A+uZ@&px|(OEIX2`A=Ir0F<W^gr%sf
zo_?%B==+aZ06?p#QiJ(PhIsvL2F(lKpkz|+y9O6hwn1~gd3USY_Q63Kl&N2~_}7MR
zuC%{M?<p#mdV*3slr^$XE!xtI%uyz8Tk#3)u9u%veu=l#Z3l3s$H<?D^|9;E+(F!2
zJe_%Z$FDn-_dqr1GT@7G$v6glAN>`6`(r<Itcvp8o5h=7nDay$?bvz6Y}a#LRx0gx
zje(~S+9`@ba|Y1-&-1JViEGe2^yC=eHq06_2>}~Lkko|&&3gqSDG6#HBO{lzlS?l$
zF))nooT$FY-(VH>NUL)`0btsBacVOtZHm-YdeFb(R`pt7_9|>pYVLfj#hDT2!IF*?
z_jG{rE%4fm(qed_+>?+fkq0P>by~;uq_E}PN1l72|1UE5XD!@r=@JrRS3F;I`@Ih9
zE9=YMv9w%Zwxu^>d1{IkX5xTE2BZ{?s{fa&m2<%OvaQ4AV;VAMBMg%Pdh(wOQP+wt
zd9`gYu-CLAS;_xE*m)Eyt%EtcVdCX^zv?)dl-r>G?z_X}K2WByHs0x2YZrK}v5s1C
z2sX2}c{L+ZxMlF!?qK?3C0Cc4T;|Hw-W~U2ThS&iRV*=k-L)?r$&;C$Ll^Fj@dYkp
z@V(|vo$g-M@ZFv{&AIHyZRy=zMtHeiw!d__db}?PKb5antfpOIGV|Sy^tv0gNo{!Z
z2IF!Irfc)vk51XWj{T)OKi`ZZLyTUWv0$f(&C2aHI<>8Mc_lc8n7_V|`z<`=<@UFI
zh;!gLm4qXHd*9`N=H!^gobp8vEAzmeH8=u>zgAnVjDIm$5j7DbeWF%=CYMwO+&tsv
zF?ba{NnXQJ16u3pssv^E`jIxq8d-NJ!z*TqE1VOT^<~hITE9tU<^GOkBFHjq^Ut6P
z<HjJze2pUS^K$E3k1K8CeZ1o0+NBl?^{XU{M>%G9R><?*E_;x+o5u^dLJE7e9x$Wo
zS5C7Ge&-!tH|gb{vTVCx$*1lzGvC)j(&rCo(<5bfFNdA{u)v!o;Ae+JTc`Wr4aEHv
z9HHxknwDoy(dD1o?_lW<=efCUEx~cgqFI}x0Kus*^$tz-!kagh&4M@ejm4z=OK9LH
z4X1V6?;)djRfE#b8bYT(w|IGr=9RD=gJpMluNpF~QcN!QXh&00XSc8uWYTNxuVI+U
z%~wHHk5vg`m5Z5^#wusiz2rT{mNGVFwXPk`f+TIq1z1tt7A5agE}BXYp!{~I4L2VS
zhufA@)z`uRv)^;42ZH%wZ>X)C+|4>Q1uwvL!5d_$9X$+JhY`%4e?!D0NW*5%+p%Tc
z_w3TKg+|B5P$@v4xV}Dc_UGQ8nVGAbeXoU`6AX0pV)dFmpU0%NcehvIVDbv#J~#?1
z;8BkplP+Zmb$97W$vsO#2>`7j(k-qu>tNR#Lz>#eQn)0g4M93w(+(C=)TMoiNjj<R
z-E|M{YP^QM%Al9B+H%;{PkTS#wK`p4HmhG@Mt~hLuRrPh#Y-zbh@K~LwjG99GJD=1
z#~1*?oyHRz@TmV#`La%h?hrb7E6so-x8G}5b~~ax+E|$8i$%#1y9>a@;|26ayNXk?
zTk~pF?j;-4DZArE?OV@tFA;yNJwXq7lyV#S3tz8`E1_?_CogF+(i5(Xsr1%jpSms|
zxNzqrTZ`~fOf3erZFn6nm$nHH)p#Dy40>6ZMc<57UiRGa*et^^Wx);|hq0cq&EiZf
z+8c2drZ8$tMvsh>u5A(->HludUCQCzFOJTo9emSKioW%65}(w$T8BSG&tOzVxnN}z
z`}@yMoI6jRXHUf=9AwY#u-3@zR!&^q+}tLL)$waQjE#*aN;HLqg%2&QtoAuuF8Zm&
z!`-iD7ShyxYvaA~Ya8|Jq~%&f){??hKcnjci(4y~MWa~#<^b$&LyNcZlyznmuP%bv
zQ=t6n$~M!stG}0o^YJ`)zM&cWmMK-~w5P`K7zPKe6L1NZFQqqq)x9J&fu45bz@Skm
z2Ck3ZJBYZ8*om-D$$I!4)|G!+y*3M6k;y+LBbB!z8dsD8VPfTZ#ReS{8e#vX^vpy%
zjMV|^KyhVSy(i=HLY<@5<cg5n*&QgUN4VNmDQ-uy+0;uK1t{-mIx_Y8sr=c#MT_+F
zid<4rl8%Ip78|<@JttQV3+<8A)<{Nd$K&>-pZ3g%p#*nM>0kI3w^NWodLRLj81T7i
zQ+XIx5!6kmawyMW^>`Jp?mnb)5XgqLXq7r`TOSsdHowL`-4Hmd17_>F@3z3}db~N5
zDSrP~Ig;NR(GsM~c(Oo0xk4r5+i*@k*fFr;_B_EFDdz00h+H}w3?4L(@lip^n6<H6
z3!m}F>Lm$8+E3XPJTJjYmlw|z-cM(rtM}CJjjS$ek7P7Uy_;yu4>X6QtgPoCPpK`Y
z#%yyNpS_){7dXD1%3mYfMg!rd?oLPfw?|Hz{`O#hJ+l?}ol9JueE52rz}q)7jr3i<
zZNm99$dGudlia?K3sXy~ijD2r7JB)&AR^Bm%clSFe4z3PAPAN(XcWQwn%9xwudK|1
zN(DA9O+hvW7+E2dIy;vO$UZgTOSk3qzrp|d!4;SsdwD#PF_5ka=J)C|7xJ8Mhd^=y
z%_!-IZ2<hS@yK^$r0xj1o$mI;&@OnK&1v6kE5lK|*X{M<A~`B=9WQ&%`n0l^i@~;h
zIC0%)=oN?-vo0H{vV4F28fRu^cvqI5b0jYJyhC~R#2*<Mc~yLF<ESU%vd(S;l+%Y1
zmqJo<gEQqR!>4gAKSQMm!qPr6B1Q>Mn8tnk?y8sZjG}$CH!{i&$rMXAWN7{Yh2K^@
zkrCeoqP$#isb&2}fj|A!70vo}SCy;$MNo}kEK&_iS})@(0e<2XO%0VF=EdxKu;B4^
znA!BL3<^c_)g5V%JBj0JZtDN)>^*>*in_K@6crGpDn*(g2ucT~NK+6H5Kxd_rI*l=
z4pBis>0P7=C<q8h?@j5UNQcm*6CeZ#EtKT$;C=sj?|gUeo%u3OoC(P}`|Pv!+Ru8{
zTF)UgwU6D1M~FFgWn7g5De9pi_F!8eB7iS|Q+sh86Vt%s404hV&3P9ufFxNpC_sA+
z)M>z!f^?#NCCG!p&y%kCMSR5!T4hWu@z9VYcZ{*mqKRzW>BnxK0AYH1#0JgN!r9r`
zL%<bOO*yHSY3wSk{<&pM<*CIl&|P(Rog(0%cv40ISM;P=Ueh7(hpe988a?Z$-se30
zme$N;ODo{gdq&xiHS45y_N^Uh?Q&E;7ZW9&lH2)yqcYVRS0ss=?=E-1p3A#0AIH^f
zT;v%~4W`VCrr2{dyj^mWm!I!hqkalZi}f^cLrGUB9$tYAxjrEHde=`b4xX5=G+C2>
z+V3aE6tLI}X3qnMNR7sw8K<P^a)8{&b}zPzaXjR8q77?z21Nv3C#Oi*5->p^n;h`e
z?+}jR`3Mrz_~=351rP$E{I$3PGRBtGlMo=(V^T6c@51|mH4S)qo&%ZBHlm8ruR~2`
zvrV3umN|rORWTB=1O^JzQ+3tong+b?c*YBp(q!^e|Ms}I&tsW0L^ZbbD~)ZzZf^%b
z=`T?xw%+{R{phsprobYhc}pL=!?Qy|`7{5A!t3_>2SoNGRsK7}SZADN#UWg;5|Om;
zIw%F!S5uI2*{KC~gvtZId^=VJpBK@-r61GgAX~G~3gb;>vlI~bQTzI**D$YO_9cJt
znvAP*$MSX3*u^_5^?CERMLJxOS`puTUk|k~fO+$pH5`R{ic4J&C`5YYAO2o#uF|GW
zN%KCDKU|$N3bWIoKGoPhjjkjywE06JT8-F6i{K{dtETVqR`)kEr}o`ZeY_M?1q5hz
z49E@6g?H>r^+3M9VsQ@ilzhk_l|7ozJ~diOq`woAmrkP#I85xg3>6uyY!K8$)fI_F
zV)qGSPrF@bYUX2+oO64-rn>O_VUVb(=I7^}^PF--H68*|@vN}<xR4=`aoWe`Ugzi1
zRt~NpiIs@wPl;dbs+F)+o`G%z;r(f;fxGJ{6C3}Per<-eV0<flsYtH}%vYi{jO>eo
zLkY*oALsr8?fzUt=Z3BF>KxCnUbkpJeoubcfA|}hS}qrMQOosh{>0BJ_C2h=<Gjkp
z5r&rw)&vO{L&@a10_IA6Z0=H^TGg}BJPo6`%ae>^eOm*xg=5M7SO>#JYIKAW1YjDS
zK8vfd99sYcvbBAv9>~L9x5zh1f49B&L3L}OIpyu=q>pr1$c!Y^dBW3b8{&5V=7tGe
z)l$DmN1t<H-}i>Xi};HFR8|PT1GNqkiWxWJ0t@vAf1XX(x&wkvrt%QZ{ySbNkSpUv
zSE6}#t<mrZ>cHl|bI)-m_Sm?%qQb-0mfMGI|3?$k9cRJrZZnyy(AXhP{cq7IK*eQf
z&v=S2cUP7E=+dNoy<M|3wqN#Kat})c-()=Gm2g(BE56mB856J(x!(NSj_+tJjwa9s
zlNxvX_M}#FV(nJ?MAJ&!6f1VA-ISEAKrAs&xHF8`RK2l6KK+;T;>vFDVq5n=)7l5%
zI{@s6<?BSiwL8dSN4#*yU51X$k^u`tO#(P6!u?4=;p;Zn`GLeuiz$juF5-}A9Bb~p
zLYH$_0?sv?qM-j3w(M4GBu5NV(&3LrVvvU?pahbsLZ`6!hJu94-#0T(C545crY6O}
zK6~zErSBnBKOr+5)F#IN$PZK(^TeF`uswOPS$1$cRV;1&ol9w|4w}^vJ_E&_&Ec?z
zdFtjxal^)@nD;KL=_36LVirMqVZlYJ^Q+vR!}hp58S{reL#Oci!c3KY)P8w-Hp{Sn
z+x{T!I|F?-F1FaqjB;Um-0Y$8iy@`odYCJ1uoyK=0wx|~lJS9T&z+ktrS4*siuycP
zXq`Z4QcLeoAJ{xmzJV5dnX9|XGCK%hCMyEe)^i4NP`^Xg(J`M@pVQXIZMCPA!oNgg
z-cYGWLfiTk%{uhQQ_N$Pt2W?j_p0le>k!>fDxBiCdUJ`&TvyVh*=RgIZ$XCIwANN4
zw7mMezo@N`h;Ze{BCqY=vA8W|(HZG#@g?S_{cYVaR$)c{Sy>i$_aat0k~Y-g#?lfo
zVqL|^cs`mXAQN@l@TjO6y6mBA{==XdGGwu|9@AjpAVVn3L}ArvRLLPvC3k5S-FsHv
z?o>TKJ|5a>CBZm^^n}F}HFleqo|jphXlrv49vfkj1LJ4SudX;K>+9<y5Qy6pGdYp4
zaTzE&ynQNAw)JyIs~`IE!YTt@X?cckF6a9Ep4WGrrC>K(h=%#m0{<EL@ElUV-m_@=
zpf{GaiALi-Q^rrp)nLovuVJzw9(tORce=-(v7MweHzo#C63Y5`t+A%k_3G!11%F>=
zUr)40HHVcxcuPaGEPvR5p2s!f1fYwM5tEcpUSr;*&23vrG18oFTRD{G7-PHgvNUhd
zI``!Af&tgu##od;3b<Z%6LCg&XkxzV$+~FmDXJ28xHB!=;I@#nNlP851sM3Y{<b^u
z`5<KgYEaVqYtaIcf%7YoVzUEcR!|UO)T7=)Gu>+rqHt&FOU>>^e(3zabj}$ZJ##pf
z)vfWWz$o4a%wxsIg{F*rBCZ#`!rDSNNohhie>7tUU|ae{+&zh?W)>7yNyAHh1D$Mf
z?e*)|GMOdW*>dW;<XnC{-PUTyhW%=)DAQHHt(LXP;>+#}cB#Q%Z_IHzr6cG~c-WUW
z;c+oa(PLKy7vk<~;lv+zW$=tS>od5$G6^qybc(_qKUt)}?dvUat#?M~`}E&syjo^b
zT!+o~GGX-rAmY!TTf<ep--Zz{JBgQ%N8R(Pw5JjUruU0s$(v%J>+QM~2HGHtf>HQy
zX~4D`4}nBh(zNc}xWK(@SjmvmJlO4MDdA2p;(%&SjprXzboBY>QVB~~KK~rGu6eRQ
zJI3npy|~y30s&QgyC@U!%IY|<l^FJy^kA3jdo2jxsz#6W_oWqyzKsg_s*hAJnpK@D
z7k~#+Yft_t=C4I8JHv2JV}(!jjie1Ww&(-<Qmq6(6u6~yxI<o@Dv%=ln1@_7$jvAy
zA`0Z;?&}QXCrXvo>J$7f;Evl5ph#G)(DInhn*v>Nd%yN!BNdJF*c5C-?7^sGnXM%j
zRJ(whcEcFHU=W@BG$^7et`jrV^h&tKEPfL?HYk!#2gAA{s;+rKrf$h&K0%M*i2jx*
zB8AwRp{|N9yEE~_9M-d^HiWMO_MadrwLXq47M0ylh;@+n^ugxMHrHQ<srfbE38uG`
zk?b5#CP5@#H~%%O;x1!p!!>8ugq1{nQ?f5NDb`%+_}n&5R8pWoCx+W+V$JO5Z>~nI
zHAj~G<o-1^ksoVNL^h~6Vas!OHa3-W3i*ESD>I+EK$Dgy2?*RfCb^Tf$7P5rPYnXT
zEM?BH{ZcGj77+KwYGK0W2ku#hJ@hAZeAUa8c~bMo;7EBMjni?|JJ_HIG!@q({m{8j
z20hu3wRMiP`na?bdX+t7b6W|sK(nFIG%hVA2tWD_9SuQgm%DN&V1EAa1!YbTAr`_Z
z70e}8apZSzgX|dqPGxgduw{_Np_+mWBz>C_<(BN(lP`x{r5$%t^TWlvjutqK7XyCx
z^2H02k(Y$7rUcQ?MhFN9XjIK=mE7gwi9b9}ZN@NyL>|if)e*S`R5*-MWvfPh1+lm1
zq42^fyj;TWKqDiL)t2i8w{X0_OeXKIDA6#p>M6hvZ_U($%p(w-^4Dj5#51$=6IDVH
zaqZG^6Q&xWA(epH%PE(+;r0`$RT-L2$=-OxKS=<}!|UC{BgUI&PeSV6`WF7V9{PRK
zV7NiUxi)2USB*-(>L3N;i<zpGtoC3Pzh{WGtH;_ICfKq*^}Inw(g!yclrZj|tW{Wj
ztdQUnBfQ1+^S2nApURkXH)fxx*%XTFdB3yaKVMi_nWf*kTR(*DG^7y2Zks~ST2Y}y
zin>LnRhX=I%3|v#WPq7JC4U*UV2hiKNk5UpHs%)>(}FG$b&0PtBO~{flKe(4EtEvw
z&s6(+2<wRw9xhUCij0afsrPhraoJuUEqwdvjn(Kra<W|M#G>t@Luy}cUz=zu68NZ?
z^dn0VnaNKA2SnE<Um0m!0WKPDCOv;}mytB5PwhVP?6c7>qGz}y{3#9B(u15l$=9>x
zx~T(Ek-<%{rjhX?+>hC4%%OHFFA3ZF#zsyh=P0Tu;YNe#yq4yE^PbL8V(FZ;CmV%_
z1xF2sd@PKb0=v{$i1Lk?gWg9LPOY@r)K33MGd8SXfWJJhRTF^LuclL2jR4~o^ID}v
zJ}QCBJ~8BESISRb*RfjKnjR`eW!{O3K=fpuX4*&|oW++ci|P6mbL8>l4h<~hd}_(Y
z%AlD>+ovlB3}OPN$XaR>+BoCD65qKRoH5T?8u<D+5n)r(+!MyEOwu@~=^7!mdKqNE
zNpE8HW!mhX2ePRG8vB^|<V@#UspIr(K%uEB)9WhRv?+huOc15GM>gt%E;YKg!OW+B
z9?sc8LK``J#R+Nd9WG_4w3)-@UIghqT#DD|;b$czBm~-laEJbM$-@pt=Pn^gYTp6)
zQ`7b@QgeTV((6@wm9e=jY`p%GrlzJ-^O>hUmM!RYgI`U>NxSVFd6BLEouqG;9tVPU
zUb2V1?%!a`=J(a}%0Wr$H3B$md|VtT!R)hkf0#}i{w+TrVQutOrJ%4d{`XY_usswM
z^enq+>*~UM?(S;Lb%m~CUnPyre;&%Lk?hki@pnL+`SN_-!tI&P9?djwtLw)^Ibupa
zNTrK^X?d5o$qhShUDU!k85HBBUj3q#`<j>MKy-TXZOdq(E}f6hhd}v_7?0H0SSq(Z
zc%{SWC7Pz++vyu9mtR}(@Sf{1S@yQQ$oC}CT{L(0KMI^0XCDY~$#{BrfbB6VDvE@s
z94yAdf&$PD;`yL>j=Q5-ul_-Y66E0la+`<dh*81sODsjdvG3P|mAPuN_M^QVhNw_)
zC6M^MMP8Qpw&b8sc|7tfgtylh;fhi7n!aoL;`(yAic*iNVEqCgE-YC+{8+W#G{QTe
z73EBR>{*oNZP5QUSvxv>Q~6%#_uEc?ftaG*9N0&Jzj3PeBj@vB44M~nrXBqHV_;ww
zr`zpih28??R$(Hynmj$LSbPng)|n)T?Gl&ord}<PIy>gjDeF4gB43P`G-vo7g;;*7
zOhO_i>|$E)i3H0nA^BFTGwyCOw9Ual{%2VZWL+{=OKIfAI=J}lZVu3e)4v);Tej-$
z{7vuDtiugCZVLT!6YQ9r!5I}uWGD(dnXJ2;V|ZoWz9ak1q{7HZ>P|jR&MdwT-H+Lf
zVOBK~4D{;sh)#Q-oE#Aqb>-rPx@0qIY7;D#z+|>bp5IDBVEvN@29Ummgj-fmS#fQK
zDH}NDTDv-;IaB*YJN<anZ1*7J6OSJGbbW5bm3WVXnQcS9Z5Sci)s!GNaAS*T$)dq8
z^81;QpQ^M(Y58ZW*!ESe;tMHK*d-P|rKH@2o-VTD=!q{?<jgdL5o2ORi<VjAzszOJ
z7*z<z{ehN~CT;u>ydgPB|Fr=B>ai5ITjdQ2g}fFtobm@)$#v^cXUOYpxG7;Dqb~pr
z_o4)qIn*~Ty%M@n>7A#)p%W*=uf}(7L=F(MC0XY7tNa{uopP*xPX<!PWuN-|F`XQ3
zL)j)g9UtP{Zlnf-eVHn?+tx1%v;pmRhBbD9g^+V2zL%6|HZbZQ73d6jmdx2Pvh;4*
zJl9%_^U^YBc+Zby=(B#0itidvAB#G%3}AlB+yr9~EX3()f@EZG>uz-S>50Klsf}-)
zG!xk};6Dj5eDz-M%^ua2WS<vUq3&p?$ZR$XzAn$2e3{d|M~V;kDQ(j61?vJP2{uq|
zl#ya86;Hz)3?aYprafdgE(I4;PS7Kp8&|$67k>OMx_+?{CWciFE?CZNn*Xreq)K=@
z4TG;M%vSVKHs|{bhQJyx@h3ZQw9|NJcDc3>$9CGYvd(Yv9A5+w|JX;_h*lfZ`Lg$n
zb|2qzBk0a8Fj$AIn!2u_-@DAJC7phsZ(T_Ps`E&g$Z!PTP|*^R+pC~$5*HVjoonAC
zOKtl>=rzlOJR*?Qd6QDr(xW;e#V7C_djVf6|0w`n*zm*<%Z3}^AwK_LW5vLSWSRB6
zxbd=W2uyWF0n^z<w8}57S-*g61P~O)a1lSWPX)HioWeexh9+_tB78R*0hOkkx6g;0
zHNRo1+_-bq%Ue#uE;d$H-K<uBS(ur>oe;fL>ul;Hyb#W5Nyi}DteI6oPC(7e$@zZX
z-mFTh!hfc*LsEkLQ(sKFb(gw`rF+-~f|hrsy9W`8)iy3nf?1qQc<pce+3_O4KuYJl
zTjKvk9UDv(c!8UCUdQ`?-oX|B_s`M2;KGWb@J`o%qkVNM>kY#NRg06Kx4>|8fIlnB
zNQsRc6#{4FdjI^)p9+Mae|xlKlafoM_7{bqZ!FH6g<BGgFtgDmxPBf1Q(XFC&tYJ%
z&;|VPKr!b@J}c3z!>9?3cZoLh#Fg`*yK-%2b(nU;s+T`G!GyYD@{!H#|C*vHGr<Xl
zk{v>CLKFS-B+fpB?iyf|UyZ8#E=cs@!|{L*;!yV~9$?qOm+}X$ypdo}kp~r~w=5~h
z7@Kjn2gmO?6_=A_7rd+l{F*eSOeIz*BCvH15HoSS)Ed5CcR70BcoKNP4am>7@4Ssr
zKBB7xE5!IdtN*XLe1mXl+z=+Z(DEbaJu87(pwc?<A>-;^Q5gub590t$Xo=pKZ7LNk
zDoZ20jK4`OZHk`<jzK}$kiRJWN{-D#r2!9elKH{_k)krG#xki|Gl|Y?U_dPq>rh0|
z<!wVS6lRAqUT+KRJdZ{*i?|2-0g}i>`o72C?>pVh40AKB7cK!a7IZ%qXl$emjLf^t
z=J$$Gdf2BMyYLC>O+7fRd$KaRQa&l`WB)-Y@iqw<g_*CF%$e=Va5VoL)2|N8<310R
zCyv3mOv;*Te(jS<Ft$`=0}W8KQVs|*P&@;_#@@6$^~z2kw()4;1y2q}o4Q|I2B#(9
zwa2nDt3U`hFU<6F5C|%Oc1VJix((xGFvhRtt<)Cw238p6DNznOUJZUd8Q)y@J&)DD
zZ*#zv@4q`K3{7&HYMK9cB*nm)U`O~#WlV?v`NYxYYA2auup9D3QHu48PtlXHE9FZN
zg5kG7G^xdi?&!UV>SjXZ`$4B3uxkVSmwrOtISpYvAMNFYpA_kTGEv@*8@nK2Yl4dj
zB%N7}_mp@QogRb0vUrvg@7beJ1cG*;D}`XC!XZy`UyNSLWkmNRS2T=*S=^pmkKf_x
zUa}@Dnj$&_F~UGfbZ)yBhB6!I9SL6-rIcbx>adX+SYAt%!-<{^OuBu;TjJc3!xPf>
zxdlwqziVWA68oD@Nr23MFuwL%Ft2^8@1d2&{MB)h-QBem^|M*%`NF!D8_egRQ0!&A
z<+pGnisXC|Q@OcE7=lSyWhzc2@!NZ5g9`Gasej5r&=&O>H0PyNafm!f>Z?#4B5UyN
z(mk=>q!6MfV?iM8b`B8EV=E{DQS050=aus-jbNgVttM3?^7RE~j6AG+W&M{C<AgiB
zLTC!Fi7CX>|I`9pniD_WdM+qFpJXcG$)WqgbD8cX?M%9p;P%kJBNa}G-B{<^yisgp
zsoebuswdhzR-7O+Ci`RV(stz!M!EQ(<Mfd}D)yy=+`;t3SEf#}87(dvw{AHjNiH2!
zrk~iIy#MScitG{iJ9+_cK)tH)HYR>6tt^9z#<owN_k%j+ML|^PXLlGrxD`14nM+`^
zSAct<dnO~E(X~AM?W6E&l$lp@=M+~k{naan8M{oITgNXZX+tXTop5b!2k8EljCmi-
zNT#16F`G&FbAvw>h1+i%UYbl}fZKmiROY|dGKlC}!x=RAYwW;DV6wZ{IR%={bORrQ
zLswS03)}lO1l$Onn`wT{_#TcoC-b;Wrz3&(-1j^6r=Bi*j|_3sJ~HF={2Ky$kuLd?
zJ=yO!2Zz%i5{@1h^J?J2o3Y_F>~8rIez>vC+l|r#8M|bcgXCNPT6Cs%Po5vI?4C5(
z)<`*so1m`VwDJeQMBmDD6{l}8Ne?p&sq<S|yz0sQHTA%OhW+!uppjkOh6zp4S#J<o
zaP;2!Ui{V0DR;N<W;4I_Hlo$DKsQmm#q_3uC4$wZKrA(b2puf3(BSRSCcr#onCSNt
zSCbFV;=`p;{<ks;V7*MnSi0?Ax5AAPu}ET@dMCj+H|LHC_9qEm@&XBd644b_V2p`)
zNa-5<x>r_edp&EG=&+2SCJG=5eF1413%@Cs?V<oj)(sTLjM+r<{#zpjD!H`76Vp^%
zKm68NqY$U4pr8Ok<Azy;$DNrTK$fKfV3fpXSOu9ZD=4^!3`|qEIf@=U09|~lDk@-y
zR#8z|3cB+?*3YGW2K<&y!&xUH1JO`}5m;Ow{-Nk-icC%RZ2r{J3csv4`GR&<%2If>
zT<F0Gmqzz{wmj|`Gw*MZ@P^a!15z4(8$vUxYu8+e3EF;v{24?vb70l27SvY4r>4|$
zQ$=oXY;277ffmGP695teU@ZJ76hfoc75MPsL+~vSXn@i%)(XIJt)go|>llo)TDKY`
zm2^P#n#Hz+=yUniWSOa{!UF6SxLdHMNj|(Gjs9Vp-1n2KI<#~j;{)9Y3&=kpxxpxM
zc)et1+!OS18cfxm8J7E`;5|iKkA3AsCi0Z3@Q)usry{^_%WAv6i9oBrx_}WViGwV?
z+9zN6gJD2M;WJv-@x`#&+3eK-6#|(EAj0t4&oPQ((k+k^xN0}@aknuA*yKOBUWcU?
zp*l-#^iV?5kl+Zk^gRw+w~8}Biy_hi@tS}dl))}ol<Mt?#f;vx@2H%FtS7q0W*N;6
znwQ=e6;<?Go2X8cDI=GBk;no}DWKBXtxO@ltW<E-#c_;+B|4yBt+~h;yLniZB+fj%
zbK`Mc>vMgN>IVNWXBP!1fjPc8I`VG<g}V<PDvb;dX7w%XzH1QXClCE?Ui$Lo%dTTc
z{S3dui(ivS&+zs#zeA@7b4pWg$45{P?yvH5*RQR4-g!&O4t7OU+nhCSDZgS3WWPme
z51?q$xf9g#K%nzR*{&b@y4B~5h`P4p*jW~nn*{H`-XH`N>|=Kfh+mkRmZ+5~D=Y8H
z`T5nin*$9AbAJ#0##<#LbY-k;!JUiw-@nJ~k3GE11&B=W%FfRI`t@s5lg!sm%PU5x
zx*v1&9JOXXzE(ax9lwCdcVC8;cp0;Ry|oha1A8w7ztT!{h_euFVk@jH(r5Y?y5o9Z
z>U*DcDCG%Mm{Q6mDCrAJvkhGf*-e|yX_S_4y>;XE`TFPNk^v>N;qBCM=A$IxTD(KV
zbJ5+FKu7;P(Ct8a`blNjN`TT-Z>fy)NTMJrid<sVt>fr5O_d#LT)3(JHitVG9j~ru
z!Z+QdjRe=|`<cvkGOGPigJ8>Tvlv}Dgpqz*L8Qm&+4sx9r)~$^?)U<I9B#S`lr$1M
zgP^N#W-A6x(JB6bfwb?>>#mlnsF<_w!2Uo(o<TnK1K8(hpg<SgEQCkD3te-Ifcjdf
ztk2_OweNfvl_&LV;U^F9`|aFS%2#alro^$<g3nBg!d^n|_mI*N-hYLMB$N~>Z$SNO
zzbKp?0+tR>0hdr0h|WfH;lc|_enuegc(9seF|i&k68mqn76-HV(52$AwK22U)d+sJ
z@NGsQu&yIZbZE>}%UHJQHm!W?7fN<EdDv^#^rTp6m9BoK=>0%)d>7yw@$=rw0@gCC
z+N$Z?oE(e8t(o=2JMqfQmy_u{6l<3MRMs`BYkL8Clg&dhp5pY3FR^JjrO;x3TvDJr
zKv$1v>><dcc-xnui0#e00|RgYu(N!uy5LYi_$T`g<0F6I4tkr`|5k|pg+oO3E5M@(
z)E0s7ARFHG4R1FWP62#Xx8;y$Ir(Jt^eCX-z+)z!T5M91hvtAj>4E~4Kmx&(XCe}Z
zuD>XmWctBoSD^EOOf8`bFx+=smajkiWhQt8PCPi9k&e0i;1II#2EXHeDvkl2rX!Ws
z6{LAEmQlB>IlKp;Q<y&)FbM@sO(w>KL#t^0qA^f0jC|_Z6}ai?2YPnXVBGHR6^Mx4
z6hJ<NgoJF*^izR2KIGhpE752G_S7dRQnklyq+iK;o}QrtahY^QcU}>O84pD=iiuw(
zsDQ|#uFKUlr~Mq%((j+&@wNsy5x3yd-rApQ{GMV-4JNm2%+t&Qv!0h$yka7K9rjUV
z-YbDWpjL;tbYazxu0vmJx3w0K*u~y{sG*bbJDvlK<V~ExdKUMa_AeV69m2I5s>;@!
zQhqw1cwR`pQW{=C;d)AM%Y}i}qT5rooqS1e33LStBALiLA)K4Fi-2N|X}7}L{;CV#
zw_C%GkpN?ev*Ik<M4Ouzw(EuNK-k*aA3ZX8J{55eR_@&GNMI88SQy%(tJeu^1W4D9
zb()8#7R2_zquh;}uh2{ltnfZrsLO%)o*f!ix5xdgJTaAzhY$c98}=Q#7mYGCy7jFk
zukrS99gZ`_RW}R27g<7giqW7m3it%oYDcrn`v@)j{=mF?Rbtn#)m+8<wnJngXW(eF
zC#bcfck8KnKBJ)55<-9;Wk5Y`fBdksodZBX4tBTA!G;}AEmSvbew;%8<9kZGSf|gy
ztD*v?Eo_F@T1y9vdbJb4RJ3LMG;9cv21$AI$ib%qviM_Boo=Hg&c?CgteF_Vm1{NE
z@d0M>mowbb<e?gc?@2C6kAlv98Icv%Cb+*R0Nns=P8zozE!ns;D7}_+L^1ST=ZlT2
z_@+lpg}EbFrI?YOu)4&8L?TEok$ZZz>z(Y3(Lr3TmdD}!-K2>MUTCpPATD|Hsaxvd
zOkG`Fq{Z7_1cIG95*{tEJ~w%8Sc%M5RL;!a`!VYAGFD2HYhI$yHqcP)(()=z!pQ26
zp6t=GBn#6iJan;89&qK{q;<~31G%Rn^i``%yJDB#m@V8Uc(y<AT}T^hw`beV0oEl+
z{n1RRBcYP_x=P<9=L%4X*SO{K<t)B!%ZOk5wg5E8_CX~(0S8{jrT`=;5WLgg%O@o#
z_H#ycjzPS<`sUA=(G*1W{6vASa?q?4aL<l+m`hL8ci_$}C>ps<2T5J~etxy->9)FP
zb(sC8k;0@_LzMhAe6;qV)MM(3m6HD7KObxvBQT+aq3UZ8%GkQShq=WgX=kenh0w9U
z$HY5JA1D^>8M7AwFohVi2Yb&qQAddo+yFal3_)(AsiFBZOTMG}jgO<kTVKC&)G7va
zMJwODapMN4-*F?%8%_ZY3T5)3+Fw&M4rtD4YGymtejk}GSLS~va-B@Zv!S+cPPUzy
z#kTpUSHttK#~EthJhljdi+$6k!mpRHKeYG(KJ};P$7<7&3$Gz4WjQ`R1`R!#1G+C>
zfWBsSr?J7o7cXACD%AsIwc4Hd{v^^VY!QX6`$6l@<2CQ@hmcjJyya*~H0ix?9W?(r
z(b1;mnP7Ex>w<*e2ciA5_0CgOi}fAwgdO1308j#o4+E<Xwzgq4DX?W~Ie-@)GDRI9
z`vQRFSx-0bkl-W;jy|iEk*Dteoj;T%ht_#6r*qmqv8(^|;#CDa^f9~nO*_fv?C(zg
z2%gn6n)h8%8BE{7!;iITd*PH|$}*L<b11Uz-2B(w6XTuJ2sTlu=h2LfhL3qZi^Rmt
z{>09TGmG;U!@%<BA%q>cudol%1TlZp(#!I<0ev5x@xiaVHv+XKC{t_kK}fr;HVTY8
zUK<W{u@<hJ?`}FTUxLvCk6u61^vN{GBWOOcjE@K|&@gte^IVWS*gLO&BYmsbNa5~4
zH=n5W%3YP>xX*dqj#zA~+0Q<dGy0GffsAETb%^Mo+u~8~+Em7`&XzU+RaTganIF20
z;MTJ?jD^~l0g$I}uD7r|H7<1k?K$@LOUq!!fL4K{YILU-xIf@#xrpQSCMFKdb@VHI
zK}0g6lsEb-I^)r!D%L?R!~OOeHx4||zZE-l4>o)c2JF$uQAQGQ3FdJ1Nxd46sgMzY
z#BUH@fHx+Po}knGT=n4h{GZ7oOJN_pmX^Mj7J%B)EEE!w04mQn#eryyd2{RG#Mu-&
zuo-)>EKv*owK?Fg;*%9VO!(jou)ptd`NcTSl>B(|F$1>+3neAcIkvgB_UY#k@CBgf
zUHj!rJX{?gM_=szZmeHXS7y6!COENza^5*0VS}Gy-<k<(ya18bOpGSv{!kVLo^)AB
z1MK9t;VQ*C-ddnCGCDd+B?eGGSQWcEDKN&HfSH`PlPnE?d{J)%dLVI$K(I6ASuTJ=
zKKdra`FlWsL#I&t01@7`iUz7;-zD|a)PP;x5dv*i?&(b5K|9o)`0iyl@SE*}Tb}jC
z9k!L`Pmn2&!YQEewcUKO(XaaVJdhIs?QP9(82|Bm#l^+yc&VuXnM@#qW^aEr6EAg8
zWeyl!6%%kqBoYaP5wy==09}aE&$pbnDAW4&lSj9UIZZzo01qtN?*#X8djS&OJ~T4Y
z7JKPU%45)5{T~^b6;Z$|nMIYIHMn14S*{MsVY0{XS1rGEV_r=pr4}^zt(DXVQ{LUk
zt2_Q%u$Q2XxXa{z{*8-=k><l<AG!3<r$ZXJ$Hb-Ps7$vS0yA=eP{85ALBKn>bN`|A
z{x`hf`Op9RApdB^^Y=Hkw*awbmT|RVqjtCPIAQ;C$hFIW04<a<VlQDu6NO+)F>oT*
zXnz-e3_to~b*7>DOL3!f2JP81pBfHK@g_h7QU2ae+gvElIXhZ8qZ8;Vzi5M)C?%!b
zS0vVdjAw4)(FFo)7T`QrN)xVK=9!(X$5&n$;nH=9g?8s9s1F(h0kYW01T2T3Z&f(q
zWwQ+M44xLbuE}JU{KQEE_E!S|<Fd&5y{E`HGAqQH1IinJW&jQ%=VRZ79ERX_&IV1r
z1iGJ>9pY{fU2Ty{^jPe;7IpCrhJU*_50wh_6Y(q=FeQ6Ek=$g&6Rfv`ZbTnz*Y7s^
zVi4RuqaQb#8xy`x2i|s3YYGE2K980Bg;mivJ|98wLHc&rMfsS9q8o<i_fw+K%i1SD
z;H%Zi!IZ+UKx*4$=fIugTwLi>K#GQJ9f!Q~)S1t(NvXK9td>3hWj-9Q75C%`?qjqv
zK2i`A!=^DLdUVQnpk#ZpKA8?R)%6|i-CmE@7tq}iA~|0~F}-=LiRe{9lt~RpE65Qq
zvlYToO!!|DJo|vaI1{tUsVEVcVY2~4*{d4zUR$el4_k>pfWoCWmxE{n@pk#{1<*46
ze=x{+a^^n<i$B)?%^l-U2^7Gu{Wu~f5Gerr5<S_vv%3)G8VQ-kS*c1NjldT@NbfT-
zdT(QbTcvtCR0QVK)c$}5YW@5DU@ZJ?OP}lyIbyazI{Gc<W@{Nu!sEz+g&HAwn$B^0
zkxJPP>a}yJs&KjRY5|V>6qkY=1Dh$u;+|-3K9=Y+;}*5HFX<Mf{}|b^&F7vtBBd}m
zXXraId*_uPJp_H&8F;CpgW?yTRBY7<$(opxG_gH*^fGvw5#HHwzuwfp_J^y{BqhVp
z18v@Y^ab+t`55_F-1W*%ms|8wvMg(r7aF+)r)kc#R1C^qIa;q=W81y1!1C-tm<T^P
zekqkJCyB$P_!f;d69)C!+HU3u&BU<ZPoy9#>G$W>l|@%z&Int-_)~?lkLIs(t7K$b
z(rYd$F{d=#Wk1g~Tne>g7*Y3a9E_`A3f56lwBk*jCt!AjoAVAc+#a)grchCkNI9q&
zW#j!szbSXU=193j_Xz`eGDcEExlb#>s1N6phhOB6Mx+9SD8*5zW_n^HGP3IZlhEQh
z;ocO<jPE11i81Wf+D{+2KQ(^xlsHQCs-zt!NnhphWybg~fsB+wZ4U8U_Zzu)mt%7N
z&J#h^6s>(fzAB$gb9#DL^rrGG-fn2~0Wf1)kVfpAkfqw{nfE6NIj!d;!q<NA8Pn5i
z*a@j`l9IHpf3Lk4(9pp4#9ZFyqCzTreo=)^_u#U(e1oU?cZ0&5rq5bZ!fbPHWLK01
z=8hjfl@1@H7Xk6QQ`6OR)Apewd@BmY`9PuGUroS6WA5hLXGB%+>8O3IR(HQXAFHZL
z{xLLI*3}{9l0wVQit&D>4$Q**R4~VSCni58{^|VgAV9^tu+SJG6t#PmROiP?MH5rJ
z<3bB-S9_npPtP2&fzj5KFC#}>ukrVAFl&}G7MaQOK_(!gp-*hwM$=T5?3r|@A~%|I
zh2Q1PJ!BK^h|lW-lS)tiVMgL(+1Mw2$o-_x<sbtFykGANzS$;#W0ajuQ|C%nK7n&G
zJ?Pa+Akm5yeFpn`qm=itB7Y^X-j-*-j+L3)YMki%4gVi;R#Ll!+W8WaNyCpatodeT
zQffzOAYj&AbIGO^`#JIBEw>h=mqH*Nb`C1{d<H1?3Q1}vJ~60j#?B$YBR^o-@3!6E
z#hh}i<wWc(6(MKoO>T0)-hOm0dh(IeJ#+7%n<>*;5F%=m{JC7U&-x-hUH2yC&jM0%
zXli%?sTHsGLBg+m@&_)x{<O!W--C=HGl{ddjQ$brB6Y4%?KYoCH7_|yCehsip)(q7
zorm4HCH;GZUOwth`4XwQocMc1Z3hN8H054Vt;D=;Xn~)T>eVGh(zf;S9Wcj^)3G@b
zpkGy@>1jrX2Nx;~eQQ|>B<ayNEpksI@@Q(J_YS6|kRrmW<$d+Lr1Z0R!M2z;kN|Kz
z+mJng{X-}&6~r@F22Hc~mqE;|&Aqy8t3TJO_Dxa^3;`PfF)<)gclm2SYG%Z%!tp<+
z%mkmHJiPKJef`vnd&j7P_6KKKke7s03YL?1{Trhs)UP;UFx%zIF^W0Qrl<xm06Y@a
zLkn|}#+b{JXkFI=d}c7r9$eYP%<C^_G%2Dz?rvTybiu1+5)I6n6$TG8iX<XFHLv2j
zciuse2|-AS+If*%R`9VVeSnkQv4p|kxj}FKc)NmVB-5lu&_nuDAvcJWP(1>~`6K|I
zeoKm43#Kq*ySxR?Ev%@-Lj|aL0JePbq2dd3QD~kw^9x43iX^~L+A`ZVKi`3Q9ag5p
zRJH9<JQAmWe<vywjt0-Hpr8i{s~T7gou7t}y6=hS3SEOy)$(tFKs(%+Blr%T_OAns
zKd3eWd@10&U`c#H85Yo8>i|g=)q9~#sv!EH*a84dZSBN^F+g&G7kJ_<#zH~ieZ!q4
zB|bpLG?v%{$^`siJwx8}TRA$$l<R=V82qv;h~FD=<@Z6_G#)AjZqMV;?~wp*tBBDD
zjs?#1bVf<}_6B@8dso+Gl`nDf{zN4Z^Y^#k$MGu6*__nuDZuDqs$+zD+i?KFRa`i3
zn9D5W7SL0byC-jn2s@pXGC*`?gTZB8Q3K(F$3vhj3Y2IqfAQbF`=T0POi+LpQ~mi_
zZkx<C@CZQ9dho{|IvWuIb)ZR>KN+^T(>oE=&)nY8Yi?m}HE3V?qzyBO7>CQuX2jR$
zd8hyA=+{aBshNA>TQYyJ-Az^QAhTf!n1Aru(QQKBd!V1QCMAnia&}sTX+o>vU^|Gy
zfzSiw0$P8gwxpLC$gTkD_G~-c)b|?%#;!2|(Bd6P=<mUi5zF$)U`in5V3JQxFg`j8
zZU7)NQyKtpUGPn~bFG#p@f<?p%Tw~jW~DXG9#=N6EF|OsBPpbiJ=&ggo~e@NIt1Z`
ztmNrdd7jkzch9=kpLyOh0<8WuZE1}q$Y~HY1;x^SB>MRrz}iQEl7jp7ly}(rgAmiP
zyYv50M)!lV4w~YCWi#>Q=j6znd@m_UMz~LwSplWC%1Th}4<uvRg8Qrsi;I)0dmI+{
zN@FIa+2gyxB>*fOaRPMAUNpmhNUb&8@r&NPYf@c;T=c4|kJ>xH$Dsk`OLu#)MN{Mz
zrY?aV$EBtZP`J}7TZcLPW~BEVe5)(C5xPMLhhj(9V`T*l&e9iNa|GGi0Sfcevpea^
zOZFeas(2+|EcK86kri1AQtvDgfiB<eAw5jo>=E!jNU%ZO)c{le{zQg{p@7%G7;;}`
zUw~ESpm3$Sr`o;dP5T=+DznnUlQj?TUw|}RJ;TCJRP47VpZp@vB0tp8)^0Cr{L%EB
zl<(*B#Lwk($M1@y&L<WmCqk!RHo6XD-eO}mF7UntcRL&oc<|Xrf1RsB0^jqkifaNE
z1I*AA(rxG!SV17R#YS|5J)j2#kUB<B7^!tOguoMg+s#u_OdWp_X_G3*-K%2z#z<C@
z;OMjIjRqD47QMrqK5%W|F*m0bXBXpdy={LJ3o&~ET>7%2!Tjk{=Yg9!vB5<-fU!Fm
zi;?i7uA~7=4Nw4ZeonS~9o=zTDOu<*;4a*~tL&I2*EROSeLxW-bG0_mCb0Fn;8QJG
zeBSnLJw5T#y?m+?8-0cV5PdkNaV_p<7rX#fq#TwTKo0OdrS#kRm<<uKD{Pk>74aLv
zWvBAV&7FhyxcKJDw1BVGq1=M6`oVK}fuvqUi44v7jHBojVEjNy7*$agD3vZ#?e6{K
z^L+A`hKco9PE*i~F(BlAswd-N*gK7%07<o6tQLVLQZ76?;bI%tr7dE)dWBUID<k12
zx4*yd_DmNU&#X>s27<xfLy8+^?=$}-INdf%ZN^nrMT68J+bdr|djBHiB6WF6O56G2
zlJmF$^#GEC@r2F=n(hY~3zgvhc1ATrg(&|%dJmR4=zcFU(3uK(5}pB;8`rx5fnZ1g
z2-5wRCqPpXkJVx5^d6v2p9|`_58R2O2HV5W?a}FSJ2k}>z(K^MHRgF6?@OolpPWUr
zxDR{qJ^|sInp$LC|3XA1mY54B`YZE|L4WLa9l3r8Kb`LCvp;q8#8RFOlRo8KE26b+
zb@%D<Gz`YpSp!+LA$>uw#de;E*AsX}?0~c1=Fy4U<!g?*n}Fc@X6-(sWV7`AFk|60
zpt&%)hQzO`r+`NQ4P3*B3LriVPs(Yq_mcG~e7IIv{>l6aTTVN?9(Di)=HTXG)YuIk
zf9waMSrERz4%$}!2v3JiOU(y`7=frg<>Y&ZL+v!E+6L|`rau7S5kQ6bUcBIIelc7n
zuK%7D0c|v(CAM;g!Or-NfefyY^$py?6S&M4$t6Ha0{;Mvlon%aq@<(-c8acBr9fZ+
zeG$0aQ5mm$@!cb{v;0$bT%4SC)g>S~GN^-ntOF%ub51<GHdzi|ZCgbrCL{z45FmEE
zm#$EG&5n#Lc|7Fu{}aOn5l+EwhsyHqiuHrl1y0K1oJna8j2BD<8{vf5oKANL-4D*G
z-zy-RZvlCTTfvlI{(u7jdl~0%vkAW*^qNq0RXeKq^xba?nNzpPq`sBS%u3TEKnjD~
z#tPUkTUaF!aqL_yok9xVH!Q-d)>NWHM(syuLFWw<dFbb#G%%;?LJ~vUn;BgfseyZ|
zo*E5K=XMh`G=647^>Ank4a_9-6fJO&v0H&T!8`WlAc|%e(+{725(TvaZy|dG^O{3!
z-5cn3JT4Dao@g=u{Z#agA=#B6BDgFD@t#zEJgD|#88E1>?+5LV{@OYZygkqW_W&Ob
zl#Zk|N^^<%D=?b9ZF~USA`=oRfj}}|z4D$CuqwdH<%Y_6nI+)EBHq$^{{VpVInzj$
z?O8z`D++~%kNf>RsRk)swkMxke@><u19I{~Y6|@QWwfIBKaA*oQnBg-;MvjXB}c^5
z4-zMm?G7Kt@i7j&<Uc6GATUNBS--2BxVmItsNQ}=oJq#+<}G<1KRJn}0$kPJ@oYuK
zoW%}(SdGEM<Oh1Nc+)^*8Yjj+5cXXmRDSaKWk%}q_Cf4yMwXdipV>r4yrF)@=YbSF
z9=ziTq7Sf{0>oAM#8MYq5O#YEp!uYo*yfC#k4Qk4iCWV#Mp`=c_PzWUsK_Grt1u^_
z9du^*glwEeQ$esNmRkN>!@#kMRt^x)icIIA@Xe=%uGTI72ac=vRL%jYr1{lY)5=hr
zWFv)cZA~CQTdQK9k-eSx(0a&B@OzZDDcL2<*P$Hawl8HV6;CUCDyl(+g_noFf#Bp#
zFe8uN4SRwa{d$<)xsf>D^{P2+F0b&yx4t=Opw!%+?Zf!*rIP22RL^$b<BpL^?~^O<
zTG3c6s55x_HHpiybnsK&S4GuP_Mv1E2_V8=b~~`@65g#|I)Yg1WgebBY8I{UTA)m?
znO;Bw=e&0*M8mf+sP#o9R@=bhelDXXkkAB=xOqd5R=mIaL=z_<JD6{p5w_!V(dQ$3
zv(zB&7MHAiPZ{L(=L=|Gd<8?V^<NnjN*00CQmR?J*X4vhq_`5~wpZGRqR^NXZY5>2
zJTkH9?G>aSx^~5FfL)>$ggJ7=K2BYNGc@#DwUwszy~7Mwf{u%x@^WRIzWdgP(p9Q=
zziw3|6mN#NG&3iFL1)PL&jMe^7#5M~Tl6L%Y<`{D65W0MtNL@ZRh{PT8p?N*N$B_f
ziQtJ<!QzPht8BmGD{khFY8aJ5XR?y4Ov%U`v?}pWOPhuCgTaj8sITCyV&xJNBI3A0
zEBAvz;A5n;r-OUL%`QtUtz7U3NpPME+Y?X|g0B8?k+nIFSH^hB>pEOlt;vP3)vSW7
zD1!wNk`P0o?y8LpePWM9fP~_8hdYwOM(1Om9j3JeXFUw*3rdiv8A+2y*i$mBEH#Mv
zWI6huN?Kkts(^3_z~oH&)WnFoIx~?ApK~YOHnOxjGvwXM13ku?{>Y)1gN{>d+?2Fe
z?9x-B9E63@nm6<t+D;Ad5m-2nCs+q`p@i^3po7}oz$3uH!J)wPYP7>OOY%O8n)27|
z%MVo46Hv}1J6hMN-`&2733!n5z)fGBw^m~5aI{H}1k>^8gfoQqE=l*WU3bXBGmBa8
z)Tiv{BGl8=lvXiu@?Ym~{}CR|_fbz;Qg_I{%Fv@-7@Bvxy#G@i_qX1%Y5i4^mWuv|
ziJ=CDQ3C7%q_H{NK8vOSU3p5kZ{o-J$?{+*7&Qn?)A?)9r4;5--)JA98)a76nn^Xi
z%Tz}D4#9f45<MEv#X2?+G)Io-MRkufo`#D_9Helx)nc((8?hVGBX9@D`O1gnD()@$
z4ObV&%oF1mjhSEfIFDIc=H_vb#&RhAu?ktyKBq$^)R;MR1k(0=-`B))#Cg1{Q8Cn_
z5@yt#`eu3Il858-q!CQcy$AQh=~u*i2-0KCYqBIfw9;1o)lDX2G$knu$1ZV^uz4tT
z?%U;EStk|<iPEhY+NR9t*2(*xC2>lE);?#QZe945>ixuFBg@f^D>!CxEeqp)pMJ>5
zih3ySnuEygOVcs3$M&#o%6WI96t(li6A+k<@y=7?i77BS{~<j8i>t&F<^LhD{*hV0
zM#1Zx!20(N&@1@hzkdF|;Y<I=LH^fc`0tVYJIMJZ{P%_YyM+I`kbh_YA7l6*7xHfg
i{{QFU-x$n3pF#ah5S(<AQ#1HEQ&v!aQY>fo?*9V1WPYmv

literal 34097
zcmbTeby!s0+crFkN=Zmcr-U?0rywOIA{~Q-ba$(ibSWKz2-3~apwtl3ok~jg49&NO
z`+I-S{d~`Re1AOq;2!24XRp2XT34Lcd0v~)mnw2NSmam`2n0t#{`o5i1g#GOxgCjd
z8+_s=<oX=^!gP?=afU$fj8OmG^74?0gg|H^3eTUt_L$z9bJW(GNME~VXqBQ3tTnRf
zic-c;W-Kv?&zBcP)R->4m~+Wh5ts^F^{h~FXKD#kNX;2hIr!2id@i??{DnEA|EcTD
zv#GwP$1WK}w($yo?J8wlJFDVHacu&u1@iT<P0&T|na~B^f#3G?r#wH?#h{aX$2eaq
z;gM5V=v^Accr-MG%=ns`l0pDx3K?dOW<veOm_ohMQGgd(Q?&oQ2HJqP?_zNN^NL?_
z3%r-&2QUA;O2Y4fxBoi*--Fzo4*Az0|D67>LH;@Ye+=@!pToaq^3Ndu>pA@UMv{wZ
zBWK7dDSghjVV3IbiD6-3R3dJJxL~#Nq-tE2A_YiDNk=NQjm-x#zCD0IB>RK6A}Itk
zU%ws%9|i;jJO$<f#~-~}+W*CR{Ch6{I{lxW`PXv3^JLXtyqwy-vYUoYU9B?IRecTg
z20JE6lEwUAd;9-40z?Xd_1mq%RoKAOv#!2<Y5%W_aJmX}>4u9iY^Jz`RjX!Z+ph)M
z5VYP~jf|X0PZ(HZ<Kl9!(|5X?*sGzg&cJo<BMwdwV2Lg$DN$)~{5)s3yRq@20z1P3
z%md!}3j5A&D?t@%1wCEel$4YUKYs%Q16uqf*Ha}5HJxJ3!m)z6hK1cky%=l`l5qVJ
zmavqZM<H!P!^6X~vy>Qjw%ojD_buFis7K8HKK7y@Ctrqj)z=GeH}LH=m>C$P_pS^#
zvyR&5U`~|g=Q}pcjZCPXd>*rdF)wJ*;>SRfRwj09^=?7(S-xO8y~}11&m35H!J@0L
zuP2g8UMEy5S#l6umUD7;c0O29;LonmX1DT6AO-(y!=Jjat99t8rIk3c?$R&^PJCP0
z9i&;P(EyF^8W|n^<UlIf&uDXO1->z7XK!z>weUa}+#5K=GfSJA4Aeg92<#xw4LB1g
zHn+F7(l*~KFC-$mzDd%81#hUYpPrfVB)CQEdStqGJM|Zn!~?TLx1R%Bwc0k7Q@Kdb
zfhtiUusCn6ZC7_THsa<cH`FYbJZAko31SaT_djQ|6c-nBaBw(fbuEQ@3sl2kh20H>
zg}s}{O#aC3V=CMbeg9mX7mh91j~YuSHb3vG=Z)l3hYqYMo~+)5^aW;Y=^It+uH=0!
z>XCm`S>UeV=y?F=4BR`v<kb+N#65kpg*;ygfhlcQl$oBbRc$fHLy0{e$F?npx8&z5
zG;>|}wM=YAo)0Og5uM7GX|o?3-8+^m2pl~ITN<(CXwTysl-!)z>ub4NsOjHre_OM#
zy0KB;FFwx=8}dp{)h#N}MhBBa;;#Csqi1eh9OE~p$JL+I{T}uX7@USz@+%as&~KpZ
z!&acKcO{NE(+(z5f)KVNmBfv)S2}()H#_UIF`A!j#XVZu!%8AuxO?W=X!dqGarRR-
zi@Lh{_VzZB<E_@T!T5CSs+Kj*XIk-Rim(eCO(GdJqLziy@xYTyKeF7hO6nE0DI5O8
z-W71WP9be?DyL@cU>(Mx=NC~kcVj4a)xqNJdn~g_oc1VsX1|$j9yPJHvB|-ufUHJl
zY*EskM_1<&_n(!gl(l*1uITm`R(U6`&{r_)fpyueoQ^N1`l-6zm%uD8Oo=PRq;OyW
znNkreg=J;E8P8^S4Z544cK7zJ1&t?nk+r_ycB=>3>?UbIX!h<XpRzfK=k3U_IWB4^
zN*o~HH*?LOZD@Q9V^EY!Nljhx@EIBK{iGm#I%Obf=R=qvF?8ql`_9hJKq0*-&eD<f
z<;laaY<Q!Hz0-X{%QLbEPqq~~7U<2<Aatn%_Jw`5tWk*1WdmT+mB8xXOST%9!<!1x
z8A=d(g<>OM=qhdY4Iw>AJ7?i^E7nmZA-(flVCK-2Fjg5gVG)rrU)Hc}Tp4;3VIi;<
zVJTabSy%n3K77$#gcx^3siU)5HlimK60GI!sJFrR$>8)h)o6M+=!;fDuCsMXR+%uw
zcS_ul8cmTuoE~AZPlRTys(U7<Cns6DevRI<7JA_u22D~w9f-*(sRz!K5UM&b$m`Z?
ze;fE04i3&J>OQ<5#W5g%O=Hf1v*b=KEG>up0tc#QH;)T^`+#?B^lV^(UnJo&!A9Hc
zPo&giiX*Q!ey-EafNz}oo!wm5l(bq@=Eu^$#X?QD+^k*%*IHQi_cjyj79&OAU0_)@
zk2$A2#~?9JT!2}vPmllp{d>l~rOBi#3S@=k@$!YsxhjcZgaA$VH>yXo*l&*!GxpeY
zZd;Sj)M?T3zZw^g<QJ^aA7qxP<Pj=}WGrbWHD496>7SRH!Fq)`>oyM#4nRZzQHa#C
zbb`%a^HtYj0)v}?fPkv%AIQ7pm+?4|@czx?dVNk1kfTcC=NHUJy`nvBG=nXxqYBH*
zhgO{OAL(60-9aT@_{)t}stVqorG0p)3aOySM`5W<-8aNqLs?Wm?Q`>o#v0($+^!em
zr=P!MWj&kkNh^YIGlMkMnq8^QuB@zV2h+=V{+$#0+i1_}FP^P_)Qr(A7G|Lw*r~q1
zPFTp?T&Af)ju3?UuKY88NFS!y6r_v@7$Z2$UVjdF<?iN|Tj)J$T>ex*V89-N?>Ss~
zf69ryzrSB4q%A87F0{*m*7`zLl^?<$o12@P+)tHuOnFpGyjtHpe$+lYHAO}`H8vCo
zfgDPM??50iZx!d_n@pERR_3En|I6{W`~QPK{s(lR)#4!o0H!sJFf|3spt;CU<H&0|
z5QS;p2|Y|t!g&5QOPif2Jctt-^^2Reh)$w~MPlLgOQXjuFOSZ$FMLv>Zoa!2Xk706
z_3P!*k_jPUevt$}4-Y@jgO9Q+8f4W1#H8^xm?#LvW?*Ag;BT|_S}@JHMkQ`Yx_mNO
zzWw#fviw>PU$qCeiYU0G9lBpHKq*cb+V>itcel#d*yH52puWjdg6e<9kiR}Tv6s%e
zi!fvc*P#1~7$1J6@`(`z9+qn-M@RRK9E^N?FcEzGJVRs{Hjg<=SXo&I*DZViE;G$;
z^ZoN^eo6UTVS$p%vs-lx_>s*bPe09{#ptAmK0oW~vgcxZLX6Lp_}Da4%-hYuAts@N
zApALfM@L5$+Hk77xAf5bpWcxPALOF8gHI=osYrj2bFMOfvgX`VXZq6Or=QjdBZOYI
z{CEh<DdKK%tA0Q6u<YR*BrI#NV#~|Mcrh^`K<|ZO{;*+o%;gVsn!zG%l0X}{w$GUr
zJ8qP!3zRaImCPMVi7m@)QLZE2BAqx8$T@Jj*)CFP?>V!vdD}~4-PO{NVq`akWa^eY
zb8yH&WEAH)PlSrn=%-J6F6~3g9NW>@5N?S%(yri6=90jreN0;m>M#|`kwDAk$CX#Q
z?zPjQhOX6T-`uLRiej>Sp8YhW-V6_``IK;w1Hh6g{S;=gn?&zo<Vco9T*hmo`m<wM
z$7<_YpK7I`9fz1Q(wW@FMLlJv400*{cPj%URK9<*{ys;xK2#uAOlbKPbP6lwhhN#p
z2~>>c$3E~EBpkT>BAyz9W0a2*sp@hRgqMl!ju){q?D-z<ma0@}rA0bd{X@H9T5?rZ
zQ&2$I%h*HE1g}&@?V%LvB-V;!<O0pv)%h)}+}P8;vkPyI+z>l;`5CzOS3<%TO3DZ`
z2M@Nvb#XPF?vp9t)8XIG7n8QfsAcyg$Md1~M{`=rBD83iP7d5e;bte#G)YM24K+LC
zKg$t9v>8Lre=^#bGCWR+N$!76pPX41gpangbg9UTWTc~L#X9z|4Z=5$w<K16&tB1x
z@&+CgD$C5I5*cCIJCuQO&xEa6^FuQCE%uTH2^VW*NBP8M8yL|ApZ-br2#tg_59-Fu
zhBsS$)A*m|?uWGL5b2qyt2h1G!^+GG1~)e}R>D78!Y|4Zh@XAFsEzCx+6gy_m<=(j
zvaQbQ$!YPe&H|fMGyKxbnEQ@t)lg`t5(Tlhra67=<FAg7GH)rs@0Hzuuh;NgWo2@R
zlthd9%bTChIXp^<>xV1tPn~SGQcL>rF->FpD2YXcY|Y(h$<v!ZSdJCUM*RjW)l1XR
zF@|}XG0vVyIF3PQdX+FTj=`iB5-dDapJps;@m9zDq4;vd<ml^bZOy9@5yH^4(C2Yl
zbu3;@6;>K`hMC@9tqFm_e19D8H(tf)-E2m`OV#|!(@EDk`FTd<a^qjE<^T=yo}*K(
z1gP($L5fJ?Z~ivyimLQVD`jXy7PREYA#Hbs=g0%zs5_SD)5;CA(osl~j4bbds^>7X
z4D-^U`1Q@5E9!375}Sq~^oc957P}M@|LSK#oSaadH^6+kGsn>ggb7270z=4BQ-p!_
z*txtO+O&@7r2IR{WQwN!Y_SJm;Z;^ygy4_;(49K>>_QEm@Iwtojn$tQmbR``XL0Wz
zg=gsS@Vk-U%^vEv0B76tC0EpI0po+Lhi&ceRC6!#zPsB!Y_(POd>e|3s~gd|MTH(?
zIR+{R-qFxMe~Ks?xtf&j-mNw>lR2HzQtOO}2C<Ipv5V(7RVMtf4nHBmq)5IesJh}^
zUu4o#DG&q?hrbnBW1%WVNv0QqA=pBw%RwMTgK5gB7zlqp8m2TbwTp{m);lVCbEcAu
zr&FBl$0z(ej-n4iw9{TcS`b%ZcsIcjQ~#3Pjfue1-Qq8ZE?3+mznHkW#WRR2jd*WD
zJMSh94b4NET^fpTp7#n^z<i6i9zPyjzbOVk3JCK!+PNGlF-p~B#Dnme9~%pt`=*91
z37@!f@CX`Mt;Gi>r`*wzEJ_Ic0s=8oNGB)DLJD&TiNuRk3hjfM&L|YF@bmHyH>UR=
z-V1k#SH&$O1(d%-DSKlASlRkY4Qzia+)C(7mS2Wt;cv%qmgnb#gaHB%m_|&Nm-Ns`
zTjCOl+tH(kqMLK!CSbP$DTJuPGB6GL$CwJB_G#<6(=Vix#hclO(^#4ps(--4aoG5A
zulR|wZ;Nb$$PKbX+25-4_8*%f5lcTEyM$yj(^DBa_;)*JO?6%46-o_@A2+%2aej^O
zkKM);X;?)Ypnq@i)Fe7G28R~=yV$R$EQ{SeGrIQmLQ?s@sZFK72d=)?YpV&pW6K8q
z$G3GT8MlZh8{eZ$<jLNoi~8ss8vH)+cd3uSYk%xx@<|hZ;nA|fPfTubtNx*%<gdP7
z@HjrB^A@HEa@LC1_Aw{0$agUc_3nbxzK%+oj$m<6ah`--`KqM&>Fea9>OvgwV-|G!
z6q`iVoHnNW_<r3RU$vox;a4Em5*G*%iOrIV&ywym3%aqJbdEznJe(d2Zs32oDey3#
z|C!e%KN0YTLELcVq0LRAAP{2APPqxZ|D1eFoX1<<E2DrvYA93$kx9%YRZ89HJy^De
zkYJ0DM<Ab{d<_Ez`Hpzm0eZ6Q=emD=#LL27(f75Qc;Vy(rwVPAgP-sC@cpgaR2^PK
zwe^#|?@9X)T8-D61xccmhjHP;S9D<(<+NP^&j|P*{={oF_H3+K<o{M?F8HDtqXM2W
z({tt3B|LrWkmRE*dS{m<I!HF`*#lMSQGTAN;0ZD_h@TPkANoM_dzNwKVee|G^QQS~
zrt2$o{zavcUy>jd@C3!@QJ)d0xSTCOv`zgmJ^*<foE`D>!#8F*qBpb>?7_gQBEp*5
z@1K&LT|EMR@=Alr_p^e3>B@9ct`onU(C4pd0dHA^pPk6o|E#JNW{XUF80UCaR!ip{
zac!Vj{1G^!f-wvqGdnsB?Nz@f!&CMAOkA}!9J;{oQ2)8Wjn$uQ`6;JV<?K@BlLSg)
zhxx9X)bkC<!)Wm{@HP<N8|snehqQC8!(Z(1Jzv~87{QbH#8xP-30;_-K5IimAN_3q
zz2Iy~%v=2W@7E2(Tzmbwd^Zt)ZD^)Lbf68`g1&EV!dBkR)xjmJ_H@s8$1pepV+a*`
zdKnMcfBlmBh;oCgp02z!Dgz8PH$DZl<hMGx8S5UGuE_yoqt;$KLUip55~rq@J99LJ
zrCh9R4F{{!P3I=Z#|H=X1&@z8{r6{`r9Q@YMK#shFTe+5lr}I0G8{y5D=Lx^wNLMR
zk_l^4qaraITi?XRS>`J;Nw7Ux`_|0Rdwe{XFC3y(T=3eje=FYYEgGGU*d!rYT@bf@
zb9sE1;?r{m^CK?LV)@Xi*`aZMg8K<cznr>hY-SMXGd11tbM)$J|3>rZdZ$^9&-y1_
zQW$hs$R7b#@QmjD<eDcV)QPfn;?n*#^kCa`a?yVz*LSNEB~It1W-L7sV96U)P-41A
zz`V7;|BZ&C;cDLddf-O@RcT8`f`v72s@#VkGSerEY9OAJNxeFE!S36K@GcO%Hc@q#
z))?#B*c6$bCdml!ci+;M(@U)pWQ?SMihd3XBJ;NCJHY^G`7@m!#(tMK;-V!A*B0+c
z$#537tG+qzYc*Hj;(ws9?Nqndn|by+bn1fi963-AW?acWZt4KfWu|DQH<rcQtM9-n
z*KafFb;iF2cU4)EpHX%K?s$3ywcu;s-u@oHTid*mou8r1vnNV3I>AJRBgH(XJC=ht
z5&o`DiJcAI^Ux{%7H?#!u<%=hC*7gZ3_pJ3;ZgpzuvQPjQK;BA6Kdee;=3pPUL~=-
z6st!DBn=W%7h_{GBxkrVNyey%p}#K=f3pXBXGDfx7%Hj6bkL#p-qDQu@TLE56T=T9
zIlUQU8y_B!1$rheEf{YGsYs=?<2n^?+1$~bU}a?sPc_T+L+%iO;J)=_!d~>$ErF$F
z!-Z@<@+LoNe_s*aR6Az&M)Z+afOQ>>+Mp;XAP8O$BxZ1=$aED^{F0l)*w#=UgkPWk
zh(?eKP=Sdq_ip!k>@2<b`uEs1Ebg>RL7D%ZOwKjJk^EMp-K`wO_bTq*Iy#z^k8g<9
zso1j3p57;(lj1j7ayki1u-ldu#8ERfb$~j+Z8PefX;w}r3=bd6$V?p-{1yTgfFzTD
zNL^Er($Hq*R+;B2#0V1vSqkz@_%cJ0fh`ML-&*qR)hzu|WJp_8n=v;rkqu93WjSp~
z49>d~3L5`^;JssEx=Pi{IVo=8a)N2A87q)=9PwMM19;vz2ta9OLl58z!9Ip#;l}EY
zj>-xO37ShFlr4YE0stf2<iP2rvkr!1?QT7FZj`f&;I9!L<-*QnXkp6nDG2TxV+F$+
z0?8#8^fsK0Zt6OJTnld{=dwt&Nals}Amw$<3?A?U-$TT*>QJPQ&887I!Fd_R?YXh&
zlxT2+$2H&#yHrtb1C)ANAB<cU>s|cp5Dnj()&ZKC(W0nwyvQBa&|nhVC1uB!DFsPC
zk81dQ>``HKCdhFNa6bz^+1~rU!bD<(^=VCb1x4J~JZ9N{kS%QfNSh;V)gl<wyU<Us
zy17Q!r#E+E=VJ13E})4nH(W8aMxd$KTu=c&@p94Xj;i+H?VR3}pHfn8UdfupJsV5r
zx~`Dk-wsf1*$oiKv;MZ1r)~b?VC%b1&Htf#y%06YrP5dLlrg1UVV-DjJ_q&XRv0yW
zT_G$JybBUo!7DKh7WqVgZ=w2)@*+2;fK1^7UXdj8Pi6pv?u38UnG;G6OYczUa=lQx
zSWXw3V;it}Dp7O337ZgG5QAgCWgROO7}q4<(X}sHFj~%0VsGa{N0%;txE(_hW(dt=
z0VB$9waIVQT?_1`_zSKDR2SrPZ~5$WbpI9|D1JI>FLty{+B{6vx3_=gG*&Yf+-1#1
zU9X(Q%8j=sE2nxNZv@x!4G+2Xix()=(z<!|$DWEJf?_HlP>k)eW+VK1k8e}8f1f~?
z)4yKFa;o~hTdRIYZmyp=ck4q2%Y8Xw;wf<*v5^8aZT39rM6J<5GH4Grhv*w@y6i45
z&FfPE!;9s}#=wN$f&QpLjHHm;>2~UlJlin|34Mq_;CW>qsY{DbG8_19^vIYKY&nuH
zdi|{oJbr3ABh0TRD557c_a5sSzxBNW+1Ag^pHKPyX+AG`rOWE4h>Fe_Y#Pc>;Sg<?
zrmVl#S1;`{k0)IbTHhAD8JcF;PZM(`&RSbN7!)PNko$&;ti|u|3iI*)B-+IOgjP+D
zO1#QZA#+2d36-DLIs=z8*(m+pb_L$nYfPO>9yFjMn$jHVvivqxTdv@366fy{mA!0v
zi|e~{fZvoCxM*x|(97;idx6E6r>3M#{XM(l$-BJfZ+yH#=|Kxd@7;BP3fhsUY+)_5
z2=DJcDs|~|u^tPNkoQB*yW;h`j7*Q`iF@g)nj<chGUI|->B^t}AMkS8jhGtM05RO?
zv^)P~nd?~9Iswl>dERN2oUXk=i_266tCV+L(SMvz=n!kjMC{D=uxK9#Ms>Pnew`Fs
z8E<Mq7Z@=F^6J<<=QqtpPIBxpzMvq0O#36FhKZcb-~$7ZfhZ~^X8Q)FYHc)*v6n<H
zNqv+_e|>NYN6);P2~f}+v@h&u3m@?5oSRWZY!OJcd!3)x_`BwwuO>&n_xEq!32A}S
z4ZOWPv&PZT^L!+2J@Vuh`4l}r-<an*f%8b8M!$;*Ho(oM4UgjuIUw>pewNO4qdd0U
z_O`NG>9{g^nYE`h_t4{ZIlYR^y<AFs{y2IHd3@pM_GJ$EGt-~W#nv~lS7%xE=VV&l
zxjI0nfw4q4pglrg<I9LGPpSx)f$`+lXfhU-6Vn?IKZ}m^oA<uEXJy%zq%W^_xveEf
zN`2I}zlQ&it~5Q`$+)Ds*52KtPcYN^rkSrqdvBGT3{WGR_wQJQ5rN^HH^NB<SRm}*
zL@U)u!ah#bmw)RvI?0;pu@Io8VD-8Lh;>z0HK@a)@BN$el5_ikcq|Au&$y$H%4Xr*
zWc&|@<zt4Yv@tW|q{OzYW*5ggNmy7`FJ3rpJ$nEu2h8g=tNL;|T<RAS_hTWvqUIB<
zSWlky=Sb!ivT+Uk@IMi)hLqtulV_;TL{eR7QWT>X;RFBHGHi}m!A}hfzy6+0j~39Z
zG%0k7gW|HV&joZ7HrC&{n4qyG=YRQ9(&)1;H^4HuZq?gk$CnIhH!_d6e@H5Yx?Yrn
zG9rdLh!tr%P)gxFPdEjEKLs@^$THIvywZMG!&~JQWqX95KU-sAuZ#gZNmi%j)RPvg
zq4cTu>_BUkDtf_jXP|*%=4H!g*?UgiDL;QYH8tPxS?u#H(_oe+PamJ13<*t;5`)mG
zi@4Qj_F29Jf!Y&VoUm8eKMprLIp>?wN$xpu`>nh1J$j@eyJY;_R;Cnc&&`;x7#qaK
z{Ncw+O3W*gl4ZA_B*3FvrLH3r$P}+O=PHYyEZA+)*O$N7kDku_UNrkkw|=i(=gUYj
z*W<IVVPV2D$<+tmby)2|l-7?X&dVOn5mox$Zv7eIO4%~FXLAr!Tw2Uou#jTTZ#aE^
znZMu5>T9Z5Jms*wyXwNoAhE?Vsckt~m&|+@E)3Y0_ZC(DKBD-Mx$`hv6d&j%Q<@2N
zbO!ufsF8!^3%TXjheG<Ou=VbU!a+O}k!<!D>fuweLhtJeWu4TnLyR?vGDy~-@05pz
zDh_?e2fpxa+?D9BO9fsn$cQWw{$Ab3qV2i4UTtsuJsi74Djr6H#Rwy3PfG!T9BrjV
zg4^xHgIsQfZNAC31k>B%I@*{<Kf>36=ODg+QFUTJ2QtMLN#=!M|2{JI=@S4v2O(`B
zkdHK90?a`2wcH~A-s^74BU3a<ujkWI_kH$Hk9<}&itGO5oiB&Rv4&CairJsDc?y+Q
zodjoID*xzwkwFA;Cb*#*rYS-rl$F>W)PpZB3Kkc^fNX9BiU6$S`~prCpxRv98o8Vt
zX|0?2xG24C4;a1t0u?lwGU~hHksg1s@5;KUGmk3PJ5P)b<4Y40z&klpy$59IOC9an
zP9ts1JbnUH5$)_|w9tE4LNR1t$mzfHr;ce08!163=E&ryzH9?o$Nwk`Sk#s7?$(ud
zW?QyGkd1re^Nfb|K}^+V2hlD-weg|Y_>&2d1o8Tri@)XvJug<vN|%}fpr`1`W!QH5
z>E@gnVah5&U0~@LQ91zy#7X(@?}1>v3&HW3%$}a$v*J}b_W@U@uD|>9*vQ=x`+VK%
z>Yg%Z=l#4*&RLi$MmgBY*i4tGtDGlp2;;4-)@d!|TR&gKq~5MUt^dNb7J~PR_T6mj
zUwTLxi;}mv<@8}Ud{a)=-_NIL$>V2L@q(u9_XS<7#g_oW{nHE3=zX5HAzd=(YdzK(
z&&Al7e=_2FpZoR=Yp>jawBu@aZ@@R^eHi6-9yA!~;ZKdYS`5zI8L=<S^}W&LlUhV8
zhhAQ*(X~$wEnv~fPeNjrt`U}~MudL}!!c7{6JqiVHzM)7zK+mnNLlTcyT6wA@%;NI
zNEN*#PE|w(p_C}BEE3joiZMVAVaEq7J~;l>4FHxJ95=7*k|1Y&d2mCm{QE2aQrrL4
zFz{~bxWqd~_W%Bmkbfqn|5}m|A}K3$8C&4FfQ0ft48+g-Wlib?8KB%&JrD3Y>*o`#
zM70>aGaEUNRX%JfovhkJ^H`;l;x8l6@I65ze-9#H-PXFu__ESt-&Koi;~X~#+~NPK
zwv#>2djp#RYHKc6qCocwP-b&;-{yYq?d_ejgVj4`4rpm<<&CUwY;5?5XX^jGQNop#
zBM}9SKzk+!YO|X@XQJC(4p((Z8L#6a1H|;!jYxR={kw_H>Y434mGoCcG6tLz<=X60
z7%dC-u$gp~yfLd|577}V1Q1gx1^D<*FRy}w?-h<s4-YTiXxKnLTX9>?1u}DyW+6fz
zTw-9&+R_s9&TY;_MYT^rW${#3R1B}zoo#}cHUqf8+lm3UN0X#*3<&dPFH7?C_dP^;
zrwe*D3`@0sHSnhv=jL{<9STw6ZubDW-VO*_7<52j4#HQ-t88qXUD^kNOs_I)7`Xn@
zV#UO!t8luL2$0e?V&NR7QhFU;=hy`gdGJv?9K&0k>d90wqKlB!V^0_#A76Jb*DdA^
z2SR@}gMP4+ZE{H4pFOW8t5r?dE>KO)rbwH&Vugp@OdA`Ry^5H}z2~&wpV^L`O>g|P
z9ed>Mw%>CLbbP!^8wMQ;(}mLK{zS9>Om=@<hxTT=%<Eb4G2l*Cw!Be5iJ!G|mzI{c
zwk{mA`s_RD2-atGnqWJZokUR|j_?q592--~8(DFxo7(|PR5(^2VMb>{)fIp~-6{Om
zrsQvV2KaeEcl+FsbE>kp%A8be7LstES^Tyni9?<KhEO<A^sK5r_kwN*6O*s{*(_n~
z*WU{Z<Hu;HZMOG4z&5a-#^f6yd<u~`cW!TJr2cwau5FqLE67J3-78L0+lX2_pvQJ~
z?96A}4@Pf3xd;k2k7wMfJr2REO1~bR|AS`Rm;hnN0kDS_8}3X3tgOR$C_%iCS=`F|
z69<W+xw*Ls>i7z}3pT|X4S~F@LG-PzPKP5W!?lGvnz>}MzOHV{s+ymle|&7rQuWho
zRiCu#Tr(7)<`m3GzaPBz;1}AtA@#PP;pxvM&$kR*k0Ie<89@BIeUK&t_U7=k>ayLb
zj#BY?L-S3W09dqbx5BR{d_YuGRyL@2QBa<AG?;LtnvufD8zIBEGs<Dn=5B7Dhc89d
zp;}UuL6~fbsmBMv&3nC@zMo~HWa!d{z+&y3U3Hi);texd#k6s`1p{c@h~5mW(Pjs2
z5py2~*3`Qu6MQ!mg{Tof32WR$8q?khMl8sIy(fMlJHJdfv$Ws-*_io(K$`rl5knke
zR6&RzQ4zsHzgadihDIeePB@9=EoFg(hk$GdXTlAU?$n=JVIMI2BmgWINTh#hA|oSt
zqu9dYa{SuXU3xIBmo0qhhqNc00<2hI+U!6Wu83eP(buEcSxK?<`TfJbx+8KBCfSBQ
z9ku}@-+&!t-$t3@mPyNb!*n4MH?7Wn`sb$44q;`m4NzY<3s&kP4w%9_P#1H8%>bh)
z(zN#W_SR3XPN0<6!&$*Z9|J7}+j|MpiE#DDW<C>9*~L)~^nvoV`JUm&h&*0y`DbI~
zhhH031bzS1FfgFw{UVsfs{$C1onUTsl&Ba2Q6#V6$~e8|_xH&?#`E7HR9z@vT^DgS
zeZFQMg^)N@@jvRVG@UAk%{Mq)H&>T#@omkDUXK4@hAL$w;T-GziryJ#S_O8NPZU^W
z>l>DaW5hEEL}Iv0>_jwUUG>q>T+clEDs07J<^iDII$A*rc-N;gX)E_ibiwhgZv#7G
z7s73if(V_S>zA#6nkmWPr!Fk@tsD{V$i9?ffKsujM2W=4D{GGPg?^=rliin#KK={S
zI4nQ|xdqPR2NDBZhd5D+@ax^YRlb<cl)p*%*-(3J_^r{=B9*1KI#48a=Rn~k*<Eij
zr+|m5r$NQ`5e)!|qN>cz8fq{1d8BUzazg6kT7h_d44Q@iVJz|3@4hPxBOa9E4Y$Bv
z|90=cTF#92)wpp`c}B8c;(Jd7Xs<$XDC<>|VpB(-&<K{0-$=iP1NB1p%{Dt-t?59C
zhE`XnCVKa{ePP0gq~XqXrt1qgaxzqSE34=u{}9;#Q-yZDY|$`CVl*Y)omas|)4mUY
zCxrJrpvQ)Ns?wTHPo5Xu^ugRDgajSr$9GERvrS?O_Aa&C@S;koexhd5hYPn*GlFXZ
zIMLAnlbeoWY~=Iob|t8=pn#B|t<Xc>v47d}r5#A?FTV(Lx6h-cxVe5d@$2IJ$!mwV
zb~tdHrcLS`RC0Ysoacc=gslz7L>%^Y_K;Gm+P4To+WnLBFd>jBsf@gblgI;8ZqXJF
zun}5Y{z;e?5O#hLWgt2Ul@nf>DbN?3zAnhdU2KT66p+qod%^b>_C|bx1>VK00`XnF
zE<DSx@N{TSX$RwGC3!(;{=79sY$?J!?f7rlV%6v_(ll|?<d^e;D%2i@Nua55Or?GD
zW%p3Mdq&R=`eX~9y$kw&wuk;yO8A;}=+1u-1l$4z;1{j@4<BrgWnD^O?MM*9xD9S#
z)zH-j!&4CmI~Qu=|E;4?lgJ8uq`2V#^*H<a7ykN-RXi{6H&vHSsZYeK6D9nETdhU^
zF+Rzfife1tMG9)Kmlt-gE=ri28VXUuXjjqG8{D7|TdS!6nrecPod0R~VHH2hT9#?g
z=N85co7{i*HI0zu_Qp4@1>z5I*er!VZTvE>%yel$5urGEVs_?Z_#QP%#Q<^v&*!>{
zZkjGs-Tzfy)0uJ6I7N5{A7fOK$=CAYkoj*zDBxuzvq{xF>&U6!GB2;*my`R}^+QnT
zxF}T^ZZPMFF=&Ci2cZl6-Cj}U0g%d*-J{a)AzN+@05Ob?jvf_)W;=slZ!1A$)+^Sm
zYpkoQ%g<l0*ZMLrF**4cd0uUOyOq~C%?2VF>?BAD($}wFzd%ol9Sr^U`@q7}OTUIW
zyUHnDUER1+Oj^v&T?WuJpn2y8C4iPsT|GUt_wTxv6lzl@Hi@OGs;UIjRqApF*1jI!
zU@`el<8sD24K*UV=b(vAzp}cT@AM2c;}~Di1XYwfro80Vuu`mBF`{NT?xvBDnAqEZ
zL3jJTq_lK^O5Q(xfYWoXuCD7Y^^&w-=5puk5@Qv`GS<C`q@MEggYSbwfS<pB;<Y=u
zyKC<DiBB`=-Ah*ipo`gE*jIUqc4XrDV}PZ6hHd%veiDN=lMZ7`3yYZI4F|#R-S^72
z1W}5Nntv_Qs|nn_6h`6L<AD6e$CigN18bl$2?e=IISO6t^u;q(HD^JSjEvgo;XFvI
zzuw0CCgX;*y{!cORizWwuo=~K;YW$gv#8wLwPa~$mryQfm!8~iE2oJD8s0#6+|Fwf
z^j-tF^eduAe^NI7(9tFKe)G$-vb0QQ6=n+?o}aga&1`OLm=0kmql1m7H3x7D5G{>X
zJF#>kZ|(yexV^Bwu>pSnnsUNEFa+?5I$DmRVyw8Ez_4yJC53$3%F+AN-Dc2evw_8#
zSXfu*IX+4OqnbLNpPqgkST<$D$Hxb5WLv@3#^yDi!s^Mo^?|U$6+y+64Y)SgtmuA{
z@NbosmF4AXQ9Sf5BDY$3v-R&t=7LlGS{8PZz3e5$I5A>B4xH50<H@rCrd`<0q}DL!
z=+@Z88Z{`yUSmAAGTI1Ud*&zOC`fq_nsHobVQUM}pdBpLiV_9+YZD^I`nO6-IBjj;
zv6siB{{Xoirwg<{T2*&`8`Dwr2*T$r&v*Ys`Zm&WL?~S)yu%naV}|D&XZ3q@qWvhB
zP%8vq0S@qnE8+nAA&AO{gAF^+P?(*zAwLhz%DFh31vCQ|7P8@Ev<KPfO#E@G0}g@U
z1W0x!5@-_3-La@<zd!-I=n8xp4%}GMS7iF0XR)BSsOM;Qax%#WJHNEf37((@nnsYS
zrIP@9YZme;pb+Ad$Q^V@ZxDbFy^><S+$Ds%8Dy;9UzX0YR-XC+>;P<)w^*f=SmD)A
zkr8TAzvZxO7cgu-majO}+<Z+bXbv^BASG^M@1p^H4eS+j-Ctl;-zK`A9x$c;T6Ga#
zku@dJA_{V*xdp13qUTS{Xvt#D9?1X9F{{=E#-Y!tWo^BAr29bU=O*$Tpg(dr!H(K?
z{W#)H9)YL1@Bsj>09L|g3^=t8)Ly)TIzNg*3xE@ntbF0Y9MJiM>P_px*g5tZi`pI^
z*Q^`dcJmgc2BpLNd_Rm7(i`RU07CZst#x$Q*X>Tksn161SVg$5=4|I!So_Lv#<9DL
zXY?o}BOn;9Vz*u1{ZV4XPaW!fSA0D^JsmW#C2|@JGi~MPvsP;Xd-|@GSHcWM1NI0$
z2i>_FTU$AkRH!Uf!q8g*Vh2BSHzC<l|BEfq(koc<Q9dhBOZm`=s%2qeaPSM~Xp|{E
z7vnlfq(}|p=kfefObGlIijTcFJu+&;uczNF2f|7XvtG&I;NWjiw@j66-OCU!`S=)Y
z?jPzEZTlV#U|hkLb^5rj^w>VtYT|Yxj$OG_RU#@I*0)hXn44zWvTsQ?qGrkl_{EGT
zrB=<Ri!kUmv$nE|3mmN_m=E;+bUauQ2>jf$X&!VxCM1x!;>u5DQEUAMo)yx@b2&*s
z{pzP&xiJ(1r0qqEYl5x6E-YU03Iigz@-jEY?XWy3(qz~mNwEJQFulsl_wYBrL7|q|
z#It(_mB>VrWNckl<1f|~L~|~4PcQhGs?p>EEjO`3V{?MS7pZZmdAF<`)t^Qe_%hG7
z%UJQ`v&~j9l5@7jew)o3M?_%cIm8J9su7U+EI|w0yQkWVsQ2JDopdhmEsYrCY{#@C
zq4g#G=i~Bw;OBTO)q)Jwpf73cZp%Gi{X<@aacF3?u<0!GUjMB3eqZ@7nmjWIZHO9x
zOc2P>|LbSq?LU<^aR6`xZGA`QhWp#sY7W=QGAVm2KYh;w@wZFGr7CZj5r9Xn0Lye=
zMB-JMqw9~(o^_aPG_tcFR|!|{g*a4I4?tHSxe>JWN8i7I*@I89?DTDYzBC_&XOc*@
z_v7BlJXc$eEb0F9(XThRXV*6}7>J#K^f-=b@&xzIHQU04_|ftEN=qt3AQ0cz{UWGQ
zqs44wS}UXf3F*ko!%l!tK_gwPpY<4^RaQTfVAC;$Fia%Gm)&&7$t(6JfGaBh1g!>;
zMde6n?gL04t9feX3D?G0*xGekU^)q?{3S^M`UVgTK-lNe$AGC`krJ{^y-|MZ9rRqg
zGYp;*y<>clj*Pg<ao*ZSDnT2&%AB9zNP%kLvo+vs_9SPtyY`bcKT>aoQg4zypfB59
zi4$v3@^%AoEo%<K_<G-G=~VSfBRN-*0)$Cz9j1UkKbQ?8s;CrvF993}Kr5MM#^CI)
z9XY<_hv&8BPT=lYi4+(Bu%S%_lRBC^9UCRA;kN?Pf=||;scxgBw|BVH{j7;2Q8fv1
zN*R|R$)~t3;V3Q#>>F?tQ$}bXg1Cc1FGVi#KlWNlGgY<lT!5|x7__R#s9iRa!S*Xu
zrpn^t2#$|Zx((5$l_kCPPKk7Ln?`&vVX+5bJYqXQ<3GU#RG`j>^Y{GHmzpW0D5EPQ
zxT`Kd>G}5SY55%Z+DdXHtDRfX(g|3Gm7b}grKU<;vzQ)XNw1bc-ZW9v39p_TW}3B%
z16?TW5^ocx0S(XV@;aI+_}b+ax4<*&W?CnOfa8fs)d&DSa2@uVjHY&TI^a@a?XRjZ
zgS`tT-6wMdent8BSg*6d{0;9gO!acfdJv0rZ$U@~elK0q25MIyGqv^qIxE(=p$V>E
z0RN16&+C=#ezCCdMEoW6;Pk8=vte8Q;3poKM)iE2DLMkcVyC^Ae}88YVb=oi9NBWr
zuGJfHQD(Bo#-T;sk)S+8Xnw%<ixBM@-E%|BePCsAW##g3)44Uyso(s34OWqsQ28DM
z>5e~>*J1>I_V3QIpz+kU(#zqC_Di|}iukq%bb+U#=kZErl#@uOxDQ*Rm5#+o)~OIs
zvOd}CSX=Y*TgH)?%iF#U?M(CCeimC+{&BafPC2Qdc_@jS*tr8m6T+{}e|1<xJ#|~m
z;_>N$OYB*<jIL|lMYhlrx^)Op#53BErN_@&`PslC;N|mI*iq&tO_!c;m(rx1s~XJ7
zDDsvMkBJv-BJ|{ccG81+vwxE*>ZYy}#c%eBW$IV34i^}~Xw`SFna&g=ZemYlgDyN#
z|12K{Dvw0c59rf<(c6H9H@H;u2G$Pjr!v8e%iDtG)aUY~<@#;T@tRd?3|JOS5e~xK
zFn_<J4|KBLt2r}@_DW0RB<1=!=ZkDz_n?0;dlkIxF3rhrB^7oXEnhHahWR>tLuTdt
zAtWBSJ}p9S7$AQu2>_o#aqP=0fnVvB?zNFp4loT&fwDB=QqG~;?7j$psjn*+7qO|)
zPt73yQqL{>6Y8YY`PqF^%UN<zIF&s==<Bd5;H-75KIoIip8dkFLHfElw2CUW#iEdJ
zwj$;!o35<q+i3jk#Zy^BX6-<G$MrPsu_MElrQhgPkKbOCDicCl<9N1aZO#_OvPkD&
zW=zGKbE>Rx=0~(M2MS*2H*Hvfx|BKiCzw=VZU$o;v46eF=~>2y$j^=PBm+<6&NDsp
zB)X_ypZk|ITeq>BwB`B!9F_1~xv;PKobjDwMdmK>O4G0k=BwV7rKXO|5=F<$#)?l&
ziJ`st`O61N>KlzAiCqL9XHk@7VLKq<>|JKPo+s~SR?<L-bC1;hR5)w7J{da<v_aWl
zz;bdpTMM#!bYjB3FvIdGbo*e@8x~C|nlly7bG&_+vkU>%OS;%+znm97rXwTntMYKr
zHYI*?HAo2cEw-UeGOZ5jzZO63?ojAg7@X)o*_jy@Yy9S&lon}vy$wiW?X$G>v!0;p
zZmBZzqR&lu-<<_EU>b$rCa0Iv2c3SMV6aEl4_=@PD_#*FMZq|S8s$HrC?$(-X$N2|
zXVX@|-GJHKZmNSowER+AjP4cgPU`Z`)T?rwz9(<+pzIm)mbFdgQA?!c8R;XPyB=(N
zO>%bGu{?w*F0AreIW-bQnO`oOrrbVPSH8Zb)YRnUVbAG70CG#)I?S9-^?4oygdQ#a
z^x&Ikh~iKb?T#17`xt=I0t~H-FhJA>KR<7~0p3XqwlF!VZHPMNCnq1mz;hafhOP~B
zQXfGjQnKq*M?jr3E3<)DW@Tf#uR9<}(6CPAJYpn!!+V6veqCBx%6e(x;_{RrRB2Ar
zPNf0?*WP+C@cWlF*Fv-9uqg{WyFMTJNRzYGfq=RBDdf9uJY%y-kb@v4e<}w)hDVcO
z!<^s7D0miPS($@l;%y^Lw-`WhfLu|vgwRM2b#xwhAd1u8#^%VY2_P)qk6pHdo5$cW
zknPx7eNI<jP+hQuH5O6lG}@GF4?nCJgN6Z$4^-B^w$-&)HX+M)!o)%bCd(RkP?q*x
zo{?8rSXf+KoR^m%^f?m~Q%S=qLV1a%<=Y-9bNr>i^<~h91!zoSUZA{L3Xf1Hji%r~
zY;jES4O@&NJ~co+f-&^pY7LwoC$l>JdlulCrrFsdyI(6#pcd_Y37*56*aRe~m26`9
z3}D)hFa25&@}mLzqTc}jlyOUF+)>oyi|u%oE#;dHxxSA8mTKS2dilMj<@#uSgjqEM
zAV#m<CZtNHxbM})=}Q)(%rY~q>k(C@&ku(^AcZ@pNhiy4p7dW*szbEz8S6YUF4qR-
zYTkkjCt$&y_^Qu!3}3P-nLgEGlO5eIWD`v9UtL`dlK37S7_#avCw)0yU97sVjz@yB
zQf(K<6`COnmlC}sZF6eRzPDb8E=!?|P}kKT5|>{Ocbk|4xs#P-WMqPZg1{?Nd?>TE
z8P=WK71`2}9jF=xHe(B$(bLldMW}vx2qSg{u}m!KC=n;u_mydq@DA|cSBdp8s%!%#
z1UNtu4xU@Hs^<KG7JZCBd1+~Jlp7LG_^`_S8H+c~$3{O!19h1XfF08QuG1s2YId;%
z`%7K<*{2uQx(!x9O4V0(n-nidD(j_ulLkd?czF2X;h`tLb87F(s{vjc@Fa_@9(byb
zD^1w-MeZ_qdIHq>pvqFMteKQ1iT4ntmQgb;{#ZMGT5O1M=k_#wesB;U_8emfkOqgR
zfDWi~go??7l9ppDL!&f{(No8*2<E@nO}eyT-R1Q-!(02_4x_Xg8Vbj_OdDF+zEv(Q
z_pZUDi?}|*o`X+b)ZjOGZo^G#CVyF*>%MWhIV>#xUYs6EeZ081SX^9;i;G)aG;T74
z>g)G8rvq}@`&h4#q?yW7C2M9IFv5<m16FAoB00n|>swpk`8m$S-nBzlH@5;F-6$I4
z%Bd>s_!u(Xw~n^`0NVrJ)~%uG+cz08>f`pVIyd`}!?k#4SzPKkItl=PTZdOu?<V@Q
zGn#kV8R{KyWO}8TY{*Ku%e6aprCBvFGchgJ*RyhU-zl#|Bi&hm?Jr|vFMJZs8{9>L
zbwe=(l^N^$ZU0qzC6c%2<(7!dXx<cN)@`t@y#xj{D;@R(u+%yNV0z2@edq06hSCZL
z6{u|Q#rM(8;jBB$w3_}t#|P6=&J;-N=jiQuKk_$BvGS%44tuk8&cNz@kaO<-mz&T`
zQO^UgSU}8BOfD-GRTn6wI$I53h@N5^PgZa2U6~QMdwu5o$0bdSRdq_#8&M=`b|}zt
z`YT^OuRk4B3aqK}4c$hjzgv8zTfRMxocZ*Ce@CfZ5X8c9+roT-?lVW3NykUi8!FXw
zGoaMgTG5H_j%MzNN%si?(5#n`5)j2J+=n3GYBcZFo<VtlF4H}itVyK+sN$koc`88q
z&Cd}6#+|hEc&6aEdMs$BK<gm+SW*3|9Nz?KL(yQa_lHZR<ZLWiV;8aN|4`{+7b-Ab
z5W*Pq@pNc=6cfbXKx?m)|As!v<(iN0{H4nd)Tsho_4Jpt_&{v|<UN%XWA{BXgAkw#
zaDBAwpyp3iRwHs1p^omds%8>8bfWe%VNZQ@Poo$p!5Nl@W9uu^y1QHKYj-h0z*81X
zwY`}5ACgX@aHvs<Zs$StUGnP_4iNohFtRAf*iJ61xfInO1%;Cq#eZoub%cKJ@<wkt
zzyruH5I;CQb)>-&2qR8)Rm2R_A1uMsmf-?G+)Go*OKW6OzzoS|ae7MNIkxR4lHR{w
zJ-xvihZ06uRUN=HaDbMudp18hHukNAX;bNy&&$}04^b7a$I>+YUnz`%PsL{*$P4fT
zd|Z+QB%w?Xo?$n}W0W2gNhxaTw^eUR?(~lGQu;C#%@tSuQE)HJzNZz<<S5RC4AJ;t
z_WCQRoNh2Dzz}*7F~!?`<NZN^4+0Dr5b(^ZtCU*1Ln;zhrg_apD5C<uegGUEpID)@
z^u61+Zg_!`U?-6K5~8%(6Go+L@UtsjT;pPVuhhNGRNefN)mn;$$7dwS?neW1|IFXX
z$o5Z*Z$Vt*qqxD2^!ymvIZ~(){|T0IT0CO*Y~ZQXIH-Q=etWphqcsD)M6-F%BSVO7
z;2=3E>E&TnW9ufJ^h?9IVfUYzF2SPq5lxpF^`>LR0;`EoWTm7#vQ7tf$R)CSl1SzL
zyrq#(AZ5h?I(bzpHc-=y7^?K`p3s+|=26yH*`05>G{sq=uUq>?<saKwJQR-;Q>M;4
z?s)NGX$qkF3VL(7uWEd%=31v+FQh#c9)pq)ICoWBBhFu(4wLeWqvOZTm6d6MI6&ff
z%4oCgW_PmQeBURcPV1`NSj~{vpnGjoQ5;&zL7I#eT`aaWUUE1K0F<i;keZ!K9Ssfj
zE@kJpJYjYxi<!u#%mlG0TW<f|*vIn)Al|Uefsz8?`Vl@3hkuGK6m&&CQ@bj6;`RzH
z81;;Pv$p#F8WA~(PHF=BGcIzR{miC9&Yeu-_IARZlj8C#U5=5=_I`gac1deTL@W0_
z4j;a5hW(#j0I@#=61&Ry74JWDo)p&{;l2~|`BMDdZ&QiEkb|H{;)`EPilHI>h#Mjy
z5tOp(l&x|t)f>T?Vr(IM6zTTxnrrKU9-n=ePsQgO6FXR2-k$U6xIl3wKBvKDC2dDX
zVaoDF-|Nd(<N}3mC))Sorx)8@_S&6zyP!~2(1bss01aJ_3l<7Wi~FBYfHDJM;Wk^j
zI6Qq~(ZQNYy3_-_9Qk`YKYsjR<B=z<#m2@y_VT?ayAgC>Ez_LS@VsZ_rdY~$FKIBG
z@XoR^!o9{rUeB55arrD3mVRqeCaL%umnFow3M%NM>M_u<k>nGW9QcA}&eqSyJ!6Wc
z-|)9Dx)eXHR|uDT8ummfG{gCL^~ieF7^6R-49FS7_Jz#|nk*}2P`HC220ZDLDM>H@
zmtdkJNtLYElJ(EO4p}@EZfbl^;>Km~sZbyMJyED(H>$Mn7;^s)kF*Ew^{1WbnBNlS
zQ<B`<9F?XbaSNs>BZ&<N`GEfF^7FvUma&V!JCCkTaRawxkb|}o_)<1gb#}c`6uQ?4
ztr4xs3TJ~63BOk*BzfyJ5_M*4^)xMe6vt6cq4<mKdNlRdTjox0+Mri|%$rYQ1J5;z
zK`*FTSG~mPVgesUsnX!aRb^(n$EGx0J95GQrMHH+k)GSE19HutKVgHg<jan*ow}~0
zKz{Vfro`EivR+TM;#=W}Wa{4)N%v(B`g#vB*BVN_bP(3wuF3uhk&PKP-zQM9J&<a1
z8xQDno7-Fk3y&KHUm&(#%x6;BiB!ZyQ}#%dJ>|`5WZae>T}uw1Xs$?HbDuZ0m&)0O
zTXgM%^}`&gp_G{M*mbj2ZC^Zvr9;OIIdq|pZT|D+SCt}IN)zG&XuD~m$mtohka=RM
z*E{vTm*(y)?z8lPCmsH0yX7x)oF=yQ5WdeE_<9q?E7x26Ud@-Evn>#V#X|=57{Ge<
z%ynD92RMz+uQ2v$7S0-K10sUxuXZm4_xihsoCEIW8m~m@@n23D<j(LHum#Rb%p999
z#3YV2G5fsx++a5`uQivSP=9yZmvARBrs={<ML&+NVXgg}&|LZTmi+b90*&vI(ujob
ztK1nKu)uAE_P!oRk@Kb3e@86+Y3;A}-2Kmvx2MKV;(x$@Yh7g>cte*m&xapv)i87`
z;)T0IJ&6r0JT(0}L5I}xXR?)p?6YD8DL2r-M?4@V8WNzhzd{2&8%~U(79)r4m3L8}
z|0Hdu4FsJ6gi@0>{H|_zb<Yj5N=tXYB^n-_`n0vRQSjT;USFN>HtaTozgtin38-qT
zUGb57@ia<ydS?f5o_M=Y$nFE)M|9dz^bh{=S@C^`@|Qp~?DcmCt%FtPx|#BZ$&QvE
zTnWCe-Md^k-EYmZ+m!HAR-WW#Q3N|dC3s}FEkL8TQ`_M(6Zzg<B`Tfbb^Oj4HjRGW
z-VB@K{Y|ziy@z7RWcpum%f?x7E(zpB`OD``kE4H_MMKpOOT6w&eVlZtSYSlQZKCj4
zGu;-r_9@X3`s0T=h3&K^mRbP&giYsn0&5ZWU5vxe1_sX>$emJpDcxwPc)E6dfLCfT
zN1hps-XFZk$(U!i_iKEC3haW1ps;oQXSuQKAMjL(i@9~eM3v0iqqm#8yLPogc0p{=
zr4mH=z}S#Le~!~b>{&%GLPXGcChH`8SzFPnQDn^TNI_Z6B&;P<<z5?T?Rw&it9HdB
z0lIWbOYPf&aU6R?LPC&RQ%^)i*-H_}$G)3-3&KvzaQEiJ^ix~>34%TP#KfX1x{b=$
z8B<AgW3<4cbg54x@TCyiA@dU6JJqlWf42~Q`ia=N0cJ!qBk5!J+3Z<I#oy`iRyCUO
zCHw~kGtr9m)BUT<{Y{7(rO8c+hOfZDvlcRr{nxsrK0dkjk>7qmGq$Csj2NN$(rC|4
zBy>GJ;~4V#{FmHlZ!iw=lQ{vqmcXsMI+f+m6RaLyuYG>aah_8h@%Hj4NY@J8Elg}`
z;~s2F9~pRAeO1X&nLER_iCqo?1@l%ZCY?&7t06^2Jk6W1pKU|6TN!)Yyn#xS0Jek;
zi(zuVy>!X^Ch>bC=^hB})IN(H-9~}~Hm`x$9G`K{68O;#iaqtoKSw1sBCiXYIUvv^
zayq63{fv2z4L$jZ*q&)YHtkZ0c^&;7`g-&CD}JR^#viFNOC%AqG_5>hd-KsNM<^jV
z%=GTZ^%@E@9XR~lyu;ibF3BGFa;NqiSO+PLl_(ep2$X_HCHT;n-e0A6Gx_Ff99?u1
zYl_-q^u~p+OYQThE5@9n^is%|$8YbZy^em?5Vc+tnS~rs;ktfF^Tnx)11+SUu;j@l
zt$K@nr?01;5Z7#K&8N)flw#G)+E2z~AukZ<AtCCKxK_t*8}7;Rg!btJVQ-_;sbVno
zFw}eo4nl0wRo2{4HIN;-XJNve9jO!?^l+6n8G%1E;6+MiY-MG&d<Mt?ouk~BT?2sU
z@@*SxDfHO+1F!^E!8o9xK>MN~zaAVnokGwXvDF>Vyx@zRgI=FafDSHU|Emj7Kw*m1
ziSXsFO=}r?nw=R$QP#ccJ)bm*p7UHwj}}8DHBL`V2yE^U*l~9RyvL^}!otDD?Md_k
z5{FNIrieQy5!NIBe6CLXV^JzLR$J&Pv6+jm{gI2+gO6MFnuX6tM=7u)vFLKOf+I30
zUI%#SMjYdSXCPqnf}1}`lfqv4xICHD;?VD&Wp-R?G%SRD$`p505aTM-jjvOGfZNX@
z!2YU66XkQ)Q?d8$58<l9=q<zaYs@3|Q=f#013?QIsGn{sKnFqa$4jzV4wyVO3_&F}
zaix0qHfRdt+y`=z(|>DGb8EO9%~#(TPnalT)p3jN&dgfT_S4F!1&t&UCI>H4@86X@
zKfgO%0AgJea(<@K(>cyHZsGqa?W@D0?z(m{K|w-ML_it=$w4V8NeM~mZs~5AK><Oe
zW9Sx8Qc{6|5ftg}&Y?p(hdFzAzW1E>yWZ=Z>s;r|A9Fpz!*B23j<xS~uY0X67<tEp
z(3lUk8>2vxeeri$gnOiJ1(~)2du2VrLV-8@SmY1Bh1Hk)WRQx8x6XSjH7`-t3GR9a
zLC@-qOs7q<LWKOW=fpfb`lu(*U@t9TCeC@aF)%>oxx`Fml^)@GR>9=kyUF4tUeBdO
zp>uqY=zOgYB#$Giw~&y}ZE&iQ@aUq)#;BpIiC$6lyD~HUJOj6_#yZ|lTUp{>RiX|<
zI2G$^dM#yjdVSTx7MNj`s2;KgA<prXO3hL#WcIPIh?g3ewru8iVq$o62i_o}Sw3M~
z4Fa?oJ<E>#<@4-n2p^y3i(tSe0}tfyy3Ang3P4Q#b`4H{8<+Fmle}X*S}QE=PX1_~
zBope_^VODkk4Mex<D9CQ&?-)Hflx)+ZcximA)vxum+xK;{sN@V3jA6I<MsPcBTs+L
zmY7-{IQreADt5!uFw6_l$Ln>&ns&8rrh3na>S5Iw<?n15E&eLpDPZiK7xq@ZXnYbQ
zpkTdw)3r)j65d5zp3n+R?clqU!M@Wsy9n8XxRy22b}2=~f$^jk)PwK*<!;^YVh6UN
z8Lu9u&6q_A+`N@;p8~9Hb|^-kMe<N8=^U05H!edzzaA9~fPSe_Psl*-%Ro$OUTfLw
z9=n4D#C9WEe;g~pvKh0Q8X5xOO`u)_(!6SnE*^<+pU>Qxs21Eq(%_@9v)LbRC*Vb*
zWX{aE*gB+v>$6%tBXlUb6}hPNdWDIAXp`*(9vN9usTO=N`psBEPi`9Fwdb>4zr1&H
zu~vFv4ToLzm`-YMu$VH631}2@k!EHHWIloSM2(-ua|)$<+Z)UhAAk^;h~IKD<J-Vr
ze#U}j35l!2;m-af<YJ$O4Q0Ooi>+Rm`uh8n=ah|u=H}ZOBQ{bMrLTG`pD^DogNSeB
z&0ke3Gxn2OL0617#gl%e|MEK=hk5V2I~($Ooi)$7toF<2V{5a1!Y-?^4TpO6Y+n1@
zq9h54mQ$txYB+NX#%y1lgIq;m!t<2~EG%p^g)qw$(<eT^BA%vMs+sm_DY+{}gHbY%
z4*?D@sT9Z{4gLiRVLW@*inry+Z$|AU{pz&XLj6~_%W`*`h~Mepg-tAh64gn)=b<6$
z*c&+A3obU?zN@TuQ_-jTC6$$~o}T<(v<ImTluR?&aS<oIpvzS&$D-&jGlKcGn~v=P
zcBWF%xir?xPHlD+=2`A5Xrjt7zAXdi$r#tPm!7#aXVnx#NO~*w;|30s&9ke*WAaov
zRT3kN1nuQ(^v$z{gmVereL>zhJVD2+Jl&lE8#JP)f0B||rTfbhf*PuMHA?*4w!bZh
z|HSY!2PQtmE5S5h<Yt8BExtv1S`wN^16l{=e2W%%(ri~%B&uh&i2$uOpi^9{L7H1%
z4-EFm&`?!nr2^;kJ|MELI}+vSP$R|$Q3Qs1R{wBjh;jxphw2Y&XrRJB3)1RsELHh_
z%bfzI`KVu>^ELHv9me1~_!eTXTU%x)#VtT9EU;L9&X2Uts~9r?GZOUbKOe8N$xt{r
zI;|m8iA-W!IDh-;WpPndKve7dJG7E)q|sAtPE(_*sFHCW%sG!&LQw39!Q+nv`+Lk}
zNg~%G<Ot`U)YWnZmm8UlBJLkVsLGJaI=v+k2%LeZt*dT_!`C?*@w|22r=#xM8LT1l
z=TpMHa+}3NS}D?!U_S|ib8>Udz4hRh5~sc*<mFX-|4>MveibN5bIm)?w||5wMZx&s
zM@e>elNls@YEQ*Z?=j|gJoSS;(NuHA1;7T}O|0uKyc<!J53plZ-6Dkolq=Da?ANyH
z@m%+t&%Q_UDSQsqr0{U`rj5(T4|Ccw%c=2{&sgS$evrTiNn#g!6`Us8rpw|3iGOjP
zn=J-Y3G5Z9*@zE1lelFG9vU`{CjYs24^`4re|U)8JYSRbD0DaWp4Qe3{wu!?<aw*N
z$L>3km5-zy*@nKT;4zK&Z7@*8ytvp#ww>q1KM`r~6VICg^hdm-nTeDN;Anxqd1A9W
zRVjHfC`FBvYWnk=Y3B}h>biB&NuW2A!t8b2m~o3ho*$z`YalslJECZzwkZtea7(nH
z_^f*mgNsYDyerZAz}^4)dq?XHes{jBJl#%dEO1DhM?=ZfXXmpi!l`{TI-T#b9!sBb
z%bk`oqHYOxZQ1t!o)K5-7Ia%qWgGvjDD2b!C@EFh0m9q#!d<9r@*(r*pX06cW@3L@
zo6lNWCiT`^q^|Cqo$w0?h}YKM)zm!BjhxOnS{^^-jLx+H;F4nBCdZsfagH$hG|JC-
zEvo)>`vP5XHo_9>eA42;REOCan5sBEM`-~TPlAT{+r*}GjK{MvU0Gh6{uyUuos-!=
zKk<YA^c=D!-BX%iU?|+McUo6yS<xY3f%R0!33($l{J9f8C!Mt15<h&Oqv*~iTP}Ne
zl~g#_nYLr}4dQetc|GHd?120Gm@PIwU{|OK^h<7G$|Z*Nj@G(W`X|}?+cmCzPA<>g
zq*R8_kwg!m=qygO7b*dHjyelvyHN`JV)*9Kg$IFY54`2EkL|l{;!}J1jBHY#!dkfo
zzET9#tX(6ePj$#g7gm&2U0q#MBLLLaNT0sJX*tDwHT#+$tY-t;yh?hv^nHt<#!I$U
zw_NAxyR1E|4-CfB{dNvx3)(H*+}hYnp@tV86$Xsm^J_ag9#8(-OtS^Zs>gS*rJKsH
zZYNg<Wf0R1B$B8+$mZY@iF_JvlackK%2n4#%@x1t?U8UI=H*_H8MS$XnV(BA+aqlR
zP}R~9_dYPgrv<8$B8~O9!mX}};4=Mq`e2=fDM4snUFoia#@47SD3=I{Q@Sownzlvw
z`gz5glE<T@2LgrE%h1z-j5XWLlG5}7Dii5w?{f8Oz~b4pKHi4)oq2rs47b934<t6V
zvm`q{g^<+8&?$`RS~Ui5AolAzM=&eMV=eTWrcGQ>8Pm?eT%mPea~DkbNlX4-L4(lf
zDQE0cw$K$`ck9m@PbDfzc&Ft;JxGh-n}^vB{Sgl=6Q&sJuHUAGZd=1U3T!e0bfCo|
z%BP&$#47Y-yJVCim>S>TE0*`UC8_G)RdZRq>+#boXFr!L@`|$A>W*o1>pBmz-Q*$t
z-mN@1O*-|SE4k?5$C|b9?wn3~b8fc6>At=!!ihu$Cc74!q|fy2uTm8)L8ZHm?;!B?
znn|sbcVf>R74f>gL=N4{mhzslN_XG_)8pAAJW`@hK~8=vNW7fl)?HF=T<%(OLu2DC
z+xmP1HEnH{?Sms2<_LtZ#&()ksF*Iyi0R`doq{_D#>U1%vnSzZF8zoMQF_r{#T;cb
z4j`M;4y1A*Zk%)cp;uDxbHt0)C=NH5i}-6)@Adthp-TR7_tE*5sF~=&tF4O2p0hWt
z#xu*-L@#J1-N&ZNuK-rl=!loR6yt``UAwtfEUreGZDI3zPoy4Yv0G66n_z6*vK%$E
zO|~&T?DIvv`h370Jy6ZuCla;6`j8~K3TfnB9fR~q=?)0x-=i^~R5A{B5O=qptgsqV
zJ#}k1?Q+BH?<w6-ZP|41*cL-EM;vC{8LQ!(pGg@X*P#srRpTVafo2fjtVZTF&Bd_R
z#26_QS+r5`Y^ru~;`sQu%4Xz4aB!(X{iEAKS{nK@4Sr{pqJP-Ffq2BY0;s_^?Y+)E
z_ECv<AvXNvF040%is`^CDlM(}&aj$&Mp+HR^rYm#g+m)-28ZZbUdVGNaRVM`t?sh0
zPu1qeZ>`0;4V!@a)1I|ofiT0WHsDj2m2n&Bs-mam3V)5{`N|b!X-<D^fb7&g;h2AC
z9~Pv!$n2~=KbL`7T01jXh@|d0U5dD|qg`U;c9Cv3PGDDX+z;18&S0inAlA$3el=ih
zH^oZFMJ06^>14iYu4$N)f`N3K&B%)g8*9%%M=sY9o~vb|k2KFX^DUp|g>v3UtH%=z
zm%}pj{+=%#3h;3nZXzl5AH--Lm>)FAV(K(@a>dTJONoN7ZXzT3#F50Gr+zq@!rlGG
zF<VI;+qc=IhSp&<TIp+_o^aNWLaokmQ<hMDcF45{;+_o#5F{?G>*DWOwbI-r7q6Y<
z^PGdHVQOrT^6yNJH~)ouRY#1;C@txXC2p_gXTbGEO7v<})zx$Bmq6siq5c3)`3nrZ
zk|=s#p0l2TDFOP%qa@D^O|>SPWX?fPzW}PLC7`GwT2JbNKlvC&9BuZFFD>8ImXo$5
z8gSpn@2OWVlje|KaIOJ?;z68DG&j)7&N%|QCZWR|gcfyl?EeH+R)RDffuhX&c-Mi$
z2$D_g?21zp=rH@?e!g4O%T`n+<miue9f;Ga?P6bLhoHfL49uP`?A&siMZO<*;=+_w
z=?Nk8w?&pZR!z)eTM~?qeFT;c0m;tg6%|$2pjpZ*YeR({2!tdu%T2PjtF7(h59~=}
z{r=;$@qYJ#2fw+;rn&hC6g9WW=~7Ib%pyx{ExHf`D>&qQBI51_WUM9N!}7G^AB7t>
z_2_xY%+o)}>2Jb~GPWnPXMLJ^fB8a>#7wEv>VOIBLOKY9?u@m#q(=UY0J=S~yJk-V
z#LyE1B@}I=dgZ6_Pf&bQviZJJDsh~wR4km0=W^BO^xA^#Yn)qt0!s(Ypd3u5Lr<ET
zQdZJo`I>CBuq=w$t@93O8I;(}%E|(1FAF<tnT!tixK0nd=#s<ELU#!->`te=5HEmg
zDTW}#AvLWucv>Z@MvPEg!%h7UJf@?kK_EPy%^CWm<9NSB^3S(>-U1K^T~j#0B(P9*
z-UqBG##cKNus<m3w&WpVuq9JZ6c-0c+~R71D%2bDvDqYguVx`=<Hocu3rW_DD;{l}
zOi`gBdRkQnO%iBpqzJJ$Ucxu&5)yegPx<Sr97T%!Zsc$zrS+W#sHA(VA_K=m*bkd(
zneOZk`9Cc!5?RDk@?3XVI5Dr_wko9q!Y<NCeb(tP;M`P80kLQKJ@-xqc0RR=AJr!@
zT^72t_3?Snpw*W1a<9C8Cwu&L@EQSfUxv-Brq1BoJOYF|&a`2@(f2ebx+AL6!a<o2
z-h`sbGL}7?_<=&ABwB&_#FUh$u=9b83)0&`l{r(0kdWgj-a-`hK;}%ghz<f0(+aEj
zG_q2}70*BR@H41`2a>nUAXdw4(!sWFT)Gp!Ux6}s?ntBG1HIP>MqC4VDm?uX1SVct
zMNO%*AU)FYA}%H_48v+8OYNYFcx7~aE(Y(bA|~wX*iF8q?8cyMw>m`gXrmt-=-g6_
ziz^)c95`ehKBgc&;nE3IpQ3*@q%?GODM}N~RJ6{v@ZWsI>e#D`xGeI>N=aOvr?4_I
zb82fW)t{SsTjWUj$=xn@+VkZ2aL|Vps(GPXVs>M4&9A^=3d=Ji^Un(ny&ysCYnsSi
zhGlGifWMc7SKE9!bGvBX$&N#}?%*|S40kdMVfIq{GZ+OPB>NgYVidaCd?@d-_i5QV
zJ0Ply7U$cfvH|3X+8Hi((xTveDE?N(@Z8Fw+~cGh7NSyh{^>j7<D`fA+b|w#Y}k63
znNzlyQSX8qZCZ|_GdQznrBv?WWxS5L=Vz{KN5sjD?Z|2-f`O%@;M|ny0rZnMR^6QV
zA}-<Wl>3BTXg1sOZe&urV)BE7g?uD4jkZ7g8dlS5q^P!3&dS-=+N!Mf97WAi*_2ol
zl00zv4tlyKC`RjaI2kkMQlQs~Y*shOX2(QfmhWH6^w?bSixGjlL|D=0&%4iBP>i3l
z&rLUST*#;k4bQeN;)(2M39#94IGt}0yLXcivFw~#=hv`uJpYF*0<{RqJCl7V#yHqz
zI%d=}@*kR|2JEo7cRs$psaw>_Rla}y9^+|o(AEte$G^BQw~j;841qu%mfT$Gpts?L
zj(L)`T-hTpuORT!{vbri4T=v!wh+S0-4ibSgo*vP39)Gx*yjJgYdB+jEM)=SY($iL
zk^i6WLK-s=_s-j1P~u)zl~%>RxG8l8h1JE>sLV3|qsx{%2PL^&Pm(g)IubHY#w59Z
zb6>#<9F3$oFe{b-nI$BYV)|ewGNKEKmrvyi2t;@Gh$B~f4ep)34i@}&=knn(ilg*h
z$k+sj4!$Y4Lrj89GEAoXGGmBF;9g^IbLmLrGYkis!{tydlC&D5`AgX2(N;~xH#E`$
z;FVMTrRb#@{c~^O1E6aZ`|N^;V?+|T@igZTwP?DB;m++G=ML(fx;>40Ep~&Vbi5CL
zFjbcG?QegBkprEj>84(<+w0%%9vCwUQqew{y-7$4@@TOMME{jG3pNz^Pq3W+=QsE>
zoQgT`9!A2e{$4Wpq98{mVla#TNlDt)W6=RBl2Tws%xqa|{9I%;-0tJv#;UOWzlK+(
z0i$MCv}udD^QXr(GGp~I37s;=9avg-H+AowM-ddDL;eEK_uF5{&wkfBnUx+k)81uz
zpOcW`$K$#vIX_|Og>G%l(pVU%0~YzyHEi;a(bjcCFAtXUM?JU25Bqtcd1u)RsE@NN
z&$7KTFE$J9Ecc)nowrSyWbT4q{M=~wJKUMHq&B>YeASaSvoyqDyiQ^B!l!R%SLeuO
z(H5s7bPFUqp<=#Ooy~b(1xp_XUI(5Apg@<9aiPEqWel&(eeC;uE4l&mW~jRJt#rkm
z>54yjHWT<+FV@RJ9ncF>00U&9t~pg^s^-0uyg&bZ!LqhR$%Ba6no4w_m~7HewA{Ic
ziIDShxd_}|W3mPKOX}HD$EaQmz+;8AYrBy%lmdf~wj%)UZyE27skqX46I%=jUw>u9
zo4pK-@b5aS1Eusg?&@ok9?#}1CN7sGTUMaoV&H{u%ertv$&udeLFAO?37@bBv~IcE
z!SG@yWlbM~)DYM_Cd3(#t^rHsWu1GiE-YY9qWInE_o|n^i^_#rp5aBbeTH&=bo{`5
z(Y0iwyxi1`uVA|-h&zK-7EB-VR!;e#r=Ozyeja`8hM%?b?mAyCWjKq#x{5)Apn8VL
zCh_r=838b%ahcem$z+`95;S#b1H-(#dz@sCo`zj^m?kTb-9`#@^^xz|^N3s60m0#R
zc6qN0h7UzuoN&wMT;b^FuQRo}R}44|5kLk0^L;fPA5U~wrF_4h7y1>L<=@mM{FEZR
zy)#?XgEO6El&m~UyP=o;c~BqL#dvBy_B1T1(6FXvrBl*zf49QnLyWe`67@D7(rq6d
z?daLgj@)T1yDBxN_C`<N+o8Jgr#z?y2NX|5_A{#48Q(@~Gh!JT?=`B7ap~5np8oFr
z0=@T>Ph*BSIk0~EE!!JNN0>D2?|8>iGJ?Zk=`#shFB=%M>ram@vNW=_t(o%tFNvDC
zmy3qk>Yhtw7(1hN-{Ec3rL_zp^0G#N=CLr)<7i=6+{@TQ#Yr97*nR#!G36jgeDk95
zSG}+SCX?8NfpQww?(^gOa(?{8!+z}H6h%ByT03znLxxg=%?Z#3deTc<z3fPj0Vlhc
z6IssvKb=Ku1<?}4714bdkFjJwbBa7R4D*|~rtEoPsb=tKKHp`t!NXJBO_>?dAAYz&
z<Jft=s~Tr_Ok|#yyHj^=?0FvL@ODlp1FWWfvQz2GPW|{D@x$X)`?Pn~ehDLHm;<BH
z4#$6K0cOx{sw6c{sUD6#(`D~2mX*9@n&zS%k3TNhfvp9OtXnltUMvI^%QN6XNBIOJ
zY0tcSasm7pQS<=TAmZ(l1hDo^<=Zxst8S6>v{R+T>Bn>D^C(`IiPbNbRzK|bh_?z_
zNISV<ZdfJkGM}^8NCOHsgUz_q4F+*`UB-2H<~V6<hi8gj{QBBVcR@*eNr_3IZZL45
z|BQEACx~Uxf|@C|uOPxfq0WwH5=xX9<)sbKPa=C*$f+$3>w}yt{-h~*5riTK<IVM)
zPxzYzNhs@gPov)jA64M|hyXhoKm_OR!|K()FK~S`*0vcYjQD?`v#v^!h^G?WoJq9y
zxy(G0O2#TbS5~4%>hww~b-jchd#SyC-Bq9tia_XqV)HpuksYYe3+`#C&|@VpFE4=V
zLDhQS-b!X?A|SV{tb8`V+Vu&%iJvZp>8nVW^&sIO($8tTgN3q4-vo?HUzbFliwFe|
zFmGXc51l`h-}x}Y`n|0Kxq;w9IrHMc`+9MLam46A9$J?LC|nOTclO^wFgIC&&r%vB
zWEZ4>65UDPKx!RO51IK#j54K%pk3YE#x~u7yvS8>2-nWup78orAaMdt>E;moWi0?Z
zYwJ)?Z<o|>oPriW-#^}M;EF&DqxIzIX8Y;aN6dLPRyU338?u{Vj!X@2&f?m`YJhQ|
z{F`=995MF-tc%B-_pnA@5;P7}NdxXyR#qUD2RJVXS%XrbW8Bqn3jn$DA3l6wQFHhW
z1(Zr4G0-TA<E3<I=;*BNc!Kxw#RKJ|?Mu6MNOU<$0*j}VF8aK?Ydr;CfTe@7oEbDU
zVakI!z1b*$v7)Gr3gowiCnpA|Z+oEw*y;(zut{dlY!5zN+$`6T==uSaT=ftZNL<GY
z9X2@7WqJ~&9m6YRhK=(J`c2OjJTfor6stWUya|ic#fIqEC-*W~PdO=ug)ljrD2`=Y
zU#TcNt~UwOPdTaA5I^y+1G7OR@J=if=w-vZxj~MWRmBekO{qfYz$y@6fn>>HAbld#
z{&W|3bU@_<xQK!ik!g6vlj8w;5~@>+l~8-jyLhza1%<bj&LyUMGTK6C5*dNr?DasN
z=3w3&oJUrd7Mwa~Wq0<+sw8NM^7i(~UlUX2xJvU9n-OM!Py2|1qKbu=o&B}J%Ekt0
z6hwv;A~*`k7UNIp4(S6yxdUuBbk|8!6BY76j#1sze+iFO+#$HY5cfL}#26r$>S|d{
zU1a|`y+(XE!ZTJI#@#hw_kV$Xwy*g#u-A1SY?O$Mi)?>0Df>J&1av)ifJ~4RkG5c*
zEh+u(*i8v_=*x96%qxC{X1vQu2Nm-G$!=06`B}@YX`bNIv+N^NND|0!>_8P17Vdcp
z8I<xFbb~mbLFxBPS0b;lP`<k+mFMMDpxrp8IF))Uv~_Z|*#@ltu_1Lq^T~f5j`skx
zCEwFJdH>QYt4XL%?e5b~ui|TH@ep?;sIKM@B)zwy?@@!B1i%dgU(k+<s<CCg1o|=r
zw(B9Ikp<P&!=Q|MJgZYhQQP3;<OWb@bBs`W?BJtM1EI&UWQ*g0<8)ErB*{b1^w(^^
z-7Ur;xnCX}NUVMH*TzujD|6`_L@!zPEqd%*nze$99_rkx`yLpBD6h7PaQI7mkmANW
z1Sb)lUI4UcmFRcMUmCE~PycmN19;JzH4F?4tYO*<>~ON9hVv8e{P9=1u-9(r;|R&H
zz+d7aaRlsqzRo(JRxa%e56CdCzy{V>#~77q6=@VgTY;#~IR!Y}*afATtia?(s%Owk
zJ_nZ=SvwR|p8}+8`OFOBhr4&Ev~y#|<C^mY-P5ku*JXm;3d^c=6?0UmEPn0fBY^dA
z{PF$C^v$BLgkqn-VcP~$D<NkE0R!XyHWg!LebpT($m?=YLqnls9b9tJX69Ts#S|+G
z&UO<~XHB1@EfAa}2`{4uB4@CTfpIyx^3x2`g_>+(k#OASKe}z2r^5(=sE<Duol0wp
zSL3<^3UGji!DQ8a3SHltX}W^-DY$em#h^A_Z^;xBgbPrl(%KuB_ZfJAHvw6%wKvTG
zjjdA*NU`zgjSw_HWdvBs0tqT$0}qg+c${kV!2j4+r%wZR5<#;~|EH0+%gX?Prj=9k
z78dpRG(qQ&gO>vuk*xYMO*f-{R8sKSJCfqO9fJ<B2mbXHHqXf~PyMxi(A`nGXFLOz
zjK}4{YH6OeoBjfs)@sowEL{S$(b4xUv$vT6`mQH7*(^Qe46m^NZ!XfS&2&C0s0?1&
z^jqKM!s^dkRi(BPZmabW;}KEi68ENv7Bb4yKe#P{#OfzEO4aQbIB$LH1UYa<!vbmD
zCvv0sEmZ?kB%eAK&mA>^nUi~IG2xw~g69O%BSIRCdZGL1>-T_Al9M!sY>=1!5~o?o
zaWMu*r2hDEgaTcia#{Y4`*@o3X$^pki|BLmh`A!w6lmocZftFitv7n6dixcFvCR=O
zooNgQ<S7#m^d4u3Z0OSE{>(?4R@7qd*T`ySg@N6_;(4;Q_gXvzoPke0Y)x()jHwxS
z`;>l{!E>_B6#aVeww&NYz=Q|G^jWE!^UH(c3dGtS?1pGw3xjn+6T6*m0Uieu9+K+$
z%%NfGnd7eN{sBE4LMy3Jl#(|WRBQuxY}Jr(oaZ6KCq-4Y+kJep;f^taf%vJG!?XSl
zB5=@)5nxW9o=A*bR6fi&X>M8LLDtZ(r@Y8^7&kExkSFp!gR`_t4fBfffkS)T-TmD4
z2q4Wd{olv?WG>__ZZGV%DSGa2mc>-&D&UcvF^N}8XJx-ZkyR4z9|emC`}o`~LVIFo
zseG5B!DUw*%+vWD>~f@8f<$&#r>7e%NHTl4*M|JgIb8U3=27kW8n`Xyn&zFlAV49R
zFPl+b=nSYti-QxS=WhQe7P&Qi4YyT6?GC4(N2>tSN%p&_-m04tT{$4SO$&w#KpD^g
zO(H09%7f;y?i@6HBy7OlCh6$f&H?q<82wF>UUS>UphiFmuaEjR)!I2*)r$oAq1KKW
z7JZpwJe-fwfCl&qK$##up8){(0K#i-jWXcNYdqw6@;+xST|Ub&qu1~)Q@v+6)K_L6
zs$4x+*Mb{FjO)A1+Ua%wYf8%K-!?1laQ#eXa85ga!LkN4&q1`}^F{#yl>muDz{lCQ
zE3LQE?wF`17_@Y%C+=MoxK44t0A@a2WXDO>PQsOPYcsl7K9HDGSh?HjW+pgaaa4a+
zVsxK|eFZ)e%rr^oPZ}J~H1Z|#nEUlF076_{TLCMl2_VmJLg9ZyKeecGRS(RA8m_Ka
z{O7-Scnuxjet01?=w*yC%X+nm)@5Cp#3rlbe2q6iSR6c6%CmN$_b7qRaZ<lFlfR5p
zjsoe+<vlwz#341>Ko<d`wpwf;%>kGw%Z}X5*~8#UfwRAhG_K`lRMp_T+HIMVY97j6
z_awu*3u0@RbST3O0iPi5y<g)`-aoy!wqsJIlcs2CXJ-d))}bMO3lLCE(!}NiaZrW`
zXskh=F1Yhz#}hth6KBWu*5&X*ZQ4@Nw}>M+J<}aCyFl$Z$QlhAA-!}+ezLU}l;{8$
zVX4Xo96ht*vzm@v4@MuEPtDmbg>7bc#amfPJ~e~;f{Kvnn4U59*ziLg+*8c_%2`jw
zE|aRTGjIR^oSqS$9u`gNu-TpwZ@Ni*?;iCPn&yAuA4<{K{g@;H(MfY(?-5{cUpZ5?
zh|$m#R)Um}&O0Ey-GM5f*aV=8HwAzYw&fA#2_w&9=ZUOte1~HZs$0Q9#M=jlh>;1k
zOlDw|%>PV^OR*?J@b5R1Q9dFJ_G`@K<af2rG_+?c=6hu5ohigv@dUeWaMZBgE36W_
zsBH81fPTd1ZfFNE6p#zYk@m$p6A(QIpSBJ#8z9Af8V<?@0Zh8-o<A_w+e_0@olvpX
z@8%O4N+BXp`~bQiEMATd5g9AsY>cPBeEQcob^UmC{riH>^VSn%Wz`kj1MYv8V&Y%$
znnQ~r03q!^ZvI~^Ie5)(I8$sJ_O_7{P^clYW#PcPZEWo3FL|Q4ktrm~(V=H<S`<L`
zgYTv7oL|~GX$&rTF(EZ~^>#(ydyRcN1BRqgMXg<_?%#%E?oornTVT;@YL3Lk4_8%H
zfx){}1U%>E-M!lOMPI+Rq}FxHk$(f9u<8u#um+>Qa{PK<n1e!N>{^+59<BCXu3h|Y
z;P<s_>(cAP_3Ea!#>dAKDLjSJBO(aB3OiBY%nIB9b<)S1X|AxXi=I(}WVcxbCal5-
z!Nw{1##;-UnZSCs-%N^k9UC1;W9tECn=9dA+fDx<MZ!0_Y!71F7B6FKu0F)Uzi3S>
zb-1$+J9(27K!;7rHm2%ERza=bD*!<o{=B50e%>7c;~xm}96&V=>#FL?N=a<;zOF8}
zD)vq+08+t~CvZ3%Kyh%K$F?z0ggF{rmW$~Swv=g^2M6bu1ONq2O~Dp%b*ctcFRZNQ
zUS0h;iS_$|Yluyu#=)n+=0b{r<5d_payNL+iSYSl&hP5eZte}{*1;Kzb3>m%V)COd
z*Mml%a-OZW?%Bl4tZw*^?YN`{HQwgrWir@Kq<Bcx87B4D4IF&p=l_RVZ&*9=pQP1)
z@rD1v8vfJAf4+g6`mZW<|2%~4`YM=*Pj6QIwo%m2EhaT>LBxBv@lJaVYWMV`(c4Aj
zcm1=KfwIW%+!;Fqz(GF^yB&;=ZOZ>EOC&shCe34~?6fSh=MLtaZ`C6yh<M~;q^2Qn
zMnUh^E!y%!sQH1-%!1ezoTbNLj2@JQy>7OdnL^UZkcjJHat!gf4|fFa9%geL_SLLs
z5Un|IL92tW-7tzaVW9Y_+Byd4*n}}1I%7^>Kwlz2ASIO292G#Z<yk3&-V&ny9o|>=
z^fi(ky-gi~vDpr#WZa&ean8hHQ?=r=pPXgJ(b^E-1_>r-M0}=4eo`G-glF?3>p{de
zuqv1e(mDY)$Sg{gJ5zCZO3jhS@_hzRKbCVI5abDEUlUzGZEdL<JpWuVuS%h_uQH3M
zp;BmMQp6{e`dRk)G7%Y2qZ#MVm}3Wp;gSGIgw^zjG|<Vkji23b8l#IT`#QRBa=?yZ
zLos{dV&IXoSOoKp>mrpr-gcQDFqqw0T6*OTHNTEGOxGd{4U6YY?*dGL!$t$1!?E0B
z+*j{NesmwCW6Km-BV8^R8y9^Z8InFQgqiA!fAgmYP*B#89Io-YxgD3U2wL@llMaWC
z98}>0v#c5h9D{#N_CEwCmVWwg)qFsIz~}P&r>{}73{Ik2!)&i3e-ZMvA@d;hqqGxK
z^~8Jwv9iv@6g#fJwK!tdWuUem0|VXh>HEgHzdsI#8<ba;s5NK<iD3lcgbD@~55HbP
z{~>dlOw9SXupVABPJQpz{^fEp&zcei;o{Q3UkygeArw}0>^qsM=3H^)E>$Co4++er
zP50q9<SeqXSGL(-SUU`GQ4*fLm-tA1rMNA?5#OPR-R<gWxGvHANA33nu3kRzh8&fT
z#<hJ}q+#B&`xHV;M1YxsB(5r;ia~0V8*!H~BZFXn+<%(-(Pkuuh4JasH(I`<55z&C
zbVPZNev^~oVi)lVgM6xaI(&zNy?dpI*rdKM(XN#^xgGa|=CVj!VTPA)<7~X6RznDV
z+1lXV{aj@gX9DuwoCt+!(rBzj6$|}UNnrg1ec>sqLnvNIp96t+Ir&HM*>Oj&hH<Yy
zv!F8=k^|gTi;GY;o1vPU-udSl&Uu31j&uDlW)E29-YsDeYFJxAW1qf-AJmN-d@7#R
z(EG)Knco==H|~DlCYdW;7)Q!oNLP?VzeYzV?ubwswR);#RWigmnt1bI!atGLAjQ>k
zRVtb@nDR!X$nm<1a|I89Zr75ngmhBht6eOa_DunpZ4~jgEE36Soox&t*PkCrks8)H
z71geD?PhZFjwXhZcp5TY>SM8}J0z+Uf@;juq_TFoRdNv=PblWA?nw;kP#p}V&P9Jb
zhMV#WK`(}h#MiVdrxSI*Fyrs3U(UzsI)0FEW7b|{*2=4YsKX(lG~3nUWKxH;CFX4U
z1UMyWBpz&j7;-DrKenrG&88UM!f<RX-8E05Mw=hqgv$92x|u*kVmggl67q_OCl5z7
zV3+3u^8))LqSN;h4YN_2Ja`4tYoBkDna4zY84K6^mi2(w_IV0@HcyT9OGfAC5rvz7
z4h+@E8!H^rh?8d97VvMLiRX}%y)abaedy8=_UE?C<7I3B!v99Wl<4Rdwa_Z{{*3_Z
z2o}EOHkDuEQb~~eR$x{QxI6BeEvk0BN7<9vs^84>;ILBr(cQwT{fAyu89>Fm0(Taz
zhPSBWl5~X|hnE-OsP><Bi+b3)bzBY~e$B;RGy;TwmWer!r+t1Uf!yVuEbHzI5GbV8
zenYIUNF1GU-7k(i2~jmgks}{qCZIf49RKR0*;eeu%cO-0f)B+@2!M`*#uR3;Gss0Y
zjt*$Kc{+g(ur3EZwR@g&fOJVP9P3DmAXykFXV83bt<^-Y=~*6GgZUD?nYM3_1|^UO
z%Wju@a*`n=Z$4WBfB@HWU*&bItHF9F_!oGZF7D}2R>Og`X^?<}CG;gitO!cAjZbK`
z5J71^Y0cZQBena?;)PdBDlUuYIB0T&!Ubh1l=jolHWRaCl@Y5OZ8@#>rt>ehaF;)+
z+4}H1Xcl?@M3Nz~F!2{f_d!DEPYfO9?`l3yk>pHSFO_gDHw(%8H3CB5!t@$?m*YmL
zYP_M?_$A+)3Vs5h*_SbIOX$1KVeSC8bupy-)g*<XU3sqtgO;mLu6;lEJ#J=rm7|ik
znYio=*cSkDzF7GvsqAcJ`|uB8k<{<!l!Vk<n*xI;NeUDI-~qoo$K?a%bOf_babo8%
zr8L|09%j+(Wwm=dmWN&;kAm3Q%&-BJ%#XPujxB(1hTtDO8RO*8dsy?BLpg4(-{*+*
zzRX}C@B~!kWn7zZzr|hk09^~~+mVw5lL9=e>Ge$#@wk*OQRazPgE!&D_rZ~>6D-I%
z3Ol>FGE>u}kbCiNH-|`6N559_^(e5gfD?cnio)tKaYvtFjvgvIh4B+27agTklMOil
zXqC&2Ub1XJTl$Qh-EOEr9k5@3>8h{F$uZrYt_L)$oV@%~c6N|4-Zz~L@CzIcN)@3Z
z0rzEaZzl&1F58^etn%2N2GGl`hor85%<Fe)>~TPeP8I9?%LIgNfH9cvDYR~c$oUH9
zc&&T<?7JBS0U${Jh7&~yU%XpZUcTWL-ErGx8X!u{;hfA-yxrd$0RWjF{@$rhlpnsP
zwc9Un9jEfjsV|pSCWu0J(6dUIt%jUW3mL4omV;7LLz$ppB><H&GBTh<s=_D=g<69H
z=|6yhyBe`=apU=O?sdRJ0>2!xPODHNy9CTW_0s}Ay+JvhiMke@#T@$i@Q@!O0_YFh
z1HhdGUR=BNhPc$2W_4NZ{<K6|2G!rRGe9sL#@mf*R!56MewH`A;`L7<KLu_CIwuQ6
z1;87A;S~dRg<9nF0=O@8j-G~NA0jckHh_@OUAhH8dPm@<t01`c6o{#bo4}2HPlv9|
z0*)4NE&yZ3dH@PnHL^dO)*~BGLx7V3Nzjl3rn==M(JzN1dg-_C=YlEg3#LXpOn7W`
za(b42U*?Qxz?`#`3Kpc<acw-EIGzZmX6IW5o%K1t*5g<dG=GInJ<RMHF<3i?8M`Hy
zo`#<8E;l8iD6D-#us6e;Yfx0V0fbge8Nt-I;RC+{;s<80-J-(H)fK4azfZ7Z>yQDg
zzN_K&x}5O=q^`vbk_27>6jXDXS=jl!rkRp}t3PPA9Suf-=vc%L^lMZM1}1%wPd+P~
za<X9oO!CN+{ETD$iN?~6kne|=j0PiWR1B#wiu+ykGkV#u`0+_$zZ-U8LqZa&FG;cf
zB*_wZ$lH!`HK{GVJ7^aPkrmr>!B3i5*@%6yll1I)5OO%m*sVUpHT0hS6q5x!A9Lvi
zpz7VZhqUp@w|)|guSLjAb_HJdK}k{OiLJl~Kaqz;uuR0(1v|qA-kPHWSR!;29c%bi
zy;tScg9NtIO2-R!@xd+=m_2(opEus6+igf5`s49W)#g{c--@WVHp%gVE}%Y(nxoDU
z^o(>nV1U5XJ)s)kJX~6`K$C}a+i-#Ebwxiy_!x?YO@X8FsTsnD9FHR^cn~W2##W-8
z_n|@IBzA3IYj7FLJ+hwB2HqMlkFY+kcd}7Goqu8*0*5HRNwFi;HWOCg_o`_aD<1Z-
zX-yRlAO8q;THRF1rs~~B5m~rv&z&?}P+Ax`3)Tyy;?)tgXM7z;D!&haIg)8IPCmT?
zfVjM#A@iF|vp<M*bMyTq1*R&e^stVO0%=fNFe$1yN4bqud_P#4zNCj6R9>zmzXHxd
z_>W?XiV@L=Xr&7c7Z8PlX8(*60F}r<sXp$SQw3EIAW;Rt_lMJhG~&Z!KBhJfMi+_Y
zm*)7QUP;=Kr-Vm`fWXcc)IWXylJ<Ir5iofEjP@RV!r~!dFG=Yc>et#cP-4l$#SIqV
z`?YiHS1Gwa)Am=<)cigw`lj^(R9vk#{4KCx6b(uv6yYSs-u#r%Uq@CoW1R{nz@sqw
z`1DO2AOrvfv-mTmpeglxsqL>5qrV>)s1Jk0DZk)>O%CqCRPV5!i1O?k0M{5x{%sD5
zD&AIJ4s!5yRKVNOK?RWT3B%f7+$98_Y*5a1VAR?C+>RWyGh*v7aeQ=^+fOI7d@%66
zMlApER`5X<VADZFLf_J<TL&f?YfjOpB*wn{%1b|(O(Nk_p$pSn5NlMTba0v4_*f&s
zQN{tzKJZT)s+CftTm2Ip`G@T1{wNL*>N<|6!UoMsGP5>Au|98;t^M(080FfuX0?+g
z^vaAYJ}o7o45--GvH&<(P!jF+gmawc=GGGjbzDEU2Wnv~CY4$~9%C1!IXMy$5rur<
z##*mZ;Gly$HSLLoWmW22l2Y=0dqZr59uv7x4|%dDfC`MjKoIxUlun5rpR+i?m?f1I
zjI?D3v~vl5VhsY|#ZT*})7b)E)-hMQ0)oVUd*DKuxx;8w?F}OTi)T{gA7CNk6B1+y
z?oQb93UGb{ZEw2dLy{CgCAso6cW|AvyAP=AkW+jfP;~L|<uvOyMVVz)k1P!_Gbd-H
z0Zu=tWOv783z3`KjB<W`l1ZX8Y6d}1Akh;L*aA591STgIu#}!V*?lnd54rvQ_Uz*A
zJ==5|c&r~Y_%mYt3~{_@J1*?@2M3um!3T6cA`&*+SA9IC_QrRAPR6i!vU(`>sG}Qy
z=6H4#OdFw_fmZcp&5p}TE9A+=8aAL<<+#@`>ndx9DlImkdVW3n73Ii37|8zYj*{7x
zkARKmr?hTTyf2Vg%#l|?Kt-_vyNEmLxmYd*)&^iha5|iWlM0YFdap+T@qT3{p9ph^
zKJxYf#zW^~Uga+Dw@I#;JF^eKFSELPpiXm5tp|?pvcUvA<Sv*ZyIn=NPrY_tBuaW$
zyom1+h_bxc!_U^NyaFH9T>z#N01jEq>??k@K)|Ss>c63kPB&kDWdp<fJ#efUI}a!i
zM?A;+Z@)vv0W<(e*d2*++@|iv4G<4u3jhL((xy~RV_u#reEFqt$9^pKv^^rS?%68<
zK;-1scz?9B%QR6tFL5c@m^i=P!DK5hTS5##Uyewp)Zz^@iv{n;()&U}TIV*|0<t<2
z>xOIrriv$bNqYKG>NgqX?u`I|kRKJH+BlIVAG^DpJ<mq5F)H9^fs^SAhknbl0ycuH
z29P?T8+2H(w-g?-!BYEVDZ~vpd?Et7Bcj@6fG-8S7$hgUGdw7%;7lBg=wqQsPX1?*
z8XUD60f)jd#58C-mCUc-x}u6Wn-xj3bcq3R7)WJ+k@;d(C#C`h9k8(6Y9Z$Z!h&_A
z%yREyYKCk%uv}YE=^%H)hhN5SMYVw2AEJuU%TYm#J=`0xSQmrU!M?dU>u`-d!s67>
z0Z>B-W}Dfwe$-*J_|33$dtk_a9-lBRqmvJv+|6LU(amcXM;;FGb&2A30a5>_bAL}}
zl?|CmNxq_FbuFGfU`^UltqM6nDKuoamTx3xuw&UBni~y?<h(1QF!NU_`esfcih97k
z00s)3)fWrJIC|<8!{Uc2!pA;@0eed<ev{@8f<$Xg`UVcp^$J<Zmv174q6aY@O0H`O
zPfVQC#j>(nIhFQBTibe~%Gq?&EPf&ku%o<3{-XysSn(8hh-B;Y?Q(#7L0B|G1bEIX
zi|qFh#Nr>e5B^M22GC9z(HybG-UH$eK#{sgd|18ceTykZkDI4=b3Yt(kh&|dvq;7U
z+-9oHZq2wmfArpC*O`A%?v$t}&kuqdLoe;CAZGR4nt2&4M>$lV2XD<-U%*%8!e_<t
zEOa>@?5b;+Ruazx*E29gvEMpTL)3}!!Yx<_K1KFuD9@T!vSti1l=^)-u@J*Zfb~Y%
z@W#h1#yi;qwM4Z^Q#?0@mpq?GwVksEhJ3jfqBtiW0?AQXa?rjZ;>D=#Tiv?NTv?jz
zJ696;YX?&4?QKQ@p%dX}RIj2Y`C64P18-E$OWJ64yzH}-l5P}el`13|m&L^qxE<p|
zG?30VFNfXt4$kK<Os?u7Y@cv;h#Pdh=EH4$lCim)oHqtt&r)kir+2P^x(+w=itnKU
zPLnLXWz?6f(vl*8zl+avUmNlJTn9Vo9Z%iQ-ZwV%MY&h%-Z6u0x8t+ilI51ss&dj>
zAq;O(dtCVTWq6Dl>P%PsoIomYHjxf$_}U^|A0GS4KH}J~ic>?bfx8#_kyc)gZr4?#
zHj8aUi+_wM=GA3c>+jbk$v-a=TkYdjD!5sv$1>JbB!8j%;5@W8H`ZF@k|(6dQo-gi
zd_urgSfPl@8ip@?(wY%pd@47$s7bxM+7|+<PEz3HCDlKh^FeTuJ_=D!qN&Ky*)C(w
z6R%cMZ;y!!5_(?<4UcMlCUbIY?T?wT-)&^qC1<np@$C*%i^xDAKi{)w&cg2X!8$(0
z><EpPmsW)Qv6?R@b&8yw%L{W9Pr6Q1CpW@eY5N!PYCC%ow=Zd@%&!5z*Mh({Cg$sx
zOjab;w6bBPvyiZ^i(`A|Uvj5CMiBNU_k%_BPf8i`Ocw|N-?L!*4+E5!T<9l=p*60_
zQ1a|E+Q6-qA0nHBVpM(2y=|eIj~+L^m|7!!{!B*HxZ(C?KR=7P-i!3K6Rv!{6-ew%
zDGZ^G45m*mRFKf9HI}ds^QYL6F|;FpOzXWxQ*rC^@i;i0DOi{3|DLSE4&uMa*Z&|k
z!TJTi{O9|U@dEBM_`yHl|1e&cfBx?e0SgCwF2DbsT>U>0X8-=@|Ivs4c*y^IAO7P-
n{&|Iedy#)y|KIeXvmApj-V$pc6+;`1eMea-MadFzqxb&<wnwrM

diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index 675c1b51a..ba0e2c61c 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -99,30 +99,31 @@ Data flow
 The data flow in Scrapy is controlled by the execution engine, and goes like
 this:
 
-1. The Engine gets the first URLs to crawl from the Spider and schedules them
-   in the Scheduler, as Requests.
+1. The Engine gets the first URLs to crawl from the Spider.
 
-2. The Engine asks the Scheduler for the next URLs to crawl.
+2. The Engine schedules the URLs in the Scheduler as Requests and asks for the
+   next URLs to crawl.
 
-3. The Scheduler returns the next URLs to crawl to the Engine and the Engine
-   sends them to the Downloader, passing through the Downloader Middleware
-   (request direction).
+3. The Scheduler returns the next URLs to crawl to the Engine.
 
-4. Once the page finishes downloading the Downloader generates a Response (with
+4. The Engine sends the URLs to the Downloader, passing through the
+   Downloader Middleware (request direction).
+
+5. Once the page finishes downloading the Downloader generates a Response (with
    that page) and sends it to the Engine, passing through the Downloader
    Middleware (response direction).
 
-5. The Engine receives the Response from the Downloader and sends it to the
+6. The Engine receives the Response from the Downloader and sends it to the
    Spider for processing, passing through the Spider Middleware (input direction).
 
-6. The Spider processes the Response and returns scraped items and new Requests
-   (to follow) to the Engine.
+7. The Spider processes the Response and returns scraped items and new Requests
+   (to follow) to the Engine, passing through the Spider Middleware
+   (output direction).
 
-7. The Engine passes scraped items and new Requests returned by a spider
-   through Spider Middleware (output direction), and then sends processed
-   items to Item Pipelines and processed Requests to the Scheduler.
+8. The Engine sends processed items to Item Pipelines and processed Requests to
+   the Scheduler.
 
-8. The process repeats (from step 1) until there are no more requests from the
+9. The process repeats (from step 1) until there are no more requests from the
    Scheduler.
 
 Event-driven networking

From 625c69fdc73c74cdb4172c4642842ed4f6f31432 Mon Sep 17 00:00:00 2001
From: Joakim Uddholm <joakim@uddholm.com>
Date: Mon, 8 Aug 2016 14:32:53 +0200
Subject: [PATCH 1036/2974] Fixed typo in error message when selecting a
 callback method for the parse command.

---
 scrapy/commands/parse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 0185bcabd..6a8978415 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -175,7 +175,7 @@ class Command(ScrapyCommand):
                     cb = cb_method
                 else:
                     logger.error('Cannot find callback %(callback)r in spider: %(spider)s',
-                                 {'callback': callback, 'spider': spider.name})
+                                 {'callback': cb, 'spider': spider.name})
                     return
 
             # parse items and requests

From d9437fd3d97430d39ea408bca7c200ec042e1ab7 Mon Sep 17 00:00:00 2001
From: rootavish <rootavish@gmail.com>
Date: Wed, 6 Apr 2016 08:47:06 +0530
Subject: [PATCH 1037/2974] Modifying existing gzip read failure recovery
 mechanism to patch read for broken archives

---
 scrapy/utils/gz.py                               |   2 +-
 .../compressed/unexpected-eof-output.txt         |   1 +
 tests/sample_data/compressed/unexpected-eof.gz   | Bin 0 -> 5134 bytes
 tests/test_utils_gz.py                           |  10 ++++++++++
 4 files changed, 12 insertions(+), 1 deletion(-)
 create mode 100644 tests/sample_data/compressed/unexpected-eof-output.txt
 create mode 100644 tests/sample_data/compressed/unexpected-eof.gz

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index afc7ed128..73c2eb73b 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -43,7 +43,7 @@ def gunzip(data):
             # contains the whole page content
             if output or getattr(f, 'extrabuf', None):
                 try:
-                    output += f.extrabuf
+                    output += f.extrabuf[-f.extrasize:]
                 finally:
                     break
             else:
diff --git a/tests/sample_data/compressed/unexpected-eof-output.txt b/tests/sample_data/compressed/unexpected-eof-output.txt
new file mode 100644
index 000000000..3b201255f
--- /dev/null
+++ b/tests/sample_data/compressed/unexpected-eof-output.txt
@@ -0,0 +1 @@
+document.write('¡¡¡¡¡°ÎҽкéС±¦£¬ÊÇÒ»¸ö·Ç³£¸ß¶Ë·Ç³£Å£±ÆµÄÖ°Òµ¶þÊÀ×棬ȴÔÚÒ»¸öÔºڷç¸ßÉìÊÖ²»¼ûÎåÖ¸µÄÒ¹ÍíÎÒĪÃûÆäÃîµÄ¾Í±»ÀÏ°Ö¸øÒ»½Åõßµ½ÁËÕâô¸öµØ·½¡­¡­¡±<p>¡¡¡¡¡°ÔÚ´©Ô½µÄÄÇÒ»¿Ì£¬ÎÒÏëÆðÁËÔÚѧУѧµÄÓïÊýÍâÎﻯÉú¡ª¡ªÎÒ¶¼²»»á¡­¡­¡±<p>¡¡¡¡¡­¡­<p>¡¡¡¡ÌìÔ˴󽣬³àÔƹú»Ê¹¬Ö®ÖУ¬»ÊµÛºéÎÄÇåµÄÊé·¿ÐþÇå¸óÄÚ¡£<p>¡¡¡¡¡°±ÝÏ£¬¡±Ò»Ãû»¤ÎÀ´Ò´ÒÀ´±¨£º¡°Íâ³öÔÆÓÎÊ®¶þÄêµÄ¹úʦ»ØÀ´ÁË£¬ÏÖÔÚÕýÔÚÐþÑôµîÄڵȺò±ÝÏ¡£¡±<p>¡¡¡¡¡°Å¶£¿¹úʦ»ØÀ´ÁË£¿¡±Ìýµ½»¤ÎÀµÄ»°£¬»ÊµÛºéÎÄÇåÏÈÊÇһ㶣¬Ëæºó·ÅÏÂÊÖÖеÄ×àÕÛ£¬ÆðÉíЦµÀ£º¡°Í¨±¨Ò»Éù£¬Ëµë޾͵½¡£¡±<p>¡¡¡¡ºéÎÄÇ壬ËÄÊ®°ËË꣬³àÔƹúµÚÊ®°ËÈλʵۣ¬ÉúµÄ·½Ãæ´ó¶ú£¬ÏàòÌÃÌã¬ÓÈÆä°®ÃñÈç×Ó£¬ÉîÊÜ°ÙÐÕ°®´÷£¬ËãµÃÉÏÊÇÖιúÓз½µÄÒ»´úÃ÷¾ý¡£<p>¡¡¡¡¡°ÊÇ¡£¡±»¤ÎÀ½ÓÁÕâ¾Í×¼±¸À뿪£¬²»¹ý×ßÖ®ºóºöÈ»¶ÙÁËÏ£¬ËæºóµÀ£º¡°¶ÔÁË£¬±ÝÏ£¬³ýÁ˹úʦ´óÈËÒÔ¼°ËûµÄÅ®¶ùÔƲÊóïС½ãÖ®Í⣬»¹ÓÐÃûÄêÇáÈË¡£¾ÝÊôÏ°µÖдò̽£¬´ËÈ˾Ý˵ÊÇÀ×Òô¸ó¶þ³¤ÀÏÒ¶·ÉºèÖ®×ÓÒ¶½úÄþ¡£¡±<p>¡¡¡¡¡°À×Òô¸ó¶þ³¤ÀÏÖ®×Ó£¿¡±ºéÎÄÇåÖåÁËÖåü£¬ËæºóÇáÇá»ÓÁË»ÓÊÖ£º¡°ÄãÏÈÏÂÈ¥°É¡£¡±<p>¡¡¡¡¡°ÊÇ¡£¡±<p>¡¡¡¡»¤ÎÀÏÂÈ¥ºó£¬ºéÎÄÇåÏëÁËÏ룬²»Ñ¡ÁúÅÛ£¬·´¶øÌØÒâ´©ÉÏÒ»Éí±ã×°£¬Ö®ºóÃþÁËÃþÏ°ÍÉϵĺúÐ룬à«à«µÀ£º¡°¹úʦÔÆÓÎÊ®¶þÄ꣬Èç½ñ»ØÀ´£¬Äª²»ÊÇÒòΪÄǸö»éÔ¼£¿¶÷£¬°´ÕÕÄê¼ÍÀ´Ë㣬ÏëÀ´Ó¦¸ÃÊÇÁË¡£Ö»²»¹ý£¬Õâ¸öÀ×Òô¸ó¶þ³¤ÀϵĶù×Ó£¬ÓÖÀ´×öʲô£¿¡±<p>¡¡¡¡¡­¡­<p>¡¡¡¡ÐþÑôÒéÊ´óµîÖ®ÄÚ¡£<p>¡¡¡¡¡°¹þ¹þ£¬Ê®¶þÄê²»¼û£¬¹úʦ·ç²ÉÓÈʤÍùÎô£¬Ïë±ØÎäѧÓÖÓнø¾³£¬¿Éϲ¿ÉºØ°¡¡£¡±»ÊµÛºéÎÄÇåÒ»½øÒéÊ´óµî£¬µÚÒ»ÑÛ¿´µ½µîÄÚÒ»ÃûÇàÅÛÄÐ×Ó£¬Á¢Ê±±ã¼±ÐÐÁ½²½£¬Á½ÈËË«ÊÖÏàÎÕ£¬¹þ¹þ´óЦ£¬ºéÎÄÇåµÀ£º¡°ÕâÒ»±ðÊ®¶þÄ꣬¿ÉÕæÊÇÏëÉ·ÎÒÒ²£¡¡±<p>¡¡¡¡ÄÇÖÐÄêÄÐ×Ó¿´Äê¼Í´óÔ¼ÎåÊ®À´Ë꣬ÊÖÖÐÒ»°Ñ¶ìëÓðÉÈ£¬Í·ÉÏÁ½³ßÇà½í£¬ò¢ÏÂÈý縳¤Ð룬ÉúµÄÏÉ·çµÀ¹Ç£¬¹ËÅÎÖ®¼ä¼«ÓÐÆøÊÆ¡£<p>¡¡¡¡ÄܽкéÎÄÇåÕâλ»ÊµÛÈç´Ë¶Ô´ý£¬´ËÈËÕýÊdzàÔƹú»¤¹úÎäÉñ£¬±»×ðΪ¹úʦµÄÆßÐÇÎäÕߣ¬ÔÆÊ¥Ðù£¡<p>¡¡¡¡¡°±ÝÏ¿ÍÆø¡£¡±ÉíΪ»¤¹úÎäÉñ£¬ÔÆÊ¥ÐùÓë»ÊµÛºéÎÄÇåÖ®¼äµÄ¹ØϵһÏò¼«ÎªÁ¼ºÃ£¬ÏòÀ´Æ½Æðƽ×ø£¬Ò²²»ÒâÍ⣬΢ЦµÀ£º¡°´Ë·¬ÔÆijÔÆÓιéÀ´£¬ÊµÊÇΪÁËСŮ²ÊóïµÄ»éʶøÀ´¡£¡±<p>¡¡¡¡¹ûÈ»£¡<p>¡¡¡¡ÌýÁËÔÆÊ¥ÐùµÄ»°£¬ºéÎÄÇåµ±¼´Ïò֮ǰһֱվÔÚÔÆÊ¥Ðù²»Ô¶´¦µÄÒ»ÃûÉÙÅ®¿´È¥¡£<p>¡¡¡¡ÄÇÉÙÅ®´©×ÅÒ»Éíµ­ÂÌÉ«µÄ³¤È¹£¬´óÔ¼¶þ°ËÄ껪£¬Ò»Ë«ËÆË®µÄíø×Ó£¬Áé¹âÉÁ¶¯¡£·ôÈçÄýÖ¬£¬Ñ©°×ÖÐ͸×ŷۺ죬ËƺõÄÜÅ¡³öË®À´¡£ÓñÊÖʮָÏËÏË£¬ÈáÈôÎ޹ǣ¬ÇáÇáµÄÎÕ×Åһ֧ͨÌå±ÌÂ̵ÄÓñóһͷ¼°ÑüÇàË¿Ëæ·çÎ趯£¬ÏÉ×Ó°ãÉñ²ÊÍÑËס£<p>¡¡¡¡¡°ÔƲÊó¼û¹ý±ÝÏ¡£¡±ÑÛ¼ûºéÎÄÇåÍûÀ´£¬ÉÙÅ®µ±¼´ÐÐÀñ¡£<p>¡¡¡¡¾øÃÀµÄÈË£¬¾øºÃµÄ³öÉí¡£<p>¡¡¡¡ÕýÊÇ»¤¹úÎäÉñÔÆÊ¥ÐùΨһµÄÕÆÉÏÃ÷Ö飬ÔƲÊó<p>¡¡¡¡¡°°¥Ñ½£¬ÕâÊ®¶þÄêû¼û£¬²Êóï¾ÓÈ»ÒѾ­³öÂäµÄÈç´ËÃÀÀö£¡¡±ºéÎÄÇå֮ǰ¾Í×¢Òâµ½ÁËÕâ¾øÃÀµÄÉÙÅ®£¬´ËʱÌýËýÈ·ÈÏ£¬Ô½·¢ÐÀϲ£¬Ð¦µÀ£º¡°´ó¼Ò¶¼²»ÊÇÍâÈË£¬½ÐÎÒÊåÊå°É£¬ÕâÑùÏÔµÃÇ×½üЩ¡£¡±<p>¡¡¡¡ÔƲÊóïÇáÇáµÄ¡°¶÷¡±ÁËÒ»Éù£¬ËãÊÇ´ðÓ¦¡£<p>¡¡¡¡ÔÆÊ¥ÐùÁ½È˺®êÑÍê±Ï£¬ºéÎÄÇå×îºó¿´ÏòÄÇÃû»¤ÎÀ¿ÚÖеÄÄêÇáÈË£¬Î¢Ð¦×ÅÎʵÀ£º¡°ÕâλÉÙÏÀÊÇ¡­¡­¡±<p>¡¡¡¡ÌýµÃºéÎÄÇåÎÊÆð£¬ÄêÇáÈ˸Ͻô±§È­ÐÐÀñ£º¡°Áù´óÕýͳ֮һ£¬À×Òô¸ó×ù϶þ³¤ÀÏÖ®×Ó£¬Ò¶½úÄþ¼û¹ý±ÝÏ¡£¡±ÕâÒ¶½úÄþÉúµÄÒDZíÌÃÌ㬽£Ã¼ÐÇÄ¿£¬Ò»Éí°×ÒÂÈçÑ©£¬¶ËµÄÒ»¸±ºÃÂôÏà¡£Ö»²»¹ýËûËäÈ»ÊÇÔÚÏòºéÎÄÇåÐÐÀñ£¬¿ÉÊÇÑÛÉñÈ´²»Ê±ÍµÍµ¿´ÏòÔƲÊóÐÐΪ¹îÒì¡£<p>¡¡¡¡¡°¶÷£¬Ó¢ÐÛ³öÉÙÄ꣬²»´í£¬²»´í¡£¡±Ò¶½úÄþµÄÑÛÉñ±»ºéÎÄÇå¿´ÔÚÑÛÀ˲Ϣ֮¼äºéÎÄÇå±ãÃ÷°×¹ýÀ´£¬Ëûüͷ΢΢ÖåÁËÖ壬²»¹ýºÜ¿ìÊæÕ¹¿ªÀ´£¬ÇáÇáµãÁ˵ãÍ·£¬Í¬Ê±ÐÄÖÐÈ´°µµÀ£º¡°²ÊóïÓ×Äê±ãÒòΪÌ츳¼«¼Ñ£¬ËÍÍùÁù´óÕýͳ֮һµÄÀ×Òô¸óÐÞÁ¶£¬Èç½ñËý»ØÀ´£¬È´¸úÀ´ÁËÕâôһ¸öÈË£¬¿´Ñù×Ó½ñÌìµÄÊÂÇ飬ÅÂÊÇûÄÇô¼òµ¥¡£¡±<p>¡¡¡¡¡°ºÇºÇ£¬±ÝÏ£¬¡±Èç½ñÈËÒѼû¹ý£¬ÔÆÊ¥ÐùÓðÉÈÇáÒ¡£¬Ð¡ÉùÌáÐѵÀ£º¡°°´ÕÕµ±ÄêµÄ»éÔ¼£¬²»Öª¾Å»Ê×Ó£¬¿É»¹°²ºÃ£¿¡±<p>¡¡¡¡³àÔƹú£¬Ã¿Èλʵۣ¬Óëµ±´ú»¤¹úÎäÉñ£¬±ØÐëÒª½á³É»éÔ¼£¬ÓÃÒÔÀ­½üË«·½Ö®¼äµÄÓѺùØϵ¡£<p>¡¡¡¡Èç¹û»¤¹úÎäÉñÑ¡µÄÊÇÄУ¬ÄǻʵÛÕâ±ß£¬±ãÑ¡Ò»¹«Ö÷ϼޣ»¶øÈôÊÇ»¤¹úÎäÉñÕâ±ßÊÇÅ®£¬ÄǻʵÛÕâ±ß£¬ÔòҪѡһÃû»Ê×ÓÓ­È¢´ËÅ®¹ýÃÅ¡£<p>¡¡¡¡±¾Èλ¤¹úÎäÉñÔÆÊ¥ÐùÏ¥ÏÂÖ»ÓÐһŮ£¬¹Ê¶ø£¬ºéÎÄÇå±ãÐèҪѡ³öÒ»Ãû»Ê×ÓÀ´ÓëÖ®³É»é¡£¶øÕâÃû±»Ñ¡¶¨µÄ»Ê×Ó£¬ÕýÊÇÓëÔƲÊóïÄêÁäÏ൱£¬×îÊܺéÎÄÇåϲ°®ÐÄÌ۵ľŻÊ×Ó£¬ºéС±¦£¡<p>¡¡¡¡Ö®ËùÒԽкéС±¦£¬ÊÇÒòΪ¾Å»Ê×Ó³öÉúʱÉí×Ó¹ÇÏà¶Ôµ¥±¡£¬ÓÖÊÇ×îСµÄ±¦±´¸í´ñ£¬ËùÒÔÆðÃûС±¦£¬Òâ˼ÊÇÏ£ÍûËûÏñ´«¼Ò±¦Ò»Ñù¼áͦ¡ª¡ªËµÆðÀ´£¬Õâ¸öÃû×Ö»¹Êǵ±ÄêÔÆÊ¥ÐùÇ××ÔÈ¡µÄ¡£<p>¡¡¡¡¡°°¦£¬ÎÒ¼ÒС±¦Õ⺢×Ó£¬¡±ÌáÆð×Ô¼ºÕâ¸ö×îÌÛ°®È´Ò²ÊÇ×î½ÐÈËÍ·Ì۵ĵھÅ×Ó£¬ºéÎÄÇåÒ²ÊÇÓÐЩÓôÃÆ£º¡°ËûÒ»Ììµ¹ÊÇͦ¿ªÐĵģ¬¾ÍÊÇÓÐЩ²»ÎñÕýÒµ£¬½ÐÈËÍ·ÌÛ¡£¡±<p>¡¡¡¡¡°Å¶£¿²»ÎñÕýÒµ£¿¡±ÌýÁËÕâ»°£¬ÔÆÊ¥Ðù¶ÙʱÖåüµÀ£º¡°Äª·Çµ±ÄêÎÒÁô¸øËûµÄÄDz¿¡¶Á÷ÔÆӥצ¹¦¡·£¬ËûÒ»Ö±¶¼Ã»ÓÐÐÞÁ¶£¿¡±<p>¡¡¡¡ºéС±¦ÌåÖÊÈõ£¬ÔÆÊ¥ÐùÊÇÖªµÀµÄ£¬Ëû³öÓÎÇ°ÌØÒâÁô¸øºéС±¦Ò»²¿¹¦·¨Ãؼ®½ÐËûÐÞÁ¶£¬ÒÔ±ã½ÐËûÇ¿Éí½¡Ì壬¿ÉÊÇÌýºéÎÄÇåµÄ»°£¬Õ⹦·¨£¬ËûûÁ·£¿<p>¡¡¡¡¡°°¦£¬Ò»ÑÔÄѾ¡°¡£¬¡±ºéÎÄÇ峤̾¿ÚÆø£¬ÎÞÄεÄÒ¡ÁËÒ¡Í·£¬µÀ£º¡°¹úʦ»¹ÊÇÓëÎÒÒ»µÀ£¬ÏÈÈ¥¿´¿´ÔÙ˵°É¡£¡±<p>¡¡¡¡ÕâÊÇÔõô»ØÊ£¬¿´Ñù×ÓÊÂÇéºÃÏñ²»´ó¶ÔÍ·¡£<p>¡¡¡¡ÔÆÊ¥ÐùÓëÅ®¶ùÔƲÊóï¶ÔÊÓÒ»ÑÛ£¬Ò»ÆðµãÍ·£º¡°ºÃ¡£¡±Õâ¾Í³öÃÅ£¬Ç°Íù¾Å»Ê×ÓËùÔÚµÄÐþÏö¸ó¡£Ò»Â·ÉÏ£¬Ò¶½úÄþ½ô¸úÔƲÊóïÉí±ß£¬ºéÎÄÇå¿´ÔÚÑÛÖУ¬ÐÄÏÂÒѾ­ÁËÈ»¡£<p>¡¡¡¡½á¹û¸ÕÒ»½øÐþÏö¸óµÄ´óÃÅ£¬¾Í¿´µ½ÎÞÊýµÄÆÍÈËÄÃ×ÅÌúDzù×Óһ··É±¼£¬Óë´ËͬʱһÕó¶ñ³ôζ´«À´£¬ºéÎÄÇåµÈËÄÈËŤͷ¿´È¥£¬¾ÓÈ»¿´µ½¼¸ÃûÆÍÈËÍƹýÀ´Ò»Á¾·à³µ£¡<p>¡¡¡¡Ò»ÏòÏÉ·çµÀ¹ÇÆÄÓеãƮƮºõÓ𻯵ÇÏÉζµÀµÄÔÆÊ¥ÐùÖ±½Ó¾Í¿´´ôÁËÑÛ£º¡°Ê²Ã´Çé¿ö£¿¡±<p>¡¡¡¡ÕýËùνʳö·´³£±ØÓÐÑý£¬Ò»½øÞͿ´µ½Õâô¸öÇé¿ö£¬ËÄÈË·´µ¹²»×ż±ÁË£¬ºéÎÄÇåÖÆÖ¹ÁËÓûÒªÇë°²µÄÆÍÈË£¬×öÁ˸öàäÉùµÄÊÖÊÆ£¬Ëæºó¸úÔÚºóÃ棬¿´¿´Õâµ½µ×ÊÇÔÚ¸ãʲôÐþ»ú¡£<p>¡¡¡¡Ò»Â·Ç°ÐУ¬ºÜ¿ì£¬ËÄÈ˾ÍÌýµ½Ò»°ÑÂÔ΢ÓÐЩÄêÇáµÄÉùÒôÏìÆ𡪡ª<p>¡¡¡¡¡°ÚÀ£¬ÊÖ½ÅÒ»¶¨Òª×Ðϸµã£¬¶¼Ìýµ½ÁËÂð£¿Õâ¿ÉÊÇ×î¹Ø¼üµÄÒ»²½£¬²»ÄÜÓаëµã²î´í£¡¡±ÄÇÉùÒôÌýÆðÀ´´óÔ¼¶þÊ®À´Ë꣬ËäÈ»²»ËãµÍ³ÁÐÛºñ£¬¿ÉÊÇÌýÔÚ¶úÖе¹Ò²Í¦Êæ·þ£¬ºéÎÄÇåЦµÀ£º¡°Ð¡±¦Õ⺢×Ó£¬²»ÖªÔÚ¸ãʲô£¬ÎÒÃÇÇÒÏÈ¿´ËûÒ»¿´£¡¡±<p>¡¡¡¡ÔÆÊ¥ÐùºÍÔƲÊóïÒ²ÊǺÃÆ棬Ҷ½úÄþÔò¹´Á˹´×ì½Ç£¬³¶³öһ˿²»Ð¼µÄЦÒâ¡£ÈýÈ˸úºéÎÄÇåÒ»ÆðÈÌס³ô棬ÕÒÁ˸öÒþ±ÎµÄµØ·½£¬ÍµÍµ¹Û¿´¡£<p>¡¡¡¡ÕâÀïµØ´¦ÐþÏö¸óµÄºóÔº£¬ÖÜΧÂÌÊ÷´Ð´Ð£¬Ð¡ÇÅÁ÷Ë®£¬·ç¾°¼«¼Ñ¡£¶ø¾Å»Ê×ÓºéС±¦£¬ÔòÕýÖ¸»Ó×ÅÆÍÈËÃÇÔÚµØÏÂÍÚ×Åʲô£¬±ßÉÏ»¹·Å×Ų»ÉÙµÄÌú¹Ü¡¢ÌÕ´ÉƬ¡¢Ä¾Í·Ö®ÀàµÄÆ÷Îï¡£×îÆæÃîµÄÊÇ£¬¾ÍÔÚºéС±¦µÄ²»Ô¶´¦£¬»¹°Ú×ÅÒ»¸öÆæ¹ÖµÄ°×É«ÌÕ´ÉÔì¾ÍµÄ¶«Î÷£¡<p>¡¡¡¡ÄǸö¶«Î÷ÓÐ×ÅÒ»¸öÍÖÔ²ÐεĿÚ×Ó£¬ÏÂÃæÊǵ××ù£¬ºóÃ滹´ø×ÅÒ»¸ö·½·½ÕýÕýµÄÏä×Ó£¬²»ÖªºÎÓá£<p>¡¡¡¡²»¹ýºÜ¿ìËÄÈ˾ÍÖªµÀÁË¡£Ö»ÌýºéС±¦ËµµÀ£º¡°¶¼Ð¡ÐÄЩ¹þ£¬Õâ¿ÉÊÇÎÒÐÂÑÐÖƳöÀ´µÄÂíÍ°£¬ºÜ½ð¹óµÄ£¬ÌÕ´É×öµÄ£¬±ðÅö»µÁË£¡¡±<p>¡¡¡¡ÂíÍ°£¿ÄÇÊÇʲô¶«Î÷£¿<p>¡¡¡¡Ç¿ÈÌסÐÄÖкÃÆ棬ºéÎÄÇåËÄÈ˼ÌÐø͵¿´¡£<p>¡¡¡¡¡°¶Ô¶Ô¶Ô£¬¾ÍÕâÑù£¬¼ÌÐøÍÚ£¬¶÷£¬ÔÙÍÚÉîµã£¬´óµã£¬ÒªÖ±ÉÏֱϵģ¡¡±ºéС±¦¼ÌÐøÖ¸»Ó£¬È´²»ÏëÆÍÈËÃÇÕýÍÚ×ÅÄØ£¬ºöȻһÃûÆÍÈË´óÉùµÀ£º¡°¾Å»Ê×Ó£¬ÎÒÍÚ³öÀ´Ò»¸ö¶«Î÷£¡¡±<p>¡¡¡¡ºéС±¦ºÃÆæµÀ£º¡°Ê²Ã´¶«Î÷£¿¸øÎÒ¿´¿´£¡¡±<p>¡¡¡¡¡°ÔÚÕâÄØ¡£¡±ÄÇÆÍÈ˸Ͻô°ÑÍÚµ½µÄ¶«Î÷½»µ½ºéС±¦Õâλ¾Å»Ê×ÓµÄÊÖÉÏ¡£¶«Î÷ºÜÔ࣬ÉÏÃæÕ´ÂúÄàÍÁ£¬ºéС±¦Ò²²»½éÒ⣬Éì³öÐä×ÓÖ±½Ó²ÁÁ˸ɾ»£¬Ö®ºóÖÕÓÚ¿´µ½Õâµ½µ×ÊǸöʲô¶«Î÷¡£<p>¡¡¡¡ÄÇÊÇÒ»¸ö·Ç³£Æ¯ÁÁµÄÊÖïí£¬ÉÏÃæÏâןì³È»ÆÂÌÇàÀ¶×Ϻڰ×Ò»¹²¾Å¸öÑÕÉ«µÄÓñʯ£¬ÄÇÓñʯÉÏ»¹É¢·¢×Åһ˿µ­µ­µÄÎíÆø£¬ÃþÔÚÊÖÀﻹÓÐЩůůµÄ£¬·Ç³£ÉñÆæ¡£<p>¡¡¡¡¡°ºÃ¶«Î÷£¡¡±ºéС±¦´óϲ£¬Ö±½Ó°ÑÄÇ´®Öé×Ó´÷ÊÖÍóÉÏ£¬Ð¦µÀ£º¡°Ã¿ÈËÉͶþÁ½£¬»ØÍ·×Ô¼ºÈ¥Á죡¡±<p>¡¡¡¡ÖÚÆÍÈ˸ж÷´÷µÂ£º¡°Ð»¾Å»Ê×Ó£¡¡±È»ºó¼ÌÐø¿ªÍÚ£¡<p>¡¡¡¡ÕâÒ»ÍÚ¾Í×ã×ãÍÚÁËС°ë¸öʱ³½£¬ÖÕÓÚ´ó¿ÓÍں㬾ŻÊ×ÓÏȽÐÈË°ÑÀïÃæÓôóÔ¼Áù³ß×óÓÒ³¤¶ÌµÄÌÕ´ÉƬµæºÃ£¬È»ºóϹÜÂñÍÁ£¬ÔÙÖ®ºó°Ñ·à³µÍÆÀ´£¬ºéС±¦ÑÚ×űÇ×Ó½ÐÆÍÈË°Ñ·à³µÀïÃæÄǶñ³ôµÄ±ãÄçÖ®ÎïÈ«²¿µ¹ÁËÏÂÈ¥¡ª¡ªÕâһϳ¡Éϸü³ôÁË¡­¡­<p>¡¡¡¡ÕâЩ»î¸ÉÍ֮꣬ºó¾ÍÊÇÉÏ°å¹Ì¶¨£¬×îºóÓÖ°ÑÂíÍ°¹Ì¶¨ºÃ£¬Ò»Õó΢·ç´µ¹ý£¬×ÜËãÒ»Çи㶨¡£<p>¡¡¡¡¡°Ð¡±¦£¬ÄãÕâ¹Äµ·Ê²Ã´ÄØ£¿¡±ÑÛ¿´ÕâʱºòÆÍÈËÒѾ­¿ªÊ¼ÔÚÖÜΧ¸ÇС·¿×Ó£¬ºéÎÄÇåËÄÈËÕâ²Å×ß³öÀ´£¬ºéÎÄÇåÖ¸×ÅÄǸöÂíÍ°£¬ÎʵÀ£º¡°ÕⶫÎ÷¸ÉʲôµÄ£¿¡±<p>¡¡¡¡ÑÛ¼û¸¸»ÊÀ´ÎÊ£¬ºéС±¦ºÙºÙЦÁËЦ£¬Óе㲻ºÃÒâ˼£¬´ðµÀ£º¡°ÕⶫÎ÷ÎÒ¹ÜËû½Ð³éË®ÂíÍ°£¬¿ÉÓÐÓÃÁËÄØ£¬È¥Ã©²ÞµÄʱºòÖ»Òª×øÔÚÕâÉÏÃæ·½±ã£¬µÈ·½±ãÍêÁË£¬Ò»°´Õ⿪¹Ø£¬ÕâºóÃæË®ÏäÀïµÄË®¾ÍÁ÷ÏÂÀ´£¬È»ºó¾Í³å¸É¾»ÁË£¬ºÜʵÓõģ¡¡±<p>¡¡¡¡¡°Äã¹Äµ·Õâô°ëÌ죬¾ÍÊÇŪÁ˸öé²Þ³öÀ´£¿£¡¡±ºéÎÄÇåËÄÈËÈ«²¿¶¼¾ª´ôÁË£¡<p>¡¡¡¡ÓÈÆäÊÇÔÆÊ¥ÐùºÍÔƲÊó¸üÊÇÌýµÄÄ¿µÉ¿Ú´ô£¡Ò¶½úÄþÔòÊÇŤ¹ýÉíÈ¥£¬¼ç°ò²»×¡¶¶¶¯¡£<p>¡¡¡¡Õâ¾Å»Ê×Ó֮ǰºéÎÄÇå¾Í˵Ëû²»ÎñÕýÒµ£¬ÏÖÔÚ¿´À´£¬»¹ÕæÊÇ£¡<p>¡¡¡¡¡°°¡£¬µ±È»£¬²»¹âÊÇé²Þ£¬¡±ºéС±¦·Ç³£µÃÒ⣬µÀ£º¡°ÕâÊÇÎÒµÄÒ»Ïîз¢Ã÷£¬ÎÒ¹ÜÕâ¸ö½ÐÕÓÆø³Ø£¡¼òµ¥À´Ëµ£¬¾ÍÊǵ±ÕâЩ¡­¡­¶÷¶÷£¬»Æ°×Ö®ÎïÔÚÕâ±ã³ØÀï·¢½ÍÖ®ºó£¬»á²úÉúÒ»ÖÖÆøÌ壬¿ÉÒÔÓÃÀ´µã»ðÔì·¹Óõģ¬·½±ã£¬»·±££¬·Ç³£ºÃÓã¡¡±<p>¡¡¡¡ÔÆÊ¥ÐùÒѾ­ÓеãÌý²»ÏÂÈ¥ÁË¡ª¡ª²»ºÃºÃÁ·Î䣬¸ãÕâЩÓеÄûµÄµÄÂÒÆß°ËÔãµÄ¶«Î÷£¬ÓÐÓã¿<p>¡¡¡¡¡°¶÷£¬Ð¡±¦°¡£¬¡±ÔÆÊ¥ÐùÉÏÇ°Ò»²½£º¡°ÎÒµ±³õÁô¸øÄãµÄÄDz¿¡¶Á÷ÔÆӥצ¹¦¡·£¬Äã¿ÉÔøÏ°Á·£¿¡±<p>¡¡¡¡¡°ÄúÊÇÔƲ®²®£¿¡±¿´µ½ÔÆÊ¥Ðù£¬ºéС±¦¼±Ã¦ÉÏÇ°¼ûÀñ£º¡°ÔƲ®²®£¬ºÃ¾Ãû¼ûÀ²£¡Äú¸øÎÒÁôµÄ¹¦·¨£¬ÎÒÊÇÒ»¶¨ÒªÁ·µÄѽ£¡¡±<p>¡¡¡¡¡°ÄÇÁ·µÄÈçºÎÁË£¿¸øÎÒʹÀ´¿´¿´£¿¡±ÔÆÊ¥Ðù¼±ÎÊ¡£<p>¡¡¡¡¡°°¡£¬ÄÇÅÂÊDz»ÐУ¬¡±ºéС±¦ÂúÁ³²»ºÃÒâ˼µÄ»ØµÀ£º¡°Á·ÊÇÁ·ÁË£¬¾ÍÊÇûÁ·³É£¡¡±<p>¡¡¡¡ÕâʱºòÒ»ÅÔµÄÔƲÊóﶼÓеãÌý²»ÏÂÈ¥ÁË£¬ÖåÁËÖåü£¬Å¤¹ýÉí×Ó¡£Ò¶½úÄþÔò¡°àÍ¡±µÄÒ»ÉùЦÁ˳öÀ´£¬´Õµ½ÔƲÊóï¶ú±ß£¬Ð¡ÉùµÀ£º¡°¿´À´ÄãµÄÕâδ»é·ò£¬¹ûÈ»¸ú´«ËµÖÐÒ»Ñù£¬²»Ö»²»ÎñÕýÒµ£¬»¹ÊǸö·Ï²Ä£¡¡±<p>¡¡¡¡¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª<p>¡¡¡¡ÐÂÊéÐÂÆøÏó£¬ÐÂÊéÉÏ´«À²£¬ºé¼ÒС±¦¸ç»ªÀöµÇ³¡£¡<p>¡¡¡¡ºÀÃ˵ÄÐÖµÜÃÇ£¬ÄãÃÇ¿É»¹ÔÚ·ñ£¿<p>¡¡¡¡ÐµÄÕ÷³Ì¿ªÆô£¬»¹¿É·ñÅãÎÒÒ»Æð³Ë·çÆÆÀË£¬Åû¾£Õ¶¼¬£¿£¡<p>¡¡¡¡ÇóÊղأ¬ÇóÍƼö£¬Çó´òÉÍ£¬ÇóÊ®·ÖÆÀ¼ÛƱ£¡ÐÂÊéÇóÒ»ÇÐÖ§³Ö£¡£¡<p>¡¡¡¡¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª<p>¡¡¡¡ÁíÍâÍÆÒ»ÏÂÍê±¾ÀÏÊ顶ÖØÉúÖ®Éñ¼¶°Ü¼Ò×Ó¡·£¬¾ù¶©ÆÆÍò×÷Æ·£¬ÖÊÁ¿±£Ö¤£¬Êé»Ä¿É¿´¡£<p>¡¡¡¡<a href=http://www.qidian.com>ÆðµãÖÐÎÄÍøwww.qidian.com»¶Ó­¹ã´óÊéÓѹâÁÙÔĶÁ£¬×îС¢×î¿ì¡¢×î»ðµÄÁ¬ÔØ×÷Æ·¾¡ÔÚÆðµãÔ­´´£¡</a><p>¡¡¡¡<a href=http://www.qidian.com>ÆðµãÖÐÎÄÍø www.qidian.com »¶Ó­¹ã´óÊéÓѹâÁÙÔĶÁ£¬×îС¢×î¿ì¡¢×î»ðµÄÁ¬ÔØ×÷Æ·¾¡ÔÚÆðµãÔ­´´£¡</a><a>ÊÖ»úÓû§Çëµ½m.qidian.comÔĶÁ¡£</a>');
\ No newline at end of file
diff --git a/tests/sample_data/compressed/unexpected-eof.gz b/tests/sample_data/compressed/unexpected-eof.gz
new file mode 100644
index 0000000000000000000000000000000000000000..96211e4324b11a3ab16c13a4fb7ea3b443389c49
GIT binary patch
literal 5134
zcmV+p6!GgHiwFSORRmW617ug*QrlV*{hXPn`GTqWnVIu24|A!hb85avD3D+jN3yl$
zizMH)ZC$o?mnCA705*hh4PXv2c1%sTg~8M$RUyV(y}MVhUaR%DfBp5hfBf>#U;p&q
z|NQ;eU;g+94*!T=qzCoWfRna(!7(GXdS5Zie05)8__NDPJwj74BFdxF%D(b~B_~5X
zm2~RE%1L$KzHri#s?ELQv`T1Hb1%J!7L0lC$hac3;Yw=D7t`uAQuXfd`$TWE!N`!S
z_hh%CbMN7=^yh#7B~K+8%G-%fD1|hdeO~Y)|GanIX2~#o8}Q*<LgVkw!X0_v)erWJ
zD{uubQnZq)9=;9)AD;f(zet!o>FB&rK1h1Cnd;Ppw3r6GpgLr)e(F*4m{8}m(hMHW
z<LZPCIsQ2v>9gPA1x|`oW3ImOT5=>uEm?ZU*E!K0mG6^Y>^Vi{k)9D+n>kzRZcA=6
zUhvaoI9MdZ;4!=-S9D0M`Yp!d__bQQ3g29NsL4tF1<~;uLRE1+Ki|T(j1^vBU+a@f
z*Y7xK8W5Tt4EA`TcWqw=Tcov2>Yem1QtbygNzT7CT%xaCnFoWfs8bXfJ6l0QhWOd?
z@Iqd&XM|RC<EuPTW>Cq2QR*9ggBN0}cO({#8|x&C;nbD$N&FZL#G?EIB94h+`;M7<
zwOJfgpn;Jp&x{|9#TuB7iMe2XdfdL^g~6!dW|~xOoi_`-P*Q7)Y(Fj5C-r+vRX(;E
zLXY+4l_?1Vn>VDqXfs4ko>(kOG?C2-?G}}Xq*rn#-vdq`r$u)J2-f1jnA0=!&|)0l
z`0U*I9g(DgeA@>;$joAm&x|-`D~s-`6Xmh|rsYRU#jbx#i&-2g`iwrlYT5chJc3Qj
zQB9BAEIl@~JZ1CHJXGU0Q)4H^<muSA{EoFL+ONXph&+8nVDL<R!V6DrKL-%b5t@)|
z&$`f?imXigyU|GAwtZ;xr8LffC<vPI*k;Bfl;_$$q4ik+&mHU>?Bw7fST9_K2|m$p
zkaoO43ndlEb?X(VS-n?J6OC^w&{0ty4CtBWwq!O27M!=_c&lnSW*dM^RcsbEuzJ6a
z2%*XYawAS#a(3^O3>oqH|B=M0h)$i3JRw(U@zuc99&3*vLd!H!Qg{JVubh<Z*mB;u
z58fRnBkg)e;oCTk1AW>k^Fq`1OHI4JD{@PQJ<kmY&UAGVFAxx|aIYy79Y+#J+C1p)
z(Ew9!Z#a@P)}$b4>m}euTele7aZ>-l8$b?_l{^WyR*lI)9gHGXx?Vxrv<F`u(?7Q>
zUKEvh?zwPkrU5;}Ynn15Do+y4J9dhIEIfvT9I0ZcTo^m?wQcc&Te1DNUfwqk^h;j2
z-2hH3PO9ZM*ioEK->#gHR?UR8Sl3I7+L5*s2feA&d&ZG|LP5t&2P02KJggI$DT%zu
zSwC1f=93z}sm;72`vxXU&91!yqD^RTKQO(c!9Fh}J!dm`;JH_d*mQeS1lqpz3NHcj
zqw)C*{Foc);;v63)xXttyh2;6gOd2SExD)nu6uem<%OtJQlk-$tGBU|F_~2$9xWqh
zYp2Ll2gVehX93qO;*f(xLSvO_OI{nH+T2o~bM1pj<T>SX`qVl}ll|MYXo``VN`po+
z%+Hk6M3J{pw~V>{v0H4)*5(VICjLs^&bl~s<b9`a?+{vkW7PmpXs7ZhGJ2+hibX2h
zVLetP^m4l7)oE>Hx0MeZU%9hR=pq$(A>0<TG;pg~x3X8i0Kw?hf2W_iT=||Av7GqE
zNu6R^^_dTfXB}F1-WO~)iVC6L0JBKny6^hOQoqv&N4%MQ=lDo<E1DSoG!IyleXX2$
z-;@skwVlP{*uE(_ZpgB=LCd9*7qoe8v2vzxPn+i`xpTxEi$=VHpmp>7Xk%)CKUt7j
zV_G)=g?sz*`Aa0$iV-=k9@r6~gP!dhKhjfV$@1P8LW!H9j#84(k+Gb4Jkp3QH3}Q=
z%l9215%Sk)weE4QR^&28@Q;mXlrI!{UYIg++yO6T3#(%JaHU?^2wHxL7oO=g(Ux0-
zW4a@(L=U{kId+ak8#4(X{Dc^0rvD7K*T|J;o;LyP=m#gY*^||d9aFxJx1M3hOGgc<
z7tccXO!8Os%dYB^CJ>OC`9jegqL1eel@CDH^_-Qyc&h!;Z-Gm!?>4X`7<n+y>0N_@
zFCVV@`knOFdXK%rhuaVGWUz3{X)#iHAy?__!SAejmyjFWIs-t`mcK)tiA?D-cYd8W
zw7{fIq_=%3W>LpF3AcGcVK^^Uscv-c{DYM*usJ)j0NP~e-{yHPa{_^M%y7@Xv19;V
za*0b2h!qcm0L14ix>%zq+EBa^>;c0*qQaI``7-|(Q_T}46hlgCeo+$^({Qf^;aJJv
zK>5jkFO7{KVpdxqsoC7n+={p1ZRD1nZyxne%?s!2pw=wlnPrp5ZAQj!1q4}eq=2S@
zC5t5Q45l1s^h_F~arQ5&Wo<_bA-3y2tOHMI4t(&Xt>mW57P;+Oa&?AVA1=bUN@Mq%
zatQXR-}EnFAMKbhEZvS{8Y*x_)^5q>tJR@y*3H#A#7P($R-q7k`K-j?Cgx$p0k)j|
z&_4{qJb83>BH_lmbbBa2Ei}5SgN$g}YE3MGM!)!W?i0LVG*=5^d`={J-8puH^QfR7
zmhC)XY(z=xz5bK!RP@{e;;?R@4Lmp3I;qW90YVPSH7P9Q)Wi!0BMFR08i!n@)^^e#
zen<PltIE{6d(jp=<Cr+=>(3;&Q30QV(SmB?_tu8Yy2aYU=ssU8HK2-D0_AV|om7p%
zgfU#rQKqX|mBzt03|PKjg@h&OJS}3K9Is5FcXJ6=GvysXxr7hpFFwyHP0!j#{sEDA
zA%j)<X`w5L#enwr;HBsZ@Tb?Vl^mu;_7Ekn-o}7n`C%A?nFF4i#WmAEP!h5!aky(a
zW_A?-hSHWKS1)B~2*&7L8}mFyG-4VoCcHambN@!((IROJ7NH02(A~lpC`NkMNJ-r0
zjCq!-HOEAL<nWkTmRJydZ7#(vd@s`?3_lpvH?R(PcRB1`XNgwFFxgdqPb^w;RKn!+
zz+!F%t0u!nH?MlRox!(p5NyYHMh}U}=8+2AhwI&~W+9-2^1}ZB00960Y*`6&+UOGe
z|35=^;+Hsn1au#O(1C;yIxGo<d9IDuBu-`R_y`WRad7RlJ`2e1D_7;f(s#`C^mGs1
zh&{s_@!fcxN#JU>+P|#S=8?)sp|sHT*)8OgVGHV<Re*b_R5y$>>t20?d2oH+<YeoC
z^v2urKs(?R5Cs&1+;#{YRRNnjMhbTyeVGggKkF^X{2WR>oH=|JmU-+qA=I$}dpa|C
zQ#(-JIn#EdlZMZX6SAT;$NzjY_}<}0Ul>h}*0nFi6~^ayuVOC4QufW?5<!_P+~Y3S
z<d6jRn($`SHD?|HIsU_qiK98{R2-;m>jseJ{F0=qL7yEZgTY%<B>FrFrQbXX^<}^K
z>4?U3Xx9t{Zql9Pj+l5M7m@OhwwGUSt(aU4bBJ!7)4Q$cl=jyh)@!<vuyp!Fx6SB@
zVLW^)W|Et}BujMxNn?{1uKQHyUR|+O$@hG!LBuE@c0P3z8-l?K2gc?|oj<i_bqpWC
z7L{Cafi1fcd_eria7}7A8icM>_D3t<=IMUeRtZ{QvP&@p`)9^@A>Rr2u`GT-sk~-~
zUK@n{H<1FdNNbW}!K!W5dB&QNhmMmOR#!O_q)m-O3)dRnbMsW1dkwFl|IBZVkCLAR
zu?!foSWa4<97h%Jt&pNfv4gAfznRM&G>+HdQ`{Na!cq~1{nj*3kJkuz=h5l2d$Ms$
z_1Y26tca4v8)d)Uf~S7fQ{F_g!u&Zq^e>w{iwbeq(hAk(+2?>nG^tXL*;`7)j4bB`
z5MhdAQeaYvhOf?Xy!6FbZ4!WB`puN`gBHuS??xDO!1dTgyod6Jh9U9xl-(y+-Yr;Z
z1M*o9CaFW3SALfrmT96&WJu*UmaszmcHbPIqh`u$ZQA|?K?48A8}4bIHZP?)e@&`^
zh@Uh(Ek9a=?3iY+Eg0kqZ-jFx60G!p`dwXv#Y|q2I@2;fIv8FkFLk796}iiQXrlG~
z!H$maXUV`6K_aHO;|5t?^@t%E6sSAiMUC;|heo166U(a(b$3BV>j{b3(l%AaL3YmV
z57wmhR<*tLZU>{112BmuGNdYdX;H;vvPQy%G?qq&cPr1&dpFWlJg93yUljWO(d0D#
zk3AYbh^dG&;{3k#5K=ExzIL0?HRyG$JnObp{1~q%s*#s{pv0lfOVh-TW@Vq<zx63d
z3{B~meV!Do<c?O_DYrKoi-C8<?YZ4ZnKXK5gt3R1oJ)D_U0_f+^ih|DtZEm@WtI$6
zoaPLkM_qas+C*}jW$*!O6<6Iza^(;4!=|<{!@vi5%3g>j1)owp%o&oD*bZ5AsVqse
z1HOhPKC<VOj?x*g$3zCn=Q^EwoI9Jy)f$n(7fjCvAz*5pHhfivay#WxH+_#O2E%2X
zrS@?;@q}ehlwA9J1S@*lxz!fv6}U&a#J^g%{iPvMJ#Mj5J#4!E&_`ISi`L}tn@BuV
zDT5CK1Wwl;rH_!-yF$9a4}uGGL!J+h%&3S+U>7IREVxEVi3|Lh9Zp(7GHE3?9jY>0
zxE|g>!XaFFBe`U0Cz(r$ai_HXlFXXHXir=aSyaKOMNXVWGh82$X6gDzC&eAuE8B~z
zk8uWe*?Ab_;qkdUMfY%h>;>1;JdS2WKot-=hSXt8!aTA4L0myXAg15zr^Olh+QhjS
zi}uPri>z9LVG7atmYvgYuY_evC;<2J7S&YHHQ0=8GR|j=P8s2;%lf!GlY;F4-TP=+
z*!<=@+cG&X`}&nIFg$zYpY`N2(Pw0)42|#h)#dRRGX?VgA0cY-H)<%@Q}zJS(3Jj5
zKW_XTZQXe(X2VJE2J_AIeg9+sBM^i=gFEkHC-b^f$D7?!T)+n7X3b;Q7aNmrWiCDp
zC(#0Zwk9@``e6{sFBWl#F^v?+y1`rPY9|$wSrz_Neh?odyJExFaO0tYp)UX#^N?W1
zZOA7xq8VBCc{z_}nB+7sDS|D5c0pvtg_%b)%!k-Q$pBKXdEC2kE^-HyZ9NH}%0-?D
zZ{!h?N<}Bq1>V4hiLIU5cgSdQf^0R9(RL$`HEi7en9$nt$zr8AJo`OXcYQQ>QqXhv
z>!cCi3_kFtq+tL(c#B?ylP9*o1&R7gpiaCq${e4k?~Q9Ui2h;_fnE)g<`-=wzk~AE
z7x{$R92GJ?!BL?l;yvXnwP5Pf<!Lz23_n%p^1Ge9F`9MfB>h}T0m`V#OiXCdZig-X
z2W|khV;&CM#S9d#`YETFTX_vK7@za~@)K%r)8<vGSVAr%qV1>wI8g)aL{Qpl3yg$R
z)fo1~cvy!~KhaOp14<wr5%PsMbO$DbDCBl`xj55K!{<&ad<z?FJ2S}B+_6GX#^9~<
zp8tdDgDx{MsIxz!xKSrcr$t004e)pL7ZeIf1-iC*s7=ADpzg4de5PLJhK+7ykwvUL
zwSo8xUr?}>t~nh_xx;8|6IpkTh(ZuGFGpA4MDme`=$trC6)AH&WI_yq2D#lmga+Gy
z8DEBw;`yza?;okZpc}|em{MTQ#QS}&ZtJwk8*LqYhfZvNZT=wj#mO?}f*T+))MKp0
z_U=ZZ6h3c_557|}MO@U-FlyeNp^^BPsOqX&1}S!2h1mh*3%npXptd?Y=t@vZ$#rTg
zd(HzWT04bv>!jn{dlaVo93~E=k0cf%sRg71`0@kVhT%0!lnQdznhVq3jk4E?PyzJ2
zKlxR#BJ(uy9+{)birk^@H#csJD@Chva>TN1D6M_QT)LAx|I$f7I3PA!Q`eztq}ro;
z1mJYXQsU1P@;MGh&$y03y2rIM?^81@=C8uw2!PwMOgKAY!t4^;BuFp}Ce^6pt5B&W
z{Df<B=l|apx^puq9+M`V*e<SCfCJy)8~7Mq;&pWXS~TsJbAXN7&ul*T`s|VJbR7&m
zd;C7<uF&)>Vj^bv;tb6Mve!~1_qg*rkbQhO_^QYF?hr2ywVqgYOtZdOyA=<2jXQ_d
zf}s*A%5)15du0iA5tHSdGMHG(khVY}0RXU`La=iJMnjp=FJM+Gv)BGTRm6y!FU1)8
z%bZNzYejI)vxsW}g~1cpgprguM&PQ|Rxo*g0{{U3|EyU{Pr^VD{wv&!7q~0(Um72Q
z1HMAS@4=cF5r{w^U@L8*Y|~;bN?8y&!GY9Qco;yuxFg|}m>3h&WVf4bX20&t&U`Zq
zrK>3%Fn!Po3G9sE@O?T4)&9(cxhS!4g#yHe&^BRlU?9aMk=RPDc!227&+KX{Z#d;V
z8e7DU;a#^##+L#mPU{_+ju4fHtd@?<kKdJCC=qs!*=9il^C)OgMY9`xZ*F&GLZ+&t
z^o<ln?|@6v`ChRN$v5hzOMC|*ne0~aGe=iBF$NB3M)L+nHAS4BHZYMmgr4c^CiF&v
z$4C|*QIh@!tqmZI;IlNQ7tntBS>@GinFwN=`IOAbmf=J=T-PT!q4z}YvMJsZg%Oj_
zeQ@Upnf=-bkr6f&DM{l~l9oAwD(Op9(@VMh$Gdx5{!r*}!{Z5u!|tQNPJr5XZyyAG
zPyoQ?Th$z|P6%8jo^3M&!>Qb5Tgrp2atg%Gaz1iWdy`(E6}qgd#Oyji$VKc1$g<-M
wVTWE1<@@9EuGJaWKLDV71{Nljf^eoiv)92T=^%`35cz0ree(+d0RR630QaH)cmMzZ

literal 0
HcmV?d00001

diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index 2b47bf8da..7148185f4 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -1,6 +1,8 @@
 import unittest
 from os.path import join
 
+from w3lib.encoding import html_to_unicode
+
 from scrapy.utils.gz import gunzip, is_gzipped
 from scrapy.http import Response, Headers
 from tests import tests_datadir
@@ -66,3 +68,11 @@ class GunzipTest(unittest.TestCase):
         hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"})
         r1 = Response("http://www.example.com", headers=hdrs)
         self.assertTrue(is_gzipped(r1))
+
+    def test_gunzip_illegal_eof(self):
+        with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f:
+            text = html_to_unicode('charset=cp1252', gunzip(f.read()))[1]
+            with open(join(SAMPLEDIR, 'unexpected-eof-output.txt'), 'rb') as o:
+                expected_text = o.read().decode("utf-8")
+                self.assertEqual(len(text), len(expected_text))
+                self.assertEqual(text, expected_text)

From bb3b806467c1a79ad0ff3c8012e9366cc6066f2a Mon Sep 17 00:00:00 2001
From: Ashish Kulkarni <ashish@advarisk.com>
Date: Mon, 8 Aug 2016 16:54:03 +0530
Subject: [PATCH 1038/2974] Use w3lib.url.canonicalize_url() from w3lib 1.15.0

Also remove code/imports which are now unused due to this change.

fixes #2157
---
 docs/topics/link-extractors.rst   |   2 +-
 requirements.txt                  |   2 +-
 scrapy/linkextractors/__init__.py |   3 +-
 scrapy/utils/request.py           |   2 +-
 scrapy/utils/url.py               | 179 +-------------------------
 setup.py                          |   2 +-
 tests/test_utils_url.py           | 206 +-----------------------------
 7 files changed, 10 insertions(+), 386 deletions(-)

diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 8253e7650..4636ddb18 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -103,7 +103,7 @@ LxmlLinkExtractor
     :type attrs: list
 
     :param canonicalize: canonicalize each extracted url (using
-        scrapy.utils.url.canonicalize_url). Defaults to ``True``.
+        w3lib.url.canonicalize_url). Defaults to ``True``.
     :type canonicalize: boolean
 
     :param unique: whether duplicate filtering should be applied to extracted
diff --git a/requirements.txt b/requirements.txt
index ad9226e87..cfa907050 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@ Twisted>=10.0.0
 lxml
 pyOpenSSL
 cssselect>=0.9
-w3lib>=1.14.2
+w3lib>=1.15.0
 queuelib
 six>=1.5.2
 PyDispatcher>=2.0.5
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index efb11e43c..f51934b00 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -9,10 +9,11 @@ import re
 
 from six.moves.urllib.parse import urlparse
 from parsel.csstranslator import HTMLTranslator
+from w3lib.url import canonicalize_url
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.url import (
-    canonicalize_url, url_is_from_any_domain, url_has_any_extension,
+    url_is_from_any_domain, url_has_any_extension,
 )
 
 
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index a98ca61ce..50bc3cb1e 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -11,7 +11,7 @@ from six.moves.urllib.parse import urlunparse
 from w3lib.http import basic_auth_header
 from scrapy.utils.python import to_bytes, to_native_str
 
-from scrapy.utils.url import canonicalize_url
+from w3lib.url import canonicalize_url
 from scrapy.utils.httpobj import urlparse_cached
 
 
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 406eb5843..dc1cce4ac 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -7,18 +7,13 @@ to the w3lib.url module. Always import those from there instead.
 """
 import posixpath
 import re
-import six
-from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag,
-                                    urlparse, parse_qsl, urlencode,
-                                    quote, unquote)
-if not six.PY2:
-    from urllib.parse import unquote_to_bytes
+from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse)
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this
 # move doesn't break old code
 from w3lib.url import *
-from w3lib.url import _safe_chars
-from scrapy.utils.python import to_bytes, to_native_str, to_unicode
+from w3lib.url import _safe_chars, _unquotepath
+from scrapy.utils.python import to_unicode
 
 
 def url_is_from_any_domain(url, domains):
@@ -40,121 +35,6 @@ def url_has_any_extension(url, extensions):
     return posixpath.splitext(parse_url(url).path)[1].lower() in extensions
 
 
-def _safe_ParseResult(parts, encoding='utf8', path_encoding='utf8'):
-    # IDNA encoding can fail for too long labels (>63 characters)
-    # or missing labels (e.g. http://.example.com)
-    try:
-        netloc = parts.netloc.encode('idna')
-    except UnicodeError:
-        netloc = parts.netloc
-
-    return (
-        to_native_str(parts.scheme),
-        to_native_str(netloc),
-
-        # default encoding for path component SHOULD be UTF-8
-        quote(to_bytes(parts.path, path_encoding), _safe_chars),
-        quote(to_bytes(parts.params, path_encoding), _safe_chars),
-
-        # encoding of query and fragment follows page encoding
-        # or form-charset (if known and passed)
-        quote(to_bytes(parts.query, encoding), _safe_chars),
-        quote(to_bytes(parts.fragment, encoding), _safe_chars)
-    )
-
-
-def canonicalize_url(url, keep_blank_values=True, keep_fragments=False,
-                     encoding=None):
-    """Canonicalize the given url by applying the following procedures:
-
-    - sort query arguments, first by key, then by value
-    - percent encode paths ; non-ASCII characters are percent-encoded
-      using UTF-8 (RFC-3986)
-    - percent encode query arguments ; non-ASCII characters are percent-encoded
-      using passed `encoding` (UTF-8 by default)
-    - normalize all spaces (in query arguments) '+' (plus symbol)
-    - normalize percent encodings case (%2f -> %2F)
-    - remove query arguments with blank values (unless `keep_blank_values` is True)
-    - remove fragments (unless `keep_fragments` is True)
-
-    The url passed can be bytes or unicode, while the url returned is
-    always a native str (bytes in Python 2, unicode in Python 3).
-
-    For examples see the tests in tests/test_utils_url.py
-    """
-    # If supplied `encoding` is not compatible with all characters in `url`,
-    # fallback to UTF-8 as safety net.
-    # UTF-8 can handle all Unicode characters,
-    # so we should be covered regarding URL normalization,
-    # if not for proper URL expected by remote website.
-    try:
-        scheme, netloc, path, params, query, fragment = _safe_ParseResult(
-            parse_url(url), encoding=encoding)
-    except UnicodeEncodeError as e:
-        scheme, netloc, path, params, query, fragment = _safe_ParseResult(
-            parse_url(url), encoding='utf8')
-
-    # 1. decode query-string as UTF-8 (or keep raw bytes),
-    #    sort values,
-    #    and percent-encode them back
-    if six.PY2:
-        keyvals = parse_qsl(query, keep_blank_values)
-    else:
-        # Python3's urllib.parse.parse_qsl does not work as wanted
-        # for percent-encoded characters that do not match passed encoding,
-        # they get lost.
-        #
-        # e.g., 'q=b%a3' becomes [('q', 'b\ufffd')]
-        # (ie. with 'REPLACEMENT CHARACTER' (U+FFFD),
-        #      instead of \xa3 that you get with Python2's parse_qsl)
-        #
-        # what we want here is to keep raw bytes, and percent encode them
-        # so as to preserve whatever encoding what originally used.
-        #
-        # See https://tools.ietf.org/html/rfc3987#section-6.4:
-        #
-        # For example, it is possible to have a URI reference of
-        # "http://www.example.org/r%E9sum%E9.xml#r%C3%A9sum%C3%A9", where the
-        # document name is encoded in iso-8859-1 based on server settings, but
-        # where the fragment identifier is encoded in UTF-8 according to
-        # [XPointer]. The IRI corresponding to the above URI would be (in XML
-        # notation)
-        # "http://www.example.org/r%E9sum%E9.xml#r&#xE9;sum&#xE9;".
-        # Similar considerations apply to query parts.  The functionality of
-        # IRIs (namely, to be able to include non-ASCII characters) can only be
-        # used if the query part is encoded in UTF-8.
-        keyvals = parse_qsl_to_bytes(query, keep_blank_values)
-    keyvals.sort()
-    query = urlencode(keyvals)
-
-    # 2. decode percent-encoded sequences in path as UTF-8 (or keep raw bytes)
-    #    and percent-encode path again (this normalizes to upper-case %XX)
-    uqp = _unquotepath(path)
-    path = quote(uqp, _safe_chars) or '/'
-
-    fragment = '' if not keep_fragments else fragment
-
-    # every part should be safe already
-    return urlunparse((scheme, netloc.lower(), path, params, query, fragment))
-
-
-def _unquotepath(path):
-    for reserved in ('2f', '2F', '3f', '3F'):
-        path = path.replace('%' + reserved, '%25' + reserved.upper())
-
-    if six.PY2:
-        # in Python 2, '%a3' becomes '\xa3', which is what we want
-        return unquote(path)
-    else:
-        # in Python 3,
-        # standard lib's unquote() does not work for non-UTF-8
-        # percent-escaped characters, they get lost.
-        # e.g., '%a3' becomes 'REPLACEMENT CHARACTER' (U+FFFD)
-        #
-        # unquote_to_bytes() returns raw bytes instead
-        return unquote_to_bytes(path)
-
-
 def parse_url(url, encoding=None):
     """Return urlparsed url from the given argument (which could be an already
     parsed url)
@@ -164,59 +44,6 @@ def parse_url(url, encoding=None):
     return urlparse(to_unicode(url, encoding))
 
 
-if not six.PY2:
-    from urllib.parse import _coerce_args, unquote_to_bytes
-
-    def parse_qsl_to_bytes(qs, keep_blank_values=False, strict_parsing=False):
-        """Parse a query given as a string argument.
-
-        Data are returned as a list of name, value pairs as bytes.
-
-        Arguments:
-
-        qs: percent-encoded query string to be parsed
-
-        keep_blank_values: flag indicating whether blank values in
-            percent-encoded queries should be treated as blank strings.  A
-            true value indicates that blanks should be retained as blank
-            strings.  The default false value indicates that blank values
-            are to be ignored and treated as if they were  not included.
-
-        strict_parsing: flag indicating what to do with parsing errors. If
-            false (the default), errors are silently ignored. If true,
-            errors raise a ValueError exception.
-
-        """
-        # This code is the same as Python3's parse_qsl()
-        # (at https://hg.python.org/cpython/rev/c38ac7ab8d9a)
-        # except for the unquote(s, encoding, errors) calls replaced
-        # with unquote_to_bytes(s)
-        qs, _coerce_result = _coerce_args(qs)
-        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
-        r = []
-        for name_value in pairs:
-            if not name_value and not strict_parsing:
-                continue
-            nv = name_value.split('=', 1)
-            if len(nv) != 2:
-                if strict_parsing:
-                    raise ValueError("bad query field: %r" % (name_value,))
-                # Handle case of a control-name with no equal sign
-                if keep_blank_values:
-                    nv.append('')
-                else:
-                    continue
-            if len(nv[1]) or keep_blank_values:
-                name = nv[0].replace('+', ' ')
-                name = unquote_to_bytes(name)
-                name = _coerce_result(name)
-                value = nv[1].replace('+', ' ')
-                value = unquote_to_bytes(value)
-                value = _coerce_result(value)
-                r.append((name, value))
-        return r
-
-
 def escape_ajax(url):
     """
     Return the crawleable url according to:
diff --git a/setup.py b/setup.py
index 000d84614..92c114a7a 100644
--- a/setup.py
+++ b/setup.py
@@ -42,7 +42,7 @@ setup(
     ],
     install_requires=[
         'Twisted>=10.0.0',
-        'w3lib>=1.14.2',
+        'w3lib>=1.15.0',
         'queuelib',
         'lxml',
         'pyOpenSSL',
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index b4819874d..f46d1d927 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -6,8 +6,7 @@ from six.moves.urllib.parse import urlparse
 
 from scrapy.spiders import Spider
 from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
-                              canonicalize_url, add_http_if_no_scheme,
-                              guess_scheme, parse_url)
+                              add_http_if_no_scheme, guess_scheme, parse_url)
 
 __doctests__ = ['scrapy.utils.url']
 
@@ -77,209 +76,6 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider))
 
 
-class CanonicalizeUrlTest(unittest.TestCase):
-
-    def test_canonicalize_url(self):
-        # simplest case
-        self.assertEqual(canonicalize_url("http://www.example.com/"),
-                                          "http://www.example.com/")
-
-    def test_return_str(self):
-        assert isinstance(canonicalize_url(u"http://www.example.com"), str)
-        assert isinstance(canonicalize_url(b"http://www.example.com"), str)
-
-    def test_append_missing_path(self):
-        self.assertEqual(canonicalize_url("http://www.example.com"),
-                                          "http://www.example.com/")
-
-    def test_typical_usage(self):
-        self.assertEqual(canonicalize_url("http://www.example.com/do?a=1&b=2&c=3"),
-                                          "http://www.example.com/do?a=1&b=2&c=3")
-        self.assertEqual(canonicalize_url("http://www.example.com/do?c=1&b=2&a=3"),
-                                          "http://www.example.com/do?a=3&b=2&c=1")
-        self.assertEqual(canonicalize_url("http://www.example.com/do?&a=1"),
-                                          "http://www.example.com/do?a=1")
-
-    def test_sorting(self):
-        self.assertEqual(canonicalize_url("http://www.example.com/do?c=3&b=5&b=2&a=50"),
-                                          "http://www.example.com/do?a=50&b=2&b=5&c=3")
-
-    def test_keep_blank_values(self):
-        self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2", keep_blank_values=False),
-                                          "http://www.example.com/do?a=2")
-        self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2"),
-                                          "http://www.example.com/do?a=2&b=")
-        self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2", keep_blank_values=False),
-                                          "http://www.example.com/do?a=2")
-        self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2"),
-                                          "http://www.example.com/do?a=2&b=&c=")
-
-        self.assertEqual(canonicalize_url(u'http://www.example.com/do?1750,4'),
-                                           'http://www.example.com/do?1750%2C4=')
-
-    def test_spaces(self):
-        self.assertEqual(canonicalize_url("http://www.example.com/do?q=a space&a=1"),
-                                          "http://www.example.com/do?a=1&q=a+space")
-        self.assertEqual(canonicalize_url("http://www.example.com/do?q=a+space&a=1"),
-                                          "http://www.example.com/do?a=1&q=a+space")
-        self.assertEqual(canonicalize_url("http://www.example.com/do?q=a%20space&a=1"),
-                                          "http://www.example.com/do?a=1&q=a+space")
-
-    def test_canonicalize_url_unicode_path(self):
-        self.assertEqual(canonicalize_url(u"http://www.example.com/résumé"),
-                                          "http://www.example.com/r%C3%A9sum%C3%A9")
-
-    def test_canonicalize_url_unicode_query_string(self):
-        # default encoding for path and query is UTF-8
-        self.assertEqual(canonicalize_url(u"http://www.example.com/résumé?q=résumé"),
-                                          "http://www.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9")
-
-        # passed encoding will affect query string
-        self.assertEqual(canonicalize_url(u"http://www.example.com/résumé?q=résumé", encoding='latin1'),
-                                          "http://www.example.com/r%C3%A9sum%C3%A9?q=r%E9sum%E9")
-
-        self.assertEqual(canonicalize_url(u"http://www.example.com/résumé?country=Россия", encoding='cp1251'),
-                                          "http://www.example.com/r%C3%A9sum%C3%A9?country=%D0%EE%F1%F1%E8%FF")
-
-    def test_canonicalize_url_unicode_query_string_wrong_encoding(self):
-        # trying to encode with wrong encoding
-        # fallback to UTF-8
-        self.assertEqual(canonicalize_url(u"http://www.example.com/résumé?currency=€", encoding='latin1'),
-                                          "http://www.example.com/r%C3%A9sum%C3%A9?currency=%E2%82%AC")
-
-        self.assertEqual(canonicalize_url(u"http://www.example.com/résumé?country=Россия", encoding='latin1'),
-                                          "http://www.example.com/r%C3%A9sum%C3%A9?country=%D0%A0%D0%BE%D1%81%D1%81%D0%B8%D1%8F")
-
-    def test_normalize_percent_encoding_in_paths(self):
-        self.assertEqual(canonicalize_url("http://www.example.com/r%c3%a9sum%c3%a9"),
-                                          "http://www.example.com/r%C3%A9sum%C3%A9")
-
-        # non-UTF8 encoded sequences: they should be kept untouched, only upper-cased
-        # 'latin1'-encoded sequence in path
-        self.assertEqual(canonicalize_url("http://www.example.com/a%a3do"),
-                                          "http://www.example.com/a%A3do")
-
-        # 'latin1'-encoded path, UTF-8 encoded query string
-        self.assertEqual(canonicalize_url("http://www.example.com/a%a3do?q=r%c3%a9sum%c3%a9"),
-                                          "http://www.example.com/a%A3do?q=r%C3%A9sum%C3%A9")
-
-        # 'latin1'-encoded path and query string
-        self.assertEqual(canonicalize_url("http://www.example.com/a%a3do?q=r%e9sum%e9"),
-                                          "http://www.example.com/a%A3do?q=r%E9sum%E9")
-
-    def test_normalize_percent_encoding_in_query_arguments(self):
-        self.assertEqual(canonicalize_url("http://www.example.com/do?k=b%a3"),
-                                          "http://www.example.com/do?k=b%A3")
-
-        self.assertEqual(canonicalize_url("http://www.example.com/do?k=r%c3%a9sum%c3%a9"),
-                                          "http://www.example.com/do?k=r%C3%A9sum%C3%A9")
-
-    def test_non_ascii_percent_encoding_in_paths(self):
-        self.assertEqual(canonicalize_url("http://www.example.com/a do?a=1"),
-                                          "http://www.example.com/a%20do?a=1"),
-        self.assertEqual(canonicalize_url("http://www.example.com/a %20do?a=1"),
-                                          "http://www.example.com/a%20%20do?a=1"),
-        self.assertEqual(canonicalize_url(u"http://www.example.com/a do£.html?a=1"),
-                                          "http://www.example.com/a%20do%C2%A3.html?a=1")
-        self.assertEqual(canonicalize_url(b"http://www.example.com/a do\xc2\xa3.html?a=1"),
-                                          "http://www.example.com/a%20do%C2%A3.html?a=1")
-
-    def test_non_ascii_percent_encoding_in_query_arguments(self):
-        self.assertEqual(canonicalize_url(u"http://www.example.com/do?price=£500&a=5&z=3"),
-                                          u"http://www.example.com/do?a=5&price=%C2%A3500&z=3")
-        self.assertEqual(canonicalize_url(b"http://www.example.com/do?price=\xc2\xa3500&a=5&z=3"),
-                                          "http://www.example.com/do?a=5&price=%C2%A3500&z=3")
-        self.assertEqual(canonicalize_url(b"http://www.example.com/do?price(\xc2\xa3)=500&a=1"),
-                                          "http://www.example.com/do?a=1&price%28%C2%A3%29=500")
-
-    def test_urls_with_auth_and_ports(self):
-        self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com:81/do?now=1"),
-                                          u"http://user:pass@www.example.com:81/do?now=1")
-
-    def test_remove_fragments(self):
-        self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com/do?a=1#frag"),
-                                          u"http://user:pass@www.example.com/do?a=1")
-        self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com/do?a=1#frag", keep_fragments=True),
-                                          u"http://user:pass@www.example.com/do?a=1#frag")
-
-    def test_dont_convert_safe_characters(self):
-        # dont convert safe characters to percent encoding representation
-        self.assertEqual(canonicalize_url(
-            "http://www.simplybedrooms.com/White-Bedroom-Furniture/Bedroom-Mirror:-Josephine-Cheval-Mirror.html"),
-            "http://www.simplybedrooms.com/White-Bedroom-Furniture/Bedroom-Mirror:-Josephine-Cheval-Mirror.html")
-
-    def test_safe_characters_unicode(self):
-        # urllib.quote uses a mapping cache of encoded characters. when parsing
-        # an already percent-encoded url, it will fail if that url was not
-        # percent-encoded as utf-8, that's why canonicalize_url must always
-        # convert the urls to string. the following test asserts that
-        # functionality.
-        self.assertEqual(canonicalize_url(u'http://www.example.com/caf%E9-con-leche.htm'),
-                                           'http://www.example.com/caf%E9-con-leche.htm')
-
-    def test_domains_are_case_insensitive(self):
-        self.assertEqual(canonicalize_url("http://www.EXAMPLE.com/"),
-                                          "http://www.example.com/")
-
-    def test_canonicalize_idns(self):
-        self.assertEqual(canonicalize_url(u'http://www.bücher.de?q=bücher'),
-                                           'http://www.xn--bcher-kva.de/?q=b%C3%BCcher')
-        # Japanese (+ reordering query parameters)
-        self.assertEqual(canonicalize_url(u'http://はじめよう.みんな/?query=サ&maxResults=5'),
-                                           'http://xn--p8j9a0d9c9a.xn--q9jyb4c/?maxResults=5&query=%E3%82%B5')
-
-    def test_quoted_slash_and_question_sign(self):
-        self.assertEqual(canonicalize_url("http://foo.com/AC%2FDC+rocks%3f/?yeah=1"),
-                         "http://foo.com/AC%2FDC+rocks%3F/?yeah=1")
-        self.assertEqual(canonicalize_url("http://foo.com/AC%2FDC/"),
-                         "http://foo.com/AC%2FDC/")
-
-    def test_canonicalize_urlparsed(self):
-        # canonicalize_url() can be passed an already urlparse'd URL
-        self.assertEqual(canonicalize_url(urlparse(u"http://www.example.com/résumé?q=résumé")),
-                                          "http://www.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9")
-        self.assertEqual(canonicalize_url(urlparse('http://www.example.com/caf%e9-con-leche.htm')),
-                                          'http://www.example.com/caf%E9-con-leche.htm')
-        self.assertEqual(canonicalize_url(urlparse("http://www.example.com/a%a3do?q=r%c3%a9sum%c3%a9")),
-                                          "http://www.example.com/a%A3do?q=r%C3%A9sum%C3%A9")
-
-    def test_canonicalize_parse_url(self):
-        # parse_url() wraps urlparse and is used in link extractors
-        self.assertEqual(canonicalize_url(parse_url(u"http://www.example.com/résumé?q=résumé")),
-                                          "http://www.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9")
-        self.assertEqual(canonicalize_url(parse_url('http://www.example.com/caf%e9-con-leche.htm')),
-                                          'http://www.example.com/caf%E9-con-leche.htm')
-        self.assertEqual(canonicalize_url(parse_url("http://www.example.com/a%a3do?q=r%c3%a9sum%c3%a9")),
-                                          "http://www.example.com/a%A3do?q=r%C3%A9sum%C3%A9")
-
-    def test_canonicalize_url_idempotence(self):
-        for url, enc in [(u'http://www.bücher.de/résumé?q=résumé', 'utf8'),
-                         (u'http://www.example.com/résumé?q=résumé', 'latin1'),
-                         (u'http://www.example.com/résumé?country=Россия', 'cp1251'),
-                         (u'http://はじめよう.みんな/?query=サ&maxResults=5', 'iso2022jp')]:
-            canonicalized = canonicalize_url(url, encoding=enc)
-
-            # if we canonicalize again, we ge the same result
-            self.assertEqual(canonicalize_url(canonicalized, encoding=enc), canonicalized)
-
-            # without encoding, already canonicalized URL is canonicalized identically
-            self.assertEqual(canonicalize_url(canonicalized), canonicalized)
-
-    def test_canonicalize_url_idna_exceptions(self):
-        # missing DNS label
-        self.assertEqual(
-            canonicalize_url(u"http://.example.com/résumé?q=résumé"),
-            "http://.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9")
-
-        # DNS label too long
-        self.assertEqual(
-            canonicalize_url(
-                u"http://www.{label}.com/résumé?q=résumé".format(
-                    label=u"example"*11)),
-            "http://www.{label}.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9".format(
-                    label=u"example"*11))
-
-
 class AddHttpIfNoScheme(unittest.TestCase):
 
     def test_add_scheme(self):

From f18c3e5ce5a7ac58f9de1407e4aa1f10430fffc8 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 18 Aug 2016 16:18:08 +0200
Subject: [PATCH 1039/2974] Add release notes for 1.1.2 version

---
 docs/news.rst | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 82b59d5a8..5395db8e3 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,20 @@
 Release notes
 =============
 
+1.1.2 (2016-08-18)
+------------------
+
+Bug fixes
+~~~~~~~~~
+
+- Introduce a missing :setting:`IMAGES_STORE_S3_ACL` setting to override
+  the default ACL policy in ``ImagesPipeline`` when uploading images to S3
+  (note that default ACL policy is "private" -- instead of "public-read" --
+  since Scrapy 1.1.0)
+- :setting:`IMAGES_EXPIRES` default value set back to 90
+  (the regression was introduced in 1.1.1)
+
+
 1.1.1 (2016-07-13)
 ------------------
 

From d61650d843f8002b79c0d3f6d203ee0fa79af4aa Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Fri, 19 Aug 2016 18:24:32 -0300
Subject: [PATCH 1040/2974] fix RANDOMIZE_DOWNLOAD_DELAY description in the
 docs

---
 docs/topics/settings.rst | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 56734368f..8540308fe 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -504,8 +504,7 @@ supported.  Example::
 
 This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY`
 setting (which is enabled by default). By default, Scrapy doesn't wait a fixed
-amount of time between requests, but uses a random interval between 0.5 and 1.5
-* :setting:`DOWNLOAD_DELAY`.
+amount of time between requests, but uses a random interval between 0.5 * :setting:`DOWNLOAD_DELAY` and 1.5 * :setting:`DOWNLOAD_DELAY`.
 
 When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced
 per ip address instead of per domain.
@@ -925,8 +924,7 @@ RANDOMIZE_DOWNLOAD_DELAY
 
 Default: ``True``
 
-If enabled, Scrapy will wait a random amount of time (between 0.5 and 1.5
-* :setting:`DOWNLOAD_DELAY`) while fetching requests from the same
+If enabled, Scrapy will wait a random amount of time (between 0.5 * :setting:`DOWNLOAD_DELAY` and 1.5 * :setting:`DOWNLOAD_DELAY`) while fetching requests from the same
 website.
 
 This randomization decreases the chance of the crawler being detected (and

From 633abfbea1bda68918618900c25d1489cc824c68 Mon Sep 17 00:00:00 2001
From: Thom Dixon <thom@uber.com>
Date: Wed, 24 Aug 2016 08:47:52 -0700
Subject: [PATCH 1041/2974] Correct documentation about Response parameters

This fixes issue #2196
---
 docs/topics/request-response.rst | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index c54685d9e..dccea32ba 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -463,7 +463,7 @@ method for this job. Here's an example spider which uses it::
 Response objects
 ================
 
-.. class:: Response(url, [status=200, headers, body, flags])
+.. class:: Response(url, [status=200, headers=None, body=b'', flags=None, request=None])
 
     A :class:`Response` object represents an HTTP response, which is usually
     downloaded (by the Downloader) and fed to the Spiders for processing.
@@ -471,28 +471,28 @@ Response objects
     :param url: the URL of this response
     :type url: string
 
-    :param headers: the headers of this response. The dict values can be strings
-       (for single valued headers) or lists (for multi-valued headers).
-    :type headers: dict
-
     :param status: the HTTP status of the response. Defaults to ``200``.
     :type status: integer
 
+    :param headers: the headers of this response. The dict values can be strings
+    (for single valued headers) or lists (for multi-valued headers).
+    :type headers: dict
+
     :param body: the response body. It must be str, not unicode, unless you're
        using a encoding-aware :ref:`Response subclass
        <topics-request-response-ref-response-subclasses>`, such as
        :class:`TextResponse`.
     :type body: str
 
-    :param meta: the initial values for the :attr:`Response.meta` attribute. If
-       given, the dict will be shallow copied.
-    :type meta: dict
-
     :param flags: is a list containing the initial values for the
        :attr:`Response.flags` attribute. If given, the list will be shallow
        copied.
     :type flags: list
 
+    :param request: the initial value of the :attr:`Response.request` attribute.
+        This represents the :class:`Request` that generated this response.
+    :type request: :class:`Request` object
+
     .. attribute:: Response.url
 
         A string containing the URL of the response.

From f68dc3026daf14c9ac1c303a90c0175eec92b73f Mon Sep 17 00:00:00 2001
From: Thom Dixon <thom@uber.com>
Date: Wed, 24 Aug 2016 09:11:27 -0700
Subject: [PATCH 1042/2974] Fix indentation

---
 docs/topics/request-response.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index dccea32ba..75b98d3b3 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -475,7 +475,7 @@ Response objects
     :type status: integer
 
     :param headers: the headers of this response. The dict values can be strings
-    (for single valued headers) or lists (for multi-valued headers).
+       (for single valued headers) or lists (for multi-valued headers).
     :type headers: dict
 
     :param body: the response body. It must be str, not unicode, unless you're

From 495d3226912bb84651b5c86183a4adad58c509cc Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 26 Aug 2016 20:16:22 +0500
Subject: [PATCH 1043/2974] DOC move Data Flow below the picture; add links to
 components

---
 docs/topics/architecture.rst | 98 +++++++++++++++++++++++-------------
 1 file changed, 63 insertions(+), 35 deletions(-)

diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index ba0e2c61c..39e54ee99 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -16,20 +16,71 @@ components and an outline of the data flow that takes place inside the system
 below with links for more detailed information about them. The data flow is
 also described below.
 
+.. _data-flow:
+
+Data flow
+=========
+
 .. image:: _images/scrapy_architecture_02.png
    :width: 700
    :height: 470
    :alt: Scrapy architecture
 
+The data flow in Scrapy is controlled by the execution engine, and goes like
+this:
+
+1. The :ref:`Engine <component-engine>` gets the first URLs to crawl from the
+   :ref:`Spider <component-spiders>`.
+
+2. The :ref:`Engine <component-engine>` schedules the URLs in the
+   :ref:`Scheduler <component-scheduler>` as Requests and asks for the
+   next URLs to crawl.
+
+3. The :ref:`Scheduler <component-scheduler>` returns the next URLs to crawl
+   to the :ref:`Engine <component-engine>`.
+
+4. The :ref:`Engine <component-engine>` sends the URLs to the
+   :ref:`Downloader <component-downloader>`, passing through the
+   :ref:`Downloader Middleware <component-downloader-middleware>`
+   (request direction).
+
+5. Once the page finishes downloading the
+   :ref:`Downloader <component-downloader>` generates a Response (with
+   that page) and sends it to the Engine, passing through the
+   :ref:`Downloader Middleware <component-downloader-middleware>`
+   (response direction).
+
+6. The :ref:`Engine <component-engine>` receives the Response from the
+   :ref:`Downloader <component-downloader>` and sends it to the
+   :ref:`Spider <component-spiders>` for processing, passing
+   through the :ref:`Spider Middleware <component-spider-middleware>`
+   (input direction).
+
+7. The :ref:`Spider <component-spiders>` processes the Response and returns
+   scraped items and new Requests (to follow) to the
+   :ref:`Engine <component-engine>`, passing through the
+   :ref:`Spider Middleware <component-spider-middleware>` (output direction).
+
+8. The :ref:`Engine <component-engine>` sends processed items to
+   :ref:`Item Pipelines <component-pipelines>` and processed Requests to
+   the :ref:`Scheduler <component-scheduler>`.
+
+9. The process repeats (from step 1) until there are no more requests from the
+   :ref:`Scheduler <component-scheduler>`.
+
 Components
 ==========
 
+.. _component-engine:
+
 Scrapy Engine
 -------------
 
 The engine is responsible for controlling the data flow between all components
-of the system, and triggering events when certain actions occur. See the Data
-Flow section below for more details.
+of the system, and triggering events when certain actions occur. See the
+:ref:`Data Flow <data-flow>` section above for more details.
+
+.. _component-scheduler:
 
 Scheduler
 ---------
@@ -37,12 +88,16 @@ Scheduler
 The Scheduler receives requests from the engine and enqueues them for feeding
 them later (also to the engine) when the engine requests them.
 
+.. _component-downloader:
+
 Downloader
 ----------
 
 The Downloader is responsible for fetching web pages and feeding them to the
 engine which, in turn, feeds them to the spiders.
 
+.. _component-spiders:
+
 Spiders
 -------
 
@@ -50,6 +105,8 @@ Spiders are custom classes written by Scrapy users to parse responses and
 extract items (aka scraped items) from them or additional URLs (requests) to
 follow. For more information see :ref:`topics-spiders`.
 
+.. _component-pipelines:
+
 Item Pipeline
 -------------
 
@@ -58,6 +115,8 @@ extracted (or scraped) by the spiders. Typical tasks include cleansing,
 validation and persistence (like storing the item in a database). For more
 information see :ref:`topics-item-pipeline`.
 
+.. _component-downloader-middleware:
+
 Downloader middlewares
 ----------------------
 
@@ -76,6 +135,8 @@ Use a Downloader middleware if you need to do one of the following:
 
 For more information see :ref:`topics-downloader-middleware`.
 
+.. _component-spider-middleware:
+
 Spider middlewares
 ------------------
 
@@ -93,39 +154,6 @@ Use a Spider middleware if you need to
 
 For more information see :ref:`topics-spider-middleware`.
 
-Data flow
-=========
-
-The data flow in Scrapy is controlled by the execution engine, and goes like
-this:
-
-1. The Engine gets the first URLs to crawl from the Spider.
-
-2. The Engine schedules the URLs in the Scheduler as Requests and asks for the
-   next URLs to crawl.
-
-3. The Scheduler returns the next URLs to crawl to the Engine.
-
-4. The Engine sends the URLs to the Downloader, passing through the
-   Downloader Middleware (request direction).
-
-5. Once the page finishes downloading the Downloader generates a Response (with
-   that page) and sends it to the Engine, passing through the Downloader
-   Middleware (response direction).
-
-6. The Engine receives the Response from the Downloader and sends it to the
-   Spider for processing, passing through the Spider Middleware (input direction).
-
-7. The Spider processes the Response and returns scraped items and new Requests
-   (to follow) to the Engine, passing through the Spider Middleware
-   (output direction).
-
-8. The Engine sends processed items to Item Pipelines and processed Requests to
-   the Scheduler.
-
-9. The process repeats (from step 1) until there are no more requests from the
-   Scheduler.
-
 Event-driven networking
 =======================
 

From 22e870e955995bf789290504292f123f760f5f1b Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 1 Sep 2016 10:19:49 +0200
Subject: [PATCH 1044/2974] Add Debian Jessie test env

---
 tox.ini | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tox.ini b/tox.ini
index f6de64b27..812302b4c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -33,6 +33,20 @@ deps =
     zope.interface==3.6.1
     -rtests/requirements.txt
 
+[testenv:jessie]
+# https://packages.debian.org/en/jessie/python/
+# https://packages.debian.org/en/jessie/zope/
+basepython = python2.7
+deps =
+    pyOpenSSL==0.14
+    lxml==3.4.0
+    Twisted==14.0.2
+    boto==2.34.0
+    Pillow==2.6.1
+    cssselect==0.9.1
+    zope.interface==4.1.1
+    -rtests/requirements.txt
+
 [testenv:trunk]
 basepython = python2.7
 commands =

From 2b2bfcea88a3b5c98a47adbd5c8d3979e6a5626d Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 1 Sep 2016 10:20:49 +0200
Subject: [PATCH 1045/2974] Add "jessie" build to Travis-CI config

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index c58ab39a5..59657b82e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,6 +9,7 @@ branches:
 env:
  - TOXENV=py27
  - TOXENV=precise
+ - TOXENV=jessie
  - TOXENV=py33
  - TOXENV=py35
  - TOXENV=docs

From 58cd7bf895321c39493cfd77feb4c07b7e614259 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 1 Sep 2016 11:17:53 +0200
Subject: [PATCH 1046/2974] Remove "precise" test env from Travis-CI config

---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 59657b82e..506f3779b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,6 @@ branches:
     - /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/
 env:
  - TOXENV=py27
- - TOXENV=precise
  - TOXENV=jessie
  - TOXENV=py33
  - TOXENV=py35

From b188f61b95d92390193bacfecda141cbf89197c9 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 1 Sep 2016 17:38:38 +0200
Subject: [PATCH 1047/2974] Update release notes for upcoming 1.2.0 version

---
 docs/news.rst | 43 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 5395db8e3..e514d9a5a 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,49 @@
 Release notes
 =============
 
+1.2.0 (YYYY-MM-DD)
+------------------
+
+New Features
+~~~~~~~~~~~~
+
+- New :setting:`FEED_EXPORT_ENCODING` setting to customize the encoding
+  used when writing items to a file. This is useful for those wanting
+  something else than UTF-8 for XML or CSV output (:issue:`2034`).
+- ``startproject`` command now supports an optional destination directory
+  to override the default one based on the project name (:issue:`2005`).
+- New :setting:`SCHEDULER_DEBUG` setting to log requests serialization
+  failures (:issue:`1610`).
+- JSON encoder now supports serialization of ``set`` instances (:issue:`2058`).
+
+Bug fixes
+~~~~~~~~~
+
+- DefaultRequestHeaders middleware now runs before UserAgent middleware
+  (:issue:`2088`). **Warning: this is technically backwards incompatible**,
+  though we consider this a bug fix.
+- ``Selector`` does not allow passing both ``response`` and ``text`` anymore
+  (:issue:`2153`).
+- Fixed logging of wrong callback name with ``scrapy parse`` (:issue:`2169`).
+- Fix for an odd gzip decompression bug (:issue:`1606`).
+
+Refactoring
+~~~~~~~~~~~
+
+- ``canonicalize_url`` has been moved to `w3lib.url`_ (:issue:`2168`).
+
+.. _w3lib.url: http://w3lib.readthedocs.io/en/latest/w3lib.html#w3lib.url.canonicalize_url
+
+Documentation
+~~~~~~~~~~~~~
+
+- Grammar fixes: :issue:`2128`, :issue:`1566`.
+- Download stats badge removed from README (:issue:`2160`).
+- New scrapy :ref:`architecture diagram <topics-architecture>` (:issue:`2165`)
+- Updated ``Response`` parameters documentation (:issue:`2197`).
+- Reworded misleading :setting:`RANDOMIZE_DOWNLOAD_DELAY` description (:issue:`2197`).
+
+
 1.1.2 (2016-08-18)
 ------------------
 

From 9cea6f07308fdbc041601aa35c097d9f6b045d73 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Fri, 2 Sep 2016 14:51:07 -0300
Subject: [PATCH 1048/2974] Add Segment Analytics to Documentation

---
 docs/_templates/layout.html | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 docs/_templates/layout.html

diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html
new file mode 100644
index 000000000..a6f6cbda8
--- /dev/null
+++ b/docs/_templates/layout.html
@@ -0,0 +1,16 @@
+{% extends "!layout.html" %}
+
+{% block footer %}
+{{ super() }}
+<script type="text/javascript">
+!function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","page","once","off","on"];analytics.factory=function(t){return function(){var e=Array.prototype.slice.call(arguments);e.unshift(t);analytics.push(e);return analytics}};for(var t=0;t<analytics.methods.length;t++){var e=analytics.methods[t];analytics[e]=analytics.factory(e)}analytics.load=function(t){var e=document.createElement("script");e.type="text/javascript";e.async=!0;e.src=("https:"===document.location.protocol?"https://":"http://")+"cdn.segment.com/analytics.js/v1/"+t+"/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(e,n)};analytics.SNIPPET_VERSION="3.1.0";
+analytics.load("8UDQfnf3cyFSTsM4YANnW5sXmgZVILbA");
+analytics.page();
+}}();
+
+analytics.ready(function () {
+    ga('require', 'linker');
+    ga('linker:autoLink', ['scrapinghub.com', 'crawlera.com']);
+});
+</script>
+{% endblock %}

From 960b1bc8f0335f34db9595532a93d2bef32df9a1 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 7 Sep 2016 04:54:32 +0500
Subject: [PATCH 1049/2974] typo fix in HttpProxyMiddleware

---
 scrapy/downloadermiddlewares/httpproxy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index b01bab76d..98c87aa9c 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -43,7 +43,7 @@ class HttpProxyMiddleware(object):
         return creds, proxy_url
 
     def process_request(self, request, spider):
-        # ignore if proxy is already seted
+        # ignore if proxy is already set
         if 'proxy' in request.meta:
             return
 

From 0ef570f6f0a152d5dc525868b1eb6fef3084823d Mon Sep 17 00:00:00 2001
From: Matti Remes <matti.remes@gmail.com>
Date: Sat, 3 Sep 2016 17:36:47 +0300
Subject: [PATCH 1050/2974] Update exceptions.rst

Added the missing dot. (+1 squashed commit)
Squashed commits:
[2198972] Update exceptions.rst

There are namely no constructors in classes in Python but an ``__init__`` method instead.
---
 docs/topics/exceptions.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index 9f8d16d84..cc02369d4 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -60,7 +60,7 @@ remain disabled. Those components include:
  * Downloader middlewares
  * Spider middlewares
 
-The exception must be raised in the component constructor.
+The exception must be raised in the component's ``__init__`` method.
 
 NotSupported
 ------------

From 743a0aa422ae2f515cbea69153e60a02c39da98d Mon Sep 17 00:00:00 2001
From: Joakim Uddholm <joakim@uddholm.com>
Date: Thu, 8 Sep 2016 21:52:14 +0200
Subject: [PATCH 1051/2974] Two fixes for when using the parse command and the
 '-r' flag (rules). 1. Use default "parse" as callback when the matching rule
 has no callback. 2. Log error and return when no rule matches the parsed url.

---
 scrapy/commands/parse.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 6a8978415..5264982b6 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -121,8 +121,8 @@ class Command(ScrapyCommand):
     def get_callback_from_rules(self, spider, response):
         if getattr(spider, 'rules', None):
             for rule in spider.rules:
-                if rule.link_extractor.matches(response.url) and rule.callback:
-                    return rule.callback
+                if rule.link_extractor.matches(response.url):
+                    return rule.callback or "parse"
         else:
             logger.error('No CrawlSpider rules found in spider %(spider)r, '
                          'please specify a callback to use for parsing',
@@ -166,6 +166,11 @@ class Command(ScrapyCommand):
             if not cb:
                 if opts.rules and self.first_response == response:
                     cb = self.get_callback_from_rules(spider, response)
+
+                    if not cb:
+                        logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',
+                                 {'url': response.url, 'spider': spider.name})
+                        return
                 else:
                     cb = 'parse'
 

From 80260824c65512edc3f60aa7eb8fead9818b1a90 Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Mon, 12 Sep 2016 00:43:58 +0300
Subject: [PATCH 1052/2974] Fix completion in `scrapy shell` for new imports

---
 scrapy/utils/console.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index a712df30e..567fd51bc 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -13,7 +13,9 @@ def _embed_ipython_shell(namespace={}, banner=''):
     @wraps(_embed_ipython_shell)
     def wrapper(namespace=namespace, banner=''):
         config = load_default_config()
-        shell = InteractiveShellEmbed(
+        # Always use .instace() to ensure _instance propagation to all parents
+        # this is needed for <TAB> completion works well for new imports
+        shell = InteractiveShellEmbed.instance(
             banner1=banner, user_ns=namespace, config=config)
         shell()
     return wrapper

From fbb555929977f91eabfa5dc28f3ae3d68972371a Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 12 Sep 2016 13:35:14 +0200
Subject: [PATCH 1053/2974] Add tests for crawl command non-default cases

---
 tests/test_commands.py | 82 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 82 insertions(+)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index d25045cb9..d13024922 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -257,6 +257,9 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
         with open(fname, 'w') as f:
             f.write("""
 import scrapy
+from scrapy.linkextractors import LinkExtractor
+from scrapy.spiders import CrawlSpider, Rule
+
 
 class MySpider(scrapy.Spider):
     name = '{0}'
@@ -265,6 +268,33 @@ class MySpider(scrapy.Spider):
         if getattr(self, 'test_arg', None):
             self.logger.debug('It Works!')
         return [scrapy.Item(), dict(foo='bar')]
+
+
+class MyGoodCrawlSpider(CrawlSpider):
+    name = 'goodcrawl{0}'
+
+    rules = (
+        Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True),
+        Rule(LinkExtractor(allow=r'/text'), follow=True),
+    )
+
+    def parse_item(self, response):
+        return [scrapy.Item(), dict(foo='bar')]
+
+    def parse(self, response):
+        return [scrapy.Item(), dict(nomatch='default')]
+
+
+class MyBadCrawlSpider(CrawlSpider):
+    '''Spider which doesn't define a parse_item callback while using it in a rule.'''
+    name = 'badcrawl{0}'
+
+    rules = (
+        Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True),
+    )
+
+    def parse(self, response):
+        return [scrapy.Item(), dict(foo='bar')]
 """.format(self.spider_name))
 
         fname = abspath(join(self.proj_mod_path, 'pipelines.py'))
@@ -309,6 +339,58 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         )
         self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
 
+    @defer.inlineCallbacks
+    def test_parse_items_no_callback_passed(self):
+        status, out, stderr = yield self.execute(
+            ['--spider', self.spider_name, self.url('/html')]
+        )
+        self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
+
+    @defer.inlineCallbacks
+    def test_wrong_callback_passed(self):
+        status, out, stderr = yield self.execute(
+            ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')]
+        )
+        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
+
+    @defer.inlineCallbacks
+    def test_crawlspider_matching_rule_callback_set(self):
+        """If a rule matches the URL, use it's defined callback."""
+        status, out, stderr = yield self.execute(
+            ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')]
+        )
+        self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
+
+    @defer.inlineCallbacks
+    def test_crawlspider_matching_rule_default_callback(self):
+        """If a rule match but it has no callback set, use the 'parse' callback."""
+        status, out, stderr = yield self.execute(
+            ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')]
+        )
+        self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out))
+
+    @defer.inlineCallbacks
+    def test_spider_with_no_rules_attribute(self):
+        """Using -r with a spider with no rule should not produce items."""
+        status, out, stderr = yield self.execute(
+            ['--spider', self.spider_name, '-r', self.url('/html')]
+        )
+        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
+
+    @defer.inlineCallbacks
+    def test_crawlspider_missing_callback(self):
+        status, out, stderr = yield self.execute(
+            ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')]
+        )
+        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
+
+    @defer.inlineCallbacks
+    def test_crawlspider_no_matching_rule(self):
+        """The requested URL has no matching rule, so no items should be scraped"""
+        status, out, stderr = yield self.execute(
+            ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')]
+        )
+        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
 
 
 class BenchCommandTest(CommandTest):

From 10f8c52f5d3023a9f60fb440b7b1fca3274c29e2 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Fri, 9 Sep 2016 10:35:28 -0300
Subject: [PATCH 1054/2974] changed tutorial examples from dmoz to
 quotes.toscrape.com

---
 docs/intro/tutorial.rst | 301 +++++++++++++++++++---------------------
 1 file changed, 141 insertions(+), 160 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 6ecd637c3..262d6b3a4 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -7,7 +7,7 @@ Scrapy Tutorial
 In this tutorial, we'll assume that Scrapy is already installed on your system.
 If that's not the case, see :ref:`intro-install`.
 
-We are going to use `Open directory project (dmoz) <https://www.dmoz.org/>`_ as
+We are going to use `quotes.toscrape.com <http://quotes.toscrape.com/>`_ as
 our example domain to scrape.
 
 This tutorial will walk you through these tasks:
@@ -16,8 +16,7 @@ This tutorial will walk you through these tasks:
 2. Defining the Items you will extract
 3. Writing a :ref:`spider <topics-spiders>` to crawl a site and extract
    :ref:`Items <topics-items>`
-4. Writing an :ref:`Item Pipeline <topics-item-pipeline>` to store the
-   extracted Items
+4. Exporting the scraped data using command line
 
 Scrapy is written in Python_. If you're new to the language you might want to
 start by getting an idea of what the language is like, to get the most out of
@@ -54,7 +53,6 @@ This will create a ``tutorial`` directory with the following contents::
 
             spiders/          # a directory where you'll later put your spiders
                 __init__.py
-                ...
 
 
 Defining our Item
@@ -72,16 +70,15 @@ its attributes as :class:`scrapy.Field <scrapy.item.Field>` objects, much like i
 easy task).
 
 We begin by modeling the item that we will use to hold the site's data obtained
-from dmoz.org. As we want to capture the name, url and description of the
-sites, we define fields for each of these three attributes. To do that, we edit
+from quotes.toscrape.com. As we want to capture the text and author from each of
+the quotes listed there, we define fields for each of these three attributes. To do that, we edit
 ``items.py``, found in the ``tutorial`` directory. Our Item class looks like this::
 
     import scrapy
 
-    class DmozItem(scrapy.Item):
-        title = scrapy.Field()
-        link = scrapy.Field()
-        desc = scrapy.Field()
+    class QuoteItem(scrapy.Item):
+        text = scrapy.Field()
+        author = scrapy.Field()
 
 This may seem complicated at first, but defining an item class allows you to use other handy
 components and helpers within Scrapy.
@@ -99,10 +96,11 @@ To create a Spider, you must subclass :class:`scrapy.Spider
 <scrapy.spiders.Spider>` and define some attributes:
 
 * :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be
-  unique, that is, you can't set the same name for different Spiders.
+  unique within a project, that is, you can't set the same name for different
+  Spiders.
 
 * :attr:`~scrapy.spiders.Spider.start_urls`: a list of URLs where the
-  Spider will begin to crawl from.  The first pages downloaded will be those
+  Spider will begin to crawl from. The first pages downloaded will be those
   listed here. The subsequent URLs will be generated successively from data
   contained in the start URLs.
 
@@ -119,20 +117,20 @@ To create a Spider, you must subclass :class:`scrapy.Spider
   objects) and more URLs to follow (as :class:`~scrapy.http.Request` objects).
 
 This is the code for our first Spider; save it in a file named
-``dmoz_spider.py`` under the ``tutorial/spiders`` directory::
+``quotes_spider.py`` under the ``tutorial/spiders`` directory::
 
     import scrapy
 
-    class DmozSpider(scrapy.Spider):
-        name = "dmoz"
-        allowed_domains = ["dmoz.org"]
+
+    class QuotesSpider(scrapy.Spider):
+        name = "quotes"
         start_urls = [
-            "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
-            "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
+            'http://quotes.toscrape.com/page/1/',
+            'http://quotes.toscrape.com/page/2/',
         ]
 
         def parse(self, response):
-            filename = response.url.split("/")[-2] + '.html'
+            filename = 'quotes-' + response.url.split("/")[-2] + '.html'
             with open(filename, 'wb') as f:
                 f.write(response.body)
 
@@ -141,24 +139,25 @@ Crawling
 
 To put our spider to work, go to the project's top level directory and run::
 
-   scrapy crawl dmoz
+   scrapy crawl quotes
 
-This command runs the spider with name ``dmoz`` that we've just added, that
-will send some requests for the ``dmoz.org`` domain. You will get an output
+This command runs the spider with name ``quotes`` that we've just added, that
+will send some requests for the ``quotes.toscrape.com`` domain. You will get an output
 similar to this::
 
-    2014-01-23 18:13:07-0400 [scrapy] INFO: Scrapy started (bot: tutorial)
-    2014-01-23 18:13:07-0400 [scrapy] INFO: Optional features available: ...
-    2014-01-23 18:13:07-0400 [scrapy] INFO: Overridden settings: {}
-    2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled extensions: ...
-    2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled downloader middlewares: ...
-    2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled spider middlewares: ...
-    2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled item pipelines: ...
-    2014-01-23 18:13:07-0400 [scrapy] INFO: Spider opened
-    2014-01-23 18:13:08-0400 [scrapy] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/> (referer: None)
-    2014-01-23 18:13:09-0400 [scrapy] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> (referer: None)
-    2014-01-23 18:13:09-0400 [scrapy] INFO: Closing spider (finished)
 
+    2016-09-01 16:51:27 [scrapy] INFO: Scrapy started (bot: tutorial)
+    2016-09-01 16:51:27 [scrapy] INFO: Overridden settings: {...}
+    2016-09-01 16:51:27 [scrapy] INFO: Enabled extensions: ...
+    2016-09-01 16:51:27 [scrapy] INFO: Enabled downloader middlewares: ...
+    2016-09-01 16:51:27 [scrapy] INFO: Enabled spider middlewares: ...
+    2016-09-01 16:51:27 [scrapy] INFO: Enabled item pipelines: ...
+    2016-09-01 16:51:27 [scrapy] INFO: Spider opened
+    2016-09-01 16:51:27 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
+    2016-09-01 16:51:28 [scrapy] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
+    2016-09-01 16:51:28 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
+    2016-09-01 16:51:29 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
+    2016-09-01 16:51:29 [scrapy] INFO: Closing spider (finished)
 
 .. note::
     At the end you can see a log line for each URL defined in ``start_urls``.
@@ -166,7 +165,7 @@ similar to this::
     shown at the end of the log line, where it says ``(referer: None)``.
 
 Now, check the files in the current directory. You should notice two new files
-have been created: *Books.html* and *Resources.html*, with the content for the respective
+have been created: *quotes-1.html* and *quotes-2.html*, with the content for the respective
 URLs, as our ``parse`` method instructs.
 
 What just happened under the hood?
@@ -197,15 +196,16 @@ mechanisms see the :ref:`Selectors documentation <topics-selectors>`.
 Here are some examples of XPath expressions and their meanings:
 
 * ``/html/head/title``: selects the ``<title>`` element, inside the ``<head>``
-  element of an HTML document
+  element of an HTML document. Equivalent CSS selector: ``html > head > title``.
 
 * ``/html/head/title/text()``: selects the text inside the aforementioned
-  ``<title>`` element.
+  ``<title>`` element. Equivalent CSS selector: ``html > head > title ::text``.
 
-* ``//td``: selects all the ``<td>`` elements
+* ``//td``: selects all the ``<td>`` elements from the whole document.
+  Equivalent CSS selector: ``td``.
 
 * ``//div[@class="mine"]``: selects all ``div`` elements which contain an
-  attribute ``class="mine"``
+  attribute ``class="mine"``. Equivalent CSS selector: ``div.mine``.
 
 These are just a couple of simple examples of what you can do with XPath, but
 XPath expressions are indeed much more powerful. To learn more about XPath, we
@@ -220,7 +220,7 @@ to think in XPath" <http://plasmasturm.org/log/xpath101/>`_.
   Because of this, we encourage you to learn about XPath even if you
   already know how to construct CSS selectors.
 
-For working with CSS and XPath expressions, Scrapy provides
+For working with CSS and XPath expressions, Scrapy provides the
 :class:`~scrapy.selector.Selector` class and convenient shortcuts to avoid
 instantiating selectors yourself every time you need to select something from a
 response.
@@ -255,7 +255,7 @@ installed on your system.
 
 To start a shell, you must go to the project's top level directory and run::
 
-    scrapy shell "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/"
+    scrapy shell "http://quotes.toscrape.com"
 
 .. note::
 
@@ -267,20 +267,20 @@ This is what the shell looks like::
 
     [ ... Scrapy log here ... ]
 
-    2014-01-23 17:11:42-0400 [scrapy] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> (referer: None)
+    2016-09-01 18:14:39 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com> (referer: None)
     [s] Available Scrapy objects:
-    [s]   crawler    <scrapy.crawler.Crawler object at 0x3636b50>
+    [s]   crawler    <scrapy.crawler.Crawler object at 0x109001c90>
     [s]   item       {}
-    [s]   request    <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
-    [s]   response   <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
-    [s]   settings   <scrapy.settings.Settings object at 0x3fadc50>
-    [s]   spider     <Spider 'default' at 0x3cebf50>
+    [s]   request    <GET http://quotes.toscrape.com>
+    [s]   response   <200 http://quotes.toscrape.com>
+    [s]   settings   <scrapy.settings.Settings object at 0x109001610>
+    [s]   spider     <DefaultSpider 'default' at 0x1092808d0>
     [s] Useful shortcuts:
     [s]   shelp()           Shell help (print this help)
     [s]   fetch(req_or_url) Fetch request (or URL) and update local objects
     [s]   view(response)    View response in a browser
-
-    In [1]:
+    
+    >>>
 
 After the shell loads, you will have the response fetched in a local
 ``response`` variable, so if you type ``response.body`` you will see the body
@@ -297,19 +297,19 @@ or ``response.css()`` which map directly to ``response.selector.xpath()`` and
 So let's try it::
 
     In [1]: response.xpath('//title')
-    Out[1]: [<Selector xpath='//title' data=u'<title>Open Directory - Computers: Progr'>]
- 
+    Out[1]: [<Selector xpath='//title' data=u'<title>Quotes to Scrape</title>'>] 
+    
     In [2]: response.xpath('//title').extract()
-    Out[2]: [u'<title>Open Directory - Computers: Programming: Languages: Python: Books</title>']
- 
+    Out[2]: [u'<title>Quotes to Scrape</title>']
+    
     In [3]: response.xpath('//title/text()')
-    Out[3]: [<Selector xpath='//title/text()' data=u'Open Directory - Computers: Programming:'>]
- 
+    Out[3]: [<Selector xpath='//title/text()' data=u'Quotes to Scrape'>]
+
     In [4]: response.xpath('//title/text()').extract()
-    Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books']
- 
-    In [5]: response.xpath('//title/text()').re('(\w+):')
-    Out[5]: [u'Computers', u'Programming', u'Languages', u'Python']
+    Out[4]: [u'Quotes to Scrape']
+    
+    In [11]: response.xpath('//title/text()').re('(\w+)')
+    Out[11]: [u'Quotes', u'to', u'Scrape']
 
 Extracting the data
 ^^^^^^^^^^^^^^^^^^^
@@ -322,35 +322,42 @@ there could become a very tedious task. To make it easier, you can
 use Firefox Developer Tools or some Firefox extensions like Firebug. For more
 information see :ref:`topics-firebug` and :ref:`topics-firefox`.
 
-After inspecting the page source, you'll find that the web site's information
-is inside a ``<ul>`` element, in fact the *second* ``<ul>`` element.
+After inspecting the page source, you'll find that every quote in the website
+is inside a separate ``<div class="quote">`` element, such as:
 
-So we can select each ``<li>`` element belonging to the site's list with this
-code::
+    <div class="quote">
+        <span class="text">“We accept the love we think we deserve.”</span>
+        <span>by <small class="author">Stephen Chbosky</small></span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords"> 
+            <a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
+            <a class="tag" href="/tag/love/page/1/">love</a>
+        </div>
+    </div>
 
-    response.xpath('//ul/li')
 
-And from them, the site's descriptions::
+So we can select each ``<div class="quote">`` element belonging to the site's 
+list with this code::
 
-    response.xpath('//ul/li/text()').extract()
+    response.xpath('//div[@class="quote"]')
 
-The site's titles::
+From each quote block, we can select text with::
 
-    response.xpath('//ul/li/a/text()').extract()
+    response.xpath('//div[@class="quote"]/span[@class="text"]/text()').extract()
 
-And the site's links::
+The authors::
 
-    response.xpath('//ul/li/a/@href').extract()
+    response.xpath('//div[@class="quote"]/span/small/text()').extract()
 
 As we've said before, each ``.xpath()`` call returns a list of selectors, so we can
 concatenate further ``.xpath()`` calls to dig deeper into a node. We are going to use
 that property here, so::
 
-    for sel in response.xpath('//ul/li'):
-        title = sel.xpath('a/text()').extract()
-        link = sel.xpath('a/@href').extract()
-        desc = sel.xpath('text()').extract()
-        print title, link, desc
+    for quote in response.xpath('//div[@class="quote"]'):
+        text = quote.xpath('span[@class="text"]/text()').extract()
+        author = quote.xpath('span/small/text()').extract()
+        print('{}: {}'.format(author, text))
 
 .. note::
 
@@ -362,26 +369,25 @@ that property here, so::
 Let's add this code to our spider::
 
     import scrapy
-     
-    class DmozSpider(scrapy.Spider):
-        name = "dmoz"
-        allowed_domains = ["dmoz.org"]
-        start_urls = [
-            "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
-            "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
-        ]
-     
-        def parse(self, response):
-            for sel in response.xpath('//ul/li'):
-                title = sel.xpath('a/text()').extract()
-                link = sel.xpath('a/@href').extract()
-                desc = sel.xpath('text()').extract()
-                print title, link, desc
 
-Now try crawling dmoz.org again and you'll see sites being printed
+
+    class QuotesSpider(scrapy.Spider):
+        name = "quotes"
+        start_urls = [
+            'http://quotes.toscrape.com/page/1/',
+            'http://quotes.toscrape.com/page/2/',
+        ]
+
+        def parse(self, response):
+            for quote in response.xpath('//div[@class="quote"]'):
+                text = quote.xpath('span[@class="text"]/text()').extract()
+                author = quote.xpath('span/small/text()').extract()
+                print('{}: {}'.format(author, text))
+
+Now try crawling quotes.toscrape.com again and you'll see sites being printed
 in your output. Run::
 
-    scrapy crawl dmoz
+    scrapy crawl quotes
 
 Using our item
 --------------
@@ -389,91 +395,83 @@ Using our item
 :class:`~scrapy.item.Item` objects are custom Python dicts; you can access the
 values of their fields (attributes of the class we defined earlier) using the
 standard dict syntax like::
-
-    >>> item = DmozItem()
-    >>> item['title'] = 'Example title'
+    
+    >>> from tutorial.items import QuoteItem
+    >>> item = QuoteItem()
+    >>> item['text'] = 'Some random quote'
     >>> item['title']
-    'Example title'
+    'Some random quote'
 
 So, in order to return the data we've scraped so far, the final code for our
 Spider would be like this::
 
     import scrapy
+    from tutorial.items import QuoteItem
 
-    from tutorial.items import DmozItem
 
-    class DmozSpider(scrapy.Spider):
-        name = "dmoz"
-        allowed_domains = ["dmoz.org"]
+    class QuotesSpider(scrapy.Spider):
+        name = "quotes"
         start_urls = [
-            "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
-            "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
+            'http://quotes.toscrape.com/page/1/',
+            'http://quotes.toscrape.com/page/2/',
         ]
 
         def parse(self, response):
-            for sel in response.xpath('//ul/li'):
-                item = DmozItem()
-                item['title'] = sel.xpath('a/text()').extract()
-                item['link'] = sel.xpath('a/@href').extract()
-                item['desc'] = sel.xpath('text()').extract()
+            for quote in response.xpath('//div[@class="quote"]'):
+                item = QuoteItem()
+                item['text'] = quote.xpath('span[@class="text"]/text()').extract()
+                item['author'] = quote.xpath('span/small/text()').extract()
                 yield item
 
-.. note:: You can find a fully-functional variant of this spider in the dirbot_
-   project available at https://github.com/scrapy/dirbot
 
-Now crawling dmoz.org yields ``DmozItem`` objects::
+Now crawling quotes.toscrape.com yields ``QuoteItem`` objects::
 
-   [scrapy] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
-        {'desc': [u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.\n],
-         'link': [u'http://gnosis.cx/TPiP/'],
-         'title': [u'Text Processing in Python']}
-   [scrapy] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
-        {'desc': [u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n'],
-         'link': [u'http://www.informit.com/store/product.aspx?isbn=0130211192'],
-         'title': [u'XML Processing with Python']}
+    2016-09-02 16:35:20 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
+    {'author': ['Oscar Wilde'],
+     'text': ['“We are all in the gutter, but some of us are looking at the '
+              'stars.”']}
+    2016-09-02 16:35:20 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
+    {'author': ['Mark Twain'],
+     'text': ['“The man who does not read has no advantage over the man who cannot '
+              'read.”']}
 
 
 Following links
 ===============
 
-Let's say, instead of just scraping the stuff in *Books* and *Resources* pages,
-you want everything that is under the `Python directory
-<http://www.dmoz.org/Computers/Programming/Languages/Python/>`_.
+Let's say, instead of just scraping the stuff from the first two pages
+from quotes.toscrape.com, you want quotes from all the pages in the website.
 
-Now that you know how to extract data from a page, why not extract the links
-for the pages you are interested, follow them and then extract the data you
+Now that you know how to extract data from a page, why not extract the
+pagination links in each page, follow them and then extract the data you
 want for all of them?
 
 Here is a modification to our spider that does just that::
 
     import scrapy
+    from tutorial.items import QuoteItem
 
-    from tutorial.items import DmozItem
 
-    class DmozSpider(scrapy.Spider):
-        name = "dmoz"
-        allowed_domains = ["dmoz.org"]
+    class QuotesSpider(scrapy.Spider):
+        name = "quotes"
         start_urls = [
-            "http://www.dmoz.org/Computers/Programming/Languages/Python/",
+            'http://quotes.toscrape.com/page/1/',
         ]
 
         def parse(self, response):
-            for href in response.css("ul.directory.dir-col > li > a::attr('href')"):
-                url = response.urljoin(href.extract())
-                yield scrapy.Request(url, callback=self.parse_dir_contents)
-
-        def parse_dir_contents(self, response):
-            for sel in response.xpath('//ul/li'):
-                item = DmozItem()
-                item['title'] = sel.xpath('a/text()').extract()
-                item['link'] = sel.xpath('a/@href').extract()
-                item['desc'] = sel.xpath('text()').extract()
+            for quote in response.xpath('//div[@class="quote"]'):
+                item = QuoteItem()
+                item['text'] = quote.xpath('span[@class="text"]/text()').extract()
+                item['author'] = quote.xpath('span/small/text()').extract()
                 yield item
+            next_page = response.xpath('//li[@class="next"]/a/@href').extract_first()
+            if next_page:
+                next_page = response.urljoin(next_page)
+                yield scrapy.Request(next_page, callback=self.parse)
 
-Now the `parse()` method only extracts the interesting links from the page,
+Now after extracting an item the `parse()` method looks for the link to the next page, 
 builds a full absolute URL using the `response.urljoin` method (since the links can
-be relative) and yields new requests to be sent later, registering as callback
-the method `parse_dir_contents()` that will ultimately scrape the data we want.
+be relative) and yields a new request to the next page, registering itself as callback to handle the data extraction for the next page and to keep the crawling going through all the pages.
 
 What you see here is Scrapy's mechanism of following links: when you yield
 a Request in a callback method, Scrapy will schedule that request to be sent
@@ -483,25 +481,8 @@ Using this, you can build complex crawlers that follow links according to rules
 you define, and extract different kinds of data depending on the page it's
 visiting.
 
-A common pattern is a callback method that extracts some items, looks for a link
-to follow to the next page and then yields a `Request` with the same callback
-for it::
-
-    def parse_articles_follow_next_page(self, response):
-        for article in response.xpath("//article"):
-            item = ArticleItem()
-
-            ... extract article data here
-
-            yield item
-
-        next_page = response.css("ul.navigation > li.next-page > a::attr('href')")
-        if next_page:
-            url = response.urljoin(next_page[0].extract())
-            yield scrapy.Request(url, self.parse_articles_follow_next_page)
-
-This creates a sort of loop, following all the links to the next page until it
-doesn't find one -- handy for crawling blogs, forums and other sites with
+In our example, it creates a sort of loop, following all the links to the next page
+until it doesn't find one -- handy for crawling blogs, forums and other sites with
 pagination.
 
 Another common pattern is to build an item with data from more than one page,
@@ -521,7 +502,7 @@ Storing the scraped data
 The simplest way to store the scraped data is by using :ref:`Feed exports
 <topics-feed-exports>`, with the following command::
 
-    scrapy crawl dmoz -o items.json
+    scrapy crawl quotes -o items.json
 
 That will generate an ``items.json`` file containing all scraped items,
 serialized in `JSON`_.

From 498a3725d18c5f8f5fd63a69ceffe9de9986aa28 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 14 Sep 2016 12:19:50 +0200
Subject: [PATCH 1055/2974] Add flush() method to StreamLogger

Fixes GH-2125
---
 scrapy/utils/log.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index cc2f0b164..a28002c08 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -145,6 +145,9 @@ class StreamLogger(object):
         for line in buf.rstrip().splitlines():
             self.logger.log(self.log_level, line.rstrip())
 
+    def flush(self):
+        pass
+
 
 class LogCounterHandler(logging.Handler):
     """Record log levels count into a crawler stats"""

From bc67cd9edd825abb6c918292cbca25bc76f1e923 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Wed, 14 Sep 2016 12:39:29 -0300
Subject: [PATCH 1056/2974] fix indentation issue

---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 262d6b3a4..f3f1bc645 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -323,7 +323,7 @@ use Firefox Developer Tools or some Firefox extensions like Firebug. For more
 information see :ref:`topics-firebug` and :ref:`topics-firefox`.
 
 After inspecting the page source, you'll find that every quote in the website
-is inside a separate ``<div class="quote">`` element, such as:
+is inside a separate ``<div class="quote">`` element, such as::
 
     <div class="quote">
         <span class="text">“We accept the love we think we deserve.”</span>

From a9a96bed8f5ec9e0e00d678e9524c5c2baf308de Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 14 Sep 2016 18:09:39 -0300
Subject: [PATCH 1057/2974] updated tutorial as per review comments

---
 docs/intro/tutorial.rst | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index f3f1bc645..f802c4e49 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -342,7 +342,7 @@ list with this code::
 
     response.xpath('//div[@class="quote"]')
 
-From each quote block, we can select text with::
+From the quote elements, we can select the texts with::
 
     response.xpath('//div[@class="quote"]/span[@class="text"]/text()').extract()
 
@@ -380,9 +380,12 @@ Let's add this code to our spider::
 
         def parse(self, response):
             for quote in response.xpath('//div[@class="quote"]'):
-                text = quote.xpath('span[@class="text"]/text()').extract()
-                author = quote.xpath('span/small/text()').extract()
-                print('{}: {}'.format(author, text))
+                text = quote.xpath('span[@class="text"]/text()').extract_first()
+                author = quote.xpath('span/small/text()').extract_first()
+                print(u'{}: {}'.format(author, text))
+
+Note how we've changed to use the method ``.extract_first()``, which extracts
+the first element from a selector list returned by ``.xpath()``.
 
 Now try crawling quotes.toscrape.com again and you'll see sites being printed
 in your output. Run::
@@ -419,21 +422,19 @@ Spider would be like this::
         def parse(self, response):
             for quote in response.xpath('//div[@class="quote"]'):
                 item = QuoteItem()
-                item['text'] = quote.xpath('span[@class="text"]/text()').extract()
-                item['author'] = quote.xpath('span/small/text()').extract()
+                item['text'] = quote.xpath('span[@class="text"]/text()').extract_first()
+                item['author'] = quote.xpath('span/small/text()').extract_first()
                 yield item
 
 
 Now crawling quotes.toscrape.com yields ``QuoteItem`` objects::
 
     2016-09-02 16:35:20 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
-    {'author': ['Oscar Wilde'],
-     'text': ['“We are all in the gutter, but some of us are looking at the '
-              'stars.”']}
+    {'author': 'Oscar Wilde',
+     'text': '“We are all in the gutter, but some of us are looking at the stars.”'}
     2016-09-02 16:35:20 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
-    {'author': ['Mark Twain'],
-     'text': ['“The man who does not read has no advantage over the man who cannot '
-              'read.”']}
+    {'author': 'Mark Twain',
+     'text': '“The man who does not read has no advantage over the man who cannot read.”'}
 
 
 Following links
@@ -461,8 +462,8 @@ Here is a modification to our spider that does just that::
         def parse(self, response):
             for quote in response.xpath('//div[@class="quote"]'):
                 item = QuoteItem()
-                item['text'] = quote.xpath('span[@class="text"]/text()').extract()
-                item['author'] = quote.xpath('span/small/text()').extract()
+                item['text'] = quote.xpath('span[@class="text"]/text()').extract_first()
+                item['author'] = quote.xpath('span/small/text()').extract_first()
                 yield item
             next_page = response.xpath('//li[@class="next"]/a/@href').extract_first()
             if next_page:

From 7d882095437a82085ee62cf55b9a2e1003a92145 Mon Sep 17 00:00:00 2001
From: pawelmhm <pawelmhm@gmail.com>
Date: Thu, 15 Sep 2016 09:30:09 +0200
Subject: [PATCH 1058/2974] [image & file pipeline] loading setting for user
 classes

if user has some custom subclass of Image pipeline and no setting for
this pipeline, he should get default settings defined for Image Pipeline.

Fixes #2198
---
 scrapy/pipelines/files.py     |  3 ++-
 scrapy/pipelines/images.py    |  3 ++-
 scrapy/pipelines/media.py     |  9 ++++++---
 tests/test_pipeline_files.py  | 25 ++++++++++++++++++++++---
 tests/test_pipeline_images.py | 27 ++++++++++++++++++++++-----
 5 files changed, 54 insertions(+), 13 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 8cdc548f6..843b4d3ec 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -233,7 +233,8 @@ class FilesPipeline(MediaPipeline):
         cls_name = "FilesPipeline"
         self.store = self._get_store(store_uri)
         resolve = functools.partial(self._key_for_pipe,
-                                    base_class_name=cls_name)
+                                    base_class_name=cls_name,
+                                    settings=settings)
         self.expires = settings.getint(
             resolve('FILES_EXPIRES'), self.EXPIRES
         )
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index af5825c0b..5796bfb80 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -55,7 +55,8 @@ class ImagesPipeline(FilesPipeline):
             settings = Settings(settings)
 
         resolve = functools.partial(self._key_for_pipe,
-                                    base_class_name="ImagesPipeline")
+                                    base_class_name="ImagesPipeline",
+                                    settings=settings)
         self.expires = settings.getint(
             resolve("IMAGES_EXPIRES"), self.EXPIRES
         )
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 82b4b462e..57f70499e 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -28,7 +28,8 @@ class MediaPipeline(object):
         self.download_func = download_func
 
 
-    def _key_for_pipe(self, key, base_class_name=None):
+    def _key_for_pipe(self, key, base_class_name=None,
+                      settings=None):
         """
         >>> MediaPipeline()._key_for_pipe("IMAGES")
         'IMAGES'
@@ -38,9 +39,11 @@ class MediaPipeline(object):
         'MYPIPE_IMAGES'
         """
         class_name = self.__class__.__name__
-        if class_name == base_class_name or not base_class_name:
+        formatted_key = "{}_{}".format(class_name.upper(), key)
+        if class_name == base_class_name or not base_class_name \
+            or (settings and not settings.get(formatted_key)):
             return key
-        return "{}_{}".format(class_name.upper(), key)
+        return formatted_key
 
     @classmethod
     def from_crawler(cls, crawler):
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index bda2a2199..157c21a89 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -255,16 +255,17 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
 
     def test_subclass_attrs_preserved_custom_settings(self):
         """
-        If file settings are defined but they are not defined for subclass class attributes
-        should be preserved.
+        If file settings are defined but they are not defined for subclass
+        settings should be preserved.
         """
         pipeline_cls = self._generate_fake_pipeline()
         settings = self._generate_fake_settings()
         pipeline = pipeline_cls.from_settings(Settings(settings))
         for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
             value = getattr(pipeline, pipe_ins_attr)
+            setting_value = settings.get(settings_attr)
             self.assertNotEqual(value, self.default_cls_settings[pipe_attr])
-            self.assertEqual(value, getattr(pipeline, pipe_attr))
+            self.assertEqual(value, setting_value)
 
     def test_no_custom_settings_for_subclasses(self):
         """
@@ -321,6 +322,24 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
         self.assertEqual(pipeline.files_urls_field, "that")
 
 
+    def test_user_defined_subclass_default_key_names(self):
+        """Test situation when user defines subclass of FilesPipeline,
+        but uses attribute names for default pipeline (without prefixing
+        them with pipeline class name).
+        """
+        settings = self._generate_fake_settings()
+
+        class UserPipe(FilesPipeline):
+            pass
+
+        pipeline_cls = UserPipe.from_settings(Settings(settings))
+
+        for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
+            expected_value = settings.get(settings_attr)
+            self.assertEqual(getattr(pipeline_cls, pipe_inst_attr),
+                             expected_value)
+
+
 class TestS3FilesStore(unittest.TestCase):
     @defer.inlineCallbacks
     def test_persist(self):
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 8286582de..6c1976b63 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -309,17 +309,19 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
 
     def test_subclass_attrs_preserved_custom_settings(self):
         """
-        If image settings are defined but they are not defined for subclass class attributes
-        should be preserved.
+        If image settings are defined but they are not defined for subclass default
+        values taken from settings should be preserved.
         """
         pipeline_cls = self._generate_fake_pipeline_subclass()
         settings = self._generate_fake_settings()
         pipeline = pipeline_cls.from_settings(Settings(settings))
         for pipe_attr, settings_attr in self.img_cls_attribute_names:
-            # Instance attribute (lowercase) must be equal to class attribute (uppercase).
+            # Instance attribute (lowercase) must be equal to
+            # value defined in settings.
             value = getattr(pipeline, pipe_attr.lower())
             self.assertNotEqual(value, self.default_pipeline_settings[pipe_attr])
-            self.assertEqual(value, getattr(pipeline, pipe_attr))
+            setings_value = settings.get(settings_attr)
+            self.assertEqual(value, setings_value)
 
     def test_no_custom_settings_for_subclasses(self):
         """
@@ -370,11 +372,26 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
         class UserDefinedImagePipeline(ImagesPipeline):
             DEFAULT_IMAGES_URLS_FIELD = "something"
             DEFAULT_IMAGES_RESULT_FIELD = "something_else"
-
         pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir}))
         self.assertEqual(pipeline.images_result_field, "something_else")
         self.assertEqual(pipeline.images_urls_field, "something")
 
+    def test_user_defined_subclass_default_key_names(self):
+        """Test situation when user defines subclass of ImagePipeline,
+        but uses attribute names for default pipeline (without prefixing
+        them with pipeline class name).
+        """
+        settings = self._generate_fake_settings()
+
+        class UserPipe(ImagesPipeline):
+            pass
+
+        pipeline_cls = UserPipe.from_settings(Settings(settings))
+
+        for pipe_attr, settings_attr in self.img_cls_attribute_names:
+            expected_value = settings.get(settings_attr)
+            self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()),
+                             expected_value)
 
 def _create_image(format, *a, **kw):
     buf = TemporaryFile()

From b828facff499e97f8d2dfec0c2fd5a76a5152237 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 15 Sep 2016 19:25:20 +0200
Subject: [PATCH 1059/2974] Add shell test for using scrapy.Request() directly
 without importing scrapy

---
 tests/test_command_shell.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index c532fc0d8..7bb7439d6 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -56,6 +56,13 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
         errcode, out, _ = yield self.execute(['-c', code.format(url)])
         self.assertEqual(errcode, 0, out)
 
+    @defer.inlineCallbacks
+    def test_scrapy_import(self):
+        url = self.url('/text')
+        code = "fetch(scrapy.Request('{0}'))"
+        errcode, out, _ = yield self.execute(['-c', code.format(url)])
+        self.assertEqual(errcode, 0, out)
+
     @defer.inlineCallbacks
     def test_local_file(self):
         filepath = join(tests_datadir, 'test_site/index.html')

From 105163fece502c7afd93d253398cfa6061ef9609 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 15 Sep 2016 19:26:53 +0200
Subject: [PATCH 1060/2974] Make scrapy available in shell without explicit
 import statement

---
 scrapy/shell.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scrapy/shell.py b/scrapy/shell.py
index 099e1af0a..4f9f06561 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -115,6 +115,9 @@ class Shell(object):
         self.populate_vars(response, request, spider)
 
     def populate_vars(self, response=None, request=None, spider=None):
+        import scrapy
+
+        self.vars['scrapy'] = scrapy
         self.vars['crawler'] = self.crawler
         self.vars['item'] = self.item_class()
         self.vars['settings'] = self.crawler.settings

From 18bd0b0886f6c09fcb03397b88ec5868fd6a9c03 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 15 Sep 2016 15:16:30 -0300
Subject: [PATCH 1061/2974] docs: update overview spider code to use
 toscrape.com and minor changes So, this will replace the spider example code
 from the overview that scrapes questions from StackOverflow by a spider
 scraping quotes (much like the one in the tutorial), and upates the text
 around it to be consistent.

There are also minor wording changes plus a small Sphinx/reST syntax fix
on the features list at the bottom (it was creating a definition list,
causing one line to be bold).
---
 docs/intro/overview.rst | 96 +++++++++++++++++++----------------------
 1 file changed, 44 insertions(+), 52 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 6c1291c1f..fb390d79d 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -19,74 +19,69 @@ Walk-through of an example spider
 In order to show you what Scrapy brings to the table, we'll walk you through an
 example of a Scrapy Spider using the simplest way to run a spider.
 
-So, here's the code for a spider that follows the links to the top
-voted questions on StackOverflow and scrapes some data from each page::
+Here's the code for a spider that scrapes famous quotes from website
+http://quotes.toscrape.com, following the pagination::
 
     import scrapy
 
 
-    class StackOverflowSpider(scrapy.Spider):
-        name = 'stackoverflow'
-        start_urls = ['http://stackoverflow.com/questions?sort=votes']
+    class QuotesSpider(scrapy.Spider):
+        name = "quotes"
+        start_urls = [
+            'http://quotes.toscrape.com/tag/humor/',
+        ]
 
         def parse(self, response):
-            for href in response.css('.question-summary h3 a::attr(href)'):
-                full_url = response.urljoin(href.extract())
-                yield scrapy.Request(full_url, callback=self.parse_question)
+            for quote in response.css('div.quote'):
+                yield {
+                    'text': quote.css('span.text::text').extract_first(),
+                    'author': quote.xpath('span/small/text()').extract_first(),
+                }
 
-        def parse_question(self, response):
-            yield {
-                'title': response.css('h1 a::text').extract_first(),
-                'votes': response.css('.question .vote-count-post::text').extract_first(),
-                'body': response.css('.question .post-text').extract_first(),
-                'tags': response.css('.question .post-tag::text').extract(),
-                'link': response.url,
-            }
+            next_page = response.css('li.next a::attr("href")').extract_first()
+            if next_page:
+                next_page = response.urljoin(next_page)
+                yield scrapy.Request(next_page, callback=self.parse)
 
 
-Put this in a file, name it to something like ``stackoverflow_spider.py``
+Put this in a text file, name it to something like ``quotes_spider.py``
 and run the spider using the :command:`runspider` command::
 
-    scrapy runspider stackoverflow_spider.py -o top-stackoverflow-questions.json
+    scrapy runspider quotes_spider.py -o quotes.json
 
 
-When this finishes you will have in the ``top-stackoverflow-questions.json`` file
-a list of the most upvoted questions in StackOverflow in JSON format, containing the
-title, link, number of upvotes, a list of the tags and the question content in HTML,
-looking like this (reformatted for easier reading)::
+When this finishes you will have in the ``quotes.json`` file a list of the
+quotes in JSON format, containing text and author, looking like this (reformatted
+here for better readability)::
 
     [{
-        "body": "... LONG HTML HERE ...",
-        "link": "http://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster-than-an-unsorted-array",
-        "tags": ["java", "c++", "performance", "optimization"],
-        "title": "Why is processing a sorted array faster than an unsorted array?",
-        "votes": "9924"
+        "author": "Jane Austen",
+        "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"
     },
     {
-        "body": "... LONG HTML HERE ...",
-        "link": "http://stackoverflow.com/questions/1260748/how-do-i-remove-a-git-submodule",
-        "tags": ["git", "git-submodules"],
-        "title": "How do I remove a Git submodule?",
-        "votes": "1764"
+        "author": "Groucho Marx",
+        "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d"
+    },
+    {
+        "author": "Steve Martin",
+        "text": "\u201cA day without sunshine is like, you know, night.\u201d"
     },
     ...]
 
 
-
 What just happened?
 -------------------
 
-When you ran the command ``scrapy runspider somefile.py``, Scrapy looked for a
+When you ran the command ``scrapy runspider quotes_spider.py``, Scrapy looked for a
 Spider definition inside it and ran it through its crawler engine.
 
 The crawl started by making requests to the URLs defined in the ``start_urls``
-attribute (in this case, only the URL for StackOverflow top questions page)
+attribute (in this case, only the URL for quotes in *humor* category)
 and called the default callback method ``parse``, passing the response object as
-an argument. In the ``parse`` callback we extract the links to the
-question pages using a CSS Selector with a custom extension that allows to get
-the value for an attribute. Then we yield a few more requests to be sent,
-registering the method ``parse_question`` as the callback to be called for each
-of them as they finish.
+an argument. In the ``parse`` callback we loop through the quote elements
+using a CSS Selector, yield a Python dict with the extracted quote text and author,
+look for a link to the next page and schedules another request using the same
+``parse`` method as callback.
 
 Here you notice one of the main advantages about Scrapy: requests are
 :ref:`scheduled and processed asynchronously <topics-architecture>`.  This
@@ -103,10 +98,6 @@ each request, limiting amount of concurrent requests per domain or per IP, and
 even :ref:`using an auto-throttling extension <topics-autothrottle>` that tries
 to figure out these automatically.
 
-Finally, the ``parse_question`` callback scrapes the question data for each
-page yielding a dict, which Scrapy then collects and writes to a JSON file as
-requested in the command line.
-
 .. note::
 
     This is using :ref:`feed exports <topics-feed-exports>` to generate the
@@ -145,12 +136,13 @@ scraping easy and efficient, such as:
   :ref:`pipelines <topics-item-pipeline>`).
 
 * Wide range of built-in extensions and middlewares for handling:
-    * cookies and session handling
-    * HTTP features like compression, authentication, caching
-    * user-agent spoofing
-    * robots.txt
-    * crawl depth restriction
-    * and more
+
+  - cookies and session handling
+  - HTTP features like compression, authentication, caching
+  - user-agent spoofing
+  - robots.txt
+  - crawl depth restriction
+  - and more
 
 * A :ref:`Telnet console <topics-telnetconsole>` for hooking into a Python
   console running inside your Scrapy process, to introspect and debug your
@@ -165,8 +157,8 @@ What's next?
 ============
 
 The next steps for you are to :ref:`install Scrapy <intro-install>`,
-:ref:`follow through the tutorial <intro-tutorial>` to learn how to organize
-your code in Scrapy projects and `join the community`_. Thanks for your
+:ref:`follow through the tutorial <intro-tutorial>` to learn how to create
+a full-blown Scrapy project and `join the community`_. Thanks for your
 interest!
 
 .. _join the community: http://scrapy.org/community/

From 1d159ae6f978a4c172fe8e0aab0f8beb74615dc8 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 15 Sep 2016 15:37:03 -0300
Subject: [PATCH 1062/2974] minor grammar fix

---
 docs/intro/overview.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index fb390d79d..953fdc968 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -78,9 +78,9 @@ Spider definition inside it and ran it through its crawler engine.
 The crawl started by making requests to the URLs defined in the ``start_urls``
 attribute (in this case, only the URL for quotes in *humor* category)
 and called the default callback method ``parse``, passing the response object as
-an argument. In the ``parse`` callback we loop through the quote elements
+an argument. In the ``parse`` callback, we loop through the quote elements
 using a CSS Selector, yield a Python dict with the extracted quote text and author,
-look for a link to the next page and schedules another request using the same
+look for a link to the next page and schedule another request using the same
 ``parse`` method as callback.
 
 Here you notice one of the main advantages about Scrapy: requests are

From effaab867e45d441b5ca99ea6b24fd423e256a33 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 15 Sep 2016 21:37:15 +0200
Subject: [PATCH 1063/2974] Update shell help with availability of scrapy
 module

---
 scrapy/shell.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/shell.py b/scrapy/shell.py
index 4f9f06561..183ee1f70 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -139,6 +139,7 @@ class Shell(object):
     def get_help(self):
         b = []
         b.append("Available Scrapy objects:")
+        b.append("  scrapy     scrapy module (contains scrapy.Request, scrapy.Selector, etc)")
         for k, v in sorted(self.vars.items()):
             if self._is_relevant(v):
                 b.append("  %-10s %s" % (k, v))

From 75531e409e5aaa00eb3146f4763f6a2a7fb1c59b Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 15 Sep 2016 16:56:13 -0300
Subject: [PATCH 1064/2974] use better condition in example spider

---
 docs/intro/overview.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 953fdc968..7195017ff 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -39,7 +39,7 @@ http://quotes.toscrape.com, following the pagination::
                 }
 
             next_page = response.css('li.next a::attr("href")').extract_first()
-            if next_page:
+            if next_page is not None:
                 next_page = response.urljoin(next_page)
                 yield scrapy.Request(next_page, callback=self.parse)
 

From 2427791287d14f14b846c67dc1edff40a1d2b778 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 15 Sep 2016 17:46:31 -0300
Subject: [PATCH 1065/2974] tutorial: remove item class definition and present
 start_requests first This changes the tutorial, removing the step of creating
 an item class and also starts by presenting the start_requests method instead
 of start_urls.

---
 docs/intro/tutorial.rst | 139 +++++++++++++---------------------------
 1 file changed, 43 insertions(+), 96 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index f802c4e49..0a3361799 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -13,10 +13,9 @@ our example domain to scrape.
 This tutorial will walk you through these tasks:
 
 1. Creating a new Scrapy project
-2. Defining the Items you will extract
-3. Writing a :ref:`spider <topics-spiders>` to crawl a site and extract
+2. Writing a :ref:`spider <topics-spiders>` to crawl a site and extract
    :ref:`Items <topics-items>`
-4. Exporting the scraped data using command line
+3. Exporting the scraped data using command line
 
 Scrapy is written in Python_. If you're new to the language you might want to
 start by getting an idea of what the language is like, to get the most out of
@@ -55,34 +54,6 @@ This will create a ``tutorial`` directory with the following contents::
                 __init__.py
 
 
-Defining our Item
-=================
-
-`Items` are containers that will be loaded with the scraped data; they work
-like simple Python dicts. While you can use plain Python dicts with Scrapy,
-`Items` provide additional protection against populating undeclared fields,
-preventing typos. They can also be used with :ref:`Item Loaders
-<topics-loaders>`, a mechanism with helpers to conveniently populate `Items`.
-
-They are declared by creating a :class:`scrapy.Item <scrapy.item.Item>` class and defining
-its attributes as :class:`scrapy.Field <scrapy.item.Field>` objects, much like in an ORM
-(don't worry if you're not familiar with ORMs, you will see that this is an
-easy task).
-
-We begin by modeling the item that we will use to hold the site's data obtained
-from quotes.toscrape.com. As we want to capture the text and author from each of
-the quotes listed there, we define fields for each of these three attributes. To do that, we edit
-``items.py``, found in the ``tutorial`` directory. Our Item class looks like this::
-
-    import scrapy
-
-    class QuoteItem(scrapy.Item):
-        text = scrapy.Field()
-        author = scrapy.Field()
-
-This may seem complicated at first, but defining an item class allows you to use other handy
-components and helpers within Scrapy.
-
 Our first Spider
 ================
 
@@ -93,20 +64,23 @@ They define an initial list of URLs to download, how to follow links, and how
 to parse the contents of pages to extract :ref:`items <topics-items>`.
 
 To create a Spider, you must subclass :class:`scrapy.Spider
-<scrapy.spiders.Spider>` and define some attributes:
+<scrapy.spiders.Spider>` and define some attributes and methods:
 
 * :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be
   unique within a project, that is, you can't set the same name for different
   Spiders.
 
-* :attr:`~scrapy.spiders.Spider.start_urls`: a list of URLs where the
-  Spider will begin to crawl from. The first pages downloaded will be those
-  listed here. The subsequent URLs will be generated successively from data
-  contained in the start URLs.
+* :meth:`~scrapy.spiders.Spider.start_requests`: must return a list
+  of requests where the Spider will begin to crawl from.
+  Subsequent requests will be generated successively from these initial requests.
+
+  As alternative to defining this method, you can define a class
+  attribute :attr:`~scrapy.spiders.Spider.start_urls`, which the default
+  implementation of this method will use to create the proper requests.
 
 * :meth:`~scrapy.spiders.Spider.parse`: a method of the spider, which will
   be called with the downloaded :class:`~scrapy.http.Response` object of each
-  start URL. The response is passed to the method as the first and only
+  initial request. The response is passed to the method as the first and only
   argument.
 
   This method is responsible for parsing the response data and extracting
@@ -124,13 +98,16 @@ This is the code for our first Spider; save it in a file named
 
     class QuotesSpider(scrapy.Spider):
         name = "quotes"
-        start_urls = [
-            'http://quotes.toscrape.com/page/1/',
-            'http://quotes.toscrape.com/page/2/',
-        ]
+
+        def start_requests(self):
+            base_url = 'http://quotes.toscrape.com'
+            for path in ['/page/1/', '/page/2/']:
+                yield scrapy.Request(url=base_url + path,
+                                     callback=self.parse)
 
         def parse(self, response):
-            filename = 'quotes-' + response.url.split("/")[-2] + '.html'
+            page = response.url.split("/")[-2]
+            filename = 'quotes-%s.html' % page
             with open(filename, 'wb') as f:
                 f.write(response.body)
 
@@ -171,13 +148,13 @@ URLs, as our ``parse`` method instructs.
 What just happened under the hood?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Scrapy creates :class:`scrapy.Request <scrapy.http.Request>` objects
-for each URL in the ``start_urls`` attribute of the Spider, and assigns
-them the ``parse`` method of the spider as their callback function.
+Scrapy will schedule the :class:`scrapy.Request <scrapy.http.Request>` objects
+returned by the ``start_requests`` method of the Spider, and when receiving
+a response for each one it will instantiate :class:`scrapy.http.Response`
+objects and call the ``parse`` callback method passing the response as argument.
 
-These Requests are scheduled, then executed, and :class:`scrapy.http.Response`
-objects are returned and then fed back to the spider, through the
-:meth:`~scrapy.spiders.Spider.parse` method.
+.. TODO: add here an explanation about how this structure is so command that
+   we can do a short version of the spider w/ start_urls and default callback
 
 Extracting Items
 ----------------
@@ -355,9 +332,13 @@ concatenate further ``.xpath()`` calls to dig deeper into a node. We are going t
 that property here, so::
 
     for quote in response.xpath('//div[@class="quote"]'):
-        text = quote.xpath('span[@class="text"]/text()').extract()
-        author = quote.xpath('span/small/text()').extract()
-        print('{}: {}'.format(author, text))
+        text = quote.xpath('span[@class="text"]/text()').extract_first()
+        author = quote.xpath('span/small/text()').extract_first()
+        print({'text': text, 'author': author})
+
+In the above snippet we've decided to use the method ``.extract_first()``
+instead of ``.extract()``, to extract the content from the first element from a
+selector list returned by ``.xpath()``.
 
 .. note::
 
@@ -366,7 +347,11 @@ that property here, so::
     :ref:`topics-selectors-relative-xpaths` in the :ref:`topics-selectors`
     documentation
 
-Let's add this code to our spider::
+Knowing to use selectors, extracting data from a page is just a matter of
+yield the Python dictionaries from the callback method instead of printing
+them.
+
+Let's add the necessary code to our spider::
 
     import scrapy
 
@@ -380,54 +365,16 @@ Let's add this code to our spider::
 
         def parse(self, response):
             for quote in response.xpath('//div[@class="quote"]'):
-                text = quote.xpath('span[@class="text"]/text()').extract_first()
-                author = quote.xpath('span/small/text()').extract_first()
-                print(u'{}: {}'.format(author, text))
+                yield {
+                    'text': quote.xpath('span[@class="text"]/text()').extract_first(),
+                    'author': quote.xpath('span/small/text()').extract_first(),
+                }
 
-Note how we've changed to use the method ``.extract_first()``, which extracts
-the first element from a selector list returned by ``.xpath()``.
-
-Now try crawling quotes.toscrape.com again and you'll see sites being printed
-in your output. Run::
+Run::
 
     scrapy crawl quotes
 
-Using our item
---------------
-
-:class:`~scrapy.item.Item` objects are custom Python dicts; you can access the
-values of their fields (attributes of the class we defined earlier) using the
-standard dict syntax like::
-    
-    >>> from tutorial.items import QuoteItem
-    >>> item = QuoteItem()
-    >>> item['text'] = 'Some random quote'
-    >>> item['title']
-    'Some random quote'
-
-So, in order to return the data we've scraped so far, the final code for our
-Spider would be like this::
-
-    import scrapy
-    from tutorial.items import QuoteItem
-
-
-    class QuotesSpider(scrapy.Spider):
-        name = "quotes"
-        start_urls = [
-            'http://quotes.toscrape.com/page/1/',
-            'http://quotes.toscrape.com/page/2/',
-        ]
-
-        def parse(self, response):
-            for quote in response.xpath('//div[@class="quote"]'):
-                item = QuoteItem()
-                item['text'] = quote.xpath('span[@class="text"]/text()').extract_first()
-                item['author'] = quote.xpath('span/small/text()').extract_first()
-                yield item
-
-
-Now crawling quotes.toscrape.com yields ``QuoteItem`` objects::
+Now crawling quotes.toscrape.com will show dictionary objects::
 
     2016-09-02 16:35:20 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
     {'author': 'Oscar Wilde',

From c508f406892f9d38860fedf1caf8a41fc69bc184 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 15 Sep 2016 18:05:09 -0300
Subject: [PATCH 1066/2974] use harcoded URLs, remove item reference on second
 spider

---
 docs/intro/tutorial.rst | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 0a3361799..d160bfc5c 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -100,10 +100,12 @@ This is the code for our first Spider; save it in a file named
         name = "quotes"
 
         def start_requests(self):
-            base_url = 'http://quotes.toscrape.com'
-            for path in ['/page/1/', '/page/2/']:
-                yield scrapy.Request(url=base_url + path,
-                                     callback=self.parse)
+            urls = [
+                'http://quotes.toscrape.com/page/1/',
+                'http://quotes.toscrape.com/page/2/',
+            ]
+            for url in urls:
+                yield scrapy.Request(url=url, callback=self.parse)
 
         def parse(self, response):
             page = response.url.split("/")[-2]
@@ -397,7 +399,6 @@ want for all of them?
 Here is a modification to our spider that does just that::
 
     import scrapy
-    from tutorial.items import QuoteItem
 
 
     class QuotesSpider(scrapy.Spider):
@@ -408,12 +409,13 @@ Here is a modification to our spider that does just that::
 
         def parse(self, response):
             for quote in response.xpath('//div[@class="quote"]'):
-                item = QuoteItem()
-                item['text'] = quote.xpath('span[@class="text"]/text()').extract_first()
-                item['author'] = quote.xpath('span/small/text()').extract_first()
-                yield item
+                yield {
+                    'text': quote.xpath('span[@class="text"]/text()').extract_first(),
+                    'author': quote.xpath('span/small/text()').extract_first(),
+                }
+
             next_page = response.xpath('//li[@class="next"]/a/@href').extract_first()
-            if next_page:
+            if next_page is not None:
                 next_page = response.urljoin(next_page)
                 yield scrapy.Request(next_page, callback=self.parse)
 

From 0da497cf7a5063accfabde3acaf2568bfd2b57e4 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Fri, 16 Sep 2016 11:55:23 -0300
Subject: [PATCH 1067/2974] updates on the first section (our first spider)

---
 docs/intro/tutorial.rst | 121 +++++++++++++++++++++++-----------------
 1 file changed, 70 insertions(+), 51 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index d160bfc5c..9ab194865 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -7,8 +7,8 @@ Scrapy Tutorial
 In this tutorial, we'll assume that Scrapy is already installed on your system.
 If that's not the case, see :ref:`intro-install`.
 
-We are going to use `quotes.toscrape.com <http://quotes.toscrape.com/>`_ as
-our example domain to scrape.
+We are going to scrape `quotes.toscrape.com <http://quotes.toscrape.com/>`_, a website
+that lists quotes from famous authors.
 
 This tutorial will walk you through these tasks:
 
@@ -57,41 +57,13 @@ This will create a ``tutorial`` directory with the following contents::
 Our first Spider
 ================
 
-Spiders are classes that you define and Scrapy uses to scrape information from a
-domain (or group of domains).
+Spiders are classes that you define and that Scrapy uses to scrape information
+from a website (or group of websites). They define an initial list of URLs to
+download, how to follow links, and how to parse the the downloaded page contents
+to extract :ref:`items <topics-items>`.
 
-They define an initial list of URLs to download, how to follow links, and how
-to parse the contents of pages to extract :ref:`items <topics-items>`.
-
-To create a Spider, you must subclass :class:`scrapy.Spider
-<scrapy.spiders.Spider>` and define some attributes and methods:
-
-* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be
-  unique within a project, that is, you can't set the same name for different
-  Spiders.
-
-* :meth:`~scrapy.spiders.Spider.start_requests`: must return a list
-  of requests where the Spider will begin to crawl from.
-  Subsequent requests will be generated successively from these initial requests.
-
-  As alternative to defining this method, you can define a class
-  attribute :attr:`~scrapy.spiders.Spider.start_urls`, which the default
-  implementation of this method will use to create the proper requests.
-
-* :meth:`~scrapy.spiders.Spider.parse`: a method of the spider, which will
-  be called with the downloaded :class:`~scrapy.http.Response` object of each
-  initial request. The response is passed to the method as the first and only
-  argument.
-
-  This method is responsible for parsing the response data and extracting
-  scraped data (as scraped items) and more URLs to follow.
-
-  The :meth:`~scrapy.spiders.Spider.parse` method is in charge of processing
-  the response and returning scraped data (as :class:`~scrapy.item.Item`
-  objects) and more URLs to follow (as :class:`~scrapy.http.Request` objects).
-
-This is the code for our first Spider; save it in a file named
-``quotes_spider.py`` under the ``tutorial/spiders`` directory::
+This is the code for our first Spider. Save it in a file named
+``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
 
     import scrapy
 
@@ -113,8 +85,29 @@ This is the code for our first Spider; save it in a file named
             with open(filename, 'wb') as f:
                 f.write(response.body)
 
-Crawling
---------
+
+As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.spiders.Spider>`
+and defines some attributes and methods:
+
+* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be
+  unique within a project, that is, you can't set the same name for different
+  Spiders.
+
+* :meth:`~scrapy.spiders.Spider.start_requests`: must return a list
+  of requests where the Spider will begin to crawl from.
+  Subsequent requests will be generated successively from these initial requests.
+
+* :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle
+  the response downloaded for each of the requests made. The response parameter
+  is an instance of :class:`~scrapy.http.Response` that holds the page content and
+  has further helpful methods to handle it.
+
+  The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting
+  the scraped data as items (:class:`~scrapy.item.Item`) and also finding new URLs to
+  follow and creating new requests (:class:`~scrapy.http.Request`) from them.
+
+How to run your spider
+----------------------
 
 To put our spider to work, go to the project's top level directory and run::
 
@@ -138,25 +131,51 @@ similar to this::
     2016-09-01 16:51:29 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
     2016-09-01 16:51:29 [scrapy] INFO: Closing spider (finished)
 
-.. note::
-    At the end you can see a log line for each URL defined in ``start_urls``.
-    Because these URLs are the starting ones, they have no referrers, which is
-    shown at the end of the log line, where it says ``(referer: None)``.
+Now, check the files in the current directory. You should notice that two new
+files have been created: *quotes-1.html* and *quotes-2.html*, with the content
+for the respective URLs, as our ``parse`` method instructs.
+
+.. note:: If you are wondering why we haven't parsed the HTML yet, hold
+  on, we will cover that soon.
 
-Now, check the files in the current directory. You should notice two new files
-have been created: *quotes-1.html* and *quotes-2.html*, with the content for the respective
-URLs, as our ``parse`` method instructs.
 
 What just happened under the hood?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Scrapy schedules the :class:`scrapy.Request <scrapy.http.Request>` objects
+returned by the ``start_requests`` method of the Spider. Upon receiving
+a response for each one, it instantiates :class:`scrapy.http.Response`
+objects and calls the ``parse`` callback method passing the response as
+argument.
 
-Scrapy will schedule the :class:`scrapy.Request <scrapy.http.Request>` objects
-returned by the ``start_requests`` method of the Spider, and when receiving
-a response for each one it will instantiate :class:`scrapy.http.Response`
-objects and call the ``parse`` callback method passing the response as argument.
 
-.. TODO: add here an explanation about how this structure is so command that
-   we can do a short version of the spider w/ start_urls and default callback
+Simplifying your spider
+-----------------------
+Instead of defining the :meth:`~scrapy.spiders.Spider.start_requests` method
+generating :class:`scrapy.Request <scrapy.http.Request>`
+objects from URLs, you can just put those URLs in the 
+:attr:`~scrapy.spiders.Spider.start_urls` attribute::
+
+    import scrapy
+
+
+    class QuotesSpider(scrapy.Spider):
+        name = "quotes"
+        start_urls = [
+          'http://quotes.toscrape.com/page/1/',
+          'http://quotes.toscrape.com/page/2/',
+        ]
+
+        def parse(self, response):
+            page = response.url.split("/")[-2]
+            filename = 'quotes-%s.html' % page
+            with open(filename, 'wb') as f:
+                f.write(response.body)
+
+The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle
+each of the requests for those URLs, even though we haven't explicitely told
+Scrapy to do so. This happens because :meth:`~scrapy.spiders.Spider.parse`
+is Scrapy's default callback method.
+
 
 Extracting Items
 ----------------

From 0cd9dfcc85433b5c09049630fe87f08e9ebd36da Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Fri, 16 Sep 2016 15:21:49 -0300
Subject: [PATCH 1068/2974] small fixes on tutorial

---
 docs/intro/tutorial.rst | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 9ab194865..2bbf71573 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -13,8 +13,7 @@ that lists quotes from famous authors.
 This tutorial will walk you through these tasks:
 
 1. Creating a new Scrapy project
-2. Writing a :ref:`spider <topics-spiders>` to crawl a site and extract
-   :ref:`Items <topics-items>`
+2. Writing a :ref:`spider <topics-spiders>` to crawl a site and extract data
 3. Exporting the scraped data using command line
 
 Scrapy is written in Python_. If you're new to the language you might want to
@@ -58,9 +57,9 @@ Our first Spider
 ================
 
 Spiders are classes that you define and that Scrapy uses to scrape information
-from a website (or group of websites). They define an initial list of URLs to
-download, how to follow links, and how to parse the the downloaded page contents
-to extract :ref:`items <topics-items>`.
+from a website (or group of websites). They define the initial requests to make,
+how to follow links in the pages, and how to parse the downloaded page content
+to extract data.
 
 This is the code for our first Spider. Save it in a file named
 ``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
@@ -103,7 +102,7 @@ and defines some attributes and methods:
   has further helpful methods to handle it.
 
   The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting
-  the scraped data as items (:class:`~scrapy.item.Item`) and also finding new URLs to
+  the scraped data as dicts and also finding new URLs to
   follow and creating new requests (:class:`~scrapy.http.Request`) from them.
 
 How to run your spider

From b2a5cddbb01e3970ce1e38e821a76c7d10520845 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Fri, 16 Sep 2016 15:44:39 -0300
Subject: [PATCH 1069/2974] tutorial: update section about following links,
 expand examples adding an AuthorSpider to demonstrate further a different
 crawling arrangement.

---
 docs/intro/tutorial.rst | 78 ++++++++++++++++++++++++++++++++++-------
 1 file changed, 65 insertions(+), 13 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 2bbf71573..f3b933d28 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -408,13 +408,13 @@ Following links
 ===============
 
 Let's say, instead of just scraping the stuff from the first two pages
-from quotes.toscrape.com, you want quotes from all the pages in the website.
+from http://quotes.toscrape.com, you want quotes from all the pages in the website.
 
-Now that you know how to extract data from a page, why not extract the
-pagination links in each page, follow them and then extract the data you
-want for all of them?
+Now that you know how to extract data from pages, let's see how to follow links
+from them.
 
-Here is a modification to our spider that does just that::
+Here is a modification of our spider that recursively follows the link to the next
+page, extracting data from it::
 
     import scrapy
 
@@ -426,18 +426,19 @@ Here is a modification to our spider that does just that::
         ]
 
         def parse(self, response):
-            for quote in response.xpath('//div[@class="quote"]'):
+            for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.xpath('span[@class="text"]/text()').extract_first(),
-                    'author': quote.xpath('span/small/text()').extract_first(),
+                    'text': quote.css('span.text::text').extract_first(),
+                    'author': quote.css('span small::text').extract_first(),
                 }
 
-            next_page = response.xpath('//li[@class="next"]/a/@href').extract_first()
+            next_page = response.css('li.next a::attr("href")').extract_first()
             if next_page is not None:
                 next_page = response.urljoin(next_page)
                 yield scrapy.Request(next_page, callback=self.parse)
 
-Now after extracting an item the `parse()` method looks for the link to the next page, 
+
+Now, after extracting the data, the `parse()` method looks for the link to the next page,
 builds a full absolute URL using the `response.urljoin` method (since the links can
 be relative) and yields a new request to the next page, registering itself as callback to handle the data extraction for the next page and to keep the crawling going through all the pages.
 
@@ -457,13 +458,64 @@ Another common pattern is to build an item with data from more than one page,
 using a :ref:`trick to pass additional data to the callbacks
 <topics-request-response-ref-request-callback-arguments>`.
 
+Another example: scraping authors
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Here is another spider that illustrates callbacks and following links,
+this time for scraping author information::
+
+
+    import scrapy
+
+
+    class AuthorSpider(scrapy.Spider):
+        name = 'author'
+
+        start_urls = ['http://quotes.toscrape.com/']
+
+        def parse(self, response):
+            # follow links to author pages
+            for href in response.css('.author a::attr("href")').extract():
+                yield scrapy.Request(response.urljoin(href),
+                                     callback=self.parse_author)
+
+            # follow pagination links
+            next_page = response.css('li.next a::attr("href")').extract_first()
+            if next_page is not None:
+                next_page = response.urljoin(next_page)
+                yield scrapy.Request(next_page, callback=self.parse)
+
+        def parse_author(self, response):
+            def extract_with_css(query):
+                return response.css(query).extract_first().strip()
+
+            yield {
+                'name': extract_with_css('h3.author-title::text'),
+                'birthdate': extract_with_css('.author-born-date::text'),
+                'bio': extract_with_css('.author-description::text'),
+            }
+
+This spider will start from the main page, it will follow all the links to the
+authors pages calling the ``parse_author`` callback for each of them, and also
+the paginations links too with the ``parse`` callback as we saw before.
+
+The ``parse_author`` callback defines a helper function to extract and cleanup the
+data from a CSS query and yields the Python dict with the author data.
+
+Another interesting this spider demonstrates is that, even if there are many
+quotes from the same author, we don't need to worry about visiting the same
+page multiple times because Scrapy by default filters out duplicated requests
+to URLs already visited, avoiding the problem of hitting servers too much
+because of a programming mistake. This can be configured by the setting
+:setting:`DUPEFILTER_CLASS`.
 
 .. note::
-    As an example spider that leverages this mechanism, check out the
-    :class:`~scrapy.spiders.CrawlSpider` class for a generic spider
-    that implements a small rules engine that you can use to write your
+    As another example spider that leverages the mechanism of following links,
+    check out the :class:`~scrapy.spiders.CrawlSpider` class for a generic
+    spider that implements a small rules engine that you can use to write your
     crawlers on top of it.
 
+
 Storing the scraped data
 ========================
 

From 21de617c77ca49d7ab09f8721676c449d793cdb7 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Fri, 16 Sep 2016 15:55:14 -0300
Subject: [PATCH 1070/2974] mention that spiders need to subclass scrapy.Spider

---
 docs/intro/tutorial.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index f3b933d28..62304de2c 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -57,9 +57,10 @@ Our first Spider
 ================
 
 Spiders are classes that you define and that Scrapy uses to scrape information
-from a website (or group of websites). They define the initial requests to make,
-how to follow links in the pages, and how to parse the downloaded page content
-to extract data.
+from a website (or group of websites). They must subclass
+:class:`scrapy.Spider` and define the initial requests to make, how to follow
+links in the pages, and how to parse the downloaded page content to extract
+data.
 
 This is the code for our first Spider. Save it in a file named
 ``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::

From 31260cf02fa60efff42aba0af185f4afe5d18557 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Fri, 16 Sep 2016 16:05:36 -0300
Subject: [PATCH 1071/2974] mentions stackoverflow as help channel (fixes
 #2255)

---
 docs/index.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/index.rst b/docs/index.rst
index 4cb3eb741..b4272e47f 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -13,13 +13,15 @@ Having trouble? We'd like to help!
 
 * Try the :doc:`FAQ <faq>` -- it's got answers to some common questions.
 * Looking for specific information? Try the :ref:`genindex` or :ref:`modindex`.
+* Ask or search questions in `StackOverflow using the scrapy tag`_,
 * Search for information in the `archives of the scrapy-users mailing list`_, or
   `post a question`_.
-* Ask a question in the `#scrapy IRC channel`_.
+* Ask a question in the `#scrapy IRC channel`_,
 * Report bugs with Scrapy in our `issue tracker`_.
 
 .. _archives of the scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
 .. _post a question: https://groups.google.com/forum/#!forum/scrapy-users
+.. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy
 .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy
 .. _issue tracker: https://github.com/scrapy/scrapy/issues
 

From 147e75602d52d66ea0d3f385dd34f9cedcab883e Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Fri, 16 Sep 2016 16:47:24 -0300
Subject: [PATCH 1072/2974] update after review comments (thanks @stummjr)

---
 docs/intro/tutorial.rst | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 62304de2c..a3a0ab390 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -439,9 +439,11 @@ page, extracting data from it::
                 yield scrapy.Request(next_page, callback=self.parse)
 
 
-Now, after extracting the data, the `parse()` method looks for the link to the next page,
-builds a full absolute URL using the `response.urljoin` method (since the links can
-be relative) and yields a new request to the next page, registering itself as callback to handle the data extraction for the next page and to keep the crawling going through all the pages.
+Now, after extracting the data, the ``parse()`` method looks for the link to
+the next page, builds a full absolute URL using the ``response.urljoin`` method
+(since the links can be relative) and yields a new request to the next page,
+registering itself as callback to handle the data extraction for the next page
+and to keep the crawling going through all the pages.
 
 What you see here is Scrapy's mechanism of following links: when you yield
 a Request in a callback method, Scrapy will schedule that request to be sent
@@ -498,16 +500,16 @@ this time for scraping author information::
 
 This spider will start from the main page, it will follow all the links to the
 authors pages calling the ``parse_author`` callback for each of them, and also
-the paginations links too with the ``parse`` callback as we saw before.
+the pagination links too with the ``parse`` callback as we saw before.
 
 The ``parse_author`` callback defines a helper function to extract and cleanup the
 data from a CSS query and yields the Python dict with the author data.
 
-Another interesting this spider demonstrates is that, even if there are many
-quotes from the same author, we don't need to worry about visiting the same
-page multiple times because Scrapy by default filters out duplicated requests
-to URLs already visited, avoiding the problem of hitting servers too much
-because of a programming mistake. This can be configured by the setting
+Another interesting thing this spider demonstrates is that, even if there are
+many quotes from the same author, we don't need to worry about visiting the
+same author page multiple times. By default, Scrapy filters out duplicated
+requests to URLs already visited, avoiding the problem of hitting servers too
+much because of a programming mistake. This can be configured by the setting
 :setting:`DUPEFILTER_CLASS`.
 
 .. note::

From 31545a9f84785edf309eb8a1c7238f46d024cdc2 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Fri, 16 Sep 2016 17:13:24 -0300
Subject: [PATCH 1073/2974] tutorial: updating extracting data section to
 introduce CSS and XPath equally

---
 docs/intro/tutorial.rst | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index a3a0ab390..34d33a9b3 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -177,8 +177,8 @@ Scrapy to do so. This happens because :meth:`~scrapy.spiders.Spider.parse`
 is Scrapy's default callback method.
 
 
-Extracting Items
-----------------
+Extracting data
+---------------
 
 Introduction to Selectors
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -191,25 +191,31 @@ mechanisms see the :ref:`Selectors documentation <topics-selectors>`.
 .. _XPath: https://www.w3.org/TR/xpath
 .. _CSS: https://www.w3.org/TR/selectors
 
-Here are some examples of XPath expressions and their meanings:
+Here are some examples of XPath expressions, their meanings and CSS
+equivalents:
 
 * ``/html/head/title``: selects the ``<title>`` element, inside the ``<head>``
-  element of an HTML document. Equivalent CSS selector: ``html > head > title``.
+  element of an HTML document. Using CSS, the equivalent would be: ``html >
+  head > title``.
 
 * ``/html/head/title/text()``: selects the text inside the aforementioned
-  ``<title>`` element. Equivalent CSS selector: ``html > head > title ::text``.
+  ``<title>`` element. In Scrapy, you can do the same with CSS using ``html >
+  head > title::text``. The ``::text`` bit isn't really CSS, but is supported
+  by Scrapy for extracting purposes.
 
 * ``//td``: selects all the ``<td>`` elements from the whole document.
-  Equivalent CSS selector: ``td``.
+  The equivalent CSS selector: ``td``.
 
-* ``//div[@class="mine"]``: selects all ``div`` elements which contain an
-  attribute ``class="mine"``. Equivalent CSS selector: ``div.mine``.
+* ``//div[@id="mine"]``: selects all ``div`` elements which contain an
+  attribute ``id="mine"``. The equivalent CSS selector would be: ``div#mine``.
 
-These are just a couple of simple examples of what you can do with XPath, but
-XPath expressions are indeed much more powerful. To learn more about XPath, we
-recommend `this tutorial to learn XPath through examples
-<http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this tutorial to learn "how
-to think in XPath" <http://plasmasturm.org/log/xpath101/>`_.
+These are just a couple of simple examples of what you can do with XPath and
+CSS. XPath expressions are very powerful, they're the foundation of Scrapy
+selectors. In fact, CSS selectors are converted to XPath expressions
+under-the-hood. To learn more about XPath, we recommend `this tutorial to learn
+XPath through examples <http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this
+tutorial to learn "how to think in XPath"
+<http://plasmasturm.org/log/xpath101/>`_.
 
 .. note:: **CSS vs XPath:** you can go a long way extracting data from web pages
   using only CSS selectors. However, XPath offers more power because besides

From 233b98d642f3c20d47917b54f6944184ee61e0cf Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Fri, 16 Sep 2016 18:08:10 -0300
Subject: [PATCH 1074/2974] include section describing spider arguments

---
 docs/intro/tutorial.rst | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 34d33a9b3..79cf502a6 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -524,6 +524,46 @@ much because of a programming mistake. This can be configured by the setting
     spider that implements a small rules engine that you can use to write your
     crawlers on top of it.
 
+Customizing behavior via spider arguments
+=========================================
+You can provide command line arguments to your spiders by using the ``-a``
+option when running them::
+
+    scrapy crawl quotes -o items.json -a tag=humor
+
+In this example, the value provided for the ``tag`` argument will be available
+via a spider attribute. Using this, you could make your spider get only quotes
+tagged with a specific tag, building the URL based on the argument::
+
+    import scrapy
+
+
+    class QuotesSpider(scrapy.Spider):
+        name = "quotes"
+
+        def start_requests(self):
+            url = 'http://quotes.toscrape.com/'
+            tag = getattr(self, 'tag', None)
+            if tag is not None:
+                url = url + 'tag/' + tag
+            yield scrapy.Request(url)
+
+        def parse(self, response):
+            for quote in response.css('div.quote'):
+                yield {
+                    'text': quote.css('span.text::text').extract_first(),
+                    'author': quote.css('span small a::text').extract_first(),
+                }
+
+            next_page = response.css('li.next a::attr("href")').extract_first()
+            if next_page is not None:
+                next_page = response.urljoin(next_page)
+                yield scrapy.Request(next_page, callback=self.parse)
+
+
+If you pass the ``tag=humor`` argument to this spider, you'll notice that it
+will only visit URLs from the ``humor`` tag, such as
+``http://quotes.toscrape.com/tag/humor``.
 
 Storing the scraped data
 ========================

From 03ab0772491c61cb7f2198b2c019c7eb7672a331 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Sat, 17 Sep 2016 01:36:56 +0200
Subject: [PATCH 1075/2974] Feed exporter: start exporting only on first item

Fixes GH-872
---
 scrapy/extensions/feedexport.py | 12 ++++++--
 tests/test_feedexport.py        | 49 ++++++++++++++++++++++++++++++---
 2 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index c3fc66de5..85d328528 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -170,6 +170,7 @@ class FeedExporter(object):
         if not self._exporter_supported(self.format):
             raise NotConfigured
         self.store_empty = settings.getbool('FEED_STORE_EMPTY')
+        self._exporting = False
         self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
         uripar = settings['FEED_URI_PARAMS']
         self._uripar = load_object(uripar) if uripar else lambda x, y: None
@@ -188,14 +189,18 @@ class FeedExporter(object):
         file = storage.open(spider)
         exporter = self._get_exporter(file, fields_to_export=self.export_fields,
             encoding=self.export_encoding)
-        exporter.start_exporting()
+        if self.store_empty:
+            exporter.start_exporting()
+            self._exporting = True
         self.slot = SpiderSlot(file, exporter, storage, uri)
 
     def close_spider(self, spider):
         slot = self.slot
         if not slot.itemcount and not self.store_empty:
             return
-        slot.exporter.finish_exporting()
+        if self._exporting:
+            slot.exporter.finish_exporting()
+            self._exporting = False
         logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
         log_args = {'format': self.format,
                     'itemcount': slot.itemcount,
@@ -210,6 +215,9 @@ class FeedExporter(object):
 
     def item_scraped(self, item, spider):
         slot = self.slot
+        if not self._exporting:
+            slot.exporter.start_exporting()
+            self._exporting = True
         slot.exporter.export_item(item)
         slot.itemcount += 1
         return item
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 353b21927..e93d2bafb 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -197,6 +197,21 @@ class FeedExportTest(unittest.TestCase):
         data = yield self.run_and_export(TestSpider, settings)
         defer.returnValue(data)
 
+    @defer.inlineCallbacks
+    def exported_no_data(self, settings):
+        """
+        Return exported data which a spider yielding no ``items`` would return.
+        """
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+            start_urls = ['http://localhost:8998/']
+
+            def parse(self, response):
+                pass
+
+        data = yield self.run_and_export(TestSpider, settings)
+        defer.returnValue(data)
+
     @defer.inlineCallbacks
     def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
         settings = settings or {}
@@ -283,6 +298,32 @@ class FeedExportTest(unittest.TestCase):
         header = self.MyItem.fields.keys()
         yield self.assertExported(items, header, rows, ordered=False)
 
+    @defer.inlineCallbacks
+    def test_export_no_items_not_store_empty(self):
+        formats = ('json',
+                   'jsonlines',
+                   'xml',
+                   'csv',)
+
+        for fmt in formats:
+            settings = {'FEED_FORMAT': fmt}
+            data = yield self.exported_no_data(settings)
+            self.assertEqual(data, b'')
+
+    @defer.inlineCallbacks
+    def test_export_no_items_store_empty(self):
+        formats = (
+            ('json', b'[\n\n]'),
+            ('jsonlines', b''),
+            ('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
+            ('csv', b''),
+        )
+
+        for fmt, expctd in formats:
+            settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True}
+            data = yield self.exported_no_data(settings)
+            self.assertEqual(data, expctd)
+
     @defer.inlineCallbacks
     def test_export_multiple_item_classes(self):
 
@@ -376,26 +417,26 @@ class FeedExportTest(unittest.TestCase):
     def test_export_encoding(self):
         items = [dict({'foo': u'Test\xd6'})]
         header = ['foo']
-        
+
         formats = {
             'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'),
             'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
             'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
         }
-        
+
         for format in formats:
             settings = {'FEED_FORMAT': format}
             data = yield self.exported_data(items, settings)
             self.assertEqual(formats[format], data)
-            
+
         formats = {
             'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'),
             'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),
             'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
         }
-        
+
         for format in formats:
             settings = {'FEED_FORMAT': format, 'FEED_EXPORT_ENCODING': 'latin-1'}
             data = yield self.exported_data(items, settings)

From cc8497abb12df79693d7f2aa63f645e17367d3ba Mon Sep 17 00:00:00 2001
From: Wayne Lovely <wayne.lovely@gmail.com>
Date: Sat, 17 Sep 2016 11:09:28 +0000
Subject: [PATCH 1076/2974] Fix a dict key in the tutorial

---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index f802c4e49..9233c828e 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -402,7 +402,7 @@ standard dict syntax like::
     >>> from tutorial.items import QuoteItem
     >>> item = QuoteItem()
     >>> item['text'] = 'Some random quote'
-    >>> item['title']
+    >>> item['text']
     'Some random quote'
 
 So, in order to return the data we've scraped so far, the final code for our

From 48f6a065b8d1ed1f9e45a78643b52874f09e2c30 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Sat, 17 Sep 2016 15:25:45 +0200
Subject: [PATCH 1077/2974] Flush StreamLogger handlers

---
 scrapy/utils/log.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index a28002c08..51f303216 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -146,7 +146,8 @@ class StreamLogger(object):
             self.logger.log(self.log_level, line.rstrip())
 
     def flush(self):
-        pass
+        for h in self.logger.handlers:
+            h.flush()
 
 
 class LogCounterHandler(logging.Handler):

From 8c38dde4e8e60371a2599523aa793a4786dc904e Mon Sep 17 00:00:00 2001
From: Joakim Uddholm <joakim@uddholm.com>
Date: Mon, 19 Sep 2016 05:33:05 +0200
Subject: [PATCH 1078/2974] Moved parse command tests to its own file. Added
 some checks to check for logged errors.

---
 tests/test_command_parse.py | 156 ++++++++++++++++++++++++++++++++++++
 tests/test_commands.py      | 147 ---------------------------------
 2 files changed, 156 insertions(+), 147 deletions(-)
 create mode 100644 tests/test_command_parse.py

diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
new file mode 100644
index 000000000..b6d6db9ee
--- /dev/null
+++ b/tests/test_command_parse.py
@@ -0,0 +1,156 @@
+from os.path import join, abspath
+from twisted.trial import unittest
+from twisted.internet import defer
+from scrapy.utils.testsite import SiteTest
+from scrapy.utils.testproc import ProcessTest
+from scrapy.utils.python import to_native_str
+from tests.test_commands import CommandTest
+
+
+class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
+    command = 'parse'
+
+    def setUp(self):
+        super(ParseCommandTest, self).setUp()
+        self.spider_name = 'parse_spider'
+        fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
+        with open(fname, 'w') as f:
+            f.write("""
+import scrapy
+from scrapy.linkextractors import LinkExtractor
+from scrapy.spiders import CrawlSpider, Rule
+
+
+class MySpider(scrapy.Spider):
+    name = '{0}'
+
+    def parse(self, response):
+        if getattr(self, 'test_arg', None):
+            self.logger.debug('It Works!')
+        return [scrapy.Item(), dict(foo='bar')]
+
+
+class MyGoodCrawlSpider(CrawlSpider):
+    name = 'goodcrawl{0}'
+
+    rules = (
+        Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True),
+        Rule(LinkExtractor(allow=r'/text'), follow=True),
+    )
+
+    def parse_item(self, response):
+        return [scrapy.Item(), dict(foo='bar')]
+
+    def parse(self, response):
+        return [scrapy.Item(), dict(nomatch='default')]
+
+
+class MyBadCrawlSpider(CrawlSpider):
+    '''Spider which doesn't define a parse_item callback while using it in a rule.'''
+    name = 'badcrawl{0}'
+
+    rules = (
+        Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True),
+    )
+
+    def parse(self, response):
+        return [scrapy.Item(), dict(foo='bar')]
+""".format(self.spider_name))
+
+        fname = abspath(join(self.proj_mod_path, 'pipelines.py'))
+        with open(fname, 'w') as f:
+            f.write("""
+import logging
+
+class MyPipeline(object):
+    component_name = 'my_pipeline'
+
+    def process_item(self, item, spider):
+        logging.info('It Works!')
+        return item
+""")
+
+        fname = abspath(join(self.proj_mod_path, 'settings.py'))
+        with open(fname, 'a') as f:
+            f.write("""
+ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
+""" % self.project_name)
+
+    @defer.inlineCallbacks
+    def test_spider_arguments(self):
+        _, _, stderr = yield self.execute(['--spider', self.spider_name,
+                                           '-a', 'test_arg=1',
+                                           '-c', 'parse',
+                                           self.url('/html')])
+        self.assertIn("DEBUG: It Works!", to_native_str(stderr))
+
+    @defer.inlineCallbacks
+    def test_pipelines(self):
+        _, _, stderr = yield self.execute(['--spider', self.spider_name,
+                                           '--pipelines',
+                                           '-c', 'parse',
+                                           self.url('/html')])
+        self.assertIn("INFO: It Works!", to_native_str(stderr))
+
+    @defer.inlineCallbacks
+    def test_parse_items(self):
+        status, out, stderr = yield self.execute(
+            ['--spider', self.spider_name, '-c', 'parse', self.url('/html')]
+        )
+        self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
+
+    @defer.inlineCallbacks
+    def test_parse_items_no_callback_passed(self):
+        status, out, stderr = yield self.execute(
+            ['--spider', self.spider_name, self.url('/html')]
+        )
+        self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
+
+    @defer.inlineCallbacks
+    def test_wrong_callback_passed(self):
+        status, out, stderr = yield self.execute(
+            ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')]
+        )
+        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
+        self.assertIn("""Cannot find callback""", to_native_str(stderr))
+
+    @defer.inlineCallbacks
+    def test_crawlspider_matching_rule_callback_set(self):
+        """If a rule matches the URL, use it's defined callback."""
+        status, out, stderr = yield self.execute(
+            ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')]
+        )
+        self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
+
+    @defer.inlineCallbacks
+    def test_crawlspider_matching_rule_default_callback(self):
+        """If a rule match but it has no callback set, use the 'parse' callback."""
+        status, out, stderr = yield self.execute(
+            ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')]
+        )
+        self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out))
+
+    @defer.inlineCallbacks
+    def test_spider_with_no_rules_attribute(self):
+        """Using -r with a spider with no rule should not produce items."""
+        status, out, stderr = yield self.execute(
+            ['--spider', self.spider_name, '-r', self.url('/html')]
+        )
+        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
+        self.assertIn("""No CrawlSpider rules found""", to_native_str(stderr))
+
+    @defer.inlineCallbacks
+    def test_crawlspider_missing_callback(self):
+        status, out, stderr = yield self.execute(
+            ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')]
+        )
+        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
+
+    @defer.inlineCallbacks
+    def test_crawlspider_no_matching_rule(self):
+        """The requested URL has no matching rule, so no items should be scraped"""
+        status, out, stderr = yield self.execute(
+            ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')]
+        )
+        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
+        self.assertIn("""Cannot find a rule that matches""", to_native_str(stderr))
diff --git a/tests/test_commands.py b/tests/test_commands.py
index d13024922..b507c46bc 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -246,153 +246,6 @@ class BadSpider(scrapy.Spider):
         self.assertIn("start_requests", log)
         self.assertIn("badspider.py", log)
 
-
-class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
-    command = 'parse'
-
-    def setUp(self):
-        super(ParseCommandTest, self).setUp()
-        self.spider_name = 'parse_spider'
-        fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
-        with open(fname, 'w') as f:
-            f.write("""
-import scrapy
-from scrapy.linkextractors import LinkExtractor
-from scrapy.spiders import CrawlSpider, Rule
-
-
-class MySpider(scrapy.Spider):
-    name = '{0}'
-
-    def parse(self, response):
-        if getattr(self, 'test_arg', None):
-            self.logger.debug('It Works!')
-        return [scrapy.Item(), dict(foo='bar')]
-
-
-class MyGoodCrawlSpider(CrawlSpider):
-    name = 'goodcrawl{0}'
-
-    rules = (
-        Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True),
-        Rule(LinkExtractor(allow=r'/text'), follow=True),
-    )
-
-    def parse_item(self, response):
-        return [scrapy.Item(), dict(foo='bar')]
-
-    def parse(self, response):
-        return [scrapy.Item(), dict(nomatch='default')]
-
-
-class MyBadCrawlSpider(CrawlSpider):
-    '''Spider which doesn't define a parse_item callback while using it in a rule.'''
-    name = 'badcrawl{0}'
-
-    rules = (
-        Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True),
-    )
-
-    def parse(self, response):
-        return [scrapy.Item(), dict(foo='bar')]
-""".format(self.spider_name))
-
-        fname = abspath(join(self.proj_mod_path, 'pipelines.py'))
-        with open(fname, 'w') as f:
-            f.write("""
-import logging
-
-class MyPipeline(object):
-    component_name = 'my_pipeline'
-
-    def process_item(self, item, spider):
-        logging.info('It Works!')
-        return item
-""")
-
-        fname = abspath(join(self.proj_mod_path, 'settings.py'))
-        with open(fname, 'a') as f:
-            f.write("""
-ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
-""" % self.project_name)
-
-    @defer.inlineCallbacks
-    def test_spider_arguments(self):
-        _, _, stderr = yield self.execute(['--spider', self.spider_name,
-                                           '-a', 'test_arg=1',
-                                           '-c', 'parse',
-                                           self.url('/html')])
-        self.assertIn("DEBUG: It Works!", to_native_str(stderr))
-
-    @defer.inlineCallbacks
-    def test_pipelines(self):
-        _, _, stderr = yield self.execute(['--spider', self.spider_name,
-                                           '--pipelines',
-                                           '-c', 'parse',
-                                           self.url('/html')])
-        self.assertIn("INFO: It Works!", to_native_str(stderr))
-
-    @defer.inlineCallbacks
-    def test_parse_items(self):
-        status, out, stderr = yield self.execute(
-            ['--spider', self.spider_name, '-c', 'parse', self.url('/html')]
-        )
-        self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
-
-    @defer.inlineCallbacks
-    def test_parse_items_no_callback_passed(self):
-        status, out, stderr = yield self.execute(
-            ['--spider', self.spider_name, self.url('/html')]
-        )
-        self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
-
-    @defer.inlineCallbacks
-    def test_wrong_callback_passed(self):
-        status, out, stderr = yield self.execute(
-            ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')]
-        )
-        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
-
-    @defer.inlineCallbacks
-    def test_crawlspider_matching_rule_callback_set(self):
-        """If a rule matches the URL, use it's defined callback."""
-        status, out, stderr = yield self.execute(
-            ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')]
-        )
-        self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
-
-    @defer.inlineCallbacks
-    def test_crawlspider_matching_rule_default_callback(self):
-        """If a rule match but it has no callback set, use the 'parse' callback."""
-        status, out, stderr = yield self.execute(
-            ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')]
-        )
-        self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out))
-
-    @defer.inlineCallbacks
-    def test_spider_with_no_rules_attribute(self):
-        """Using -r with a spider with no rule should not produce items."""
-        status, out, stderr = yield self.execute(
-            ['--spider', self.spider_name, '-r', self.url('/html')]
-        )
-        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
-
-    @defer.inlineCallbacks
-    def test_crawlspider_missing_callback(self):
-        status, out, stderr = yield self.execute(
-            ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')]
-        )
-        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
-
-    @defer.inlineCallbacks
-    def test_crawlspider_no_matching_rule(self):
-        """The requested URL has no matching rule, so no items should be scraped"""
-        status, out, stderr = yield self.execute(
-            ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')]
-        )
-        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
-
-
 class BenchCommandTest(CommandTest):
 
     def test_run(self):

From bb8740a55228c28d9a2c3c54ceebc7135fe326ca Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 19 Sep 2016 12:26:36 +0200
Subject: [PATCH 1079/2974] Update 1.2 release notes with latest changes

---
 docs/news.rst | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index e514d9a5a..985eeb796 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -17,6 +17,9 @@ New Features
 - New :setting:`SCHEDULER_DEBUG` setting to log requests serialization
   failures (:issue:`1610`).
 - JSON encoder now supports serialization of ``set`` instances (:issue:`2058`).
+- Interpret ``application/json-amazonui-streaming`` as ``TextResponse`` (:issue:`1503`).
+- ``scrapy`` is imported by default when using shell tools (:command:`shell`,
+  :ref:`inspect_response <topics-shell-inspect-response>`) (:issue:`2248`).
 
 Bug fixes
 ~~~~~~~~~
@@ -28,6 +31,10 @@ Bug fixes
   (:issue:`2153`).
 - Fixed logging of wrong callback name with ``scrapy parse`` (:issue:`2169`).
 - Fix for an odd gzip decompression bug (:issue:`1606`).
+- Fix for selected callbacks when using ``CrawlSpider`` with :command:`scrapy parse <parse>`
+  (:issue:`2225`).
+- Fix for invalid JSON and XML files when spider yields no items (:issue:`872`).
+- Implement ``flush()`` fpr ``StreamLogger`` avoiding a warning in logs (:issue:`2125`).
 
 Refactoring
 ~~~~~~~~~~~
@@ -39,11 +46,14 @@ Refactoring
 Documentation
 ~~~~~~~~~~~~~
 
+- :ref:`Overview <intro-overview>` and :ref:`tutorial <intro-tutorial>`
+  rewritten to use http://toscrape.com websites (:issue:`2236`, :issue:`2249`).
 - Grammar fixes: :issue:`2128`, :issue:`1566`.
 - Download stats badge removed from README (:issue:`2160`).
-- New scrapy :ref:`architecture diagram <topics-architecture>` (:issue:`2165`)
+- New scrapy :ref:`architecture diagram <topics-architecture>` (:issue:`2165`).
 - Updated ``Response`` parameters documentation (:issue:`2197`).
-- Reworded misleading :setting:`RANDOMIZE_DOWNLOAD_DELAY` description (:issue:`2197`).
+- Reworded misleading :setting:`RANDOMIZE_DOWNLOAD_DELAY` description (:issue:`2190`).
+- Add StackOverflow as a support channel (:issue:`2257`).
 
 
 1.1.2 (2016-08-18)

From 2a409d1d951d3c968231785cbc1c2c398445888a Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Mon, 19 Sep 2016 17:13:04 -0300
Subject: [PATCH 1080/2974] [wip] changing introduction to scraping with
 selectors

---
 docs/intro/tutorial.rst | 291 ++++++++++++----------------------------
 1 file changed, 88 insertions(+), 203 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 79cf502a6..f6aa6476c 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -180,235 +180,120 @@ is Scrapy's default callback method.
 Extracting data
 ---------------
 
-Introduction to Selectors
-^^^^^^^^^^^^^^^^^^^^^^^^^
+The best way to learn how to extract data with Scrapy is trying selectors
+using the shell :ref:`Scrapy shell <topics-shell>`. Run::
 
-There are several ways to extract data from web pages. Scrapy uses a mechanism
-based on `XPath`_ or `CSS`_ expressions called :ref:`Scrapy Selectors
-<topics-selectors>`.  For more information about selectors and other extraction
-mechanisms see the :ref:`Selectors documentation <topics-selectors>`.
+    scrapy crawl http://quotes.toscrape.com/page/1/
 
-.. _XPath: https://www.w3.org/TR/xpath
-.. _CSS: https://www.w3.org/TR/selectors
-
-Here are some examples of XPath expressions, their meanings and CSS
-equivalents:
-
-* ``/html/head/title``: selects the ``<title>`` element, inside the ``<head>``
-  element of an HTML document. Using CSS, the equivalent would be: ``html >
-  head > title``.
-
-* ``/html/head/title/text()``: selects the text inside the aforementioned
-  ``<title>`` element. In Scrapy, you can do the same with CSS using ``html >
-  head > title::text``. The ``::text`` bit isn't really CSS, but is supported
-  by Scrapy for extracting purposes.
-
-* ``//td``: selects all the ``<td>`` elements from the whole document.
-  The equivalent CSS selector: ``td``.
-
-* ``//div[@id="mine"]``: selects all ``div`` elements which contain an
-  attribute ``id="mine"``. The equivalent CSS selector would be: ``div#mine``.
-
-These are just a couple of simple examples of what you can do with XPath and
-CSS. XPath expressions are very powerful, they're the foundation of Scrapy
-selectors. In fact, CSS selectors are converted to XPath expressions
-under-the-hood. To learn more about XPath, we recommend `this tutorial to learn
-XPath through examples <http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this
-tutorial to learn "how to think in XPath"
-<http://plasmasturm.org/log/xpath101/>`_.
-
-.. note:: **CSS vs XPath:** you can go a long way extracting data from web pages
-  using only CSS selectors. However, XPath offers more power because besides
-  navigating the structure, it can also look at the content: you're
-  able to select things like: *the link that contains the text 'Next Page'*.
-  Because of this, we encourage you to learn about XPath even if you
-  already know how to construct CSS selectors.
-
-For working with CSS and XPath expressions, Scrapy provides the
-:class:`~scrapy.selector.Selector` class and convenient shortcuts to avoid
-instantiating selectors yourself every time you need to select something from a
-response.
-
-You can see selectors as objects that represent nodes in the document
-structure. So, the first instantiated selectors are associated with the root
-node, or the entire document.
-
-Selectors have four basic methods (click on the method to see the complete API
-documentation):
-
-* :meth:`~scrapy.selector.Selector.xpath`: returns a list of selectors, each of
-  which represents the nodes selected by the xpath expression given as
-  argument.
-
-* :meth:`~scrapy.selector.Selector.css`: returns a list of selectors, each of
-  which represents the nodes selected by the CSS expression given as argument.
-
-* :meth:`~scrapy.selector.Selector.extract`: returns a unicode string with the
-  selected data.
-
-* :meth:`~scrapy.selector.Selector.re`: returns a list of unicode strings
-  extracted by applying the regular expression given as argument.
-
-
-Trying Selectors in the Shell
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-To illustrate the use of Selectors we're going to use the built-in :ref:`Scrapy
-shell <topics-shell>`, which also requires `IPython <http://ipython.org/>`_ (an extended Python console)
-installed on your system.
-
-To start a shell, you must go to the project's top level directory and run::
-
-    scrapy shell "http://quotes.toscrape.com"
-
-.. note::
-
-   Remember to always enclose urls in quotes when running Scrapy shell from
-   command-line, otherwise urls containing arguments (ie. ``&`` character)
-   will not work.
-
-This is what the shell looks like::
+You will see something like::
 
     [ ... Scrapy log here ... ]
-
-    2016-09-01 18:14:39 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com> (referer: None)
+    2016-09-19 12:09:27 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
     [s] Available Scrapy objects:
-    [s]   crawler    <scrapy.crawler.Crawler object at 0x109001c90>
+    [s]   crawler    <scrapy.crawler.Crawler object at 0x7fa91d888c90>
     [s]   item       {}
-    [s]   request    <GET http://quotes.toscrape.com>
-    [s]   response   <200 http://quotes.toscrape.com>
-    [s]   settings   <scrapy.settings.Settings object at 0x109001610>
-    [s]   spider     <DefaultSpider 'default' at 0x1092808d0>
+    [s]   request    <GET http://quotes.toscrape.com/page/1/>
+    [s]   response   <200 http://quotes.toscrape.com/page/1/>
+    [s]   settings   <scrapy.settings.Settings object at 0x7fa91d888c10>
+    [s]   spider     <DefaultSpider 'default' at 0x7fa91c8af990>
     [s] Useful shortcuts:
     [s]   shelp()           Shell help (print this help)
     [s]   fetch(req_or_url) Fetch request (or URL) and update local objects
     [s]   view(response)    View response in a browser
-    
-    >>>
+    >>> 
 
-After the shell loads, you will have the response fetched in a local
-``response`` variable, so if you type ``response.body`` you will see the body
-of the response, or you can type ``response.headers`` to see its headers.
+Using the shell, you can try selecting elements using `CSS`_ with the response
+object::
 
-More importantly ``response`` has a ``selector`` attribute which is an instance of
-:class:`~scrapy.selector.Selector` class, instantiated with this particular ``response``.
-You can run queries on ``response`` by calling ``response.selector.xpath()`` or
-``response.selector.css()``. There are also some convenience shortcuts like ``response.xpath()``
-or ``response.css()`` which map directly to ``response.selector.xpath()`` and
-``response.selector.css()``.
+    >>> response.css('title')
+    [<Selector xpath=u'descendant-or-self::title' data=u'<title>Quotes to Scrape</title>'>]
 
+The result of running ``response.css('title')`` is a list-like object called
+:class:`~scrapy.selector.SelectorList`, which represents a list of
+:class:`~scrapy.selector.Selector` objects that wrap around XML/HTML elements
+and allow you to run further queries to fine-grain the selection or extract the
+data.
 
-So let's try it::
+To extract the text from the title above, you can do::
 
-    In [1]: response.xpath('//title')
-    Out[1]: [<Selector xpath='//title' data=u'<title>Quotes to Scrape</title>'>] 
-    
-    In [2]: response.xpath('//title').extract()
-    Out[2]: [u'<title>Quotes to Scrape</title>']
-    
-    In [3]: response.xpath('//title/text()')
-    Out[3]: [<Selector xpath='//title/text()' data=u'Quotes to Scrape'>]
+    >>> response.css('title::text').extract()
+    [u'Quotes to Scrape']
 
-    In [4]: response.xpath('//title/text()').extract()
-    Out[4]: [u'Quotes to Scrape']
-    
-    In [11]: response.xpath('//title/text()').re('(\w+)')
-    Out[11]: [u'Quotes', u'to', u'Scrape']
+There are two things to note here: one is that we've added ``::text`` to the
+CSS query, to mean that we want to select the text from inside the title element.
 
-Extracting the data
-^^^^^^^^^^^^^^^^^^^
+The other is that the result of calling ``.extract()`` is a list, because we're
+dealing with an instance :class:`~scrapy.selector.SelectorList`.  When you know
+you just want the first result, as in this case, you can do::
 
-Now, let's try to extract some real information from those pages.
+    >>> response.css('title::text').extract_first()
+    u'Quotes to Scrape'
 
-You could type ``response.body`` in the console, and inspect the source code to
-figure out the XPaths you need to use. However, inspecting the raw HTML code
-there could become a very tedious task. To make it easier, you can
-use Firefox Developer Tools or some Firefox extensions like Firebug. For more
+As an alternative, you could've written::
+
+    >>> response.css('title::text')[0].extract()
+    u'Quotes to Scrape'
+
+However, using ``.extract_first()`` avoids an ``IndexError`` and returns
+``None`` when it doesn't find any element matching the selection.
+
+There's a lesson here: for most scraping code, you want it to be resilient to
+errors due to things not being found on a page, so that even if some parts fail
+to be scraped, you can at least get **some** data.
+
+Besides the :meth:`~scrapy.selector.Selector.extract` and
+:meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use
+the :meth:`~scrapy.selector.Selector.re` method to extract using a regular
+expression::
+
+    >>> response.css('title::text').re('Quotes.*')
+    [u'Quotes to Scrape']
+    >>> response.css('title::text').re('Q\w+')
+    [u'Quotes']
+    >>> response.css('title::text').re('(\w+) to (\w+)')
+    [u'Quotes', u'Scrape']
+
+In order to find the proper CSS selectors to use, you might find useful opening
+the response page from the shell in your web browser using ``view(response)``.
+You can use your browser developer tools or extensions like Firebug. For more
 information see :ref:`topics-firebug` and :ref:`topics-firefox`.
 
-After inspecting the page source, you'll find that every quote in the website
-is inside a separate ``<div class="quote">`` element, such as::
 
-    <div class="quote">
-        <span class="text">“We accept the love we think we deserve.”</span>
-        <span>by <small class="author">Stephen Chbosky</small></span>
-        <div class="tags">
-            Tags:
-            <meta class="keywords"> 
-            <a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
-            <a class="tag" href="/tag/love/page/1/">love</a>
-        </div>
-    </div>
+XPath: a brief intro
+^^^^^^^^^^^^^^^^^^^^
+
+Besides CSS, Scrapy selectors also support using `XPath`_ expressions::
+
+    >>> response.xpath('//title')
+    [<Selector xpath='//title' data=u'<title>Quotes to Scrape</title>'>]
+    >>> response.xpath('//title/text()').extract_first()
+    u'Quotes to Scrape'
+
+XPath expressions are very powerful, and are the foundation of Scrapy
+Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You
+can see that if you read closely the text representation of the selector
+objects in the shell.
+
+While perhaps not as popular as CSS selectors, XPath expressions offer more
+power because besides navigating the structure, it can also look at the
+content. Using XPath, you're able to select things like: **select the link
+that contains the text "Next Page"**. This makes XPath very fitting to the task
+of scraping, and we encourage you to learn XPath even if you already know how to
+construct CSS selectors, it will make scraping much easier.
+
+We won't cover much of XPath here. To learn more about XPath, we recommend `this tutorial to learn
+XPath through examples <http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this
+tutorial to learn "how to think in XPath"
+<http://plasmasturm.org/log/xpath101/>`_.
 
 
-So we can select each ``<div class="quote">`` element belonging to the site's 
-list with this code::
+Extraction wrap-up
+^^^^^^^^^^^^^^^^^^
 
-    response.xpath('//div[@class="quote"]')
+Now that you know a bit about selection and extraction, let's complete our
+spider by writing the code to extract the quotes from the webpage.
 
-From the quote elements, we can select the texts with::
-
-    response.xpath('//div[@class="quote"]/span[@class="text"]/text()').extract()
-
-The authors::
-
-    response.xpath('//div[@class="quote"]/span/small/text()').extract()
-
-As we've said before, each ``.xpath()`` call returns a list of selectors, so we can
-concatenate further ``.xpath()`` calls to dig deeper into a node. We are going to use
-that property here, so::
-
-    for quote in response.xpath('//div[@class="quote"]'):
-        text = quote.xpath('span[@class="text"]/text()').extract_first()
-        author = quote.xpath('span/small/text()').extract_first()
-        print({'text': text, 'author': author})
-
-In the above snippet we've decided to use the method ``.extract_first()``
-instead of ``.extract()``, to extract the content from the first element from a
-selector list returned by ``.xpath()``.
-
-.. note::
-
-    For a more detailed description of using nested selectors, see
-    :ref:`topics-selectors-nesting-selectors` and
-    :ref:`topics-selectors-relative-xpaths` in the :ref:`topics-selectors`
-    documentation
-
-Knowing to use selectors, extracting data from a page is just a matter of
-yield the Python dictionaries from the callback method instead of printing
-them.
-
-Let's add the necessary code to our spider::
-
-    import scrapy
-
-
-    class QuotesSpider(scrapy.Spider):
-        name = "quotes"
-        start_urls = [
-            'http://quotes.toscrape.com/page/1/',
-            'http://quotes.toscrape.com/page/2/',
-        ]
-
-        def parse(self, response):
-            for quote in response.xpath('//div[@class="quote"]'):
-                yield {
-                    'text': quote.xpath('span[@class="text"]/text()').extract_first(),
-                    'author': quote.xpath('span/small/text()').extract_first(),
-                }
-
-Run::
-
-    scrapy crawl quotes
-
-Now crawling quotes.toscrape.com will show dictionary objects::
-
-    2016-09-02 16:35:20 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
-    {'author': 'Oscar Wilde',
-     'text': '“We are all in the gutter, but some of us are looking at the stars.”'}
-    2016-09-02 16:35:20 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
-    {'author': 'Mark Twain',
-     'text': '“The man who does not read has no advantage over the man who cannot read.”'}
+TODO: show how to extract quotes and integrate spider code here.
 
 
 Following links

From fee07835f2f9504acc7f0952088ca2ea201027c3 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Mon, 19 Sep 2016 19:19:31 -0300
Subject: [PATCH 1081/2974] Completing the data extraction section

---
 docs/intro/tutorial.rst | 122 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 112 insertions(+), 10 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index f6aa6476c..473183be3 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -106,8 +106,8 @@ and defines some attributes and methods:
   the scraped data as dicts and also finding new URLs to
   follow and creating new requests (:class:`~scrapy.http.Request`) from them.
 
-How to run your spider
-----------------------
+How to run our spider
+---------------------
 
 To put our spider to work, go to the project's top level directory and run::
 
@@ -148,12 +148,14 @@ objects and calls the ``parse`` callback method passing the response as
 argument.
 
 
-Simplifying your spider
------------------------
-Instead of defining the :meth:`~scrapy.spiders.Spider.start_requests` method
-generating :class:`scrapy.Request <scrapy.http.Request>`
-objects from URLs, you can just put those URLs in the 
-:attr:`~scrapy.spiders.Spider.start_urls` attribute::
+A shortcut to the start_requests method
+---------------------------------------
+Instead of implementing a :meth:`~scrapy.spiders.Spider.start_requests` method
+that generates :class:`scrapy.Request <scrapy.http.Request>` objects from URLs,
+you can just define a :attr:`~scrapy.spiders.Spider.start_urls` class attribute
+with a list of URLs. This list will then be used by the default implementation
+of :meth:`~scrapy.spiders.Spider.start_requests` to create the initial requests
+for your spider::
 
     import scrapy
 
@@ -174,7 +176,8 @@ objects from URLs, you can just put those URLs in the
 The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle
 each of the requests for those URLs, even though we haven't explicitely told
 Scrapy to do so. This happens because :meth:`~scrapy.spiders.Spider.parse`
-is Scrapy's default callback method.
+is Scrapy's default callback method that is called for any request that have
+been generated with no callback explicitely assigned to handle it.
 
 
 Extracting data
@@ -293,7 +296,104 @@ Extraction wrap-up
 Now that you know a bit about selection and extraction, let's complete our
 spider by writing the code to extract the quotes from the webpage.
 
-TODO: show how to extract quotes and integrate spider code here.
+Each quote in http://quotes.toscrape.com is represented by HTML code that looks
+like this::
+
+    <div class="quote">
+        <span class="text">“The world as we have created it is a process of our
+        thinking. It cannot be changed without changing our thinking.”</span>
+        <span>
+            by <small class="author">Albert Einstein</small>
+            <a href="/author/Albert-Einstein">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <a class="tag" href="/tag/change/page/1/">change</a>
+            <a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
+            <a class="tag" href="/tag/thinking/page/1/">thinking</a>
+            <a class="tag" href="/tag/world/page/1/">world</a>
+        </div>
+    </div>
+
+Let's open up scrapy shell and play a bit to find out how to extract the data
+we want::
+
+    $ scrapy shell http://quotes.toscrape.com
+
+We get a list of selectors to the quotes using::
+
+    >>> response.css("div.quote")
+
+Each of the selectors returned by the query above allows us to run further
+queries over the quotes itselves. Let's assign the first selector to a
+variable, so that we can run our CSS selectors directly on a particular quote::
+
+    >>> quote = response.css("div.quote")[0]
+
+Now, let's extract ``title``, ``author`` and the ``tags`` from that quote
+using the ``quote`` object we just created::
+
+    >>> title = quote.css("span.text ::text").extract_first()
+    >>> title
+    '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
+    >>> author = quote.css("small.author ::text").extract_first()
+    >>> author
+    'Albert Einstein'
+
+Given that the tags is a list of strings, we can use the ``.extract()`` method
+to get all of them::
+
+    >>> tags = quote.css("div.tags a.tag ::text").extract()
+    >>> tags
+    ['change', 'deep-thoughts', 'thinking', 'world']
+
+Now, we can iterate over all the quotes in the page and use the CSS selectors
+we defined to extract data::
+
+    >>> for quote in response.css("div.quote"):
+    ...     text = quote.css("span.text ::text").extract_first()
+    ...     author = quote.css("small.author ::text").extract_first()
+    ...     tags = quote.css("div.tags a.tag ::text").extract()
+    ...     print("{} - {} - {}".format(text, author, tags))
+
+
+Extracting data in our spider
+------------------------------
+
+Until now, the spider we built doesn't extract any data in particular. I just
+saves the whole HTML page to a local file. Now, let's integrate the extraction
+logic above in our spider.
+
+A Scrapy spider typically generates many dictionaries containing the data
+extracted from the page. To do that, we use the ``yield`` Python keyword, as
+you can see below::
+
+    import scrapy
+
+
+    class QuotesSpider(scrapy.Spider):
+        name = "quotes"
+        start_urls = [
+            'http://quotes.toscrape.com/page/1/',
+            'http://quotes.toscrape.com/page/2/',
+        ]
+
+        def parse(self, response):
+            for quote in response.css('div.quote'):
+                yield {
+                    'text': quote.css('span.text::text').extract_first(),
+                    'author': quote.css('span small::text').extract_first(),
+                    'tags': quote.css("div.tags a.tag ::text").extract(),
+                }
+
+If you run this spider, it will output the extracted data with the log::
+
+    2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
+    {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'}
+    2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
+    {'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"}
+
+:ref:`Later in the tutorial <storing-data>`, we will see how to save this data to a file.
 
 
 Following links
@@ -450,6 +550,8 @@ If you pass the ``tag=humor`` argument to this spider, you'll notice that it
 will only visit URLs from the ``humor`` tag, such as
 ``http://quotes.toscrape.com/tag/humor``.
 
+.. _storing-data:
+
 Storing the scraped data
 ========================
 

From 8893527a2f48fbcf9b9fb51e069bdd7ea79dd8b3 Mon Sep 17 00:00:00 2001
From: Ashish Kulkarni <ashish@advarisk.com>
Date: Fri, 5 Aug 2016 12:54:18 +0530
Subject: [PATCH 1082/2974] temporarily deprecate official Ubuntu packages

They are not currently updated and fail to install on
Ubuntu 16.04. Also update the instructions to refer to
the earliest supported LTS (Ubuntu 12.04).

fixes #2137 and closes #2076
---
 docs/intro/install.rst | 11 +++++------
 docs/topics/ubuntu.rst |  3 +++
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 3364c3b31..59c8285a4 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -89,15 +89,14 @@ Windows
      Python 3 is not supported on Windows. This is because Scrapy core requirement Twisted does not support
      Python 3 on Windows.
 
-Ubuntu 9.10 or above
---------------------
+Ubuntu 12.04 or above
+---------------------
 
 **Don't** use the ``python-scrapy`` package provided by Ubuntu, they are
-typically too old and slow to catch up with latest Scrapy.
+typically too old and slow to catch up with latest Scrapy. You will need
+to install the following system packages::
 
-Instead, use the official :ref:`Ubuntu Packages <topics-ubuntu>`, which already
-solve all dependencies for you and are continuously updated with the latest bug
-fixes.
+    sudo apt-get install python-pip python-twisted python-openssl python-lxml
 
 If you prefer to build the python dependencies locally instead of relying on
 system packages you'll need to install their required non-python dependencies
diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst
index 102df581c..f1a870d62 100644
--- a/docs/topics/ubuntu.rst
+++ b/docs/topics/ubuntu.rst
@@ -11,6 +11,9 @@ those in Ubuntu, and more stable too since they're continuously built from
 `GitHub repo`_ (master & stable branches) and so they contain the latest bug
 fixes.
 
+.. caution:: These packages are currently not updated and may not work on
+   Ubuntu 16.04 and above, see :issue:`2076` and :issue:`2137`.
+
 To use the packages:
 
 1. Import the GPG key used to sign Scrapy packages into APT keyring::

From a135dbaf19f45b9cf59b4e89954f93c74bdf7771 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 20 Sep 2016 12:47:33 +0200
Subject: [PATCH 1083/2974] Log warning when request cannot be serialized
 (instead of error)

Fixes GH-2035
---
 scrapy/core/scheduler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index dcd6fb989..a54b4daf0 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -89,8 +89,8 @@ class Scheduler(object):
                 msg = ("Unable to serialize request: %(request)s - reason:"
                        " %(reason)s - no more unserializable requests will be"
                        " logged (stats being collected)")
-                logger.error(msg, {'request': request, 'reason': e},
-                             exc_info=True, extra={'spider': self.spider})
+                logger.warning(msg, {'request': request, 'reason': e},
+                               exc_info=True, extra={'spider': self.spider})
                 self.logunser = False
             self.stats.inc_value('scheduler/unserializable',
                                  spider=self.spider)

From 81a0e3cd93954ecd0a728ef5fcf5dc92a281fdd5 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 20 Sep 2016 13:42:28 +0200
Subject: [PATCH 1084/2974] Raise log level for HttpErrorMiddleware to INFO
 (from DEBUG)

Fixes GH-910
---
 scrapy/spidermiddlewares/httperror.py    |  2 +-
 tests/test_spidermiddleware_httperror.py | 27 ++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index 6b7c50fce..e34c265c1 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -46,7 +46,7 @@ class HttpErrorMiddleware(object):
 
     def process_spider_exception(self, response, exception, spider):
         if isinstance(exception, HttpError):
-            logger.debug(
+            logger.info(
                 "Ignoring response %(response)r: HTTP status code is not handled or not allowed",
                 {'response': response}, extra={'spider': spider},
             )
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index a64400482..319746350 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -1,3 +1,4 @@
+import logging
 from unittest import TestCase
 
 from testfixtures import LogCapture
@@ -185,3 +186,29 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
         self.assertIn('Ignoring response <500', str(log))
         self.assertNotIn('Ignoring response <200', str(log))
         self.assertNotIn('Ignoring response <402', str(log))
+
+    @defer.inlineCallbacks
+    def test_logging_level(self):
+        # HttpError logs ignored responses with level INFO
+        crawler = get_crawler(_HttpErrorSpider)
+        with LogCapture(level=logging.INFO) as log:
+            yield crawler.crawl()
+        self.assertEqual(crawler.spider.parsed, {'200'})
+        self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
+
+        self.assertIn('Ignoring response <402', str(log))
+        self.assertIn('Ignoring response <404', str(log))
+        self.assertIn('Ignoring response <500', str(log))
+        self.assertNotIn('Ignoring response <200', str(log))
+
+        # with level WARNING, we shouldn't capture anything from HttpError
+        crawler = get_crawler(_HttpErrorSpider)
+        with LogCapture(level=logging.WARNING) as log:
+            yield crawler.crawl()
+        self.assertEqual(crawler.spider.parsed, {'200'})
+        self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
+
+        self.assertNotIn('Ignoring response <402', str(log))
+        self.assertNotIn('Ignoring response <404', str(log))
+        self.assertNotIn('Ignoring response <500', str(log))
+        self.assertNotIn('Ignoring response <200', str(log))

From f4f93c5c266648317ff2c2e474b7d5fd08918c07 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 20 Sep 2016 09:19:48 -0300
Subject: [PATCH 1085/2974] fix tox docs build, adjust title

---
 docs/intro/tutorial.rst | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 473183be3..6a5e99d7a 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -265,7 +265,7 @@ information see :ref:`topics-firebug` and :ref:`topics-firefox`.
 XPath: a brief intro
 ^^^^^^^^^^^^^^^^^^^^
 
-Besides CSS, Scrapy selectors also support using `XPath`_ expressions::
+Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions::
 
     >>> response.xpath('//title')
     [<Selector xpath='//title' data=u'<title>Quotes to Scrape</title>'>]
@@ -289,6 +289,8 @@ XPath through examples <http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this
 tutorial to learn "how to think in XPath"
 <http://plasmasturm.org/log/xpath101/>`_.
 
+.. _XPath: https://www.w3.org/TR/xpath
+.. _CSS: https://www.w3.org/TR/selectors
 
 Extraction wrap-up
 ^^^^^^^^^^^^^^^^^^
@@ -453,7 +455,7 @@ using a :ref:`trick to pass additional data to the callbacks
 <topics-request-response-ref-request-callback-arguments>`.
 
 Another example: scraping authors
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+---------------------------------
 
 Here is another spider that illustrates callbacks and following links,
 this time for scraping author information::
@@ -509,8 +511,9 @@ much because of a programming mistake. This can be configured by the setting
     spider that implements a small rules engine that you can use to write your
     crawlers on top of it.
 
-Customizing behavior via spider arguments
-=========================================
+Adding a spider argument
+========================
+
 You can provide command line arguments to your spiders by using the ``-a``
 option when running them::
 

From e59d79bf37a8ecbef6679145815b000008bb6624 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 20 Sep 2016 17:17:22 +0200
Subject: [PATCH 1086/2974] Add note on "to" and "cc" as lists for sending
 emails

Fixes GH-2244
---
 docs/topics/email.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index 62ebc4c08..18d2f8084 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -35,6 +35,12 @@ And here is how to use it to send an e-mail (without attachments)::
 
     mailer.send(to=["someone@example.com"], subject="Some subject", body="Some body", cc=["another@example.com"])
 
+.. note::
+    As shown in the example above, ``to`` and ``cc`` need to be lists
+    of email addresses, not single addresses, and even for one recipient,
+    i.e. ``to="someone@example.com"`` will not work.
+
+
 MailSender class reference
 ==========================
 

From 125b691102320864c635608618636253074eae1a Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 20 Sep 2016 12:47:03 -0300
Subject: [PATCH 1087/2974] more reviewing and editing, minor restructure,
 syntax fixes

---
 docs/intro/tutorial.rst | 141 ++++++++++++++++++++++++----------------
 1 file changed, 84 insertions(+), 57 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 6a5e99d7a..b4a5e3b48 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -14,7 +14,9 @@ This tutorial will walk you through these tasks:
 
 1. Creating a new Scrapy project
 2. Writing a :ref:`spider <topics-spiders>` to crawl a site and extract data
-3. Exporting the scraped data using command line
+3. Exporting the scraped data using the command line
+4. Change spider to recursively follow links
+5. Using spider arguments
 
 Scrapy is written in Python_. If you're new to the language you might want to
 start by getting an idea of what the language is like, to get the most out of
@@ -43,7 +45,7 @@ This will create a ``tutorial`` directory with the following contents::
         tutorial/             # project's Python module, you'll import your code from here
             __init__.py
 
-            items.py          # project items file
+            items.py          # project items definition file
 
             pipelines.py      # project pipelines file
 
@@ -109,7 +111,8 @@ and defines some attributes and methods:
 How to run our spider
 ---------------------
 
-To put our spider to work, go to the project's top level directory and run::
+To put our spider to work, go to the project's top level directory (``cd
+tutorial``) and run::
 
    scrapy crawl quotes
 
@@ -141,6 +144,7 @@ for the respective URLs, as our ``parse`` method instructs.
 
 What just happened under the hood?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
 Scrapy schedules the :class:`scrapy.Request <scrapy.http.Request>` objects
 returned by the ``start_requests`` method of the Spider. Upon receiving
 a response for each one, it instantiates :class:`scrapy.http.Response`
@@ -173,11 +177,11 @@ for your spider::
             with open(filename, 'wb') as f:
                 f.write(response.body)
 
-The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle
-each of the requests for those URLs, even though we haven't explicitely told
-Scrapy to do so. This happens because :meth:`~scrapy.spiders.Spider.parse`
-is Scrapy's default callback method that is called for any request that have
-been generated with no callback explicitely assigned to handle it.
+The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each
+of the requests for those URLs, even though we haven't explicitely told Scrapy
+to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's
+default callback method, which is called for requests without an explicitely
+assigned callback.
 
 
 Extracting data
@@ -224,10 +228,14 @@ To extract the text from the title above, you can do::
 
 There are two things to note here: one is that we've added ``::text`` to the
 CSS query, to mean that we want to select the text from inside the title element.
+If we don't specify ``::text``, we'd get the HTML tags::
 
-The other is that the result of calling ``.extract()`` is a list, because we're
-dealing with an instance :class:`~scrapy.selector.SelectorList`.  When you know
-you just want the first result, as in this case, you can do::
+    >>> response.css('title').extract()
+    [u'<title>Quotes to Scrape</title>']
+
+The other thing is that the result of calling ``.extract()`` is a list, because
+we're dealing with an instance of :class:`~scrapy.selector.SelectorList`.  When
+you know you just want the first result, as in this case, you can do::
 
     >>> response.css('title::text').extract_first()
     u'Quotes to Scrape'
@@ -284,22 +292,24 @@ that contains the text "Next Page"**. This makes XPath very fitting to the task
 of scraping, and we encourage you to learn XPath even if you already know how to
 construct CSS selectors, it will make scraping much easier.
 
-We won't cover much of XPath here. To learn more about XPath, we recommend `this tutorial to learn
-XPath through examples <http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this
-tutorial to learn "how to think in XPath"
-<http://plasmasturm.org/log/xpath101/>`_.
+We won't cover much of XPath here. To learn more about XPath, we recommend
+`this tutorial to learn XPath through examples
+<http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this tutorial to learn "how
+to think in XPath" <http://plasmasturm.org/log/xpath101/>`_.
 
 .. _XPath: https://www.w3.org/TR/xpath
 .. _CSS: https://www.w3.org/TR/selectors
 
-Extraction wrap-up
-^^^^^^^^^^^^^^^^^^
+Extracting quotes and authors
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Now that you know a bit about selection and extraction, let's complete our
 spider by writing the code to extract the quotes from the webpage.
 
-Each quote in http://quotes.toscrape.com is represented by HTML code that looks
-like this::
+Each quote in http://quotes.toscrape.com is represented by HTML elements that look
+like this:
+
+.. code-block:: html
 
     <div class="quote">
         <span class="text">“The world as we have created it is a process of our
@@ -322,12 +332,12 @@ we want::
 
     $ scrapy shell http://quotes.toscrape.com
 
-We get a list of selectors to the quotes using::
+We get a list of selectors for the quote HTML elements with::
 
     >>> response.css("div.quote")
 
 Each of the selectors returned by the query above allows us to run further
-queries over the quotes itselves. Let's assign the first selector to a
+queries over their sub-elements. Let's assign the first selector to a
 variable, so that we can run our CSS selectors directly on a particular quote::
 
     >>> quote = response.css("div.quote")[0]
@@ -342,33 +352,33 @@ using the ``quote`` object we just created::
     >>> author
     'Albert Einstein'
 
-Given that the tags is a list of strings, we can use the ``.extract()`` method
+Given that the tags are a list of strings, we can use the ``.extract()`` method
 to get all of them::
 
     >>> tags = quote.css("div.tags a.tag ::text").extract()
     >>> tags
     ['change', 'deep-thoughts', 'thinking', 'world']
 
-Now, we can iterate over all the quotes in the page and use the CSS selectors
-we defined to extract data::
+Having figured out how to extract each bit, we can now iterate over all the
+quotes elements and put them together into a Python dictionary::
 
     >>> for quote in response.css("div.quote"):
     ...     text = quote.css("span.text ::text").extract_first()
     ...     author = quote.css("small.author ::text").extract_first()
     ...     tags = quote.css("div.tags a.tag ::text").extract()
-    ...     print("{} - {} - {}".format(text, author, tags))
+    ...     print(dict(text=text, author=author, tags=tags))
 
 
 Extracting data in our spider
 ------------------------------
 
-Until now, the spider we built doesn't extract any data in particular. I just
-saves the whole HTML page to a local file. Now, let's integrate the extraction
-logic above in our spider.
+Let's get back to our spider. Until now, it doesn't extract any data in
+particular, just saves the whole HTML page to a local file. Let's integrate the
+extraction logic above into our spider.
 
 A Scrapy spider typically generates many dictionaries containing the data
-extracted from the page. To do that, we use the ``yield`` Python keyword, as
-you can see below::
+extracted from the page. To do that, we use the ``yield`` Python keyword
+in the callback, as you can see below::
 
     import scrapy
 
@@ -395,7 +405,38 @@ If you run this spider, it will output the extracted data with the log::
     2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
     {'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"}
 
-:ref:`Later in the tutorial <storing-data>`, we will see how to save this data to a file.
+
+.. _storing-data:
+
+Storing the scraped data
+========================
+
+The simplest way to store the scraped data is by using :ref:`Feed exports
+<topics-feed-exports>`, with the following command::
+
+    scrapy crawl quotes -o items.json
+
+That will generate an ``items.json`` file containing all scraped items,
+serialized in `JSON`_.
+
+You could've also used other formats, like `JSON Lines`_::
+
+    scrapy crawl quotes -o items.jl
+
+The `JSON Lines`_ format is useful because it's stream-like, you can easily
+append new records to it. As each record is a separate line, you can also
+process big files without having to fit everything in memory, there are tools
+like `JQ`_ to help doing that at the command-line.
+
+In small projects (like the one in this tutorial), that should be enough.
+However, if you want to perform more complex things with the scraped items, you
+can write an :ref:`Item Pipeline <topics-item-pipeline>`. As with Items, a
+placeholder file for Item Pipelines has been set up for you when the project is
+created, in ``tutorial/pipelines.py``. Though you don't need to implement any item
+pipelines if you just want to store the scraped items.
+
+.. _JSON Lines: http://jsonlines.org
+.. _JQ: https://stedolan.github.io/jq
 
 
 Following links
@@ -511,17 +552,20 @@ much because of a programming mistake. This can be configured by the setting
     spider that implements a small rules engine that you can use to write your
     crawlers on top of it.
 
-Adding a spider argument
-========================
+Using spider arguments
+======================
 
 You can provide command line arguments to your spiders by using the ``-a``
 option when running them::
 
     scrapy crawl quotes -o items.json -a tag=humor
 
+These arguments are passed to the Spider's ``__init__`` method and become
+spider attributes by default.  
+
 In this example, the value provided for the ``tag`` argument will be available
-via a spider attribute. Using this, you could make your spider get only quotes
-tagged with a specific tag, building the URL based on the argument::
+via ``self.tag``. You can use this to make your spider fetch only quotes
+with a specific tag, building the URL based on the argument::
 
     import scrapy
 
@@ -553,25 +597,7 @@ If you pass the ``tag=humor`` argument to this spider, you'll notice that it
 will only visit URLs from the ``humor`` tag, such as
 ``http://quotes.toscrape.com/tag/humor``.
 
-.. _storing-data:
-
-Storing the scraped data
-========================
-
-The simplest way to store the scraped data is by using :ref:`Feed exports
-<topics-feed-exports>`, with the following command::
-
-    scrapy crawl quotes -o items.json
-
-That will generate an ``items.json`` file containing all scraped items,
-serialized in `JSON`_.
-
-In small projects (like the one in this tutorial), that should be enough.
-However, if you want to perform more complex things with the scraped items, you
-can write an :ref:`Item Pipeline <topics-item-pipeline>`. As with Items, a
-placeholder file for Item Pipelines has been set up for you when the project is
-created, in ``tutorial/pipelines.py``. Though you don't need to implement any item
-pipelines if you just want to store the scraped items.
+You can :ref:`learn more about handling spider arguments here <spiderargs>`.
 
 Next steps
 ==========
@@ -580,9 +606,10 @@ This tutorial covered only the basics of Scrapy, but there's a lot of other
 features not mentioned here. Check the :ref:`topics-whatelse` section in
 :ref:`intro-overview` chapter for a quick overview of the most important ones.
 
-Then, we recommend you continue by playing with an example project (see
-:ref:`intro-examples`), and then continue with the section
-:ref:`section-basics`.
+You can continue from the section :ref:`section-basics` to know more about the
+command-line tool, spiders and other things the tutorial haven't covered like
+modeling the scraped data. If you prefer to play with an example project, check
+the :ref:`intro-examples` section.
 
 .. _JSON: https://en.wikipedia.org/wiki/JSON
 .. _dirbot: https://github.com/scrapy/dirbot

From 40293551b22c0354b3bb20e8b7756e09aadf2dc3 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 20 Sep 2016 18:14:43 +0200
Subject: [PATCH 1088/2974] Remove mention of odd-numbered versions for
 development releases

Fixes GH-1317
---
 docs/versioning.rst | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/docs/versioning.rst b/docs/versioning.rst
index 8e7908762..0421ba544 100644
--- a/docs/versioning.rst
+++ b/docs/versioning.rst
@@ -7,22 +7,33 @@ Versioning and API Stability
 Versioning
 ==========
 
-Scrapy uses the `odd-numbered versions for development releases`_.
-
 There are 3 numbers in a Scrapy version: *A.B.C*
 
 * *A* is the major version. This will rarely change and will signify very
   large changes.
 * *B* is the release number. This will include many changes including features
-  and things that possibly break backwards compatibility. Even Bs will be
-  stable branches, and odd Bs will be development.
+  and things that possibly break backwards compatibility, although we strive to
+  keep theses cases at a minimum.
 * *C* is the bugfix release number.
 
+Backward-incompatibilities are explicitly mentioned in the :ref:`release notes <news>`,
+and may require special attention before upgrading.
+
+Development releases do not follow 3-numbers version and are generally
+released as ``dev`` suffixed versions, e.g. ``1.3dev``.
+
+.. note::
+    With Scrapy 0.* series, Scrapy used `odd-numbered versions for development releases`_.
+    This is not the case anymore from Scrapy 1.0 onwards.
+
+    Starting with Scrapy 1.0, all releases should be considered production-ready.
+
 For example:
 
-* *0.14.1* is the first bugfix release of the *0.14* series (safe to use in
+* *1.1.1* is the first bugfix release of the *1.1* series (safe to use in
   production)
 
+
 API Stability
 =============
 

From bc41fdf20e76ec06677a3d488323796ff2e126f7 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 20 Sep 2016 15:04:08 -0300
Subject: [PATCH 1089/2974] address review comments, add debug log to initial
 spider

---
 docs/intro/tutorial.rst | 94 ++++++++++++++++++++++++++---------------
 1 file changed, 61 insertions(+), 33 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index b4a5e3b48..162fa242e 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -15,7 +15,7 @@ This tutorial will walk you through these tasks:
 1. Creating a new Scrapy project
 2. Writing a :ref:`spider <topics-spiders>` to crawl a site and extract data
 3. Exporting the scraped data using the command line
-4. Change spider to recursively follow links
+4. Changing spider to recursively follow links
 5. Using spider arguments
 
 Scrapy is written in Python_. If you're new to the language you might want to
@@ -60,9 +60,9 @@ Our first Spider
 
 Spiders are classes that you define and that Scrapy uses to scrape information
 from a website (or group of websites). They must subclass
-:class:`scrapy.Spider` and define the initial requests to make, how to follow
-links in the pages, and how to parse the downloaded page content to extract
-data.
+:class:`scrapy.Spider` and define the initial requests to make, optionally how
+to follow links in the pages, and how to parse the downloaded page content to
+extract data.
 
 This is the code for our first Spider. Save it in a file named
 ``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
@@ -86,6 +86,7 @@ This is the code for our first Spider. Save it in a file named
             filename = 'quotes-%s.html' % page
             with open(filename, 'wb') as f:
                 f.write(response.body)
+            self.log('Saved file %s' % filename)
 
 
 As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.spiders.Spider>`
@@ -120,19 +121,17 @@ This command runs the spider with name ``quotes`` that we've just added, that
 will send some requests for the ``quotes.toscrape.com`` domain. You will get an output
 similar to this::
 
-
-    2016-09-01 16:51:27 [scrapy] INFO: Scrapy started (bot: tutorial)
-    2016-09-01 16:51:27 [scrapy] INFO: Overridden settings: {...}
-    2016-09-01 16:51:27 [scrapy] INFO: Enabled extensions: ...
-    2016-09-01 16:51:27 [scrapy] INFO: Enabled downloader middlewares: ...
-    2016-09-01 16:51:27 [scrapy] INFO: Enabled spider middlewares: ...
-    2016-09-01 16:51:27 [scrapy] INFO: Enabled item pipelines: ...
-    2016-09-01 16:51:27 [scrapy] INFO: Spider opened
-    2016-09-01 16:51:27 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
-    2016-09-01 16:51:28 [scrapy] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
-    2016-09-01 16:51:28 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
-    2016-09-01 16:51:29 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
-    2016-09-01 16:51:29 [scrapy] INFO: Closing spider (finished)
+    ... (omitted for brevity)
+    2016-09-20 14:48:00 [scrapy] INFO: Spider opened
+    2016-09-20 14:48:00 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
+    2016-09-20 14:48:00 [scrapy] DEBUG: Telnet console listening on 127.0.0.1:6023
+    2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
+    2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
+    2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-1.html
+    2016-09-20 14:48:01 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
+    2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-2.html
+    2016-09-20 14:48:01 [scrapy] INFO: Closing spider (finished)
+    ...
 
 Now, check the files in the current directory. You should notice that two new
 files have been created: *quotes-1.html* and *quotes-2.html*, with the content
@@ -178,9 +177,9 @@ for your spider::
                 f.write(response.body)
 
 The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each
-of the requests for those URLs, even though we haven't explicitely told Scrapy
+of the requests for those URLs, even though we haven't explicitly told Scrapy
 to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's
-default callback method, which is called for requests without an explicitely
+default callback method, which is called for requests without an explicitly
 assigned callback.
 
 
@@ -190,7 +189,7 @@ Extracting data
 The best way to learn how to extract data with Scrapy is trying selectors
 using the shell :ref:`Scrapy shell <topics-shell>`. Run::
 
-    scrapy crawl http://quotes.toscrape.com/page/1/
+    scrapy shell http://quotes.toscrape.com/page/1/
 
 You will see something like::
 
@@ -304,7 +303,7 @@ Extracting quotes and authors
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Now that you know a bit about selection and extraction, let's complete our
-spider by writing the code to extract the quotes from the webpage.
+spider by writing the code to extract the quotes from the web page.
 
 Each quote in http://quotes.toscrape.com is represented by HTML elements that look
 like this:
@@ -345,17 +344,17 @@ variable, so that we can run our CSS selectors directly on a particular quote::
 Now, let's extract ``title``, ``author`` and the ``tags`` from that quote
 using the ``quote`` object we just created::
 
-    >>> title = quote.css("span.text ::text").extract_first()
+    >>> title = quote.css("span.text::text").extract_first()
     >>> title
     '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
-    >>> author = quote.css("small.author ::text").extract_first()
+    >>> author = quote.css("small.author::text").extract_first()
     >>> author
     'Albert Einstein'
 
 Given that the tags are a list of strings, we can use the ``.extract()`` method
 to get all of them::
 
-    >>> tags = quote.css("div.tags a.tag ::text").extract()
+    >>> tags = quote.css("div.tags a.tag::text").extract()
     >>> tags
     ['change', 'deep-thoughts', 'thinking', 'world']
 
@@ -363,10 +362,14 @@ Having figured out how to extract each bit, we can now iterate over all the
 quotes elements and put them together into a Python dictionary::
 
     >>> for quote in response.css("div.quote"):
-    ...     text = quote.css("span.text ::text").extract_first()
-    ...     author = quote.css("small.author ::text").extract_first()
-    ...     tags = quote.css("div.tags a.tag ::text").extract()
+    ...     text = quote.css("span.text::text").extract_first()
+    ...     author = quote.css("small.author::text").extract_first()
+    ...     tags = quote.css("div.tags a.tag::text").extract()
     ...     print(dict(text=text, author=author, tags=tags))
+    {'text': u'\u201cThe world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.\u201d', 'tags': [u'change', u'deep-thoughts', u'thinking', u'world'], 'author': u'Albert Einstein'}
+    {'text': u'\u201cIt is our choices, Harry, that show what we truly are, far more than our abilities.\u201d', 'tags': [u'abilities', u'choices'], 'author': u'J.K. Rowling'}
+        ... a few more of these, omitted for brevity
+    >>>
 
 
 Extracting data in our spider
@@ -395,7 +398,7 @@ in the callback, as you can see below::
                 yield {
                     'text': quote.css('span.text::text').extract_first(),
                     'author': quote.css('span small::text').extract_first(),
-                    'tags': quote.css("div.tags a.tag ::text").extract(),
+                    'tags': quote.css("div.tags a.tag::text").extract(),
                 }
 
 If you run this spider, it will output the extracted data with the log::
@@ -430,9 +433,9 @@ like `JQ`_ to help doing that at the command-line.
 
 In small projects (like the one in this tutorial), that should be enough.
 However, if you want to perform more complex things with the scraped items, you
-can write an :ref:`Item Pipeline <topics-item-pipeline>`. As with Items, a
-placeholder file for Item Pipelines has been set up for you when the project is
-created, in ``tutorial/pipelines.py``. Though you don't need to implement any item
+can write an :ref:`Item Pipeline <topics-item-pipeline>`. A placeholder file
+for Item Pipelines has been set up for you when the project is created, in
+``tutorial/pipelines.py``. Though you don't need to implement any item
 pipelines if you just want to store the scraped items.
 
 .. _JSON Lines: http://jsonlines.org
@@ -448,7 +451,31 @@ from http://quotes.toscrape.com, you want quotes from all the pages in the websi
 Now that you know how to extract data from pages, let's see how to follow links
 from them.
 
-Here is a modification of our spider that recursively follows the link to the next
+First thing is to extract the link to the page we want to follow.  Examining
+our page, we can see there is a link to the next page with the following
+markup:
+
+.. code-block:: html
+
+    <ul class="pager">
+        <li class="next">
+            <a href="/page/2/">Next <span aria-hidden="true">&rarr;</span></a>
+        </li>
+    </ul>
+
+We can try extracting it in the shell::
+
+    >>> response.css('li.next a').extract_first()
+    u'<a href="/page/2/">Next <span aria-hidden="true">\u2192</span></a>'
+
+This gets the anchor element, but we want the attribute ``href``. For that,
+Scrapy supports a CSS extension that let's you select the attribute contents,
+like this::
+
+    >>> response.css('li.next a::attr("href")').extract_first()
+    u'/page/2/'
+
+Let's see now our spider modified to recursively follows the link to the next
 page, extracting data from it::
 
     import scrapy
@@ -465,6 +492,7 @@ page, extracting data from it::
                 yield {
                     'text': quote.css('span.text::text').extract_first(),
                     'author': quote.css('span small::text').extract_first(),
+                    'tags': quote.css("div.tags a.tag::text").extract(),
                 }
 
             next_page = response.css('li.next a::attr("href")').extract_first()
@@ -534,7 +562,7 @@ this time for scraping author information::
 
 This spider will start from the main page, it will follow all the links to the
 authors pages calling the ``parse_author`` callback for each of them, and also
-the pagination links too with the ``parse`` callback as we saw before.
+the pagination links with the ``parse`` callback as we saw before.
 
 The ``parse_author`` callback defines a helper function to extract and cleanup the
 data from a CSS query and yields the Python dict with the author data.

From a876ea5bd2911d5f7d06dfbb4ddbbdde8c51bc27 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 20 Sep 2016 15:10:49 -0300
Subject: [PATCH 1090/2974] minor grammar fix

---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 162fa242e..f4b2d0693 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -475,7 +475,7 @@ like this::
     >>> response.css('li.next a::attr("href")').extract_first()
     u'/page/2/'
 
-Let's see now our spider modified to recursively follows the link to the next
+Let's see now our spider modified to recursively follow the link to the next
 page, extracting data from it::
 
     import scrapy

From c126c593619ebbab6367f22557a44df93486942f Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 20 Sep 2016 18:19:25 -0300
Subject: [PATCH 1091/2974] address more review comments

---
 docs/intro/tutorial.rst | 119 +++++++++++++++++++++++-----------------
 1 file changed, 69 insertions(+), 50 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index f4b2d0693..65746c389 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -59,7 +59,7 @@ Our first Spider
 ================
 
 Spiders are classes that you define and that Scrapy uses to scrape information
-from a website (or group of websites). They must subclass
+from a website (or a group of websites). They must subclass
 :class:`scrapy.Spider` and define the initial requests to make, optionally how
 to follow links in the pages, and how to parse the downloaded page content to
 extract data.
@@ -96,7 +96,7 @@ and defines some attributes and methods:
   unique within a project, that is, you can't set the same name for different
   Spiders.
 
-* :meth:`~scrapy.spiders.Spider.start_requests`: must return a list
+* :meth:`~scrapy.spiders.Spider.start_requests`: must generate or return a list
   of requests where the Spider will begin to crawl from.
   Subsequent requests will be generated successively from these initial requests.
 
@@ -112,8 +112,7 @@ and defines some attributes and methods:
 How to run our spider
 ---------------------
 
-To put our spider to work, go to the project's top level directory (``cd
-tutorial``) and run::
+To put our spider to work, go to the project's top level directory and run::
 
    scrapy crawl quotes
 
@@ -145,10 +144,10 @@ What just happened under the hood?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Scrapy schedules the :class:`scrapy.Request <scrapy.http.Request>` objects
-returned by the ``start_requests`` method of the Spider. Upon receiving
-a response for each one, it instantiates :class:`scrapy.http.Response`
-objects and calls the ``parse`` callback method passing the response as
-argument.
+returned by the ``start_requests`` method of the Spider. Upon receiving a
+response for each one, it instantiates :class:`scrapy.http.Response` objects
+and calls the callback method associated with the request (in this case, the
+``parse`` method) passing the response as argument.
 
 
 A shortcut to the start_requests method
@@ -166,8 +165,8 @@ for your spider::
     class QuotesSpider(scrapy.Spider):
         name = "quotes"
         start_urls = [
-          'http://quotes.toscrape.com/page/1/',
-          'http://quotes.toscrape.com/page/2/',
+            'http://quotes.toscrape.com/page/1/',
+            'http://quotes.toscrape.com/page/2/',
         ]
 
         def parse(self, response):
@@ -189,13 +188,20 @@ Extracting data
 The best way to learn how to extract data with Scrapy is trying selectors
 using the shell :ref:`Scrapy shell <topics-shell>`. Run::
 
-    scrapy shell http://quotes.toscrape.com/page/1/
+    scrapy shell 'http://quotes.toscrape.com/page/1/'
+
+.. note::
+
+   Remember to always enclose urls in quotes when running Scrapy shell from
+   command-line, otherwise urls containing arguments (ie. ``&`` character)
+   will not work.
 
 You will see something like::
 
     [ ... Scrapy log here ... ]
     2016-09-19 12:09:27 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
     [s] Available Scrapy objects:
+    [s]   scrapy     scrapy module (contains scrapy.Request, scrapy.Selector, etc)
     [s]   crawler    <scrapy.crawler.Crawler object at 0x7fa91d888c90>
     [s]   item       {}
     [s]   request    <GET http://quotes.toscrape.com/page/1/>
@@ -212,7 +218,7 @@ Using the shell, you can try selecting elements using `CSS`_ with the response
 object::
 
     >>> response.css('title')
-    [<Selector xpath=u'descendant-or-self::title' data=u'<title>Quotes to Scrape</title>'>]
+    [<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
 
 The result of running ``response.css('title')`` is a list-like object called
 :class:`~scrapy.selector.SelectorList`, which represents a list of
@@ -223,26 +229,27 @@ data.
 To extract the text from the title above, you can do::
 
     >>> response.css('title::text').extract()
-    [u'Quotes to Scrape']
+    ['Quotes to Scrape']
 
 There are two things to note here: one is that we've added ``::text`` to the
-CSS query, to mean that we want to select the text from inside the title element.
-If we don't specify ``::text``, we'd get the HTML tags::
+CSS query, to mean we want to select only the text elements directly inside
+``<title>`` element.  If we don't specify ``::text``, we'd get the full title
+element, including its tags::
 
     >>> response.css('title').extract()
-    [u'<title>Quotes to Scrape</title>']
+    ['<title>Quotes to Scrape</title>']
 
 The other thing is that the result of calling ``.extract()`` is a list, because
 we're dealing with an instance of :class:`~scrapy.selector.SelectorList`.  When
 you know you just want the first result, as in this case, you can do::
 
     >>> response.css('title::text').extract_first()
-    u'Quotes to Scrape'
+    'Quotes to Scrape'
 
 As an alternative, you could've written::
 
     >>> response.css('title::text')[0].extract()
-    u'Quotes to Scrape'
+    'Quotes to Scrape'
 
 However, using ``.extract_first()`` avoids an ``IndexError`` and returns
 ``None`` when it doesn't find any element matching the selection.
@@ -253,21 +260,27 @@ to be scraped, you can at least get **some** data.
 
 Besides the :meth:`~scrapy.selector.Selector.extract` and
 :meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use
-the :meth:`~scrapy.selector.Selector.re` method to extract using a regular
-expression::
+the :meth:`~scrapy.selector.Selector.re` method to extract using `regular
+expressions`::
 
-    >>> response.css('title::text').re('Quotes.*')
-    [u'Quotes to Scrape']
-    >>> response.css('title::text').re('Q\w+')
-    [u'Quotes']
-    >>> response.css('title::text').re('(\w+) to (\w+)')
-    [u'Quotes', u'Scrape']
+    >>> response.css('title::text').re(r'Quotes.*')
+    ['Quotes to Scrape']
+    >>> response.css('title::text').re(r'Q\w+')
+    ['Quotes']
+    >>> response.css('title::text').re(r'(\w+) to (\w+)')
+    ['Quotes', 'Scrape']
 
 In order to find the proper CSS selectors to use, you might find useful opening
 the response page from the shell in your web browser using ``view(response)``.
-You can use your browser developer tools or extensions like Firebug. For more
+You can use your browser developer tools or extensions like Firebug.  For more
 information see :ref:`topics-firebug` and :ref:`topics-firefox`.
 
+`Selector Gadget`_ is also a nice tool to quickly find CSS selector for
+visually selected elements.
+
+.. _regular expressions: https://docs.python.org/3/library/re.html
+.. _Selector Gadget: http://selectorgadget.com/
+
 
 XPath: a brief intro
 ^^^^^^^^^^^^^^^^^^^^
@@ -275,9 +288,9 @@ XPath: a brief intro
 Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions::
 
     >>> response.xpath('//title')
-    [<Selector xpath='//title' data=u'<title>Quotes to Scrape</title>'>]
+    [<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
     >>> response.xpath('//title/text()').extract_first()
-    u'Quotes to Scrape'
+    'Quotes to Scrape'
 
 XPath expressions are very powerful, and are the foundation of Scrapy
 Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You
@@ -291,8 +304,9 @@ that contains the text "Next Page"**. This makes XPath very fitting to the task
 of scraping, and we encourage you to learn XPath even if you already know how to
 construct CSS selectors, it will make scraping much easier.
 
-We won't cover much of XPath here. To learn more about XPath, we recommend
-`this tutorial to learn XPath through examples
+We won't cover much of XPath here, but you can read more about `using XPath
+with Scrapy Selectors here <topics-selectors>`_. To learn more about XPath, we
+recommend `this tutorial to learn XPath through examples
 <http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this tutorial to learn "how
 to think in XPath" <http://plasmasturm.org/log/xpath101/>`_.
 
@@ -366,8 +380,8 @@ quotes elements and put them together into a Python dictionary::
     ...     author = quote.css("small.author::text").extract_first()
     ...     tags = quote.css("div.tags a.tag::text").extract()
     ...     print(dict(text=text, author=author, tags=tags))
-    {'text': u'\u201cThe world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.\u201d', 'tags': [u'change', u'deep-thoughts', u'thinking', u'world'], 'author': u'Albert Einstein'}
-    {'text': u'\u201cIt is our choices, Harry, that show what we truly are, far more than our abilities.\u201d', 'tags': [u'abilities', u'choices'], 'author': u'J.K. Rowling'}
+    {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
+    {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
         ... a few more of these, omitted for brevity
     >>>
 
@@ -417,19 +431,24 @@ Storing the scraped data
 The simplest way to store the scraped data is by using :ref:`Feed exports
 <topics-feed-exports>`, with the following command::
 
-    scrapy crawl quotes -o items.json
+    scrapy crawl quotes -o quotes.json
 
-That will generate an ``items.json`` file containing all scraped items,
+That will generate an ``quotes.json`` file containing all scraped items,
 serialized in `JSON`_.
 
-You could've also used other formats, like `JSON Lines`_::
+For historic reasons, Scrapy appends to a given file instead of overwriting
+its contents. If you run this command twice without removing the file
+before the second time, you'll end up with a broken JSON file.
 
-    scrapy crawl quotes -o items.jl
+You can also used other formats, like `JSON Lines`_::
+
+    scrapy crawl quotes -o quotes.jl
 
 The `JSON Lines`_ format is useful because it's stream-like, you can easily
-append new records to it. As each record is a separate line, you can also
-process big files without having to fit everything in memory, there are tools
-like `JQ`_ to help doing that at the command-line.
+append new records to it. It doesn't have the same problem of JSON when you run
+twice. Also, as each record is a separate line, you can process big files
+without having to fit everything in memory, there are tools like `JQ`_ to help
+doing that at the command-line.
 
 In small projects (like the one in this tutorial), that should be enough.
 However, if you want to perform more complex things with the scraped items, you
@@ -466,14 +485,14 @@ markup:
 We can try extracting it in the shell::
 
     >>> response.css('li.next a').extract_first()
-    u'<a href="/page/2/">Next <span aria-hidden="true">\u2192</span></a>'
+    '<a href="/page/2/">Next <span aria-hidden="true">→</span></a>'
 
 This gets the anchor element, but we want the attribute ``href``. For that,
 Scrapy supports a CSS extension that let's you select the attribute contents,
 like this::
 
-    >>> response.css('li.next a::attr("href")').extract_first()
-    u'/page/2/'
+    >>> response.css('li.next a::attr(href)').extract_first()
+    '/page/2/'
 
 Let's see now our spider modified to recursively follow the link to the next
 page, extracting data from it::
@@ -495,7 +514,7 @@ page, extracting data from it::
                     'tags': quote.css("div.tags a.tag::text").extract(),
                 }
 
-            next_page = response.css('li.next a::attr("href")').extract_first()
+            next_page = response.css('li.next a::attr(href)').extract_first()
             if next_page is not None:
                 next_page = response.urljoin(next_page)
                 yield scrapy.Request(next_page, callback=self.parse)
@@ -540,12 +559,12 @@ this time for scraping author information::
 
         def parse(self, response):
             # follow links to author pages
-            for href in response.css('.author a::attr("href")').extract():
+            for href in response.css('.author a::attr(href)').extract():
                 yield scrapy.Request(response.urljoin(href),
                                      callback=self.parse_author)
 
             # follow pagination links
-            next_page = response.css('li.next a::attr("href")').extract_first()
+            next_page = response.css('li.next a::attr(href)').extract_first()
             if next_page is not None:
                 next_page = response.urljoin(next_page)
                 yield scrapy.Request(next_page, callback=self.parse)
@@ -586,7 +605,7 @@ Using spider arguments
 You can provide command line arguments to your spiders by using the ``-a``
 option when running them::
 
-    scrapy crawl quotes -o items.json -a tag=humor
+    scrapy crawl quotes -o quotes-humor.json -a tag=humor
 
 These arguments are passed to the Spider's ``__init__`` method and become
 spider attributes by default.  
@@ -606,7 +625,7 @@ with a specific tag, building the URL based on the argument::
             tag = getattr(self, 'tag', None)
             if tag is not None:
                 url = url + 'tag/' + tag
-            yield scrapy.Request(url)
+            yield scrapy.Request(url, self.parse)
 
         def parse(self, response):
             for quote in response.css('div.quote'):
@@ -615,10 +634,10 @@ with a specific tag, building the URL based on the argument::
                     'author': quote.css('span small a::text').extract_first(),
                 }
 
-            next_page = response.css('li.next a::attr("href")').extract_first()
+            next_page = response.css('li.next a::attr(href)').extract_first()
             if next_page is not None:
                 next_page = response.urljoin(next_page)
-                yield scrapy.Request(next_page, callback=self.parse)
+                yield scrapy.Request(next_page, self.parse)
 
 
 If you pass the ``tag=humor`` argument to this spider, you'll notice that it

From 9c9690c76cf95df53c1eb2d3ad0360209eb2a50a Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 21 Sep 2016 10:30:08 -0300
Subject: [PATCH 1092/2974] add better messages for when response content isn't
 text (closes #2264)

---
 scrapy/http/response/__init__.py | 21 +++++++++++++++++++++
 tests/test_http_response.py      | 13 +++++++++++++
 2 files changed, 34 insertions(+)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 983154001..58ad414f1 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -9,6 +9,8 @@ from six.moves.urllib.parse import urljoin
 from scrapy.http.headers import Headers
 from scrapy.utils.trackref import object_ref
 from scrapy.http.common import obsolete_setter
+from scrapy.exceptions import NotSupported
+
 
 class Response(object_ref):
 
@@ -80,3 +82,22 @@ class Response(object_ref):
         """Join this Response's url with a possible relative url to form an
         absolute interpretation of the latter."""
         return urljoin(self.url, url)
+
+    @property
+    def text(self):
+        """For subclasses of TextResponse, this will return the body
+        as text (unicode object in Python 2 and str in Python 3)
+        """
+        raise AttributeError("Response content isn't text")
+
+    def css(self, *a, **kw):
+        """Shortcut method implemented only by responses whose content
+        is text (subclasses of TextResponse).
+        """
+        raise NotSupported("Response content isn't text")
+
+    def xpath(self, *a, **kw):
+        """Shortcut method implemented only by responses whose content
+        is text (subclasses of TextResponse).
+        """
+        raise NotSupported("Response content isn't text")
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index c7f36687a..7624aa4c4 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -7,6 +7,7 @@ from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
                          XmlResponse, Headers)
 from scrapy.selector import Selector
 from scrapy.utils.python import to_native_str
+from scrapy.exceptions import NotSupported
 
 
 class BaseResponseTest(unittest.TestCase):
@@ -127,6 +128,18 @@ class BaseResponseTest(unittest.TestCase):
         absolute = 'http://www.example.com/test'
         self.assertEqual(joined, absolute)
 
+    def test_shortcut_attributes(self):
+        r = self.response_class("http://example.com", body=b'hello')
+        if self.response_class == Response:
+            msg = "Response content isn't text"
+            self.assertRaisesRegexp(AttributeError, msg, getattr, r, 'text')
+            self.assertRaisesRegexp(NotSupported, msg, r.css, 'body')
+            self.assertRaisesRegexp(NotSupported, msg, r.xpath, '//body')
+        else:
+            r.text
+            r.css('body')
+            r.xpath('//body')
+
 
 class TextResponseTest(BaseResponseTest):
 

From 38266cc949f594c7f596728876151a65b481c966 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 21 Sep 2016 11:02:24 -0300
Subject: [PATCH 1093/2974] recommend Dive into Python and Python tutorial
 instead of LPTHW for non-beginners

---
 docs/intro/tutorial.rst | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 65746c389..ec68bf922 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -20,14 +20,17 @@ This tutorial will walk you through these tasks:
 
 Scrapy is written in Python_. If you're new to the language you might want to
 start by getting an idea of what the language is like, to get the most out of
-Scrapy.  If you're already familiar with other languages, and want to learn
-Python quickly, we recommend `Learn Python The Hard Way`_.  If you're new to programming
-and want to start with Python, take a look at `this list of Python resources
-for non-programmers`_.
+Scrapy. If you're already familiar with other languages, and want to learn
+Python quickly, we recommend reading through `Dive Into Python 3`_.
+Alternatively, you can follow the `Python Tutorial`_. If you're new to
+programming and want to start with Python, take a look at `this list of Python
+resources for non-programmers`_.
 
 .. _Python: https://www.python.org/
 .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
-.. _Learn Python The Hard Way: http://learnpythonthehardway.org/book/
+.. _Dive Into Python 3: http://www.diveintopython3.net
+.. _Python Tutorial: https://docs.python.org/3/tutorial
+
 
 Creating a project
 ==================

From 32017a76f8560d8cba2746d541adb63e03e68f62 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 21 Sep 2016 11:06:36 -0300
Subject: [PATCH 1094/2974] recommend learn python the hard way for beginners

---
 docs/intro/tutorial.rst | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index ec68bf922..31228017b 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -20,16 +20,21 @@ This tutorial will walk you through these tasks:
 
 Scrapy is written in Python_. If you're new to the language you might want to
 start by getting an idea of what the language is like, to get the most out of
-Scrapy. If you're already familiar with other languages, and want to learn
-Python quickly, we recommend reading through `Dive Into Python 3`_.
-Alternatively, you can follow the `Python Tutorial`_. If you're new to
-programming and want to start with Python, take a look at `this list of Python
-resources for non-programmers`_.
+Scrapy.
+
+If you're already familiar with other languages, and want to learn Python
+quickly, we recommend reading through `Dive Into Python 3`_.  Alternatively,
+you can follow the `Python Tutorial`_.
+
+If you're new to programming and want to start with Python, you may find useful
+the online book `Learn Python The Hard Way`_. You can also take a look at `this
+list of Python resources for non-programmers`_.
 
 .. _Python: https://www.python.org/
 .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
 .. _Dive Into Python 3: http://www.diveintopython3.net
 .. _Python Tutorial: https://docs.python.org/3/tutorial
+.. _Learn Python The Hard Way: http://learnpythonthehardway.org/book/
 
 
 Creating a project

From d636e5baa8a077e2869bfe3b76525efec42392ec Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 21 Sep 2016 18:54:12 -0300
Subject: [PATCH 1095/2974] better description for start_requests expected
 return value

---
 docs/intro/tutorial.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 31228017b..e85219e06 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -104,9 +104,10 @@ and defines some attributes and methods:
   unique within a project, that is, you can't set the same name for different
   Spiders.
 
-* :meth:`~scrapy.spiders.Spider.start_requests`: must generate or return a list
-  of requests where the Spider will begin to crawl from.
-  Subsequent requests will be generated successively from these initial requests.
+* :meth:`~scrapy.spiders.Spider.start_requests`: must return an iterable of
+  Requests (you can return a list of requests or write a generator function)
+  which the Spider will begin to crawl from. Subsequent requests will be
+  generated successively from these initial requests.
 
 * :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle
   the response downloaded for each of the requests made. The response parameter

From f4a22089168c31a2b6c2f03c0053073eb80e33b3 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 22 Sep 2016 11:04:45 -0300
Subject: [PATCH 1096/2974] addressing review comments and other minor editing

---
 docs/intro/tutorial.rst | 63 +++++++++++++++++++++--------------------
 1 file changed, 33 insertions(+), 30 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index e85219e06..4f2736709 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -111,8 +111,8 @@ and defines some attributes and methods:
 
 * :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle
   the response downloaded for each of the requests made. The response parameter
-  is an instance of :class:`~scrapy.http.Response` that holds the page content and
-  has further helpful methods to handle it.
+  is an instance of :class:`~scrapy.http.TextResponse` that holds
+  the page content and has further helpful methods to handle it.
 
   The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting
   the scraped data as dicts and also finding new URLs to
@@ -154,7 +154,7 @@ What just happened under the hood?
 
 Scrapy schedules the :class:`scrapy.Request <scrapy.http.Request>` objects
 returned by the ``start_requests`` method of the Spider. Upon receiving a
-response for each one, it instantiates :class:`scrapy.http.Response` objects
+response for each one, it instantiates :class:`~scrapy.http.Response` objects
 and calls the callback method associated with the request (in this case, the
 ``parse`` method) passing the response as argument.
 
@@ -281,11 +281,11 @@ expressions`::
 
 In order to find the proper CSS selectors to use, you might find useful opening
 the response page from the shell in your web browser using ``view(response)``.
-You can use your browser developer tools or extensions like Firebug.  For more
-information see :ref:`topics-firebug` and :ref:`topics-firefox`.
+You can use your browser developer tools or extensions like Firebug (see
+sections about :ref:`topics-firebug` and :ref:`topics-firefox`).
 
 `Selector Gadget`_ is also a nice tool to quickly find CSS selector for
-visually selected elements.
+visually selected elements, which works in many browsers.
 
 .. _regular expressions: https://docs.python.org/3/library/re.html
 .. _Selector Gadget: http://selectorgadget.com/
@@ -308,13 +308,13 @@ objects in the shell.
 
 While perhaps not as popular as CSS selectors, XPath expressions offer more
 power because besides navigating the structure, it can also look at the
-content. Using XPath, you're able to select things like: **select the link
-that contains the text "Next Page"**. This makes XPath very fitting to the task
+content. Using XPath, you're able to select things like: *select the link
+that contains the text "Next Page"*. This makes XPath very fitting to the task
 of scraping, and we encourage you to learn XPath even if you already know how to
 construct CSS selectors, it will make scraping much easier.
 
-We won't cover much of XPath here, but you can read more about `using XPath
-with Scrapy Selectors here <topics-selectors>`_. To learn more about XPath, we
+We won't cover much of XPath here, but you can read more about :ref:`using XPath
+with Scrapy Selectors here <topics-selectors>`. To learn more about XPath, we
 recommend `this tutorial to learn XPath through examples
 <http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this tutorial to learn "how
 to think in XPath" <http://plasmasturm.org/log/xpath101/>`_.
@@ -352,7 +352,7 @@ like this:
 Let's open up scrapy shell and play a bit to find out how to extract the data
 we want::
 
-    $ scrapy shell http://quotes.toscrape.com
+    $ scrapy shell 'http://quotes.toscrape.com'
 
 We get a list of selectors for the quote HTML elements with::
 
@@ -394,7 +394,6 @@ quotes elements and put them together into a Python dictionary::
         ... a few more of these, omitted for brevity
     >>>
 
-
 Extracting data in our spider
 ------------------------------
 
@@ -421,7 +420,7 @@ in the callback, as you can see below::
                 yield {
                     'text': quote.css('span.text::text').extract_first(),
                     'author': quote.css('span small::text').extract_first(),
-                    'tags': quote.css("div.tags a.tag::text").extract(),
+                    'tags': quote.css('div.tags a.tag::text').extract(),
                 }
 
 If you run this spider, it will output the extracted data with the log::
@@ -520,7 +519,7 @@ page, extracting data from it::
                 yield {
                     'text': quote.css('span.text::text').extract_first(),
                     'author': quote.css('span small::text').extract_first(),
-                    'tags': quote.css("div.tags a.tag::text").extract(),
+                    'tags': quote.css('div.tags a.tag::text').extract(),
                 }
 
             next_page = response.css('li.next a::attr(href)').extract_first()
@@ -530,10 +529,11 @@ page, extracting data from it::
 
 
 Now, after extracting the data, the ``parse()`` method looks for the link to
-the next page, builds a full absolute URL using the ``response.urljoin`` method
-(since the links can be relative) and yields a new request to the next page,
-registering itself as callback to handle the data extraction for the next page
-and to keep the crawling going through all the pages.
+the next page, builds a full absolute URL using the
+:meth:`~scrapy.http.Response.urljoin` method (since the links can be
+relative) and yields a new request to the next page, registering itself as
+callback to handle the data extraction for the next page and to keep the
+crawling going through all the pages.
 
 What you see here is Scrapy's mechanism of following links: when you yield
 a Request in a callback method, Scrapy will schedule that request to be sent
@@ -547,12 +547,8 @@ In our example, it creates a sort of loop, following all the links to the next p
 until it doesn't find one -- handy for crawling blogs, forums and other sites with
 pagination.
 
-Another common pattern is to build an item with data from more than one page,
-using a :ref:`trick to pass additional data to the callbacks
-<topics-request-response-ref-request-callback-arguments>`.
-
-Another example: scraping authors
----------------------------------
+More examples and patterns
+--------------------------
 
 Here is another spider that illustrates callbacks and following links,
 this time for scraping author information::
@@ -602,11 +598,18 @@ requests to URLs already visited, avoiding the problem of hitting servers too
 much because of a programming mistake. This can be configured by the setting
 :setting:`DUPEFILTER_CLASS`.
 
-.. note::
-    As another example spider that leverages the mechanism of following links,
-    check out the :class:`~scrapy.spiders.CrawlSpider` class for a generic
-    spider that implements a small rules engine that you can use to write your
-    crawlers on top of it.
+Hopefully by now you have a good understanding of how to use the mechanism
+of following links and callbacks with Scrapy.
+
+As yet another example spider that leverages the mechanism of following links,
+check out the :class:`~scrapy.spiders.CrawlSpider` class for a generic
+spider that implements a small rules engine that you can use to write your
+crawlers on top of it.
+
+Also, a common pattern is to build an item with data from more than one page,
+using a :ref:`trick to pass additional data to the callbacks
+<topics-request-response-ref-request-callback-arguments>`.
+
 
 Using spider arguments
 ======================
@@ -663,7 +666,7 @@ features not mentioned here. Check the :ref:`topics-whatelse` section in
 :ref:`intro-overview` chapter for a quick overview of the most important ones.
 
 You can continue from the section :ref:`section-basics` to know more about the
-command-line tool, spiders and other things the tutorial haven't covered like
+command-line tool, spiders, selectors and other things the tutorial hasn't covered like
 modeling the scraped data. If you prefer to play with an example project, check
 the :ref:`intro-examples` section.
 

From b2bfd1e5c536997a2c260e56e8e9fa1303691384 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Fri, 23 Sep 2016 10:36:03 +0200
Subject: [PATCH 1097/2974] [docs] document that process_item can return
 Deferred

---
 docs/topics/item-pipeline.rst | 55 +++++++++++++++++++++++++++++++++--
 1 file changed, 52 insertions(+), 3 deletions(-)

diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index b9b4c2058..a31fe7423 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -27,9 +27,10 @@ Each item pipeline component is a Python class that must implement the following
 
 .. method:: process_item(self, item, spider)
 
-   This method is called for every item pipeline component and must either return
-   a dict with data, :class:`~scrapy.item.Item` (or any descendant class) object 
-   or raise a :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
+   This method is called for every item pipeline component. :meth:`process_item`
+   must either: return a dict with data, return an :class:`~scrapy.item.Item`
+   (or any descendant class) object, return a `Twisted Deferred`_ or raise
+   :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
    processed by further pipeline components.
 
    :param item: the item scraped
@@ -66,6 +67,8 @@ Additionally, they may also implement the following methods:
    :type crawler: :class:`~scrapy.crawler.Crawler` object
 
 
+.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
+
 Item pipeline example
 =====================
 
@@ -163,6 +166,52 @@ method and how to clean up the resources properly.
 .. _MongoDB: https://www.mongodb.org/
 .. _pymongo: https://api.mongodb.org/python/current/
 
+
+Take screenshot of item
+-----------------------
+
+This example demonstrates how to return Deferred_ from :meth:`process_item` method.
+It uses Splash_ to render screenshot of item url. Pipeline
+makes request to locally running instance of Splash_. After request is downloaded
+and Deferred callback fires, it saves item to a file and adds filename to an item.
+
+::
+
+    import scrapy
+
+
+    class ScreenshotPipeline(object):
+        """Pipeline that uses Splash to render screenshot of
+        every Scrapy item."""
+
+        SPLASH_URL = "http://localhost:8050/render.png?url={}"
+
+        def process_item(self, item, spider):
+            item_url = item["url"]
+            screenshot_url = self.SPLASH_URL.format(item_url)
+            request = scrapy.Request(screenshot_url)
+            dfd = spider.crawler.engine.download(request, spider)
+            dfd.addBoth(self.return_item, item)
+            return dfd
+
+        def return_item(self, response, item):
+            if response.status != 200:
+                # Error happened, return item.
+                return item
+
+            # Save screenshot to file.
+            filename = "item_file_name.png"
+            with open(filename, "wb") as f:
+                f.write(response.body)
+
+            # Store filename in item.
+            item["screenshot_filename"] = filename
+            return item
+
+
+.. _Splash: http://splash.readthedocs.io/en/stable/
+.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
+
 Duplicates filter
 -----------------
 

From a0f87d2f45865158a6ea687a1603e4b38106ff3f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 22 Sep 2016 17:21:26 +0200
Subject: [PATCH 1098/2974] Update release notes for upcoming 1.1.3 bugfix
 release

---
 docs/news.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 5395db8e3..0992ebf89 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,22 @@
 Release notes
 =============
 
+1.1.3 (YYYY-MM-DD)
+------------------
+
+Bug fixes
+~~~~~~~~~
+
+- Class attributes for subclasses of ``ImagesPipeline`` and ``FilesPipeline``
+  work at they did before 1.1.1 (:issue:`2243`, fixes :issue:`2198`)
+
+Documentation
+~~~~~~~~~~~~~
+
+- :ref:`Overview <intro-overview>` and :ref:`tutorial <intro-tutorial>`
+  rewritten to use http://toscrape.com websites (:issue:`2236`, :issue:`2249`).
+
+
 1.1.2 (2016-08-18)
 ------------------
 

From 80c1e5dc252f09b3ab864405c87586d7db42dfac Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 22 Sep 2016 21:33:05 +0200
Subject: [PATCH 1099/2974] Set release date, fix typo and add tutorial
 improvement issue number

---
 docs/news.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 0992ebf89..0e9efe6f6 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,20 +3,21 @@
 Release notes
 =============
 
-1.1.3 (YYYY-MM-DD)
+1.1.3 (2016-09-22)
 ------------------
 
 Bug fixes
 ~~~~~~~~~
 
 - Class attributes for subclasses of ``ImagesPipeline`` and ``FilesPipeline``
-  work at they did before 1.1.1 (:issue:`2243`, fixes :issue:`2198`)
+  work as they did before 1.1.1 (:issue:`2243`, fixes :issue:`2198`)
 
 Documentation
 ~~~~~~~~~~~~~
 
 - :ref:`Overview <intro-overview>` and :ref:`tutorial <intro-tutorial>`
-  rewritten to use http://toscrape.com websites (:issue:`2236`, :issue:`2249`).
+  rewritten to use http://toscrape.com websites
+  (:issue:`2236`, :issue:`2249`, :issue:`2252`).
 
 
 1.1.2 (2016-08-18)

From 7eeee6f616da6b44bfeb24b47b8b4c1ccef21baf Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 23 Sep 2016 16:39:09 +0200
Subject: [PATCH 1100/2974] Mention \uXXXX escapes in regards to
 FEED_EXPORT_ENCODING

---
 docs/news.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 0ee7642cf..a6f1ff09d 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -10,8 +10,10 @@ New Features
 ~~~~~~~~~~~~
 
 - New :setting:`FEED_EXPORT_ENCODING` setting to customize the encoding
-  used when writing items to a file. This is useful for those wanting
-  something else than UTF-8 for XML or CSV output (:issue:`2034`).
+  used when writing items to a file.
+  This can be used to turn off ``\uXXXX`` escapes in JSON output.
+  This is also useful for those wanting something else than UTF-8
+  for XML or CSV output (:issue:`2034`).
 - ``startproject`` command now supports an optional destination directory
   to override the default one based on the project name (:issue:`2005`).
 - New :setting:`SCHEDULER_DEBUG` setting to log requests serialization

From bba31b545d89f60a3f1d4d6d972f6d413e688821 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 23 Sep 2016 17:23:37 +0200
Subject: [PATCH 1101/2974] Mention new Debian Jessie baseline

---
 docs/news.rst | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index a6f1ff09d..25e251ed6 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -6,6 +6,19 @@ Release notes
 1.2.0 (YYYY-MM-DD)
 ------------------
 
+Requirements
+~~~~~~~~~~~~
+
+Scrapy's new requirements baseline is Debian 7 Jessie. It was previously
+Ubuntu 12.04 Precise.
+What this means in practice is that we run continuous integration tests
+with these (main) packages versions at a minimum:
+Twisted 14.0, pyOpenSSL 0.14, lxml 3.4.
+
+Scrapy may very well work with older versions of these packages
+(the code base still has switches for older Twisted versions for example)
+but it is not guaranteed (because it's not tested anymore).
+
 New Features
 ~~~~~~~~~~~~
 

From 39dbd890b063af1b153824214fa973993560416c Mon Sep 17 00:00:00 2001
From: pawelmhm <pawelmhm@gmail.com>
Date: Sat, 24 Sep 2016 08:36:09 +0200
Subject: [PATCH 1102/2974] [docs/item_pipeline] process_item returning
 Deferred docs

* quote url
* use hash of url as filename
---
 docs/topics/item-pipeline.rst | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index a31fe7423..6b43fe258 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -178,6 +178,8 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
 ::
 
     import scrapy
+    import hashlib
+    from urllib.parse import quote
 
 
     class ScreenshotPipeline(object):
@@ -187,8 +189,8 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
         SPLASH_URL = "http://localhost:8050/render.png?url={}"
 
         def process_item(self, item, spider):
-            item_url = item["url"]
-            screenshot_url = self.SPLASH_URL.format(item_url)
+            encoded_item_url = quote(item["url"])
+            screenshot_url = self.SPLASH_URL.format(encoded_item_url)
             request = scrapy.Request(screenshot_url)
             dfd = spider.crawler.engine.download(request, spider)
             dfd.addBoth(self.return_item, item)
@@ -199,8 +201,10 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
                 # Error happened, return item.
                 return item
 
-            # Save screenshot to file.
-            filename = "item_file_name.png"
+            # Save screenshot to file, filename will be hash of url.
+            url = item["url"]
+            url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
+            filename = "{}.png".format(url_hash)
             with open(filename, "wb") as f:
                 f.write(response.body)
 
@@ -208,7 +212,6 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
             item["screenshot_filename"] = filename
             return item
 
-
 .. _Splash: http://splash.readthedocs.io/en/stable/
 .. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
 

From c707c313458d7d01a61ab81e11dab6579d19a27f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 26 Sep 2016 17:20:13 +0200
Subject: [PATCH 1103/2974] Fix Debian number

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 25e251ed6..882f3eab0 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -9,7 +9,7 @@ Release notes
 Requirements
 ~~~~~~~~~~~~
 
-Scrapy's new requirements baseline is Debian 7 Jessie. It was previously
+Scrapy's new requirements baseline is Debian 8 "Jessie". It was previously
 Ubuntu 12.04 Precise.
 What this means in practice is that we run continuous integration tests
 with these (main) packages versions at a minimum:

From 8d130b295a53508fd81c561a79ad5e40bffeed7b Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 28 Sep 2016 17:09:17 +0200
Subject: [PATCH 1104/2974] Update installation guide

---
 docs/intro/install.rst | 179 ++++++++++++++++++++++++++++++-----------
 1 file changed, 130 insertions(+), 49 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 59c8285a4..5f7e4fa04 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -7,48 +7,104 @@ Installation guide
 Installing Scrapy
 =================
 
-.. note:: Check :ref:`intro-install-platform-notes` first.
+Scrapy runs on Python 2.7 and Python 3.3 or above
+(except on Windows where Python 3 is not supported yet).
 
-The installation steps assume that you have the following things installed:
+If you’re already familiar with installation of Python packages,
+you can install Scrapy and its dependencies from PyPI with::
 
-* `Python`_ 2.7 or above 3.3
+    pip install Scrapy
 
-* `pip`_ and `setuptools`_ Python packages. Nowadays `pip`_ requires and
-  installs `setuptools`_ if not installed. Python 2.7.9 and later include
-  `pip`_ by default, so you may have it already.
+We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
+to avoid conflicting with your system packages.
 
-* `lxml`_. Most Linux distributions ships prepackaged versions of lxml.
-  Otherwise refer to http://lxml.de/installation.html
+For more detailed and platform specifics instructions, read on.
 
-* `OpenSSL`_. This comes preinstalled in all operating systems, except Windows
-  where the Python installer ships it bundled.
 
-You can install Scrapy using pip (which is the canonical way to install Python
-packages). To install using ``pip`` run::
+Things that are good to know
+----------------------------
+
+Scrapy is written in pure Python and depends on a few key Python packages (among others):
+
+* `lxml`_, an efficient XML and HTML parser
+* `parsel`_, an HTML/XML data extraction library written on top of lxml,
+* `w3lib`_, a multi-purpose helper for dealing with URLs and web page encodings
+* `twisted`_, an asynchronous networking framework
+* `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs
+
+The minimal versions which Scrapy is tested against are:
+
+* Twisted 14.0
+* lxml 3.4
+* pyOpenSSL 0.14
+
+Scrapy may work with older versions of these packages
+but it is not guaranteed it will continue working
+because it’s not being tested against them.
+
+Some of these packages themselves depends on non-Python packages
+that might require additional installation steps depending on your platform.
+Please check :ref:`platform-specific guides below <intro-install-platform-notes>`.
+
+In case of any trouble related to these dependencies,
+please refer to their respective installation instructions:
+
+* `lxml installation`_
+* `cryptography installation`_
+
+.. _lxml installation: http://lxml.de/installation.html
+.. _cryptography installation: https://cryptography.io/en/latest/installation/
+
+
+.. _intro-using-virtualenv:
+
+Using a virtual environment (recommended)
+-----------------------------------------
+
+TL;DR: We recommend installing Scrapy inside a virtual environment
+on all platforms.
+
+Python packages can be installed either globally (a.k.a system wide),
+or in user-space. We do not recommend installing scrapy system wide.
+
+Instead, we recommend that you install scrapy within a so-called
+"virtual environment" (`virtualenv`_).
+Virtualenvs allow you to not conflict with already-installed Python
+system packages (which could break some of your system tools and scripts),
+and still install packages normally with ``pip`` (without ``sudo`` and the likes).
+
+To get started with virtual environments, see `virtualenv installation instructions`_.
+To install it globally (having it globally installed actually helps here),
+it should be a matter of running::
+
+    $ [sudo] pip install virtualenv
+
+Check this `user guide`_ on how to create your virtualenv.
+
+.. note::
+    If you use Linux or OS X, `virtualenvwrapper`_ is a handy tool to create virtualenvs.
+
+Once you have created a virtualenv, you can install scrapy inside it with ``pip``,
+just like any other Python package.
+(See :ref:`platform-specific guides <intro-install-platform-notes>`
+below for non-Python dependencies that you may need to install beforehand).
+
+Python virtualenvs can be created to use Python 2 by default, or Python 3 by default.
+
+* If you want to install scrapy with Python 3, install scrapy within a Python 3 virtualenv.
+* And if you want to install scrapy with Python 2, install scrapy within a Python 2 virtualenv.
+
+.. _virtualenv: https://virtualenv.pypa.io
+.. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/
+.. _virtualenvwrapper: http://virtualenvwrapper.readthedocs.io/en/latest/install.html
+.. _user guide: https://virtualenv.pypa.io/en/stable/userguide/
 
-   pip install Scrapy
 
 .. _intro-install-platform-notes:
 
 Platform specific installation notes
 ====================================
 
-Anaconda
---------
-
-.. note::
-
-  For Windows users, or if you have issues installing through `pip`, this is
-  the recommended way to install Scrapy.
-
-If you already have installed `Anaconda`_ or `Miniconda`_, the company
-`Scrapinghub`_ maintains official conda packages for Linux, Windows and OS X.
-
-To install Scrapy using ``conda``, run::
-
-  conda install -c scrapinghub scrapy 
-
-
 Windows
 -------
 
@@ -76,7 +132,7 @@ Windows
 * *(Only required for Python<2.7.9)* Install `pip`_ from
   https://pip.pypa.io/en/latest/installing/
 
-  Now open a Command prompt to check ``pip`` is installed correctly:: 
+  Now open a Command prompt to check ``pip`` is installed correctly::
 
       pip --version
 
@@ -90,35 +146,39 @@ Windows
      Python 3 on Windows.
 
 Ubuntu 12.04 or above
----------------------
+--------------------
+
+Scrapy is currently tested with recent-enough versions of lxml,
+twisted and pyOpenSSL, and is compatible with with recent Ubuntu distributions.
+But it should support older versions of Ubuntu too, like Ubuntu 12.04,
+albeit with potential issues with TLS connections.
 
 **Don't** use the ``python-scrapy`` package provided by Ubuntu, they are
-typically too old and slow to catch up with latest Scrapy. You will need
-to install the following system packages::
+typically too old and slow to catch up with latest Scrapy.
 
-    sudo apt-get install python-pip python-twisted python-openssl python-lxml
 
-If you prefer to build the python dependencies locally instead of relying on
-system packages you'll need to install their required non-python dependencies
-first::
+To install scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
+these dependencies::
 
     sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
 
-You can install Scrapy with ``pip`` after that::
+- ``python-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
+  are required for ``lxml``
+- ``libssl-dev`` and ``libffi-dev`` are required for ``cryptography``
 
-    pip install Scrapy
+If you want to install scrapy on Python 3, you’ll also need Python 3 development headers::
+
+    sudo apt-get install python3 python3-dev
+
+Inside a :ref:`virtualenv <intro-using-virtualenv>`,
+you can install Scrapy with ``pip`` after that::
+
+    pip install scrapy
 
 .. note::
-
     The same non-python dependencies can be used to install Scrapy in Debian
     Wheezy (7.0) and above.
 
-Archlinux
----------
-
-You can follow the generic instructions or install Scrapy from `AUR Scrapy package`::
-
-    yaourt -S scrapy
 
 Mac OS X
 --------
@@ -173,17 +233,38 @@ After any of these workarounds you should be able to install Scrapy::
 
   pip install Scrapy
 
+
+Anaconda
+--------
+
+
+Using Anaconda is an alternative to using a virtualenv and installing with ``pip``.
+
+.. note::
+
+  For Windows users, or if you have issues installing through ``pip``, this is
+  the recommended way to install Scrapy.
+
+If you already have `Anaconda`_ or `Miniconda`_ installed,
+`Scrapinghub`_ maintains official conda packages for Linux, Windows and OS X.
+
+To install Scrapy using ``conda``, run::
+
+  conda install -c scrapinghub scrapy
+
 .. _Python: https://www.python.org/
 .. _pip: https://pip.pypa.io/en/latest/installing/
-.. _easy_install: https://pypi.python.org/pypi/setuptools
 .. _Control Panel: https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx
 .. _lxml: http://lxml.de/
-.. _OpenSSL: https://pypi.python.org/pypi/pyOpenSSL
+.. _parsel: https://pypi.python.org/pypi/parsel
+.. _w3lib: https://pypi.python.org/pypi/w3lib
+.. _twisted: https://twistedmatrix.com/
+.. _cryptography: https://cryptography.io/
+.. _pyOpenSSL: https://pypi.python.org/pypi/pyOpenSSL
 .. _setuptools: https://pypi.python.org/pypi/setuptools
 .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
 .. _homebrew: http://brew.sh/
 .. _zsh: http://www.zsh.org/
-.. _virtualenv: https://virtualenv.pypa.io/en/latest/
 .. _Scrapinghub: http://scrapinghub.com
 .. _Anaconda: http://docs.continuum.io/anaconda/index
 .. _Miniconda: http://conda.pydata.org/docs/install/quick.html

From 23abf2d31b204f5b92ea00af7a3e72672a395840 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 28 Sep 2016 18:31:27 +0200
Subject: [PATCH 1105/2974] Fix title underline

---
 docs/intro/install.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 5f7e4fa04..bf2d4ec3f 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -146,7 +146,7 @@ Windows
      Python 3 on Windows.
 
 Ubuntu 12.04 or above
---------------------
+---------------------
 
 Scrapy is currently tested with recent-enough versions of lxml,
 twisted and pyOpenSSL, and is compatible with with recent Ubuntu distributions.

From 5680c610632284bcb686553225bc723601ff050f Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Wed, 28 Sep 2016 15:45:01 -0300
Subject: [PATCH 1106/2974] Doc: update examples section with
 quotes.toscrape.com spiders

---
 docs/intro/examples.rst | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/docs/intro/examples.rst b/docs/intro/examples.rst
index c56348714..96363c7d5 100644
--- a/docs/intro/examples.rst
+++ b/docs/intro/examples.rst
@@ -5,21 +5,16 @@ Examples
 ========
 
 The best way to learn is with examples, and Scrapy is no exception. For this
-reason, there is an example Scrapy project named dirbot_, that you can use to
-play and learn more about Scrapy. It contains the dmoz spider described in the
-tutorial.
+reason, there is an example Scrapy project named quotesbot_, that you can use to
+play and learn more about Scrapy. It contains two spiders for
+http://quotes.toscrape.com, one using CSS selectors and another one using XPath
+expressions.
 
-This dirbot_ project is available at: https://github.com/scrapy/dirbot
-
-It contains a README file with a detailed description of the project contents.
+The quotesbot_ project is available at: https://github.com/scrapy/quotesbot.
+You can find more information about it in the project's README.
 
 If you're familiar with git, you can checkout the code. Otherwise you can
-download a tarball or zip file of the project by clicking on `Downloads`_.
+download the project as a zip file by clicking
+`here <https://github.com/scrapy/quotesbot/archive/master.zip>`_.
 
-The `scrapy tag on Snipplr`_ is used for sharing code snippets such as spiders,
-middlewares, extensions, or scripts. Feel free (and encouraged!) to share any
-code there.
-
-.. _dirbot: https://github.com/scrapy/dirbot
-.. _Downloads: https://github.com/scrapy/dirbot/downloads
-.. _scrapy tag on Snipplr: http://snipplr.com/all/tags/scrapy/
+.. _quotesbot: https://github.com/scrapy/quotesbot

From 3ac3ac4d925ee7ef1eb8be8dc6519606c0fbf9b0 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Wed, 28 Sep 2016 16:38:45 -0300
Subject: [PATCH 1107/2974] docs: update data flow description and image
 (fixes: #2278) This fixes the explanation to use Requests instead of URLs,
 which is what actually happens, and is also consistent with the new tutorial,
 which already explains how URLs become Request objects.

I've also changed the "loop", jumping from 9 to step 2.
---
 .../topics/_images/scrapy_architecture_02.png | Bin 34433 -> 53978 bytes
 docs/topics/architecture.rst                  |  23 +++++++++---------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/docs/topics/_images/scrapy_architecture_02.png b/docs/topics/_images/scrapy_architecture_02.png
index 80be02078a101e8fbf58ed605e2e365bd43ed346..5fe393f2266966d612eff57e2eca252e214fc5b2 100644
GIT binary patch
literal 53978
zcmeFZbyStz*EYIEB&C%`5EK+7B&Ab8TBHOC=@g`;1t}2_k&*`KmhKV}knR?xyF1R@
zJioWj`=0NNZ=7-dIAff9JP)_9?^tWD8P}ZanmbTQK?(<p91DR!;5?C*P(~oGIU^8P
zb1_ihFP_9t4B&)mBmLYCfgrw%{Qt@m<vW`Q1P$Vegs6)1*y^OGP3(a?!=V)I<ELMx
zMT!jhl^^u&?-Hs`eXM-MQB+JkvqWF&#Qtz?^?PModWuuoD~?gEeyi=)E-nIxu2&bC
z)oFZDPk7coBs<zg8FF(EcE-6oRHMa7u3N=Dr?nW|>uR}nM>dd|7ezWKg+{qX_}0pO
z)N_1Kfjd_=_rA!Xxo!S#B+(V2ueWxqo8Cx|iHU&`BWQ4A9j?F`pG5R3@<j3fpZ?z$
zLF_!a_4Gmgg}XK3K$4=zb~qm_LA3ni%N2;7uQE;BP)K&n<N(zoPD(*8Pe5?e|2IzZ
zc0RQU!txi3-63@~``ssv36u50r(SA#HL5U;0j`@ka1yaZ{eO1);&8=*Azoo&=ZFSF
zIZK5xLh~?xQ+n5cKD^D4z4xfNpzy2Rd<|1#cSipbua*!e^0+Y-fE)7R!-tR%-0Rov
zot(OzwiZv@(bejvl-OhB(v)j!Yq8^sH&;OD_?(3VK7N+m6sV1hi>8B0M`2R8vM8n1
z6DY6Y_~TKBbn&_!+R%oP-nw;*lr-}?0kLOSoT&I*bB?#4J)icy8;l0NGcz-m+_7@>
z=;-JRHO@sEA6Q1Fr*&iKBntACb#<qI{*-2qdGR~0o#pa%BKT-H5yEPbw;$vd6tKR(
zjpf@!7b?&<IjPOHtn-T(o8Nx1XVqtNa?)4qc4D`bsQ4ixqpqwoKh_OK-C-D_YNMFZ
zv!E7zQl_uUS=n8IU197XW^c6U@i@Bu{QP{e<igFX&$3iFeP8P7{a)-%y?XVk>JB!s
zT~||fI<G`jr*ZL9W=Ww$)(9C@QSl;;>vVzb!w%puQl&zT*<;$9-e;$yiL$Y=Z6@Qn
z-;3%9{_tV<;Gp{rE&|c8Tts?jFB^tku|IykxvyNGNqy@UE=N!{p;uZ$vGhwf{A<L$
z*X$}^*eg8hPtH!c9jEbn5^Z;X-%qf3I~}I~z$fXg%S|*aRSsOjKnWF$LTOk;%9zj(
zA982o9%IFlaPiml_V*J_;&zZV(ItaerSG`7x>i?J*;c-AZgNP3XNdG5Wbr*_s8u;<
zcaC7TH4f-0&#yBHn2>efEEA5yd0MF<K3EgD+2VKMIjQWM%>?4fy(@Zkk1<AWe|J}6
zdefh7e-!&J%ap#@y9jig#?l;VcoK-Nefh)Y{p@-9cMK8~Ccm8K^ot&`v3=~AgQ3o$
zAhp;+vPXm}hceOsqQ2(6U(0u*jsP}p)l50`CQTnX`ezS_n;Ac}EBmsUY9yt+bnN-4
zkO2$99y3v7(At|S<wFBn6jD}xol3~9;+vq507#UvXa&{b0l9OAa@Jz(MT@m9Sr&OX
z>q#8r%s1R(>RYb_o<1OE5`R}$S7(FsGdmD3B7(d)Dh=Z%mT#s6ZJ4wnd(3E&w7Pm!
z+YJQ5E9nZcGH(P;REH`{QpKI1(WIATv5$U{yiv(RCsSp5H?HHyA=(s094VlZZ$s8a
zoHa4Pf#EIgkBftYGq-c3WnEjU!q^!Tc>VhIktsq1LgQ+>f>LW$bv03-1b3|V)Q`xp
zmG!@cKkbO8SJ_N6<JJjOnGB0ZL2#SD?tPHfzNwHK&C#^DzK)w38DSj^POQ+<8*N$Z
zkq}Z#)F?S0oNl(n6@TUSf5wmkg4!biS$~#ETUAX{z8_s*LpxDR8?SJp2ZCOC+#}+i
z6DL@-%FK}MchDhZvQ<@8^=;HLSt($QQqAO@EU)Bi$-}RTf_nqMR9R~_#n>*sez3_1
zw0Y&*jRr4fLB$y;w@K1rjEJlqc?rH^&bHBe5G#n+<Vi}pnjc;7nrWDC`yee-Hzozg
zf>ARe=3`jk1{VTOrRkKgi|ux@?W5D*zxD7@-QC@H=57S>GDI4yQ&Cb;;V{UkB!tV+
zH{Yb3XEJAV1&!3ub&OA1_oWbL$r_@GS1@6Z(Jvy6M7u*6I7+wElKAY+ofph0jE^#F
zc;s+W8dA37?kNR$`~4mhWKVU)_7$thT3C%5nx4Km{2NMw<=fla+x|}9@tLcuE0Y#B
zgl$tVC-h&kIOJHqV#!Jv*3<&dHm0Vgj#*90-I%Jc6v8@<bGky;aadJLsZFA>;jK2y
z-`%hk!R&7FJ6hK5(KL?|RlZ<}S5TkFD2@eGQ%KI;`Rmt!?MdCm*#^4kW}BjiZP>zv
z*|C(T<L+Fk^#Z!9<Hh;rRn5|EwWlYM#|5AaBTBiwurP@4l0E;!=|?^hqUX{37Jd5g
zK<@OVwY9a4jRL-xT~-#LJ|-J^M5v-diQNl3Gi!$IWLCZ$e0-Xf-Ce6h^VL7&QcQIF
zjObERQ)m3X&$OMLo!Qk9BaIyKpmB3^v!_QXOC?)iWN?s+on4s>f5LU6)=o$uOd6b$
z`r3S~LIxhS5arbR$_lpA6ppu^jg8GdeO>aX7K*x#p@Kgy6*)P%=CY9w&Ci7$q$k9C
zbGp?^?Ydgj&g^l;tJH8A6~XD9oSfWB?ODTUgyEf}bPxQhFE*A~n(`8OPY4xukpd5N
z9*{i$Wdv^5+Il0d@p~AZRMRX9Lc`S?4Uy^EqW}8sr?<*kC5KO)V6b1Tims<Ao7vc~
z?MFQyENg6R9Bq`D-JEF-3RFjUjbM@^lGeeiN|X}?et!lU+=R_4u-S067wLh!JX91F
zyRku|k$>GAcn0d%y9Mk1rg4SOdGj~&lV2R&Dmr&!JG-6_^b8I<m`18Fmj3KeE6e*9
zh9|HoawtX^_&shC1(8c;!HE!=%sE~5&_f<H7~WG;Q|s&Ni~JYht+)~t>BD=&-oAYc
ze-cZ^kFPnb@`HbF^35=Q=zYvM?3XF<CT#BJj<%z{J*TCX5wAF3yaIjrOLd7BDmps4
zoTnOB5SF<~GwK^BB8nRcrJhr3<60$*k!n{8#b%uOYY6CZ+YG-|SG&#xQ#5Uzg3MAH
zPqz~UBV-;A5)~hWYLvB?J;qk<I?6sdIgyf*Y9^`P=A{8QcDrI}J|y<ldQbMT)lCGG
z3NNm1Ul(?-hWlMcC*OeAf<ITvn6rl$*VecUP!K~=Tu6bcsv&(5&bML%MMXt>R}lUW
z6fA_^T|gg72gfHT5JFO@sXb4(?D=X3f>elUREX&ZyxZA>@2O%k_`QSh!Wi+13n9Xc
z4>F_6%+`0F4U;0SPcu5)G-wU!y!ET~6-dD$Z$;=1Sa|X1s1*ezW$puWjAAq`O_YYJ
z;uWMKou8lkpyR2%GI(>iIpZc&FJdPCXdd*w*AF*BT`SEoQYNi^K<~<uJzsd6N_?F3
z9mes<%OFv5CczT5vbNS~$eu&;8G#s111v}`!51&&ys0-A2AZMP#Tsq*OPJs2k^Q0)
zvt0bW8(773fq*^QeQ!5w+zyjgsmoMhy!n^!GonEmsbjCd_Adz^`UqzVe6tx+x*H}<
z$!Bl+#ExaCkE2_F<D~T6bMZM!tmjt|xvb9^5J}0`adE*<g0`B9HA1v{mNl??+l4j^
zBZCwW6dVWp`{~MA8r+pHM09m^=h{QC7nhgGNlB$M`im`Cvf*)_o}MaKo-#(Dp`jV#
zfDcvyQKpgsZ2F?aW)dd$NT}P8eaYU%)HJic-m~<XUs~P!Le<_dIF1xRv0~)XZRe#~
zqO^VtQ6L)Hk#o4Rx4*B3Ta4}7M7x4v-bw4@*R#241H*>YB^Mih4Ajcea|j3saB@N*
z0sq4(Q9$W1Y}WHpCM~7!gL2l5PCqbw&{lDA@tNKCn|kaqxw(@Xv-{v;YLFhyVPnd5
zBlwknteRgrsG(iugl5^D#BOzC)S?%~_4UGIWgPJeczAgJJh<Rz1kFlNT9&PTVIm%U
zdXF@Hq<OBjAE87vpK~ces9vG}qq&vLUKn7k_^w044<gpC3=@S?rOY)%p)pb(lQ|PM
zGvHSM-~pJ8SCH~Wh`Q}vD>h<}v1t+;#3isJCIK|UtDgQsPq7p9N$S>{J2e0_CBZ;&
z*a3=^6ia$~UVvXL(#WmpTCO#`be6#nCX#NGJ5UR`J^G@Czr4JxS^Lz9J2t+STnttV
zR&O;NLXGOU7&a;nSY<{A!<VasT?x|k;oudeLMSou=4~s{soiWG%MUFP4PzV*D2Qbp
z_L%GHEYZ(s!z$K2aJH70mI4pRnCN^K+d-TWIofKx&YMYl<On@&3|K4z01g5>zc{qD
zwN+bBj*kIq&%H;>$<E1H=}8ukk)zMQMlKdHM>?|_NQ3dI&5%AEZr2-Z3zYZ+pYNBU
zdhLBPGd50-96vbQ@D#@IMy5+sjGG078Xx57iC6+i^9R@#PJPkByB9cO<b)Ezf6Z*;
z2%#YgLy${}b2mc9swB+O$*Fvf6p`c@jZC%vJz%8^r7OI%Yl1&{k!SpqXvyboTwXFj
z@V}j^Kvn|JW_n%sqVAEP`)ilu)47hBQ0%yx)U>-8-c(v;mKNq?LjyO?rRhdFIXF^D
z;A)LeJ9FEt?Q=U(tgMtw4}OG1M#6<P8EKCfdq^*{qz6Mswnx8aXSKq^pXIPcT27TL
z{TLdW%*!M6uc9}FNBrhXH#Wv1VTp^0i;GLp@yW2TG9Q~98ylROsxb@iD&U8Bgh0$Y
zz<~C!NxELtRIqf{xsI2OkBygealN5i4nXFXUc9;1-<6(BDCu6lRpn%fe<Lm~IyyV%
zios60yzBVbtLeH)t!F2fAD{Zbu$c@DEQ*Pnl}ov~CVNNCvVM{?T#iS(AUHHJE;@4y
zN)r~Yo>KliImTeP2m}U;g5tvamaVL;DpM;dyPb21l+?;m3UwcNW?}mKsC2sp`h^-M
zr&r(X&8GMU_22M0Iodu{45MaEq^zG{6D%t?kacQJkuh|2ETz|)*JgN7RGMXCvNC};
zG5Te~{Ov<Mao3f;G@`UV6>S{6SRSjfMCpkNhn(_)A<)I&$KNtoRn9d?SxJ7ds0Xj*
zgrJIU{)xe^%J;adIgdeK#h3FeG{1{kd-&GUcz^zZ*Rk#VpB5qm1BbE0v-tSxa;qKs
zp&Q^|D5Q<JM3=_ey?ZTkFbz?^&o!)j{-h$*igjUX3QCC23(SeQk<I9}XRB|SOUzyP
zB5-oNY;9w8S8WIv?#8)EUE^H3o5cR;Be{}pie(CWbbvBu7%F}$nlzc%H@`2mrii4W
zn?E{^iaMSK;4`cC=rIvAC_m0omk4@BInsX%RrDZtloDZQ9{u_D9g9pVSYC6n^$EkR
zKKqJ&(HnO&CA=aqnCO~FX<x|F@?N};ZjkKPyDu(1q(ldD&MjSC!k#(Cm_d1uMWh$(
zn}$Er6gl(ZI<`iW<g?)%VU}NGA4>sgH7qlJh_W+bfF&u;VEuU`$=G*Ed^%wt<|kOK
zDPQM>wm+06Td804R##7v-K@J~@2)9=#-q|M-1Gc+WU;c1L{ax%M1}*#K)i%(|69Se
zT7E5I2d#`z;vQ*GgNx2_YelYIw!%04>7i3L&1Li}^ZCfiOHo@w-DZZT(|NZpH3^Fd
z-IEed_vNw>-IE4fjt5CsPT{^#MR~5%j#blF=ZiQz&eg*~Nmsl`XS?Lj_whW8*Cb6X
z(H+Y#&ILDAe<Su*O1HMciHIm>TmDTK-I0DNpfO+AL-poWOVq%~^y2zDMPU&^K3$f2
zxARj!mo}&Mk)qU=_Z%yF#ViQDxv$rMQ}ZkR)iLX_qOX}Hzk^hAQyXFTh4{859FiLf
zFq_?Yr<xNB#U3baBy800?NCe8Yd+Y%j&rxSZ}$lAPU1$rs=7%0moG7+k7PMiczGW_
zG^+IC8qUQ=1jvF=>jc*t3n0H*PFQ134y)l7s(Xn({)~eI3~2)t=b?ryEb^b>;$klH
zm>WMjDvQHG&ZGS#+kfZ6|3>ZM4J6f4X;txS9`BbNtXi&(4vr1JJmNIPC&|d@273>4
z{r#&YsJyh?+Cuqtr6W=c^7BI?;XlR6p`(Z#DKL!BEb<Fo^7E}7zC|5<=o$tu(|ZOh
z)2dU`OiV2FEuKv$I8LGjFYjS<C*tbV+lW4G)+FRa6b?1r?#y^qJ%8Z-j{?4|c|ltJ
z;(_8qmjW6X7nvtT<2KdEB<O5+{*a;UGxY`JTW5#pcWCs)W4^?^v9MJZj;nOuI-{cM
zjOEsjOGt=Is1xAe@OTr#%3EeRHPJgkiTYOl;WZpUQaL_}yme~KAEI^)qI_taw|0GL
zFj7dx?xRw?(1^x=j46u$k9{~T5ED^*nR;2>|K8J4f7o(3Fz~0_+REHTQC1e*4)|C$
zcfGA$5=1FWoEyOlX5Z*MXEQ#h^~nN@CN^3O?_Rs`G}(=O@ho=lV?s4rRMC$kV*{?G
zh?$1LZqM$np;<MK;J1(OTMl&68*3@I<D)hjwu}+y$;<P+nyhji=MHS9U809b`Ay&N
zsOecJ6J4Nk%i8ZTJ=%}*g;-glsy6tNXn{FlZx0axlSj61Y{#Y)A4bO|2=P9AZL6PB
znw6E4XLV3!1p-+j29rQ-EDe)jvxpD4W~ite&i~HhiOQ|EZazX((e;V1lY%SiDMlf4
z*PE{0$O%#Dd{$UUt3=nO5r6wtcz-`A5K=ey%Ss(e%Y!K9=}Zw`E3sVmFV3aEIa}R*
zoHkLadUFu3rs`3P1J}s%&0ZPQ;hW5=Y)&?87kmNPm|lK{WIt8exj1%?eX$uB=)dFa
z5O5+IzQ}adM<iJ{T?yzo9QbTRqpo^`XI1getBERAf0Z%NY6OD(gRNibLoSYRCSs|e
z$3+Th@*GuA-%Kj10UJBAeJ-3EGcO<O65d9|>_~!(av5xcrP>RBKrqu0@;v`~CgDQ^
zxEq1sjNZz+A799#LqRS4kUO3FG63Nioo*Y&7wSMbJKR3np>S(M14$u7#48^O<`E6`
zLd=}gJ=Lc(!9hkpuOH-#*`^xSf1f=|ZTaKpesmcU14cpk&H2MLmj+#OJ6|F5nhA@}
zMD&RgxCjJ7PZep2SL~>fPe$ZM_UgY*Wxif%W^Axgbw%WXX5G0AKI$CnR5DXTy}h3H
z2Sxc{-8#2_;(ofqz!z<;@_qVniiJIH*A9rV%e|n75XC?Fjydi3rIoX2SG`^t!r68K
z-9{kxX5A{?jBU5`+S=~#E$#i4$l6WNuKofs41-k-nO-ZQgJfX38qv|9D?=1}hx<2K
zUR1cB><4e^Up|6s?a%J+3JJ@-mzhqmR#ju;JYr9Mkik7kQoi#SOvYpW&tvUg(v5kv
z`{tLj9Y{&6+v#cx#8kZ=Z0K`?>vG=a9U#4kqK6g_ITLJ>gKMp<g3s7_Tr=Q7=B*Zg
z|FR|uvl^MI_yz|0c?L5gC1u{!vb6M~=#dat8WZ`*?C$R2fQpnI9o`QZ*MAG_8wKy@
zmL^8qS_B?@)VSH6+BWZwOocE{QZWbd7MrMBd`mN{b0vak>R8rw))1f9mM1S1*Kq7|
z8QDMGB;)qxAYrYmPfdNRn*@`~XLUUmn+x^bvVFN~J2+CeERB!KnoyavqP9MJIQMv-
ze#_Ar3`8#_r<iYVae1Q8VJ!^&?G%?D8C^F-_v4p7&MSRT3q)`Rw3~bPl}21k9abay
zi2>1T;gB@$`L+Y@>bd{KZNM7kv^9&PqhBgCpCo6aaDRilxeF(h$UXMoK?w4DkEc_k
z>7qnNDUG8crAl3Yae6R(@q@jcp3x`pUIhLV5uY%2OSy%PIW$}-wnU?#idZpS`m$8~
z+nU))jcT-BY46nv0*Esn6=n!xc&|S%=Jon=*Jz~B(R#eBzPR|QZalo_%SV2m=QW&E
z`R?HI7zy<5Hnj(^`C$xylJ2Ufr7zt7dY$6A#j}1vekbnb93rHX2KWAMh3*~pJw`b%
zF5*bBci38f9PAJDFfF>5L)}i(W6Af@0HA4TgAmtO4Z)S~?_P^R8OUXGUmWkBDlRTH
zQO{{3Da@X#vcIqXVA&sMy^D(9v4nHp(-KzrJFC;HZ3x#Cf;>lmmax;dOK{&-g`1l3
z6=u{rnXF{ITHzc=`Gdrt)>9s??IM=VFsg<<BSqcA{(An4g@RCQlBBRZnqPz*v|0<?
z_58_*@q_B$oUDP}pEaED^MU)~nw5o{@O=^^+v~S<6||}e$j9rKD`$SAJKnSLig31a
zJDvUh&I8IB0BfzM>uPW7ju~&TAeauug%4vsUm>6J6y`ZnOgM5qLv%RSy@+HlB)C-e
zp(duX7(D~927I6C`h{9_$JiPioOS9o%d-x2>p{Fify+Q|W<gqX`w}?RT^tJQ2-wXW
zoofkSyfsWod*c5ON;$vUws2y7CTwf%Im>^Mi2K$k`b*41Gw<Nk=OhUHD9|}~*D)X6
zqT0Iyg5z3ZHQ>+66Yo;^!;&8Ety%_4(*Gy%`<^-<%^n(^UF9;hZjD?IiG<q)`LzJ}
zIy#gr5(RXC!GRepqjMuMWll~`YFbX}+_Q2AkGbVAKzM1g{Qwb;kVf%s_sWn)^nH7W
zS4-=rOQ60G%*tEInF`<Q)ax10&ll!py)=o%g7vqU`8W@cP`mXjKK6mm7nToE*alHS
zm&QtiVHtzTM8MKJR-D5%+BX2$ASd;1WoerLhK2zkmC|QjoAX)EJh$RE097Bcx@T>-
zE=n)5*6qaeD8?UuZ^rX4U#J<7FQ3>B^r!Bk5<U<A+iEg>(pbH*UU{-kgK@sv#*lMs
z(C%Q8XVSJ~mA-S>Z*w+#MWTtj=FRdz+<{~#J}1b3Bp@HxWByRHluf5AR2$d(cS}Es
zH%)*(MjF1Q&C8c!H}0Z}+UO~|;-gN+YstBJjXljwv-L|%<hvO%(tO)}*8m^2d}G@4
z;OwBD|L4Vix#xDDEOi{y{X_qUr`k^Le`fqWZBGy57~IZfb(do<X7J9B3p{568M41{
z>QWHnk_$N>OwPpEO;K>Aa+iY4^lIOQvwG*G;TPXzMFJ?l^5*}BAgqXE|IhfrwOVBI
z5eM&clC{&B;%-Tw)1RH?6NIW{r!D36XY&-+Zl@fHm2WntXfPOa2!))7wFRk9GzJgr
zL8woK-RC0yb_*W-qQ2PLmq_{M@$xPAuQ&1S#2$Z&6LAvH78n$TE+#k7ZblGL=gxe-
ztpsUs>4WDFo84xz<|D!+vi>-#vfYl=<5Kp?{!R1YI-xq<)vYU>Um>HLwRc~w+dXxv
zKk;Qq7)g*;xmA-zYx?tS)8D#o>j#6-$>=B-3Jhgrbs$N3E;ZqNp66mQN7wy0b~^6t
zBkwrdw8Qz#?`FLE@ut76up6dXmYNnO+YJ%hvPYUWx>Vsi%D0SyHmau5JB*>TCRCW-
zG)}Mj))V%adf8iT(I6*LmCByKuc1K@Fd)m@jJ{E|!oXT<e_FYaB6_m?X0cZa`MO#*
z?nr)*8g{M<tBdnt#W;;v?g8(RdC6NJu#s7wF%DIf@TUhlT2G$dcd}A_ymz&LMd7tJ
zPMOQ&62eA@$FetnQBnS?$4xOu(Tk{2We{yfPAsuzw{OcNQM*<2Ts&0hc-9~^&zfQg
zS<s>3ZaXR`+?9VMLdUDiqS*Os;PJDKx{W#-j1b|23I4P`6b=%7f;IftC+BA?gMONu
z34Y+;PY&>4Be7cdbH`raz?zk=rv_B*FS!7L5}aMSgA6V^16kr%uZrTkZRS^J_r2q?
z5N94Kb+CYB8|M_|GN5nqdsZeG7x|703Y>LXv-~ac+@B3ktT>uq^ek39{GndAorX?*
z`n{9K^PG<Q&Dr<B{jM%k*N#!E!ys!<*E)vdZE?KE-R#qTQ4TXd^0X*UnysO<jSxBQ
z8diKPj*pt?d^lt6Hpf(u_=rH>N~0!r>Y%QDlEuUo9<{g+(zm!5pI~PzG2(}mj++%y
z-fQt2EVDJ6977cpTF57|Le__d1VKaqf@qX=?~5<4pRBu{S5KdTX0z<QO*~vRA(?u*
zeURW$wXj~@9Lzi|;9&mtE-CS=W|9PF{r&w&2NOWM!GhMbY*1iA&^VH0tXRAL4fW+9
z0r#O=dmd|Dv(>Mz^Sl7oc`{h13;)t{>#K)~HcuW*d-k79mDh|C1-<TPE}tm#^eog1
zDIBxXaN^h0BzHCF_=rS6G{$?SgobmEpn}9R?Q9T{e*ta?+{dAJF^5_;&WZ)yXAaa$
zRLJP#HCP-yN|ZC{=TMTGW&}f{nDMzx`U0K{vY?TnHm*5Myhgs%Ww?GRQfnP_L9g-h
zhP~CP{e7J6qjD(xA`lxcoBt@RcAxn9pYiD~u%qD*nHv><u@wa#F0?sKdtPiUB=DcF
z7VpFticr*TMHEPkYxxOnjdl;oUi+S(r^VB>eEs$%{M<MmIXhtGGIibw<{_^CDyGqk
z-<42t7Q!Dn;0HF4$=A126gt@2;M$+gKR-QCyk+?%CT7bT6Q(ZNU4N9WK+_j4H&x-V
z^5aeI5EAeP^nbRt%PNFI5KaU;a_aiM+k5H&YTtVxF8Z!y4)ji(o`-}4?`L1?;%oD%
zk|zxK9)$Z0gcO@Qmzdcrn#+33e!+FBzqf|KN$(O-iu^)ez+r@O3DfZlpmIWCb>*iF
zQnp^vYo=zsm=w=I$Ej;){GcvkwWzSk3#0LhfU4E9JU!3qhrr8v6pb-|nI+iUihuB4
zdU?q%efT~lCOot7$A7I8xlEx)W<Z^SWaq}w)^^)g2Ts3%D}R~2(}Qf3%Ks}gDe|Gr
zl3Ui@pfa!aY=3QhPh1)h_HdM-d-8T`lXE4TW2L#JuEQM+Z<6j8Jq}Dxwx`>)cHz$a
z0=?`~Y`ufe<6V%^v}#`((&7#BBxeut+F^<}K7qVT*a+^A|63@M{6h;3PY9Gid=xT~
zm7nB4J|qQ^w^Y#3m9Gvx9c<kYzBn6kpe8gTifEMhEE8}iNoUp6tVpg|(mBeZ3HDME
z=Grf7;;5@BeKC5Kui=ytHh$Y%GZQY%IQzSCs(-LM*VTLR)L}zo<HA1OtYc?<dIvC?
zDv?O$^N7*!D2);&;1(aXK!HXS-@f}x=at-U`~jDbU}1Vxb6xVptn&UA)Dlm<W#0Mj
z_)3MVAM3bV_5Ylfzva2tKJr}-|MNWGPOa#sh3Mv@_6fK9AJ49J*9)GY*t@vFw>4}3
zP@kurE`A~UFAx>K`k<tw#-q{Gnqne^S<Al3GS=AGcozj>aUY<3g^*Z<S4z`6BZ=!5
zj<5rr*d-@$@YhJ7W-s5}SL%G}!qLdT;$<_I?CmCB=4LW7;=*^T+f7>5@?<t#CI8gv
z`tejjQ*!g0oOzub*_~doyB>SXz4Wc()2w^FGa!8Yqd%OrwyH_CbgFd2`r^nvRA=$Q
z2d0&~82RaVfYDqX%l22zxsFiuU+Y`0LgnN*`K|k|AH^!q!&oF(1F@VRtr(;((CZ4g
zC#xh3-7Y#Du!`5cX!DFe^)b<}nIH1p%F#XFl&ascdwzA0kXarb3N!1LXK<CVR>QY=
z)+k*ePaR>-!q92^aD`}jWZeir`svbi{Kd|2t1ouZN%Pd~Royd}qwj4a1ry&qH+FPS
zb<gLzxmOJ50a_mqSW(sQzojzPXkb76JyF?aR!>aP4}}eWr_~<3VgU02jf%2!(}%1X
zd6N|~b^SLdj<lrmKgj3y?eRF3)t+upyH+G+REBsC+BG-gtt$OS$e!Fh>f(=lXQ={(
zqUk#KYLDo@irCt)>2d~jHF_hByE*af0IrNI|8!0BZO603cRHLN*o9xr#?%kv72eTa
zlX*)%6rAThTzUC=JWuXqBDH4feR_3%BMjf?&51P!?}bdRI5NI9GMy@E%SR19?$SiF
z6+*h6%_zJn7rk<8Oa1+<aX#w9<>6r(BMfb_*u%r&ivoY^vSGvyUt#qbD|L;kaD(z>
zdS)`geG?5Y*yaleZlN(480_617Cw1W<0|*y2U7Y6ZogCaYrcMJp?NvCYB%b<4txu#
zAbXEj@U}lDt{o94^`2%stY>5O+EE!EXDgu>%iZp>#l53bg#VGL{LXn=0=4a8y9|%$
zP+0jvRZ>={Y|VGc8!PJm42@C@(H&wK9xTr|6TWpeG%H(>-%(t?I(=esv65>1*P8Rv
z50&h$nQe@Y0&XtNX&$!6=NMPy{B+=CziMUCkB6+LT;^y0rTpeUdVU<aEgyP*uHRKN
z^V8bW^RVGh!Rfq@u9lp~)_zZeRL1m5u<5E?a*my>lawr!q`nW)+tKP-{i6%M9u5Ui
zS9Nt<Lpp8sU*nIf5Rmxcnn8kPGS>*|@+WQSA3vv&&=ntl-CNJ#Rm>>NW551%#fjKy
zdmyVgr`J#LqWfapY18gK`LcF}gvEa^;ZZ@|{|L%2$H`NZBV&w48qvu7)W%_xqjfkN
zvKfz6TVyg`hmR_HL!8t6{`Y!k-4*`nvqoCQx>K2#Tw`oUeL0~*j(1(h(2LW3eKDVY
zhGN=3zR2m-!>udkNS%fP1Ovk=ruMWY9nwRtJ2;fQOHDt@t`54~F=!rV)j1>d*qtz3
z>Z_j1+EXRlQq*y^+VXsxuZ%=F?ge$ksIKm(Lyt?ITS6&wV-V%tRdwMDNT<TJVs{A#
zvS2mhSfg4q((NC|kJVa@m(gDk)ZxNLdEr|XNy4=E<Bg?_5``Y45>TqWu$zquT=1YA
zzQ#oII-1X6e^&~P<VZwT>_-B*$~E+th94`)$pY~7{*oqsaXBKXFTDC#@Ec#VKEDhJ
zqVV;d0`1*=bExF01eZ7eXAo?gRqc9fq+jTbKv^gKm+A6i+pM2`8}8Oyq0EVo-smZw
zNaB(cGTS{c!DmU*aQJGT;O%?!t*-+o&xYb1+m@94On%s*`Fb6+JD@c;`R}&WrxpvQ
z`;6eK_RBvPd}!fz;@O&0lq%0qh}S;t!%NmZ8>LW0T@fA>)(rezm6C=vzaD%vEUF@p
z$StBMe>FKh`bOVUPJJM3C_R0=+<F)<=r)uNxgz2`nHnJ#4R6(T=g6PlV!CI7akl-P
ze~$ImX5^JzWJHgzcRK=~S(TzQ(L0(&L~I5%bV$QOtVsfIMslEvvhDqNzJbxOJUTsE
z3ANHQB@#&Q*G!9&yY<0uB>BA{3o?;^H&nf!R~UE2i;Ko06Pv2Db7Nz%2QLWrm%PwQ
zwr$Z6Lw>o*KLC9Ugv%}a-vuiYNj0mr&|R54NuiG5KnzqsBsl53C>V8O2pbtPZ2X;e
zdCk)<gpw0fQ-;Fjh9-vsXUflT;y$fztk%3<QA_N;>&a$+1u@XWI<MuJw~5tGhFBg5
z)+&GH+U=_FSq6%S4!yra=I0X(*-y*rZ#(sj)LO+pXYNRaokR1V?LK~_9b~X{9sDST
zKg17^yBbqvWn<RM@DqL!=gdgjQu~6u=jcJl1(6sDLS)T!wBG41UTAsT%MKmUs||e4
zo>Hyw#yWOxZeBrcpH2WpRjI{x`#PaBg*6YC$-iD_`igwp5{X?(?U6fw)Y+!9y2*%U
z*mFc-q+?0RM6un16>uO*&BXy@AsspVVfFLv0a6DXWk1sMgx+firDvK^{59~-<UN-A
z@Xl%|AtJ{Ye0|c%dfc%)U%MO%D4S+`bv}MnBKpX6AN#h-=k44<RyKL?a^qS<)%egr
zNqzAS^~OYH$m`~HtMT)#uJ{vvev;=pPlx$UHO?LqgC=G&TOD{B+AVyFx)S7G99ZUg
zOt5iHBpBnbpDO_Rvi<#`oE6smGNgu$jc?R${eU$ak39SeRdkn2Q2Bz#0>!VxW~_~!
zh1drbM#0-V<T&jgr5TEE%_~0cPW`IRDB}PnsSM3JmV2_!{bX&tJdri3TTY>li<g&+
zqnNzg*1%$AeQ4;2SYM;B=j@-&Ag-w*FqWz5-kGb!sq!2-o4u$AHT~s*(SB;O7t5)d
z8)M~jA^DN~RV+d$+u!Ykb)H;(<#Ft!R!2C?KmFz*hW8pC^`;+6?tg6E!b{biv#6Mv
z0kc*Xm2&N~zuTF!om@+5#6s#caqU^bg{iM9X2(U&O1?Xf;0`}ieb*8{HMX~2cZXC>
zEW+Y9RDYpB1_j)sj;Lm8l8~F;G$Tbu9cAh1R`T)|Y<=hdz8^bd0!s@M$geKD+p(&n
zjX-eS+nV#_-kJz4_>d=FxfD|1?lDI~b~11G%sLL++Fr;dk>`fq;Kj;h-?vo1Sf`Wy
zHQ3B_J!^sli#6ZX0PJBOkbad`{<jc~>x2mNr#sC2MD=GqW5U*d%Bjsqr%RiH<l_0X
zT(&0%@5w59ST7Gex4wtitMEbVFo2A5AfJc&bPO5)M*B-4{!P^%PmaV=4ZpaKApbG&
zOKt|<6drC->?jU(Nmz~ieaEAwF-R9g)~b}hIPWAgv@<B=H`-aPTaNFf(!~;4G1Vw+
zwQWQsxK%CdwH8btBrphV;9N_R(Oykzyd<)u%g6H_6zd-G<kV_3&v%u@CY;Ad@~iQg
z{fXbGIovn6c-d9`>d`iZV-iA7sAwZ;D<P8js0pZi2$8&4K8N+yhK&LpZSfcPn`gJw
zr`;AUu;XYf5PA>I-OzFJWg#1I_$_1hXSJbTH+YX+bAO|DX{jLB?Z<aefUQ&Y^VO;d
zl5c1TQ)kC`gqO>qZ@g*sQq1(|>%&Crb<3cwZ-LVui?yD6)kR;gTMO?~r+iKZ^vWZ6
zw!Rz+Kl$^V)`Gg+PKqwr_B$79k`X-(NE(P>44*^(uIt58{KmPS?=bkYX&leP*v|6R
z7jMF+&WAm3lx4tUw1mqF*6R41pOgB=3X`CU4tLkzL~VqKP1JpU_@|gPO7R8&J<E$<
zf;^`))~_;o^1t!Gz6(+j&ylNmT)QEUMAQ!mE6+F%EH5mlhpG=+`aUtG1#Sn5b)4sq
z8WCV$&tnL!=UWgbM@IC^3Z}@}*?FDOFmgovGDPsM#l9_d<VAZwDt)^=r@P=OR<*9W
z<7d=?PR{jUs#UwcQn=EiuOc#AgjJA^HLEkQY%OtZ{CpIOC;6XQJ}bu6C|4;7Jq3#0
z?rgs({;JFJR%hUT<e4iE!<YjEOm4vOJs8{!W6kVu%T6|~=cD1wSV0^*E0#dAClIe;
zLE$L+&!<93EFMpmiLEHKOL_!CA2z+<__tF(Y)>UAxSvj0|8g2me6jJ4s}Rk~`P`TF
zkQ8*|>HmVz_%D_tlGg~^L1Focp0W&_0{`1Ik;}yRzf$0H!<sr1nI=wo&w57whmZg3
z#sBNykR%jK!xrtp7t`qKf7|PCOcaj=TOS=Wt@N37A54#xXH5E|>ycG?oHNJwY=YXs
zQG}c|swn>FdS`!yiu}FoWP}%A%0GOV|GDv?h31M^CD4<Af^!xPyumCL;6S1*NL-Tf
zv8;iZZ_gLsX4u38^q)K7Z7sr&@ljj!AI2*HtNT}-7(Obn-GERfXy!Zz>_VU$wg&-i
z85puUU#qM6tA6(M^u)z0IgkU<6@iGP=A;C6^s*kX`1s;OzIpL>2Y&NoyM|=rrTDal
zPz8zkAKjW79wsrynh6EcDDW+TwEV_*^FDB`?2V0@kB^U=l7Kl6jA|e?PQ`W_13~l@
zIFmpa^_L=cz6sF^S-mJ)0?Me=#>>YdK)l=7pr)eA%e^1al_f*=zU9v)^B2gYGHJ>{
z?ld(89y|^%ZhHUH;w;de*>EuuX=6M04-TXvaD9Ay(v-6@GEAJEA61FXVMg4S1Y(w@
zhM|p3q0+r~B$(cM&d!G;MFv-|UOjTF+df4S1Y59_{~5(=ctdXql7M<i!enD-CzOAZ
z$(Nx_AAUbV2FWf10%~pfl*<ZG^8Ee%QwcnPif7-{(9qy7v<n1zem=gj@$p!obd6fl
zh6TuwHZe&sNOVU?-+?EqHZyVr6bjBfZsPMDWi`~(OJaO@g+}M!eF^N~MI)EDMFyBu
zg>Lrt_74~QGgNUWtAVxY;zC>@W^HW^^f6K_z;ZA&R8gS#88L1(T*Z@cdffB`UlpBc
zZ+BOPm{sB#V@)0cpA{^mf$*Kdo_<k9ba?Z7ALx;Iz5zV=pq4+YH>q>pDL&-S#_$wy
zT-6vH2n2Q;5F3G{{2;Nrn|Jy)5Z9+}60>st_!Y3t<jZ706yADKw(PtzSzl9GoHrN3
zlBk6MlMxBVk~$BfOoSDdpnZDp2G9*t9+MH1J%pz(^!uu3V)1`!o4txBV6!?>1mr@S
zmjN;}+B5CAFV&E37Q^?O|K&jYw(8>r&X*&K0(radUsJ4)eKb*OJ$~j?HhD1vu<+>m
zRisMJ@1J7pMsMUdb$ZrrVj?$OsHu-mS_HH!lnp9I8sk=IenraB!`e{4nMD#c{qPd)
z(tgOyxP&(jGWyvR2Ocrr$PNMHc;~A83zyu_-yg_pmt;@*jDCk&d}LH=tv^~tqK_tV
z9T$h~H1%%Jy>7rJQHR?ZS{qXfiFk;dR&o%ngXH<l4IkK#6F5KoD~gXK2&>XvH2dRO
zXDe|XdM;;Wc+)%r;R4?;&f*^M2tl#n072WZ{%XDDn!A0U__<5Z^mL;1ou>PQs!k40
zPC!>LtoZ{h$oP1Pck8tWQ;VttdZbT1(l#Lpb=^rWV7_mZDML90fzw*(u(a)O!LjAl
zLK&nx!M=Y3Na?qaw|q%-q6JYlfOSZNO-f1%Jpj-x0}NF>p~K$>p@PKg51$|1K(^C(
z#JM;9@nz!rMP>s0^!c457@8%w>+H`ka4+(y0V*t{Zzk{GvdBH0J&;ipqUuPa*9D0r
z7dV_AySK5ryV#|4`cry2XTib{+b;GbLvP#8LU%p?M7#yrPjD`BzM~94;06-CO?d#F
zREWwPhM=Gz>k9{9A{b>{Zye;M+R?rpBL}T6NOCNOH?&LmQmCq6nP|LI>-*rWeP8}`
zRsCH4#eH$Z)6hhiKSO)L4Y!|HkUqnL`GxsI4V#OfD>^R?4QGEh`iSCR$H7rWBZf8>
zhrzg@M-OayIxqW_)Ya9Kl{9|e2KRt$PryeVwdB@J`DPrUdq3{{r@<T%gZkVb28Ft#
zo?$~A1eD;s9Pm9Z>Q|F)e^$r<+mlJ_qb=wk9K899F%`peb!9~>DSjp;V&_SkGWaFU
zqmJiRbLv>uLc66$+z|mS5k;85<lO?E4kCyU_cuu*0jfI5r`&9Ei~<hV+8?~CuWZSl
zI|p@+^6Nua6!3VnuhBF|(9g$VD%0hErdQ)Pay|$&#AzU3K0Yc(28=VL>T7hZ2D%9)
zb{dliZ@nyIi^$YGxNX_sRxJi=2dy+Jx9$@Mjarud2qG~?3To(|I*#*6kuYGrO<r`p
zIWX4-ecu+8AnGFj=OW<IvCx5~tMR+=XD40i8~mr5VQq%My|y1lS<AJQtSB-vvU*%G
zWRPdYOmJW5s8&@o>ZK6KDoD|z8<styxb`lhPy>1d0wrkU1fTMDDFXSJC-$cyJH!F#
zON#CE8`lCB^JQz63wSUxP9QG}?~fP&#T7&c9ZG)JKz?8BMt^3;hYFuN&xHZw84O5x
z`zIX`kRc*LUs>_!rt_egY7`YIX|y1T!C>oY!43Us1O10TE)FXJn^@MmCWi7s(-rJd
z3VCes%vY}{<~xaftzaV9b+>u)ycwJ|Y>5GF|K2abW?=!2*v?zLws!>jhKI|)#?yhX
z0DlPAq%sM>(11&e-AeSbfq)A0pz@LJ>`(ZlnGR+M9UIKM_(L)(nuaul1v|P&#YVzq
zMi*0^@oO_{ZBaVf$ImX7TrZZL&i<n4-9`3vEM7yHmL$y#*28v(?)ph<5GJ{;JN0$%
zq|+9DwSp{Ulf`PFA$w)bDAtji&<kNCJHO6|hQu?>hyjCH*Auv_u&=!~-e30>o%q%H
z3w+d|=NFdShdD?^Gb*t<F<D=7-aUtWDLcVF3kkZ%sfB(|lOQ_M0YYdvFzI5wyn7^8
zL-O2;q%K~&?<kgke2iMC=V;L#pzZstE3C-9XH3}T5Xq~6bYP)cSmX4Fs@_9)TQ+ej
z&NQe92g<d44$GD=imo9jK58=n;iToe9V_e7Daic;kCAgZS@-#f>y?IY0CM587LzDq
zPJp@;Q}8@e$b;%N8W}%Du(&a-)FSjo<>H|??)Ngge{MnBU)ga6Wa<a8Wy@NK|1yV?
zFPXa8-Srt%td^vK@%ibWiUC&S=;V>*9w>T$y2XCpxd5K1mOMB{;M^RF4mJ0YZERN%
z+$FQ4%F~|5jpi@jY-w4tjPJ0zC5KG~Wj$MnJpP!5MkH(YYZOUqDx^ei!enBCy_WVz
zo9>|_G#D@6QNivg%7ExW#x&Te`yU!5xss0}wfU}ip&^Yi6O-#Er?enK<D8WM5<w+w
z&3&iAp6LK@G(89Vvr{d;<Qb$_%nTOpAhhb-7=KO|YzBJjgz7bi#C%Fjnj2FGT~3~b
zhdV^NE{k7;SM!<0Gi#iVXGdFfE0Og9L<f@8n|V`9c)vpyi~N)B@$R1u?bC?zNl!?<
zy^N5Y@Fca8Nv8$K!c?mc|121lOkO~h0MUVj-<dJ>zX+|Y4)TU*Ji}J_kp%=MQIx`z
z=|@!vpkv>uJ6&q&e(k)|SipSV{w$-5WxgatYU6+fQAh}u<fVE1cQsObT-dIgw%8A}
zJNRH!Yh%o$9uuMVa{v<x6Q3X&s#~Y|`K~$t**T{6x{3651CbD^?-EZa=ttbC+g0kI
z(*E~P;<}_ZENU&}GuTR2r(<7)+Fu<pg>)g`W@lkn+yE#hh=COCrD=1h%LJd(u3A<@
z-nVFn_;Xa60<1Rkt%4GX@%&xuyheQ-P~%rF$4-!(?D8VEPK}|of+)>`#o0K^@hs1F
zz<JrOX9m^g)49uA)pnSx;d-W=5*_i5^IBIAdx>`FH$bj)>Dhv2%38XKhe_+j`H^$R
zQy6s*(uO;u5j5x1FOZ_zcMx)!hQCc)L1M>Lud<?jvVUqia*GI&`^gx&^wgf^liybS
zgikxaM+s~HE6D@ABwO3G-#yU}ITs{oOL7RcEs{tUSeT6ZsaK;8&D7`fA${M$nskq6
zrWI~^IxGj<VaH$8#pgL~j2{>2zN+3(Ai9gt2+~zW4rAR?p~MQx(fj^*uosZV`Bdm>
z#bpW?pkNAh0^Z)KZ}qYjhz9aYQI<o(XTvQ4@M#$*>4(FmLT-6JT!LL*1Pu!;Fr2vm
zb^6yH5N9-`LfAvDw=9`U8u?*QNb#Sx=>Pk!Y>Th3h)u!Hrh5PN!T<WL6NbM$(7JK<
z!_DceZBIetszs0Ag6^SrC}gJO|H|S3z_1nkgXUb%l&a511$QQQWE%y8Y9zNUvlvc@
z+%XZ~AOCAaaL)_fqadfncRzN%d%CX#QrD(Yi-E^9a!CBmbU}~dtyh|8gd;fZX#x9$
ziIfQAXUOcb<Z-DO%=?lH0G$Z%u!d?k*?LlD9F?dmxwZdeS_v*oj~AJ4g)d#gDCn;_
za`Wiwzs$Fwo9)Za6yO|Wqn5HL{yX*9vj?h9H-lO(lQ4*_j1gF@>?eP~)dTdH!CbPh
z+RlfzFKC>Wqfe!S3}iV@+URd{#w{|kCxH4Rm{<wRxZT=zs6OcQ#!;z`B2&D*2g7R%
z*KT1WQ?jHXA3unX2YY+I&4-Y0OvhxYEJHKf&g4xroL)>-(Prq<g9cY`8jQqyl+ypC
zT?rrmNxKG~DXI#xQAR?;6pYzTNVxh+poBhTN-hI&3o6%e{w;y?6-H(O<bCS<PY$cH
z`55^$j`U3qBu7?}-7oOVKjT>;!bT#cObSS)dVBaNA5-($XWDL3LSvks>QdvX%sYU6
zFxU2{E`yjbd+3+^CohwsMMFo2?w%aKerN>}#fMZdG$h1NDn$+Y@Mdi*;VqDhR)6!2
zSop>L=#d6vL{kO}aS#ON$`|?2tJH<%>$|tNrz&WDXi%g<NlAG<HVhh!OvorH<udx8
z3PTQz?9ke?6H3(mT_lv&!CgI;mTh`Bx}G*YTiOItfkOSX<v*(E3pwUlZYuQQWN=VX
z$<v1elzDoU_!$Amw80f56JHY@r+1CbG}<m-!symRE!wbq5i+0(SmfJBZdoc=p;8&X
z*Km8wcLDB{m-Ae9XR8D+0tiR8WQnkt4?<rMbO*tDK%SBFg>(8rC@4~=F|_dR4*Ziz
zT5!kaVM!4i)GgfsVL+E7z}Vhe=trVH?(q+~B}k5j^VFazDS4X~dbQrR9)HfiW17h*
z_D>%?C?gDcC!Xj%<7@B7D&Hk;GC>#lT;sKD&H0=FvRk)w93RW=YET1{83!?tsKV#~
zIT%2D4`%8Y%qGyXmq$+J;NYMd;s!l=&?r8rUiy@`*-eq>(IZl!N!nNvZ8n!>&~5Zv
z&c7jNh5<t3D=}$5+Ay)pKcEwq%O=@h)#btP;NYO@b8@A7H*UFYHeT#cpKHx&>Aq3e
zy`6QM1dVp^2xK>??;ib*>UT(m*zN~pYeKGQTRCyrR|=c``ozw;K-~eav8)K0E;O&E
z^FnQ8N&!-5QSoFYXdUatXQ32Q?QFt8w}#HC<aW(K-^al*9F&1ML`~`sp(Rx{8~VNq
zVvSBORwvyx!cFQ}kF~y&8qkO)9r``dYLyuzY;h!ZG;4ZQ9aKD)L(ttDv!v0SplZM2
z7%{S}W8qTLv}kUkT}nK$HZ_?49EaV#$Rv5DgriSzjlK8&s#%L4L9lqd0&U$>5kicI
zUjr6GW2c0K-psTyJvsDXVJ)~LVEu+rY<+s?<ZNAf=-O>o93DEqI%W%V^NN=!>+AxY
z+yFwaqg@SFEpiLb&StEil=H#(OdyVz{_WeRvjJ1i$j_e(MXtK{SW8~7r83p7K@w>0
zL?ipZ3GWSw_+VH*n&mH>u6RusO!J5mm%HlkHF8UCEjV7jeED&UVp0~)(r?kG(3TJh
zR8MA0H1p)>Ei5el>X079)6p%jh-I2i$tOjk*Dc1z#1e?_$OqI*-XZu?H$fikM@)?=
zpi;jgxkUxvE`R|68S(w5g_P7rpWYSg7#J*HTsj$kBf5jAq`Q66Bjfg~;&#-hk}P9m
zV=0d$zRI(#EH76ye%2_!4!b@tj>Tz1{naUz)HF$|Wjv|Pkk4^-BpRL4P`hRVfv>$E
zqc~<F6u9Fhf~1b$`26{^+;PHm<dt2~FZ{oG;@{%#fi=p}7mu&qy?Zw|`Ko#n8cvDb
z?AIsZGBPrYrv!DpGvSZiq`Z>TE6|m-wL2#X4y5?v*W;b0ZolK?=4OnLIevlKw`5;X
zf9~SEB1Zbw2ft2re64PS+C|-2^LzB7>y#z89X8)Plke|pAtob`UnsyPgoTYgIK4B@
z<s~HTZ}S!f$Nph#XS!Mr=IzqbQpSk)u6t86Hps`{z?#;M?lg{cFSJ`wcqNP>*-22n
z6(N@Vd;KSm4%9gwekrXp?)>tRadyu|&`JBtUqr7F8|h*oj>CTI!wje3kIGqJ&!{>&
zIvn`4rJpm0A$j|kqWFNvr+RD73GpDRW9b1VuD_4u6%>bupF8Hj7<6i_VIk>RQDdzC
z%y-7}SWmhA31pD|{R4|WWMofiV99=h>WD?e<VupirnWX!-*?mrhjPnN?@H!WZv}-X
z9Qy4Y9ZPa4H6lKvmiw^;e|f&POxk9m6WyUp&(6*U{liyQS10*e^Roan7gi++bdT^x
zP6Atu9F{Nb;a%NYor^(r-LUbU+E&XR5ziYs&qYZMnhK$rKebmnR<{q6;BPU@;ha6+
z8?uLW1e7sy^E`C)hesqrdzeH5bm{5o^FPDra%=8=m40>y`DGt8NOFB(06oFLz`&z;
zh1;0dK*sZ~#SL$JO+U98dRVYjyhW>?yrrBgnyhpfLb{|q9j0@G>^hpS*o-}2b43kh
zTUA4<T8^3Ibt|Ri@?1aTD~HD4QA4A8%hjZzYSn9Anp=}uQzP)T+~q%Bi0O}pK~6yt
zSdWIo=^j6f_{1XFdMcUPyJU}+C@aPi)&I@=qok63?oW1du%2GiUH*l;iqd)KAR|t4
z9P35tot;*>^q!Tymj&3jakepU-_~BnJI4)l$w1o|;)#)akaPFdf6VT2mP)%J`zzG9
zt(M&AnE0q>Jq<S{_fW0r#r&HaHJP_V!g;ds92BlhGe{fU6H8~f?y8Nk7vD29MEt{&
zpyoi55=DfIiwnM$<W;$U*r4*WXW|z3J`#Y@AnjAc*5I$>bRQ<Pa7zhIthCveD6(RQ
zjwZe3R|EXfWPoEI@#BXi-=nVvY?|8Lj;+R>dN+xQgIX-Pv3c*T6N4Wb_imm^QskC?
zC_TI4Z6lNRNloz`XDWK0gkBM*=84FLWYNOCPV<=`tl;Mx)TcM9P^Qaa(FB$rfrj8y
zyVNE)nP~$eQozsbH3g|&<q`tPKq#1-1C@P}rSe6Np6}D}${xJ+ReArnfZwKj5(WmT
zy{5A`X>L+RFgLK(EEP31wV|OQb53&T<}T7O{VYl!4iy?cwE^?e-Q^+(?nJ14uRUJA
zylu^9#zyK^KnP#(lBXuSFsZ~C0d4RsQ54tF;2USEuhl1iEFAXzxM*L+tW`!DEICtz
z2GZG)Up#?tT8a4lxo4K@m0_nP_sQB-+RdBxk0_0cV|ZS^Z%|9(j%8j&wvpmP?&P)A
z$hVJ9Hdy>nHL0UeRaF(&Keeo^3|hB@Xv5n~A9xXmW7nYN44RuIy*+q@qjeavYx=!$
z#%2I!J~Tuqw)mCQxcJRdUmDaoUe1y7#pmbc<yBRE$ye3TSfqGXIKJjsJ-P9=*9J^6
z-Szgn%t7<FF(lvyn#@YaVBjNbjzKN7Y^X_4;*M1-7|wyOG5A6rEcw@-&xoCjmF|Zr
zr<$s2@5i_n{p5Nsl!KIBQ|K`^H#ax^8t8|&IBxw*0e_c=wg%=SmQr!dq9E2DYriuu
z9|x<p+2LlkN$^(}^Y)YpF~h)HkaMJOmlz?e(2Px{N_KSFouul>jycAN_8A*f&Mz?U
zcz!2%U6IjZ7{1t|{>#<7pO`UporX!K*N?Zg-jo;joMvdm<#?8!zs27^xaLVw9rgF1
z_1vO_`>soIJpQ)pTKqC^e`RQ_T>7U^gr%CDIHUlW;JNMXGG7KnqM+zh#}FskI3leB
zUx84CNiO=a&G12B!;B5Ut&ILGrF$BZANrSO*ELgn!Rt=P=$dVMqg`8dbp6O&n0oh}
zO+9Mgbo|3%9FKg^&3lGO%h-m-_d50Gf1lAbOqRZ2Dc5*_=F{jjKr*94JBosJ_fLcM
z-n;&#>z@b=*hnjm=mD~^M3tA8_I6=_5p?Y(4Vtex@^=Jm#fB5!q5Hv;A*rVp^nwM?
z74kjAt;;uvV8-?$p1%EnmVfvnE#rv3=0h}mHNk5OUkx<TS+a*7;<jN0K_4K|!oVd%
zRumAgn{X(5ZlI<1{-o%Z5|YJW*M~+Yx7$c32%!fIY~wo<{Id=BE!;#((fJvfq#;P3
zkt={!ULF$I?Bw7!Y4+llOs~fLymkLb_pQebf5{J7Z)Ac!Tb{s00r<#EE)N>>kI1A4
zfxpB1VvQs5&bxd0bwST?5JZ$hcnm1WIb0qh1DAgI^50K=kn#BPkic<4UU4U9n);;R
zt%qUak^gV(@1nGn$P2zx|EISg4F-95nUcS*66_peQzLs_^?F|!Ap_+i<U#YL&+t<t
z-Y;#V>VEdSDUaWq<wVJ<k)$mwY!mO4jZexm_zRW0Xy(4dU#g1u;~tB=_woyV;9h*A
z(GzaP=eet{SyTP`dnFntI@u&7BtOgY(uDUowc-o7dk&ixw@YKcuBgw{q@khyYxXq9
z>`XHKhM%U|&vx@OT9)sR%Ue6Sd-~viioD-8R(j>BpNjXt&GkDf4t1f1k8Y*kLJJd?
zO};Yb7g1!M3~H1sJzUV9KY3#@W3)6D-$+{U5*8spx&8z``9G+8%cv~7sB4r4=}-_5
zNkJMBq@>#b0R^N*Bt*KqOQgGz5K$@VmQuQ<ySqEj0-yIg-=Fu#`FR}Acm@OR`-*+-
zz4lsj%{edoivB~3;Q`K<&ob!#&zVc~*G8z=YFMpAH8iLuD~Ju~?|qiO8a0WC(s7~Q
z%>%BKOgjoEGO5}Zxn(@8N;GOeela2;#K!(SG^yC>4x~Ld6_Zi>LgzF|pBVAd=lUuo
z-j@}Qx)j-bJ^qn3{G?Wtvs2Jd+AOq(d}hW|gy<VUWK4KMyH@Yi2M?-W%439?Xa?3I
zF@ETUG{J%xPObRKWg)^Fzv-{t+iF_*LYt{y3U*4yw^w^_{AG~A%P*^@0S~>6-km36
z&504kytf#!ujed2u32TZnRInl6WIarlJ^P`(-<7=r^!-V=*jhCcMs(|;L4Bp$LKK^
z|J|3llVjYcN0_LN;NI_RL{E=&JtkBrv~qST!)%*zSUuxwzeb;ncNi2J^Q5cO!7%=8
z*o%fkQbm@IyDnd}ew8Mb;&ELNQyDfjtE5;Yg}-F~TOsvGi0cQrKgX+N@1+@MOa`gK
zh>H9aPrX_jDBcK(8>;YCJMZgXhyVNa?}1k(%(x0@?=vRH)sF}K`-1u)u?_w7@5&$W
z9&j~&Wf!NKct~=`iOhTfjucLCJ@4?_%?<2|^nYTvz||(}Gj|HZxq3yf&*i!7V(K<W
zPC!E9ABZKU`Q7Y~XrdcT{)~uVCAbs>5BvC><mjzF%#2lKIrVCdpN@Rg8J8kTUlVqc
z%cuCQH7Z<hX<@(@=>E+=6s{u=zN%Z9tIDk<InTSg-i7!RGuQLG>tS+z;vJuOXuJK<
z^Mwilt71kAoqa80W6=N1p?N_zEekW4Cskd{2s;1n$^EU)pdW*n=F=iWE!%bd$-HH1
zoLW-(K%26{1oC_tJtNMRDf9&xMWZ;v9-5}2zis!v2Rh?TjGEf<B;7f2=2DlaFwL}|
z)UTRE&^7tLrW~0e83hBWi7YJ}8^#_p_P^>N!;*^X%U~C*UvWV?StA8F<HYZAX?ZPS
zN6xj*;EnX0xLcVmJ862GqmSPe{M^iRln?ydr@QWytpzJ%v?ThXzwd)h&B!lS>XzJb
z?-6`=)m<)iblf{OJjK5Gt?RY*(>_lS>3_&Vs@wP)##TnLx~qErWXpSV-|DoT;Rf}`
z@0)~fgI$F_n!=j{>p06!<rziy2e%#{5)6Kx?NlN6Wj~PExg#04&~dUBHRq_Q;TL?U
z9NZ)EkX0$X*cdqt8{xscN$KJG*N}`sY#(#ve_@A6=F;O!P0WhJq;F-bEs@v`u35<3
ziNm627V<li(<dnj4_6CSRM{H*`^Ixd$&X{4K1w}PWElvxvll>Pg(D92;g1hHJWOVb
zGSdtUX8LciYE(6ZDL2+?tZTK79+>-e(8?V~`MDCjp%KUY)Z$ge_B9s4Kbs({^81IF
z!*-vqW_&Nm$q>rc)zPoQzsS#sT<s{$O2#8jh*IKY!?<?7`1JTBB<9G75LQ(qsUeQA
zvB|W3T0)wF$s1n7Mk;^0%tk$`=6fGmWX*nbJ!T{ClVqd^{*0j2=kFg%OUHwEvGGx^
zBV7g(zd4TO`Q!di?x8hO2IleH6}S|K$>mgPT<qz(#rBej#zS(a)dGiH80>#!4n&Zt
z^?v8#Ocsxp-=mwEK{_5Nw*Q0F$ky8Zszl&Z7wNRr$v9lpF0Prh7P5#KI@!wQ*~2Qr
z(3_9OcF7Y_5M<_5y<iq^*%EDTZ{?<h?BQW*tHL^|6g;Zi9<wVkrzAb^3`<FPt-z#N
z%OrK&euLR*@(VLwwTz#jQC#?%OTv~Gs{bpDU(3NZksjp6Ls~YAJ_0>?h~d8qbX^h;
z{d+-s;okq??I4=Pwn-e8MVV{o=-Y{~^%@0V^~uM2;Z+R+y^9Oa5=rg}3acEPKcyP2
zh|aV}5b3vxTK^~>=|Q$^CtSUm7hXrsyN=*`F*bV>CSS(YiB-kDk2Hi2O@0Z@SXbgw
zz@%GOE37o`*qB(Hm?74MHTu(p=({adVH%f<socp}V*5wMDoI4$#)2&B&SAylN2Lc|
zPkDC{YP!?msZteR6fG{cmgHs<Y5iV2IgT9t9trE`KzXBfJxv$O!o^;9Ad9`wQGfa5
z)myoUFkO1J52$7BF=&(zE^aMS)71`6WG6q=JlbhU8t+MJQB1u&vlz^#)K5}R^oYTB
z$GVQ;+4M|<T|UzHO*P7Ll-}AMuevJ5Bsr0l**Tz+Fv>$T(vL^*PnuZ#&1!o*m&ZPJ
zhP+iPIdy%<$<28wSa@7p^z}Dc;}?0H92%E{8|o0~I5!^#S`hxGb1W3>fAzRSd1H+2
z?F_vt>aae<ZEE$l((=m#VaJQD6w9p<#IcixpsUCj>PS;-&!$FaJ@ROiRKQG<`du$|
z$9oSEWV&E4U-s2iIp3m>v?@~lhtvH&daKb=4y8&{^_TOOe?Eo^O`<$|O4=Lw3wGbF
zRO_p~`OHN!R2HQZ=RxnLEoC+LM!1Q3TZMw<^ULj=p+B$yWCTaWPJZJd5Ph8cy@k10
zS}^o;>)3)gnY8)X79$V!SB^Gi;?;35xT@@2?8TbHk<sIaBBPg&%N_hm?N$$PwX0q!
zhHYCj^0d_3EZJlDDSWyQC|XhfU-E+rS~(tehCt@$^wcCirtgO-DK1a9A6MN$NY_R)
zzp|<Qm6we-!OYP;WmIrLX{cMxAQ^beN^Zb0K4-$V(y)L=P}ooXbjqaEum6tg$RRa0
z0)qBa^jx2!$2{|`ea<D7XZv68{BFHmf*r^G->!2O5y`gM_Tx~%$2$)qlNmOi!FlPQ
zA=;T+s5STSlJ$*CP0bAb3M$~%GMKKcIOB#i0W9q?e981G^c~beP;o7p0Q-jHHo$-5
zo^n|cZS(bZq8JvucOWq~<T^3+z}Mxt4bGcpS=MbpR>|tK;L+ubyMv29?ys1wN1pxd
z+l~{VeW1@uvZ#Sfn-zICimU;~tD26W!-xOECI?#A&2Z(P_8Hq)9&Lx0JLkr~g+IN#
zka3<&A`H+QDKa8#$zk?`sS1G^<%^8DCxzq^%8hIa!>_ry3xcYus)ii5Rat&*a<UAF
zq$5B9xmE|&E3q|3QH5I0mj|ehgSrKVTx3w@*<o3eK;6dLBFIs!_8b}KQNr`|o}u-0
zal67{4NXm#xn=?=4k>T!u+-yn%{ec|J>pEH%3G*E@URMexRr}`4Q(Jujh9(8tlMhW
zx{j~g_K|ysw#bjg+~e&Q!H~UFVmLl_-ApR?I-C!gdxtw2#k{fW?}*+PoKiV#!S^Y>
z^{Z-744PsxTv)nYJOR|}U;e&lCC1{(a%XJ_<OFE0T1y^dxZEmBDjkLxDo7I(6UXWa
z9gcjTTbQUg=-pTNQI2vOC&iv)b>A`z3(Jq}>^JIDY`=nZ;oBP?58%^KF$dv`L0Jct
zwGDq#hyE;Et;MXAmMV3)weQ*{&HFl;tnpi{@iEbSNf6iHlEgO53$JXi<5R;zxz6f2
zJRGXEh?UGM$F=eX<@c!Bt(HDJWx3iuD#{~qTbH{u<~a;Ycti2ItPElI`7epo9z)gF
zqOV>>fqUNQx>R~t6x~+zG;}|}6)i`(Fy-6JZvvZDg+RLc^2Il3E9%KTb&;$-R?Uij
z03lh3qn-dUtzEzH8U0&EY-_9!h$ZX*0s$U`{yRvtd-gN=X9vt0d5NsPZ5sNp*hNxo
zo2b1rbt_p&vf~Skkq@(AawO}2-~EebDlqy2QZW&XFQuU8i{%|NwF5-hz`#IS+OJ8M
z&*fxK`(h`CStS?7s#d|br#M-|!s?{of^X`_C^`ZHk%tCbR)Zzf(0UFPx9yg4Uov4~
zVR?NJ1#Z~i_hdJzykSX*`73{)m%5j1KXc=N)|Xgx>+T#A3(Fimdw%%Gp`=_R9<khn
z*vFg7+|h;w^$gfy%$l|DJ6lG6XwU4TBV5&J2xa3z<UD|3*PI`yCv?~ztPKwgC<7``
zI1IlTg%%4rk158uoYt4qmYE~U%OW@X50-TrRlyt&-r{)U`!!JOy@|K@l@p$77eXPh
zFuEP)w^7gYvTkGN34>wk=YoO<FV<3Xa)M52ljQ6GRIFXb3t%ryds$M9y(#7XFA!2m
zVlet(bnP=l2dH(*^SL}Q?Rt0AIo8>WnnWBZL#OL<?;c?Qu#2#87(;^3P;J?|4Y}YM
znU=0v^Anrm-_}j7+ydGks#d-@Ta55_BT-Ng1dFgozEf$!X-O1@KP0kWO7*%?OZf_w
z9vD0dP`L?v9AtK|`TGhuL0!ifc&m5+d)c_by!s%ntNG(kVAOrRzNlUo#3G$*+lPzY
zKCydDefkH_A&EqHZecIG)9tminNu~)$Tlf~2nKn53-pN}#0ZS;w(m1Ka8c-yn&)@o
z<KuUBb}B0PD=H@Cf;5UI>5hd=Jmelt^U#Qp%mT7&Er&TNBL4h&P>bGsV`HE}qIpq&
z47#1xc3mZ&pQUF1EnhU>5KyQ49#i<|OODo6zA+K^Uou8tTPW6uLvhx1|8h{)OZ>rP
zPu>9P3a03)C_GM69ON2#+f*KU`Q%Tfm~<5UF&Q2DZ>_|^uS1|J%+1Dz+s?5W@K;Op
zOYcy)N{CWmFv)Ry=u`No4*m2FDhLEPdVpzJ+EH`ti+ukJ2%uH~%BW|HUfM$qK=|QL
zy~_;t0A&xE6sxTZ6iCWAN-Atn^_8Y4lN-uJP{-f+2aIC|XfjF+Q6v1h911P?{=n7&
zhpUytStWPtRW5|B^OOCF2`xvGOk#XZD6#JN#@kjV`P{~!24d#!b989Mi0UE&ItCp=
zYyQe{K%~hgMQg@Q7k@J%Ng|^BCIigqti=0e_Dcw$-~YyUER2k(MYJ@w-oI_y(EQ-(
z<KO)cCGU(6(u9izBny0*Q9-+&_U?^@U5fr&W6f>W7i|^onn49igB;EVxx3l}lDrM5
zPgtowI;J=opM?Cih<$6b*qL;VpV2Zi2cj4vXZJ0i$(gl491m;~drA(E4uVK&#pnF|
zP)XvKOn+nNZ;38h^Yhz%nVi8b*D&RbgYZ}KVYqi-KuZ0QImy@T>;T<Yl8gPz3U@)A
z0=pxXW$cx9Stk)~T_OG!O{ZaRosJKEg_Tx?o7$TFTk-zIp1EYb?|-39;Rq|VQMe@(
z;{=bZT<9@@-+g2|{|!6Eme3L%E;1g|F&T`{VOSd>T$8)x@9=pO=z|P_H3tZ^-mPja
zH&}sU{=**@I{XPVH(HjfmBdcR@JY~}o;wt-K*^(c6y3$z`HLMF!3R(u_qhd=(vZDM
z!489C?F{9yglNX?B<efAcblH%EBec5xar0nnk_t?FiAcc9>(C=9}h3jWzFQ-;heRe
zyI+iecvY!Xzf6fH-Mo4G3h1=<*Q14vTDPF6xR^9Zikjk{QNMePl<4NSzbhhtUaPA=
zbrM7gkX{=rFRZ9gn|kBD-bFYJc~5;zm%vXKsN`&xj=C31E5yFMd(y~^$;qT_CH@p*
z?O{9WX9ZtLf6nXuBDCFCpmJPV-7mfik`(|VYTI~QrhELwE$aT&ie*Z+qRw=#j1qLF
zuq{5;UU9YbnH$&dxjcS)`#Fu@-HO58=CsY|xscgHk!;kXAkIDI4F+$2V&FSLT!e;4
zl3W%*>LyFK@Bh%@xNXf(VaJj8_x<^S@!r$ym%V!^rjp`YjE|r#Vq(I2ND$bRHn{qV
z=~n5!msEOoZtj8l4Z;qQhCO^F9Ihm4NZxB~Scl0zv_RPeb@;TO#fsnSqjAKm_(R3$
z5TApL9$b;2Ca#*<v;wJE^-Fd(wonsw@-D8afDs|X2athZa{=_74Sa4-nm1(Bea>cG
z%sR`qYF!RAnmj;1-Efs&uzR}TYNvhiV*BhX$rFir$;KbV{+!+Sp$H-~IPf@1{Y%7^
zEvl1R2N_Anf=mTd??e!EYexG#^k1R8|B|Yh4PB-32yOm5xP`+X3Ww`A9*~ygWef|;
zovs#zy%F_FaTaC0EtR;D`%|;Nj}(JC%k}=BJiP0iO7{1t>E@`yrrV0yG=bmGaSR@b
zT}j!`ivAemi9&yOw`jT=0x_r~(ENI9k8Q(^@1G~AM<`)WMmZ_*+Z^Qz5OS2^-+97H
zA89PGf<9rYYX{Z-j2m`QLi*Oqjd*T&0MOJh+(q?k5FMq#SStq@8X88uLqmpu!T+iU
zLw72%P#r(HtkIwEunsvOs~k7YTE4X>N@nfv{61i6ETXTrHI|wqyUywN#x=A7$>!C}
zUH^&~$+0Bdy5)W+i3;%wn~>oQ7Q_M2D*Y!TnQ60q#Y=c{U(oUX{`!T0!<M1PC~?yZ
zw}`A>{ju<VvSz3vR+vm=m<P?e9{1Sr`01FdDTj)N1_#?M1GOAw`iNs?`P<lN6MHyZ
zG?1}Cj{s2gKK0glORqYe(oxvyQPIdEpS**ACMG#j>7O4Uk!1~)TN#A5FdEu>p89L$
z<>ltGv9m+B28f+-TOIl*yk-w3^c`#{-yB_9F5bZ9(pbtFn=DP&s!G#o)%7t9qj~;_
z-^?fLcr6HPeB;YNs2FztrK>Z&Yru8GK!)s%o*5<?N9{OQzw@I?+lLjBXrfQ4%j!@Q
ziMCuKj#!IRMbRVG=mIa0pLYu4R841o7#-Q>zni11uA=e>iwgk%%JB_5hl-5h5Be@d
z{qJpL?qNf71o{rpbTIk+^C#!ghj@Ud(Rhbwt&;;Gn1KU#kJbGMXoqQ~qnp;D-a7se
zBdFBa@l(Lt?)~j$yUGML+50yBqKWE}9>??u_VnXb4pw7$X%}6>4a|tLZyE$^>tx;S
zcS=S9<nXgFl1wdnCOa{`?$15OKRp>ER;G~)T;%`_OvD}<!hkPwNzkZKcpnxE&sRX(
z<aW&vQD60$!h=l%c|lo>ZJ|P5Tc-k57Op`lwirK^9G${p!%LY=MbfW08CyZ#6r%Xm
z!*UqM9d&kYhP9zLH`8Jc*ec8!I8mrSQ!o$waR_OKNfMi?fF{-~Qwtc#p+qpZdEu=x
zl1#+3gTS?Y$7-#ON@iMGWZ3Mm!b-*b>1y!DgXjomPlh3O9f@77c!D6_@q%E^PaB3b
zO=K@H5d#o1iEl_&xJFr@Gu37iL1A~gp5WBI_0DCyCG=pXIcoZ)-^!~d!g4*Wuf`^Y
zDK&rAKaGmoGTr-4*;N;adbI_MSS~S*-hzE1kb6(LCd$rUdS%~Nf)RyIi2pk(^;~Nd
zGfF1$EFC0eKP&tPv3rl-dmS~~CMM49ER1=YkjO1Mq0Q#!6+LTVt|XaBM83+&L+=3E
zm^N{4@d@!b4hvSrLt7J>10_F23>n+|#kWg08=(f3F0slw+AdP9UbnR_kY$7ICa+Wd
z=O5cAH)oqUO)Tf`@_Jg~USA;L?TxkIfoIX5ae5|A!+&UP@W}JN`Fs<J9w}0-F~WIu
zWbbSYK=AbRfiDp~I9%P2{P0&*8=OLf6t_E;hkVw|iWN=kj;~81D$+fnQsGa<jZLV_
z$9xfs^Mqoi8cTOK)~4wZu=-19r+##4$<9T1DPkZ*s%;4qkF3KEA;0_jB5UTO6CnF<
zyl(&s53s)N>Ot^cC4l<jIre?`QZAPJc5yeC0HB+dpN^FXT*%ZbpuUQ4_+~9msPtcm
z0-8LthRJ$M(_jWlNg6HmeRrwd2K4jXjFN)4o33im3$`gh^dQkBT=Y)8iKO#Md-TTZ
z_YcN~*M&y-`<(~+KH%&TEYhWCzX3q}|Fjn{$7<{)MF(<ms-sTRxt~SZ3H-furKt}Z
zQ{g==j%PqR=lVn(r4SY?Er}r;A-fa&;c#@}A>q%QcYtGWw;*$fBI5|gpjEP$FRd?=
zra7R#-f+!3*U;W7UJ4&zukd77-V(WMMl$v%4r`#M4w2n#)}w^fFssVsL{qDP<7qe}
zjW>#8aJf|mpVb$vNuy8$vboov9Av6&$`}0vpL_T<j{ENivK_j|;M26=-x5sRJd8u4
z3xh@pwIr!$J8>9&M006wRIc{loKfb-F4kLK+t^%i^Hi4En(wELN^tM+u9&{Zc@sta
zJ*#QtP_eqKOr*pl=NHn06WJm4a<|6i-^HtQlB3T2+9u{Mw~bPzH8eu2%_6qUU;W^d
zOFDd5)D}yBUU$1tXYY>i`pHS}Qbar-t#mb=CV$Wbg{`KYjmAzT<3>mZ)&k1wCad}Q
z#n$4U^{#m8WbTH|2W>GOcr|}Sx<<DDE=l!f_E{>kPMf9#R=(S~Yv#gf9=uyJ>C#z9
zzUzIOd@PbMzfR%dt7kLQtQ}ls5kH}2e@VeUAgn7qc-X3xf~FnF#<!r0-X+*Fin>2Q
za>+xXsZ{H5JXg+MJ8$y1`;ph0^+f#-?k+Y)M_%FXom$=_4hmOvhv4u}jBR&Y*-!mW
z%VxJYl3|1&EKtw4C)XSb7rA!fjfZWCr%`{3H)tPw_zc70Fls<LMeS%VQ1YmdjJ73;
zBJ|Vi+mJWQOnTcZpdI!^0N5_(pJC1C^jJ`sr_y#e%1`^|MUZn@p@6A&2E-G5V@9R-
z)Bo^dx~5-HXhg0P-!%=*)#lzie!WTRWXYX}cwxvO%;PkvKU`7_ILFxYoHu$rGJ`pZ
z7jJFSs&QDlYL+jLpEOwweqYR0ISr1^yEq8kEtdAxOdh>^zIeEptIn-m>x(;Hp758e
zw$oJmv0lmf36Z1IcHYI)&TsejO1{q$s!P`Fnf^`a(mEd}`OuOW6-aW~)^&dO;dt5R
zibrzZ>gv<=%Om5izoWU|lJmG1erQyg|B*+2{F?TVV55)P+dL%V@?%$uAe@JU48ywK
z0eq$lTWi_hWkKf=ztuH2#Hl)o2Mu@B$I7L+>+Yrcy5_A~CR``l4G&O9o-OnnB(dLh
zrjyZ?e73J~x5%~U)_rN(qO88n@!WIGu6M#jBrd=Lq^1kd#VM0RDy=R}5Nl?SiX<*D
ztOvyWRY_nbEAQc*=v?)<YO`I#_ICwhifVLcNU5iW#`=;*pOR^xQ0rzD+kA>C`})<t
z28DuxVlV7<p?*iQU;=A*PfwlJ%?b_`Y3V;L5e%AaB-9d2QNSD6sBp#eGMU?%Z?leM
zHqWX{c`7CK+Y=RQ>!Tl^hJ2?Y)`lKFLE5zMr>v(?s0JT1@H$N{5M+^?B}Crf(c^<H
zJwiMs1v-j;><1|7uBO5El6C#)n^BQ$^0|5M8`whD2UGT9<-cDn7LH9H{+)ES*hrZ?
ztZZ$<3&E?*Wgojx67#p$Cnj%q%GG##j5PLrFE@|9pojC&V$#_=nP|Nsch!{$MQrBe
zBz!)O=ZHFd)9%f&nb}q{%jPdBVJW#Jl33*_ez)PBwFIool$7?fO7yg&-m*?@)5|Qk
zw{Iv=$V9$cB`WNp8f#ngT%gbLNaA(P&vSbdnA#s_Pi~$+YwM`;&m!`x4S13|9pSZj
zCPg|VPdQ%3!TA0@RAbDD$a5UQxs*86-5fSe)ngc_dA#-e(*d78{$(M2@UFlU1D6Z?
zE~hkex4WXhJz#Z8Tc~o$k7dyhKD3}>5my~fIDPOCbUH}nxA|DffrVEMFGN_A?w_c3
za@&bWxq3Z<C@u{DY>2Rs`IP%s5}F%ZTQuuib+6`o_SvZHp=8Ad=eB(>c8AwSI^Nm_
zHtxH>45yC(+$YWE0j*r=;U||yKA8wa#vU5U_zxA3Z4Cpz;JB^C%|sV@<jvszZgHLd
zOw~yxyTQd0;_=VpUdr}ua>J=;YI8HE+zGO-(;VHU2PwhMWv(>~{*(8Soqm$w#YG9T
z&o|LKr{ALf%bs|;csSp-fX>u`F$&LY@rN2G0TUzrlT>QEVC#;;bPt|Y%1YG=(ayDH
z=C%#8YC-8%^;NTCyZO#r{945q1;!nREkbV(yf9{3v~Tugcqyf5WehH6)0!pXGInnT
zdf^d?IkMp-^PZ6$IFBPw7Q5P7%*R*KIDB~{u>YEYT5Olo^G<I-!({#_)@JS74&kxI
zjl-JrK;pw+cIgGa6;=T${oQV<E&J0qY&%HZ@n?*1sqe*}&f7!f)>XP|?xZcb<8wC?
zza<kmy6Nd}+t=KmCp;>k!^i~}E;mpT0zI!kNvp<r*z6Gsou@Zb@z!--CtVOhxmlR|
z{exYo98c(uD7s^T^iM~JpTaT6QEDT1re~-NSI^!(N_}<V!dTro`Btcd;%nc$pXOlK
zx@(A_>%N_Kb4*7JTWp_|ZqZTy_TQYF2qmv4ZYPslZ;ifzZ(DhLpmjwV2j^YXZ*-pZ
zoIiBiLkJSePhWYEvVclf_C0?cWdt=Ripm?d7?<sa*)gxf1Q(87>BC(l{F%6i-e%Fs
z?@`DUB{~zAjgzZ4Po3N<ZW)%XThv6ov%67?<mN%7k$+LZye6Kc5-Mq@wk4tkpWLkJ
zU8!`T9tz9Tuo~RGY-v_R_~Ev;gTv*$_pD*z?$JhLn3?n8-IbQbFArayy+?hnwN%d4
z+!;0Q$XYiUnk)VXI0W)-jgo{^V>+TT+FzhXf6f01lEldj-3{^&OS|3Q;sraDJ<1>5
z_mQ!sblD3tQ${jFtW`i;feG#kZ!B?HW$|>nJU{f>yh-?nH(ITct5qauuadyn{U+6U
z-5q}O4h8n(1~=C*a@5oit6kiTU4t&oybIS~X9(m7eHLZT%g$m#(?RVH!-!*2H%g=A
zShk1^&68@|J3e8scblufl&tlYl8<!L9=VXeTp&lkaZ_ol!=j9vt}|9j{^of~QiRC&
z?zNBt=@G<7<W+feNH|jlH^;4vLUA4069eUzC3+f;KWsh2DA!rX%4?`^{aL9h{wH~q
zq0PT=Kwbc<BNW|xgs6CPtWJixX;gvuihD0yX=-0xw@0t*^?TccWY=v3c#nfz*2CM_
zj}kgIb%BAD-h)yx1t3o3lY8Sgsl81*?AWU)`R?FC<#p98@PlfLTsA;NTwbOP&(%D!
zBR?uRCE>I)x`tx^$4umCU7gFG-q-a$fEjLV6B&vyO8m56EBJ%*1smfA7px6p7W-)*
z62RcJoYuJqtfoQ#h{fFvhxSG%b53dd@w#w}FeLpbQVp_0NcE`(5EyxjBIH1&l%UzZ
z_=V0~y4Ka~6b)<0Tu`PK`B6PdS372ZAoU9^k=tcXH+oMz$f<NR8Jw-|oZ+!4vzFf?
zz%P07HDSKENFo*Sw|=P>&}oS$+@$F=sU3pQ%68w;XDg*>S|2*457DLV_sOX#aOP1x
zz5K$z-Gvayi7)o$p4GSH7QJt|xd3l)b?ZDrPmEMZ?|G@Bf__4P!<CO}@8JtbCIEzD
z7~=%gjrxT~Tx69lMTxy19)TpW_6ktE6%kfCepiiUN>_f<%B*08yk%uR{BN74UohSs
z-X{j}x6j`We0XP8j*kqO@r-Sy(X+aF%kuFJz?P@dg%~lqIL*s7fNKX&=4@=CNdN2C
zuR0X>hV#~ApWk=UEVsOOB-h}?#=&NvNeozPv<zKQzZe~%EZv)hjsM!6O=OPDldS?(
z$m_QeAd~CO)K!zO^Ii2gp)g0r7qujI3{DAFOBGesM>%wd7^;0f9<#Rnz@Qrig^Um*
zkUsHrW;D%jH%I%^!1H$XSf7a5Rak&`VeesLNW|aSXv=|_7Np|T_Cj-1K^RcjO6J!|
zG}=iv{1oII-35O-lLTJh_h2o#>TIldn9+MS=_+Ku95Y8VL#);^yRPSfKdnk9mN#fz
ztX8g(+@EEU;l|PbN`o>P>Rf|CQ+@5=*CEVIYeMViofl?&R)`|+iG&sm9CpUSgp>Se
zYz2JgEN6e?Xtg4<7K4&(rO2ocC0Y&m9}?B4Z_WSlKY+A@pS~_&`OakJlxEC7F7%S%
z!@`2m+@%`nrOIPl=~spq5mloBli5cR7dp7r;lgV>-F&ppO6NxY9^{4?ichA>M;M%c
z`+c3V4Q_s2?2StHSS6Rb+g(JQQ{iTN>a5lXnfr6T2WD@Sy=^daQfw_ZeaEFfpU&va
zB<40F<G`k)@8O%==EW|&zZ{58DYvEC0By`otJCu3Rj6s8ii^BjAza;4rOKY<AzSw%
zS`y{5TJ&^1&=NYJ$Fe;8;X(WqtQ134(exhR@s*T30=gWMEY6!PE_JW=ZD5OR?n^Q>
z9MuT%@-9h5zNtTgH(Y7MdJFxyPHQ%A$QcLgj?k7<(w}Bh5sTP*OZ}?AqR5G35f|tz
zO#mGIEz^Aj*nEHJx!XGEAfjshY*@dAS!aneGj<V55$n7>K1PSJ(U`b>QUCi!YsS+i
z46bL^n>AxhEwS|!lPBx!zfS0#DNS$VaLsil-IY!Kyx2=+{6hqzC5~d@2YyO_(y7<>
z02dCba68s%oD_}0f%Qn^c71tG@>8$R$!96mWr<0PQN|l585?)L%a@zynpGayo$&mf
z=Fa!``VUvR`2zCpqg%7RhFs9dfK`z<SZIMkEs?3Xy~24D6P3q7^!wkE;$qbIgkd9e
znUX9?b=L@+u*tq<%L)x~17Y6%*!^MVe>@YEiMRO9Lsu^X^DBBHDJdy2iKLm!VT;M=
z4zuS@ut+9xb=Nv&-rLnt3-bYkyhdvhpj6HZa?mT}-@M_^x50hcLm05>gqgpFKo)Sj
zkce(Iq}%GudMwy&$R>#*-+lmlqt7CdU*7euk7i;XU%iRh1MOOR`!w&xR_@U}vwPfa
zF`X$GW<V|ToZh$I6Odr=o@<X|Yl#(GRIbul7dUiTZmdV&sgHH_#LagR5Xd*;MRjy&
z*n|Y+xFl1fd|@L$#fa|X-A;o-8&^d;w*H8&Rr$|uO4XXD>W@<1b<Nksq@0|?Y?Ccg
z^;+$=Sp1IjLzprGy56~QTk!fK*Z~SuhiLaoq|&ER$<}Nup$+nmgiC>j`$W|DKu~A=
z$v~>dzW(7gD=!8T`r=haGIvm%mlafh=skYQBk$r>N{X`6BTd=F*Nv}bjjV{5*RR{U
zEce>K!?^sXYdLk{ObO#g#lRZz1kRw-f+t*rDI(b~l*Z?XTN>rRa}r`JwK`bEr}Vjw
zD2!kW71B1Q^Tw`xtm1YmnONg$wL-sS71o*ZJV6;PZkn@5b4;kEY0Ld`VhFPD*U1hI
z`}_Ac#?rX7fY*q@ZT5y32S>GbNjHL>seYwr%I^2aL~tH*VJfmju6=%=(&asMUY*{U
z3}+ULli@~o7aO6Nn(+FkEe^|tXKve5w6HXth>gwhoJ(CgHj`a5c~{DB7jQs8D}60a
zIJ6yj)JWE@{}79&`xBZLdkeG%2mbywDppe#{QQ#1OM-OmVal_Xu0+5%26D58#!bhy
zOj|##HW1}PF;T7GFtyg+({-|gJ`bygF0c9^{w32pJ9tHwMXk7F_RA*%ERcJzKz*~#
z_ozgvaF|6et7`G5&FTQ8=TrCOh6y@1yS!^nEfCpT@qO#?@D1Gqf@6Aq&EU?&y{@7x
zu&A`YSQAP3=%(mpcI5Sr(mwZ~CzoiOFZ@l+AG-<6<K9Yxi*aP4>7Yp8$;_`hj0g<D
z&JAM7Ju77STNeUaKiueca&jDOTO!zl4Gu;51yQIo7I$>!9=dljH;<ggMGso+u4@<N
z-zB}VEnDsS7f6J!YsW&P1EdDtrc>NeMQ#SVZ+*Sk1Sd3u;H6K{)zlYxHe|)^a?TW=
zKV@6#q(0(OsNX!*J*Q1sxu3O>I~FFu>0{<b$4+F{oH5YTWAV}Jx#aR;Oz_81_LpA;
z<^@Ui?mvG6(DxVUX1ALa#f57$?1oW#Wi_>a8ApuGpL@D_5JcbY-(Vs<^YT8*$jwT}
zNh=z;TMtTYf>Zyr8{yS;*!kDS30Jsn*Pc0hRt1Lk;J35m3bEv0(vWZb+_oCxJKH7J
z@0mb>cod=Mk9oHl%S;K9cc73yl#Um$bL_lh)FW=?<;~<R+$Zw`QqfBGlng!$79#3!
z@o>G5T91#mJhY3tC2o9MAl)^c4I%u-qo*&Kt!EQv_OQ8%b~IukhMMeeI0#(UO4M==
zD0I5#P}LsP^^)4RbuV))W5>W>qN1X#PXa`4s!TqQwt75cZ8x!7)GeH*SGIWin)=IG
zxf1adv%uj-P*r=8*(s{&B^nYAQN!AB;otY%IdYDWp&E3=b1yJK4hw70rO}BagT-sO
z(12YvKNS>-_&f0OJ*&c7I-Ut5mO^VbSyfs>v9`2c_yC3&{ag;~XQ&7%kxC{-4DBx?
zlF#qG!LLEm6)#$If<^7EjyYW_+OX7pvU#`cXnNcJ?wLsvtt*G$*u>Xx+a!N;l5af8
z1V6*9v+Wl`?wY%}1SAaNmpWrYvhSUy<J(3128}G)x!^w2+Y6FaO4(mT#cQ9PO$}O`
zWjQ4wrlVhE`<Rqm%`hT$iAx?<VV=!q+YJq`Wa96o<7{d_VtH%neSCa=1kl>ywyCF#
zFx|p>fP%@@4bp}L<?q}H9U$0&CQOt9T`^Z2or`I3${31US)rJ7DcvP>psVtcsINNF
zW0et=Cq9(zc29S<>x4w}AKh2U)gTCGF`sXZQp%FszKhM>HpL)(e1kP$WoZf8O1Blt
zJbYK>SytotqO7fDq<ZNtw`zBAO@rn*(D34BNexrony#2Pattpqe_|gV#+>@~>KFd)
z?^k6Z2JS(Rp?$=Io*SQ`?^QYxdYC#Xn{ICm?L5%WtllZ#l|@yHsdP-WnSI;1tY0YF
z5bNQqPP@B#O3;m-DI6=_9OI22-6`T}%M%GHh;bM`<(NV>pb5?Ef;Y>KaV>a*cK3}n
zw6*!mExML??*%`9Kwv0CFy<QCUq8fvL6@H9w<yKaE%NH@+fjYFP_$gQyN7R&{4{WO
zcdeY5rB9726XfK+UNzO9Z{^*zEU(JX&(F!}m^SaOLL*b!s?2~)H#>Xy=ewuiKW#D`
zij*Z&l_oN^haab+*(<P7=6cRgF@^KNfopbOzO2AEp)ac<MWb#*j&Gdtanf|<J*(|E
zQELaAliDxozA6WgqWP4_`O902AIbR$lgQXA$!TN1&5^~@pRf6mKuh>ePj64?5OfH#
zU0nfl9QCn;y!9f;rMCz`LUBbDDy!S%Tq_Nj;JA$qRH40+;Sagrzk`(vnwJNwZ0RrK
zly)bMX00V8&|W|Xr7{M>ac?z>T1?Jez44>1<N`cb9etNb>LC(Bw@m!>pj+qMF4&6v
zq8M?<c6i;<p%%XaX^%=|{N0vON)?*&+xo1|xZ*{{3{{0?n5yJLiKx0my8@fNzb*Er
zjytYHlClx#1#F)@hXbLLqX>55>Ye_qJ=-}PD&3>kg|4q0UD-^=dG^YqJ~SnGUTS8*
zu<90owih(9us5`;XAiD&lXdiFJ_VIZoz9eIg{+hkWtECf>YF2BxkZ}_L(}}hRhvnd
zL!E8iuFKQo45i7jQ2&}r_DN_0no4=ijYs3xe7x25N8Mtkxw%>AFOjFv@4$~s&O|_`
z^&gdcLlZmpcQYZ5`wR3Ue%M_;5EMuNkfEeE<i(>Vv!TZw1HAIMnINTV2Gs@h*2u&`
zF)`*WSn|m%5*nL5EuWd0nTyp}tD#-Bm@r7y_&bQ)K{I+;)2r@<iV8zS3v}tBRBY0P
z*?~3Mc9qhrHhy@-d=s<87@9zanmJef-&w10$-tYMl*zFtQ`Jj{eDt+8h+K_RhthS(
zIDFru8(F(K7ODuh+jFLp<sYg0&GtK>kWq0#{C(@;d*9acnIM6(e39$O@dk4aOIZUF
z4r)t;7>$-fXtA-l*}SVf%0v51CdYjcn*Cn3#%%ct#0CMo4S3<wBeEbye8bRd3f*ES
zC4mI6<)G7*)`S@*121TuleAPU^2fg_5lTk|IXI^F9mh=%_g4oW9z61~PU=48fi0P(
zCZZV)kE{F4Qw9m=^Tthj&myy=H$S3)MvTJ@)hJ|)w{pBlH`eGgs5)zXf9wGy1Hz_}
zb=!%QN1*)N+x()Ivv9!5#Lk}g?HlH7$lKI3(1OSuN|r)c0Giz?QM;gP;qZ@_Oy$z#
zhA^=Br6R~V38?{@jCYKuqHjp_;<~GB{z_8u)J&nIkaU~8%Q>Yo`TAknE5Qf0cSJ?r
zc?0n4y347{VDOia*l>x@)v?C4!~5vvAIcKLX*oB1TZZO<RIZ<CXdC6#=DB%`y-Rv&
zYf_2;Ujwl?!1Y1By655ZKzwNb%A9c<=Bu)RBvh+tI%me5S8<x?HGNnl=5qqdWjnj5
zb=x(gsp>%B$0UY@J#aeskg4bp^R{>G@?w+P#^UgzV|)pPujk4XY1=Fke{tl4jPvrO
z_v1k?>+vU#G}dyQpX!8=*LnC_(1Og-Fj1KRQ`B#JtTx0N%eGSUNb0x5Wxr(vh2bX=
znSI?<n{<17G2q%`P`e1M2zF#U6#TK8A2I1cbZam&rSo)3xIDFiSKV!BPU!n!kEC;d
zrcFw*+?&#L#Z4~GrStweVK3Ep#($o5cAQ>{gd_T$A@^bfYx=>>&$a!DT=-poYwQn#
zarz7YhYJvJyG!2Y><!jT<Fr|)OHY%N`82YFD|@olsVGx8YxmS)T&{+d9iUKS(g66~
zb(oc}YEqT+*pj(|4h3V-8w4^u(Ra$4NVvOV2On293T}-v*esf^(~zzxLw5|v#dW~x
z_ua+0c9bkg@Hpc@gv5mlGSi~rSKpEmp?X33&S3bXu6%Lpmj^v?|AAixiW#n9YcYS|
zOpgt_oG-O-)YxhdvNfEVBSD>Wl_EYqj13Fh9(o(n<d79=C_&THpREMjjRZ|O_3={~
zK{4pLy<2zuII$tzJ!ks<dFDfLW_pz0-4wWfFwxH|Ww9bzhyy3l+TT##N;);Nrpi(T
zQ!AE%RjHnSbgaj$%4FGHx#N+X4mc`qhd#64cjpe`r~oE-R5Q<QS9D$>TVq`VlW$D4
zxOS^9rc{?JaRy`AlW(k?$!qX=zDPstU!r8@!4I{jGj0#h92nW>EKCQTpamkRlADw#
zXt^3%BVRTZJYC-4WNPWRxKb=qc>u}3A|8f;X{QLoS;^v2IyzF)N9*MxXm$+CR-TNq
z(~Bu0BjMB?b!=(x{<vguF%@pkSVdVrjAL`A3ZXPCdu%@!)JK3uhCwZfx9DcMr7`5U
zxAfa*je{+5^}9>7@H;tOThG0yYO{x&WM{bchEiUeB(y+9ifCE06)2@e#^K-kCL3`i
z&L^w_iij;$WE`F{XsC15=!M*5;+RB%nWRzU95?V>BJ=$HSvon##+T;_^kFh!fhHy8
zD?t<Y*kLE9$MR&Z`@(m?^h?~<gss!$(|tzUv?Uu?54Vw}dd|2{Jctn>ZR|NVq=-`f
zZt~#+gsO#3TWSu0fE<XH!;^<c29IQ)#1tOaIx-*P+1a+0<~|P)X^5%8>(tC!z>O{6
z$1Txz6KiG=jXn$#Fkg%cWl!qT)Rd7zHp^HS?lv#(I#`u=M#ibKagr-KHTAoDmcM$E
z-!^LpIIQ(m-6?dC+m|^b5^&p99x%x>ss|)c-j^oVH-1ePNu`d5&Cc2r+Z2H8-`HFz
z1q;KcYQojGzs}B1zTZ?UU&z(tyG8Uq{$90K*C{T%-qzVz+9*{tM;U!Y222GU9Liht
z6q%xmW^a}YYI6;@l;nV}19Pn(U(-r!)Ot}DwX|0JYag+g=h0Gnh?@Fw0mFb+-Da#d
zQ`AWk!qrCdL8xs|%ZZhi)=Oe6^-X(H@Ug~ScvUpyud_aK&fwWFHYHZ{C}F=1$9m)o
zP=HEza$fSkEh44EDf8>ss^j4Y>{RMOH;5>)(LiX)#>fbST!YYK7Ox+b*2}$v+HNjY
z7w-d!nW5qRLpIu~Yx;9n4#>?UCq%e<WEQ)WFV0F{t1Ye$7G3t$qN=%WRJ^JZ<zx?W
zkLkhJYY~Bt=Z(9D7`ipCL9iq|JTZ1M?fU!?O6fgS_Cu2`nA1gd=IhT|xx={1w_HwG
zvb>b#8_VTk=CnLl;FukYnM@;^8zLZm^Os1p%6@BlvTCfg%`sH(nS=FgJqITXl@O3T
zoc%~WYM4+fS`}Wsnm6YKy4>*1*F`=yMLS|dGgdG;OKeQe*t=UwqI=jktT{rHlAcp$
ziK3T|u3MJ-`ubLBu}`!IbeVk-0rVo&fd+Y=<SHt&bWd59mzV}E=<$h#lbx}>c=^&>
zL>y@*TC96YjOBoak+t>5Z3<D4b~QRYYQ6p7Ha3`eK^8!RA=V81--fnp%C;P}14v4#
z)p2RNUnnZpo5GYeMT@xR<LDQbgW6Rth4FEEXinw3HsbW2+r1S%XKzbQa<V?~RIVBo
zXgMrN4i6NsJ&!`7_KkBpL|_;6z_os5_Snkw&}mF4*gMoZTlIKXadN&dvrm;!qT*#c
z4p+w0tjwoP)<Y?5x3KO6Eq;v&b#oQ2Vz`&g{nlZ;#Hw;~H!0T=9#d=W<pr{bQs*xL
zrj4(zL-X6ClO9Q-iLah6<f=&X;8P`YdnLT=1t_K!os9m*S?I^?6X8p2w*+cz==bMn
zTkmUFhW)C3m}U$lnYh$RipdWQ-u5Z46RG~ZEgT-=$r7OskPujSVUsgV(d8fu(G~PX
zx2T`?zGekprd&GSV4n6L0|igj8fS+sRHo~HYU=9nM!7~8?}?42rK4qew^m{yhaI}g
z^S+deqV)6zJ1=N)R#?tsQtMq_oU?e>31XLoA*18w1EePRoSulsvM!o(x7_>?W7c1v
z8PXO<FLg`;CSXNa^Se*=a^AD5`H^|-4h(9*q!_$`rK?2DJj6MsXAd7KDxB?J9I!}S
zTMqf?AnddvTk94D)UUE><Q1^>7^^k4Cgop!=oA`s`J7%!R`v!v&94F75cznn7Q418
z^yP7=+@n3dDrP&M(iFUnzIs=10Lkj&I!}g8xm%r^)z&!W$SjE@D5GdcOOzEW9%p>$
z3y+>NCYy(L=l9ug2<g_#T+4V1M@hMyZyaC)h2WFG(j3zXttFzS46lCMAG{Mr63vG0
zbkPqD90^~^iri$3YK=5%#h1yQJ_csocDZ5<C^rSF#HZ@5m9b1QxdH{cFU`ivJW35Y
z)$(-^vF;oNuK=2f8a*f?czL#aIA(<x_}W_<VDS+~x>RTu7XlqwI9^-UXFfNOm~n`g
z1G@a1%?DF$g{E$}y`jy8Px_vq_%(z@MM)wRdCbXClPUoNtFDsa<*Qc!<K!qCmyU9i
z1vTjsTuAeU+fXxQ2**ZL4fZw_>Pr|SVO^09b!eUdj|6wK%-ZeUj|A?FcTcXgV;Pzl
z)yd21??q>E?-%=%o8pS$!*cY9K7+V-CX^f&W-;~4BRDu1;Q0!TTxfL{w4zQvKOX1h
z2^4LXLlq+N=MIx;)Ws}d_R%5mZgiLdCKGxb(Msxc+et<DL|U*$Txipr<uQ@`;~q%f
zzGJ8DXz2EvD}a|NU)Jcz$eyEMlJ>(-Hs8N}gFrahr1gp=?CFA!o&DjhPS>1;8@0ka
zT<Vp)nM8Y=I!sY=|C9r78_D!t4lE&4UfUr9L4FZ${N$!24J6FEMy2<QWk{h{qYC5R
zDgHiCmr>$&iv6;c)$hbey9~GPzQ<C!NIX)YaOpE&2fkcAp8g$!<S&H}y}3-Y42|`2
z@0WP&J-rg``({3k=^cog&Vjmu<bF3kEpR4GopJ)P6n~P^T~i?Oj~?>FbJ_|v9iLFu
zJNro7(I!wuJD`-3+K)u_rm5&A4K|2i-pXyP3R-#C@r}gHtW>VqP6nU65EgD~LY=iI
zEK)Opy{K>bW^(^8OJY#Cu!;i-9_mCnC(<VU7HjFTPAsOVZ`Z|c&^rGon^^}(KY_~f
zO+O$D7_5!Bhae$<kX7e#&%!f&10zmZP?DXa$0q`dOH~R4c8|0o{L=f8bjE~!&#Gb)
z_=5IFKtGil_lk2K?1Y!+=ovf&3pk0O@57_-FPXCXT5!04X<0bjzJBq`6l*P9grNzt
z?XfA;@#a`{jGv~D%R?sOnSp^qWIT<C&~u3v2>PkeBM>wjxE%bn9&c^txyQy9yZf%g
zUXt`I*cl!@Kt7u_ryKX4Mg#H>h~V7fB8VmA6MU6>5yFDvvYxkfc6N4kY3J*BJjqo1
zsn;@YHwo?h571391kN3>UL4fE=$+xc-RG4z6!Su*ekI@i%1}V|I^K(?W4|ws_d!^~
z*|<qqqz~Q<H?X<7f1sF(KLLcebo3qP>t};OX{*xmG_7C?0Zcykf_s~EVT}6@Nht}{
zW`!R{)c{%s)+EPI@+(>FWr5=z;}9QU2#e6w3f*{xN$`y;PC`+Qt40<lyf@8)eQ@x7
zF{l(u9qV;Cj^u93Qi3}NtVO5}fM{Q=Mx0>|x(`3EM;-8cNHz2eUxU7hi!7l-Sw%$}
zqi6LvVq$sUalpV2m2)YPL@K?YpunP?_2{GCxS?wb-&^TO#A_z|7Nu#kWyQ7rTzJ)}
z1Z^`ZT8Kv)7@sR%hrawg_A^&kNOeOClK>48^DuNW0IC;oSbW;PswQD9LMH(L-fbit
zpl+HTURr~KA2z>bYf$<C`s5u7LDR3H2b4jI*<kvlm<{een^n8PT6sJP9Y3t}-o4A*
zjEt*sAu<&5RLthOjm@8A`tKU=gZ|Lq-117EYQeD#Rxh)z`1@`bq*^y*Fv&NLapaKb
z;t8@woqh|=+*xCDOTwoaRg;p+icjfGG8cOFMj0#i#cvB`|FUJ5jhyW#??ELxM=>n4
zpgaBCZ1_t}NTZN=C_OJ&D`YClW`%-L2*?HA;?PpyqMH!55Mw#9Pn(D~R}?Y4{Cq6G
zV`^Z(Shu+sovgnK%3)AIrrXy)ReQfxjv_m)S|HTvBq4Rhl6)E=TlN<}pwF3F=*4@)
z?DGS;6_5ykN*^bWpT+WZi!dT0)_r_2us!S0hi>xNc9jO(A{)`M)=asnw?yF0nXgiy
z^}Z6-#NLMkv@q{Kp{_qRjZa}7tti>dJmK9`DJgzdq#CBnpu`sQr~HN(&C}zX!tRYH
z9Df+E#GZG{djeROG$YkS#NyzMgm0RE+^=@s%K+heE#61TU3xWs@ECt~EhQ(yc*3gV
zJn2f5f0O1hsA+-z>d>X(-%taasDq!~R3b8h#I+8b<MZc!BwciPPdDDhyDQNs+qgFA
zsy+^AP^%h7XQN-rkKRE-$as9MayKO>P3uRh_f_;a^hMAMf1Of22IAmKK+fLDR1bi}
zZ}!^P!QCAxN)1eE%Qe^{)uz^|!(=iPm!Voyz$A#+%)U||+F7q2{7%#MGVG~>Q_e)R
zQEl3OyA<r@6F8R}ZuWk#tpAd3K>_if=B~wCD^K69T5A(*yMam#TyG?0OfXq~V6*Zv
zQmj!l{#m%4p3|iV{bI83{b+^elIYK;v<M<=*Wxhzo@OOFLJOt4Wukr_9*Bu)8PQ>3
zbvSHO-k)MXw+j6*Shcx$v>jM5g+Rm|nVb?>h>3tNaBYGh;b8btF_6wj(ck-`GyZ>B
z=W11H2n<9BR11_NSB^0V1zw7Q5->({rwi9#v5M|nH|6I8`W%=N9O=sE&o3_ctK59%
zvZW=vyi@j^6;!g<XKS?43selW@K*Hi54|Q1h39HZVt~;^wZR%~r|2f>X^wW$b)DQp
zgX#%(O0!5;v&r4!?nzodjnoUw2LzB36_JU7&bSi|LB#W)M#&?!pN8<h4_uiH%Jq5Z
z_L@mEiA8~C34j3N^?V7{6JNcd{@w#<uORdPC(I#yx~_;7DF)x8qrHCoiqb!0)c$=7
z^?GnaTM5hib;Y8@h75#cF%&dhNH)QrL&o9yEYGNB|L1@K|1>oL%mbOk*B27&{!h!V
zPQzII>jyrZLge2ss61oTx>13LNYt91iG}sbYNl2@`=xjK$5sVIkvrEXMG!lVRzrBk
z<?!&~j=uiX*YLxy2voR)G~M~r$&*u2a4fh5I?ErbArS~uD*v~!$p83lAmDRfU*Xj!
z|5s;_|KENXpB|E7AOKn1y&4E+ima>Kte<r*_Q0pnti{ZNhmLUj+8ITJ7c@dR+H0gW
zLw#<kHw>=)J8u88v(USeunOYrl~~C`i=L+w#O4Sut_Mw|*)e5QxOekQsr1QJ8vch5
zrY{2KwN?-#=&k%!bIPT$Pq(@x{jLGbP6UkWk;Es2s{(C=3nis}<&B3g;13A@WWV0v
zhv3AFtSk#M8tETb2fLmM2%^B$0g*DAHd?{eVXo&g!tVus>#EWnWn>u(3sHbK_iW*R
zAkQz+190@2JzOS7Smw2qUBvvq{x-D)s4DjBkYtxFdvsAxv+X54j+zgqJ53LJ24FjP
z>>2{9)Mf5?G~gt16d-Iy@xHbrL3uc1Eq__SZ#<|3`A~k|&Es;gdVv4HYh{>!#u{Dw
zYOyO4{=2<r@y`II#a~f-^W$oO8F%IpuLmA}Mo2@w9-XTn{mHJ!_v%Ng_y6t_esur(
zZz!&Qord=R<-_#<9xqk4!{~oR^@-V;TZbTkk_f!TD+yyM|8J_(rTn|&&kKdH&B@e`
zZgR?5#!f)cdv;~uQE^{Tm?R6FLv(SjPw%b=!1@0|z6PrVIa=_?!Tq;HTK=&MHjN5=
zt5L!HLaBs&8XlNG<JFxiegsbz-W#G<wmcCCvaJPKfclxgd?eX`Ln<HB3Rq8sthp)Z
z8Y=z`hK%9=?xP|>a&<yRq`?(0@mzE|MzC!G&R?o+NSNtayAK$CK*>Nm07Iw{>3eNc
z5*n*Z_Ad3GQ3Hq!5>4Q$i2sd0VUK?w#p?ZrrC@yHtMR|mR=^`6_<zk24wY-TXRK$7
zVCSFv@7j1l$e^%!aXS&1`hSwq5*^@pcm54Rz&l`2t37<}t2ebyFJE|TkrDxc#{2cP
zIS}DfV7N4>f?r9pVVLetm4;X|)C&_24>&AoPHgWa0u|$G;!+X*_oVRU3;*^(Tiye9
zN8ql8_s$z}-q@7>Cn^U%5Pl6e*v!BR&jdRl2<z_opRer#s64P;b%;;6*_6UHD~*7w
z0o2o-AYgFy?phTdng9(;kX4DW74goW?{KeS{}vRL0^s`fMEH=Oh$-TMYR(d;3riU7
z+D?T$4`#SGP9_n293h~7+*<Ld9{A(7PXA0($_h^_ZBGF_ZDiiKm3Lo982%TUtIqh?
zds=8D{<Xr#7zil!|J;^D(#l-9yhR#hYOh-|z}BBCvDqq8O`0<YOC3;$=-^!$$=Ij9
z*$O4Snt>SP{|zin|M0!M$9sVyN>8z?#v@-V#3!db-TbIB#_U}9=9vmSwqL2&&O;(%
zpc45ai&>8vf`M&!kUvU$?nQ0F*B=U6X#}q;lPUyv?_WV!5{ph+uDF%!U`=4pG44IK
zvXehFL}lJ&yRKFh39{-t|7@QK#l9A}PeCUCv*HG!?jbrS8HjffFwp;9)(z_Z6<@0V
zAm{P@l2IQ!PY`kYgJ=&vg}^8Y<}?%26!74x1)!t7dN5N2A#CjWO3yCW$KA}87Lzu&
z(TNotIqtlpo$R;O%_dPpk$oQ(tCxC%xSJA~Ymz>4mYXFkGa)7qtxY+S_8wJI7gtd`
zjT(I+7-sFGbW_zDaVkx^7iOo=F!n_!;HNoJrNjh2%a=FbW*coEDw9!Qt9?{;zO<i-
z@X~7uch+m$IpvP~L}#5bOZ$3Ymv3_M6yYm{O~_;GQ>+kD6|)uSdI(v~CecZwA#fOj
z=`9#iH=4K=I?bz4KFXc2clywN=`!#Q1)-hu%JbNL<w#9`WnA@J6LZukp4;=`iT7jS
zuH~22=fX@j)$JM$%huZAze5q+6)(~(Age(5He>v{T(luvfMWl%8B6saqp@(ujo6E^
zz=y+e%$G*<m%-W?ct*6#y=fo2E_dd0aDpmTT+(C}?4Kj*cw9)%US$a{y)x|t+D6`g
z90(cg#O*XUs5w4;YSzdeQ!p*Vf8O`h?0l~m`nF_td1}>l3SIwCdtV(DRrs|#CLJo$
zDS~u&hk_CU(jC$r(xp<0q|y=s(xo7s0+ItLDFPw_gMf7R-GjgH`|euz@B7bPcb028
z%*;9GJ@47?j_29$-mEwX9K523=mR^)cr<lg^RnM7FHw7UrScev@(MMWo*~t*(zR1$
zq}oV1yFM{rTU0+C$_pK`F)~2tsOF;|cs_BjKuP1~D`OIGtDTo=yLUXuw97Kaw%4e7
zwTWG}Hb;Kf%QS6t|8002S?K6IEXdqDRhY7);$lei(0D~L%5ZpRGHgjD%6}%2bIq-6
zYQd;;83uJ3im@Z4D)+mrMsL#aC7}l~=$q=<@PunRtt^eq&vPO37R`aZVs^vVrTjg9
z(Q1Ca(huXCQ{zdL-09t^q*Pc0HYdV;^wI4YDiKi)>n)7rf`}FwDcE|&&nn`#<2Ezx
zLMgGn4F&Mp3HnKz##o!8tNGa%tW8Uq2&$W!H~%0L!@D#eU3zi0U^Dsc@8p7!_9KoD
zt(IH-qW*pTGfxihAIl8HSe}l(s%t9R14g7pu1)|U)t8N4YEmP9yy`dSXO395yYBtu
zZTh@JaOOW&v8>-)Dq;O7&T6dsbb7$asgYm3=5_a9aeo#+JH!P<zBrV>mRhMl-SHK|
z9MTgp{3u)5+_O;mHgRaI-=M3`e>~Uyl{O11gg|Y6*lCr;b+^6VSWIJ(p3|s+NS*cD
zuC}qR<(JE0jQ6m_YlOw#`!7U3V@-~5Ir{8&B7<qn3$ACrF1p@1Qd{sRwPN$}TgmiV
z=M|($(}njOMsXww4hBcxR!Vwyj}j*D&G@(n3fC7kt;U9hcayW}8s6$HkF-m4(6gJl
zbeNLnsOi6C&|IrI2j^A}Z>7XW<ZGhq`kS2AdRRIm&rW9WR@MTKtPl5{!iAOnSN{Gw
znwAi0$UK?iq^|R@jCrin2e+Wt;Odyd_;VNO*h@21@<W!Os-)JvHvDX3NjIVK@>9Kg
z@VHw1Tu&NyS4jvO>$g<lbUnVoc2*eEQ{FFXGE>-Vi;NoG@4Xz)l=xU7Q(>?5-CkF>
zEnW$=p*^s!cS%Ya8df4qe%=@NwBN1>ba?Q&A%fy`ef!&65v%y3Z#`ua4feA?#{D{S
zJOw9SDGIJ)(bV7ZD<19~=p*M?I*i@FVq^!8WgD=J5T@(U{o&edAw>B;jnvV3E&jMr
zyI^^TTpQ{*bCE`TG2=+=UEVo73D`=z^z12N>)KwuL8+f$Pf;yWKcsEOSt#}XZ3B+P
z=`}$Q+UY7(-<E@D^W^Z*+NvMw@U85q#NU0DsIv`Z$OM6;Ty+<V2F;-ygV8oKyHUa9
z#_yr{fP|X@6Cgq-c`9U$bRgHBeDx_rA&C?S+wQEFFR`L7B8K1pSsxh)3A~f5=YES2
z^NELt<<$4V!Ry!|5jCbgN0@XXA|eqp{!Bza>OIR2jQ9SI-@p!$IOeUejwQ^ht{xL>
z7^qV->a&>qytW4&jZ_ZU4yyPCI6pWL*DkgHTOegsiJOPsD^qD<;Wq7DLDHztJmptc
z9vc(O2nC+;228O$DRx|1oWVO{-5X5YyBaf(yWnr}a252H>kL-Gh#HNLGw&K9BA)?7
zy`>e(WHQQc7HB{zT|8XeeZM9hHp}V>x3$MyrHt2K?(TZfwV$3G`AcLp{Fs_b(;rif
z0Jtith&q^X)0T+{YEo-y@{{5MT&lz7@vG&+tS;Q(TpE4seLixil+vM&egG4P&@E*^
z(AX$fArDvP1uUkuRWS{t(WgCfh{8eV-%g#!(`5#Rvg6$9<9>bp>oKDlly9_;-aPj2
z=GJ<5UWi(Xech)~&OwHj5R$A@azolo-IPq4R1!?M@|S&}>M|E8pW|e`rxPt96TNkp
zU;!N^e(%ETXaJv}B88Pz-lWRbseHCtVz0%|3BL(wL2_`N51xGdL$WUN!jt+RhC7}O
z``bxpNAWsH*Ttzm^j~J~RaP8g^-ViC&3@xGKA^Q><*F)LU&TTY%3#({amdd?ORFyB
zIW8k&ep1tCgfc^)e$$xiL6V@7mf(FJ`lRx}yRj2`HNNq?tNm7%bQWCYp?kY$F+XEQ
z_T7~=SG^X0f$f_3!OuBz$3q}>-?$=!2a_&^$U=TAEOxZ}+qHzc%jhcHQVy1Q9lsiq
zxEpe9TKV3{0Q*$oahr3mhmzGxEFDTukh#TGuf&Oy7pgARKCT$T{OWwtmpSa#xo?>N
zkhS^Hx$R|hiSJxyvg9yD#*^djg0=6!`YE?b?1^lRq-S2{<UVixWgDr4QBKZ4)KpPe
zhziAbz0DbX|27*1Zx?xRfB!=_qnOIgFP|n-f}(F#r@4Xp8p`l>+0F1gzS9ZjU~Vi^
z1XRK4E(x{W63(djrmFV{-W>svjN=a3v+e5o)#Eusx1$O^W`!b~y1QQYXLqig-`Wc7
zP5x~1Aumr}>0uz#qtWmrv+@@$t?xb$?{>?A!+Bm8U~R9L<t3D3CDK8buUBQ9Zs+wy
z-bRJ&QJ<-i>9bGNRDp^dzT;7-^4DM^k$@9Eh?x0vq;httnP`j7QONITrffuf^DCJ3
zp~<s{omqx&)2H30>8H;s*4Ap*B2`9@Qi`%}gn0|f4cb_$fB`rz_)Umtd43b>Eyv!!
z6&v%0Hki=R&`|d#URqij>K?}{1ESi9cc!qls$(AVyFs}NQSKi2L7Asf{$QERTfH2}
z32Gr{T1x!HPb-w5Z9ly45A12%V-I{%QQqj@6R6xgDS0|O`$h8Cj$l~n@v+8KcCnDo
z2yv0ZRQ6lgJEyK<9H?GjVBC0$3Uj}pxaPs3mB`sRQLCfbD*K~MiB-YT&bm01##CQ+
zR(fcpX--F-xAFdkr|&KxxH4=~EbqcHgNpx?!dD*Vxp~N;*xA`T*dmH$u%L#<%exn!
zNdYL$N511F1~fu-l9>YUxCg%BNvHM4z^=6?!(mNoUImyIzD4DYJB^)zUh^c)96lJB
z<S5-gAMA(p4)j#K!<xD)Z-)@tX}E{<Y2np3wp!|E7m~P7eSbg$Zo_Z~4I};q1e*@*
z8N6}h##-}PQ-g>F#r@`K8t#J0{>wqqv%0ydC_>E3Mu8{pP+4}lZfu#oz}k#)qQB{_
zPu>3AUmc>|x1a|0Nl0g0*c$7mX;~Uo!^SRYGd+Ks7;9!B@oT;{T=hE$^c}0x|MfVE
zq7rSNbB#~PSFOGm-Ff5H7d<__Cx(KYMY>B>IlDed7K^hd3{%F87eR3Ak7sAs#82Bi
zo4xLz?uF$=8%YK#28mb3X<i-VJPVzUO8MqgXl?x{L3<!7*|VeJ&AAF{IcC(h0`2lb
zu7dTL)sPJ;9L)cYD|ff>p78MW)ZXgdSmT7IJK(hK|D2Vi{Q33tv^v`8;7b_{J?<>>
zY;OOL)!C1R;JdMv1cM_P@&_aN4;(d1{>*mn49}LOm%L89aJ~#tJl0>`8~2Mp=j(F4
zz9P&Z&2iht^hWUH2$MLqilX8xrdz{I9~}5u-QBxEndG-<y!t&MZeDweB5WpS()7B2
z^l0tnn{4`N@5fJ}wjHRfry)fR<2PK}9r+w%N{BJ8UPVd=85j;uo_ww`FpusAlNu>!
z5fHmWJ0v0V%%NyBvbxjJsLD0BZiPDW;Irb7s^P9nTQc(^(xG9Wf))-#2;Bwry&b4#
zeZomF(Tyh%{9tGk$@#jJFFt!pS?B5QzF5c6c7^_83<<K2YJTpDrsgYLa`OkzYl~pR
z%Qrq{_N?DN@T3jD#(Y};29KAQliKB=vr}=B422wfx1p4Koxt~YfOIadod*B*FUx9M
z&ROZ632cO_gANqnhovZyp`xiN>?_m%p3C*w<QK*{_JC$dQe4{<)P@(Bnrn}s)tLqh
z!&u?=(Ft^LfMwZl225?aq>LkLC)l*d-iq{H>qjTgq+o^aZEq_bVb)rfuNZXOPV2N9
zZ;Mj(o=t=$xBiarjbGVguU&3SI4xL+iaZN`cj4y7aP4V2odb`Bb<yj2-hUHZJ3wAn
zUtjV3s`3z@!%}A1lp{e{TR=1H2mA#sL1bPeBqV6C@Iyk|h8Hv=?495AX#{Puva$j^
z#UIMrz_Zgycbd6W9A4dMNC=S0jG11`*uCc3XFgHu3hw)+K*Pqv!E`INm)pH3vyu`4
zD{Nb|Qaymt<Go^JWE6s#d6U(qElF|;)yEFU)(ZaI=)Q=N*bCc}Ah~gDdBnXDTnp>=
zB!!#^Ijl`b3rR)qXjNpDVxY4T=0^D1tb?*!?Y=SCjzc%jglY$M<F5v-v{S=FaMLG8
zy(fY{nY|x{9=1R9{d|+VGTL7kbLRMDVBh`i=p=QKZ);~n?{r<Y#a`hEZ{Vk`Cd1j}
zh_ff#S*1919(<x1hSPdWvQ&-Rqn^qR^aDp88(J5hl1o9$7n58ht0>qnkU#7Y<g9o7
zRC?uM8*YY$Ky{kDV?!DeI$RX>WJpvTp>EW3oXyRb)8isNOQH@>=P+)Vh7M`7dn1$c
zaCJq8mhu<kFFyT&jx0VTO*xjK7%}<(f`24&d?_&R@uGPDe6*uhRW&vpaZz}&GOe9X
zQVLN{;HInbV9F?kT+fGrMvn|%LvI4JpUI5(X3(c<DFi%0BX;t2@aCk~on(pAw;vk^
z&Sn;~J0c;vVy_+gEN%2uASwI`$5I^wj!IYD%-6%wa`n&+B{_-vs}C<CQ2gllv+#wc
zbkNMwfJ$;%z!TqP%2iyH!YJu!`=1vK1ise(MP=i<yC~gyX&89rmVYSNJARca;r3&%
zkE<S9+Z!ZB3>tYi_=Y1G@7ZX?#Xiw9!XvQZ#lj3mufRW_`y59k-e6!6_}649XJ>Ii
z@g>`<=q5fILo}9}%<dj-5G~*wFK@~VzAZcLRK?obg4m&#^${<!TPD!e-izl6QxF4l
zo3qH2lkHcowqKVJs+tf_K1~@J{nRjsqu)=zWlh&%@pcrg#9p1q2IS2~CE^=NmKh!#
z6d=7<evC`cW+eQaI);cvcUj#?lvn4kqy24*d}h%UL;2q{O^5QoNqY3}$Y64B#YwM9
zjjnExUy$lUH|=^?=Qqc@tbj?rT1X-TarQyp-rw}y{=`7PW3=L@seSK=pC6zcUS65G
zF=wtH3=0nZgeq=Yrt{$D4T<M^-+%}@S?~g)GPnT!s7U5#M!&GWqI%*#$NOcJ$^b^V
z*X;i(c<OrqP1(K3@kZOqgC+Ha!v$NF@^lfqP6T3$3LW2S?pEt+#)ZA2nu|W$-=|5J
z@R?naNh5J=5Z?tQ?&owC9X^c#QRJV?y1oKJOZv*JqT6>mkt?KD%3k+*J*}+Whbw+w
zMhuhYLIkhs{-2nzuXk}HgOcW~dxODKB*8)N=uY3xKO2riSDG<~K6@uGNoljN#u9$<
z{*s4LNXqu#!%@!TO^ewj&!42uMN7y$86wWoyr!)VsZoxNI_T~RUC0sd4o}6VHI4VW
zWWRXY63T#V>JFaZ=v29|FDnlhGTs|sFtU3R((t-l%)L}{+hhX!tpjnozt|h^C4;kg
zrH&el3}5ysGH9iRmX@cs?*rVlH`2(9-6weKg-`kIGkmAWHTg@8hTYE=|J2y58`J&Q
zTBfKjRvYnOb#J(ZQ1wOY{CqjZ)k1A2!(>;Fd~mybqTkFKPZ)WbsHYVKn-Z6YPR3>S
zTl&sCQ*Iyz?R0*`BXC96%Ofj9LQKKK-mSea$km%dG`~ju-MZ<XC7z48=F7ZS%LT=8
z(wG~b1oq~GFt04<%ShY*4~62IV|vcoW!896Bfpf#O;YyG!!&i3IJ5T*_cA0b4n&A%
z==uT=HMDx1t^HY9l@U^S=%&Z-!p|NS^-yt2h~xB;6PIbMdTsnNi|0k+yeJz_j^{V%
zHpfq2h7<6oJ{&mS2ph6FTU5tzS~+D37W&jQGKSoyMC31Wasj=FC_GfDEaXewp9ml2
z?}~diii;CdHa@=d3st9|nEK}*d?M*y+Wfc1R46R~CY&VTz%|en>F1l)x2vLG70UzI
z1Xvz8(`nhxe+L|Z5$T=&g1c*){gPK;q4NR6EZX7VFSrEVDN|p0-|yhsltlR$SGeiR
zKIUFFj<VvpBPZV~tEbgOTcS2XLjC=&@l}Klp|PP5fO{IvUkY726)`x^_G?ZjKSHRb
zC0-1gyG{~Nswg;Z?0yLS#^2jTlEM&Vi88g5Yfk;jcY2giuc?XovDK1StKh}*q4S}w
zu`3&EA=ku$QLjP*;)63=N>b7<CP3TLcRzCC(4{t?jr*NRgCOSyu6>U;KK2QX-b$%B
z2s|A-<FVH!`So_@=5*&oO2QUCk|9q_NjF-xN=yC(VrOP^#M3Jw;h-E*BgU)sOHP3^
z+&gdU0jpA^Ia4MZ!Q-r&JTmLeGjbPj1DfQ4nhb&Cs$vw{fOLP0C6o9u8rm^cq*Vct
z5nFp1EBe8prl;4=PxNtOdwcn8!C0kn`z^+GS>=PG(GWt#)hBEdpLXdpbo0a}eWWz;
z*@kDUwlNmGx0LLHvIF3sP@06#V)Y$)c{v4ptnqxNCd-8=OA<|7xb5bAXG}1&d0>d*
zB`3B^&a4S#432sNn@)2*)SG=ylvmnZcBcBjEJ4bl=;F?cNBqn^Z}zIhyq$VMttL_~
zD@Nf#0ax+(O068itbR+kqFu&&u~=6QVD*9L`c&_Ojwa{_)?n4DYF9p!Oi=W;9_5S-
zh<Cwd4Cs4<mkGKx^Z6UHb0zZ@o%?g0yA(3o0sPqT!k+AeM0u#<=)zC@3idJRu%Tef
zh*24!9Mi&tn3#0O0nNShJ)YSODZLGv=zNW~mv>v|T@OwI9%F1e*!>inu7a!JD}nVy
z++N+^YNrY`@|Dz{OJl8S4s6@qbDBF<mzS4cojw6S29h-!*ZqSD!L#r1htgN&r|tvk
z;;1{X1)=vTaSiwM9^ov$@9y93J+>=N9bJ7m_HXY|7j=nhdtqEes0st-fKcs)gvz<a
z6Ac!KuVAaIpM)_RF?1NJ-Qy+&B&QvUSy*htEdi+G>+1_Bh@+t_H&hIKT<Sw8(^ZtI
znqOb9uQ_7E^#|?We1CQc1Dr2FC<YdI2V-DrBITgC-NrCSV$pgPx^GjGKN@ix=7DX?
zO5&K8FLER*4t;VKvX8<-?sZ>>E;fXs9)G!3V-#|LOoEuC2a|kL*%mXbZL@anF;str
zYKY_GWY{6#S$dzrmfqfZo;8OV2X8PRTt{d$phd<MVgYLirif}^Qg*!M4MF23!MQdc
zb&!QriaC$W%Il<GlSy-{aKCZ5fRJK9H_iR<G|5Vdh&ObMuaC5j0%oL`s=K$5r#ffb
zOAI=a!ChuefRrep&85kOWY@@-e&%uS3T;PM%KhH$m%p&-rllvvO=){o9M8$}ksUPh
zWwe|S2g9`D!iI+Y8h4iVPvu(eL~o>nQY%cbny05B2WI&&Px8cO{HB>l6%Z%!XrD=n
z9VbKvfV6qnDUS&U*r!4$1IauV>?dEx$7{RVYZ{&7R6B;z@i*zfYfTqI{IL0m2*`-+
znr9CW{GMzNBtKgx+F=^zQ64hSxKaQu*gPLT8(Kt$Q};w&$F{Sv;ZC`0epBK5$}T(}
z%*!I8qA<SBSwQn@zYwXXbc6i0xc(CjfRDhX{TX<6a&d3t=;Yds+1*wLe1#=e<}3ea
z)aX-CiN!lk3>|r2pjXomtLh1;&q^2k!I?c2J02IDOC7e+!NII$A6mksN|^emnrKU;
zLpZmDJL6n4jAwCnpSL$p!~EmK3J)Rl6-@6#|A^t=kIcWmTlU$|4dQlhO$w#WyWLep
zti?B1Dz~`d1;Hgt<NwV0<;S~sAKb(H-$JZl=FpfuZca5*gY@!F^V0Jes{)%L+(5#O
z@qZp`Ce!2H;OR9InadY12tR>7(Sz=Wn2@QIP#F+hdlb}ATxc6B9fV{<8s3DsA=jp`
zFyW(v5alRe-{O<a@Jk4;JLrJuX-MnB*2+}9c7J+IW>73n63JVx&_f<;%&S2ux!z*b
z83R_qL=X?fD|8qCgB2m<>C(l0c~@pv-rC7&zy}Fyx44o86vgx!!R3qhBxC@z>v;k8
z1TCpg6<~T`NyGf~W0UhijKe<7xY?hQOC{l_gZXW6@U=n?EMqZh6-Z*KH83ixE-ET&
zY!oMtBu}H?Tbs_PUkyriGi6-9X23C~$k{lzca-NJY;gYIG+O@yoW4WZ&>WQ$wjLa1
zjC>!sN!4^%yS^4oIMy7XaxddPu}}@*w!P5L5foeLGOQ1|yinM-!KV=8$$F#<fV5ea
z;bG90%nNNl8Yerweo;ciQ|wG=zKc*AXebW?S^Hu(<NRGL9Wt_YizJcjx|VMGu(0UR
zYetm^TC6Kwg7xqL@o@?68eSi8(eB~-8-N)Q`B%#O=s-$Sv(e{-mrm%p4f|M?J0EZK
zy&REX569J>(=qUE0@_(azy9yM8IF4EbND0qGmk)RL7uP=`Z_wTFudUiu>LzV*#NJ8
z^2#aSc3hCJu4b87(|6VOADsfvbFlO1Y`-iBd=6|n;Fzq(aiU$e;czx|2$IiS-MpT(
zLOljNfe+9Hl@&q2AN+wRV#I0n^_kBK^4)3UV;<xEU#DTsT6=#iqbNW)G>Mg!l$?nm
zbV2nDs5Dle$%m{}c;&c)OZkAzQs(G;VhCaX^jU=L=Z_Z=zuKI1!K_KG6!<>bT7k3d
zO90Up!_%Vo1Pc@++ACEpGJg?MRywoWW~^Uj!!D0CnxzOf_&FNrX<MS6@;hk|IDIp#
zTV15`_kF^Jn=*-n2*e*~u#@ky7O)ZHZBPw-f~b*5m+8mL{urB?aIgs-ni6`~F?HHH
z20ma!=F|$OU*zNG3fbphqW#3-8V1jiA{9T?no^06te&l6s#qZvSL=eo;~@n>c(+?}
zywrwvC2EI0ZWa-^@p0L~SQ>tF=6UsgW4Me^oj@A`9l{7lzZUz$AuA87emh-5nZGLQ
zj^M@euz=RVkpXB=I4nX~8b;hABMUQ*P?bjCHya<?Kq2Dqp}%-86AdEyH2i#QGdg_S
z+@ICpOL!%i{GYP<-$MF-?G5reAbAM>Y);{XwG0j`9+Dl60s=4LH!qic$b5fmjl>J+
z7w9`U4=0|KfxE5nUydS|b_Z|re#z`v@duNh%m4!w!)|jSADI^-Z~IDqR5&6dh0A5I
zt8I<deDJ}|2}jG{)A)#_76a*+o5V;{rLTHEYEopCSo0&&+Y6kXNquc*r`}`v9<w|<
zAZtV~(fWQ7d@K6JA`}kl<EQ!0(%D&jEWJ3<yA<*B%Hcek9I8PJOK!^&<whj3jvd^u
zn?zne)UV-1kz>!ga~nnZm4V$t&pR7!Iqv6tY0`&z;QlLi<fFRYJy46F`-464sJigs
zTk;lH-SN8yW1YNXk%6#Px|ZE<F>-7wKl+^z2r>-2rGaEj9pWVJE<g6<)NYeOQrXf^
z;}H!lq~fJXg?0$k(OY>Vf!vOE_Y+sf$KCaR0>WgKLX{gJGXs5jjcdvDU1t;w+d2Km
zf}w$Onhh?uQt+QMwIofO#&wW7weqxLjtj^%VOd<29>@9THuhY_F8^bguESBF*h;Qx
zXsT~ZM2{SLzB{>lIkai9{N}8^A3v9N$lRN?oMg|(YsC*lHbY1U44QSGtSODt$x!nN
zUkF3|8@&{k`G}9Pt5P~8$0>3>rQ0;gMW#j}E4a8R6!nF9GY64S&0FkHTf*m7fxG2;
zJ<HZs(;U~7qnlYGC`Pi<ug%GMnD+|gpP(lKgMPiHlpwE+ID@$h+fBIF5QSNh%BV5n
zlQ6_#Vzc@{mjP)=x|Lk{g>%`FDoeZDHgVH|@r+Y6g2+wE{Ls0%jZRh&bqWa#uv#jW
z(yM@*?v_S<%FZpmQ58?UNOp3@QkQOF*l)8iX~9;}->C94^4cs$uE7CM^~2kq<Gl(E
zFKUYnwd%@SFloa%4{Y3rdq3C&WvAPa`~z^sMiVJ{3vXg~V#jk_6b#LkdtrjqV)9+L
zp?0vmx%BqZHS8Zn3EnnXE_N9>TjmML&9V{IVccESVgfAgt!hQtKT8Ugdt9pv%eNW-
z0na&eCPtw+5V)G8R#>_Q)2mGB^Yn9PU4A@RUyS5*kF$;<nEQ+CVs}YK&H^X-np_fv
zTaPy$uXhERo4$bGhxK57?qDPp$HV@IEME^Z)zl^n4MXvV`~S^=o7ifKfG$@Dmz{?1
zn=Itqc7Rq2t%_*ku%Mbi1fR@h;b=ouPlO^_<@NH~>}zsypS}mPZE+XUv31k__OTU8
zOZX<s-CRLIxpUj@ABAQC%jM0C49j%P11*bs&HB_EMjr2IRXP%Wgd2OvAw}*X8=h!P
zti`gu=wfMME<elheiqM9Q|zZ}iBLlx=kgaWX|3=)`L&06{<J?x8K~?qyz%e;IQIIn
zx>hJf_w(aGvgGHKp553XW*vGZG^tZ3&J2nQ*HsmvSweDzcVF)<-WOW(jm1x6nu#pl
z6Kr&wh56OWruBI=pT}WAZJ(Ve&nMPIx5gxOhK*mBONE@1Q9WrZp6yxTqmGnBxkFVs
zMWSS(xgwKWPyTr)x&M+478Rq?NU6H<Nc5<7_CzTe`o<Ho<6^f*#$I=~rV*KIiwRGg
zhUWZgxp>^ipG)4YOByoqsArV`e(G9~_?!2lu)|WNIgj8}e#LkdKObaCIN4dx36h-4
zB95VfN*X1VFt^BS_l|O44Lb^NxTdPV%X$8BDVS|N_x^?a%LlzocNlIOxtm266ez@!
z89&R)<1ef$$#k)krMJL8pD`Wh0~xRTqQWAtHLYn`Ny5o)6V;~YcysrXm5N+232oki
z(_YC6d!$_-XKLY?Bb%LN?_j;nFM3ddfu_n@p7)ZF-?V^$)lPQyB<ip=D7@9J>dV~i
z7d(aK7m(c;&$*kKr-NVp_AQUF;wfY3d>9qA&CppcVS4}Htg70D4lM#RAkTR+dc4xy
zg$=F3=_1WdBW)8$#wZ<3P6JficZTnY$8&lz1;~xWxs}7^_6k*C6mvlr&u3!m`^#^!
z>2xb|e_jY1uk+Vq2aY9vzT4z1U24VLTS&%*s7w>>T2xaPcp8@}%T2>LlNl`R^8Lrx
zn(Xn~lFZm3+;cv{`?1&?NDe_=zpC5V;5%{$8y9P?@J4MU#a&%Ugvn>PqLmgy!|6Km
z^5V<3Z;H+Qfel?3F^ntTgs}Wub}QG=AB!K3>dy*F<lp@cxgkHk>X;y$&7RyP7Ph*{
zNywq00d;}Cm~EB=*nV_!uuAsoOY6*s2K(v>2oxb2t^;3{o5NR(8nz7BzXe>}DwYEH
zZrC+yG94k$C0YF0D|PY&>0er$2azdt^quKQ-P|b^psgkc{0Lt4gy6fT^_odI`-Jj&
zyMMVpu{u)ZS$25Y<<hywTqte^a&&4ovP9Ka>c@Z!i)1|}A)b@S&GI@g5q8LriHRNk
zdK;0nen?5+=YoNrlLfTP*Pto`4enK^gG<fUu`acM+Ar_)^ukUB^2nZE2)Un0Nnu_)
z51?@VW^Q1j!uZuSE5RFXY@E8!w<)eYIs`i&x>ZejC@PfdGde1WzN=&mkYHZURgyD#
z$;_H*{`;4m2kI4b*wA8nX1k<;k7B?nA!Ow=R0X5KXJ)bYR4<KVjJGLn0#Z0t723hv
zT;3~oOYD98H`40~?{R(iR{I(ahxp3IC|=%;D2A+guSO&P!Vz(N8i`be9GT3o1}8*y
zrJf!hp=~p;L#dy<FtT(RC>U4C0>}Iq2fWA$a`KFZ>MxUsO$ElGb8ss%{5On=5fjkf
zfksD=H%J|d%>V#7Xa<e?rj?C%tDSSjEe7Nd^D;!nIB}Ae?W^vrIdt#w`vZ7$z1c*A
zt@nQG*k3l)9R!p)D1dP0V)kzc!e^mk`ACqir)M>wAuJ@Q_LdeYNEKiqDHL`D{-@*a
zO^p}&mGzoOK!xj;&<-gj(^9@9She<8bt~&V%-1!oI7mrrva|wLt)Jo&{o*#{+RV_c
zjPI2AF<n4_&t>8m9QJALbmD0$>&Jmc0pl4Lt#;a?6v*vTK51opb@vwP@2`V}j6E;0
zf3U?*Kqqdz4DOwWO&{Xo;vM1;uR?G3bcGxUzBVv3RjpCdK@d4R5OB}my;NHb5=n?@
z{R3kyM;C`2WRE$}@3Cai=Xc!8<|Zbz#IMwbxkSw?TP61DYSKk~7TJX*6^VtE$OJLz
zAdv#58`k>neZLQq!n-+dnh!_w^q3s}*+P3?NFGAskLE7{D3_;%2i_V@JLfSZIc%~x
z{9H&lcz>8K?rZKIl*(<2JN5Wu5-b{#6Kd>`IovDF(}DZzGn4YHZ=pLW5A(X0n5)U!
zj*sN&MszQQyJ7U*TNSE0Y8?SRrO9#}Zze8YibrX!43s+g4}QXoilAsJx7`_=S}vCe
zfn1@Z$>k~mJ_+~Iv#e4Yv9akyN?#9u*05{m>ZT%%=dZN~>S#1C@fY+!zWd;=v<63j
z*^{{fx0CX*6#K~*veqTjQu1|l7kDT;Kd;a9`A^WVt%Hbnbkz^$idSH_86UJ3&3S1{
zPmud9<GPzgNMDuj$Pyrk6eZ>X=M#3Tda+o)l4fF4_uzn0@^=^cxsu^3Vs6XaDLE()
zqTj#)Su}G!cnSD7WwkBoA$g)~W{gX?gdNt)r3KeoBm=2LMw!UoPy^vradK3ueWx?9
zmmC5g&qgYaChhW$t$jTFm#@m7eC{R?DZh$9BRU$R-xQsc-|{3`X`(aZeiUjB)Qr^>
z+vq_8OXcThQ;z5s)Axy)WABqD>dura{B<jv`1aT5c!Zb4{p7x%vrt+(k9-V`BafYP
z(JtJL+aJU`TXq!04-EpT4{696XoL5A3P-Nkl11p=?5y%jh|Bcx5~I%BYSS*MsHhlb
zU;#3@Jy=#RsWv<_C2_{U8j$$!2?HSd*mORjpWvp|fA)?2J%*FWyr2&v303ahxQHmk
z1A>XG8PM2?4T@xuw=2uuxiUb=xOUuq_7g7&dc#|zb0S6>efQ*U#Xg!1Z3R^qDZTS)
z>k2TDwf(KL8yX8@Q9Qdt4NJk3!PUl^u{5gUvOi+?O`7g`G_M+wP!ED6W+w(5%{8)9
zLh~e;E1fkkH-x`AwMzP{?SsVxzvnagF_%3AoB=AZR#Xht<kZ)T#w(C5D<BOXDBp57
z%ph60{%TsKHbH(=T2-^smf``MxysLp!9+@m2YJ!$oJwR7kg14P&-5Fg!65qOUXU6w
zgWwA3M8Ein&@3ohVw&o=j{@e$%FNg+Y(-kMgx{M3n-3Dw)2YkG^ecM>k(`$<U8+@k
zRmo5Pp$AA7JK5}w&-@GrE`qJYScrTxaO_oQfg2=jE#aYREW8r*8H(Q&&v55Wvc9jb
zPJFCiITsdx{OckDseb-ba9Xl{ElO*kLAr?v*dOYm$VblZel6T@cIoAVAr3VFg#=6=
zRpK$enjU}(UQto_ZbQmHq_lVsOh;0h6>!cSjQnE$LY815a8uUs!LgBfu^uD$W_M{}
z$A>=lg|HpqK_Pckq0Y)vtWK^(F&<QZEb?u>9wfm6l_gP=ds}jKG$}wCffxye=n7R0
z>LbK0c&bQ3&8h;TH|oG0q!(k`wIWwAhm`mf+U|ye#(nYkZ;L1kJv~TTP#}_cpTs(w
zj0EPQ_I3C0xIN^?n)Mu!J_tVk^Up38Tg_prq%CJ`;<MTdj5nNQl^&_ccas+?+`f)=
zHE7eX{^R=lvE`|ckHdr&eo6JE)@Df$@kQC*%*8z&QX5a10lj;WG6u?`+4(x;<AZD<
z<6`N7klVDt1H(d~Y>Ag~G~as#rE8K<SzS#d<ovFpf|oopC9H`$+A83L7L_JQuvi4e
zHUv8biTE|*Z)DS>58$x?wuj%ivK1IYNIT0i`On#PTDq>mV(`-qDjWCSe->H3YkF2>
zMH9Hg#!gZA3~*qMw-gvkzD-lslP1H`m~_Y!d@FS};vv31vG)VYl*mg*wS|oDlx1&%
z;ew<7o@x*Dh=jBe*n?NMm4hoTt2c8S5D$g~U>V+B?J7fI4D9UT+PH}Cv|f6fLc|~#
z?d)_=EXLsF`ebhIPgj)X0x0@PU$b2!p^P#}nKD{kT(og=+B@(AG_$^bq9(f7v8&Jp
zQsp4CAzmSwLQ3xu23@_P<rF~W#{q$(k(sKHMEU*^^VYkm#V6$Trcr8u+|A*lcN3p3
z<{DVK_V{<z<?NsENsa-o9&z&E_H~qa`(JrnpmNPUXoG?8l&o)Z?}^Ta{9DNIB;CP3
zcntx;&VPW4r#}d(2b+$ZF3k$^xC=}G7czGwLJc2nb+p8HTjGhL&FR8b;x}(`&NcR8
zUHTgS!@MD(v$egWKG|&paAi6k9Nl|Gtn6JswhR@yHeI3B7-bR0#b>RNw_y|Y++i##
zEbaA1(wAc>#WH=e>goDPaz#Q(Nhz*614T1E?@IQCWI*!F&(~MfsdaoTkrIc%My_3T
zZeb*xp~}g>Fa$Q8qOJgyBDJ#4FIrFiqd`uy%$x_N{zxU`HS6HQ&z1NAr|AVFeX9yA
zv@UGSQ(0L$83Bd;jm0%#d}X(_?;`Tc#m^?Hea-Ku!AhbUgj?Z1q~=wRdT~-*;+nIq
z0ZGYT{wt`9*p@2D@(m@_pPXusol{##-sZ6Y*KoIXtZDOLMZE*VMSNb;f9Y54DQhy4
zTyw?8nwR<CT0uS4uMDHM2L_Hc7|kvg?6re*x=nBwe?TAJ2#1G_RtvC%6aLOB4cv=-
z7J0Tg5Z&w}Aof012uWTxHZd{rcETvLq-14h%vjJ01I8A6AG#J<OsQ**z%KV%DuL=x
zp8)BfUk#dG3tY@ebPW{I)U2<ovuu9XLdn-Pexu=%6&*U^9Zu^p+kCv`IZYbFOG|!p
z9<m*@C7S(#lXukb1W5pUY`i2+r!NV96Q1~YF!z*7h$PM=5g_Q9&spr|?y^Oh7p*I#
z4<n+sac%B68Ah5x90P0~sF^388B|cRCE=vrfFrt7{oRZVz};WW%JJTJ&Tc^7=h>G+
zfPT$oF(B!3FIl?*wG{#@x13686r(rF(yGZZ3WA69hCTJWxc?O4E%3N$+A$()GDp;!
z!QT5BZ_K~!Tvz@5yX4K%SpE`twk3Fxep48#I{zKo9dUyn3hv&)^IkZ)JZGcD>4(EQ
zU-rhw{|WZXe}~H5+u#?4tl!O>LSXfPGIYHW239aY{sheX)v%zi7oJ|@*#57mSlcnp
zonb2ku5?Ah!6)=O__OA&p(NM-^MKVSO;)!#!jmXXi6j@IXd=FTE-A#tdU|Ew`})`p
zbK8}J8Pz$fSKek<Si$*0`<g~Q;(<)zcAd1WS;mTtkv*6dquuSFOMYyn)m`Q)lN14y
zg~B<IxV-8qggx*vg~vFlssoz90W_0DBe7p6$-@&s|0Z4tm~)cT4)C;qot(#^TCx|Y
zwr!rC(aHYNP4(0}jm=<Ky^P_Fi@3W*VZ60@&4<rw#x=OWX#)*wF^|KV@V!bPfY8PD
z(|Nj~s_3SBXd)hM9+l*a?#1y<jfMODA}MmKzqB;TL{v%eH1e9}l5k0W*QImnT5AyL
z)@UeY6DQwC351&8qc~ULY{l(#b{*~y@Mt@E$+BxiO7n7&RC78xYPtc<ZQt_Q?)V^3
zTw0_LjQ!b@DIXZKuTQW9$s}_sQQ*zl;p!^Q0$K`_^Ue~&|5*2a-NyvNb4&pvSjnPY
zUt!ScuIVAdKF&q9Z^|c23VELX3PG^ZT?#qJsXE*nrHapWnxJwh&X?ObSth#b*;!rV
zI>_jAFBu_0nvqIL#r##rD1Cvg>9#Z@KFLlT<(BzP4J+O3LRNdlHBTX}?+JC9kOtY2
zlG5_2*=#mh=^QMCCj^jT-U<7g#h>IO99to`GVVWo^<vrQ8G{XWu`Sm5y_=+5jj>~7
zEt=Q;`kO{8-44AQqu(4Gi7VLk12z%Zq%=q&o?W3}>N3ulf%G3H)jV3IsYhBr>;cxv
z8O0+Tc>sdW-1h1gh3vloory?<=+8aQ8)1*!FlIM6(Qs{?5=}0ZG+7W6UHb`s7@=9D
zt+7o7_mxXqDG>y7YtIRPSDHV<-^U4aY2_oa$?0PZB<UeNrs$Sw+F^^_vA_iE3=LDV
zcDvk%oPg52&6_s8lhRt&B*M$0)9(~VgGs!x#~LolVCrJf1;+&4QlL}CYnA}KhcJs%
zm~ncp4LqtWDcLuZQl>e5C@oFlw*0@&eSh+A9SO}%JOhV?qB1U#?mwfqD(2}edKl+F
zIwt0!XR6N<9AowLi@9_#Vb))r%Q4OPGMKY{%&bp>35Tj4auumoJ`B-<9S`SeuZp#@
zGo#UR+q45cdi?3!az5qwt)!@UBH5%pRZO}8YL%b+GbYi48zI?FoA%9=`Mtd^hRvzn
zm~`QspSmNjn-4Pt;sm=3C#sduiryvuDVr^RyZawY&m}2m4R^u4SHEs;%uzA+p2<Rm
zhtsuPsO?6-`f9Dqc>Zfb8PZE#-9^Z$q{kh|EA(stwk9S<3I=J+8HD=D?)fqB-t^J#
zt^SnyLd|O+5<eC)x3)HK{Rih`v(xk94!H_R9*UOl6UpV0<TE?&`lxBS+>ONwxm`40
zF-L(ndd&2VNsW1(C%N03&yu$rCzmp-w^#0nz%4;zF|lp3^6sd*($Nsv$5vd5T%dH(
zYhERyVD?^AT6>9|QBUm8R3e<D&N@`D9F(ejJoTuu-|LU4#X6luJR&Yn0YA$7xssMS
z*O5&WTRl@YUSVvH?jSA@Vt_kIt;8&RR8*!xX3OiAbA;(@@|)7_5jEEO1ootw^<&Iw
zG6_m`D`4O4Z@8&uktSi0V)8^FK34g`nwwj$RKb4|9bCH#FB}Bw-Zmaw*Y8gilzR{9
z$lm_H1&bD?h_fl~jU$KTa;YW~d;S=3<sAGRY}ILbv>P3FpjI#?W&Zxwvceoiya)^j
z&2m)mW$`AaBz~Lxmh{XiA|q1$H=oO0V)|)r>_-tIaXfWiU6@F*n=hn*Pl8#HitiCA
zVq2!|(TV<QLbO7$g6MlJbL-`g7X<AkihAVyR2C{G+**q9l~|Q(%5Zk{6}&MLI`SVe
zUgI@bc~n@>|Afm^6R9Wqm^fij$2!8=rl$yyaW-AKcAbGiuXu~+QtOw`7c*kq(ydZa
z*0Dl%{3p+H$7DcQZeWU_?BqoUMpsdbaIqpojV#?<VHTdKGy762*iDJP7LMLdnMZQ8
zi_A}!gt%axQroeL%LtREf4NKBQz@e6HxCq1j{;>w!kG(|4bum4w={^DJ+ETN0xMGy
zgr-5r1=G9RUm1<K_E<kgDcFk5Xsxhrmbt+N3MtW-fY%448JfjGzr+5okN$_?{QsZ)
l?{x3~pN;=_X9IbLF<MsdJ?%RQVJ-sk;NHW#r7|YZ{})2eY4895

literal 34433
zcmb?@1yq$?w=RMrh=fwoDWH_njglfI(v5U?H&W7FQX<{C=@4l)A>G}v0qN$h&G(;k
z{(H_C_l|MLeFttC8}@qVJJ*b7&Sx%yKFNrqKOuaAfPjE5@j>J>0s>MW0s>+f3L<#N
zmD@oC{D*4&LDdcc0aFM5?}4kcXcz(lIf8`9dj;p|-8nCn<TH=`y9fj{AFO3$Jzhui
z&<{tMr&3B$&ZjjLv?Zf4mJY7biqsQv-29p{9m6k8CfM<N!z=e4*yq2NIK$%WVdS(s
zn57M$?Y}c*@b=dzr6f7pz<yLNPD^i(|DJ3OZ&g8|cu?WJPeyiiYC++LK;h>saC`7_
zuKuLX%7gkql5GBLBQrTA1qaBAFiac41pgmp3jRt#1YXGXk^b}5&m4T~9f|&*uUHii
z!1qEN;N_p!ciqU~+kgA`zb|tCafE+c<e!iK+amvb{C_O+|1N_cmJp8qp-$9?RbNbj
zA0y=|!NI}frP{KB;5kyBOi@u$(aNS}f%jj(ehrtJd_t(Y%3`KEIQU6DDS_kp!51Fz
zg!;(r|6L*fzn?`%=1B$2Z}-fd7G^=}yi>krO%4>Pk1F<G8vb{+{F{dVLpj1IH8aH{
zTfI=|7-Mm9F;A*Y?nqAqPwD{7vCyqI73SEmu&<o{v!Ng-M{JMyY__(|#HDHe@X$5^
z`qIg|PJ^AD{jhpsXKc&<@ZDk2_V)I3j%<F*-@*rCNC@6VpWop#Job6_YyV800gu2U
zW_N0M_&Lf$HFfo!?QNM<o@~0{**$A6bB^S}jToD{{QOrgRNEGXW2O+bvF^PS*BehS
za;#4{qLvDSALVe;2NTL(xy-&J|1&$Aa)8y?0XM5KDS4cZof9B9feNlD>O(}-v)C&x
zVhnEQq5!qRvDakJg=#gJa!2yl_qPb;R2lHt`JqGt>4S!q7dM{lWLNG2xg%SS4NY35
z{f3g03T9@7t^ziJK#Se4V?!9wY_ZfdNQTFACjA)jLcniutrydrj7C@j0nLm1X9dz_
zG4u+Jvcp#7TWF=ufJAVSM0Fr)2D<Tz1T517=QtLs)f`@E6wd`>sVktl-3l;kF8%26
zLY*l8m#D(kgBwTAd>DbgHO~<cuqvWNDb$sOfKY{F-?N5=+2^{;-7OE_8=ORoBy|`_
ze*<a~SB|MtWBQpEcWFRgl3!oXBPv(rF!?W$NA;P}Dg@ehPI|c-{shX8lo97C;fQLj
z=Z*WF-*^(_j76EdC~+kBui55P%~rf>Szs<~gDjvgV*q=7*WF1OxzeRoS_AeWF4&a&
z!H3_l<!$Fv`y8z}y1-!%0d0Bt7~-l^XTIgj_j}{xN_Ca4VL9fnu~2H~EF-#D=N?6Y
z=bs9&javI{SlNS<y!@fsRZ*81*63Cld`=oci-ppi32mr&mUNz+oUE<}zjCy$^5}xw
zmLfBV?^Va33sl!BolUMDZ%3Wtr>`WTX;7|$=;Pg?8zrtRE2h!3J>;VJ+l-6^L!2Co
z+z41%_E&I2A)0l9HXE>e8lhrxIIv*WiWK47LdJ;kc$d5kjX6qAH3mEf1a;}_<!x){
zSxDk`qLW5<JTA@lLn?X9R`sXw@YBu4ZMP80s?W*6(W@!|gsak<Cyfxtb7te@G;6Dw
zWwNS)K=Q-#c&i`~U`PaC$1=@Wo#2ag)?$YgzDROKnb$=?h~&4AiHRZLo&5Fd*HT+x
zjm3<ToZQ<vfm65UnA4Sixhb_1h!Pa-)fO|GAQ6=P$?N%VArS7M!-Fe-+1S`N?SXYG
zD=X7(l*Zam6c!fR)oDzdIIcQ2%%xf1eg95aGsEw2$94G4v4Q`h9Ow$VR*;`Rv}M1x
zwq_H7i14{}ml7ee&aZg><ig#t;pUeQ66(fv6!eE{(>&5+j4ZC2Cg|FsGhb?IY9)Vp
zepkf|Z2PI8ZI5F5pwr^72lFBbA4sUC5F$}6Yt&IP=sv2et4m6T%;u&3)kOIMQNtwW
z`axp_oYNhh#=(o*ztM62vX&&PHt7&sU}EKOw;#-&XN>i)Ym+?f_(KHh)(;#A`d^>s
zOG`_$j9%W{=I=1Q3Z@h;?$FK8&yU&~-?EpP$f<A9=#Y>~($kuwpg2*P-nhK+yvVzz
z0e%IALM=GdXJbzhykFu~uyP>T?~$Oq<B-RBiG`B)thx?<xzwkQoy2`+2vCyzIC?72
zIahw|+0vmahx0N9`U>IQzz>aw2rTb0h-#{<O-r3pWN4V#LV4qP4vRIFae*5kdz+7r
zj#66z)we6k$Y$?FW=&FcAi$bT>laC#7(h@#Q)|>JowGF3dWMilWLoin9J@%#Oj1%(
zPfw2=E7G@JKBO1~6q#K4{nx>iM|&wf+)}HseAXl>r=s|HJm7V?#8L7;vRK+s5#EJF
znIm*#fdDl?KRh{^=+37u^Yka46f>Qu91kz=nBJ@4pE9D9dORyr_o5XIOo4r})u}Bn
ze_JA;o@%o{oGl67IZE$uGpv->Vuu{Af(^E1^DqsFFIN}XKDwx4bw`3{*GLEb==hoy
z<jS>BH;!aR)-?i-WNAiK2`&mJu%Bw_{e1u2Vt4uMYR<9-y7t+wTB@dJ;(L$Oa)OUm
z{~ywbE|3;za|T!wd$Z{7led0&*NRP@2C$5<f-S~sr5*nDKSjiaO3Ixk=H~Ww8rxWI
zhQYrdz|(&syTdX@0(9-KGl>6duXm18_(W1OWK&0YN<!6n{TJ+Cl5NYcZbH<^C|jQA
zjZGe<NMRww6KT>U91^~G!R3B_0K5fc+)Y>gN#IxhSTofYne=6Js~;R^#T?Z5xs%iS
z4dtPb6)#oo7GK9xehZDg-QCeWGI0U;(>!pvxDv!diA_j=I?s)dub*9k9F$Y?iFGdM
z0m5@#Y9e3~C7DctVQX2VK%ZNLq}3`6{r&xR(AMCe-IdX3<0=gC0|PRFZ4_pZVx^rF
zb!6|g&?s{T{7~Sk-l1Q>3jYCori#gr0%N@(k69vkUV@0=OVN;*x4E8dmGcjn*n9}y
zd_d52@C<xJ1>=7>zlze6O@Icy*JwDn3F_Tdn+i8ij#2NCRplEN0n%?`!sRM-cxal<
z#L0&9Xu_VJa-PlkKYl5E`muQ;dU0Vtm-4m&1%!0VCmp&QVq%w_onl>G8wd6T5KbKY
zm}*owFo9~C+g4cFA1iS2)tINgv9O|J(t_Gz!!e5Y8bPVs?<Ll#59SXL(<lU6I*qAK
zvEiU1KB~FAe8Ep>i?B)NIfOGI-Zq08E*E)M2Muk+W8WJ+Jp$h4faT@y(b0&h;RCe#
zNbfj!Mu`Rvo^i&P<gvVc_-Hl!9oyyJq7K7K&_HCmmz!!*zIXmMu+KNp#u@D3)2Etg
z^&1WN;=Xz#x9_W0G`Nf|^?gk`I=8T0EV47C(sc%WE*&~+84dg|AoS?&o!(Wv*g@Ac
zIBM<KlL~4d<2WdY-tFz6ippw-b+nfMEeXjSdAR~!TQ|A*t5vFt8kv45$7v<3L*Cx(
zy^137w%0Y@obB%ZnI6z#T=eSI5)6c1mFRk=aE$OUZPYHkmFYixFGvJ)P)E8I4R0Vz
z)(%)r+omUyu#dv36!I5e{hGMO@o+6(Gp<Qf=p%KD0mwk&8Q(#aD?G%S0ky~YtlTnA
zSAr3iTpFMMh8W}dsnE<P4fl|qc6H%y<g#;@`b<u+{{A-EO9?8yiy4hXud9!~#CJ3<
zdO<wwe7l%ND-g0#BcOf#>BUKBajWw`b=Jyv#!Q;{-?A-zO+;b6KN}_p%#;bcXH}jL
zgoUQOAnctUDU`>_`9lNb5vjgyz48@w7JS#wR*|ln?v>tg++F;{gh<s)-~BoP38|a`
z!|D;woMv2HU%E7q$QwNRv=+<|8Iytg#4K$7s&MPEeug&2Q$O8u2Em@*-@j*|z0Q?~
zz(UbhKUpREg=K#WrqRy@)^bW~P9btPsiKWMkwy))=$KYij3s3LruYYancr@aAt8%?
zX4K!m@@;LBYv!^Keaj8B>03W0BH)ehlh$SLEN|}IW(YTZK}Ly6U0sVvFHhXlGAd%(
zI$7Y6NhhB<Z(DkaqplYd!@qREqMZ>CiLTQ+R>C$~!p2AChxK!3*?6KO3T$-WY@leB
zL3T!OR*Oe=hGTX{_3$SHU6zOX5O8nvWL!5fBg&}PnKs<o4<x|zqWGvch%&8B4iVug
z<Nx^5YM1p^Dq-tr?KhjP<!+d=^0$Zak-I6lyxcGyM{>f{dSHvev*C-tPeRBVI>sJd
zqz#!TppBzY=-<Xe$5HU)0t4#*)NAO987rw85elq^POj=NY(`&aJJCGn#txz`wUs+|
zHRuSoZ4x*$oPcl9XGNNuP!ox8cc{obt9|z2YSlVNko&9XouAql7)mB|BOBUNo+mNC
zjM?Jjnums7x<32zI+x^|7}8LlATiUQEhU1dM8P4_kGP{{fM>_Qk-n8Nnfx4|ljj5(
z(r`UnHnwJH{_MZQE??7HrMnV)UWndnV{_4qL5~i6=$JI}Ptpf{?#8~!)z!MFC_5^u
zU@dJuw)J1XfG7Q8d9I9+rpqS#wxSC)Jj|)z6HBu=ST(VfGM@#1U3RN*=S}=meFj&i
zpt|xAI;=29vCLpCuaKUqEO7{sC9<TObYB3Cl?f>^;iXzwYJA=e)aOCSP_6yO1ASEo
z5NQ}3bd8OBq~{Z~a=w0@uBiBVovkd_85{uul<I4HmpWz|tl$n$?l)yPW`_6m%M{Vu
zEkB^rjciHD68eAMqFW+lS~b6~D9fbmfY=})5y{cBup71Ta@8utuLH*fbx|f18{1we
zW$5qUl~!`_Y__>~<2F1G+%0`Z{dLP(qq-Y>eOriM90aOba`@bqm%l83LK89sBw?rk
zQ4hQUl>Y)PF>7)3;9&aaPY~bWRjs!()yfJ8vI8QN`a7yY(lTj;q|F`rl<-?F);q&6
z0~0Px!;freo=<(!Na?44{~R7jS+++H-egXg^h{0hCHDKnpO=6VAN4G-je7S2df0;W
zCK?@qfSrqD-ob`VQYNS9O^ai;59g!qP#~w35XB{hrfy6OxGji1&j13EJn{RP8_3jU
zvAZe##%26Iey*Sragzk6p=u6{F%@W=rR-%)e-us<Xl58rp=(}f8t4v0hjL)t(%}3=
z<b{D80hC#`SmoQ^y7M@6<+n{2w_$o}Dm6dH#=xwOTKQ0x^1f1>Rs&JOki>?{HmaLR
z@#AK{hd+mk_&J3*ykEW}G1)^EbqioZqma>VdatWhKj6ot&3Wo&=~$;BJ%6HX71{Cr
zw^qO$(Kj*NCIPfTX-<hbe(zoDJWd9`&3Z0EVj^W$wg-fj*~m6Ybs(Pg`Ckx8Dk~%%
z+hcr~tlPkgF%h<T3-;DeYM5Vm2snC0a3*m2H#{;m<g6DBa4#R8xQ_?1xa%W~DqN{P
z?>Ht{R|6oTqRAg6;jWRN_UZVO)%ho@S`ci?i;~ol-s~KKTfyUuwo%sf!>yC#Pn%NI
zLV^rzqNSzsIN-t7*4jSocB~{c-+7V;=FYuino`xgBna7xQpc@FJR0TX7-#4DAgJ%1
z|6>vJz8?@DCET=Rb-mGXH`10>rlLz)T6})lW%YyXjZt||Px{$e<*Zcaq;l!N?@HYc
zaRf9F_>8$Y-V5sV{Kfn%65_&5`ZT(%6~_|+uQvvQTl3m~PFT)V6a>(E$gWj&aRyk&
z$GFPNJk%5PNq!+-+FxyLrd#dRbGhbmUe|klQx%C!tuA8ghLY_UGBMKkO?d4%9}jYl
zFeVb=$vg(xF`&C{_&zcp`~B3s$+SIePWu#_Bo^snEB1`$uuX`IljV2OB!OY~;BdM&
z&_HMb%T}S(<rTxkI$K;JcW*ZF8X=qSkj-z-VceXKF-4Qr-;sSIf&S=Inb&*0Rg}OH
zse|T(sJ{u=qJOAU3;V;6?a3pO$_awBC(Hq@AQ`PR%XHkS{U=I?E`f^cdW_jPU%lIz
zZ-n4&54)KkR)e;Cx#xzALj%@wHox0)+F>Q8U@UK;fRgsY?D%P>$JJ=6RbIhmn`Q&4
z`?bvpojj2;W`V`V+0G6&X;dW0VUbOv@5N!=HRFq8*NlFfb%d?{Shf@rF3BRzB6gKn
zKY^*_v>YzvYLIB6$T_l0AWa#{#iwmK-#8@sj*XhR-nvk<F9@C}+zh0W6_zr;qHj2C
zgV=3!ZSSnX*cBDG<(sdskA%L(bcHun84VsR{*c;6C6b*C&o45sr!ownV3yqIr2_fy
zckh!UcCQ=R{HH&Fy)8~cLYRHOCwjiD74K@eT)A76d;|?0oetgMz*}DsFwna>c|;T|
zLS=Mp(5RHvZoX%1MTUVa*=Ja{Mx|cut`Ef{a1s2heihi@n9tiar`QwbMdydLyl}d&
zEPOiJnQLYQZZ1|QAbDpl3i4W`6#CZ@^q5>aCWtIMdzX>bCz&siiBeCgWhV9XlG2G9
zZkw-e2XAjkN?X$6jm=n+1ASW`nYu60fy|~tlkZXBvhIv9#f&sIDL?l~@ud9nGW_)P
zVq<ZEly`h+zJw7&LT?2vx1#Kg*<cth85|qxcjF0NFO_KC)np3uBJ!suobFz%erHeD
zs9fJqcm3F)X?cAl>sirmetS6I#L6L{$s3}ph!<!M+EonXO2-K4>I~Q+?EKY@$bZ2@
zrQ67!eC>!B9XGA<y13XFJUhNNUT79~+7G91k(oTVhcO5RkAd7*amHc{5B<Ly8OGmK
zWBqJ5FIlKN66EDEoMr9tk7#RMj5vku7+WdlqDu^)|J0aE{PWs3<6SOvTK3TzC5XF6
zEGz=e-^3F{NseT<fi{}4!kCFe-WE>8-^Xx<rN|rb@U<)v+<7hg%}Y9EnN1!PgxV-(
zA;A@n1cX7h0L6P}OUu+GxTh~TI^<u)#>RCNv&gsD83j$HIG-f#5XsKV1RNI1B&@Yf
z-s9^FwW2~&MXxHeiBO7$%qC5eyylF8*IyE4RNg+HI1h#Czmt}bh{x1Jqp*)5NarE}
z`7FU688O5ewql#PdEyWpeX}7U&Efqh>t7k{L6)SCjFX${R5X|3eIAS#`&~lMJh*Oh
zb)O<l^j<Gg)=1TM=&K270uMnzNKD9PsjklgxK&eKp(<7Yx6ugbqI-S+O-(}FE7zOn
zycYA^2;JTw&id-DOt}XDunpisqx>)8*qxJG>qJi#6p*r|4UaxEbc6g#e%sl#koKv{
zXw!DqG8fxIK^;Y`D?wP>LKOfQT8w3<QdXL$d+R&CymF-6J&}XCuJ6t1m`Yx~QvEr`
z1(d?AYi-`Rvz*sRGCGSVM7-T(>6gDkj3i@cFEvtVvp!yE=-~<b=<8ZjVBaA&64!zP
zq6oi&lJlJ9yXsfHgUo-nbc^_-l_X10k^wN>ih89=lsY!sNG%p+AO>PPj!>ciF-mcM
zq|rPp4(z)?HPhqP&2xvX>)Ae8Iq{Kx|BISZHsh7^&S6|d8h$}$6`WwL1wjrm!K|}F
zXtaPA2!loBT^VC97+H_EJ{?PK9u9lR0e#5Qt)RDDSzI(2Wp>r;@q0{*pQsm{K*pAm
zIU5}}y0kV4H&M$qG(J{kZo%e+CBD`Y<114+|Ne?cuBn8J_q<{RTBX|o){j?wT!w;i
z5@3)eFiJ!k8cqhS8=9^7ech5ud_H<N@$9N2h`?!pl}OSNc3@e0RftLRZ5r*PQf~Kw
zX2b9&#e&=?kG6mv--d+XZw1=F`m?pC3JC`AgcW2~kQdzZIA<Mf^d&H5a&_JQuAcGE
z;|O{fDGm0r67|dOUQfdunC;OIe0s7jA)V77+vd!|PIf{yVuLQu*M7aD!^<Qa78=^w
z!SM$x26?;);=l8{(mujmb1ze3#|=mwY+-wrvI2<<xa@zNemR>^|5a6BLAzwSV)#Jc
zqJygDwf&>@MNne}WdPj5$RMUCEdj4)pF7V99{1}Z*|54rmKkL&eD?TD)}ni}#VsHd
zvUXJSiV*}F%*0}VO9!v6lJuw<Z-4yM&1Y5`tk?#*aqG{T0lPyd{*lm;?4;1Q&GNlC
zuhWwR0UIi^jE=+gDbR$s`}AYPf{rPNY<M_ElK%^+p%{M9n$h1*ggdCmcljG6qtV<R
zHduhqn-1I_-+8X|Og|A2B`_S{*ax|`|8c?AlRoLVp;{4TnV#2$PwNiI9)pO1ry)ys
z`CTw}Mw$O|;MQpCZ2p5)^2@v7<Huw#60N9_M$O3fD`SU=5_3VJ!%!|D%l(O72qYyF
zTY=s1`7(`cK2AL9JM*PUyM`8lxy;_Zz-U4KvW+#r8T6B6uTw}DsmBqL0LK;tp!G}%
zuQMJ!#jW7!IMKf4z~f+P8B)?0FGx0~*SJ|$Hx?ka1{jT4z&Jac@;HfWp7swvS-5jq
z7W}BSmAE%lfa#TZy~S@vnzolQW+8hwZ^G|8Wyk6uK^X5RVt5=D=75edjA>Unsw}Tu
zzD(uaL7T7~<CX6vR_SqIdKgbXfkH9g<VxlC**Q)^`!dH$_mbpiSN)0ly2=WXrQl)G
z;57xS5pB;OAryj4e=HCn)_=RVE{Cg_7d}Coz{5=?BFLaOghr;P6QQaDJgburFUv7C
z^N*A&YdONvDDdp%dhjX9+_(2nX{X+|Wb>3V2pkbOSu##Rgi)UDZ0*cl;*aoi4w^r`
z?m}}Ayqsfk;oO<f+R2;lr6*8JPDA~p@*1Gv)C8t-j(+pCzBko6)I`5ND;o?KK0<ia
z$!x)tz>_1jUfjP~Fp{x3ipF`D?sK~u7(%*}Mwm*k?f!WlW6w<Xa5hNieB5pW`sAq6
z-Gb19tNC<XgG6WCGN^Ez{4JF-?xuZm-}-=LLLvvR0|<F367V1n<xdzC+}1QUPKS}#
zT?D1N7-nKoj*oCQ-Yq^QXexBu?-Cqmt@1XQQp{I;js4AMN~{yq%{)jC>M2FRaw(TT
zJ9wWUB*hwW^a?$DJMyNQ?HOpgni1R}Gl&Ljwd7aDLdNC4yd|{LnQq?koH@!t{N*0%
zNSr=)yZz==<PEXQ4GIa$YV|#~P4e<U27-Z}3$dFVkga=hU!(uc>A|h<mKw$Lckvs$
zJfdG-c5mTq+B>YRai625*We+n>nV*KnFN;mE|}+@c^km}yBCUIPOK|z^OsCPLI2C2
z1XNzy79YU0>3Ft?K|ePUM}9Ha;bz6f8I4OY@;nViOf9J{+Qp0MQTP<;uZ(BOl5AIm
z1EMd&mR#cDhtt`B@{PtwbZ>Kc;DxUliy7%zS2v8-zs33cJ1Huayk>}XQpUwIl=Z3I
zML(IfJ%FuaV<H@tQ$n!#Holk(-x%@`biV|a1U<P-vp$!iuCT5&rg!xS8_L^UG1|~j
zSI}20*Bzogh*zYn;|Tys&6|zxBjdwCix&;f8%nM(x?9h)N5*1f6$rx6o<VS%Jd0Z+
zvzSEjN>Wge@y*_URIQDkK~?|6goto-!>(q?a1rBXIB8VUHYxg452=n6vR_a%tDwen
z*RJ|wq?qybrEhRBk7!c)p<DTrcK;N!mlIuOFXyNWJrI9ggN<qVbD(h;SzKDoRIrd@
z#Gy5P<J(gXeRY$d3Nf9Fvg+HG3h)}0D{CGuyWyC}(di)|Aj1^N^w87uF=jg{w~&AD
z^K*jU^=39W`Kq&1=Am!WE0QS5{tfGA@N#yzbm2m1S~))7{cq1ye8};*N4}27+P1RO
zQ8<DmzjZ`-gm$o)KUR5<1f(SMU2crtFVF=9U-FVA8{{0^Q`i_+ZD_m;QJ}-cLYOe7
zzcdQ^a)fkuM;SxHT_ard+J6U%E2SrW-Mome7Ik-hL!#0h@w^JcIUlLi4<}|A4SeO%
zl$Pk3{z>V@<!-4@4TF_3zOHx?CYtp9(@Qsask-&dL~!`dAAk7BOKYrv$g^5pi!SNs
z=HIlV>Xzu5qmLn0&PP^g<I0CQ2#In3bQ;n==-a!yr%4kW9u)?MfOx?;lg$Kg8gR5-
zVPf!|4D%87uo5Xl_%&5{L$=CkzedgTHnY9Ok*nF{v{1R}Wp!ECoyS420}X;pG4ZRe
zPD5U#p!^>{D$1<mTFZ!;EI<qDotT}Y&LGsTglNbrpULZXXo_YR4U><E--f37l`V(2
zJ&H(XhRW_?yU3#La!7R?F1^y~T94$LEhd1X7-B(E@hNS_;UDLpp^X=49=SfxJ?i~v
zXj8PpkEDGmo>YcrsgY#Flol#26(Fuhra=t|DX?9((#Chm!dITpceW%AjDRH{t?(K(
z)ao5O5Bfho-Q-j(p42dy^i=<wyUjG)B92miIW`A09)4ZL>UgTVYkmiI*2v4_baQ53
z(^+S5TiKEIj@*0p7vOQ&{JFCM8oZwcC?l6@5co2E64bpTF6Qa!UbW)<yQ+BMvqj#5
zMx)(M(YA02zlYt}WehWod)~RcgVY;DIFa5h-En5Y*lbmZrAzrD-0{Ny54xKt>FM=y
zKspR)DaY%ctxs&=s~q}t=O*-RD6{36P-NKy@-8pf$Rhv}|Bap}Z#XG(NtF4APM!)u
zBYU9pWorK-{B+Ob#%M3O;;m-dHaD%Gv!`bmne)!ud3J<E+W!+fofrsm)(6`EJG=jn
z46Hk<M{iR9o#O`D|K9{esLS(a0#vl^Wu~H?=Uz(@D++~7vhv`@!!-Wsgj4*JzSKfz
zqX+OChU9Ey60P5Alz+LUJ=o>-1F^2)@*vP25x%2StWHI}NeY*{L(TplU<i?G|D$OS
z*FnWMnKsLsEBEI7nKfbChKZfx{QM-#4ZxQJnw<kM+JJMg=h4*EOr6+?BLcL9Ol~2d
z>gMBWkbK?=3;UXPt=XD5{rQ}X=)D#MS0<`87ct;!uuz=&NE_xnw?|)z%8kNo$wE3P
z{SK~f00p>ZuU0$`u6*7r!0?7AfB$~%=>?V=Gqo@=`RmMwj*brKj0%#e*ujlTwc=@3
zK-U0{H#-{;W`G$NmHYnv`>HS_BjdKE%^+grjq5mH-$N>3CMsG*N`)g^vG410a&jOL
zMKN`UhTfIGc`g%%m1Co$&P`f@Z5eY@`)5!*K<M~mJ;or$65>i7I6OQA;;gS5PR0JD
z1FSC-lN%pt?#Qc|zZdS?N!@rA8Kt`X9*-3JUZB8fp$|zseS9R`#*E@W81W?4)n`U?
zRf?68Isk#jgT*9VSKZgPWDTC4-L4P@p3p@-+@IMsE4+$~U}rm=bSL~ogH!p3rzE7g
zy53sSp{Inz-4+n(56E+gfxW@zTAKth_V}@D{?;3x7q5))<Dfj$=Z0_)blGqbTh=sW
zWMmW<_nSg`-|lh&GQXH{hy(D=%R-K3W@Z`=kdZCGVS!)E&+jpXFzKpx8+ZL(Y(5<d
zH))`EE#h@8a#Q*CD=h(l@;9*QRuoR}(52tv2r7sgObM?2x`0(9v{-XYm6@2Fq^qF|
zri2QD2?>4nZyf4?daT!AQaz-QZy>%<N+&{jINAVsdEN5vw^oYs@GAzo28dXjw#jY9
z2sHji{%23myq5gY&<9^iOxj#w!psEkdgg?$rY{&CbUy_yrGpwd!B9VE8Pf5S(L;&Z
zeKed-9ZpycbiyyOmW9%BH=xFdu1va*c6OC&ANW^n0F%2CTnxzgVkI-cT>`T_6_Pml
z-9=!gwoc_*boF(mE=b22V~7u+)2Q&3IA(z)BaXui#IheRdXPcKPpfj%nhQ{&jwvdZ
z33froXUaI)K%HCmKn@4Imi6`Zo}M1b1g3mcUdv4$OC5ETC3_wuAMfd}9PUnzj!!5?
zRj*eEw%Mbm;>#2>NxC+s5dYb`?rTEjmS=FbQE}AvDQ{2g7+#1d(G-{mnI*64&@ERh
zHbv7aRZi7Y56?FOu=;}5+=ACV#ha^dK^))!gO6hKs}v0o2zv<*Z&2-)$TAI4R8@jZ
zZ(TXQf!H(Ez|@jwn;^D>t-BC;puz>R;jhojU@oICPI%x@P3$7_Cmlwh@WAa5*tnJ2
zpL7!2oGCK78f9Vw<FwEF9T<regzGwVO^l5r4XAC_rl+l$&!kq+Fg+{zEd!ce2&X5s
z6k4V}wIpHU!Y7;<_yPBy+2p2umvoh5-Z)^i|NX5#a0*pr#!`b=OfX9TNX43lIrvOP
z;h5{<FP~Z#Ge+*HYQkP@nSx@vV4g+BT&Ac5h2Ha$R8jTRB@vXUvV884ljCfbw;$e*
zy$0!Jjn8B0lnNxy0j$|G2)xq4np<6^ZUSv(^eOzo>xdZvfw|j20k;Vd$XwS4h?9mc
zrZPPSn@8|(jtDAt|0~rHRWERy4Ef}0eN0OP1xbIz{wU3Qh6gMk$FIoJ?RyHk4h1~n
zcRNN+L{Pg_9sf-Q70mj$ZS4&TXh#Yy0ta~KNV|JV^-61>%faBprPB!^krh}d`HuVD
z&997k;-u~idpe#$yw3W1qrPz<bHFKCDCv&Y-05-AxJ)`q04SwF6@4-K0F*9<c!EHM
zt3|<*8Goa`%G-Vapr>fBfHIWJ*|+h~)Kn?w@0vN=7U%clBNGZ&^MFYHX&b(dlQHa?
z1<(Y8lldAo7mE!b^vm<NJQw2AK5;wZ_Cd1ZuP)+A86$hZ@DT9#2&xtWnWJ!i9pnyp
zzl4GBNzQ#v_;BW>U?2v@xj+u^wpZJXG8gN85Oi@h95N*e7>eRD#{evi3bU@@LOZ>}
z82z0Z1*icz^VxBKEj4fIQ+Rw+(K3A6+L@}JyQi18P-H3fyduQOGCL65xEHK2bwd_6
zr2-akSNl+btpBWS5&;hx3|n}c8_m|S_ZZvP-gJi+3d!I%ZxAfp)95M%PfwD;5@A(j
z66eN;(L2=oL%_uDSCjOvxXR$O<JmZ*pRbGiY7Wo#zbceIK?-#DGpD8qBZ)wCm%7ic
zP#FOEa0hcK2sV<!;Ep|SV}tPRjztz9H1eLfr0N3Hgxz{DtM<n#zhv@?4($(uiZPt&
zj#nAbg1NX_zN!N|Z|BK`(Dwu`?@O_k>`MS32<E-s)Ryv(k0kWU13`iH=-Y_H<v@sj
zH3iAkp9ueO6$N4m@OJDRi<@{;fR0u?)Crp4dxb!*xGm+5-D+L6Gab$5Vj%XF{eD95
zDB6hQGdg0Q_*kG|7WVLkWYB*N9KKV^1mhfd<32T{Bl9G)S1XJ2tTsd=Yrk*Y{IDKt
zd4yU%_U`BaLnrVFr~lYrpcNr$4g~T{&~eGN48@Q|F5sYRi@6ku7!X(e0{Toya~OSj
zM*VGl%OIX0FesjS!DOy~QDKHG$b4IH%ssQdXGT@ykl}_Ax#KvecG_M15gFq+p@mD<
z3JG9=cNi55M<lShtINSzP6NGqzA-x)l)u;4gxWv09zi(1AA^RN;g2Ke@r64g+|(HO
z0Y`h_dxpFfNG)?CmwLJ<S*|9_;5r`3r5hgJ&Ozp$AcF+7;cZY1T5Mf));DTYnr~CT
ze~&}upqV}b4uw@O6>)E<epp}8pu@}mxOYvF7RV!jf&oKG^7{aNIB+KSkI&AUL)6wz
zL4*d$eMi7Y%Sul#b76mC4;3Ngn+x0{dY7!Ls;a=~(__R34RfR@51;RWnF%lu*l%cN
zVj|H@>rI{55&<`GynCErZfa_3W`_0<5sVR{I)R}!p>JT;1?2s+W7no*())ev0V5RD
z5nv|9@8H7ScCfm-di4qpn12ngI5rrt7h685nPDUwi}npatai++Xl&FF5Jh>0@{o{_
z5TGU(mpbU$FzLX?5lAra$bVRjfU%D5?6kpAswWhLL_}aT2wX%&L}SQfzMXTaB_yR2
z>dMMsz7J&G)%Ur3uXy#h(>{oT5gn^~Fw0l#+qP7u!T<nLg)#}oV=%3WM}jYas6w|*
zw#QE@3$i!XZ-5MRUhjXL=36~+1+$S@zxU6~pr)Ek2>`$0)-WYV%Hf>Ub4GOORZgv{
z<bd_2%)z|l%HK)YseG<X4-7S85?s^M)3dV-dN#8HBSB{3N=UrK7+~OsQ~~cXRJdFI
zGtSGUwCybliUer3XRfJ<NfHweeXuPUy=s`#0%KnxVw*g058{R{Sp$}NRnacu=@hCS
z$WhzpC4(FCIHLVi6^||$(g!D|rwh=hOwB6%lu7Hs#Abu~>@x!&D*z_yeW7cnmX>L6
zPdXt~f%7xd(|%YS$;CN2ojWH5RQX|HVcV~Ihy@*F1!#LfyiM+hIRf;sIy9wQZtv*G
z0Vd+O#a5kY5mZECLlC<0z%T5d0W_A>0f9WQrB<1@G&TK96+5wWa&;r0xgGkl&Q#OK
z)!1RBC`$`nYVLrzyA+;Kn}=AN@hvPYz>q1-Q9cX_9v>&7%Xt!EU`h${prTy}16jRT
ziJjQ3Nvm*-FhR+PVF_2LyFEuhZa(pCr}UaV4<la4oMi&lu|$E7`~tRogN#P15H$rr
z*W*)AW}QAJ#J=}~-vhTE0!PV~ZX*7aY=$ce;DSIpGZsp_zq#tfl5O3tW5ekyFxkf&
z3bU<S?TE6h;Vtpw-1}C6CCZ2q#rZi&!(}@O;FI3tSul~+=s$X<A)r54n7Va#m8-O?
z!hi>+GpY3fgp=WHAVXll0~v$?@#L_z_Pw!-K^C!3I8k6c^qif8V|;9^r-}!nL@j~3
zZ!ey5mqZ^tJT>(y;Wcmx0Ln*j9dnyE4wtOC0A6eqQecctx8wxVVGWw&YtTS26${XJ
zST{on-IECcMEF!ncn<U(3ni2?(oX_v3IVIvX_Npb%7;)^_)YAeHOyJEN516y$zxwk
z(kAorP{wH;Oxc0G0ti;4WVJFjt9iqgoEA9>pP^Sm@3~ebop5#6O4PGifa;NEz`odK
z&cG}!EFfyd0L(AWpm~dm*P(bbP*ysC$x6o;Gr(4`m7x@-sy0N*^v@G~V-)be?Sz!^
zlSY_ZWBATa+r>Qds$a=z-e7Sb;pOF}rKM%otk0631fgd;uNq9??aOG%dLo00gGsZV
zjhlOFYD%J<7akfk#N!P>ps|pN#%Jl&jN3fAcs@W+`I&A>ainp!YyXUsjV+G|!kW}g
zBoQFGGQcR2?Xc~Jh+vBV@5lO_val!j<0&650W+eSvJDIfKn8&a>bUmiVAMra!RBQn
zY6RVet9(g?S}}_h6lPn1t-4qkc4z}@Pfd0l-s8&OM}^O<|I#o{cpH)YcG|GgrAZ66
zXrnPzGO}fEX<3q=FY)~I%138xQn_(q=yQzZ{`J4kAm)H6)=_ryHE&l9A1e)Bj6l8Z
zx|e*}ex<roo(LefbPP2=CEm2Bs~^hRpvCwg`cMA)z{$-m!HketX%unFe<h<0$XVtS
ze9By(o!woH<AKI2uRHevM!vBldW!ZV#TL%=5w3%AJ+w;}FbI>@qgAYJc08}$AMx7z
zsU8?Q1Jml7)g)+)OUe~PH46;~ztYA^cR9>k!k7>vk!tTp%s#;3SJwLgG>`<rV|E*W
z=KkO}#be>;g6Fpm4)Zl4U%^*t>Fm3~`{Mq2gIns~JqNK%ilFcuQrO(xwAXrbh*9|5
zsf{Z#g6)g@!FDVF&T_Et*`<GqaTN|w6MOsP)>K0m=gA(QrK$dSq<!Dtuvr}I+I0Nq
zQ~-G<^4aZV*-hr79c8?t(v(Z%V`o{=@@W5*wc6@>3Fj;jyklQfdsW@FczNj3eYRft
zxFUR#t$iMgMd?^Aczzv0Wpz#yKzS)m^t$B?m*<~Pj0&Gx6lza-Ecx)4V=^J1H9t4?
z5Jnkt>q=P51iz`&awwHnml*z?mB}3p->HNjCl@NURPdHosi3@li&Iyf>UYnIy;HNT
zRseT=4)3~QS=sKgq$21janD?j2`cR?H69pI?^gYwAwq~RQ<2rqy6IL?b~;vZD$13$
zjD=Qj)hO(k6{6KBB<xPV7k}{K=cvnmw=AzXW{c6~@1_7)%@75=u5K3N4S#$Fi2%)9
z%Ro?tfUhqUh;CX;Q;@bQGjV|DFVWu#;WD$O-L-4SnFLh52Pk&Ufbs?D`L<-#DOLWy
zU{$Gy^WFO+Y+gyS1&POqA0kmEluJAfIQ=P2L;$W~csZmu*W`37x{az%1VM7sYW{OD
zEAmE)6EX2)OI<L$UGs}YdJ>=b=wr{n$}V^)rrvFb{Yk$cGhMT!e6e@;8*Piw2Nc;D
zyDmvukKlgFVAg6?UC4Tz-SKHSNdj=E?eN)R7%1pLDJnl(2k_l*#xWOz&RgcjH^SBn
z6P9@kQfG*81BuVnd4kG2XwD{BUI~7Uukvk&p*U|iP=j_+3JL!_>xu$${w`A185-kA
zk1r4hD!O_?K(jk?4?y!gB~KvEu0`sAB><K81$G-u9FJZf^Dd7{-%Ys1#Me_k?r%mz
zY18CCeYC>>S(FUEhLFgxM`5G#_)>~qIVx{(kFvm_l`MP&ou#LI7ZF0=aFaO+s~2(>
zG}vp;x>}n`6+F&`3FtCuT{~}A_<_~E#2lxUnS_Z)<9lw_D3qS^F+MJ%{kpVt^H}%_
zGe_lDsu4%Yo%D|GN#8*P4|4OU?V6(bG!|$swhZ&lli>)h#^IHMZ9yeM)9-vF4-uqd
zNPGgcsGk|6v-%rpP)s)EnGnuwZN(o~`E!W&_m<0dc_(2(8Q-veUynVGW-uV0JhF}c
zy31E-1KBX)4j|zlp94KsLpP6AbitWS5=8Cx6c3naR9U&VU__<*8^xRVdaVk5c$Fat
zzYy=z*zrp}*9dNM(1pHDKE`h=z4@H0{cQFsSZWgY?oT517wBfnsvIyP1Cj8PV@+#5
z{GBZ*E0GcI_e{;XkV$Uv@XFW1db48BIq*=>awuc)%4poqZN1J9MeDg}i<z3kWDpFK
zE64us_G}u|X<kg1Y!!%nR%o;<Y<xC5Vd<%B1WJr;G6>k@$ES^ada9>C0|TNT?{wX+
zOte`DDqs2eg+2N>X`=c@@%m^5|LNOHR@yl&S3N>?^<TDNkgt@O$l0HhffiQ3mcI2+
zr<&$#)KR)K&uMGiP+)XG!)vYcN`myU%Jro!(${%vkiBlsRhVBTrPRYVeB!~EgHxa^
z#k+}EG#z|VJy#7ntnLbt{-|JR$6^fI3wriz_ISLSIk%i0lwst;hkYHU1x&Rz*@wGS
zN9LcB3(yLbM5iOVTZ{|YXDcG@C5}T6cbO<eI?%%0Wy%m1mWS(f3bvXAx^6Ky75lE~
zzk?oQ=v(ua{$qbPVdiFFdGv!CDZTQalY(SXY;I#3#^!+6M_>lT&0(TS^Gn3j-zuaY
zLNp)k&|gF{r2H!pgg50O-4$Q%Ik5Jjr=LEOzuSyEYMKA+`m;I~6Bk6<NXFjg%dUio
zGaRo1rjDDV-p^_pG)%%-(a=ii{39&^Qi6j%5^fgUxvfw3cUQjo*T&S^Zxp9){@71E
zp(VJxZBrh^u7F1qhG-+ElIwZp&kNcu*{=0d?vR;~{TZIdyC3Uxoue}w(mfAA%YxPJ
z^^I}d`DTF8USCwCM;_p|ZdyxkJIoAjTB#fOZEu}=n3-g3rN1^iPNz3ZYYHdV2Xs$;
zIGsrY+UVQZ9d%+VlE(Wwrc=D@`9ZTmt|eQIv16QO-w)5ML!6v-TgWbbli@=75AS@l
z$QdR4)Dn5&OUzauer6CxK?C5ZuM@DAvJcy^eZQQ1{lHyk6`ji*7i^&uj_S6l``7v7
z2rTTLc$qCCY*~T*IwPI=K&!_Cs?>Lmci%bSTsa^?0l9aL<mRf)&&I{|B1q)0PYlL4
zwINu?dlC8sFg?sxJm{eHa&)ok2vB50+Qbg7kaN>KI9PLk=pw<|2>kTyYU~sg7XMjc
ztYB>abslYiABX_u$>@zHlk)GF*uW1rkCehNm^JnZ0`-Vw%Q?_<;Dx7tsxcc62M690
ze=#vJ%Bb$u;<a^ZH_V7mQZtfTlasX>PN~GF{{H^-V!HKCGSb(VyHD>PC=`EuzECa&
zPBH;xgYpnG?&d579kxMB^)Mqo-i&@8;N+GCtC|@SjKGnt6w3|Ja|JTjHI<hSfm0*w
z>^<b96F<OkJ{R$Q%Xj<szF2Wl=a+Dq(Jjw<t6%8#DwuQ!j1+)rh2_galIR)d*S7VY
z-MJ_rNp*jY`k)Ukb8~ZNXRhZ#p%D?<X9qB%P61E5C1x3#D~sC(v_~xs<W{mzqIy~M
zY{LZ^>_GEs%f6RfU0!|!6e42kb(&+-)35!E&CIZcCe5qCNtM!xLS%!w%2_5Pd(dD7
zZOG~G_jO5phK3wYL>D&DWhj$g4Zyuupt^u1<<Ztze<WMTZ$h*M<c78ORjA1fSKn|%
zA|AkfWhg=!pR~0XwnlPg)A+WIF6Q0#o51(!f?fhRPdaGLlYR?&P3Q+vgwd8Q&qoBw
zUXxlqRA3Fqg1%!Y6k+&Rv$>nm=iR%#91KAyk*R-B=(=ggx9R43-+S1Xb{BGK>}+i*
z%Qk8Sx>Iak+pX^K-WM8h9xB4HfolK1?CVRk<~bXl(eXzF+Ipnt*JLSP`}`DsJ^J9N
z!fOTwhS#rIlB60{q4J>Y#3eChNb9|Ck2cKBkB=KxPAv!XQ>#XT9$#Zdj<t(0;{-T$
z0FGr~Xci7yfQlOEA6%A})oR5`h3rVoeD@6VkLGHg=`luToj+QSGIl-b4lkqZ8F5iT
zX3=Rjq`OOuDUVnva{H}xD80`&lf>}WFYUI9hl%uwzfeZxWgdq$K0f}&#>SE5ItM5@
zwG1YQhku4u1ACw60v0CK14<J>EP&!V%Fv=bkb%a;!a@;8bUgT5q3^<W7^u9%c0ti>
zqwa?V>c)F6L6qohECw7?n*O~<aPs0~bb>*b>m_oU>#JKMn2J-!QAF>rRQ&>TILJVg
z;r&?LU$n*Ux{pWU_qU<t4|>EvvHzrWVPJY$hou_-90oXG2`*n>-;Kd^e4*8i4Ipnq
zY%C5w{?5te^0Myy4#PA4`Am?t?;Kaz{Vi*NLAQ_b7uqP1L@MEAS5|`RvO$BXbYf;|
zD!<d1+v{7u;fSS_kfAZCSGi{lE7{rDK!Fb06DRJYXk<pbjU)r<k^`W1^(o+9#-x7E
zBc{7V*@5u6s`}MrtrM26(J=SZ@NX;Os$-535fNSJ4)R*j=(<K%%zR}=H$C_p3$cwk
zf9^{#&}nC^Oxsgy%gLch?l-KN0o?+^c3RrfhG`{|`tk5D;4B&OK|Cvxsp|cK5>-xJ
z&^);4yZGHjDlBnW0nhfP?B5)1FN0e3A(T>#U-i?O>Jfg$-Br6?Zu)JBP)7Oq_}Fa>
zr1H5O&N(gIjFoyi9Io~pZ46>!U>LG|(RRrNYCOlp#1#BOq^&3O_f>+`0@ii%V8DpY
zNmir&(e6~`VRXOt-Q`3sIE3l6H*23@P6-3)_4e-8;U=emt@;&}`E1-b8_?P>no*Ls
zioRJ1E?fCnHRfw7Ts;F*+V3qSjY*dcrr^E6z;z>w!{;L0=7gH8gMgdo{#W9pMi+qM
z=-kR8o@g+Tq0$`1Gwi8fR0#MIUM4NjBmD;@GjGuEu*%=QCCfasBxrJy1aYP0e2|rX
z9@caBCh<+UcC$y@q$VeAK4ubauOv%)88hF3`EfhJ2>zb5UL{^ghbg3BM1dACI-q=R
zTl$oXB{BS?Xy?GSX&e1*_9QqC1<p=YgHQyHlg4Pca6fM0i0{YpEl>hDUACrVd(_sV
z*r__<$mbyikvC0BKii!d5c^o=tPBgJAb{b&Ohbe<Mcybg>Rxp5g?U`B1Gi%E1NMq{
zy*HhZ-4p%WA8lOXlU79;+2}nX+_f3RDumK@Ix{>|3s@^~OfUGEMQc^RVN3@R@EC9)
zj7cjd%THJihnpy<amTSCRc2#U&En*_1ce`ndH7u5lww2|m3k!Rg}Z<Qr>ar(a!Bwh
zzI|+QKH0T3MjIBAY*XxYw0J-{NaEHVB5(K(#u6xD;-441jJ2=8xW_teiFyvj<L)WJ
zQ7^k6179lItuIc|Rh@PFw4#}Wi>)4>-8$>iQTox<+86-x8IMxm>~ST;ECU>D<R%UW
zhXK1yO_IE0WDOXh5Q70nE@D)pM>agErmjj80Z$(yDixl9vt_1qXU=@4Ti-f#CHVEx
zD0n25X;9$_RXtpPBY^<qZ7>E0TE{BNXQ3^$SIYReSG4pODwL?2vvE&MMjigf=^>T=
z>EJe*V7!x}W#9&-3czdqF29w9Z`OP9)B9MNdkWn<ySm2s7_AOmx}V1;WF4m-<fdkf
zbBbZM<XWoQK*KK9CT>)J6z<H1!mgpGytf83BtwD=DMWL&d7S2V6MGxeT31b#O+9BC
zym)WJv}N{(T22(u*C^{YiY>Hd^vnQ8px%vzk8LbETS7LrUPe`B4&{B%O;Eb%vK4lb
zcnu27^2(dzwv>|{^p4E(>*KK3&6Q0le(0dF#6}I+CeCYfS-j@XS^7JbYC7hmg4Nbb
zfNddblrc*~V}Y~N9%(HbysXuB(~Uf5JR56cV^*vEDNZ{9Ot-50S0>!cn><Hnq=n1k
zxPWJ_OGXK}YB><9kAL(}wWSI$+j28quV6f;A+uZ@&px|(OEIX2`A=Ir0F<W^gr%sf
zo_?%B==+aZ06?p#QiJ(PhIsvL2F(lKpkz|+y9O6hwn1~gd3USY_Q63Kl&N2~_}7MR
zuC%{M?<p#mdV*3slr^$XE!xtI%uyz8Tk#3)u9u%veu=l#Z3l3s$H<?D^|9;E+(F!2
zJe_%Z$FDn-_dqr1GT@7G$v6glAN>`6`(r<Itcvp8o5h=7nDay$?bvz6Y}a#LRx0gx
zje(~S+9`@ba|Y1-&-1JViEGe2^yC=eHq06_2>}~Lkko|&&3gqSDG6#HBO{lzlS?l$
zF))nooT$FY-(VH>NUL)`0btsBacVOtZHm-YdeFb(R`pt7_9|>pYVLfj#hDT2!IF*?
z_jG{rE%4fm(qed_+>?+fkq0P>by~;uq_E}PN1l72|1UE5XD!@r=@JrRS3F;I`@Ih9
zE9=YMv9w%Zwxu^>d1{IkX5xTE2BZ{?s{fa&m2<%OvaQ4AV;VAMBMg%Pdh(wOQP+wt
zd9`gYu-CLAS;_xE*m)Eyt%EtcVdCX^zv?)dl-r>G?z_X}K2WByHs0x2YZrK}v5s1C
z2sX2}c{L+ZxMlF!?qK?3C0Cc4T;|Hw-W~U2ThS&iRV*=k-L)?r$&;C$Ll^Fj@dYkp
z@V(|vo$g-M@ZFv{&AIHyZRy=zMtHeiw!d__db}?PKb5antfpOIGV|Sy^tv0gNo{!Z
z2IF!Irfc)vk51XWj{T)OKi`ZZLyTUWv0$f(&C2aHI<>8Mc_lc8n7_V|`z<`=<@UFI
zh;!gLm4qXHd*9`N=H!^gobp8vEAzmeH8=u>zgAnVjDIm$5j7DbeWF%=CYMwO+&tsv
zF?ba{NnXQJ16u3pssv^E`jIxq8d-NJ!z*TqE1VOT^<~hITE9tU<^GOkBFHjq^Ut6P
z<HjJze2pUS^K$E3k1K8CeZ1o0+NBl?^{XU{M>%G9R><?*E_;x+o5u^dLJE7e9x$Wo
zS5C7Ge&-!tH|gb{vTVCx$*1lzGvC)j(&rCo(<5bfFNdA{u)v!o;Ae+JTc`Wr4aEHv
z9HHxknwDoy(dD1o?_lW<=efCUEx~cgqFI}x0Kus*^$tz-!kagh&4M@ejm4z=OK9LH
z4X1V6?;)djRfE#b8bYT(w|IGr=9RD=gJpMluNpF~QcN!QXh&00XSc8uWYTNxuVI+U
z%~wHHk5vg`m5Z5^#wusiz2rT{mNGVFwXPk`f+TIq1z1tt7A5agE}BXYp!{~I4L2VS
zhufA@)z`uRv)^;42ZH%wZ>X)C+|4>Q1uwvL!5d_$9X$+JhY`%4e?!D0NW*5%+p%Tc
z_w3TKg+|B5P$@v4xV}Dc_UGQ8nVGAbeXoU`6AX0pV)dFmpU0%NcehvIVDbv#J~#?1
z;8BkplP+Zmb$97W$vsO#2>`7j(k-qu>tNR#Lz>#eQn)0g4M93w(+(C=)TMoiNjj<R
z-E|M{YP^QM%Al9B+H%;{PkTS#wK`p4HmhG@Mt~hLuRrPh#Y-zbh@K~LwjG99GJD=1
z#~1*?oyHRz@TmV#`La%h?hrb7E6so-x8G}5b~~ax+E|$8i$%#1y9>a@;|26ayNXk?
zTk~pF?j;-4DZArE?OV@tFA;yNJwXq7lyV#S3tz8`E1_?_CogF+(i5(Xsr1%jpSms|
zxNzqrTZ`~fOf3erZFn6nm$nHH)p#Dy40>6ZMc<57UiRGa*et^^Wx);|hq0cq&EiZf
z+8c2drZ8$tMvsh>u5A(->HludUCQCzFOJTo9emSKioW%65}(w$T8BSG&tOzVxnN}z
z`}@yMoI6jRXHUf=9AwY#u-3@zR!&^q+}tLL)$waQjE#*aN;HLqg%2&QtoAuuF8Zm&
z!`-iD7ShyxYvaA~Ya8|Jq~%&f){??hKcnjci(4y~MWa~#<^b$&LyNcZlyznmuP%bv
zQ=t6n$~M!stG}0o^YJ`)zM&cWmMK-~w5P`K7zPKe6L1NZFQqqq)x9J&fu45bz@Skm
z2Ck3ZJBYZ8*om-D$$I!4)|G!+y*3M6k;y+LBbB!z8dsD8VPfTZ#ReS{8e#vX^vpy%
zjMV|^KyhVSy(i=HLY<@5<cg5n*&QgUN4VNmDQ-uy+0;uK1t{-mIx_Y8sr=c#MT_+F
zid<4rl8%Ip78|<@JttQV3+<8A)<{Nd$K&>-pZ3g%p#*nM>0kI3w^NWodLRLj81T7i
zQ+XIx5!6kmawyMW^>`Jp?mnb)5XgqLXq7r`TOSsdHowL`-4Hmd17_>F@3z3}db~N5
zDSrP~Ig;NR(GsM~c(Oo0xk4r5+i*@k*fFr;_B_EFDdz00h+H}w3?4L(@lip^n6<H6
z3!m}F>Lm$8+E3XPJTJjYmlw|z-cM(rtM}CJjjS$ek7P7Uy_;yu4>X6QtgPoCPpK`Y
z#%yyNpS_){7dXD1%3mYfMg!rd?oLPfw?|Hz{`O#hJ+l?}ol9JueE52rz}q)7jr3i<
zZNm99$dGudlia?K3sXy~ijD2r7JB)&AR^Bm%clSFe4z3PAPAN(XcWQwn%9xwudK|1
zN(DA9O+hvW7+E2dIy;vO$UZgTOSk3qzrp|d!4;SsdwD#PF_5ka=J)C|7xJ8Mhd^=y
z%_!-IZ2<hS@yK^$r0xj1o$mI;&@OnK&1v6kE5lK|*X{M<A~`B=9WQ&%`n0l^i@~;h
zIC0%)=oN?-vo0H{vV4F28fRu^cvqI5b0jYJyhC~R#2*<Mc~yLF<ESU%vd(S;l+%Y1
zmqJo<gEQqR!>4gAKSQMm!qPr6B1Q>Mn8tnk?y8sZjG}$CH!{i&$rMXAWN7{Yh2K^@
zkrCeoqP$#isb&2}fj|A!70vo}SCy;$MNo}kEK&_iS})@(0e<2XO%0VF=EdxKu;B4^
znA!BL3<^c_)g5V%JBj0JZtDN)>^*>*in_K@6crGpDn*(g2ucT~NK+6H5Kxd_rI*l=
z4pBis>0P7=C<q8h?@j5UNQcm*6CeZ#EtKT$;C=sj?|gUeo%u3OoC(P}`|Pv!+Ru8{
zTF)UgwU6D1M~FFgWn7g5De9pi_F!8eB7iS|Q+sh86Vt%s404hV&3P9ufFxNpC_sA+
z)M>z!f^?#NCCG!p&y%kCMSR5!T4hWu@z9VYcZ{*mqKRzW>BnxK0AYH1#0JgN!r9r`
zL%<bOO*yHSY3wSk{<&pM<*CIl&|P(Rog(0%cv40ISM;P=Ueh7(hpe988a?Z$-se30
zme$N;ODo{gdq&xiHS45y_N^Uh?Q&E;7ZW9&lH2)yqcYVRS0ss=?=E-1p3A#0AIH^f
zT;v%~4W`VCrr2{dyj^mWm!I!hqkalZi}f^cLrGUB9$tYAxjrEHde=`b4xX5=G+C2>
z+V3aE6tLI}X3qnMNR7sw8K<P^a)8{&b}zPzaXjR8q77?z21Nv3C#Oi*5->p^n;h`e
z?+}jR`3Mrz_~=351rP$E{I$3PGRBtGlMo=(V^T6c@51|mH4S)qo&%ZBHlm8ruR~2`
zvrV3umN|rORWTB=1O^JzQ+3tong+b?c*YBp(q!^e|Ms}I&tsW0L^ZbbD~)ZzZf^%b
z=`T?xw%+{R{phsprobYhc}pL=!?Qy|`7{5A!t3_>2SoNGRsK7}SZADN#UWg;5|Om;
zIw%F!S5uI2*{KC~gvtZId^=VJpBK@-r61GgAX~G~3gb;>vlI~bQTzI**D$YO_9cJt
znvAP*$MSX3*u^_5^?CERMLJxOS`puTUk|k~fO+$pH5`R{ic4J&C`5YYAO2o#uF|GW
zN%KCDKU|$N3bWIoKGoPhjjkjywE06JT8-F6i{K{dtETVqR`)kEr}o`ZeY_M?1q5hz
z49E@6g?H>r^+3M9VsQ@ilzhk_l|7ozJ~diOq`woAmrkP#I85xg3>6uyY!K8$)fI_F
zV)qGSPrF@bYUX2+oO64-rn>O_VUVb(=I7^}^PF--H68*|@vN}<xR4=`aoWe`Ugzi1
zRt~NpiIs@wPl;dbs+F)+o`G%z;r(f;fxGJ{6C3}Per<-eV0<flsYtH}%vYi{jO>eo
zLkY*oALsr8?fzUt=Z3BF>KxCnUbkpJeoubcfA|}hS}qrMQOosh{>0BJ_C2h=<Gjkp
z5r&rw)&vO{L&@a10_IA6Z0=H^TGg}BJPo6`%ae>^eOm*xg=5M7SO>#JYIKAW1YjDS
zK8vfd99sYcvbBAv9>~L9x5zh1f49B&L3L}OIpyu=q>pr1$c!Y^dBW3b8{&5V=7tGe
z)l$DmN1t<H-}i>Xi};HFR8|PT1GNqkiWxWJ0t@vAf1XX(x&wkvrt%QZ{ySbNkSpUv
zSE6}#t<mrZ>cHl|bI)-m_Sm?%qQb-0mfMGI|3?$k9cRJrZZnyy(AXhP{cq7IK*eQf
z&v=S2cUP7E=+dNoy<M|3wqN#Kat})c-()=Gm2g(BE56mB856J(x!(NSj_+tJjwa9s
zlNxvX_M}#FV(nJ?MAJ&!6f1VA-ISEAKrAs&xHF8`RK2l6KK+;T;>vFDVq5n=)7l5%
zI{@s6<?BSiwL8dSN4#*yU51X$k^u`tO#(P6!u?4=;p;Zn`GLeuiz$juF5-}A9Bb~p
zLYH$_0?sv?qM-j3w(M4GBu5NV(&3LrVvvU?pahbsLZ`6!hJu94-#0T(C545crY6O}
zK6~zErSBnBKOr+5)F#IN$PZK(^TeF`uswOPS$1$cRV;1&ol9w|4w}^vJ_E&_&Ec?z
zdFtjxal^)@nD;KL=_36LVirMqVZlYJ^Q+vR!}hp58S{reL#Oci!c3KY)P8w-Hp{Sn
z+x{T!I|F?-F1FaqjB;Um-0Y$8iy@`odYCJ1uoyK=0wx|~lJS9T&z+ktrS4*siuycP
zXq`Z4QcLeoAJ{xmzJV5dnX9|XGCK%hCMyEe)^i4NP`^Xg(J`M@pVQXIZMCPA!oNgg
z-cYGWLfiTk%{uhQQ_N$Pt2W?j_p0le>k!>fDxBiCdUJ`&TvyVh*=RgIZ$XCIwANN4
zw7mMezo@N`h;Ze{BCqY=vA8W|(HZG#@g?S_{cYVaR$)c{Sy>i$_aat0k~Y-g#?lfo
zVqL|^cs`mXAQN@l@TjO6y6mBA{==XdGGwu|9@AjpAVVn3L}ArvRLLPvC3k5S-FsHv
z?o>TKJ|5a>CBZm^^n}F}HFleqo|jphXlrv49vfkj1LJ4SudX;K>+9<y5Qy6pGdYp4
zaTzE&ynQNAw)JyIs~`IE!YTt@X?cckF6a9Ep4WGrrC>K(h=%#m0{<EL@ElUV-m_@=
zpf{GaiALi-Q^rrp)nLovuVJzw9(tORce=-(v7MweHzo#C63Y5`t+A%k_3G!11%F>=
zUr)40HHVcxcuPaGEPvR5p2s!f1fYwM5tEcpUSr;*&23vrG18oFTRD{G7-PHgvNUhd
zI``!Af&tgu##od;3b<Z%6LCg&XkxzV$+~FmDXJ28xHB!=;I@#nNlP851sM3Y{<b^u
z`5<KgYEaVqYtaIcf%7YoVzUEcR!|UO)T7=)Gu>+rqHt&FOU>>^e(3zabj}$ZJ##pf
z)vfWWz$o4a%wxsIg{F*rBCZ#`!rDSNNohhie>7tUU|ae{+&zh?W)>7yNyAHh1D$Mf
z?e*)|GMOdW*>dW;<XnC{-PUTyhW%=)DAQHHt(LXP;>+#}cB#Q%Z_IHzr6cG~c-WUW
z;c+oa(PLKy7vk<~;lv+zW$=tS>od5$G6^qybc(_qKUt)}?dvUat#?M~`}E&syjo^b
zT!+o~GGX-rAmY!TTf<ep--Zz{JBgQ%N8R(Pw5JjUruU0s$(v%J>+QM~2HGHtf>HQy
zX~4D`4}nBh(zNc}xWK(@SjmvmJlO4MDdA2p;(%&SjprXzboBY>QVB~~KK~rGu6eRQ
zJI3npy|~y30s&QgyC@U!%IY|<l^FJy^kA3jdo2jxsz#6W_oWqyzKsg_s*hAJnpK@D
z7k~#+Yft_t=C4I8JHv2JV}(!jjie1Ww&(-<Qmq6(6u6~yxI<o@Dv%=ln1@_7$jvAy
zA`0Z;?&}QXCrXvo>J$7f;Evl5ph#G)(DInhn*v>Nd%yN!BNdJF*c5C-?7^sGnXM%j
zRJ(whcEcFHU=W@BG$^7et`jrV^h&tKEPfL?HYk!#2gAA{s;+rKrf$h&K0%M*i2jx*
zB8AwRp{|N9yEE~_9M-d^HiWMO_MadrwLXq47M0ylh;@+n^ugxMHrHQ<srfbE38uG`
zk?b5#CP5@#H~%%O;x1!p!!>8ugq1{nQ?f5NDb`%+_}n&5R8pWoCx+W+V$JO5Z>~nI
zHAj~G<o-1^ksoVNL^h~6Vas!OHa3-W3i*ESD>I+EK$Dgy2?*RfCb^Tf$7P5rPYnXT
zEM?BH{ZcGj77+KwYGK0W2ku#hJ@hAZeAUa8c~bMo;7EBMjni?|JJ_HIG!@q({m{8j
z20hu3wRMiP`na?bdX+t7b6W|sK(nFIG%hVA2tWD_9SuQgm%DN&V1EAa1!YbTAr`_Z
z70e}8apZSzgX|dqPGxgduw{_Np_+mWBz>C_<(BN(lP`x{r5$%t^TWlvjutqK7XyCx
z^2H02k(Y$7rUcQ?MhFN9XjIK=mE7gwi9b9}ZN@NyL>|if)e*S`R5*-MWvfPh1+lm1
zq42^fyj;TWKqDiL)t2i8w{X0_OeXKIDA6#p>M6hvZ_U($%p(w-^4Dj5#51$=6IDVH
zaqZG^6Q&xWA(epH%PE(+;r0`$RT-L2$=-OxKS=<}!|UC{BgUI&PeSV6`WF7V9{PRK
zV7NiUxi)2USB*-(>L3N;i<zpGtoC3Pzh{WGtH;_ICfKq*^}Inw(g!yclrZj|tW{Wj
ztdQUnBfQ1+^S2nApURkXH)fxx*%XTFdB3yaKVMi_nWf*kTR(*DG^7y2Zks~ST2Y}y
zin>LnRhX=I%3|v#WPq7JC4U*UV2hiKNk5UpHs%)>(}FG$b&0PtBO~{flKe(4EtEvw
z&s6(+2<wRw9xhUCij0afsrPhraoJuUEqwdvjn(Kra<W|M#G>t@Luy}cUz=zu68NZ?
z^dn0VnaNKA2SnE<Um0m!0WKPDCOv;}mytB5PwhVP?6c7>qGz}y{3#9B(u15l$=9>x
zx~T(Ek-<%{rjhX?+>hC4%%OHFFA3ZF#zsyh=P0Tu;YNe#yq4yE^PbL8V(FZ;CmV%_
z1xF2sd@PKb0=v{$i1Lk?gWg9LPOY@r)K33MGd8SXfWJJhRTF^LuclL2jR4~o^ID}v
zJ}QCBJ~8BESISRb*RfjKnjR`eW!{O3K=fpuX4*&|oW++ci|P6mbL8>l4h<~hd}_(Y
z%AlD>+ovlB3}OPN$XaR>+BoCD65qKRoH5T?8u<D+5n)r(+!MyEOwu@~=^7!mdKqNE
zNpE8HW!mhX2ePRG8vB^|<V@#UspIr(K%uEB)9WhRv?+huOc15GM>gt%E;YKg!OW+B
z9?sc8LK``J#R+Nd9WG_4w3)-@UIghqT#DD|;b$czBm~-laEJbM$-@pt=Pn^gYTp6)
zQ`7b@QgeTV((6@wm9e=jY`p%GrlzJ-^O>hUmM!RYgI`U>NxSVFd6BLEouqG;9tVPU
zUb2V1?%!a`=J(a}%0Wr$H3B$md|VtT!R)hkf0#}i{w+TrVQutOrJ%4d{`XY_usswM
z^enq+>*~UM?(S;Lb%m~CUnPyre;&%Lk?hki@pnL+`SN_-!tI&P9?djwtLw)^Ibupa
zNTrK^X?d5o$qhShUDU!k85HBBUj3q#`<j>MKy-TXZOdq(E}f6hhd}v_7?0H0SSq(Z
zc%{SWC7Pz++vyu9mtR}(@Sf{1S@yQQ$oC}CT{L(0KMI^0XCDY~$#{BrfbB6VDvE@s
z94yAdf&$PD;`yL>j=Q5-ul_-Y66E0la+`<dh*81sODsjdvG3P|mAPuN_M^QVhNw_)
zC6M^MMP8Qpw&b8sc|7tfgtylh;fhi7n!aoL;`(yAic*iNVEqCgE-YC+{8+W#G{QTe
z73EBR>{*oNZP5QUSvxv>Q~6%#_uEc?ftaG*9N0&Jzj3PeBj@vB44M~nrXBqHV_;ww
zr`zpih28??R$(Hynmj$LSbPng)|n)T?Gl&ord}<PIy>gjDeF4gB43P`G-vo7g;;*7
zOhO_i>|$E)i3H0nA^BFTGwyCOw9Ual{%2VZWL+{=OKIfAI=J}lZVu3e)4v);Tej-$
z{7vuDtiugCZVLT!6YQ9r!5I}uWGD(dnXJ2;V|ZoWz9ak1q{7HZ>P|jR&MdwT-H+Lf
zVOBK~4D{;sh)#Q-oE#Aqb>-rPx@0qIY7;D#z+|>bp5IDBVEvN@29Ummgj-fmS#fQK
zDH}NDTDv-;IaB*YJN<anZ1*7J6OSJGbbW5bm3WVXnQcS9Z5Sci)s!GNaAS*T$)dq8
z^81;QpQ^M(Y58ZW*!ESe;tMHK*d-P|rKH@2o-VTD=!q{?<jgdL5o2ORi<VjAzszOJ
z7*z<z{ehN~CT;u>ydgPB|Fr=B>ai5ITjdQ2g}fFtobm@)$#v^cXUOYpxG7;Dqb~pr
z_o4)qIn*~Ty%M@n>7A#)p%W*=uf}(7L=F(MC0XY7tNa{uopP*xPX<!PWuN-|F`XQ3
zL)j)g9UtP{Zlnf-eVHn?+tx1%v;pmRhBbD9g^+V2zL%6|HZbZQ73d6jmdx2Pvh;4*
zJl9%_^U^YBc+Zby=(B#0itidvAB#G%3}AlB+yr9~EX3()f@EZG>uz-S>50Klsf}-)
zG!xk};6Dj5eDz-M%^ua2WS<vUq3&p?$ZR$XzAn$2e3{d|M~V;kDQ(j61?vJP2{uq|
zl#ya86;Hz)3?aYprafdgE(I4;PS7Kp8&|$67k>OMx_+?{CWciFE?CZNn*Xreq)K=@
z4TG;M%vSVKHs|{bhQJyx@h3ZQw9|NJcDc3>$9CGYvd(Yv9A5+w|JX;_h*lfZ`Lg$n
zb|2qzBk0a8Fj$AIn!2u_-@DAJC7phsZ(T_Ps`E&g$Z!PTP|*^R+pC~$5*HVjoonAC
zOKtl>=rzlOJR*?Qd6QDr(xW;e#V7C_djVf6|0w`n*zm*<%Z3}^AwK_LW5vLSWSRB6
zxbd=W2uyWF0n^z<w8}57S-*g61P~O)a1lSWPX)HioWeexh9+_tB78R*0hOkkx6g;0
zHNRo1+_-bq%Ue#uE;d$H-K<uBS(ur>oe;fL>ul;Hyb#W5Nyi}DteI6oPC(7e$@zZX
z-mFTh!hfc*LsEkLQ(sKFb(gw`rF+-~f|hrsy9W`8)iy3nf?1qQc<pce+3_O4KuYJl
zTjKvk9UDv(c!8UCUdQ`?-oX|B_s`M2;KGWb@J`o%qkVNM>kY#NRg06Kx4>|8fIlnB
zNQsRc6#{4FdjI^)p9+Mae|xlKlafoM_7{bqZ!FH6g<BGgFtgDmxPBf1Q(XFC&tYJ%
z&;|VPKr!b@J}c3z!>9?3cZoLh#Fg`*yK-%2b(nU;s+T`G!GyYD@{!H#|C*vHGr<Xl
zk{v>CLKFS-B+fpB?iyf|UyZ8#E=cs@!|{L*;!yV~9$?qOm+}X$ypdo}kp~r~w=5~h
z7@Kjn2gmO?6_=A_7rd+l{F*eSOeIz*BCvH15HoSS)Ed5CcR70BcoKNP4am>7@4Ssr
zKBB7xE5!IdtN*XLe1mXl+z=+Z(DEbaJu87(pwc?<A>-;^Q5gub590t$Xo=pKZ7LNk
zDoZ20jK4`OZHk`<jzK}$kiRJWN{-D#r2!9elKH{_k)krG#xki|Gl|Y?U_dPq>rh0|
z<!wVS6lRAqUT+KRJdZ{*i?|2-0g}i>`o72C?>pVh40AKB7cK!a7IZ%qXl$emjLf^t
z=J$$Gdf2BMyYLC>O+7fRd$KaRQa&l`WB)-Y@iqw<g_*CF%$e=Va5VoL)2|N8<310R
zCyv3mOv;*Te(jS<Ft$`=0}W8KQVs|*P&@;_#@@6$^~z2kw()4;1y2q}o4Q|I2B#(9
zwa2nDt3U`hFU<6F5C|%Oc1VJix((xGFvhRtt<)Cw238p6DNznOUJZUd8Q)y@J&)DD
zZ*#zv@4q`K3{7&HYMK9cB*nm)U`O~#WlV?v`NYxYYA2auup9D3QHu48PtlXHE9FZN
zg5kG7G^xdi?&!UV>SjXZ`$4B3uxkVSmwrOtISpYvAMNFYpA_kTGEv@*8@nK2Yl4dj
zB%N7}_mp@QogRb0vUrvg@7beJ1cG*;D}`XC!XZy`UyNSLWkmNRS2T=*S=^pmkKf_x
zUa}@Dnj$&_F~UGfbZ)yBhB6!I9SL6-rIcbx>adX+SYAt%!-<{^OuBu;TjJc3!xPf>
zxdlwqziVWA68oD@Nr23MFuwL%Ft2^8@1d2&{MB)h-QBem^|M*%`NF!D8_egRQ0!&A
z<+pGnisXC|Q@OcE7=lSyWhzc2@!NZ5g9`Gasej5r&=&O>H0PyNafm!f>Z?#4B5UyN
z(mk=>q!6MfV?iM8b`B8EV=E{DQS050=aus-jbNgVttM3?^7RE~j6AG+W&M{C<AgiB
zLTC!Fi7CX>|I`9pniD_WdM+qFpJXcG$)WqgbD8cX?M%9p;P%kJBNa}G-B{<^yisgp
zsoebuswdhzR-7O+Ci`RV(stz!M!EQ(<Mfd}D)yy=+`;t3SEf#}87(dvw{AHjNiH2!
zrk~iIy#MScitG{iJ9+_cK)tH)HYR>6tt^9z#<owN_k%j+ML|^PXLlGrxD`14nM+`^
zSAct<dnO~E(X~AM?W6E&l$lp@=M+~k{naan8M{oITgNXZX+tXTop5b!2k8EljCmi-
zNT#16F`G&FbAvw>h1+i%UYbl}fZKmiROY|dGKlC}!x=RAYwW;DV6wZ{IR%={bORrQ
zLswS03)}lO1l$Onn`wT{_#TcoC-b;Wrz3&(-1j^6r=Bi*j|_3sJ~HF={2Ky$kuLd?
zJ=yO!2Zz%i5{@1h^J?J2o3Y_F>~8rIez>vC+l|r#8M|bcgXCNPT6Cs%Po5vI?4C5(
z)<`*so1m`VwDJeQMBmDD6{l}8Ne?p&sq<S|yz0sQHTA%OhW+!uppjkOh6zp4S#J<o
zaP;2!Ui{V0DR;N<W;4I_Hlo$DKsQmm#q_3uC4$wZKrA(b2puf3(BSRSCcr#onCSNt
zSCbFV;=`p;{<ks;V7*MnSi0?Ax5AAPu}ET@dMCj+H|LHC_9qEm@&XBd644b_V2p`)
zNa-5<x>r_edp&EG=&+2SCJG=5eF1413%@Cs?V<oj)(sTLjM+r<{#zpjD!H`76Vp^%
zKm68NqY$U4pr8Ok<Azy;$DNrTK$fKfV3fpXSOu9ZD=4^!3`|qEIf@=U09|~lDk@-y
zR#8z|3cB+?*3YGW2K<&y!&xUH1JO`}5m;Ow{-Nk-icC%RZ2r{J3csv4`GR&<%2If>
zT<F0Gmqzz{wmj|`Gw*MZ@P^a!15z4(8$vUxYu8+e3EF;v{24?vb70l27SvY4r>4|$
zQ$=oXY;277ffmGP695teU@ZJ76hfoc75MPsL+~vSXn@i%)(XIJt)go|>llo)TDKY`
zm2^P#n#Hz+=yUniWSOa{!UF6SxLdHMNj|(Gjs9Vp-1n2KI<#~j;{)9Y3&=kpxxpxM
zc)et1+!OS18cfxm8J7E`;5|iKkA3AsCi0Z3@Q)usry{^_%WAv6i9oBrx_}WViGwV?
z+9zN6gJD2M;WJv-@x`#&+3eK-6#|(EAj0t4&oPQ((k+k^xN0}@aknuA*yKOBUWcU?
zp*l-#^iV?5kl+Zk^gRw+w~8}Biy_hi@tS}dl))}ol<Mt?#f;vx@2H%FtS7q0W*N;6
znwQ=e6;<?Go2X8cDI=GBk;no}DWKBXtxO@ltW<E-#c_;+B|4yBt+~h;yLniZB+fj%
zbK`Mc>vMgN>IVNWXBP!1fjPc8I`VG<g}V<PDvb;dX7w%XzH1QXClCE?Ui$Lo%dTTc
z{S3dui(ivS&+zs#zeA@7b4pWg$45{P?yvH5*RQR4-g!&O4t7OU+nhCSDZgS3WWPme
z51?q$xf9g#K%nzR*{&b@y4B~5h`P4p*jW~nn*{H`-XH`N>|=Kfh+mkRmZ+5~D=Y8H
z`T5nin*$9AbAJ#0##<#LbY-k;!JUiw-@nJ~k3GE11&B=W%FfRI`t@s5lg!sm%PU5x
zx*v1&9JOXXzE(ax9lwCdcVC8;cp0;Ry|oha1A8w7ztT!{h_euFVk@jH(r5Y?y5o9Z
z>U*DcDCG%Mm{Q6mDCrAJvkhGf*-e|yX_S_4y>;XE`TFPNk^v>N;qBCM=A$IxTD(KV
zbJ5+FKu7;P(Ct8a`blNjN`TT-Z>fy)NTMJrid<sVt>fr5O_d#LT)3(JHitVG9j~ru
z!Z+QdjRe=|`<cvkGOGPigJ8>Tvlv}Dgpqz*L8Qm&+4sx9r)~$^?)U<I9B#S`lr$1M
zgP^N#W-A6x(JB6bfwb?>>#mlnsF<_w!2Uo(o<TnK1K8(hpg<SgEQCkD3te-Ifcjdf
ztk2_OweNfvl_&LV;U^F9`|aFS%2#alro^$<g3nBg!d^n|_mI*N-hYLMB$N~>Z$SNO
zzbKp?0+tR>0hdr0h|WfH;lc|_enuegc(9seF|i&k68mqn76-HV(52$AwK22U)d+sJ
z@NGsQu&yIZbZE>}%UHJQHm!W?7fN<EdDv^#^rTp6m9BoK=>0%)d>7yw@$=rw0@gCC
z+N$Z?oE(e8t(o=2JMqfQmy_u{6l<3MRMs`BYkL8Clg&dhp5pY3FR^JjrO;x3TvDJr
zKv$1v>><dcc-xnui0#e00|RgYu(N!uy5LYi_$T`g<0F6I4tkr`|5k|pg+oO3E5M@(
z)E0s7ARFHG4R1FWP62#Xx8;y$Ir(Jt^eCX-z+)z!T5M91hvtAj>4E~4Kmx&(XCe}Z
zuD>XmWctBoSD^EOOf8`bFx+=smajkiWhQt8PCPi9k&e0i;1II#2EXHeDvkl2rX!Ws
z6{LAEmQlB>IlKp;Q<y&)FbM@sO(w>KL#t^0qA^f0jC|_Z6}ai?2YPnXVBGHR6^Mx4
z6hJ<NgoJF*^izR2KIGhpE752G_S7dRQnklyq+iK;o}QrtahY^QcU}>O84pD=iiuw(
zsDQ|#uFKUlr~Mq%((j+&@wNsy5x3yd-rApQ{GMV-4JNm2%+t&Qv!0h$yka7K9rjUV
z-YbDWpjL;tbYazxu0vmJx3w0K*u~y{sG*bbJDvlK<V~ExdKUMa_AeV69m2I5s>;@!
zQhqw1cwR`pQW{=C;d)AM%Y}i}qT5rooqS1e33LStBALiLA)K4Fi-2N|X}7}L{;CV#
zw_C%GkpN?ev*Ik<M4Ouzw(EuNK-k*aA3ZX8J{55eR_@&GNMI88SQy%(tJeu^1W4D9
zb()8#7R2_zquh;}uh2{ltnfZrsLO%)o*f!ix5xdgJTaAzhY$c98}=Q#7mYGCy7jFk
zukrS99gZ`_RW}R27g<7giqW7m3it%oYDcrn`v@)j{=mF?Rbtn#)m+8<wnJngXW(eF
zC#bcfck8KnKBJ)55<-9;Wk5Y`fBdksodZBX4tBTA!G;}AEmSvbew;%8<9kZGSf|gy
ztD*v?Eo_F@T1y9vdbJb4RJ3LMG;9cv21$AI$ib%qviM_Boo=Hg&c?CgteF_Vm1{NE
z@d0M>mowbb<e?gc?@2C6kAlv98Icv%Cb+*R0Nns=P8zozE!ns;D7}_+L^1ST=ZlT2
z_@+lpg}EbFrI?YOu)4&8L?TEok$ZZz>z(Y3(Lr3TmdD}!-K2>MUTCpPATD|Hsaxvd
zOkG`Fq{Z7_1cIG95*{tEJ~w%8Sc%M5RL;!a`!VYAGFD2HYhI$yHqcP)(()=z!pQ26
zp6t=GBn#6iJan;89&qK{q;<~31G%Rn^i``%yJDB#m@V8Uc(y<AT}T^hw`beV0oEl+
z{n1RRBcYP_x=P<9=L%4X*SO{K<t)B!%ZOk5wg5E8_CX~(0S8{jrT`=;5WLgg%O@o#
z_H#ycjzPS<`sUA=(G*1W{6vASa?q?4aL<l+m`hL8ci_$}C>ps<2T5J~etxy->9)FP
zb(sC8k;0@_LzMhAe6;qV)MM(3m6HD7KObxvBQT+aq3UZ8%GkQShq=WgX=kenh0w9U
z$HY5JA1D^>8M7AwFohVi2Yb&qQAddo+yFal3_)(AsiFBZOTMG}jgO<kTVKC&)G7va
zMJwODapMN4-*F?%8%_ZY3T5)3+Fw&M4rtD4YGymtejk}GSLS~va-B@Zv!S+cPPUzy
z#kTpUSHttK#~EthJhljdi+$6k!mpRHKeYG(KJ};P$7<7&3$Gz4WjQ`R1`R!#1G+C>
zfWBsSr?J7o7cXACD%AsIwc4Hd{v^^VY!QX6`$6l@<2CQ@hmcjJyya*~H0ix?9W?(r
z(b1;mnP7Ex>w<*e2ciA5_0CgOi}fAwgdO1308j#o4+E<Xwzgq4DX?W~Ie-@)GDRI9
z`vQRFSx-0bkl-W;jy|iEk*Dteoj;T%ht_#6r*qmqv8(^|;#CDa^f9~nO*_fv?C(zg
z2%gn6n)h8%8BE{7!;iITd*PH|$}*L<b11Uz-2B(w6XTuJ2sTlu=h2LfhL3qZi^Rmt
z{>09TGmG;U!@%<BA%q>cudol%1TlZp(#!I<0ev5x@xiaVHv+XKC{t_kK}fr;HVTY8
zUK<W{u@<hJ?`}FTUxLvCk6u61^vN{GBWOOcjE@K|&@gte^IVWS*gLO&BYmsbNa5~4
zH=n5W%3YP>xX*dqj#zA~+0Q<dGy0GffsAETb%^Mo+u~8~+Em7`&XzU+RaTganIF20
z;MTJ?jD^~l0g$I}uD7r|H7<1k?K$@LOUq!!fL4K{YILU-xIf@#xrpQSCMFKdb@VHI
zK}0g6lsEb-I^)r!D%L?R!~OOeHx4||zZE-l4>o)c2JF$uQAQGQ3FdJ1Nxd46sgMzY
z#BUH@fHx+Po}knGT=n4h{GZ7oOJN_pmX^Mj7J%B)EEE!w04mQn#eryyd2{RG#Mu-&
zuo-)>EKv*owK?Fg;*%9VO!(jou)ptd`NcTSl>B(|F$1>+3neAcIkvgB_UY#k@CBgf
zUHj!rJX{?gM_=szZmeHXS7y6!COENza^5*0VS}Gy-<k<(ya18bOpGSv{!kVLo^)AB
z1MK9t;VQ*C-ddnCGCDd+B?eGGSQWcEDKN&HfSH`PlPnE?d{J)%dLVI$K(I6ASuTJ=
zKKdra`FlWsL#I&t01@7`iUz7;-zD|a)PP;x5dv*i?&(b5K|9o)`0iyl@SE*}Tb}jC
z9k!L`Pmn2&!YQEewcUKO(XaaVJdhIs?QP9(82|Bm#l^+yc&VuXnM@#qW^aEr6EAg8
zWeyl!6%%kqBoYaP5wy==09}aE&$pbnDAW4&lSj9UIZZzo01qtN?*#X8djS&OJ~T4Y
z7JKPU%45)5{T~^b6;Z$|nMIYIHMn14S*{MsVY0{XS1rGEV_r=pr4}^zt(DXVQ{LUk
zt2_Q%u$Q2XxXa{z{*8-=k><l<AG!3<r$ZXJ$Hb-Ps7$vS0yA=eP{85ALBKn>bN`|A
z{x`hf`Op9RApdB^^Y=Hkw*awbmT|RVqjtCPIAQ;C$hFIW04<a<VlQDu6NO+)F>oT*
zXnz-e3_to~b*7>DOL3!f2JP81pBfHK@g_h7QU2ae+gvElIXhZ8qZ8;Vzi5M)C?%!b
zS0vVdjAw4)(FFo)7T`QrN)xVK=9!(X$5&n$;nH=9g?8s9s1F(h0kYW01T2T3Z&f(q
zWwQ+M44xLbuE}JU{KQEE_E!S|<Fd&5y{E`HGAqQH1IinJW&jQ%=VRZ79ERX_&IV1r
z1iGJ>9pY{fU2Ty{^jPe;7IpCrhJU*_50wh_6Y(q=FeQ6Ek=$g&6Rfv`ZbTnz*Y7s^
zVi4RuqaQb#8xy`x2i|s3YYGE2K980Bg;mivJ|98wLHc&rMfsS9q8o<i_fw+K%i1SD
z;H%Zi!IZ+UKx*4$=fIugTwLi>K#GQJ9f!Q~)S1t(NvXK9td>3hWj-9Q75C%`?qjqv
zK2i`A!=^DLdUVQnpk#ZpKA8?R)%6|i-CmE@7tq}iA~|0~F}-=LiRe{9lt~RpE65Qq
zvlYToO!!|DJo|vaI1{tUsVEVcVY2~4*{d4zUR$el4_k>pfWoCWmxE{n@pk#{1<*46
ze=x{+a^^n<i$B)?%^l-U2^7Gu{Wu~f5Gerr5<S_vv%3)G8VQ-kS*c1NjldT@NbfT-
zdT(QbTcvtCR0QVK)c$}5YW@5DU@ZJ?OP}lyIbyazI{Gc<W@{Nu!sEz+g&HAwn$B^0
zkxJPP>a}yJs&KjRY5|V>6qkY=1Dh$u;+|-3K9=Y+;}*5HFX<Mf{}|b^&F7vtBBd}m
zXXraId*_uPJp_H&8F;CpgW?yTRBY7<$(opxG_gH*^fGvw5#HHwzuwfp_J^y{BqhVp
z18v@Y^ab+t`55_F-1W*%ms|8wvMg(r7aF+)r)kc#R1C^qIa;q=W81y1!1C-tm<T^P
zekqkJCyB$P_!f;d69)C!+HU3u&BU<ZPoy9#>G$W>l|@%z&Int-_)~?lkLIs(t7K$b
z(rYd$F{d=#Wk1g~Tne>g7*Y3a9E_`A3f56lwBk*jCt!AjoAVAc+#a)grchCkNI9q&
zW#j!szbSXU=193j_Xz`eGDcEExlb#>s1N6phhOB6Mx+9SD8*5zW_n^HGP3IZlhEQh
z;ocO<jPE11i81Wf+D{+2KQ(^xlsHQCs-zt!NnhphWybg~fsB+wZ4U8U_Zzu)mt%7N
z&J#h^6s>(fzAB$gb9#DL^rrGG-fn2~0Wf1)kVfpAkfqw{nfE6NIj!d;!q<NA8Pn5i
z*a@j`l9IHpf3Lk4(9pp4#9ZFyqCzTreo=)^_u#U(e1oU?cZ0&5rq5bZ!fbPHWLK01
z=8hjfl@1@H7Xk6QQ`6OR)Apewd@BmY`9PuGUroS6WA5hLXGB%+>8O3IR(HQXAFHZL
z{xLLI*3}{9l0wVQit&D>4$Q**R4~VSCni58{^|VgAV9^tu+SJG6t#PmROiP?MH5rJ
z<3bB-S9_npPtP2&fzj5KFC#}>ukrVAFl&}G7MaQOK_(!gp-*hwM$=T5?3r|@A~%|I
zh2Q1PJ!BK^h|lW-lS)tiVMgL(+1Mw2$o-_x<sbtFykGANzS$;#W0ajuQ|C%nK7n&G
zJ?Pa+Akm5yeFpn`qm=itB7Y^X-j-*-j+L3)YMki%4gVi;R#Ll!+W8WaNyCpatodeT
zQffzOAYj&AbIGO^`#JIBEw>h=mqH*Nb`C1{d<H1?3Q1}vJ~60j#?B$YBR^o-@3!6E
z#hh}i<wWc(6(MKoO>T0)-hOm0dh(IeJ#+7%n<>*;5F%=m{JC7U&-x-hUH2yC&jM0%
zXli%?sTHsGLBg+m@&_)x{<O!W--C=HGl{ddjQ$brB6Y4%?KYoCH7_|yCehsip)(q7
zorm4HCH;GZUOwth`4XwQocMc1Z3hN8H054Vt;D=;Xn~)T>eVGh(zf;S9Wcj^)3G@b
zpkGy@>1jrX2Nx;~eQQ|>B<ayNEpksI@@Q(J_YS6|kRrmW<$d+Lr1Z0R!M2z;kN|Kz
z+mJng{X-}&6~r@F22Hc~mqE;|&Aqy8t3TJO_Dxa^3;`PfF)<)gclm2SYG%Z%!tp<+
z%mkmHJiPKJef`vnd&j7P_6KKKke7s03YL?1{Trhs)UP;UFx%zIF^W0Qrl<xm06Y@a
zLkn|}#+b{JXkFI=d}c7r9$eYP%<C^_G%2Dz?rvTybiu1+5)I6n6$TG8iX<XFHLv2j
zciuse2|-AS+If*%R`9VVeSnkQv4p|kxj}FKc)NmVB-5lu&_nuDAvcJWP(1>~`6K|I
zeoKm43#Kq*ySxR?Ev%@-Lj|aL0JePbq2dd3QD~kw^9x43iX^~L+A`ZVKi`3Q9ag5p
zRJH9<JQAmWe<vywjt0-Hpr8i{s~T7gou7t}y6=hS3SEOy)$(tFKs(%+Blr%T_OAns
zKd3eWd@10&U`c#H85Yo8>i|g=)q9~#sv!EH*a84dZSBN^F+g&G7kJ_<#zH~ieZ!q4
zB|bpLG?v%{$^`siJwx8}TRA$$l<R=V82qv;h~FD=<@Z6_G#)AjZqMV;?~wp*tBBDD
zjs?#1bVf<}_6B@8dso+Gl`nDf{zN4Z^Y^#k$MGu6*__nuDZuDqs$+zD+i?KFRa`i3
zn9D5W7SL0byC-jn2s@pXGC*`?gTZB8Q3K(F$3vhj3Y2IqfAQbF`=T0POi+LpQ~mi_
zZkx<C@CZQ9dho{|IvWuIb)ZR>KN+^T(>oE=&)nY8Yi?m}HE3V?qzyBO7>CQuX2jR$
zd8hyA=+{aBshNA>TQYyJ-Az^QAhTf!n1Aru(QQKBd!V1QCMAnia&}sTX+o>vU^|Gy
zfzSiw0$P8gwxpLC$gTkD_G~-c)b|?%#;!2|(Bd6P=<mUi5zF$)U`in5V3JQxFg`j8
zZU7)NQyKtpUGPn~bFG#p@f<?p%Tw~jW~DXG9#=N6EF|OsBPpbiJ=&ggo~e@NIt1Z`
ztmNrdd7jkzch9=kpLyOh0<8WuZE1}q$Y~HY1;x^SB>MRrz}iQEl7jp7ly}(rgAmiP
zyYv50M)!lV4w~YCWi#>Q=j6znd@m_UMz~LwSplWC%1Th}4<uvRg8Qrsi;I)0dmI+{
zN@FIa+2gyxB>*fOaRPMAUNpmhNUb&8@r&NPYf@c;T=c4|kJ>xH$Dsk`OLu#)MN{Mz
zrY?aV$EBtZP`J}7TZcLPW~BEVe5)(C5xPMLhhj(9V`T*l&e9iNa|GGi0Sfcevpea^
zOZFeas(2+|EcK86kri1AQtvDgfiB<eAw5jo>=E!jNU%ZO)c{le{zQg{p@7%G7;;}`
zUw~ESpm3$Sr`o;dP5T=+DznnUlQj?TUw|}RJ;TCJRP47VpZp@vB0tp8)^0Cr{L%EB
zl<(*B#Lwk($M1@y&L<WmCqk!RHo6XD-eO}mF7UntcRL&oc<|Xrf1RsB0^jqkifaNE
z1I*AA(rxG!SV17R#YS|5J)j2#kUB<B7^!tOguoMg+s#u_OdWp_X_G3*-K%2z#z<C@
z;OMjIjRqD47QMrqK5%W|F*m0bXBXpdy={LJ3o&~ET>7%2!Tjk{=Yg9!vB5<-fU!Fm
zi;?i7uA~7=4Nw4ZeonS~9o=zTDOu<*;4a*~tL&I2*EROSeLxW-bG0_mCb0Fn;8QJG
zeBSnLJw5T#y?m+?8-0cV5PdkNaV_p<7rX#fq#TwTKo0OdrS#kRm<<uKD{Pk>74aLv
zWvBAV&7FhyxcKJDw1BVGq1=M6`oVK}fuvqUi44v7jHBojVEjNy7*$agD3vZ#?e6{K
z^L+A`hKco9PE*i~F(BlAswd-N*gK7%07<o6tQLVLQZ76?;bI%tr7dE)dWBUID<k12
zx4*yd_DmNU&#X>s27<xfLy8+^?=$}-INdf%ZN^nrMT68J+bdr|djBHiB6WF6O56G2
zlJmF$^#GEC@r2F=n(hY~3zgvhc1ATrg(&|%dJmR4=zcFU(3uK(5}pB;8`rx5fnZ1g
z2-5wRCqPpXkJVx5^d6v2p9|`_58R2O2HV5W?a}FSJ2k}>z(K^MHRgF6?@OolpPWUr
zxDR{qJ^|sInp$LC|3XA1mY54B`YZE|L4WLa9l3r8Kb`LCvp;q8#8RFOlRo8KE26b+
zb@%D<Gz`YpSp!+LA$>uw#de;E*AsX}?0~c1=Fy4U<!g?*n}Fc@X6-(sWV7`AFk|60
zpt&%)hQzO`r+`NQ4P3*B3LriVPs(Yq_mcG~e7IIv{>l6aTTVN?9(Di)=HTXG)YuIk
zf9waMSrERz4%$}!2v3JiOU(y`7=frg<>Y&ZL+v!E+6L|`rau7S5kQ6bUcBIIelc7n
zuK%7D0c|v(CAM;g!Or-NfefyY^$py?6S&M4$t6Ha0{;Mvlon%aq@<(-c8acBr9fZ+
zeG$0aQ5mm$@!cb{v;0$bT%4SC)g>S~GN^-ntOF%ub51<GHdzi|ZCgbrCL{z45FmEE
zm#$EG&5n#Lc|7Fu{}aOn5l+EwhsyHqiuHrl1y0K1oJna8j2BD<8{vf5oKANL-4D*G
z-zy-RZvlCTTfvlI{(u7jdl~0%vkAW*^qNq0RXeKq^xba?nNzpPq`sBS%u3TEKnjD~
z#tPUkTUaF!aqL_yok9xVH!Q-d)>NWHM(syuLFWw<dFbb#G%%;?LJ~vUn;BgfseyZ|
zo*E5K=XMh`G=647^>Ank4a_9-6fJO&v0H&T!8`WlAc|%e(+{725(TvaZy|dG^O{3!
z-5cn3JT4Dao@g=u{Z#agA=#B6BDgFD@t#zEJgD|#88E1>?+5LV{@OYZygkqW_W&Ob
zl#Zk|N^^<%D=?b9ZF~USA`=oRfj}}|z4D$CuqwdH<%Y_6nI+)EBHq$^{{VpVInzj$
z?O8z`D++~%kNf>RsRk)swkMxke@><u19I{~Y6|@QWwfIBKaA*oQnBg-;MvjXB}c^5
z4-zMm?G7Kt@i7j&<Uc6GATUNBS--2BxVmItsNQ}=oJq#+<}G<1KRJn}0$kPJ@oYuK
zoW%}(SdGEM<Oh1Nc+)^*8Yjj+5cXXmRDSaKWk%}q_Cf4yMwXdipV>r4yrF)@=YbSF
z9=ziTq7Sf{0>oAM#8MYq5O#YEp!uYo*yfC#k4Qk4iCWV#Mp`=c_PzWUsK_Grt1u^_
z9du^*glwEeQ$esNmRkN>!@#kMRt^x)icIIA@Xe=%uGTI72ac=vRL%jYr1{lY)5=hr
zWFv)cZA~CQTdQK9k-eSx(0a&B@OzZDDcL2<*P$Hawl8HV6;CUCDyl(+g_noFf#Bp#
zFe8uN4SRwa{d$<)xsf>D^{P2+F0b&yx4t=Opw!%+?Zf!*rIP22RL^$b<BpL^?~^O<
zTG3c6s55x_HHpiybnsK&S4GuP_Mv1E2_V8=b~~`@65g#|I)Yg1WgebBY8I{UTA)m?
znO;Bw=e&0*M8mf+sP#o9R@=bhelDXXkkAB=xOqd5R=mIaL=z_<JD6{p5w_!V(dQ$3
zv(zB&7MHAiPZ{L(=L=|Gd<8?V^<NnjN*00CQmR?J*X4vhq_`5~wpZGRqR^NXZY5>2
zJTkH9?G>aSx^~5FfL)>$ggJ7=K2BYNGc@#DwUwszy~7Mwf{u%x@^WRIzWdgP(p9Q=
zziw3|6mN#NG&3iFL1)PL&jMe^7#5M~Tl6L%Y<`{D65W0MtNL@ZRh{PT8p?N*N$B_f
ziQtJ<!QzPht8BmGD{khFY8aJ5XR?y4Ov%U`v?}pWOPhuCgTaj8sITCyV&xJNBI3A0
zEBAvz;A5n;r-OUL%`QtUtz7U3NpPME+Y?X|g0B8?k+nIFSH^hB>pEOlt;vP3)vSW7
zD1!wNk`P0o?y8LpePWM9fP~_8hdYwOM(1Om9j3JeXFUw*3rdiv8A+2y*i$mBEH#Mv
zWI6huN?Kkts(^3_z~oH&)WnFoIx~?ApK~YOHnOxjGvwXM13ku?{>Y)1gN{>d+?2Fe
z?9x-B9E63@nm6<t+D;Ad5m-2nCs+q`p@i^3po7}oz$3uH!J)wPYP7>OOY%O8n)27|
z%MVo46Hv}1J6hMN-`&2733!n5z)fGBw^m~5aI{H}1k>^8gfoQqE=l*WU3bXBGmBa8
z)Tiv{BGl8=lvXiu@?Ym~{}CR|_fbz;Qg_I{%Fv@-7@Bvxy#G@i_qX1%Y5i4^mWuv|
ziJ=CDQ3C7%q_H{NK8vOSU3p5kZ{o-J$?{+*7&Qn?)A?)9r4;5--)JA98)a76nn^Xi
z%Tz}D4#9f45<MEv#X2?+G)Io-MRkufo`#D_9Helx)nc((8?hVGBX9@D`O1gnD()@$
z4ObV&%oF1mjhSEfIFDIc=H_vb#&RhAu?ktyKBq$^)R;MR1k(0=-`B))#Cg1{Q8Cn_
z5@yt#`eu3Il858-q!CQcy$AQh=~u*i2-0KCYqBIfw9;1o)lDX2G$knu$1ZV^uz4tT
z?%U;EStk|<iPEhY+NR9t*2(*xC2>lE);?#QZe945>ixuFBg@f^D>!CxEeqp)pMJ>5
zih3ySnuEygOVcs3$M&#o%6WI96t(li6A+k<@y=7?i77BS{~<j8i>t&F<^LhD{*hV0
zM#1Zx!20(N&@1@hzkdF|;Y<I=LH^fc`0tVYJIMJZ{P%_YyM+I`kbh_YA7l6*7xHfg
i{{QFU-x$n3pF#ah5S(<AQ#1HEQ&v!aQY>fo?*9V1WPYmv

diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index 39e54ee99..84c053660 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -29,20 +29,20 @@ Data flow
 The data flow in Scrapy is controlled by the execution engine, and goes like
 this:
 
-1. The :ref:`Engine <component-engine>` gets the first URLs to crawl from the
+1. The :ref:`Engine <component-engine>` gets the first Requests to crawl from the
    :ref:`Spider <component-spiders>`.
 
-2. The :ref:`Engine <component-engine>` schedules the URLs in the
-   :ref:`Scheduler <component-scheduler>` as Requests and asks for the
-   next URLs to crawl.
+2. The :ref:`Engine <component-engine>` schedules the Requests in the
+   :ref:`Scheduler <component-scheduler>` and asks for the
+   next Requests to crawl.
 
-3. The :ref:`Scheduler <component-scheduler>` returns the next URLs to crawl
+3. The :ref:`Scheduler <component-scheduler>` returns the next Requests
    to the :ref:`Engine <component-engine>`.
 
-4. The :ref:`Engine <component-engine>` sends the URLs to the
+4. The :ref:`Engine <component-engine>` sends the Requests to the
    :ref:`Downloader <component-downloader>`, passing through the
    :ref:`Downloader Middleware <component-downloader-middleware>`
-   (request direction).
+   (requests direction).
 
 5. Once the page finishes downloading the
    :ref:`Downloader <component-downloader>` generates a Response (with
@@ -62,10 +62,11 @@ this:
    :ref:`Spider Middleware <component-spider-middleware>` (output direction).
 
 8. The :ref:`Engine <component-engine>` sends processed items to
-   :ref:`Item Pipelines <component-pipelines>` and processed Requests to
-   the :ref:`Scheduler <component-scheduler>`.
+   :ref:`Item Pipelines <component-pipelines>`, then sends processed Requests to
+   the :ref:`Scheduler <component-scheduler>` and asks for the next
+   Requests to crawl.
 
-9. The process repeats (from step 1) until there are no more requests from the
+9. The process repeats (from step 2) until there are no more requests from the
    :ref:`Scheduler <component-scheduler>`.
 
 Components
@@ -102,7 +103,7 @@ Spiders
 -------
 
 Spiders are custom classes written by Scrapy users to parse responses and
-extract items (aka scraped items) from them or additional URLs (requests) to
+extract items (aka scraped items) from them or additional requests to
 follow. For more information see :ref:`topics-spiders`.
 
 .. _component-pipelines:

From 33d04684e5a3a101b3ae1013878c46994b9963ba Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 29 Sep 2016 10:25:21 +0200
Subject: [PATCH 1108/2974] Fix typo

---
 docs/intro/install.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index bf2d4ec3f..7c806a049 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -149,7 +149,7 @@ Ubuntu 12.04 or above
 ---------------------
 
 Scrapy is currently tested with recent-enough versions of lxml,
-twisted and pyOpenSSL, and is compatible with with recent Ubuntu distributions.
+twisted and pyOpenSSL, and is compatible with recent Ubuntu distributions.
 But it should support older versions of Ubuntu too, like Ubuntu 12.04,
 albeit with potential issues with TLS connections.
 

From c670e3cdf643e36d00de4619c09968b5f8dadc81 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 29 Sep 2016 14:22:08 +0200
Subject: [PATCH 1109/2974] Move mention

---
 docs/news.rst | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 882f3eab0..450ab5542 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -6,19 +6,6 @@ Release notes
 1.2.0 (YYYY-MM-DD)
 ------------------
 
-Requirements
-~~~~~~~~~~~~
-
-Scrapy's new requirements baseline is Debian 8 "Jessie". It was previously
-Ubuntu 12.04 Precise.
-What this means in practice is that we run continuous integration tests
-with these (main) packages versions at a minimum:
-Twisted 14.0, pyOpenSSL 0.14, lxml 3.4.
-
-Scrapy may very well work with older versions of these packages
-(the code base still has switches for older Twisted versions for example)
-but it is not guaranteed (because it's not tested anymore).
-
 New Features
 ~~~~~~~~~~~~
 
@@ -58,6 +45,19 @@ Refactoring
 
 .. _w3lib.url: http://w3lib.readthedocs.io/en/latest/w3lib.html#w3lib.url.canonicalize_url
 
+Tests & Requirements
+~~~~~~~~~~~~~~~~~~~~
+
+Scrapy's new requirements baseline is Debian 8 "Jessie". It was previously
+Ubuntu 12.04 Precise.
+What this means in practice is that we run continuous integration tests
+with these (main) packages versions at a minimum:
+Twisted 14.0, pyOpenSSL 0.14, lxml 3.4.
+
+Scrapy may very well work with older versions of these packages
+(the code base still has switches for older Twisted versions for example)
+but it is not guaranteed (because it's not tested anymore).
+
 Documentation
 ~~~~~~~~~~~~~
 

From fd0b6faecb97ce34a4d5472a075d9487adceba8f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 29 Sep 2016 14:36:13 +0200
Subject: [PATCH 1110/2974] Change release section titles to have correct links
 in ToC

---
 docs/news.rst | 184 +++++++++++++++++++++++++-------------------------
 1 file changed, 92 insertions(+), 92 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 450ab5542..b922690c1 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,8 +3,8 @@
 Release notes
 =============
 
-1.2.0 (YYYY-MM-DD)
-------------------
+Scrapy 1.2.0 (YYYY-MM-DD)
+-------------------------
 
 New Features
 ~~~~~~~~~~~~
@@ -69,8 +69,8 @@ Documentation
 - Add StackOverflow as a support channel (:issue:`2257`).
 
 
-1.1.3 (2016-09-22)
-------------------
+Scrapy 1.1.3 (2016-09-22)
+-------------------------
 
 Bug fixes
 ~~~~~~~~~
@@ -86,8 +86,8 @@ Documentation
   (:issue:`2236`, :issue:`2249`, :issue:`2252`).
 
 
-1.1.2 (2016-08-18)
-------------------
+Scrapy 1.1.2 (2016-08-18)
+-------------------------
 
 Bug fixes
 ~~~~~~~~~
@@ -100,8 +100,8 @@ Bug fixes
   (the regression was introduced in 1.1.1)
 
 
-1.1.1 (2016-07-13)
-------------------
+Scrapy 1.1.1 (2016-07-13)
+-------------------------
 
 Bug fixes
 ~~~~~~~~~
@@ -152,8 +152,8 @@ Tests
 - Upgrade py.test requirement on Travis CI and Pin pytest-cov to 2.2.1 (:issue:`2095`)
 
 
-1.1.0 (2016-05-11)
-------------------
+Scrapy 1.1.0 (2016-05-11)
+-------------------------
 
 This 1.1 release brings a lot of interesting features and bug fixes:
 
@@ -341,8 +341,8 @@ Bugfixes
   to same remote host (:issue:`1912`).
 
 
-1.0.6 (2016-05-04)
-------------------
+Scrapy 1.0.6 (2016-05-04)
+-------------------------
 
 - FIX: RetryMiddleware is now robust to non-standard HTTP status codes (:issue:`1857`)
 - FIX: Filestorage HTTP cache was checking wrong modified time (:issue:`1875`)
@@ -350,8 +350,8 @@ Bugfixes
 - DOC: Consistency in selectors examples (:issue:`1869`)
 
 
-1.0.5 (2016-02-04)
-------------------
+Scrapy 1.0.5 (2016-02-04)
+-------------------------
 
 - FIX: [Backport] Ignore bogus links in LinkExtractors (fixes :issue:`907`, :commit:`108195e`)
 - TST: Changed buildbot makefile to use 'pytest' (:commit:`1f3d90a`)
@@ -359,8 +359,8 @@ Bugfixes
 - DOC: Add AjaxCrawlMiddleware to DOWNLOADER_MIDDLEWARES_BASE in settings docs (:commit:`aa94121`)
 
 
-1.0.4 (2015-12-30)
-------------------
+Scrapy 1.0.4 (2015-12-30)
+-------------------------
 
 - Ignoring xlib/tx folder, depending on Twisted version. (:commit:`7dfa979`)
 - Run on new travis-ci infra (:commit:`6e42f0b`)
@@ -411,14 +411,14 @@ Bugfixes
 - Small grammatical change (:commit:`8752294`)
 - Add openssl version to version command (:commit:`13c45ac`)
 
-1.0.3 (2015-08-11)
-------------------
+Scrapy 1.0.3 (2015-08-11)
+-------------------------
 
 - add service_identity to scrapy install_requires (:commit:`cbc2501`)
 - Workaround for travis#296 (:commit:`66af9cd`)
 
-1.0.2 (2015-08-06)
-------------------
+Scrapy 1.0.2 (2015-08-06)
+-------------------------
 
 - Twisted 15.3.0 does not raises PicklingError serializing lambda functions (:commit:`b04dd7d`)
 - Minor method name fix (:commit:`6f85c7f`)
@@ -427,8 +427,8 @@ Bugfixes
 - Fixed typos (:commit:`a9ae7b0`)
 - Fix doc reference. (:commit:`7c8a4fe`)
 
-1.0.1 (2015-07-01)
-------------------
+Scrapy 1.0.1 (2015-07-01)
+-------------------------
 
 - Unquote request path before passing to FTPClient, it already escape paths (:commit:`cc00ad2`)
 - include tests/ to source distribution in MANIFEST.in (:commit:`eca227e`)
@@ -437,8 +437,8 @@ Bugfixes
 - DOC remove version suffix from ubuntu package (:commit:`5303c66`)
 - DOC Update release date for 1.0 (:commit:`c89fa29`)
 
-1.0.0 (2015-06-19)
-------------------
+Scrapy 1.0.0 (2015-06-19)
+-------------------------
 
 You will find a lot of new features and bugfixes in this major release.  Make
 sure to check our updated :ref:`overview <intro-overview>` to get a glance of
@@ -803,8 +803,8 @@ Code refactoring
   (:issue:`805`)
 - rename "sflo" local variables to less cryptic "log_observer" (:issue:`775`)
 
-0.24.6 (2015-04-20)
--------------------
+Scrapy 0.24.6 (2015-04-20)
+--------------------------
 
 - encode invalid xpath with unicode_escape under PY2 (:commit:`07cb3e5`)
 - fix IPython shell scope issue and load IPython user config (:commit:`2c8e573`)
@@ -813,8 +813,8 @@ Code refactoring
 - Converted sel.xpath() calls to response.xpath() in Extracting the data (:commit:`c2c6d15`)
 
 
-0.24.5 (2015-02-25)
--------------------
+Scrapy 0.24.5 (2015-02-25)
+--------------------------
 
 - Support new _getEndpoint Agent signatures on Twisted 15.0.0 (:commit:`540b9bc`)
 - DOC a couple more references are fixed (:commit:`b4c454b`)
@@ -839,14 +839,14 @@ Code refactoring
 - Update request-response.rst (:commit:`3f3263d`)
 - SgmlLinkExtractor - fix for parsing <area> tag with Unicode present (:commit:`49b40f0`)
 
-0.24.4 (2014-08-09)
--------------------
+Scrapy 0.24.4 (2014-08-09)
+--------------------------
 
 - pem file is used by mockserver and required by scrapy bench (:commit:`5eddc68`)
 - scrapy bench needs scrapy.tests* (:commit:`d6cb999`)
 
-0.24.3 (2014-08-09)
--------------------
+Scrapy 0.24.3 (2014-08-09)
+--------------------------
 
 - no need to waste travis-ci time on py3 for 0.24 (:commit:`8e080c1`)
 - Update installation docs (:commit:`1d0c096`)
@@ -878,23 +878,23 @@ Code refactoring
 - better testcase for settings.overrides.setdefault (:commit:`e22daaf`)
 - Using CRLF as line marker according to http 1.1 definition (:commit:`5ec430b`)
 
-0.24.2 (2014-07-08)
--------------------
+Scrapy 0.24.2 (2014-07-08)
+--------------------------
 
 - Use a mutable mapping to proxy deprecated settings.overrides and settings.defaults attribute (:commit:`e5e8133`)
 - there is not support for python3 yet (:commit:`3cd6146`)
 - Update python compatible version set to debian packages (:commit:`fa5d76b`)
 - DOC fix formatting in release notes (:commit:`c6a9e20`)
 
-0.24.1 (2014-06-27)
--------------------
+Scrapy 0.24.1 (2014-06-27)
+--------------------------
 
 - Fix deprecated CrawlerSettings and increase backwards compatibility with
   .defaults attribute (:commit:`8e3f20a`)
 
 
-0.24.0 (2014-06-26)
--------------------
+Scrapy 0.24.0 (2014-06-26)
+--------------------------
 
 Enhancements
 ~~~~~~~~~~~~
@@ -973,15 +973,15 @@ Bugfixes
 - Testsuite doesn't require PIL anymore (:issue:`585`)
 
 
-0.22.2 (released 2014-02-14)
-----------------------------
+Scrapy 0.22.2 (released 2014-02-14)
+-----------------------------------
 
 - fix a reference to unexistent engine.slots. closes #593 (:commit:`13c099a`)
 - downloaderMW doc typo (spiderMW doc copy remnant) (:commit:`8ae11bf`)
 - Correct typos (:commit:`1346037`)
 
-0.22.1 (released 2014-02-08)
-----------------------------
+Scrapy 0.22.1 (released 2014-02-08)
+-----------------------------------
 
 - localhost666 can resolve under certain circumstances (:commit:`2ec2279`)
 - test inspect.stack failure (:commit:`cc3eda3`)
@@ -1009,8 +1009,8 @@ Bugfixes
 - fix 0.22.0 release date (:commit:`af0219a`)
 - fix typos in news.rst and remove (not released yet) header (:commit:`b7f58f4`)
 
-0.22.0 (released 2014-01-17)
-----------------------------
+Scrapy 0.22.0 (released 2014-01-17)
+-----------------------------------
 
 Enhancements
 ~~~~~~~~~~~~
@@ -1053,20 +1053,20 @@ Fixes
 - Fix tests runner under pip 1.5 (:issue:`513`)
 - Fix logging error when spider name is unicode (:issue:`479`)
 
-0.20.2 (released 2013-12-09)
-----------------------------
+Scrapy 0.20.2 (released 2013-12-09)
+-----------------------------------
 
 - Update CrawlSpider Template with Selector changes (:commit:`6d1457d`)
 - fix method name in tutorial. closes GH-480 (:commit:`b4fc359`
 
-0.20.1 (released 2013-11-28)
-----------------------------
+Scrapy 0.20.1 (released 2013-11-28)
+-----------------------------------
 
 - include_package_data is required to build wheels from published sources (:commit:`5ba1ad5`)
 - process_parallel was leaking the failures on its internal deferreds.  closes #458 (:commit:`419a780`)
 
-0.20.0 (released 2013-11-08)
-----------------------------
+Scrapy 0.20.0 (released 2013-11-08)
+-----------------------------------
 
 Enhancements
 ~~~~~~~~~~~~
@@ -1153,15 +1153,15 @@ List of contributors sorted by number of commits::
       1 cacovsky <amarquesferraz@...>
       1 Berend Iwema <berend@...>
 
-0.18.4 (released 2013-10-10)
-----------------------------
+Scrapy 0.18.4 (released 2013-10-10)
+-----------------------------------
 
 - IPython refuses to update the namespace. fix #396 (:commit:`3d32c4f`)
 - Fix AlreadyCalledError replacing a request in shell command. closes #407 (:commit:`b1d8919`)
 - Fix start_requests laziness and early hangs (:commit:`89faf52`)
 
-0.18.3 (released 2013-10-03)
-----------------------------
+Scrapy 0.18.3 (released 2013-10-03)
+-----------------------------------
 
 - fix regression on lazy evaluation of start requests (:commit:`12693a5`)
 - forms: do not submit reset inputs (:commit:`e429f63`)
@@ -1169,14 +1169,14 @@ List of contributors sorted by number of commits::
 - backport master fixes to json exporter (:commit:`cfc2d46`)
 - Fix permission and set umask before generating sdist tarball (:commit:`06149e0`)
 
-0.18.2 (released 2013-09-03)
-----------------------------
+Scrapy 0.18.2 (released 2013-09-03)
+-----------------------------------
 
 - Backport `scrapy check` command fixes and backward compatible multi
   crawler process(:issue:`339`)
 
-0.18.1 (released 2013-08-27)
-----------------------------
+Scrapy 0.18.1 (released 2013-08-27)
+-----------------------------------
 
 - remove extra import added by cherry picked changes (:commit:`d20304e`)
 - fix crawling tests under twisted pre 11.0.0 (:commit:`1994f38`)
@@ -1194,8 +1194,8 @@ List of contributors sorted by number of commits::
 - minor updates to 0.18 release notes (:commit:`c45e5f1`)
 - fix contributters list format (:commit:`0b60031`)
 
-0.18.0 (released 2013-08-09)
-----------------------------
+Scrapy 0.18.0 (released 2013-08-09)
+-----------------------------------
 
 - Lot of improvements to testsuite run using Tox, including a way to test on pypi
 - Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`)
@@ -1286,8 +1286,8 @@ contributors sorted by number of commits::
       1 Berend Iwema <berend@...>
 
 
-0.16.5 (released 2013-05-30)
-----------------------------
+Scrapy 0.16.5 (released 2013-05-30)
+-----------------------------------
 
 - obey request method when scrapy deploy is redirected to a new endpoint (:commit:`8c4fcee`)
 - fix inaccurate downloader middleware documentation. refs #280 (:commit:`40667cb`)
@@ -1295,8 +1295,8 @@ contributors sorted by number of commits::
 - Find form nodes in invalid html5 documents (:commit:`e3d6945`)
 - Fix typo labeling attrs type bool instead of list (:commit:`a274276`)
 
-0.16.4 (released 2013-01-23)
-----------------------------
+Scrapy 0.16.4 (released 2013-01-23)
+-----------------------------------
 
 - fixes spelling errors in documentation (:commit:`6d2b3aa`)
 - add doc about disabling an extension. refs #132 (:commit:`c90de33`)
@@ -1307,8 +1307,8 @@ contributors sorted by number of commits::
 - fix bug in scrapy parse command when spider is not specified explicitly. closes #209 (:commit:`c72e682`)
 - Update docs/topics/commands.rst (:commit:`28eac7a`)
 
-0.16.3 (released 2012-12-07)
-----------------------------
+Scrapy 0.16.3 (released 2012-12-07)
+-----------------------------------
 
 - Remove concurrency limitation when using download delays and still ensure inter-request delays are enforced (:commit:`487b9b5`)
 - add error details when image pipeline fails (:commit:`8232569`)
@@ -1320,8 +1320,8 @@ contributors sorted by number of commits::
 - Fixed docs typo in SpiderOpenCloseLogging example (:commit:`7184094`)
 
 
-0.16.2 (released 2012-11-09)
-----------------------------
+Scrapy 0.16.2 (released 2012-11-09)
+-----------------------------------
 
 - scrapy contracts: python2.6 compat (:commit:`a4a9199`)
 - scrapy contracts verbose option (:commit:`ec41673`)
@@ -1331,8 +1331,8 @@ contributors sorted by number of commits::
 - Fix SpiderState bug in Windows platforms (:commit:`58998f4`)
 
 
-0.16.1 (released 2012-10-26)
-----------------------------
+Scrapy 0.16.1 (released 2012-10-26)
+-----------------------------------
 
 - fixed LogStats extension, which got broken after a wrong merge before the 0.16 release (:commit:`8c780fd`)
 - better backwards compatibility for scrapy.conf.settings (:commit:`3403089`)
@@ -1342,8 +1342,8 @@ contributors sorted by number of commits::
 - set release date for 0.16.0 in news (:commit:`e292246`)
 
 
-0.16.0 (released 2012-10-18)
-----------------------------
+Scrapy 0.16.0 (released 2012-10-18)
+-----------------------------------
 
 Scrapy changes:
 
@@ -1384,8 +1384,8 @@ Scrapy changes:
 - number received responses are now tracked through Scrapy stats (stat name: ``response_received_count``)
 - removed ``scrapy.log.started`` attribute
 
-0.14.4
-------
+Scrapy 0.14.4
+-------------
 
 - added precise to supported ubuntu distros (:commit:`b7e46df`)
 - fixed bug in json-rpc webservice reported in https://groups.google.com/forum/#!topic/scrapy-users/qgVBmFybNAQ/discussion. also removed no longer supported 'run' command from extras/scrapy-ws.py (:commit:`340fbdb`)
@@ -1393,8 +1393,8 @@ Scrapy changes:
 - replace "import Image" by more standard "from PIL import Image". closes #88 (:commit:`4d17048`)
 - return trial status as bin/runtests.sh exit value. #118 (:commit:`b7b2e7f`)
 
-0.14.3
-------
+Scrapy 0.14.3
+-------------
 
 - forgot to include pydispatch license. #118 (:commit:`fd85f9c`)
 - include egg files used by testsuite in source distribution. #118 (:commit:`c897793`)
@@ -1406,8 +1406,8 @@ Scrapy changes:
 - fixed minor defect in link extractors documentation (:commit:`ba14f38`)
 - removed some obsolete remaining code related to sqlite support in scrapy (:commit:`0665175`)
 
-0.14.2
-------
+Scrapy 0.14.2
+-------------
 
 - move buffer pointing to start of file before computing checksum. refs #92 (:commit:`6a5bef2`)
 - Compute image checksum before persisting images. closes #92 (:commit:`9817df1`)
@@ -1421,8 +1421,8 @@ Scrapy changes:
 - scrapyd: fixed documentation link (:commit:`2b4e4c3`)
 - extras/makedeb.py: no longer obtaining version from git (:commit:`caffe0e`)
 
-0.14.1
-------
+Scrapy 0.14.1
+-------------
 
 - extras/makedeb.py: no longer obtaining version from git (:commit:`caffe0e`)
 - bumped version to 0.14.1 (:commit:`6cb9e1c`)
@@ -1436,8 +1436,8 @@ Scrapy changes:
 - Avoid _disconnectedDeferred AttributeError exception in Twisted>=11.1.0 (:commit:`98f3f87`)
 - allow spider to set autothrottle max concurrency (:commit:`175a4b5`)
 
-0.14
-----
+Scrapy 0.14
+-----------
 
 New features and settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1504,8 +1504,8 @@ Code rearranged and removed
 - Renamed attributes of core components: downloader.sites -> downloader.slots, scraper.sites -> scraper.slots (:rev:`2717`, :rev:`2718`)
 - Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backwards compatibility kept.
 
-0.12
-----
+Scrapy 0.12
+-----------
 
 The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available.
 
@@ -1547,8 +1547,8 @@ Deprecated/obsoleted functionality
 - Deprecated ``queue`` command in favor of using Scrapyd ``schedule.json`` API. See also: Scrapyd changes
 - Removed the !LxmlItemLoader (experimental contrib which never graduated to main contrib)
 
-0.10
-----
+Scrapy 0.10
+-----------
 
 The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available.
 
@@ -1620,8 +1620,8 @@ Changes to settings
 - Removed ``COMMANDS_SETTINGS_MODULE`` setting (#201)
 - Renamed ``REQUEST_HANDLERS`` to ``DOWNLOAD_HANDLERS`` and make download handlers classes (instead of functions)
 
-0.9
----
+Scrapy 0.9
+----------
 
 The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available.
 
@@ -1660,8 +1660,8 @@ Changes to default settings
 
 - Changed default ``SCHEDULER_ORDER`` to ``DFO`` (:rev:`1939`)
 
-0.8
----
+Scrapy 0.8
+----------
 
 The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available.
 
@@ -1706,8 +1706,8 @@ Backwards-incompatible changes
 - Renamed extension: ``DelayedCloseDomain`` to ``SpiderCloseDelay`` (:rev:`1861` | #121)
 - Removed obsolete ``scrapy.utils.markup.remove_escape_chars`` function - use ``scrapy.utils.markup.replace_escape_chars`` instead (:rev:`1865`)
 
-0.7
----
+Scrapy 0.7
+----------
 
 First release of Scrapy.
 

From 8e4947ef0dd4bad368784f0d56e6b0f675ea2a18 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 3 Nov 2015 15:46:05 -0200
Subject: [PATCH 1111/2974] make utils.project.data_path return path unmodified
 when outside project

---
 scrapy/utils/project.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index a15a0d90f..9258469ee 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -40,10 +40,10 @@ def project_data_dir(project='default'):
     return d
 
 def data_path(path, createdir=False):
-    """If path is relative, return the given path inside the project data dir,
-    otherwise return the path unmodified
+    """If inside the project and path is relative, return the given path
+    as relative the project data dir, otherwise return it unmodified
     """
-    if not isabs(path):
+    if inside_project() and not isabs(path):
         path = join(project_data_dir(), path)
     if createdir and not exists(path):
         os.makedirs(path)

From 25bd3b3fea444f9f20b089436f94c4c91d37c05e Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 29 Sep 2016 18:30:42 -0300
Subject: [PATCH 1112/2974] add .scrapy when outside spider too, add tests

---
 scrapy/utils/project.py     | 13 ++++++++++---
 tests/test_utils_project.py | 34 ++++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 3 deletions(-)
 create mode 100644 tests/test_utils_project.py

diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index 9258469ee..fd457fdcb 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -12,6 +12,7 @@ from scrapy.exceptions import NotConfigured
 ENVVAR = 'SCRAPY_SETTINGS_MODULE'
 DATADIR_CFG_SECTION = 'datadir'
 
+
 def inside_project():
     scrapy_module = os.environ.get('SCRAPY_SETTINGS_MODULE')
     if scrapy_module is not None:
@@ -23,6 +24,7 @@ def inside_project():
             return True
     return bool(closest_scrapy_cfg())
 
+
 def project_data_dir(project='default'):
     """Return the current project data dir, creating it if it doesn't exist"""
     if not inside_project():
@@ -39,16 +41,21 @@ def project_data_dir(project='default'):
         os.makedirs(d)
     return d
 
+
 def data_path(path, createdir=False):
     """If inside the project and path is relative, return the given path
-    as relative the project data dir, otherwise return it unmodified
+    as relative to the project data dir, otherwise return it unmodified
     """
-    if inside_project() and not isabs(path):
-        path = join(project_data_dir(), path)
+    if not isabs(path):
+        if inside_project():
+            path = join(project_data_dir(), path)
+        else:
+            path = join('.scrapy', path)
     if createdir and not exists(path):
         os.makedirs(path)
     return path
 
+
 def get_project_settings():
     if ENVVAR not in os.environ:
         project = os.environ.get('SCRAPY_PROJECT', 'default')
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
new file mode 100644
index 000000000..d99a22272
--- /dev/null
+++ b/tests/test_utils_project.py
@@ -0,0 +1,34 @@
+import unittest
+import os
+import tempfile
+import shutil
+import contextlib
+from scrapy.utils.project import data_path
+
+
+@contextlib.contextmanager
+def inside_a_project():
+    prev_dir = os.getcwd()
+    project_dir = tempfile.mkdtemp()
+
+    try:
+        os.chdir(project_dir)
+        with open('scrapy.cfg', 'w') as f:
+            # create an empty scrapy.cfg
+            f.close()
+
+        yield project_dir
+    finally:
+        os.chdir(prev_dir)
+        shutil.rmtree(project_dir)
+
+
+class ProjectUtilsTest(unittest.TestCase):
+    def test_data_path_outside_project(self):
+        self.assertEquals('.scrapy/somepath', data_path('somepath'))
+
+    def test_data_path_inside_project(self):
+        with inside_a_project() as proj_path:
+            expected = os.path.join(proj_path, '.scrapy', 'somepath')
+            self.assertEquals(expected, data_path('somepath'))
+            self.assertEquals('/absolute/path', data_path('/absolute/path'))

From e107212043ec3c9e7704d08e58bd41004657999a Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 29 Sep 2016 19:02:43 -0300
Subject: [PATCH 1113/2974] restore explanation loop to step 1 (see comment in
 3ac3ac4)

---
 docs/topics/architecture.rst | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index 84c053660..31b1c4ddd 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -62,11 +62,10 @@ this:
    :ref:`Spider Middleware <component-spider-middleware>` (output direction).
 
 8. The :ref:`Engine <component-engine>` sends processed items to
-   :ref:`Item Pipelines <component-pipelines>`, then sends processed Requests to
-   the :ref:`Scheduler <component-scheduler>` and asks for the next
-   Requests to crawl.
+   :ref:`Item Pipelines <component-pipelines>` and processed Requests to
+   the :ref:`Scheduler <component-scheduler>`.
 
-9. The process repeats (from step 2) until there are no more requests from the
+9. The process repeats (from step 1) until there are no more requests from the
    :ref:`Scheduler <component-scheduler>`.
 
 Components

From 9c90d9515a50ede29415b8b5d6ba11229f333b49 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Thu, 29 Sep 2016 19:11:17 -0300
Subject: [PATCH 1114/2974] update data_path dosctring

---
 scrapy/utils/project.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index fd457fdcb..95c6a8035 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -43,8 +43,9 @@ def project_data_dir(project='default'):
 
 
 def data_path(path, createdir=False):
-    """If inside the project and path is relative, return the given path
-    as relative to the project data dir, otherwise return it unmodified
+    """
+    Return the given path joined with the .scrapy data directory.
+    If given an absolute path, return it unmodified.
     """
     if not isabs(path):
         if inside_project():

From 8bafcf33a10ea788f6affe311a87bdf37f343afb Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Fri, 30 Sep 2016 11:20:30 -0300
Subject: [PATCH 1115/2974] improve description of engine

---
 docs/topics/architecture.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index 31b1c4ddd..91c80acc0 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -29,7 +29,7 @@ Data flow
 The data flow in Scrapy is controlled by the execution engine, and goes like
 this:
 
-1. The :ref:`Engine <component-engine>` gets the first Requests to crawl from the
+1. The :ref:`Engine <component-engine>` gets the initial Requests to crawl from the
    :ref:`Spider <component-spiders>`.
 
 2. The :ref:`Engine <component-engine>` schedules the Requests in the
@@ -62,8 +62,9 @@ this:
    :ref:`Spider Middleware <component-spider-middleware>` (output direction).
 
 8. The :ref:`Engine <component-engine>` sends processed items to
-   :ref:`Item Pipelines <component-pipelines>` and processed Requests to
-   the :ref:`Scheduler <component-scheduler>`.
+   :ref:`Item Pipelines <component-pipelines>`, then send processed Requests to
+   the :ref:`Scheduler <component-scheduler>` and asks for possible next Requests
+   to crawl.
 
 9. The process repeats (from step 1) until there are no more requests from the
    :ref:`Scheduler <component-scheduler>`.

From 559b4edaec9b0b73376178427e2684f25b7d78c0 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Fri, 30 Sep 2016 11:42:50 -0300
Subject: [PATCH 1116/2974] update release notes with changes from #1581

---
 docs/news.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 450ab5542..0876695be 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -29,6 +29,9 @@ Bug fixes
 - DefaultRequestHeaders middleware now runs before UserAgent middleware
   (:issue:`2088`). **Warning: this is technically backwards incompatible**,
   though we consider this a bug fix.
+- HTTP cache extension and plugins that use the ``.scrapy`` data directory now
+  work outside projects (:issue:`1581`).  **Warning: this is technically
+  backwards incompatible**, though we consider this a bug fix.
 - ``Selector`` does not allow passing both ``response`` and ``text`` anymore
   (:issue:`2153`).
 - Fixed logging of wrong callback name with ``scrapy parse`` (:issue:`2169`).

From 2d932c173cdf7dc530c2193870262bf57b8f2f30 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Fri, 30 Sep 2016 15:07:58 -0300
Subject: [PATCH 1117/2974] test abs path outside project as well

---
 tests/test_utils_project.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index d99a22272..1239195e8 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -26,6 +26,7 @@ def inside_a_project():
 class ProjectUtilsTest(unittest.TestCase):
     def test_data_path_outside_project(self):
         self.assertEquals('.scrapy/somepath', data_path('somepath'))
+        self.assertEquals('/absolute/path', data_path('/absolute/path'))
 
     def test_data_path_inside_project(self):
         with inside_a_project() as proj_path:

From e8edc6c2bb3dae67098ffc72913c479a2101c498 Mon Sep 17 00:00:00 2001
From: Wieland Hoffmann <mineo@users.noreply.github.com>
Date: Sun, 2 Oct 2016 16:08:10 +0200
Subject: [PATCH 1118/2974] Remove duplicate colons from the feed export
 settings docs

---
 docs/topics/feed-exports.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index f0d3e660d..efdd8c46b 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -280,7 +280,7 @@ Whether to export empty feeds (ie. feeds with no items).
 FEED_STORAGES
 -------------
 
-Default:: ``{}``
+Default: ``{}``
 
 A dict containing additional feed storage backends supported by your project.
 The keys are URI schemes and the values are paths to storage classes.
@@ -314,7 +314,7 @@ can disable any of these backends by assigning ``None`` to their URI scheme in
 FEED_EXPORTERS
 --------------
 
-Default:: ``{}``
+Default: ``{}``
 
 A dict containing additional exporters supported by your project. The keys are
 serialization formats and the values are paths to :ref:`Item exporter

From e2137d77cef15f9740a7f0b02e32e45719048384 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 3 Oct 2016 12:41:18 +0200
Subject: [PATCH 1119/2974] Add release date for scrapy 1.2

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index ae8553b9a..8647041f6 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,7 +3,7 @@
 Release notes
 =============
 
-Scrapy 1.2.0 (YYYY-MM-DD)
+Scrapy 1.2.0 (2016-10-03)
 -------------------------
 
 New Features

From 3235bfeb1ea46f4e99269469cecbb715c93b747e Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 3 Oct 2016 13:04:11 +0200
Subject: [PATCH 1120/2974] =?UTF-8?q?Bump=20version:=201.2.0dev2=20?=
 =?UTF-8?q?=E2=86=92=201.2.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 49a7e239f..3dcab980c 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.2.0dev2
+current_version = 1.2.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index a25fc280b..26aaba0e8 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.2.0dev2
+1.2.0

From eb91cb8ea2c0fa3cd852a50fb45482d7fd887dd5 Mon Sep 17 00:00:00 2001
From: Bernardas <bernardas.alisauskas@gmail.com>
Date: Mon, 3 Oct 2016 20:31:41 +0000
Subject: [PATCH 1121/2974] fix JsonWriterPipeline example

---
 docs/topics/item-pipeline.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 6b43fe258..11b56f1c4 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -106,9 +106,12 @@ format::
 
    class JsonWriterPipeline(object):
 
-       def __init__(self):
+       def open_spider(self, spider):
            self.file = open('items.jl', 'wb')
 
+       def close_spider(self, spider):
+           self.file.close()
+
        def process_item(self, item, spider):
            line = json.dumps(dict(item)) + "\n"
            self.file.write(line)

From dfba151f59366f461ceb0d0ea1b6be6c89dc6d10 Mon Sep 17 00:00:00 2001
From: Bernardas <bernardas.alisauskas@gmail.com>
Date: Wed, 5 Oct 2016 16:35:23 +0000
Subject: [PATCH 1122/2974] Remove unnecessary note for the JsonWriterPipeline
 example

---
 docs/topics/item-pipeline.rst | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 11b56f1c4..8c7aa361f 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -129,14 +129,7 @@ MongoDB address and database name are specified in Scrapy settings;
 MongoDB collection is named after item class.
 
 The main point of this example is to show how to use :meth:`from_crawler`
-method and how to clean up the resources properly.
-
-.. note::
-
-    Previous example (JsonWriterPipeline) doesn't clean up resources properly.
-    Fixing it is left as an exercise for the reader.
-
-::
+method and how to clean up the resources properly.::
 
     import pymongo
 

From dd778892d09f4b504bfc65aae83dd70d87f0d93e Mon Sep 17 00:00:00 2001
From: Raul Gallegos <raul.ogh@gmail.com>
Date: Wed, 5 Oct 2016 12:16:45 -0500
Subject: [PATCH 1123/2974] genspider: removing www. from starturl templates

---
 scrapy/templates/spiders/basic.tmpl   | 2 +-
 scrapy/templates/spiders/crawl.tmpl   | 2 +-
 scrapy/templates/spiders/csvfeed.tmpl | 2 +-
 scrapy/templates/spiders/xmlfeed.tmpl | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl
index 2d3b96e38..abaa34ef5 100644
--- a/scrapy/templates/spiders/basic.tmpl
+++ b/scrapy/templates/spiders/basic.tmpl
@@ -6,7 +6,7 @@ class $classname(scrapy.Spider):
     name = "$name"
     allowed_domains = ["$domain"]
     start_urls = (
-        'http://www.$domain/',
+        'http://$domain/',
     )
 
     def parse(self, response):
diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 154237d9c..802cb88a1 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -7,7 +7,7 @@ from scrapy.spiders import CrawlSpider, Rule
 class $classname(CrawlSpider):
     name = '$name'
     allowed_domains = ['$domain']
-    start_urls = ['http://www.$domain/']
+    start_urls = ['http://$domain/']
 
     rules = (
         Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl
index 0544e0ae7..c2e4bacfe 100644
--- a/scrapy/templates/spiders/csvfeed.tmpl
+++ b/scrapy/templates/spiders/csvfeed.tmpl
@@ -5,7 +5,7 @@ from scrapy.spiders import CSVFeedSpider
 class $classname(CSVFeedSpider):
     name = '$name'
     allowed_domains = ['$domain']
-    start_urls = ['http://www.$domain/feed.csv']
+    start_urls = ['http://$domain/feed.csv']
     # headers = ['id', 'name', 'description', 'image_link']
     # delimiter = '\t'
 
diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl
index d8ff61f6e..7c2ff8850 100644
--- a/scrapy/templates/spiders/xmlfeed.tmpl
+++ b/scrapy/templates/spiders/xmlfeed.tmpl
@@ -5,7 +5,7 @@ from scrapy.spiders import XMLFeedSpider
 class $classname(XMLFeedSpider):
     name = '$name'
     allowed_domains = ['$domain']
-    start_urls = ['http://www.$domain/feed.xml']
+    start_urls = ['http://$domain/feed.xml']
     iterator = 'iternodes' # you can change this; see the docs
     itertag = 'item' # change it accordingly
 

From c3411373e8a8ee2786588bdad7be469c69a25e2a Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 7 Oct 2016 12:28:16 +0200
Subject: [PATCH 1124/2974] Use OpenSSL default ciphers

Twisted default TLS options restricts the ciphers list a bit -- "a secure default"
https://github.com/twisted/twisted/blob/e38cc25a67747899c6984d6ebaa8d3d134799415/src/twisted/internet/_sslverify.py#L1861

We want to be a bit more permissive with Scrapy
(at least as permissive as Scrapy 1.0 was, and which used a default OpenSSL Context)

See https://www.openssl.org/docs/manmaster/apps/ciphers.html#CIPHER_STRINGS

OpenSSL's 'DEFAULT' seems to be reasonable enough.

Fixes #2311
---
 scrapy/core/downloader/contextfactory.py | 6 ++++--
 scrapy/core/downloader/tls.py            | 3 +++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index b643d935b..4bd4c6166 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -13,7 +13,7 @@ try:
     from twisted.web.client import BrowserLikePolicyForHTTPS
     from twisted.web.iweb import IPolicyForHTTPS
 
-    from scrapy.core.downloader.tls import ScrapyClientTLSOptions
+    from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
 
 
     @implementer(IPolicyForHTTPS)
@@ -46,7 +46,9 @@ try:
             #   not calling super(..., self).__init__
             return CertificateOptions(verify=False,
                         method=getattr(self, 'method',
-                                       getattr(self, '_ssl_method', None)))
+                                       getattr(self, '_ssl_method', None)),
+                        fixBrokenPeers=True,
+                        acceptableCiphers=DEFAULT_CIPHERS)
 
         # kept for old-style HTTP/1.0 downloader context twisted calls,
         # e.g. connectSSL()
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 00c94ee2e..955b7630c 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -28,6 +28,7 @@ try:
         SSL_CB_HANDSHAKE_START = 0x10
         SSL_CB_HANDSHAKE_DONE = 0x20
 
+    from twisted.internet.ssl import AcceptableCiphers
     from twisted.internet._sslverify import (ClientTLSOptions,
                                              _maybeSetHostNameIndication,
                                              verifyHostname,
@@ -60,6 +61,8 @@ try:
                         'from host "{}" (exception: {})'.format(
                             self._hostnameASCII, repr(e)))
 
+    DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
+
 except ImportError:
     # ImportError should not matter for older Twisted versions
     # as the above is not used in the fallback ScrapyClientContextFactory

From 118b42ab594fcde8f04ddd570ac6b3a7a00bdae5 Mon Sep 17 00:00:00 2001
From: Raul Gallegos <raul.ogh@gmail.com>
Date: Tue, 11 Oct 2016 22:08:05 -0500
Subject: [PATCH 1125/2974] making start_urls a list in basic genspider
 template

---
 scrapy/templates/spiders/basic.tmpl | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl
index abaa34ef5..99e5d43b2 100644
--- a/scrapy/templates/spiders/basic.tmpl
+++ b/scrapy/templates/spiders/basic.tmpl
@@ -5,9 +5,7 @@ import scrapy
 class $classname(scrapy.Spider):
     name = "$name"
     allowed_domains = ["$domain"]
-    start_urls = (
-        'http://$domain/',
-    )
+    start_urls = ['http://$domain/']
 
     def parse(self, response):
         pass

From bfe28ae7077b08da53bfbd863a3eae558848da68 Mon Sep 17 00:00:00 2001
From: Bo Pace <bopace@gmail.com>
Date: Mon, 17 Oct 2016 14:10:05 -0600
Subject: [PATCH 1126/2974] Added documentation about accessing header values

---
 docs/topics/request-response.rst | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 75b98d3b3..5d9ac79f6 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -507,7 +507,13 @@ Response objects
 
     .. attribute:: Response.headers
 
-        A dictionary-like object which contains the response headers.
+        A dictionary-like object which contains the response headers. Values can
+        be accessed from the :class:`headers` object by using :meth:`get` to
+        return the first header value with the specified name or :meth:`getlist`
+        to return all header values with the specified name. For example, this
+        call will give you all cookies in the headers::
+
+            response.headers.getlist('Set-Cookie')
 
     .. attribute:: Response.body
 

From bebcd5081ce50be17434d8815e9037130d709f6e Mon Sep 17 00:00:00 2001
From: Jose Ricardo <zehzinho@gmail.com>
Date: Tue, 18 Oct 2016 11:22:55 -0200
Subject: [PATCH 1127/2974] Add downloader middleware ordering details to the
 docs

Add more details, making it easier to understand what are the effects of
setting a downloader middleware order.
---
 docs/topics/downloader-middleware.rst | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 31545d548..15069e56e 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -27,7 +27,11 @@ The :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the
 :setting:`DOWNLOADER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant
 to be overridden) and then sorted by order to get the final sorted list of
 enabled middlewares: the first middleware is the one closer to the engine and
-the last is the one closer to the downloader.
+the last is the one closer to the downloader. In other words,
+the :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request`
+method of each middleware will be invoked in increasing
+middleware order (100, 200, 300, ...) and the :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response` method
+of each middleware will be invoked in decreasing order.
 
 To decide which order to assign to your middleware see the
 :setting:`DOWNLOADER_MIDDLEWARES_BASE` setting and pick a value according to

From ea7bd39529347af3b8e30c0588428ac442151673 Mon Sep 17 00:00:00 2001
From: Jose Ricardo <zehzinho@gmail.com>
Date: Tue, 18 Oct 2016 11:48:58 -0200
Subject: [PATCH 1128/2974] Make architecture overview references a little more
 clear on the docs

Expliciting what actually happens by adding links to the respective methods
that are invoked in each processing phase.
---
 docs/topics/architecture.rst | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index 91c80acc0..ea0cb0ea7 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -41,25 +41,26 @@ this:
 
 4. The :ref:`Engine <component-engine>` sends the Requests to the
    :ref:`Downloader <component-downloader>`, passing through the
-   :ref:`Downloader Middleware <component-downloader-middleware>`
-   (requests direction).
+   :ref:`Downloader Middlewares <component-downloader-middleware>` (see
+   :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request`).
 
 5. Once the page finishes downloading the
    :ref:`Downloader <component-downloader>` generates a Response (with
    that page) and sends it to the Engine, passing through the
-   :ref:`Downloader Middleware <component-downloader-middleware>`
-   (response direction).
+   :ref:`Downloader Middlewares <component-downloader-middleware>` (see
+   :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`).
 
 6. The :ref:`Engine <component-engine>` receives the Response from the
    :ref:`Downloader <component-downloader>` and sends it to the
    :ref:`Spider <component-spiders>` for processing, passing
-   through the :ref:`Spider Middleware <component-spider-middleware>`
-   (input direction).
+   through the :ref:`Spider Middleware <component-spider-middleware>` (see
+   :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_input`).
 
 7. The :ref:`Spider <component-spiders>` processes the Response and returns
    scraped items and new Requests (to follow) to the
    :ref:`Engine <component-engine>`, passing through the
-   :ref:`Spider Middleware <component-spider-middleware>` (output direction).
+   :ref:`Spider Middleware <component-spider-middleware>` (see
+   :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`).
 
 8. The :ref:`Engine <component-engine>` sends processed items to
    :ref:`Item Pipelines <component-pipelines>`, then send processed Requests to

From 7c33e0cb55613b7ec805e58e64fc92a8d2a7d371 Mon Sep 17 00:00:00 2001
From: Luiz Fernando Mattos Schlindwein <l.f.mattos.sch@gmail.com>
Date: Tue, 18 Oct 2016 11:55:51 -0200
Subject: [PATCH 1129/2974] added a note about invalid spider names in python 2

---
 docs/topics/spiders.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 0e473709a..af676fccd 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -72,6 +72,8 @@ scrapy.Spider
        spider that crawls ``mywebsite.com`` would often be called
        ``mywebsite``.
 
+       .. note:: In Python 2 this must be ASCII only.
+
    .. attribute:: allowed_domains
 
        An optional list of strings containing domains that this spider is

From 45e95b79ce17eef0b3b4d324daa8b0d220fdb2fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mois=C3=A9s=20Guimar=C3=A3es?= <moises@wolfssl.com>
Date: Tue, 18 Oct 2016 11:06:55 -0300
Subject: [PATCH 1130/2974] (fixes #2272) using arg_to_iter() to wrap single
 values and list() to avoid consuming from generators.

---
 scrapy/mail.py     |  6 ++++++
 tests/test_mail.py | 18 ++++++++++++++----
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/scrapy/mail.py b/scrapy/mail.py
index c6339f25b..283c3b8e1 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -21,6 +21,8 @@ else:
 
 from twisted.internet import defer, reactor, ssl
 
+from utils.misc import arg_to_iter
+
 logger = logging.getLogger(__name__)
 
 
@@ -48,6 +50,10 @@ class MailSender(object):
             msg = MIMEMultipart()
         else:
             msg = MIMENonMultipart(*mimetype.split('/', 1))
+
+        to = list(arg_to_iter(to))
+        cc = list(arg_to_iter(cc))
+
         msg['From'] = self.mailfrom
         msg['To'] = COMMASPACE.join(to)
         msg['Date'] = formatdate(localtime=True)
diff --git a/tests/test_mail.py b/tests/test_mail.py
index bd7e49621..b139e98d8 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -10,7 +10,8 @@ class MailSenderTest(unittest.TestCase):
 
     def test_send(self):
         mailsender = MailSender(debug=True)
-        mailsender.send(to=['test@scrapy.org'], subject='subject', body='body', _callback=self._catch_mail_sent)
+        mailsender.send(to=['test@scrapy.org'], subject='subject', body='body',
+                        _callback=self._catch_mail_sent)
 
         assert self.catched_msg
 
@@ -24,9 +25,16 @@ class MailSenderTest(unittest.TestCase):
         self.assertEqual(msg.get_payload(), 'body')
         self.assertEqual(msg.get('Content-Type'), 'text/plain')
 
+    def test_send_single_values_to_and_cc(self):
+        mailsender = MailSender(debug=True)
+        mailsender.send(to='test@scrapy.org', subject='subject', body='body',
+                        cc='test@scrapy.org', _callback=self._catch_mail_sent)
+
     def test_send_html(self):
         mailsender = MailSender(debug=True)
-        mailsender.send(to=['test@scrapy.org'], subject='subject', body='<p>body</p>', mimetype='text/html', _callback=self._catch_mail_sent)
+        mailsender.send(to=['test@scrapy.org'], subject='subject',
+                        body='<p>body</p>', mimetype='text/html',
+                        _callback=self._catch_mail_sent)
 
         msg = self.catched_msg['msg']
         self.assertEqual(msg.get_payload(), '<p>body</p>')
@@ -90,7 +98,8 @@ class MailSenderTest(unittest.TestCase):
 
         mailsender = MailSender(debug=True)
         mailsender.send(to=['test@scrapy.org'], subject=subject, body=body,
-                        attachs=attachs, charset='utf-8', _callback=self._catch_mail_sent)
+                        attachs=attachs, charset='utf-8',
+                        _callback=self._catch_mail_sent)
 
         assert self.catched_msg
         self.assertEqual(self.catched_msg['subject'], subject)
@@ -99,7 +108,8 @@ class MailSenderTest(unittest.TestCase):
         msg = self.catched_msg['msg']
         self.assertEqual(msg['subject'], subject)
         self.assertEqual(msg.get_charset(), Charset('utf-8'))
-        self.assertEqual(msg.get('Content-Type'), 'multipart/mixed; charset="utf-8"')
+        self.assertEqual(msg.get('Content-Type'),
+                         'multipart/mixed; charset="utf-8"')
 
         payload = msg.get_payload()
         assert isinstance(payload, list)

From e12e364a40e951ffa76ef15b3f24fa64abb0f1bb Mon Sep 17 00:00:00 2001
From: Jose Ricardo <zehzinho@gmail.com>
Date: Tue, 18 Oct 2016 12:29:30 -0200
Subject: [PATCH 1131/2974] Add details to the spider middlewares docs

Document the effects of the middleware order in a more detailed way.
---
 docs/topics/spider-middleware.rst | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index a38c1ab65..604f1864f 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -28,7 +28,12 @@ The :setting:`SPIDER_MIDDLEWARES` setting is merged with the
 :setting:`SPIDER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant to
 be overridden) and then sorted by order to get the final sorted list of enabled
 middlewares: the first middleware is the one closer to the engine and the last
-is the one closer to the spider.
+is the one closer to the spider. In other words,
+the :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_input`
+method of each middleware will be invoked in increasing
+middleware order (100, 200, 300, ...), and the
+:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method
+of each middleware will be invoked in decreasing order.
 
 To decide which order to assign to your middleware see the
 :setting:`SPIDER_MIDDLEWARES_BASE` setting and pick a value according to where

From 3fb6e52457a7f24d580ed32c07aa12d9547d2970 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mois=C3=A9s=20Guimar=C3=A3es?= <moises@wolfssl.com>
Date: Tue, 18 Oct 2016 12:24:11 -0300
Subject: [PATCH 1132/2974] fixes import for py35 env.

---
 scrapy/mail.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/mail.py b/scrapy/mail.py
index 283c3b8e1..0bb395521 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -21,7 +21,7 @@ else:
 
 from twisted.internet import defer, reactor, ssl
 
-from utils.misc import arg_to_iter
+from .utils.misc import arg_to_iter
 
 logger = logging.getLogger(__name__)
 

From fd016ee71b4d85f6b4e02a0da4a7ebd11f787e0b Mon Sep 17 00:00:00 2001
From: bopace <bopace@gmail.com>
Date: Tue, 18 Oct 2016 09:37:45 -0600
Subject: [PATCH 1133/2974] Fixed wording of documentation

---
 docs/topics/request-response.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 5d9ac79f6..a45ea6939 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -508,10 +508,10 @@ Response objects
     .. attribute:: Response.headers
 
         A dictionary-like object which contains the response headers. Values can
-        be accessed from the :class:`headers` object by using :meth:`get` to
-        return the first header value with the specified name or :meth:`getlist`
-        to return all header values with the specified name. For example, this
-        call will give you all cookies in the headers::
+        be accessed using :meth:`get` to return the first header value with the
+        specified name or :meth:`getlist` to return all header values with the
+        specified name. For example, this call will give you all cookies in the
+        headers::
 
             response.headers.getlist('Set-Cookie')
 

From f74051e69e870d9e29deec62c121f8a593c503ae Mon Sep 17 00:00:00 2001
From: Victor Sued <suedx1000@suedx1000>
Date: Tue, 18 Oct 2016 16:36:43 -0200
Subject: [PATCH 1134/2974] update documentation to explain the use of double
 quotes on Windows.

---
 docs/intro/tutorial.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 4f2736709..5aed25e4e 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -205,6 +205,10 @@ using the shell :ref:`Scrapy shell <topics-shell>`. Run::
    command-line, otherwise urls containing arguments (ie. ``&`` character)
    will not work.
 
+   On Windows, use double quotes instead::
+
+       scrapy shell "http://quotes.toscrape.com/page/1/"
+
 You will see something like::
 
     [ ... Scrapy log here ... ]

From d5bd44a5b9fa28a44c44396d1f87c3697a5e2d42 Mon Sep 17 00:00:00 2001
From: muriloviana <murilo.vianamo@gmail.com>
Date: Tue, 18 Oct 2016 17:29:16 -0200
Subject: [PATCH 1135/2974] add middleware to template project

---
 scrapy/commands/startproject.py               |  1 +
 .../project/module/middleware.py.tmpl         | 23 +++++++++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 scrapy/templates/project/module/middleware.py.tmpl

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index e3989baaf..1001b8682 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -17,6 +17,7 @@ TEMPLATES_TO_RENDER = (
     ('${project_name}', 'settings.py.tmpl'),
     ('${project_name}', 'items.py.tmpl'),
     ('${project_name}', 'pipelines.py.tmpl'),
+    ('${project_name}', 'middleware.py.tmpl'),
 )
 
 IGNORE = ignore_patterns('*.pyc', '.svn')
diff --git a/scrapy/templates/project/module/middleware.py.tmpl b/scrapy/templates/project/module/middleware.py.tmpl
new file mode 100644
index 000000000..0ae9d7160
--- /dev/null
+++ b/scrapy/templates/project/module/middleware.py.tmpl
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# http://doc.scrapy.org/en/latest/topics/spider-middleware.html
+
+
+class ${ProjectName}SpiderMiddleware(object):
+    # This method is called for each response that goes through the
+    # spider middleware and into the spider, for processing.
+
+    def process_spider_input(response, spider):
+        # Called for each response that goes to spider
+        pass
+
+    def process_spider_output(response, result, spider):
+        # Called for each response with the spider results
+        pass
+
+    def process_spider_exception(response, exception, spider):
+        # Called whenever a spider raises an exception
+        pass

From 34f2014c55ed25615066f7017ddd36df871095bd Mon Sep 17 00:00:00 2001
From: muriloviana <murilo.vianamo@gmail.com>
Date: Wed, 19 Oct 2016 00:37:31 -0200
Subject: [PATCH 1136/2974] change settings middleware name and updating
 middleware template

---
 scrapy/commands/startproject.py               |  2 +-
 .../project/module/middleware.py.tmpl         | 23 -----------
 .../project/module/middlewares.py.tmpl        | 39 +++++++++++++++++++
 .../templates/project/module/settings.py.tmpl |  2 +-
 4 files changed, 41 insertions(+), 25 deletions(-)
 delete mode 100644 scrapy/templates/project/module/middleware.py.tmpl
 create mode 100644 scrapy/templates/project/module/middlewares.py.tmpl

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 1001b8682..594106632 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -17,7 +17,7 @@ TEMPLATES_TO_RENDER = (
     ('${project_name}', 'settings.py.tmpl'),
     ('${project_name}', 'items.py.tmpl'),
     ('${project_name}', 'pipelines.py.tmpl'),
-    ('${project_name}', 'middleware.py.tmpl'),
+    ('${project_name}', 'middlewares.py.tmpl'),
 )
 
 IGNORE = ignore_patterns('*.pyc', '.svn')
diff --git a/scrapy/templates/project/module/middleware.py.tmpl b/scrapy/templates/project/module/middleware.py.tmpl
deleted file mode 100644
index 0ae9d7160..000000000
--- a/scrapy/templates/project/module/middleware.py.tmpl
+++ /dev/null
@@ -1,23 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Define here the models for your spider middleware
-#
-# See documentation in:
-# http://doc.scrapy.org/en/latest/topics/spider-middleware.html
-
-
-class ${ProjectName}SpiderMiddleware(object):
-    # This method is called for each response that goes through the
-    # spider middleware and into the spider, for processing.
-
-    def process_spider_input(response, spider):
-        # Called for each response that goes to spider
-        pass
-
-    def process_spider_output(response, result, spider):
-        # Called for each response with the spider results
-        pass
-
-    def process_spider_exception(response, exception, spider):
-        # Called whenever a spider raises an exception
-        pass
diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
new file mode 100644
index 000000000..91d05baa6
--- /dev/null
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# http://doc.scrapy.org/en/latest/topics/spider-middleware.html
+
+
+class ${ProjectName}SpiderMiddleware(object):
+
+    def process_spider_input(response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        pass
+
+    def process_spider_output(response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, dict or Item objects.
+        pass
+
+    def process_spider_exception(response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Response, dict
+        # or Item objects.
+        pass
+
+    def process_start_requests(start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        pass
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index f13e85871..72f25ebef 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -47,7 +47,7 @@ ROBOTSTXT_OBEY = True
 # Enable or disable spider middlewares
 # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
 #SPIDER_MIDDLEWARES = {
-#    '$project_name.middlewares.MyCustomSpiderMiddleware': 543,
+#    '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543,
 #}
 
 # Enable or disable downloader middlewares

From 38e292a13207443c9f966fb7168e37fb69965a7e Mon Sep 17 00:00:00 2001
From: muriloviana <murilo.vianamo@gmail.com>
Date: Wed, 19 Oct 2016 12:47:23 -0200
Subject: [PATCH 1137/2974] add from_crawler method to template and turn the
 returns methods explicit

---
 .../project/module/middlewares.py.tmpl        | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
index 91d05baa6..4f7c8d191 100644
--- a/scrapy/templates/project/module/middlewares.py.tmpl
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -5,22 +5,34 @@
 # See documentation in:
 # http://doc.scrapy.org/en/latest/topics/spider-middleware.html
 
+from scrapy import signals
+
 
 class ${ProjectName}SpiderMiddleware(object):
+    # If any of methods bellow is not defined, in practice, the middleware
+    # manager only add the methods that the middleware has defined.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls(crawler.stats)
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
 
     def process_spider_input(response, spider):
         # Called for each response that goes through the spider
         # middleware and into the spider.
 
         # Should return None or raise an exception.
-        pass
+        return None
 
     def process_spider_output(response, result, spider):
         # Called with the results returned from the Spider, after
         # it has processed the response.
 
         # Must return an iterable of Request, dict or Item objects.
-        pass
+        for i in result:
+            yield i
 
     def process_spider_exception(response, exception, spider):
         # Called when a spider or process_spider_input() method
@@ -36,4 +48,5 @@ class ${ProjectName}SpiderMiddleware(object):
         # that it doesn’t have a response associated.
 
         # Must return only requests (not items).
-        pass
+        for r in start_requests:
+            yield r

From 32fd69281066f16db7aab8741da4c08051884049 Mon Sep 17 00:00:00 2001
From: muriloviana <murilo.vianamo@gmail.com>
Date: Wed, 19 Oct 2016 13:50:09 -0200
Subject: [PATCH 1138/2974] define method spider_opened and update class
 instructions

---
 scrapy/templates/project/module/middlewares.py.tmpl | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
index 4f7c8d191..8b9827527 100644
--- a/scrapy/templates/project/module/middlewares.py.tmpl
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -9,8 +9,9 @@ from scrapy import signals
 
 
 class ${ProjectName}SpiderMiddleware(object):
-    # If any of methods bellow is not defined, in practice, the middleware
-    # manager only add the methods that the middleware has defined.
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -50,3 +51,6 @@ class ${ProjectName}SpiderMiddleware(object):
         # Must return only requests (not items).
         for r in start_requests:
             yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)

From 09c401bf8e1381f2e9109b799fb2889b8c07a651 Mon Sep 17 00:00:00 2001
From: muriloviana <murilo.vianamo@gmail.com>
Date: Wed, 19 Oct 2016 14:09:01 -0200
Subject: [PATCH 1139/2974] use crawler only to register signals in
 from_crawler()

---
 scrapy/templates/project/module/middlewares.py.tmpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
index 8b9827527..42318fec2 100644
--- a/scrapy/templates/project/module/middlewares.py.tmpl
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -16,7 +16,7 @@ class ${ProjectName}SpiderMiddleware(object):
     @classmethod
     def from_crawler(cls, crawler):
         # This method is used by Scrapy to create your spiders.
-        s = cls(crawler.stats)
+        s = cls()
         crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
         return s
 

From db408528928b2d15043593032913fe40d6eb6783 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 20 Oct 2016 04:26:12 +0200
Subject: [PATCH 1140/2974] Do not interpret non-ASCII bytes in "Location" and
 percent-encode them (#2322)

* Do not interpret non-ASCII bytes in "Location" and percent-encode them

Fixes GH-2321

The idea is to not guess the encoding of "Location" header value
and simply percent-encode non-ASCII bytes,
which should then be re-interpreted correctly by the remote website
in whatever encoding was used originally.

See https://tools.ietf.org/html/rfc3987#section-3.2

This is similar to the changes to safe_url_string in
https://github.com/scrapy/w3lib/pull/45

* Remove unused import
---
 scrapy/downloadermiddlewares/redirect.py    | 6 +++---
 tests/test_downloadermiddleware_redirect.py | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 4ed7e4c24..db276eefb 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -1,9 +1,10 @@
 import logging
 from six.moves.urllib.parse import urljoin
 
+from w3lib.url import safe_url_string
+
 from scrapy.http import HtmlResponse
 from scrapy.utils.response import get_meta_refresh
-from scrapy.utils.python import to_native_str
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 
 logger = logging.getLogger(__name__)
@@ -65,8 +66,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
         if 'Location' not in response.headers or response.status not in allowed_status:
             return response
 
-        # HTTP header is ascii or latin1, redirected url will be percent-encoded utf-8
-        location = to_native_str(response.headers['location'].decode('latin1'))
+        location = safe_url_string(response.headers['location'])
 
         redirected_url = urljoin(request.url, location)
 
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 9db073cc5..e8c92affa 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -157,15 +157,15 @@ class RedirectMiddlewareTest(unittest.TestCase):
         latin1_location = u'/ação'.encode('latin1')  # HTTP historically supports latin1
         resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302)
         req_result = self.mw.process_response(req, resp, self.spider)
-        perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
+        perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o'
         self.assertEquals(perc_encoded_utf8_url, req_result.url)
 
-    def test_location_with_wrong_encoding(self):
+    def test_utf8_location(self):
         req = Request('http://scrapytest.org/first')
-        utf8_location = u'/ação'  # header with wrong encoding (utf-8)
+        utf8_location = u'/ação'.encode('utf-8')  # header using UTF-8 encoding
         resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302)
         req_result = self.mw.process_response(req, resp, self.spider)
-        perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%83%C2%A7%C3%83%C2%A3o'
+        perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
         self.assertEquals(perc_encoded_utf8_url, req_result.url)
 
 

From f357ccd0d7ab164d107ba22f6e45e9f172aae712 Mon Sep 17 00:00:00 2001
From: Jay Patel <jaympatel@users.noreply.github.com>
Date: Thu, 20 Oct 2016 15:52:28 -0400
Subject: [PATCH 1141/2974] Typo (through was misspelled)

---
 scrapy/core/scraper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 67198179d..c08e37367 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -119,7 +119,7 @@ class Scraper(object):
             self._scrape(response, request, spider).chainDeferred(deferred)
 
     def _scrape(self, response, request, spider):
-        """Handle the downloaded response or failure trough the spider
+        """Handle the downloaded response or failure through the spider
         callback/errback"""
         assert isinstance(response, (Response, Failure))
 

From c7dfb5eb88e2cb9c489725e5ad81bc1c69f7b3fc Mon Sep 17 00:00:00 2001
From: gustavodeandrade <gu_andrade1@yahoo.com.br>
Date: Fri, 21 Oct 2016 00:08:08 -0200
Subject: [PATCH 1142/2974] Fix issue 1704

---
 docs/topics/selectors.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 39ec9b73c..293628953 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -626,6 +626,11 @@ Built-in Selectors reference
      ``regex`` can be either a compiled regular expression or a string which
      will be compiled to a regular expression using ``re.compile(regex)``
 
+    .. note::
+
+        Note that ``re`` and ``re_first`` both escape HTML entities. If you want to
+        get raw unescaped content, use extract() or extract_first()
+
   .. method:: register_namespace(prefix, uri)
 
      Register the given namespace to be used in this :class:`Selector`.

From 22772a61c8252493ae2bdd344a2e916048e92db2 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 20 Oct 2016 13:12:27 +0200
Subject: [PATCH 1143/2974] Update release notes for upcoming 1.2.1

---
 docs/news.rst | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 8647041f6..b375694e2 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,30 @@
 Release notes
 =============
 
+Scrapy 1.2.1 (2016-MM-DD)
+-------------------------
+
+Bug fixes
+~~~~~~~~~
+
+- Include OpenSSL's more permissive default ciphers when establishing
+  TLS/SSL connections (:issue:`2314`).
+- Fix "Location" HTTP header decoding on non-ASCII URL redirects (:issue:`2321`).
+
+Documentation
+~~~~~~~~~~~~~
+
+- Fix JsonWriterPipeline example (:issue:`2302`).
+- Various notes: :issue:`2330` on spider names,
+  :issue:`2329` on middleware methods processing order,
+  :issue:`2327` on getting multi-valued HTTP headers as lists.
+
+Other changes
+~~~~~~~~~~~~~
+
+- Removed ``www.`` from ``start_urls`` in built-in spider templates (:issue:`2299`).
+
+
 Scrapy 1.2.0 (2016-10-03)
 -------------------------
 

From c559a64c207fce0c57bed1a94a319aeb18017848 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 21 Oct 2016 10:17:46 +0200
Subject: [PATCH 1144/2974] Set date for 1.2.1 release

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index b375694e2..c302d2e17 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,7 +3,7 @@
 Release notes
 =============
 
-Scrapy 1.2.1 (2016-MM-DD)
+Scrapy 1.2.1 (2016-10-21)
 -------------------------
 
 Bug fixes

From 6df48d57e039d98c3df049c8bc58f0f97d5f6665 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 21 Oct 2016 10:37:45 +0200
Subject: [PATCH 1145/2974] =?UTF-8?q?Bump=20version:=201.2.0=20=E2=86=92?=
 =?UTF-8?q?=201.2.1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 3dcab980c..ffc933d13 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.2.0
+current_version = 1.2.1
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 26aaba0e8..6085e9465 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.2.0
+1.2.1

From eacc5937e41e57ccabee7549b06c774cebfc7dea Mon Sep 17 00:00:00 2001
From: Randy Pen <penzm@foxmail.com>
Date: Fri, 7 Oct 2016 16:48:57 +0800
Subject: [PATCH 1146/2974] fix example code

In the AuthorSpider, original css selector failed to get links of author pages
---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 4f2736709..89671910a 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -564,7 +564,7 @@ this time for scraping author information::
 
         def parse(self, response):
             # follow links to author pages
-            for href in response.css('.author a::attr(href)').extract():
+            for href in response.css('span a::attr(href)').extract():
                 yield scrapy.Request(response.urljoin(href),
                                      callback=self.parse_author)
 

From c7d245b90b4ee9c6a006965dd91fca14f2efeef7 Mon Sep 17 00:00:00 2001
From: Randy Pen <penzm@foxmail.com>
Date: Fri, 7 Oct 2016 22:52:58 +0800
Subject: [PATCH 1147/2974] update

Thx for your advice.
---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 89671910a..0fb2c0b86 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -564,7 +564,7 @@ this time for scraping author information::
 
         def parse(self, response):
             # follow links to author pages
-            for href in response.css('span a::attr(href)').extract():
+            for href in response.css('.author+a::attr(href)').extract():
                 yield scrapy.Request(response.urljoin(href),
                                      callback=self.parse_author)
 

From a958e549547f12364fcaf1754a4db7e24f140135 Mon Sep 17 00:00:00 2001
From: Steven Almeroth <sroth77@gmail.com>
Date: Fri, 21 Oct 2016 16:13:40 -0700
Subject: [PATCH 1148/2974] Doc: remove trailing spaces

---
 docs/topics/spiders.rst | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index af676fccd..d17cea941 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -24,8 +24,8 @@ For spiders, the scraping cycle goes through something like this:
    Requests.
 
 2. In the callback function, you parse the response (web page) and return either
-   dicts with extracted data, :class:`~scrapy.item.Item` objects, 
-   :class:`~scrapy.http.Request` objects, or an iterable of these objects. 
+   dicts with extracted data, :class:`~scrapy.item.Item` objects,
+   :class:`~scrapy.http.Request` objects, or an iterable of these objects.
    Those Requests will also contain a callback (maybe
    the same) and will then be downloaded by Scrapy and then their
    response handled by the specified callback.
@@ -56,7 +56,7 @@ scrapy.Spider
    must inherit (including spiders that come bundled with Scrapy, as well as spiders
    that you write yourself). It doesn't provide any special functionality. It just
    provides a default :meth:`start_requests` implementation which sends requests from
-   the :attr:`start_urls` spider attribute and calls the spider's method ``parse`` 
+   the :attr:`start_urls` spider attribute and calls the spider's method ``parse``
    for each of the resulting responses.
 
    .. attribute:: name
@@ -161,7 +161,7 @@ scrapy.Spider
 
            class MySpider(scrapy.Spider):
                name = 'myspider'
-                
+
                def start_requests(self):
                    return [scrapy.FormRequest("http://www.example.com/login",
                                               formdata={'user': 'john', 'pass': 'secret'},
@@ -247,8 +247,8 @@ Return multiple Requests and items from a single callback::
 
             for url in response.xpath('//a/@href').extract():
                 yield scrapy.Request(url, callback=self.parse)
-                
-Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; 
+
+Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
 to give data more structure you can use :ref:`topics-items`::
 
     import scrapy
@@ -257,7 +257,7 @@ to give data more structure you can use :ref:`topics-items`::
     class MySpider(scrapy.Spider):
         name = 'example.com'
         allowed_domains = ['example.com']
-        
+
         def start_requests(self):
             yield scrapy.Request('http://www.example.com/1.html', self.parse)
             yield scrapy.Request('http://www.example.com/2.html', self.parse)
@@ -269,7 +269,7 @@ to give data more structure you can use :ref:`topics-items`::
 
             for url in response.xpath('//a/@href').extract():
                 yield scrapy.Request(url, callback=self.parse)
-    
+
 .. _spiderargs:
 
 Spider arguments
@@ -301,7 +301,7 @@ Spider arguments can also be passed through the Scrapyd ``schedule.json`` API.
 See `Scrapyd documentation`_.
 
 .. _builtin-spiders:
-                
+
 Generic Spiders
 ===============
 

From 99daea495be7c445229b1851d4f217c0a4d9221e Mon Sep 17 00:00:00 2001
From: Steven Almeroth <sroth77@gmail.com>
Date: Fri, 21 Oct 2016 16:14:57 -0700
Subject: [PATCH 1149/2974] Doc: wording

---
 docs/topics/spiders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index d17cea941..0179e9284 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -285,7 +285,7 @@ Spider arguments are passed through the :command:`crawl` command using the
 
     scrapy crawl myspider -a category=electronics
 
-Spiders receive arguments in their constructors::
+Spiders can access arguments in their `__init__` methods::
 
     import scrapy
 

From d85da273be0e62f98b95ec1c78ac8862534749c7 Mon Sep 17 00:00:00 2001
From: John O'Connor <tehjcon@gmail.com>
Date: Fri, 28 Oct 2016 19:44:46 -0700
Subject: [PATCH 1150/2974] Fix typo in media pipeline docs

---
 docs/topics/media-pipeline.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 4ee4f1758..5e1ceff34 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -312,7 +312,7 @@ See here the methods that you can override in your custom Files Pipeline:
       By default the :meth:`get_media_requests` method returns ``None`` which
       means there are no files to download for the item.
 
-   .. method:: FilesPipeline.item_completed(results, items, info)
+   .. method:: FilesPipeline.item_completed(results, item, info)
 
       The :meth:`FilesPipeline.item_completed` method called when all file
       requests for a single item have completed (either finished downloading, or
@@ -355,7 +355,7 @@ See here the methods that you can override in your custom Images Pipeline:
 
       Must return a Request for each image URL.
 
-   .. method:: ImagesPipeline.item_completed(results, items, info)
+   .. method:: ImagesPipeline.item_completed(results, item, info)
 
       The :meth:`ImagesPipeline.item_completed` method is called when all image
       requests for a single item have completed (either finished downloading, or

From f2e49bc23cd2044551721bf81f6443a391b30f6f Mon Sep 17 00:00:00 2001
From: Gustavo de Andrade <gu_andrade1@yahoo.com.br>
Date: Tue, 1 Nov 2016 21:32:17 -0200
Subject: [PATCH 1151/2974] Update selectors.rst

Decode instead escape, exceptions &lt; and &amp; (kmike)
Second sentence droped (Digenis)
---
 docs/topics/selectors.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 293628953..9f8143db1 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -628,8 +628,7 @@ Built-in Selectors reference
 
     .. note::
 
-        Note that ``re`` and ``re_first`` both escape HTML entities. If you want to
-        get raw unescaped content, use extract() or extract_first()
+        Note that ``re()`` and ``re_first()`` both decode HTML entities (except ``&lt;`` and ``&amp;``).
 
   .. method:: register_namespace(prefix, uri)
 

From f56aef99c231e954387e59d6c5dffe500a2b21ca Mon Sep 17 00:00:00 2001
From: Rahul Kant <rahulkant13may@gmail.com>
Date: Mon, 7 Nov 2016 17:49:22 +0530
Subject: [PATCH 1152/2974] Add closing tag in <tbody>

---
 docs/topics/firefox.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/firefox.rst b/docs/topics/firefox.rst
index beda3b8db..0cf45861a 100644
--- a/docs/topics/firefox.rst
+++ b/docs/topics/firefox.rst
@@ -17,7 +17,7 @@ when inspecting the page source is not the original HTML, but a modified one
 after applying some browser clean up and executing Javascript code.  Firefox,
 in particular, is known for adding ``<tbody>`` elements to tables.  Scrapy, on
 the other hand, does not modify the original page HTML, so you won't be able to
-extract any data if you use ``<tbody`` in your XPath expressions. 
+extract any data if you use ``<tbody>`` in your XPath expressions. 
 
 Therefore, you should keep in mind the following things when working with
 Firefox and XPath:

From 7727d87f646d03a9f4d0d30d3dcc1161dd1cb1b5 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 7 Nov 2016 16:44:57 +0100
Subject: [PATCH 1153/2974] Test Slot's heartbeat state before stopping it

Also add a test on state of looping task in LogStats extension

Fixes #2011 and #2362
---
 scrapy/core/engine.py         | 3 ++-
 scrapy/extensions/logstats.py | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 3c4bc662c..2b5770138 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -48,7 +48,8 @@ class Slot(object):
         if self.closing and not self.inprogress:
             if self.nextcall:
                 self.nextcall.cancel()
-                self.heartbeat.stop()
+                if self.heartbeat.running:
+                    self.heartbeat.stop()
             self.closing.callback(None)
 
 
diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py
index 647e50f8d..b685e7b19 100644
--- a/scrapy/extensions/logstats.py
+++ b/scrapy/extensions/logstats.py
@@ -15,6 +15,7 @@ class LogStats(object):
         self.stats = stats
         self.interval = interval
         self.multiplier = 60.0 / self.interval
+        self.task = None
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -47,5 +48,5 @@ class LogStats(object):
         logger.info(msg, log_args, extra={'spider': spider})
 
     def spider_closed(self, spider, reason):
-        if self.task.running:
+        if self.task and self.task.running:
             self.task.stop()

From 61efacdd1f6fb55cf1f694e56502a3d8a1d5a325 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 8 Nov 2016 11:35:42 +0100
Subject: [PATCH 1154/2974] Add testcase for catching exception from
 open_spider() from pipeline

---
 tests/pipelines.py  | 11 +++++++++++
 tests/test_crawl.py | 12 ++++++++++++
 2 files changed, 23 insertions(+)
 create mode 100644 tests/pipelines.py

diff --git a/tests/pipelines.py b/tests/pipelines.py
new file mode 100644
index 000000000..d81cfa93d
--- /dev/null
+++ b/tests/pipelines.py
@@ -0,0 +1,11 @@
+"""
+Some pipelines used for testing and benchmarking
+"""
+
+class ZeroDivisionErrorPipeline(object):
+
+    def open_spider(self, spider):
+        a = 1/0
+
+    def process_item(self, item, spider):
+        return item
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 90fd921c8..a0f3c9997 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -250,6 +250,18 @@ with multiples lines
         yield self.assertFailure(crawler.crawl(), TestError)
         self.assertFalse(crawler.crawling)
 
+    @defer.inlineCallbacks
+    def test_open_spider_error_on_faulty_pipeline(self):
+        settings = {
+            "ITEM_PIPELINES": {
+                "tests.pipelines.ZeroDivisionErrorPipeline": 300,
+            }
+        }
+        crawler = CrawlerRunner(settings).create_crawler(SimpleSpider)
+        yield self.assertFailure(
+            self.runner.crawl(crawler, "http://localhost:8998/status?n=200"),
+            ZeroDivisionError)
+
     @defer.inlineCallbacks
     def test_crawlerrunner_accepts_crawler(self):
         crawler = self.runner.create_crawler(SimpleSpider)

From 27456996a904ba25f780d2e3824059e836b6cb90 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 8 Nov 2016 11:46:16 +0100
Subject: [PATCH 1155/2974] Add assertion on crawler not running

---
 tests/test_crawl.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index a0f3c9997..1b4a4b3b0 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -261,6 +261,7 @@ with multiples lines
         yield self.assertFailure(
             self.runner.crawl(crawler, "http://localhost:8998/status?n=200"),
             ZeroDivisionError)
+        self.assertFalse(crawler.crawling)
 
     @defer.inlineCallbacks
     def test_crawlerrunner_accepts_crawler(self):

From af2280e695f9430fdb0ba452594f3f9c4cb9592d Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 8 Nov 2016 13:30:51 +0100
Subject: [PATCH 1156/2974] Update docstring

---
 tests/pipelines.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/pipelines.py b/tests/pipelines.py
index d81cfa93d..ddfbc7a99 100644
--- a/tests/pipelines.py
+++ b/tests/pipelines.py
@@ -1,5 +1,5 @@
 """
-Some pipelines used for testing and benchmarking
+Some pipelines used for testing
 """
 
 class ZeroDivisionErrorPipeline(object):

From 3cd56da0cc68a1c1d0ac4ca38dabc5b8aab3e461 Mon Sep 17 00:00:00 2001
From: Rolando Espinoza <rndmax84@gmail.com>
Date: Tue, 8 Nov 2016 20:52:32 -0300
Subject: [PATCH 1157/2974] Ignore explicitly compiled python files.

This avoids to include compiled files in the templates directory.
---
 MANIFEST.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index 04b3e1fb9..94de4f3bf 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -12,3 +12,4 @@ prune docs/build
 recursive-include extras *
 recursive-include bin *
 recursive-include tests *
+global-exclude __pycache__ *.py[cod]

From 76459e1969245f214dd39868c11090cdf6e7cf04 Mon Sep 17 00:00:00 2001
From: Rolando Espinoza <rndmax84@gmail.com>
Date: Tue, 8 Nov 2016 21:30:27 -0300
Subject: [PATCH 1158/2974] Update conda channel to conda-forge and show conda
 version badge.

---
 README.rst             | 4 ++++
 docs/intro/install.rst | 7 ++++---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index 74ae1f30d..b72ebf53d 100644
--- a/README.rst
+++ b/README.rst
@@ -22,6 +22,10 @@ Scrapy
    :target: http://codecov.io/github/scrapy/scrapy?branch=master
    :alt: Coverage report
 
+.. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg
+   :target: https://anaconda.org/conda-forge/scrapy
+   :alt: Conda Version
+
 
 Overview
 ========
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 7c806a049..767749ec5 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -245,12 +245,12 @@ Using Anaconda is an alternative to using a virtualenv and installing with ``pip
   For Windows users, or if you have issues installing through ``pip``, this is
   the recommended way to install Scrapy.
 
-If you already have `Anaconda`_ or `Miniconda`_ installed,
-`Scrapinghub`_ maintains official conda packages for Linux, Windows and OS X.
+If you already have `Anaconda`_ or `Miniconda`_ installed, the `conda-forge`_
+community have up-to-date packages for Linux, Windows and OS X.
 
 To install Scrapy using ``conda``, run::
 
-  conda install -c scrapinghub scrapy
+  conda install -c conda-forge scrapy
 
 .. _Python: https://www.python.org/
 .. _pip: https://pip.pypa.io/en/latest/installing/
@@ -268,3 +268,4 @@ To install Scrapy using ``conda``, run::
 .. _Scrapinghub: http://scrapinghub.com
 .. _Anaconda: http://docs.continuum.io/anaconda/index
 .. _Miniconda: http://conda.pydata.org/docs/install/quick.html
+.. _conda-forge: https://conda-forge.github.io/

From 28155dfcccc0cd7883ffde714570e8c13b0920a6 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 9 Nov 2016 12:20:06 +0100
Subject: [PATCH 1159/2974] Parse robots.txt content as native str

Fixes #2373
---
 scrapy/downloadermiddlewares/robotstxt.py    |  5 ++++-
 tests/test_downloadermiddleware_robotstxt.py | 13 ++++++++++---
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 698f394ad..e26c22a09 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -13,6 +13,7 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.log import failure_to_exc_info
+from scrapy.utils.python import to_native_str
 
 logger = logging.getLogger(__name__)
 
@@ -94,7 +95,9 @@ class RobotsTxtMiddleware(object):
                 # Running rp.parse() will set rp state from
                 # 'disallow all' to 'allow any'.
                 pass
-        rp.parse(body.splitlines())
+        # stdlib's robotparser expects native 'str' ;
+        # with unicode input, non-ASCII encoded bytes decoding fails in Python2
+        rp.parse(to_native_str(body).splitlines())
 
         rp_dfd = self._parsers[netloc]
         self._parsers[netloc] = rp
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index f2e94e171..95208c41f 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 import re
 from twisted.internet import reactor, error
@@ -30,11 +31,15 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
     def _get_successful_crawler(self):
         crawler = self.crawler
         crawler.settings.set('ROBOTSTXT_OBEY', True)
-        ROBOTS = re.sub(b'^\s+(?m)', b'', b'''
+        ROBOTS = re.sub(b'^\s+(?m)', b'', u'''
         User-Agent: *
         Disallow: /admin/
         Disallow: /static/
-        ''')
+
+        # taken from https://en.wikipedia.org/robots.txt
+        Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
+        Disallow: /wiki/Käyttäjä:
+        '''.encode('utf-8'))
         response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
         def return_response(request, spider):
             deferred = Deferred()
@@ -48,7 +53,9 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         return DeferredList([
             self.assertNotIgnored(Request('http://site.local/allowed'), middleware),
             self.assertIgnored(Request('http://site.local/admin/main'), middleware),
-            self.assertIgnored(Request('http://site.local/static/'), middleware)
+            self.assertIgnored(Request('http://site.local/static/'), middleware),
+            self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware),
+            self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware)
         ], fireOnOneErrback=True)
 
     def test_robotstxt_ready_parser(self):

From e8205f67339bdd3cdd6450e104b8969c9b98692d Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Wed, 29 Jul 2015 15:34:27 +0000
Subject: [PATCH 1160/2974] LinkExtractor PY3 'unicode' type fix

---
 scrapy/linkextractors/htmlparser.py | 3 ++-
 scrapy/linkextractors/sgml.py       | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index dcc261b31..9867e1179 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -3,6 +3,7 @@ HTMLParser-based link extractor
 """
 
 import warnings
+import six
 from six.moves.html_parser import HTMLParser
 from six.moves.urllib.parse import urljoin
 
@@ -39,7 +40,7 @@ class HtmlParserLinkExtractor(HTMLParser):
         ret = []
         base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
         for link in links:
-            if isinstance(link.url, unicode):
+            if isinstance(link.url, six.text_type):
                 link.url = link.url.encode(response_encoding)
             try:
                 link.url = urljoin(base_url, link.url)
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 9938e071f..c68dae4c8 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -1,6 +1,7 @@
 """
 SGMLParser-based Link extractors
 """
+import six
 from six.moves.urllib.parse import urljoin
 import warnings
 from sgmllib import SGMLParser
@@ -40,7 +41,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
         if base_url is None:
             base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
         for link in self.links:
-            if isinstance(link.url, unicode):
+            if isinstance(link.url, six.text_type):
                 link.url = link.url.encode(response_encoding)
             try:
                 link.url = urljoin(base_url, link.url)

From 7025d6656a127d18ac1ef4f29a912181308c6971 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Tue, 18 Oct 2016 11:51:13 -0200
Subject: [PATCH 1161/2974] document download_latency meta key

---
 docs/topics/request-response.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 75b98d3b3..7e700c7cb 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -299,6 +299,7 @@ Those are:
 * :reqmeta:`dont_obey_robotstxt`
 * :reqmeta:`download_timeout`
 * :reqmeta:`download_maxsize`
+* :reqmeta:`download_latency`
 * :reqmeta:`proxy`
 
 .. reqmeta:: bindaddress
@@ -316,6 +317,15 @@ download_timeout
 The amount of time (in secs) that the downloader will wait before timing out.
 See also: :setting:`DOWNLOAD_TIMEOUT`.
 
+.. reqmeta:: download_latency
+
+download_latency
+----------------
+
+The amount of time spent to fetch the response, since the request has been
+started, i.e. HTTP message sent over the network. This meta key only becomes
+available when the response has been downloaded. While most other meta keys are
+used to control Scrapy behavior, this one is supposed to be read-only.
 
 .. _topics-request-response-ref-request-subclasses:
 

From 6cd35c77da6601f218b5042f4b9a9919e642e15c Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 15 Nov 2016 17:38:32 +0100
Subject: [PATCH 1162/2974] Pass user-agent as native str when checking URLs
 against robots.txt

---
 scrapy/downloadermiddlewares/robotstxt.py    | 3 ++-
 tests/test_downloadermiddleware_robotstxt.py | 3 +++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index e26c22a09..c3dfa7819 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -41,7 +41,8 @@ class RobotsTxtMiddleware(object):
         return d
 
     def process_request_2(self, rp, request, spider):
-        if rp is not None and not rp.can_fetch(self._useragent, request.url):
+        if rp is not None and not rp.can_fetch(
+                 to_native_str(self._useragent), request.url):
             logger.debug("Forbidden by robots.txt: %(request)s",
                          {'request': request}, extra={'spider': spider})
             raise IgnoreRequest()
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 95208c41f..60306eacb 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -39,6 +39,9 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         # taken from https://en.wikipedia.org/robots.txt
         Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
         Disallow: /wiki/Käyttäjä:
+
+        User-Agent: UnicödeBöt
+        Disallow: /some/randome/page.html
         '''.encode('utf-8'))
         response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
         def return_response(request, spider):

From 11fe3751cf96ebbf3af3c1446a736cd577883495 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 16 Nov 2016 11:55:09 +0100
Subject: [PATCH 1163/2974] DOC Remove "Ubuntu" section from sidebar/ToC

---
 docs/index.rst | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/docs/index.rst b/docs/index.rst
index b4272e47f..289fb2b1b 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -155,7 +155,6 @@ Solving specific problems
    topics/firebug
    topics/leaks
    topics/media-pipeline
-   topics/ubuntu
    topics/deploy
    topics/autothrottle
    topics/benchmarking
@@ -188,9 +187,6 @@ Solving specific problems
 :doc:`topics/media-pipeline`
     Download files and/or images associated with your scraped items.
 
-:doc:`topics/ubuntu`
-    Install latest Scrapy packages easily on Ubuntu
-
 :doc:`topics/deploy`
     Deploying your Scrapy spiders and run them in a remote server.
 

From 8db85453939b4431be8142a7b6733acbc6d71416 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 16 Nov 2016 13:56:58 +0100
Subject: [PATCH 1164/2974] Add "orphan" metadata for Ubuntu packages page

As described in http://www.sphinx-doc.org/en/latest/markup/misc.html#metadata
---
 docs/topics/ubuntu.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst
index f1a870d62..679bb56ff 100644
--- a/docs/topics/ubuntu.rst
+++ b/docs/topics/ubuntu.rst
@@ -1,3 +1,5 @@
+:orphan: Ubuntu packages are obsolete
+
 .. _topics-ubuntu:
 
 ===============

From d62776a8589a7ae8fd98403020ce85d01e6b8977 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Wed, 16 Nov 2016 12:19:32 -0300
Subject: [PATCH 1165/2974] mention scrapoxy in best practices doc

---
 docs/topics/practices.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 7dae68470..25ae4b5ba 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -238,7 +238,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
 * if possible, use `Google cache`_ to fetch pages, instead of hitting the sites
   directly
 * use a pool of rotating IPs. For example, the free `Tor project`_ or paid
-  services like `ProxyMesh`_
+  services like `ProxyMesh`_. An open source alterantive is `scrapoxy`_, a
+  super proxy that you can attach your own proxies to.
 * use a highly distributed downloader that circumvents bans internally, so you
   can just focus on parsing clean pages. One example of such downloaders is
   `Crawlera`_
@@ -253,3 +254,4 @@ If you are still unable to prevent your bot getting banned, consider contacting
 .. _testspiders: https://github.com/scrapinghub/testspiders
 .. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
 .. _Crawlera: http://scrapinghub.com/crawlera
+.. _scrapoxy: http://scrapoxy.io/

From 01142e2ae5e3b82d5f8701858931ba0354808720 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 22 Nov 2016 14:48:33 +0100
Subject: [PATCH 1166/2974] Print more dependencies versions in "scrapy
 version" verbose output

---
 scrapy/commands/version.py    | 13 +++++++++++++
 tests/test_command_version.py |  6 ++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 4bf085c9e..a9954edb0 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -26,12 +26,25 @@ class Command(ScrapyCommand):
 
     def run(self, args, opts):
         if opts.verbose:
+            import cssselect
+            import parsel
             import lxml.etree
+            import w3lib
+
             lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
             libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
+
+            try:
+                w3lib_version = w3lib.__version__
+            except AttributeError:
+                w3lib_version = "<1.14.3"
+
             print("Scrapy    : %s" % scrapy.__version__)
             print("lxml      : %s" % lxml_version)
             print("libxml2   : %s" % libxml2_version)
+            print("cssselect : %s" % cssselect.__version__)
+            print("parsel    : %s" % parsel.__version__)
+            print("w3lib     : %s" % w3lib_version)
             print("Twisted   : %s" % twisted.version.short())
             print("Python    : %s" % sys.version.replace("\n", "- "))
             print("pyOpenSSL : %s" % self._get_openssl_version())
diff --git a/tests/test_command_version.py b/tests/test_command_version.py
index 37e1f2543..2789d207c 100644
--- a/tests/test_command_version.py
+++ b/tests/test_command_version.py
@@ -25,5 +25,7 @@ class VersionTest(ProcessTest, unittest.TestCase):
         _, out, _ = yield self.execute(['-v'])
         headers = [l.partition(":")[0].strip()
                    for l in out.strip().decode(encoding).splitlines()]
-        self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', 'Twisted',
-                                   'Python', 'pyOpenSSL', 'Platform'])
+        self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2',
+                                   'cssselect', 'parsel', 'w3lib',
+                                   'Twisted', 'Python', 'pyOpenSSL',
+                                   'Platform'])

From 35b655d2f84d652440c393166f6e19d7384b4f1c Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 24 Nov 2016 12:23:22 +0100
Subject: [PATCH 1167/2974] Handle redirects transparently by default in shell
 and fetch

Adds --no-status-aware command line option to have previous behaviour
---
 scrapy/commands/fetch.py |  5 ++++-
 scrapy/commands/shell.py |  4 +++-
 scrapy/shell.py          | 12 ++++++------
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index f09a873c1..a157b19f8 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -27,6 +27,8 @@ class Command(ScrapyCommand):
             help="use this spider")
         parser.add_option("--headers", dest="headers", action="store_true", \
             help="print response HTTP headers instead of body")
+        parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \
+            default=False, help="do not handle status codes like redirects and print response as-is")
 
     def _print_headers(self, headers, prefix):
         for key, values in headers.items():
@@ -50,7 +52,8 @@ class Command(ScrapyCommand):
             raise UsageError()
         cb = lambda x: self._print_response(x, opts)
         request = Request(args[0], callback=cb, dont_filter=True)
-        request.meta['handle_httpstatus_all'] = True
+        if opts.no_status_aware:
+            request.meta['handle_httpstatus_all'] = True
 
         spidercls = DefaultSpider
         spider_loader = self.crawler_process.spider_loader
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 7be7f7256..bc0203d89 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -36,6 +36,8 @@ class Command(ScrapyCommand):
             help="evaluate the code in the shell, print the result and exit")
         parser.add_option("--spider", dest="spider",
             help="use this spider")
+        parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \
+            default=False, help="do not transparently handle status codes like redirects")
 
     def update_vars(self, vars):
         """You can use this function to update the Scrapy objects that will be
@@ -68,7 +70,7 @@ class Command(ScrapyCommand):
         self._start_crawler_thread()
 
         shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
-        shell.start(url=url)
+        shell.start(url=url, handle_statuses=opts.no_status_aware)
 
     def _start_crawler_thread(self):
         t = Thread(target=self.crawler_process.start,
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 183ee1f70..966003f17 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -40,11 +40,11 @@ class Shell(object):
         self.code = code
         self.vars = {}
 
-    def start(self, url=None, request=None, response=None, spider=None):
+    def start(self, url=None, request=None, response=None, spider=None, handle_statuses=True):
         # disable accidental Ctrl-C key press from shutting down the engine
         signal.signal(signal.SIGINT, signal.SIG_IGN)
         if url:
-            self.fetch(url, spider)
+            self.fetch(url, spider, handle_statuses=handle_statuses)
         elif request:
             self.fetch(request, spider)
         elif response:
@@ -98,14 +98,14 @@ class Shell(object):
         self.spider = spider
         return spider
 
-    def fetch(self, request_or_url, spider=None):
+    def fetch(self, request_or_url, spider=None, handle_statuses=False, **kwargs):
         if isinstance(request_or_url, Request):
             request = request_or_url
-            url = request.url
         else:
             url = any_to_uri(request_or_url)
-            request = Request(url, dont_filter=True)
-            request.meta['handle_httpstatus_all'] = True
+            request = Request(url, dont_filter=True, **kwargs)
+            if handle_statuses:
+                request.meta['handle_httpstatus_all'] = True
         response = None
         try:
             response, spider = threads.blockingCallFromThread(

From 9aefc0a886ca571a49f087e5349e2557fd78d943 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 24 Nov 2016 13:41:51 +0100
Subject: [PATCH 1168/2974] Add test for fetch command with redirections
 disabled

---
 scrapy/utils/testsite.py    |  8 ++++++++
 tests/test_command_fetch.py | 12 ++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index ad0375443..e50a989b3 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -20,12 +20,20 @@ class SiteTest(object):
         return urljoin(self.baseurl, path)
 
 
+class NoMetaRefreshRedirect(util.Redirect):
+    def render(self, request):
+        content = util.Redirect.render(self, request)
+        return content.replace(b'http-equiv=\"refresh\"',
+            b'http-no-equiv=\"do-not-refresh-me\"')
+
+
 def test_site():
     r = resource.Resource()
     r.putChild(b"text", static.Data(b"Works", "text/plain"))
     r.putChild(b"html", static.Data(b"<body><p class='one'>Works</p><p class='two'>World</p></body>", "text/html"))
     r.putChild(b"enc-gb18030", static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"))
     r.putChild(b"redirect", util.Redirect(b"/redirected"))
+    r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected"))
     r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
     return server.Site(r)
 
diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py
index 4843a9a2f..45d03a129 100644
--- a/tests/test_command_fetch.py
+++ b/tests/test_command_fetch.py
@@ -14,6 +14,18 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
         _, out, _ = yield self.execute([self.url('/text')])
         self.assertEqual(out.strip(), b'Works')
 
+    @defer.inlineCallbacks
+    def test_redirect_default(self):
+        _, out, _ = yield self.execute([self.url('/redirect')])
+        self.assertEqual(out.strip(), b'Redirected here')
+
+    @defer.inlineCallbacks
+    def test_redirect_disabled(self):
+        _, out, err = yield self.execute(['--no-status-aware', self.url('/redirect-no-meta-refresh')])
+        err = err.strip()
+        self.assertIn(b'downloader/response_status_count/302', err, err)
+        self.assertNotIn(b'downloader/response_status_count/200', err, err)
+
     @defer.inlineCallbacks
     def test_headers(self):
         _, out, _ = yield self.execute([self.url('/text'), '--headers'])

From 03cf5f1bd2019127933ea4a6358a7d47743efcf6 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 24 Nov 2016 17:18:57 +0100
Subject: [PATCH 1169/2974] Remove ChunkedTransferMiddleware from default
 settings

---
 docs/topics/downloader-middleware.rst | 8 +++++++-
 docs/topics/settings.rst              | 1 -
 scrapy/settings/default_settings.py   | 1 -
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 15069e56e..dca5ec6a0 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -665,7 +665,13 @@ ChunkedTransferMiddleware
 
 .. class:: ChunkedTransferMiddleware
 
-   This middleware adds support for `chunked transfer encoding`_
+   This middleware adds support for `chunked transfer encoding`_.
+
+.. note::
+    This middleware is not enabled nor used by Scrapy downloader anymore.
+    In fact, Scrapy downloader has built-in support for chunked transfers,
+    so this middleware has no effect in practice.
+
 
 HttpProxyMiddleware
 -------------------
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 8540308fe..2195e4233 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -468,7 +468,6 @@ Default::
         'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
         'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
         'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
-        'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830,
         'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
         'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
     }
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 18d5ebbbb..61f4bd567 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -102,7 +102,6 @@ DOWNLOADER_MIDDLEWARES_BASE = {
     'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
     'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
     'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
-    'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830,
     'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
     'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
     # Downloader side

From e6f174b01535810fffeae2d020bc66a6a25ba4e8 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 24 Nov 2016 17:33:27 +0100
Subject: [PATCH 1170/2974] Add deprecation warning for ChunkedTransfer
 middleware

---
 scrapy/downloadermiddlewares/chunked.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py
index 57e97e4d2..fd90aab2a 100644
--- a/scrapy/downloadermiddlewares/chunked.py
+++ b/scrapy/downloadermiddlewares/chunked.py
@@ -1,6 +1,14 @@
+import warnings
+
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.http import decode_chunked_transfer
 
 
+warnings.warn("Module `scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware` "
+              "is deprecated, chunked transfers are supported by default.",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+
 class ChunkedTransferMiddleware(object):
     """This middleware adds support for chunked transfer encoding, as
     documented in: http://en.wikipedia.org/wiki/Chunked_transfer_encoding

From 8cffb4bbefcebd10ec12ee2678fd490edf149576 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 24 Nov 2016 17:50:21 +0100
Subject: [PATCH 1171/2974] Update warning wording

---
 scrapy/downloadermiddlewares/chunked.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py
index fd90aab2a..64d94c489 100644
--- a/scrapy/downloadermiddlewares/chunked.py
+++ b/scrapy/downloadermiddlewares/chunked.py
@@ -4,8 +4,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.http import decode_chunked_transfer
 
 
-warnings.warn("Module `scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware` "
-              "is deprecated, chunked transfers are supported by default.",
+warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, "
+              "chunked transfers are supported by default.",
               ScrapyDeprecationWarning, stacklevel=2)
 
 

From 059085b5b4a54901c4fd0730e43b156a6d8eeb51 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 24 Nov 2016 18:23:34 +0100
Subject: [PATCH 1172/2974] Remove docs for deprecated ChunkedTransfer
 middleware

---
 docs/topics/downloader-middleware.rst | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index dca5ec6a0..29d9b0298 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -657,22 +657,6 @@ Default: ``True``
 Whether the Compression middleware will be enabled.
 
 
-ChunkedTransferMiddleware
--------------------------
-
-.. module:: scrapy.downloadermiddlewares.chunked
-   :synopsis: Chunked Transfer Middleware
-
-.. class:: ChunkedTransferMiddleware
-
-   This middleware adds support for `chunked transfer encoding`_.
-
-.. note::
-    This middleware is not enabled nor used by Scrapy downloader anymore.
-    In fact, Scrapy downloader has built-in support for chunked transfers,
-    so this middleware has no effect in practice.
-
-
 HttpProxyMiddleware
 -------------------
 
@@ -976,4 +960,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
 
 .. _DBM: https://en.wikipedia.org/wiki/Dbm
 .. _anydbm: https://docs.python.org/2/library/anydbm.html
-.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding

From f98ffb53b66cf2e5b38b393f811f59ccc1d68992 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Tue, 29 Nov 2016 16:52:54 +0100
Subject: [PATCH 1173/2974] add docs for some scheduler settings

---
 docs/faq.rst             |  2 +-
 docs/topics/settings.rst | 29 +++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 415331515..ad11b071b 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -317,6 +317,6 @@ I'm scraping a XML document and my XPath selector doesn't return any items
 You may need to remove namespaces. See :ref:`removing-namespaces`.
 
 .. _user agents: https://en.wikipedia.org/wiki/User_agent
-.. _LIFO: https://en.wikipedia.org/wiki/LIFO
+.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
 .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
 .. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 8540308fe..91f3b37c9 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1033,6 +1033,35 @@ Example entry in logs::
     (type Request)> - no more unserializable requests will be logged
     (see 'scheduler/unserializable' stats counter)
 
+
+.. setting:: SCHEDULER_DISK_QUEUE
+
+    SCHEDULER_DISK_QUEUE
+--------------------
+
+Default: ``'scrapy.squeues.PickleLifoDiskQueue'``
+
+Type of disk queue that will be used by scheduler. Other available types are
+``scrapy.squeues.PickleFifoDiskQueue``, ``scrapy.squeues.MarshalFifoDiskQueue``,
+``scrapy.squeues.MarshalLifoDiskQueue``.
+
+.. setting:: SCHEDULER_MEMORY_QUEUE
+
+SCHEDULER_MEMORY_QUEUE
+----------------------
+Default: ``'scrapy.squeues.LifoMemoryQueue'``
+
+Type of in-memory queue used by scheduler. Other available type is:
+``scrapy.squeues.FifoMemoryQueue``.
+
+.. setting:: SCHEDULER_PRIORITY_QUEUE
+
+SCHEDULER_PRIORITY_QUEUE
+------------------------
+Default: ``'queuelib.PriorityQueue'``
+
+Type of priority queue used by scheduler.
+
 .. setting:: SPIDER_CONTRACTS
 
 SPIDER_CONTRACTS

From 624284e85145aa62b0e7fc21bf0d61bda2cbbefa Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 29 Nov 2016 18:18:59 +0100
Subject: [PATCH 1174/2974] Fix indent

---
 docs/topics/settings.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 91f3b37c9..fe3767a53 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1036,7 +1036,7 @@ Example entry in logs::
 
 .. setting:: SCHEDULER_DISK_QUEUE
 
-    SCHEDULER_DISK_QUEUE
+SCHEDULER_DISK_QUEUE
 --------------------
 
 Default: ``'scrapy.squeues.PickleLifoDiskQueue'``

From 27606aad1166afe3eebbee8dc365b1a122639e3c Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Wed, 30 Nov 2016 09:47:02 +0200
Subject: [PATCH 1175/2974] Fix #396 re-triggered issue

The InteractiveShellEmbed class is a singleton
and we need to drop the instance with its clear_instance() method
to rebuild the instance from scratch with fresh environment
for each subsequent Scrapy's shell drop in.
---
 scrapy/utils/console.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 567fd51bc..1888d9599 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -15,6 +15,9 @@ def _embed_ipython_shell(namespace={}, banner=''):
         config = load_default_config()
         # Always use .instace() to ensure _instance propagation to all parents
         # this is needed for <TAB> completion works well for new imports
+        # and clear the instance to always have the fresh env
+        # on repeated breaks like with inspect_response()
+        InteractiveShellEmbed.clear_instance()
         shell = InteractiveShellEmbed.instance(
             banner1=banner, user_ns=namespace, config=config)
         shell()

From 5ff64ad015aff872ffae78193b67525dd53ae80d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 15 Nov 2016 11:47:25 -0300
Subject: [PATCH 1176/2974] handle relative sitemap urls in robots.txt

---
 scrapy/spiders/sitemap.py   | 2 +-
 scrapy/utils/sitemap.py     | 7 +++++--
 tests/test_spider.py        | 6 +++++-
 tests/test_utils_sitemap.py | 9 +++++++--
 4 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 89d96c330..9e45637c3 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -32,7 +32,7 @@ class SitemapSpider(Spider):
 
     def _parse_sitemap(self, response):
         if response.url.endswith('/robots.txt'):
-            for url in sitemap_urls_from_robots(response.text):
+            for url in sitemap_urls_from_robots(response.text, base_url=response.url):
                 yield Request(url, callback=self._parse_sitemap)
         else:
             body = self._get_sitemap_body(response)
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index 008196435..4742b3e13 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -4,7 +4,9 @@ Module for processing Sitemaps.
 Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
+
 import lxml.etree
+from six.moves.urllib.parse import urljoin
 
 
 class Sitemap(object):
@@ -34,10 +36,11 @@ class Sitemap(object):
                 yield d
 
 
-def sitemap_urls_from_robots(robots_text):
+def sitemap_urls_from_robots(robots_text, base_url=None):
     """Return an iterator over all sitemap urls contained in the given
     robots.txt file
     """
     for line in robots_text.splitlines():
         if line.lstrip().lower().startswith('sitemap:'):
-            yield line.split(':', 1)[1].strip()
+            url = line.split(':', 1)[1].strip()
+            yield urljoin(base_url, url)
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 1d22c1212..079734a69 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -332,13 +332,17 @@ class SitemapSpiderTest(SpiderTest):
         robots = b"""# Sitemap files
 Sitemap: http://example.com/sitemap.xml
 Sitemap: http://example.com/sitemap-product-index.xml
+Sitemap: HTTP://example.com/sitemap-uppercase.xml
+Sitemap: /sitemap-relative-url.xml
 """
 
         r = TextResponse(url="http://www.example.com/robots.txt", body=robots)
         spider = self.spider_class("example.com")
         self.assertEqual([req.url for req in spider._parse_sitemap(r)],
                          ['http://example.com/sitemap.xml',
-                          'http://example.com/sitemap-product-index.xml'])
+                          'http://example.com/sitemap-product-index.xml',
+                          'http://example.com/sitemap-uppercase.xml',
+                          'http://www.example.com/sitemap-relative-url.xml'])
 
 
 class BaseSpiderDeprecationTest(unittest.TestCase):
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index bd2677956..716bb44eb 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -119,13 +119,18 @@ Disallow: /s*/*tags
 # Sitemap files
 Sitemap: http://example.com/sitemap.xml
 Sitemap: http://example.com/sitemap-product-index.xml
+Sitemap: HTTP://example.com/sitemap-uppercase.xml
+Sitemap: /sitemap-relative-url.xml
 
 # Forums
 Disallow: /forum/search/
 Disallow: /forum/active/
 """
-        self.assertEqual(list(sitemap_urls_from_robots(robots)),
-             ['http://example.com/sitemap.xml', 'http://example.com/sitemap-product-index.xml'])
+        self.assertEqual(list(sitemap_urls_from_robots(robots, base_url='http://example.com')),
+                         ['http://example.com/sitemap.xml',
+                          'http://example.com/sitemap-product-index.xml',
+                          'http://example.com/sitemap-uppercase.xml',
+                          'http://example.com/sitemap-relative-url.xml'])
 
     def test_sitemap_blanklines(self):
         """Assert we can deal with starting blank lines before <xml> tag"""

From d9f43e21ba510ee3c95c89c3d551bc30a8d0f2c9 Mon Sep 17 00:00:00 2001
From: Rolando Espinoza <rndmax84@gmail.com>
Date: Thu, 1 Dec 2016 11:56:33 -0300
Subject: [PATCH 1177/2974] TST: Fix duplicated test name.

---
 tests/test_spiderloader/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 68dca2e98..83c3a3670 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -66,7 +66,7 @@ class SpiderLoaderTest(unittest.TestCase):
         self.spider_loader = SpiderLoader.from_settings(settings)
         assert len(self.spider_loader._spiders) == 1
 
-    def test_load_spider_module(self):
+    def test_load_spider_module_multiple(self):
         prefix = 'tests.test_spiderloader.test_spiders.'
         module = ','.join(prefix + s for s in ('spider1', 'spider2'))
         settings = Settings({'SPIDER_MODULES': module})

From 6431e7a1386e03fff203ae85b6c47a4f0f4cf797 Mon Sep 17 00:00:00 2001
From: Rolando Espinoza <rndmax84@gmail.com>
Date: Thu, 1 Dec 2016 13:24:12 -0300
Subject: [PATCH 1178/2974] DOC State explicitly that spiders are loaded
 recursively.

---
 docs/topics/api.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index d470a0d41..985cc0433 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -171,7 +171,8 @@ SpiderLoader API
 
        This class method is used by Scrapy to create an instance of the class.
        It's called with the current project settings, and it loads the spiders
-       found in the modules of the :setting:`SPIDER_MODULES` setting.
+       found recursively in the modules of the :setting:`SPIDER_MODULES`
+       setting.
 
        :param settings: project settings
        :type settings: :class:`~scrapy.settings.Settings` instance

From 923b974f0a10c3eb007ee4d183aa63a41d6a6db2 Mon Sep 17 00:00:00 2001
From: Rolando Espinoza <rndmax84@gmail.com>
Date: Thu, 1 Dec 2016 12:52:52 -0300
Subject: [PATCH 1179/2974] TST Include nested a nested spider in spider loader
 test.

---
 tests/test_spiderloader/__init__.py                     | 6 +++---
 tests/test_spiderloader/test_spiders/nested/__init__.py | 0
 tests/test_spiderloader/test_spiders/nested/spider4.py  | 9 +++++++++
 3 files changed, 12 insertions(+), 3 deletions(-)
 create mode 100644 tests/test_spiderloader/test_spiders/nested/__init__.py
 create mode 100644 tests/test_spiderloader/test_spiders/nested/spider4.py

diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 68dca2e98..74e461215 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -9,6 +9,7 @@ from twisted.trial import unittest
 # ugly hack to avoid cyclic imports of scrapy.spiders when running this test
 # alone
 import scrapy
+import tempfile
 from scrapy.interfaces import ISpiderLoader
 from scrapy.spiderloader import SpiderLoader
 from scrapy.settings import Settings
@@ -22,8 +23,7 @@ class SpiderLoaderTest(unittest.TestCase):
 
     def setUp(self):
         orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
-        self.tmpdir = self.mktemp()
-        os.mkdir(self.tmpdir)
+        self.tmpdir = tempfile.mkdtemp()
         self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
         shutil.copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(self.tmpdir)
@@ -40,7 +40,7 @@ class SpiderLoaderTest(unittest.TestCase):
 
     def test_list(self):
         self.assertEqual(set(self.spider_loader.list()),
-            set(['spider1', 'spider2', 'spider3']))
+            set(['spider1', 'spider2', 'spider3', 'spider4']))
 
     def test_load(self):
         spider1 = self.spider_loader.load("spider1")
diff --git a/tests/test_spiderloader/test_spiders/nested/__init__.py b/tests/test_spiderloader/test_spiders/nested/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_spiderloader/test_spiders/nested/spider4.py b/tests/test_spiderloader/test_spiders/nested/spider4.py
new file mode 100644
index 000000000..35b71870a
--- /dev/null
+++ b/tests/test_spiderloader/test_spiders/nested/spider4.py
@@ -0,0 +1,9 @@
+from scrapy.spiders import Spider
+
+class Spider4(Spider):
+    name = "spider4"
+    allowed_domains = ['spider4.com']
+
+    @classmethod
+    def handles_request(cls, request):
+        return request.url == 'http://spider4.com/onlythis'

From e1ea0c433a96c2949b0a7c26cde65aea13fb79b4 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Thu, 1 Dec 2016 22:02:10 +0000
Subject: [PATCH 1180/2974] Strip xlib.tx code of Twisted 10

---
 scrapy/xlib/tx/__init__.py   | 12 ++++-----
 scrapy/xlib/tx/_newclient.py | 31 +++++++++++++++++-------
 scrapy/xlib/tx/client.py     | 28 +++++++++++++++------
 scrapy/xlib/tx/endpoints.py  | 26 +++++++++++++++-----
 scrapy/xlib/tx/interfaces.py | 47 ++++++++++++++++++++++++++----------
 scrapy/xlib/tx/iweb.py       | 18 ++++++++++----
 6 files changed, 115 insertions(+), 47 deletions(-)

diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx/__init__.py
index 1ac4e0108..1c9bf09e5 100644
--- a/scrapy/xlib/tx/__init__.py
+++ b/scrapy/xlib/tx/__init__.py
@@ -15,9 +15,9 @@ else:
     client = endpoints = _Mock()
 
 
-Agent = client.Agent
-ProxyAgent = client.ProxyAgent
-ResponseDone = client.ResponseDone
-ResponseFailed = client.ResponseFailed
-HTTPConnectionPool = client.HTTPConnectionPool
-TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint
+Agent = client.Agent  # since < 11.1
+ProxyAgent = client.ProxyAgent  # since 11.1
+ResponseDone = client.ResponseDone  # since 11.1
+ResponseFailed = client.ResponseFailed  # since 11.1
+HTTPConnectionPool = client.HTTPConnectionPool  # since 12.1
+TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint  # since 10.1
diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py
index 16d0ca6b4..e902d6683 100644
--- a/scrapy/xlib/tx/_newclient.py
+++ b/scrapy/xlib/tx/_newclient.py
@@ -39,12 +39,25 @@ from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred
 from twisted.internet.defer import CancelledError
 from twisted.internet.protocol import Protocol
 from twisted.protocols.basic import LineReceiver
+from twisted.web.iweb import UNKNOWN_LENGTH
 from twisted.web.http_headers import Headers
 from twisted.web.http import NO_CONTENT, NOT_MODIFIED
 from twisted.web.http import _DataLoss, PotentialDataLoss
 from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder
 
-from .iweb import IResponse, UNKNOWN_LENGTH
+from twisted.web._newclient import (
+    BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException,
+    RequestGenerationFailed, RequestTransmissionFailed,
+    WrongBodyLength, ResponseDone, RequestNotSent,
+    LengthEnforcingConsumer, makeStatefulDispatcher, ChunkedEncoder,
+    TransportProxyProducer,
+)
+# newer than 10.0.0
+#from twisted.web._newclient import (
+#    ConnectionAborted, ResponseFailed, ResponseNeverReceived, HTTPParser,
+#    HTTPClientParser, Request, Response, HTTP11ClientProtocol,
+#)
+from .iweb import IResponse
 
 # States HTTPParser can be in
 STATUS = 'STATUS'
@@ -52,7 +65,7 @@ HEADER = 'HEADER'
 BODY = 'BODY'
 DONE = 'DONE'
 
-
+''' {{{
 class BadHeaders(Exception):
     """
     Headers passed to L{Request} were in some way invalid.
@@ -117,7 +130,7 @@ class RequestTransmissionFailed(_WrapperException):
     @ivar reasons: A C{list} of one or more L{Failure} instances giving the
         reasons the request transmission was considered to have failed.
     """
-
+}}} '''
 
 
 class ConnectionAborted(Exception):
@@ -126,7 +139,7 @@ class ConnectionAborted(Exception):
     """
 
 
-
+''' {{{
 class WrongBodyLength(Exception):
     """
     An L{IBodyProducer} declared the number of bytes it was going to
@@ -142,7 +155,7 @@ class ResponseDone(Exception):
     protocol passed to L{Response.deliverBody} and indicates that the entire
     response has been delivered.
     """
-
+}}} '''
 
 
 class ResponseFailed(_WrapperException):
@@ -169,7 +182,7 @@ class ResponseNeverReceived(ResponseFailed):
     """
 
 
-
+''' {{{
 class RequestNotSent(Exception):
     """
     L{RequestNotSent} indicates that an attempt was made to issue a request but
@@ -178,7 +191,7 @@ class RequestNotSent(Exception):
     to send a request using a protocol which is no longer connected to a
     server.
     """
-
+}}} '''
 
 
 def _callAppFunction(function):
@@ -764,7 +777,7 @@ class Request:
         _callAppFunction(self.bodyProducer.stopProducing)
 
 
-
+''' {{{
 class LengthEnforcingConsumer:
     """
     An L{IConsumer} proxy which enforces an exact length requirement on the
@@ -1188,7 +1201,7 @@ class TransportProxyProducer:
         """
         if self._producer is not None:
             self._producer.pauseProducing()
-
+}}} '''
 
 
 class HTTP11ClientProtocol(Protocol):
diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py
index c3830dc47..396115985 100644
--- a/scrapy/xlib/tx/client.py
+++ b/scrapy/xlib/tx/client.py
@@ -32,19 +32,29 @@ from twisted.internet.interfaces import IProtocol
 from twisted.python import failure
 from twisted.python.components import proxyForInterface
 from twisted.web import error
+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
 from twisted.web.http_headers import Headers
 
+from twisted.web.client import (
+    PartialDownloadError,
+)
+# newer than 10.0.0
+#from twisted.web.client import (
+#    CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, FileBodyProducer,
+#    HTTPConnectionPool, Agent, ProxyAgent,
+#)
+
 from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint
-from .iweb import IResponse, UNKNOWN_LENGTH, IBodyProducer
-
+from .iweb import IResponse
 
+''' {{{
 class PartialDownloadError(error.Error):
     """
     Page was only partially downloaded, we got disconnected in middle.
 
     @ivar response: All of the response body which was downloaded.
     """
-
+}}} '''
 
 class _URL(tuple):
     """
@@ -136,10 +146,13 @@ def _makeGetterFactory(url, factoryFactory, contextFactory=None,
 
 from twisted.web.error import SchemeNotSupported
 from ._newclient import Request, Response, HTTP11ClientProtocol
-from ._newclient import ResponseDone, ResponseFailed
-from ._newclient import RequestNotSent, RequestTransmissionFailed
+from twisted.web._newclient import ResponseDone
+from ._newclient import ResponseFailed
+from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed
+from twisted.web._newclient import (
+    PotentialDataLoss, _WrapperException)
 from ._newclient import (
-    ResponseNeverReceived, PotentialDataLoss, _WrapperException)
+    ResponseNeverReceived)
 
 try:
     from twisted.internet.ssl import ClientContextFactory
@@ -1161,8 +1174,7 @@ def readBody(response):
 
 
 __all__ = [
-    'PartialDownloadError', 'HTTPPageGetter', 'HTTPPageDownloader',
-    'HTTPClientFactory', 'HTTPDownloader', 'getPage', 'downloadPage',
+    'PartialDownloadError',
     'ResponseDone', 'Response', 'ResponseFailed', 'Agent', 'CookieAgent',
     'ProxyAgent', 'ContentDecoderAgent', 'GzipDecoder', 'RedirectAgent',
     'HTTPConnectionPool', 'readBody']
diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py
index d8a92ccd0..21a467433 100644
--- a/scrapy/xlib/tx/endpoints.py
+++ b/scrapy/xlib/tx/endpoints.py
@@ -15,23 +15,37 @@ parsed by the L{clientFromString} and L{serverFromString} functions.
 from __future__ import division, absolute_import
 
 import os
-import socket
+#import socket
 
 from zope.interface import implementer, directlyProvides
 import warnings
 
-from twisted.internet import interfaces, defer, error, fdesc, threads
+from twisted.internet import interfaces, defer, error, fdesc
 from twisted.internet.protocol import (
-        ClientFactory, Protocol, ProcessProtocol, Factory)
+        ClientFactory, Protocol, Factory)
+#from twisted.internet import threads, ProcessProtocol
 from twisted.internet.interfaces import IStreamServerEndpointStringParser
 from twisted.internet.interfaces import IStreamClientEndpointStringParser
 from twisted.python.filepath import FilePath
-from twisted.python.failure import Failure
-from twisted.python import log
+#from twisted.python.failure import Failure
+#from twisted.python import log
 from twisted.python.components import proxyForInterface
 
 from twisted.plugin import IPlugin, getPlugins
-from twisted.internet import stdio
+#from twisted.internet import stdio
+
+# newer than 10.0.0
+#from twisted.internet.endpoints import (
+#    TCP4ServerEndpoint, TCP6ServerEndpoint, TCP4ClientEndpoint, SSL4ServerEndpoint, SSL4ClientEndpoint,
+#    UNIXServerEndpoint, UNIXClientEndpoint, AdoptedStreamServerEndpoint, connectProtocol,
+#    quoteStringArgument,
+#    serverFromString, #> using newer _parseSSL, _tokenize in _serverParsers
+#    clientFromString, #> using newer _clientParsers
+#    _WrappingProtocol, _WrappingFactory, _TCPServerEndpoint,
+#    _parseTCP, _parseUNIX, _loadCAsFromDir,
+#    _parseSSL, _tokenize,
+#    _parseClientTCP, _parseClientSSL, _parseClientUNIX,
+#)
 
 from .interfaces import IFileDescriptorReceiver
 
diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py
index f3e4ed5d8..a715d4a05 100644
--- a/scrapy/xlib/tx/interfaces.py
+++ b/scrapy/xlib/tx/interfaces.py
@@ -11,7 +11,28 @@ from __future__ import division, absolute_import
 
 from zope.interface import Interface, Attribute
 
+from twisted.internet.interfaces import (
+    IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL,
+    IReactorUDP, IReactorMulticast, IReactorProcess,
+    IReactorTime, IDelayedCall, IReactorThreads, IReactorCore,
+    IReactorPluggableResolver, IReactorFDSet,
+    IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor,
+    IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor,
+    ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer,
+    IProtocol, IProcessProtocol, IHalfCloseableProtocol,
+    IProtocolFactory, ITransport, IProcessTransport, IServiceCollection,
+    IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport,
+    IMulticastTransport,
+)
+# newer than 10.0.0
+#from twisted.internet.interfaces import (
+#    IResolver, IReactorUNIX, IReactorUNIXDatagram, IReactorWin32Events, IReactorSocket,
+#    IReactorDaemonize, IFileDescriptorReceiver, ITCPTransport, IUNIXTransport,
+#    ITLSTransport, ISSLTransport, IStreamClientEndpoint, IStreamServerEndpoint,
+#    IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,
+#)
 
+''' {{{
 class IAddress(Interface):
     """
     An address, e.g. a TCP C{(host, port)}.
@@ -74,7 +95,7 @@ class IResolverSimple(Interface):
         @raise twisted.internet.defer.TimeoutError: Raised (asynchronously)
         if the name cannot be resolved within the specified timeout period.
         """
-
+}}} '''
 
 
 class IResolver(IResolverSimple):
@@ -614,7 +635,7 @@ class IResolver(IResolverSimple):
         """
 
 
-
+''' {{{
 class IReactorTCP(Interface):
 
     def listenTCP(port, factory, backlog=50, interface=''):
@@ -701,7 +722,7 @@ class IReactorSSL(Interface):
 
         @param interface: the hostname to bind to, defaults to '' (all)
         """
-
+}}} '''
 
 
 class IReactorUNIX(Interface):
@@ -829,7 +850,7 @@ class IReactorWin32Events(Interface):
         """
 
 
-
+''' {{{
 class IReactorUDP(Interface):
     """
     UDP socket methods.
@@ -868,7 +889,7 @@ class IReactorMulticast(Interface):
         @see: L{twisted.internet.interfaces.IMulticastTransport}
         @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html}
         """
-
+}}} '''
 
 
 class IReactorSocket(Interface):
@@ -970,7 +991,7 @@ class IReactorSocket(Interface):
         """
 
 
-
+''' {{{
 class IReactorProcess(Interface):
 
     def spawnProcess(processProtocol, executable, args=(), env={}, path=None,
@@ -1347,7 +1368,7 @@ class IReactorPluggableResolver(Interface):
 
         @return: The previously installed resolver.
         """
-
+}}} '''
 
 class IReactorDaemonize(Interface):
     """
@@ -1379,7 +1400,7 @@ class IReactorDaemonize(Interface):
         """
 
 
-
+''' {{{
 class IReactorFDSet(Interface):
     """
     Implement me to be able to use L{IFileDescriptor} type resources.
@@ -1863,7 +1884,7 @@ class IHalfCloseableProtocol(Interface):
         This will never be called for TCP connections as TCP does not
         support notification of this type of half-close.
         """
-
+}}} '''
 
 
 class IFileDescriptorReceiver(Interface):
@@ -1884,7 +1905,7 @@ class IFileDescriptorReceiver(Interface):
         """
 
 
-
+''' {{{
 class IProtocolFactory(Interface):
     """
     Interface for protocol factories.
@@ -1974,7 +1995,7 @@ class ITransport(Interface):
 
         @return: An L{IAddress} provider.
         """
-
+}}} '''
 
 class ITCPTransport(ITransport):
     """
@@ -2095,7 +2116,7 @@ class ISSLTransport(ITCPTransport):
         Return an object with the peer's certificate info.
         """
 
-
+''' {{{
 class IProcessTransport(ITransport):
     """
     A process transport.
@@ -2324,7 +2345,7 @@ class IMulticastTransport(Interface):
         """
         Leave multicast group, return L{Deferred} of success.
         """
-
+}}} '''
 
 class IStreamClientEndpoint(Interface):
     """
diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py
index ddcb6ed7a..57a111411 100644
--- a/scrapy/xlib/tx/iweb.py
+++ b/scrapy/xlib/tx/iweb.py
@@ -12,8 +12,16 @@ Interface definitions for L{twisted.web}.
 
 from zope.interface import Interface, Attribute
 
-from twisted.internet.interfaces import IPushProducer
+#from twisted.internet.interfaces import IPushProducer
 
+from twisted.web.iweb import (
+    ICredentialFactory, IBodyProducer,
+    UNKNOWN_LENGTH,
+)
+# newer than 10.0.0
+#from twisted.web.iweb import (
+#    IRequest, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory,
+#)
 
 class IRequest(Interface):
     """
@@ -320,7 +328,7 @@ class IRequest(Interface):
         """
 
 
-
+''' {{{
 class ICredentialFactory(Interface):
     """
     A credential factory defines a way to generate a particular kind of
@@ -424,7 +432,7 @@ class IBodyProducer(IPushProducer):
         L{Deferred<twisted.internet.defer.Deferred>} returned by
         C{startProducing} is never fired.
         """
-
+}}} '''
 
 
 class IRenderable(Interface):
@@ -576,9 +584,9 @@ class _IRequestEncoderFactory(Interface):
         """
 
 
-
+''' {{{
 UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH"
-
+}}} '''
 __all__ = [
     "ICredentialFactory", "IRequest",
     "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder",

From c8cf1a303d2272aa44422035fd1e3e6048cb5606 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Thu, 1 Dec 2016 22:02:11 +0000
Subject: [PATCH 1181/2974] Bump Twisted dependency to 13.1.0 (released June
 2013)

---
 requirements.txt             |  2 +-
 scrapy/xlib/tx/_newclient.py | 36 +++++++++++++---------------
 scrapy/xlib/tx/client.py     | 38 +++++++++++++----------------
 scrapy/xlib/tx/endpoints.py  | 38 +++++++++++++----------------
 scrapy/xlib/tx/interfaces.py | 46 +++++++++++++++++-------------------
 scrapy/xlib/tx/iweb.py       | 15 +++++-------
 setup.py                     |  2 +-
 7 files changed, 79 insertions(+), 98 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index cfa907050..64b6e771c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-Twisted>=10.0.0
+Twisted>=13.1.0
 lxml
 pyOpenSSL
 cssselect>=0.9
diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py
index e902d6683..d20eda34f 100644
--- a/scrapy/xlib/tx/_newclient.py
+++ b/scrapy/xlib/tx/_newclient.py
@@ -38,26 +38,21 @@ from twisted.internet.error import ConnectionDone
 from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred
 from twisted.internet.defer import CancelledError
 from twisted.internet.protocol import Protocol
-from twisted.protocols.basic import LineReceiver
-from twisted.web.iweb import UNKNOWN_LENGTH
-from twisted.web.http_headers import Headers
+#from twisted.protocols.basic import LineReceiver
+from twisted.web.iweb import UNKNOWN_LENGTH, IResponse
+#from twisted.web.http_headers import Headers
 from twisted.web.http import NO_CONTENT, NOT_MODIFIED
 from twisted.web.http import _DataLoss, PotentialDataLoss
 from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder
 
 from twisted.web._newclient import (
     BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException,
-    RequestGenerationFailed, RequestTransmissionFailed,
-    WrongBodyLength, ResponseDone, RequestNotSent,
-    LengthEnforcingConsumer, makeStatefulDispatcher, ChunkedEncoder,
-    TransportProxyProducer,
+    RequestGenerationFailed, RequestTransmissionFailed, ConnectionAborted,
+    WrongBodyLength, ResponseDone, ResponseFailed, RequestNotSent,
+    ResponseNeverReceived, HTTPParser, HTTPClientParser, Request,
+    LengthEnforcingConsumer, makeStatefulDispatcher, Response, ChunkedEncoder,
+    TransportProxyProducer, HTTP11ClientProtocol
 )
-# newer than 10.0.0
-#from twisted.web._newclient import (
-#    ConnectionAborted, ResponseFailed, ResponseNeverReceived, HTTPParser,
-#    HTTPClientParser, Request, Response, HTTP11ClientProtocol,
-#)
-from .iweb import IResponse
 
 # States HTTPParser can be in
 STATUS = 'STATUS'
@@ -130,7 +125,7 @@ class RequestTransmissionFailed(_WrapperException):
     @ivar reasons: A C{list} of one or more L{Failure} instances giving the
         reasons the request transmission was considered to have failed.
     """
-}}} '''
+
 
 
 class ConnectionAborted(Exception):
@@ -139,7 +134,7 @@ class ConnectionAborted(Exception):
     """
 
 
-''' {{{
+
 class WrongBodyLength(Exception):
     """
     An L{IBodyProducer} declared the number of bytes it was going to
@@ -155,7 +150,7 @@ class ResponseDone(Exception):
     protocol passed to L{Response.deliverBody} and indicates that the entire
     response has been delivered.
     """
-}}} '''
+
 
 
 class ResponseFailed(_WrapperException):
@@ -182,7 +177,7 @@ class ResponseNeverReceived(ResponseFailed):
     """
 
 
-''' {{{
+
 class RequestNotSent(Exception):
     """
     L{RequestNotSent} indicates that an attempt was made to issue a request but
@@ -191,7 +186,7 @@ class RequestNotSent(Exception):
     to send a request using a protocol which is no longer connected to a
     server.
     """
-}}} '''
+
 
 
 def _callAppFunction(function):
@@ -777,7 +772,7 @@ class Request:
         _callAppFunction(self.bodyProducer.stopProducing)
 
 
-''' {{{
+
 class LengthEnforcingConsumer:
     """
     An L{IConsumer} proxy which enforces an exact length requirement on the
@@ -1201,7 +1196,7 @@ class TransportProxyProducer:
         """
         if self._producer is not None:
             self._producer.pauseProducing()
-}}} '''
+
 
 
 class HTTP11ClientProtocol(Protocol):
@@ -1527,3 +1522,4 @@ class HTTP11ClientProtocol(Protocol):
         d = Deferred()
         self._abortDeferreds.append(d)
         return d
+}}} '''
diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py
index 396115985..8e0b1df8b 100644
--- a/scrapy/xlib/tx/client.py
+++ b/scrapy/xlib/tx/client.py
@@ -29,23 +29,18 @@ from twisted.python.failure import Failure
 from twisted.web import http
 from twisted.internet import defer, protocol, task, reactor
 from twisted.internet.interfaces import IProtocol
+from twisted.internet.endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint
 from twisted.python import failure
 from twisted.python.components import proxyForInterface
 from twisted.web import error
-from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse
 from twisted.web.http_headers import Headers
 
 from twisted.web.client import (
-    PartialDownloadError,
+    PartialDownloadError, FileBodyProducer,
+    CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent,
+    Agent, ProxyAgent, HTTPConnectionPool, readBody,
 )
-# newer than 10.0.0
-#from twisted.web.client import (
-#    CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, FileBodyProducer,
-#    HTTPConnectionPool, Agent, ProxyAgent,
-#)
-
-from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint
-from .iweb import IResponse
 
 ''' {{{
 class PartialDownloadError(error.Error):
@@ -54,7 +49,7 @@ class PartialDownloadError(error.Error):
 
     @ivar response: All of the response body which was downloaded.
     """
-}}} '''
+
 
 class _URL(tuple):
     """
@@ -138,22 +133,21 @@ def _makeGetterFactory(url, factoryFactory, contextFactory=None,
     else:
         reactor.connectTCP(host, port, factory)
     return factory
-
+}}} '''
 
 # The code which follows is based on the new HTTP client implementation.  It
 # should be significantly better than anything above, though it is not yet
 # feature equivalent.
 
-from twisted.web.error import SchemeNotSupported
-from ._newclient import Request, Response, HTTP11ClientProtocol
-from twisted.web._newclient import ResponseDone
-from ._newclient import ResponseFailed
-from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed
-from twisted.web._newclient import (
-    PotentialDataLoss, _WrapperException)
-from ._newclient import (
-    ResponseNeverReceived)
+#from twisted.web.error import SchemeNotSupported
+from twisted.web._newclient import Response
+#from twisted.web._newclient import Request, HTTP11ClientProtocol
+from twisted.web._newclient import ResponseDone, ResponseFailed
+#from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed
+#from twisted.web._newclient import (
+#    ResponseNeverReceived, PotentialDataLoss, _WrapperException)
 
+''' {{{
 try:
     from twisted.internet.ssl import ClientContextFactory
 except ImportError:
@@ -1170,7 +1164,7 @@ def readBody(response):
     d = defer.Deferred()
     response.deliverBody(_ReadBodyProtocol(response.code, response.phrase, d))
     return d
-
+}}} '''
 
 
 __all__ = [
diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py
index 21a467433..3f4704064 100644
--- a/scrapy/xlib/tx/endpoints.py
+++ b/scrapy/xlib/tx/endpoints.py
@@ -21,38 +21,33 @@ from zope.interface import implementer, directlyProvides
 import warnings
 
 from twisted.internet import interfaces, defer, error, fdesc
-from twisted.internet.protocol import (
-        ClientFactory, Protocol, Factory)
+#from twisted.internet.protocol import (
+#        ClientFactory, Protocol)
+from twisted.internet.protocol import Factory
 #from twisted.internet import threads, ProcessProtocol
 from twisted.internet.interfaces import IStreamServerEndpointStringParser
-from twisted.internet.interfaces import IStreamClientEndpointStringParser
+#from twisted.internet.interfaces import IStreamClientEndpointStringParser
 from twisted.python.filepath import FilePath
 #from twisted.python.failure import Failure
 #from twisted.python import log
-from twisted.python.components import proxyForInterface
+#from twisted.python.components import proxyForInterface
 
 from twisted.plugin import IPlugin, getPlugins
 #from twisted.internet import stdio
 
-# newer than 10.0.0
-#from twisted.internet.endpoints import (
-#    TCP4ServerEndpoint, TCP6ServerEndpoint, TCP4ClientEndpoint, SSL4ServerEndpoint, SSL4ClientEndpoint,
-#    UNIXServerEndpoint, UNIXClientEndpoint, AdoptedStreamServerEndpoint, connectProtocol,
-#    quoteStringArgument,
-#    serverFromString, #> using newer _parseSSL, _tokenize in _serverParsers
-#    clientFromString, #> using newer _clientParsers
-#    _WrappingProtocol, _WrappingFactory, _TCPServerEndpoint,
-#    _parseTCP, _parseUNIX, _loadCAsFromDir,
-#    _parseSSL, _tokenize,
-#    _parseClientTCP, _parseClientSSL, _parseClientUNIX,
-#)
-
-from .interfaces import IFileDescriptorReceiver
+from twisted.internet.endpoints import (
+    clientFromString, serverFromString, quoteStringArgument,
+    TCP4ServerEndpoint, TCP6ServerEndpoint,
+    TCP4ClientEndpoint, TCP6ClientEndpoint,
+    UNIXServerEndpoint, UNIXClientEndpoint,
+    SSL4ServerEndpoint, SSL4ClientEndpoint,
+    AdoptedStreamServerEndpoint, connectProtocol,
+)
 
 
 __all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"]
 
-
+''' {{{
 class _WrappingProtocol(Protocol):
     """
     Wrap another protocol in order to notify my user when a connection has
@@ -71,7 +66,7 @@ class _WrappingProtocol(Protocol):
         self._wrappedProtocol = wrappedProtocol
 
         for iface in [interfaces.IHalfCloseableProtocol,
-                      IFileDescriptorReceiver]:
+                      interfaces.IFileDescriptorReceiver]:
             if iface.providedBy(self._wrappedProtocol):
                 directlyProvides(self, iface)
 
@@ -609,6 +604,7 @@ class AdoptedStreamServerEndpoint(object):
 
 
 
+
 def _parseTCP(factory, port, interface="", backlog=50):
     """
     Internal parser function for L{_parseServer} to convert the string
@@ -1280,4 +1276,4 @@ def connectProtocol(endpoint, protocol):
         def buildProtocol(self, addr):
             return protocol
     return endpoint.connect(OneShotFactory())
-
+}}} '''
diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py
index a715d4a05..7b2a78632 100644
--- a/scrapy/xlib/tx/interfaces.py
+++ b/scrapy/xlib/tx/interfaces.py
@@ -13,24 +13,21 @@ from zope.interface import Interface, Attribute
 
 from twisted.internet.interfaces import (
     IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL,
-    IReactorUDP, IReactorMulticast, IReactorProcess,
+    IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess,
     IReactorTime, IDelayedCall, IReactorThreads, IReactorCore,
-    IReactorPluggableResolver, IReactorFDSet,
+    IReactorPluggableResolver, IReactorDaemonize, IReactorFDSet,
     IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor,
     IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor,
     ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer,
     IProtocol, IProcessProtocol, IHalfCloseableProtocol,
-    IProtocolFactory, ITransport, IProcessTransport, IServiceCollection,
+    IFileDescriptorReceiver, IProtocolFactory, ITransport, ITCPTransport,
+    IUNIXTransport,
+    ITLSTransport, ISSLTransport, IProcessTransport, IServiceCollection,
     IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport,
-    IMulticastTransport,
+    IMulticastTransport, IStreamClientEndpoint, IStreamServerEndpoint,
+    IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,
+    IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver
 )
-# newer than 10.0.0
-#from twisted.internet.interfaces import (
-#    IResolver, IReactorUNIX, IReactorUNIXDatagram, IReactorWin32Events, IReactorSocket,
-#    IReactorDaemonize, IFileDescriptorReceiver, ITCPTransport, IUNIXTransport,
-#    ITLSTransport, ISSLTransport, IStreamClientEndpoint, IStreamServerEndpoint,
-#    IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,
-#)
 
 ''' {{{
 class IAddress(Interface):
@@ -95,7 +92,7 @@ class IResolverSimple(Interface):
         @raise twisted.internet.defer.TimeoutError: Raised (asynchronously)
         if the name cannot be resolved within the specified timeout period.
         """
-}}} '''
+
 
 
 class IResolver(IResolverSimple):
@@ -635,7 +632,7 @@ class IResolver(IResolverSimple):
         """
 
 
-''' {{{
+
 class IReactorTCP(Interface):
 
     def listenTCP(port, factory, backlog=50, interface=''):
@@ -722,7 +719,7 @@ class IReactorSSL(Interface):
 
         @param interface: the hostname to bind to, defaults to '' (all)
         """
-}}} '''
+
 
 
 class IReactorUNIX(Interface):
@@ -850,7 +847,7 @@ class IReactorWin32Events(Interface):
         """
 
 
-''' {{{
+
 class IReactorUDP(Interface):
     """
     UDP socket methods.
@@ -889,7 +886,7 @@ class IReactorMulticast(Interface):
         @see: L{twisted.internet.interfaces.IMulticastTransport}
         @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html}
         """
-}}} '''
+
 
 
 class IReactorSocket(Interface):
@@ -991,7 +988,7 @@ class IReactorSocket(Interface):
         """
 
 
-''' {{{
+
 class IReactorProcess(Interface):
 
     def spawnProcess(processProtocol, executable, args=(), env={}, path=None,
@@ -1368,7 +1365,7 @@ class IReactorPluggableResolver(Interface):
 
         @return: The previously installed resolver.
         """
-}}} '''
+
 
 class IReactorDaemonize(Interface):
     """
@@ -1400,7 +1397,7 @@ class IReactorDaemonize(Interface):
         """
 
 
-''' {{{
+
 class IReactorFDSet(Interface):
     """
     Implement me to be able to use L{IFileDescriptor} type resources.
@@ -1884,7 +1881,7 @@ class IHalfCloseableProtocol(Interface):
         This will never be called for TCP connections as TCP does not
         support notification of this type of half-close.
         """
-}}} '''
+
 
 
 class IFileDescriptorReceiver(Interface):
@@ -1905,7 +1902,7 @@ class IFileDescriptorReceiver(Interface):
         """
 
 
-''' {{{
+
 class IProtocolFactory(Interface):
     """
     Interface for protocol factories.
@@ -1995,7 +1992,7 @@ class ITransport(Interface):
 
         @return: An L{IAddress} provider.
         """
-}}} '''
+
 
 class ITCPTransport(ITransport):
     """
@@ -2116,7 +2113,7 @@ class ISSLTransport(ITCPTransport):
         Return an object with the peer's certificate info.
         """
 
-''' {{{
+
 class IProcessTransport(ITransport):
     """
     A process transport.
@@ -2345,7 +2342,7 @@ class IMulticastTransport(Interface):
         """
         Leave multicast group, return L{Deferred} of success.
         """
-}}} '''
+
 
 class IStreamClientEndpoint(Interface):
     """
@@ -2461,3 +2458,4 @@ class IStreamClientEndpointStringParser(Interface):
         @return: a client endpoint
         @rtype: L{IStreamClientEndpoint}
         """
+}}} '''
diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py
index 57a111411..32c88ff2d 100644
--- a/scrapy/xlib/tx/iweb.py
+++ b/scrapy/xlib/tx/iweb.py
@@ -15,14 +15,11 @@ from zope.interface import Interface, Attribute
 #from twisted.internet.interfaces import IPushProducer
 
 from twisted.web.iweb import (
-    ICredentialFactory, IBodyProducer,
-    UNKNOWN_LENGTH,
+    IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader,
+    IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH,
 )
-# newer than 10.0.0
-#from twisted.web.iweb import (
-#    IRequest, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory,
-#)
 
+''' {{{
 class IRequest(Interface):
     """
     An HTTP request.
@@ -328,7 +325,7 @@ class IRequest(Interface):
         """
 
 
-''' {{{
+
 class ICredentialFactory(Interface):
     """
     A credential factory defines a way to generate a particular kind of
@@ -432,7 +429,7 @@ class IBodyProducer(IPushProducer):
         L{Deferred<twisted.internet.defer.Deferred>} returned by
         C{startProducing} is never fired.
         """
-}}} '''
+
 
 
 class IRenderable(Interface):
@@ -584,7 +581,7 @@ class _IRequestEncoderFactory(Interface):
         """
 
 
-''' {{{
+
 UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH"
 }}} '''
 __all__ = [
diff --git a/setup.py b/setup.py
index 92c114a7a..5e32d4240 100644
--- a/setup.py
+++ b/setup.py
@@ -41,7 +41,7 @@ setup(
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
     install_requires=[
-        'Twisted>=10.0.0',
+        'Twisted>=13.1.0',
         'w3lib>=1.15.0',
         'queuelib',
         'lxml',

From 985755d1fe0aabf922bcdb0e8bc22d67948820cb Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Thu, 1 Dec 2016 23:44:15 +0000
Subject: [PATCH 1182/2974] Remove obsolete xlib code for Twisted 13.1.0

---
 scrapy/xlib/tx/_newclient.py | 1466 --------------------
 scrapy/xlib/tx/client.py     | 1116 ----------------
 scrapy/xlib/tx/endpoints.py  | 1253 -----------------
 scrapy/xlib/tx/interfaces.py | 2433 ----------------------------------
 scrapy/xlib/tx/iweb.py       |  569 --------
 5 files changed, 6837 deletions(-)

diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py
index d20eda34f..39cd20f95 100644
--- a/scrapy/xlib/tx/_newclient.py
+++ b/scrapy/xlib/tx/_newclient.py
@@ -38,9 +38,7 @@ from twisted.internet.error import ConnectionDone
 from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred
 from twisted.internet.defer import CancelledError
 from twisted.internet.protocol import Protocol
-#from twisted.protocols.basic import LineReceiver
 from twisted.web.iweb import UNKNOWN_LENGTH, IResponse
-#from twisted.web.http_headers import Headers
 from twisted.web.http import NO_CONTENT, NOT_MODIFIED
 from twisted.web.http import _DataLoss, PotentialDataLoss
 from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder
@@ -59,1467 +57,3 @@ STATUS = 'STATUS'
 HEADER = 'HEADER'
 BODY = 'BODY'
 DONE = 'DONE'
-
-''' {{{
-class BadHeaders(Exception):
-    """
-    Headers passed to L{Request} were in some way invalid.
-    """
-
-
-
-class ExcessWrite(Exception):
-    """
-    The body L{IBodyProducer} for a request tried to write data after
-    indicating it had finished writing data.
-    """
-
-
-class ParseError(Exception):
-    """
-    Some received data could not be parsed.
-
-    @ivar data: The string which could not be parsed.
-    """
-    def __init__(self, reason, data):
-        Exception.__init__(self, reason, data)
-        self.data = data
-
-
-
-class BadResponseVersion(ParseError):
-    """
-    The version string in a status line was unparsable.
-    """
-
-
-
-class _WrapperException(Exception):
-    """
-    L{_WrapperException} is the base exception type for exceptions which
-    include one or more other exceptions as the low-level causes.
-
-    @ivar reasons: A list of exceptions.  See subclass documentation for more
-        details.
-    """
-    def __init__(self, reasons):
-        Exception.__init__(self, reasons)
-        self.reasons = reasons
-
-
-
-class RequestGenerationFailed(_WrapperException):
-    """
-    There was an error while creating the bytes which make up a request.
-
-    @ivar reasons: A C{list} of one or more L{Failure} instances giving the
-        reasons the request generation was considered to have failed.
-    """
-
-
-
-class RequestTransmissionFailed(_WrapperException):
-    """
-    There was an error while sending the bytes which make up a request.
-
-    @ivar reasons: A C{list} of one or more L{Failure} instances giving the
-        reasons the request transmission was considered to have failed.
-    """
-
-
-
-class ConnectionAborted(Exception):
-    """
-    The connection was explicitly aborted by application code.
-    """
-
-
-
-class WrongBodyLength(Exception):
-    """
-    An L{IBodyProducer} declared the number of bytes it was going to
-    produce (via its C{length} attribute) and then produced a different number
-    of bytes.
-    """
-
-
-
-class ResponseDone(Exception):
-    """
-    L{ResponseDone} may be passed to L{IProtocol.connectionLost} on the
-    protocol passed to L{Response.deliverBody} and indicates that the entire
-    response has been delivered.
-    """
-
-
-
-class ResponseFailed(_WrapperException):
-    """
-    L{ResponseFailed} indicates that all of the response to a request was not
-    received for some reason.
-
-    @ivar reasons: A C{list} of one or more L{Failure} instances giving the
-        reasons the response was considered to have failed.
-
-    @ivar response: If specified, the L{Response} received from the server (and
-        in particular the status code and the headers).
-    """
-
-    def __init__(self, reasons, response=None):
-        _WrapperException.__init__(self, reasons)
-        self.response = response
-
-
-
-class ResponseNeverReceived(ResponseFailed):
-    """
-    A L{ResponseFailed} that knows no response bytes at all have been received.
-    """
-
-
-
-class RequestNotSent(Exception):
-    """
-    L{RequestNotSent} indicates that an attempt was made to issue a request but
-    for reasons unrelated to the details of the request itself, the request
-    could not be sent.  For example, this may indicate that an attempt was made
-    to send a request using a protocol which is no longer connected to a
-    server.
-    """
-
-
-
-def _callAppFunction(function):
-    """
-    Call C{function}.  If it raises an exception, log it with a minimal
-    description of the source.
-
-    @return: C{None}
-    """
-    try:
-        function()
-    except:
-        log.err(None, "Unexpected exception from %s" % (
-                fullyQualifiedName(function),))
-
-
-
-class HTTPParser(LineReceiver):
-    """
-    L{HTTPParser} handles the parsing side of HTTP processing. With a suitable
-    subclass, it can parse either the client side or the server side of the
-    connection.
-
-    @ivar headers: All of the non-connection control message headers yet
-        received.
-
-    @ivar state: State indicator for the response parsing state machine.  One
-        of C{STATUS}, C{HEADER}, C{BODY}, C{DONE}.
-
-    @ivar _partialHeader: C{None} or a C{list} of the lines of a multiline
-        header while that header is being received.
-    """
-
-    # NOTE: According to HTTP spec, we're supposed to eat the
-    # 'Proxy-Authenticate' and 'Proxy-Authorization' headers also, but that
-    # doesn't sound like a good idea to me, because it makes it impossible to
-    # have a non-authenticating transparent proxy in front of an authenticating
-    # proxy. An authenticating proxy can eat them itself. -jknight
-    #
-    # Further, quoting
-    # http://homepages.tesco.net/J.deBoynePollard/FGA/web-proxy-connection-header.html
-    # regarding the 'Proxy-Connection' header:
-    #
-    #    The Proxy-Connection: header is a mistake in how some web browsers
-    #    use HTTP. Its name is the result of a false analogy. It is not a
-    #    standard part of the protocol. There is a different standard
-    #    protocol mechanism for doing what it does. And its existence
-    #    imposes a requirement upon HTTP servers such that no proxy HTTP
-    #    server can be standards-conforming in practice.
-    #
-    # -exarkun
-
-    # Some servers (like http://news.ycombinator.com/) return status lines and
-    # HTTP headers delimited by \n instead of \r\n.
-    delimiter = '\n'
-
-    CONNECTION_CONTROL_HEADERS = set([
-            'content-length', 'connection', 'keep-alive', 'te', 'trailers',
-            'transfer-encoding', 'upgrade', 'proxy-connection'])
-
-    def connectionMade(self):
-        self.headers = Headers()
-        self.connHeaders = Headers()
-        self.state = STATUS
-        self._partialHeader = None
-
-
-    def switchToBodyMode(self, decoder):
-        """
-        Switch to body parsing mode - interpret any more bytes delivered as
-        part of the message body and deliver them to the given decoder.
-        """
-        if self.state == BODY:
-            raise RuntimeError("already in body mode")
-
-        self.bodyDecoder = decoder
-        self.state = BODY
-        self.setRawMode()
-
-
-    def lineReceived(self, line):
-        """
-        Handle one line from a response.
-        """
-        # Handle the normal CR LF case.
-        if line[-1:] == '\r':
-            line = line[:-1]
-
-        if self.state == STATUS:
-            self.statusReceived(line)
-            self.state = HEADER
-        elif self.state == HEADER:
-            if not line or line[0] not in ' \t':
-                if self._partialHeader is not None:
-                    header = ''.join(self._partialHeader)
-                    name, value = header.split(':', 1)
-                    value = value.strip()
-                    self.headerReceived(name, value)
-                if not line:
-                    # Empty line means the header section is over.
-                    self.allHeadersReceived()
-                else:
-                    # Line not beginning with LWS is another header.
-                    self._partialHeader = [line]
-            else:
-                # A line beginning with LWS is a continuation of a header
-                # begun on a previous line.
-                self._partialHeader.append(line)
-
-
-    def rawDataReceived(self, data):
-        """
-        Pass data from the message body to the body decoder object.
-        """
-        self.bodyDecoder.dataReceived(data)
-
-
-    def isConnectionControlHeader(self, name):
-        """
-        Return C{True} if the given lower-cased name is the name of a
-        connection control header (rather than an entity header).
-
-        According to RFC 2616, section 14.10, the tokens in the Connection
-        header are probably relevant here.  However, I am not sure what the
-        practical consequences of either implementing or ignoring that are.
-        So I leave it unimplemented for the time being.
-        """
-        return name in self.CONNECTION_CONTROL_HEADERS
-
-
-    def statusReceived(self, status):
-        """
-        Callback invoked whenever the first line of a new message is received.
-        Override this.
-
-        @param status: The first line of an HTTP request or response message
-            without trailing I{CR LF}.
-        @type status: C{str}
-        """
-
-
-    def headerReceived(self, name, value):
-        """
-        Store the given header in C{self.headers}.
-        """
-        name = name.lower()
-        if self.isConnectionControlHeader(name):
-            headers = self.connHeaders
-        else:
-            headers = self.headers
-        headers.addRawHeader(name, value)
-
-
-    def allHeadersReceived(self):
-        """
-        Callback invoked after the last header is passed to C{headerReceived}.
-        Override this to change to the C{BODY} or C{DONE} state.
-        """
-        self.switchToBodyMode(None)
-
-
-
-class HTTPClientParser(HTTPParser):
-    """
-    An HTTP parser which only handles HTTP responses.
-
-    @ivar request: The request with which the expected response is associated.
-    @type request: L{Request}
-
-    @ivar NO_BODY_CODES: A C{set} of response codes which B{MUST NOT} have a
-        body.
-
-    @ivar finisher: A callable to invoke when this response is fully parsed.
-
-    @ivar _responseDeferred: A L{Deferred} which will be called back with the
-        response when all headers in the response have been received.
-        Thereafter, C{None}.
-
-    @ivar _everReceivedData: C{True} if any bytes have been received.
-    """
-    NO_BODY_CODES = set([NO_CONTENT, NOT_MODIFIED])
-
-    _transferDecoders = {
-        'chunked': _ChunkedTransferDecoder,
-        }
-
-    bodyDecoder = None
-
-    def __init__(self, request, finisher):
-        self.request = request
-        self.finisher = finisher
-        self._responseDeferred = Deferred()
-        self._everReceivedData = False
-
-
-    def dataReceived(self, data):
-        """
-        Override so that we know if any response has been received.
-        """
-        self._everReceivedData = True
-        HTTPParser.dataReceived(self, data)
-
-
-    def parseVersion(self, strversion):
-        """
-        Parse version strings of the form Protocol '/' Major '.' Minor. E.g.
-        'HTTP/1.1'.  Returns (protocol, major, minor).  Will raise ValueError
-        on bad syntax.
-        """
-        try:
-            proto, strnumber = strversion.split('/')
-            major, minor = strnumber.split('.')
-            major, minor = int(major), int(minor)
-        except ValueError as e:
-            raise BadResponseVersion(str(e), strversion)
-        if major < 0 or minor < 0:
-            raise BadResponseVersion("version may not be negative", strversion)
-        return (proto, major, minor)
-
-
-    def statusReceived(self, status):
-        """
-        Parse the status line into its components and create a response object
-        to keep track of this response's state.
-        """
-        parts = status.split(' ', 2)
-        if len(parts) != 3:
-            raise ParseError("wrong number of parts", status)
-
-        try:
-            statusCode = int(parts[1])
-        except ValueError:
-            raise ParseError("non-integer status code", status)
-
-        self.response = Response(
-            self.parseVersion(parts[0]),
-            statusCode,
-            parts[2],
-            self.headers,
-            self.transport)
-
-
-    def _finished(self, rest):
-        """
-        Called to indicate that an entire response has been received.  No more
-        bytes will be interpreted by this L{HTTPClientParser}.  Extra bytes are
-        passed up and the state of this L{HTTPClientParser} is set to I{DONE}.
-
-        @param rest: A C{str} giving any extra bytes delivered to this
-            L{HTTPClientParser} which are not part of the response being
-            parsed.
-        """
-        self.state = DONE
-        self.finisher(rest)
-
-
-    def isConnectionControlHeader(self, name):
-        """
-        Content-Length in the response to a HEAD request is an entity header,
-        not a connection control header.
-        """
-        if self.request.method == 'HEAD' and name == 'content-length':
-            return False
-        return HTTPParser.isConnectionControlHeader(self, name)
-
-
-    def allHeadersReceived(self):
-        """
-        Figure out how long the response body is going to be by examining
-        headers and stuff.
-        """
-        if (self.response.code in self.NO_BODY_CODES
-            or self.request.method == 'HEAD'):
-            self.response.length = 0
-            self._finished(self.clearLineBuffer())
-        else:
-            transferEncodingHeaders = self.connHeaders.getRawHeaders(
-                'transfer-encoding')
-            if transferEncodingHeaders:
-
-                # This could be a KeyError.  However, that would mean we do not
-                # know how to decode the response body, so failing the request
-                # is as good a behavior as any.  Perhaps someday we will want
-                # to normalize/document/test this specifically, but failing
-                # seems fine to me for now.
-                transferDecoder = self._transferDecoders[transferEncodingHeaders[0].lower()]
-
-                # If anyone ever invents a transfer encoding other than
-                # chunked (yea right), and that transfer encoding can predict
-                # the length of the response body, it might be sensible to
-                # allow the transfer decoder to set the response object's
-                # length attribute.
-            else:
-                contentLengthHeaders = self.connHeaders.getRawHeaders('content-length')
-                if contentLengthHeaders is None:
-                    contentLength = None
-                elif len(contentLengthHeaders) == 1:
-                    contentLength = int(contentLengthHeaders[0])
-                    self.response.length = contentLength
-                else:
-                    # "HTTP Message Splitting" or "HTTP Response Smuggling"
-                    # potentially happening.  Or it's just a buggy server.
-                    raise ValueError(
-                        "Too many Content-Length headers; response is invalid")
-
-                if contentLength == 0:
-                    self._finished(self.clearLineBuffer())
-                    transferDecoder = None
-                else:
-                    transferDecoder = lambda x, y: _IdentityTransferDecoder(
-                        contentLength, x, y)
-
-            if transferDecoder is None:
-                self.response._bodyDataFinished()
-            else:
-                # Make sure as little data as possible from the response body
-                # gets delivered to the response object until the response
-                # object actually indicates it is ready to handle bytes
-                # (probably because an application gave it a way to interpret
-                # them).
-                self.transport.pauseProducing()
-                self.switchToBodyMode(transferDecoder(
-                        self.response._bodyDataReceived,
-                        self._finished))
-
-        # This must be last.  If it were first, then application code might
-        # change some state (for example, registering a protocol to receive the
-        # response body).  Then the pauseProducing above would be wrong since
-        # the response is ready for bytes and nothing else would ever resume
-        # the transport.
-        self._responseDeferred.callback(self.response)
-        del self._responseDeferred
-
-
-    def connectionLost(self, reason):
-        if self.bodyDecoder is not None:
-            try:
-                try:
-                    self.bodyDecoder.noMoreData()
-                except PotentialDataLoss:
-                    self.response._bodyDataFinished(Failure())
-                except _DataLoss:
-                    self.response._bodyDataFinished(
-                        Failure(ResponseFailed([reason, Failure()],
-                                               self.response)))
-                else:
-                    self.response._bodyDataFinished()
-            except:
-                # Handle exceptions from both the except suites and the else
-                # suite.  Those functions really shouldn't raise exceptions,
-                # but maybe there's some buggy application code somewhere
-                # making things difficult.
-                log.err()
-        elif self.state != DONE:
-            if self._everReceivedData:
-                exceptionClass = ResponseFailed
-            else:
-                exceptionClass = ResponseNeverReceived
-            self._responseDeferred.errback(Failure(exceptionClass([reason])))
-            del self._responseDeferred
-
-
-
-class Request:
-    """
-    A L{Request} instance describes an HTTP request to be sent to an HTTP
-    server.
-
-    @ivar method: The HTTP method to for this request, ex: 'GET', 'HEAD',
-        'POST', etc.
-    @type method: C{str}
-
-    @ivar uri: The relative URI of the resource to request.  For example,
-        C{'/foo/bar?baz=quux'}.
-    @type uri: C{str}
-
-    @ivar headers: Headers to be sent to the server.  It is important to
-        note that this object does not create any implicit headers.  So it
-        is up to the HTTP Client to add required headers such as 'Host'.
-    @type headers: L{twisted.web.http_headers.Headers}
-
-    @ivar bodyProducer: C{None} or an L{IBodyProducer} provider which
-        produces the content body to send to the remote HTTP server.
-
-    @ivar persistent: Set to C{True} when you use HTTP persistent connection.
-    @type persistent: C{bool}
-    """
-    def __init__(self, method, uri, headers, bodyProducer, persistent=False):
-        self.method = method
-        self.uri = uri
-        self.headers = headers
-        self.bodyProducer = bodyProducer
-        self.persistent = persistent
-
-
-    def _writeHeaders(self, transport, TEorCL):
-        hosts = self.headers.getRawHeaders('host', ())
-        if len(hosts) != 1:
-            raise BadHeaders("Exactly one Host header required")
-
-        # In the future, having the protocol version be a parameter to this
-        # method would probably be good.  It would be nice if this method
-        # weren't limited to issuing HTTP/1.1 requests.
-        requestLines = []
-        requestLines.append(
-            '%s %s HTTP/1.1\r\n' % (self.method, self.uri))
-        if not self.persistent:
-            requestLines.append('Connection: close\r\n')
-        if TEorCL is not None:
-            requestLines.append(TEorCL)
-        for name, values in self.headers.getAllRawHeaders():
-            requestLines.extend(['%s: %s\r\n' % (name, v) for v in values])
-        requestLines.append('\r\n')
-        transport.writeSequence(requestLines)
-
-
-    def _writeToChunked(self, transport):
-        """
-        Write this request to the given transport using chunked
-        transfer-encoding to frame the body.
-        """
-        self._writeHeaders(transport, 'Transfer-Encoding: chunked\r\n')
-        encoder = ChunkedEncoder(transport)
-        encoder.registerProducer(self.bodyProducer, True)
-        d = self.bodyProducer.startProducing(encoder)
-
-        def cbProduced(ignored):
-            encoder.unregisterProducer()
-        def ebProduced(err):
-            encoder._allowNoMoreWrites()
-            # Don't call the encoder's unregisterProducer because it will write
-            # a zero-length chunk.  This would indicate to the server that the
-            # request body is complete.  There was an error, though, so we
-            # don't want to do that.
-            transport.unregisterProducer()
-            return err
-        d.addCallbacks(cbProduced, ebProduced)
-        return d
-
-
-    def _writeToContentLength(self, transport):
-        """
-        Write this request to the given transport using content-length to frame
-        the body.
-        """
-        self._writeHeaders(
-            transport,
-            'Content-Length: %d\r\n' % (self.bodyProducer.length,))
-
-        # This Deferred is used to signal an error in the data written to the
-        # encoder below.  It can only errback and it will only do so before too
-        # many bytes have been written to the encoder and before the producer
-        # Deferred fires.
-        finishedConsuming = Deferred()
-
-        # This makes sure the producer writes the correct number of bytes for
-        # the request body.
-        encoder = LengthEnforcingConsumer(
-            self.bodyProducer, transport, finishedConsuming)
-
-        transport.registerProducer(self.bodyProducer, True)
-
-        finishedProducing = self.bodyProducer.startProducing(encoder)
-
-        def combine(consuming, producing):
-            # This Deferred is returned and will be fired when the first of
-            # consuming or producing fires. If it's cancelled, forward that
-            # cancellation to the producer.
-            def cancelConsuming(ign):
-                finishedProducing.cancel()
-            ultimate = Deferred(cancelConsuming)
-
-            # Keep track of what has happened so far.  This initially
-            # contains None, then an integer uniquely identifying what
-            # sequence of events happened.  See the callbacks and errbacks
-            # defined below for the meaning of each value.
-            state = [None]
-
-            def ebConsuming(err):
-                if state == [None]:
-                    # The consuming Deferred failed first.  This means the
-                    # overall writeTo Deferred is going to errback now.  The
-                    # producing Deferred should not fire later (because the
-                    # consumer should have called stopProducing on the
-                    # producer), but if it does, a callback will be ignored
-                    # and an errback will be logged.
-                    state[0] = 1
-                    ultimate.errback(err)
-                else:
-                    # The consuming Deferred errbacked after the producing
-                    # Deferred fired.  This really shouldn't ever happen.
-                    # If it does, I goofed.  Log the error anyway, just so
-                    # there's a chance someone might notice and complain.
-                    log.err(
-                        err,
-                        "Buggy state machine in %r/[%d]: "
-                        "ebConsuming called" % (self, state[0]))
-
-            def cbProducing(result):
-                if state == [None]:
-                    # The producing Deferred succeeded first.  Nothing will
-                    # ever happen to the consuming Deferred.  Tell the
-                    # encoder we're done so it can check what the producer
-                    # wrote and make sure it was right.
-                    state[0] = 2
-                    try:
-                        encoder._noMoreWritesExpected()
-                    except:
-                        # Fail the overall writeTo Deferred - something the
-                        # producer did was wrong.
-                        ultimate.errback()
-                    else:
-                        # Success - succeed the overall writeTo Deferred.
-                        ultimate.callback(None)
-                # Otherwise, the consuming Deferred already errbacked.  The
-                # producing Deferred wasn't supposed to fire, but it did
-                # anyway.  It's buggy, but there's not really anything to be
-                # done about it.  Just ignore this result.
-
-            def ebProducing(err):
-                if state == [None]:
-                    # The producing Deferred failed first.  This means the
-                    # overall writeTo Deferred is going to errback now.
-                    # Tell the encoder that we're done so it knows to reject
-                    # further writes from the producer (which should not
-                    # happen, but the producer may be buggy).
-                    state[0] = 3
-                    encoder._allowNoMoreWrites()
-                    ultimate.errback(err)
-                else:
-                    # The producing Deferred failed after the consuming
-                    # Deferred failed.  It shouldn't have, so it's buggy.
-                    # Log the exception in case anyone who can fix the code
-                    # is watching.
-                    log.err(err, "Producer is buggy")
-
-            consuming.addErrback(ebConsuming)
-            producing.addCallbacks(cbProducing, ebProducing)
-
-            return ultimate
-
-        d = combine(finishedConsuming, finishedProducing)
-        def f(passthrough):
-            # Regardless of what happens with the overall Deferred, once it
-            # fires, the producer registered way up above the definition of
-            # combine should be unregistered.
-            transport.unregisterProducer()
-            return passthrough
-        d.addBoth(f)
-        return d
-
-
-    def writeTo(self, transport):
-        """
-        Format this L{Request} as an HTTP/1.1 request and write it to the given
-        transport.  If bodyProducer is not None, it will be associated with an
-        L{IConsumer}.
-
-        @return: A L{Deferred} which fires with C{None} when the request has
-            been completely written to the transport or with a L{Failure} if
-            there is any problem generating the request bytes.
-        """
-        if self.bodyProducer is not None:
-            if self.bodyProducer.length is UNKNOWN_LENGTH:
-                return self._writeToChunked(transport)
-            else:
-                return self._writeToContentLength(transport)
-        else:
-            self._writeHeaders(transport, None)
-            return succeed(None)
-
-
-    def stopWriting(self):
-        """
-        Stop writing this request to the transport.  This can only be called
-        after C{writeTo} and before the L{Deferred} returned by C{writeTo}
-        fires.  It should cancel any asynchronous task started by C{writeTo}.
-        The L{Deferred} returned by C{writeTo} need not be fired if this method
-        is called.
-        """
-        # If bodyProducer is None, then the Deferred returned by writeTo has
-        # fired already and this method cannot be called.
-        _callAppFunction(self.bodyProducer.stopProducing)
-
-
-
-class LengthEnforcingConsumer:
-    """
-    An L{IConsumer} proxy which enforces an exact length requirement on the
-    total data written to it.
-
-    @ivar _length: The number of bytes remaining to be written.
-
-    @ivar _producer: The L{IBodyProducer} which is writing to this
-        consumer.
-
-    @ivar _consumer: The consumer to which at most C{_length} bytes will be
-        forwarded.
-
-    @ivar _finished: A L{Deferred} which will be fired with a L{Failure} if too
-        many bytes are written to this consumer.
-    """
-    def __init__(self, producer, consumer, finished):
-        self._length = producer.length
-        self._producer = producer
-        self._consumer = consumer
-        self._finished = finished
-
-
-    def _allowNoMoreWrites(self):
-        """
-        Indicate that no additional writes are allowed.  Attempts to write
-        after calling this method will be met with an exception.
-        """
-        self._finished = None
-
-
-    def write(self, bytes):
-        """
-        Write C{bytes} to the underlying consumer unless
-        C{_noMoreWritesExpected} has been called or there are/have been too
-        many bytes.
-        """
-        if self._finished is None:
-            # No writes are supposed to happen any more.  Try to convince the
-            # calling code to stop calling this method by calling its
-            # stopProducing method and then throwing an exception at it.  This
-            # exception isn't documented as part of the API because you're
-            # never supposed to expect it: only buggy code will ever receive
-            # it.
-            self._producer.stopProducing()
-            raise ExcessWrite()
-
-        if len(bytes) <= self._length:
-            self._length -= len(bytes)
-            self._consumer.write(bytes)
-        else:
-            # No synchronous exception is raised in *this* error path because
-            # we still have _finished which we can use to report the error to a
-            # better place than the direct caller of this method (some
-            # arbitrary application code).
-            _callAppFunction(self._producer.stopProducing)
-            self._finished.errback(WrongBodyLength("too many bytes written"))
-            self._allowNoMoreWrites()
-
-
-    def _noMoreWritesExpected(self):
-        """
-        Called to indicate no more bytes will be written to this consumer.
-        Check to see that the correct number have been written.
-
-        @raise WrongBodyLength: If not enough bytes have been written.
-        """
-        if self._finished is not None:
-            self._allowNoMoreWrites()
-            if self._length:
-                raise WrongBodyLength("too few bytes written")
-
-
-
-def makeStatefulDispatcher(name, template):
-    """
-    Given a I{dispatch} name and a function, return a function which can be
-    used as a method and which, when called, will call another method defined
-    on the instance and return the result.  The other method which is called is
-    determined by the value of the C{_state} attribute of the instance.
-
-    @param name: A string which is used to construct the name of the subsidiary
-        method to invoke.  The subsidiary method is named like C{'_%s_%s' %
-        (name, _state)}.
-
-    @param template: A function object which is used to give the returned
-        function a docstring.
-
-    @return: The dispatcher function.
-    """
-    def dispatcher(self, *args, **kwargs):
-        func = getattr(self, '_' + name + '_' + self._state, None)
-        if func is None:
-            raise RuntimeError(
-                "%r has no %s method in state %s" % (self, name, self._state))
-        return func(*args, **kwargs)
-    dispatcher.__doc__ = template.__doc__
-    return dispatcher
-
-
-
-class Response:
-    """
-    A L{Response} instance describes an HTTP response received from an HTTP
-    server.
-
-    L{Response} should not be subclassed or instantiated.
-
-    @ivar _transport: The transport which is delivering this response.
-
-    @ivar _bodyProtocol: The L{IProtocol} provider to which the body is
-        delivered.  C{None} before one has been registered with
-        C{deliverBody}.
-
-    @ivar _bodyBuffer: A C{list} of the strings passed to C{bodyDataReceived}
-        before C{deliverBody} is called.  C{None} afterwards.
-
-    @ivar _state: Indicates what state this L{Response} instance is in,
-        particularly with respect to delivering bytes from the response body
-        to an application-suppled protocol object.  This may be one of
-        C{'INITIAL'}, C{'CONNECTED'}, C{'DEFERRED_CLOSE'}, or C{'FINISHED'},
-        with the following meanings:
-
-          - INITIAL: This is the state L{Response} objects start in.  No
-            protocol has yet been provided and the underlying transport may
-            still have bytes to deliver to it.
-
-          - DEFERRED_CLOSE: If the underlying transport indicates all bytes
-            have been delivered but no application-provided protocol is yet
-            available, the L{Response} moves to this state.  Data is
-            buffered and waiting for a protocol to be delivered to.
-
-          - CONNECTED: If a protocol is provided when the state is INITIAL,
-            the L{Response} moves to this state.  Any buffered data is
-            delivered and any data which arrives from the transport
-            subsequently is given directly to the protocol.
-
-          - FINISHED: If a protocol is provided in the DEFERRED_CLOSE state,
-            the L{Response} moves to this state after delivering all
-            buffered data to the protocol.  Otherwise, if the L{Response} is
-            in the CONNECTED state, if the transport indicates there is no
-            more data, the L{Response} moves to this state.  Nothing else
-            can happen once the L{Response} is in this state.
-    """
-    implements(IResponse)
-
-    length = UNKNOWN_LENGTH
-
-    _bodyProtocol = None
-    _bodyFinished = False
-
-    def __init__(self, version, code, phrase, headers, _transport):
-        self.version = version
-        self.code = code
-        self.phrase = phrase
-        self.headers = headers
-        self._transport = _transport
-        self._bodyBuffer = []
-        self._state = 'INITIAL'
-
-
-    def deliverBody(self, protocol):
-        """
-        Dispatch the given L{IProtocol} depending of the current state of the
-        response.
-        """
-    deliverBody = makeStatefulDispatcher('deliverBody', deliverBody)
-
-
-    def _deliverBody_INITIAL(self, protocol):
-        """
-        Deliver any buffered data to C{protocol} and prepare to deliver any
-        future data to it.  Move to the C{'CONNECTED'} state.
-        """
-        # Now that there's a protocol to consume the body, resume the
-        # transport.  It was previously paused by HTTPClientParser to avoid
-        # reading too much data before it could be handled.
-        self._transport.resumeProducing()
-
-        protocol.makeConnection(self._transport)
-        self._bodyProtocol = protocol
-        for data in self._bodyBuffer:
-            self._bodyProtocol.dataReceived(data)
-        self._bodyBuffer = None
-        self._state = 'CONNECTED'
-
-
-    def _deliverBody_CONNECTED(self, protocol):
-        """
-        It is invalid to attempt to deliver data to a protocol when it is
-        already being delivered to another protocol.
-        """
-        raise RuntimeError(
-            "Response already has protocol %r, cannot deliverBody "
-            "again" % (self._bodyProtocol,))
-
-
-    def _deliverBody_DEFERRED_CLOSE(self, protocol):
-        """
-        Deliver any buffered data to C{protocol} and then disconnect the
-        protocol.  Move to the C{'FINISHED'} state.
-        """
-        # Unlike _deliverBody_INITIAL, there is no need to resume the
-        # transport here because all of the response data has been received
-        # already.  Some higher level code may want to resume the transport if
-        # that code expects further data to be received over it.
-
-        protocol.makeConnection(self._transport)
-
-        for data in self._bodyBuffer:
-            protocol.dataReceived(data)
-        self._bodyBuffer = None
-        protocol.connectionLost(self._reason)
-        self._state = 'FINISHED'
-
-
-    def _deliverBody_FINISHED(self, protocol):
-        """
-        It is invalid to attempt to deliver data to a protocol after the
-        response body has been delivered to another protocol.
-        """
-        raise RuntimeError(
-            "Response already finished, cannot deliverBody now.")
-
-
-    def _bodyDataReceived(self, data):
-        """
-        Called by HTTPClientParser with chunks of data from the response body.
-        They will be buffered or delivered to the protocol passed to
-        deliverBody.
-        """
-    _bodyDataReceived = makeStatefulDispatcher('bodyDataReceived',
-                                               _bodyDataReceived)
-
-
-    def _bodyDataReceived_INITIAL(self, data):
-        """
-        Buffer any data received for later delivery to a protocol passed to
-        C{deliverBody}.
-
-        Little or no data should be buffered by this method, since the
-        transport has been paused and will not be resumed until a protocol
-        is supplied.
-        """
-        self._bodyBuffer.append(data)
-
-
-    def _bodyDataReceived_CONNECTED(self, data):
-        """
-        Deliver any data received to the protocol to which this L{Response}
-        is connected.
-        """
-        self._bodyProtocol.dataReceived(data)
-
-
-    def _bodyDataReceived_DEFERRED_CLOSE(self, data):
-        """
-        It is invalid for data to be delivered after it has been indicated
-        that the response body has been completely delivered.
-        """
-        raise RuntimeError("Cannot receive body data after _bodyDataFinished")
-
-
-    def _bodyDataReceived_FINISHED(self, data):
-        """
-        It is invalid for data to be delivered after the response body has
-        been delivered to a protocol.
-        """
-        raise RuntimeError("Cannot receive body data after protocol disconnected")
-
-
-    def _bodyDataFinished(self, reason=None):
-        """
-        Called by HTTPClientParser when no more body data is available.  If the
-        optional reason is supplied, this indicates a problem or potential
-        problem receiving all of the response body.
-        """
-    _bodyDataFinished = makeStatefulDispatcher('bodyDataFinished',
-                                               _bodyDataFinished)
-
-
-    def _bodyDataFinished_INITIAL(self, reason=None):
-        """
-        Move to the C{'DEFERRED_CLOSE'} state to wait for a protocol to
-        which to deliver the response body.
-        """
-        self._state = 'DEFERRED_CLOSE'
-        if reason is None:
-            reason = Failure(ResponseDone("Response body fully received"))
-        self._reason = reason
-
-
-    def _bodyDataFinished_CONNECTED(self, reason=None):
-        """
-        Disconnect the protocol and move to the C{'FINISHED'} state.
-        """
-        if reason is None:
-            reason = Failure(ResponseDone("Response body fully received"))
-        self._bodyProtocol.connectionLost(reason)
-        self._bodyProtocol = None
-        self._state = 'FINISHED'
-
-
-    def _bodyDataFinished_DEFERRED_CLOSE(self):
-        """
-        It is invalid to attempt to notify the L{Response} of the end of the
-        response body data more than once.
-        """
-        raise RuntimeError("Cannot finish body data more than once")
-
-
-    def _bodyDataFinished_FINISHED(self):
-        """
-        It is invalid to attempt to notify the L{Response} of the end of the
-        response body data more than once.
-        """
-        raise RuntimeError("Cannot finish body data after protocol disconnected")
-
-
-
-class ChunkedEncoder:
-    """
-    Helper object which exposes L{IConsumer} on top of L{HTTP11ClientProtocol}
-    for streaming request bodies to the server.
-    """
-    implements(IConsumer)
-
-    def __init__(self, transport):
-        self.transport = transport
-
-
-    def _allowNoMoreWrites(self):
-        """
-        Indicate that no additional writes are allowed.  Attempts to write
-        after calling this method will be met with an exception.
-        """
-        self.transport = None
-
-
-    def registerProducer(self, producer, streaming):
-        """
-        Register the given producer with C{self.transport}.
-        """
-        self.transport.registerProducer(producer, streaming)
-
-
-    def write(self, data):
-        """
-        Write the given request body bytes to the transport using chunked
-        encoding.
-
-        @type data: C{str}
-        """
-        if self.transport is None:
-            raise ExcessWrite()
-        self.transport.writeSequence(("%x\r\n" % len(data), data, "\r\n"))
-
-
-    def unregisterProducer(self):
-        """
-        Indicate that the request body is complete and finish the request.
-        """
-        self.write('')
-        self.transport.unregisterProducer()
-        self._allowNoMoreWrites()
-
-
-
-class TransportProxyProducer:
-    """
-    An L{IPushProducer} implementation which wraps another such thing and
-    proxies calls to it until it is told to stop.
-
-    @ivar _producer: The wrapped L{IPushProducer} provider or C{None} after
-        this proxy has been stopped.
-    """
-    implements(IPushProducer)
-
-    # LineReceiver uses this undocumented attribute of transports to decide
-    # when to stop calling lineReceived or rawDataReceived (if it finds it to
-    # be true, it doesn't bother to deliver any more data).  Set disconnecting
-    # to False here and never change it to true so that all data is always
-    # delivered to us and so that LineReceiver doesn't fail with an
-    # AttributeError.
-    disconnecting = False
-
-    def __init__(self, producer):
-        self._producer = producer
-
-
-    def _stopProxying(self):
-        """
-        Stop forwarding calls of L{IPushProducer} methods to the underlying
-        L{IPushProvider} provider.
-        """
-        self._producer = None
-
-
-    def stopProducing(self):
-        """
-        Proxy the stoppage to the underlying producer, unless this proxy has
-        been stopped.
-        """
-        if self._producer is not None:
-            self._producer.stopProducing()
-
-
-    def resumeProducing(self):
-        """
-        Proxy the resumption to the underlying producer, unless this proxy has
-        been stopped.
-        """
-        if self._producer is not None:
-            self._producer.resumeProducing()
-
-
-    def pauseProducing(self):
-        """
-        Proxy the pause to the underlying producer, unless this proxy has been
-        stopped.
-        """
-        if self._producer is not None:
-            self._producer.pauseProducing()
-
-
-
-class HTTP11ClientProtocol(Protocol):
-    """
-    L{HTTP11ClientProtocol} is an implementation of the HTTP 1.1 client
-    protocol.  It supports as few features as possible.
-
-    @ivar _parser: After a request is issued, the L{HTTPClientParser} to
-        which received data making up the response to that request is
-        delivered.
-
-    @ivar _finishedRequest: After a request is issued, the L{Deferred} which
-        will fire when a L{Response} object corresponding to that request is
-        available.  This allows L{HTTP11ClientProtocol} to fail the request
-        if there is a connection or parsing problem.
-
-    @ivar _currentRequest: After a request is issued, the L{Request}
-        instance used to make that request.  This allows
-        L{HTTP11ClientProtocol} to stop request generation if necessary (for
-        example, if the connection is lost).
-
-    @ivar _transportProxy: After a request is issued, the
-        L{TransportProxyProducer} to which C{_parser} is connected.  This
-        allows C{_parser} to pause and resume the transport in a way which
-        L{HTTP11ClientProtocol} can exert some control over.
-
-    @ivar _responseDeferred: After a request is issued, the L{Deferred} from
-        C{_parser} which will fire with a L{Response} when one has been
-        received.  This is eventually chained with C{_finishedRequest}, but
-        only in certain cases to avoid double firing that Deferred.
-
-    @ivar _state: Indicates what state this L{HTTP11ClientProtocol} instance
-        is in with respect to transmission of a request and reception of a
-        response.  This may be one of the following strings:
-
-          - QUIESCENT: This is the state L{HTTP11ClientProtocol} instances
-            start in.  Nothing is happening: no request is being sent and no
-            response is being received or expected.
-
-          - TRANSMITTING: When a request is made (via L{request}), the
-            instance moves to this state.  L{Request.writeTo} has been used
-            to start to send a request but it has not yet finished.
-
-          - TRANSMITTING_AFTER_RECEIVING_RESPONSE: The server has returned a
-            complete response but the request has not yet been fully sent
-            yet.  The instance will remain in this state until the request
-            is fully sent.
-
-          - GENERATION_FAILED: There was an error while the request.  The
-            request was not fully sent to the network.
-
-          - WAITING: The request was fully sent to the network.  The
-            instance is now waiting for the response to be fully received.
-
-          - ABORTING: Application code has requested that the HTTP connection
-            be aborted.
-
-          - CONNECTION_LOST: The connection has been lost.
-
-    @ivar _abortDeferreds: A list of C{Deferred} instances that will fire when
-        the connection is lost.
-    """
-    _state = 'QUIESCENT'
-    _parser = None
-    _finishedRequest = None
-    _currentRequest = None
-    _transportProxy = None
-    _responseDeferred = None
-
-
-    def __init__(self, quiescentCallback=lambda c: None):
-        self._quiescentCallback = quiescentCallback
-        self._abortDeferreds = []
-
-
-    @property
-    def state(self):
-        return self._state
-
-
-    def request(self, request):
-        """
-        Issue C{request} over C{self.transport} and return a L{Deferred} which
-        will fire with a L{Response} instance or an error.
-
-        @param request: The object defining the parameters of the request to
-           issue.
-        @type request: L{Request}
-
-        @rtype: L{Deferred}
-        @return: The deferred may errback with L{RequestGenerationFailed} if
-            the request was not fully written to the transport due to a local
-            error.  It may errback with L{RequestTransmissionFailed} if it was
-            not fully written to the transport due to a network error.  It may
-            errback with L{ResponseFailed} if the request was sent (not
-            necessarily received) but some or all of the response was lost.  It
-            may errback with L{RequestNotSent} if it is not possible to send
-            any more requests using this L{HTTP11ClientProtocol}.
-        """
-        if self._state != 'QUIESCENT':
-            return fail(RequestNotSent())
-
-        self._state = 'TRANSMITTING'
-        _requestDeferred = maybeDeferred(request.writeTo, self.transport)
-
-        def cancelRequest(ign):
-            # Explicitly cancel the request's deferred if it's still trying to
-            # write when this request is cancelled.
-            if self._state in (
-                    'TRANSMITTING', 'TRANSMITTING_AFTER_RECEIVING_RESPONSE'):
-                _requestDeferred.cancel()
-            else:
-                self.transport.abortConnection()
-                self._disconnectParser(Failure(CancelledError()))
-        self._finishedRequest = Deferred(cancelRequest)
-
-        # Keep track of the Request object in case we need to call stopWriting
-        # on it.
-        self._currentRequest = request
-
-        self._transportProxy = TransportProxyProducer(self.transport)
-        self._parser = HTTPClientParser(request, self._finishResponse)
-        self._parser.makeConnection(self._transportProxy)
-        self._responseDeferred = self._parser._responseDeferred
-
-        def cbRequestWrotten(ignored):
-            if self._state == 'TRANSMITTING':
-                self._state = 'WAITING'
-                self._responseDeferred.chainDeferred(self._finishedRequest)
-
-        def ebRequestWriting(err):
-            if self._state == 'TRANSMITTING':
-                self._state = 'GENERATION_FAILED'
-                self.transport.abortConnection()
-                self._finishedRequest.errback(
-                    Failure(RequestGenerationFailed([err])))
-            else:
-                log.err(err, 'Error writing request, but not in valid state '
-                             'to finalize request: %s' % self._state)
-
-        _requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting)
-
-        return self._finishedRequest
-
-
-    def _finishResponse(self, rest):
-        """
-        Called by an L{HTTPClientParser} to indicate that it has parsed a
-        complete response.
-
-        @param rest: A C{str} giving any trailing bytes which were given to
-            the L{HTTPClientParser} which were not part of the response it
-            was parsing.
-        """
-    _finishResponse = makeStatefulDispatcher('finishResponse', _finishResponse)
-
-
-    def _finishResponse_WAITING(self, rest):
-        # Currently the rest parameter is ignored. Don't forget to use it if
-        # we ever add support for pipelining. And maybe check what trailers
-        # mean.
-        if self._state == 'WAITING':
-            self._state = 'QUIESCENT'
-        else:
-            # The server sent the entire response before we could send the
-            # whole request.  That sucks.  Oh well.  Fire the request()
-            # Deferred with the response.  But first, make sure that if the
-            # request does ever finish being written that it won't try to fire
-            # that Deferred.
-            self._state = 'TRANSMITTING_AFTER_RECEIVING_RESPONSE'
-            self._responseDeferred.chainDeferred(self._finishedRequest)
-
-        # This will happen if we're being called due to connection being lost;
-        # if so, no need to disconnect parser again, or to call
-        # _quiescentCallback.
-        if self._parser is None:
-            return
-
-        reason = ConnectionDone("synthetic!")
-        connHeaders = self._parser.connHeaders.getRawHeaders('connection', ())
-        if (('close' in connHeaders) or self._state != "QUIESCENT" or
-            not self._currentRequest.persistent):
-            self._giveUp(Failure(reason))
-        else:
-            # We call the quiescent callback first, to ensure connection gets
-            # added back to connection pool before we finish the request.
-            try:
-                self._quiescentCallback(self)
-            except:
-                # If callback throws exception, just log it and disconnect;
-                # keeping persistent connections around is an optimisation:
-                log.err()
-                self.transport.loseConnection()
-            self._disconnectParser(reason)
-
-
-    _finishResponse_TRANSMITTING = _finishResponse_WAITING
-
-
-    def _disconnectParser(self, reason):
-        """
-        If there is still a parser, call its C{connectionLost} method with the
-        given reason.  If there is not, do nothing.
-
-        @type reason: L{Failure}
-        """
-        if self._parser is not None:
-            parser = self._parser
-            self._parser = None
-            self._currentRequest = None
-            self._finishedRequest = None
-            self._responseDeferred = None
-
-            # The parser is no longer allowed to do anything to the real
-            # transport.  Stop proxying from the parser's transport to the real
-            # transport before telling the parser it's done so that it can't do
-            # anything.
-            self._transportProxy._stopProxying()
-            self._transportProxy = None
-            parser.connectionLost(reason)
-
-
-    def _giveUp(self, reason):
-        """
-        Lose the underlying connection and disconnect the parser with the given
-        L{Failure}.
-
-        Use this method instead of calling the transport's loseConnection
-        method directly otherwise random things will break.
-        """
-        self.transport.loseConnection()
-        self._disconnectParser(reason)
-
-
-    def dataReceived(self, bytes):
-        """
-        Handle some stuff from some place.
-        """
-        try:
-            self._parser.dataReceived(bytes)
-        except:
-            self._giveUp(Failure())
-
-
-    def connectionLost(self, reason):
-        """
-        The underlying transport went away.  If appropriate, notify the parser
-        object.
-        """
-    connectionLost = makeStatefulDispatcher('connectionLost', connectionLost)
-
-
-    def _connectionLost_QUIESCENT(self, reason):
-        """
-        Nothing is currently happening.  Move to the C{'CONNECTION_LOST'}
-        state but otherwise do nothing.
-        """
-        self._state = 'CONNECTION_LOST'
-
-
-    def _connectionLost_GENERATION_FAILED(self, reason):
-        """
-        The connection was in an inconsistent state.  Move to the
-        C{'CONNECTION_LOST'} state but otherwise do nothing.
-        """
-        self._state = 'CONNECTION_LOST'
-
-
-    def _connectionLost_TRANSMITTING(self, reason):
-        """
-        Fail the L{Deferred} for the current request, notify the request
-        object that it does not need to continue transmitting itself, and
-        move to the C{'CONNECTION_LOST'} state.
-        """
-        self._state = 'CONNECTION_LOST'
-        self._finishedRequest.errback(
-            Failure(RequestTransmissionFailed([reason])))
-        del self._finishedRequest
-
-        # Tell the request that it should stop bothering now.
-        self._currentRequest.stopWriting()
-
-
-    def _connectionLost_TRANSMITTING_AFTER_RECEIVING_RESPONSE(self, reason):
-        """
-        Move to the C{'CONNECTION_LOST'} state.
-        """
-        self._state = 'CONNECTION_LOST'
-
-
-    def _connectionLost_WAITING(self, reason):
-        """
-        Disconnect the response parser so that it can propagate the event as
-        necessary (for example, to call an application protocol's
-        C{connectionLost} method, or to fail a request L{Deferred}) and move
-        to the C{'CONNECTION_LOST'} state.
-        """
-        self._disconnectParser(reason)
-        self._state = 'CONNECTION_LOST'
-
-
-    def _connectionLost_ABORTING(self, reason):
-        """
-        Disconnect the response parser with a L{ConnectionAborted} failure, and
-        move to the C{'CONNECTION_LOST'} state.
-        """
-        self._disconnectParser(Failure(ConnectionAborted()))
-        self._state = 'CONNECTION_LOST'
-        for d in self._abortDeferreds:
-            d.callback(None)
-        self._abortDeferreds = []
-
-
-    def abort(self):
-        """
-        Close the connection and cause all outstanding L{request} L{Deferred}s
-        to fire with an error.
-        """
-        if self._state == "CONNECTION_LOST":
-            return succeed(None)
-        self.transport.loseConnection()
-        self._state = 'ABORTING'
-        d = Deferred()
-        self._abortDeferreds.append(d)
-        return d
-}}} '''
diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py
index 8e0b1df8b..c2d50648a 100644
--- a/scrapy/xlib/tx/client.py
+++ b/scrapy/xlib/tx/client.py
@@ -42,1129 +42,13 @@ from twisted.web.client import (
     Agent, ProxyAgent, HTTPConnectionPool, readBody,
 )
 
-''' {{{
-class PartialDownloadError(error.Error):
-    """
-    Page was only partially downloaded, we got disconnected in middle.
-
-    @ivar response: All of the response body which was downloaded.
-    """
-
-
-class _URL(tuple):
-    """
-    A parsed URL.
-
-    At some point this should be replaced with a better URL implementation.
-    """
-    def __new__(self, scheme, host, port, path):
-        return tuple.__new__(_URL, (scheme, host, port, path))
-
-
-    def __init__(self, scheme, host, port, path):
-        self.scheme = scheme
-        self.host = host
-        self.port = port
-        self.path = path
-
-
-def _parse(url, defaultPort=None):
-    """
-    Split the given URL into the scheme, host, port, and path.
-
-    @type url: C{bytes}
-    @param url: An URL to parse.
-
-    @type defaultPort: C{int} or C{None}
-    @param defaultPort: An alternate value to use as the port if the URL does
-    not include one.
-
-    @return: A four-tuple of the scheme, host, port, and path of the URL.  All
-    of these are C{bytes} instances except for port, which is an C{int}.
-    """
-    url = url.strip()
-    parsed = http.urlparse(url)
-    scheme = parsed[0]
-    path = urlunparse((b'', b'') + parsed[2:])
-
-    if defaultPort is None:
-        if scheme == b'https':
-            defaultPort = 443
-        else:
-            defaultPort = 80
-
-    host, port = parsed[1], defaultPort
-    if b':' in host:
-        host, port = host.split(b':')
-        try:
-            port = int(port)
-        except ValueError:
-            port = defaultPort
-
-    if path == b'':
-        path = b'/'
-
-    return _URL(scheme, host, port, path)
-
-
-def _makeGetterFactory(url, factoryFactory, contextFactory=None,
-                       *args, **kwargs):
-    """
-    Create and connect an HTTP page getting factory.
-
-    Any additional positional or keyword arguments are used when calling
-    C{factoryFactory}.
-
-    @param factoryFactory: Factory factory that is called with C{url}, C{args}
-        and C{kwargs} to produce the getter
-
-    @param contextFactory: Context factory to use when creating a secure
-        connection, defaulting to C{None}
-
-    @return: The factory created by C{factoryFactory}
-    """
-    scheme, host, port, path = _parse(url)
-    factory = factoryFactory(url, *args, **kwargs)
-    if scheme == b'https':
-        from twisted.internet import ssl
-        if contextFactory is None:
-            contextFactory = ssl.ClientContextFactory()
-        reactor.connectSSL(host, port, factory, contextFactory)
-    else:
-        reactor.connectTCP(host, port, factory)
-    return factory
-}}} '''
 
 # The code which follows is based on the new HTTP client implementation.  It
 # should be significantly better than anything above, though it is not yet
 # feature equivalent.
 
-#from twisted.web.error import SchemeNotSupported
 from twisted.web._newclient import Response
-#from twisted.web._newclient import Request, HTTP11ClientProtocol
 from twisted.web._newclient import ResponseDone, ResponseFailed
-#from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed
-#from twisted.web._newclient import (
-#    ResponseNeverReceived, PotentialDataLoss, _WrapperException)
-
-''' {{{
-try:
-    from twisted.internet.ssl import ClientContextFactory
-except ImportError:
-    class WebClientContextFactory(object):
-        """
-        A web context factory which doesn't work because the necessary SSL
-        support is missing.
-        """
-        def getContext(self, hostname, port):
-            raise NotImplementedError("SSL support unavailable")
-else:
-    class WebClientContextFactory(ClientContextFactory):
-        """
-        A web context factory which ignores the hostname and port and does no
-        certificate verification.
-        """
-        def getContext(self, hostname, port):
-            return ClientContextFactory.getContext(self)
-
-
-
-class _WebToNormalContextFactory(object):
-    """
-    Adapt a web context factory to a normal context factory.
-
-    @ivar _webContext: A web context factory which accepts a hostname and port
-        number to its C{getContext} method.
-
-    @ivar _hostname: The hostname which will be passed to
-        C{_webContext.getContext}.
-
-    @ivar _port: The port number which will be passed to
-        C{_webContext.getContext}.
-    """
-    def __init__(self, webContext, hostname, port):
-        self._webContext = webContext
-        self._hostname = hostname
-        self._port = port
-
-
-    def getContext(self):
-        """
-        Called the wrapped web context factory's C{getContext} method with a
-        hostname and port number and return the resulting context object.
-        """
-        return self._webContext.getContext(self._hostname, self._port)
-
-
-
-@implementer(IBodyProducer)
-class FileBodyProducer(object):
-    """
-    L{FileBodyProducer} produces bytes from an input file object incrementally
-    and writes them to a consumer.
-
-    Since file-like objects cannot be read from in an event-driven manner,
-    L{FileBodyProducer} uses a L{Cooperator} instance to schedule reads from
-    the file.  This process is also paused and resumed based on notifications
-    from the L{IConsumer} provider being written to.
-
-    The file is closed after it has been read, or if the producer is stopped
-    early.
-
-    @ivar _inputFile: Any file-like object, bytes read from which will be
-        written to a consumer.
-
-    @ivar _cooperate: A method like L{Cooperator.cooperate} which is used to
-        schedule all reads.
-
-    @ivar _readSize: The number of bytes to read from C{_inputFile} at a time.
-    """
-
-    # Python 2.4 doesn't have these symbolic constants
-    _SEEK_SET = getattr(os, 'SEEK_SET', 0)
-    _SEEK_END = getattr(os, 'SEEK_END', 2)
-
-    def __init__(self, inputFile, cooperator=task, readSize=2 ** 16):
-        self._inputFile = inputFile
-        self._cooperate = cooperator.cooperate
-        self._readSize = readSize
-        self.length = self._determineLength(inputFile)
-
-
-    def _determineLength(self, fObj):
-        """
-        Determine how many bytes can be read out of C{fObj} (assuming it is not
-        modified from this point on).  If the determination cannot be made,
-        return C{UNKNOWN_LENGTH}.
-        """
-        try:
-            seek = fObj.seek
-            tell = fObj.tell
-        except AttributeError:
-            return UNKNOWN_LENGTH
-        originalPosition = tell()
-        seek(0, self._SEEK_END)
-        end = tell()
-        seek(originalPosition, self._SEEK_SET)
-        return end - originalPosition
-
-
-    def stopProducing(self):
-        """
-        Permanently stop writing bytes from the file to the consumer by
-        stopping the underlying L{CooperativeTask}.
-        """
-        self._inputFile.close()
-        self._task.stop()
-
-
-    def startProducing(self, consumer):
-        """
-        Start a cooperative task which will read bytes from the input file and
-        write them to C{consumer}.  Return a L{Deferred} which fires after all
-        bytes have been written.
-
-        @param consumer: Any L{IConsumer} provider
-        """
-        self._task = self._cooperate(self._writeloop(consumer))
-        d = self._task.whenDone()
-        def maybeStopped(reason):
-            # IBodyProducer.startProducing's Deferred isn't support to fire if
-            # stopProducing is called.
-            reason.trap(task.TaskStopped)
-            return defer.Deferred()
-        d.addCallbacks(lambda ignored: None, maybeStopped)
-        return d
-
-
-    def _writeloop(self, consumer):
-        """
-        Return an iterator which reads one chunk of bytes from the input file
-        and writes them to the consumer for each time it is iterated.
-        """
-        while True:
-            bytes = self._inputFile.read(self._readSize)
-            if not bytes:
-                self._inputFile.close()
-                break
-            consumer.write(bytes)
-            yield None
-
-
-    def pauseProducing(self):
-        """
-        Temporarily suspend copying bytes from the input file to the consumer
-        by pausing the L{CooperativeTask} which drives that activity.
-        """
-        self._task.pause()
-
-
-    def resumeProducing(self):
-        """
-        Undo the effects of a previous C{pauseProducing} and resume copying
-        bytes to the consumer by resuming the L{CooperativeTask} which drives
-        the write activity.
-        """
-        self._task.resume()
-
-
-
-class _HTTP11ClientFactory(protocol.Factory):
-    """
-    A factory for L{HTTP11ClientProtocol}, used by L{HTTPConnectionPool}.
-
-    @ivar _quiescentCallback: The quiescent callback to be passed to protocol
-        instances, used to return them to the connection pool.
-
-    @since: 11.1
-    """
-    def __init__(self, quiescentCallback):
-        self._quiescentCallback = quiescentCallback
-
-
-    def buildProtocol(self, addr):
-        return HTTP11ClientProtocol(self._quiescentCallback)
-
-
-
-class _RetryingHTTP11ClientProtocol(object):
-    """
-    A wrapper for L{HTTP11ClientProtocol} that automatically retries requests.
-
-    @ivar _clientProtocol: The underlying L{HTTP11ClientProtocol}.
-
-    @ivar _newConnection: A callable that creates a new connection for a
-        retry.
-    """
-
-    def __init__(self, clientProtocol, newConnection):
-        self._clientProtocol = clientProtocol
-        self._newConnection = newConnection
-
-
-    def _shouldRetry(self, method, exception, bodyProducer):
-        """
-        Indicate whether request should be retried.
-
-        Only returns C{True} if method is idempotent, no response was
-        received, the reason for the failed request was not due to
-        user-requested cancellation, and no body was sent. The latter
-        requirement may be relaxed in the future, and PUT added to approved
-        method list.
-        """
-        if method not in ("GET", "HEAD", "OPTIONS", "DELETE", "TRACE"):
-            return False
-        if not isinstance(exception, (RequestNotSent, RequestTransmissionFailed,
-                                      ResponseNeverReceived)):
-            return False
-        if isinstance(exception, _WrapperException):
-            for failure in exception.reasons:
-                if failure.check(defer.CancelledError):
-                    return False
-        if bodyProducer is not None:
-            return False
-        return True
-
-
-    def request(self, request):
-        """
-        Do a request, and retry once (with a new connection) it it fails in
-        a retryable manner.
-
-        @param request: A L{Request} instance that will be requested using the
-            wrapped protocol.
-        """
-        d = self._clientProtocol.request(request)
-
-        def failed(reason):
-            if self._shouldRetry(request.method, reason.value,
-                                 request.bodyProducer):
-                return self._newConnection().addCallback(
-                    lambda connection: connection.request(request))
-            else:
-                return reason
-        d.addErrback(failed)
-        return d
-
-
-
-class HTTPConnectionPool(object):
-    """
-    A pool of persistent HTTP connections.
-
-    Features:
-     - Cached connections will eventually time out.
-     - Limits on maximum number of persistent connections.
-
-    Connections are stored using keys, which should be chosen such that any
-    connections stored under a given key can be used interchangeably.
-
-    Failed requests done using previously cached connections will be retried
-    once if they use an idempotent method (e.g. GET), in case the HTTP server
-    timed them out.
-
-    @ivar persistent: Boolean indicating whether connections should be
-        persistent. Connections are persistent by default.
-
-    @ivar maxPersistentPerHost: The maximum number of cached persistent
-        connections for a C{host:port} destination.
-    @type maxPersistentPerHost: C{int}
-
-    @ivar cachedConnectionTimeout: Number of seconds a cached persistent
-        connection will stay open before disconnecting.
-
-    @ivar retryAutomatically: C{boolean} indicating whether idempotent
-        requests should be retried once if no response was received.
-
-    @ivar _factory: The factory used to connect to the proxy.
-
-    @ivar _connections: Map (scheme, host, port) to lists of
-        L{HTTP11ClientProtocol} instances.
-
-    @ivar _timeouts: Map L{HTTP11ClientProtocol} instances to a
-        C{IDelayedCall} instance of their timeout.
-
-    @since: 12.1
-    """
-
-    _factory = _HTTP11ClientFactory
-    maxPersistentPerHost = 2
-    cachedConnectionTimeout = 240
-    retryAutomatically = True
-
-    def __init__(self, reactor, persistent=True):
-        self._reactor = reactor
-        self.persistent = persistent
-        self._connections = {}
-        self._timeouts = {}
-
-
-    def getConnection(self, key, endpoint):
-        """
-        Supply a connection, newly created or retrieved from the pool, to be
-        used for one HTTP request.
-
-        The connection will remain out of the pool (not available to be
-        returned from future calls to this method) until one HTTP request has
-        been completed over it.
-
-        Afterwards, if the connection is still open, it will automatically be
-        added to the pool.
-
-        @param key: A unique key identifying connections that can be used
-            interchangeably.
-
-        @param endpoint: An endpoint that can be used to open a new connection
-            if no cached connection is available.
-
-        @return: A C{Deferred} that will fire with a L{HTTP11ClientProtocol}
-           (or a wrapper) that can be used to send a single HTTP request.
-        """
-        # Try to get cached version:
-        connections = self._connections.get(key)
-        while connections:
-            connection = connections.pop(0)
-            # Cancel timeout:
-            self._timeouts[connection].cancel()
-            del self._timeouts[connection]
-            if connection.state == "QUIESCENT":
-                if self.retryAutomatically:
-                    newConnection = lambda: self._newConnection(key, endpoint)
-                    connection = _RetryingHTTP11ClientProtocol(
-                        connection, newConnection)
-                return defer.succeed(connection)
-
-        return self._newConnection(key, endpoint)
-
-
-    def _newConnection(self, key, endpoint):
-        """
-        Create a new connection.
-
-        This implements the new connection code path for L{getConnection}.
-        """
-        def quiescentCallback(protocol):
-            self._putConnection(key, protocol)
-        factory = self._factory(quiescentCallback)
-        return endpoint.connect(factory)
-
-
-    def _removeConnection(self, key, connection):
-        """
-        Remove a connection from the cache and disconnect it.
-        """
-        connection.transport.loseConnection()
-        self._connections[key].remove(connection)
-        del self._timeouts[connection]
-
-
-    def _putConnection(self, key, connection):
-        """
-        Return a persistent connection to the pool. This will be called by
-        L{HTTP11ClientProtocol} when the connection becomes quiescent.
-        """
-        if connection.state != "QUIESCENT":
-            # Log with traceback for debugging purposes:
-            try:
-                raise RuntimeError(
-                    "BUG: Non-quiescent protocol added to connection pool.")
-            except:
-                log.err()
-            return
-        connections = self._connections.setdefault(key, [])
-        if len(connections) == self.maxPersistentPerHost:
-            dropped = connections.pop(0)
-            dropped.transport.loseConnection()
-            self._timeouts[dropped].cancel()
-            del self._timeouts[dropped]
-        connections.append(connection)
-        cid = self._reactor.callLater(self.cachedConnectionTimeout,
-                                      self._removeConnection,
-                                      key, connection)
-        self._timeouts[connection] = cid
-
-
-    def closeCachedConnections(self):
-        """
-        Close all persistent connections and remove them from the pool.
-
-        @return: L{defer.Deferred} that fires when all connections have been
-            closed.
-        """
-        results = []
-        for protocols in self._connections.itervalues():
-            for p in protocols:
-                results.append(p.abort())
-        self._connections = {}
-        for dc in self._timeouts.values():
-            dc.cancel()
-        self._timeouts = {}
-        return defer.gatherResults(results).addCallback(lambda ign: None)
-
-
-
-class _AgentBase(object):
-    """
-    Base class offering common facilities for L{Agent}-type classes.
-
-    @ivar _reactor: The C{IReactorTime} implementation which will be used by
-        the pool, and perhaps by subclasses as well.
-
-    @ivar _pool: The L{HTTPConnectionPool} used to manage HTTP connections.
-    """
-
-    def __init__(self, reactor, pool):
-        if pool is None:
-            pool = HTTPConnectionPool(reactor, False)
-        self._reactor = reactor
-        self._pool = pool
-
-
-    def _computeHostValue(self, scheme, host, port):
-        """
-        Compute the string to use for the value of the I{Host} header, based on
-        the given scheme, host name, and port number.
-        """
-        if (scheme, port) in (('http', 80), ('https', 443)):
-            return host
-        return '%s:%d' % (host, port)
-
-
-    def _requestWithEndpoint(self, key, endpoint, method, parsedURI,
-                             headers, bodyProducer, requestPath):
-        """
-        Issue a new request, given the endpoint and the path sent as part of
-        the request.
-        """
-        # Create minimal headers, if necessary:
-        if headers is None:
-            headers = Headers()
-        if not headers.hasHeader('host'):
-            #headers = headers.copy()  # not supported in twisted <= 11.1, and it doesn't affects us
-            headers.addRawHeader(
-                'host', self._computeHostValue(parsedURI.scheme, parsedURI.host,
-                                               parsedURI.port))
-
-        d = self._pool.getConnection(key, endpoint)
-        def cbConnected(proto):
-            return proto.request(
-                Request(method, requestPath, headers, bodyProducer,
-                        persistent=self._pool.persistent))
-        d.addCallback(cbConnected)
-        return d
-
-
-
-class Agent(_AgentBase):
-    """
-    L{Agent} is a very basic HTTP client.  It supports I{HTTP} and I{HTTPS}
-    scheme URIs (but performs no certificate checking by default).
-
-    @param pool: A L{HTTPConnectionPool} instance, or C{None}, in which case a
-        non-persistent L{HTTPConnectionPool} instance will be created.
-
-    @ivar _contextFactory: A web context factory which will be used to create
-        SSL context objects for any SSL connections the agent needs to make.
-
-    @ivar _connectTimeout: If not C{None}, the timeout passed to C{connectTCP}
-        or C{connectSSL} for specifying the connection timeout.
-
-    @ivar _bindAddress: If not C{None}, the address passed to C{connectTCP} or
-        C{connectSSL} for specifying the local address to bind to.
-
-    @since: 9.0
-    """
-
-    def __init__(self, reactor, contextFactory=WebClientContextFactory(),
-                 connectTimeout=None, bindAddress=None,
-                 pool=None):
-        _AgentBase.__init__(self, reactor, pool)
-        self._contextFactory = contextFactory
-        self._connectTimeout = connectTimeout
-        self._bindAddress = bindAddress
-
-
-    def _wrapContextFactory(self, host, port):
-        """
-        Create and return a normal context factory wrapped around
-        C{self._contextFactory} in such a way that C{self._contextFactory} will
-        have the host and port information passed to it.
-
-        @param host: A C{str} giving the hostname which will be connected to in
-            order to issue a request.
-
-        @param port: An C{int} giving the port number the connection will be
-            on.
-
-        @return: A context factory suitable to be passed to
-            C{reactor.connectSSL}.
-        """
-        return _WebToNormalContextFactory(self._contextFactory, host, port)
-
-
-    def _getEndpoint(self, scheme, host, port):
-        """
-        Get an endpoint for the given host and port, using a transport
-        selected based on scheme.
-
-        @param scheme: A string like C{'http'} or C{'https'} (the only two
-            supported values) to use to determine how to establish the
-            connection.
-
-        @param host: A C{str} giving the hostname which will be connected to in
-            order to issue a request.
-
-        @param port: An C{int} giving the port number the connection will be
-            on.
-
-        @return: An endpoint which can be used to connect to given address.
-        """
-        kwargs = {}
-        if self._connectTimeout is not None:
-            kwargs['timeout'] = self._connectTimeout
-        kwargs['bindAddress'] = self._bindAddress
-        if scheme == 'http':
-            return TCP4ClientEndpoint(self._reactor, host, port, **kwargs)
-        elif scheme == 'https':
-            return SSL4ClientEndpoint(self._reactor, host, port,
-                                      self._wrapContextFactory(host, port),
-                                      **kwargs)
-        else:
-            raise SchemeNotSupported("Unsupported scheme: %r" % (scheme,))
-
-
-    def request(self, method, uri, headers=None, bodyProducer=None):
-        """
-        Issue a new request.
-
-        @param method: The request method to send.
-        @type method: C{str}
-
-        @param uri: The request URI send.
-        @type uri: C{str}
-
-        @param headers: The request headers to send.  If no I{Host} header is
-            included, one will be added based on the request URI.
-        @type headers: L{Headers}
-
-        @param bodyProducer: An object which will produce the request body or,
-            if the request body is to be empty, L{None}.
-        @type bodyProducer: L{IBodyProducer} provider
-
-        @return: A L{Deferred} which fires with the result of the request (a
-            L{twisted.web.iweb.IResponse} provider), or fails if there is a
-            problem setting up a connection over which to issue the request.
-            It may also fail with L{SchemeNotSupported} if the scheme of the
-            given URI is not supported.
-        @rtype: L{Deferred}
-        """
-        parsedURI = _parse(uri)
-        try:
-            endpoint = self._getEndpoint(parsedURI.scheme, parsedURI.host,
-                                         parsedURI.port)
-        except SchemeNotSupported:
-            return defer.fail(Failure())
-        key = (parsedURI.scheme, parsedURI.host, parsedURI.port)
-        return self._requestWithEndpoint(key, endpoint, method, parsedURI,
-                                         headers, bodyProducer, parsedURI.path)
-
-
-
-class ProxyAgent(_AgentBase):
-    """
-    An HTTP agent able to cross HTTP proxies.
-
-    @ivar _proxyEndpoint: The endpoint used to connect to the proxy.
-
-    @since: 11.1
-    """
-
-    def __init__(self, endpoint, reactor=None, pool=None):
-        if reactor is None:
-            from twisted.internet import reactor
-        _AgentBase.__init__(self, reactor, pool)
-        self._proxyEndpoint = endpoint
-
-
-    def request(self, method, uri, headers=None, bodyProducer=None):
-        """
-        Issue a new request via the configured proxy.
-        """
-        # Cache *all* connections under the same key, since we are only
-        # connecting to a single destination, the proxy:
-        key = ("http-proxy", self._proxyEndpoint)
-
-        # To support proxying HTTPS via CONNECT, we will use key
-        # ("http-proxy-CONNECT", scheme, host, port), and an endpoint that
-        # wraps _proxyEndpoint with an additional callback to do the CONNECT.
-        return self._requestWithEndpoint(key, self._proxyEndpoint, method,
-                                         _parse(uri), headers, bodyProducer,
-                                         uri)
-
-
-
-class _FakeUrllib2Request(object):
-    """
-    A fake C{urllib2.Request} object for C{cookielib} to work with.
-
-    @see: U{http://docs.python.org/library/urllib2.html#request-objects}
-
-    @type uri: C{str}
-    @ivar uri: Request URI.
-
-    @type headers: L{twisted.web.http_headers.Headers}
-    @ivar headers: Request headers.
-
-    @type type: C{str}
-    @ivar type: The scheme of the URI.
-
-    @type host: C{str}
-    @ivar host: The host[:port] of the URI.
-
-    @since: 11.1
-    """
-    def __init__(self, uri):
-        self.uri = uri
-        self.headers = Headers()
-        self.type, rest = splittype(self.uri)
-        self.host, rest = splithost(rest)
-
-
-    def has_header(self, header):
-        return self.headers.hasHeader(header)
-
-
-    def add_unredirected_header(self, name, value):
-        self.headers.addRawHeader(name, value)
-
-
-    def get_full_url(self):
-        return self.uri
-
-
-    def get_header(self, name, default=None):
-        headers = self.headers.getRawHeaders(name, default)
-        if headers is not None:
-            return headers[0]
-        return None
-
-
-    def get_host(self):
-        return self.host
-
-
-    def get_type(self):
-        return self.type
-
-
-    def is_unverifiable(self):
-        # In theory this shouldn't be hardcoded.
-        return False
-
-
-
-class _FakeUrllib2Response(object):
-    """
-    A fake C{urllib2.Response} object for C{cookielib} to work with.
-
-    @type response: C{twisted.web.iweb.IResponse}
-    @ivar response: Underlying Twisted Web response.
-
-    @since: 11.1
-    """
-    def __init__(self, response):
-        self.response = response
-
-
-    def info(self):
-        class _Meta(object):
-            def getheaders(zelf, name):
-                return self.response.headers.getRawHeaders(name, [])
-        return _Meta()
-
-
-
-class CookieAgent(object):
-    """
-    L{CookieAgent} extends the basic L{Agent} to add RFC-compliant
-    handling of HTTP cookies.  Cookies are written to and extracted
-    from a C{cookielib.CookieJar} instance.
-
-    The same cookie jar instance will be used for any requests through this
-    agent, mutating it whenever a I{Set-Cookie} header appears in a response.
-
-    @type _agent: L{twisted.web.client.Agent}
-    @ivar _agent: Underlying Twisted Web agent to issue requests through.
-
-    @type cookieJar: C{cookielib.CookieJar}
-    @ivar cookieJar: Initialized cookie jar to read cookies from and store
-        cookies to.
-
-    @since: 11.1
-    """
-    def __init__(self, agent, cookieJar):
-        self._agent = agent
-        self.cookieJar = cookieJar
-
-
-    def request(self, method, uri, headers=None, bodyProducer=None):
-        """
-        Issue a new request to the wrapped L{Agent}.
-
-        Send a I{Cookie} header if a cookie for C{uri} is stored in
-        L{CookieAgent.cookieJar}. Cookies are automatically extracted and
-        stored from requests.
-
-        If a C{'cookie'} header appears in C{headers} it will override the
-        automatic cookie header obtained from the cookie jar.
-
-        @see: L{Agent.request}
-        """
-        if headers is None:
-            headers = Headers()
-        lastRequest = _FakeUrllib2Request(uri)
-        # Setting a cookie header explicitly will disable automatic request
-        # cookies.
-        if not headers.hasHeader('cookie'):
-            self.cookieJar.add_cookie_header(lastRequest)
-            cookieHeader = lastRequest.get_header('Cookie', None)
-            if cookieHeader is not None:
-                headers = headers.copy()
-                headers.addRawHeader('cookie', cookieHeader)
-
-        d = self._agent.request(method, uri, headers, bodyProducer)
-        d.addCallback(self._extractCookies, lastRequest)
-        return d
-
-
-    def _extractCookies(self, response, request):
-        """
-        Extract response cookies and store them in the cookie jar.
-
-        @type response: L{twisted.web.iweb.IResponse}
-        @param response: Twisted Web response.
-
-        @param request: A urllib2 compatible request object.
-        """
-        resp = _FakeUrllib2Response(response)
-        self.cookieJar.extract_cookies(resp, request)
-        return response
-
-
-
-class GzipDecoder(proxyForInterface(IResponse)):
-    """
-    A wrapper for a L{Response} instance which handles gzip'ed body.
-
-    @ivar original: The original L{Response} object.
-
-    @since: 11.1
-    """
-
-    def __init__(self, response):
-        self.original = response
-        self.length = UNKNOWN_LENGTH
-
-
-    def deliverBody(self, protocol):
-        """
-        Override C{deliverBody} to wrap the given C{protocol} with
-        L{_GzipProtocol}.
-        """
-        self.original.deliverBody(_GzipProtocol(protocol, self.original))
-
-
-
-class _GzipProtocol(proxyForInterface(IProtocol)):
-    """
-    A L{Protocol} implementation which wraps another one, transparently
-    decompressing received data.
-
-    @ivar _zlibDecompress: A zlib decompress object used to decompress the data
-        stream.
-
-    @ivar _response: A reference to the original response, in case of errors.
-
-    @since: 11.1
-    """
-
-    def __init__(self, protocol, response):
-        self.original = protocol
-        self._response = response
-        self._zlibDecompress = zlib.decompressobj(16 + zlib.MAX_WBITS)
-
-
-    def dataReceived(self, data):
-        """
-        Decompress C{data} with the zlib decompressor, forwarding the raw data
-        to the original protocol.
-        """
-        try:
-            rawData = self._zlibDecompress.decompress(data)
-        except zlib.error:
-            raise ResponseFailed([failure.Failure()], self._response)
-        if rawData:
-            self.original.dataReceived(rawData)
-
-
-    def connectionLost(self, reason):
-        """
-        Forward the connection lost event, flushing remaining data from the
-        decompressor if any.
-        """
-        try:
-            rawData = self._zlibDecompress.flush()
-        except zlib.error:
-            raise ResponseFailed([reason, failure.Failure()], self._response)
-        if rawData:
-            self.original.dataReceived(rawData)
-        self.original.connectionLost(reason)
-
-
-
-class ContentDecoderAgent(object):
-    """
-    An L{Agent} wrapper to handle encoded content.
-
-    It takes care of declaring the support for content in the
-    I{Accept-Encoding} header, and automatically decompresses the received data
-    if it's effectively using compression.
-
-    @param decoders: A list or tuple of (name, decoder) objects. The name
-        declares which decoding the decoder supports, and the decoder must
-        return a response object when called/instantiated. For example,
-        C{(('gzip', GzipDecoder))}. The order determines how the decoders are
-        going to be advertized to the server.
-
-    @since: 11.1
-    """
-
-    def __init__(self, agent, decoders):
-        self._agent = agent
-        self._decoders = dict(decoders)
-        self._supported = ','.join([decoder[0] for decoder in decoders])
-
-
-    def request(self, method, uri, headers=None, bodyProducer=None):
-        """
-        Send a client request which declares supporting compressed content.
-
-        @see: L{Agent.request}.
-        """
-        if headers is None:
-            headers = Headers()
-        else:
-            headers = headers.copy()
-        headers.addRawHeader('accept-encoding', self._supported)
-        deferred = self._agent.request(method, uri, headers, bodyProducer)
-        return deferred.addCallback(self._handleResponse)
-
-
-    def _handleResponse(self, response):
-        """
-        Check if the response is encoded, and wrap it to handle decompression.
-        """
-        contentEncodingHeaders = response.headers.getRawHeaders(
-            'content-encoding', [])
-        contentEncodingHeaders = ','.join(contentEncodingHeaders).split(',')
-        while contentEncodingHeaders:
-            name = contentEncodingHeaders.pop().strip()
-            decoder = self._decoders.get(name)
-            if decoder is not None:
-                response = decoder(response)
-            else:
-                # Add it back
-                contentEncodingHeaders.append(name)
-                break
-        if contentEncodingHeaders:
-            response.headers.setRawHeaders(
-                'content-encoding', [','.join(contentEncodingHeaders)])
-        else:
-            response.headers.removeHeader('content-encoding')
-        return response
-
-
-
-class RedirectAgent(object):
-    """
-    An L{Agent} wrapper which handles HTTP redirects.
-
-    The implementation is rather strict: 301 and 302 behaves like 307, not
-    redirecting automatically on methods different from C{GET} and C{HEAD}.
-
-    @param redirectLimit: The maximum number of times the agent is allowed to
-        follow redirects before failing with a L{error.InfiniteRedirection}.
-
-    @since: 11.1
-    """
-
-    def __init__(self, agent, redirectLimit=20):
-        self._agent = agent
-        self._redirectLimit = redirectLimit
-
-
-    def request(self, method, uri, headers=None, bodyProducer=None):
-        """
-        Send a client request following HTTP redirects.
-
-        @see: L{Agent.request}.
-        """
-        deferred = self._agent.request(method, uri, headers, bodyProducer)
-        return deferred.addCallback(
-            self._handleResponse, method, uri, headers, 0)
-
-
-    def _handleRedirect(self, response, method, uri, headers, redirectCount):
-        """
-        Handle a redirect response, checking the number of redirects already
-        followed, and extracting the location header fields.
-        """
-        if redirectCount >= self._redirectLimit:
-            err = error.InfiniteRedirection(
-                response.code,
-                'Infinite redirection detected',
-                location=uri)
-            raise ResponseFailed([failure.Failure(err)], response)
-        locationHeaders = response.headers.getRawHeaders('location', [])
-        if not locationHeaders:
-            err = error.RedirectWithNoLocation(
-                response.code, 'No location header field', uri)
-            raise ResponseFailed([failure.Failure(err)], response)
-        location = locationHeaders[0]
-        deferred = self._agent.request(method, location, headers)
-        return deferred.addCallback(
-            self._handleResponse, method, uri, headers, redirectCount + 1)
-
-
-    def _handleResponse(self, response, method, uri, headers, redirectCount):
-        """
-        Handle the response, making another request if it indicates a redirect.
-        """
-        if response.code in (http.MOVED_PERMANENTLY, http.FOUND,
-                             http.TEMPORARY_REDIRECT):
-            if method not in ('GET', 'HEAD'):
-                err = error.PageRedirect(response.code, location=uri)
-                raise ResponseFailed([failure.Failure(err)], response)
-            return self._handleRedirect(response, method, uri, headers,
-                                        redirectCount)
-        elif response.code == http.SEE_OTHER:
-            return self._handleRedirect(response, 'GET', uri, headers,
-                                        redirectCount)
-        return response
-
-
-
-class _ReadBodyProtocol(protocol.Protocol):
-    """
-    Protocol that collects data sent to it.
-
-    This is a helper for L{IResponse.deliverBody}, which collects the body and
-    fires a deferred with it.
-
-    @ivar deferred: See L{__init__}.
-    @ivar status: See L{__init__}.
-    @ivar message: See L{__init__}.
-
-    @ivar dataBuffer: list of byte-strings received
-    @type dataBuffer: L{list} of L{bytes}
-    """
-
-    def __init__(self, status, message, deferred):
-        """
-        @param status: Status of L{IResponse}
-        @ivar status: L{int}
-
-        @param message: Message of L{IResponse}
-        @type message: L{bytes}
-
-        @param deferred: deferred to fire when response is complete
-        @type deferred: L{Deferred} firing with L{bytes}
-        """
-        self.deferred = deferred
-        self.status = status
-        self.message = message
-        self.dataBuffer = []
-
-
-    def dataReceived(self, data):
-        """
-        Accumulate some more bytes from the response.
-        """
-        self.dataBuffer.append(data)
-
-
-    def connectionLost(self, reason):
-        """
-        Deliver the accumulated response bytes to the waiting L{Deferred}, if
-        the response body has been completely received without error.
-        """
-        if reason.check(ResponseDone):
-            self.deferred.callback(b''.join(self.dataBuffer))
-        elif reason.check(PotentialDataLoss):
-            self.deferred.errback(
-                PartialDownloadError(self.status, self.message,
-                                     b''.join(self.dataBuffer)))
-        else:
-            self.deferred.errback(reason)
-
-
-
-def readBody(response):
-    """
-    Get the body of an L{IResponse} and return it as a byte string.
-
-    This is a helper function for clients that don't want to incrementally
-    receive the body of an HTTP response.
-
-    @param response: The HTTP response for which the body will be read.
-    @type response: L{IResponse} provider
-
-    @return: A L{Deferred} which will fire with the body of the response.
-    """
-    d = defer.Deferred()
-    response.deliverBody(_ReadBodyProtocol(response.code, response.phrase, d))
-    return d
-}}} '''
 
 
 __all__ = [
diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py
index 3f4704064..197e43ed3 100644
--- a/scrapy/xlib/tx/endpoints.py
+++ b/scrapy/xlib/tx/endpoints.py
@@ -14,27 +14,6 @@ parsed by the L{clientFromString} and L{serverFromString} functions.
 
 from __future__ import division, absolute_import
 
-import os
-#import socket
-
-from zope.interface import implementer, directlyProvides
-import warnings
-
-from twisted.internet import interfaces, defer, error, fdesc
-#from twisted.internet.protocol import (
-#        ClientFactory, Protocol)
-from twisted.internet.protocol import Factory
-#from twisted.internet import threads, ProcessProtocol
-from twisted.internet.interfaces import IStreamServerEndpointStringParser
-#from twisted.internet.interfaces import IStreamClientEndpointStringParser
-from twisted.python.filepath import FilePath
-#from twisted.python.failure import Failure
-#from twisted.python import log
-#from twisted.python.components import proxyForInterface
-
-from twisted.plugin import IPlugin, getPlugins
-#from twisted.internet import stdio
-
 from twisted.internet.endpoints import (
     clientFromString, serverFromString, quoteStringArgument,
     TCP4ServerEndpoint, TCP6ServerEndpoint,
@@ -44,1236 +23,4 @@ from twisted.internet.endpoints import (
     AdoptedStreamServerEndpoint, connectProtocol,
 )
 
-
 __all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"]
-
-''' {{{
-class _WrappingProtocol(Protocol):
-    """
-    Wrap another protocol in order to notify my user when a connection has
-    been made.
-    """
-
-    def __init__(self, connectedDeferred, wrappedProtocol):
-        """
-        @param connectedDeferred: The L{Deferred} that will callback
-            with the C{wrappedProtocol} when it is connected.
-
-        @param wrappedProtocol: An L{IProtocol} provider that will be
-            connected.
-        """
-        self._connectedDeferred = connectedDeferred
-        self._wrappedProtocol = wrappedProtocol
-
-        for iface in [interfaces.IHalfCloseableProtocol,
-                      interfaces.IFileDescriptorReceiver]:
-            if iface.providedBy(self._wrappedProtocol):
-                directlyProvides(self, iface)
-
-
-    def logPrefix(self):
-        """
-        Transparently pass through the wrapped protocol's log prefix.
-        """
-        if interfaces.ILoggingContext.providedBy(self._wrappedProtocol):
-            return self._wrappedProtocol.logPrefix()
-        return self._wrappedProtocol.__class__.__name__
-
-
-    def connectionMade(self):
-        """
-        Connect the C{self._wrappedProtocol} to our C{self.transport} and
-        callback C{self._connectedDeferred} with the C{self._wrappedProtocol}
-        """
-        self._wrappedProtocol.makeConnection(self.transport)
-        self._connectedDeferred.callback(self._wrappedProtocol)
-
-
-    def dataReceived(self, data):
-        """
-        Proxy C{dataReceived} calls to our C{self._wrappedProtocol}
-        """
-        return self._wrappedProtocol.dataReceived(data)
-
-
-    def fileDescriptorReceived(self, descriptor):
-        """
-        Proxy C{fileDescriptorReceived} calls to our C{self._wrappedProtocol}
-        """
-        return self._wrappedProtocol.fileDescriptorReceived(descriptor)
-
-
-    def connectionLost(self, reason):
-        """
-        Proxy C{connectionLost} calls to our C{self._wrappedProtocol}
-        """
-        return self._wrappedProtocol.connectionLost(reason)
-
-
-    def readConnectionLost(self):
-        """
-        Proxy L{IHalfCloseableProtocol.readConnectionLost} to our
-        C{self._wrappedProtocol}
-        """
-        self._wrappedProtocol.readConnectionLost()
-
-
-    def writeConnectionLost(self):
-        """
-        Proxy L{IHalfCloseableProtocol.writeConnectionLost} to our
-        C{self._wrappedProtocol}
-        """
-        self._wrappedProtocol.writeConnectionLost()
-
-
-
-class _WrappingFactory(ClientFactory):
-    """
-    Wrap a factory in order to wrap the protocols it builds.
-
-    @ivar _wrappedFactory: A provider of I{IProtocolFactory} whose buildProtocol
-        method will be called and whose resulting protocol will be wrapped.
-
-    @ivar _onConnection: A L{Deferred} that fires when the protocol is
-        connected
-
-    @ivar _connector: A L{connector <twisted.internet.interfaces.IConnector>}
-        that is managing the current or previous connection attempt.
-    """
-    protocol = _WrappingProtocol
-
-    def __init__(self, wrappedFactory):
-        """
-        @param wrappedFactory: A provider of I{IProtocolFactory} whose
-            buildProtocol method will be called and whose resulting protocol
-            will be wrapped.
-        """
-        self._wrappedFactory = wrappedFactory
-        self._onConnection = defer.Deferred(canceller=self._canceller)
-
-
-    def startedConnecting(self, connector):
-        """
-        A connection attempt was started.  Remember the connector which started
-        said attempt, for use later.
-        """
-        self._connector = connector
-
-
-    def _canceller(self, deferred):
-        """
-        The outgoing connection attempt was cancelled.  Fail that L{Deferred}
-        with an L{error.ConnectingCancelledError}.
-
-        @param deferred: The L{Deferred <defer.Deferred>} that was cancelled;
-            should be the same as C{self._onConnection}.
-        @type deferred: L{Deferred <defer.Deferred>}
-
-        @note: This relies on startedConnecting having been called, so it may
-            seem as though there's a race condition where C{_connector} may not
-            have been set.  However, using public APIs, this condition is
-            impossible to catch, because a connection API
-            (C{connectTCP}/C{SSL}/C{UNIX}) is always invoked before a
-            L{_WrappingFactory}'s L{Deferred <defer.Deferred>} is returned to
-            C{connect()}'s caller.
-
-        @return: C{None}
-        """
-        deferred.errback(
-            error.ConnectingCancelledError(
-                self._connector.getDestination()))
-        self._connector.stopConnecting()
-
-
-    def doStart(self):
-        """
-        Start notifications are passed straight through to the wrapped factory.
-        """
-        self._wrappedFactory.doStart()
-
-
-    def doStop(self):
-        """
-        Stop notifications are passed straight through to the wrapped factory.
-        """
-        self._wrappedFactory.doStop()
-
-
-    def buildProtocol(self, addr):
-        """
-        Proxy C{buildProtocol} to our C{self._wrappedFactory} or errback
-        the C{self._onConnection} L{Deferred}.
-
-        @return: An instance of L{_WrappingProtocol} or C{None}
-        """
-        try:
-            proto = self._wrappedFactory.buildProtocol(addr)
-        except:
-            self._onConnection.errback()
-        else:
-            return self.protocol(self._onConnection, proto)
-
-
-    def clientConnectionFailed(self, connector, reason):
-        """
-        Errback the C{self._onConnection} L{Deferred} when the
-        client connection fails.
-        """
-        if not self._onConnection.called:
-            self._onConnection.errback(reason)
-
-
-
-
-
-@implementer(interfaces.ITransport)
-class _ProcessEndpointTransport(proxyForInterface(
-                                interfaces.IProcessTransport, '_process')):
-    """
-    An L{ITransport} provider for the L{IProtocol} instance passed to the
-    process endpoint.
-
-    @ivar _process: An active process transport which will be used by write
-        methods on this object to write data to a child process.
-    @type _process: L{interfaces.IProcessTransport} provider
-    """
-
-    def write(self, data):
-        """
-        Write to the child process's standard input.
-
-        @param data: The data to write on stdin.
-        """
-        self._process.writeToChild(0, data)
-
-
-    def writeSequence(self, data):
-        """
-        Write a list of strings to child process's stdin.
-
-        @param data: The list of chunks to write on stdin.
-        """
-        for chunk in data:
-            self._process.writeToChild(0, chunk)
-
-
-@implementer(interfaces.IStreamServerEndpoint)
-class _TCPServerEndpoint(object):
-    """
-    A TCP server endpoint interface
-    """
-
-    def __init__(self, reactor, port, backlog, interface):
-        """
-        @param reactor: An L{IReactorTCP} provider.
-
-        @param port: The port number used for listening
-        @type port: int
-
-        @param backlog: Size of the listen queue
-        @type backlog: int
-
-        @param interface: The hostname to bind to
-        @type interface: str
-        """
-        self._reactor = reactor
-        self._port = port
-        self._backlog = backlog
-        self._interface = interface
-
-
-    def listen(self, protocolFactory):
-        """
-        Implement L{IStreamServerEndpoint.listen} to listen on a TCP
-        socket
-        """
-        return defer.execute(self._reactor.listenTCP,
-                             self._port,
-                             protocolFactory,
-                             backlog=self._backlog,
-                             interface=self._interface)
-
-
-
-class TCP4ServerEndpoint(_TCPServerEndpoint):
-    """
-    Implements TCP server endpoint with an IPv4 configuration
-    """
-    def __init__(self, reactor, port, backlog=50, interface=''):
-        """
-        @param reactor: An L{IReactorTCP} provider.
-
-        @param port: The port number used for listening
-        @type port: int
-
-        @param backlog: Size of the listen queue
-        @type backlog: int
-
-        @param interface: The hostname to bind to, defaults to '' (all)
-        @type interface: str
-        """
-        _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface)
-
-
-
-class TCP6ServerEndpoint(_TCPServerEndpoint):
-    """
-    Implements TCP server endpoint with an IPv6 configuration
-    """
-    def __init__(self, reactor, port, backlog=50, interface='::'):
-        """
-        @param reactor: An L{IReactorTCP} provider.
-
-        @param port: The port number used for listening
-        @type port: int
-
-        @param backlog: Size of the listen queue
-        @type backlog: int
-
-        @param interface: The hostname to bind to, defaults to '' (all)
-        @type interface: str
-        """
-        _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface)
-
-
-
-@implementer(interfaces.IStreamClientEndpoint)
-class TCP4ClientEndpoint(object):
-    """
-    TCP client endpoint with an IPv4 configuration.
-    """
-
-    def __init__(self, reactor, host, port, timeout=30, bindAddress=None):
-        """
-        @param reactor: An L{IReactorTCP} provider
-
-        @param host: A hostname, used when connecting
-        @type host: str
-
-        @param port: The port number, used when connecting
-        @type port: int
-
-        @param timeout: The number of seconds to wait before assuming the
-            connection has failed.
-        @type timeout: int
-
-        @param bindAddress: A (host, port) tuple of local address to bind to,
-            or None.
-        @type bindAddress: tuple
-        """
-        self._reactor = reactor
-        self._host = host
-        self._port = port
-        self._timeout = timeout
-        self._bindAddress = bindAddress
-
-
-    def connect(self, protocolFactory):
-        """
-        Implement L{IStreamClientEndpoint.connect} to connect via TCP.
-        """
-        try:
-            wf = _WrappingFactory(protocolFactory)
-            self._reactor.connectTCP(
-                self._host, self._port, wf,
-                timeout=self._timeout, bindAddress=self._bindAddress)
-            return wf._onConnection
-        except:
-            return defer.fail()
-
-
-
-
-@implementer(interfaces.IStreamServerEndpoint)
-class SSL4ServerEndpoint(object):
-    """
-    SSL secured TCP server endpoint with an IPv4 configuration.
-    """
-
-    def __init__(self, reactor, port, sslContextFactory,
-                 backlog=50, interface=''):
-        """
-        @param reactor: An L{IReactorSSL} provider.
-
-        @param port: The port number used for listening
-        @type port: int
-
-        @param sslContextFactory: An instance of
-            L{twisted.internet.ssl.ContextFactory}.
-
-        @param backlog: Size of the listen queue
-        @type backlog: int
-
-        @param interface: The hostname to bind to, defaults to '' (all)
-        @type interface: str
-        """
-        self._reactor = reactor
-        self._port = port
-        self._sslContextFactory = sslContextFactory
-        self._backlog = backlog
-        self._interface = interface
-
-
-    def listen(self, protocolFactory):
-        """
-        Implement L{IStreamServerEndpoint.listen} to listen for SSL on a
-        TCP socket.
-        """
-        return defer.execute(self._reactor.listenSSL, self._port,
-                             protocolFactory,
-                             contextFactory=self._sslContextFactory,
-                             backlog=self._backlog,
-                             interface=self._interface)
-
-
-
-@implementer(interfaces.IStreamClientEndpoint)
-class SSL4ClientEndpoint(object):
-    """
-    SSL secured TCP client endpoint with an IPv4 configuration
-    """
-
-    def __init__(self, reactor, host, port, sslContextFactory,
-                 timeout=30, bindAddress=None):
-        """
-        @param reactor: An L{IReactorSSL} provider.
-
-        @param host: A hostname, used when connecting
-        @type host: str
-
-        @param port: The port number, used when connecting
-        @type port: int
-
-        @param sslContextFactory: SSL Configuration information as an instance
-            of L{twisted.internet.ssl.ContextFactory}.
-
-        @param timeout: Number of seconds to wait before assuming the
-            connection has failed.
-        @type timeout: int
-
-        @param bindAddress: A (host, port) tuple of local address to bind to,
-            or None.
-        @type bindAddress: tuple
-        """
-        self._reactor = reactor
-        self._host = host
-        self._port = port
-        self._sslContextFactory = sslContextFactory
-        self._timeout = timeout
-        self._bindAddress = bindAddress
-
-
-    def connect(self, protocolFactory):
-        """
-        Implement L{IStreamClientEndpoint.connect} to connect with SSL over
-        TCP.
-        """
-        try:
-            wf = _WrappingFactory(protocolFactory)
-            self._reactor.connectSSL(
-                self._host, self._port, wf, self._sslContextFactory,
-                timeout=self._timeout, bindAddress=self._bindAddress)
-            return wf._onConnection
-        except:
-            return defer.fail()
-
-
-
-@implementer(interfaces.IStreamServerEndpoint)
-class UNIXServerEndpoint(object):
-    """
-    UnixSocket server endpoint.
-    """
-    def __init__(self, reactor, address, backlog=50, mode=0o666, wantPID=0):
-        """
-        @param reactor: An L{IReactorUNIX} provider.
-        @param address: The path to the Unix socket file, used when listening
-        @param backlog: number of connections to allow in backlog.
-        @param mode: mode to set on the unix socket.  This parameter is
-            deprecated.  Permissions should be set on the directory which
-            contains the UNIX socket.
-        @param wantPID: If True, create a pidfile for the socket.
-        """
-        self._reactor = reactor
-        self._address = address
-        self._backlog = backlog
-        self._mode = mode
-        self._wantPID = wantPID
-
-
-    def listen(self, protocolFactory):
-        """
-        Implement L{IStreamServerEndpoint.listen} to listen on a UNIX socket.
-        """
-        return defer.execute(self._reactor.listenUNIX, self._address,
-                             protocolFactory,
-                             backlog=self._backlog,
-                             mode=self._mode,
-                             wantPID=self._wantPID)
-
-
-
-@implementer(interfaces.IStreamClientEndpoint)
-class UNIXClientEndpoint(object):
-    """
-    UnixSocket client endpoint.
-    """
-    def __init__(self, reactor, path, timeout=30, checkPID=0):
-        """
-        @param reactor: An L{IReactorUNIX} provider.
-
-        @param path: The path to the Unix socket file, used when connecting
-        @type path: str
-
-        @param timeout: Number of seconds to wait before assuming the
-            connection has failed.
-        @type timeout: int
-
-        @param checkPID: If True, check for a pid file to verify that a server
-            is listening.
-        @type checkPID: bool
-        """
-        self._reactor = reactor
-        self._path = path
-        self._timeout = timeout
-        self._checkPID = checkPID
-
-
-    def connect(self, protocolFactory):
-        """
-        Implement L{IStreamClientEndpoint.connect} to connect via a
-        UNIX Socket
-        """
-        try:
-            wf = _WrappingFactory(protocolFactory)
-            self._reactor.connectUNIX(
-                self._path, wf,
-                timeout=self._timeout,
-                checkPID=self._checkPID)
-            return wf._onConnection
-        except:
-            return defer.fail()
-
-
-
-@implementer(interfaces.IStreamServerEndpoint)
-class AdoptedStreamServerEndpoint(object):
-    """
-    An endpoint for listening on a file descriptor initialized outside of
-    Twisted.
-
-    @ivar _used: A C{bool} indicating whether this endpoint has been used to
-        listen with a factory yet.  C{True} if so.
-    """
-    _close = os.close
-    _setNonBlocking = staticmethod(fdesc.setNonBlocking)
-
-    def __init__(self, reactor, fileno, addressFamily):
-        """
-        @param reactor: An L{IReactorSocket} provider.
-
-        @param fileno: An integer file descriptor corresponding to a listening
-            I{SOCK_STREAM} socket.
-
-        @param addressFamily: The address family of the socket given by
-            C{fileno}.
-        """
-        self.reactor = reactor
-        self.fileno = fileno
-        self.addressFamily = addressFamily
-        self._used = False
-
-
-    def listen(self, factory):
-        """
-        Implement L{IStreamServerEndpoint.listen} to start listening on, and
-        then close, C{self._fileno}.
-        """
-        if self._used:
-            return defer.fail(error.AlreadyListened())
-        self._used = True
-
-        try:
-            self._setNonBlocking(self.fileno)
-            port = self.reactor.adoptStreamPort(
-                self.fileno, self.addressFamily, factory)
-            self._close(self.fileno)
-        except:
-            return defer.fail()
-        return defer.succeed(port)
-
-
-
-
-def _parseTCP(factory, port, interface="", backlog=50):
-    """
-    Internal parser function for L{_parseServer} to convert the string
-    arguments for a TCP(IPv4) stream endpoint into the structured arguments.
-
-    @param factory: the protocol factory being parsed, or C{None}.  (This was a
-        leftover argument from when this code was in C{strports}, and is now
-        mostly None and unused.)
-
-    @type factory: L{IProtocolFactory} or C{NoneType}
-
-    @param port: the integer port number to bind
-    @type port: C{str}
-
-    @param interface: the interface IP to listen on
-    @param backlog: the length of the listen queue
-    @type backlog: C{str}
-
-    @return: a 2-tuple of (args, kwargs), describing  the parameters to
-        L{IReactorTCP.listenTCP} (or, modulo argument 2, the factory, arguments
-        to L{TCP4ServerEndpoint}.
-    """
-    return (int(port), factory), {'interface': interface,
-                                  'backlog': int(backlog)}
-
-
-
-def _parseUNIX(factory, address, mode='666', backlog=50, lockfile=True):
-    """
-    Internal parser function for L{_parseServer} to convert the string
-    arguments for a UNIX (AF_UNIX/SOCK_STREAM) stream endpoint into the
-    structured arguments.
-
-    @param factory: the protocol factory being parsed, or C{None}.  (This was a
-        leftover argument from when this code was in C{strports}, and is now
-        mostly None and unused.)
-
-    @type factory: L{IProtocolFactory} or C{NoneType}
-
-    @param address: the pathname of the unix socket
-    @type address: C{str}
-
-    @param backlog: the length of the listen queue
-    @type backlog: C{str}
-
-    @param lockfile: A string '0' or '1', mapping to True and False
-        respectively.  See the C{wantPID} argument to C{listenUNIX}
-
-    @return: a 2-tuple of (args, kwargs), describing  the parameters to
-        L{IReactorTCP.listenUNIX} (or, modulo argument 2, the factory,
-        arguments to L{UNIXServerEndpoint}.
-    """
-    return (
-        (address, factory),
-        {'mode': int(mode, 8), 'backlog': int(backlog),
-         'wantPID': bool(int(lockfile))})
-
-
-
-def _parseSSL(factory, port, privateKey="server.pem", certKey=None,
-              sslmethod=None, interface='', backlog=50):
-    """
-    Internal parser function for L{_parseServer} to convert the string
-    arguments for an SSL (over TCP/IPv4) stream endpoint into the structured
-    arguments.
-
-    @param factory: the protocol factory being parsed, or C{None}.  (This was a
-        leftover argument from when this code was in C{strports}, and is now
-        mostly None and unused.)
-    @type factory: L{IProtocolFactory} or C{NoneType}
-
-    @param port: the integer port number to bind
-    @type port: C{str}
-
-    @param interface: the interface IP to listen on
-    @param backlog: the length of the listen queue
-    @type backlog: C{str}
-
-    @param privateKey: The file name of a PEM format private key file.
-    @type privateKey: C{str}
-
-    @param certKey: The file name of a PEM format certificate file.
-    @type certKey: C{str}
-
-    @param sslmethod: The string name of an SSL method, based on the name of a
-        constant in C{OpenSSL.SSL}.  Must be one of: "SSLv23_METHOD",
-        "SSLv2_METHOD", "SSLv3_METHOD", "TLSv1_METHOD".
-    @type sslmethod: C{str}
-
-    @return: a 2-tuple of (args, kwargs), describing  the parameters to
-        L{IReactorSSL.listenSSL} (or, modulo argument 2, the factory, arguments
-        to L{SSL4ServerEndpoint}.
-    """
-    from twisted.internet import ssl
-    if certKey is None:
-        certKey = privateKey
-    kw = {}
-    if sslmethod is not None:
-        kw['method'] = getattr(ssl.SSL, sslmethod)
-    else:
-        kw['method'] = ssl.SSL.SSLv23_METHOD
-    certPEM = FilePath(certKey).getContent()
-    keyPEM = FilePath(privateKey).getContent()
-    privateCertificate = ssl.PrivateCertificate.loadPEM(certPEM + keyPEM)
-    cf = ssl.CertificateOptions(
-        privateKey=privateCertificate.privateKey.original,
-        certificate=privateCertificate.original,
-        **kw
-    )
-    return ((int(port), factory, cf),
-            {'interface': interface, 'backlog': int(backlog)})
-
-
-
-@implementer(IPlugin, IStreamServerEndpointStringParser)
-class _StandardIOParser(object):
-    """
-    Stream server endpoint string parser for the Standard I/O type.
-
-    @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}.
-    """
-    prefix = "stdio"
-
-    def _parseServer(self, reactor):
-        """
-        Internal parser function for L{_parseServer} to convert the string
-        arguments into structured arguments for the L{StandardIOEndpoint}
-
-        @param reactor: Reactor for the endpoint
-        """
-        return StandardIOEndpoint(reactor)
-
-
-    def parseStreamServer(self, reactor, *args, **kwargs):
-        # Redirects to another function (self._parseServer), tricks zope.interface
-        # into believing the interface is correctly implemented.
-        return self._parseServer(reactor)
-
-
-
-
-@implementer(IPlugin, IStreamServerEndpointStringParser)
-class _TCP6ServerParser(object):
-    """
-    Stream server endpoint string parser for the TCP6ServerEndpoint type.
-
-    @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}.
-    """
-    prefix = "tcp6"     # Used in _parseServer to identify the plugin with the endpoint type
-
-    def _parseServer(self, reactor, port, backlog=50, interface='::'):
-        """
-        Internal parser function for L{_parseServer} to convert the string
-        arguments into structured arguments for the L{TCP6ServerEndpoint}
-
-        @param reactor: An L{IReactorTCP} provider.
-
-        @param port: The port number used for listening
-        @type port: int
-
-        @param backlog: Size of the listen queue
-        @type backlog: int
-
-        @param interface: The hostname to bind to
-        @type interface: str
-        """
-        port = int(port)
-        backlog = int(backlog)
-        return TCP6ServerEndpoint(reactor, port, backlog, interface)
-
-
-    def parseStreamServer(self, reactor, *args, **kwargs):
-        # Redirects to another function (self._parseServer), tricks zope.interface
-        # into believing the interface is correctly implemented.
-        return self._parseServer(reactor, *args, **kwargs)
-
-
-
-_serverParsers = {"tcp": _parseTCP,
-                  "unix": _parseUNIX,
-                  "ssl": _parseSSL,
-                  }
-
-_OP, _STRING = range(2)
-
-def _tokenize(description):
-    """
-    Tokenize a strports string and yield each token.
-
-    @param description: a string as described by L{serverFromString} or
-        L{clientFromString}.
-
-    @return: an iterable of 2-tuples of (L{_OP} or L{_STRING}, string).  Tuples
-        starting with L{_OP} will contain a second element of either ':' (i.e.
-        'next parameter') or '=' (i.e. 'assign parameter value').  For example,
-        the string 'hello:greet\=ing=world' would result in a generator
-        yielding these values::
-
-            _STRING, 'hello'
-            _OP, ':'
-            _STRING, 'greet=ing'
-            _OP, '='
-            _STRING, 'world'
-    """
-    current = ''
-    ops = ':='
-    nextOps = {':': ':=', '=': ':'}
-    description = iter(description)
-    for n in description:
-        if n in ops:
-            yield _STRING, current
-            yield _OP, n
-            current = ''
-            ops = nextOps[n]
-        elif n == '\\':
-            current += next(description)
-        else:
-            current += n
-    yield _STRING, current
-
-
-
-def _parse(description):
-    """
-    Convert a description string into a list of positional and keyword
-    parameters, using logic vaguely like what Python does.
-
-    @param description: a string as described by L{serverFromString} or
-        L{clientFromString}.
-
-    @return: a 2-tuple of C{(args, kwargs)}, where 'args' is a list of all
-        ':'-separated C{str}s not containing an '=' and 'kwargs' is a map of
-        all C{str}s which do contain an '='.  For example, the result of
-        C{_parse('a:b:d=1:c')} would be C{(['a', 'b', 'c'], {'d': '1'})}.
-    """
-    args, kw = [], {}
-    def add(sofar):
-        if len(sofar) == 1:
-            args.append(sofar[0])
-        else:
-            kw[sofar[0]] = sofar[1]
-    sofar = ()
-    for (type, value) in _tokenize(description):
-        if type is _STRING:
-            sofar += (value,)
-        elif value == ':':
-            add(sofar)
-            sofar = ()
-    add(sofar)
-    return args, kw
-
-
-# Mappings from description "names" to endpoint constructors.
-_endpointServerFactories = {
-    'TCP': TCP4ServerEndpoint,
-    'SSL': SSL4ServerEndpoint,
-    'UNIX': UNIXServerEndpoint,
-    }
-
-_endpointClientFactories = {
-    'TCP': TCP4ClientEndpoint,
-    'SSL': SSL4ClientEndpoint,
-    'UNIX': UNIXClientEndpoint,
-    }
-
-
-_NO_DEFAULT = object()
-
-def _parseServer(description, factory, default=None):
-    """
-    Parse a stports description into a 2-tuple of arguments and keyword values.
-
-    @param description: A description in the format explained by
-        L{serverFromString}.
-    @type description: C{str}
-
-    @param factory: A 'factory' argument; this is left-over from
-        twisted.application.strports, it's not really used.
-    @type factory: L{IProtocolFactory} or L{None}
-
-    @param default: Deprecated argument, specifying the default parser mode to
-        use for unqualified description strings (those which do not have a ':'
-        and prefix).
-    @type default: C{str} or C{NoneType}
-
-    @return: a 3-tuple of (plugin or name, arguments, keyword arguments)
-    """
-    args, kw = _parse(description)
-    if not args or (len(args) == 1 and not kw):
-        deprecationMessage = (
-            "Unqualified strport description passed to 'service'."
-            "Use qualified endpoint descriptions; for example, 'tcp:%s'."
-            % (description,))
-        if default is None:
-            default = 'tcp'
-            warnings.warn(
-                deprecationMessage, category=DeprecationWarning, stacklevel=4)
-        elif default is _NO_DEFAULT:
-            raise ValueError(deprecationMessage)
-        # If the default has been otherwise specified, the user has already
-        # been warned.
-        args[0:0] = [default]
-    endpointType = args[0]
-    parser = _serverParsers.get(endpointType)
-    if parser is None:
-        # If the required parser is not found in _server, check if
-        # a plugin exists for the endpointType
-        for plugin in getPlugins(IStreamServerEndpointStringParser):
-            if plugin.prefix == endpointType:
-                return (plugin, args[1:], kw)
-        raise ValueError("Unknown endpoint type: '%s'" % (endpointType,))
-    return (endpointType.upper(),) + parser(factory, *args[1:], **kw)
-
-
-
-def _serverFromStringLegacy(reactor, description, default):
-    """
-    Underlying implementation of L{serverFromString} which avoids exposing the
-    deprecated 'default' argument to anything but L{strports.service}.
-    """
-    nameOrPlugin, args, kw = _parseServer(description, None, default)
-    if type(nameOrPlugin) is not str:
-        plugin = nameOrPlugin
-        return plugin.parseStreamServer(reactor, *args, **kw)
-    else:
-        name = nameOrPlugin
-    # Chop out the factory.
-    args = args[:1] + args[2:]
-    return _endpointServerFactories[name](reactor, *args, **kw)
-
-
-
-def serverFromString(reactor, description):
-    """
-    Construct a stream server endpoint from an endpoint description string.
-
-    The format for server endpoint descriptions is a simple string.  It is a
-    prefix naming the type of endpoint, then a colon, then the arguments for
-    that endpoint.
-
-    For example, you can call it like this to create an endpoint that will
-    listen on TCP port 80::
-
-        serverFromString(reactor, "tcp:80")
-
-    Additional arguments may be specified as keywords, separated with colons.
-    For example, you can specify the interface for a TCP server endpoint to
-    bind to like this::
-
-        serverFromString(reactor, "tcp:80:interface=127.0.0.1")
-
-    SSL server endpoints may be specified with the 'ssl' prefix, and the
-    private key and certificate files may be specified by the C{privateKey} and
-    C{certKey} arguments::
-
-        serverFromString(reactor, "ssl:443:privateKey=key.pem:certKey=crt.pem")
-
-    If a private key file name (C{privateKey}) isn't provided, a "server.pem"
-    file is assumed to exist which contains the private key. If the certificate
-    file name (C{certKey}) isn't provided, the private key file is assumed to
-    contain the certificate as well.
-
-    You may escape colons in arguments with a backslash, which you will need to
-    use if you want to specify a full pathname argument on Windows::
-
-        serverFromString(reactor,
-            "ssl:443:privateKey=C\\:/key.pem:certKey=C\\:/cert.pem")
-
-    finally, the 'unix' prefix may be used to specify a filesystem UNIX socket,
-    optionally with a 'mode' argument to specify the mode of the socket file
-    created by C{listen}::
-
-        serverFromString(reactor, "unix:/var/run/finger")
-        serverFromString(reactor, "unix:/var/run/finger:mode=660")
-
-    This function is also extensible; new endpoint types may be registered as
-    L{IStreamServerEndpointStringParser} plugins.  See that interface for more
-    information.
-
-    @param reactor: The server endpoint will be constructed with this reactor.
-
-    @param description: The strports description to parse.
-
-    @return: A new endpoint which can be used to listen with the parameters
-        given by by C{description}.
-
-    @rtype: L{IStreamServerEndpoint<twisted.internet.interfaces.IStreamServerEndpoint>}
-
-    @raise ValueError: when the 'description' string cannot be parsed.
-
-    @since: 10.2
-    """
-    return _serverFromStringLegacy(reactor, description, _NO_DEFAULT)
-
-
-
-def quoteStringArgument(argument):
-    """
-    Quote an argument to L{serverFromString} and L{clientFromString}.  Since
-    arguments are separated with colons and colons are escaped with
-    backslashes, some care is necessary if, for example, you have a pathname,
-    you may be tempted to interpolate into a string like this::
-
-        serverFromString("ssl:443:privateKey=%s" % (myPathName,))
-
-    This may appear to work, but will have portability issues (Windows
-    pathnames, for example).  Usually you should just construct the appropriate
-    endpoint type rather than interpolating strings, which in this case would
-    be L{SSL4ServerEndpoint}.  There are some use-cases where you may need to
-    generate such a string, though; for example, a tool to manipulate a
-    configuration file which has strports descriptions in it.  To be correct in
-    those cases, do this instead::
-
-        serverFromString("ssl:443:privateKey=%s" %
-                         (quoteStringArgument(myPathName),))
-
-    @param argument: The part of the endpoint description string you want to
-        pass through.
-
-    @type argument: C{str}
-
-    @return: The quoted argument.
-
-    @rtype: C{str}
-    """
-    return argument.replace('\\', '\\\\').replace(':', '\\:')
-
-
-
-def _parseClientTCP(*args, **kwargs):
-    """
-    Perform any argument value coercion necessary for TCP client parameters.
-
-    Valid positional arguments to this function are host and port.
-
-    Valid keyword arguments to this function are all L{IReactorTCP.connectTCP}
-    arguments.
-
-    @return: The coerced values as a C{dict}.
-    """
-
-    if len(args) == 2:
-        kwargs['port'] = int(args[1])
-        kwargs['host'] = args[0]
-    elif len(args) == 1:
-        if 'host' in kwargs:
-            kwargs['port'] = int(args[0])
-        else:
-            kwargs['host'] = args[0]
-
-    try:
-        kwargs['port'] = int(kwargs['port'])
-    except KeyError:
-        pass
-
-    try:
-        kwargs['timeout'] = int(kwargs['timeout'])
-    except KeyError:
-        pass
-    return kwargs
-
-
-
-def _loadCAsFromDir(directoryPath):
-    """
-    Load certificate-authority certificate objects in a given directory.
-
-    @param directoryPath: a L{FilePath} pointing at a directory to load .pem
-        files from.
-
-    @return: a C{list} of L{OpenSSL.crypto.X509} objects.
-    """
-    from twisted.internet import ssl
-
-    caCerts = {}
-    for child in directoryPath.children():
-        if not child.basename().split('.')[-1].lower() == 'pem':
-            continue
-        try:
-            data = child.getContent()
-        except IOError:
-            # Permission denied, corrupt disk, we don't care.
-            continue
-        try:
-            theCert = ssl.Certificate.loadPEM(data)
-        except ssl.SSL.Error:
-            # Duplicate certificate, invalid certificate, etc.  We don't care.
-            pass
-        else:
-            caCerts[theCert.digest()] = theCert.original
-    return caCerts.values()
-
-
-
-def _parseClientSSL(*args, **kwargs):
-    """
-    Perform any argument value coercion necessary for SSL client parameters.
-
-    Valid keyword arguments to this function are all L{IReactorSSL.connectSSL}
-    arguments except for C{contextFactory}.  Instead, C{certKey} (the path name
-    of the certificate file) C{privateKey} (the path name of the private key
-    associated with the certificate) are accepted and used to construct a
-    context factory.
-
-    Valid positional arguments to this function are host and port.
-
-    @param caCertsDir: The one parameter which is not part of
-        L{IReactorSSL.connectSSL}'s signature, this is a path name used to
-        construct a list of certificate authority certificates.  The directory
-        will be scanned for files ending in C{.pem}, all of which will be
-        considered valid certificate authorities for this connection.
-
-    @type caCertsDir: C{str}
-
-    @return: The coerced values as a C{dict}.
-    """
-    from twisted.internet import ssl
-    kwargs = _parseClientTCP(*args, **kwargs)
-    certKey = kwargs.pop('certKey', None)
-    privateKey = kwargs.pop('privateKey', None)
-    caCertsDir = kwargs.pop('caCertsDir', None)
-    if certKey is not None:
-        certx509 = ssl.Certificate.loadPEM(
-            FilePath(certKey).getContent()).original
-    else:
-        certx509 = None
-    if privateKey is not None:
-        privateKey = ssl.PrivateCertificate.loadPEM(
-            FilePath(privateKey).getContent()).privateKey.original
-    else:
-        privateKey = None
-    if caCertsDir is not None:
-        verify = True
-        caCerts = _loadCAsFromDir(FilePath(caCertsDir))
-    else:
-        verify = False
-        caCerts = None
-    kwargs['sslContextFactory'] = ssl.CertificateOptions(
-        method=ssl.SSL.SSLv23_METHOD,
-        certificate=certx509,
-        privateKey=privateKey,
-        verify=verify,
-        caCerts=caCerts
-    )
-    return kwargs
-
-
-
-def _parseClientUNIX(*args, **kwargs):
-    """
-    Perform any argument value coercion necessary for UNIX client parameters.
-
-    Valid keyword arguments to this function are all L{IReactorUNIX.connectUNIX}
-    keyword arguments except for C{checkPID}.  Instead, C{lockfile} is accepted
-    and has the same meaning.  Also C{path} is used instead of C{address}.
-
-    Valid positional arguments to this function are C{path}.
-
-    @return: The coerced values as a C{dict}.
-    """
-    if len(args) == 1:
-        kwargs['path'] = args[0]
-
-    try:
-        kwargs['checkPID'] = bool(int(kwargs.pop('lockfile')))
-    except KeyError:
-        pass
-    try:
-        kwargs['timeout'] = int(kwargs['timeout'])
-    except KeyError:
-        pass
-    return kwargs
-
-_clientParsers = {
-    'TCP': _parseClientTCP,
-    'SSL': _parseClientSSL,
-    'UNIX': _parseClientUNIX,
-    }
-
-
-
-def clientFromString(reactor, description):
-    """
-    Construct a client endpoint from a description string.
-
-    Client description strings are much like server description strings,
-    although they take all of their arguments as keywords, aside from host and
-    port.
-
-    You can create a TCP client endpoint with the 'host' and 'port' arguments,
-    like so::
-
-        clientFromString(reactor, "tcp:host=www.example.com:port=80")
-
-    or, without specifying host and port keywords::
-
-        clientFromString(reactor, "tcp:www.example.com:80")
-
-    Or you can specify only one or the other, as in the following 2 examples::
-
-        clientFromString(reactor, "tcp:host=www.example.com:80")
-        clientFromString(reactor, "tcp:www.example.com:port=80")
-
-    or an SSL client endpoint with those arguments, plus the arguments used by
-    the server SSL, for a client certificate::
-
-        clientFromString(reactor, "ssl:web.example.com:443:"
-                                  "privateKey=foo.pem:certKey=foo.pem")
-
-    to specify your certificate trust roots, you can identify a directory with
-    PEM files in it with the C{caCertsDir} argument::
-
-        clientFromString(reactor, "ssl:host=web.example.com:port=443:"
-                                  "caCertsDir=/etc/ssl/certs")
-
-    You can create a UNIX client endpoint with the 'path' argument and optional
-    'lockfile' and 'timeout' arguments::
-
-        clientFromString(reactor, "unix:path=/var/foo/bar:lockfile=1:timeout=9")
-
-    or, with the path as a positional argument with or without optional
-    arguments as in the following 2 examples::
-
-        clientFromString(reactor, "unix:/var/foo/bar")
-        clientFromString(reactor, "unix:/var/foo/bar:lockfile=1:timeout=9")
-
-    This function is also extensible; new endpoint types may be registered as
-    L{IStreamClientEndpointStringParser} plugins.  See that interface for more
-    information.
-
-    @param reactor: The client endpoint will be constructed with this reactor.
-
-    @param description: The strports description to parse.
-
-    @return: A new endpoint which can be used to connect with the parameters
-        given by by C{description}.
-    @rtype: L{IStreamClientEndpoint<twisted.internet.interfaces.IStreamClientEndpoint>}
-
-    @since: 10.2
-    """
-    args, kwargs = _parse(description)
-    aname = args.pop(0)
-    name = aname.upper()
-    for plugin in getPlugins(IStreamClientEndpointStringParser):
-        if plugin.prefix.upper() == name:
-            return plugin.parseStreamClient(*args, **kwargs)
-    if name not in _clientParsers:
-        raise ValueError("Unknown endpoint type: %r" % (aname,))
-    kwargs = _clientParsers[name](*args, **kwargs)
-    return _endpointClientFactories[name](reactor, **kwargs)
-
-
-
-def connectProtocol(endpoint, protocol):
-    """
-    Connect a protocol instance to an endpoint.
-
-    This allows using a client endpoint without having to create a factory.
-
-    @param endpoint: A client endpoint to connect to.
-
-    @param protocol: A protocol instance.
-
-    @return: The result of calling C{connect} on the endpoint, i.e. a
-    L{Deferred} that will fire with the protocol when connected, or an
-    appropriate error.
-    """
-    class OneShotFactory(Factory):
-        def buildProtocol(self, addr):
-            return protocol
-    return endpoint.connect(OneShotFactory())
-}}} '''
diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py
index 7b2a78632..fdcbf3977 100644
--- a/scrapy/xlib/tx/interfaces.py
+++ b/scrapy/xlib/tx/interfaces.py
@@ -9,8 +9,6 @@ Maintainer: Itamar Shtull-Trauring
 
 from __future__ import division, absolute_import
 
-from zope.interface import Interface, Attribute
-
 from twisted.internet.interfaces import (
     IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL,
     IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess,
@@ -28,2434 +26,3 @@ from twisted.internet.interfaces import (
     IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,
     IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver
 )
-
-''' {{{
-class IAddress(Interface):
-    """
-    An address, e.g. a TCP C{(host, port)}.
-
-    Default implementations are in L{twisted.internet.address}.
-    """
-
-### Reactor Interfaces
-
-class IConnector(Interface):
-    """
-    Object used to interface between connections and protocols.
-
-    Each L{IConnector} manages one connection.
-    """
-
-    def stopConnecting():
-        """
-        Stop attempting to connect.
-        """
-
-    def disconnect():
-        """
-        Disconnect regardless of the connection state.
-
-        If we are connected, disconnect, if we are trying to connect,
-        stop trying.
-        """
-
-    def connect():
-        """
-        Try to connect to remote address.
-        """
-
-    def getDestination():
-        """
-        Return destination this will try to connect to.
-
-        @return: An object which provides L{IAddress}.
-        """
-
-
-
-class IResolverSimple(Interface):
-    def getHostByName(name, timeout = (1, 3, 11, 45)):
-        """
-        Resolve the domain name C{name} into an IP address.
-
-        @type name: C{str}
-        @type timeout: C{tuple}
-        @rtype: L{twisted.internet.defer.Deferred}
-        @return: The callback of the Deferred that is returned will be
-        passed a string that represents the IP address of the specified
-        name, or the errback will be called if the lookup times out.  If
-        multiple types of address records are associated with the name,
-        A6 records will be returned in preference to AAAA records, which
-        will be returned in preference to A records.  If there are multiple
-        records of the type to be returned, one will be selected at random.
-
-        @raise twisted.internet.defer.TimeoutError: Raised (asynchronously)
-        if the name cannot be resolved within the specified timeout period.
-        """
-
-
-
-class IResolver(IResolverSimple):
-    def query(query, timeout=None):
-        """
-        Dispatch C{query} to the method which can handle its type.
-
-        @type query: L{twisted.names.dns.Query}
-        @param query: The DNS query being issued, to which a response is to be
-            generated.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupAddress(name, timeout=None):
-        """
-        Perform an A record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupAddress6(name, timeout=None):
-        """
-        Perform an A6 record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupIPV6Address(name, timeout=None):
-        """
-        Perform an AAAA record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupMailExchange(name, timeout=None):
-        """
-        Perform an MX record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupNameservers(name, timeout=None):
-        """
-        Perform an NS record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupCanonicalName(name, timeout=None):
-        """
-        Perform a CNAME record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupMailBox(name, timeout=None):
-        """
-        Perform an MB record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupMailGroup(name, timeout=None):
-        """
-        Perform an MG record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupMailRename(name, timeout=None):
-        """
-        Perform an MR record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupPointer(name, timeout=None):
-        """
-        Perform a PTR record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupAuthority(name, timeout=None):
-        """
-        Perform an SOA record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupNull(name, timeout=None):
-        """
-        Perform a NULL record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupWellKnownServices(name, timeout=None):
-        """
-        Perform a WKS record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupHostInfo(name, timeout=None):
-        """
-        Perform a HINFO record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupMailboxInfo(name, timeout=None):
-        """
-        Perform an MINFO record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupText(name, timeout=None):
-        """
-        Perform a TXT record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupResponsibility(name, timeout=None):
-        """
-        Perform an RP record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupAFSDatabase(name, timeout=None):
-        """
-        Perform an AFSDB record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupService(name, timeout=None):
-        """
-        Perform an SRV record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupAllRecords(name, timeout=None):
-        """
-        Perform an ALL_RECORD lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupSenderPolicy(name, timeout= 10):
-        """
-        Perform a SPF record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupNamingAuthorityPointer(name, timeout=None):
-        """
-        Perform a NAPTR record lookup.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: Sequence of C{int}
-        @param timeout: Number of seconds after which to reissue the query.
-            When the last timeout expires, the query is considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.  The first element of the
-            tuple gives answers.  The second element of the tuple gives
-            authorities.  The third element of the tuple gives additional
-            information.  The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-    def lookupZone(name, timeout=None):
-        """
-        Perform an AXFR record lookup.
-
-        NB This is quite different from other DNS requests. See
-        U{http://cr.yp.to/djbdns/axfr-notes.html} for more
-        information.
-
-        NB Unlike other C{lookup*} methods, the timeout here is not a
-        list of ints, it is a single int.
-
-        @type name: C{str}
-        @param name: DNS name to resolve.
-
-        @type timeout: C{int}
-        @param timeout: When this timeout expires, the query is
-            considered failed.
-
-        @rtype: L{Deferred}
-        @return: A L{Deferred} which fires with a three-tuple of lists of
-            L{twisted.names.dns.RRHeader} instances.
-            The first element of the tuple gives answers.
-            The second and third elements are always empty.
-            The L{Deferred} may instead fail with one of the
-            exceptions defined in L{twisted.names.error} or with
-            C{NotImplementedError}.
-        """
-
-
-
-class IReactorTCP(Interface):
-
-    def listenTCP(port, factory, backlog=50, interface=''):
-        """
-        Connects a given protocol factory to the given numeric TCP/IP port.
-
-        @param port: a port number on which to listen
-
-        @param factory: a L{twisted.internet.protocol.ServerFactory} instance
-
-        @param backlog: size of the listen queue
-
-        @param interface: The local IPv4 or IPv6 address to which to bind;
-            defaults to '', ie all IPv4 addresses.  To bind to all IPv4 and IPv6
-            addresses, you must call this method twice.
-
-        @return: an object that provides L{IListeningPort}.
-
-        @raise CannotListenError: as defined here
-                                  L{twisted.internet.error.CannotListenError},
-                                  if it cannot listen on this port (e.g., it
-                                  cannot bind to the required port number)
-        """
-
-    def connectTCP(host, port, factory, timeout=30, bindAddress=None):
-        """
-        Connect a TCP client.
-
-        @param host: a host name
-
-        @param port: a port number
-
-        @param factory: a L{twisted.internet.protocol.ClientFactory} instance
-
-        @param timeout: number of seconds to wait before assuming the
-                        connection has failed.
-
-        @param bindAddress: a (host, port) tuple of local address to bind
-                            to, or None.
-
-        @return: An object which provides L{IConnector}. This connector will
-                 call various callbacks on the factory when a connection is
-                 made, failed, or lost - see
-                 L{ClientFactory<twisted.internet.protocol.ClientFactory>}
-                 docs for details.
-        """
-
-class IReactorSSL(Interface):
-
-    def connectSSL(host, port, factory, contextFactory, timeout=30, bindAddress=None):
-        """
-        Connect a client Protocol to a remote SSL socket.
-
-        @param host: a host name
-
-        @param port: a port number
-
-        @param factory: a L{twisted.internet.protocol.ClientFactory} instance
-
-        @param contextFactory: a L{twisted.internet.ssl.ClientContextFactory} object.
-
-        @param timeout: number of seconds to wait before assuming the
-                        connection has failed.
-
-        @param bindAddress: a (host, port) tuple of local address to bind to,
-                            or C{None}.
-
-        @return: An object which provides L{IConnector}.
-        """
-
-    def listenSSL(port, factory, contextFactory, backlog=50, interface=''):
-        """
-        Connects a given protocol factory to the given numeric TCP/IP port.
-        The connection is a SSL one, using contexts created by the context
-        factory.
-
-        @param port: a port number on which to listen
-
-        @param factory: a L{twisted.internet.protocol.ServerFactory} instance
-
-        @param contextFactory: a L{twisted.internet.ssl.ContextFactory} instance
-
-        @param backlog: size of the listen queue
-
-        @param interface: the hostname to bind to, defaults to '' (all)
-        """
-
-
-
-class IReactorUNIX(Interface):
-    """
-    UNIX socket methods.
-    """
-
-    def connectUNIX(address, factory, timeout=30, checkPID=0):
-        """
-        Connect a client protocol to a UNIX socket.
-
-        @param address: a path to a unix socket on the filesystem.
-
-        @param factory: a L{twisted.internet.protocol.ClientFactory} instance
-
-        @param timeout: number of seconds to wait before assuming the connection
-            has failed.
-
-        @param checkPID: if True, check for a pid file to verify that a server
-            is listening.  If C{address} is a Linux abstract namespace path,
-            this must be C{False}.
-
-        @return: An object which provides L{IConnector}.
-        """
-
-
-    def listenUNIX(address, factory, backlog=50, mode=0o666, wantPID=0):
-        """
-        Listen on a UNIX socket.
-
-        @param address: a path to a unix socket on the filesystem.
-
-        @param factory: a L{twisted.internet.protocol.Factory} instance.
-
-        @param backlog: number of connections to allow in backlog.
-
-        @param mode: The mode (B{not} umask) to set on the unix socket.  See
-            platform specific documentation for information about how this
-            might affect connection attempts.
-        @type mode: C{int}
-
-        @param wantPID: if True, create a pidfile for the socket.  If C{address}
-            is a Linux abstract namespace path, this must be C{False}.
-
-        @return: An object which provides L{IListeningPort}.
-        """
-
-
-
-class IReactorUNIXDatagram(Interface):
-    """
-    Datagram UNIX socket methods.
-    """
-
-    def connectUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666, bindAddress=None):
-        """
-        Connect a client protocol to a datagram UNIX socket.
-
-        @param address: a path to a unix socket on the filesystem.
-
-        @param protocol: a L{twisted.internet.protocol.ConnectedDatagramProtocol} instance
-
-        @param maxPacketSize: maximum packet size to accept
-
-        @param mode: The mode (B{not} umask) to set on the unix socket.  See
-            platform specific documentation for information about how this
-            might affect connection attempts.
-        @type mode: C{int}
-
-        @param bindAddress: address to bind to
-
-        @return: An object which provides L{IConnector}.
-        """
-
-
-    def listenUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666):
-        """
-        Listen on a datagram UNIX socket.
-
-        @param address: a path to a unix socket on the filesystem.
-
-        @param protocol: a L{twisted.internet.protocol.DatagramProtocol} instance.
-
-        @param maxPacketSize: maximum packet size to accept
-
-        @param mode: The mode (B{not} umask) to set on the unix socket.  See
-            platform specific documentation for information about how this
-            might affect connection attempts.
-        @type mode: C{int}
-
-        @return: An object which provides L{IListeningPort}.
-        """
-
-
-
-class IReactorWin32Events(Interface):
-    """
-    Win32 Event API methods
-
-    @since: 10.2
-    """
-
-    def addEvent(event, fd, action):
-        """
-        Add a new win32 event to the event loop.
-
-        @param event: a Win32 event object created using win32event.CreateEvent()
-
-        @param fd: an instance of L{twisted.internet.abstract.FileDescriptor}
-
-        @param action: a string that is a method name of the fd instance.
-                       This method is called in response to the event.
-
-        @return: None
-        """
-
-
-    def removeEvent(event):
-        """
-        Remove an event.
-
-        @param event: a Win32 event object added using L{IReactorWin32Events.addEvent}
-
-        @return: None
-        """
-
-
-
-class IReactorUDP(Interface):
-    """
-    UDP socket methods.
-    """
-
-    def listenUDP(port, protocol, interface='', maxPacketSize=8192):
-        """
-        Connects a given DatagramProtocol to the given numeric UDP port.
-
-        @return: object which provides L{IListeningPort}.
-        """
-
-
-
-class IReactorMulticast(Interface):
-    """
-    UDP socket methods that support multicast.
-
-    IMPORTANT: This is an experimental new interface. It may change
-    without backwards compatibility. Suggestions are welcome.
-    """
-
-    def listenMulticast(port, protocol, interface='', maxPacketSize=8192,
-                        listenMultiple=False):
-        """
-        Connects a given
-        L{DatagramProtocol<twisted.internet.protocol.DatagramProtocol>} to the
-        given numeric UDP port.
-
-        @param listenMultiple: If set to True, allows multiple sockets to
-            bind to the same address and port number at the same time.
-        @type listenMultiple: C{bool}
-
-        @returns: An object which provides L{IListeningPort}.
-
-        @see: L{twisted.internet.interfaces.IMulticastTransport}
-        @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html}
-        """
-
-
-
-class IReactorSocket(Interface):
-    """
-    Methods which allow a reactor to use externally created sockets.
-
-    For example, to use C{adoptStreamPort} to implement behavior equivalent
-    to that of L{IReactorTCP.listenTCP}, you might write code like this::
-
-        from socket import SOMAXCONN, AF_INET, SOCK_STREAM, socket
-        portSocket = socket(AF_INET, SOCK_STREAM)
-        # Set FD_CLOEXEC on port, left as an exercise.  Then make it into a
-        # non-blocking listening port:
-        portSocket.setblocking(False)
-        portSocket.bind(('192.168.1.2', 12345))
-        portSocket.listen(SOMAXCONN)
-
-        # Now have the reactor use it as a TCP port
-        port = reactor.adoptStreamPort(
-            portSocket.fileno(), AF_INET, YourFactory())
-
-        # portSocket itself is no longer necessary, and needs to be cleaned
-        # up by us.
-        portSocket.close()
-
-        # Whenever the server is no longer needed, stop it as usual.
-        stoppedDeferred = port.stopListening()
-
-    Another potential use is to inherit a listening descriptor from a parent
-    process (for example, systemd or launchd), or to receive one over a UNIX
-    domain socket.
-
-    Some plans for extending this interface exist.  See:
-
-        - U{http://twistedmatrix.com/trac/ticket/5570}: established connections
-        - U{http://twistedmatrix.com/trac/ticket/5573}: AF_UNIX ports
-        - U{http://twistedmatrix.com/trac/ticket/5574}: SOCK_DGRAM sockets
-    """
-
-    def adoptStreamPort(fileDescriptor, addressFamily, factory):
-        """
-        Add an existing listening I{SOCK_STREAM} socket to the reactor to
-        monitor for new connections to accept and handle.
-
-        @param fileDescriptor: A file descriptor associated with a socket which
-            is already bound to an address and marked as listening.  The socket
-            must be set non-blocking.  Any additional flags (for example,
-            close-on-exec) must also be set by application code.  Application
-            code is responsible for closing the file descriptor, which may be
-            done as soon as C{adoptStreamPort} returns.
-        @type fileDescriptor: C{int}
-
-        @param addressFamily: The address family (or I{domain}) of the socket.
-            For example, L{socket.AF_INET6}.
-
-        @param factory: A L{ServerFactory} instance to use to create new
-            protocols to handle connections accepted via this socket.
-
-        @return: An object providing L{IListeningPort}.
-
-        @raise UnsupportedAddressFamily: If the given address family is not
-            supported by this reactor, or not supported with the given socket
-            type.
-
-        @raise UnsupportedSocketType: If the given socket type is not supported
-            by this reactor, or not supported with the given socket type.
-        """
-
-
-    def adoptStreamConnection(fileDescriptor, addressFamily, factory):
-        """
-        Add an existing connected I{SOCK_STREAM} socket to the reactor to
-        monitor for data.
-
-        Note that the given factory won't have its C{startFactory} and
-        C{stopFactory} methods called, as there is no sensible time to call
-        them in this situation.
-
-        @param fileDescriptor: A file descriptor associated with a socket which
-            is already connected.  The socket must be set non-blocking.  Any
-            additional flags (for example, close-on-exec) must also be set by
-            application code.  Application code is responsible for closing the
-            file descriptor, which may be done as soon as
-            C{adoptStreamConnection} returns.
-        @type fileDescriptor: C{int}
-
-        @param addressFamily: The address family (or I{domain}) of the socket.
-            For example, L{socket.AF_INET6}.
-
-        @param factory: A L{ServerFactory} instance to use to create a new
-            protocol to handle the connection via this socket.
-
-        @raise UnsupportedAddressFamily: If the given address family is not
-            supported by this reactor, or not supported with the given socket
-            type.
-
-        @raise UnsupportedSocketType: If the given socket type is not supported
-            by this reactor, or not supported with the given socket type.
-        """
-
-
-
-class IReactorProcess(Interface):
-
-    def spawnProcess(processProtocol, executable, args=(), env={}, path=None,
-                     uid=None, gid=None, usePTY=0, childFDs=None):
-        """
-        Spawn a process, with a process protocol.
-
-        @type processProtocol: L{IProcessProtocol} provider
-        @param processProtocol: An object which will be notified of all
-            events related to the created process.
-
-        @param executable: the file name to spawn - the full path should be
-                           used.
-
-        @param args: the command line arguments to pass to the process; a
-                     sequence of strings. The first string should be the
-                     executable's name.
-
-        @type env: a C{dict} mapping C{str} to C{str}, or C{None}.
-        @param env: the environment variables to pass to the child process. The
-                    resulting behavior varies between platforms. If
-                      - C{env} is not set:
-                        - On POSIX: pass an empty environment.
-                        - On Windows: pass C{os.environ}.
-                      - C{env} is C{None}:
-                        - On POSIX: pass C{os.environ}.
-                        - On Windows: pass C{os.environ}.
-                      - C{env} is a C{dict}:
-                        - On POSIX: pass the key/value pairs in C{env} as the
-                          complete environment.
-                        - On Windows: update C{os.environ} with the key/value
-                          pairs in the C{dict} before passing it. As a
-                          consequence of U{bug #1640
-                          <http://twistedmatrix.com/trac/ticket/1640>}, passing
-                          keys with empty values in an effort to unset
-                          environment variables I{won't} unset them.
-
-        @param path: the path to run the subprocess in - defaults to the
-                     current directory.
-
-        @param uid: user ID to run the subprocess as. (Only available on
-                    POSIX systems.)
-
-        @param gid: group ID to run the subprocess as. (Only available on
-                    POSIX systems.)
-
-        @param usePTY: if true, run this process in a pseudo-terminal.
-                       optionally a tuple of C{(masterfd, slavefd, ttyname)},
-                       in which case use those file descriptors.
-                       (Not available on all systems.)
-
-        @param childFDs: A dictionary mapping file descriptors in the new child
-                         process to an integer or to the string 'r' or 'w'.
-
-                         If the value is an integer, it specifies a file
-                         descriptor in the parent process which will be mapped
-                         to a file descriptor (specified by the key) in the
-                         child process.  This is useful for things like inetd
-                         and shell-like file redirection.
-
-                         If it is the string 'r', a pipe will be created and
-                         attached to the child at that file descriptor: the
-                         child will be able to write to that file descriptor
-                         and the parent will receive read notification via the
-                         L{IProcessProtocol.childDataReceived} callback.  This
-                         is useful for the child's stdout and stderr.
-
-                         If it is the string 'w', similar setup to the previous
-                         case will occur, with the pipe being readable by the
-                         child instead of writeable.  The parent process can
-                         write to that file descriptor using
-                         L{IProcessTransport.writeToChild}.  This is useful for
-                         the child's stdin.
-
-                         If childFDs is not passed, the default behaviour is to
-                         use a mapping that opens the usual stdin/stdout/stderr
-                         pipes.
-
-        @see: L{twisted.internet.protocol.ProcessProtocol}
-
-        @return: An object which provides L{IProcessTransport}.
-
-        @raise OSError: Raised with errno C{EAGAIN} or C{ENOMEM} if there are
-                        insufficient system resources to create a new process.
-        """
-
-class IReactorTime(Interface):
-    """
-    Time methods that a Reactor should implement.
-    """
-
-    def seconds():
-        """
-        Get the current time in seconds.
-
-        @return: A number-like object of some sort.
-        """
-
-
-    def callLater(delay, callable, *args, **kw):
-        """
-        Call a function later.
-
-        @type delay:  C{float}
-        @param delay: the number of seconds to wait.
-
-        @param callable: the callable object to call later.
-
-        @param args: the arguments to call it with.
-
-        @param kw: the keyword arguments to call it with.
-
-        @return: An object which provides L{IDelayedCall} and can be used to
-                 cancel the scheduled call, by calling its C{cancel()} method.
-                 It also may be rescheduled by calling its C{delay()} or
-                 C{reset()} methods.
-        """
-
-
-    def getDelayedCalls():
-        """
-        Retrieve all currently scheduled delayed calls.
-
-        @return: A tuple of all L{IDelayedCall} providers representing all
-                 currently scheduled calls. This is everything that has been
-                 returned by C{callLater} but not yet called or canceled.
-        """
-
-
-class IDelayedCall(Interface):
-    """
-    A scheduled call.
-
-    There are probably other useful methods we can add to this interface;
-    suggestions are welcome.
-    """
-
-    def getTime():
-        """
-        Get time when delayed call will happen.
-
-        @return: time in seconds since epoch (a float).
-        """
-
-    def cancel():
-        """
-        Cancel the scheduled call.
-
-        @raises twisted.internet.error.AlreadyCalled: if the call has already
-            happened.
-        @raises twisted.internet.error.AlreadyCancelled: if the call has already
-            been cancelled.
-        """
-
-    def delay(secondsLater):
-        """
-        Delay the scheduled call.
-
-        @param secondsLater: how many seconds from its current firing time to delay
-
-        @raises twisted.internet.error.AlreadyCalled: if the call has already
-            happened.
-        @raises twisted.internet.error.AlreadyCancelled: if the call has already
-            been cancelled.
-        """
-
-    def reset(secondsFromNow):
-        """
-        Reset the scheduled call's timer.
-
-        @param secondsFromNow: how many seconds from now it should fire,
-            equivalent to C{.cancel()} and then doing another
-            C{reactor.callLater(secondsLater, ...)}
-
-        @raises twisted.internet.error.AlreadyCalled: if the call has already
-            happened.
-        @raises twisted.internet.error.AlreadyCancelled: if the call has already
-            been cancelled.
-        """
-
-    def active():
-        """
-        @return: True if this call is still active, False if it has been
-                 called or cancelled.
-        """
-
-class IReactorThreads(Interface):
-    """
-    Dispatch methods to be run in threads.
-
-    Internally, this should use a thread pool and dispatch methods to them.
-    """
-
-    def getThreadPool():
-        """
-        Return the threadpool used by L{callInThread}.  Create it first if
-        necessary.
-
-        @rtype: L{twisted.python.threadpool.ThreadPool}
-        """
-
-
-    def callInThread(callable, *args, **kwargs):
-        """
-        Run the callable object in a separate thread.
-        """
-
-
-    def callFromThread(callable, *args, **kw):
-        """
-        Cause a function to be executed by the reactor thread.
-
-        Use this method when you want to run a function in the reactor's thread
-        from another thread.  Calling L{callFromThread} should wake up the main
-        thread (where L{reactor.run()<reactor.run>} is executing) and run the
-        given callable in that thread.
-
-        If you're writing a multi-threaded application the C{callable} may need
-        to be thread safe, but this method doesn't require it as such. If you
-        want to call a function in the next mainloop iteration, but you're in
-        the same thread, use L{callLater} with a delay of 0.
-        """
-
-
-    def suggestThreadPoolSize(size):
-        """
-        Suggest the size of the internal threadpool used to dispatch functions
-        passed to L{callInThread}.
-        """
-
-
-class IReactorCore(Interface):
-    """
-    Core methods that a Reactor must implement.
-    """
-
-    running = Attribute(
-        "A C{bool} which is C{True} from I{during startup} to "
-        "I{during shutdown} and C{False} the rest of the time.")
-
-
-    def resolve(name, timeout=10):
-        """
-        Return a L{twisted.internet.defer.Deferred} that will resolve a hostname.
-        """
-
-    def run():
-        """
-        Fire 'startup' System Events, move the reactor to the 'running'
-        state, then run the main loop until it is stopped with C{stop()} or
-        C{crash()}.
-        """
-
-    def stop():
-        """
-        Fire 'shutdown' System Events, which will move the reactor to the
-        'stopped' state and cause C{reactor.run()} to exit.
-        """
-
-    def crash():
-        """
-        Stop the main loop *immediately*, without firing any system events.
-
-        This is named as it is because this is an extremely "rude" thing to do;
-        it is possible to lose data and put your system in an inconsistent
-        state by calling this.  However, it is necessary, as sometimes a system
-        can become wedged in a pre-shutdown call.
-        """
-
-    def iterate(delay=0):
-        """
-        Run the main loop's I/O polling function for a period of time.
-
-        This is most useful in applications where the UI is being drawn "as
-        fast as possible", such as games. All pending L{IDelayedCall}s will
-        be called.
-
-        The reactor must have been started (via the C{run()} method) prior to
-        any invocations of this method.  It must also be stopped manually
-        after the last call to this method (via the C{stop()} method).  This
-        method is not re-entrant: you must not call it recursively; in
-        particular, you must not call it while the reactor is running.
-        """
-
-    def fireSystemEvent(eventType):
-        """
-        Fire a system-wide event.
-
-        System-wide events are things like 'startup', 'shutdown', and
-        'persist'.
-        """
-
-    def addSystemEventTrigger(phase, eventType, callable, *args, **kw):
-        """
-        Add a function to be called when a system event occurs.
-
-        Each "system event" in Twisted, such as 'startup', 'shutdown', and
-        'persist', has 3 phases: 'before', 'during', and 'after' (in that
-        order, of course).  These events will be fired internally by the
-        Reactor.
-
-        An implementor of this interface must only implement those events
-        described here.
-
-        Callbacks registered for the "before" phase may return either None or a
-        Deferred.  The "during" phase will not execute until all of the
-        Deferreds from the "before" phase have fired.
-
-        Once the "during" phase is running, all of the remaining triggers must
-        execute; their return values must be ignored.
-
-        @param phase: a time to call the event -- either the string 'before',
-                      'after', or 'during', describing when to call it
-                      relative to the event's execution.
-
-        @param eventType: this is a string describing the type of event.
-
-        @param callable: the object to call before shutdown.
-
-        @param args: the arguments to call it with.
-
-        @param kw: the keyword arguments to call it with.
-
-        @return: an ID that can be used to remove this call with
-                 removeSystemEventTrigger.
-        """
-
-    def removeSystemEventTrigger(triggerID):
-        """
-        Removes a trigger added with addSystemEventTrigger.
-
-        @param triggerID: a value returned from addSystemEventTrigger.
-
-        @raise KeyError: If there is no system event trigger for the given
-            C{triggerID}.
-
-        @raise ValueError: If there is no system event trigger for the given
-            C{triggerID}.
-
-        @raise TypeError: If there is no system event trigger for the given
-            C{triggerID}.
-        """
-
-    def callWhenRunning(callable, *args, **kw):
-        """
-        Call a function when the reactor is running.
-
-        If the reactor has not started, the callable will be scheduled
-        to run when it does start. Otherwise, the callable will be invoked
-        immediately.
-
-        @param callable: the callable object to call later.
-
-        @param args: the arguments to call it with.
-
-        @param kw: the keyword arguments to call it with.
-
-        @return: None if the callable was invoked, otherwise a system
-                 event id for the scheduled call.
-        """
-
-
-class IReactorPluggableResolver(Interface):
-    """
-    A reactor with a pluggable name resolver interface.
-    """
-
-    def installResolver(resolver):
-        """
-        Set the internal resolver to use to for name lookups.
-
-        @type resolver: An object implementing the L{IResolverSimple} interface
-        @param resolver: The new resolver to use.
-
-        @return: The previously installed resolver.
-        """
-
-
-class IReactorDaemonize(Interface):
-    """
-    A reactor which provides hooks that need to be called before and after
-    daemonization.
-
-    Notes:
-       - This interface SHOULD NOT be called by applications.
-       - This interface should only be implemented by reactors as a workaround
-         (in particular, it's implemented currently only by kqueue()).
-         For details please see the comments on ticket #1918.
-    """
-
-    def beforeDaemonize():
-        """
-        Hook to be called immediately before daemonization. No reactor methods
-        may be called until L{afterDaemonize} is called.
-
-        @return: C{None}.
-        """
-
-
-    def afterDaemonize():
-        """
-        Hook to be called immediately after daemonization. This may only be
-        called after L{beforeDaemonize} had been called previously.
-
-        @return: C{None}.
-        """
-
-
-
-class IReactorFDSet(Interface):
-    """
-    Implement me to be able to use L{IFileDescriptor} type resources.
-
-    This assumes that your main-loop uses UNIX-style numeric file descriptors
-    (or at least similarly opaque IDs returned from a .fileno() method)
-    """
-
-    def addReader(reader):
-        """
-        I add reader to the set of file descriptors to get read events for.
-
-        @param reader: An L{IReadDescriptor} provider that will be checked for
-                       read events until it is removed from the reactor with
-                       L{removeReader}.
-
-        @return: C{None}.
-        """
-
-    def addWriter(writer):
-        """
-        I add writer to the set of file descriptors to get write events for.
-
-        @param writer: An L{IWriteDescriptor} provider that will be checked for
-                       write events until it is removed from the reactor with
-                       L{removeWriter}.
-
-        @return: C{None}.
-        """
-
-    def removeReader(reader):
-        """
-        Removes an object previously added with L{addReader}.
-
-        @return: C{None}.
-        """
-
-    def removeWriter(writer):
-        """
-        Removes an object previously added with L{addWriter}.
-
-        @return: C{None}.
-        """
-
-    def removeAll():
-        """
-        Remove all readers and writers.
-
-        Should not remove reactor internal reactor connections (like a waker).
-
-        @return: A list of L{IReadDescriptor} and L{IWriteDescriptor} providers
-                 which were removed.
-        """
-
-    def getReaders():
-        """
-        Return the list of file descriptors currently monitored for input
-        events by the reactor.
-
-        @return: the list of file descriptors monitored for input events.
-        @rtype: C{list} of C{IReadDescriptor}
-        """
-
-    def getWriters():
-        """
-        Return the list file descriptors currently monitored for output events
-        by the reactor.
-
-        @return: the list of file descriptors monitored for output events.
-        @rtype: C{list} of C{IWriteDescriptor}
-        """
-
-
-class IListeningPort(Interface):
-    """
-    A listening port.
-    """
-
-    def startListening():
-        """
-        Start listening on this port.
-
-        @raise CannotListenError: If it cannot listen on this port (e.g., it is
-                                  a TCP port and it cannot bind to the required
-                                  port number).
-        """
-
-    def stopListening():
-        """
-        Stop listening on this port.
-
-        If it does not complete immediately, will return Deferred that fires
-        upon completion.
-        """
-
-    def getHost():
-        """
-        Get the host that this port is listening for.
-
-        @return: An L{IAddress} provider.
-        """
-
-
-class ILoggingContext(Interface):
-    """
-    Give context information that will be used to log events generated by
-    this item.
-    """
-
-    def logPrefix():
-        """
-        @return: Prefix used during log formatting to indicate context.
-        @rtype: C{str}
-        """
-
-
-
-class IFileDescriptor(ILoggingContext):
-    """
-    An interface representing a UNIX-style numeric file descriptor.
-    """
-
-    def fileno():
-        """
-        @raise: If the descriptor no longer has a valid file descriptor
-            number associated with it.
-
-        @return: The platform-specified representation of a file descriptor
-            number.  Or C{-1} if the descriptor no longer has a valid file
-            descriptor number associated with it.  As long as the descriptor
-            is valid, calls to this method on a particular instance must
-            return the same value.
-        """
-
-
-    def connectionLost(reason):
-        """
-        Called when the connection was lost.
-
-        This is called when the connection on a selectable object has been
-        lost.  It will be called whether the connection was closed explicitly,
-        an exception occurred in an event handler, or the other end of the
-        connection closed it first.
-
-        See also L{IHalfCloseableDescriptor} if your descriptor wants to be
-        notified separately of the two halves of the connection being closed.
-
-        @param reason: A failure instance indicating the reason why the
-                       connection was lost.  L{error.ConnectionLost} and
-                       L{error.ConnectionDone} are of special note, but the
-                       failure may be of other classes as well.
-        """
-
-
-
-class IReadDescriptor(IFileDescriptor):
-    """
-    An L{IFileDescriptor} that can read.
-
-    This interface is generally used in conjunction with L{IReactorFDSet}.
-    """
-
-    def doRead():
-        """
-        Some data is available for reading on your descriptor.
-
-        @return: If an error is encountered which causes the descriptor to
-            no longer be valid, a L{Failure} should be returned.  Otherwise,
-            C{None}.
-        """
-
-
-class IWriteDescriptor(IFileDescriptor):
-    """
-    An L{IFileDescriptor} that can write.
-
-    This interface is generally used in conjunction with L{IReactorFDSet}.
-    """
-
-    def doWrite():
-        """
-        Some data can be written to your descriptor.
-
-        @return: If an error is encountered which causes the descriptor to
-            no longer be valid, a L{Failure} should be returned.  Otherwise,
-            C{None}.
-        """
-
-
-class IReadWriteDescriptor(IReadDescriptor, IWriteDescriptor):
-    """
-    An L{IFileDescriptor} that can both read and write.
-    """
-
-
-class IHalfCloseableDescriptor(Interface):
-    """
-    A descriptor that can be half-closed.
-    """
-
-    def writeConnectionLost(reason):
-        """
-        Indicates write connection was lost.
-        """
-
-    def readConnectionLost(reason):
-        """
-        Indicates read connection was lost.
-        """
-
-
-class ISystemHandle(Interface):
-    """
-    An object that wraps a networking OS-specific handle.
-    """
-
-    def getHandle():
-        """
-        Return a system- and reactor-specific handle.
-
-        This might be a socket.socket() object, or some other type of
-        object, depending on which reactor is being used. Use and
-        manipulate at your own risk.
-
-        This might be used in cases where you want to set specific
-        options not exposed by the Twisted APIs.
-        """
-
-
-class IConsumer(Interface):
-    """
-    A consumer consumes data from a producer.
-    """
-
-    def registerProducer(producer, streaming):
-        """
-        Register to receive data from a producer.
-
-        This sets self to be a consumer for a producer.  When this object runs
-        out of data (as when a send(2) call on a socket succeeds in moving the
-        last data from a userspace buffer into a kernelspace buffer), it will
-        ask the producer to resumeProducing().
-
-        For L{IPullProducer} providers, C{resumeProducing} will be called once
-        each time data is required.
-
-        For L{IPushProducer} providers, C{pauseProducing} will be called
-        whenever the write buffer fills up and C{resumeProducing} will only be
-        called when it empties.
-
-        @type producer: L{IProducer} provider
-
-        @type streaming: C{bool}
-        @param streaming: C{True} if C{producer} provides L{IPushProducer},
-        C{False} if C{producer} provides L{IPullProducer}.
-
-        @raise RuntimeError: If a producer is already registered.
-
-        @return: C{None}
-        """
-
-
-    def unregisterProducer():
-        """
-        Stop consuming data from a producer, without disconnecting.
-        """
-
-
-    def write(data):
-        """
-        The producer will write data by calling this method.
-
-        The implementation must be non-blocking and perform whatever
-        buffering is necessary.  If the producer has provided enough data
-        for now and it is a L{IPushProducer}, the consumer may call its
-        C{pauseProducing} method.
-        """
-
-
-
-class IProducer(Interface):
-    """
-    A producer produces data for a consumer.
-
-    Typically producing is done by calling the write method of an class
-    implementing L{IConsumer}.
-    """
-
-    def stopProducing():
-        """
-        Stop producing data.
-
-        This tells a producer that its consumer has died, so it must stop
-        producing data for good.
-        """
-
-
-class IPushProducer(IProducer):
-    """
-    A push producer, also known as a streaming producer is expected to
-    produce (write to this consumer) data on a continuous basis, unless
-    it has been paused. A paused push producer will resume producing
-    after its resumeProducing() method is called.   For a push producer
-    which is not pauseable, these functions may be noops.
-    """
-
-    def pauseProducing():
-        """
-        Pause producing data.
-
-        Tells a producer that it has produced too much data to process for
-        the time being, and to stop until resumeProducing() is called.
-        """
-    def resumeProducing():
-        """
-        Resume producing data.
-
-        This tells a producer to re-add itself to the main loop and produce
-        more data for its consumer.
-        """
-
-class IPullProducer(IProducer):
-    """
-    A pull producer, also known as a non-streaming producer, is
-    expected to produce data each time resumeProducing() is called.
-    """
-
-    def resumeProducing():
-        """
-        Produce data for the consumer a single time.
-
-        This tells a producer to produce data for the consumer once
-        (not repeatedly, once only). Typically this will be done
-        by calling the consumer's write() method a single time with
-        produced data.
-        """
-
-class IProtocol(Interface):
-
-    def dataReceived(data):
-        """
-        Called whenever data is received.
-
-        Use this method to translate to a higher-level message.  Usually, some
-        callback will be made upon the receipt of each complete protocol
-        message.
-
-        @param data: a string of indeterminate length.  Please keep in mind
-            that you will probably need to buffer some data, as partial
-            (or multiple) protocol messages may be received!  I recommend
-            that unit tests for protocols call through to this method with
-            differing chunk sizes, down to one byte at a time.
-        """
-
-    def connectionLost(reason):
-        """
-        Called when the connection is shut down.
-
-        Clear any circular references here, and any external references
-        to this Protocol.  The connection has been closed. The C{reason}
-        Failure wraps a L{twisted.internet.error.ConnectionDone} or
-        L{twisted.internet.error.ConnectionLost} instance (or a subclass
-        of one of those).
-
-        @type reason: L{twisted.python.failure.Failure}
-        """
-
-    def makeConnection(transport):
-        """
-        Make a connection to a transport and a server.
-        """
-
-    def connectionMade():
-        """
-        Called when a connection is made.
-
-        This may be considered the initializer of the protocol, because
-        it is called when the connection is completed.  For clients,
-        this is called once the connection to the server has been
-        established; for servers, this is called after an accept() call
-        stops blocking and a socket has been received.  If you need to
-        send any greeting or initial message, do it here.
-        """
-
-
-class IProcessProtocol(Interface):
-    """
-    Interface for process-related event handlers.
-    """
-
-    def makeConnection(process):
-        """
-        Called when the process has been created.
-
-        @type process: L{IProcessTransport} provider
-        @param process: An object representing the process which has been
-            created and associated with this protocol.
-        """
-
-
-    def childDataReceived(childFD, data):
-        """
-        Called when data arrives from the child process.
-
-        @type childFD: C{int}
-        @param childFD: The file descriptor from which the data was
-            received.
-
-        @type data: C{str}
-        @param data: The data read from the child's file descriptor.
-        """
-
-
-    def childConnectionLost(childFD):
-        """
-        Called when a file descriptor associated with the child process is
-        closed.
-
-        @type childFD: C{int}
-        @param childFD: The file descriptor which was closed.
-        """
-
-
-    def processExited(reason):
-        """
-        Called when the child process exits.
-
-        @type reason: L{twisted.python.failure.Failure}
-        @param reason: A failure giving the reason the child process
-            terminated.  The type of exception for this failure is either
-            L{twisted.internet.error.ProcessDone} or
-            L{twisted.internet.error.ProcessTerminated}.
-
-        @since: 8.2
-        """
-
-
-    def processEnded(reason):
-        """
-        Called when the child process exits and all file descriptors associated
-        with it have been closed.
-
-        @type reason: L{twisted.python.failure.Failure}
-        @param reason: A failure giving the reason the child process
-            terminated.  The type of exception for this failure is either
-            L{twisted.internet.error.ProcessDone} or
-            L{twisted.internet.error.ProcessTerminated}.
-        """
-
-
-
-class IHalfCloseableProtocol(Interface):
-    """
-    Implemented to indicate they want notification of half-closes.
-
-    TCP supports the notion of half-closing the connection, e.g.
-    closing the write side but still not stopping reading. A protocol
-    that implements this interface will be notified of such events,
-    instead of having connectionLost called.
-    """
-
-    def readConnectionLost():
-        """
-        Notification of the read connection being closed.
-
-        This indicates peer did half-close of write side. It is now
-        the responsibility of the this protocol to call
-        loseConnection().  In addition, the protocol MUST make sure a
-        reference to it still exists (i.e. by doing a callLater with
-        one of its methods, etc.)  as the reactor will only have a
-        reference to it if it is writing.
-
-        If the protocol does not do so, it might get garbage collected
-        without the connectionLost method ever being called.
-        """
-
-    def writeConnectionLost():
-        """
-        Notification of the write connection being closed.
-
-        This will never be called for TCP connections as TCP does not
-        support notification of this type of half-close.
-        """
-
-
-
-class IFileDescriptorReceiver(Interface):
-    """
-    Protocols may implement L{IFileDescriptorReceiver} to receive file
-    descriptors sent to them.  This is useful in conjunction with
-    L{IUNIXTransport}, which allows file descriptors to be sent between
-    processes on a single host.
-    """
-    def fileDescriptorReceived(descriptor):
-        """
-        Called when a file descriptor is received over the connection.
-
-        @param descriptor: The descriptor which was received.
-        @type descriptor: C{int}
-
-        @return: C{None}
-        """
-
-
-
-class IProtocolFactory(Interface):
-    """
-    Interface for protocol factories.
-    """
-
-    def buildProtocol(addr):
-        """
-        Called when a connection has been established to addr.
-
-        If None is returned, the connection is assumed to have been refused,
-        and the Port will close the connection.
-
-        @type addr: (host, port)
-        @param addr: The address of the newly-established connection
-
-        @return: None if the connection was refused, otherwise an object
-                 providing L{IProtocol}.
-        """
-
-    def doStart():
-        """
-        Called every time this is connected to a Port or Connector.
-        """
-
-    def doStop():
-        """
-        Called every time this is unconnected from a Port or Connector.
-        """
-
-
-class ITransport(Interface):
-    """
-    I am a transport for bytes.
-
-    I represent (and wrap) the physical connection and synchronicity
-    of the framework which is talking to the network.  I make no
-    representations about whether calls to me will happen immediately
-    or require returning to a control loop, or whether they will happen
-    in the same or another thread.  Consider methods of this class
-    (aside from getPeer) to be 'thrown over the wall', to happen at some
-    indeterminate time.
-    """
-
-    def write(data):
-        """
-        Write some data to the physical connection, in sequence, in a
-        non-blocking fashion.
-
-        If possible, make sure that it is all written.  No data will
-        ever be lost, although (obviously) the connection may be closed
-        before it all gets through.
-        """
-
-    def writeSequence(data):
-        """
-        Write a list of strings to the physical connection.
-
-        If possible, make sure that all of the data is written to
-        the socket at once, without first copying it all into a
-        single string.
-        """
-
-    def loseConnection():
-        """
-        Close my connection, after writing all pending data.
-
-        Note that if there is a registered producer on a transport it
-        will not be closed until the producer has been unregistered.
-        """
-
-    def getPeer():
-        """
-        Get the remote address of this connection.
-
-        Treat this method with caution.  It is the unfortunate result of the
-        CGI and Jabber standards, but should not be considered reliable for
-        the usual host of reasons; port forwarding, proxying, firewalls, IP
-        masquerading, etc.
-
-        @return: An L{IAddress} provider.
-        """
-
-    def getHost():
-        """
-        Similar to getPeer, but returns an address describing this side of the
-        connection.
-
-        @return: An L{IAddress} provider.
-        """
-
-
-class ITCPTransport(ITransport):
-    """
-    A TCP based transport.
-    """
-
-    def loseWriteConnection():
-        """
-        Half-close the write side of a TCP connection.
-
-        If the protocol instance this is attached to provides
-        IHalfCloseableProtocol, it will get notified when the operation is
-        done. When closing write connection, as with loseConnection this will
-        only happen when buffer has emptied and there is no registered
-        producer.
-        """
-
-
-    def abortConnection():
-        """
-        Close the connection abruptly.
-
-        Discards any buffered data, stops any registered producer,
-        and, if possible, notifies the other end of the unclean
-        closure.
-
-        @since: 11.1
-        """
-
-
-    def getTcpNoDelay():
-        """
-        Return if C{TCP_NODELAY} is enabled.
-        """
-
-    def setTcpNoDelay(enabled):
-        """
-        Enable/disable C{TCP_NODELAY}.
-
-        Enabling C{TCP_NODELAY} turns off Nagle's algorithm. Small packets are
-        sent sooner, possibly at the expense of overall throughput.
-        """
-
-    def getTcpKeepAlive():
-        """
-        Return if C{SO_KEEPALIVE} is enabled.
-        """
-
-    def setTcpKeepAlive(enabled):
-        """
-        Enable/disable C{SO_KEEPALIVE}.
-
-        Enabling C{SO_KEEPALIVE} sends packets periodically when the connection
-        is otherwise idle, usually once every two hours. They are intended
-        to allow detection of lost peers in a non-infinite amount of time.
-        """
-
-    def getHost():
-        """
-        Returns L{IPv4Address} or L{IPv6Address}.
-        """
-
-    def getPeer():
-        """
-        Returns L{IPv4Address} or L{IPv6Address}.
-        """
-
-
-
-class IUNIXTransport(ITransport):
-    """
-    Transport for stream-oriented unix domain connections.
-    """
-    def sendFileDescriptor(descriptor):
-        """
-        Send a duplicate of this (file, socket, pipe, etc) descriptor to the
-        other end of this connection.
-
-        The send is non-blocking and will be queued if it cannot be performed
-        immediately.  The send will be processed in order with respect to other
-        C{sendFileDescriptor} calls on this transport, but not necessarily with
-        respect to C{write} calls on this transport.  The send can only be
-        processed if there are also bytes in the normal connection-oriented send
-        buffer (ie, you must call C{write} at least as many times as you call
-        C{sendFileDescriptor}).
-
-        @param descriptor: An C{int} giving a valid file descriptor in this
-            process.  Note that a I{file descriptor} may actually refer to a
-            socket, a pipe, or anything else POSIX tries to treat in the same
-            way as a file.
-
-        @return: C{None}
-        """
-
-
-
-class ITLSTransport(ITCPTransport):
-    """
-    A TCP transport that supports switching to TLS midstream.
-
-    Once TLS mode is started the transport will implement L{ISSLTransport}.
-    """
-
-    def startTLS(contextFactory):
-        """
-        Initiate TLS negotiation.
-
-        @param contextFactory: A context factory (see L{ssl.py<twisted.internet.ssl>})
-        """
-
-class ISSLTransport(ITCPTransport):
-    """
-    A SSL/TLS based transport.
-    """
-
-    def getPeerCertificate():
-        """
-        Return an object with the peer's certificate info.
-        """
-
-
-class IProcessTransport(ITransport):
-    """
-    A process transport.
-    """
-
-    pid = Attribute(
-        "From before L{IProcessProtocol.makeConnection} is called to before "
-        "L{IProcessProtocol.processEnded} is called, C{pid} is an L{int} "
-        "giving the platform process ID of this process.  C{pid} is L{None} "
-        "at all other times.")
-
-    def closeStdin():
-        """
-        Close stdin after all data has been written out.
-        """
-
-    def closeStdout():
-        """
-        Close stdout.
-        """
-
-    def closeStderr():
-        """
-        Close stderr.
-        """
-
-    def closeChildFD(descriptor):
-        """
-        Close a file descriptor which is connected to the child process, identified
-        by its FD in the child process.
-        """
-
-    def writeToChild(childFD, data):
-        """
-        Similar to L{ITransport.write} but also allows the file descriptor in
-        the child process which will receive the bytes to be specified.
-
-        @type childFD: C{int}
-        @param childFD: The file descriptor to which to write.
-
-        @type data: C{str}
-        @param data: The bytes to write.
-
-        @return: C{None}
-
-        @raise KeyError: If C{childFD} is not a file descriptor that was mapped
-            in the child when L{IReactorProcess.spawnProcess} was used to create
-            it.
-        """
-
-    def loseConnection():
-        """
-        Close stdin, stderr and stdout.
-        """
-
-    def signalProcess(signalID):
-        """
-        Send a signal to the process.
-
-        @param signalID: can be
-          - one of C{"KILL"}, C{"TERM"}, or C{"INT"}.
-              These will be implemented in a
-              cross-platform manner, and so should be used
-              if possible.
-          - an integer, where it represents a POSIX
-              signal ID.
-
-        @raise twisted.internet.error.ProcessExitedAlready: If the process has
-            already exited.
-        @raise OSError: If the C{os.kill} call fails with an errno different
-            from C{ESRCH}.
-        """
-
-
-class IServiceCollection(Interface):
-    """
-    An object which provides access to a collection of services.
-    """
-
-    def getServiceNamed(serviceName):
-        """
-        Retrieve the named service from this application.
-
-        Raise a C{KeyError} if there is no such service name.
-        """
-
-    def addService(service):
-        """
-        Add a service to this collection.
-        """
-
-    def removeService(service):
-        """
-        Remove a service from this collection.
-        """
-
-
-class IUDPTransport(Interface):
-    """
-    Transport for UDP DatagramProtocols.
-    """
-
-    def write(packet, addr=None):
-        """
-        Write packet to given address.
-
-        @param addr: a tuple of (ip, port). For connected transports must
-                     be the address the transport is connected to, or None.
-                     In non-connected mode this is mandatory.
-
-        @raise twisted.internet.error.MessageLengthError: C{packet} was too
-        long.
-        """
-
-    def connect(host, port):
-        """
-        Connect the transport to an address.
-
-        This changes it to connected mode. Datagrams can only be sent to
-        this address, and will only be received from this address. In addition
-        the protocol's connectionRefused method might get called if destination
-        is not receiving datagrams.
-
-        @param host: an IP address, not a domain name ('127.0.0.1', not 'localhost')
-        @param port: port to connect to.
-        """
-
-    def getHost():
-        """
-        Returns L{IPv4Address}.
-        """
-
-    def stopListening():
-        """
-        Stop listening on this port.
-
-        If it does not complete immediately, will return L{Deferred} that fires
-        upon completion.
-        """
-
-
-
-class IUNIXDatagramTransport(Interface):
-    """
-    Transport for UDP PacketProtocols.
-    """
-
-    def write(packet, address):
-        """
-        Write packet to given address.
-        """
-
-    def getHost():
-        """
-        Returns L{UNIXAddress}.
-        """
-
-
-class IUNIXDatagramConnectedTransport(Interface):
-    """
-    Transport for UDP ConnectedPacketProtocols.
-    """
-
-    def write(packet):
-        """
-        Write packet to address we are connected to.
-        """
-
-    def getHost():
-        """
-        Returns L{UNIXAddress}.
-        """
-
-    def getPeer():
-        """
-        Returns L{UNIXAddress}.
-        """
-
-
-class IMulticastTransport(Interface):
-    """
-    Additional functionality for multicast UDP.
-    """
-
-    def getOutgoingInterface():
-        """
-        Return interface of outgoing multicast packets.
-        """
-
-    def setOutgoingInterface(addr):
-        """
-        Set interface for outgoing multicast packets.
-
-        Returns Deferred of success.
-        """
-
-    def getLoopbackMode():
-        """
-        Return if loopback mode is enabled.
-        """
-
-    def setLoopbackMode(mode):
-        """
-        Set if loopback mode is enabled.
-        """
-
-    def getTTL():
-        """
-        Get time to live for multicast packets.
-        """
-
-    def setTTL(ttl):
-        """
-        Set time to live on multicast packets.
-        """
-
-    def joinGroup(addr, interface=""):
-        """
-        Join a multicast group. Returns L{Deferred} of success or failure.
-
-        If an error occurs, the returned L{Deferred} will fail with
-        L{error.MulticastJoinError}.
-        """
-
-    def leaveGroup(addr, interface=""):
-        """
-        Leave multicast group, return L{Deferred} of success.
-        """
-
-
-class IStreamClientEndpoint(Interface):
-    """
-    A stream client endpoint is a place that L{ClientFactory} can connect to.
-    For example, a remote TCP host/port pair would be a TCP client endpoint.
-
-    @since: 10.1
-    """
-
-    def connect(protocolFactory):
-        """
-        Connect the C{protocolFactory} to the location specified by this
-        L{IStreamClientEndpoint} provider.
-
-        @param protocolFactory: A provider of L{IProtocolFactory}
-        @return: A L{Deferred} that results in an L{IProtocol} upon successful
-            connection otherwise a L{ConnectError}
-        """
-
-
-
-class IStreamServerEndpoint(Interface):
-    """
-    A stream server endpoint is a place that a L{Factory} can listen for
-    incoming connections.
-
-    @since: 10.1
-    """
-
-    def listen(protocolFactory):
-        """
-        Listen with C{protocolFactory} at the location specified by this
-        L{IStreamServerEndpoint} provider.
-
-        @param protocolFactory: A provider of L{IProtocolFactory}
-        @return: A L{Deferred} that results in an L{IListeningPort} or an
-            L{CannotListenError}
-        """
-
-
-
-class IStreamServerEndpointStringParser(Interface):
-    """
-    An L{IStreamServerEndpointStringParser} is like an
-    L{IStreamClientEndpointStringParser}, except for L{IStreamServerEndpoint}s
-    instead of clients.  It integrates with L{endpoints.serverFromString} in
-    much the same way.
-    """
-
-    prefix = Attribute(
-        """
-        @see: L{IStreamClientEndpointStringParser.prefix}
-        """
-    )
-
-
-    def parseStreamServer(reactor, *args, **kwargs):
-        """
-        Parse a stream server endpoint from a reactor and string-only arguments
-        and keyword arguments.
-
-        @see: L{IStreamClientEndpointStringParser.parseStreamClient}
-
-        @return: a stream server endpoint
-        @rtype: L{IStreamServerEndpoint}
-        """
-
-
-
-class IStreamClientEndpointStringParser(Interface):
-    """
-    An L{IStreamClientEndpointStringParser} is a parser which can convert
-    a set of string C{*args} and C{**kwargs} into an L{IStreamClientEndpoint}
-    provider.
-
-    This interface is really only useful in the context of the plugin system
-    for L{endpoints.clientFromString}.  See the document entitled "I{The
-    Twisted Plugin System}" for more details on how to write a plugin.
-
-    If you place an L{IStreamClientEndpointStringParser} plugin in the
-    C{twisted.plugins} package, that plugin's C{parseStreamClient} method will
-    be used to produce endpoints for any description string that begins with
-    the result of that L{IStreamClientEndpointStringParser}'s prefix attribute.
-    """
-
-    prefix = Attribute(
-        """
-        A C{str}, the description prefix to respond to.  For example, an
-        L{IStreamClientEndpointStringParser} plugin which had C{"foo"} for its
-        C{prefix} attribute would be called for endpoint descriptions like
-        C{"foo:bar:baz"} or C{"foo:"}.
-        """
-    )
-
-
-    def parseStreamClient(*args, **kwargs):
-        """
-        This method is invoked by L{endpoints.clientFromString}, if the type of
-        endpoint matches the return value from this
-        L{IStreamClientEndpointStringParser}'s C{prefix} method.
-
-        @param args: The string arguments, minus the endpoint type, in the
-            endpoint description string, parsed according to the rules
-            described in L{endpoints.quoteStringArgument}.  For example, if the
-            description were C{"my-type:foo:bar:baz=qux"}, C{args} would be
-            C{('foo','bar')}
-
-        @param kwargs: The string arguments from the endpoint description
-            passed as keyword arguments.  For example, if the description were
-            C{"my-type:foo:bar:baz=qux"}, C{kwargs} would be
-            C{dict(baz='qux')}.
-
-        @return: a client endpoint
-        @rtype: L{IStreamClientEndpoint}
-        """
-}}} '''
diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py
index 32c88ff2d..fd814dc22 100644
--- a/scrapy/xlib/tx/iweb.py
+++ b/scrapy/xlib/tx/iweb.py
@@ -10,580 +10,11 @@ Interface definitions for L{twisted.web}.
     body is not known in advance.
 """
 
-from zope.interface import Interface, Attribute
-
-#from twisted.internet.interfaces import IPushProducer
-
 from twisted.web.iweb import (
     IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader,
     IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH,
 )
 
-''' {{{
-class IRequest(Interface):
-    """
-    An HTTP request.
-
-    @since: 9.0
-    """
-
-    method = Attribute("A C{str} giving the HTTP method that was used.")
-    uri = Attribute(
-        "A C{str} giving the full encoded URI which was requested (including "
-        "query arguments).")
-    path = Attribute(
-        "A C{str} giving the encoded query path of the request URI.")
-    args = Attribute(
-        "A mapping of decoded query argument names as C{str} to "
-        "corresponding query argument values as C{list}s of C{str}.  "
-        "For example, for a URI with C{'foo=bar&foo=baz&quux=spam'} "
-        "for its query part, C{args} will be C{{'foo': ['bar', 'baz'], "
-        "'quux': ['spam']}}.")
-
-    received_headers = Attribute(
-        "Backwards-compatibility access to C{requestHeaders}.  Use "
-        "C{requestHeaders} instead.  C{received_headers} behaves mostly "
-        "like a C{dict} and does not provide access to all header values.")
-
-    requestHeaders = Attribute(
-        "A L{http_headers.Headers} instance giving all received HTTP request "
-        "headers.")
-
-    content = Attribute(
-        "A file-like object giving the request body.  This may be a file on "
-        "disk, a C{StringIO}, or some other type.  The implementation is free "
-        "to decide on a per-request basis.")
-
-    headers = Attribute(
-        "Backwards-compatibility access to C{responseHeaders}.  Use"
-        "C{responseHeaders} instead.  C{headers} behaves mostly like a "
-        "C{dict} and does not provide access to all header values nor "
-        "does it allow multiple values for one header to be set.")
-
-    responseHeaders = Attribute(
-        "A L{http_headers.Headers} instance holding all HTTP response "
-        "headers to be sent.")
-
-    def getHeader(key):
-        """
-        Get an HTTP request header.
-
-        @type key: C{str}
-        @param key: The name of the header to get the value of.
-
-        @rtype: C{str} or C{NoneType}
-        @return: The value of the specified header, or C{None} if that header
-            was not present in the request.
-        """
-
-
-    def getCookie(key):
-        """
-        Get a cookie that was sent from the network.
-        """
-
-
-    def getAllHeaders():
-        """
-        Return dictionary mapping the names of all received headers to the last
-        value received for each.
-
-        Since this method does not return all header information,
-        C{requestHeaders.getAllRawHeaders()} may be preferred.
-        """
-
-
-    def getRequestHostname():
-        """
-        Get the hostname that the user passed in to the request.
-
-        This will either use the Host: header (if it is available) or the
-        host we are listening on if the header is unavailable.
-
-        @returns: the requested hostname
-        @rtype: C{str}
-        """
-
-
-    def getHost():
-        """
-        Get my originally requesting transport's host.
-
-        @return: An L{IAddress<twisted.internet.interfaces.IAddress>}.
-        """
-
-
-    def getClientIP():
-        """
-        Return the IP address of the client who submitted this request.
-
-        @returns: the client IP address or C{None} if the request was submitted
-            over a transport where IP addresses do not make sense.
-        @rtype: L{str} or C{NoneType}
-        """
-
-
-    def getClient():
-        """
-        Return the hostname of the IP address of the client who submitted this
-        request, if possible.
-
-        This method is B{deprecated}.  See L{getClientIP} instead.
-
-        @rtype: C{NoneType} or L{str}
-        @return: The canonical hostname of the client, as determined by
-            performing a name lookup on the IP address of the client.
-        """
-
-
-    def getUser():
-        """
-        Return the HTTP user sent with this request, if any.
-
-        If no user was supplied, return the empty string.
-
-        @returns: the HTTP user, if any
-        @rtype: C{str}
-        """
-
-
-    def getPassword():
-        """
-        Return the HTTP password sent with this request, if any.
-
-        If no password was supplied, return the empty string.
-
-        @returns: the HTTP password, if any
-        @rtype: C{str}
-        """
-
-
-    def isSecure():
-        """
-        Return True if this request is using a secure transport.
-
-        Normally this method returns True if this request's HTTPChannel
-        instance is using a transport that implements ISSLTransport.
-
-        This will also return True if setHost() has been called
-        with ssl=True.
-
-        @returns: True if this request is secure
-        @rtype: C{bool}
-        """
-
-
-    def getSession(sessionInterface=None):
-        """
-        Look up the session associated with this request or create a new one if
-        there is not one.
-
-        @return: The L{Session} instance identified by the session cookie in
-            the request, or the C{sessionInterface} component of that session
-            if C{sessionInterface} is specified.
-        """
-
-
-    def URLPath():
-        """
-        @return: A L{URLPath} instance which identifies the URL for which this
-            request is.
-        """
-
-
-    def prePathURL():
-        """
-        @return: At any time during resource traversal, a L{str} giving an
-            absolute URL to the most nested resource which has yet been
-            reached.
-        """
-
-
-    def rememberRootURL():
-        """
-        Remember the currently-processed part of the URL for later
-        recalling.
-        """
-
-
-    def getRootURL():
-        """
-        Get a previously-remembered URL.
-        """
-
-
-    # Methods for outgoing response
-    def finish():
-        """
-        Indicate that the response to this request is complete.
-        """
-
-
-    def write(data):
-        """
-        Write some data to the body of the response to this request.  Response
-        headers are written the first time this method is called, after which
-        new response headers may not be added.
-        """
-
-
-    def addCookie(k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None):
-        """
-        Set an outgoing HTTP cookie.
-
-        In general, you should consider using sessions instead of cookies, see
-        L{twisted.web.server.Request.getSession} and the
-        L{twisted.web.server.Session} class for details.
-        """
-
-
-    def setResponseCode(code, message=None):
-        """
-        Set the HTTP response code.
-        """
-
-
-    def setHeader(k, v):
-        """
-        Set an HTTP response header.  Overrides any previously set values for
-        this header.
-
-        @type name: C{str}
-        @param name: The name of the header for which to set the value.
-
-        @type value: C{str}
-        @param value: The value to set for the named header.
-        """
-
-
-    def redirect(url):
-        """
-        Utility function that does a redirect.
-
-        The request should have finish() called after this.
-        """
-
-
-    def setLastModified(when):
-        """
-        Set the C{Last-Modified} time for the response to this request.
-
-        If I am called more than once, I ignore attempts to set Last-Modified
-        earlier, only replacing the Last-Modified time if it is to a later
-        value.
-
-        If I am a conditional request, I may modify my response code to
-        L{NOT_MODIFIED<http.NOT_MODIFIED>} if appropriate for the time given.
-
-        @param when: The last time the resource being returned was modified, in
-            seconds since the epoch.
-        @type when: L{int}, L{long} or L{float}
-
-        @return: If I am a C{If-Modified-Since} conditional request and the time
-            given is not newer than the condition, I return
-            L{CACHED<http.CACHED>} to indicate that you should write no body.
-            Otherwise, I return a false value.
-        """
-
-
-    def setETag(etag):
-        """
-        Set an C{entity tag} for the outgoing response.
-
-        That's "entity tag" as in the HTTP/1.1 I{ETag} header, "used for
-        comparing two or more entities from the same requested resource."
-
-        If I am a conditional request, I may modify my response code to
-        L{NOT_MODIFIED<http.NOT_MODIFIED>} or
-        L{PRECONDITION_FAILED<http.PRECONDITION_FAILED>}, if appropriate for the
-        tag given.
-
-        @param etag: The entity tag for the resource being returned.
-        @type etag: C{str}
-
-        @return: If I am a C{If-None-Match} conditional request and the tag
-            matches one in the request, I return L{CACHED<http.CACHED>} to
-            indicate that you should write no body.  Otherwise, I return a
-            false value.
-        """
-
-
-    def setHost(host, port, ssl=0):
-        """
-        Change the host and port the request thinks it's using.
-
-        This method is useful for working with reverse HTTP proxies (e.g.  both
-        Squid and Apache's mod_proxy can do this), when the address the HTTP
-        client is using is different than the one we're listening on.
-
-        For example, Apache may be listening on https://www.example.com, and
-        then forwarding requests to http://localhost:8080, but we don't want
-        HTML produced by Twisted to say 'http://localhost:8080', they should
-        say 'https://www.example.com', so we do::
-
-           request.setHost('www.example.com', 443, ssl=1)
-        """
-
-
-
-class ICredentialFactory(Interface):
-    """
-    A credential factory defines a way to generate a particular kind of
-    authentication challenge and a way to interpret the responses to these
-    challenges.  It creates
-    L{ICredentials<twisted.cred.credentials.ICredentials>} providers from
-    responses.  These objects will be used with L{twisted.cred} to authenticate
-    an authorize requests.
-    """
-    scheme = Attribute(
-        "A C{str} giving the name of the authentication scheme with which "
-        "this factory is associated.  For example, C{'basic'} or C{'digest'}.")
-
-
-    def getChallenge(request):
-        """
-        Generate a new challenge to be sent to a client.
-
-        @type peer: L{twisted.web.http.Request}
-        @param peer: The request the response to which this challenge will be
-            included.
-
-        @rtype: C{dict}
-        @return: A mapping from C{str} challenge fields to associated C{str}
-            values.
-        """
-
-
-    def decode(response, request):
-        """
-        Create a credentials object from the given response.
-
-        @type response: C{str}
-        @param response: scheme specific response string
-
-        @type request: L{twisted.web.http.Request}
-        @param request: The request being processed (from which the response
-            was taken).
-
-        @raise twisted.cred.error.LoginFailed: If the response is invalid.
-
-        @rtype: L{twisted.cred.credentials.ICredentials} provider
-        @return: The credentials represented by the given response.
-        """
-
-
-
-class IBodyProducer(IPushProducer):
-    """
-    Objects which provide L{IBodyProducer} write bytes to an object which
-    provides L{IConsumer<twisted.internet.interfaces.IConsumer>} by calling its
-    C{write} method repeatedly.
-
-    L{IBodyProducer} providers may start producing as soon as they have an
-    L{IConsumer<twisted.internet.interfaces.IConsumer>} provider.  That is, they
-    should not wait for a C{resumeProducing} call to begin writing data.
-
-    L{IConsumer.unregisterProducer<twisted.internet.interfaces.IConsumer.unregisterProducer>}
-    must not be called.  Instead, the
-    L{Deferred<twisted.internet.defer.Deferred>} returned from C{startProducing}
-    must be fired when all bytes have been written.
-
-    L{IConsumer.write<twisted.internet.interfaces.IConsumer.write>} may
-    synchronously invoke any of C{pauseProducing}, C{resumeProducing}, or
-    C{stopProducing}.  These methods must be implemented with this in mind.
-
-    @since: 9.0
-    """
-
-    # Despite the restrictions above and the additional requirements of
-    # stopProducing documented below, this interface still needs to be an
-    # IPushProducer subclass.  Providers of it will be passed to IConsumer
-    # providers which only know about IPushProducer and IPullProducer, not
-    # about this interface.  This interface needs to remain close enough to one
-    # of those interfaces for consumers to work with it.
-
-    length = Attribute(
-        """
-        C{length} is a C{int} indicating how many bytes in total this
-        L{IBodyProducer} will write to the consumer or L{UNKNOWN_LENGTH}
-        if this is not known in advance.
-        """)
-
-    def startProducing(consumer):
-        """
-        Start producing to the given
-        L{IConsumer<twisted.internet.interfaces.IConsumer>} provider.
-
-        @return: A L{Deferred<twisted.internet.defer.Deferred>} which fires with
-            C{None} when all bytes have been produced or with a
-            L{Failure<twisted.python.failure.Failure>} if there is any problem
-            before all bytes have been produced.
-        """
-
-
-    def stopProducing():
-        """
-        In addition to the standard behavior of
-        L{IProducer.stopProducing<twisted.internet.interfaces.IProducer.stopProducing>}
-        (stop producing data), make sure the
-        L{Deferred<twisted.internet.defer.Deferred>} returned by
-        C{startProducing} is never fired.
-        """
-
-
-
-class IRenderable(Interface):
-    """
-    An L{IRenderable} is an object that may be rendered by the
-    L{twisted.web.template} templating system.
-    """
-
-    def lookupRenderMethod(name):
-        """
-        Look up and return the render method associated with the given name.
-
-        @type name: C{str}
-        @param name: The value of a render directive encountered in the
-            document returned by a call to L{IRenderable.render}.
-
-        @return: A two-argument callable which will be invoked with the request
-            being responded to and the tag object on which the render directive
-            was encountered.
-        """
-
-
-    def render(request):
-        """
-        Get the document for this L{IRenderable}.
-
-        @type request: L{IRequest} provider or C{NoneType}
-        @param request: The request in response to which this method is being
-            invoked.
-
-        @return: An object which can be flattened.
-        """
-
-
-
-class ITemplateLoader(Interface):
-    """
-    A loader for templates; something usable as a value for
-    L{twisted.web.template.Element}'s C{loader} attribute.
-    """
-
-    def load():
-        """
-        Load a template suitable for rendering.
-
-        @return: a C{list} of C{list}s, C{unicode} objects, C{Element}s and
-            other L{IRenderable} providers.
-        """
-
-
-
-class IResponse(Interface):
-    """
-    An object representing an HTTP response received from an HTTP server.
-
-    @since: 11.1
-    """
-
-    version = Attribute(
-        "A three-tuple describing the protocol and protocol version "
-        "of the response.  The first element is of type C{str}, the second "
-        "and third are of type C{int}.  For example, C{('HTTP', 1, 1)}.")
-
-
-    code = Attribute("The HTTP status code of this response, as a C{int}.")
-
-
-    phrase = Attribute(
-        "The HTTP reason phrase of this response, as a C{str}.")
-
-
-    headers = Attribute("The HTTP response L{Headers} of this response.")
-
-
-    length = Attribute(
-        "The C{int} number of bytes expected to be in the body of this "
-        "response or L{UNKNOWN_LENGTH} if the server did not indicate how "
-        "many bytes to expect.  For I{HEAD} responses, this will be 0; if "
-        "the response includes a I{Content-Length} header, it will be "
-        "available in C{headers}.")
-
-
-    def deliverBody(protocol):
-        """
-        Register an L{IProtocol<twisted.internet.interfaces.IProtocol>} provider
-        to receive the response body.
-
-        The protocol will be connected to a transport which provides
-        L{IPushProducer}.  The protocol's C{connectionLost} method will be
-        called with:
-
-            - ResponseDone, which indicates that all bytes from the response
-              have been successfully delivered.
-
-            - PotentialDataLoss, which indicates that it cannot be determined
-              if the entire response body has been delivered.  This only occurs
-              when making requests to HTTP servers which do not set
-              I{Content-Length} or a I{Transfer-Encoding} in the response.
-
-            - ResponseFailed, which indicates that some bytes from the response
-              were lost.  The C{reasons} attribute of the exception may provide
-              more specific indications as to why.
-        """
-
-
-
-class _IRequestEncoder(Interface):
-    """
-    An object encoding data passed to L{IRequest.write}, for example for
-    compression purpose.
-
-    @since: 12.3
-    """
-
-    def encode(data):
-        """
-        Encode the data given and return the result.
-
-        @param data: The content to encode.
-        @type data: C{str}
-
-        @return: The encoded data.
-        @rtype: C{str}
-        """
-
-
-    def finish():
-        """
-        Callback called when the request is closing.
-
-        @return: If necessary, the pending data accumulated from previous
-            C{encode} calls.
-        @rtype: C{str}
-        """
-
-
-
-class _IRequestEncoderFactory(Interface):
-    """
-    A factory for returing L{_IRequestEncoder} instances.
-
-    @since: 12.3
-    """
-
-    def encoderForRequest(request):
-        """
-        If applicable, returns a L{_IRequestEncoder} instance which will encode
-        the request.
-        """
-
-
-
-UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH"
-}}} '''
 __all__ = [
     "ICredentialFactory", "IRequest",
     "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder",

From 534772f6ea8389ec4f51d5ea4c4572080a3d3d7b Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Fri, 2 Dec 2016 01:15:37 +0000
Subject: [PATCH 1183/2974] Import xlib.tx code from twisted proper

---
 scrapy/core/downloader/handlers/http11.py | 5 +++--
 scrapy/downloadermiddlewares/httpcache.py | 2 +-
 scrapy/downloadermiddlewares/retry.py     | 2 +-
 tests/test_downloadermiddleware_retry.py  | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 404e9160b..54aa359fb 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -13,8 +13,9 @@ from twisted.web.http_headers import Headers as TxHeaders
 from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
 from twisted.internet.error import TimeoutError
 from twisted.web.http import PotentialDataLoss
-from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \
-    HTTPConnectionPool, TCP4ClientEndpoint
+from twisted.web.client import Agent, ProxyAgent, ResponseDone, \
+    HTTPConnectionPool
+from twisted.internet.endpoints import TCP4ClientEndpoint
 
 from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 521327bfe..30e49b886 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -3,10 +3,10 @@ from twisted.internet import defer
 from twisted.internet.error import TimeoutError, DNSLookupError, \
         ConnectionRefusedError, ConnectionDone, ConnectError, \
         ConnectionLost, TCPTimedOutError
+from twisted.web.client import ResponseFailed
 from scrapy import signals
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.utils.misc import load_object
-from scrapy.xlib.tx import ResponseFailed
 
 
 class HttpCacheMiddleware(object):
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 74938067f..c9c512be8 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -17,10 +17,10 @@ from twisted.internet import defer
 from twisted.internet.error import TimeoutError, DNSLookupError, \
         ConnectionRefusedError, ConnectionDone, ConnectError, \
         ConnectionLost, TCPTimedOutError
+from twisted.web.client import ResponseFailed
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.response import response_status_message
-from scrapy.xlib.tx import ResponseFailed
 from scrapy.core.downloader.handlers.http11 import TunnelError
 
 logger = logging.getLogger(__name__)
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 3de9399cf..eb17974bf 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -3,10 +3,10 @@ from twisted.internet import defer
 from twisted.internet.error import TimeoutError, DNSLookupError, \
         ConnectionRefusedError, ConnectionDone, ConnectError, \
         ConnectionLost, TCPTimedOutError
+from twisted.web.client import ResponseFailed
 
 from scrapy import twisted_version
 from scrapy.downloadermiddlewares.retry import RetryMiddleware
-from scrapy.xlib.tx import ResponseFailed
 from scrapy.spiders import Spider
 from scrapy.http import Request, Response
 from scrapy.utils.test import get_crawler

From 67cf64edbee99287bc8f663be033d1343e3a77ae Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Fri, 2 Dec 2016 20:53:06 +0000
Subject: [PATCH 1184/2974] Deprecate scrapy.xlib.tx

---
 scrapy/xlib/tx/__init__.py | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx/__init__.py
index 1c9bf09e5..0d94307b7 100644
--- a/scrapy/xlib/tx/__init__.py
+++ b/scrapy/xlib/tx/__init__.py
@@ -1,19 +1,10 @@
-from scrapy import twisted_version
-if twisted_version > (13, 0, 0):
-    from twisted.web import client
-    from twisted.internet import endpoints
-if twisted_version >= (11, 1, 0):
-    from . import client, endpoints
-else:
-    from scrapy.exceptions import NotSupported
-    class _Mocked(object):
-        def __init__(self, *args, **kw):
-            raise NotSupported('HTTP1.1 not supported')
-    class _Mock(object):
-        def __getattr__(self, name):
-            return _Mocked
-    client = endpoints = _Mock()
+from __future__ import absolute_import
 
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+
+from twisted.web import client
+from twisted.internet import endpoints
 
 Agent = client.Agent  # since < 11.1
 ProxyAgent = client.ProxyAgent  # since 11.1
@@ -21,3 +12,8 @@ ResponseDone = client.ResponseDone  # since 11.1
 ResponseFailed = client.ResponseFailed  # since 11.1
 HTTPConnectionPool = client.HTTPConnectionPool  # since 12.1
 TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint  # since 10.1
+
+warnings.warn("Importing from scrapy.xlib.tx is deprecated and will"
+              " no longer be supported in future Scrapy versions."
+              " Update your code to import from twisted proper.",
+              ScrapyDeprecationWarning, stacklevel=2)

From 67bc2e0b18d990f08bfeb913be0c5bd7f299015a Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Fri, 2 Dec 2016 21:00:39 +0000
Subject: [PATCH 1185/2974] Wipe scrapy.xlib.tx

---
 scrapy/xlib/{tx/__init__.py => tx.py} |  0
 scrapy/xlib/tx/LICENSE                | 57 --------------------------
 scrapy/xlib/tx/README                 |  2 -
 scrapy/xlib/tx/_newclient.py          | 59 ---------------------------
 scrapy/xlib/tx/client.py              | 58 --------------------------
 scrapy/xlib/tx/endpoints.py           | 26 ------------
 scrapy/xlib/tx/interfaces.py          | 28 -------------
 scrapy/xlib/tx/iweb.py                | 23 -----------
 8 files changed, 253 deletions(-)
 rename scrapy/xlib/{tx/__init__.py => tx.py} (100%)
 delete mode 100644 scrapy/xlib/tx/LICENSE
 delete mode 100644 scrapy/xlib/tx/README
 delete mode 100644 scrapy/xlib/tx/_newclient.py
 delete mode 100644 scrapy/xlib/tx/client.py
 delete mode 100644 scrapy/xlib/tx/endpoints.py
 delete mode 100644 scrapy/xlib/tx/interfaces.py
 delete mode 100644 scrapy/xlib/tx/iweb.py

diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx.py
similarity index 100%
rename from scrapy/xlib/tx/__init__.py
rename to scrapy/xlib/tx.py
diff --git a/scrapy/xlib/tx/LICENSE b/scrapy/xlib/tx/LICENSE
deleted file mode 100644
index 8529f6edf..000000000
--- a/scrapy/xlib/tx/LICENSE
+++ /dev/null
@@ -1,57 +0,0 @@
-Copyright (c) 2001-2013
-Allen Short
-Andy Gayton
-Andrew Bennetts
-Antoine Pitrou
-Apple Computer, Inc.
-Benjamin Bruheim
-Bob Ippolito
-Canonical Limited
-Christopher Armstrong
-David Reid
-Donovan Preston
-Eric Mangold
-Eyal Lotem
-Itamar Turner-Trauring
-James Knight
-Jason A. Mobarak
-Jean-Paul Calderone
-Jessica McKellar
-Jonathan Jacobs
-Jonathan Lange
-Jonathan D. Simms
-Jürgen Hermann
-Kevin Horn
-Kevin Turner
-Mary Gardiner
-Matthew Lefkowitz
-Massachusetts Institute of Technology
-Moshe Zadka
-Paul Swartz
-Pavel Pergamenshchik
-Ralph Meijer
-Sean Riley
-Software Freedom Conservancy
-Travis B. Hartwell
-Thijs Triemstra
-Thomas Herve
-Timothy Allen
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/scrapy/xlib/tx/README b/scrapy/xlib/tx/README
deleted file mode 100644
index 75ef485ce..000000000
--- a/scrapy/xlib/tx/README
+++ /dev/null
@@ -1,2 +0,0 @@
-This source files are adapted copies from Twisted trunk to support HTTP1.1
-handler under Twisted >= 11.1 and Twisted <= 13.0.0
diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py
deleted file mode 100644
index 39cd20f95..000000000
--- a/scrapy/xlib/tx/_newclient.py
+++ /dev/null
@@ -1,59 +0,0 @@
-# -*- test-case-name: twisted.web.test.test_newclient -*-
-# Copyright (c) Twisted Matrix Laboratories.
-# See LICENSE for details.
-
-"""
-An U{HTTP 1.1<http://www.w3.org/Protocols/rfc2616/rfc2616.html>} client.
-
-The way to use the functionality provided by this module is to:
-
-  - Connect a L{HTTP11ClientProtocol} to an HTTP server
-  - Create a L{Request} with the appropriate data
-  - Pass the request to L{HTTP11ClientProtocol.request}
-  - The returned Deferred will fire with a L{Response} object
-  - Create a L{IProtocol} provider which can handle the response body
-  - Connect it to the response with L{Response.deliverBody}
-  - When the protocol's C{connectionLost} method is called, the response is
-    complete.  See L{Response.deliverBody} for details.
-
-Various other classes in this module support this usage:
-
-  - HTTPParser is the basic HTTP parser.  It can handle the parts of HTTP which
-    are symmetric between requests and responses.
-
-  - HTTPClientParser extends HTTPParser to handle response-specific parts of
-    HTTP.  One instance is created for each request to parse the corresponding
-    response.
-"""
-
-__metaclass__ = type
-
-from zope.interface import implements
-
-from twisted.python import log
-from twisted.python.reflect import fullyQualifiedName
-from twisted.python.failure import Failure
-from twisted.internet.interfaces import IConsumer, IPushProducer
-from twisted.internet.error import ConnectionDone
-from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred
-from twisted.internet.defer import CancelledError
-from twisted.internet.protocol import Protocol
-from twisted.web.iweb import UNKNOWN_LENGTH, IResponse
-from twisted.web.http import NO_CONTENT, NOT_MODIFIED
-from twisted.web.http import _DataLoss, PotentialDataLoss
-from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder
-
-from twisted.web._newclient import (
-    BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException,
-    RequestGenerationFailed, RequestTransmissionFailed, ConnectionAborted,
-    WrongBodyLength, ResponseDone, ResponseFailed, RequestNotSent,
-    ResponseNeverReceived, HTTPParser, HTTPClientParser, Request,
-    LengthEnforcingConsumer, makeStatefulDispatcher, Response, ChunkedEncoder,
-    TransportProxyProducer, HTTP11ClientProtocol
-)
-
-# States HTTPParser can be in
-STATUS = 'STATUS'
-HEADER = 'HEADER'
-BODY = 'BODY'
-DONE = 'DONE'
diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py
deleted file mode 100644
index c2d50648a..000000000
--- a/scrapy/xlib/tx/client.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# -*- test-case-name: twisted.web.test.test_webclient,twisted.web.test.test_agent -*-
-# Copyright (c) Twisted Matrix Laboratories.
-# See LICENSE for details.
-
-"""
-HTTP client.
-"""
-
-from __future__ import division, absolute_import
-
-import os
-
-try:
-    from urlparse import urlunparse
-    from urllib import splithost, splittype
-except ImportError:
-    from urllib.parse import splithost, splittype
-    from urllib.parse import urlunparse as _urlunparse
-
-    def urlunparse(parts):
-        result = _urlunparse(tuple([p.decode("charmap") for p in parts]))
-        return result.encode("charmap")
-import zlib
-
-from zope.interface import implementer
-
-from twisted.python import log
-from twisted.python.failure import Failure
-from twisted.web import http
-from twisted.internet import defer, protocol, task, reactor
-from twisted.internet.interfaces import IProtocol
-from twisted.internet.endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint
-from twisted.python import failure
-from twisted.python.components import proxyForInterface
-from twisted.web import error
-from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse
-from twisted.web.http_headers import Headers
-
-from twisted.web.client import (
-    PartialDownloadError, FileBodyProducer,
-    CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent,
-    Agent, ProxyAgent, HTTPConnectionPool, readBody,
-)
-
-
-# The code which follows is based on the new HTTP client implementation.  It
-# should be significantly better than anything above, though it is not yet
-# feature equivalent.
-
-from twisted.web._newclient import Response
-from twisted.web._newclient import ResponseDone, ResponseFailed
-
-
-__all__ = [
-    'PartialDownloadError',
-    'ResponseDone', 'Response', 'ResponseFailed', 'Agent', 'CookieAgent',
-    'ProxyAgent', 'ContentDecoderAgent', 'GzipDecoder', 'RedirectAgent',
-    'HTTPConnectionPool', 'readBody']
diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py
deleted file mode 100644
index 197e43ed3..000000000
--- a/scrapy/xlib/tx/endpoints.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# -*- test-case-name: twisted.internet.test.test_endpoints -*-
-# Copyright (c) Twisted Matrix Laboratories.
-# See LICENSE for details.
-
-"""
-Implementations of L{IStreamServerEndpoint} and L{IStreamClientEndpoint} that
-wrap the L{IReactorTCP}, L{IReactorSSL}, and L{IReactorUNIX} interfaces.
-
-This also implements an extensible mini-language for describing endpoints,
-parsed by the L{clientFromString} and L{serverFromString} functions.
-
-@since: 10.1
-"""
-
-from __future__ import division, absolute_import
-
-from twisted.internet.endpoints import (
-    clientFromString, serverFromString, quoteStringArgument,
-    TCP4ServerEndpoint, TCP6ServerEndpoint,
-    TCP4ClientEndpoint, TCP6ClientEndpoint,
-    UNIXServerEndpoint, UNIXClientEndpoint,
-    SSL4ServerEndpoint, SSL4ClientEndpoint,
-    AdoptedStreamServerEndpoint, connectProtocol,
-)
-
-__all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"]
diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py
deleted file mode 100644
index fdcbf3977..000000000
--- a/scrapy/xlib/tx/interfaces.py
+++ /dev/null
@@ -1,28 +0,0 @@
-# Copyright (c) Twisted Matrix Laboratories.
-# See LICENSE for details.
-
-"""
-Interface documentation.
-
-Maintainer: Itamar Shtull-Trauring
-"""
-
-from __future__ import division, absolute_import
-
-from twisted.internet.interfaces import (
-    IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL,
-    IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess,
-    IReactorTime, IDelayedCall, IReactorThreads, IReactorCore,
-    IReactorPluggableResolver, IReactorDaemonize, IReactorFDSet,
-    IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor,
-    IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor,
-    ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer,
-    IProtocol, IProcessProtocol, IHalfCloseableProtocol,
-    IFileDescriptorReceiver, IProtocolFactory, ITransport, ITCPTransport,
-    IUNIXTransport,
-    ITLSTransport, ISSLTransport, IProcessTransport, IServiceCollection,
-    IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport,
-    IMulticastTransport, IStreamClientEndpoint, IStreamServerEndpoint,
-    IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,
-    IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver
-)
diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py
deleted file mode 100644
index fd814dc22..000000000
--- a/scrapy/xlib/tx/iweb.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# -*- test-case-name: twisted.web.test -*-
-# Copyright (c) Twisted Matrix Laboratories.
-# See LICENSE for details.
-
-"""
-Interface definitions for L{twisted.web}.
-
-@var UNKNOWN_LENGTH: An opaque object which may be used as the value of
-    L{IBodyProducer.length} to indicate that the length of the entity
-    body is not known in advance.
-"""
-
-from twisted.web.iweb import (
-    IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader,
-    IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH,
-)
-
-__all__ = [
-    "ICredentialFactory", "IRequest",
-    "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder",
-    "_IRequestEncoderFactory",
-
-    "UNKNOWN_LENGTH"]

From c58ea021b816f20fda1b522f80d8b6a1c22f2454 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mois=C3=A9s=20Guimar=C3=A3es?= <moises@wolfssl.com>
Date: Sun, 4 Dec 2016 11:56:14 -0300
Subject: [PATCH 1186/2974] fixes docs

---
 docs/topics/email.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index 18d2f8084..2380a340c 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -87,13 +87,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
         Send email to the given recipients.
 
         :param to: the e-mail recipients
-        :type to: list
+        :type to: str or iterable
 
         :param subject: the subject of the e-mail
         :type subject: str
 
         :param cc: the e-mails to CC
-        :type cc: list
+        :type cc: str or iterable
 
         :param body: the e-mail body
         :type body: str

From a4178f99daf69e930fcc635421cc3a9ba519e36c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mois=C3=A9s=20Guimar=C3=A3es?= <moises@wolfssl.com>
Date: Mon, 5 Dec 2016 15:24:26 -0300
Subject: [PATCH 1187/2974] fixes params types in docs.

---
 docs/topics/email.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index 2380a340c..f68448276 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -87,13 +87,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
         Send email to the given recipients.
 
         :param to: the e-mail recipients
-        :type to: str or iterable
+        :type to: str or list of str
 
         :param subject: the subject of the e-mail
         :type subject: str
 
         :param cc: the e-mails to CC
-        :type cc: str or iterable
+        :type cc: str or list of str
 
         :param body: the e-mail body
         :type body: str

From c08d278c0c7549b3377df995cdf724e3fbb8c02d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mois=C3=A9s=20Guimar=C3=A3es?= <moises@wolfssl.com>
Date: Mon, 5 Dec 2016 16:47:24 -0300
Subject: [PATCH 1188/2974] removes note from docs.

---
 docs/topics/email.rst | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index f68448276..aac93a91a 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -35,12 +35,6 @@ And here is how to use it to send an e-mail (without attachments)::
 
     mailer.send(to=["someone@example.com"], subject="Some subject", body="Some body", cc=["another@example.com"])
 
-.. note::
-    As shown in the example above, ``to`` and ``cc`` need to be lists
-    of email addresses, not single addresses, and even for one recipient,
-    i.e. ``to="someone@example.com"`` will not work.
-
-
 MailSender class reference
 ==========================
 

From 89d5f5acd3627cef040c421fce516bb87cfd4b5e Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 1 Dec 2016 22:42:16 +0100
Subject: [PATCH 1189/2974] Update changelog for upcoming 1.2.2 release

---
 docs/news.rst | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index c302d2e17..db5856d31 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,41 @@
 Release notes
 =============
 
+Scrapy 1.2.2 (2016-12-XX)
+-------------------------
+
+Bug fixes
+~~~~~~~~~
+
+- Fix a cryptic traceback when a pipeline fails on ``open_spider()`` (:issue:`2011`)
+- Fix embedded IPython shell variables (fixing :issue:`396` that re-appeared
+  in 1.2.0, fixed in :issue:`2418`)
+- A couple of patches when dealing with robots.txt:
+
+  - handle (non-standard) relative sitemap URLs (:issue:`2390`)
+  - handle non-ASCII URLs and User-Agents in Python 2 (:issue:`2373`)
+
+Documentation
+~~~~~~~~~~~~~
+
+- Document ``"download_latency"`` key in ``Request``'s ``meta`` dict (:issue:`2033`)
+- Remove page on (deprecated & unsupported) Ubuntu packages from ToC (:issue:`2335`)
+- A few fixed typos (:issue:`2346`, :issue:`2369`, :issue:`2369`, :issue:`2380`)
+  and clarifications (:issue:`2354`, :issue:`2325`)
+
+Other changes
+~~~~~~~~~~~~~
+
+- Advertize `conda-forge`_ as Scrapy's official conda channel (:issue:`2387`)
+- More helpful error messages when trying to use ``.css()`` or ``.xpath()``
+  on non-Text Responses (:issue:`2264`)
+- ``startproject`` command now generates a sample ``middlewares.py`` file (:issue:`2335`)
+- Add more dependencies' version info in ``scrapy version`` verbose output (:issue:`2404`)
+- Remove all ``*.pyc`` files from source distribution (:issue:`2386`)
+
+.. _conda-forge: https://anaconda.org/conda-forge/scrapy
+
+
 Scrapy 1.2.1 (2016-10-21)
 -------------------------
 

From aa2e1b030d717951475c3e4d06fcb447ab1e1181 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 6 Dec 2016 14:44:19 +0100
Subject: [PATCH 1190/2974] Add reference to fixed scheduler settings
 documentation

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index db5856d31..3c75a0228 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -23,7 +23,7 @@ Documentation
 - Document ``"download_latency"`` key in ``Request``'s ``meta`` dict (:issue:`2033`)
 - Remove page on (deprecated & unsupported) Ubuntu packages from ToC (:issue:`2335`)
 - A few fixed typos (:issue:`2346`, :issue:`2369`, :issue:`2369`, :issue:`2380`)
-  and clarifications (:issue:`2354`, :issue:`2325`)
+  and clarifications (:issue:`2354`, :issue:`2325`, :issue:`2414`)
 
 Other changes
 ~~~~~~~~~~~~~

From 09e310d0b7db40a244cf3508c722ab251ac452eb Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 6 Dec 2016 15:03:38 +0100
Subject: [PATCH 1191/2974] Set release date for 1.2.2

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 3c75a0228..5e28fb130 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,7 +3,7 @@
 Release notes
 =============
 
-Scrapy 1.2.2 (2016-12-XX)
+Scrapy 1.2.2 (2016-12-06)
 -------------------------
 
 Bug fixes

From f3d599532943ebf6a4639e9e99781c2f51cd24c5 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 6 Dec 2016 15:21:00 +0100
Subject: [PATCH 1192/2974] =?UTF-8?q?Bump=20version:=201.2.1=20=E2=86=92?=
 =?UTF-8?q?=201.2.2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index ffc933d13..ee039790d 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.2.1
+current_version = 1.2.2
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 6085e9465..23aa83906 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.2.1
+1.2.2

From 5efd65255c88ebb156b07f5f09a0a0fdb66f8e7e Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 6 Dec 2016 18:49:53 +0100
Subject: [PATCH 1193/2974] TST: Randomize IMAGES_EXPIRES above 90 days

---
 tests/test_pipeline_images.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 6c1976b63..342f25ea9 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -244,7 +244,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
             return "".join([chr(random.randint(97, 123)) for _ in range(10)])
 
         settings = {
-            "IMAGES_EXPIRES": random.randint(1, 1000),
+            "IMAGES_EXPIRES": random.randint(100, 1000),
             "IMAGES_STORE": self.tempdir,
             "IMAGES_RESULT_FIELD": random_string(),
             "IMAGES_URLS_FIELD": random_string(),

From 778bed07bf771dd3942ea8cd51b7944065f4e2cd Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 7 Dec 2016 17:56:13 +0100
Subject: [PATCH 1194/2974] Let framework handle only HTTP redirects by default
 for fetch and shell commands

---
 scrapy/commands/fetch.py    | 11 ++++++++---
 scrapy/commands/shell.py    |  6 +++---
 scrapy/shell.py             | 12 ++++++++----
 scrapy/utils/datatypes.py   | 10 ++++++++++
 tests/test_command_fetch.py |  2 +-
 5 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index a157b19f8..6fe6d73b9 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -5,6 +5,7 @@ from w3lib.url import is_url
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
 from scrapy.exceptions import UsageError
+from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 
 class Command(ScrapyCommand):
@@ -27,8 +28,8 @@ class Command(ScrapyCommand):
             help="use this spider")
         parser.add_option("--headers", dest="headers", action="store_true", \
             help="print response HTTP headers instead of body")
-        parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \
-            default=False, help="do not handle status codes like redirects and print response as-is")
+        parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
+            default=False, help="do not handle HTTP 3xx status codes and print response as-is")
 
     def _print_headers(self, headers, prefix):
         for key, values in headers.items():
@@ -52,7 +53,11 @@ class Command(ScrapyCommand):
             raise UsageError()
         cb = lambda x: self._print_response(x, opts)
         request = Request(args[0], callback=cb, dont_filter=True)
-        if opts.no_status_aware:
+        # by default, let the framework handle redirects,
+        # i.e. command handles all codes expect 3xx
+        if not opts.no_redirect:
+            request.meta['handle_httpstatus_list'] = SequenceExclude(six.moves.range(300, 400))
+        else:
             request.meta['handle_httpstatus_all'] = True
 
         spidercls = DefaultSpider
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index bc0203d89..40a58d94a 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -36,8 +36,8 @@ class Command(ScrapyCommand):
             help="evaluate the code in the shell, print the result and exit")
         parser.add_option("--spider", dest="spider",
             help="use this spider")
-        parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \
-            default=False, help="do not transparently handle status codes like redirects")
+        parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
+            default=False, help="do not handle HTTP 3xx status codes and print response as-is")
 
     def update_vars(self, vars):
         """You can use this function to update the Scrapy objects that will be
@@ -70,7 +70,7 @@ class Command(ScrapyCommand):
         self._start_crawler_thread()
 
         shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
-        shell.start(url=url, handle_statuses=opts.no_status_aware)
+        shell.start(url=url, redirect=not opts.no_redirect)
 
     def _start_crawler_thread(self):
         t = Thread(target=self.crawler_process.start,
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 966003f17..6c78722be 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -7,6 +7,7 @@ from __future__ import print_function
 
 import os
 import signal
+from six.moves import range
 import warnings
 
 from twisted.internet import reactor, threads, defer
@@ -20,6 +21,7 @@ from scrapy.item import BaseItem
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.utils.console import start_python_console
+from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
 from scrapy.utils.conf import get_config
@@ -40,11 +42,11 @@ class Shell(object):
         self.code = code
         self.vars = {}
 
-    def start(self, url=None, request=None, response=None, spider=None, handle_statuses=True):
+    def start(self, url=None, request=None, response=None, spider=None, redirect=True):
         # disable accidental Ctrl-C key press from shutting down the engine
         signal.signal(signal.SIGINT, signal.SIG_IGN)
         if url:
-            self.fetch(url, spider, handle_statuses=handle_statuses)
+            self.fetch(url, spider, redirect=redirect)
         elif request:
             self.fetch(request, spider)
         elif response:
@@ -98,13 +100,15 @@ class Shell(object):
         self.spider = spider
         return spider
 
-    def fetch(self, request_or_url, spider=None, handle_statuses=False, **kwargs):
+    def fetch(self, request_or_url, spider=None, redirect=True, **kwargs):
         if isinstance(request_or_url, Request):
             request = request_or_url
         else:
             url = any_to_uri(request_or_url)
             request = Request(url, dont_filter=True, **kwargs)
-            if handle_statuses:
+            if redirect:
+                request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400))
+            else:
                 request.meta['handle_httpstatus_all'] = True
         response = None
         try:
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index d04b43176..e516185bd 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -304,3 +304,13 @@ class LocalCache(OrderedDict):
         while len(self) >= self.limit:
             self.popitem(last=False)
         super(LocalCache, self).__setitem__(key, value)
+
+
+class SequenceExclude(object):
+    """Object to test if an item is NOT within some sequence."""
+
+    def __init__(self, seq):
+        self.seq = seq
+
+    def __contains__(self, item):
+        return item not in self.seq
diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py
index 45d03a129..3fa3ed930 100644
--- a/tests/test_command_fetch.py
+++ b/tests/test_command_fetch.py
@@ -21,7 +21,7 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_redirect_disabled(self):
-        _, out, err = yield self.execute(['--no-status-aware', self.url('/redirect-no-meta-refresh')])
+        _, out, err = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh')])
         err = err.strip()
         self.assertIn(b'downloader/response_status_count/302', err, err)
         self.assertNotIn(b'downloader/response_status_count/200', err, err)

From 7e54de24550df658690277839cbee99c9afe4bc8 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 7 Dec 2016 18:41:24 +0100
Subject: [PATCH 1195/2974] Add tests for shell command with and without
 --no-redirect

---
 tests/test_command_shell.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 7bb7439d6..ee6e8ad8e 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -49,6 +49,16 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
         _, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url'])
         assert out.strip().endswith(b'/redirected')
 
+    @defer.inlineCallbacks
+    def test_redirect_follow_302(self):
+        _, out, _ = yield self.execute([self.url('/redirect-no-meta-refresh'), '-c', 'response.status'])
+        assert out.strip().endswith(b'200')
+
+    @defer.inlineCallbacks
+    def test_redirect_not_follow_302(self):
+        _, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status'])
+        assert out.strip().endswith(b'302')
+
     @defer.inlineCallbacks
     def test_request_replace(self):
         url = self.url('/text')

From 2cd579a7748d0c37eac557216b857c38fbcf80df Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 7 Dec 2016 19:07:32 +0100
Subject: [PATCH 1196/2974] Add test for fetch(url) within shell with and
 without redirect

---
 tests/test_command_shell.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index ee6e8ad8e..3e27d6abd 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -59,6 +59,25 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
         _, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status'])
         assert out.strip().endswith(b'302')
 
+    @defer.inlineCallbacks
+    def test_fetch_redirect_follow_302(self):
+        """Test that calling `fetch(url)` follows HTTP redirects by default."""
+        url = self.url('/redirect-no-meta-refresh')
+        code = "fetch('{0}')"
+        errcode, out, errout = yield self.execute(['-c', code.format(url)])
+        self.assertEqual(errcode, 0, out)
+        assert b'Redirecting (302)' in errout
+        assert b'Crawled (200)' in errout
+
+    @defer.inlineCallbacks
+    def test_fetch_redirect_not_follow_302(self):
+        """Test that calling `fetch(url, redirect=False)` disables automatic redirects."""
+        url = self.url('/redirect-no-meta-refresh')
+        code = "fetch('{0}', redirect=False)"
+        errcode, out, errout = yield self.execute(['-c', code.format(url)])
+        self.assertEqual(errcode, 0, out)
+        assert b'Crawled (302)' in errout
+
     @defer.inlineCallbacks
     def test_request_replace(self):
         url = self.url('/text')

From 948e3cd00328f9a23410b3a7197975f56efadd16 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 8 Dec 2016 12:50:26 +0100
Subject: [PATCH 1197/2974] Warn user instead of failing for wrong
 SPIDER_MODULES setting

---
 scrapy/spiderloader.py              | 13 ++++++++++---
 tests/test_spiderloader/__init__.py | 12 ++++++++++++
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index fbf68cec4..265182329 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
+import warnings
 
 from zope.interface import implementer
 
@@ -18,15 +19,21 @@ class SpiderLoader(object):
         self.spider_modules = settings.getlist('SPIDER_MODULES')
         self._spiders = {}
         self._load_all_spiders()
-            
+
     def _load_spiders(self, module):
         for spcls in iter_spider_classes(module):
             self._spiders[spcls.name] = spcls
 
     def _load_all_spiders(self):
         for name in self.spider_modules:
-            for module in walk_modules(name):
-                self._load_spiders(module)
+            try:
+                for module in walk_modules(name):
+                    self._load_spiders(module)
+            except ImportError as e:
+                msg = ("Could not load spiders from module '{}'; "
+                       "Check SPIDER_MODULES setting "
+                       "(exception: {})".format(name, str(e)))
+                warnings.warn(msg, RuntimeWarning)
 
     @classmethod
     def from_settings(cls, settings):
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index fbd2c1669..b2ad93b3f 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -1,6 +1,7 @@
 import sys
 import os
 import shutil
+import warnings
 
 from zope.interface.verify import verifyObject
 from twisted.trial import unittest
@@ -89,3 +90,14 @@ class SpiderLoaderTest(unittest.TestCase):
         crawler = runner.create_crawler('spider1')
         self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
         self.assertEqual(crawler.spidercls.name, 'spider1')
+
+    def test_bad_spider_modules_warning(self):
+
+        with warnings.catch_warnings(record=True) as w:
+            module = 'tests.test_spiderloader.test_spiders.doesnotexist'
+            settings = Settings({'SPIDER_MODULES': [module]})
+            spider_loader = SpiderLoader.from_settings(settings)
+            self.assertIn("Could not load spiders from module", str(w[0].message))
+
+            spiders = spider_loader.list()
+            self.assertEqual(spiders, [])

From 7d1783603251923bb549d34a64d43952fe03b3bc Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 8 Dec 2016 17:27:25 +0100
Subject: [PATCH 1198/2974] Update documentation about --no-redirect option

---
 docs/topics/commands.rst | 28 ++++++++++++++++++
 docs/topics/shell.rst    | 62 ++++++++++++++++++++++++++--------------
 scrapy/shell.py          |  9 ++++--
 3 files changed, 75 insertions(+), 24 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 32669104c..3a26b19ae 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -322,6 +322,14 @@ So this command can be used to "see" how your spider would fetch a certain page.
 If used outside a project, no particular per-spider behaviour would be applied
 and it will just use the default Scrapy downloader settings.
 
+Supported options:
+
+* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
+
+* ``--headers``: print the response's HTTP headers instead of the response's body
+
+* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
+
 Usage examples::
 
     $ scrapy fetch --nolog http://www.example.com/some/page.html
@@ -368,11 +376,31 @@ given. Also supports UNIX-style local file paths, either relative with
 ``./`` or ``../`` prefixes or absolute file paths.
 See :ref:`topics-shell` for more info.
 
+Supported options:
+
+* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
+
+* ``-c code``: evaluate the code in the shell, print the result and exit
+
+* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
+
 Usage example::
 
     $ scrapy shell http://www.example.com/some/page.html
     [ ... scrapy shell starts ... ]
 
+    $ scrapy shell --nolog http://www.example.com/ -c '(response.status, response.url)'
+    (200, 'http://www.example.com/')
+
+    # shell follows HTTP redirects by default
+    $ scrapy shell --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)'
+    (200, 'http://example.com/')
+
+    # you can disable this with --no-redirect
+    $ scrapy shell --no-redirect --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)'
+    (302, 'http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F')
+
+
 .. command:: parse
 
 parse
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 322c3ddfa..6eb81a71f 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -97,8 +97,12 @@ Available Shortcuts
 
  * ``shelp()`` - print a help with the list of available objects and shortcuts
 
- * ``fetch(request_or_url)`` - fetch a new response from the given request or
-   URL and update all related objects accordingly.
+ * ``fetch(url[, redirect=True])`` - fetch a new response from the given
+   URL and update all related objects accordingly. You can optionaly ask for
+   HTTP 3xx redirections to not be followed by passing ``redirect=False``
+
+ * ``fetch(request)`` - fetch a new response from the given request and
+   update all related objects accordingly.
 
  * ``view(response)`` - open the given response in your local web browser, for
    inspection. This will add a `\<base\> tag`_ to the response body in order
@@ -157,36 +161,28 @@ list of available objects and useful shortcuts (you'll notice that these lines
 all start with the ``[s]`` prefix)::
 
     [s] Available Scrapy objects:
-    [s]   crawler    <scrapy.crawler.Crawler object at 0x1e16b50>
+    [s]   scrapy     scrapy module (contains scrapy.Request, scrapy.Selector, etc)
+    [s]   crawler    <scrapy.crawler.Crawler object at 0x7f07395dd690>
     [s]   item       {}
     [s]   request    <GET http://scrapy.org>
-    [s]   response   <200 http://scrapy.org>
-    [s]   settings   <scrapy.settings.Settings object at 0x2bfd650>
-    [s]   spider     <Spider 'default' at 0x20c6f50>
+    [s]   response   <200 https://scrapy.org/>
+    [s]   settings   <scrapy.settings.Settings object at 0x7f07395dd710>
+    [s]   spider     <DefaultSpider 'default' at 0x7f0735891690>
     [s] Useful shortcuts:
+    [s]   fetch(url[, redirect=True]) Fetch URL and update local objects (by default, redirects are followed)
+    [s]   fetch(req)                  Fetch a scrapy.Request and update local objects
     [s]   shelp()           Shell help (print this help)
-    [s]   fetch(req_or_url) Fetch request (or URL) and update local objects
     [s]   view(response)    View response in a browser
 
     >>>
 
+
 After that, we can start playing with the objects::
 
     >>> response.xpath('//title/text()').extract_first()
     u'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
 
     >>> fetch("http://reddit.com")
-    [s] Available Scrapy objects:
-    [s]   crawler    <scrapy.crawler.Crawler object at 0x7fb3ed9c9c90>
-    [s]   item       {}
-    [s]   request    <GET http://reddit.com>
-    [s]   response   <200 https://www.reddit.com/>
-    [s]   settings   <scrapy.settings.Settings object at 0x7fb3ed9c9c10>
-    [s]   spider     <DefaultSpider 'default' at 0x7fb3ecdd3390>
-    [s] Useful shortcuts:
-    [s]   shelp()           Shell help (print this help)
-    [s]   fetch(req_or_url) Fetch request (or URL) and update local objects
-    [s]   view(response)    View response in a browser
 
     >>> response.xpath('//title/text()').extract()
     [u'reddit: the front page of the internet']
@@ -194,12 +190,36 @@ After that, we can start playing with the objects::
     >>> request = request.replace(method="POST")
 
     >>> fetch(request)
-    [s] Available Scrapy objects:
-    [s]   crawler    <scrapy.crawler.Crawler object at 0x1e16b50>
-    ...
 
+    >>> response.status
+    404
+
+    >>> from pprint import pprint
+
+    >>> pprint(response.headers)
+    {'Accept-Ranges': ['bytes'],
+     'Cache-Control': ['max-age=0, must-revalidate'],
+     'Content-Type': ['text/html; charset=UTF-8'],
+     'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
+     'Server': ['snooserv'],
+     'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
+                    'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
+                    'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
+                    'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
+     'Vary': ['accept-encoding'],
+     'Via': ['1.1 varnish'],
+     'X-Cache': ['MISS'],
+     'X-Cache-Hits': ['0'],
+     'X-Content-Type-Options': ['nosniff'],
+     'X-Frame-Options': ['SAMEORIGIN'],
+     'X-Moose': ['majestic'],
+     'X-Served-By': ['cache-cdg8730-CDG'],
+     'X-Timer': ['S1481214079.394283,VS0,VE159'],
+     'X-Ua-Compatible': ['IE=edge'],
+     'X-Xss-Protection': ['1; mode=block']}
     >>>
 
+
 .. _topics-shell-inspect-response:
 
 Invoking the shell from spiders to inspect responses
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 6c78722be..babc267c7 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -148,10 +148,13 @@ class Shell(object):
             if self._is_relevant(v):
                 b.append("  %-10s %s" % (k, v))
         b.append("Useful shortcuts:")
-        b.append("  shelp()           Shell help (print this help)")
         if self.inthread:
-            b.append("  fetch(req_or_url) Fetch request (or URL) and "
-                     "update local objects")
+            b.append("  fetch(url[, redirect=True]) "
+                     "Fetch URL and update local objects "
+                     "(by default, redirects are followed)")
+            b.append("  fetch(req)                  "
+                     "Fetch a scrapy.Request and update local objects ")
+        b.append("  shelp()           Shell help (print this help)")
         b.append("  view(response)    View response in a browser")
 
         return "\n".join("[s] %s" % l for l in b)

From a75ad2bbc63e2fd351c43069a034461c1ab673cf Mon Sep 17 00:00:00 2001
From: Akhil Lb <akhil.lb1@gmail.com>
Date: Wed, 4 Nov 2015 01:59:57 +0530
Subject: [PATCH 1199/2974] LOG_SHORT_NAMES option

---
 docs/topics/logging.rst             |  5 +++++
 docs/topics/settings.rst            | 10 ++++++++++
 scrapy/settings/default_settings.py |  1 +
 scrapy/utils/log.py                 |  3 ++-
 4 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index b7aa6d985..231f5186b 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -150,6 +150,7 @@ These settings can be used to configure the logging:
 * :setting:`LOG_FORMAT`
 * :setting:`LOG_DATEFORMAT`
 * :setting:`LOG_STDOUT`
+* :setting:`LOG_SHORT_NAMES`
 
 The first couple of settings define a destination for log messages. If
 :setting:`LOG_FILE` is set, messages sent through the root logger will be
@@ -170,6 +171,10 @@ listed in `logging's logrecord attributes docs
 <https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_
 respectively.
 
+If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the scrapy
+component that prints the log. It is unset by default, hence logs contain the 
+scrapy component responsible for that log output.
+
 Command-line options
 --------------------
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index a17472564..c528987ec 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -788,6 +788,16 @@ If ``True``, all standard output (and error) of your process will be redirected
 to the log. For example if you ``print 'hello'`` it will appear in the Scrapy
 log.
 
+.. setting:: LOG_SHORT_NAMES
+
+LOG_SHORT_NAMES
+____________
+
+Default: ``False``
+
+If ``True``, the logs will just contain the root path. If it is set to ``False``
+then it displays the component responsible for the log output
+
 .. setting:: MEMDEBUG_ENABLED
 
 MEMDEBUG_ENABLED
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 61f4bd567..24714a7a8 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -191,6 +191,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'
 LOG_STDOUT = False
 LOG_LEVEL = 'DEBUG'
 LOG_FILE = None
+LOG_SHORT_NAMES = False
 
 SCHEDULER_DEBUG = False
 
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 51f303216..f33ce7017 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -118,7 +118,8 @@ def _get_handler(settings):
     )
     handler.setFormatter(formatter)
     handler.setLevel(settings.get('LOG_LEVEL'))
-    handler.addFilter(TopLevelFormatter(['scrapy']))
+    if settings.getbool('LOG_SHORT_NAMES'):
+        handler.addFilter(TopLevelFormatter(['scrapy']))
     return handler
 
 

From 05cec0f2f348345e3d32242968450fb523d25dff Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 27 Jan 2016 15:21:05 +0500
Subject: [PATCH 1200/2974] fixed ReST syntax

---
 docs/topics/settings.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index c528987ec..503f4afb1 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -791,7 +791,7 @@ log.
 .. setting:: LOG_SHORT_NAMES
 
 LOG_SHORT_NAMES
-____________
+---------------
 
 Default: ``False``
 

From 6eab59cbac6a35e7d92a7391541ad2f16493338b Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 9 Dec 2016 02:14:12 +0500
Subject: [PATCH 1201/2974] TST cleanup runspider tests

---
 tests/test_commands.py | 47 +++++++++++++++++++++---------------------
 1 file changed, 24 insertions(+), 23 deletions(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index b507c46bc..bcd7215a0 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -38,11 +38,11 @@ class ProjectTest(unittest.TestCase):
             return subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd,
                 env=self.env, **kwargs)
 
-    def proc(self, *new_args, **kwargs):
+    def proc(self, *new_args, **popen_kwargs):
         args = (sys.executable, '-m', 'scrapy.cmdline') + new_args
         p = subprocess.Popen(args, cwd=self.cwd, env=self.env,
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                             **kwargs)
+                             **popen_kwargs)
 
         waited = 0
         interval = 0.2
@@ -182,6 +182,17 @@ class MiscCommandsTest(CommandTest):
 
 class RunSpiderCommandTest(CommandTest):
 
+    debug_log_spider = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+
+    def start_requests(self):
+        self.logger.debug("It Works!")
+        return []
+"""
+
     @contextmanager
     def _create_file(self, content, name):
         tmpdir = self.mktemp()
@@ -194,32 +205,23 @@ class RunSpiderCommandTest(CommandTest):
         finally:
             rmtree(tmpdir)
 
-    def runspider(self, code, name='myspider.py'):
+    def runspider(self, code, name='myspider.py', args=()):
         with self._create_file(code, name) as fname:
-            return self.proc('runspider', fname)
+            return self.proc('runspider', fname, *args)
+
+    def get_log(self, code, name='myspider.py', args=()):
+        p = self.runspider(code, name=name, args=args)
+        return to_native_str(p.stderr.read())
 
     def test_runspider(self):
-        spider = """
-import scrapy
-
-class MySpider(scrapy.Spider):
-    name = 'myspider'
-
-    def start_requests(self):
-        self.logger.debug("It Works!")
-        return []
-"""
-        p = self.runspider(spider)
-        log = to_native_str(p.stderr.read())
-
+        log = self.get_log(self.debug_log_spider)
         self.assertIn("DEBUG: It Works!", log)
         self.assertIn("INFO: Spider opened", log)
         self.assertIn("INFO: Closing spider (finished)", log)
         self.assertIn("INFO: Spider closed (finished)", log)
 
     def test_runspider_no_spider_found(self):
-        p = self.runspider("from scrapy.spiders import Spider\n")
-        log = to_native_str(p.stderr.read())
+        log = self.get_log("from scrapy.spiders import Spider\n")
         self.assertIn("No spider found in file", log)
 
     def test_runspider_file_not_found(self):
@@ -228,12 +230,11 @@ class MySpider(scrapy.Spider):
         self.assertIn("File not found: some_non_existent_file", log)
 
     def test_runspider_unable_to_load(self):
-        p = self.runspider('', 'myspider.txt')
-        log = to_native_str(p.stderr.read())
+        log = self.get_log('', name='myspider.txt')
         self.assertIn('Unable to load', log)
 
     def test_start_requests_errors(self):
-        p = self.runspider("""
+        log = self.get_log("""
 import scrapy
 
 class BadSpider(scrapy.Spider):
@@ -241,11 +242,11 @@ class BadSpider(scrapy.Spider):
     def start_requests(self):
         raise Exception("oops!")
         """, name="badspider.py")
-        log = to_native_str(p.stderr.read())
         print(log)
         self.assertIn("start_requests", log)
         self.assertIn("badspider.py", log)
 
+
 class BenchCommandTest(CommandTest):
 
     def test_run(self):

From e46572d6f2de1533b1df2ab206971351c22bbbbe Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 9 Dec 2016 02:19:33 +0500
Subject: [PATCH 1202/2974] TST end-to-end test for LOG_LEVEL option

there were no end-to-end tests for this option
---
 tests/test_commands.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index bcd7215a0..1dd88f342 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -220,6 +220,12 @@ class MySpider(scrapy.Spider):
         self.assertIn("INFO: Closing spider (finished)", log)
         self.assertIn("INFO: Spider closed (finished)", log)
 
+    def test_runspider_log_level(self):
+        log = self.get_log(self.debug_log_spider,
+                           args=('-s', 'LOG_LEVEL=INFO'))
+        self.assertNotIn("DEBUG: It Works!", log)
+        self.assertIn("INFO: Spider opened", log)
+
     def test_runspider_no_spider_found(self):
         log = self.get_log("from scrapy.spiders import Spider\n")
         self.assertIn("No spider found in file", log)

From 05b4555f3932afb04ab3b15893a5858fca9dec04 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 9 Dec 2016 02:19:51 +0500
Subject: [PATCH 1203/2974] TST tests for LOG_SHORT_NAMES

---
 tests/test_commands.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 1dd88f342..922098668 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -226,6 +226,21 @@ class MySpider(scrapy.Spider):
         self.assertNotIn("DEBUG: It Works!", log)
         self.assertIn("INFO: Spider opened", log)
 
+    def test_runspider_log_short_names(self):
+        log1 = self.get_log(self.debug_log_spider,
+                            args=('-s', 'LOG_SHORT_NAMES=1'))
+        print(log1)
+        self.assertIn("[myspider] DEBUG: It Works!", log1)
+        self.assertIn("[scrapy]", log1)
+        self.assertNotIn("[scrapy.core.engine]", log1)
+
+        log2 = self.get_log(self.debug_log_spider,
+                            args=('-s', 'LOG_SHORT_NAMES=0'))
+        print(log2)
+        self.assertIn("[myspider] DEBUG: It Works!", log2)
+        self.assertNotIn("[scrapy]", log2)
+        self.assertIn("[scrapy.core.engine]", log2)
+
     def test_runspider_no_spider_found(self):
         log = self.get_log("from scrapy.spiders import Spider\n")
         self.assertIn("No spider found in file", log)

From f457379a54cdbad60bcf24947e053340abc7e16b Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 9 Dec 2016 16:56:26 +0100
Subject: [PATCH 1204/2974] Add stacktrace in warning message

---
 scrapy/spiderloader.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 265182329..d4f0f663f 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
+import traceback
 import warnings
 
 from zope.interface import implementer
@@ -30,9 +31,9 @@ class SpiderLoader(object):
                 for module in walk_modules(name):
                     self._load_spiders(module)
             except ImportError as e:
-                msg = ("Could not load spiders from module '{}'; "
-                       "Check SPIDER_MODULES setting "
-                       "(exception: {})".format(name, str(e)))
+                msg = ("\n{tb}Could not load spiders from module '{modname}'. "
+                       "Check SPIDER_MODULES setting".format(
+                            modname=name, tb=traceback.format_exc()))
                 warnings.warn(msg, RuntimeWarning)
 
     @classmethod

From f7e4081414d318ddb5297fb98a120e57044dc622 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 12 Dec 2016 22:37:53 +0100
Subject: [PATCH 1205/2974] Add tests for SequenceExclude container

---
 tests/test_utils_datatypes.py | 63 ++++++++++++++++++++++++++++++++++-
 1 file changed, 62 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index b31d2179c..80f797227 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,7 +1,7 @@
 import copy
 import unittest
 
-from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.datatypes import CaselessDict, SequenceExclude
 
 __doctests__ = ['scrapy.utils.datatypes']
 
@@ -128,6 +128,67 @@ class CaselessDictTest(unittest.TestCase):
         assert isinstance(h2, CaselessDict)
 
 
+class SequenceExcludeTest(unittest.TestCase):
+
+    def test_list(self):
+        seq = [1, 2, 3]
+        d = SequenceExclude(seq)
+        self.assertIn(0, d)
+        self.assertIn(4, d)
+        self.assertNotIn(2, d)
+
+    def test_range(self):
+        seq = range(10, 20)
+        d = SequenceExclude(seq)
+        self.assertIn(5, d)
+        self.assertIn(20, d)
+        self.assertNotIn(15, d)
+
+    def test_six_range(self):
+        import six.moves
+        seq = six.moves.range(10**3, 10**6)
+        d = SequenceExclude(seq)
+        self.assertIn(10**2, d)
+        self.assertIn(10**7, d)
+        self.assertNotIn(10**4, d)
+
+    def test_range_step(self):
+        seq = range(10, 20, 3)
+        d = SequenceExclude(seq)
+        are_not_in = [v for v in range(10, 20, 3) if v in d]
+        self.assertEquals([], are_not_in)
+
+        are_not_in = [v for v in range(10, 20) if v in d]
+        self.assertEquals([11, 12, 14, 15, 17, 18], are_not_in)
+
+    def test_string_seq(self):
+        seq = "cde"
+        d = SequenceExclude(seq)
+        chars = "".join(v for v in "abcdefg" if v in d)
+        self.assertEquals("abfg", chars)
+
+    def test_stringset_seq(self):
+        seq = set("cde")
+        d = SequenceExclude(seq)
+        chars = "".join(v for v in "abcdefg" if v in d)
+        self.assertEquals("abfg", chars)
+
+    def test_set(self):
+        """Anything that is not in the supplied sequence will evaluate as 'in' the container."""
+        seq = set([-3, "test", 1.1])
+        d = SequenceExclude(seq)
+        self.assertIn(0, d)
+        self.assertIn("foo", d)
+        self.assertIn(3.14, d)
+        self.assertIn(set("bar"), d)
+
+        # supplied sequence is a set, so checking for list (non)inclusion fails
+        self.assertRaises(TypeError, (0, 1, 2) in d)
+        self.assertRaises(TypeError, d.__contains__, ['a', 'b', 'c'])
+
+        for v in [-3, "test", 1.1]:
+            self.assertNotIn(v, d)
+
 if __name__ == "__main__":
     unittest.main()
 

From 70a69d2199c3c08a7e16f33ea5d35fd4066eb14b Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 12 Dec 2016 22:40:48 +0100
Subject: [PATCH 1206/2974] Use built-in range()

---
 scrapy/commands/fetch.py | 2 +-
 scrapy/shell.py          | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 6fe6d73b9..7d4840529 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -56,7 +56,7 @@ class Command(ScrapyCommand):
         # by default, let the framework handle redirects,
         # i.e. command handles all codes expect 3xx
         if not opts.no_redirect:
-            request.meta['handle_httpstatus_list'] = SequenceExclude(six.moves.range(300, 400))
+            request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400))
         else:
             request.meta['handle_httpstatus_all'] = True
 
diff --git a/scrapy/shell.py b/scrapy/shell.py
index babc267c7..6f94635a1 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -7,7 +7,6 @@ from __future__ import print_function
 
 import os
 import signal
-from six.moves import range
 import warnings
 
 from twisted.internet import reactor, threads, defer

From 0fc73a9d558158f1686f9cc9c289fe364b5df536 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 16 Dec 2016 21:47:58 +0500
Subject: [PATCH 1207/2974] DOC update examples with long longger names

---
 docs/intro/tutorial.rst               | 24 +++----
 docs/topics/benchmarking.rst          | 92 +++++++++++++++++----------
 docs/topics/downloader-middleware.rst |  8 +--
 docs/topics/settings.rst              |  2 +-
 docs/topics/shell.rst                 | 10 +--
 5 files changed, 81 insertions(+), 55 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 0941eb1e5..8e14d1b7c 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -130,15 +130,15 @@ will send some requests for the ``quotes.toscrape.com`` domain. You will get an
 similar to this::
 
     ... (omitted for brevity)
-    2016-09-20 14:48:00 [scrapy] INFO: Spider opened
-    2016-09-20 14:48:00 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
-    2016-09-20 14:48:00 [scrapy] DEBUG: Telnet console listening on 127.0.0.1:6023
-    2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
-    2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
-    2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-1.html
-    2016-09-20 14:48:01 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
-    2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-2.html
-    2016-09-20 14:48:01 [scrapy] INFO: Closing spider (finished)
+    2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened
+    2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
+    2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023
+    2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
+    2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
+    2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
+    2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html
+    2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html
+    2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished)
     ...
 
 Now, check the files in the current directory. You should notice that two new
@@ -212,7 +212,7 @@ using the shell :ref:`Scrapy shell <topics-shell>`. Run::
 You will see something like::
 
     [ ... Scrapy log here ... ]
-    2016-09-19 12:09:27 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
+    2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
     [s] Available Scrapy objects:
     [s]   scrapy     scrapy module (contains scrapy.Request, scrapy.Selector, etc)
     [s]   crawler    <scrapy.crawler.Crawler object at 0x7fa91d888c90>
@@ -429,9 +429,9 @@ in the callback, as you can see below::
 
 If you run this spider, it will output the extracted data with the log::
 
-    2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
+    2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
     {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'}
-    2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
+    2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
     {'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"}
 
 
diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst
index 632190067..99469ebf1 100644
--- a/docs/topics/benchmarking.rst
+++ b/docs/topics/benchmarking.rst
@@ -18,40 +18,66 @@ To run it use::
 
 You should see an output like this::
 
-    2013-05-16 13:08:46-0300 [scrapy] INFO: Scrapy 0.17.0 started (bot: scrapybot)
-    2013-05-16 13:08:47-0300 [scrapy] INFO: Spider opened
-    2013-05-16 13:08:47-0300 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:48-0300 [scrapy] INFO: Crawled 74 pages (at 4440 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:49-0300 [scrapy] INFO: Crawled 143 pages (at 4140 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:50-0300 [scrapy] INFO: Crawled 210 pages (at 4020 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:51-0300 [scrapy] INFO: Crawled 274 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:52-0300 [scrapy] INFO: Crawled 343 pages (at 4140 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:53-0300 [scrapy] INFO: Crawled 410 pages (at 4020 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:54-0300 [scrapy] INFO: Crawled 474 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:55-0300 [scrapy] INFO: Crawled 538 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:56-0300 [scrapy] INFO: Crawled 602 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:57-0300 [scrapy] INFO: Closing spider (closespider_timeout)
-    2013-05-16 13:08:57-0300 [scrapy] INFO: Crawled 666 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
-    2013-05-16 13:08:57-0300 [scrapy] INFO: Dumping Scrapy stats:
-        {'downloader/request_bytes': 231508,
-         'downloader/request_count': 682,
-         'downloader/request_method_count/GET': 682,
-         'downloader/response_bytes': 1172802,
-         'downloader/response_count': 682,
-         'downloader/response_status_count/200': 682,
-         'finish_reason': 'closespider_timeout',
-         'finish_time': datetime.datetime(2013, 5, 16, 16, 8, 57, 985539),
-         'log_count/INFO': 14,
-         'request_depth_max': 34,
-         'response_received_count': 682,
-         'scheduler/dequeued': 682,
-         'scheduler/dequeued/memory': 682,
-         'scheduler/enqueued': 12767,
-         'scheduler/enqueued/memory': 12767,
-         'start_time': datetime.datetime(2013, 5, 16, 16, 8, 47, 676539)}
-    2013-05-16 13:08:57-0300 [scrapy] INFO: Spider closed (closespider_timeout)
+    2016-12-16 21:18:48 [scrapy.utils.log] INFO: Scrapy 1.2.2 started (bot: quotesbot)
+    2016-12-16 21:18:48 [scrapy.utils.log] INFO: Overridden settings: {'CLOSESPIDER_TIMEOUT': 10, 'ROBOTSTXT_OBEY': True, 'SPIDER_MODULES': ['quotesbot.spiders'], 'LOGSTATS_INTERVAL': 1, 'BOT_NAME': 'quotesbot', 'LOG_LEVEL': 'INFO', 'NEWSPIDER_MODULE': 'quotesbot.spiders'}
+    2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled extensions:
+    ['scrapy.extensions.closespider.CloseSpider',
+     'scrapy.extensions.logstats.LogStats',
+     'scrapy.extensions.telnet.TelnetConsole',
+     'scrapy.extensions.corestats.CoreStats']
+    2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled downloader middlewares:
+    ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware',
+     'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',
+     'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',
+     'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',
+     'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',
+     'scrapy.downloadermiddlewares.retry.RetryMiddleware',
+     'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',
+     'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',
+     'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',
+     'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',
+     'scrapy.downloadermiddlewares.stats.DownloaderStats']
+    2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled spider middlewares:
+    ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
+     'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
+     'scrapy.spidermiddlewares.referer.RefererMiddleware',
+     'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
+     'scrapy.spidermiddlewares.depth.DepthMiddleware']
+    2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled item pipelines:
+    []
+    2016-12-16 21:18:49 [scrapy.core.engine] INFO: Spider opened
+    2016-12-16 21:18:49 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
+    2016-12-16 21:18:50 [scrapy.extensions.logstats] INFO: Crawled 70 pages (at 4200 pages/min), scraped 0 items (at 0 items/min)
+    2016-12-16 21:18:51 [scrapy.extensions.logstats] INFO: Crawled 134 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
+    2016-12-16 21:18:52 [scrapy.extensions.logstats] INFO: Crawled 198 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
+    2016-12-16 21:18:53 [scrapy.extensions.logstats] INFO: Crawled 254 pages (at 3360 pages/min), scraped 0 items (at 0 items/min)
+    2016-12-16 21:18:54 [scrapy.extensions.logstats] INFO: Crawled 302 pages (at 2880 pages/min), scraped 0 items (at 0 items/min)
+    2016-12-16 21:18:55 [scrapy.extensions.logstats] INFO: Crawled 358 pages (at 3360 pages/min), scraped 0 items (at 0 items/min)
+    2016-12-16 21:18:56 [scrapy.extensions.logstats] INFO: Crawled 406 pages (at 2880 pages/min), scraped 0 items (at 0 items/min)
+    2016-12-16 21:18:57 [scrapy.extensions.logstats] INFO: Crawled 438 pages (at 1920 pages/min), scraped 0 items (at 0 items/min)
+    2016-12-16 21:18:58 [scrapy.extensions.logstats] INFO: Crawled 470 pages (at 1920 pages/min), scraped 0 items (at 0 items/min)
+    2016-12-16 21:18:59 [scrapy.core.engine] INFO: Closing spider (closespider_timeout)
+    2016-12-16 21:18:59 [scrapy.extensions.logstats] INFO: Crawled 518 pages (at 2880 pages/min), scraped 0 items (at 0 items/min)
+    2016-12-16 21:19:00 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
+    {'downloader/request_bytes': 229995,
+     'downloader/request_count': 534,
+     'downloader/request_method_count/GET': 534,
+     'downloader/response_bytes': 1565504,
+     'downloader/response_count': 534,
+     'downloader/response_status_count/200': 534,
+     'finish_reason': 'closespider_timeout',
+     'finish_time': datetime.datetime(2016, 12, 16, 16, 19, 0, 647725),
+     'log_count/INFO': 17,
+     'request_depth_max': 19,
+     'response_received_count': 534,
+     'scheduler/dequeued': 533,
+     'scheduler/dequeued/memory': 533,
+     'scheduler/enqueued': 10661,
+     'scheduler/enqueued/memory': 10661,
+     'start_time': datetime.datetime(2016, 12, 16, 16, 18, 49, 799869)}
+    2016-12-16 21:19:00 [scrapy.core.engine] INFO: Spider closed (closespider_timeout)
 
-That tells you that Scrapy is able to crawl about 3900 pages per minute in the
+That tells you that Scrapy is able to crawl about 3000 pages per minute in the
 hardware where you run it. Note that this is a very simple spider intended to
 follow links, any custom spider you write will probably do more stuff which
 results in slower crawl rates. How slower depends on how much your spider does
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 29d9b0298..3b9a5335a 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -238,14 +238,14 @@ header) and all cookies received in responses (ie. ``Set-Cookie`` header).
 
 Here's an example of a log with :setting:`COOKIES_DEBUG` enabled::
 
-    2011-04-06 14:35:10-0300 [scrapy] INFO: Spider opened
-    2011-04-06 14:35:10-0300 [scrapy] DEBUG: Sending cookies to: <GET http://www.diningcity.com/netherlands/index.html>
+    2011-04-06 14:35:10-0300 [scrapy.core.engine] INFO: Spider opened
+    2011-04-06 14:35:10-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Sending cookies to: <GET http://www.diningcity.com/netherlands/index.html>
             Cookie: clientlanguage_nl=en_EN
-    2011-04-06 14:35:14-0300 [scrapy] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html>
+    2011-04-06 14:35:14-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html>
             Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/
             Set-Cookie: ip_isocode=US
             Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/
-    2011-04-06 14:49:50-0300 [scrapy] DEBUG: Crawled (200) <GET http://www.diningcity.com/netherlands/index.html> (referer: None)
+    2011-04-06 14:49:50-0300 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://www.diningcity.com/netherlands/index.html> (referer: None)
     [...]
 
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 503f4afb1..0515a9e0d 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1037,7 +1037,7 @@ Stats counter (``scheduler/unserializable``) tracks the number of times this hap
 
 Example entry in logs::
 
-    1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request:
+    1956-01-31 00:00:00+0800 [scrapy.core.scheduler] ERROR: Unable to serialize request:
     <GET http://example.com> - reason: cannot serialize <Request at 0x9a7c7ec>
     (type Request)> - no more unserializable requests will be logged
     (see 'scheduler/unserializable' stats counter)
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 322c3ddfa..da91108b2 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -173,7 +173,7 @@ all start with the ``[s]`` prefix)::
 After that, we can start playing with the objects::
 
     >>> response.xpath('//title/text()').extract_first()
-    u'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
+    'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
 
     >>> fetch("http://reddit.com")
     [s] Available Scrapy objects:
@@ -189,7 +189,7 @@ After that, we can start playing with the objects::
     [s]   view(response)    View response in a browser
 
     >>> response.xpath('//title/text()').extract()
-    [u'reddit: the front page of the internet']
+    ['reddit: the front page of the internet']
 
     >>> request = request.replace(method="POST")
 
@@ -234,8 +234,8 @@ Here's an example of how you would call it from your spider::
 
 When you run the spider, you will get something similar to this::
 
-    2014-01-23 17:48:31-0400 [scrapy] DEBUG: Crawled (200) <GET http://example.com> (referer: None)
-    2014-01-23 17:48:31-0400 [scrapy] DEBUG: Crawled (200) <GET http://example.org> (referer: None)
+    2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://example.com> (referer: None)
+    2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://example.org> (referer: None)
     [s] Available Scrapy objects:
     [s]   crawler    <scrapy.crawler.Crawler object at 0x1e16b50>
     ...
@@ -258,7 +258,7 @@ Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the
 crawling::
 
     >>> ^D
-    2014-01-23 17:50:03-0400 [scrapy] DEBUG: Crawled (200) <GET http://example.net> (referer: None)
+    2014-01-23 17:50:03-0400 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://example.net> (referer: None)
     ...
 
 Note that you can't use the ``fetch`` shortcut here since the Scrapy engine is

From da19f0b7b73ca4fd78d828e710e111c60bc658e3 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 16 Dec 2016 22:14:54 +0500
Subject: [PATCH 1208/2974] DOC how to override log level for a specific Scrapy
 component

---
 docs/topics/logging.rst | 39 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 231f5186b..ac3b614fc 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -10,7 +10,7 @@ Logging
     about the new logging system.
 
 Scrapy uses `Python's builtin logging system
-<https://docs.python.org/2/library/logging.html>`_ for event logging. We'll
+<https://docs.python.org/3/library/logging.html>`_ for event logging. We'll
 provide some simple examples to get you started, but for more advanced
 use-cases it's strongly suggested to read thoroughly its documentation.
 
@@ -193,6 +193,43 @@ to override some of the Scrapy settings regarding logging.
     Module `logging.handlers <https://docs.python.org/2/library/logging.handlers.html>`_
         Further documentation on available handlers
 
+Advanced customization
+----------------------
+
+Because Scrapy uses stdlib logging module, you can customize logging using
+all features of stdlib logging.
+
+For example, let's say you're scraping a website which returns many
+HTTP 404 and 500 responses, and you want to hide all messages like this::
+
+    2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring
+    response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code
+    is not handled or not allowed
+
+The first thing to note is a logger name - it is in brackets:
+``[scrapy.spidermiddlewares.httperror]``. If you get just ``[scrapy]`` then
+:setting:`LOG_SHORT_NAMES` is likely set to True; set it to False and re-run
+the crawl.
+
+Next, we can see that the message has INFO level. To hide it
+we should set logging level for ``scrapy.spidermiddlewares.httperror``
+higher than INFO; next level after INFO is WARNING. It could be done
+e.g. in the spider's ``__init__`` method::
+
+    import logging
+    import scrapy
+
+
+    class MySpider(scrapy.Spider):
+        # ...
+        def __init__(self, *args, **kwargs):
+            logger = logging.getLogger('scrapy.spidermiddlewares.httperror')
+            logger.setLevel(logging.WARNING)
+            super().__init__(*args, **kwargs)
+
+If you run this spider again then INFO messages from
+``scrapy.spidermiddlewares.httperror`` logger will be gone.
+
 scrapy.utils.log module
 =======================
 

From 2b3abdb7006a2875744a89d45f46e858c981d751 Mon Sep 17 00:00:00 2001
From: zhouyc <zhouyc@knownsec.com>
Date: Mon, 19 Dec 2016 11:27:57 +0800
Subject: [PATCH 1209/2974] update parsel version which will cause attribute
 error

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 4eb8d2318..3ae7915b4 100644
--- a/setup.py
+++ b/setup.py
@@ -44,7 +44,7 @@ setup(
         'pyOpenSSL',
         'cssselect>=0.9',
         'six>=1.5.2',
-        'parsel>=0.9.3',
+        'parsel>=0.9.5',
         'PyDispatcher>=2.0.5',
         'service_identity',
     ],

From 140a57d7b00f0b044598715ab640bd7e96e32318 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 19 Dec 2016 17:51:30 +0100
Subject: [PATCH 1210/2974] Amend note on --no-redirect option for shell tool

---
 docs/topics/commands.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 3a26b19ae..6636c30cb 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -382,7 +382,9 @@ Supported options:
 
 * ``-c code``: evaluate the code in the shell, print the result and exit
 
-* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
+* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them);
+  this only affects the URL you may pass as argument on the command line;
+  once you are inside the shell, ``fetch(url)`` will still follow HTTP redirects by default.
 
 Usage example::
 
@@ -397,6 +399,7 @@ Usage example::
     (200, 'http://example.com/')
 
     # you can disable this with --no-redirect
+    # (only for the URL passed as command line argument)
     $ scrapy shell --no-redirect --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)'
     (302, 'http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F')
 

From 6dec4a3ccb455511decc4c1414b9042c57a205ea Mon Sep 17 00:00:00 2001
From: Rolando Espinoza <rndmax84@gmail.com>
Date: Tue, 20 Dec 2016 20:02:31 -0400
Subject: [PATCH 1211/2974] ENH Pass arguments to logger rather than formatted
 message.

This not only use the standard form but helps error aggregation
libraries (i.e.: Sentry) to avoid duplicating the message.
---
 scrapy/core/downloader/handlers/http11.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 54aa359fb..ecd7f90d3 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -319,14 +319,13 @@ class ScrapyAgent(object):
         expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1
 
         if maxsize and expected_size > maxsize:
-            error_message = ("Cancelling download of {url}: expected response "
-                             "size ({size}) larger than "
-                             "download max size ({maxsize})."
-            ).format(url=request.url, size=expected_size, maxsize=maxsize)
+            error_msg = ("Cancelling download of %(url)s: expected response "
+                         "size (%(size)s) larger than download max size (%(maxsize)s).")
+            error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
 
-            logger.error(error_message)
+            logger.error(error_msg, error_args)
             txresponse._transport._producer.loseConnection()
-            raise defer.CancelledError(error_message)
+            raise defer.CancelledError(error_msg % error_args)
 
         if warnsize and expected_size > warnsize:
             logger.warning("Expected response size (%(size)s) larger than "

From e9b3cf01f4393c50b4a10f2032db0fecae5aa5f4 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 9 Dec 2016 17:23:59 +0100
Subject: [PATCH 1212/2974] Update changelog for upcoming 1.3.0 release

---
 docs/news.rst | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 5e28fb130..5f759f0fc 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,26 @@
 Release notes
 =============
 
+Scrapy 1.3.0 (2016-12-XX)
+-------------------------
+
+New Features
+~~~~~~~~~~~~
+
+- ``MailSender`` now accepts single strings as values for ``to`` and ``cc``
+  arguments (:issue:`2272`)
+
+Dependencies & Cleanups
+~~~~~~~~~~~~~~~~~~~~~~~
+
+- Scrapy now requires Twisted >= 13.1 which is the case for many Linux
+  distributions already.
+- As a consequence, we got rid of ``scrapy.xlib.tx.*`` modules, which
+  copied some of Twisted code for users stuck with an "old" Twisted version
+- ``ChunkedTransferMiddleware`` is deprecated and removed from the default
+  downloader middlewares.
+
+
 Scrapy 1.2.2 (2016-12-06)
 -------------------------
 

From 9d5afd8c35410e2874a97a17b1a2b5f4deef2b38 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 16 Dec 2016 18:46:49 +0100
Subject: [PATCH 1213/2974] Add note on HttpErrorMiddleware new logging level

---
 docs/news.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 5f759f0fc..a7cc30b0b 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -22,6 +22,12 @@ Dependencies & Cleanups
 - ``ChunkedTransferMiddleware`` is deprecated and removed from the default
   downloader middlewares.
 
+Logging
+~~~~~~~
+
+- ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``;
+  this is technically **backwards incompatible** so please check your log parsers.
+
 
 Scrapy 1.2.2 (2016-12-06)
 -------------------------

From 9098001888a57ac4db030fc6563d541174f47d47 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 19 Dec 2016 17:09:47 +0100
Subject: [PATCH 1214/2974] Add note on LOG_SHORT_NAMES

---
 docs/news.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index a7cc30b0b..02227aa8e 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -27,6 +27,11 @@ Logging
 
 - ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``;
   this is technically **backwards incompatible** so please check your log parsers.
+- By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``,
+  instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``);
+  this is **backwards incompatible** if you have log parsers expecting the short
+  logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES`
+  set to ``True``.
 
 
 Scrapy 1.2.2 (2016-12-06)

From 49a84c2d414386b27a3f9e4439c5a967ce6cb9bc Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 21 Dec 2016 14:32:21 +0100
Subject: [PATCH 1215/2974] Add note on HTTP redirects with shell and fetch

---
 docs/news.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 02227aa8e..02976fedf 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -11,6 +11,9 @@ New Features
 
 - ``MailSender`` now accepts single strings as values for ``to`` and ``cc``
   arguments (:issue:`2272`)
+- ``scrapy fetch url``, ``scrapy shell url`` and ``fetch(url)`` inside
+  scrapy shell now follow HTTP redirections by default (:issue:`2290`);
+  See :command:`fetch` and :command:`shell` for details.
 
 Dependencies & Cleanups
 ~~~~~~~~~~~~~~~~~~~~~~~

From 4eeec3e42cf4a48dd3481e43ca064576a4a825d7 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 21 Dec 2016 15:59:18 +0100
Subject: [PATCH 1216/2974] Add preamble on why 1.3.0 comes so soon after 1.2.2

---
 docs/news.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 02976fedf..09c47dc30 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -6,6 +6,14 @@ Release notes
 Scrapy 1.3.0 (2016-12-XX)
 -------------------------
 
+This release comes rather soon after 1.2.2 for one main reason:
+it was found out that releases since 0.18 up to 1.2.2 (included) use
+some backported code from Twisted, even if newer Twisted modules are available.
+Scrapy now uses ``twisted.web.client`` and ``twisted.internet.endpoints`` directly.
+
+As it is a major change, we wanted to get the bug fix out quickly
+while not breaking any projects using the 1.2 series.
+
 New Features
 ~~~~~~~~~~~~
 

From d60a6899dfe0fa5ac08298fa466b71e4f221b93e Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 21 Dec 2016 16:01:36 +0100
Subject: [PATCH 1217/2974] Merge logging changes into "New features" section

---
 docs/news.rst | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 09c47dc30..f42edba30 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -22,6 +22,13 @@ New Features
 - ``scrapy fetch url``, ``scrapy shell url`` and ``fetch(url)`` inside
   scrapy shell now follow HTTP redirections by default (:issue:`2290`);
   See :command:`fetch` and :command:`shell` for details.
+- ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``;
+  this is technically **backwards incompatible** so please check your log parsers.
+- By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``,
+  instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``);
+  this is **backwards incompatible** if you have log parsers expecting the short
+  logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES`
+  set to ``True``.
 
 Dependencies & Cleanups
 ~~~~~~~~~~~~~~~~~~~~~~~
@@ -33,17 +40,6 @@ Dependencies & Cleanups
 - ``ChunkedTransferMiddleware`` is deprecated and removed from the default
   downloader middlewares.
 
-Logging
-~~~~~~~
-
-- ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``;
-  this is technically **backwards incompatible** so please check your log parsers.
-- By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``,
-  instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``);
-  this is **backwards incompatible** if you have log parsers expecting the short
-  logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES`
-  set to ``True``.
-
 
 Scrapy 1.2.2 (2016-12-06)
 -------------------------

From b9e7ca044cfb82fc9592cef7e2fdcfa422fe8d44 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 21 Dec 2016 16:11:19 +0100
Subject: [PATCH 1218/2974] Reword things a tiny bit

---
 docs/news.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index f42edba30..6690ca558 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -8,8 +8,10 @@ Scrapy 1.3.0 (2016-12-XX)
 
 This release comes rather soon after 1.2.2 for one main reason:
 it was found out that releases since 0.18 up to 1.2.2 (included) use
-some backported code from Twisted, even if newer Twisted modules are available.
+some backported code from Twisted (``scrapy.xlib.tx.*``),
+even if newer Twisted modules are available.
 Scrapy now uses ``twisted.web.client`` and ``twisted.internet.endpoints`` directly.
+(See also cleanups below.)
 
 As it is a major change, we wanted to get the bug fix out quickly
 while not breaking any projects using the 1.2 series.

From f8793e2460977a0c218900687cd7c6d592a5c7d5 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 21 Dec 2016 16:27:01 +0100
Subject: [PATCH 1219/2974] Set release date for 1.3.0

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 6690ca558..cce46599b 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,7 +3,7 @@
 Release notes
 =============
 
-Scrapy 1.3.0 (2016-12-XX)
+Scrapy 1.3.0 (2016-12-21)
 -------------------------
 
 This release comes rather soon after 1.2.2 for one main reason:

From ac74d5a467908a8c3db05f80e9d533be2c9deb64 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 21 Dec 2016 16:28:44 +0100
Subject: [PATCH 1220/2974] =?UTF-8?q?Bump=20version:=201.2.2=20=E2=86=92?=
 =?UTF-8?q?=201.3.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index ee039790d..57ff603fa 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.2.2
+current_version = 1.3.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 23aa83906..f0bb29e76 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.2.2
+1.3.0

From 07f9985a941d2fce4d7115a35dc983bc21ded4be Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 21 Dec 2016 17:03:11 +0100
Subject: [PATCH 1221/2974] TST: Randomize FILES_EXPIRES above 90 days

---
 tests/test_pipeline_files.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 157c21a89..e3ec04b8d 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -208,7 +208,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
             return "".join([chr(random.randint(97, 123)) for _ in range(10)])
 
         settings = {
-            "FILES_EXPIRES": random.randint(1, 1000),
+            "FILES_EXPIRES": random.randint(100, 1000),
             "FILES_URLS_FIELD": random_string(),
             "FILES_RESULT_FIELD": random_string(),
             "FILES_STORE": self.tempdir

From e7c7e055ff3c661cd60b15fe5ad910a2765db707 Mon Sep 17 00:00:00 2001
From: Raul Gallegos <raul.ogh@gmail.com>
Date: Sat, 24 Dec 2016 11:55:04 -0500
Subject: [PATCH 1222/2974] settings: fixing name of the pipeline template

---
 scrapy/templates/project/module/settings.py.tmpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 72f25ebef..486df6b71 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -65,7 +65,7 @@ ROBOTSTXT_OBEY = True
 # Configure item pipelines
 # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
 #ITEM_PIPELINES = {
-#    '$project_name.pipelines.SomePipeline': 300,
+#    '$project_name.pipelines.${ProjectName}Pipeline': 300,
 #}
 
 # Enable and configure the AutoThrottle extension (disabled by default)

From 2240f00a136e264cc1856860fddba85ec97432c1 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Tue, 1 Mar 2016 07:12:19 +0000
Subject: [PATCH 1223/2974] Remove dependency on os.environ from default
 settings

Avoid loading settings from environment in scrapy core.
Instead it's better to populate them from the starting
shell or an embedding script.
---
 docs/topics/commands.rst            |  6 +++---
 docs/topics/settings.rst            |  8 ++++----
 scrapy/cmdline.py                   |  8 +++++++-
 scrapy/commands/edit.py             |  4 +++-
 scrapy/settings/default_settings.py | 11 +++--------
 5 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 6636c30cb..935e3281e 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -291,12 +291,12 @@ edit
 * Syntax: ``scrapy edit <spider>``
 * Requires project: *yes*
 
-Edit the given spider using the editor defined in the :setting:`EDITOR`
-setting.
+Edit the given spider using the editor defined in the ``EDITOR`` environment
+variable or (if unset) the :setting:`EDITOR` setting.
 
 This command is provided only as a convenience shortcut for the most common
 case, the developer is of course free to choose any tool or IDE to write and
-debug his spiders.
+debug spiders.
 
 Usage example::
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0515a9e0d..1bb7a9d2b 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -635,11 +635,11 @@ Setting :setting:`DUPEFILTER_DEBUG` to ``True`` will make it log all duplicate r
 EDITOR
 ------
 
-Default: `depends on the environment`
+Default: ``vi`` (on Unix systems) or the IDLE editor (on Windows)
 
-The editor to use for editing spiders with the :command:`edit` command. It
-defaults to the ``EDITOR`` environment variable, if set. Otherwise, it defaults
-to ``vi`` (on Unix systems) or the IDLE editor (on Windows).
+The editor to use for editing spiders with the :command:`edit` command.
+Additionally, if the ``EDITOR`` environment variable is set, the :command:`edit`
+command will prefer it over the default setting.
 
 .. setting:: EXTENSIONS
 
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index cb7bbd64d..dca931e99 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -1,5 +1,5 @@
 from __future__ import print_function
-import sys
+import sys, os
 import optparse
 import cProfile
 import inspect
@@ -106,6 +106,12 @@ def execute(argv=None, settings=None):
 
     if settings is None:
         settings = get_project_settings()
+        # set EDITOR from environment if available
+        try:
+            editor = os.environ['EDITOR']
+        except KeyError: pass
+        else:
+            settings['EDITOR'] = editor
     check_deprecated_settings(settings)
 
     # --- backwards compatibility for scrapy.conf.settings singleton ---
diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py
index 2df6a730c..a7f8983b4 100644
--- a/scrapy/commands/edit.py
+++ b/scrapy/commands/edit.py
@@ -3,6 +3,7 @@ import sys, os
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 
+
 class Command(ScrapyCommand):
 
     requires_project = True
@@ -15,7 +16,8 @@ class Command(ScrapyCommand):
         return "Edit spider"
 
     def long_desc(self):
-        return "Edit a spider using the editor defined in EDITOR setting"
+        return ("Edit a spider using the editor defined in the EDITOR environment"
+                " variable or else the EDITOR setting")
 
     def _err(self, msg):
         sys.stderr.write(msg + os.linesep)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 24714a7a8..f687ef6b1 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -13,7 +13,6 @@ Scrapy developers, if you add a setting here remember to:
 
 """
 
-import os
 import sys
 from importlib import import_module
 from os.path import join, abspath, dirname
@@ -111,13 +110,9 @@ DOWNLOADER_STATS = True
 
 DUPEFILTER_CLASS = 'scrapy.dupefilters.RFPDupeFilter'
 
-try:
-    EDITOR = os.environ['EDITOR']
-except KeyError:
-    if sys.platform == 'win32':
-        EDITOR = '%s -m idlelib.idle'
-    else:
-        EDITOR = 'vi'
+EDITOR = 'vi'
+if sys.platform == 'win32':
+    EDITOR = '%s -m idlelib.idle'
 
 EXTENSIONS = {}
 

From 9922ec15d7a79ac109bf02ccd9ac154bca5d5167 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 27 Dec 2016 09:52:35 -0200
Subject: [PATCH 1224/2974] mention contributing document before CoC in the
 README

---
 README.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index b72ebf53d..38ad43eba 100644
--- a/README.rst
+++ b/README.rst
@@ -73,14 +73,14 @@ See http://scrapy.org/community/
 Contributing
 ============
 
+See http://doc.scrapy.org/en/master/contributing.html
+
 Please note that this project is released with a Contributor Code of Conduct
 (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
 
 By participating in this project you agree to abide by its terms.
 Please report unacceptable behavior to opensource@scrapinghub.com.
 
-See http://doc.scrapy.org/en/master/contributing.html
-
 Companies using Scrapy
 ======================
 

From dabcb17d72e5bf0ab3a0d7cfe50f70aa2529cd09 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 27 Dec 2016 11:32:15 -0200
Subject: [PATCH 1225/2974] update code of conduct
 http://contributor-covenant.org/version/1/4

---
 CODE_OF_CONDUCT.md | 80 ++++++++++++++++++++++++++++++----------------
 1 file changed, 52 insertions(+), 28 deletions(-)

diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
index 95b4a7e3c..162602248 100644
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -1,24 +1,41 @@
-# Contributor Code of Conduct
+# Contributor Covenant Code of Conduct
 
-As contributors and maintainers of this project, and in the interest of
-fostering an open and welcoming community, we pledge to respect all people who
-contribute through reporting issues, posting feature requests, updating
-documentation, submitting pull requests or patches, and other activities.
+## Our Pledge
 
-We are committed to making participation in this project a harassment-free
-experience for everyone, regardless of level of experience, gender, gender
-identity and expression, sexual orientation, disability, personal appearance,
-body size, race, ethnicity, age, religion, or nationality.
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
 
 Examples of unacceptable behavior by participants include:
 
-* The use of sexualized language or imagery
-* Personal attacks
-* Trolling or insulting/derogatory comments
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
 * Public or private harassment
-* Publishing other's private information, such as physical or electronic
-  addresses, without explicit permission
-* Other unethical or unprofessional conduct
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
 
 Project maintainers have the right and responsibility to remove, edit, or
 reject comments, commits, code, wiki edits, issues, and other contributions
@@ -26,25 +43,32 @@ that are not aligned to this Code of Conduct, or to ban temporarily or
 permanently any contributor for other behaviors that they deem inappropriate,
 threatening, offensive, or harmful.
 
-By adopting this Code of Conduct, project maintainers commit themselves to
-fairly and consistently applying these principles to every aspect of managing
-this project. Project maintainers who do not follow or enforce the Code of
-Conduct may be permanently removed from the project team.
+## Scope
 
 This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community.
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting a project maintainer at opensource@scrapinghub.com. All
+reported by contacting the project team at opensource@scrapinghub.com. All
 complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. Maintainers are
-obligated to maintain confidentiality with regard to the reporter of an
-incident.
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
 
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
 
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 1.3.0, available at
-[http://contributor-covenant.org/version/1/3/0/][version]
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
 
 [homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/3/0/
+[version]: http://contributor-covenant.org/version/1/4/

From b7dd089bf9d041218012ce1da7bce5045e76d37a Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 27 Dec 2016 11:33:23 -0200
Subject: [PATCH 1226/2974] show CoC in its own section

---
 README.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.rst b/README.rst
index 38ad43eba..38dda62e5 100644
--- a/README.rst
+++ b/README.rst
@@ -75,6 +75,9 @@ Contributing
 
 See http://doc.scrapy.org/en/master/contributing.html
 
+Code of Conduct
+---------------
+
 Please note that this project is released with a Contributor Code of Conduct
 (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
 

From 8a86574394172a8221a95d9e2fcc229a26be2103 Mon Sep 17 00:00:00 2001
From: Alex <chekunkov@gmail.com>
Date: Wed, 28 Dec 2016 14:10:50 +0000
Subject: [PATCH 1227/2974] .devN release suffix must be preceded with a dot

https://packaging.python.org/distributing/#standards-compliance-for-interoperability
---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 506f3779b..ac79a0425 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,7 +5,7 @@ branches:
   only:
     - master
     - /^\d\.\d+$/
-    - /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/
+    - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
 env:
  - TOXENV=py27
  - TOXENV=jessie
@@ -35,4 +35,4 @@ deploy:
   on:
     tags: true
     repo: scrapy/scrapy
-    condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|dev[0-9]+)?$"
+    condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"

From b6ab1ae9c3ffa7aeee0e7ee77e2dd55d7b663a30 Mon Sep 17 00:00:00 2001
From: Elias Dorneles <eliasdorneles@gmail.com>
Date: Tue, 3 Jan 2017 15:14:59 -0200
Subject: [PATCH 1228/2974] docs: installation instructions, mention conda in
 the beginning (closes #2475)

---
 docs/intro/install.rst | 74 +++++++++++-------------------------------
 1 file changed, 19 insertions(+), 55 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 767749ec5..86387ef5e 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -7,14 +7,25 @@ Installation guide
 Installing Scrapy
 =================
 
-Scrapy runs on Python 2.7 and Python 3.3 or above
-(except on Windows where Python 3 is not supported yet).
+Scrapy runs on Python 2.7 and Python 3.3 or above.
 
-If you’re already familiar with installation of Python packages,
+If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
+the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
+and OS X.
+
+To install Scrapy using ``conda``, run::
+
+  conda install -c conda-forge scrapy
+
+Alternatively, if you’re already familiar with installation of Python packages,
 you can install Scrapy and its dependencies from PyPI with::
 
     pip install Scrapy
 
+Note that sometimes this may require solving compilation issues for some Scrapy
+dependencies depending on your operating system, so be sure to check the
+:ref:`intro-install-platform-notes`.
+
 We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
 to avoid conflicting with your system packages.
 
@@ -108,42 +119,14 @@ Platform specific installation notes
 Windows
 -------
 
-* Install Python 2.7 from https://www.python.org/downloads/
+Though it's possible to install Scrapy on Windows using pip, we recommend you
+to install `Anaconda`_ or `Miniconda`_ and use the package from the
+`conda-forge`_ channel, which will avoid most installation issues.
 
-  You need to adjust ``PATH`` environment variable to include paths to
-  the Python executable and additional scripts. The following paths need to be
-  added to ``PATH``::
+Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with::
 
-      C:\Python27\;C:\Python27\Scripts\;
+  conda install -c conda-forge scrapy
 
-  To update the ``PATH`` open a Command prompt and run::
-
-      c:\python27\python.exe c:\python27\tools\scripts\win_add2path.py
-
-  Close the command prompt window and reopen it so changes take effect, run the
-  following command and check it shows the expected Python version::
-
-      python --version
-
-* Install `pywin32` from http://sourceforge.net/projects/pywin32/
-
-  Be sure you download the architecture (win32 or amd64) that matches your system
-
-* *(Only required for Python<2.7.9)* Install `pip`_ from
-  https://pip.pypa.io/en/latest/installing/
-
-  Now open a Command prompt to check ``pip`` is installed correctly::
-
-      pip --version
-
-* At this point Python 2.7 and ``pip`` package manager must be working, let's
-  install Scrapy::
-
-      pip install Scrapy
-
-.. note::
-     Python 3 is not supported on Windows. This is because Scrapy core requirement Twisted does not support
-     Python 3 on Windows.
 
 Ubuntu 12.04 or above
 ---------------------
@@ -234,27 +217,8 @@ After any of these workarounds you should be able to install Scrapy::
   pip install Scrapy
 
 
-Anaconda
---------
-
-
-Using Anaconda is an alternative to using a virtualenv and installing with ``pip``.
-
-.. note::
-
-  For Windows users, or if you have issues installing through ``pip``, this is
-  the recommended way to install Scrapy.
-
-If you already have `Anaconda`_ or `Miniconda`_ installed, the `conda-forge`_
-community have up-to-date packages for Linux, Windows and OS X.
-
-To install Scrapy using ``conda``, run::
-
-  conda install -c conda-forge scrapy
-
 .. _Python: https://www.python.org/
 .. _pip: https://pip.pypa.io/en/latest/installing/
-.. _Control Panel: https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx
 .. _lxml: http://lxml.de/
 .. _parsel: https://pypi.python.org/pypi/parsel
 .. _w3lib: https://pypi.python.org/pypi/w3lib

From a21473147160e4b176fedb32c92e1b68b4af04f6 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 6 Jan 2017 15:38:35 +0100
Subject: [PATCH 1229/2974] Add Python 3.6 tox env + Travis CI build for it

---
 .travis.yml | 1 +
 tox.ini     | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 506f3779b..b0ac3afde 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,6 +11,7 @@ env:
  - TOXENV=jessie
  - TOXENV=py33
  - TOXENV=py35
+ - TOXENV=py36
  - TOXENV=docs
 install:
  - pip install -U tox twine wheel codecov
diff --git a/tox.ini b/tox.ini
index 812302b4c..bdc14a128 100644
--- a/tox.ini
+++ b/tox.ini
@@ -70,6 +70,10 @@ deps = {[testenv:py33]deps}
 basepython = python3.5
 deps = {[testenv:py33]deps}
 
+[testenv:py36]
+basepython = python3.6
+deps = {[testenv:py33]deps}
+
 [docs]
 changedir = docs
 deps =

From 40851a3d4cde6b44cc92e03b7272b3acd33ed7f6 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 6 Jan 2017 15:44:32 +0100
Subject: [PATCH 1230/2974] Use Python 3.6-dev on Travis

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index b0ac3afde..5c9eb8cc7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,5 @@
 language: python
-python: 3.5
+python: 3.6-dev
 sudo: false
 branches:
   only:

From 53769245f553355ad1907ad0b47a05093ec65cbe Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 6 Jan 2017 16:02:14 +0100
Subject: [PATCH 1231/2974] Use python 3.6 directly

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 5c9eb8cc7..4c4adb948 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,5 @@
 language: python
-python: 3.6-dev
+python: 3.6
 sudo: false
 branches:
   only:

From 6b838b02966902311ab869bb6a35e023265ed274 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 6 Jan 2017 16:10:14 +0100
Subject: [PATCH 1232/2974] Use matrix build config

---
 .travis.yml | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 4c4adb948..52fbf02ae 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,18 +1,24 @@
 language: python
-python: 3.6
 sudo: false
 branches:
   only:
     - master
     - /^\d\.\d+$/
     - /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/
-env:
- - TOXENV=py27
- - TOXENV=jessie
- - TOXENV=py33
- - TOXENV=py35
- - TOXENV=py36
- - TOXENV=docs
+matrix:
+  include:
+    - python: 2.7
+      env: TOXENV=py27
+    - python: 2.7
+      env: TOXENV=jessie
+    - python: 3.3
+      env: TOXENV=py33
+    - python: 3.5
+      env: TOXENV=py35
+    - python: 3.6
+      env: TOXENV=py36
+    - python: 3.6
+      env: TOXENV=docs
 install:
  - pip install -U tox twine wheel codecov
 script: tox

From b3406677b980656751af9a12ddf2d33ad884fbc1 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 9 Jan 2017 14:40:02 +0100
Subject: [PATCH 1233/2974] Update classifiers in setup.py

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index f673b1dc4..388cf0dec 100644
--- a/setup.py
+++ b/setup.py
@@ -36,6 +36,7 @@ setup(
         'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',

From 900b6710d3c9d70c740707d72c611c50d909616b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?=
 =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?=
 =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?=
 <digenishjkl@gmail.com>
Date: Thu, 5 Jan 2017 12:53:00 +0200
Subject: [PATCH 1234/2974] Document copying of spider arguments to attributes

---
 docs/topics/spiders.rst | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 0179e9284..29106e87a 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -297,6 +297,37 @@ Spiders can access arguments in their `__init__` methods::
             self.start_urls = ['http://www.example.com/categories/%s' % category]
             # ...
 
+The default `__init__` method will take any spider arguments
+and copy them to the spider as attributes.
+The above example can also be written as follows::
+
+    import scrapy
+
+    class MySpider(scrapy.Spider):
+        name = 'myspider'
+
+        def start_requests(self):
+            yield scrapy.Request('http://www.example.com/categories/%s' % self.category)
+
+Keep in mind that spider arguments are only strings.
+The spider will not do any parsing on its own.
+If you were to set the `start_urls` attribute from the command line,
+you would have to parse it on you own into a list
+using something like
+`ast.literal_eval <https://docs.python.org/library/ast.html#ast.literal_eval>`_
+or `json.loads <https://docs.python.org/library/json.html#json.loads>`_
+and then set it as an attribute.
+Otherwise, you would cause iteration over a `start_urls` string
+(a very common python pitfall)
+resulting in each character being seen as a separate url.
+
+A valid use case is to set the http auth credentials
+used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`
+or the user agent
+used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`::
+
+    scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword -a user_agent=mybot
+
 Spider arguments can also be passed through the Scrapyd ``schedule.json`` API.
 See `Scrapyd documentation`_.
 

From bf2277a028aee0feaecd90a3a64c53bba4e5fbdf Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 10 Jan 2017 10:27:38 +0100
Subject: [PATCH 1235/2974] Update spiders.rst

---
 docs/topics/spiders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 29106e87a..c123c2635 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -312,7 +312,7 @@ The above example can also be written as follows::
 Keep in mind that spider arguments are only strings.
 The spider will not do any parsing on its own.
 If you were to set the `start_urls` attribute from the command line,
-you would have to parse it on you own into a list
+you would have to parse it on your own into a list
 using something like
 `ast.literal_eval <https://docs.python.org/library/ast.html#ast.literal_eval>`_
 or `json.loads <https://docs.python.org/library/json.html#json.loads>`_

From 5586fc7e3876d5a5e8d1797c73cb9b743c8097d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Paulius=20Aleksi=C5=ABnas?= <paulius.aleksiunas@gmail.com>
Date: Tue, 10 Jan 2017 11:12:42 +0200
Subject: [PATCH 1236/2974] Update architecture.rst

In the data flow image arrows are red.
---
 docs/topics/architecture.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index ea0cb0ea7..4ac39ad2d 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -12,7 +12,7 @@ Overview
 
 The following diagram shows an overview of the Scrapy architecture with its
 components and an outline of the data flow that takes place inside the system
-(shown by the green arrows). A brief description of the components is included
+(shown by the red arrows). A brief description of the components is included
 below with links for more detailed information about them. The data flow is
 also described below.
 

From df1a42419f8bce48b605087937320af1ec968116 Mon Sep 17 00:00:00 2001
From: Raul Gallegos <raul.ogh@gmail.com>
Date: Sat, 14 Jan 2017 20:45:20 -0500
Subject: [PATCH 1237/2974] adding formid to FormRequest documentation

---
 docs/topics/request-response.rst | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 664a7239f..a1bd1e146 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -358,7 +358,7 @@ fields with form data from :class:`Response` objects.
     The :class:`FormRequest` objects support the following class method in
     addition to the standard :class:`Request` methods:
 
-    .. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...])
+    .. classmethod:: FormRequest.from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...])
 
        Returns a new :class:`FormRequest` object with its form field values
        pre-populated with those found in the HTML ``<form>`` element contained
@@ -383,6 +383,9 @@ fields with form data from :class:`Response` objects.
        :param formname: if given, the form with name attribute set to this value will be used.
        :type formname: string
 
+       :param formid: if given, the form with id attribute set to this value will be used.
+       :type formid: string
+
        :param formxpath: if given, the first form that matches the xpath will be used.
        :type formxpath: string
 
@@ -421,6 +424,9 @@ fields with form data from :class:`Response` objects.
        .. versionadded:: 1.1.0
           The ``formcss`` parameter.
 
+       .. versionadded:: 1.1.0
+          The ``formid`` parameter.
+
 Request usage examples
 ----------------------
 

From b279bc8546994f5610836e63490b4a7a262b3d77 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 19 Jan 2017 16:28:52 +0100
Subject: [PATCH 1238/2974] Fix view command against new --no-redirect option

---
 scrapy/commands/view.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index 4eb44f77d..59592d08b 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -11,7 +11,7 @@ class Command(fetch.Command):
             "contents in a browser"
 
     def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+        super(Command, self).add_options(parser)
         parser.add_option("--spider", dest="spider",
             help="use this spider")
 

From 299544416a05c126838db3c75b6fc154b6a08de1 Mon Sep 17 00:00:00 2001
From: Raul Gallegos <raul.ogh@gmail.com>
Date: Sun, 22 Jan 2017 19:23:44 -0500
Subject: [PATCH 1239/2974] changing README to README.rst

---
 artwork/{README => README.rst} | 2 ++
 docs/{README => README.rst}    | 2 ++
 sep/{README => README.rst}     | 2 ++
 3 files changed, 6 insertions(+)
 rename artwork/{README => README.rst} (97%)
 rename docs/{README => README.rst} (99%)
 rename sep/{README => README.rst} (95%)

diff --git a/artwork/README b/artwork/README.rst
similarity index 97%
rename from artwork/README
rename to artwork/README.rst
index c185d57da..016462f2c 100644
--- a/artwork/README
+++ b/artwork/README.rst
@@ -1,3 +1,5 @@
+:orphan:
+
 Scrapy artwork
 ==============
 
diff --git a/docs/README b/docs/README.rst
similarity index 99%
rename from docs/README
rename to docs/README.rst
index cf04965ac..733af2af4 100644
--- a/docs/README
+++ b/docs/README.rst
@@ -1,3 +1,5 @@
+:orphan:
+
 ======================================
 Scrapy documentation quick start guide
 ======================================
diff --git a/sep/README b/sep/README.rst
similarity index 95%
rename from sep/README
rename to sep/README.rst
index 668772492..e2d2e6274 100644
--- a/sep/README
+++ b/sep/README.rst
@@ -1,3 +1,5 @@
+:orphan:
+
 Scrapy Enhancement Proposals
 ============================
 

From 53757e51e56ace34bef8616451cca921da16791c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 24 Jan 2017 11:29:11 -0300
Subject: [PATCH 1240/2974] Preserve request class when converting to/from
 dicts (utils.reqser)

---
 scrapy/utils/reqser.py     |  6 +++++-
 tests/test_utils_reqser.py | 13 ++++++++++++-
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 7e1e99e48..2fceb0d94 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -5,6 +5,7 @@ import six
 
 from scrapy.http import Request
 from scrapy.utils.python import to_unicode, to_native_str
+from scrapy.utils.misc import load_object
 
 
 def request_to_dict(request, spider=None):
@@ -32,6 +33,8 @@ def request_to_dict(request, spider=None):
         'priority': request.priority,
         'dont_filter': request.dont_filter,
     }
+    if type(request) is not Request:
+        d['_class'] = request.__module__ + '.' + request.__class__.__name__
     return d
 
 
@@ -47,7 +50,8 @@ def request_from_dict(d, spider=None):
     eb = d['errback']
     if eb and spider:
         eb = _get_method(spider, eb)
-    return Request(
+    request_cls = load_object(d['_class']) if '_class' in d else Request
+    return request_cls(
         url=to_native_str(d['url']),
         callback=cb,
         errback=eb,
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index a62f13e21..5b889ab5d 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import unittest
 
-from scrapy.http import Request
+from scrapy.http import Request, FormRequest
 from scrapy.spiders import Spider
 from scrapy.utils.reqser import request_to_dict, request_from_dict
 
@@ -42,6 +42,7 @@ class RequestSerializationTest(unittest.TestCase):
         self._assert_same_request(request, request2)
 
     def _assert_same_request(self, r1, r2):
+        self.assertEqual(r1.__class__, r2.__class__)
         self.assertEqual(r1.url, r2.url)
         self.assertEqual(r1.callback, r2.callback)
         self.assertEqual(r1.errback, r2.errback)
@@ -54,6 +55,12 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertEqual(r1.priority, r2.priority)
         self.assertEqual(r1.dont_filter, r2.dont_filter)
 
+    def test_request_class(self):
+        r = FormRequest("http://www.example.com")
+        self._assert_serializes_ok(r, spider=self.spider)
+        r = CustomRequest("http://www.example.com")
+        self._assert_serializes_ok(r, spider=self.spider)
+
     def test_callback_serialization(self):
         r = Request("http://www.example.com", callback=self.spider.parse_item,
                     errback=self.spider.handle_error)
@@ -77,3 +84,7 @@ class TestSpider(Spider):
 
     def handle_error(self, failure):
         pass
+
+
+class CustomRequest(Request):
+    pass

From bae12870bbb4e6f589887ad6036e50e5db64369f Mon Sep 17 00:00:00 2001
From: Michael Fladischer <FladischerMichael@fladi.at>
Date: Tue, 24 Jan 2017 22:20:37 +0100
Subject: [PATCH 1241/2974] Fix spelling error in scrapy.1 manpage.

The word "intepreted" should be "interpreted".
---
 extras/scrapy.1 | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/extras/scrapy.1 b/extras/scrapy.1
index a4f29569b..84f693fc3 100644
--- a/extras/scrapy.1
+++ b/extras/scrapy.1
@@ -28,16 +28,16 @@ Query Scrapy settings
 Print raw setting value
 .TP
 .I --getbool=SETTING
-Print setting value, intepreted as a boolean
+Print setting value, interpreted as a boolean
 .TP
 .I --getint=SETTING
-Print setting value, intepreted as an integer
+Print setting value, interpreted as an integer
 .TP
 .I --getfloat=SETTING
-Print setting value, intepreted as an float
+Print setting value, interpreted as an float
 .TP
 .I --getlist=SETTING
-Print setting value, intepreted as an float
+Print setting value, interpreted as an float
 .TP
 .I --init
 Print initial setting value (before loading extensions and spiders)

From 87472346df6ce0ca63842e50d548a423b606767d Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 25 Jan 2017 11:28:20 +0100
Subject: [PATCH 1242/2974] Update scrapy.1

---
 extras/scrapy.1 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/extras/scrapy.1 b/extras/scrapy.1
index 84f693fc3..2fa8d8231 100644
--- a/extras/scrapy.1
+++ b/extras/scrapy.1
@@ -34,10 +34,10 @@ Print setting value, interpreted as a boolean
 Print setting value, interpreted as an integer
 .TP
 .I --getfloat=SETTING
-Print setting value, interpreted as an float
+Print setting value, interpreted as a float
 .TP
 .I --getlist=SETTING
-Print setting value, interpreted as an float
+Print setting value, interpreted as a float
 .TP
 .I --init
 Print initial setting value (before loading extensions and spiders)

From fc07711614549ce8f0464b3fca5b0f4acd746681 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 30 Jan 2017 15:54:28 +0100
Subject: [PATCH 1243/2974] Remove unused --headers option for view command

---
 scrapy/commands/view.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index 59592d08b..59e665016 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -12,8 +12,7 @@ class Command(fetch.Command):
 
     def add_options(self, parser):
         super(Command, self).add_options(parser)
-        parser.add_option("--spider", dest="spider",
-            help="use this spider")
+        parser.remove_option("--headers")
 
     def _print_response(self, response, opts):
         open_in_browser(response)

From 4156a86148f07c108d3ea4248a2d4a7e2a58ffa9 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 30 Jan 2017 15:57:37 +0100
Subject: [PATCH 1244/2974] Update docs on view command

---
 docs/topics/commands.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 6636c30cb..eaeeee113 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -358,6 +358,12 @@ Opens the given URL in a browser, as your Scrapy spider would "see" it.
 Sometimes spiders see pages differently from regular users, so this can be used
 to check what the spider "sees" and confirm it's what you expect.
 
+Supported options:
+
+* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
+
+* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
+
 Usage example::
 
     $ scrapy view http://www.example.com/some/page.html

From ae6d8d728e12e8efd704ba529de1db2eacfb494f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 30 Jan 2017 16:33:08 +0100
Subject: [PATCH 1245/2974] Support 'True' and 'False' strings as boolean
 settings values

---
 scrapy/settings/__init__.py     | 16 +++++++++++++---
 tests/test_settings/__init__.py | 12 ++++++++++++
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 7b7808959..28446a372 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -114,8 +114,8 @@ class BaseSettings(MutableMapping):
         """
         Get a setting value as a boolean.
 
-        ``1``, ``'1'``, and ``True`` return ``True``, while ``0``, ``'0'``,
-        ``False`` and ``None`` return ``False``.
+        ``1``, ``'1'``, `True`` and ``'True'`` return ``True``,
+        while ``0``, ``'0'``, ``False``, ``'False'`` and ``None`` return ``False``.
 
         For example, settings populated through environment variables set to
         ``'0'`` will return ``False`` when using this method.
@@ -126,7 +126,17 @@ class BaseSettings(MutableMapping):
         :param default: the value to return if no setting is found
         :type default: any
         """
-        return bool(int(self.get(name, default)))
+        got = self.get(name, default)
+        try:
+            return bool(int(got))
+        except ValueError:
+            if got in ("True", "true"):
+                return True
+            if got in ("False", "false"):
+                return False
+            raise ValueError("Supported values for boolean settings "
+                             "are 0/1, True/False, '0'/'1', "
+                             "'True'/'False' and 'true'/'false'")
 
     def getint(self, name, default=0):
         """
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 4acf22cba..863684075 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -211,9 +211,15 @@ class BaseSettingsTest(unittest.TestCase):
             'TEST_ENABLED1': '1',
             'TEST_ENABLED2': True,
             'TEST_ENABLED3': 1,
+            'TEST_ENABLED4': 'True',
+            'TEST_ENABLED5': 'true',
+            'TEST_ENABLED_WRONG': 'on',
             'TEST_DISABLED1': '0',
             'TEST_DISABLED2': False,
             'TEST_DISABLED3': 0,
+            'TEST_DISABLED4': 'False',
+            'TEST_DISABLED5': 'false',
+            'TEST_DISABLED_WRONG': 'off',
             'TEST_INT1': 123,
             'TEST_INT2': '123',
             'TEST_FLOAT1': 123.45,
@@ -231,11 +237,15 @@ class BaseSettingsTest(unittest.TestCase):
         self.assertTrue(settings.getbool('TEST_ENABLED1'))
         self.assertTrue(settings.getbool('TEST_ENABLED2'))
         self.assertTrue(settings.getbool('TEST_ENABLED3'))
+        self.assertTrue(settings.getbool('TEST_ENABLED4'))
+        self.assertTrue(settings.getbool('TEST_ENABLED5'))
         self.assertFalse(settings.getbool('TEST_ENABLEDx'))
         self.assertTrue(settings.getbool('TEST_ENABLEDx', True))
         self.assertFalse(settings.getbool('TEST_DISABLED1'))
         self.assertFalse(settings.getbool('TEST_DISABLED2'))
         self.assertFalse(settings.getbool('TEST_DISABLED3'))
+        self.assertFalse(settings.getbool('TEST_DISABLED4'))
+        self.assertFalse(settings.getbool('TEST_DISABLED5'))
         self.assertEqual(settings.getint('TEST_INT1'), 123)
         self.assertEqual(settings.getint('TEST_INT2'), 123)
         self.assertEqual(settings.getint('TEST_INTx'), 0)
@@ -258,6 +268,8 @@ class BaseSettingsTest(unittest.TestCase):
         self.assertEqual(settings.getdict('TEST_DICT3'), {})
         self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5})
         self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
+        self.assertRaises(ValueError, settings.getbool, 'TEST_ENABLED_WRONG')
+        self.assertRaises(ValueError, settings.getbool, 'TEST_DISABLED_WRONG')
 
     def test_getpriority(self):
         settings = BaseSettings({'key': 'value'}, priority=99)

From d2e9ea0c88b7578c5fc8d4d37e5df9d078e9b884 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 13 Jan 2017 16:17:51 +0100
Subject: [PATCH 1246/2974] Enforce DNS resolution timeout

---
 scrapy/resolver.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 3954fd977..4f4f0b04f 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -16,8 +16,11 @@ class CachingThreadedResolver(ThreadedResolver):
     def getHostByName(self, name, timeout=None):
         if name in dnscache:
             return defer.succeed(dnscache[name])
-        if not timeout:
-            timeout = self.timeout
+        # in Twisted<=16.6, getHostByName() is always called with
+        # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple),
+        # so the input argument above is simply overridden
+        # to enforce Scrapy's DNS_TIMEOUT setting's value
+        timeout = (self.timeout,)
         d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
         d.addCallback(self._cache_result, name)
         return d

From a58624375824d821f0bdadb7bc04fad53171ca70 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 13 Jan 2017 15:51:44 +0100
Subject: [PATCH 1247/2974] Buffer CONNECT response bytes from proxy until all
 HTTP headers are received

---
 scrapy/core/downloader/handlers/http11.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index ecd7f90d3..4b02cb16f 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -105,6 +105,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         self._tunneledHost = host
         self._tunneledPort = port
         self._contextFactory = contextFactory
+        self._connectBuffer = b''
 
     def requestTunnel(self, protocol):
         """Asks the proxy to open a tunnel."""
@@ -121,8 +122,16 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         created, notifies the client that we are ready to send requests. If not
         raises a TunnelError.
         """
+        self._connectBuffer += rcvd_bytes
+        # make sure that enough (all) bytes are consumed
+        # and that we've got all HTTP headers (ending with a blank line)
+        # from the proxy so that we don't send those bytes to the TLS layer
+        #
+        # see https://github.com/scrapy/scrapy/issues/2491
+        if b'\r\n\r\n' not in self._connectBuffer:
+            return
         self._protocol.dataReceived = self._protocolDataReceived
-        respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(rcvd_bytes)
+        respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
         if respm and int(respm.group('status')) == 200:
             try:
                 # this sets proper Server Name Indication extension

From 8c4f614d2101c21d3560fe68dcec6d3910417cb7 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 1 Sep 2016 15:25:38 +0200
Subject: [PATCH 1248/2974] Enable PyPy tests on Travis

---
 .travis.yml | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 6f3ab511f..2df02ea43 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,10 +17,30 @@ matrix:
       env: TOXENV=py35
     - python: 3.6
       env: TOXENV=py36
+    - python: 2.7
+      env: TOXENV=pypy
     - python: 3.6
       env: TOXENV=docs
+  allow_failures:
+    - python: 2.7
+      env: TOXENV=pypy
 install:
- - pip install -U tox twine wheel codecov
+  - |
+      if [ "$TOXENV" = "pypy" ]; then
+        export PYENV_ROOT="$HOME/.pyenv"
+        if [ -f "$PYENV_ROOT/bin/pyenv" ]; then
+          pushd "$PYENV_ROOT" && git pull && popd
+        else
+          rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT"
+        fi
+        # get latest PyPy from pyenv directly (thanks to natural version sort option -V)
+        export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy-[0-9][\.0-9]*$' |sort -V |tail -1`
+        "$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION"
+        virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION"
+        source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
+      fi
+  - pip install -U tox twine wheel codecov
+
 script: tox
 after_success:
   - codecov

From 70f260d3537d788e4efaf4a060fce7f0fc183ed9 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 1 Feb 2017 15:14:32 +0100
Subject: [PATCH 1249/2974] Don't run coverage stats when on PyPy

---
 tox.ini | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tox.ini b/tox.ini
index bdc14a128..0fdea11bb 100644
--- a/tox.ini
+++ b/tox.ini
@@ -54,6 +54,11 @@ commands =
     pip install -U https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib
     py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
 
+[testenv:pypy]
+basepython = pypy
+commands =
+    py.test {posargs:scrapy tests}
+
 [testenv:py33]
 basepython = python3.3
 deps =

From 55742c0392e8c582ed622f0ea28d8d62ece2c401 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 1 Feb 2017 22:43:28 +0500
Subject: [PATCH 1250/2974] DOC mention LevelDB cache storage backend

---
 docs/topics/downloader-middleware.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 3b9a5335a..912671d19 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -318,10 +318,11 @@ HttpCacheMiddleware
     This middleware provides low-level cache to all HTTP requests and responses.
     It has to be combined with a cache storage backend as well as a cache policy.
 
-    Scrapy ships with two HTTP cache storage backends:
+    Scrapy ships with three HTTP cache storage backends:
 
         * :ref:`httpcache-storage-fs`
         * :ref:`httpcache-storage-dbm`
+        * :ref:`httpcache-storage-leveldb`
 
     You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE`
     setting. Or you can also implement your own storage backend.

From 0cf6344cc22a193079ea1fd2fbe28187cc212c7a Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 9 Dec 2016 17:58:38 +0100
Subject: [PATCH 1251/2974] Support kwargs for response.xpath()

---
 scrapy/http/response/text.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index afa430329..5a6507aa8 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -111,8 +111,8 @@ class TextResponse(Response):
             self._cached_selector = Selector(self)
         return self._cached_selector
 
-    def xpath(self, query):
-        return self.selector.xpath(query)
+    def xpath(self, query, **kwargs):
+        return self.selector.xpath(query, **kwargs)
 
     def css(self, query):
         return self.selector.css(query)

From 803d8c4b5709d792f1c314bbde124e69903ff23f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 20 Dec 2016 11:26:42 +0100
Subject: [PATCH 1252/2974] Add tests for passing kwargs on response .xpath()
 shortcut

---
 tests/test_http_response.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 7624aa4c4..9df3bf6e7 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -320,6 +320,20 @@ class TextResponseTest(BaseResponseTest):
             response.selector.css("title::text").extract(),
         )
 
+    def test_selector_shortcuts_kwargs(self):
+        body = b"<html><head><title>Some page</title><body><p class=\"content\">A nice paragraph.</p></body></html>"
+        response = self.response_class("http://www.example.com", body=body)
+
+        self.assertEqual(
+            response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(),
+            response.xpath("normalize-space(//p[@class=\"content\"])").extract(),
+        )
+        self.assertEqual(
+            response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()",
+                pclass="content", pcount=1).extract(),
+            response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(),
+        )
+
     def test_urljoin_with_base_url(self):
         """Test urljoin shortcut which also evaluates base-url through get_base_url()."""
         body = b'<html><body><base href="https://example.net"></body></html>'
@@ -428,3 +442,21 @@ class XmlResponseTest(TextResponseTest):
             response.xpath("//elem/text()").extract(),
             response.selector.xpath("//elem/text()").extract(),
         )
+
+    def test_selector_shortcuts_kwargs(self):
+        body = b'''<?xml version="1.0" encoding="utf-8"?>
+        <xml xmlns:somens="http://scrapy.org">
+        <somens:elem>value</somens:elem>
+        </xml>'''
+        response = self.response_class("http://www.example.com", body=body)
+
+        self.assertEqual(
+            response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
+            response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
+        )
+
+        response.selector.register_namespace('s2', 'http://scrapy.org')
+        self.assertEqual(
+            response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(),
+            response.selector.xpath("//s2:elem/text()").extract(),
+        )

From 1c0b8053579e9680e4bd4c788a0a2bdf88a8f175 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 11 Jan 2017 19:44:21 +0100
Subject: [PATCH 1253/2974] DOC Mention XPath variables in Selectors section

---
 docs/topics/selectors.rst | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 39ec9b73c..43370d479 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -283,6 +283,40 @@ XPath specification.
 
 .. _Location Paths: https://www.w3.org/TR/xpath#location-paths
 
+.. _topics-selectors-xpath-variables:
+
+Variables in XPath expressions
+------------------------------
+
+XPath allows you to reference variables in your XPath expressions, using
+the ``$somevariable`` syntax. This is somewhat similar to parameterized
+queries or prepared statements in the SQL world where you replace
+some arguments in your queries with placeholders like ``?``,
+which are then substituted with values passed with the query.
+
+Here's an example to match an element based on its "id" attribute value,
+without hard-coding it (that was shown previously)::
+
+    >>> # `$val` used in the expression, a `val` argument needs to be passed
+    >>> response.xpath('//div[@id=$val]/a/text()', val='images').extract_first()
+    u'Name: My image 1 '
+
+Here's another example, to find the "id" attribute of a ``<div>`` tag containing
+five ``<a>`` children (here we pass the value ``5`` as an integer)::
+
+    >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).extract_first()
+    u'images'
+
+All variable references must have a binding value when calling ``.xpath()``
+(otherwise you'll get a ``ValueError: XPath error:`` exception).
+This is done by passing as many named arguments as necessary.
+
+`parsel`_, the library powering Scrapy selectors, has more details and examples
+on `XPath variables`_.
+
+.. _parsel: https://parsel.readthedocs.io/
+.. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions
+
 Using EXSLT extensions
 ----------------------
 

From 1295c17a26544f451261f4c6558b344162f3c4d4 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 2 Feb 2017 17:47:22 +0100
Subject: [PATCH 1254/2974] Bump parsel requirement to at least parsel v1.1

---
 requirements.txt | 2 +-
 setup.py         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 64b6e771c..f92603d3d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,4 +7,4 @@ queuelib
 six>=1.5.2
 PyDispatcher>=2.0.5
 service_identity
-parsel>=0.9.5
+parsel>=1.1
diff --git a/setup.py b/setup.py
index 388cf0dec..a6e6f9615 100644
--- a/setup.py
+++ b/setup.py
@@ -49,7 +49,7 @@ setup(
         'pyOpenSSL',
         'cssselect>=0.9',
         'six>=1.5.2',
-        'parsel>=0.9.5',
+        'parsel>=1.1',
         'PyDispatcher>=2.0.5',
         'service_identity',
     ],

From 3358254c5c78a1b8de1d02cb3b1db85fe824c798 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 2 Feb 2017 17:53:28 +0100
Subject: [PATCH 1255/2974] Make DNS retry test compatible with Twisted 17+

---
 tests/test_crawl.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 1b4a4b3b0..c7f4c0e35 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -91,12 +91,11 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_retry_dns_error(self):
-        with mock.patch('socket.gethostbyname',
-                        side_effect=socket.gaierror(-5, 'No address associated with hostname')):
-            crawler = self.runner.create_crawler(SimpleSpider)
-            with LogCapture() as l:
-                yield crawler.crawl("http://example.com/")
-            self._assert_retried(l)
+        crawler = self.runner.create_crawler(SimpleSpider)
+        with LogCapture() as l:
+            # try to fetch the homepage of a non-existent domain
+            yield crawler.crawl("http://dns.resolution.invalid/")
+        self._assert_retried(l)
 
     @defer.inlineCallbacks
     def test_start_requests_bug_before_yield(self):

From e604c0f3abeafdd896606960170ab5bad699d79d Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 2 Feb 2017 18:26:41 +0100
Subject: [PATCH 1256/2974] Remove unused imports

---
 tests/test_crawl.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index c7f4c0e35..0c64948fa 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -1,5 +1,4 @@
 import json
-import socket
 import logging
 
 from testfixtures import LogCapture
@@ -9,7 +8,6 @@ from twisted.trial.unittest import TestCase
 from scrapy.http import Request
 from scrapy.crawler import CrawlerRunner
 from scrapy.utils.python import to_unicode
-from tests import mock
 from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
     BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
 from tests.mockserver import MockServer

From 02e1d2b1fd0257a926f438a7758a9e65a0114325 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 2 Feb 2017 22:28:37 +0100
Subject: [PATCH 1257/2974] Add trailing dot

---
 tests/test_crawl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 0c64948fa..d5babdded 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -92,7 +92,7 @@ class CrawlTestCase(TestCase):
         crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
             # try to fetch the homepage of a non-existent domain
-            yield crawler.crawl("http://dns.resolution.invalid/")
+            yield crawler.crawl("http://dns.resolution.invalid./")
         self._assert_retried(l)
 
     @defer.inlineCallbacks

From 09643796b4d6ef25b23c1d7366d22ca55bd091d8 Mon Sep 17 00:00:00 2001
From: Lukas Anzinger <lukas@lukasanzinger.at>
Date: Fri, 3 Feb 2017 20:05:17 +0100
Subject: [PATCH 1258/2974] Fix typo in downloader-middleware.rst.

---
 docs/topics/downloader-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 3b9a5335a..2e32eb280 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -748,7 +748,7 @@ REDIRECT_MAX_TIMES
 
 Default: ``20``
 
-The maximum number of redirections that will be follow for a single request.
+The maximum number of redirections that will be followed for a single request.
 
 MetaRefreshMiddleware
 ---------------------

From 3021084f376a643f129388f76f039ab1301abb11 Mon Sep 17 00:00:00 2001
From: djrobust <djrobust@users.noreply.github.com>
Date: Sat, 4 Feb 2017 20:07:05 -0800
Subject: [PATCH 1259/2974] Use 'yield' when parsing multiple responses

Use 'yield' consistently across examples of parse functions.
---
 docs/topics/request-response.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 664a7239f..3853f5935 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -207,12 +207,12 @@ different fields from different pages::
         request = scrapy.Request("http://www.example.com/some_page.html",
                                  callback=self.parse_page2)
         request.meta['item'] = item
-        return request
+        yield request
 
     def parse_page2(self, response):
         item = response.meta['item']
         item['other_url'] = response.url
-        return item
+        yield item
 
 
 .. _topics-request-response-ref-errbacks:

From fcb3daf4fa3e4d97e9e66462395a021b3bc4363d Mon Sep 17 00:00:00 2001
From: Takehiro Shiozaki <shiozaki@vasily.jp>
Date: Mon, 6 Feb 2017 14:03:41 +0900
Subject: [PATCH 1260/2974] fix typo

---
 docs/topics/spider-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 604f1864f..8360827e8 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -112,7 +112,7 @@ following methods:
 
     .. method:: process_spider_exception(response, exception, spider)
 
-        This method is called when when a spider or :meth:`process_spider_input`
+        This method is called when a spider or :meth:`process_spider_input`
         method (from other spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an

From 16c4b4e184c1b213eafacecca0097ff44effa696 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Mon, 6 Feb 2017 11:41:08 +0100
Subject: [PATCH 1261/2974] [httpcompression] add support for br - brotli
 content encoding

---
 requirements.txt                                 |   1 +
 scrapy/downloadermiddlewares/httpcompression.py  |   6 +++++-
 tests/sample_data/compressed/html-br.bin         | Bin 0 -> 4027 bytes
 .../test_downloadermiddleware_httpcompression.py |  13 ++++++++++++-
 4 files changed, 18 insertions(+), 2 deletions(-)
 create mode 100644 tests/sample_data/compressed/html-br.bin

diff --git a/requirements.txt b/requirements.txt
index f92603d3d..362d05013 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,3 +8,4 @@ six>=1.5.2
 PyDispatcher>=2.0.5
 service_identity
 parsel>=1.1
+brotlipy==0.6
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index bcf20f10c..9202fd8da 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,5 +1,7 @@
 import zlib
 
+import brotli
+
 from scrapy.utils.gz import gunzip, is_gzipped
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
@@ -17,7 +19,7 @@ class HttpCompressionMiddleware(object):
         return cls()
 
     def process_request(self, request, spider):
-        request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
+        request.headers.setdefault('Accept-Encoding', 'gzip,deflate,br')
 
     def process_response(self, request, response, spider):
 
@@ -55,5 +57,7 @@ class HttpCompressionMiddleware(object):
                 # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
                 # http://www.gzip.org/zlib/zlib_faq.html#faq38
                 body = zlib.decompress(body, -15)
+        if encoding == b"br":
+            body = brotli.decompress(body)
         return body
 
diff --git a/tests/sample_data/compressed/html-br.bin b/tests/sample_data/compressed/html-br.bin
new file mode 100644
index 0000000000000000000000000000000000000000..c7eea4bb826a1a1851d047e64dd98423138cf996
GIT binary patch
literal 4027
zcmV;s4@B@=R3tGHVT)KTxseDvm_{W4YMH#TqFGI-3`~9)H9xksFwjQ_FSnd3&0Y8=
z@_ThUt$zy9SegPwSgY7y5IE<ZP6fjb*t`kh(h4F;QVOQVZF*h<(<~ds=3;v^jN$$L
zn5$c|wzw#%7<yeb^e^wfiCaR{u9M6omAZuF1pNOy|1~NY%ColF7E{={?1i<sGk?na
zLk{y}17lVKxXKh3IMA>>Uo9&E0Ix0^<i;%c5k*=t1j~#J<;LY14~QHgRIs5CLAGkJ
zH)WjrRtE+a>mRs=<3xGiyIT}Yr9fO2kCiRXNW-0{i+Lu3MI!yIoUSf-Z226AD}r0^
zdA#v**@y{+8$OqPhc=zMEOz;5{^K$+BPXfRJ1|wzaHbBHwx3!Jh1BbRnT%?EOzxB`
zc@o=4lizjLI=g3zEFMR~-@xbc=^ZH>n1uLg<R8j-{B@x3lyW?rM>XP=Rip&%H2-eg
z$?YM9bB=&I5l()^leA~+n16V4g_^DL-W4w{9=Ts$j!|s<ER#W{OIFUs$vy*_*4jjp
zAJbP~G~2u5$sahBwFn5>*p|9)8b6}Tq?$r33#pv6Q<C75y2?Wb(vI-karLN_dQ#fk
zS<F1PzRzZq9EXzDP{dWvSlXG-Z77heqkpp!o5s2(_@efsvl~qGcu3o+r*T^*tcZry
z^d6o~&(_;E-IdDLJ3!@urZ3FHcFwJA8Ji&P$NQt=*&pGd^nwSc&SE1y^xkXlQJL2!
z8}ytwYO^KYs`xuaKOP|_Dqczk>dYC{6y7(EBcDcrg)1%XlD<j_541nR%?IwF>FWLE
zIxI2y!eR*(8>O@}u3Hx&$4JYjr`~J*l_%PUM_=!22LDWBb0`w>VTQ3!mL%lo=0kh>
z;=f_ZiDy6Qi&t|`{$gtJAKjV<!HmA!76!i@v<4n#&Hc^&b>rTHwK@8XlMYrk*v45K
zR(OuASJjF_HuS)^Z3_;}-x*kbam!jCxmOBSZRDe}&Ao{=EGz4)%9_~X?Reh+%sac7
zGuk9hsij`&OiZ72ftz#WrA9ZS6`R$ubDx3drEzSlcoi3GIyjXkI?`smByM(WtKW-H
zWt7nbG9EvTV{l{PV!0a@LvYsd{#$Oc7=PHO*Tv<VQtFp#uo+e;^iEe&#PN<{GY)OD
zSi;8rKbTU)>u0%JJ>8p$>ryS1;SQzlXd;_<=@yK5Wg8buXuZIS?L!lfDPB6Z^`*eg
zutTl2T*u?5VH;d&i+*2*Ep~8PI;ShzaWl4DOzYKVhmHR_p&Q&<N^88WHBMZTi&-i-
zZO!SFhMI0-_o3_mT>md=hBvsdBnWMe#f`~|qjj-zg?1*K|BSawH>RT%cD#arQ7^TI
z6KvdkI?!<pMQX2KEHMSA)H~X+z0)#0<nY9;eW?{@^rCaPxw_Fz9P3ze7qcli9t@q5
zGKMN%XH3juy|066{E14}8fvBk#fjm4F*<@{zzXHc<aKi&FC9_7vr82xqZLcEqt~(%
z#}k{5qy885VhzW})U}q}xHaiA*CLi-hEgZmm1KN9ma`4!uJ!HV7`SCFrgsNVv$16_
zHMoo^vHq6>mmV|~lieEkzilzWsWq#SbmLLQWoW|u<XsJhJ6KRL-8!wA7@jToo5dE+
z5c~i1R#F8v=<F8t$yzb$=!FkNTy{3fIc>l_2z*TXMh3@&sY}_3+cF$~=<QuU{P-yu
z*6@V3-f6Av4oW9+SZ%RdZ*({=Y@Ns`mM30ATfFhumQXXg(LGLFn(ENjC>~WaRuLRH
zUEPy7hC`>~;lwtyrg)YdG@M|=iOVCKn3mX%);O^=MJtD6aba>hXdREsw_u5NhA+4<
z^sKWQ6|@y+i+U>g{@V(_mUeYdG6iL&*_d)M3>B2UhLP}Q%IjQ<SlVf6J?yZGU|3^t
zX_u}5ewvD5-9qPCIP@a5jL?a;_Lpu_@^1D#@p{#)ViG#mF?W9yl&=ok;4QId6~fY-
z%buZg+=4I~JyCNO7@FP>-WWddTE~_uq2D4R*$|#Mo;&1hp{C*{ib`vZ;l|QC2kzlp
zcW8qz7Ow>GBmI5eG5r=Ij!~P~x`Wh@iT^RIF&vn>rIiM8`^4oaS=lALY>4}Rd9{?x
z2~pZ28W!2T#&BcGbi5XgW9h~am$ddO1B-hYldX6)DLps$m>P*^fkBPe_i$Q=Q0L+p
zOUieVrI$zYIdT_{Z#*_}8RPXWf?<u}!--1|9YhCgqi5N%R|(vj=t|nL-7%O~k3vmw
zv%u6y>%)XCZqB)4+m?*aBFFV!egD_UB)01g%>2+HtLY+JuQ7OeW5{mY#yF<9sft#;
zm%%GGhDL{_4vFSs8EZt&f=bTu_)ucGPgpfh2}P@v-36Ci;&yf@H5?~v3Z8`Zj(6eG
zjSk}BrDb}So@WuVuC8=js+hJX*q5eTw&5(OwWr;~oDPn5VsJ{<b`{y(xM9U|YdUT3
zn?1WW+h`V;xDKbuv7k&_sKe$>uu7xQS<N0cw0sGPqjOH{n#AD*7cL!lptewhrcGuM
zvl+?7c1sw;Y753oH@=L`Vz=RuOH8{h)H)u=OO;h7So;Z6cQh<0U!7^#W@o`F&j0C-
zv(M-=+DmPG@HWXRYd3IX=&*z8KxwWfR+G>Km$c*Z({v2&xOGa_`jLDebo8PVxh;rI
z2UgQ<wz*mCHf~JavF^>BA(pbl-m4reT$<JP9XK7*8noW#W<jYNP027`uNIsW$kT&)
zFAEg1j@#!jr9)e*@$wpprV#7@_7{j44xK4j54VKG3gvfnwb8Te6^x&*UIP9t+uSVZ
zXsvsPjSk~}Y=IJJR$*{s>bA~mr(motI4NglL8WtC&D?=w#mzb8)5KR1N}Ykemgi!-
z3hdhf_2FiLf%Ql}hp$6dS8>!O%UY!A9R`oCJ6Jks!6_NlBmI5QXm&re)=b=%c&$oo
zvx-AU(;S);+M$Ei>6-SN1vad4`F?b$H7Xd}F3~E3_3QWlz>ediMS+aksu44b-GoQ)
z_&Q)MV!KOdT4mwF(zEQ2%TJ&mUDTA{EHF4Q_NzmuL(_ro5-$?Arx)GPNe8L3<839-
zdL-WmUT$2P)9h1}+xS^HyHySzOzlc=0=?-@fxP-Kv%toX^4065!>~oPTM_!b1ZeB-
z;e;NwQvhF$YSuK1-NKDqR~;hW0_t^vs}xL3%}HHQes@Ud%o7b~feS<9!x?UK58t-5
zSC{MkX>8-n*BAU+-r&+iR~?29txo6-4RlD~c>J{JU};j)>^G(5!G7VwrHMu{Y-(iw
zsbJ`xoI<!yh*<}_3;ULuPV{`Zrl$_BMm6O9One5Hc6DL7n>nkJa=s}Cx9;kkOioxY
z9R%4f##`(GFWl~~E}$ul`~eUzbZ=NMu<51=K(I^Fdd;(|*!Khv9lLfQIJ&2LFDE&?
z)~qCzEv7b`BAX`>Uc%H}_i=W!ho#^j20V6#$7;*ba;7oQZ!n*1cd1l&a4}Tb_O0hT
zT%VKq=MOB;ZtceUZb|pcSC4UwJ5Pvze0Ak9i`Z1ZCA-|C{*PIiO2tBNT!#C4Np@;!
zlWnIr#`Pw-uM&~yvo21Luh=1OOKh@t-LC5~S*)BW^o$9e`+u%i$?RI&WS@1B|35Cd
zrc$ACKL1O#xqi7`l5w@P$(}gwD^6UMh(^D4dyLCDC%(RK{hw_a?7nk7C1)!)4n9}2
zUNMV#MaHNd+ijA)cfCqpRXQ(T9UhF=;$`!{+5eGeYej!9r=E}L9BZSJIWS$^KJgyO
z?5-{OOOAIQHs?8axl@UJa9g(NK6c34F1BG%J!c)MtSQ{n23odvO^<mjjM}j+%Vn{i
zvzzHk=YnAvhH<2GU2VgE?*HNCeekQ_9ZPqmGGVzLH}jlc#?{g`TppMA*mYM{6h6G+
z*mHOe?32E`@7>y#?e^HmQeB(eFigv0KW8rG_HO78cg@E%tYcSQJK1o1Sbbw2>9n`k
zXS*!Jb{$K*5^>NQR==~1V<GJAvS)MVK%L&PA@}M3q?w&!URWM(XL8z8WpOXtHY|@O
zuiZ6D&dzcS`fOFH^Q>6~^<Jz!n=_l3eZ*~+(09+=<%8*A_c_$L{!?YUw&<R-ENyOi
zsw8fV&JB27rkpkIno7#v9h>ewmy{XuR*ib+Bo|Dt;c^*OW7L*yOfBC|Czkr9rW1Fk
z6^QfxUoq#j8RM;@w718$4HtZ#JG%<PPF)senX`%8J*zUQ+n8KaOWQS%7oCTkaaE19
zlM@^L?#;A^yl2#O!tJfIJ?5*^1KBrK^i(n%?ipUx4|&E@1!*tFCO=F#vl;ZQ5~C7%
zV0c_owyctwwJGqQ-4nWR=C19j<X*F?)jP9!;hDvmkISf{tF~;n%cVrQ_i8F7JJ&e?
z&-8y8)?^;{R-C+d&Nk-do6YU1TC=j_q4zwNlq+_P)81ak_NsH%w5+Q36zY5c+jI#T
zYgSPmdcP>YXIrYc?t7+D1(&&&Twr~8$A|sXz0u_EqyI~jtXW;#@LC?1Q?(w~+C-vH
zG%sUQ%NbAANSU*0+v7GZmtEC{Ypt_jOSwnPJMJ1KC)cdXwp=d5WmT2!T4QdwgsADf
zJmac@lrgJi6MbViF5*^;%e97R^hBF!P4x6!&4$kD+&ndBdp(A2I#uH_*OCXeI+xYC
z*NZ;y|5NAe>e_D0uq=<|byX@aOjU+eS+QMm(&f$S*e=5`+?HW^-D?iPKcoNK9I<lK
z$vm>}nR1?6ZQJp>9NXnJOnXiF@OqZ=HLuIy^o!wIrES}$9BR+`f3`L;*}GKd3%s7D
z{hHf!+_uMDg>9LZlo79GsdsjA;q?^F*G#v^Wt!}#a<*Z(U50HKhOw4NeEQCV0i4eb
zyY;$Ws$|XT+OKD=cs`cpcs-TMg2&@ha?bSH>dadm+m>^$*LK-%(_V9KxZRe=Ww=e3
zYqf22)^=0hYx4H&5AP~Vm2f*N*>IV*+wGW!*S#v+aJj51uzF(~Ym9@wH>2|YglgW(
zYj~xMT`imGIm=~n`kZZ6Dla^aVHmdKHjGu+_PpEFJ@ZxbRBkWy=e(SL^Wyxt9JoEE
zV_6=Tr&hKlXT)omwqYA<$p_PI&r`py!R~WX&aAHOvRt<1vb<)Y&I1@Tx2ZdGGYr#X
z7^_3~o~x##w(q&*l;qtt=YnInEXy%0Z>?<XpY#8CXDUL^nYOXkneaGnRpxpfmpad`
z%C@~O!|m}JW@Tr=JrD1ETsEgCF4rn;%dY8N6<eG3b~?FXx?PT4)gH%QE!#DVGcVI}
h+a7Zb*)UAECgV)YEyE7>;W^uy8!P|j4f_LbK_s`0D?<PP

literal 0
HcmV?d00001

diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index b2426946d..c47de54ed 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -17,6 +17,7 @@ FORMAT = {
         'x-gzip': ('html-gzip.bin', 'gzip'),
         'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
         'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
+        'br': ('html-br.bin', 'br')
         }
 
 class HttpCompressionTest(TestCase):
@@ -50,7 +51,8 @@ class HttpCompressionTest(TestCase):
         request = Request('http://scrapytest.org')
         assert 'Accept-Encoding' not in request.headers
         self.mw.process_request(request, self.spider)
-        self.assertEqual(request.headers.get('Accept-Encoding'), b'gzip,deflate')
+        self.assertEqual(request.headers.get('Accept-Encoding'),
+                         b'gzip,deflate,br')
 
     def test_process_response_gzip(self):
         response = self._getresponse('gzip')
@@ -62,6 +64,15 @@ class HttpCompressionTest(TestCase):
         assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
 
+    def test_process_response_br(self):
+        response = self._getresponse('br')
+        request = response.request
+        self.assertEqual(response.headers['Content-Encoding'], b'br')
+        newresponse = self.mw.process_response(request, response, self.spider)
+        assert newresponse is not response
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        assert 'Content-Encoding' not in newresponse.headers
+
     def test_process_response_rawdeflate(self):
         response = self._getresponse('rawdeflate')
         request = response.request

From f8f8bbe080163ba39db3bb78efbc36f8e5cd5883 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Mon, 6 Feb 2017 12:17:45 +0100
Subject: [PATCH 1262/2974] [httpcompression] import brotli when available

---
 requirements.txt                                |  1 -
 scrapy/downloadermiddlewares/httpcompression.py | 17 ++++++++++++-----
 ...test_downloadermiddleware_httpcompression.py |  3 ++-
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 362d05013..f92603d3d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,4 +8,3 @@ six>=1.5.2
 PyDispatcher>=2.0.5
 service_identity
 parsel>=1.1
-brotlipy==0.6
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 9202fd8da..04c9e355d 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,6 +1,5 @@
 import zlib
 
-import brotli
 
 from scrapy.utils.gz import gunzip, is_gzipped
 from scrapy.http import Response, TextResponse
@@ -8,10 +7,18 @@ from scrapy.responsetypes import responsetypes
 from scrapy.exceptions import NotConfigured
 
 
+ACCEPTED_ENCODINGS = [b'gzip', b'deflate']
+
+try:
+    import brotli
+    ACCEPTED_ENCODINGS.append(b'br')
+except ImportError:
+    pass
+
+
 class HttpCompressionMiddleware(object):
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
-
     @classmethod
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('COMPRESSION_ENABLED'):
@@ -19,7 +26,8 @@ class HttpCompressionMiddleware(object):
         return cls()
 
     def process_request(self, request, spider):
-        request.headers.setdefault('Accept-Encoding', 'gzip,deflate,br')
+        request.headers.setdefault('Accept-Encoding',
+                                   ",".join(ACCEPTED_ENCODINGS))
 
     def process_response(self, request, response, spider):
 
@@ -57,7 +65,6 @@ class HttpCompressionMiddleware(object):
                 # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
                 # http://www.gzip.org/zlib/zlib_faq.html#faq38
                 body = zlib.decompress(body, -15)
-        if encoding == b"br":
+        if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:
             body = brotli.decompress(body)
         return body
-
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index c47de54ed..b47b267e2 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -1,6 +1,6 @@
 from io import BytesIO
 from unittest import TestCase
-from os.path import join, abspath, dirname
+from os.path import join
 from gzip import GzipFile
 
 from scrapy.spiders import Spider
@@ -20,6 +20,7 @@ FORMAT = {
         'br': ('html-br.bin', 'br')
         }
 
+
 class HttpCompressionTest(TestCase):
 
     def setUp(self):

From 3daf473686aab89aa03ebd0ebc59a73b4b6e00f1 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Mon, 6 Feb 2017 12:29:33 +0100
Subject: [PATCH 1263/2974] [httpcompression] skip test if no brotli

---
 scrapy/downloadermiddlewares/httpcompression.py    |  2 +-
 tests/requirements.txt                             |  1 +
 tests/test_downloadermiddleware_httpcompression.py | 11 ++++++++---
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 04c9e355d..2fc1bb8eb 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -27,7 +27,7 @@ class HttpCompressionMiddleware(object):
 
     def process_request(self, request, spider):
         request.headers.setdefault('Accept-Encoding',
-                                   ",".join(ACCEPTED_ENCODINGS))
+                                   b",".join(ACCEPTED_ENCODINGS))
 
     def process_response(self, request, response, spider):
 
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 9d0c3c996..9baa4be21 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -6,6 +6,7 @@ pytest==2.9.2
 pytest-twisted
 pytest-cov==2.2.1
 jmespath
+brotlipy==0.6
 testfixtures
 # optional for shell wrapper tests
 bpython
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index b47b267e2..7924fb3b5 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -1,11 +1,12 @@
 from io import BytesIO
-from unittest import TestCase
+from unittest import TestCase, SkipTest
 from os.path import join
 from gzip import GzipFile
 
 from scrapy.spiders import Spider
 from scrapy.http import Response, Request, HtmlResponse
-from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware
+from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \
+    ACCEPTED_ENCODINGS
 from tests import tests_datadir
 from w3lib.encoding import resolve_encoding
 
@@ -53,7 +54,7 @@ class HttpCompressionTest(TestCase):
         assert 'Accept-Encoding' not in request.headers
         self.mw.process_request(request, self.spider)
         self.assertEqual(request.headers.get('Accept-Encoding'),
-                         b'gzip,deflate,br')
+                         b','.join(ACCEPTED_ENCODINGS))
 
     def test_process_response_gzip(self):
         response = self._getresponse('gzip')
@@ -66,6 +67,10 @@ class HttpCompressionTest(TestCase):
         assert 'Content-Encoding' not in newresponse.headers
 
     def test_process_response_br(self):
+        try:
+            import brotli
+        except ImportError:
+            raise SkipTest("no brotli")
         response = self._getresponse('br')
         request = response.request
         self.assertEqual(response.headers['Content-Encoding'], b'br')

From af802bad14f833178bc4e03e3db3a86c44dcb735 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Mon, 6 Feb 2017 15:45:21 +0100
Subject: [PATCH 1264/2974] [httpcompression] add brotlipy for python 3

---
 tests/requirements-py3.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index ed189c66c..d73a2300f 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -8,3 +8,4 @@ botocore
 # optional for shell wrapper tests
 bpython
 ipython
+brotlipy==0.6

From f73eb715ac4374b1eafb1da3c1311c5a7c153b5e Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 7 Feb 2017 02:16:06 +0500
Subject: [PATCH 1265/2974] =?UTF-8?q?LinkExtractor:=20don=E2=80=99t=20chec?=
 =?UTF-8?q?k=20all=20regexes=20if=20one=20of=20them=20matches?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scrapy/linkextractors/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index f51934b00..e5d21e174 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -40,7 +40,7 @@ IGNORED_EXTENSIONS = [
 
 
 _re_type = type(re.compile("", 0))
-_matches = lambda url, regexs: any((r.search(url) for r in regexs))
+_matches = lambda url, regexs: any(r.search(url) for r in regexs)
 _is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'}
 
 
@@ -93,8 +93,8 @@ class FilteringLinkExtractor(object):
         if self.deny_domains and url_is_from_any_domain(url, self.deny_domains):
             return False
 
-        allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True]
-        denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else []
+        allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True]
+        denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else []
         return any(allowed) and not any(denied)
 
     def _process_links(self, links):

From 85a124970ad406c2680402256853dd3ff86d191f Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 7 Feb 2017 03:32:54 +0500
Subject: [PATCH 1266/2974] Enable memusage extension by default. Fixes
 GH-2187.

---
 docs/topics/settings.rst            | 10 ++++++----
 scrapy/settings/default_settings.py |  2 +-
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0515a9e0d..c1f488f73 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -827,13 +827,15 @@ Example::
 MEMUSAGE_ENABLED
 ----------------
 
-Default: ``False``
+Default: ``True``
 
 Scope: ``scrapy.extensions.memusage``
 
-Whether to enable the memory usage extension that will shutdown the Scrapy
-process when it exceeds a memory limit, and also notify by email when that
-happened.
+Whether to enable the memory usage extension. This extension keeps track of
+a peak memory used by the process (it writes it to stats). It can also
+optionally shutdown the Scrapy process when it exceeds a memory limit
+(see :setting:`MEMUSAGE_LIMIT_MB`), and notify by email when that happened
+(see :setting:`MEMUSAGE_NOTIFY_MAIL`).
 
 See :ref:`topics-extensions-ref-memusage`.
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 24714a7a8..1cc169c9a 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -207,7 +207,7 @@ MEMDEBUG_ENABLED = False        # enable memory debugging
 MEMDEBUG_NOTIFY = []            # send memory debugging report by mail at engine shutdown
 
 MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
-MEMUSAGE_ENABLED = False
+MEMUSAGE_ENABLED = True
 MEMUSAGE_LIMIT_MB = 0
 MEMUSAGE_NOTIFY_MAIL = []
 MEMUSAGE_REPORT = False

From fb4ef21a1dcdc0c70fa80443e7379150639c42f6 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Tue, 7 Feb 2017 10:22:42 +0100
Subject: [PATCH 1267/2974] [httpcompression] minor style edits

---
 scrapy/downloadermiddlewares/httpcompression.py | 1 -
 tests/requirements-py3.txt                      | 2 +-
 tests/requirements.txt                          | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 2fc1bb8eb..19d6345e4 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,6 +1,5 @@
 import zlib
 
-
 from scrapy.utils.gz import gunzip, is_gzipped
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index d73a2300f..51a25f5e5 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -8,4 +8,4 @@ botocore
 # optional for shell wrapper tests
 bpython
 ipython
-brotlipy==0.6
+brotlipy
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 9baa4be21..c1576a2e7 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -6,7 +6,7 @@ pytest==2.9.2
 pytest-twisted
 pytest-cov==2.2.1
 jmespath
-brotlipy==0.6
+brotlipy
 testfixtures
 # optional for shell wrapper tests
 bpython

From 24e82bfe75ad1a7b5ceb7a2840ecbae82d05268d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 23 Dec 2016 12:58:29 -0300
Subject: [PATCH 1268/2974] Validate values for components order

---
 scrapy/utils/conf.py     |  9 +++++++++
 tests/test_utils_conf.py | 21 +++++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index e8af90f11..435e9a6b3 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -1,5 +1,6 @@
 import os
 import sys
+import numbers
 from operator import itemgetter
 
 import six
@@ -34,6 +35,13 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
             _check_components(compdict)
             return {convert(k): v for k, v in six.iteritems(compdict)}
 
+    def _validate_values(compdict):
+        """Fail if a value in the components dict is not a real number or None."""
+        for name, value in six.iteritems(compdict):
+            if value is not None and not isinstance(value, numbers.Real):
+                raise ValueError('Invalid value {} for component {}, please provide ' \
+                                 'a real number or None instead'.format(value, name))
+
     # BEGIN Backwards compatibility for old (base, custom) call signature
     if isinstance(custom, (list, tuple)):
         _check_components(custom)
@@ -43,6 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
         compdict.update(custom)
     # END Backwards compatibility
 
+    _validate_values(compdict)
     compdict = without_none_values(_map_keys(compdict))
     return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
 
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index dab41ac8d..f203c32ef 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -62,6 +62,27 @@ class BuildComponentListTest(unittest.TestCase):
         self.assertRaises(ValueError, build_component_list, duplicate_bs,
                           convert=lambda x: x.lower())
 
+    def test_valid_numbers(self):
+        # work well with None and numeric values
+        d = {'a': 10, 'b': None, 'c': 15, 'd': 5.0}
+        self.assertEqual(build_component_list(d, convert=lambda x: x),
+                         ['d', 'a', 'c'])
+        d = {'a': 33333333333333333333, 'b': 11111111111111111111, 'c': 22222222222222222222}
+        self.assertEqual(build_component_list(d, convert=lambda x: x),
+                         ['b', 'c', 'a'])
+        # raise exception for invalid values
+        d = {'one': '5'}
+        self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
+        d = {'one': '1.0'}
+        self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
+        d = {'one': [1, 2, 3]}
+        self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
+        d = {'one': {'a': 'a', 'b': 2}}
+        self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
+        d = {'one': 'lorem ipsum',}
+        self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
+
+
 
 class UtilsConfTestCase(unittest.TestCase):
 

From eaf62ab69c23459fa36dafe298bcfabc5952b7f7 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 7 Feb 2017 17:56:43 +0500
Subject: [PATCH 1269/2974] cleanup MetaRefreshMiddleware: remove redundant
 check

---
 scrapy/downloadermiddlewares/redirect.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index db276eefb..26677e527 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -53,8 +53,10 @@ class BaseRedirectMiddleware(object):
 
 
 class RedirectMiddleware(BaseRedirectMiddleware):
-    """Handle redirection of requests based on response status and meta-refresh html tag"""
-
+    """
+    Handle redirection of requests based on response status
+    and meta-refresh html tag.
+    """
     def process_response(self, request, response, spider):
         if (request.meta.get('dont_redirect', False) or
                 response.status in getattr(spider, 'handle_httpstatus_list', []) or
@@ -92,10 +94,9 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
                 not isinstance(response, HtmlResponse):
             return response
 
-        if isinstance(response, HtmlResponse):
-            interval, url = get_meta_refresh(response)
-            if url and interval < self._maxdelay:
-                redirected = self._redirect_request_using_get(request, url)
-                return self._redirect(redirected, request, spider, 'meta refresh')
+        interval, url = get_meta_refresh(response)
+        if url and interval < self._maxdelay:
+            redirected = self._redirect_request_using_get(request, url)
+            return self._redirect(redirected, request, spider, 'meta refresh')
 
         return response

From 04b2f79e7a1316d29b2cb94c8fb2623e040b64ec Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 7 Feb 2017 22:30:58 +0500
Subject: [PATCH 1270/2974] Remove code required to support ancient twisted
 versions. See GH-1887.

---
 scrapy/core/downloader/handlers/http.py  |  8 +---
 tests/mockserver.py                      | 39 +++++--------------
 tests/test_downloader_handlers.py        | 49 +++++++++---------------
 tests/test_downloadermiddleware_retry.py |  5 +--
 4 files changed, 30 insertions(+), 71 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index 81da2615a..e4a7d8564 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -1,10 +1,6 @@
-from scrapy import twisted_version
+from __future__ import absolute_import
 from .http10 import HTTP10DownloadHandler
-
-if twisted_version >= (11, 1, 0):
-    from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
-else:
-    HTTPDownloadHandler = HTTP10DownloadHandler
+from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
 
 
 # backwards compatibility
diff --git a/tests/mockserver.py b/tests/mockserver.py
index a40e2e501..e611cc3ec 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -2,34 +2,18 @@ from __future__ import print_function
 import sys, time, random, os, json
 from six.moves.urllib.parse import urlencode
 from subprocess import Popen, PIPE
+
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
-from twisted.internet import reactor, defer, ssl
-from scrapy import twisted_version
+from twisted.web.test.test_webclient import PayloadResource
+from twisted.web.server import GzipEncoderFactory
+from twisted.web.resource import EncodingResourceWrapper
+from twisted.internet import reactor, ssl
+from twisted.internet.task import deferLater
+
 from scrapy.utils.python import to_bytes, to_unicode
 
 
-if twisted_version < (11, 0, 0):
-    def deferLater(clock, delay, func, *args, **kw):
-        def _cancel_method():
-            _cancel_cb(None)
-            d.errback(Exception())
-
-        def _cancel_cb(result):
-            if cl.active():
-                cl.cancel()
-            return result
-
-        d = defer.Deferred()
-        d.cancel = _cancel_method
-        d.addCallback(lambda ignored: func(*args, **kw))
-        d.addBoth(_cancel_cb)
-        cl = clock.callLater(delay, d.callback, None)
-        return d
-else:
-    from twisted.internet.task import deferLater
-
-
 def getarg(request, name, default=None, type=None):
     if name in request.args:
         value = request.args[name][0]
@@ -174,13 +158,8 @@ class Root(Resource):
         self.putChild(b"drop", Drop())
         self.putChild(b"raw", Raw())
         self.putChild(b"echo", Echo())
-
-        if twisted_version > (12, 3, 0):
-            from twisted.web.test.test_webclient import PayloadResource
-            from twisted.web.server import GzipEncoderFactory
-            from twisted.web.resource import EncodingResourceWrapper
-            self.putChild(b"payload", PayloadResource())
-            self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
+        self.putChild(b"payload", PayloadResource())
+        self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
 
     def getChild(self, name, request):
         return self
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 8d3b49d6a..6333efceb 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -17,7 +17,6 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \
 from twisted.cred import portal, checkers, credentials
 from w3lib.url import path_to_file_uri
 
-from scrapy import twisted_version
 from scrapy.core.downloader.handlers import DownloadHandlers
 from scrapy.core.downloader.handlers.file import FileDownloadHandler
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler
@@ -281,8 +280,6 @@ class Https10TestCase(Http10TestCase):
 class Http11TestCase(HttpTestCase):
     """HTTP 1.1 test case"""
     download_handler_cls = HTTP11DownloadHandler
-    if twisted_version < (11, 1, 0):
-        skip = 'HTTP1.1 not supported in twisted < 11.1.0'
 
     def test_download_without_maxsize_limit(self):
         request = Request(self.getURL('file'))
@@ -366,8 +363,6 @@ class Https11InvalidDNSId(Https11TestCase):
 
 class Http11MockServerTestCase(unittest.TestCase):
     """HTTP 1.1 test case with MockServer"""
-    if twisted_version < (11, 1, 0):
-        skip = 'HTTP1.1 not supported in twisted < 11.1.0'
 
     def setUp(self):
         self.mockserver = MockServer()
@@ -396,31 +391,27 @@ class Http11MockServerTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_download_gzip_response(self):
+        crawler = get_crawler(SingleRequestSpider)
+        body = b'1' * 100  # PayloadResource requires body length to be 100
+        request = Request('http://localhost:8998/payload', method='POST',
+                          body=body, meta={'download_maxsize': 50})
+        yield crawler.crawl(seed=request)
+        failure = crawler.spider.meta['failure']
+        # download_maxsize < 100, hence the CancelledError
+        self.assertIsInstance(failure.value, defer.CancelledError)
 
-        if twisted_version > (12, 3, 0):
-
-            crawler = get_crawler(SingleRequestSpider)
-            body = b'1'*100 # PayloadResource requires body length to be 100
-            request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50})
+        if six.PY2:
+            request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
+            request = request.replace(url='http://localhost:8998/xpayload')
             yield crawler.crawl(seed=request)
-            failure = crawler.spider.meta['failure']
-            # download_maxsize < 100, hence the CancelledError
-            self.assertIsInstance(failure.value, defer.CancelledError)
-
-            if six.PY2:
-                request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
-                request = request.replace(url='http://localhost:8998/xpayload')
-                yield crawler.crawl(seed=request)
-                # download_maxsize = 50 is enough for the gzipped response
-                failure = crawler.spider.meta.get('failure')
-                self.assertTrue(failure == None)
-                reason = crawler.spider.meta['close_reason']
-                self.assertTrue(reason, 'finished')
-            else:
-                # See issue https://twistedmatrix.com/trac/ticket/8175
-                raise unittest.SkipTest("xpayload only enabled for PY2")
+            # download_maxsize = 50 is enough for the gzipped response
+            failure = crawler.spider.meta.get('failure')
+            self.assertTrue(failure == None)
+            reason = crawler.spider.meta['close_reason']
+            self.assertTrue(reason, 'finished')
         else:
-            raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0")
+            # See issue https://twistedmatrix.com/trac/ticket/8175
+            raise unittest.SkipTest("xpayload only enabled for PY2")
 
 
 class UriResource(resource.Resource):
@@ -500,8 +491,6 @@ class Http10ProxyTestCase(HttpProxyTestCase):
 
 class Http11ProxyTestCase(HttpProxyTestCase):
     download_handler_cls = HTTP11DownloadHandler
-    if twisted_version < (11, 1, 0):
-        skip = 'HTTP1.1 not supported in twisted < 11.1.0'
 
     @defer.inlineCallbacks
     def test_download_with_proxy_https_timeout(self):
@@ -692,8 +681,6 @@ class FTPTestCase(unittest.TestCase):
     username = "scrapy"
     password = "passwd"
 
-    if twisted_version < (10, 2, 0):
-        skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home"
     if six.PY3:
         skip = "Twisted missing ftp support for PY3"
 
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index eb17974bf..e129b71f8 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -5,7 +5,6 @@ from twisted.internet.error import TimeoutError, DNSLookupError, \
         ConnectionLost, TCPTimedOutError
 from twisted.web.client import ResponseFailed
 
-from scrapy import twisted_version
 from scrapy.downloadermiddlewares.retry import RetryMiddleware
 from scrapy.spiders import Spider
 from scrapy.http import Request, Response
@@ -74,9 +73,7 @@ class RetryTest(unittest.TestCase):
     def test_twistederrors(self):
         exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError,
                 DNSLookupError, ConnectionRefusedError, ConnectionDone,
-                ConnectError, ConnectionLost]
-        if twisted_version >= (11, 1, 0): # http11 available
-            exceptions.append(ResponseFailed)
+                ConnectError, ConnectionLost, ResponseFailed]
 
         for exc in exceptions:
             req = Request('http://www.scrapytest.org/%s' % exc.__name__)

From 4e765acaed7a914630ee5320fa6f6523890a2b9d Mon Sep 17 00:00:00 2001
From: Rolando Espinoza <rndmax84@gmail.com>
Date: Tue, 24 Jan 2017 10:30:47 -0400
Subject: [PATCH 1271/2974] BUG: Fix __classcell__ propagation.

Python 3.6 added simpler customization of class creation but this
requires to propagate correctly the __classcell__ attribute in custom
__new__ methods.

See https://docs.python.org/3.6/whatsnew/3.6.html#pep-487-simpler-
customization-of-class-creation
---
 scrapy/item.py     |  3 +++
 tests/test_item.py | 52 +++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/scrapy/item.py b/scrapy/item.py
index 138728a9a..aa05e9c69 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -25,6 +25,7 @@ class Field(dict):
 class ItemMeta(ABCMeta):
 
     def __new__(mcs, class_name, bases, attrs):
+        classcell = attrs.pop('__classcell__', None)
         new_bases = tuple(base._class for base in bases if hasattr(base, '_class'))
         _class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs)
 
@@ -39,6 +40,8 @@ class ItemMeta(ABCMeta):
 
         new_attrs['fields'] = fields
         new_attrs['_class'] = _class
+        if classcell is not None:
+            new_attrs['__classcell__'] = classcell
         return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs)
 
 
diff --git a/tests/test_item.py b/tests/test_item.py
index dcb169c3a..85a554de0 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -1,8 +1,14 @@
+import sys
 import unittest
 
-from scrapy.item import Item, Field
 import six
 
+from scrapy.item import ABCMeta, Item, ItemMeta, Field
+from tests import mock
+
+
+PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6)
+
 
 class ItemTest(unittest.TestCase):
 
@@ -244,5 +250,49 @@ class ItemTest(unittest.TestCase):
         self.assertNotEqual(item['name'], copied_item['name'])
 
 
+class ItemMetaTest(unittest.TestCase):
+
+    def test_new_method_propagates_classcell(self):
+        new_mock = mock.Mock(side_effect=ABCMeta.__new__)
+        base = ItemMeta.__bases__[0]
+
+        with mock.patch.object(base, '__new__', new_mock):
+
+            class MyItem(Item):
+                if not PY36_PLUS:
+                    # This attribute is an internal attribute in Python 3.6+
+                    # and must be propagated properly. See
+                    # https://docs.python.org/3.6/reference/datamodel.html#creating-the-class-object
+                    # In <3.6, we add a dummy attribute just to ensure the
+                    # __new__ method propagates it correctly.
+                    __classcell__ = object()
+
+                def f(self):
+                    # For rationale of this see:
+                    # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222
+                    return __class__
+
+            MyItem()
+
+        (first_call, second_call) = new_mock.call_args_list[-2:]
+
+        mcs, class_name, bases, attrs = first_call[0]
+        assert '__classcell__' not in attrs
+        mcs, class_name, bases, attrs = second_call[0]
+        assert '__classcell__' in attrs
+
+
+class ItemMetaClassCellRegression(unittest.TestCase):
+
+    def test_item_meta_classcell_regression(self):
+        class MyItem(six.with_metaclass(ItemMeta, Item)):
+            def __init__(self, *args, **kwargs):
+                # This call to super() trigger the __classcell__ propagation
+                # requirement. When not done properly raises an error:
+                # TypeError: __class__ set to <class '__main__.MyItem'>
+                # defining 'MyItem' as <class '__main__.MyItem'>
+                super(MyItem, self).__init__(*args, **kwargs)
+
+
 if __name__ == "__main__":
     unittest.main()

From 48c8c679de72da295aa753ffd9ed68b3958d3cbb Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 6 Feb 2017 18:03:48 +0100
Subject: [PATCH 1272/2974] Update changelog for upcoming 1.3.1 release

---
 docs/news.rst | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index cce46599b..41374f970 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,54 @@
 Release notes
 =============
 
+Scrapy 1.3.1 (2017-02-XX)
+-------------------------
+
+New features
+~~~~~~~~~~~~
+
+- Support ``'True'`` and ``'False'`` string values for boolean settings (:issue:`2519`);
+  you can now do something like ``scrapy crawl myspider -s REDIRECT_ENABLED=False``.
+- Support kwargs with ``response.xpath()`` to use :ref:`XPath variables <topics-selectors-xpath-variables>`
+  and ad-hoc namespaces declarations ;
+  this requires at least Parsel v1.1 (:issue:`2457`).
+- Add support for Python 3.6 (:issue:`2485`).
+- Run tests on PyPy (warning: some tests still fail, so PyPy is not supported yet).
+
+Bug fixes
+~~~~~~~~~
+
+- Enforce ``DNS_TIMEOUT`` setting (:issue:`2496`).
+- Fix :command:`view` command ; it was a regression in v1.3.0 (:issue:`2503`).
+- Fix tests regarding ``*_EXPIRES settings`` with Files/Images pipelines (:issue:`2460`).
+- Fix name of generated pipeline class when using basic project template (:issue:`2466`).
+- Fix compatiblity with Twisted 17+ (:issue:`2496`, :issue:`2528`).
+- Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`).
+- Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``,
+  ``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`).
+
+Documentation
+~~~~~~~~~~~~~
+
+- Reword Code of Coduct section and upgrade to Contributor Covenant v1.4
+  (:issue:`2469`).
+- Clarify that passing spider arguments converts them to spider attributes
+  (:issue:`2483`).
+- Document ``formid`` argument on ``FormRequest.from_response()`` (:issue:`2497`).
+- Add .rst extension to README files (:issue:`2507`).
+- Mention LevelDB cache storage backend (:issue:`2525`).
+- Use ``yield`` in sample callback code (:issue:`2533`).
+- Add note about HTML entities decoding with ``.re()/.re_first()`` (:issue:`1704`).
+- Typos (:issue:`2512`, :issue:`2534`, :issue:`2531`).
+
+Cleanups
+~~~~~~~~
+
+- Remove reduntant check in ``MetaRefreshMiddleware`` (:issue:`2542`).
+- Faster checks in ``LinkExtractor`` for allow/deny patterns (:issue:`2538`).
+- Remove dead code supporting old Twisted versions (:issue:`2544`).
+
+
 Scrapy 1.3.0 (2016-12-21)
 -------------------------
 

From ff8a564b1a775ec16143b7c00199d40875868337 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 8 Feb 2017 17:05:06 +0100
Subject: [PATCH 1273/2974] Set date for 1.3.1 release

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 41374f970..3c1d24561 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,7 +3,7 @@
 Release notes
 =============
 
-Scrapy 1.3.1 (2017-02-XX)
+Scrapy 1.3.1 (2017-02-08)
 -------------------------
 
 New features

From af55a8713e561015b4e50eb5dd7061709770c67a Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 8 Feb 2017 17:08:19 +0100
Subject: [PATCH 1274/2974] =?UTF-8?q?Bump=20version:=201.3.0=20=E2=86=92?=
 =?UTF-8?q?=201.3.1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 57ff603fa..0a8a71e8e 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.3.0
+current_version = 1.3.1
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index f0bb29e76..3a3cd8cc8 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.3.0
+1.3.1

From 9c0aae724ed821fd954a14db83902a86f7fe7731 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 3 Feb 2017 10:32:36 -0300
Subject: [PATCH 1275/2974] Use credentials from request.meta['proxy'] if
 present

---
 docs/topics/downloader-middleware.rst        |  5 ++-
 scrapy/downloadermiddlewares/httpproxy.py    | 24 +++++++----
 tests/test_downloadermiddleware_httpproxy.py | 43 +++++++++++++++++---
 3 files changed, 57 insertions(+), 15 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 1ca78ccc6..f0ff3c77c 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -681,7 +681,10 @@ HttpProxyMiddleware
    * ``no_proxy``
 
    You can also set the meta key ``proxy`` per-request, to a value like
-   ``http://some_proxy_server:port``.
+   ``http://username:password@some_proxy_server:port``. Keep in mind
+   this value will take precedence over ``http_proxy``/``https_proxy``
+   environment variables, and it will also ignore ``no_proxy`` environment
+   variable.
 
 .. _urllib: https://docs.python.org/2/library/urllib.html
 .. _urllib2: https://docs.python.org/2/library/urllib2.html
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 98c87aa9c..edc1c52ed 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -8,7 +8,6 @@ except ImportError:
 from six.moves.urllib.parse import urlunparse
 
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.exceptions import NotConfigured
 from scrapy.utils.python import to_bytes
 
 
@@ -20,23 +19,23 @@ class HttpProxyMiddleware(object):
         for type, url in getproxies().items():
             self.proxies[type] = self._get_proxy(url, type)
 
-        if not self.proxies:
-            raise NotConfigured
-
     @classmethod
     def from_crawler(cls, crawler):
         auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')
         return cls(auth_encoding)
 
+    def _basic_auth_header(self, username, password):
+        user_pass = to_bytes(
+            '%s:%s' % (unquote(username), unquote(password)),
+            encoding=self.auth_encoding)
+        return base64.b64encode(user_pass).strip()
+
     def _get_proxy(self, url, orig_type):
         proxy_type, user, password, hostport = _parse_proxy(url)
         proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
 
         if user:
-            user_pass = to_bytes(
-                '%s:%s' % (unquote(user), unquote(password)),
-                encoding=self.auth_encoding)
-            creds = base64.b64encode(user_pass).strip()
+            creds = self._basic_auth_header(user, password)
         else:
             creds = None
 
@@ -45,6 +44,15 @@ class HttpProxyMiddleware(object):
     def process_request(self, request, spider):
         # ignore if proxy is already set
         if 'proxy' in request.meta:
+            if request.meta['proxy'] is None:
+                return
+            # extract credentials if present
+            creds, proxy_url = self._get_proxy(request.meta['proxy'], '')
+            request.meta['proxy'] = proxy_url
+            if creds and not request.headers.get('Proxy-Authorization'):
+                request.headers['Proxy-Authorization'] = b'Basic ' + creds
+            return
+        elif not self.proxies:
             return
 
         parsed = urlparse_cached(request)
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 2b26431a4..dd09e4dd0 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -20,10 +20,6 @@ class TestDefaultHeadersMiddleware(TestCase):
     def tearDown(self):
         os.environ = self._oldenv
 
-    def test_no_proxies(self):
-        os.environ = {}
-        self.assertRaises(NotConfigured, HttpProxyMiddleware)
-
     def test_no_enviroment_proxies(self):
         os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
         mw = HttpProxyMiddleware()
@@ -47,6 +43,13 @@ class TestDefaultHeadersMiddleware(TestCase):
             self.assertEquals(req.url, url)
             self.assertEquals(req.meta.get('proxy'), proxy)
 
+    def test_proxy_precedence_meta(self):
+        os.environ['http_proxy'] = 'https://proxy.com'
+        mw = HttpProxyMiddleware()
+        req = Request('http://scrapytest.org', meta={'proxy': 'https://new.proxy:3128'})
+        assert mw.process_request(req, spider) is None
+        self.assertEquals(req.meta, {'proxy': 'https://new.proxy:3128'})
+
     def test_proxy_auth(self):
         os.environ['http_proxy'] = 'https://user:pass@proxy:3128'
         mw = HttpProxyMiddleware()
@@ -54,6 +57,11 @@ class TestDefaultHeadersMiddleware(TestCase):
         assert mw.process_request(req, spider) is None
         self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
         self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz')
+        # proxy from request.meta
+        req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'})
+        assert mw.process_request(req, spider) is None
+        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=')
 
     def test_proxy_auth_empty_passwd(self):
         os.environ['http_proxy'] = 'https://user:@proxy:3128'
@@ -62,6 +70,11 @@ class TestDefaultHeadersMiddleware(TestCase):
         assert mw.process_request(req, spider) is None
         self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
         self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=')
+        # proxy from request.meta
+        req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'})
+        assert mw.process_request(req, spider) is None
+        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6')
 
     def test_proxy_auth_encoding(self):
         # utf-8 encoding
@@ -72,6 +85,12 @@ class TestDefaultHeadersMiddleware(TestCase):
         self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
         self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
 
+        # proxy from request.meta
+        req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
+        assert mw.process_request(req, spider) is None
+        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==')
+
         # default latin-1 encoding
         mw = HttpProxyMiddleware(auth_encoding='latin-1')
         req = Request('http://scrapytest.org')
@@ -79,15 +98,21 @@ class TestDefaultHeadersMiddleware(TestCase):
         self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
         self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
 
+        # proxy from request.meta, latin-1 encoding
+        req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
+        assert mw.process_request(req, spider) is None
+        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz')
+
     def test_proxy_already_seted(self):
-        os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
+        os.environ['http_proxy'] = 'https://proxy.for.http:3128'
         mw = HttpProxyMiddleware()
         req = Request('http://noproxy.com', meta={'proxy': None})
         assert mw.process_request(req, spider) is None
         assert 'proxy' in req.meta and req.meta['proxy'] is None
 
     def test_no_proxy(self):
-        os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
+        os.environ['http_proxy'] = 'https://proxy.for.http:3128'
         mw = HttpProxyMiddleware()
 
         os.environ['no_proxy'] = '*'
@@ -104,3 +129,9 @@ class TestDefaultHeadersMiddleware(TestCase):
         req = Request('http://noproxy.com')
         assert mw.process_request(req, spider) is None
         assert 'proxy' not in req.meta
+
+        # proxy from meta['proxy'] takes precedence
+        os.environ['no_proxy'] = '*'
+        req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'})
+        assert mw.process_request(req, spider) is None
+        self.assertEquals(req.meta, {'proxy': 'http://proxy.com'})

From 29e60213db19030907dc8afd50173aa4421132df Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 9 Feb 2017 10:41:21 +0100
Subject: [PATCH 1276/2974] Use consistent selectors for author field in
 tutorial

---
 docs/intro/tutorial.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 8e14d1b7c..3dc5ad2ed 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -225,7 +225,7 @@ You will see something like::
     [s]   shelp()           Shell help (print this help)
     [s]   fetch(req_or_url) Fetch request (or URL) and update local objects
     [s]   view(response)    View response in a browser
-    >>> 
+    >>>
 
 Using the shell, you can try selecting elements using `CSS`_ with the response
 object::
@@ -423,7 +423,7 @@ in the callback, as you can see below::
             for quote in response.css('div.quote'):
                 yield {
                     'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.css('span small::text').extract_first(),
+                    'author': quote.css('small.author::text').extract_first(),
                     'tags': quote.css('div.tags a.tag::text').extract(),
                 }
 
@@ -522,7 +522,7 @@ page, extracting data from it::
             for quote in response.css('div.quote'):
                 yield {
                     'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.css('span small::text').extract_first(),
+                    'author': quote.css('small.author::text').extract_first(),
                     'tags': quote.css('div.tags a.tag::text').extract(),
                 }
 
@@ -568,7 +568,7 @@ this time for scraping author information::
 
         def parse(self, response):
             # follow links to author pages
-            for href in response.css('.author+a::attr(href)').extract():
+            for href in response.css('.author + a::attr(href)').extract():
                 yield scrapy.Request(response.urljoin(href),
                                      callback=self.parse_author)
 
@@ -624,7 +624,7 @@ option when running them::
     scrapy crawl quotes -o quotes-humor.json -a tag=humor
 
 These arguments are passed to the Spider's ``__init__`` method and become
-spider attributes by default.  
+spider attributes by default.
 
 In this example, the value provided for the ``tag`` argument will be available
 via ``self.tag``. You can use this to make your spider fetch only quotes
@@ -647,7 +647,7 @@ with a specific tag, building the URL based on the argument::
             for quote in response.css('div.quote'):
                 yield {
                     'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.css('span small a::text').extract_first(),
+                    'author': quote.css('small.author::text').extract_first(),
                 }
 
             next_page = response.css('li.next a::attr(href)').extract_first()

From 9956f198db54fa181856ad788e2ad8ee76ae5437 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 13 Feb 2017 18:40:53 +0500
Subject: [PATCH 1277/2974] add a couple more lines to gitignore

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index b116640b4..406146e5f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,8 @@ dist
 .idea
 htmlcov/
 .coverage
+.coverage.*
+.cache/
 
 # Windows
 Thumbs.db

From de65ad3fb1e90f6fcffbcc74bfa6aff8ff65e14f Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 13 Feb 2017 18:44:39 +0500
Subject: [PATCH 1278/2974] TST replace Ubuntu 12.04 tox environment with 14.04

---
 tox.ini | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tox.ini b/tox.ini
index 0fdea11bb..bbf50b733 100644
--- a/tox.ini
+++ b/tox.ini
@@ -21,16 +21,16 @@ passenv =
 commands =
     py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
 
-[testenv:precise]
+[testenv:trusty]
 basepython = python2.7
 deps =
     pyOpenSSL==0.13
-    lxml==2.3.2
-    Twisted==11.1.0
-    boto==2.2.2
-    Pillow<2.0
+    lxml==3.3.3
+    Twisted==13.2.0
+    boto==2.20.1
+    Pillow==2.3.0
     cssselect==0.9.1
-    zope.interface==3.6.1
+    zope.interface==4.0.5
     -rtests/requirements.txt
 
 [testenv:jessie]

From 1bc4d8b6b6cf84c1785a6ad69abf37b4f0114bb3 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 13 Feb 2017 20:03:53 +0500
Subject: [PATCH 1279/2974] fixed tls in Twisted 17+

---
 scrapy/core/downloader/tls.py | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 955b7630c..498e3d60f 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -1,6 +1,8 @@
 import logging
 from OpenSSL import SSL
 
+from scrapy import twisted_version
+
 
 logger = logging.getLogger(__name__)
 
@@ -18,11 +20,17 @@ openssl_methods = {
     METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
 }
 
-# ClientTLSOptions requires a recent-enough version of Twisted
-try:
+if twisted_version >= (14, 0, 0):
+    # ClientTLSOptions requires a recent-enough version of Twisted.
+    # Not having ScrapyClientTLSOptions should not matter for older
+    # Twisted versions because it is not used in the fallback
+    # ScrapyClientContextFactory.
 
     # taken from twisted/twisted/internet/_sslverify.py
+
     try:
+        # XXX: this try-except is not needed in Twisted 17.0.0+ because
+        # it requires pyOpenSSL 0.16+.
         from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START
     except ImportError:
         SSL_CB_HANDSHAKE_START = 0x10
@@ -30,10 +38,17 @@ try:
 
     from twisted.internet.ssl import AcceptableCiphers
     from twisted.internet._sslverify import (ClientTLSOptions,
-                                             _maybeSetHostNameIndication,
                                              verifyHostname,
                                              VerificationError)
 
+    if twisted_version < (17, 0, 0):
+        from twisted.internet._sslverify import _maybeSetHostNameIndication
+        set_tlsext_host_name = _maybeSetHostNameIndication
+    else:
+        def set_tlsext_host_name(connection, hostNameBytes):
+            connection.set_tlsext_host_name(hostNameBytes)
+
+
     class ScrapyClientTLSOptions(ClientTLSOptions):
         """
         SSL Client connection creator ignoring certificate verification errors
@@ -46,7 +61,7 @@ try:
 
         def _identityVerifyingInfoCallback(self, connection, where, ret):
             if where & SSL_CB_HANDSHAKE_START:
-                _maybeSetHostNameIndication(connection, self._hostnameBytes)
+                set_tlsext_host_name(connection, self._hostnameBytes)
             elif where & SSL_CB_HANDSHAKE_DONE:
                 try:
                     verifyHostname(connection, self._hostnameASCII)
@@ -62,8 +77,3 @@ try:
                             self._hostnameASCII, repr(e)))
 
     DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
-
-except ImportError:
-    # ImportError should not matter for older Twisted versions
-    # as the above is not used in the fallback ScrapyClientContextFactory
-    pass

From 9315e944a225a4010870333ff751fb2ed512f489 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 13 Feb 2017 14:56:29 -0300
Subject: [PATCH 1280/2974] Release notes for 1.3.2

---
 docs/news.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 3c1d24561..ff1e4ce03 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,16 @@
 Release notes
 =============
 
+Scrapy 1.3.2 (2017-02-13)
+-------------------------
+
+Bug fixes
+~~~~~~~~~
+
+- Preserve crequest class when converting to/from dicts (utils.reqser) (:issue:`2510`).
+- Use consistent selectors for author field in tutorial (:issue:`2551`).
+- Fix TLS compatibility in Twisted 17+ (:issue:`2558`)
+
 Scrapy 1.3.1 (2017-02-08)
 -------------------------
 

From 7dd7646e6563798d408b8062059e826f08256b39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 13 Feb 2017 14:57:55 -0300
Subject: [PATCH 1281/2974] =?UTF-8?q?Bump=20version:=201.3.1=20=E2=86=92?=
 =?UTF-8?q?=201.3.2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 0a8a71e8e..b95e0bad5 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.3.1
+current_version = 1.3.2
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 3a3cd8cc8..1892b9267 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.3.1
+1.3.2

From 5b31dfe3c970f7e16dc911bb8b028a0fff54a7f9 Mon Sep 17 00:00:00 2001
From: terut <terut.dev+github@gmail.com>
Date: Mon, 13 Feb 2017 23:51:43 -0800
Subject: [PATCH 1282/2974] Separate building request from _requests_to_follow
 in CrawlSpider

You just overwrite buiding request if you can use another request class
because of something like splash-plugin.
---
 scrapy/spiders/crawl.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 031f649d6..e5ac72e18 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -48,6 +48,11 @@ class CrawlSpider(Spider):
     def process_results(self, response, results):
         return results
 
+    def _build_request(self, rule, link):
+        r = Request(url=link.url, callback=self._response_downloaded)
+        r.meta.update(rule=rule, link_text=link.text)
+        return r
+
     def _requests_to_follow(self, response):
         if not isinstance(response, HtmlResponse):
             return
@@ -59,8 +64,7 @@ class CrawlSpider(Spider):
                 links = rule.process_links(links)
             for link in links:
                 seen.add(link)
-                r = Request(url=link.url, callback=self._response_downloaded)
-                r.meta.update(rule=n, link_text=link.text)
+                r = self._build_request(n, link)
                 yield rule.process_request(r)
 
     def _response_downloaded(self, response):

From ae0ea31abd4e65f7decb4482d1669d7af17bca0a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 8 Feb 2017 13:21:10 -0300
Subject: [PATCH 1283/2974] Add HTTPPROXY_ENABLED setting (default True)

---
 docs/topics/downloader-middleware.rst        | 15 +++++++++++----
 scrapy/downloadermiddlewares/httpproxy.py    |  3 +++
 scrapy/settings/default_settings.py          |  1 +
 tests/test_downloadermiddleware_httpproxy.py |  8 ++++++++
 4 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index f0ff3c77c..0ef3fb071 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -681,10 +681,9 @@ HttpProxyMiddleware
    * ``no_proxy``
 
    You can also set the meta key ``proxy`` per-request, to a value like
-   ``http://username:password@some_proxy_server:port``. Keep in mind
-   this value will take precedence over ``http_proxy``/``https_proxy``
-   environment variables, and it will also ignore ``no_proxy`` environment
-   variable.
+   ``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``.
+   Keep in mind this value will take precedence over ``http_proxy``/``https_proxy``
+   environment variables, and it will also ignore ``no_proxy`` environment variable.
 
 .. _urllib: https://docs.python.org/2/library/urllib.html
 .. _urllib2: https://docs.python.org/2/library/urllib2.html
@@ -952,8 +951,16 @@ enable it for :ref:`broad crawls <topics-broad-crawls>`.
 HttpProxyMiddleware settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+.. setting:: HTTPPROXY_ENABLED
 .. setting:: HTTPPROXY_AUTH_ENCODING
 
+HTTPPROXY_ENABLED
+^^^^^^^^^^^^^^^^^
+
+Default: ``True``
+
+Whether or not to enable the :class:`HttpProxyMiddleware`.
+
 HTTPPROXY_AUTH_ENCODING
 ^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index edc1c52ed..0d5320bf8 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -8,6 +8,7 @@ except ImportError:
 from six.moves.urllib.parse import urlunparse
 
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.exceptions import NotConfigured
 from scrapy.utils.python import to_bytes
 
 
@@ -21,6 +22,8 @@ class HttpProxyMiddleware(object):
 
     @classmethod
     def from_crawler(cls, crawler):
+        if not crawler.settings.getbool('HTTPPROXY_ENABLED'):
+            raise NotConfigured
         auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')
         return cls(auth_encoding)
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 24714a7a8..cb88bc2bf 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -174,6 +174,7 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
 HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
 HTTPCACHE_GZIP = False
 
+HTTPPROXY_ENABLED = True
 HTTPPROXY_AUTH_ENCODING = 'latin-1'
 
 IMAGES_STORE_S3_ACL = 'private'
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index dd09e4dd0..c77179ceb 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -1,11 +1,14 @@
 import os
 import sys
+from functools import partial
 from twisted.trial.unittest import TestCase, SkipTest
 
 from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response, Request
 from scrapy.spiders import Spider
+from scrapy.crawler import Crawler
+from scrapy.settings import Settings
 
 spider = Spider('foo')
 
@@ -20,6 +23,11 @@ class TestDefaultHeadersMiddleware(TestCase):
     def tearDown(self):
         os.environ = self._oldenv
 
+    def test_not_enabled(self):
+        settings = Settings({'HTTPPROXY_ENABLED': False})
+        crawler = Crawler(spider, settings)
+        self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler))
+
     def test_no_enviroment_proxies(self):
         os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
         mw = HttpProxyMiddleware()

From e285b1d6c2aaa1fdfe788f1894b0196bc64d1be1 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 7 Feb 2017 18:17:07 +0500
Subject: [PATCH 1284/2974] retry stats

---
 scrapy/downloadermiddlewares/retry.py    | 14 ++++++++++++--
 scrapy/downloadermiddlewares/stats.py    |  4 +++-
 scrapy/utils/misc.py                     |  2 +-
 scrapy/utils/python.py                   | 11 +++++++++++
 scrapy/utils/response.py                 |  3 ++-
 tests/test_downloadermiddleware_retry.py | 15 ++++++++++++---
 tests/test_proxy_connect.py              |  4 +++-
 7 files changed, 44 insertions(+), 9 deletions(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index c9c512be8..d84697b14 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -22,6 +22,7 @@ from twisted.web.client import ResponseFailed
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.response import response_status_message
 from scrapy.core.downloader.handlers.http11 import TunnelError
+from scrapy.utils.python import global_object_name
 
 logger = logging.getLogger(__name__)
 
@@ -35,16 +36,18 @@ class RetryMiddleware(object):
                            ConnectionLost, TCPTimedOutError, ResponseFailed,
                            IOError, TunnelError)
 
-    def __init__(self, settings):
+    def __init__(self, crawler):
+        settings = crawler.settings
         if not settings.getbool('RETRY_ENABLED'):
             raise NotConfigured
         self.max_retry_times = settings.getint('RETRY_TIMES')
         self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES'))
         self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
+        self.stats = crawler.stats
 
     @classmethod
     def from_crawler(cls, crawler):
-        return cls(crawler.settings)
+        return cls(crawler)
 
     def process_response(self, request, response, spider):
         if request.meta.get('dont_retry', False):
@@ -70,8 +73,15 @@ class RetryMiddleware(object):
             retryreq.meta['retry_times'] = retries
             retryreq.dont_filter = True
             retryreq.priority = request.priority + self.priority_adjust
+
+            if isinstance(reason, Exception):
+                reason = global_object_name(reason.__class__)
+
+            self.stats.inc_value('retry/count')
+            self.stats.inc_value('retry/reason_count/%s' % reason)
             return retryreq
         else:
+            self.stats.inc_value('retry/max_reached')
             logger.debug("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
                          {'request': request, 'retries': retries, 'reason': reason},
                          extra={'spider': spider})
diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py
index 9c0ad90a5..ef0aafce0 100644
--- a/scrapy/downloadermiddlewares/stats.py
+++ b/scrapy/downloadermiddlewares/stats.py
@@ -1,6 +1,8 @@
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.request import request_httprepr
 from scrapy.utils.response import response_httprepr
+from scrapy.utils.python import global_object_name
+
 
 class DownloaderStats(object):
 
@@ -27,6 +29,6 @@ class DownloaderStats(object):
         return response
 
     def process_exception(self, request, exception, spider):
-        ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__)
+        ex_class = global_object_name(exception.__class__)
         self.stats.inc_value('downloader/exception_count', spider=spider)
         self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider)
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 30c9e5058..35f855007 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -113,7 +113,7 @@ def md5sum(file):
         m.update(d)
     return m.hexdigest()
 
+
 def rel_has_nofollow(rel):
     """Return True if link rel attribute has nofollow type"""
     return True if rel is not None and 'nofollow' in rel.split() else False
-    
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 42fbbda7f..4c500abf4 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -344,3 +344,14 @@ def without_none_values(iterable):
         return {k: v for k, v in six.iteritems(iterable) if v is not None}
     except AttributeError:
         return type(iterable)((v for v in iterable if v is not None))
+
+
+def global_object_name(obj):
+    """
+    Return full name of a global object.
+
+    >>> from scrapy import Request
+    >>> global_object_name(Request)
+    'scrapy.http.request.Request'
+    """
+    return "%s.%s" % (obj.__module__, obj.__name__)
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index deb5741be..bf276b5ca 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -43,7 +43,8 @@ def get_meta_refresh(response):
 def response_status_message(status):
     """Return status code plus status text descriptive message
     """
-    return '%s %s' % (status, to_native_str(http.RESPONSES.get(int(status), "Unknown Status")))
+    message = http.RESPONSES.get(int(status), "Unknown Status")
+    return '%s %s' % (status, to_native_str(message))
 
 
 def response_httprepr(response):
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index e129b71f8..b833cb448 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -13,9 +13,9 @@ from scrapy.utils.test import get_crawler
 
 class RetryTest(unittest.TestCase):
     def setUp(self):
-        crawler = get_crawler(Spider)
-        self.spider = crawler._create_spider('foo')
-        self.mw = RetryMiddleware.from_crawler(crawler)
+        self.crawler = get_crawler(Spider)
+        self.spider = self.crawler._create_spider('foo')
+        self.mw = RetryMiddleware.from_crawler(self.crawler)
         self.mw.max_retry_times = 2
 
     def test_priority_adjust(self):
@@ -70,6 +70,10 @@ class RetryTest(unittest.TestCase):
         # discard it
         assert self.mw.process_response(req, rsp, self.spider) is rsp
 
+        assert self.crawler.stats.get_value('retry/max_reached') == 1
+        assert self.crawler.stats.get_value('retry/reason_count/503 Service Unavailable') == 2
+        assert self.crawler.stats.get_value('retry/count') == 2
+
     def test_twistederrors(self):
         exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError,
                 DNSLookupError, ConnectionRefusedError, ConnectionDone,
@@ -79,6 +83,11 @@ class RetryTest(unittest.TestCase):
             req = Request('http://www.scrapytest.org/%s' % exc.__name__)
             self._test_retry_exception(req, exc('foo'))
 
+        stats = self.crawler.stats
+        assert stats.get_value('retry/max_reached') == len(exceptions)
+        assert stats.get_value('retry/count') == len(exceptions) * 2
+        assert stats.get_value('retry/reason_count/twisted.internet.defer.TimeoutError') == 2
+
     def _test_retry_exception(self, req, exception):
         # first retry
         req = self.mw.process_exception(req, exception, self.spider)
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 0f06fd53d..6213a51e8 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -101,7 +101,9 @@ class ProxyConnectTestCase(TestCase):
         self._assert_got_response_code(407, l)
 
     def _assert_got_response_code(self, code, log):
+        print(log)
         self.assertEqual(str(log).count('Crawled (%d)' % code), 1)
 
     def _assert_got_tunnel_error(self, log):
-        self.assertEqual(str(log).count('TunnelError'), 1)
+        print(log)
+        self.assertIn('TunnelError', str(log))

From 922d3fec54f40b0349c8c98dd8441aee3075cd65 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 14 Feb 2017 12:11:06 -0300
Subject: [PATCH 1285/2974] Doc: binary mode is required for exporters

---
 docs/topics/exporters.rst | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index af469eb7b..85c73222d 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -225,7 +225,8 @@ XmlItemExporter
 
    Exports Items in XML format to the specified file object.
 
-   :param file: the file-like object to use for exporting the data.
+   :param file: the file-like object to use for exporting the data. Its ``write`` method should
+                accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
 
    :param root_element: The name of root element in the exported XML.
    :type root_element: str
@@ -281,7 +282,8 @@ CsvItemExporter
    CSV columns and their order. The :attr:`export_empty_fields` attribute has
    no effect on this exporter.
 
-   :param file: the file-like object to use for exporting the data.
+   :param file: the file-like object to use for exporting the data. Its ``write`` method should
+                accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
 
    :param include_headers_line: If enabled, makes the exporter output a header
       line with the field names taken from
@@ -312,7 +314,8 @@ PickleItemExporter
 
    Exports Items in pickle format to the given file-like object.
 
-   :param file: the file-like object to use for exporting the data.
+   :param file: the file-like object to use for exporting the data. Its ``write`` method should
+                accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
 
    :param protocol: The pickle protocol to use.
    :type protocol: int
@@ -333,7 +336,8 @@ PprintItemExporter
 
    Exports Items in pretty print format to the specified file object.
 
-   :param file: the file-like object to use for exporting the data.
+   :param file: the file-like object to use for exporting the data. Its ``write`` method should
+                accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
 
    The additional keyword arguments of this constructor are passed to the
    :class:`BaseItemExporter` constructor.
@@ -356,7 +360,8 @@ JsonItemExporter
    arguments to the `JSONEncoder`_ constructor, so you can use any
    `JSONEncoder`_ constructor argument to customize this exporter.
 
-   :param file: the file-like object to use for exporting the data.
+   :param file: the file-like object to use for exporting the data. Its ``write`` method should
+                accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
 
    A typical output of this exporter would be::
 
@@ -386,7 +391,8 @@ JsonLinesItemExporter
    the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_
    constructor argument to customize this exporter.
 
-   :param file: the file-like object to use for exporting the data.
+   :param file: the file-like object to use for exporting the data. Its ``write`` method should
+                accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
 
    A typical output of this exporter would be::
 

From 39df675f091cf904dc904acd9538a0bebe2e55cd Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 14 Feb 2017 23:28:50 +0500
Subject: [PATCH 1286/2974] make retry middleware changes backwards compatible

---
 scrapy/downloadermiddlewares/retry.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index d84697b14..549d74f46 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -36,18 +36,16 @@ class RetryMiddleware(object):
                            ConnectionLost, TCPTimedOutError, ResponseFailed,
                            IOError, TunnelError)
 
-    def __init__(self, crawler):
-        settings = crawler.settings
+    def __init__(self, settings):
         if not settings.getbool('RETRY_ENABLED'):
             raise NotConfigured
         self.max_retry_times = settings.getint('RETRY_TIMES')
         self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES'))
         self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
-        self.stats = crawler.stats
 
     @classmethod
     def from_crawler(cls, crawler):
-        return cls(crawler)
+        return cls(crawler.settings)
 
     def process_response(self, request, response, spider):
         if request.meta.get('dont_retry', False):
@@ -65,6 +63,7 @@ class RetryMiddleware(object):
     def _retry(self, request, reason, spider):
         retries = request.meta.get('retry_times', 0) + 1
 
+        stats = spider.crawler.stats
         if retries <= self.max_retry_times:
             logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",
                          {'request': request, 'retries': retries, 'reason': reason},
@@ -77,11 +76,11 @@ class RetryMiddleware(object):
             if isinstance(reason, Exception):
                 reason = global_object_name(reason.__class__)
 
-            self.stats.inc_value('retry/count')
-            self.stats.inc_value('retry/reason_count/%s' % reason)
+            stats.inc_value('retry/count')
+            stats.inc_value('retry/reason_count/%s' % reason)
             return retryreq
         else:
-            self.stats.inc_value('retry/max_reached')
+            stats.inc_value('retry/max_reached')
             logger.debug("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
                          {'request': request, 'retries': retries, 'reason': reason},
                          extra={'spider': spider})

From e1ceaf3b5fa29326f032c4ed3f50943384b9e63d Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 13 Feb 2017 21:06:05 +0500
Subject: [PATCH 1287/2974] require w3lib 1.17+

---
 requirements-py3.txt | 2 +-
 requirements.txt     | 2 +-
 setup.py             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements-py3.txt b/requirements-py3.txt
index 08ccf1958..cc0a7f644 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -3,5 +3,5 @@ lxml>=3.2.4
 pyOpenSSL>=0.13.1
 cssselect>=0.9
 queuelib>=1.1.1
-w3lib>=1.14.2
+w3lib>=1.17.0
 service_identity
diff --git a/requirements.txt b/requirements.txt
index f92603d3d..392f83dd6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@ Twisted>=13.1.0
 lxml
 pyOpenSSL
 cssselect>=0.9
-w3lib>=1.15.0
+w3lib>=1.17.0
 queuelib
 six>=1.5.2
 PyDispatcher>=2.0.5
diff --git a/setup.py b/setup.py
index a6e6f9615..086ab8142 100644
--- a/setup.py
+++ b/setup.py
@@ -43,7 +43,7 @@ setup(
     ],
     install_requires=[
         'Twisted>=13.1.0',
-        'w3lib>=1.15.0',
+        'w3lib>=1.17.0',
         'queuelib',
         'lxml',
         'pyOpenSSL',

From 877057fac0d47e5ece95a55594706c91c8855883 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 7 Feb 2017 06:00:09 +0500
Subject: [PATCH 1288/2974] initial response.follow implementation

---
 docs/intro/overview.rst          |  3 +-
 docs/intro/tutorial.rst          | 46 +++++++++++++++++++------
 docs/topics/request-response.rst |  4 +++
 scrapy/http/response/text.py     | 59 ++++++++++++++++++++++++++++++++
 4 files changed, 100 insertions(+), 12 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 7195017ff..1da1a4059 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -40,8 +40,7 @@ http://quotes.toscrape.com, following the pagination::
 
             next_page = response.css('li.next a::attr("href")').extract_first()
             if next_page is not None:
-                next_page = response.urljoin(next_page)
-                yield scrapy.Request(next_page, callback=self.parse)
+                yield response.follow(next_page, self.parse)
 
 
 Put this in a text file, name it to something like ``quotes_spider.py``
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 3dc5ad2ed..d47bf69e5 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -551,13 +551,40 @@ In our example, it creates a sort of loop, following all the links to the next p
 until it doesn't find one -- handy for crawling blogs, forums and other sites with
 pagination.
 
+As a shortcut for creating Request objects you can use
+:meth:`response.follow <scrapy.http.TextResponse.follow>` method::
+
+    import scrapy
+
+
+    class QuotesSpider(scrapy.Spider):
+        name = "quotes"
+        start_urls = [
+            'http://quotes.toscrape.com/page/1/',
+        ]
+
+        def parse(self, response):
+            for quote in response.css('div.quote'):
+                yield {
+                    'text': quote.css('span.text::text').extract_first(),
+                    'author': quote.css('span small::text').extract_first(),
+                    'tags': quote.css('div.tags a.tag::text').extract(),
+                }
+
+            for href in response.css('li.next a::attr(href)'):
+                yield response.follow(href, callback=self.parse)
+
+Unlike scrapy.Request, ``response.follow`` supports
+relative URLs directly; you can also pass a selector to it instead of
+a string. Note that ``response.follow`` just returns a Request instance;
+you still have to yield this Request.
+
 More examples and patterns
 --------------------------
 
 Here is another spider that illustrates callbacks and following links,
 this time for scraping author information::
 
-
     import scrapy
 
 
@@ -568,15 +595,12 @@ this time for scraping author information::
 
         def parse(self, response):
             # follow links to author pages
-            for href in response.css('.author + a::attr(href)').extract():
-                yield scrapy.Request(response.urljoin(href),
-                                     callback=self.parse_author)
+            for href in response.css('.author + a::attr(href)'):
+                yield response.follow(href, self.parse_author)
 
             # follow pagination links
-            next_page = response.css('li.next a::attr(href)').extract_first()
-            if next_page is not None:
-                next_page = response.urljoin(next_page)
-                yield scrapy.Request(next_page, callback=self.parse)
+            for href in response.css('li.next a::attr(href)'):
+                yield response.follow(href, self.parse)
 
         def parse_author(self, response):
             def extract_with_css(query):
@@ -592,6 +616,9 @@ This spider will start from the main page, it will follow all the links to the
 authors pages calling the ``parse_author`` callback for each of them, and also
 the pagination links with the ``parse`` callback as we saw before.
 
+Here we're passing callbacks to ``response.follow`` as positional arguments
+to make the code shorter; it also works for ``scrapy.Request``.
+
 The ``parse_author`` callback defines a helper function to extract and cleanup the
 data from a CSS query and yields the Python dict with the author data.
 
@@ -652,8 +679,7 @@ with a specific tag, building the URL based on the argument::
 
             next_page = response.css('li.next a::attr(href)').extract_first()
             if next_page is not None:
-                next_page = response.urljoin(next_page)
-                yield scrapy.Request(next_page, self.parse)
+                yield response.follow(next_page, self.parse)
 
 
 If you pass the ``tag=humor`` argument to this spider, you'll notice that it
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 1fdd26043..71050fddd 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -683,6 +683,10 @@ TextResponse objects
 
             response.css('p')
 
+    .. method:: TextResponse.follow(url, ...)
+
+        Return a scrapy.Request instance to follow a link ``url``.
+
     .. method:: TextResponse.body_as_unicode()
 
         The same as :attr:`text`, but available as a method. This method is
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 5a6507aa8..1718b1f3b 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -8,8 +8,12 @@ See documentation in docs/topics/request-response.rst
 import six
 from six.moves.urllib.parse import urljoin
 
+import parsel
 from w3lib.encoding import html_to_unicode, resolve_encoding, \
     html_body_declared_encoding, http_content_type_encoding
+
+from scrapy.link import Link
+from scrapy.http.request import Request
 from scrapy.http.response import Response
 from scrapy.utils.response import get_base_url
 from scrapy.utils.python import memoizemethod_noargs, to_native_str
@@ -116,3 +120,58 @@ class TextResponse(Response):
 
     def css(self, query):
         return self.selector.css(query)
+
+    def follow(self, url, callback=None, method='GET', headers=None, body=None,
+               cookies=None, meta=None, encoding=None, priority=0,
+               dont_filter=False, errback=None):
+        # type: (...) -> Request
+        """
+        Return a scrapy.Request instance to follow a link ``url``.
+
+        ``url`` can be:
+
+        * absolute URL;
+        * relative URL;
+        * scrapy.link.Link object (e.g. a link extractor result);
+        * attribute Selector (not SelectorList) - e.g.
+          ``response.css('a::attr(href)')[0]`` or
+          ``response.xpath('//img/@src')[0]``.
+        * a Selector for ``<a>`` element, e.g.
+          ``response.css('a.my_link')[0]``.
+        """
+        if isinstance(url, Link):
+            url = url.url
+        elif isinstance(url, parsel.Selector):
+            url = _url_from_selector(url)
+        elif isinstance(url, parsel.SelectorList):
+            raise ValueError("Please pass either string")
+
+
+        encoding = self.encoding if encoding is None else encoding
+        url = self.urljoin(url)
+        return Request(url, callback,
+                       method=method,
+                       headers=headers,
+                       body=body,
+                       cookies=cookies,
+                       meta=meta,
+                       encoding=encoding,
+                       priority=priority,
+                       dont_filter=dont_filter,
+                       errback=errback)
+
+
+def _url_from_selector(sel):
+    # type: (parsel.Selector) -> str
+    if isinstance(sel.root, six.string_types):
+        # e.g. ::attr(href) result
+        return sel.root
+    if not hasattr(sel.root, 'tag'):
+        raise ValueError("Unsupported selector: %s" % sel)
+    if sel.root.tag != 'a':
+        raise ValueError("Only <a> elements are supported; got <%s>" %
+                         sel.root.tag)
+    href = sel.root.get('href')
+    if href is None:
+        raise ValueError("<a> element has no href attribute: %s" % sel)
+    return href

From 71dd5d0bf9d0c41d70e72b0fd0a89528ef246065 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 7 Feb 2017 06:11:08 +0500
Subject: [PATCH 1289/2974] strip URL extracted from selectors (as per html5
 standard)

---
 scrapy/http/response/text.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 1718b1f3b..5bfd2debb 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -142,10 +142,9 @@ class TextResponse(Response):
         if isinstance(url, Link):
             url = url.url
         elif isinstance(url, parsel.Selector):
-            url = _url_from_selector(url)
+            url = _url_from_selector(url).strip()
         elif isinstance(url, parsel.SelectorList):
-            raise ValueError("Please pass either string")
-
+            raise ValueError("SelectorList is not supported")
 
         encoding = self.encoding if encoding is None else encoding
         url = self.urljoin(url)

From 608c3f0c452bd508aa24bc9d4bf759e0b11e1683 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Feb 2017 04:17:41 +0500
Subject: [PATCH 1290/2974] handle whitespace in response.follow; add tests

---
 scrapy/http/response/text.py |   7 ++-
 tests/__init__.py            |   7 ++-
 tests/test_http_response.py  | 105 +++++++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+), 5 deletions(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 5bfd2debb..6eacfbd35 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -11,6 +11,7 @@ from six.moves.urllib.parse import urljoin
 import parsel
 from w3lib.encoding import html_to_unicode, resolve_encoding, \
     html_body_declared_encoding, http_content_type_encoding
+from w3lib.html import strip_html5_whitespace
 
 from scrapy.link import Link
 from scrapy.http.request import Request
@@ -142,7 +143,7 @@ class TextResponse(Response):
         if isinstance(url, Link):
             url = url.url
         elif isinstance(url, parsel.Selector):
-            url = _url_from_selector(url).strip()
+            url = _url_from_selector(url)
         elif isinstance(url, parsel.SelectorList):
             raise ValueError("SelectorList is not supported")
 
@@ -164,7 +165,7 @@ def _url_from_selector(sel):
     # type: (parsel.Selector) -> str
     if isinstance(sel.root, six.string_types):
         # e.g. ::attr(href) result
-        return sel.root
+        return strip_html5_whitespace(sel.root)
     if not hasattr(sel.root, 'tag'):
         raise ValueError("Unsupported selector: %s" % sel)
     if sel.root.tag != 'a':
@@ -173,4 +174,4 @@ def _url_from_selector(sel):
     href = sel.root.get('href')
     if href is None:
         raise ValueError("<a> element has no href attribute: %s" % sel)
-    return href
+    return strip_html5_whitespace(href)
diff --git a/tests/__init__.py b/tests/__init__.py
index d940f28ea..c2e4fd2bf 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -26,9 +26,12 @@ try:
 except ImportError:
     import mock
 
-tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data')
+tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+                             'sample_data')
+
 
 def get_testdata(*paths):
     """Return test data"""
     path = os.path.join(tests_datadir, *paths)
-    return open(path, 'rb').read()
+    with open(path, 'rb') as f:
+        return f.read()
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 9df3bf6e7..2a9baf5ed 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 import unittest
 
 import six
@@ -8,6 +9,8 @@ from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
 from scrapy.selector import Selector
 from scrapy.utils.python import to_native_str
 from scrapy.exceptions import NotSupported
+from scrapy.link import Link
+from tests import get_testdata
 
 
 class BaseResponseTest(unittest.TestCase):
@@ -356,6 +359,11 @@ class HtmlResponseTest(TextResponseTest):
 
     response_class = HtmlResponse
 
+    def _links_response(self):
+        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
+        resp = self.response_class('http://example.com/index', body=body)
+        return resp
+
     def test_html_encoding(self):
 
         body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
@@ -388,6 +396,103 @@ class HtmlResponseTest(TextResponseTest):
         r1 = self.response_class("http://www.example.com", body=body)
         self._assert_response_values(r1, 'gb2312', body)
 
+    def assert_followed_url(self, follow_obj, target_url, response=None):
+        if response is None:
+            response = self._links_response()
+        req = response.follow(follow_obj)
+        self.assertEqual(req.url, target_url)
+        return req
+
+    def test_follow_url_absolute(self):
+        self.assert_followed_url('http://foo.example.com',
+                                 'http://foo.example.com')
+
+    def test_follow_url_relative(self):
+        self.assert_followed_url('foo',
+                                 'http://example.com/foo')
+
+    def test_follow_link(self):
+        self.assert_followed_url(Link('http://example.com/foo'),
+                                 'http://example.com/foo')
+
+    def test_follow_selector(self):
+        resp = self._links_response()
+        urls = [
+            'http://example.com/sample2.html',
+            'http://example.com/sample3.html',
+            'http://example.com/sample3.html',
+            'http://www.google.com/something',
+            'http://example.com/innertag.html'
+        ]
+
+        # select <a> elements
+        for sellist in [resp.css('a'), resp.xpath('//a')]:
+            for sel, url in zip(sellist, urls):
+                self.assert_followed_url(sel, url, response=resp)
+
+        # href attributes should work
+        for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]:
+            for sel, url in zip(sellist, urls):
+                self.assert_followed_url(sel, url, response=resp)
+
+        # non-a elements are not supported
+        self.assertRaises(ValueError, resp.follow, resp.css('div')[0])
+
+    def test_follow_selector_list(self):
+        resp = self._links_response()
+        self.assertRaisesRegex(ValueError, 'SelectorList',
+                               resp.follow, resp.css('a'))
+
+    def test_follow_selector_attribute(self):
+        resp = self._links_response()
+        for src in resp.css('img::attr(src)'):
+            self.assert_followed_url(src, 'http://example.com/sample2.jpg')
+
+    def test_follow_whitespace_url(self):
+        self.assert_followed_url('foo ',
+                                 'http://example.com/foo%20')
+
+    def test_follow_whitespace_link(self):
+        self.assert_followed_url(Link('http://example.com/foo '),
+                                 'http://example.com/foo%20')
+
+    def test_follow_whitespace_selector(self):
+        resp = self.response_class(
+            'http://example.com',
+            body=b'''<html><body><a href=" foo\n">click me</a></body></html>'''
+        )
+        self.assert_followed_url(resp.css('a')[0],
+                                 'http://example.com/foo',
+                                 response=resp)
+        self.assert_followed_url(resp.css('a::attr(href)')[0],
+                                 'http://example.com/foo',
+                                 response=resp)
+
+    def test_follow_encoding(self):
+        resp1 = self.response_class(
+            'http://example.com',
+            encoding='utf8',
+            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
+        )
+        req = self.assert_followed_url(
+            resp1.css('a')[0],
+            'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82',
+            response=resp1,
+        )
+        self.assertEqual(req.encoding, 'utf8')
+
+        resp2 = self.response_class(
+            'http://example.com',
+            encoding='cp1251',
+            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
+        )
+        req = self.assert_followed_url(
+            resp2.css('a')[0],
+            'http://example.com/foo?%EF%F0%E8%E2%E5%F2',
+            response=resp2,
+        )
+        self.assertEqual(req.encoding, 'cp1251')
+
 
 class XmlResponseTest(TextResponseTest):
 

From 2674f317df9e4970f5953db3a6df04331246d8c9 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Feb 2017 04:39:47 +0500
Subject: [PATCH 1291/2974] Response.follow

---
 scrapy/http/response/__init__.py |  29 +++++
 scrapy/http/response/text.py     |  27 ++--
 tests/test_http_response.py      | 204 +++++++++++++++----------------
 3 files changed, 143 insertions(+), 117 deletions(-)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 58ad414f1..e5fb4eef8 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -6,7 +6,9 @@ See documentation in docs/topics/request-response.rst
 """
 from six.moves.urllib.parse import urljoin
 
+from scrapy.http.request import Request
 from scrapy.http.headers import Headers
+from scrapy.link import Link
 from scrapy.utils.trackref import object_ref
 from scrapy.http.common import obsolete_setter
 from scrapy.exceptions import NotSupported
@@ -101,3 +103,30 @@ class Response(object_ref):
         is text (subclasses of TextResponse).
         """
         raise NotSupported("Response content isn't text")
+
+    def follow(self, url, callback=None, method='GET', headers=None, body=None,
+               cookies=None, meta=None, encoding='utf-8', priority=0,
+               dont_filter=False, errback=None):
+        # type: (...) -> Request
+        """
+        Return a scrapy.Request instance to follow a link ``url``.
+
+        ``url`` can be:
+
+        * absolute URL;
+        * relative URL;
+        * scrapy.link.Link object.
+        """
+        if isinstance(url, Link):
+            url = url.url
+        url = self.urljoin(url)
+        return Request(url, callback,
+                       method=method,
+                       headers=headers,
+                       body=body,
+                       cookies=cookies,
+                       meta=meta,
+                       encoding=encoding,
+                       priority=priority,
+                       dont_filter=dont_filter,
+                       errback=errback)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 6eacfbd35..3c360bcf9 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -140,25 +140,22 @@ class TextResponse(Response):
         * a Selector for ``<a>`` element, e.g.
           ``response.css('a.my_link')[0]``.
         """
-        if isinstance(url, Link):
-            url = url.url
-        elif isinstance(url, parsel.Selector):
+        if isinstance(url, parsel.Selector):
             url = _url_from_selector(url)
         elif isinstance(url, parsel.SelectorList):
             raise ValueError("SelectorList is not supported")
-
         encoding = self.encoding if encoding is None else encoding
-        url = self.urljoin(url)
-        return Request(url, callback,
-                       method=method,
-                       headers=headers,
-                       body=body,
-                       cookies=cookies,
-                       meta=meta,
-                       encoding=encoding,
-                       priority=priority,
-                       dont_filter=dont_filter,
-                       errback=errback)
+        return super(TextResponse, self).follow(url, callback,
+            method=method,
+            headers=headers,
+            body=body,
+            cookies=cookies,
+            meta=meta,
+            encoding=encoding,
+            priority=priority,
+            dont_filter=dont_filter,
+            errback=errback
+        )
 
 
 def _url_from_selector(sel):
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 2a9baf5ed..e64d0eeba 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -143,6 +143,38 @@ class BaseResponseTest(unittest.TestCase):
             r.css('body')
             r.xpath('//body')
 
+    def test_follow_url_absolute(self):
+        self._assert_followed_url('http://foo.example.com',
+                                  'http://foo.example.com')
+
+    def test_follow_url_relative(self):
+        self._assert_followed_url('foo',
+                                  'http://example.com/foo')
+
+    def test_follow_link(self):
+        self._assert_followed_url(Link('http://example.com/foo'),
+                                  'http://example.com/foo')
+
+    def test_follow_whitespace_url(self):
+        self._assert_followed_url('foo ',
+                                  'http://example.com/foo%20')
+
+    def test_follow_whitespace_link(self):
+        self._assert_followed_url(Link('http://example.com/foo '),
+                                  'http://example.com/foo%20')
+
+    def _assert_followed_url(self, follow_obj, target_url, response=None):
+        if response is None:
+            response = self._links_response()
+        req = response.follow(follow_obj)
+        self.assertEqual(req.url, target_url)
+        return req
+
+    def _links_response(self):
+        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
+        resp = self.response_class('http://example.com/index', body=body)
+        return resp
+
 
 class TextResponseTest(BaseResponseTest):
 
@@ -354,16 +386,81 @@ class TextResponseTest(BaseResponseTest):
         absolute = 'http://www.example.com/elsewhere/test'
         self.assertEqual(joined, absolute)
 
+    def test_follow_selector(self):
+        resp = self._links_response()
+        urls = [
+            'http://example.com/sample2.html',
+            'http://example.com/sample3.html',
+            'http://example.com/sample3.html',
+            'http://www.google.com/something',
+            'http://example.com/innertag.html'
+        ]
+
+        # select <a> elements
+        for sellist in [resp.css('a'), resp.xpath('//a')]:
+            for sel, url in zip(sellist, urls):
+                self._assert_followed_url(sel, url, response=resp)
+
+        # href attributes should work
+        for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]:
+            for sel, url in zip(sellist, urls):
+                self._assert_followed_url(sel, url, response=resp)
+
+        # non-a elements are not supported
+        self.assertRaises(ValueError, resp.follow, resp.css('div')[0])
+
+    def test_follow_selector_list(self):
+        resp = self._links_response()
+        self.assertRaisesRegex(ValueError, 'SelectorList',
+                               resp.follow, resp.css('a'))
+
+    def test_follow_selector_attribute(self):
+        resp = self._links_response()
+        for src in resp.css('img::attr(src)'):
+            self._assert_followed_url(src, 'http://example.com/sample2.jpg')
+
+    def test_follow_whitespace_selector(self):
+        resp = self.response_class(
+            'http://example.com',
+            body=b'''<html><body><a href=" foo\n">click me</a></body></html>'''
+        )
+        self._assert_followed_url(resp.css('a')[0],
+                                 'http://example.com/foo',
+                                  response=resp)
+        self._assert_followed_url(resp.css('a::attr(href)')[0],
+                                 'http://example.com/foo',
+                                  response=resp)
+
+    def test_follow_encoding(self):
+        resp1 = self.response_class(
+            'http://example.com',
+            encoding='utf8',
+            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
+        )
+        req = self._assert_followed_url(
+            resp1.css('a')[0],
+            'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82',
+            response=resp1,
+        )
+        self.assertEqual(req.encoding, 'utf8')
+
+        resp2 = self.response_class(
+            'http://example.com',
+            encoding='cp1251',
+            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
+        )
+        req = self._assert_followed_url(
+            resp2.css('a')[0],
+            'http://example.com/foo?%EF%F0%E8%E2%E5%F2',
+            response=resp2,
+        )
+        self.assertEqual(req.encoding, 'cp1251')
+
 
 class HtmlResponseTest(TextResponseTest):
 
     response_class = HtmlResponse
 
-    def _links_response(self):
-        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
-        resp = self.response_class('http://example.com/index', body=body)
-        return resp
-
     def test_html_encoding(self):
 
         body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
@@ -396,103 +493,6 @@ class HtmlResponseTest(TextResponseTest):
         r1 = self.response_class("http://www.example.com", body=body)
         self._assert_response_values(r1, 'gb2312', body)
 
-    def assert_followed_url(self, follow_obj, target_url, response=None):
-        if response is None:
-            response = self._links_response()
-        req = response.follow(follow_obj)
-        self.assertEqual(req.url, target_url)
-        return req
-
-    def test_follow_url_absolute(self):
-        self.assert_followed_url('http://foo.example.com',
-                                 'http://foo.example.com')
-
-    def test_follow_url_relative(self):
-        self.assert_followed_url('foo',
-                                 'http://example.com/foo')
-
-    def test_follow_link(self):
-        self.assert_followed_url(Link('http://example.com/foo'),
-                                 'http://example.com/foo')
-
-    def test_follow_selector(self):
-        resp = self._links_response()
-        urls = [
-            'http://example.com/sample2.html',
-            'http://example.com/sample3.html',
-            'http://example.com/sample3.html',
-            'http://www.google.com/something',
-            'http://example.com/innertag.html'
-        ]
-
-        # select <a> elements
-        for sellist in [resp.css('a'), resp.xpath('//a')]:
-            for sel, url in zip(sellist, urls):
-                self.assert_followed_url(sel, url, response=resp)
-
-        # href attributes should work
-        for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]:
-            for sel, url in zip(sellist, urls):
-                self.assert_followed_url(sel, url, response=resp)
-
-        # non-a elements are not supported
-        self.assertRaises(ValueError, resp.follow, resp.css('div')[0])
-
-    def test_follow_selector_list(self):
-        resp = self._links_response()
-        self.assertRaisesRegex(ValueError, 'SelectorList',
-                               resp.follow, resp.css('a'))
-
-    def test_follow_selector_attribute(self):
-        resp = self._links_response()
-        for src in resp.css('img::attr(src)'):
-            self.assert_followed_url(src, 'http://example.com/sample2.jpg')
-
-    def test_follow_whitespace_url(self):
-        self.assert_followed_url('foo ',
-                                 'http://example.com/foo%20')
-
-    def test_follow_whitespace_link(self):
-        self.assert_followed_url(Link('http://example.com/foo '),
-                                 'http://example.com/foo%20')
-
-    def test_follow_whitespace_selector(self):
-        resp = self.response_class(
-            'http://example.com',
-            body=b'''<html><body><a href=" foo\n">click me</a></body></html>'''
-        )
-        self.assert_followed_url(resp.css('a')[0],
-                                 'http://example.com/foo',
-                                 response=resp)
-        self.assert_followed_url(resp.css('a::attr(href)')[0],
-                                 'http://example.com/foo',
-                                 response=resp)
-
-    def test_follow_encoding(self):
-        resp1 = self.response_class(
-            'http://example.com',
-            encoding='utf8',
-            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
-        )
-        req = self.assert_followed_url(
-            resp1.css('a')[0],
-            'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82',
-            response=resp1,
-        )
-        self.assertEqual(req.encoding, 'utf8')
-
-        resp2 = self.response_class(
-            'http://example.com',
-            encoding='cp1251',
-            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
-        )
-        req = self.assert_followed_url(
-            resp2.css('a')[0],
-            'http://example.com/foo?%EF%F0%E8%E2%E5%F2',
-            response=resp2,
-        )
-        self.assertEqual(req.encoding, 'cp1251')
-
 
 class XmlResponseTest(TextResponseTest):
 

From 160da6abab8954906181ce69593ff6e84d950ac1 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Feb 2017 04:41:53 +0500
Subject: [PATCH 1292/2974] fixed tests in Python 2

---
 tests/test_http_response.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index e64d0eeba..fa74b468b 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -411,8 +411,8 @@ class TextResponseTest(BaseResponseTest):
 
     def test_follow_selector_list(self):
         resp = self._links_response()
-        self.assertRaisesRegex(ValueError, 'SelectorList',
-                               resp.follow, resp.css('a'))
+        self.assertRaisesRegexp(ValueError, 'SelectorList',
+                                resp.follow, resp.css('a'))
 
     def test_follow_selector_attribute(self):
         resp = self._links_response()
@@ -435,7 +435,7 @@ class TextResponseTest(BaseResponseTest):
         resp1 = self.response_class(
             'http://example.com',
             encoding='utf8',
-            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
+            body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
         )
         req = self._assert_followed_url(
             resp1.css('a')[0],
@@ -447,7 +447,7 @@ class TextResponseTest(BaseResponseTest):
         resp2 = self.response_class(
             'http://example.com',
             encoding='cp1251',
-            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
+            body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
         )
         req = self._assert_followed_url(
             resp2.css('a')[0],

From 5b79c6a679b66868c89302a1693e5dedc62b6f61 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 16 Feb 2017 00:06:52 +0500
Subject: [PATCH 1293/2974] DOC document response.follow methods; expand the
 tutorial

---
 docs/intro/tutorial.rst          | 41 +++++++++++++++++++++++++-------
 docs/topics/request-response.rst |  7 +++---
 scrapy/http/response/__init__.py | 15 ++++++------
 scrapy/http/response/text.py     | 18 +++++++-------
 4 files changed, 54 insertions(+), 27 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index d47bf69e5..3b3bd8d21 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -399,7 +399,7 @@ quotes elements and put them together into a Python dictionary::
     >>>
 
 Extracting data in our spider
-------------------------------
+-----------------------------
 
 Let's get back to our spider. Until now, it doesn't extract any data in
 particular, just saves the whole HTML page to a local file. Let's integrate the
@@ -551,8 +551,14 @@ In our example, it creates a sort of loop, following all the links to the next p
 until it doesn't find one -- handy for crawling blogs, forums and other sites with
 pagination.
 
+
+.. _response-follow-example:
+
+A shortcut for creating Requests
+--------------------------------
+
 As a shortcut for creating Request objects you can use
-:meth:`response.follow <scrapy.http.TextResponse.follow>` method::
+:meth:`response.follow <scrapy.http.TextResponse.follow>`::
 
     import scrapy
 
@@ -571,13 +577,32 @@ As a shortcut for creating Request objects you can use
                     'tags': quote.css('div.tags a.tag::text').extract(),
                 }
 
-            for href in response.css('li.next a::attr(href)'):
-                yield response.follow(href, callback=self.parse)
+            next_page = response.css('li.next a::attr(href)').extract_first()
+            if next_page is not None:
+                yield response.follow(next_page, callback=self.parse)
 
-Unlike scrapy.Request, ``response.follow`` supports
-relative URLs directly; you can also pass a selector to it instead of
-a string. Note that ``response.follow`` just returns a Request instance;
-you still have to yield this Request.
+Unlike scrapy.Request, ``response.follow`` supports relative URLs directly - no
+need to call urljoin. Note that ``response.follow`` just returns a Request
+instance; you still have to yield this Request.
+
+You can also pass a selector to ``response.follow`` instead of a string;
+this selector should extract necessary attributes::
+
+    for href in response.css('li.next a::attr(href)'):
+        yield response.follow(href, callback=self.parse)
+
+For ``<a>`` elements there is a shortcut: ``response.follow`` uses their href
+attribute automatically. So the code can be shortened further::
+
+    for a in response.css('li.next a'):
+        yield response.follow(a, callback=self.parse)
+
+.. note::
+
+    ``response.follow(response.css('li.next a'))`` is not valid because
+    ``response.css`` returns a list-like object with selectors for all results,
+    not a single selector. A ``for`` loop like in the example above, or
+    ``response.follow(response.css('li.next a')[0])`` is fine.
 
 More examples and patterns
 --------------------------
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 71050fddd..3e80f18b5 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -597,6 +597,9 @@ Response objects
 
             urlparse.urljoin(response.url, url)
 
+    .. automethod:: Response.follow
+
+
 .. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
 
 .. _topics-request-response-ref-response-subclasses:
@@ -683,9 +686,7 @@ TextResponse objects
 
             response.css('p')
 
-    .. method:: TextResponse.follow(url, ...)
-
-        Return a scrapy.Request instance to follow a link ``url``.
+    .. automethod:: TextResponse.follow
 
     .. method:: TextResponse.body_as_unicode()
 
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index e5fb4eef8..434d87eab 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -109,13 +109,14 @@ class Response(object_ref):
                dont_filter=False, errback=None):
         # type: (...) -> Request
         """
-        Return a scrapy.Request instance to follow a link ``url``.
-
-        ``url`` can be:
-
-        * absolute URL;
-        * relative URL;
-        * scrapy.link.Link object.
+        Return a :class:`~.Request` instance to follow a link ``url``.
+        It accepts the same arguments as ``Request.__init__`` method,
+        but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
+        not only an absolute URL.
+        
+        :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow` 
+        method which supports selectors in addition to absolute/relative URLs
+        and Link objects.
         """
         if isinstance(url, Link):
             url = url.url
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 3c360bcf9..6415e191a 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -13,7 +13,6 @@ from w3lib.encoding import html_to_unicode, resolve_encoding, \
     html_body_declared_encoding, http_content_type_encoding
 from w3lib.html import strip_html5_whitespace
 
-from scrapy.link import Link
 from scrapy.http.request import Request
 from scrapy.http.response import Response
 from scrapy.utils.response import get_base_url
@@ -127,18 +126,19 @@ class TextResponse(Response):
                dont_filter=False, errback=None):
         # type: (...) -> Request
         """
-        Return a scrapy.Request instance to follow a link ``url``.
-
-        ``url`` can be:
-
-        * absolute URL;
-        * relative URL;
-        * scrapy.link.Link object (e.g. a link extractor result);
-        * attribute Selector (not SelectorList) - e.g.
+        Return a :class:`~.Request` instance to follow a link ``url``.
+        It accepts the same arguments as ``Request.__init__`` method,
+        but ``url`` can be not only an absolute URL, but also
+        
+        * a relative URL;
+        * a scrapy.link.Link object (e.g. a link extractor result);
+        * an attribute Selector (not SelectorList) - e.g.
           ``response.css('a::attr(href)')[0]`` or
           ``response.xpath('//img/@src')[0]``.
         * a Selector for ``<a>`` element, e.g.
           ``response.css('a.my_link')[0]``.
+          
+        See :ref:`response-follow-example` for usage examples.
         """
         if isinstance(url, parsel.Selector):
             url = _url_from_selector(url)

From fade5763af3d03f076f3317589038201bbdeccaf Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 16 Feb 2017 02:02:50 +0500
Subject: [PATCH 1294/2974] TST more response.follow tests

---
 tests/test_http_response.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index fa74b468b..924bb7979 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -414,11 +414,24 @@ class TextResponseTest(BaseResponseTest):
         self.assertRaisesRegexp(ValueError, 'SelectorList',
                                 resp.follow, resp.css('a'))
 
+    def test_follow_selector_invalid(self):
+        resp = self._links_response()
+        self.assertRaisesRegexp(ValueError, 'Unsupported',
+                                resp.follow, resp.xpath('count(//div)')[0])
+
     def test_follow_selector_attribute(self):
         resp = self._links_response()
         for src in resp.css('img::attr(src)'):
             self._assert_followed_url(src, 'http://example.com/sample2.jpg')
 
+    def test_follow_selector_no_href(self):
+        resp = self.response_class(
+            url='http://example.com',
+            body=b'<html><body><a name=123>click me</a></body></html>',
+        )
+        self.assertRaisesRegexp(ValueError, 'no href',
+                                resp.follow, resp.css('a')[0])
+
     def test_follow_whitespace_selector(self):
         resp = self.response_class(
             'http://example.com',

From 074caf434e255bc96f106e57e3e288028f372485 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 9 Feb 2017 00:17:56 +0500
Subject: [PATCH 1295/2974] FormRequest: handle whitespaces in action attribute
 properly

---
 scrapy/http/request/form.py | 10 ++++++++--
 tests/test_http_request.py  | 10 ++++++++--
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 2862dc096..905d8412f 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -5,10 +5,13 @@ This module implements the FormRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 
+import six
 from six.moves.urllib.parse import urljoin, urlencode
+
 import lxml.html
 from parsel.selector import create_root_node
-import six
+from w3lib.html import strip_html5_whitespace
+
 from scrapy.http.request import Request
 from scrapy.utils.python import to_bytes, is_listlike
 from scrapy.utils.response import get_base_url
@@ -51,7 +54,10 @@ class FormRequest(Request):
 
 def _get_form_url(form, url):
     if url is None:
-        return urljoin(form.base_url, form.action)
+        action = form.get('action')
+        if action is None:
+            return form.base_url
+        return urljoin(form.base_url, strip_html5_whitespace(action))
     return urljoin(form.base_url, url)
 
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index d7216e1d2..7eadb874f 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -556,7 +556,6 @@ class FormRequestTest(RequestTest):
         fs = _qs(req, to_unicode=True, encoding='latin1')
         self.assertTrue(fs[u'price in \u00a5'])
 
-
     def test_from_response_multiple_forms_clickdata(self):
         response = _buildresponse(
             """<form name="form1">
@@ -989,7 +988,7 @@ class FormRequestTest(RequestTest):
             """
             <html>
                 <head>
-                    <base href="http://b.com/">
+                    <base href=" http://b.com/">
                 </head>
                 <body>
                     <form action="test_form">
@@ -1002,6 +1001,11 @@ class FormRequestTest(RequestTest):
         req = self.request_class.from_response(response)
         self.assertEqual(req.url, 'http://b.com/test_form')
 
+    def test_spaces_in_action(self):
+        resp = _buildresponse('<body><form action=" path\n"></form></body>')
+        req = self.request_class.from_response(resp)
+        self.assertEqual(req.url, 'http://example.com/path')
+
     def test_from_response_css(self):
         response = _buildresponse(
             """<form action="post.php" method="POST">
@@ -1023,12 +1027,14 @@ class FormRequestTest(RequestTest):
         self.assertRaises(ValueError, self.request_class.from_response,
                           response, formcss="input[name='abc']")
 
+
 def _buildresponse(body, **kwargs):
     kwargs.setdefault('body', body)
     kwargs.setdefault('url', 'http://example.com')
     kwargs.setdefault('encoding', 'utf-8')
     return HtmlResponse(**kwargs)
 
+
 def _qs(req, encoding='utf-8', to_unicode=False):
     if req.method == 'POST':
         qs = req.body

From ad36a4a6ae8376a779f9feb08adfb2ca4a59dbb4 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 8 Feb 2017 16:58:38 +0500
Subject: [PATCH 1296/2974] RegexLinkExtractor: add \x0c to whitespace
 characters, as per html5 standard

---
 scrapy/linkextractors/regex.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py
index 0fc7b079f..e689b4727 100644
--- a/scrapy/linkextractors/regex.py
+++ b/scrapy/linkextractors/regex.py
@@ -10,9 +10,10 @@ linkre = re.compile(
         "<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
         re.DOTALL | re.IGNORECASE)
 
+
 def clean_link(link_text):
     """Remove leading and trailing whitespace and punctuation"""
-    return link_text.strip("\t\r\n '\"")
+    return link_text.strip("\t\r\n '\"\x0c")
 
 
 class RegexLinkExtractor(SgmlLinkExtractor):

From d079e15fe2fcf269d040ac435e2eb414d2b4c334 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 8 Feb 2017 17:03:11 +0500
Subject: [PATCH 1297/2974] Strip leading/trailing whitespaces in link
 extractors. Fixes GH-838.

---
 docs/topics/link-extractors.rst               | 10 +++++++++-
 scrapy/linkextractors/htmlparser.py           |  8 ++++++--
 scrapy/linkextractors/lxmlhtml.py             | 16 ++++++++++-----
 scrapy/linkextractors/sgml.py                 | 13 ++++++++----
 scrapy/utils/url.py                           | 13 ++++++++++++
 .../link_extractor/sgml_linkextractor.html    |  1 +
 tests/test_linkextractors.py                  |  3 +++
 tests/test_linkextractors_deprecated.py       | 20 +++++++++++++------
 8 files changed, 66 insertions(+), 18 deletions(-)

diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 4636ddb18..2486e0982 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -51,7 +51,7 @@ LxmlLinkExtractor
    :synopsis: lxml's HTMLParser-based link extractors
 
 
-.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None)
+.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, strip=True)
 
     LxmlLinkExtractor is the recommended link extractor with handy filtering
     options. It is implemented using lxml's robust HTMLParser.
@@ -132,4 +132,12 @@ LxmlLinkExtractor
 
     :type process_value: callable
 
+    :param strip: whether to strip whitespaces from extracted attributes.
+        According to HTML5 standard, leading and trailing whitespaces
+        must be stripped from ``href`` attributes of ``<a>`` and ``<area>``
+        elements, so LinkExtractor strips them by default. Set ``strip=False``
+        to turn it off (e.g. if you're extracting urls from elements or
+        attributes which allow leading/trailing whitespaces).
+    :type strip: boolean
+
 .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py
diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 9867e1179..4841e4a54 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -1,7 +1,6 @@
 """
 HTMLParser-based link extractor
 """
-
 import warnings
 import six
 from six.moves.html_parser import HTMLParser
@@ -11,12 +10,14 @@ from w3lib.url import safe_url_string
 
 from scrapy.link import Link
 from scrapy.utils.python import unique as unique_list
+from scrapy.utils.url import trim_href_attribute
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
 class HtmlParserLinkExtractor(HTMLParser):
 
-    def __init__(self, tag="a", attr="href", process=None, unique=False):
+    def __init__(self, tag="a", attr="href", process=None, unique=False,
+                 strip=True):
         HTMLParser.__init__(self)
 
         warnings.warn(
@@ -29,6 +30,7 @@ class HtmlParserLinkExtractor(HTMLParser):
         self.scan_attr = attr if callable(attr) else lambda a: a == attr
         self.process_attr = process if callable(process) else lambda v: v
         self.unique = unique
+        self.strip = strip
 
     def _extract_links(self, response_text, response_url, response_encoding):
         self.reset()
@@ -70,6 +72,8 @@ class HtmlParserLinkExtractor(HTMLParser):
             for attr, value in attrs:
                 if self.scan_attr(attr):
                     url = self.process_attr(value)
+                    if self.strip:
+                        url = trim_href_attribute(url)
                     link = Link(url=url)
                     self.links.append(link)
                     self.current_link = link
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 71d57b392..f753033ab 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -9,8 +9,9 @@ import lxml.etree as etree
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list, to_native_str
-from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.response import get_base_url
+from scrapy.utils.url import trim_href_attribute
+from scrapy.linkextractors import FilteringLinkExtractor
 
 
 # from lxml/src/lxml/html/__init__.py
@@ -27,11 +28,13 @@ def _nons(tag):
 
 
 class LxmlParserLinkExtractor(object):
-    def __init__(self, tag="a", attr="href", process=None, unique=False):
+    def __init__(self, tag="a", attr="href", process=None, unique=False,
+                 strip=True):
         self.scan_tag = tag if callable(tag) else lambda t: t == tag
         self.scan_attr = attr if callable(attr) else lambda a: a == attr
         self.process_attr = process if callable(process) else lambda v: v
         self.unique = unique
+        self.strip = strip
 
     def _iter_links(self, document):
         for el in document.iter(etree.Element):
@@ -49,9 +52,11 @@ class LxmlParserLinkExtractor(object):
         for el, attr, attr_val in self._iter_links(selector.root):
             # pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
             try:
+                if self.strip:
+                    attr_val = trim_href_attribute(attr_val)
                 attr_val = urljoin(base_url, attr_val)
             except ValueError:
-                continue # skipping bogus links
+                continue  # skipping bogus links
             else:
                 url = self.process_attr(attr_val)
                 if url is None:
@@ -85,12 +90,13 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
 
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
                  tags=('a', 'area'), attrs=('href',), canonicalize=True,
-                 unique=True, process_value=None, deny_extensions=None, restrict_css=()):
+                 unique=True, process_value=None, deny_extensions=None, restrict_css=(),
+                 strip=True):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
         tag_func = lambda x: x in tags
         attr_func = lambda x: x in attrs
         lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func,
-            unique=unique, process=process_value)
+            unique=unique, process=process_value, strip=strip)
 
         super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
             allow_domains=allow_domains, deny_domains=deny_domains,
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index c68dae4c8..6ecfd52aa 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -7,18 +7,19 @@ import warnings
 from sgmllib import SGMLParser
 
 from w3lib.url import safe_url_string
-from scrapy.selector import Selector
 from scrapy.link import Link
 from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list, to_unicode
 from scrapy.utils.response import get_base_url
+from scrapy.utils.url import trim_href_attribute
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
 class BaseSgmlLinkExtractor(SGMLParser):
 
-    def __init__(self, tag="a", attr="href", unique=False, process_value=None):
+    def __init__(self, tag="a", attr="href", unique=False, process_value=None,
+                 strip=True):
         warnings.warn(
             "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
             "Please use scrapy.linkextractors.LinkExtractor",
@@ -30,6 +31,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
         self.process_value = (lambda v: v) if process_value is None else process_value
         self.current_link = None
         self.unique = unique
+        self.strip = strip
 
     def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
         """ Do the real extraction work """
@@ -81,6 +83,8 @@ class BaseSgmlLinkExtractor(SGMLParser):
                 if self.scan_attr(attr):
                     url = self.process_value(value)
                     if url is not None:
+                        if self.strip:
+                            url = trim_href_attribute(url)
                         link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
                         self.links.append(link)
                         self.current_link = link
@@ -103,7 +107,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
 
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
                  tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True,
-                 process_value=None, deny_extensions=None, restrict_css=()):
+                 process_value=None, deny_extensions=None, restrict_css=(),
+                 strip=True):
 
         warnings.warn(
             "SgmlLinkExtractor is deprecated and will be removed in future releases. "
@@ -118,7 +123,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
         with warnings.catch_warnings():
             warnings.simplefilter('ignore', ScrapyDeprecationWarning)
             lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
-                unique=unique, process_value=process_value)
+                unique=unique, process_value=process_value, strip=strip)
 
         super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
             allow_domains=allow_domains, deny_domains=deny_domains,
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index dc1cce4ac..090f65f80 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -103,3 +103,16 @@ def guess_scheme(url):
         return any_to_uri(url)
     else:
         return add_http_if_no_scheme(url)
+
+
+def trim_href_attribute(href):
+    """
+    Process href attribute of ``a`` or ``area`` elements according to HTML5
+    standards (strip all leading and trailing whitespaces). References:
+
+    * https://www.w3.org/TR/html5/links.html#links-created-by-a-and-area-elements
+    * https://www.w3.org/TR/html5/infrastructure.html#valid-url-potentially-surrounded-by-spaces
+    * https://www.w3.org/TR/html5/infrastructure.html#strip-leading-and-trailing-whitespace
+    * https://www.w3.org/TR/html5/infrastructure.html#space-character
+    """
+    return href.strip(' \t\n\r\x0c')
diff --git a/tests/sample_data/link_extractor/sgml_linkextractor.html b/tests/sample_data/link_extractor/sgml_linkextractor.html
index 35aa457ee..fbb803f2d 100644
--- a/tests/sample_data/link_extractor/sgml_linkextractor.html
+++ b/tests/sample_data/link_extractor/sgml_linkextractor.html
@@ -13,6 +13,7 @@
 <a href='sample3.html'>sample 3 repetition</a>
 <a href='http://www.google.com/something'></a>
 <a href='http://example.com/innertag.html'><b>inner</b> tag</a>
+<a href=' page 4.html '>href with whitespaces</a>
 </div>
 </body>
 </html>
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 129336d14..340c64f35 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -32,6 +32,7 @@ class Base:
                 Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
                 Link(url='http://www.google.com/something', text=u''),
                 Link(url='http://example.com/innertag.html', text=u'inner tag'),
+                Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
             ])
 
         def test_extract_filter_allow(self):
@@ -281,6 +282,7 @@ class Base:
                 Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
                 Link(url='http://www.google.com/something', text=u''),
                 Link(url='http://example.com/innertag.html', text=u'inner tag'),
+                Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
             ])
 
             lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())
@@ -291,6 +293,7 @@ class Base:
                 Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
                 Link(url='http://www.google.com/something', text=u''),
                 Link(url='http://example.com/innertag.html', text=u'inner tag'),
+                Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
             ])
 
             lx = self.extractor_cls(attrs=None)
diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py
index 36dfe174f..fef227aa1 100644
--- a/tests/test_linkextractors_deprecated.py
+++ b/tests/test_linkextractors_deprecated.py
@@ -117,12 +117,14 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase):
     def test_extraction(self):
         # Default arguments
         lx = HtmlParserLinkExtractor()
-        self.assertEqual(lx.extract_links(self.response),
-                         [Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                          Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-                          Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
-                          Link(url='http://www.google.com/something', text=u''),
-                          Link(url='http://example.com/innertag.html', text=u'inner tag'),])
+        self.assertEqual(lx.extract_links(self.response), [
+            Link(url='http://example.com/sample2.html', text=u'sample 2'),
+            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+            Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+            Link(url='http://www.google.com/something', text=u''),
+            Link(url='http://example.com/innertag.html', text=u'inner tag'),
+            Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
+        ])
 
     def test_link_wrong_href(self):
         html = """
@@ -220,3 +222,9 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
         self.assertEqual([link for link in lx.extract_links(response)], [
             Link(url='http://b.com/test.html', text=u'', nofollow=False),
         ])
+
+    @unittest.expectedFailure
+    def test_extraction(self):
+        # RegexLinkExtractor doesn't parse URLs with leading/trailing
+        # whitespaces correctly.
+        super(RegexLinkExtractorTestCase, self).test_extraction()

From d09eed7674b5df2a1883a7c7abad40fdfd062c74 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 8 Feb 2017 23:44:55 +0500
Subject: [PATCH 1298/2974] use w3lib.html.strip_html5_whitespace function;
 expand docs; strip consistently before calling process_value

---
 docs/topics/link-extractors.rst     |  9 +++++----
 scrapy/linkextractors/htmlparser.py |  6 +++---
 scrapy/linkextractors/lxmlhtml.py   |  6 +++---
 scrapy/linkextractors/sgml.py       |  7 ++++---
 scrapy/utils/url.py                 | 13 -------------
 5 files changed, 15 insertions(+), 26 deletions(-)

diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 2486e0982..75bdb4142 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -134,10 +134,11 @@ LxmlLinkExtractor
 
     :param strip: whether to strip whitespaces from extracted attributes.
         According to HTML5 standard, leading and trailing whitespaces
-        must be stripped from ``href`` attributes of ``<a>`` and ``<area>``
-        elements, so LinkExtractor strips them by default. Set ``strip=False``
-        to turn it off (e.g. if you're extracting urls from elements or
-        attributes which allow leading/trailing whitespaces).
+        must be stripped from ``href`` attributes of ``<a>``, ``<area>``
+        and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``
+        elements, etc., so LinkExtractor strips space chars by default.
+        Set ``strip=False`` to turn it off (e.g. if you're extracting urls
+        from elements or attributes which allow leading/trailing whitespaces).
     :type strip: boolean
 
 .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py
diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 4841e4a54..27978a8a1 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -7,10 +7,10 @@ from six.moves.html_parser import HTMLParser
 from six.moves.urllib.parse import urljoin
 
 from w3lib.url import safe_url_string
+from w3lib.html import strip_html5_whitespace
 
 from scrapy.link import Link
 from scrapy.utils.python import unique as unique_list
-from scrapy.utils.url import trim_href_attribute
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
@@ -71,9 +71,9 @@ class HtmlParserLinkExtractor(HTMLParser):
         if self.scan_tag(tag):
             for attr, value in attrs:
                 if self.scan_attr(attr):
-                    url = self.process_attr(value)
                     if self.strip:
-                        url = trim_href_attribute(url)
+                        value = strip_html5_whitespace(value)
+                    url = self.process_attr(value)
                     link = Link(url=url)
                     self.links.append(link)
                     self.current_link = link
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index f753033ab..c284f1905 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -2,15 +2,15 @@
 Link extractor based on lxml.html
 """
 import six
-from six.moves.urllib.parse import urlparse, urljoin
+from six.moves.urllib.parse import urljoin
 
 import lxml.etree as etree
+from w3lib.html import strip_html5_whitespace
 
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list, to_native_str
 from scrapy.utils.response import get_base_url
-from scrapy.utils.url import trim_href_attribute
 from scrapy.linkextractors import FilteringLinkExtractor
 
 
@@ -53,7 +53,7 @@ class LxmlParserLinkExtractor(object):
             # pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
             try:
                 if self.strip:
-                    attr_val = trim_href_attribute(attr_val)
+                    attr_val = strip_html5_whitespace(attr_val)
                 attr_val = urljoin(base_url, attr_val)
             except ValueError:
                 continue  # skipping bogus links
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 6ecfd52aa..11ff7a261 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -7,12 +7,13 @@ import warnings
 from sgmllib import SGMLParser
 
 from w3lib.url import safe_url_string
+from w3lib.html import strip_html5_whitespace
+
 from scrapy.link import Link
 from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list, to_unicode
 from scrapy.utils.response import get_base_url
-from scrapy.utils.url import trim_href_attribute
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
@@ -81,10 +82,10 @@ class BaseSgmlLinkExtractor(SGMLParser):
         if self.scan_tag(tag):
             for attr, value in attrs:
                 if self.scan_attr(attr):
+                    if self.strip and value is not None:
+                        value = strip_html5_whitespace(value)
                     url = self.process_value(value)
                     if url is not None:
-                        if self.strip:
-                            url = trim_href_attribute(url)
                         link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
                         self.links.append(link)
                         self.current_link = link
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 090f65f80..dc1cce4ac 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -103,16 +103,3 @@ def guess_scheme(url):
         return any_to_uri(url)
     else:
         return add_http_if_no_scheme(url)
-
-
-def trim_href_attribute(href):
-    """
-    Process href attribute of ``a`` or ``area`` elements according to HTML5
-    standards (strip all leading and trailing whitespaces). References:
-
-    * https://www.w3.org/TR/html5/links.html#links-created-by-a-and-area-elements
-    * https://www.w3.org/TR/html5/infrastructure.html#valid-url-potentially-surrounded-by-spaces
-    * https://www.w3.org/TR/html5/infrastructure.html#strip-leading-and-trailing-whitespace
-    * https://www.w3.org/TR/html5/infrastructure.html#space-character
-    """
-    return href.strip(' \t\n\r\x0c')

From 692975acb40c6394424dfb728b1ffa46b3b3c55d Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 27 Jan 2016 00:32:15 +0500
Subject: [PATCH 1299/2974] deprecate Spider.make_requests_from_url. Fixes
 #1495.

---
 docs/topics/spiders.rst    | 26 +++++---------------------
 scrapy/spiders/__init__.py | 14 ++++++++++++--
 tests/spiders.py           |  5 +----
 tests/test_engine.py       |  4 ++--
 4 files changed, 20 insertions(+), 29 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index c123c2635..5e69055d1 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -144,16 +144,12 @@ scrapy.Spider
    .. method:: start_requests()
 
        This method must return an iterable with the first Requests to crawl for
-       this spider.
+       this spider. It is called by Scrapy when the spider is opened for
+       scraping. Scrapy calls it only once, so it is safe to implement
+       :meth:`start_requests` as a generator.
 
-       This is the method called by Scrapy when the spider is opened for
-       scraping when no particular URLs are specified. If particular URLs are
-       specified, the :meth:`make_requests_from_url` is used instead to create
-       the Requests. This method is also called only once from Scrapy, so it's
-       safe to implement it as a generator.
-
-       The default implementation uses :meth:`make_requests_from_url` to
-       generate Requests for each url in :attr:`start_urls`.
+       The default implementation generates ``Request(url, dont_filter=True)``
+       for each url in :attr:`start_urls`.
 
        If you want to change the Requests used to start scraping a domain, this is
        the method to override. For example, if you need to start by logging in using
@@ -172,18 +168,6 @@ scrapy.Spider
                    # each of them, with another callback
                    pass
 
-   .. method:: make_requests_from_url(url)
-
-       A method that receives a URL and returns a :class:`~scrapy.http.Request`
-       object (or a list of :class:`~scrapy.http.Request` objects) to scrape. This
-       method is used to construct the initial requests in the
-       :meth:`start_requests` method, and is typically used to convert urls to
-       requests.
-
-       Unless overridden, this method returns Requests with the :meth:`parse`
-       method as their callback function, and with dont_filter parameter enabled
-       (see :class:`~scrapy.http.Request` class for more info).
-
    .. method:: parse(response)
 
        This is the default callback used by Scrapy to process downloaded
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index c08bb964a..138cdbc67 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -66,10 +66,20 @@ class Spider(object_ref):
         crawler.signals.connect(self.close, signals.spider_closed)
 
     def start_requests(self):
-        for url in self.start_urls:
-            yield self.make_requests_from_url(url)
+        if self.make_requests_from_url is not Spider.make_requests_from_url:
+            warnings.warn(
+                "Spider.make_requests_from_url method is deprecated; "
+                "it won't be called in future Scrapy releases. "
+                "Please override start_requests method instead."
+            )
+            for url in self.start_urls:
+                yield self.make_requests_from_url(url)
+        else:
+            for url in self.start_urls:
+                yield Request(url, dont_filter=True)
 
     def make_requests_from_url(self, url):
+        """ This method is deprecated. """
         return Request(url, dont_filter=True)
 
     def parse(self, response):
diff --git a/tests/spiders.py b/tests/spiders.py
index 711d80cac..1038b69de 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -170,10 +170,7 @@ class DuplicateStartRequestsSpider(Spider):
         for i in range(0, self.distinct_urls):
             for j in range(0, self.dupe_factor):
                 url = "http://localhost:8998/echo?headers=1&body=test%d" % i
-                yield self.make_requests_from_url(url)
-
-    def make_requests_from_url(self, url):
-        return Request(url, dont_filter=self.dont_filter)
+                yield Request(url, dont_filter=self.dont_filter)
 
     def __init__(self, url="http://localhost:8998", *args, **kwargs):
         super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs)
diff --git a/tests/test_engine.py b/tests/test_engine.py
index baf6ef1bf..04113ddcf 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -66,8 +66,8 @@ class TestSpider(Spider):
 
 
 class TestDupeFilterSpider(TestSpider):
-    def make_requests_from_url(self, url):
-        return Request(url)  # dont_filter=False
+    def start_requests(self):
+        return (Request(url) for url in self.start_urls)  # no dont_filter=True
 
 
 class DictItemsSpider(TestSpider):

From 71dda2700b1566d09c08ed1c211240baf4e6e7f1 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 16 Feb 2017 04:22:19 +0500
Subject: [PATCH 1300/2974] =?UTF-8?q?DOC=20remove=20=E2=80=9CPython=203=20?=
 =?UTF-8?q?progress=E2=80=9D=20badge?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Badge doesn’t work, and Scrapy already works fine in Python 3
for practical purposes.
---
 README.rst | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/README.rst b/README.rst
index 38dda62e5..4eb36b44a 100644
--- a/README.rst
+++ b/README.rst
@@ -13,10 +13,6 @@ Scrapy
 .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
    :target: https://pypi.python.org/pypi/Scrapy
    :alt: Wheel Status
-   
-.. image:: http://static.scrapy.org/py3progress/badge.svg
-   :target: https://github.com/scrapy/scrapy/wiki/Python-3-Porting
-   :alt: Python 3 Porting Status
 
 .. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg
    :target: http://codecov.io/github/scrapy/scrapy?branch=master

From f0c4e5dfd587a2b522a48c141c5af0a35ea8e56e Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 16 Feb 2017 16:32:23 +0500
Subject: [PATCH 1301/2974] TST fixed ProjectUtilsTest on OS X

Temp folder can be a symlink on OS X.
---
 tests/test_utils_project.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index 1239195e8..6b7fcd4c2 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -31,5 +31,8 @@ class ProjectUtilsTest(unittest.TestCase):
     def test_data_path_inside_project(self):
         with inside_a_project() as proj_path:
             expected = os.path.join(proj_path, '.scrapy', 'somepath')
-            self.assertEquals(expected, data_path('somepath'))
+            self.assertEquals(
+                os.path.realpath(expected),
+                os.path.realpath(data_path('somepath'))
+            )
             self.assertEquals('/absolute/path', data_path('/absolute/path'))

From b9928558b293be71b02afc4da7ba4e42896b6a87 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 16 Feb 2017 17:50:38 +0500
Subject: [PATCH 1302/2974] TST remove temporary files and folders

---
 tests/test_downloader_handlers.py | 21 ++++++++++++++-------
 tests/test_feedexport.py          | 22 ++++++++++++++--------
 tests/test_spiderstate.py         | 28 ++++++++++++++++------------
 tests/test_webclient.py           | 19 +++++++++----------
 4 files changed, 53 insertions(+), 37 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 6333efceb..f9980f7b2 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1,6 +1,7 @@
 import os
 import six
 import contextlib
+import shutil
 try:
     from unittest import mock
 except ImportError:
@@ -84,11 +85,13 @@ class FileTestCase(unittest.TestCase):
 
     def setUp(self):
         self.tmpname = self.mktemp()
-        fd = open(self.tmpname + '^', 'w')
-        fd.write('0123456789')
-        fd.close()
+        with open(self.tmpname + '^', 'w') as f:
+            f.write('0123456789')
         self.download_request = FileDownloadHandler(Settings()).download_request
 
+    def tearDown(self):
+        os.unlink(self.tmpname + '^')
+
     def test_download(self):
         def _test(response):
             self.assertEquals(response.url, request.url)
@@ -134,10 +137,10 @@ class HttpTestCase(unittest.TestCase):
     certfile = 'keys/cert.pem'
 
     def setUp(self):
-        name = self.mktemp()
-        os.mkdir(name)
-        FilePath(name).child("file").setContent(b"0123456789")
-        r = static.File(name)
+        self.tmpname = self.mktemp()
+        os.mkdir(self.tmpname)
+        FilePath(self.tmpname).child("file").setContent(b"0123456789")
+        r = static.File(self.tmpname)
         r.putChild(b"redirect", util.Redirect(b"/file"))
         r.putChild(b"wait", ForeverTakingResource())
         r.putChild(b"hang-after-headers", ForeverTakingResource(write=True))
@@ -165,6 +168,7 @@ class HttpTestCase(unittest.TestCase):
         yield self.port.stopListening()
         if hasattr(self.download_handler, 'close'):
             yield self.download_handler.close()
+        shutil.rmtree(self.tmpname)
 
     def getURL(self, path):
         return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path)
@@ -709,6 +713,9 @@ class FTPTestCase(unittest.TestCase):
         self.download_handler = FTPDownloadHandler(Settings())
         self.addCleanup(self.port.stopListening)
 
+    def tearDown(self):
+        shutil.rmtree(self.directory)
+
     def _add_test_callbacks(self, deferred, callback=None, errback=None):
         def _clean(data):
             self.download_handler.client.transport.loseConnection()
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e93d2bafb..2d137edf4 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -57,8 +57,11 @@ class FileFeedStorageTest(unittest.TestCase):
         file.write(b"content")
         yield storage.store(file)
         self.assertTrue(os.path.exists(path))
-        with open(path, 'rb') as fp:
-            self.assertEqual(fp.read(), b"content")
+        try:
+            with open(path, 'rb') as fp:
+                self.assertEqual(fp.read(), b"content")
+        finally:
+            os.unlink(path)
 
 
 class FTPFeedStorageTest(unittest.TestCase):
@@ -79,12 +82,15 @@ class FTPFeedStorageTest(unittest.TestCase):
         file.write(b"content")
         yield storage.store(file)
         self.assertTrue(os.path.exists(path))
-        with open(path, 'rb') as fp:
-            self.assertEqual(fp.read(), b"content")
-        # again, to check s3 objects are overwritten
-        yield storage.store(BytesIO(b"new content"))
-        with open(path, 'rb') as fp:
-            self.assertEqual(fp.read(), b"new content")
+        try:
+            with open(path, 'rb') as fp:
+                self.assertEqual(fp.read(), b"content")
+            # again, to check s3 objects are overwritten
+            yield storage.store(BytesIO(b"new content"))
+            with open(path, 'rb') as fp:
+                self.assertEqual(fp.read(), b"new content")
+        finally:
+            os.unlink(path)
 
 
 class BlockingFeedStorageTest(unittest.TestCase):
diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index d1d6debec..383fadfeb 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -1,5 +1,6 @@
 import os
 from datetime import datetime
+import shutil
 from twisted.trial import unittest
 
 from scrapy.extensions.spiderstate import SpiderState
@@ -13,20 +14,23 @@ class SpiderStateTest(unittest.TestCase):
     def test_store_load(self):
         jobdir = self.mktemp()
         os.mkdir(jobdir)
-        spider = Spider(name='default')
-        dt = datetime.now()
+        try:
+            spider = Spider(name='default')
+            dt = datetime.now()
 
-        ss = SpiderState(jobdir)
-        ss.spider_opened(spider)
-        spider.state['one'] = 1
-        spider.state['dt'] = dt
-        ss.spider_closed(spider)
+            ss = SpiderState(jobdir)
+            ss.spider_opened(spider)
+            spider.state['one'] = 1
+            spider.state['dt'] = dt
+            ss.spider_closed(spider)
 
-        spider2 = Spider(name='default')
-        ss2 = SpiderState(jobdir)
-        ss2.spider_opened(spider2)
-        self.assertEqual(spider.state, {'one': 1, 'dt': dt})
-        ss2.spider_closed(spider2)
+            spider2 = Spider(name='default')
+            ss2 = SpiderState(jobdir)
+            ss2.spider_opened(spider2)
+            self.assertEqual(spider.state, {'one': 1, 'dt': dt})
+            ss2.spider_closed(spider2)
+        finally:
+            shutil.rmtree(jobdir)
 
     def test_state_attribute(self):
         # state attribute must be present if jobdir is not set, to provide a
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 9b5beda4c..af531c81f 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -4,7 +4,7 @@ Tests borrowed from the twisted.web.client tests.
 """
 import os
 import six
-from six.moves.urllib.parse import urlparse
+import shutil
 
 from twisted.trial import unittest
 from twisted.web import server, static, util, resource
@@ -12,6 +12,7 @@ from twisted.internet import reactor, defer
 from twisted.test.proto_helpers import StringTransport
 from twisted.python.filepath import FilePath
 from twisted.protocols.policies import WrappingFactory
+from twisted.internet.defer import inlineCallbacks
 
 from scrapy.core.downloader import webclient as client
 from scrapy.http import Request, Headers
@@ -229,10 +230,10 @@ class WebClientTestCase(unittest.TestCase):
         return reactor.listenTCP(0, site, interface="127.0.0.1")
 
     def setUp(self):
-        name = self.mktemp()
-        os.mkdir(name)
-        FilePath(name).child("file").setContent(b"0123456789")
-        r = static.File(name)
+        self.tmpname = self.mktemp()
+        os.mkdir(self.tmpname)
+        FilePath(self.tmpname).child("file").setContent(b"0123456789")
+        r = static.File(self.tmpname)
         r.putChild(b"redirect", util.Redirect(b"/file"))
         r.putChild(b"wait", ForeverTakingResource())
         r.putChild(b"error", ErrorResource())
@@ -246,8 +247,10 @@ class WebClientTestCase(unittest.TestCase):
         self.port = self._listen(self.wrapper)
         self.portno = self.port.getHost().port
 
+    @inlineCallbacks
     def tearDown(self):
-        return self.port.stopListening()
+        yield self.port.stopListening()
+        shutil.rmtree(self.tmpname)
 
     def getURL(self, path):
         return "http://127.0.0.1:%d/%s" % (self.portno, path)
@@ -266,7 +269,6 @@ class WebClientTestCase(unittest.TestCase):
             getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(
                 self.assertEquals, to_bytes("www.example.com"))])
 
-
     def test_getPage(self):
         """
         L{client.getPage} returns a L{Deferred} which is called back with
@@ -276,7 +278,6 @@ class WebClientTestCase(unittest.TestCase):
         d.addCallback(self.assertEquals, b"0123456789")
         return d
 
-
     def test_getPageHead(self):
         """
         L{client.getPage} returns a L{Deferred} which is called back with
@@ -289,7 +290,6 @@ class WebClientTestCase(unittest.TestCase):
             _getPage("head").addCallback(self.assertEqual, b""),
             _getPage("HEAD").addCallback(self.assertEqual, b"")])
 
-
     def test_timeoutNotTriggering(self):
         """
         When a non-zero timeout is passed to L{getPage} and the page is
@@ -301,7 +301,6 @@ class WebClientTestCase(unittest.TestCase):
             self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno))
         return d
 
-
     def test_timeoutTriggering(self):
         """
         When a non-zero timeout is passed to L{getPage} and that many

From 78cb46a0797e836c2a7c7366bffd3663b75b0afe Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 16 Feb 2017 18:22:04 +0500
Subject: [PATCH 1303/2974] TST fix a weird OS X testing issue
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

pytest intercepts temp file creation to provide readable file/folder
names; path is built from method name; in case of conflicts
pytests uses increasing numbers, but it seems it doesn’t account
for case-insensitive (but case preserving) OS X filesystem. There
are methods named test_encoding, pytest thinks test_Encoding is
different and fails to create a test folder
---
 tests/test_webclient.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index af531c81f..3ad1aa70e 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -350,7 +350,7 @@ class WebClientTestCase(unittest.TestCase):
                 b'    </head>\n    <body bgcolor="#FFFFFF" text="#000000">\n    '
                 b'<a href="/file">click here</a>\n    </body>\n</html>\n')
 
-    def test_Encoding(self):
+    def test_encoding(self):
         """ Test that non-standart body encoding matches
         Content-Encoding header """
         body = b'\xd0\x81\xd1\x8e\xd0\xaf'

From a1e8a8525d2312842c7e1cca8ba6e4e1a83084b7 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 17 Feb 2017 00:18:29 +0500
Subject: [PATCH 1304/2974] fix make_requests_from_url deprcation
 implementation, add tests

---
 scrapy/spiders/__init__.py | 11 +++++++----
 tests/test_spider.py       | 29 ++++++++++++++++++++++++++---
 2 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 138cdbc67..80768b1b0 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -66,11 +66,14 @@ class Spider(object_ref):
         crawler.signals.connect(self.close, signals.spider_closed)
 
     def start_requests(self):
-        if self.make_requests_from_url is not Spider.make_requests_from_url:
+        cls = self.__class__
+        if cls.make_requests_from_url is not Spider.make_requests_from_url:
             warnings.warn(
-                "Spider.make_requests_from_url method is deprecated; "
-                "it won't be called in future Scrapy releases. "
-                "Please override start_requests method instead."
+                "Spider.make_requests_from_url method is deprecated; it "
+                "won't be called in future Scrapy releases. Please "
+                "override Spider.start_requests method instead (see %s.%s)." % (
+                    cls.__module__, cls.__name__
+                ),
             )
             for url in self.start_urls:
                 yield self.make_requests_from_url(url)
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 079734a69..371b8c1ac 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -345,7 +345,7 @@ Sitemap: /sitemap-relative-url.xml
                           'http://www.example.com/sitemap-relative-url.xml'])
 
 
-class BaseSpiderDeprecationTest(unittest.TestCase):
+class DeprecationTest(unittest.TestCase):
 
     def test_basespider_is_deprecated(self):
         with warnings.catch_warnings(record=True) as w:
@@ -399,6 +399,29 @@ class BaseSpiderDeprecationTest(unittest.TestCase):
         assert isinstance(CrawlSpider(name='foo'), Spider)
         assert isinstance(CrawlSpider(name='foo'), BaseSpider)
 
+    def test_make_requests_from_url_deprecated(self):
+        class MySpider4(Spider):
+            name = 'spider1'
+            start_urls = ['http://example.com']
 
-if __name__ == '__main__':
-    unittest.main()
+        class MySpider5(Spider):
+            name = 'spider2'
+            start_urls = ['http://example.com']
+
+            def make_requests_from_url(self, url):
+                return Request(url + "/foo", dont_filter=True)
+
+        with warnings.catch_warnings(record=True) as w:
+            # spider without overridden make_requests_from_url method
+            # doesn't issue a warning
+            spider1 = MySpider4()
+            self.assertEqual(len(list(spider1.start_requests())), 1)
+            self.assertEqual(len(w), 0)
+
+            # spider with overridden make_requests_from_url issues a warning,
+            # but the method still works
+            spider2 = MySpider5()
+            requests = list(spider2.start_requests())
+            self.assertEqual(len(requests), 1)
+            self.assertEqual(requests[0].url, 'http://example.com/foo')
+            self.assertEqual(len(w), 1)

From 5adacc4dde1ae0dff866803c6d437ebcb4db2730 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 17 Feb 2017 02:35:02 +0500
Subject: [PATCH 1305/2974] fixed method override check in Python 2

---
 scrapy/spiders/__init__.py |  3 ++-
 scrapy/utils/deprecate.py  | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 80768b1b0..30cb7590a 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -12,6 +12,7 @@ from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
 from scrapy.utils.deprecate import create_deprecated_class
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.deprecate import method_is_overridden
 
 
 class Spider(object_ref):
@@ -67,7 +68,7 @@ class Spider(object_ref):
 
     def start_requests(self):
         cls = self.__class__
-        if cls.make_requests_from_url is not Spider.make_requests_from_url:
+        if method_is_overridden(cls, Spider, 'make_requests_from_url'):
             warnings.warn(
                 "Spider.make_requests_from_url method is deprecated; it "
                 "won't be called in future Scrapy releases. Please "
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index 0fc33e0c4..7ab39c97e 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -156,3 +156,35 @@ def update_classpath(path):
                           ScrapyDeprecationWarning)
             return new_path
     return path
+
+
+def method_is_overridden(subclass, base_class, method_name):
+    """ 
+    Return True if a method named ``method_name`` of a ``base_class`` 
+    is overridden in a ``subclass``. 
+    
+    >>> class Base(object):
+    ...     def foo(self):
+    ...         pass
+    >>> class Sub1(Base):
+    ...     pass
+    >>> class Sub2(Base):
+    ...     def foo(self):
+    ...         pass
+    >>> class Sub3(Sub1):
+    ...     def foo(self):
+    ...         pass
+    >>> class Sub4(Sub2):
+    ...     pass
+    >>> method_is_overridden(Sub1, Base, 'foo')
+    False
+    >>> method_is_overridden(Sub2, Base, 'foo')
+    True
+    >>> method_is_overridden(Sub3, Base, 'foo')
+    True
+    >>> method_is_overridden(Sub4, Base, 'foo')
+    True
+    """
+    base_method = getattr(base_class, method_name)
+    sub_method = getattr(subclass, method_name)
+    return base_method.__code__ is not sub_method.__code__

From 165e2cb8c917c570bfb7cb98155588ccdddb154a Mon Sep 17 00:00:00 2001
From: Ashish Kulkarni <ashish@advarisk.com>
Date: Fri, 17 Feb 2017 14:54:22 +0530
Subject: [PATCH 1306/2974] document issue with FormRequest.from_response due
 to bug in lxml

This can make the spider fail due to incorrect values being posted
server-side, which is extremely hard to debug because it is easy
to miss leading/trailing whitespace, even with a logging proxy.

The fix was merged for lxml 3.8 in lxml/lxml#228 so document that
as well.
---
 docs/topics/request-response.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 1fdd26043..a003026bb 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -376,6 +376,10 @@ fields with form data from :class:`Response` objects.
        control clicked (instead of disabling it) you can also use the
        ``clickdata`` argument.
 
+       .. caution:: Using this method with select elements which have leading
+          or trailing whitespace in the option values will not work due to a
+          `bug in lxml`_, which should be fixed in lxml 3.8 and above.
+
        :param response: the response containing a HTML form which will be used
           to pre-populate the form fields
        :type response: :class:`Response` object
@@ -710,3 +714,4 @@ XmlResponse objects
     line.  See :attr:`TextResponse.encoding`.
 
 .. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
+.. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241

From 3ffa2c5759d068e298d94be406fc26faffac6034 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 20 Oct 2016 18:01:50 +0200
Subject: [PATCH 1307/2974] Support Anonymous FTP

Fixes GH-2342
---
 scrapy/core/downloader/handlers/ftp.py | 20 ++++++----
 scrapy/settings/default_settings.py    |  3 ++
 tests/test_downloader_handlers.py      | 52 +++++++++++++++++++++++---
 3 files changed, 62 insertions(+), 13 deletions(-)

diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 93fa0a3a7..603c1e2e3 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
 
 import re
 from io import BytesIO
-from six.moves.urllib.parse import urlparse, unquote
+from six.moves.urllib.parse import unquote
 
 from twisted.internet import reactor
 from twisted.protocols.ftp import FTPClient, CommandFailed
@@ -38,6 +38,8 @@ from twisted.internet.protocol import Protocol, ClientCreator
 
 from scrapy.http import Response
 from scrapy.responsetypes import responsetypes
+from scrapy.utils.httpobj import urlparse_cached
+
 
 class ReceivedDataProtocol(Protocol):
     def __init__(self, filename=None):
@@ -64,14 +66,18 @@ class FTPDownloadHandler(object):
         "default": 503,
     }
 
-    def __init__(self, setting):
-        pass
+    def __init__(self, settings):
+        self.anonymous_user = settings['FTP_ANONYMOUS_USER']
+        self.anonymous_password = settings['FTP_ANONYMOUS_PASSWORD']
 
     def download_request(self, request, spider):
-        parsed_url = urlparse(request.url)
-        creator = ClientCreator(reactor, FTPClient, request.meta["ftp_user"],
-                                    request.meta["ftp_password"],
-                                    passive=request.meta.get("ftp_passive", 1))
+        parsed_url = urlparse_cached(request)
+        user = request.meta.get("ftp_user", self.anonymous_user)
+        password = request.meta.get("ftp_password")
+        if user == self.anonymous_user and password is None:
+            password = self.anonymous_password
+        creator = ClientCreator(reactor, FTPClient, user, password,
+            passive=request.meta.get("ftp_passive", 1))
         return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient,
                                 request, unquote(parsed_url.path))
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index cb88bc2bf..a21c2584a 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -161,6 +161,9 @@ FEED_EXPORTERS_BASE = {
 
 FILES_STORE_S3_ACL = 'private'
 
+FTP_ANONYMOUS_USER = 'anonymous'
+FTP_ANONYMOUS_PASSWORD = 'anonymous@example.com'
+
 HTTPCACHE_ENABLED = False
 HTTPCACHE_DIR = 'httpcache'
 HTTPCACHE_IGNORE_MISSING = False
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 6333efceb..2d97573b3 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -676,10 +676,11 @@ class S3TestCase(unittest.TestCase):
             b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')
 
 
-class FTPTestCase(unittest.TestCase):
+class BaseFTPTestCase(unittest.TestCase):
 
     username = "scrapy"
     password = "passwd"
+    req_meta = {"ftp_user": username, "ftp_password": password}
 
     if six.PY3:
         skip = "Twisted missing ftp support for PY3"
@@ -722,7 +723,7 @@ class FTPTestCase(unittest.TestCase):
 
     def test_ftp_download_success(self):
         request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
-                meta={"ftp_user": self.username, "ftp_password": self.password})
+                          meta=self.req_meta)
         d = self.download_handler.download_request(request, None)
 
         def _test(r):
@@ -734,7 +735,7 @@ class FTPTestCase(unittest.TestCase):
     def test_ftp_download_path_with_spaces(self):
         request = Request(
             url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum,
-            meta={"ftp_user": self.username, "ftp_password": self.password}
+            meta=self.req_meta
         )
         d = self.download_handler.download_request(request, None)
 
@@ -746,7 +747,7 @@ class FTPTestCase(unittest.TestCase):
 
     def test_ftp_download_notexist(self):
         request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum,
-                meta={"ftp_user": self.username, "ftp_password": self.password})
+                          meta=self.req_meta)
         d = self.download_handler.download_request(request, None)
 
         def _test(r):
@@ -755,8 +756,10 @@ class FTPTestCase(unittest.TestCase):
 
     def test_ftp_local_filename(self):
         local_fname = "/tmp/file.txt"
+        meta = {"ftp_local_filename": local_fname}
+        meta.update(self.req_meta)
         request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
-                meta={"ftp_user": self.username, "ftp_password": self.password, "ftp_local_filename": local_fname})
+                          meta=meta)
         d = self.download_handler.download_request(request, None)
 
         def _test(r):
@@ -768,13 +771,50 @@ class FTPTestCase(unittest.TestCase):
             os.remove(local_fname)
         return self._add_test_callbacks(d, _test)
 
+
+class FTPTestCase(BaseFTPTestCase):
+
     def test_invalid_credentials(self):
         from twisted.protocols.ftp import ConnectionLost
 
+        meta = dict(self.req_meta)
+        meta.update({"ftp_password": 'invalid'})
         request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
-                meta={"ftp_user": self.username, "ftp_password": 'invalid'})
+                          meta=meta)
         d = self.download_handler.download_request(request, None)
 
         def _test(r):
             self.assertEqual(r.type, ConnectionLost)
         return self._add_test_callbacks(d, errback=_test)
+
+
+class AnonymousFTPTestCase(BaseFTPTestCase):
+
+    username = "anonymous"
+    req_meta = {}
+
+    def setUp(self):
+        from twisted.protocols.ftp import FTPRealm, FTPFactory
+        from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
+
+        # setup dir and test file
+        self.directory = self.mktemp()
+        os.mkdir(self.directory)
+
+        fp = FilePath(self.directory)
+        fp.child('file.txt').setContent("I have the power!")
+        fp.child('file with spaces.txt').setContent("Moooooooooo power!")
+
+        # setup server for anonymous access
+        realm = FTPRealm(anonymousRoot=self.directory)
+        p = portal.Portal(realm)
+        p.registerChecker(checkers.AllowAnonymousAccess(),
+                          credentials.IAnonymous)
+
+        self.factory = FTPFactory(portal=p,
+                                  userAnonymous=self.username)
+        print("self.factory.allowAnonymous=%r" % self.factory.allowAnonymous)
+        self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
+        self.portNum = self.port.getHost().port
+        self.download_handler = FTPDownloadHandler(Settings())
+        self.addCleanup(self.port.stopListening)

From 26b7c039a9ba3a975b94472dbd54d6fa7bbc358a Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 20 Oct 2016 18:06:59 +0200
Subject: [PATCH 1308/2974] Remove debug print statement

---
 tests/test_downloader_handlers.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 2d97573b3..519a3052e 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -813,7 +813,6 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
 
         self.factory = FTPFactory(portal=p,
                                   userAnonymous=self.username)
-        print("self.factory.allowAnonymous=%r" % self.factory.allowAnonymous)
         self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
         self.portNum = self.port.getHost().port
         self.download_handler = FTPDownloadHandler(Settings())

From 565baae1e2b5f4b805d8f5354d164d61423f59ad Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 24 Oct 2016 15:07:55 +0200
Subject: [PATCH 1309/2974] Change FTP_* settings names

---
 scrapy/core/downloader/handlers/ftp.py | 15 ++++++++-------
 scrapy/settings/default_settings.py    |  5 +++--
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 603c1e2e3..1398140b4 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -67,17 +67,18 @@ class FTPDownloadHandler(object):
     }
 
     def __init__(self, settings):
-        self.anonymous_user = settings['FTP_ANONYMOUS_USER']
-        self.anonymous_password = settings['FTP_ANONYMOUS_PASSWORD']
+        self.default_user = settings['FTP_USER']
+        self.default_password = settings['FTP_PASSWORD']
+        self.passive_mode = settings['FTP_PASSIVE_MODE']
 
     def download_request(self, request, spider):
         parsed_url = urlparse_cached(request)
-        user = request.meta.get("ftp_user", self.anonymous_user)
-        password = request.meta.get("ftp_password")
-        if user == self.anonymous_user and password is None:
-            password = self.anonymous_password
+        user = request.meta.get("ftp_user", self.default_user)
+        password = request.meta.get("ftp_password", self.default_password)
+        passive_mode = 1 if bool(request.meta.get("ftp_passive",
+                                                  self.passive_mode)) else 0
         creator = ClientCreator(reactor, FTPClient, user, password,
-            passive=request.meta.get("ftp_passive", 1))
+            passive=passive_mode)
         return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient,
                                 request, unquote(parsed_url.path))
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index a21c2584a..9ecefbbb5 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -161,8 +161,9 @@ FEED_EXPORTERS_BASE = {
 
 FILES_STORE_S3_ACL = 'private'
 
-FTP_ANONYMOUS_USER = 'anonymous'
-FTP_ANONYMOUS_PASSWORD = 'anonymous@example.com'
+FTP_USER = 'anonymous'
+FTP_PASSWORD = 'anonymous@example.com'
+FTP_PASSIVE_MODE = True
 
 HTTPCACHE_ENABLED = False
 HTTPCACHE_DIR = 'httpcache'

From b80e1bb6c5605406cfff45ae8058cd0f142ef3f4 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 24 Oct 2016 15:08:24 +0200
Subject: [PATCH 1310/2974] Document new FTP_* settings

---
 docs/topics/settings.rst | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0515a9e0d..78fbc79fe 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -686,6 +686,34 @@ The Feed Temp dir allows you to set a custom folder to save crawler
 temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
 :ref:`Amazon S3 <topics-feed-storage-s3>`.
 
+.. setting:: FTP_PASSIVE_MODE
+
+FTP_PASSIVE_MODE
+----------------
+
+Default: ``True``
+
+Whether or not to use passive mode when initiating FTP transfers.
+
+.. setting:: FTP_PASSWORD
+
+FTP_PASSWORD
+------------
+
+Default: ``"anonymous@example.com"``
+
+The password to use for FTP connections when there is no ``"ftp_password"``
+in ``Request`` meta.
+
+.. setting:: FTP_USER
+
+FTP_USER
+--------
+
+Default: ``"anonymous"``
+
+The username to use for FTP connections when there is no ``"ftp_user"``
+in ``Request`` meta.
 
 .. setting:: ITEM_PIPELINES
 

From d35a01a1036c94b13a3828b34bd1752000b5c0fc Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 20 Feb 2017 14:23:23 +0100
Subject: [PATCH 1311/2974] Update default password

---
 docs/topics/settings.rst            | 2 +-
 scrapy/settings/default_settings.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 78fbc79fe..72e7a881f 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -700,7 +700,7 @@ Whether or not to use passive mode when initiating FTP transfers.
 FTP_PASSWORD
 ------------
 
-Default: ``"anonymous@example.com"``
+Default: ``"guest"``
 
 The password to use for FTP connections when there is no ``"ftp_password"``
 in ``Request`` meta.
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 9ecefbbb5..6ae0fc288 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -162,7 +162,7 @@ FEED_EXPORTERS_BASE = {
 FILES_STORE_S3_ACL = 'private'
 
 FTP_USER = 'anonymous'
-FTP_PASSWORD = 'anonymous@example.com'
+FTP_PASSWORD = 'guest'
 FTP_PASSIVE_MODE = True
 
 HTTPCACHE_ENABLED = False

From f2e200120bc20b6d4792ade2b295395296340ee0 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 20 Feb 2017 14:50:05 +0100
Subject: [PATCH 1312/2974] Explicitly remove test directories in FTP tests

---
 tests/test_downloader_handlers.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 519a3052e..459c1a179 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -5,6 +5,7 @@ try:
     from unittest import mock
 except ImportError:
     import mock
+import shutil
 
 from twisted.trial import unittest
 from twisted.protocols.policies import WrappingFactory
@@ -710,6 +711,9 @@ class BaseFTPTestCase(unittest.TestCase):
         self.download_handler = FTPDownloadHandler(Settings())
         self.addCleanup(self.port.stopListening)
 
+    def tearDown(self):
+        shutil.rmtree(self.directory)
+
     def _add_test_callbacks(self, deferred, callback=None, errback=None):
         def _clean(data):
             self.download_handler.client.transport.loseConnection()
@@ -817,3 +821,6 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
         self.portNum = self.port.getHost().port
         self.download_handler = FTPDownloadHandler(Settings())
         self.addCleanup(self.port.stopListening)
+
+    def tearDown(self):
+        shutil.rmtree(self.directory)

From ff3e299eb0c6fe486ecdaafaae340bb55a0c226e Mon Sep 17 00:00:00 2001
From: Omer Schleifer <foromer4@gmail.com>
Date: Mon, 20 Feb 2017 16:42:29 +0200
Subject: [PATCH 1313/2974] [MRG+2] add flags to request (#2082)

* add flags to request

* fxi test - add flags to request

* fix test(2) - add flags to request

* fix test(2) - add flags to request

* Updated test to reqser with flags field of request

* Updated documntation with flags field of request

* fix test identation

* fix test failed

* make the change backward comptaible

* remove  unrequired  spaces, fix documentation request flags

* remove  unrequired  space

* fx assert equal

* flags default is empty list

* Add flags to request

* add flags to request

* fxi test - add flags to request

* fix test(2) - add flags to request

* fix test(2) - add flags to request

* Updated test to reqser with flags field of request

* Updated documntation with flags field of request

* fix test identation

* fix test failed

* make the change backward comptaible

* remove  unrequired  spaces, fix documentation request flags

* remove  unrequired  space

* fx assert equal

* flags default is empty list

* add flags to request squashed commits
---
 docs/topics/request-response.rst | 3 +++
 scrapy/http/request/__init__.py  | 3 ++-
 scrapy/logformatter.py           | 8 +++++---
 scrapy/utils/reqser.py           | 4 +++-
 tests/test_logformatter.py       | 8 ++++++++
 tests/test_utils_reqser.py       | 4 +++-
 6 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 9e0dee000..9a6e0d1b6 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -121,6 +121,9 @@ Request objects
        see :ref:`topics-request-response-ref-errbacks` below.
     :type errback: callable
 
+    :param flags:  Flags sent to the request, can be used for logging or similar purposes.
+    :type flags: list
+
     .. attribute:: Request.url
 
         A string containing the URL of this request. Keep in mind that this
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index f48325a0f..1435d91de 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -18,7 +18,7 @@ class Request(object_ref):
 
     def __init__(self, url, callback=None, method='GET', headers=None, body=None,
                  cookies=None, meta=None, encoding='utf-8', priority=0,
-                 dont_filter=False, errback=None):
+                 dont_filter=False, errback=None, flags=None):
 
         self._encoding = encoding  # this one has to be set first
         self.method = str(method).upper()
@@ -36,6 +36,7 @@ class Request(object_ref):
         self.dont_filter = dont_filter
 
         self._meta = dict(meta) if meta else None
+        self.flags = [] if flags is None else list(flags)
 
     @property
     def meta(self):
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 2160d9ab0..e7bf7942e 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -7,7 +7,7 @@ from scrapy.utils.request import referer_str
 
 SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s"
 DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
-CRAWLEDMSG = u"Crawled (%(status)s) %(request)s (referer: %(referer)s)%(flags)s"
+CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
 
 
 class LogFormatter(object):
@@ -32,15 +32,17 @@ class LogFormatter(object):
     """
 
     def crawled(self, request, response, spider):
-        flags = ' %s' % str(response.flags) if response.flags else ''
+        request_flags = ' %s' % str(request.flags) if request.flags else ''
+        response_flags = ' %s' % str(response.flags) if response.flags else ''
         return {
             'level': logging.DEBUG,
             'msg': CRAWLEDMSG,
             'args': {
                 'status': response.status,
                 'request': request,
+                'request_flags' : request_flags,
                 'referer': referer_str(request),
-                'flags': flags,
+                'response_flags': response_flags,
             }
         }
 
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 2fceb0d94..959dddbd5 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -32,6 +32,7 @@ def request_to_dict(request, spider=None):
         '_encoding': request._encoding,
         'priority': request.priority,
         'dont_filter': request.dont_filter,
+        'flags': request.flags
     }
     if type(request) is not Request:
         d['_class'] = request.__module__ + '.' + request.__class__.__name__
@@ -62,7 +63,8 @@ def request_from_dict(d, spider=None):
         meta=d['meta'],
         encoding=d['_encoding'],
         priority=d['priority'],
-        dont_filter=d['dont_filter'])
+        dont_filter=d['dont_filter'],
+        flags=d.get('flags'))
 
 
 def _find_method(obj, func):
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 50e9662c6..11fe7b653 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -36,6 +36,14 @@ class LoggingContribTest(unittest.TestCase):
         self.assertEqual(logline,
             "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
 
+    def test_flags_in_request(self):
+        req = Request("http://www.example.com", flags=['test','flag'])
+        res = Response("http://www.example.com")
+        logkws = self.formatter.crawled(req, res, self.spider)
+        logline = logkws['msg'] % logkws['args']
+        self.assertEqual(logline,
+        "Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")
+
     def test_dropped(self):
         item = {}
         exception = Exception(u"\u2018")
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 5b889ab5d..073baadc2 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -25,7 +25,8 @@ class RequestSerializationTest(unittest.TestCase):
             cookies={'currency': u'руб'},
             encoding='latin-1',
             priority=20,
-            meta={'a': 'b'})
+            meta={'a': 'b'},
+            flags=['testFlag'])
         self._assert_serializes_ok(r)
 
     def test_latin1_body(self):
@@ -54,6 +55,7 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertEqual(r1._encoding, r2._encoding)
         self.assertEqual(r1.priority, r2.priority)
         self.assertEqual(r1.dont_filter, r2.dont_filter)
+        self.assertEqual(r1.flags, r2.flags)
 
     def test_request_class(self):
         r = FormRequest("http://www.example.com")

From adb180fbaf7de9f720e76b33c742f6cdaa083cdd Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 20 Feb 2017 16:25:12 +0100
Subject: [PATCH 1314/2974] Use bytearray for the CONNECT message bytes buffer

---
 scrapy/core/downloader/handlers/http11.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 4b02cb16f..b96c8c6fe 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -105,7 +105,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         self._tunneledHost = host
         self._tunneledPort = port
         self._contextFactory = contextFactory
-        self._connectBuffer = b''
+        self._connectBuffer = bytearray()
 
     def requestTunnel(self, protocol):
         """Asks the proxy to open a tunnel."""

From 301847d862a1412f3aa8302b242e3f6b9f1442e4 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 20 Feb 2017 16:40:37 +0100
Subject: [PATCH 1315/2974] Set context factory implementation based on Twisted
 version

---
 scrapy/core/downloader/contextfactory.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 4bd4c6166..a94a89205 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,15 +1,15 @@
 from OpenSSL import SSL
 from twisted.internet.ssl import ClientContextFactory
 
-try:
+from scrapy import twisted_version
+
+if twisted_version >= (14, 0, 0):
 
     from zope.interface.declarations import implementer
 
-    # the following should be available from Twisted 14.0.0
     from twisted.internet.ssl import (optionsForClientTLS,
                                       CertificateOptions,
                                       platformTrust)
-
     from twisted.web.client import BrowserLikePolicyForHTTPS
     from twisted.web.iweb import IPolicyForHTTPS
 
@@ -86,7 +86,7 @@ try:
                                             'method': self._ssl_method,
                                        })
 
-except ImportError:
+else:
 
     class ScrapyClientContextFactory(ClientContextFactory):
         "A SSL context factory which is more permissive against SSL bugs."

From f3a7567443d9e52add6224291e98999740d4c33d Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 20 Feb 2017 17:15:05 +0100
Subject: [PATCH 1316/2974] Add note on FTP_PASSWORD default value

---
 docs/topics/settings.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 72e7a881f..522843fee 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -705,6 +705,14 @@ Default: ``"guest"``
 The password to use for FTP connections when there is no ``"ftp_password"``
 in ``Request`` meta.
 
+.. note::
+    Paraphrasing `RFC 1635`_, although it is common to use either the password
+    "guest" or one's e-mail address for anonymous FTP,
+    some FTP servers explicitly ask for the user's e-mail address
+    and will not allow login with the "guest" password.
+
+.. _RFC 1635: https://tools.ietf.org/html/rfc1635
+
 .. setting:: FTP_USER
 
 FTP_USER

From bb7d99ed81e70a5a59d47e69995ac5c025a6a8c0 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 7 Feb 2017 01:30:45 +0500
Subject: [PATCH 1317/2974] drop unneeded urlparse call

---
 scrapy/linkextractors/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index e5d21e174..8676c3b92 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -101,7 +101,7 @@ class FilteringLinkExtractor(object):
         links = [x for x in links if self._link_allowed(x)]
         if self.canonicalize:
             for link in links:
-                link.url = canonicalize_url(urlparse(link.url))
+                link.url = canonicalize_url(link.url)
         links = self.link_extractor._process_links(links)
         return links
 

From 47f7da8724b5a453979c2b0d10114c23ac1bd170 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 7 Feb 2017 02:03:26 +0500
Subject: [PATCH 1318/2974] canonicalize=False by default for LinkExtractor.
 Fixes GH-1941.

---
 docs/topics/link-extractors.rst               |  9 +++++--
 scrapy/linkextractors/lxmlhtml.py             | 22 +++++++++++----
 .../link_extractor/sgml_linkextractor.html    |  1 +
 tests/test_linkextractors.py                  | 27 +++++++++++++++++++
 4 files changed, 52 insertions(+), 7 deletions(-)

diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 75bdb4142..01d7f0b97 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -51,7 +51,7 @@ LxmlLinkExtractor
    :synopsis: lxml's HTMLParser-based link extractors
 
 
-.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, strip=True)
+.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, strip=True)
 
     LxmlLinkExtractor is the recommended link extractor with handy filtering
     options. It is implemented using lxml's robust HTMLParser.
@@ -103,7 +103,12 @@ LxmlLinkExtractor
     :type attrs: list
 
     :param canonicalize: canonicalize each extracted url (using
-        w3lib.url.canonicalize_url). Defaults to ``True``.
+        w3lib.url.canonicalize_url). Defaults to ``False``.
+        Note that canonicalize_url is meant for duplicate checking;
+        it can change the URL visible at server side, so the response can be
+        different for requests with canonicalized and raw URLs. If you're
+        using LinkExtractor to follow links it is more robust to
+        keep the default ``canonicalize=False``.
     :type canonicalize: boolean
 
     :param unique: whether duplicate filtering should be applied to extracted
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index c284f1905..a7092f9b8 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -6,6 +6,7 @@ from six.moves.urllib.parse import urljoin
 
 import lxml.etree as etree
 from w3lib.html import strip_html5_whitespace
+from w3lib.url import canonicalize_url
 
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
@@ -29,12 +30,17 @@ def _nons(tag):
 
 class LxmlParserLinkExtractor(object):
     def __init__(self, tag="a", attr="href", process=None, unique=False,
-                 strip=True):
+                 strip=True, canonicalized=False):
         self.scan_tag = tag if callable(tag) else lambda t: t == tag
         self.scan_attr = attr if callable(attr) else lambda a: a == attr
         self.process_attr = process if callable(process) else lambda v: v
         self.unique = unique
         self.strip = strip
+        if canonicalized:
+            self.link_key = lambda link: link.url
+        else:
+            self.link_key = lambda link: canonicalize_url(link.url,
+                                                          keep_fragments=True)
 
     def _iter_links(self, document):
         for el in document.iter(etree.Element):
@@ -82,21 +88,27 @@ class LxmlParserLinkExtractor(object):
 
     def _deduplicate_if_needed(self, links):
         if self.unique:
-            return unique_list(links, key=lambda link: link.url)
+            return unique_list(links, key=self.link_key)
         return links
 
 
 class LxmlLinkExtractor(FilteringLinkExtractor):
 
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
-                 tags=('a', 'area'), attrs=('href',), canonicalize=True,
+                 tags=('a', 'area'), attrs=('href',), canonicalize=False,
                  unique=True, process_value=None, deny_extensions=None, restrict_css=(),
                  strip=True):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
         tag_func = lambda x: x in tags
         attr_func = lambda x: x in attrs
-        lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func,
-            unique=unique, process=process_value, strip=strip)
+        lx = LxmlParserLinkExtractor(
+            tag=tag_func,
+            attr=attr_func,
+            unique=unique,
+            process=process_value,
+            strip=strip,
+            canonicalized=canonicalize
+        )
 
         super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
             allow_domains=allow_domains, deny_domains=deny_domains,
diff --git a/tests/sample_data/link_extractor/sgml_linkextractor.html b/tests/sample_data/link_extractor/sgml_linkextractor.html
index fbb803f2d..7d5db368a 100644
--- a/tests/sample_data/link_extractor/sgml_linkextractor.html
+++ b/tests/sample_data/link_extractor/sgml_linkextractor.html
@@ -11,6 +11,7 @@
 </div>
 <a href='http://example.com/sample3.html' title='sample 3'>sample 3 text</a>
 <a href='sample3.html'>sample 3 repetition</a>
+<a href='sample3.html#foo'>sample 3 repetition with fragment</a>
 <a href='http://www.google.com/something'></a>
 <a href='http://example.com/innertag.html'><b>inner</b> tag</a>
 <a href=' page 4.html '>href with whitespaces</a>
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 340c64f35..50484f060 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -30,6 +30,7 @@ class Base:
                 Link(url='http://example.com/sample1.html', text=u''),
                 Link(url='http://example.com/sample2.html', text=u'sample 2'),
                 Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
                 Link(url='http://www.google.com/something', text=u''),
                 Link(url='http://example.com/innertag.html', text=u'inner tag'),
                 Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
@@ -41,6 +42,7 @@ class Base:
                 Link(url='http://example.com/sample1.html', text=u''),
                 Link(url='http://example.com/sample2.html', text=u'sample 2'),
                 Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
             ])
 
         def test_extract_filter_allow_with_duplicates(self):
@@ -50,6 +52,27 @@ class Base:
                 Link(url='http://example.com/sample2.html', text=u'sample 2'),
                 Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
                 Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+                Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
+            ])
+
+        def test_extract_filter_allow_with_duplicates_canonicalize(self):
+            lx = self.extractor_cls(allow=('sample', ), unique=False,
+                                    canonicalize=True)
+            self.assertEqual([link for link in lx.extract_links(self.response)], [
+                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+                Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment')
+            ])
+
+        def test_extract_filter_allow_no_duplicates_canonicalize(self):
+            lx = self.extractor_cls(allow=('sample',), unique=True,
+                                    canonicalize=True)
+            self.assertEqual([link for link in lx.extract_links(self.response)], [
+                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
             ])
 
         def test_extract_filter_allow_and_deny(self):
@@ -73,6 +96,8 @@ class Base:
                 Link(url='http://example.com/sample1.html', text=u''),
                 Link(url='http://example.com/sample2.html', text=u'sample 2'),
                 Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample3.html#foo',
+                     text='sample 3 repetition with fragment')
             ])
 
             lx = self.extractor_cls(allow='sample', deny='3')
@@ -280,6 +305,7 @@ class Base:
                 Link(url='http://example.com/sample1.html', text=u''),
                 Link(url='http://example.com/sample2.html', text=u'sample 2'),
                 Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
                 Link(url='http://www.google.com/something', text=u''),
                 Link(url='http://example.com/innertag.html', text=u'inner tag'),
                 Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
@@ -291,6 +317,7 @@ class Base:
                 Link(url='http://example.com/sample2.html', text=u'sample 2'),
                 Link(url='http://example.com/sample2.jpg', text=u''),
                 Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
                 Link(url='http://www.google.com/something', text=u''),
                 Link(url='http://example.com/innertag.html', text=u'inner tag'),
                 Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),

From df446d167f6b539c4a79fb64d91cbc1607f30acb Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 7 Feb 2017 03:11:59 +0500
Subject: [PATCH 1319/2974] fix deprecated link extractors

---
 scrapy/linkextractors/sgml.py           | 18 +++++++++++-------
 tests/test_linkextractors_deprecated.py |  2 ++
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 11ff7a261..f4ca4262a 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -6,7 +6,7 @@ from six.moves.urllib.parse import urljoin
 import warnings
 from sgmllib import SGMLParser
 
-from w3lib.url import safe_url_string
+from w3lib.url import safe_url_string, canonicalize_url
 from w3lib.html import strip_html5_whitespace
 
 from scrapy.link import Link
@@ -20,7 +20,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 class BaseSgmlLinkExtractor(SGMLParser):
 
     def __init__(self, tag="a", attr="href", unique=False, process_value=None,
-                 strip=True):
+                 strip=True, canonicalized=False):
         warnings.warn(
             "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
             "Please use scrapy.linkextractors.LinkExtractor",
@@ -33,6 +33,11 @@ class BaseSgmlLinkExtractor(SGMLParser):
         self.current_link = None
         self.unique = unique
         self.strip = strip
+        if canonicalized:
+            self.link_key = lambda link: link.url
+        else:
+            self.link_key = lambda link: canonicalize_url(link.url,
+                                                          keep_fragments=True)
 
     def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
         """ Do the real extraction work """
@@ -61,8 +66,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
 
         The subclass should override it if necessary
         """
-        links = unique_list(links, key=lambda link: link.url) if self.unique else links
-        return links
+        return unique_list(links, key=self.link_key) if self.unique else links
 
     def extract_links(self, response):
         # wrapper needed to allow to work directly with text
@@ -107,10 +111,9 @@ class BaseSgmlLinkExtractor(SGMLParser):
 class SgmlLinkExtractor(FilteringLinkExtractor):
 
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
-                 tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True,
+                 tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True,
                  process_value=None, deny_extensions=None, restrict_css=(),
                  strip=True):
-
         warnings.warn(
             "SgmlLinkExtractor is deprecated and will be removed in future releases. "
             "Please use scrapy.linkextractors.LinkExtractor",
@@ -124,7 +127,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
         with warnings.catch_warnings():
             warnings.simplefilter('ignore', ScrapyDeprecationWarning)
             lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
-                unique=unique, process_value=process_value, strip=strip)
+                unique=unique, process_value=process_value, strip=strip,
+                canonicalized=canonicalize)
 
         super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
             allow_domains=allow_domains, deny_domains=deny_domains,
diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py
index fef227aa1..794f85e0f 100644
--- a/tests/test_linkextractors_deprecated.py
+++ b/tests/test_linkextractors_deprecated.py
@@ -121,6 +121,7 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase):
             Link(url='http://example.com/sample2.html', text=u'sample 2'),
             Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
             Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+            Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
             Link(url='http://www.google.com/something', text=u''),
             Link(url='http://example.com/innertag.html', text=u'inner tag'),
             Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
@@ -190,6 +191,7 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
         self.assertEqual(lx.extract_links(self.response),
                          [Link(url='http://example.com/sample2.html', text=u'sample 2'),
                           Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                          Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
                           Link(url='http://www.google.com/something', text=u''),
                           Link(url='http://example.com/innertag.html', text=u'inner tag'),])
 

From 2b4d46315f2ce7ce20e7355fe4ad184bb041d6f4 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 21 Feb 2017 00:05:40 +0500
Subject: [PATCH 1320/2974] TST fixed compatibility with new link extractor
 whitespace handling

---
 tests/test_http_response.py             |  1 +
 tests/test_linkextractors.py            | 17 ++++++++++++++---
 tests/test_linkextractors_deprecated.py |  1 +
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 924bb7979..779f5a71c 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -392,6 +392,7 @@ class TextResponseTest(BaseResponseTest):
             'http://example.com/sample2.html',
             'http://example.com/sample3.html',
             'http://example.com/sample3.html',
+            'http://example.com/sample3.html#foo',
             'http://www.google.com/something',
             'http://example.com/innertag.html'
         ]
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 50484f060..1d7c4f311 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -13,6 +13,7 @@ from tests import get_testdata
 class Base:
     class LinkExtractorTestCase(unittest.TestCase):
         extractor_cls = None
+        escapes_whitespace = False
 
         def setUp(self):
             body = get_testdata('link_extractor', 'sgml_linkextractor.html')
@@ -26,6 +27,11 @@ class Base:
 
         def test_extract_all_links(self):
             lx = self.extractor_cls()
+            if self.escapes_whitespace:
+                page4_url = 'http://example.com/page%204.html'
+            else:
+                page4_url = 'http://example.com/page 4.html'
+
             self.assertEqual([link for link in lx.extract_links(self.response)], [
                 Link(url='http://example.com/sample1.html', text=u''),
                 Link(url='http://example.com/sample2.html', text=u'sample 2'),
@@ -33,7 +39,7 @@ class Base:
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
                 Link(url='http://www.google.com/something', text=u''),
                 Link(url='http://example.com/innertag.html', text=u'inner tag'),
-                Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
+                Link(url=page4_url, text=u'href with whitespaces'),
             ])
 
         def test_extract_filter_allow(self):
@@ -301,6 +307,11 @@ class Base:
 
         def test_attrs(self):
             lx = self.extractor_cls(attrs="href")
+            if self.escapes_whitespace:
+                page4_url = 'http://example.com/page%204.html'
+            else:
+                page4_url = 'http://example.com/page 4.html'
+
             self.assertEqual(lx.extract_links(self.response), [
                 Link(url='http://example.com/sample1.html', text=u''),
                 Link(url='http://example.com/sample2.html', text=u'sample 2'),
@@ -308,7 +319,7 @@ class Base:
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
                 Link(url='http://www.google.com/something', text=u''),
                 Link(url='http://example.com/innertag.html', text=u'inner tag'),
-                Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
+                Link(url=page4_url, text=u'href with whitespaces'),
             ])
 
             lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())
@@ -320,7 +331,7 @@ class Base:
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
                 Link(url='http://www.google.com/something', text=u''),
                 Link(url='http://example.com/innertag.html', text=u'inner tag'),
-                Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
+                Link(url=page4_url, text=u'href with whitespaces'),
             ])
 
             lx = self.extractor_cls(attrs=None)
diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py
index 794f85e0f..1366971be 100644
--- a/tests/test_linkextractors_deprecated.py
+++ b/tests/test_linkextractors_deprecated.py
@@ -143,6 +143,7 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase):
 
 class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
     extractor_cls = SgmlLinkExtractor
+    escapes_whitespace = True
 
     def test_deny_extensions(self):
         html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""

From 26c1256f95e21f474783338bd1f1dda6ea3afecd Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 16 Feb 2017 03:11:28 +0500
Subject: [PATCH 1321/2974] HttpErrorMiddleware stats

---
 scrapy/spidermiddlewares/httperror.py    |  4 ++++
 tests/test_spidermiddleware_httperror.py | 13 ++++++++++---
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index e34c265c1..def697c2b 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -46,6 +46,10 @@ class HttpErrorMiddleware(object):
 
     def process_spider_exception(self, response, exception, spider):
         if isinstance(exception, HttpError):
+            spider.crawler.stats.inc_value('httperror/response_ignored_count')
+            spider.crawler.stats.inc_value(
+                'httperror/response_ignored_status_count/%s' % response.status
+            )
             logger.info(
                 "Ignoring response %(response)r: HTTP status code is not handled or not allowed",
                 {'response': response}, extra={'spider': spider},
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 319746350..e1407e6b3 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -60,7 +60,8 @@ def _responses(request, status_codes):
 class TestHttpErrorMiddleware(TestCase):
 
     def setUp(self):
-        self.spider = Spider('foo')
+        crawler = get_crawler(Spider)
+        self.spider = Spider.from_crawler(crawler, name='foo')
         self.mw = HttpErrorMiddleware(Settings({}))
         self.req = Request('http://scrapytest.org')
         self.res200, self.res404 = _responses(self.req, [200, 404])
@@ -73,10 +74,10 @@ class TestHttpErrorMiddleware(TestCase):
 
     def test_process_spider_exception(self):
         self.assertEquals([],
-                self.mw.process_spider_exception(self.res404, \
+                self.mw.process_spider_exception(self.res404,
                         HttpError(self.res404), self.spider))
         self.assertEquals(None,
-                self.mw.process_spider_exception(self.res404, \
+                self.mw.process_spider_exception(self.res404,
                         Exception(), self.spider))
 
     def test_handle_httpstatus_list(self):
@@ -173,6 +174,12 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
         self.assertEqual(crawler.spider.parsed, {'200'})
         self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
 
+        get_value = crawler.stats.get_value
+        self.assertEqual(get_value('httperror/response_ignored_count'), 3)
+        self.assertEqual(get_value('httperror/response_ignored_status_count/404'), 1)
+        self.assertEqual(get_value('httperror/response_ignored_status_count/402'), 1)
+        self.assertEqual(get_value('httperror/response_ignored_status_count/500'), 1)
+
     @defer.inlineCallbacks
     def test_logging(self):
         crawler = get_crawler(_HttpErrorSpider)

From d60642e1755b004e46b87e2011c1873f25895bb8 Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Fri, 12 Aug 2016 00:45:42 -0300
Subject: [PATCH 1322/2974] data URI download handler.

---
 scrapy/core/downloader/handlers/data.py | 93 +++++++++++++++++++++++++
 tests/test_downloader_handlers.py       | 58 ++++++++++++++-
 2 files changed, 150 insertions(+), 1 deletion(-)
 create mode 100644 scrapy/core/downloader/handlers/data.py

diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.py
new file mode 100644
index 000000000..637a6f9c0
--- /dev/null
+++ b/scrapy/core/downloader/handlers/data.py
@@ -0,0 +1,93 @@
+import base64
+import re
+from six.moves.urllib.parse import unquote
+
+from scrapy.http import TextResponse
+from scrapy.responsetypes import responsetypes
+from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.decorators import defers
+
+
+# ASCII characters.
+_char = set(map(chr, range(127)))
+
+# RFC 2045 token.
+_token = r'[{}]+'.format(re.escape(''.join(_char -
+                                           # Control characters.
+                                           set(map(chr, range(0, 32))) -
+                                           # tspecials and space.
+                                           set('()<>@,;:\\"/[]?= '))))
+
+# RFC 822 quoted-string, without surrounding quotation marks.
+_quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format(
+    re.escape(''.join(_char - {'"', '\\', '\r'})),
+    re.escape(''.join(_char))
+)
+
+# RFC 2397 mediatype.
+_mediatype_pattern = re.compile(r'{token}/{token}'.format(token=_token))
+
+_mediatype_parameter_pattern = re.compile(
+    r';({token})=(?:({token})|"({quoted})")'.format(token=_token,
+                                                    quoted=_quoted_string)
+)
+
+
+class DataURIDownloadHandler(object):
+    def __init__(self, settings):
+        super(DataURIDownloadHandler, self).__init__()
+
+    @defers
+    def download_request(self, request, spider):
+        url = request.url
+
+        scheme, url = url.split(':', 1)
+        if scheme != 'data':
+            raise ValueError("not a data URI")
+
+        # RFC 3986 section 2.1 allows percent encoding to escape characters
+        # that would be interpreted as delimiters, implying that actual
+        # delimiters should not be percent-encoded.
+        # Decoding before parsing will allow malformed URIs with
+        # percent-encoded delimiters, but it makes parsing easier and should
+        # not affect well-formed URIs, as the delimiters used in this URI
+        # scheme are not allowed, percent-encoded or not, in tokens.
+        url = unquote(url)
+
+        media_type = "text/plain"
+        media_type_params = CaselessDict()
+
+        m = _mediatype_pattern.match(url)
+        if m:
+            media_type = m.group()
+            url = url[m.end():]
+        else:
+            media_type_params['charset'] = "US-ASCII"
+
+        while True:
+            m = _mediatype_parameter_pattern.match(url)
+            if m:
+                attribute, value, value_quoted = m.groups()
+                if value_quoted:
+                    value = re.sub(r'\\(.)', '\1', value_quoted)
+                media_type_params[attribute] = value
+                url = url[m.end():]
+            else:
+                break
+
+        is_base64, data = url.split(',', 1)
+        if is_base64:
+            if is_base64 != ";base64":
+                raise ValueError("invalid data URI")
+            data = base64.b64decode(data)
+
+        respcls = responsetypes.from_mimetype(media_type)
+
+        resp_kwargs = {}
+
+        if media_type:
+            media_type = media_type.split('/')
+            if issubclass(respcls, TextResponse) and media_type[0] == 'text':
+                resp_kwargs['encoding'] = media_type_params.get('charset')
+
+        return respcls(url=request.url, body=data, **resp_kwargs)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index c1683fb3e..c21a1670f 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -6,7 +6,6 @@ try:
     from unittest import mock
 except ImportError:
     import mock
-import shutil
 
 from twisted.trial import unittest
 from twisted.protocols.policies import WrappingFactory
@@ -20,6 +19,7 @@ from twisted.cred import portal, checkers, credentials
 from w3lib.url import path_to_file_uri
 
 from scrapy.core.downloader.handlers import DownloadHandlers
+from scrapy.core.downloader.handlers.data import DataURIDownloadHandler
 from scrapy.core.downloader.handlers.file import FileDownloadHandler
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler
 from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
@@ -29,6 +29,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.http.response.text import TextResponse
+from scrapy.responsetypes import responsetypes
 from scrapy.settings import Settings
 from scrapy.utils.test import get_crawler, skip_if_no_boto
 from scrapy.utils.python import to_bytes
@@ -828,3 +829,58 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
 
     def tearDown(self):
         shutil.rmtree(self.directory)
+
+
+class DataURITestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.download_handler = DataURIDownloadHandler(Settings())
+        self.download_request = self.download_handler.download_request
+        self.spider = Spider('foo')
+
+    def test_default_mediatype_encoding(self):
+        def _test(response):
+            self.assertEquals(response.text, 'A brief note')
+            self.assertEquals(type(response),
+                              responsetypes.from_mimetype("text/plain"))
+            self.assertEquals(response.encoding, "US-ASCII")
+
+        request = Request("data:,A%20brief%20note")
+        return self.download_request(request, self.spider).addCallback(_test)
+
+    def test_default_mediatype(self):
+        def _test(response):
+            self.assertEquals(response.text, u'\u038e\u03a3\u038e')
+            self.assertEquals(type(response),
+                              responsetypes.from_mimetype("text/plain"))
+            self.assertEquals(response.encoding, "iso-8859-7")
+
+        request = Request("data:;charset=iso-8859-7,%be%d3%be")
+        return self.download_request(request, self.spider).addCallback(_test)
+
+    def test_text_charset(self):
+        def _test(response):
+            self.assertEquals(response.text, u'\u038e\u03a3\u038e')
+            self.assertEquals(response.encoding, "iso-8859-7")
+
+        request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be")
+        return self.download_request(request, self.spider).addCallback(_test)
+
+    def test_mediatype_parameters(self):
+        def _test(response):
+            self.assertEquals(response.text, u'\u038e\u03a3\u038e')
+            self.assertEquals(type(response),
+                              responsetypes.from_mimetype("text/plain"))
+            self.assertEquals(response.encoding, "utf-8")
+
+        request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;'
+                          'charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22'
+                          ',%CE%8E%CE%A3%CE%8E')
+        return self.download_request(request, self.spider).addCallback(_test)
+
+    def test_base64(self):
+        def _test(response):
+            self.assertEquals(response.text, 'Hello, world.')
+
+        request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D')
+        return self.download_request(request, self.spider).addCallback(_test)

From 7e9f2c31d78dcc798a281e5ba6ddf91714dc9cc7 Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Fri, 12 Aug 2016 02:09:35 -0300
Subject: [PATCH 1323/2974] Ensure bytes objects when needed in data URI
 downloader.

---
 scrapy/core/downloader/handlers/data.py | 29 ++++++++++++++++++-------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.py
index 637a6f9c0..30c39865f 100644
--- a/scrapy/core/downloader/handlers/data.py
+++ b/scrapy/core/downloader/handlers/data.py
@@ -1,6 +1,12 @@
 import base64
 import re
-from six.moves.urllib.parse import unquote
+
+import six
+
+if six.PY2:
+    from urllib import unquote
+else:
+    from urllib.parse import unquote_to_bytes as unquote
 
 from scrapy.http import TextResponse
 from scrapy.responsetypes import responsetypes
@@ -24,12 +30,19 @@ _quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format(
     re.escape(''.join(_char))
 )
 
+# Encode the regular expression strings to make them into bytes, as Python 3
+# bytes have no format() method, but bytes must be passed to re.compile() in
+# order to make a pattern object that can be used to match on bytes.
+
 # RFC 2397 mediatype.
-_mediatype_pattern = re.compile(r'{token}/{token}'.format(token=_token))
+_mediatype_pattern = re.compile(
+    r'{token}/{token}'.format(token=_token).encode()
+)
 
 _mediatype_parameter_pattern = re.compile(
     r';({token})=(?:({token})|"({quoted})")'.format(token=_token,
-                                                    quoted=_quoted_string)
+                                                    quoted=_quoted_string
+                                                    ).encode()
 )
 
 
@@ -59,7 +72,7 @@ class DataURIDownloadHandler(object):
 
         m = _mediatype_pattern.match(url)
         if m:
-            media_type = m.group()
+            media_type = m.group().decode()
             url = url[m.end():]
         else:
             media_type_params['charset'] = "US-ASCII"
@@ -69,15 +82,15 @@ class DataURIDownloadHandler(object):
             if m:
                 attribute, value, value_quoted = m.groups()
                 if value_quoted:
-                    value = re.sub(r'\\(.)', '\1', value_quoted)
-                media_type_params[attribute] = value
+                    value = re.sub(br'\\(.)', r'\1', value_quoted)
+                media_type_params[attribute.decode()] = value.decode()
                 url = url[m.end():]
             else:
                 break
 
-        is_base64, data = url.split(',', 1)
+        is_base64, data = url.split(b',', 1)
         if is_base64:
-            if is_base64 != ";base64":
+            if is_base64 != b";base64":
                 raise ValueError("invalid data URI")
             data = base64.b64decode(data)
 

From 3397d27574d03a222b6a790004fcaa1a03711d75 Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Wed, 8 Feb 2017 12:32:00 -0200
Subject: [PATCH 1324/2974] Test for binary body content from data URI
 downloader.

---
 tests/test_downloader_handlers.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index c21a1670f..cfcdcd8f8 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -861,6 +861,7 @@ class DataURITestCase(unittest.TestCase):
     def test_text_charset(self):
         def _test(response):
             self.assertEquals(response.text, u'\u038e\u03a3\u038e')
+            self.assertEquals(response.body, b'\xbe\xd3\xbe')
             self.assertEquals(response.encoding, "iso-8859-7")
 
         request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be")

From c847e7d4d00647a953a3c5458d198439c30b87f7 Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Wed, 8 Feb 2017 12:38:49 -0200
Subject: [PATCH 1325/2974] Use w3lib data URI parser.

---
 scrapy/core/downloader/handlers/data.py | 99 ++-----------------------
 1 file changed, 8 insertions(+), 91 deletions(-)

diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.py
index 30c39865f..d102f2b73 100644
--- a/scrapy/core/downloader/handlers/data.py
+++ b/scrapy/core/downloader/handlers/data.py
@@ -1,106 +1,23 @@
-import base64
-import re
-
-import six
-
-if six.PY2:
-    from urllib import unquote
-else:
-    from urllib.parse import unquote_to_bytes as unquote
+from w3lib.url import parse_data_uri
 
 from scrapy.http import TextResponse
 from scrapy.responsetypes import responsetypes
-from scrapy.utils.datatypes import CaselessDict
 from scrapy.utils.decorators import defers
 
 
-# ASCII characters.
-_char = set(map(chr, range(127)))
-
-# RFC 2045 token.
-_token = r'[{}]+'.format(re.escape(''.join(_char -
-                                           # Control characters.
-                                           set(map(chr, range(0, 32))) -
-                                           # tspecials and space.
-                                           set('()<>@,;:\\"/[]?= '))))
-
-# RFC 822 quoted-string, without surrounding quotation marks.
-_quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format(
-    re.escape(''.join(_char - {'"', '\\', '\r'})),
-    re.escape(''.join(_char))
-)
-
-# Encode the regular expression strings to make them into bytes, as Python 3
-# bytes have no format() method, but bytes must be passed to re.compile() in
-# order to make a pattern object that can be used to match on bytes.
-
-# RFC 2397 mediatype.
-_mediatype_pattern = re.compile(
-    r'{token}/{token}'.format(token=_token).encode()
-)
-
-_mediatype_parameter_pattern = re.compile(
-    r';({token})=(?:({token})|"({quoted})")'.format(token=_token,
-                                                    quoted=_quoted_string
-                                                    ).encode()
-)
-
-
 class DataURIDownloadHandler(object):
     def __init__(self, settings):
         super(DataURIDownloadHandler, self).__init__()
 
     @defers
     def download_request(self, request, spider):
-        url = request.url
-
-        scheme, url = url.split(':', 1)
-        if scheme != 'data':
-            raise ValueError("not a data URI")
-
-        # RFC 3986 section 2.1 allows percent encoding to escape characters
-        # that would be interpreted as delimiters, implying that actual
-        # delimiters should not be percent-encoded.
-        # Decoding before parsing will allow malformed URIs with
-        # percent-encoded delimiters, but it makes parsing easier and should
-        # not affect well-formed URIs, as the delimiters used in this URI
-        # scheme are not allowed, percent-encoded or not, in tokens.
-        url = unquote(url)
-
-        media_type = "text/plain"
-        media_type_params = CaselessDict()
-
-        m = _mediatype_pattern.match(url)
-        if m:
-            media_type = m.group().decode()
-            url = url[m.end():]
-        else:
-            media_type_params['charset'] = "US-ASCII"
-
-        while True:
-            m = _mediatype_parameter_pattern.match(url)
-            if m:
-                attribute, value, value_quoted = m.groups()
-                if value_quoted:
-                    value = re.sub(br'\\(.)', r'\1', value_quoted)
-                media_type_params[attribute.decode()] = value.decode()
-                url = url[m.end():]
-            else:
-                break
-
-        is_base64, data = url.split(b',', 1)
-        if is_base64:
-            if is_base64 != b";base64":
-                raise ValueError("invalid data URI")
-            data = base64.b64decode(data)
-
-        respcls = responsetypes.from_mimetype(media_type)
+        uri = parse_data_uri(request.url)
+        respcls = responsetypes.from_mimetype(uri.media_type)
 
         resp_kwargs = {}
+        if (issubclass(respcls, TextResponse) and
+                uri.media_type.split('/')[0] == 'text'):
+            charset = uri.media_type_parameters.get('charset')
+            resp_kwargs['encoding'] = charset
 
-        if media_type:
-            media_type = media_type.split('/')
-            if issubclass(respcls, TextResponse) and media_type[0] == 'text':
-                resp_kwargs['encoding'] = media_type_params.get('charset')
-
-        return respcls(url=request.url, body=data, **resp_kwargs)
+        return respcls(url=request.url, body=uri.data, **resp_kwargs)

From 121a668a479c3b681581a36c7da090518fbc2120 Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Sun, 12 Feb 2017 11:23:21 -0200
Subject: [PATCH 1326/2974] Rename data URI downloader module.

---
 scrapy/core/downloader/handlers/{data.py => datauri.py} | 0
 tests/test_downloader_handlers.py                       | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename scrapy/core/downloader/handlers/{data.py => datauri.py} (100%)

diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/datauri.py
similarity index 100%
rename from scrapy/core/downloader/handlers/data.py
rename to scrapy/core/downloader/handlers/datauri.py
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index cfcdcd8f8..b27245a36 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -19,7 +19,7 @@ from twisted.cred import portal, checkers, credentials
 from w3lib.url import path_to_file_uri
 
 from scrapy.core.downloader.handlers import DownloadHandlers
-from scrapy.core.downloader.handlers.data import DataURIDownloadHandler
+from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler
 from scrapy.core.downloader.handlers.file import FileDownloadHandler
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler
 from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler

From 3139f4a5f700a100b6ab6890677dac808b01835f Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Sun, 12 Feb 2017 11:23:56 -0200
Subject: [PATCH 1327/2974] Add data URI download handler to settings.

---
 scrapy/settings/default_settings.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index e0e39120c..2251d3db5 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -67,6 +67,7 @@ DOWNLOAD_DELAY = 0
 
 DOWNLOAD_HANDLERS = {}
 DOWNLOAD_HANDLERS_BASE = {
+    'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler',
     'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
     'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
     'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',

From 96a570a93a78c977f7554179face6123d338ade0 Mon Sep 17 00:00:00 2001
From: MikeinRealLife <mloo87@gmail.com>
Date: Wed, 22 Feb 2017 21:17:34 -0800
Subject: [PATCH 1328/2974] fixed ticket #2574

---
 docs/topics/request-response.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 9a6e0d1b6..018d14100 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -304,6 +304,8 @@ Those are:
 * :reqmeta:`download_maxsize`
 * :reqmeta:`download_latency`
 * :reqmeta:`proxy`
+* ``ftp_user`` (See :setting:`FTP_USER` for more info)
+* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
 
 .. reqmeta:: bindaddress
 
@@ -332,6 +334,8 @@ used to control Scrapy behavior, this one is supposed to be read-only.
 
 .. _topics-request-response-ref-request-subclasses:
 
+.. _topics-request-response-ref-request-subclasses:
+
 Request subclasses
 ==================
 

From 441f25507ea37b098e14bf3a1f9de7ccbe5a1cc5 Mon Sep 17 00:00:00 2001
From: MikeinRealLife <mloo87@gmail.com>
Date: Wed, 22 Feb 2017 21:23:27 -0800
Subject: [PATCH 1329/2974] fixed typo

removed duplicate line
---
 docs/topics/request-response.rst | 2 --
 1 file changed, 2 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 018d14100..06e245b16 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -334,8 +334,6 @@ used to control Scrapy behavior, this one is supposed to be read-only.
 
 .. _topics-request-response-ref-request-subclasses:
 
-.. _topics-request-response-ref-request-subclasses:
-
 Request subclasses
 ==================
 

From 4274f0d4d4b3a79ebb76f61f4f9d5d3fb379d42b Mon Sep 17 00:00:00 2001
From: mangogao <jjmangogao@gmail.com>
Date: Sat, 25 Feb 2017 15:44:20 +0800
Subject: [PATCH 1330/2974] Add omitted "self" arguments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

“self” argument is omitted in some methods.
---
 scrapy/templates/project/module/middlewares.py.tmpl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
index 42318fec2..292bf572e 100644
--- a/scrapy/templates/project/module/middlewares.py.tmpl
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -20,14 +20,14 @@ class ${ProjectName}SpiderMiddleware(object):
         crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
         return s
 
-    def process_spider_input(response, spider):
+    def process_spider_input(self, response, spider):
         # Called for each response that goes through the spider
         # middleware and into the spider.
 
         # Should return None or raise an exception.
         return None
 
-    def process_spider_output(response, result, spider):
+    def process_spider_output(self, response, result, spider):
         # Called with the results returned from the Spider, after
         # it has processed the response.
 
@@ -35,7 +35,7 @@ class ${ProjectName}SpiderMiddleware(object):
         for i in result:
             yield i
 
-    def process_spider_exception(response, exception, spider):
+    def process_spider_exception(self, response, exception, spider):
         # Called when a spider or process_spider_input() method
         # (from other spider middleware) raises an exception.
 
@@ -43,7 +43,7 @@ class ${ProjectName}SpiderMiddleware(object):
         # or Item objects.
         pass
 
-    def process_start_requests(start_requests, spider):
+    def process_start_requests(self, start_requests, spider):
         # Called with the start requests of the spider, and works
         # similarly to the process_spider_output() method, except
         # that it doesn’t have a response associated.

From e85f0db1285d7338330e0a7575dcc59c76375d6a Mon Sep 17 00:00:00 2001
From: Arvind Chembarpu <arvindch@users.noreply.github.com>
Date: Sat, 25 Feb 2017 16:48:17 +0530
Subject: [PATCH 1331/2974] Use single quotes uniformly

The default spider template mixes single and double quotes.
---
 scrapy/templates/spiders/basic.tmpl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl
index 99e5d43b2..1cfe9cc9d 100644
--- a/scrapy/templates/spiders/basic.tmpl
+++ b/scrapy/templates/spiders/basic.tmpl
@@ -3,8 +3,8 @@ import scrapy
 
 
 class $classname(scrapy.Spider):
-    name = "$name"
-    allowed_domains = ["$domain"]
+    name = '$name'
+    allowed_domains = ['$domain']
     start_urls = ['http://$domain/']
 
     def parse(self, response):

From 0b90c3b43c4eedc891056fa433e0608afbe6cd32 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 27 Feb 2017 17:42:00 +0100
Subject: [PATCH 1332/2974] Re-enable FTP tests on Python 3

---
 scrapy/core/downloader/handlers/ftp.py |  7 ++++---
 tests/py3-ignores.txt                  |  7 -------
 tests/test_downloader_handlers.py      | 27 ++++++++++++--------------
 3 files changed, 16 insertions(+), 25 deletions(-)

diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 1398140b4..933bc7e8d 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -39,12 +39,13 @@ from twisted.internet.protocol import Protocol, ClientCreator
 from scrapy.http import Response
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_bytes
 
 
 class ReceivedDataProtocol(Protocol):
     def __init__(self, filename=None):
         self.__filename = filename
-        self.body = open(filename, "w") if filename else BytesIO()
+        self.body = open(filename, "wb") if filename else BytesIO()
         self.size = 0
 
     def dataReceived(self, data):
@@ -97,7 +98,7 @@ class FTPDownloadHandler(object):
         protocol.close()
         body = protocol.filename or protocol.body.read()
         headers = {"local filename": protocol.filename or '', "size": protocol.size}
-        return respcls(url=request.url, status=200, body=body, headers=headers)
+        return respcls(url=request.url, status=200, body=to_bytes(body), headers=headers)
 
     def _failed(self, result, request):
         message = result.getErrorMessage()
@@ -106,6 +107,6 @@ class FTPDownloadHandler(object):
             if m:
                 ftpcode = m.group()
                 httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])
-                return Response(url=request.url, status=httpcode, body=message)
+                return Response(url=request.url, status=httpcode, body=to_bytes(message))
         raise result.type(result.value)
 
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index ec2947003..313e74ec9 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,13 +1,6 @@
 tests/test_linkextractors_deprecated.py
 tests/test_proxy_connect.py
 
-scrapy/xlib/tx/iweb.py
-scrapy/xlib/tx/interfaces.py
-scrapy/xlib/tx/endpoints.py
-scrapy/xlib/tx/client.py
-scrapy/xlib/tx/_newclient.py
-scrapy/xlib/tx/__init__.py
-scrapy/core/downloader/handlers/ftp.py
 scrapy/linkextractors/sgml.py
 scrapy/linkextractors/regex.py
 scrapy/linkextractors/htmlparser.py
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index c1683fb3e..e49a514b8 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -687,9 +687,6 @@ class BaseFTPTestCase(unittest.TestCase):
     password = "passwd"
     req_meta = {"ftp_user": username, "ftp_password": password}
 
-    if six.PY3:
-        skip = "Twisted missing ftp support for PY3"
-
     def setUp(self):
         from twisted.protocols.ftp import FTPRealm, FTPFactory
         from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
@@ -700,8 +697,8 @@ class BaseFTPTestCase(unittest.TestCase):
         userdir = os.path.join(self.directory, self.username)
         os.mkdir(userdir)
         fp = FilePath(userdir)
-        fp.child('file.txt').setContent("I have the power!")
-        fp.child('file with spaces.txt').setContent("Moooooooooo power!")
+        fp.child('file.txt').setContent(b"I have the power!")
+        fp.child('file with spaces.txt').setContent(b"Moooooooooo power!")
 
         # setup server
         realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory)
@@ -736,8 +733,8 @@ class BaseFTPTestCase(unittest.TestCase):
 
         def _test(r):
             self.assertEqual(r.status, 200)
-            self.assertEqual(r.body, 'I have the power!')
-            self.assertEqual(r.headers, {'Local Filename': [''], 'Size': ['17']})
+            self.assertEqual(r.body, b'I have the power!')
+            self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']})
         return self._add_test_callbacks(d, _test)
 
     def test_ftp_download_path_with_spaces(self):
@@ -749,8 +746,8 @@ class BaseFTPTestCase(unittest.TestCase):
 
         def _test(r):
             self.assertEqual(r.status, 200)
-            self.assertEqual(r.body, 'Moooooooooo power!')
-            self.assertEqual(r.headers, {'Local Filename': [''], 'Size': ['18']})
+            self.assertEqual(r.body, b'Moooooooooo power!')
+            self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'18']})
         return self._add_test_callbacks(d, _test)
 
     def test_ftp_download_notexist(self):
@@ -763,7 +760,7 @@ class BaseFTPTestCase(unittest.TestCase):
         return self._add_test_callbacks(d, _test)
 
     def test_ftp_local_filename(self):
-        local_fname = "/tmp/file.txt"
+        local_fname = b"/tmp/file.txt"
         meta = {"ftp_local_filename": local_fname}
         meta.update(self.req_meta)
         request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
@@ -772,10 +769,10 @@ class BaseFTPTestCase(unittest.TestCase):
 
         def _test(r):
             self.assertEqual(r.body, local_fname)
-            self.assertEqual(r.headers, {'Local Filename': ['/tmp/file.txt'], 'Size': ['17']})
+            self.assertEqual(r.headers, {b'Local Filename': [b'/tmp/file.txt'], b'Size': [b'17']})
             self.assertTrue(os.path.exists(local_fname))
-            with open(local_fname) as f:
-                self.assertEqual(f.read(), "I have the power!")
+            with open(local_fname, "rb") as f:
+                self.assertEqual(f.read(), b"I have the power!")
             os.remove(local_fname)
         return self._add_test_callbacks(d, _test)
 
@@ -810,8 +807,8 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
         os.mkdir(self.directory)
 
         fp = FilePath(self.directory)
-        fp.child('file.txt').setContent("I have the power!")
-        fp.child('file with spaces.txt').setContent("Moooooooooo power!")
+        fp.child('file.txt').setContent(b"I have the power!")
+        fp.child('file with spaces.txt').setContent(b"Moooooooooo power!")
 
         # setup server for anonymous access
         realm = FTPRealm(anonymousRoot=self.directory)

From f01ae6ffcd431b73f5358f9f876f8e9ee9be0113 Mon Sep 17 00:00:00 2001
From: Rolando Espinoza <rndmax84@gmail.com>
Date: Thu, 23 Feb 2017 11:42:34 -0400
Subject: [PATCH 1333/2974] Handle data loss gracefully.

Websites that return a wrong ``Content-Length`` header may cause a data
loss error. Also when a chunked response is not finished properly.

This change adds a new setting ``DOWNLOAD_FAIL_ON_DATALOSS`` (default:
``True``) and request.meta key ``download_fail_on_dataloss``.
---
 docs/topics/request-response.rst          |  9 +++
 docs/topics/settings.rst                  | 26 ++++++
 scrapy/core/downloader/handlers/http11.py | 41 +++++++---
 scrapy/settings/default_settings.py       |  2 +
 tests/test_downloader_handlers.py         | 99 ++++++++++++++++++++++-
 5 files changed, 167 insertions(+), 10 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 9a6e0d1b6..214ac5640 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -303,6 +303,7 @@ Those are:
 * :reqmeta:`download_timeout`
 * :reqmeta:`download_maxsize`
 * :reqmeta:`download_latency`
+* :reqmeta:`download_fail_on_dataloss`
 * :reqmeta:`proxy`
 
 .. reqmeta:: bindaddress
@@ -330,6 +331,14 @@ started, i.e. HTTP message sent over the network. This meta key only becomes
 available when the response has been downloaded. While most other meta keys are
 used to control Scrapy behavior, this one is supposed to be read-only.
 
+.. reqmeta:: download_fail_on_dataloss
+
+download_fail_on_dataloss
+-------------------------
+
+Whether or not to fail on broken responses. See:
+:setting:`DOWNLOAD_FAIL_ON_DATALOSS`.
+
 .. _topics-request-response-ref-request-subclasses:
 
 Request subclasses
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index f616742c4..ccdd02c4e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -604,6 +604,32 @@ If you want to disable it set to 0.
 
     This feature needs Twisted >= 11.1.
 
+.. setting:: DOWNLOAD_FAIL_ON_DATALOSS
+
+DOWNLOAD_FAIL_ON_DATALOSS
+-------------------------
+
+Default: ``True``
+
+Whether or not to fail on broken responses, that is, declared
+``Content-Length`` does not match content sent by the server or chunked
+response was not properly finish. If ``True``, these responses raise a
+``ResponseFailed([_DataLoss])`` error. If ``False``, these responses
+are passed through and the flag ``dataloss`` is added to the response, i.e.:
+``'dataloss' in response.flags`` is ``True``.
+
+Optionally, this can be set per-request basis by using the
+:reqmeta:`download_fail_on_dataloss` Request.meta key to ``False``.
+
+.. note::
+
+  A broken response, or data loss error, may happen under several
+  circumstances, from server misconfiguration to network errors to data
+  corruption. It is up to the user to decide if it makes sense to process
+  broken responses considering they may contain partial or incomplete content.
+  If setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``,
+  the ``ResponseFailed([_DataLoss])`` failure will be retried as usual.
+
 .. setting:: DUPEFILTER_CLASS
 
 DUPEFILTER_CLASS
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index b96c8c6fe..37e836809 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -12,9 +12,9 @@ from twisted.internet import defer, reactor, protocol
 from twisted.web.http_headers import Headers as TxHeaders
 from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
 from twisted.internet.error import TimeoutError
-from twisted.web.http import PotentialDataLoss
+from twisted.web.http import _DataLoss, PotentialDataLoss
 from twisted.web.client import Agent, ProxyAgent, ResponseDone, \
-    HTTPConnectionPool
+    HTTPConnectionPool, ResponseFailed
 from twisted.internet.endpoints import TCP4ClientEndpoint
 
 from scrapy.http import Headers
@@ -51,13 +51,15 @@ class HTTP11DownloadHandler(object):
             warnings.warn(msg)
         self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
         self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
+        self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
         self._disconnect_timeout = 1
 
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
         agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool,
             maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
-            warnsize=getattr(spider, 'download_warnsize', self._default_warnsize))
+            warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
+            fail_on_dataloss=self._fail_on_dataloss)
         return agent.download_request(request)
 
     def close(self):
@@ -233,13 +235,14 @@ class ScrapyAgent(object):
     _TunnelingAgent = TunnelingAgent
 
     def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
-                 maxsize=0, warnsize=0):
+                 maxsize=0, warnsize=0, fail_on_dataloss=True):
         self._contextFactory = contextFactory
         self._connectTimeout = connectTimeout
         self._bindAddress = bindAddress
         self._pool = pool
         self._maxsize = maxsize
         self._warnsize = warnsize
+        self._fail_on_dataloss = fail_on_dataloss
         self._txresponse = None
 
     def _get_agent(self, request, timeout):
@@ -326,6 +329,7 @@ class ScrapyAgent(object):
         maxsize = request.meta.get('download_maxsize', self._maxsize)
         warnsize = request.meta.get('download_warnsize', self._warnsize)
         expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1
+        fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss)
 
         if maxsize and expected_size > maxsize:
             error_msg = ("Cancelling download of %(url)s: expected response "
@@ -345,7 +349,8 @@ class ScrapyAgent(object):
             txresponse._transport._producer.loseConnection()
 
         d = defer.Deferred(_cancel)
-        txresponse.deliverBody(_ResponseReader(d, txresponse, request, maxsize, warnsize))
+        txresponse.deliverBody(_ResponseReader(
+            d, txresponse, request, maxsize, warnsize, fail_on_dataloss))
 
         # save response for timeouts
         self._txresponse = txresponse
@@ -380,13 +385,16 @@ class _RequestBodyProducer(object):
 
 class _ResponseReader(protocol.Protocol):
 
-    def __init__(self, finished, txresponse, request, maxsize, warnsize):
+    def __init__(self, finished, txresponse, request, maxsize, warnsize,
+                 fail_on_dataloss):
         self._finished = finished
         self._txresponse = txresponse
         self._request = request
         self._bodybuf = BytesIO()
         self._maxsize  = maxsize
         self._warnsize  = warnsize
+        self._fail_on_dataloss = fail_on_dataloss
+        self._fail_on_dataloss_warned = False
         self._reached_warnsize = False
         self._bytes_received = 0
 
@@ -415,7 +423,22 @@ class _ResponseReader(protocol.Protocol):
         body = self._bodybuf.getvalue()
         if reason.check(ResponseDone):
             self._finished.callback((self._txresponse, body, None))
-        elif reason.check(PotentialDataLoss):
+            return
+
+        if reason.check(PotentialDataLoss):
             self._finished.callback((self._txresponse, body, ['partial']))
-        else:
-            self._finished.errback(reason)
+            return
+
+        if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
+            if not self._fail_on_dataloss:
+                self._finished.callback((self._txresponse, body, ['dataloss']))
+                return
+
+            elif not self._fail_on_dataloss_warned:
+                logger.warn("Got data loss in %s. If you want to process broken "
+                            "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
+                            " -- This message won't be shown in further requests",
+                            self._txresponse.request.absoluteURI.decode())
+                self._fail_on_dataloss_warned = True
+
+        self._finished.errback(reason)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index e0e39120c..a5931a3d5 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -79,6 +79,8 @@ DOWNLOAD_TIMEOUT = 180      # 3mins
 DOWNLOAD_MAXSIZE = 1024*1024*1024   # 1024m
 DOWNLOAD_WARNSIZE = 32*1024*1024    # 32m
 
+DOWNLOAD_FAIL_ON_DATALOSS = True
+
 DOWNLOADER = 'scrapy.core.downloader.Downloader'
 
 DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index e49a514b8..4e63b2038 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -13,9 +13,11 @@ from twisted.protocols.policies import WrappingFactory
 from twisted.python.filepath import FilePath
 from twisted.internet import reactor, defer, error
 from twisted.web import server, static, util, resource
+from twisted.web._newclient import ResponseFailed
+from twisted.web.http import _DataLoss
 from twisted.web.test.test_webclient import ForeverTakingResource, \
         NoLengthResource, HostHeaderResource, \
-        PayloadResource, BrokenDownloadResource
+        PayloadResource
 from twisted.cred import portal, checkers, credentials
 from w3lib.url import path_to_file_uri
 
@@ -118,6 +120,52 @@ class ContentLengthHeaderResource(resource.Resource):
         return request.requestHeaders.getRawHeaders(b"content-length")[0]
 
 
+class ChunkedResource(resource.Resource):
+
+    def render(self, request):
+        def response():
+            request.write(b"chunked ")
+            request.write(b"content\n")
+            request.finish()
+        reactor.callLater(0, response)
+        return server.NOT_DONE_YET
+
+
+class BrokenChunkedResource(resource.Resource):
+
+    def render(self, request):
+        def response():
+            request.write(b"chunked ")
+            request.write(b"content\n")
+            # Disable terminating chunk on finish.
+            request.chunked = False
+            closeConnection(request)
+        reactor.callLater(0, response)
+        return server.NOT_DONE_YET
+
+
+class BrokenDownloadResource(resource.Resource):
+
+    def render(self, request):
+        def response():
+            request.setHeader(b"Content-Length", b"20")
+            request.write(b"partial")
+            closeConnection(request)
+
+        reactor.callLater(0, response)
+        return server.NOT_DONE_YET
+
+
+def closeConnection(request):
+    # We have to force a disconnection for HTTP/1.1 clients. Otherwise
+    # client keeps the connection open waiting for more data.
+    if hasattr(request.channel, 'loseConnection'):  # twisted >=16.3.0
+        request.channel.loseConnection()
+    else:
+        request.channel.transport.loseConnection()
+    request.finish()
+
+
 class EmptyContentTypeHeaderResource(resource.Resource):
     """
     A testing resource which renders itself as the value of request body
@@ -149,6 +197,8 @@ class HttpTestCase(unittest.TestCase):
         r.putChild(b"host", HostHeaderResource())
         r.putChild(b"payload", PayloadResource())
         r.putChild(b"broken", BrokenDownloadResource())
+        r.putChild(b"chunked", ChunkedResource())
+        r.putChild(b"broken-chunked", BrokenChunkedResource())
         r.putChild(b"contentlength", ContentLengthHeaderResource())
         r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource())
         self.site = server.Site(r, timeout=None)
@@ -341,6 +391,53 @@ class Http11TestCase(HttpTestCase):
         d.addCallback(self.assertEquals, b"0123456789")
         return d
 
+    def test_download_chunked_content(self):
+        request = Request(self.getURL('chunked'))
+        d = self.download_request(request, Spider('foo'))
+        d.addCallback(lambda r: r.body)
+        d.addCallback(self.assertEquals, b"chunked content\n")
+        return d
+
+    def test_download_broken_content_cause_data_loss(self, url='broken'):
+        request = Request(self.getURL(url))
+        d = self.download_request(request, Spider('foo'))
+
+        def checkDataLoss(failure):
+            if failure.check(ResponseFailed):
+                if any(r.check(_DataLoss) for r in failure.value.reasons):
+                    return None
+            return failure
+
+        d.addCallback(lambda _: self.fail("No DataLoss exception"))
+        d.addErrback(checkDataLoss)
+        return d
+
+    def test_download_broken_chunked_content_cause_data_loss(self):
+        return self.test_download_broken_content_cause_data_loss('broken-chunked')
+
+    def test_download_broken_content_allow_data_loss(self, url='broken'):
+        request = Request(self.getURL(url), meta={'download_fail_on_dataloss': False})
+        d = self.download_request(request, Spider('foo'))
+        d.addCallback(lambda r: r.flags)
+        d.addCallback(self.assertEqual, ['dataloss'])
+        return d
+
+    def test_download_broken_chunked_content_allow_data_loss(self):
+        return self.test_download_broken_content_allow_data_loss('broken-chunked')
+
+    def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
+        download_handler = self.download_handler_cls(Settings({
+            'DOWNLOAD_FAIL_ON_DATALOSS': False,
+        }))
+        request = Request(self.getURL(url))
+        d = download_handler.download_request(request, Spider('foo'))
+        d.addCallback(lambda r: r.flags)
+        d.addCallback(self.assertEqual, ['dataloss'])
+        return d
+
+    def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
+        return self.test_download_broken_content_allow_data_loss_via_setting('broken-chunked')
+
 
 class Https11TestCase(Http11TestCase):
     scheme = 'https'

From 706ed0e049fe008cac12f243371b67ee0230a08a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 14 Jun 2016 12:34:07 -0300
Subject: [PATCH 1334/2974] Spider middleware: process_spider_exception on
 generators

---
 docs/topics/exceptions.rst        |  11 +
 docs/topics/spider-middleware.rst |  10 +-
 scrapy/core/spidermw.py           |  37 +++-
 scrapy/exceptions.py              |   5 +
 tests/test_spidermiddleware.py    | 340 ++++++++++++++++++++++++++++++
 5 files changed, 389 insertions(+), 14 deletions(-)
 create mode 100644 tests/test_spidermiddleware.py

diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index cc02369d4..a3ff72827 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -62,6 +62,17 @@ remain disabled. Those components include:
 
 The exception must be raised in the component's ``__init__`` method.
 
+InvalidOutput
+-------------
+
+.. exception:: InvalidOutput
+
+This exception can be raised by a downloader or spider middleware to
+indicate that some method returned a value not suported by the processing
+chain.
+See :ref:`topics-spider-middleware` and :ref:`topics-downloader-middleware`
+for a list of supported output values.
+
 NotSupported
 ------------
 
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 8360827e8..fc7669437 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -112,11 +112,12 @@ following methods:
 
     .. method:: process_spider_exception(response, exception, spider)
 
-        This method is called when a spider or :meth:`process_spider_input`
-        method (from other spider middleware) raises an exception.
+        This method is called when when a spider or :meth:`process_spider_input`/
+        :meth:`process_spider_output` method (from other spider middleware)
+        raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
-        iterable of :class:`~scrapy.http.Response`, dict or
+        iterable of :class:`~scrapy.http.Request`, dict or
         :class:`~scrapy.item.Item` objects.
 
         If it returns ``None``, Scrapy will continue processing this exception,
@@ -125,7 +126,8 @@ following methods:
         exception reaches the engine (where it's logged and discarded).
 
         If it returns an iterable the :meth:`process_spider_output` pipeline
-        kicks in, and no other :meth:`process_spider_exception` will be called.
+        kicks in, starting with the last non-executed method, and no other
+        :meth:`process_spider_exception` will be called.
 
         :param response: the response being processed when the exception was
           raised
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index a206e4b0c..0f03a7b36 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -5,6 +5,7 @@ See documentation in docs/topics/spider-middleware.rst
 """
 import six
 from twisted.python.failure import Failure
+from scrapy.exceptions import InvalidOutput
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.conf import build_component_list
@@ -40,31 +41,47 @@ class SpiderMiddlewareManager(MiddlewareManager):
             for method in self.methods['process_spider_input']:
                 try:
                     result = method(response=response, spider=spider)
-                    assert result is None, \
-                            'Middleware %s must returns None or ' \
-                            'raise an exception, got %s ' \
-                            % (fname(method), type(result))
+                    if result is not None:
+                        raise InvalidOutput('Middleware {} must return None or raise ' \
+                            'an exception, got {}'.format(fname(method), type(result)))
                 except:
                     return scrape_func(Failure(), request, spider)
             return scrape_func(response, request, spider)
 
         def process_spider_exception(_failure):
             exception = _failure.value
+            # don't handle InvalidOutput exception
+            if isinstance(exception, InvalidOutput):
+                return _failure
             for method in self.methods['process_spider_exception']:
                 result = method(response=response, exception=exception, spider=spider)
-                assert result is None or _isiterable(result), \
-                    'Middleware %s must returns None, or an iterable object, got %s ' % \
-                    (fname(method), type(result))
+                if result is not None and not _isiterable(result):
+                    raise InvalidOutput('Middleware {} must return None or an iterable ' \
+                        'object, got {}'.format(fname(method), type(result)))
+                # stop exception handling by handing control over to the
+                # process_spider_output chain if an iterable has been returned
                 if result is not None:
                     return result
             return _failure
 
         def process_spider_output(result):
+            def wrapper(result_iterable):
+                try:
+                    for r in result_iterable:
+                        yield r
+                except Exception as ex:
+                    exception_result = process_spider_exception(Failure(ex))
+                    if exception_result is None or isinstance(exception_result, Failure):
+                        raise
+                    for output in exception_result:
+                        yield output
             for method in self.methods['process_spider_output']:
                 result = method(response=response, result=result, spider=spider)
-                assert _isiterable(result), \
-                    'Middleware %s must returns an iterable object, got %s ' % \
-                    (fname(method), type(result))
+                if _isiterable(result):
+                    result = wrapper(result)
+                else:
+                    raise InvalidOutput('Middleware {} must return an iterable object, ' \
+                        'got {}'.format(fname(method), type(result)))
             return result
 
         dfd = mustbe_deferred(process_spider_input, response)
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 4bcecd994..ba7272255 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -11,6 +11,11 @@ class NotConfigured(Exception):
     """Indicates a missing configuration situation"""
     pass
 
+class InvalidOutput(TypeError):
+    """Indicates an invalid value has been returned
+    by a middleware's processing method"""
+    pass
+
 # HTTP and crawling
 
 class IgnoreRequest(Exception):
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
new file mode 100644
index 000000000..8ec9583d8
--- /dev/null
+++ b/tests/test_spidermiddleware.py
@@ -0,0 +1,340 @@
+
+import logging
+
+from testfixtures import LogCapture
+from twisted.trial.unittest import TestCase
+from twisted.internet import defer
+
+from scrapy.spiders import Spider
+from scrapy.item import Item, Field
+from scrapy.http import Request
+from scrapy.utils.test import get_crawler
+
+
+class TestItem(Item):
+    value = Field()
+
+
+# ================================================================================
+# exceptions from a spider's parse method
+class BaseExceptionFromParseMethodSpider(Spider):
+    start_urls = ["http://example.com/"]
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {'tests.test_spidermiddleware.CatchExceptionMiddleware': 540}
+    }
+
+
+class NotAGeneratorSpider(BaseExceptionFromParseMethodSpider):
+    """ return value is NOT a generator """
+    name = 'not_a_generator'
+
+    def parse(self, response):
+        raise AssertionError
+
+
+class GeneratorErrorBeforeItemsSpider(BaseExceptionFromParseMethodSpider):
+    """ return value is a generator; the exception is raised
+    before the items are yielded: no items should be scraped """
+    name = 'generator_error_before_items'
+
+    def parse(self, response):
+        raise ValueError
+        for i in range(3):
+            yield {'value': i}
+
+
+class GeneratorErrorAfterItemsSpider(BaseExceptionFromParseMethodSpider):
+    """ return value is a generator; the exception is raised
+    after the items are yielded: 3 items should be scraped """
+    name = 'generator_error_after_items'
+
+    def parse(self, response):
+        for i in range(3):
+            yield {'value': i}
+        raise FloatingPointError
+
+
+class CatchExceptionMiddleware(object):
+    def process_spider_exception(self, response, exception, spider):
+        """ catch an exception and log it """
+        logging.warn('{} exception caught'.format(exception.__class__.__name__))
+        return None
+
+
+# ================================================================================
+# exception from a previous middleware's process_spider_input method
+# process_spider_input is not expected to return an iterable, so there are no
+# separate tests for generator/non-generator implementations
+class FromPreviousMiddlewareInputSpider(Spider):
+    start_urls = ["http://example.com/"]
+    name = 'not_a_generator_from_previous_middleware_input'
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            # engine side
+            'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,
+            'tests.test_spidermiddleware.RaiseExceptionOnInputMiddleware': 545,
+            # spider side
+        }
+    }
+
+    def parse(self, response):
+        return None
+
+
+class RaiseExceptionOnInputMiddleware(object):
+    def process_spider_input(self, response, spider):
+        raise LookupError
+
+
+# ================================================================================
+# exception from a previous middleware's process_spider_output method (not a generator)
+class NotAGeneratorFromPreviousMiddlewareOutputSpider(Spider):
+    start_urls = ["http://example.com/"]
+    name = 'not_a_generator_from_previous_middleware_output'
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            # engine side
+            'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,
+            'tests.test_spidermiddleware.RaiseExceptionOnOutputNotAGeneratorMiddleware': 545,
+            # spider side
+        }
+    }
+
+    def parse(self, response):
+        return [{'value': i} for i in range(3)]
+
+
+class RaiseExceptionOnOutputNotAGeneratorMiddleware(object):
+    def process_spider_output(self, response, result, spider):
+        raise UnicodeError
+
+
+# ================================================================================
+# exception from a previous middleware's process_spider_output method (generator)
+class GeneratorFromPreviousMiddlewareOutputSpider(Spider):
+    start_urls = ["http://example.com/"]
+    name = 'generator_from_previous_middleware_output'
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            # engine side
+            'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,
+            'tests.test_spidermiddleware.RaiseExceptionOnOutputGeneratorMiddleware': 545,
+            # spider side
+        }
+    }
+
+    def parse(self, response):
+        return [{'value': i} for i in range(10, 13)]
+
+
+class RaiseExceptionOnOutputGeneratorMiddleware(object):
+    def process_spider_output(self, response, result, spider):
+        for r in result:
+            yield r
+        raise NameError
+
+
+# ================================================================================
+# do something useful from the exception handler
+class DoSomethingSpider(Spider):
+    start_urls = ["http://example.com"]
+    name = 'do_something'
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            # engine side
+            'tests.test_spidermiddleware.DoSomethingMiddleware': 540,
+            'tests.test_spidermiddleware.CatchExceptionMiddleware': 545,
+            # spider side
+        }
+    }
+
+    def parse(self, response):
+        yield {'value': response.url}
+        raise ImportError
+
+
+class DoSomethingMiddleware(object):
+    def process_spider_exception(self, response, exception, spider):
+        return [Request('http://example.org'), {'value': 10}, TestItem(value='asdf')]
+
+
+# ================================================================================
+# don't catch InvalidOutput from scrapy's spider middleware manager
+class InvalidReturnValueFromPreviousMiddlewareInputSpider(Spider):
+    start_urls = ["http://example.com/"]
+    name = 'invalid_return_value_from_previous_middleware_input'
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            # engine side
+            'tests.test_spidermiddleware.InvalidReturnValueInputMiddleware': 540,
+            'tests.test_spidermiddleware.CatchExceptionMiddleware': 545,
+            # spider side
+        }
+    }
+
+    def parse(self, response):
+        return None
+
+
+class InvalidReturnValueInputMiddleware(object):
+    def process_spider_input(self, response, spider):
+        return 1.0  # <type 'float'>, not None
+
+
+class InvalidReturnValueFromPreviousMiddlewareOutputSpider(Spider):
+    start_urls = ["http://example.com/"]
+    name = 'invalid_return_value_from_previous_middleware_output'
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            # engine side
+            'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,
+            'tests.test_spidermiddleware.InvalidReturnValueOutputMiddleware': 545,
+            # spider side
+        }
+    }
+
+    def parse(self, response):
+        return None
+
+
+class InvalidReturnValueOutputMiddleware(object):
+    def process_spider_output(self, response, result, spider):
+        return 1  # <type 'int'>, not an iterable
+
+
+# ================================================================================
+# make sure only non already called process_spider_output methods
+# are called if process_spider_exception returns an iterable
+class ExecutionChainSpider(Spider):
+    start_urls = ["http://example.com"]
+    name = 'execution_chain'
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            # engine side
+            'tests.test_spidermiddleware.ThirdMiddleware': 540,
+            'tests.test_spidermiddleware.SecondMiddleware': 541,
+            'tests.test_spidermiddleware.FirstMiddleware': 542
+            # spider side
+        },
+    }
+
+    def parse(self, response):
+        return None
+
+
+class FirstMiddleware(object):
+    def process_spider_output(self, response, result, spider):
+        for r in result:
+            if isinstance(r, dict):
+                r['handled_by_first_middleware'] = True
+            yield r
+
+    def process_spider_exception(self, response, exception, spider):
+        # log exception, handle control to the next middleware's process_spider_exception
+        logging.warn('{} exception caught'.format(exception.__class__.__name__))
+        return None
+
+
+class SecondMiddleware(object):
+    def process_spider_output(self, response, result, spider):
+        for r in result:
+            if isinstance(r, dict):
+                r['handled_by_second_middleware'] = True
+            yield r
+        raise MemoryError
+
+
+class ThirdMiddleware(object):
+    def process_spider_output(self, response, result, spider):
+        for r in result:
+            if isinstance(r, dict):
+                r['handled_by_third_middleware'] = True
+            yield r
+
+    def process_spider_exception(self, response, exception, spider):
+        # handle control to the next middleware's process_spider_output
+        return [{'item': i} for i in range(3)]
+
+
+class TestSpiderMiddleware(TestCase):
+
+    @defer.inlineCallbacks
+    def test_process_spider_exception_from_parse_method(self):
+        # non-generator return value
+        crawler = get_crawler(NotAGeneratorSpider)
+        with LogCapture() as log:
+            yield crawler.crawl()
+        self.assertIn("AssertionError exception caught", str(log))
+        self.assertIn("spider_exceptions/AssertionError", str(log))
+        # generator return value, no items before the error
+        crawler = get_crawler(GeneratorErrorBeforeItemsSpider)
+        with LogCapture() as log:
+            yield crawler.crawl()
+        self.assertIn("ValueError exception caught", str(log))
+        self.assertIn("spider_exceptions/ValueError", str(log))
+        # generator return value, 3 items before the error
+        crawler = get_crawler(GeneratorErrorAfterItemsSpider)
+        with LogCapture() as log:
+            yield crawler.crawl()
+        self.assertIn("'item_scraped_count': 3", str(log))
+        self.assertIn("FloatingPointError exception caught", str(log))
+        self.assertIn("spider_exceptions/FloatingPointError", str(log))
+
+    @defer.inlineCallbacks
+    def test_process_spider_exception_from_previous_middleware_input(self):
+        crawler = get_crawler(FromPreviousMiddlewareInputSpider)
+        with LogCapture() as log:
+            yield crawler.crawl()
+        self.assertIn("LookupError exception caught", str(log))
+
+    @defer.inlineCallbacks
+    def test_process_spider_exception_from_previous_middleware_output(self):
+        # non-generator output value
+        crawler = get_crawler(NotAGeneratorFromPreviousMiddlewareOutputSpider)
+        with LogCapture() as log:
+            yield crawler.crawl()
+        self.assertNotIn("UnicodeError exception caught", str(log))
+        # generator output value
+        crawler = get_crawler(GeneratorFromPreviousMiddlewareOutputSpider)
+        with LogCapture() as log:
+            yield crawler.crawl()
+        self.assertIn("'item_scraped_count': 3", str(log))
+        self.assertIn("NameError exception caught", str(log))
+
+    @defer.inlineCallbacks
+    def test_process_spider_exception_do_something(self):
+        crawler = get_crawler(DoSomethingSpider)
+        with LogCapture() as log:
+            yield crawler.crawl()
+        self.assertIn("ImportError exception caught", str(log))
+        self.assertIn("{'value': 10}", str(log))
+        self.assertIn("{'value': 'asdf'}", str(log))
+        self.assertIn("{'value': 'http://example.com'}", str(log))
+        self.assertIn("{'value': 'http://example.org'}", str(log))
+
+    @defer.inlineCallbacks
+    def test_process_spider_exception_invalid_return_value_previous_middleware(self):
+        """ don't catch InvalidOutput from middleware """
+        # on middleware's input
+        crawler1 = get_crawler(InvalidReturnValueFromPreviousMiddlewareInputSpider)
+        with LogCapture() as log1:
+            yield crawler1.crawl()
+        self.assertNotIn("InvalidOutput exception caught", str(log1))
+        self.assertIn("'spider_exceptions/InvalidOutput'", str(log1))
+        # on middleware's output
+        crawler2 = get_crawler(InvalidReturnValueFromPreviousMiddlewareOutputSpider)
+        with LogCapture() as log2:
+            yield crawler2.crawl()
+        self.assertNotIn("InvalidOutput exception caught", str(log2))
+        self.assertIn("'spider_exceptions/InvalidOutput'", str(log2))
+
+    @defer.inlineCallbacks
+    def test_process_spider_exception_execution_chain(self):
+        # on middleware's input
+        crawler1 = get_crawler(ExecutionChainSpider)
+        with LogCapture() as log1:
+            yield crawler1.crawl()
+        self.assertNotIn("handled_by_first_middleware", str(log1))
+        self.assertNotIn("handled_by_second_middleware", str(log1))
+        self.assertIn("MemoryError exception caught", str(log1))
+        self.assertIn("handled_by_third_middleware", str(log1))

From baed7c436f8c6e52a1aab62fbaf900c0a0f3bbda Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 5 Oct 2016 11:18:26 +0200
Subject: [PATCH 1335/2974] WIP Add Referrer policies

---
 scrapy/spidermiddlewares/referer.py    | 206 ++++++++++++++++++++++++-
 tests/test_spidermiddleware_referer.py |  74 +++++++++
 2 files changed, 279 insertions(+), 1 deletion(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 6a8c46543..21b340f22 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -2,22 +2,226 @@
 RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
+from six.moves.urllib.parse import urlsplit, urlunsplit
 
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
+from scrapy.utils.python import to_native_str
+
+
+LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)
+
+class ReferrerPolicy(object):
+
+    NOREFERRER_SCHEMES = LOCAL_SCHEMES
+
+    def referrer(self, response, request):
+        raise NotImplementedError()
+
+    def strip_url(self, url, origin_only=False):
+        """
+        https://www.w3.org/TR/referrer-policy/#strip-url
+
+        If url is null, return no referrer.
+        If url's scheme is a local scheme, then return no referrer.
+        Set url's username to the empty string.
+        Set url's password to null.
+        Set url's fragment to null.
+        If the origin-only flag is true, then:
+            Set url's path to null.
+            Set url's query to null.
+        Return url.
+        """
+        if url is None or not url:
+            return None
+        parsed = urlsplit(url, allow_fragments=True)
+
+        if parsed.scheme in self.NOREFERRER_SCHEMES:
+            return None
+        if parsed.username or parsed.password:
+            netloc = parsed.netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '')
+        else:
+            netloc = parsed.netloc
+        return urlunsplit((
+            parsed.scheme,
+            netloc,
+            '' if origin_only else parsed.path,
+            '' if origin_only else parsed.query,
+            ''))
+
+
+class NoReferrerPolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer
+
+    The simplest policy is "no-referrer", which specifies that no referrer information
+    is to be sent along with requests made from a particular request client to any origin.
+    The header will be omitted entirely.
+    """
+    name = "no-referrer"
+
+    def referrer(self, response, request):
+        return None
+
+
+class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
+
+    The "no-referrer-when-downgrade" policy sends a full URL
+    along with requests from a TLS-protected environment settings object
+    to a a priori authenticated URL,
+    and requests from request clients which are not TLS-protected to any origin.
+
+    Requests from TLS-protected request clients to non-a priori authenticated URLs,
+    on the other hand, will contain no referrer information.
+    A Referer HTTP header will not be sent.
+
+    This is a user agent's default behavior, if no policy is otherwise specified.
+    """
+    name = "no-referrer-when-downgrade"
+
+    def referrer(self, response, request):
+        target_url = request.url
+
+        referrer_source = response.url
+        referrer_url = self.strip_url(referrer_source)
+
+        # https://www.w3.org/TR/referrer-policy/#determine-requests-referrer:
+        #
+        # If environment is TLS-protected
+        # and the origin of request's current URL is not an a priori authenticated URL,
+        # then return no referrer.
+        if urlsplit(referrer_source).scheme in ('https', 'ftps') and \
+            urlsplit(target_url).scheme in ('http',):
+                return None
+        return referrer_url
+
+
+class SameOriginPolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin
+
+    The "same-origin" policy specifies that a full URL, stripped for use as a referrer,
+    is sent as referrer information when making same-origin requests from a particular request client.
+
+    Cross-origin requests, on the other hand, will contain no referrer information.
+    A Referer HTTP header will not be sent.
+    """
+    name = "same-origin"
+
+    def referrer(self, response, request):
+        target_url = request.url
+        referrer_source = response.url
+        if urlsplit(referrer_source).netloc == urlsplit(target_url).netloc:
+            return self.strip_url(referrer_source)
+        else:
+            return None
+
+
+class OriginPolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-origin
+
+    The "origin" policy specifies that only the ASCII serialization
+    of the origin of the request client is sent as referrer information
+    when making both same-origin requests and cross-origin requests
+    from a particular request client.
+    """
+    name = "origin"
+
+    def referrer(self, response, request):
+        return self.strip_url(referrer_source, origin_only=True)
+
+
+class OriginWhenCrossOriginPolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin
+
+    The "origin-when-cross-origin" policy specifies that a full URL,
+    stripped for use as a referrer, is sent as referrer information
+    when making same-origin requests from a particular request client,
+    and only the ASCII serialization of the origin of the request client
+    is sent as referrer information when making cross-origin requests
+    from a particular request client.
+    """
+    name = "origin-when-cross-origin"
+
+    def referrer(self, response, request):
+        target_url = request.url
+        referrer_source = response.url
+
+        # same origin --> send full referrer
+        # different origin --> send only "origin" as referrer
+        if urlsplit(referrer_source).netloc != urlsplit(target_url).netloc:
+            origin_only = True
+        return self.strip_url(referrer_source, origin_only=origin_only)
+
+
+class UnsafeUrlPolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url
+
+    The "unsafe-url" policy specifies that a full URL, stripped for use as a referrer,
+    is sent along with both cross-origin requests
+    and same-origin requests made from a particular request client.
+
+    Note: The policy's name doesn't lie; it is unsafe.
+    This policy will leak origins and paths from TLS-protected resources
+    to insecure origins.
+    Carefully consider the impact of setting such a policy for potentially sensitive documents.
+    """
+    name = "unsafe-url"
+
+    def referrer(self, response, request):
+        referrer_source = response.url
+        return self.strip_url(referrer_source)
+
+
+class LegacyPolicy(ReferrerPolicy):
+    def referrer(self, response, request):
+        return response.url
+
+
+class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
+
+    NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3')
+
+
+_policies = {p.name: p for p in (
+    NoReferrerPolicy,
+    NoReferrerWhenDowngradePolicy,
+    SameOriginPolicy,
+    OriginPolicy,
+    OriginWhenCrossOriginPolicy,
+    UnsafeUrlPolicy,
+)}
 
 class RefererMiddleware(object):
 
+    def __init__(self, policy_class=DefaultReferrerPolicy):
+        self.default_policy = policy_class
+
     @classmethod
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('REFERER_ENABLED'):
             raise NotConfigured
         return cls()
 
+    def policy(self, response, request):
+        policy_name = request.meta.get('referrer_policy')
+        if policy_name is None:
+            policy_name = to_native_str(response.headers.get('Referrer-Policy', '').decode('latin1'))
+
+        policy_class = _policies.get(policy_name.lower(), self.default_policy)
+        return policy_class()
+
     def process_spider_output(self, response, result, spider):
         def _set_referer(r):
             if isinstance(r, Request):
-                r.headers.setdefault('Referer', response.url)
+                referrer = self.policy(response, r).referrer(response, r)
+                if referrer is not None:
+                    r.headers.setdefault('Referer', referrer)
             return r
         return (_set_referer(r) for r in result or ())
 
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index bd7673efb..f109bb248 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -19,3 +19,77 @@ class TestRefererMiddleware(TestCase):
         self.assertEquals(out[0].headers.get('Referer'),
                           b'http://scrapytest.org')
 
+    def test_policy_default(self):
+        """
+        Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
+
+        with some additional filtering of s3://
+        """
+        # a) https:// --> https://  -- include Referer header
+        origin = Response('https://example.com/')
+        target = Request('https://scrapy.org/')
+
+        out = list(self.mw.process_spider_output(origin, [target], self.spider))
+        self.assertEquals(out[0].headers.get('Referer'),
+                          b'https://example.com/')
+
+        # b.1) http:// --> http://  -- include Referer header
+        origin = Response('http://example.com/')
+        target = Request('http://scrapy.org/')
+
+        out = list(self.mw.process_spider_output(origin, [target], self.spider))
+        self.assertEquals(out[0].headers.get('Referer'),
+                          b'http://example.com/')
+
+        # b.2) http:// --> https://  -- include Referer header
+        origin = Response('http://example.com/')
+        target = Request('https://scrapy.org/')
+
+        out = list(self.mw.process_spider_output(origin, [target], self.spider))
+        self.assertEquals(out[0].headers.get('Referer'),
+                          b'http://example.com/')
+
+        # c) https:// --> http://  -- Referer header NOT sent
+        origin = Response('https://example.com/')
+        target = Request('http://scrapy.org/')
+
+        out = list(self.mw.process_spider_output(origin, [target], self.spider))
+        self.assertEquals(out[0].headers.get('Referer'), None)
+
+    def test_policy_default_no_credentials_leak(self):
+        origin = Response('http://user:password@example.com/')
+        target = Request('https://scrapy.org/')
+
+        out = list(self.mw.process_spider_output(origin, [target], self.spider))
+        self.assertEquals(out[0].headers.get('Referer'),
+                          b'http://example.com/')
+
+    def test_policy_default_file_no_referrer_leak(self):
+        # file:// --> https://  -- Referrer NOT sent
+        origin = Response('file:///home/path/to/somefile.html')
+        target = Request('https://scrapy.org/')
+
+        out = list(self.mw.process_spider_output(origin, [target], self.spider))
+        self.assertEquals(out[0].headers.get('Referer'), None)
+
+        # file:// --> http://  -- Referrer NOT sent
+        origin = Response('file:///home/path/to/somefile.html')
+        target = Request('http://scrapy.org/')
+
+        out = list(self.mw.process_spider_output(origin, [target], self.spider))
+        self.assertEquals(out[0].headers.get('Referer'), None)
+
+    def test_policy_default_s3_no_referrer_leak(self):
+        # s3:// --> https://  -- Referrer NOT sent
+        origin = Response('s3://mybucket/path/to/data.csv')
+        target = Request('https://scrapy.org/')
+
+        out = list(self.mw.process_spider_output(origin, [target], self.spider))
+        self.assertEquals(out[0].headers.get('Referer'), None)
+
+        # s3:// --> http://  -- Referrer NOT sent
+        origin = Response('s3://mybucket/path/to/data.csv')
+        target = Request('http://scrapy.org/')
+
+        out = list(self.mw.process_spider_output(origin, [target], self.spider))
+        self.assertEquals(out[0].headers.get('Referer'), None)

From 7ec1b5f6c316f6c251821441f513e3f71cf63da0 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 5 Oct 2016 17:12:44 +0200
Subject: [PATCH 1336/2974] Add tests for the different referrer policies

---
 scrapy/spidermiddlewares/referer.py    |  66 ++++---
 tests/test_spidermiddleware_referer.py | 242 +++++++++++++++++++------
 2 files changed, 235 insertions(+), 73 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 21b340f22..1895aa95d 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -11,6 +11,15 @@ from scrapy.utils.python import to_native_str
 
 LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)
 
+POLICY_NO_REFERRER = "no-referrer"
+POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade"
+POLICY_SAME_ORIGIN = "same-origin"
+POLICY_ORIGIN = "origin"
+POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin"
+POLICY_UNSAFE_URL = "unsafe-url"
+POLICY_SCRAPY_DEFAULT = "scrapy-default"
+
+
 class ReferrerPolicy(object):
 
     NOREFERRER_SCHEMES = LOCAL_SCHEMES
@@ -38,17 +47,29 @@ class ReferrerPolicy(object):
 
         if parsed.scheme in self.NOREFERRER_SCHEMES:
             return None
+
+        netloc = parsed.netloc
+        # strip username and password if present
         if parsed.username or parsed.password:
-            netloc = parsed.netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '')
-        else:
-            netloc = parsed.netloc
+            netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '')
+
+        # strip standard protocol numbers
+        # Note: strictly speaking, standard port numbers should only be
+        # stripped when comparing origins
+        if parsed.port:
+            if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)):
+                netloc = netloc.replace(':{p.port}'.format(p=parsed), '')
+
         return urlunsplit((
             parsed.scheme,
             netloc,
-            '' if origin_only else parsed.path,
+            '/' if origin_only else parsed.path,
             '' if origin_only else parsed.query,
             ''))
 
+    def origin(self, url):
+        return self.strip_url(url, origin_only=True)
+
 
 class NoReferrerPolicy(ReferrerPolicy):
     """
@@ -58,7 +79,7 @@ class NoReferrerPolicy(ReferrerPolicy):
     is to be sent along with requests made from a particular request client to any origin.
     The header will be omitted entirely.
     """
-    name = "no-referrer"
+    name = POLICY_NO_REFERRER
 
     def referrer(self, response, request):
         return None
@@ -79,7 +100,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
 
     This is a user agent's default behavior, if no policy is otherwise specified.
     """
-    name = "no-referrer-when-downgrade"
+    name = POLICY_NO_REFERRER_WHEN_DOWNGRADE
 
     def referrer(self, response, request):
         target_url = request.url
@@ -108,12 +129,12 @@ class SameOriginPolicy(ReferrerPolicy):
     Cross-origin requests, on the other hand, will contain no referrer information.
     A Referer HTTP header will not be sent.
     """
-    name = "same-origin"
+    name = POLICY_SAME_ORIGIN
 
     def referrer(self, response, request):
         target_url = request.url
         referrer_source = response.url
-        if urlsplit(referrer_source).netloc == urlsplit(target_url).netloc:
+        if self.origin(referrer_source) == self.origin(target_url):
             return self.strip_url(referrer_source)
         else:
             return None
@@ -128,10 +149,10 @@ class OriginPolicy(ReferrerPolicy):
     when making both same-origin requests and cross-origin requests
     from a particular request client.
     """
-    name = "origin"
+    name = POLICY_ORIGIN
 
     def referrer(self, response, request):
-        return self.strip_url(referrer_source, origin_only=True)
+        return self.strip_url(response.url, origin_only=True)
 
 
 class OriginWhenCrossOriginPolicy(ReferrerPolicy):
@@ -145,17 +166,17 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
     is sent as referrer information when making cross-origin requests
     from a particular request client.
     """
-    name = "origin-when-cross-origin"
+    name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
 
     def referrer(self, response, request):
         target_url = request.url
         referrer_source = response.url
+        source_origin = self.origin(referrer_source)
+        if source_origin == self.origin(target_url):
+            return self.strip_url(referrer_source, origin_only=False)
+        else:
+            return source_origin
 
-        # same origin --> send full referrer
-        # different origin --> send only "origin" as referrer
-        if urlsplit(referrer_source).netloc != urlsplit(target_url).netloc:
-            origin_only = True
-        return self.strip_url(referrer_source, origin_only=origin_only)
 
 
 class UnsafeUrlPolicy(ReferrerPolicy):
@@ -171,7 +192,7 @@ class UnsafeUrlPolicy(ReferrerPolicy):
     to insecure origins.
     Carefully consider the impact of setting such a policy for potentially sensitive documents.
     """
-    name = "unsafe-url"
+    name = POLICY_UNSAFE_URL
 
     def referrer(self, response, request):
         referrer_source = response.url
@@ -186,15 +207,17 @@ class LegacyPolicy(ReferrerPolicy):
 class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
 
     NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3')
+    name = POLICY_SCRAPY_DEFAULT
 
 
-_policies = {p.name: p for p in (
+_policy_classes = {p.name: p for p in (
     NoReferrerPolicy,
     NoReferrerWhenDowngradePolicy,
     SameOriginPolicy,
     OriginPolicy,
     OriginWhenCrossOriginPolicy,
     UnsafeUrlPolicy,
+    DefaultReferrerPolicy,
 )}
 
 class RefererMiddleware(object):
@@ -211,10 +234,11 @@ class RefererMiddleware(object):
     def policy(self, response, request):
         policy_name = request.meta.get('referrer_policy')
         if policy_name is None:
-            policy_name = to_native_str(response.headers.get('Referrer-Policy', '').decode('latin1'))
+            policy_name = to_native_str(
+                response.headers.get('Referrer-Policy', '').decode('latin1'))
 
-        policy_class = _policies.get(policy_name.lower(), self.default_policy)
-        return policy_class()
+        cls = _policy_classes.get(policy_name.lower(), self.default_policy)
+        return cls()
 
     def process_spider_output(self, response, result, spider):
         def _set_referer(r):
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index f109bb248..8458fe90b 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -2,7 +2,10 @@ from unittest import TestCase
 
 from scrapy.http import Response, Request
 from scrapy.spiders import Spider
-from scrapy.spidermiddlewares.referer import RefererMiddleware
+from scrapy.spidermiddlewares.referer import RefererMiddleware, \
+    POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \
+    POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \
+    POLICY_UNSAFE_URL
 
 
 class TestRefererMiddleware(TestCase):
@@ -25,71 +28,206 @@ class TestRefererMiddleware(TestCase):
 
         with some additional filtering of s3://
         """
-        # a) https:// --> https://  -- include Referer header
-        origin = Response('https://example.com/')
-        target = Request('https://scrapy.org/')
+        for origin, target, referrer in [
+                ('https://example.com/',    'https://scrapy.org/',  b'https://example.com/'),
+                ('http://example.com/',     'http://scrapy.org/',   b'http://example.com/'),
+                ('http://example.com/',     'https://scrapy.org/',  b'http://example.com/'),
+                ('https://example.com/',    'http://scrapy.org/',   None),
 
-        out = list(self.mw.process_spider_output(origin, [target], self.spider))
-        self.assertEquals(out[0].headers.get('Referer'),
-                          b'https://example.com/')
+                # no credentials leak
+                ('http://user:password@example.com/',  'https://scrapy.org/', b'http://example.com/'),
 
-        # b.1) http:// --> http://  -- include Referer header
-        origin = Response('http://example.com/')
-        target = Request('http://scrapy.org/')
+                # no referrer leak for local schemes
+                ('file:///home/path/to/somefile.html',  'https://scrapy.org/', None),
+                ('file:///home/path/to/somefile.html',  'http://scrapy.org/',  None),
 
-        out = list(self.mw.process_spider_output(origin, [target], self.spider))
-        self.assertEquals(out[0].headers.get('Referer'),
-                          b'http://example.com/')
+                # no referrer leak for s3 origins
+                ('s3://mybucket/path/to/data.csv',  'https://scrapy.org/', None),
+                ('s3://mybucket/path/to/data.csv',  'http://scrapy.org/',  None),
+            ]:
+            response = Response(origin)
+            request = Request(target)
 
-        # b.2) http:// --> https://  -- include Referer header
-        origin = Response('http://example.com/')
-        target = Request('https://scrapy.org/')
+            out = list(self.mw.process_spider_output(response, [request], self.spider))
+            self.assertEquals(out[0].headers.get('Referer'), referrer)
 
-        out = list(self.mw.process_spider_output(origin, [target], self.spider))
-        self.assertEquals(out[0].headers.get('Referer'),
-                          b'http://example.com/')
+    def test_policy_no_referrer(self):
 
-        # c) https:// --> http://  -- Referer header NOT sent
-        origin = Response('https://example.com/')
-        target = Request('http://scrapy.org/')
+        for origin, target, referrer in [
+                ('https://example.com/page.html',       'https://example.com/', None),
+                ('http://www.example.com/',             'https://scrapy.org/',  None),
+                ('http://www.example.com/',             'http://scrapy.org/',   None),
+                ('https://www.example.com/',            'http://scrapy.org/',   None),
+                ('file:///home/path/to/somefile.html',  'http://scrapy.org/',   None),
+            ]:
+            response = Response(origin)
+            request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER})
 
-        out = list(self.mw.process_spider_output(origin, [target], self.spider))
-        self.assertEquals(out[0].headers.get('Referer'), None)
+            out = list(self.mw.process_spider_output(response, [request], self.spider))
+            self.assertEquals(out[0].headers.get('Referer'), referrer)
 
-    def test_policy_default_no_credentials_leak(self):
-        origin = Response('http://user:password@example.com/')
-        target = Request('https://scrapy.org/')
+    def test_policy_no_referrer_when_downgrade(self):
 
-        out = list(self.mw.process_spider_output(origin, [target], self.spider))
-        self.assertEquals(out[0].headers.get('Referer'),
-                          b'http://example.com/')
+        for origin, target, referrer in [
+                # TLS to TLS: send non-empty referrer
+                ('https://example.com/page.html',       'https://not.example.com/', b'https://example.com/page.html'),
+                ('https://example.com/page.html',       'https://scrapy.org/',      b'https://example.com/page.html'),
+                ('https://example.com:443/page.html',   'https://scrapy.org/',      b'https://example.com/page.html'),
+                ('https://example.com:444/page.html',   'https://scrapy.org/',      b'https://example.com:444/page.html'),
+                ('ftps://example.com/urls.zip',         'https://scrapy.org/',      b'ftps://example.com/urls.zip'),
 
-    def test_policy_default_file_no_referrer_leak(self):
-        # file:// --> https://  -- Referrer NOT sent
-        origin = Response('file:///home/path/to/somefile.html')
-        target = Request('https://scrapy.org/')
+                # TLS to non-TLS: do not send referrer
+                ('https://example.com/page.html',       'http://not.example.com/',  None),
+                ('https://example.com/page.html',       'http://scrapy.org/',       None),
+                ('ftps://example.com/urls.zip',         'http://scrapy.org/',       None),
 
-        out = list(self.mw.process_spider_output(origin, [target], self.spider))
-        self.assertEquals(out[0].headers.get('Referer'), None)
+                # non-TLS to TLS or non-TLS: send referrer
+                ('http://example.com/page.html',        'https://not.example.com/', b'http://example.com/page.html'),
+                ('http://example.com/page.html',        'https://scrapy.org/',      b'http://example.com/page.html'),
+                ('http://example.com:8080/page.html',   'https://scrapy.org/',      b'http://example.com:8080/page.html'),
+                ('http://example.com:80/page.html',     'http://not.example.com/',  b'http://example.com/page.html'),
+                ('http://example.com/page.html',        'http://scrapy.org/',       b'http://example.com/page.html'),
+                ('http://example.com:443/page.html',    'http://scrapy.org/',       b'http://example.com:443/page.html'),
+                ('ftp://example.com/urls.zip',          'http://scrapy.org/',       b'ftp://example.com/urls.zip'),
+                ('ftp://example.com/urls.zip',          'https://scrapy.org/',      b'ftp://example.com/urls.zip'),
 
-        # file:// --> http://  -- Referrer NOT sent
-        origin = Response('file:///home/path/to/somefile.html')
-        target = Request('http://scrapy.org/')
+                # test for user/password stripping
+                ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
+            ]:
+            response = Response(origin)
+            request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE})
 
-        out = list(self.mw.process_spider_output(origin, [target], self.spider))
-        self.assertEquals(out[0].headers.get('Referer'), None)
+            out = list(self.mw.process_spider_output(response, [request], self.spider))
+            self.assertEquals(out[0].headers.get('Referer'), referrer)
 
-    def test_policy_default_s3_no_referrer_leak(self):
-        # s3:// --> https://  -- Referrer NOT sent
-        origin = Response('s3://mybucket/path/to/data.csv')
-        target = Request('https://scrapy.org/')
+    def test_policy_same_origin(self):
 
-        out = list(self.mw.process_spider_output(origin, [target], self.spider))
-        self.assertEquals(out[0].headers.get('Referer'), None)
+        for origin, target, referrer in [
+                # Same origin (protocol, host, port): send referrer
+                ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
+                ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
+                ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
+                ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
+                ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
+                ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
 
-        # s3:// --> http://  -- Referrer NOT sent
-        origin = Response('s3://mybucket/path/to/data.csv')
-        target = Request('http://scrapy.org/')
+                # Different host: do NOT send referrer
+                ('https://example.com/page.html',       'https://not.example.com/otherpage.html',   None),
+                ('http://example.com/page.html',        'http://not.example.com/otherpage.html',    None),
+                ('http://example.com/page.html',        'http://www.example.com/otherpage.html',    None),
 
-        out = list(self.mw.process_spider_output(origin, [target], self.spider))
-        self.assertEquals(out[0].headers.get('Referer'), None)
+                # Different port: do NOT send referrer
+                ('https://example.com:444/page.html',   'https://example.com/not-page.html',    None),
+                ('http://example.com:81/page.html',     'http://example.com/not-page.html',     None),
+                ('http://example.com/page.html',        'http://example.com:81/not-page.html',  None),
+
+                # Different protocols: do NOT send refferer
+                ('https://example.com/page.html',   'http://example.com/not-page.html',     None),
+                ('https://example.com/page.html',   'http://not.example.com/',              None),
+                ('ftps://example.com/urls.zip',     'https://example.com/not-page.html',    None),
+                ('ftp://example.com/urls.zip',      'http://example.com/not-page.html',     None),
+                ('ftps://example.com/urls.zip',     'https://example.com/not-page.html',    None),
+
+                # test for user/password stripping
+                ('https://user:password@example.com/page.html', 'https://example.com/not-page.html',    b'https://example.com/page.html'),
+                ('https://user:password@example.com/page.html', 'http://example.com/not-page.html',     None),
+            ]:
+            response = Response(origin)
+            request = Request(target, meta={'referrer_policy': POLICY_SAME_ORIGIN})
+
+            out = list(self.mw.process_spider_output(response, [request], self.spider))
+            self.assertEquals(out[0].headers.get('Referer'), referrer)
+
+    def test_policy_origin(self):
+
+        for origin, target, referrer in [
+                # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)
+                ('https://example.com/page.html',   'https://example.com/not-page.html',    b'https://example.com/'),
+                ('https://example.com/page.html',   'https://scrapy.org',                   b'https://example.com/'),
+                ('https://example.com/page.html',   'http://scrapy.org',                    b'https://example.com/'),
+                ('http://example.com/page.html',    'http://scrapy.org',                    b'http://example.com/'),
+
+                # test for user/password stripping
+                ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
+            ]:
+            response = Response(origin)
+            request = Request(target, meta={'referrer_policy': POLICY_ORIGIN})
+
+            out = list(self.mw.process_spider_output(response, [request], self.spider))
+            self.assertEquals(out[0].headers.get('Referer'), referrer)
+
+    def test_policy_origin_when_cross_origin(self):
+
+        for origin, target, referrer in [
+                # Same origin (protocol, host, port): send referrer
+                ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
+                ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
+                ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
+                ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
+                ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
+                ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+
+                # Different host: send origin as referrer
+                ('https://example2.com/page.html',  'https://scrapy.org/otherpage.html',        b'https://example2.com/'),
+                ('https://example2.com/page.html',  'https://not.example2.com/otherpage.html',  b'https://example2.com/'),
+                ('http://example2.com/page.html',   'http://not.example2.com/otherpage.html',   b'http://example2.com/'),
+                # exact match required
+                ('http://example2.com/page.html',   'http://www.example2.com/otherpage.html',   b'http://example2.com/'),
+
+                # Different port: send origin as referrer
+                ('https://example3.com:444/page.html',  'https://example3.com/not-page.html',   b'https://example3.com:444/'),
+                ('http://example3.com:81/page.html',    'http://example3.com/not-page.html',    b'http://example3.com:81/'),
+
+                # Different protocols: send origin as referrer
+                ('https://example4.com/page.html',  'http://example4.com/not-page.html',    b'https://example4.com/'),
+                ('https://example4.com/page.html',  'http://not.example4.com/',             b'https://example4.com/'),
+                ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
+                ('ftp://example4.com/urls.zip',     'http://example4.com/not-page.html',    b'ftp://example4.com/'),
+                ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
+
+                # test for user/password stripping
+                ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html',  b'https://example5.com/page.html'),
+                # TLS to non-TLS downgrade: send origin
+                ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html',   b'https://example5.com/'),
+            ]:
+            response = Response(origin)
+            request = Request(target, meta={'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN})
+
+            out = list(self.mw.process_spider_output(response, [request], self.spider))
+            self.assertEquals(out[0].headers.get('Referer'), referrer)
+
+    def test_policy_unsafe_url(self):
+
+        for origin, target, referrer in [
+                # TLS to TLS: send referrer
+                ('https://example.com/sekrit.html',     'http://not.example.com/',      b'https://example.com/sekrit.html'),
+                ('https://example1.com/page.html',      'https://not.example1.com/',    b'https://example1.com/page.html'),
+                ('https://example1.com/page.html',      'https://scrapy.org/',          b'https://example1.com/page.html'),
+                ('https://example1.com:443/page.html',  'https://scrapy.org/',          b'https://example1.com/page.html'),
+                ('https://example1.com:444/page.html',  'https://scrapy.org/',          b'https://example1.com:444/page.html'),
+                ('ftps://example1.com/urls.zip',        'https://scrapy.org/',          b'ftps://example1.com/urls.zip'),
+
+                # TLS to non-TLS: send referrer (yes, it's unsafe)
+                ('https://example2.com/page.html',  'http://not.example2.com/', b'https://example2.com/page.html'),
+                ('https://example2.com/page.html',  'http://scrapy.org/',       b'https://example2.com/page.html'),
+                ('ftps://example2.com/urls.zip',    'http://scrapy.org/',       b'ftps://example2.com/urls.zip'),
+
+                # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)
+                ('http://example3.com/page.html',       'https://not.example3.com/',    b'http://example3.com/page.html'),
+                ('http://example3.com/page.html',       'https://scrapy.org/',          b'http://example3.com/page.html'),
+                ('http://example3.com:8080/page.html',  'https://scrapy.org/',          b'http://example3.com:8080/page.html'),
+                ('http://example3.com:80/page.html',    'http://not.example3.com/',     b'http://example3.com/page.html'),
+                ('http://example3.com/page.html',       'http://scrapy.org/',           b'http://example3.com/page.html'),
+                ('http://example3.com:443/page.html',   'http://scrapy.org/',           b'http://example3.com:443/page.html'),
+                ('ftp://example3.com/urls.zip',         'http://scrapy.org/',           b'ftp://example3.com/urls.zip'),
+                ('ftp://example3.com/urls.zip',         'https://scrapy.org/',          b'ftp://example3.com/urls.zip'),
+
+                # test for user/password stripping
+                ('http://user:password@example4.com/page.html',     'https://not.example4.com/',    b'http://example4.com/page.html'),
+                ('https://user:password@example4.com/page.html',    'http://scrapy.org/',           b'https://example4.com/page.html'),
+            ]:
+            response = Response(origin)
+            request = Request(target, meta={'referrer_policy': POLICY_UNSAFE_URL})
+
+            out = list(self.mw.process_spider_output(response, [request], self.spider))
+            self.assertEquals(out[0].headers.get('Referer'), referrer)

From 3af88a2877f947c74c6d9003da620629b27d5a17 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 5 Oct 2016 18:27:18 +0200
Subject: [PATCH 1337/2974] Use urlparse_cached() on request and responses

---
 scrapy/spidermiddlewares/referer.py | 79 ++++++++++++++++++++++-------
 1 file changed, 60 insertions(+), 19 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 1895aa95d..cbbcd3e97 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -2,11 +2,12 @@
 RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
-from six.moves.urllib.parse import urlsplit, urlunsplit
+from six.moves.urllib.parse import urlparse, urlunparse, ParseResult
 
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.python import to_native_str
+from scrapy.utils.httpobj import urlparse_cached
 
 
 LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)
@@ -27,7 +28,7 @@ class ReferrerPolicy(object):
     def referrer(self, response, request):
         raise NotImplementedError()
 
-    def strip_url(self, url, origin_only=False):
+    def strip_url_parsed(self, req_or_resp, origin_only=False):
         """
         https://www.w3.org/TR/referrer-policy/#strip-url
 
@@ -41,9 +42,9 @@ class ReferrerPolicy(object):
             Set url's query to null.
         Return url.
         """
-        if url is None or not url:
+        if req_or_resp.url is None or not req_or_resp.url:
             return None
-        parsed = urlsplit(url, allow_fragments=True)
+        parsed = urlparse_cached(req_or_resp)
 
         if parsed.scheme in self.NOREFERRER_SCHEMES:
             return None
@@ -60,16 +61,61 @@ class ReferrerPolicy(object):
             if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)):
                 netloc = netloc.replace(':{p.port}'.format(p=parsed), '')
 
-        return urlunsplit((
+        return ParseResult(parsed.scheme,
+                           netloc,
+                           '/' if origin_only else parsed.path,
+                           '' if origin_only else parsed.params,
+                           '' if origin_only else parsed.query,
+                           '')
+
+    def strip_url(self, url, origin_only=False):
+        """
+        https://www.w3.org/TR/referrer-policy/#strip-url
+
+        If url is null, return no referrer.
+        If url's scheme is a local scheme, then return no referrer.
+        Set url's username to the empty string.
+        Set url's password to null.
+        Set url's fragment to null.
+        If the origin-only flag is true, then:
+            Set url's path to null.
+            Set url's query to null.
+        Return url.
+        """
+        if url is None or not url:
+            return None
+        parsed = urlparse(url, allow_fragments=True)
+
+        if parsed.scheme in self.NOREFERRER_SCHEMES:
+            return None
+
+        netloc = parsed.netloc
+        # strip username and password if present
+        if parsed.username or parsed.password:
+            netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '')
+
+        # strip standard protocol numbers
+        # Note: strictly speaking, standard port numbers should only be
+        # stripped when comparing origins
+        if parsed.port:
+            if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)):
+                netloc = netloc.replace(':{p.port}'.format(p=parsed), '')
+
+        return urlunparse((
             parsed.scheme,
             netloc,
             '/' if origin_only else parsed.path,
+            '' if origin_only else parsed.params,
             '' if origin_only else parsed.query,
             ''))
 
     def origin(self, url):
         return self.strip_url(url, origin_only=True)
 
+    def origin_parsed(self, req_or_resp):
+        """Return (scheme, host, path) tuple for a request or response URL."""
+        return tuple(self.strip_url_parsed(req_or_resp, origin_only=True)[:3])
+
 
 class NoReferrerPolicy(ReferrerPolicy):
     """
@@ -103,20 +149,17 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
     name = POLICY_NO_REFERRER_WHEN_DOWNGRADE
 
     def referrer(self, response, request):
-        target_url = request.url
-
-        referrer_source = response.url
-        referrer_url = self.strip_url(referrer_source)
-
         # https://www.w3.org/TR/referrer-policy/#determine-requests-referrer:
         #
         # If environment is TLS-protected
         # and the origin of request's current URL is not an a priori authenticated URL,
         # then return no referrer.
-        if urlsplit(referrer_source).scheme in ('https', 'ftps') and \
-            urlsplit(target_url).scheme in ('http',):
+        if urlparse_cached(response).scheme in ('https', 'ftps') and \
+            urlparse_cached(request).scheme in ('http',):
                 return None
-        return referrer_url
+        stripped = self.strip_url_parsed(response)
+        if stripped is not None:
+            return urlunparse(stripped)
 
 
 class SameOriginPolicy(ReferrerPolicy):
@@ -132,12 +175,10 @@ class SameOriginPolicy(ReferrerPolicy):
     name = POLICY_SAME_ORIGIN
 
     def referrer(self, response, request):
-        target_url = request.url
-        referrer_source = response.url
-        if self.origin(referrer_source) == self.origin(target_url):
-            return self.strip_url(referrer_source)
-        else:
-            return None
+        if self.origin_parsed(response) == self.origin_parsed(request):
+            stripped = self.strip_url_parsed(response)
+            if stripped is not None:
+                return urlunparse(stripped)
 
 
 class OriginPolicy(ReferrerPolicy):

From f2ee6be3bb311bb8f89ccd929bc0ebd8412d0183 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 5 Oct 2016 18:33:31 +0200
Subject: [PATCH 1338/2974] Use urlparse_cached() for OriginPolicy

---
 scrapy/spidermiddlewares/referer.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index cbbcd3e97..bf2a3c037 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -193,7 +193,9 @@ class OriginPolicy(ReferrerPolicy):
     name = POLICY_ORIGIN
 
     def referrer(self, response, request):
-        return self.strip_url(response.url, origin_only=True)
+        stripped = self.strip_url_parsed(response, origin_only=True)
+        if stripped is not None:
+            return urlunparse(stripped)
 
 
 class OriginWhenCrossOriginPolicy(ReferrerPolicy):

From 59cb884ace1cb1c3339f1d0f05895501dd4d0447 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 5 Oct 2016 18:44:46 +0200
Subject: [PATCH 1339/2974] Use urlparse_cached() for
 OriginWhenCrossOriginPolicy

---
 scrapy/spidermiddlewares/referer.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index bf2a3c037..60f21ae84 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -193,9 +193,9 @@ class OriginPolicy(ReferrerPolicy):
     name = POLICY_ORIGIN
 
     def referrer(self, response, request):
-        stripped = self.strip_url_parsed(response, origin_only=True)
-        if stripped is not None:
-            return urlunparse(stripped)
+        origin = self.strip_url_parsed(response, origin_only=True)
+        if origin is not None:
+            return urlunparse(origin)
 
 
 class OriginWhenCrossOriginPolicy(ReferrerPolicy):
@@ -212,14 +212,13 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
     name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
 
     def referrer(self, response, request):
-        target_url = request.url
-        referrer_source = response.url
-        source_origin = self.origin(referrer_source)
-        if source_origin == self.origin(target_url):
-            return self.strip_url(referrer_source, origin_only=False)
+        origin = self.origin_parsed(response)
+        if origin == self.origin_parsed(request):
+            stripped = self.strip_url_parsed(response)
+            if stripped is not None:
+                return urlunparse(stripped)
         else:
-            return source_origin
-
+            return urlunparse(origin + ('', '', ''))
 
 
 class UnsafeUrlPolicy(ReferrerPolicy):

From f6a800fde67447d94480d4271de62f5e9a60bfb0 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 5 Oct 2016 18:55:35 +0200
Subject: [PATCH 1340/2974] Remove all non-cached urlparsing references

---
 scrapy/spidermiddlewares/referer.py | 71 ++++++-----------------------
 1 file changed, 14 insertions(+), 57 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 60f21ae84..46351576f 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -2,7 +2,7 @@
 RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
-from six.moves.urllib.parse import urlparse, urlunparse, ParseResult
+from six.moves.urllib.parse import ParseResult, urlunparse
 
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
@@ -28,7 +28,7 @@ class ReferrerPolicy(object):
     def referrer(self, response, request):
         raise NotImplementedError()
 
-    def strip_url_parsed(self, req_or_resp, origin_only=False):
+    def strip_url(self, req_or_resp, origin_only=False):
         """
         https://www.w3.org/TR/referrer-policy/#strip-url
 
@@ -68,53 +68,9 @@ class ReferrerPolicy(object):
                            '' if origin_only else parsed.query,
                            '')
 
-    def strip_url(self, url, origin_only=False):
-        """
-        https://www.w3.org/TR/referrer-policy/#strip-url
-
-        If url is null, return no referrer.
-        If url's scheme is a local scheme, then return no referrer.
-        Set url's username to the empty string.
-        Set url's password to null.
-        Set url's fragment to null.
-        If the origin-only flag is true, then:
-            Set url's path to null.
-            Set url's query to null.
-        Return url.
-        """
-        if url is None or not url:
-            return None
-        parsed = urlparse(url, allow_fragments=True)
-
-        if parsed.scheme in self.NOREFERRER_SCHEMES:
-            return None
-
-        netloc = parsed.netloc
-        # strip username and password if present
-        if parsed.username or parsed.password:
-            netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '')
-
-        # strip standard protocol numbers
-        # Note: strictly speaking, standard port numbers should only be
-        # stripped when comparing origins
-        if parsed.port:
-            if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)):
-                netloc = netloc.replace(':{p.port}'.format(p=parsed), '')
-
-        return urlunparse((
-            parsed.scheme,
-            netloc,
-            '/' if origin_only else parsed.path,
-            '' if origin_only else parsed.params,
-            '' if origin_only else parsed.query,
-            ''))
-
-    def origin(self, url):
-        return self.strip_url(url, origin_only=True)
-
-    def origin_parsed(self, req_or_resp):
+    def origin(self, req_or_resp):
         """Return (scheme, host, path) tuple for a request or response URL."""
-        return tuple(self.strip_url_parsed(req_or_resp, origin_only=True)[:3])
+        return tuple(self.strip_url(req_or_resp, origin_only=True)[:3])
 
 
 class NoReferrerPolicy(ReferrerPolicy):
@@ -157,7 +113,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
         if urlparse_cached(response).scheme in ('https', 'ftps') and \
             urlparse_cached(request).scheme in ('http',):
                 return None
-        stripped = self.strip_url_parsed(response)
+        stripped = self.strip_url(response)
         if stripped is not None:
             return urlunparse(stripped)
 
@@ -175,8 +131,8 @@ class SameOriginPolicy(ReferrerPolicy):
     name = POLICY_SAME_ORIGIN
 
     def referrer(self, response, request):
-        if self.origin_parsed(response) == self.origin_parsed(request):
-            stripped = self.strip_url_parsed(response)
+        if self.origin(response) == self.origin(request):
+            stripped = self.strip_url(response)
             if stripped is not None:
                 return urlunparse(stripped)
 
@@ -193,7 +149,7 @@ class OriginPolicy(ReferrerPolicy):
     name = POLICY_ORIGIN
 
     def referrer(self, response, request):
-        origin = self.strip_url_parsed(response, origin_only=True)
+        origin = self.strip_url(response, origin_only=True)
         if origin is not None:
             return urlunparse(origin)
 
@@ -212,9 +168,9 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
     name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
 
     def referrer(self, response, request):
-        origin = self.origin_parsed(response)
-        if origin == self.origin_parsed(request):
-            stripped = self.strip_url_parsed(response)
+        origin = self.origin(response)
+        if origin == self.origin(request):
+            stripped = self.strip_url(response)
             if stripped is not None:
                 return urlunparse(stripped)
         else:
@@ -237,8 +193,9 @@ class UnsafeUrlPolicy(ReferrerPolicy):
     name = POLICY_UNSAFE_URL
 
     def referrer(self, response, request):
-        referrer_source = response.url
-        return self.strip_url(referrer_source)
+        stripped = self.strip_url(response)
+        if stripped is not None:
+            return urlunparse(stripped)
 
 
 class LegacyPolicy(ReferrerPolicy):

From f6205778f31dae3ac0f60528a7aa4ef3ef6a181a Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 5 Oct 2016 19:06:26 +0200
Subject: [PATCH 1341/2974] Refactor ReferrerPolicy methods

---
 scrapy/spidermiddlewares/referer.py | 30 ++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 46351576f..44a599433 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -28,6 +28,16 @@ class ReferrerPolicy(object):
     def referrer(self, response, request):
         raise NotImplementedError()
 
+    def stripped_referrer(self, req_or_resp):
+        stripped = self.strip_url(req_or_resp)
+        if stripped is not None:
+            return urlunparse(stripped)
+
+    def origin_referrer(self, req_or_resp):
+        stripped = self.strip_url(req_or_resp, origin_only=True)
+        if stripped is not None:
+            return urlunparse(stripped)
+
     def strip_url(self, req_or_resp, origin_only=False):
         """
         https://www.w3.org/TR/referrer-policy/#strip-url
@@ -113,9 +123,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
         if urlparse_cached(response).scheme in ('https', 'ftps') and \
             urlparse_cached(request).scheme in ('http',):
                 return None
-        stripped = self.strip_url(response)
-        if stripped is not None:
-            return urlunparse(stripped)
+        return self.stripped_referrer(response)
 
 
 class SameOriginPolicy(ReferrerPolicy):
@@ -132,9 +140,7 @@ class SameOriginPolicy(ReferrerPolicy):
 
     def referrer(self, response, request):
         if self.origin(response) == self.origin(request):
-            stripped = self.strip_url(response)
-            if stripped is not None:
-                return urlunparse(stripped)
+            return self.stripped_referrer(response)
 
 
 class OriginPolicy(ReferrerPolicy):
@@ -149,9 +155,7 @@ class OriginPolicy(ReferrerPolicy):
     name = POLICY_ORIGIN
 
     def referrer(self, response, request):
-        origin = self.strip_url(response, origin_only=True)
-        if origin is not None:
-            return urlunparse(origin)
+        return self.origin_referrer(response)
 
 
 class OriginWhenCrossOriginPolicy(ReferrerPolicy):
@@ -170,9 +174,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
     def referrer(self, response, request):
         origin = self.origin(response)
         if origin == self.origin(request):
-            stripped = self.strip_url(response)
-            if stripped is not None:
-                return urlunparse(stripped)
+            return self.stripped_referrer(response)
         else:
             return urlunparse(origin + ('', '', ''))
 
@@ -193,9 +195,7 @@ class UnsafeUrlPolicy(ReferrerPolicy):
     name = POLICY_UNSAFE_URL
 
     def referrer(self, response, request):
-        stripped = self.strip_url(response)
-        if stripped is not None:
-            return urlunparse(stripped)
+        return self.stripped_referrer(response)
 
 
 class LegacyPolicy(ReferrerPolicy):

From 842ce131aa666dce0db61aeee701c420050a5a9c Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 11 Oct 2016 18:27:31 +0200
Subject: [PATCH 1342/2974] Make default referrer policy customizable via
 settings

---
 scrapy/settings/default_settings.py |  1 +
 scrapy/spidermiddlewares/referer.py | 18 +++++++++++++++---
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index a5931a3d5..15a134dd3 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -234,6 +234,7 @@ REDIRECT_MAX_TIMES = 20  # uses Firefox default setting
 REDIRECT_PRIORITY_ADJUST = +2
 
 REFERER_ENABLED = True
+REFERER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
 
 RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 44a599433..88041d8f0 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -8,6 +8,7 @@ from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.python import to_native_str
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.misc import load_object
 
 
 LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)
@@ -221,16 +222,27 @@ _policy_classes = {p.name: p for p in (
 
 class RefererMiddleware(object):
 
-    def __init__(self, policy_class=DefaultReferrerPolicy):
-        self.default_policy = policy_class
+    def __init__(self, settings={}):
+        policy = settings.get('REFERER_POLICY')
+        if policy is not None:
+            try:
+                self.default_policy = load_object(policy)
+            except ValueError:
+                try:
+                    self.default_policy = _policy_classes[policy]
+                except:
+                    raise NotConfigured("Unknown referrer policy name %r" % policy)
+        else:
+            self.default_policy = DefaultReferrerPolicy
 
     @classmethod
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('REFERER_ENABLED'):
             raise NotConfigured
-        return cls()
+        return cls(crawler.settings)
 
     def policy(self, response, request):
+        # policy set in request's meta dict takes precedence over default policy
         policy_name = request.meta.get('referrer_policy')
         if policy_name is None:
             policy_name = to_native_str(

From e72b6e33611f61dbb358f1ff97c5c195a7db696f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 11 Oct 2016 18:27:56 +0200
Subject: [PATCH 1343/2974] Add tests for referrer policy via settings and via
 Request meta

---
 tests/test_spidermiddleware_referer.py | 501 ++++++++++++++-----------
 1 file changed, 292 insertions(+), 209 deletions(-)

diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 8458fe90b..b724d7999 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,233 +1,316 @@
 from unittest import TestCase
 
+from scrapy.exceptions import NotConfigured
 from scrapy.http import Response, Request
+from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.spidermiddlewares.referer import RefererMiddleware, \
     POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \
     POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \
-    POLICY_UNSAFE_URL
+    POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \
+    DefaultReferrerPolicy, \
+    NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \
+    OriginWhenCrossOriginPolicy, OriginPolicy, \
+    SameOriginPolicy, UnsafeUrlPolicy
 
 
 class TestRefererMiddleware(TestCase):
 
+    req_meta = {}
+    resp_headers = {}
+    settings = {}
+    scenarii = [
+        ('http://scrapytest.org', 'http://scrapytest.org/',  b'http://scrapytest.org'),
+    ]
+
     def setUp(self):
         self.spider = Spider('foo')
-        self.mw = RefererMiddleware()
+        settings = Settings(self.settings)
+        self.mw = RefererMiddleware(settings)
 
-    def test_process_spider_output(self):
-        res = Response('http://scrapytest.org')
-        reqs = [Request('http://scrapytest.org/')]
+    def get_request(self, target):
+        return Request(target, meta=self.req_meta)
 
-        out = list(self.mw.process_spider_output(res, reqs, self.spider))
-        self.assertEquals(out[0].headers.get('Referer'),
-                          b'http://scrapytest.org')
+    def get_response(self, origin):
+        return Response(origin, headers=self.resp_headers)
 
-    def test_policy_default(self):
-        """
-        Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
+    def test(self):
 
-        with some additional filtering of s3://
-        """
-        for origin, target, referrer in [
-                ('https://example.com/',    'https://scrapy.org/',  b'https://example.com/'),
-                ('http://example.com/',     'http://scrapy.org/',   b'http://example.com/'),
-                ('http://example.com/',     'https://scrapy.org/',  b'http://example.com/'),
-                ('https://example.com/',    'http://scrapy.org/',   None),
-
-                # no credentials leak
-                ('http://user:password@example.com/',  'https://scrapy.org/', b'http://example.com/'),
-
-                # no referrer leak for local schemes
-                ('file:///home/path/to/somefile.html',  'https://scrapy.org/', None),
-                ('file:///home/path/to/somefile.html',  'http://scrapy.org/',  None),
-
-                # no referrer leak for s3 origins
-                ('s3://mybucket/path/to/data.csv',  'https://scrapy.org/', None),
-                ('s3://mybucket/path/to/data.csv',  'http://scrapy.org/',  None),
-            ]:
-            response = Response(origin)
-            request = Request(target)
+        for origin, target, referrer in self.scenarii:
+            response = self.get_response(origin)
+            request = self.get_request(target)
 
             out = list(self.mw.process_spider_output(response, [request], self.spider))
             self.assertEquals(out[0].headers.get('Referer'), referrer)
 
-    def test_policy_no_referrer(self):
 
-        for origin, target, referrer in [
-                ('https://example.com/page.html',       'https://example.com/', None),
-                ('http://www.example.com/',             'https://scrapy.org/',  None),
-                ('http://www.example.com/',             'http://scrapy.org/',   None),
-                ('https://www.example.com/',            'http://scrapy.org/',   None),
-                ('file:///home/path/to/somefile.html',  'http://scrapy.org/',   None),
+class MixinDefault(object):
+    """
+    Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
+
+    with some additional filtering of s3://
+    """
+    scenarii = [
+        ('https://example.com/',    'https://scrapy.org/',  b'https://example.com/'),
+        ('http://example.com/',     'http://scrapy.org/',   b'http://example.com/'),
+        ('http://example.com/',     'https://scrapy.org/',  b'http://example.com/'),
+        ('https://example.com/',    'http://scrapy.org/',   None),
+
+        # no credentials leak
+        ('http://user:password@example.com/',  'https://scrapy.org/', b'http://example.com/'),
+
+        # no referrer leak for local schemes
+        ('file:///home/path/to/somefile.html',  'https://scrapy.org/', None),
+        ('file:///home/path/to/somefile.html',  'http://scrapy.org/',  None),
+
+        # no referrer leak for s3 origins
+        ('s3://mybucket/path/to/data.csv',  'https://scrapy.org/', None),
+        ('s3://mybucket/path/to/data.csv',  'http://scrapy.org/',  None),
+    ]
+
+
+class MixinNoReferrer(object):
+    scenarii = [
+        ('https://example.com/page.html',       'https://example.com/', None),
+        ('http://www.example.com/',             'https://scrapy.org/',  None),
+        ('http://www.example.com/',             'http://scrapy.org/',   None),
+        ('https://www.example.com/',            'http://scrapy.org/',   None),
+        ('file:///home/path/to/somefile.html',  'http://scrapy.org/',   None),
+    ]
+
+
+class MixinNoReferrerWhenDowngrade(object):
+    scenarii = [
+        # TLS to TLS: send non-empty referrer
+        ('https://example.com/page.html',       'https://not.example.com/', b'https://example.com/page.html'),
+        ('https://example.com/page.html',       'https://scrapy.org/',      b'https://example.com/page.html'),
+        ('https://example.com:443/page.html',   'https://scrapy.org/',      b'https://example.com/page.html'),
+        ('https://example.com:444/page.html',   'https://scrapy.org/',      b'https://example.com:444/page.html'),
+        ('ftps://example.com/urls.zip',         'https://scrapy.org/',      b'ftps://example.com/urls.zip'),
+
+        # TLS to non-TLS: do not send referrer
+        ('https://example.com/page.html',       'http://not.example.com/',  None),
+        ('https://example.com/page.html',       'http://scrapy.org/',       None),
+        ('ftps://example.com/urls.zip',         'http://scrapy.org/',       None),
+
+        # non-TLS to TLS or non-TLS: send referrer
+        ('http://example.com/page.html',        'https://not.example.com/', b'http://example.com/page.html'),
+        ('http://example.com/page.html',        'https://scrapy.org/',      b'http://example.com/page.html'),
+        ('http://example.com:8080/page.html',   'https://scrapy.org/',      b'http://example.com:8080/page.html'),
+        ('http://example.com:80/page.html',     'http://not.example.com/',  b'http://example.com/page.html'),
+        ('http://example.com/page.html',        'http://scrapy.org/',       b'http://example.com/page.html'),
+        ('http://example.com:443/page.html',    'http://scrapy.org/',       b'http://example.com:443/page.html'),
+        ('ftp://example.com/urls.zip',          'http://scrapy.org/',       b'ftp://example.com/urls.zip'),
+        ('ftp://example.com/urls.zip',          'https://scrapy.org/',      b'ftp://example.com/urls.zip'),
+
+        # test for user/password stripping
+        ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
+    ]
+
+
+class MixinSameOrigin(object):
+    scenarii = [
+        # Same origin (protocol, host, port): send referrer
+        ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
+        ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
+        ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
+        ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
+        ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
+        ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+
+        # Different host: do NOT send referrer
+        ('https://example.com/page.html',       'https://not.example.com/otherpage.html',   None),
+        ('http://example.com/page.html',        'http://not.example.com/otherpage.html',    None),
+        ('http://example.com/page.html',        'http://www.example.com/otherpage.html',    None),
+
+        # Different port: do NOT send referrer
+        ('https://example.com:444/page.html',   'https://example.com/not-page.html',    None),
+        ('http://example.com:81/page.html',     'http://example.com/not-page.html',     None),
+        ('http://example.com/page.html',        'http://example.com:81/not-page.html',  None),
+
+        # Different protocols: do NOT send refferer
+        ('https://example.com/page.html',   'http://example.com/not-page.html',     None),
+        ('https://example.com/page.html',   'http://not.example.com/',              None),
+        ('ftps://example.com/urls.zip',     'https://example.com/not-page.html',    None),
+        ('ftp://example.com/urls.zip',      'http://example.com/not-page.html',     None),
+        ('ftps://example.com/urls.zip',     'https://example.com/not-page.html',    None),
+
+        # test for user/password stripping
+        ('https://user:password@example.com/page.html', 'https://example.com/not-page.html',    b'https://example.com/page.html'),
+        ('https://user:password@example.com/page.html', 'http://example.com/not-page.html',     None),
+    ]
+
+
+class MixinOrigin(object):
+    scenarii = [
+        # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)
+        ('https://example.com/page.html',   'https://example.com/not-page.html',    b'https://example.com/'),
+        ('https://example.com/page.html',   'https://scrapy.org',                   b'https://example.com/'),
+        ('https://example.com/page.html',   'http://scrapy.org',                    b'https://example.com/'),
+        ('http://example.com/page.html',    'http://scrapy.org',                    b'http://example.com/'),
+
+        # test for user/password stripping
+        ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
+    ]
+
+
+class MixinOriginWhenCrossOrigin(object):
+    scenarii = [
+        # Same origin (protocol, host, port): send referrer
+        ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
+        ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
+        ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
+        ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
+        ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
+        ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+
+        # Different host: send origin as referrer
+        ('https://example2.com/page.html',  'https://scrapy.org/otherpage.html',        b'https://example2.com/'),
+        ('https://example2.com/page.html',  'https://not.example2.com/otherpage.html',  b'https://example2.com/'),
+        ('http://example2.com/page.html',   'http://not.example2.com/otherpage.html',   b'http://example2.com/'),
+        # exact match required
+        ('http://example2.com/page.html',   'http://www.example2.com/otherpage.html',   b'http://example2.com/'),
+
+        # Different port: send origin as referrer
+        ('https://example3.com:444/page.html',  'https://example3.com/not-page.html',   b'https://example3.com:444/'),
+        ('http://example3.com:81/page.html',    'http://example3.com/not-page.html',    b'http://example3.com:81/'),
+
+        # Different protocols: send origin as referrer
+        ('https://example4.com/page.html',  'http://example4.com/not-page.html',    b'https://example4.com/'),
+        ('https://example4.com/page.html',  'http://not.example4.com/',             b'https://example4.com/'),
+        ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
+        ('ftp://example4.com/urls.zip',     'http://example4.com/not-page.html',    b'ftp://example4.com/'),
+        ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
+
+        # test for user/password stripping
+        ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html',  b'https://example5.com/page.html'),
+        # TLS to non-TLS downgrade: send origin
+        ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html',   b'https://example5.com/'),
+    ]
+
+
+class MixinUnsafeUrl(object):
+    scenarii = [
+        # TLS to TLS: send referrer
+        ('https://example.com/sekrit.html',     'http://not.example.com/',      b'https://example.com/sekrit.html'),
+        ('https://example1.com/page.html',      'https://not.example1.com/',    b'https://example1.com/page.html'),
+        ('https://example1.com/page.html',      'https://scrapy.org/',          b'https://example1.com/page.html'),
+        ('https://example1.com:443/page.html',  'https://scrapy.org/',          b'https://example1.com/page.html'),
+        ('https://example1.com:444/page.html',  'https://scrapy.org/',          b'https://example1.com:444/page.html'),
+        ('ftps://example1.com/urls.zip',        'https://scrapy.org/',          b'ftps://example1.com/urls.zip'),
+
+        # TLS to non-TLS: send referrer (yes, it's unsafe)
+        ('https://example2.com/page.html',  'http://not.example2.com/', b'https://example2.com/page.html'),
+        ('https://example2.com/page.html',  'http://scrapy.org/',       b'https://example2.com/page.html'),
+        ('ftps://example2.com/urls.zip',    'http://scrapy.org/',       b'ftps://example2.com/urls.zip'),
+
+        # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)
+        ('http://example3.com/page.html',       'https://not.example3.com/',    b'http://example3.com/page.html'),
+        ('http://example3.com/page.html',       'https://scrapy.org/',          b'http://example3.com/page.html'),
+        ('http://example3.com:8080/page.html',  'https://scrapy.org/',          b'http://example3.com:8080/page.html'),
+        ('http://example3.com:80/page.html',    'http://not.example3.com/',     b'http://example3.com/page.html'),
+        ('http://example3.com/page.html',       'http://scrapy.org/',           b'http://example3.com/page.html'),
+        ('http://example3.com:443/page.html',   'http://scrapy.org/',           b'http://example3.com:443/page.html'),
+        ('ftp://example3.com/urls.zip',         'http://scrapy.org/',           b'ftp://example3.com/urls.zip'),
+        ('ftp://example3.com/urls.zip',         'https://scrapy.org/',          b'ftp://example3.com/urls.zip'),
+
+        # test for user/password stripping
+        ('http://user:password@example4.com/page.html',     'https://not.example4.com/',    b'http://example4.com/page.html'),
+        ('https://user:password@example4.com/page.html',    'http://scrapy.org/',           b'https://example4.com/page.html'),
+    ]
+
+
+class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware):
+    pass
+
+
+# --- Tests using settings to set policy using class path
+class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'}
+
+
+class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
+
+
+class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
+
+
+class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}
+
+
+class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+
+
+class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
+
+
+# --- Tests using Request meta dict to set policy
+class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT}
+
+
+class TestRefererMiddlewareNoReferrer(MixinNoReferrer, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_NO_REFERRER}
+
+
+class TestRefererMiddlewareNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}
+
+
+class TestRefererMiddlewareSameOrigin(MixinSameOrigin, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_SAME_ORIGIN}
+
+
+class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_ORIGIN}
+
+
+class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
+
+
+class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
+
+
+
+class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
+    req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
+
+
+class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
+    req_meta = {'referrer_policy': POLICY_NO_REFERRER}
+
+
+class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+    req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
+
+
+class TestRefererMiddlewareSettingsPolicyByName(TestCase):
+
+    def test_valid_name(self):
+        for s, p in [
+                (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
+                (POLICY_NO_REFERRER, NoReferrerPolicy),
+                (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
+                (POLICY_SAME_ORIGIN, SameOriginPolicy),
+                (POLICY_ORIGIN, OriginPolicy),
+                (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
+                (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
             ]:
-            response = Response(origin)
-            request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER})
+            settings = Settings({'REFERER_POLICY': s})
+            mw = RefererMiddleware(settings)
+            self.assertEquals(mw.default_policy, p)
 
-            out = list(self.mw.process_spider_output(response, [request], self.spider))
-            self.assertEquals(out[0].headers.get('Referer'), referrer)
-
-    def test_policy_no_referrer_when_downgrade(self):
-
-        for origin, target, referrer in [
-                # TLS to TLS: send non-empty referrer
-                ('https://example.com/page.html',       'https://not.example.com/', b'https://example.com/page.html'),
-                ('https://example.com/page.html',       'https://scrapy.org/',      b'https://example.com/page.html'),
-                ('https://example.com:443/page.html',   'https://scrapy.org/',      b'https://example.com/page.html'),
-                ('https://example.com:444/page.html',   'https://scrapy.org/',      b'https://example.com:444/page.html'),
-                ('ftps://example.com/urls.zip',         'https://scrapy.org/',      b'ftps://example.com/urls.zip'),
-
-                # TLS to non-TLS: do not send referrer
-                ('https://example.com/page.html',       'http://not.example.com/',  None),
-                ('https://example.com/page.html',       'http://scrapy.org/',       None),
-                ('ftps://example.com/urls.zip',         'http://scrapy.org/',       None),
-
-                # non-TLS to TLS or non-TLS: send referrer
-                ('http://example.com/page.html',        'https://not.example.com/', b'http://example.com/page.html'),
-                ('http://example.com/page.html',        'https://scrapy.org/',      b'http://example.com/page.html'),
-                ('http://example.com:8080/page.html',   'https://scrapy.org/',      b'http://example.com:8080/page.html'),
-                ('http://example.com:80/page.html',     'http://not.example.com/',  b'http://example.com/page.html'),
-                ('http://example.com/page.html',        'http://scrapy.org/',       b'http://example.com/page.html'),
-                ('http://example.com:443/page.html',    'http://scrapy.org/',       b'http://example.com:443/page.html'),
-                ('ftp://example.com/urls.zip',          'http://scrapy.org/',       b'ftp://example.com/urls.zip'),
-                ('ftp://example.com/urls.zip',          'https://scrapy.org/',      b'ftp://example.com/urls.zip'),
-
-                # test for user/password stripping
-                ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
-            ]:
-            response = Response(origin)
-            request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE})
-
-            out = list(self.mw.process_spider_output(response, [request], self.spider))
-            self.assertEquals(out[0].headers.get('Referer'), referrer)
-
-    def test_policy_same_origin(self):
-
-        for origin, target, referrer in [
-                # Same origin (protocol, host, port): send referrer
-                ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
-                ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
-                ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
-                ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
-                ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
-                ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
-
-                # Different host: do NOT send referrer
-                ('https://example.com/page.html',       'https://not.example.com/otherpage.html',   None),
-                ('http://example.com/page.html',        'http://not.example.com/otherpage.html',    None),
-                ('http://example.com/page.html',        'http://www.example.com/otherpage.html',    None),
-
-                # Different port: do NOT send referrer
-                ('https://example.com:444/page.html',   'https://example.com/not-page.html',    None),
-                ('http://example.com:81/page.html',     'http://example.com/not-page.html',     None),
-                ('http://example.com/page.html',        'http://example.com:81/not-page.html',  None),
-
-                # Different protocols: do NOT send refferer
-                ('https://example.com/page.html',   'http://example.com/not-page.html',     None),
-                ('https://example.com/page.html',   'http://not.example.com/',              None),
-                ('ftps://example.com/urls.zip',     'https://example.com/not-page.html',    None),
-                ('ftp://example.com/urls.zip',      'http://example.com/not-page.html',     None),
-                ('ftps://example.com/urls.zip',     'https://example.com/not-page.html',    None),
-
-                # test for user/password stripping
-                ('https://user:password@example.com/page.html', 'https://example.com/not-page.html',    b'https://example.com/page.html'),
-                ('https://user:password@example.com/page.html', 'http://example.com/not-page.html',     None),
-            ]:
-            response = Response(origin)
-            request = Request(target, meta={'referrer_policy': POLICY_SAME_ORIGIN})
-
-            out = list(self.mw.process_spider_output(response, [request], self.spider))
-            self.assertEquals(out[0].headers.get('Referer'), referrer)
-
-    def test_policy_origin(self):
-
-        for origin, target, referrer in [
-                # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)
-                ('https://example.com/page.html',   'https://example.com/not-page.html',    b'https://example.com/'),
-                ('https://example.com/page.html',   'https://scrapy.org',                   b'https://example.com/'),
-                ('https://example.com/page.html',   'http://scrapy.org',                    b'https://example.com/'),
-                ('http://example.com/page.html',    'http://scrapy.org',                    b'http://example.com/'),
-
-                # test for user/password stripping
-                ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
-            ]:
-            response = Response(origin)
-            request = Request(target, meta={'referrer_policy': POLICY_ORIGIN})
-
-            out = list(self.mw.process_spider_output(response, [request], self.spider))
-            self.assertEquals(out[0].headers.get('Referer'), referrer)
-
-    def test_policy_origin_when_cross_origin(self):
-
-        for origin, target, referrer in [
-                # Same origin (protocol, host, port): send referrer
-                ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
-                ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
-                ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
-                ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
-                ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
-                ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
-
-                # Different host: send origin as referrer
-                ('https://example2.com/page.html',  'https://scrapy.org/otherpage.html',        b'https://example2.com/'),
-                ('https://example2.com/page.html',  'https://not.example2.com/otherpage.html',  b'https://example2.com/'),
-                ('http://example2.com/page.html',   'http://not.example2.com/otherpage.html',   b'http://example2.com/'),
-                # exact match required
-                ('http://example2.com/page.html',   'http://www.example2.com/otherpage.html',   b'http://example2.com/'),
-
-                # Different port: send origin as referrer
-                ('https://example3.com:444/page.html',  'https://example3.com/not-page.html',   b'https://example3.com:444/'),
-                ('http://example3.com:81/page.html',    'http://example3.com/not-page.html',    b'http://example3.com:81/'),
-
-                # Different protocols: send origin as referrer
-                ('https://example4.com/page.html',  'http://example4.com/not-page.html',    b'https://example4.com/'),
-                ('https://example4.com/page.html',  'http://not.example4.com/',             b'https://example4.com/'),
-                ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
-                ('ftp://example4.com/urls.zip',     'http://example4.com/not-page.html',    b'ftp://example4.com/'),
-                ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
-
-                # test for user/password stripping
-                ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html',  b'https://example5.com/page.html'),
-                # TLS to non-TLS downgrade: send origin
-                ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html',   b'https://example5.com/'),
-            ]:
-            response = Response(origin)
-            request = Request(target, meta={'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN})
-
-            out = list(self.mw.process_spider_output(response, [request], self.spider))
-            self.assertEquals(out[0].headers.get('Referer'), referrer)
-
-    def test_policy_unsafe_url(self):
-
-        for origin, target, referrer in [
-                # TLS to TLS: send referrer
-                ('https://example.com/sekrit.html',     'http://not.example.com/',      b'https://example.com/sekrit.html'),
-                ('https://example1.com/page.html',      'https://not.example1.com/',    b'https://example1.com/page.html'),
-                ('https://example1.com/page.html',      'https://scrapy.org/',          b'https://example1.com/page.html'),
-                ('https://example1.com:443/page.html',  'https://scrapy.org/',          b'https://example1.com/page.html'),
-                ('https://example1.com:444/page.html',  'https://scrapy.org/',          b'https://example1.com:444/page.html'),
-                ('ftps://example1.com/urls.zip',        'https://scrapy.org/',          b'ftps://example1.com/urls.zip'),
-
-                # TLS to non-TLS: send referrer (yes, it's unsafe)
-                ('https://example2.com/page.html',  'http://not.example2.com/', b'https://example2.com/page.html'),
-                ('https://example2.com/page.html',  'http://scrapy.org/',       b'https://example2.com/page.html'),
-                ('ftps://example2.com/urls.zip',    'http://scrapy.org/',       b'ftps://example2.com/urls.zip'),
-
-                # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)
-                ('http://example3.com/page.html',       'https://not.example3.com/',    b'http://example3.com/page.html'),
-                ('http://example3.com/page.html',       'https://scrapy.org/',          b'http://example3.com/page.html'),
-                ('http://example3.com:8080/page.html',  'https://scrapy.org/',          b'http://example3.com:8080/page.html'),
-                ('http://example3.com:80/page.html',    'http://not.example3.com/',     b'http://example3.com/page.html'),
-                ('http://example3.com/page.html',       'http://scrapy.org/',           b'http://example3.com/page.html'),
-                ('http://example3.com:443/page.html',   'http://scrapy.org/',           b'http://example3.com:443/page.html'),
-                ('ftp://example3.com/urls.zip',         'http://scrapy.org/',           b'ftp://example3.com/urls.zip'),
-                ('ftp://example3.com/urls.zip',         'https://scrapy.org/',          b'ftp://example3.com/urls.zip'),
-
-                # test for user/password stripping
-                ('http://user:password@example4.com/page.html',     'https://not.example4.com/',    b'http://example4.com/page.html'),
-                ('https://user:password@example4.com/page.html',    'http://scrapy.org/',           b'https://example4.com/page.html'),
-            ]:
-            response = Response(origin)
-            request = Request(target, meta={'referrer_policy': POLICY_UNSAFE_URL})
-
-            out = list(self.mw.process_spider_output(response, [request], self.spider))
-            self.assertEquals(out[0].headers.get('Referer'), referrer)
+    def test_invalid_name(self):
+        settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'})
+        with self.assertRaises(NotConfigured):
+            mw = RefererMiddleware(settings)

From 0344f57fefc0877bf9048a084002fc719335e31c Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 11 Oct 2016 19:53:15 +0200
Subject: [PATCH 1344/2974] Support case-insensitive policy names in settings

---
 scrapy/spidermiddlewares/referer.py    |  2 +-
 tests/test_spidermiddleware_referer.py | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 88041d8f0..deda7b284 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -229,7 +229,7 @@ class RefererMiddleware(object):
                 self.default_policy = load_object(policy)
             except ValueError:
                 try:
-                    self.default_policy = _policy_classes[policy]
+                    self.default_policy = _policy_classes[policy.lower()]
                 except:
                     raise NotConfigured("Unknown referrer policy name %r" % policy)
         else:
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index b724d7999..b1ab366a7 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -310,6 +310,20 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):
             mw = RefererMiddleware(settings)
             self.assertEquals(mw.default_policy, p)
 
+    def test_valid_name_casevariants(self):
+        for s, p in [
+                (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
+                (POLICY_NO_REFERRER, NoReferrerPolicy),
+                (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
+                (POLICY_SAME_ORIGIN, SameOriginPolicy),
+                (POLICY_ORIGIN, OriginPolicy),
+                (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
+                (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
+            ]:
+            settings = Settings({'REFERER_POLICY': s.upper()})
+            mw = RefererMiddleware(settings)
+            self.assertEquals(mw.default_policy, p)
+
     def test_invalid_name(self):
         settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'})
         with self.assertRaises(NotConfigured):

From ec8b4c1a9bfd004960d730b41078b9f0633627d8 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 11 Oct 2016 20:00:34 +0200
Subject: [PATCH 1345/2974] Change __init__ default "settings" arg handling

---
 scrapy/spidermiddlewares/referer.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index deda7b284..2a3790bde 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -222,18 +222,18 @@ _policy_classes = {p.name: p for p in (
 
 class RefererMiddleware(object):
 
-    def __init__(self, settings={}):
-        policy = settings.get('REFERER_POLICY')
-        if policy is not None:
-            try:
-                self.default_policy = load_object(policy)
-            except ValueError:
+    def __init__(self, settings=None):
+        self.default_policy = DefaultReferrerPolicy
+        if settings is not None:
+            policy = settings.get('REFERER_POLICY')
+            if policy is not None:
                 try:
-                    self.default_policy = _policy_classes[policy.lower()]
-                except:
-                    raise NotConfigured("Unknown referrer policy name %r" % policy)
-        else:
-            self.default_policy = DefaultReferrerPolicy
+                    self.default_policy = load_object(policy)
+                except ValueError:
+                    try:
+                        self.default_policy = _policy_classes[policy.lower()]
+                    except:
+                        raise NotConfigured("Unknown referrer policy name %r" % policy)
 
     @classmethod
     def from_crawler(cls, crawler):

From e50e670eff2fe1b109f6b5dd026c4c706a93585c Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 12 Oct 2016 16:16:53 +0200
Subject: [PATCH 1346/2974] Add test for custom referrer policy via settings

---
 scrapy/spidermiddlewares/referer.py    |  4 ++++
 tests/test_spidermiddleware_referer.py | 30 ++++++++++++++++++++++++--
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 2a3790bde..0bba63cb7 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -227,9 +227,12 @@ class RefererMiddleware(object):
         if settings is not None:
             policy = settings.get('REFERER_POLICY')
             if policy is not None:
+                # expect a string for the path to the policy class
                 try:
                     self.default_policy = load_object(policy)
                 except ValueError:
+                    # otherwise try to interpret the string as standard
+                    # https://www.w3.org/TR/referrer-policy/#referrer-policies
                     try:
                         self.default_policy = _policy_classes[policy.lower()]
                     except:
@@ -239,6 +242,7 @@ class RefererMiddleware(object):
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('REFERER_ENABLED'):
             raise NotConfigured
+
         return cls(crawler.settings)
 
     def policy(self, response, request):
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index b1ab366a7..cfc4b5296 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -11,7 +11,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \
     DefaultReferrerPolicy, \
     NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \
     OriginWhenCrossOriginPolicy, OriginPolicy, \
-    SameOriginPolicy, UnsafeUrlPolicy
+    SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy
 
 
 class TestRefererMiddleware(TestCase):
@@ -249,6 +249,33 @@ class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddlewa
     settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
 
 
+class CustomPythonOrgPolicy(ReferrerPolicy):
+    """
+    A dummy policy that returns referrer as http(s)://python.org
+    depending on the scheme of the target URL.
+    """
+    def referrer(self, response, request):
+        from scrapy.utils.httpobj import urlparse_cached
+
+        scheme = urlparse_cached(request).scheme
+        if scheme == 'https':
+            return b'https://python.org/'
+        elif scheme == 'http':
+            return b'http://python.org/'
+
+
+class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}
+    scenarii = [
+        ('https://example.com/',    'https://scrapy.org/',  b'https://python.org/'),
+        ('http://example.com/',     'http://scrapy.org/',   b'http://python.org/'),
+        ('http://example.com/',     'https://scrapy.org/',  b'https://python.org/'),
+        ('https://example.com/',    'http://scrapy.org/',   b'http://python.org/'),
+        ('file:///home/path/to/somefile.html',  'https://scrapy.org/', b'https://python.org/'),
+        ('file:///home/path/to/somefile.html',  'http://scrapy.org/',  b'http://python.org/'),
+
+    ]
+
 # --- Tests using Request meta dict to set policy
 class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT}
@@ -278,7 +305,6 @@ class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
 
 
-
 class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}

From d3d4d66ce8e5d01aa1fd6013a6d63337bb931460 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 12 Oct 2016 16:30:25 +0200
Subject: [PATCH 1347/2974] Add tests for referrer-policy set in response HTTP
 headers

---
 tests/test_spidermiddleware_referer.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index cfc4b5296..9555817d9 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -354,3 +354,16 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):
         settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'})
         with self.assertRaises(NotConfigured):
             mw = RefererMiddleware(settings)
+
+
+class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
+    resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}
+
+class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
+    resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}
+
+class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+    resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}

From 285d5bc03a7ebe8eaa558a5c24ff0693353c3e87 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 12 Oct 2016 17:34:12 +0200
Subject: [PATCH 1348/2974] Patch "Referer" header on HTTP redirects if
 necessary

---
 scrapy/spidermiddlewares/referer.py | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 0bba63cb7..01f1fdf85 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -4,8 +4,9 @@ originated it.
 """
 from six.moves.urllib.parse import ParseResult, urlunparse
 
-from scrapy.http import Request
+from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
+from scrapy import signals
 from scrapy.utils.python import to_native_str
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
@@ -242,8 +243,9 @@ class RefererMiddleware(object):
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('REFERER_ENABLED'):
             raise NotConfigured
-
-        return cls(crawler.settings)
+        mw = cls(crawler.settings)
+        crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled)
+        return mw
 
     def policy(self, response, request):
         # policy set in request's meta dict takes precedence over default policy
@@ -264,3 +266,18 @@ class RefererMiddleware(object):
             return r
         return (_set_referer(r) for r in result or ())
 
+    def request_scheduled(self, request, spider):
+        # check redirected request to patch "Referer" header if necessary
+        redirected_urls = request.meta.get('redirect_urls', [])
+        if redirected_urls:
+            request_referrer = request.headers.get('Referer')
+            # we don't patch the referrer value if there is none
+            if request_referrer is not None:
+                faked_response = Response(redirected_urls[0])
+                policy_referrer = self.policy(faked_response,
+                    request).referrer(faked_response, request)
+                if policy_referrer != request_referrer:
+                    if policy_referrer is None:
+                        request.headers.pop('Referer')
+                    else:
+                        request.headers['Referer'] = policy_referrer

From c9c59db489575131d573e130015fd1fb6b133882 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 12 Oct 2016 18:29:47 +0200
Subject: [PATCH 1349/2974] Update documentation about REFERER_POLICY setting

---
 docs/topics/spider-middleware.rst | 38 ++++++++++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 8360827e8..a9d3d4568 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -95,7 +95,7 @@ following methods:
         it has processed the response.
 
         :meth:`process_spider_output` must return an iterable of
-        :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` 
+        :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item`
         objects.
 
         :param response: the response which generated this output from the
@@ -328,6 +328,42 @@ Default: ``True``
 
 Whether to enable referer middleware.
 
+.. setting:: REFERER_POLICY
+
+REFERER_POLICY
+^^^^^^^^^^^^^^
+
+.. versionadded:: 1.3
+
+Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
+
+`Referrer Policy`_ to apply when populating Request "Referer" header.
+
+This setting accepts:
+
+- a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass,
+  either a custom one or one of the built-in ones
+  (see ``scrapy.spidermiddlewares.referer``),
+- or one of the standard W3C-defined string values, i.e. ``"no-referrer"``,
+  ``"no-referrer-when-downgrade"``, ``"same-origin"``, ``"origin"``,
+  ``"origin-when-cross-origin"`` or ``"unsafe-url"``.
+  (It can also be the non-standard value ``"scrapy-default"`` to use
+  Scrapy's default referrer policy.)
+
+Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_,
+with the addition that "Referrer" is not sent if the parent request was
+using ``file://`` or ``s3://`` scheme.
+
+.. warning::
+    By default, Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_,
+    will send a non-empty "Referer" header from any ``https://`` to any ``https://`` URL,
+    even if the domain is different.
+    ``same-origin`` may be a better choice if you want to remove referrer
+    information for cross-domain requests.
+
+.. _Referrer Policy: https://www.w3.org/TR/referrer-policy
+.. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
+
 UrlLengthMiddleware
 -------------------
 

From 5dd7311cd48e676147138746229d7ab2b429b8a9 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 19 Oct 2016 14:45:33 +0200
Subject: [PATCH 1350/2974] Move URL credentials stripping to a helper function

---
 scrapy/spidermiddlewares/referer.py | 43 +++-----------
 scrapy/utils/url.py                 | 34 ++++++++++-
 tests/test_utils_url.py             | 92 ++++++++++++++++++++++++++++-
 3 files changed, 133 insertions(+), 36 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 01f1fdf85..e40e798b8 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -2,14 +2,13 @@
 RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
-from six.moves.urllib.parse import ParseResult, urlunparse
-
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
 from scrapy.utils.python import to_native_str
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
+from scrapy.utils.url import strip_url_credentials
 
 
 LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)
@@ -31,14 +30,10 @@ class ReferrerPolicy(object):
         raise NotImplementedError()
 
     def stripped_referrer(self, req_or_resp):
-        stripped = self.strip_url(req_or_resp)
-        if stripped is not None:
-            return urlunparse(stripped)
+        return self.strip_url(req_or_resp)
 
     def origin_referrer(self, req_or_resp):
-        stripped = self.strip_url(req_or_resp, origin_only=True)
-        if stripped is not None:
-            return urlunparse(stripped)
+        return self.strip_url(req_or_resp, origin_only=True)
 
     def strip_url(self, req_or_resp, origin_only=False):
         """
@@ -56,33 +51,13 @@ class ReferrerPolicy(object):
         """
         if req_or_resp.url is None or not req_or_resp.url:
             return None
-        parsed = urlparse_cached(req_or_resp)
-
-        if parsed.scheme in self.NOREFERRER_SCHEMES:
-            return None
-
-        netloc = parsed.netloc
-        # strip username and password if present
-        if parsed.username or parsed.password:
-            netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '')
-
-        # strip standard protocol numbers
-        # Note: strictly speaking, standard port numbers should only be
-        # stripped when comparing origins
-        if parsed.port:
-            if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)):
-                netloc = netloc.replace(':{p.port}'.format(p=parsed), '')
-
-        return ParseResult(parsed.scheme,
-                           netloc,
-                           '/' if origin_only else parsed.path,
-                           '' if origin_only else parsed.params,
-                           '' if origin_only else parsed.query,
-                           '')
+        parsed_url = urlparse_cached(req_or_resp)
+        if parsed_url.scheme not in self.NOREFERRER_SCHEMES:
+            return strip_url_credentials(parsed_url, origin_only=origin_only)
 
     def origin(self, req_or_resp):
-        """Return (scheme, host, path) tuple for a request or response URL."""
-        return tuple(self.strip_url(req_or_resp, origin_only=True)[:3])
+        """Return serialized origin (scheme, host, path) for a request or response URL."""
+        return self.strip_url(req_or_resp, origin_only=True)
 
 
 class NoReferrerPolicy(ReferrerPolicy):
@@ -178,7 +153,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
         if origin == self.origin(request):
             return self.stripped_referrer(response)
         else:
-            return urlunparse(origin + ('', '', ''))
+            return origin
 
 
 class UnsafeUrlPolicy(ReferrerPolicy):
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index dc1cce4ac..f3ccfb0e8 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead.
 """
 import posixpath
 import re
-from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse)
+from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse)
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this
 # move doesn't break old code
@@ -103,3 +103,35 @@ def guess_scheme(url):
         return any_to_uri(url)
     else:
         return add_http_if_no_scheme(url)
+
+
+def strip_url_credentials(url, origin_only=False, keep_fragments=False):
+
+    if url is None:
+        return None
+
+    if not isinstance(url, ParseResult):
+        parsed_url = urlparse(url)
+    else:
+        parsed_url = url
+
+    netloc = parsed_url.netloc
+    # strip username and password if present
+    if parsed_url.username or parsed_url.password:
+        netloc = netloc.split('@')[-1]
+
+    # strip standard protocol numbers
+    # Note: strictly speaking, standard port numbers should only be
+    # stripped when comparing origins
+    if parsed_url.port:
+        if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443)):
+            netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '')
+
+    return urlunparse((
+        parsed_url.scheme,
+        netloc,
+        '/' if origin_only else parsed_url.path,
+        '' if origin_only else parsed_url.params,
+        '' if origin_only else parsed_url.query,
+        '' if not keep_fragments else parsed_url.fragment
+    ))
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index f46d1d927..f1a5c3196 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -6,7 +6,8 @@ from six.moves.urllib.parse import urlparse
 
 from scrapy.spiders import Spider
 from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
-                              add_http_if_no_scheme, guess_scheme, parse_url)
+                              add_http_if_no_scheme, guess_scheme,
+                              parse_url, strip_url_credentials)
 
 __doctests__ = ['scrapy.utils.url']
 
@@ -241,5 +242,94 @@ for k, args in enumerate ([
     setattr (GuessSchemeTest, t_method.__name__, t_method)
 
 
+class StripUrlCredentials(unittest.TestCase):
+
+    def test_noop(self):
+        self.assertEqual(strip_url_credentials(
+            'http://www.example.com/index.html'),
+            'http://www.example.com/index.html')
+
+    def test_noop_query_string(self):
+        self.assertEqual(strip_url_credentials(
+            'http://www.example.com/index.html?somekey=somevalue'),
+            'http://www.example.com/index.html?somekey=somevalue')
+
+    def test_fragments(self):
+        self.assertEqual(strip_url_credentials(
+            'http://www.example.com/index.html?somekey=somevalue#section', keep_fragments=True),
+            'http://www.example.com/index.html?somekey=somevalue#section')
+
+    def test_noop_trailing_path(self):
+        self.assertEqual(strip_url_credentials(
+            'http://www.example.com/'),
+            'http://www.example.com/')
+
+    def test_noop_trailing_path2(self):
+        self.assertEqual(strip_url_credentials(
+            'http://www.example.com'),
+            'http://www.example.com')
+
+    def test_trailing_path_origin(self):
+        self.assertEqual(strip_url_credentials(
+            'http://www.example.com', origin_only=True),
+            'http://www.example.com/')
+
+    def test_username(self):
+        # username is stripped (and fragment too)
+        self.assertEqual(strip_url_credentials(
+            'http://username@www.example.com/index.html?somekey=somevalue#section'),
+            'http://www.example.com/index.html?somekey=somevalue')
+
+    def test_username_empty_pass(self):
+        # same as above
+        self.assertEqual(strip_url_credentials(
+            'https://username:@www.example.com/index.html?somekey=somevalue#section'),
+            'https://www.example.com/index.html?somekey=somevalue')
+
+    def test_username_password(self):
+        self.assertEqual(strip_url_credentials(
+            'ftp://username:password@www.example.com/index.html?somekey=somevalue#section'),
+            'ftp://www.example.com/index.html?somekey=somevalue')
+
+    def test_default_http_port(self):
+        self.assertEqual(strip_url_credentials(
+            'http://username:password@www.example.com:80/index.html'),
+            'http://www.example.com/index.html')
+
+    def test_non_default_http_port(self):
+        self.assertEqual(strip_url_credentials(
+            'http://username:password@www.example.com:8080/index.html'),
+            'http://www.example.com:8080/index.html')
+
+    def test_default_https_port(self):
+        self.assertEqual(strip_url_credentials(
+            'https://username:password@www.example.com:443/index.html'),
+            'https://www.example.com/index.html')
+
+    def test_non_default_https_port(self):
+        self.assertEqual(strip_url_credentials(
+            'https://username:password@www.example.com:442/index.html'),
+            'https://www.example.com:442/index.html')
+
+    def test_origin_only(self):
+        self.assertEqual(strip_url_credentials(
+            'http://username:password@www.example.com/index.html', origin_only=True),
+            'http://www.example.com/')
+
+    def test_default_http_port_origin_only(self):
+        self.assertEqual(strip_url_credentials(
+            'http://username:password@www.example.com:80/index.html', origin_only=True),
+            'http://www.example.com/')
+
+    def test_non_default_http_port_origin_only(self):
+        self.assertEqual(strip_url_credentials(
+            'http://username:password@www.example.com:8008/index.html', origin_only=True),
+            'http://www.example.com:8008/')
+
+    def test_default_https_port_origin_only(self):
+        self.assertEqual(strip_url_credentials(
+            'https://username:password@www.example.com:443/index.html', origin_only=True),
+            'https://www.example.com/')
+
 if __name__ == "__main__":
     unittest.main()

From 8864d0e8c19d0ddb52175be1952a4013b13cd86f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 24 Oct 2016 18:21:49 +0200
Subject: [PATCH 1351/2974] Rename helper function to strip_url() + add more
 tests

---
 scrapy/spidermiddlewares/referer.py |   8 +-
 scrapy/utils/url.py                 |  30 +++--
 tests/test_utils_url.py             | 190 +++++++++++++++++++---------
 3 files changed, 150 insertions(+), 78 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index e40e798b8..24e5eac40 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -8,7 +8,7 @@ from scrapy import signals
 from scrapy.utils.python import to_native_str
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
-from scrapy.utils.url import strip_url_credentials
+from scrapy.utils.url import strip_url
 
 
 LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)
@@ -53,7 +53,11 @@ class ReferrerPolicy(object):
             return None
         parsed_url = urlparse_cached(req_or_resp)
         if parsed_url.scheme not in self.NOREFERRER_SCHEMES:
-            return strip_url_credentials(parsed_url, origin_only=origin_only)
+            return strip_url(parsed_url,
+                             strip_credentials=True,
+                             strip_fragment=True,
+                             strip_default_port=True,
+                             origin_only=origin_only)
 
     def origin(self, req_or_resp):
         """Return serialized origin (scheme, host, path) for a request or response URL."""
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index f3ccfb0e8..9864f353d 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -105,33 +105,37 @@ def guess_scheme(url):
         return add_http_if_no_scheme(url)
 
 
-def strip_url_credentials(url, origin_only=False, keep_fragments=False):
+def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True):
+
+    """Strip URL string from some of its components:
+
+    - `strip_credentials` removes "user:password@"
+    - `strip_default_port` removes ":80" (resp. ":443", ":21")
+      from http:// (resp. https://, ftp://) URLs
+    - `origin_only` replaces path component with "/", also dropping
+      query and fragment components ; it also strips credentials
+    - `strip_fragment` drops any #fragment component
+    """
 
     if url is None:
         return None
-
     if not isinstance(url, ParseResult):
         parsed_url = urlparse(url)
     else:
         parsed_url = url
-
     netloc = parsed_url.netloc
-    # strip username and password if present
-    if parsed_url.username or parsed_url.password:
+    if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password):
         netloc = netloc.split('@')[-1]
-
-    # strip standard protocol numbers
-    # Note: strictly speaking, standard port numbers should only be
-    # stripped when comparing origins
-    if parsed_url.port:
-        if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443)):
+    if strip_default_port and parsed_url.port:
+        if (parsed_url.scheme, parsed_url.port) in (('http', 80),
+                                                    ('https', 443),
+                                                    ('ftp', 21)):
             netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '')
-
     return urlunparse((
         parsed_url.scheme,
         netloc,
         '/' if origin_only else parsed_url.path,
         '' if origin_only else parsed_url.params,
         '' if origin_only else parsed_url.query,
-        '' if not keep_fragments else parsed_url.fragment
+        '' if strip_fragment else parsed_url.fragment
     ))
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index f1a5c3196..1f9845d82 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -7,7 +7,7 @@ from six.moves.urllib.parse import urlparse
 from scrapy.spiders import Spider
 from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
                               add_http_if_no_scheme, guess_scheme,
-                              parse_url, strip_url_credentials)
+                              parse_url, strip_url)
 
 __doctests__ = ['scrapy.utils.url']
 
@@ -242,94 +242,158 @@ for k, args in enumerate ([
     setattr (GuessSchemeTest, t_method.__name__, t_method)
 
 
-class StripUrlCredentials(unittest.TestCase):
+class StripUrl(unittest.TestCase):
 
     def test_noop(self):
-        self.assertEqual(strip_url_credentials(
+        self.assertEqual(strip_url(
             'http://www.example.com/index.html'),
             'http://www.example.com/index.html')
 
     def test_noop_query_string(self):
-        self.assertEqual(strip_url_credentials(
+        self.assertEqual(strip_url(
             'http://www.example.com/index.html?somekey=somevalue'),
             'http://www.example.com/index.html?somekey=somevalue')
 
     def test_fragments(self):
-        self.assertEqual(strip_url_credentials(
-            'http://www.example.com/index.html?somekey=somevalue#section', keep_fragments=True),
+        self.assertEqual(strip_url(
+            'http://www.example.com/index.html?somekey=somevalue#section', strip_fragment=False),
             'http://www.example.com/index.html?somekey=somevalue#section')
 
-    def test_noop_trailing_path(self):
-        self.assertEqual(strip_url_credentials(
-            'http://www.example.com/'),
-            'http://www.example.com/')
+    def test_path(self):
+        for input_url, origin, output_url in [
+            ('http://www.example.com/',
+             False,
+             'http://www.example.com/'),
 
-    def test_noop_trailing_path2(self):
-        self.assertEqual(strip_url_credentials(
-            'http://www.example.com'),
-            'http://www.example.com')
+            ('http://www.example.com',
+             False,
+             'http://www.example.com'),
 
-    def test_trailing_path_origin(self):
-        self.assertEqual(strip_url_credentials(
-            'http://www.example.com', origin_only=True),
-            'http://www.example.com/')
+            ('http://www.example.com',
+             True,
+             'http://www.example.com/'),
+            ]:
+            self.assertEqual(strip_url(input_url, origin_only=origin), output_url)
+            self.assertEqual(strip_url(urlparse(input_url), origin_only=origin), output_url)
 
-    def test_username(self):
-        # username is stripped (and fragment too)
-        self.assertEqual(strip_url_credentials(
-            'http://username@www.example.com/index.html?somekey=somevalue#section'),
-            'http://www.example.com/index.html?somekey=somevalue')
+    def test_credentials(self):
+        for i, o in [
+            ('http://username@www.example.com/index.html?somekey=somevalue#section',
+             'http://www.example.com/index.html?somekey=somevalue'),
 
-    def test_username_empty_pass(self):
-        # same as above
-        self.assertEqual(strip_url_credentials(
-            'https://username:@www.example.com/index.html?somekey=somevalue#section'),
-            'https://www.example.com/index.html?somekey=somevalue')
+            ('https://username:@www.example.com/index.html?somekey=somevalue#section',
+             'https://www.example.com/index.html?somekey=somevalue'),
 
-    def test_username_password(self):
-        self.assertEqual(strip_url_credentials(
-            'ftp://username:password@www.example.com/index.html?somekey=somevalue#section'),
-            'ftp://www.example.com/index.html?somekey=somevalue')
+            ('ftp://username:password@www.example.com/index.html?somekey=somevalue#section',
+             'ftp://www.example.com/index.html?somekey=somevalue'),
+            ]:
+            self.assertEqual(strip_url(i, strip_credentials=True), o)
+            self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o)
 
-    def test_default_http_port(self):
-        self.assertEqual(strip_url_credentials(
-            'http://username:password@www.example.com:80/index.html'),
-            'http://www.example.com/index.html')
+    def test_default_ports_creds_off(self):
+        for i, o in [
+            ('http://username:password@www.example.com:80/index.html?somekey=somevalue#section',
+             'http://www.example.com/index.html?somekey=somevalue'),
 
-    def test_non_default_http_port(self):
-        self.assertEqual(strip_url_credentials(
-            'http://username:password@www.example.com:8080/index.html'),
-            'http://www.example.com:8080/index.html')
+            ('http://username:password@www.example.com:8080/index.html#section',
+             'http://www.example.com:8080/index.html'),
 
-    def test_default_https_port(self):
-        self.assertEqual(strip_url_credentials(
-            'https://username:password@www.example.com:443/index.html'),
-            'https://www.example.com/index.html')
+            ('http://username:password@www.example.com:443/index.html?somekey=somevalue&someotherkey=sov#section',
+             'http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov'),
 
-    def test_non_default_https_port(self):
-        self.assertEqual(strip_url_credentials(
-            'https://username:password@www.example.com:442/index.html'),
-            'https://www.example.com:442/index.html')
+            ('https://username:password@www.example.com:443/index.html',
+             'https://www.example.com/index.html'),
+
+            ('https://username:password@www.example.com:442/index.html',
+             'https://www.example.com:442/index.html'),
+
+            ('https://username:password@www.example.com:80/index.html',
+             'https://www.example.com:80/index.html'),
+
+            ('ftp://username:password@www.example.com:21/file.txt',
+             'ftp://www.example.com/file.txt'),
+
+            ('ftp://username:password@www.example.com:221/file.txt',
+             'ftp://www.example.com:221/file.txt'),
+            ]:
+            self.assertEqual(strip_url(i), o)
+            self.assertEqual(strip_url(urlparse(i)), o)
+
+    def test_default_ports(self):
+        for i, o in [
+            ('http://username:password@www.example.com:80/index.html',
+             'http://username:password@www.example.com/index.html'),
+
+            ('http://username:password@www.example.com:8080/index.html',
+             'http://username:password@www.example.com:8080/index.html'),
+
+            ('http://username:password@www.example.com:443/index.html',
+             'http://username:password@www.example.com:443/index.html'),
+
+            ('https://username:password@www.example.com:443/index.html',
+             'https://username:password@www.example.com/index.html'),
+
+            ('https://username:password@www.example.com:442/index.html',
+             'https://username:password@www.example.com:442/index.html'),
+
+            ('https://username:password@www.example.com:80/index.html',
+             'https://username:password@www.example.com:80/index.html'),
+
+            ('ftp://username:password@www.example.com:21/file.txt',
+             'ftp://username:password@www.example.com/file.txt'),
+
+            ('ftp://username:password@www.example.com:221/file.txt',
+             'ftp://username:password@www.example.com:221/file.txt'),
+            ]:
+            self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o)
+            self.assertEqual(strip_url(urlparse(i), strip_default_port=True, strip_credentials=False), o)
+
+    def test_default_ports_keep(self):
+        for i, o in [
+            ('http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section',
+             'http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov'),
+
+            ('http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov#section',
+             'http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov'),
+
+            ('http://username:password@www.example.com:443/index.html',
+             'http://username:password@www.example.com:443/index.html'),
+
+            ('https://username:password@www.example.com:443/index.html',
+             'https://username:password@www.example.com:443/index.html'),
+
+            ('https://username:password@www.example.com:442/index.html',
+             'https://username:password@www.example.com:442/index.html'),
+
+            ('https://username:password@www.example.com:80/index.html',
+             'https://username:password@www.example.com:80/index.html'),
+
+            ('ftp://username:password@www.example.com:21/file.txt',
+             'ftp://username:password@www.example.com:21/file.txt'),
+
+            ('ftp://username:password@www.example.com:221/file.txt',
+             'ftp://username:password@www.example.com:221/file.txt'),
+            ]:
+            self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o)
+            self.assertEqual(strip_url(urlparse(i), strip_default_port=False, strip_credentials=False), o)
 
     def test_origin_only(self):
-        self.assertEqual(strip_url_credentials(
-            'http://username:password@www.example.com/index.html', origin_only=True),
-            'http://www.example.com/')
+        for i, o in [
+            ('http://username:password@www.example.com/index.html',
+             'http://www.example.com/'),
 
-    def test_default_http_port_origin_only(self):
-        self.assertEqual(strip_url_credentials(
-            'http://username:password@www.example.com:80/index.html', origin_only=True),
-            'http://www.example.com/')
+            ('http://username:password@www.example.com:80/foo/bar?query=value#somefrag',
+             'http://www.example.com/'),
 
-    def test_non_default_http_port_origin_only(self):
-        self.assertEqual(strip_url_credentials(
-            'http://username:password@www.example.com:8008/index.html', origin_only=True),
-            'http://www.example.com:8008/')
+            ('http://username:password@www.example.com:8008/foo/bar?query=value#somefrag',
+             'http://www.example.com:8008/'),
+
+            ('https://username:password@www.example.com:443/index.html',
+             'https://www.example.com/'),
+            ]:
+            self.assertEqual(strip_url(i, origin_only=True), o)
+            self.assertEqual(strip_url(urlparse(i), origin_only=True), o)
 
-    def test_default_https_port_origin_only(self):
-        self.assertEqual(strip_url_credentials(
-            'https://username:password@www.example.com:443/index.html', origin_only=True),
-            'https://www.example.com/')
 
 if __name__ == "__main__":
     unittest.main()

From 0a0b60a59f75ef1b4976b811011bd50b78f9cec8 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 26 Oct 2016 12:41:00 +0200
Subject: [PATCH 1352/2974] Add tests for stripping userinfo with
 percent-encoded delimiters

---
 tests/test_utils_url.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 1f9845d82..9182d0fda 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -290,6 +290,26 @@ class StripUrl(unittest.TestCase):
             self.assertEqual(strip_url(i, strip_credentials=True), o)
             self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o)
 
+    def test_credentials_encoded_delims(self):
+        for i, o in [
+            # user: "username@"
+            # password: none
+            ('http://username%40@www.example.com/index.html?somekey=somevalue#section',
+             'http://www.example.com/index.html?somekey=somevalue'),
+
+            # user: "username:pass"
+            # password: ""
+            ('https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section',
+             'https://www.example.com/index.html?somekey=somevalue'),
+
+            # user: "me"
+            # password: "user@domain.com"
+            ('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section',
+             'ftp://www.example.com/index.html?somekey=somevalue'),
+            ]:
+            self.assertEqual(strip_url(i, strip_credentials=True), o)
+            self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o)
+
     def test_default_ports_creds_off(self):
         for i, o in [
             ('http://username:password@www.example.com:80/index.html?somekey=somevalue#section',

From c808a97c74718f72f53a3c00eca77f60c26fb6ba Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 12 Jan 2017 18:22:18 +0100
Subject: [PATCH 1353/2974] Add new "strict-" policies

---
 scrapy/spidermiddlewares/referer.py | 49 +++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 24e5eac40..5d2a0e7b1 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -17,7 +17,9 @@ POLICY_NO_REFERRER = "no-referrer"
 POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade"
 POLICY_SAME_ORIGIN = "same-origin"
 POLICY_ORIGIN = "origin"
+POLICY_STRICT_ORIGIN = "strict-origin"
 POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin"
+POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN = "strict-origin-when-cross-origin"
 POLICY_UNSAFE_URL = "unsafe-url"
 POLICY_SCRAPY_DEFAULT = "scrapy-default"
 
@@ -139,6 +141,26 @@ class OriginPolicy(ReferrerPolicy):
         return self.origin_referrer(response)
 
 
+class StrictOriginPolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin
+
+    The "strict-origin" policy sends the ASCII serialization
+    of the origin of the request client when making requests:
+    - from a TLS-protected environment settings object to a potentially trustworthy URL, and
+    - from non-TLS-protected environment settings objects to any origin.
+
+    Requests from TLS-protected request clients to non- potentially trustworthy URLs,
+    on the other hand, will contain no referrer information.
+    A Referer HTTP header will not be sent.
+    """
+    name = POLICY_STRICT_ORIGIN
+
+    def referrer(self, response, request):
+        if urlparse_cached(response).scheme == urlparse_cached(request).scheme:
+            return self.origin_referrer(response)
+
+
 class OriginWhenCrossOriginPolicy(ReferrerPolicy):
     """
     https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin
@@ -160,6 +182,33 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
             return origin
 
 
+class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin
+
+    The "strict-origin-when-cross-origin" policy specifies that a full URL,
+    stripped for use as a referrer, is sent as referrer information
+    when making same-origin requests from a particular request client,
+    and only the ASCII serialization of the origin of the request client
+    when making cross-origin requests:
+
+    - from a TLS-protected environment settings object to a potentially trustworthy URL, and
+    - from non-TLS-protected environment settings objects to any origin.
+
+    Requests from TLS-protected clients to non- potentially trustworthy URLs,
+    on the other hand, will contain no referrer information.
+    A Referer HTTP header will not be sent.
+    """
+    name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
+
+    def referrer(self, response, request):
+        origin = self.origin(response)
+        if origin == self.origin(request):
+            return self.stripped_referrer(response)
+        else:
+            return origin
+
+
 class UnsafeUrlPolicy(ReferrerPolicy):
     """
     https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url

From 5cef67ae75dca229bd1b1f3ac61f52309d1ca5a4 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 17 Jan 2017 14:18:33 +0100
Subject: [PATCH 1354/2974] Update Referrer tests for "strict-" policies

---
 tests/test_spidermiddleware_referer.py | 75 +++++++++++++++++++++++++-
 1 file changed, 74 insertions(+), 1 deletion(-)

diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 9555817d9..df20dfbb9 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -8,6 +8,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \
     POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \
     POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \
     POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \
+    POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, \
     DefaultReferrerPolicy, \
     NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \
     OriginWhenCrossOriginPolicy, OriginPolicy, \
@@ -39,7 +40,6 @@ class TestRefererMiddleware(TestCase):
         for origin, target, referrer in self.scenarii:
             response = self.get_response(origin)
             request = self.get_request(target)
-
             out = list(self.mw.process_spider_output(response, [request], self.spider))
             self.assertEquals(out[0].headers.get('Referer'), referrer)
 
@@ -154,6 +154,25 @@ class MixinOrigin(object):
     ]
 
 
+class MixinStrictOrigin(object):
+    scenarii = [
+        # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades
+        ('https://example.com/page.html',   'https://example.com/not-page.html',    b'https://example.com/'),
+        ('https://example.com/page.html',   'https://scrapy.org',                   b'https://example.com/'),
+        ('http://example.com/page.html',    'http://scrapy.org',                    b'http://example.com/'),
+
+        # downgrade: send nothing
+        ('https://example.com/page.html',   'http://scrapy.org',                    None),
+
+        # upgrade: send origin
+        ('http://example.com/page.html',    'https://scrapy.org',                   b'http://example.com/'),
+
+        # test for user/password stripping
+        ('https://user:password@example.com/page.html', 'https://scrapy.org',       b'https://example.com/'),
+        ('https://user:password@example.com/page.html', 'http://scrapy.org',        None),
+    ]
+
+
 class MixinOriginWhenCrossOrigin(object):
     scenarii = [
         # Same origin (protocol, host, port): send referrer
@@ -189,6 +208,44 @@ class MixinOriginWhenCrossOrigin(object):
     ]
 
 
+class MixinStrictOriginWhenCrossOrigin(object):
+    scenarii = [
+        # Same origin (protocol, host, port): send referrer
+        ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
+        ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
+        ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
+        ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
+        ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
+        ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+
+        # Different host: send origin as referrer
+        ('https://example2.com/page.html',  'https://scrapy.org/otherpage.html',        b'https://example2.com/'),
+        ('https://example2.com/page.html',  'https://not.example2.com/otherpage.html',  b'https://example2.com/'),
+        ('http://example2.com/page.html',   'http://not.example2.com/otherpage.html',   b'http://example2.com/'),
+        # exact match required
+        ('http://example2.com/page.html',   'http://www.example2.com/otherpage.html',   b'http://example2.com/'),
+
+        # Different port: send origin as referrer
+        ('https://example3.com:444/page.html',  'https://example3.com/not-page.html',   b'https://example3.com:444/'),
+        ('http://example3.com:81/page.html',    'http://example3.com/not-page.html',    b'http://example3.com:81/'),
+
+        # downgrade
+        ('https://example4.com/page.html',  'http://example4.com/not-page.html',    None),
+        ('https://example4.com/page.html',  'http://not.example4.com/',             None),
+        ('ftp://example4.com/urls.zip',     'http://example4.com/not-page.html',    None),
+
+        # Different protocols: send origin as referrer
+        ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
+        ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
+
+        # test for user/password stripping
+        ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html',  b'https://example5.com/page.html'),
+
+        # TLS to non-TLS downgrade: send nothing
+        ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html',   None),
+    ]
+
+
 class MixinUnsafeUrl(object):
     scenarii = [
         # TLS to TLS: send referrer
@@ -241,10 +298,18 @@ class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware):
     settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}
 
 
+class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'}
+
+
 class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
     settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
 
 
+class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'}
+
+
 class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
 
@@ -297,10 +362,18 @@ class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_ORIGIN}
 
 
+class TestRefererMiddlewareSrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN}
+
+
 class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
 
 
+class TestRefererMiddlewareStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
+
+
 class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
 

From 77aec5a79681128c0608e65bc77e317252172d98 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 17 Jan 2017 14:19:19 +0100
Subject: [PATCH 1355/2974] Fix implementation

---
 scrapy/spidermiddlewares/referer.py | 40 ++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 5d2a0e7b1..a6316cd0c 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -31,13 +31,13 @@ class ReferrerPolicy(object):
     def referrer(self, response, request):
         raise NotImplementedError()
 
-    def stripped_referrer(self, req_or_resp):
-        return self.strip_url(req_or_resp)
+    def stripped_referrer(self, r):
+        return self.strip_url(r)
 
-    def origin_referrer(self, req_or_resp):
-        return self.strip_url(req_or_resp, origin_only=True)
+    def origin_referrer(self, r):
+        return self.strip_url(r, origin_only=True)
 
-    def strip_url(self, req_or_resp, origin_only=False):
+    def strip_url(self, r, origin_only=False):
         """
         https://www.w3.org/TR/referrer-policy/#strip-url
 
@@ -51,9 +51,9 @@ class ReferrerPolicy(object):
             Set url's query to null.
         Return url.
         """
-        if req_or_resp.url is None or not req_or_resp.url:
+        if r is None or not r.url:
             return None
-        parsed_url = urlparse_cached(req_or_resp)
+        parsed_url = urlparse_cached(r)
         if parsed_url.scheme not in self.NOREFERRER_SCHEMES:
             return strip_url(parsed_url,
                              strip_credentials=True,
@@ -61,9 +61,19 @@ class ReferrerPolicy(object):
                              strip_default_port=True,
                              origin_only=origin_only)
 
-    def origin(self, req_or_resp):
+    def origin(self, r):
         """Return serialized origin (scheme, host, path) for a request or response URL."""
-        return self.strip_url(req_or_resp, origin_only=True)
+        return self.strip_url(r, origin_only=True)
+
+    def potentially_trustworthy(self, r):
+        # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy
+        parsed_url = urlparse_cached(r)
+        if parsed_url.scheme in ('data',):
+            return False
+        return self.tls_protected(r)
+
+    def tls_protected(self, r):
+        return urlparse_cached(r).scheme in ('https', 'ftps')
 
 
 class NoReferrerPolicy(ReferrerPolicy):
@@ -157,7 +167,9 @@ class StrictOriginPolicy(ReferrerPolicy):
     name = POLICY_STRICT_ORIGIN
 
     def referrer(self, response, request):
-        if urlparse_cached(response).scheme == urlparse_cached(request).scheme:
+        if ((urlparse_cached(response).scheme == 'https' and
+             self.potentially_trustworthy(request))
+             or urlparse_cached(response).scheme == 'http'):
             return self.origin_referrer(response)
 
 
@@ -205,8 +217,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
         origin = self.origin(response)
         if origin == self.origin(request):
             return self.stripped_referrer(response)
-        else:
-            return origin
+        elif ((urlparse_cached(response).scheme in ('https', 'ftps') and
+               self.potentially_trustworthy(request))
+              or urlparse_cached(response).scheme == 'http'):
+            return self.origin_referrer(response)
 
 
 class UnsafeUrlPolicy(ReferrerPolicy):
@@ -244,7 +258,9 @@ _policy_classes = {p.name: p for p in (
     NoReferrerWhenDowngradePolicy,
     SameOriginPolicy,
     OriginPolicy,
+    StrictOriginPolicy,
     OriginWhenCrossOriginPolicy,
+    StrictOriginWhenCrossOriginPolicy,
     UnsafeUrlPolicy,
     DefaultReferrerPolicy,
 )}

From deb8567116db8488a8f5b890aad975df095ddcfe Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 17 Jan 2017 16:22:49 +0100
Subject: [PATCH 1356/2974] Update NoReferrerWhenDowngradePolicy

---
 scrapy/spidermiddlewares/referer.py | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index a6316cd0c..d64c79108 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -94,12 +94,11 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
     """
     https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
 
-    The "no-referrer-when-downgrade" policy sends a full URL
-    along with requests from a TLS-protected environment settings object
-    to a a priori authenticated URL,
-    and requests from request clients which are not TLS-protected to any origin.
+    The "no-referrer-when-downgrade" policy sends a full URL along with requests
+    from a TLS-protected environment settings object to a potentially trustworthy URL,
+    and requests from clients which are not TLS-protected to any origin.
 
-    Requests from TLS-protected request clients to non-a priori authenticated URLs,
+    Requests from TLS-protected clients to non-potentially trustworthy URLs,
     on the other hand, will contain no referrer information.
     A Referer HTTP header will not be sent.
 
@@ -108,15 +107,8 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
     name = POLICY_NO_REFERRER_WHEN_DOWNGRADE
 
     def referrer(self, response, request):
-        # https://www.w3.org/TR/referrer-policy/#determine-requests-referrer:
-        #
-        # If environment is TLS-protected
-        # and the origin of request's current URL is not an a priori authenticated URL,
-        # then return no referrer.
-        if urlparse_cached(response).scheme in ('https', 'ftps') and \
-            urlparse_cached(request).scheme in ('http',):
-                return None
-        return self.stripped_referrer(response)
+        if not self.tls_protected(response) or self.tls_protected(request):
+            return self.stripped_referrer(response)
 
 
 class SameOriginPolicy(ReferrerPolicy):

From ebcacd3f549de034d05ed946c446e0f71b11bc6f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 17 Jan 2017 17:18:00 +0100
Subject: [PATCH 1357/2974] Update StrictOriginPolicy

---
 scrapy/spidermiddlewares/referer.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index d64c79108..8ee04120a 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -159,9 +159,8 @@ class StrictOriginPolicy(ReferrerPolicy):
     name = POLICY_STRICT_ORIGIN
 
     def referrer(self, response, request):
-        if ((urlparse_cached(response).scheme == 'https' and
-             self.potentially_trustworthy(request))
-             or urlparse_cached(response).scheme == 'http'):
+        if ((self.tls_protected(response) and self.potentially_trustworthy(request))
+            or not self.tls_protected(response)):
             return self.origin_referrer(response)
 
 

From b6c761d2b4c9cafe94010075ebf39807921dec9c Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 17 Jan 2017 17:57:17 +0100
Subject: [PATCH 1358/2974] Fix tests

---
 scrapy/spidermiddlewares/referer.py    | 7 ++++---
 tests/test_spidermiddleware_referer.py | 8 +++++++-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 8ee04120a..4f50db689 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -159,7 +159,8 @@ class StrictOriginPolicy(ReferrerPolicy):
     name = POLICY_STRICT_ORIGIN
 
     def referrer(self, response, request):
-        if ((self.tls_protected(response) and self.potentially_trustworthy(request))
+        if ((self.tls_protected(response) and
+             self.potentially_trustworthy(request))
             or not self.tls_protected(response)):
             return self.origin_referrer(response)
 
@@ -208,9 +209,9 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
         origin = self.origin(response)
         if origin == self.origin(request):
             return self.stripped_referrer(response)
-        elif ((urlparse_cached(response).scheme in ('https', 'ftps') and
+        elif ((self.tls_protected(response) and
                self.potentially_trustworthy(request))
-              or urlparse_cached(response).scheme == 'http'):
+              or not self.tls_protected(response)):
             return self.origin_referrer(response)
 
 
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index df20dfbb9..4779b0ed1 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -232,7 +232,13 @@ class MixinStrictOriginWhenCrossOrigin(object):
         # downgrade
         ('https://example4.com/page.html',  'http://example4.com/not-page.html',    None),
         ('https://example4.com/page.html',  'http://not.example4.com/',             None),
-        ('ftp://example4.com/urls.zip',     'http://example4.com/not-page.html',    None),
+
+        # non-TLS to non-TLS
+        ('ftp://example4.com/urls.zip',     'http://example4.com/not-page.html',    b'ftp://example4.com/'),
+
+        # upgrade
+        ('http://example4.com/page.html',  'https://example4.com/not-page.html',    b'http://example4.com/'),
+        ('http://example4.com/page.html',  'https://not.example4.com/',             b'http://example4.com/'),
 
         # Different protocols: send origin as referrer
         ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),

From c86f568b9cd1fc69c67762ba39714e38c9ba70fb Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 17 Jan 2017 22:31:29 +0100
Subject: [PATCH 1359/2974] Update docs with "strict-..." policies

---
 docs/topics/spider-middleware.rst | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index a9d3d4568..e8325d7ef 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -344,9 +344,18 @@ This setting accepts:
 - a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass,
   either a custom one or one of the built-in ones
   (see ``scrapy.spidermiddlewares.referer``),
-- or one of the standard W3C-defined string values, i.e. ``"no-referrer"``,
-  ``"no-referrer-when-downgrade"``, ``"same-origin"``, ``"origin"``,
-  ``"origin-when-cross-origin"`` or ``"unsafe-url"``.
+- or one of the standard W3C-defined string values:
+
+  - `"no-referrer" <https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer>`_,
+  - `"no-referrer-when-downgrade" <https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade>`_,
+  - `"same-origin" <https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin>`_,
+  - `"origin" <https://www.w3.org/TR/referrer-policy/#referrer-policy-origin>`_,
+  - `"strict-origin" <https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin>`_,
+  - `"origin-when-cross-origin" <https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin>`_,
+  - `"strict-origin-when-cross-origin" <https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin>`_,
+  - or `"unsafe-url" <https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url>`_
+    (not recommended).
+
   (It can also be the non-standard value ``"scrapy-default"`` to use
   Scrapy's default referrer policy.)
 

From e249abc32bcb171e4676b8fd9f11f04201f80a75 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 18 Jan 2017 15:48:28 +0100
Subject: [PATCH 1360/2974] Update docs

---
 docs/topics/spider-middleware.rst | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index e8325d7ef..0ddf027ea 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -333,7 +333,7 @@ Whether to enable referer middleware.
 REFERER_POLICY
 ^^^^^^^^^^^^^^
 
-.. versionadded:: 1.3
+.. versionadded:: 1.4
 
 Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
 
@@ -346,14 +346,14 @@ This setting accepts:
   (see ``scrapy.spidermiddlewares.referer``),
 - or one of the standard W3C-defined string values:
 
-  - `"no-referrer" <https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer>`_,
-  - `"no-referrer-when-downgrade" <https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade>`_,
-  - `"same-origin" <https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin>`_,
-  - `"origin" <https://www.w3.org/TR/referrer-policy/#referrer-policy-origin>`_,
-  - `"strict-origin" <https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin>`_,
-  - `"origin-when-cross-origin" <https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin>`_,
-  - `"strict-origin-when-cross-origin" <https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin>`_,
-  - or `"unsafe-url" <https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url>`_
+  - `"no-referrer"`_,
+  - `"no-referrer-when-downgrade"`_,
+  - `"same-origin"`_,
+  - `"origin"`_,
+  - `"strict-origin"`_,
+  - `"origin-when-cross-origin"`_,
+  - `"strict-origin-when-cross-origin"`_,
+  - or `"unsafe-url"`_
     (not recommended).
 
   (It can also be the non-standard value ``"scrapy-default"`` to use
@@ -364,14 +364,22 @@ with the addition that "Referrer" is not sent if the parent request was
 using ``file://`` or ``s3://`` scheme.
 
 .. warning::
-    By default, Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_,
+    Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_,
     will send a non-empty "Referer" header from any ``https://`` to any ``https://`` URL,
     even if the domain is different.
     ``same-origin`` may be a better choice if you want to remove referrer
     information for cross-domain requests.
 
 .. _Referrer Policy: https://www.w3.org/TR/referrer-policy
+.. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer
 .. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
+.. _"same-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin
+.. _"origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin
+.. _"strict-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin
+.. _"origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin
+.. _"strict-origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin
+.. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url
+
 
 UrlLengthMiddleware
 -------------------

From 03ff19d1882edb709cf627795f3ce22934235254 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 18 Jan 2017 16:29:20 +0100
Subject: [PATCH 1361/2974] Update docs for new "referrer_policy" Request.meta
 key

---
 docs/topics/request-response.rst  | 1 +
 docs/topics/spider-middleware.rst | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 3d110b02d..67f8ec285 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -307,6 +307,7 @@ Those are:
 * :reqmeta:`proxy`
 * ``ftp_user`` (See :setting:`FTP_USER` for more info)
 * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
+* :reqmeta:`referrer_policy`
 
 .. reqmeta:: bindaddress
 
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 0ddf027ea..a4ac45b41 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -337,6 +337,8 @@ REFERER_POLICY
 
 Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
 
+.. reqmeta:: referrer_policy
+
 `Referrer Policy`_ to apply when populating Request "Referer" header.
 
 This setting accepts:
@@ -370,6 +372,11 @@ using ``file://`` or ``s3://`` scheme.
     ``same-origin`` may be a better choice if you want to remove referrer
     information for cross-domain requests.
 
+.. note::
+    You can also override the Referrer Policy per request,
+    using the special ``"referrer_policy"`` :ref:`Request.meta <topics-request-meta>` key,
+    with the same acceptable values as for the ``REFERER_POLICY`` setting.
+
 .. _Referrer Policy: https://www.w3.org/TR/referrer-policy
 .. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer
 .. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade

From eb07285a63d6fdeefe113051943f4bf7e36f7b33 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 18 Jan 2017 17:20:35 +0100
Subject: [PATCH 1362/2974] Reword warning on no-referrer-when-downgrade policy

---
 docs/topics/spider-middleware.rst | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index a4ac45b41..ada4c46c0 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -349,7 +349,8 @@ This setting accepts:
 - or one of the standard W3C-defined string values:
 
   - `"no-referrer"`_,
-  - `"no-referrer-when-downgrade"`_,
+  - `"no-referrer-when-downgrade"`_
+    (the W3C-recommended default, used by major web browsers),
   - `"same-origin"`_,
   - `"origin"`_,
   - `"strict-origin"`_,
@@ -358,18 +359,19 @@ This setting accepts:
   - or `"unsafe-url"`_
     (not recommended).
 
-  (It can also be the non-standard value ``"scrapy-default"`` to use
-  Scrapy's default referrer policy.)
+It can also be the non-standard value ``"scrapy-default"`` to use
+Scrapy's default referrer policy.
 
 Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_,
 with the addition that "Referrer" is not sent if the parent request was
 using ``file://`` or ``s3://`` scheme.
 
 .. warning::
-    Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_,
-    will send a non-empty "Referer" header from any ``https://`` to any ``https://`` URL,
+    Scrapy's default referrer policy—just like `"no-referrer-when-downgrade"`_,
+    the W3C-recommended value for browsers—will send a non-empty
+    "Referer" header from any ``http(s)://`` to any ``https://`` URL,
     even if the domain is different.
-    ``same-origin`` may be a better choice if you want to remove referrer
+    `"same-origin"`_ may be a better choice if you want to remove referrer
     information for cross-domain requests.
 
 .. note::

From 605935f015b1862b83f051380cd08b2931cce784 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 26 Jan 2017 12:09:34 +0100
Subject: [PATCH 1363/2974] Edit text

---
 docs/topics/spider-middleware.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index ada4c46c0..a792364b3 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -363,12 +363,12 @@ It can also be the non-standard value ``"scrapy-default"`` to use
 Scrapy's default referrer policy.
 
 Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_,
-with the addition that "Referrer" is not sent if the parent request was
+with the addition that "Referer" is not sent if the parent request was
 using ``file://`` or ``s3://`` scheme.
 
 .. warning::
-    Scrapy's default referrer policy—just like `"no-referrer-when-downgrade"`_,
-    the W3C-recommended value for browsers—will send a non-empty
+    Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_,
+    the W3C-recommended value for browsers — will send a non-empty
     "Referer" header from any ``http(s)://`` to any ``https://`` URL,
     even if the domain is different.
     `"same-origin"`_ may be a better choice if you want to remove referrer

From 3dc09eeceb16ca97a0c5851bfdc41f921772910b Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 30 Jan 2017 15:25:49 +0100
Subject: [PATCH 1364/2974] Use table for referrer policy options

---
 docs/topics/spider-middleware.rst | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index a792364b3..12aa53012 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -346,21 +346,21 @@ This setting accepts:
 - a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass,
   either a custom one or one of the built-in ones
   (see ``scrapy.spidermiddlewares.referer``),
-- or one of the standard W3C-defined string values:
+- or one of the standard W3C-defined string values
 
-  - `"no-referrer"`_,
-  - `"no-referrer-when-downgrade"`_
-    (the W3C-recommended default, used by major web browsers),
-  - `"same-origin"`_,
-  - `"origin"`_,
-  - `"strict-origin"`_,
-  - `"origin-when-cross-origin"`_,
-  - `"strict-origin-when-cross-origin"`_,
-  - or `"unsafe-url"`_
-    (not recommended).
-
-It can also be the non-standard value ``"scrapy-default"`` to use
-Scrapy's default referrer policy.
+=======================================  ========================================================================  =======================================================
+String value                             Class name
+=======================================  ========================================================================  =======================================================
+`"no-referrer"`_                         ``'scrapy.spidermiddlewares.referer.NoReferrerPolicy'``
+`"no-referrer-when-downgrade"`_          ``'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'``      the W3C-recommended default, used by major web browsers
+`"same-origin"`_                         ``'scrapy.spidermiddlewares.referer.SameOriginPolicy'``
+`"origin"`_                              ``'scrapy.spidermiddlewares.referer.OriginPolicy'``
+`"strict-origin"`_                       ``'scrapy.spidermiddlewares.referer.StrictOriginPolicy'``
+`"origin-when-cross-origin"`_            ``'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'``
+`"strict-origin-when-cross-origin"`_     ``'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'``
+`"unsafe-url"`_                          ``'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'``                    NOT recommended
+``"scrapy-default"``                     ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``              Scrapy's default policy (see below)
+=======================================  ========================================================================  =======================================================
 
 Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_,
 with the addition that "Referer" is not sent if the parent request was

From 537683f945906ff9865cf4e2704a432612f10eb4 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 21 Feb 2017 16:47:57 +0100
Subject: [PATCH 1365/2974] Add autoclass directives to document built-in
 policies

---
 docs/topics/spider-middleware.rst   | 70 +++++++++++++++++++----------
 scrapy/spidermiddlewares/referer.py |  6 ++-
 2 files changed, 51 insertions(+), 25 deletions(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 12aa53012..349dbb3a1 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -341,43 +341,65 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
 
 `Referrer Policy`_ to apply when populating Request "Referer" header.
 
-This setting accepts:
+.. note::
+    You can also set the Referrer Policy per request,
+    using the special ``"referrer_policy"`` :ref:`Request.meta <topics-request-meta>` key,
+    with the same acceptable values as for the ``REFERER_POLICY`` setting.
 
-- a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass,
-  either a custom one or one of the built-in ones
-  (see ``scrapy.spidermiddlewares.referer``),
-- or one of the standard W3C-defined string values
+Acceptable values for REFERER_POLICY
+************************************
 
-=======================================  ========================================================================  =======================================================
-String value                             Class name
-=======================================  ========================================================================  =======================================================
-`"no-referrer"`_                         ``'scrapy.spidermiddlewares.referer.NoReferrerPolicy'``
-`"no-referrer-when-downgrade"`_          ``'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'``      the W3C-recommended default, used by major web browsers
-`"same-origin"`_                         ``'scrapy.spidermiddlewares.referer.SameOriginPolicy'``
-`"origin"`_                              ``'scrapy.spidermiddlewares.referer.OriginPolicy'``
-`"strict-origin"`_                       ``'scrapy.spidermiddlewares.referer.StrictOriginPolicy'``
-`"origin-when-cross-origin"`_            ``'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'``
-`"strict-origin-when-cross-origin"`_     ``'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'``
-`"unsafe-url"`_                          ``'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'``                    NOT recommended
-``"scrapy-default"``                     ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``              Scrapy's default policy (see below)
-=======================================  ========================================================================  =======================================================
+- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy``
+  subclass — a custom policy or one of the built-in ones (see classes below),
+- or one of the standard W3C-defined string values,
+- or the special ``"scrapy-default"``.
 
-Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_,
-with the addition that "Referer" is not sent if the parent request was
-using ``file://`` or ``s3://`` scheme.
+=======================================  ========================================================================
+String value                             Class name (as a string)
+=======================================  ========================================================================
+``"scrapy-default"`` (default)           :class:`scrapy.spidermiddlewares.referer.DefaultReferrerPolicy`
+`"no-referrer"`_                         :class:`scrapy.spidermiddlewares.referer.NoReferrerPolicy`
+`"no-referrer-when-downgrade"`_          :class:`scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy`
+`"same-origin"`_                         :class:`scrapy.spidermiddlewares.referer.SameOriginPolicy`
+`"origin"`_                              :class:`scrapy.spidermiddlewares.referer.OriginPolicy`
+`"strict-origin"`_                       :class:`scrapy.spidermiddlewares.referer.StrictOriginPolicy`
+`"origin-when-cross-origin"`_            :class:`scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy`
+`"strict-origin-when-cross-origin"`_     :class:`scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy`
+`"unsafe-url"`_                          :class:`scrapy.spidermiddlewares.referer.UnsafeUrlPolicy`
+=======================================  ========================================================================
 
+.. autoclass:: DefaultReferrerPolicy
 .. warning::
     Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_,
     the W3C-recommended value for browsers — will send a non-empty
     "Referer" header from any ``http(s)://`` to any ``https://`` URL,
     even if the domain is different.
+
     `"same-origin"`_ may be a better choice if you want to remove referrer
     information for cross-domain requests.
 
+.. autoclass:: NoReferrerPolicy
+
+.. autoclass:: NoReferrerWhenDowngradePolicy
 .. note::
-    You can also override the Referrer Policy per request,
-    using the special ``"referrer_policy"`` :ref:`Request.meta <topics-request-meta>` key,
-    with the same acceptable values as for the ``REFERER_POLICY`` setting.
+    "no-referrer-when-downgrade" policy is the W3C-recommended default,
+    and is used by major web browsers.
+
+    However, it is NOT Scrapy's default referrer policy (see :class:`DefaultReferrerPolicy`).
+
+.. autoclass:: SameOriginPolicy
+
+.. autoclass:: OriginPolicy
+
+.. autoclass:: StrictOriginPolicy
+
+.. autoclass:: OriginWhenCrossOriginPolicy
+
+.. autoclass:: StrictOriginWhenCrossOriginPolicy
+
+.. autoclass:: UnsafeUrlPolicy
+.. warning::
+    "unsafe-url" policy is NOT recommended.
 
 .. _Referrer Policy: https://www.w3.org/TR/referrer-policy
 .. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 4f50db689..c015e13c8 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -240,7 +240,11 @@ class LegacyPolicy(ReferrerPolicy):
 
 
 class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
-
+    """
+    A variant of "no-referrer-when-downgrade",
+    with the addition that "Referer" is not sent if the parent request was
+    using ``file://`` or ``s3://`` scheme.
+    """
     NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3')
     name = POLICY_SCRAPY_DEFAULT
 

From bc200d1155e0f93f37897ac206aa32945daf89bd Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 21 Feb 2017 17:08:39 +0100
Subject: [PATCH 1366/2974] Rename setting to REFERRER_POLICY (with 2 Rs)

---
 docs/topics/spider-middleware.rst      | 12 ++++-----
 scrapy/settings/default_settings.py    |  2 +-
 scrapy/spidermiddlewares/referer.py    |  2 +-
 tests/test_spidermiddleware_referer.py | 36 +++++++++++++-------------
 4 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 349dbb3a1..9a0ccd0c1 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -328,10 +328,10 @@ Default: ``True``
 
 Whether to enable referer middleware.
 
-.. setting:: REFERER_POLICY
+.. setting:: REFERRER_POLICY
 
-REFERER_POLICY
-^^^^^^^^^^^^^^
+REFERRER_POLICY
+^^^^^^^^^^^^^^^
 
 .. versionadded:: 1.4
 
@@ -344,10 +344,10 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
 .. note::
     You can also set the Referrer Policy per request,
     using the special ``"referrer_policy"`` :ref:`Request.meta <topics-request-meta>` key,
-    with the same acceptable values as for the ``REFERER_POLICY`` setting.
+    with the same acceptable values as for the ``REFERRER_POLICY`` setting.
 
-Acceptable values for REFERER_POLICY
-************************************
+Acceptable values for REFERRER_POLICY
+*************************************
 
 - either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy``
   subclass — a custom policy or one of the built-in ones (see classes below),
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 15a134dd3..35d9844a7 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -234,7 +234,7 @@ REDIRECT_MAX_TIMES = 20  # uses Firefox default setting
 REDIRECT_PRIORITY_ADJUST = +2
 
 REFERER_ENABLED = True
-REFERER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
+REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
 
 RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index c015e13c8..24c163089 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -266,7 +266,7 @@ class RefererMiddleware(object):
     def __init__(self, settings=None):
         self.default_policy = DefaultReferrerPolicy
         if settings is not None:
-            policy = settings.get('REFERER_POLICY')
+            policy = settings.get('REFERRER_POLICY')
             if policy is not None:
                 # expect a string for the path to the policy class
                 try:
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 4779b0ed1..81868efab 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -289,35 +289,35 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware):
 
 # --- Tests using settings to set policy using class path
 class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware):
-    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'}
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'}
 
 
 class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
-    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
 
 
 class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware):
-    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
 
 
 class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware):
-    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}
 
 
 class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
-    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'}
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'}
 
 
 class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
-    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
 
 
 class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
-    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'}
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'}
 
 
 class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
-    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
 
 
 class CustomPythonOrgPolicy(ReferrerPolicy):
@@ -336,7 +336,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy):
 
 
 class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware):
-    settings = {'REFERER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}
+    settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}
     scenarii = [
         ('https://example.com/',    'https://scrapy.org/',  b'https://python.org/'),
         ('http://example.com/',     'http://scrapy.org/',   b'http://python.org/'),
@@ -385,17 +385,17 @@ class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
 
 
 class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
-    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
 
 
 class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):
-    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
     req_meta = {'referrer_policy': POLICY_NO_REFERRER}
 
 
 class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):
-    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
 
 
@@ -411,7 +411,7 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):
                 (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
                 (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
             ]:
-            settings = Settings({'REFERER_POLICY': s})
+            settings = Settings({'REFERRER_POLICY': s})
             mw = RefererMiddleware(settings)
             self.assertEquals(mw.default_policy, p)
 
@@ -425,24 +425,24 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):
                 (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
                 (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
             ]:
-            settings = Settings({'REFERER_POLICY': s.upper()})
+            settings = Settings({'REFERRER_POLICY': s.upper()})
             mw = RefererMiddleware(settings)
             self.assertEquals(mw.default_policy, p)
 
     def test_invalid_name(self):
-        settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'})
+        settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'})
         with self.assertRaises(NotConfigured):
             mw = RefererMiddleware(settings)
 
 
 class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
-    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
     resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}
 
 class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
-    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
     resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}
 
 class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
-    settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
     resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}

From 04e4d08612364480c4d72d2b55e1cac161397136 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 27 Feb 2017 16:04:37 +0100
Subject: [PATCH 1367/2974] Pass URLs around instead of Request/Responses

---
 scrapy/spidermiddlewares/referer.py | 53 ++++++++++++++++-------------
 scrapy/utils/url.py                 |  7 +---
 2 files changed, 31 insertions(+), 29 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 24c163089..f2910a0c8 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -2,6 +2,8 @@
 RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
+from six.moves.urllib.parse import urlparse
+
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
@@ -32,12 +34,14 @@ class ReferrerPolicy(object):
         raise NotImplementedError()
 
     def stripped_referrer(self, r):
-        return self.strip_url(r)
+        if urlparse(r).scheme not in self.NOREFERRER_SCHEMES:
+            return self.strip_url(r)
 
     def origin_referrer(self, r):
-        return self.strip_url(r, origin_only=True)
+        if urlparse(r).scheme not in self.NOREFERRER_SCHEMES:
+            return self.origin(r)
 
-    def strip_url(self, r, origin_only=False):
+    def strip_url(self, url, origin_only=False):
         """
         https://www.w3.org/TR/referrer-policy/#strip-url
 
@@ -51,29 +55,27 @@ class ReferrerPolicy(object):
             Set url's query to null.
         Return url.
         """
-        if r is None or not r.url:
+        if not url:
             return None
-        parsed_url = urlparse_cached(r)
-        if parsed_url.scheme not in self.NOREFERRER_SCHEMES:
-            return strip_url(parsed_url,
-                             strip_credentials=True,
-                             strip_fragment=True,
-                             strip_default_port=True,
-                             origin_only=origin_only)
+        return strip_url(url,
+                         strip_credentials=True,
+                         strip_fragment=True,
+                         strip_default_port=True,
+                         origin_only=origin_only)
 
     def origin(self, r):
         """Return serialized origin (scheme, host, path) for a request or response URL."""
         return self.strip_url(r, origin_only=True)
 
-    def potentially_trustworthy(self, r):
+    def potentially_trustworthy(self, url):
         # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy
-        parsed_url = urlparse_cached(r)
+        parsed_url = urlparse(url)
         if parsed_url.scheme in ('data',):
             return False
-        return self.tls_protected(r)
+        return self.tls_protected(url)
 
-    def tls_protected(self, r):
-        return urlparse_cached(r).scheme in ('https', 'ftps')
+    def tls_protected(self, url):
+        return urlparse(url).scheme in ('https', 'ftps')
 
 
 class NoReferrerPolicy(ReferrerPolicy):
@@ -287,12 +289,17 @@ class RefererMiddleware(object):
         crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled)
         return mw
 
-    def policy(self, response, request):
+    def policy(self, resp_or_url, request):
+        """
+        Determine Referrer-Policy to use from a parent Response (or URL),
+        and a Request to be sent.
+        """
         # policy set in request's meta dict takes precedence over default policy
         policy_name = request.meta.get('referrer_policy')
         if policy_name is None:
-            policy_name = to_native_str(
-                response.headers.get('Referrer-Policy', '').decode('latin1'))
+            if isinstance(resp_or_url, Response):
+                policy_name = to_native_str(
+                    resp_or_url.headers.get('Referrer-Policy', '').decode('latin1'))
 
         cls = _policy_classes.get(policy_name.lower(), self.default_policy)
         return cls()
@@ -300,7 +307,7 @@ class RefererMiddleware(object):
     def process_spider_output(self, response, result, spider):
         def _set_referer(r):
             if isinstance(r, Request):
-                referrer = self.policy(response, r).referrer(response, r)
+                referrer = self.policy(response, r).referrer(response.url, r.url)
                 if referrer is not None:
                     r.headers.setdefault('Referer', referrer)
             return r
@@ -313,9 +320,9 @@ class RefererMiddleware(object):
             request_referrer = request.headers.get('Referer')
             # we don't patch the referrer value if there is none
             if request_referrer is not None:
-                faked_response = Response(redirected_urls[0])
-                policy_referrer = self.policy(faked_response,
-                    request).referrer(faked_response, request)
+                initial_url = redirected_urls[0]
+                policy_referrer = self.policy(initial_url,
+                    request).referrer(orig_url, request.url)
                 if policy_referrer != request_referrer:
                     if policy_referrer is None:
                         request.headers.pop('Referer')
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 9864f353d..8eed31060 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -117,12 +117,7 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=
     - `strip_fragment` drops any #fragment component
     """
 
-    if url is None:
-        return None
-    if not isinstance(url, ParseResult):
-        parsed_url = urlparse(url)
-    else:
-        parsed_url = url
+    parsed_url = urlparse(url)
     netloc = parsed_url.netloc
     if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password):
         netloc = netloc.split('@')[-1]

From d2aa51c0fb81b4e264fc351b9184d9ce06855021 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 27 Feb 2017 16:05:22 +0100
Subject: [PATCH 1368/2974] Update tests

---
 tests/test_spidermiddleware_referer.py | 43 ++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 3 deletions(-)

diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 81868efab..39bbaab5d 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,3 +1,4 @@
+from six.moves.urllib.parse import urlparse
 from unittest import TestCase
 
 from scrapy.exceptions import NotConfigured
@@ -326,9 +327,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy):
     depending on the scheme of the target URL.
     """
     def referrer(self, response, request):
-        from scrapy.utils.httpobj import urlparse_cached
-
-        scheme = urlparse_cached(request).scheme
+        scheme = urlparse(request).scheme
         if scheme == 'https':
             return b'https://python.org/'
         elif scheme == 'http':
@@ -446,3 +445,41 @@ class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefere
 class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
     resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
+
+
+class TestReferrerPolicyOnRedirect(TestRefererMiddleware):
+
+    req_meta = {}
+    resp_headers = {}
+    #settings = {}
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
+    scenarii = [
+        (   # origin
+            'http://scrapytest.org/1',
+
+            # target + redirection
+            ['http://scrapytest.org/2',
+             'http://scrapytest.org/3',],
+
+            b'http://scrapytest.org/1', # expected initial referer
+            b'http://scrapytest.org/1', # expected referer for the redirection request
+
+        ),
+    ]
+
+    def setUp(self):
+        self.spider = Spider('foo')
+        settings = Settings(self.settings)
+        self.mw = RefererMiddleware(settings)
+
+    def test_(self):
+
+        for origin, target_chain, init_referrer, final_referrer in self.scenarii:
+            response = self.get_response(origin)
+            request = self.get_request(target_chain.pop())
+
+
+            out = list(self.mw.process_spider_output(response, [request], self.spider))
+            self.assertEquals(out[0].headers.get('Referer'), init_referrer)
+
+            request.meta['redirected_urls'] = target_chain

From 8226e77010d79f64c7ec5ebdb9e7cee7803600ed Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 1 Mar 2017 12:41:33 +0100
Subject: [PATCH 1369/2974] Add test for Referer header on HTTP redirections

---
 scrapy/spidermiddlewares/referer.py    |  16 +-
 tests/test_spidermiddleware_referer.py | 398 ++++++++++++++++++++++---
 2 files changed, 364 insertions(+), 50 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index f2910a0c8..bb184cc12 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -4,6 +4,8 @@ originated it.
 """
 from six.moves.urllib.parse import urlparse
 
+from w3lib.url import safe_url_string
+
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
@@ -300,8 +302,7 @@ class RefererMiddleware(object):
             if isinstance(resp_or_url, Response):
                 policy_name = to_native_str(
                     resp_or_url.headers.get('Referrer-Policy', '').decode('latin1'))
-
-        cls = _policy_classes.get(policy_name.lower(), self.default_policy)
+        cls = _policy_classes.get(policy_name.lower()) if policy_name else self.default_policy
         return cls()
 
     def process_spider_output(self, response, result, spider):
@@ -320,9 +321,14 @@ class RefererMiddleware(object):
             request_referrer = request.headers.get('Referer')
             # we don't patch the referrer value if there is none
             if request_referrer is not None:
-                initial_url = redirected_urls[0]
-                policy_referrer = self.policy(initial_url,
-                    request).referrer(orig_url, request.url)
+                # the request's referrer header value acts as a surrogate
+                # for the parent response URL
+                #
+                # Note: if the 3xx response contained a Referrer-Policy header,
+                #       the information is not available using this hook
+                parent_url = safe_url_string(request_referrer)
+                policy_referrer = self.policy(parent_url, request).referrer(
+                    parent_url, request.url)
                 if policy_referrer != request_referrer:
                     if policy_referrer is None:
                         request.headers.pop('Referer')
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 39bbaab5d..28c694169 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -5,6 +5,7 @@ from scrapy.exceptions import NotConfigured
 from scrapy.http import Response, Request
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
+from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
 from scrapy.spidermiddlewares.referer import RefererMiddleware, \
     POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \
     POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \
@@ -13,6 +14,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \
     DefaultReferrerPolicy, \
     NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \
     OriginWhenCrossOriginPolicy, OriginPolicy, \
+    StrictOriginWhenCrossOriginPolicy, StrictOriginPolicy, \
     SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy
 
 
@@ -289,35 +291,35 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware):
 
 
 # --- Tests using settings to set policy using class path
-class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware):
+class TestSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'}
 
 
-class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
+class TestSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
 
 
-class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware):
+class TestSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
 
 
-class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware):
+class TestSettingsOrigin(MixinOrigin, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}
 
 
-class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
+class TestSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'}
 
 
-class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
+class TestSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
 
 
-class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
+class TestSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'}
 
 
-class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
+class TestSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
 
 
@@ -334,7 +336,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy):
             return b'http://python.org/'
 
 
-class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware):
+class TestSettingsCustomPolicy(TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}
     scenarii = [
         ('https://example.com/',    'https://scrapy.org/',  b'https://python.org/'),
@@ -347,58 +349,58 @@ class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware):
     ]
 
 # --- Tests using Request meta dict to set policy
-class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware):
+class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT}
 
 
-class TestRefererMiddlewareNoReferrer(MixinNoReferrer, TestRefererMiddleware):
+class TestRequestMetaNoReferrer(MixinNoReferrer, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_NO_REFERRER}
 
 
-class TestRefererMiddlewareNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
+class TestRequestMetaNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}
 
 
-class TestRefererMiddlewareSameOrigin(MixinSameOrigin, TestRefererMiddleware):
+class TestRequestMetaSameOrigin(MixinSameOrigin, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_SAME_ORIGIN}
 
 
-class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware):
+class TestRequestMetaOrigin(MixinOrigin, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_ORIGIN}
 
 
-class TestRefererMiddlewareSrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
+class TestRequestMetaSrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN}
 
 
-class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
+class TestRequestMetaOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
 
 
-class TestRefererMiddlewareStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
+class TestRequestMetaStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
 
 
-class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
+class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
 
 
-class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
+class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
 
 
-class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):
+class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
     req_meta = {'referrer_policy': POLICY_NO_REFERRER}
 
 
-class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):
+class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
 
 
-class TestRefererMiddlewareSettingsPolicyByName(TestCase):
+class TestSettingsPolicyByName(TestCase):
 
     def test_valid_name(self):
         for s, p in [
@@ -407,7 +409,9 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):
                 (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
                 (POLICY_SAME_ORIGIN, SameOriginPolicy),
                 (POLICY_ORIGIN, OriginPolicy),
+                (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
                 (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
+                (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
                 (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
             ]:
             settings = Settings({'REFERRER_POLICY': s})
@@ -421,7 +425,9 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):
                 (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
                 (POLICY_SAME_ORIGIN, SameOriginPolicy),
                 (POLICY_ORIGIN, OriginPolicy),
+                (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
                 (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
+                (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
                 (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
             ]:
             settings = Settings({'REFERRER_POLICY': s.upper()})
@@ -434,52 +440,354 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):
             mw = RefererMiddleware(settings)
 
 
-class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
+class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
     resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}
 
-class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
+class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
     resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}
 
-class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
+class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
     resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
 
 
-class TestReferrerPolicyOnRedirect(TestRefererMiddleware):
+class TestReferrerOnRedirect(TestRefererMiddleware):
 
-    req_meta = {}
-    resp_headers = {}
-    #settings = {}
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
     scenarii = [
-        (   # origin
-            'http://scrapytest.org/1',
-
-            # target + redirection
-            ['http://scrapytest.org/2',
-             'http://scrapytest.org/3',],
-
+        (   'http://scrapytest.org/1',      # parent
+            'http://scrapytest.org/2',      # target
+            (
+                # redirections: code, URL
+                (301, 'http://scrapytest.org/3'),
+                (301, 'http://scrapytest.org/4'),
+            ),
             b'http://scrapytest.org/1', # expected initial referer
             b'http://scrapytest.org/1', # expected referer for the redirection request
-
+        ),
+        (   'https://scrapytest.org/1',
+            'https://scrapytest.org/2',
+            (
+                # redirecting to non-secure URL
+                (301, 'http://scrapytest.org/3'),
+            ),
+            b'https://scrapytest.org/1',
+            b'https://scrapytest.org/1',
+        ),
+        (   'https://scrapytest.org/1',
+            'https://scrapytest.com/2',
+            (
+                # redirecting to non-secure URL: different origin
+                (301, 'http://scrapytest.com/3'),
+            ),
+            b'https://scrapytest.org/1',
+            b'https://scrapytest.org/1',
         ),
     ]
 
     def setUp(self):
         self.spider = Spider('foo')
         settings = Settings(self.settings)
-        self.mw = RefererMiddleware(settings)
+        self.referrermw = RefererMiddleware(settings)
+        self.redirectmw = RedirectMiddleware(settings)
 
-    def test_(self):
+    def test(self):
 
-        for origin, target_chain, init_referrer, final_referrer in self.scenarii:
-            response = self.get_response(origin)
-            request = self.get_request(target_chain.pop())
+        for parent, target, redirections, init_referrer, final_referrer in self.scenarii:
+            response = self.get_response(parent)
+            request = self.get_request(target)
 
-
-            out = list(self.mw.process_spider_output(response, [request], self.spider))
+            out = list(self.referrermw.process_spider_output(response, [request], self.spider))
             self.assertEquals(out[0].headers.get('Referer'), init_referrer)
 
-            request.meta['redirected_urls'] = target_chain
+            for status, url in redirections:
+                response = Response(request.url, headers={'Location': url}, status=status)
+                request = self.redirectmw.process_response(request, response, self.spider)
+                self.referrermw.request_scheduled(request, self.spider)
+
+            assert isinstance(request, Request)
+            self.assertEquals(request.headers.get('Referer'), final_referrer)
+
+
+class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
+    """
+    No Referrer policy never sets the "Referer" header.
+    HTTP redirections should not change that.
+    """
+    settings = {'REFERRER_POLICY': 'no-referrer'}
+    scenarii = [
+        (   'http://scrapytest.org/1',      # parent
+            'http://scrapytest.org/2',      # target
+            (
+                # redirections: code, URL
+                (301, 'http://scrapytest.org/3'),
+                (301, 'http://scrapytest.org/4'),
+            ),
+            None, # expected initial "Referer"
+            None, # expected "Referer" for the redirection request
+        ),
+        (   'https://scrapytest.org/1',
+            'https://scrapytest.org/2',
+            (
+                (301, 'http://scrapytest.org/3'),
+            ),
+            None,
+            None,
+        ),
+        (   'https://scrapytest.org/1',
+            'https://example.com/2',    # different origin
+            (
+                (301, 'http://scrapytest.com/3'),
+            ),
+            None,
+            None,
+        ),
+    ]
+
+
+class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
+    """
+    Same Origin policy sends the full URL as "Referer" if the target origin
+    is the same as the parent response (same protocol, same domain, same port).
+
+    HTTP redirections to a different domain or a lower secure level
+    should have the "Referer" removed.
+    """
+    settings = {'REFERRER_POLICY': 'same-origin'}
+    scenarii = [
+        (   'http://scrapytest.org/101',      # origin
+            'http://scrapytest.org/102',      # target
+            (
+                # redirections: code, URL
+                (301, 'http://scrapytest.org/103'),
+                (301, 'http://scrapytest.org/104'),
+            ),
+            b'http://scrapytest.org/101', # expected initial "Referer"
+            b'http://scrapytest.org/101', # expected referer for the redirection request
+        ),
+        (   'https://scrapytest.org/201',
+            'https://scrapytest.org/202',
+            (
+                # redirecting from secure to non-secure URL == different origin
+                (301, 'http://scrapytest.org/203'),
+            ),
+            b'https://scrapytest.org/201',
+            None,
+        ),
+        (   'https://scrapytest.org/301',
+            'https://scrapytest.org/302',
+            (
+                # different domain == different origin
+                (301, 'http://example.com/303'),
+            ),
+            b'https://scrapytest.org/301',
+            None,
+        ),
+    ]
+
+
+class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
+    """
+    Strict Origin policy will always send the "origin" as referrer
+    (think of it as the parent URL without the path part),
+    unless the security level is lower and no "Referer" is sent.
+
+    Redirections from secure to non-secure URLs should have the
+    "Referrer" header removed if necessary.
+    """
+    settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN}
+    scenarii = [
+        (   'http://scrapytest.org/101',
+            'http://scrapytest.org/102',
+            (
+                (301, 'http://scrapytest.org/103'),
+                (301, 'http://scrapytest.org/104'),
+            ),
+            b'http://scrapytest.org/',  # send origin
+            b'http://scrapytest.org/',  # redirects to same origin: send origin
+        ),
+        (   'https://scrapytest.org/201',
+            'https://scrapytest.org/202',
+            (
+                # redirecting to non-secure URL: no referrer
+                (301, 'http://scrapytest.org/203'),
+            ),
+            b'https://scrapytest.org/',
+            None,
+        ),
+        (   'https://scrapytest.org/301',
+            'https://scrapytest.org/302',
+            (
+                # redirecting to non-secure URL (different domain): no referrer
+                (301, 'http://example.com/303'),
+            ),
+            b'https://scrapytest.org/',
+            None,
+        ),
+        (   'http://scrapy.org/401',
+            'http://example.com/402',
+            (
+                (301, 'http://scrapytest.org/403'),
+            ),
+            b'http://scrapy.org/',
+            b'http://scrapy.org/',
+        ),
+        (   'https://scrapy.org/501',
+            'https://example.com/502',
+            (
+                # HTTPS all along, so origin referrer is kept as-is
+                (301, 'https://google.com/503'),
+                (301, 'https://facebook.com/504'),
+            ),
+            b'https://scrapy.org/',
+            b'https://scrapy.org/',
+        ),
+        (   'https://scrapytest.org/601',
+            'http://scrapytest.org/602',                # TLS to non-TLS: no referrer
+            (
+                (301, 'https://scrapytest.org/603'),    # TLS URL again: (still) no referrer
+            ),
+            None,
+            None,
+        ),
+    ]
+
+
+class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
+    """
+    Origin When Cross-Origin policy sends the full URL as "Referer",
+    unless the target's origin is different (different domain, different protocol)
+    in which case only the origin is sent.
+
+    Redirections to a different origin should strip the "Referer"
+    to the parent origin.
+    """
+    settings = {'REFERRER_POLICY': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
+    scenarii = [
+        (   'http://scrapytest.org/101',      # origin
+            'http://scrapytest.org/102',      # target + redirection
+            (
+                # redirections: code, URL
+                (301, 'http://scrapytest.org/103'),
+                (301, 'http://scrapytest.org/104'),
+            ),
+            b'http://scrapytest.org/101', # expected initial referer
+            b'http://scrapytest.org/101', # expected referer for the redirection request
+        ),
+        (   'https://scrapytest.org/201',
+            'https://scrapytest.org/202',
+            (
+                # redirecting to non-secure URL: send origin
+                (301, 'http://scrapytest.org/203'),
+            ),
+            b'https://scrapytest.org/201',
+            b'https://scrapytest.org/',
+        ),
+        (   'https://scrapytest.org/301',
+            'https://scrapytest.org/302',
+            (
+                # redirecting to non-secure URL (different domain): send origin
+                (301, 'http://example.com/303'),
+            ),
+            b'https://scrapytest.org/301',
+            b'https://scrapytest.org/',
+        ),
+        (   'http://scrapy.org/401',
+            'http://example.com/402',
+            (
+                (301, 'http://scrapytest.org/403'),
+            ),
+            b'http://scrapy.org/',
+            b'http://scrapy.org/',
+        ),
+        (   'https://scrapy.org/501',
+            'https://example.com/502',
+            (
+                # all different domains: send origin
+                (301, 'https://google.com/503'),
+                (301, 'https://facebook.com/504'),
+            ),
+            b'https://scrapy.org/',
+            b'https://scrapy.org/',
+        ),
+        (   'https://scrapytest.org/301',
+            'http://scrapytest.org/302',                # TLS to non-TLS: send origin
+            (
+                (301, 'https://scrapytest.org/303'),    # TLS URL again: send origin (also)
+            ),
+            b'https://scrapytest.org/',
+            b'https://scrapytest.org/',
+        ),
+    ]
+
+
+class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
+    """
+    Strict Origin When Cross-Origin policy sends the full URL as "Referer",
+    unless the target's origin is different (different domain, different protocol)
+    in which case only the origin is sent...
+    Unless there's also a downgrade in security and then the "Referer" header
+    is not sent.
+
+    Redirections to a different origin should strip the "Referer" to the parent origin,
+    and from https:// to http:// will remove the "Referer" header.
+    """
+    settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
+    scenarii = [
+        (   'http://scrapytest.org/101',      # origin
+            'http://scrapytest.org/102',      # target + redirection
+            (
+                # redirections: code, URL
+                (301, 'http://scrapytest.org/103'),
+                (301, 'http://scrapytest.org/104'),
+            ),
+            b'http://scrapytest.org/101', # expected initial referer
+            b'http://scrapytest.org/101', # expected referer for the redirection request
+        ),
+        (   'https://scrapytest.org/201',
+            'https://scrapytest.org/202',
+            (
+                # redirecting to non-secure URL: do not send the "Referer" header
+                (301, 'http://scrapytest.org/203'),
+            ),
+            b'https://scrapytest.org/201',
+            None,
+        ),
+        (   'https://scrapytest.org/301',
+            'https://scrapytest.org/302',
+            (
+                # redirecting to non-secure URL (different domain): send origin
+                (301, 'http://example.com/303'),
+            ),
+            b'https://scrapytest.org/301',
+            None,
+        ),
+        (   'http://scrapy.org/401',
+            'http://example.com/402',
+            (
+                (301, 'http://scrapytest.org/403'),
+            ),
+            b'http://scrapy.org/',
+            b'http://scrapy.org/',
+        ),
+        (   'https://scrapy.org/501',
+            'https://example.com/502',
+            (
+                # all different domains: send origin
+                (301, 'https://google.com/503'),
+                (301, 'https://facebook.com/504'),
+            ),
+            b'https://scrapy.org/',
+            b'https://scrapy.org/',
+        ),
+        (   'https://scrapytest.org/601',
+            'http://scrapytest.org/602',                # TLS to non-TLS: do not send "Referer"
+            (
+                (301, 'https://scrapytest.org/603'),    # TLS URL again: (still) send nothing
+            ),
+            None,
+            None,
+        ),
+    ]

From 6916dd6240940cab06e2d02b3fceeb32c70691de Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 1 Mar 2017 17:42:46 +0100
Subject: [PATCH 1370/2974] Warn or fail with exception on unknown policies

---
 scrapy/spidermiddlewares/referer.py | 55 +++++++++++++++++++++--------
 1 file changed, 40 insertions(+), 15 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index bb184cc12..5ed40791b 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -3,6 +3,7 @@ RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
 from six.moves.urllib.parse import urlparse
+import warnings
 
 from w3lib.url import safe_url_string
 
@@ -265,45 +266,69 @@ _policy_classes = {p.name: p for p in (
     DefaultReferrerPolicy,
 )}
 
+
+def _load_policy_class(policy, warning_only=False):
+    """
+    Expect a string for the path to the policy class,
+    otherwise try to interpret the string as a standard value
+    from https://www.w3.org/TR/referrer-policy/#referrer-policies
+    """
+    try:
+        return load_object(policy)
+    except ValueError:
+        try:
+            return _policy_classes[policy.lower()]
+        except KeyError:
+            msg = "Could not load referrer policy %r" % policy
+            if not warning_only:
+                raise RuntimeError(msg)
+            else:
+                warnings.warn(msg, RuntimeWarning)
+                return None
+
+
 class RefererMiddleware(object):
 
     def __init__(self, settings=None):
         self.default_policy = DefaultReferrerPolicy
         if settings is not None:
-            policy = settings.get('REFERRER_POLICY')
-            if policy is not None:
-                # expect a string for the path to the policy class
-                try:
-                    self.default_policy = load_object(policy)
-                except ValueError:
-                    # otherwise try to interpret the string as standard
-                    # https://www.w3.org/TR/referrer-policy/#referrer-policies
-                    try:
-                        self.default_policy = _policy_classes[policy.lower()]
-                    except:
-                        raise NotConfigured("Unknown referrer policy name %r" % policy)
+            self.default_policy = _load_policy_class(
+                settings.get('REFERRER_POLICY'))
 
     @classmethod
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('REFERER_ENABLED'):
             raise NotConfigured
         mw = cls(crawler.settings)
+
+        # Note: this hook is a bit of a hack to intercept redirections
         crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled)
+
         return mw
 
     def policy(self, resp_or_url, request):
         """
         Determine Referrer-Policy to use from a parent Response (or URL),
         and a Request to be sent.
+
+        - if a valid policy is set in Request meta, it is used.
+        - if the policy is set in meta but is wrong (e.g. a typo error),
+          the policy from settings is used
+        - if the policy is not set in Request meta,
+          but there is a Referrer-policy header in the parent response,
+          it is used if valid
+        - otherwise, the policy from settings is used.
         """
-        # policy set in request's meta dict takes precedence over default policy
         policy_name = request.meta.get('referrer_policy')
         if policy_name is None:
             if isinstance(resp_or_url, Response):
                 policy_name = to_native_str(
                     resp_or_url.headers.get('Referrer-Policy', '').decode('latin1'))
-        cls = _policy_classes.get(policy_name.lower()) if policy_name else self.default_policy
-        return cls()
+        if policy_name is None:
+            return self.default_policy()
+
+        cls = _load_policy_class(policy_name, warning_only=True)
+        return cls() if cls else self.default_policy()
 
     def process_spider_output(self, response, result, spider):
         def _set_referer(r):

From efa50039ec4dc8d5cd5103e64c47ad8c86b9ed4e Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 1 Mar 2017 17:43:47 +0100
Subject: [PATCH 1371/2974] Add tests for policy fallback on unknown policies
 from meta and headers

---
 tests/test_spidermiddleware_referer.py | 76 +++++++++++++++++++++++++-
 1 file changed, 75 insertions(+), 1 deletion(-)

diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 28c694169..b1c815876 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,5 +1,6 @@
 from six.moves.urllib.parse import urlparse
 from unittest import TestCase
+import warnings
 
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response, Request
@@ -400,6 +401,79 @@ class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
 
 
+class TestRequestMetaSettingFallback(TestCase):
+
+    params = [
+        (
+            # When an unknown policy is referenced in Request.meta
+            # (here, a typo error),
+            # the policy defined in settings takes precedence
+            {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'},
+            {},
+            {'referrer_policy': 'ssscrapy-default'},
+            OriginWhenCrossOriginPolicy,
+            True
+        ),
+        (
+            # same as above but with string value for settings policy
+            {'REFERRER_POLICY': 'origin-when-cross-origin'},
+            {},
+            {'referrer_policy': 'ssscrapy-default'},
+            OriginWhenCrossOriginPolicy,
+            True
+        ),
+        (
+            # request meta references a wrong policy but it is set,
+            # so the Referrer-Policy header in response is not used,
+            # and the settings' policy is applied
+            {'REFERRER_POLICY': 'origin-when-cross-origin'},
+            {'Referrer-Policy': 'unsafe-url'},
+            {'referrer_policy': 'ssscrapy-default'},
+            OriginWhenCrossOriginPolicy,
+            True
+        ),
+        (
+            # here, request meta does not set the policy
+            # so response headers take precedence
+            {'REFERRER_POLICY': 'origin-when-cross-origin'},
+            {'Referrer-Policy': 'unsafe-url'},
+            {},
+            UnsafeUrlPolicy,
+            False
+        ),
+        (
+            # here, request meta does not set the policy,
+            # but response headers also use an unknown policy,
+            # so the settings' policy is used
+            {'REFERRER_POLICY': 'origin-when-cross-origin'},
+            {'Referrer-Policy': 'unknown'},
+            {},
+            OriginWhenCrossOriginPolicy,
+            True
+        )
+    ]
+
+    def test(self):
+
+        origin = 'http://www.scrapy.org'
+        target = 'http://www.example.com'
+
+        for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]:
+            spider = Spider('foo')
+            mw = RefererMiddleware(Settings(settings))
+
+            response = Response(origin, headers=response_headers)
+            request = Request(target, meta=request_meta)
+
+            with warnings.catch_warnings(record=True) as w:
+                policy = mw.policy(response, request)
+                self.assertIsInstance(policy, policy_class)
+
+                if check_warning:
+                    self.assertEqual(len(w), 1)
+                    self.assertEqual(w[0].category, RuntimeWarning, w[0].message)
+
+
 class TestSettingsPolicyByName(TestCase):
 
     def test_valid_name(self):
@@ -436,7 +510,7 @@ class TestSettingsPolicyByName(TestCase):
 
     def test_invalid_name(self):
         settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'})
-        with self.assertRaises(NotConfigured):
+        with self.assertRaises(RuntimeError):
             mw = RefererMiddleware(settings)
 
 

From 2d55d838ca1e2936e5729e67a6ab34a4bb58ba3b Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 1 Mar 2017 20:59:52 +0100
Subject: [PATCH 1372/2974] Fix strip_url() tests

---
 tests/test_utils_url.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 9182d0fda..c2b9fc176 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -274,7 +274,6 @@ class StripUrl(unittest.TestCase):
              'http://www.example.com/'),
             ]:
             self.assertEqual(strip_url(input_url, origin_only=origin), output_url)
-            self.assertEqual(strip_url(urlparse(input_url), origin_only=origin), output_url)
 
     def test_credentials(self):
         for i, o in [
@@ -288,7 +287,6 @@ class StripUrl(unittest.TestCase):
              'ftp://www.example.com/index.html?somekey=somevalue'),
             ]:
             self.assertEqual(strip_url(i, strip_credentials=True), o)
-            self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o)
 
     def test_credentials_encoded_delims(self):
         for i, o in [
@@ -308,7 +306,6 @@ class StripUrl(unittest.TestCase):
              'ftp://www.example.com/index.html?somekey=somevalue'),
             ]:
             self.assertEqual(strip_url(i, strip_credentials=True), o)
-            self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o)
 
     def test_default_ports_creds_off(self):
         for i, o in [
@@ -337,7 +334,6 @@ class StripUrl(unittest.TestCase):
              'ftp://www.example.com:221/file.txt'),
             ]:
             self.assertEqual(strip_url(i), o)
-            self.assertEqual(strip_url(urlparse(i)), o)
 
     def test_default_ports(self):
         for i, o in [
@@ -366,7 +362,6 @@ class StripUrl(unittest.TestCase):
              'ftp://username:password@www.example.com:221/file.txt'),
             ]:
             self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o)
-            self.assertEqual(strip_url(urlparse(i), strip_default_port=True, strip_credentials=False), o)
 
     def test_default_ports_keep(self):
         for i, o in [
@@ -395,7 +390,6 @@ class StripUrl(unittest.TestCase):
              'ftp://username:password@www.example.com:221/file.txt'),
             ]:
             self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o)
-            self.assertEqual(strip_url(urlparse(i), strip_default_port=False, strip_credentials=False), o)
 
     def test_origin_only(self):
         for i, o in [
@@ -412,7 +406,6 @@ class StripUrl(unittest.TestCase):
              'https://www.example.com/'),
             ]:
             self.assertEqual(strip_url(i, origin_only=True), o)
-            self.assertEqual(strip_url(urlparse(i), origin_only=True), o)
 
 
 if __name__ == "__main__":

From a70ec30e19eff13d3bf18e5c392612db018d1b4f Mon Sep 17 00:00:00 2001
From: Oto Brglez <otobrglez@gmail.com>
Date: Wed, 1 Mar 2017 23:02:42 +0100
Subject: [PATCH 1373/2974] Adding new options.

---
 scrapy/pipelines/files.py | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 843b4d3ec..7041a0a78 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -40,7 +40,6 @@ class FileException(Exception):
 
 
 class FSFilesStore(object):
-
     def __init__(self, basedir):
         if '://' in basedir:
             basedir = basedir.split('://', 1)[1]
@@ -79,9 +78,12 @@ class FSFilesStore(object):
 
 
 class S3FilesStore(object):
-
     AWS_ACCESS_KEY_ID = None
     AWS_SECRET_ACCESS_KEY = None
+    AWS_ENDPOINT_URL = None
+    AWS_REGION_NAME = None
+    AWS_USE_SSL = None
+    AWS_VERIFY = None
 
     POLICY = 'private'  # Overriden from settings.FILES_STORE_S3_ACL in
                         # FilesPipeline.from_settings.
@@ -95,8 +97,14 @@ class S3FilesStore(object):
             import botocore.session
             session = botocore.session.get_session()
             self.s3_client = session.create_client(
-                's3', aws_access_key_id=self.AWS_ACCESS_KEY_ID,
-                aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY)
+                's3',
+                aws_access_key_id=self.AWS_ACCESS_KEY_ID,
+                aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
+                endpoint_url=self.AWS_ENDPOINT_URL,
+                region_name=self.AWS_REGION_NAME,
+                use_ssl=self.AWS_USE_SSL,
+                verify=self.AWS_VERIFY
+            )
         else:
             from boto.s3.connection import S3Connection
             self.S3Connection = S3Connection
@@ -181,7 +189,7 @@ class S3FilesStore(object):
             'X-Amz-Grant-Read': 'GrantRead',
             'X-Amz-Grant-Read-ACP': 'GrantReadACP',
             'X-Amz-Grant-Write-ACP': 'GrantWriteACP',
-            })
+        })
         extra = {}
         for key, value in six.iteritems(headers):
             try:
@@ -226,7 +234,7 @@ class FilesPipeline(MediaPipeline):
     def __init__(self, store_uri, download_func=None, settings=None):
         if not store_uri:
             raise NotConfigured
-        
+
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
 
@@ -256,6 +264,10 @@ class FilesPipeline(MediaPipeline):
         s3store = cls.STORE_SCHEMES['s3']
         s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
+        s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']
+        s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']
+        s3store.AWS_USE_SSL = settings['AWS_USE_SSL']
+        s3store.AWS_VERIFY = settings['AWS_VERIFY']
         s3store.POLICY = settings['FILES_STORE_S3_ACL']
 
         store_uri = settings['FILES_STORE']
@@ -423,4 +435,5 @@ class FilesPipeline(MediaPipeline):
     # deprecated
     def file_key(self, url):
         return self.file_path(url)
+
     file_key._base = True

From 97d84d920bf97f3ec1becd291596b3f0ee966328 Mon Sep 17 00:00:00 2001
From: jorenham <jhammudoglu@gmail.com>
Date: Thu, 2 Mar 2017 11:04:16 +0100
Subject: [PATCH 1374/2974] Logging the cache directory at HttpCacheMiddleware
 instantiation #2604

---
 scrapy/downloadermiddlewares/httpcache.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 30e49b886..6f1ccce68 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -1,3 +1,5 @@
+import logging
+
 from email.utils import formatdate
 from twisted.internet import defer
 from twisted.internet.error import TimeoutError, DNSLookupError, \
@@ -9,6 +11,9 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.utils.misc import load_object
 
 
+logger = logging.getLogger(__name__)
+
+
 class HttpCacheMiddleware(object):
 
     DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
@@ -24,6 +29,8 @@ class HttpCacheMiddleware(object):
         self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING')
         self.stats = stats
 
+        logger.debug("Using cache directory %(cachedir)s" % {'cachedir': self.storage.cachedir})
+
     @classmethod
     def from_crawler(cls, crawler):
         o = cls(crawler.settings, crawler.stats)

From f3b75c940d2fefc3ff5bb4435e507c5959ff903c Mon Sep 17 00:00:00 2001
From: MrMenezes <sr.tama@outlook.com>
Date: Fri, 21 Oct 2016 18:02:16 -0300
Subject: [PATCH 1375/2974] Fix warning to duplicated spider. Issue 2181

---
 scrapy/spiderloader.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index d4f0f663f..e6c3e64a5 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -23,7 +23,12 @@ class SpiderLoader(object):
 
     def _load_spiders(self, module):
         for spcls in iter_spider_classes(module):
-            self._spiders[spcls.name] = spcls
+            if spcls.name in self._spiders.keys():
+                import warnings
+                warnings.warn("There are several spiders with the same name (" + spcls.name +
+                              "), this can cause unexpected behavior", UserWarning)
+        self._spiders[spcls.name] = spcls
+    
 
     def _load_all_spiders(self):
         for name in self.spider_modules:

From 6abd9ba843e54ecb869af9571192c3f33375a2b6 Mon Sep 17 00:00:00 2001
From: Erick <sr.tama@outlook.com>
Date: Fri, 21 Oct 2016 18:24:59 -0300
Subject: [PATCH 1376/2974] Fix warning to duplicated spider. Issue 2181

---
 scrapy/spiderloader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index e6c3e64a5..6093c07c6 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -27,7 +27,7 @@ class SpiderLoader(object):
                 import warnings
                 warnings.warn("There are several spiders with the same name (" + spcls.name +
                               "), this can cause unexpected behavior", UserWarning)
-        self._spiders[spcls.name] = spcls
+            self._spiders[spcls.name] = spcls
     
 
     def _load_all_spiders(self):

From 5be5ef57f38b5d6f8ed01ccf9c9cac1e1e02fe03 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 23 Nov 2016 11:01:31 +0100
Subject: [PATCH 1377/2974] Remove extra blank line

---
 scrapy/spiderloader.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 6093c07c6..1322c01d1 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -28,7 +28,6 @@ class SpiderLoader(object):
                 warnings.warn("There are several spiders with the same name (" + spcls.name +
                               "), this can cause unexpected behavior", UserWarning)
             self._spiders[spcls.name] = spcls
-    
 
     def _load_all_spiders(self):
         for name in self.spider_modules:

From e71803c833edd67400848a3b0d4dbb01e0ec80f7 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 23 Nov 2016 11:52:02 +0100
Subject: [PATCH 1378/2974] Add tests for duplicate spider name warnings

---
 tests/test_spiderloader/__init__.py | 49 +++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index b2ad93b3f..ac5f0ddab 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -101,3 +101,52 @@ class SpiderLoaderTest(unittest.TestCase):
 
             spiders = spider_loader.list()
             self.assertEqual(spiders, [])
+
+
+class DuplicateSpiderNameLoaderTest(unittest.TestCase):
+
+    def setUp(self):
+        orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
+        self.tmpdir = self.mktemp()
+        os.mkdir(self.tmpdir)
+        self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
+        shutil.copytree(orig_spiders_dir, self.spiders_dir)
+        sys.path.append(self.tmpdir)
+        self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
+
+    def tearDown(self):
+        del sys.modules['test_spiders_xxx']
+        sys.path.remove(self.tmpdir)
+
+    def test_dupename_warning(self):
+        # copy 1 spider module so as to have duplicate spider name
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider3.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider3dupe.py'))
+
+        with warnings.catch_warnings(record=True) as w:
+            spider_loader = SpiderLoader.from_settings(self.settings)
+
+            self.assertEqual(len(w), 1)
+            self.assertIn("several spiders with the same name (spider3)", str(w[0].message))
+
+            spiders = set(spider_loader.list())
+            self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3']))
+
+    def test_multiple_dupename_warning(self):
+        # copy 2 spider modules so as to have duplicate spider name
+        # This should issue 2 warning, 1 for each duplicate spider name
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider1.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider1dupe.py'))
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider2.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider2dupe.py'))
+
+        with warnings.catch_warnings(record=True) as w:
+            spider_loader = SpiderLoader.from_settings(self.settings)
+
+            self.assertEqual(len(w), 2)
+            msgs = sorted(str(wrn.message) for wrn in w)
+            self.assertIn("several spiders with the same name (spider1)", msgs[0])
+            self.assertIn("several spiders with the same name (spider2)", msgs[1])
+
+            spiders = set(spider_loader.list())
+            self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3']))

From 12a8ddecab8c64923d1789571955699270255211 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 2 Mar 2017 13:03:18 +0100
Subject: [PATCH 1379/2974] Fix tests

---
 tests/test_spiderloader/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index ac5f0ddab..302bf9a10 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -130,7 +130,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             self.assertIn("several spiders with the same name (spider3)", str(w[0].message))
 
             spiders = set(spider_loader.list())
-            self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3']))
+            self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))
 
     def test_multiple_dupename_warning(self):
         # copy 2 spider modules so as to have duplicate spider name
@@ -149,4 +149,4 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             self.assertIn("several spiders with the same name (spider2)", msgs[1])
 
             spiders = set(spider_loader.list())
-            self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3']))
+            self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))

From f96490df2ce532e3b8757b2464f5f3cc54181ce5 Mon Sep 17 00:00:00 2001
From: jorenham <jhammudoglu@gmail.com>
Date: Thu, 2 Mar 2017 16:17:51 +0100
Subject: [PATCH 1380/2974] Move cache storage logging to the individual
 storage classes

---
 scrapy/downloadermiddlewares/httpcache.py |  7 -------
 scrapy/extensions/httpcache.py            | 10 ++++++++++
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 6f1ccce68..30e49b886 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -1,5 +1,3 @@
-import logging
-
 from email.utils import formatdate
 from twisted.internet import defer
 from twisted.internet.error import TimeoutError, DNSLookupError, \
@@ -11,9 +9,6 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.utils.misc import load_object
 
 
-logger = logging.getLogger(__name__)
-
-
 class HttpCacheMiddleware(object):
 
     DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
@@ -29,8 +24,6 @@ class HttpCacheMiddleware(object):
         self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING')
         self.stats = stats
 
-        logger.debug("Using cache directory %(cachedir)s" % {'cachedir': self.storage.cachedir})
-
     @classmethod
     def from_crawler(cls, crawler):
         o = cls(crawler.settings, crawler.stats)
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 247cac64e..8025efe77 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,6 +1,7 @@
 from __future__ import print_function
 import os
 import gzip
+import logging
 from six.moves import cPickle as pickle
 from importlib import import_module
 from time import time
@@ -15,6 +16,9 @@ from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_bytes, to_unicode
 
 
+logger = logging.getLogger(__name__)
+
+
 class DummyPolicy(object):
 
     def __init__(self, settings):
@@ -216,6 +220,8 @@ class DbmCacheStorage(object):
         self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE'])
         self.db = None
 
+        logger.debug("Using DBM cache storage in %(cachedir)s" % {'cachedir': self.cachedir})
+
     def open_spider(self, spider):
         dbpath = os.path.join(self.cachedir, '%s.db' % spider.name)
         self.db = self.dbmodule.open(dbpath, 'c')
@@ -271,6 +277,8 @@ class FilesystemCacheStorage(object):
         self.use_gzip = settings.getbool('HTTPCACHE_GZIP')
         self._open = gzip.open if self.use_gzip else open
 
+        logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir})
+
     def open_spider(self, spider):
         pass
 
@@ -344,6 +352,8 @@ class LeveldbCacheStorage(object):
         self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
         self.db = None
 
+        logger.debug("Using LevelDB cache storage in %(cachedir)s" % {'cachedir': self.cachedir})
+
     def open_spider(self, spider):
         dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name)
         self.db = self._leveldb.LevelDB(dbpath)

From b50d0370f4ebdb485f4625ef6d71398ac8538c79 Mon Sep 17 00:00:00 2001
From: Artur Gaspar <artur.gaspar.00@gmail.com>
Date: Thu, 2 Mar 2017 14:46:33 -0300
Subject: [PATCH 1381/2974] Test response attributes in data URI download
 handler.

---
 tests/test_downloader_handlers.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index b27245a36..74203dbfe 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -838,6 +838,16 @@ class DataURITestCase(unittest.TestCase):
         self.download_request = self.download_handler.download_request
         self.spider = Spider('foo')
 
+    def test_response_attrs(self):
+        uri = "data:,A%20brief%20note"
+
+        def _test(response):
+            self.assertEquals(response.url, uri)
+            self.assertFalse(response.headers)
+
+        request = Request(uri)
+        return self.download_request(request, self.spider).addCallback(_test)
+
     def test_default_mediatype_encoding(self):
         def _test(response):
             self.assertEquals(response.text, 'A brief note')

From c2c503192f24a198b4515ab006e3174e28062731 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 2 Mar 2017 22:53:27 +0100
Subject: [PATCH 1382/2974] Rename arguments

---
 scrapy/spidermiddlewares/referer.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 5ed40791b..2bc8b1782 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -36,13 +36,13 @@ class ReferrerPolicy(object):
     def referrer(self, response, request):
         raise NotImplementedError()
 
-    def stripped_referrer(self, r):
-        if urlparse(r).scheme not in self.NOREFERRER_SCHEMES:
-            return self.strip_url(r)
+    def stripped_referrer(self, url):
+        if urlparse(url).scheme not in self.NOREFERRER_SCHEMES:
+            return self.strip_url(url)
 
-    def origin_referrer(self, r):
-        if urlparse(r).scheme not in self.NOREFERRER_SCHEMES:
-            return self.origin(r)
+    def origin_referrer(self, url):
+        if urlparse(url).scheme not in self.NOREFERRER_SCHEMES:
+            return self.origin(url)
 
     def strip_url(self, url, origin_only=False):
         """
@@ -66,9 +66,9 @@ class ReferrerPolicy(object):
                          strip_default_port=True,
                          origin_only=origin_only)
 
-    def origin(self, r):
+    def origin(self, url):
         """Return serialized origin (scheme, host, path) for a request or response URL."""
-        return self.strip_url(r, origin_only=True)
+        return self.strip_url(url, origin_only=True)
 
     def potentially_trustworthy(self, url):
         # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy
@@ -241,7 +241,7 @@ class UnsafeUrlPolicy(ReferrerPolicy):
 
 class LegacyPolicy(ReferrerPolicy):
     def referrer(self, response, request):
-        return response.url
+        return response
 
 
 class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):

From db176f872b91c4f51f10ea1e732875dc40a9e7bb Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 2 Mar 2017 22:56:23 +0100
Subject: [PATCH 1383/2974] Remove Legacy Policy which is equivalent to
 UnsafeUrl Policy

---
 scrapy/spidermiddlewares/referer.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 2bc8b1782..7b41bfaf7 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -239,11 +239,6 @@ class UnsafeUrlPolicy(ReferrerPolicy):
         return self.stripped_referrer(response)
 
 
-class LegacyPolicy(ReferrerPolicy):
-    def referrer(self, response, request):
-        return response
-
-
 class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
     """
     A variant of "no-referrer-when-downgrade",

From fad499ab602e7f12bf56a514f5aae1a2f80670f7 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 2 Mar 2017 23:06:04 +0100
Subject: [PATCH 1384/2974] Rename arguments (bis)

---
 scrapy/spidermiddlewares/referer.py | 58 ++++++++++++++---------------
 1 file changed, 29 insertions(+), 29 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 7b41bfaf7..b444e34bb 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -33,7 +33,7 @@ class ReferrerPolicy(object):
 
     NOREFERRER_SCHEMES = LOCAL_SCHEMES
 
-    def referrer(self, response, request):
+    def referrer(self, response_url, request_url):
         raise NotImplementedError()
 
     def stripped_referrer(self, url):
@@ -91,7 +91,7 @@ class NoReferrerPolicy(ReferrerPolicy):
     """
     name = POLICY_NO_REFERRER
 
-    def referrer(self, response, request):
+    def referrer(self, response_url, request_url):
         return None
 
 
@@ -111,9 +111,9 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
     """
     name = POLICY_NO_REFERRER_WHEN_DOWNGRADE
 
-    def referrer(self, response, request):
-        if not self.tls_protected(response) or self.tls_protected(request):
-            return self.stripped_referrer(response)
+    def referrer(self, response_url, request_url):
+        if not self.tls_protected(response_url) or self.tls_protected(request_url):
+            return self.stripped_referrer(response_url)
 
 
 class SameOriginPolicy(ReferrerPolicy):
@@ -128,9 +128,9 @@ class SameOriginPolicy(ReferrerPolicy):
     """
     name = POLICY_SAME_ORIGIN
 
-    def referrer(self, response, request):
-        if self.origin(response) == self.origin(request):
-            return self.stripped_referrer(response)
+    def referrer(self, response_url, request_url):
+        if self.origin(response_url) == self.origin(request_url):
+            return self.stripped_referrer(response_url)
 
 
 class OriginPolicy(ReferrerPolicy):
@@ -144,8 +144,8 @@ class OriginPolicy(ReferrerPolicy):
     """
     name = POLICY_ORIGIN
 
-    def referrer(self, response, request):
-        return self.origin_referrer(response)
+    def referrer(self, response_url, request_url):
+        return self.origin_referrer(response_url)
 
 
 class StrictOriginPolicy(ReferrerPolicy):
@@ -163,11 +163,11 @@ class StrictOriginPolicy(ReferrerPolicy):
     """
     name = POLICY_STRICT_ORIGIN
 
-    def referrer(self, response, request):
-        if ((self.tls_protected(response) and
-             self.potentially_trustworthy(request))
-            or not self.tls_protected(response)):
-            return self.origin_referrer(response)
+    def referrer(self, response_url, request_url):
+        if ((self.tls_protected(response_url) and
+             self.potentially_trustworthy(request_url))
+            or not self.tls_protected(response_url)):
+            return self.origin_referrer(response_url)
 
 
 class OriginWhenCrossOriginPolicy(ReferrerPolicy):
@@ -183,10 +183,10 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
     """
     name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
 
-    def referrer(self, response, request):
-        origin = self.origin(response)
-        if origin == self.origin(request):
-            return self.stripped_referrer(response)
+    def referrer(self, response_url, request_url):
+        origin = self.origin(response_url)
+        if origin == self.origin(request_url):
+            return self.stripped_referrer(response_url)
         else:
             return origin
 
@@ -210,14 +210,14 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
     """
     name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
 
-    def referrer(self, response, request):
-        origin = self.origin(response)
-        if origin == self.origin(request):
-            return self.stripped_referrer(response)
-        elif ((self.tls_protected(response) and
-               self.potentially_trustworthy(request))
-              or not self.tls_protected(response)):
-            return self.origin_referrer(response)
+    def referrer(self, response_url, request_url):
+        origin = self.origin(response_url)
+        if origin == self.origin(request_url):
+            return self.stripped_referrer(response_url)
+        elif ((self.tls_protected(response_url) and
+               self.potentially_trustworthy(request_url))
+              or not self.tls_protected(response_url)):
+            return self.origin_referrer(response_url)
 
 
 class UnsafeUrlPolicy(ReferrerPolicy):
@@ -235,8 +235,8 @@ class UnsafeUrlPolicy(ReferrerPolicy):
     """
     name = POLICY_UNSAFE_URL
 
-    def referrer(self, response, request):
-        return self.stripped_referrer(response)
+    def referrer(self, response_url, request_url):
+        return self.stripped_referrer(response_url)
 
 
 class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):

From 42b429dc37e17a3a24d9f5f6d9365b6d78479630 Mon Sep 17 00:00:00 2001
From: jorenham <jhammudoglu@gmail.com>
Date: Fri, 3 Mar 2017 15:15:59 +0100
Subject: [PATCH 1385/2974] Log full cache file path instead of cache directory
 for the storages that cache to single files.

---
 scrapy/extensions/httpcache.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 8025efe77..fe8c55c6b 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -220,12 +220,12 @@ class DbmCacheStorage(object):
         self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE'])
         self.db = None
 
-        logger.debug("Using DBM cache storage in %(cachedir)s" % {'cachedir': self.cachedir})
-
     def open_spider(self, spider):
         dbpath = os.path.join(self.cachedir, '%s.db' % spider.name)
         self.db = self.dbmodule.open(dbpath, 'c')
 
+        logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
+
     def close_spider(self, spider):
         self.db.close()
 
@@ -352,12 +352,12 @@ class LeveldbCacheStorage(object):
         self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
         self.db = None
 
-        logger.debug("Using LevelDB cache storage in %(cachedir)s" % {'cachedir': self.cachedir})
-
     def open_spider(self, spider):
         dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name)
         self.db = self._leveldb.LevelDB(dbpath)
 
+        logger.debug("Using LevelDB cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
+
     def close_spider(self, spider):
         # Do compactation each time to save space and also recreate files to
         # avoid them being removed in storages with timestamp-based autoremoval.

From 5e89db548419c4114ff71bd0504155c9a83868fa Mon Sep 17 00:00:00 2001
From: jorenham <jhammudoglu@gmail.com>
Date: Fri, 3 Mar 2017 15:32:20 +0100
Subject: [PATCH 1386/2974] Moved cache dir logging to `open_spider` in
 FilesystemCacheStorage for consistency

---
 scrapy/extensions/httpcache.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index fe8c55c6b..2fb4b6a15 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -277,10 +277,9 @@ class FilesystemCacheStorage(object):
         self.use_gzip = settings.getbool('HTTPCACHE_GZIP')
         self._open = gzip.open if self.use_gzip else open
 
-        logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir})
-
     def open_spider(self, spider):
-        pass
+        logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir},
+                     extra={'spider': spider})
 
     def close_spider(self, spider):
         pass

From 5d0058492c07d3f89f46ccfd2b8e33849f8bd30e Mon Sep 17 00:00:00 2001
From: Bernardas <bernardas.alisauskas@gmail.com>
Date: Tue, 23 Aug 2016 16:54:10 +0000
Subject: [PATCH 1387/2974] add media pipeline settings to enable redirection
 and handling of certain http statuses

---
 scrapy/pipelines/files.py |  2 +-
 scrapy/pipelines/media.py | 34 +++++++++++++++++++++++++++++++---
 2 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 843b4d3ec..4ae7e1d89 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -249,7 +249,7 @@ class FilesPipeline(MediaPipeline):
             resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD
         )
 
-        super(FilesPipeline, self).__init__(download_func=download_func)
+        super(FilesPipeline, self).__init__(download_func=download_func, settings=settings)
 
     @classmethod
     def from_settings(cls, settings):
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 57f70499e..4177d294f 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -1,5 +1,6 @@
 from __future__ import print_function
 
+import functools
 import logging
 from collections import defaultdict
 from twisted.internet.defer import Deferred, DeferredList
@@ -16,6 +17,7 @@ logger = logging.getLogger(__name__)
 class MediaPipeline(object):
 
     LOG_FAILED_RESULTS = True
+    ALLOW_REDIRECTS = False
 
     class SpiderInfo(object):
         def __init__(self, spider):
@@ -24,9 +26,16 @@ class MediaPipeline(object):
             self.downloaded = {}
             self.waiting = defaultdict(list)
 
-    def __init__(self, download_func=None):
+    def __init__(self, download_func=None, settings=None):
         self.download_func = download_func
-
+        resolve = functools.partial(self._key_for_pipe,
+                                    base_class_name="MediaPipeline")
+        self.allow_redirects = settings.getbool(
+            resolve('MEDIA_ALLOW_REDIRECTS'), self.ALLOW_REDIRECTS
+        )
+        self.allow_httpstatus_list = settings.getlist(
+            resolve('MEDIA_HTTPSTATUS_LIST'), []
+        )
 
     def _key_for_pipe(self, key, base_class_name=None,
                       settings=None):
@@ -93,6 +102,25 @@ class MediaPipeline(object):
         )
         return dfd.addBoth(lambda _: wad)  # it must return wad at last
 
+    def _modify_media_request(self, request):
+        httpstatus_list = []
+        if self.allow_httpstatus_list:
+            httpstatus_list = self.allow_httpstatus_list
+        elif self.allow_redirects:
+            if not httpstatus_list:
+                httpstatus_list = list(range(0, 300)) + list(range(400, 1000))
+            else:
+                for i in range(300, 400):
+                    try:
+                        httpstatus_list.remove(i)
+                    except ValueError:
+                        pass
+        if httpstatus_list:
+            request.meta['handle_httpstatus_list'] = httpstatus_list
+        else:
+            request.meta['handle_httpstatus_all'] = True
+        return request
+
     def _check_media_to_download(self, result, request, info):
         if result is not None:
             return result
@@ -103,7 +131,7 @@ class MediaPipeline(object):
                 callback=self.media_downloaded, callbackArgs=(request, info),
                 errback=self.media_failed, errbackArgs=(request, info))
         else:
-            request.meta['handle_httpstatus_all'] = True
+            request = self._modify_media_request(request)
             dfd = self.crawler.engine.download(request, info.spider)
             dfd.addCallbacks(
                 callback=self.media_downloaded, callbackArgs=(request, info),

From 25ed491219924b5fc98a67f64d355249a1b68f50 Mon Sep 17 00:00:00 2001
From: Bernardas <bernardas.alisauskas@gmail.com>
Date: Tue, 23 Aug 2016 16:55:34 +0000
Subject: [PATCH 1388/2974] add description for media pipeline
 MEDIA_ALLOW_REDIRECTS and MEDIA_HTTPSTATUS_LIST settings

---
 docs/topics/media-pipeline.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 82c0aaa88..a86bab4bf 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -322,6 +322,22 @@ By default, there are no size constraints, so all images are processed.
 
 .. _topics-media-pipeline-override:
 
+Allowing redirection and handling various http statuses
+-------------------------------------------------------
+
+.. setting:: MEDIA_ALLOW_REDIRECTS
+.. setting:: MEDIA_HTTPSTATUS_LIST
+
+By default media pipelines ignore redirects. To allow redirecting(all 300 codes) set:
+
+    MEDIA_ALLOW_REDIRECTS = True
+
+To only allow specific codes through set:
+
+    MEDIA_HTTpSTATUS_LIST = <LIST>
+    # example:
+    MEDIA_HTTPSTATUS_LIST = [303, 404]
+
 Extending the Media Pipelines
 =============================
 

From 6a4221471610b7f3a5d44d93306294c21d550406 Mon Sep 17 00:00:00 2001
From: Bernardas <bernardas.alisauskas@gmail.com>
Date: Tue, 23 Aug 2016 16:56:31 +0000
Subject: [PATCH 1389/2974] add tests for media pipeline MEDIA_ALLOW_REDIRECTS
 and MEDIA_HTTPSTATUS_LIST settings

---
 tests/test_pipeline_media.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index f30b4fea3..41ee9962f 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -6,6 +6,7 @@ from twisted.internet import reactor
 from twisted.internet.defer import Deferred, inlineCallbacks
 
 from scrapy.http import Request, Response
+from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.utils.request import request_fingerprint
 from scrapy.pipelines.media import MediaPipeline
@@ -25,7 +26,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 
     def setUp(self):
         self.spider = Spider('media.com')
-        self.pipe = self.pipeline_class(download_func=_mocked_download_func)
+        self.pipe = self.pipeline_class(download_func=_mocked_download_func,
+                                        settings=Settings())
         self.pipe.open_spider(self.spider)
         self.info = self.pipe.spiderinfo
 
@@ -82,6 +84,21 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         new_item = yield self.pipe.process_item(item, self.spider)
         assert new_item is item
 
+    def test_modify_media_request(self):
+        request = Request('http://url')
+        assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True}
+
+        request = Request('http://url')
+        self.pipe.allow_httpstatus_list = list(range(100))
+        assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_list': list(range(100))}
+        self.pipe.allow_httpstatus_list = None
+
+        request = Request('http://url')
+        self.pipe.allow_redirects = True
+        correct = {'handle_httpstatus_list': list(range(300)) + list(range(400,1000))}
+        assert self.pipe._modify_media_request(request).meta == correct
+        self.pipe.allow_redirects = False
+
 
 class MockedMediaPipeline(MediaPipeline):
 

From 854278085494053361a8cf070237683557b642ae Mon Sep 17 00:00:00 2001
From: Bernardas <bernardas.alisauskas@gmail.com>
Date: Tue, 23 Aug 2016 17:09:43 +0000
Subject: [PATCH 1390/2974] typo and clarify handling

---
 docs/topics/media-pipeline.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index a86bab4bf..bfc405d9e 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -332,11 +332,11 @@ By default media pipelines ignore redirects. To allow redirecting(all 300 codes)
 
     MEDIA_ALLOW_REDIRECTS = True
 
-To only allow specific codes through set:
+To only allow handling only specific codes set (default: any code):
 
-    MEDIA_HTTpSTATUS_LIST = <LIST>
+    MEDIA_HTTPSTATUS_LIST = <LIST>
     # example:
-    MEDIA_HTTPSTATUS_LIST = [303, 404]
+    MEDIA_HTTPSTATUS_LIST = [303, 404]  # will not go through pipelines
 
 Extending the Media Pipelines
 =============================

From 2e052c86150502c7441c8f92cd1488de6232ec7f Mon Sep 17 00:00:00 2001
From: Bernardas <bernardas.alisauskas@gmail.com>
Date: Tue, 23 Aug 2016 17:13:09 +0000
Subject: [PATCH 1391/2974] fix error when settings are not provided

---
 scrapy/pipelines/media.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 4177d294f..976b5032b 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -6,6 +6,7 @@ from collections import defaultdict
 from twisted.internet.defer import Deferred, DeferredList
 from twisted.python.failure import Failure
 
+from scrapy.settings import Settings
 from scrapy.utils.defer import mustbe_deferred, defer_result
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import arg_to_iter
@@ -28,6 +29,8 @@ class MediaPipeline(object):
 
     def __init__(self, download_func=None, settings=None):
         self.download_func = download_func
+        if isinstance(settings, dict) or settings is None:
+            settings = Settings(settings)
         resolve = functools.partial(self._key_for_pipe,
                                     base_class_name="MediaPipeline")
         self.allow_redirects = settings.getbool(

From f0b4077f812619640df078a2485f8a460fafa58a Mon Sep 17 00:00:00 2001
From: Bernardas <bernardas.alisauskas@gmail.com>
Date: Wed, 24 Aug 2016 08:39:30 +0000
Subject: [PATCH 1392/2974] expose allowed_status tuple for media pipeline

---
 scrapy/downloadermiddlewares/redirect.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 26677e527..ae4ad8891 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -57,6 +57,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):
     Handle redirection of requests based on response status
     and meta-refresh html tag.
     """
+    allowed_status = (301, 302, 303, 307)
+
     def process_response(self, request, response, spider):
         if (request.meta.get('dont_redirect', False) or
                 response.status in getattr(spider, 'handle_httpstatus_list', []) or
@@ -64,8 +66,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
                 request.meta.get('handle_httpstatus_all', False)):
             return response
 
-        allowed_status = (301, 302, 303, 307)
-        if 'Location' not in response.headers or response.status not in allowed_status:
+        if 'Location' not in response.headers or response.status not in self.allowed_status:
             return response
 
         location = safe_url_string(response.headers['location'])

From 3cef1cd451c8f28df8075aa4e3b65cedf5ecedfa Mon Sep 17 00:00:00 2001
From: Bernardas <bernardas.alisauskas@gmail.com>
Date: Wed, 24 Aug 2016 08:40:46 +0000
Subject: [PATCH 1393/2974] adjust variable wording and redirect logic

---
 scrapy/pipelines/media.py    | 19 +++++++++----------
 tests/test_pipeline_media.py |  4 ++--
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 976b5032b..b11e7095b 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -6,6 +6,7 @@ from collections import defaultdict
 from twisted.internet.defer import Deferred, DeferredList
 from twisted.python.failure import Failure
 
+from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
 from scrapy.settings import Settings
 from scrapy.utils.defer import mustbe_deferred, defer_result
 from scrapy.utils.request import request_fingerprint
@@ -36,7 +37,7 @@ class MediaPipeline(object):
         self.allow_redirects = settings.getbool(
             resolve('MEDIA_ALLOW_REDIRECTS'), self.ALLOW_REDIRECTS
         )
-        self.allow_httpstatus_list = settings.getlist(
+        self.handle_httpstatus_list = settings.getlist(
             resolve('MEDIA_HTTPSTATUS_LIST'), []
         )
 
@@ -107,17 +108,15 @@ class MediaPipeline(object):
 
     def _modify_media_request(self, request):
         httpstatus_list = []
-        if self.allow_httpstatus_list:
-            httpstatus_list = self.allow_httpstatus_list
-        elif self.allow_redirects:
+        if self.handle_httpstatus_list:
+            httpstatus_list = self.handle_httpstatus_list
+        if self.allow_redirects:
             if not httpstatus_list:
-                httpstatus_list = list(range(0, 300)) + list(range(400, 1000))
+                httpstatus_list = [i for i in range(1000)
+                                   if i not in RedirectMiddleware.allowed_status]
             else:
-                for i in range(300, 400):
-                    try:
-                        httpstatus_list.remove(i)
-                    except ValueError:
-                        pass
+                httpstatus_list = [i for i in httpstatus_list
+                                   if i not in RedirectMiddleware.allowed_status]
         if httpstatus_list:
             request.meta['handle_httpstatus_list'] = httpstatus_list
         else:
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 41ee9962f..66e98db29 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -89,9 +89,9 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True}
 
         request = Request('http://url')
-        self.pipe.allow_httpstatus_list = list(range(100))
+        self.pipe.handle_httpstatus_list = list(range(100))
         assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_list': list(range(100))}
-        self.pipe.allow_httpstatus_list = None
+        self.pipe.handle_httpstatus_list = None
 
         request = Request('http://url')
         self.pipe.allow_redirects = True

From 11b31c9fbddd75fe7fec60152a1bec57acccc039 Mon Sep 17 00:00:00 2001
From: Bernardas <bernardas.alisauskas@gmail.com>
Date: Wed, 24 Aug 2016 08:44:56 +0000
Subject: [PATCH 1394/2974] fix redirect change

---
 tests/test_pipeline_media.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 66e98db29..f1b8076fd 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -5,6 +5,7 @@ from twisted.python.failure import Failure
 from twisted.internet import reactor
 from twisted.internet.defer import Deferred, inlineCallbacks
 
+from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
@@ -95,7 +96,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 
         request = Request('http://url')
         self.pipe.allow_redirects = True
-        correct = {'handle_httpstatus_list': list(range(300)) + list(range(400,1000))}
+        correct = {'handle_httpstatus_list': [i for i in range(1000)
+                                              if i not in RedirectMiddleware.allowed_status]}
         assert self.pipe._modify_media_request(request).meta == correct
         self.pipe.allow_redirects = False
 

From c64ebee06253ef385564c83806bd2422bff6f6d6 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 2 Mar 2017 22:40:10 +0100
Subject: [PATCH 1395/2974] Refactor (WIP)

---
 scrapy/pipelines/media.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index b11e7095b..0712101b0 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -19,7 +19,6 @@ logger = logging.getLogger(__name__)
 class MediaPipeline(object):
 
     LOG_FAILED_RESULTS = True
-    ALLOW_REDIRECTS = False
 
     class SpiderInfo(object):
         def __init__(self, spider):
@@ -35,12 +34,23 @@ class MediaPipeline(object):
         resolve = functools.partial(self._key_for_pipe,
                                     base_class_name="MediaPipeline")
         self.allow_redirects = settings.getbool(
-            resolve('MEDIA_ALLOW_REDIRECTS'), self.ALLOW_REDIRECTS
+            resolve('MEDIA_ALLOW_REDIRECTS'), False
         )
         self.handle_httpstatus_list = settings.getlist(
             resolve('MEDIA_HTTPSTATUS_LIST'), []
         )
 
+        self.httpstatus_list = []
+        if self.handle_httpstatus_list:
+            self.httpstatus_list = self.handle_httpstatus_list
+        if self.allow_redirects:
+            if not self.httpstatus_list:
+                self.httpstatus_list = [i for i in range(1000)
+                                   if i not in RedirectMiddleware.allowed_status]
+            else:
+                self.httpstatus_list = [i for i in self.httpstatus_list
+                                   if i not in RedirectMiddleware.allowed_status]
+
     def _key_for_pipe(self, key, base_class_name=None,
                       settings=None):
         """
@@ -107,18 +117,8 @@ class MediaPipeline(object):
         return dfd.addBoth(lambda _: wad)  # it must return wad at last
 
     def _modify_media_request(self, request):
-        httpstatus_list = []
-        if self.handle_httpstatus_list:
-            httpstatus_list = self.handle_httpstatus_list
-        if self.allow_redirects:
-            if not httpstatus_list:
-                httpstatus_list = [i for i in range(1000)
-                                   if i not in RedirectMiddleware.allowed_status]
-            else:
-                httpstatus_list = [i for i in httpstatus_list
-                                   if i not in RedirectMiddleware.allowed_status]
-        if httpstatus_list:
-            request.meta['handle_httpstatus_list'] = httpstatus_list
+        if self.httpstatus_list:
+            request.meta['handle_httpstatus_list'] = self.httpstatus_list
         else:
             request.meta['handle_httpstatus_all'] = True
         return request

From 72fbb687d7a35ddf52f82b165df0a842d0b653e0 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 3 Mar 2017 12:31:05 +0100
Subject: [PATCH 1396/2974] Revert "expose allowed_status tuple for media
 pipeline"

This reverts commit 052809c73ed20b9a728a8fd7df3de5f45f2dad8d.
---
 scrapy/downloadermiddlewares/redirect.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index ae4ad8891..26677e527 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -57,8 +57,6 @@ class RedirectMiddleware(BaseRedirectMiddleware):
     Handle redirection of requests based on response status
     and meta-refresh html tag.
     """
-    allowed_status = (301, 302, 303, 307)
-
     def process_response(self, request, response, spider):
         if (request.meta.get('dont_redirect', False) or
                 response.status in getattr(spider, 'handle_httpstatus_list', []) or
@@ -66,7 +64,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):
                 request.meta.get('handle_httpstatus_all', False)):
             return response
 
-        if 'Location' not in response.headers or response.status not in self.allowed_status:
+        allowed_status = (301, 302, 303, 307)
+        if 'Location' not in response.headers or response.status not in allowed_status:
             return response
 
         location = safe_url_string(response.headers['location'])

From ecde166ee18935456bcf4c6fce89ab909bbffbaf Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 3 Mar 2017 15:50:34 +0100
Subject: [PATCH 1397/2974] Refactor without MEDIA_HTTPSTATUS_LIST setting

---
 docs/topics/media-pipeline.rst | 15 +++++--------
 scrapy/pipelines/media.py      | 25 ++++++++-------------
 tests/test_pipeline_media.py   | 41 +++++++++++++++++++++++++---------
 3 files changed, 46 insertions(+), 35 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index bfc405d9e..f258ff748 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -322,21 +322,18 @@ By default, there are no size constraints, so all images are processed.
 
 .. _topics-media-pipeline-override:
 
-Allowing redirection and handling various http statuses
--------------------------------------------------------
+Allowing redirections
+---------------------
 
 .. setting:: MEDIA_ALLOW_REDIRECTS
-.. setting:: MEDIA_HTTPSTATUS_LIST
 
-By default media pipelines ignore redirects. To allow redirecting(all 300 codes) set:
+By default media pipelines ignore redirects, i.e. an HTTP redirection
+to a media file URL request will mean the media download is considered failed.
+
+To handle media redirections, set this settings to ``True``:
 
     MEDIA_ALLOW_REDIRECTS = True
 
-To only allow handling only specific codes set (default: any code):
-
-    MEDIA_HTTPSTATUS_LIST = <LIST>
-    # example:
-    MEDIA_HTTPSTATUS_LIST = [303, 404]  # will not go through pipelines
 
 Extending the Media Pipelines
 =============================
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 0712101b0..02daf8d2c 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -6,8 +6,8 @@ from collections import defaultdict
 from twisted.internet.defer import Deferred, DeferredList
 from twisted.python.failure import Failure
 
-from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
 from scrapy.settings import Settings
+from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.defer import mustbe_deferred, defer_result
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import arg_to_iter
@@ -29,6 +29,7 @@ class MediaPipeline(object):
 
     def __init__(self, download_func=None, settings=None):
         self.download_func = download_func
+
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
         resolve = functools.partial(self._key_for_pipe,
@@ -36,20 +37,12 @@ class MediaPipeline(object):
         self.allow_redirects = settings.getbool(
             resolve('MEDIA_ALLOW_REDIRECTS'), False
         )
-        self.handle_httpstatus_list = settings.getlist(
-            resolve('MEDIA_HTTPSTATUS_LIST'), []
-        )
+        self._handle_statuses(self.allow_redirects)
 
-        self.httpstatus_list = []
-        if self.handle_httpstatus_list:
-            self.httpstatus_list = self.handle_httpstatus_list
-        if self.allow_redirects:
-            if not self.httpstatus_list:
-                self.httpstatus_list = [i for i in range(1000)
-                                   if i not in RedirectMiddleware.allowed_status]
-            else:
-                self.httpstatus_list = [i for i in self.httpstatus_list
-                                   if i not in RedirectMiddleware.allowed_status]
+    def _handle_statuses(self, allow_redirects):
+        self.handle_httpstatus_list = None
+        if allow_redirects:
+            self.handle_httpstatus_list = SequenceExclude(range(300, 400))
 
     def _key_for_pipe(self, key, base_class_name=None,
                       settings=None):
@@ -117,8 +110,8 @@ class MediaPipeline(object):
         return dfd.addBoth(lambda _: wad)  # it must return wad at last
 
     def _modify_media_request(self, request):
-        if self.httpstatus_list:
-            request.meta['handle_httpstatus_list'] = self.httpstatus_list
+        if self.handle_httpstatus_list:
+            request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list
         else:
             request.meta['handle_httpstatus_all'] = True
         return request
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index f1b8076fd..4797956a0 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -24,11 +24,12 @@ def _mocked_download_func(request, info):
 class BaseMediaPipelineTestCase(unittest.TestCase):
 
     pipeline_class = MediaPipeline
+    settings = None
 
     def setUp(self):
         self.spider = Spider('media.com')
         self.pipe = self.pipeline_class(download_func=_mocked_download_func,
-                                        settings=Settings())
+                                        settings=Settings(self.settings))
         self.pipe.open_spider(self.spider)
         self.info = self.pipe.spiderinfo
 
@@ -89,17 +90,37 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         request = Request('http://url')
         assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True}
 
-        request = Request('http://url')
-        self.pipe.handle_httpstatus_list = list(range(100))
-        assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_list': list(range(100))}
-        self.pipe.handle_httpstatus_list = None
 
+class MediaPipelineAllowRedirectsTestCase(BaseMediaPipelineTestCase):
+
+    pipeline_class = MediaPipeline
+    settings = {
+        'MEDIA_ALLOW_REDIRECTS': True
+    }
+
+    def test_modify_media_request(self):
         request = Request('http://url')
-        self.pipe.allow_redirects = True
-        correct = {'handle_httpstatus_list': [i for i in range(1000)
-                                              if i not in RedirectMiddleware.allowed_status]}
-        assert self.pipe._modify_media_request(request).meta == correct
-        self.pipe.allow_redirects = False
+        meta = self.pipe._modify_media_request(request).meta
+        self.assertIn('handle_httpstatus_list', meta)
+        for status, check in [
+                (200, True),
+
+                # These are the status codes we want
+                # the downloader to handle itself
+                (301, False),
+                (302, False),
+                (302, False),
+                (307, False),
+                (308, False),
+
+                # we still want to get 4xx and 5xx
+                (400, True),
+                (404, True),
+                (500, True)]:
+            if check:
+                self.assertIn(status, meta['handle_httpstatus_list'])
+            else:
+                self.assertNotIn(status, meta['handle_httpstatus_list'])
 
 
 class MockedMediaPipeline(MediaPipeline):

From f7e11b198efd0213bb51205b6829123476ccf2ba Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 3 Mar 2017 16:00:59 +0100
Subject: [PATCH 1398/2974] Cleanup

---
 scrapy/pipelines/media.py    |  3 +--
 tests/test_pipeline_media.py | 12 ++++++------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 02daf8d2c..921e9e1c9 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -114,7 +114,6 @@ class MediaPipeline(object):
             request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list
         else:
             request.meta['handle_httpstatus_all'] = True
-        return request
 
     def _check_media_to_download(self, result, request, info):
         if result is not None:
@@ -126,7 +125,7 @@ class MediaPipeline(object):
                 callback=self.media_downloaded, callbackArgs=(request, info),
                 errback=self.media_failed, errbackArgs=(request, info))
         else:
-            request = self._modify_media_request(request)
+            self._modify_media_request(request)
             dfd = self.crawler.engine.download(request, info.spider)
             dfd.addCallbacks(
                 callback=self.media_downloaded, callbackArgs=(request, info),
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 4797956a0..cfa2fc42b 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -5,7 +5,6 @@ from twisted.python.failure import Failure
 from twisted.internet import reactor
 from twisted.internet.defer import Deferred, inlineCallbacks
 
-from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
@@ -88,7 +87,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 
     def test_modify_media_request(self):
         request = Request('http://url')
-        assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True}
+        self.pipe._modify_media_request(request)
+        assert request.meta == {'handle_httpstatus_all': True}
 
 
 class MediaPipelineAllowRedirectsTestCase(BaseMediaPipelineTestCase):
@@ -100,8 +100,8 @@ class MediaPipelineAllowRedirectsTestCase(BaseMediaPipelineTestCase):
 
     def test_modify_media_request(self):
         request = Request('http://url')
-        meta = self.pipe._modify_media_request(request).meta
-        self.assertIn('handle_httpstatus_list', meta)
+        self.pipe._modify_media_request(request)
+        self.assertIn('handle_httpstatus_list', request.meta)
         for status, check in [
                 (200, True),
 
@@ -118,9 +118,9 @@ class MediaPipelineAllowRedirectsTestCase(BaseMediaPipelineTestCase):
                 (404, True),
                 (500, True)]:
             if check:
-                self.assertIn(status, meta['handle_httpstatus_list'])
+                self.assertIn(status, request.meta['handle_httpstatus_list'])
             else:
-                self.assertNotIn(status, meta['handle_httpstatus_list'])
+                self.assertNotIn(status, request.meta['handle_httpstatus_list'])
 
 
 class MockedMediaPipeline(MediaPipeline):

From ef04cfd237ec3d072a487f92e217bad68195f2d8 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 21 Feb 2017 19:55:52 +0300
Subject: [PATCH 1399/2974] Respect log settings in custom_settings: fixes
 GH-1612

A new root logger is installed when a crawler is created
if one was already installed before.
This allows to respect custom settings related to logging,
such as LOG_LEVEL, LOG_FILE, etc.
---
 scrapy/crawler.py     |  9 +++++++--
 scrapy/utils/log.py   | 22 ++++++++++++++++++---
 tests/test_crawl.py   |  4 ++--
 tests/test_crawler.py | 46 +++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 74 insertions(+), 7 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 443a9aa2f..7b8518832 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -16,7 +16,9 @@ from scrapy.signalmanager import SignalManager
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 from scrapy.utils.misc import load_object
-from scrapy.utils.log import LogCounterHandler, configure_logging, log_scrapy_info
+from scrapy.utils.log import (
+    LogCounterHandler, configure_logging, log_scrapy_info,
+    get_scrapy_root_handler, install_scrapy_root_handler)
 from scrapy import signals
 
 logger = logging.getLogger(__name__)
@@ -35,8 +37,11 @@ class Crawler(object):
         self.signals = SignalManager(self)
         self.stats = load_object(self.settings['STATS_CLASS'])(self)
 
-        handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL'))
+        handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
         logging.root.addHandler(handler)
+        if get_scrapy_root_handler() is not None:
+            # scrapy root handler alread installed: update it with new settings
+            install_scrapy_root_handler(self.settings)
         # lambda is assigned to Crawler attribute because this way it is not
         # garbage collected after leaving __init__ scope
         self.__remove_handler = lambda: logging.root.removeHandler(handler)
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index f33ce7017..6ceb61a82 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -96,9 +96,25 @@ def configure_logging(settings=None, install_root_handler=True):
         sys.stdout = StreamLogger(logging.getLogger('stdout'))
 
     if install_root_handler:
-        logging.root.setLevel(logging.NOTSET)
-        handler = _get_handler(settings)
-        logging.root.addHandler(handler)
+        install_scrapy_root_handler(settings)
+
+
+def install_scrapy_root_handler(settings):
+    global _scrapy_root_handler
+
+    if (_scrapy_root_handler is not None
+            and _scrapy_root_handler in logging.root.handlers):
+        logging.root.removeHandler(_scrapy_root_handler)
+    logging.root.setLevel(logging.NOTSET)
+    _scrapy_root_handler = _get_handler(settings)
+    logging.root.addHandler(_scrapy_root_handler)
+
+
+def get_scrapy_root_handler():
+    return _scrapy_root_handler
+
+
+_scrapy_root_handler = None
 
 
 def _get_handler(settings):
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index d5babdded..3c5d9b958 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -97,8 +97,8 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_start_requests_bug_before_yield(self):
+        crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
         with LogCapture('scrapy', level=logging.ERROR) as l:
-            crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_before_yield=1)
 
         self.assertEqual(len(l.records), 1)
@@ -108,8 +108,8 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_start_requests_bug_yielding(self):
+        crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
         with LogCapture('scrapy', level=logging.ERROR) as l:
-            crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_yielding=1)
 
         self.assertEqual(len(l.records), 1)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 53a1202e3..ba0d709ff 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,3 +1,6 @@
+import logging
+import os
+import tempfile
 import warnings
 import unittest
 
@@ -5,6 +8,7 @@ import scrapy
 from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
 from scrapy.settings import Settings, default_settings
 from scrapy.spiderloader import SpiderLoader
+from scrapy.utils.log import configure_logging, get_scrapy_root_handler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
 from scrapy.extensions.throttle import AutoThrottle
@@ -74,6 +78,48 @@ class SpiderSettingsTestCase(unittest.TestCase):
         self.assertIn(AutoThrottle, enabled_exts)
 
 
+class CrawlerLoggingTestCase(unittest.TestCase):
+    def test_no_root_handler_installed(self):
+        handler = get_scrapy_root_handler()
+        if handler is not None:
+            logging.root.removeHandler(handler)
+
+        class MySpider(scrapy.Spider):
+            name = 'spider'
+
+        crawler = Crawler(MySpider, {})
+        assert get_scrapy_root_handler() is None
+
+    def test_spider_custom_settings_log_level(self):
+        with tempfile.NamedTemporaryFile() as log_file:
+            class MySpider(scrapy.Spider):
+                name = 'spider'
+                custom_settings = {
+                    'LOG_LEVEL': 'INFO',
+                    'LOG_FILE': log_file.name,
+                }
+
+            configure_logging()
+            self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
+            crawler = Crawler(MySpider, {})
+            self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
+            info_count = crawler.stats.get_value('log_count/INFO')
+            logging.debug('debug message')
+            logging.info('info message')
+            logging.warning('warning message')
+            logging.error('error message')
+            logged = log_file.read().decode('utf8')
+        self.assertNotIn('debug message', logged)
+        self.assertIn('info message', logged)
+        self.assertIn('warning message', logged)
+        self.assertIn('error message', logged)
+        self.assertEqual(crawler.stats.get_value('log_count/ERROR'), 1)
+        self.assertEqual(crawler.stats.get_value('log_count/WARNING'), 1)
+        self.assertEqual(
+            crawler.stats.get_value('log_count/INFO') - info_count, 1)
+        self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0)
+
+
 class SpiderLoaderWithWrongInterface(object):
 
     def unneeded_method(self):

From c3b6feca0e15309386c080d71b0b03a07d4c8635 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 3 Mar 2017 16:29:07 +0100
Subject: [PATCH 1400/2974] Fix setting lookup for MEDIA_ALLOWED_REDIRECTS

---
 scrapy/pipelines/media.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 921e9e1c9..404bbf5bf 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -33,7 +33,8 @@ class MediaPipeline(object):
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
         resolve = functools.partial(self._key_for_pipe,
-                                    base_class_name="MediaPipeline")
+                                    base_class_name="MediaPipeline",
+                                    settings=settings)
         self.allow_redirects = settings.getbool(
             resolve('MEDIA_ALLOW_REDIRECTS'), False
         )

From c68f99eed843bd35224d8bf0e22b0c66bdc2122b Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 3 Mar 2017 17:03:25 +0100
Subject: [PATCH 1401/2974] Refactor settings tests

---
 tests/test_pipeline_media.py | 90 +++++++++++++++++++++++-------------
 1 file changed, 58 insertions(+), 32 deletions(-)

diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index cfa2fc42b..5f6a6d9e6 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -91,38 +91,6 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         assert request.meta == {'handle_httpstatus_all': True}
 
 
-class MediaPipelineAllowRedirectsTestCase(BaseMediaPipelineTestCase):
-
-    pipeline_class = MediaPipeline
-    settings = {
-        'MEDIA_ALLOW_REDIRECTS': True
-    }
-
-    def test_modify_media_request(self):
-        request = Request('http://url')
-        self.pipe._modify_media_request(request)
-        self.assertIn('handle_httpstatus_list', request.meta)
-        for status, check in [
-                (200, True),
-
-                # These are the status codes we want
-                # the downloader to handle itself
-                (301, False),
-                (302, False),
-                (302, False),
-                (307, False),
-                (308, False),
-
-                # we still want to get 4xx and 5xx
-                (400, True),
-                (404, True),
-                (500, True)]:
-            if check:
-                self.assertIn(status, request.meta['handle_httpstatus_list'])
-            else:
-                self.assertNotIn(status, request.meta['handle_httpstatus_list'])
-
-
 class MockedMediaPipeline(MediaPipeline):
 
     def __init__(self, *args, **kwargs):
@@ -289,3 +257,61 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         self.assertEqual(new_item['results'], [(True, 'ITSME')])
         self.assertEqual(self.pipe._mockcalled, \
                 ['get_media_requests', 'media_to_download', 'item_completed'])
+
+
+class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase):
+
+    def _assert_request_no3xx(self, pipeline_class, settings):
+        pipe = pipeline_class(settings=Settings(settings))
+        request = Request('http://url')
+        pipe._modify_media_request(request)
+
+        self.assertIn('handle_httpstatus_list', request.meta)
+        for status, check in [
+                (200, True),
+
+                # These are the status codes we want
+                # the downloader to handle itself
+                (301, False),
+                (302, False),
+                (302, False),
+                (307, False),
+                (308, False),
+
+                # we still want to get 4xx and 5xx
+                (400, True),
+                (404, True),
+                (500, True)]:
+            if check:
+                self.assertIn(status, request.meta['handle_httpstatus_list'])
+            else:
+                self.assertNotIn(status, request.meta['handle_httpstatus_list'])
+
+    def test_standard_setting(self):
+        self._assert_request_no3xx(
+            MediaPipeline,
+            {
+                'MEDIA_ALLOW_REDIRECTS': True
+            })
+
+    def test_subclass_standard_setting(self):
+
+        class UserDefinedPipeline(MediaPipeline):
+            pass
+
+        self._assert_request_no3xx(
+            UserDefinedPipeline,
+            {
+                'MEDIA_ALLOW_REDIRECTS': True
+            })
+
+    def test_subclass_specific_setting(self):
+
+        class UserDefinedPipeline(MediaPipeline):
+            pass
+
+        self._assert_request_no3xx(
+            UserDefinedPipeline,
+            {
+                'USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS': True
+            })

From 30d812eea233914b3aead55b8053a9c804e594d3 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 3 Mar 2017 17:15:37 +0100
Subject: [PATCH 1402/2974] Remove redundant slot.add_request() call in
 ExecutionEngine

---
 scrapy/core/engine.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 2b5770138..37fe0a873 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -218,10 +218,8 @@ class ExecutionEngine(object):
                                         request=request, spider=spider)
 
     def download(self, request, spider):
-        slot = self.slot
-        slot.add_request(request)
         d = self._download(request, spider)
-        d.addBoth(self._downloaded, slot, request, spider)
+        d.addBoth(self._downloaded, self.slot, request, spider)
         return d
 
     def _downloaded(self, response, slot, request, spider):

From a8b47d6c689cd8f221acb4caf3974ff0057b6a2d Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 6 Mar 2017 14:25:52 +0100
Subject: [PATCH 1403/2974] Update release notes for 1.0.7, 1.1.4 and 1.2.3

---
 docs/news.rst | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index ff1e4ce03..31f4d3026 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -101,6 +101,12 @@ Dependencies & Cleanups
   downloader middlewares.
 
 
+Scrapy 1.2.3 (2017-03-03)
+-------------------------
+
+- Packaging fix: disallow unsupported Twisted versions in setup.py
+
+
 Scrapy 1.2.2 (2016-12-06)
 -------------------------
 
@@ -229,6 +235,12 @@ Documentation
 - Add StackOverflow as a support channel (:issue:`2257`).
 
 
+Scrapy 1.1.4 (2017-03-03)
+-------------------------
+
+- Packaging fix: disallow unsupported Twisted versions in setup.py
+
+
 Scrapy 1.1.3 (2016-09-22)
 -------------------------
 
@@ -501,6 +513,12 @@ Bugfixes
   to same remote host (:issue:`1912`).
 
 
+Scrapy 1.0.7 (2017-03-03)
+-------------------------
+
+- Packaging fix: disallow unsupported Twisted versions in setup.py
+
+
 Scrapy 1.0.6 (2016-05-04)
 -------------------------
 

From 768f3155e57ed54419f0c137fc066a95197d1b46 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 6 Mar 2017 16:20:37 +0100
Subject: [PATCH 1404/2974] Fix referrer policy from response headers and
 support explicit empty string

---
 scrapy/spidermiddlewares/referer.py    | 8 ++++++--
 tests/test_spidermiddleware_referer.py | 7 +++++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index b444e34bb..1ddfb37f4 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -261,6 +261,9 @@ _policy_classes = {p.name: p for p in (
     DefaultReferrerPolicy,
 )}
 
+# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string
+_policy_classes[''] = NoReferrerWhenDowngradePolicy
+
 
 def _load_policy_class(policy, warning_only=False):
     """
@@ -317,8 +320,9 @@ class RefererMiddleware(object):
         policy_name = request.meta.get('referrer_policy')
         if policy_name is None:
             if isinstance(resp_or_url, Response):
-                policy_name = to_native_str(
-                    resp_or_url.headers.get('Referrer-Policy', '').decode('latin1'))
+                policy_header = resp_or_url.headers.get('Referrer-Policy')
+                if policy_header is not None:
+                    policy_name = to_native_str(policy_header.decode('latin1'))
         if policy_name is None:
             return self.default_policy()
 
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index b1c815876..f27f31b74 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -526,6 +526,13 @@ class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMid
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
     resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
 
+class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
+    """
+    The empty string means "no-referrer-when-downgrade"
+    """
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+    resp_headers = {'Referrer-Policy': ''}
+
 
 class TestReferrerOnRedirect(TestRefererMiddleware):
 

From 2a7d391e0b379143810fdcaed241664e159e0d1d Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 6 Mar 2017 17:30:32 +0100
Subject: [PATCH 1405/2974] DOC Mention brotli support in
 HttpCompressionMiddleware section

---
 docs/topics/downloader-middleware.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 0ef3fb071..c3a454279 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -645,6 +645,12 @@ HttpCompressionMiddleware
    This middleware allows compressed (gzip, deflate) traffic to be
    sent/received from web sites.
 
+   This middleware also supports decoding `brotli-compressed`_ responses,
+   provided `brotlipy`_ is installed.
+
+.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
+.. _brotlipy: https://pypi.python.org/pypi/brotlipy
+
 HttpCompressionMiddleware Settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From e42b846a9fc68a624e94052aac2ba44224b05cad Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 10 Nov 2016 16:26:55 +0100
Subject: [PATCH 1406/2974] Use body to chose response type after decompression
 content

---
 scrapy/downloadermiddlewares/httpcompression.py   |  2 +-
 .../test_downloadermiddleware_httpcompression.py  | 15 +++++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 19d6345e4..eb00d8923 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -38,7 +38,7 @@ class HttpCompressionMiddleware(object):
                 encoding = content_encoding.pop()
                 decoded_body = self._decode(response.body, encoding.lower())
                 respcls = responsetypes.from_args(headers=response.headers, \
-                    url=response.url)
+                    url=response.url, body=decoded_body)
                 kwargs = dict(cls=respcls, body=decoded_body)
                 if issubclass(respcls, TextResponse):
                     # force recalculating the encoding until we make sure the
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 7924fb3b5..5403e8f52 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -7,6 +7,7 @@ from scrapy.spiders import Spider
 from scrapy.http import Response, Request, HtmlResponse
 from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \
     ACCEPTED_ENCODINGS
+from scrapy.responsetypes import responsetypes
 from tests import tests_datadir
 from w3lib.encoding import resolve_encoding
 
@@ -152,6 +153,20 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
 
+    def test_process_response_no_content_type_header(self):
+        headers = {
+            'Content-Encoding': 'identity',
+        }
+        plainbody = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
+        respcls = responsetypes.from_args(url="http://www.example.com/index", headers=headers, body=plainbody)
+        response = respcls("http://www.example.com/index", headers=headers, body=plainbody)
+        request = Request("http://www.example.com/index")
+
+        newresponse = self.mw.process_response(request, response, self.spider)
+        assert isinstance(newresponse, respcls)
+        self.assertEqual(newresponse.body, plainbody)
+        self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
+
     def test_process_response_gzipped_contenttype(self):
         response = self._getresponse('gzip')
         response.headers['Content-Type'] = 'application/gzip'

From 11cdf58abe26a9f65d8c1aefa717b6735e5734c9 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 9 Nov 2016 23:08:23 +0100
Subject: [PATCH 1407/2974] Always decompress Content-Encoding: gzip at
 HttpCompression stage

Let SitemapSpider handle decoding of .xml.gz files if necessary
---
 .../downloadermiddlewares/httpcompression.py  |  2 +-
 scrapy/spiders/sitemap.py                     | 23 +++++--
 scrapy/utils/gz.py                            |  4 ++
 ...st_downloadermiddleware_httpcompression.py | 61 ++++++++++++++++---
 tests/test_spider.py                          |  4 ++
 5 files changed, 78 insertions(+), 16 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index eb00d8923..dd32c62de 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -34,7 +34,7 @@ class HttpCompressionMiddleware(object):
             return response
         if isinstance(response, Response):
             content_encoding = response.headers.getlist('Content-Encoding')
-            if content_encoding and not is_gzipped(response):
+            if content_encoding:
                 encoding = content_encoding.pop()
                 decoded_body = self._decode(response.body, encoding.lower())
                 respcls = responsetypes.from_args(headers=response.headers, \
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 9e45637c3..10af90259 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -5,7 +5,8 @@ import six
 from scrapy.spiders import Spider
 from scrapy.http import Request, XmlResponse
 from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
-from scrapy.utils.gz import gunzip, is_gzipped
+from scrapy.utils.gz import gunzip, gzip_magic_number
+
 
 logger = logging.getLogger(__name__)
 
@@ -59,12 +60,22 @@ class SitemapSpider(Spider):
         """
         if isinstance(response, XmlResponse):
             return response.body
-        elif is_gzipped(response):
-            return gunzip(response.body)
-        elif response.url.endswith('.xml'):
+        elif gzip_magic_number(response):
+            try:
+                return gunzip(response.body)
+            except:
+                pass
+        # actual gzipped sitemap files are decompressed above ;
+        # if we are here (response body is not gzipped)
+        # and have a response for .xml.gz,
+        # it usually means that it was already gunzipped
+        # by HttpCompression middleware,
+        # the HTTP response being sent with "Content-Encoding: gzip"
+        # without actually being a .xml.gz file in the first place,
+        # merely XML gzip-compressed on the fly,
+        # in other word, here, we have plain XML
+        elif response.url.endswith('.xml') or response.url.endswith('.xml.gz'):
             return response.body
-        elif response.url.endswith('.xml.gz'):
-            return gunzip(response.body)
 
 
 def regex(x):
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 73c2eb73b..22cf58986 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -59,3 +59,7 @@ def is_gzipped(response):
     cenc = response.headers.get('Content-Encoding', b'').lower()
     return (_is_gzipped(ctype) or
             (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')))
+
+
+def gzip_magic_number(response):
+    return response.body[:2] == b'\x1f\x8b'
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 5403e8f52..5f56c99ec 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -8,6 +8,7 @@ from scrapy.http import Response, Request, HtmlResponse
 from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \
     ACCEPTED_ENCODINGS
 from scrapy.responsetypes import responsetypes
+from scrapy.utils.gz import gunzip
 from tests import tests_datadir
 from w3lib.encoding import resolve_encoding
 
@@ -173,9 +174,9 @@ class HttpCompressionTest(TestCase):
         request = response.request
 
         newresponse = self.mw.process_response(request, response, self.spider)
-        self.assertIs(newresponse, response)
-        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
-        self.assertEqual(response.headers['Content-Type'], b'application/gzip')
+        assert newresponse is not response
+        assert newresponse.body.startswith(b'<!DOCTYPE')
+        assert 'Content-Encoding' not in newresponse.headers
 
     def test_process_response_gzip_app_octetstream_contenttype(self):
         response = self._getresponse('gzip')
@@ -183,9 +184,9 @@ class HttpCompressionTest(TestCase):
         request = response.request
 
         newresponse = self.mw.process_response(request, response, self.spider)
-        self.assertIs(newresponse, response)
-        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
-        self.assertEqual(response.headers['Content-Type'], b'application/octet-stream')
+        assert newresponse is not response
+        assert newresponse.body.startswith(b'<!DOCTYPE')
+        assert 'Content-Encoding' not in newresponse.headers
 
     def test_process_response_gzip_binary_octetstream_contenttype(self):
         response = self._getresponse('x-gzip')
@@ -193,9 +194,51 @@ class HttpCompressionTest(TestCase):
         request = response.request
 
         newresponse = self.mw.process_response(request, response, self.spider)
-        self.assertIs(newresponse, response)
-        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
-        self.assertEqual(response.headers['Content-Type'], b'binary/octet-stream')
+        assert newresponse is not response
+        assert newresponse.body.startswith(b'<!DOCTYPE')
+        assert 'Content-Encoding' not in newresponse.headers
+
+    def test_process_response_gzipped_gzip_file(self):
+        """Test that a gzip Content-Encoded .gz file is gunzipped
+        only once by the middleware, leaving gunzipping of the file
+        to upper layers.
+        """
+        headers = {
+            'Content-Type': 'application/gzip',
+            'Content-Encoding': 'gzip',
+        }
+        # build a gzipped file (here, a sitemap)
+        f = BytesIO()
+        plainbody = b"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
+  <url>
+    <loc>http://www.example.com/</loc>
+    <lastmod>2009-08-16</lastmod>
+    <changefreq>daily</changefreq>
+    <priority>1</priority>
+  </url>
+  <url>
+    <loc>http://www.example.com/Special-Offers.html</loc>
+    <lastmod>2009-08-16</lastmod>
+    <changefreq>weekly</changefreq>
+    <priority>0.8</priority>
+  </url>
+</urlset>"""
+        gz_file = GzipFile(fileobj=f, mode='wb')
+        gz_file.write(plainbody)
+        gz_file.close()
+
+        # build a gzipped response body containing this gzipped file
+        r = BytesIO()
+        gz_resp = GzipFile(fileobj=r, mode='wb')
+        gz_resp.write(f.getvalue())
+        gz_resp.close()
+
+        response = Response("http;//www.example.com/", headers=headers, body=r.getvalue())
+        request = Request("http://www.example.com/")
+
+        newresponse = self.mw.process_response(request, response, self.spider)
+        self.assertEqual(gunzip(newresponse.body), plainbody)
 
     def test_process_response_head_request_no_decode_required(self):
         response = self._getresponse('gzip')
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 371b8c1ac..e55f0fa6d 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -328,6 +328,10 @@ class SitemapSpiderTest(SpiderTest):
         r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY)
         self.assertSitemapBody(r, self.BODY)
 
+        # .xml.gz but body decoded by HttpCompression middleware already
+        r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.BODY)
+        self.assertSitemapBody(r, self.BODY)
+
     def test_get_sitemap_urls_from_robotstxt(self):
         robots = b"""# Sitemap files
 Sitemap: http://example.com/sitemap.xml

From b174744b80fd12efc6e4008ea5eb38256f21038a Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 10 Nov 2016 10:50:34 +0100
Subject: [PATCH 1408/2974] Do not silently fail on gzip unzipping

---
 scrapy/downloadermiddlewares/httpcompression.py | 2 +-
 scrapy/spiders/sitemap.py                       | 5 +----
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index dd32c62de..203dee42d 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,6 +1,6 @@
 import zlib
 
-from scrapy.utils.gz import gunzip, is_gzipped
+from scrapy.utils.gz import gunzip
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
 from scrapy.exceptions import NotConfigured
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 10af90259..e54001d88 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -61,10 +61,7 @@ class SitemapSpider(Spider):
         if isinstance(response, XmlResponse):
             return response.body
         elif gzip_magic_number(response):
-            try:
-                return gunzip(response.body)
-            except:
-                pass
+            return gunzip(response.body)
         # actual gzipped sitemap files are decompressed above ;
         # if we are here (response body is not gzipped)
         # and have a response for .xml.gz,

From 4caceccd594f2563a3d32ea708579ce8e70132f3 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 7 Mar 2017 10:51:34 +0100
Subject: [PATCH 1409/2974] Use 3-bytes for gzip archive type sniffing

---
 scrapy/utils/gz.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 22cf58986..16c9ce539 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -62,4 +62,4 @@ def is_gzipped(response):
 
 
 def gzip_magic_number(response):
-    return response.body[:2] == b'\x1f\x8b'
+    return response.body[:3] == b'\x1f\x8b\x08'

From 6f55ca4643dfff7a163714d696eef66d06cb81a8 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Tue, 7 Mar 2017 14:20:52 +0300
Subject: [PATCH 1410/2974] Revert unneeded test_crawl changes

---
 tests/test_crawl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3c5d9b958..d5babdded 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -97,8 +97,8 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_start_requests_bug_before_yield(self):
-        crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
         with LogCapture('scrapy', level=logging.ERROR) as l:
+            crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_before_yield=1)
 
         self.assertEqual(len(l.records), 1)
@@ -108,8 +108,8 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_start_requests_bug_yielding(self):
-        crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
         with LogCapture('scrapy', level=logging.ERROR) as l:
+            crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_yielding=1)
 
         self.assertEqual(len(l.records), 1)

From b6378c7ef6393412165239fd6bf489d45a1c5196 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 7 Mar 2017 12:28:24 +0100
Subject: [PATCH 1411/2974] Revert to using self.assert methods

---
 ...est_downloadermiddleware_httpcompression.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 5f56c99ec..0678fcb14 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -174,9 +174,9 @@ class HttpCompressionTest(TestCase):
         request = response.request
 
         newresponse = self.mw.process_response(request, response, self.spider)
-        assert newresponse is not response
-        assert newresponse.body.startswith(b'<!DOCTYPE')
-        assert 'Content-Encoding' not in newresponse.headers
+        self.assertIsNot(newresponse, response)
+        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
+        self.assertNotIn('Content-Encoding', newresponse.headers)
 
     def test_process_response_gzip_app_octetstream_contenttype(self):
         response = self._getresponse('gzip')
@@ -184,9 +184,9 @@ class HttpCompressionTest(TestCase):
         request = response.request
 
         newresponse = self.mw.process_response(request, response, self.spider)
-        assert newresponse is not response
-        assert newresponse.body.startswith(b'<!DOCTYPE')
-        assert 'Content-Encoding' not in newresponse.headers
+        self.assertIsNot(newresponse, response)
+        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
+        self.assertNotIn('Content-Encoding', newresponse.headers)
 
     def test_process_response_gzip_binary_octetstream_contenttype(self):
         response = self._getresponse('x-gzip')
@@ -194,9 +194,9 @@ class HttpCompressionTest(TestCase):
         request = response.request
 
         newresponse = self.mw.process_response(request, response, self.spider)
-        assert newresponse is not response
-        assert newresponse.body.startswith(b'<!DOCTYPE')
-        assert 'Content-Encoding' not in newresponse.headers
+        self.assertIsNot(newresponse, response)
+        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
+        self.assertNotIn('Content-Encoding', newresponse.headers)
 
     def test_process_response_gzipped_gzip_file(self):
         """Test that a gzip Content-Encoded .gz file is gunzipped

From a017f7b93298f8fcf5252afd8674466de95f04f9 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 7 Mar 2017 14:44:27 +0100
Subject: [PATCH 1412/2974] Warn about modules where duplicate spider names
 were found

---
 scrapy/spiderloader.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 1322c01d1..27a909c9f 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
+from collections import defaultdict
 import traceback
 import warnings
 
@@ -19,14 +20,21 @@ class SpiderLoader(object):
     def __init__(self, settings):
         self.spider_modules = settings.getlist('SPIDER_MODULES')
         self._spiders = {}
+        self._found = defaultdict(list)
         self._load_all_spiders()
 
     def _load_spiders(self, module):
         for spcls in iter_spider_classes(module):
+            self._found[spcls.name].append((module.__name__, spcls.__name__))
             if spcls.name in self._spiders.keys():
                 import warnings
-                warnings.warn("There are several spiders with the same name (" + spcls.name +
-                              "), this can cause unexpected behavior", UserWarning)
+                msg = ("There are several spiders with the same name {!r}:\n"
+                       "{}\n    This can cause unexpected behavior.".format(
+                            spcls.name,
+                            "\n".join(
+                                "        {1} (in {0})".format(mod, cls)
+                                for (mod, cls) in self._found[spcls.name])))
+                warnings.warn(msg, UserWarning)
             self._spiders[spcls.name] = spcls
 
     def _load_all_spiders(self):

From 978306a2236403f6275bad11e85d1e42ca37d6f1 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 7 Mar 2017 14:48:16 +0100
Subject: [PATCH 1413/2974] Fix dupe spider name warning string tests

---
 tests/test_spiderloader/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 302bf9a10..4600e53dc 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -127,7 +127,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             spider_loader = SpiderLoader.from_settings(self.settings)
 
             self.assertEqual(len(w), 1)
-            self.assertIn("several spiders with the same name (spider3)", str(w[0].message))
+            self.assertIn("several spiders with the same name 'spider3'", str(w[0].message))
 
             spiders = set(spider_loader.list())
             self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))
@@ -145,8 +145,8 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
 
             self.assertEqual(len(w), 2)
             msgs = sorted(str(wrn.message) for wrn in w)
-            self.assertIn("several spiders with the same name (spider1)", msgs[0])
-            self.assertIn("several spiders with the same name (spider2)", msgs[1])
+            self.assertIn("several spiders with the same name 'spider1'", msgs[0])
+            self.assertIn("several spiders with the same name 'spider2'", msgs[1])
 
             spiders = set(spider_loader.list())
             self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))

From 0b9a18e1a120751b8a2b2f1cc04c62b97967d612 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 7 Mar 2017 15:41:17 +0100
Subject: [PATCH 1414/2974] Warn only once for all spiders

---
 scrapy/spiderloader.py              | 22 +++++++++++++---------
 tests/test_spiderloader/__init__.py | 13 ++++++++-----
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 27a909c9f..486a4637e 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -23,18 +23,21 @@ class SpiderLoader(object):
         self._found = defaultdict(list)
         self._load_all_spiders()
 
+    def _check_name_duplicates(self):
+        dupes = ["\n".join("  {cls} named {name!r} (in {module})".format(
+                                module=mod, cls=cls, name=name)
+                           for (mod, cls) in locations)
+                 for name, locations in self._found.items()
+                 if len(locations)>1]
+        if dupes:
+            msg = ("There are several spiders with the same name:\n\n"
+                   "{}\n\n  This can cause unexpected behavior.".format(
+                        "\n\n".join(dupes)))
+            warnings.warn(msg, UserWarning)
+
     def _load_spiders(self, module):
         for spcls in iter_spider_classes(module):
             self._found[spcls.name].append((module.__name__, spcls.__name__))
-            if spcls.name in self._spiders.keys():
-                import warnings
-                msg = ("There are several spiders with the same name {!r}:\n"
-                       "{}\n    This can cause unexpected behavior.".format(
-                            spcls.name,
-                            "\n".join(
-                                "        {1} (in {0})".format(mod, cls)
-                                for (mod, cls) in self._found[spcls.name])))
-                warnings.warn(msg, UserWarning)
             self._spiders[spcls.name] = spcls
 
     def _load_all_spiders(self):
@@ -47,6 +50,7 @@ class SpiderLoader(object):
                        "Check SPIDER_MODULES setting".format(
                             modname=name, tb=traceback.format_exc()))
                 warnings.warn(msg, RuntimeWarning)
+        self._check_name_duplicates()
 
     @classmethod
     def from_settings(cls, settings):
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 4600e53dc..673a2d302 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -127,7 +127,9 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             spider_loader = SpiderLoader.from_settings(self.settings)
 
             self.assertEqual(len(w), 1)
-            self.assertIn("several spiders with the same name 'spider3'", str(w[0].message))
+            msg = str(w[0].message)
+            self.assertIn("several spiders with the same name", msg)
+            self.assertIn("'spider3'", msg)
 
             spiders = set(spider_loader.list())
             self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))
@@ -143,10 +145,11 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
         with warnings.catch_warnings(record=True) as w:
             spider_loader = SpiderLoader.from_settings(self.settings)
 
-            self.assertEqual(len(w), 2)
-            msgs = sorted(str(wrn.message) for wrn in w)
-            self.assertIn("several spiders with the same name 'spider1'", msgs[0])
-            self.assertIn("several spiders with the same name 'spider2'", msgs[1])
+            self.assertEqual(len(w), 1)
+            msg = str(w[0].message)
+            self.assertIn("several spiders with the same name", msg)
+            self.assertIn("'spider1'", msg)
+            self.assertIn("'spider2'", msg)
 
             spiders = set(spider_loader.list())
             self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))

From 7e9153b38d7ef70f1f19a82506b669433d134b01 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 19 Dec 2016 10:43:04 -0300
Subject: [PATCH 1415/2974] Feed exports: beautify JSON and XML

---
 docs/topics/exporters.rst           |   9 ++-
 docs/topics/feed-exports.rst        |  14 ++++
 scrapy/exporters.py                 |  30 ++++++--
 scrapy/extensions/feedexport.py     |   3 +-
 scrapy/settings/default_settings.py |   1 +
 tests/test_feedexport.py            | 113 ++++++++++++++++++++++++++--
 6 files changed, 156 insertions(+), 14 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 85c73222d..4114eda58 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -140,7 +140,7 @@ output examples, which assume you're exporting these two items::
 BaseItemExporter
 ----------------
 
-.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8')
+.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent_width=None)
 
    This is the (abstract) base class for all Item Exporters. It provides
    support for common features used by all (concrete) Item Exporters, such as
@@ -149,7 +149,7 @@ BaseItemExporter
 
    These features can be configured through the constructor arguments which
    populate their respective instance attributes: :attr:`fields_to_export`,
-   :attr:`export_empty_fields`, :attr:`encoding`.
+   :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent_width`.
 
    .. method:: export_item(item)
 
@@ -216,6 +216,11 @@ BaseItemExporter
       encoding). Other value types are passed unchanged to the specific
       serialization library.
 
+   .. attribute:: indent_width
+
+      Amount of spaces used to indent the output on each level.
+      Defaults to ``None``, which disables indentation.
+
 .. highlight:: none
 
 XmlItemExporter
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index efdd8c46b..ce3b5fd75 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -209,6 +209,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_STORE_EMPTY`
  * :setting:`FEED_EXPORT_ENCODING`
  * :setting:`FEED_EXPORT_FIELDS`
+ * :setting:`FEED_EXPORT_INDENT_WIDTH`
 
 .. currentmodule:: scrapy.extensions.feedexport
 
@@ -266,6 +267,19 @@ If an exporter requires a fixed set of fields (this is the case for
 is empty or None, then Scrapy tries to infer field names from the
 exported data - currently it uses field names from the first item.
 
+.. setting:: FEED_EXPORT_INDENT_WIDTH
+
+FEED_EXPORT_INDENT_WIDTH
+------------------------
+
+Default: ``None``
+
+Amount of spaces to indent on each level.
+Set to `None` to disable indentation.
+
+Currently used by :class:`~scrapy.exporters.JsonItemExporter`
+and :class:`~scrapy.exporters.XmlItemExporter`
+
 .. setting:: FEED_STORE_EMPTY
 
 FEED_STORE_EMPTY
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index c4b1b3476..69e6c15e0 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -36,6 +36,7 @@ class BaseItemExporter(object):
         self.encoding = options.pop('encoding', None)
         self.fields_to_export = options.pop('fields_to_export', None)
         self.export_empty_fields = options.pop('export_empty_fields', False)
+        self.indent_width = options.pop('indent_width', None)
         if not dont_fail and options:
             raise TypeError("Unexpected options: %s" % ', '.join(options.keys()))
 
@@ -99,7 +100,7 @@ class JsonItemExporter(BaseItemExporter):
         self._configure(kwargs, dont_fail=True)
         self.file = file
         kwargs.setdefault('ensure_ascii', not self.encoding)
-        self.encoder = ScrapyJSONEncoder(**kwargs)
+        self.encoder = ScrapyJSONEncoder(indent=self.indent_width, **kwargs)
         self.first_item = True
 
     def start_exporting(self):
@@ -128,33 +129,52 @@ class XmlItemExporter(BaseItemExporter):
             self.encoding = 'utf-8'
         self.xg = XMLGenerator(file, encoding=self.encoding)
 
+    def _beautify_newline(self):
+        if self.indent_width:
+            self._xg_characters('\n')
+
+    def _beautify_indent(self, depth=1):
+        if self.indent_width:
+            self._xg_characters(' ' * self.indent_width * depth)
+
     def start_exporting(self):
         self.xg.startDocument()
         self.xg.startElement(self.root_element, {})
+        self._beautify_newline()
 
     def export_item(self, item):
+        self._beautify_indent(depth=1)
         self.xg.startElement(self.item_element, {})
+        self._beautify_newline()
         for name, value in self._get_serialized_fields(item, default_value=''):
-            self._export_xml_field(name, value)
+            self._export_xml_field(name, value, depth=2)
+        self._beautify_indent(depth=1)
         self.xg.endElement(self.item_element)
+        self._beautify_newline()
 
     def finish_exporting(self):
         self.xg.endElement(self.root_element)
         self.xg.endDocument()
 
-    def _export_xml_field(self, name, serialized_value):
+    def _export_xml_field(self, name, serialized_value, depth):
+        self._beautify_indent(depth=depth)
         self.xg.startElement(name, {})
         if hasattr(serialized_value, 'items'):
+            self._beautify_newline()
             for subname, value in serialized_value.items():
-                self._export_xml_field(subname, value)
+                self._export_xml_field(subname, value, depth=depth+1)
+            self._beautify_indent(depth=depth)
         elif is_listlike(serialized_value):
+            self._beautify_newline()
             for value in serialized_value:
-                self._export_xml_field('value', value)
+                self._export_xml_field('value', value, depth=depth+1)
+            self._beautify_indent(depth=depth)
         elif isinstance(serialized_value, six.text_type):
             self._xg_characters(serialized_value)
         else:
             self._xg_characters(str(serialized_value))
         self.xg.endElement(name)
+        self._beautify_newline()
 
     # Workaround for http://bugs.python.org/issue17606
     # Before Python 2.7.4 xml.sax.saxutils required bytes;
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 85d328528..26024e5e9 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -172,6 +172,7 @@ class FeedExporter(object):
         self.store_empty = settings.getbool('FEED_STORE_EMPTY')
         self._exporting = False
         self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
+        self.indent_width = settings.getint('FEED_EXPORT_INDENT_WIDTH') or None
         uripar = settings['FEED_URI_PARAMS']
         self._uripar = load_object(uripar) if uripar else lambda x, y: None
 
@@ -188,7 +189,7 @@ class FeedExporter(object):
         storage = self._get_storage(uri)
         file = storage.open(spider)
         exporter = self._get_exporter(file, fields_to_export=self.export_fields,
-            encoding=self.export_encoding)
+            encoding=self.export_encoding, indent_width=self.indent_width)
         if self.store_empty:
             exporter.start_exporting()
             self._exporting = True
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index d73c595d2..cca0d3889 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -161,6 +161,7 @@ FEED_EXPORTERS_BASE = {
     'marshal': 'scrapy.exporters.MarshalItemExporter',
     'pickle': 'scrapy.exporters.PickleItemExporter',
 }
+FEED_EXPORT_INDENT_WIDTH = None
 
 FILES_STORE_S3_ACL = 'private'
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 2d137edf4..bf002bec7 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -431,10 +431,10 @@ class FeedExportTest(unittest.TestCase):
             'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
         }
 
-        for format in formats:
-            settings = {'FEED_FORMAT': format}
+        for format, expected in formats.items():
+            settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT_WIDTH': None}
             data = yield self.exported_data(items, settings)
-            self.assertEqual(formats[format], data)
+            self.assertEqual(expected, data)
 
         formats = {
             'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'),
@@ -443,7 +443,108 @@ class FeedExportTest(unittest.TestCase):
             'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
         }
 
-        for format in formats:
-            settings = {'FEED_FORMAT': format, 'FEED_EXPORT_ENCODING': 'latin-1'}
+        settings = {'FEED_EXPORT_INDENT_WIDTH': None, 'FEED_EXPORT_ENCODING': 'latin-1'}
+        for format, expected in formats.items():
+            settings['FEED_FORMAT'] = format
             data = yield self.exported_data(items, settings)
-            self.assertEqual(formats[format], data)
+            self.assertEqual(expected, data)
+
+    @defer.inlineCallbacks
+    def test_export_indentation(self):
+        items = [dict({'foo': ['bar']})]
+
+        output = [
+            # JSON
+            {
+                'format': 'json',
+                'indent_width': None,
+                'expected': b'[\n{"foo": ["bar"]}\n]',
+            },
+            {
+                'format': 'json',
+                'indent_width': 2,
+                'expected': b"""
+[
+{
+  "foo": [
+    "bar"
+  ]
+}
+]""",
+            },
+            {
+                'format': 'json',
+                'indent_width': 4,
+                'expected': b"""
+[
+{
+    "foo": [
+        "bar"
+    ]
+}
+]""",
+            },
+            {
+                'format': 'json',
+                'indent_width': 5,
+                'expected': b"""
+[
+{
+     "foo": [
+          "bar"
+     ]
+}
+]""",
+            },
+
+            # XML
+            {
+                'format': 'xml',
+                'indent_width': None,
+                'expected': b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo><value>bar</value></foo></item></items>',
+            },
+            {
+                'format': 'xml',
+                'indent_width': 2,
+                'expected': b"""
+<?xml version="1.0" encoding="utf-8"?>
+<items>
+  <item>
+    <foo>
+      <value>bar</value>
+    </foo>
+  </item>
+</items>""",
+            },
+            {
+                'format': 'xml',
+                'indent_width': 4,
+                'expected': b"""
+<?xml version="1.0" encoding="utf-8"?>
+<items>
+    <item>
+        <foo>
+            <value>bar</value>
+        </foo>
+    </item>
+</items>""",
+            },
+            {
+                'format': 'xml',
+                'indent_width': 5,
+                'expected': b"""
+<?xml version="1.0" encoding="utf-8"?>
+<items>
+     <item>
+          <foo>
+               <value>bar</value>
+          </foo>
+     </item>
+</items>""",
+            },
+        ]
+
+        for row in output:
+            settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT_WIDTH': row['indent_width']}
+            data = yield self.exported_data(items, settings)
+            self.assertEqual(row['expected'].strip(), data)

From 766b2c84539d58ee871e1f301df1ad0ae0d44079 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 23 Feb 2017 10:21:33 -0300
Subject: [PATCH 1416/2974] Feed exports: enforce difference between None and 0
 on indent

Also rename params and settings from "indent_width" to just "indent"
---
 docs/topics/exporters.rst           |  14 ++--
 docs/topics/feed-exports.rst        |  14 ++--
 scrapy/exporters.py                 |  24 ++++--
 scrapy/extensions/feedexport.py     |   6 +-
 scrapy/settings/default_settings.py |   2 +-
 tests/test_feedexport.py            | 116 ++++++++++++++++++++++++----
 6 files changed, 137 insertions(+), 39 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 4114eda58..ad559fb35 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -140,7 +140,7 @@ output examples, which assume you're exporting these two items::
 BaseItemExporter
 ----------------
 
-.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent_width=None)
+.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=None)
 
    This is the (abstract) base class for all Item Exporters. It provides
    support for common features used by all (concrete) Item Exporters, such as
@@ -149,7 +149,7 @@ BaseItemExporter
 
    These features can be configured through the constructor arguments which
    populate their respective instance attributes: :attr:`fields_to_export`,
-   :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent_width`.
+   :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
 
    .. method:: export_item(item)
 
@@ -216,10 +216,14 @@ BaseItemExporter
       encoding). Other value types are passed unchanged to the specific
       serialization library.
 
-   .. attribute:: indent_width
+   .. attribute:: indent
 
-      Amount of spaces used to indent the output on each level.
-      Defaults to ``None``, which disables indentation.
+      Amount of spaces used to indent the output on each level. Defaults to ``None``,
+      which disables indentation. This argument behaves like ``indent`` in python's
+      JSON module (both for JSON and XML exporters): "If ``indent`` is a non-negative
+      integer, then array elements and object members will be pretty-printed with that
+      indent level. An indent level of 0, or negative, will only insert newlines.
+      ``None`` (the default) selects the most compact representation"
 
 .. highlight:: none
 
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index ce3b5fd75..afaa972e5 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -209,7 +209,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_STORE_EMPTY`
  * :setting:`FEED_EXPORT_ENCODING`
  * :setting:`FEED_EXPORT_FIELDS`
- * :setting:`FEED_EXPORT_INDENT_WIDTH`
+ * :setting:`FEED_EXPORT_INDENT`
 
 .. currentmodule:: scrapy.extensions.feedexport
 
@@ -267,15 +267,17 @@ If an exporter requires a fixed set of fields (this is the case for
 is empty or None, then Scrapy tries to infer field names from the
 exported data - currently it uses field names from the first item.
 
-.. setting:: FEED_EXPORT_INDENT_WIDTH
+.. setting:: FEED_EXPORT_INDENT
 
-FEED_EXPORT_INDENT_WIDTH
-------------------------
+FEED_EXPORT_INDENT
+------------------
 
 Default: ``None``
 
-Amount of spaces to indent on each level.
-Set to `None` to disable indentation.
+Amount of spaces used to indent the output on each level. If ``FEED_EXPORT_INDENT``
+is a non-negative integer, then array elements and object members will be pretty-printed
+with that indent level. An indent level of 0, or negative, will only insert newlines.
+``None`` (the default) selects the most compact representation
 
 Currently used by :class:`~scrapy.exporters.JsonItemExporter`
 and :class:`~scrapy.exporters.XmlItemExporter`
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 69e6c15e0..1dfa2af85 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -36,7 +36,7 @@ class BaseItemExporter(object):
         self.encoding = options.pop('encoding', None)
         self.fields_to_export = options.pop('fields_to_export', None)
         self.export_empty_fields = options.pop('export_empty_fields', False)
-        self.indent_width = options.pop('indent_width', None)
+        self.indent = options.pop('indent', None)
         if not dont_fail and options:
             raise TypeError("Unexpected options: %s" % ', '.join(options.keys()))
 
@@ -100,20 +100,28 @@ class JsonItemExporter(BaseItemExporter):
         self._configure(kwargs, dont_fail=True)
         self.file = file
         kwargs.setdefault('ensure_ascii', not self.encoding)
-        self.encoder = ScrapyJSONEncoder(indent=self.indent_width, **kwargs)
+        kwargs.setdefault('indent', self.indent)
+        self.encoder = ScrapyJSONEncoder(**kwargs)
         self.first_item = True
 
+    def _beautify_newline(self):
+        if self.indent is not None:
+            self.file.write(b'\n')
+
     def start_exporting(self):
-        self.file.write(b"[\n")
+        self.file.write(b"[")
+        self._beautify_newline()
 
     def finish_exporting(self):
-        self.file.write(b"\n]")
+        self._beautify_newline()
+        self.file.write(b"]")
 
     def export_item(self, item):
         if self.first_item:
             self.first_item = False
         else:
-            self.file.write(b',\n')
+            self.file.write(b',')
+            self._beautify_newline()
         itemdict = dict(self._get_serialized_fields(item))
         data = self.encoder.encode(itemdict)
         self.file.write(to_bytes(data, self.encoding))
@@ -130,12 +138,12 @@ class XmlItemExporter(BaseItemExporter):
         self.xg = XMLGenerator(file, encoding=self.encoding)
 
     def _beautify_newline(self):
-        if self.indent_width:
+        if self.indent is not None:
             self._xg_characters('\n')
 
     def _beautify_indent(self, depth=1):
-        if self.indent_width:
-            self._xg_characters(' ' * self.indent_width * depth)
+        if self.indent:
+            self._xg_characters(' ' * self.indent * depth)
 
     def start_exporting(self):
         self.xg.startDocument()
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 26024e5e9..5f133fbde 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -172,7 +172,9 @@ class FeedExporter(object):
         self.store_empty = settings.getbool('FEED_STORE_EMPTY')
         self._exporting = False
         self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
-        self.indent_width = settings.getint('FEED_EXPORT_INDENT_WIDTH') or None
+        self.indent = None
+        if settings.get('FEED_EXPORT_INDENT') is not None:
+            self.indent = settings.getint('FEED_EXPORT_INDENT')
         uripar = settings['FEED_URI_PARAMS']
         self._uripar = load_object(uripar) if uripar else lambda x, y: None
 
@@ -189,7 +191,7 @@ class FeedExporter(object):
         storage = self._get_storage(uri)
         file = storage.open(spider)
         exporter = self._get_exporter(file, fields_to_export=self.export_fields,
-            encoding=self.export_encoding, indent_width=self.indent_width)
+            encoding=self.export_encoding, indent=self.indent)
         if self.store_empty:
             exporter.start_exporting()
             self._exporting = True
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index cca0d3889..fc265e2ba 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -161,7 +161,7 @@ FEED_EXPORTERS_BASE = {
     'marshal': 'scrapy.exporters.MarshalItemExporter',
     'pickle': 'scrapy.exporters.PickleItemExporter',
 }
-FEED_EXPORT_INDENT_WIDTH = None
+FEED_EXPORT_INDENT = None
 
 FILES_STORE_S3_ACL = 'private'
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index bf002bec7..2b82bba0c 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -319,7 +319,7 @@ class FeedExportTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_export_no_items_store_empty(self):
         formats = (
-            ('json', b'[\n\n]'),
+            ('json', b'[]'),
             ('jsonlines', b''),
             ('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
             ('csv', b''),
@@ -425,25 +425,25 @@ class FeedExportTest(unittest.TestCase):
         header = ['foo']
 
         formats = {
-            'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'),
+            'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
             'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
             'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
         }
 
         for format, expected in formats.items():
-            settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT_WIDTH': None}
+            settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT': None}
             data = yield self.exported_data(items, settings)
             self.assertEqual(expected, data)
 
         formats = {
-            'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'),
+            'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'),
             'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),
             'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
         }
 
-        settings = {'FEED_EXPORT_INDENT_WIDTH': None, 'FEED_EXPORT_ENCODING': 'latin-1'}
+        settings = {'FEED_EXPORT_INDENT': None, 'FEED_EXPORT_ENCODING': 'latin-1'}
         for format, expected in formats.items():
             settings['FEED_FORMAT'] = format
             data = yield self.exported_data(items, settings)
@@ -451,48 +451,89 @@ class FeedExportTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_export_indentation(self):
-        items = [dict({'foo': ['bar']})]
+        items = [dict({'foo': ['bar']}), dict({'key': 'value'})]
 
         output = [
             # JSON
             {
                 'format': 'json',
-                'indent_width': None,
-                'expected': b'[\n{"foo": ["bar"]}\n]',
+                'indent': None,
+                'expected': b'[{"foo": ["bar"]},{"key": "value"}]',
             },
             {
                 'format': 'json',
-                'indent_width': 2,
+                'indent': -1,
+                'expected': b"""
+[
+{
+"foo": [
+"bar"
+]
+},
+{
+"key": "value"
+}
+]
+""",
+            },
+            {
+                'format': 'json',
+                'indent': 0,
+                'expected': b"""
+[
+{
+"foo": [
+"bar"
+]
+},
+{
+"key": "value"
+}
+]
+""",
+            },
+            {
+                'format': 'json',
+                'indent': 2,
                 'expected': b"""
 [
 {
   "foo": [
     "bar"
   ]
+},
+{
+  "key": "value"
 }
 ]""",
             },
             {
                 'format': 'json',
-                'indent_width': 4,
+                'indent': 4,
                 'expected': b"""
 [
 {
     "foo": [
         "bar"
     ]
+},
+{
+    "key": "value"
 }
 ]""",
             },
             {
                 'format': 'json',
-                'indent_width': 5,
+                'indent': 5,
                 'expected': b"""
 [
 {
      "foo": [
           "bar"
      ]
+},
+{
+     "key": "value"
 }
 ]""",
             },
@@ -500,12 +541,44 @@ class FeedExportTest(unittest.TestCase):
             # XML
             {
                 'format': 'xml',
-                'indent_width': None,
-                'expected': b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo><value>bar</value></foo></item></items>',
+                'indent': None,
+                'expected': b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo><value>bar</value></foo></item><item><key>value</key></item></items>',
             },
             {
                 'format': 'xml',
-                'indent_width': 2,
+                'indent': -1,
+                'expected': b"""
+<?xml version="1.0" encoding="utf-8"?>
+<items>
+<item>
+<foo>
+<value>bar</value>
+</foo>
+</item>
+<item>
+<key>value</key>
+</item>
+</items>""",
+            },
+            {
+                'format': 'xml',
+                'indent': 0,
+                'expected': b"""
+<?xml version="1.0" encoding="utf-8"?>
+<items>
+<item>
+<foo>
+<value>bar</value>
+</foo>
+</item>
+<item>
+<key>value</key>
+</item>
+</items>""",
+            },
+            {
+                'format': 'xml',
+                'indent': 2,
                 'expected': b"""
 <?xml version="1.0" encoding="utf-8"?>
 <items>
@@ -514,11 +587,14 @@ class FeedExportTest(unittest.TestCase):
       <value>bar</value>
     </foo>
   </item>
+  <item>
+    <key>value</key>
+  </item>
 </items>""",
             },
             {
                 'format': 'xml',
-                'indent_width': 4,
+                'indent': 4,
                 'expected': b"""
 <?xml version="1.0" encoding="utf-8"?>
 <items>
@@ -527,11 +603,14 @@ class FeedExportTest(unittest.TestCase):
             <value>bar</value>
         </foo>
     </item>
+    <item>
+        <key>value</key>
+    </item>
 </items>""",
             },
             {
                 'format': 'xml',
-                'indent_width': 5,
+                'indent': 5,
                 'expected': b"""
 <?xml version="1.0" encoding="utf-8"?>
 <items>
@@ -540,11 +619,14 @@ class FeedExportTest(unittest.TestCase):
                <value>bar</value>
           </foo>
      </item>
+     <item>
+          <key>value</key>
+     </item>
 </items>""",
             },
         ]
 
         for row in output:
-            settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT_WIDTH': row['indent_width']}
+            settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']}
             data = yield self.exported_data(items, settings)
             self.assertEqual(row['expected'].strip(), data)

From c7bb2fa8ce2633d92a7ec2840f84b174a5494428 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 7 Mar 2017 11:55:26 -0300
Subject: [PATCH 1417/2974] Feed exports: consistent and backwards compatible
 behaviour on indent

---
 docs/topics/exporters.rst           | 14 ++++----
 docs/topics/feed-exports.rst        |  6 ++--
 scrapy/exporters.py                 | 14 +++++---
 scrapy/settings/default_settings.py |  2 +-
 tests/test_feedexport.py            | 56 ++++++++++-------------------
 5 files changed, 39 insertions(+), 53 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index ad559fb35..2ad77c905 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -140,7 +140,7 @@ output examples, which assume you're exporting these two items::
 BaseItemExporter
 ----------------
 
-.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=None)
+.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0)
 
    This is the (abstract) base class for all Item Exporters. It provides
    support for common features used by all (concrete) Item Exporters, such as
@@ -218,12 +218,12 @@ BaseItemExporter
 
    .. attribute:: indent
 
-      Amount of spaces used to indent the output on each level. Defaults to ``None``,
-      which disables indentation. This argument behaves like ``indent`` in python's
-      JSON module (both for JSON and XML exporters): "If ``indent`` is a non-negative
-      integer, then array elements and object members will be pretty-printed with that
-      indent level. An indent level of 0, or negative, will only insert newlines.
-      ``None`` (the default) selects the most compact representation"
+      Amount of spaces used to indent the output on each level. Defaults to ``0``.
+
+      * ``indent=None`` selects the most compact representation,
+        all items in the same line with no indentation
+      * ``indent<=0`` each item on it's own line, no indentation
+      * ``indent>0`` each item on it's own line, indentated with the provided numeric value
 
 .. highlight:: none
 
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index afaa972e5..e57a4e776 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -272,12 +272,12 @@ exported data - currently it uses field names from the first item.
 FEED_EXPORT_INDENT
 ------------------
 
-Default: ``None``
+Default: ``0``
 
 Amount of spaces used to indent the output on each level. If ``FEED_EXPORT_INDENT``
 is a non-negative integer, then array elements and object members will be pretty-printed
-with that indent level. An indent level of 0, or negative, will only insert newlines.
-``None`` (the default) selects the most compact representation
+with that indent level. An indent level of ``0``, or negative, will put each item on a new line.
+``None`` selects the most compact representation
 
 Currently used by :class:`~scrapy.exporters.JsonItemExporter`
 and :class:`~scrapy.exporters.XmlItemExporter`
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 1dfa2af85..e2d42b6ab 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -99,8 +99,12 @@ class JsonItemExporter(BaseItemExporter):
     def __init__(self, file, **kwargs):
         self._configure(kwargs, dont_fail=True)
         self.file = file
+        # there is a small difference between the behaviour or JsonItemExporter.indent
+        # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent
+        # the addition of newlines everywhere
+        json_indent = self.indent if self.indent is not None and self.indent > 0 else None
+        kwargs.setdefault('indent', json_indent)
         kwargs.setdefault('ensure_ascii', not self.encoding)
-        kwargs.setdefault('indent', self.indent)
         self.encoder = ScrapyJSONEncoder(**kwargs)
         self.first_item = True
 
@@ -137,8 +141,8 @@ class XmlItemExporter(BaseItemExporter):
             self.encoding = 'utf-8'
         self.xg = XMLGenerator(file, encoding=self.encoding)
 
-    def _beautify_newline(self):
-        if self.indent is not None:
+    def _beautify_newline(self, new_item=False):
+        if self.indent is not None and (self.indent > 0 or new_item):
             self._xg_characters('\n')
 
     def _beautify_indent(self, depth=1):
@@ -148,7 +152,7 @@ class XmlItemExporter(BaseItemExporter):
     def start_exporting(self):
         self.xg.startDocument()
         self.xg.startElement(self.root_element, {})
-        self._beautify_newline()
+        self._beautify_newline(new_item=True)
 
     def export_item(self, item):
         self._beautify_indent(depth=1)
@@ -158,7 +162,7 @@ class XmlItemExporter(BaseItemExporter):
             self._export_xml_field(name, value, depth=2)
         self._beautify_indent(depth=1)
         self.xg.endElement(self.item_element)
-        self._beautify_newline()
+        self._beautify_newline(new_item=True)
 
     def finish_exporting(self):
         self.xg.endElement(self.root_element)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index fc265e2ba..bbc02cfdb 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -161,7 +161,7 @@ FEED_EXPORTERS_BASE = {
     'marshal': 'scrapy.exporters.MarshalItemExporter',
     'pickle': 'scrapy.exporters.PickleItemExporter',
 }
-FEED_EXPORT_INDENT = None
+FEED_EXPORT_INDENT = 0
 
 FILES_STORE_S3_ACL = 'private'
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 2b82bba0c..c66c470a8 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -326,7 +326,7 @@ class FeedExportTest(unittest.TestCase):
         )
 
         for fmt, expctd in formats:
-            settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True}
+            settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None}
             data = yield self.exported_no_data(settings)
             self.assertEqual(data, expctd)
 
@@ -451,9 +451,12 @@ class FeedExportTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_export_indentation(self):
-        items = [dict({'foo': ['bar']}), dict({'key': 'value'})]
+        items = [
+            {'foo': ['bar']},
+            {'key': 'value'},
+        ]
 
-        output = [
+        test_cases = [
             # JSON
             {
                 'format': 'json',
@@ -465,14 +468,8 @@ class FeedExportTest(unittest.TestCase):
                 'indent': -1,
                 'expected': b"""
 [
-{
-"foo": [
-"bar"
-]
-},
-{
-"key": "value"
-}
+{"foo": ["bar"]},
+{"key": "value"}
 ]
 """,
             },
@@ -481,14 +478,8 @@ class FeedExportTest(unittest.TestCase):
                 'indent': 0,
                 'expected': b"""
 [
-{
-"foo": [
-"bar"
-]
-},
-{
-"key": "value"
-}
+{"foo": ["bar"]},
+{"key": "value"}
 ]
 """,
             },
@@ -542,7 +533,9 @@ class FeedExportTest(unittest.TestCase):
             {
                 'format': 'xml',
                 'indent': None,
-                'expected': b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo><value>bar</value></foo></item><item><key>value</key></item></items>',
+                'expected': b"""
+<?xml version="1.0" encoding="utf-8"?>
+<items><item><foo><value>bar</value></foo></item><item><key>value</key></item></items>""",
             },
             {
                 'format': 'xml',
@@ -550,14 +543,8 @@ class FeedExportTest(unittest.TestCase):
                 'expected': b"""
 <?xml version="1.0" encoding="utf-8"?>
 <items>
-<item>
-<foo>
-<value>bar</value>
-</foo>
-</item>
-<item>
-<key>value</key>
-</item>
+<item><foo><value>bar</value></foo></item>
+<item><key>value</key></item>
 </items>""",
             },
             {
@@ -566,14 +553,8 @@ class FeedExportTest(unittest.TestCase):
                 'expected': b"""
 <?xml version="1.0" encoding="utf-8"?>
 <items>
-<item>
-<foo>
-<value>bar</value>
-</foo>
-</item>
-<item>
-<key>value</key>
-</item>
+<item><foo><value>bar</value></foo></item>
+<item><key>value</key></item>
 </items>""",
             },
             {
@@ -626,7 +607,8 @@ class FeedExportTest(unittest.TestCase):
             },
         ]
 
-        for row in output:
+        for row in test_cases:
             settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']}
             data = yield self.exported_data(items, settings)
+            print(row['format'], row['indent'])
             self.assertEqual(row['expected'].strip(), data)

From 7be773e14ae27501914c3f3922f5504a7ac72f48 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 7 Mar 2017 17:40:40 +0100
Subject: [PATCH 1418/2974] Add SPIDER_LOADER_WARN_ONLY to toggle between
 spiderloader failure and warning

---
 scrapy/commands/list.py             |  3 ++-
 scrapy/commands/runspider.py        |  1 +
 scrapy/commands/settings.py         |  3 ++-
 scrapy/commands/startproject.py     |  5 +++--
 scrapy/commands/version.py          |  3 ++-
 scrapy/settings/default_settings.py |  1 +
 scrapy/spiderloader.py              | 12 ++++++++----
 tests/test_spiderloader/__init__.py | 11 +++++++++--
 8 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index a255b3b94..185a77a40 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -4,7 +4,8 @@ from scrapy.commands import ScrapyCommand
 class Command(ScrapyCommand):
 
     requires_project = True
-    default_settings = {'LOG_ENABLED': False}
+    default_settings = {'LOG_ENABLED': False,
+                        'SPIDER_LOADER_WARN_ONLY': True}
 
     def short_desc(self):
         return "List available spiders"
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 1da09e4da..a98033dd1 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -28,6 +28,7 @@ def _import_file(filepath):
 class Command(ScrapyCommand):
 
     requires_project = False
+    default_settings = {'SPIDER_LOADER_WARN_ONLY': True}
 
     def syntax(self):
         return "[options] <spider_file>"
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index bce4e6086..bee52f06a 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -7,7 +7,8 @@ from scrapy.settings import BaseSettings
 class Command(ScrapyCommand):
 
     requires_project = False
-    default_settings = {'LOG_ENABLED': False}
+    default_settings = {'LOG_ENABLED': False,
+                        'SPIDER_LOADER_WARN_ONLY': True}
 
     def syntax(self):
         return "[options]"
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 594106632..c17aaf442 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -26,7 +26,8 @@ IGNORE = ignore_patterns('*.pyc', '.svn')
 class Command(ScrapyCommand):
 
     requires_project = False
-    default_settings = {'LOG_ENABLED': False}
+    default_settings = {'LOG_ENABLED': False,
+                        'SPIDER_LOADER_WARN_ONLY': True}
 
     def syntax(self):
         return "<project_name> [project_dir]"
@@ -118,4 +119,4 @@ class Command(ScrapyCommand):
         _templates_base_dir = self.settings['TEMPLATES_DIR'] or \
             join(scrapy.__path__[0], 'templates')
         return join(_templates_base_dir, 'project')
-    
+
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index a9954edb0..e22f98f5a 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -11,7 +11,8 @@ from scrapy.commands import ScrapyCommand
 
 class Command(ScrapyCommand):
 
-    default_settings = {'LOG_ENABLED': False}
+    default_settings = {'LOG_ENABLED': False,
+                        'SPIDER_LOADER_WARN_ONLY': True}
 
     def syntax(self):
         return "[-v]"
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index d73c595d2..854cefc9c 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -250,6 +250,7 @@ SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
 SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'
 
 SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
+SPIDER_LOADER_WARN_ONLY = False
 
 SPIDER_MIDDLEWARES = {}
 
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 486a4637e..7478faa78 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -19,6 +19,7 @@ class SpiderLoader(object):
     """
     def __init__(self, settings):
         self.spider_modules = settings.getlist('SPIDER_MODULES')
+        self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')
         self._spiders = {}
         self._found = defaultdict(list)
         self._load_all_spiders()
@@ -46,10 +47,13 @@ class SpiderLoader(object):
                 for module in walk_modules(name):
                     self._load_spiders(module)
             except ImportError as e:
-                msg = ("\n{tb}Could not load spiders from module '{modname}'. "
-                       "Check SPIDER_MODULES setting".format(
-                            modname=name, tb=traceback.format_exc()))
-                warnings.warn(msg, RuntimeWarning)
+                if self.warn_only:
+                    msg = ("\n{tb}Could not load spiders from module '{modname}'. "
+                           "See above traceback for details.".format(
+                                modname=name, tb=traceback.format_exc()))
+                    warnings.warn(msg, RuntimeWarning)
+                else:
+                    raise
         self._check_name_duplicates()
 
     @classmethod
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 673a2d302..99a61daea 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -91,18 +91,25 @@ class SpiderLoaderTest(unittest.TestCase):
         self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
         self.assertEqual(crawler.spidercls.name, 'spider1')
 
+    def test_bad_spider_modules_exception(self):
+
+        module = 'tests.test_spiderloader.test_spiders.doesnotexist'
+        settings = Settings({'SPIDER_MODULES': [module]})
+        with self.assertRaises(ImportError):
+            SpiderLoader.from_settings(settings)
+
     def test_bad_spider_modules_warning(self):
 
         with warnings.catch_warnings(record=True) as w:
             module = 'tests.test_spiderloader.test_spiders.doesnotexist'
-            settings = Settings({'SPIDER_MODULES': [module]})
+            settings = Settings({'SPIDER_MODULES': [module],
+                                 'SPIDER_LOADER_WARN_ONLY': True})
             spider_loader = SpiderLoader.from_settings(settings)
             self.assertIn("Could not load spiders from module", str(w[0].message))
 
             spiders = spider_loader.list()
             self.assertEqual(spiders, [])
 
-
 class DuplicateSpiderNameLoaderTest(unittest.TestCase):
 
     def setUp(self):

From 4090cc3990636337964a6e157679d6be15ba6f3a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 8 Mar 2017 18:11:20 -0300
Subject: [PATCH 1419/2974] Spider middleware: use Mockserver to test
 process_spider_exception

---
 tests/test_spidermiddleware.py | 42 +++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 8ec9583d8..3981a8d75 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -9,16 +9,20 @@ from scrapy.spiders import Spider
 from scrapy.item import Item, Field
 from scrapy.http import Request
 from scrapy.utils.test import get_crawler
+from tests.mockserver import MockServer
 
 
 class TestItem(Item):
     value = Field()
 
 
+class LocalhostSpider(Spider):
+    start_urls = ['http://localhost:8998']  # tests.mockserver.MockServer
+
+
 # ================================================================================
 # exceptions from a spider's parse method
-class BaseExceptionFromParseMethodSpider(Spider):
-    start_urls = ["http://example.com/"]
+class BaseExceptionFromParseMethodSpider(LocalhostSpider):
     custom_settings = {
         'SPIDER_MIDDLEWARES': {'tests.test_spidermiddleware.CatchExceptionMiddleware': 540}
     }
@@ -65,8 +69,7 @@ class CatchExceptionMiddleware(object):
 # exception from a previous middleware's process_spider_input method
 # process_spider_input is not expected to return an iterable, so there are no
 # separate tests for generator/non-generator implementations
-class FromPreviousMiddlewareInputSpider(Spider):
-    start_urls = ["http://example.com/"]
+class FromPreviousMiddlewareInputSpider(LocalhostSpider):
     name = 'not_a_generator_from_previous_middleware_input'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
@@ -88,8 +91,7 @@ class RaiseExceptionOnInputMiddleware(object):
 
 # ================================================================================
 # exception from a previous middleware's process_spider_output method (not a generator)
-class NotAGeneratorFromPreviousMiddlewareOutputSpider(Spider):
-    start_urls = ["http://example.com/"]
+class NotAGeneratorFromPreviousMiddlewareOutputSpider(LocalhostSpider):
     name = 'not_a_generator_from_previous_middleware_output'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
@@ -111,8 +113,7 @@ class RaiseExceptionOnOutputNotAGeneratorMiddleware(object):
 
 # ================================================================================
 # exception from a previous middleware's process_spider_output method (generator)
-class GeneratorFromPreviousMiddlewareOutputSpider(Spider):
-    start_urls = ["http://example.com/"]
+class GeneratorFromPreviousMiddlewareOutputSpider(LocalhostSpider):
     name = 'generator_from_previous_middleware_output'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
@@ -136,8 +137,7 @@ class RaiseExceptionOnOutputGeneratorMiddleware(object):
 
 # ================================================================================
 # do something useful from the exception handler
-class DoSomethingSpider(Spider):
-    start_urls = ["http://example.com"]
+class DoSomethingSpider(LocalhostSpider):
     name = 'do_something'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
@@ -155,13 +155,12 @@ class DoSomethingSpider(Spider):
 
 class DoSomethingMiddleware(object):
     def process_spider_exception(self, response, exception, spider):
-        return [Request('http://example.org'), {'value': 10}, TestItem(value='asdf')]
+        return [Request('http://localhost:8998?processed=true'), {'value': 10}, TestItem(value='asdf')]
 
 
 # ================================================================================
 # don't catch InvalidOutput from scrapy's spider middleware manager
-class InvalidReturnValueFromPreviousMiddlewareInputSpider(Spider):
-    start_urls = ["http://example.com/"]
+class InvalidReturnValueFromPreviousMiddlewareInputSpider(LocalhostSpider):
     name = 'invalid_return_value_from_previous_middleware_input'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
@@ -181,8 +180,7 @@ class InvalidReturnValueInputMiddleware(object):
         return 1.0  # <type 'float'>, not None
 
 
-class InvalidReturnValueFromPreviousMiddlewareOutputSpider(Spider):
-    start_urls = ["http://example.com/"]
+class InvalidReturnValueFromPreviousMiddlewareOutputSpider(LocalhostSpider):
     name = 'invalid_return_value_from_previous_middleware_output'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
@@ -205,8 +203,7 @@ class InvalidReturnValueOutputMiddleware(object):
 # ================================================================================
 # make sure only non already called process_spider_output methods
 # are called if process_spider_exception returns an iterable
-class ExecutionChainSpider(Spider):
-    start_urls = ["http://example.com"]
+class ExecutionChainSpider(LocalhostSpider):
     name = 'execution_chain'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
@@ -258,6 +255,13 @@ class ThirdMiddleware(object):
 
 class TestSpiderMiddleware(TestCase):
 
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
     @defer.inlineCallbacks
     def test_process_spider_exception_from_parse_method(self):
         # non-generator return value
@@ -309,8 +313,8 @@ class TestSpiderMiddleware(TestCase):
         self.assertIn("ImportError exception caught", str(log))
         self.assertIn("{'value': 10}", str(log))
         self.assertIn("{'value': 'asdf'}", str(log))
-        self.assertIn("{'value': 'http://example.com'}", str(log))
-        self.assertIn("{'value': 'http://example.org'}", str(log))
+        self.assertIn("{'value': 'http://localhost:8998'}", str(log))
+        self.assertIn("{'value': 'http://localhost:8998?processed=true'}", str(log))
 
     @defer.inlineCallbacks
     def test_process_spider_exception_invalid_return_value_previous_middleware(self):

From ac63d3a3cf0a45b9ae66c9847d07d287154078e6 Mon Sep 17 00:00:00 2001
From: jorenham <jhammudoglu@gmail.com>
Date: Thu, 9 Mar 2017 10:56:23 +0100
Subject: [PATCH 1420/2974] Removed contrib section; contrib is deprecated

---
 docs/contributing.rst | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index b0a435ad2..ab3779395 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -124,15 +124,6 @@ Scrapy:
 * Don't put your name in the code you contribute. Our policy is to keep
   the contributor's name in the `AUTHORS`_ file distributed with Scrapy.
 
-Scrapy Contrib
-==============
-
-Scrapy contrib shares a similar rationale as Django contrib, which is explained
-in `this post <https://jacobian.org/writing/what-is-django-contrib/>`_. If you
-are working on a new functionality, please follow that rationale to decide
-whether it should be a Scrapy contrib. If unsure, you can ask in
-`scrapy-users`_.
-
 Documentation policies
 ======================
 

From 9cfe9ae0989069b1a7a2ae0af916ffccb5a8bcee Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 9 Mar 2017 12:21:03 +0100
Subject: [PATCH 1421/2974] Do not use self.assertRaises() as context manager

---
 tests/test_spiderloader/__init__.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 99a61daea..1cd59b99a 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -95,8 +95,7 @@ class SpiderLoaderTest(unittest.TestCase):
 
         module = 'tests.test_spiderloader.test_spiders.doesnotexist'
         settings = Settings({'SPIDER_MODULES': [module]})
-        with self.assertRaises(ImportError):
-            SpiderLoader.from_settings(settings)
+        self.assertRaises(ImportError, SpiderLoader.from_settings, settings)
 
     def test_bad_spider_modules_warning(self):
 

From f2ac24eb7b353d1140729ba7d8c81ad9635d5899 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 9 Mar 2017 17:38:15 +0100
Subject: [PATCH 1422/2974] Do not only warn on wrong spider modules for
 "scrapy list"

---
 scrapy/commands/list.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index 185a77a40..a255b3b94 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -4,8 +4,7 @@ from scrapy.commands import ScrapyCommand
 class Command(ScrapyCommand):
 
     requires_project = True
-    default_settings = {'LOG_ENABLED': False,
-                        'SPIDER_LOADER_WARN_ONLY': True}
+    default_settings = {'LOG_ENABLED': False}
 
     def short_desc(self):
         return "List available spiders"

From 9628a739723983f2d3c4079d228dcaa79f558e73 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 9 Mar 2017 17:40:34 +0100
Subject: [PATCH 1423/2974] Update settings docs for new
 SPIDER_LOADER_WARN_ONLY

---
 docs/topics/settings.rst | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index ccdd02c4e..569b71518 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1180,6 +1180,29 @@ Default: ``'scrapy.spiderloader.SpiderLoader'``
 The class that will be used for loading spiders, which must implement the
 :ref:`topics-api-spiderloader`.
 
+.. setting:: SPIDER_LOADER_WARN_ONLY
+
+SPIDER_LOADER_WARN_ONLY
+-----------------------
+
+.. versionadded:: 1.4
+
+Default: ``False``
+
+By default, when scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
+it will fail loudly if there is any ``ImportError`` exception.
+But you can choose to silence this exception and turn it into a simple
+warning by setting ``SPIDER_LOADER_WARN_ONLY = True``.
+
+.. note::
+    Some :ref:`scrapy commands <topics-commands>` run with this setting to ``True``
+    already (i.e. they will only issue a warning and will not fail)
+    since they do not actually need to load spider classes to work:
+    :command:`scrapy runspider <runspider>`,
+    :command:`scrapy settings <settings>`,
+    :command:`scrapy startproject <startproject>`,
+    :command:`scrapy version <version>`.
+
 .. setting:: SPIDER_MIDDLEWARES
 
 SPIDER_MIDDLEWARES

From d8865b33045c239e18704d7ab5012e334b128a32 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 10 Mar 2017 14:02:00 +0100
Subject: [PATCH 1424/2974] Update changelog for upcoming 1.3.3

---
 docs/news.rst | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 31f4d3026..f9a900771 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,13 +3,25 @@
 Release notes
 =============
 
+Scrapy 1.3.3 (2017-03-XX)
+-------------------------
+
+Bug fixes
+~~~~~~~~~
+
+- Make ``SpiderLoader`` raise ``ImportError`` again by default for missing
+  dependencies and wrong :setting:`SPIDER_MODULES`.
+  These exceptions were silenced as warnings since 1.3.0.
+  A new setting is introduced to toggle between warning or exception if needed ;
+  see :setting:`SPIDER_LOADER_WARN_ONLY` for details.
+
 Scrapy 1.3.2 (2017-02-13)
 -------------------------
 
 Bug fixes
 ~~~~~~~~~
 
-- Preserve crequest class when converting to/from dicts (utils.reqser) (:issue:`2510`).
+- Preserve request class when converting to/from dicts (utils.reqser) (:issue:`2510`).
 - Use consistent selectors for author field in tutorial (:issue:`2551`).
 - Fix TLS compatibility in Twisted 17+ (:issue:`2558`)
 

From b2c505d8ec71fd1226782c65edd06d901c3ef211 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 10 Mar 2017 16:29:44 +0100
Subject: [PATCH 1425/2974] Set release date in changelog for v1.3.3

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index f9a900771..da856d883 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,7 +3,7 @@
 Release notes
 =============
 
-Scrapy 1.3.3 (2017-03-XX)
+Scrapy 1.3.3 (2017-03-10)
 -------------------------
 
 Bug fixes

From a7f5207e9f2837040a3a08c3a8f4e76265b68bb2 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 10 Mar 2017 12:25:58 +0100
Subject: [PATCH 1426/2974] Update version added for SPIDER_LOADER_WARN_ONLY

---
 docs/topics/settings.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 569b71518..8367b1092 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1185,7 +1185,7 @@ The class that will be used for loading spiders, which must implement the
 SPIDER_LOADER_WARN_ONLY
 -----------------------
 
-.. versionadded:: 1.4
+.. versionadded:: 1.3.3
 
 Default: ``False``
 

From 9c256cf693d73e854d409d717854b3f354b5e0a9 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 10 Mar 2017 15:41:57 -0300
Subject: [PATCH 1427/2974] Undocument _InvalidOutput exception

---
 docs/topics/exceptions.rst     | 11 -----------
 scrapy/core/spidermw.py        | 12 ++++++------
 scrapy/exceptions.py           |  8 +++++---
 tests/test_spidermiddleware.py | 12 ++++++------
 4 files changed, 17 insertions(+), 26 deletions(-)

diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index a3ff72827..cc02369d4 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -62,17 +62,6 @@ remain disabled. Those components include:
 
 The exception must be raised in the component's ``__init__`` method.
 
-InvalidOutput
--------------
-
-.. exception:: InvalidOutput
-
-This exception can be raised by a downloader or spider middleware to
-indicate that some method returned a value not suported by the processing
-chain.
-See :ref:`topics-spider-middleware` and :ref:`topics-downloader-middleware`
-for a list of supported output values.
-
 NotSupported
 ------------
 
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 0f03a7b36..50677670b 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -5,7 +5,7 @@ See documentation in docs/topics/spider-middleware.rst
 """
 import six
 from twisted.python.failure import Failure
-from scrapy.exceptions import InvalidOutput
+from scrapy.exceptions import _InvalidOutput
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.conf import build_component_list
@@ -42,7 +42,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 try:
                     result = method(response=response, spider=spider)
                     if result is not None:
-                        raise InvalidOutput('Middleware {} must return None or raise ' \
+                        raise _InvalidOutput('Middleware {} must return None or raise ' \
                             'an exception, got {}'.format(fname(method), type(result)))
                 except:
                     return scrape_func(Failure(), request, spider)
@@ -50,13 +50,13 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
         def process_spider_exception(_failure):
             exception = _failure.value
-            # don't handle InvalidOutput exception
-            if isinstance(exception, InvalidOutput):
+            # don't handle _InvalidOutput exception
+            if isinstance(exception, _InvalidOutput):
                 return _failure
             for method in self.methods['process_spider_exception']:
                 result = method(response=response, exception=exception, spider=spider)
                 if result is not None and not _isiterable(result):
-                    raise InvalidOutput('Middleware {} must return None or an iterable ' \
+                    raise _InvalidOutput('Middleware {} must return None or an iterable ' \
                         'object, got {}'.format(fname(method), type(result)))
                 # stop exception handling by handing control over to the
                 # process_spider_output chain if an iterable has been returned
@@ -80,7 +80,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 if _isiterable(result):
                     result = wrapper(result)
                 else:
-                    raise InvalidOutput('Middleware {} must return an iterable object, ' \
+                    raise _InvalidOutput('Middleware {} must return an iterable object, ' \
                         'got {}'.format(fname(method), type(result)))
             return result
 
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index ba7272255..96949bdd9 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -11,9 +11,11 @@ class NotConfigured(Exception):
     """Indicates a missing configuration situation"""
     pass
 
-class InvalidOutput(TypeError):
-    """Indicates an invalid value has been returned
-    by a middleware's processing method"""
+class _InvalidOutput(TypeError):
+    """
+    Indicates an invalid value has been returned by a middleware's processing method.
+    Internal and undocumented, it should not be raised or caught by user code.
+    """
     pass
 
 # HTTP and crawling
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 3981a8d75..2d05c335c 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -159,7 +159,7 @@ class DoSomethingMiddleware(object):
 
 
 # ================================================================================
-# don't catch InvalidOutput from scrapy's spider middleware manager
+# don't catch _InvalidOutput from scrapy's spider middleware manager
 class InvalidReturnValueFromPreviousMiddlewareInputSpider(LocalhostSpider):
     name = 'invalid_return_value_from_previous_middleware_input'
     custom_settings = {
@@ -318,19 +318,19 @@ class TestSpiderMiddleware(TestCase):
 
     @defer.inlineCallbacks
     def test_process_spider_exception_invalid_return_value_previous_middleware(self):
-        """ don't catch InvalidOutput from middleware """
+        """ don't catch _InvalidOutput from middleware """
         # on middleware's input
         crawler1 = get_crawler(InvalidReturnValueFromPreviousMiddlewareInputSpider)
         with LogCapture() as log1:
             yield crawler1.crawl()
-        self.assertNotIn("InvalidOutput exception caught", str(log1))
-        self.assertIn("'spider_exceptions/InvalidOutput'", str(log1))
+        self.assertNotIn("_InvalidOutput exception caught", str(log1))
+        self.assertIn("'spider_exceptions/_InvalidOutput'", str(log1))
         # on middleware's output
         crawler2 = get_crawler(InvalidReturnValueFromPreviousMiddlewareOutputSpider)
         with LogCapture() as log2:
             yield crawler2.crawl()
-        self.assertNotIn("InvalidOutput exception caught", str(log2))
-        self.assertIn("'spider_exceptions/InvalidOutput'", str(log2))
+        self.assertNotIn("_InvalidOutput exception caught", str(log2))
+        self.assertIn("'spider_exceptions/_InvalidOutput'", str(log2))
 
     @defer.inlineCallbacks
     def test_process_spider_exception_execution_chain(self):

From 4cfbe8204480214b65d48caaf080feda30fe91ae Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 10 Mar 2017 15:47:47 -0300
Subject: [PATCH 1428/2974] Downloader middleware: raise _InvalidOutput

Instead of AssertionError, to make it consistent with spider middleware
---
 scrapy/core/downloader/middleware.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index c3b23e284..cf0c1f869 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -7,6 +7,7 @@ import six
 
 from twisted.internet import defer
 
+from scrapy.exceptions import _InvalidOutput
 from scrapy.http import Request, Response
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.defer import mustbe_deferred
@@ -35,9 +36,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         def process_request(request):
             for method in self.methods['process_request']:
                 response = yield method(request=request, spider=spider)
-                assert response is None or isinstance(response, (Response, Request)), \
-                        'Middleware %s.process_request must return None, Response or Request, got %s' % \
-                        (six.get_method_self(method).__class__.__name__, response.__class__.__name__)
+                if response is not None and not isinstance(response, (Response, Request)):
+                    raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
+                                         (six.get_method_self(method).__class__.__name__, response.__class__.__name__))
                 if response:
                     defer.returnValue(response)
             defer.returnValue((yield download_func(request=request,spider=spider)))
@@ -51,9 +52,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             for method in self.methods['process_response']:
                 response = yield method(request=request, response=response,
                                         spider=spider)
-                assert isinstance(response, (Response, Request)), \
-                    'Middleware %s.process_response must return Response or Request, got %s' % \
-                    (six.get_method_self(method).__class__.__name__, type(response))
+                if not isinstance(response, (Response, Request)):
+                    raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
+                                         (six.get_method_self(method).__class__.__name__, type(response)))
                 if isinstance(response, Request):
                     defer.returnValue(response)
             defer.returnValue(response)
@@ -64,9 +65,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             for method in self.methods['process_exception']:
                 response = yield method(request=request, exception=exception,
                                         spider=spider)
-                assert response is None or isinstance(response, (Response, Request)), \
-                    'Middleware %s.process_exception must return None, Response or Request, got %s' % \
-                    (six.get_method_self(method).__class__.__name__, type(response))
+                if response is not None and not isinstance(response, (Response, Request)):
+                    raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
+                                         (six.get_method_self(method).__class__.__name__, type(response)))
                 if response:
                     defer.returnValue(response)
             defer.returnValue(_failure)

From 7dcc86e61adf37fdfb77b00375040fe25e7ad832 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 10 Mar 2017 21:35:25 +0100
Subject: [PATCH 1429/2974] Add file listing resource + redirecting resource to
 MockServer

---
 tests/mockserver.py                             |  16 ++++++++++++++++
 .../python-logo-master-v3-TM-flattened.png      | Bin 0 -> 11155 bytes
 .../files/images/python-powered-h-50x65.png     | Bin 0 -> 3243 bytes
 .../test_site/files/images/scrapy.png           | Bin 0 -> 2710 bytes
 4 files changed, 16 insertions(+)
 create mode 100644 tests/sample_data/test_site/files/images/python-logo-master-v3-TM-flattened.png
 create mode 100644 tests/sample_data/test_site/files/images/python-powered-h-50x65.png
 create mode 100644 tests/sample_data/test_site/files/images/scrapy.png

diff --git a/tests/mockserver.py b/tests/mockserver.py
index e611cc3ec..26ab51183 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -5,13 +5,17 @@ from subprocess import Popen, PIPE
 
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
+from twisted.web.static import File
 from twisted.web.test.test_webclient import PayloadResource
 from twisted.web.server import GzipEncoderFactory
 from twisted.web.resource import EncodingResourceWrapper
+from twisted.web.util import redirectTo
 from twisted.internet import reactor, ssl
 from twisted.internet.task import deferLater
 
+
 from scrapy.utils.python import to_bytes, to_unicode
+from tests import tests_datadir
 
 
 def getarg(request, name, default=None, type=None):
@@ -120,6 +124,16 @@ class Echo(LeafResource):
         return to_bytes(json.dumps(output))
 
 
+class RedirectTo(LeafResource):
+
+    def render(self, request):
+        goto = getarg(request, b'goto', b'/')
+        # we force the body content, otherwise Twisted redirectTo()
+        # returns HTML with <meta http-equiv="refresh"
+        redirectTo(goto, request)
+        return b'redirecting...'
+
+
 class Partial(LeafResource):
 
     def render_GET(self, request):
@@ -160,6 +174,8 @@ class Root(Resource):
         self.putChild(b"echo", Echo())
         self.putChild(b"payload", PayloadResource())
         self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
+        self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/')))
+        self.putChild(b"redirect-to", RedirectTo())
 
     def getChild(self, name, request):
         return self
diff --git a/tests/sample_data/test_site/files/images/python-logo-master-v3-TM-flattened.png b/tests/sample_data/test_site/files/images/python-logo-master-v3-TM-flattened.png
new file mode 100644
index 0000000000000000000000000000000000000000..738f6ed41f499d1f57fd4db356dc1d64769bf725
GIT binary patch
literal 11155
zcmdUVWn5IzyDmd_NQaal2t$J+ol*jVbf+L4(%q$k_$O4P6i{-=p&38~1f+zaLz<ys
z7@D&@=iU$Z!~K5m{PqkpdoR{r>)mfY@00XQM~#e_i5LqDi%dgZMIQ?b+Z22sAjAis
zmi_Vf!5>^-B@F{Y@Cqe#NCdx$ywpv7v9O2_{=Fee<>J}kB;6}jlUK?<j<&B}dU~)M
zymWWO5`HKoEFmN;uHC-w4MuSP8}Z!p#j5~YA4e>0TYDc*cSnbpw(L({`Zxx7`ndYC
zYn!l(3JZ%T_?Hi3VX<Rrs3;l)=IqW#yk<s5u^m*=OpB_?Lq9Ojy1M?Uz3Y9)Vl2%k
zJ@>(%->L@DUDIuIY)C`>CCbyk_-T<}6^)I*C_H@QqdIY1*i>j@X(n9|*<id+>#px%
zxcYZq+^^6(T%6O_J8ke+2j`4c&-7?jhPr8u#C44H+_G1AD~D4&%N_=2YL#@cHE7OK
zpyP7k(<5fmZeax#@G6*@+Q+A)qM{<Z7Z2s+<SZl<b^yPqR3kZ35)%`7u}Mw;4WqK@
z0~a}yv6Sh+MLk@G8gMa@6OpD0F492$U;9_(Z&p>xhK7TKgZV~f=$6MSsIlVkN0Et%
zT?Hs9z6@e<ZRL71_g#6fB?UHf)w*yIuxIXU#&QEg$5h(UNL#R3O_=_7T<!6Tc~VXv
zL|VO0&f@?^EqW#s88|iq@u7&eT)Ywo>$he33Oa=hAq-Vxh^)RKw0WgQ9I&5eHcPSa
z2n^f&Ex<7Up(}Uv?o?(o_s;+zEAc6Yn)s0qzWkEcM?pE^Ycjuc%sa<=g3WEPM~DsL
zn}#<-$ug0df`6)CExWk39F2J*f0a+R3HXsLNCZo>w(frpJPX)YT+hAxu2mj9E-srs
zraMl*E0(s{ZkRHR1#PyJa5f2b1tA06p-rs&qqFs6jc<RYbf~WNEdKp|Zq4pHO#XCd
zHbzbUK)S6%5zG*4+o#z$US)wmc3?PlI(!z=Gn$D^;%^UGAp=`yJxIYN{)lUP8T8=s
z*<bl9%2PQnC*%EhKsAyYb&R5~d*Bi*a)-A+urqCcP{K%iSF!{xdJ3w-=yk(3&i590
zfwo|aIB@^T3q#RoUoV@U;pLp}TH$0u_9NqdAN`y;eLgIcS~YmXyJ^l1q|QZ7e}4q5
z<c^M!b%H12<?Jc3i6V9u@^F*Xq@ji<;^+<*j&;tN>?mAIMdd&0JN=s3MJZ%5=r(#P
zs14&N0d&CYtM&Z9aW6V<`Ij+I!6=n~qpBjRrm0uy?4AV7#U~|%6MPumooWuE&J=uj
zaN;Ypa-!9&9F!I57(sTB`<~WINrn{1`g&6L@L+$hYxAx-PB<RW_w*<Ze%s0WO#$)Q
zvH_mmhY?pCZf)-FdLy!5W)BZS_PYAVB|_ICu1NPd`ikJh&J=o<x{3QTH{bWKSgd8v
z(w3xoG~0vQ$TiY|S{3&FXTWh$bY*3QH*0`_kx@}DFF>H=C6l;zcYD}nnodc#+Y2{I
zi)?l+dgfhM8&mAb$=Qc~xX>~+%&)USwNpvbLsPiGFb*CRM-hh(E9dHv#6&8h=Te)R
zJEsx_<_x?#?5&2uM)ub!Rb9H{$=q*;hKBrha3Y(U9{)#^d$W1Z3^_N)vcA49Ge9bJ
z>cHLCl!9D(NQ!aZvqDBXe(YOJEuk~o_oj1J3Q0Y#XBl|JmKW9zgtxdd?NRvkI}-KG
zFyoPXv7eKhTe*9Sg<!yr*qhg{^UW&kZf~ydR&6!Q(cJno)ZB4cn|U+g^qFIQIp9+0
zq-nRoJdLsKZc3H4MH#`Si;m%C!}FkOR@k^uu6<u&!Lur)UWJj~cY|jYMwjy}g;B=5
zGzWuX!;<mk=jTDI$3cU`!<Yic<z+A1=$yy!vBy6Mrj*j*0-uH~?y4a~TN4<_Y>L7y
z8+TP=+-z+jMfypQ6!kRdKpGd($6~+M;!-~1I3+eSTEX~ouER#{-+#v(3;6higzmLQ
zI!gYBUM9SLCj%e2xzS)lUYh(;m)UlgdScT|YU%0uWDb7s^7oT(CS>t-a>FMueeX+#
zIZY6vEWcWwCSr(9SY4e3+B4^f-l(Yj`D7tTU}&gUg*3l3FhsX*3u`dcr1#m<V#GEC
zm&<q}mV^8Lj&X;gBd+KCuUry|MN57uI3c3zTLtooFqu@~_U|=|(uOw<=X{4D&N_ie
z@i!bP7e7`WKCT^Zi<Q3~mhT*ktM4E*s+>pkk2`s9_OmVhs%LJ~HlzRG={VIOP~j&<
zF9x=8(crl#Cv!X~h815%PqH)-N3_BU8~ky!Bb0Rj-Y|57Hp4u}^57Wvt9aI^NZMba
z0{bguPoSKIenRj2v780vg17%XV37-L92Ov^P&WF@klFfoVc=DfH;xli7K^p3TwrYe
z2zF*jwmkxbi18>K0<XV6h}6?@O*1d&b4I)xg>6-^jV^O|XI&LGa!L`90WaP~M+7nK
z3KxW(l2EzSwY9YZlZMV@SyQ6F%&;qL9dpK#BS<`Ti_C(T3jr8Z#FH1Q?;OFG;R8ka
zhw2MEa3>*~4RCxuu~e~&Y()0JKp5&y;^0qeCX@^#G0CiNotu3EoNXwOd2tm)YxV3j
z(lIczeo`Bpv;Zo(D#q5o1l~V<DwVCx5nJ>NkzNR&oQO631~ek_-)t!yDLFxC8zLc5
zzA5u31+Ya2zcit^Nxv}OJ*_AGdSw$c9`klgjEsz>d8(Gk>ekk|>WH)mHd3*8=+W}S
z&V}WA4{SOyBldbTGH^Hz=c-m1#Z;n3{BAGhi^pH~)sU*{r46iI92MyQx55{c@`-C}
zT^TB)l4KD$G1mp|3m;X->n1xG<v%<vMtiF0(Ws}>C~LJiwfC;AIXZvL{I-aP>c{V1
zsz32=_SwbP=?G;_*$}kfQu?w>C;6mTtb{}AvcWhtZTPESx6$O$RUA^tuIQ$CpebjY
zj>zP9@9pN|?fd%qEFCPQ^~<Wo@1I{-ep)1%^WkB?mbF=-BfTf~AA>T(-i{wF1-!(k
z`-`a-g7lhU9)mW9|81MIRw-f*z@rpc2!>=bva$6}O=Wvpj*pMOEx7kIxzNVPNB>*7
zddORr$75!Ovs!Ido+LaeDJku}eKoJ2-j4~pJVndC`d+$yRy+9y=s>6(g{q#Cv+4fe
zzBY)>aizk%$z#D)G;*g7J*y6D5;B(qi%Zkirt6P`<`RTYG*I4)_jyJR>5vg*t>+}>
zh?)u8FgKZwhBh=NN?H!jt+}>#A$C-pth6+qseA|R`Vn7MPgq2R>LS5Do6Vm_po$&Z
zwe!Ht#3Ztimw}l%MK=3eX|hifi`juN%D}MRbHASkxgYJ*^xCdkQn<j!MA$)f=Y)Ub
z&}lJrVa$Gl(QeWw)&}A8aa>#--cQ6dGBXtd!;Q{7Z#=SG+NOv)u}7R<t8;3P3qN{9
zAEcXv4pmEKRjoE5MGH;qf2T3Cq#?6J>Ir5kUmrY_hhczpV+bhX^-HpX+upA9rm9`+
z_gv)u?CrH@e)9S<AgU6Totyi1Hw114*0K7l9)3W-ctfjWWQ8Ub`3qwGQr!g7DxlXj
zjLd|*SNe@X05fbbX0e^P_0VwXr)T6zi4=<M8PLPp+S=-CrF#Ah%@J|39=mFxjgqrJ
zD8Iqqzm*Ip`hJ6O5!utD`cy~f37sYgM29QA_qOJ}Yrtuj`sD5e3N<MxQ$S-)%1Q`4
z7pw$$9JtrF6MK*xHNhikXWV^Qw*8rENOm7SR>U?+lr^1P8NSV0x2V{SiC`-S!fa}4
zs!R>|7`fxmok4%BoTca*I5a$L@8KbsX`^2@AuKC<w?Co2v@~o>Kq4fB{H_$u1?(f(
z{O8ZAMfyac<>uziWG;H)`;*M^<!N{%CafI1@t1`7((SbsU66q)c)4;aWFS-q;~6!C
zd34LH&|GQ%;>DBAUp!^N9joe@QNNv^s#>Q8#`fgNli2%_RmL7|3<CoL?Y_}tTjJnc
zvz!_{Gb;-`nU;%@46g5}XGUccURP({%Nkdl(fCl)O!M0p{<||1bWG*O9LeOLCQ*J*
zb#)V@ez1yPt@}K7s5RwH{u|=A^!G0(NHh%$$xgz=Ly+fc#-0o{N~goOu03~PQp&IG
zg|{BC6UAl=<76(n`|R{yHV72vLXQvM++y|J;>Y~F($YK0$_>JM_V5$!&47OT*GWU=
zkHrNBd_>x$f>{H74x%|&C_6j5F7J*$+Siw-yC|(lS~BjfSIvGeJD>x1*_P_fDkL~8
zPMnO##i3BB&F#&?h6X;7Ns|TnC1xLU^;oB5ydvTrI`}RL2}yQd9(}MUP>wXmrV@p6
zRIux}g;XvLUK(O>s2AyXZGansg8`WN)m7{M@VAW8(!Lk9@2ulw2Eo3>FT1T*P6pN!
zOiUmHU*KgMk+j0NMDq!-Pr^tmZD$Axz8BgvN^2Mk&bBFIoQF$favsVGG9TG0$1l9Z
z#l^+o!3+HHHR8(<mXPQvi1y>m8mL!lr8mtK5fO<xJoE<n%5p(PNnSY9v|B?--ilep
zeHs{-#<q*3ZaUZNH*YxdhDq?)YSmm_TuRF@MWuW%oSecUC|)sF?&yCvK=Ji|_`tZF
zCL=99kTwzX=Gp@RGijbeJYNj6^&8;MW`N;Ef5XXKer9zLn9#tqez_@WtPUxwc-(+k
zrH7Ha2AO~Q9Wiii5ymbG1&Nu6ghZTOHN%WuR63U}aC4hNo|edvk33KMLA>_fnG#`<
zde!&H#<WQP0r<u{;$@n~O49-i8!Tecrv_>d7Fl97^-AhZf4@cn=0!u<r)7h83kt`n
zU#hFCbFniG@4qJ!NB&~bFWM7!h+9S7Zs1uo`Qx~Abg{ykcNLFFMWSV&S#7}UCUz~>
zrM=~{%x9PM7wAUHcOVbEbTcco@grS@u#i?i@+r^bETDAvnxZw*xj+av*`(jOx(fY<
zA|G9PMuVsJmSyYl5Be0<WvjROzR~46Ig-nlwazzGRFiDng}#-85?zIqPlOt%O*3(Z
z>i-4@%-i=7gK7mIIG&j2h%Et|xDl;CBK>o8Be-1_7|6hZ^QY0%k>W2r15CN)sSO|r
zG<_?7tDQhyp}Y0UEYRwiP#cAKAeV+xwaJgw)vqJO2b{@0g<e&CzQeNgjp5qkvGLp#
z|BU{iVN<DX&FAp}9`u^dq0(?{VV2z%+`G1ev^lw@Nmo3U5FgszzWWURd^pX1o)a>#
z$t%Qi-#lSPKl8alO$V|;w88(?i9>~lwH%(=b6%R*8zm#dj5bqcpjpdP)%FF8a@-l3
zO;)5f$e|-#&STis{l);TbgpI0LuqN|i}Nq{FBTYTntRh|qu*y3f6|S`&a6JoGbVf~
z+Zf9b#UZ!ftcsnf97yC9I_ulpz|^<n)0oqC&*uC@e|mWN&j9)~BZ^6s1vf5wiplQy
z_?QYFTj+nVr1(!C>Vjnb&qJ-g(LdNss8{dH;vxc;5VcL#)mnp0ZK6?0&wiA0N{x)J
z>Q3Y+Q!}odH?gd-6`;gzrznt(#X8vGCU#V6ZQ7-UE%~-0&*xDM;*{n^L(a#iw6E}?
zuF~a@fdZYXAoHCemwPi`sMdZh(WFXx%t58}WYmYh^?9L{DP(ER4Pk{Gf=To#2fsR<
zMI6wZAhfwdNTNX~iN)IGC{E#!+i0F^kWQuV(0;2tJJZnCPOgoatFCZoMq2w5nl!MM
zf%FE;VtG}YUn4!Xl2?pgg8PV5%7TZfMqHbkFJwvKWT5uz_{)L_{z7+FE(;1FGg4Do
zsoGaT=nqToKCMcLrax(>X%ny^19B`|0ih#J4rQk=_AF<;$V}u?Cyzs9t*%Xb&ymuj
z*0A}(q5?E`-yKR$mEdQe>7Kqxb{_T|i${K8QYyY5U(WhD{9I-dJCl2<F%GMWl9L9;
z5}cN!^x9Kl+s!rKa88-f1nVGQ!a*W;U*>*!q~qkQFjDlMf{X*VdnFNCEjvo>D@MGW
zpG$Ss=W&5;|JQ5tBzxW<S^K{2QC(}R2HbMPO65>a4lWxCeI%r;tzvoMz8cg<_UPu)
z8}8TW<ydg-O7D*lMJzy>!D5>$c{~_m|MK`)KcNEYj?(*zm0Vq}JKHYxOt%%!NR*8z
zK*{-0H#Q`HiCRfT>Gkt-c1T5SMJ~MGyNPcE?vWRXheaTNbW{HZz5uhhrq@H*>1N&=
zUBI&XsqIa*OP1V1A1~WMLjxxI4Yho75E)k=7KLL(0jUrWmdJbL>MNCHKF_&7QPeur
zncB)0f|;vrSh=#Un7B9genJ}|ie+K#<31kDJ#z`b8*i#MTc0`1Bn_G06<q0`ETMvy
zW?^(Nf_||dgr^F4WF`#@g54EvsDe+CYcwNLj{a|!s@3G{8onI*E*(XO2yZv=3~V*h
zT5&t^NTtR{e12}BqV%iMvc=9>LwYn98V?l(+Liy<f_DbT(E~H3<q$Cx9)4*UMZU2u
zXOk4GF&k3dzr)mN9llIaE#c?t+dEW-IC{Pd`A+-V%~}X`<AI%des)TWJgrbGY=c;&
zrTyhRs-+7MGNL3kb+GH(21QU=LnN(u>>2KV`-1>fAPS}9mjZr$ahJ2L@%t8nnEBKM
zm$iW)j^4ZaR_82~Jx91mbWeH|<iGH;$|Orra866S$w8w^CEq=MeH9b?^wY(BBv;E=
zBV?I!EZB=kq+k$vB-f5HoU{DBPhHti^l+s*HG-15<M!s1e&LK{X)HA4dNLnLRmU<j
zLsz)7>+?7_5#Q*`=w<-55-1cJWdHzzE)3J5o6N=ORC?+R8IT=}^ZmrK(L9fB^a4pJ
zcQev*xVhg}(uN^UWVbn=un)5Mj%D6$De2v|L_qK^+m<Nh4@h`&&toYO91M-Eboz*y
z%gUCtCJeco_Wa+p2sde#kbu_>1RIo&l6w!0y@q#~wn&W>e90%rI<U7RNM7M`WgR?_
z+5Z))y(GC!EH9au_yeq|tBT-z1<*h&S@qmr<|#1c;U)na;@<xIYV^YdFEK*Fo8fhR
zGEZyOGYG$p>>trS>cphP^w%6^PSM@YwmOM$xBW1rW#EqezS>??OK=FYUQ-aQ$p-?#
z%qZrLEdd5RBD37ygtMD?tc2Ud#KO+B3F>!|T%xNReyz`gaAx)4f5q4?Te1VsWcP+b
zP*58SKO)-rjp7x>_+ixg9y}apE)eJ2En(oX>a7i}Kd=NN_Ewj)L|!IR)3)s(os8!}
z`z!DwkDkMBww~M&|Ct$mg32(qz~f8Ao7zpO$XGfD+_(E$2)mJ*ziaC%%k~r^5a5OS
z=i26%iM!b${DBI-q$W@9ev6=PJdIgD$Z2c;hMkT)v_(?;d)+IH;tD2+R=izd&(K7-
z5Q$?k)C9vM{06qg;E@jJ3ZUM<D+`5SPntVLWke9Pw#fuSUOvO2^EE=M$YUinb6<!}
zR@r*IJ3g|zBkJ9QVST;#AIczv=xd`7LgAu|tgy;Fe8Fo}C#gxi4STYz4I|>TEnh-0
z7+|Zs<ec(O*Je5@tF0`2T-3nSp9E-jVrNPiOj)BO7o7Po`!4;Lmoa8H%k6%2CP8<E
zw?|J{;;7Q>h(hDF!s6B1*hzFkOBBUKMgN)?v&qAoD@ii!`#As2B>F{~lGmm_qaF|1
z{eZWQuZv#pkYnIQ+jI=AA%eI}4$kf{zaQAjF-7j`T*F3z35IVdxT=KI(5NOga;s<;
z%DZ|KRDc4sz~_GqP|>%M(u<Te7eeAXJakk!i_dz!gwq(Zky%e|eO*R~ooNmuVoGz{
zg%fauDEPP~ICi*7yY;?e*?;yE3l>E4N9QCaSvig-OR?=`Q}IiswKD>u5B~)1baHyi
zj#ZbCkc(q<yu5TTvmUl!<smpeLgg?Q92|<BAv+a_rXj-PO32@$4Zr#&+0ec0x_c(!
zH8p&0a#uzEoB#g8U2m<#k2R*%At}l6xv)f1dQf=ZC@Kj69heEZl?I66V_de>NQvNH
zx}BztxAiVn*Kf4ct3NB(mGmtLR#~wl4^|9*vhL-)^?sR9AC|j3V)pZ%gINcKK7L)X
zQ5lgg`)`M?>ltrBhM@kns|554xhdy{y%3!#uq|HS;oTvg1`at)BwoUi2aropUIixq
z@VWsxn@h0wpr(F6qHk&&P3$t+3&YmWKJN9GBQ?8s!NIMezj^#Lu=eE<=h&wrRIr<^
zCDN}Tw_K@!QrVUF0gz1{sBrIMr>_NpaP;zz6n&ZBCzcW{yId`>><2bYvQl`^sz@%g
zssV3B)S>jkm6SW{S`G+KY1~qw)Ulgfc4m1h!8KS*f4)Dbf`9$4|NSgeqhvhiM8_RW
z{-?Ezl9I%v0frrj4`sj?#hAqtFIt@vvMYupLsP>QdDYp8@4%vQzgo@WVM((=Dn|Y8
zPPiC;+X0{@_JqKZA~oh21-BMohLT(!MxdnBg|vc7dKiF0cFsiHM9M~~dkYVy<4N1F
zfj5NzGPkP5gSxt7MZ&}1^kJ*ZV;h<5Qj@vtv*1)v?LyQ`#-ldulga}Hm=2t;nb_Dc
z0^`o&oD^msi_gimG}F0gh+l?*v{3OfjbldstomY;9_H3&8D9?_F@3m$_192W9tTV~
zD?{uyjpI^^ME$FC_-2Q!H#m)rCs63AU=Dn|1oarOd!_DpkZz8JFlPP(W#g2hF#KQ1
zNliHdPdYCrAMSKV008Op>%FnrH5S-N&{Gr04^-f2KHWc)3Hy>)7r}w7*Z<BXe%LzO
z@msYDmVD7IqQkjw`l`pvOiV;%+58F8Z&{uxdQ6-QBn7#SpxFHRbLzRyXHY!BHgfL{
z5p7pO1{C;h>Iren8;&MRmC4z?$sRwM5WBBG{JH-E?^~&h!!l@TEEhB<B_&nfNYQ_;
zD@RFzG6x6A77GmLeP#&vk>B4M+Rh@7b=h`<N_(WE9h7%sVq)!U<g79b+d(e_|CJMB
zY+?J`VhNG<9`Hus1d3Ygw+XQmtz~U->%(<&BQ|=Y0^C&70UFZF79n%8TxV7h54L7t
zl(|;9k)5bxhW~NjuufZXv-z$5_t>qiEt9S{P*6aRZlk3(3jJ)|+@5aQCG+G4a@NV~
zfOEwqC1D|<Bymby#*1TKHhskXU)UNcz66xtM59}yNBC_z&(G@VyC~QTArD?FtWJ|B
zRB5~BJr9cyY-^EE;c^jY;?2-3h!Ze0^gh|tTRsA)Magl}iCngUVb96QE8gyxV3TV;
zi~wJG6ZSi+FTvfQ+pN3N1bPua_2diHuVKA5E4A+iM`Rl{P=brcCFleWewF9c6z90m
zx&*?4$TiBk^0i=M_CoZBHL|Q=+~_o0{zWR-(g-V#t2KVswUy$K@W<|0B5US>op<MR
z8*&K%B|5`VLHfxY{-BNuPD$>u8%A9xk{@RS#ANh~T1#;1`XN9o@@}s-L>@gtW;I!H
zYFRpej0cZ|!A#7;!IX~p24FGPH%D15R-7v1;s8LnJeAZ;kF~4r>)%^_`MiT!_^A)Y
zT#}l6puEn_YR~#x&@mZLEAZ=Zg}tPGMIm~6ZH3OBmA$(^NuGe&W>{xAm7cVpUZ^8$
z^t{tuIO=TU;X&@}>zn4t_~9u&0OK%w^X|sS6DYqwE)wAQR>~Ck4VpKc;Q!!aYKj45
z>^F!qB&q_FJ@f*AeGaDnEXm{IWX+Yz<KhNo6H(a;dHoImtqk3W%ouP46NFxj=h2z%
zMSrZBhJp%ipN;fIs?q)Too8o_BdSChnyP{Ag<KlhIXSCcUsh8%_#Ldzeh_%AqmNBO
z(O265W_HZ0ny4PWsL`doL(JZA&53Tvs75lgv-{=nFCFIY51*b@Mi9AaPFg_qUL*pP
zj<9|qRBrKDxL|&{I|0aD7neL_!bRfUaaM8<r?~bpSy+Z99ykE501$$E(D!1;6j<+a
zr~jc@y1H;9y8vR<btaQSc#s{UkR4cE1G>T7>uN;J+j(8uB-GunubX2q5f5rUvJG75
z4yLsCW@;3_1TzVM>X1O+llKZ2-RgX*dUn=SvdNKn;yj&fk9}}~ZK3QkFa~^apppmo
zAwgGuFP(W90R60U@6*sxvIGTKPde8>*e2}F&?WgP)65Wu7C!V&&tp@nso+@w$?!8Q
za=EKq8^}PU@)XQFXLyxH!49MZ%F;PA^u~zh{X7VUCnVb|G?eA}^XFYCZS?k1XUscX
z=zB()zwJSlO}~-V_PutpRnP$-?VVC5O{8Dz#daB0ol7IYqMmAN_s|7`hDZ6vmt{n5
zd7@j}uF=)FZYX7HtLLNKD}UOOVQA-TlOY0g&kPBbDP`49tT^gv3e;a06u)hp5<t#t
zhk7||!k-BV3Km)js*RC+<t4s<|GwSjQg(K>f?!tnqxQo$>h$Y_Yd!!E>Ra;(`;-kG
z5N$sx5?bCW$+HUJpY7%TtOFnq;?hz(fB%Q+K|$WP%mvF`X|&aII04IN%c;wA=lo|K
z^71Kp?XcL43%{Ho?Ox3lpozlRMpKISOPA=@4`(rza&m-ohqg9auuBxTD$upnRW49e
zt#527e^{;H)%-?17Xu(FN&g)N1H;`5O{JDsQe>)n`MjFQPQK*xXWtEod+ezGr`?7l
zrWi4`dVD^KM*xcRg7TEi;f-p~#dZW{m#b#RhQULu{N<9xjVEHQ<1LTP?v1ZPuyx(O
z)8~)5xu?4sQa-DR8$uYD`k7`I-4h(u%rzs5VfJf7k2YfcuW+4zK>eSOeb9lLX9~eP
z)x9#-n9Gj|1y-o`48Sv({Ko|tb*^i7`dDlm6f`a3z?U4|uTjn6`A%cD2HHFJ<q@yD
z$cFXHkMXrm!^C(&0JX$u8qjbbJtY5yB#zv7kHHdl^G}jB)u7%;V<9VddpO&0P;{<U
zV^mfjtoc7EW}0RZKA*vKf6bhCgw&<bwN*rCN@(H_jttHE_5j;?lt)@SJZ3BtAW+wr
z7<<sIs|fRu#{uA>kg?ytyVyh~-vOPTAG9!ry7V7c+>fCh(1f+Hk)@?!8Z?fwQQ;3^
zo&hN4Qx$~yYyFWsTC%Rho%uJq%fHohvE|wmp-CfAIj5~+Yu~4tE}1~gEOcU}o9|DB
zMzWJFQjUTkp!snmI<wyR!C?N2GPqw(ebzujUSgs%A?OtAFhKg(0u$n%TjHeK%Ua?>
zUqF*PkB(|KP&+-Gg}k8mqX|Z^wnwx6bAHS6y@pG7PpMT0_wR%+eEpDEh)B!ey4$8$
zc1yx<gGZ!(cf%*p0Z|C)%!P7p6U3K)^(Hf6RlMH^D?}Y;UR(7Bv;<4*#a;#10Pxq&
zg@~$xhtxFbFTL~9oitn=BO!S2bSBQixf<ea$Ko7j{Z9U^=Td#uC7UjdNJqx%^m|nm
zy??t=Y@{e?3%y{&irBEOgZm!ydJX4^cO!);E|JPK?_BD)&ErW7@95DyJZ*u{D42Ur
zY`<;9$+YE<K+Pj6hnwI*vsOzB^0tV=_U{IKxmEnhcb;n{I=6w={Ks$tXh>MV4b{Qh
ziUs%fkiuHl<1=%$o`pwx^xNoWPhBIytd#E(Ds>A$GW|aVhSa_Q2z=pv#>~p%UVxkH
zsq^D{)Wdja^6w;S9<xV-{`PI752^IBC3*NYJld)*SxaEpMo8r_LG&3qn#xbZ&~UM!
z3-n`NN!@*(@^1;cM(JM{W<zOq`;(<?81bK3pLk--`4L6qztL{DkxQR=?v#}BX%?k>
zHXR-wGMFi=plFqi#@uE*F1WttLWhG%Z!dliUBH93wtfc>t&FqL1&*uy{k2x(g|HK%
zqf$-kj?<`Jp@`HPF846$O14)JP4Q!fEm6W=Pu?fRX$&V1+~P-ETfg$ypHN+zxZRwq
zl<xhgB}*BnFfI<-7KKJ-=b45yc=q=8B6Z1$lVacrPlvL>jwvEZ7PmQVrm^q1tx02N
zwy`Fxkl~of5w)I6d0`Z7Htr2wdkR842O(8*TNeAp?ufnN(~}~0?2pSX9v;5~I`!T|
z2P8w!yna-0a@zH27N5vtSKg2Y9v3ZO>l;G`tcJHw*T#o#=eDcm7v%e`pxYmdUw$*-
z`({+u4j-&ecVq<Msv=)TGR#j!MVr)Aqt2TIb~zkX8$oOlQVj}7fyLt)Q^8%B{EPNE
zP~_dof64U9vUj@H2_OcrMqaP#xESyhKvPK6hDy&3^f7;tnpNrI@Vi7t#wKECB36ps
zhghNbP#@dVXCM%6HnV=q`t=52_H-#06e)>zy=fT^%y6uhcHErX`v<ePe=PVQg!1fh
zR4|WQ1u+}<W!b-dmS08@U0q7fXj)>t*Qu$snB>HZXbx9~xx)m+E*swOy99z)+P>B%
zknhO+hM>B~ouAu`x!7M{5`t6(EKaEa!OfKTtqL$&dZZh$FszWhtCbfcW;gqYS^iOs
zL(_!sWej4oq>(%`MyUWxBR$mko^>kdet`_gEWT4G(~*W44}fl`7F44OfXDI>J~u&X
z7H}itbxz~8MKE-mgH2EAfoQ%4k1OF*XR<MQ^Dhu%Uer2^Gd(k;Ez$YB&1XQ$uc{Z@
z4uY(r9u04DwA;f0D}GKUNoLY}2ilvwNlZf6#(W4(T4yM>XRflYBI@oe)=LW5b?1x5
znf6DO0iyqTW$B40Xo>V*?W1xKMfu*C)tmJU4nCc22j21BJ1Wqv1fLF~jD0n8C3JVF
zl5G3TsutRb#ea*_leM+~A=TwCF>f01dBB<W3+q!uC0^Okm!0p)fV>dHrGaX2DjszP
z9HjbqasZqp|1VBetD^Y-!J;Z7LB>o_EjRYYJ;Fl7lZoVI0ADB!?D?~8Vq8gf%p`uN
zuSg=`1i*kM5?2XxQ`Y@$Z|_97L>|E)k=3>_j^7DqQKZ&@>taJ{it|f?rU{o1eq^=G
zH@LJ$llW1~g|V`#WpY9$|4Y~o`X6g(`3`wZ*8>?Me7+LTf=#)qYtGiL_v5aoVRvVT
zKDS?9{RWNlKc1wf*IW$ctx6fzOD?mgj6S$OHr&q9d_i#!7V{kiP+qLPVWbwIqi(}c
zbCJKIwMSj9qHKg}YLlC7FrJ^^lv2P8TbNsJ-fc)tO#H^iP-C~sT#4J2=J~BgCP3<E
zS-VG4Y&m-rpYxf|oW~wb7zyA<71;MBD^n;(o(iwh*N}F5mCx>m+Qyf&L&|26GI@^z
zBpadt^~i(NRPT7J!e=|EMvk7y!o%^pVYgS(y@hHws%GDJwgLYuyW-%Dg~KN>7F2Om
z)SnYsxt>mAlH$wQ3kGkRXEUNbAnM{4)9`E7>r=?iD3D1f!mp-qRhXd6ef*FTgRb6j
zyu!{<pE{b^_(`f7Z(G+j@be9`^lv8tCsrrfEQzzaZPC}0Os32%th?4_lkF2LkrQ#x
z1ZKe=t`TXp1#g2)Jl1gU7n}1mv`j45lV_?UnuCsuT^YrMr<LC`zVcXTO?tyTzyD~D
zro6n$9fr?^G@CnT?HvK+0}&vEwCkHN56p2bKK-TK<vj}zW&3?@pOB#ieusId{WnL<
zoOSIU6ec>yg>&&cQt=nfBi=dJ7NjPCjR#yH?=;Rk_t^)p9^(my1eN4Fu)2DfhkVY7
z-z(n`n{N$z9=Ml28w!OZV>c{_2K#-acx0lPR%ofH&^SEpYD3`?9MRh0aJl_#m`9_~
zJiCvCE!>zPS+Ig~)kXPT5liE196*-?wfxho=-18&pScanKhd$@t-^7yGW8E;dG1tN
z3`jhhJiT<IrlRTrv<#h7D}g+#N{{F)b<+;}=mTQY$*{^!b+07+D&K10bQ1BYl#S%i
z$T%+C2*MVI8hD3u%hz>!-QN{-<fElCI5;`C>Ep}gP-jQdxWbq19ylM6a+l{a=eUej
z`3{TZ`gC4IM%eEE_=(KQlUUwYMuf3~aQhhia&~#*aM>sA#yHJ8zR>Z|&Hy%GM4*wx
zWT3PB%T<0(Ieif>S)R17_^UCFtLorKfdZhQ-T6lhA~eaZ!h1k{rz5v`lHVih*G3>v
z5yjp#2~w5r_d!LrvD$|ux_8$YzXD5D2k4$H%r@qrbbWq-SvWCY5l|<c8AKc^cpzO2
yz|zq#zC=*{<4+R*Zx8VQC9CrPN&`K(h4jP)6lgG(_XBz?mWHa1N~Mxb^nU@y+ffq$

literal 0
HcmV?d00001

diff --git a/tests/sample_data/test_site/files/images/python-powered-h-50x65.png b/tests/sample_data/test_site/files/images/python-powered-h-50x65.png
new file mode 100644
index 0000000000000000000000000000000000000000..37f6b4719e3b10d0e1f5b8866c39557fd0f85d91
GIT binary patch
literal 3243
zcmV;c3{>-pP)<h;3K|Lk000e1NJLTq001%o002P<1^@s6lZp4<00004b3#c}2nYxW
zd<bNS0000PbVXQnQ*UN;cVTj60C#tHE@^ISb7Ns}WiD@WXPfRk8UO$cJxN4CRA_<S
zT5WJs*O`8<?v?J<w=7wbzXifZHbxGR#Neh7z_4ZtC3KT)7Mga_q`UF5vn0$8(@ZCA
zmuYs>&ayO<on$i!O}g1+ongAmz_hG)T@uO`6R-m$U?Yg}hipr-EL-2wmG0O6aGdx<
zvSe4JOP@c|oO{l>?{nYtKJPj2y@Vu5-S{Jb><?`~p>W5EBoYbo#EBCnf*=rg@;-N*
zh}Y|-PoF+r&hz};MI;uBk;jf5tKvBBjv77}1t$^=2IUtoUNnoM_@xq|vR#NqqhxPy
zuh!@D(GG_rR$E&e2Y^T<LiP0YXquXuLeqWw`}>v2WRk3}ua8~1azzgS=g*%ns;#Y!
z&75<3tkda?YBU;Qx;+pG$h*6{4XIR0=5#tEcDp?}eN7ZaT)uo+?{>SD6h%p!Hf@?P
z7!3Sedoz0Ewbx!-_R1@-*h8TZ#q&Jz^2;x;?C9ty27s=vF8%4#r^{yc@9gX}wzs!i
z0f1#$82}`cNiv;If9{o+UV3T8n{U2Z)z#HyeD1mDoC5;`N&x8V>r=h(!V7gnLqjTx
zqNG<}eYN_`nKLE;xOwxY>ctmdbbS2r$9lb9&)v9jL-+ji&##WfVq{((xq9`g@tJ3y
zxmj0N7Y6{Erp0sT&RMo^-yX=+e<qrmnt~l29cA0LZ41yeE#AI;TLl1*KmNG8yu3UG
z07s4-sXlk^oN4*;<s%&(9hSz%#)*CV_KgBSNl6L&-h1y=H8(d;^z`)TD2fuFdg>`R
z05mo>h7KJ%)Ntj>75&DI8z*ymWG-wrTf*n_DZ}A#)}@#W#l^+k^#6*AibNm~pgE2s
zdV6~fD_5?JPG769ua89{5rxa;QqP@3tJMkygCRZB(~BZfR8+(R!0_-evospBK@d}R
z#N%<2Wm$6OnpY?kB1KW+<m9AcF>~fcWN>g$Ns^>kQ&W=wfOI-t5VbQH417gJMSNgj
zKsDXwcDort5M&Jv4UxsnmDM1Z%cV^u5@a|WraC)2P1V)aQJSVjr_&h?g+hw1t}Z<=
zr8r|_W6J4qMMXs%MNy*1<Dv60prN55bm`J1lOPBL$8p5jvuDkAyM30b<%6ti-skf%
zM~@z@_WS*e#bQaewzducfW=}-@7c3w`1tYTm8VXfDk~}~N+**^nZaP-06?SB2)lOe
z8f|N9t88g$@tI5}POVmRGMP-8o|~p=L8Vd&0I+-a?y*QDLLEMQxR&EMnb~Y+pMLu3
zfzSGhDwS%M)T-5LUM`o5bL}NE?S5NZTh~pimLLcONs`jsJ~Q#wTW?jWR4U=I#~$-6
z{M;6XD2kBDmbPkhj><(s6h(ME9;MZ4WtU{ZOk<Wv0$5;a<w7_drd%$U`s~@W7NgOa
zo|)<Y5Xea6p@$wC*6DQfwgEFSIy%Y>4GpQbZ{I$)Zr!>?Da@Tf#u&L9vE-$d4S`6K
z>b~XGjD_R!iD*(DiY6&aMo0!V!yB~>Z#AgXTh=U#P&s06u_zdkes75W@$rkcUiY{b
zk_1Q+fX{wE|1ScP1dBnH`sNm=?~w=VCkjQVAWKUWfro#3s@@ZbF{^DR#FHru_(I4=
zWV(IV`j!44d~3__@67@QI#>;ig=o)2oLOVH;O7Tg@b00nqgb!XIqp{<x-6ksa^9AF
zDJTdLe<Udf0FIy9RS2R8L710(Gk}0=IH1hkFE<JjBcdP?0MO%}z#}i7LNb*`FqU6_
z6okSYi3_e;!-42jEPTl*2$8ibEb+Gw?&$fVWd;!m`1U`ZsxQQS7Q&*5^bQA=$9~gO
z+UE_cqDhvTdic}oB;pHon$ezf((lwF<zXO6fV?;b<CaO3KRE{EuLFxB@|#aa^gn33
zv_cdmB9okEli8dQ6C^~Igd_s2TLsTARFHTA6@TT+*S5S>E5>rNAOG_5a#2E#;LK+D
zBJ-aAqy<7MZz0H=$jM8CMwSx_qWLqSQPKj$47r34%Q1e!l<z!w6X_id&Z}xH8J{_a
zQINu_5<vhTk!S1|yO#Go^Cse%OyqS2UkqAztsjvG-nC@yz<Tq(XWqf?4JGGHntjb{
z897f7pLO_L1f~Rllo42F)+PRI%WD7D^6M1b?Ut-jml`WPkr>M@*jLR^d}~X?NOMC;
z#H1LfAo0XJvJwFBK;pI%;eRn><g#Hw>EI{jQ#mR3!pWP6S;wS3lktoNIxQpTe{o>5
ztAhJ?3p~HIK@x~*56sRlGo?9?>`xVO`L6!FW2D5ON@csY|7-UQSES#uBJx`cP{7v-
zOj+Iqjl7AJWg$Wkkajc_M-lm*Ip4YR22)jdBFzou!A$LDy)v!B5S=ScE(ylwyel(#
z6Zwk68go_|=T%Ecf{>;HyTDM@MN#xXG<U}-Eb3tO*Db3?89BMAn(UH5P$DW@7ge=f
z)TVK*-JE#)z}DWAmj+G4fiNQoB2lbWrgIL|+#j6M-Z?4;Y0fTLlAtI_qj<+8)T@#U
zP5FBf0RWp>n?A79>7CUW`$)UsAeAHbx!g|gil<2!w}n8L<$f4BQ1BCCM)=>U%$P?c
z2~gG~3qj<||AeDx8({uZfGF%nx-cFg^L!~omjhLYrv9frK<GRm*A+xO3gZ!FbrPhO
z%h`_`2U4Q|9|n^B0IBAobSx^Uv!QTB?AS+IcwV+ch*Gm+VBTI!0w4njnSiSA^~12S
zAeH>WR^x?)41r%-5p1_Wj8hqj5DX7v%Q&oi$3W=|mXq$>YlW0gf%G5^c7y?;5J8ux
zz*MCmH|Ji0OTb<Ce&k25pQwDaxiPSMMft4Qy6e868}dvrnaZqp9g%Fvm`6tZlZy7v
zYkJp^Pi4}n`DgdFc$KtV6h#s37q08iUAd*-vw7{rhSe+Lw}!@;&g-9Qp4j=-008{-
zUw>Wp<kvU)%|;#nLAOg=QDRJ&nsnSN$Iq0AqD1W5v0>a+TAT)eK~F$&?9~5SgVC7$
zpa1ro{UiRM;^(K%SuF-#dhd?)0i$jzKeheOMbn3$^y#WfOl&Y3&scHE=#g*z@D0bN
zwYAZ|-~X^X9B0X+@3xl$!1rJM&#HmZapu{*+r96czF>LnKR&SNwMzc&Uv-v>qKE;{
zIDPy~cge5Me_R9r|9#<_vDBpF{_gdYmG*KA`|X`y@gF?ewn7jD;?~d@bKuB3)jJ<>
z246b3cR(hS;n|<H*&l6g419IX%E-Z=y=Mo2_q%&_ZS5DV`ybuv-Mx9;q?Dy{nUP2`
z&5<i@R+g4i((aA-OkTd})}}a~xNz-L!*{>&pufbV=l<rgW^en2YbL8v$CsFNsf#`R
znzP+5-QMku-uJs*22m7|Omi}gk`b@mbnC*g1o^>7E<Ht(QrC?^RWKGKH7bU0a#qLW
z6eam4!tzigA^+&cp!(YFVYPQWNGFmh+56o$^$%~o-|w)MvNaW!lu@V2sCqfKD8&XH
z?;Q^@k!XSxMTwy0lmq}Ky@n5j;|c)Sa$jBOzdElM#S%&KAND^o)N<gToM$h)bdD-(
zB9>%j!AM-b-nk;Cph)TdRm)=~Mm^_pduJ6gT!UVQQBdOg)%F+w+*h}}kEBS+7YNZ0
zuB{2@vNLC}U%x$~TxBndnT<N$Y}BPb9iFN+b-M=D8&=yx0I>6cdxF2addnb_$s~rB
zOHHd+gpd4Nd*zn<>cSc&BkDCuE|ubB4qGYfu$8iUt!iE(bl6H+|728AWi_U2mzA=$
z%Su^VPD$$=Rnf~Xw<edJSv_*_XxlPNk(T?UZ$xw8&$o>Nz;}0V^6vl1+ci6yoD=7-
z_301)<(?q`C^H*4vq76)<FLg5;7^+BgIzZUbal(8svzJ0&d#A9y>ZgE_1@axcrdEi
z+p@tw_kN|tdhRd2v1#=2AHMF`d~a=Vcx;mX-V<LN-Lv`L$tMoK;Yg;^GOJO?J+P)G
zY_BlSD@tTch#Q`KVa<ncK7aM*ke8{hw6f%kZ$u)oB-uYYPCJ&BvP$}kuXF;TC{=9G
z@&F(R0uhcUWz(ty036Q~*ZVw5TbVIs)M*4Z#Sy`1f+{W6&sy`*1S{)v`x%GL%BqyK
z2mow~BZkL=ikb>b){@kMM6wO~|A>q+qEXU!zQ9mmWb_Ef^MstD3i^q~okm8F+-)R)
dKcc<X{{i$llc%HTFv0);002ovPDHLkV1k(DQPcnc

literal 0
HcmV?d00001

diff --git a/tests/sample_data/test_site/files/images/scrapy.png b/tests/sample_data/test_site/files/images/scrapy.png
new file mode 100644
index 0000000000000000000000000000000000000000..66e86f567f56b2e06bc993615a01a8c6e7d98b68
GIT binary patch
literal 2710
zcmV;H3TgF;P)<h;3K|Lk000e1NJLTq002h-002h_1^@s6Rj^tg0000PbVXQnQ*UN;
zcVTj606}DLVr3vnZDD6+Qe|Oed2z{QJOBU+I7vi7RCwC#oPSUh=NZQz;5g)`2Uz0)
z1~8JwVp<?cW38nWM<>`unPw&?X(v;o(_+&^9j6^TNhdm!bdrCJ#A%1=M8%mVjhQAz
zMQEpt+<>-36!8aE0rdjN9X}3^-*9-^_j&i-^S$4D+}_^O=9$^qTlQ{upAX;Xd7t;)
zcNgF*^J)>{zJPyy_?nKa@;vNnUA^s~_5TJA0Z~L&xO|H&Sze&HK_ZKhESJb)Aixa&
zGjIryQnK1a7F_FG_s1lpKult)+wW$^EHE=Z<mr&8rYf==C!nh49UK-aJIS(x2NIu}
z4oORLAW6TRKuM+B{%pc>hd{ExsHqc1%^h^x1yng%4um2EBSHa1qm6y*u5HSp4E@R?
zNXh&r{T!^Jnufl-45q6FURfc56d@xRRLWbTQj+vJur$Afsw7l~YU;jNOV%Acq<sX?
z0ad^uP%yC`VTo1|C=wuH(%wQK8N&vLC0hieBcTKt1tp-*Eec!HYict<PfIPZkhU^F
zRf@wQV1CBHP_-;C{D~|%MUI@*+3*psA_P>a7#soyp&%Tpl&n=hR$X;#Z2FukiCHB9
zv?b(lcrd7zujM?g+VX#(;|r>yxdMs`5X@dyzo;IJP*-KZg2WWk^k2cvysaCbjyQDU
zoI!Tw@Hu8afbpRq$vG%eA{gugy^o)lk``$`fkDIT#p`DFKJ!N&IAU{+Rc=Op$-EqX
z!0X2AXGQ7}S>UidlZVapBP5*Wvpw>l+{j~o%{}L^>i0H&&fW03qE&AwtLe6y55&!E
zUsZIZgm9vAoG)M|to>lTcpBR3kAc;6+igGk!NLptg;#{1fvI}@T(EhsDym<fEz<3)
z6i7c=i)ORU!ei^V!INuC-Kyx8V~(i$fwuuPU&`4bR={coGdn#2tu=?C{_yW8n5_I_
z(CM@3KCLzy41Xzi`wbmNdC&*k5l~pMTtji=Sd5xO2%)t9`=sM<`5Rt=N5AtU$F?TI
z4!yUoyZx5)tCaccW*T3r!TLb0s=+Wi^YOb>IlTStou4{wTZTp;HX+fYdeA>C5ShQb
zq=r?gkcb~wB^Xo+D#jIzTj#UO3>ja~hvdbchhsVaN+pFAs~)qEESJ?)lV8q#evsxE
zwDrH-58r)hkK6a`_InhJR;P!T-#F;?c-+u!mZz%OQ&<&=V>D{ZFK0;@@y&B<?P0g?
zjg3!IFv|-H;rX4vhuHW;pT|W%S!ki(t^VL_Qd>xQ%P+3YcCXn2c1;4j@az4MyK1eZ
zW22u84*Gq-ADlcjDF^clhEZ<u3v|2f^e1j1)Hg6pmLVwKydBnTeA%yBT$RK7?KS7H
zCQXNNh-1QP$}gS_TcXR?ZbDk_8u-nh{sHS=d^I32R2{@mmUncnO}eXc=uS^}AeAt?
z-44^!(_ovOg}Nhe!~LO~j(xmh5DJD6wnVS({v)I<dn~B)qQC4O82WGY2`4@!J>(z|
z!1UBKOc59p6BBfea2y@Pm~>ch?-oQy-gip22xW2hO4w5N9xQwODOs)s{bd94{*|MN
zi46%1gfLDPTw?<DWlt5ZhuTk!kPTfhIW+^_cltpSmkcG_-gN6L1#6tYSyC!wSq#-M
zJZyC+BT!pivKgxX@CH%PB&>Vsb((CIG)*Y-#<0~2gBFWlbx2CgsViXbyYB^@XjKIV
z!=|slpDIPRX1u)}^1i(~EMTMzc1eQLedi8bztJu$7_{bd&G7xg?}e?L2ue8^KD{Qh
zQ!68vUAcS(($mu*Gcz-6aGaD=V;L0sG0qc&yxPd<Xh^`IH8wWFbI&Prn_2W%^xq>K
z(~OO-#feA$Y-xHrL`6l(ez9j7&p=Mja`?uI6^g(S{T2Q92<K3*K#-fE;L{Kc0+Q4E
z`ugC)g$uB~>=%lKkLa)Hzqf>JG}&eX3azB1M7qdf*md~qub;v0-Mb(zE>6*qVtyNa
z!nrs$CeSo@NIrpF;dQF|6l~h~tYYEB`^o(F4W~tfW9pQA0y)B~y1E+lOO`-M$uo+!
zNTQ!De&JL!85^;ECWbRMr6%TJ!>hTu8B%n~P*(PHMZutd=qGI*(Qo3v&mwTlez&#3
zIxN&N?1!NYFRW)~Nh_f=7~Vhh7yXt}As=rL#KTV;3rTn(m@!f-g#!%lAM^LP6gY;O
z!9f@&#T{nyKyptY;Z^s~I!H;@g$)e)h5n(R=r0J<bN#@nXrd2d93%Ax8qV(x)cl24
z{posm;)w#qlP_<gU(8P<^IKBK^XRcYGo^|JQlRSb7G7A#oIG&?N;f?l)($0oj)M4w
z{-K}fZy>$$O5ulsdO92a9PkCWh8I@8moHs{T|0M#wKK{4gZ_;ReZxxWaD*nU3aMh{
zh=b2PL4?<}wl>g4N5i(QuP9dedNcF`{XxH&YN}kDa4F+C<w*6N+rk~j?v~m)g_mw&
z609vQ4r>_k;e>vmp<hfj<zBDf>ygg)7~#o`B9qOQl@O(Yl?e&7E_i6g8i<UFmL6V-
zi4Y$j7j`gM@LfH*AMV-j(r~I87!X>hil%*%#&bLvo$+1X-31+!lk^1FYCZ_4a#n-!
z*NuLlKfHf|D@Ra|m8b`&#ua@*inSjuS_Bc(<1jta4~v!+%o{iz^+#z)VHo|Y{e~D$
z`-p0)<_<>yWFUY_$pM!%jgG*&lvJ>f4$!2A_n89;v2ZBn*4a<Na;t>`dG*G1_)@q<
z)Jnj-=<oMZ2iQ@5R+J>308fN76Vx1HVkqHtieT{ifjx&MsX>u+rt>{!q8==IN{41=
zK~xZi82(pKLE5+^)qufn0fLz#;qL+lZ97rT-{pfN0BR+GM#!gUL_jO!6JX)}yA)8Q
zCK?hKXN4UMb|!DseF%1zeD~b!<iYGBU@9b+A_1NGd`}Yrgaq;sP~FfFWX3LlrO_I&
z_WTD%&7BaRk`Wdap;|f&N1^ZfS;sKMp3Y7%+Z==Dl?2Rh13Bj09%C$9<#-?stx%T?
zZ)Rsh0_$vwPpK#|6BQc<EbQ3qEE+~{lLAj1lj6XH{z*L!5W;axk4G>&AAT4%q%LxN
z1tH;NXekfZ&qXO1LLwA2yzg|Nt4nOEA2cQvO_g&wM{ti$m#uO<9W4Gy9}@#x)6$$E
zi8E6XRxqFt2*SXjP5xC52<1?3FPLU#_^(?T80DW1<^W{@_gpy->G{PE(ZVr-8{5HY
zW^t(*u#mp8A%(uS0WFR|!+qEUPqN!3_?iZUFhUBrNlTAY*?{UeH8=>L_V>9$5eKfe
zc<WFq2j>AKr#BMthB>J)SqJ%~pQYEv%)KQdJ!!m!(lBgw%_+QEu<}*M8wp$hd3?c#
ztD^K;EdiB874@i(A&hp?1fxPOj*Sv9HctxSq0o0(5HRX{mqnnEWU`Pz2`wD=r0w<n
zIdcR;-=7mqvQ-BhXHZ2D<eQfW5Q9>e8}@g0u@Ln5&aO~|%2y_n{a1hi00-6?7X^q(
Q0RR9107*qoM6N<$g8M56QUCw|

literal 0
HcmV?d00001


From 708f1b009b9b23971d73bfc7bc09163969ab6e00 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 10 Mar 2017 21:36:33 +0100
Subject: [PATCH 1430/2974] Add integration tests for MEDIA_ALLOW_REDIRECTS

---
 tests/test_pipeline_crawl.py | 163 +++++++++++++++++++++++++++++++++++
 1 file changed, 163 insertions(+)
 create mode 100644 tests/test_pipeline_crawl.py

diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
new file mode 100644
index 000000000..1f5b80954
--- /dev/null
+++ b/tests/test_pipeline_crawl.py
@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+import os
+import shutil
+
+from testfixtures import LogCapture
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+from w3lib.url import add_or_replace_parameter
+
+from scrapy.crawler import CrawlerRunner
+from scrapy import signals
+from tests.mockserver import MockServer
+from tests.spiders import SimpleSpider
+
+
+class MediaDownloadSpider(SimpleSpider):
+    name = 'mediadownload'
+
+    def _process_url(self, url):
+        return url
+
+    def parse(self, response):
+        self.logger.info(response.headers)
+        self.logger.info(response.text)
+        item = {
+            'images': [],
+            'image_urls': [
+                self._process_url(response.urljoin(href))
+                    for href in response.xpath('''
+                        //table[thead/tr/th="Filename"]
+                            /tbody//a/@href
+                        ''').extract()],
+        }
+        yield item
+
+
+class BrokenLinksMediaDownloadSpider(MediaDownloadSpider):
+    name = 'brokenmedia'
+
+    def _process_url(self, url):
+        return url + '.foo'
+
+
+class RedirectedMediaDownloadSpider(MediaDownloadSpider):
+    name = 'redirectedmedia'
+
+    def _process_url(self, url):
+        return add_or_replace_parameter(
+                    'http://localhost:8998/redirect-to',
+                    'goto', url)
+
+
+class MediaDownloadCrawlTestCase(TestCase):
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+
+        # prepare a directory for storing files
+        self.tmpmediastore = self.mktemp()
+        os.mkdir(self.tmpmediastore)
+        self.settings = {
+            'ITEM_PIPELINES': {'scrapy.pipelines.images.ImagesPipeline': 1},
+            'IMAGES_STORE': self.tmpmediastore,
+        }
+        self.runner = CrawlerRunner(self.settings)
+        self.items = []
+        # these are the checksums for images in test_site/files/images
+        # - scrapy.png
+        # - python-powered-h-50x65.png
+        # - python-logo-master-v3-TM-flattened.png
+        self.expected_checksums = set([
+            'a7020c30837f971084834e603625af58',
+            'acac52d42b63cf2c3b05832641f3a53c',
+            '195672ac5888feb400fbf7b352553afe'])
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpmediastore)
+        self.items = []
+        self.mockserver.__exit__(None, None, None)
+
+    def _on_item_scraped(self, item):
+        self.items.append(item)
+
+    def _create_crawler(self, spider_class):
+        crawler = self.runner.create_crawler(spider_class)
+        crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
+        return crawler
+
+    def _assert_files_downloaded(self, items, logs):
+        self.assertEqual(len(items), 1)
+        self.assertIn('images', items[0])
+
+        # check that logs show the expected number of successful file downloads
+        file_dl_success = 'File (downloaded): Downloaded file from'
+        self.assertEqual(logs.count(file_dl_success), 3)
+
+        # check that the images checksums are what we know they should be
+        checksums = set(
+            i['checksum']
+                for item in items
+                    for i in item['images'])
+        self.assertEqual(checksums, self.expected_checksums)
+
+        # check that the image files where actually written to the media store
+        for item in items:
+            for i in item['images']:
+                self.assertTrue(
+                    os.path.exists(
+                        os.path.join(self.tmpmediastore, i['path'])))
+
+    def _assert_files_download_failure(self, crawler, items, code, logs):
+
+        # check that the item does NOT have the "images" field populated
+        self.assertEqual(len(items), 1)
+        self.assertIn('images', items[0])
+        self.assertFalse(items[0]['images'])
+
+        # check that there was 1 successful fetch and 3 other responses with non-200 code
+        self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4)
+        self.assertEqual(crawler.stats.get_value('downloader/response_count'), 4)
+        self.assertEqual(crawler.stats.get_value('downloader/response_status_count/200'), 1)
+        self.assertEqual(crawler.stats.get_value('downloader/response_status_count/%d' % code), 3)
+
+        # check that logs do show the failure on the file downloads
+        file_dl_failure = 'File (code: %d): Error downloading file from' % code
+        self.assertEqual(logs.count(file_dl_failure), 3)
+
+        # check that no files were written to the media store
+        self.assertEqual(os.listdir(self.tmpmediastore), [])
+
+    @defer.inlineCallbacks
+    def test_download_media(self):
+        crawler = self._create_crawler(MediaDownloadSpider)
+        with LogCapture() as log:
+            yield crawler.crawl("http://localhost:8998/files/images/")
+        self._assert_files_downloaded(self.items, str(log))
+
+    @defer.inlineCallbacks
+    def test_download_media_wrong_urls(self):
+        crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
+        with LogCapture() as log:
+            yield crawler.crawl("http://localhost:8998/files/images/")
+        self._assert_files_download_failure(crawler, self.items, 404, str(log))
+
+    @defer.inlineCallbacks
+    def test_download_media_redirected_default_failure(self):
+        crawler = self._create_crawler(RedirectedMediaDownloadSpider)
+        with LogCapture() as log:
+            yield crawler.crawl("http://localhost:8998/files/images/")
+        self._assert_files_download_failure(crawler, self.items, 302, str(log))
+
+    @defer.inlineCallbacks
+    def test_download_media_redirected_allowed(self):
+        settings = dict(self.settings)
+        settings.update({'MEDIA_ALLOW_REDIRECTS': True})
+        self.runner = CrawlerRunner(settings)
+
+        crawler = self._create_crawler(RedirectedMediaDownloadSpider)
+        with LogCapture() as log:
+            yield crawler.crawl("http://localhost:8998/files/images/")
+        self._assert_files_downloaded(self.items, str(log))
+        self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3)

From 810658bcc5b1897d57c0882a0f7ab4a0d264a778 Mon Sep 17 00:00:00 2001
From: harshasrinivas <harshsrinivas@gmail.com>
Date: Sun, 12 Mar 2017 05:06:12 +0530
Subject: [PATCH 1431/2974] Add feature to set RETRY_TIMES per request (#2642)

---
 scrapy/downloadermiddlewares/retry.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 549d74f46..a5342995f 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -63,6 +63,9 @@ class RetryMiddleware(object):
     def _retry(self, request, reason, spider):
         retries = request.meta.get('retry_times', 0) + 1
 
+        if 'max_retry_times' in request.meta:
+            self.max_retry_times = request.meta['max_retry_times']
+
         stats = spider.crawler.stats
         if retries <= self.max_retry_times:
             logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",

From 871134ee22653f7f074fbfb8f3c393ba3555a6d4 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Sun, 12 Mar 2017 17:30:24 +0100
Subject: [PATCH 1432/2974] Refactor to also test FilesPipeline

---
 tests/test_pipeline_crawl.py | 79 ++++++++++++++++++++++--------------
 1 file changed, 49 insertions(+), 30 deletions(-)

diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 1f5b80954..9b81f827d 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -23,8 +23,8 @@ class MediaDownloadSpider(SimpleSpider):
         self.logger.info(response.headers)
         self.logger.info(response.text)
         item = {
-            'images': [],
-            'image_urls': [
+            self.media_key: [],
+            self.media_urls_key: [
                 self._process_url(response.urljoin(href))
                     for href in response.xpath('''
                         //table[thead/tr/th="Filename"]
@@ -50,7 +50,15 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):
                     'goto', url)
 
 
-class MediaDownloadCrawlTestCase(TestCase):
+class FileDownloadCrawlTestCase(TestCase):
+    pipeline_class = 'scrapy.pipelines.files.FilesPipeline'
+    store_setting_key = 'FILES_STORE'
+    media_key = 'files'
+    media_urls_key = 'file_urls'
+    expected_checksums = set([
+        '5547178b89448faf0015a13f904c936e',
+        'c2281c83670e31d8aaab7cb642b824db',
+        'ed3f6538dc15d4d9179dae57319edc5f'])
 
     def setUp(self):
         self.mockserver = MockServer()
@@ -60,19 +68,11 @@ class MediaDownloadCrawlTestCase(TestCase):
         self.tmpmediastore = self.mktemp()
         os.mkdir(self.tmpmediastore)
         self.settings = {
-            'ITEM_PIPELINES': {'scrapy.pipelines.images.ImagesPipeline': 1},
-            'IMAGES_STORE': self.tmpmediastore,
+            'ITEM_PIPELINES': {self.pipeline_class: 1},
+            self.store_setting_key: self.tmpmediastore,
         }
         self.runner = CrawlerRunner(self.settings)
         self.items = []
-        # these are the checksums for images in test_site/files/images
-        # - scrapy.png
-        # - python-powered-h-50x65.png
-        # - python-logo-master-v3-TM-flattened.png
-        self.expected_checksums = set([
-            'a7020c30837f971084834e603625af58',
-            'acac52d42b63cf2c3b05832641f3a53c',
-            '195672ac5888feb400fbf7b352553afe'])
 
     def tearDown(self):
         shutil.rmtree(self.tmpmediastore)
@@ -82,39 +82,40 @@ class MediaDownloadCrawlTestCase(TestCase):
     def _on_item_scraped(self, item):
         self.items.append(item)
 
-    def _create_crawler(self, spider_class):
-        crawler = self.runner.create_crawler(spider_class)
+    def _create_crawler(self, spider_class, **kwargs):
+        crawler = self.runner.create_crawler(spider_class, **kwargs)
         crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
         return crawler
 
     def _assert_files_downloaded(self, items, logs):
         self.assertEqual(len(items), 1)
-        self.assertIn('images', items[0])
+        self.assertIn(self.media_key, items[0])
 
         # check that logs show the expected number of successful file downloads
         file_dl_success = 'File (downloaded): Downloaded file from'
         self.assertEqual(logs.count(file_dl_success), 3)
 
-        # check that the images checksums are what we know they should be
-        checksums = set(
-            i['checksum']
-                for item in items
-                    for i in item['images'])
-        self.assertEqual(checksums, self.expected_checksums)
+        # check that the images/files checksums are what we know they should be
+        if self.expected_checksums is not None:
+            checksums = set(
+                i['checksum']
+                    for item in items
+                        for i in item[self.media_key])
+            self.assertEqual(checksums, self.expected_checksums)
 
         # check that the image files where actually written to the media store
         for item in items:
-            for i in item['images']:
+            for i in item[self.media_key]:
                 self.assertTrue(
                     os.path.exists(
                         os.path.join(self.tmpmediastore, i['path'])))
 
     def _assert_files_download_failure(self, crawler, items, code, logs):
 
-        # check that the item does NOT have the "images" field populated
+        # check that the item does NOT have the "images/files" field populated
         self.assertEqual(len(items), 1)
-        self.assertIn('images', items[0])
-        self.assertFalse(items[0]['images'])
+        self.assertIn(self.media_key, items[0])
+        self.assertFalse(items[0][self.media_key])
 
         # check that there was 1 successful fetch and 3 other responses with non-200 code
         self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4)
@@ -133,21 +134,27 @@ class MediaDownloadCrawlTestCase(TestCase):
     def test_download_media(self):
         crawler = self._create_crawler(MediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl("http://localhost:8998/files/images/")
+            yield crawler.crawl("http://localhost:8998/files/images/",
+                media_key=self.media_key,
+                media_urls_key=self.media_urls_key)
         self._assert_files_downloaded(self.items, str(log))
 
     @defer.inlineCallbacks
     def test_download_media_wrong_urls(self):
         crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl("http://localhost:8998/files/images/")
+            yield crawler.crawl("http://localhost:8998/files/images/",
+                media_key=self.media_key,
+                media_urls_key=self.media_urls_key)
         self._assert_files_download_failure(crawler, self.items, 404, str(log))
 
     @defer.inlineCallbacks
     def test_download_media_redirected_default_failure(self):
         crawler = self._create_crawler(RedirectedMediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl("http://localhost:8998/files/images/")
+            yield crawler.crawl("http://localhost:8998/files/images/",
+                media_key=self.media_key,
+                media_urls_key=self.media_urls_key)
         self._assert_files_download_failure(crawler, self.items, 302, str(log))
 
     @defer.inlineCallbacks
@@ -158,6 +165,18 @@ class MediaDownloadCrawlTestCase(TestCase):
 
         crawler = self._create_crawler(RedirectedMediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl("http://localhost:8998/files/images/")
+            yield crawler.crawl("http://localhost:8998/files/images/",
+                media_key=self.media_key,
+                media_urls_key=self.media_urls_key)
         self._assert_files_downloaded(self.items, str(log))
         self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3)
+
+
+class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase):
+    pipeline_class = 'scrapy.pipelines.images.ImagesPipeline'
+    store_setting_key = 'IMAGES_STORE'
+    media_key = 'images'
+    media_urls_key = 'image_urls'
+
+    # somehow checksums for images are different for Python 3.3
+    expected_checksums = None

From 0d57b5cd43343a335fcf2e923b29e76d09dd0b51 Mon Sep 17 00:00:00 2001
From: harshasrinivas <harshsrinivas@gmail.com>
Date: Mon, 13 Mar 2017 02:10:23 +0530
Subject: [PATCH 1433/2974] Prevent max_retry_times override

---
 scrapy/downloadermiddlewares/retry.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index a5342995f..07e979628 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -63,11 +63,13 @@ class RetryMiddleware(object):
     def _retry(self, request, reason, spider):
         retries = request.meta.get('retry_times', 0) + 1
 
+        retry_times = self.max_retry_times
+
         if 'max_retry_times' in request.meta:
-            self.max_retry_times = request.meta['max_retry_times']
+            retry_times = request.meta['max_retry_times']
 
         stats = spider.crawler.stats
-        if retries <= self.max_retry_times:
+        if retries <= retry_times:
             logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",
                          {'request': request, 'retries': retries, 'reason': reason},
                          extra={'spider': spider})

From 3cd9185aa12430708b17eb8c02a6bdc3709ed5f9 Mon Sep 17 00:00:00 2001
From: jorenham <jhammudoglu@gmail.com>
Date: Wed, 8 Mar 2017 20:44:39 +0100
Subject: [PATCH 1434/2974] Fixed the FIXME; more specific exception catching

---
 scrapy/pipelines/files.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 843b4d3ec..bdc0f24e5 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -58,7 +58,7 @@ class FSFilesStore(object):
         absolute_path = self._get_filesystem_path(path)
         try:
             last_modified = os.path.getmtime(absolute_path)
-        except:  # FIXME: catching everything!
+        except os.error:
             return {}
 
         with open(absolute_path, 'rb') as f:

From fbb411a805724fec50b786f369be79dc221c798e Mon Sep 17 00:00:00 2001
From: woxcab <woxcab@users.noreply.github.com>
Date: Mon, 13 Mar 2017 14:16:39 +0300
Subject: [PATCH 1435/2974] Allowed passing objects of Mapping class or its
 subclass to the CaselessDict initializer

---
 scrapy/utils/datatypes.py     |  4 ++--
 tests/test_utils_datatypes.py | 43 +++++++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index e516185bd..eb373c501 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -8,7 +8,7 @@ This module must not depend on any module outside the Standard Library.
 import copy
 import six
 import warnings
-from collections import OrderedDict
+from collections import OrderedDict, Mapping
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
@@ -224,7 +224,7 @@ class CaselessDict(dict):
         return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
 
     def update(self, seq):
-        seq = seq.items() if isinstance(seq, dict) else seq
+        seq = seq.items() if isinstance(seq, Mapping) else seq
         iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
         super(CaselessDict, self).update(iseq)
 
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 80f797227..3a4137942 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,5 +1,6 @@
 import copy
 import unittest
+from collections import Mapping, MutableMapping
 
 from scrapy.utils.datatypes import CaselessDict, SequenceExclude
 
@@ -18,6 +19,48 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(d['red'], 1)
         self.assertEqual(d['black'], 3)
 
+        class MyMapping(Mapping):
+            def __init__(self, **kwargs):
+                self._d = kwargs
+
+            def __getitem__(self, key):
+                return self._d[key]
+
+            def __iter__(self):
+                return iter(self._d)
+
+            def __len__(self):
+                return len(self._d)
+
+        seq = MyMapping(red=1, black=3)
+        d = CaselessDict(seq)
+        self.assertEqual(d['red'], 1)
+        self.assertEqual(d['black'], 3)
+
+        class MyMutableMapping(MutableMapping):
+            def __init__(self, **kwargs):
+                self._d = kwargs
+
+            def __getitem__(self, key):
+                return self._d[key]
+
+            def __setitem__(self, key, value):
+                self._d[key] = value
+
+            def __delitem__(self, key):
+                del self._d[key]
+
+            def __iter__(self):
+                return iter(self._d)
+
+            def __len__(self):
+                return len(self._d)
+
+        seq = MyMutableMapping(red=1, black=3)
+        d = CaselessDict(seq)
+        self.assertEqual(d['red'], 1)
+        self.assertEqual(d['black'], 3)
+
     def test_caseless(self):
         d = CaselessDict()
         d['key_Lower'] = 1

From 0f2a5cdb8edb3d4d6ce542f13b5d7e5858905bae Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Mon, 13 Mar 2017 15:13:24 +0100
Subject: [PATCH 1436/2974] [logformatter] 'flags' format spec backward
 compatibility

pass 'flags' kwarg to logger so that it is compatible with old
format of CRAWLEDMSG.
---
 scrapy/logformatter.py     |  1 +
 tests/test_logformatter.py | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index e7bf7942e..2a89c00c5 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -43,6 +43,7 @@ class LogFormatter(object):
                 'request_flags' : request_flags,
                 'referer': referer_str(request),
                 'response_flags': response_flags,
+                'flags': response_flags
             }
         }
 
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 11fe7b653..52646ec1b 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -64,5 +64,32 @@ class LoggingContribTest(unittest.TestCase):
         assert all(isinstance(x, six.text_type) for x in lines)
         self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])
 
+
+class LogFormatterSubclass(LogFormatter):
+    # Formatter with format spec that is same as in Scrapy before 1.3 version.
+    def crawled(self, request, response, spider):
+        kwargs = super(LogFormatterSubclass, self).crawled(
+        request, response, spider)
+        CRAWLEDMSG = (
+            u"Crawled (%(status)s) %(request)s (referer: "
+            u"%(referer)s)%(flags)s"
+        )
+        return {
+            'level': kwargs['level'],
+            'msg': CRAWLEDMSG,
+            'args': kwargs['args']
+        }
+
+
+class LogformatterSubclassTest(LoggingContribTest):
+    # Test if old crawledmsg format string still works fine
+    def setUp(self):
+        self.formatter = LogFormatterSubclass()
+        self.spider = Spider('default')
+
+    def test_flags_in_request(self):
+        pass
+
+
 if __name__ == "__main__":
     unittest.main()

From 694c6d3d7460ab80ace979c4563af8f310614d37 Mon Sep 17 00:00:00 2001
From: harshasrinivas <harshsrinivas@gmail.com>
Date: Tue, 14 Mar 2017 16:14:40 +0530
Subject: [PATCH 1437/2974] Simplify retry_times assignment statement

---
 scrapy/downloadermiddlewares/retry.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 07e979628..c22437ff1 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -63,10 +63,7 @@ class RetryMiddleware(object):
     def _retry(self, request, reason, spider):
         retries = request.meta.get('retry_times', 0) + 1
 
-        retry_times = self.max_retry_times
-
-        if 'max_retry_times' in request.meta:
-            retry_times = request.meta['max_retry_times']
+        retry_times = request.meta.get('max_retry_times') or self.max_retry_times
 
         stats = spider.crawler.stats
         if retries <= retry_times:

From 966bd49c421fdf40a8b21b49c76cd54ded06fe50 Mon Sep 17 00:00:00 2001
From: harshasrinivas <harshsrinivas@gmail.com>
Date: Tue, 14 Mar 2017 16:23:47 +0530
Subject: [PATCH 1438/2974] Update unittest for meta['max_retry_times']

---
 tests/test_downloadermiddleware_retry.py | 28 ++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index b833cb448..cc3d37075 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -103,6 +103,34 @@ class RetryTest(unittest.TestCase):
         req = self.mw.process_exception(req, exception, self.spider)
         self.assertEqual(req, None)
 
+    def test_different_retry(self):
+        
+        req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 1})
+        self._test_retry(req, DNSLookupError('foo'))
+        req2 = Request('http://www.scrapytest.org/invalid_url')
+        self._test_retry(req2, DNSLookupError('foo'))
+
+        stats = self.crawler.stats
+        assert stats.get_value('retry/max_reached') == 2
+        assert stats.get_value('retry/count') == 3
+
+    def _test_retry(self, req, exception):
+        
+        req = self.mw.process_exception(req, exception, self.spider)
+        assert isinstance(req, Request)
+
+        retry_times = req.meta.get('max_retry_times') or self.mw.max_retry_times
+
+        while req.meta['retry_times'] != retry_times:
+            req = self.mw.process_exception(req, exception, self.spider)
+            assert isinstance(req, Request)
+        
+        self.assertEqual(req.meta['retry_times'], retry_times)
+
+        # discard it
+        req = self.mw.process_exception(req, exception, self.spider)
+        self.assertEqual(req, None)
+
 
 if __name__ == "__main__":
     unittest.main()

From e321ac9931007360a38dbd6a5933794af415274b Mon Sep 17 00:00:00 2001
From: harshasrinivas <harshsrinivas@gmail.com>
Date: Wed, 15 Mar 2017 04:12:32 +0530
Subject: [PATCH 1439/2974] Update unittests for max_retry_times

---
 tests/test_downloadermiddleware_retry.py | 37 ++++++++++++++----------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index cc3d37075..064c740c9 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -105,27 +105,34 @@ class RetryTest(unittest.TestCase):
 
     def test_different_retry(self):
         
-        req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 1})
-        self._test_retry(req, DNSLookupError('foo'))
+        req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3})
+
+        # SETINGS: meta(max_retry_times) = 3, RETRY_TIMES = 2
+        self._test_retry(req, DNSLookupError('foo'), 3)
+
         req2 = Request('http://www.scrapytest.org/invalid_url')
-        self._test_retry(req2, DNSLookupError('foo'))
-
-        stats = self.crawler.stats
-        assert stats.get_value('retry/max_reached') == 2
-        assert stats.get_value('retry/count') == 3
-
-    def _test_retry(self, req, exception):
         
-        req = self.mw.process_exception(req, exception, self.spider)
-        assert isinstance(req, Request)
+        # SETINGS: RETRY_TIMES < meta(max_retry_times)
+        self._test_retry(req2, DNSLookupError('foo'), 2)
 
-        retry_times = req.meta.get('max_retry_times') or self.mw.max_retry_times
+        # SETINGS: RETRY_TIMES = 0
+        self.mw.max_retry_times = 0
+        self._test_retry(req2, DNSLookupError('foo'), 0)
+        
+        # SETINGS: RETRY_TIMES > meta(max_retry_times)
+        self.mw.max_retry_times = 4
+        self._test_retry(req2, DNSLookupError('foo'), 4)
 
-        while req.meta['retry_times'] != retry_times:
+        # RESET RETRY_TIMES SETTINGS
+        self.mw.max_retry_times = 2
+
+    def _test_retry(self, req, exception, max_retry_times):
+        
+        while max_retry_times > 0:
             req = self.mw.process_exception(req, exception, self.spider)
             assert isinstance(req, Request)
-        
-        self.assertEqual(req.meta['retry_times'], retry_times)
+            if req.meta['retry_times'] == max_retry_times:
+                break
 
         # discard it
         req = self.mw.process_exception(req, exception, self.spider)

From 9d97d788c06c2e8fbbdfbcfbee65321ac2dfb517 Mon Sep 17 00:00:00 2001
From: harshasrinivas <harshsrinivas@gmail.com>
Date: Wed, 15 Mar 2017 04:13:47 +0530
Subject: [PATCH 1440/2974] Update docs for meta key

---
 docs/topics/downloader-middleware.rst | 5 +++++
 docs/topics/request-response.rst      | 8 ++++++++
 2 files changed, 13 insertions(+)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index c3a454279..b808a6448 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -852,6 +852,11 @@ Default: ``2``
 
 Maximum number of times to retry, in addition to the first download.
 
+.. reqmeta:: max_retry_times
+
+If :attr:`Request.meta <scrapy.http.Request.meta>` has ``max_retry_times`` key
+set to some value, this setting will be ignored by this middleware for the corresponding request.
+
 .. setting:: RETRY_HTTP_CODES
 
 RETRY_HTTP_CODES
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 67f8ec285..64a1e55fa 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -308,6 +308,7 @@ Those are:
 * ``ftp_user`` (See :setting:`FTP_USER` for more info)
 * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
 * :reqmeta:`referrer_policy`
+* :reqmeta:`max_retry_times`
 
 .. reqmeta:: bindaddress
 
@@ -342,6 +343,13 @@ download_fail_on_dataloss
 Whether or not to fail on broken responses. See:
 :setting:`DOWNLOAD_FAIL_ON_DATALOSS`.
 
+.. reqmeta:: max_retry_times
+
+max_retry_times
+---------------
+
+The meta key is used set retry times per request. When initialized, the :setting:`RETRY_TIMES` setting will be ignored by the downloader middleware.
+
 .. _topics-request-response-ref-request-subclasses:
 
 Request subclasses

From 7ba4b0a21b0bb2efe0523bec0c1db07771816347 Mon Sep 17 00:00:00 2001
From: Bernardas <bernardas.alisauskas@gmail.com>
Date: Wed, 15 Mar 2017 07:50:31 +0000
Subject: [PATCH 1441/2974] add support for embeded ptpython shell

---
 scrapy/utils/console.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 1888d9599..a9d73aada 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -31,6 +31,15 @@ def _embed_bpython_shell(namespace={}, banner=''):
         bpython.embed(locals_=namespace, banner=banner)
     return wrapper
 
+def _embed_ptpython_shell(namespace={}, banner=''):
+    """Start a ptpython shell"""
+    import ptpython.repl
+    @wraps(_embed_ptpython_shell)
+    def wrapper(namespace=namespace, banner=''):
+        print(banner)
+        ptpython.repl.embed(locals=namespace)
+    return wrapper
+
 def _embed_standard_shell(namespace={}, banner=''):
     """Start a standard python shell"""
     import code
@@ -49,7 +58,8 @@ def _embed_standard_shell(namespace={}, banner=''):
 DEFAULT_PYTHON_SHELLS = OrderedDict([
     ('ipython', _embed_ipython_shell),
     ('bpython', _embed_bpython_shell),
-    ( 'python', _embed_standard_shell),
+    ('ptpython', _embed_ptpython_shell),
+    ('python', _embed_standard_shell),
 ])
 
 def get_shell_embed_func(shells=None, known_shells=None):

From a84652e775fda1135fe959465b27cf6ed2c25e1d Mon Sep 17 00:00:00 2001
From: woxcab <woxcab@users.noreply.github.com>
Date: Wed, 15 Mar 2017 12:39:48 +0300
Subject: [PATCH 1442/2974] Init tests are split by initializer' input

---
 tests/test_utils_datatypes.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 3a4137942..49323f0ff 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -8,17 +8,19 @@ __doctests__ = ['scrapy.utils.datatypes']
 
 class CaselessDictTest(unittest.TestCase):
 
-    def test_init(self):
+    def test_init_dict(self):
         seq = {'red': 1, 'black': 3}
         d = CaselessDict(seq)
         self.assertEqual(d['red'], 1)
         self.assertEqual(d['black'], 3)
 
+    def test_init_pair_sequence(self):
         seq = (('red', 1), ('black', 3))
         d = CaselessDict(seq)
         self.assertEqual(d['red'], 1)
         self.assertEqual(d['black'], 3)
 
+    def test_init_mapping(self):
         class MyMapping(Mapping):
             def __init__(self, **kwargs):
                 self._d = kwargs
@@ -37,6 +39,7 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(d['red'], 1)
         self.assertEqual(d['black'], 3)
 
+    def test_init_mutable_mapping(self):
         class MyMutableMapping(MutableMapping):
             def __init__(self, **kwargs):
                 self._d = kwargs

From 4345eaf1b640bfecb2340a0e27649b1ab1079da6 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Fri, 17 Mar 2017 08:11:20 +0100
Subject: [PATCH 1443/2974] [logformatter] backward compat comments

---
 scrapy/logformatter.py     | 1 +
 tests/test_logformatter.py | 2 --
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 2a89c00c5..075a6d862 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -43,6 +43,7 @@ class LogFormatter(object):
                 'request_flags' : request_flags,
                 'referer': referer_str(request),
                 'response_flags': response_flags,
+                # backward compatibility with Scrapy logformatter below 1.4 version
                 'flags': response_flags
             }
         }
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 52646ec1b..94e6c9fde 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -66,7 +66,6 @@ class LoggingContribTest(unittest.TestCase):
 
 
 class LogFormatterSubclass(LogFormatter):
-    # Formatter with format spec that is same as in Scrapy before 1.3 version.
     def crawled(self, request, response, spider):
         kwargs = super(LogFormatterSubclass, self).crawled(
         request, response, spider)
@@ -82,7 +81,6 @@ class LogFormatterSubclass(LogFormatter):
 
 
 class LogformatterSubclassTest(LoggingContribTest):
-    # Test if old crawledmsg format string still works fine
     def setUp(self):
         self.formatter = LogFormatterSubclass()
         self.spider = Spider('default')

From 49c5afc5ff6c810c78678ea1c86beb19ca3487ad Mon Sep 17 00:00:00 2001
From: harshasrinivas <harshsrinivas@gmail.com>
Date: Sun, 19 Mar 2017 06:08:35 +0530
Subject: [PATCH 1444/2974] Fix bug involving OR condition

---
 scrapy/downloadermiddlewares/retry.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index c22437ff1..07e979628 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -63,7 +63,10 @@ class RetryMiddleware(object):
     def _retry(self, request, reason, spider):
         retries = request.meta.get('retry_times', 0) + 1
 
-        retry_times = request.meta.get('max_retry_times') or self.max_retry_times
+        retry_times = self.max_retry_times
+
+        if 'max_retry_times' in request.meta:
+            retry_times = request.meta['max_retry_times']
 
         stats = spider.crawler.stats
         if retries <= retry_times:

From 0d9ebd6e1ed58654ea1996d5a236b4d0240590df Mon Sep 17 00:00:00 2001
From: harshasrinivas <harshsrinivas@gmail.com>
Date: Sun, 19 Mar 2017 06:15:46 +0530
Subject: [PATCH 1445/2974] Update tests for max_retry_times

---
 tests/test_downloadermiddleware_retry.py | 76 +++++++++++++++++++-----
 1 file changed, 61 insertions(+), 15 deletions(-)

diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 064c740c9..5f1760fd1 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -103,29 +103,75 @@ class RetryTest(unittest.TestCase):
         req = self.mw.process_exception(req, exception, self.spider)
         self.assertEqual(req, None)
 
-    def test_different_retry(self):
+
+class MaxRetryTimesTest(unittest.TestCase):
+    def setUp(self):
+        self.crawler = get_crawler(Spider)
+        self.spider = self.crawler._create_spider('foo')
+        self.mw = RetryMiddleware.from_crawler(self.crawler)
+        self.mw.max_retry_times = 2
+
+    def test_without_metakey(self):
         
-        req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3})
+        req = Request('http://www.scrapytest.org/invalid_url')
 
-        # SETINGS: meta(max_retry_times) = 3, RETRY_TIMES = 2
-        self._test_retry(req, DNSLookupError('foo'), 3)
+        # SETTINGS: RETRY_TIMES is NON-ZERO
+        self.mw.max_retry_times = 5
+        self._test_retry(req, DNSLookupError('foo'), 5)
 
-        req2 = Request('http://www.scrapytest.org/invalid_url')
-        
-        # SETINGS: RETRY_TIMES < meta(max_retry_times)
-        self._test_retry(req2, DNSLookupError('foo'), 2)
-
-        # SETINGS: RETRY_TIMES = 0
+        # SETTINGS: RETRY_TIMES = 0
         self.mw.max_retry_times = 0
-        self._test_retry(req2, DNSLookupError('foo'), 0)
-        
-        # SETINGS: RETRY_TIMES > meta(max_retry_times)
-        self.mw.max_retry_times = 4
-        self._test_retry(req2, DNSLookupError('foo'), 4)
+        self._test_retry(req, DNSLookupError('foo'), 0)
 
         # RESET RETRY_TIMES SETTINGS
         self.mw.max_retry_times = 2
 
+    def test_with_metakey_preceding(self):
+        # request with meta(max_retry_times) is called first
+        
+        req1 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3})
+        req2 = Request('http://www.scrapytest.org/invalid_url')
+        req3 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 4})
+
+        # SETINGS: RETRY_TIMES < meta(max_retry_times)
+        self.mw.max_retry_times = 2
+        self._test_retry(req1, DNSLookupError('foo'), 3)
+        self._test_retry(req2, DNSLookupError('foo'), 2)
+
+        # SETINGS: RETRY_TIMES > meta(max_retry_times)
+        self.mw.max_retry_times = 5
+        self._test_retry(req3, DNSLookupError('foo'), 4)
+        self._test_retry(req2, DNSLookupError('foo'), 5)
+
+        # RESET RETRY_TIMES SETTINGS
+        self.mw.max_retry_times = 2
+
+    def test_with_metakey_succeeding(self):
+        # request with meta(max_retry_times) is called second
+        
+        req1 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3})
+        req2 = Request('http://www.scrapytest.org/invalid_url')
+        req3 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 4})
+
+        # SETINGS: RETRY_TIMES < meta(max_retry_times)
+        self.mw.max_retry_times = 2
+        self._test_retry(req2, DNSLookupError('foo'), 2)
+        self._test_retry(req1, DNSLookupError('foo'), 3)
+
+        # SETINGS: RETRY_TIMES > meta(max_retry_times)
+        self.mw.max_retry_times = 5
+        self._test_retry(req2, DNSLookupError('foo'), 5)
+        self._test_retry(req3, DNSLookupError('foo'), 4)
+
+        # RESET RETRY_TIMES SETTINGS
+        self.mw.max_retry_times = 2
+
+    def test_with_metakey_zero(self):
+        
+        req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 0})
+        self._test_retry(req, DNSLookupError('foo'), 0)
+
+
     def _test_retry(self, req, exception, max_retry_times):
         
         while max_retry_times > 0:

From 10741aca720293a12dedda4d1872cf0604b49f0b Mon Sep 17 00:00:00 2001
From: harshasrinivas <harshsrinivas@gmail.com>
Date: Sun, 19 Mar 2017 06:17:28 +0530
Subject: [PATCH 1446/2974] Update docs - improve clarity

---
 docs/topics/downloader-middleware.rst | 8 ++++----
 docs/topics/request-response.rst      | 4 +++-
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index b808a6448..0d168017f 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -852,10 +852,10 @@ Default: ``2``
 
 Maximum number of times to retry, in addition to the first download.
 
-.. reqmeta:: max_retry_times
-
-If :attr:`Request.meta <scrapy.http.Request.meta>` has ``max_retry_times`` key
-set to some value, this setting will be ignored by this middleware for the corresponding request.
+Maximum number of retries can also be specified per-request using
+:reqmeta:`max_retry_times` attribute of :attr:`Request.meta <scrapy.http.Request.meta>`.
+When initialized, the :reqmeta:`max_retry_times` meta key takes higher
+precedence over the :setting:`RETRY_TIMES` setting.
 
 .. setting:: RETRY_HTTP_CODES
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 64a1e55fa..03918fd2d 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -348,7 +348,9 @@ Whether or not to fail on broken responses. See:
 max_retry_times
 ---------------
 
-The meta key is used set retry times per request. When initialized, the :setting:`RETRY_TIMES` setting will be ignored by the downloader middleware.
+The meta key is used set retry times per request. When initialized, the
+:reqmeta:`max_retry_times` meta key takes higher precedence over the
+:setting:`RETRY_TIMES` setting.
 
 .. _topics-request-response-ref-request-subclasses:
 

From 605691792f5198d75524a1ad952489193970d4a7 Mon Sep 17 00:00:00 2001
From: Oto Brglez <otobrglez@gmail.com>
Date: Sun, 19 Mar 2017 12:35:39 +0100
Subject: [PATCH 1447/2974] Updating media-pipeline docs for S3-like storage.

---
 docs/topics/media-pipeline.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 82c0aaa88..733a7fe2b 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -169,6 +169,18 @@ policy::
 
 For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
 
+Because Scrapy uses ``boto`` / ``botocore`` internally you can also use other S3-like storages. Storages like
+self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
+
+    AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
+
+For self-hosting you also might feel the need not to use SSL and not to verify SSL connection::
+
+    AWS_USE_SSL = False # or True (None by default)
+    AWS_VERIFY = False # or True (None by default)
+
+.. _Minio: https://github.com/minio/minio
+.. _s3.scality: https://s3.scality.com/
 .. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
 Usage example

From 11cf6ad4258250b8a2410029eb794ed892dc4cea Mon Sep 17 00:00:00 2001
From: Oto Brglez <otobrglez@gmail.com>
Date: Sun, 19 Mar 2017 12:48:06 +0100
Subject: [PATCH 1448/2974] Comments for AWS_ENDPOINT_URL setting.

---
 docs/topics/settings.rst | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index ccdd02c4e..3ac3fd5af 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -180,6 +180,34 @@ such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
 
 .. setting:: BOT_NAME
 
+AWS_ENDPOINT_URL
+----------------
+
+Default: ``None``
+
+Endpoint URL used for S3-like self-hosted storage. Storage like Minio or s3.scality.
+
+.. setting:: AWS_ENDPOINT_URL
+
+AWS_USE_SSL
+-----------
+
+Default: ``None``
+
+Use this option if you want to disable SSL connection for communication with S3 or S3-like storage.
+By default SSL will be used.
+
+.. setting:: AWS_USE_SSL
+
+AWS_VERIFY
+----------
+
+Default: ``None``
+
+Verify SSL connection between Scrapy and S3 or S3-like storage. By default SSL verification will occur.
+
+.. setting:: AWS_VERIFY
+
 BOT_NAME
 --------
 

From a57e49d55b7d703c8e5153811d6f5de3eacd2272 Mon Sep 17 00:00:00 2001
From: harshasrinivas <harshsrinivas@gmail.com>
Date: Mon, 20 Mar 2017 19:49:31 +0530
Subject: [PATCH 1449/2974] Add sphinx_rtd_theme to docs setup readme

---
 docs/README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/README.rst b/docs/README.rst
index 733af2af4..af8bf4297 100644
--- a/docs/README.rst
+++ b/docs/README.rst
@@ -15,7 +15,7 @@ and all its dependencies run
 
 ::
 
-    pip install 'Sphinx >= 1.3'
+    pip install 'Sphinx >= 1.3' sphinx_rtd_theme
 
 
 Compile the documentation

From 4ec07ae7640df6fdb13a24a4265232fd11e53755 Mon Sep 17 00:00:00 2001
From: harshasrinivas <harshsrinivas@gmail.com>
Date: Mon, 20 Mar 2017 22:21:08 +0530
Subject: [PATCH 1450/2974] Create docs/requirements.txt

---
 docs/README.rst       | 2 +-
 docs/requirements.txt | 2 ++
 tox.ini               | 3 +--
 3 files changed, 4 insertions(+), 3 deletions(-)
 create mode 100644 docs/requirements.txt

diff --git a/docs/README.rst b/docs/README.rst
index af8bf4297..f6011b2c6 100644
--- a/docs/README.rst
+++ b/docs/README.rst
@@ -15,7 +15,7 @@ and all its dependencies run
 
 ::
 
-    pip install 'Sphinx >= 1.3' sphinx_rtd_theme
+    pip install -r requirements.txt
 
 
 Compile the documentation
diff --git a/docs/requirements.txt b/docs/requirements.txt
new file mode 100644
index 000000000..44e97ceb1
--- /dev/null
+++ b/docs/requirements.txt
@@ -0,0 +1,2 @@
+'Sphinx >= 1.3'
+sphinx_rtd_theme
\ No newline at end of file
diff --git a/tox.ini b/tox.ini
index bbf50b733..6987847f8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -82,8 +82,7 @@ deps = {[testenv:py33]deps}
 [docs]
 changedir = docs
 deps =
-    Sphinx
-    sphinx_rtd_theme
+    -rdocs/requirements.txt
 
 [testenv:docs]
 changedir = {[docs]changedir}

From 83aa0c5e1a971c07d0beab89fde91ce1184f098c Mon Sep 17 00:00:00 2001
From: harshasrinivas <harshsrinivas@gmail.com>
Date: Mon, 20 Mar 2017 22:36:09 +0530
Subject: [PATCH 1451/2974] Clarify docs readme

---
 docs/README.rst       | 2 +-
 docs/requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/README.rst b/docs/README.rst
index f6011b2c6..0a343cd19 100644
--- a/docs/README.rst
+++ b/docs/README.rst
@@ -11,7 +11,7 @@ Setup the environment
 ---------------------
 
 To compile the documentation you need Sphinx Python library. To install it
-and all its dependencies run
+and all its dependencies run the following command from this dir
 
 ::
 
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 44e97ceb1..d3dcb97be 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,2 +1,2 @@
-'Sphinx >= 1.3'
+Sphinx>=1.3
 sphinx_rtd_theme
\ No newline at end of file

From c5f74f7d1a5245b65e5c7863b02a379667bee0dd Mon Sep 17 00:00:00 2001
From: Qiwei Huang <hqiwei11@hotmail.com>
Date: Mon, 20 Mar 2017 18:52:33 -0700
Subject: [PATCH 1452/2974] Update spiders.rst

Added a note to allowed_domains attribute, reminding users not to add urls into the list.
---
 docs/topics/spiders.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 5e69055d1..9e27614d8 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -80,6 +80,8 @@ scrapy.Spider
        allowed to crawl. Requests for URLs not belonging to the domain names
        specified in this list (or their subdomains) won't be followed if
        :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled.
+       .. note:: If you are scraping an url ``https://www.example.com/1.html`` 
+                 you should add ``'example.com'`` to allowed_domains list.
 
    .. attribute:: start_urls
 

From 8ecc307e8f04b058f2fc8a1a47759203ff6ebca0 Mon Sep 17 00:00:00 2001
From: Qiwei Huang <hqiwei11@hotmail.com>
Date: Mon, 20 Mar 2017 19:37:07 -0700
Subject: [PATCH 1453/2974] Update spiders.rst

Added note to allowed_domain attribute with an example explaining what goes in the list
---
 docs/topics/spiders.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 9e27614d8..49c0cefb5 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -80,8 +80,9 @@ scrapy.Spider
        allowed to crawl. Requests for URLs not belonging to the domain names
        specified in this list (or their subdomains) won't be followed if
        :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled.
-       .. note:: If you are scraping an url ``https://www.example.com/1.html`` 
-                 you should add ``'example.com'`` to allowed_domains list.
+       
+       Let's say your target url is ``https://www.example.com/1.html``,
+       then add ``'example.com'`` to the list.
 
    .. attribute:: start_urls
 

From b040df5ac09cf133cd07b505d20469a56409129c Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 21 Mar 2017 15:56:18 +0500
Subject: [PATCH 1454/2974] TST cleanup spider middleware tests

---
 tests/test_spidermiddleware.py | 73 ++++++++++++++++------------------
 1 file changed, 34 insertions(+), 39 deletions(-)

diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 2d05c335c..109bcc250 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -255,61 +255,62 @@ class ThirdMiddleware(object):
 
 class TestSpiderMiddleware(TestCase):
 
-    def setUp(self):
-        self.mockserver = MockServer()
-        self.mockserver.__enter__()
+    @classmethod
+    def setUpClass(cls):
+        cls.mockserver = MockServer()
+        cls.mockserver.__enter__()
 
-    def tearDown(self):
-        self.mockserver.__exit__(None, None, None)
+    @classmethod
+    def tearDownClass(cls):
+        cls.mockserver.__exit__(None, None, None)
 
     @defer.inlineCallbacks
-    def test_process_spider_exception_from_parse_method(self):
-        # non-generator return value
-        crawler = get_crawler(NotAGeneratorSpider)
+    def crawl_log(self, spider):
+        crawler = get_crawler(spider)
         with LogCapture() as log:
             yield crawler.crawl()
+        raise defer.returnValue(log)
+
+    @defer.inlineCallbacks
+    def test_process_spider_exception_from_parse_method_non_generator(self):
+        # non-generator return value
+        log = yield self.crawl_log(NotAGeneratorSpider)
         self.assertIn("AssertionError exception caught", str(log))
         self.assertIn("spider_exceptions/AssertionError", str(log))
+
+    @defer.inlineCallbacks
+    def test_process_spider_exception_from_parse_method_generator_no_items(self):
         # generator return value, no items before the error
-        crawler = get_crawler(GeneratorErrorBeforeItemsSpider)
-        with LogCapture() as log:
-            yield crawler.crawl()
+        log = yield self.crawl_log(GeneratorErrorBeforeItemsSpider)
         self.assertIn("ValueError exception caught", str(log))
         self.assertIn("spider_exceptions/ValueError", str(log))
+
+    @defer.inlineCallbacks
+    def test_process_spider_exception_from_parse_method_generator_with_items(self):
         # generator return value, 3 items before the error
-        crawler = get_crawler(GeneratorErrorAfterItemsSpider)
-        with LogCapture() as log:
-            yield crawler.crawl()
+        log = yield self.crawl_log(GeneratorErrorAfterItemsSpider)
         self.assertIn("'item_scraped_count': 3", str(log))
         self.assertIn("FloatingPointError exception caught", str(log))
         self.assertIn("spider_exceptions/FloatingPointError", str(log))
 
     @defer.inlineCallbacks
     def test_process_spider_exception_from_previous_middleware_input(self):
-        crawler = get_crawler(FromPreviousMiddlewareInputSpider)
-        with LogCapture() as log:
-            yield crawler.crawl()
+        log = yield self.crawl_log(FromPreviousMiddlewareInputSpider)
         self.assertIn("LookupError exception caught", str(log))
 
     @defer.inlineCallbacks
     def test_process_spider_exception_from_previous_middleware_output(self):
         # non-generator output value
-        crawler = get_crawler(NotAGeneratorFromPreviousMiddlewareOutputSpider)
-        with LogCapture() as log:
-            yield crawler.crawl()
+        log = yield self.crawl_log(NotAGeneratorFromPreviousMiddlewareOutputSpider)
         self.assertNotIn("UnicodeError exception caught", str(log))
         # generator output value
-        crawler = get_crawler(GeneratorFromPreviousMiddlewareOutputSpider)
-        with LogCapture() as log:
-            yield crawler.crawl()
+        log = yield self.crawl_log(GeneratorFromPreviousMiddlewareOutputSpider)
         self.assertIn("'item_scraped_count': 3", str(log))
         self.assertIn("NameError exception caught", str(log))
 
     @defer.inlineCallbacks
     def test_process_spider_exception_do_something(self):
-        crawler = get_crawler(DoSomethingSpider)
-        with LogCapture() as log:
-            yield crawler.crawl()
+        log = yield self.crawl_log(DoSomethingSpider)
         self.assertIn("ImportError exception caught", str(log))
         self.assertIn("{'value': 10}", str(log))
         self.assertIn("{'value': 'asdf'}", str(log))
@@ -320,25 +321,19 @@ class TestSpiderMiddleware(TestCase):
     def test_process_spider_exception_invalid_return_value_previous_middleware(self):
         """ don't catch _InvalidOutput from middleware """
         # on middleware's input
-        crawler1 = get_crawler(InvalidReturnValueFromPreviousMiddlewareInputSpider)
-        with LogCapture() as log1:
-            yield crawler1.crawl()
+        log1 = yield self.crawl_log(InvalidReturnValueFromPreviousMiddlewareInputSpider)
         self.assertNotIn("_InvalidOutput exception caught", str(log1))
         self.assertIn("'spider_exceptions/_InvalidOutput'", str(log1))
         # on middleware's output
-        crawler2 = get_crawler(InvalidReturnValueFromPreviousMiddlewareOutputSpider)
-        with LogCapture() as log2:
-            yield crawler2.crawl()
+        log2 = yield self.crawl_log(InvalidReturnValueFromPreviousMiddlewareOutputSpider)
         self.assertNotIn("_InvalidOutput exception caught", str(log2))
         self.assertIn("'spider_exceptions/_InvalidOutput'", str(log2))
 
     @defer.inlineCallbacks
     def test_process_spider_exception_execution_chain(self):
         # on middleware's input
-        crawler1 = get_crawler(ExecutionChainSpider)
-        with LogCapture() as log1:
-            yield crawler1.crawl()
-        self.assertNotIn("handled_by_first_middleware", str(log1))
-        self.assertNotIn("handled_by_second_middleware", str(log1))
-        self.assertIn("MemoryError exception caught", str(log1))
-        self.assertIn("handled_by_third_middleware", str(log1))
+        log = yield self.crawl_log(ExecutionChainSpider)
+        self.assertNotIn("handled_by_first_middleware", str(log))
+        self.assertNotIn("handled_by_second_middleware", str(log))
+        self.assertIn("MemoryError exception caught", str(log))
+        self.assertIn("handled_by_third_middleware", str(log))

From 21d794d35ae1347918e6bf5b2ffe13515ea28795 Mon Sep 17 00:00:00 2001
From: Simon Diviani Gartz <simonspublic@gmail.com>
Date: Tue, 21 Mar 2017 17:04:30 +0100
Subject: [PATCH 1455/2974] Fixes conversion of transparent PNG with palette
 images to jpg #2452

---
 scrapy/pipelines/images.py    | 5 +++++
 tests/test_pipeline_images.py | 8 ++++++++
 2 files changed, 13 insertions(+)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 5796bfb80..bc449431f 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -132,6 +132,11 @@ class ImagesPipeline(FilesPipeline):
             background = Image.new('RGBA', image.size, (255, 255, 255))
             background.paste(image, image)
             image = background.convert('RGB')
+        elif image.mode == 'P':
+            image = image.convert("RGBA")
+            background = Image.new('RGBA', image.size, (255, 255, 255))
+            background.paste(image, image)
+            image = background.convert('RGB')
         elif image.mode != 'RGB':
             image = image.convert('RGB')
 
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 342f25ea9..0f3047602 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -96,6 +96,14 @@ class ImagesPipelineTestCase(unittest.TestCase):
         self.assertEquals(converted.mode, 'RGB')
         self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))])
 
+        # transparency case with palette: P and PNG
+        COLOUR = (0, 127, 255, 50)
+        im = _create_image('PNG', 'RGBA', SIZE, COLOUR)
+        im = im.convert('P')
+        converted, _ = self.pipeline.convert_image(im)
+        self.assertEquals(converted.mode, 'RGB')
+        self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))])
+
 
 class DeprecatedImagesPipeline(ImagesPipeline):
     def file_key(self, url):

From 99e3c0d653e23d1af3e4236b88a747317e1c8a8a Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 23 Mar 2017 11:52:01 +0100
Subject: [PATCH 1456/2974] Set bodyproducer with empty content for POST

---
 scrapy/core/downloader/handlers/http11.py | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 37e836809..bff4a30c9 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -280,19 +280,7 @@ class ScrapyAgent(object):
         if request.body:
             bodyproducer = _RequestBodyProducer(request.body)
         else:
-            bodyproducer = None
-            # Setting Content-Length: 0 even for POST requests is not a
-            # MUST per HTTP RFCs, but it's common behavior, and some
-            # servers require this, otherwise returning HTTP 411 Length required
-            #
-            # RFC 7230#section-3.3.2:
-            # "a Content-Length header field is normally sent in a POST
-            # request even when the value is 0 (indicating an empty payload body)."
-            #
-            # Twisted Agent will not add "Content-Length: 0" by itself
-            if method == b'POST':
-                headers.addRawHeader(b'Content-Length', b'0')
-
+            bodyproducer = _RequestBodyProducer(b'') if method == b'POST' else None
         start_time = time()
         d = agent.request(
             method, to_bytes(url, encoding='ascii'), headers, bodyproducer)

From 38e6857c957ef023533128f054688152be223c87 Mon Sep 17 00:00:00 2001
From: harshasrinivas <harshsrinivas@gmail.com>
Date: Thu, 23 Mar 2017 19:45:04 +0530
Subject: [PATCH 1457/2974] Improvise the clarity of test cases

---
 tests/test_downloadermiddleware_retry.py | 105 +++++++++++------------
 1 file changed, 51 insertions(+), 54 deletions(-)

diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 5f1760fd1..51b79b6c3 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -110,75 +110,72 @@ class MaxRetryTimesTest(unittest.TestCase):
         self.spider = self.crawler._create_spider('foo')
         self.mw = RetryMiddleware.from_crawler(self.crawler)
         self.mw.max_retry_times = 2
+        self.invalid_url = 'http://www.scrapytest.org/invalid_url'
 
-    def test_without_metakey(self):
-        
-        req = Request('http://www.scrapytest.org/invalid_url')
-
-        # SETTINGS: RETRY_TIMES is NON-ZERO
-        self.mw.max_retry_times = 5
-        self._test_retry(req, DNSLookupError('foo'), 5)
+    def test_with_settings_zero(self):
 
         # SETTINGS: RETRY_TIMES = 0
         self.mw.max_retry_times = 0
-        self._test_retry(req, DNSLookupError('foo'), 0)
 
-        # RESET RETRY_TIMES SETTINGS
-        self.mw.max_retry_times = 2
-
-    def test_with_metakey_preceding(self):
-        # request with meta(max_retry_times) is called first
-        
-        req1 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3})
-        req2 = Request('http://www.scrapytest.org/invalid_url')
-        req3 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 4})
-
-        # SETINGS: RETRY_TIMES < meta(max_retry_times)
-        self.mw.max_retry_times = 2
-        self._test_retry(req1, DNSLookupError('foo'), 3)
-        self._test_retry(req2, DNSLookupError('foo'), 2)
-
-        # SETINGS: RETRY_TIMES > meta(max_retry_times)
-        self.mw.max_retry_times = 5
-        self._test_retry(req3, DNSLookupError('foo'), 4)
-        self._test_retry(req2, DNSLookupError('foo'), 5)
-
-        # RESET RETRY_TIMES SETTINGS
-        self.mw.max_retry_times = 2
-
-    def test_with_metakey_succeeding(self):
-        # request with meta(max_retry_times) is called second
-        
-        req1 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3})
-        req2 = Request('http://www.scrapytest.org/invalid_url')
-        req3 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 4})
-
-        # SETINGS: RETRY_TIMES < meta(max_retry_times)
-        self.mw.max_retry_times = 2
-        self._test_retry(req2, DNSLookupError('foo'), 2)
-        self._test_retry(req1, DNSLookupError('foo'), 3)
-
-        # SETINGS: RETRY_TIMES > meta(max_retry_times)
-        self.mw.max_retry_times = 5
-        self._test_retry(req2, DNSLookupError('foo'), 5)
-        self._test_retry(req3, DNSLookupError('foo'), 4)
-
-        # RESET RETRY_TIMES SETTINGS
-        self.mw.max_retry_times = 2
+        req = Request(self.invalid_url)
+        self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times)
 
     def test_with_metakey_zero(self):
+
+        # SETTINGS: meta(max_retry_times) = 0
+        meta_max_retry_times = 0
         
-        req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 0})
+        req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
+        self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times)
+
+    def test_without_metakey(self):
+
+        # SETTINGS: RETRY_TIMES is NON-ZERO
+        self.mw.max_retry_times = 5
+
+        req = Request(self.invalid_url)
+        self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times)
+
+    def test_with_metakey_greater(self):
+        
+        # SETINGS: RETRY_TIMES < meta(max_retry_times)
+        self.mw.max_retry_times = 2
+        meta_max_retry_times = 3
+
+        req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
+        req2 = Request(self.invalid_url)
+
+        self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times)
+        self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times)
+
+    def test_with_metakey_lesser(self):
+        
+        # SETINGS: RETRY_TIMES > meta(max_retry_times)
+        self.mw.max_retry_times = 5
+        meta_max_retry_times = 4
+
+        req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
+        req2 = Request(self.invalid_url)
+
+        self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times)
+        self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times)
+
+    def test_with_dont_retry(self):
+
+        # SETTINGS: meta(max_retry_times) = 4
+        meta_max_retry_times = 4
+
+        req = Request(self.invalid_url, meta= \
+            {'max_retry_times': meta_max_retry_times, 'dont_retry': True})
+
         self._test_retry(req, DNSLookupError('foo'), 0)
 
 
     def _test_retry(self, req, exception, max_retry_times):
         
-        while max_retry_times > 0:
+        for i in range(0, max_retry_times):
             req = self.mw.process_exception(req, exception, self.spider)
             assert isinstance(req, Request)
-            if req.meta['retry_times'] == max_retry_times:
-                break
 
         # discard it
         req = self.mw.process_exception(req, exception, self.spider)

From 0298bcbe79ec9c010282eb59d79848f17bada7ed Mon Sep 17 00:00:00 2001
From: harshasrinivas <harshsrinivas@gmail.com>
Date: Fri, 24 Mar 2017 18:13:08 +0530
Subject: [PATCH 1458/2974] Update Makefile to open webbrowser in MacOS (#2661)

---
 docs/Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/Makefile b/docs/Makefile
index eaba3ba2b..a3d1611f9 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -82,7 +82,8 @@ pydoc-topics: build
 	      "into the Lib/ directory"
 
 htmlview: html
-	 $(PYTHON) -c "import webbrowser; webbrowser.open('build/html/index.html')"
+	 $(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \
+	 os.path.realpath('build/html/index.html'))"
 
 clean:
 	-rm -rf build/*

From 2ff6b0572318742f1b16ee7f9f0c3b836b020633 Mon Sep 17 00:00:00 2001
From: harshasrinivas <harshsrinivas@gmail.com>
Date: Fri, 24 Mar 2017 20:43:28 +0530
Subject: [PATCH 1459/2974] Remove __nonzero__ from SelectorList docs

---
 docs/topics/selectors.rst | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 8a5d44aac..61206a193 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -714,10 +714,6 @@ SelectorList objects
        Call the ``.re()`` method for each element in this list and return
        their results flattened, as a list of unicode strings.
 
-   .. method:: __nonzero__()
-
-        returns True if the list is not empty, False otherwise.
-
 
 Selector examples on HTML response
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

From 163618c9b792ded60e451e929341bb502d9cc19a Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 29 Mar 2017 12:02:44 +0200
Subject: [PATCH 1460/2974] FAQ Rewrite note on Python 3 support on Windows

---
 docs/faq.rst           | 3 ++-
 docs/intro/install.rst | 6 ++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index ad11b071b..f0ee20b5e 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -74,7 +74,8 @@ Python 2.6 support was dropped starting at Scrapy 0.20.
 Python 3 support was added in Scrapy 1.1.
 
 .. note::
-    Python 3 is not yet supported on Windows.
+    For Python 3 support on Windows, it is recommended to use
+    Anaconda/Miniconda as :ref:`outlined in the installation guide <intro-install-windows>`.
 
 Did Scrapy "steal" X from Django?
 ---------------------------------
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 86387ef5e..9cec2eaee 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -116,6 +116,8 @@ Python virtualenvs can be created to use Python 2 by default, or Python 3 by def
 Platform specific installation notes
 ====================================
 
+.. _intro-install-windows:
+
 Windows
 -------
 
@@ -128,6 +130,8 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with::
   conda install -c conda-forge scrapy
 
 
+.. _intro-install-ubuntu:
+
 Ubuntu 12.04 or above
 ---------------------
 
@@ -163,6 +167,8 @@ you can install Scrapy with ``pip`` after that::
     Wheezy (7.0) and above.
 
 
+.. _intro-install-macos:
+
 Mac OS X
 --------
 

From 6352c2e9b2028473acdbd58175bbc5638258e29d Mon Sep 17 00:00:00 2001
From: LMKight <LMKight@users.noreply.github.com>
Date: Sun, 2 Apr 2017 15:11:13 +0200
Subject: [PATCH 1461/2974] fixed command list

---
 scrapy/cmdline.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index cb7bbd64d..05b0a12e0 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -27,8 +27,9 @@ def _get_commands_from_module(module, inproject):
     d = {}
     for cmd in _iter_command_classes(module):
         if inproject or not cmd.requires_project:
-            cmdname = cmd.__module__.split('.')[-1]
-            d[cmdname] = cmd()
+            if not cmd.__module__ == module:
+                cmdname = cmd.__module__.split('.')[-1]
+                d[cmdname] = cmd()
     return d
 
 def _get_commands_from_entry_points(inproject, group='scrapy.commands'):

From 05ce1296c6a60f23e81af7ec38ac1855e78be79f Mon Sep 17 00:00:00 2001
From: LMKight <LMKight@users.noreply.github.com>
Date: Mon, 3 Apr 2017 19:47:01 +0200
Subject: [PATCH 1462/2974] changed code according to request

---
 scrapy/cmdline.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 05b0a12e0..8edc1ad2d 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -20,16 +20,16 @@ def _iter_command_classes(module_name):
         for obj in vars(module).values():
             if inspect.isclass(obj) and \
                     issubclass(obj, ScrapyCommand) and \
-                    obj.__module__ == module.__name__:
+                    obj.__module__ == module.__name__ and \
+                    not obj == ScrapyCommand:
                 yield obj
 
 def _get_commands_from_module(module, inproject):
     d = {}
     for cmd in _iter_command_classes(module):
         if inproject or not cmd.requires_project:
-            if not cmd.__module__ == module:
-                cmdname = cmd.__module__.split('.')[-1]
-                d[cmdname] = cmd()
+            cmdname = cmd.__module__.split('.')[-1]
+            d[cmdname] = cmd()
     return d
 
 def _get_commands_from_entry_points(inproject, group='scrapy.commands'):

From 422b38f65ccbbe4479b6249c89bc57bd2d22d092 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 11 Apr 2017 16:55:43 +0200
Subject: [PATCH 1463/2974] DOC Rearrange selector sections

---
 docs/topics/selectors.rst | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 8a5d44aac..68d8120a8 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -593,6 +593,9 @@ Built-in Selectors reference
 .. module:: scrapy.selector
    :synopsis: Selector class
 
+Selector objects
+----------------
+
 .. class:: Selector(response=None, text=None, type=None)
 
   An instance of :class:`Selector` is a wrapper over response to select
@@ -720,7 +723,7 @@ SelectorList objects
 
 
 Selector examples on HTML response
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+----------------------------------
 
 Here's a couple of :class:`Selector` examples to illustrate several concepts.
 In all cases, we assume there is already a :class:`Selector` instantiated with
@@ -745,7 +748,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
           print node.xpath("@class").extract()
 
 Selector examples on XML response
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+---------------------------------
 
 Here's a couple of examples to illustrate several concepts. In both cases we
 assume there is already a :class:`Selector` instantiated with an
@@ -767,7 +770,7 @@ assume there is already a :class:`Selector` instantiated with an
 .. _removing-namespaces:
 
 Removing namespaces
-~~~~~~~~~~~~~~~~~~~
+-------------------
 
 When dealing with scraping projects, it is often quite convenient to get rid of
 namespaces altogether and just work with element names, to write more

From f3f7a4186150119377891069a3155cea8cabb142 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 12 Apr 2017 16:32:21 +0200
Subject: [PATCH 1464/2974] Travis CI: use portable pypy for Linux

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 2df02ea43..906115096 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -33,8 +33,8 @@ install:
         else
           rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT"
         fi
-        # get latest PyPy from pyenv directly (thanks to natural version sort option -V)
-        export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy-[0-9][\.0-9]*$' |sort -V |tail -1`
+        # get latest portable PyPy from pyenv directly (thanks to natural version sort option -V)
+        export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy-portable-[0-9][\.0-9]*$' |sort -V |tail -1`
         "$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION"
         virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"

From 30eec559104649653d4c0e182fb2d3e2252dca1e Mon Sep 17 00:00:00 2001
From: Julien Palard <julien@palard.fr>
Date: Sat, 22 Apr 2017 00:24:18 +0200
Subject: [PATCH 1465/2974] [PEDANTIC] FIX trailing whitespaces in LICENSE.

---
 LICENSE | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/LICENSE b/LICENSE
index 68ccf9762..6ead05ece 100644
--- a/LICENSE
+++ b/LICENSE
@@ -4,10 +4,10 @@ All rights reserved.
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:
 
-    1. Redistributions of source code must retain the above copyright notice, 
+    1. Redistributions of source code must retain the above copyright notice,
        this list of conditions and the following disclaimer.
-    
-    2. Redistributions in binary form must reproduce the above copyright 
+
+    2. Redistributions in binary form must reproduce the above copyright
        notice, this list of conditions and the following disclaimer in the
        documentation and/or other materials provided with the distribution.
 

From 00ee9eaeafcd3b3af43132c4513754627f97a886 Mon Sep 17 00:00:00 2001
From: Tiago Cardoso <tbcardoso@outlook.com>
Date: Sat, 22 Apr 2017 14:36:44 +0100
Subject: [PATCH 1466/2974] Mention how to disable request filtering in
 documentation of DUPEFILTER_CLASS setting

---
 docs/topics/settings.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 8367b1092..9bf07588b 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -646,6 +646,13 @@ override its ``request_fingerprint`` method. This method should accept
 scrapy :class:`~scrapy.http.Request` object and return its fingerprint
 (a string).
 
+You can disable filtering of duplicate requests by setting
+:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
+Be very careful about this however, because you can get into crawling loops.
+It's usually a better idea to set the ``dont_filter`` parameter to
+``True`` on the specific :class:`~scrapy.http.Request` that should not be
+filtered.
+
 .. setting:: DUPEFILTER_DEBUG
 
 DUPEFILTER_DEBUG

From 97fc68fa1699b1c782c6e6d888e21d995bdf071d Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 24 Apr 2017 22:08:39 +0200
Subject: [PATCH 1467/2974] Refactor conditions on body producer

---
 scrapy/core/downloader/handlers/http11.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index bff4a30c9..46493f87f 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -279,8 +279,10 @@ class ScrapyAgent(object):
             headers.removeHeader(b'Proxy-Authorization')
         if request.body:
             bodyproducer = _RequestBodyProducer(request.body)
+        elif method == b'POST':
+            bodyproducer = _RequestBodyProducer(b'')
         else:
-            bodyproducer = _RequestBodyProducer(b'') if method == b'POST' else None
+            bodyproducer = None
         start_time = time()
         d = agent.request(
             method, to_bytes(url, encoding='ascii'), headers, bodyproducer)

From b1a0a6e25810353f46d314cea1fd34bd37109b0c Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 25 Apr 2017 17:01:54 +0200
Subject: [PATCH 1468/2974] Make mockserver runnable outside of tox

Add POST support for Echo resource
---
 tests/mockserver.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 26ab51183..b95a6c3c4 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -15,7 +15,6 @@ from twisted.internet.task import deferLater
 
 
 from scrapy.utils.python import to_bytes, to_unicode
-from tests import tests_datadir
 
 
 def getarg(request, name, default=None, type=None):
@@ -122,6 +121,7 @@ class Echo(LeafResource):
             'body': to_unicode(request.content.read()),
         }
         return to_bytes(json.dumps(output))
+    render_POST = render_GET
 
 
 class RedirectTo(LeafResource):
@@ -174,7 +174,11 @@ class Root(Resource):
         self.putChild(b"echo", Echo())
         self.putChild(b"payload", PayloadResource())
         self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
-        self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/')))
+        try:
+            from tests import tests_datadir
+            self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/')))
+        except:
+            pass
         self.putChild(b"redirect-to", RedirectTo())
 
     def getChild(self, name, request):

From a63d9f502f50fbd948154fc65c8a94ffd4722d11 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 25 Apr 2017 17:03:03 +0200
Subject: [PATCH 1469/2974] Restore comments on why POST needs `Content-Length:
 0`

---
 scrapy/core/downloader/handlers/http11.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 46493f87f..55bd31303 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -280,6 +280,18 @@ class ScrapyAgent(object):
         if request.body:
             bodyproducer = _RequestBodyProducer(request.body)
         elif method == b'POST':
+            # Setting Content-Length: 0 even for POST requests is not a
+            # MUST per HTTP RFCs, but it's common behavior, and some
+            # servers require this, otherwise returning HTTP 411 Length required
+            #
+            # RFC 7230#section-3.3.2:
+            # "a Content-Length header field is normally sent in a POST
+            # request even when the value is 0 (indicating an empty payload body)."
+            #
+            # Twisted < 17 will not add "Content-Length: 0" by itself;
+            # Twisted >= 17 fixes this;
+            # Using a producer with an empty-string sends `0` as Content-Length
+            # for all versions of Twisted.
             bodyproducer = _RequestBodyProducer(b'')
         else:
             bodyproducer = None

From c3d0f9b6c10b68e436054ce9421d2ddadfc47087 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 25 Apr 2017 17:03:41 +0200
Subject: [PATCH 1470/2974] Add test for non-duplicated `Content-Length: 0` for
 bodyless POST

---
 tests/test_downloader_handlers.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 3efcf6e9c..0f28037ba 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -29,7 +29,7 @@ from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 
 from scrapy.spiders import Spider
-from scrapy.http import Request
+from scrapy.http import Headers, Request
 from scrapy.http.response.text import TextResponse
 from scrapy.responsetypes import responsetypes
 from scrapy.settings import Settings
@@ -37,7 +37,7 @@ from scrapy.utils.test import get_crawler, skip_if_no_boto
 from scrapy.utils.python import to_bytes
 from scrapy.exceptions import NotConfigured
 
-from tests.mockserver import MockServer, ssl_context_factory
+from tests.mockserver import MockServer, ssl_context_factory, Echo
 from tests.spiders import SingleRequestSpider
 
 class DummyDH(object):
@@ -202,6 +202,7 @@ class HttpTestCase(unittest.TestCase):
         r.putChild(b"broken-chunked", BrokenChunkedResource())
         r.putChild(b"contentlength", ContentLengthHeaderResource())
         r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource())
+        r.putChild(b"echo", Echo())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
         self.host = 'localhost'
@@ -310,6 +311,17 @@ class HttpTestCase(unittest.TestCase):
         request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'})
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
+    def test_content_length_zero_bodyless_post_only_one(self):
+        def _test(response):
+            import json
+            headers = Headers(json.loads(response.text)['headers'])
+            contentlengths = headers.getlist('Content-Length')
+            self.assertEquals(len(contentlengths), 1)
+            self.assertEquals(contentlengths, [b"0"])
+
+        request = Request(self.getURL('echo'), method='POST')
+        return self.download_request(request, Spider('foo')).addCallback(_test)
+
     def test_payload(self):
         body = b'1'*100 # PayloadResource requires body length to be 100
         request = Request(self.getURL('payload'), method='POST', body=body)

From 4bc0c6b0f4c5dae033bd83549b6d0a573fcf4805 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mart=C3=ADn=20Gait=C3=A1n?= <gaitan@gmail.com>
Date: Tue, 25 Apr 2017 22:33:22 -0300
Subject: [PATCH 1471/2974] Update practices.rst

fix a typo
---
 docs/topics/practices.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 25ae4b5ba..63913d3c4 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -238,7 +238,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
 * if possible, use `Google cache`_ to fetch pages, instead of hitting the sites
   directly
 * use a pool of rotating IPs. For example, the free `Tor project`_ or paid
-  services like `ProxyMesh`_. An open source alterantive is `scrapoxy`_, a
+  services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
   super proxy that you can attach your own proxies to.
 * use a highly distributed downloader that circumvents bans internally, so you
   can just focus on parsing clean pages. One example of such downloaders is

From 2b34c6edffcf14d81b5f11369648f4661e067ccc Mon Sep 17 00:00:00 2001
From: Rolando Espinoza <rndmax84@gmail.com>
Date: Sat, 4 Mar 2017 21:29:24 -0300
Subject: [PATCH 1472/2974] Abort connection earlier and avoid to buffer data

A symptom of this issue was having the log message "Received (X) bytes
larger than download max size (Y)" several times printed, with increased
X values.
---
 scrapy/core/downloader/handlers/http11.py | 11 ++++++++-
 tests/test_downloader_handlers.py         | 30 +++++++++++++++++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 55bd31303..9bfdd803c 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -348,7 +348,8 @@ class ScrapyAgent(object):
                            {'size': expected_size, 'warnsize': warnsize})
 
         def _cancel(_):
-            txresponse._transport._producer.loseConnection()
+            # Abort connection inmediately.
+            txresponse._transport._producer.abortConnection()
 
         d = defer.Deferred(_cancel)
         txresponse.deliverBody(_ResponseReader(
@@ -401,6 +402,11 @@ class _ResponseReader(protocol.Protocol):
         self._bytes_received = 0
 
     def dataReceived(self, bodyBytes):
+        # This maybe called several times after cancel was called with buffered
+        # data.
+        if self._finished.called:
+            return
+
         self._bodybuf.write(bodyBytes)
         self._bytes_received += len(bodyBytes)
 
@@ -409,6 +415,9 @@ class _ResponseReader(protocol.Protocol):
                          "max size (%(maxsize)s).",
                          {'bytes': self._bytes_received,
                           'maxsize': self._maxsize})
+            # Clear buffer earlier to avoid keeping data in memory for a long
+            # time.
+            self._bodybuf.truncate(0)
             self._finished.cancel()
 
         if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize:
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 0f28037ba..b52dac499 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -177,6 +177,16 @@ class EmptyContentTypeHeaderResource(resource.Resource):
         return request.content.read()
 
 
+class LargeChunkedFileResource(resource.Resource):
+    def render(self, request):
+        def response():
+            for i in range(1024):
+                request.write(b"x" * 1024)
+            request.finish()
+        reactor.callLater(0, response)
+        return server.NOT_DONE_YET
+
+
 class HttpTestCase(unittest.TestCase):
 
     scheme = 'http'
@@ -202,6 +212,7 @@ class HttpTestCase(unittest.TestCase):
         r.putChild(b"broken-chunked", BrokenChunkedResource())
         r.putChild(b"contentlength", ContentLengthHeaderResource())
         r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource())
+        r.putChild(b"largechunkedfile", LargeChunkedFileResource())
         r.putChild(b"echo", Echo())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
@@ -384,6 +395,25 @@ class Http11TestCase(HttpTestCase):
         d = self.download_request(request, Spider('foo', download_maxsize=9))
         yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
 
+    @defer.inlineCallbacks
+    def test_download_with_maxsize_very_large_file(self):
+        with mock.patch('scrapy.core.downloader.handlers.http11.logger') as logger:
+            request = Request(self.getURL('largechunkedfile'))
+
+            def check(logger):
+                logger.error.assert_called_once_with(mock.ANY, mock.ANY)
+
+            d = self.download_request(request, Spider('foo', download_maxsize=1500))
+            yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+
+            # As the error message is logged in the dataReceived callback, we
+            # have to give a bit of time to the reactor to process the queue
+            # after closing the connection.
+            d = defer.Deferred()
+            d.addCallback(check)
+            reactor.callLater(.1, d.callback, logger)
+            yield d
+
     @defer.inlineCallbacks
     def test_download_with_maxsize_per_req(self):
         meta = {'download_maxsize': 2}

From e6ab8bc9a5460c5af29a330598f7afa5a6efb7e2 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 27 Apr 2017 23:10:52 +0200
Subject: [PATCH 1473/2974] Change "localhost" test server certificate

---
 tests/keys/localhost.crt          | 20 ++++++++++++++++++++
 tests/keys/localhost.gen.README   | 21 +++++++++++++++++++++
 tests/keys/localhost.key          | 28 ++++++++++++++++++++++++++++
 tests/test_downloader_handlers.py |  4 ++--
 4 files changed, 71 insertions(+), 2 deletions(-)
 create mode 100644 tests/keys/localhost.crt
 create mode 100644 tests/keys/localhost.gen.README
 create mode 100644 tests/keys/localhost.key

diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt
new file mode 100644
index 000000000..13c5b5bd6
--- /dev/null
+++ b/tests/keys/localhost.crt
@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV
+BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x
+NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD
+VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K
+o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y
+njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY
+ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/
+8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52
+gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O
+BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j
+xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES
+9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i
+jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+
+TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own
+tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM
+am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U
+Mjybr/hQK3mnVdE=
+-----END CERTIFICATE-----
diff --git a/tests/keys/localhost.gen.README b/tests/keys/localhost.gen.README
new file mode 100644
index 000000000..19c29a725
--- /dev/null
+++ b/tests/keys/localhost.gen.README
@@ -0,0 +1,21 @@
+$ openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt
+Generating a 2048 bit RSA private key
+...................................................................................................+++
+.....+++
+writing new private key to 'localhost.key'
+-----
+You are about to be asked to enter information that will be incorporated
+into your certificate request.
+What you are about to enter is what is called a Distinguished Name or a DN.
+There are quite a few fields but you can leave some blank
+For some fields there will be a default value,
+If you enter '.', the field will be left blank.
+-----
+Country Name (2 letter code) [AU]:IE
+State or Province Name (full name) [Some-State]:.
+Locality Name (eg, city) []:.
+Organization Name (eg, company) [Internet Widgits Pty Ltd]:Scrapy
+Organizational Unit Name (eg, section) []:.
+Common Name (e.g. server FQDN or YOUR name) []:localhost
+Email Address []:.
+
diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key
new file mode 100644
index 000000000..da975e6d3
--- /dev/null
+++ b/tests/keys/localhost.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5
+ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4
+JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz
+vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2
+RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY
+b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/
+XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0
+gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk
+i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu
+Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss
+ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4
+hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD
+7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa
+EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf
+uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG
+HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR
+UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO
+j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri
+fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ
+6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI
+xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g
+DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f
+npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM
+8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6
+J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj
+Z+pMCFLbwAbFZ9Qb/NFlUv0=
+-----END PRIVATE KEY-----
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 0f28037ba..c138bec80 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -183,8 +183,8 @@ class HttpTestCase(unittest.TestCase):
     download_handler_cls = HTTPDownloadHandler
 
     # only used for HTTPS tests
-    keyfile = 'keys/cert.pem'
-    certfile = 'keys/cert.pem'
+    keyfile = 'keys/localhost.key'
+    certfile = 'keys/localhost.crt'
 
     def setUp(self):
         self.tmpname = self.mktemp()

From 6d14e392f1096d94669c00456855b64164dba6bd Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 27 Apr 2017 23:35:01 +0200
Subject: [PATCH 1474/2974] Remove old test certificate+key

---
 tests/keys/cert.pem | 36 ------------------------------------
 tests/mockserver.py |  2 +-
 2 files changed, 1 insertion(+), 37 deletions(-)
 delete mode 100644 tests/keys/cert.pem

diff --git a/tests/keys/cert.pem b/tests/keys/cert.pem
deleted file mode 100644
index 65478765e..000000000
--- a/tests/keys/cert.pem
+++ /dev/null
@@ -1,36 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDBjCCAm+gAwIBAgIBATANBgkqhkiG9w0BAQQFADB7MQswCQYDVQQGEwJTRzER
-MA8GA1UEChMITTJDcnlwdG8xFDASBgNVBAsTC00yQ3J5cHRvIENBMSQwIgYDVQQD
-ExtNMkNyeXB0byBDZXJ0aWZpY2F0ZSBNYXN0ZXIxHTAbBgkqhkiG9w0BCQEWDm5n
-cHNAcG9zdDEuY29tMB4XDTAwMDkxMDA5NTEzMFoXDTAyMDkxMDA5NTEzMFowUzEL
-MAkGA1UEBhMCU0cxETAPBgNVBAoTCE0yQ3J5cHRvMRIwEAYDVQQDEwlsb2NhbGhv
-c3QxHTAbBgkqhkiG9w0BCQEWDm5ncHNAcG9zdDEuY29tMFwwDQYJKoZIhvcNAQEB
-BQADSwAwSAJBAKy+e3dulvXzV7zoTZWc5TzgApr8DmeQHTYC8ydfzH7EECe4R1Xh
-5kwIzOuuFfn178FBiS84gngaNcrFi0Z5fAkCAwEAAaOCAQQwggEAMAkGA1UdEwQC
-MAAwLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRl
-MB0GA1UdDgQWBBTPhIKSvnsmYsBVNWjj0m3M2z0qVTCBpQYDVR0jBIGdMIGagBT7
-hyNp65w6kxXlxb8pUU/+7Sg4AaF/pH0wezELMAkGA1UEBhMCU0cxETAPBgNVBAoT
-CE0yQ3J5cHRvMRQwEgYDVQQLEwtNMkNyeXB0byBDQTEkMCIGA1UEAxMbTTJDcnlw
-dG8gQ2VydGlmaWNhdGUgTWFzdGVyMR0wGwYJKoZIhvcNAQkBFg5uZ3BzQHBvc3Qx
-LmNvbYIBADANBgkqhkiG9w0BAQQFAAOBgQA7/CqT6PoHycTdhEStWNZde7M/2Yc6
-BoJuVwnW8YxGO8Sn6UJ4FeffZNcYZddSDKosw8LtPOeWoK3JINjAk5jiPQ2cww++
-7QGG/g5NDjxFZNDJP1dGiLAxPW6JXwov4v0FmdzfLOZ01jDcgQQZqEpYlgpuI5JE
-WUQ9Ho4EzbYCOQ==
------END CERTIFICATE-----
------BEGIN RSA PRIVATE KEY-----
-MIIBPAIBAAJBAKy+e3dulvXzV7zoTZWc5TzgApr8DmeQHTYC8ydfzH7EECe4R1Xh
-5kwIzOuuFfn178FBiS84gngaNcrFi0Z5fAkCAwEAAQJBAIqm/bz4NA1H++Vx5Ewx
-OcKp3w19QSaZAwlGRtsUxrP7436QjnREM3Bm8ygU11BjkPVmtrKm6AayQfCHqJoT
-ZIECIQDW0BoMoL0HOYM/mrTLhaykYAVqgIeJsPjvkEhTFXWBuQIhAM3deFAvWNu4
-nklUQ37XsCT2c9tmNt1LAT+slG2JOTTRAiAuXDtC/m3NYVwyHfFm+zKHRzHkClk2
-HjubeEgjpj32AQIhAJqMGTaZVOwevTXvvHwNEH+vRWsAYU/gbx+OQB+7VOcBAiEA
-oolb6NMg/R3enNPvS1O4UU1H8wpaF77L4yiSWlE0p4w=
------END RSA PRIVATE KEY-----
------BEGIN CERTIFICATE REQUEST-----
-MIIBDTCBuAIBADBTMQswCQYDVQQGEwJTRzERMA8GA1UEChMITTJDcnlwdG8xEjAQ
-BgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3DQEJARYObmdwc0Bwb3N0MS5jb20w
-XDANBgkqhkiG9w0BAQEFAANLADBIAkEArL57d26W9fNXvOhNlZzlPOACmvwOZ5Ad
-NgLzJ1/MfsQQJ7hHVeHmTAjM664V+fXvwUGJLziCeBo1ysWLRnl8CQIDAQABoAAw
-DQYJKoZIhvcNAQEEBQADQQA7uqbrNTjVWpF6By5ZNPvhZ4YdFgkeXFVWi5ao/TaP
-Vq4BG021fJ9nlHRtr4rotpgHDX1rr+iWeHKsx4+5DRSy
------END CERTIFICATE REQUEST-----
\ No newline at end of file
diff --git a/tests/mockserver.py b/tests/mockserver.py
index b95a6c3c4..98723846e 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -203,7 +203,7 @@ class MockServer():
         time.sleep(0.2)
 
 
-def ssl_context_factory(keyfile='keys/cert.pem', certfile='keys/cert.pem'):
+def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt'):
     return ssl.DefaultOpenSSLContextFactory(
          os.path.join(os.path.dirname(__file__), keyfile),
          os.path.join(os.path.dirname(__file__), certfile),

From 6c1cacb5d5f0e5523d429cb89808dabfd824dcd5 Mon Sep 17 00:00:00 2001
From: Liu Siyuan <liusy182@hotmail.com>
Date: Sat, 6 May 2017 05:47:06 +0800
Subject: [PATCH 1475/2974] [MRG+1] doc: fix documentation error in
 link-extractor.rst (#2676)

* fix doc error in link-extractor.rst

* remove the import clause

* update based on suggestion
---
 docs/topics/link-extractors.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 01d7f0b97..f40a36d31 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -8,7 +8,7 @@ Link extractors are objects whose only purpose is to extract links from web
 pages (:class:`scrapy.http.Response` objects) which will be eventually
 followed.
 
-There is ``scrapy.linkextractors import LinkExtractor`` available
+There is ``scrapy.linkextractors.LinkExtractor`` available
 in Scrapy, but you can create your own custom Link Extractors to suit your
 needs by implementing a simple interface.
 

From 4966dd7a7fa544f6c5b3bbeb1f09426d9adcdb16 Mon Sep 17 00:00:00 2001
From: yandongxu <nextocean@gmail.com>
Date: Mon, 8 May 2017 18:50:30 +0800
Subject: [PATCH 1476/2974] Fix doc: open file with "wb" mode will get an error
 in python 3

---
 docs/topics/item-pipeline.rst | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 8c7aa361f..33e4d7429 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -107,7 +107,7 @@ format::
    class JsonWriterPipeline(object):
 
        def open_spider(self, spider):
-           self.file = open('items.jl', 'wb')
+           self.file = open('items.jl', 'w')
 
        def close_spider(self, spider):
            self.file.close()
@@ -134,7 +134,7 @@ method and how to clean up the resources properly.::
     import pymongo
 
     class MongoPipeline(object):
-    
+
         collection_name = 'scrapy_items'
 
         def __init__(self, mongo_uri, mongo_db):
@@ -248,4 +248,3 @@ To activate an Item Pipeline component you must add its class to the
 The integer values you assign to classes in this setting determine the
 order in which they run: items go through from lower valued to higher
 valued classes. It's customary to define these numbers in the 0-1000 range.
-

From 63b8caf5debf84e8da7f299782d15d0a41bf8a14 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 9 May 2017 11:58:53 -0300
Subject: [PATCH 1477/2974] Feed exports: rewrite indentation test without
 .strip()

---
 tests/test_feedexport.py | 41 ++++++++++++++--------------------------
 1 file changed, 14 insertions(+), 27 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c66c470a8..f55927121 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -466,28 +466,23 @@ class FeedExportTest(unittest.TestCase):
             {
                 'format': 'json',
                 'indent': -1,
-                'expected': b"""
-[
+                'expected': b"""[
 {"foo": ["bar"]},
 {"key": "value"}
-]
-""",
+]""",
             },
             {
                 'format': 'json',
                 'indent': 0,
-                'expected': b"""
-[
+                'expected': b"""[
 {"foo": ["bar"]},
 {"key": "value"}
-]
-""",
+]""",
             },
             {
                 'format': 'json',
                 'indent': 2,
-                'expected': b"""
-[
+                'expected': b"""[
 {
   "foo": [
     "bar"
@@ -501,8 +496,7 @@ class FeedExportTest(unittest.TestCase):
             {
                 'format': 'json',
                 'indent': 4,
-                'expected': b"""
-[
+                'expected': b"""[
 {
     "foo": [
         "bar"
@@ -516,8 +510,7 @@ class FeedExportTest(unittest.TestCase):
             {
                 'format': 'json',
                 'indent': 5,
-                'expected': b"""
-[
+                'expected': b"""[
 {
      "foo": [
           "bar"
@@ -533,15 +526,13 @@ class FeedExportTest(unittest.TestCase):
             {
                 'format': 'xml',
                 'indent': None,
-                'expected': b"""
-<?xml version="1.0" encoding="utf-8"?>
+                'expected': b"""<?xml version="1.0" encoding="utf-8"?>
 <items><item><foo><value>bar</value></foo></item><item><key>value</key></item></items>""",
             },
             {
                 'format': 'xml',
                 'indent': -1,
-                'expected': b"""
-<?xml version="1.0" encoding="utf-8"?>
+                'expected': b"""<?xml version="1.0" encoding="utf-8"?>
 <items>
 <item><foo><value>bar</value></foo></item>
 <item><key>value</key></item>
@@ -550,8 +541,7 @@ class FeedExportTest(unittest.TestCase):
             {
                 'format': 'xml',
                 'indent': 0,
-                'expected': b"""
-<?xml version="1.0" encoding="utf-8"?>
+                'expected': b"""<?xml version="1.0" encoding="utf-8"?>
 <items>
 <item><foo><value>bar</value></foo></item>
 <item><key>value</key></item>
@@ -560,8 +550,7 @@ class FeedExportTest(unittest.TestCase):
             {
                 'format': 'xml',
                 'indent': 2,
-                'expected': b"""
-<?xml version="1.0" encoding="utf-8"?>
+                'expected': b"""<?xml version="1.0" encoding="utf-8"?>
 <items>
   <item>
     <foo>
@@ -576,8 +565,7 @@ class FeedExportTest(unittest.TestCase):
             {
                 'format': 'xml',
                 'indent': 4,
-                'expected': b"""
-<?xml version="1.0" encoding="utf-8"?>
+                'expected': b"""<?xml version="1.0" encoding="utf-8"?>
 <items>
     <item>
         <foo>
@@ -592,8 +580,7 @@ class FeedExportTest(unittest.TestCase):
             {
                 'format': 'xml',
                 'indent': 5,
-                'expected': b"""
-<?xml version="1.0" encoding="utf-8"?>
+                'expected': b"""<?xml version="1.0" encoding="utf-8"?>
 <items>
      <item>
           <foo>
@@ -611,4 +598,4 @@ class FeedExportTest(unittest.TestCase):
             settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']}
             data = yield self.exported_data(items, settings)
             print(row['format'], row['indent'])
-            self.assertEqual(row['expected'].strip(), data)
+            self.assertEqual(row['expected'], data)

From 548a432951ef48f142d6091ecffd8e54eaab3fc4 Mon Sep 17 00:00:00 2001
From: Kurt Peek <kurt.peek@gmail.com>
Date: Fri, 5 May 2017 13:03:56 +0200
Subject: [PATCH 1478/2974] Minor grammatical changes

---
 docs/topics/logging.rst | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index ac3b614fc..a3281dd6b 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -27,7 +27,7 @@ Scrapy from scripts as described in :ref:`run-from-script`.
 Log levels
 ==========
 
-Python's builtin logging defines 5 different levels to indicate severity on a
+Python's builtin logging defines 5 different levels to indicate the severity of a
 given log message. Here are the standard ones, listed in decreasing order:
 
 1. ``logging.CRITICAL`` - for critical errors (highest severity)
@@ -47,20 +47,20 @@ level::
 
 There are shortcuts for issuing log messages on any of the standard 5 levels,
 and there's also a general ``logging.log`` method which takes a given level as
-argument.  If you need so, last example could be rewrote as::
+argument.  If needed, the last example could be rewritten as::
 
     import logging
     logging.log(logging.WARNING, "This is a warning")
 
-On top of that, you can create different "loggers" to encapsulate messages (For
-example, a common practice it's to create different loggers for every module).
+On top of that, you can create different "loggers" to encapsulate messages. (For
+example, a common practice is to create different loggers for every module).
 These loggers can be configured independently, and they allow hierarchical
 constructions.
 
-Last examples use the root logger behind the scenes, which is a top level
+The previous examples use the root logger behind the scenes, which is a top level
 logger where all messages are propagated to (unless otherwise specified). Using
 ``logging`` helpers is merely a shortcut for getting the root logger
-explicitly, so this is also an equivalent of last snippets::
+explicitly, so this is also an equivalent of the last snippets::
 
     import logging
     logger = logging.getLogger()
@@ -95,7 +95,7 @@ Logging from Spiders
 ====================
 
 Scrapy provides a :data:`~scrapy.spiders.Spider.logger` within each Spider
-instance, that can be accessed and used like this::
+instance, which can be accessed and used like this::
 
     import scrapy
 

From 25535dba9ca7e6f6f3c2279dd77240a21b1cc672 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 10 May 2017 16:45:15 -0300
Subject: [PATCH 1479/2974] Feed exports: edit note, fix typos

---
 docs/topics/exporters.rst    | 4 ++--
 docs/topics/feed-exports.rst | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 2ad77c905..b6139af92 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -222,8 +222,8 @@ BaseItemExporter
 
       * ``indent=None`` selects the most compact representation,
         all items in the same line with no indentation
-      * ``indent<=0`` each item on it's own line, no indentation
-      * ``indent>0`` each item on it's own line, indentated with the provided numeric value
+      * ``indent<=0`` each item on its own line, no indentation
+      * ``indent>0`` each item on its own line, indented with the provided numeric value
 
 .. highlight:: none
 
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index e57a4e776..d760b1a28 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -279,7 +279,7 @@ is a non-negative integer, then array elements and object members will be pretty
 with that indent level. An indent level of ``0``, or negative, will put each item on a new line.
 ``None`` selects the most compact representation
 
-Currently used by :class:`~scrapy.exporters.JsonItemExporter`
+Currently implemented only by :class:`~scrapy.exporters.JsonItemExporter`
 and :class:`~scrapy.exporters.XmlItemExporter`
 
 .. setting:: FEED_STORE_EMPTY

From 3a0a86ed31df1d22fea3b5b05e853f212adc40c8 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 12 May 2017 17:26:17 +0200
Subject: [PATCH 1480/2974] Clarify FEED_EXPORT_INDENT section

---
 docs/topics/feed-exports.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index d760b1a28..135d05c93 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -276,11 +276,12 @@ Default: ``0``
 
 Amount of spaces used to indent the output on each level. If ``FEED_EXPORT_INDENT``
 is a non-negative integer, then array elements and object members will be pretty-printed
-with that indent level. An indent level of ``0``, or negative, will put each item on a new line.
-``None`` selects the most compact representation
+with that indent level. An indent level of ``0`` (the default), or negative,
+will put each item on a new line. ``None`` selects the most compact representation.
 
 Currently implemented only by :class:`~scrapy.exporters.JsonItemExporter`
-and :class:`~scrapy.exporters.XmlItemExporter`
+and :class:`~scrapy.exporters.XmlItemExporter`, i.e. when you are exporting
+to ``.json`` or ``.xml``.
 
 .. setting:: FEED_STORE_EMPTY
 

From 26f723e4e63185b35b2f11285f50cfc3ac52b8fc Mon Sep 17 00:00:00 2001
From: Harrison Gregg <hcg28@cornell.edu>
Date: Sun, 30 Apr 2017 19:07:29 -0400
Subject: [PATCH 1481/2974] Allow formdata value to be None to drop field
 generated from response

---
 scrapy/http/request/form.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 905d8412f..d9d178a3e 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -135,7 +135,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
         if clickable and clickable[0] not in formdata and not clickable[0] is None:
             values.append(clickable)
 
-    values.extend(formdata.items())
+    values.extend((k, v) for k, v in formdata.items() if v is not None)
     return values
 
 

From 45a323024c7a7008df3e319a8a1437fae53826f9 Mon Sep 17 00:00:00 2001
From: Harrison Gregg <hcg28@cornell.edu>
Date: Sun, 30 Apr 2017 19:14:47 -0400
Subject: [PATCH 1482/2974] Add documentation for dropping fields in
 from_response request body

---
 docs/topics/request-response.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 67f8ec285..5410654ef 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -417,7 +417,9 @@ fields with form data from :class:`Response` objects.
 
        :param formdata: fields to override in the form data. If a field was
           already present in the response ``<form>`` element, its value is
-          overridden by the one passed in this parameter.
+          overridden by the one passed in this parameter. If a value passed in
+          this parameter is ``None``, the field will not be included in the
+          request, even if it was present in the response ``<form>`` element.
        :type formdata: dict
 
        :param clickdata: attributes to lookup the control clicked. If it's not

From ffef828a8deb86520e3bd6a50d76b2a4ecf3ae71 Mon Sep 17 00:00:00 2001
From: Harrison Gregg <hcg28@cornell.edu>
Date: Sun, 30 Apr 2017 19:33:51 -0400
Subject: [PATCH 1483/2974] Add test for dropping fields in from_response
 request body

---
 tests/test_http_request.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 7eadb874f..bbce537f4 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -425,6 +425,17 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs[b'one'], [b'1'])
         self.assertEqual(fs[b'two'], [b'2'])
 
+    def test_from_response_drop_params(self):
+        response = _buildresponse(
+            """<form action="get.php" method="POST">
+            <input type="hidden" name="one" value="1">
+            <input type="hidden" name="two" value="3">
+            </form>""")
+        req = self.request_class.from_response(response, formdata={'two': None})
+        fs = _qs(req)
+        self.assertEqual(fs[b'one'], [b'1'])
+        self.assertNotIn(b'two', fs)
+
     def test_from_response_override_method(self):
         response = _buildresponse(
                 '''<html><body>

From df7a5c4aa4f5898de3c70cef17c3c5031f7e05a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?F=C3=A1bio=20C=2E=20Barrionuevo=20da=20Luz?=
 <bnafta@gmail.com>
Date: Mon, 15 May 2017 22:52:23 -0300
Subject: [PATCH 1484/2974] Add support for executing scrapy using -m option of
 python

python -m scrapy
---
 scrapy/__main__.py | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 scrapy/__main__.py

diff --git a/scrapy/__main__.py b/scrapy/__main__.py
new file mode 100644
index 000000000..e467e057f
--- /dev/null
+++ b/scrapy/__main__.py
@@ -0,0 +1,4 @@
+from scrapy.cmdline import execute
+
+if __name__ == '__main__':
+    execute()

From b74b98fa3e7149c27bd3a541940457864a28d5d1 Mon Sep 17 00:00:00 2001
From: Eli Atzaba <eliat123@gmail.com>
Date: Tue, 16 May 2017 13:59:58 +0300
Subject: [PATCH 1485/2974] cleanup: removed unused MEMUSAGE_REPORT

Signed-off-by: Eli Atzaba <eliat123@gmail.com>
---
 docs/topics/extensions.rst          |  1 -
 docs/topics/settings.rst            | 13 -------------
 scrapy/extensions/memusage.py       |  1 -
 scrapy/settings/default_settings.py |  1 -
 4 files changed, 16 deletions(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 7f2952f4c..6036db0f5 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -220,7 +220,6 @@ can be configured with the following settings:
 * :setting:`MEMUSAGE_LIMIT_MB`
 * :setting:`MEMUSAGE_WARNING_MB`
 * :setting:`MEMUSAGE_NOTIFY_MAIL`
-* :setting:`MEMUSAGE_REPORT`
 * :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS`
 
 Memory debugger extension
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 9bf07588b..2cf6ffe75 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -959,19 +959,6 @@ Example::
 
 See :ref:`topics-extensions-ref-memusage`.
 
-.. setting:: MEMUSAGE_REPORT
-
-MEMUSAGE_REPORT
----------------
-
-Default: ``False``
-
-Scope: ``scrapy.extensions.memusage``
-
-Whether to send a memory usage report after each spider has been closed.
-
-See :ref:`topics-extensions-ref-memusage`.
-
 .. setting:: MEMUSAGE_WARNING_MB
 
 MEMUSAGE_WARNING_MB
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 322213cf0..c0570567e 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -35,7 +35,6 @@ class MemoryUsage(object):
         self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
         self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
         self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
-        self.report = crawler.settings.getbool('MEMUSAGE_REPORT')
         self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
         self.mail = MailSender.from_settings(crawler.settings)
         crawler.signals.connect(self.engine_started, signal=signals.engine_started)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 26ff4257e..03c36a0f9 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -219,7 +219,6 @@ MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
 MEMUSAGE_ENABLED = True
 MEMUSAGE_LIMIT_MB = 0
 MEMUSAGE_NOTIFY_MAIL = []
-MEMUSAGE_REPORT = False
 MEMUSAGE_WARNING_MB = 0
 
 METAREFRESH_ENABLED = True

From 1a452c038cc0547924051a7cd0786215ec7c2104 Mon Sep 17 00:00:00 2001
From: Bernardas <bernardas.alisauskas@gmail.com>
Date: Thu, 18 May 2017 16:57:13 +0000
Subject: [PATCH 1486/2974] increase ptpython priority since it can use other
 shells as backend

---
 scrapy/utils/console.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index a9d73aada..2e9981556 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -56,9 +56,9 @@ def _embed_standard_shell(namespace={}, banner=''):
     return wrapper
 
 DEFAULT_PYTHON_SHELLS = OrderedDict([
+    ('ptpython', _embed_ptpython_shell),
     ('ipython', _embed_ipython_shell),
     ('bpython', _embed_bpython_shell),
-    ('ptpython', _embed_ptpython_shell),
     ('python', _embed_standard_shell),
 ])
 

From 9ce03d096d7ed569071a1951d9e327d7294e1a83 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 19 May 2017 00:01:27 +0500
Subject: [PATCH 1487/2974] codecov config: disable project check, tweak PR
 comments

---
 codecov.yml | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 codecov.yml

diff --git a/codecov.yml b/codecov.yml
new file mode 100644
index 000000000..d8aa6b984
--- /dev/null
+++ b/codecov.yml
@@ -0,0 +1,6 @@
+comment:
+  layout: "header, diff, tree"
+
+coverage:
+  status:
+    project: false

From 851adcedf2220a783cdfb2d6b5872580e18b8d88 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 6 Mar 2017 23:36:21 +0100
Subject: [PATCH 1488/2974] List merged pull requests since 1.3.3

---
 docs/news.rst | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index da856d883..675a6d595 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,78 @@
 Release notes
 =============
 
+Scrapy 1.4.0 (2017-XX-XX)
+-------------------------
+
+New Features
+~~~~~~~~~~~~
+
+- Use credentials from request.meta['proxy'] #2530
+- [httpcompression] add support for br - brotli content encoding #2535
+- Enable memusage extension by default. #2539
+- response.follow #2540
+- add flags to request #2082
+- Support Anonymous FTP #2343
+- HttpErrorMiddleware stats #2566
+- Retry stats #2543
+- Set canonicalize=False for LinkExtractor #2537
+- Referrer policies in RefererMiddleware #2306
+- Fix referrer policy from response headers and support explicit empty string #2627
+- Data URI download handler. #2334
+- HttpCacheMiddleware: log cache directory at instantiation #2611
+- Add warning on duplicate spider name #2612
+- Allowed passing objects of Mapping class or its subclass to the CaselessDict initializer #2646
+- Allow redirections in media files downloads #2616
+- Travis CI: use portable pypy for Linux #2710
+
+
+Bug fixes
+~~~~~~~~~
+
+- LinkExtractors: strip whitespaces #2547
+- FormRequest: handle whitespaces in action attribute properly #2548
+- Buffer CONNECT response bytes from proxy until all HTTP headers are received #2495
+- Fix FTP downloader and re-enable FTP tests on Python 3 #2599
+- Handle data loss gracefully. #2590
+- Use body to choose response type after decompression content #2393
+- Always decompress Content-Encoding: gzip at HttpCompression stage #2391
+- Respect custom log level (#2581, fixes #1612)
+- [logformatter] 'flags' format spec backward compatibility #2649
+- 'make htmlview' does not open the webbrowser #2661
+- Remove "commands" from the command list  #2695
+
+Cleanups
+~~~~~~~~
+
+- TST remove temp files and folders #2570
+- TST fixed ProjectUtilsTest on OS X #2569
+- Separate building request from _requests_to_follow in CrawlSpider #2562
+- remove “Python 3 progress” badge #2567
+- add a couple more lines to gitignore #2557
+- deprecate Spider.make_requests_from_url. #1728
+- Remove bumpversion prerelease configuration #2159
+- Set context factory implementation based on Twisted version #2577
+- Add omitted "self" arguments #2595
+- Remove redundant slot.add_request() call in ExecutionEngine #2617
+- Removed contrib section in contribution documentation #2636
+- More specific exception catching: os.path.getmtime can only raise os.error in FSFilesStore #2644
+
+Documentation
+~~~~~~~~~~~~~
+
+- Doc: binary mode is required for exporters #2564
+- document issue with FormRequest.from_response due to bug in lxml #2572
+- Use single quotes uniformly #2596
+- Document ftp_user and ftp_password meta keys #2587
+- Update release notes for 1.0.7, 1.1.4 and 1.2.3 #2625
+- DOC Mention brotli support in HttpCompressionMiddleware section #2628
+- Removed contrib section in contribution documentation #2636
+- docs: installation instructions, mention conda in the beginning (closes #2475) #2477
+- FAQ Rewrite note on Python 3 support on Windows #2690
+- DOC Rearrange selector sections #2705
+- Remove __nonzero__ from SelectorList docs #2683
+
+
 Scrapy 1.3.3 (2017-03-10)
 -------------------------
 
@@ -15,6 +87,7 @@ Bug fixes
   A new setting is introduced to toggle between warning or exception if needed ;
   see :setting:`SPIDER_LOADER_WARN_ONLY` for details.
 
+
 Scrapy 1.3.2 (2017-02-13)
 -------------------------
 

From 8729a91f7a136decb66b00ada858a67c41796e11 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 19 Apr 2017 17:22:37 +0200
Subject: [PATCH 1489/2974] Rephrase "New features" section

---
 docs/news.rst                    | 50 +++++++++++++++++++++-----------
 docs/topics/media-pipeline.rst   |  5 ++--
 docs/topics/request-response.rst |  2 +-
 3 files changed, 36 insertions(+), 21 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 675a6d595..979e9a953 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -9,24 +9,40 @@ Scrapy 1.4.0 (2017-XX-XX)
 New Features
 ~~~~~~~~~~~~
 
-- Use credentials from request.meta['proxy'] #2530
-- [httpcompression] add support for br - brotli content encoding #2535
-- Enable memusage extension by default. #2539
-- response.follow #2540
-- add flags to request #2082
-- Support Anonymous FTP #2343
-- HttpErrorMiddleware stats #2566
-- Retry stats #2543
-- Set canonicalize=False for LinkExtractor #2537
-- Referrer policies in RefererMiddleware #2306
-- Fix referrer policy from response headers and support explicit empty string #2627
-- Data URI download handler. #2334
-- HttpCacheMiddleware: log cache directory at instantiation #2611
-- Add warning on duplicate spider name #2612
-- Allowed passing objects of Mapping class or its subclass to the CaselessDict initializer #2646
-- Allow redirections in media files downloads #2616
-- Travis CI: use portable pypy for Linux #2710
+- Accept proxy credentials in request.meta['proxy'] (:issue:`2526`)
+- Support `brotli`_-compressed content; requires optional `brotlipy`_
+  (:issue:`2535`)
+- Enable memusage extension by default (:issue:`2187`) ;
+  **this is technically backwards-incompatible** so please check if you have
+  any non-default ``MEMUSAGE_***`` settings set.
+- New :meth:`Response.follow <scrapy.http.Response.follow>` shortcur
+  for creating requests (:issue:`1940`)
+- Added ``flags`` argument and attribute to :class:`Request <scrapy.http.Request>`
+  (:issue:`2047`)
+- Support Anonymous FTP (:issue:`2342`)
+- Added ``retry/count``, ``retry/max_reached`` and ``retry/reason_count/***``
+  stats to :class:`RetryMiddleware <scrapy.downloadermiddlewares.retry.RetryMiddleware>`
+  (:issue:`2543`)
+- Added ``httperror/response_ignored_count`` and ``httperror/response_ignored_status_count/***``
+  stats to :class:`HttpErrorMiddleware <scrapy.spidermiddlewares.httperror.HttpErrorMiddleware>`
+  (:issue:`2566`)
+- Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor`
+  (:issue:`2537`, fixes :issue:`1941` and :issue:`1982`):
+  **warning, this istechnically backwards-incompatible**
+- Customizable :setting:`Referrer policy <REFERRER_POLICY>` in
+  :class:`RefererMiddleware <scrapy.spidermiddlewares.referer.RefererMiddleware>`
+  (:issue:`2306`)
+- New ``data:`` URI download handler (:issue:`2334`, fixes :issue:`2156`)
+- Log cache directory when HTTP Cache is used (:issue:`2611`, fixes :issue:`2604`)
+- Warn users when project contains duplicate spider names (fixes :issue:`2181`)
+- :class:`CaselessDict` now accepts ``Mapping`` instances and not only dicts (:issue:`2646`)
+- :ref:`Media downloads <topics-media-pipeline>`, with :class:`FilesPipelines`
+  or :class:`ImagesPipelines`, can now optionally handle HTTP redirects
+  using the new :setting:`MEDIA_ALLOW_REDIRECTS` (:issue:`2616`, fixes :issue:`2004`)
+- Use portable pypy for Linux on Travis CI (:issue:`2710`)
 
+.. _brotli: https://github.com/google/brotli
+.. _brotlipy: https://github.com/python-hyper/brotlipy/
 
 Bug fixes
 ~~~~~~~~~
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index f258ff748..e948913a4 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -320,8 +320,6 @@ all be dropped because at least one dimension is shorter than the constraint.
 
 By default, there are no size constraints, so all images are processed.
 
-.. _topics-media-pipeline-override:
-
 Allowing redirections
 ---------------------
 
@@ -330,10 +328,11 @@ Allowing redirections
 By default media pipelines ignore redirects, i.e. an HTTP redirection
 to a media file URL request will mean the media download is considered failed.
 
-To handle media redirections, set this settings to ``True``:
+To handle media redirections, set this setting to ``True``::
 
     MEDIA_ALLOW_REDIRECTS = True
 
+.. _topics-media-pipeline-override:
 
 Extending the Media Pipelines
 =============================
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index f1552572a..6ca37b7c9 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
 Request objects
 ===============
 
-.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback])
+.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags])
 
     A :class:`Request` object represents an HTTP request, which is usually
     generated in the Spider and executed by the Downloader, and thus generating

From cba55cd190c573a03b8458cda7ba2a20da5651e6 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 21 Apr 2017 16:52:32 +0200
Subject: [PATCH 1490/2974] Rephrase other sections

---
 docs/news.rst | 96 ++++++++++++++++++++++++++++-----------------------
 1 file changed, 52 insertions(+), 44 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 979e9a953..a19d7a93a 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -9,26 +9,26 @@ Scrapy 1.4.0 (2017-XX-XX)
 New Features
 ~~~~~~~~~~~~
 
-- Accept proxy credentials in request.meta['proxy'] (:issue:`2526`)
+- Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`)
 - Support `brotli`_-compressed content; requires optional `brotlipy`_
   (:issue:`2535`)
 - Enable memusage extension by default (:issue:`2187`) ;
   **this is technically backwards-incompatible** so please check if you have
   any non-default ``MEMUSAGE_***`` settings set.
-- New :meth:`Response.follow <scrapy.http.Response.follow>` shortcur
+- New :meth:`Response.follow <scrapy.http.Response.follow>` shortcut
   for creating requests (:issue:`1940`)
 - Added ``flags`` argument and attribute to :class:`Request <scrapy.http.Request>`
-  (:issue:`2047`)
+  objects (:issue:`2047`)
 - Support Anonymous FTP (:issue:`2342`)
-- Added ``retry/count``, ``retry/max_reached`` and ``retry/reason_count/***``
+- Added ``retry/count``, ``retry/max_reached`` and ``retry/reason_count/<reason>``
   stats to :class:`RetryMiddleware <scrapy.downloadermiddlewares.retry.RetryMiddleware>`
   (:issue:`2543`)
-- Added ``httperror/response_ignored_count`` and ``httperror/response_ignored_status_count/***``
+- Added ``httperror/response_ignored_count`` and ``httperror/response_ignored_status_count/<status>``
   stats to :class:`HttpErrorMiddleware <scrapy.spidermiddlewares.httperror.HttpErrorMiddleware>`
   (:issue:`2566`)
 - Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor`
   (:issue:`2537`, fixes :issue:`1941` and :issue:`1982`):
-  **warning, this istechnically backwards-incompatible**
+  **warning, this is technically backwards-incompatible**
 - Customizable :setting:`Referrer policy <REFERRER_POLICY>` in
   :class:`RefererMiddleware <scrapy.spidermiddlewares.referer.RefererMiddleware>`
   (:issue:`2306`)
@@ -38,8 +38,9 @@ New Features
 - :class:`CaselessDict` now accepts ``Mapping`` instances and not only dicts (:issue:`2646`)
 - :ref:`Media downloads <topics-media-pipeline>`, with :class:`FilesPipelines`
   or :class:`ImagesPipelines`, can now optionally handle HTTP redirects
-  using the new :setting:`MEDIA_ALLOW_REDIRECTS` (:issue:`2616`, fixes :issue:`2004`)
-- Use portable pypy for Linux on Travis CI (:issue:`2710`)
+  using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting (:issue:`2616`, fixes :issue:`2004`)
+- Accept non-complete responses from websites using a new
+  :setting:`DOWNLOAD_FAIL_ON_DATALOSS` setting (:issue:`2590`, fixes :issue:`2586`)
 
 .. _brotli: https://github.com/google/brotli
 .. _brotlipy: https://github.com/python-hyper/brotlipy/
@@ -47,48 +48,55 @@ New Features
 Bug fixes
 ~~~~~~~~~
 
-- LinkExtractors: strip whitespaces #2547
-- FormRequest: handle whitespaces in action attribute properly #2548
-- Buffer CONNECT response bytes from proxy until all HTTP headers are received #2495
-- Fix FTP downloader and re-enable FTP tests on Python 3 #2599
-- Handle data loss gracefully. #2590
-- Use body to choose response type after decompression content #2393
-- Always decompress Content-Encoding: gzip at HttpCompression stage #2391
-- Respect custom log level (#2581, fixes #1612)
-- [logformatter] 'flags' format spec backward compatibility #2649
-- 'make htmlview' does not open the webbrowser #2661
-- Remove "commands" from the command list  #2695
+- LinkExtractor now strips leading and trailing whitespaces from attributes
+  (:issue:`2547`, fixes :issue:`1614`)
+- Properly handle whitespaces in action attribute in :class:`FormRequest`
+  (:issue:`2548`)
+- Buffer CONNECT response bytes from proxy until all HTTP headers are received
+  (:issue:`2495`, fixes :issue:`2491`)
+- FTP downloader now works on Python 3, provided you use Twisted>=17.1
+  (:issue:`2599`)
+- Use body to choose response type after decompressing content (:issue:`2393`,
+  fixes :issue:`2145`)
+- Always decompress ``Content-Encoding: gzip`` at :class:`HttpCompressionMiddleware
+  <scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware>` stage (:issue:`2391`)
+- Respect custom log level in ``Spider.custom_settings`` (:issue:`2581`,
+  fixes :issue:`1612`)
+- 'make htmlview' fix for macOS (:issue:`2661`)
+- Remove "commands" from the command list  (:issue:`2695`)
 
-Cleanups
-~~~~~~~~
+Cleanups & Refactoring
+~~~~~~~~~~~~~~~~~~~~~~
 
-- TST remove temp files and folders #2570
-- TST fixed ProjectUtilsTest on OS X #2569
-- Separate building request from _requests_to_follow in CrawlSpider #2562
-- remove “Python 3 progress” badge #2567
-- add a couple more lines to gitignore #2557
-- deprecate Spider.make_requests_from_url. #1728
-- Remove bumpversion prerelease configuration #2159
-- Set context factory implementation based on Twisted version #2577
-- Add omitted "self" arguments #2595
-- Remove redundant slot.add_request() call in ExecutionEngine #2617
-- Removed contrib section in contribution documentation #2636
-- More specific exception catching: os.path.getmtime can only raise os.error in FSFilesStore #2644
+- Tests: remove temp files and folders (:issue:`2570`),
+  fixed ProjectUtilsTest on OS X (:issue:`2569`),
+  use portable pypy for Linux on Travis CI (:issue:`2710`)
+
+- Separate building request from ``_requests_to_follow`` in CrawlSpider (:issue:`2562`)
+- Remove “Python 3 progress” badge (:issue:`2567`)
+- Add a couple more lines to ``.gitignore`` (:issue:`2557`)
+- Deprecate ``Spider.make_requests_from_url`` (:issue:`1728`)
+- Remove bumpversion prerelease configuration (:issue:`2159`)
+- Set context factory implementation based on Twisted version (:issue:`2577`,
+  fixes :issue:`2560`)
+- Add omitted ``self`` arguments in default project middleware template (:issue:`2595`)
+- Remove redundant ``slot.add_request()`` call in ExecutionEngine (:issue:`2617`)
+- Catch more specific ``os.error`` exception in :class:`FSFilesStore` (:issue:`2644`)
 
 Documentation
 ~~~~~~~~~~~~~
 
-- Doc: binary mode is required for exporters #2564
-- document issue with FormRequest.from_response due to bug in lxml #2572
-- Use single quotes uniformly #2596
-- Document ftp_user and ftp_password meta keys #2587
-- Update release notes for 1.0.7, 1.1.4 and 1.2.3 #2625
-- DOC Mention brotli support in HttpCompressionMiddleware section #2628
-- Removed contrib section in contribution documentation #2636
-- docs: installation instructions, mention conda in the beginning (closes #2475) #2477
-- FAQ Rewrite note on Python 3 support on Windows #2690
-- DOC Rearrange selector sections #2705
-- Remove __nonzero__ from SelectorList docs #2683
+- Binary mode is required for exporters (:issue:`2564`, fixes :issue:`2553`)
+- Mention issue with :meth:`FormRequest.from_response
+  <scrapy.http.FormRequest.from_response>` due to bug in lxml (:issue:`2572`)
+- Use single quotes uniformly in templates (:issue:`2596`)
+- Document :reqmeta:`ftp_user` and :reqmeta:`ftp_password` meta keys (:issue:`2587`)
+- Removed section on deprecated ``contrib/`` (:issue:`2636`)
+- Recommend Anaconda when installing Scrapy on Windows
+  (:issue:`2477`, fixes :issue:`2475`)
+- FAQ: rewrite note on Python 3 support on Windows (:issue:`2690`)
+- Rearrange selector sections (:issue:`2705`)
+- Remove ``__nonzero__`` from :class:`SelectorList` docs (:issue:`2683`)
 
 
 Scrapy 1.3.3 (2017-03-10)

From e139d990fca1e1e2144992bb2e05665d17c3f989 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 21 Apr 2017 16:53:20 +0200
Subject: [PATCH 1491/2974] Fix sphinx-build warning on deprecated
 latex_paper_size

---
 docs/Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/Makefile b/docs/Makefile
index a3d1611f9..187f03c4c 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -10,7 +10,8 @@ PAPER        =
 SOURCES      =
 SHELL        = /bin/bash
 
-ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \
+ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \
+                -D latex_elements.papersize=$(PAPER) \
                 $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES)
 
 .PHONY: help update build html htmlhelp clean

From c6464cc4f5a152c2d23a79457b7e01ee90b18069 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 9 May 2017 20:16:41 +0200
Subject: [PATCH 1492/2974] Add verbose introduction to new features

---
 docs/news.rst | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index a19d7a93a..f225fef1d 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -6,6 +6,52 @@ Release notes
 Scrapy 1.4.0 (2017-XX-XX)
 -------------------------
 
+Scrapy 1.4 does not bring that many breathtaking new features
+but quite a few handy improvements nonetheless.
+
+Scrapy now supports anonymous FTP sessions with customizable user and
+password via the new :setting:`FTP_USER` and :setting:`FTP_PASSWORD` settings.
+**And if you're using Twisted version 17.1.0 or above, FTP is now available
+with Python 3.**
+
+Link extractors now work similarly to what a regular modern browser would
+do. Especially, leading and trailing whitespace are removed from attributes
+(think ``href="   http://example.com"``) when building ``Link`` objects.
+This whitespace-stripping also happens for ``action`` attributes with
+``FormRequest``.
+**Please also note that link extractors do not canonicalize URLs by default
+anymore.** This was puzzling users every now and then, and it's not what
+browsers do in fact, so we removed that extra transformation on extractred
+links.
+
+There's a new ``response.follow()`` shortcut for creating URLs directly
+from the response instance in callbacks.
+For example, instead of::
+
+    scrapy.Request(response.urljoin(somehrefvalue))
+
+you can now use the simpler::
+
+    response.follow(somehrefvalue)
+
+
+For those of you wanting more control on the ``Referer:`` header that Scrapy
+sends when following links, you can set your own ``Referrer Policy``.
+Prior to Scrapy 1.4, the default ``RefererMiddleware`` would simply and
+blindly set it to the URL of the response that generated the HTTP request
+(which could leak information on your URL seeds).
+By default, Scrapy now behaves much like your regular browser does.
+And this policy is fully customizable with W3C standard values
+(or with something really custom of your own if you wish).
+See :setting:`REFERRER_POLICY` for details.
+
+Last but not least, Scrapy now has the option to make JSON and XML items
+more human-readable, with newlines between items and even custom indenting
+offset, using the new :setting:`FEED_EXPORT_INDENT` setting.
+
+Enjoy! (Or read on for the rest of changes in this release.)
+
+
 New Features
 ~~~~~~~~~~~~
 

From 7d72394794bea63fdaa08d1264a50b0b83998424 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 10 May 2017 18:55:09 +0200
Subject: [PATCH 1493/2974] Reword mention of new response.follow() shortcut

---
 docs/news.rst | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index f225fef1d..27baff754 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -24,8 +24,8 @@ anymore.** This was puzzling users every now and then, and it's not what
 browsers do in fact, so we removed that extra transformation on extractred
 links.
 
-There's a new ``response.follow()`` shortcut for creating URLs directly
-from the response instance in callbacks.
+There's a new ``response.follow()`` shortcut for creating requests directly
+from a response instance and a relative URL.
 For example, instead of::
 
     scrapy.Request(response.urljoin(somehrefvalue))
@@ -34,7 +34,6 @@ you can now use the simpler::
 
     response.follow(somehrefvalue)
 
-
 For those of you wanting more control on the ``Referer:`` header that Scrapy
 sends when following links, you can set your own ``Referrer Policy``.
 Prior to Scrapy 1.4, the default ``RefererMiddleware`` would simply and

From 55d10823603e8033a4c4c441b8e78da217a951bf Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 11 May 2017 14:25:11 +0200
Subject: [PATCH 1494/2974] Reference recent fixes and commits

---
 docs/news.rst | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 27baff754..64995dd10 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -109,6 +109,8 @@ Bug fixes
   fixes :issue:`1612`)
 - 'make htmlview' fix for macOS (:issue:`2661`)
 - Remove "commands" from the command list  (:issue:`2695`)
+- Fix duplicate Content-Length header for POST requests with empty body (:issue:`2677`)
+- Properly cancel large downloads, i.e. above :setting:`DOWNLOAD_MAXSIZE` (:issue:`1616`)
 
 Cleanups & Refactoring
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -116,7 +118,6 @@ Cleanups & Refactoring
 - Tests: remove temp files and folders (:issue:`2570`),
   fixed ProjectUtilsTest on OS X (:issue:`2569`),
   use portable pypy for Linux on Travis CI (:issue:`2710`)
-
 - Separate building request from ``_requests_to_follow`` in CrawlSpider (:issue:`2562`)
 - Remove “Python 3 progress” badge (:issue:`2567`)
 - Add a couple more lines to ``.gitignore`` (:issue:`2557`)
@@ -127,6 +128,7 @@ Cleanups & Refactoring
 - Add omitted ``self`` arguments in default project middleware template (:issue:`2595`)
 - Remove redundant ``slot.add_request()`` call in ExecutionEngine (:issue:`2617`)
 - Catch more specific ``os.error`` exception in :class:`FSFilesStore` (:issue:`2644`)
+- Change "localhost" test server certificate (:issue:`2720`)
 
 Documentation
 ~~~~~~~~~~~~~
@@ -142,6 +144,10 @@ Documentation
 - FAQ: rewrite note on Python 3 support on Windows (:issue:`2690`)
 - Rearrange selector sections (:issue:`2705`)
 - Remove ``__nonzero__`` from :class:`SelectorList` docs (:issue:`2683`)
+- Mention how to disable request filtering in documentation of
+  :setting:`DUPEFILTER_CLASS` setting (:issue:`2714`)
+- Add sphinx_rtd_theme to docs setup readme (:issue:`2668`)
+- Open file in text mode in JSON item writer example (:issue:`2729`)
 
 
 Scrapy 1.3.3 (2017-03-10)

From 896c30a8eb1ea5d03898b27f238243ea629a3e42 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Fri, 12 May 2017 19:39:31 +0200
Subject: [PATCH 1495/2974] Reference items pretty-printing issue number

---
 docs/news.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 64995dd10..c249c038a 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -86,6 +86,8 @@ New Features
   using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting (:issue:`2616`, fixes :issue:`2004`)
 - Accept non-complete responses from websites using a new
   :setting:`DOWNLOAD_FAIL_ON_DATALOSS` setting (:issue:`2590`, fixes :issue:`2586`)
+- Optional pretty-printing of JSON and XML items via
+  :setting:`FEED_EXPORT_INDENT` setting (:issue:`2456`, fixes :issue:`1327`)
 
 .. _brotli: https://github.com/google/brotli
 .. _brotlipy: https://github.com/python-hyper/brotlipy/

From 432668acf7b8236229fca3b7726139acdc0c05f2 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 16 May 2017 16:00:26 +0200
Subject: [PATCH 1496/2974] Mention implementation of #667

---
 docs/news.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index c249c038a..2bd6640b3 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -88,6 +88,8 @@ New Features
   :setting:`DOWNLOAD_FAIL_ON_DATALOSS` setting (:issue:`2590`, fixes :issue:`2586`)
 - Optional pretty-printing of JSON and XML items via
   :setting:`FEED_EXPORT_INDENT` setting (:issue:`2456`, fixes :issue:`1327`)
+- Allow dropping fields in ``FormRequest.from_response`` formdata when
+  ``None`` value is passed (:issue:`667`)
 
 .. _brotli: https://github.com/google/brotli
 .. _brotlipy: https://github.com/python-hyper/brotlipy/

From a3d3cd4cb7d256a0b876019279ec88c47d9957ff Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 17 May 2017 19:52:18 +0200
Subject: [PATCH 1497/2974] Update with latest merges

---
 docs/news.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 2bd6640b3..7debe176b 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -90,6 +90,8 @@ New Features
   :setting:`FEED_EXPORT_INDENT` setting (:issue:`2456`, fixes :issue:`1327`)
 - Allow dropping fields in ``FormRequest.from_response`` formdata when
   ``None`` value is passed (:issue:`667`)
+- Per-request retry times with the new :reqmeta:`max_retry_times` meta key
+  (:issue:`2642`)
 
 .. _brotli: https://github.com/google/brotli
 .. _brotlipy: https://github.com/python-hyper/brotlipy/
@@ -133,6 +135,7 @@ Cleanups & Refactoring
 - Remove redundant ``slot.add_request()`` call in ExecutionEngine (:issue:`2617`)
 - Catch more specific ``os.error`` exception in :class:`FSFilesStore` (:issue:`2644`)
 - Change "localhost" test server certificate (:issue:`2720`)
+- Remove unused ``MEMUSAGE_REPORT`` setting (:issue:`2576`)
 
 Documentation
 ~~~~~~~~~~~~~

From edcde7a2cf1b7abbc0b27d51d83f40fe6fe8f5fa Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 18 May 2017 23:11:34 +0500
Subject: [PATCH 1498/2974] DOC tweak release notes: promote response.follow,
 mention logging/stats changes

---
 docs/news.rst | 53 ++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 35 insertions(+), 18 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 7debe176b..b5e31e444 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -11,29 +11,42 @@ but quite a few handy improvements nonetheless.
 
 Scrapy now supports anonymous FTP sessions with customizable user and
 password via the new :setting:`FTP_USER` and :setting:`FTP_PASSWORD` settings.
-**And if you're using Twisted version 17.1.0 or above, FTP is now available
-with Python 3.**
+And if you're using Twisted version 17.1.0 or above, FTP is now available
+with Python 3. 
+
+There's a new :meth:`response.follow <scrapy.http.TextResponse.follow>` method
+for creating requests; **it is now a recommended way to create Requests
+in Scrapy spiders**. This method makes it easier to write correct
+spiders; ``response.follow`` has several advantages over creating
+``scrapy.Request`` objects directly:
+
+* it handles relative URLs;
+* it works properly with non-ascii URLs on non-UTF8 pages;
+* in addition to absolute and relative URLs it supports Selectors;
+  for ``<a>`` elements it can also extract their href values.
+
+For example, instead of this::
+
+    for href in response.css('li.page a::attr(href)').extract():
+        url = response.urljoin(href)
+        yield scrapy.Request(url, self.parse, encoding=response.encoding)
+
+One can now write this::
+
+    for a in response.css('li.page a'):
+        yield response.follow(a, self.parse)
+
+Link extractors are also improved. They work similarly to what a regular
+modern browser would do: leading and trailing whitespace are removed
+from attributes (think ``href="   http://example.com"``) when building
+``Link`` objects. This whitespace-stripping also happens for ``action``
+attributes with ``FormRequest``.
 
-Link extractors now work similarly to what a regular modern browser would
-do. Especially, leading and trailing whitespace are removed from attributes
-(think ``href="   http://example.com"``) when building ``Link`` objects.
-This whitespace-stripping also happens for ``action`` attributes with
-``FormRequest``.
 **Please also note that link extractors do not canonicalize URLs by default
 anymore.** This was puzzling users every now and then, and it's not what
 browsers do in fact, so we removed that extra transformation on extractred
 links.
 
-There's a new ``response.follow()`` shortcut for creating requests directly
-from a response instance and a relative URL.
-For example, instead of::
-
-    scrapy.Request(response.urljoin(somehrefvalue))
-
-you can now use the simpler::
-
-    response.follow(somehrefvalue)
-
 For those of you wanting more control on the ``Referer:`` header that Scrapy
 sends when following links, you can set your own ``Referrer Policy``.
 Prior to Scrapy 1.4, the default ``RefererMiddleware`` would simply and
@@ -44,6 +57,10 @@ And this policy is fully customizable with W3C standard values
 (or with something really custom of your own if you wish).
 See :setting:`REFERRER_POLICY` for details.
 
+To make Scrapy spiders easier to debug, Scrapy logs more stats by default
+in 1.4: memory usage stats, detailed retry stats, detailed HTTP error code
+stats. A similar change is that HTTP cache path is also visible in logs now.
+
 Last but not least, Scrapy now has the option to make JSON and XML items
 more human-readable, with newlines between items and even custom indenting
 offset, using the new :setting:`FEED_EXPORT_INDENT` setting.
@@ -60,7 +77,7 @@ New Features
 - Enable memusage extension by default (:issue:`2187`) ;
   **this is technically backwards-incompatible** so please check if you have
   any non-default ``MEMUSAGE_***`` settings set.
-- New :meth:`Response.follow <scrapy.http.Response.follow>` shortcut
+- New :ref:`response.follow <response-follow-example>` shortcut
   for creating requests (:issue:`1940`)
 - Added ``flags`` argument and attribute to :class:`Request <scrapy.http.Request>`
   objects (:issue:`2047`)

From 76e5b0f65c256373411c8fa6e005a85c48cad6ee Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 19 May 2017 01:13:32 +0500
Subject: [PATCH 1499/2974] DOC 1.4 deprecations and backwards incompatible
 changes, add recent commits to news.

---
 docs/news.rst | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index b5e31e444..3c641af44 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -67,6 +67,21 @@ offset, using the new :setting:`FEED_EXPORT_INDENT` setting.
 
 Enjoy! (Or read on for the rest of changes in this release.)
 
+Deprecations and Backwards Incompatible Changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor`
+  (:issue:`2537`, fixes :issue:`1941` and :issue:`1982`):
+  **warning, this is technically backwards-incompatible**
+- Enable memusage extension by default (:issue:`2539`, fixes :issue:`2187`);
+  **this is technically backwards-incompatible** so please check if you have
+  any non-default ``MEMUSAGE_***`` options set.
+- ``EDITOR`` environment variable now takes precedence over ``EDITOR``
+  option defined in settings.py (:issue:`1829`); Scrapy default settings
+  no longer depend on environment variables. **This is technically a backwards
+  incompatible change**.
+- ``Spider.make_requests_from_url`` is deprecated
+  (:issue:`1728`, fixes :issue:`1495`).
 
 New Features
 ~~~~~~~~~~~~
@@ -74,9 +89,6 @@ New Features
 - Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`)
 - Support `brotli`_-compressed content; requires optional `brotlipy`_
   (:issue:`2535`)
-- Enable memusage extension by default (:issue:`2187`) ;
-  **this is technically backwards-incompatible** so please check if you have
-  any non-default ``MEMUSAGE_***`` settings set.
 - New :ref:`response.follow <response-follow-example>` shortcut
   for creating requests (:issue:`1940`)
 - Added ``flags`` argument and attribute to :class:`Request <scrapy.http.Request>`
@@ -88,9 +100,6 @@ New Features
 - Added ``httperror/response_ignored_count`` and ``httperror/response_ignored_status_count/<status>``
   stats to :class:`HttpErrorMiddleware <scrapy.spidermiddlewares.httperror.HttpErrorMiddleware>`
   (:issue:`2566`)
-- Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor`
-  (:issue:`2537`, fixes :issue:`1941` and :issue:`1982`):
-  **warning, this is technically backwards-incompatible**
 - Customizable :setting:`Referrer policy <REFERRER_POLICY>` in
   :class:`RefererMiddleware <scrapy.spidermiddlewares.referer.RefererMiddleware>`
   (:issue:`2306`)
@@ -109,6 +118,8 @@ New Features
   ``None`` value is passed (:issue:`667`)
 - Per-request retry times with the new :reqmeta:`max_retry_times` meta key
   (:issue:`2642`)
+- ``python -m scrapy`` as a more explicit alternative to ``scrapy`` command
+  (:issue:`2740`)
 
 .. _brotli: https://github.com/google/brotli
 .. _brotlipy: https://github.com/python-hyper/brotlipy/
@@ -134,6 +145,8 @@ Bug fixes
 - Remove "commands" from the command list  (:issue:`2695`)
 - Fix duplicate Content-Length header for POST requests with empty body (:issue:`2677`)
 - Properly cancel large downloads, i.e. above :setting:`DOWNLOAD_MAXSIZE` (:issue:`1616`)
+- ImagesPipeline: fixed processing of transparent PNG images with palette
+  (:issue:`2675`)
 
 Cleanups & Refactoring
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -144,8 +157,8 @@ Cleanups & Refactoring
 - Separate building request from ``_requests_to_follow`` in CrawlSpider (:issue:`2562`)
 - Remove “Python 3 progress” badge (:issue:`2567`)
 - Add a couple more lines to ``.gitignore`` (:issue:`2557`)
-- Deprecate ``Spider.make_requests_from_url`` (:issue:`1728`)
 - Remove bumpversion prerelease configuration (:issue:`2159`)
+- Add codecov.yml file (:issue:`2750`)
 - Set context factory implementation based on Twisted version (:issue:`2577`,
   fixes :issue:`2560`)
 - Add omitted ``self`` arguments in default project middleware template (:issue:`2595`)
@@ -172,6 +185,7 @@ Documentation
   :setting:`DUPEFILTER_CLASS` setting (:issue:`2714`)
 - Add sphinx_rtd_theme to docs setup readme (:issue:`2668`)
 - Open file in text mode in JSON item writer example (:issue:`2729`)
+- Clarify ``allowed_domains`` example (:issue:`2670`)
 
 
 Scrapy 1.3.3 (2017-03-10)

From fc2846d637ca1ebc31e574284d9bc2537969bbe0 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 18 May 2017 22:59:46 +0200
Subject: [PATCH 1500/2974] Set release date for v1.4.0

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 3c641af44..e0f8eee0b 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,7 +3,7 @@
 Release notes
 =============
 
-Scrapy 1.4.0 (2017-XX-XX)
+Scrapy 1.4.0 (2017-05-18)
 -------------------------
 
 Scrapy 1.4 does not bring that many breathtaking new features

From 5f69ec98f70e1e1e5f65fb36eb1cfb23d0be5b45 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 18 May 2017 23:01:05 +0200
Subject: [PATCH 1501/2974] =?UTF-8?q?Bump=20version:=201.3.2=20=E2=86=92?=
 =?UTF-8?q?=201.4.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 3 ++-
 scrapy/VERSION   | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 36484c49f..21800f616 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,7 +1,8 @@
 [bumpversion]
-current_version = 1.3.2
+current_version = 1.4.0
 commit = True
 tag = True
 tag_name = {new_version}
 
 [bumpversion:file:scrapy/VERSION]
+
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 1892b9267..88c5fb891 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.3.2
+1.4.0

From af2963d0eb79b1241e04b9fc7972bc4a31adf67c Mon Sep 17 00:00:00 2001
From: Kurt Peek <kurt.peek@gmail.com>
Date: Wed, 24 May 2017 15:50:47 +0200
Subject: [PATCH 1502/2974] Update autothrottle.rst

Added missing bullet point for the AUTOTHROTTLE_TARGET_CONCURRENCY setting.
---
 docs/topics/autothrottle.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst
index b83946a58..c9bece753 100644
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@@ -88,6 +88,7 @@ The settings used to control the AutoThrottle extension are:
 * :setting:`AUTOTHROTTLE_ENABLED`
 * :setting:`AUTOTHROTTLE_START_DELAY`
 * :setting:`AUTOTHROTTLE_MAX_DELAY`
+* :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY`
 * :setting:`AUTOTHROTTLE_DEBUG`
 * :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
 * :setting:`CONCURRENT_REQUESTS_PER_IP`

From 80b160d0d739bc3667adf65817a43111f14b9599 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Mon, 29 May 2017 14:56:49 -0300
Subject: [PATCH 1503/2974] include references to scrapy subreddit in the docs

---
 docs/contributing.rst | 13 +++++++------
 docs/index.rst        | 10 +++++-----
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index ab3779395..c969bd842 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -22,7 +22,7 @@ There are many ways to contribute to Scrapy. Here are some of them:
   `Writing patches`_ and `Submitting patches`_ below for details on how to
   write and submit a patch.
 
-* Join the `scrapy-users`_ mailing list and share your ideas on how to
+* Join the `Scrapy subreddit`_ and share your ideas on how to
   improve Scrapy. We're always open to suggestions.
 
 Reporting bugs
@@ -44,9 +44,9 @@ guidelines when reporting a new bug.
   don't dismiss the report but check the ticket history and comments, you may
   find additional useful information to contribute.
 
-* search the `scrapy-users`_ list to see if it has been discussed there, or
-  if you're not sure if what you're seeing is a bug. You can also ask in the
-  `#scrapy` IRC channel.
+* search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has
+  been discussed there, or if you're not sure if what you're seeing is a bug.
+  You can also ask in the `#scrapy` IRC channel.
 
 * write **complete, reproducible, specific bug reports**. The smaller the test
   case, the better. Remember that other developers won't have your project to
@@ -98,8 +98,8 @@ patch, but it's always good to have a patch ready to illustrate your arguments
 and show that you have put some additional thought into the subject. A good
 starting point is to send a pull request on GitHub. It can be simple enough to
 illustrate your idea, and leave documentation/tests for later, after the idea
-has been validated and proven useful. Alternatively, you can send an email to
-`scrapy-users`_ to discuss your idea first.
+has been validated and proven useful. Alternatively, you can start a
+conversation in the `Scrapy subreddit`_ to discuss your idea first.
 When writing GitHub pull requests, try to keep titles short but descriptive.
 E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests"
 prefer "Fix hanging when exception occurs in start_requests (#411)"
@@ -188,6 +188,7 @@ And their unit-tests are in::
 
 .. _issue tracker: https://github.com/scrapy/scrapy/issues
 .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
+.. _Scrapy subreddit: http://reddit.com/r/scrapy
 .. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
 .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
 .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
diff --git a/docs/index.rst b/docs/index.rst
index 289fb2b1b..7e8c979c4 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -13,14 +13,14 @@ Having trouble? We'd like to help!
 
 * Try the :doc:`FAQ <faq>` -- it's got answers to some common questions.
 * Looking for specific information? Try the :ref:`genindex` or :ref:`modindex`.
-* Ask or search questions in `StackOverflow using the scrapy tag`_,
-* Search for information in the `archives of the scrapy-users mailing list`_, or
-  `post a question`_.
+* Ask or search questions in `StackOverflow using the scrapy tag`_.
+* Ask or search questions in the `Scrapy subreddit`_.
+* Search for questions on the archives of the `scrapy-users mailing list`_.
 * Ask a question in the `#scrapy IRC channel`_,
 * Report bugs with Scrapy in our `issue tracker`_.
 
-.. _archives of the scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
-.. _post a question: https://groups.google.com/forum/#!forum/scrapy-users
+.. _scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
+.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/
 .. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy
 .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy
 .. _issue tracker: https://github.com/scrapy/scrapy/issues

From 083880888b3d4916c3713aae251f42ae4c77e7f7 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 30 May 2017 00:44:11 +0500
Subject: [PATCH 1504/2974] DOC fixed rst syntax in DOWNLOAD_FAIL_ON_DATALOSS
 docs

---
 docs/topics/settings.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 9cd639db4..37e3828a4 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -627,7 +627,7 @@ Optionally, this can be set per-request basis by using the
   circumstances, from server misconfiguration to network errors to data
   corruption. It is up to the user to decide if it makes sense to process
   broken responses considering they may contain partial or incomplete content.
-  If setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``,
+  If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``,
   the ``ResponseFailed([_DataLoss])`` failure will be retried as usual.
 
 .. setting:: DUPEFILTER_CLASS

From 5e1f7a9eadbb5e6bd1ef1f200e7422b77e3377c9 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 30 May 2017 00:50:32 +0500
Subject: [PATCH 1505/2974] DOC change "releases" section content

---
 README.rst | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/README.rst b/README.rst
index 4eb36b44a..27fab8e29 100644
--- a/README.rst
+++ b/README.rst
@@ -49,18 +49,17 @@ The quick way::
 For more details see the install section in the documentation:
 http://doc.scrapy.org/en/latest/intro/install.html
 
-Releases
-========
-
-You can download the latest stable and development releases from:
-http://scrapy.org/download/
-
 Documentation
 =============
 
 Documentation is available online at http://doc.scrapy.org/ and in the ``docs``
 directory.
 
+Releases
+========
+
+You can find release notes at https://doc.scrapy.org/en/latest/news.html
+
 Community (blog, twitter, mail list, IRC)
 =========================================
 

From c8dc158697453b610f168f97ce94945892163674 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 30 May 2017 17:22:23 +0200
Subject: [PATCH 1506/2974] Use HTTP pool and proper endpoint key for
 ProxyAgent

---
 scrapy/core/downloader/handlers/http11.py | 32 +++++++++++++++++++----
 1 file changed, 27 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 9bfdd803c..a4b077b57 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -14,7 +14,7 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
 from twisted.internet.error import TimeoutError
 from twisted.web.http import _DataLoss, PotentialDataLoss
 from twisted.web.client import Agent, ProxyAgent, ResponseDone, \
-    HTTPConnectionPool, ResponseFailed
+    HTTPConnectionPool, ResponseFailed, URI
 from twisted.internet.endpoints import TCP4ClientEndpoint
 
 from scrapy.http import Headers
@@ -228,10 +228,33 @@ class TunnelingAgent(Agent):
             headers, bodyProducer, requestPath)
 
 
+class ScrapyProxyAgent(Agent):
+
+    def __init__(self, reactor, proxyURI,
+                 connectTimeout=None, bindAddress=None, pool=None):
+        super(ScrapyProxyAgent, self).__init__(reactor,
+                                               connectTimeout=connectTimeout,
+                                               bindAddress=bindAddress,
+                                               pool=pool)
+        self._proxyURI = URI.fromBytes(proxyURI)
+
+    def request(self, method, uri, headers=None, bodyProducer=None):
+        """
+        Issue a new request via the configured proxy.
+        """
+        # Cache *all* connections under the same key, since we are only
+        # connecting to a single destination, the proxy:
+        proxyEndpoint = self._getEndpoint(self._proxyURI)
+        key = ("http-proxy", self._proxyURI.host, self._proxyURI.port)
+        return self._requestWithEndpoint(key, proxyEndpoint, method,
+                                         URI.fromBytes(uri), headers,
+                                         bodyProducer, uri)
+
+
 class ScrapyAgent(object):
 
     _Agent = Agent
-    _ProxyAgent = ProxyAgent
+    _ProxyAgent = ScrapyProxyAgent
     _TunnelingAgent = TunnelingAgent
 
     def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
@@ -260,9 +283,8 @@ class ScrapyAgent(object):
                     contextFactory=self._contextFactory, connectTimeout=timeout,
                     bindAddress=bindaddress, pool=self._pool)
             else:
-                endpoint = TCP4ClientEndpoint(reactor, proxyHost, proxyPort,
-                    timeout=timeout, bindAddress=bindaddress)
-                return self._ProxyAgent(endpoint)
+                return self._ProxyAgent(reactor, proxyURI=proxy,
+                    connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)
 
         return self._Agent(reactor, contextFactory=self._contextFactory,
             connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)

From 60727dedf605fad2ed4be844cb2ec44e305257f0 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Wed, 31 May 2017 15:00:38 -0300
Subject: [PATCH 1507/2974] verify if Request callback is callable

---
 scrapy/http/request/__init__.py |  4 ++++
 tests/test_http_request.py      | 20 ++++++++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 1435d91de..b9c5f8541 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -27,6 +27,10 @@ class Request(object_ref):
         assert isinstance(priority, int), "Request priority not an integer: %r" % priority
         self.priority = priority
 
+        if callback is not None and not callable(callback):
+            raise TypeError('callback must be a function, got %s' % type(callback).__name__)
+        if errback is not None and not callable(errback):
+            raise TypeError('errback must be a function, got %s' % type(errback).__name__)
         assert callback or not errback, "Cannot use errback without a callback"
         self.callback = callback
         self.errback = errback
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index bbce537f4..9b0ee63dc 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -235,6 +235,26 @@ class RequestTest(unittest.TestCase):
         self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com')
         self.assertRaises(AttributeError, setattr, r, 'body', 'xxx')
 
+    def test_callback_is_callable(self):
+        def a_function():
+            pass
+        r = self.request_class('http://example.com')
+        self.assertIsNone(r.callback)
+        r = self.request_class('http://example.com', a_function)
+        self.assertIs(r.callback, a_function)
+        with self.assertRaises(TypeError):
+            self.request_class('http://example.com', 'a_function')
+
+    def test_errback_is_callable(self):
+        def a_function():
+            pass
+        r = self.request_class('http://example.com')
+        self.assertIsNone(r.errback)
+        r = self.request_class('http://example.com', a_function, errback=a_function)
+        self.assertIs(r.errback, a_function)
+        with self.assertRaises(TypeError):
+            self.request_class('http://example.com', a_function, errback='a_function')
+
 
 class FormRequestTest(RequestTest):
 

From e162c1ff40bf74802c1a9f7b93fdbb56a5ff4a13 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 1 Jun 2017 11:58:17 +0200
Subject: [PATCH 1508/2974] Pass proxy URI to ProxyAgent as bytes

---
 scrapy/core/downloader/handlers/http11.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index a4b077b57..7b77d82da 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -283,7 +283,7 @@ class ScrapyAgent(object):
                     contextFactory=self._contextFactory, connectTimeout=timeout,
                     bindAddress=bindaddress, pool=self._pool)
             else:
-                return self._ProxyAgent(reactor, proxyURI=proxy,
+                return self._ProxyAgent(reactor, proxyURI=to_bytes(proxy, encoding='ascii'),
                     connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)
 
         return self._Agent(reactor, contextFactory=self._contextFactory,

From 6b092c66809ec11245d03063961ca64e488580e1 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 1 Jun 2017 12:29:01 +0200
Subject: [PATCH 1509/2974] Handle Twisted versions before 15.0

---
 scrapy/core/downloader/handlers/http11.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 7b77d82da..216671f82 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -14,7 +14,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
 from twisted.internet.error import TimeoutError
 from twisted.web.http import _DataLoss, PotentialDataLoss
 from twisted.web.client import Agent, ProxyAgent, ResponseDone, \
-    HTTPConnectionPool, ResponseFailed, URI
+    HTTPConnectionPool, ResponseFailed
+try:
+    from twisted.web.client import URI
+except ImportError:
+    from twisted.web.client import _URI as URI
 from twisted.internet.endpoints import TCP4ClientEndpoint
 
 from scrapy.http import Headers
@@ -244,7 +248,12 @@ class ScrapyProxyAgent(Agent):
         """
         # Cache *all* connections under the same key, since we are only
         # connecting to a single destination, the proxy:
-        proxyEndpoint = self._getEndpoint(self._proxyURI)
+        if twisted_version >= (15, 0, 0):
+            proxyEndpoint = self._getEndpoint(self._proxyURI)
+        else:
+            proxyEndpoint = self._getEndpoint(self._proxyURI.scheme,
+                                              self._proxyURI.host,
+                                              self._proxyURI.port)
         key = ("http-proxy", self._proxyURI.host, self._proxyURI.port)
         return self._requestWithEndpoint(key, proxyEndpoint, method,
                                          URI.fromBytes(uri), headers,

From fad6b70d92825b7530e39cc66020273f7f6b836b Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 1 Jun 2017 16:37:28 +0200
Subject: [PATCH 1510/2974] Use https:// for readthedocs links

---
 docs/intro/install.rst        | 2 +-
 docs/news.rst                 | 4 ++--
 docs/topics/commands.rst      | 2 +-
 docs/topics/deploy.rst        | 4 ++--
 docs/topics/item-pipeline.rst | 2 +-
 docs/topics/scrapyd.rst       | 2 +-
 docs/topics/spiders.rst       | 4 ++--
 7 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 9cec2eaee..47af8292e 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -107,7 +107,7 @@ Python virtualenvs can be created to use Python 2 by default, or Python 3 by def
 
 .. _virtualenv: https://virtualenv.pypa.io
 .. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/
-.. _virtualenvwrapper: http://virtualenvwrapper.readthedocs.io/en/latest/install.html
+.. _virtualenvwrapper: https://virtualenvwrapper.readthedocs.io/en/latest/install.html
 .. _user guide: https://virtualenv.pypa.io/en/stable/userguide/
 
 
diff --git a/docs/news.rst b/docs/news.rst
index e0f8eee0b..577c93b8e 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -12,7 +12,7 @@ but quite a few handy improvements nonetheless.
 Scrapy now supports anonymous FTP sessions with customizable user and
 password via the new :setting:`FTP_USER` and :setting:`FTP_PASSWORD` settings.
 And if you're using Twisted version 17.1.0 or above, FTP is now available
-with Python 3. 
+with Python 3.
 
 There's a new :meth:`response.follow <scrapy.http.TextResponse.follow>` method
 for creating requests; **it is now a recommended way to create Requests
@@ -407,7 +407,7 @@ Refactoring
 
 - ``canonicalize_url`` has been moved to `w3lib.url`_ (:issue:`2168`).
 
-.. _w3lib.url: http://w3lib.readthedocs.io/en/latest/w3lib.html#w3lib.url.canonicalize_url
+.. _w3lib.url: https://w3lib.readthedocs.io/en/latest/w3lib.html#w3lib.url.canonicalize_url
 
 Tests & Requirements
 ~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 3e69c4e6f..8de858f8a 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -543,7 +543,7 @@ Example::
 
     COMMANDS_MODULE = 'mybot.commands'
 
-.. _Deploying your project: http://scrapyd.readthedocs.org/en/latest/deploy.html
+.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
 
 Register commands via setup.py entry points
 -------------------------------------------
diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst
index bc48ddce7..f4186ea7a 100644
--- a/docs/topics/deploy.rst
+++ b/docs/topics/deploy.rst
@@ -50,10 +50,10 @@ them as needed - the configuration is read from the ``scrapy.cfg`` file
 just like ``scrapyd-deploy``.
 
 .. _Scrapyd: https://github.com/scrapy/scrapyd
-.. _Deploying your project: https://scrapyd.readthedocs.org/en/latest/deploy.html
+.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
 .. _Scrapy Cloud: http://scrapinghub.com/scrapy-cloud/
 .. _scrapyd-client: https://github.com/scrapy/scrapyd-client
 .. _shub: http://doc.scrapinghub.com/shub.html
-.. _scrapyd-deploy documentation: http://scrapyd.readthedocs.org/en/latest/deploy.html
+.. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html
 .. _Scrapy Cloud documentation: http://doc.scrapinghub.com/scrapy-cloud.html
 .. _Scrapinghub: http://scrapinghub.com/
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 33e4d7429..ac0a5973b 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -208,7 +208,7 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
             item["screenshot_filename"] = filename
             return item
 
-.. _Splash: http://splash.readthedocs.io/en/stable/
+.. _Splash: https://splash.readthedocs.io/en/stable/
 .. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
 
 Duplicates filter
diff --git a/docs/topics/scrapyd.rst b/docs/topics/scrapyd.rst
index 57921b901..a3d6f7698 100644
--- a/docs/topics/scrapyd.rst
+++ b/docs/topics/scrapyd.rst
@@ -10,4 +10,4 @@ Scrapyd has been moved into a separate project.
 
 Its documentation is now hosted at:
 
-    http://scrapyd.readthedocs.org/en/latest/
+    https://scrapyd.readthedocs.io/en/latest/
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 49c0cefb5..6ac946003 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -80,7 +80,7 @@ scrapy.Spider
        allowed to crawl. Requests for URLs not belonging to the domain names
        specified in this list (or their subdomains) won't be followed if
        :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled.
-       
+
        Let's say your target url is ``https://www.example.com/1.html``,
        then add ``'example.com'`` to the list.
 
@@ -756,4 +756,4 @@ Combine SitemapSpider with other sources of urls::
 .. _Sitemap index files: http://www.sitemaps.org/protocol.html#index
 .. _robots.txt: http://www.robotstxt.org/
 .. _TLD: https://en.wikipedia.org/wiki/Top-level_domain
-.. _Scrapyd documentation: http://scrapyd.readthedocs.org/en/latest/
+.. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/

From 0f6f486769d7761054274509de08fcf455d492de Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Mon, 5 Jun 2017 16:19:00 -0300
Subject: [PATCH 1511/2974] fix parse command issue with callback as a string

---
 scrapy/commands/parse.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 5264982b6..a90095146 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -142,7 +142,8 @@ class Command(ScrapyCommand):
                 logger.error('Unable to find spider for: %(url)s',
                              {'url': url})
 
-        request = Request(url, opts.callback)
+        # Request requires callback argument as callable or None, not string
+        request = Request(url, None)
         _start_requests = lambda s: [self.prepare_request(s, request, opts)]
         self.spidercls.start_requests = _start_requests
 
@@ -164,7 +165,9 @@ class Command(ScrapyCommand):
             # determine real callback
             cb = response.meta['_callback']
             if not cb:
-                if opts.rules and self.first_response == response:
+                if opts.callback:
+                    cb = opts.callback
+                elif opts.rules and self.first_response == response:
                     cb = self.get_callback_from_rules(spider, response)
 
                     if not cb:

From 4b6f68b9ee2830534c25b824103b13d502005fe1 Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Mon, 5 Jun 2017 17:26:52 -0300
Subject: [PATCH 1512/2974] make reqser tests create Request with proper
 callback/errback

---
 tests/test_utils_reqser.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 073baadc2..dcc070b8f 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -17,8 +17,8 @@ class RequestSerializationTest(unittest.TestCase):
 
     def test_all_attributes(self):
         r = Request("http://www.example.com",
-            callback='parse_item',
-            errback='handle_error',
+            callback=self.spider.parse_item,
+            errback=self.spider.handle_error,
             method="POST",
             body=b"some body",
             headers={'content-encoding': 'text/html; charset=latin-1'},
@@ -27,7 +27,7 @@ class RequestSerializationTest(unittest.TestCase):
             priority=20,
             meta={'a': 'b'},
             flags=['testFlag'])
-        self._assert_serializes_ok(r)
+        self._assert_serializes_ok(r, spider=self.spider)
 
     def test_latin1_body(self):
         r = Request("http://www.example.com", body=b"\xa3")

From 3f8542eb566dd06d35b17574d65955de2682a29e Mon Sep 17 00:00:00 2001
From: Chuan Jin <chuan.jin.813@gmail.com>
Date: Mon, 5 Jun 2017 23:31:37 +0200
Subject: [PATCH 1513/2974] Update extensions.rst

#2759
---
 docs/topics/extensions.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 6036db0f5..d24f579ee 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -277,9 +277,10 @@ CLOSESPIDER_ITEMCOUNT
 Default: ``0``
 
 An integer which specifies a number of items. If the spider scrapes more than
-that amount if items and those items are passed by the item pipeline, the
-spider will be closed with the reason ``closespider_itemcount``. If zero (or
-non set), spiders won't be closed by number of passed items.
+that amount and those items are passed by the item pipeline, the
+spider will be closed with the reason ``closespider_itemcount``, requests which 
+are currently in the downloader queue (up to CONCURRENT_REQUEST requests) are still processed. 
+If zero (or non set), spiders won't be closed by number of passed items.
 
 .. setting:: CLOSESPIDER_PAGECOUNT
 

From e7061f7a4193ea4bd7587c6cac5c9a52f847bb58 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 6 Jun 2017 10:47:43 +0200
Subject: [PATCH 1514/2974] Reformat a bit

---
 docs/topics/extensions.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index d24f579ee..03c5f2316 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -278,8 +278,9 @@ Default: ``0``
 
 An integer which specifies a number of items. If the spider scrapes more than
 that amount and those items are passed by the item pipeline, the
-spider will be closed with the reason ``closespider_itemcount``, requests which 
-are currently in the downloader queue (up to CONCURRENT_REQUEST requests) are still processed. 
+spider will be closed with the reason ``closespider_itemcount``.
+Requests which  are currently in the downloader queue (up to
+:setting:`CONCURRENT_REQUEST` requests) are still processed. 
 If zero (or non set), spiders won't be closed by number of passed items.
 
 .. setting:: CLOSESPIDER_PAGECOUNT

From 39ad0d0bddbc3d15b4186598bc69386e7321e17b Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 6 Jun 2017 10:48:30 +0200
Subject: [PATCH 1515/2974] Fix setting name reference

---
 docs/topics/extensions.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 03c5f2316..7a67cf295 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -280,7 +280,7 @@ An integer which specifies a number of items. If the spider scrapes more than
 that amount and those items are passed by the item pipeline, the
 spider will be closed with the reason ``closespider_itemcount``.
 Requests which  are currently in the downloader queue (up to
-:setting:`CONCURRENT_REQUEST` requests) are still processed. 
+:setting:`CONCURRENT_REQUESTS` requests) are still processed. 
 If zero (or non set), spiders won't be closed by number of passed items.
 
 .. setting:: CLOSESPIDER_PAGECOUNT

From ae679f6499b7d63061f11ca11592dcaff5919a00 Mon Sep 17 00:00:00 2001
From: Casker <100347755@alumnos.uc3m.es>
Date: Fri, 9 Jun 2017 16:12:20 +0800
Subject: [PATCH 1516/2974] Create item-pipeline.rst

---
 docs/topics/item-pipeline.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index ac0a5973b..38265b474 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -156,7 +156,7 @@ method and how to clean up the resources properly.::
             self.client.close()
 
         def process_item(self, item, spider):
-            self.db[self.collection_name].insert(dict(item))
+            self.db[self.collection_name].insert_one(dict(item))
             return item
 
 .. _MongoDB: https://www.mongodb.org/

From b33e0d5a54a90588e02396e7a2162d4ea12ae7dd Mon Sep 17 00:00:00 2001
From: Pengyu CHEN <pengyu@libstarrify.so>
Date: Wed, 14 Jun 2017 12:17:20 +0800
Subject: [PATCH 1517/2974] Added: Now supporting <link> tags in
 Response.follow

---
 scrapy/http/response/text.py | 9 +++++----
 tests/test_http_response.py  | 8 +++++++-
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 6415e191a..74a042f2c 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -135,7 +135,7 @@ class TextResponse(Response):
         * an attribute Selector (not SelectorList) - e.g.
           ``response.css('a::attr(href)')[0]`` or
           ``response.xpath('//img/@src')[0]``.
-        * a Selector for ``<a>`` element, e.g.
+        * a Selector for ``<a>`` or ``<link>`` element, e.g.
           ``response.css('a.my_link')[0]``.
           
         See :ref:`response-follow-example` for usage examples.
@@ -165,10 +165,11 @@ def _url_from_selector(sel):
         return strip_html5_whitespace(sel.root)
     if not hasattr(sel.root, 'tag'):
         raise ValueError("Unsupported selector: %s" % sel)
-    if sel.root.tag != 'a':
-        raise ValueError("Only <a> elements are supported; got <%s>" %
+    if sel.root.tag not in ('a', 'link'):
+        raise ValueError("Only <a> and <link> elements are supported; got <%s>" %
                          sel.root.tag)
     href = sel.root.get('href')
     if href is None:
-        raise ValueError("<a> element has no href attribute: %s" % sel)
+        raise ValueError("<%s> element has no href attribute: %s" %
+                         (sel.root.tag, sel))
     return strip_html5_whitespace(href)
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 779f5a71c..a36ec3af6 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -162,7 +162,6 @@ class BaseResponseTest(unittest.TestCase):
     def test_follow_whitespace_link(self):
         self._assert_followed_url(Link('http://example.com/foo '),
                                   'http://example.com/foo%20')
-
     def _assert_followed_url(self, follow_obj, target_url, response=None):
         if response is None:
             response = self._links_response()
@@ -402,6 +401,13 @@ class TextResponseTest(BaseResponseTest):
             for sel, url in zip(sellist, urls):
                 self._assert_followed_url(sel, url, response=resp)
 
+        # select <link> elements
+        self._assert_followed_url(
+            Selector(text='<link href="foo"></link>').css('link')[0],
+            'http://example.com/foo',
+            response=resp
+        )
+
         # href attributes should work
         for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]:
             for sel, url in zip(sellist, urls):

From f712513ed709e2e6d71a19b4e259934fd6f86955 Mon Sep 17 00:00:00 2001
From: Pengyu CHEN <pengyu@libstarrify.so>
Date: Thu, 15 Jun 2017 10:41:02 +0800
Subject: [PATCH 1518/2974] Added doc for `scrapy.exceptions.DontCloseSpider`.
 Also fixes inaccurate doc for `scrapy.signals.spider_idle`.

---
 docs/topics/exceptions.rst |  8 ++++++++
 docs/topics/signals.rst    | 10 ++++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index cc02369d4..09cb8ed66 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -39,6 +39,14 @@ For example::
         if 'Bandwidth exceeded' in response.body:
             raise CloseSpider('bandwidth_exceeded')
 
+DontCloseSpider
+---------------
+
+.. exception:: DontCloseSpider
+
+This exception can be raised in a :signal:`spider_idle` signal handler to
+prevent the spider from being closed.
+
 IgnoreRequest
 -------------
 
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 0306ee4a5..cf1588df8 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -189,14 +189,20 @@ spider_idle
     the engine starts closing the spider. After the spider has finished
     closing, the :signal:`spider_closed` signal is sent.
 
-    You can, for example, schedule some requests in your :signal:`spider_idle`
-    handler to prevent the spider from being closed.
+    You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
+    prevent the spider from being closed.
 
     This signal does not support returning deferreds from their handlers.
 
     :param spider: the spider which has gone idle
     :type spider: :class:`~scrapy.spiders.Spider` object
 
+.. note:: Scheduling some requests in your :signal:`spider_idle` handler does
+    **not** guarantee that it can prevent the spider from being closed,
+    although it sometimes can. That's because the spider may still remain idle
+    if all the scheduled requests are rejected by the scheduler (e.g. filtered
+    due to duplication).
+
 spider_error
 ------------
 

From 5a08cf3b9606bf77ef02a37dca1e2bc76f74558f Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 2 Sep 2016 00:22:22 +0300
Subject: [PATCH 1519/2974] Fix test_start_requests_errors for PyPy

Twisted prints errors in DebugInfo.__del__, but PyPy does not run
gc.collect() on exit:
http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies
---
 scrapy/cmdline.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index e4dc7f2de..b546d030e 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -4,6 +4,7 @@ import optparse
 import cProfile
 import inspect
 import pkg_resources
+import gc
 
 import scrapy
 from scrapy.crawler import CrawlerProcess
@@ -165,4 +166,9 @@ def _run_command_profiled(cmd, args, opts):
         p.dump_stats(opts.profile)
 
 if __name__ == '__main__':
-    execute()
+    try:
+        execute()
+    finally:
+        # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()
+        # on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies
+        gc.collect()

From 6014856df5717496b57aaac6c8a2e64b125ac32b Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 1 Sep 2016 18:39:25 +0300
Subject: [PATCH 1520/2974] Fix test_output_processor_error undere PyPy

For float(u'$10') PyPy includes "u'" in the error message,
and it's more fair to check error message on input we are really
passing.
---
 tests/test_loader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_loader.py b/tests/test_loader.py
index 2693a18d9..9d07eb95b 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -290,7 +290,7 @@ class BasicItemLoaderTest(unittest.TestCase):
         il = TestItemLoader()
         il.add_value('name', [u'$10'])
         try:
-            float('$10')
+            float(u'$10')
         except Exception as e:
             expected_exc_str = str(e)
 

From c3d17659b33432ba8cd2e5b2da57105c1cc0da22 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 1 Sep 2016 15:39:16 +0300
Subject: [PATCH 1521/2974] Fix queue serialization test on PyPy

It is not affected by Twisted bug #7989 and is more permissive
with pickling (especially with protocol=2).
---
 tests/test_squeues.py | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 48871ceeb..3a24348b4 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -1,3 +1,5 @@
+import pickle
+
 from queuelib.tests import test_queue as t
 from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue
 from scrapy.item import Item, Field
@@ -14,6 +16,22 @@ class TestLoader(ItemLoader):
     default_item_class = TestItem
     name_out = staticmethod(_test_procesor)
 
+def nonserializable_object_test(self):
+    try:
+        pickle.dumps(lambda x: x)
+    except Exception:
+        # Trigger Twisted bug #7989
+        import twisted.persisted.styles  # NOQA
+        q = self.queue()
+        self.assertRaises(ValueError, q.push, lambda x: x)
+    else:
+        # Use a different unpickleable object
+        class A(object): pass
+        a = A()
+        a.__reduce__ = a.__reduce_ex__ = None
+        q = self.queue()
+        self.assertRaises(ValueError, q.push, a)
+
 class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
 
     chunksize = 100000
@@ -30,11 +48,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
         self.assertEqual(q.pop(), 123)
         self.assertEqual(q.pop(), {'a': 'dict'})
 
-    def test_nonserializable_object(self):
-        # Trigger Twisted bug #7989
-        import twisted.persisted.styles  # NOQA
-        q = self.queue()
-        self.assertRaises(ValueError, q.push, lambda x: x)
+    test_nonserializable_object = nonserializable_object_test
 
 class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
     chunksize = 1
@@ -110,11 +124,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
         self.assertEqual(q.pop(), 123)
         self.assertEqual(q.pop(), 'a')
 
-    def test_nonserializable_object(self):
-        # Trigger Twisted bug #7989
-        import twisted.persisted.styles  # NOQA
-        q = self.queue()
-        self.assertRaises(ValueError, q.push, lambda x: x)
+    test_nonserializable_object = nonserializable_object_test
 
 
 class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest):

From 5abb70c8d71365adbf3a035b1f6d07f5fbbbf446 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 1 Sep 2016 15:43:57 +0300
Subject: [PATCH 1522/2974] Fix test_weakkeycache on PyPy: run gc.collect()

One gc.collect() seems to be enough, but it's more reliable
to run it several times (at most 100), until all objects are collected.
---
 tests/test_utils_python.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 9a0cc975d..e22bd8eb6 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -1,3 +1,4 @@
+import gc
 import functools
 import operator
 import unittest
@@ -144,6 +145,9 @@ class UtilsPythonTestCase(unittest.TestCase):
         self.assertNotEqual(v, wk[_Weakme()])
         self.assertEqual(v, wk[k])
         del k
+        for _ in range(100):
+            if wk._weakdict:
+                gc.collect()
         self.assertFalse(len(wk._weakdict))
 
     @unittest.skipUnless(six.PY2, "deprecated function")

From 7c67047e77914dfe0d6666e4dc535c684aa77090 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 1 Sep 2016 18:35:57 +0300
Subject: [PATCH 1523/2974] Fix get_func_args tests under PyPy

On CPython get_func_args does not work correctly for built-in
methods.
---
 tests/test_utils_python.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index e22bd8eb6..8becca0f1 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -3,6 +3,7 @@ import functools
 import operator
 import unittest
 from itertools import count
+import platform
 import six
 
 from scrapy.utils.python import (
@@ -212,10 +213,16 @@ class UtilsPythonTestCase(unittest.TestCase):
         self.assertEqual(get_func_args(cal), ['a', 'b', 'c'])
         self.assertEqual(get_func_args(object), [])
 
-        # TODO: how do we fix this to return the actual argument names?
-        self.assertEqual(get_func_args(six.text_type.split), [])
-        self.assertEqual(get_func_args(" ".join), [])
-        self.assertEqual(get_func_args(operator.itemgetter(2)), [])
+        if platform.python_implementation() == 'CPython':
+            # TODO: how do we fix this to return the actual argument names?
+            self.assertEqual(get_func_args(six.text_type.split), [])
+            self.assertEqual(get_func_args(" ".join), [])
+            self.assertEqual(get_func_args(operator.itemgetter(2)), [])
+        else:
+            self.assertEqual(get_func_args(six.text_type.split), ['sep', 'maxsplit'])
+            self.assertEqual(get_func_args(" ".join), ['list'])
+            self.assertEqual(get_func_args(operator.itemgetter(2)), ['obj'])
+
 
     def test_without_none_values(self):
         self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])

From 19ca986aa1340b08658caaac0ce677aa22be9814 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Fri, 2 Sep 2016 00:32:33 +0300
Subject: [PATCH 1524/2974] Move garbage_collect to scrapy.utils.python

---
 scrapy/cmdline.py      |  4 ++--
 scrapy/utils/python.py | 20 ++++++++++++++++++++
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index b546d030e..dc6b59fe0 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -4,7 +4,6 @@ import optparse
 import cProfile
 import inspect
 import pkg_resources
-import gc
 
 import scrapy
 from scrapy.crawler import CrawlerProcess
@@ -12,6 +11,7 @@ from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import inside_project, get_project_settings
+from scrapy.utils.python import garbage_collect
 from scrapy.settings.deprecated import check_deprecated_settings
 
 def _iter_command_classes(module_name):
@@ -171,4 +171,4 @@ if __name__ == '__main__':
     finally:
         # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()
         # on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies
-        gc.collect()
+        garbage_collect()
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 4c500abf4..d28d71bd3 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -1,6 +1,7 @@
 """
 This module contains essential stuff that should've come with Python itself ;)
 """
+import gc
 import os
 import re
 import inspect
@@ -8,6 +9,8 @@ import weakref
 import errno
 import six
 from functools import partial, wraps
+import sys
+import time
 
 from scrapy.utils.decorators import deprecated
 
@@ -355,3 +358,20 @@ def global_object_name(obj):
     'scrapy.http.request.Request'
     """
     return "%s.%s" % (obj.__module__, obj.__name__)
+
+
+if sys.platform.startswith('java'):
+    def garbage_collect():
+        # Some JVM GCs will execute finalizers in a different thread, meaning
+        # we need to wait for that to complete before we go on looking for the
+        # effects of that.
+        gc.collect()
+        time.sleep(0.1)
+elif hasattr(sys, "pypy_version_info"):
+    def garbage_collect():
+        # Collecting weakreferences can take two collections on PyPy.
+        gc.collect()
+        gc.collect()
+else:
+    def garbage_collect():
+        gc.collect()

From b4eb60e5270498a64200332587a065ec9978a333 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 15 Jun 2017 13:24:06 +0300
Subject: [PATCH 1525/2974] Install PyPyDispatcher for PyPy tests

Using https://github.com/lopuhin/pydispatcher, pypy branch.
This is executed as a separate step to avoid changing
default requirements.txt and setup.py. If just added to "deps"
in tox, this install command will be executed as one command
and PyPyDispatcher will not override PyDispatcher.
---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 6987847f8..c559f1e47 100644
--- a/tox.ini
+++ b/tox.ini
@@ -57,6 +57,7 @@ commands =
 [testenv:pypy]
 basepython = pypy
 commands =
+    pip install PyPyDispatcher>=2.0.6
     py.test {posargs:scrapy tests}
 
 [testenv:py33]

From a8df0900713a1a56cffca8774bc137a6e781877e Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 15 Jun 2017 14:29:28 +0300
Subject: [PATCH 1526/2974] Fix httpcache leveldb tests: gc.collect after del

LevelDB does not have "official" close method, so we have
to rely on garbage collection to close it.
---
 scrapy/extensions/httpcache.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 2fb4b6a15..648b32ec7 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -13,7 +13,7 @@ from scrapy.responsetypes import responsetypes
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.project import data_path
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_bytes, to_unicode
+from scrapy.utils.python import to_bytes, to_unicode, garbage_collect
 
 
 logger = logging.getLogger(__name__)
@@ -362,6 +362,7 @@ class LeveldbCacheStorage(object):
         # avoid them being removed in storages with timestamp-based autoremoval.
         self.db.CompactRange()
         del self.db
+        garbage_collect()
 
     def retrieve_response(self, spider, request):
         data = self._read_data(spider, request)

From ea08b952801b9af9bc45ae9918fcf5ab9bcb5aaf Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 19 Jun 2017 16:45:29 +0300
Subject: [PATCH 1527/2974] Remove Jython gc branch: it's not supported

---
 scrapy/utils/python.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index d28d71bd3..72f8f4311 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -360,14 +360,7 @@ def global_object_name(obj):
     return "%s.%s" % (obj.__module__, obj.__name__)
 
 
-if sys.platform.startswith('java'):
-    def garbage_collect():
-        # Some JVM GCs will execute finalizers in a different thread, meaning
-        # we need to wait for that to complete before we go on looking for the
-        # effects of that.
-        gc.collect()
-        time.sleep(0.1)
-elif hasattr(sys, "pypy_version_info"):
+if hasattr(sys, "pypy_version_info"):
     def garbage_collect():
         # Collecting weakreferences can take two collections on PyPy.
         gc.collect()

From 271b3a485cbdce1b0a866d9fb9938d46d7ddb497 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 19 Jun 2017 16:46:16 +0300
Subject: [PATCH 1528/2974] Require pypy build to pass

---
 .travis.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 906115096..449eee96b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,6 +11,8 @@ matrix:
       env: TOXENV=py27
     - python: 2.7
       env: TOXENV=jessie
+    - python: 2.7
+      env: TOXENV=pypy
     - python: 3.3
       env: TOXENV=py33
     - python: 3.5
@@ -21,9 +23,6 @@ matrix:
       env: TOXENV=pypy
     - python: 3.6
       env: TOXENV=docs
-  allow_failures:
-    - python: 2.7
-      env: TOXENV=pypy
 install:
   - |
       if [ "$TOXENV" = "pypy" ]; then

From 5ba8e5adc0e1aa7d9430b63a0094c6a71c51564a Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 19 Jun 2017 17:45:28 +0300
Subject: [PATCH 1529/2974] Remove duplicate PyPy toxenv from Travis config

Thanks for the catch @redapple
---
 .travis.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 449eee96b..4f44d1e6d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,8 +19,6 @@ matrix:
       env: TOXENV=py35
     - python: 3.6
       env: TOXENV=py36
-    - python: 2.7
-      env: TOXENV=pypy
     - python: 3.6
       env: TOXENV=docs
 install:

From b0a9236357dd74381b5a014e3de15a3a52de4f7d Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 19 Jun 2017 19:16:50 +0300
Subject: [PATCH 1530/2974] Use environment markers for custom PyPy
 requirements

---
 setup.py | 23 ++++++++++++++++++++++-
 tox.ini  |  1 -
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 086ab8142..c03f0b9f7 100644
--- a/setup.py
+++ b/setup.py
@@ -1,11 +1,31 @@
 from os.path import dirname, join
-from setuptools import setup, find_packages
+from pkg_resources import parse_version
+from setuptools import setup, find_packages, __version__ as setuptools_version
 
 
 with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f:
     version = f.read().decode('ascii').strip()
 
 
+def has_environment_marker_platform_impl_support():
+    """Code extracted from 'pytest/setup.py'
+    https://github.com/pytest-dev/pytest/blob/7538680c/setup.py#L31
+
+    The first known release to support environment marker with range operators
+    it is 18.5, see:
+    https://setuptools.readthedocs.io/en/latest/history.html#id235
+    """
+    return parse_version(setuptools_version) >= parse_version('18.5')
+
+
+extras_require = {}
+
+if has_environment_marker_platform_impl_support():
+    extras_require[':platform_python_implementation == "PyPy"'] = [
+        'PyPyDispatcher>=2.1.0',
+    ]
+
+
 setup(
     name='Scrapy',
     version=version,
@@ -53,4 +73,5 @@ setup(
         'PyDispatcher>=2.0.5',
         'service_identity',
     ],
+    extras_require=extras_require,
 )
diff --git a/tox.ini b/tox.ini
index c559f1e47..6987847f8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -57,7 +57,6 @@ commands =
 [testenv:pypy]
 basepython = pypy
 commands =
-    pip install PyPyDispatcher>=2.0.6
     py.test {posargs:scrapy tests}
 
 [testenv:py33]

From 793b2376f83caa01d3e883f7642147de8e174c01 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 3 Jul 2017 11:28:04 -0300
Subject: [PATCH 1531/2974] Populate spider variable when using
 shell.inspect_response

---
 scrapy/shell.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/shell.py b/scrapy/shell.py
index 6f94635a1..80b625633 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -164,7 +164,7 @@ class Shell(object):
 
 def inspect_response(response, spider):
     """Open a shell to inspect the given response"""
-    Shell(spider.crawler).start(response=response)
+    Shell(spider.crawler).start(response=response, spider=spider)
 
 
 def _request_deferred(request):

From 1f08d9a64884a05a541fe9649e3349cce5aa47be Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 4 Jul 2017 23:10:19 +0200
Subject: [PATCH 1532/2974] Add test for DNS cache disabling

---
 tests/test_commands.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 922098668..cb1301c95 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -226,6 +226,27 @@ class MySpider(scrapy.Spider):
         self.assertNotIn("DEBUG: It Works!", log)
         self.assertIn("INFO: Spider opened", log)
 
+    def test_runspider_dnscache_disabled(self):
+        # see https://github.com/scrapy/scrapy/issues/2811
+        # The spider below should not be able to connect to localhost:12345,
+        # which is intended,
+        # but this should not be because of DNS lookup error
+        # assumption: localhost will resolve in all cases (true?)
+        log = self.get_log("""
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+    start_urls = ['http://localhost:12345']
+
+    def parse(self, response):
+        return {'test': 'value'}
+""",
+                           args=('-s', 'DNSCACHE_ENABLED=False'))
+        print(log)
+        self.assertNotIn("DNSLookupError", log)
+        self.assertIn("INFO: Spider opened", log)
+
     def test_runspider_log_short_names(self):
         log1 = self.get_log(self.debug_log_spider,
                             args=('-s', 'LOG_SHORT_NAMES=1'))

From f0ded6b7759c9c2ec4b8946cdfc4abab97bb14d9 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 4 Jul 2017 23:18:15 +0200
Subject: [PATCH 1533/2974] Do not cache DNS responses when cache size is 0

---
 scrapy/resolver.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 4f4f0b04f..0aaced7e4 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -22,7 +22,8 @@ class CachingThreadedResolver(ThreadedResolver):
         # to enforce Scrapy's DNS_TIMEOUT setting's value
         timeout = (self.timeout,)
         d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
-        d.addCallback(self._cache_result, name)
+        if dnscache.limit:
+            d.addCallback(self._cache_result, name)
         return d
 
     def _cache_result(self, result, name):

From dedc4a8b8f7cd3870f89bec2bf89a5ec11a95e4f Mon Sep 17 00:00:00 2001
From: Danny Guo <dguo@users.noreply.github.com>
Date: Thu, 13 Jul 2017 22:58:10 -0500
Subject: [PATCH 1534/2974] Tweak the CSVFeedSpider documentation

---
 docs/topics/spiders.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 6ac946003..bf1532d1b 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -578,8 +578,7 @@ CSVFeedSpider
 
    .. attribute:: headers
 
-       A list of the rows contained in the file CSV feed which will be used to
-       extract fields from it.
+       A list of the column names in the CSV file.
 
    .. method:: parse_row(response, row)
 

From 18b96dd82af0ac58a2fc6e1ba2227b4468ab2bb7 Mon Sep 17 00:00:00 2001
From: Claus Conrad <mail@clausconrad.com>
Date: Sat, 15 Jul 2017 11:31:09 +0200
Subject: [PATCH 1535/2974] Spelling mistake

---
 docs/topics/broad-crawls.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index 28ed7c064..040c8cfde 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -85,8 +85,8 @@ When doing broad crawls you are often only interested in the crawl rates you
 get and any errors found. These stats are reported by Scrapy when using the
 ``INFO`` log level. In order to save CPU (and log storage requirements) you
 should not use ``DEBUG`` log level when preforming large broad crawls in
-production. Using ``DEBUG`` level when developing your (broad) crawler may fine
-though.
+production. Using ``DEBUG`` level when developing your (broad) crawler may be 
+fine though.
 
 To set the log level use::
 

From 26c488970c51c06506cfc88c4a76440c009cdd48 Mon Sep 17 00:00:00 2001
From: Danny Guo <dguo@users.noreply.github.com>
Date: Tue, 18 Jul 2017 19:56:51 -0500
Subject: [PATCH 1536/2974] Fix a typo in the Items documentation

---
 docs/topics/items.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 4a8f47e93..4423bbda2 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -21,7 +21,7 @@ their available fields.
 Various Scrapy components use extra information provided by Items: 
 exporters look at declared fields to figure out columns to export,
 serialization can be customized using Item fields metadata, :mod:`trackref`
-tracks Item instances to help finding memory leaks 
+tracks Item instances to help find memory leaks 
 (see :ref:`topics-leaks-trackrefs`), etc.
 
 .. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict

From bb0bd691d9dd42725af6291a15e357f0146de17f Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Mon, 24 Jul 2017 11:12:09 -0300
Subject: [PATCH 1537/2974] Improve error message when callback is not callable

---
 scrapy/http/request/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index b9c5f8541..13a92ffa0 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -28,9 +28,9 @@ class Request(object_ref):
         self.priority = priority
 
         if callback is not None and not callable(callback):
-            raise TypeError('callback must be a function, got %s' % type(callback).__name__)
+            raise TypeError('callback must be a callable, got %s' % type(callback).__name__)
         if errback is not None and not callable(errback):
-            raise TypeError('errback must be a function, got %s' % type(errback).__name__)
+            raise TypeError('errback must be a callable, got %s' % type(errback).__name__)
         assert callback or not errback, "Cannot use errback without a callback"
         self.callback = callback
         self.errback = errback

From 1a18587d41cc7dabc0a829fc0b69016b054d64f8 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 24 Jul 2017 19:30:08 +0200
Subject: [PATCH 1538/2974] Jessi toxenv: Add cryptography as per
 https://packages.debian.org/jessie/python-cryptography

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 6987847f8..c7e1e43c9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -38,6 +38,7 @@ deps =
 # https://packages.debian.org/en/jessie/zope/
 basepython = python2.7
 deps =
+    cryptography==0.6.1
     pyOpenSSL==0.14
     lxml==3.4.0
     Twisted==14.0.2

From 33dfac50185b1940dda89bec3e3480a7e76e9ca7 Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@bluewin.ch>
Date: Mon, 24 Jul 2017 22:06:17 +0200
Subject: [PATCH 1539/2974] xrange() --> range() for Python 3

Either this PR or #2845.
---
 extras/qpsclient.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extras/qpsclient.py b/extras/qpsclient.py
index bb83588dd..7554f7eec 100644
--- a/extras/qpsclient.py
+++ b/extras/qpsclient.py
@@ -41,7 +41,7 @@ class QPSSpider(Spider):
 
         slots = int(self.slots)
         if slots > 1:
-            urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in xrange(slots)]
+            urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)]
         else:
             urls = [url]
 

From 815d6160cfac5c0b764d01fc9d4ebc4ab7793aac Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Mon, 16 Nov 2015 18:28:49 +0100
Subject: [PATCH 1540/2974] Add from_crawler constructor for feed exporters and
 storages

---
 scrapy/extensions/feedexport.py   | 12 +++++--
 scrapy/middleware.py              |  9 ++---
 scrapy/utils/misc.py              | 24 +++++++++++++
 tests/test_feedexport.py          | 30 ++++++++++++++++
 tests/test_utils_misc/__init__.py | 58 ++++++++++++++++++++++++++++++-
 5 files changed, 122 insertions(+), 11 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 5f133fbde..70c302fba 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -21,7 +21,7 @@ from w3lib.url import file_uri_to_path
 from scrapy import signals
 from scrapy.utils.ftp import ftp_makedirs_cwd
 from scrapy.exceptions import NotConfigured
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.python import without_none_values
 from scrapy.utils.boto import is_botocore
@@ -181,6 +181,7 @@ class FeedExporter(object):
     @classmethod
     def from_crawler(cls, crawler):
         o = cls(crawler.settings)
+        o.crawler = crawler
         crawler.signals.connect(o.open_spider, signals.spider_opened)
         crawler.signals.connect(o.close_spider, signals.spider_closed)
         crawler.signals.connect(o.item_scraped, signals.item_scraped)
@@ -253,11 +254,16 @@ class FeedExporter(object):
             logger.error("Unknown feed storage scheme: %(scheme)s",
                          {'scheme': scheme})
 
+    def _get_instance(self, objcls, *args, **kwargs):
+        return create_instance(
+            objcls, self.settings, getattr(self, 'crawler', None),
+            *args, **kwargs)
+
     def _get_exporter(self, *args, **kwargs):
-        return self.exporters[self.format](*args, **kwargs)
+        return self._get_instance(self.exporters[self.format], *args, **kwargs)
 
     def _get_storage(self, uri):
-        return self.storages[urlparse(uri).scheme](uri)
+        return self._get_instance(self.storages[urlparse(uri).scheme], uri)
 
     def _get_uri_params(self, spider):
         params = {}
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index be36f977e..f2240984c 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -3,7 +3,7 @@ import logging
 import pprint
 
 from scrapy.exceptions import NotConfigured
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.defer import process_parallel, process_chain, process_chain_both
 
 logger = logging.getLogger(__name__)
@@ -32,12 +32,7 @@ class MiddlewareManager(object):
         for clspath in mwlist:
             try:
                 mwcls = load_object(clspath)
-                if crawler and hasattr(mwcls, 'from_crawler'):
-                    mw = mwcls.from_crawler(crawler)
-                elif hasattr(mwcls, 'from_settings'):
-                    mw = mwcls.from_settings(settings)
-                else:
-                    mw = mwcls()
+                mw = create_instance(mwcls, settings, crawler)
                 middlewares.append(mw)
                 enabled.append(clspath)
             except NotConfigured as e:
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 35f855007..8eb1aabb5 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -117,3 +117,27 @@ def md5sum(file):
 def rel_has_nofollow(rel):
     """Return True if link rel attribute has nofollow type"""
     return True if rel is not None and 'nofollow' in rel.split() else False
+
+def create_instance(objcls, settings, crawler, *args, **kwargs):
+    """Construct a class instance using its ``from_crawler`` or
+    ``from_settings`` constructors, if available.
+
+    At least one of ``settings`` and ``crawler`` needs to be different from
+    ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
+    If ``crawler`` is ``None``, only the ``from_settings`` constructor will be
+    tried.
+
+    ``*args`` and ``**kwargs`` are forwarded to the constructors.
+
+    Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
+    """
+    if settings is None:
+        if crawler is None:
+            raise ValueError("Specifiy at least one of settings and crawler.")
+        settings = crawler.settings
+    if crawler and hasattr(objcls, 'from_crawler'):
+        return objcls.from_crawler(crawler, *args, **kwargs)
+    elif hasattr(objcls, 'from_settings'):
+        return objcls.from_settings(settings, *args, **kwargs)
+    else:
+        return objcls(*args, **kwargs)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index f55927121..08f7e4d8d 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -16,6 +16,7 @@ from tests.mockserver import MockServer
 from w3lib.url import path_to_file_uri
 
 import scrapy
+from scrapy.exporters import CsvItemExporter
 from scrapy.extensions.feedexport import (
     IFeedStorage, FileFeedStorage, FTPFeedStorage,
     S3FeedStorage, StdoutFeedStorage,
@@ -159,6 +160,23 @@ class StdoutFeedStorageTest(unittest.TestCase):
         self.assertEqual(out.getvalue(), b"content")
 
 
+class FromCrawlerMixin(object):
+    init_with_crawler = False
+
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        cls.init_with_crawler = True
+        return cls(*args, **kwargs)
+
+
+class FromCrawlerCsvItemExporter(CsvItemExporter, FromCrawlerMixin):
+    pass
+
+
+class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
+    pass
+
+
 class FeedExportTest(unittest.TestCase):
 
     class MyItem(scrapy.Item):
@@ -599,3 +617,15 @@ class FeedExportTest(unittest.TestCase):
             data = yield self.exported_data(items, settings)
             print(row['format'], row['indent'])
             self.assertEqual(row['expected'], data)
+
+    @defer.inlineCallbacks
+    def test_init_exporters_storages_with_crawler(self):
+        settings = {
+            'FEED_EXPORTERS': {'csv': 'tests.test_feedexport.'
+                                      'FromCrawlerCsvItemExporter'},
+            'FEED_STORAGES': {'file': 'tests.test_feedexport.'
+                                      'FromCrawlerFileFeedStorage'},
+        }
+        yield self.exported_data({}, settings)
+        self.assertTrue(FromCrawlerCsvItemExporter.init_with_crawler)
+        self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler)
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index 01460a10b..b95bba5c1 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -3,7 +3,9 @@ import os
 import unittest
 
 from scrapy.item import Item, Field
-from scrapy.utils.misc import load_object, arg_to_iter, walk_modules
+from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules
+
+from tests import mock
 
 __doctests__ = ['scrapy.utils.misc']
 
@@ -74,5 +76,59 @@ class UtilsMiscTestCase(unittest.TestCase):
         self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}])
         self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")])
 
+    def test_create_instance(self):
+        settings = mock.MagicMock()
+        crawler = mock.MagicMock(spec_set=['settings'])
+        args = (True, 100.)
+        kwargs = {'key': 'val'}
+
+        def _test_with_settings(mock, settings):
+            create_instance(mock, settings, None, *args, **kwargs)
+            if hasattr(mock, 'from_crawler'):
+                self.assertEqual(mock.from_crawler.call_count, 0)
+            if hasattr(mock, 'from_settings'):
+                mock.from_settings.assert_called_once_with(settings, *args,
+                                                           **kwargs)
+                self.assertEqual(mock.call_count, 0)
+            else:
+                mock.assert_called_once_with(*args, **kwargs)
+
+        def _test_with_crawler(mock, settings, crawler):
+            create_instance(mock, settings, crawler, *args, **kwargs)
+            if hasattr(mock, 'from_crawler'):
+                mock.from_crawler.assert_called_once_with(crawler, *args,
+                                                          **kwargs)
+                if hasattr(mock, 'from_settings'):
+                    self.assertEqual(mock.from_settings.call_count, 0)
+                self.assertEqual(mock.call_count, 0)
+            elif hasattr(mock, 'from_settings'):
+                mock.from_settings.assert_called_once_with(settings, *args,
+                                                           **kwargs)
+                self.assertEqual(mock.call_count, 0)
+            else:
+                mock.assert_called_once_with(*args, **kwargs)
+
+        # Check usage of correct constructor using four mocks:
+        #   1. with no alternative constructors
+        #   2. with from_settings() constructor
+        #   3. with from_crawler() constructor
+        #   4. with from_settings() and from_crawler() constructor
+        spec_sets = ([], ['from_settings'], ['from_crawler'],
+                     ['from_settings', 'from_crawler'])
+        for specs in spec_sets:
+            m = mock.MagicMock(spec_set=specs)
+            _test_with_settings(m, settings)
+            m.reset_mock()
+            _test_with_crawler(m, settings, crawler)
+
+        # Check adoption of crawler settings
+        m = mock.MagicMock(spec_set=['from_settings'])
+        create_instance(m, None, crawler, *args, **kwargs)
+        m.from_settings.assert_called_once_with(crawler.settings, *args,
+                                                **kwargs)
+
+        with self.assertRaises(ValueError):
+            create_instance(m, None, None)
+
 if __name__ == "__main__":
     unittest.main()

From 4d77c3084e64d351e3acf9092b4171d4418925fe Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Mon, 23 Nov 2015 12:24:05 +0100
Subject: [PATCH 1541/2974] Add from_crawler constructor to S3FeedStorage

---
 scrapy/extensions/feedexport.py | 27 ++++++++++++++++++++---
 tests/test_feedexport.py        | 38 ++++++++++++++++++++++++++++++++-
 2 files changed, 61 insertions(+), 4 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 70c302fba..067887d94 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -93,12 +93,28 @@ class FileFeedStorage(object):
 
 class S3FeedStorage(BlockingFeedStorage):
 
-    def __init__(self, uri):
+    def __init__(self, uri, access_key=None, secret_key=None):
+        # BEGIN Backwards compatibility for initialising without keys (and
+        # without using from_crawler)
         from scrapy.conf import settings
+        no_defaults = access_key is None and secret_key is None
+        if no_defaults and ('AWS_ACCESS_KEY_ID' in settings or
+                            'AWS_SECRET_ACCESS_KEY' in settings):
+            import warnings
+            from scrapy.exceptions import ScrapyDeprecationWarning
+            warnings.warn(
+                "Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
+                "without AWS keys is deprecated. Please supply credentials or "
+                "use the `from_crawler()` constructor.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2
+            )
+            access_key = settings['AWS_ACCESS_KEY_ID']
+            secret_key = settings['AWS_SECRET_ACCESS_KEY']
         u = urlparse(uri)
         self.bucketname = u.hostname
-        self.access_key = u.username or settings['AWS_ACCESS_KEY_ID']
-        self.secret_key = u.password or settings['AWS_SECRET_ACCESS_KEY']
+        self.access_key = u.username or access_key
+        self.secret_key = u.password or secret_key
         self.is_botocore = is_botocore()
         self.keyname = u.path[1:]  # remove first "/"
         if self.is_botocore:
@@ -111,6 +127,11 @@ class S3FeedStorage(BlockingFeedStorage):
             import boto
             self.connect_s3 = boto.connect_s3
 
+    @classmethod
+    def from_crawler(cls, crawler, uri):
+        return cls(uri, crawler.settings['AWS_ACCESS_KEY_ID'],
+                   crawler.settings['AWS_SECRET_ACCESS_KEY'])
+
     def _store_in_thread(self, file):
         file.seek(0)
         if self.is_botocore:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 08f7e4d8d..eeb1bc2a4 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -6,12 +6,14 @@ from io import BytesIO
 import tempfile
 import shutil
 from six.moves.urllib.parse import urlparse
+import warnings
 
 from zope.interface.verify import verifyObject
 from twisted.trial import unittest
 from twisted.internet import defer
 from scrapy.crawler import CrawlerRunner
 from scrapy.settings import Settings
+from tests import mock
 from tests.mockserver import MockServer
 from w3lib.url import path_to_file_uri
 
@@ -131,13 +133,47 @@ class BlockingFeedStorageTest(unittest.TestCase):
 
 class S3FeedStorageTest(unittest.TestCase):
 
+    @mock.patch('scrapy.conf.settings', new={'AWS_ACCESS_KEY_ID': 'conf_key',
+                'AWS_SECRET_ACCESS_KEY': 'conf_secret'}, create=True)
+    def test_parse_credentials(self):
+        try:
+            import boto
+        except ImportError:
+            raise unittest.SkipTest("S3FeedStorage requires boto")
+        aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key',
+                           'AWS_SECRET_ACCESS_KEY': 'settings_secret'}
+        crawler = get_crawler(settings_dict=aws_credentials)
+        # Instantiate with crawler
+        storage = S3FeedStorage.from_crawler(crawler,
+                                             's3://mybucket/export.csv')
+        self.assertEqual(storage.access_key, 'settings_key')
+        self.assertEqual(storage.secret_key, 'settings_secret')
+        # Instantiate directly
+        storage = S3FeedStorage('s3://mybucket/export.csv',
+                                aws_credentials['AWS_ACCESS_KEY_ID'],
+                                aws_credentials['AWS_SECRET_ACCESS_KEY'])
+        self.assertEqual(storage.access_key, 'settings_key')
+        self.assertEqual(storage.secret_key, 'settings_secret')
+        # URI priority > settings priority
+        storage = S3FeedStorage('s3://uri_key:uri_secret@mybucket/export.csv',
+                                aws_credentials['AWS_ACCESS_KEY_ID'],
+                                aws_credentials['AWS_SECRET_ACCESS_KEY'])
+        self.assertEqual(storage.access_key, 'uri_key')
+        self.assertEqual(storage.secret_key, 'uri_secret')
+        # Backwards compatibility for initialising without settings
+        with warnings.catch_warnings(record=True) as w:
+            storage = S3FeedStorage('s3://mybucket/export.csv')
+            self.assertEqual(storage.access_key, 'conf_key')
+            self.assertEqual(storage.secret_key, 'conf_secret')
+            self.assertTrue('without AWS keys' in str(w[-1].message))
+
     @defer.inlineCallbacks
     def test_store(self):
         assert_aws_environ()
         uri = os.environ.get('S3_TEST_FILE_URI')
         if not uri:
             raise unittest.SkipTest("No S3 URI available for testing")
-        storage = S3FeedStorage(uri)
+        storage = S3FeedStorage(uri, Settings())
         verifyObject(IFeedStorage, storage)
         file = storage.open(scrapy.Spider("default"))
         expected_content = b"content: \xe2\x98\x83"

From 11a1f970b7b7b68b3d968df4b29c4269ab220ac6 Mon Sep 17 00:00:00 2001
From: Pengyu Chen <pengyu@libstarrify.so>
Date: Wed, 26 Jul 2017 16:11:13 +0800
Subject: [PATCH 1542/2974] Added: HTTP status code 522/524 to retry.

---
 scrapy/settings/default_settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 87dbf6974..697314b7f 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -234,7 +234,7 @@ REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
 
 RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
-RETRY_HTTP_CODES = [500, 502, 503, 504, 408]
+RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408]
 RETRY_PRIORITY_ADJUST = -1
 
 ROBOTSTXT_OBEY = False

From 5dc9a88c347db3497b03949938184ca339f4e9cb Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 24 Jul 2017 18:10:58 +0200
Subject: [PATCH 1543/2974] Handle HTTP 308 Permanent Redirect

---
 scrapy/downloadermiddlewares/redirect.py    |  4 ++--
 tests/test_downloadermiddleware_redirect.py | 12 ++++++++----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 26677e527..30cae3fee 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -64,7 +64,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
                 request.meta.get('handle_httpstatus_all', False)):
             return response
 
-        allowed_status = (301, 302, 303, 307)
+        allowed_status = (301, 302, 303, 307, 308)
         if 'Location' not in response.headers or response.status not in allowed_status:
             return response
 
@@ -72,7 +72,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
 
         redirected_url = urljoin(request.url, location)
 
-        if response.status in (301, 307) or request.method == 'HEAD':
+        if response.status in (301, 307, 308) or request.method == 'HEAD':
             redirected = request.replace(url=redirected_url)
             return self._redirect(redirected, request, spider, response.status)
 
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index e8c92affa..a2da4aa8f 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -22,12 +22,12 @@ class RedirectMiddlewareTest(unittest.TestCase):
         req2 = self.mw.process_response(req, rsp, self.spider)
         assert req2.priority > req.priority
 
-    def test_redirect_301(self):
-        def _test(method):
-            url = 'http://www.example.com/301'
+    def test_redirect_3xx_permanent(self):
+        def _test(method, status=301):
+            url = 'http://www.example.com/{}'.format(status)
             url2 = 'http://www.example.com/redirected'
             req = Request(url, method=method)
-            rsp = Response(url, headers={'Location': url2}, status=301)
+            rsp = Response(url, headers={'Location': url2}, status=status)
 
             req2 = self.mw.process_response(req, rsp, self.spider)
             assert isinstance(req2, Request)
@@ -42,6 +42,10 @@ class RedirectMiddlewareTest(unittest.TestCase):
         _test('POST')
         _test('HEAD')
 
+        _test('GET', status=308)
+        _test('POST', status=308)
+        _test('HEAD', status=308)
+
     def test_dont_redirect(self):
         url = 'http://www.example.com/301'
         url2 = 'http://www.example.com/redirected'

From 1fdc10684fc427e5446350cada465ec934330a3f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 24 Jul 2017 18:25:11 +0200
Subject: [PATCH 1544/2974] HTTP Cache: treat 308 as 301

---
 tests/test_downloadermiddleware_httpcache.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 12b69860a..22946b98c 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -322,6 +322,7 @@ class RFC2616PolicyTest(DefaultStorageTest):
             (True, 203, {'Last-Modified': self.yesterday}),
             (True, 300, {'Last-Modified': self.yesterday}),
             (True, 301, {'Last-Modified': self.yesterday}),
+            (True, 308, {'Last-Modified': self.yesterday}),
             (True, 401, {'Last-Modified': self.yesterday}),
             (True, 404, {'Cache-Control': 'public, max-age=600'}),
             (True, 302, {'Expires': self.tomorrow}),

From 15a5c533fa6f448b7e5cd72ef099725c2295ef6f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 26 Jul 2017 19:07:57 +0200
Subject: [PATCH 1545/2974] Add tests for HTTP 307 permanent redirects

---
 tests/test_downloadermiddleware_redirect.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index a2da4aa8f..35e474418 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -42,6 +42,10 @@ class RedirectMiddlewareTest(unittest.TestCase):
         _test('POST')
         _test('HEAD')
 
+        _test('GET', status=307)
+        _test('POST', status=307)
+        _test('HEAD', status=307)
+
         _test('GET', status=308)
         _test('POST', status=308)
         _test('HEAD', status=308)

From 219c8aa0b622260b9814379be41132367ef33e39 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 27 Jul 2017 17:30:30 +0200
Subject: [PATCH 1546/2974] Log versions information at startup

---
 scrapy/commands/version.py | 40 +++----------------------------
 scrapy/utils/log.py        | 48 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 50 insertions(+), 38 deletions(-)

diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index e22f98f5a..92b70d88b 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -1,12 +1,8 @@
 from __future__ import print_function
-import sys
-import platform
-
-import twisted
-import OpenSSL
 
 import scrapy
 from scrapy.commands import ScrapyCommand
+from scrapy.utils.log import scrapy_components_versions
 
 
 class Command(ScrapyCommand):
@@ -27,38 +23,8 @@ class Command(ScrapyCommand):
 
     def run(self, args, opts):
         if opts.verbose:
-            import cssselect
-            import parsel
-            import lxml.etree
-            import w3lib
-
-            lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
-            libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
-
-            try:
-                w3lib_version = w3lib.__version__
-            except AttributeError:
-                w3lib_version = "<1.14.3"
-
-            print("Scrapy    : %s" % scrapy.__version__)
-            print("lxml      : %s" % lxml_version)
-            print("libxml2   : %s" % libxml2_version)
-            print("cssselect : %s" % cssselect.__version__)
-            print("parsel    : %s" % parsel.__version__)
-            print("w3lib     : %s" % w3lib_version)
-            print("Twisted   : %s" % twisted.version.short())
-            print("Python    : %s" % sys.version.replace("\n", "- "))
-            print("pyOpenSSL : %s" % self._get_openssl_version())
-            print("Platform  : %s" % platform.platform())
+            for name, version in scrapy_components_versions():
+                print("%-9s : %s" % (name, version))
         else:
             print("Scrapy %s" % scrapy.__version__)
 
-    def _get_openssl_version(self):
-        try:
-            openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\
-                .decode('ascii', errors='replace')
-        # pyOpenSSL 0.12 does not expose openssl version
-        except AttributeError:
-            openssl = 'Unknown OpenSSL version'
-
-        return '{} ({})'.format(OpenSSL.version.__version__, openssl)
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 6ceb61a82..660b3c9f5 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -139,10 +139,56 @@ def _get_handler(settings):
     return handler
 
 
+def scrapy_components_versions():
+    import platform
+
+    import cssselect
+    import parsel
+    import lxml.etree
+    import twisted
+    import w3lib
+
+    lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
+    libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
+
+    try:
+        w3lib_version = w3lib.__version__
+    except AttributeError:
+        w3lib_version = "<1.14.3"
+
+    return [
+        ("Scrapy", scrapy.__version__),
+        ("lxml", lxml_version),
+        ("libxml2", libxml2_version),
+        ("cssselect", cssselect.__version__),
+        ("parsel", parsel.__version__),
+        ("w3lib", w3lib_version),
+        ("Twisted", twisted.version.short()),
+        ("Python", sys.version.replace("\n", "- ")),
+        ("pyOpenSSL", _get_openssl_version()),
+        ("Platform",  platform.platform()),
+    ]
+
+
+def _get_openssl_version():
+    try:
+        import OpenSSL
+        openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\
+            .decode('ascii', errors='replace')
+    # pyOpenSSL 0.12 does not expose openssl version
+    except AttributeError:
+        openssl = 'Unknown OpenSSL version'
+
+    return '{} ({})'.format(OpenSSL.version.__version__, openssl)
+
+
 def log_scrapy_info(settings):
     logger.info("Scrapy %(version)s started (bot: %(bot)s)",
                 {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
-
+    logger.info("Versions: %(versions)s}",
+                {'versions': ", ".join("%s %s" % (name, version)
+                    for name, version in scrapy_components_versions()
+                    if name != "Scrapy")})
     d = dict(overridden_settings(settings))
     logger.info("Overridden settings: %(settings)r", {'settings': d})
 

From bf7ef3e4c3ea70a60f6e5f9e5c8a6d842e2b72b9 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 27 Jul 2017 20:07:14 +0200
Subject: [PATCH 1547/2974] Move methods to a new scrapy.utils.versions

---
 scrapy/commands/version.py |  2 +-
 scrapy/utils/log.py        | 45 ++------------------------------------
 scrapy/utils/versions.py   | 44 +++++++++++++++++++++++++++++++++++++
 3 files changed, 47 insertions(+), 44 deletions(-)
 create mode 100644 scrapy/utils/versions.py

diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 92b70d88b..71b1026fa 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -2,7 +2,7 @@ from __future__ import print_function
 
 import scrapy
 from scrapy.commands import ScrapyCommand
-from scrapy.utils.log import scrapy_components_versions
+from scrapy.utils.versions import scrapy_components_versions
 
 
 class Command(ScrapyCommand):
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 660b3c9f5..905c1bfc1 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -11,6 +11,8 @@ from twisted.python import log as twisted_log
 import scrapy
 from scrapy.settings import overridden_settings, Settings
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.versions import scrapy_components_versions
+
 
 logger = logging.getLogger(__name__)
 
@@ -139,49 +141,6 @@ def _get_handler(settings):
     return handler
 
 
-def scrapy_components_versions():
-    import platform
-
-    import cssselect
-    import parsel
-    import lxml.etree
-    import twisted
-    import w3lib
-
-    lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
-    libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
-
-    try:
-        w3lib_version = w3lib.__version__
-    except AttributeError:
-        w3lib_version = "<1.14.3"
-
-    return [
-        ("Scrapy", scrapy.__version__),
-        ("lxml", lxml_version),
-        ("libxml2", libxml2_version),
-        ("cssselect", cssselect.__version__),
-        ("parsel", parsel.__version__),
-        ("w3lib", w3lib_version),
-        ("Twisted", twisted.version.short()),
-        ("Python", sys.version.replace("\n", "- ")),
-        ("pyOpenSSL", _get_openssl_version()),
-        ("Platform",  platform.platform()),
-    ]
-
-
-def _get_openssl_version():
-    try:
-        import OpenSSL
-        openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\
-            .decode('ascii', errors='replace')
-    # pyOpenSSL 0.12 does not expose openssl version
-    except AttributeError:
-        openssl = 'Unknown OpenSSL version'
-
-    return '{} ({})'.format(OpenSSL.version.__version__, openssl)
-
-
 def log_scrapy_info(settings):
     logger.info("Scrapy %(version)s started (bot: %(bot)s)",
                 {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py
new file mode 100644
index 000000000..d2cff09fe
--- /dev/null
+++ b/scrapy/utils/versions.py
@@ -0,0 +1,44 @@
+import platform
+import sys
+
+import cssselect
+import lxml.etree
+import parsel
+import twisted
+import w3lib
+
+import scrapy
+
+
+def scrapy_components_versions():
+    lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
+    libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
+    try:
+        w3lib_version = w3lib.__version__
+    except AttributeError:
+        w3lib_version = "<1.14.3"
+
+    return [
+        ("Scrapy", scrapy.__version__),
+        ("lxml", lxml_version),
+        ("libxml2", libxml2_version),
+        ("cssselect", cssselect.__version__),
+        ("parsel", parsel.__version__),
+        ("w3lib", w3lib_version),
+        ("Twisted", twisted.version.short()),
+        ("Python", sys.version.replace("\n", "- ")),
+        ("pyOpenSSL", _get_openssl_version()),
+        ("Platform",  platform.platform()),
+    ]
+
+
+def _get_openssl_version():
+    try:
+        import OpenSSL
+        openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\
+            .decode('ascii', errors='replace')
+    # pyOpenSSL 0.12 does not expose openssl version
+    except AttributeError:
+        openssl = 'Unknown OpenSSL version'
+
+    return '{} ({})'.format(OpenSSL.version.__version__, openssl)

From aaaa4da7a4e1fe75396028189de88fb9a6604200 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 24 May 2017 12:42:54 +0200
Subject: [PATCH 1548/2974] Add template for a downloader middleware

---
 .../project/module/middlewares.py.tmpl        | 47 +++++++++++++++++++
 .../templates/project/module/settings.py.tmpl |  2 +-
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
index 292bf572e..1a4b0caa5 100644
--- a/scrapy/templates/project/module/middlewares.py.tmpl
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -54,3 +54,50 @@ class ${ProjectName}SpiderMiddleware(object):
 
     def spider_opened(self, spider):
         spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class ${ProjectName}DownloaderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 486df6b71..35a0f9a45 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -53,7 +53,7 @@ ROBOTSTXT_OBEY = True
 # Enable or disable downloader middlewares
 # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
 #DOWNLOADER_MIDDLEWARES = {
-#    '$project_name.middlewares.MyCustomDownloaderMiddleware': 543,
+#    '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543,
 #}
 
 # Enable or disable extensions

From a65fec050ae4a07d233ae886457daf10f14929fe Mon Sep 17 00:00:00 2001
From: simik-ru <simik@simik.ru>
Date: Sun, 30 Jul 2017 17:04:02 +0300
Subject: [PATCH 1549/2974] Small fix in description of startproject arguments

---
 docs/topics/commands.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 8de858f8a..dc8067d7e 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -187,7 +187,7 @@ startproject
 
 Creates a new Scrapy project named ``project_name``, under the ``project_dir``
 directory.
-If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``myproject``.
+If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``project_name``.
 
 Usage example::
 

From 6e6b5cc29f15dbf4f1941fca70dd9c126e4ba556 Mon Sep 17 00:00:00 2001
From: Andrei Petre <p31andrei@gmail.com>
Date: Tue, 1 Aug 2017 17:14:43 +0300
Subject: [PATCH 1550/2974] Use getfullargspec under the scenes for py3 to stop
 DeprecationWarning (#2864)

Use getfullargspec under the scenes for py3 to stop DeprecationWarning.

Closes #2862
---
 scrapy/utils/python.py | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 72f8f4311..732ca13a0 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -10,7 +10,6 @@ import errno
 import six
 from functools import partial, wraps
 import sys
-import time
 
 from scrapy.utils.decorators import deprecated
 
@@ -198,10 +197,30 @@ def binary_is_text(data):
     return all(c not in _BINARYCHARS for c in data)
 
 
+def _getargspec_py23(func):
+    """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords,
+                                                        defaults)
+
+    Identical to inspect.getargspec() in python2, but uses
+    inspect.getfullargspec() for python3 behind the scenes to avoid
+    DeprecationWarning.
+
+    >>> def f(a, b=2, *ar, **kw):
+    ...     pass
+
+    >>> _getargspec_py23(f)
+    ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,))
+    """
+    if six.PY2:
+        return inspect.getargspec(func)
+
+    return inspect.ArgSpec(*inspect.getfullargspec(func)[:4])
+
+
 def get_func_args(func, stripself=False):
     """Return the argument name list of a callable"""
     if inspect.isfunction(func):
-        func_args, _, _, _ = inspect.getargspec(func)
+        func_args, _, _, _ = _getargspec_py23(func)
     elif inspect.isclass(func):
         return get_func_args(func.__init__, True)
     elif inspect.ismethod(func):
@@ -248,9 +267,9 @@ def get_spec(func):
     """
 
     if inspect.isfunction(func) or inspect.ismethod(func):
-        spec = inspect.getargspec(func)
+        spec = _getargspec_py23(func)
     elif hasattr(func, '__call__'):
-        spec = inspect.getargspec(func.__call__)
+        spec = _getargspec_py23(func.__call__)
     else:
         raise TypeError('%s is not callable' % type(func))
 

From 71d5b7d75a579360ab02b9f9594199993ff91f61 Mon Sep 17 00:00:00 2001
From: david watson <davidthewatson@gmail.com>
Date: Tue, 1 Aug 2017 13:49:22 -0400
Subject: [PATCH 1551/2974] fix typo (#2867)

---
 docs/topics/broad-crawls.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index 040c8cfde..eb02086dc 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -20,7 +20,7 @@ These are some common properties often found in broad crawls:
 
 * they crawl many domains (often, unbounded) instead of a specific set of sites
 
-* they don't necessarily crawl domains to completion, because it would
+* they don't necessarily crawl domains to completion, because it would be
   impractical (or impossible) to do so, and instead limit the crawl by time or
   number of pages crawled
 

From 01ac8838934071b89c5c711688f3d833387e27d4 Mon Sep 17 00:00:00 2001
From: Eugene Vorobev <jenya.vv@gmail.com>
Date: Wed, 26 Jul 2017 19:51:27 +0300
Subject: [PATCH 1552/2974] Follow alternate link for all types of sitemaps

---
 scrapy/spiders/sitemap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index e54001d88..0ee8ba5e7 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -48,7 +48,7 @@ class SitemapSpider(Spider):
                     if any(x.search(loc) for x in self._follow):
                         yield Request(loc, callback=self._parse_sitemap)
             elif s.type == 'urlset':
-                for loc in iterloc(s):
+                for loc in iterloc(s, self.sitemap_alternate_links):
                     for r, c in self._cbs:
                         if r.search(loc):
                             yield Request(loc, callback=c)

From 0cb3085f8453265a8f37de684e62eda8b5398c75 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 3 Aug 2017 16:35:47 +0200
Subject: [PATCH 1553/2974] Add test for alternate links

---
 tests/test_spider.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/tests/test_spider.py b/tests/test_spider.py
index e55f0fa6d..2507964b5 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -348,6 +348,33 @@ Sitemap: /sitemap-relative-url.xml
                           'http://example.com/sitemap-uppercase.xml',
                           'http://www.example.com/sitemap-relative-url.xml'])
 
+    def test_alternate_url_locs(self):
+        sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:xhtml="http://www.w3.org/1999/xhtml">
+        <url>
+            <loc>http://www.example.com/english/</loc>
+            <xhtml:link rel="alternate" hreflang="de"
+                href="http://www.example.com/deutsch/"/>
+            <xhtml:link rel="alternate" hreflang="de-ch"
+                href="http://www.example.com/schweiz-deutsch/"/>
+            <xhtml:link rel="alternate" hreflang="it"
+                href="http://www.example.com/italiano/"/>
+            <xhtml:link rel="alternate" hreflang="it"/><!-- wrong tag without href -->
+        </url>
+    </urlset>"""
+        r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
+        spider = self.spider_class("example.com")
+        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
+                         ['http://www.example.com/english/'])
+
+        spider.sitemap_alternate_links = True
+        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
+                         ['http://www.example.com/english/',
+                          'http://www.example.com/deutsch/',
+                          'http://www.example.com/schweiz-deutsch/',
+                          'http://www.example.com/italiano/'])
+
 
 class DeprecationTest(unittest.TestCase):
 

From c016a4309dbb045c17842e329043b7d9951e8f14 Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@bluewin.ch>
Date: Fri, 4 Aug 2017 01:44:23 +0200
Subject: [PATCH 1554/2974] # noqa to close #2836

Marks #2836 as will not fix.
---
 tests/test_item.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_item.py b/tests/test_item.py
index 85a554de0..3c645649c 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -270,7 +270,7 @@ class ItemMetaTest(unittest.TestCase):
                 def f(self):
                     # For rationale of this see:
                     # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222
-                    return __class__
+                    return __class__  # noqa https://github.com/scrapy/scrapy/issues/2836
 
             MyItem()
 

From 0a69a32b5a2cab7575fdbb5f2cd4b7c7b900aabc Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@bluewin.ch>
Date: Fri, 4 Aug 2017 14:35:43 +0200
Subject: [PATCH 1555/2974] Force Travis CI to test again

---
 tests/test_item.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_item.py b/tests/test_item.py
index 3c645649c..2c1eb0dd3 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -270,7 +270,7 @@ class ItemMetaTest(unittest.TestCase):
                 def f(self):
                     # For rationale of this see:
                     # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222
-                    return __class__  # noqa https://github.com/scrapy/scrapy/issues/2836
+                    return __class__  # noqa  https://github.com/scrapy/scrapy/issues/2836
 
             MyItem()
 

From be71f98e92688c759d3af48101617229dcdfe05f Mon Sep 17 00:00:00 2001
From: kirankoduru <crazycreature11@gmail.com>
Date: Sat, 29 Jul 2017 20:51:54 -0400
Subject: [PATCH 1556/2974] Explicit message for scrapy parse callback

The scrapy parse method raises a NotImplementedError when not defined,
but for new comers it can be hard to debug what might be going wrong.

Adding an explicit message for NotImplementedError will help new users.
---
 scrapy/spiders/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 30cb7590a..c6b92f8eb 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -87,7 +87,7 @@ class Spider(object_ref):
         return Request(url, dont_filter=True)
 
     def parse(self, response):
-        raise NotImplementedError
+        raise NotImplementedError('Spider.parse callback is not defined')
 
     @classmethod
     def update_settings(cls, settings):

From 7adab61a7a5f88c78311cc44a468c7b8d0a4c954 Mon Sep 17 00:00:00 2001
From: kirankoduru <crazycreature11@gmail.com>
Date: Tue, 1 Aug 2017 22:42:26 -0400
Subject: [PATCH 1557/2974] Added test for NotImplemented Spider.parse method

---
 tests/test_spider.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tests/test_spider.py b/tests/test_spider.py
index e55f0fa6d..6c845d826 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -429,3 +429,17 @@ class DeprecationTest(unittest.TestCase):
             self.assertEqual(len(requests), 1)
             self.assertEqual(requests[0].url, 'http://example.com/foo')
             self.assertEqual(len(w), 1)
+
+
+class NoParseMethodSpiderTest(unittest.TestCase):
+
+    spider_class = Spider
+
+    def test_undefined_parse_method(self):
+        spider = self.spider_class('example.com')
+        text = 'Random text response'
+        resp = TextResponse(url="http://www.example.com/random_url", body=text)
+
+        exc_msg = 'Spider.parse callback is not defined'
+        with self.assertRaisesRegexp(NotImplementedError, exc_msg):
+            spider.parse(resp)

From 12409a0cf6c37ff5c19588bb064690549798bb37 Mon Sep 17 00:00:00 2001
From: Kiran Koduru <crazycreature11@gmail.com>
Date: Wed, 2 Aug 2017 08:32:38 -0400
Subject: [PATCH 1558/2974] Fix broken encoding on text for py 3

---
 tests/test_spider.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_spider.py b/tests/test_spider.py
index 6c845d826..6a52b3ea7 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -437,7 +437,7 @@ class NoParseMethodSpiderTest(unittest.TestCase):
 
     def test_undefined_parse_method(self):
         spider = self.spider_class('example.com')
-        text = 'Random text response'
+        text = b'Random text'
         resp = TextResponse(url="http://www.example.com/random_url", body=text)
 
         exc_msg = 'Spider.parse callback is not defined'

From 2960c9b5683dceb149823e7f927d8c86ee83deb8 Mon Sep 17 00:00:00 2001
From: Kiran Koduru <crazycreature11@gmail.com>
Date: Sat, 5 Aug 2017 16:29:41 -0400
Subject: [PATCH 1559/2974] Use self.__class__.__name__ instead of showing
 generic Spider class name

---
 scrapy/spiders/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index c6b92f8eb..e9c131e3b 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -87,7 +87,7 @@ class Spider(object_ref):
         return Request(url, dont_filter=True)
 
     def parse(self, response):
-        raise NotImplementedError('Spider.parse callback is not defined')
+        raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))
 
     @classmethod
     def update_settings(cls, settings):

From 4ca61a20512b7306e7266e6667a0c02ae5ebe557 Mon Sep 17 00:00:00 2001
From: Chomba Ng'ang'a <choeringa@gmail.com>
Date: Mon, 7 Aug 2017 18:29:36 +0300
Subject: [PATCH 1560/2974] Update deprecated test aliases

- change ``failIf`` to ``assertFalse``
- change ``asertEquals`` to ``assertEqual``
- change ``assert_`` to ``assertTrue``

https://docs.python.org/2/library/unittest.html#deprecated-aliases
---
 tests/test_cmdline/__init__.py                |  2 +-
 tests/test_downloader_handlers.py             | 88 +++++++++----------
 tests/test_downloadermiddleware_cookies.py    | 20 ++---
 ...est_downloadermiddleware_defaultheaders.py |  6 +-
 ...st_downloadermiddleware_downloadtimeout.py |  8 +-
 tests/test_downloadermiddleware_httpauth.py   |  4 +-
 ...st_downloadermiddleware_httpcompression.py |  2 +-
 tests/test_downloadermiddleware_httpproxy.py  | 44 +++++-----
 tests/test_downloadermiddleware_redirect.py   |  4 +-
 tests/test_downloadermiddleware_useragent.py  |  6 +-
 tests/test_http_cookies.py                    |  2 +-
 tests/test_http_request.py                    |  4 +-
 tests/test_loader.py                          | 18 ++--
 tests/test_pipeline_images.py                 | 16 ++--
 tests/test_selector.py                        |  2 +-
 tests/test_spider.py                          |  6 +-
 tests/test_spidermiddleware_depth.py          |  8 +-
 tests/test_spidermiddleware_httperror.py      | 24 ++---
 tests/test_spidermiddleware_offsite.py        |  6 +-
 tests/test_spidermiddleware_referer.py        | 10 +--
 tests/test_spidermiddleware_urllength.py      |  2 +-
 tests/test_urlparse_monkeypatches.py          |  8 +-
 tests/test_utils_datatypes.py                 |  8 +-
 tests/test_utils_defer.py                     |  2 +-
 tests/test_utils_iterators.py                 |  4 +-
 tests/test_utils_misc/__init__.py             |  8 +-
 tests/test_utils_project.py                   |  8 +-
 tests/test_utils_python.py                    | 26 +++---
 tests/test_utils_signal.py                    |  2 +-
 tests/test_webclient.py                       | 28 +++---
 30 files changed, 188 insertions(+), 188 deletions(-)

diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 7733e7180..10076bbca 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -68,4 +68,4 @@ class CmdlineTest(unittest.TestCase):
             settingsstr = settingsstr.replace(char, '"')
         settingsdict = json.loads(settingsstr)
         six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys())
-        self.assertEquals(200, settingsdict[EXT_PATH])
+        self.assertEqual(200, settingsdict[EXT_PATH])
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 7d88dbcba..bd2c86292 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -98,9 +98,9 @@ class FileTestCase(unittest.TestCase):
 
     def test_download(self):
         def _test(response):
-            self.assertEquals(response.url, request.url)
-            self.assertEquals(response.status, 200)
-            self.assertEquals(response.body, b'0123456789')
+            self.assertEqual(response.url, request.url)
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.body, b'0123456789')
 
         request = Request(path_to_file_uri(self.tmpname + '^'))
         assert request.url.upper().endswith('%5E')
@@ -241,28 +241,28 @@ class HttpTestCase(unittest.TestCase):
         request = Request(self.getURL('file'))
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, b"0123456789")
+        d.addCallback(self.assertEqual, b"0123456789")
         return d
 
     def test_download_head(self):
         request = Request(self.getURL('file'), method='HEAD')
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, b'')
+        d.addCallback(self.assertEqual, b'')
         return d
 
     def test_redirect_status(self):
         request = Request(self.getURL('redirect'))
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.status)
-        d.addCallback(self.assertEquals, 302)
+        d.addCallback(self.assertEqual, 302)
         return d
 
     def test_redirect_status_head(self):
         request = Request(self.getURL('redirect'), method='HEAD')
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.status)
-        d.addCallback(self.assertEquals, 302)
+        d.addCallback(self.assertEqual, 302)
         return d
 
     @defer.inlineCallbacks
@@ -285,24 +285,24 @@ class HttpTestCase(unittest.TestCase):
 
     def test_host_header_not_in_request_headers(self):
         def _test(response):
-            self.assertEquals(
+            self.assertEqual(
                 response.body, to_bytes('%s:%d' % (self.host, self.portno)))
-            self.assertEquals(request.headers, {})
+            self.assertEqual(request.headers, {})
 
         request = Request(self.getURL('host'))
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
     def test_host_header_seted_in_request_headers(self):
         def _test(response):
-            self.assertEquals(response.body, b'example.com')
-            self.assertEquals(request.headers.get('Host'), b'example.com')
+            self.assertEqual(response.body, b'example.com')
+            self.assertEqual(request.headers.get('Host'), b'example.com')
 
         request = Request(self.getURL('host'), headers={'Host': 'example.com'})
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, b'example.com')
+        d.addCallback(self.assertEqual, b'example.com')
         return d
 
     def test_content_length_zero_bodyless_post_request_headers(self):
@@ -317,7 +317,7 @@ class HttpTestCase(unittest.TestCase):
         https://bugs.python.org/issue14721
         """
         def _test(response):
-            self.assertEquals(response.body, b'0')
+            self.assertEqual(response.body, b'0')
 
         request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'})
         return self.download_request(request, Spider('foo')).addCallback(_test)
@@ -327,8 +327,8 @@ class HttpTestCase(unittest.TestCase):
             import json
             headers = Headers(json.loads(response.text)['headers'])
             contentlengths = headers.getlist('Content-Length')
-            self.assertEquals(len(contentlengths), 1)
-            self.assertEquals(contentlengths, [b"0"])
+            self.assertEqual(len(contentlengths), 1)
+            self.assertEqual(contentlengths, [b"0"])
 
         request = Request(self.getURL('echo'), method='POST')
         return self.download_request(request, Spider('foo')).addCallback(_test)
@@ -338,7 +338,7 @@ class HttpTestCase(unittest.TestCase):
         request = Request(self.getURL('payload'), method='POST', body=body)
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, body)
+        d.addCallback(self.assertEqual, body)
         return d
 
 
@@ -364,7 +364,7 @@ class Http11TestCase(HttpTestCase):
         request = Request(self.getURL('file'))
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, b"0123456789")
+        d.addCallback(self.assertEqual, b"0123456789")
         return d
 
     def test_response_class_choosing_request(self):
@@ -374,7 +374,7 @@ class Http11TestCase(HttpTestCase):
         body = b'Some plain text\ndata with tabs\t and null bytes\0'
 
         def _test_type(response):
-            self.assertEquals(type(response), TextResponse)
+            self.assertEqual(type(response), TextResponse)
 
         request = Request(self.getURL('nocontenttype'), body=body)
         d = self.download_request(request, Spider('foo'))
@@ -389,7 +389,7 @@ class Http11TestCase(HttpTestCase):
         # response body. (regardless of headers)
         d = self.download_request(request, Spider('foo', download_maxsize=10))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, b"0123456789")
+        d.addCallback(self.assertEqual, b"0123456789")
         yield d
 
         d = self.download_request(request, Spider('foo', download_maxsize=9))
@@ -431,14 +431,14 @@ class Http11TestCase(HttpTestCase):
         request = Request(self.getURL('file'))
         d = self.download_request(request, Spider('foo', download_maxsize=100))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, b"0123456789")
+        d.addCallback(self.assertEqual, b"0123456789")
         return d
 
     def test_download_chunked_content(self):
         request = Request(self.getURL('chunked'))
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, b"chunked content\n")
+        d.addCallback(self.assertEqual, b"chunked content\n")
         return d
 
     def test_download_broken_content_cause_data_loss(self, url='broken'):
@@ -597,9 +597,9 @@ class HttpProxyTestCase(unittest.TestCase):
 
     def test_download_with_proxy(self):
         def _test(response):
-            self.assertEquals(response.status, 200)
-            self.assertEquals(response.url, request.url)
-            self.assertEquals(response.body, b'http://example.com')
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.url, request.url)
+            self.assertEqual(response.body, b'http://example.com')
 
         http_proxy = self.getURL('')
         request = Request('http://example.com', meta={'proxy': http_proxy})
@@ -607,9 +607,9 @@ class HttpProxyTestCase(unittest.TestCase):
 
     def test_download_with_proxy_https_noconnect(self):
         def _test(response):
-            self.assertEquals(response.status, 200)
-            self.assertEquals(response.url, request.url)
-            self.assertEquals(response.body, b'https://example.com')
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.url, request.url)
+            self.assertEqual(response.body, b'https://example.com')
 
         http_proxy = '%s?noconnect' % self.getURL('')
         request = Request('https://example.com', meta={'proxy': http_proxy})
@@ -617,9 +617,9 @@ class HttpProxyTestCase(unittest.TestCase):
 
     def test_download_without_proxy(self):
         def _test(response):
-            self.assertEquals(response.status, 200)
-            self.assertEquals(response.url, request.url)
-            self.assertEquals(response.body, b'/path/to/resource')
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.url, request.url)
+            self.assertEqual(response.body, b'/path/to/resource')
 
         request = Request(self.getURL('path/to/resource'))
         return self.download_request(request, Spider('foo')).addCallback(_test)
@@ -978,7 +978,7 @@ class DataURITestCase(unittest.TestCase):
         uri = "data:,A%20brief%20note"
 
         def _test(response):
-            self.assertEquals(response.url, uri)
+            self.assertEqual(response.url, uri)
             self.assertFalse(response.headers)
 
         request = Request(uri)
@@ -986,39 +986,39 @@ class DataURITestCase(unittest.TestCase):
 
     def test_default_mediatype_encoding(self):
         def _test(response):
-            self.assertEquals(response.text, 'A brief note')
-            self.assertEquals(type(response),
+            self.assertEqual(response.text, 'A brief note')
+            self.assertEqual(type(response),
                               responsetypes.from_mimetype("text/plain"))
-            self.assertEquals(response.encoding, "US-ASCII")
+            self.assertEqual(response.encoding, "US-ASCII")
 
         request = Request("data:,A%20brief%20note")
         return self.download_request(request, self.spider).addCallback(_test)
 
     def test_default_mediatype(self):
         def _test(response):
-            self.assertEquals(response.text, u'\u038e\u03a3\u038e')
-            self.assertEquals(type(response),
+            self.assertEqual(response.text, u'\u038e\u03a3\u038e')
+            self.assertEqual(type(response),
                               responsetypes.from_mimetype("text/plain"))
-            self.assertEquals(response.encoding, "iso-8859-7")
+            self.assertEqual(response.encoding, "iso-8859-7")
 
         request = Request("data:;charset=iso-8859-7,%be%d3%be")
         return self.download_request(request, self.spider).addCallback(_test)
 
     def test_text_charset(self):
         def _test(response):
-            self.assertEquals(response.text, u'\u038e\u03a3\u038e')
-            self.assertEquals(response.body, b'\xbe\xd3\xbe')
-            self.assertEquals(response.encoding, "iso-8859-7")
+            self.assertEqual(response.text, u'\u038e\u03a3\u038e')
+            self.assertEqual(response.body, b'\xbe\xd3\xbe')
+            self.assertEqual(response.encoding, "iso-8859-7")
 
         request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be")
         return self.download_request(request, self.spider).addCallback(_test)
 
     def test_mediatype_parameters(self):
         def _test(response):
-            self.assertEquals(response.text, u'\u038e\u03a3\u038e')
-            self.assertEquals(type(response),
+            self.assertEqual(response.text, u'\u038e\u03a3\u038e')
+            self.assertEqual(type(response),
                               responsetypes.from_mimetype("text/plain"))
-            self.assertEquals(response.encoding, "utf-8")
+            self.assertEqual(response.encoding, "utf-8")
 
         request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;'
                           'charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22'
@@ -1027,7 +1027,7 @@ class DataURITestCase(unittest.TestCase):
 
     def test_base64(self):
         def _test(response):
-            self.assertEquals(response.text, 'Hello, world.')
+            self.assertEqual(response.text, 'Hello, world.')
 
         request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D')
         return self.download_request(request, self.spider).addCallback(_test)
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 26d9794b6..17801e502 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -36,7 +36,7 @@ class CookiesMiddlewareTest(TestCase):
 
         req2 = Request('http://scrapytest.org/sub1/')
         assert self.mw.process_request(req2, self.spider) is None
-        self.assertEquals(req2.headers.get('Cookie'), b"C1=value1")
+        self.assertEqual(req2.headers.get('Cookie'), b"C1=value1")
 
     def test_setting_false_cookies_enabled(self):
         self.assertRaises(
@@ -131,12 +131,12 @@ class CookiesMiddlewareTest(TestCase):
         # check that cookies are merged back
         req = Request('http://scrapytest.org/mergeme')
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEquals(req.headers.get('Cookie'), b'C1=value1')
+        self.assertEqual(req.headers.get('Cookie'), b'C1=value1')
 
         # check that cookies are merged when dont_merge_cookies is passed as 0
         req = Request('http://scrapytest.org/mergeme', meta={'dont_merge_cookies': 0})
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEquals(req.headers.get('Cookie'), b'C1=value1')
+        self.assertEqual(req.headers.get('Cookie'), b'C1=value1')
 
     def test_complex_cookies(self):
         # merge some cookies into jar
@@ -157,7 +157,7 @@ class CookiesMiddlewareTest(TestCase):
         # embed C2 for scrapytest.org/bar
         req = Request('http://scrapytest.org/bar')
         self.mw.process_request(req, self.spider)
-        self.assertEquals(req.headers.get('Cookie'), b'C2=value2')
+        self.assertEqual(req.headers.get('Cookie'), b'C2=value2')
 
         # embed nothing for scrapytest.org/baz
         req = Request('http://scrapytest.org/baz')
@@ -167,7 +167,7 @@ class CookiesMiddlewareTest(TestCase):
     def test_merge_request_cookies(self):
         req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'})
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEquals(req.headers.get('Cookie'), b'galleta=salada')
+        self.assertEqual(req.headers.get('Cookie'), b'galleta=salada')
 
         headers = {'Set-Cookie': 'C1=value1; path=/'}
         res = Response('http://scrapytest.org/', headers=headers)
@@ -181,7 +181,7 @@ class CookiesMiddlewareTest(TestCase):
     def test_cookiejar_key(self):
         req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}, meta={'cookiejar': "store1"})
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEquals(req.headers.get('Cookie'), b'galleta=salada')
+        self.assertEqual(req.headers.get('Cookie'), b'galleta=salada')
 
         headers = {'Set-Cookie': 'C1=value1; path=/'}
         res = Response('http://scrapytest.org/', headers=headers, request=req)
@@ -193,7 +193,7 @@ class CookiesMiddlewareTest(TestCase):
 
         req3 = Request('http://scrapytest.org/', cookies={'galleta': 'dulce'}, meta={'cookiejar': "store2"})
         assert self.mw.process_request(req3, self.spider) is None
-        self.assertEquals(req3.headers.get('Cookie'), b'galleta=dulce')
+        self.assertEqual(req3.headers.get('Cookie'), b'galleta=dulce')
 
         headers = {'Set-Cookie': 'C2=value2; path=/'}
         res2 = Response('http://scrapytest.org/', headers=headers, request=req3)
@@ -213,16 +213,16 @@ class CookiesMiddlewareTest(TestCase):
 
         req5_2 = Request('http://scrapytest.org:1104/some-redirected-path')
         assert self.mw.process_request(req5_2, self.spider) is None
-        self.assertEquals(req5_2.headers.get('Cookie'), b'C1=value1')
+        self.assertEqual(req5_2.headers.get('Cookie'), b'C1=value1')
 
         req5_3 = Request('http://scrapytest.org/some-redirected-path')
         assert self.mw.process_request(req5_3, self.spider) is None
-        self.assertEquals(req5_3.headers.get('Cookie'), b'C1=value1')
+        self.assertEqual(req5_3.headers.get('Cookie'), b'C1=value1')
 
         #skip cookie retrieval for not http request
         req6 = Request('file:///scrapy/sometempfile')
         assert self.mw.process_request(req6, self.spider) is None
-        self.assertEquals(req6.headers.get('Cookie'), None)
+        self.assertEqual(req6.headers.get('Cookie'), None)
 
     def test_local_domain(self):
         request = Request("http://example-host/", cookies={'currencyCookie': 'USD'})
diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py
index 80efa83f9..6a31dfcf8 100644
--- a/tests/test_downloadermiddleware_defaultheaders.py
+++ b/tests/test_downloadermiddleware_defaultheaders.py
@@ -22,15 +22,15 @@ class TestDefaultHeadersMiddleware(TestCase):
         defaults, spider, mw = self.get_defaults_spider_mw()
         req = Request('http://www.scrapytest.org')
         mw.process_request(req, spider)
-        self.assertEquals(req.headers, defaults)
+        self.assertEqual(req.headers, defaults)
 
     def test_update_headers(self):
         defaults, spider, mw = self.get_defaults_spider_mw()
         headers = {'Accept-Language': ['es'], 'Test-Header': ['test']}
         bytes_headers = {b'Accept-Language': [b'es'], b'Test-Header': [b'test']}
         req = Request('http://www.scrapytest.org', headers=headers)
-        self.assertEquals(req.headers, bytes_headers)
+        self.assertEqual(req.headers, bytes_headers)
 
         mw.process_request(req, spider)
         defaults.update(bytes_headers)
-        self.assertEquals(req.headers, defaults)
+        self.assertEqual(req.headers, defaults)
diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py
index 446a99f36..586bdc0d1 100644
--- a/tests/test_downloadermiddleware_downloadtimeout.py
+++ b/tests/test_downloadermiddleware_downloadtimeout.py
@@ -18,20 +18,20 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase):
         req, spider, mw = self.get_request_spider_mw()
         mw.spider_opened(spider)
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.meta.get('download_timeout'), 180)
+        self.assertEqual(req.meta.get('download_timeout'), 180)
 
     def test_string_download_timeout(self):
         req, spider, mw = self.get_request_spider_mw({'DOWNLOAD_TIMEOUT': '20.1'})
         mw.spider_opened(spider)
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.meta.get('download_timeout'), 20.1)
+        self.assertEqual(req.meta.get('download_timeout'), 20.1)
 
     def test_spider_has_download_timeout(self):
         req, spider, mw = self.get_request_spider_mw()
         spider.download_timeout = 2
         mw.spider_opened(spider)
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.meta.get('download_timeout'), 2)
+        self.assertEqual(req.meta.get('download_timeout'), 2)
 
     def test_request_has_download_timeout(self):
         req, spider, mw = self.get_request_spider_mw()
@@ -39,4 +39,4 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase):
         mw.spider_opened(spider)
         req.meta['download_timeout'] = 1
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.meta.get('download_timeout'), 1)
+        self.assertEqual(req.meta.get('download_timeout'), 1)
diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py
index 425a5cc79..3381632b0 100644
--- a/tests/test_downloadermiddleware_httpauth.py
+++ b/tests/test_downloadermiddleware_httpauth.py
@@ -23,10 +23,10 @@ class HttpAuthMiddlewareTest(unittest.TestCase):
     def test_auth(self):
         req = Request('http://scrapytest.org/')
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEquals(req.headers['Authorization'], b'Basic Zm9vOmJhcg==')
+        self.assertEqual(req.headers['Authorization'], b'Basic Zm9vOmJhcg==')
 
     def test_auth_already_set(self):
         req = Request('http://scrapytest.org/',
                       headers=dict(Authorization='Digest 123'))
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEquals(req.headers['Authorization'], b'Digest 123')
+        self.assertEqual(req.headers['Authorization'], b'Digest 123')
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 0678fcb14..0745c8dd3 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -248,4 +248,4 @@ class HttpCompressionTest(TestCase):
         response = response.replace(body = None)
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIs(newresponse, response)
-        self.assertEquals(response.body, b'')
+        self.assertEqual(response.body, b'')
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index c77179ceb..0ea83aaf9 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -35,8 +35,8 @@ class TestDefaultHeadersMiddleware(TestCase):
         for url in ('http://e.com', 'https://e.com', 'file:///tmp/a'):
             req = Request(url)
             assert mw.process_request(req, spider) is None
-            self.assertEquals(req.url, url)
-            self.assertEquals(req.meta, {})
+            self.assertEqual(req.url, url)
+            self.assertEqual(req.meta, {})
 
     def test_enviroment_proxies(self):
         os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
@@ -48,41 +48,41 @@ class TestDefaultHeadersMiddleware(TestCase):
                 ('https://e.com', https_proxy), ('file://tmp/a', None)]:
             req = Request(url)
             assert mw.process_request(req, spider) is None
-            self.assertEquals(req.url, url)
-            self.assertEquals(req.meta.get('proxy'), proxy)
+            self.assertEqual(req.url, url)
+            self.assertEqual(req.meta.get('proxy'), proxy)
 
     def test_proxy_precedence_meta(self):
         os.environ['http_proxy'] = 'https://proxy.com'
         mw = HttpProxyMiddleware()
         req = Request('http://scrapytest.org', meta={'proxy': 'https://new.proxy:3128'})
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.meta, {'proxy': 'https://new.proxy:3128'})
+        self.assertEqual(req.meta, {'proxy': 'https://new.proxy:3128'})
 
     def test_proxy_auth(self):
         os.environ['http_proxy'] = 'https://user:pass@proxy:3128'
         mw = HttpProxyMiddleware()
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
-        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz')
+        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz')
         # proxy from request.meta
         req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'})
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
-        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=')
+        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=')
 
     def test_proxy_auth_empty_passwd(self):
         os.environ['http_proxy'] = 'https://user:@proxy:3128'
         mw = HttpProxyMiddleware()
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
-        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=')
+        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=')
         # proxy from request.meta
         req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'})
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
-        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6')
+        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6')
 
     def test_proxy_auth_encoding(self):
         # utf-8 encoding
@@ -90,27 +90,27 @@ class TestDefaultHeadersMiddleware(TestCase):
         mw = HttpProxyMiddleware(auth_encoding='utf-8')
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
-        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
+        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
 
         # proxy from request.meta
         req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
-        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==')
+        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==')
 
         # default latin-1 encoding
         mw = HttpProxyMiddleware(auth_encoding='latin-1')
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
-        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
+        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
 
         # proxy from request.meta, latin-1 encoding
         req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
-        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz')
+        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz')
 
     def test_proxy_already_seted(self):
         os.environ['http_proxy'] = 'https://proxy.for.http:3128'
@@ -142,4 +142,4 @@ class TestDefaultHeadersMiddleware(TestCase):
         os.environ['no_proxy'] = '*'
         req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'})
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.meta, {'proxy': 'http://proxy.com'})
+        self.assertEqual(req.meta, {'proxy': 'http://proxy.com'})
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 35e474418..74137b4cd 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -166,7 +166,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
         resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302)
         req_result = self.mw.process_response(req, resp, self.spider)
         perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o'
-        self.assertEquals(perc_encoded_utf8_url, req_result.url)
+        self.assertEqual(perc_encoded_utf8_url, req_result.url)
 
     def test_utf8_location(self):
         req = Request('http://scrapytest.org/first')
@@ -174,7 +174,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
         resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302)
         req_result = self.mw.process_response(req, resp, self.spider)
         perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
-        self.assertEquals(perc_encoded_utf8_url, req_result.url)
+        self.assertEqual(perc_encoded_utf8_url, req_result.url)
 
 
 class MetaRefreshMiddlewareTest(unittest.TestCase):
diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py
index 1e41fdace..a286764fd 100644
--- a/tests/test_downloadermiddleware_useragent.py
+++ b/tests/test_downloadermiddleware_useragent.py
@@ -17,7 +17,7 @@ class UserAgentMiddlewareTest(TestCase):
         spider, mw = self.get_spider_and_mw('default_useragent')
         req = Request('http://scrapytest.org/')
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.headers['User-Agent'], b'default_useragent')
+        self.assertEqual(req.headers['User-Agent'], b'default_useragent')
 
     def test_remove_agent(self):
         # settings UESR_AGENT to None should remove the user agent
@@ -34,7 +34,7 @@ class UserAgentMiddlewareTest(TestCase):
         mw.spider_opened(spider)
         req = Request('http://scrapytest.org/')
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.headers['User-Agent'], b'spider_useragent')
+        self.assertEqual(req.headers['User-Agent'], b'spider_useragent')
 
     def test_header_agent(self):
         spider, mw = self.get_spider_and_mw('default_useragent')
@@ -43,7 +43,7 @@ class UserAgentMiddlewareTest(TestCase):
         req = Request('http://scrapytest.org/',
                       headers={'User-Agent': 'header_useragent'})
         assert mw.process_request(req, spider) is None
-        self.assertEquals(req.headers['User-Agent'], b'header_useragent')
+        self.assertEqual(req.headers['User-Agent'], b'header_useragent')
 
     def test_no_agent(self):
         spider, mw = self.get_spider_and_mw(None)
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index 549f779d8..caa6fe83e 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -62,7 +62,7 @@ class WrappedResponseTest(TestCase):
         self.wrapped = WrappedResponse(self.response)
 
     def test_info(self):
-        self.assert_(self.wrapped.info() is self.wrapped)
+        self.assertTrue(self.wrapped.info() is self.wrapped)
 
     def test_getheaders(self):
         self.assertEqual(self.wrapped.getheaders('content-type'), ['text/html'])
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 9b0ee63dc..21c0dd746 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -64,9 +64,9 @@ class RequestTest(unittest.TestCase):
         h = Headers({'key1': u'val1', u'key2': 'val2'})
         h[u'newkey'] = u'newval'
         for k, v in h.iteritems():
-            self.assert_(isinstance(k, bytes))
+            self.assertTrue(isinstance(k, bytes))
             for s in v:
-                self.assert_(isinstance(s, bytes))
+                self.assertTrue(isinstance(s, bytes))
 
     def test_eq(self):
         url = 'http://www.scrapy.org'
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 9d07eb95b..2569ccf5e 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -437,7 +437,7 @@ class ProcessorsTest(unittest.TestCase):
         self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
         self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
         self.assertEqual(proc(['hello', 'world']), u'hello world')
-        self.assert_(isinstance(proc(['hello', 'world']), six.text_type))
+        self.assertTrue(isinstance(proc(['hello', 'world']), six.text_type))
 
     def test_compose(self):
         proc = Compose(lambda v: v[0], str.upper)
@@ -482,7 +482,7 @@ class SelectortemLoaderTest(unittest.TestCase):
     def test_constructor_with_selector(self):
         sel = Selector(text=u"<html><body><div>marta</div></body></html>")
         l = TestItemLoader(selector=sel)
-        self.assert_(l.selector is sel)
+        self.assertTrue(l.selector is sel)
 
         l.add_xpath('name', '//div/text()')
         self.assertEqual(l.get_output_value('name'), [u'Marta'])
@@ -490,21 +490,21 @@ class SelectortemLoaderTest(unittest.TestCase):
     def test_constructor_with_selector_css(self):
         sel = Selector(text=u"<html><body><div>marta</div></body></html>")
         l = TestItemLoader(selector=sel)
-        self.assert_(l.selector is sel)
+        self.assertTrue(l.selector is sel)
 
         l.add_css('name', 'div::text')
         self.assertEqual(l.get_output_value('name'), [u'Marta'])
 
     def test_constructor_with_response(self):
         l = TestItemLoader(response=self.response)
-        self.assert_(l.selector)
+        self.assertTrue(l.selector)
 
         l.add_xpath('name', '//div/text()')
         self.assertEqual(l.get_output_value('name'), [u'Marta'])
 
     def test_constructor_with_response_css(self):
         l = TestItemLoader(response=self.response)
-        self.assert_(l.selector)
+        self.assertTrue(l.selector)
 
         l.add_css('name', 'div::text')
         self.assertEqual(l.get_output_value('name'), [u'Marta'])
@@ -526,7 +526,7 @@ class SelectortemLoaderTest(unittest.TestCase):
 
     def test_replace_xpath(self):
         l = TestItemLoader(response=self.response)
-        self.assert_(l.selector)
+        self.assertTrue(l.selector)
         l.add_xpath('name', '//div/text()')
         self.assertEqual(l.get_output_value('name'), [u'Marta'])
         l.replace_xpath('name', '//p/text()')
@@ -552,7 +552,7 @@ class SelectortemLoaderTest(unittest.TestCase):
 
     def test_replace_xpath_re(self):
         l = TestItemLoader(response=self.response)
-        self.assert_(l.selector)
+        self.assertTrue(l.selector)
         l.add_xpath('name', '//div/text()')
         self.assertEqual(l.get_output_value('name'), [u'Marta'])
         l.replace_xpath('name', '//div/text()', re='ma')
@@ -568,7 +568,7 @@ class SelectortemLoaderTest(unittest.TestCase):
 
     def test_replace_css(self):
         l = TestItemLoader(response=self.response)
-        self.assert_(l.selector)
+        self.assertTrue(l.selector)
         l.add_css('name', 'div::text')
         self.assertEqual(l.get_output_value('name'), [u'Marta'])
         l.replace_css('name', 'p::text')
@@ -606,7 +606,7 @@ class SelectortemLoaderTest(unittest.TestCase):
 
     def test_replace_css_re(self):
         l = TestItemLoader(response=self.response)
-        self.assert_(l.selector)
+        self.assertTrue(l.selector)
         l.add_css('url', 'a::attr(href)')
         self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
         l.replace_css('url', 'a::attr(href)', re='http://www\.(.+)')
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 0f3047602..03c6d8059 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -81,28 +81,28 @@ class ImagesPipelineTestCase(unittest.TestCase):
         COLOUR = (0, 127, 255)
         im = _create_image('JPEG', 'RGB', SIZE, COLOUR)
         converted, _ = self.pipeline.convert_image(im)
-        self.assertEquals(converted.mode, 'RGB')
-        self.assertEquals(converted.getcolors(), [(10000, COLOUR)])
+        self.assertEqual(converted.mode, 'RGB')
+        self.assertEqual(converted.getcolors(), [(10000, COLOUR)])
 
         # check that thumbnail keep image ratio
         thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25))
-        self.assertEquals(thumbnail.mode, 'RGB')
-        self.assertEquals(thumbnail.size, (10, 10))
+        self.assertEqual(thumbnail.mode, 'RGB')
+        self.assertEqual(thumbnail.size, (10, 10))
 
         # transparency case: RGBA and PNG
         COLOUR = (0, 127, 255, 50)
         im = _create_image('PNG', 'RGBA', SIZE, COLOUR)
         converted, _ = self.pipeline.convert_image(im)
-        self.assertEquals(converted.mode, 'RGB')
-        self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))])
+        self.assertEqual(converted.mode, 'RGB')
+        self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
 
         # transparency case with palette: P and PNG
         COLOUR = (0, 127, 255, 50)
         im = _create_image('PNG', 'RGBA', SIZE, COLOUR)
         im = im.convert('P')
         converted, _ = self.pipeline.convert_image(im)
-        self.assertEquals(converted.mode, 'RGB')
-        self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))])
+        self.assertEqual(converted.mode, 'RGB')
+        self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
 
 
 class DeprecatedImagesPipeline(ImagesPipeline):
diff --git a/tests/test_selector.py b/tests/test_selector.py
index af0cc4de2..526660cc8 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -84,7 +84,7 @@ class SelectorTestCase(unittest.TestCase):
         headers = {'Content-Type': ['text/html; charset=utf-8']}
         response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
         x = Selector(response)
-        self.assertEquals(x.xpath("//span[@id='blank']/text()").extract(),
+        self.assertEqual(x.xpath("//span[@id='blank']/text()").extract(),
                           [u'\xa3'])
 
     def test_badly_encoded_body(self):
diff --git a/tests/test_spider.py b/tests/test_spider.py
index e55f0fa6d..0a343549e 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -207,7 +207,7 @@ class CrawlSpiderTest(SpiderTest):
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEquals([r.url for r in output],
+        self.assertEqual([r.url for r in output],
                           ['http://example.org/somepage/item/12.html',
                            'http://example.org/about.html',
                            'http://example.org/nofollow.html'])
@@ -234,7 +234,7 @@ class CrawlSpiderTest(SpiderTest):
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 2)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEquals([r.url for r in output],
+        self.assertEqual([r.url for r in output],
                           ['http://example.org/somepage/item/12.html',
                            'http://example.org/about.html'])
 
@@ -258,7 +258,7 @@ class CrawlSpiderTest(SpiderTest):
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEquals([r.url for r in output],
+        self.assertEqual([r.url for r in output],
                           ['http://example.org/somepage/item/12.html',
                            'http://example.org/about.html',
                            'http://example.org/nofollow.html'])
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index a3cdc0114..3685d5a6f 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -25,18 +25,18 @@ class TestDepthMiddleware(TestCase):
         result = [Request('http://scrapytest.org')]
 
         out = list(self.mw.process_spider_output(resp, result, self.spider))
-        self.assertEquals(out, result)
+        self.assertEqual(out, result)
 
         rdc = self.stats.get_value('request_depth_count/1', spider=self.spider)
-        self.assertEquals(rdc, 1)
+        self.assertEqual(rdc, 1)
 
         req.meta['depth'] = 1
 
         out2 = list(self.mw.process_spider_output(resp, result, self.spider))
-        self.assertEquals(out2, [])
+        self.assertEqual(out2, [])
 
         rdm = self.stats.get_value('request_depth_max', spider=self.spider)
-        self.assertEquals(rdm, 1)
+        self.assertEqual(rdm, 1)
 
     def tearDown(self):
         self.stats.close_spider(self.spider, '')
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index e1407e6b3..19e6bbdcd 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -67,16 +67,16 @@ class TestHttpErrorMiddleware(TestCase):
         self.res200, self.res404 = _responses(self.req, [200, 404])
 
     def test_process_spider_input(self):
-        self.assertEquals(None,
+        self.assertEqual(None,
                 self.mw.process_spider_input(self.res200, self.spider))
         self.assertRaises(HttpError,
                 self.mw.process_spider_input, self.res404, self.spider)
 
     def test_process_spider_exception(self):
-        self.assertEquals([],
+        self.assertEqual([],
                 self.mw.process_spider_exception(self.res404,
                         HttpError(self.res404), self.spider))
-        self.assertEquals(None,
+        self.assertEqual(None,
                 self.mw.process_spider_exception(self.res404,
                         Exception(), self.spider))
 
@@ -84,11 +84,11 @@ class TestHttpErrorMiddleware(TestCase):
         res = self.res404.copy()
         res.request = Request('http://scrapytest.org',
                               meta={'handle_httpstatus_list': [404]})
-        self.assertEquals(None,
+        self.assertEqual(None,
             self.mw.process_spider_input(res, self.spider))
 
         self.spider.handle_httpstatus_list = [404]
-        self.assertEquals(None,
+        self.assertEqual(None,
             self.mw.process_spider_input(self.res404, self.spider))
 
 
@@ -102,11 +102,11 @@ class TestHttpErrorMiddlewareSettings(TestCase):
         self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
 
     def test_process_spider_input(self):
-        self.assertEquals(None,
+        self.assertEqual(None,
                 self.mw.process_spider_input(self.res200, self.spider))
         self.assertRaises(HttpError,
                 self.mw.process_spider_input, self.res404, self.spider)
-        self.assertEquals(None,
+        self.assertEqual(None,
                 self.mw.process_spider_input(self.res402, self.spider))
 
     def test_meta_overrides_settings(self):
@@ -117,14 +117,14 @@ class TestHttpErrorMiddlewareSettings(TestCase):
         res402 = self.res402.copy()
         res402.request = request
 
-        self.assertEquals(None,
+        self.assertEqual(None,
             self.mw.process_spider_input(res404, self.spider))
         self.assertRaises(HttpError,
                 self.mw.process_spider_input, res402, self.spider)
 
     def test_spider_override_settings(self):
         self.spider.handle_httpstatus_list = [404]
-        self.assertEquals(None,
+        self.assertEqual(None,
             self.mw.process_spider_input(self.res404, self.spider))
         self.assertRaises(HttpError,
                 self.mw.process_spider_input, self.res402, self.spider)
@@ -139,9 +139,9 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
         self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
 
     def test_process_spider_input(self):
-        self.assertEquals(None,
+        self.assertEqual(None,
                 self.mw.process_spider_input(self.res200, self.spider))
-        self.assertEquals(None,
+        self.assertEqual(None,
                 self.mw.process_spider_input(self.res404, self.spider))
 
     def test_meta_overrides_settings(self):
@@ -152,7 +152,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
         res402 = self.res402.copy()
         res402.request = request
 
-        self.assertEquals(None,
+        self.assertEqual(None,
             self.mw.process_spider_input(res404, self.spider))
         self.assertRaises(HttpError,
                 self.mw.process_spider_input, res402, self.spider)
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 37c3a450b..9ad86313c 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -37,7 +37,7 @@ class TestOffsiteMiddleware(TestCase):
         reqs = onsite_reqs + offsite_reqs
 
         out = list(self.mw.process_spider_output(res, reqs, self.spider))
-        self.assertEquals(out, onsite_reqs)
+        self.assertEqual(out, onsite_reqs)
 
 
 class TestOffsiteMiddleware2(TestOffsiteMiddleware):
@@ -49,7 +49,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware):
         res = Response('http://scrapytest.org')
         reqs = [Request('http://a.com/b.html'), Request('http://b.com/1')]
         out = list(self.mw.process_spider_output(res, reqs, self.spider))
-        self.assertEquals(out, reqs)
+        self.assertEqual(out, reqs)
 
 class TestOffsiteMiddleware3(TestOffsiteMiddleware2):
 
@@ -67,4 +67,4 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
       res = Response('http://scrapytest.org')
       reqs = [Request('http://scrapytest.org/1')]
       out = list(self.mw.process_spider_output(res, reqs, self.spider))
-      self.assertEquals(out, reqs)
+      self.assertEqual(out, reqs)
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index f27f31b74..21439c20e 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -45,7 +45,7 @@ class TestRefererMiddleware(TestCase):
             response = self.get_response(origin)
             request = self.get_request(target)
             out = list(self.mw.process_spider_output(response, [request], self.spider))
-            self.assertEquals(out[0].headers.get('Referer'), referrer)
+            self.assertEqual(out[0].headers.get('Referer'), referrer)
 
 
 class MixinDefault(object):
@@ -490,7 +490,7 @@ class TestSettingsPolicyByName(TestCase):
             ]:
             settings = Settings({'REFERRER_POLICY': s})
             mw = RefererMiddleware(settings)
-            self.assertEquals(mw.default_policy, p)
+            self.assertEqual(mw.default_policy, p)
 
     def test_valid_name_casevariants(self):
         for s, p in [
@@ -506,7 +506,7 @@ class TestSettingsPolicyByName(TestCase):
             ]:
             settings = Settings({'REFERRER_POLICY': s.upper()})
             mw = RefererMiddleware(settings)
-            self.assertEquals(mw.default_policy, p)
+            self.assertEqual(mw.default_policy, p)
 
     def test_invalid_name(self):
         settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'})
@@ -581,7 +581,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
             request = self.get_request(target)
 
             out = list(self.referrermw.process_spider_output(response, [request], self.spider))
-            self.assertEquals(out[0].headers.get('Referer'), init_referrer)
+            self.assertEqual(out[0].headers.get('Referer'), init_referrer)
 
             for status, url in redirections:
                 response = Response(request.url, headers={'Location': url}, status=status)
@@ -589,7 +589,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
                 self.referrermw.request_scheduled(request, self.spider)
 
             assert isinstance(request, Request)
-            self.assertEquals(request.headers.get('Referer'), final_referrer)
+            self.assertEqual(request.headers.get('Referer'), final_referrer)
 
 
 class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index dca868ecf..a0aae0fdd 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -17,5 +17,5 @@ class TestUrlLengthMiddleware(TestCase):
         mw = UrlLengthMiddleware(maxlength=25)
         spider = Spider('foo')
         out = list(mw.process_spider_output(res, reqs, spider))
-        self.assertEquals(out, [short_url_req])
+        self.assertEqual(out, [short_url_req])
 
diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py
index 052dde37f..22e39821c 100644
--- a/tests/test_urlparse_monkeypatches.py
+++ b/tests/test_urlparse_monkeypatches.py
@@ -6,7 +6,7 @@ class UrlparseTestCase(unittest.TestCase):
 
     def test_s3_url(self):
         p = urlparse('s3://bucket/key/name?param=value')
-        self.assertEquals(p.scheme, 's3')
-        self.assertEquals(p.hostname, 'bucket')
-        self.assertEquals(p.path, '/key/name')
-        self.assertEquals(p.query, 'param=value')
+        self.assertEqual(p.scheme, 's3')
+        self.assertEqual(p.hostname, 'bucket')
+        self.assertEqual(p.path, '/key/name')
+        self.assertEqual(p.query, 'param=value')
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 49323f0ff..5b83869b8 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -202,22 +202,22 @@ class SequenceExcludeTest(unittest.TestCase):
         seq = range(10, 20, 3)
         d = SequenceExclude(seq)
         are_not_in = [v for v in range(10, 20, 3) if v in d]
-        self.assertEquals([], are_not_in)
+        self.assertEqual([], are_not_in)
 
         are_not_in = [v for v in range(10, 20) if v in d]
-        self.assertEquals([11, 12, 14, 15, 17, 18], are_not_in)
+        self.assertEqual([11, 12, 14, 15, 17, 18], are_not_in)
 
     def test_string_seq(self):
         seq = "cde"
         d = SequenceExclude(seq)
         chars = "".join(v for v in "abcdefg" if v in d)
-        self.assertEquals("abfg", chars)
+        self.assertEqual("abfg", chars)
 
     def test_stringset_seq(self):
         seq = set("cde")
         d = SequenceExclude(seq)
         chars = "".join(v for v in "abcdefg" if v in d)
-        self.assertEquals("abfg", chars)
+        self.assertEqual("abfg", chars)
 
     def test_set(self):
         """Anything that is not in the supplied sequence will evaluate as 'in' the container."""
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index f49bbfafe..003bb9b02 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -89,7 +89,7 @@ class IterErrbackTest(unittest.TestCase):
         errors = []
         out = list(iter_errback(itergood(), errors.append))
         self.assertEqual(out, list(range(10)))
-        self.failIf(errors)
+        self.assertFalse(errors)
 
     def test_iter_errback_bad(self):
         def iterbad():
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index b2e3889a4..b2e8610f8 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -252,8 +252,8 @@ class UtilsCsvTestCase(unittest.TestCase):
 
         # explicit type check cuz' we no like stinkin' autocasting! yarrr
         for result_row in result:
-            self.assert_(all((isinstance(k, six.text_type) for k in result_row.keys())))
-            self.assert_(all((isinstance(v, six.text_type) for v in result_row.values())))
+            self.assertTrue(all((isinstance(k, six.text_type) for k in result_row.keys())))
+            self.assertTrue(all((isinstance(v, six.text_type) for v in result_row.values())))
 
     def test_csviter_delimiter(self):
         body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index 01460a10b..832253aa4 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -23,20 +23,20 @@ class UtilsMiscTestCase(unittest.TestCase):
             'tests.test_utils_misc.test_walk_modules.mod.mod0',
             'tests.test_utils_misc.test_walk_modules.mod1',
         ]
-        self.assertEquals(set([m.__name__ for m in mods]), set(expected))
+        self.assertEqual(set([m.__name__ for m in mods]), set(expected))
 
         mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod')
         expected = [
             'tests.test_utils_misc.test_walk_modules.mod',
             'tests.test_utils_misc.test_walk_modules.mod.mod0',
         ]
-        self.assertEquals(set([m.__name__ for m in mods]), set(expected))
+        self.assertEqual(set([m.__name__ for m in mods]), set(expected))
 
         mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod1')
         expected = [
             'tests.test_utils_misc.test_walk_modules.mod1',
         ]
-        self.assertEquals(set([m.__name__ for m in mods]), set(expected))
+        self.assertEqual(set([m.__name__ for m in mods]), set(expected))
 
         self.assertRaises(ImportError, walk_modules, 'nomodule999')
 
@@ -51,7 +51,7 @@ class UtilsMiscTestCase(unittest.TestCase):
                 'testegg.spiders.b',
                 'testegg'
             ]
-            self.assertEquals(set([m.__name__ for m in mods]), set(expected))
+            self.assertEqual(set([m.__name__ for m in mods]), set(expected))
         finally:
             sys.path.remove(egg)
 
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index 6b7fcd4c2..7e2caace8 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -25,14 +25,14 @@ def inside_a_project():
 
 class ProjectUtilsTest(unittest.TestCase):
     def test_data_path_outside_project(self):
-        self.assertEquals('.scrapy/somepath', data_path('somepath'))
-        self.assertEquals('/absolute/path', data_path('/absolute/path'))
+        self.assertEqual('.scrapy/somepath', data_path('somepath'))
+        self.assertEqual('/absolute/path', data_path('/absolute/path'))
 
     def test_data_path_inside_project(self):
         with inside_a_project() as proj_path:
             expected = os.path.join(proj_path, '.scrapy', 'somepath')
-            self.assertEquals(
+            self.assertEqual(
                 os.path.realpath(expected),
                 os.path.realpath(data_path('somepath'))
             )
-            self.assertEquals('/absolute/path', data_path('/absolute/path'))
+            self.assertEqual('/absolute/path', data_path('/absolute/path'))
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 8becca0f1..c2e4037e8 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -97,9 +97,9 @@ class UtilsPythonTestCase(unittest.TestCase):
         a = Obj()
         b = Obj()
         # no attributes given return False
-        self.failIf(equal_attributes(a, b, []))
+        self.assertFalse(equal_attributes(a, b, []))
         # not existent attributes
-        self.failIf(equal_attributes(a, b, ['x', 'y']))
+        self.assertFalse(equal_attributes(a, b, ['x', 'y']))
 
         a.x = 1
         b.x = 1
@@ -108,7 +108,7 @@ class UtilsPythonTestCase(unittest.TestCase):
 
         b.y = 2
         # obj1 has no attribute y
-        self.failIf(equal_attributes(a, b, ['x', 'y']))
+        self.assertFalse(equal_attributes(a, b, ['x', 'y']))
 
         a.y = 2
         # equal attributes
@@ -116,7 +116,7 @@ class UtilsPythonTestCase(unittest.TestCase):
 
         a.y = 1
         # differente attributes
-        self.failIf(equal_attributes(a, b, ['x', 'y']))
+        self.assertFalse(equal_attributes(a, b, ['x', 'y']))
 
         # test callable
         a.meta = {}
@@ -134,7 +134,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         self.assertTrue(equal_attributes(a, b, [compare_z, 'x']))
         # fail z equality
         a.meta['z'] = 2
-        self.failIf(equal_attributes(a, b, [compare_z, 'x']))
+        self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
 
     def test_weakkeycache(self):
         class _Weakme(object): pass
@@ -156,9 +156,9 @@ class UtilsPythonTestCase(unittest.TestCase):
         d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'}
         d2 = stringify_dict(d, keys_only=False)
         self.assertEqual(d, d2)
-        self.failIf(d is d2)  # shouldn't modify in place
-        self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()))
-        self.failIf(any(isinstance(x, six.text_type) for x in d2.values()))
+        self.assertFalse(d is d2)  # shouldn't modify in place
+        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
+        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
 
     @unittest.skipUnless(six.PY2, "deprecated function")
     def test_stringify_dict_tuples(self):
@@ -166,17 +166,17 @@ class UtilsPythonTestCase(unittest.TestCase):
         d = dict(tuples)
         d2 = stringify_dict(tuples, keys_only=False)
         self.assertEqual(d, d2)
-        self.failIf(d is d2)  # shouldn't modify in place
-        self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())
-        self.failIf(any(isinstance(x, six.text_type) for x in d2.values()))
+        self.assertFalse(d is d2)  # shouldn't modify in place
+        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())
+        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
 
     @unittest.skipUnless(six.PY2, "deprecated function")
     def test_stringify_dict_keys_only(self):
         d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
         d2 = stringify_dict(d)
         self.assertEqual(d, d2)
-        self.failIf(d is d2)  # shouldn't modify in place
-        self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()))
+        self.assertFalse(d is d2)  # shouldn't modify in place
+        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
 
     def test_get_func_args(self):
         def f1(a, b, c):
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index b7de85049..dea81adf6 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -29,7 +29,7 @@ class SendCatchLogTest(unittest.TestCase):
         self.assertIn('error_handler', record.getMessage())
         self.assertEqual(record.levelname, 'ERROR')
         self.assertEqual(result[0][0], self.error_handler)
-        self.assert_(isinstance(result[0][1], Failure))
+        self.assertTrue(isinstance(result[0][1], Failure))
         self.assertEqual(result[1], (self.ok_handler, "OK"))
 
         dispatcher.disconnect(self.error_handler, signal=test_signal)
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 3ad1aa70e..fedac2634 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -71,7 +71,7 @@ class ParseUrlTestCase(unittest.TestCase):
         for url, test in tests:
             test = tuple(
                 to_bytes(x) if not isinstance(x, int) else x for x in test)
-            self.assertEquals(client._parse(url), test, url)
+            self.assertEqual(client._parse(url), test, url)
 
     def test_externalUnicodeInterference(self):
         """
@@ -258,16 +258,16 @@ class WebClientTestCase(unittest.TestCase):
     def testPayload(self):
         s = "0123456789" * 10
         return getPage(self.getURL("payload"), body=s).addCallback(
-            self.assertEquals, to_bytes(s))
+            self.assertEqual, to_bytes(s))
 
     def testHostHeader(self):
         # if we pass Host header explicitly, it should be used, otherwise
         # it should extract from url
         return defer.gatherResults([
             getPage(self.getURL("host")).addCallback(
-                self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno)),
+                self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno)),
             getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(
-                self.assertEquals, to_bytes("www.example.com"))])
+                self.assertEqual, to_bytes("www.example.com"))])
 
     def test_getPage(self):
         """
@@ -275,7 +275,7 @@ class WebClientTestCase(unittest.TestCase):
         the body of the response if the default method B{GET} is used.
         """
         d = getPage(self.getURL("file"))
-        d.addCallback(self.assertEquals, b"0123456789")
+        d.addCallback(self.assertEqual, b"0123456789")
         return d
 
     def test_getPageHead(self):
@@ -298,7 +298,7 @@ class WebClientTestCase(unittest.TestCase):
         """
         d = getPage(self.getURL("host"), timeout=100)
         d.addCallback(
-            self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno))
+            self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno))
         return d
 
     def test_timeoutTriggering(self):
@@ -326,7 +326,7 @@ class WebClientTestCase(unittest.TestCase):
         return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile)
 
     def _cbNoSuchFile(self, pageData):
-        self.assert_(b'404 - No Such Resource' in pageData)
+        self.assertTrue(b'404 - No Such Resource' in pageData)
 
     def testFactoryInfo(self):
         url = self.getURL('file')
@@ -336,16 +336,16 @@ class WebClientTestCase(unittest.TestCase):
         return factory.deferred.addCallback(self._cbFactoryInfo, factory)
 
     def _cbFactoryInfo(self, ignoredResult, factory):
-        self.assertEquals(factory.status, b'200')
-        self.assert_(factory.version.startswith(b'HTTP/'))
-        self.assertEquals(factory.message, b'OK')
-        self.assertEquals(factory.response_headers[b'content-length'], b'10')
+        self.assertEqual(factory.status, b'200')
+        self.assertTrue(factory.version.startswith(b'HTTP/'))
+        self.assertEqual(factory.message, b'OK')
+        self.assertEqual(factory.response_headers[b'content-length'], b'10')
 
     def testRedirect(self):
         return getPage(self.getURL("redirect")).addCallback(self._cbRedirect)
 
     def _cbRedirect(self, pageData):
-        self.assertEquals(pageData,
+        self.assertEqual(pageData,
                 b'\n<html>\n    <head>\n        <meta http-equiv="refresh" content="0;URL=/file">\n'
                 b'    </head>\n    <body bgcolor="#FFFFFF" text="#000000">\n    '
                 b'<a href="/file">click here</a>\n    </body>\n</html>\n')
@@ -360,6 +360,6 @@ class WebClientTestCase(unittest.TestCase):
 
     def _check_Encoding(self, response, original_body):
         content_encoding = to_unicode(response.headers[b'Content-Encoding'])
-        self.assertEquals(content_encoding, EncodingResource.out_encoding)
-        self.assertEquals(
+        self.assertEqual(content_encoding, EncodingResource.out_encoding)
+        self.assertEqual(
             response.body.decode(content_encoding), to_unicode(original_body))

From fd27cde24d273e30f72f53f2711515403270838a Mon Sep 17 00:00:00 2001
From: Chomba Ng'ang'a <choeringa@gmail.com>
Date: Tue, 8 Aug 2017 19:08:53 +0300
Subject: [PATCH 1561/2974] Update asserts to use more generic ones

---
 tests/test_http_cookies.py | 2 +-
 tests/test_http_request.py | 4 ++--
 tests/test_loader.py       | 6 +++---
 tests/test_utils_python.py | 6 +++---
 tests/test_utils_signal.py | 2 +-
 tests/test_webclient.py    | 2 +-
 6 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index caa6fe83e..0a9ed500a 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -62,7 +62,7 @@ class WrappedResponseTest(TestCase):
         self.wrapped = WrappedResponse(self.response)
 
     def test_info(self):
-        self.assertTrue(self.wrapped.info() is self.wrapped)
+        self.assertIs(self.wrapped.info(), self.wrapped)
 
     def test_getheaders(self):
         self.assertEqual(self.wrapped.getheaders('content-type'), ['text/html'])
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 21c0dd746..fca8ff411 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -64,9 +64,9 @@ class RequestTest(unittest.TestCase):
         h = Headers({'key1': u'val1', u'key2': 'val2'})
         h[u'newkey'] = u'newval'
         for k, v in h.iteritems():
-            self.assertTrue(isinstance(k, bytes))
+            self.assertIsInstance(k, bytes)
             for s in v:
-                self.assertTrue(isinstance(s, bytes))
+                self.assertIsInstance(s, bytes)
 
     def test_eq(self):
         url = 'http://www.scrapy.org'
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 2569ccf5e..3b5714058 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -437,7 +437,7 @@ class ProcessorsTest(unittest.TestCase):
         self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
         self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
         self.assertEqual(proc(['hello', 'world']), u'hello world')
-        self.assertTrue(isinstance(proc(['hello', 'world']), six.text_type))
+        self.assertIsInstance(proc(['hello', 'world']), six.text_type)
 
     def test_compose(self):
         proc = Compose(lambda v: v[0], str.upper)
@@ -482,7 +482,7 @@ class SelectortemLoaderTest(unittest.TestCase):
     def test_constructor_with_selector(self):
         sel = Selector(text=u"<html><body><div>marta</div></body></html>")
         l = TestItemLoader(selector=sel)
-        self.assertTrue(l.selector is sel)
+        self.assertIs(l.selector, sel)
 
         l.add_xpath('name', '//div/text()')
         self.assertEqual(l.get_output_value('name'), [u'Marta'])
@@ -490,7 +490,7 @@ class SelectortemLoaderTest(unittest.TestCase):
     def test_constructor_with_selector_css(self):
         sel = Selector(text=u"<html><body><div>marta</div></body></html>")
         l = TestItemLoader(selector=sel)
-        self.assertTrue(l.selector is sel)
+        self.assertIs(l.selector, sel)
 
         l.add_css('name', 'div::text')
         self.assertEqual(l.get_output_value('name'), [u'Marta'])
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index c2e4037e8..115f523e9 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -156,7 +156,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'}
         d2 = stringify_dict(d, keys_only=False)
         self.assertEqual(d, d2)
-        self.assertFalse(d is d2)  # shouldn't modify in place
+        self.assertIsNot(d, d2)  # shouldn't modify in place
         self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
         self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
 
@@ -166,7 +166,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         d = dict(tuples)
         d2 = stringify_dict(tuples, keys_only=False)
         self.assertEqual(d, d2)
-        self.assertFalse(d is d2)  # shouldn't modify in place
+        self.assertIsNot(d, d2)  # shouldn't modify in place
         self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())
         self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
 
@@ -175,7 +175,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
         d2 = stringify_dict(d)
         self.assertEqual(d, d2)
-        self.assertFalse(d is d2)  # shouldn't modify in place
+        self.assertIsNot(d, d2)  # shouldn't modify in place
         self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
 
     def test_get_func_args(self):
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index dea81adf6..62edd420d 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -29,7 +29,7 @@ class SendCatchLogTest(unittest.TestCase):
         self.assertIn('error_handler', record.getMessage())
         self.assertEqual(record.levelname, 'ERROR')
         self.assertEqual(result[0][0], self.error_handler)
-        self.assertTrue(isinstance(result[0][1], Failure))
+        self.assertIsInstance(result[0][1], Failure)
         self.assertEqual(result[1], (self.ok_handler, "OK"))
 
         dispatcher.disconnect(self.error_handler, signal=test_signal)
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index fedac2634..766329b57 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -326,7 +326,7 @@ class WebClientTestCase(unittest.TestCase):
         return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile)
 
     def _cbNoSuchFile(self, pageData):
-        self.assertTrue(b'404 - No Such Resource' in pageData)
+        self.assertIn(b'404 - No Such Resource', pageData)
 
     def testFactoryInfo(self):
         url = self.getURL('file')

From 1dcea6a9d4615afc463fa9839d002d861aed5274 Mon Sep 17 00:00:00 2001
From: kim minji <minji.dev@gmail.com>
Date: Wed, 16 Aug 2017 18:07:52 +0900
Subject: [PATCH 1562/2974] fix typo

---
 scrapy/crawler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 7b8518832..1367536ab 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -40,7 +40,7 @@ class Crawler(object):
         handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
         logging.root.addHandler(handler)
         if get_scrapy_root_handler() is not None:
-            # scrapy root handler alread installed: update it with new settings
+            # scrapy root handler already installed: update it with new settings
             install_scrapy_root_handler(self.settings)
         # lambda is assigned to Crawler attribute because this way it is not
         # garbage collected after leaving __init__ scope

From 1968a8ec02913273a99a3137cb419e2649d69a5f Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 23 Aug 2017 15:08:10 +0200
Subject: [PATCH 1563/2974] Move logging of overriden settings to Crawler init

---
 scrapy/crawler.py   | 5 ++++-
 scrapy/utils/log.py | 4 +---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 1367536ab..0a56ef57a 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -11,7 +11,7 @@ from scrapy.core.engine import ExecutionEngine
 from scrapy.resolver import CachingThreadedResolver
 from scrapy.interfaces import ISpiderLoader
 from scrapy.extension import ExtensionManager
-from scrapy.settings import Settings
+from scrapy.settings import overridden_settings, Settings
 from scrapy.signalmanager import SignalManager
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
@@ -34,6 +34,9 @@ class Crawler(object):
         self.settings = settings.copy()
         self.spidercls.update_settings(self.settings)
 
+        d = dict(overridden_settings(self.settings))
+        logger.info("Overridden settings: %(settings)r", {'settings': d})
+
         self.signals = SignalManager(self)
         self.stats = load_object(self.settings['STATS_CLASS'])(self)
 
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 905c1bfc1..c6d1cdf46 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -9,7 +9,7 @@ from twisted.python.failure import Failure
 from twisted.python import log as twisted_log
 
 import scrapy
-from scrapy.settings import overridden_settings, Settings
+from scrapy.settings import Settings
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.versions import scrapy_components_versions
 
@@ -148,8 +148,6 @@ def log_scrapy_info(settings):
                 {'versions': ", ".join("%s %s" % (name, version)
                     for name, version in scrapy_components_versions()
                     if name != "Scrapy")})
-    d = dict(overridden_settings(settings))
-    logger.info("Overridden settings: %(settings)r", {'settings': d})
 
 
 class StreamLogger(object):

From 7a35a1ad4ad38b2f413528b0184d8709dea2a495 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 23 Aug 2017 17:08:21 +0200
Subject: [PATCH 1564/2974] Remove trailing bracket from components versions
 log

---
 scrapy/utils/log.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 905c1bfc1..61b978c55 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -144,7 +144,7 @@ def _get_handler(settings):
 def log_scrapy_info(settings):
     logger.info("Scrapy %(version)s started (bot: %(bot)s)",
                 {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
-    logger.info("Versions: %(versions)s}",
+    logger.info("Versions: %(versions)s",
                 {'versions': ", ".join("%s %s" % (name, version)
                     for name, version in scrapy_components_versions()
                     if name != "Scrapy")})

From a429d78019a379fff29c7aa3fff0a0f0427b6995 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Thu, 24 Aug 2017 16:03:36 -0300
Subject: [PATCH 1565/2974] update scrapinghub.com urls to use https

---
 docs/intro/install.rst    | 2 +-
 docs/topics/deploy.rst    | 6 +++---
 docs/topics/logging.rst   | 4 ++--
 docs/topics/practices.rst | 2 +-
 docs/topics/ubuntu.rst    | 2 +-
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 47af8292e..deb0118d4 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -235,7 +235,7 @@ After any of these workarounds you should be able to install Scrapy::
 .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
 .. _homebrew: http://brew.sh/
 .. _zsh: http://www.zsh.org/
-.. _Scrapinghub: http://scrapinghub.com
+.. _Scrapinghub: https://scrapinghub.com
 .. _Anaconda: http://docs.continuum.io/anaconda/index
 .. _Miniconda: http://conda.pydata.org/docs/install/quick.html
 .. _conda-forge: https://conda-forge.github.io/
diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst
index f4186ea7a..f2e11fe8f 100644
--- a/docs/topics/deploy.rst
+++ b/docs/topics/deploy.rst
@@ -51,9 +51,9 @@ just like ``scrapyd-deploy``.
 
 .. _Scrapyd: https://github.com/scrapy/scrapyd
 .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
-.. _Scrapy Cloud: http://scrapinghub.com/scrapy-cloud/
+.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud
 .. _scrapyd-client: https://github.com/scrapy/scrapyd-client
-.. _shub: http://doc.scrapinghub.com/shub.html
+.. _shub: https://doc.scrapinghub.com/shub.html
 .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html
 .. _Scrapy Cloud documentation: http://doc.scrapinghub.com/scrapy-cloud.html
-.. _Scrapinghub: http://scrapinghub.com/
+.. _Scrapinghub: https://scrapinghub.com/
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index a3281dd6b..0986929ad 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -102,7 +102,7 @@ instance, which can be accessed and used like this::
     class MySpider(scrapy.Spider):
 
         name = 'myspider'
-        start_urls = ['http://scrapinghub.com']
+        start_urls = ['https://scrapinghub.com']
 
         def parse(self, response):
             self.logger.info('Parse function called on %s', response.url)
@@ -118,7 +118,7 @@ Python logger you want. For example::
     class MySpider(scrapy.Spider):
 
         name = 'myspider'
-        start_urls = ['http://scrapinghub.com']
+        start_urls = ['https://scrapinghub.com']
 
         def parse(self, response):
             logger.info('Parse function called on %s', response.url)
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 63913d3c4..21aa4a0a7 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -253,5 +253,5 @@ If you are still unable to prevent your bot getting banned, consider contacting
 .. _Google cache: http://www.googleguide.com/cached_pages.html
 .. _testspiders: https://github.com/scrapinghub/testspiders
 .. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
-.. _Crawlera: http://scrapinghub.com/crawlera
+.. _Crawlera: https://scrapinghub.com/crawlera
 .. _scrapoxy: http://scrapoxy.io/
diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst
index 679bb56ff..81ce800aa 100644
--- a/docs/topics/ubuntu.rst
+++ b/docs/topics/ubuntu.rst
@@ -37,5 +37,5 @@ To use the packages:
 .. warning:: `python-scrapy` is a different package provided by official debian
    repositories, it's very outdated and it isn't supported by Scrapy team.
 
-.. _Scrapinghub: http://scrapinghub.com/
+.. _Scrapinghub: https://scrapinghub.com/
 .. _GitHub repo: https://github.com/scrapy/scrapy

From 9f16f040b661f18f2e61a35427199c099bfd2f90 Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@bluewin.ch>
Date: Fri, 1 Sep 2017 11:53:59 +0200
Subject: [PATCH 1566/2974] ur'string' not needed in Py 2, syntax error in Py3

Convert `ur'Scrapy Documentation'`--> `u'Scrapy Documentation'`to be compatible with both Python 2 and Python 3.  See #2891
---
 docs/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index 640dcd7cb..5780db65d 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -191,7 +191,7 @@ htmlhelp_basename = 'Scrapydoc'
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, document class [howto/manual]).
 latex_documents = [
-  ('index', 'Scrapy.tex', ur'Scrapy Documentation',
+  ('index', 'Scrapy.tex', u'Scrapy Documentation',
    ur'Scrapy developers', 'manual'),
 ]
 

From b7022360824cd1a8aa19fb0d65ea01700f06a208 Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@bluewin.ch>
Date: Fri, 1 Sep 2017 11:56:09 +0200
Subject: [PATCH 1567/2974] ur'string' not needed in Py 2, syntax error in Py3

Convert `u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))'`--> `u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))'`to be compatible with both Python 2 and Python 3.  See #2891
---
 docs/utils/linkfix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
index 40316968f..6290adbe2 100755
--- a/docs/utils/linkfix.py
+++ b/docs/utils/linkfix.py
@@ -20,7 +20,7 @@ _filename = None
 _contents = None
 
 # A regex that matches standard linkcheck output lines
-line_re = re.compile(ur'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
+line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
 
 # Read lines from the linkcheck output file
 try:

From b8fabeed8652d22725959345700b9e7d00073de4 Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@bluewin.ch>
Date: Fri, 1 Sep 2017 13:55:05 +0200
Subject: [PATCH 1568/2974] ur'string' not needed in Py 2, syntax error in Py3

This instance was missed in #2909 --> ur'Scrapy developers' --> u'Scrapy developers'
---
 docs/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index 5780db65d..007dc2788 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -192,7 +192,7 @@ htmlhelp_basename = 'Scrapydoc'
 # (source start file, target name, title, author, document class [howto/manual]).
 latex_documents = [
   ('index', 'Scrapy.tex', u'Scrapy Documentation',
-   ur'Scrapy developers', 'manual'),
+   u'Scrapy developers', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of

From abaf466bb311f6416a58763ad7974825d88f4855 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 7 Sep 2017 11:37:40 +0200
Subject: [PATCH 1569/2974] Print cryptography package version

---
 scrapy/commands/version.py | 7 +++++--
 scrapy/utils/versions.py   | 6 ++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 71b1026fa..577365c3b 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -23,8 +23,11 @@ class Command(ScrapyCommand):
 
     def run(self, args, opts):
         if opts.verbose:
-            for name, version in scrapy_components_versions():
-                print("%-9s : %s" % (name, version))
+            versions = scrapy_components_versions()
+            width = max(len(n) for (n, _) in versions)
+            patt = "%-{}s : %s".format(width)
+            for name, version in versions:
+                print(patt % (name, version))
         else:
             print("Scrapy %s" % scrapy.__version__)
 
diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py
index d2cff09fe..58c7aef85 100644
--- a/scrapy/utils/versions.py
+++ b/scrapy/utils/versions.py
@@ -17,6 +17,11 @@ def scrapy_components_versions():
         w3lib_version = w3lib.__version__
     except AttributeError:
         w3lib_version = "<1.14.3"
+    try:
+        import cryptography
+        cryptography_version = cryptography.__version__
+    except ImportError:
+        cryptography_version = "unknown"
 
     return [
         ("Scrapy", scrapy.__version__),
@@ -28,6 +33,7 @@ def scrapy_components_versions():
         ("Twisted", twisted.version.short()),
         ("Python", sys.version.replace("\n", "- ")),
         ("pyOpenSSL", _get_openssl_version()),
+        ("cryptography", cryptography_version),
         ("Platform",  platform.platform()),
     ]
 

From aab98080a06281ca3a88646990b81b92f492c517 Mon Sep 17 00:00:00 2001
From: Iulian Onofrei <iulian.onofrei@yahoo.com>
Date: Mon, 11 Sep 2017 00:40:55 +0300
Subject: [PATCH 1570/2974] Add option to disable automatic log handler install

---
 scrapy/crawler.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 0a56ef57a..a33ce9805 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -237,15 +237,18 @@ class CrawlerProcess(CrawlerRunner):
     The CrawlerProcess object must be instantiated with a
     :class:`~scrapy.settings.Settings` object.
 
+    :param install_root_handler: whether to install root logging handler
+        (default: True)
+
     This class shouldn't be needed (since Scrapy is responsible of using it
     accordingly) unless writing scripts that manually handle the crawling
     process. See :ref:`run-from-script` for an example.
     """
 
-    def __init__(self, settings=None):
+    def __init__(self, settings=None, install_root_handler=True):
         super(CrawlerProcess, self).__init__(settings)
         install_shutdown_handlers(self._signal_shutdown)
-        configure_logging(self.settings)
+        configure_logging(self.settings, install_root_handler)
         log_scrapy_info(self.settings)
 
     def _signal_shutdown(self, signum, _):

From 3637b75a6702cb3fb4962477c0f2ec38e366f3e2 Mon Sep 17 00:00:00 2001
From: Steven Almeroth <sroth77@gmail.com>
Date: Tue, 12 Sep 2017 15:54:09 -0400
Subject: [PATCH 1571/2974] [Doc] Update Response.body type

---
 docs/topics/request-response.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 6ca37b7c9..92aae1ad0 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -525,11 +525,11 @@ Response objects
        (for single valued headers) or lists (for multi-valued headers).
     :type headers: dict
 
-    :param body: the response body. It must be str, not unicode, unless you're
-       using a encoding-aware :ref:`Response subclass
-       <topics-request-response-ref-response-subclasses>`, such as
-       :class:`TextResponse`.
-    :type body: str
+    :param body: the response body. To access the decoded text as str (unicode
+       in Python 2) you can use ``response.text`` from an encoding-aware
+       :ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
+       such as :class:`TextResponse`.
+    :type body: bytes
 
     :param flags: is a list containing the initial values for the
        :attr:`Response.flags` attribute. If given, the list will be shallow

From d71a0634039d637dc10509ebb63fa2f4ef595ebb Mon Sep 17 00:00:00 2001
From: rhoboro <rhoboro@gmail.com>
Date: Tue, 12 Sep 2017 18:30:15 +0900
Subject: [PATCH 1572/2974] Support for Google Cloud Storage

---
 scrapy/pipelines/files.py | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index eae03752a..304d89bcf 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -194,6 +194,41 @@ class S3FilesStore(object):
         return extra
 
 
+class GCSFilesStore(object):
+
+    GCS_PROJECT_ID = None
+
+    CACHE_CONTROL = 'max-age=172800'
+
+    def __init__(self, uri):
+        from google.cloud import storage
+        client = storage.Client(project=self.GCS_PROJECT_ID)
+        bucket, prefix = uri[5:].split('/', 1)
+        self.bucket = client.bucket(bucket)
+        self.prefix = prefix
+
+    def stat_file(self, path, info):
+        def _onsuccess(blob):
+            if blob:
+                checksum = blob.md5_hash
+                last_modified = time.mktime(blob.updated.timetuple())
+                return {'checksum': checksum, 'last_modified': last_modified}
+            else:
+                return {}
+
+        return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)
+
+    def persist_file(self, path, buf, info, meta=None, headers=None):
+        blob = self.bucket.blob(self.prefix + path)
+        blob.cache_control = self.CACHE_CONTROL
+        blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}
+        return threads.deferToThread(
+            blob.upload_from_string,
+            data=buf.getvalue(),
+            content_type='application/octet-stream'
+        )
+
+
 class FilesPipeline(MediaPipeline):
     """Abstract pipeline that implement the file downloading
 
@@ -219,6 +254,7 @@ class FilesPipeline(MediaPipeline):
         '': FSFilesStore,
         'file': FSFilesStore,
         's3': S3FilesStore,
+        'gs': GCSFilesStore,
     }
     DEFAULT_FILES_URLS_FIELD = 'file_urls'
     DEFAULT_FILES_RESULT_FIELD = 'files'
@@ -258,6 +294,9 @@ class FilesPipeline(MediaPipeline):
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
         s3store.POLICY = settings['FILES_STORE_S3_ACL']
 
+        gcs_store = cls.STORE_SCHEMES['gs']
+        gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
+
         store_uri = settings['FILES_STORE']
         return cls(store_uri, settings=settings)
 

From e5d4364b2a0e7ae205605905ac0c5ac6fd8d15db Mon Sep 17 00:00:00 2001
From: rhoboro <rhoboro@gmail.com>
Date: Wed, 13 Sep 2017 16:24:04 +0900
Subject: [PATCH 1573/2974] Add tests for GCS Storage

---
 scrapy/utils/test.py         | 15 +++++++++++++++
 tests/test_pipeline_files.py | 28 +++++++++++++++++++++++++++-
 tox.ini                      |  3 +++
 3 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index d2ef68912..60b931f48 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -20,6 +20,12 @@ def assert_aws_environ():
     if 'AWS_ACCESS_KEY_ID' not in os.environ:
         raise SkipTest("AWS keys not found")
 
+
+def assert_gcs_environ():
+    if 'GCS_PROJECT_ID' not in os.environ:
+        raise SkipTest("GCS_PROJECT_ID not found")
+
+
 def skip_if_no_boto():
     try:
         is_botocore()
@@ -45,6 +51,15 @@ def get_s3_content_and_delete(bucket, path, with_key=False):
         bucket.delete_key(path)
     return (content, key) if with_key else content
 
+def get_gcs_content_and_delete(bucket, path):
+    from google.cloud import storage
+    client = storage.Client(project=os.environ.get('GCS_PROJECT_ID'))
+    bucket = client.get_bucket(bucket)
+    blob = bucket.get_blob(path)
+    content = blob.download_as_string()
+    bucket.delete_blob(path)
+    return content, blob
+
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index e3ec04b8d..c761bd606 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -11,12 +11,13 @@ from six import BytesIO
 from twisted.trial import unittest
 from twisted.internet import defer
 
-from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore
+from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore
 from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
+from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete
 from scrapy.utils.boto import is_botocore
 
 from tests import mock
@@ -375,6 +376,31 @@ class TestS3FilesStore(unittest.TestCase):
             self.assertEqual(key.content_type, 'image/png')
 
 
+class TestGCSFilesStore(unittest.TestCase):
+    @defer.inlineCallbacks
+    def test_persist(self):
+        assert_gcs_environ()
+        uri = os.environ.get('GCS_TEST_FILE_URI')
+        if not uri:
+            raise unittest.SkipTest("No GCS URI available for testing")
+        data = b"TestGCSFilesStore: \xe2\x98\x83"
+        buf = BytesIO(data)
+        meta = {'foo': 'bar'}
+        path = 'full/filename'
+        store = GCSFilesStore(uri)
+        yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
+        s = yield store.stat_file(path, info=None)
+        self.assertIn('last_modified', s)
+        self.assertIn('checksum', s)
+        self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==')
+        u = urlparse(uri)
+        content, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path)
+        self.assertEqual(content, data)
+        self.assertEqual(blob.metadata, {'foo': 'bar'})
+        self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL)
+        self.assertEqual(blob.content_type, 'application/octet-stream')
+
+
 class ItemWithFiles(Item):
     file_urls = Field()
     files = Field()
diff --git a/tox.ini b/tox.ini
index c7e1e43c9..0608693ba 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,6 +11,7 @@ deps =
     -rrequirements.txt
     # Extras
     botocore
+    google-cloud-storage
     Pillow != 3.0.0
     leveldb
     -rtests/requirements.txt
@@ -18,6 +19,8 @@ passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY
+    GCS_TEST_FILE_URI
+    GCS_PROJECT_ID
 commands =
     py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
 

From ee166ec44f38da7f5b99c6c164a7c6ff02b37c16 Mon Sep 17 00:00:00 2001
From: rhoboro <rhoboro@gmail.com>
Date: Wed, 13 Sep 2017 17:35:46 +0900
Subject: [PATCH 1574/2974] Support for ImagesPipeline

---
 scrapy/pipelines/files.py  | 8 +++++++-
 scrapy/pipelines/images.py | 3 +++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 304d89bcf..7fdb8a086 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -218,6 +218,12 @@ class GCSFilesStore(object):
 
         return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)
 
+    def _get_content_type(self, headers):
+        if headers and 'Content-Type' in headers:
+            return headers['Content-Type']
+        else:
+            return 'application/octet-stream'
+
     def persist_file(self, path, buf, info, meta=None, headers=None):
         blob = self.bucket.blob(self.prefix + path)
         blob.cache_control = self.CACHE_CONTROL
@@ -225,7 +231,7 @@ class GCSFilesStore(object):
         return threads.deferToThread(
             blob.upload_from_string,
             data=buf.getvalue(),
-            content_type='application/octet-stream'
+            content_type=self._get_content_type(headers)
         )
 
 
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index bc449431f..c5fc12afe 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -91,6 +91,9 @@ class ImagesPipeline(FilesPipeline):
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
         s3store.POLICY = settings['IMAGES_STORE_S3_ACL']
 
+        gcs_store = cls.STORE_SCHEMES['gs']
+        gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
+
         store_uri = settings['IMAGES_STORE']
         return cls(store_uri, settings=settings)
 

From 088b80d41a12a7b79e440cc4d4a3aae678d5c4af Mon Sep 17 00:00:00 2001
From: Renze Yu <renzeyu@microsoft.com>
Date: Wed, 13 Sep 2017 23:29:22 +0800
Subject: [PATCH 1575/2974] minor fix typo

---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 3b3bd8d21..29f54bc86 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -452,7 +452,7 @@ For historic reasons, Scrapy appends to a given file instead of overwriting
 its contents. If you run this command twice without removing the file
 before the second time, you'll end up with a broken JSON file.
 
-You can also used other formats, like `JSON Lines`_::
+You can also use other formats, like `JSON Lines`_::
 
     scrapy crawl quotes -o quotes.jl
 

From dcb279bd6cc85cf1743b548e44b050edba6a2ed8 Mon Sep 17 00:00:00 2001
From: djunzu <djunzu@users.noreply.github.com>
Date: Sun, 17 Sep 2017 16:09:22 -0300
Subject: [PATCH 1576/2974] Add m4v extension to IGNORED_EXTENSIONS in
 LinkExtractor.

	modified:   scrapy/linkextractors/__init__.py
---
 scrapy/linkextractors/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 8676c3b92..2d7115cc5 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -28,7 +28,7 @@ IGNORED_EXTENSIONS = [
 
     # video
     '3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv',
-    'm4a',
+    'm4a', 'm4v',
 
     # office suites
     'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg',

From 84111969c4250b486b15729b42f88300fc983511 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Wed, 20 Sep 2017 13:35:48 +0200
Subject: [PATCH 1577/2974] Update pypy version regexp to get last release

PyPy changed naming conention since 5.8 release, not it's called
pypy2.7-x.x.x
---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 4f44d1e6d..9c51fafb2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,7 +31,7 @@ install:
           rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT"
         fi
         # get latest portable PyPy from pyenv directly (thanks to natural version sort option -V)
-        export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy-portable-[0-9][\.0-9]*$' |sort -V |tail -1`
+        export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy2.7-portable-[0-9][\.0-9]*$' |sort -V |tail -1`
         "$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION"
         virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"

From 00c81a32ee58d0a59c14373471bf152c46131aec Mon Sep 17 00:00:00 2001
From: Craig Rodrigues <rodrigc@FreeBSD.org>
Date: Sat, 23 Sep 2017 11:01:34 -0700
Subject: [PATCH 1578/2974] Bump Twisted requirement to 17.9.0 to catch many
 Python 3 fixes.

---
 requirements-py3.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-py3.txt b/requirements-py3.txt
index cc0a7f644..2aae3ae65 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -1,4 +1,4 @@
-Twisted >= 15.5.0
+Twisted >= 17.9.0
 lxml>=3.2.4
 pyOpenSSL>=0.13.1
 cssselect>=0.9

From e733f51d4b04f209bfec32d1bd7559a258f45d0c Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 25 Sep 2017 12:49:27 +0200
Subject: [PATCH 1579/2974] Fix test

---
 tests/test_command_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_command_version.py b/tests/test_command_version.py
index 2789d207c..4ac7fb786 100644
--- a/tests/test_command_version.py
+++ b/tests/test_command_version.py
@@ -28,4 +28,4 @@ class VersionTest(ProcessTest, unittest.TestCase):
         self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2',
                                    'cssselect', 'parsel', 'w3lib',
                                    'Twisted', 'Python', 'pyOpenSSL',
-                                   'Platform'])
+                                   'cryptography', 'Platform'])

From d4555b2bcc387292e5fd5bd8321c946e2e374fb7 Mon Sep 17 00:00:00 2001
From: rhoboro <rhoboro@gmail.com>
Date: Fri, 29 Sep 2017 12:07:29 +0900
Subject: [PATCH 1580/2974] update docs for supporting google cloud storage

---
 docs/topics/media-pipeline.rst | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index e948913a4..9580a15d9 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -15,7 +15,8 @@ typically you'll either use the Files Pipeline or the Images Pipeline.
 Both pipelines implement these features:
 
 * Avoid re-downloading media that was downloaded recently
-* Specifying where to store the media (filesystem directory, Amazon S3 bucket)
+* Specifying where to store the media (filesystem directory, Amazon S3 bucket,
+  Google Cloud Storage bucket)
 
 The Images Pipeline has a few extra functions for processing images:
 
@@ -116,10 +117,11 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
 Supported Storage
 =================
 
-File system is currently the only officially supported storage, but there is
-also support for storing files in `Amazon S3`_.
+File system is currently the only officially supported storage, but there are
+also support for storing files in `Amazon S3`_ and `Google Cloud Storage`_.
 
 .. _Amazon S3: https://aws.amazon.com/s3/
+.. _Google Cloud Storage: https://cloud.google.com/storage/
 
 File system storage
 -------------------
@@ -171,6 +173,25 @@ For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
 
 .. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
+Google Cloud Storage
+---------------------
+
+.. setting:: GCS_PROJECT_ID
+
+:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage
+bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ )
+
+.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
+
+For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings::
+
+    IMAGES_STORE = 'gs://bucket/images/'
+    GCS_PROJECT_ID = 'project_id'
+
+For information about authentication, see this `documentation`_.
+
+.. _documentation: https://cloud.google.com/docs/authentication/production
+
 Usage example
 =============
 

From 59c3f6f095d7605a825a90f376d41493a78e5da7 Mon Sep 17 00:00:00 2001
From: Lucas Moauro <lagenar@gmail.com>
Date: Sun, 1 Oct 2017 12:24:56 -0300
Subject: [PATCH 1581/2974] Fix typos in tests

---
 tests/test_downloadermiddleware_httpproxy.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 0ea83aaf9..17be875c1 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -28,7 +28,7 @@ class TestDefaultHeadersMiddleware(TestCase):
         crawler = Crawler(spider, settings)
         self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler))
 
-    def test_no_enviroment_proxies(self):
+    def test_no_environment_proxies(self):
         os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
         mw = HttpProxyMiddleware()
 
@@ -38,7 +38,7 @@ class TestDefaultHeadersMiddleware(TestCase):
             self.assertEqual(req.url, url)
             self.assertEqual(req.meta, {})
 
-    def test_enviroment_proxies(self):
+    def test_environment_proxies(self):
         os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
         os.environ['https_proxy'] = https_proxy = 'http://proxy.for.https:8080'
         os.environ.pop('file_proxy', None)

From fc406801f1783392935fcd7faf603d8339c74675 Mon Sep 17 00:00:00 2001
From: Craig Rodrigues <rodrigc@FreeBSD.org>
Date: Tue, 21 Mar 2017 00:21:41 -0700
Subject: [PATCH 1582/2974] ESMTPSenderFactory takes a message of bytes

---
 scrapy/mail.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/scrapy/mail.py b/scrapy/mail.py
index 0bb395521..7365f25b7 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -5,10 +5,10 @@ See documentation in docs/topics/email.rst
 """
 import logging
 
-from six.moves import cStringIO as StringIO
 import six
 
 from email.utils import COMMASPACE, formatdate
+from io import BytesIO
 from six.moves.email_mime_multipart import MIMEMultipart
 from six.moves.email_mime_text import MIMEText
 from six.moves.email_mime_base import MIMEBase
@@ -21,6 +21,14 @@ else:
 
 from twisted.internet import defer, reactor, ssl
 
+try:
+    from twisted.mail.smtp import ESMTPSenderFactory
+except ImportError:
+    """
+    twisted.mail.smtp was not available in
+    older versions of Twisted on Python 3
+    """
+
 from .utils.misc import arg_to_iter
 
 logger = logging.getLogger(__name__)
@@ -110,9 +118,7 @@ class MailSender(object):
                       'mailattachs': nattachs, 'mailerr': errstr})
 
     def _sendmail(self, to_addrs, msg):
-        # Import twisted.mail here because it is not available in python3
-        from twisted.mail.smtp import ESMTPSenderFactory
-        msg = StringIO(msg)
+        msg = BytesIO(msg.encode('utf-8'))
         d = defer.Deferred()
         factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \
             to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \

From 12c7628fcbcfb3927f595e1fd6806ea6aefdf6fd Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 8 Aug 2017 17:13:45 +0200
Subject: [PATCH 1583/2974] Encode message using supplied charset

---
 scrapy/mail.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/scrapy/mail.py b/scrapy/mail.py
index 7365f25b7..7f237820f 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -5,10 +5,13 @@ See documentation in docs/topics/email.rst
 """
 import logging
 
+try:
+    from cStringIO import StringIO as BytesIO
+except ImportError:
+    from io import BytesIO
 import six
 
 from email.utils import COMMASPACE, formatdate
-from io import BytesIO
 from six.moves.email_mime_multipart import MIMEMultipart
 from six.moves.email_mime_text import MIMEText
 from six.moves.email_mime_base import MIMEBase
@@ -21,14 +24,6 @@ else:
 
 from twisted.internet import defer, reactor, ssl
 
-try:
-    from twisted.mail.smtp import ESMTPSenderFactory
-except ImportError:
-    """
-    twisted.mail.smtp was not available in
-    older versions of Twisted on Python 3
-    """
-
 from .utils.misc import arg_to_iter
 
 logger = logging.getLogger(__name__)
@@ -96,7 +91,7 @@ class MailSender(object):
                           'mailattachs': len(attachs)})
             return
 
-        dfd = self._sendmail(rcpts, msg.as_string())
+        dfd = self._sendmail(rcpts, msg.as_string().encode(charset or 'utf-8'))
         dfd.addCallbacks(self._sent_ok, self._sent_failed,
             callbackArgs=[to, cc, subject, len(attachs)],
             errbackArgs=[to, cc, subject, len(attachs)])
@@ -118,7 +113,9 @@ class MailSender(object):
                       'mailattachs': nattachs, 'mailerr': errstr})
 
     def _sendmail(self, to_addrs, msg):
-        msg = BytesIO(msg.encode('utf-8'))
+        # Import twisted.mail here because it is not available in python3
+        from twisted.mail.smtp import ESMTPSenderFactory
+        msg = BytesIO(msg)
         d = defer.Deferred()
         factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \
             to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \

From 80bb4fcf9710598138d7604190636d723d6392df Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 8 Aug 2017 17:40:36 +0200
Subject: [PATCH 1584/2974] Convert SMTP credentials to bytes if needed

---
 scrapy/mail.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scrapy/mail.py b/scrapy/mail.py
index 7f237820f..6d809dc99 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -24,7 +24,8 @@ else:
 
 from twisted.internet import defer, reactor, ssl
 
-from .utils.misc import arg_to_iter
+from scrapy.utils.misc import arg_to_iter
+from scrapy.utils.python import to_bytes
 
 logger = logging.getLogger(__name__)
 
@@ -35,8 +36,8 @@ class MailSender(object):
             smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):
         self.smtphost = smtphost
         self.smtpport = smtpport
-        self.smtpuser = smtpuser
-        self.smtppass = smtppass
+        self.smtpuser = to_bytes(smtpuser)
+        self.smtppass = to_bytes(smtppass)
         self.smtptls = smtptls
         self.smtpssl = smtpssl
         self.mailfrom = mailfrom

From 0d8a33fddccd7083c0f72fe75f89e9e7fd52cd82 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 8 Aug 2017 17:42:56 +0200
Subject: [PATCH 1585/2974] Update docs

---
 docs/topics/email.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index aac93a91a..949cdc638 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -54,10 +54,10 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
     :param smtpuser: the SMTP user. If omitted, the :setting:`MAIL_USER`
       setting will be used. If not given, no SMTP authentication will be
       performed.
-    :type smtphost: str
+    :type smtphost: str or bytes
 
     :param smtppass: the SMTP pass for authentication.
-    :type smtppass: str
+    :type smtppass: str or bytes
 
     :param smtpport: the SMTP port to connect to
     :type smtpport: int

From 9cd348d94af782008f1a561a0b36da9231878833 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 22 Aug 2017 12:29:47 +0200
Subject: [PATCH 1586/2974] Handle None values for smtp user and password

---
 scrapy/mail.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/mail.py b/scrapy/mail.py
index 6d809dc99..0cfb4ec79 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -36,8 +36,8 @@ class MailSender(object):
             smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):
         self.smtphost = smtphost
         self.smtpport = smtpport
-        self.smtpuser = to_bytes(smtpuser)
-        self.smtppass = to_bytes(smtppass)
+        self.smtpuser = to_bytes(smtpuser) if smtpuser is not None else None
+        self.smtppass = to_bytes(smtppass) if smtppass is not None else None
         self.smtptls = smtptls
         self.smtpssl = smtpssl
         self.mailfrom = mailfrom

From f729d74886be2290fcbbcaa21d366b770ff21008 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 28 Aug 2017 11:21:11 +0200
Subject: [PATCH 1587/2974] Use a helper for to_bytes() and None input

---
 scrapy/mail.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/scrapy/mail.py b/scrapy/mail.py
index 0cfb4ec79..5b944e1c4 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -30,14 +30,20 @@ from scrapy.utils.python import to_bytes
 logger = logging.getLogger(__name__)
 
 
+def _to_bytes_or_none(text):
+    if text is None:
+        return None
+    return to_bytes(text)
+
+
 class MailSender(object):
 
     def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost',
             smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):
         self.smtphost = smtphost
         self.smtpport = smtpport
-        self.smtpuser = to_bytes(smtpuser) if smtpuser is not None else None
-        self.smtppass = to_bytes(smtppass) if smtppass is not None else None
+        self.smtpuser = _to_bytes_or_none(smtpuser)
+        self.smtppass = _to_bytes_or_none(smtppass)
         self.smtptls = smtptls
         self.smtpssl = smtpssl
         self.mailfrom = mailfrom

From e914556adf8e556d4184db415c49266cc4c91bf5 Mon Sep 17 00:00:00 2001
From: NoExitTV <fredrikopettersson@hotmail.com>
Date: Thu, 5 Oct 2017 15:12:01 +0200
Subject: [PATCH 1588/2974] Changed the log message to make it more clear. As
 requested in issue #2927

---
 scrapy/core/downloader/handlers/http11.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 9bfdd803c..23343d92a 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -412,9 +412,10 @@ class _ResponseReader(protocol.Protocol):
 
         if self._maxsize and self._bytes_received > self._maxsize:
             logger.error("Received (%(bytes)s) bytes larger than download "
-                         "max size (%(maxsize)s).",
+                         "max size (%(maxsize)s) in request %(request)s.",
                          {'bytes': self._bytes_received,
-                          'maxsize': self._maxsize})
+                          'maxsize': self._maxsize,
+                          'request': self._request})
             # Clear buffer earlier to avoid keeping data in memory for a long
             # time.
             self._bodybuf.truncate(0)

From 938bc18405ca2cf60836bb8490c391c0fe445af1 Mon Sep 17 00:00:00 2001
From: NoExitTV <fredrikopettersson@hotmail.com>
Date: Thu, 5 Oct 2017 15:31:00 +0200
Subject: [PATCH 1589/2974] Changed the log message to make it more clear. As
 requested in issue #2927

---
 scrapy/core/downloader/handlers/http11.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 23343d92a..0a5538947 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -344,8 +344,8 @@ class ScrapyAgent(object):
 
         if warnsize and expected_size > warnsize:
             logger.warning("Expected response size (%(size)s) larger than "
-                           "download warn size (%(warnsize)s).",
-                           {'size': expected_size, 'warnsize': warnsize})
+                           "download warn size (%(warnsize)s) in request (%(request)s).",
+                           {'size': expected_size, 'warnsize': warnsize, 'request': request})
 
         def _cancel(_):
             # Abort connection inmediately.
@@ -412,10 +412,9 @@ class _ResponseReader(protocol.Protocol):
 
         if self._maxsize and self._bytes_received > self._maxsize:
             logger.error("Received (%(bytes)s) bytes larger than download "
-                         "max size (%(maxsize)s) in request %(request)s.",
+                         "max size (%(maxsize)s).",
                          {'bytes': self._bytes_received,
-                          'maxsize': self._maxsize,
-                          'request': self._request})
+                          'maxsize': self._maxsize})
             # Clear buffer earlier to avoid keeping data in memory for a long
             # time.
             self._bodybuf.truncate(0)

From 345d948f2f55ad81ec7de7cb1f5619d80971a6c0 Mon Sep 17 00:00:00 2001
From: NoExitTV <fredrikopettersson@hotmail.com>
Date: Thu, 5 Oct 2017 15:37:05 +0200
Subject: [PATCH 1590/2974] Changed the log message to make it more clear. As
 requested in issue #2927

---
 scrapy/core/downloader/handlers/http11.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 0a5538947..4e1bb0cd5 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -344,7 +344,7 @@ class ScrapyAgent(object):
 
         if warnsize and expected_size > warnsize:
             logger.warning("Expected response size (%(size)s) larger than "
-                           "download warn size (%(warnsize)s) in request (%(request)s).",
+                           "download warn size (%(warnsize)s) in request %(request)s.",
                            {'size': expected_size, 'warnsize': warnsize, 'request': request})
 
         def _cancel(_):

From 9b8503011e1da0507e82e15631194ed99b7e699a Mon Sep 17 00:00:00 2001
From: NoExitTV <fredrikopettersson@hotmail.com>
Date: Fri, 6 Oct 2017 13:45:35 +0200
Subject: [PATCH 1591/2974] Changed log message to include information about
 request as user djunzu commented

---
 scrapy/core/downloader/handlers/http11.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 4e1bb0cd5..48d2481b4 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -412,9 +412,10 @@ class _ResponseReader(protocol.Protocol):
 
         if self._maxsize and self._bytes_received > self._maxsize:
             logger.error("Received (%(bytes)s) bytes larger than download "
-                         "max size (%(maxsize)s).",
+                         "max size (%(maxsize)s) in request %(request)s.",
                          {'bytes': self._bytes_received,
-                          'maxsize': self._maxsize})
+                          'maxsize': self._maxsize,
+                          'request': self._request})
             # Clear buffer earlier to avoid keeping data in memory for a long
             # time.
             self._bodybuf.truncate(0)

From 9cdf34b7c791359b1f86678f758cf64368723c53 Mon Sep 17 00:00:00 2001
From: Aditya <code.aditya@gmail.com>
Date: Tue, 10 Oct 2017 22:49:22 +0530
Subject: [PATCH 1592/2974] Link "Debugging in Python" article to its new
 location

Reference: https://web.archive.org/web/20170203104051/http://www.ferg.org/papers/debugging_in_python.html
---
 docs/topics/extensions.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 7a67cf295..c421a5e05 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -280,7 +280,7 @@ An integer which specifies a number of items. If the spider scrapes more than
 that amount and those items are passed by the item pipeline, the
 spider will be closed with the reason ``closespider_itemcount``.
 Requests which  are currently in the downloader queue (up to
-:setting:`CONCURRENT_REQUESTS` requests) are still processed. 
+:setting:`CONCURRENT_REQUESTS` requests) are still processed.
 If zero (or non set), spiders won't be closed by number of passed items.
 
 .. setting:: CLOSESPIDER_PAGECOUNT
@@ -373,4 +373,4 @@ For more info see `Debugging in Python`.
 This extension only works on POSIX-compliant platforms (ie. not Windows).
 
 .. _Python debugger: https://docs.python.org/2/library/pdb.html
-.. _Debugging in Python: http://www.ferg.org/papers/debugging_in_python.html
+.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/

From 8288f78a39939dbffea467bf110c64e005847117 Mon Sep 17 00:00:00 2001
From: djunzu <djunzu@users.noreply.github.com>
Date: Mon, 16 Oct 2017 21:34:37 -0200
Subject: [PATCH 1593/2974] Add note about request.meta['depth'] in
 DepthMiddleware

---
 docs/topics/spider-middleware.rst | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 9a0ccd0c1..a2d2556c5 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -188,9 +188,13 @@ DepthMiddleware
 
 .. class:: DepthMiddleware
 
-   DepthMiddleware is a scrape middleware used for tracking the depth of each
-   Request inside the site being scraped. It can be used to limit the maximum
-   depth to scrape or things like that.
+   DepthMiddleware is used for tracking the depth of each Request inside the
+   site being scraped. It works by setting `request.meta['depth'] = 0` whenever
+   there is no value previously set (usually just the first Request) and
+   incrementing it by 1 otherwise.
+
+   It can be used to limit the maximum depth to scrape, control Request
+   priority based on their depth, and things like that.
 
    The :class:`DepthMiddleware` can be configured through the following
    settings (see the settings documentation for more info):

From 169dc2860e9f7054c50c84f5adcd8a0d5afe161e Mon Sep 17 00:00:00 2001
From: Weldon Malbrough <weldon0405@yahoo.com>
Date: Mon, 16 Oct 2017 22:46:32 -0400
Subject: [PATCH 1594/2974] Update tutorial.rst startproject files

Added middlewares.py to accurately reflect the file structure created by "scrapy startproject tutorial"
---
 docs/intro/tutorial.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 29f54bc86..a02c759bb 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -54,6 +54,8 @@ This will create a ``tutorial`` directory with the following contents::
             __init__.py
 
             items.py          # project items definition file
+            
+            middlewares.py    # project middlewares file
 
             pipelines.py      # project pipelines file
 

From 95815d27e89a6eea4676358697e2846959e4725b Mon Sep 17 00:00:00 2001
From: Weldon Malbrough <weldon0405@yahoo.com>
Date: Wed, 25 Oct 2017 23:16:30 -0400
Subject: [PATCH 1595/2974] updated file structure to include middlewares.py

---
 docs/topics/commands.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index dc8067d7e..b15349598 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -55,6 +55,7 @@ structure by default, similar to this::
    myproject/
        __init__.py
        items.py
+       middlewares.py
        pipelines.py
        settings.py
        spiders/

From 9dd680d5c94340ac308f1450d9d3dc226a015326 Mon Sep 17 00:00:00 2001
From: Aditya <code.aditya@gmail.com>
Date: Thu, 26 Oct 2017 23:32:20 +0530
Subject: [PATCH 1596/2974] Use https for external links wherever possible in
 docs

---
 README.rst                       |  4 ++--
 artwork/README.rst               |  4 ++--
 docs/contributing.rst            |  2 +-
 docs/faq.rst                     |  2 +-
 docs/intro/install.rst           | 12 ++++++------
 docs/intro/overview.rst          |  2 +-
 docs/intro/tutorial.rst          |  2 +-
 docs/topics/debug.rst            |  2 +-
 docs/topics/deploy.rst           |  2 +-
 docs/topics/firebug.rst          |  2 +-
 docs/topics/firefox.rst          |  8 ++++----
 docs/topics/jobs.rst             |  2 +-
 docs/topics/loaders.rst          |  4 ++--
 docs/topics/media-pipeline.rst   |  2 +-
 docs/topics/practices.rst        |  4 ++--
 docs/topics/request-response.rst |  2 +-
 docs/topics/selectors.rst        |  2 +-
 docs/topics/settings.rst         |  4 ++--
 docs/topics/shell.rst            |  6 +++---
 docs/topics/spiders.rst          |  4 ++--
 extras/coverage-report.sh        |  2 +-
 21 files changed, 37 insertions(+), 37 deletions(-)

diff --git a/README.rst b/README.rst
index 27fab8e29..da63f2b93 100644
--- a/README.rst
+++ b/README.rst
@@ -7,7 +7,7 @@ Scrapy
    :alt: PyPI Version
 
 .. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
-   :target: http://travis-ci.org/scrapy/scrapy
+   :target: https://travis-ci.org/scrapy/scrapy
    :alt: Build Status
 
 .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
@@ -15,7 +15,7 @@ Scrapy
    :alt: Wheel Status
 
 .. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg
-   :target: http://codecov.io/github/scrapy/scrapy?branch=master
+   :target: https://codecov.io/github/scrapy/scrapy?branch=master
    :alt: Coverage report
 
 .. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg
diff --git a/artwork/README.rst b/artwork/README.rst
index 016462f2c..92f6ecb7e 100644
--- a/artwork/README.rst
+++ b/artwork/README.rst
@@ -10,10 +10,10 @@ scrapy-logo.jpg
 
 Main Scrapy logo, in JPEG format.
 
-qlassik.zip 
+qlassik.zip
 -----------
 
-Font used for Scrapy logo. Homepage: http://www.dafont.com/qlassik.font
+Font used for Scrapy logo. Homepage: https://www.dafont.com/qlassik.font
 
 scrapy-blog.logo.xcf
 --------------------
diff --git a/docs/contributing.rst b/docs/contributing.rst
index c969bd842..291a1054e 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -188,7 +188,7 @@ And their unit-tests are in::
 
 .. _issue tracker: https://github.com/scrapy/scrapy/issues
 .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
-.. _Scrapy subreddit: http://reddit.com/r/scrapy
+.. _Scrapy subreddit: https://reddit.com/r/scrapy
 .. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
 .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
 .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
diff --git a/docs/faq.rst b/docs/faq.rst
index f0ee20b5e..42c3abbfa 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -21,7 +21,7 @@ Python code.
 In other words, comparing `BeautifulSoup`_ (or `lxml`_) to Scrapy is like
 comparing `jinja2`_ to `Django`_.
 
-.. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/
+.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
 .. _lxml: http://lxml.de/
 .. _jinja2: http://jinja.pocoo.org/
 .. _Django: https://www.djangoproject.com/
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index deb0118d4..12d489612 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -188,7 +188,7 @@ solutions:
   that doesn't conflict with the rest of your system. Here's how to do it using
   the `homebrew`_ package manager:
 
-  * Install `homebrew`_ following the instructions in http://brew.sh/
+  * Install `homebrew`_ following the instructions in https://brew.sh/
 
   * Update your ``PATH`` variable to state that homebrew packages should be
     used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly
@@ -233,9 +233,9 @@ After any of these workarounds you should be able to install Scrapy::
 .. _pyOpenSSL: https://pypi.python.org/pypi/pyOpenSSL
 .. _setuptools: https://pypi.python.org/pypi/setuptools
 .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
-.. _homebrew: http://brew.sh/
-.. _zsh: http://www.zsh.org/
+.. _homebrew: https://brew.sh/
+.. _zsh: https://www.zsh.org/
 .. _Scrapinghub: https://scrapinghub.com
-.. _Anaconda: http://docs.continuum.io/anaconda/index
-.. _Miniconda: http://conda.pydata.org/docs/install/quick.html
-.. _conda-forge: https://conda-forge.github.io/
+.. _Anaconda: https://docs.anaconda.com/anaconda/
+.. _Miniconda: https://conda.io/docs/user-guide/install/index.html
+.. _conda-forge: https://conda-forge.org/
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 1da1a4059..d0ce07a8e 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -164,4 +164,4 @@ interest!
 .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
 .. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html
 .. _Amazon S3: https://aws.amazon.com/s3/
-.. _Sitemaps: http://www.sitemaps.org
+.. _Sitemaps: https://www.sitemaps.org/index.html
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index a02c759bb..20538e90f 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -34,7 +34,7 @@ list of Python resources for non-programmers`_.
 .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
 .. _Dive Into Python 3: http://www.diveintopython3.net
 .. _Python Tutorial: https://docs.python.org/3/tutorial
-.. _Learn Python The Hard Way: http://learnpythonthehardway.org/book/
+.. _Learn Python The Hard Way: https://learnpythonthehardway.org/book/
 
 
 Creating a project
diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst
index a3e72097c..d1991c02f 100644
--- a/docs/topics/debug.rst
+++ b/docs/topics/debug.rst
@@ -142,4 +142,4 @@ available in all future runs should they be necessary again::
 
 For more information, check the :ref:`topics-logging` section.
 
-.. _base tag: http://www.w3schools.com/tags/tag_base.asp
+.. _base tag: https://www.w3schools.com/tags/tag_base.asp
diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst
index f2e11fe8f..361914a29 100644
--- a/docs/topics/deploy.rst
+++ b/docs/topics/deploy.rst
@@ -55,5 +55,5 @@ just like ``scrapyd-deploy``.
 .. _scrapyd-client: https://github.com/scrapy/scrapyd-client
 .. _shub: https://doc.scrapinghub.com/shub.html
 .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html
-.. _Scrapy Cloud documentation: http://doc.scrapinghub.com/scrapy-cloud.html
+.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html
 .. _Scrapinghub: https://scrapinghub.com/
diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst
index 8f0a5767b..4ea8d3bd0 100644
--- a/docs/topics/firebug.rst
+++ b/docs/topics/firebug.rst
@@ -23,7 +23,7 @@ In this example, we'll show how to use `Firebug`_ to scrape data from the
 Project`_ used in the :ref:`tutorial <intro-tutorial>` but with a different
 face.
 
-.. _Firebug: http://getfirebug.com
+.. _Firebug: https://getfirebug.com/
 .. _Google Directory: http://directory.google.com/
 .. _Open Directory Project: http://www.dmoz.org
 
diff --git a/docs/topics/firefox.rst b/docs/topics/firefox.rst
index 0cf45861a..2c85848be 100644
--- a/docs/topics/firefox.rst
+++ b/docs/topics/firefox.rst
@@ -17,7 +17,7 @@ when inspecting the page source is not the original HTML, but a modified one
 after applying some browser clean up and executing Javascript code.  Firefox,
 in particular, is known for adding ``<tbody>`` elements to tables.  Scrapy, on
 the other hand, does not modify the original page HTML, so you won't be able to
-extract any data if you use ``<tbody>`` in your XPath expressions. 
+extract any data if you use ``<tbody>`` in your XPath expressions.
 
 Therefore, you should keep in mind the following things when working with
 Firefox and XPath:
@@ -71,11 +71,11 @@ Firecookie
 
 `Firecookie`_ makes it easier to view and manage cookies. You can use this
 extension to create a new cookie, delete existing cookies, see a list of cookies
-for the current site, manage cookies permissions and a lot more. 
+for the current site, manage cookies permissions and a lot more.
 
-.. _Firebug: http://getfirebug.com
+.. _Firebug: https://getfirebug.com/
 .. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA
-.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/ 
+.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/
 .. _XPath Checker: https://addons.mozilla.org/en-US/firefox/addon/xpath-checker/
 .. _Tamper Data: https://addons.mozilla.org/en-US/firefox/addon/tamper-data/
 .. _Firecookie: https://addons.mozilla.org/en-US/firefox/addon/firecookie/
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 4f9e38086..06c7fff3d 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -100,4 +100,4 @@ If you wish to log the requests that couldn't be serialized, you can set the
 :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
 It is ``False`` by default.
 
-.. _pickle: http://docs.python.org/library/pickle.html
+.. _pickle: https://docs.python.org/library/pickle.html
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index ad86dba63..0849090b4 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -518,8 +518,8 @@ a footer of a page that looks something like:
 Example::
 
     <footer>
-        <a class="social" href="http://facebook.com/whatever">Like Us</a>
-        <a class="social" href="http://twitter.com/whatever">Follow Us</a>
+        <a class="social" href="https://facebook.com/whatever">Like Us</a>
+        <a class="social" href="https://twitter.com/whatever">Follow Us</a>
         <a class="email" href="mailto:whatever@example.com">Email Us</a>
     </footer>
 
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 9580a15d9..4c634ace5 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -171,7 +171,7 @@ policy::
 
 For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
 
-.. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
 Google Cloud Storage
 ---------------------
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 21aa4a0a7..e0dd4000f 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -249,9 +249,9 @@ If you are still unable to prevent your bot getting banned, consider contacting
 
 .. _Tor project: https://www.torproject.org/
 .. _commercial support: http://scrapy.org/support/
-.. _ProxyMesh: http://proxymesh.com/
+.. _ProxyMesh: https://proxymesh.com/
 .. _Google cache: http://www.googleguide.com/cached_pages.html
 .. _testspiders: https://github.com/scrapinghub/testspiders
 .. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
 .. _Crawlera: https://scrapinghub.com/crawlera
-.. _scrapoxy: http://scrapoxy.io/
+.. _scrapoxy: https://scrapoxy.io/
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 92aae1ad0..121abe6b5 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -734,7 +734,7 @@ HtmlResponse objects
     which adds encoding auto-discovering support by looking into the HTML `meta
     http-equiv`_ attribute.  See :attr:`TextResponse.encoding`.
 
-.. _meta http-equiv: http://www.w3schools.com/TAGS/att_meta_http_equiv.asp
+.. _meta http-equiv: https://www.w3schools.com/TAGS/att_meta_http_equiv.asp
 
 XmlResponse objects
 -------------------
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 9bae53f45..cb4c25391 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -36,7 +36,7 @@ documents.
 For a complete reference of the selectors API see
 :ref:`Selector reference <topics-selectors-ref>`
 
-.. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/
+.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
 .. _lxml: http://lxml.de/
 .. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html
 .. _cssselect: https://pypi.python.org/pypi/cssselect/
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 37e3828a4..4b15cb607 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1002,7 +1002,7 @@ The randomization policy is the same used by `wget`_ ``--random-wait`` option.
 
 If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect.
 
-.. _wget: http://www.gnu.org/software/wget/manual/wget.html
+.. _wget: https://www.gnu.org/software/wget/manual/wget.html
 
 .. setting:: REACTOR_THREADPOOL_MAXSIZE
 
@@ -1317,7 +1317,7 @@ Default: ``2083``
 Scope: ``spidermiddlewares.urllength``
 
 The maximum URL length to allow for crawled URLs. For more information about
-the default value for this setting see: http://www.boutell.com/newfaq/misc/urllength.html
+the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html
 
 .. setting:: USER_AGENT
 
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index ef6aeeed3..527116418 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -39,9 +39,9 @@ variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
     [settings]
     shell = bpython
 
-.. _IPython: http://ipython.org/
-.. _IPython installation guide: http://ipython.org/install.html
-.. _bpython: http://www.bpython-interpreter.org/
+.. _IPython: https://ipython.org/
+.. _IPython installation guide: https://ipython.org/install.html
+.. _bpython: https://www.bpython-interpreter.org/
 
 Launch the shell
 ================
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index bf1532d1b..c2c271245 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -751,8 +751,8 @@ Combine SitemapSpider with other sources of urls::
         def parse_other(self, response):
             pass # ... scrape other here ...
 
-.. _Sitemaps: http://www.sitemaps.org
-.. _Sitemap index files: http://www.sitemaps.org/protocol.html#index
+.. _Sitemaps: https://www.sitemaps.org/index.html
+.. _Sitemap index files: https://www.sitemaps.org/protocol.html#index
 .. _robots.txt: http://www.robotstxt.org/
 .. _TLD: https://en.wikipedia.org/wiki/Top-level_domain
 .. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/
diff --git a/extras/coverage-report.sh b/extras/coverage-report.sh
index dc20e16e4..842d0e46e 100755
--- a/extras/coverage-report.sh
+++ b/extras/coverage-report.sh
@@ -1,6 +1,6 @@
 # Run tests, generate coverage report and open it on a browser
 #
-# Requires: coverage 3.3 or above from http://pypi.python.org/pypi/coverage
+# Requires: coverage 3.3 or above from https://pypi.python.org/pypi/coverage
 
 coverage run --branch $(which trial) --reporter=text tests
 coverage html -i

From 9d9d83a8c31b6a18d7aaac35a30ffb69db4bb81d Mon Sep 17 00:00:00 2001
From: Aditya <code.aditya@gmail.com>
Date: Sat, 28 Oct 2017 16:24:40 +0530
Subject: [PATCH 1597/2974] Use https links wherever possible

---
 CONTRIBUTING.md                               |  4 ++--
 INSTALL                                       |  2 +-
 README.rst                                    | 14 ++++++-------
 debian/control                                |  6 +++---
 debian/copyright                              |  8 ++++----
 docs/contributing.rst                         |  2 +-
 docs/intro/overview.rst                       |  2 +-
 docs/topics/practices.rst                     |  2 +-
 docs/topics/selectors.rst                     |  4 ++--
 docs/topics/shell.rst                         |  8 ++++----
 scrapy/_monkeypatches.py                      |  4 ++--
 scrapy/core/downloader/contextfactory.py      |  4 ++--
 scrapy/crawler.py                             |  2 +-
 scrapy/downloadermiddlewares/chunked.py       |  2 +-
 scrapy/downloadermiddlewares/httpcache.py     |  2 +-
 scrapy/exporters.py                           |  2 +-
 scrapy/extensions/httpcache.py                | 10 +++++-----
 scrapy/extensions/telnet.py                   |  2 +-
 scrapy/pipelines/files.py                     |  4 ++--
 scrapy/signalmanager.py                       |  2 +-
 scrapy/templates/project/module/items.py.tmpl |  2 +-
 .../project/module/middlewares.py.tmpl        |  2 +-
 .../project/module/pipelines.py.tmpl          |  2 +-
 .../templates/project/module/settings.py.tmpl | 20 +++++++++----------
 scrapy/utils/defer.py                         |  2 +-
 scrapy/utils/deprecate.py                     | 12 +++++------
 scrapy/utils/http.py                          |  2 +-
 scrapy/utils/log.py                           |  2 +-
 scrapy/utils/url.py                           |  2 +-
 sep/sep-001.rst                               |  2 +-
 sep/sep-006.rst                               |  4 ++--
 sep/sep-013.rst                               |  2 +-
 sep/sep-017.rst                               |  2 +-
 sep/sep-020.rst                               |  2 +-
 setup.py                                      |  2 +-
 tests/__init__.py                             |  2 +-
 tests/keys/example-com.conf                   |  4 ++--
 37 files changed, 76 insertions(+), 76 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 88c472f6f..0a11b05d2 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,6 +1,6 @@
 The guidelines for contributing are available here:
-http://doc.scrapy.org/en/master/contributing.html
+https://doc.scrapy.org/en/master/contributing.html
 
 Please do not abuse the issue tracker for support questions.
 If your issue topic can be rephrased to "How to ...?", please use the
-support channels to get it answered: http://scrapy.org/community/
+support channels to get it answered: https://scrapy.org/community/
diff --git a/INSTALL b/INSTALL
index 84803a933..a3c7899c6 100644
--- a/INSTALL
+++ b/INSTALL
@@ -1,4 +1,4 @@
 For information about installing Scrapy see:
 
 * docs/intro/install.rst (local file)
-* http://doc.scrapy.org/en/latest/intro/install.html (online version)
+* https://doc.scrapy.org/en/latest/intro/install.html (online version)
diff --git a/README.rst b/README.rst
index da63f2b93..45135c7a2 100644
--- a/README.rst
+++ b/README.rst
@@ -31,7 +31,7 @@ crawl websites and extract structured data from their pages. It can be used for
 a wide range of purposes, from data mining to monitoring and automated testing.
 
 For more information including a list of features check the Scrapy homepage at:
-http://scrapy.org
+https://scrapy.org
 
 Requirements
 ============
@@ -47,12 +47,12 @@ The quick way::
     pip install scrapy
 
 For more details see the install section in the documentation:
-http://doc.scrapy.org/en/latest/intro/install.html
+https://doc.scrapy.org/en/latest/intro/install.html
 
 Documentation
 =============
 
-Documentation is available online at http://doc.scrapy.org/ and in the ``docs``
+Documentation is available online at https://doc.scrapy.org/ and in the ``docs``
 directory.
 
 Releases
@@ -63,12 +63,12 @@ You can find release notes at https://doc.scrapy.org/en/latest/news.html
 Community (blog, twitter, mail list, IRC)
 =========================================
 
-See http://scrapy.org/community/
+See https://scrapy.org/community/
 
 Contributing
 ============
 
-See http://doc.scrapy.org/en/master/contributing.html
+See https://doc.scrapy.org/en/master/contributing.html
 
 Code of Conduct
 ---------------
@@ -82,9 +82,9 @@ Please report unacceptable behavior to opensource@scrapinghub.com.
 Companies using Scrapy
 ======================
 
-See http://scrapy.org/companies/
+See https://scrapy.org/companies/
 
 Commercial Support
 ==================
 
-See http://scrapy.org/support/
+See https://scrapy.org/support/
diff --git a/debian/control b/debian/control
index f3a31753b..2cc8eedf4 100644
--- a/debian/control
+++ b/debian/control
@@ -4,7 +4,7 @@ Priority: optional
 Maintainer: Scrapinghub Team <info@scrapinghub.com>
 Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2)
 Standards-Version: 3.8.4
-Homepage: http://scrapy.org/
+Homepage: https://scrapy.org/
 
 Package: scrapy
 Architecture: all
@@ -15,6 +15,6 @@ Conflicts: python-scrapy, scrapy-0.25
 Provides: python-scrapy, scrapy-0.25
 Description: Python web crawling and web scraping framework
  Scrapy is a fast high-level web crawling and web scraping framework,
- used to crawl websites and extract structured data from their pages. 
- It can be used for a wide range of purposes, from data mining to 
+ used to crawl websites and extract structured data from their pages.
+ It can be used for a wide range of purposes, from data mining to
  monitoring and automated testing.
diff --git a/debian/copyright b/debian/copyright
index 4cc239002..c1bf47565 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -1,6 +1,6 @@
 This package was debianized by the Scrapinghub team <info@scrapinghub.com>.
 
-It was downloaded from http://scrapy.org
+It was downloaded from https://scrapy.org
 
 Upstream Author: Scrapy Developers
 
@@ -14,10 +14,10 @@ All rights reserved.
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:
 
-    1. Redistributions of source code must retain the above copyright notice, 
+    1. Redistributions of source code must retain the above copyright notice,
        this list of conditions and the following disclaimer.
-    
-    2. Redistributions in binary form must reproduce the above copyright 
+
+    2. Redistributions in binary form must reproduce the above copyright
        notice, this list of conditions and the following disclaimer in the
        documentation and/or other materials provided with the distribution.
 
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 291a1054e..f3732ab06 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -7,7 +7,7 @@ Contributing to Scrapy
 .. important::
 
     Double check you are reading the most recent version of this document at
-    http://doc.scrapy.org/en/master/contributing.html
+    https://doc.scrapy.org/en/master/contributing.html
 
 There are many ways to contribute to Scrapy. Here are some of them:
 
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index d0ce07a8e..6f1c2c43f 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -160,7 +160,7 @@ The next steps for you are to :ref:`install Scrapy <intro-install>`,
 a full-blown Scrapy project and `join the community`_. Thanks for your
 interest!
 
-.. _join the community: http://scrapy.org/community/
+.. _join the community: https://scrapy.org/community/
 .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
 .. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html
 .. _Amazon S3: https://aws.amazon.com/s3/
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index e0dd4000f..02cfa9b05 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -248,7 +248,7 @@ If you are still unable to prevent your bot getting banned, consider contacting
 `commercial support`_.
 
 .. _Tor project: https://www.torproject.org/
-.. _commercial support: http://scrapy.org/support/
+.. _commercial support: https://scrapy.org/support/
 .. _ProxyMesh: https://proxymesh.com/
 .. _Google cache: http://www.googleguide.com/cached_pages.html
 .. _testspiders: https://github.com/scrapinghub/testspiders
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index cb4c25391..8ac40c3cc 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -86,7 +86,7 @@ To explain how to use the selectors we'll use the `Scrapy shell` (which
 provides interactive testing) and an example page located in the Scrapy
 documentation server:
 
-    http://doc.scrapy.org/en/latest/_static/selectors-sample1.html
+    https://doc.scrapy.org/en/latest/_static/selectors-sample1.html
 
 .. _topics-selectors-htmlcode:
 
@@ -99,7 +99,7 @@ Here's its HTML code:
 
 First, let's open the shell::
 
-    scrapy shell http://doc.scrapy.org/en/latest/_static/selectors-sample1.html
+    scrapy shell https://doc.scrapy.org/en/latest/_static/selectors-sample1.html
 
 Then, after the shell loads, you'll have the response available as ``response``
 shell variable, and its attached selector in ``response.selector`` attribute.
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 527116418..11ab199f2 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -142,7 +142,7 @@ Example of shell session
 ========================
 
 Here's an example of a typical shell session where we start by scraping the
-http://scrapy.org page, and then proceed to scrape the https://reddit.com
+https://scrapy.org page, and then proceed to scrape the https://reddit.com
 page. Finally, we modify the (Reddit) request method to POST and re-fetch it
 getting an error. We end the session by typing Ctrl-D (in Unix systems) or
 Ctrl-Z in Windows.
@@ -154,7 +154,7 @@ shell works.
 
 First, we launch the shell::
 
-    scrapy shell 'http://scrapy.org' --nolog
+    scrapy shell 'https://scrapy.org' --nolog
 
 Then, the shell fetches the URL (using the Scrapy downloader) and prints the
 list of available objects and useful shortcuts (you'll notice that these lines
@@ -164,7 +164,7 @@ all start with the ``[s]`` prefix)::
     [s]   scrapy     scrapy module (contains scrapy.Request, scrapy.Selector, etc)
     [s]   crawler    <scrapy.crawler.Crawler object at 0x7f07395dd690>
     [s]   item       {}
-    [s]   request    <GET http://scrapy.org>
+    [s]   request    <GET https://scrapy.org>
     [s]   response   <200 https://scrapy.org/>
     [s]   settings   <scrapy.settings.Settings object at 0x7f07395dd710>
     [s]   spider     <DefaultSpider 'default' at 0x7f0735891690>
@@ -182,7 +182,7 @@ After that, we can start playing with the objects::
     >>> response.xpath('//title/text()').extract_first()
     'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
 
-    >>> fetch("http://reddit.com")
+    >>> fetch("https://reddit.com")
 
     >>> response.xpath('//title/text()').extract()
     ['reddit: the front page of the internet']
diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index 60e0de1f2..f55ecc213 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -4,12 +4,12 @@ from six.moves import copyreg
 if sys.version_info[0] == 2:
     from urlparse import urlparse
 
-    # workaround for http://bugs.python.org/issue7904 - Python < 2.7
+    # workaround for https://bugs.python.org/issue7904 - Python < 2.7
     if urlparse('s3://bucket/key').netloc != 'bucket':
         from urlparse import uses_netloc
         uses_netloc.append('s3')
 
-    # workaround for http://bugs.python.org/issue9374 - Python < 2.7.4
+    # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
     if urlparse('s3://bucket/key?key=value').query != 'key=value':
         from urlparse import uses_query
         uses_query.append('s3')
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index a94a89205..783d4c383 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -64,7 +64,7 @@ if twisted_version >= (14, 0, 0):
         """
         Twisted-recommended context factory for web clients.
 
-        Quoting http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
+        Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
         "The default is to use a BrowserLikePolicyForHTTPS,
         so unless you have special requirements you can leave this as-is."
 
@@ -100,6 +100,6 @@ else:
         def getContext(self, hostname=None, port=None):
             ctx = ClientContextFactory.getContext(self)
             # Enable all workarounds to SSL bugs as documented by
-            # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html
+            # https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_options.html
             ctx.set_options(SSL.OP_ALL)
             return ctx
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index a33ce9805..5cbc2d7c5 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -83,7 +83,7 @@ class Crawler(object):
             yield defer.maybeDeferred(self.engine.start)
         except Exception:
             # In Python 2 reraising an exception after yield discards
-            # the original traceback (see http://bugs.python.org/issue7563),
+            # the original traceback (see https://bugs.python.org/issue7563),
             # so sys.exc_info() workaround is used.
             # This workaround also works in Python 3, but it is not needed,
             # and it is slower, so in Python 3 we use native `raise`.
diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py
index 64d94c489..6748d0265 100644
--- a/scrapy/downloadermiddlewares/chunked.py
+++ b/scrapy/downloadermiddlewares/chunked.py
@@ -11,7 +11,7 @@ warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, "
 
 class ChunkedTransferMiddleware(object):
     """This middleware adds support for chunked transfer encoding, as
-    documented in: http://en.wikipedia.org/wiki/Chunked_transfer_encoding
+    documented in: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
     """
 
     def process_response(self, request, response, spider):
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 30e49b886..495b103d1 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -75,7 +75,7 @@ class HttpCacheMiddleware(object):
             return response
 
         # RFC2616 requires origin server to set Date header,
-        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18
+        # https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18
         if 'Date' not in response.headers:
             response.headers['Date'] = formatdate(usegmt=1)
 
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index e2d42b6ab..07f43b494 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -188,7 +188,7 @@ class XmlItemExporter(BaseItemExporter):
         self.xg.endElement(name)
         self._beautify_newline()
 
-    # Workaround for http://bugs.python.org/issue17606
+    # Workaround for https://bugs.python.org/issue17606
     # Before Python 2.7.4 xml.sax.saxutils required bytes;
     # since 2.7.4 it requires unicode. The bug is likely to be
     # fixed in 2.7.6, but 2.7.6 will still support unicode,
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 648b32ec7..1b5e05b1b 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -70,8 +70,8 @@ class RFC2616Policy(object):
         return True
 
     def should_cache_response(self, response, request):
-        # What is cacheable - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1
-        # Response cacheability - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4
+        # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1
+        # Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4
         # Status code 206 is not included because cache can not deal with partial contents
         cc = self._parse_cachecontrol(response)
         # obey directive "Cache-Control: no-store"
@@ -163,7 +163,7 @@ class RFC2616Policy(object):
 
     def _compute_freshness_lifetime(self, response, request, now):
         # Reference nsHttpResponseHead::ComputeFreshnessLifetime
-        # http://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#410
+        # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#706
         cc = self._parse_cachecontrol(response)
         maxage = self._get_max_age(cc)
         if maxage is not None:
@@ -194,7 +194,7 @@ class RFC2616Policy(object):
 
     def _compute_current_age(self, response, request, now):
         # Reference nsHttpResponseHead::ComputeCurrentAge
-        # http://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#366
+        # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#658
         currentage = 0
         # If Date header is not set we assume it is a fast connection, and
         # clock is in sync with the server
@@ -414,7 +414,7 @@ class LeveldbCacheStorage(object):
 def parse_cachecontrol(header):
     """Parse Cache-Control header
 
-    http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
+    https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
 
     >>> parse_cachecontrol(b'public, max-age=3600') == {b'public': None,
     ...                                                 b'max-age': b'3600'}
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index d9add1d97..5ca0d19a0 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -82,7 +82,7 @@ class TelnetConsole(protocol.ServerFactory):
             'prefs': print_live_refs,
             'hpy': hpy,
             'help': "This is Scrapy telnet console. For more info see: " \
-                "http://doc.scrapy.org/en/latest/topics/telnetconsole.html",
+                "https://doc.scrapy.org/en/latest/topics/telnetconsole.html",
         }
         self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars)
         return telnet_vars
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 7fdb8a086..9f1faa313 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -120,7 +120,7 @@ class S3FilesStore(object):
 
     def _get_boto_bucket(self):
         # disable ssl (is_secure=False) because of this python bug:
-        # http://bugs.python.org/issue5103
+        # https://bugs.python.org/issue5103
         c = self.S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False)
         return c.get_bucket(self.bucket, validate=False)
 
@@ -268,7 +268,7 @@ class FilesPipeline(MediaPipeline):
     def __init__(self, store_uri, download_func=None, settings=None):
         if not store_uri:
             raise NotConfigured
-        
+
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
 
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index fd79905e9..296d27ed8 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -55,7 +55,7 @@ class SignalManager(object):
         The keyword arguments are passed to the signal handlers (connected
         through the :meth:`connect` method).
 
-        .. _deferreds: http://twistedmatrix.com/documents/current/core/howto/defer.html
+        .. _deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html
         """
         kwargs.setdefault('sender', self.sender)
         return _signal.send_catch_log_deferred(signal, **kwargs)
diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl
index 2c746138f..7d766f4fc 100644
--- a/scrapy/templates/project/module/items.py.tmpl
+++ b/scrapy/templates/project/module/items.py.tmpl
@@ -3,7 +3,7 @@
 # Define here the models for your scraped items
 #
 # See documentation in:
-# http://doc.scrapy.org/en/latest/topics/items.html
+# https://doc.scrapy.org/en/latest/topics/items.html
 
 import scrapy
 
diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
index 1a4b0caa5..c5b542bd6 100644
--- a/scrapy/templates/project/module/middlewares.py.tmpl
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -3,7 +3,7 @@
 # Define here the models for your spider middleware
 #
 # See documentation in:
-# http://doc.scrapy.org/en/latest/topics/spider-middleware.html
+# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
 
 from scrapy import signals
 
diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl
index 4e9b32e9e..e58dab089 100644
--- a/scrapy/templates/project/module/pipelines.py.tmpl
+++ b/scrapy/templates/project/module/pipelines.py.tmpl
@@ -3,7 +3,7 @@
 # Define your item pipelines here
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
-# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
+# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 
 
 class ${ProjectName}Pipeline(object):
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 35a0f9a45..a0557473e 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -5,9 +5,9 @@
 # For simplicity, this file contains only settings considered important or
 # commonly used. You can find more settings consulting the documentation:
 #
-#     http://doc.scrapy.org/en/latest/topics/settings.html
-#     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
-#     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
+#     https://doc.scrapy.org/en/latest/topics/settings.html
+#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html
 
 BOT_NAME = '$project_name'
 
@@ -25,7 +25,7 @@ ROBOTSTXT_OBEY = True
 #CONCURRENT_REQUESTS = 32
 
 # Configure a delay for requests for the same website (default: 0)
-# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
+# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
 # See also autothrottle settings and docs
 #DOWNLOAD_DELAY = 3
 # The download delay setting will honor only one of:
@@ -45,31 +45,31 @@ ROBOTSTXT_OBEY = True
 #}
 
 # Enable or disable spider middlewares
-# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
+# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
 #SPIDER_MIDDLEWARES = {
 #    '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543,
 #}
 
 # Enable or disable downloader middlewares
-# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
+# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
 #DOWNLOADER_MIDDLEWARES = {
 #    '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543,
 #}
 
 # Enable or disable extensions
-# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
+# See https://doc.scrapy.org/en/latest/topics/extensions.html
 #EXTENSIONS = {
 #    'scrapy.extensions.telnet.TelnetConsole': None,
 #}
 
 # Configure item pipelines
-# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
+# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 #ITEM_PIPELINES = {
 #    '$project_name.pipelines.${ProjectName}Pipeline': 300,
 #}
 
 # Enable and configure the AutoThrottle extension (disabled by default)
-# See http://doc.scrapy.org/en/latest/topics/autothrottle.html
+# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
 #AUTOTHROTTLE_ENABLED = True
 # The initial download delay
 #AUTOTHROTTLE_START_DELAY = 5
@@ -82,7 +82,7 @@ ROBOTSTXT_OBEY = True
 #AUTOTHROTTLE_DEBUG = False
 
 # Enable and configure HTTP caching (disabled by default)
-# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
 #HTTPCACHE_ENABLED = True
 #HTTPCACHE_EXPIRATION_SECS = 0
 #HTTPCACHE_DIR = 'httpcache'
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index bb4c74a6e..aa6dcffda 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -57,7 +57,7 @@ def parallel(iterable, count, callable, *args, **named):
     """Execute a callable over the objects in the given iterable, in parallel,
     using no more than ``count`` concurrent calls.
 
-    Taken from: http://jcalderone.livejournal.com/24285.html
+    Taken from: https://jcalderone.livejournal.com/24285.html
     """
     coop = task.Cooperator()
     work = (callable(elem, *args, **named) for elem in iterable)
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index 7ab39c97e..f76161a68 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -71,8 +71,8 @@ def create_deprecated_class(name, new_class, clsdict=None,
                 warnings.warn(msg, warn_category, stacklevel=2)
             super(DeprecatedClass, cls).__init__(name, bases, clsdict_)
 
-        # see http://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass
-        # and http://docs.python.org/2/reference/datamodel.html#customizing-instance-and-subclass-checks
+        # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass
+        # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks
         # for implementation details
         def __instancecheck__(cls, inst):
             return any(cls.__subclasscheck__(c)
@@ -159,10 +159,10 @@ def update_classpath(path):
 
 
 def method_is_overridden(subclass, base_class, method_name):
-    """ 
-    Return True if a method named ``method_name`` of a ``base_class`` 
-    is overridden in a ``subclass``. 
-    
+    """
+    Return True if a method named ``method_name`` of a ``base_class``
+    is overridden in a ``subclass``.
+
     >>> class Base(object):
     ...     def foo(self):
     ...         pass
diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py
index 8b659a22a..7cc8d1884 100644
--- a/scrapy/utils/http.py
+++ b/scrapy/utils/http.py
@@ -11,7 +11,7 @@ def decode_chunked_transfer(chunked_body):
     decoded body.
 
     For more info see:
-    http://en.wikipedia.org/wiki/Chunked_transfer_encoding
+    https://en.wikipedia.org/wiki/Chunked_transfer_encoding
 
     """
     body, h, t = '', '', chunked_body
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 7c95e1e50..828880709 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -154,7 +154,7 @@ class StreamLogger(object):
     """Fake file-like stream object that redirects writes to a logger instance
 
     Taken from:
-        http://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/
+        https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/
     """
     def __init__(self, logger, log_level=logging.INFO):
         self.logger = logger
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 8eed31060..657c53815 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -47,7 +47,7 @@ def parse_url(url, encoding=None):
 def escape_ajax(url):
     """
     Return the crawleable url according to:
-    http://code.google.com/web/ajaxcrawling/docs/getting-started.html
+    https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
 
     >>> escape_ajax("www.example.com/ajax.html#!key=value")
     'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
diff --git a/sep/sep-001.rst b/sep/sep-001.rst
index 2f0fe3500..3766f38fc 100644
--- a/sep/sep-001.rst
+++ b/sep/sep-001.rst
@@ -61,7 +61,7 @@ ItemForm
 --------
 
 Pros:
-- same API used for Items (see http://doc.scrapy.org/en/latest/topics/items.html)
+- same API used for Items (see https://doc.scrapy.org/en/latest/topics/items.html)
 - some people consider setitem API more elegant than methods API
 
 Cons:
diff --git a/sep/sep-006.rst b/sep/sep-006.rst
index c0f945b66..522bba134 100644
--- a/sep/sep-006.rst
+++ b/sep/sep-006.rst
@@ -16,7 +16,7 @@ Motivation
 ==========
 
 When you use Selectors in Scrapy, your final goal is to "extract" the data that
-you've selected, as the [http://doc.scrapy.org/en/latest/topics/selectors.html
+you've selected, as the [https://doc.scrapy.org/en/latest/topics/selectors.html
 XPath Selectors documentation] says (bolding by me):
 
    When you’re scraping web pages, the most common task you need to perform is
@@ -71,5 +71,5 @@ webpage or set of pages.
 References
 ==========
 
- 1. XPath Selectors (http://doc.scrapy.org/topics/selectors.html)
+ 1. XPath Selectors (https://doc.scrapy.org/topics/selectors.html)
  2. XPath and XSLT with lxml (http://codespeak.net/lxml/xpathxslt.html)
diff --git a/sep/sep-013.rst b/sep/sep-013.rst
index 4c11a0762..5b18b7501 100644
--- a/sep/sep-013.rst
+++ b/sep/sep-013.rst
@@ -44,7 +44,7 @@ Overview of changes proposed
 
 Most of the inconsistencies come from the fact that middlewares don't follow
 the typical
-[http://twistedmatrix.com/projects/core/documentation/howto/defer.html
+[https://twistedmatrix.com/projects/core/documentation/howto/defer.html
 deferred] callback/errback chaining logic. Twisted logic is fine and quite
 intuitive, and also fits middlewares very well. Due to some bad design choices
 the integration between middleware calls and deferred is far from optional. So
diff --git a/sep/sep-017.rst b/sep/sep-017.rst
index 7707a1622..86005e3c9 100644
--- a/sep/sep-017.rst
+++ b/sep/sep-017.rst
@@ -13,7 +13,7 @@ SEP-017: Spider Contracts
 The motivation for Spider Contracts is to build a lightweight mechanism for
 testing your spiders, and be able to run the tests quickly without having to
 wait for all the spider to run. It's partially based on the
-[http://en.wikipedia.org/wiki/Design_by_contract Design by contract]  approach
+[https://en.wikipedia.org/wiki/Design_by_contract Design by contract]  approach
 (hence its name) where you define certain conditions that spider callbacks must
 met, and you give example testing pages.
 
diff --git a/sep/sep-020.rst b/sep/sep-020.rst
index 49d068479..52d78097b 100644
--- a/sep/sep-020.rst
+++ b/sep/sep-020.rst
@@ -29,7 +29,7 @@ the rows and the further embedded ``<td>`` elements denoting the individual
 fields.
 
 One pattern that is particularly well suited for auto-populating an Item Loader
-is the `definition list <http://www.w3.org/TR/html401/struct/lists.html#h-10.3>`_::
+is the `definition list <https://www.w3.org/TR/html401/struct/lists.html#h-10.3>`_::
 
     <div class="geeks">
         <dl>
diff --git a/setup.py b/setup.py
index c03f0b9f7..327286f5a 100644
--- a/setup.py
+++ b/setup.py
@@ -29,7 +29,7 @@ if has_environment_marker_platform_impl_support():
 setup(
     name='Scrapy',
     version=version,
-    url='http://scrapy.org',
+    url='https://scrapy.org',
     description='A high-level Web Crawling and Web Scraping framework',
     long_description=open('README.rst').read(),
     author='Scrapy developers',
diff --git a/tests/__init__.py b/tests/__init__.py
index c2e4fd2bf..55b1ecde8 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -1,7 +1,7 @@
 """
 tests: this package contains all Scrapy unittests
 
-see http://doc.scrapy.org/en/latest/contributing.html#running-tests
+see https://doc.scrapy.org/en/latest/contributing.html#running-tests
 """
 
 import os
diff --git a/tests/keys/example-com.conf b/tests/keys/example-com.conf
index 8aa338cd5..1f9c25e43 100644
--- a/tests/keys/example-com.conf
+++ b/tests/keys/example-com.conf
@@ -1,4 +1,4 @@
-# this is copied from http://stackoverflow.com/a/27931596
+# this is copied from https://stackoverflow.com/a/27931596
 [ req ]
 default_bits        = 2048
 default_keyfile     = server-key.pem
@@ -24,7 +24,7 @@ organizationName_default    = Example, LLC
 
 # Use a friendly name here because its presented to the user. The server's DNS
 #   names are placed in Subject Alternate Names. Plus, DNS names here is deprecated
-#   by both IETF and CA/Browser Forums. If you place a DNS name here, then you 
+#   by both IETF and CA/Browser Forums. If you place a DNS name here, then you
 #   must include the DNS name in the SAN too (otherwise, Chrome and others that
 #   strictly follow the CA/Browser Baseline Requirements will fail).
 commonName          = Common Name (e.g. server FQDN or YOUR name)

From 23c7437e4629199e8ee1ae6bcdf75b7062466010 Mon Sep 17 00:00:00 2001
From: Aditya <code.aditya@gmail.com>
Date: Sat, 28 Oct 2017 16:34:49 +0530
Subject: [PATCH 1598/2974] Fix link for 'XPath and XSLT with lxml'

---
 sep/sep-006.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sep/sep-006.rst b/sep/sep-006.rst
index 522bba134..7425c0930 100644
--- a/sep/sep-006.rst
+++ b/sep/sep-006.rst
@@ -58,7 +58,7 @@ As the name of the method for performing selection (the ``x`` method) is not
 descriptive nor mnemotechnic enough and clearly clashes with ``extract`` method
 (x sounds like a short for extract in english), we propose to rename it to
 `select`, `sel` (is shortness if required), or `xpath` after `lxml's
-<http://codespeak.net/lxml/xpathxslt.html>`_ ``xpath`` method.
+<http://lxml.de/xpathxslt.html>`_ ``xpath`` method.
 
 Bonus (ItemBuilder)
 ===================
@@ -72,4 +72,4 @@ References
 ==========
 
  1. XPath Selectors (https://doc.scrapy.org/topics/selectors.html)
- 2. XPath and XSLT with lxml (http://codespeak.net/lxml/xpathxslt.html)
+ 2. XPath and XSLT with lxml (http://lxml.de/xpathxslt.html)

From 97d047a055b3af080047768b196ce677fbfaa12e Mon Sep 17 00:00:00 2001
From: Aditya <code.aditya@gmail.com>
Date: Sat, 28 Oct 2017 16:48:41 +0530
Subject: [PATCH 1599/2974] Fix link for Tox

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 0608693ba..f35b894f3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,4 +1,4 @@
-# Tox (http://tox.testrun.org/) is a tool for running tests
+# Tox (https://tox.readthedocs.io/) is a tool for running tests
 # in multiple virtualenvs. This configuration file will run the
 # test suite on all supported python versions. To use it, "pip install tox"
 # and then run "tox" from this directory.

From dae7b1cdd06649db3c692eccd70195a499733448 Mon Sep 17 00:00:00 2001
From: Aditya <code.aditya@gmail.com>
Date: Sat, 28 Oct 2017 16:53:32 +0530
Subject: [PATCH 1600/2974] Migrate all subdomains on readthedocs.org to
 readthedocs.io

---
 scrapy/templates/project/scrapy.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/templates/project/scrapy.cfg b/scrapy/templates/project/scrapy.cfg
index d7f02e0a2..1daeaa541 100644
--- a/scrapy/templates/project/scrapy.cfg
+++ b/scrapy/templates/project/scrapy.cfg
@@ -1,7 +1,7 @@
 # Automatically created by: scrapy startproject
 #
 # For more information about the [deploy] section see:
-# https://scrapyd.readthedocs.org/en/latest/deploy.html
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
 
 [settings]
 default = ${project_name}.settings

From df7e0a4315f9db2c74fa9e9a0654f44277da2e55 Mon Sep 17 00:00:00 2001
From: Aditya <code.aditya@gmail.com>
Date: Sat, 28 Oct 2017 23:37:44 +0530
Subject: [PATCH 1601/2974] Use https link in default user agent

---
 docs/topics/settings.rst            | 2 +-
 scrapy/settings/default_settings.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 4b15cb607..afa666659 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1324,7 +1324,7 @@ the default value for this setting see: https://boutell.com/newfaq/misc/urllengt
 USER_AGENT
 ----------
 
-Default: ``"Scrapy/VERSION (+http://scrapy.org)"``
+Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
 
 The default User-Agent to use when crawling, unless overridden.
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 697314b7f..ead511473 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -270,7 +270,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))
 
 URLLENGTH_LIMIT = 2083
 
-USER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % import_module('scrapy').__version__
+USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__version__
 
 TELNETCONSOLE_ENABLED = 1
 TELNETCONSOLE_PORT = [6023, 6073]

From 8a7552370de2d31fbd3564e47fa92049c8efee6f Mon Sep 17 00:00:00 2001
From: colinmorris <colin.morris2@gmail.com>
Date: Tue, 31 Oct 2017 17:14:53 -0400
Subject: [PATCH 1602/2974] revise/modernize item exporter example in docs

---
 docs/topics/exporters.rst | 50 +++++++++++++++++++--------------------
 1 file changed, 24 insertions(+), 26 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index b6139af92..28f5ad9c2 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -36,38 +36,36 @@ to export
 3. and finally call the :meth:`~BaseItemExporter.finish_exporting` to signal
 the end of the exporting process
 
-Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses an Item
-Exporter to export scraped items to different files, one per spider::
+Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
+Item Exporters to group scraped items to different files according to the
+value of one of their fields::
 
-   from scrapy import signals
-   from scrapy.exporters import XmlItemExporter
+    from scrapy.exporters import XmlItemExporter
 
-   class XmlExportPipeline(object):
+    class PerYearXmlExportPipeline(object):
+        """Distribute items across multiple XML files according to their 'year' field"""
 
-       def __init__(self):
-           self.files = {}
+        def open_spider(self, spider):
+            self.year_to_exporter = {}
 
-        @classmethod
-        def from_crawler(cls, crawler):
-            pipeline = cls()
-            crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
-            crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
-            return pipeline
+        def close_spider(self, spider):
+            for exporter in self.year_to_exporter.itervalues():
+                exporter.finish_exporting()
+                exporter.file.close()
 
-       def spider_opened(self, spider):
-           file = open('%s_products.xml' % spider.name, 'w+b')
-           self.files[spider] = file
-           self.exporter = XmlItemExporter(file)
-           self.exporter.start_exporting()
+        def _exporter_for_item(self, item):
+            year = item['year']
+            if year not in self.year_to_exporter:
+                f = open('{}.xml'.format(year), 'w+b')
+                exporter = XmlItemExporter(f)
+                exporter.start_exporting()
+                self.year_to_exporter[year] = exporter
+            return self.year_to_exporter[year]
 
-       def spider_closed(self, spider):
-           self.exporter.finish_exporting()
-           file = self.files.pop(spider)
-           file.close()
-
-       def process_item(self, item, spider):
-           self.exporter.export_item(item)
-           return item
+        def process_item(self, item, spider):
+            exporter = self._exporter_for_item(item)
+            exporter.export_item(item)
+            return item
 
 
 .. _topics-exporters-field-serialization:

From 23e571e860729fad1f4351cde69b77d88837e628 Mon Sep 17 00:00:00 2001
From: colinmorris <colin.morris2@gmail.com>
Date: Tue, 31 Oct 2017 18:08:47 -0400
Subject: [PATCH 1603/2974] fix issues identified in review

---
 docs/topics/exporters.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 28f5ad9c2..95f7920f8 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -49,14 +49,14 @@ value of one of their fields::
             self.year_to_exporter = {}
 
         def close_spider(self, spider):
-            for exporter in self.year_to_exporter.itervalues():
+            for exporter in self.year_to_exporter.values():
                 exporter.finish_exporting()
                 exporter.file.close()
 
         def _exporter_for_item(self, item):
             year = item['year']
             if year not in self.year_to_exporter:
-                f = open('{}.xml'.format(year), 'w+b')
+                f = open('{}.xml'.format(year), 'wb')
                 exporter = XmlItemExporter(f)
                 exporter.start_exporting()
                 self.year_to_exporter[year] = exporter

From abb6d0a1c1dd4d1ab6f734f2b366b9d851a52a48 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Wed, 1 Nov 2017 17:26:59 +0300
Subject: [PATCH 1604/2974] Use portable pypy directly

They are provided by https://github.com/squeaky-pl/portable-pypy
---
 .travis.yml | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 9c51fafb2..d4f30814d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,16 +24,10 @@ matrix:
 install:
   - |
       if [ "$TOXENV" = "pypy" ]; then
-        export PYENV_ROOT="$HOME/.pyenv"
-        if [ -f "$PYENV_ROOT/bin/pyenv" ]; then
-          pushd "$PYENV_ROOT" && git pull && popd
-        else
-          rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT"
-        fi
-        # get latest portable PyPy from pyenv directly (thanks to natural version sort option -V)
-        export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy2.7-portable-[0-9][\.0-9]*$' |sort -V |tail -1`
-        "$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION"
-        virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION"
+        export PYPY_VERSION="pypy-5.9-linux_x86_64-portable"
+        wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
+        tar -jxf ${PYPY_VERSION}.tar.bz2
+        virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
       fi
   - pip install -U tox twine wheel codecov

From 846fd83512bb45335195b060d76b46060b4b6e3d Mon Sep 17 00:00:00 2001
From: IAlwaysBeCoding <erik.dominguez1003@gmail.com>
Date: Sat, 11 Nov 2017 18:30:01 -0500
Subject: [PATCH 1605/2974] removed commented out code, wrapped line to pep-8
 and removed backlashes

---
 docs/topics/commands.rst    |  3 +++
 scrapy/commands/parse.py    | 23 ++++++++++++++++++++++
 tests/test_command_parse.py | 39 +++++++++++++++++++++++++++++++++++++
 3 files changed, 65 insertions(+)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index dc8067d7e..07c69ddda 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -430,6 +430,9 @@ Supported options:
 * ``--callback`` or ``-c``: spider method to use as callback for parsing the
   response
 
+* ``--meta`` or ``-m``: additional request meta that will be pass to the callback 
+  request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
+
 * ``--pipelines``: process items through pipelines
 
 * ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider`
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index a90095146..69418a478 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,4 +1,5 @@
 from __future__ import print_function
+import json
 import logging
 
 from w3lib.url import is_url
@@ -48,6 +49,8 @@ class Command(ScrapyCommand):
             help="use CrawlSpider rules to discover the callback")
         parser.add_option("-c", "--callback", dest="callback",
             help="use this callback for parsing, instead looking for a callback")
+        parser.add_option("-m", "--meta", dest="meta",
+            help="inject extra meta into the Request, it must be a valid raw json string")
         parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
             help="maximum depth for parsing requests [default: %default]")
         parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
@@ -204,6 +207,10 @@ class Command(ScrapyCommand):
                     req.callback = callback
                 return requests
 
+        #update request meta if any extra meta was passed through the --meta/-m opts.
+        if opts.meta:
+            request.meta.update(opts.meta)
+
         request.meta['_depth'] = 1
         request.meta['_callback'] = request.callback
         request.callback = callback
@@ -211,11 +218,27 @@ class Command(ScrapyCommand):
 
     def process_options(self, args, opts):
         ScrapyCommand.process_options(self, args, opts)
+
+        self.process_spider_arguments(opts)
+        self.process_request_meta(opts)
+
+    def process_spider_arguments(self, opts):
+
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
 
+    def process_request_meta(self, opts):
+
+        if opts.meta:
+            try:
+                opts.meta = json.loads(opts.meta)
+            except ValueError:
+                raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \
+                                "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
+
+
     def run(self, args, opts):
         # parse arguments
         if not len(args) == 1 or not is_url(args[0]):
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index b6d6db9ee..66dd17110 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -29,6 +29,21 @@ class MySpider(scrapy.Spider):
             self.logger.debug('It Works!')
         return [scrapy.Item(), dict(foo='bar')]
 
+    def parse_request_with_meta(self, response):
+        foo = response.meta.get('foo', 'bar')
+
+        if foo == 'bar':
+            self.logger.debug('It Does Not Work :(')
+        else:
+            self.logger.debug('It Works!')
+
+    def parse_request_without_meta(self, response):
+        foo = response.meta.get('foo', 'bar')
+
+        if foo == 'bar':
+            self.logger.debug('It Works!')
+        else:
+            self.logger.debug('It Does Not Work :(')
 
 class MyGoodCrawlSpider(CrawlSpider):
     name = 'goodcrawl{0}'
@@ -84,6 +99,30 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", to_native_str(stderr))
 
+    @defer.inlineCallbacks
+    def test_request_with_meta(self):
+        raw_json_string = '{"foo" : "baz"}'
+        _, _, stderr = yield self.execute(['--spider', self.spider_name,
+                                           '--meta', raw_json_string,
+                                           '-c', 'parse_request_with_meta',
+                                           self.url('/html')])
+        self.assertIn("DEBUG: It Works!", to_native_str(stderr))
+
+        _, _, stderr = yield self.execute(['--spider', self.spider_name,
+                                           '-m', raw_json_string,
+                                           '-c', 'parse_request_with_meta',
+                                           self.url('/html')])
+        self.assertIn("DEBUG: It Works!", to_native_str(stderr))
+
+
+    @defer.inlineCallbacks
+    def test_request_without_meta(self):
+        _, _, stderr = yield self.execute(['--spider', self.spider_name,
+                                          '-c', 'parse_request_without_meta',
+                                           self.url('/html')])
+        self.assertIn("DEBUG: It Works!", to_native_str(stderr))
+
+
     @defer.inlineCallbacks
     def test_pipelines(self):
         _, _, stderr = yield self.execute(['--spider', self.spider_name,

From 62a626102877de4998538717f34e61d2f7d2622c Mon Sep 17 00:00:00 2001
From: Jana Cavojska <cavojska_jana@yahoo.com>
Date: Sat, 18 Nov 2017 20:03:59 +0100
Subject: [PATCH 1606/2974] Issues a warning when user puts a URL into
 allowed_domains (#2250)

---
 scrapy/spidermiddlewares/offsite.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index ea1c9270f..f51b0a2b0 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -52,6 +52,10 @@ class OffsiteMiddleware(object):
         allowed_domains = getattr(spider, 'allowed_domains', None)
         if not allowed_domains:
             return re.compile('') # allow all by default
+        for domainIndex in range(0, len(allowed_domains)):
+            url_pattern = re.compile("^https?://.*$")
+            if url_pattern.match(allowed_domains[domainIndex]):
+                logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % allowed_domains[domainIndex])
         regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None)
         return re.compile(regex)
 

From 5441cc18e43cba6b8196ece49d6b454badb246f0 Mon Sep 17 00:00:00 2001
From: KosayJabre <Kosay.jabre@gmail.com>
Date: Sun, 19 Nov 2017 18:09:38 -0400
Subject: [PATCH 1607/2974] Separated import statements

Just separated the import statements. Tiny change - testing GitHub!
---
 scrapy/commands/edit.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py
index a7f8983b4..25d843a53 100644
--- a/scrapy/commands/edit.py
+++ b/scrapy/commands/edit.py
@@ -1,4 +1,5 @@
-import sys, os
+import sys
+import os
 
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError

From 91ff194d1e9477d2196817ea1dc8beb220c3e058 Mon Sep 17 00:00:00 2001
From: Jana Cavojska <cavojska_jana@yahoo.com>
Date: Mon, 20 Nov 2017 21:23:31 +0100
Subject: [PATCH 1608/2974] looping over allowed_domains directly instead of
 via index

---
 scrapy/spidermiddlewares/offsite.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index f51b0a2b0..8ff35e29f 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -52,10 +52,10 @@ class OffsiteMiddleware(object):
         allowed_domains = getattr(spider, 'allowed_domains', None)
         if not allowed_domains:
             return re.compile('') # allow all by default
-        for domainIndex in range(0, len(allowed_domains)):
+        for domain in allowed_domains:
             url_pattern = re.compile("^https?://.*$")
-            if url_pattern.match(allowed_domains[domainIndex]):
-                logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % allowed_domains[domainIndex])
+            if url_pattern.match(domain):
+                logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain)
         regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None)
         return re.compile(regex)
 

From 0b14cb44aabcc8322e09a46c4e14913c94886e12 Mon Sep 17 00:00:00 2001
From: Jesse Bakker <j.bakker@scisports.com>
Date: Thu, 23 Nov 2017 15:25:43 +0100
Subject: [PATCH 1609/2974] Added from_crawler to middleware docs

---
 docs/topics/downloader-middleware.rst | 11 +++++++++++
 docs/topics/spider-middleware.rst     | 11 +++++++++++
 2 files changed, 22 insertions(+)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 0d168017f..983a93290 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -157,6 +157,17 @@ more of the following methods:
       :param spider: the spider for which this request is intended
       :type spider: :class:`~scrapy.spiders.Spider` object
 
+   .. method:: from_crawler(cls, crawler)
+    
+      If present, this classmethod is called to create a middleware instance
+      from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
+      of the middleware. Crawler object provides access to all Scrapy core
+      components like settings and signals; it is a way for middleware to
+      access them and hook its functionality into Scrapy.
+   
+      :param crawler: crawler that uses this middleware
+      :type crawler: :class:`~scrapy.crawler.Crawler` object
+
 .. _topics-downloader-middleware-ref:
 
 Built-in downloader middleware reference
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index a2d2556c5..c297ed556 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -164,6 +164,17 @@ following methods:
         :param spider: the spider to whom the start requests belong
         :type spider: :class:`~scrapy.spiders.Spider` object
 
+    .. method:: from_crawler(cls, crawler)
+    
+       If present, this classmethod is called to create a middleware instance
+       from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
+       of the middleware. Crawler object provides access to all Scrapy core
+       components like settings and signals; it is a way for middleware to
+       access them and hook its functionality into Scrapy.
+    
+       :param crawler: crawler that uses this middleware
+       :type crawler: :class:`~scrapy.crawler.Crawler` object
+
 
 .. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception
 

From 6af323d7c85eeee40d90a20133504df26a593304 Mon Sep 17 00:00:00 2001
From: IAlwaysBeCoding <erik.dominguez1003@gmail.com>
Date: Sun, 26 Nov 2017 00:24:52 +0100
Subject: [PATCH 1610/2974] Fix spelling mistake on scrapy parse command docs

Fixed spelling mistake from "will be pass" to "will be passed"
---
 docs/topics/commands.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 07c69ddda..06f9a485b 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -430,7 +430,7 @@ Supported options:
 * ``--callback`` or ``-c``: spider method to use as callback for parsing the
   response
 
-* ``--meta`` or ``-m``: additional request meta that will be pass to the callback 
+* ``--meta`` or ``-m``: additional request meta that will be passed to the callback 
   request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
 
 * ``--pipelines``: process items through pipelines

From 8ec3b476b03d6b8424f6dfc556758392e7a5a61f Mon Sep 17 00:00:00 2001
From: Jana Cavojska <jana.cavojska9@gmail.com>
Date: Sun, 26 Nov 2017 16:36:15 +0100
Subject: [PATCH 1611/2974] triggering a warning when user puts URL in
 allowed_domains now covered by test

---
 scrapy/spidermiddlewares/offsite.py    |  3 ++-
 tests/test_spidermiddleware_offsite.py | 11 +++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 8ff35e29f..647792e5d 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/spider-middleware.rst
 
 import re
 import logging
+import warnings
 
 from scrapy import signals
 from scrapy.http import Request
@@ -55,7 +56,7 @@ class OffsiteMiddleware(object):
         for domain in allowed_domains:
             url_pattern = re.compile("^https?://.*$")
             if url_pattern.match(domain):
-                logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain)
+                warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, Warning)
         regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None)
         return re.compile(regex)
 
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 9ad86313c..b532cc2ec 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -6,6 +6,7 @@ from scrapy.http import Response, Request
 from scrapy.spiders import Spider
 from scrapy.spidermiddlewares.offsite import OffsiteMiddleware
 from scrapy.utils.test import get_crawler
+import warnings
 
 class TestOffsiteMiddleware(TestCase):
 
@@ -68,3 +69,13 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
       reqs = [Request('http://scrapytest.org/1')]
       out = list(self.mw.process_spider_output(res, reqs, self.spider))
       self.assertEqual(out, reqs)
+
+
+class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
+    
+    def test_get_host_regex(self):
+        self.spider.allowed_domains = ['http://scrapytest.org', 'scrapy.org', 'scrapy.test.org']
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+            self.mw.get_host_regex(self.spider)
+            assert "allowed_domains accepts only domains, not URLs." in str(w[-1].message)

From 454d5e57333e9f33c8d684e4e21f8f7e9493f310 Mon Sep 17 00:00:00 2001
From: Jana Cavojska <jana.cavojska9@gmail.com>
Date: Sun, 26 Nov 2017 20:07:04 +0100
Subject: [PATCH 1612/2974] checking for subclass of URLWarning instead of
 checking error message text when URL in allowed_domains

---
 scrapy/spidermiddlewares/offsite.py    | 7 ++++++-
 tests/test_spidermiddleware_offsite.py | 3 ++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 647792e5d..f595eef42 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -56,10 +56,15 @@ class OffsiteMiddleware(object):
         for domain in allowed_domains:
             url_pattern = re.compile("^https?://.*$")
             if url_pattern.match(domain):
-                warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, Warning)
+                warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, URLWarning)
+                
         regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None)
         return re.compile(regex)
 
     def spider_opened(self, spider):
         self.host_regex = self.get_host_regex(spider)
         self.domains_seen = set()
+
+
+class URLWarning(Warning):
+    pass
\ No newline at end of file
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index b532cc2ec..7e4af0d4c 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -5,6 +5,7 @@ from six.moves.urllib.parse import urlparse
 from scrapy.http import Response, Request
 from scrapy.spiders import Spider
 from scrapy.spidermiddlewares.offsite import OffsiteMiddleware
+from scrapy.spidermiddlewares.offsite import URLWarning
 from scrapy.utils.test import get_crawler
 import warnings
 
@@ -78,4 +79,4 @@ class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
         with warnings.catch_warnings(record=True) as w:
             warnings.simplefilter("always")
             self.mw.get_host_regex(self.spider)
-            assert "allowed_domains accepts only domains, not URLs." in str(w[-1].message)
+            assert issubclass(w[-1].category, URLWarning)

From 22c68baf990f15d249f38c481f24a984977be3e5 Mon Sep 17 00:00:00 2001
From: Jana Cavojska <cavojska_jana@yahoo.com>
Date: Thu, 7 Dec 2017 18:38:29 +0100
Subject: [PATCH 1613/2974] url_pattern is now being compiled before entering
 the loop

---
 scrapy/spidermiddlewares/offsite.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index f595eef42..310166cad 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -53,8 +53,8 @@ class OffsiteMiddleware(object):
         allowed_domains = getattr(spider, 'allowed_domains', None)
         if not allowed_domains:
             return re.compile('') # allow all by default
+        url_pattern = re.compile("^https?://.*$")
         for domain in allowed_domains:
-            url_pattern = re.compile("^https?://.*$")
             if url_pattern.match(domain):
                 warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, URLWarning)
                 
@@ -67,4 +67,4 @@ class OffsiteMiddleware(object):
 
 
 class URLWarning(Warning):
-    pass
\ No newline at end of file
+    pass

From f716843a66829350063e55f8df768eb538c6b05c Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 12 Dec 2017 16:19:43 +0500
Subject: [PATCH 1614/2974] DOC update "Contributing" docs:

* suggest Stack Overflow for Scrapy usage questions;
* encourage users to submit test-only pull requests with reproducable examples;
* encourage users to pick up stalled pull requests;
* we don't use AUTHORS file as a main acknowledgement source;
* suggest using Sphinx autodocs extension
---
 docs/contributing.rst | 58 +++++++++++++++++++++++++++++++++++--------
 1 file changed, 48 insertions(+), 10 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index f3732ab06..eb736bf30 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -19,12 +19,16 @@ There are many ways to contribute to Scrapy. Here are some of them:
   the guidelines detailed in `Reporting bugs`_ below.
 
 * Submit patches for new functionality and/or bug fixes. Please read
-  `Writing patches`_ and `Submitting patches`_ below for details on how to
+  :ref:`writing-patches` and `Submitting patches`_ below for details on how to
   write and submit a patch.
 
 * Join the `Scrapy subreddit`_ and share your ideas on how to
   improve Scrapy. We're always open to suggestions.
 
+* Answer Scrapy questions at
+  `Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__.
+
+
 Reporting bugs
 ==============
 
@@ -40,9 +44,14 @@ guidelines when reporting a new bug.
 * check the :ref:`FAQ <faq>` first to see if your issue is addressed in a
   well-known question
 
+* if you have a general question about scrapy usage, please ask it at
+  `Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__
+  (use "scrapy" tag).
+
 * check the `open issues`_ to see if it has already been reported. If it has,
-  don't dismiss the report but check the ticket history and comments, you may
-  find additional useful information to contribute.
+  don't dismiss the report, but check the ticket history and comments. If you
+  have additional useful information, please leave a comment, or consider
+  :ref:`sending a pull request <writing-patches>` with a fix.
 
 * search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has
   been discussed there, or if you're not sure if what you're seeing is a bug.
@@ -54,12 +63,20 @@ guidelines when reporting a new bug.
   it. See for example StackOverflow's guide on creating a
   `Minimal, Complete, and Verifiable example`_ exhibiting the issue.
 
+* the most awesome way to provide a complete reproducible example is to
+  send a pull request which adds a failing test case to the
+  Scrapy testing suite (see :ref:`submitting-patches`).
+  This is helpful even if you don't have an intention to
+  fix the issue yourselves.
+
 * include the output of ``scrapy version -v`` so developers working on your bug
   know exactly which version and platform it occurred on, which is often very
   helpful for reproducing it, or knowing if it was already fixed.
 
 .. _Minimal, Complete, and Verifiable example: https://stackoverflow.com/help/mcve
 
+.. _writing-patches:
+
 Writing patches
 ===============
 
@@ -83,6 +100,8 @@ Well-written patches should:
   the documentation changes in the same patch.  See `Documentation policies`_
   below.
 
+.. _submitting-patches:
+
 Submitting patches
 ==================
 
@@ -100,11 +119,22 @@ starting point is to send a pull request on GitHub. It can be simple enough to
 illustrate your idea, and leave documentation/tests for later, after the idea
 has been validated and proven useful. Alternatively, you can start a
 conversation in the `Scrapy subreddit`_ to discuss your idea first.
+
+Sometimes there is an existing pull request for the problem you'd like to
+solve, which is stalled for some reason. Often the pull request is in a
+right direction, but changes are requested by Scrapy maintainers, and the
+original pull request author haven't had time to address them.
+In this case consider picking up this pull request: open
+a new pull request with all commits from the original pull request, as well as
+additional changes to address the raised issues. Doing so helps a lot; it is
+not considered rude as soon as original the author is acknowledged by keeping
+his/her commits.
+
 When writing GitHub pull requests, try to keep titles short but descriptive.
 E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests"
 prefer "Fix hanging when exception occurs in start_requests (#411)"
-instead of "Fix for #411".
-Complete titles make it easy to skim through the issue tracker.
+instead of "Fix for #411". Complete titles make it easy to skim through
+the issue tracker.
 
 Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports
 removal, etc) in separate commits than functional changes. This will make pull
@@ -121,21 +151,29 @@ Scrapy:
 * It's OK to use lines longer than 80 chars if it improves the code
   readability.
 
-* Don't put your name in the code you contribute. Our policy is to keep
-  the contributor's name in the `AUTHORS`_ file distributed with Scrapy.
+* Don't put your name in the code you contribute; git provides enough
+  metadata to identify author of the code.
+  See https://help.github.com/articles/setting-your-username-in-git/ for
+  setup instructions.
 
 Documentation policies
 ======================
 
 * **Don't** use docstrings for documenting classes, or methods which are
-  already documented in the official (sphinx) documentation. For example, the
-  :meth:`ItemLoader.add_value` method should be documented in the sphinx
-  documentation, not its docstring.
+  already documented in the official (sphinx) documentation. Alternatively,
+  **do** provide a docstring, but make sure sphinx documentation uses
+  autodoc_ extension to pull the docstring. For example, the
+  :meth:`ItemLoader.add_value` method should be either
+  documented only in the sphinx documentation (not it a docstring), or
+  it should have a docstring which is pulled to sphinx documentation using
+  autodoc_ extension.
 
 * **Do** use docstrings for documenting functions not present in the official
   (sphinx) documentation, such as functions from ``scrapy.utils`` package and
   its sub-modules.
 
+.. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html
+
 Tests
 =====
 

From 9aa9dd8d45a2ce0c8e6ae0732e610f020735df7e Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 12 Dec 2017 19:17:00 +0500
Subject: [PATCH 1615/2974] DOC mention an easier way to track pull requests
 locally. Thanks @eliasdorneles!

---
 docs/contributing.rst | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index eb736bf30..9a02634cb 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -127,9 +127,16 @@ original pull request author haven't had time to address them.
 In this case consider picking up this pull request: open
 a new pull request with all commits from the original pull request, as well as
 additional changes to address the raised issues. Doing so helps a lot; it is
-not considered rude as soon as original the author is acknowledged by keeping
+not considered rude as soon as the original author is acknowledged by keeping
 his/her commits.
 
+You can pull an existing pull request to a local branch
+by running ``git fetch upstream pull/$PR_NUMBER/head:$BRANCH_NAME_TO_CREATE``
+(replace 'upstream' with a remote name for scrapy repository,
+``$PR_NUMBER`` with an ID of the pull request, and ``$BRANCH_NAME_TO_CREATE``
+with a name of the branch you want to create locally).
+See also: https://help.github.com/articles/checking-out-pull-requests-locally/#modifying-an-inactive-pull-request-locally.
+
 When writing GitHub pull requests, try to keep titles short but descriptive.
 E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests"
 prefer "Fix hanging when exception occurs in start_requests (#411)"

From 44623687ab8936c5696f68f74e438a2891880c82 Mon Sep 17 00:00:00 2001
From: Hugo <hugovk@users.noreply.github.com>
Date: Tue, 19 Dec 2017 17:59:05 +0200
Subject: [PATCH 1616/2974] Drop support for EOL Python 3.3

---
 .travis.yml            |  2 --
 README.rst             |  6 +++++-
 docs/faq.rst           |  2 +-
 docs/intro/install.rst | 10 +++++-----
 setup.py               |  2 +-
 tox.ini                | 12 ++++--------
 6 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index d4f30814d..66de9ed03 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,8 +13,6 @@ matrix:
       env: TOXENV=jessie
     - python: 2.7
       env: TOXENV=pypy
-    - python: 3.3
-      env: TOXENV=py33
     - python: 3.5
       env: TOXENV=py35
     - python: 3.6
diff --git a/README.rst b/README.rst
index 45135c7a2..1361eac26 100644
--- a/README.rst
+++ b/README.rst
@@ -6,6 +6,10 @@ Scrapy
    :target: https://pypi.python.org/pypi/Scrapy
    :alt: PyPI Version
 
+.. image:: https://img.shields.io/pypi/pyversions/Scrapy.svg
+   :target: https://pypi.python.org/pypi/Scrapy
+   :alt: Supported Python Versions
+
 .. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
    :target: https://travis-ci.org/scrapy/scrapy
    :alt: Build Status
@@ -36,7 +40,7 @@ https://scrapy.org
 Requirements
 ============
 
-* Python 2.7 or Python 3.3+
+* Python 2.7 or Python 3.4+
 * Works on Linux, Windows, Mac OSX, BSD
 
 Install
diff --git a/docs/faq.rst b/docs/faq.rst
index 42c3abbfa..484226979 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
 What Python versions does Scrapy support?
 -----------------------------------------
 
-Scrapy is supported under Python 2.7 and Python 3.3+.
+Scrapy is supported under Python 2.7 and Python 3.4+.
 Python 2.6 support was dropped starting at Scrapy 0.20.
 Python 3 support was added in Scrapy 1.1.
 
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 12d489612..a2e3f506e 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -7,7 +7,7 @@ Installation guide
 Installing Scrapy
 =================
 
-Scrapy runs on Python 2.7 and Python 3.3 or above.
+Scrapy runs on Python 2.7 and Python 3.4 or above.
 
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
 the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
@@ -132,12 +132,12 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with::
 
 .. _intro-install-ubuntu:
 
-Ubuntu 12.04 or above
+Ubuntu 14.04 or above
 ---------------------
 
 Scrapy is currently tested with recent-enough versions of lxml,
 twisted and pyOpenSSL, and is compatible with recent Ubuntu distributions.
-But it should support older versions of Ubuntu too, like Ubuntu 12.04,
+But it should support older versions of Ubuntu too, like Ubuntu 14.04,
 albeit with potential issues with TLS connections.
 
 **Don't** use the ``python-scrapy`` package provided by Ubuntu, they are
@@ -163,8 +163,8 @@ you can install Scrapy with ``pip`` after that::
     pip install scrapy
 
 .. note::
-    The same non-python dependencies can be used to install Scrapy in Debian
-    Wheezy (7.0) and above.
+    The same non-Python dependencies can be used to install Scrapy in Debian
+    Jessue (8.0) and above.
 
 
 .. _intro-install-macos:
diff --git a/setup.py b/setup.py
index 327286f5a..2619bd544 100644
--- a/setup.py
+++ b/setup.py
@@ -53,7 +53,6 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
@@ -61,6 +60,7 @@ setup(
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     install_requires=[
         'Twisted>=13.1.0',
         'w3lib>=1.17.0',
diff --git a/tox.ini b/tox.ini
index f35b894f3..5c543475c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -63,25 +63,21 @@ basepython = pypy
 commands =
     py.test {posargs:scrapy tests}
 
-[testenv:py33]
-basepython = python3.3
+[testenv:py34]
+basepython = python3.4
 deps =
     -rrequirements-py3.txt
     # Extras
     Pillow
     -rtests/requirements-py3.txt
 
-[testenv:py34]
-basepython = python3.4
-deps = {[testenv:py33]deps}
-
 [testenv:py35]
 basepython = python3.5
-deps = {[testenv:py33]deps}
+deps = {[testenv:py34]deps}
 
 [testenv:py36]
 basepython = python3.6
-deps = {[testenv:py33]deps}
+deps = {[testenv:py34]deps}
 
 [docs]
 changedir = docs

From f11c21c6fc62b64a2bbee0e19e2098ed6257cf19 Mon Sep 17 00:00:00 2001
From: Hugo <hugovk@users.noreply.github.com>
Date: Wed, 20 Dec 2017 17:05:56 +0200
Subject: [PATCH 1617/2974] Test on Python 3.4

---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 66de9ed03..e2e9e0cc1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,6 +13,8 @@ matrix:
       env: TOXENV=jessie
     - python: 2.7
       env: TOXENV=pypy
+    - python: 3.4
+      env: TOXENV=py34
     - python: 3.5
       env: TOXENV=py35
     - python: 3.6

From cbcf80b98ff66db1ccf625fa52c4de8935331972 Mon Sep 17 00:00:00 2001
From: Hugo <hugovk@users.noreply.github.com>
Date: Wed, 20 Dec 2017 17:34:13 +0200
Subject: [PATCH 1618/2974] Fix typo

[CI skip]
---
 docs/intro/install.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index a2e3f506e..22bc84a40 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -164,7 +164,7 @@ you can install Scrapy with ``pip`` after that::
 
 .. note::
     The same non-Python dependencies can be used to install Scrapy in Debian
-    Jessue (8.0) and above.
+    Jessie (8.0) and above.
 
 
 .. _intro-install-macos:

From ea41114cf0ab2782650792ad204cf43fc148c749 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 25 Dec 2017 12:29:02 +0300
Subject: [PATCH 1619/2974] Mention PyPy support, add PyPy to install docs

---
 docs/faq.rst           |  4 +++-
 docs/intro/install.rst | 25 ++++++++++++++++++++++++-
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 484226979..7eecc999f 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -69,9 +69,11 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
 What Python versions does Scrapy support?
 -----------------------------------------
 
-Scrapy is supported under Python 2.7 and Python 3.4+.
+Scrapy is supported under Python 2.7 and Python 3.4+
+under CPython (default Python implementation) and PyPy (only for Python 2.7).
 Python 2.6 support was dropped starting at Scrapy 0.20.
 Python 3 support was added in Scrapy 1.1.
+PyPy support was added in Scrapy 1.4, PyPy version tested is PyPy2-v5.9.0.
 
 .. note::
     For Python 3 support on Windows, it is recommended to use
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 22bc84a40..b00dc2cd6 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -7,7 +7,8 @@ Installation guide
 Installing Scrapy
 =================
 
-Scrapy runs on Python 2.7 and Python 3.4 or above.
+Scrapy runs on Python 2.7 and Python 3.4 or above
+under CPython (default Python implementation) and PyPy (only for Python 2.7).
 
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
 the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
@@ -223,6 +224,28 @@ After any of these workarounds you should be able to install Scrapy::
   pip install Scrapy
 
 
+PyPy
+----
+
+We recommend using the latest PyPy version. The version tested is PyPy2-v5.9.0.
+
+Most scrapy dependencides now have binary wheels for CPython, but not for PyPy.
+This means that these dependecies will be built during installation.
+On OS X, you are likely to face an issue with building Cryptography dependency,
+solution to this problem is described
+`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
+that is to ``brew install openssl`` and then export the flags that this command
+recommends (only needed when installing scrapy). Installing on Linux has no special
+issues besides installing build dependencies.
+Installing scrapy with PyPy on Windows is not tested.
+
+You can check that scrapy is installed correctly by running ``scrapy bench``.
+If this command gives errors such as
+``TypeError: ... got 2 unexpected keyword arguments``, this means
+that setuptools was unable to pick up one PyPy-specific dependency.
+To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``.
+
+
 .. _Python: https://www.python.org/
 .. _pip: https://pip.pypa.io/en/latest/installing/
 .. _lxml: http://lxml.de/

From 1058169f0e3a8646dbd20f9b4c0b599ed9f6d08e Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 25 Dec 2017 15:31:07 +0500
Subject: [PATCH 1620/2974] setup.py: mention that we support PyPy. See
 GH-2213.

---
 setup.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/setup.py b/setup.py
index 2619bd544..06a36e2ba 100644
--- a/setup.py
+++ b/setup.py
@@ -56,6 +56,8 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: Implementation :: CPython',
+        'Programming Language :: Python :: Implementation :: PyPy',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',

From f71df6f9addca10b562bb22890b5ea1c37efde5c Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 25 Dec 2017 13:46:22 +0300
Subject: [PATCH 1621/2974] Run tests for PyPy3

---
 .travis.yml | 9 +++++++++
 tox.ini     | 6 ++++++
 2 files changed, 15 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index e2e9e0cc1..6635f5d3b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,6 +13,8 @@ matrix:
       env: TOXENV=jessie
     - python: 2.7
       env: TOXENV=pypy
+    - python: 2.7
+      env: TOXENV=pypy3
     - python: 3.4
       env: TOXENV=py34
     - python: 3.5
@@ -30,6 +32,13 @@ install:
         virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
       fi
+      if [ "$TOXENV" = "pypy3" ]; then
+        export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
+        wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
+        tar -jxf ${PYPY_VERSION}.tar.bz2
+        virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
+        source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
+      fi
   - pip install -U tox twine wheel codecov
 
 script: tox
diff --git a/tox.ini b/tox.ini
index 5c543475c..60ff8c15e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -79,6 +79,12 @@ deps = {[testenv:py34]deps}
 basepython = python3.6
 deps = {[testenv:py34]deps}
 
+[testenv:pypy3]
+basepython = pypy3
+deps = {[testenv:py34]deps}
+commands =
+    py.test {posargs:scrapy tests}
+
 [docs]
 changedir = docs
 deps =

From 041308afe7c40de7088f75b0e0c312ecd5de428a Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 25 Dec 2017 14:27:20 +0300
Subject: [PATCH 1622/2974] Fix get_func_args test for pypy3

These built-in functions are exposed as methods in PyPy3.
For scrapy this does not matter as:
1) they do not work for CPython at all
2) get_func_args is checked for presense of an argument in scrapy,
   extra "self" does not matter.
But it still makes sense to leave these tests so that we know we
shouldn't use get_func_args for built-in functions/methods.
---
 tests/test_utils_python.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 115f523e9..f6133657b 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -219,9 +219,12 @@ class UtilsPythonTestCase(unittest.TestCase):
             self.assertEqual(get_func_args(" ".join), [])
             self.assertEqual(get_func_args(operator.itemgetter(2)), [])
         else:
-            self.assertEqual(get_func_args(six.text_type.split), ['sep', 'maxsplit'])
-            self.assertEqual(get_func_args(" ".join), ['list'])
-            self.assertEqual(get_func_args(operator.itemgetter(2)), ['obj'])
+            stripself = not six.PY2  # PyPy3 exposes them as methods
+            self.assertEqual(
+                get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit'])
+            self.assertEqual(get_func_args(" ".join, stripself), ['list'])
+            self.assertEqual(
+                get_func_args(operator.itemgetter(2), stripself), ['obj'])
 
 
     def test_without_none_values(self):

From bb1f31189128cb2272c1302350387075fbbb730a Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 25 Dec 2017 15:46:05 +0300
Subject: [PATCH 1623/2974] Add PyPy3 support to faq and install doc

---
 docs/faq.rst           | 4 ++--
 docs/intro/install.rst | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 7eecc999f..7a0628f88 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -70,10 +70,10 @@ What Python versions does Scrapy support?
 -----------------------------------------
 
 Scrapy is supported under Python 2.7 and Python 3.4+
-under CPython (default Python implementation) and PyPy (only for Python 2.7).
+under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
 Python 2.6 support was dropped starting at Scrapy 0.20.
 Python 3 support was added in Scrapy 1.1.
-PyPy support was added in Scrapy 1.4, PyPy version tested is PyPy2-v5.9.0.
+PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
 
 .. note::
     For Python 3 support on Windows, it is recommended to use
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index b00dc2cd6..4a9aa3cfb 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -8,7 +8,7 @@ Installing Scrapy
 =================
 
 Scrapy runs on Python 2.7 and Python 3.4 or above
-under CPython (default Python implementation) and PyPy (only for Python 2.7).
+under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
 
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
 the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
@@ -227,7 +227,8 @@ After any of these workarounds you should be able to install Scrapy::
 PyPy
 ----
 
-We recommend using the latest PyPy version. The version tested is PyPy2-v5.9.0.
+We recommend using the latest PyPy version. The version tested is 5.9.0.
+For PyPy3, only Linux installation was tested.
 
 Most scrapy dependencides now have binary wheels for CPython, but not for PyPy.
 This means that these dependecies will be built during installation.

From a1cc5a63d3e253c325159fdc6ebf4cd3faa37c49 Mon Sep 17 00:00:00 2001
From: Raphael <raphapassini@gmail.com>
Date: Wed, 27 Dec 2017 18:54:17 -0200
Subject: [PATCH 1624/2974] Add mention to dont_merge_cookies in
 CookiesMiddlewares docs (#2999) (#3030)

Add mention to dont_merge_cookies in CookiesMiddlewares docs (#2999)
---
 docs/topics/downloader-middleware.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 983a93290..863620900 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -237,6 +237,17 @@ Default: ``True``
 Whether to enable the cookies middleware. If disabled, no cookies will be sent
 to web servers.
 
+Notice that if the :class:`~scrapy.http.Request` 
+has ``meta['dont_merge_cookies']`` evaluated to ``True``. 
+despite the value of :setting:`COOKIES_ENABLED` the cookies will **not** be 
+sent to web servers and received cookies in 
+:class:`~scrapy.http.Response` will **not** be merged with the existing 
+cookies.
+
+For more detailed information see the ``cookies`` parameter in 
+:class:`~scrapy.http.Request`
+
+
 .. setting:: COOKIES_DEBUG
 
 COOKIES_DEBUG

From 461f9daff5747728e26cd60e9dfe531092f58132 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 27 Jul 2017 19:56:17 +0200
Subject: [PATCH 1625/2974] Update release notes for upcoming 1.4.1 version

---
 docs/news.rst | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 577c93b8e..7ecf22470 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,53 @@
 Release notes
 =============
 
+Scrapy 1.4.1 (2017-XX-XX)
+-------------------------
+
+New features
+~~~~~~~~~~~~
+
+- Support ``<link>`` tags in ``Response.follow`` (:issue:`2785`)
+- Support for ``ptpython`` REPL (:issue:`2654`)
+- Populate spider variable when using ``shell.inspect_response`` (:issue:`2812`)
+- Handle HTTP 308 Permanent Redirect (:issue:`2844`)
+- Add 522 and 524 to ``RETRY_HTTP_CODES`` (:issue:`2851`)
+- Log versions information at startup (:issue:`2857`)
+- Add template for a downloader middleware (:issue:`2755`)
+- Explicit message for NotImplementedError when parse callback not defined (:issue:`2831`)
+
+Bug fixes
+~~~~~~~~~
+
+- Fix PyPy test failures (:issue:`2793`)
+- Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`)
+- Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`)
+- Add verification to check if Request callback is callable (:issue:`2766`)
+- Port ``extras/qpsclient.py`` to Python 3 (:issue:`2849`)
+- Use getfullargspec under the scenes for Python 3 to stop DeprecationWarning (:issue:`2862`)
+- Update deprecated test aliases (:issue:`2876`)
+- Fix ``SitemapSpider`` support for alternate links (:issue:`2853`)
+- Fix logging of settings overridden by ``custom_settings``;
+  **this is technically backwards-incompatible** because the logger
+  changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``, so please
+  update your log parsers if needed (:issue:`1343`)
+
+Docs
+~~~~
+
+- Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting. (:issue:`2756`)
+- Include references to Scrapy subreddit in the docs (:issue:`2762`)
+- Use https:// for readthedocs links
+- Document CloseSpider extension better (:issue:`2759`)
+- Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example (:issue:`2781`)
+- Spelling mistake and typos (:issue:`2828`, :issue:`2837`, :issue:`#2884`)
+- Clarify ``CSVFeedSpider.headers`` documentation (:issue:`2826`)
+- Document ``DontCloseSpider`` exception and clarify ``spider_idle`` (:issue:`2791`)
+- Update "Releases" section in README (:issue:`2764`)
+- Fix rst syntax in ``DOWNLOAD_FAIL_ON_DATALOSS`` docs (:issue:`2763`)
+- Small fix in description of startproject arguments (:issue:`2866`)
+
+
 Scrapy 1.4.0 (2017-05-18)
 -------------------------
 

From 45b0e1a0e4c51a773b39be14334a999cc5f0fe56 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 28 Dec 2017 07:33:43 +0500
Subject: [PATCH 1626/2974] DOC draft 1.5 release notes

---
 docs/news.rst | 78 ++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 62 insertions(+), 16 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 7ecf22470..df0f10a32 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,51 +3,97 @@
 Release notes
 =============
 
-Scrapy 1.4.1 (2017-XX-XX)
+Scrapy 1.5.0 (2017-XX-XX)
 -------------------------
 
+Supported Environments
+----------------------
+
+* Scrapy 1.5 drops support for Python 3.3;
+* this release also improves Python 3.x support (especially 3.6);
+* PyPy and PyPy3 are now supported officially, by running tests on CI.
+* Ubuntu 12.04 and Debian 7.0 are baseline Linux distributions.
+
 New features
 ~~~~~~~~~~~~
 
 - Support ``<link>`` tags in ``Response.follow`` (:issue:`2785`)
 - Support for ``ptpython`` REPL (:issue:`2654`)
+- Google Cloud Storage support for FilesPipeline and ImagesPipeline
+  (:issue:`2923`).
+- New ``--meta`` option of the "scrapy parse" command allows to pass additional
+  request.meta (:issue:`2883`)
 - Populate spider variable when using ``shell.inspect_response`` (:issue:`2812`)
 - Handle HTTP 308 Permanent Redirect (:issue:`2844`)
 - Add 522 and 524 to ``RETRY_HTTP_CODES`` (:issue:`2851`)
 - Log versions information at startup (:issue:`2857`)
 - Add template for a downloader middleware (:issue:`2755`)
-- Explicit message for NotImplementedError when parse callback not defined (:issue:`2831`)
+- Explicit message for NotImplementedError when parse callback not defined
+  (:issue:`2831`)
+- Connections to proxy servers are reused (:issue:`2743`)
+- CrawlerProcess got an option to disable installation of root log handler
+  (:issue:`2921`)
+- LinkExtractor now ignores ``m4v`` extension by default
+- ``scrapy.mail.MailSender`` now works in Python 3 (it requires Twisted 17.9.0)
+- Better log messages for responses over :setting:`DOWNLOAD_WARNSIZE` and
+  :setting:`DOWNLOAD_MAXSIZE` limits (:issue:`2927`)
+- Show warning when a URL is put to ``Spider.allowed_domains`` instead of
+  a domain (:issue:`2250`).
+
 
 Bug fixes
 ~~~~~~~~~
 
-- Fix PyPy test failures (:issue:`2793`)
-- Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`)
-- Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`)
-- Add verification to check if Request callback is callable (:issue:`2766`)
-- Port ``extras/qpsclient.py`` to Python 3 (:issue:`2849`)
-- Use getfullargspec under the scenes for Python 3 to stop DeprecationWarning (:issue:`2862`)
-- Update deprecated test aliases (:issue:`2876`)
-- Fix ``SitemapSpider`` support for alternate links (:issue:`2853`)
 - Fix logging of settings overridden by ``custom_settings``;
   **this is technically backwards-incompatible** because the logger
   changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``, so please
   update your log parsers if needed (:issue:`1343`)
+- Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`).
+  **This is technically backwards-incompatible**; override
+  :setting:`USER_AGENT` if you relied on old value.
+- Fix PyPy and PyPy3 test failures, support them officially
+  (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`, :issue:`3048`)
+- Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`)
+- Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`)
+- Add verification to check if Request callback is callable (:issue:`2766`)
+- Port ``extras/qpsclient.py`` to Python 3 (:issue:`2849`)
+- Use getfullargspec under the scenes for Python 3 to stop DeprecationWarning
+  (:issue:`2862`)
+- Update deprecated test aliases (:issue:`2876`)
+- Fix ``SitemapSpider`` support for alternate links (:issue:`2853`)
+
 
 Docs
 ~~~~
 
-- Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting. (:issue:`2756`)
-- Include references to Scrapy subreddit in the docs (:issue:`2762`)
-- Use https:// for readthedocs links
+- Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY``
+  setting. (:issue:`2756`)
+- Update Contributing docs, document new support channels
+  (:issue:`2762`, issue:`3038`)
+- Include references to Scrapy subreddit in the docs
+- Fix broken links; use https:// for external links
+  (:issue:`2978`, :issue:`2982`, :issue:`2958`)
 - Document CloseSpider extension better (:issue:`2759`)
-- Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example (:issue:`2781`)
-- Spelling mistake and typos (:issue:`2828`, :issue:`2837`, :issue:`#2884`)
+- Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example
+  (:issue:`2781`)
+- Spelling mistake and typos
+  (:issue:`2828`, :issue:`2837`, :issue:`#2884`, :issue:`2924`)
 - Clarify ``CSVFeedSpider.headers`` documentation (:issue:`2826`)
-- Document ``DontCloseSpider`` exception and clarify ``spider_idle`` (:issue:`2791`)
+- Document ``DontCloseSpider`` exception and clarify ``spider_idle``
+  (:issue:`2791`)
 - Update "Releases" section in README (:issue:`2764`)
 - Fix rst syntax in ``DOWNLOAD_FAIL_ON_DATALOSS`` docs (:issue:`2763`)
 - Small fix in description of startproject arguments (:issue:`2866`)
+- Clarify data types in Response.body docs (:issue:`2922`)
+- Add a note about ``request.meta['depth']`` to DepthMiddleware docs (:issue:`2374`)
+- Add a note about ``request.meta['dont_merge_cookies']`` to CookiesMiddleware
+  docs (:issue:`2999`)
+- Up-to-date example of project structure (:issue:`2964`, :issue:`2976`)
+- A better example of ItemExporters usage (:issue:`2989`)
+- Document ``from_crawler`` methods for spider and downloader middlewares
+  (:issue:`3019`)
+
+
 
 
 Scrapy 1.4.0 (2017-05-18)

From d4e5671d07a8dcf18b665ed3ce4136dccae222fb Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 29 Dec 2017 07:06:00 +0500
Subject: [PATCH 1627/2974] make release docs more readable, add highlights

---
 docs/news.rst | 45 ++++++++++++++++++++++++++++++++-------------
 1 file changed, 32 insertions(+), 13 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index df0f10a32..2283b00ae 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -6,13 +6,35 @@ Release notes
 Scrapy 1.5.0 (2017-XX-XX)
 -------------------------
 
-Supported Environments
-----------------------
+This release brings small new features and improvements across the codebase.
+Some highlights:
 
-* Scrapy 1.5 drops support for Python 3.3;
-* this release also improves Python 3.x support (especially 3.6);
-* PyPy and PyPy3 are now supported officially, by running tests on CI.
-* Ubuntu 12.04 and Debian 7.0 are baseline Linux distributions.
+* Google Cloud Storage is supported in FilesPipeline and ImagesPipeline.
+* Crawling with proxy servers becomes more efficient, as connections
+  to proxies can be reused now.
+* Warnings, exception and logging messages are improved to make debugging
+  easier.
+* ``scrapy parse`` command now allows to set custom request meta via
+  ``--meta`` argument.
+* Compatibility with Python 3.6, PyPy and PyPy3 is improved;
+  PyPy and PyPy3 are now supported officially, by running tests on CI.
+* Better default handling of HTTP 308, 522 and 524 status codes.
+* Documentation is improved, as usual.
+
+Backwards Incompatible Changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Scrapy 1.5 drops support for Python 3.3.
+* Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`).
+  **This is technically backwards-incompatible**; override
+  :setting:`USER_AGENT` if you relied on old value.
+* Logging of settings overridden by ``custom_settings`` is fixed;
+  **this is technically backwards-incompatible** because the logger
+  changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``. If you're
+  parsing Scrapy logs, please update your log parsers (:issue:`1343`).
+* LinkExtractor now ignores ``m4v`` extension by default, this is change
+  in behavior.
+* 522 and 524 status codes are added to ``RETRY_HTTP_CODES`` (:issue:`2851`)
 
 New features
 ~~~~~~~~~~~~
@@ -27,20 +49,19 @@ New features
 - Handle HTTP 308 Permanent Redirect (:issue:`2844`)
 - Add 522 and 524 to ``RETRY_HTTP_CODES`` (:issue:`2851`)
 - Log versions information at startup (:issue:`2857`)
+- ``scrapy.mail.MailSender`` now works in Python 3 (it requires Twisted 17.9.0)
+- Connections to proxy servers are reused (:issue:`2743`)
 - Add template for a downloader middleware (:issue:`2755`)
 - Explicit message for NotImplementedError when parse callback not defined
   (:issue:`2831`)
-- Connections to proxy servers are reused (:issue:`2743`)
 - CrawlerProcess got an option to disable installation of root log handler
   (:issue:`2921`)
 - LinkExtractor now ignores ``m4v`` extension by default
-- ``scrapy.mail.MailSender`` now works in Python 3 (it requires Twisted 17.9.0)
 - Better log messages for responses over :setting:`DOWNLOAD_WARNSIZE` and
   :setting:`DOWNLOAD_MAXSIZE` limits (:issue:`2927`)
 - Show warning when a URL is put to ``Spider.allowed_domains`` instead of
   a domain (:issue:`2250`).
 
-
 Bug fixes
 ~~~~~~~~~
 
@@ -52,7 +73,8 @@ Bug fixes
   **This is technically backwards-incompatible**; override
   :setting:`USER_AGENT` if you relied on old value.
 - Fix PyPy and PyPy3 test failures, support them officially
-  (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`, :issue:`3048`)
+  (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`,
+   :issue:`3048`)
 - Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`)
 - Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`)
 - Add verification to check if Request callback is callable (:issue:`2766`)
@@ -62,7 +84,6 @@ Bug fixes
 - Update deprecated test aliases (:issue:`2876`)
 - Fix ``SitemapSpider`` support for alternate links (:issue:`2853`)
 
-
 Docs
 ~~~~
 
@@ -94,8 +115,6 @@ Docs
   (:issue:`3019`)
 
 
-
-
 Scrapy 1.4.0 (2017-05-18)
 -------------------------
 

From c107059ef82a4b7b491b23b740b71353f03ab891 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 29 Dec 2017 07:07:43 +0500
Subject: [PATCH 1628/2974] DOC fix rst syntax

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 2283b00ae..65bccc12d 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -74,7 +74,7 @@ Bug fixes
   :setting:`USER_AGENT` if you relied on old value.
 - Fix PyPy and PyPy3 test failures, support them officially
   (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`,
-   :issue:`3048`)
+  :issue:`3048`)
 - Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`)
 - Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`)
 - Add verification to check if Request callback is callable (:issue:`2766`)

From d07fe11981a07e493faf7454db79b98c02a53118 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 30 Dec 2017 02:09:41 +0500
Subject: [PATCH 1629/2974] set release date

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 65bccc12d..36ead3aba 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,7 +3,7 @@
 Release notes
 =============
 
-Scrapy 1.5.0 (2017-XX-XX)
+Scrapy 1.5.0 (2017-12-29)
 -------------------------
 
 This release brings small new features and improvements across the codebase.

From aa83e159c97b441167d0510064204681bbc93f21 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 30 Dec 2017 02:09:52 +0500
Subject: [PATCH 1630/2974] =?UTF-8?q?Bump=20version:=201.4.0=20=E2=86=92?=
 =?UTF-8?q?=201.5.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 21800f616..6e7be142e 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.4.0
+current_version = 1.5.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 88c5fb891..bc80560fa 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.4.0
+1.5.0

From a0836b8fd9720a9439cb3b940aca53b6844a094b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jes=C3=BAs=20Losada?= <dev@jesuslosada.com>
Date: Mon, 1 Jan 2018 15:59:38 +0000
Subject: [PATCH 1631/2974] Fix link in news.rst

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 36ead3aba..1629510b2 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -98,7 +98,7 @@ Docs
 - Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example
   (:issue:`2781`)
 - Spelling mistake and typos
-  (:issue:`2828`, :issue:`2837`, :issue:`#2884`, :issue:`2924`)
+  (:issue:`2828`, :issue:`2837`, :issue:`2884`, :issue:`2924`)
 - Clarify ``CSVFeedSpider.headers`` documentation (:issue:`2826`)
 - Document ``DontCloseSpider`` exception and clarify ``spider_idle``
   (:issue:`2791`)

From 61c0b1478284b02a4fcfd2cc4931587c348c5d3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jes=C3=BAs=20Losada?= <dev@jesuslosada.com>
Date: Mon, 1 Jan 2018 16:03:55 +0000
Subject: [PATCH 1632/2974] Fix typo in comment

---
 scrapy/core/downloader/handlers/http11.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 8e9727093..038db7b47 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -379,7 +379,7 @@ class ScrapyAgent(object):
                            {'size': expected_size, 'warnsize': warnsize, 'request': request})
 
         def _cancel(_):
-            # Abort connection inmediately.
+            # Abort connection immediately.
             txresponse._transport._producer.abortConnection()
 
         d = defer.Deferred(_cancel)

From 1d1581266c3df99ebf870d6c3e10ac09f2ee3673 Mon Sep 17 00:00:00 2001
From: Yash Sharma <yashrsharma44@gmail.com>
Date: Fri, 26 Jan 2018 01:42:17 +0530
Subject: [PATCH 1633/2974] Changed some documentations (#3089)

DOC typo fix in defer_fail docstring
---
 scrapy/utils/defer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index aa6dcffda..bcf209511 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -11,7 +11,7 @@ def defer_fail(_failure):
     """Same as twisted.internet.defer.fail but delay calling errback until
     next reactor loop
 
-    It delays by 100ms so reactor has a chance to go trough readers and writers
+    It delays by 100ms so reactor has a chance to go through readers and writers
     before attending pending delayed calls, so do not set delay to zero.
     """
     d = defer.Deferred()

From ba15b63ed696dbbd6aa6082c035754a15ca8e03c Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 26 Jan 2018 02:11:49 +0500
Subject: [PATCH 1634/2974] TST fix tests to account for changes in w3lib 1.19

---
 tests/test_http_response.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index a36ec3af6..b228344b5 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -272,7 +272,10 @@ class TextResponseTest(BaseResponseTest):
                                  headers={"Content-type": ["text/html; charset=utf-8"]},
                                  body=b"\xef\xbb\xbfWORD\xe3\xab")
         self.assertEqual(r6.encoding, 'utf-8')
-        self.assertEqual(r6.text, u'WORD\ufffd\ufffd')
+        self.assertIn(r6.text, {
+            u'WORD\ufffd\ufffd',  # w3lib < 1.19.0
+            u'WORD\ufffd',        # w3lib >= 1.19.0
+        })
 
     def test_bom_is_removed_from_body(self):
         # Inferring encoding from body also cache decoded body as sideeffect,

From c1916626c1b5c06f1b3c89cd29db1b7b5bc9996c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jes=C3=BAs=20Losada?= <dev@jesuslosada.com>
Date: Sat, 27 Jan 2018 21:24:15 +0000
Subject: [PATCH 1635/2974] Fix OS signal names

---
 scrapy/utils/ossignal.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py
index df4eee5ec..f87d5a803 100644
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@@ -1,17 +1,18 @@
 
 from __future__ import absolute_import
+import signal
 
 from twisted.internet import reactor
 
-import signal
 
 signal_names = {}
 for signame in dir(signal):
-    if signame.startswith("SIG"):
+    if signame.startswith('SIG') and not signame.startswith('SIG_'):
         signum = getattr(signal, signame)
         if isinstance(signum, int):
             signal_names[signum] = signame
 
+
 def install_shutdown_handlers(function, override_sigint=True):
     """Install the given function as a signal handler for all common shutdown
     signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the
@@ -24,5 +25,5 @@ def install_shutdown_handlers(function, override_sigint=True):
             override_sigint:
         signal.signal(signal.SIGINT, function)
     # Catch Ctrl-Break in windows
-    if hasattr(signal, "SIGBREAK"):
+    if hasattr(signal, 'SIGBREAK'):
         signal.signal(signal.SIGBREAK, function)

From 6f264ab190882d9bfb375688a7e44d03716242ba Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 30 Jan 2018 05:47:28 +0500
Subject: [PATCH 1636/2974] more stats for RobotsTxtMiddleware

---
 scrapy/downloadermiddlewares/robotstxt.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index c3dfa7819..b86c09c14 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -41,10 +41,12 @@ class RobotsTxtMiddleware(object):
         return d
 
     def process_request_2(self, rp, request, spider):
-        if rp is not None and not rp.can_fetch(
-                 to_native_str(self._useragent), request.url):
+        if rp is None:
+            return
+        if not rp.can_fetch(to_native_str(self._useragent), request.url):
             logger.debug("Forbidden by robots.txt: %(request)s",
                          {'request': request}, extra={'spider': spider})
+            self.crawler.stats.inc_value('robotstxt/forbidden')
             raise IgnoreRequest()
 
     def robot_parser(self, request, spider):
@@ -63,6 +65,7 @@ class RobotsTxtMiddleware(object):
             dfd.addCallback(self._parse_robots, netloc)
             dfd.addErrback(self._logerror, robotsreq, spider)
             dfd.addErrback(self._robots_error, netloc)
+            self.crawler.stats.inc_value('robotstxt/request_count')
 
         if isinstance(self._parsers[netloc], Deferred):
             d = Deferred()
@@ -83,11 +86,14 @@ class RobotsTxtMiddleware(object):
         return failure
 
     def _parse_robots(self, response, netloc):
+        self.crawler.stats.inc_value('robotstxt/response_count')
+        self.crawler.stats.inc_value(
+            'robotstxt/response_status_count/{}'.format(response.status))
         rp = robotparser.RobotFileParser(response.url)
         body = ''
         if hasattr(response, 'text'):
             body = response.text
-        else: # last effort try
+        else:  # last effort try
             try:
                 body = response.body.decode('utf-8')
             except UnicodeDecodeError:
@@ -95,7 +101,7 @@ class RobotsTxtMiddleware(object):
                 # but keep the lookup cached (in self._parsers)
                 # Running rp.parse() will set rp state from
                 # 'disallow all' to 'allow any'.
-                pass
+                self.crawler.stats.inc_value('robotstxt/unicode_error_count')
         # stdlib's robotparser expects native 'str' ;
         # with unicode input, non-ASCII encoded bytes decoding fails in Python2
         rp.parse(to_native_str(body).splitlines())
@@ -105,6 +111,9 @@ class RobotsTxtMiddleware(object):
         rp_dfd.callback(rp)
 
     def _robots_error(self, failure, netloc):
+        if failure.type is not IgnoreRequest:
+            key = 'robotstxt/exception_count/{}'.format(failure.type)
+            self.crawler.stats.inc_value(key)
         rp_dfd = self._parsers[netloc]
         self._parsers[netloc] = None
         rp_dfd.callback(None)

From 4d5e5378bd2ccea2879102614536410d9338b3f1 Mon Sep 17 00:00:00 2001
From: Wenbin Zhang <WenbinZhang@users.noreply.github.com>
Date: Wed, 7 Feb 2018 10:59:32 -0500
Subject: [PATCH 1637/2974] Update robotstxt.py

Add message to IgnoreRequest exception so that it can be detectedin the errbak method of a spider
---
 scrapy/downloadermiddlewares/robotstxt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index c3dfa7819..4f3b5d64f 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -45,7 +45,7 @@ class RobotsTxtMiddleware(object):
                  to_native_str(self._useragent), request.url):
             logger.debug("Forbidden by robots.txt: %(request)s",
                          {'request': request}, extra={'spider': spider})
-            raise IgnoreRequest()
+            raise IgnoreRequest("Forbidden by robots.txt")
 
     def robot_parser(self, request, spider):
         url = urlparse_cached(request)

From 0c374c00fb7c8d24a60bec8a94f7cbb06accb980 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 8 Feb 2018 05:09:02 +0500
Subject: [PATCH 1638/2974] use INFO log level to show telnet host/port

---
 scrapy/extensions/telnet.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 5ca0d19a0..e78afa1fc 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -55,9 +55,9 @@ class TelnetConsole(protocol.ServerFactory):
     def start_listening(self):
         self.port = listen_tcp(self.portrange, self.host, self)
         h = self.port.getHost()
-        logger.debug("Telnet console listening on %(host)s:%(port)d",
-                     {'host': h.host, 'port': h.port},
-                     extra={'crawler': self.crawler})
+        logger.info("Telnet console listening on %(host)s:%(port)d",
+                    {'host': h.host, 'port': h.port},
+                    extra={'crawler': self.crawler})
 
     def stop_listening(self):
         self.port.stopListening()

From a56540877c2c24d3ba787cc43ca1f81d91b386fd Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 16 Jan 2018 16:14:35 -0300
Subject: [PATCH 1639/2974] Do not serialize unpickable objects (py3)

---
 scrapy/squeues.py     | 7 ++++---
 tests/test_squeues.py | 9 +++++++--
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 21520f454..0b8f6af7d 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -25,9 +25,10 @@ def _serializable_queue(queue_class, serialize, deserialize):
 def _pickle_serialize(obj):
     try:
         return pickle.dumps(obj, protocol=2)
-    # Python>=3.5 raises AttributeError here while
-    # Python<=3.4 raises pickle.PicklingError
-    except (pickle.PicklingError, AttributeError) as e:
+    # Python<=3.4 raises pickle.PicklingError here while
+    # Python>=3.5 raises AttributeError and
+    # Python>=3.6 raises TypeError
+    except (pickle.PicklingError, AttributeError, TypeError) as e:
         raise ValueError(str(e))
 
 PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 3a24348b4..d2f721241 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -1,3 +1,4 @@
+from sys import version_info
 import pickle
 
 from queuelib.tests import test_queue as t
@@ -5,6 +6,7 @@ from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFif
 from scrapy.item import Item, Field
 from scrapy.http import Request
 from scrapy.loader import ItemLoader
+from scrapy.selector import Selector
 
 class TestItem(Item):
     name = Field()
@@ -17,20 +19,23 @@ class TestLoader(ItemLoader):
     name_out = staticmethod(_test_procesor)
 
 def nonserializable_object_test(self):
+    q = self.queue()
     try:
         pickle.dumps(lambda x: x)
     except Exception:
         # Trigger Twisted bug #7989
         import twisted.persisted.styles  # NOQA
-        q = self.queue()
         self.assertRaises(ValueError, q.push, lambda x: x)
     else:
         # Use a different unpickleable object
         class A(object): pass
         a = A()
         a.__reduce__ = a.__reduce_ex__ = None
-        q = self.queue()
         self.assertRaises(ValueError, q.push, a)
+    if version_info.major == 3 and version_info.minor >= 6:
+        # Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
+        sel = Selector(text='<html><body><p>some text</p></body></html>')
+        self.assertRaises(ValueError, q.push, sel)
 
 class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
 

From e4558cb27e8eeec8432a06124acf8c2569784ccc Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 19 Jan 2018 10:51:30 -0300
Subject: [PATCH 1640/2974] Update test for unpickable objects

---
 tests/test_squeues.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index d2f721241..3ded5c027 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -1,4 +1,3 @@
-from sys import version_info
 import pickle
 
 from queuelib.tests import test_queue as t
@@ -32,10 +31,9 @@ def nonserializable_object_test(self):
         a = A()
         a.__reduce__ = a.__reduce_ex__ = None
         self.assertRaises(ValueError, q.push, a)
-    if version_info.major == 3 and version_info.minor >= 6:
-        # Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
-        sel = Selector(text='<html><body><p>some text</p></body></html>')
-        self.assertRaises(ValueError, q.push, sel)
+    # Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
+    sel = Selector(text='<html><body><p>some text</p></body></html>')
+    self.assertRaises(ValueError, q.push, sel)
 
 class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
 

From 0d87e77afeb506c69f4717744917b95234a86650 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 8 Feb 2018 14:49:26 -0300
Subject: [PATCH 1641/2974] Bump parsel dependency

---
 requirements.txt | 2 +-
 setup.py         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 392f83dd6..2a94d742d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,4 +7,4 @@ queuelib
 six>=1.5.2
 PyDispatcher>=2.0.5
 service_identity
-parsel>=1.1
+parsel>=1.4
diff --git a/setup.py b/setup.py
index 06a36e2ba..c37919cda 100644
--- a/setup.py
+++ b/setup.py
@@ -71,7 +71,7 @@ setup(
         'pyOpenSSL',
         'cssselect>=0.9',
         'six>=1.5.2',
-        'parsel>=1.1',
+        'parsel>=1.4',
         'PyDispatcher>=2.0.5',
         'service_identity',
     ],

From 6edd4114c4e715a3a0c440af455fff089a099620 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 8 Feb 2018 15:47:20 -0300
Subject: [PATCH 1642/2974] Clarify comment about Pyhton versions

---
 scrapy/squeues.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 0b8f6af7d..d2074a457 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -25,9 +25,9 @@ def _serializable_queue(queue_class, serialize, deserialize):
 def _pickle_serialize(obj):
     try:
         return pickle.dumps(obj, protocol=2)
-    # Python<=3.4 raises pickle.PicklingError here while
-    # Python>=3.5 raises AttributeError and
-    # Python>=3.6 raises TypeError
+    # Python <= 3.4 raises pickle.PicklingError here while
+    # 3.5 <= Python < 3.6 raises AttributeError and
+    # Python >= 3.6 raises TypeError
     except (pickle.PicklingError, AttributeError, TypeError) as e:
         raise ValueError(str(e))
 

From dc0304fde1b29b4973fabf9d189eb5c4084bf899 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 13 Feb 2018 19:47:41 +0500
Subject: [PATCH 1643/2974] fix docs building with recent sphinx: don't use
 deprecated sphinx options and imports

---
 docs/_ext/scrapydocs.py | 6 +++++-
 docs/conf.py            | 4 ----
 docs/requirements.txt   | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index 83b0d2cc6..192123473 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -1,6 +1,6 @@
 from docutils.parsers.rst.roles import set_classes
 from docutils import nodes
-from sphinx.util.compat import Directive
+from docutils.parsers.rst import Directive
 from sphinx.util.nodes import make_refnode
 from operator import itemgetter
 
@@ -110,24 +110,28 @@ def setup(app):
     app.connect('doctree-read', collect_scrapy_settings_refs)
     app.connect('doctree-resolved', replace_settingslist_nodes)
 
+
 def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
     ref = 'https://github.com/scrapy/scrapy/blob/master/' + text
     set_classes(options)
     node = nodes.reference(rawtext, text, refuri=ref, **options)
     return [node], []
 
+
 def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
     ref = 'https://github.com/scrapy/scrapy/issues/' + text
     set_classes(options)
     node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options)
     return [node], []
 
+
 def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
     ref = 'https://github.com/scrapy/scrapy/commit/' + text
     set_classes(options)
     node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options)
     return [node], []
 
+
 def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
     ref = 'http://hg.scrapy.org/scrapy/changeset/' + text
     set_classes(options)
diff --git a/docs/conf.py b/docs/conf.py
index 007dc2788..594740f39 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -144,10 +144,6 @@ html_static_path = ['_static']
 # using the given strftime format.
 html_last_updated_fmt = '%b %d, %Y'
 
-# If true, SmartyPants will be used to convert quotes and dashes to
-# typographically correct entities.
-html_use_smartypants = True
-
 # Custom sidebar templates, maps document names to template names.
 #html_sidebars = {}
 
diff --git a/docs/requirements.txt b/docs/requirements.txt
index d3dcb97be..8e7611d21 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,2 +1,2 @@
-Sphinx>=1.3
+Sphinx>=1.6
 sphinx_rtd_theme
\ No newline at end of file

From 6954da136604ef2a5bf37f8d792e0de6ec90469a Mon Sep 17 00:00:00 2001
From: Anjali Jain <aj2966@gmail.com>
Date: Thu, 15 Feb 2018 23:27:40 +0530
Subject: [PATCH 1644/2974] Updated contributing.rst

Rectified grammatical errors
---
 docs/contributing.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 9a02634cb..44068baa9 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -6,7 +6,7 @@ Contributing to Scrapy
 
 .. important::
 
-    Double check you are reading the most recent version of this document at
+    Double check that you are reading the most recent version of this document at
     https://doc.scrapy.org/en/master/contributing.html
 
 There are many ways to contribute to Scrapy. Here are some of them:
@@ -18,7 +18,7 @@ There are many ways to contribute to Scrapy. Here are some of them:
 * Report bugs and request features in the `issue tracker`_, trying to follow
   the guidelines detailed in `Reporting bugs`_ below.
 
-* Submit patches for new functionality and/or bug fixes. Please read
+* Submit patches for new functionalities and/or bug fixes. Please read
   :ref:`writing-patches` and `Submitting patches`_ below for details on how to
   write and submit a patch.
 
@@ -80,8 +80,8 @@ guidelines when reporting a new bug.
 Writing patches
 ===============
 
-The better written a patch is, the higher chance that it'll get accepted and
-the sooner that will be merged.
+The better written a patch is, higher is the chance that it'll get accepted and 
+sooner it will be merged.
 
 Well-written patches should:
 

From bbc2a3569f153b20bf306375883688d99b565cb3 Mon Sep 17 00:00:00 2001
From: Anjali Jain <aj2966@gmail.com>
Date: Fri, 16 Feb 2018 23:33:10 +0530
Subject: [PATCH 1645/2974] further edited

---
 docs/contributing.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 44068baa9..f4f9e393f 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -80,8 +80,7 @@ guidelines when reporting a new bug.
 Writing patches
 ===============
 
-The better written a patch is, higher is the chance that it'll get accepted and 
-sooner it will be merged.
+The better a patch is written, the higher the chances that it'll get accepted and the sooner it will be merged.
 
 Well-written patches should:
 

From acd2b8d43b5ebec7ffd364b6f335427041a0b98d Mon Sep 17 00:00:00 2001
From: NewUserHa <32261870+NewUserHa@users.noreply.github.com>
Date: Thu, 22 Feb 2018 06:37:26 +0800
Subject: [PATCH 1646/2974] [MRG+1] Fix part of issue #3128 - None should not
 be a valid type for 'url' in Response.follow (#3131)

* fix one issue of issue#3128

because @kmike posted: 'If url is '', Scrapy should follow the same page, this is an intended behavior.'

*  fix one issue of issue#3128

because @kmike posted: 'If url is '', Scrapy should follow the same page, this is an intended behavior.'
---
 scrapy/http/response/__init__.py | 2 ++
 tests/test_http_response.py      | 4 ++++
 2 files changed, 6 insertions(+)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 434d87eab..1974259b5 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -120,6 +120,8 @@ class Response(object_ref):
         """
         if isinstance(url, Link):
             url = url.url
+        elif url is None:
+            raise ValueError("url can't be None")
         url = self.urljoin(url)
         return Request(url, callback,
                        method=method,
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index b228344b5..820758dc9 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -155,6 +155,10 @@ class BaseResponseTest(unittest.TestCase):
         self._assert_followed_url(Link('http://example.com/foo'),
                                   'http://example.com/foo')
 
+    def test_follow_None_url(self):
+        r = self.response_class("http://example.com")
+        self.assertRaises(ValueError, r.follow, None)
+
     def test_follow_whitespace_url(self):
         self._assert_followed_url('foo ',
                                   'http://example.com/foo%20')

From aca2655c12d806759c6e0821a40d0277d200e0ea Mon Sep 17 00:00:00 2001
From: Patience Shyu <patienceshyu@gmail.com>
Date: Fri, 2 Mar 2018 14:57:39 +0100
Subject: [PATCH 1647/2974] [WIP] Run tests for Python 3.7

---
 .travis.yml | 2 ++
 tox.ini     | 4 ++++
 2 files changed, 6 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 6635f5d3b..e4df22139 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,6 +23,8 @@ matrix:
       env: TOXENV=py36
     - python: 3.6
       env: TOXENV=docs
+    - python: 3.7
+      env: TOXENV=py37
 install:
   - |
       if [ "$TOXENV" = "pypy" ]; then
diff --git a/tox.ini b/tox.ini
index 60ff8c15e..5301624ee 100644
--- a/tox.ini
+++ b/tox.ini
@@ -79,6 +79,10 @@ deps = {[testenv:py34]deps}
 basepython = python3.6
 deps = {[testenv:py34]deps}
 
+[testenv:py37]
+basepython = python3.7
+deps = {[testenv:py34]deps}
+
 [testenv:pypy3]
 basepython = pypy3
 deps = {[testenv:py34]deps}

From fab68ff6260b9ce4f55ca7b211a1aeb3e8e6df3d Mon Sep 17 00:00:00 2001
From: Patience Shyu <patienceshyu@gmail.com>
Date: Fri, 2 Mar 2018 17:05:14 +0100
Subject: [PATCH 1648/2974] Use 3.7-dev version for travis

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index e4df22139..aa1a3c4c3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,7 +23,7 @@ matrix:
       env: TOXENV=py36
     - python: 3.6
       env: TOXENV=docs
-    - python: 3.7
+    - python: 3.7-dev
       env: TOXENV=py37
 install:
   - |

From 4c05441450bc1f8438239af0310ab76777a2dacf Mon Sep 17 00:00:00 2001
From: nctl144 <nctl144@gmail.com>
Date: Sat, 3 Mar 2018 00:00:03 -0500
Subject: [PATCH 1649/2974] add ftp to the scheme list

---
 scrapy/linkextractors/__init__.py |  3 ++-
 tests/test_linkextractors.py      | 12 +++++++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 2d7115cc5..cda6ddc7e 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -41,7 +41,8 @@ IGNORED_EXTENSIONS = [
 
 _re_type = type(re.compile("", 0))
 _matches = lambda url, regexs: any(r.search(url) for r in regexs)
-_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'}
+_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', \
+                                                       'file', 'ftp'}
 
 
 class FilteringLinkExtractor(object):
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 1d7c4f311..903032b52 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -451,6 +451,17 @@ class Base:
                 Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
             ])
 
+        def test_ftp_links(self):
+            body = b"""
+            <html><body>
+            <div><a href="ftp://www.external.com/">An Item</a></div>
+            </body></html>"""
+            response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8')
+            lx = self.extractor_cls()
+            self.assertEqual(lx.extract_links(response), [
+                Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False),
+            ])
+
 
 class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
     extractor_cls = LxmlLinkExtractor
@@ -471,4 +482,3 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
     @pytest.mark.xfail
     def test_restrict_xpaths_with_html_entities(self):
         super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()
-

From d5b7ebcfdcfd40d29990712b587893fcc6e84ce8 Mon Sep 17 00:00:00 2001
From: Viral Mehta <virmht@gmail.com>
Date: Sat, 3 Mar 2018 18:17:49 +0530
Subject: [PATCH 1650/2974] Fixed bug FormRequest.from_response() clickdata
 ignores input[type=image]

---
 scrapy/http/request/form.py | 10 ++++++----
 tests/test_http_request.py  | 10 ++++++++++
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index d9d178a3e..184ee2599 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -170,10 +170,12 @@ def _get_clickable(clickdata, form):
     """
     clickables = [
         el for el in form.xpath(
-            'descendant::*[(self::input or self::button)'
-            ' and re:test(@type, "^submit$", "i")]'
-            '|descendant::button[not(@type)]',
-            namespaces={"re": "http://exslt.org/regular-expressions"})
+             'descendant::*[(self::input or self::button)'
+             ' and re:test(@type, "^submit$", "i")]'
+             '|descendant::*[(self::input or self::button)'
+             ' and re:test(@type, "^image$", "i")]'
+             '|descendant::button[not(@type)]',
+             namespaces={"re": "http://exslt.org/regular-expressions"})
         ]
     if not clickables:
         return
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index fca8ff411..73a74cd5d 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -532,6 +532,16 @@ class FormRequestTest(RequestTest):
         req = self.request_class.from_response(response, dont_click=True)
         fs = _qs(req)
         self.assertEqual(fs, {b'i1': [b'i1v']})
+    
+    def test_from_response_clickdata_does_not_ignore_image(self):
+        response = _buildresponse(
+            """<form>
+            <input type="text" name="i1" value="i1v">
+            <input id="image" name="i2" type="image" value="i2v" alt="Login" src="http://my.image.org/1.jpg">
+            </form>""")
+        req = self.request_class.from_response(response, dont_click=True)
+        fs = _qs(req)
+        self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']})
 
     def test_from_response_dont_submit_reset_as_input(self):
         response = _buildresponse(

From 65744c2199fc6a5bccfa11eec40a867c1401aee9 Mon Sep 17 00:00:00 2001
From: Viral Mehta <virmht@gmail.com>
Date: Sat, 3 Mar 2018 20:07:50 +0530
Subject: [PATCH 1651/2974] Corrected Test

---
 tests/test_http_request.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 73a74cd5d..a042f03b6 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -532,16 +532,6 @@ class FormRequestTest(RequestTest):
         req = self.request_class.from_response(response, dont_click=True)
         fs = _qs(req)
         self.assertEqual(fs, {b'i1': [b'i1v']})
-    
-    def test_from_response_clickdata_does_not_ignore_image(self):
-        response = _buildresponse(
-            """<form>
-            <input type="text" name="i1" value="i1v">
-            <input id="image" name="i2" type="image" value="i2v" alt="Login" src="http://my.image.org/1.jpg">
-            </form>""")
-        req = self.request_class.from_response(response, dont_click=True)
-        fs = _qs(req)
-        self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']})
 
     def test_from_response_dont_submit_reset_as_input(self):
         response = _buildresponse(
@@ -554,6 +544,16 @@ class FormRequestTest(RequestTest):
         req = self.request_class.from_response(response, dont_click=True)
         fs = _qs(req)
         self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']})
+    
+    def test_from_response_clickdata_does_not_ignore_image(self):
+        response = _buildresponse(
+            """<form>
+            <input type="text" name="i1" value="i1v">
+            <input id="image" name="i2" type="image" value="i2v" alt="Login" src="http://my.image.org/1.jpg">
+            </form>""")
+        req = self.request_class.from_response(response)
+        fs = _qs(req)
+        self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']})
 
     def test_from_response_multiple_clickdata(self):
         response = _buildresponse(

From ca7d79c29a55be4482bf2d4f704fc145ba801337 Mon Sep 17 00:00:00 2001
From: Patience Shyu <patienceshyu@gmail.com>
Date: Mon, 5 Mar 2018 10:46:51 +0100
Subject: [PATCH 1652/2974] Install Twisted from branch to bypass syntax issue

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 2a94d742d..47eddf1fc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-Twisted>=13.1.0
+git+https://github.com/twisted/twisted.git@dcaf946
 lxml
 pyOpenSSL
 cssselect>=0.9

From 5d1f5245f2699745e73449b013a29cc370f424c9 Mon Sep 17 00:00:00 2001
From: Patience Shyu <patienceshyu@gmail.com>
Date: Mon, 5 Mar 2018 11:14:50 +0100
Subject: [PATCH 1653/2974] [WIP] Install Twisted from branch to bypass syntax
 issue

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 47eddf1fc..95cd37772 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-git+https://github.com/twisted/twisted.git@dcaf946
+git+https://github.com/lopuhin/twisted.git@9384-remove-async-param
 lxml
 pyOpenSSL
 cssselect>=0.9

From f10a43d562dee32f324703fddb19bee5266912ce Mon Sep 17 00:00:00 2001
From: Patience Shyu <patienceshyu@gmail.com>
Date: Mon, 5 Mar 2018 11:43:39 +0100
Subject: [PATCH 1654/2974] [WIP] Install Twisted from branch for py3.7

---
 requirements-py3.txt | 2 +-
 requirements.txt     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements-py3.txt b/requirements-py3.txt
index 2aae3ae65..c3357e970 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -1,4 +1,4 @@
-Twisted >= 17.9.0
+git+https://github.com/lopuhin/twisted.git@9384-remove-async-param
 lxml>=3.2.4
 pyOpenSSL>=0.13.1
 cssselect>=0.9
diff --git a/requirements.txt b/requirements.txt
index 95cd37772..2a94d742d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-git+https://github.com/lopuhin/twisted.git@9384-remove-async-param
+Twisted>=13.1.0
 lxml
 pyOpenSSL
 cssselect>=0.9

From 13a74d77e2e4d1719bf984d99ca2ae6874752e41 Mon Sep 17 00:00:00 2001
From: Lucy Wang <wxitb2017@gmail.com>
Date: Mon, 12 Mar 2018 22:25:19 +0800
Subject: [PATCH 1655/2974] catch CertificateError in tls verification

---
 scrapy/core/downloader/tls.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 498e3d60f..e1c4f4908 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -40,6 +40,7 @@ if twisted_version >= (14, 0, 0):
     from twisted.internet._sslverify import (ClientTLSOptions,
                                              verifyHostname,
                                              VerificationError)
+    from service_identity.exceptions import CertificateError
 
     if twisted_version < (17, 0, 0):
         from twisted.internet._sslverify import _maybeSetHostNameIndication
@@ -65,7 +66,7 @@ if twisted_version >= (14, 0, 0):
             elif where & SSL_CB_HANDSHAKE_DONE:
                 try:
                     verifyHostname(connection, self._hostnameASCII)
-                except VerificationError as e:
+                except (CertificateError, VerificationError) as e:
                     logger.warning(
                         'Remote certificate is not valid for hostname "{}"; {}'.format(
                             self._hostnameASCII, e))

From e487100987496474de4d2b5fc509ff47d01cd0b6 Mon Sep 17 00:00:00 2001
From: Lucy Wang <wxitb2017@gmail.com>
Date: Tue, 13 Mar 2018 08:59:03 +0800
Subject: [PATCH 1656/2974] add a test case

---
 tests/keys/localhost-ip.gen.README | 21 ++++++++++++
 tests/keys/localhost.crt           | 36 ++++++++++-----------
 tests/keys/localhost.ip.crt        | 20 ++++++++++++
 tests/keys/localhost.ip.key        | 28 ++++++++++++++++
 tests/keys/localhost.key           | 52 +++++++++++++++---------------
 tests/test_downloader_handlers.py  |  9 ++++++
 6 files changed, 122 insertions(+), 44 deletions(-)
 create mode 100644 tests/keys/localhost-ip.gen.README
 create mode 100644 tests/keys/localhost.ip.crt
 create mode 100644 tests/keys/localhost.ip.key

diff --git a/tests/keys/localhost-ip.gen.README b/tests/keys/localhost-ip.gen.README
new file mode 100644
index 000000000..8e94e1217
--- /dev/null
+++ b/tests/keys/localhost-ip.gen.README
@@ -0,0 +1,21 @@
+$ openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt
+Generating a 2048 bit RSA private key
+...................................................................................................+++
+.....+++
+writing new private key to 'localhost.key'
+-----
+You are about to be asked to enter information that will be incorporated
+into your certificate request.
+What you are about to enter is what is called a Distinguished Name or a DN.
+There are quite a few fields but you can leave some blank
+For some fields there will be a default value,
+If you enter '.', the field will be left blank.
+-----
+Country Name (2 letter code) [AU]:IE
+State or Province Name (full name) [Some-State]:.
+Locality Name (eg, city) []:.
+Organization Name (eg, company) [Internet Widgits Pty Ltd]:Scrapy
+Organizational Unit Name (eg, section) []:.
+Common Name (e.g. server FQDN or YOUR name) []:127.0.0.1
+Email Address []:.
+
diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt
index 13c5b5bd6..48d7bd9a3 100644
--- a/tests/keys/localhost.crt
+++ b/tests/keys/localhost.crt
@@ -1,20 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV
-BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x
-NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD
-VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB
-BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K
-o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y
-njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY
-ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/
-8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52
-gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O
-BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j
-xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES
-9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i
-jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+
-TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own
-tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM
-am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U
-Mjybr/hQK3mnVdE=
+MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV
+BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x
+ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD
+VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL
+JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev
+y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV
+ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR
+v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4
+JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O
+BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ
+dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr
+idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP
+8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X
+kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ
+FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb
+smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO
+zh6AcsnjkLumOkM=
 -----END CERTIFICATE-----
diff --git a/tests/keys/localhost.ip.crt b/tests/keys/localhost.ip.crt
new file mode 100644
index 000000000..48d7bd9a3
--- /dev/null
+++ b/tests/keys/localhost.ip.crt
@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV
+BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x
+ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD
+VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL
+JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev
+y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV
+ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR
+v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4
+JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O
+BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ
+dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr
+idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP
+8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X
+kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ
+FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb
+smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO
+zh6AcsnjkLumOkM=
+-----END CERTIFICATE-----
diff --git a/tests/keys/localhost.ip.key b/tests/keys/localhost.ip.key
new file mode 100644
index 000000000..1e12c1255
--- /dev/null
+++ b/tests/keys/localhost.ip.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo
++g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC
+T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb
+rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR
+3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi
+JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj
+uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES
+FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc
+BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud
+JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD
+bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT
+ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx
+KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N
+yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA
+hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+
+32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie
+PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY
+r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2
+X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz
+ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ
+Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ
+g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5
+WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+
+inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u
+BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz
+m1Q+9xJdtRxlYc1YGTK1YGCq
+-----END PRIVATE KEY-----
diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key
index da975e6d3..1e12c1255 100644
--- a/tests/keys/localhost.key
+++ b/tests/keys/localhost.key
@@ -1,28 +1,28 @@
 -----BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5
-ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4
-JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz
-vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2
-RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY
-b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/
-XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0
-gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk
-i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu
-Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss
-ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4
-hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD
-7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa
-EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf
-uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG
-HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR
-UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO
-j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri
-fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ
-6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI
-xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g
-DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f
-npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM
-8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6
-J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj
-Z+pMCFLbwAbFZ9Qb/NFlUv0=
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo
++g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC
+T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb
+rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR
+3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi
+JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj
+uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES
+FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc
+BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud
+JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD
+bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT
+ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx
+KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N
+yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA
+hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+
+32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie
+PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY
+r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2
+X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz
+ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ
+Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ
+g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5
+WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+
+inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u
+BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz
+m1Q+9xJdtRxlYc1YGTK1YGCq
 -----END PRIVATE KEY-----
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index bd2c86292..ceb03f945 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -505,6 +505,15 @@ class Https11InvalidDNSId(Https11TestCase):
         super(Https11InvalidDNSId, self).setUp()
         self.host = '127.0.0.1'
 
+class Https11InvalidDNSPattern(Https11TestCase):
+    """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
+
+    keyfile = 'keys/localhost.ip.key'
+    certfile = 'keys/localhost.ip.crt'
+
+    def setUp(self):
+        super(Https11InvalidDNSPattern, self).setUp()
+
 
 class Http11MockServerTestCase(unittest.TestCase):
     """HTTP 1.1 test case with MockServer"""

From d9e6c73fb3ef787e39474bda5f008b309b65c65b Mon Sep 17 00:00:00 2001
From: Lucy Wang <wxitb2017@gmail.com>
Date: Tue, 13 Mar 2018 13:05:37 +0800
Subject: [PATCH 1657/2974] revert wrong changes

---
 tests/keys/localhost.crt | 36 ++++++++++++++--------------
 tests/keys/localhost.key | 52 ++++++++++++++++++++--------------------
 2 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt
index 48d7bd9a3..13c5b5bd6 100644
--- a/tests/keys/localhost.crt
+++ b/tests/keys/localhost.crt
@@ -1,20 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV
-BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x
-ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD
-VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB
-BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL
-JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev
-y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV
-ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR
-v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4
-JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O
-BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ
-dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr
-idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP
-8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X
-kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ
-FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb
-smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO
-zh6AcsnjkLumOkM=
+MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV
+BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x
+NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD
+VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K
+o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y
+njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY
+ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/
+8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52
+gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O
+BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j
+xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES
+9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i
+jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+
+TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own
+tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM
+am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U
+Mjybr/hQK3mnVdE=
 -----END CERTIFICATE-----
diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key
index 1e12c1255..da975e6d3 100644
--- a/tests/keys/localhost.key
+++ b/tests/keys/localhost.key
@@ -1,28 +1,28 @@
 -----BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo
-+g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC
-T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb
-rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR
-3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi
-JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj
-uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES
-FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc
-BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud
-JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD
-bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT
-ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx
-KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N
-yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA
-hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+
-32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie
-PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY
-r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2
-X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz
-ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ
-Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ
-g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5
-WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+
-inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u
-BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz
-m1Q+9xJdtRxlYc1YGTK1YGCq
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5
+ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4
+JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz
+vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2
+RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY
+b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/
+XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0
+gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk
+i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu
+Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss
+ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4
+hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD
+7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa
+EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf
+uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG
+HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR
+UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO
+j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri
+fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ
+6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI
+xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g
+DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f
+npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM
+8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6
+J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj
+Z+pMCFLbwAbFZ9Qb/NFlUv0=
 -----END PRIVATE KEY-----

From 6a7cdf9a6c4162cf4dd91721d28974c80e0f68cd Mon Sep 17 00:00:00 2001
From: siulkilulki <jurkiewiczdawid@gmail.com>
Date: Tue, 13 Mar 2018 08:35:27 +0100
Subject: [PATCH 1658/2974] [MRG+1] Add 'flv' to ignored video extensions.
 (#3165)

---
 scrapy/linkextractors/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 2d7115cc5..c3c79cf25 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -28,7 +28,7 @@ IGNORED_EXTENSIONS = [
 
     # video
     '3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv',
-    'm4a', 'm4v',
+    'm4a', 'm4v', 'flv',
 
     # office suites
     'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg',

From 1a2f0193a30c1aceb4743e33a90b3c264e0b09c4 Mon Sep 17 00:00:00 2001
From: Lucy Wang <wxitb2017@gmail.com>
Date: Tue, 13 Mar 2018 19:14:52 +0800
Subject: [PATCH 1659/2974] fix tests on jessie

---
 scrapy/core/downloader/tls.py     | 11 +++++++++--
 tests/test_downloader_handlers.py |  4 ++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index e1c4f4908..c97c6a9a9 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -40,7 +40,14 @@ if twisted_version >= (14, 0, 0):
     from twisted.internet._sslverify import (ClientTLSOptions,
                                              verifyHostname,
                                              VerificationError)
-    from service_identity.exceptions import CertificateError
+    try:
+        # XXX: this import would fail on Debian jessie with system installed
+        # service_identity library, due to lack of cryptography.x509 dependency
+        # See https://github.com/pyca/service_identity/issues/21
+        from service_identity.exceptions import CertificateError
+        verification_errors = (CertificateError, VerificationError)
+    except ImportError:
+        verification_errors = VerificationError
 
     if twisted_version < (17, 0, 0):
         from twisted.internet._sslverify import _maybeSetHostNameIndication
@@ -66,7 +73,7 @@ if twisted_version >= (14, 0, 0):
             elif where & SSL_CB_HANDSHAKE_DONE:
                 try:
                     verifyHostname(connection, self._hostnameASCII)
-                except (CertificateError, VerificationError) as e:
+                except verification_errors as e:
                     logger.warning(
                         'Remote certificate is not valid for hostname "{}"; {}'.format(
                             self._hostnameASCII, e))
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index ceb03f945..b34faa7e7 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -512,6 +512,10 @@ class Https11InvalidDNSPattern(Https11TestCase):
     certfile = 'keys/localhost.ip.crt'
 
     def setUp(self):
+        try:
+            from service_identity.exceptions import CertificateError
+        except ImportError:
+            raise unittest.SkipTest("cryptography lib is too old")
         super(Https11InvalidDNSPattern, self).setUp()
 
 

From 2c58da19a6f85f4532d80d3a941cede8f9d0bab8 Mon Sep 17 00:00:00 2001
From: Lucy Wang <wxitb2017@gmail.com>
Date: Wed, 14 Mar 2018 09:27:59 +0800
Subject: [PATCH 1660/2974] update docstring of ScrapyClientTLSOptions

---
 scrapy/core/downloader/tls.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index c97c6a9a9..df8051182 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -63,8 +63,9 @@ if twisted_version >= (14, 0, 0):
         (for genuinely invalid certificates or bugs in verification code).
 
         Same as Twisted's private _sslverify.ClientTLSOptions,
-        except that VerificationError and ValueError exceptions are caught,
-        so that the connection is not closed, only logging warnings.
+        except that VerificationError, CertificateError and ValueError
+        exceptions are caught, so that the connection is not closed, only
+        logging warnings.
         """
 
         def _identityVerifyingInfoCallback(self, connection, where, ret):

From ff5f717f7a2aaf0a1a1101019485201427edd536 Mon Sep 17 00:00:00 2001
From: Viral Mehta <virmht@gmail.com>
Date: Sat, 17 Mar 2018 18:17:48 +0530
Subject: [PATCH 1661/2974] Fixed formatting issues

---
 scrapy/http/request/form.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 184ee2599..22846ad77 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -168,15 +168,17 @@ def _get_clickable(clickdata, form):
     if the latter is given. If not, it returns the first
     clickable element found
     """
+    print("form =", form.__dict__)
     clickables = [
         el for el in form.xpath(
-             'descendant::*[(self::input or self::button)'
-             ' and re:test(@type, "^submit$", "i")]'
-             '|descendant::*[(self::input or self::button)'
-             ' and re:test(@type, "^image$", "i")]'
-             '|descendant::button[not(@type)]',
-             namespaces={"re": "http://exslt.org/regular-expressions"})
+            'descendant::*[(self::input or self::button)'
+            ' and re:test(@type, "^submit$", "i")]'
+            '|descendant::*[(self::input or self::button)'
+            ' and re:test(@type, "^image$", "i")]'
+            '|descendant::button[not(@type)]',
+            namespaces={"re": "http://exslt.org/regular-expressions"})
         ]
+    print("clickables =", clickables)
     if not clickables:
         return
 

From e25e2afe174bbe70ecbf88ea684937890e8ba4d5 Mon Sep 17 00:00:00 2001
From: Viral Mehta <virmht@gmail.com>
Date: Sat, 17 Mar 2018 18:20:14 +0530
Subject: [PATCH 1662/2974] Removed unnecessary print statements

---
 scrapy/http/request/form.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 22846ad77..238dd44b3 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -168,7 +168,6 @@ def _get_clickable(clickdata, form):
     if the latter is given. If not, it returns the first
     clickable element found
     """
-    print("form =", form.__dict__)
     clickables = [
         el for el in form.xpath(
             'descendant::*[(self::input or self::button)'
@@ -178,7 +177,6 @@ def _get_clickable(clickdata, form):
             '|descendant::button[not(@type)]',
             namespaces={"re": "http://exslt.org/regular-expressions"})
         ]
-    print("clickables =", clickables)
     if not clickables:
         return
 

From a5acc9373f8735e27c22de6fbe345fbed8f268c1 Mon Sep 17 00:00:00 2001
From: Viral Mehta <virmht@gmail.com>
Date: Mon, 19 Mar 2018 18:19:39 +0530
Subject: [PATCH 1663/2974] Resolving Comments

---
 scrapy/http/request/form.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 238dd44b3..d033a830e 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -170,11 +170,8 @@ def _get_clickable(clickdata, form):
     """
     clickables = [
         el for el in form.xpath(
-            'descendant::*[(self::input or self::button)'
-            ' and re:test(@type, "^submit$", "i")]'
-            '|descendant::*[(self::input or self::button)'
-            ' and re:test(@type, "^image$", "i")]'
-            '|descendant::button[not(@type)]',
+            'descendant::input[re.test(@type, "^(submit|image)$", "i")]'
+            '|descendant::button[not(@type) or re.test(@type, "^submit$", "i")]',
             namespaces={"re": "http://exslt.org/regular-expressions"})
         ]
     if not clickables:

From dd064413a46356940151ac3f9ccd8a45bca2cbd8 Mon Sep 17 00:00:00 2001
From: Viral Mehta <virmht@gmail.com>
Date: Mon, 19 Mar 2018 19:28:41 +0530
Subject: [PATCH 1664/2974] corrected syntax error in XPath

---
 scrapy/http/request/form.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index d033a830e..95b38e990 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -170,8 +170,8 @@ def _get_clickable(clickdata, form):
     """
     clickables = [
         el for el in form.xpath(
-            'descendant::input[re.test(@type, "^(submit|image)$", "i")]'
-            '|descendant::button[not(@type) or re.test(@type, "^submit$", "i")]',
+            'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
+            '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
             namespaces={"re": "http://exslt.org/regular-expressions"})
         ]
     if not clickables:

From c6d20bdd826070c6c808421cd6331b5d208f8aa7 Mon Sep 17 00:00:00 2001
From: Steven Almeroth <sroth77@gmail.com>
Date: Tue, 27 Mar 2018 16:21:07 -0400
Subject: [PATCH 1665/2974] Doc: update wording for COOKIES_ENABLED

---
 docs/topics/downloader-middleware.rst | 20 +++++++++-----------
 docs/topics/request-response.rst      |  4 +++-
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 863620900..dfe4c13b4 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -158,13 +158,13 @@ more of the following methods:
       :type spider: :class:`~scrapy.spiders.Spider` object
 
    .. method:: from_crawler(cls, crawler)
-    
+
       If present, this classmethod is called to create a middleware instance
       from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
       of the middleware. Crawler object provides access to all Scrapy core
       components like settings and signals; it is a way for middleware to
       access them and hook its functionality into Scrapy.
-   
+
       :param crawler: crawler that uses this middleware
       :type crawler: :class:`~scrapy.crawler.Crawler` object
 
@@ -237,16 +237,14 @@ Default: ``True``
 Whether to enable the cookies middleware. If disabled, no cookies will be sent
 to web servers.
 
-Notice that if the :class:`~scrapy.http.Request` 
-has ``meta['dont_merge_cookies']`` evaluated to ``True``. 
-despite the value of :setting:`COOKIES_ENABLED` the cookies will **not** be 
-sent to web servers and received cookies in 
-:class:`~scrapy.http.Response` will **not** be merged with the existing 
-cookies.
-
-For more detailed information see the ``cookies`` parameter in 
-:class:`~scrapy.http.Request`
+Notice that despite the value of :setting:`COOKIES_ENABLED` setting if
+``Request.``:reqmeta:`meta['dont_merge_cookies'] <dont_merge_cookies>`
+evaluates to ``True`` the request cookies will **not** be sent to the
+web server and received cookies in :class:`~scrapy.http.Response` will
+**not** be merged with the existing cookies.
 
+For more detailed information see the ``cookies`` parameter in
+:class:`~scrapy.http.Request`.
 
 .. setting:: COOKIES_DEBUG
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 121abe6b5..e29914dbf 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -80,6 +80,8 @@ Request objects
         attributes of the cookie. This is only useful if the cookies are saved
         for later requests.
 
+        .. reqmeta:: dont_merge_cookies
+
         When some site returns cookies (in a response) those are stored in the
         cookies for that domain and will be sent again in future requests. That's
         the typical behaviour of any regular web browser. However, if, for some
@@ -294,7 +296,7 @@ Those are:
 * :reqmeta:`dont_retry`
 * :reqmeta:`handle_httpstatus_list`
 * :reqmeta:`handle_httpstatus_all`
-* ``dont_merge_cookies`` (see ``cookies`` parameter of :class:`Request` constructor)
+* :reqmeta:`dont_merge_cookies`
 * :reqmeta:`cookiejar`
 * :reqmeta:`dont_cache`
 * :reqmeta:`redirect_urls`

From 8e8994c6b55fa8e975ce30b25ae326f829a58aed Mon Sep 17 00:00:00 2001
From: rhoboro <rhoboro@gmail.com>
Date: Mon, 2 Apr 2018 15:36:47 +0900
Subject: [PATCH 1666/2974] add acl support for gcs

---
 scrapy/pipelines/files.py           | 6 +++++-
 scrapy/pipelines/images.py          | 1 +
 scrapy/settings/default_settings.py | 2 ++
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index ab18a727d..af1d5488a 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -207,6 +207,8 @@ class GCSFilesStore(object):
     GCS_PROJECT_ID = None
 
     CACHE_CONTROL = 'max-age=172800'
+    POLICY = 'projectPrivate'  # Overriden from settings.FILES_STORE_GCS_ACL in
+                               # FilesPipeline.from_settings.
 
     def __init__(self, uri):
         from google.cloud import storage
@@ -239,7 +241,8 @@ class GCSFilesStore(object):
         return threads.deferToThread(
             blob.upload_from_string,
             data=buf.getvalue(),
-            content_type=self._get_content_type(headers)
+            content_type=self._get_content_type(headers),
+            predefined_acl=self.POLICY
         )
 
 
@@ -314,6 +317,7 @@ class FilesPipeline(MediaPipeline):
 
         gcs_store = cls.STORE_SCHEMES['gs']
         gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
+        gcs_store.POLICY = settings['FILES_STORE_GCS_ACL']
 
         store_uri = settings['FILES_STORE']
         return cls(store_uri, settings=settings)
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index c5fc12afe..5cdddce49 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -93,6 +93,7 @@ class ImagesPipeline(FilesPipeline):
 
         gcs_store = cls.STORE_SCHEMES['gs']
         gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
+        gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL']
 
         store_uri = settings['IMAGES_STORE']
         return cls(store_uri, settings=settings)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index ead511473..7916b9704 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -159,6 +159,7 @@ FEED_EXPORTERS_BASE = {
 FEED_EXPORT_INDENT = 0
 
 FILES_STORE_S3_ACL = 'private'
+FILES_STORE_GCS_ACL = 'projectPrivate'
 
 FTP_USER = 'anonymous'
 FTP_PASSWORD = 'guest'
@@ -181,6 +182,7 @@ HTTPPROXY_ENABLED = True
 HTTPPROXY_AUTH_ENCODING = 'latin-1'
 
 IMAGES_STORE_S3_ACL = 'private'
+IMAGES_STORE_GCS_ACL = 'projectPrivate'
 
 ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
 

From 5254ac393bdf712db698bfa41caf5fb2e682f883 Mon Sep 17 00:00:00 2001
From: rhoboro <rhoboro@gmail.com>
Date: Tue, 3 Apr 2018 18:00:08 +0900
Subject: [PATCH 1667/2974] added test for gcs policy

---
 scrapy/utils/test.py         | 3 ++-
 tests/test_pipeline_files.py | 5 ++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 60b931f48..4b935c51b 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -57,8 +57,9 @@ def get_gcs_content_and_delete(bucket, path):
     bucket = client.get_bucket(bucket)
     blob = bucket.get_blob(path)
     content = blob.download_as_string()
+    acl = list(blob.acl)  # loads acl before it will be deleted
     bucket.delete_blob(path)
-    return content, blob
+    return content, acl, blob
 
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index c761bd606..728a74803 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -388,17 +388,20 @@ class TestGCSFilesStore(unittest.TestCase):
         meta = {'foo': 'bar'}
         path = 'full/filename'
         store = GCSFilesStore(uri)
+        store.POLICY = 'authenticatedRead'
+        expected_policy = {'role': 'READER', 'entity': 'allAuthenticatedUsers'}
         yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
         s = yield store.stat_file(path, info=None)
         self.assertIn('last_modified', s)
         self.assertIn('checksum', s)
         self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==')
         u = urlparse(uri)
-        content, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path)
+        content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path)
         self.assertEqual(content, data)
         self.assertEqual(blob.metadata, {'foo': 'bar'})
         self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL)
         self.assertEqual(blob.content_type, 'application/octet-stream')
+        self.assertIn(expected_policy, acl)
 
 
 class ItemWithFiles(Item):

From 74a9c65290888b3ce712c3cfa7132f91ffa3c576 Mon Sep 17 00:00:00 2001
From: rhoboro <rhoboro@gmail.com>
Date: Tue, 3 Apr 2018 18:20:37 +0900
Subject: [PATCH 1668/2974] update docs for support gcs acl

---
 docs/topics/media-pipeline.rst | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 41beebe98..284ec1a25 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -189,6 +189,8 @@ Google Cloud Storage
 ---------------------
 
 .. setting:: GCS_PROJECT_ID
+.. setting:: FILES_STORE_GCS_ACL
+.. setting:: IMAGES_STORE_GCS_ACL
 
 :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage
 bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ )
@@ -204,6 +206,18 @@ For information about authentication, see this `documentation`_.
 
 .. _documentation: https://cloud.google.com/docs/authentication/production
 
+You can modify the Access Control List (ACL) policy used for the stored files,
+which is defined by the :setting:`FILES_STORE_GCS_ACL` and
+:setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to
+``projectPrivate``. To make the files publicly available use the ``publicRead``
+policy::
+
+    IMAGES_STORE_GCS_ACL = 'publicRead'
+
+For more information, see `Predefined ACLs`_ in the Google Cloud Platform Developer Guide.
+
+.. _Predefined ACLs: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
+
 Usage example
 =============
 

From cb76b88331e1e0cff30de9a6961de3e28e94ff44 Mon Sep 17 00:00:00 2001
From: grammy-jiang <grammy.jiang@gmail.com>
Date: Wed, 4 Apr 2018 05:56:05 -0400
Subject: [PATCH 1669/2974] fix a mistake in topic spider-middleware.rst

---
 docs/topics/spider-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index c297ed556..1d451af21 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -116,7 +116,7 @@ following methods:
         method (from other spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
-        iterable of :class:`~scrapy.http.Response`, dict or
+        iterable of :class:`~scrapy.http.Request`, dict or
         :class:`~scrapy.item.Item` objects.
 
         If it returns ``None``, Scrapy will continue processing this exception,

From 464973489e9f457570d7dd1a2e4aab8c4c1778fd Mon Sep 17 00:00:00 2001
From: rhoboro <rhoboro@gmail.com>
Date: Fri, 13 Apr 2018 12:06:39 +0900
Subject: [PATCH 1670/2974] Using bucket's default object ACL

---
 docs/topics/media-pipeline.rst      | 3 ++-
 scrapy/pipelines/files.py           | 6 ++++--
 scrapy/settings/default_settings.py | 4 ++--
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 284ec1a25..0872ac0cd 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -209,7 +209,8 @@ For information about authentication, see this `documentation`_.
 You can modify the Access Control List (ACL) policy used for the stored files,
 which is defined by the :setting:`FILES_STORE_GCS_ACL` and
 :setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to
-``projectPrivate``. To make the files publicly available use the ``publicRead``
+None which means that Cloud Storage applies the bucket's default object ACL to the object.
+To make the files publicly available use the ``publicRead``
 policy::
 
     IMAGES_STORE_GCS_ACL = 'publicRead'
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index af1d5488a..8ea70e5d1 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -207,8 +207,10 @@ class GCSFilesStore(object):
     GCS_PROJECT_ID = None
 
     CACHE_CONTROL = 'max-age=172800'
-    POLICY = 'projectPrivate'  # Overriden from settings.FILES_STORE_GCS_ACL in
-                               # FilesPipeline.from_settings.
+
+    # The bucket's default object ACL will be applied to the object.
+    # Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
+    POLICY = None
 
     def __init__(self, uri):
         from google.cloud import storage
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 7916b9704..d7ca8a835 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -159,7 +159,7 @@ FEED_EXPORTERS_BASE = {
 FEED_EXPORT_INDENT = 0
 
 FILES_STORE_S3_ACL = 'private'
-FILES_STORE_GCS_ACL = 'projectPrivate'
+FILES_STORE_GCS_ACL = None
 
 FTP_USER = 'anonymous'
 FTP_PASSWORD = 'guest'
@@ -182,7 +182,7 @@ HTTPPROXY_ENABLED = True
 HTTPPROXY_AUTH_ENCODING = 'latin-1'
 
 IMAGES_STORE_S3_ACL = 'private'
-IMAGES_STORE_GCS_ACL = 'projectPrivate'
+IMAGES_STORE_GCS_ACL = None
 
 ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
 

From 560ee623fd84c3db986fccacaef7e9a31a8ea02c Mon Sep 17 00:00:00 2001
From: rhoboro <rhoboro@gmail.com>
Date: Fri, 13 Apr 2018 19:00:27 +0900
Subject: [PATCH 1671/2974] set defalut value "" to FILES_STORE_GCS_ACL

---
 scrapy/pipelines/files.py           | 2 +-
 scrapy/pipelines/images.py          | 2 +-
 scrapy/settings/default_settings.py | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 8ea70e5d1..510cc23c7 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -319,7 +319,7 @@ class FilesPipeline(MediaPipeline):
 
         gcs_store = cls.STORE_SCHEMES['gs']
         gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
-        gcs_store.POLICY = settings['FILES_STORE_GCS_ACL']
+        gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] or None
 
         store_uri = settings['FILES_STORE']
         return cls(store_uri, settings=settings)
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 5cdddce49..95323c613 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -93,7 +93,7 @@ class ImagesPipeline(FilesPipeline):
 
         gcs_store = cls.STORE_SCHEMES['gs']
         gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
-        gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL']
+        gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] or None
 
         store_uri = settings['IMAGES_STORE']
         return cls(store_uri, settings=settings)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index d7ca8a835..36e17ef6b 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -159,7 +159,7 @@ FEED_EXPORTERS_BASE = {
 FEED_EXPORT_INDENT = 0
 
 FILES_STORE_S3_ACL = 'private'
-FILES_STORE_GCS_ACL = None
+FILES_STORE_GCS_ACL = ''
 
 FTP_USER = 'anonymous'
 FTP_PASSWORD = 'guest'
@@ -182,7 +182,7 @@ HTTPPROXY_ENABLED = True
 HTTPPROXY_AUTH_ENCODING = 'latin-1'
 
 IMAGES_STORE_S3_ACL = 'private'
-IMAGES_STORE_GCS_ACL = None
+IMAGES_STORE_GCS_ACL = ''
 
 ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
 

From 6ef6585b5a187d4a8dcec99ba7cae9b6cee91b30 Mon Sep 17 00:00:00 2001
From: rhoboro <rhoboro@gmail.com>
Date: Fri, 13 Apr 2018 19:06:29 +0900
Subject: [PATCH 1672/2974] update docs

---
 docs/topics/media-pipeline.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 0872ac0cd..a1f518cbd 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -209,7 +209,7 @@ For information about authentication, see this `documentation`_.
 You can modify the Access Control List (ACL) policy used for the stored files,
 which is defined by the :setting:`FILES_STORE_GCS_ACL` and
 :setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to
-None which means that Cloud Storage applies the bucket's default object ACL to the object.
+``''`` (empty string) which means that Cloud Storage applies the bucket's default object ACL to the object.
 To make the files publicly available use the ``publicRead``
 policy::
 

From 57b0e6b6955705efcbe2d3da5501b27716ef1baf Mon Sep 17 00:00:00 2001
From: Lucy Wang <wxitb2017@gmail.com>
Date: Thu, 19 Apr 2018 13:35:46 +0800
Subject: [PATCH 1673/2974] improve document about functions as processors

---
 docs/topics/loaders.rst | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index ad86dba63..cdb066e57 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -136,6 +136,20 @@ accept one (and only one) positional argument, which will be an iterator.
    containing the collected values (for that field). The result of the output
    processors is the value that will be finally assigned to the item.
 
+If you want to use a plain function as a processor, make sure it receives
+``self`` as the first argument::
+
+    def lowercase_processor(self, values):
+        for v in values:
+            yield v.lower()
+
+    class MyItemLoader(ItemLoader):
+        name_in = lowercase_processor
+
+This is because whenever a function is assigned as a class variable, it becomes
+a method and would be passed the instance as the the first argument when being
+called. See `this answer on stackoverflow`_ for more details.
+
 The other thing you need to keep in mind is that the values returned by input
 processors are collected internally (in lists) and then passed to output
 processors to populate the fields.
@@ -143,6 +157,7 @@ processors to populate the fields.
 Last, but not least, Scrapy comes with some :ref:`commonly used processors
 <topics-loaders-available-processors>` built-in for convenience.
 
+.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635
 
 Declaring Item Loaders
 ======================

From e75f721c04446f8f28d3bdfcd69f967f65981407 Mon Sep 17 00:00:00 2001
From: Pengyu Chen <pengyu@libstarrify.so>
Date: Mon, 23 Apr 2018 22:08:28 +0800
Subject: [PATCH 1674/2974] Added: Allowing optional arguments for
 `scrapy.http.cookies.CookieJar.clear`

---
 scrapy/http/cookies.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index a1e95102e..4e8056750 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -58,8 +58,8 @@ class CookieJar(object):
     def clear_session_cookies(self, *args, **kwargs):
         return self.jar.clear_session_cookies(*args, **kwargs)
 
-    def clear(self):
-        return self.jar.clear()
+    def clear(self, domain=None, path=None, name=None):
+        return self.jar.clear(domain, path, name)
 
     def __iter__(self):
         return iter(self.jar)

From 0d015e5c0f0dcb8936044aec163dd12e33482730 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Wed, 16 May 2018 09:36:07 +0000
Subject: [PATCH 1675/2974] blacklist twisted version with regression

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 2a94d742d..7d857a8c9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-Twisted>=13.1.0
+Twisted>=13.1.0,!=18.4.0
 lxml
 pyOpenSSL
 cssselect>=0.9

From c5ddfddb7e5b42e45b6c8f86a381ee52fbb6f797 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Thu, 17 May 2018 08:53:42 +0000
Subject: [PATCH 1676/2974] blacklist twisted version with regression in
 constraints file

---
 requirements.txt      | 2 +-
 tests/constraints.txt | 1 +
 tox.ini               | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)
 create mode 100644 tests/constraints.txt

diff --git a/requirements.txt b/requirements.txt
index 7d857a8c9..2a94d742d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-Twisted>=13.1.0,!=18.4.0
+Twisted>=13.1.0
 lxml
 pyOpenSSL
 cssselect>=0.9
diff --git a/tests/constraints.txt b/tests/constraints.txt
new file mode 100644
index 000000000..3bc30de15
--- /dev/null
+++ b/tests/constraints.txt
@@ -0,0 +1 @@
+Twisted!=18.4.0
diff --git a/tox.ini b/tox.ini
index 60ff8c15e..c2fa9af28 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,6 +8,7 @@ envlist = py27
 
 [testenv]
 deps =
+    -ctests/constraints.txt
     -rrequirements.txt
     # Extras
     botocore

From 2dfc5d128bba42e2fe2bb24c0326fe47b0d3cd97 Mon Sep 17 00:00:00 2001
From: Ryan P Kilby <rpkilby@ncsu.edu>
Date: Wed, 9 May 2018 11:59:38 -0400
Subject: [PATCH 1677/2974] Update DEPTH_STATS refs to DEPTH_STATS_VERBOSE

---
 docs/topics/settings.rst            | 11 -----------
 docs/topics/spider-middleware.rst   |  3 ++-
 scrapy/settings/default_settings.py |  2 +-
 3 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 076dc6bfd..1f1217770 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -335,17 +335,6 @@ See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO.
     other priority settings :setting:`REDIRECT_PRIORITY_ADJUST`
     and :setting:`RETRY_PRIORITY_ADJUST`.
 
-.. setting:: DEPTH_STATS
-
-DEPTH_STATS
------------
-
-Default: ``True``
-
-Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
-
-Whether to collect maximum depth stats.
-
 .. setting:: DEPTH_STATS_VERBOSE
 
 DEPTH_STATS_VERBOSE
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index c297ed556..265acdb43 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -212,7 +212,8 @@ DepthMiddleware
 
       * :setting:`DEPTH_LIMIT` - The maximum depth that will be allowed to
         crawl for any site. If zero, no limit will be imposed.
-      * :setting:`DEPTH_STATS` - Whether to collect depth stats.
+      * :setting:`DEPTH_STATS_VERBOSE` - Whether to collect the number of
+        requests for each depth.
       * :setting:`DEPTH_PRIORITY` - Whether to prioritize the requests based on
         their depth.
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 36e17ef6b..ca004aedd 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -55,7 +55,7 @@ DEFAULT_REQUEST_HEADERS = {
 }
 
 DEPTH_LIMIT = 0
-DEPTH_STATS = True
+DEPTH_STATS_VERBOSE = False
 DEPTH_PRIORITY = 0
 
 DNSCACHE_ENABLED = True

From 6a182c955273745daf334033944c82da3aa4eb12 Mon Sep 17 00:00:00 2001
From: Ryan P Kilby <rpkilby@ncsu.edu>
Date: Wed, 9 May 2018 12:00:18 -0400
Subject: [PATCH 1678/2974] Depth stats are not optional

---
 scrapy/spidermiddlewares/depth.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index e2f039146..34a87f2df 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
 
 class DepthMiddleware(object):
 
-    def __init__(self, maxdepth, stats=None, verbose_stats=False, prio=1):
+    def __init__(self, maxdepth, stats, verbose_stats=False, prio=1):
         self.maxdepth = maxdepth
         self.stats = stats
         self.verbose_stats = verbose_stats
@@ -41,7 +41,7 @@ class DepthMiddleware(object):
                         extra={'spider': spider}
                     )
                     return False
-                elif self.stats:
+                else:
                     if self.verbose_stats:
                         self.stats.inc_value('request_depth_count/%s' % depth,
                                              spider=spider)
@@ -50,7 +50,7 @@ class DepthMiddleware(object):
             return True
 
         # base case (depth=0)
-        if self.stats and 'depth' not in response.meta:
+        if 'depth' not in response.meta:
             response.meta['depth'] = 0
             if self.verbose_stats:
                 self.stats.inc_value('request_depth_count/0', spider=spider)

From b364d27247b2d9b86c164569c7e0459fa3f8391b Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Wed, 23 May 2018 21:25:50 +0300
Subject: [PATCH 1679/2974] [MRG+1] Automatic port selection for servicies in
 unit tests (#3210)

* ability to pass port as a parameter

* try to find free ports

* use environment variables to pass mock server address

* get mock server address from environment variables

* ability to select ports for proxy in runtime

* use common method for URLs from mock server

* https support

* get mock server address

* get mock address

* replace hand-written mechanism by kernel-based one

* use ephemeral ports in mockserver

* strip EOL from addresses

* use ephemeral port in proxy

* no need to restore environment as it is restored in tearDown

* decode bytes

* use mockserver address as a variable

* ability to pass address as variable

* per test-case mockserver

* use base class

* remove obsolete environment manipulation

* return usage of proxy for http cases

* common method for broking proxy auth credentials

* python version-independent url methods
---
 tests/mockserver.py                      | 25 +++++++++---
 tests/spiders.py                         | 17 ++++++---
 tests/test_closespider.py                |  8 ++--
 tests/test_crawl.py                      | 48 ++++++++++++------------
 tests/test_downloader_handlers.py        |  8 ++--
 tests/test_feedexport.py                 |  3 +-
 tests/test_pipeline_crawl.py             | 16 ++++----
 tests/test_proxy_connect.py              | 41 ++++++++++++--------
 tests/test_spidermiddleware_httperror.py | 23 ++++++------
 9 files changed, 111 insertions(+), 78 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 98723846e..f36ce3c44 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -192,9 +192,15 @@ class MockServer():
 
     def __enter__(self):
         from scrapy.utils.test import get_testenv
+
         self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'],
                           stdout=PIPE, env=get_testenv())
-        self.proc.stdout.readline()
+        http_address = self.proc.stdout.readline().strip().decode('ascii')
+        https_address = self.proc.stdout.readline().strip().decode('ascii')
+
+        self.http_address = http_address
+        self.https_address = https_address
+
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
@@ -202,6 +208,12 @@ class MockServer():
         self.proc.wait()
         time.sleep(0.2)
 
+    def url(self, path, is_secure=False):
+        host = self.http_address
+        if is_secure:
+            host = self.https_address
+        return host + path
+
 
 def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt'):
     return ssl.DefaultOpenSSLContextFactory(
@@ -213,14 +225,17 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c
 if __name__ == "__main__":
     root = Root()
     factory = Site(root)
-    httpPort = reactor.listenTCP(8998, factory)
+    httpPort = reactor.listenTCP(0, factory)
     contextFactory = ssl_context_factory()
-    httpsPort = reactor.listenSSL(8999, factory, contextFactory)
+    httpsPort = reactor.listenSSL(0, factory, contextFactory)
 
     def print_listening():
         httpHost = httpPort.getHost()
         httpsHost = httpsPort.getHost()
-        print("Mock server running at http://%s:%d and https://%s:%d" % (
-            httpHost.host, httpHost.port, httpsHost.host, httpsHost.port))
+        httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port)
+        httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port)
+        print(httpAddress)
+        print(httpsAddress)
+
     reactor.callWhenRunning(print_listening)
     reactor.run()
diff --git a/tests/spiders.py b/tests/spiders.py
index 1038b69de..7816bf7c7 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -11,7 +11,12 @@ from scrapy.item import Item
 from scrapy.linkextractors import LinkExtractor
 
 
-class MetaSpider(Spider):
+class MockServerSpider(Spider):
+    def __init__(self, mockserver=None, *args, **kwargs):
+        super(MockServerSpider, self).__init__(*args, **kwargs)
+        self.mockserver = mockserver
+
+class MetaSpider(MockServerSpider):
 
     name = 'meta'
 
@@ -33,7 +38,7 @@ class FollowAllSpider(MetaSpider):
         self.urls_visited = []
         self.times = []
         qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency}
-        url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1)
+        url = self.mockserver.url("/follow?%s" % urlencode(qargs, doseq=1))
         self.start_urls = [url]
 
     def parse(self, response):
@@ -55,7 +60,7 @@ class DelaySpider(MetaSpider):
 
     def start_requests(self):
         self.t1 = time.time()
-        url = "http://localhost:8998/delay?n=%s&b=%s" % (self.n, self.b)
+        url = self.mockserver.url("/delay?n=%s&b=%s" % (self.n, self.b))
         yield Request(url, callback=self.parse, errback=self.errback)
 
     def parse(self, response):
@@ -121,7 +126,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
 
         for s in range(100):
             qargs = {'total': 10, 'seed': s}
-            url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1)
+            url = self.mockserver.url("/follow?%s") % urlencode(qargs, doseq=1)
             yield Request(url, meta={'seed': s})
             if self.fail_yielding:
                 2 / 0
@@ -160,7 +165,7 @@ class SingleRequestSpider(MetaSpider):
             return self.errback_func(failure)
 
 
-class DuplicateStartRequestsSpider(Spider):
+class DuplicateStartRequestsSpider(MockServerSpider):
     dont_filter = True
     name = 'duplicatestartrequests'
     distinct_urls = 2
@@ -169,7 +174,7 @@ class DuplicateStartRequestsSpider(Spider):
     def start_requests(self):
         for i in range(0, self.distinct_urls):
             for j in range(0, self.dupe_factor):
-                url = "http://localhost:8998/echo?headers=1&body=test%d" % i
+                url = self.mockserver.url("/echo?headers=1&body=test%d" % i)
                 yield Request(url, dont_filter=self.dont_filter)
 
     def __init__(self, url="http://localhost:8998", *args, **kwargs):
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index fa0b48998..0eb1b7944 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -18,7 +18,7 @@ class TestCloseSpider(TestCase):
     def test_closespider_itemcount(self):
         close_on = 5
         crawler = get_crawler(ItemSpider, {'CLOSESPIDER_ITEMCOUNT': close_on})
-        yield crawler.crawl()
+        yield crawler.crawl(mockserver=self.mockserver)
         reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_itemcount')
         itemcount = crawler.stats.get_value('item_scraped_count')
@@ -28,7 +28,7 @@ class TestCloseSpider(TestCase):
     def test_closespider_pagecount(self):
         close_on = 5
         crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_PAGECOUNT': close_on})
-        yield crawler.crawl()
+        yield crawler.crawl(mockserver=self.mockserver)
         reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_pagecount')
         pagecount = crawler.stats.get_value('response_received_count')
@@ -38,7 +38,7 @@ class TestCloseSpider(TestCase):
     def test_closespider_errorcount(self):
         close_on = 5
         crawler = get_crawler(ErrorSpider, {'CLOSESPIDER_ERRORCOUNT': close_on})
-        yield crawler.crawl(total=1000000)
+        yield crawler.crawl(total=1000000, mockserver=self.mockserver)
         reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_errorcount')
         key = 'spider_exceptions/{name}'\
@@ -50,7 +50,7 @@ class TestCloseSpider(TestCase):
     def test_closespider_timeout(self):
         close_on = 0.1
         crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_TIMEOUT': close_on})
-        yield crawler.crawl(total=1000000)
+        yield crawler.crawl(total=1000000, mockserver=self.mockserver)
         reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_timeout')
         stats = crawler.stats
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index d5babdded..3fc13eeb7 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -26,7 +26,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_follow_all(self):
         crawler = self.runner.create_crawler(FollowAllSpider)
-        yield crawler.crawl()
+        yield crawler.crawl(mockserver=self.mockserver)
         self.assertEqual(len(crawler.spider.urls_visited), 11)  # 10 + start_url
 
     @defer.inlineCallbacks
@@ -42,7 +42,7 @@ class CrawlTestCase(TestCase):
     def _test_delay(self, delay, randomize):
         settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize}
         crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
-        yield crawler.crawl(maxlatency=delay * 2)
+        yield crawler.crawl(maxlatency=delay * 2, mockserver=self.mockserver)
         t = crawler.spider.times
         totaltime = t[-1] - t[0]
         avgd = totaltime / (len(t) - 1)
@@ -53,7 +53,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_timeout_success(self):
         crawler = self.runner.create_crawler(DelaySpider)
-        yield crawler.crawl(n=0.5)
+        yield crawler.crawl(n=0.5, mockserver=self.mockserver)
         self.assertTrue(crawler.spider.t1 > 0)
         self.assertTrue(crawler.spider.t2 > 0)
         self.assertTrue(crawler.spider.t2 > crawler.spider.t1)
@@ -61,13 +61,13 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_timeout_failure(self):
         crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider)
-        yield crawler.crawl(n=0.5)
+        yield crawler.crawl(n=0.5, mockserver=self.mockserver)
         self.assertTrue(crawler.spider.t1 > 0)
         self.assertTrue(crawler.spider.t2 == 0)
         self.assertTrue(crawler.spider.t2_err > 0)
         self.assertTrue(crawler.spider.t2_err > crawler.spider.t1)
         # server hangs after receiving response headers
-        yield crawler.crawl(n=0.5, b=1)
+        yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver)
         self.assertTrue(crawler.spider.t1 > 0)
         self.assertTrue(crawler.spider.t2 == 0)
         self.assertTrue(crawler.spider.t2_err > 0)
@@ -77,14 +77,14 @@ class CrawlTestCase(TestCase):
     def test_retry_503(self):
         crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
-            yield crawler.crawl("http://localhost:8998/status?n=503")
+            yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
         self._assert_retried(l)
 
     @defer.inlineCallbacks
     def test_retry_conn_failed(self):
         crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
-            yield crawler.crawl("http://localhost:65432/status?n=503")
+            yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver)
         self._assert_retried(l)
 
     @defer.inlineCallbacks
@@ -92,14 +92,14 @@ class CrawlTestCase(TestCase):
         crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
             # try to fetch the homepage of a non-existent domain
-            yield crawler.crawl("http://dns.resolution.invalid./")
+            yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)
         self._assert_retried(l)
 
     @defer.inlineCallbacks
     def test_start_requests_bug_before_yield(self):
         with LogCapture('scrapy', level=logging.ERROR) as l:
             crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
-            yield crawler.crawl(fail_before_yield=1)
+            yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
 
         self.assertEqual(len(l.records), 1)
         record = l.records[0]
@@ -110,7 +110,7 @@ class CrawlTestCase(TestCase):
     def test_start_requests_bug_yielding(self):
         with LogCapture('scrapy', level=logging.ERROR) as l:
             crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
-            yield crawler.crawl(fail_yielding=1)
+            yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
 
         self.assertEqual(len(l.records), 1)
         record = l.records[0]
@@ -121,7 +121,7 @@ class CrawlTestCase(TestCase):
     def test_start_requests_lazyness(self):
         settings = {"CONCURRENT_REQUESTS": 1}
         crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider)
-        yield crawler.crawl()
+        yield crawler.crawl(mockserver=self.mockserver)
         #self.assertTrue(False, crawler.spider.seedsseen)
         #self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
         #                crawler.spider.seedsseen)
@@ -130,10 +130,10 @@ class CrawlTestCase(TestCase):
     def test_start_requests_dupes(self):
         settings = {"CONCURRENT_REQUESTS": 1}
         crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider)
-        yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3)
+        yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver)
         self.assertEqual(crawler.spider.visited, 6)
 
-        yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4)
+        yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4, mockserver=self.mockserver)
         self.assertEqual(crawler.spider.visited, 3)
 
     @defer.inlineCallbacks
@@ -160,7 +160,7 @@ with multiples lines
 '''})
         crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
-            yield crawler.crawl("http://localhost:8998/raw?{0}".format(query))
+            yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver)
         self.assertEqual(str(l).count("Got response 200"), 1)
 
     @defer.inlineCallbacks
@@ -168,7 +168,7 @@ with multiples lines
         # connection lost after receiving data
         crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
-            yield crawler.crawl("http://localhost:8998/drop?abort=0")
+            yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver)
         self._assert_retried(l)
 
     @defer.inlineCallbacks
@@ -176,7 +176,7 @@ with multiples lines
         # connection lost before receiving data
         crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
-            yield crawler.crawl("http://localhost:8998/drop?abort=1")
+            yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver)
         self._assert_retried(l)
 
     def _assert_retried(self, log):
@@ -186,7 +186,7 @@ with multiples lines
     @defer.inlineCallbacks
     def test_referer_header(self):
         """Referer header is set by RefererMiddleware unless it is already set"""
-        req0 = Request('http://localhost:8998/echo?headers=1&body=0', dont_filter=1)
+        req0 = Request(self.mockserver.url('/echo?headers=1&body=0'), dont_filter=1)
         req1 = req0.replace()
         req2 = req0.replace(headers={'Referer': None})
         req3 = req0.replace(headers={'Referer': 'http://example.com'})
@@ -194,7 +194,7 @@ with multiples lines
         req1.meta['next'] = req2
         req2.meta['next'] = req3
         crawler = self.runner.create_crawler(SingleRequestSpider)
-        yield crawler.crawl(seed=req0)
+        yield crawler.crawl(seed=req0, mockserver=self.mockserver)
         # basic asserts in case of weird communication errors
         self.assertIn('responses', crawler.spider.meta)
         self.assertNotIn('failures', crawler.spider.meta)
@@ -220,7 +220,7 @@ with multiples lines
             est.append(get_engine_status(crawler.engine))
 
         crawler = self.runner.create_crawler(SingleRequestSpider)
-        yield crawler.crawl(seed='http://localhost:8998/', callback_func=cb)
+        yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)
         self.assertEqual(len(est), 1, est)
         s = dict(est[0])
         self.assertEqual(s['engine.spider.name'], crawler.spider.name)
@@ -244,7 +244,7 @@ with multiples lines
                 raise TestError
 
         crawler = self.runner.create_crawler(FaultySpider)
-        yield self.assertFailure(crawler.crawl(), TestError)
+        yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError)
         self.assertFalse(crawler.crawling)
 
     @defer.inlineCallbacks
@@ -256,7 +256,7 @@ with multiples lines
         }
         crawler = CrawlerRunner(settings).create_crawler(SimpleSpider)
         yield self.assertFailure(
-            self.runner.crawl(crawler, "http://localhost:8998/status?n=200"),
+            self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver),
             ZeroDivisionError)
         self.assertFalse(crawler.crawling)
 
@@ -264,13 +264,13 @@ with multiples lines
     def test_crawlerrunner_accepts_crawler(self):
         crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as log:
-            yield self.runner.crawl(crawler, "http://localhost:8998/status?n=200")
+            yield self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
         self.assertIn("Got response 200", str(log))
 
     @defer.inlineCallbacks
     def test_crawl_multiple(self):
-        self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=200")
-        self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=503")
+        self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
 
         with LogCapture() as log:
             yield self.runner.join()
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index b34faa7e7..c91be2c0c 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -534,14 +534,14 @@ class Http11MockServerTestCase(unittest.TestCase):
         crawler = get_crawler(SingleRequestSpider)
         # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid
         # download it
-        yield crawler.crawl(seed=Request(url='http://localhost:8998/partial', meta={'download_maxsize': 1000}))
+        yield crawler.crawl(seed=Request(url=self.mockserver.url('/partial'), meta={'download_maxsize': 1000}))
         failure = crawler.spider.meta['failure']
         self.assertIsInstance(failure.value, defer.CancelledError)
 
     @defer.inlineCallbacks
     def test_download(self):
         crawler = get_crawler(SingleRequestSpider)
-        yield crawler.crawl(seed=Request(url='http://localhost:8998'))
+        yield crawler.crawl(seed=Request(url=self.mockserver.url('')))
         failure = crawler.spider.meta.get('failure')
         self.assertTrue(failure == None)
         reason = crawler.spider.meta['close_reason']
@@ -551,7 +551,7 @@ class Http11MockServerTestCase(unittest.TestCase):
     def test_download_gzip_response(self):
         crawler = get_crawler(SingleRequestSpider)
         body = b'1' * 100  # PayloadResource requires body length to be 100
-        request = Request('http://localhost:8998/payload', method='POST',
+        request = Request(self.mockserver.url('/payload'), method='POST',
                           body=body, meta={'download_maxsize': 50})
         yield crawler.crawl(seed=request)
         failure = crawler.spider.meta['failure']
@@ -560,7 +560,7 @@ class Http11MockServerTestCase(unittest.TestCase):
 
         if six.PY2:
             request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
-            request = request.replace(url='http://localhost:8998/xpayload')
+            request = request.replace(url=self.mockserver.url('/xpayload'))
             yield crawler.crawl(seed=request)
             # download_maxsize = 50 is enough for the gzipped response
             failure = crawler.spider.meta.get('failure')
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index f55927121..0d9f1e83c 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -179,6 +179,7 @@ class FeedExportTest(unittest.TestCase):
         try:
             with MockServer() as s:
                 runner = CrawlerRunner(Settings(defaults))
+                spider_cls.start_urls = [s.url('/')]
                 yield runner.crawl(spider_cls)
 
             with open(res_name, 'rb') as f:
@@ -194,7 +195,6 @@ class FeedExportTest(unittest.TestCase):
         """
         class TestSpider(scrapy.Spider):
             name = 'testspider'
-            start_urls = ['http://localhost:8998/']
 
             def parse(self, response):
                 for item in items:
@@ -210,7 +210,6 @@ class FeedExportTest(unittest.TestCase):
         """
         class TestSpider(scrapy.Spider):
             name = 'testspider'
-            start_urls = ['http://localhost:8998/']
 
             def parse(self, response):
                 pass
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 9b81f827d..5985a6f3e 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -46,7 +46,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):
 
     def _process_url(self, url):
         return add_or_replace_parameter(
-                    'http://localhost:8998/redirect-to',
+                    self.mockserver.url('/redirect-to'),
                     'goto', url)
 
 
@@ -134,7 +134,7 @@ class FileDownloadCrawlTestCase(TestCase):
     def test_download_media(self):
         crawler = self._create_crawler(MediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl("http://localhost:8998/files/images/",
+            yield crawler.crawl(self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key)
         self._assert_files_downloaded(self.items, str(log))
@@ -143,7 +143,7 @@ class FileDownloadCrawlTestCase(TestCase):
     def test_download_media_wrong_urls(self):
         crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl("http://localhost:8998/files/images/",
+            yield crawler.crawl(self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key)
         self._assert_files_download_failure(crawler, self.items, 404, str(log))
@@ -152,9 +152,10 @@ class FileDownloadCrawlTestCase(TestCase):
     def test_download_media_redirected_default_failure(self):
         crawler = self._create_crawler(RedirectedMediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl("http://localhost:8998/files/images/",
+            yield crawler.crawl(self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
-                media_urls_key=self.media_urls_key)
+                media_urls_key=self.media_urls_key,
+                mockserver=self.mockserver)
         self._assert_files_download_failure(crawler, self.items, 302, str(log))
 
     @defer.inlineCallbacks
@@ -165,9 +166,10 @@ class FileDownloadCrawlTestCase(TestCase):
 
         crawler = self._create_crawler(RedirectedMediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl("http://localhost:8998/files/images/",
+            yield crawler.crawl(self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
-                media_urls_key=self.media_urls_key)
+                media_urls_key=self.media_urls_key,
+                mockserver=self.mockserver)
         self._assert_files_downloaded(self.items, str(log))
         self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3)
 
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 6213a51e8..ae1236bcb 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -2,6 +2,7 @@ import json
 import os
 import time
 
+from six.moves.urllib.parse import urlsplit, urlunsplit
 from threading import Thread
 from libmproxy import controller, proxy
 from netlib import http_auth
@@ -17,7 +18,7 @@ from tests.mockserver import MockServer
 
 class HTTPSProxy(controller.Master, Thread):
 
-    def __init__(self, port):
+    def __init__(self):
         password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy')
         authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy")
         cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
@@ -25,10 +26,19 @@ class HTTPSProxy(controller.Master, Thread):
         server = proxy.ProxyServer(proxy.ProxyConfig(
             authenticator = authenticator,
             cacert = cert_path),
-            port)
+            0)
+        self.server = server
         Thread.__init__(self)
         controller.Master.__init__(self, server)
 
+    def http_address(self):
+        return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname()
+
+
+def _wrong_credentials(proxy_url):
+    bad_auth_proxy = list(urlsplit(proxy_url))
+    bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')
+    return urlunsplit(bad_auth_proxy)
 
 class ProxyConnectTestCase(TestCase):
 
@@ -36,12 +46,14 @@ class ProxyConnectTestCase(TestCase):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
         self._oldenv = os.environ.copy()
-        self._proxy = HTTPSProxy(8888)
+
+        self._proxy = HTTPSProxy()
         self._proxy.start()
+
         # Wait for the proxy to start.
         time.sleep(1.0)
-        os.environ['http_proxy'] = 'http://scrapy:scrapy@localhost:8888'
-        os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888'
+        os.environ['https_proxy'] = self._proxy.http_address()
+        os.environ['http_proxy'] = self._proxy.http_address()
 
     def tearDown(self):
         self.mockserver.__exit__(None, None, None)
@@ -52,17 +64,17 @@ class ProxyConnectTestCase(TestCase):
     def test_https_connect_tunnel(self):
         crawler = get_crawler(SimpleSpider)
         with LogCapture() as l:
-            yield crawler.crawl("https://localhost:8999/status?n=200")
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, l)
 
     @defer.inlineCallbacks
     def test_https_noconnect(self):
-        os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888?noconnect'
+        proxy = os.environ['https_proxy']
+        os.environ['https_proxy'] = proxy + '?noconnect'
         crawler = get_crawler(SimpleSpider)
         with LogCapture() as l:
-            yield crawler.crawl("https://localhost:8999/status?n=200")
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, l)
-        os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888'
 
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
@@ -73,18 +85,17 @@ class ProxyConnectTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_https_tunnel_auth_error(self):
-        os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888'
+        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
         crawler = get_crawler(SimpleSpider)
         with LogCapture() as l:
-            yield crawler.crawl("https://localhost:8999/status?n=200")
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         # The proxy returns a 407 error code but it does not reach the client;
         # he just sees a TunnelError.
         self._assert_got_tunnel_error(l)
-        os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888'
 
     @defer.inlineCallbacks
     def test_https_tunnel_without_leak_proxy_authorization_header(self):
-        request = Request("https://localhost:8999/echo")
+        request = Request(self.mockserver.url("/echo", is_secure=True))
         crawler = get_crawler(SingleRequestSpider)
         with LogCapture() as l:
             yield crawler.crawl(seed=request)
@@ -94,10 +105,10 @@ class ProxyConnectTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_https_noconnect_auth_error(self):
-        os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888?noconnect'
+        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
         crawler = get_crawler(SimpleSpider)
         with LogCapture() as l:
-            yield crawler.crawl("https://localhost:8999/status?n=200")
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(407, l)
 
     def _assert_got_response_code(self, code, log):
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 19e6bbdcd..dacd0147f 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -11,20 +11,21 @@ from scrapy.http import Response, Request
 from scrapy.spiders import Spider
 from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError
 from scrapy.settings import Settings
+from tests.spiders import MockServerSpider
 
 
-class _HttpErrorSpider(Spider):
+class _HttpErrorSpider(MockServerSpider):
     name = 'httperror'
-    start_urls = [
-        "http://localhost:8998/status?n=200",
-        "http://localhost:8998/status?n=404",
-        "http://localhost:8998/status?n=402",
-        "http://localhost:8998/status?n=500",
-    ]
     bypass_status_codes = set()
 
     def __init__(self, *args, **kwargs):
         super(_HttpErrorSpider, self).__init__(*args, **kwargs)
+        self.start_urls = [
+           self.mockserver.url("/status?n=200"),
+           self.mockserver.url("/status?n=404"),
+           self.mockserver.url("/status?n=402"),
+           self.mockserver.url("/status?n=500"),
+        ]
         self.failed = set()
         self.skipped = set()
         self.parsed = set()
@@ -169,7 +170,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
     @defer.inlineCallbacks
     def test_middleware_works(self):
         crawler = get_crawler(_HttpErrorSpider)
-        yield crawler.crawl()
+        yield crawler.crawl(mockserver=self.mockserver)
         assert not crawler.spider.skipped, crawler.spider.skipped
         self.assertEqual(crawler.spider.parsed, {'200'})
         self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
@@ -184,7 +185,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
     def test_logging(self):
         crawler = get_crawler(_HttpErrorSpider)
         with LogCapture() as log:
-            yield crawler.crawl(bypass_status_codes={402})
+            yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402})
         self.assertEqual(crawler.spider.parsed, {'200', '402'})
         self.assertEqual(crawler.spider.skipped, {'402'})
         self.assertEqual(crawler.spider.failed, {'404', '500'})
@@ -199,7 +200,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
         # HttpError logs ignored responses with level INFO
         crawler = get_crawler(_HttpErrorSpider)
         with LogCapture(level=logging.INFO) as log:
-            yield crawler.crawl()
+            yield crawler.crawl(mockserver=self.mockserver)
         self.assertEqual(crawler.spider.parsed, {'200'})
         self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
 
@@ -211,7 +212,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
         # with level WARNING, we shouldn't capture anything from HttpError
         crawler = get_crawler(_HttpErrorSpider)
         with LogCapture(level=logging.WARNING) as log:
-            yield crawler.crawl()
+            yield crawler.crawl(mockserver=self.mockserver)
         self.assertEqual(crawler.spider.parsed, {'200'})
         self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
 

From ffa7bede17088e5eebd50305704906bf1451ab3a Mon Sep 17 00:00:00 2001
From: Kevin Tewouda <rollandkev@yahoo.fr>
Date: Wed, 30 May 2018 06:33:18 +0200
Subject: [PATCH 1680/2974] Update spiders.rst

I changed URLs to :class:`~scrapy.http.Request` in start_urls explanation of the default spider
---
 docs/topics/spiders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index c2c271245..697732b47 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -88,7 +88,7 @@ scrapy.Spider
 
        A list of URLs where the spider will begin to crawl from, when no
        particular URLs are specified. So, the first pages downloaded will be those
-       listed here. The subsequent URLs will be generated successively from data
+       listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data
        contained in the start URLs.
 
    .. attribute:: custom_settings

From ecdd888ff4614a0e994c9dadb31b7c9c85c88e8a Mon Sep 17 00:00:00 2001
From: Chris Slothouber <cms@dreaming.org>
Date: Fri, 1 Jun 2018 09:25:34 -0400
Subject: [PATCH 1681/2974] Minor edits to contributing.rst

Corrected minor grammatical issues and increased clarity of instructions.
---
 docs/contributing.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index f4f9e393f..6615840f7 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -48,9 +48,9 @@ guidelines when reporting a new bug.
   `Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__
   (use "scrapy" tag).
 
-* check the `open issues`_ to see if it has already been reported. If it has,
-  don't dismiss the report, but check the ticket history and comments. If you
-  have additional useful information, please leave a comment, or consider
+* check the `open issues`_ to see if the issue has already been reported. If it
+  has, don't dismiss the report, but check the ticket history and comments. If 
+  you have additional useful information, please leave a comment, or consider
   :ref:`sending a pull request <writing-patches>` with a fix.
 
 * search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has
@@ -122,7 +122,7 @@ conversation in the `Scrapy subreddit`_ to discuss your idea first.
 Sometimes there is an existing pull request for the problem you'd like to
 solve, which is stalled for some reason. Often the pull request is in a
 right direction, but changes are requested by Scrapy maintainers, and the
-original pull request author haven't had time to address them.
+original pull request author hasn't had time to address them.
 In this case consider picking up this pull request: open
 a new pull request with all commits from the original pull request, as well as
 additional changes to address the raised issues. Doing so helps a lot; it is
@@ -143,7 +143,7 @@ instead of "Fix for #411". Complete titles make it easy to skim through
 the issue tracker.
 
 Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports
-removal, etc) in separate commits than functional changes. This will make pull
+removal, etc) in separate commits from functional changes. This will make pull
 requests easier to review and more likely to get merged.
 
 Coding style
@@ -170,7 +170,7 @@ Documentation policies
   **do** provide a docstring, but make sure sphinx documentation uses
   autodoc_ extension to pull the docstring. For example, the
   :meth:`ItemLoader.add_value` method should be either
-  documented only in the sphinx documentation (not it a docstring), or
+  documented only in the sphinx documentation (not as a docstring), or
   it should have a docstring which is pulled to sphinx documentation using
   autodoc_ extension.
 

From 6a2d2c3b77bde1d74b46d7dbfb9488cb06e5021f Mon Sep 17 00:00:00 2001
From: Fredrik Bergenlid <fredrik.bergenlid@gmail.com>
Date: Fri, 1 Jun 2018 21:38:07 +0200
Subject: [PATCH 1682/2974] Improve gunzip performance for big files

---
 scrapy/utils/gz.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 16c9ce539..ec3949651 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -30,25 +30,25 @@ def gunzip(data):
     This is resilient to CRC checksum errors.
     """
     f = GzipFile(fileobj=BytesIO(data))
-    output = b''
+    output_list = []
     chunk = b'.'
     while chunk:
         try:
             chunk = read1(f, 8196)
-            output += chunk
+            output_list.append(chunk)
         except (IOError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise
             # see issue 87 about catching struct.error
-            # some pages are quite small so output is '' and f.extrabuf
+            # some pages are quite small so output_list is empty and f.extrabuf
             # contains the whole page content
-            if output or getattr(f, 'extrabuf', None):
+            if output_list or getattr(f, 'extrabuf', None):
                 try:
-                    output += f.extrabuf[-f.extrasize:]
+                    output_list.append(f.extrabuf[-f.extrasize:])
                 finally:
                     break
             else:
                 raise
-    return output
+    return b''.join(output_list)
 
 _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
 _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search

From 98d9093dc7241b32b927e79ff2f8a475ae5659a6 Mon Sep 17 00:00:00 2001
From: Colton Herinckx <herinckc@flip3.engr.oregonstate.edu>
Date: Mon, 14 May 2018 13:37:16 -0700
Subject: [PATCH 1683/2974] minor grammatical fixes in CODE_OF_CONDUCT.md

---
 CODE_OF_CONDUCT.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
index 162602248..d477168eb 100644
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -3,7 +3,7 @@
 ## Our Pledge
 
 In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to making participation in our project and
+contributors and maintainers pledge to make participation in our project and
 our community a harassment-free experience for everyone, regardless of age, body
 size, disability, ethnicity, gender identity and expression, level of experience,
 nationality, personal appearance, race, religion, or sexual identity and

From 9bd5444a42e82a05d2791fd10a205a9dd99303a6 Mon Sep 17 00:00:00 2001
From: Colton Herinckx <herinckc@flip3.engr.oregonstate.edu>
Date: Mon, 14 May 2018 13:48:28 -0700
Subject: [PATCH 1684/2974] added oxford commas to LICENSE

---
 LICENSE | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/LICENSE b/LICENSE
index 6ead05ece..4d0a0863a 100644
--- a/LICENSE
+++ b/LICENSE
@@ -5,10 +5,10 @@ Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:
 
     1. Redistributions of source code must retain the above copyright notice,
-       this list of conditions and the following disclaimer.
+       this list of conditions, and the following disclaimer.
 
     2. Redistributions in binary form must reproduce the above copyright
-       notice, this list of conditions and the following disclaimer in the
+       notice, this list of conditions, and the following disclaimer in the
        documentation and/or other materials provided with the distribution.
 
     3. Neither the name of Scrapy nor the names of its contributors may be used

From 12d10eec2cb10fb13cf199d73363093478e80f9a Mon Sep 17 00:00:00 2001
From: Colton Herinckx <herinckc@flip3.engr.oregonstate.edu>
Date: Mon, 14 May 2018 13:53:53 -0700
Subject: [PATCH 1685/2974] changed Twisted >= 17.9.0 to Twisted>=17.9.0

---
 requirements-py3.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-py3.txt b/requirements-py3.txt
index 2aae3ae65..1f342cfbb 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -1,4 +1,4 @@
-Twisted >= 17.9.0
+Twisted>=17.9.0
 lxml>=3.2.4
 pyOpenSSL>=0.13.1
 cssselect>=0.9

From 596f39600dfa94504b585ef65b1b04571a631447 Mon Sep 17 00:00:00 2001
From: Colton Herinckx <herinckc@flip3.engr.oregonstate.edu>
Date: Sat, 19 May 2018 16:32:55 -0700
Subject: [PATCH 1686/2974] reversed earlier change that seemed to cause Travis
 CI build failure

---
 requirements-py3.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-py3.txt b/requirements-py3.txt
index 1f342cfbb..2aae3ae65 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -1,4 +1,4 @@
-Twisted>=17.9.0
+Twisted >= 17.9.0
 lxml>=3.2.4
 pyOpenSSL>=0.13.1
 cssselect>=0.9

From d4511667fb5df63058accfe731e3d4160e795ee8 Mon Sep 17 00:00:00 2001
From: mugayoshi <muga.y.j3039@gmail.com>
Date: Sat, 9 Jun 2018 18:17:11 +0900
Subject: [PATCH 1687/2974] Update debugging memory leaks section in the docs

Add Python3 tools description.
---
 docs/topics/leaks.rst | 45 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 92590c180..af14d14e8 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -202,6 +202,7 @@ memory leaks (Requests, Responses, Items, and Selectors). However, there are
 other cases where the memory leaks could come from other (more or less obscure)
 objects. If this is your case, and you can't find your leaks using ``trackref``,
 you still have another resource: the `Guppy library`_.
+If you're using Python3, see :ref:`topics-leaks-muppy`.
 
 .. _Guppy library: https://pypi.python.org/pypi/guppy
 
@@ -253,6 +254,50 @@ knowledge about Python internals. For more info about Guppy, refer to the
 
 .. _Guppy documentation: http://guppy-pe.sourceforge.net/
 
+.. _topics-leaks-muppy:
+
+Debugging memory leaks with muppy
+=================================
+If you're using Python 3, you can use muppy from `Pympler`_.
+
+.. _Pympler: https://pypi.org/project/Pympler/
+
+If you use ``pip``, you can install muppy with the following command::
+
+    pip install Pympler
+
+Here's an example to view all Python objects available in
+the heap using muppy::
+
+    >>> from pympler import muppy
+    >>> all_objects = muppy.get_objects()
+    >>> len(all_objects)
+    28667
+    >>> from pympler import summary
+    >>> suml = summary.summarize(all_objects)
+    >>> summary.print_(suml)
+                                   types |   # objects |   total size
+    ==================================== | =========== | ============
+                             <class 'str |        9822 |      1.10 MB
+                            <class 'dict |        1658 |    856.62 KB
+                            <class 'type |         436 |    443.60 KB
+                            <class 'code |        2974 |    419.56 KB
+              <class '_io.BufferedWriter |           2 |    256.34 KB
+                             <class 'set |         420 |    159.88 KB
+              <class '_io.BufferedReader |           1 |    128.17 KB
+              <class 'wrapper_descriptor |        1130 |     88.28 KB
+                           <class 'tuple |        1304 |     86.57 KB
+                         <class 'weakref |        1013 |     79.14 KB
+      <class 'builtin_function_or_method |         958 |     67.36 KB
+               <class 'method_descriptor |         865 |     60.82 KB
+                     <class 'abc.ABCMeta |          62 |     59.96 KB
+                            <class 'list |         446 |     58.52 KB
+                             <class 'int |        1425 |     43.20 KB
+
+For more info about muppy, refer to the `muppy documentation`_.
+
+.. _muppy documentation: https://pythonhosted.org/Pympler/muppy.html
+
 .. _topics-leaks-without-leaks:
 
 Leaks without leaks

From e2bb218e9ad89e9b367604f5f11a6fd7106cd79a Mon Sep 17 00:00:00 2001
From: Renne Rocha <renne@scrapinghub.com>
Date: Wed, 13 Jun 2018 18:11:43 -0300
Subject: [PATCH 1688/2974] Include Python version indication to each required
 library used in S3 storage

---
 docs/topics/feed-exports.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 135d05c93..b64dbfbfd 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -177,7 +177,7 @@ The feeds are stored on `Amazon S3`_.
    * ``s3://mybucket/path/to/export.csv``
    * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
 
- * Required external libraries: `botocore`_ or `boto`_
+ * Required external libraries: `botocore`_ (Python 2 and Python 3) or `boto`_ (Python 2 only)
 
 The AWS credentials can be passed as user/password in the URI, or they can be
 passed through the following settings:

From 72d0899bce06190de5a453b24dd66c8910e6d0ee Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Thu, 14 Jun 2018 17:58:48 +0300
Subject: [PATCH 1689/2974] Return non-zero exit code from scrapy commands in
 case of spider bootstrap errors

* method to detect spider creation in crawler

* correct method name

* method to know if crawlers has spiders

* we do not need to issue requests

* set exit code accordingly to spiders in crawlers

* more portable way to check ofr exceptions

* more clear way

* test cases for several spiders per crawler

* grammatically correct name for method

* method is private

* grammatically correct name for method

* method is private

* remove unused import

* correct order of imports

* changes mechanism of obtaining spider status from method to object member

* rename tests
---
 scrapy/commands/crawl.py     |  3 ++
 scrapy/commands/runspider.py |  3 ++
 scrapy/crawler.py            |  2 ++
 tests/test_commands.py       | 12 +++++++
 tests/test_crawler.py        | 64 +++++++++++++++++++++++++++++++++++-
 5 files changed, 83 insertions(+), 1 deletion(-)

diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 4b986bf9d..8093fd402 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -56,3 +56,6 @@ class Command(ScrapyCommand):
 
         self.crawler_process.crawl(spname, **opts.spargs)
         self.crawler_process.start()
+
+        if self.crawler_process.bootstrap_failed:
+            self.exitcode = 1
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index a98033dd1..376d3c84e 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -87,3 +87,6 @@ class Command(ScrapyCommand):
 
         self.crawler_process.crawl(spidercls, **opts.spargs)
         self.crawler_process.start()
+
+        if self.crawler_process.bootstrap_failed:
+            self.exitcode = 1
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 5cbc2d7c5..04aee18ed 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -137,6 +137,7 @@ class CrawlerRunner(object):
         self.spider_loader = _get_spider_loader(settings)
         self._crawlers = set()
         self._active = set()
+        self.bootstrap_failed = False
 
     @property
     def spiders(self):
@@ -178,6 +179,7 @@ class CrawlerRunner(object):
         def _done(result):
             self.crawlers.discard(crawler)
             self._active.discard(d)
+            self.bootstrap_failed |= not getattr(crawler, 'spider', None)
             return result
 
         return d.addBoth(_done)
diff --git a/tests/test_commands.py b/tests/test_commands.py
index cb1301c95..7d9071b64 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -1,3 +1,4 @@
+import inspect
 import os
 import sys
 import subprocess
@@ -17,6 +18,7 @@ from scrapy.utils.python import retry_on_eintr
 from scrapy.utils.test import get_testenv
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
+from tests.test_crawler import ExceptionSpider, NoRequestsSpider
 
 
 class ProjectTest(unittest.TestCase):
@@ -220,6 +222,16 @@ class MySpider(scrapy.Spider):
         self.assertIn("INFO: Closing spider (finished)", log)
         self.assertIn("INFO: Spider closed (finished)", log)
 
+    def test_run_fail_spider(self):
+        proc = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider))
+        ret = proc.returncode
+        self.assertNotEqual(ret, 0)
+
+    def test_run_good_spider(self):
+        proc = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider))
+        ret = proc.returncode
+        self.assertEqual(ret, 0)
+
     def test_runspider_log_level(self):
         log = self.get_log(self.debug_log_spider,
                            args=('-s', 'LOG_LEVEL=INFO'))
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index ba0d709ff..d3b80f460 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -4,6 +4,9 @@ import tempfile
 import warnings
 import unittest
 
+from twisted.internet import defer
+import twisted.trial.unittest
+
 import scrapy
 from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
 from scrapy.settings import Settings, default_settings
@@ -11,9 +14,9 @@ from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.log import configure_logging, get_scrapy_root_handler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
+from scrapy.utils.test import get_crawler
 from scrapy.extensions.throttle import AutoThrottle
 
-
 class BaseCrawlerTest(unittest.TestCase):
 
     def assertOptionIsDefault(self, settings, key):
@@ -181,3 +184,62 @@ class CrawlerProcessTest(BaseCrawlerTest):
     def test_crawler_process_accepts_None(self):
         runner = CrawlerProcess()
         self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED')
+
+
+class ExceptionSpider(scrapy.Spider):
+    name = 'exception'
+
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        raise ValueError('Exception in from_crawler method')
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+class CrawlerRunnerHasSpider(twisted.trial.unittest.TestCase):
+
+    @defer.inlineCallbacks
+    def test_crawler_runner_bootstrap_successful(self):
+        runner = CrawlerRunner()
+        yield runner.crawl(NoRequestsSpider)
+        self.assertEqual(runner.bootstrap_failed, False)
+
+    @defer.inlineCallbacks
+    def test_crawler_runner_bootstrap_successful_for_several(self):
+        runner = CrawlerRunner()
+        yield runner.crawl(NoRequestsSpider)
+        yield runner.crawl(NoRequestsSpider)
+        self.assertEqual(runner.bootstrap_failed, False)
+
+    @defer.inlineCallbacks
+    def test_crawler_runner_bootstrap_failed(self):
+        runner = CrawlerRunner()
+
+        try:
+            yield runner.crawl(ExceptionSpider)
+        except ValueError:
+            pass
+        else:
+            self.fail('Exception should be raised from spider')
+
+        self.assertEqual(runner.bootstrap_failed, True)
+
+    @defer.inlineCallbacks
+    def test_crawler_runner_bootstrap_failed_for_several(self):
+        runner = CrawlerRunner()
+
+        try:
+            yield runner.crawl(ExceptionSpider)
+        except ValueError:
+            pass
+        else:
+            self.fail('Exception should be raised from spider')
+
+        yield runner.crawl(NoRequestsSpider)
+
+        self.assertEqual(runner.bootstrap_failed, True)

From 7a601d76de7adc37571815a7d08f84e1e26f7507 Mon Sep 17 00:00:00 2001
From: Leo <leochoa90@gmail.com>
Date: Tue, 19 Jun 2018 10:51:55 +0200
Subject: [PATCH 1690/2974] fix typo

extractred --> extracted
---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 1629510b2..1b8d121a1 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -156,7 +156,7 @@ attributes with ``FormRequest``.
 
 **Please also note that link extractors do not canonicalize URLs by default
 anymore.** This was puzzling users every now and then, and it's not what
-browsers do in fact, so we removed that extra transformation on extractred
+browsers do in fact, so we removed that extra transformation on extracted
 links.
 
 For those of you wanting more control on the ``Referer:`` header that Scrapy

From 88bd067912ee94e2a6d2e1ba5b0d5db1241b7621 Mon Sep 17 00:00:00 2001
From: Grammy Jiang <grammy.jiang@gmail.com>
Date: Wed, 20 Jun 2018 16:56:46 +0800
Subject: [PATCH 1691/2974] fix the test case name of HttpProxyMiddleware

---
 tests/test_downloadermiddleware_httpproxy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 17be875c1..537126613 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -13,7 +13,7 @@ from scrapy.settings import Settings
 spider = Spider('foo')
 
 
-class TestDefaultHeadersMiddleware(TestCase):
+class TestHttpProxyMiddleware(TestCase):
 
     failureException = AssertionError
 

From 9ad3af9d88bcefa18394c1cb2c833902ac20c533 Mon Sep 17 00:00:00 2001
From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com>
Date: Sat, 23 Jun 2018 17:31:54 +0800
Subject: [PATCH 1692/2974] Update requirements.txt

make the version of ipython less than 6.0 in python 2.7
---
 tests/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/requirements.txt b/tests/requirements.txt
index c1576a2e7..790f29d34 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -10,4 +10,4 @@ brotlipy
 testfixtures
 # optional for shell wrapper tests
 bpython
-ipython
+ipython<6.0

From 4740dca8f260bef83eed849b692b3a2c1aaec6cf Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 24 Jun 2018 20:59:18 -0300
Subject: [PATCH 1693/2974] Deferred-like process_output/process_exception
 chain

---
 scrapy/core/spidermw.py | 58 +++++++++++++++++++++++++----------------
 1 file changed, 36 insertions(+), 22 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 50677670b..98e264bd3 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -17,6 +17,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
     component_name = 'spider middleware'
 
+    # List of dicts. Each dict represents a spider middleware and contains the
+    # 'process_spider_output', 'process_spider_exception' methods.
+    # The idea is to simulate the behaviour of a Twisted deferred's callback/errback chain
+    output_methods = []
+
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
         return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
@@ -25,12 +30,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
         super(SpiderMiddlewareManager, self)._add_middleware(mw)
         if hasattr(mw, 'process_spider_input'):
             self.methods['process_spider_input'].append(mw.process_spider_input)
-        if hasattr(mw, 'process_spider_output'):
-            self.methods['process_spider_output'].insert(0, mw.process_spider_output)
-        if hasattr(mw, 'process_spider_exception'):
-            self.methods['process_spider_exception'].insert(0, mw.process_spider_exception)
         if hasattr(mw, 'process_start_requests'):
             self.methods['process_start_requests'].insert(0, mw.process_start_requests)
+        self.output_methods.insert(0, dict(
+            process_spider_output=getattr(mw, 'process_spider_output', None),
+            process_spider_exception=getattr(mw, 'process_spider_exception', None),
+        ))
 
     def scrape_response(self, scrape_func, response, request, spider):
         fname = lambda f:'%s.%s' % (
@@ -48,45 +53,54 @@ class SpiderMiddlewareManager(MiddlewareManager):
                     return scrape_func(Failure(), request, spider)
             return scrape_func(response, request, spider)
 
-        def process_spider_exception(_failure):
+        def process_spider_exception(_failure, mw_index):
             exception = _failure.value
             # don't handle _InvalidOutput exception
             if isinstance(exception, _InvalidOutput):
                 return _failure
-            for method in self.methods['process_spider_exception']:
-                result = method(response=response, exception=exception, spider=spider)
-                if result is not None and not _isiterable(result):
-                    raise _InvalidOutput('Middleware {} must return None or an iterable ' \
-                        'object, got {}'.format(fname(method), type(result)))
-                # stop exception handling by handing control over to the
-                # process_spider_output chain if an iterable has been returned
-                if result is not None:
-                    return result
+            for index, mw in enumerate(self.output_methods):
+                if index < mw_index or mw['process_spider_exception'] is None:
+                    continue
+                result = mw['process_spider_exception'](response=response, exception=exception, spider=spider)
+                mw_index += 1
+                if _isiterable(result):
+                    # stop exception handling by handing control over to the
+                    # process_spider_output chain if an iterable has been returned
+                    return process_spider_output(result, mw_index)
+                elif result is None:
+                    continue
+                else:
+                    raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \
+                                         .format(fname(mw['process_spider_exception']), type(result)))
             return _failure
 
-        def process_spider_output(result):
+        def process_spider_output(result, mw_index):
             def wrapper(result_iterable):
                 try:
                     for r in result_iterable:
                         yield r
                 except Exception as ex:
-                    exception_result = process_spider_exception(Failure(ex))
+                    # process the exception with the method from the next middleware
+                    exception_result = process_spider_exception(Failure(ex), mw_index)
                     if exception_result is None or isinstance(exception_result, Failure):
                         raise
                     for output in exception_result:
                         yield output
-            for method in self.methods['process_spider_output']:
-                result = method(response=response, result=result, spider=spider)
+            for index, mw in enumerate(self.output_methods):
+                if index < mw_index or mw['process_spider_output'] is None:
+                    continue
+                result = mw['process_spider_output'](response=response, result=result, spider=spider)
+                mw_index += 1
                 if _isiterable(result):
                     result = wrapper(result)
                 else:
-                    raise _InvalidOutput('Middleware {} must return an iterable object, ' \
-                        'got {}'.format(fname(method), type(result)))
+                    raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
+                                         .format(fname(mw['process_spider_output']), type(result)))
             return result
 
         dfd = mustbe_deferred(process_spider_input, response)
-        dfd.addErrback(process_spider_exception)
-        dfd.addCallback(process_spider_output)
+        dfd.addErrback(process_spider_exception, mw_index=0)
+        dfd.addCallback(process_spider_output, mw_index=0)
         return dfd
 
     def process_start_requests(self, start_requests, spider):

From ba294351381c0dd81476603246d2cea6c31486be Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 25 Jun 2018 15:01:12 -0300
Subject: [PATCH 1694/2974] Default values for OffsiteMiddleware

For some reason test_crawl.py seems to be skipping the spider_opened
method, which initializes the host_regex instance variable
---
 scrapy/spidermiddlewares/offsite.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 310166cad..3b7f194e4 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -19,6 +19,9 @@ class OffsiteMiddleware(object):
 
     def __init__(self, stats):
         self.stats = stats
+        # default values
+        self.host_regex = re.compile('')  # allow all by default
+        self.domains_seen = set()
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -52,7 +55,7 @@ class OffsiteMiddleware(object):
         """Override this method to implement a different offsite policy"""
         allowed_domains = getattr(spider, 'allowed_domains', None)
         if not allowed_domains:
-            return re.compile('') # allow all by default
+            return
         url_pattern = re.compile("^https?://.*$")
         for domain in allowed_domains:
             if url_pattern.match(domain):
@@ -62,8 +65,9 @@ class OffsiteMiddleware(object):
         return re.compile(regex)
 
     def spider_opened(self, spider):
-        self.host_regex = self.get_host_regex(spider)
-        self.domains_seen = set()
+        host_regex = self.get_host_regex(spider)
+        if host_regex:
+            self.host_regex = host_regex
 
 
 class URLWarning(Warning):

From fac1b2f3516f3db6ce669664943336c08d95d0aa Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 27 Jun 2018 03:23:47 +0500
Subject: [PATCH 1695/2974] TST remove workaround for old Pillow versions which
 don't support BytesIO

---
 tests/test_pipeline_images.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 03c6d8059..a7c652959 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -1,8 +1,8 @@
-import os
+import io
 import hashlib
 import random
 import warnings
-from tempfile import mkdtemp, TemporaryFile
+from tempfile import mkdtemp
 from shutil import rmtree
 
 from twisted.trial import unittest
@@ -401,8 +401,9 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
             self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()),
                              expected_value)
 
+
 def _create_image(format, *a, **kw):
-    buf = TemporaryFile()
+    buf = io.BytesIO()
     Image.new(*a, **kw).save(buf, format)
     buf.seek(0)
     return Image.open(buf)

From 45f67eb64d54f2ac9fcd69233d7bddbdcec88a37 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 27 Jun 2018 14:51:01 +0500
Subject: [PATCH 1696/2974] TST exclude lxml==4.2.2 from tests, as it doesn't
 play well with Pillow

---
 tests/constraints.txt | 1 +
 tox.ini               | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tests/constraints.txt b/tests/constraints.txt
index 3bc30de15..e59e68b3f 100644
--- a/tests/constraints.txt
+++ b/tests/constraints.txt
@@ -1 +1,2 @@
 Twisted!=18.4.0
+lxml!=4.2.2
\ No newline at end of file
diff --git a/tox.ini b/tox.ini
index c2fa9af28..82348eb24 100644
--- a/tox.ini
+++ b/tox.ini
@@ -67,6 +67,7 @@ commands =
 [testenv:py34]
 basepython = python3.4
 deps =
+    -ctests/constraints.txt
     -rrequirements-py3.txt
     # Extras
     Pillow

From 8782901fc865ca6f505bcd1a7d17314076507028 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 28 Jun 2018 01:11:15 +0500
Subject: [PATCH 1697/2974] [MRG+1] TST test agains latest pypy (#3309)

pypy3 is not upgraded, as tests segfault with pypy3 6.0 for some reason
---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 6635f5d3b..065f23805 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -26,7 +26,7 @@ matrix:
 install:
   - |
       if [ "$TOXENV" = "pypy" ]; then
-        export PYPY_VERSION="pypy-5.9-linux_x86_64-portable"
+        export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable"
         wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
         tar -jxf ${PYPY_VERSION}.tar.bz2
         virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"

From f11d65f7d66cf2d8560c707c4bb2b76079d45e5f Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 29 Jun 2018 18:34:11 +0500
Subject: [PATCH 1698/2974] TST make it clear which requirements are Python
 2-only

* rename requirements.txt to requirements-py2.txt, to make it clear they are Python 2-only
* make requirements-py3.txt consistent with requirements-py2.txt
---
 requirements.txt => requirements-py2.txt         | 4 ++--
 requirements-py3.txt                             | 3 +++
 tests/{requirements.txt => requirements-py2.txt} | 0
 tox.ini                                          | 8 ++++----
 4 files changed, 9 insertions(+), 6 deletions(-)
 rename requirements.txt => requirements-py2.txt (100%)
 rename tests/{requirements.txt => requirements-py2.txt} (100%)

diff --git a/requirements.txt b/requirements-py2.txt
similarity index 100%
rename from requirements.txt
rename to requirements-py2.txt
index 2a94d742d..03b33d02d 100644
--- a/requirements.txt
+++ b/requirements-py2.txt
@@ -2,9 +2,9 @@ Twisted>=13.1.0
 lxml
 pyOpenSSL
 cssselect>=0.9
-w3lib>=1.17.0
 queuelib
+w3lib>=1.17.0
 six>=1.5.2
 PyDispatcher>=2.0.5
-service_identity
 parsel>=1.4
+service_identity
diff --git a/requirements-py3.txt b/requirements-py3.txt
index 2aae3ae65..d76d9412f 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -4,4 +4,7 @@ pyOpenSSL>=0.13.1
 cssselect>=0.9
 queuelib>=1.1.1
 w3lib>=1.17.0
+six>=1.5.2
+PyDispatcher>=2.0.5
+parsel>=1.4
 service_identity
diff --git a/tests/requirements.txt b/tests/requirements-py2.txt
similarity index 100%
rename from tests/requirements.txt
rename to tests/requirements-py2.txt
diff --git a/tox.ini b/tox.ini
index 82348eb24..ee40983de 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,13 +9,13 @@ envlist = py27
 [testenv]
 deps =
     -ctests/constraints.txt
-    -rrequirements.txt
+    -rrequirements-py2.txt
     # Extras
     botocore
     google-cloud-storage
     Pillow != 3.0.0
     leveldb
-    -rtests/requirements.txt
+    -rtests/requirements-py2.txt
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
@@ -35,7 +35,7 @@ deps =
     Pillow==2.3.0
     cssselect==0.9.1
     zope.interface==4.0.5
-    -rtests/requirements.txt
+    -rtests/requirements-py2.txt
 
 [testenv:jessie]
 # https://packages.debian.org/en/jessie/python/
@@ -50,7 +50,7 @@ deps =
     Pillow==2.6.1
     cssselect==0.9.1
     zope.interface==4.1.1
-    -rtests/requirements.txt
+    -rtests/requirements-py2.txt
 
 [testenv:trunk]
 basepython = python2.7

From df75a0942e004f9645182a0260769f4337f843e5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 1 Jul 2018 13:30:50 -0300
Subject: [PATCH 1699/2974] Update docs

---
 docs/topics/spider-middleware.rst | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 0af26be73..dde1786af 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -112,9 +112,8 @@ following methods:
 
     .. method:: process_spider_exception(response, exception, spider)
 
-        This method is called when when a spider or :meth:`process_spider_input`/
-        :meth:`process_spider_output` method (from other spider middleware)
-        raises an exception.
+        This method is called when a spider or :meth:`process_spider_output`
+        method (from a previous spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
         iterable of :class:`~scrapy.http.Request`, dict or
@@ -126,7 +125,7 @@ following methods:
         exception reaches the engine (where it's logged and discarded).
 
         If it returns an iterable the :meth:`process_spider_output` pipeline
-        kicks in, starting with the last non-executed method, and no other
+        kicks in, starting from the next spider middleware, and no other
         :meth:`process_spider_exception` will be called.
 
         :param response: the response being processed when the exception was

From 735de8167d3e6b0085710d406c8c2976913baa43 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 30 Jun 2018 20:55:17 -0300
Subject: [PATCH 1700/2974] Test for exceptions on process_spider_input

---
 tests/test_spider_mw.py | 77 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 tests/test_spider_mw.py

diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py
new file mode 100644
index 000000000..4a431d379
--- /dev/null
+++ b/tests/test_spider_mw.py
@@ -0,0 +1,77 @@
+
+import logging
+
+from testfixtures import LogCapture
+from twisted.trial.unittest import TestCase
+from twisted.internet import defer
+
+from scrapy import Spider, Request
+from scrapy.utils.test import get_crawler
+from tests.mockserver import MockServer
+
+
+class CommonTestCase(TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.mockserver = MockServer()
+        cls.mockserver.__enter__()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def crawl_log(self, spider):
+        crawler = get_crawler(spider)
+        with LogCapture() as log:
+            yield crawler.crawl()
+        raise defer.returnValue(log)
+
+
+class LogExceptionMiddleware(object):
+    def process_spider_exception(self, response, exception, spider):
+        logging.warn('Middleware: %s exception caught', exception.__class__.__name__)
+        return None
+
+
+# ================================================================================
+# (1) exceptions from a spider middleware's process_spider_input method
+class ProcessSpiderInputSpider(Spider):
+    name = 'ProcessSpiderInputSpider'
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            # spider
+            __name__ + '.LogExceptionMiddleware': 10,
+            __name__ + '.FailProcessSpiderInputMiddleware': 8,
+            __name__ + '.LogExceptionMiddleware': 6,
+            # engine
+        }
+    }
+
+    def start_requests(self):
+        yield Request('http://localhost:8998', callback=self.parse, errback=self.errback)
+
+    def parse(self, response):
+        return [{'test': 1}, {'test': 2}]
+
+    def errback(self, failure):
+        self.logger.warn('Got a Failure on the Request errback')
+
+
+class FailProcessSpiderInputMiddleware:
+    def process_spider_input(self, response, spider):
+        logging.warn('Middleware: will raise ZeroDivisionError')
+        raise ZeroDivisionError()
+
+
+class TestProcessSpiderInputSpider(CommonTestCase):
+    @defer.inlineCallbacks
+    def test_process_spider_input_errback(self):
+        """
+        (1) An exception from the process_spider_input chain should not be caught by the
+        process_spider_exception chain, it should go directly to the Request errback
+        """
+        log = yield self.crawl_log(ProcessSpiderInputSpider)
+        self.assertNotIn('Middleware: ZeroDivisionError exception caught', str(log))
+        self.assertIn('Middleware: will raise ZeroDivisionError', str(log))
+        self.assertIn('Got a Failure on the Request errback', str(log))

From 6ed9440ed528ab5c5eece50512e19929e4320b42 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 30 Jun 2018 21:27:10 -0300
Subject: [PATCH 1701/2974] Tests for exceptions on spider callbacks

---
 tests/test_spider_mw.py | 72 +++++++++++++++++++++++++++++++++++++----
 1 file changed, 66 insertions(+), 6 deletions(-)

diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py
index 4a431d379..092546291 100644
--- a/tests/test_spider_mw.py
+++ b/tests/test_spider_mw.py
@@ -60,11 +60,11 @@ class ProcessSpiderInputSpider(Spider):
 
 class FailProcessSpiderInputMiddleware:
     def process_spider_input(self, response, spider):
-        logging.warn('Middleware: will raise ZeroDivisionError')
-        raise ZeroDivisionError()
+        logging.warn('Middleware: will raise IndexError')
+        raise IndexError()
 
 
-class TestProcessSpiderInputSpider(CommonTestCase):
+class TestProcessSpiderInput(CommonTestCase):
     @defer.inlineCallbacks
     def test_process_spider_input_errback(self):
         """
@@ -72,6 +72,66 @@ class TestProcessSpiderInputSpider(CommonTestCase):
         process_spider_exception chain, it should go directly to the Request errback
         """
         log = yield self.crawl_log(ProcessSpiderInputSpider)
-        self.assertNotIn('Middleware: ZeroDivisionError exception caught', str(log))
-        self.assertIn('Middleware: will raise ZeroDivisionError', str(log))
-        self.assertIn('Got a Failure on the Request errback', str(log))
+        self.assertNotIn("Middleware: IndexError exception caught", str(log))
+        self.assertIn("Middleware: will raise IndexError", str(log))
+        self.assertIn("Got a Failure on the Request errback", str(log))
+
+
+# ================================================================================
+# (2) exceptions from a spider callback (generator)
+class GeneratorCallbackSpider(Spider):
+    name = 'GeneratorCallbackSpider'
+    start_urls = ['http://localhost:8998']
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            # spider
+            __name__ + '.LogExceptionMiddleware': 10,
+            # engine
+        },
+    }
+
+    def parse(self, response):
+        yield {'test': 1}
+        yield {'test': 2}
+        raise ImportError()
+
+
+class TestGeneratorCallback(CommonTestCase):
+    @defer.inlineCallbacks
+    def test_generator_callback(self):
+        """
+        (2) An exception from a spider's callback should
+        be caught by the process_spider_exception chain
+        """
+        log = yield self.crawl_log(GeneratorCallbackSpider)
+        self.assertIn("Middleware: ImportError exception caught", str(log))
+        self.assertIn("'item_scraped_count': 2", str(log))
+
+
+# ================================================================================
+# (3) exceptions from a spider callback (not a generator)
+class NotAGeneratorCallbackSpider(Spider):
+    name = 'NotAGeneratorCallbackSpider'
+    start_urls = ['http://localhost:8998']
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            # spider
+            __name__ + '.LogExceptionMiddleware': 10,
+            # engine
+        },
+    }
+
+    def parse(self, response):
+        return [{'test': 1}, {'test': 1/0}]
+
+
+class TestNotAGeneratorCallback(CommonTestCase):
+    @defer.inlineCallbacks
+    def test_not_a_generator_callback(self):
+        """
+        (3) An exception from a spider's callback should
+        be caught by the process_spider_exception chain
+        """
+        log = yield self.crawl_log(NotAGeneratorCallbackSpider)
+        self.assertIn("Middleware: ZeroDivisionError exception caught", str(log))
+        self.assertNotIn("item_scraped_count", str(log))

From 4fca9aba851133fcdc12bb46c7ae229d9537079a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 1 Jul 2018 13:18:29 -0300
Subject: [PATCH 1702/2974] Recover from a callback exception

---
 tests/test_spider_mw.py | 137 +++++++++++++++++++++++++---------------
 1 file changed, 85 insertions(+), 52 deletions(-)

diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py
index 092546291..49a60d08b 100644
--- a/tests/test_spider_mw.py
+++ b/tests/test_spider_mw.py
@@ -10,22 +10,8 @@ from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
 
 
-class CommonTestCase(TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.mockserver = MockServer()
-        cls.mockserver.__enter__()
-
-    @classmethod
-    def tearDownClass(cls):
-        cls.mockserver.__exit__(None, None, None)
-
-    @defer.inlineCallbacks
-    def crawl_log(self, spider):
-        crawler = get_crawler(spider)
-        with LogCapture() as log:
-            yield crawler.crawl()
-        raise defer.returnValue(log)
+# TEST_URL = 'http://example.org'
+TEST_URL = 'http://localhost:8998'
 
 
 class LogExceptionMiddleware(object):
@@ -34,6 +20,32 @@ class LogExceptionMiddleware(object):
         return None
 
 
+# ================================================================================
+# recover from an exception on a spider callback
+class RecoverySpider(Spider):
+    name = 'RecoverySpider'
+    start_urls = [TEST_URL]
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            __name__ + '.RecoveryMiddleware': 10,
+        },
+    }
+
+    def parse(self, response):
+        yield {'test': 1}
+        self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail'))
+        if not response.meta.get('dont_fail'):
+            raise ModuleNotFoundError()
+
+class RecoveryMiddleware(object):
+    def process_spider_exception(self, response, exception, spider):
+        logging.warn('Middleware: %s exception caught', exception.__class__.__name__)
+        return [
+            {'from': 'process_spider_exception'},
+            Request(response.url, meta={'dont_fail': True}, dont_filter=True),
+        ]
+
+
 # ================================================================================
 # (1) exceptions from a spider middleware's process_spider_input method
 class ProcessSpiderInputSpider(Spider):
@@ -49,7 +61,7 @@ class ProcessSpiderInputSpider(Spider):
     }
 
     def start_requests(self):
-        yield Request('http://localhost:8998', callback=self.parse, errback=self.errback)
+        yield Request(TEST_URL, callback=self.parse, errback=self.errback)
 
     def parse(self, response):
         return [{'test': 1}, {'test': 2}]
@@ -64,29 +76,14 @@ class FailProcessSpiderInputMiddleware:
         raise IndexError()
 
 
-class TestProcessSpiderInput(CommonTestCase):
-    @defer.inlineCallbacks
-    def test_process_spider_input_errback(self):
-        """
-        (1) An exception from the process_spider_input chain should not be caught by the
-        process_spider_exception chain, it should go directly to the Request errback
-        """
-        log = yield self.crawl_log(ProcessSpiderInputSpider)
-        self.assertNotIn("Middleware: IndexError exception caught", str(log))
-        self.assertIn("Middleware: will raise IndexError", str(log))
-        self.assertIn("Got a Failure on the Request errback", str(log))
-
-
 # ================================================================================
 # (2) exceptions from a spider callback (generator)
 class GeneratorCallbackSpider(Spider):
     name = 'GeneratorCallbackSpider'
-    start_urls = ['http://localhost:8998']
+    start_urls = [TEST_URL]
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            # spider
             __name__ + '.LogExceptionMiddleware': 10,
-            # engine
         },
     }
 
@@ -96,28 +93,14 @@ class GeneratorCallbackSpider(Spider):
         raise ImportError()
 
 
-class TestGeneratorCallback(CommonTestCase):
-    @defer.inlineCallbacks
-    def test_generator_callback(self):
-        """
-        (2) An exception from a spider's callback should
-        be caught by the process_spider_exception chain
-        """
-        log = yield self.crawl_log(GeneratorCallbackSpider)
-        self.assertIn("Middleware: ImportError exception caught", str(log))
-        self.assertIn("'item_scraped_count': 2", str(log))
-
-
 # ================================================================================
 # (3) exceptions from a spider callback (not a generator)
 class NotAGeneratorCallbackSpider(Spider):
     name = 'NotAGeneratorCallbackSpider'
-    start_urls = ['http://localhost:8998']
+    start_urls = [TEST_URL]
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            # spider
             __name__ + '.LogExceptionMiddleware': 10,
-            # engine
         },
     }
 
@@ -125,13 +108,63 @@ class NotAGeneratorCallbackSpider(Spider):
         return [{'test': 1}, {'test': 1/0}]
 
 
-class TestNotAGeneratorCallback(CommonTestCase):
+# ================================================================================
+class TestSpiderMiddleware(TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.mockserver = MockServer()
+        cls.mockserver.__enter__()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.mockserver.__exit__(None, None, None)
+    
+    @defer.inlineCallbacks
+    def crawl_log(self, spider):
+        crawler = get_crawler(spider)
+        with LogCapture() as log:
+            yield crawler.crawl()
+        raise defer.returnValue(log)
+
+    # @defer.inlineCallbacks
+    # def test_recovery(self):
+    #     """
+    #     Recover from an exception from a spider's callback. The final item count should be 3
+    #     (one from the spider before raising the exception, one from the middleware and one
+    #     from the spider when processing the response that was enqueued from the middleware)
+    #     """
+    #     log = yield self.crawl_log(RecoverySpider)
+    #     self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log))
+    #     self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1)
+    #     self.assertIn("'item_scraped_count': 3", str(log))
+
+    @defer.inlineCallbacks
+    def test_process_spider_input_errback(self):
+        """
+        (1) An exception from the process_spider_input chain should not be caught by the
+        process_spider_exception chain, it should go directly to the Request errback
+        """
+        log1 = yield self.crawl_log(ProcessSpiderInputSpider)
+        self.assertNotIn("Middleware: IndexError exception caught", str(log1))
+        self.assertIn("Middleware: will raise IndexError", str(log1))
+        self.assertIn("Got a Failure on the Request errback", str(log1))
+    
+    @defer.inlineCallbacks
+    def test_generator_callback(self):
+        """
+        (2) An exception from a spider's callback should
+        be caught by the process_spider_exception chain
+        """
+        log2 = yield self.crawl_log(GeneratorCallbackSpider)
+        self.assertIn("Middleware: ImportError exception caught", str(log2))
+        self.assertIn("'item_scraped_count': 2", str(log2))
+    
     @defer.inlineCallbacks
     def test_not_a_generator_callback(self):
         """
         (3) An exception from a spider's callback should
         be caught by the process_spider_exception chain
         """
-        log = yield self.crawl_log(NotAGeneratorCallbackSpider)
-        self.assertIn("Middleware: ZeroDivisionError exception caught", str(log))
-        self.assertNotIn("item_scraped_count", str(log))
+        log3 = yield self.crawl_log(NotAGeneratorCallbackSpider)
+        self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3))
+        self.assertNotIn("item_scraped_count", str(log3))

From 985ab636cfa0825f100b02583bcfd106d1f4cef6 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 1 Jul 2018 17:49:30 -0300
Subject: [PATCH 1703/2974] Store output methods on the 'methods' dict

---
 scrapy/core/spidermw.py | 43 +++++++++++++++++------------------------
 1 file changed, 18 insertions(+), 25 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 98e264bd3..c9dd8c91e 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -17,11 +17,6 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
     component_name = 'spider middleware'
 
-    # List of dicts. Each dict represents a spider middleware and contains the
-    # 'process_spider_output', 'process_spider_exception' methods.
-    # The idea is to simulate the behaviour of a Twisted deferred's callback/errback chain
-    output_methods = []
-
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
         return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
@@ -32,10 +27,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
             self.methods['process_spider_input'].append(mw.process_spider_input)
         if hasattr(mw, 'process_start_requests'):
             self.methods['process_start_requests'].insert(0, mw.process_start_requests)
-        self.output_methods.insert(0, dict(
-            process_spider_output=getattr(mw, 'process_spider_output', None),
-            process_spider_exception=getattr(mw, 'process_spider_exception', None),
-        ))
+        self.methods['process_spider_output'].insert(0, getattr(mw, 'process_spider_output', None))
+        self.methods['process_spider_exception'].insert(0, getattr(mw, 'process_spider_exception', None))
 
     def scrape_response(self, scrape_func, response, request, spider):
         fname = lambda f:'%s.%s' % (
@@ -53,54 +46,54 @@ class SpiderMiddlewareManager(MiddlewareManager):
                     return scrape_func(Failure(), request, spider)
             return scrape_func(response, request, spider)
 
-        def process_spider_exception(_failure, mw_index):
+        def process_spider_exception(_failure, index):
             exception = _failure.value
             # don't handle _InvalidOutput exception
             if isinstance(exception, _InvalidOutput):
                 return _failure
-            for index, mw in enumerate(self.output_methods):
-                if index < mw_index or mw['process_spider_exception'] is None:
+            for i, method in enumerate(self.methods['process_spider_exception']):
+                if i < index or method is None:
                     continue
-                result = mw['process_spider_exception'](response=response, exception=exception, spider=spider)
-                mw_index += 1
+                result = method(response=response, exception=exception, spider=spider)
+                index += 1
                 if _isiterable(result):
                     # stop exception handling by handing control over to the
                     # process_spider_output chain if an iterable has been returned
-                    return process_spider_output(result, mw_index)
+                    return process_spider_output(result, index)
                 elif result is None:
                     continue
                 else:
                     raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \
-                                         .format(fname(mw['process_spider_exception']), type(result)))
+                                         .format(fname(method), type(result)))
             return _failure
 
-        def process_spider_output(result, mw_index):
+        def process_spider_output(result, index):
             def wrapper(result_iterable):
                 try:
                     for r in result_iterable:
                         yield r
                 except Exception as ex:
                     # process the exception with the method from the next middleware
-                    exception_result = process_spider_exception(Failure(ex), mw_index)
+                    exception_result = process_spider_exception(Failure(ex), index)
                     if exception_result is None or isinstance(exception_result, Failure):
                         raise
                     for output in exception_result:
                         yield output
-            for index, mw in enumerate(self.output_methods):
-                if index < mw_index or mw['process_spider_output'] is None:
+            for i, method in enumerate(self.methods['process_spider_output']):
+                if i < index or method is None:
                     continue
-                result = mw['process_spider_output'](response=response, result=result, spider=spider)
-                mw_index += 1
+                result = method(response=response, result=result, spider=spider)
+                index += 1
                 if _isiterable(result):
                     result = wrapper(result)
                 else:
                     raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
-                                         .format(fname(mw['process_spider_output']), type(result)))
+                                         .format(fname(method), type(result)))
             return result
 
         dfd = mustbe_deferred(process_spider_input, response)
-        dfd.addErrback(process_spider_exception, mw_index=0)
-        dfd.addCallback(process_spider_output, mw_index=0)
+        dfd.addErrback(process_spider_exception, index=0)
+        dfd.addCallback(process_spider_output, index=0)
         return dfd
 
     def process_start_requests(self, start_requests, spider):

From d05c8677c5079b88db4405fe2bed83dc437c9204 Mon Sep 17 00:00:00 2001
From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com>
Date: Wed, 4 Jul 2018 02:58:43 +0800
Subject: [PATCH 1704/2974] [MRG+1] change the bad smell code (#3304)

Change the bad smell code
---
 scrapy/downloadermiddlewares/httpproxy.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 0d5320bf8..1dd47359f 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -1,14 +1,13 @@
 import base64
+from six.moves.urllib.parse import unquote, urlunparse
 from six.moves.urllib.request import getproxies, proxy_bypass
-from six.moves.urllib.parse import unquote
 try:
     from urllib2 import _parse_proxy
 except ImportError:
     from urllib.request import _parse_proxy
-from six.moves.urllib.parse import urlunparse
 
-from scrapy.utils.httpobj import urlparse_cached
 from scrapy.exceptions import NotConfigured
+from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_bytes
 
 
@@ -17,8 +16,8 @@ class HttpProxyMiddleware(object):
     def __init__(self, auth_encoding='latin-1'):
         self.auth_encoding = auth_encoding
         self.proxies = {}
-        for type, url in getproxies().items():
-            self.proxies[type] = self._get_proxy(url, type)
+        for type_, url in getproxies().items():
+            self.proxies[type_] = self._get_proxy(url, type_)
 
     @classmethod
     def from_crawler(cls, crawler):

From 74ce1561542dac9be5d1363a4a3e623653855e05 Mon Sep 17 00:00:00 2001
From: chainly <1258626769@qq.com>
Date: Wed, 4 Jul 2018 03:00:59 +0800
Subject: [PATCH 1705/2974] add item_error to be catchable (#3256)

---
 docs/topics/signals.rst | 23 +++++++++++++++++++++++
 scrapy/core/scraper.py  |  3 +++
 scrapy/signals.py       |  1 +
 tests/pipelines.py      |  6 ++++++
 tests/test_engine.py    | 31 +++++++++++++++++++++++++++++++
 5 files changed, 64 insertions(+)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index cf1588df8..d40c0e1df 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -135,6 +135,29 @@ item_dropped
         to be dropped
     :type exception: :exc:`~scrapy.exceptions.DropItem` exception
 
+item_error
+------------
+
+.. signal:: item_error
+.. function:: item_error(item, response, spider, failure)
+
+    Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises
+    an exception), except :exc:`~scrapy.exceptions.DropItem` exception.
+
+    This signal supports returning deferreds from their handlers.
+
+    :param item: the item dropped from the :ref:`topics-item-pipeline`
+    :type item: dict or :class:`~scrapy.item.Item` object
+
+    :param response: the response being processed when the exception was raised
+    :type response: :class:`~scrapy.http.Response` object
+
+    :param spider: the spider which raised the exception
+    :type spider: :class:`~scrapy.spiders.Spider` object
+
+    :param failure: the exception raised as a Twisted `Failure`_ object
+    :type failure: `Failure`_ object
+
 spider_closed
 -------------
 
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index c08e37367..ee1e95a0c 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -232,6 +232,9 @@ class Scraper(object):
                 logger.error('Error processing %(item)s', {'item': item},
                              exc_info=failure_to_exc_info(output),
                              extra={'spider': spider})
+                return self.signals.send_catch_log_deferred(
+                    signal=signals.item_error, item=item, response=response,
+                    spider=spider, failure=output)
         else:
             logkws = self.logformatter.scraped(output, response, spider)
             logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
diff --git a/scrapy/signals.py b/scrapy/signals.py
index de0886fb6..e36c27203 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -17,6 +17,7 @@ response_received = object()
 response_downloaded = object()
 item_scraped = object()
 item_dropped = object()
+item_error = object()
 
 # for backwards compatibility
 stats_spider_opened = spider_opened
diff --git a/tests/pipelines.py b/tests/pipelines.py
index ddfbc7a99..7e2895a5c 100644
--- a/tests/pipelines.py
+++ b/tests/pipelines.py
@@ -9,3 +9,9 @@ class ZeroDivisionErrorPipeline(object):
 
     def process_item(self, item, spider):
         return item
+
+
+class ProcessWithZeroDivisionErrorPipiline(object):
+
+    def process_item(self, item, spider):
+        1/0
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 04113ddcf..719c0c60c 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -74,6 +74,14 @@ class DictItemsSpider(TestSpider):
     item_cls = dict
 
 
+class ItemZeroDivisionErrorSpider(TestSpider):
+    custom_settings = {
+        "ITEM_PIPELINES": {
+            "tests.pipelines.ProcessWithZeroDivisionErrorPipiline": 300,
+        }
+    }
+
+
 def start_test_site(debug=False):
     root_dir = os.path.join(tests_datadir, "test_site")
     r = static.File(root_dir)
@@ -95,6 +103,7 @@ class CrawlerRun(object):
         self.respplug = []
         self.reqplug = []
         self.reqdropped = []
+        self.itemerror = []
         self.itemresp = []
         self.signals_catched = {}
         self.spider_class = spider_class
@@ -112,6 +121,7 @@ class CrawlerRun(object):
 
         self.crawler = get_crawler(self.spider_class)
         self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
+        self.crawler.signals.connect(self.item_error, signals.item_error)
         self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
         self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
         self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded)
@@ -136,6 +146,9 @@ class CrawlerRun(object):
         u = urlparse(url)
         return u.path
 
+    def item_error(self, item, response, spider, failure):
+        self.itemerror.append((item, response, spider, failure))
+
     def item_scraped(self, item, spider, response):
         self.itemresp.append((item, response))
 
@@ -175,6 +188,10 @@ class EngineTest(unittest.TestCase):
         self._assert_scheduled_requests(urls_to_visit=7)
         self._assert_dropped_requests()
 
+        self.run = CrawlerRun(ItemZeroDivisionErrorSpider)
+        yield self.run.run()
+        self._assert_items_error()
+
     def _assert_visited_urls(self):
         must_be_visited = ["/", "/redirect", "/redirected",
                            "/item1.html", "/item2.html", "/item999.html"]
@@ -209,6 +226,20 @@ class EngineTest(unittest.TestCase):
             if self.run.getpath(response.url) == '/redirect':
                 self.assertEqual(302, response.status)
 
+    def _assert_items_error(self):
+        self.assertEqual(2, len(self.run.itemerror))
+        for item, response, spider, failure in self.run.itemerror:
+            self.assertEqual(failure.value.__class__, ZeroDivisionError)
+            self.assertEqual(spider, self.run.spider)
+
+            self.assertEqual(item['url'], response.url)
+            if 'item1.html' in item['url']:
+                self.assertEqual('Item 1 name', item['name'])
+                self.assertEqual('100', item['price'])
+            if 'item2.html' in item['url']:
+                self.assertEqual('Item 2 name', item['name'])
+                self.assertEqual('200', item['price'])
+
     def _assert_scraped_items(self):
         self.assertEqual(2, len(self.run.itemresp))
         for item, response in self.run.itemresp:

From 6f5c39d65f3e5d74c292f2143b4f84c7f43b155a Mon Sep 17 00:00:00 2001
From: Oz T <rreloc@yahoo.com>
Date: Wed, 4 Jul 2018 00:22:24 +0300
Subject: [PATCH 1706/2974] Fix for CSV export unnecessary blank lines problem
 on Windows (#3039)

---
 scrapy/exporters.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 07f43b494..695c74fec 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -214,7 +214,8 @@ class CsvItemExporter(BaseItemExporter):
             file,
             line_buffering=False,
             write_through=True,
-            encoding=self.encoding
+            encoding=self.encoding,
+            newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
         ) if six.PY3 else file
         self.csv_writer = csv.writer(self.stream, **kwargs)
         self._headers_not_written = True

From 0b2870634af6cc14191faafcba7d58a5f3cc3016 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 4 Jul 2018 16:14:51 -0300
Subject: [PATCH 1707/2974] Do not inherit from object

---
 tests/test_spider_mw.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py
index 49a60d08b..2565ef7af 100644
--- a/tests/test_spider_mw.py
+++ b/tests/test_spider_mw.py
@@ -14,7 +14,7 @@ from tests.mockserver import MockServer
 TEST_URL = 'http://localhost:8998'
 
 
-class LogExceptionMiddleware(object):
+class LogExceptionMiddleware:
     def process_spider_exception(self, response, exception, spider):
         logging.warn('Middleware: %s exception caught', exception.__class__.__name__)
         return None
@@ -37,7 +37,7 @@ class RecoverySpider(Spider):
         if not response.meta.get('dont_fail'):
             raise ModuleNotFoundError()
 
-class RecoveryMiddleware(object):
+class RecoveryMiddleware:
     def process_spider_exception(self, response, exception, spider):
         logging.warn('Middleware: %s exception caught', exception.__class__.__name__)
         return [

From 0a0e62272e67aaebe29666017e9b0623b81bf369 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 4 Jul 2018 16:19:19 -0300
Subject: [PATCH 1708/2974] New tests

---
 tests/test_spider_mw.py        | 170 --------------
 tests/test_spidermiddleware.py | 405 ++++++++++-----------------------
 2 files changed, 118 insertions(+), 457 deletions(-)
 delete mode 100644 tests/test_spider_mw.py

diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py
deleted file mode 100644
index 2565ef7af..000000000
--- a/tests/test_spider_mw.py
+++ /dev/null
@@ -1,170 +0,0 @@
-
-import logging
-
-from testfixtures import LogCapture
-from twisted.trial.unittest import TestCase
-from twisted.internet import defer
-
-from scrapy import Spider, Request
-from scrapy.utils.test import get_crawler
-from tests.mockserver import MockServer
-
-
-# TEST_URL = 'http://example.org'
-TEST_URL = 'http://localhost:8998'
-
-
-class LogExceptionMiddleware:
-    def process_spider_exception(self, response, exception, spider):
-        logging.warn('Middleware: %s exception caught', exception.__class__.__name__)
-        return None
-
-
-# ================================================================================
-# recover from an exception on a spider callback
-class RecoverySpider(Spider):
-    name = 'RecoverySpider'
-    start_urls = [TEST_URL]
-    custom_settings = {
-        'SPIDER_MIDDLEWARES': {
-            __name__ + '.RecoveryMiddleware': 10,
-        },
-    }
-
-    def parse(self, response):
-        yield {'test': 1}
-        self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail'))
-        if not response.meta.get('dont_fail'):
-            raise ModuleNotFoundError()
-
-class RecoveryMiddleware:
-    def process_spider_exception(self, response, exception, spider):
-        logging.warn('Middleware: %s exception caught', exception.__class__.__name__)
-        return [
-            {'from': 'process_spider_exception'},
-            Request(response.url, meta={'dont_fail': True}, dont_filter=True),
-        ]
-
-
-# ================================================================================
-# (1) exceptions from a spider middleware's process_spider_input method
-class ProcessSpiderInputSpider(Spider):
-    name = 'ProcessSpiderInputSpider'
-    custom_settings = {
-        'SPIDER_MIDDLEWARES': {
-            # spider
-            __name__ + '.LogExceptionMiddleware': 10,
-            __name__ + '.FailProcessSpiderInputMiddleware': 8,
-            __name__ + '.LogExceptionMiddleware': 6,
-            # engine
-        }
-    }
-
-    def start_requests(self):
-        yield Request(TEST_URL, callback=self.parse, errback=self.errback)
-
-    def parse(self, response):
-        return [{'test': 1}, {'test': 2}]
-
-    def errback(self, failure):
-        self.logger.warn('Got a Failure on the Request errback')
-
-
-class FailProcessSpiderInputMiddleware:
-    def process_spider_input(self, response, spider):
-        logging.warn('Middleware: will raise IndexError')
-        raise IndexError()
-
-
-# ================================================================================
-# (2) exceptions from a spider callback (generator)
-class GeneratorCallbackSpider(Spider):
-    name = 'GeneratorCallbackSpider'
-    start_urls = [TEST_URL]
-    custom_settings = {
-        'SPIDER_MIDDLEWARES': {
-            __name__ + '.LogExceptionMiddleware': 10,
-        },
-    }
-
-    def parse(self, response):
-        yield {'test': 1}
-        yield {'test': 2}
-        raise ImportError()
-
-
-# ================================================================================
-# (3) exceptions from a spider callback (not a generator)
-class NotAGeneratorCallbackSpider(Spider):
-    name = 'NotAGeneratorCallbackSpider'
-    start_urls = [TEST_URL]
-    custom_settings = {
-        'SPIDER_MIDDLEWARES': {
-            __name__ + '.LogExceptionMiddleware': 10,
-        },
-    }
-
-    def parse(self, response):
-        return [{'test': 1}, {'test': 1/0}]
-
-
-# ================================================================================
-class TestSpiderMiddleware(TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.mockserver = MockServer()
-        cls.mockserver.__enter__()
-
-    @classmethod
-    def tearDownClass(cls):
-        cls.mockserver.__exit__(None, None, None)
-    
-    @defer.inlineCallbacks
-    def crawl_log(self, spider):
-        crawler = get_crawler(spider)
-        with LogCapture() as log:
-            yield crawler.crawl()
-        raise defer.returnValue(log)
-
-    # @defer.inlineCallbacks
-    # def test_recovery(self):
-    #     """
-    #     Recover from an exception from a spider's callback. The final item count should be 3
-    #     (one from the spider before raising the exception, one from the middleware and one
-    #     from the spider when processing the response that was enqueued from the middleware)
-    #     """
-    #     log = yield self.crawl_log(RecoverySpider)
-    #     self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log))
-    #     self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1)
-    #     self.assertIn("'item_scraped_count': 3", str(log))
-
-    @defer.inlineCallbacks
-    def test_process_spider_input_errback(self):
-        """
-        (1) An exception from the process_spider_input chain should not be caught by the
-        process_spider_exception chain, it should go directly to the Request errback
-        """
-        log1 = yield self.crawl_log(ProcessSpiderInputSpider)
-        self.assertNotIn("Middleware: IndexError exception caught", str(log1))
-        self.assertIn("Middleware: will raise IndexError", str(log1))
-        self.assertIn("Got a Failure on the Request errback", str(log1))
-    
-    @defer.inlineCallbacks
-    def test_generator_callback(self):
-        """
-        (2) An exception from a spider's callback should
-        be caught by the process_spider_exception chain
-        """
-        log2 = yield self.crawl_log(GeneratorCallbackSpider)
-        self.assertIn("Middleware: ImportError exception caught", str(log2))
-        self.assertIn("'item_scraped_count': 2", str(log2))
-    
-    @defer.inlineCallbacks
-    def test_not_a_generator_callback(self):
-        """
-        (3) An exception from a spider's callback should
-        be caught by the process_spider_exception chain
-        """
-        log3 = yield self.crawl_log(NotAGeneratorCallbackSpider)
-        self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3))
-        self.assertNotIn("item_scraped_count", str(log3))
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 109bcc250..2565ef7af 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -5,256 +5,111 @@ from testfixtures import LogCapture
 from twisted.trial.unittest import TestCase
 from twisted.internet import defer
 
-from scrapy.spiders import Spider
-from scrapy.item import Item, Field
-from scrapy.http import Request
+from scrapy import Spider, Request
 from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
 
 
-class TestItem(Item):
-    value = Field()
+# TEST_URL = 'http://example.org'
+TEST_URL = 'http://localhost:8998'
 
 
-class LocalhostSpider(Spider):
-    start_urls = ['http://localhost:8998']  # tests.mockserver.MockServer
-
-
-# ================================================================================
-# exceptions from a spider's parse method
-class BaseExceptionFromParseMethodSpider(LocalhostSpider):
-    custom_settings = {
-        'SPIDER_MIDDLEWARES': {'tests.test_spidermiddleware.CatchExceptionMiddleware': 540}
-    }
-
-
-class NotAGeneratorSpider(BaseExceptionFromParseMethodSpider):
-    """ return value is NOT a generator """
-    name = 'not_a_generator'
-
-    def parse(self, response):
-        raise AssertionError
-
-
-class GeneratorErrorBeforeItemsSpider(BaseExceptionFromParseMethodSpider):
-    """ return value is a generator; the exception is raised
-    before the items are yielded: no items should be scraped """
-    name = 'generator_error_before_items'
-
-    def parse(self, response):
-        raise ValueError
-        for i in range(3):
-            yield {'value': i}
-
-
-class GeneratorErrorAfterItemsSpider(BaseExceptionFromParseMethodSpider):
-    """ return value is a generator; the exception is raised
-    after the items are yielded: 3 items should be scraped """
-    name = 'generator_error_after_items'
-
-    def parse(self, response):
-        for i in range(3):
-            yield {'value': i}
-        raise FloatingPointError
-
-
-class CatchExceptionMiddleware(object):
+class LogExceptionMiddleware:
     def process_spider_exception(self, response, exception, spider):
-        """ catch an exception and log it """
-        logging.warn('{} exception caught'.format(exception.__class__.__name__))
+        logging.warn('Middleware: %s exception caught', exception.__class__.__name__)
         return None
 
 
 # ================================================================================
-# exception from a previous middleware's process_spider_input method
-# process_spider_input is not expected to return an iterable, so there are no
-# separate tests for generator/non-generator implementations
-class FromPreviousMiddlewareInputSpider(LocalhostSpider):
-    name = 'not_a_generator_from_previous_middleware_input'
+# recover from an exception on a spider callback
+class RecoverySpider(Spider):
+    name = 'RecoverySpider'
+    start_urls = [TEST_URL]
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            # engine side
-            'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,
-            'tests.test_spidermiddleware.RaiseExceptionOnInputMiddleware': 545,
-            # spider side
-        }
-    }
-
-    def parse(self, response):
-        return None
-
-
-class RaiseExceptionOnInputMiddleware(object):
-    def process_spider_input(self, response, spider):
-        raise LookupError
-
-
-# ================================================================================
-# exception from a previous middleware's process_spider_output method (not a generator)
-class NotAGeneratorFromPreviousMiddlewareOutputSpider(LocalhostSpider):
-    name = 'not_a_generator_from_previous_middleware_output'
-    custom_settings = {
-        'SPIDER_MIDDLEWARES': {
-            # engine side
-            'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,
-            'tests.test_spidermiddleware.RaiseExceptionOnOutputNotAGeneratorMiddleware': 545,
-            # spider side
-        }
-    }
-
-    def parse(self, response):
-        return [{'value': i} for i in range(3)]
-
-
-class RaiseExceptionOnOutputNotAGeneratorMiddleware(object):
-    def process_spider_output(self, response, result, spider):
-        raise UnicodeError
-
-
-# ================================================================================
-# exception from a previous middleware's process_spider_output method (generator)
-class GeneratorFromPreviousMiddlewareOutputSpider(LocalhostSpider):
-    name = 'generator_from_previous_middleware_output'
-    custom_settings = {
-        'SPIDER_MIDDLEWARES': {
-            # engine side
-            'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,
-            'tests.test_spidermiddleware.RaiseExceptionOnOutputGeneratorMiddleware': 545,
-            # spider side
-        }
-    }
-
-    def parse(self, response):
-        return [{'value': i} for i in range(10, 13)]
-
-
-class RaiseExceptionOnOutputGeneratorMiddleware(object):
-    def process_spider_output(self, response, result, spider):
-        for r in result:
-            yield r
-        raise NameError
-
-
-# ================================================================================
-# do something useful from the exception handler
-class DoSomethingSpider(LocalhostSpider):
-    name = 'do_something'
-    custom_settings = {
-        'SPIDER_MIDDLEWARES': {
-            # engine side
-            'tests.test_spidermiddleware.DoSomethingMiddleware': 540,
-            'tests.test_spidermiddleware.CatchExceptionMiddleware': 545,
-            # spider side
-        }
-    }
-
-    def parse(self, response):
-        yield {'value': response.url}
-        raise ImportError
-
-
-class DoSomethingMiddleware(object):
-    def process_spider_exception(self, response, exception, spider):
-        return [Request('http://localhost:8998?processed=true'), {'value': 10}, TestItem(value='asdf')]
-
-
-# ================================================================================
-# don't catch _InvalidOutput from scrapy's spider middleware manager
-class InvalidReturnValueFromPreviousMiddlewareInputSpider(LocalhostSpider):
-    name = 'invalid_return_value_from_previous_middleware_input'
-    custom_settings = {
-        'SPIDER_MIDDLEWARES': {
-            # engine side
-            'tests.test_spidermiddleware.InvalidReturnValueInputMiddleware': 540,
-            'tests.test_spidermiddleware.CatchExceptionMiddleware': 545,
-            # spider side
-        }
-    }
-
-    def parse(self, response):
-        return None
-
-
-class InvalidReturnValueInputMiddleware(object):
-    def process_spider_input(self, response, spider):
-        return 1.0  # <type 'float'>, not None
-
-
-class InvalidReturnValueFromPreviousMiddlewareOutputSpider(LocalhostSpider):
-    name = 'invalid_return_value_from_previous_middleware_output'
-    custom_settings = {
-        'SPIDER_MIDDLEWARES': {
-            # engine side
-            'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,
-            'tests.test_spidermiddleware.InvalidReturnValueOutputMiddleware': 545,
-            # spider side
-        }
-    }
-
-    def parse(self, response):
-        return None
-
-
-class InvalidReturnValueOutputMiddleware(object):
-    def process_spider_output(self, response, result, spider):
-        return 1  # <type 'int'>, not an iterable
-
-
-# ================================================================================
-# make sure only non already called process_spider_output methods
-# are called if process_spider_exception returns an iterable
-class ExecutionChainSpider(LocalhostSpider):
-    name = 'execution_chain'
-    custom_settings = {
-        'SPIDER_MIDDLEWARES': {
-            # engine side
-            'tests.test_spidermiddleware.ThirdMiddleware': 540,
-            'tests.test_spidermiddleware.SecondMiddleware': 541,
-            'tests.test_spidermiddleware.FirstMiddleware': 542
-            # spider side
+            __name__ + '.RecoveryMiddleware': 10,
         },
     }
 
     def parse(self, response):
-        return None
-
-
-class FirstMiddleware(object):
-    def process_spider_output(self, response, result, spider):
-        for r in result:
-            if isinstance(r, dict):
-                r['handled_by_first_middleware'] = True
-            yield r
+        yield {'test': 1}
+        self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail'))
+        if not response.meta.get('dont_fail'):
+            raise ModuleNotFoundError()
 
+class RecoveryMiddleware:
     def process_spider_exception(self, response, exception, spider):
-        # log exception, handle control to the next middleware's process_spider_exception
-        logging.warn('{} exception caught'.format(exception.__class__.__name__))
-        return None
+        logging.warn('Middleware: %s exception caught', exception.__class__.__name__)
+        return [
+            {'from': 'process_spider_exception'},
+            Request(response.url, meta={'dont_fail': True}, dont_filter=True),
+        ]
 
 
-class SecondMiddleware(object):
-    def process_spider_output(self, response, result, spider):
-        for r in result:
-            if isinstance(r, dict):
-                r['handled_by_second_middleware'] = True
-            yield r
-        raise MemoryError
+# ================================================================================
+# (1) exceptions from a spider middleware's process_spider_input method
+class ProcessSpiderInputSpider(Spider):
+    name = 'ProcessSpiderInputSpider'
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            # spider
+            __name__ + '.LogExceptionMiddleware': 10,
+            __name__ + '.FailProcessSpiderInputMiddleware': 8,
+            __name__ + '.LogExceptionMiddleware': 6,
+            # engine
+        }
+    }
+
+    def start_requests(self):
+        yield Request(TEST_URL, callback=self.parse, errback=self.errback)
+
+    def parse(self, response):
+        return [{'test': 1}, {'test': 2}]
+
+    def errback(self, failure):
+        self.logger.warn('Got a Failure on the Request errback')
 
 
-class ThirdMiddleware(object):
-    def process_spider_output(self, response, result, spider):
-        for r in result:
-            if isinstance(r, dict):
-                r['handled_by_third_middleware'] = True
-            yield r
-
-    def process_spider_exception(self, response, exception, spider):
-        # handle control to the next middleware's process_spider_output
-        return [{'item': i} for i in range(3)]
+class FailProcessSpiderInputMiddleware:
+    def process_spider_input(self, response, spider):
+        logging.warn('Middleware: will raise IndexError')
+        raise IndexError()
 
 
+# ================================================================================
+# (2) exceptions from a spider callback (generator)
+class GeneratorCallbackSpider(Spider):
+    name = 'GeneratorCallbackSpider'
+    start_urls = [TEST_URL]
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            __name__ + '.LogExceptionMiddleware': 10,
+        },
+    }
+
+    def parse(self, response):
+        yield {'test': 1}
+        yield {'test': 2}
+        raise ImportError()
+
+
+# ================================================================================
+# (3) exceptions from a spider callback (not a generator)
+class NotAGeneratorCallbackSpider(Spider):
+    name = 'NotAGeneratorCallbackSpider'
+    start_urls = [TEST_URL]
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            __name__ + '.LogExceptionMiddleware': 10,
+        },
+    }
+
+    def parse(self, response):
+        return [{'test': 1}, {'test': 1/0}]
+
+
+# ================================================================================
 class TestSpiderMiddleware(TestCase):
-
     @classmethod
     def setUpClass(cls):
         cls.mockserver = MockServer()
@@ -263,7 +118,7 @@ class TestSpiderMiddleware(TestCase):
     @classmethod
     def tearDownClass(cls):
         cls.mockserver.__exit__(None, None, None)
-
+    
     @defer.inlineCallbacks
     def crawl_log(self, spider):
         crawler = get_crawler(spider)
@@ -271,69 +126,45 @@ class TestSpiderMiddleware(TestCase):
             yield crawler.crawl()
         raise defer.returnValue(log)
 
-    @defer.inlineCallbacks
-    def test_process_spider_exception_from_parse_method_non_generator(self):
-        # non-generator return value
-        log = yield self.crawl_log(NotAGeneratorSpider)
-        self.assertIn("AssertionError exception caught", str(log))
-        self.assertIn("spider_exceptions/AssertionError", str(log))
+    # @defer.inlineCallbacks
+    # def test_recovery(self):
+    #     """
+    #     Recover from an exception from a spider's callback. The final item count should be 3
+    #     (one from the spider before raising the exception, one from the middleware and one
+    #     from the spider when processing the response that was enqueued from the middleware)
+    #     """
+    #     log = yield self.crawl_log(RecoverySpider)
+    #     self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log))
+    #     self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1)
+    #     self.assertIn("'item_scraped_count': 3", str(log))
 
     @defer.inlineCallbacks
-    def test_process_spider_exception_from_parse_method_generator_no_items(self):
-        # generator return value, no items before the error
-        log = yield self.crawl_log(GeneratorErrorBeforeItemsSpider)
-        self.assertIn("ValueError exception caught", str(log))
-        self.assertIn("spider_exceptions/ValueError", str(log))
-
+    def test_process_spider_input_errback(self):
+        """
+        (1) An exception from the process_spider_input chain should not be caught by the
+        process_spider_exception chain, it should go directly to the Request errback
+        """
+        log1 = yield self.crawl_log(ProcessSpiderInputSpider)
+        self.assertNotIn("Middleware: IndexError exception caught", str(log1))
+        self.assertIn("Middleware: will raise IndexError", str(log1))
+        self.assertIn("Got a Failure on the Request errback", str(log1))
+    
     @defer.inlineCallbacks
-    def test_process_spider_exception_from_parse_method_generator_with_items(self):
-        # generator return value, 3 items before the error
-        log = yield self.crawl_log(GeneratorErrorAfterItemsSpider)
-        self.assertIn("'item_scraped_count': 3", str(log))
-        self.assertIn("FloatingPointError exception caught", str(log))
-        self.assertIn("spider_exceptions/FloatingPointError", str(log))
-
+    def test_generator_callback(self):
+        """
+        (2) An exception from a spider's callback should
+        be caught by the process_spider_exception chain
+        """
+        log2 = yield self.crawl_log(GeneratorCallbackSpider)
+        self.assertIn("Middleware: ImportError exception caught", str(log2))
+        self.assertIn("'item_scraped_count': 2", str(log2))
+    
     @defer.inlineCallbacks
-    def test_process_spider_exception_from_previous_middleware_input(self):
-        log = yield self.crawl_log(FromPreviousMiddlewareInputSpider)
-        self.assertIn("LookupError exception caught", str(log))
-
-    @defer.inlineCallbacks
-    def test_process_spider_exception_from_previous_middleware_output(self):
-        # non-generator output value
-        log = yield self.crawl_log(NotAGeneratorFromPreviousMiddlewareOutputSpider)
-        self.assertNotIn("UnicodeError exception caught", str(log))
-        # generator output value
-        log = yield self.crawl_log(GeneratorFromPreviousMiddlewareOutputSpider)
-        self.assertIn("'item_scraped_count': 3", str(log))
-        self.assertIn("NameError exception caught", str(log))
-
-    @defer.inlineCallbacks
-    def test_process_spider_exception_do_something(self):
-        log = yield self.crawl_log(DoSomethingSpider)
-        self.assertIn("ImportError exception caught", str(log))
-        self.assertIn("{'value': 10}", str(log))
-        self.assertIn("{'value': 'asdf'}", str(log))
-        self.assertIn("{'value': 'http://localhost:8998'}", str(log))
-        self.assertIn("{'value': 'http://localhost:8998?processed=true'}", str(log))
-
-    @defer.inlineCallbacks
-    def test_process_spider_exception_invalid_return_value_previous_middleware(self):
-        """ don't catch _InvalidOutput from middleware """
-        # on middleware's input
-        log1 = yield self.crawl_log(InvalidReturnValueFromPreviousMiddlewareInputSpider)
-        self.assertNotIn("_InvalidOutput exception caught", str(log1))
-        self.assertIn("'spider_exceptions/_InvalidOutput'", str(log1))
-        # on middleware's output
-        log2 = yield self.crawl_log(InvalidReturnValueFromPreviousMiddlewareOutputSpider)
-        self.assertNotIn("_InvalidOutput exception caught", str(log2))
-        self.assertIn("'spider_exceptions/_InvalidOutput'", str(log2))
-
-    @defer.inlineCallbacks
-    def test_process_spider_exception_execution_chain(self):
-        # on middleware's input
-        log = yield self.crawl_log(ExecutionChainSpider)
-        self.assertNotIn("handled_by_first_middleware", str(log))
-        self.assertNotIn("handled_by_second_middleware", str(log))
-        self.assertIn("MemoryError exception caught", str(log))
-        self.assertIn("handled_by_third_middleware", str(log))
+    def test_not_a_generator_callback(self):
+        """
+        (3) An exception from a spider's callback should
+        be caught by the process_spider_exception chain
+        """
+        log3 = yield self.crawl_log(NotAGeneratorCallbackSpider)
+        self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3))
+        self.assertNotIn("item_scraped_count", str(log3))

From d4c7cc848b83c2ec38ea90b76369885daba7375c Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 6 Jul 2018 03:19:43 +0500
Subject: [PATCH 1709/2974] remove backwards compatibility shims for relocated
 modules

---
 .coveragerc                                    |  9 ---------
 conftest.py                                    |  9 +--------
 scrapy/command.py                              |  7 -------
 scrapy/contrib/__init__.py                     |  0
 scrapy/contrib/closespider.py                  |  7 -------
 scrapy/contrib/corestats.py                    |  7 -------
 scrapy/contrib/debug.py                        |  7 -------
 .../contrib/downloadermiddleware/__init__.py   |  0
 .../contrib/downloadermiddleware/ajaxcrawl.py  |  7 -------
 scrapy/contrib/downloadermiddleware/chunked.py |  7 -------
 scrapy/contrib/downloadermiddleware/cookies.py |  7 -------
 .../downloadermiddleware/decompression.py      |  7 -------
 .../downloadermiddleware/defaultheaders.py     |  7 -------
 .../downloadermiddleware/downloadtimeout.py    |  7 -------
 .../contrib/downloadermiddleware/httpauth.py   |  7 -------
 .../contrib/downloadermiddleware/httpcache.py  |  7 -------
 .../downloadermiddleware/httpcompression.py    |  7 -------
 .../contrib/downloadermiddleware/httpproxy.py  |  7 -------
 .../contrib/downloadermiddleware/redirect.py   |  7 -------
 scrapy/contrib/downloadermiddleware/retry.py   |  7 -------
 .../contrib/downloadermiddleware/robotstxt.py  |  7 -------
 scrapy/contrib/downloadermiddleware/stats.py   |  7 -------
 .../contrib/downloadermiddleware/useragent.py  |  7 -------
 scrapy/contrib/exporter/__init__.py            |  8 --------
 scrapy/contrib/feedexport.py                   |  7 -------
 scrapy/contrib/httpcache.py                    |  7 -------
 scrapy/contrib/linkextractors/__init__.py      |  7 -------
 scrapy/contrib/linkextractors/htmlparser.py    |  7 -------
 scrapy/contrib/linkextractors/lxmlhtml.py      |  7 -------
 scrapy/contrib/linkextractors/regex.py         |  7 -------
 scrapy/contrib/linkextractors/sgml.py          |  7 -------
 scrapy/contrib/loader/__init__.py              |  7 -------
 scrapy/contrib/loader/common.py                |  7 -------
 scrapy/contrib/loader/processor.py             |  7 -------
 scrapy/contrib/logstats.py                     |  7 -------
 scrapy/contrib/memdebug.py                     |  7 -------
 scrapy/contrib/memusage.py                     |  7 -------
 scrapy/contrib/pipeline/__init__.py            |  7 -------
 scrapy/contrib/pipeline/files.py               |  7 -------
 scrapy/contrib/pipeline/images.py              |  7 -------
 scrapy/contrib/pipeline/media.py               |  7 -------
 scrapy/contrib/spidermiddleware/__init__.py    |  0
 scrapy/contrib/spidermiddleware/depth.py       |  7 -------
 scrapy/contrib/spidermiddleware/httperror.py   |  7 -------
 scrapy/contrib/spidermiddleware/offsite.py     |  7 -------
 scrapy/contrib/spidermiddleware/referer.py     |  7 -------
 scrapy/contrib/spidermiddleware/urllength.py   |  7 -------
 scrapy/contrib/spiders/__init__.py             |  7 -------
 scrapy/contrib/spiders/crawl.py                |  7 -------
 scrapy/contrib/spiders/feed.py                 |  7 -------
 scrapy/contrib/spiders/init.py                 |  7 -------
 scrapy/contrib/spiders/sitemap.py              |  7 -------
 scrapy/contrib/spiderstate.py                  |  7 -------
 scrapy/contrib/statsmailer.py                  |  7 -------
 scrapy/contrib/throttle.py                     |  7 -------
 scrapy/contrib_exp/__init__.py                 |  0
 .../downloadermiddleware/__init__.py           |  0
 .../downloadermiddleware/decompression.py      |  7 -------
 scrapy/contrib_exp/iterators.py                |  6 ------
 scrapy/dupefilter.py                           |  7 -------
 scrapy/linkextractor.py                        |  7 -------
 scrapy/spider.py                               |  7 -------
 scrapy/squeue.py                               |  7 -------
 scrapy/statscol.py                             |  7 -------
 scrapy/utils/decorator.py                      |  7 -------
 scrapy/utils/deprecate.py                      | 18 ------------------
 66 files changed, 1 insertion(+), 441 deletions(-)
 delete mode 100644 scrapy/command.py
 delete mode 100644 scrapy/contrib/__init__.py
 delete mode 100644 scrapy/contrib/closespider.py
 delete mode 100644 scrapy/contrib/corestats.py
 delete mode 100644 scrapy/contrib/debug.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/__init__.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/ajaxcrawl.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/chunked.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/cookies.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/decompression.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/defaultheaders.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/downloadtimeout.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/httpauth.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/httpcache.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/httpcompression.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/httpproxy.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/redirect.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/retry.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/robotstxt.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/stats.py
 delete mode 100644 scrapy/contrib/downloadermiddleware/useragent.py
 delete mode 100644 scrapy/contrib/exporter/__init__.py
 delete mode 100644 scrapy/contrib/feedexport.py
 delete mode 100644 scrapy/contrib/httpcache.py
 delete mode 100644 scrapy/contrib/linkextractors/__init__.py
 delete mode 100644 scrapy/contrib/linkextractors/htmlparser.py
 delete mode 100644 scrapy/contrib/linkextractors/lxmlhtml.py
 delete mode 100644 scrapy/contrib/linkextractors/regex.py
 delete mode 100644 scrapy/contrib/linkextractors/sgml.py
 delete mode 100644 scrapy/contrib/loader/__init__.py
 delete mode 100644 scrapy/contrib/loader/common.py
 delete mode 100644 scrapy/contrib/loader/processor.py
 delete mode 100644 scrapy/contrib/logstats.py
 delete mode 100644 scrapy/contrib/memdebug.py
 delete mode 100644 scrapy/contrib/memusage.py
 delete mode 100644 scrapy/contrib/pipeline/__init__.py
 delete mode 100644 scrapy/contrib/pipeline/files.py
 delete mode 100644 scrapy/contrib/pipeline/images.py
 delete mode 100644 scrapy/contrib/pipeline/media.py
 delete mode 100644 scrapy/contrib/spidermiddleware/__init__.py
 delete mode 100644 scrapy/contrib/spidermiddleware/depth.py
 delete mode 100644 scrapy/contrib/spidermiddleware/httperror.py
 delete mode 100644 scrapy/contrib/spidermiddleware/offsite.py
 delete mode 100644 scrapy/contrib/spidermiddleware/referer.py
 delete mode 100644 scrapy/contrib/spidermiddleware/urllength.py
 delete mode 100644 scrapy/contrib/spiders/__init__.py
 delete mode 100644 scrapy/contrib/spiders/crawl.py
 delete mode 100644 scrapy/contrib/spiders/feed.py
 delete mode 100644 scrapy/contrib/spiders/init.py
 delete mode 100644 scrapy/contrib/spiders/sitemap.py
 delete mode 100644 scrapy/contrib/spiderstate.py
 delete mode 100644 scrapy/contrib/statsmailer.py
 delete mode 100644 scrapy/contrib/throttle.py
 delete mode 100644 scrapy/contrib_exp/__init__.py
 delete mode 100644 scrapy/contrib_exp/downloadermiddleware/__init__.py
 delete mode 100644 scrapy/contrib_exp/downloadermiddleware/decompression.py
 delete mode 100644 scrapy/contrib_exp/iterators.py
 delete mode 100644 scrapy/dupefilter.py
 delete mode 100644 scrapy/linkextractor.py
 delete mode 100644 scrapy/spider.py
 delete mode 100644 scrapy/squeue.py
 delete mode 100644 scrapy/statscol.py
 delete mode 100644 scrapy/utils/decorator.py

diff --git a/.coveragerc b/.coveragerc
index 3105409ba..aeadccb25 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -7,13 +7,4 @@ omit =
   scrapy/conf.py
   scrapy/stats.py
   scrapy/project.py
-  scrapy/utils/decorator.py
-  scrapy/statscol.py
-  scrapy/squeue.py
   scrapy/log.py
-  scrapy/dupefilter.py
-  scrapy/command.py
-  scrapy/linkextractor.py
-  scrapy/spider.py
-  scrapy/contrib/*
-  scrapy/contrib_exp/*
diff --git a/conftest.py b/conftest.py
index 8b4faf8fc..c733db646 100644
--- a/conftest.py
+++ b/conftest.py
@@ -13,19 +13,12 @@ collect_ignore = [
     "scrapy/conf.py",
     "scrapy/stats.py",
     "scrapy/project.py",
-    "scrapy/utils/decorator.py",
-    "scrapy/statscol.py",
-    "scrapy/squeue.py",
     "scrapy/log.py",
-    "scrapy/dupefilter.py",
-    "scrapy/command.py",
-    "scrapy/linkextractor.py",
-    "scrapy/spider.py",
 
     # not a test, but looks like a test
     "scrapy/utils/testsite.py",
 
-] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp")
+]
 
 if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, 5, 0):
     collect_ignore += _py_files("scrapy/xlib/tx")
diff --git a/scrapy/command.py b/scrapy/command.py
deleted file mode 100644
index 3e1219bbc..000000000
--- a/scrapy/command.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.command` is deprecated, "
-              "use `scrapy.commands` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.commands import *
diff --git a/scrapy/contrib/__init__.py b/scrapy/contrib/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py
deleted file mode 100644
index 9c52c418f..000000000
--- a/scrapy/contrib/closespider.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.closespider` is deprecated, "
-              "use `scrapy.extensions.closespider` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.closespider import *
diff --git a/scrapy/contrib/corestats.py b/scrapy/contrib/corestats.py
deleted file mode 100644
index 2f5354239..000000000
--- a/scrapy/contrib/corestats.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.corestats` is deprecated, "
-              "use `scrapy.extensions.corestats` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.corestats import *
diff --git a/scrapy/contrib/debug.py b/scrapy/contrib/debug.py
deleted file mode 100644
index a38f059ce..000000000
--- a/scrapy/contrib/debug.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.debug` is deprecated, "
-              "use `scrapy.extensions.debug` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.debug import *
diff --git a/scrapy/contrib/downloadermiddleware/__init__.py b/scrapy/contrib/downloadermiddleware/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
deleted file mode 100644
index 90ebc46b6..000000000
--- a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.ajaxcrawl` is deprecated, "
-              "use `scrapy.downloadermiddlewares.ajaxcrawl` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.ajaxcrawl import *
diff --git a/scrapy/contrib/downloadermiddleware/chunked.py b/scrapy/contrib/downloadermiddleware/chunked.py
deleted file mode 100644
index 1322c9083..000000000
--- a/scrapy/contrib/downloadermiddleware/chunked.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.chunked` is deprecated, "
-              "use `scrapy.downloadermiddlewares.chunked` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.chunked import *
diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/contrib/downloadermiddleware/cookies.py
deleted file mode 100644
index bad970690..000000000
--- a/scrapy/contrib/downloadermiddleware/cookies.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.cookies` is deprecated, "
-              "use `scrapy.downloadermiddlewares.cookies` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.cookies import *
diff --git a/scrapy/contrib/downloadermiddleware/decompression.py b/scrapy/contrib/downloadermiddleware/decompression.py
deleted file mode 100644
index a541aa61e..000000000
--- a/scrapy/contrib/downloadermiddleware/decompression.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.decompression` is deprecated, "
-              "use `scrapy.downloadermiddlewares.decompression` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.decompression import *
diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/contrib/downloadermiddleware/defaultheaders.py
deleted file mode 100644
index cf023dc8f..000000000
--- a/scrapy/contrib/downloadermiddleware/defaultheaders.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.defaultheaders` is deprecated, "
-              "use `scrapy.downloadermiddlewares.defaultheaders` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.defaultheaders import *
diff --git a/scrapy/contrib/downloadermiddleware/downloadtimeout.py b/scrapy/contrib/downloadermiddleware/downloadtimeout.py
deleted file mode 100644
index 84bd06acf..000000000
--- a/scrapy/contrib/downloadermiddleware/downloadtimeout.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.downloadtimeout` is deprecated, "
-              "use `scrapy.downloadermiddlewares.downloadtimeout` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.downloadtimeout import *
diff --git a/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/contrib/downloadermiddleware/httpauth.py
deleted file mode 100644
index a37ffa0dc..000000000
--- a/scrapy/contrib/downloadermiddleware/httpauth.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpauth` is deprecated, "
-              "use `scrapy.downloadermiddlewares.httpauth` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.httpauth import *
diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py
deleted file mode 100644
index f5f068204..000000000
--- a/scrapy/contrib/downloadermiddleware/httpcache.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcache` is deprecated, "
-              "use `scrapy.downloadermiddlewares.httpcache` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.httpcache import *
diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/contrib/downloadermiddleware/httpcompression.py
deleted file mode 100644
index 8a52ec50b..000000000
--- a/scrapy/contrib/downloadermiddleware/httpcompression.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcompression` is deprecated, "
-              "use `scrapy.downloadermiddlewares.httpcompression` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.httpcompression import *
diff --git a/scrapy/contrib/downloadermiddleware/httpproxy.py b/scrapy/contrib/downloadermiddleware/httpproxy.py
deleted file mode 100644
index d94d85076..000000000
--- a/scrapy/contrib/downloadermiddleware/httpproxy.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpproxy` is deprecated, "
-              "use `scrapy.downloadermiddlewares.httpproxy` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.httpproxy import *
diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py
deleted file mode 100644
index 824eee8ae..000000000
--- a/scrapy/contrib/downloadermiddleware/redirect.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.redirect` is deprecated, "
-              "use `scrapy.downloadermiddlewares.redirect` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.redirect import *
diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py
deleted file mode 100644
index aafe0f508..000000000
--- a/scrapy/contrib/downloadermiddleware/retry.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.retry` is deprecated, "
-              "use `scrapy.downloadermiddlewares.retry` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.retry import *
diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py
deleted file mode 100644
index 408f760a0..000000000
--- a/scrapy/contrib/downloadermiddleware/robotstxt.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.robotstxt` is deprecated, "
-              "use `scrapy.downloadermiddlewares.robotstxt` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.robotstxt import *
diff --git a/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/contrib/downloadermiddleware/stats.py
deleted file mode 100644
index fa84a8206..000000000
--- a/scrapy/contrib/downloadermiddleware/stats.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.stats` is deprecated, "
-              "use `scrapy.downloadermiddlewares.stats` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.stats import *
diff --git a/scrapy/contrib/downloadermiddleware/useragent.py b/scrapy/contrib/downloadermiddleware/useragent.py
deleted file mode 100644
index 893d5241c..000000000
--- a/scrapy/contrib/downloadermiddleware/useragent.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.useragent` is deprecated, "
-              "use `scrapy.downloadermiddlewares.useragent` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.useragent import *
diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py
deleted file mode 100644
index 12adaaddd..000000000
--- a/scrapy/contrib/exporter/__init__.py
+++ /dev/null
@@ -1,8 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.exporter` is deprecated, "
-              "use `scrapy.exporters` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.exporters import *
-from scrapy.exporters import PythonItemExporter
diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py
deleted file mode 100644
index 19651998a..000000000
--- a/scrapy/contrib/feedexport.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.feedexport` is deprecated, "
-              "use `scrapy.extensions.feedexport` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.feedexport import *
diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py
deleted file mode 100644
index 196372fcb..000000000
--- a/scrapy/contrib/httpcache.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.httpcache` is deprecated, "
-              "use `scrapy.extensions.httpcache` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.httpcache import *
diff --git a/scrapy/contrib/linkextractors/__init__.py b/scrapy/contrib/linkextractors/__init__.py
deleted file mode 100644
index 976658df3..000000000
--- a/scrapy/contrib/linkextractors/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.linkextractors` is deprecated, "
-              "use `scrapy.linkextractors` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.linkextractors import *
diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/contrib/linkextractors/htmlparser.py
deleted file mode 100644
index ff03da98f..000000000
--- a/scrapy/contrib/linkextractors/htmlparser.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.linkextractors.htmlparser` is deprecated, "
-              "use `scrapy.linkextractors.htmlparser` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.linkextractors.htmlparser import *
diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py
deleted file mode 100644
index fc2b7de3c..000000000
--- a/scrapy/contrib/linkextractors/lxmlhtml.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.linkextractors.lxmlhtml` is deprecated, "
-              "use `scrapy.linkextractors.lxmlhtml` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.linkextractors.lxmlhtml import *
diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py
deleted file mode 100644
index 97bda29c1..000000000
--- a/scrapy/contrib/linkextractors/regex.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.linkextractors.regex` is deprecated, "
-              "use `scrapy.linkextractors.regex` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.linkextractors.regex import *
diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py
deleted file mode 100644
index a5a598208..000000000
--- a/scrapy/contrib/linkextractors/sgml.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.linkextractors.sgml` is deprecated, "
-              "use `scrapy.linkextractors.sgml` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.linkextractors.sgml import *
diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py
deleted file mode 100644
index 2b9453e18..000000000
--- a/scrapy/contrib/loader/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.loader` is deprecated, "
-              "use `scrapy.loader` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.loader import *
diff --git a/scrapy/contrib/loader/common.py b/scrapy/contrib/loader/common.py
deleted file mode 100644
index a59b2b7b1..000000000
--- a/scrapy/contrib/loader/common.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.loader.common` is deprecated, "
-              "use `scrapy.loader.common` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.loader.common import *
diff --git a/scrapy/contrib/loader/processor.py b/scrapy/contrib/loader/processor.py
deleted file mode 100644
index da7e484a5..000000000
--- a/scrapy/contrib/loader/processor.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.loader.processor` is deprecated, "
-              "use `scrapy.loader.processors` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.loader.processors import *
diff --git a/scrapy/contrib/logstats.py b/scrapy/contrib/logstats.py
deleted file mode 100644
index 62bc9b860..000000000
--- a/scrapy/contrib/logstats.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.logstats` is deprecated, "
-              "use `scrapy.extensions.logstats` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.logstats import *
diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py
deleted file mode 100644
index 4f6e4760e..000000000
--- a/scrapy/contrib/memdebug.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.memdebug` is deprecated, "
-              "use `scrapy.extensions.memdebug` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.memdebug import *
diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py
deleted file mode 100644
index e13bd78f3..000000000
--- a/scrapy/contrib/memusage.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.memusage` is deprecated, "
-              "use `scrapy.extensions.memusage` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.memusage import *
diff --git a/scrapy/contrib/pipeline/__init__.py b/scrapy/contrib/pipeline/__init__.py
deleted file mode 100644
index aedf34a3f..000000000
--- a/scrapy/contrib/pipeline/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.pipeline` is deprecated, "
-              "use `scrapy.pipelines` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.pipelines import *
diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py
deleted file mode 100644
index cd1238b5d..000000000
--- a/scrapy/contrib/pipeline/files.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.pipeline.files` is deprecated, "
-              "use `scrapy.pipelines.files` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.pipelines.files import *
diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py
deleted file mode 100644
index 4f5ce4c40..000000000
--- a/scrapy/contrib/pipeline/images.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.pipeline.images` is deprecated, "
-              "use `scrapy.pipelines.images` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.pipelines.images import *
diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py
deleted file mode 100644
index 4b4fea560..000000000
--- a/scrapy/contrib/pipeline/media.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.pipeline.media` is deprecated, "
-              "use `scrapy.pipelines.media` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.pipelines.media import *
diff --git a/scrapy/contrib/spidermiddleware/__init__.py b/scrapy/contrib/spidermiddleware/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py
deleted file mode 100644
index 718803148..000000000
--- a/scrapy/contrib/spidermiddleware/depth.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spidermiddleware.depth` is deprecated, "
-              "use `scrapy.spidermiddlewares.depth` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spidermiddlewares.depth import *
diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py
deleted file mode 100644
index e39fb3f56..000000000
--- a/scrapy/contrib/spidermiddleware/httperror.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spidermiddleware.httperror` is deprecated, "
-              "use `scrapy.spidermiddlewares.httperror` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spidermiddlewares.httperror import *
diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py
deleted file mode 100644
index a5ed9ea7e..000000000
--- a/scrapy/contrib/spidermiddleware/offsite.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spidermiddleware.offsite` is deprecated, "
-              "use `scrapy.spidermiddlewares.offsite` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spidermiddlewares.offsite import *
diff --git a/scrapy/contrib/spidermiddleware/referer.py b/scrapy/contrib/spidermiddleware/referer.py
deleted file mode 100644
index fdf8d6659..000000000
--- a/scrapy/contrib/spidermiddleware/referer.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spidermiddleware.referer` is deprecated, "
-              "use `scrapy.spidermiddlewares.referer` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spidermiddlewares.referer import *
diff --git a/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/contrib/spidermiddleware/urllength.py
deleted file mode 100644
index 5e51add59..000000000
--- a/scrapy/contrib/spidermiddleware/urllength.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spidermiddleware.urllength` is deprecated, "
-              "use `scrapy.spidermiddlewares.urllength` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spidermiddlewares.urllength import *
diff --git a/scrapy/contrib/spiders/__init__.py b/scrapy/contrib/spiders/__init__.py
deleted file mode 100644
index 56780533b..000000000
--- a/scrapy/contrib/spiders/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spiders` is deprecated, "
-              "use `scrapy.spiders` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spiders import *
diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py
deleted file mode 100644
index d20a8bb16..000000000
--- a/scrapy/contrib/spiders/crawl.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spiders.crawl` is deprecated, "
-              "use `scrapy.spiders.crawl` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spiders.crawl import *
diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py
deleted file mode 100644
index 5eea9a062..000000000
--- a/scrapy/contrib/spiders/feed.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spiders.feed` is deprecated, "
-              "use `scrapy.spiders.feed` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spiders.feed import *
diff --git a/scrapy/contrib/spiders/init.py b/scrapy/contrib/spiders/init.py
deleted file mode 100644
index 6d1ec0aa9..000000000
--- a/scrapy/contrib/spiders/init.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spiders.init` is deprecated, "
-              "use `scrapy.spiders.init` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spiders.init import *
diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py
deleted file mode 100644
index 2ad231fd8..000000000
--- a/scrapy/contrib/spiders/sitemap.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spiders.sitemap` is deprecated, "
-              "use `scrapy.spiders.sitemap` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spiders.sitemap import *
diff --git a/scrapy/contrib/spiderstate.py b/scrapy/contrib/spiderstate.py
deleted file mode 100644
index 06afc8bfc..000000000
--- a/scrapy/contrib/spiderstate.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spiderstate` is deprecated, "
-              "use `scrapy.extensions.spiderstate` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.spiderstate import *
diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py
deleted file mode 100644
index f9c9a37f5..000000000
--- a/scrapy/contrib/statsmailer.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.statsmailer` is deprecated, "
-              "use `scrapy.extensions.statsmailer` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.statsmailer import *
diff --git a/scrapy/contrib/throttle.py b/scrapy/contrib/throttle.py
deleted file mode 100644
index d5c234871..000000000
--- a/scrapy/contrib/throttle.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.throttle` is deprecated, "
-              "use `scrapy.extensions.throttle` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.throttle import *
diff --git a/scrapy/contrib_exp/__init__.py b/scrapy/contrib_exp/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/scrapy/contrib_exp/downloadermiddleware/__init__.py b/scrapy/contrib_exp/downloadermiddleware/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py
deleted file mode 100644
index 1f8490587..000000000
--- a/scrapy/contrib_exp/downloadermiddleware/decompression.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib_exp.downloadermiddleware.decompression` is deprecated, "
-              "use `scrapy.downloadermiddlewares.decompression` instead",
-    ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware
diff --git a/scrapy/contrib_exp/iterators.py b/scrapy/contrib_exp/iterators.py
deleted file mode 100644
index c59f47bcc..000000000
--- a/scrapy/contrib_exp/iterators.py
+++ /dev/null
@@ -1,6 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib_exp.iterators` is deprecated, use `scrapy.utils.iterators` instead",
-    ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.utils.iterators import xmliter_lxml
diff --git a/scrapy/dupefilter.py b/scrapy/dupefilter.py
deleted file mode 100644
index 232d96288..000000000
--- a/scrapy/dupefilter.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.dupefilter` is deprecated, "
-              "use `scrapy.dupefilters` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.dupefilters import *
diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py
deleted file mode 100644
index b744aff8e..000000000
--- a/scrapy/linkextractor.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.linkextractor` is deprecated, "
-              "use `scrapy.linkextractors` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.linkextractors import *
diff --git a/scrapy/spider.py b/scrapy/spider.py
deleted file mode 100644
index 56a5a0a0b..000000000
--- a/scrapy/spider.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.spider` is deprecated, "
-              "use `scrapy.spiders` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spiders import *
diff --git a/scrapy/squeue.py b/scrapy/squeue.py
deleted file mode 100644
index a4a3f4238..000000000
--- a/scrapy/squeue.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.squeue` is deprecated, "
-              "use `scrapy.squeues` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.squeues import *
diff --git a/scrapy/statscol.py b/scrapy/statscol.py
deleted file mode 100644
index b4ddcce28..000000000
--- a/scrapy/statscol.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.statscol` is deprecated, "
-              "use `scrapy.statscollectors` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.statscollectors import *
diff --git a/scrapy/utils/decorator.py b/scrapy/utils/decorator.py
deleted file mode 100644
index e8c8eae39..000000000
--- a/scrapy/utils/decorator.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.utils.decorator` is deprecated, "
-              "use `scrapy.utils.decorators` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.utils.decorators import *
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index f76161a68..8c72cc556 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -124,25 +124,7 @@ def _clspath(cls, forced=None):
 
 
 DEPRECATION_RULES = [
-    ('scrapy.contrib_exp.downloadermiddleware.decompression.', 'scrapy.downloadermiddlewares.decompression.'),
-    ('scrapy.contrib_exp.iterators.', 'scrapy.utils.iterators.'),
-    ('scrapy.contrib.downloadermiddleware.', 'scrapy.downloadermiddlewares.'),
-    ('scrapy.contrib.exporter.', 'scrapy.exporters.'),
-    ('scrapy.contrib.linkextractors.', 'scrapy.linkextractors.'),
-    ('scrapy.contrib.loader.processor.', 'scrapy.loader.processors.'),
-    ('scrapy.contrib.loader.', 'scrapy.loader.'),
-    ('scrapy.contrib.pipeline.', 'scrapy.pipelines.'),
-    ('scrapy.contrib.spidermiddleware.', 'scrapy.spidermiddlewares.'),
-    ('scrapy.contrib.spiders.', 'scrapy.spiders.'),
-    ('scrapy.contrib.', 'scrapy.extensions.'),
-    ('scrapy.command.', 'scrapy.commands.'),
-    ('scrapy.dupefilter.', 'scrapy.dupefilters.'),
-    ('scrapy.linkextractor.', 'scrapy.linkextractors.'),
     ('scrapy.telnet.', 'scrapy.extensions.telnet.'),
-    ('scrapy.spider.', 'scrapy.spiders.'),
-    ('scrapy.squeue.', 'scrapy.squeues.'),
-    ('scrapy.statscol.', 'scrapy.statscollectors.'),
-    ('scrapy.utils.decorator.', 'scrapy.utils.decorators.'),
     ('scrapy.spidermanager.SpiderManager', 'scrapy.spiderloader.SpiderLoader'),
 ]
 

From 36453348fad9babc96558ab10af9b2942eb5431e Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 6 Jul 2018 03:23:37 +0500
Subject: [PATCH 1710/2974] remove ancient modules kept only for error messages

---
 .coveragerc       |  2 --
 conftest.py       |  2 --
 scrapy/project.py | 17 -----------------
 scrapy/stats.py   |  8 --------
 4 files changed, 29 deletions(-)
 delete mode 100644 scrapy/project.py
 delete mode 100644 scrapy/stats.py

diff --git a/.coveragerc b/.coveragerc
index aeadccb25..1fde07e7e 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -5,6 +5,4 @@ omit =
   tests/*
   scrapy/xlib/*
   scrapy/conf.py
-  scrapy/stats.py
-  scrapy/project.py
   scrapy/log.py
diff --git a/conftest.py b/conftest.py
index c733db646..2d015f5e9 100644
--- a/conftest.py
+++ b/conftest.py
@@ -11,8 +11,6 @@ def _py_files(folder):
 collect_ignore = [
     # deprecated or moved modules
     "scrapy/conf.py",
-    "scrapy/stats.py",
-    "scrapy/project.py",
     "scrapy/log.py",
 
     # not a test, but looks like a test
diff --git a/scrapy/project.py b/scrapy/project.py
deleted file mode 100644
index d8973a6c7..000000000
--- a/scrapy/project.py
+++ /dev/null
@@ -1,17 +0,0 @@
-
-"""
-Obsolete module, kept for giving a meaningful error message when trying to
-import.
-"""
-
-raise ImportError("""scrapy.project usage has become obsolete.
-
-If you want to get the Scrapy crawler from your extension, middleware or
-pipeline implement the `from_crawler` class method (or look up for extending
-components that have already done it, such as spiders).
-
-For example:
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler)""")
diff --git a/scrapy/stats.py b/scrapy/stats.py
deleted file mode 100644
index 710601430..000000000
--- a/scrapy/stats.py
+++ /dev/null
@@ -1,8 +0,0 @@
-
-"""
-Obsolete module, kept for giving a meaningful error message when trying to
-import.
-"""
-
-raise ImportError("scrapy.stats usage has become obsolete, use "
-                  "`crawler.stats` attribute instead")

From f531b66822491140740a6d86af2f3f11f0443d38 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 6 Jul 2018 03:28:01 +0500
Subject: [PATCH 1711/2974] SpiderManager shim is removed

---
 scrapy/interfaces.py      | 4 ----
 scrapy/spidermanager.py   | 7 -------
 scrapy/utils/deprecate.py | 1 -
 3 files changed, 12 deletions(-)
 delete mode 100644 scrapy/spidermanager.py

diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py
index eb93c6f7e..89ad2b14f 100644
--- a/scrapy/interfaces.py
+++ b/scrapy/interfaces.py
@@ -16,7 +16,3 @@ class ISpiderLoader(Interface):
     def find_by_request(request):
         """Return the list of spiders names that can handle the given request"""
 
-
-# ISpiderManager is deprecated, don't use it!
-# An alias is kept for backwards compatibility.
-ISpiderManager = ISpiderLoader
diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py
deleted file mode 100644
index 220257bb1..000000000
--- a/scrapy/spidermanager.py
+++ /dev/null
@@ -1,7 +0,0 @@
-"""
-Backwards compatibility shim. Use scrapy.spiderloader instead.
-"""
-from scrapy.spiderloader import SpiderLoader
-from scrapy.utils.deprecate import create_deprecated_class
-
-SpiderManager = create_deprecated_class('SpiderManager', SpiderLoader)
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index 8c72cc556..2d3db431d 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -125,7 +125,6 @@ def _clspath(cls, forced=None):
 
 DEPRECATION_RULES = [
     ('scrapy.telnet.', 'scrapy.extensions.telnet.'),
-    ('scrapy.spidermanager.SpiderManager', 'scrapy.spiderloader.SpiderLoader'),
 ]
 
 

From 722e1afcdb337bf11652167f02435c81fc68ecfb Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 9 Jul 2018 12:21:19 +0300
Subject: [PATCH 1712/2974] Update ancient pytest on python 3

2.9 gives collection errors on python 3.7 due to PEP 479.
---
 tests/requirements-py3.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 51a25f5e5..8d9ce5231 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,6 +1,6 @@
-pytest==2.9.2
+pytest==3.6.3
 pytest-twisted
-pytest-cov==2.2.1
+pytest-cov==2.5.1
 testfixtures
 jmespath
 leveldb

From 17e9914b8a12e5a96cc40016b74f301fb9835cbf Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 9 Jul 2018 12:26:09 +0300
Subject: [PATCH 1713/2974] Catch SyntaxError as well when importing manhole

Also give a more detailed reason why telnet is not enabled (for the
future).
---
 requirements-py3.txt        | 2 +-
 scrapy/extensions/telnet.py | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/requirements-py3.txt b/requirements-py3.txt
index b941fd867..b38c4cc09 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -1,4 +1,4 @@
-git+https://github.com/lopuhin/twisted.git@9384-remove-async-param
+Twisted>=17.9.0
 lxml>=3.2.4
 pyOpenSSL>=0.13.1
 cssselect>=0.9
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index e78afa1fc..7cc8f823a 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -12,7 +12,7 @@ try:
     from twisted.conch import manhole, telnet
     from twisted.conch.insults import insults
     TWISTED_CONCH_AVAILABLE = True
-except ImportError:
+except (ImportError, SyntaxError):
     TWISTED_CONCH_AVAILABLE = False
 
 from scrapy.exceptions import NotConfigured
@@ -40,7 +40,8 @@ class TelnetConsole(protocol.ServerFactory):
         if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'):
             raise NotConfigured
         if not TWISTED_CONCH_AVAILABLE:
-            raise NotConfigured
+            raise NotConfigured('TelnetConsole not enabled: failed to import '
+                                'required twisted modules.')
         self.crawler = crawler
         self.noisy = False
         self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]

From cf9399acc149cf5eafb2d00d310416ab2ba185e5 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 9 Jul 2018 12:26:56 +0300
Subject: [PATCH 1714/2974] Use python 3.7 on travis

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 29f9f0065..f6ea670ae 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,7 +23,7 @@ matrix:
       env: TOXENV=py36
     - python: 3.6
       env: TOXENV=docs
-    - python: 3.7-dev
+    - python: 3.7
       env: TOXENV=py37
 install:
   - |

From 2773fe09e4b4fac51dbc06725f1fafb2e5d9a271 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 9 Jul 2018 12:36:58 +0300
Subject: [PATCH 1715/2974] Make "docs" the last build, even though it still
 uses python3.6 for now

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index f6ea670ae..88c72b08e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,10 +21,10 @@ matrix:
       env: TOXENV=py35
     - python: 3.6
       env: TOXENV=py36
-    - python: 3.6
-      env: TOXENV=docs
     - python: 3.7
       env: TOXENV=py37
+    - python: 3.6
+      env: TOXENV=docs
 install:
   - |
       if [ "$TOXENV" = "pypy" ]; then

From f4f39057cbbfa4daf66f82061e57101b88d88d05 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 9 Jul 2018 12:46:45 +0300
Subject: [PATCH 1716/2974] Make csviter work on python 3.7

PEP 479 does not allow for StopIteration in generators.  Instead,
handle it explicitly, also use a for loop which looks simpler.
---
 scrapy/utils/iterators.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 73857b410..a12e14005 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -98,8 +98,9 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     """
 
     encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8'
-    def _getrow(csv_r):
-        return [to_unicode(field, encoding) for field in next(csv_r)]
+
+    def row_to_unicode(row_):
+        return [to_unicode(field, encoding) for field in row_]
 
     # Python 3 csv reader input object needs to return strings
     if six.PY3:
@@ -113,10 +114,14 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     csv_r = csv.reader(lines, **kwargs)
 
     if not headers:
-        headers = _getrow(csv_r)
+        try:
+            row = next(csv_r)
+        except StopIteration:
+            return
+        headers = row_to_unicode(row)
 
-    while True:
-        row = _getrow(csv_r)
+    for row in csv_r:
+        row = row_to_unicode(row)
         if len(row) != len(headers):
             logger.warning("ignoring row %(csvlnum)d (length: %(csvrow)d, "
                            "should be: %(csvheader)d)",

From b3cd12dc48592fb8b1d4c6883315fe2b341ca5c2 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 9 Jul 2018 12:53:40 +0300
Subject: [PATCH 1717/2974] Try to get python3.7 by using xenial base and sudo

See https://github.com/travis-ci/travis-ci/issues/9815#issuecomment-401756442
and https://github.com/travis-ci/travis-ci/issues/9815#issuecomment-402045581
---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 88c72b08e..4218d13bf 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,6 +23,8 @@ matrix:
       env: TOXENV=py36
     - python: 3.7
       env: TOXENV=py37
+      dist: xenial
+      sudo: true
     - python: 3.6
       env: TOXENV=docs
 install:

From 92b504eae5379dadade2d78efba4e54b201cbd93 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 9 Jul 2018 13:43:36 +0300
Subject: [PATCH 1718/2974] Fix telnet warnings in tests

Disable telnet console if it's not available, else we'll get an extra
warning about failure to enable it, and tests will fail.
---
 tests/test_crawler.py   | 6 ++++--
 tests/test_utils_log.py | 7 ++++++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index d3b80f460..6a8e11363 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,5 +1,4 @@
 import logging
-import os
 import tempfile
 import warnings
 import unittest
@@ -14,8 +13,9 @@ from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.log import configure_logging, get_scrapy_root_handler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
-from scrapy.utils.test import get_crawler
 from scrapy.extensions.throttle import AutoThrottle
+from scrapy.extensions import telnet
+
 
 class BaseCrawlerTest(unittest.TestCase):
 
@@ -100,6 +100,8 @@ class CrawlerLoggingTestCase(unittest.TestCase):
                 custom_settings = {
                     'LOG_LEVEL': 'INFO',
                     'LOG_FILE': log_file.name,
+                    # disable telnet if not available to avoid an extra warning
+                    'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
                 }
 
             configure_logging()
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 45527b03b..742e04803 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -10,6 +10,7 @@ from twisted.python.failure import Failure
 from scrapy.utils.log import (failure_to_exc_info, TopLevelFormatter,
                               LogCounterHandler, StreamLogger)
 from scrapy.utils.test import get_crawler
+from scrapy.extensions import telnet
 
 
 class FailureToExcInfoTest(unittest.TestCase):
@@ -65,10 +66,14 @@ class TopLevelFormatterTest(unittest.TestCase):
 class LogCounterHandlerTest(unittest.TestCase):
 
     def setUp(self):
+        settings = {'LOG_LEVEL': 'WARNING'}
+        if not telnet.TWISTED_CONCH_AVAILABLE:
+            # disable it to avoid the extra warning
+            settings['TELNETCONSOLE_ENABLED'] = False
         self.logger = logging.getLogger('test')
         self.logger.setLevel(logging.NOTSET)
         self.logger.propagate = False
-        self.crawler = get_crawler(settings_dict={'LOG_LEVEL': 'WARNING'})
+        self.crawler = get_crawler(settings_dict=settings)
         self.handler = LogCounterHandler(self.crawler)
         self.logger.addHandler(self.handler)
 

From 4f6778aa7332aecd15f7672a0852d1f49596969b Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Mon, 9 Jul 2018 17:16:31 +0000
Subject: [PATCH 1719/2974] Remove deprecated CrawlerSettings class and
 Settings attributes

---
 docs/news.rst                   |  9 +++++
 scrapy/settings/__init__.py     | 48 --------------------------
 tests/test_settings/__init__.py | 60 +--------------------------------
 3 files changed, 10 insertions(+), 107 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 1b8d121a1..633e5c72f 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,15 @@
 Release notes
 =============
 
+Scrapy 1.6.0 (unreleased)
+-------------------------
+
+Cleanups
+~~~~~~~~
+
+* Remove deprecated ``CrawlerSettings`` class.
+* Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes.
+
 Scrapy 1.5.0 (2017-12-29)
 -------------------------
 
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 28446a372..7d6d20164 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -6,7 +6,6 @@ from collections import MutableMapping
 from importlib import import_module
 from pprint import pformat
 
-from scrapy.utils.deprecate import create_deprecated_class
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 from . import default_settings
@@ -405,30 +404,6 @@ class BaseSettings(MutableMapping):
         else:
             p.text(pformat(self.copy_to_dict()))
 
-    @property
-    def overrides(self):
-        warnings.warn("`Settings.overrides` attribute is deprecated and won't "
-                      "be supported in Scrapy 0.26, use "
-                      "`Settings.set(name, value, priority='cmdline')` instead",
-                      category=ScrapyDeprecationWarning, stacklevel=2)
-        try:
-            o = self._overrides
-        except AttributeError:
-            self._overrides = o = _DictProxy(self, 'cmdline')
-        return o
-
-    @property
-    def defaults(self):
-        warnings.warn("`Settings.defaults` attribute is deprecated and won't "
-                      "be supported in Scrapy 0.26, use "
-                      "`Settings.set(name, value, priority='default')` instead",
-                      category=ScrapyDeprecationWarning, stacklevel=2)
-        try:
-            o = self._defaults
-        except AttributeError:
-            self._defaults = o = _DictProxy(self, 'default')
-        return o
-
 
 class _DictProxy(MutableMapping):
 
@@ -479,29 +454,6 @@ class Settings(BaseSettings):
         self.update(values, priority)
 
 
-class CrawlerSettings(Settings):
-
-    def __init__(self, settings_module=None, **kw):
-        self.settings_module = settings_module
-        Settings.__init__(self, **kw)
-
-    def __getitem__(self, opt_name):
-        if opt_name in self.overrides:
-            return self.overrides[opt_name]
-        if self.settings_module and hasattr(self.settings_module, opt_name):
-            return getattr(self.settings_module, opt_name)
-        if opt_name in self.defaults:
-            return self.defaults[opt_name]
-        return Settings.__getitem__(self, opt_name)
-
-    def __str__(self):
-        return "<CrawlerSettings module=%r>" % self.settings_module
-
-CrawlerSettings = create_deprecated_class(
-    'CrawlerSettings', CrawlerSettings,
-    new_class_path='scrapy.settings.Settings')
-
-
 def iter_default_settings():
     """Return the default settings as an iterator of (name, value) tuples"""
     for name in dir(default_settings):
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 863684075..1dbacbea3 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -3,8 +3,7 @@ import unittest
 import warnings
 
 from scrapy.settings import (BaseSettings, Settings, SettingsAttribute,
-                             CrawlerSettings, SETTINGS_PRIORITIES,
-                             get_settings_priority)
+                             SETTINGS_PRIORITIES, get_settings_priority)
 from tests import mock
 from . import default_settings
 
@@ -341,35 +340,6 @@ class BaseSettingsTest(unittest.TestCase):
         self.assertTrue(frozencopy.frozen)
         self.assertIsNot(frozencopy, self.settings)
 
-    def test_deprecated_attribute_overrides(self):
-        self.settings.set('BAR', 'fuz', priority='cmdline')
-        with warnings.catch_warnings(record=True) as w:
-            self.settings.overrides['BAR'] = 'foo'
-            self.assertIn("Settings.overrides", str(w[0].message))
-            self.assertEqual(self.settings.get('BAR'), 'foo')
-            self.assertEqual(self.settings.overrides.get('BAR'), 'foo')
-            self.assertIn('BAR', self.settings.overrides)
-
-            self.settings.overrides.update(BAR='bus')
-            self.assertEqual(self.settings.get('BAR'), 'bus')
-            self.assertEqual(self.settings.overrides.get('BAR'), 'bus')
-
-            self.settings.overrides.setdefault('BAR', 'fez')
-            self.assertEqual(self.settings.get('BAR'), 'bus')
-
-            self.settings.overrides.setdefault('FOO', 'fez')
-            self.assertEqual(self.settings.get('FOO'), 'fez')
-            self.assertEqual(self.settings.overrides.get('FOO'), 'fez')
-
-    def test_deprecated_attribute_defaults(self):
-        self.settings.set('BAR', 'fuz', priority='default')
-        with warnings.catch_warnings(record=True) as w:
-            self.settings.defaults['BAR'] = 'foo'
-            self.assertIn("Settings.defaults", str(w[0].message))
-            self.assertEqual(self.settings.get('BAR'), 'foo')
-            self.assertEqual(self.settings.defaults.get('BAR'), 'foo')
-            self.assertIn('BAR', self.settings.defaults)
-
 
 class SettingsTest(unittest.TestCase):
 
@@ -422,33 +392,5 @@ class SettingsTest(unittest.TestCase):
         self.assertEqual(mydict['key'], 'val')
 
 
-class CrawlerSettingsTest(unittest.TestCase):
-
-    def test_deprecated_crawlersettings(self):
-        def _get_settings(settings_dict=None):
-            settings_module = type('SettingsModuleMock', (object,), settings_dict or {})
-            return CrawlerSettings(settings_module)
-
-        with warnings.catch_warnings(record=True) as w:
-            settings = _get_settings()
-            self.assertIn("CrawlerSettings is deprecated", str(w[0].message))
-
-            # test_global_defaults
-            self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 180)
-
-            # test_defaults
-            settings.defaults['DOWNLOAD_TIMEOUT'] = '99'
-            self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 99)
-
-            # test_settings_module
-            settings = _get_settings({'DOWNLOAD_TIMEOUT': '3'})
-            self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 3)
-
-            # test_overrides
-            settings = _get_settings({'DOWNLOAD_TIMEOUT': '3'})
-            settings.overrides['DOWNLOAD_TIMEOUT'] = '15'
-            self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 15)
-
-
 if __name__ == "__main__":
     unittest.main()

From 9428a4a3aa4c679c15eb2c606de7b49ad832ee6e Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 9 Jul 2018 21:03:26 +0300
Subject: [PATCH 1720/2974] More visible telnet conch message

Capture traceback when trying to import required twisted modules,
print it in case telnet is enabled, and mention settings variable
that can be used to supress the message.
Thanks @kmike!
---
 scrapy/extensions/telnet.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 7cc8f823a..3024ddfaa 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/telnetconsole.rst
 
 import pprint
 import logging
+import traceback
 
 from twisted.internet import protocol
 try:
@@ -13,6 +14,7 @@ try:
     from twisted.conch.insults import insults
     TWISTED_CONCH_AVAILABLE = True
 except (ImportError, SyntaxError):
+    _TWISTED_CONCH_TRACEBACK = traceback.format_exc()
     TWISTED_CONCH_AVAILABLE = False
 
 from scrapy.exceptions import NotConfigured
@@ -40,8 +42,9 @@ class TelnetConsole(protocol.ServerFactory):
         if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'):
             raise NotConfigured
         if not TWISTED_CONCH_AVAILABLE:
-            raise NotConfigured('TelnetConsole not enabled: failed to import '
-                                'required twisted modules.')
+            raise NotConfigured(
+                'TELNETCONSOLE_ENABLED setting is True but required twisted '
+                'modules failed to import:\n' + _TWISTED_CONCH_TRACEBACK)
         self.crawler = crawler
         self.noisy = False
         self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]

From c86213317daf25aec04f3afc69c327a43033987f Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 12 Jul 2018 02:10:24 +0500
Subject: [PATCH 1721/2974] 1.5.1 release notes

---
 docs/news.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 633e5c72f..01016e2e6 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -12,6 +12,22 @@ Cleanups
 * Remove deprecated ``CrawlerSettings`` class.
 * Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes.
 
+
+Scrapy 1.5.1 (2018-07-12)
+-------------------------
+
+This is a maintenance release with important bug fixes, but no new features:
+
+* ``O(N^2)`` gzip decompression issue which affected Python 3 and PyPy
+  is fixed (:issue:`3281`);
+* skipping of TLS validation errors is improved (:issue:`3166`);
+* Ctrl-C handling is fixed in Python 3.5+ (:issue:`3096`);
+* testing fixes (:issue:`3092`, :issue:`3263`);
+* documentation improvements (:issue:`3058`, :issue:`3059`, :issue:`3089`,
+  :issue:`3123`, :issue:`3127`, :issue:`3189`, :issue:`3224`, :issue:`3280`,
+  :issue:`3279`, :issue:`3201`, :issue:`3260`, :issue:`3284`, :issue:`3298`,
+  :issue:`3294`).
+
 Scrapy 1.5.0 (2017-12-29)
 -------------------------
 

From c61e8a617f1291bfcbe56d54a2f80f8fb79b7ddb Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Fri, 13 Jul 2018 11:55:16 -0700
Subject: [PATCH 1722/2974] [doc] update default RETRY_HTTP_CODES

---
 docs/topics/downloader-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index dfe4c13b4..8dbe249fa 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -882,7 +882,7 @@ precedence over the :setting:`RETRY_TIMES` setting.
 RETRY_HTTP_CODES
 ^^^^^^^^^^^^^^^^
 
-Default: ``[500, 502, 503, 504, 408]``
+Default: ``[500, 502, 503, 504, 522, 524, 408]``
 
 Which HTTP response codes to retry. Other errors (DNS lookup issues,
 connections lost, etc) are always retried.

From e7e18db179f2e45aa38a5bbdb0abba7d983cdce7 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 11 Jul 2018 14:04:35 -0300
Subject: [PATCH 1723/2974] Fix tests

---
 tests/test_spidermiddleware.py | 57 +++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 25 deletions(-)

diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 2565ef7af..5622c3179 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -8,10 +8,7 @@ from twisted.internet import defer
 from scrapy import Spider, Request
 from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
-
-
-# TEST_URL = 'http://example.org'
-TEST_URL = 'http://localhost:8998'
+from tests.spiders import MockServerSpider
 
 
 class LogExceptionMiddleware:
@@ -21,16 +18,18 @@ class LogExceptionMiddleware:
 
 
 # ================================================================================
-# recover from an exception on a spider callback
+# (0) recover from an exception on a spider callback
 class RecoverySpider(Spider):
     name = 'RecoverySpider'
-    start_urls = [TEST_URL]
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
             __name__ + '.RecoveryMiddleware': 10,
         },
     }
 
+    def start_requests(self):
+        yield Request(self.mockserver.url('/status?n=200'))
+
     def parse(self, response):
         yield {'test': 1}
         self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail'))
@@ -61,10 +60,11 @@ class ProcessSpiderInputSpider(Spider):
     }
 
     def start_requests(self):
-        yield Request(TEST_URL, callback=self.parse, errback=self.errback)
+        yield Request(url=self.mockserver.url('/status?n=200'),
+                      callback=self.parse, errback=self.errback)
 
     def parse(self, response):
-        return [{'test': 1}, {'test': 2}]
+        return {'from': 'callback'}
 
     def errback(self, failure):
         self.logger.warn('Got a Failure on the Request errback')
@@ -80,13 +80,15 @@ class FailProcessSpiderInputMiddleware:
 # (2) exceptions from a spider callback (generator)
 class GeneratorCallbackSpider(Spider):
     name = 'GeneratorCallbackSpider'
-    start_urls = [TEST_URL]
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
             __name__ + '.LogExceptionMiddleware': 10,
         },
     }
 
+    def start_requests(self):
+        yield Request(self.mockserver.url('/status?n=200'))
+
     def parse(self, response):
         yield {'test': 1}
         yield {'test': 2}
@@ -97,13 +99,15 @@ class GeneratorCallbackSpider(Spider):
 # (3) exceptions from a spider callback (not a generator)
 class NotAGeneratorCallbackSpider(Spider):
     name = 'NotAGeneratorCallbackSpider'
-    start_urls = [TEST_URL]
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
             __name__ + '.LogExceptionMiddleware': 10,
         },
     }
 
+    def start_requests(self):
+        yield Request(self.mockserver.url('/status?n=200'))
+
     def parse(self, response):
         return [{'test': 1}, {'test': 1/0}]
 
@@ -123,20 +127,20 @@ class TestSpiderMiddleware(TestCase):
     def crawl_log(self, spider):
         crawler = get_crawler(spider)
         with LogCapture() as log:
-            yield crawler.crawl()
+            yield crawler.crawl(mockserver=self.mockserver)
         raise defer.returnValue(log)
 
-    # @defer.inlineCallbacks
-    # def test_recovery(self):
-    #     """
-    #     Recover from an exception from a spider's callback. The final item count should be 3
-    #     (one from the spider before raising the exception, one from the middleware and one
-    #     from the spider when processing the response that was enqueued from the middleware)
-    #     """
-    #     log = yield self.crawl_log(RecoverySpider)
-    #     self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log))
-    #     self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1)
-    #     self.assertIn("'item_scraped_count': 3", str(log))
+    @defer.inlineCallbacks
+    def test_recovery(self):
+        """
+        (0) Recover from an exception in a spider callback. The final item count should be 2
+        (one directly from the recovery middleware and one from the spider when processing
+        the request that was enqueued from the recovery middleware)
+        """
+        log = yield self.crawl_log(RecoverySpider)
+        self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log))
+        self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1)
+        self.assertIn("'item_scraped_count': 2", str(log))
 
     @defer.inlineCallbacks
     def test_process_spider_input_errback(self):
@@ -148,21 +152,24 @@ class TestSpiderMiddleware(TestCase):
         self.assertNotIn("Middleware: IndexError exception caught", str(log1))
         self.assertIn("Middleware: will raise IndexError", str(log1))
         self.assertIn("Got a Failure on the Request errback", str(log1))
+        self.assertIn("{'from': 'errback'}", str(log1))
+        self.assertNotIn("{'from': 'callback'}", str(log1))
+        self.assertIn("'item_scraped_count': 1", str(log1))
     
     @defer.inlineCallbacks
     def test_generator_callback(self):
         """
-        (2) An exception from a spider's callback should
+        (2) An exception from a spider callback (returning a generator) should
         be caught by the process_spider_exception chain
         """
         log2 = yield self.crawl_log(GeneratorCallbackSpider)
         self.assertIn("Middleware: ImportError exception caught", str(log2))
-        self.assertIn("'item_scraped_count': 2", str(log2))
+        self.assertNotIn("item_scraped_count", str(log2))
     
     @defer.inlineCallbacks
     def test_not_a_generator_callback(self):
         """
-        (3) An exception from a spider's callback should
+        (3) An exception from a spider callback (returning a list) should
         be caught by the process_spider_exception chain
         """
         log3 = yield self.crawl_log(NotAGeneratorCallbackSpider)

From 0c579b5276f502832f375be316094b4244cf87c5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 14 Jul 2018 19:58:42 -0300
Subject: [PATCH 1724/2974] Untested experiment

---
 scrapy/core/spidermw.py | 37 +++++++++++++++++++++++++++++++------
 1 file changed, 31 insertions(+), 6 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index c9dd8c91e..a9aeb6dcc 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,6 +3,8 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
+from itertools import chain
+
 import six
 from twisted.python.failure import Failure
 from scrapy.exceptions import _InvalidOutput
@@ -10,9 +12,28 @@ from scrapy.middleware import MiddlewareManager
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.conf import build_component_list
 
+
 def _isiterable(possible_iterator):
     return hasattr(possible_iterator, '__iter__')
 
+
+class MutableChain:
+    def __init__(self, *args):
+        self.data = chain(*args)
+
+    def extend(self, iterable):
+        self.data = chain(self.data, iterable)
+
+    def __iter__(self):
+        return self.data.__iter__()
+
+    def __next__(self):  # py3
+        return self.data.__next__()
+
+    def next(self):  # py2
+        return self.data.next()
+
+
 class SpiderMiddlewareManager(MiddlewareManager):
 
     component_name = 'spider middleware'
@@ -68,28 +89,32 @@ class SpiderMiddlewareManager(MiddlewareManager):
             return _failure
 
         def process_spider_output(result, index):
-            def wrapper(result_iterable):
+            # items in this iterable do not need to go through the process_spider_output
+            # chain, they went through it already from the process_spider_exception method
+            recovered = MutableChain()
+
+            def evaluate_result(result_iterable, index):
                 try:
                     for r in result_iterable:
                         yield r
                 except Exception as ex:
-                    # process the exception with the method from the next middleware
                     exception_result = process_spider_exception(Failure(ex), index)
                     if exception_result is None or isinstance(exception_result, Failure):
                         raise
-                    for output in exception_result:
-                        yield output
+                    recovered.extend(exception_result)
+
             for i, method in enumerate(self.methods['process_spider_output']):
                 if i < index or method is None:
                     continue
                 result = method(response=response, result=result, spider=spider)
                 index += 1
                 if _isiterable(result):
-                    result = wrapper(result)
+                    result = evaluate_result(result, index)
                 else:
                     raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
                                          .format(fname(method), type(result)))
-            return result
+
+            return chain(result, recovered)
 
         dfd = mustbe_deferred(process_spider_input, response)
         dfd.addErrback(process_spider_exception, index=0)

From c5fa0ae6bc536d6bc5370c6f3b634c33848971e5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 14 Jul 2018 19:58:42 -0300
Subject: [PATCH 1725/2974] Untested experiment

---
 scrapy/core/spidermw.py | 40 ++++++++++++++++++++++++++++++++++------
 1 file changed, 34 insertions(+), 6 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index c9dd8c91e..8ee42c2cf 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,6 +3,8 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
+from itertools import chain
+
 import six
 from twisted.python.failure import Failure
 from scrapy.exceptions import _InvalidOutput
@@ -10,9 +12,31 @@ from scrapy.middleware import MiddlewareManager
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.conf import build_component_list
 
+
 def _isiterable(possible_iterator):
     return hasattr(possible_iterator, '__iter__')
 
+
+class MutableChain:
+    """
+    Thin wrapper around itertools.chain, allowing to add iterables "in-place"
+    """
+    def __init__(self, *args):
+        self.data = chain(*args)
+
+    def extend(self, *iterables):
+        self.data = chain(self.data, *iterables)
+
+    def __iter__(self):
+        return self.data.__iter__()
+
+    def __next__(self):  # py3
+        return self.data.__next__()
+
+    def next(self):  # py2
+        return self.data.next()
+
+
 class SpiderMiddlewareManager(MiddlewareManager):
 
     component_name = 'spider middleware'
@@ -68,28 +92,32 @@ class SpiderMiddlewareManager(MiddlewareManager):
             return _failure
 
         def process_spider_output(result, index):
-            def wrapper(result_iterable):
+            # items in this iterable do not need to go through the process_spider_output
+            # chain, they went through it already from the process_spider_exception method
+            recovered = MutableChain()
+
+            def evaluate_result(result_iterable, index):
                 try:
                     for r in result_iterable:
                         yield r
                 except Exception as ex:
-                    # process the exception with the method from the next middleware
                     exception_result = process_spider_exception(Failure(ex), index)
                     if exception_result is None or isinstance(exception_result, Failure):
                         raise
-                    for output in exception_result:
-                        yield output
+                    recovered.extend(exception_result)
+
             for i, method in enumerate(self.methods['process_spider_output']):
                 if i < index or method is None:
                     continue
                 result = method(response=response, result=result, spider=spider)
                 index += 1
                 if _isiterable(result):
-                    result = wrapper(result)
+                    result = evaluate_result(result, index)
                 else:
                     raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
                                          .format(fname(method), type(result)))
-            return result
+
+            return chain(result, recovered)
 
         dfd = mustbe_deferred(process_spider_input, response)
         dfd.addErrback(process_spider_exception, index=0)

From cff9e8762512033da181293bab379b485aeffa66 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 15 Jul 2018 16:21:08 -0300
Subject: [PATCH 1726/2974] Fix tests

---
 tests/test_spidermiddleware.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 5622c3179..c33eb28ca 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -68,6 +68,7 @@ class ProcessSpiderInputSpider(Spider):
 
     def errback(self, failure):
         self.logger.warn('Got a Failure on the Request errback')
+        return {'from': 'errback'}
 
 
 class FailProcessSpiderInputMiddleware:
@@ -133,14 +134,15 @@ class TestSpiderMiddleware(TestCase):
     @defer.inlineCallbacks
     def test_recovery(self):
         """
-        (0) Recover from an exception in a spider callback. The final item count should be 2
-        (one directly from the recovery middleware and one from the spider when processing
-        the request that was enqueued from the recovery middleware)
+        (0) Recover from an exception in a spider callback. The final item count should be 3
+        (one yielded from the callback method before the exception is raised, one directly
+        from the recovery middleware and one from the spider when processing the request that
+        was enqueued from the recovery middleware)
         """
         log = yield self.crawl_log(RecoverySpider)
         self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log))
         self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1)
-        self.assertIn("'item_scraped_count': 2", str(log))
+        self.assertIn("'item_scraped_count': 3", str(log))
 
     @defer.inlineCallbacks
     def test_process_spider_input_errback(self):
@@ -164,7 +166,7 @@ class TestSpiderMiddleware(TestCase):
         """
         log2 = yield self.crawl_log(GeneratorCallbackSpider)
         self.assertIn("Middleware: ImportError exception caught", str(log2))
-        self.assertNotIn("item_scraped_count", str(log2))
+        self.assertIn("'item_scraped_count': 2", str(log2))
     
     @defer.inlineCallbacks
     def test_not_a_generator_callback(self):

From 60c2ef86f0c40d17219d3e3320072fb5b1ded412 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 15 Jul 2018 16:47:55 -0300
Subject: [PATCH 1727/2974] Revert "Default values for OffsiteMiddleware"

This reverts commit ba294351381c0dd81476603246d2cea6c31486be.
---
 scrapy/spidermiddlewares/offsite.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 3b7f194e4..310166cad 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -19,9 +19,6 @@ class OffsiteMiddleware(object):
 
     def __init__(self, stats):
         self.stats = stats
-        # default values
-        self.host_regex = re.compile('')  # allow all by default
-        self.domains_seen = set()
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -55,7 +52,7 @@ class OffsiteMiddleware(object):
         """Override this method to implement a different offsite policy"""
         allowed_domains = getattr(spider, 'allowed_domains', None)
         if not allowed_domains:
-            return
+            return re.compile('') # allow all by default
         url_pattern = re.compile("^https?://.*$")
         for domain in allowed_domains:
             if url_pattern.match(domain):
@@ -65,9 +62,8 @@ class OffsiteMiddleware(object):
         return re.compile(regex)
 
     def spider_opened(self, spider):
-        host_regex = self.get_host_regex(spider)
-        if host_regex:
-            self.host_regex = host_regex
+        self.host_regex = self.get_host_regex(spider)
+        self.domains_seen = set()
 
 
 class URLWarning(Warning):

From b8e8922d5436247f7be66c40e1a16a0acab7986e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 15 Jul 2018 17:50:55 -0300
Subject: [PATCH 1728/2974] Simplify stuff. Add more tests.

---
 scrapy/core/spidermw.py        | 10 +++---
 tests/test_spidermiddleware.py | 63 ++++++++++++++++++++++++++++++++--
 2 files changed, 66 insertions(+), 7 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 8ee42c2cf..c733402b9 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -75,8 +75,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
             # don't handle _InvalidOutput exception
             if isinstance(exception, _InvalidOutput):
                 return _failure
-            for i, method in enumerate(self.methods['process_spider_exception']):
-                if i < index or method is None:
+            for method in self.methods['process_spider_exception'][index:]:
+                if method is None:
                     continue
                 result = method(response=response, exception=exception, spider=spider)
                 index += 1
@@ -101,13 +101,13 @@ class SpiderMiddlewareManager(MiddlewareManager):
                     for r in result_iterable:
                         yield r
                 except Exception as ex:
-                    exception_result = process_spider_exception(Failure(ex), index)
+                    exception_result = process_spider_exception(Failure(ex), index+1)
                     if exception_result is None or isinstance(exception_result, Failure):
                         raise
                     recovered.extend(exception_result)
 
-            for i, method in enumerate(self.methods['process_spider_output']):
-                if i < index or method is None:
+            for method in self.methods['process_spider_output'][index:]:
+                if method is None:
                     continue
                 result = method(response=response, result=result, spider=spider)
                 index += 1
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index c33eb28ca..645d95059 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -113,6 +113,49 @@ class NotAGeneratorCallbackSpider(Spider):
         return [{'test': 1}, {'test': 1/0}]
 
 
+# ================================================================================
+# (4) exceptions from a middleware process_spider_output method (generator)
+class GeneratorOutputChainSpider(Spider):
+    name = 'GeneratorOutputChainSpider'
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            __name__ + '.GeneratorFailOutputChainMiddleware': 10,
+            __name__ + '.GeneratorRecoverOutputChainMiddleware': 5,
+        },
+    }
+
+    def start_requests(self):
+        yield Request(self.mockserver.url('/status?n=200'))
+
+    def parse(self, response):
+        yield {'processed': ['parse']}
+
+
+class GeneratorFailOutputChainMiddleware:
+    def process_spider_output(self, response, result, spider):
+        for r in result:
+            r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__))
+            yield r
+            raise LookupError()
+    
+    def process_spider_exception(self, response, exception, spider):
+        method = '{}.process_spider_exception'.format(self.__class__.__name__)
+        logging.info('%s: %s caught', method, exception.__class__.__name__)
+        yield {'processed': [method]}
+
+
+class GeneratorRecoverOutputChainMiddleware:
+    def process_spider_output(self, response, result, spider):
+        for r in result:
+            r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__))
+            yield r
+
+    def process_spider_exception(self, response, exception, spider):
+        method = '{}.process_spider_exception'.format(self.__class__.__name__)
+        logging.info('%s: %s caught', method, exception.__class__.__name__)
+        yield {'processed': [method]}
+
+
 # ================================================================================
 class TestSpiderMiddleware(TestCase):
     @classmethod
@@ -162,7 +205,8 @@ class TestSpiderMiddleware(TestCase):
     def test_generator_callback(self):
         """
         (2) An exception from a spider callback (returning a generator) should
-        be caught by the process_spider_exception chain
+        be caught by the process_spider_exception chain. Items yielded before the
+        exception is raised should be processed normally.
         """
         log2 = yield self.crawl_log(GeneratorCallbackSpider)
         self.assertIn("Middleware: ImportError exception caught", str(log2))
@@ -172,8 +216,23 @@ class TestSpiderMiddleware(TestCase):
     def test_not_a_generator_callback(self):
         """
         (3) An exception from a spider callback (returning a list) should
-        be caught by the process_spider_exception chain
+        be caught by the process_spider_exception chain. No items should be processed.
         """
         log3 = yield self.crawl_log(NotAGeneratorCallbackSpider)
         self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3))
         self.assertNotIn("item_scraped_count", str(log3))
+
+    @defer.inlineCallbacks
+    def test_generator_output_chain(self):
+        """
+        (4) An exception from a middleware's process_spider_output method should be sent
+        to the process_spider_exception method from the next middleware in the chain.
+        The final item count should be 2 (one from the spider callback and one from the
+        process_spider_exception chain)
+        """
+        log4 = yield self.crawl_log(GeneratorOutputChainSpider)
+        self.assertIn("'item_scraped_count': 2", str(log4))
+        self.assertIn("GeneratorRecoverOutputChainMiddleware.process_spider_exception: LookupError caught", str(log4))
+        self.assertNotIn("GeneratorFailOutputChainMiddleware.process_spider_exception: LookupError caught", str(log4))
+        self.assertIn("{'processed': ['parse', 'GeneratorFailOutputChainMiddleware.process_spider_output', 'GeneratorRecoverOutputChainMiddleware.process_spider_output']}", str(log4))
+        self.assertIn("{'processed': ['GeneratorRecoverOutputChainMiddleware.process_spider_exception']}", str(log4))

From 56e92d90fda3e812aca270327e513391591a10cc Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 17 Jul 2018 15:15:38 -0300
Subject: [PATCH 1729/2974] Update tests

---
 tests/test_spidermiddleware.py | 54 ++++++++++++++++++++++++++++------
 1 file changed, 45 insertions(+), 9 deletions(-)

diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 645d95059..9bb7f62fd 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -119,8 +119,10 @@ class GeneratorOutputChainSpider(Spider):
     name = 'GeneratorOutputChainSpider'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            __name__ + '.GeneratorFailOutputChainMiddleware': 10,
-            __name__ + '.GeneratorRecoverOutputChainMiddleware': 5,
+            __name__ + '.GeneratorFailMiddleware': 10,
+            __name__ + '.GeneratorDoNothingAfterFailureMiddleware': 8,
+            __name__ + '.GeneratorRecoverMiddleware': 5,
+            __name__ + '.GeneratorDoNothingAfterRecoveryMiddleware': 3,
         },
     }
 
@@ -128,10 +130,23 @@ class GeneratorOutputChainSpider(Spider):
         yield Request(self.mockserver.url('/status?n=200'))
 
     def parse(self, response):
-        yield {'processed': ['parse']}
+        yield {'processed': ['parse-first-item']}
+        yield {'processed': ['parse-second-item']}
 
 
-class GeneratorFailOutputChainMiddleware:
+class _GeneratorDoNothingMiddleware:
+    def process_spider_output(self, response, result, spider):
+        for r in result:
+            r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__))
+            yield r
+
+    def process_spider_exception(self, response, exception, spider):
+        method = '{}.process_spider_exception'.format(self.__class__.__name__)
+        logging.info('%s: %s caught', method, exception.__class__.__name__)
+        return None
+
+
+class GeneratorFailMiddleware:
     def process_spider_output(self, response, result, spider):
         for r in result:
             r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__))
@@ -144,7 +159,11 @@ class GeneratorFailOutputChainMiddleware:
         yield {'processed': [method]}
 
 
-class GeneratorRecoverOutputChainMiddleware:
+class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware):
+    pass
+
+
+class GeneratorRecoverMiddleware:
     def process_spider_output(self, response, result, spider):
         for r in result:
             r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__))
@@ -155,6 +174,9 @@ class GeneratorRecoverOutputChainMiddleware:
         logging.info('%s: %s caught', method, exception.__class__.__name__)
         yield {'processed': [method]}
 
+class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
+    pass
+
 
 # ================================================================================
 class TestSpiderMiddleware(TestCase):
@@ -227,12 +249,26 @@ class TestSpiderMiddleware(TestCase):
         """
         (4) An exception from a middleware's process_spider_output method should be sent
         to the process_spider_exception method from the next middleware in the chain.
+        The result of the recovery by the process_spider_exception method should be handled
+        by the process_spider_output method from the next middleware.
         The final item count should be 2 (one from the spider callback and one from the
         process_spider_exception chain)
         """
         log4 = yield self.crawl_log(GeneratorOutputChainSpider)
         self.assertIn("'item_scraped_count': 2", str(log4))
-        self.assertIn("GeneratorRecoverOutputChainMiddleware.process_spider_exception: LookupError caught", str(log4))
-        self.assertNotIn("GeneratorFailOutputChainMiddleware.process_spider_exception: LookupError caught", str(log4))
-        self.assertIn("{'processed': ['parse', 'GeneratorFailOutputChainMiddleware.process_spider_output', 'GeneratorRecoverOutputChainMiddleware.process_spider_output']}", str(log4))
-        self.assertIn("{'processed': ['GeneratorRecoverOutputChainMiddleware.process_spider_exception']}", str(log4))
+        self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", str(log4))
+        self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", str(log4))
+        self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: LookupError caught", str(log4))
+        self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", str(log4))
+        item_from_callback = {'processed': [
+            'parse-first-item',
+            'GeneratorFailMiddleware.process_spider_output',
+            'GeneratorDoNothingAfterFailureMiddleware.process_spider_output',
+            'GeneratorRecoverMiddleware.process_spider_output',
+            'GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']}
+        item_recovered = {'processed': [
+            'GeneratorRecoverMiddleware.process_spider_exception',
+            'GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']}
+        self.assertIn(str(item_from_callback), str(log4))
+        self.assertIn(str(item_recovered), str(log4))
+        self.assertNotIn('parse-second-item', str(log4))

From 610f589662ca9c5929527e15cad2c347c8d5d335 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 17 Jul 2018 19:13:03 -0300
Subject: [PATCH 1730/2974] Add callback and errback in the same step

---
 scrapy/core/spidermw.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index c733402b9..da51bc974 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -70,7 +70,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                     return scrape_func(Failure(), request, spider)
             return scrape_func(response, request, spider)
 
-        def process_spider_exception(_failure, index):
+        def process_spider_exception(_failure, index=0):
             exception = _failure.value
             # don't handle _InvalidOutput exception
             if isinstance(exception, _InvalidOutput):
@@ -91,7 +91,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                                          .format(fname(method), type(result)))
             return _failure
 
-        def process_spider_output(result, index):
+        def process_spider_output(result, index=0):
             # items in this iterable do not need to go through the process_spider_output
             # chain, they went through it already from the process_spider_exception method
             recovered = MutableChain()
@@ -120,8 +120,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             return chain(result, recovered)
 
         dfd = mustbe_deferred(process_spider_input, response)
-        dfd.addErrback(process_spider_exception, index=0)
-        dfd.addCallback(process_spider_output, index=0)
+        dfd.addCallbacks(callback=process_spider_output, errback=process_spider_exception)
         return dfd
 
     def process_start_requests(self, start_requests, spider):

From a3af0bfd56770aab0a056ae6e29efffa8b7d88c4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 18 Jul 2018 15:15:55 -0300
Subject: [PATCH 1731/2974] More tests

---
 scrapy/core/spidermw.py        | 21 +++++---
 tests/test_spidermiddleware.py | 99 ++++++++++++++++++++++++++++++++--
 2 files changed, 110 insertions(+), 10 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index da51bc974..96488806d 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -64,8 +64,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 try:
                     result = method(response=response, spider=spider)
                     if result is not None:
-                        raise _InvalidOutput('Middleware {} must return None or raise ' \
-                            'an exception, got {}'.format(fname(method), type(result)))
+                        raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \
+                                             .format(fname(method), type(result)))
                 except:
                     return scrape_func(Failure(), request, spider)
             return scrape_func(response, request, spider)
@@ -78,8 +78,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
             for method in self.methods['process_spider_exception'][index:]:
                 if method is None:
                     continue
-                result = method(response=response, exception=exception, spider=spider)
                 index += 1
+                result = method(response=response, exception=exception, spider=spider)
                 if _isiterable(result):
                     # stop exception handling by handing control over to the
                     # process_spider_output chain if an iterable has been returned
@@ -96,9 +96,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
             # chain, they went through it already from the process_spider_exception method
             recovered = MutableChain()
 
-            def evaluate_result(result_iterable, index):
+            def evaluate_iterable(iterable, index):
                 try:
-                    for r in result_iterable:
+                    for r in iterable:
                         yield r
                 except Exception as ex:
                     exception_result = process_spider_exception(Failure(ex), index+1)
@@ -109,10 +109,17 @@ class SpiderMiddlewareManager(MiddlewareManager):
             for method in self.methods['process_spider_output'][index:]:
                 if method is None:
                     continue
-                result = method(response=response, result=result, spider=spider)
                 index += 1
+                # the following might fail directly if the output value is not a generator
+                try:
+                    result = method(response=response, result=result, spider=spider)
+                except Exception as ex:
+                    exception_result = process_spider_exception(Failure(ex), index+1)
+                    if exception_result is None or isinstance(exception_result, Failure):
+                        raise
+                    return exception_result
                 if _isiterable(result):
-                    result = evaluate_result(result, index)
+                    result = evaluate_iterable(result, index)
                 else:
                     raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
                                          .format(fname(method), type(result)))
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 9bb7f62fd..2f431ddc7 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -98,8 +98,8 @@ class GeneratorCallbackSpider(Spider):
 
 # ================================================================================
 # (3) exceptions from a spider callback (not a generator)
-class NotAGeneratorCallbackSpider(Spider):
-    name = 'NotAGeneratorCallbackSpider'
+class NotGeneratorCallbackSpider(Spider):
+    name = 'NotGeneratorCallbackSpider'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
             __name__ + '.LogExceptionMiddleware': 10,
@@ -178,6 +178,76 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
     pass
 
 
+# ================================================================================
+# (5) exceptions from a middleware process_spider_output method (not generator)
+class NotGeneratorOutputChainSpider(Spider):
+    name = 'NotGeneratorOutputChainSpider'
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            __name__ + '.NotGeneratorFailMiddleware': 10,
+            __name__ + '.NotGeneratorDoNothingAfterFailureMiddleware': 8,
+            __name__ + '.NotGeneratorRecoverMiddleware': 5,
+            __name__ + '.NotGeneratorDoNothingAfterRecoveryMiddleware': 3,
+        },
+    }
+
+    def start_requests(self):
+        return [Request(self.mockserver.url('/status?n=200'))]
+
+    def parse(self, response):
+        return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}]
+
+
+class _NotGeneratorDoNothingMiddleware:
+    def process_spider_output(self, response, result, spider):
+        out = []
+        for r in result:
+            r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__))
+            out.append(r)
+        return out
+
+    def process_spider_exception(self, response, exception, spider):
+        method = '{}.process_spider_exception'.format(self.__class__.__name__)
+        logging.info('%s: %s caught', method, exception.__class__.__name__)
+        return None
+
+
+class NotGeneratorFailMiddleware:
+    def process_spider_output(self, response, result, spider):
+        out = []
+        for r in result:
+            r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__))
+            out.append(r)
+        raise ReferenceError()
+        return out
+
+    def process_spider_exception(self, response, exception, spider):
+        method = '{}.process_spider_exception'.format(self.__class__.__name__)
+        logging.info('%s: %s caught', method, exception.__class__.__name__)
+        return [{'processed': [method]}]
+
+
+class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddleware):
+    pass
+
+
+class NotGeneratorRecoverMiddleware:
+    def process_spider_output(self, response, result, spider):
+        out = []
+        for r in result:
+            r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__))
+            out.append(r)
+        return out
+
+    def process_spider_exception(self, response, exception, spider):
+        method = '{}.process_spider_exception'.format(self.__class__.__name__)
+        logging.info('%s: %s caught', method, exception.__class__.__name__)
+        return [{'processed': [method]}]
+
+class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware):
+    pass
+
+
 # ================================================================================
 class TestSpiderMiddleware(TestCase):
     @classmethod
@@ -240,7 +310,7 @@ class TestSpiderMiddleware(TestCase):
         (3) An exception from a spider callback (returning a list) should
         be caught by the process_spider_exception chain. No items should be processed.
         """
-        log3 = yield self.crawl_log(NotAGeneratorCallbackSpider)
+        log3 = yield self.crawl_log(NotGeneratorCallbackSpider)
         self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3))
         self.assertNotIn("item_scraped_count", str(log3))
 
@@ -272,3 +342,26 @@ class TestSpiderMiddleware(TestCase):
         self.assertIn(str(item_from_callback), str(log4))
         self.assertIn(str(item_recovered), str(log4))
         self.assertNotIn('parse-second-item', str(log4))
+
+    @defer.inlineCallbacks
+    def test_not_a_generator_output_chain(self):
+        """
+        (5) An exception from a middleware's process_spider_output method should be sent
+        to the process_spider_exception method from the next middleware in the chain.
+        The result of the recovery by the process_spider_exception method should be handled
+        by the process_spider_output method from the next middleware.
+        The final item count should be 1 (from the process_spider_exception chain, the items
+        from the spider callback are lost)
+        """
+        log5 = yield self.crawl_log(NotGeneratorOutputChainSpider)
+        self.assertIn("'item_scraped_count': 1", str(log5))
+        self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", str(log5))
+        self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", str(log5))
+        self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", str(log5))
+        self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", str(log5))
+        item_recovered = {'processed': [
+            'NotGeneratorRecoverMiddleware.process_spider_exception',
+            'NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']}
+        self.assertIn(str(item_recovered), str(log5))
+        self.assertNotIn('parse-first-item', str(log5))
+        self.assertNotIn('parse-second-item', str(log5))

From 6329441c826bec97aeec82d3e7ec0bcd91c60a47 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 18 Jul 2018 16:59:24 -0300
Subject: [PATCH 1732/2974] ModuleNotFoundError was added in py3.6

---
 tests/test_spidermiddleware.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 2f431ddc7..0451dfd27 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -34,7 +34,7 @@ class RecoverySpider(Spider):
         yield {'test': 1}
         self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail'))
         if not response.meta.get('dont_fail'):
-            raise ModuleNotFoundError()
+            raise TabError()
 
 class RecoveryMiddleware:
     def process_spider_exception(self, response, exception, spider):
@@ -275,8 +275,8 @@ class TestSpiderMiddleware(TestCase):
         was enqueued from the recovery middleware)
         """
         log = yield self.crawl_log(RecoverySpider)
-        self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log))
-        self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1)
+        self.assertIn("Middleware: TabError exception caught", str(log))
+        self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)
         self.assertIn("'item_scraped_count': 3", str(log))
 
     @defer.inlineCallbacks

From 71a1406c99e7d4cced0693389e537c98a38104aa Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 18 Jul 2018 17:40:30 -0300
Subject: [PATCH 1733/2974] Logging changes

---
 tests/test_spidermiddleware.py | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 0451dfd27..0f5646a72 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -1,6 +1,4 @@
 
-import logging
-
 from testfixtures import LogCapture
 from twisted.trial.unittest import TestCase
 from twisted.internet import defer
@@ -13,7 +11,7 @@ from tests.spiders import MockServerSpider
 
 class LogExceptionMiddleware:
     def process_spider_exception(self, response, exception, spider):
-        logging.warn('Middleware: %s exception caught', exception.__class__.__name__)
+        spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
         return None
 
 
@@ -32,13 +30,13 @@ class RecoverySpider(Spider):
 
     def parse(self, response):
         yield {'test': 1}
-        self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail'))
+        self.logger.info('DONT_FAIL: %s', response.meta.get('dont_fail'))
         if not response.meta.get('dont_fail'):
             raise TabError()
 
 class RecoveryMiddleware:
     def process_spider_exception(self, response, exception, spider):
-        logging.warn('Middleware: %s exception caught', exception.__class__.__name__)
+        spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
         return [
             {'from': 'process_spider_exception'},
             Request(response.url, meta={'dont_fail': True}, dont_filter=True),
@@ -67,13 +65,13 @@ class ProcessSpiderInputSpider(Spider):
         return {'from': 'callback'}
 
     def errback(self, failure):
-        self.logger.warn('Got a Failure on the Request errback')
+        self.logger.info('Got a Failure on the Request errback')
         return {'from': 'errback'}
 
 
 class FailProcessSpiderInputMiddleware:
     def process_spider_input(self, response, spider):
-        logging.warn('Middleware: will raise IndexError')
+        spider.logger.info('Middleware: will raise IndexError')
         raise IndexError()
 
 
@@ -142,7 +140,7 @@ class _GeneratorDoNothingMiddleware:
 
     def process_spider_exception(self, response, exception, spider):
         method = '{}.process_spider_exception'.format(self.__class__.__name__)
-        logging.info('%s: %s caught', method, exception.__class__.__name__)
+        spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
         return None
 
 
@@ -155,7 +153,7 @@ class GeneratorFailMiddleware:
     
     def process_spider_exception(self, response, exception, spider):
         method = '{}.process_spider_exception'.format(self.__class__.__name__)
-        logging.info('%s: %s caught', method, exception.__class__.__name__)
+        spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
         yield {'processed': [method]}
 
 
@@ -171,7 +169,7 @@ class GeneratorRecoverMiddleware:
 
     def process_spider_exception(self, response, exception, spider):
         method = '{}.process_spider_exception'.format(self.__class__.__name__)
-        logging.info('%s: %s caught', method, exception.__class__.__name__)
+        spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
         yield {'processed': [method]}
 
 class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
@@ -208,7 +206,7 @@ class _NotGeneratorDoNothingMiddleware:
 
     def process_spider_exception(self, response, exception, spider):
         method = '{}.process_spider_exception'.format(self.__class__.__name__)
-        logging.info('%s: %s caught', method, exception.__class__.__name__)
+        spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
         return None
 
 
@@ -223,7 +221,7 @@ class NotGeneratorFailMiddleware:
 
     def process_spider_exception(self, response, exception, spider):
         method = '{}.process_spider_exception'.format(self.__class__.__name__)
-        logging.info('%s: %s caught', method, exception.__class__.__name__)
+        spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
         return [{'processed': [method]}]
 
 
@@ -241,7 +239,7 @@ class NotGeneratorRecoverMiddleware:
 
     def process_spider_exception(self, response, exception, spider):
         method = '{}.process_spider_exception'.format(self.__class__.__name__)
-        logging.info('%s: %s caught', method, exception.__class__.__name__)
+        spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
         return [{'processed': [method]}]
 
 class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware):

From 20defa2e16628b4b432a1cc44ad37182dfc764ee Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 19 Jul 2018 10:31:06 -0300
Subject: [PATCH 1734/2974] Better handling of method indexes

---
 scrapy/core/spidermw.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 96488806d..8607ed620 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -70,20 +70,19 @@ class SpiderMiddlewareManager(MiddlewareManager):
                     return scrape_func(Failure(), request, spider)
             return scrape_func(response, request, spider)
 
-        def process_spider_exception(_failure, index=0):
+        def process_spider_exception(_failure, start_index=0):
             exception = _failure.value
             # don't handle _InvalidOutput exception
             if isinstance(exception, _InvalidOutput):
                 return _failure
-            for method in self.methods['process_spider_exception'][index:]:
+            for method_index, method in enumerate(self.methods['process_spider_exception'][start_index:], start=start_index):
                 if method is None:
                     continue
-                index += 1
                 result = method(response=response, exception=exception, spider=spider)
                 if _isiterable(result):
                     # stop exception handling by handing control over to the
                     # process_spider_output chain if an iterable has been returned
-                    return process_spider_output(result, index)
+                    return process_spider_output(result, method_index+1)
                 elif result is None:
                     continue
                 else:
@@ -91,7 +90,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                                          .format(fname(method), type(result)))
             return _failure
 
-        def process_spider_output(result, index=0):
+        def process_spider_output(result, start_index=0):
             # items in this iterable do not need to go through the process_spider_output
             # chain, they went through it already from the process_spider_exception method
             recovered = MutableChain()
@@ -106,20 +105,19 @@ class SpiderMiddlewareManager(MiddlewareManager):
                         raise
                     recovered.extend(exception_result)
 
-            for method in self.methods['process_spider_output'][index:]:
+            for method_index, method in enumerate(self.methods['process_spider_output'][start_index:], start=start_index):
                 if method is None:
                     continue
-                index += 1
                 # the following might fail directly if the output value is not a generator
                 try:
                     result = method(response=response, result=result, spider=spider)
                 except Exception as ex:
-                    exception_result = process_spider_exception(Failure(ex), index+1)
+                    exception_result = process_spider_exception(Failure(ex), method_index+1)
                     if exception_result is None or isinstance(exception_result, Failure):
                         raise
                     return exception_result
                 if _isiterable(result):
-                    result = evaluate_iterable(result, index)
+                    result = evaluate_iterable(result, method_index)
                 else:
                     raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
                                          .format(fname(method), type(result)))

From 6a38fc39f8fd1344bc41fcf50fe0e0af27ec74c4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 19 Jul 2018 11:56:23 -0300
Subject: [PATCH 1735/2974] Include flags when copying requests

---
 scrapy/http/request/__init__.py | 2 +-
 tests/test_http_request.py      | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 13a92ffa0..cd4360483 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -91,7 +91,7 @@ class Request(object_ref):
         """Create a new Request with the same attributes except for those
         given new values.
         """
-        for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta',
+        for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',
                   'encoding', 'priority', 'dont_filter', 'callback', 'errback']:
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index a042f03b6..fc89229c6 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -174,7 +174,8 @@ class RequestTest(unittest.TestCase):
         def somecallback():
             pass
 
-        r1 = self.request_class("http://www.example.com", callback=somecallback, errback=somecallback)
+        r1 = self.request_class("http://www.example.com", flags=['f1', 'f2'],
+                                callback=somecallback, errback=somecallback)
         r1.meta['foo'] = 'bar'
         r2 = r1.copy()
 
@@ -184,6 +185,10 @@ class RequestTest(unittest.TestCase):
         assert r2.callback is r1.callback
         assert r2.errback is r2.errback
 
+        # make sure flags list is shallow copied
+        assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
+        self.assertEqual(r1.flags, r2.flags)
+
         # make sure meta dict is shallow copied
         assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical"
         self.assertEqual(r1.meta, r2.meta)

From 7020c3e4523ef445f0d279cdb85fc7570d9e2c2e Mon Sep 17 00:00:00 2001
From: Andrei Korigodski <akorigod@gmail.com>
Date: Fri, 20 Jul 2018 14:46:57 +0300
Subject: [PATCH 1736/2974] Doc: update copyright notice

The years are updated. The hyphen is replaced with an en dash.
---
 docs/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index 594740f39..a54a6bbe9 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -45,7 +45,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Scrapy'
-copyright = u'2008-2016, Scrapy developers'
+copyright = u'2008–2018, Scrapy developers'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

From 98d74d1083be6afa7553a5950f89fc5ac446272f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 20 Jul 2018 12:08:49 -0300
Subject: [PATCH 1737/2974] Requested changes

---
 scrapy/extensions/feedexport.py | 29 +++++++++++++++--------------
 scrapy/utils/misc.py            |  1 +
 tests/test_feedexport.py        |  4 +++-
 3 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 067887d94..7c7db387e 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -96,21 +96,22 @@ class S3FeedStorage(BlockingFeedStorage):
     def __init__(self, uri, access_key=None, secret_key=None):
         # BEGIN Backwards compatibility for initialising without keys (and
         # without using from_crawler)
-        from scrapy.conf import settings
         no_defaults = access_key is None and secret_key is None
-        if no_defaults and ('AWS_ACCESS_KEY_ID' in settings or
-                            'AWS_SECRET_ACCESS_KEY' in settings):
-            import warnings
-            from scrapy.exceptions import ScrapyDeprecationWarning
-            warnings.warn(
-                "Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
-                "without AWS keys is deprecated. Please supply credentials or "
-                "use the `from_crawler()` constructor.",
-                category=ScrapyDeprecationWarning,
-                stacklevel=2
-            )
-            access_key = settings['AWS_ACCESS_KEY_ID']
-            secret_key = settings['AWS_SECRET_ACCESS_KEY']
+        if no_defaults:
+            from scrapy.conf import settings
+            if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:
+                import warnings
+                from scrapy.exceptions import ScrapyDeprecationWarning
+                warnings.warn(
+                    "Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
+                    "without AWS keys is deprecated. Please supply credentials or "
+                    "use the `from_crawler()` constructor.",
+                    category=ScrapyDeprecationWarning,
+                    stacklevel=2
+                )
+                access_key = settings['AWS_ACCESS_KEY_ID']
+                secret_key = settings['AWS_SECRET_ACCESS_KEY']
+        # END Backwards compatibility
         u = urlparse(uri)
         self.bucketname = u.hostname
         self.access_key = u.username or access_key
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 8eb1aabb5..5ccfdcd72 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -118,6 +118,7 @@ def rel_has_nofollow(rel):
     """Return True if link rel attribute has nofollow type"""
     return True if rel is not None and 'nofollow' in rel.split() else False
 
+
 def create_instance(objcls, settings, crawler, *args, **kwargs):
     """Construct a class instance using its ``from_crawler`` or
     ``from_settings`` constructors, if available.
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index eeb1bc2a4..380ed971b 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -173,7 +173,9 @@ class S3FeedStorageTest(unittest.TestCase):
         uri = os.environ.get('S3_TEST_FILE_URI')
         if not uri:
             raise unittest.SkipTest("No S3 URI available for testing")
-        storage = S3FeedStorage(uri, Settings())
+        access_key = os.environ.get('AWS_ACCESS_KEY_ID')
+        secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
+        storage = S3FeedStorage(uri, access_key, secret_key)
         verifyObject(IFeedStorage, storage)
         file = storage.open(scrapy.Spider("default"))
         expected_content = b"content: \xe2\x98\x83"

From 784eed113021a1a787787a354add242e7abbf6f9 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 20 Jul 2018 19:08:46 -0300
Subject: [PATCH 1738/2974] Improve test coverage (downloader middleware)

---
 scrapy/core/downloader/middleware.py |  6 +--
 tests/test_downloadermiddleware.py   | 64 ++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index cf0c1f869..2fa277e7d 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -50,8 +50,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 defer.returnValue(response)
 
             for method in self.methods['process_response']:
-                response = yield method(request=request, response=response,
-                                        spider=spider)
+                response = yield method(request=request, response=response, spider=spider)
                 if not isinstance(response, (Response, Request)):
                     raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
                                          (six.get_method_self(method).__class__.__name__, type(response)))
@@ -63,8 +62,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         def process_exception(_failure):
             exception = _failure.value
             for method in self.methods['process_exception']:
-                response = yield method(request=request, exception=exception,
-                                        spider=spider)
+                response = yield method(request=request, exception=exception, spider=spider)
                 if response is not None and not isinstance(response, (Response, Request)):
                     raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
                                          (six.get_method_self(method).__class__.__name__, type(response)))
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index fb51392b2..0f420b70d 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -3,6 +3,7 @@ from twisted.python.failure import Failure
 
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
+from scrapy.exceptions import _InvalidOutput
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.utils.test import get_crawler
 from scrapy.utils.python import to_bytes
@@ -115,3 +116,66 @@ class ResponseFromProcessRequestTest(ManagerTestCase):
 
         self.assertIs(results[0], resp)
         self.assertFalse(download_func.called)
+
+
+class ProcessRequestInvalidOutput(ManagerTestCase):
+    """Invalid return value for process_request method should raise an exception"""
+
+    def test_invalid_process_request(self):
+        req = Request('http://example.com/index.html')
+        resp = Response('http://example.com/index.html')
+
+        class InvalidProcessRequestMiddleware:
+            def process_request(self, request, spider):
+                return 1
+
+        self.mwman._add_middleware(InvalidProcessRequestMiddleware())
+        download_func = mock.MagicMock()
+        dfd = self.mwman.download(download_func, req, self.spider)
+        results = []
+        dfd.addBoth(results.append)
+        self.assertIsInstance(results[0], Failure)
+        self.assertIsInstance(results[0].value, _InvalidOutput)
+
+
+class ProcessResponseInvalidOutput(ManagerTestCase):
+    """Invalid return value for process_response method should raise an exception"""
+
+    def test_invalid_process_response(self):
+        req = Request('http://example.com/index.html')
+        resp = Response('http://example.com/index.html')
+
+        class InvalidProcessResponseMiddleware:
+            def process_response(self, request, response, spider):
+                return 1
+
+        self.mwman._add_middleware(InvalidProcessResponseMiddleware())
+        download_func = mock.MagicMock()
+        dfd = self.mwman.download(download_func, req, self.spider)
+        results = []
+        dfd.addBoth(results.append)
+        self.assertIsInstance(results[0], Failure)
+        self.assertIsInstance(results[0].value, _InvalidOutput)
+
+
+class ProcessExceptionInvalidOutput(ManagerTestCase):
+    """Invalid return value for process_exception method should raise an exception"""
+
+    def test_invalid_process_exception(self):
+        req = Request('http://example.com/index.html')
+        resp = Response('http://example.com/index.html')
+
+        class InvalidProcessExceptionMiddleware:
+            def process_request(self, request, spider):
+                raise Exception()
+
+            def process_exception(self, request, exception, spider):
+                return 1
+
+        self.mwman._add_middleware(InvalidProcessExceptionMiddleware())
+        download_func = mock.MagicMock()
+        dfd = self.mwman.download(download_func, req, self.spider)
+        results = []
+        dfd.addBoth(results.append)
+        self.assertIsInstance(results[0], Failure)
+        self.assertIsInstance(results[0].value, _InvalidOutput)

From 48866457b309ebb5edfe2dac8966af1c825b9d65 Mon Sep 17 00:00:00 2001
From: Malcolm Granado Ho Yong Liang <malcolmho3065@gmail.com>
Date: Wed, 25 Jul 2018 14:38:37 +0800
Subject: [PATCH 1739/2974] make amendments to grammer

---
 docs/contributing.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 6615840f7..2369c3436 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -12,7 +12,7 @@ Contributing to Scrapy
 There are many ways to contribute to Scrapy. Here are some of them:
 
 * Blog about Scrapy. Tell the world how you're using Scrapy. This will help
-  newcomers with more examples and the Scrapy project to increase its
+  newcomers with more examples and will help the Scrapy project to increase its
   visibility.
 
 * Report bugs and request features in the `issue tracker`_, trying to follow
@@ -39,7 +39,7 @@ Reporting bugs
     trusted Scrapy developers, and its archives are not public.
 
 Well-written bug reports are very helpful, so keep in mind the following
-guidelines when reporting a new bug.
+guidelines when you're going to report a new bug.
 
 * check the :ref:`FAQ <faq>` first to see if your issue is addressed in a
   well-known question

From 782f866572d8bf8b1673ea962ae57d7631d1d9de Mon Sep 17 00:00:00 2001
From: CCInCharge <charles.l.chen.clc@gmail.com>
Date: Thu, 7 Jun 2018 16:39:48 -0700
Subject: [PATCH 1740/2974] Fix #3247: Allow scrapy.FormRequest.from_response
 method to handle duplicate keys

---
 scrapy/http/request/form.py | 11 ++++++++---
 tests/test_http_request.py  | 23 +++++++++++++++++++++++
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 95b38e990..c2413b431 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -114,10 +114,12 @@ def _get_form(response, formname, formid, formnumber, formxpath):
 
 def _get_inputs(form, formdata, dont_click, clickdata, response):
     try:
-        formdata = dict(formdata or ())
+        formdata_keys = dict(formdata or ()).keys()
     except (ValueError, TypeError):
         raise ValueError('formdata should be a dict or iterable of tuples')
 
+    if not formdata:
+        formdata = ()
     inputs = form.xpath('descendant::textarea'
                         '|descendant::select'
                         '|descendant::input[not(@type) or @type['
@@ -128,14 +130,17 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
                             "re": "http://exslt.org/regular-expressions"})
     values = [(k, u'' if v is None else v)
               for k, v in (_value(e) for e in inputs)
-              if k and k not in formdata]
+              if k and k not in formdata_keys]
 
     if not dont_click:
         clickable = _get_clickable(clickdata, form)
         if clickable and clickable[0] not in formdata and not clickable[0] is None:
             values.append(clickable)
 
-    values.extend((k, v) for k, v in formdata.items() if v is not None)
+    if isinstance(formdata, dict):
+        formdata = formdata.items()
+
+    values.extend((k, v) for k, v in formdata if v is not None)
     return values
 
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index a042f03b6..18fc6413c 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -401,6 +401,29 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs[u'test2'], [u'xxx µ'])
         self.assertEqual(fs[u'six'], [u'seven'])
 
+    def test_from_response_duplicate_form_key(self):
+        response = _buildresponse(
+                '<form></form>',
+                url='http://www.example.com')
+        req = self.request_class.from_response(response,
+                method='GET',
+                formdata=(('foo', 'bar'), ('foo', 'baz')))
+        self.assertEqual(urlparse(req.url).hostname, 'www.example.com')
+        self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz')
+    
+    def test_from_response_override_duplicate_form_key(self):
+        response = _buildresponse(
+            """<form action="get.php" method="POST">
+            <input type="hidden" name="one" value="1">
+            <input type="hidden" name="two" value="3">
+            </form>""")
+        req = self.request_class.from_response(
+            response,
+            formdata=(('two', '2'), ('two', '4')))
+        fs = _qs(req)
+        self.assertEqual(fs[b'one'], [b'1'])
+        self.assertEqual(fs[b'two'], [b'2', b'4'])
+
     def test_from_response_extra_headers(self):
         response = _buildresponse(
             """<form action="post.php" method="POST">

From 701cd2ff9d4be34fcf63a7410d09f425410635f9 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 9 Oct 2017 09:42:34 -0300
Subject: [PATCH 1741/2974] Add from_crawler support to dupefilters

---
 scrapy/core/scheduler.py  |  7 ++++++-
 tests/test_dupefilters.py | 21 +++++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index a54b4daf0..faed27fd1 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -26,7 +26,12 @@ class Scheduler(object):
     def from_crawler(cls, crawler):
         settings = crawler.settings
         dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
-        dupefilter = dupefilter_cls.from_settings(settings)
+        if hasattr(dupefilter_cls, 'from_crawler'):
+            dupefilter = dupefilter_cls.from_crawler(crawler)
+        elif hasattr(dupefilter_cls, 'from_settings'):
+            dupefilter = dupefilter_cls.from_settings(crawler.settings)
+        else:
+            dupefilter = dupefilter_cls()
         pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
         dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
         mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 2d1a4bfff..81524fddd 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -6,10 +6,31 @@ import shutil
 from scrapy.dupefilters import RFPDupeFilter
 from scrapy.http import Request
 from scrapy.utils.python import to_bytes
+from scrapy.utils.job import job_dir
+from scrapy.utils.test import get_crawler
 
 
 class RFPDupeFilterTest(unittest.TestCase):
 
+    def test_dupefilter_from_crawler(self):
+
+        class FromCrawlerRFPDupeFilter(RFPDupeFilter):
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                debug = crawler.settings.getbool('DUPEFILTER_DEBUG')
+                df = cls(job_dir(crawler.settings), debug)
+                df.user_agent = crawler.settings.get('USER_AGENT')
+                return df
+
+        crawler = get_crawler(settings_dict={'DUPEFILTER_DEBUG': True, 'USER_AGENT': 'test ua'})
+        dupefilter = FromCrawlerRFPDupeFilter.from_crawler(crawler)
+
+        self.assertTrue(dupefilter.debug)
+        self.assertEqual(dupefilter.user_agent, 'test ua')
+
+        dupefilter.close('finished')
+
     def test_filter(self):
         dupefilter = RFPDupeFilter()
         dupefilter.open()

From d306fe30ac08401e74fd20cf90e5164a4125d8e1 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 30 Dec 2017 22:49:22 -0300
Subject: [PATCH 1742/2974] Test dupefilter creation by the Scheduler

---
 tests/test_dupefilters.py | 52 ++++++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 17 deletions(-)

diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 81524fddd..990b5141c 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -5,31 +5,49 @@ import shutil
 
 from scrapy.dupefilters import RFPDupeFilter
 from scrapy.http import Request
+from scrapy.core.scheduler import Scheduler
 from scrapy.utils.python import to_bytes
 from scrapy.utils.job import job_dir
 from scrapy.utils.test import get_crawler
 
 
+class FromCrawlerRFPDupeFilter(RFPDupeFilter):
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        debug = crawler.settings.getbool('DUPEFILTER_DEBUG')
+        df = cls(job_dir(crawler.settings), debug)
+        df.method = crawler.settings.get('METHOD')
+        return df
+
+
+class FromSettingsRFPDupeFilter(RFPDupeFilter):
+
+    @classmethod
+    def from_settings(cls, settings):
+        debug = settings.getbool('DUPEFILTER_DEBUG')
+        df = cls(job_dir(settings), debug)
+        df.method = settings.get('METHOD')
+        return df
+
+
 class RFPDupeFilterTest(unittest.TestCase):
 
-    def test_dupefilter_from_crawler(self):
+    def test_from_crawler_scheduler(self):
+        settings = {'DUPEFILTER_DEBUG': True, 'METHOD': 'from_crawler',
+                    'DUPEFILTER_CLASS': __name__  + '.FromCrawlerRFPDupeFilter'}
+        crawler = get_crawler(settings_dict=settings)
+        scheduler = Scheduler.from_crawler(crawler)
+        self.assertTrue(scheduler.df.debug)
+        self.assertEqual(scheduler.df.method, 'from_crawler')
 
-        class FromCrawlerRFPDupeFilter(RFPDupeFilter):
-
-            @classmethod
-            def from_crawler(cls, crawler):
-                debug = crawler.settings.getbool('DUPEFILTER_DEBUG')
-                df = cls(job_dir(crawler.settings), debug)
-                df.user_agent = crawler.settings.get('USER_AGENT')
-                return df
-
-        crawler = get_crawler(settings_dict={'DUPEFILTER_DEBUG': True, 'USER_AGENT': 'test ua'})
-        dupefilter = FromCrawlerRFPDupeFilter.from_crawler(crawler)
-
-        self.assertTrue(dupefilter.debug)
-        self.assertEqual(dupefilter.user_agent, 'test ua')
-
-        dupefilter.close('finished')
+    def test_from_settings_scheduler(self):
+        settings = {'DUPEFILTER_DEBUG': True, 'METHOD': 'from_settings',
+                    'DUPEFILTER_CLASS': __name__  + '.FromSettingsRFPDupeFilter'}
+        crawler = get_crawler(settings_dict=settings)
+        scheduler = Scheduler.from_crawler(crawler)
+        self.assertTrue(scheduler.df.debug)
+        self.assertEqual(scheduler.df.method, 'from_settings')
 
     def test_filter(self):
         dupefilter = RFPDupeFilter()

From 0089a4ab31d1764dd38c30c4448a4c62efd9b9c3 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 23 Mar 2018 13:19:31 -0300
Subject: [PATCH 1743/2974] Add test for direct creation of dupefilter (no
 from_crawler/from_settings)

---
 tests/test_dupefilters.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 990b5141c..02a01fc94 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -17,7 +17,7 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter):
     def from_crawler(cls, crawler):
         debug = crawler.settings.getbool('DUPEFILTER_DEBUG')
         df = cls(job_dir(crawler.settings), debug)
-        df.method = crawler.settings.get('METHOD')
+        df.method = 'from_crawler'
         return df
 
 
@@ -27,28 +27,40 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter):
     def from_settings(cls, settings):
         debug = settings.getbool('DUPEFILTER_DEBUG')
         df = cls(job_dir(settings), debug)
-        df.method = settings.get('METHOD')
+        df.method = 'from_settings'
         return df
 
 
+class DirectRFPDupeFilter(RFPDupeFilter):
+    method = 'n/a'
+
+
 class RFPDupeFilterTest(unittest.TestCase):
 
-    def test_from_crawler_scheduler(self):
-        settings = {'DUPEFILTER_DEBUG': True, 'METHOD': 'from_crawler',
+    def test_df_from_crawler_scheduler(self):
+        settings = {'DUPEFILTER_DEBUG': True,
                     'DUPEFILTER_CLASS': __name__  + '.FromCrawlerRFPDupeFilter'}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
         self.assertEqual(scheduler.df.method, 'from_crawler')
 
-    def test_from_settings_scheduler(self):
-        settings = {'DUPEFILTER_DEBUG': True, 'METHOD': 'from_settings',
+    def test_df_from_settings_scheduler(self):
+        settings = {'DUPEFILTER_DEBUG': True,
                     'DUPEFILTER_CLASS': __name__  + '.FromSettingsRFPDupeFilter'}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
         self.assertEqual(scheduler.df.method, 'from_settings')
 
+    def test_df_direct_scheduler(self):
+        settings = {'DUPEFILTER_DEBUG': True,
+                    'DUPEFILTER_CLASS': __name__  + '.DirectRFPDupeFilter'}
+        crawler = get_crawler(settings_dict=settings)
+        scheduler = Scheduler.from_crawler(crawler)
+        self.assertTrue(scheduler.df.debug)
+        self.assertEqual(scheduler.df.method, 'n/a')
+
     def test_filter(self):
         dupefilter = RFPDupeFilter()
         dupefilter.open()

From 9e14f8c7e4141fd5216efc378c9d003ea981a4d5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 23 Mar 2018 21:19:57 -0300
Subject: [PATCH 1744/2974] Fix test for dupefilter

---
 tests/test_dupefilters.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 02a01fc94..db69597a2 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -31,7 +31,7 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter):
         return df
 
 
-class DirectRFPDupeFilter(RFPDupeFilter):
+class DirectDupeFilter(object):
     method = 'n/a'
 
 
@@ -54,11 +54,9 @@ class RFPDupeFilterTest(unittest.TestCase):
         self.assertEqual(scheduler.df.method, 'from_settings')
 
     def test_df_direct_scheduler(self):
-        settings = {'DUPEFILTER_DEBUG': True,
-                    'DUPEFILTER_CLASS': __name__  + '.DirectRFPDupeFilter'}
+        settings = {'DUPEFILTER_CLASS': __name__  + '.DirectDupeFilter'}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
-        self.assertTrue(scheduler.df.debug)
         self.assertEqual(scheduler.df.method, 'n/a')
 
     def test_filter(self):

From 999341b60bd5289ca97c8187d0ff380a8555ba5e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 20 Jul 2018 22:17:55 -0300
Subject: [PATCH 1745/2974] Simplify dupefilter creation

---
 scrapy/core/scheduler.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index faed27fd1..eb790a67e 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -4,7 +4,7 @@ import logging
 from os.path import join, exists
 
 from scrapy.utils.reqser import request_to_dict, request_from_dict
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, create_instance
 from scrapy.utils.job import job_dir
 
 logger = logging.getLogger(__name__)
@@ -26,12 +26,7 @@ class Scheduler(object):
     def from_crawler(cls, crawler):
         settings = crawler.settings
         dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
-        if hasattr(dupefilter_cls, 'from_crawler'):
-            dupefilter = dupefilter_cls.from_crawler(crawler)
-        elif hasattr(dupefilter_cls, 'from_settings'):
-            dupefilter = dupefilter_cls.from_settings(crawler.settings)
-        else:
-            dupefilter = dupefilter_cls()
+        dupefilter = create_instance(dupefilter_cls, settings, crawler)
         pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
         dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
         mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])

From d6d3e87e3a4fd306829a84f934a971fd4e337a26 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 27 Jul 2018 14:47:52 -0300
Subject: [PATCH 1746/2974] Rename test file

---
 ..._spidermiddleware.py => test_spidermiddleware_output_chain.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename tests/{test_spidermiddleware.py => test_spidermiddleware_output_chain.py} (100%)

diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware_output_chain.py
similarity index 100%
rename from tests/test_spidermiddleware.py
rename to tests/test_spidermiddleware_output_chain.py

From 801d3c07b4b7e57d50429e714a8255e7747568f4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 27 Jul 2018 15:06:25 -0300
Subject: [PATCH 1747/2974] Fix bad exception handling, add tests

---
 scrapy/core/spidermw.py                       |  2 +
 tests/test_spidermiddleware_invalid_values.py | 82 +++++++++++++++++++
 2 files changed, 84 insertions(+)
 create mode 100644 tests/test_spidermiddleware_invalid_values.py

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 8607ed620..1b67af130 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -66,6 +66,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
                     if result is not None:
                         raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \
                                              .format(fname(method), type(result)))
+                except _InvalidOutput:
+                    raise
                 except:
                     return scrape_func(Failure(), request, spider)
             return scrape_func(response, request, spider)
diff --git a/tests/test_spidermiddleware_invalid_values.py b/tests/test_spidermiddleware_invalid_values.py
new file mode 100644
index 000000000..0d9af8951
--- /dev/null
+++ b/tests/test_spidermiddleware_invalid_values.py
@@ -0,0 +1,82 @@
+from twisted.trial.unittest import TestCase
+from twisted.python.failure import Failure
+
+from scrapy.spiders import Spider
+from scrapy.http import Request, Response
+from scrapy.exceptions import _InvalidOutput
+from scrapy.utils.test import get_crawler
+from scrapy.core.spidermw import SpiderMiddlewareManager
+from tests import mock
+
+
+class SpiderMiddlewareTestCase(TestCase):
+
+    def setUp(self):
+        self.request = Request('http://example.com/index.html')
+        self.response = Response(self.request.url, request=self.request)
+        self.crawler = get_crawler(Spider)
+        self.spider = self.crawler._create_spider('foo')
+        self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
+
+    def _scrape_response(self):
+        """Execute spider mw manager's scrape_response method and return the result.
+        Raise exception in case of failure.
+        """
+        scrape_func = mock.MagicMock()
+        dfd = self.mwman.scrape_response(scrape_func, self.response, self.request, self.spider)
+        # catch deferred result and return the value
+        results = []
+        dfd.addBoth(results.append)
+        self._wait(dfd)
+        ret = results[0]
+        return ret
+
+
+class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase):
+    """Invalid return value for process_spider_input method"""
+
+    def test_invalid_process_spider_input(self):
+
+        class InvalidProcessSpiderInputMiddleware:
+            def process_spider_input(self, response, spider):
+                return 1
+
+        self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware())
+        result = self._scrape_response()
+        self.assertIsInstance(result, Failure)
+        self.assertIsInstance(result.value, _InvalidOutput)
+
+
+class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase):
+    """Invalid return value for process_spider_output method"""
+
+    def test_invalid_process_spider_output(self):
+
+        class InvalidProcessSpiderOutputMiddleware:
+            def process_spider_output(self, response, result, spider):
+                return 1
+
+        self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware())
+        result = self._scrape_response()
+        self.assertIsInstance(result, Failure)
+        self.assertIsInstance(result.value, _InvalidOutput)
+
+
+class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):
+    """Invalid return value for process_spider_exception method"""
+
+    def test_invalid_process_spider_exception(self):
+
+        class InvalidProcessSpiderOutputExceptionMiddleware:
+            def process_spider_exception(self, response, exception, spider):
+                return 1
+
+        class RaiseExceptionProcessSpiderOutputMiddleware:
+            def process_spider_output(self, response, result, spider):
+                raise Exception()
+
+        self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())
+        self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
+        result = self._scrape_response()
+        self.assertIsInstance(result, Failure)
+        self.assertIsInstance(result.value, _InvalidOutput)

From 980be4cb4b70c71e5502c8f31c85882540f3f078 Mon Sep 17 00:00:00 2001
From: Kevin Lloyd Bernal <kevinoxy@gmail.com>
Date: Tue, 31 Jul 2018 19:05:04 +0800
Subject: [PATCH 1748/2974] provide better error message when disabling s3
 exporter

---
 scrapy/extensions/feedexport.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 7c7db387e..22ebf3b3f 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -269,9 +269,10 @@ class FeedExporter(object):
             try:
                 self._get_storage(uri)
                 return True
-            except NotConfigured:
-                logger.error("Disabled feed storage scheme: %(scheme)s",
-                             {'scheme': scheme})
+            except NotConfigured as e:
+                logger.error("Disabled feed storage scheme: %(scheme)s. "
+                             "Reason: %(reason)s",
+                             {'scheme': scheme, 'reason': str(e)})
         else:
             logger.error("Unknown feed storage scheme: %(scheme)s",
                          {'scheme': scheme})

From c87a4f5c6fd97d22345bb444bc90b56ef0624aa5 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 1 Aug 2018 01:45:16 +0500
Subject: [PATCH 1749/2974] remove unused imports from
 scrapy/settings/__init__.py

This is a follow-up to https://github.com/scrapy/scrapy/pull/3327
---
 scrapy/settings/__init__.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 7d6d20164..14c93bef2 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,13 +1,10 @@
 import six
 import json
 import copy
-import warnings
 from collections import MutableMapping
 from importlib import import_module
 from pprint import pformat
 
-from scrapy.exceptions import ScrapyDeprecationWarning
-
 from . import default_settings
 
 

From 8c55f5eb159ae85d468a89b74ffaff3e824144ab Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 3 Aug 2018 15:16:26 -0300
Subject: [PATCH 1750/2974] Simplify check for re-raised exception. Add tests.

---
 scrapy/core/spidermw.py                       |  4 ++--
 ...lid_values.py => test_spidermiddleware.py} | 20 +++++++++++++++++++
 2 files changed, 22 insertions(+), 2 deletions(-)
 rename tests/{test_spidermiddleware_invalid_values.py => test_spidermiddleware.py} (79%)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 1b67af130..4268c91d6 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -103,7 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                         yield r
                 except Exception as ex:
                     exception_result = process_spider_exception(Failure(ex), index+1)
-                    if exception_result is None or isinstance(exception_result, Failure):
+                    if isinstance(exception_result, Failure):
                         raise
                     recovered.extend(exception_result)
 
@@ -115,7 +115,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                     result = method(response=response, result=result, spider=spider)
                 except Exception as ex:
                     exception_result = process_spider_exception(Failure(ex), method_index+1)
-                    if exception_result is None or isinstance(exception_result, Failure):
+                    if isinstance(exception_result, Failure):
                         raise
                     return exception_result
                 if _isiterable(result):
diff --git a/tests/test_spidermiddleware_invalid_values.py b/tests/test_spidermiddleware.py
similarity index 79%
rename from tests/test_spidermiddleware_invalid_values.py
rename to tests/test_spidermiddleware.py
index 0d9af8951..54756f2ff 100644
--- a/tests/test_spidermiddleware_invalid_values.py
+++ b/tests/test_spidermiddleware.py
@@ -80,3 +80,23 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):
         result = self._scrape_response()
         self.assertIsInstance(result, Failure)
         self.assertIsInstance(result.value, _InvalidOutput)
+
+
+class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
+    """Re raise the exception by returning None"""
+
+    def test_process_spider_exception_return_none(self):
+
+        class ProcessSpiderOutputExceptionReturnNoneMiddleware:
+            def process_spider_exception(self, response, exception, spider):
+                return None
+
+        class RaiseExceptionProcessSpiderOutputMiddleware:
+            def process_spider_output(self, response, result, spider):
+                1/0
+
+        self.mwman._add_middleware(ProcessSpiderOutputExceptionReturnNoneMiddleware())
+        self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
+        result = self._scrape_response()
+        self.assertIsInstance(result, Failure)
+        self.assertIsInstance(result.value, ZeroDivisionError)

From 40449fa0eb707bac1ae2b78f0f812372e90f17b7 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 3 Aug 2018 18:20:25 -0300
Subject: [PATCH 1751/2974] Update docs, add tests, remove FIXME comment

---
 docs/topics/spider-middleware.rst           |  3 +-
 scrapy/core/scraper.py                      |  1 -
 tests/test_spidermiddleware.py              |  4 +-
 tests/test_spidermiddleware_output_chain.py | 43 ++++++++++++++-------
 4 files changed, 33 insertions(+), 18 deletions(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 915833c54..7db623cf4 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -78,7 +78,8 @@ following methods:
 
         If it raises an exception, Scrapy won't bother calling any other spider
         middleware :meth:`process_spider_input` and will call the request
-        errback.  The output of the errback is chained back in the other
+        errback if there is one, otherwise it will start the :meth:`process_spider_exception`
+        chain. The output of the errback is chained back in the other
         direction for :meth:`process_spider_output` to process it, or
         :meth:`process_spider_exception` if it raised an exception.
 
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index ee1e95a0c..d7fe721fb 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -135,7 +135,6 @@ class Scraper(object):
             return self.spidermw.scrape_response(
                 self.call_spider, request_result, request, spider)
         else:
-            # FIXME: don't ignore errors in spider middleware
             dfd = self.call_spider(request_result, request, spider)
             return dfd.addErrback(
                 self._log_download_errors, request_result, request, spider)
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 54756f2ff..832fd3330 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -87,7 +87,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
 
     def test_process_spider_exception_return_none(self):
 
-        class ProcessSpiderOutputExceptionReturnNoneMiddleware:
+        class ProcessSpiderExceptionReturnNoneMiddleware:
             def process_spider_exception(self, response, exception, spider):
                 return None
 
@@ -95,7 +95,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
             def process_spider_output(self, response, result, spider):
                 1/0
 
-        self.mwman._add_middleware(ProcessSpiderOutputExceptionReturnNoneMiddleware())
+        self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())
         self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
         result = self._scrape_response()
         self.assertIsInstance(result, Failure)
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 0f5646a72..6f8727a15 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -45,8 +45,13 @@ class RecoveryMiddleware:
 
 # ================================================================================
 # (1) exceptions from a spider middleware's process_spider_input method
-class ProcessSpiderInputSpider(Spider):
-    name = 'ProcessSpiderInputSpider'
+class FailProcessSpiderInputMiddleware:
+    def process_spider_input(self, response, spider):
+        spider.logger.info('Middleware: will raise IndexError')
+        raise IndexError()
+
+class ProcessSpiderInputSpiderWithoutErrback(Spider):
+    name = 'ProcessSpiderInputSpiderWithoutErrback'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
             # spider
@@ -58,23 +63,23 @@ class ProcessSpiderInputSpider(Spider):
     }
 
     def start_requests(self):
-        yield Request(url=self.mockserver.url('/status?n=200'),
-                      callback=self.parse, errback=self.errback)
+        yield Request(url=self.mockserver.url('/status?n=200'), callback=self.parse)
 
     def parse(self, response):
         return {'from': 'callback'}
 
+
+class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback):
+    name = 'ProcessSpiderInputSpiderWithErrback'
+
+    def start_requests(self):
+        yield Request(url=self.mockserver.url('/status?n=200'), callback=self.parse, errback=self.errback)
+
     def errback(self, failure):
         self.logger.info('Got a Failure on the Request errback')
         return {'from': 'errback'}
 
 
-class FailProcessSpiderInputMiddleware:
-    def process_spider_input(self, response, spider):
-        spider.logger.info('Middleware: will raise IndexError')
-        raise IndexError()
-
-
 # ================================================================================
 # (2) exceptions from a spider callback (generator)
 class GeneratorCallbackSpider(Spider):
@@ -278,12 +283,22 @@ class TestSpiderMiddleware(TestCase):
         self.assertIn("'item_scraped_count': 3", str(log))
 
     @defer.inlineCallbacks
-    def test_process_spider_input_errback(self):
+    def test_process_spider_input_without_errback(self):
         """
-        (1) An exception from the process_spider_input chain should not be caught by the
-        process_spider_exception chain, it should go directly to the Request errback
+        (1.1) An exception from the process_spider_input chain should be caught by the
+        process_spider_exception chain from the start if the Request has no errback
         """
-        log1 = yield self.crawl_log(ProcessSpiderInputSpider)
+        log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback)
+        self.assertIn("Middleware: will raise IndexError", str(log1))
+        self.assertIn("Middleware: IndexError exception caught", str(log1))
+
+    @defer.inlineCallbacks
+    def test_process_spider_input_with_errback(self):
+        """
+        (1.2) An exception from the process_spider_input chain should not be caught by the
+        process_spider_exception chain if the Request has an errback
+        """
+        log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback)
         self.assertNotIn("Middleware: IndexError exception caught", str(log1))
         self.assertIn("Middleware: will raise IndexError", str(log1))
         self.assertIn("Got a Failure on the Request errback", str(log1))

From d3aa1e86664b02a20522abbb265dc9b91b95f111 Mon Sep 17 00:00:00 2001
From: Raphael Wuillemier <wuillemr@essilor.de>
Date: Mon, 6 Aug 2018 17:40:31 +0200
Subject: [PATCH 1753/2974] Updated tutorial.rst to include more and up-to-date
 beginner resources

---
 docs/intro/tutorial.rst | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 20538e90f..0db6a6218 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -26,15 +26,26 @@ If you're already familiar with other languages, and want to learn Python
 quickly, we recommend reading through `Dive Into Python 3`_.  Alternatively,
 you can follow the `Python Tutorial`_.
 
-If you're new to programming and want to start with Python, you may find useful
-the online book `Learn Python The Hard Way`_. You can also take a look at `this
-list of Python resources for non-programmers`_.
+If you're new to programming and want to start with Python, the following books
+may be useful to you: 
+
+* `Automate the Boring Stuff With Python`_
+
+* `How To Think Like a Computer Scientist`_ 
+
+* `Learn Python 3 The Hard Way`_ 
+
+You can also take a look at `this list of Python resources for non-programmers`_,
+as well as the `suggested resources in the learnpython-subreddit`_. 
 
 .. _Python: https://www.python.org/
 .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
 .. _Dive Into Python 3: http://www.diveintopython3.net
 .. _Python Tutorial: https://docs.python.org/3/tutorial
-.. _Learn Python The Hard Way: https://learnpythonthehardway.org/book/
+.. _Automate the Boring Stuff With Python: https://automatetheboringstuff.com/
+.. _How To Think Like a Computer Scientist: http://openbookproject.net/thinkcs/python/english3e/
+.. _Learn Python 3 The Hard Way: https://learnpythonthehardway.org/python3/
+.. _suggested resources in the learnpython-subreddit: https://www.reddit.com/r/learnpython/wiki/index#wiki_new_to_python.3F
 
 
 Creating a project

From 16dad81715d3970149c0cf7a318e73a0d84be1ff Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Thu, 9 Aug 2018 21:07:25 +0300
Subject: [PATCH 1754/2974] Fix contract errback

---
 scrapy/contracts/__init__.py |  2 +-
 tests/test_contracts.py      | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 5eaee3d11..8315d21d2 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -84,7 +84,7 @@ class ContractsManager(object):
 
         def eb_wrapper(failure):
             case = _create_testcase(method, 'errback')
-            exc_info = failure.value, failure.type, failure.getTracebackObject()
+            exc_info = failure.type, failure.value, failure.getTracebackObject()
             results.addError(case, exc_info)
 
         request.callback = cb_wrapper
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 1cea2afb7..b07cbee1e 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -1,7 +1,9 @@
 from unittest import TextTestResult
 
+from twisted.python import failure
 from twisted.trial import unittest
 
+from scrapy.spidermiddlewares.httperror import HttpError
 from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.item import Item, Field
@@ -185,3 +187,18 @@ class ContractsManagerTest(unittest.TestCase):
                 self.results)
         request.callback(response)
         self.should_fail()
+
+    def test_errback(self):
+        spider = TestSpider()
+        response = ResponseMock()
+
+        try:
+            raise HttpError(response, 'Ignoring non-200 response')
+        except HttpError:
+            failure_mock = failure.Failure()
+
+        request = self.conman.from_method(spider.returns_request, self.results)
+        request.errback(failure_mock)
+
+        self.assertFalse(self.results.failures)
+        self.assertTrue(self.results.errors)

From fb7d4cbce379c4e3fac2ca89b6e0772d0c690935 Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Sat, 11 Aug 2018 16:08:26 +0300
Subject: [PATCH 1755/2974] Add error handling in contracts

---
 scrapy/contracts/__init__.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 5eaee3d11..18014e290 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -41,7 +41,11 @@ class ContractsManager(object):
         requests = []
         for method in self.tested_methods_from_spidercls(type(spider)):
             bound_method = spider.__getattribute__(method)
-            requests.append(self.from_method(bound_method, results))
+            try:
+                requests.append(self.from_method(bound_method, results))
+            except:
+                case = _create_testcase(bound_method, 'contract')
+                results.addError(case, sys.exc_info())
 
         return requests
 

From ebbde57eca310d3c5c0f530a2e668572dde4d952 Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Sat, 11 Aug 2018 17:50:56 +0300
Subject: [PATCH 1756/2974] Add custom contracts tests

---
 tests/test_contracts.py | 55 +++++++++++++++++++++++++++++++++++++++--
 1 file changed, 53 insertions(+), 2 deletions(-)

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 1cea2afb7..078ef6e0d 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -5,7 +5,7 @@ from twisted.trial import unittest
 from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.item import Item, Field
-from scrapy.contracts import ContractsManager
+from scrapy.contracts import ContractsManager, Contract
 from scrapy.contracts.default import (
     UrlContract,
     ReturnsContract,
@@ -22,6 +22,21 @@ class ResponseMock(object):
     url = 'http://scrapy.org'
 
 
+class CustomSuccessContract(Contract):
+    name = 'custom_success_contract'
+
+    def adjust_request_args(self, args):
+        args['url'] = 'http://scrapy.org'
+        return args
+
+
+class CustomFailContract(Contract):
+    name = 'custom_fail_contract'
+
+    def adjust_request_args(self, args):
+        raise TypeError('Error in adjust_request_args')
+
+
 class TestSpider(Spider):
     name = 'demo_spider'
 
@@ -99,8 +114,34 @@ class TestSpider(Spider):
         pass
 
 
+class CustomContractSuccessSpider(Spider):
+    name = 'custom_contract_success_spider'
+
+    def parse(self, response):
+        """
+        @custom_success_contract
+        """
+        pass
+
+
+class CustomContractFailSpider(Spider):
+    name = 'custom_contract_fail_spider'
+
+    def parse(self, response):
+        """
+        @custom_fail_contract
+        """
+        pass
+
+
 class ContractsManagerTest(unittest.TestCase):
-    contracts = [UrlContract, ReturnsContract, ScrapesContract]
+    contracts = [
+        UrlContract,
+        ReturnsContract,
+        ScrapesContract,
+        CustomSuccessContract,
+        CustomFailContract
+    ]
 
     def setUp(self):
         self.conman = ContractsManager(self.contracts)
@@ -114,6 +155,9 @@ class ContractsManagerTest(unittest.TestCase):
         self.assertTrue(self.results.failures)
         self.assertFalse(self.results.errors)
 
+    def should_error(self):
+        self.assertTrue(self.results.errors)
+
     def test_contracts(self):
         spider = TestSpider()
 
@@ -185,3 +229,10 @@ class ContractsManagerTest(unittest.TestCase):
                 self.results)
         request.callback(response)
         self.should_fail()
+
+    def test_custom_contracts(self):
+        self.conman.from_spider(CustomContractSuccessSpider(), self.results)
+        self.should_succeed()
+
+        self.conman.from_spider(CustomContractFailSpider(), self.results)
+        self.should_error()

From 76220e8733b1462ecaa5db1cdd4a58ad4e93ceb3 Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Sat, 11 Aug 2018 18:49:12 +0300
Subject: [PATCH 1757/2974] Use inspect.getmembers in
 tested_methods_from_spidercls

---
 scrapy/contracts/__init__.py |  3 ++-
 tests/test_contracts.py      | 10 ++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 8315d21d2..de7ac4503 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -1,6 +1,7 @@
 import sys
 import re
 from functools import wraps
+from inspect import getmembers
 from unittest import TestCase
 
 from scrapy.http import Request
@@ -17,7 +18,7 @@ class ContractsManager(object):
 
     def tested_methods_from_spidercls(self, spidercls):
         methods = []
-        for key, value in vars(spidercls).items():
+        for key, value in getmembers(spidercls):
             if (callable(value) and value.__doc__ and
                     re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
                 methods.append(key)
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index b07cbee1e..322d20c4c 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -101,6 +101,10 @@ class TestSpider(Spider):
         pass
 
 
+class InheritsTestSpider(TestSpider):
+    name = 'inherits_demo_spider'
+
+
 class ContractsManagerTest(unittest.TestCase):
     contracts = [UrlContract, ReturnsContract, ScrapesContract]
 
@@ -202,3 +206,9 @@ class ContractsManagerTest(unittest.TestCase):
 
         self.assertFalse(self.results.failures)
         self.assertTrue(self.results.errors)
+
+    def test_inherited_contracts(self):
+        spider = InheritsTestSpider()
+
+        requests = self.conman.from_spider(spider, self.results)
+        self.assertTrue(requests)

From 8fc017d345af1e6fa53334fc9df92ebd8f9dc32e Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Sat, 11 Aug 2018 19:25:33 +0300
Subject: [PATCH 1758/2974] Add dont_filter to ContractsManager requests

---
 scrapy/contracts/__init__.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 8315d21d2..2569ab151 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -50,7 +50,12 @@ class ContractsManager(object):
         if contracts:
             # calculate request args
             args, kwargs = get_spec(Request.__init__)
+
+            # Don't filter requests to allow
+            # testing different callbacks on the same URL.
+            kwargs['dont_filter'] = True
             kwargs['callback'] = method
+
             for contract in contracts:
                 kwargs = contract.adjust_request_args(kwargs)
 

From b4b1e4834376e8565d727a1a5087cae31450f931 Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Sat, 11 Aug 2018 22:18:43 +0300
Subject: [PATCH 1759/2974] Add ability to use FormRequest in contracts

---
 docs/topics/contracts.rst    |  7 +++++--
 scrapy/contracts/__init__.py |  7 ++++++-
 tests/test_contracts.py      | 26 ++++++++++++++++++++++++--
 3 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index ba1421c42..cac52042a 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -86,8 +86,11 @@ override three methods:
     .. method:: Contract.adjust_request_args(args)
 
         This receives a ``dict`` as an argument containing default arguments
-        for :class:`~scrapy.http.Request` object. Must return the same or a
-        modified version of it.
+        for request object. If ``formdata`` is in ``args``, then
+        :class:`~scrapy.http.FormRequest` object is created,
+        otherwise :class:`~scrapy.http.Request` is used.
+
+        Must return the same or a modified version of it.
 
     .. method:: Contract.pre_process(response)
 
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 8315d21d2..ca2a8d384 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -3,6 +3,7 @@ import re
 from functools import wraps
 from unittest import TestCase
 
+from scrapy import FormRequest
 from scrapy.http import Request
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.python import get_spec
@@ -57,7 +58,11 @@ class ContractsManager(object):
             # create and prepare request
             args.remove('self')
             if set(args).issubset(set(kwargs)):
-                request = Request(**kwargs)
+                if 'formdata' in kwargs:
+                    kwargs['method'] = 'POST'
+                    request = FormRequest(**kwargs)
+                else:
+                    request = Request(**kwargs)
 
                 # execute pre and post hooks in order
                 for contract in reversed(contracts):
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index b07cbee1e..f2085f711 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -3,11 +3,12 @@ from unittest import TextTestResult
 from twisted.python import failure
 from twisted.trial import unittest
 
+from scrapy import FormRequest
 from scrapy.spidermiddlewares.httperror import HttpError
 from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.item import Item, Field
-from scrapy.contracts import ContractsManager
+from scrapy.contracts import ContractsManager, Contract
 from scrapy.contracts.default import (
     UrlContract,
     ReturnsContract,
@@ -24,6 +25,14 @@ class ResponseMock(object):
     url = 'http://scrapy.org'
 
 
+class CustomFormContract(Contract):
+    name = 'custom_form'
+
+    def adjust_request_args(self, args):
+        args['formdata'] = {'name': 'scrapy'}
+        return args
+
+
 class TestSpider(Spider):
     name = 'demo_spider'
 
@@ -100,9 +109,16 @@ class TestSpider(Spider):
         """
         pass
 
+    def custom_form(self, response):
+        """
+        @url http://scrapy.org
+        @custom_form
+        """
+        pass
+
 
 class ContractsManagerTest(unittest.TestCase):
-    contracts = [UrlContract, ReturnsContract, ScrapesContract]
+    contracts = [UrlContract, ReturnsContract, ScrapesContract, CustomFormContract]
 
     def setUp(self):
         self.conman = ContractsManager(self.contracts)
@@ -202,3 +218,9 @@ class ContractsManagerTest(unittest.TestCase):
 
         self.assertFalse(self.results.failures)
         self.assertTrue(self.results.errors)
+
+    def test_form_contract(self):
+        spider = TestSpider()
+        request = self.conman.from_method(spider.custom_form, self.results)
+        self.assertEqual(request.method, 'POST')
+        self.assertIsInstance(request, FormRequest)

From 1d25c98eb337763f613d73fd534d3dc5e64eb66f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Tue, 3 Jul 2018 16:41:53 -0300
Subject: [PATCH 1760/2974] Add appveyor.yml

---
 appveyor.yml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 appveyor.yml

diff --git a/appveyor.yml b/appveyor.yml
new file mode 100644
index 000000000..81432be39
--- /dev/null
+++ b/appveyor.yml
@@ -0,0 +1,14 @@
+platform: x86
+version: '{branch}-{build}'
+environment:
+  matrix:
+    - PYTHON: "C:\\Python36"
+      TOX_ENV: py36
+
+install:
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE"
+  - "pip install -U tox twine wheel"
+build: false
+test_script:
+  - "tox -e %TOX_ENV%"

From 4c53957f5bd0f64b00d9488f171f0f8a9620af02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Tue, 3 Jul 2018 16:56:05 -0300
Subject: [PATCH 1761/2974] Skip leveldb tests on windows

---
 tests/requirements-py3.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 8d9ce5231..1e4a4b641 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -3,7 +3,7 @@ pytest-twisted
 pytest-cov==2.5.1
 testfixtures
 jmespath
-leveldb
+leveldb; sys_platform != "win32"
 botocore
 # optional for shell wrapper tests
 bpython

From dd75297e3fb37800c0ef763003a54081e9fad4cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Tue, 3 Jul 2018 16:58:02 -0300
Subject: [PATCH 1762/2974] Run Appveyor CI for master and release branches
 only, but also PRs

---
 appveyor.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/appveyor.yml b/appveyor.yml
index 81432be39..4f3c69847 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -5,6 +5,11 @@ environment:
     - PYTHON: "C:\\Python36"
       TOX_ENV: py36
 
+branches:
+  only:
+    - master
+    - /d+\.\d+\.\d+[\w\-]*$/
+
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
   - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE"

From 19ad94105f70a26ce47683d488b66fb77b79067b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Tue, 3 Jul 2018 17:15:48 -0300
Subject: [PATCH 1763/2974] pywin32 is required to run tests under windows

---
 tests/requirements-py3.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 1e4a4b641..7c1aacd81 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -9,3 +9,4 @@ botocore
 bpython
 ipython
 brotlipy
+pywin32; sys_platform == "win32"

From 152fde70b12f3c6e0e73230370f79f4b4a7ea906 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Tue, 2 Feb 2016 18:23:23 +0000
Subject: [PATCH 1764/2974] Fix FTPTestCase by using Windows-friendly temporary
 file name

---
 tests/test_downloader_handlers.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index c91be2c0c..fe76989f4 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1,7 +1,8 @@
 import os
 import six
-import contextlib
 import shutil
+import tempfile
+import contextlib
 try:
     from unittest import mock
 except ImportError:
@@ -913,7 +914,8 @@ class BaseFTPTestCase(unittest.TestCase):
         return self._add_test_callbacks(d, _test)
 
     def test_ftp_local_filename(self):
-        local_fname = b"/tmp/file.txt"
+        f, local_fname = tempfile.mkstemp()
+        os.close(f)
         meta = {"ftp_local_filename": local_fname}
         meta.update(self.req_meta)
         request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
@@ -922,7 +924,8 @@ class BaseFTPTestCase(unittest.TestCase):
 
         def _test(r):
             self.assertEqual(r.body, local_fname)
-            self.assertEqual(r.headers, {b'Local Filename': [b'/tmp/file.txt'], b'Size': [b'17']})
+            self.assertEqual(r.headers, {b'Local Filename': [local_fname],
+                                         b'Size': [b'17']})
             self.assertTrue(os.path.exists(local_fname))
             with open(local_fname, "rb") as f:
                 self.assertEqual(f.read(), b"I have the power!")

From 57a1d66c61423c85703a4008d351e9d566580810 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Wed, 3 Feb 2016 16:56:43 +0000
Subject: [PATCH 1765/2974] Fix test issues caused by Windows pipe buffer
 filling up

---
 tests/test_commands.py | 48 +++++++++++++++++++++---------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 7d9071b64..84c38c0e9 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -3,18 +3,17 @@ import os
 import sys
 import subprocess
 import tempfile
-from time import sleep
 from os.path import exists, join, abspath
 from shutil import rmtree, copytree
 from tempfile import mkdtemp
 from contextlib import contextmanager
+from threading import Timer
 
 from twisted.trial import unittest
 from twisted.internet import defer
 
 import scrapy
 from scrapy.utils.python import to_native_str
-from scrapy.utils.python import retry_on_eintr
 from scrapy.utils.test import get_testenv
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
@@ -46,16 +45,18 @@ class ProjectTest(unittest.TestCase):
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                              **popen_kwargs)
 
-        waited = 0
-        interval = 0.2
-        while p.poll() is None:
-            sleep(interval)
-            waited += interval
-            if waited > 15:
-                p.kill()
-                assert False, 'Command took too much time to complete'
+        def kill_proc():
+            p.kill()
+            assert False, 'Command took too much time to complete'
 
-        return p
+        timer = Timer(15, kill_proc)
+        try:
+            timer.start()
+            stdout, stderr = p.communicate()
+        finally:
+            timer.cancel()
+
+        return to_native_str(stdout), to_native_str(stderr)
 
 
 class StartprojectTest(ProjectTest):
@@ -111,8 +112,7 @@ class StartprojectTemplatesTest(ProjectTest):
         assert exists(join(self.tmpl_proj, 'root_template'))
 
         args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl]
-        p = self.proc('startproject', self.project_name, *args)
-        out = to_native_str(retry_on_eintr(p.stdout.read))
+        out, err = self.proc('startproject', self.project_name, *args)
         self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out)
         self.assertIn(self.tmpl_proj, out)
         assert exists(join(self.proj_path, 'root_template'))
@@ -140,12 +140,10 @@ class GenspiderCommandTest(CommandTest):
     def test_template(self, tplname='crawl'):
         args = ['--template=%s' % tplname] if tplname else []
         spname = 'test_spider'
-        p = self.proc('genspider', spname, 'test.com', *args)
-        out = to_native_str(retry_on_eintr(p.stdout.read))
+        out, err = self.proc('genspider', spname, 'test.com', *args)
         self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out)
         self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
-        p = self.proc('genspider', spname, 'test.com', *args)
-        out = to_native_str(retry_on_eintr(p.stdout.read))
+        out, err = self.proc('genspider', spname, 'test.com', *args)
         self.assertIn("Spider %r already exists in module" % spname, out)
 
     def test_template_basic(self):
@@ -212,8 +210,8 @@ class MySpider(scrapy.Spider):
             return self.proc('runspider', fname, *args)
 
     def get_log(self, code, name='myspider.py', args=()):
-        p = self.runspider(code, name=name, args=args)
-        return to_native_str(p.stderr.read())
+        stdout, stderr = self.runspider(code, name=name, args=args)
+        return stderr
 
     def test_runspider(self):
         log = self.get_log(self.debug_log_spider)
@@ -279,14 +277,17 @@ class MySpider(scrapy.Spider):
         self.assertIn("No spider found in file", log)
 
     def test_runspider_file_not_found(self):
-        p = self.proc('runspider', 'some_non_existent_file')
-        log = to_native_str(p.stderr.read())
+        _, log = self.proc('runspider', 'some_non_existent_file')
         self.assertIn("File not found: some_non_existent_file", log)
 
     def test_runspider_unable_to_load(self):
         log = self.get_log('', name='myspider.txt')
         self.assertIn('Unable to load', log)
 
+
+class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
+    command = 'parse'
+
     def test_start_requests_errors(self):
         log = self.get_log("""
 import scrapy
@@ -304,8 +305,7 @@ class BadSpider(scrapy.Spider):
 class BenchCommandTest(CommandTest):
 
     def test_run(self):
-        p = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
-                '-s', 'CLOSESPIDER_TIMEOUT=0.01')
-        log = to_native_str(p.stderr.read())
+        _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
+                           '-s', 'CLOSESPIDER_TIMEOUT=0.01')
         self.assertIn('INFO: Crawled', log)
         self.assertNotIn('Unhandled Error', log)

From ed8255bde04d7e07747792022858f161c2296096 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 6 Jul 2018 17:53:56 -0300
Subject: [PATCH 1766/2974] Fix merge issues with stderr/out fixes for windows
 buffering

---
 tests/test_commands.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 84c38c0e9..4963ef99c 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -56,7 +56,7 @@ class ProjectTest(unittest.TestCase):
         finally:
             timer.cancel()
 
-        return to_native_str(stdout), to_native_str(stderr)
+        return p, to_native_str(stdout), to_native_str(stderr)
 
 
 class StartprojectTest(ProjectTest):
@@ -112,7 +112,7 @@ class StartprojectTemplatesTest(ProjectTest):
         assert exists(join(self.tmpl_proj, 'root_template'))
 
         args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl]
-        out, err = self.proc('startproject', self.project_name, *args)
+        p, out, err = self.proc('startproject', self.project_name, *args)
         self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out)
         self.assertIn(self.tmpl_proj, out)
         assert exists(join(self.proj_path, 'root_template'))
@@ -140,10 +140,10 @@ class GenspiderCommandTest(CommandTest):
     def test_template(self, tplname='crawl'):
         args = ['--template=%s' % tplname] if tplname else []
         spname = 'test_spider'
-        out, err = self.proc('genspider', spname, 'test.com', *args)
+        p, out, err = self.proc('genspider', spname, 'test.com', *args)
         self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out)
         self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
-        out, err = self.proc('genspider', spname, 'test.com', *args)
+        p, out, err = self.proc('genspider', spname, 'test.com', *args)
         self.assertIn("Spider %r already exists in module" % spname, out)
 
     def test_template_basic(self):
@@ -210,7 +210,7 @@ class MySpider(scrapy.Spider):
             return self.proc('runspider', fname, *args)
 
     def get_log(self, code, name='myspider.py', args=()):
-        stdout, stderr = self.runspider(code, name=name, args=args)
+        p, stdout, stderr = self.runspider(code, name=name, args=args)
         return stderr
 
     def test_runspider(self):
@@ -221,12 +221,12 @@ class MySpider(scrapy.Spider):
         self.assertIn("INFO: Spider closed (finished)", log)
 
     def test_run_fail_spider(self):
-        proc = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider))
+        proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider))
         ret = proc.returncode
         self.assertNotEqual(ret, 0)
 
     def test_run_good_spider(self):
-        proc = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider))
+        proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider))
         ret = proc.returncode
         self.assertEqual(ret, 0)
 
@@ -277,7 +277,7 @@ class MySpider(scrapy.Spider):
         self.assertIn("No spider found in file", log)
 
     def test_runspider_file_not_found(self):
-        _, log = self.proc('runspider', 'some_non_existent_file')
+        _, _, log = self.proc('runspider', 'some_non_existent_file')
         self.assertIn("File not found: some_non_existent_file", log)
 
     def test_runspider_unable_to_load(self):
@@ -305,7 +305,7 @@ class BadSpider(scrapy.Spider):
 class BenchCommandTest(CommandTest):
 
     def test_run(self):
-        _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
+        _, _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
                            '-s', 'CLOSESPIDER_TIMEOUT=0.01')
         self.assertIn('INFO: Crawled', log)
         self.assertNotIn('Unhandled Error', log)

From 034152961d187930eb659bef9e87ceb3e589c103 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Wed, 3 Feb 2016 18:12:08 +0000
Subject: [PATCH 1767/2974] Fix Feedexport test in Windows by using proper file
 URI

---
 tests/test_feedexport.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 2b57449d9..6eefa14bf 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -2,11 +2,12 @@ from __future__ import absolute_import
 import os
 import csv
 import json
+import warnings
 from io import BytesIO
 import tempfile
 import shutil
-from six.moves.urllib.parse import urlparse
-import warnings
+from six.moves.urllib.parse import urljoin, urlparse
+from six.moves.urllib.request import pathname2url
 
 from zope.interface.verify import verifyObject
 from twisted.trial import unittest
@@ -226,9 +227,10 @@ class FeedExportTest(unittest.TestCase):
     def run_and_export(self, spider_cls, settings=None):
         """ Run spider with specified settings; return exported data. """
         tmpdir = tempfile.mkdtemp()
-        res_name = tmpdir + '/res'
+        res_path = os.path.join(tmpdir, 'res')
+        res_uri = urljoin('file:', pathname2url(res_path))
         defaults = {
-            'FEED_URI': 'file://' + res_name,
+            'FEED_URI': res_uri,
             'FEED_FORMAT': 'csv',
         }
         defaults.update(settings or {})
@@ -238,7 +240,7 @@ class FeedExportTest(unittest.TestCase):
                 spider_cls.start_urls = [s.url('/')]
                 yield runner.crawl(spider_cls)
 
-            with open(res_name, 'rb') as f:
+            with open(res_path, 'rb') as f:
                 defer.returnValue(f.read())
 
         finally:

From 22505a34a9d0095dfc6e133bbb79d3b7aa651082 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Wed, 3 Feb 2016 18:42:51 +0000
Subject: [PATCH 1768/2974] Fix cmdline profiling test on Windows by using
 proper path composing

---
 tests/test_cmdline/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 10076bbca..68dfb1cca 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -52,7 +52,8 @@ class CmdlineTest(unittest.TestCase):
             stats.print_stats()
             out.seek(0)
             stats = out.read()
-            self.assertIn('scrapy/commands/version.py', stats)
+            self.assertIn(os.path.join('scrapy', 'commands', 'version.py'),
+                          stats)
             self.assertIn('tottime', stats)
         finally:
             shutil.rmtree(path)

From fb09148c91f118e3a71243e527a0d39294ba59cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 6 Jul 2018 18:10:56 -0300
Subject: [PATCH 1769/2974] Fix bad merge on ParseCommandTest

---
 tests/test_commands.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 4963ef99c..78aa2a776 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -284,10 +284,6 @@ class MySpider(scrapy.Spider):
         log = self.get_log('', name='myspider.txt')
         self.assertIn('Unable to load', log)
 
-
-class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
-    command = 'parse'
-
     def test_start_requests_errors(self):
         log = self.get_log("""
 import scrapy

From a21abac743865b628e52f11882db7b7ab70e0342 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Sat, 7 Jul 2018 10:15:20 -0300
Subject: [PATCH 1770/2974] fix ftp tests on windows

---
 tests/test_downloader_handlers.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index fe76989f4..2f8973054 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -915,6 +915,7 @@ class BaseFTPTestCase(unittest.TestCase):
 
     def test_ftp_local_filename(self):
         f, local_fname = tempfile.mkstemp()
+        local_fname = to_bytes(local_fname)
         os.close(f)
         meta = {"ftp_local_filename": local_fname}
         meta.update(self.req_meta)

From ed068e59b746de6395c33f6f0f5454e69e88d1ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Sat, 7 Jul 2018 10:28:07 -0300
Subject: [PATCH 1771/2974] Cache pip cache and do not rebuild tags on appveyor
 and travis

---
 appveyor.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index 4f3c69847..93cfd469e 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -13,7 +13,12 @@ branches:
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
   - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE"
-  - "pip install -U tox twine wheel"
+  - "pip install -U tox"
+
 build: false
+skip_tags: true
 test_script:
   - "tox -e %TOX_ENV%"
+
+cache:
+  - '%LOCALAPPDATA%\pip\cache'

From 0e532e3dd8b4dd07b1a287d6321c1e6ae6786c64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Sat, 7 Jul 2018 10:46:15 -0300
Subject: [PATCH 1772/2974] Creating a connection to 0.0.0.0 fails on windows
 but not on linux nor mac

---
 tests/mockserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index f36ce3c44..bf62fe907 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -209,7 +209,7 @@ class MockServer():
         time.sleep(0.2)
 
     def url(self, path, is_secure=False):
-        host = self.http_address
+        host = self.http_address.replace('0.0.0.0', '127.0.0.1')
         if is_secure:
             host = self.https_address
         return host + path

From ca53a8699a8cfc2a0a3be4a65d91bada8a43f94c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Sat, 7 Jul 2018 11:07:05 -0300
Subject: [PATCH 1773/2974] Fix presentation of template directory in
 startproject command

---
 scrapy/commands/startproject.py | 4 ++--
 tests/test_commands.py          | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index c17aaf442..67337c26e 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -107,8 +107,8 @@ class Command(ScrapyCommand):
                 string.Template(path).substitute(project_name=project_name))
             render_templatefile(tplfile, project_name=project_name,
                 ProjectName=string_camelcase(project_name))
-        print("New Scrapy project %r, using template directory %r, created in:" % \
-              (project_name, self.templates_dir))
+        print("New Scrapy project '%s', using template directory '%s', "
+              "created in:" % (project_name, self.templates_dir))
         print("    %s\n" % abspath(project_dir))
         print("You can start your first spider with:")
         print("    cd %s" % project_dir)
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 78aa2a776..b8445ae6c 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -113,7 +113,8 @@ class StartprojectTemplatesTest(ProjectTest):
 
         args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl]
         p, out, err = self.proc('startproject', self.project_name, *args)
-        self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out)
+        self.assertIn("New Scrapy project '%s', using template directory"
+                      % self.project_name, out)
         self.assertIn(self.tmpl_proj, out)
         assert exists(join(self.proj_path, 'root_template'))
 

From cb281757500b2e114fb554aa3bf618e3a8f8eb71 Mon Sep 17 00:00:00 2001
From: Jakob de Maeyer <jakobdm1@gmail.com>
Date: Wed, 3 Feb 2016 18:35:35 +0000
Subject: [PATCH 1774/2974] Fix csviter tests by explicitly using newline only

---
 tests/test_utils_iterators.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index b2e8610f8..f953076b8 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -7,7 +7,7 @@ from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
 from scrapy.http import XmlResponse, TextResponse, Response
 from tests import get_testdata
 
-FOOBAR_NL = u"foo" + os.linesep + u"bar"
+FOOBAR_NL = u"foo\nbar"
 
 
 class XmliterTestCase(unittest.TestCase):

From d93d960319e22badccd68499df11f2a728dbbc04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 15 Aug 2018 01:53:20 -0300
Subject: [PATCH 1775/2974] Fix test_utils_project under Windows

---
 tests/test_utils_project.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index 7e2caace8..bd74b0c34 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -25,8 +25,12 @@ def inside_a_project():
 
 class ProjectUtilsTest(unittest.TestCase):
     def test_data_path_outside_project(self):
-        self.assertEqual('.scrapy/somepath', data_path('somepath'))
-        self.assertEqual('/absolute/path', data_path('/absolute/path'))
+        self.assertEqual(
+            os.path.join('.scrapy', 'somepath'),
+            data_path('somepath')
+        )
+        abspath = os.path.join(os.path.sep, 'absolute', 'path')
+        self.assertEqual(abspath, data_path(abspath))
 
     def test_data_path_inside_project(self):
         with inside_a_project() as proj_path:
@@ -35,4 +39,5 @@ class ProjectUtilsTest(unittest.TestCase):
                 os.path.realpath(expected),
                 os.path.realpath(data_path('somepath'))
             )
-            self.assertEqual('/absolute/path', data_path('/absolute/path'))
+            abspath = os.path.join(os.path.sep, 'absolute', 'path')
+            self.assertEqual(abspath, data_path(abspath))

From 96517cb7de93da990f4f7dcb1cb7b7129e8c2064 Mon Sep 17 00:00:00 2001
From: Daniel Grana <dangra@gmail.com>
Date: Wed, 15 Aug 2018 01:08:40 -0700
Subject: [PATCH 1776/2974] Fix test_command_parse under windows

---
 tests/test_command_parse.py | 38 +++++++++++++++++++++----------------
 1 file changed, 22 insertions(+), 16 deletions(-)

diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 66dd17110..02037b866 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,3 +1,4 @@
+import os
 from os.path import join, abspath
 from twisted.trial import unittest
 from twisted.internet import defer
@@ -7,6 +8,11 @@ from scrapy.utils.python import to_native_str
 from tests.test_commands import CommandTest
 
 
+def _textmode(bstr):
+    """Normalize input the same as writing to a file
+    and reading from it in text mode"""
+    return to_native_str(bstr).replace(os.linesep, '\n')
+
 class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
     command = 'parse'
 
@@ -97,7 +103,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
                                            '-a', 'test_arg=1',
                                            '-c', 'parse',
                                            self.url('/html')])
-        self.assertIn("DEBUG: It Works!", to_native_str(stderr))
+        self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_request_with_meta(self):
@@ -106,13 +112,13 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
                                            '--meta', raw_json_string,
                                            '-c', 'parse_request_with_meta',
                                            self.url('/html')])
-        self.assertIn("DEBUG: It Works!", to_native_str(stderr))
+        self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                            '-m', raw_json_string,
                                            '-c', 'parse_request_with_meta',
                                            self.url('/html')])
-        self.assertIn("DEBUG: It Works!", to_native_str(stderr))
+        self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
 
     @defer.inlineCallbacks
@@ -120,7 +126,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                           '-c', 'parse_request_without_meta',
                                            self.url('/html')])
-        self.assertIn("DEBUG: It Works!", to_native_str(stderr))
+        self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
 
     @defer.inlineCallbacks
@@ -129,29 +135,29 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
                                            '--pipelines',
                                            '-c', 'parse',
                                            self.url('/html')])
-        self.assertIn("INFO: It Works!", to_native_str(stderr))
+        self.assertIn("INFO: It Works!", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_parse_items(self):
         status, out, stderr = yield self.execute(
             ['--spider', self.spider_name, '-c', 'parse', self.url('/html')]
         )
-        self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
+        self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
 
     @defer.inlineCallbacks
     def test_parse_items_no_callback_passed(self):
         status, out, stderr = yield self.execute(
             ['--spider', self.spider_name, self.url('/html')]
         )
-        self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
+        self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
 
     @defer.inlineCallbacks
     def test_wrong_callback_passed(self):
         status, out, stderr = yield self.execute(
             ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')]
         )
-        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
-        self.assertIn("""Cannot find callback""", to_native_str(stderr))
+        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
+        self.assertIn("""Cannot find callback""", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_crawlspider_matching_rule_callback_set(self):
@@ -159,7 +165,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         status, out, stderr = yield self.execute(
             ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')]
         )
-        self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
+        self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
 
     @defer.inlineCallbacks
     def test_crawlspider_matching_rule_default_callback(self):
@@ -167,7 +173,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         status, out, stderr = yield self.execute(
             ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')]
         )
-        self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out))
+        self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out))
 
     @defer.inlineCallbacks
     def test_spider_with_no_rules_attribute(self):
@@ -175,15 +181,15 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         status, out, stderr = yield self.execute(
             ['--spider', self.spider_name, '-r', self.url('/html')]
         )
-        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
-        self.assertIn("""No CrawlSpider rules found""", to_native_str(stderr))
+        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
+        self.assertIn("""No CrawlSpider rules found""", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_crawlspider_missing_callback(self):
         status, out, stderr = yield self.execute(
             ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')]
         )
-        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
+        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
 
     @defer.inlineCallbacks
     def test_crawlspider_no_matching_rule(self):
@@ -191,5 +197,5 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         status, out, stderr = yield self.execute(
             ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')]
         )
-        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
-        self.assertIn("""Cannot find a rule that matches""", to_native_str(stderr))
+        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
+        self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))

From e7fe243c3e6f71660620441ec6a8ca4605d4333b Mon Sep 17 00:00:00 2001
From: Daniel Grana <dangra@gmail.com>
Date: Wed, 15 Aug 2018 01:09:23 -0700
Subject: [PATCH 1777/2974] Fix test_crawler under windows

---
 tests/test_crawler.py | 46 ++++++++++++++++++++++---------------------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 6a8e11363..0aeb12e58 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,10 +1,9 @@
 import logging
 import tempfile
 import warnings
-import unittest
 
 from twisted.internet import defer
-import twisted.trial.unittest
+from twisted.trial import unittest
 
 import scrapy
 from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
@@ -94,26 +93,29 @@ class CrawlerLoggingTestCase(unittest.TestCase):
         assert get_scrapy_root_handler() is None
 
     def test_spider_custom_settings_log_level(self):
-        with tempfile.NamedTemporaryFile() as log_file:
-            class MySpider(scrapy.Spider):
-                name = 'spider'
-                custom_settings = {
-                    'LOG_LEVEL': 'INFO',
-                    'LOG_FILE': log_file.name,
-                    # disable telnet if not available to avoid an extra warning
-                    'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
-                }
+        log_file = self.mktemp()
+        class MySpider(scrapy.Spider):
+            name = 'spider'
+            custom_settings = {
+                'LOG_LEVEL': 'INFO',
+                'LOG_FILE': log_file,
+                # disable telnet if not available to avoid an extra warning
+                'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
+            }
+
+        configure_logging()
+        self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
+        crawler = Crawler(MySpider, {})
+        self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
+        info_count = crawler.stats.get_value('log_count/INFO')
+        logging.debug('debug message')
+        logging.info('info message')
+        logging.warning('warning message')
+        logging.error('error message')
+
+        with open(log_file, 'rb') as fo:
+            logged = fo.read().decode('utf8')
 
-            configure_logging()
-            self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
-            crawler = Crawler(MySpider, {})
-            self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
-            info_count = crawler.stats.get_value('log_count/INFO')
-            logging.debug('debug message')
-            logging.info('info message')
-            logging.warning('warning message')
-            logging.error('error message')
-            logged = log_file.read().decode('utf8')
         self.assertNotIn('debug message', logged)
         self.assertIn('info message', logged)
         self.assertIn('warning message', logged)
@@ -203,7 +205,7 @@ class NoRequestsSpider(scrapy.Spider):
         return []
 
 
-class CrawlerRunnerHasSpider(twisted.trial.unittest.TestCase):
+class CrawlerRunnerHasSpider(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_crawler_runner_bootstrap_successful(self):

From a304d6b692664f06c67fb23eb08fd985eda72e21 Mon Sep 17 00:00:00 2001
From: Daniel Grana <dangra@gmail.com>
Date: Wed, 15 Aug 2018 02:02:20 -0700
Subject: [PATCH 1778/2974] Workaround to pass tests/test_feedexporter.py under
 windows

---
 tests/test_feedexport.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 6eefa14bf..76452d450 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -241,10 +241,18 @@ class FeedExportTest(unittest.TestCase):
                 yield runner.crawl(spider_cls)
 
             with open(res_path, 'rb') as f:
-                defer.returnValue(f.read())
+                content = f.read()
 
         finally:
-            shutil.rmtree(tmpdir)
+            # FIXME: Windows fails to remove the file because FeedExporter 
+            # keeps a reference to the temporal file even after
+            # the spider finished.
+            try:
+                shutil.rmtree(tmpdir)
+            except OSError:
+                pass
+
+        defer.returnValue(content)
 
     @defer.inlineCallbacks
     def exported_data(self, items, settings):

From 4eaf8690b14527d8d74ddd8103810de7c4dcdee6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 15 Aug 2018 08:54:18 -0300
Subject: [PATCH 1779/2974] Twisted's unittest.Testcase assertRaiess can't be
 used as context manager

---
 tests/test_crawler.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 0aeb12e58..268948a70 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -143,9 +143,8 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
         settings = Settings({
             'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface'
         })
-        with warnings.catch_warnings(record=True) as w, \
-                self.assertRaises(AttributeError):
-            CrawlerRunner(settings)
+        with warnings.catch_warnings(record=True) as w:
+            self.assertRaises(AttributeError, CrawlerRunner, settings)
             self.assertEqual(len(w), 1)
             self.assertIn("SPIDER_LOADER_CLASS", str(w[0].message))
             self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message))

From 38608bc2495189bd7de2dc48c0036df217138377 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 15 Aug 2018 11:59:09 -0300
Subject: [PATCH 1780/2974] Use ignore_errors option from rmtree

---
 tests/test_feedexport.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 76452d450..e46c8c14e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -244,13 +244,7 @@ class FeedExportTest(unittest.TestCase):
                 content = f.read()
 
         finally:
-            # FIXME: Windows fails to remove the file because FeedExporter 
-            # keeps a reference to the temporal file even after
-            # the spider finished.
-            try:
-                shutil.rmtree(tmpdir)
-            except OSError:
-                pass
+            shutil.rmtree(tmpdir, ignore_errors=True)
 
         defer.returnValue(content)
 

From 4de493efdd80d8ff85a78009dcc120ac46b9c55c Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Wed, 15 Aug 2018 20:24:00 +0300
Subject: [PATCH 1781/2974] Add test_same_url

---
 tests/test_contracts.py | 40 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index b07cbee1e..e4f1a777b 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -1,8 +1,10 @@
 from unittest import TextTestResult
 
+from twisted.internet import defer
 from twisted.python import failure
 from twisted.trial import unittest
 
+from scrapy.crawler import CrawlerRunner
 from scrapy.spidermiddlewares.httperror import HttpError
 from scrapy.spiders import Spider
 from scrapy.http import Request
@@ -101,6 +103,29 @@ class TestSpider(Spider):
         pass
 
 
+class TestSameUrlSpider(Spider):
+
+    name = 'test_same_url'
+
+    def __init__(self, *args, **kwargs):
+        super(TestSameUrlSpider, self).__init__(*args, **kwargs)
+        self.visited = 0
+
+    def parse_first(self, response):
+        """first callback
+        @url http://scrapy.org
+        """
+        self.visited += 1
+        return TestItem()
+
+    def parse_second(self, response):
+        """second callback
+        @url http://scrapy.org
+        """
+        self.visited += 1
+        return TestItem()
+
+
 class ContractsManagerTest(unittest.TestCase):
     contracts = [UrlContract, ReturnsContract, ScrapesContract]
 
@@ -177,14 +202,12 @@ class ContractsManagerTest(unittest.TestCase):
         self.should_succeed()
 
         # scrapes_item_fail
-        request = self.conman.from_method(spider.scrapes_item_fail,
-                self.results)
+        request = self.conman.from_method(spider.scrapes_item_fail, self.results)
         request.callback(response)
         self.should_fail()
 
         # scrapes_dict_item_fail
-        request = self.conman.from_method(spider.scrapes_dict_item_fail,
-                self.results)
+        request = self.conman.from_method(spider.scrapes_dict_item_fail, self.results)
         request.callback(response)
         self.should_fail()
 
@@ -202,3 +225,12 @@ class ContractsManagerTest(unittest.TestCase):
 
         self.assertFalse(self.results.failures)
         self.assertTrue(self.results.errors)
+
+    @defer.inlineCallbacks
+    def test_same_url(self):
+        TestSameUrlSpider.start_requests = lambda s: self.conman.from_spider(s, self.results)
+
+        crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
+        yield crawler.crawl()
+
+        self.assertEqual(crawler.spider.visited, 2)

From 2cb4decb6ad52548b3441c877235c540cef12082 Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Wed, 15 Aug 2018 20:36:10 +0300
Subject: [PATCH 1782/2974] Move TestSameUrlSpider to test method

---
 tests/test_contracts.py | 45 ++++++++++++++++++++---------------------
 1 file changed, 22 insertions(+), 23 deletions(-)

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index e4f1a777b..430d89253 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -103,29 +103,6 @@ class TestSpider(Spider):
         pass
 
 
-class TestSameUrlSpider(Spider):
-
-    name = 'test_same_url'
-
-    def __init__(self, *args, **kwargs):
-        super(TestSameUrlSpider, self).__init__(*args, **kwargs)
-        self.visited = 0
-
-    def parse_first(self, response):
-        """first callback
-        @url http://scrapy.org
-        """
-        self.visited += 1
-        return TestItem()
-
-    def parse_second(self, response):
-        """second callback
-        @url http://scrapy.org
-        """
-        self.visited += 1
-        return TestItem()
-
-
 class ContractsManagerTest(unittest.TestCase):
     contracts = [UrlContract, ReturnsContract, ScrapesContract]
 
@@ -228,6 +205,28 @@ class ContractsManagerTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_same_url(self):
+
+        class TestSameUrlSpider(Spider):
+            name = 'test_same_url'
+
+            def __init__(self, *args, **kwargs):
+                super(TestSameUrlSpider, self).__init__(*args, **kwargs)
+                self.visited = 0
+
+            def parse_first(self, response):
+                """first callback
+                @url http://scrapy.org
+                """
+                self.visited += 1
+                return TestItem()
+
+            def parse_second(self, response):
+                """second callback
+                @url http://scrapy.org
+                """
+                self.visited += 1
+                return TestItem()
+
         TestSameUrlSpider.start_requests = lambda s: self.conman.from_spider(s, self.results)
 
         crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)

From ddd69f4c10578975658ef6ac450cbe1bed85df80 Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Wed, 15 Aug 2018 20:39:43 +0300
Subject: [PATCH 1783/2974] Use MockServer in test_same_url

---
 tests/test_contracts.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 430d89253..223a926f1 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -15,6 +15,7 @@ from scrapy.contracts.default import (
     ReturnsContract,
     ScrapesContract,
 )
+from tests.mockserver import MockServer
 
 
 class TestItem(Item):
@@ -213,6 +214,9 @@ class ContractsManagerTest(unittest.TestCase):
                 super(TestSameUrlSpider, self).__init__(*args, **kwargs)
                 self.visited = 0
 
+            def start_requests(s):
+                return self.conman.from_spider(s, self.results)
+
             def parse_first(self, response):
                 """first callback
                 @url http://scrapy.org
@@ -227,9 +231,8 @@ class ContractsManagerTest(unittest.TestCase):
                 self.visited += 1
                 return TestItem()
 
-        TestSameUrlSpider.start_requests = lambda s: self.conman.from_spider(s, self.results)
-
         crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
-        yield crawler.crawl()
+        with MockServer() as mockserver:
+            yield crawler.crawl(mockserver=mockserver)
 
         self.assertEqual(crawler.spider.visited, 2)

From 2b212d426668c02704bc23c4e988fa108e0738ef Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 17 Aug 2018 14:39:06 +0000
Subject: [PATCH 1784/2974] ignore cache for pytests

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 406146e5f..ff6e2ea65 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@ dist
 .idea
 htmlcov/
 .coverage
+.pytest_cache/
 .coverage.*
 .cache/
 

From d95762db7c9abe59d8edfc8c17397a5a5bbfc661 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 17 Aug 2018 14:39:24 +0000
Subject: [PATCH 1785/2974] new signal

---
 scrapy/signals.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/signals.py b/scrapy/signals.py
index e36c27203..c0e4bb74e 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -13,6 +13,7 @@ spider_closed = object()
 spider_error = object()
 request_scheduled = object()
 request_dropped = object()
+request_reached_downloader = object()
 response_received = object()
 response_downloaded = object()
 item_scraped = object()

From 597b8a97ad468123432dba3d6f2c3dd943f4aa36 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 17 Aug 2018 14:39:42 +0000
Subject: [PATCH 1786/2974] documentation for new signal

---
 docs/topics/signals.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index d40c0e1df..cf7b8db2f 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -279,6 +279,22 @@ request_dropped
     :param spider: the spider that yielded the request
     :type spider: :class:`~scrapy.spiders.Spider` object
 
+request_reached_downloader
+---------------------------
+
+.. signal:: request_reached_downloader
+.. function:: request_reached_downloader(request, spider)
+
+    Sent when a :class:`~scrapy.http.Request`, reached downloader.
+
+    The signal does not support returning deferreds from their handlers.
+
+    :param request: the request that reached downloader
+    :type request: :class:`~scrapy.http.Request` object
+
+    :param spider: the spider that yielded the request
+    :type spider: :class:`~scrapy.spiders.Spider` object
+
 response_received
 -----------------
 

From afb1458bd3b75fad78f053ac062910052971cdfc Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 17 Aug 2018 14:39:54 +0000
Subject: [PATCH 1787/2974] tests for new signal

---
 tests/test_engine.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index 719c0c60c..856465161 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -103,6 +103,7 @@ class CrawlerRun(object):
         self.respplug = []
         self.reqplug = []
         self.reqdropped = []
+        self.reqreached = []
         self.itemerror = []
         self.itemresp = []
         self.signals_catched = {}
@@ -124,6 +125,7 @@ class CrawlerRun(object):
         self.crawler.signals.connect(self.item_error, signals.item_error)
         self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
         self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
+        self.crawler.signals.connect(self.request_reached, signals.request_reached_downloader)
         self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded)
         self.crawler.crawl(start_urls=start_urls)
         self.spider = self.crawler.spider
@@ -155,6 +157,9 @@ class CrawlerRun(object):
     def request_scheduled(self, request, spider):
         self.reqplug.append((request, spider))
 
+    def request_reached(self, request, spider):
+        self.reqreached.append((request, spider))
+
     def request_dropped(self, request, spider):
         self.reqdropped.append((request, spider))
 
@@ -212,6 +217,8 @@ class EngineTest(unittest.TestCase):
         responses_count = len(self.run.respplug)
         self.assertEqual(scheduled_requests_count,
                          dropped_requests_count + responses_count)
+        self.assertEqual(len(self.run.reqreached),
+                         responses_count)
 
     def _assert_dropped_requests(self):
         self.assertEqual(len(self.run.reqdropped), 1)
@@ -219,6 +226,7 @@ class EngineTest(unittest.TestCase):
     def _assert_downloaded_responses(self):
         # response tests
         self.assertEqual(8, len(self.run.respplug))
+        self.assertEqual(8, len(self.run.reqreached))
 
         for response, _ in self.run.respplug:
             if self.run.getpath(response.url) == '/item999.html':

From 561ad3b63c539b560bbaed1df37d0b60043ab3b3 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 17 Aug 2018 14:40:24 +0000
Subject: [PATCH 1788/2974] emit new signal

---
 scrapy/core/downloader/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index d835e65f7..59c3ad074 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -129,6 +129,9 @@ class Downloader(object):
             return response
 
         slot.active.add(request)
+        self.signals.send_catch_log(signal=signals.request_reached_downloader,
+                                    request=request,
+                                    spider=spider)
         deferred = defer.Deferred().addBoth(_deactivate)
         slot.queue.append((request, deferred))
         self._process_queue(spider, slot)

From 5bac43676425d25169830d4410db60a98a11911f Mon Sep 17 00:00:00 2001
From: Henrique Coura <coura.henrique@gmail.com>
Date: Fri, 17 Aug 2018 15:07:37 -0300
Subject: [PATCH 1789/2974] Make lazy loading Download Handlers optional

---
 scrapy/core/downloader/handlers/__init__.py | 14 ++++++++++--
 scrapy/core/downloader/handlers/s3.py       |  1 +
 tests/test_downloader_handlers.py           | 25 ++++++++++++++++-----
 3 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index bc5cd742e..ebe6f5b78 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -24,6 +24,13 @@ class DownloadHandlers(object):
             crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
         for scheme, clspath in six.iteritems(handlers):
             self._schemes[scheme] = clspath
+        for scheme in self._schemes:
+            path = self._schemes[scheme]
+            dhcls = load_object(path)
+            lazy = getattr(dhcls, 'lazy', False)
+            if lazy:
+                continue
+            self._load_handler(scheme, dhcls)
 
         crawler.signals.connect(self._close, signals.engine_stopped)
 
@@ -40,8 +47,12 @@ class DownloadHandlers(object):
             return None
 
         path = self._schemes[scheme]
+        dhcls = load_object(path)
+        self._load_handler(scheme, dhcls)
+        return self._handlers[scheme]
+
+    def _load_handler(self, scheme, dhcls):
         try:
-            dhcls = load_object(path)
             dh = dhcls(self._crawler.settings)
         except NotConfigured as ex:
             self._notconfigured[scheme] = str(ex)
@@ -54,7 +65,6 @@ class DownloadHandlers(object):
             return None
         else:
             self._handlers[scheme] = dh
-        return self._handlers[scheme]
 
     def download_request(self, request, spider):
         scheme = urlparse_cached(request).scheme
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index d8bbdd326..e723e616d 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -31,6 +31,7 @@ def _get_boto_connection():
 
 
 class S3DownloadHandler(object):
+    lazy = True
 
     def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
             httpdownloadhandler=HTTPDownloadHandler, **kw):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 2f8973054..116942ebe 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -41,12 +41,20 @@ from scrapy.exceptions import NotConfigured
 from tests.mockserver import MockServer, ssl_context_factory, Echo
 from tests.spiders import SingleRequestSpider
 
+
 class DummyDH(object):
 
     def __init__(self, crawler):
         pass
 
 
+class DummyLazyDH(object):
+    lazy = True
+
+    def __init__(self, crawler):
+        pass
+
+
 class OffDH(object):
 
     def __init__(self, crawler):
@@ -60,8 +68,6 @@ class LoadTestCase(unittest.TestCase):
         crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
         dh = DownloadHandlers(crawler)
         self.assertIn('scheme', dh._schemes)
-        for scheme in handlers: # force load handlers
-            dh._get_handler(scheme)
         self.assertIn('scheme', dh._handlers)
         self.assertNotIn('scheme', dh._notconfigured)
 
@@ -70,8 +76,6 @@ class LoadTestCase(unittest.TestCase):
         crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
         dh = DownloadHandlers(crawler)
         self.assertIn('scheme', dh._schemes)
-        for scheme in handlers: # force load handlers
-            dh._get_handler(scheme)
         self.assertNotIn('scheme', dh._handlers)
         self.assertIn('scheme', dh._notconfigured)
 
@@ -80,11 +84,22 @@ class LoadTestCase(unittest.TestCase):
         crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
         dh = DownloadHandlers(crawler)
         self.assertNotIn('scheme', dh._schemes)
-        for scheme in handlers: # force load handlers
+        for scheme in handlers:  # force load handlers
             dh._get_handler(scheme)
         self.assertNotIn('scheme', dh._handlers)
         self.assertIn('scheme', dh._notconfigured)
 
+    def test_lazy_handlers(self):
+        handlers = {'scheme': 'tests.test_downloader_handlers.DummyLazyDH'}
+        crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
+        dh = DownloadHandlers(crawler)
+        self.assertIn('scheme', dh._schemes)
+        self.assertNotIn('scheme', dh._handlers)
+        for scheme in handlers:  # force load lazy handler
+            dh._get_handler(scheme)
+        self.assertIn('scheme', dh._handlers)
+        self.assertNotIn('scheme', dh._notconfigured)
+
 
 class FileTestCase(unittest.TestCase):
 

From e2de0a7203b99d8fc71e62539fdd75dad439982a Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Sat, 18 Aug 2018 15:24:30 +0300
Subject: [PATCH 1790/2974] Use except Exception

---
 scrapy/contracts/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index c62df5ab0..18c59ff22 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -44,7 +44,7 @@ class ContractsManager(object):
             bound_method = spider.__getattribute__(method)
             try:
                 requests.append(self.from_method(bound_method, results))
-            except:
+            except Exception:
                 case = _create_testcase(bound_method, 'contract')
                 results.addError(case, sys.exc_info())
 

From 0467737cf0bc4c5603bd63a87e220482eb724600 Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Sat, 18 Aug 2018 15:42:21 +0300
Subject: [PATCH 1791/2974] Fix mockserver usage

---
 tests/test_contracts.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index c5820f256..0a216b745 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -222,22 +222,20 @@ class ContractsManagerTest(unittest.TestCase):
                 return self.conman.from_spider(s, self.results)
 
             def parse_first(self, response):
-                """first callback
-                @url {}
-                """.format(self.mockserver.url('/status?n=200'))
                 self.visited += 1
                 return TestItem()
 
             def parse_second(self, response):
-                """second callback
-                @url {}
-                """.format(self.mockserver.url('/status?n=200'))
                 self.visited += 1
                 return TestItem()
 
-        crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
         with MockServer() as mockserver:
-            yield crawler.crawl(mockserver=mockserver)
+            mock_endpoint = mockserver.url('/status?n=200')
+            TestSameUrlSpider.parse_first.__func__.__doc__ = '@url {}'.format(mock_endpoint)
+            TestSameUrlSpider.parse_second.__func__.__doc__ = '@url {}'.format(mock_endpoint)
+
+            crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
+            yield crawler.crawl()
 
         self.assertEqual(crawler.spider.visited, 2)
 

From 57824600a8295f0d8537de2a8d1c3eb9977ed36d Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Sun, 19 Aug 2018 16:56:41 +0300
Subject: [PATCH 1792/2974] Use six.get_unbound_function in test_same_url

---
 tests/test_contracts.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 0a216b745..b4209e1f6 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -1,5 +1,6 @@
 from unittest import TextTestResult
 
+from six import get_unbound_function
 from twisted.internet import defer
 from twisted.python import failure
 from twisted.trial import unittest
@@ -230,9 +231,10 @@ class ContractsManagerTest(unittest.TestCase):
                 return TestItem()
 
         with MockServer() as mockserver:
-            mock_endpoint = mockserver.url('/status?n=200')
-            TestSameUrlSpider.parse_first.__func__.__doc__ = '@url {}'.format(mock_endpoint)
-            TestSameUrlSpider.parse_second.__func__.__doc__ = '@url {}'.format(mock_endpoint)
+            contract_doc = '@url {}'.format(mockserver.url('/status?n=200'))
+
+            get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc
+            get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc
 
             crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
             yield crawler.crawl()

From 167211ffb0e7fa7756483aa9f20e1ab5589c7c4a Mon Sep 17 00:00:00 2001
From: Henrique Coura <coura.henrique@gmail.com>
Date: Mon, 20 Aug 2018 15:54:04 -0300
Subject: [PATCH 1793/2974] Default is lazy, load_object exception handling,
 code improvements

---
 scrapy/core/downloader/handlers/__init__.py | 22 +++++++++------------
 scrapy/core/downloader/handlers/datauri.py  |  2 ++
 scrapy/core/downloader/handlers/file.py     |  2 ++
 scrapy/core/downloader/handlers/ftp.py      |  3 +++
 scrapy/core/downloader/handlers/http10.py   |  1 +
 scrapy/core/downloader/handlers/http11.py   |  1 +
 scrapy/core/downloader/handlers/s3.py       |  1 -
 tests/test_downloader_handlers.py           |  4 +++-
 8 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index ebe6f5b78..0b55d32fa 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -24,13 +24,7 @@ class DownloadHandlers(object):
             crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
         for scheme, clspath in six.iteritems(handlers):
             self._schemes[scheme] = clspath
-        for scheme in self._schemes:
-            path = self._schemes[scheme]
-            dhcls = load_object(path)
-            lazy = getattr(dhcls, 'lazy', False)
-            if lazy:
-                continue
-            self._load_handler(scheme, dhcls)
+            self._load_handler(scheme, skip_lazy=True)
 
         crawler.signals.connect(self._close, signals.engine_stopped)
 
@@ -46,13 +40,14 @@ class DownloadHandlers(object):
             self._notconfigured[scheme] = 'no handler available for that scheme'
             return None
 
-        path = self._schemes[scheme]
-        dhcls = load_object(path)
-        self._load_handler(scheme, dhcls)
-        return self._handlers[scheme]
+        return self._load_handler(scheme)
 
-    def _load_handler(self, scheme, dhcls):
+    def _load_handler(self, scheme, skip_lazy=False):
+        path = self._schemes[scheme]
         try:
+            dhcls = load_object(path)
+            if skip_lazy and getattr(dhcls, 'lazy', True):
+                return None
             dh = dhcls(self._crawler.settings)
         except NotConfigured as ex:
             self._notconfigured[scheme] = str(ex)
@@ -60,11 +55,12 @@ class DownloadHandlers(object):
         except Exception as ex:
             logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"',
                          {"clspath": path, "scheme": scheme},
-                         exc_info=True,  extra={'crawler': self._crawler})
+                         exc_info=True, extra={'crawler': self._crawler})
             self._notconfigured[scheme] = str(ex)
             return None
         else:
             self._handlers[scheme] = dh
+            return dh
 
     def download_request(self, request, spider):
         scheme = urlparse_cached(request).scheme
diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py
index d102f2b73..ad25beb3b 100644
--- a/scrapy/core/downloader/handlers/datauri.py
+++ b/scrapy/core/downloader/handlers/datauri.py
@@ -6,6 +6,8 @@ from scrapy.utils.decorators import defers
 
 
 class DataURIDownloadHandler(object):
+    lazy = False
+
     def __init__(self, settings):
         super(DataURIDownloadHandler, self).__init__()
 
diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py
index 9346ce08d..23f25d28d 100644
--- a/scrapy/core/downloader/handlers/file.py
+++ b/scrapy/core/downloader/handlers/file.py
@@ -2,7 +2,9 @@ from w3lib.url import file_uri_to_path
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.decorators import defers
 
+
 class FileDownloadHandler(object):
+    lazy = False
 
     def __init__(self, settings):
         pass
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 933bc7e8d..c342d4ab1 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -60,7 +60,10 @@ class ReceivedDataProtocol(Protocol):
         self.body.close() if self.filename else self.body.seek(0)
 
 _CODE_RE = re.compile("\d+")
+
+
 class FTPDownloadHandler(object):
+    lazy = False
 
     CODE_MAPPING = {
         "550": 404,
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index 0322bbe49..d875fb1e4 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -6,6 +6,7 @@ from scrapy.utils.python import to_unicode
 
 
 class HTTP10DownloadHandler(object):
+    lazy = False
 
     def __init__(self, settings):
         self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 038db7b47..0673188a1 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -33,6 +33,7 @@ logger = logging.getLogger(__name__)
 
 
 class HTTP11DownloadHandler(object):
+    lazy = False
 
     def __init__(self, settings):
         self._pool = HTTPConnectionPool(reactor, persistent=True)
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index e723e616d..d8bbdd326 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -31,7 +31,6 @@ def _get_boto_connection():
 
 
 class S3DownloadHandler(object):
-    lazy = True
 
     def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
             httpdownloadhandler=HTTPDownloadHandler, **kw):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 116942ebe..0d0829793 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -43,19 +43,21 @@ from tests.spiders import SingleRequestSpider
 
 
 class DummyDH(object):
+    lazy = False
 
     def __init__(self, crawler):
         pass
 
 
 class DummyLazyDH(object):
-    lazy = True
+    # Default is lazy for backwards compatibility
 
     def __init__(self, crawler):
         pass
 
 
 class OffDH(object):
+    lazy = False
 
     def __init__(self, crawler):
         raise NotConfigured

From af555cab23958501f07f8b2771c082ac3f3a5961 Mon Sep 17 00:00:00 2001
From: Raphael Wuillemier <wuillemr@essilor.de>
Date: Wed, 22 Aug 2018 14:15:53 +0200
Subject: [PATCH 1794/2974] Added general guide for developer tools instead of
 Firefox and Firebug-sections

---
 docs/index.rst                       |  10 +-
 docs/intro/tutorial.rst              |   3 +-
 docs/topics/_images/inspector_01.png | Bin 0 -> 53922 bytes
 docs/topics/_images/network_01.png   | Bin 0 -> 10720 bytes
 docs/topics/_images/network_02.png   | Bin 0 -> 82702 bytes
 docs/topics/_images/network_03.png   | Bin 0 -> 45506 bytes
 docs/topics/developer-tools.rst      | 248 +++++++++++++++++++++++++++
 docs/topics/firebug.rst              | 167 ------------------
 docs/topics/firefox.rst              |  82 ---------
 9 files changed, 252 insertions(+), 258 deletions(-)
 create mode 100644 docs/topics/_images/inspector_01.png
 create mode 100644 docs/topics/_images/network_01.png
 create mode 100644 docs/topics/_images/network_02.png
 create mode 100644 docs/topics/_images/network_03.png
 create mode 100644 docs/topics/developer-tools.rst
 delete mode 100644 docs/topics/firebug.rst
 delete mode 100644 docs/topics/firefox.rst

diff --git a/docs/index.rst b/docs/index.rst
index 7e8c979c4..0a96aa88e 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -151,8 +151,7 @@ Solving specific problems
    topics/contracts
    topics/practices
    topics/broad-crawls
-   topics/firefox
-   topics/firebug
+   topics/developer-tools
    topics/leaks
    topics/media-pipeline
    topics/deploy
@@ -175,11 +174,8 @@ Solving specific problems
 :doc:`topics/broad-crawls`
     Tune Scrapy for crawling a lot domains in parallel.
 
-:doc:`topics/firefox`
-    Learn how to scrape with Firefox and some useful add-ons.
-
-:doc:`topics/firebug`
-    Learn how to scrape efficiently using Firebug.
+:doc:`topics/developer-tools`
+    Learn how to scrape with your browser's developer tools.
 
 :doc:`topics/leaks`
     Learn how to find and get rid of memory leaks in your crawler.
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 0db6a6218..fa6dc274d 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -298,8 +298,7 @@ expressions`::
 
 In order to find the proper CSS selectors to use, you might find useful opening
 the response page from the shell in your web browser using ``view(response)``.
-You can use your browser developer tools or extensions like Firebug (see
-sections about :ref:`topics-firebug` and :ref:`topics-firefox`).
+You can use your browser developer tools (see section about :ref:`topics-developer-tools`).
 
 `Selector Gadget`_ is also a nice tool to quickly find CSS selector for
 visually selected elements, which works in many browsers.
diff --git a/docs/topics/_images/inspector_01.png b/docs/topics/_images/inspector_01.png
new file mode 100644
index 0000000000000000000000000000000000000000..edb8795dcb8890bac8311db1445435f969761aa0
GIT binary patch
literal 53922
zcmcG#1$0zfmo-{3;!23S6L)tZfe?cbcXxMpcZd)dNJ2az?jcUx-QC^oKNar1U-$pL
z?jHT#cr{>9>^gPMmL+q}wJKCzRuUNj4*>)MA-|Op{{R9(!Gk~$6L2qpk^1YJ6W||s
z8!1(L5D29M{1+mg4h0_=gmri;BLTY#hlPNEHF6oA`#dC};vnMaU}9|t`ccv^1q>lM
z07K%2_WE|FHV&rNR-l*gq~yRj0(e}^+Q!w+)Y!xURD{~k3yh*XkJ=gPf-fa;Fts!U
zaa^GS^8vwwKlx~F<lwApX9&`?c>-UK_<VCydwmOC;8HtKO>1Q|FboGCR@Sw%GPN=W
zwc>9E17om%zVB$KZwR`0^56UWGBImQOG7IM&}Dx@3o!Nqe3_!DgM}d||8SHV1R?>w
z6&F!-N!?$tRKZkE=X$g@m#@$<kVwbGFks%M{f(sB8H(OD;{GL_M42_9>pjNWMN6zW
zt8I^P>q$DceG2=pAP#(tSnMtewqemYN&~#qR}oZr!;z{Rq$d3&=xFt>wIZGQ6hEHA
z56uday`Ym}AG}HmnYMX9<;kWU76`WRZaL6Fi=YXMi#vaNy2*HI%XmDeZoFD_KR!K8
z-d;sRAHig?%ti!0FCm44Ty)(cY&`yjh=djr((@V#BP8Td-j5_RD+_{7sVHeX8+>^?
ziEqd@%Eel=P<Cl)bQ7Go_&WOwc;NmtvOMHu*|8t)*XB6A!qCuxS8V9&Wbxb@9>u+V
z$(+mKVadhQC@gh7@Evh+b_v^B@HOb@y%Fwmkk9XAtkz~f+id)fKba?d9*6`5<3GQN
zMe>Ix1Ha;QV}Qv5FV>H&QruI#A@;@hVRB^Iy(Oiks60IGXoQ5w*4EZ8?(XEgym)-h
zkLXKYcMPD+=4PUci;I^UAAnmV$BBsE%%so=x!Y%l+jgt`G>Y75pBs^Ma5xvoPBBTk
zy>%0plq4r3(^F9s7l+Qy$(g|+l$4N=xSkpE3bJl#inzb`I6ptn%*%tlxw%1uhDhPG
z6qOw$;=dUG_)=3buI%D#tGSR275LFkhxg>1mSJL2QXxB@^`;+F_uoKA_Lr3kYyLDs
z#>2-?`)#l@`kmYBzINfFM5E5n^=y+T{;K6sTVQvVV2?wEXgz4G!J|00d)*;xlengQ
zmzvFnzP<a0-F$CjQ`5-U*jk+BhYug*)zlC`Qc_aFA|h=ezqvmwE-rFA9}+-9L3LYZ
z08{T780g&SOFSsK>%zCzIolj?nmW6@Ob}qAr|$sH@^IK`&GY`{YceuOU_~lTZdIKQ
z#Y*8qk0;UG?w3Uijk0OHfz#6}V`s6c<b&f)H|81&3cm4l%4!4HpadpuK~wFe>}<08
zlW1>oS67~2?d=RDXDh+PWDE?*#NPMi6OQ7W7i@8BSlVNuwOBHg)g?!R9$LUD>2D3D
zx$JyT4EpYQ+y*NpE1O<<v{bGrFYojCSco-dG}Mgx`t@ts4dY@h>h=pS{yup{#emdQ
z0*iSZ#P6(1P76Y|tI(CR2qx^EE*tU61h=V#2OLkb4$(J$U0q%1o;daIH0MaH4l<60
z>)C3m=H3e>toVPOo3&k@@g@q-j{Mzkd0;Yo5R8_g%hVQtvo}>%T#Ve8$OIgB;7)rj
z@E^dFDyEO?KCiE@FE?E+{+!$XQCcdgp+NvF@}LB<2VVl%YZFt`^6M3ME^|xEDm)U9
z&xao!V{$^4Tp88XoE=?V$t8iKS0!{p&}qF~%*^sm#py`M&<fhx#7+mZhHS$5rKQ~&
zPxr>bkkGL%9>gBU5Kc#n2M%IFoSc(p#}XyBC#w>bRaMCD+Emoka`K^F89zmQ$QiX7
zxwD7HdL7bmih&*RI3JO8S@wku`d+Nngjc9s`sUql&W?_bX=Dtl-DIH|nckH*?3W$7
zpeVupnI`wEGSs!!*4BpA_PVO7iL5trE-u`sXJ_cx*ul-sf_ay=hx0s*x4VVp46B5D
zE5Y7TFR5i!WWc*k<aNE}bY^O8Eh#GMw{x>_x=2WOcXia=6^U=S)s`;cems`(h)GIH
z3Y=!wXF0(gd1lLAz{a9RM?V0o;AuLzzlmCKsGl33nj)v5fd290htrgdjEn@}B4T1!
zSd1DWAM<2;6ViAc5%>TbwpE39xtuXRxtz0>b6}wwAN?*}?(Lm5;D~^5Nlru4W2M!&
zsqUl<3lE&{T5--4i@kcoj<>>l*+kQu*she^i)i-6%33e`sjJ+LO#!@3*U+7C=bht*
zY{Brk@H@<_JK$Y|0Oe;1b#?rA($bdcI+m7}6}Ba#sOUf>@c_&+w-QzGy7fy!TpX(f
zzPjto?3O{MB5iD3oS#ezCr_28jt&V121e1OucEtqLudUt;8+|L--f>5gq8;WG##}A
z?0l_iO3U7~>FVNQWxaRE@{)pHvmTRL&dZ96M!n9C5_qoR1q{+fErWJ5L21(^9^ldj
zM>W7cs5Y8&CtszzdZr{wzE4c(8x<TFVo`1;QY%(cLUq5~Z|nU`Ew7{$#ME++F;$}3
z{X3pc5|cz_wdYP&O;S&Q_oJj<y@RIt)axC=&MC3}_E}CJ{V6VGu*rRt69{x(@|?<g
z@kw4DZPD{e6CA*tpaKD_(QSHR-SWVR@$#i%pG8wp)MP_WX<p7&aUl?TRnpFMO$FmA
z&$foTe+422HwB9)?2g^@p3It;A{UbHRC!cRVVqq(sklGhY@K90-H-y2>SWm;5uLcD
z_uS>UE%5mAvP=8%l&Iq8&x6g4Z4P3a62T%(#5boryy#SxF*YTRXbn$Bu1FaS4Np!J
zKdc3Lmu>_ACys7;7{qT88b<|jkGj0<gIusNfAW~06(ZhCtWBBpmT*U|xg3%lvl6<0
zlov{FXol4>Ip#XJWZR&8fyTn4>9U&TZs;1?o|o)&w<5&p{<ZpP?x<15_32i~?Y%2h
zE7_Af0g)329wHV!&gamy(U5rzTbd7Ya;Sr8e8E8UQ}wU<{vAd@Kp;OqpV_lD6);dl
zY)1AAqY#mhrp9k~{LY7|ueHswr%o<U`<TGCnZ@TaIn7&Lv*b0W$kp51i?ZMuKl$UQ
z>Ev&80)qWfmUKCx2TWF0R*RCgjg7#F2qgZqhzC+h;1S$*>mmDM!t_sd_FKoJ(o8Ee
z#u@P!8mZj2kVArZ`(=#;o=1SOcud3k7vCl%d{SdPJD^X)F7Xfml8PaOu&pi2q6;}O
zD*rL4v9YntdPzOMpa9SI5gc6rV|~By6xDe2&5>eo%eFeI!~REh_Ahp)nR{Ls5L456
zoM}dB^KHR`Ilc}Y+GMG=^~o(K(NwX}!;YNG?M~jv?5vsd!sMh9|HJv%HQ?>*@{53{
zet{7H1tBlktm7G#E)seQ5DHjpZtTzeRC3=T{K`b>288({tucm)yc<Woi`|J_H*Uw>
zFZ*3%KR)K65fE7X?wc`x`|ceAEiJ7EFU94}Rz`OMgSyTAwQWbYP!NzsM$SW`y^(-7
zEFV68Ecd)~3O==8Pe$cdCY~&TA%7qsw3dZ|2eLd3BO{R$HY1PgToTqh3OIyB>!12f
z)Sx6t-_4Svlb1{GWlz^WMR~#$gT`<tb+B{U0>`KiR~5p#o7GPDfGOBCkCin7x%T8<
zO)kX{tp>Y~WI(-=bbevATAx8kXep~f>@wtYJOjEM#)%j+Gcy{nc5(MRdAU%R#J3uG
zK+XVrMqz0)@BUycU>AF#k!67OR903-Y@&6ilE7Umv$A<VdNg!bn@=%<Cil8DfY1Xz
z6i3=L5=*85B_67eQ^h%sn)32Gw`gVKVpVn<?`FzM!?BrA_+*V2-B)4#9W;unZmT<E
z&-7TEh@vp|n0a_qIj7zxZ*JdNQ8-ehORw6Uwdkdf@-1!BYtD5nDxXa`^Ovf7Kb~cz
z9lpIj%Xr$CUknWm&COy7Mj^zTBym2+Gb&t6a}^-uvCaz%3qv5_b&#s=AGsR1zc~YE
zAdr~vXe_xT2qYOr&;|}O4(a=gP566LrOvKtmseNhoSazFiHs5e5c-nnZQ-`QzTY<F
z{b&#5KVFX1sYQ;ZncFf42AqMdZ5Mm9NefR;0aiA)*sothkB`5*0NJZtxBX52guW1`
zn`;by0&E@6Ldvl>Q5wgtXYe)7;cU2A?BgY$wW_sWeu*VoG$XN^l~<nj>`UfPH1Y4w
zudddjz2`$uPopKj1Uu05t7uVlgeL1vX#k0QY%QF6t>zVAim-`H+Ff}WUc1MfdO6vd
znXL{Aw(HYCCN;6=9#l!n=k>Z;xH|rvaSsi|(~1d4q@|e*fA9M>Vw5d>9Gs9}zeE8C
z{Iz;lS5`KRSzDsrvS=pw`J`!!Ryu)R1XvJor;tEIY_iv{;alz&5U-=3T%dj$4|SDj
zHe`OFT$!_O>HNxOjE0LF3S^F%rf@U!(`ADn+Gp@F`YaKK{82LdZ)3a~!a4NheziQz
zweWnYxlRV>L2M?iN;hX$er++yp}BJQ$t62z9K3Hpgs$#<QQUNkshtP_jj_E`AV>DO
zxxZmOf^Ds3x6pRoctt@$vD^krEdKsI65z<u$Hz86+FfgT+-m`QmdDLTVoz_cEv>fu
zl!s*GrW&(qiLvYj1#o71IyxX%SJ#lRTb?L_AHT?>`Tc^4y{0z}+J5ZbRHa{Eb<EV}
z<D2ahmvHsJLUV_oWU}S*6pM8o5(^m;P`UVwi{(zyKLw?e96UUKA#h1dT!xcWs|M_m
zot@qBZ@L`B`*hFC8Qs|0YfFexR3zIQz+rZed#|t8qI1g1<S!#}FHbS1Qs~7Z#MO$i
zU|?Wqn3$Z+_{s}0%DgQ`>thFBy_J$;UU-OUmi6`ZrD0&;VEDXg4t5@mfxf=J!;X!h
zpcnl2$3rU{8-{G9U`QcI5KX|Nx9i4L1NajGk6nku*1){UZl{vx+#Cwd4G*W<yYCpl
zlR_dQI)Jo#vKr2`x+Ua|1Hk)Va^ANe`;#4so%UVOCnhA(adF`z-}nPN;v#;BXZlcI
z5cTaPNvHi_Yz466NPIyj{COHPpFv2TR4QgM(Ve!ucW$$M)|#Z0lrT&!sk$dD$vSpG
zzOGmv{Cca=;?=Y>*m8Y8oF$gKMb0DFkxs1KV!gMA&+K{YfP{p!y!0e6bnYKHGBPsV
zaJ9t3%<MDs(>SNOxy?!|O6x4`2dep$U++DS+29Y7w!S{ZmfMNa<4$btW$h;pjHf&A
zCvTV2Uh0v>#VFuw4FwVUR8)D0pt6gDCu4hoo%Qu4;@w1KY~Cf)Bpv!eQnXNe%#MQ&
z_}*?-4}-o=VIB3#V2jlXZd8%}2Gg3QB%+3#bhq6f=-z7~<3Jpc(R9*No;zG<kfDxa
zmm%;rj;@?z<l&ham)J7VT21nh{v6WvL0&#ati5V?Z|~$6B2#8#<GfYQtC(nIAs}P!
zXB+HItEs8IcO-v-)jAwpT~l*a7F5N3IW=knuWq{bse5udrMA)S!Yq~lrf$JQsYK&-
z;Su0v$&1$DtRtg`h=kMu0FiQ=712WHI>KUX0PCQ_t6wk}YL&#M>gyZMRa*inyLF;a
zx!hsr0|CDaT6uYSesQtH$B!@5J)ICA&xD@F9MWDhGFJ0tbR@(!A89B^uDZU~I@=!@
z!gz&=*%KvVAsjKqbFt`tqzmLOPWM%DAVRPzm1;!-Y<g(rUb#dgG#~)FP;9enmHV$K
zif}Sm^QG#Ry+t<UrPDyL-dFEtZuj)Y^HFJVc9*6Oz{WZRhz3kSVIh!GgBBOHz^t;S
z2D`Y}GQXsx3$Q^2mD(jMtwD`5!eUhQ=QF0|*zX>1Y_fckf2W{;{&<>rbqu!ChpVOT
zm3D71ZxM@pr~H&OTOrJKfAv)UsBf0TG2=C-Q#9>9&~28|5O@v66!jkDaNa#m6?dJ2
zouFl{Zu+uqG5SXu04@MDE-QoL&|zNM6a?&-+>Gr&3CROX)!ktOs_|ed%R}oj@TlmX
z9x06`w^zW`)76%9--J*PsQ^yrJX+=5I5-y)0yncIA}>Fjr0swB#@04^_3@6l6|fM7
zMU%a~y>>*VMu6TmqoZaJp?*HFSNy#*X=PzSujR4=qo|~`dc4x2KUp*~1X0%cA?NAI
z%LBP}x?hN-{aV+!#mKxjj*^jfV7vw(%6tWF_S?gxh0bD@-(>=nwOnBUd*}dW9qb7w
zDK5U;e0M13<@K0!_>ng~G00-E5f_NXZvg7{<sBt3>!5BTcO;)I?1N!a<ghR#0XXN;
z8#`)%yxZhy+;yKd8VWDt9W~z(UGDd`f!%-J`{|+f>G2Mos2TI(0=d@XhLusf87ZC?
z-n#LmOB{P5xA&5VmZOMTcOaR=#KNMow&LCQUn$njp<ljW5(s+W5PMsM^b-(fv&N-E
zOLvrVYZ9$<?lz{{=3P>teU*OuHa7Bm!s1F<yi*PO-m29DhpgoJ#f9$E<HN97Rqjv%
zrF6pbN-%OuklD`Ehfs~st%J0!j?kATCi#-yPoMhJJ&6+-k)3Z69U{6EZwLsnD7aUF
zgaZQ_>(nIh7SIy;BeK*DhWG`<>iv;X*l%QJwKb@~%1U-<8zAj~Uj|){e@QbkUi)uJ
zz#o#~f;ArCMPV-|DcLQa_~%M0^ufPBN8x(D?q3&zg`(%X;ZXb$gZ`0lY-y!{4{Sd1
zJb!KyXG1GxVrl94&;OTg0PO{M{Y&csOHsf}D(J5@^wL-@)H5V8MF6)ki?ajR%V4?y
zUoOn^_p(EES)|Ik=H~DL7+pe-`bSz~{+}lp@liw+h*O2XMV^=XIpE`x)$==XXTksa
z+lM>b+ezE3fHnr5!edVbOpIfslY}h$tvDLA$g=k1#((|;LqZer^t>VH5X-55L@m~T
z_UaIKY~CR_w)z62+9ac$)mA~`MRoJd#W1h#YwnmD2(Q(%dQW+yhlT5P5euOhf0d7V
zP^a+?@~;IKYVl{szvuTwODa*W|N3A<a%9Z?IU!<`!B9$*Y6xnV_1i7so7qW74Q5M<
zg^ZbIkcFc3S-EY$yHDcxZi+D<j}2oL@*gjQ#}E8Z$0RdlG<W-zp4XnkSwYgK)~@0V
zkNpd)Ct)YKC4r-Q?{ux4Luh#Tx_A!O5G!*s0T*xB@h)@T9w*ine>snheOqkYa5%ZD
zB=LKKNQzm{o{kjhsh+2dY3v?RM@j}KiQ~+NwMVBcS9j9W1c>^uoLU_uc91?ue^whm
zc>oG0cy|?pF@RssGo^k=9=93<Ke!GrJmmzhOB$^%g{wRKd_e0YdBONQw5*x)Zt&{#
z?75DXiD=b_#&H{Nl^jALXuV;57$4A-Zg6*peeR2-r@e$nr!_C(IJIh_3D5JV_2JNp
zmU^dhigDrK&*K>`!ZpnMm*gku_c>wE>o-f$u~;jcwU;5^t}v~LN@dzZ___3xTJ|V+
ziT#KOW^HLpvcWO^?IOn18whFf2^j1b4nzUr#bh|)yt?T!e07(p*hO9}YRy~C_DLI+
zkJJCV)F&v_-=6vQ6-Ce68f8N!HJn4nK%}0M5j44~X@Q8WzL1z-Uk38sWH>5=IWD|I
zipS4LKw^N*G#9g<hP>Ws7M}>z){Puc-*98sj)5fsfm$+jHK&(OOgnY=UF{*pb60qS
zd<-?esWiV>H+~?*ufg#n)SRQtv`fs&VzS(y%QAn&!(MC*;Fx7KLH$NjRENw9I-c|4
zx5ORCM{ZB31cs}+vSjKx*|lkeKS_~}<F|CBd_v4ze9W9FpW%Kq99n6;l<s-QXEBWp
zcB&B^JB!-b$Sff-dUgv2WFcOAOl;Bg7qd|)#Iocl3nQMK77Z(gX29&#D3H{9r<m8b
z2OupJ6W{vb3M4amME)*5ae3*ZD=7MvMT!NQ8)ZNw#<VprzV%HNi^-_wFBmB|>W*~G
zv+_6f(zR#_eo*CZUucOtE}`&!Mj-==PT~nt4oKg5mGf}>AX&)xpNV~Y4iXkNMbvH9
zP^T#0GI^v481;KUd`rGyMk4ka_Zj7C>FVN0pjz8iusuFf8l^8|;qPIxfhe=BX>4N}
z<QYz0nBY2`(<=>FV8uCouR<kAC^%m*BPthWDxjbFa)+8Y!4U471+r({`7X0CI51|j
zY+l=tw;=cMm0^8SXMja;ylD^K-cJ7B9v!hnL}A*foqs#Zu93?qglipyGj@+fvj0mV
zwVWlPiy6ChG|7|Bw0e^u32$%m%Gg!%lz*e!ynkxcXw@zE;KC%x<0<!a>Oid7;neq#
zdGt$=waqFQF?i!+>A0^B-gx21<TbOOBa3q?bH1l9DT?6oEZ%<8YeU&GEu`cAiM5&*
zF(OoKi5I#B*~Rrv!GZ%B8S+<kUrVRd)cwWqF#%h`=&s;HL%cPjUsk!-%8{61TWO}#
z@|^$rlCXiUN$k>48DkMh^><53l#t`huv<M_VwvrLz0V<YGG$v3;2Rn}?}AQc%C$NO
z!?an{yR12%qjsVKoFB=12h!)mFgaajVtn~L=YPWgKMMZ8MRjJW|0~)26BoNkq_!oU
z+_=mnQ>}VI>^!_QqmJ_2RhF>4ij*cwUouq{0<H2ccOZcvq<AO>__tKJq*|bt-R4`c
zw^6WUlVmXp5#2cIYhu1e&1bzQS+5s67<Ynz8|L>#BPC$dP-M*uWbny2S&9+XPTW#W
z9BO^eUT?ODwv?s-KgIpDhY(xyK7G_qG5S+HBuMxnf|xZCvdVizcY<dPVTZSUY2y&h
zix1yqJ7|~Q$6jbTCK^9%3z=+Zez>}1I8D>9P*rTmc%0w2l?G%YnvzX87H+-V@GS-8
zW7-)_qVh^drTpg$28iNog}6j|Pw_!=zIq}(+i`1368q5FM#8qQP*T5}VL(#-9?;2=
zk0=}U^i^nkqj6ali^!G*;k0q!?LIV-g@#A&Q9cwsyA)mTOkP5Nf_SSuPr5$%;=D<o
za9?6;lqcl2m^fhRlAO;Ijit^W-LQHh%U)QfgpX|0I(wf~&Y>T_73Bd0idG;>FtlgO
zXq|t*_IVJm5?V1!T_v~<<fG*=g%O3OTTW^ziC9tyxg1JW3FDw^)cq3f7*<|Aa8v&b
z8Nv@R*J`X~KL_gPwAE&l=RIRTyorm|y&3;jX*YnXiIAoHr4C#nc$Iin-QK1}Ia(B&
zD8V)!=))foKRF@ta^1Es8KskHcWlO!R**rx+knik8q)n5SoLgHMYi-(%$z|$l4*BC
zL(0wBtoe!jbSo<<K1HGinr^yknnzXuMd|W76<vU$P`K3&V}BHjo)u{I`teje9rbBg
zTu&__UJ;me%OEeC(g^|nJ7#6mOd!BO!?zGFHu60#kWFD8O57Fy_lKXBvAX3o*krOb
zd`cOk*`1rh__e6-3|~|e_lOi_JXk)@2PEkacv;Wb3o>ooX!dXc_>UqlXO$LOoAjc6
zY(ru_!&gd5x+i=v*r7n1%Ytn<@2B_PmKjQ*@!I92nqBjWDaZO2TQmBorL?C%BLcgy
zm+g@WWS`VS!YU2Ewao~xNLFt9=D)>McQBq><J@drSPV<!sP*E6YyMTR(u6>M@b>sp
zv}4DOz#@WmPz*=i@#duGmz7CX>gVTb*`U4MYK3m2yN(zC>E7>{-C>5^ndu6JXdey2
zIG?Q0KF$krzK2}X4@{$uKGFERteewyXNv)x5&~BE84F5Zmd*6gJU9xMO()5ji@D+0
zmRpaho<1E%(D-VNgk>Ee*>yJ&+IA6z#;OGLquyn@6=5RSj^JF-k8gCZ0hzz@V?|9c
zc%P`C0!iS0iDT=1C<Kl}3spZ;7wj2(Z8;B4f1q1$sED8D=^{MHhnIkKmBJVtQL_2|
z2_3j2e?^m#wD7;7;C~%7QV}rf>+6}hxwi`{o)Omw#qu~4<J9EjukLQx*w|Pbdwcv9
z#ukwqZ9eqNb+dp-W?1EewX@IEBcr2!ds8Jd8bkJz^Z|i^q^DWIZxriGa$t{Hh&m^$
z1j!i-?Ys1e!NJ8L55TV=q$YTP9of>#U++FlolLYehAiEOk8R+yX*WIbnr&Hda`p$`
z-LJnrcW_=CspalZ=IArh0{DM>d5$DRt68SbLsHAHliNGAi|h@%2v~Q=XNcp>mFExL
z`c*g!Q4$q1i_(RyI+-KEkveMmv$Zt@Xj9@I&sMtQi?#gY)LE^b2bD~N&9(LQH)9&q
z87JlD4sD@2zbruz(~Rbs_sxKj0};)yy`$3ZaIEb3QYMYoHF$U@C}>a;hdGwsWJzZ?
zv8Ms4RGC*lXxOn5OSzpm&~?;hGPtS!InOoBh4?|Y#tN^{H9p;<YarJ+bl0P5Yib94
zp%z1qoJnRHX<9c<LKF~u$b6|Q_lRc;LPB1DSzFe@{GQ69y)+o_@p(e%W6dHr)B<p~
zou%bQ>9v}9GlHIFPJ7{SLT;eamd96<E!X&}Vh@)Gb0{!y*{VN{R&Fo$WC_?n4q44G
za9$4Cd+kn|;IkZAu<Q_l)Pcjiy}eyt)1TZv;0=S4xTv*$2**0gh0?n243ckeIXi4G
zN=e&>V7_{XuV#I@QrPl{oRKt{246}iJk1Y^q?Wg-mP<edWtP8GZp0?BN`GLzD_RJ{
zepu?>D<u}*+APNTiO%H|joYf+t>CSV85GT)oOElW`ezDXI15O0PgXnIfn7XS4KHT8
zU3-|)?)H_X?rv@vYPTkO!rerXzJBo#R^a`2FTjRzOMA=oK0{gQ-Zlj0#8U7DlU0@L
zRpZ@Q%WC^13IF9cVl#whpDF9~fTJxK?zIbi3oW)rhs)?XL`m~fvU>&UqnU;>L4smL
zuxr#eL|i*#U@>V$3=hlWo8KdPovj){*I=%+jxct!ra2QEEj0QK;UAQ`tsD<?d)}W3
z=_<)lGlD?p*XK}glRK1;TB5QSX&Se;kCRosu<*93r@1#v+$u?MRtD#bopn9AHL;&<
zd<sWF&hG&vVt{k^IB{3H_E1W2<CG)c@vut!c-y-qZ!=lj<3KHdsk^v!(o5~#t_(5~
zU`9Q2v%;DDdc}6X>r3vk#7%Fk)j`(JN7quw=#?`RuH?pH)Su)#S7WW;6@d%#sWVR(
zuV6VC!CdQks>K{5@G+k%X&W0GThV{qbrCoWcS9eH+_shRoG$!p?XQ8(1_I5x`g%N|
z%HiV=@oU^}Gn@(NA9&Sqzlt5@3A0O`p|jZN8ug$gI;B8xjpO35r$+s!ks$kIOEJbK
zkMFD#yVmBhG0>VJ?(F<rd^%f#{E>y@IqdXC{2gK<Tvl1Z7zTae>%WoN{~FD_7mgW@
z-aoe*mreF~&Jt2)8h!!w_b%&Gk=^MQiKNA%Lgj(ah$k8H(AMDl?ylK<t!?`zuY!WY
z@yQ8`dV!>b8f(GiD>Cb^{NFe7i3(|vTh<b4M_wjZ5p~J9W&I}XAYF6YHNKCj!^9f%
z*~w>NctYp1+af~%F@t0V9V<IfEF1m<PT2kMhlv}aTu|^Iw?c7_d3ty(1M+BfCB)4;
zeIK+v^Mp&NxRU;Wm_1n1DMIvksNrphOyZfkC33-qp_qijpr1$E40#*%Vhl!Gd@oM~
zg_c$PPwF1Aes9*H!e9r*qopVZL6IJb{+JJjKb-!vwn_p73{f$$a+edma8p4+K|oy%
zNJ*I+7G2Q&_SzX@mlY8L67*hjz}gu9sRg!85K`?Ng!dx0%7aRAm8#jkOusIk9TzoU
zFOPiBLI{+3+0ra||CDTLDsc@Y<H8peUvTq>XM~)d&#WP`-5!G;1_2A~k^r?iup;Rj
zaRnhjt6SXj>HfeU-+G}C?nTgGhpJb!o>$o8!-^<xZjZLd_1hEU`G9>>lWXabF^gWn
zQAx)ZFm%)x7|i*KwmVDhCL6(Fc;u0u%rmSVlYwrv_671{z9Hka448nDQ&W244B7En
z`n+4qTaX~j?U$`}m4O^QJ$uhyIlEQXFUtFDKM)8aXwb%7ySux7iHV7kAdwOPm1dN@
z7TaZQJ)ZbJ`r^@HJc)_`3ILI-X~Z3?S=<X#>>c@M`9u(1y@R{P^S;h6(8(8^rlE~o
zaZ!R)l8OIF6(7hEFsKR#bKLfQx`(+apPv;wMsSNT-a9znUN%qo%Nm#D`RpUR2q*qv
z0`VL!#+KUmVD}dP2>WmCHU9?A=W#{+nz0KU2rKZZ>PZYf$uTI7eDASJ-Sz|DI9A{z
z^Is61+F*%rTRtAZ<bFmHuj;A4Az}PySiacF#~;1Fy$fI9P95Ri1pfMhDwSc3d?Io5
zJufxqaT=6N34pHai=^KC3!yjBN&bz{m*5$~WEK^M3FL$~<7{U$BHe!U883s8jzPGz
zmn}FVa9Pcl)yXO<VtROZ^e5~p#3n=(I0p8pQ)$j4emW#U`HJ;k5*efe9NnJOE-PbS
zS~$%Nr;uK-=~vx_1eI>T(8GNUjR96cg<x_D`*Qm^P?EwKG-M~F<76gf<|`q*3|Tra
z=(YmTooSAILknhMScDvBu-J5U&#^bKG0|1YyB{ACrd@=9bvi*CqmaD5!vc}8oS~a=
z-u9h^F%l-^(Q76cQ?u`6knXU9`b7p#v{9Pga9(nK(TXn8b%m_D>l>+<#WozmkJf+2
ziOZ;W3E#2wG}TzY3^ZiPIJz2usSW!^?K*a1%grV}I|qlV%+<+>h=Bpc{QNwHG=5C!
zeBkCmU{84b;?04i(=7vZ-ks<MoglGz1UpGttp%`l2%T_SK2Ih^Sl=dG`+ZJ|hM&?l
zuc{&asVZO3t}qt0YnXp^02ZtRO8>>%F9DFPC*=J|vDUJ*IjONfa#9Q<qI8>YBXVxT
z`sDZ_QzxwyB-H1#o~TE>HsUUk4P^b+Ov(FTTX|P}{0+UemOQKi3!Ty*CzdAT-931>
zSm)!S7t<go6*xG(AcmjSEwUz~_~@y0B}_}rPTsGTJlQaDz;`X57`#y2re!U=7=V#C
z2ZXV^LNKXZ;LsTTjD`oaA7VklghTk1vk+u6e!e0B3Ad9}Bn$;CpG)DPcJQ<N5F$g#
z7heb)w_ml25uwbRTKyWJcb=cqZQ0UEnx$ng-HoSD(!a;|Q9bby%!FoWcZ0Y1c4H}=
z?&N>A(dtS-mmIo^Isk-}$b+8QdlG2tHTxr){b`Lb{WMS`vSA526rwegc&e2oqE5>q
zUSXDtme)QX0c~OY!$oE0z5^3VXcHF5=PS)2M0SLsvNi55PsnV74xdeVD-$Q|Y^$d{
zEnC@xFpA!RgcjHvo#7Lq;e9rQ+w4#wBO+IX*Xh<`IRRC9{eEgtTzrthp#&z%lLeWN
zm7ie#G-G+Qr}ouXs><SeUw6KA=1VvnH+=E*0&bTApS@%|b~>xyLCmT$_!4rE%f33+
z4Rn#QXFXnCpiW{c95?C+=LgAwP*QPnmhsOqe~}OPL|F|KWaQ4(&Y+;O2~eBC>kn0d
zD$4Qn8H&4n$e~*=BA4>nfb=3`pxUcjI}`HtN+613UUY_^3-*YeDX5_ydy0b)<)}06
zaNljdx@`XAtKgU@lJ|Ub!F!jZ_nahh>Mx4VSLA)l;8;@J%WEp^UE*3`F~XaJa=Wh^
z^RV2*jmV}~60SjNZWg5q4K0hIQ#}aJ+uJFE?P+MH>BV2oDTK@rj0X;L$ZHkp&<Z9d
zG)6<|$V^O3_z-?Q*yXN<aQ)GeUYfWc5+RP${G0GE;H0J?K;uKTVh@9vwhvwgdD4DC
z4us2VK`6bA{`n!MDi?z!<3s(vIh#g7Yq_gz?9{l}Nr$&wSu!l+*Db{~Z4(_N*Jes>
zFaJg5l9UJnaB-MRUN528fC<oXWeTUA!rU)(^aotbnJ$>sTJIwolf{Cd)$$KGDArVA
zoj34K48IckOVLTJ%#&*yf%Wf||E}d{VS_UpEAOKwCa)iFBl45zxi3R4=zf=lh8?Le
zBnqq-B?wcL?~TB-AnS~m`2H`!{)hR8N7+B3z6`<N`usnV$^VR0HW?8yZe!CHE$=Tj
zV{v?_Bw;1$PfijaS?HgeE=SgvPqj6gbJ#tCzquZwu^P!aynV(o8b)SU6x;4u@213Z
zoEeKZ@)yVPGkJMQ-tGWVMqW<D$={<v_90p-)tIzo^h(swuwk}kJIgfY+tzvZSla{m
z*wAGHeh^Yb^q@-Bv%VB9Ar8=R7FR^UfeM8%v3r63dC&M=J6}-N+qAAsvdj;yS-<zX
z-ZbI(oRjlixp(B@*GN0J&p3Z;3-$Sipi|1Sl<exyJ+0&J4R#*!m!@W@T&Lg^^3!xo
z;Y{7Oc31%JvMv-$05mVEI)%#prQ~x_p>R1-h}s^aZZWCKGz6;2q#IK4D04{B1{>)$
z%7JsY9bwcC1Ww`){jJueB!UZdA#n(w#jZz>=XA@io9h(AOoa)Dg})(awcUebHrz1p
zauf~ToF0ge?zJOuTNFv)Z7VSiB58BxC9Chq5|%wq3+k}13A)TV>RhysYi-|kBA@>w
z=Y=l>3cw5L#G8f;IgGClav(jmSs3I#fj*U>AlQDAJUqV}^xgbwMPuhE1qq^&Pd!A8
zUmz9x=i6W$OjVEeyn1%m_XjI){=D7%pH1MOg38}4`s>?Q6b_D#x;8e$7kt3s#g~Y2
zdLJa2+#fdr3_Y2ZYU2MU&^a1+v1x)KIAct7(RnBi`GW1v!5|%!!<Un~hqZ|!*;d3~
zRDIO$xMxS>6sk(lT+5xj@n&Jb$`2p{dGGw1>X^8*QFzop#`E+#r2y){HV?BLDSNJi
z$$@2M%;JLQ!y#b>g*~lA_igWPs7}Gt$R0^n1jygE47dGe#7AGSiwJ6d3>r5?IqYyo
zVXq@+%2|U%%XEbtX3vrt5hUWPWU_#`2USAA!Ka3ReD^C(@Fu|0M94+!3YhJWEksJL
ztumWW(x)`kIUV3d5eguHl%Qbtwv?dWEot~Ls8KJ9k#isI6EXp&Ti5js&x!&A3g%8M
z8W0-U*vSkcan{OSNYtZ*ou~A$QX&a|CTBc)!_Z$7%g9!=w>=yJfC+ZGi0FN)tr~?d
znKEjbfTAWfs>V(j^*vJl;L*+m&pa!PF_5etGwfW=AuLoNZUc<1*s*dd2nv47stP6~
zsE&P56{MH*KkKP36B6SIPC%WLRT^~b-Lqnn!|y)qVCjjPdzY3~ju*CVXX1t=$4A$f
z6;hJWf^DgipcdlzsaJke-Amuga|IG0`&IZPpw=SQpJ+dw7#LH9tVt(+*BO2*bip2&
zBMzqTh1p7LTB8I2H8wg`?CGi9Oipg@n_ADi>iDX?KrTHl_?D^N{)8*|U7_s&y|i=J
zM0qv~!ZX9Ez*T{`0|x=o)pVk)c3(T^AE@w8^4^B}k);7j1nBh^hNCBSHocXDuU6iz
zO8$Bk&Q}vVsrvd}(CIT8o#%b?hi!*t95aM>(Dzwg4viczlz#O9T+VtcUwR`&bI3{M
zFa;O7dKN?r0VC7pSP+0Gg2qcL$ZBOQ{W(+Xqu%^vQl9A&q38*50Y&M6{9-%!@?7?x
zcE78CrsZAeJFnsx+|?);`{|S~rcV5j!Pe7X!WjI@u+(CmG3JWA)HPd2ISN0!l!vG`
zgWp%Rl~(n<1ye~5?UCOA9cg%K3YM6d*!7RNNg}Qp#DCV1itOK`Civ2CEaxRZ+SxG3
zi5*F}Y7#09c?k;DwRP#xNv8JeMwB}r7GHx{W;Mb<p&KRQCeG6f9VPxuR2mWK6LL5u
z7^%tf*v%RU*IDGj#CEAB%HIx+3Ue4p`n~vbu~j!BaEo2g|G<rVffzWDfM;tM1c5oU
z9wSs69l^{T!MKIGLH`&cdfwa!WNS+vjA5R;Els^*#o>F*E0Dq>^L(hcmX<Leef-DY
zTIrKfgh6}{5HmGFzv)x_NqAnkkh*j-R60OzS1K4Uf)egA2onL@X@<E_hbPDTd4!z;
zWw|U!6OTLXMFLyyMc&hap}v)ne+boxd&*up>REr?Xg_lT(5G05&%9D9jV>q0AP7L6
zg^DeWket0N$1mM)aiSu#v&yn(u-OkcosL~uSx$I@=DHgR_y@NL`?Q?Y87%tc(W0c&
zF4yN041}eCHNdaomU+1NF4*51;Gc1(jcUqnir{!Uk?=8=cBO`3bb@sk$<Qm2?ViXA
zw9GWg%QGmNM`j3G>$Niz*ey4eficdfNLw^c-ulb;(05Q#k)KX|+}La`fjWQ34i~D!
zE8OYZO+Z`bDdTvR_uq*y`nq^c3-;2{Vg89_4;x#lj6XS|Qk|oSz!4!b@+}WF1T!>T
zR#q10q)M1#v2LRsP<}8eKVqjy$7heSCGw%#%AJ{ZP&XnsTD;<(v_UjHn<r>&Z@@X+
z)zTo|X!QB-z!J+WsBH-<R6;wYYd8mtkRl>=!H9^cmBod`=q<A!Q%w#G5nQzc*OJ1b
zRtWb(n1AG@a`6JwSLfv;-;QjjwY^1nnR<$WP=3jxa`d-(k=<5>gMjLV2a&i%Za;_6
zLHy8fAsFQOFUm?_&OCH#f2v+pl&TKOyu<jU5-xSd6^}T?mlvG~<heJNA1*1TQPISk
zLk8%?o^|DjrPftGSa+^n^q3MTP=1DO@GYS&q=Qh!_m{L|f9c9d$Q+?bL7$G0L?8MI
zxXrALxUFaf#q8s3eRoQ{U(RBRMLx}iVQ9!8`IQ>I;x7kw$4vv7MuZJ0l87YvLaZg9
z9)JMgjt(o=V9K~m2s5BQA<wUb(7rMteiCfk#do6xr=<}6t?w)r-Wz(2je-;|2Qd+g
z^sHSlpNK9{RKs8wMkOF=;6lZaURuuxZCo$SRZ`d5hzBx4ENt95P|UM_!Ixh$2fOgS
zSQ3R;ti^<?J_qIy6h}95`xnK!m7XZZKw2<~n`Mf;wiHsMYcggt>Tf-mt6|aaLISnU
zbI6g+Bwd=kOIpdO#l(4PuQH!n-Z9--&|jH{lvh^siD58ohmylibH0oM@1lknDV!hk
zNFdrLW#u(rSRxy6RngwK`5F>vClcvr@>-cz&h$Ek-yBHUnzp9gMj^LEJ+7b&E(MHk
z<Qwx3KWH+%CN^ykU&O7UwOhwC2s69%^mI7&N|O$uTrdi7X8ao$vYAr<Da4I*==}W?
znt$W?|Db>dHmTv~<nPojjvW97HYTTah<*V8aCXjd61`mTMfn7u{0gYODQ3Bg{w7hV
zL4CI&I4%+$R_XHX)Amu(Gs+uDswMwR@!~1>Jt8o%dx)tUH<VyS5cb)G4}e=qTG`d)
zp|$YcnX(gu^rUw1cDz9%06&c?$DOJk1d`X(jKt@%n%1DEr9}XwBO~dHd@1QS6ZN9&
z44)Yf18JSng5M+N=Dhf$cTqUvCT<K$e6dI&E^n{wFSC*^(r*)Yc7iW4!p+EUM~|zU
zuCoO7=et7s>iP1T4`U365l719y%O@%`Q-CZiC(av9hETAOyk}>A#Y9RHyn!(;|<#W
z5x85f^wG0m>7DQ<M&{270=_axpahreLC^5hNY#vV@;jy2Oe6lRVxpu47<jE`zC8)=
zTk<$VPEJlP%Dub3CShg8u(Gm>`~I)m+`A+BE1>3r<u~S;b^0m;mFx{3nu-P|K&wDy
zowaFz(Bj{^7{OCEHGeS3*!F1}0V#Pn*Ce>148^|Ka8%Vl!3V+xVUHObP+A_i%D7M<
z+MTS(Wt?$L)e&t(>gnlWU{5)d4mZ_5g2jnuTm&SutHjmmU1$SF9zc>PMZec2P+Vkw
zf#lZVjRX71m;;k~Q*8Q$fn*K&<~0+#hH8}mRX3@9Xd2<WaLvDpQ+EUjGumn3)6uZ_
za83~r6a?whDcq*#qho;-oZq)D00$cW1EIME;<}(%P?2>c+d+!v0>d-M0>?4I3m^!S
z{{`=6{$L%-N@zV^4J`(mn2stMKovdaq;iqe?cV%Aum=qUHgVAQ4wMqovI;qxJuaNq
z%DmTuo_!|rJnVN?s5#Vil1VLbG4O#fa+q#{r>l3O=PEY$hg2^JPfkafUTO2aMAx1d
zrGD|8n93*H+ZdkUtuAxm*<?)A9igtAfby{Jbt8=2Cj!v`YG&*w%cSen1p^`QPXBhH
z`5s-_KFp)eZtO7@JUa#jx44S<oRU(9M3rM9z@0Iy&us$9NK`U0pdKigLI^*gAHfW<
zyo6Sjh~`>WY_(2gtp*oa__O^>RV6zdD^9&l&uD$*8hOqk8CJ~RgiagftJV74_LU};
z(KOBIIjjWtf>pFp^Z--iJDUZq<fRwurhS?5f&@5b2A}_pMicvhE0?ItIQ(coDR9~n
z0eFB4?LPNg9;`cpJ5LhCBA?wY@-54}6fHYDCeR=k^hcUp51(#ih2HEYZz)0g(FeL7
zY+H<UgZ_qpm{AB0?P(%ww+}tCiO|S9ysh2HgmJF03g}~HyJ;f%8DF6!2L_P3Wjc*b
zWLX)RWi+frQ}psryww%_3PVMXCyM;2vD!5?qYEJ@O<p`#6q$82J~k6B14wCg*_3Ml
zrZFeewGc0?`<*U#<5wE2E$UXpHoDzd2UkJXm1rH0APVTV5t|>tB4EW3vLXqCtyRJD
zFIlimSZE*k$i)F3RPZ+sI{AOWgZ}AvYbR;T{S);%=1_${=lGK?TwhB+;Uqk7To1?L
z*KNr85xi!xQGRDQ+@I9-6wl8`X=rf60E+J4_p(O#5-05xt#^^-2?=Aq0UXvdKaZqA
zq**iXQIcZtCtC21dsan-=7IM4FuT;Kz1zsn@+0_ALb`q}y!!u#BkS)R;#sZp!6s`M
zkAbV_t2SBgX#4gr8Qeg%ffXghHZ+>ls>;540%%o}*uT`o%F3XKhKnw6)V}MYsrDI*
z9uv+UMQ!jBj6pFoe(WCfZDHZrt?>$h1T{rI-1vd#Zqh%mDe@~xo<zms%Ux|2xqiOL
zk=<I85crhp=@vBGBj0dSw|!gTqi~dAgNAP&Lm#0;fwA)g1EvO!N@7o@%k<CCL`pUI
zD}Od4wIdz6!K;;a3@u|Pzc<Ys=h1(r=lv5zfd(PwYC+We&Q%)J0NaB6`fdy~NIsK$
z0h_i#Ea*+`^ApLa|1(FP2c8d}Wk}CI`SE|Dn*@pYkv|`8%z!o@xOXzU0puT`3zkwM
zXW?g@|3g#CA^dB9@dW@p-^s%Y(2lOgDrFHH2Bi&uOB6-dM38W(3g5lvuOJ*~D4$iq
zY8U%7!}Eo6vs)Hyz>f3rB4B2Fx|cfL3TCxlJsP_eWVkRS!e`K;0XF?M7bOI*szF{|
z-azh@{Q6Z$u?`+BDp*Wl)-H0^)j$Y!oIuXUW?nuwWdDt;jqcj?-YG6OAhwUB?WKn@
zwn`}im;?CD9_LhOSeRdQbo9&zV1|D<df&Hj_vh-;nBPu51PCCs_JnUWy9|Mb5z4tu
z$$e%9wxT(VJr(AW=f;*U>Rp=v14bt+d=Q&ix^SinnBPKIu1c<r{ngweWQhqRAiYnq
zC%v0CaC2C=k4OJ8t;Q+r=a=IZyaa{0*_ue8g^8)hDpK@LS$MY+tnC?xWBVzO>suRA
zv*SPbb~zao9}5TjOQ?Zqt!b}3=TQ*n#Ks6Y7!uWf_|toodNIJYl=<UFuUUO`G%6W6
z`O4-dmFg_e<;chm0B5bD^Aa1OiZN?au$gF|+>q#fgYfM%gc@E)0;xyL0S?Ivw?B{s
z<;}No^#39KX8uXPB=AV@0lnka92&^A@mvweE&{8aTe*(2==S%CakrIV(!S-Jymn?w
zUr%JrM%<>A+J#c&W2B$q;VrvqiQvGg&Wm;T4^;u$U>+ty0praLcs=vKm%o9Ue<nQF
z&v{$x;D>a7U9-aB#scLEov0IQkWnYptq(>e(jH~mKZF!&%eKgbDom(yIN!=ba&i-v
zGz7xe8Pmyvv9nwjwAikkxLZUoDSXK=Rz4wxf19#Jj?l`|-xd}5DEvMCfhYA(jpH>t
z!LJjE`~__U2{&S(>xIAtIT&2N`wzV|P1|9>noJOq;s9xfN4{*{AEd9(jdt_f<s0(e
zTO6LJDmL_3d&D$zUw!TO?NLamur>G8wLD`OFd+6U%L(N9xs!l$$=2aHl2gUbwvR9x
zQ(tmaZ9j4b^qS4mxM6@$q32A``TD%|2?8S5z8K3P{+1mQktrb^4xPP(3mX$^Z|kDT
z3wncmL9t#&4Q~EA#QEs9Y6nQmSWKMk0mc8JB6tII2AfLpbD5-;@%}#w;oU)lIxO=O
zO|>Pq4(APq3o{qBmb!HSw<Tqt8P)i^7l2SC$*CH3>fk%L)KHD2w$q*k(nZ(fywWQ@
zpYsuaB$A@P#Y-kyJ4;vFXGjj!$@3?Jy>TZg&tJH{brk)qb=0xP<%3n8x-Hgcy7gNX
zsGb!M$cgo3fq}n6@YgtdjDAN)N8tA$eQ$h7l!1@sroNC)Ojqpj7f1?!&%y)baHyk@
zGgKgOSz)nX*Jg?F9%W|XhdF!s+~OKHKjgz(BD^JEa+q4O-~my*XjA@)kRBtakq+qI
z?r|?7&anDx?7_l!91*O!X^t(?%t+jeI*0e(V9FHUo^6z^!l~U_gx*5d^N5zwWkw_0
zA6$l<d-I|#exr*A{;`5oVz=g(?-pUoXA<JD^?B$$pf*l`gxjD6{0trVMgMvf+@H2@
zll}KQf9LYc|5YxxfdPtJJG=&5?;S);`hnV=go0o7Y$#)orjMfxJI}?q=qt^z{jN|(
zIP^LU{{V$OaE~EXF5%dJLvL=F)Ehcn4J4KMw67Rn8^4^TYBy&_x0u-a@?-Dq`K@X(
zoBNTo`VKk2GQqg2DoXr!ZJnCWo7__g-WBDF##MLHSs_Gjwbj0YUKg4TOTWt5mBh-Q
z?rH1}fVMb-WForWwMSJ6c8&$M5tt-|e($Uq#wDKN_4>*`wNMBXFb^P(CY;3vLO*C0
z6aX!|s!dL0%JZO1s-m({@IgkidiS&ALE{fQA_&mnnoc#W6G6>+!%p>RPsNI(5||I5
zp{!hI!_y}QIAgMRA3WqX>H8gJg#JwA^QBz7Cn~p%dR9llE!)(0b&~69&Jr>I4HJ6{
z1r6+Soj<pM8Z#jW$On+{OUQrp)zUjNW*f}ihY;o(205zhfvMjki-U3EvRT4O=(Dtw
zHus`6BAFVE1}pzh@8ig|gW9tsHZ6z$BV8?@$UJ_0hG3lFi89*;Ft39Uo=0i@rX&Hb
zIoSs#NY+FgA@=YH#2jg4Z!VztO63)jF6rbTP|;tFU-y?1jbzv&2lOp=VVVjo0gY_5
zI+u7HSuzUIb6v){Q(4sr4Ba4ui`DK9l6m}*u%il!9Vac=!Xx;f2SHlSV$3{E*Dso-
z>rF}J=r#zP*VLyDyDipd{;;p)-#RpNsC?<dI^!MV#?pp7N`_02%hNYHcN#3I*5H0k
z*3GX;H*oCul*rZKwTzKJwsYq<1}EnIJT*@p(C~eaXjqkG<RqJNB&Vb`!((|fja-y^
z6geTE29h5S7o+o&CzuWUMn-qRNiT<Aq>+fU!6eydru{|0`n@)KK-4_Vwfgkn=m~1!
z&*uQ)k4FdLH}Ei1i=xKoG<TE;)KZ1SMoo3L@feFgL;T6v6%zA#_EYN}cTE&GQJE!_
zymWo&G02$euBa*M|1qEXqMBs$#oJ)lx}0RoYiU_sD17`#KHuy)XHn?35;-mN>!^zN
z=Z2@>Eh=x83bp4V<v5ahx5tcQmP@n&pJsGSZJy+|P3FDkuGlZ3aqStEHM*=<k^zkB
zUmZLcHCT)7C2HfYV{JND2Q9xjDj~E)FHm&6BW@|h<UJq~0o2^HSowJdPc6eD;+?02
zQHnW7K+07*izO_QzlSw<jY_Dhudcoiv!PU|*Qn4MKwlx|CUbz-hA-x((4cK5)1Z5l
zs#WxmD0{%0ag+0)tQ4)#?B4oCIcDGP6;%DX#5^aJ+-pJ7ym5C{D?K<s>11FG(52rh
zd9SHv$oRR*g!=eVwT~U*k)EJ(#F>W_U)5Ey|Ktx&EA$IIlPwUVdGu#9t)9dVNQN|3
zwfOSUX`gfO3Q(oZ0ZDr8GV~RpsAQ+4s9s>;%ff~?QV!d^x_9-8ALYq}PwU4qzrgLj
zK%C=HROMDeycm<G>>EnGc;K-mByiflEcu!+I`I2&*_{2{O+}qEuAYci8n_kb-cW+9
zWjz}HU4o4j^c=<l{9#yn0MPIS68`vctc)EHrN!U8`86<LX_EEacnfILR7WBppCAzf
zh79<&9WAMUFM_upx*$yjq^KU&)~r1lA1(N2*N+AsX{XTOk&tE%yCAMyM0^NkQ~Y}3
zUmy<O>-?@PNPZ9~X1+_hk4=)EAXS=eE2tUda@<wq7@_U^A%^*&AM~E^dmtOuDOM7%
zFSLCuW3()8fX`R>OCE%U0S0kW1*wV_p?Z8L=cA^(U!PpPK4}h{A5>lbOn2M?+O3<!
zN(rEKt+V;urC!BsKcu|#AU*#+xeTX9$s8g*)3p+08QV89((WrDBnGelqMKRs1J-wU
zGgDiLH?GLM`3>A1rc3lVAQnYbQKfIJ?_VYQH{=ChKUO*i;rV>$U!JqF(CoBZrV8zN
zmFG9$9H~HvTM6%usz2#XvM^QFKhvRVUb_a8)0bXf301v<CZeftk46DO@eC+T-dVh?
zDv6Y$tYVKnOpxvQt&>@hk%V!9l|M6SOjg|d_2L^eE7Rq&4|7*6^BIM2<$$wo{+-GP
ztm-q~w|u<QO|p5`=3QTVdjrP_tHxnURE8z`e&7@gZCm~rdEKDk4Lkiaj6O3^{M}i<
zjVyycTQ1jqch85k2Dw~#0Y+76D&wC&wdUSVBTBuwWfwm4%Iaz}`E3=FXC=*4Pcz82
zRzD3}m(oN>^P-|Ca^u{%eVW%z2er~c7(S+M5{Z!T<M4&1(^39}rY2nQmwSKqBEhc9
zgFDx=XxLu4P_maIwZW=rvXCZx2+1grE|Dt}iWW=;ODSVDOe3lIVw-IZwRTw>qh#-{
zx`V!+l442n|8lfryW>mXagH^+^iob!520fuDZzPTap911cx^c|9C^koH#J_P+L9<l
zwAj^%&zE=H&U}DbOgfVwqoGsV&iz$^P-2RzQsp<4Dd~D+;;z6uP5ij3Fv=MLq%Uo0
zbaYb8<9834LpY=VhrPEBitGFK1tA0oK@;3bAh-s12o@lCaCeu+od7|Cy99^e?(V_e
z-CdjD)-Z?Um*4%~_ukCAx90sbRa4bPbvJ!Z%h`Lc^^vu9hnYIPg<{s{cvIYqThMmX
zvlyNcEVPN~MridQJ@Me~$e9DM|4w!zrI&upJ~MZ-x-5@+sZkj6=3%|WWtf^HOzIJr
z8Dw{NkU(;*Yiu`Ja0z5%)NeI>hnt86U&3Q@Lm4+NRKOjOM|x)*aW+q_7UtAGPZj*^
zr>rK{!0{=i1AHC+)mgt=xV|OF$2zgsuxp-rg1k#6OAQVN2F@Y0n((?l6fT8p&qvss
z_a8TXH$^Z!n2tU6J1J{?ojB5?g&4ylDo^3n=E3r{wxEPcNnF}$6ec}8e-CUrxPFQ&
z(ZX-QZ?{zMFQV*^i+~wl5!#9Dz>>$mhQqe45+r@$?6BQoRF9WlDMzrJD?1xrxIS2p
zk#cKxih$|s7?%8!MTdMsIwD43y9@vG(Cx-ei$2dNB@@NT>K;Cl%o@|1b_at|@^L*?
z`&DZW58FC5T+a0O8dI~}z?_(wV~k>+cTJNs`Y?m%@{G@8g$nE~kDRsA(=?hZ>h*$V
zSW*nWkpT{8-`6py){KYBXjG7}Z_@<#LKLJm=mJY=JS}^ErCXX(=bg+eMDXog{Nu#r
z3Z$Eem~g0F=JiW+jd7jP;mFp~b{9{I{+kraeaL-!Y2^==m+YvKgnJX7Z%2cT<~mIo
z1T9F6HHqm6QTC^cztg5g(gNRnH^WcSpxm3l)#S}|)`*G2k^Y$-rPv$pAPUxMMT}F4
zD@hiXCVCW~G96V$8U()SV)Y)NF#QA2h{oG@l{E3b)1jz@@<-;-oqg7(aM<QWq!|cC
z$HRqc`l_dS#!IqZWQ#>*u}qI?81gN8AR{rj*VBvOVBp)=fzyi6MJ5l4D=X`^pCJ{G
z?Y&G2$hG7~49qHK&Hcrbcn#hnQ|2lic~=sT_GK}K!_P{8>0@BO6IUgmDx{cdoV4I=
zcy%DMQZxl8(ckiRin~$ia}8Xf%<vCd3$Y|7slleM=L9@wGTfY`_1+cCofzP}uFXdK
zei}s42Do05j9cq-mkVB1G29|Yl#LTV-v^}1cF3~>!CI+=N^x_`Ot*E<RQr5%SmO0K
zuuaKI#;VCyGJ=u%jHrvNidIFp+ZRs8CHL`MEw2}t?828n*&@Dl=E2QAZonnJ<7T~!
z@l1YPQ`AmNU8Jvw94t=LqrQTBPM|SOKL2pcUkPTv#Cc__F)pLrP!cy;%>LOs+VsM~
zDCc^VS?300V!9ALBLh_xfA8Jrxr=#80f;5?@^JKpJ?)(JVw)?=00Vy~7Z>YKOl#fo
z>98nGlqusttr@daXOykHo;k!Tod)opDDtU2sjj!aJjT0SV;MA~xHbwrgi2rIfjRl9
z37@YgIOS8!Z4AD{{)>0FAr%ZL^HlT!q-y?ka*H2wvTzj?i_!!w`LzZjt4DMe4$)E&
zd>FwhHZ%x6Brn$9>lNl&*MfO^F^FN@`6FYpAL((j>ih=L3(K^c_FJRMj7cR@aT$Y$
zp9>~9^-2JT{JDA3=J^i%ee_i{8L&|Nx!p7V3X`nCe>(ZojnTV^>|clYtOjuWeVXL@
zjkFJRA(58wAdDTIpN-7fd6v5!rQfio9bMq>BR^aRM(ea)hTx<=st!H_VGylvc!>*g
z#`xG$e&Zji=4gy+vUT3PH3AD2^akuDJK`drvcwp|9Xs*cq)n|v`Z$*ms~jH@-=-bF
z$DKVwTFd3)*V8Ub5^^rzJ$)VpA*~!NA4OD#^_#S=(eo+I;|CnA@BFIry|>ovxU9@H
z3+JbVBM)az&LPEJc61>DV6XL;5pZ?>EsgAkH;4E{5DR~U@8Ct!iz^DrmwvU@XC~P-
zA!0Ar`!uQ(jA>fie}(=ciMZT_`3$HdeFp1SkDQq<p%2$5jBbxJGLx#?(#je3fOthh
zOYl?th3^TWU%>UX_Ilr8^H7!<%<7)l&9>h=GUrwlucNzHmpGdxrPz^Ytyr)>T|CPq
zqbMr(ZW+>ab(85aJZ~;N!uLjtd&jT)^gg$sf#lNT^gA;!Bz*c$7*hh`%HPNiYl>60
zY6uU^d8R@Jnf2W^y&%G|hG-wQagkGNA}Wzq;R|Se?;Y{_PlogbPJPMT`9%kg-ffcI
ztqWCWWB!0#p&C4lt7ET(%iWHwoWQ>*$l5bARvYZOy*tWS_yLjR3EC1IVtLo9A-cWR
z9)a@g%SIA0Rk?@cz`-{>X`kWi)vdJ4Zk-2L`^*%mN<PQ0<3JUQPnk`GSUTRx{$u4J
z*lnaZ4T%6Y9Hfu(KJ)29fAf1h$6;49W&4D~f;!<s2{)q}YM$m9W38nkoMITU0+-`<
zE7x)(cHKO(%e*QRhatiEKAlJOtES;3+!{r&+zrzDvk>C86&qGzneL|_Hg4_OSlhYH
zp)pF92*1bvMd-P13^(=pX=o-A@RTQlVjQTV$@1vx=c${hOVz1hXU#cqS>OGNzD-FU
zLNT$E;qPZ>;OUL6gz(%Y3f^r#6!KSZWDE-d<FRi&jIwl<#dX6MwYcTIlYT%dJunl#
z`uP@-3AYslV3+eazT&O*JPkj=YbtrPH6KLt6w|b(71R2-mPApF^p!a8JL9ZZ+XdwP
zAWgHj)4PoOQ^8hq)(MYG(h_W}XxxhDsL`?a=yb+e32!E2yYk%OK+NUHJ9{|q<9R=Z
zS+n7ONweWac88lpl5cQ=?avUghs3*i;_-_bLha)3)Cy4W%PQXx9x8&p*i9q_l<$7L
zxn2;`f^nSDV>@4Z*Qt!1<{_l%_)-;p&;GQi{&63S8VkACRes%{f%(G}cVSGAgU@-c
zIwX0)iRos^?zlS)cjD>#*^H1#e|WCZ#6)yR@?~b#BVkyG$~_wAUjHGW-}B*%v7<C6
z2C*K^PnkMbXri36_+S&xgUbg2nHg2@G#lZ2Uu#<6GTCqBR7L3s^zwLS7NQP*Lxn!G
zU#Zr@ee3xls+3NOVwnfrd7G5X3ud<$crXRO3@f_#9`|~PY+jyo6RT<0dO+KN;rZDk
zL!Y_D@7wdW9*?_Y>i1`Rd@IBLGCG*OyMFx6w;i*#w`_5{zu$Q&H~K!Kf_P$%$$x`*
zHLZ$%7WiI<^P=}@w9$7}Svy#lE$~#y>k|s+229l2vWvzmCwX0=;*f8>Z5qP!6VQTb
zkU;af*O&Xa6OB<Eulc3BGvt7}(HUpje9=^92@Jy**uFKJVqh60ik_2lnXp{qY(~|d
zdf6QpU`a}&r6Ty&8cel##9HeWGK%{ihd4FtO*C;Hb6(*b0r%VK-=AFjEe(P$+PvV&
zn#+e#;1K)cTWJSbSqLcv7y);RaNWK!pDapciEWzwni>3}u;pd7zO0q@>`J6I$XqQ5
ztwcLeE;G|06eNm!ak3LgWEkiv&3welI`yM=t6I!FJ-{UR)z7TzuWpd+@x@x}qu@>U
zmAp1C`Q@3Lk!~7f2Oh@A@!~WDi_}9@)j#5D(p(bG{SrC6<kN>=eQU*8eMj%)HgThF
z6unv~ZWHWL47}XL-st9kTjNgMP;Z&@_<|ICl{P|N%b(4A3-t&I?GDP?zJ~JZ*YYV`
zDwb=l<o}P!?>hj8VN9Q02o7K*{A!Yr-&xVop-(;^RVDLf<x!tNeFFPzVWPIf_4$5U
z5Ye{#RoyYEEF^F>1Loo5($fOY99%?(qtj?>Tq1(@TsQ0n;uUss=M|E*bs<dTHt8(-
z(As+Qtw3@mV27lw!!z9fDHfGR#{Nf~iujNC^%RVL2OZ$AX99*u<dNv#$Dc@k$I0Ih
z|I^EW=p^9f#lNw!VMq6L@~7J%^=ayJm_tWypxvj=M2-}#R+Do{SE3`m`vWIRfN$hN
zEplmaaJVt#_QC>a0F!sTVYN8Y*wndpznM3V_~$JvgE<i1?1I5Nr&|NBIA2;~`ngwR
zht+!Wb$0kgWKcuCmgst}osHlU;H)ac!v5^(!QXbcyR=l}yL}-rBR9%m$6XEPl@{+@
z<b3fbzyP1?nsgzwqiu)s>$2;l*aly?;Jz{K6|$TKNdMF`h#&7l)LpXP5Nbip0}rJ0
zc-6;39-MUQ*dYOba#kyS;yzJmQEDO<Tw0<@O2N+l!xsvTm{>;Iihib{p@X3k+i4?h
z&+}~NT4bn#@5A1QKWV^;Sn^j!gd@r-g>s1E_f*v6a-uG!3mCsRj2NNRa5{D+%tz8j
zH~l7%WFHTEQ1J+Q^hGPh1Z7;8hKm<eqY~GEUY$L5d>T*RCUMC|_@0;1-KBmN>_={(
zMOGY;sPgfp0_BH{1BY4R15zjVIrY;4<xi{b)6MIrcp%Un#nV3^z!$K1mRa9DLYt@f
zmlH@1E0o>Io>;U7z?(o;!kr(zBC0T7qLj*gx@+4UyvYDbA7o2<ei?j7voYCBjq-|s
zqam1WNfULN6+sTwO>C}$_+5=_zvUMp<Z!x8VAA|^%@>UWqtT#oadC@O%HMLlHN+3V
zf51e+s3<LGpqx)m3bCd1BMosy0(zfJuhNr`M?tq_-$T;!QWHShL98zmmqPF6JfVt^
z;bB4?yx{ma`@Th|F}ATjSZ|5*5q>ehh)upV5a1RJd@n>CsyO3fw-YVMWS1Ro<M5O_
z08w=G6~&uRKm$CLTC0?HOqo+9Ut@4%&DbGp7+o;cK7>ifJR<%QJh#-^YmRZJ|9Hhs
z5GcD{6ZAoA&-0tV{|v$bl7sup0n!Fv!Neu6>~9)uxHub?`Qd?o&;KPd7vE3N1in4M
z7<{)M^nrTbUFP;C&Ng^HomI|Kyqo72M680YrI`VB)YXJj6^iG>FvFq&v27(jWxsmJ
zMQ~b2+}xFL8eB@T0MOz~*TNzM=)wY3<~|VIv>$pmwS1B4_40BZ5)M?cp<`oX8zp8G
z*F>F1art<BotqJ({MV2C;sKbIa@%TwHxhPs%;CJ3DhVhbSe{00ax#LD^JSg;-s|S(
z=1rbE3+HeF$hAJu;|O=T@ilPy!J5}gYnDJB^2#n3_0I^=eXT0axLUuHnq>aUC`3wz
z7dD>$m)A(@l(ShRKL`XWS}A*TAD^FXv2n6yiYEVOL?@sdy!&%F@Rxscwc_6kMqu>c
zyFSG0|F_BR-`9Bl$HKwvmDTblh|a2i!T3R`<C*Y${O=dNdJ7h3Y_qDZ)a!c5W;y?J
zy(>7EYNp;@Hv!%FSFXDk9!X?C?2anm!TUTD?(Rz%9m{K0_N!)zi8Bfgo?l9v^!>G$
z5q1ytb7RpNv}+(>u>f%pge<mLI%{gU^8oGUP#>Gop`Y+MJ$Sd5g5d}H)3nju2gb1v
zFmHgCJOq(hqe^nU6zcoV2RpjUWSp-SGIkA-(mu;uHnvUoNhyq2EHbhj5L3X`A^a!}
z8>CE=Qk|!MnJ+QpMDVoM`NF{57o?Lkv8#BUzxq;=mA>LXpYH(%N#MO5Rm3==`RoGw
z^i$Tg0t>C~svX7$Yu^1VP+CvjDf4mD0`Dg*cJcv2luyDTdfdB+#BI_#cn%3W^ES01
zNHH^`mZzTm+@v!41#3BKB+4`>ipibZNDTBZ4|~j+ESvkG{d<?yf=$ioayB%fe?_OA
zqA0Ela(AgiHiAd$SvJT!I#+kBQ&w+sl;^ZEG@-RM8%3q9tUkg+lK_iK@}fqGgM;bM
z{i-(%$KCuBknPf6!QZ_vF$<c0ug;4*ls@!8heXdbpn0RAg~g|x{lfm%t=9CWrB##{
z#7afrx>&Q?TesG6JbNjf*+6+n33ZB_ES0O1EXe<jQ|m?5Y4wGqFHOs5tCNgz>zHq6
zKDHj~-?g%9cZ<dgUF>{0W;nNw?y+OsFLsXXf@nhCQ%dCOEsbNOqa(wq#b$mx;KDqJ
ztCYC-!d}uWo>4KOOp<*|sHhQBhes>n`5?Cy{N-e$Crl(+*#@tfZL_nOrO8qb?;eAr
z@<_7uUteLRSK`BSpxqxc4-Wys+QS!*g-2AC1D{&ss&KniY$gn`CUerb<lOWU4-6>f
zJJgfUNZS#6T=jN3R09(4-!{LQO(Iu*EpL4?AuLS&%kRXaV4B)nP@|s>g?(}?{efUv
zRAMvgW@3z|UcPJtF1J&{fSdx`5gcd%`zGRvLsJ2Ipg>p}E=uvQH_0fZJv*Rgu3KX%
zZtS5-rj%{>qoI$Ui(2-IEoeJ4K;E5By-Z%a#I*W5UeGAsJbPN~%JoESgrDB>o`!(Z
zKwE#MYH8$&a?5V4!^wW4-0bu_Ki!}O3%Q{~+yCX<N)F#@&h@@lw9o)S@AcEiooNfI
zC)jU1g$TDd3sY$x#Eh)!mBFX%qWOH4=v(*FF=zPY(myRpj7I<#@PrG^THwGOpImPU
z;Ba=FY`+Erhj(->)vdU--K4jcmx#DaT*G$?)nxGD6_%WB4^xn$&wjC)R~FvFe41cu
zErKZY0pg;fa0k=H%ysMH%Vr#c8!gIn0sC#nr*DH=aC#nE-vrB9z=LhRz?&c+lmHZ1
zMR=TJo_alwx^q;;q-*S}r0A`#eBJC(z4Uu(eX<G`5}uN}FLlXE1lgS-P#{qd)dI(2
z8J!uf4n_}&!Mi<eU4GA47&6y$PLrM9m6=tZG{d@ey<r-GIg8vncO2}m<y;#X#hDQ>
z_cX}J8ORq~>ROwM;K$ba#^p1acCBgroN-ejsN>dgVft1?mPz9>4-t8-r>To}%<u$T
zM^Svj4Ng&K${R8xg4n>(rcvmeQ8Qu%UCdrqzgZ&>`1ICn4Uf^ub$`rY(6LN1qekU>
z>`2H1$8<b;XPlGEoak`^U$$E<ds{=~c?D8>bW%=Xh>LH}C)P!3HK7W+B#u%$5KXne
zm5$s6UL50o^Ef{ttzlwi+W`qqmy{oJOpJ%f{u5z&I2U+2564potK4_#j~!dwz8}p@
z6+G@x{lls`qo2ZmpBWZ|!a=U(qF`UPl!E6qJdjI0N9I!Zj-yrWn4QG|qSIIi>LyS*
z|Fg@~R6GkA+Oj(KNBktL_0;!_|5yN4{^vmC!?<J5qF)fy<ziy{uHCaKj?_G@Jr4WV
zLt)_mCN9}0@B9faFJM5gzupD5O+EWegT_3vB^h5gJ=N9Q8@T51uXwZ{pZ|wD8D#lq
zjNw`O_~#5jO!_JQ_ij5X==IZ(d%B$n{ezo9qxjdezh9nUdFxMEQ%7Z;W-j>m<@li2
zIf0EJ{yV%od%f1|#rI!4l-~X8WMA{&NPoboQpAuS%fXzGK5`3Jh)N^yH0t{{i;^vS
zX<%?Q<0SmA6Y09XqG*rc!@I(ikM;Da^&}!Vu)1envVOp8r35>^jqVEVvlY9zTmgo=
z#~O4YqK6R1ou5m|g{AoIjXH&M%bsG5axc-pBChWCD+<AQN9vl^-$AaZUVgM^i`?0q
z>Sys^FMww>*Rgs(DnuaT9qg9u%l?dP6^eB|s%2uF?RT919hm%UCv;fnOW9#vF`tUq
z$fCXK!uWF@6OR_cGwSpH<WeLrCcbhYsMYc7tXw|AOmh*!Lz~<og0owNJ%gx=Jd$Br
z12Z^$mc)qk`dnV_&x=9ZR6jz=bm^efQNMd6K~~}@^WYCX{s{hiwlx4%U_DL9lTsT2
zFs;C!1kU(5JmLqP;m|i{<C+>F<F=-+vg^wQXMk{A34v0<N4e#u_yYud7()Z2(+8rP
zrdBx-(KadT9wpH0Enc!rMq|sJchy~4v;$oEJ~Kjw^Vj59{?T#I_=ecvzL$ZPg_N7I
z$MGg6&V->Gul}CP^6$8nRu_4bW|5!7(L`{q8G4Y#CqLry*$kMhP5jcs8CwRw=(ggZ
za&^1(lJC$dYB8%(bzC3CiWu(rA$i3ecFt-}2Y=I@Po(Q)!a;X1m(7T++)c=Hfbje$
zNQVD2BeEs`=aBnvjZmzZ=+Tvxk?ZU0!IeDgKs^AJTqd^0G7i+KffCH>Oo3sG1(j(o
zn2?4vPqnNj>C&Tt`1bjm%K*kuT1tv`t6IC8(=Ag+j22)Rw=K>xNhf!)+9#RqvavET
zQv6Atj(SY8!n-~E+N}#x9UCZD<_YSLjfID$t%d`*5)ebqT}F0YiKfy@hCft0WnEL=
zNhH(_Pbs($V}dlU2aE2{km*w9!e{j6xo#3wVLUF+E%{62;=+tR^)%*06e1>N6D_1m
zpR<HWV)H#FD^SSTW>$=!*zIqM(`z?iIEXKX-XU)$N!|&myx`%Oc{a!ffj~4`+!KM`
zwm`N2df7Nmv#x6Nrm#goW;~O-ym<A|Gj$4fDc(v@=V*~vGR7)-M2jm?>l-dL&Y`&<
zRCCPlrkaBz`9x+S?Mlx{p&Ih2!?VwuI6F85lXRXwXVj1=5Yx70=`qDsXNQfYB$Z6f
z_3KXg>7DiB76Ju{%?l}i0IOoaS@qqr-w404jm{TsObh_-Svw~I3Hrq8m>3iQT&af4
z&n-h7%~sr!qx@fVf1x}Km)*j|qDM4)*B&q;_o6v-HPJ9nf`4trrFp=GJQzQE3k9D1
zt!O#nxgWgq)xV}XvXi(?`eLn~tdNt7@uumsT1)3z+v{}#He+Yd*t%wFYBIvvkUZ?L
z3amTo$DJh5>0p887L<^|-8#f5wSh8iT9qB5h##I>`o0^jmgOKXI!Wz9$GSsqd2`GG
zL^FK)<r?iGeS!ShxYTr`am}J>;Y^=`*s!Xxql0BaZXsqs8WPbQ&&_ZsAK}hE={{<=
zN7uxPMFEem5d{7S!+I($)*<CCh{jMR-SE9xH|)7-C+^voIfLQ~nK+@WRwot(_og{J
z{$Rod5M8qflk{$gWGc@=xkKJMoudZVUR6Kwk4iyqG3j^Xx8G}gb)5NqL9(TX-A7%1
zL$5jf3kPyg&NR4PMtZw9GpiB4b0MnFP;2b$GvQXP33xtCBcxip#vWnL^N;>7@>A2|
zklk4vt;JBUoVy|I4Qy7JOR>Dumxwrq(7=`m;44#0OGRMs2K$nhf_wD`#Qp{3<1Z(=
zUAJNC`HqFHS2V?fGtd1)DW|{%<Mr^dnNIMS#yGZD>M}Ii>a8qw>yle=#MBVcL=FbZ
z8H$oUMc3eA>gMn4?16SE3d_S@f_=(f%Y3<YAL7y#%B?sHKcnWgvb$VlJ!sQN$6BQ;
zQjHJwl$J?caJ#xlWAbVq_+3#qi|pG>ITRC9YoBF|m%8UKu2$JLHz?O@dMMRvEfwO-
zM9L*y%h!8Tr~FQb8+Z7k_a<lXdT||^Wn$;X9rAc&XxQCDdt3zTk8@k+CXIoVSUyX3
zZtsR`v;d=dq5w}QWWHWYLm*YqLvd`^$t)Ro`^cZTnx&M*v1Y1Bo98r*f?n3v^0VVV
zs$3_qS-CKcH&nCp<g)aUtEfeEL(#i8<joC3eG0VHXv{Ip8$}x~$wMp5c5|Ei7+M9+
z!PxB_IG-<N^>dWmzZ~jLj~RIE`iW#6h^r=t8Y5pAe=Tg!+}t5nY<cm5-}6xArweTE
zSS6EtU95jCorLd78)M~VfR>A&-+<48)sLLk*0i-XU8N$GHR7l2yYzFERq=*Ry3-2Z
zu($Tlbm1myY=EiAim3^gPsR@}gupvf$BiNur7Utj>1H7xtzTS{H&h0EhW@tPn*S5~
zgSHuR*uL>c8fisMeYD>~=)&9htgHYSH&6d4mM(j7)tKY_>@HZGE!N%i<8`#$+UbI4
z`gufOi#{4AP}9RZ$)Ns%?BQ8SJ*VG-Ua^mNz#Q;SB73cz{E}^jRrE;*J>(5hee|~1
z?tE{IZd|+;2E!9d7a1@BU~cF1T@-#UD5OmmF;#G-)MNwO+t|r*;RbcCH#EN91)h2m
zlj`Y4>S+bITmK!MKbF^*PKM3}n=V%`k1wk>TQv7j6hf#oLS69%>owI7b0=qY$WX%a
z2{Eap!!chX`FWiIUElY-=%N`%vyuvv9m{kX7#1+L89_+Sp}+Hf&3R_<IuCP6ut!AW
z9BF(ct+_rYWR9^WjA#~yj!^cQK?$pnSPzE1NBk1EneG{0azI7qJ}K1=ys3&u$kD>p
z*)}n1RN8A1ay6Ve9KQL74!1pqvGkT5qy=JTC9SnoLqTo?fB<U$&n5tw--zVDX9D=*
zK)R=;FUCI1U8bq9Vg%88aCXK&v!YGxK<%tjsPZ>ZX9Ib!p==n4I$&rfjp|XM{_u-9
z(<->4bg=w5O&w=&#V8BsR8mCJE(^Z$<<tL&SZG;arWh)(nPWJ4(35!fQufqcUEsJQ
zHIx!vT7uSejN{Z5vUJj9kG*9bBJW{D(IvhNRyRJin1Qps)OhD6M%M`a@y8T6#5-&l
zP|!6ALWglD3E_=%4!4`qB27W7{l0PGdQ`kW8tekOD!kx0ON%qNoUw~iDkuezrVPsu
z>BHrnoKcWlPnS1<<VMe)473^@om^k;Psorns^;?T>q^tdQ$GoMihF-v^*DHn7pD5>
zV=4|Q>|R%{81m-mVUSl=z|`sV;faPKg4YH718~Ybed@@m9<H7;SRb{7$FMh+n?15-
zQ3|e}`>(&UReBJRywluQqDQfFr1H(4ej}RWOt13<m+;m=U9dm6XUcZ_8A&tN<!eB7
z8DVf26a<~!fXgADsB~CkmR1?Yo-L@eQF+o+1JgJxdv{Vf`tBqA5dkNrN!zEK6cQ@$
z!C>~h?-xC19zD+Ka(dFM?Gx-csU4o@t6c9*&`rx}xlY)V`8YL-LkMJXOUw=l-M!`y
zEO7d8Et^LunKR5od7#r;z%f?>`Tk>IeXU)EEdH;UJK0&jOV}_-4$aI6z9{=2$;Afp
z*n`fVkQ!&De`YJH+tE+Jn}NIj@rTTZ?>P_0?3dTOCMe7N#M5;@+S5}=y|jJLo&YPi
z+%P}3jgpX<vA8LQ713Mn@mz>-I`&XSz=W31tiW&spBfjzWU)G{Dj%sTa6!j}5DR<t
zU^vf5KNN|jjqT6(Z12d1eQ?B(F*Eyj9?8q&%gG0e?9>KDh!4{SQU2a*sIZ#O=zA&l
z6A1gy5pCbSsrt|EflcrK!9DOV92n927YAyf3~2~o-fgI8OOt8D#anKM?I|=kxxtVd
z#c<b~erI`jLV({Bng@0AFQ{wf&3ECj6iamxvX*He6YcVGoQO@`;c^-=WQL$0MX8^`
z@|736qD9yql}h-ORCht2!G~&cb9hC3oY56!gqGbbOlMd^7m3fngUUAZeYkzdF)6q?
z&SG@rE7=NKs*K~T0&oaZURfiB;6AQ?ooYLr00x3{;{@bmY!%c7%Z_9zHFNYO^sBQU
zk|~sRJ2oZ8h@ak~(u}xy&1S~hktbMc;&1Kgy7b`@<Y)b)^4{By=DWBR%HtS;R%|jf
zL1xi%D0}QdelO{0c1Z=P1NgZDWO<qY<R-Z8%NMJHEO{BNSRXRI-K_hOR${M7m#=P}
z_l#m<|6uUbuMFe1Q>vU|Wr`2QVovgxUqLtL7q1HrU60<SKh(fk@ASEH#T~L+>F|Ev
zicixeOu<_Zs%11a7we-B*k=|lHD{IK39y~qIP{xNgkPEP#ItvNMXOrzwH)ZBN~`*;
z?Uat;w1E9s*=@${foeARir;FbMM@)CX=J91=}XPL#JJ?X4+V-*@52j4E!{P}-%bRu
zXLfykBG3e!_7l8d0w<5;0;et>X@~vVoNbMLM+9bv;ZLOzs8%E{d|U7VbAJnmNp2lM
zm<jH>C<rkTNRk!HJ>jY$&U9kae7=^ILiUcRpque`u5NC9E|+sBMBcZ*dWtma<*#1l
zc|L)%#*;=)?PAQYu-jxW&R{{WxBc(h^HMYc`_%an3J>b~hUaMRdDk4N4W=KXvc%V$
z-oG&V<Ep5p*ZTUpN~^ArGYTNnF#iK8hr_zrsQ!f|mjC$2e+5JT8=*MwBIZi0pmkxf
zZyhmv(dfp3s@EB(Z0ufI%VKp1+ta-`iv~zYZ|h8~ZFoJe7Zbfcwyy!52-r;{)kthj
zy`K)|RqLsTtIaxXNyfrwq1ya!-kgzwA`474*8HOQ9xTT+NS;&oRTCmN)7!lc{G_&y
zwsw4z?HW2iM%+Ya&hj4Y9k2?T@w>$4lskSpl%%fPl>}~d7#nS+S)1$)dNyHAO!tip
z;_X)ETHk>}D*(QNYnjzgLvxn)ViwLJGgnum;pAo~nad>#JvP=q?Fp*fjA!Rbu3Ify
zldNdW6L)-AZLi1DsoDFZOj*oVed`Lw4v5VG*dv*mPc83x$)0Ec5eEJ1A5mrtuxule
zDci7rDdN<gQs;0bX*)JkiOWyo4q&|>GGh*#i!_iIU+4?^^{IGEFG_T>{u4|&+w3xb
zLMbvM2Up|M-8NfZ8Ye$#YhU^DO)JTLRJ=t%*)hLrMojY>UTuSFrKWrV4jmx&c%FXu
zfz<;)D`ozQDEG%3A10EQcch<x=U!^!Cu;!JTuSL|gj+B!bM2%BwoUC@?>4B<a`oK?
zuyo4_6u1R|nXQZ?)ob(!C^2_RNM^hA;1C|E)f!~55{mX8rdh3NU?Q4Xw?(y}4Y%+M
z7<$j0Svpm<x{yF8&_cUuWl#ZoYl7<_r`2p@>`bv6XFbk{(7{%rkmyi~qd$qKf!?Z1
zl?aVElgNHSSfHWL^=dGV+wy|TWxtp6V6dbF`OLn0{Uo}bG>KxRek^xG!$TO-URUaT
zn<N!$JX-Xap0ZwiTUk+Jrwz(JL>OXcGyi`BP72P9j5_adLwsEg*{vApE#lgkG&I?s
z*!Dg*Q8seyVyV$d)!7E6H7@1wzI0^x(|;?1qYvH>`}3@}wst~XFQ9(?`n9>aZCfm9
zZYbQ-eX{<S>&e3{<Npw=tkOn_7zk_&*eYXgUihAkEywOcackvNTXxyX2L3B5&9u{^
zM727kS=$u3ft=AoINwt_c9=I0qnSI8qcfs$#Z~QQ{x-WEQ=6LoT6T}U|0e(i_&`#n
z<LdTGds9vIg2@NTr;ZPZ5-qo9mz-Ug70*+G90pXgT?HV`eTBpqk7}R-dzfO0NGm>1
zKgEKTu>re+-90siCgnXk`GCR_bBOHCchp$VEdc!F#vV*H_aE(7_FwF4b=8H1%>Rt<
zNB+E*j2&};zQTKz_$jw#T3|Tuz@uO=u0&|dMXi3#oM#xZ-+>=8l*7jj5cGI{KyA#7
zw^H_QiqaI0-sJGb`m<A{{lXy8R)vkv1uNgkhId^>Op8Ac=#k`V&mrHsxkY!xk!MdO
zwd9TEbKs9qi-h74ngd|!rFk+VQT!7s`Ft_1rkbU)z&c*dZW^N~#D65^c>|m2)%v2l
zJ-s3TkGXms#sIZ4jQl`|ffi3wrhnan$2^apR0A379sgtAsffV!Fa4gk!!#VBsiA9s
zl>s>a#A^QYei~X0tVKssR)Bv%LPdQNo*rLZ!5_M1N7<Vb2Ea7mU4l^V-@?6HSFBgD
zO~uQu@`zbqRM9x%Z&N7$s-<(ZIqaQHYiA_vDpbZkmcskv<;U{}sdp`CxANvTpZSG=
zUw<FZw=Tw#z-kt&o`8^-VD^kT<_A@dWPhf;3>_VP8)c~=yaRRsM{~6jCXUbriGc`p
z=VuUSottYQSZ2n?{H+ZY>UB<IQK1UTB3WeJ`scEn5~c!DUp8H^?%*xzk2}0L&Lj?V
zWG8}EQ;Tmr8rzcbZ7?EE*JIE+B)OdZu4$H;)77u?f5efGFO4Ip;v#*%Kv&3Ie#W|R
z`+10x;$4JGIXwJVrm(Mx6bL^oTY`m_dZH@IQf9>G$J&n_xIEOEhd)Wv|2WReE;+=t
za53v^G~vV)y1;FwW92er-v>jvDMi9oMSL_(#U0K!wCFzff5cHOumiO8jGlMq;?u$@
z2F*1JcCjMd)-WD;=`O9Z8#*k8>^~K1v#~WXb`n9FhrWwLMD-yCs`vwYBmU(M=WRqL
zdtY9Y*cIb)apj~RG#)O?AtnJ7ehj7}v3MW<$Ig9yi<Oo%_DAQ4`xwgiV;#NSCu5h`
z%aUH-;4XNLXCD&EPhVcqBF>BV_V&JTL~AogD$UVzdvmv(+~VQ7y@>N3b-61RkuDxf
zu7RIzFb1`pWBpZ0bkf{P`CxtU=gO<b{lyP}R08T^1U!NZ_V7jc;#YXw4WMQ3qwlwD
z2Un*(oeK$0`h6IlI5{bqtYixn-w=_Ua447yvUcacVoAD?(0xt5l5ZBvHy5f<Dkp0U
z@P1!sX9$bn@7hXyjw__5VmaVfY!<Wso-Rv^e>hqBOVrN7Gh6IS3*kf?`TvDWnM(T9
zbA>C~xREWQa))}3+iVV5;BB#|F_OK#5s<2{o7fMhA9HXp8?x^}LOun<C=*q=V@U43
znMx6wjD#@vx>#b;^)%`n#fU)b<<SQlI=RF&blRC*$6gWj%-f)4k$!_oOWqs|Wj%Ce
z@|*6LB)i`hiDLL{%elsWI9I{LA@~s=APWl)nr<3rvLe)yY!1V1254vrur?E)pK!eR
zx2$t=IC17yA@Ez%xjH_=?{buFreMVQUpboJciBP1Y$@YTya@j<`<a_FFE<J}>}W-D
zTfQ%faRJIsz5tBT`=jUL2nJeBf>TBOsbKDIn9$dD&rnla+D|Ypv*2eU1(QAo8;*Im
zuATR-nHg?=>9<kT2d~~uXc>2Y95y-d5KTnTG~YzHKKICNf|p<o@gzR`kcJ0&Q^dhQ
zNO;(Sqmy@#B93D(EB2mKO|)4q8QV?^x<8Q>1L84jPGJRTWN2b<JzkHqTsXCs_lSwv
zojlx>b-zP~J0iHb!%N<A%1mDQX8Tb8r4o;eOHv}s<_v8vlJAJ(S+hwTZ}R2qpK!L*
z7h~gV=~hbxeW$p6dSTgtL~1G`Sopme0a$vxw;@V1d~Eyywip#z&hiQXiCt>nDlF7_
zu!cMMi^`-91`goF{wP&R&*1%VtN7p2mzI&JM9?5)5I!e<@{#B}h`hnHT&6%Jqp&3_
z4gVaB4|OmgS>6KXhJw&-?fVL2u@|X?*m<pQ+L~;?tym%MxwU<}R?%cZ>hvjXym<ki
zktu7jPio@FFMgwfwdtRShH#R_%Js2oLhNOEuE>%&7g~F1C~*~os;jJ?A?ulBRKcW%
z7e8F%zZZ+K>oxBtRnH5E&nf?KHJ%CTpVqnPON{Y5GGWg7mRJi^G+;OzxA;s*4}Y{u
zKOAK2;x=Qtp-(yO_lYcle*ZD=0pCN*LuOhIt}vyO!+kBCzth5|was<Z5`EiZTrFug
zwJwc9b6>cfH^7cpUFWrnip%m{CsuJXbGyaCJf#fkfaLV*N5Xf~2%p%~&hux8KXkdE
z092)rdHSycUkJZdgldICrOaLrAdo3qM!4RdZ#qVX8dyOv6zbXAWTbgBkH8Picb@v;
zuio^$gV<*un29gB3Sn|2Drp-V1)=i7#K{2SJJ%~S-iGK3ijaZa(s)_J8FL2ci&|D^
zZkg$ca)VY7%)uCK>pjxJ?j)LWlb^Ucw}d0-=d?GjnuRXwoH1EM#tc=%JlCn06X}(~
zI>0tXA)%q=27Rxg;;daFm776-RmrAD5MX#(U$1g)@ehTrno^3=lE>x&F$(>mL9wGV
zDFyl6u4smahIlS#^>w*kbPTSk&JQC@;JnH~9Eq#UM)xm5qbIF#g}XK!or`eOPLsRr
z0@Cv>8{q*o_8|4%-*VmsbKX&pIcLQsu0;}IW~E!;Ks^1}!h+;*<%s{)i^*_)BVeQD
zhH!z4P6#wWm#+-nm=%-IGExX8Fa$<3vM-{F9v%L6S39k`pb-OXOg6(Xk#`~G*LwN4
zgJOUH@UyyvUR3lM`~EH9eCfGG#_zBLY#6Ek^rF$u{)b7;g!CUKwSOus|I`EjVX!Vw
zPyPznIl6m#gtR#n|0;bq>Ijt~0Q;TqpQPaITY*OYS3f~<N3Uk&N4pbB5Cu?;PVGXy
z`^_l}_sbP|E8M{ASnz}|x-kfODF?4XN(yQ48xr+=7_fyT)BJ80NL%-$a62~zz_n~*
zC3m=ymC;FO87SM&NIjOB8&btCatkYEkCrI;`deO;m)Kokq}hoOVDmB4Jh}@>8eeWy
zC)7xL8chqN@p>$To5y5l;ZFQ2q1xQ<Fv{;?%R}cy-WUhOh@Hv4=;XX2mis4=0$d6a
z&DTyFo-jAt;}2+9&rpY}p(>>SKOTUnjQ%+9IEq+)o97HqWB1;OOaFAFbj9tYi2b}k
zU*@8<&%&Vc$U)U6MdES6<Jsj_W&6EST%p2PqQDKC)FY^bbVqAX49Z?kkw|jSrPv7;
zzYAgh`2hf=;w;!QlhY^|v?TjG96epNTH|Zj{AnIHd?<|SLvhZ_0c#ZiE^Iio8v*0d
zy&W7p+_j>hptH5c!!9Y*5_ZAf0?yH7%N55v<X0ika<jx4zjLoi2_Mw7EHqb<cIE-u
zSkI2uGuQaDKQ@>td=|0)Zwe<SE#<1r^yxDA!wK+%<mRy+6$E&Y=B&i^jiJ;#)x+5^
z^F6vC&<A6U(@oXzVj)cH@{MuvT*U{H6y$kMTk1%q6n19(<q;i%Nt#BG8%VeEdDz;1
zMSdsGjPx>d1yj9=rBBXq$8`0}nR?exaF!eqX{`=EJ4srGZhn`j=gKoBk<m%`<AP9u
z;o%e%?D6KBYBZ!=Z2tc+EPjE!Dro9>CaNj`?CE&tp{A}L3-BQ+g{9PwagRQyj=tO^
z?keRi8%hOB`(TKK`6#QmE~&&tV}bp#VD|2kU>C0cB4m1^e`cBixdc!!+5;q7<;Vdj
zaPiIDHFi0(QlfNw2zgy&C#LpHTlRD+t#*W}B;wuf=OXoC>M59gu#^@g?{OwbAsgRB
zG06Vx7fq2oBupz+K}2T8b*)*#I+9DnLGvJ>f5tuUoCW|t9K{-?$BI_UX^JJvSJd4O
z29xA~l_*_erzu?{?fSTW#DMFf8qQW!6RcoZV^2suGSQE$y#dmaExFsXX1kcnSE-$(
z{Osu3f&%Cs;HrXdv@FLGZ0rbQS#3p@+Z4Q9d5AO_NOXoq!aQ3I3x*TYZ#BBIkc3gg
zLRGwcE)pu;HRr5EWj4910)Qzwm5r~z>LvkHl3ox$-H+EbUlIwplCp~MZcVL${d>_M
zcOr}^aIw6T3Hg}BByI@~EULt=x?{Vp@>qcWA_T!*Gb^LKxe<$Hc-9e4%#8)oYBxQ%
zKq9uvUb74yUu9<;EP`Yx19VfxY7Krqz&sc`J11b$C|!#^(Lpq^KhzI62zHlhpL|QU
z-vrM7dI6XyUzPTEMV(DjW*=o?>~yc2Jwm4jy*qzqPT_<Cm`9jN$MJodfezJg6a_M`
ztN|C(XuO(l)-CRMkb>e2&QHMjXVY2JyS2N!)|Q%<K_<oYKb1@F6;~Iu;S|8Xp^?YU
z2^wf$nh<jLYVcP__m*^NV_!1-ua--`*KYW)mP_WEiE=)%&%7KYAt16r9&r-OGfjeh
zzBnL@Xp$ZeLvbvf*jxAxGP1RO;~+-oU^t>Ppdh*Y->R3aAm1lDs80D_NIUO>zdHdn
zbmxZ$pGDR9CbfaCc{h_|B;-Jkc+={#*N|*rUPaoAEu=UL{!%~^J_ah$d}wrx308B(
z@G4Agwh3-0qbTq8qr`(Co)WSC-6f>yrpI?PS2*$;mMEKlcunhx)~zH<9+<Q9vL_@~
zxz1R1RQiLcsA!oll$O3TE9L^?+|~B^(FF5OcZACceTjxieO2H%q+B=0J+SLlxJ}C9
z;6s?<u;!2N%`1vKSnENEfi(WF)OnY3xOJ93Hq6Y9g`S~U{Ii%+GUxwbG35$YL?cid
z)&Dj6wWJzLSd*3)I}gK>$PIqh=d?L-P}J4kbRy3}HBjNF+_x+@U<LKs^nzPIJ&PGg
z+p)#6?BE=F+7qc7Il;yTY6Nws>04PU5~%sd1esxnX@bM%g(jTKu;#h6FVTb?RB0Or
zvL*>rg*r={fX;{hzBq*91b}zW*s4gkCzz^AN$c$zJ42LvM$`a8tL{kLRy)Fnc^nTG
zCC#0om1ZKp$@#DPUSHLLX;elsLiVLc@E=y;wA7Vo*r?xr5-01GCqD_V_kZ$}bS7)|
z>*PBGMpC6B;!w8sCx|*fx#xyiX&c@Te$4v~>GQG#AqYKXOg-}U1wDn#*lYt+P${A7
zC&!r8#L8WX`czyOQ=5F_Pii<U9%HmA#YYK}vvuSGBgilJ6(%B+m;`j^lXGPM+%jU`
zx-g`9VKv%dlzZe{85Ct2QuxpQg?Mi8D`}Tp_@cI=>R{&nPrGWQGHtnF?k!%~0^Rew
z1nk?%T~ysJm%Uw`t1c)ZP4K6VbYHHhzOncFq3cZvfao<`l(7r$)ZzxgaJu<(yxxnB
z$jNWL3+3hG`tEx*62MAt%J?sXT<ag>-w15-)aaP1Jf2@tKyTQ&DgUcok;=ohs(2Cr
zdzOU9p};#FC`rYQe8d7S==o>|b;Yaz3U3wK!I*JX!)5smkj_RBc13>&T(K_Mckl4<
zv`|NM!ej(H2^j@?x}j_yW65pjFgPd&(;=@wE`G4>J{saLj!3RQl0VuI7Sg=TH#y^T
zA>oW9b@BM2wg{!7trc|Pu>!qoO{GjsR#EpadrrI%^ItC=Lj0B?e!c!}%<<Clmv6XR
z(pKNeLm&T-7h%eJ<g{_0VJ17;irT^$8v9c%%LVs2?Zqf#h1erU2!6O&**)8Ls|b6q
zazxO*h6oSAHS)cKo%Xd#Vl<APo)^CQ3E|n3r+9hHnS3z!EV{R||7@AK=06o<!inuu
zf+TbrZhpu#KcG<$+X-M=<Ep`gDia<)f_w*~Qoegg@RtOi7MuA_PUKS523wHX9ponT
zH$!{c?!U71x=Ne7!gmXK!AZZDOQWeQxj&6x(AHOV%f+nVzKdJt9%Onj^-cC*EG33l
zSA)-^`73p!mK#D|YjOL6OGRqfySq?)m2jgQU_VE(T>rXtx=-`-auov^w@70LYD4HF
z83Y#`ZW#{S?N?QRt6lT?|K8P}J9Wltft3E+*WR%D%@?=gHzisJII<9K=Nh$5t*o}o
zdx#Jx1ksF8M#deb6S@N#HJxjc=~5UC{A_Uld&QkgLK-dh-+--fZBsqin#9sQ@0n<n
zOC~*@7Z4w#&R)^qTMa9l!|Hl-B6PS5CV#D8^JFwLjt5pwk4Bgg-|IT!b1Dze<qHjw
zH-oXSb@>1p;2mIzW_tg+PHrZxu8w2cdKN9!m*mA6(my<g`t3v-SpTpP{=NJEU_SgS
zxwMDFZjn9Nk@@(DZaebF*@eryq+IR>S0oY)HHtX^6)?F2KgN&ol_p}xy}_t7Xf-L%
z@5XzUxPzr4J^6X#((u+meD0jwb)4;hz|T_mG@@jVVDnwJsZmQaJsoKS*)c~z!w(-5
zB@VrN>8s7ke`GJ$Qq5_*G!mGcvY3qOz=(~rca}PK!BfDmnc`;#7X2l68OsPtDa^f2
zrdh8$NmRYr!B^v|Brff$ayVtKgu6Z5TkzD#q9q(k2y>Epqm0ca$y+0svTKSH|73p8
zl}_ChHq2go0xZC#pxTcDynXn+?D3ZYWoXVeC9r_)^6m~mSMPysvGsuMk94z50jV{b
zF1qv+s<`dj8lkZP9KQS?^N!v_XY#o&?YBMR&{og(3U_|dEN8_|Bujmd;#H7eYQ))7
zuqWialHh-q9$av$A~9m#OfyR<JxZ?f1gZ=wUv1s<xcpLk>ZLGocH3N4&7^TdjLpL|
zVEHpW9H<6=GG=z(%zJ=4cG8$R7~4u;zU*+@I29v?-ffYUTIn{)y=?BGG7Z#03hekJ
z`&FbXy>`DQ@EfIhnX2kvaE|_vGSirBTv5lQdYreM&KY!+&cM^w2*%HxWf-)xtdsPC
z*=oV`#cLP<4EyMHmYt2Uo!C2NoSnhICR3;9o1YS_<EU-AsaM?uxK=`<jg^k5;c?zB
zV1wDuCPi+lh8y-u=bg{pW10NxvQ@=W7AFhf-O$5M$-_&6)j@IaKLhgMm)KrKSXC_Z
zR&GhFH%q5-*$#767MgeRXclwa8=UlR_c>Xe?{O#-%Xu*#)1*EaDQw}g9PpduIu||^
z6&)-pYf!m;I13&64ubZWx9R55VN)$UjNTgE1rrWo5L;g^j_CC`YDikkg`jh7L{W^m
zS`ZdE2NT9ia}}S$?^ytK`U>Bw_L~ytPCVf@(zWJ77lIL!-q|22$4e1z*E^o3Muzr@
zn4_~eVAYf;0@gwO_a>d*_xHu<JS*qa3p7<;>8%Zpp~;JjK-sK-yE_kOeZ9uPxEk(Q
zIa5;|{^O^sq}FZ*Gq^tnJI4NJnJq2L;EDP90)QLY)9_HdsRfD!YVjAL*E)kP2y5k!
zl)S*oYQ7~1NceIED~j26s6d{NLOsm5d0d;S)KcSVYSRU%8GN5_L*GBa0vU~o!gH(?
zMORj9E3_T1+@_RApQzG@CJ$+lAFUwZv$9pnCzAEDKHV3Cv??W2F9{UN|40hg){WhF
zIzLUOc5RqO1Opba$+dz5HU7^n+d|peHvkYa-Ews(wQamDdNO)3m5^q!{mQn5QRLcz
z-ManhG~z)%kzdLh!MyL9MqQbGAnTH<WOa=f4%okzWV|Z#N2I$NvPFUNhBF61198n&
zWcA0Xwrj5Y!sB~ekU0jP<N)6O?KkEIEjcxssS2FlJ!MDVhvnrTW!d7U7dZ@O_SD1O
zl@fAUqCI#3u<F302<gcN5DKhfbCuFG<+Nj{{gD)&;nb*%TeP;P_wEXV4~Q}EFr)iZ
z@io+3XHiz)?I>cf=uC74pvB<rkGxR!73mw{FK~Z_=6OF`9(eXrI?Gu9+J*+%T0I7?
z)mzu-2(=k-;Kjq%E`5ArAv!xdfpV}8{tkc|(#j`WQtMe00Jr?M5DR{DR-^~&D@3U*
ziL*J_v3{3W&}Lw(aQ`l`NXCs^Mmv<UH%85W_PW_i;Tyx(mPtH<sgVe!<bsB768fhC
zvl<b4F)>v5%X_4Era^mA6_iT~lEH|K{pO>BOI>fIt&0_5yJdM_s7CfPDr%q{rxLB-
zO#p6+7aILAEqn9_)NwHE;$$Tpm=-POA^W}QQnPM2l@Bm`E&hhX<il#6gz(OI>Tglw
zfu;XMV2sey9tx+|G^}FTI4RG;QkXhF`<s2O{RbsD7)8j>n7mqnp&C9#=Kd%4`LG>Y
zH-uDoTl|@65_nzqsvKzFu;UQrXkX@*BE2#l!W<oqn9psNPvZuso~m|AuO{6k>mh0_
z87e&&dduCGs%QXBM<)N>*x$|!H;JivjbAiwf6?Q;)Gnw`Es9l+Lbdd0Qp0d;ctIcy
z&SaTkk^@I-4*A!l<(>$3my3}G_8o~+r5HV_>U6j5ZKbXSA#WMH8dv)BUJY~qurR9K
zTlz+)$&PJS4pc*tDKYs|)bX-goP}EO&JaHl8NbL~gSU*d?RGbwV^hKh^R0KUJ<s0u
zVN=&9pWt6ws1hZ&{f`REi`u4+4&DFX{q<jTf4x5asOSC?@AwPb8_SSCMKWNma1|ro
zj&lCD)W6w!<JWB)Bt`1A!>d6+dzki0nGtZ`v%X-;TfN1`T5F62!Hd*3S)j(SpV-8-
zRdT7nOLv&wuxyg+XD`MGEK-cN;V06kiWvJHJS<IIBeKu`u1or=Kbrs`k*STTM8u>m
zvoFJ<Am^)~`F}Ttu_fLEY*e|=kQ4f<(8&Nj@E4q7<%4qz7zdR_Vi1AH`4+BIVK$sd
zLk!OHEnS=l{Dc1lML=O}Bn)+@Vra83SjH^{nUpiD_zKsM+`c%HVs+vw<&=nWit*NC
z5YfZzJfLj7$=<%~PVY-bi4!@n1^0isC(PYWCtf@YPL<sz*xV)+P*I-R!t6X7Hrw5$
zr3&pB5;73Gtx9`wo~e3iGcSpT->W(!CK=${;7~|DU`!Uw8TZ!JL>E)lf+T=QE%0KO
zSxxh9_MV_$t~YF5TneB_o5mrpk&#OSSQ6r&h<3zjn1h?CWo#EC&LW1{!sZ}G@32tl
z)dw1>%?c_T*#yBm(kNi93xKVUda6^8{PeKKrKAkhglv193fR6CJH7o_T?bv((u8qv
z*8(%TB45Jy(E!vL)WvZ{Ax$4(ME~{<0xDvlcuhYrz-HTXPPNq742{m*pR&jzh4-kR
z12|C!wT#MuYA5x4)<-WlFdoKKBE$W|yzo01qgj&iK)er}zoHoRQw(UG%`yDR+=7(|
z1_%S<npiXL8Kb}DN2#>0=AVJmCWKcis%qqV;Riku?!mps(tX*h?$d6dcV*re6B%qm
zJ_{zXx8oJo%}h9S;47Ls-#@Na(ylY(Z!MtT-MRI87|DJdv7~eTB2qI&tNQ_Aj~I2`
zxxZO~$us0WlZ-QI1CVy8)Q6@OR8>lym5ZkpG#rCj3>}KgELfFFll$goWvX?)tGTEL
z9o<VCZqaa*Sc};58t(Lz9hP)$B6A(3z>;dhSdffeBKrolcsNvfD;_gxXgQ5)>$3GS
zlmX_Dubcr$V$8e>H+{dxb6(PG566bLk^QPV530DuEKaU$*8qeasl=_WLQu9;1PN@n
z`p4VJ1JsQBO~e99L!0M#rCc2Rdxi?Xh=%yG!81}WCG#Y)cU9oJk61~ztv^4PaQyY0
z8UFR05tkZR(wWvbQ;TV9c%&qj6D&Q;*4FE{OtY;{ofn!r>)x7}k<6<&sE+?~JAlkL
z91SUSwp~8%a;08i;O*opz}xyRpAf=|z8T|t#Pfuf&KO8jq9_2gM84rMFXKB*<pUJ|
zGIDYP+Kyz$>$T-(H!?Dl3wLCEm^v+WK_ignNk=>F+!H4rsQD?13>I?8TT)e3bu!32
z@|9B%ZII}6+EEKYM>;tX;mKW6xD`G&xV1CPgABgM{yeu&t#86O%40e~bV7^03~_rF
z?e=WTBk7Jx)3)wIScgfPAB@A%ER{>gS5{<}OT7q}NoDyTlVfR=_xYYpg?o>1TtheU
zgBz=#35H0$evSysB%VM_1fS(-_4Fon>?eGI@jZfRnGC~(-i?5aqwrQGuu8MJf9Tk%
zy9Wk!|2F#nYVR$hs_MJ8Z>5nEknWaF=@9AeZYh!85*rW@knUC_lx`5DySrN&_NKd2
zp0zp8>wK>39?x^X_lI|k_si=S8OUNwto5IB9>?#PF)^+6=er%>C{Ym5&^hIvqaeWf
zy`}{I^6A{`vi$2!;62|XztH~kPjF(LwBoRjdHm2TDtgqvuYvPx#rn_BF-dFo?p;VQ
z7DW&yzY&tm&jd`>CC*zt)D6OD_~e2hp$S0Hp;Y`_is&EM7lNw*G)jnQ@MgX!Rxw{v
zX!+BWfB*h%1<pJogXVO=+-6oz>(!H^=7m3cCI^oid%v&U*tqTS54j@Q2-h-SX`ON_
z^FixaV|RVDA&qTmfvU)-ezBqyL`Ll2ZziVn@-G}96qY8axGEYVN%Sh6KC9dfC+s}G
zTsDe}yApP92TT1#-EZ8q-zVf$B5GJ(@~EGivX}ITX0D6Cua85^nlpD61Vjt-Iqo#m
zgk<KYiqw)Vz8}Z<eLS`CMS1JTuxaqA&I<T)tE+j9KORc~J6cJ#Q+FBMkGr&&nI|MF
zs8MgnU`#U4_+ATQ!<~8_Vq3kn9OA)~uSNJre>F-HK?&Rj>LT?(qqHRmB2V-|c+F=$
zu+CVQ4Y<u{#Qb>M1V3m`$4TIi=OCV}RYC0blC3t*y@+g+K>HNrV4o-$6Vu=WBCzHb
zTbp&;Z+5>fDkCH!->6Cx>Hs>WuJ9-I?3xoboPzoHM|xo-)P(Svk9Mc$4V{z9{sf1T
zRA*8Cjr~r%r!)`4MRuEC<&ilOmvoxMKfv8$mwC1B`8d4@&-g~lyTIDE3ObAu#}I+?
zQAgu~?IZ;HSI<ApEv{Zmg;$8|8=#(+ssrhF016YbY2+NIGxb|w@TGvUZ%bmHL%+4l
zl#=g^^z;Xz_y~|rWt4-HOc8nNP;LJ=OCtl)+5z}GiAi7rgA%e?<nB?mU|%*P2aqrV
z)(!v(liTArpn{~wE3kMymRD3Dn)%0cK?m~3N+*Zt*N)g)XIxPW)2>Z@S5?L4tL53^
zZGzD}44pxE?f9zx1Lk09-$T1L^*QQIot;kOf}EO{-Kc5Ka>|wDUU~g3lc6#U>Dt!;
zF1%|Huoey`OwQS`=D^%ZT&!&ZmnS1V$~lXW@@8Qgl6bTk+V?vwc1A@<JpbyF(xSmR
z*Oz<z1^4jYyBkU;C#Mq1&z~f*8b}-ai1Ia5xk<a&p$GbPi;Y!2EFYhyAi}JEh@25G
zBp&4)Qp{i;0f7SCdZU|>{g&7zdqNZ^iumF~4B6_Z=a*D{*e~cu?F}ouzlZU0P~-V<
zP}g<Dc=hVDdfV=Ep^VX;gT#rs<U76f>P7H!O?KGtG<RkO7g|RqXqy`D`NrMOHy7#q
za4Qh_Xm_Vo=;q6ByeczcPBWiEa0h0ADj!)(yL|QK*?1}&&?J30Ouj*zIw*9CLL09!
zj0wtSD@cC7U0q^QZ+)v-Z@-%yvD=%k<RaIUzgEYtP?Og@aDBB_(D4@olUxTdFg!&$
z#*XXGCP@x}l*4xJAyMH_5}RL3<xA!!8+a(7X&#~`G;@k3%bi~)PkgT&X;9Hb5SpO|
zmIG<#TXM!sl<gq_;+x)E+e(P)8Ub%To%?#8x*4oP0<M2VVB)eerQc|=M}-mh{r-Vn
zmZb+%a<<=HC;&>%9W^0xKtK8W!O0V>7?DB2ld0fz1D^VeKEUZWsp=~YCYI0HpI4DH
zL9W$F7bSDnZ*vBe9Cmp1j)oj{7y{D(LtqepDh!hGsdMx2bOOXJBO~J<a`+5qClkc4
z(}Gcm!v{j-peq7}Sn8$3aE8MiRCz)Aa!Akm_SIA4ForXvU5o>YsCCB6AX#n%Yi<#N
zET&*us3Y-!Tbrpq2HQM77mH0H2a-?0JpXh#D@Z`_mf!G<RQ-f!oz7ZAT~nb5nM2;T
zQ))3$t?+}bR+Pfe$k%Q_)+Ir7IPz51M-!utMaCZf^dmNRy$L<c1*%X41>)Hzv&a63
z|MttTKM{(K9bB6+rX{P*Xprut&I!vUaHy~vID^9I2@)tJtN5~x{=E@NthBxyjz|d~
zr5AjMb+E2}u_)ZDnH$ZLmkbOQ+wv4_DCVTVatxfl*}Esn_0e?J<}KN{Rh98gwyrnT
zxfx#@Kg!M<DHzS+?;!#vGn*3qu>HoRSmLw_l5_V@nJyK6+Ho%#IV9rsyYUqrZ=+-B
z1nX|sK9eUGcg<<I#@z>tq%^Snwt3WU?%qTOj3q2SuX9i+1((10a%@QS`EIDfP`aWx
zH1DR&d{*f8JBq&055n1B-RNm)6Qk$w7YC(*Lq*OMLs5m%*$!fFItnNDW}QRst)t3C
zw1gIp2sZhb8&Vb>v)A%#<=1u{%hy1N_Mv4Ozg#_>5NJp`2F}G!%{C6-scs4kT*%Zy
z@j5X%MdW77%_NQxp1v>#<1x7Jt*!j|DwJnEY|@`ggLMRv*qxF}APvLf-cZpcJw|a4
z5E6KRFkU*!aWgM#UE<@g2+5VV);6V+Pk5JG#TF0>)%*0fx@n{h)z|QUup>bO;FxZ#
zjtGs~6ub*k9JocJ?y-oSm-W0qq)1gPKF6G@{RPLk2`YeY2-yN|XVhhRCU4ijkK|Z<
zo~jm}9Z@!Ap5WR!zaK;tihb57^P2p%8vghsEnRxVJNidBrrCuL>ZoEW)Pk%~aU$Z*
zd+1ieP?n#Tx>&V`NQoM}Oa@Pu_H~`Hkqdj#Le-r#HLc<sTDrML0h-FwU&ax6kbI`i
z=dO@DX2?BN+H}8zF1Nkfr<~dTwK)UwoP!v4jKkX_y<!X*6c0LQJtpFbr}lt+6Tp++
zH1y~4f$-;`Cm`QQY8s0xK96_Dd7VyKY}7DK*@@OTL74%wnYD&Vz3{nL6`1GqyAQch
zG=1qKzXWtg`{38G?x>W_o^n{Q|BB}j5k0wvyI?ilbWSeU>s5GZBqw=4yDLc#@y<I{
z-*!&B!Q&U&=Zu1<rtJ1uqjLZYR-9j{4z-TTE8j^vB;5{)U)S<OieNu@(HCl0KRz%g
z;48c==0ONkZ#*jMlYiQz%3zfOhRj&(k<5<em5csWXAU1-%i2O`w-fIwuD?SwwX7s&
zh;87Ey_eY!18{)Q%mM$$(98jqAHY4J0|yW=LZ~!sJG$cONotOeJX5yWP#2$9fxZ+P
zd!6SHyhxDnI0My?sxL6&Z*-|f$jHEg<z;sE9k-Nqd)r`*=ZMHA8)McM{m80>pu5?V
z(=q%z0i&z3^jlnC$Y`=e=`s-}-UtVtjmZnn6*EMO>W3zu6P1)U+4&G{zko;{XXgUd
z#?P`%cOITQ&g{0vkGmvyKAVE3WMxg~NCrtA^=Xbaht!Hc%*~&m-$h!TcyI#Ek<?>z
z^fvg*f*gv7_)&%be$S4sD~N!hj9G<TH@5TGUwm91`AK;*QSiC>A>!8IOC1whCbCZn
zUA0(r*VPhytkj$9!Me+e)BqG?>AZYKC*lQ|b?~4h3!K*x4{{n9?s?kme9x`zHy!jb
zL|Uom-zRbW_<R<P>?{mjDj*b-_EC#V9XhV?98V~*+AUxy)@dx#-eqKKkV>hj*Ve5j
zF47qW3Y|QC4yBSt)-E@nyowV%(wQ3Sy7Y=ub6p@6?gT0AVi;yS<BfTD8ze5Z+${LJ
zmFbd@txRaEjDrO7HxeTel=U^e;0^+|(LhXQ-|Uw$jRWg@O6lE)X%Ektmb%&}<PM&!
zP1TOm%f`U1v(kF+oCmA=oFHl?aPh+dVoqwtSw#Uj+A3rGayM64$3CXt`?tFxZMagM
z$y+8J(mTi`4Un$cQt_{J&1jv9>ZD%_#)(<%rOm^Gx={{(<LsQ!YLz;DmHi)k9_uID
zyMHHaHb*+={L2e)574ZeeSfrs1spHmiX`qQr<;*!MS_$V6}wYVLH%uR>07QP8`sIP
z%6;)oqk{3rqWgK`n=jgc4&_T~ss+%gn1wnWE{Wja{AD*YEGwa`tYrhQnkGFCWjopE
z<>LH(F35F&&70tjuh#9UlqgnM9=1a4isq6W8oe^ZYq{up;<XX;yF?qw;_edU-rmx4
z1Tv|*{j0S}R1Oe=0hel1L6SpjRUu|ufo3bTiy^ijl`<jKMLK}xf#Gnb1@jO)AK)DN
z6PaBp{C0X;tx4~mEM;6S>ibc4)1&$a`hTGv#EK436RLr|$N~wNqH|#VyUG~BW>l-u
zi}nRZY!=DirN&SzcfNICOPcGwoUE+r;?4Q0(sKlV#GwCZM&4emRXnyVur6d&S`p`8
zEexC=#oM@l`V^GA=kW;tG&CzRib^;^K|$cFBeShf6P8YZ>-GthtTH1JSc}2$^!0r$
zcVu!FYP;l?KBexL2>zc4%@)e#9M3O;O_4$i<oqYLvr9_sgBYGIs~HQD9?z3nBk?&X
zV2I=rdnyj0kA$N$m1bD=L`@i4$8-lY_%Blnth;ECcYWirmKi2l&_i+Mpa3+S*s*iC
ze`z>>U>eRbP)Uw?XjH_STn(1qp#iME0JAM8^hUJO_WMX$b6O&?CD0?j*e1!nKQd*X
z0xQ%e69F2I?>U~<=J2taRN!hJ(4zp~xyX82_{d;%$TyW+iRWmY3na<GK|$|qZP&X9
zOe|muPFEe!C79TqH(_wnGRKuuL>hM=$<Yk`F(@Axu{C*K?3JXsf*Diy{%K1SC&20p
zSjY-g(oJOntbQP1HB*CG%|28Mr^hMimp$b^!F`8Y%x!JSDqulnq)&OMDLdTlV`H^c
z9Q)IXapupe-{xj5U^U}BBngS$ZuGl4(J|rV5+1Y_au2%XstPA~>av{p-O=1d^>(%L
z1)n4blBAjMXQnbrx6pz#1ooxSH(O+g9*L`V8!~^=bmoS+6ERKITaeOF_hcFmo>tzr
zsj>JtfM%U)5P}DqgpgjHB`k*sE3f>-fW8c#8L@N^-Xvh{zxd@yH`pMLSNj(4NQ5%o
zy-Utm%XX6^3j)5s*G{iz%ox++yEh`y8`)Rn>Ngq^tZySL@pR%{XL3w(wRc`&!#cuJ
z1IRjR%Ji}&=oF}?$(TWjOkgv$ZmP;Gg&dhpjL2dR;<71|QJQ-baueWe5_|j2DA8)P
z@BwISOaBxW=WXN9^rz2!B>I+X@;ZapkP}N+X_~2_3}!K_L{zX0Pc<pP^~y<F$|PiP
z>82AH^4v>fdQfK|PXp^ivBu|M=b0S8i*dcxmy9<4d>yy-i?udm;zu;!ExzOCB*IRS
z%`iT{pt9Vxq<up3Z~DbubS-gXxpF-laF~^R<Let3!4eREqFZf8<Z*qyus8i1rrf+h
zB7Tq@`QzI&VvGORLxMvc{SnOvg1E*ZkZk2Gw{5YS2Wih5Gs;$P#foM=$~mOT<zQMF
z)i1AWock{Pn3h^YG_Tr1Aea&dRJK)TB~=5IlF5?;h&H`1u`eyRFEqdS6to;^6qs;-
zfEQDVKRdWsmIX45+`rMP^x`bU6dQ;o+I8$44u$zlKG<FttYHXczgcv-d&WGsG@<aJ
z9)Mp)nt-N8v!SS6fK7H8=yl@ivyBUL=Bs_S)Au!CtqB)QwgLK}%&^U7mc{fNw$1hM
zJWN|k>&bIbqYe3)ALGSIo=uTGK!>!S*HSmRiP2YIC7rnzbmuxth1a~P8c8L+Z#cqE
zIrP1m30NKT%AxPVnAS&nm99_iM^e{o<(3MPv&$%}(2}7ziBOZ-t)zv>4EaD9`2t)Y
z^h%j9S^;QAP_>%)ncmad21_?aSv^wuS##`7ug^}yO-a$2yz)6U2E<-paP0Q#P<T3w
zZT)5lZ#dR~nagZmCIa*voj2KvvB!-|rww8cK;<{0%VHcR$z%AyckKP9@B}JUHcs~i
z*`d+V=jP_-+1c4wUg#gY+r05C^mqC3`5O+wy{jfHQej{nw)Dni`j^{ZTH`39^VX5U
zjOn8!#eB8C{84!eB-BICQ-{tV5rw322`x4(^N0a9uWIcKddss5+6k|69M*Lo4P|@l
zPGcWr`z9iiQ<db@SWC!TUzT7Kxavq343}sLuyygm%>kN?kuV#%X7q`-D{$D8u7lef
z7A%#l1)PFuaae3jFM}Srm(@M<@f&WgCX9hp3ID2~TecVy100oPk9E2e$yIM3m#2S8
zHk&Ch$%fd5PVVg2(x*KP^bE4MHa~ArCJlErsxnhkdvlq+Yajh&JaV+=r|^b=pDZ=<
zK2CXZSoxxK3h<NpSe=|u*}DUNGJ|r0e7eKjs`rf5r=BwJUHTm+m!F1>_aW)aWoEac
zE4?Q&;Aq+hB%3YBK{&3bVFkXm%9Pv%#l^dWig$UBO#Ug_D@<t(Hwj2RYu#F#x@q?Z
z%jYM210bTl3i|Y&mMF8sy#VTW`=8&MP7rguxG77_V9ho(;PV2t9gk3<Xk7h03PzD~
zrFjq4MTS88#pFwzN9+Z3Kl1U*wGi>7$8g0xr!5puxT5on)~q84qYW29HDXmOb>U}t
zz^(N`YPx_g<F_c`)_ANE9RfAbOVPnEVrI^rGz%C9kh}Q2g%-M@;*UCu2D0UQv~6(f
z7oMJqmu)Fhxo`KjZW!U`m0Wpumw+gFdzRXt7Q|xc+{&x(wP!!bqPVyd$^6Z|I19&O
zzZsS(Sjw8LqMF-g>DxB1ML#f6pc+#>j1CY@j>FcTKQR>(@+(ms$e+37CfJi?IwF7S
z@kB~bRwF(E%$51IclX@A<92*8p+&@R|Bjrcum5sku5rF9y)n}b<Tg`5@QIex$Ndp|
z$vf(obcu(>^Skvsuz{nLa@d3sM?E>y<FX1<0qo_pEao!UMW4CBZ0MKN)1<e|;nt1-
z_9D^D(|Fin{gaopZA*UbzwPTj$MAMjMp`>oKHQ5*Q&sV<&Qh!<={y(egrp(FFre7Z
zl)geahCAfx-8{QXXDNS+9cUt^IeejHDwZjVOBj|BR|Dku3z4hBa{P-FNJ1*3xI~A%
zo_0X@72nq~=(@#i8xLjLnnwhmC@gmW3=$-s%y2>{0V;DX@33LvS%rCk$oEqaVBRU#
zyaw8=;(Dl%j38{H^!@32xR4Zx`djH4$ZeQn<WhwkiI{FfGQIe~(vEm@EMOOqHZ3|U
zw~qK~n?PXF2nuZg$WKlCXi}oc8v%)^Ub0S{8?BN~`J}UaIdJ=3vSaJAg7U0jjH3NS
zV{6aiFcIb@Q*I9l7u1f)vz#QkX8eNYMxAD&w7T5$LdaRfLn@0j8i?=bevI#DPh)M{
z{7b5F2HhPNIh*_raDjCs$?yj-sYW2c=$lMm=)Zf(rsHeeVy2Y|Tx}=3q1<s@q$*V2
zzv7g6cl)xUwbv}YrHgFYznMr2A_}Mp7i{138SU*}lVM=^qK}vwCxvypJOcTqD~>$z
zBjJ7l?JU13u#<J3<90%n-)y1!C*#?PY4;SyH}HEHd7QL{(MA!V$7u^b`E)+SUrgb}
z{@Pcj?uJ@z`$;xSxA)7w8nSvDYyk?f&KoNnfO=sGzs+Ueo-5*4EP1)S64hsYr;_r%
zR;bFe&W|4`H74F~J;k5XuKbKjm+dPY0*qypyRJA-!s8%zQOQ&BxpuD%R4mP!){Lt3
zRh;Ko#}lTi+AF4Xn?Ur|=3-O+Ht(VKz+*%xF|ll=wi&PQJRyyuqoM&no^R`yJIqT~
z?k73Vht70!F*SyyUDxB}u_*2vI7j(6^Fqo+I&`K8W$01(##H-JKt~iVzH?%nxgX2m
z7ibviLy2t`k7W`;S$86@xc>`6N2+a&GO^Dm@$C7T32oT#8h2LivXhm%ym)RCnoWQq
z_xk|NgNf#KYCXIDdmN3rwH1au^05o|heU6HWeSN|kMn7atE$R|UKEmgI|I@F-5?jV
z1fNjG8t?uo&-YlCBP7WxChR}@OJ|JXcG&r@_*E{-*7`ge2zCqXTF%gS^WP`f%A0Xb
zE5>MGR-eLP{E?Oj7x{D;j304nBZeWEhXRYCgFt8jJK0my>uJ8LrnSr82D=bnx8}eT
zipzsV8Y=OkP1{+**7M)bxWae+2{5htV6GhTy|;0+j9b=+&N(?bf!66B^n<AWrWY|a
z)u-(|(g@svA0aFb{V{l-2Y?}pnGcZ*A7pICI>FnyGc<=Z$`*j}eSRuhqvci}12BGN
zMbf%$wdwL6j-c1)cMYD206=o$bH59N@vk@k1>?gGNUW86kL40Pl(%+ql)5iN<l66!
z_yLaw>_ys|{4T{Cg)whlcsn1UEC5EbG@Q0K+50<M4sPz;qU2!JnT*e$5vEH_D3%b!
zDRoUl<oy3>IgV(o*xAtjX*%rxr{c(eevVvvR0#<Q;g44wF0I>K{9!pRpH8CYM&vnq
zz@)W+rR-}rNdlr7wF~QW7Dee)UASdQR$W4rTM3SU&Pm#~ZI|)sY2FUpz{8j9WKjV7
z!W}?TlM_U)0dyo*L7P%Ycrz-%8Dvw+ExX=k)K->~;atx+VjpUFSxLQ#alrCf@C#PN
z*>&%h=hMHV!)P90(P7|}ou51&)E8(ObBVq`e&oPv{9MCjYd4Iux`~=f^(>Y&)-OyA
zd9i@a$wuS?o9ys^(vAoVEW-`H0>0c|RaM7p;p`Q#<ghhEvAS^f>Ko&)<Z*a%v(eHp
zE<e`lfpr#|EXay7Euz5WgxqUb^~vOzyd-sllq)-RE^cS6OE%-pOzdesEIdrWfLx`I
z4#vJ*qy9pj{tEnVkXFR7vBnUT!KRix+-1uHx|IpG4@a%19~Pw1Z)VSdN6oiu?4D`>
z%pWlE?&>|_DBE8kS|UkiH%1us{*lrgIVQdzJfgnvyp8aAAPm0npQX!n!)XLFJp1AP
z0-_aE`IX|@-PlrSL{son!CCVTMhfn1G<Xr->M|fp_ih1FzUqLAne62bfccCq*dD2^
z<|O@Um}MCQ2bd%5`U(3sfNjt{k;$nG<|z|C42y3poE|ol?mND@EfIFr5X~xbxM|?6
zNT2^|k<gIh>JlKNf4*YZt+m9m`+Kd`h<xR5E??NBX`l$|;MJs@0$9ofJ^+?79uZ7K
zAFketzU|>?Q6r^u>~rMxX6j462Bx9!S`XL^Nrv}YdMXpFrHLYC*WFf1EuTc~Yu}QW
zrxa_snF#RQcv+(*_2Ub)s54y1|0^;KlyW{Rr1d99Sy=G+F)|FBp{FR$X$vrvar~1Q
z7SP1=XeisK4?c$y&3!SfrB~CiHrv$?%qt*|7bn4lY#x5>WFBg<?vdN4*fQXAj7!d-
z0Qo;Dx0~YvM3cI?k}B>@bMu$zJbyyd?;oNvW>_A<qY$%f!bd~dx)XnPU!Pps=>E8`
z!$ahmL&xh+w_hy09bZ>q<hNkSMSbV+b6Ahthu3QK!!9@q!0sapT9BB2zinWfEFJ<#
z8EE4%YSae<<NF01_v*f@dFm}z8%RFs^xKPo@KRkQcE~H29z$14rB~y_G#50r{0$Gr
zy_{%tkpSXD{q_}}c12TO@`lUUL{++1%cM!Zn@->d!<c5MKM$>|T$83kjs5+&$b8<o
zy+MnGp^UVBroxUC-Uv*aIX4>aAy6Uvi^#7ZOe~ei(PKULB#FMQaG6)}{ZKGM66b0g
zDcOZ_y<0B9^l{|JV_I0c$?Fs#Eo>Y}3tNcWZ<4)>Xjh2Kdczv-m+E852ebd-7<{7(
z#%U=ZW33|*OMFK1N^lyu9{mx>C&hb(Q|RfeIVn4h83`sEdi0XXRGN7DZEgM&7MA1-
z3k!=a6FpnV9IH~n2LzX(8}9ZARVMpoANd=>)~1uQ?U0FQQGk~WOcX=B#F|KahZHpT
z2;_$ty*;RnP5JSI3W#!(m6JQZB)p0UxMk}7HSNse<hFbq)6t?E_TsqIP}qZANp+ml
zk4}tSr3dRsUHAu|uA|d~6RAoapJWOZ|F`WvA}s-0M&AkDYBZ#Y_fB)wF=lcaft*K=
z^%iU~*j&$Q=4MP>9ud31x42AweUVRU`oIvjG(tVipa^6bW{*%olW0~CHWbb$hu-cG
zA1DiSAAfeQ8%P-jtzgR0>W;S_V3=zH)|mFWp6qjL3r~_HfnWq%j7uX*zG~WZw$p9}
zNND@*-4+~2%mk@r)5t5e-QOWR<MMG^Hce0GNC-uRI>+ChW9QdR=e?H_5QtUMxp5LF
zT$lY~6DQlffm}Q3%_2Z@OP7|PmbPb9G*R>Feh>nc_<f+i(_d}MuBx!4WILs9w|O2l
zxs|Y(-va+4Cm!QhoTQu|QOYD&wW;U4Zv@L&-y#76EGld`y)b@HY;SDAa!mQrNw)EK
zR9LU80jMh=r!``;pAzDj1m%jeR1%)@CF@w4LzWu-joQ_k`^su-FEyx?qTJn`)y_Eb
z_MMxiH?8X}HnDaZFrkvz6Z`Oazd(&PGcY!C6?&6#&2j3^lYg3Od22r1&kVK*-Ii+)
z!XWWyK>1GCQN&+oPJnzlvLGPx;g32mna3vv3jH#+Oo-~oFPy&ZYaueVBny4l3@UZL
z3MqQmrS&y2!1Uf9v3vjg?8`yGt8m4x*|BpCr0Q=wcGFD&X)&+sWZuW)Lui%SK2==4
z8S{MWfGf&MQ)a9Rm30kFP4hW5`fnV6LN9Kj$BAg6CFxD(k0lCZZ#Fub;%aD{j~kr9
z*Ce&FLu+90uDK!tck(2xA|{uf@z_F=`*CU5WljU5uNU0>wTNz9;$dB-M~y8|2=7Do
zN-f!+$noDD5W=>GsjMc3ZqWzSudnPSEE*n>ev>x>Kl^_ER+vskO90p!wrN6jh_(Go
z+pQzTCLE5XOB;V=3D_DI{Lu!y96a*Yp5K2gv2&Gu*tyyLRl~Aumzr~8=ghKc7xdY0
z9Fx3vyC*9sfQL++cx1`y0oeNR=Dj`OA!}y^hFl?iLa3A~&!<a`Y@e?oF4@G4Y+sA$
znRZb>Yw^iX-T&6=Bk4`g@j{Hw5sLN8P58S5bd5tNnfJLhndQfnOSqtc`LB~sDnR=;
z(Uc-J(@ls4O|FkL|7W=v_*v1-i!_h&<Z9$CJGf+!_M~3H33rc%uIqq%lnbCk6uyS<
zCU6gfNb(wOL-$ev7C%E69k7sv3hONn%<>HX#p3sdSdXnbI(b2Xs!6OESS`jcblt4~
z_3xZ8rb-|ujOS@t^I`I(cqot)W-0!d69&y2o$Fg-ei~H2l=c~z8x&Kj06AeN8XQ0m
zDcF+I-V)8|w&r;fCM`e?ATCz<FvP`J(7=D>FT`aHa<#0p-uw%Bdz?I5fgO1PhPZ%`
z8hE4QaQ}U;STTd+K4nQ7+s<_EZ9mLGmj34E9bKB;bi52jN#_6z#W&^!3fO0$OtAU{
zaO%&_oC*;W{~=6oYh=NT-2U`bkQb@7b@?5A1_3;Bpxz^kPe~urm-cxS7lEC5)~7g1
zi^1Lbr2YirhbKqMPDoqg^5n_xH)mb33*%3JqCn?Tnd47o^&)On;~a%tCa6K8J|X8r
zwC$E2dia-0CgYVl&R_J^n%l_YN3Jq>rkvd0pM&6?r`-;6r$*sfDoU(Saph$nz|VU$
z4${Jo%^?JIeRHJjMGQ)F@2k`M+daD)-dClsRTyM57TE?`7X9>uDgrasLd1#5{(0ar
z;?Bfu(&p^V9Jk9TBo<`p&Pwk%YIDd!xJ_&dA6o55sCl+ul!Z|Yie5db5BT$}QryQ6
z7BxC3V8NfV{k2~uDw$Wy=^9k}5rIB5FGnZgk#ND-(4UdE>1Sa#j%2p?M_}Ywdh@t&
z49Z}lD#aK;GYXz+yOv}B7IyjWg3-Ihq5s1aQo%x=caK!XY{D&FE~u;RZxEmP9}pi*
zXKO1ru&*U>vj&8OML@UkUjhw`!)-#*`6PLNIv-m`v+<imqp~K1NfU=7=kz_3uJ}_o
z_GyhdpKYRPwW*gg3GEieUn%oQ?KVSUTgQBfwiATs&`P+84q+K#%8wagKxyz-Mp%jC
zkyqDYRn2tY!`??UsMx_`gUlQMzcRvhtMDxFXAs#*2jl)K?7C>>)JI2_*5wrx&_;w$
z6~EvjwCzzj+HVHhM4(tbyYAInNT6ayNp|)U>A2(Ca_G$}8+m!V2F!N_NN2b@Bk$gR
z9mZF*UcX0w57fy$Gj?swb962jC}VEtOh8Tz<o;%6RRr$*bVRQXlmlB113MroT5nIA
z4Va&I&eDpPt3Hqad=1Eu?>Ccx7YXo!KMVkSke)H#tSw;IhQIkDMCTs}3{8s`_71E*
z3Lkwh6+=ZuEjDNoZAA5dUEcbi3df+aA^tz6t>6FAZb<yUr;_l!MEiG@Bvk+Zu97e>
z|Mw~hfVu!GN%}kvkZ3IR|0I*xoJj5wh*mN`yaAB>;eLI^y5^BR;;j#Pj@CKSdV&TV
zT5c*dT38q3o_jQcOWYi6Mt^pg3y0Izv%;Qrs7%#;0E}?Nu5w7kSVX;i85hBn_+U^o
z;ul*$6#75vBtTr)f6+<UUG@K6Cs`Bu?{$*HXT1MKCjpTBN1X&f^8Z68`3cCl&VU0e
z%6{g+E4k$+EOyn*c)pIik&n%!5#{f^uz=rK89BB8<b?r76~%p8@UilXH-|5Ki_ifB
z*vVmIzgv2N%M*=lZ-B{XD5p5CM;<&tmic*A*JswTh(pSDc|bThs^LR(!td9^n~+6n
zES9`TmTvOv<@`i*BF7<<)=HRG>*Xi#5*~wxR!Le%MBMuMrGdvz?nh~z#WQz%3y~V`
zzL)N=cDIn+f9<r*MHDsaO;>gKH3+wgNs4S$hkgP#dnW4)EM9n~@wh@#KWu2v1lf3!
z*4|mfSjJDJQFzy<{1s&^b>Qfxl_FmBhenyD?xv_BJ=egH*~=IPy7<6Amo8z`R{-eJ
z40?BVIgHPe$O`hwQQk|`_Y8lVI!crb;o1G^UU;XF26kDqNfeW{rcsX4p<5mG;I1_L
zhjRgX)D^-~wr{_cFn0%FJie1s=3hKM;0UA-;3e%HS-uX@N%0+tiU)#>YaEtn*~P>l
z2v1GtDl8$;%@36Zw$?ycm@47BJ6w_52QslTy}#UZ9kQH~-rh~iJ*!T#@$vD3Kjw2D
z7XFZIF9!KNx#aA-S^8u_qUuW#M=EXxx7bZLk6Z|br%_);ril|muL~}o8n2mV3wUm%
z&Q6d`l_j~5oHMUsc9Zj`T_wk7Xb|I0SZTar-j~K6$T7B$6Q)vd0<2W`niUvDEqu&3
zY7b@5yfD7Km}jmN0FH0rrNTb6otbAVatY`ys{5tSHft%p0U1ViC(g4V$zlE`$E#We
zrkeD@lEMIqCY&i9=sdi38d9U#(*=wDQq^W|UA{6K1<P0J@0|0J1L93}#-9!K447|>
zI5dppWX3F5^D159k#P}e`NRV-F4+L%QcE}Q6?;V#m`p1Z_z#OOltNd{tAI)8W0qwA
z;`3p`<9+W^3|vn*&2J{mCIILpn&)fz`LV@h#K(duulfsrWJFoDoWYHa{DZi#KEMzc
z8WZoWI;kw%43>j#6<CIG-Ve>+&G!jsuAjW1+Y{J%c%(8T-A7!MuNn=QUR`0JSK9h+
zCF%gq#sy~@*t7uvkYeNF)($z3h>$O}Vr)v*byxP9*QtVif4*Q`GpYlh7^i&CHiuL2
z5l)YySjhpuaV%e2`LoKO4ss0Pk#OP#GV}^Tvg&_wf7%hC=T5Cu*i&`H<vufCd91f!
z8OD{-9a1AybSm7&AYMADs$ZT@vVKV$P8t#A2-avrYhTQxSjaGSxE$AlRYm$Qr(P|j
zfg;vX4?)SV(x9zmakAyvev4ai!6Z+fVAIJ-iffW|=L2~0=gMfg-I3liUX~A-sMEv;
z4X+*S<b;hjB9nV@123<~5dTgLvxOyw*<>fuj!k&NSMYIsbPLJp+M!YXP@*yw26`Mj
zulmE*7EjQ|Mhsy&72N8AcrO$L)r5rR8nl76#W!F327-SrR!t__w;6v=YWy}`m@_-P
zOZNBX;+OIaB=6`#<KVb?fW)x2vuYqQEHl1=T^R<uq>5-U%=yk{)eU5(0jusYMhE|f
zU1E2uAz{c0encaQ;=Ri(0|1Dh0xo<I3~H*x3j1D#8|J5EJ2*NgfMFzNdUybU_{5iE
zdjN>Pb3wyXEI1TJ7Ch^_fika=B`b;cUkzJ5#fdscy*Lx`pD{>y5mcVP<_ywt2KBIE
z9%R&g@Z7k|u@gLbq9Xds*hsSP#}$Y9pU*H7pRYFlR3#nski=7YV&v*b)m9OYfCd;5
zwmBD(ZSlW`{mk0UL$~9bnk)7S=kBTgxzZhHbH_2rA6TSkHbO-;x@bu{lpmF8i>u#x
zEGdm5K#}7atv(ZEkylBZ7rn|=>&qG_DbXZ`d1ELq4J3vI-0vY&!xF=mM-0nx3tGJp
zAWvUdQYk5^ul0pkQJ2juV;m~DYy7!Ds6O>7r)6ILV14Q^0~E!-`{s-WjGx3gT`h8s
zj(rtKZ)e6GQ-Xo`bBqmjD+0WU^+zvW?{<OZqY6QoYBIwuZ<Z*=ZA$inUQOU5>V}oD
zq&Gp;b}+BUkD9SEx_&z$WHE<!n53?MJbdrg5*wDa#uQrC(iS#=_lvGsd2AZmhW+nG
zu75au@xA}w$o2o7kxMwh<I!XX4_Jpk`L&EGin1eNKC67^D{1&>u@gf5!GL0{U4E}i
zR-}Be262V<!86D48*|9&Z$Ei5&9`3ev)A<Go(#GZE|~)#Q$cw7gLc_%D_08g@$UKV
zH#yiQWR`<(&DqvvYaCUX9|MHwGmgb<3Re#%cc4gX#rq3NML&>rEewWm=o4c898&`7
zu@K&RPcOTCT%~}qwn|FHl$}Uo;CYz))O(RG%Nkh??U`o-+UDo|UjboO6D_wOu?BdI
z9BOJe`>)?<)6QtP`JfLR=qoOLpICxL8Mh+g*e`SBZ_Vq-kmy3mJD{$}_jpzTcYkd|
z{?O~hGr^shLCS7lxN-tDyhc&PQBf#+DP}j;FF-N~>}nrR&@*l%*7=Qqs&3jC_z2<m
zt2BQPQ%914g@v8Q8}s5Fj_qmO_$(%D0zt-gDz{PL7C#v;6%?Ui0U4>DlJm>=Iv2n~
zYnnGv3OC2{)6;QD^F^SuuBn-?L<3{2syB4G3m1$!y9FW%B98x#;4iGK3(vKLF>LM|
z7w1*nk8>Ai6{?jo2KMQ4iz^Gv4&yeJP6<D7jWc>w5a?TTE#=+wY^74`R#>0{Wde|9
zjPMC+u)J~^wruNI68|uFjo<IP(BK~Z8KES-%*_1Bj4g97ODUQ${};MPr})osxw!1_
z8BINjNlEzW9vfdd7*PD@B$j>;qXAK2gD*~ED{C1gw4bv9I&jdn<>+FJ%wtp-FzW^J
zms*C9Tz7E+6Z_%rTcSyzW{xVG^Y^Y)*xJL+1hfu-@$S1lFYE9^*xAge1VQB&Wxn6Q
z4ok65mz&--+LR$b<@`;5P|~BGcC4FD4zmI;f^><rgnWF$zol{3RY&7Q5Xj*!kAMaY
zP<86&7Kg6DE8<4JI5;@EH9p>8Tm(N;zyF{`qQ5lNviu7qLHXrz;`?yuJ6MAbb8>yL
zPWNPrI&5hXTYd}cVod!gqexKyky*q#ooW8Jn=AM~9AW=c1!hYTN07g6{dpujciQ3`
z758dLIQ0Vn)$o^9eC`{S8cyZ!==yPuPfJQJelOi+m|PjIVpW3iS-v4$MLBh)GZZ)V
zOQjnj^S&(1WcE|>g%E0`qpa_-6afv*r4@UC-dky?GE5+*u~SL=3ZZ6D>|m$M@J7Cf
z&EUGl5i!1*V$kjvz;*Ek`8DWCk@7ozf-e(Qx;0_LDeM#Vxk>v&Yrh&@2Q!$3csrG@
zKSO{M->aaECb{3UwlS=qlyY}}_!y3Z9bLE({p@GvvM%H18BzNOcOoLae)Ly#p_XQy
zI5!y5TavK<ou9DHme6C}<AkeQ^8mCK6c9+%Lpc2>?}n@L6|r|<WaWb^+^^FgQ?Lfi
z(28C6>OO>cQ!M!x)H1i@GqLWQTqXNnqbghT-QUyP)jU+E*VcCZph4<upzbA;YIgT2
z#oJ`%dP<mmymGkBs2$kglR{>l0R?;qTdc&=bTw9gC=X}vLI=4I+mKiLxjMVJwSF<9
z`EQqzubfBr%J>)lJ}$*ouAAs$U`VC46jAc>qIfb;5jLAF*2%!YHi~?vmG)iaUJJ9a
zIkr?q+3ANP6E^VnShDn(=!wSJxJCrOn83KldF!s7=2|?ax3>zj1DaY3F=%J2YBwI&
z#U6dsIksY+%O#dOd2r9ms4U^sNI9gDEbz@(TM<z$Tkr9NiOB2HO>`}Jde40eJ#BoI
z-P~_)d@d$Z^KTGN{UTkos!q>}kWs1%ULD?Gu$VLqv^;?4!?fp7$9<EnOr{QOxeMX}
zKU()((Vj72hRxZ7Npaq<yT7TZUmi;o`nCHb>|Gx};eS|u3ToEQe4xhMZh3g&k@)bG
z2r+zi{)`5pGxj>_Wc-#CiC?hQ*l__cwaGv~uAq|2WVE|lq}C@VT#<ErA$|yd_#nIX
zfEvLJ{8as82={0g);cPWd?d&)rhV5pf_*mx*^iVvd{tXN?1^c}bO^D(3$hn%495aT
zBem4V9*-;7dR!eLH{VZpbcL^WYWF#9uwEVczLd!XZiO;`McXu%k*_sd0v}~34!zm>
zaVfRSo357M-DH|Go$s35sXPliXFwV2D@kriak7lX5Yh<(ovnT9m(_~MgGa`6`u;M!
zfnZVw_h3x4UQM7hcDU(6!R2S{3|6DO;6N{v;CDqN)Ae0>Vq{~wrjddVjpEEAWu$Ce
zVF^6Oh?|;lm$l!{h>_nAa-9|}zZN(@uXi>V^fm$&Z11cca5+^CSdzVW7gyh^)QdJ2
zde$ZKJMAcCob84!b%nh#cS8B&bz@MtXqE-vMqE{KG#c=GBO^%Ck@9OW&xeL3Ls>ZE
zPt{GKn#lwMb(+h^k)=duSs9Jks9g1pd&bvq9cf6#Qo=Or2~+5EqpmF;E;Z>sl1`UV
z<VQWo?(Lhf+%{b!Oerm|pAGEgC6GmSgrOSP6gq;O&yd{`nQWR-d)WP<s-)jintS@(
z6MWI5B$%&S0@i?=){l}DogQ`M=YuyC`C!xF<qEI6%OMp)kav`hv8ML1MEBH&?|qr$
z)v*j<#(i;e>Ps%@uOBpN|AJs4)A|WvtpwIXx3(jWpU>{n+vv6A`g`{m;P~5X{AVf6
zn)G~RAIH|w_{>2h&@y^by`bODiwX=vN1)c2;D-0>jC2`$XI>G?bLa!T{aV6&4zy_v
z;IqBZQk4X>OagRtlC-lR;RPu9zE(fo1e*84H=H-F7PJVbxxhC+@+>8;v!a1K=u>HF
zgN23lr=m2fpm*-r+IkdCZV(LjSZ)o1P%$-h-%pknO-bx!CyooU_4>(vz}<ttM4u3s
zp;Yhi`{c6~&%8Cd$<csD#`Q#*_;mz|&Vz>8`Av!__O=CV!rdGVU6cG2jB4PFWAzD7
zF5&J?J2Va51>Ky9tQE3|r`mI^(*^c5<JA5$Kp+#{Mo|~kxs(MDN}+g=b$!5URblkP
z1<4<Fd>iP))O!y*5Vp)dF1$<M+9zvUzkaxyDw{rSN3+e8HH#4vX`SAdJ%;nTcnHNJ
zM)t@M8$0dyqH@R1WxpKk<61{If(#;p&s(4J!X8h)q14zV=2)v=T)3hmd^9S0SJ;1H
zxpI2)1uR7Btu%IystU1jfmu6AS7RCT@ENc7gkr=d3U;y=rDjADR~tE#SvF0M(A%Wp
z$;8HQFusk-L`+H+$HRne|IIbZ+;L}Ut(xg~RKxpLdPtTyMZo$yra47ZF&zH7kRS#p
zQas$JurrVqAJPRqI1swqZE*-YLcYTvvhP2c0Or)q1$*S#rPT2ceFRI3r@)ytpMBDx
zCtk9~H+sd;VvbP2z9Dg%QDEO)!_5^B3hrHI-%4N^Q5SY?kvP|kPhjCnI!B~nmgPL<
z@dm53aJXPwn_Ts9<q{G@mnvvI=)n18I@wrn)F3FEvjOC%h0Cwe&r^W>gxWuA5C>^|
z#_X6x=j2j_te9B5lfgUl#i%#2Z#TBR_6>JVtk%(;No80ygi5_s%rSqgTu+s$U8~FL
z=l&+!+BvzF%6xcvqwiwQ_T^M(g^k16e$@6%Vbb;G1D5D_hb`0-<0nWD;X4UZ7K(Mj
z@%3#Qq1wW2^ga;qsB#c!aO)lEM`2PYJ2}{q7W=$QxgE`pwYJM_stfvcV@k&tTdH`o
zckh~FZ<fj=iLHE^%JEA}tC025+Dfz0o@)EP*oV$d>w7*o(j_U}lfX_mwwpM`MQP^^
zHq28Bp3gsN3F{W9PITgLe2<>tef)Aw@vcZX@XovJq%V|7`PTs3NmqAdqcLPvaSB!0
z@QJRp_E<BAHX&aH=RF)jE}O(g9JDxdM^cDIfk7m-rxzicVPG|0rTLE3=NasA7IB^u
zm-uwQw1DOnqaM*h2`Dy}<R*V?eJYCDlBTTXcULGHNOYj0LbfM|Vp@e>Z(5aWsLUs_
zKlzQ1+f4$+_I6>Gtn>}mUXiD_k_k8VZnin$M2{GgNqVa@5BU`)$Ik*^IE`(7N|SfP
zNZ?gQyU6wIM9kOyxVkhx7AGoww_Q&J3I}>DbHv^X2u2mi4TqZuhn74bm*UZ?N(&Pq
z24s^Fb$P#Jd|3)}w!ATqqm@f>q{8Q#Q8`!(FDYb50*bJ<2<$7#EEyuGoKy~T`R4}?
zA=n+qqKXf3&O+>HwNAx2ZU4uMe^d)piYe&hU?&2h3nNJ4rG(gpChP|XtM0(5&wC>O
oDdqs9KEwZvDE^Nr_WA>|$bBkX(D@(PC%_*$DP_shH>Upo3!Z#UQ2+n{

literal 0
HcmV?d00001

diff --git a/docs/topics/_images/network_01.png b/docs/topics/_images/network_01.png
new file mode 100644
index 0000000000000000000000000000000000000000..1788ea76a884912208ea2d41cade5866e0b4ab9d
GIT binary patch
literal 10720
zcmeHtcTkhxzh=HR6cH3eP$?FA6G1?_iWneLL+C|7KmwtMUPJ{{q)3-4og{P;=>e4@
zAcT^D5Fkh=n9zI44gT)_v9mKfclPeg-rd<hl9N2|Ip;m?Q=jurPX~B`@fsrl0Jxy3
z@z?+WIC%~L_($mM89KtY(IiVR&be!tdI12;t;e5#l0=!W(?JGrO>K3Csk2wkpZ*6Q
z>i6j{q;BH<5bpiL%@cqq`>8>Pn7rxGV|y<<PbYV8CpT9B+qs*!={VytuIA?M@9E_D
z!W&S+@>7A1GXF(A?QM>$-tcyU*#l&@S?J>doI8F3c60FdwehqESh~}WtDpbdxs#Wj
ziw(W1C!nSU9ZQGL9>c~qo~}-=j)0cybCGn6;a~0Ho_6+tP1@_F|5i=S4F<D!^#*MH
zOm3!QXO62GI(fU;1B$5wf&jn`fac?ehJG28@z5afcJT6Q&2!Tu{tx<`cePjT@!H^|
zdm>k<uczRMJNYxf%nqY%0e*3N?y!Qp&hZjr!Ee@n^~8>U*f3BSbGV7-zjt$FN+!C7
z&vn4x)S1WW=V#aM%UTw(&Ry<o7u(N*RH_R0BSR<4y`Oh~d4AnbnZ~&1v7ro9Sltfp
z-QM3-m|FF-ppOwC`9psE!rvd&7g>)>&a=l$1Ymk|>bN-b=)`ex^5#GE@0Ve5$A4NL
z9B&5T*$uwqLh`?FR)#pNRWY?Zthx;DARc<&`2lgls%5cm<7H2#>Yu?yX*7?<w9S~t
zp$39FEmH;Mis(lp^-PSP5)Y0n*qTc|us6A|zmMP1vB*h1>~8e(LI;>;83orqpn^7z
zBH#xZW;DgsC3j?yQ%>XHK<=mCxk?hR=i4Ua_VC_P5~6M0W3G`0=Pi*4u{hfMF=5y(
z0HD^}{x07J!YfoG_Ndl647)*^^^rGh%GFy+nR6^Ml{Rs-({tVu4B?g^-NUf%78R&V
ze{X|Z*fMgfpn%p3;O_gq=QTY8GtBniognPoY4SP=06Z=IV^Ll64LU|Bm5H7{e>VR4
zW*da$r{nCC2>~LzQ7xwty4=x@-o*837W%_*+tFRzy!U#H1jbwE8NEHgsVhk1Y`9@f
z9+0T>lzK*e{v60U;NZiF945^9REMv_I5KG8PB`*PK{uIfJQ!NQS;GJT#QpQ>!nvF{
zGS4zZO_!WfYss8+MZpkW06uyY*_elJAyWqni|PrQp=UYj`%EcgDGE3I*0_--iFssE
zFEkRLGte&}mNEmSNIBd)33&OZ^BFwPN=A7Q74i14x?tV1emmk(dS}XA(~dy&K0_07
zSt}%){R5fms5T030A~G^xsqy-*`qHr^2V}exS%v!JRaxlCGsSI;BzB)Ts#<h)dN^M
z$HtT7P>!Ha@l!*>#h@l2aiTJ?JqL-rMKP6AREJ+F8Z-7ohfKRr%&6#6BQ}?1$oTcA
zdM)81xyhtoOwC>ja2kgLk>QkP7Y@9;HWILtH4;D_tV8}x8CxGing+c~rqRzNSU}pB
z_0~;F@L-5g8xo>k@9Sh`rY($0E2S8v3{i9?&W+%?j5dMeYy2CruV6PZnL2(RRxt;>
z5`D!drEb|^j*zO0UX9D^#Y*G#X*m@q#FNuvwovxZuUjzhc6%E7!?O;e)}Gv1y!J=G
z+wF|RT?XBerW<7~M{)f++ktBXLXlS!;7TQhL&au_wu~RRP8codseb}Nj)Hat42+~%
zjh2?^9d4p9#GtP239en$I25CGPGiE=O9`0i0l_7c^0bgoK^97FIuJ2;#2U+*p^99>
z?2~=`CA7a^8R!V@8e2EB$?gpDH{o8Mi?06Toc#)RNrr~h73aPS|B$+@ly!&^RZI77
zEVaT=VvBD|t~3<y<r_P55M9KsKq{Jy#ha<1BN%i=&KqM6nLX`P%4RbxyRJk9SBRIi
z%kAUGt`7HUU)8MyCoOkF2px=uX{!B`Np0&)fR|2*A6bjP77o_Ktk+B!ao?0Ey9a)e
z=S-oyN2V~9w1cOF&9-`4-W)G+$Wg}$9Qf@#$P?pex4NJvrQ^LswbVb&%2S42lp;u6
z&3k954BfGLtQcER=BE1mXM7n$gZClm(9Wf8#@wu!f?_mKx(;4&*MOHlCM34!yZLIy
zK#i}wAV8c>$&2PDa|CQmeyrR!*XgY3D;-ULZ)hbI#{BCZFGs8#a65D~@BP5jnFYE;
zkRfVQ-zO}TeT_YmhXOdQ&rMZs4VN)dcBi?)D-UX9<+f0;R@#~P-hxSa8SDTm69_I(
zv2|B4^oL=oO<27w{g6jKu<J*EdaFmbN($Qhv0g<pqPW3ioZpko+xA^k{$~I!OmSfz
z=WS#TzftdC!AowbKa@_?h`72Fhnn3KI6sb`9zn2y=a-5wbLKXLf=d_k*ZD2oLJmHR
zR#_s1ujsFDH_-LLBHSWBOiP}a#j*&ZdAn}Jm<#3CbI+sYGhg1&yer)2x(FhX2YRiy
z3Wrl&-h~s;&Osggm;VgU3F2N1f|R>Php+qrfBz_!D0N}8_-IvnTI8WpMe}$-g|JdD
zOB7<g!x24aFMWfv<gMA!&EoWC_Wmzh_fEdzYny^j@D9Ng1&n6JyemM?Sm`GZ;?9Ji
zfB~4f>z2?(tvrtrB^+HD-xT0w9Tk`Pb&%st*WQG&bFN;lSR}_UrI0G!QrU#`q9;!?
zqb<$02Y<amikk}VZxnPi;L}QZ9K}i^I&h0$^PGI9*wCkMnQf0(3vY>+L=-(KH%qFw
z^pH8amDUuqm72|K^nF@6V57A)2DJ3GVn~@S?fHJDa70flW_#CF`In#0*QeTb=fVxQ
z{Y(^fb!K`zWR{ou$D<b+!Y7v;(bVX>UG?pIc&V?Xf_s%Bjz*3(vdsgk*zFCbQw93C
z{*o+50N^KA0uSgC`7L5_ZncYLCR#=CnPDAr=OtJnc_-yw;Sj0S5Kh7Cjxlln85)=M
z;*XFo7zm2Qa~L}!In?(sy9ifGeYHt}TEird&^b}fs;MgT+ui01k}(Bxq>n3%pi*1}
z#L+9gjpA9nm*;PWX_b*I;U6e}fGQYgh%Y}lqia&XAI_t^z4*v$$?XYR+JmSQi7f9P
zCOfas_V{X57C-6`m!U-#>^*qOtw!`wgwuF+Cu0nfW4l*6YPr_x`9)ccCDl8`e@&oz
zqUH0L?5}~aA{^n6&|h;16N{ex+fbE@`Xh~8kbTeDW&vIMtaBM@m0u-Oa=)eW)r~Xz
zTN(((@4XLs7HR@t_cUyH{y}=DqyWY+V6uIal$Yi#Ci8)l)zFAMK-Qf>Of|XsN6eix
zNme*vLpd5&kr6y)Aj%0D*A3WQfiGSdc2c^i9Uzt&Y@BbMo-L13zV0x>bh4XzED%qn
z7UTLnx~S!)YP0qU>BaLB-THO5P>^3E#kVP5VAM4IvxY{4^->KlIdK`kmvkDr8k_2M
z@f>Qlt%C9^Sd^2Hr)>kXzFW%@<>-u?oY85hBpAd{X<F|nE2!Tvo<IGsDV<<Y<NEk$
zy+d*r>s|mmiV>pEZgiNSteHuBb|bfmw{w`Rt|lx{_`uu~*w<X5f$0W%|H`Oa6Z=6x
zo>n)5gUUy(DE+S)Y}d+#4B8$;haFaHHR64-ply-`A8m#6pdYk)hWQP_T{{dpAL|R`
z>lC)OFAD<ZGUk&r+Vf+2LkRFKkoKiIOXgjUI>TVtwFD9a9Gj+aG0FN<k$BDJnAFe2
zqGz9hbPblEFk@h$y&njvZdE2XX0`l&Gc)J?Lsb<BF`h5>jsan-)^(Gt=8XtpR;ML!
zgyd_nhf3@<=Jp{qUlLi$YT_^Yn1%4@cfkEcKq2$RF56AwB~5osp$j@8O`9<VnQl)K
z=2Y#sl}-YlIegl%7V6$Y7O2^?k)K9Ok`SKhkwl$XHz{WU?*ml9ne^@gs%^>_l6(i#
z+4w~3&+pAa8)MP(UlI&XGL+ym+dR&rev|eSBES1tjPA9~#B@vghC#7$MdbWlya(k_
z6ZclH&)NJ_QOWZ}(fVGcr1G>pW#0hw(deSthG?6|M?-4&a2dQBc<Pi>?d4@l9_2=%
z)tEQ=CKe6pIPFIH^{CC23tP<$ibmEJ;=tU|0~ZZBTJ~pIWiO+!Ng*~Sy6IMV#_^U5
ziS5sGBat<WC#6m9I;mlyO7R&6rcQJlz`jtoKexbe-JziQ%Im?LG)VYfN_hZgGpZT`
zMF5Z1g1p{But1EjKibi9^%vWSGEZqa+F!fay7Y^EGWe=JsoN3ZWB2ikSjBFcy&O%L
zIDZuYkOn17*vSvnugQxP3EZ_1P6)sU{lczezBnWY375w!&enY~F%Oc>yol1hKwliS
zryB3&Z$fxQ4M$8!VI6rGUQozKxEONo?p(;d=A`!f!EO^9;xALCYK!@d&5vq@;0+o6
z$Q>6f)aAW?c#H{@$6g3$q6E@HFAZ5hVtf-d#ma_!K)#_%AFvy*4qJ!&aDO<Q*Z0;S
zffS*zUU2@VZ|^{J(54inKJPIBbe<t1wm?)B$U=hTZi3FYE6pqCo$fwG4VPczSH5Q4
z;Wp@&=H~QuVW0Zbcs4Dl6tjOf;{$SW;ccky;IVjmcN%EC?N0lXw5AcXC(sb%t>hns
z=JPM>0{5`najO&sg(f;H@)g#JOS6Y6uxt~5p7TILodUi$De-zDZY}nwnoV3|yer`!
z6;I-DSth_p_e;n31MpY79))LVR;fz7MzP24V%U-Gt`i`!!XeFx6KfWxCF0wDXjGR>
zj*ST{@>V_NU8`oJKIRM2nA3b8=)@Nvw0%mgNliwM9c6oq-<3UqFplc0^n#A?8h~z;
z=oy!n<bak^6g!kle$3#>OM{%r3u1;n9T~D5d{a<Y#q6`yp;`X`B&(kW=)59+IS94j
z7R-MyDp2_w>;eC!WMQU+vGXg9Im-9H6mM83)Z>52%L}rI!KX<+?_gwpx}y`Q5m{P1
zssN*U)=ohKupn_K*lkzhC)eHO;(QCuI^@dy4=*ME;&Ar&PHte6!1QmP4cu#YxJaCP
ze>6QTW)f9BWU!YTqr7H_93V4b9wn3M=R1!%go}R`j}=b>Tq<{nR)JqJYBp2TDlegS
z7E?o1@|9}>&M4OC!}%nHgf!da*L2a}Ow2Nn8tZ<{oT=OWJM{%kK(O{@mQIRKv1g@M
z{A|o%bNV8-y`aljT!vqXXJ=e*C-eQWXZFc{;AYMrsiOSW<}D8l{BeP1iPW_KBG!xA
zy`WWVcRs6<7ixHv<*f+Z6%my$%IDEF7I^e1`<2i4nf)oKf>d4I^_(BO%lM{zJ@eQ|
zZ(nCJStnUWG6FlRVdxn#W1fn_(5fYE#%PB0$!dAF^IVg9KGBkq+8TH9ep09WU7fo5
zE6h*bE%}9Q5qV9<J*rX2qt=C`bEjfWp!Y=xVu|^wuY6{m<`o2Y_q%S0mb5F=Jt6(~
zLsj&b-w*eSU9%Lq^97%H|6QH$=TfS$!m-1lDKP$eJ?I7yn=N4N0Sge#4NRFWOf~L9
z6^_rN_Y-$tjL4i0@KX=Vw~}eLx0dOwi9@-9aiwdzNegw`*@L~I82n|f>G!@*8PPxt
z;_!nOM?Ge9$W$crD%(?sFF`e3L9V4+ny6zxnZ=p7gt#Ich1zvDkvU&fJ23pX^&8Z$
zO3;3`;A4RcP*go#`-?AkZyP)Q`D-NrfTOU}dM+~?rKNk%UmZWKFO#MBKK`RW68TOf
zPH(PaJ-UKCcOMdH`gqeP>S0o?;oT9W=2cd$z$QfM;Q>>PUBuI$Iw{~Rl*8f#DnIbG
zGSo@a?d{Wfeb7?YQ<Lkq^(NCM08xi#>#WtM<V+^ttge*#MdI+Ul#@xUROqkqzUx?4
zl}ua4#%JI(!eSrNe%H+0RLITtq}nCS;jLU?`W`Hj%fYHy!9^>lOYt?8o>P%*lajoW
z{{S-l80}@f7MGV<o&h0nIF1;{ulgAV7V+hsylnivdmeA_LN#QVmeVLq%d_0$DH<c|
zE=absFDxWoDFB!9(+}|%|JvtOY|{LQ2lnd$`T2~(I?4wvT#)A|)g^BE$A!kQW-<iq
zoPCW{$$QK0bl&LhlkzWOy3O@HGx_%=l<#8Tn)duoE>rH!X)vBG%Vg=Vvg+)Nvp=S<
z&ZkVRz6iEAyr9ZMS*r)x^EY*>jZQ=(#08?AM;{czGY2OgOvNcjBHdQ<tx9?l%GYLt
z*K|erTG`)gRAovheQvTAmyO&Gx{*=%@3D%i+r-isvT`@jjk^Y2=aC4a^29=o=LNiN
zr^x8*))XHnSQ@GnDX21+-!WNEnOEWy?yfA=!LOmLh&K{u^bDhAQ}RKigoKZ4pJ!8u
z!Qpo?qovgJ3%mXz)@CJ}zJ8mp_mxJmc&COd57x^|g-g$6lWXrCh-I2TxO@7j8dq4K
ztcI;RcP5j~26p6-;zQ*Py#Vpe3KplWug%`+_C$=o+tapQYn@<Cfwr6V_z&13-L%#l
zRQenPC5#*A5Bl1jRF4J0_EX)NIk5q9EHUX);)(r0^kP94K0&fv!|fvi8OxC$s*Wcc
zPc8`Ic$bG>@5dWo`U06UA}FPIG~|vRwsV@l-F&g^yp`EQMh%K5s|LuM#*)g=x%idp
z>cylu<q#clSd(3H$pv_Ueib28kc1ow@T=60+#LJ$6&57mV{wAv7L;eH1)WWhZRpJw
z1SUnb_qW>U%g}_755&s$+P&)ifhpHcE+)4`Bc^K)V`8}W41M4+luSfnb_RHfbN~9-
za6!+V49KL(oSnsdgDe|uJf`yG+PfLTP>1Nsdc`#Zt_GV9wNWQxzb|s?^}O0Q!^~q}
z{qofYeX6NZzW8}cwzD$l5R9Z$x;Ny$#;e(F(6<=pmT2?Ogt|Y{(jj!Vazm;CRYLbq
z5QnzbIk(3H+}-ZYzVCo7^H~M|1QR?{U`2%p;e41|eyx;CIAP_nNdxTK@2nIQGJMMo
zbOpSQ6Q3pxt@6N`gIzQ~8d=+~uXf}iL4+^Awdu9L|7byCV>1R|vDrftg>ee?tjszK
zC1fo7K$_(^6ci-!4$RXKMOpM_>ae%%P_Ino37$fyku)oLfeo0&5f!411UqkM_OFYT
zdCH@~e-2bj%R*rLMPW&ayunV1y>$fCgXya<C<ax*4>>$~o8VxBH6n(OXf(^J!k@st
zD37E*YF5hYDc?rT#uYU9W)n7JPhl1t9PykL^2UNKip04Cr4hLR_sVZb+QWMY&R2Y=
z6$Z1I=&>D=I9|{c2|iVo=g4qm{cJrk@L`z<kQtQz*J@mCy%gI)3B18}iQGR?^lenR
zo!?FutHx7Ach|oD;7szgpr^i)Sa-qEIy?3$t|50ldRqX?7AZp#-ASQ2YF}&i@t%^%
zJ>T}!Msi}8ip6Kk>}&o7B!hKd9Qo_C@K#599wsCuXt_H{krOgGtWRLDC09^R-bWur
z@qamA8E&Oos=A8^?)a)WjOc2tHJgY=4aOtg((*?6UlY$}67ooB7vr(bJDtOaqEV~@
zk?ofi0(2B<od$MOeG*0BZMW1s<nO&beaz8!Q<#F(j<kn@;qPw3sH%uw-LQie$XEle
zE1-)O-{g3KD{61RuS`<yf7kvH-n}x1+*6a-jR@f_)hGBC?gS^!Kyj9ApMsS4+&X@x
zT!MtOE6J@{jvghA%#CBF$Lj-q-vg9(njoPq{MmA4Uq~AWgiKnMK~i9rd7$jiYuN9@
zBb@lO!*?{)kN%<Zq+iS(*M~x$IA>|{^iFTpqYuwlLtObfR8F5%Mh5aj8(#j7lWO)4
z8LjCtRKOJwKo)CFs=%TC;uEy1gx)Pf+8y^76biptlhE((Uh=WW_A`DY?uKTL*XVBM
zTQUV1{xr^4Kg%b0=r&P4yONHYJl>q<5&o@*At92D(vu81W)}%Wwz9xT1LIFuSx5tq
zVAe^wF0(>mf*(7KKl29d1p$TW(f*PZxrTROI#H`h2ja(VU~LgbYR5b5-(-f65+!rc
zhaZNN?LzIxDLy}FRz(rXk1FczpkFc@|MczQ0vVk5bRT?)0o7+P6UxN(H&aDji1+=C
z(z7dE1lx5odbasA-y^CkK}%b)dG8ari_|dI8`qLL1-Mu*8-0)AA@FPO5gy#gPPrXg
zneTNMb0AUHdnwAdfk9N3p91=}>qL@`4C41V{YS-@X+m`UiI|~d*XZA5OTunFSJtgk
zk_lZgNE=T{(_>*cZj^;LzFLDY{6VHt`jeYe7?jo$5??AF3&dEvqeL>H!U&Z?>(7zk
z)2B4eGEn4uEM^Nea|Badylrrx`uoFonwf%j(E9`B5nFiflrB{<4QlQEOc9XB$riec
z;<U{Bi~2D{#%I)6ZcSV~)$c)Em9NFXZ|@frM<q6TmiL}-G7lE@0CQf-S7j$<BuF=&
zuSm-{>BX?r{_aC34+Zqm*3{*KD8bwZ1i89Dkxp&&@bN1NxBBL#-k*UX+zc}4TlNy3
ztg0#YUQ)J3&X}lb$9pkf>~F?GazuDyp6<ub)8jNpPl!x!*n(>;7?a&Qy!WR0+t;m$
z#H4tw@;iF4`Wi|medA2-xy#Tzhl{;##{Wz&*k2aYqa1kn?9g{<%p{+ZEy5LSfvwg+
z8OJEUGmpoQXS*lig_o=31E}k+kQgt+*oHpDsJZf}XQuW`)B7NEs_?)NfA6_cuSsjE
zCz4vgGcny|=3X74bT4pYg`3|jc$QDmn?D?BwM1wh`&cxp&b+T^yNK?X?IW8jjn><u
zrxy(u-_ujI(h|EJ#D_d%q_ps>R{VkeVzB1Bf)tYOVo(h>c;02V%)3+dogNjNYeIDM
zMM?F*5_^4%Dzkvez0v@HL+Vc!Ijw=BXXYl+s_&3@ZSH@)SVO!?dz1ndIX}~ETao*g
zQc)1b974g!O552rL4yl`L^C@wx-O04BR|vJMb8l`&5@fs=!$7x%!>&<?tXjy`>{IN
zELSJQ*}`7t93t!6O9_xHkCOXb$EiV{AdK}t4R*7h#cSW|kfzDDIxqR!a>+`4dG&Kx
zeGbKSzYJvA4{%S_w+)3dqw>)$pdqb0L?)ZdULy)Dl0HAMr^}AWy$9y*eae+$1#i66
z-<#YY8|CX7V|1t6c=LHcabBeu9KDI$oezfU$n)99fkigwiXHya8Z9CAlZH?JxOEG$
zg)2{Rm0GIo9q9zs_Wy`}gWk_BO<Y$wra+D&PC&%DJVNMpzi?b16Qk5IaLx5zF3jk$
z_?>W!beq!6nhpdeTS%|RuX)+!uT2GjFS4?lBtKGC<6Vqv_@#`E!I{|7LsD}_Z|0CE
z`Y}Pv@J?`sVpsa!bNGd5e_qFQ`qTP-{HhOa#a(EdE2Rr$x#lM@_>h`FV22PVeFPh|
zyA2k6``0_v;(Z^MX@NWi>jK=fOUhK>TokZ-?d4{*f@K`Usjzu4AJ^03@y>*?DLu%d
ztI^+6cT*$I#4v}NdU?^9SeYs&Z@3qd(x@}f*-Z3E)!e^ciQhE`aThThEeiKQc4za@
zb_+WiRf?*E%#!p(+45tC*vfp1qOPuSkxab5f{^oZAU|kFRyzTBuJ?rMNv9f4rqt5E
z|0CR0-0uR&KhL0b>)ayRW`}+ms9ZW5b8_ilRF>q}S$Z%35t{3NksbSgsoCT$HoATX
z*B^b-+fpSzODPL229=J`@`Y#9sK<OSfTM(;KLFn8Eq80ZHsKg(X~{e$6)mdFvgHf3
zDf2YT;6Zz&43+5(p&}0mw|q^rNJNc3GlWklH3c*{KCA5q4bYA357nK~!tHqknKH4y
zEZj-Rif$9hOtb7d`<I)Pc&L_1%lOKSJ)?L~#CPFz_}5{-^$qfq*bgm}+D)@9M^;HB
z?~xYCd^WmuH#Urwjlcv$9&Y)Hs&Tr~NAnVd`)Wmi%NsVhO5V^Ih-Zaa^2z*JGMZm;
z_C#;`;3m&A8d@fZl1zwJiM~g!tp~QHmVBotVwhdlPdq&4L0kBX)-zN2in`{~WA{~4
zZ^Jhcvv2TwM$Lv(_ae{r;2@CB1tL8FO87mbqx`QCYrp#k@cl2%T6+)PX2A517WTV4
z;^y%goQwx6;Dm(>W7y`VZ%1-oU{cmP%PN=)B@d{_9+R*1;l?Wwl3O7)&aKc+h6Xgk
z5^1GkQs%9h1qJP4ocw0zDz1_97#F5eeI>7HeWo*IlT-K!)DkaQJHYRm_?r_DH!-~t
z=_2%E4)`_hzF`AZ3c|u7BciwYL{l*!qGsAWt!T6+kZ_NklKteEmd&O<p}!Zv5phhi
zB*kB`Y^eR0m{j!BU7<~u(*$?h8N1|biFsX7_it#7X&%=$fCkSc%3@-Jp0_ggaNPZO
z68U@d@w5LV!Tf){2q}v64$$uvWFOxJ_@5Bz|F!37k>J0+ILIJHV5aZrWUicJFJ^H3
z4ccAs?>zyV_?tQA<w_L~MU7qi*dnC{KfiMtW)qcrG26u*FgqRwJfX{sUJB!d>pFg?
zw|{w_+4o;}4+LZcZ3NcGSKnAQ*wvdK_hsQ%D8_fn+<K5rw+T6?HeSW}e%3ss>1ZP+
zX)O@gxG?@5Nux#ugjhGcvgxdzylu9*YravGoqA`x&v`EpzctN)neH5BzYi87y*Eey
zE*r?lomZ-RXnh1kOx@ufyqzt^bFghY-ms0^%u%~T4kVF^=IoT$f>soQXTxyFwY41I
zCxq3!fJ0Kzmhej1jAf&RbF)PrZPko2xi3eKAazqK=|To5987%krk9t}mmPc<SCYf<
zZJhC-)@qxgSQ3~$R(xF|Wuqaue%UJ!uP_yyW2b6-Na8X8(nt#*+1uiC8I_tK)**YX
zlcE8acUG(S*2?EmVx`F&h|NwQnnKcHQe_QTf*&qtE)Sl!M#TXQ=3|end}CfXuk<aA
zZgB`gJT{-l&<6o<|NSFG<Njc-AT~rJc@9Y~r}i5g|4iCzAgpZ@H(FR&oQS<w4u21e
z%+xhLvfkZS_q>yOXL%C#x-TqX+@IzQ_5b}@!N{VhN?LVzT{k-}5%oUs$Sij3zO8x3
zfYVPp^+^mOC~aMUt#EWfONX<^x6>3t2XdGpKjg-Y(kV2_sE%U`n)<Lx#Y280${JxB
zFXD=iVE1C+`6+DFHt%X!z;`&KjXA|HY|MjM`qW&nD_1)9If<~nh%I7aX*~L&8UDm|
zsp}f7)}!?cwOAjXMiHj1Ul^UdlgL*WY>apm@BN6jJbsBtdloIL<GxRSSxdOYk}qqC
zJFX>EETjD6Aj`eLvIqy(jV;iS`5kh1=xUV$2dNmCWrkSGRXNRq&1Ycd22CV4|Kzfo
zkx)`~v|+N+)$iKA;6@`$U9IM7uMDPlI{7GNQ}G=2RTyfE%vF0`^3P9AIP;OCRm|m`
z?G(aI#2(J|BTYGc?Wcs%h^;geliQLp_xI%<E%R#hznaHw#!c4Ek-yrg(+}1D84lT{
zWuxxFJ1lfYdIt#k9o=;;LwIb}S`gPf$7>Ca;#+((8l#SLTPh8p`0c(a3AtdXxc3_b
zjpN(YtYkvV%x=+z;boL{d&G4DsruojW-N0e)A&k?8*+9j62CIM{V9|3s;%Dl@B!pz
zcJm`@sH=WAN%?b9sNBxMtJvbT*Mj(ip_@%t&!YAc*uN8@yY1FXg>Fo+7uc>V8r&Vs
z&gN<juC;dktv<Fs;jXl)Zs-y9oY(&OGe|)=Q*(u-Zpnc;fmYr;;Or|FKQUg!V6^=4
zgC|OKvu7AOlWqc{FWE~cX?{_Oln~E6EX^bud4+@J1LclTvYUD|4^i3V$S{6>+IeHg
z+Def0=j*&G<_T$cDBF3OH}o-&W|w?(>9P{qfW0|IaRtj_^y0hSHhFd5Ro_2sCTnIU
zYz?RKd6@~L7A|5Q62-H>2Ybzy@pnD4Z}Y@<2=2`d7O_095|>tMjyTy_AuYZ~k|F6=
zUs*!YD)b>%4RXf<nwKs;Kf37p$gFTi=iBMqV`a<Z@L22t-UiNGTIi6eH<)mHUiYcp
z@rimVEJ>e<wFzA=4=)5CExw5a{>-N1011bLYp&k<WYvO0(!M#2_OLqSFor+r0E<Mb
z7U9N2qEv!LyycEn;t!R7w|QRwJ1$lI*R}=!xeH_ee{TQ79lHPXOY)m`@>Q7PRY#9h
R-{W(ysjl<5?9sDV{|1*77K;D?

literal 0
HcmV?d00001

diff --git a/docs/topics/_images/network_02.png b/docs/topics/_images/network_02.png
new file mode 100644
index 0000000000000000000000000000000000000000..5d39ae601ae7c29c8f69fa4709859e0a2006e54f
GIT binary patch
literal 82702
zcmX_H1z1%<w*^s=P^7z4KvKFyy1N?*>F$zlkOt`zknZm8lJ4&Ay1e22?|<j}1n+R}
ziJ2X1t-U8$R$AmeJPte*6x4e$Q9*eqC>S{K|Fd^*!Ox!+4~F0uoVBQ$9Te1uu9tt%
zaWo%r!62-?n3NFg`a28+BzBuPtp5xNsoL{9*#EGyg(@f+5(Q@=+JhlML%Z*`Cf4>Q
zR+dnxaKxlw9R6ioz{=Xi*2MUSJya3W5GNS@@V}$BhI%ij64{$r7(%h!AiX>f+{<4S
ztc>iP^lS~Gw5*?BPDl9Pi<{VeH`fEF+CtT~RYii~cQ3;#dbXA(mc~$RxZ8nX4E8_Y
zcd-3#2zB-Bd+^`W1gtD93@z=Uu7~1V!PwiE(-ckY%?+XQk0&UhpopNv1o;)6Qw~#|
zG~>4*w=1Irhvc}gk&zYg>SY!BtME7`m_CdAW;MSl&UTRAP%(-BJXjHvn-otl97i)L
zzo>yn3oUfL4V}j0>6IZvg!C#T+wS=0pj#sSp`~X>J>ATs<N%}j>6)viVA7cAKZo_j
z*l*%$LHow;y6jVV{<6X&$53J-qE)B@Me6REg$u_)@Ch8JI>Y_$$Q!kyR<&48Xfa%B
zclJSB|92|C2Hd-sWAT-bmW4a#tozHwg3zX^-n>3mwVWf%tf)ZAl@FGrXjGDe|Ie9>
z&INk5MY`&w;L0~&-3v)7RB30;eQeP_{`aVxtT&8#4gLRzcPp&ZW76yFaZx=z;^X6E
zb}p_JOPklPUqb>vQBmpkM&PG#*u5=QmT-1v=W)HvFq<h+R8)joJveYn&i&72>uhjR
z$r6XRDU#rV1xd(I;7G>9wN8vG=k^mR&@nM9J)d0fuMR})?W>kj1xfg6Au9{-k{I+}
z-QC@VH(Xp^ZgwK^k<ro~Vu(7pm8lo27EhZ^k3LVit_LAt(w&ZQoqWW^?7iHZgzoF>
zTP?0#)}JlY3&Z08@9vVE+r(V1UVOOJm>|RV6x-tQz~p-QBPAt8R9bqwmr$k5<AIe<
zyQvS?>QNz)Oq0^8<&FXl4$dL1f6KNfl5jNklY(WWY%N^jZ~$pF#jCFM=V2PfVr3fR
zkrcg+9w{=)^z4j`kN)4JXG)f^zU*qdUhdNE(tC6HoU}ftDHST~eZe_swsUmc?#9$H
zSZ(vJ<*(Q#wJ?`l$&(>mTw20?pa1u7h*j%T{Poeo%CE2e{r!^Zyh|-+)`n1ZHXFhZ
zuLVnIE*>vNnT>`LT_Fkz3XE8YegOgN`}?i}6^p?+d3hJ$%3t=6D@S=NOU*g6^7DfZ
z56z)`e0)yN&P*7w&Fr>^u(PtVeu-;%PMUUypciS>M(ppK2A*8G^2AWeY>$f&3ehES
z+1qXQAwN9Bz`y{T`oYj*w=;r_j6ANZ>cmOk`p5)6k~1+)KxnzSQ|>NyF1C}cP*70?
z0zcs6-A^4{<>cmO&K#Vbon<;4Oo#W(#6JElDcN4Ex4T|pWMs^r+UGn$fQNsFh)DMN
zb2wNfjkIUw!nmV_+De0d6yAqJW*=YQqm!)cY&Zmjjl$9v{fV4@K|w)w#ElMr1T=JX
z=+&()p&vh}P=a&D>q0|9`bI~eOQ(#h#Y9DQXG*jj(o(ryq74V*wI+4s<dC<vw!j=s
z7vQ3!qiZ(U@BP_JPwqbp6&~44ELT6^MY@i)wvya{Sqmd!#wZN0Y4OrRv*lb6%t=S{
zzucB)g8zasgn<jOjK)YED^e}@;d_3J(d!I2`==nYzP07ATu4q#?49nZ-fz`vv9si~
zVlmdLT)1^OR|!>Rxpe>M%H`Fo;Nak)WVXnWRIVAfTJt%kii!$LUqxM}B&yG!ySuw{
zrTGwK<>kXP>3X|!E%_$%x9x3h^&hT}!i6F0q4fF;Sie8ndLX`q0S>hqBg1PlHe6vi
zm|s+6e{1=4fAx6QOXv=M)aEE?H9CIo40zvYNP}f)Xjr6H`A5zP8yovWx&|$VbUN?w
za<g0i=%@%S#$>IP&i(a~M(IS=YZ=}<vSe0^j?T{P?oRDWqxUPG4;H=yEx>lzmYsc-
zi`8wOpKiGDVtLN{NUzT)O{<4~vxGVx&2u1HO&6&>UQWt*EyU@Us~(o&X=-X(+u2!I
zV~+j!QeiSl5kbHc(ydQSO&tP<N&&pfF#r*JLbusO+%q>f_iTuHbPPO*0TZ^jx3@!w
zgoMQR@hqXHhvUY_>jf)6KfhmTX%X$Z8Q*&&q;P($P`!Ec=5W3`6jR&Tdv#SeBqW60
z^^yuqptG~H^?DavPXxZiN{a_B;s!8(Qa-+P@aUQH1&JJXMlp_e+oS1`-!gRvVn02{
z#k{VZlhoHIsjshpzkk`(#O-;vg{lAjyUFAqspcDv8gsC~?}yU)T1Uu^4oX{%z%ZNC
z^P=RjY4K{wbjZQ=^Z2zmQ|OTUa~-9pC5y#oFqmzZ`52<e`z1*FC)aO0xqf$xw+5z@
zVm#eS7}`xUZ-fvm*N;Km`K0h?qI5;)NU~U&*`V*^<I~e!>$6)mf2(VA8o5OLuhi7=
zegpln4E{y8lV<b9y0iG}l~=E7W6sZQJ#Y1}iHV6333*>pQc^;_1gi5bJ`n9h+#h)y
z9v_?IReravw)=8;LEKZPT9W%I)7`(hyFY$bVB+WJcRy{14R3He>&C?ArT_Fv4>-!A
z!;IEY3I}Srs-%+A)cfS%r4Q6JG^O=8n;RRQ8<Bh?A?P%x7Z)2PjmLj?dPYYhYnNRS
zkdTm+OSOl6-XPopx3~N=Tl>bh{pXyy;}2KEa#e<;p<R2<Ku;PP8V-;9FJR)brIJ~%
z?<=g=zOApXTRPj$)!P{_yX>TzrE$m?PBmRlNI<;=p^6G7;P6)cWNRQ!+}_@P1o1>j
z=uM$=aaM)Az}K%tG%66mn`0+N>(w@4C-y(msp8L&C$$=LY`2?Za(4E*ar9%4hwHOJ
z`qtH*otbE{hVSe)>#xnGi};I`v&VnnHGpWk2HcX2jO;Yh>#=h!M}Z6_DmK<2oeYE&
zIb~%>1;bV)ya9$X_g0K2rF0c*g<#cF>(0hUIf%UHtj=)Z4I6L0riWnf6MpPxCb2p-
zM10OzB)*i<`D(M%D+~Xa7*yfxsKrHYCT8Z~&``56D={%RN|`i+{%G<<Ns7OJ|Nhii
z@j`gFD4mj;3WsFXIbE#5fFs3yJ^S6j(D3Z(W<^d>QE;Iq>grK*hWmNn4(4kpB2#5W
z#cZ98K+O6|trdN5Bw_2HMqs69AP0UVBpd~ERgj-gL`fN#m*yIBd+S_dKKISmmZ`Mm
zo*IODVqV@fFe!SKb0E?f?*A+3>g~N)3nbL9oMT8HZn~Z~0|S-LXZjQ}X_7`p6fa*h
zF)`WP*<rR`{kp!fad$pQUr<s~lPtq?w;7X@le1_|pryCg;s4^Gs-?@3mVP-oWbp9t
zL7|}n6fwk6qQLzHa^+d5p6y0?dL`(ayV|^Aj_xji7ctpxioV=OTKWTUe#s;zfAG9U
zqiIrda&oEnYgBwzZ$SJq1ZlBIvp%}1smZJXpV<geT3Xt`*qDv#1oHH7bXCyQl(M;L
zu<Uy9xk~*rR1Ad_$YvdBgs2pfy30*2v^p(#G5uX>JZ@O*)<hROqsE7`<!G3g3TkTo
z3s$XC?(W<mcea^UpSQyjmg%;?N^?K&$LJ4aF`NE~ff109pwjF#GBR>`yp)_P(dx82
zGd%pha&8D*uS}J;uA!mg$2f_+f`ZF=KjqziVJTP{QuShtV&%b}aNL6`DUrRx@NWHf
zU4~gF60=4=``uqlOI0S5?AEJLgkFz~Ah3Sc@w_8{>+21&6EY#840s+fF|k?m8jRuA
z#YIhbcXu#<FQF?Glb*?B!hk|LrH?c+OeA|eL-vcslnPw}X~N(qkTp!JD~v}8zzSi5
z_XL-4xD*1{!lc(W0JHEiUt+D?AY8j&O{pN_vH!X{n5lHXJLkMF&B^ief?TA7b!P@)
z!kiTsm(^VNc(GnGk-__HqxS_L{B*sS&YPTyxe;c>2~Qk0nwXr#{bxuK6WrA$2$ltB
z(PDSH7<a|vCiN+7;$^Bi?em|Sv#+@==D)C7%o~i3f^`IMH+nJ3XT=6vH=(_;u`ySs
z=gZRymU|c=8Pz&#;siRaSEiC$3AAdqoi4U4M~?>F_$NaMH`7mUS*?>SPgiDz8KU2}
zbQ>}wTBdf3hSW6pPj29oE-gN_;!RdS{L)lr6i?Tt#p92QDM}Z^ri?Z+OU0=QDrDsi
z*OK`P84}GO-JH;ymuYfl-2CwxWl^silc}Ar__id~<V_=c4ayleH%MyI$7@A-A0ZDX
zmxpsPeSM-JNB4p-%>fciS{gxDcXw2x^I!f}US8hV`1lH|6<#QS(_F6SjFXqVyzck%
z9PaDp?~U)yw;V2?k9JR9LmEXTxw}hSEwjqXJ|+(fYikoM)L4*y{`@|)`<s#yI+x=i
zFz~0{tMFRpIPvbWOaW|e=XV;lmTN1|5VmK?4O?t%?CIs?D7&U+Yl+x5fU@}@x1`Bz
zRun8O{u2}O0Z4?>y1J`ZuWu3<^cm$%CV6WpqC|1EgocKOx;i_3tIcMjqocoxi^Bt}
zc(LQp8Z`{|yRw(AJb_4rM!+2aEC`6?y$|y7dG+E(4UR_x$yP1Fk&#FsLOyN~KI<;l
z*}UZ0Bxd6ZfL1y>IuiN3c=h!3!U*|Za?~ks=IGSaVE~`U4z2*aKfOOFW&6=%%kvTu
z0nD9?(^iNY-177HpVDaml05u=AU!Q@WGIPQt-=7Vv=zb$68THWTXs7s2RC>yeS|^c
zZUdM%I4sOy_&3Wne|oomK|=b@Xu4H-Qym6t^81*6KSgQ_dlH&BaT{CP4s%vEs+aKE
z3c07j2@mKDM9Q8y=pP=gMzUsNVq!>=biW)I3BzU%0C&R)7Zw1g0;j9D+o{m+h8K>&
z>-R^%1ZNd#H-o=^=TmeCaQFP;0$h3>yqsl}`>#R6#Nqc|dsF7Dq%j=Ya0%kMP0x^r
zzAf7qmjo8_VgulDH(%hv<L%kz)|Q;S{5O!XaKd{m#lKgp<NqmT1>E7q3&U_&;0vb)
z0^Z|VJ2)6#Y!5r^n}BG$wzl??{s6>JHBNI0P{@-8NY1Ow@kkxOs!qV8sOab{#~)FH
z1!v7!2L=bn53a7?z$6a`OqmMNVkpSTc`Fw-Z-4s=)|$)dI42|HCp64ksM8JvdIsYW
zyfiO}3xHqhB04W?Sysld?0zBj`1trcM@Kr18$jiEG|EM*zDT@8BqUi85IZNQonXqe
zmrzio)fnH=@eSOu-g@m-x)+24iRbd;iu=X;x4xNaEis~GSDVdpUvS=obT>OY`}V5-
zTV{q%t0x&HB^y?RdhvF7FTwYd<>qTw=k6~B#7{9vNn-r`(0~eV^+l29{P|Pia$y9X
zWVZY}ELfEpZMSfA8s##;`#f*gU~O$}sp#o@^~+C>YF8!-6ezDxR*?Sv+lc24OVqH;
zl(*^>`qU$(bg@%{fx7*fc;IB-BUh3{-;|GK@W!mav<ynOx;?~KeV|%-n`c*e<|Fn9
zJaTpeRTAHHI3+B*Ud_DdCx*(6A_C0ZAef!^c<feRDZDq-ggv<mjW;UO%-yGBRGt<1
zwf+4U;c1Sk871<j+<EI|zprggZP7Z<h7;qf8^vjt&Y<Gr4g#!o_juu7r2RzLa<{!A
znQ}{&%H`DZ$1`DY>jl#kYt%BCP16Eu^pZzG#E=0&78qf4TwHfBDkZzyHSLQ}9?e%*
zS*<LWJX@)FQCja<gAn`ypHuOKo|(C_Uk@aoWB}3q@VOjyS(DXTJ@E-W+0y~x2o}y>
z>k38%%p$>Q*`?RBr;1UL8vSLdd|{E+cGA5_XlOzKRzO8X-57gvIQl$q&g%L6<U~zP
z4cLF%%1Zj`>MAZT6fPQC7l=c+JoywT!Mjt1nC0c=;Q@2mB1qia+^46f86f5U`t?yd
zm9u-4?^&``yBR7dER2ka3B}mh`2KQF0B|e-UV-^WB_~Vh>Jnju`hzDp2UL<HEDwkd
z0BZv^01EIy$H@qqvs$v2mKH%FAsdK04xle)6@0;h-vB63FRqD5>Z{&F0=$rc;Ul>C
z!`I2l$(W7aNW1GpH9%KFwM!|vx$%-Dg=;OB`arzXFW5`GboFR{Y{K3l#IpfTe0{t$
z2$1V?et&>+VQxkSRJHn2MiZ%U*w<p^$wHOVw<>`k5@u#*cJ=fC+AQAactk}*lMP<c
z^XYyLNu;r%0WcZBt;l$IQiqe-HrChmOA}NEvdFPGY|#K3{ga=cuZ>$=RJ6G}p3Qx~
zn>~a0eIiH9e*a%E@RX#nG1-R;8V4>O9{g(c@Vvadr@870Cx76Wr@*;44iCe|e{@Yv
z#ews{Vl6c|6jml%vjB?fFhApQvt-Oh{-fX#;oZAc;3+BWHm|_k0dA%WVD9qr^5*vT
zM5Qr3uqVL#&1cKJK@{Ue+yQ3z@|R{eCqQ9<3X{>$@Eh3StFcOBnYQwlAAdd1&d*8c
z=sp1G4sHs<j>Dynu<%=u!SQA7Ft*?U^X>)Dbb4xCW3eC%;24jGin8*y>@qHYt~_y^
zc>l<Vr8Q-{IjbGy@lJTb3IOjrKukdZ1;Wqj(b0QNjpr~d#*Q>TaP53W>Q}E`y-aF;
zejfx^_i}eUHX#9q#o=47JTfq~Oo2dlF9<gXB|;0dU&UMCI+g6)3b3+Z4A@!pE~pnZ
z#5FZI+(vO3ep@7dmd&h;);2Ke{Dd!Ui>yuOB{iaGi!2R~{03cF+5p=nzcJ)HrWDlT
zq&9zMPe^2;pK=L{Kw#awH(T>KJhP0urDB8A%x{J5<tK>Cw&<|ycGBFOpRQ(rtUyXj
z8wRinpj*Rt7bZD*#vn(Q33oal6xYr+xiBx(T2YdbbwQdw9Z3{U&5w$kR;#1IzX6sF
zDD=kKnjV{RbLvxSTWTj*bQ3-*on(NGDONb=3RUO<FMS6O|BV&{6$3*U2x3Q=KF-c{
zAYO!u=bBDazM|8t`|9e-37DK{uKY+Ej{pD&N5*KTc3Ls72?+@~9S?(qvokFgYCxnc
z1A>UAme#>yiY+AyI=T?>F2HEk{uL-xnaz9%7xop-Zr3ljzCG3ZBb6MWNk>UX7Xg?y
z6btK8DyPFbpgSLoHBGB=fQLCcvH}yr<8@Eu_kFumWe>O|a<E{BFH(d>d@GoW3`J@X
zTzHUpZ;4QX{j#!1a^y{7wp6IluBSGD4%XwjAqg&l&uPC3NQY{%a%1>4I%??r-5d(Q
z#qzSUW2^1608+F(9CHDwOa{C-?)wvvUdm`512%_8MtX*a!%0OVUa-hNQx*sWf`$0@
z0qPrFLJWu+AccX4)2y?GW@BRmA_BYPA(oJkkfMr;f|gb|PyvDW=1O2{yKJEchF6jQ
z8r+&SR}L(!wynnt7M#fF4+AsS(0~J`PrcgIKQdCLxp3*Y_1W`<*Z<_KjbLH(051iE
zE?Yahz42_3)Z|FKHQ?G-t*pu6AXwG>!1w))n*i98PrpHje)&fNf)|vv)vs&TrH@=v
zQsVw_RLlK%+W9_!+2!2eEfRtG^U?l(NMj=xv++pP3Wme^rdXh-hIlK`zSeejUi3#7
zpvmCc=P4H+>~V>lFH_mprw|JSd_+cWZ})}8ZwL+wTHQ*}sVSXh^9v0{(AL&|@n~#j
zquGOtt9?RDOiZBofH}yYG6h;ocbQ%%n3Tk&@bq-TsHmtHB?=HCHjcwTObq)y%4>19
zj7u7WYIT(%Me+zJjO;Xm_U~fPwqZ)@{+hn+kU%lD42lsK^mCC=bJ7ePR%XWLNRgYI
zhFK&}9AXtOERI!H`}*40R`PY-%*-bb(#@c5Yxd5sk*5cIKedv9EuIa~l+MB)Di@-I
zIbL6P{m!bP+F)<u>sgxET4OpD($@ABs9J}is$*5xaWOFh0O}HY-V&d*JkSC32JEar
zA<uc$8}`u1n60oC_129MBt{Uf#Kgtd0Yl_IZG!=%|4YL=?c}_4Pn_^SD13Z;KpTF#
zUy^#b9(P1Nb3Dp&2I56jzk!r?MyvA~$%@xAer_&#)8jb}898}5kZx%)XaFJny8~&U
zfCI~2SI6FRXT*rl+4U<rx`XFtagsz_zCh9T@zw?qC+I_shw78YTJ_><fiRUyfJ4C{
zDkfIIybJCT)U6MUp+khw!%w?(2M`%p(?g}_yHPJ&$fFD7s?-Zay7m`}3*^ax+7(X;
zkVu^Oaub|^onZ62vjf)pVlI-uzXUfPH-x@moacwu=M5m=tpNfqOcBGV-}MeC9{qiN
zUn$If!zn40&6xw~<R#6?7YI_saBy3zYd}x;089^_BK8FWcs)O?0B2)z*jEHJV43U=
z9sUi_`Dj(kem*_iZ0ziaC@P|j^4#D77N^(l^X3{lQ<6d;Q<f;EA8*?jgc&O;`WN^L
zfH6*$0xM29_r(x1A{Ijizd`_DpTdy@;i|MoJz>}Yu>R$D0apj#*#Jgr)b<k^Koo*t
zK_B@7gP}x50QCQ*Ek}`xoCCf6B}f4s>nNNP$PJ9&dl;Ngb;E>7^~)32Is=)Vjx_=B
z4FL24D907eEj47e^*%t923~$>2*N60vqGgaY*^_4I0Erz6HI0&2o-yic_4oF0J7-3
zUy$bvc<kPCGgnCo3mcghBbOo-+WS8z1C~CbWF$a-1@|Gv2u)(MBH(em-ULBMcj~WF
z7y-{b-9g+n9zMQB?L{iD2PepEAWgl>=Lm8p_xK#=1z6pp*YgvtdQ}iupcjq-!l?lh
zal+u~`MExL_t&tnY;_ICq6C1d0UH?ggF}_2j(agyK&s2M-5SrW6c+NbpLK%yNJxC*
zXjGzOWBrnoab>edOA{SHbpuG7w)KmdEsg5SK($;8IIjUHSd<I}jGI<Bq|q^BAv(Ld
zqTt|!w6?Z(GMEnX{TL%MU`p~R-&eoym4p2IAd@Tq;woTKd<9ALwuh2{*!eG5rdSyj
z4NVYWW#5?<Z6uv1xo(-jszh3r!iUnOlL^{N{G`_<i@fuH;$q#h(arhHzETIsX(>LM
zH3zuyQg<w<bs*euPBKf5z-rKZnpDT078A2k8gA8=)>l0L{j({htM1cQ*J~`SM;peV
z)zIMMy25UU^_ZZ{)bv<5o~fp)1|PlDA@1pi9!x%vo&Y4$?FvE$nZHY)5rhce_BNo0
zMD^nkjMUZFpKZp-*sgWF0iwPOV9ZB_&r(ONzP`SI&H(Cz4?qdX2Y_9IuM3A^>7{>M
zEP_a{o`|@(aCkkQ!S3ztiRPjJ;b?;PSzCMA*~MkSi;II}O6><~X!mQ_K=$)};NB?l
z^v~WvLrilq!WbVP2h{>1lqt=3U}n{=@aSEEfNcTHPu$mg8KA5y%Sk!vxYuvro_C?j
zWXOLi`1|*dOggV*8h3rW+xFn#pj_e9aLYy$P{2U}W4u(S6&Lc>mlV*Z{w$%;!|e}F
zoHE>JuYr0D3R36i>p=A{uFR#jJ^)gg5m2y!a{W)uaAaUWA1s^sLd{Wpe{^DE-;9oz
zR3rh<Az@wv;8iEhHw3<K-&Z8dYasz`C08QB7PxLzU4M7#Hy}-eiV-i6QF4Ja05SlO
z6gfN}u`)A@lek^Zy|!3#YePV${t3+ep9@98AU2!jB7P%@S+)9$asjdi(08)s3oL6z
zpR0dVtG7JbA_MKPv%A}Wb(P50*0f~gJrE#4lG;z?Up9@7i2+<c!b3!9)hhv0$33b)
zhVq5qfL}nZr9`Ea?pj${X^CNQIp5TvQvoE`?fMYlIIND2{p(2(`MP@Zlz0k5g(=})
zWG-@Ya)&e*H#cS=_yMdc24wy9uHY9Invp^D;N4VMzbxi6{v)TZ4vUSgHd#O9%qf}d
z-6Kz3MYpP^rlg6<r<d8an6J_UR2k?5Q9#cC)sQ3(yX_;4s{o)<=p9TKtJ20lhg@og
z0;J-2vMdD<gnDuOuV1o2&kzUT!_N%u8FminbD;jQ=(OSi2*;A84TKUDCP0}?L0dZ@
zCWaYUUF}il51=3bb|tH-+E@O)*ZFoWu>NM*%_@}kmD2~Srn9&51^+TX*@HC!LDm>>
z{g<lU<SyBoy5<~?a)Bb4U&OZpzOeSCPSMttAe@nds#b|sV@<x=ueN8_q@*M=3JU!%
zH~@845}HbgiW2eh5rP=Bv$yBM)oqSkV*d-Eb;~g_!LO^WQGR}}H*M?3ydERvm6W={
zBvXAZuD^Exl1!}{i8q%wD7{6cq#TYNYS1N27HhD9(w?QS!SGH11>EqObSl~Z7Iy#h
z<6o|!jIcILu;6*^<KNPs|2^8CacU27535f$G5%ZS{rIuYuHKLf4gmN69$rW}MwLlG
zvS0Wd_}_2-{Mm-@qevY$&!<BlCypHIZwK?=(_g)MF2o9_XkQfl`u|4;T?ZNe|9|?L
z@uBw6krdjS3m-@r!$xN1r-CF=)<^&MU~MbW8bDM(pW+A>w57>Hy~PC@O1Ua6+tER;
zaCUh+(*53*E1<6G=2@$on|_6bW}$xMpdvno^6sg;ZCEg0=UHnJ#g4RDJ5sx}6<rSQ
zahNfA7y!tXs#6+jY8>~xDN_x5O8&yB8dcSor~d3cMu@N|>34s%;%`mC|8<q=GY5Yi
z_S%1n=<lGVId%Wr3-E4{`nRFwC7?VuhqH1}|4b=cZuN;lLG&~AuVJH3(`rsU-C)72
z!kV%otU`xGy65P=xV~=IA){Xd-Ktkcg5n&}2TTaH`qNX_UR$+@QFDInKRV!py7bZD
z(NWH(j^9{fy!C~K^8Y5c<6>yICG}8I+ntj;PtvaBLN1N)HXORV5<aRwsGCCg&#&=I
zT)$$hzn4K%rUusq4R1Vhk%Q%Z<Vriup9ZT{{|FYx>gP|8$a7@g52CV$!ljM`sb&h&
z)#!gwjVeH6lSQ&fe9u|siptv67fY=O|A{2jJA|5th(A+4>z711PPO8m<WID?O8EL5
zmxB+=p?mZ$#aEqHwkWg|e-T;#nR4j2=}+jDiOY%CQ}dZ8h9?Y4&<U3oR2(2B4PaL{
zxoB)EhazjmeYn_Kv8|R!t*B>tRMIA?$1#p8+PiZMEmsfQt)eiXMz=L7<}_!O&1K29
zcU+&JQz?8j$S7qcby6m7_I|A~EG((~tA8^{sQMrfkx@B}f;lL+Pw3A<X?jBW>;i%)
znrTX$O0tqR`txVHKH&eXR&zdz>Rvx|`E(2`^_Y^eS?#GnZvTlMBON&MqbJ~9u%!3C
z>t*DWUya(9S~h(yn<^wcDmG;Z;09clbf9u`^E&Kr=_n&AP3?YW5pNRq8jK1uPYe^L
z75&AQ>3*;)(x^d0k&0ZX81T)JS<=<g@m8fQW9r}0C|+JIqh&yo%Dpjp;%{y3=Wx;E
z`kk~|;5SeoL%Xj&s8f)U{fdtl26P)p`rDw$ljrpeox<sW0*Y=R!(q}JWy*iDaddRV
z3hr0TM#HUmQyM=0gN&XY3CNa)hPh8N+2bHG{doQ(@Sfb<%Zag(-td?EHKO4Fl)HsL
z8D;!B4(-m+Iv;~m#8)+?U}a<ezvj}!X6%qF!bC2Sb-LmwF1)4SC-XZ)F3OR-U+%}W
zMzx}k+?Odu8``U`psE2%Q?cTPzLub<CnYbB0s>*th5O;Gjb!3!A4}(qX*D|s2g~h<
zY4x<1-nh;7&1hmXN6hW9maG_*?JNVtY~bV|;(A8vkV9QaG*35HWiLN~^0P}(a(QBi
z_tU#_N|u=1Nfmu_`t8U#opRSnWzWY)O2>5G?mKGqBhn#S*WLFY9;$LRoC^niKOq?I
zIG!!@VML*#NLV|N;I?phtCap&EXgBKmx|rWwK<(pdb~}SdW^RYbGM-(d8M>@G~d_S
z@)YdBg*u-9Xo5<$))u*Pw)a=+@QDZAM34-5dEZD#CAOhfc)8K%)6>P@Q-S6vHisrH
z*LW8RPjadn+%$ds9Y<P2yBbvD?SF;m0x}bSJ|FTb35iw(3w5R&ZtW=?K3niSAUp*f
zU`X0YccSG&MtWsN44;C(A1*cH3jEN?(qJjsDbdZR&Lt{*m6U8p>bqg4x4zzz?);E@
zhZog5k@vmOD0fX+KK7o=PDf2xD4A;hu3n4aU1HzUg@{q}C8~GlN#PU$o?PLhy2{z$
z4D8wNG(jY;sI}oeohL_pNTd#~rXeTF%yXa)QXDD)YRH6Y*EA1Tovy8b>v}El3aEK^
z&nFwk^wPxv2u~xsqnOA+P>pkbQLNEs7V;-+<=<vgqeQW6dMz%+kUG#|GB6+xXoiTc
z>_-47;NDwE2`0bu@##E!|1Q@oAtE*|4lr1!`cZmPy>?^TTN_e1xCi!F#czd<QB}64
z+XmH~NpU(H)G`Xij|Ij5OfPfwt?Av9mp6Y)fog|#TAKYKnuN7z5Rc3^0gXw{Dwg8T
zj$ji)DOX|6@>EW(v+f4$(hIc?=nuztsUCUfWiGFg*GMVI_1Jpp)!^W-OsXKWSKskH
zw1qo-(nS^TCwX^`Qd}C#Pm_6<2RA$AN^RPh{Yvzp`^ewt&LF#<%NXmOUw$R8_jEbf
z*ZDjbTp!f5C>KP3GwS3d;FPnsHKhsaA-}|P^A-OJYfej-`mSt?GmUD#k{MM$8MUFf
zAeo!vP*gLlW9K4n7WYh*76Hn<c<zLriyG~8j}yuVLf)a}<_9P&W}`0vguKZBI_FD$
z)C&>_(Cd3ED|(=0;|$04)K$cQwiG4G4NyWK-?P73Y;<)n8rmejosy^4dV(&QP1B%~
z;*e?SkK3&a!cgCGB5&Av66_U{b%9U?M&@N(;LoE#BYc5wUS2(QadEkm3LaIAA2E^Q
z>Jdl1`>fF085Pd5i`&Uk#WtyP^gid}=zH=m$H^N`PMDkxgDXK5``?%6j`pX~J_R9_
zn9Ni(=yt>9N~q&zbT-xtu75j3nB3gmQo3CSXlTmNDbJ>qX{<bJetuQpxQiqD#NHLK
z>DctJhv1A0UzcIiBsxY$jpt|e=63_b=2EcnF(jtiOEXJp#yz^K9dVx!x|@9FavOa%
z0du?gtB3^2^T}Rn4)1jG$nbzDxkY2-dhkpSD^s0=q|mjAV+pl>o7s1``ePvm4!x(q
zkIx>wkegm>JF}hetgadTZFxru!y_1zTnt~jNWM|b^(?bLwvHis=Y@=Hv{8jtW`83L
zW#tMwb+Uv+2$IwZtEMOv<HZedP`xLLa0}D1>MwicKWN;aE&K4^enQloJwxX!W$$FJ
z{3fQ^95>-DfIT^R5HLB3Vqmx<udE$>d%j<v7l_-4q)k#Q+c^|$^Wp8Y=i<%3jj%AK
za<gIA5rW~2BM-*>WafNZ%Y1Gcvn`piEAsp|qSAwH%7foDXi<n`#Gq)rJzGAwS8WI1
z7){P-N%TZGxL+<2xVqne96!53^%%@)skDi^&!FMxOd6`W46-*17?nrCDC&5goV#>r
zh1?$OuW;9R)CH&MXzf`+2Tw}Tl)hAdKpB!5@IM0AUE;;Xk~m1=q{dAx<1^NR3Vzr3
za|CX@f~H3Zm$YL9#jNrWxd+w8pNOnVouxMupvL!7J^>9dgTIIc@aRTZImMjswi3=x
zPhS(U)}R8x2q?s_0NiUkX+1HxKAe*pP7nqqIzUBV%3e-RB4j9ffQ!@8k`WPQ$UQf#
zYtrQ^qs~?;4WC3ykP$A5{FV~Wk{pOVhThCg*&G{3qc-X7>uu1NZWgt)B<Sy^LnOeJ
zu-neU2!5YjYX?K)L=o)>85-Aocl<9VjWH_UPb;7m+eo|uUbfzOlBNcMk1QuEG_2VI
zhB2;@c|_kB9<`DY(LB7q>eo3nGQR%&SQCqTZ>*Lid2JE?O>cFt+S`n-B`vrRc%Na)
zx%4m5Hb+Zm7q2&$>tZd}hq|E=-(BTakG>Zyax5kEWIo&f5_L+LP+=XzHm>D{x|%7d
zuqZfcM5@%T?OfF*9#tOHe87785G<L)3mx|R91*`se6IBr>2J3A;iAi{m6DsY*)r=I
zdd>bCJks9y##1k(D<YF_k;5UMJ_xA`Lkp~BNKbZQsCSV7Wgm4h*}>u`CRs-FTO#@A
z#kSxfcvqRU&c18u*c7$Vi7OGRjzEHJ3dTRal!}z)yU$OeLaJC6CPHIv?5@>vQB77f
z2Q?`fK6KoCR$nGIpB^hLag)$b+b&L!cUHLJc)mT2r<W4t#RMqp(V^uU62rq2*TfZQ
zL+W3D_&!i+<Gfcfdpc|?y-WK#|IzyDZJ%+ku}Dw~<EtHe*gX?!X<qw^G$I`Zp`d&p
z>Pm!6VJf;(VZ+Y?=^OqhvQ|FBpm^!IUvXzqm?(E~znTXnJJ#@w<@2mL+Rig~$kCJ>
zn)GQ?)d(3$HnlrI<|bNCnPE3-@AAFU+#n@_*L@^`n7a&i<MNWQ_D`JW#O1CV#6NRW
zTR5O66m%&>4d!4Cqm1NWtgn@P`;;&?W<nDzFZHt$mfSEdu}Ie4lX9?iK?vFco6GEr
zMZeMWJZa9la5}7;T+Lai`W25~rw$>pZO=#c8qi$F7d+P;XgO{%Yc-nwyfIg2Mk}oT
z*KBG={c2L%i-?*#o~ZR>!zG;l^ZHHl)bb)d(k7WcBU2Ju?za&BQUO~jW(k%x7+9N?
zV<e8>Y|;5es0P^Fcs=PgYYx+Rup}o&)4njjV6Z}FA1-QhD2yiz&+&w7tf-o`oIRIS
zP1$<)*T1mR>kOQ|+U2}95w7vqGQ%wALf_>Ka6xi_3DaD3BX5PlV%a*kY@!GjdcF+_
zDH{nRV2-9D<mfJN7`!OS*hV}&$hN=Ab;!pw5rc(8!c)A-ZEn%g;*CqWGel^#jrb1P
zRDkb&-#eMvlxtqR#LKBzo+Ir{Tzy#@&ilsrj)y*JBh<^_ypABjzphB;J%Rn+Mh^tZ
zNgl&;{r2q3x8tOa$(x$u?9z^t`t)D&jEvbEr}aI;1E4okt;v}Yq!Xa_M*;SI+<c=Q
zXyh6GeN$eaMJ}5wD(Y7VUB%?qSup*LOcKA216}LTf`CvxhN%?FOTq7a>!sUd3zVhF
zxVf9s9NvDpknCuMTv6EVw2ThK(Lg6M=yf=d?al+ew5hqde`p94fnjB2WW>eAt(}~l
zoKp>4&g7Im<3c1~;|FpJvN^CAa%@#qOi7}kS$YaO?uW(9`PVQ)D{dJ_CFPUfAA9H1
zOmf8LlKwd)!5qxJ++D%w4WPQx-%{+LLy3^EIbw{rX?~ImD%|Iyks#rp4<8qJ0%8r7
zA8tFuaW+gzlWKZ>a~ioc=(=j*7PWs1f7ceK5YsDa#CVjq9mS32i*7azPwAQftrI!-
zw{I?U90S$FSm!YXvBJn}N#7(EDfN1MW+@SC3(-@kQ+iiiLaNib2hC>)Bon{HVXa7W
z-|4eF->sWmcZV6CClbG|*WRI>)w>PZp8Gk_9JG%nJYa-+H1DTnX~h|R^E`B&v_Nl#
zt}z}Ly2t5=1J#`X!0lvkP{;l0;-%bcwgvM{%UjeHNJ>Vf28YUFd#6lzTWpK((jeO!
z#DL8<!otvMEOf||`{zk3-*EYCO_vl_h>^ml=D^-HjfOxcL*GQ6LY@-Kk_1YV;cjum
zWo*yoziVs8y(noF(f$%O3?}D-+Pgq|ZoN%CMi7~xZlcfb6^Ti=iDqs+Ca28e>Rr|h
z+v%)yY5NpRi18$BxKkpSLuq537r-0I9U!yjseIBRnnO&eG<wE8vkc32(nH>WQ(*To
zHLHR>IaMa_t0+BHcAPOg6}^uX(1ATt=&$s;vOUJNxQEhrzMVc?P{R9cB#jGom!vC4
zZ7qGs-(DZLfBeI*+kDCWRae2uk<7B{L!Ysm7zg8k_~p8GSFe$EyJ~m?clc0ZOPDo$
zUHQGqxcv4~#MpeYg{{iJeKa!u?7g`)Zj1G|OeE8xkNCNVJjx25QT?qI7E3nzX$X{S
zS5mjq?qS~35{LAInW7u~1LZKch@PP>_)QxBx+KX^h!Z4i9OX$9B<AY*V%9563GkQD
z%lo{=C@}2fa6;&n?EW$kss&o+cPBmGjx%PZxraUc9Vt6AhbGQUU{8$}n8^!7{?6VP
z>ag#oE#(r?+?URQ;u_(elBmn32#+5R<Fz$fO(o6uwIhP|EjI*DFH3<owH52ei6G0|
zW$C_h<x5xu=A_!f&@}}{Xq0$2m4W=GEw*hEnt6Or=|e2%+Mc$w_q4d7Hr?D5dsyqz
zPZP;w)uHnA4|cT0q|K=gWO>FDHm7JtmvC#1{(NKCPl0B+H_l7fE@de@Zd%7FMY-lr
z+h*oxmCl*fIC068Oi6LaH$2h@%Px<QcIgPjJwKfR3|7iytn1my+lEgz4!-)fYOS5Y
zPH8T&Fu2bKE18Jj0;y;1m0Xs0$)L-_GG>q2pX>bnDqSucnYt>s%FnKZ>SWw~8C&ii
z%g<#T@82^1F@wsWSWol3R8983`5tNO^eY@SViTX@q0pl%aaNh0yH)+(iNS(QZ!;}w
z(q++_^Ij=hDJIw}E<tLb+{26L076YKdAY5F;22{g^Rb7oCQ5O?A9zB>?r7<(eJ0FW
z#Bb5hFVZn6tQ>8FI^u{_9aGJ<bGzu=8z*h(qN)0nj#Nux{HQdyz)lXrF}{y&l1eVv
zcm7ERb?WaNN*3ycgoP_Wi{pcplT)wnZ=<VSobb1y-TJNbX9FH}pTos-NkA8SRn-^J
z3kM2)uiw1kNHMH;Tj}=8gD|oE9h&~<LLWcyt1#7?iDRQAKKnB*1<S^loxjW<VU|6(
zppI|d8Yc!Mw9IZ)!z8r|VawDR=5zkhV%4E8SfPzCX|Xr4;!7n!;o+#WKcJ|U1={Pp
zX<sHm3o-iO`r4AKVL~E@f`ETPJ=nH1MbnNG)-jdRNK&F`p?2-gzq(e!Y$&uO9P7kU
zYs`{YP+9q{e_$jE%e~7Nlh4dzzvjDC;pye|4#O1iQ%Sn}X~M*;=r@P1w~AKE&?DhU
z-|btojkaQTmz=b0u5kIdJhV%{f3u`I6w&Eq(lgY+8QBbbopI9~x<M)bW2qG;gM`|W
zTV{UP+df6BHEuG&YVPVo#Lm?yg>2Qn6Fjy3jLp)5(poajY<fl)a~V&ufCv9ns?T}2
z;*UmX3oJH<w|qD9u8pagRf}Pv30!p4sLKR&eM7~2?j|q~F;iVN7UAl}mJKWt_}qJL
zPOzLp)jdQ<oj;j{{A%I4WUxg}M%m=#3j|{AW64{Og%srdja7AMr^s{e5%pZ1bQYtK
zF=NF)WzaOXMa4X@{}uc8_I3E7O#aX};eEA{es$E1{Xm6Ekvx=d_lqLu>)**>cd7nT
z{Shd&j17h=jGCB0z?)C*2>#a`q~*SQq##*iN4ef0g7T3zPr@EEr`k%aW|xZ7PRaE*
zd1O%AwUdpz0P<3?$z~NnKhfwb)|CEUiFA=oEnZQ}Do=66I?O*&O7D49l(Qt`b}#Ww
zPEG_{=Hapf*|epZ&JSXBL^?*d<<z!?P5a+WDAVcDfBgsb_QtkzuAV0~>&eCh&Os5u
z$w@_?X@K3PBzATe3T1R_oZ??#3BK>R;t#xADx=YXQuh6Af!^h8Y-G%kB6q@GX6f&0
z+KQE(<2-2Jly2{8NSQid<2WAw-g|OTRpX`(y-AVgjYNR<t48XoSB7XJXsMxN<XKjQ
zUUtW{U!E|fc4R$#vR3qJ`Uc=p4hoz!HHU3IVs4gEAZR4>9iG008ZF9|oQ%@dS-spc
zR_Uc@v$o$5hS*)B{Bg&1`u;{yQ0OK@ru4_FSc^T>%HJI!trT!q^E+7h*7XA8WKk*e
z$pv;Dbrm;RG@^z(Uehtw#OZ%+k+p`@>2k46ww98DSiKWeG9<jAgn4Gur!%z`VIOQ8
zvhcnq<$Ai*K%pcN!Vv6r;2vC~j$c*G#n<@#i>3?PKMo~qSzJ0`bHt%8z5Tkk?Nce~
zA6Uy~Iq-I~`u1^(*W6gN5HqbL9^%j`wdY+a8J>{GWm>;+Fxb4r1zKOoJV!<r2bpXK
z!-Be(t0KZPE%NELd0=hVzR6{qOdni<tsrpO<L`d9w(^1Q`0#((^Bew}5N1PcUE=3M
zJ$NQNj=XqTlWX!V>$lA!T+kekO{+Ytb*?DSZ36bErCgbXB_$Nh%##o<T3SS%m3t#Q
zr=qjLIA1gpx#87@T;39Lj6yiX@Z8D7mRsuXpTUa)5t@I`EHy@w5Vl^|+Vv`?NC!kx
zmRGb&a}iyT|JivnA+xpWk^n<Wo<#2yNzl{H+{lr1EJK)}x`^IEyLKYj?mvkq!H7!6
zGZ<{nkhpj%%JntvI&Fb6b|}<xjU?VmJK*!G+*_AUZw&Z((){<9(loO!R3VJb^J`?|
z3UUR|FpBwFa5l<ogeW0Yr6>ZP6lxXkzQv|hg<tL@NW_JUSfe-ml^<wok~Kr<XVO7$
z8JC<4ZwWVeDo<*<9=%X?X634(jGE&Oqfq<WuRd<9$S#pUNI^rf+c1JLiySQ1XRoLd
zug~C}TNE@A4MmcS^2ZKQW(aP5^E%gjQKKU#>IQk$9aXwy-*&dFHufR!1zJRX<`Vz<
zr>l7P3wG|!q?mk1cL++(Pr)@|!Aq!srwwG1BTBf1u|H4Pkjg_ECN0hj!HW+z|1uEQ
zmv^<?>y0qWop=O)A>jEB-tF-BLXgf*b0E&2x^)LOW=I3;RL6pU5R%{dCs5h^HFVvl
z;u-GdN#xxsygMrMYl&%=t*`37*wMLdUJNbIR?7H9CgBcqrGVbme1of6!x>zou&^Oc
zzT<i4AFAzvUE=fD^Een=7u<w(wtj$Et1@=EomUy$mn)AbjKGB=-`_`_Tb2`ojk?ih
zBtUKV#r)K;Z>Ydw6p<?B-DSj|=fi|Ok2?l0z4PPN<(a}03$wIA^>MFth0D6SbRj;7
zXv<(|1m+qZkz7#uo^Ad=@4Q`<q$G=722EilL*=LYjekwo6fxqm)P_`>wrB~%S!Rx!
z2G+x*^PF^Kha_54Y3w`dh8_-&OG4(N!Rx6=grqiSI0M~K!e5lSfBYS8dn2zfKlTvy
zThCp}6~B6v=I$gs@PMEKKK!-z$n#>=cTIW3)hDD^v*-6nMA2*OmbpbmP{q9PQ`OP~
zTvVeNom{^uJFd$%`An#kj_77?B+n&?JF0z(LPzQx8=$)vC)Tw1$%OT2DGQkhQ$P7t
z6e(7m{rIN=T0lU*+)H;yU0of+_Z~#hBar+I&1r|e6N(rcC-EyMtzEIWt=HX;dkKmt
zcJKK_i8J=t_~IHh1?h~k=b2t332b_K*$xKE+^5&qMnJakDiwS6FD1HGl~#)V&f_x~
zyGwF?l@)AW4hq51Ze*k}CztZ$PbG>TW=kwB+|!+<Gdf8PFQYqR77TwHx#s&-!D6vj
ztLyk7pO}MVn?%hw%Z~C0WkcHwm$Vj_gnJr3!<PMy^>d66_7vHv`AJ&A>@?{)z{Ne@
ziwT!OBds?zJ)AP^Q1#of@YmW*E)67k@?9!QgwMRM9nW5`*H#~HuUv5EO)N)Pgyp@r
zey#|W#|CT}Bw?zlQae!UNb<oGiQ!KIGbQHdt<ZIOdvP|`4>&?UBhzv#cYb}U%&>u9
zZ7(0L|4<+6k^b4xE;9w9aMCWIGyw5e@jscr^!JXQ#J#2NOj1dM@02){5J|UEOS2A1
z_+3|?c^Yb%@yy#Y7u$8PF|YHSgc|(2h9{e&))v?9x%`d40@c}f!Y&;P`(4ew!-$f7
zH{Qqgq?u#o^E;<jy{S(>xcV?1V%`shYl_~cE_L@>uS9k2_Ebn6^W(0(Vg)T>CWY4O
zTn|6Fe`UfbfndN<6>2S@s7L<l)O5+46~b438$J`2PxgA5FDb3z>G9kc=g-8)9?4d1
zT=6%VFb|Wi&sO$_$B|(jzkAnzKXSEPs%#I>Mf7E(^N2`gc(EZZKix@uXrQ)1u3yDo
z7IppoIfBn4>#y^XC>wzTY>v0Rjl?>0t=j8PP=)a^Y_fc26I}k%G&83?g@bZgy?pvm
z8w3Y_G^~%Lg>!YL&<R7MjD1l~E<^(9KYB$@9%FdYn1Y&;cY2na2AM$Xr`GpZcNe;E
z^P=tc@X$pSadiYqkSG3pFi^C>W-x~>e7<^npqV>nfmqr#?V4VjK9~Ne-rAy~rGGSk
zg~yp3D32o`O2dYB{pb?6^YCcsp!3`5C;bX-m(r`u8)-e)n-5jjTHPvk+q&Jsb&Lu>
z4)%n8X5Ytv>mcBFtw1pqp!rJ%NKy-tM9osd$rn_4Tdf<EUmkIo#}>Q`KWxhoYd*`G
zCR^LT)!^7Lh9uV8$*lclL|uaY8kT$est|K@Con5i@clieh1S|YgHiJ}$N60Axd{5m
zPvqsI`jls&K?XoC^ZroqT1}OiUzaN;7cI(>I#C_*YE64%SCb9@PXFJ(w?ln#SA%wt
zPA4s{$=_i<V~|LcN;lR<6XVho9f;irTqUN*`Rk4LvUy`U?so_Ay)%SflI?Y{KQ8N9
zJ8YZ>{+_p9o(S-qzGH7o&~anj$~41%n*LVw$710tM72JR?Yfw<$U`({<SjUuz?KlO
zPfc_*jc0GA)hovGxd}(W5KdTv{6;cWY!r$EL+XRZHH6=E1pUGZ?l^O47rQ0+gpI)9
zddTe%^*AW#9ccU*0=peS_iaw+#XG31n-#B>>+``<C?ZgS<|+%kJ8nFI2A!*URx2$)
zdw}_fj^3Tfh!p*+Z09#z{rQ`vfRZ%xAW-puM?g5cDJ|bCjeH}J3s+D%;b|5%W!_<4
zMXV&osDk^EtKg$X_^tz|BnrBvIJQwXLF#f4`mg*HIlX#!`s&yQwzhc&MYFMF=wLj`
zYLo&jTH%-_9#ylNFfw^TSNw*-8xo04@qbLnaZ{Y8$5YCgza)-l9|lj*KqnOFZ03Jp
z{|PqXz%ZEp%jX=;llDjf4Ian8*D%r1wOSS$dBHBm>w|@oj8eKD>Kj3iJ5TZCsW9bI
z5waN4iCttjcGBqs#yK*osjK1s$0uCQDHDDT_3*AGbbpVt9!ZCS5A}_MTL%oKR|_GT
z@{wIa;b=5(!!ZiwgEbAZDvF|M1mxqA+2}%yN`x1Ne$W)=Pve#clToE=+Q@~D$0^Gi
zvid4|gC<1kBy=e~lklV%&G5ib)Cvk6LL+b=?+@4|$8>#NEUgCtEB5uD1H=cU<eO5I
z96TFox^^8f(CGFDoeCBh_AV36^<$~w8Y)`PXH4^R7O8rdgvJdjNkjXsm2Mp_WN4M1
zysnqIn@S3~FCj-B*GVGy!`Uoz@tchCn`DmMWH#Tv^~s`q^M=yN^p7oFQBkLA{`_RQ
zz!ht|=VXE%BTps!D)u9hAKAz%ne?G^P@eM`w~UAg=LPq%$McwOf5mr==+AS@tvhGm
zk1DU#A3SE%AGA6>XVjT5lc`TCw+zRq1Uo(T6Z;rLZheuzb{JIaw`^W~K$|FM$o%oD
zUAL3ulK~pCuUuqw(x(;0r}D4*>z+bpUHMa(4kZu&?FGnSOZrv8*yE1uclmCV&m}A*
z()}mb{<&xmsUInv!B?gQMrxamax~pGHQmOdq(5_a<T39k$JC{M7_}gEgxyn$V--1?
z%fHnjlEi>+p8iQD#yoKH>R$6tIZy2)Gb0~y5xW$M!j(hIm0gHEw;vz(8rF}pZCz$*
z#4i>74Bpmw?XkW}eUj6ob^|hE>o>0{N3O<_`pWmk-k>rccIKXGAC)g0ZO@)<6scF^
zsKuZ8ueFu)TDBB~Oet~nsbR~ZaoFE}Otx$=vE4p1_h3z9MvWH}f_syNx)X#|w25Qf
z6d7}MWbT0(bho~1{&szwA=uz;xIz2cmv(yyJLMn@k=--|O1;t-FXl^ji)s1JmgMY=
zISP0i3iQp}&<8k9+`BdpCBa8buNCPuP9f@~t{+U<lj6JfuJ*3@V5^+m$6tnT#G;*#
zokq{jdI^%lSh8C(+I7oy8TXO>v9CpvyO|H}aP&Qq#*6%<qsNk^cKc9}dc~s1WBT=N
zFm$7hHfxnro;=l!wnw{M7!{7Z+kg8C6@G+!Bbw1k1ix!>E#`(sks|6N96F^JxU1-5
zYBVdsW9#bZf-$=-ou}#Qb+UN<UVa>Eyy(M0(LFEovr$*X-B~vpqB70B!8!N&NX<!~
z-6ZPfrP_lTOH6PC>d0=nx)jwKJ5C@vqf5)B(Jak`UkB=y!{ue+A(wnd;IlquJth~7
zKs?xBw0m`raA{?1Yw4x8u3m*h9P>3h{hwSvij4H72<2vaosM+xyRJ4pwqetSiM=mT
zq6YfIMV`l09C6}0^F(WWkPS+(=_dd*0}a7F+|N8GC1}q-rR@(p{mQW2u=(Wc=WksP
ze%P+@`-BYGlK7TBAQorKbv)&9>`t8SQhKHbxKA1BSZSL7I^EW=DmNPLQqFmW8tQ;9
z#Gl5f;R&{t)L1PzYTx*6iNueIBH_bq_%(gUbF1~gYP|_uRMW8*8_}nfN$uFOmDARS
zq}|}NTmO{t+E)lAJ!csTpIV6te$cdcxIq<f-&2909^rfSd|#lc_YrZ@WRWkp%$0h@
z!otGD%iA-a-Mj6_1G(J@rAq|O@F4?JWD?F*3Go<&vTYA5t^RTQP-_95=Bx_J>f)y)
z#v`$D@!A5ZUz6&r{7SAZJ3f)()_BAgC$N8oqN1WA!U@*=B7?Jku)?gl<tj*weKx)_
z6+rW9<bC>@$4ozzUWmkfbeXJWQB>~0H`M`pxN_AnSt@(z=ZV2ybJldH1#PhHKQ8y1
zMWj**ndwu5XWb?}9D<=9LZp%<S(W+``jIMEDkkrn{vTh`X6UV2g4WSzlY~JBogsAo
zQTafeV6n~&ri$UO499pyI&}VzAN^=*n;x2LuuxyGo!1h9U08DU0kR9<<7k4_TTAHw
zO(=qq|2vZWIYnjv`$0}F$v?K=;MAF0;&b53w18v4!8?lz!8Ubxfa&*}hU5Ps>n(%g
zh}x~~pn>4-uE8z1ySpd2dvJGmcMBFAf;$9faCdii*SGVW?>Xnkw+pMGrmClB_nzJB
zUiY==R6}Gb1Ig(}X}w8Us3QXY{m&p{In20|W$3zZNdqo<%WER_&7v=Bge&Rlm`o}`
z6mUc;kK_S8u>XdG4s!UwpESn3*R>l~ZuZX2&Lv@!!{eiZ##V>*tGXq+QYYMkZl>zT
zQq>4awh&gc_cnsx%4lXpM@}~<Q7^A(-!~XUL)=LH<;%0{_L(pB6gD5n2i%K2zbURo
z1ATt~9j1ARsQ>*N_-$&xhJ*@C=sZR9(tb_$k|%qctvioY?VytkJbK@14sYNq1&_ny
zef{t`S7wv6SbhTruy&({_bEr(&o!TahW(qL{mZlC*ZyEbptib3%K!gaBMTo-Co=Rs
z367haV2Jp<e`XDTe6tM$E`<Lb!$pp2@;}u7d%9a(fQ5)i(-I{mON~~8z5zxf843xN
zrQ5(o;0C>me=?HVZkI9lk&H&!UO56famkO<%t|v|^|qrYm6pNhrC878B4*+;ITUwN
z&L>dn>G^RFZDX+GS&08lAFa{Rfh|)SIo6bUaZJ*XNZ}1MV*NZgp=j5VW%k+a<$D*|
z&y%zEt{SHr^j69%r_O;y-#3xgGgCJ3RRAYbYx~Nyrdnr}8f*qM`o5~A8@jyzZaIo)
zyqP-N<BW>r%#Fj}o6BwY_qsV-?P&aYHLWAPi!OA5%*cE7O&9M|1pC#anSiWOx=ks?
zch&ok7owZqraJNfX5&B<v!gGBueVr>YF0lir!a$qx^fQyC*di7HpzbYFMDq4vW?L}
zq8I9uh+8;$0X)L}{*PC%uG8wxNZ)HFD>$qX#g@mjTz<JN-xuGOrA|pHT?F*>K1n!%
z*P|KyBkY1dr(paKSR7N&bymInL+Rusxv`q!J;$Tnmd%|9I(T?^W*@(X$n!}8q~h^I
zp|VYXJ4$ExjmR-&cjtWf>op`LxZ8v)F1}#mfvdq|n98Hb{Cg0s@3dc&Ha4}VtgOs(
z&?L(n#)wW97z9dTUDKs;2r(a;NE3*VrzNqt({yw|HX;$OAt&Jq;_odwVNKtF)e4<6
z6n4b$^2f}Y6QTcnBxil#>w6)ySa4SFp6Ep26U;hgc>X>)ws^oxBDIk7@I|kRy3mMX
z7qO!!{z}dIYZkJX*CTc)qGw6zumk^vnBE0X`|B;|O6Q{x4LnP!dBQ1gPPP=el$qLx
zq}aT2%N7a$dq0vgbI08^L<Phzvz(`Lmj2UM=bKqQ??9pf0nh=VcCc*?VLqebLb0!J
zjq-PBW7zxCJpmtwu-}7SBM%XNkHj|oV`4_SR@QHU7qxdtukB9?T|HfI-n`8%)}><F
zs_<O2ZjTCGBfU|VqeO2(`PJUxl{XuN-iV#?nBp%tPvwk033HLl=^B!QsK!+dl%xc^
zo1N+o3%*fyZ~Zj2?=J1(R)hRe^_{~CJ}t3775jV!3%uU_5hBf~Qc0?mu!<<msA!CG
z*nQr1A0FNR`1?Lj8&!<wa%pWEU39(U4WbHBulEcJTz%RGnLUB<L<9r!$lk0!1~0Ba
z&igW+Zuz;X?~aUk{P#GxS&?P)CMYpMV1KlVDgBh;v)#h=I@SYdvkLzl6?mQJIj6A^
z4+dtry;-;rIg>>&|Iqmo`x@Jo|9VTy>swDyP4)LT*&VNw%zM!s9@qHaa1Qh6)H+8`
zvisnP?>~~DT&KhcMDqjP-l7*B)m1QgdaKu0C$8#)<H0}Ywk?6CI=c4zi2rzej;$Q~
zX)OL&yl`^Ji&NzAqU}(x8}phUUHh3sWRU;9eMV|NY0&24gLrPlSh%X<ch;0QUmu}v
za5(3i8T8ksn)GeZ<>sc6i-0e>fXS(s8^PK{dr#gtb}wV{^=DUZ;!OR`N4zY~8UL0e
zU7zYw{An=xj+R(gZizU2*4yZZn-iiA(fU4EfAskxRj;LlfwgIW9>B-hiX%X^cItWm
zP5z};b>7b8C5C3-w{SFCn#9%H5R37QD}c5!0;5ON+X-S1uc@Y>h7i2NiI9LKAa}eI
z1^42ENqTSu-}*s?Q>)A9Tz%c^g#MW~uItW#P}i)`>;h+du3XdLa?1Tzs?cjxl$)04
z#9&M1HEw3I{0rB!1IONjZ^HKoN?+C#6^$=PLyvSX507VatjAVRT6!0SZ{Ni(Cmm1u
z&nT*Eb8J?clsL0DTC7()qX$C3Q#O{ev|b^KkKm3pTUrU>P_&oZB$pge){jEIsV}c{
zekr%x&-p%YxKM=6=^Vy^<brH~LFC~@)gn{bLxHkA%Q|?;IYZvBZA(M6edk>X`BTw%
zh3~W^L}db>tOYy<|A^R3Sx?6@D6mbha1E*4^_=*=yor+wS%mAAL9D3~Y9buxtlwoA
z``~t+J%p*wMFPVgOJQehF7bOq%0L-s7f&&NALR<3dMdJ<@z>U7(h)cz(ehZ9V|%pN
zg}B5zT-F+k7k$UB?i-0!w1=wo?wu9d6s5QD8J9~r8lpEme$B~&g|6x0M+iL5mEe2F
z%R$epeetj~^(pJK?Y)#!WA7T2i<OC3mj_*yvl##-Gjj0$=s2^G?u03!gi*wGaMFNv
zy}9x3yUODdVJG06F{PhG(BzxZ)kTm{@cC(3Alr0g7zZNAO@NVaO@^{5u)CYZWnBZ5
zez0ylsot~ZdY0d9`m4kk&Lo8Sp3uQaK6hX8bA@$h;MvKS>I@reD=P?n$D4hSp7ChI
z!=<mCN09ymzphUunt}%b2JU2<W6RkCzr;=lY;AA#<=jQF*7p<{i@S@N%TM?o8yap~
zvH}nC2&lF8(DKextgGFA3z6fKL(6Ig^7yC3ELtB3DY%wd!D*z$?#I#O6zK)^66*FF
zd>ksadasR+-9)qz$Bp&}!_)#FdcGh9B&0mnJ;mDmfa_Xq+p<xS*>vjfbE1xp7~y(n
zc`7*sgbxT)4!;6_ViFIPV#RJ<AzTS-_+=eCTWe}4jH_YO>(B(xl`|LLv!H^LW@kB7
zZq5&9fhJMb-L8>S8sykCsNCxKge^o6OvWAN=TZ!&!ymy3vIv7(E%_MV-H6D@j|A#N
zUA4VbtL^etPqrG0H+r(KFZVg#F%&G&oVmE4dl_b<Wx%$b2)t9OU-z({>I7a$?P82C
zU7V-ak4eArMPlynR<9dN(&W3peVM)@Y+WraLVwN>vun_wj762ps*D}OqrqN#!L)OY
zLVIw&UdiQGaODoYD}sIB)9aM`{Ebav!EQ3+ro_fPzojL~pQuaJju$k^Lv`d3Ott-C
z%g8UnHC6=c^BgGRmZE($LV-6}Oj|%|S9LPfa)04u?Bbo>=zEUoGx>FTbyZZv2FkQD
zl%wkk)xjHWtepNf3UMovZqUvcT$}{-T}Pz3S1MtPIi&SI9(~&B)F*?e>!{dN1gi7x
zGw0+P1=H$4Jf8^qzaw!9!%~n)(e{@i7alw>=E9^eY@GJzDh_bQ%g{`xoq6jqSS<~|
z%ym7n(A5z<IgxO8W$QD)Kv|D3H@CITr(v{f<tx!q3ex5AkoVjoIX^M}IFUfUjsUTW
zxgWF~V9#Nyq~z*<WIZ$W(QZ6JIXI(zgXGL<RljzC)`r7tJlB`Ihxz#MeM7a2`IOaw
z)R@4Xk*QH|A!iU@Xrk`R+bqwisbeXYswZu;I5tFQFO0Sqim~yock>(<EiICkqQL{R
zwe=T?`>Tzvf=!VT4&wVn?8;e=vkk4*pf^!_<Z^A+&HBdGb<-2$hFAJ(G@h1*gUi16
z<syk})0*hsXkc0*S^M3#o#bv66{AlORp&MCe6;hSgt)oD5otR9*5tW7DJ<hf`zyMq
zgrjCUQb#rZpd&`+gE^Zt8*cVhr2nfIo=1lyd%E!WGV|!yNyxM64&fsRSH+S*w~Ozu
z)c}ZtS3`9)R6x<y#jm6g+rm4$mw<%k<OFB^hR^3gJ*}+49sczin@l<)ejbs#wcQvz
zoA+i!4|!ZCy0am9rbENja~aB4YBK%C;`y(HR+BTDg7=;8b-BPR&51_RPAq1fff$3d
zQy9=!J8XyOg5s=YF@K=QLdHAi-6?v9-pXSkh4E9xn<2#Hc5XdOFQn>M)cZ_!P^ho?
z5`?eZc{*P+SO7%3y&EI{!&fKikl8V)*Y$f#X@%Ij-@no8sy<7|Lu8(>4TyUpxV~;5
z+&-a(1+p|=%UO3i(?_gG-k+_x4LWc!toBH3bcOTg1x)%Y1;|vs*8UZod-im@{b<Q`
zfH;hrbUvpCeac?DsBQC3;)6z5N@(C!Ux5r>xCz~)4a}O#Ur;wRh|*XtYkr8#)T9{R
z9sEddBK7%+T~f9*WaGLjot)>Fnd_c27tK-Aogkpp1)-HSrKJ0`R)>FvFOJHAoISVj
zO;N2i#O8Zu3Cr%!TMfponA2U}x3j%S;{3apE3rh!fTU*M#<!8b?wKkD@fbRxoLX)1
z4;sr;{Sut(E<t43gUoRhI=&oDw$`6g;+Y`@`xILf1`ak1hxP`ooyg#t>N;otBAcu}
zOv>ZlKsk_ojK2T$ldK5l^VB1sZzPhESGnCH_;auQ^^_&UR!71{K1A~1tIu<tnSBds
z(rj03Auo#020^C9C<3zA!NTKULB%ZX5CA2dull`g-v2{jif~pH-7pM4Yejb#kD9uo
zz}}JG{kQJuz8c%QvPAmgFzdm5=rQtJ%f-P=8@qwV^`VEQc`2(oq4}R(joO`ukF?AS
z?t!MxX9^bIcdQu#G*2b^{&&KVi>jUZ0UFX`-^Qq81xyMfA)d9!;kab84FmS-{HsRO
zS9*xY$m7M`$(^S?Vt-M-c?`aG?&fg%xezPUOwdZBZAZmRxvNh^SE=0rmVi$v*<SE!
z2Y0TuTb!#rm&J2O1XmzgojY76$41l|r=;r~P2)FpgJb-@y{oviC0lgs-IV=IEi#vF
zf~8mi2c&)t5L9V?`zvpkX;@VDZc*{Q%0sap<pn-#_Epo>aZR+>p$a0opo%@K@AUzP
z{cE?~&B_PwIMEx3-JzIP2t`c|f$dvm_c^zu)su_u8n)?fs6YWDr>E&g<MW&KMti!y
zKLkEph{1BP$(80MF$u;sD(n8%klkk08HHDYXVe{QPv|&)A!AG6$6NMCWXH#jgyVK3
zE{YNnMK&0z&yP>fsOO%qGN;xXV;Wso-)vuctAD!AU2C=zOJbHqp0;~za<t=iW_CTr
ziGHMm1S?|_63$+`7x+g!26r=p@0(2fvBY32)dPtwCjpQZX#%Aj4GvWz6c-@{;YhcB
z9FwTd<Mt+c{v#^$c#QxDH8^f1VO@Vf|0m>J&q(S#w2|hAD}mkK@3hsmUJGa_+z$k#
zrpqxih26xDF#TXg0e%yHQ+ZI3K>wE%8L2tjAtVB>l3(dF-;rZh(-w2*ojYIG>%~OQ
zO;q?JZZr%of_BnP)tT1(;my))(&oYEhnli&lwo>7?jg5TNvD`Mw76TX+ABycoCFjB
z-A^}`{BHsu@#7kuV`u`HZiibuOfU_vQyTme_pc`W%@~g6?JGmBh_0kR(G6~Pu*Haw
zNl;T!cCo93VApWz^XKJ7W_nj2f<KqCF*bos^iL97K!zb)kJ{<1zJ&V9WlwkKEc}?+
z;;w`iMX_*@h}lLU+MST$c^XsM*wqW=A1f|1BX{+`7e@aUmUtCh`XH6CKOcI?T&Reb
zN1m1lHo#&rzTmAjZ>%heX3w<D4BR=kUx_V?zgA?!%RfT96=Pi6wFG+s+@;zw0jOP$
z$DdEz72|H$<CUf-T((mIzk~`<P=do*FaQ846w}P>(MQN$u<y>GLd(-3(T3QTa=9Zc
zz_44p@D9xCVdWG>=Goh8k>n=BEKWrID#MZj%@Wm!UoZ&Pv^$}bL<XbHmhV?GMSRp?
z9u&@-8CW7#j)H=W+*iH=6h&rpZe${A8UtVNL+xtM;e1EN#00q|UusCQI4Vi>QuFOT
z@ZdLUlaE0xm~7m#rUWiaQS_*!>Hc+*sBGq<2m!>z78K-_LDXSkJ*$mMaZZx?e$=`x
zE{@%N7k^I3C(h2vt7KU5Yi{CWF^$fw3?BO;UtjQC2<chSuG@!WebvQ#(2U?v=YQC9
zzs7Q4<TrH;D}3BZUDH6Jeq!dZYrn&+^wzyTvl^<QD?YCYX+O3cwZk`ZcI2u{Sxqam
zu=a`H@Y;dBKd>^mL*&=CBEk7U5LnR}COYMw_`*3QxcAJWox3rC)a3|YY5nWB1y;X1
zqOBgmX4{5Wg0soy{Xj{pHjVCyx#MCpU+l-?8~g^Jh5sss^+rwhx8Q|<g#+b8!)7A^
zftPhob}0pWAB^u8rGtiHKVKl&eeTAn+y_YmReP(lgEu>dPK$o^CeS)w4%t0+RtL#G
zcL0cmDW!EU#*>b&?NU4b@HMcsZ0q(jTlK~924C(7%j>JIpMJi+`3O@<lrpf8o^H<z
zFW8PE@dc=L1z{^hf36CA-2+&QJg7H@dU-gM^tEzgU_)v%HE#x*mU=PY3$2GkiRE50
zyY}1s!M4+J&nxNr@}v~?eNu9LPhd~k!`mxkFZ1pe2Fijg*eJ95eGqvc(5dJ!ozmf|
ztqH9>v(0obh%xAj`rKo>{<zzEUJV7aSF<K%74xgk==)|+b>5LX)BYuxpv#HTfTQRi
z0(qbEaf@n8Z)2Q1DMk}KbcS!HaQO7CW7Q@_BVF1Wkj2_oYl+_t_C-YZ^FBmgOrp*Z
zZ+K6bRKUVITpwY%k7f5Mr3YT&rsDvD@#2{o&x5}|+3EY<NIkrmluJYHo4>>);nIi&
z?&IVc+3Rk)x3Q7^c8c1n-npMs(0M#|QNI<aiS7mz3&-TVT-(iLa4`MOulop6aHm(5
zeO?gIcQRpJ$rRP~+U8*M^+tHY!5Y-Tbg{|KA_;48H@Xo0U_XUXZ19HQ8sVf3@_h{T
zRqVVBW*q17R~;P@T=^T+<z)+%slJ;K<AIHjPp)5~@12*jwQk$KI5Zc*sjg}uosF=e
zYTD1rJ><s&*&-idgj=D;^(YQ~1-XqaYce~#$GTwxd*qa%NylDpI1x*3WMg~#4P&uE
zi_}O6Jrk;2NMv@0EMsnGv?)Hk`S1)kJ8fgUQeUJde#+lv6<t}wfNJq<u(cLGl8uay
zxqESwm_ZK`lr|BOY@$g{%s*Z;PvWEw#~y_ZcQe$4!$5j4Hym3;f&sUQC$mF6k?SNf
zGCt%>zjL-8{z@^dZ$`9pg`Re*nv|Morkg`h@zAq#6_=XoI+w$7;yoZc)~dExTuV<-
z_@;`a<g}ZTqLN`JB0owO%UPv<!FAUL?`@sc-Hp<tP1P@sFrOgILbg*#Ab+a74SrP}
z(;1o3HS+|4ojt3bY)HyJ_`Vd*!o+E?QQz{kOY4Cjto`f}8b<LqUPIhH7~*X7);f4r
z4c%~>-n!M^{#FhRJB_+4{l4TSeQ-(T#+kov3MR1)0*C(txgSK`*yBJGaXkr(6R^_@
zkN6@9_Pdt*_{)CR;`wj~5$oAo%K~sVMQ+=?hW5R#={@kD*oCuIw#;q@vs`B8FCeTd
zN@xeAtyA8lgG`&StZu$x`4Ek(5Wyf#&0qAUo%Eq9{i!VDed%NX*I0iy4)<7vM{Evf
zRwF7rdPRGyy!E8A`-W9xYr<b6Rw013WB6=_niu}S!>`oY^1H61LvmKajX1=4@98u<
zTG0$i#A1lH`gad=Qz)P~%PR@J2kgd*8flK+<l>+FkMjKd98Q(Hdo|(X^6a7h26bJ2
zTvPNa%B-R<=a6sWCo5jqToy9zg$E!yc5V{qmOxFpJEqBq(;M0OB@9Gz##!-&;iToz
zt{=!A-@)XIdw#k`<l~~9Ur{3#0~{o;PU8Jn2Zq`9Vr4W|74EfHUHDwzo`sEcR%V>b
z@p!UNY#iUf;^cyW$=rt}4~APYZa6F>7v&mlR9@L9pEOQF^b`!mDeEDb+wgHj)lu<u
zUsZPoTr8aDWJSz7qpXRSh^TX^iuxeg<y_QDxsXgFCpV`Gx~4d6JkpA>VotLwDq$>M
zNzL;gC)wNm;$yK^k>z}n%~Q=&*REH0YLbJ!kD&@TBO3Xy?`9W0U%@rZ;iep0riL#p
z9SaUFgm3lCKYMc_qQ2i>yr*=1v~_juJ}z0OsB%Y7F?E5)RB6P>aWl_<O)y6H6OgyR
z51f2j?Yy~{ktfTM{54}aWp1i)4ax5cKIqCmE>l(BfBo~eD*z_IPu=ruj9BmM>}zx5
z<Vot(yzT-ChxR)|XD3T{9&7vK>(_a&&3F(PxL^GB*j*%dxi`OU{dL>&>fw`T!HF-!
z$$VAYquc@)fawD?ceTA%B67)uAab1tOI;RApC8{fE02$`H4*@s9!pOT!JXD_FEO|4
zE^EVa@m1o1@w#r`p&)r<ZeM(215f9}(S0`_xK|CLFq7-eD8k$-BQ;^cY1h5zzxdxu
zNegw2TbA)QZ`*|qslVW-N>hW(0tVHPpPcQ#-VN9@DY8NOX$M&8sx)%E@q+OOzT&bp
zd`Gry3P=(9=G#Gg!9#J9lvC2v_^t)2yh1MN4_!r7RScaUZ@xR#AJ5gP+L%BI=-)`T
z%8?`oc*LEL(O81&zyAC*sG^5UCP5FUmkhCHm!U}>Fwfe^2r>rFIx5A2`+M&)L$Ipl
zcHy^dTc7VRQi*|)!9&QgRFg;hRZ^E3&>$Us96gId0D1%fdV;BSX$Rc_62^Z9ot~)D
z55+Fayelj+1R-QhbUmdcC=e5dYu*#*%||(bFU|SWt)}_AE<&FyrQ<lt4A9yB-@(CR
zNQ}G*!14aCPV}!-#3BSV7VZF0H7MeQh|8Ij;=_B_gdxiP<X_ZbN#8YoD%|l^xXkCF
zRQuNrsHs+wp_^<!j^7ZPRsT<IDBl4#%*e>-AM^E7`{E5R)j$l2e5lmOk^7sBX`&Mp
zw_DXo(<5ppg}Q>+-m{nzde0UUe{QTIwJ3Y07|2)>={BB#D3g%xzpwBILjnLxG0)<h
zZ|7KSGlxYA4V5(B@4B1MPeho$8n<}vg=O0R)9Ulhq~%LxHNK(obxOjwvE9dwt<j++
z^Monkvt`O*0**(lc<Z<r*AS3}C9xi#sR2j{9V&IC|L~ByRP_tMRRsvdf1=vLPuBwT
zs=#JaY+$(Wn11z>qu>sHODp`<34bHDkSTV^e@NQoPQlUor0LU#!#NvUQ*&N>y0n-4
z;Vj7Bt*AgG<jqez;xIKf%f&K8`<eUGP-O+ci$V{CmV~mfAQ({8m1=O@x9D=*2o*IC
zN)f4)wE=H=p64n3^8=~_C~z^r0u7yjfCKMWA^Y`{H|u}B0H+X{W}^oMcd=LfV|v>n
z<wV(?E&Q$sdaJJ#oy|`&t9mcQl%OCf7JJ{bz?rUv19^P5cqJBUy2SKR>#!II`bW87
zHnv%%GbWP3USlbybv!CxH39?#=Oj*ePU&NF^O7D^X)>-6O`Y#zF-p!Wf=tSqI#GM#
z=GfI>!g<KOy}bZ0$kNK{>({SBCj?Zl*B6tTm4_PDLe1cz`|cgO>mF7DN1%>k#!vtT
z7_Uj!c*sIesdRx{{NRe*`Zs0qNE6vLH*}XGy%Dl%-XQ<li#&Yt{?#CaK_F9&Asl_D
zG7yrn&YGe6wc1<$+&8940SOfs&heZ!9<vfS>?y_U2}vt~z;|k5v};cspC&j3a%cId
zmJ1PXOcMM_#m3f{kYx4Ndne?U9&P|q{DXTMV2=m@45g>*)GppPqxhDvnH-f@vz3X*
z$gz)L9-8ybze7TVCEjsOp&O=5Y=1*f1Z11K;1o+7=Wm=|7)$=BUrr8(9>U$DPS7Qe
z`z08(H=%VcNsD+Rs8{+avzTmkNNU49Q4}S~XV)3f^e3T_SrSsD=t)Y&8<moG2~!vD
zz)owW%hCvaL5b{_rNN|(PhehhtT_||oE6<a1K@F^)Opl9pexBq<ogxO!irU>!_sUJ
zqqMiO%tT?t(5Ul+=FSCWI~JS~J)D2v)n0mZ098zO>qS3xbSalcG@m2)!fj>ihxJ(V
zhHO_kVU~1qY<L5vL#g5!PJBe=QuNvieHI*$F-zZ$Jsl=oME>TTd^3E8thg*Cn(YHN
zLu#ZTao9}yKa4UqJXl=fjvz85@HVH*VwTUbyd|^XBeJrv1PtyJ{ZR}AsPo7$aUkQV
z6K732OaR5{<Ll?d&G*Kum10&zC55_mDaz4f*#In7K!+zVODfbyvb;+eL$ah$V4<Xp
z|LWJF5;(GQ<Mtxc36T{c%gW25#>s6*f<~ov<WVn!BuNZvwVCj~^?~c=f1OOH(;V|7
zO^gV+`i*Kn>Vj`dK#jA#1v4dzt5mD+iaSMZLHdqd1AiDz=>&;`heL6EW#B5`?i(2>
zOo>5-8j~;_(4QVE?^Av93j3Bye{h@_D-@1~C?RPoSi8H)Cgms0f=Y}%LVScEOHi08
zCw>KGsLy0nhaC?Ytja6RnLr!~X6BD!A?{w3cq5>Aq-}g=^RVO96g_oQLXUHnlz~r;
z1Gj#zTdxJhh0=|7PkLPW<MEIt-I!BW6Ck~b+Bqm{YDcJ`nDa0R(m|w{=p3l3+Ts_!
zzIwTwuX3FX0&{T2w|EyFoHX+bM;@9pJ`P`BHtl&&SzS$oHC_Vs)Rq-lqFfE79lbcH
z@A(x0-+`ccxa`L#3OVw!?uwDY5{$5B&qMoS(ATedoLI{CNNdFkTXULUtLZ5Q*}7Do
zst8!3nsWDV711LZB02W^mD|7A^e7DR2)F*kda$J@-?=!LJt6_WqMOf0u(qk6eqQNa
z6lg_Ld9jdx2pz)4n2smrqO~&{;if`!(ILShEG|ET7s-LdP0ViKs5EPTfea)waKap1
zJ=v#6PiG))9A*O<J9vn%UtIliL`dYTM{>w2WVfUEYQe0dKXsiaa<O^k27qOE`@$2w
z0NQUsd3mNA@A&@Mm|KS~(Cq$?lO`u2@!7_P;PB4FiJJgman;w?Z{Iy1o}UjGYE1y{
zj>{)6V-_u3JUijy8OPg0B}gITwblkz)g*|?g#{p8U}rQP6L{v8Ta@btUog2;SpHvt
zkq+Q_O*T5;!ty1ZoiS)N=BMZ8#Psw8mMTVDovo&h>H!umEdW^t)>%p5VG)tuLS?B&
z+cjaK?mYrowENrI4|5(EZs~<;inZynI4ZejRwOZtg6c!v^D?$LT!G$oepVjZV(yc#
zyl0j87q}FalAL8HOPZwy7w>_@*hA&E`Z|^2syz~RBRJS6W}<B|66fV2_feA`wTU&N
zxY4SDy*5@g9v)n4%L+^d`VJ{^)sdSK(FIxDfwtacCzwZ2AaXMJ`YxFX$*?54=|6Wa
zeKqXzu2Fu(V;$=*NXheJ?zb2QG5jm%d7P~B3+b4vTafl=xaTc((652@4fQ*SBbsm{
znr)5vVUru{<A;(&4ALz&gW_L`m~;l^S&^lQsSsw*m!-8F{8R?HuL`?!j7Vc_O6NK)
z!}6i8z#B!8jaT{Tn}xr=_RFg;!UsW!Jnpc6IX&XiBcxQ643RkEoa^~h8B^JhL;8-g
ztYE=3*hAe^RUk+}Tv9@ZN3NAptWe04I?zLjF1BBPfNpiYfI8CW_Um4#9+!acodH=)
z78ecbxA1voXDoy7XZ1H0Yo#ENuODgg<^3I+<PpeGTM<1Sz{!b&NH>_^_%P5bqg!oG
z3~nJ`6htjNHV`9%eSBVSY{8q*mGv0~?QTiLUi$`kMKMlJA5*{^SLgkB9Wr(*$wU`8
zYa;!!dS*Bz(;RHu%AxDX1ue7A^-(3p#;i+6vOOyuv>*e8g!96eJRdTv6Iprv@=C%s
z%f2;Bh*M)g0(+o?J{iiysMuIXfM1#bGUjyreHSnxSu_oxErXDJAC=&t`6OcS`%ms1
zS8Tp>dOgBqvabc@Q{!_vL;-ASaPYUcK!6Beuz<H%yN5ZtUq|I#1+aPxBx37qR@4FJ
zVw)?G$-n8WrZ&B_6g;@oug?VFBbsl<SL`|Q$rGhwh<sCDF1nB+O?s5vot)4CK0JMs
z^?+ebTxMqE%Gto41Im|RM$fo7`NyHdktK%{=<88Z0}HhD0u=Rcb33v)haM(zZhK#N
zcfOp^!o^?8ijG2?^DIjL;v{OrFR-(YTILN`obI0IX_Tunxc(J}+A?>dxzQY;;X6I7
z*@|oOE46b9`8O{uXq0%ZiZqKr2cGJ$0qRvy2fB4d@jPL7rz2j&4G;Bgd!JWtv-q9$
zsymW9GzyE-a=wAn6N2@UX1G+;M5k__nI7!^Aw*oQQT;w~{~!nv0ZkGMjejZjM`!qh
z-AHB>4kSAK(>Loi#r~Q#v&#@PP3*z~8+17_!sk<Js+ZTpqEFA;9xjzyLM10mgAg)B
zXV)U*#&J+6a@106+2rh;L5&^=&jySKHx$5K(5*AT*J{b(8U7*})08)~6wa<Wz4NmL
zCh7a!H(nZ=5lGO%js4*yiEY>0y%t!sm|T1HwQi^Hv1hZ9ZP`r2E;c@V5cH$XfnBpv
zKf`s@6FIBY7kaVw)fd#kRf*;zrxDar2QWy4Wjb<Y*l8BcNGna9cw=%_lnM?E#fGcj
zb?4$7kRA?>XV`bBI0b=TI*=W@kkJ@aZ-uNv%NsbNse1Avw){KqyQ_~bv>ufqQBzS`
z;c?l7dZ5At{mG0esYEhX8$>*gRZ-*mz+zvt&3&GW4DXH;UFVu*6Mz8)GOb&bfCinM
ziE3!e>(Ak?g3xR8fBfta<jx+jZ8BiO{uH-vG$sserIpY}_rq~cv$9$wrWKl@(t&Uy
zzE6~iKr51;IN9X@o?Wej$=U8FL-txavJn>3I#?xiAhD=$2B$(D4p@ru1$@BE=cMK2
z5dN+5eoVpg#UjNsz<ve*PeNF)wF2JCNlV5L3K#ThlStk^P{%~RUPt!bK92tY%RD;%
zueR?J+BJ1Omj0K!#6qAr%G_@!ct1ZK(sNpCyL?MO;;G4WS)sAj$qfWlF@c4slCijQ
zP(lI*TSQL4w6=Q1uox<ZEast2lX6RKQ0)&iN$68V^EPW626^^u$R&S^GLcGVWvw=_
z14%~4kNecwwWp!e6P;M)rYJs!gF`YZ?4whvgg+`xjcIwZNO7cutebzb+QK%ZrD_PG
z*+X<_Q4=xH(dou%$`+E1KGmy3#4ACV7GP9D!@E$i&sq7PBJ`Kr=5yk0@1CnfLES|W
zaf|uNAURNn@S>AGBt9te?=&Q-^uoX3?`mjHSrCT@eQU~6<FM)b8pQS51LtN~{*$h^
zMI0j;52XemKF50PlGXYA$nEm~Y(=zPw2n0ufr{vj_*wjZC!!UmSeR|MvS?O$z)`1b
ztL5A*{CB=_tp$Jcv53<XPikyAir=rIW}4jQwpf+yx8}7SNHhyJZhG)Ixxvfm=q;Dn
zt=ziFV`z%>9*Ux=BR1-w@DK(H<+uU?kL0Q-LbLcB#gf==f2DyAepH;`3Sz&yjJtly
zo~hf@6U}{TD<>TiW7b{x+qi{=V8zaWnvlCF$^@)8R8#g$90KQ=r6n;94XmrjzH=ik
zE<|Ik?mb+SpFgob1q#K^903H{<h(qC*49>gPV2DzFuD<7Jq29ol!;(n8yrXM<noAg
zqX3c_pqye85bOcCDyn2@V2WOPcn{Ue%Etue<naGsz*<{dGiM`BAP17100e+g@eB}N
z1yG&$0BB(In$gU)JTU2MwCTh01v&9a5<!!5b7Mf<hb%AQ!DoR-5<tX&_>AbV-Pd+7
zh9;&yl#yLQLra;MfFg>*g*Si~7B?=vXyt&zJtIDE{MR^(xudj3Ic>*ivFO*7q`4@H
zPzvanJ}F66KUtI|{iZS^nd!SIY7Ns-6o5t+N|N1OM2m{0qMDvIM}|Vs7Rm4%lmR3J
zpG$F1Fm4f~wQPpfb7sTz@G}Z@PuOT$hby2rv|41ubMF@4TUqOM3DTkfIvbtj^fM$6
z#T=|h8?=4f;3c$hA>$u?Ewfr;SbkC^gIYSLG**-l5AH)ELZiaAz^(GKt!dP&*d4fF
z*ieY5mx7@}9Ih({iI`hdnCG8ugFlg?Su%|B{;H2==f2S8K0kh5(rt?wYhzb&8Rrce
zS0Ajz`yvIOTR2D{E>4Mov_pW`%Qr2-sS-W+hpn}&q-0b~MNjX;wU(~Vpcnccp+k%b
zr?0HZATDY*{kpVXg9L^NAIFq;-=M+qn|xqmhlm{$HX(7c90`nKW*urY@%7Ant}K<$
z_a*kz=c1etRAdBp^(365lha>zf5oPgL~L}vxpc^yTc;U3P%|u0Gwbl1w1^8*W)I~%
za{tkDwL(X)Ev3t!idwT3PGU92ppOye;U<w`Dbuf}DVBtpT$zj1`#u!mfQJ~0L6Oq0
zoD)jo*!ifQqzNtKPSS$N^ZX}wyozuD9Hl~@r|MjNxeZ#<LZkM#IAC5IaP}Veq;pV7
z56_ZK;Bv>eF{4#BL0AvDd(5BGdXR0|%8De00dA0%mzSAw5u&1+HJfbd0cLRSgoJ_u
zqr4d+Kxod-7q+x40|=Npz#9vI%7ADK*aUKv*g;sxfPcvg>gwWFRy+);aAN6?!k6c5
zRItH46<RZX>g8f7CP3py&C;optE*Anl4eV7N_V%=^UG(@$VgFVXN)G@z`rf=&C|aa
z9S)9E0D~)^Yuml&U{Is^8%q@n6BEF}U%ItN#l$3}rb<XjF#{&{XmN7Dx#`9$50EgA
zoW4Z%2dvrVCQ`-M$Rp++8q--9b>_i`YDtjJ%QWfbQ&0Fy1qm7>e?Rhw7)6zKDeJ^i
zY-J&uC{4U=7PqG$kBEXrB;pHPLF&h#r5>G&kD*x{xnYU`%!imV?kG_1i1o0HI&0qW
zo`L%G26JVk#u|*3Zz8}xaFnYb{Q%1R1Ruf*3@WAHU4%Rkr4A<W?CiEpoEakp&9%X?
zwkWhllUWWg+wH~d2M0!77ROX}fx?odbTkexgwCya#jNV`p973IjkH+PKcOn#5JAdu
ziK&rkqlRA@|I+0&JUq!$nW|*?$1p(a6a>i?&1!IlhYTDZ9Sth#PJBtUnAT-zxYx*e
zPqdrCl1>DP;>22&GK>gGnV>|A14cyodc;lK31zjmgll0qTp>YZbQTcZ^C~`m-SPxo
zX^HUqu^Q*zCxUZw6DUgXbAu3Z3HxX%>vmVx-~}loV9zymw0}{J6)sw%J&M(J^<{YT
z4r<7G{4K((lsZC~owi_Z+%8WF&<B&HjWSk08FgBGO6@4^M+er$?jE>!U{R8leGjW^
z3XS692snCA_-ABBCiH!^-)mg4-7zA1!PeusRr3hsi%j^VKKoC@$zN6Pq&@<sg5dUT
zThK#$lyWEdM1*B~%~IV-KDDy0F7S@`i_JW5A^DDy#d`gW;fWpA>RD%K{k%O}p%Z$?
zeFQLLOMbYM##5(2IbG`%7@=e7tmBsVr{oosr>Zrm-NXpYfjDpv9^9$Zr2+gp7Mo(a
zwj<9*cf!KKXU#bHD>v|Lc^<Q0+fSn?*Y6D$>qoi>g+M+Ta3A#LZ|kdbB)g=hfu6rq
zxvf?~er=lh^%tgZ8v%_5^psLEPh1S*0mO9X`;sOR@PQI8j{1LvZn87A3d6nrKlKyZ
zLSNtSUr-dI@6$+o^|7vC;hblPC?#_>q?k})k9v1Z#ELy&>ZGlV7)FRu)eKVAg!DTH
z|Bj|hll!_mh$T{Zf{tJk{cx2h{>dNjZymr?{b!WN1WpVgK+OM4X-aZY+8OFucA;fh
z1L$VsgHMQ(-w`czd|^tK3jX6zCnM|W+XhJX|2IB?cCP*f$jmG*HqX}nue^BEw|xcW
z9UV2eBRa9va&9>iCrOS;F7i(e9w#2TSs4JjCKUF2Ul*2&C?^nFC!d6{j4V&i{%OJF
zQpH7FaH%s#w*e!M(qzfW$yb-XNMs{ei=5j3Ibz_)UP3xI8(@l04(`{3$3Yj<N<&Np
z(q?esq6Lx%Mn9@uUQ3&6>2eqPU0WVH06z5yz-&*|M*P3SPBsaE8Sy3m&#U?$h1KCM
z%~>As?6)sIfb(_RqAoqz_?MxG4%1<*s<%dys>HVj0y3_esU;cs-P*st0bryv{u9N3
zEJ`-L#~v)VybJex_|Do*(a&UoQT^gil50lCTCg-U8Zcr?Kv;lrotCJQXz>gQfYE*X
zHAkv#b1W$IKd;C{S5m$>5eTe+n8Y~Lt`GW53em^Pte*~#x=T%_6Bsq9@~ci!SzS6u
z&ERqk@inn&H)+8z+==;608g%fRasR@BuDE9Y7JH{osykjS_K)^M1)i214%NxL2N2E
zr9<^+Zw3@GB^XHX<~7>{5IKO_1z^jvV2t73cE3t1#5TO(0@V3v+?b{M10dURtczN(
z87xszqg)>U%%}WZON%`TLEj<RIkb|lv0(6daH<%g`8e^j{&KO%$qW*=&%uyVlnm;C
zM~X(45(1Uo?c$l_=NLeXzYQhCPnRWg-p;@x-^H-<vWervsaciRSWH8Ph-2SFqeQ!_
zj(P};FYsq~2tZ)l$V-n5GnN0A6Wfa*i^vaisU?e7WBJ5+%eAWF%otnw1lgv(`pw%h
zxg%q6-YekWx6K|8lQ}##8+2pxMcRAEur7k2wmcbQ>J8Q|cA!}+Heyj6IV=}03I3OO
z3h9k_-TeF@aw&62Fd;G$DVEN0A@N=da;ZQ!9cNyX9x{fIkhj7E9bTzbdOr9kqXXCx
zWPIFw-@h`)H1y8E&gbfxrZ5sUvL724R~YzQ`tk)pzHj1CT|LuM*9QD<zK!q_gMk*w
z;&smeiULdaNQAtiq|5>w1Lb0*oZ=VhB>7k+ZKZ!w{TorXQ*X~jk9ej<%yln>vit*W
zqV#;&%YUH4hQzvTcZ}Ayv|o4KAMrQDid?wl*Z@&t+ciwEO2X1n4wsk2HO5hjEa4UL
zVbOQ?wa!ty#;>xZqmsUO{z+lThx>}fp>99Z*7FRMBSK25b5rt&%_6tF&Qx#}JU>J|
zE0kd1W0_cZne~CjmX=ycAla+^ZvOkK!wCJ1fci46lyO4)d{<~}^%HrXY8VhT&{Dk*
z(Oh5z?3tT8wh=ub);?L&cSJ9I*6q!k*y_wcx;!)55H(2GR1%{+JT{_3n6sLXvzjF?
zo-qbPAaNS_p+3u@NGG?RJ0mD>0sm$m;ig9KTy#%?{{AJOdT5DGOH2?MN>nKAQ7%R4
zlMpfr;1n@2#H;@*;Ho65I1MQ=5cZt_eI*(QyCPLcY9<~Q?cnFt!fT<BuLqwo$fx!r
zB~#$l-%iiE%hNzN$?p{?a4e@D&iYB|iw23JG^;-j>V-tBV%LI#G!2?Kr>2bvU@0n~
zCjINF8#ywoLj`ECvXrIO)r0122?PWrEguNG6#+-*4|!kda)!zs)0U^@ks;B=gwVw(
zYc<jdRb_YA`ZiQDae)#YF^p&dGxX<B&4w};kBNg)&pyyWcC;h=hl$qjwzkmcXOCNt
zH}@G#NQ3-$Iey<Nc+vo-7x!T&9c0t~7S-`eq*BeWt1aqzH41Dc!L%%LMfBP)b(PDv
zFDy^i_O<8KW+o;qZ`Lm2-dSF`l*bKzB`7H4?hnM_#e>1#JJJr{BDIi`rLvQ8oZmaD
zQeVhHEzjnZd$~h0kSJrKVEIFdF0#nKot2ZqYtw5b(8SnHE=-f6Sym9&snIp6#m1wM
z)1QUN494h<oc%GsW(FQ$c?b2QUPqKs(@h~aT9{klQ034IR{I-M%aUqJt~GL3I3W6)
z@!+uRl5s_rlvR;)tqWhc;TLUtomB*6B@3y3f`C$MsQAjv;!sB5$|S1YNVW!}=cuZh
zg8VP>PlC|hcs>>=St)#Er_69VA=-Xj3Ydu-Nu=r=NlB>O#l2~6dWHxKkj!jKcCo`F
z`f<n9zmLj}wb4Ve;9p)j@}#5;>4gIyVrVUn0Q!TaM;<w4I!Gf#NzlLJ;9oeHV>MXV
z7G*cFK&NH+ZV+?9A55EFWZHXHQOzFj)O7irgP5-j{oS5GPR`2SlGG`evnM_0(^nEH
z=b4TqW=SgrJbn$L^D);NU5>sfG#gQ6++z+77MD{<2=hb9-nV3&n2V!6J4XGr%Dc%Y
z@X(i9@+geemN!a#5Xt(Qz$X`|hezrbga?NcZmL$0I4}3SH@lZpSUUpGA1H2fB5V=5
zr5+KUk*gew!Q=*|ruES&A_8Jvt8NZ9HXPltrRr&x_bnIs9L{JS8pW0;MBEA?#7=K)
zy&sP2Cz<6b;cct%Ea||;!Ykh~X)NLNjI!;*x%LRY=Y>cVM9X>c32YNkf&qmWxSr+B
z%&>rDIGBzO@4g*aK;H%I5)QYA_AY0w*;7+WK(<mSxfBqVIXzn+&gZF1h8yj$`Paa<
zvnBlgo?&@eJKiK6XkPp^bF|oGCjj)A{r)X(X-U~hN66;^-zD%jhp)QE-4RW@GRFI5
zoibY<h^4f67C2pV)yFsUe|)>R?<`l-xns~u@l5m#2ROu8oFx7~kzu$tC1c>u4v~im
zTqT;ZIAah65PO9D0tJ6=vr#qfEY{F#HAdLnHPW;tsB7EUvDRG>B@nDQWbF2l=1lmu
zvEGc{@Ul*f-i-22u+pZ)Cm8OO7Uz)QMk^~R0TCHD_HAof{A+^bs3!dpdqCJB5S#W1
zxa0U{WqEjbnBUZd)v^9J1mQ$duAwWh9X{qx*Qs5Dn1D}SMl=70LbKwqN5cco%JCbF
z5k2qvZ)2IIC!}MTpb#GSt4SO$DKe<Yl$!d{zu|8R+lUrFRq0LkUeSY^b90`JvW|~P
z66#V`*s<c_W2PNy<Hu11v{ksitzhZH!vB2MBzx;7KYB7TE*~(zb0Om;7;H1-qrH}`
z<1%4D4#w_XGc)I22xR=Ne{|2MNj0uR-8JKL(VQ5OR9<_+(O-Xi+MEbplL;g`vvS4@
zCU)hlSzQzJ%A57tXM83t+{LUj6yrkfU7HphY+-*bO}5r<i(|71%K1H$bwo%(Sy;Cp
zgTRT}n^kF3XvHQfZVzS&zrej@hf^>YBgP?o`P%dcU3%7%B98p{;a3$ksa%2KHBP3e
zDvaSUV-Jfno?L>cY>wS)hEirWbrc5VvNk8}BkVlS*A8#6nTuAb{CC?JzK!q(Q=G5l
zFj3Ll3A6Ats*4V+OhX8e>8EnVghWKeot-%VL)!vw<P76ez55DvLn<mtkM-APf70*F
zf&O59g<(=gePs;}R!f8dSHzvo7nSn8Rx>4v<*L+vLurXKcMNnzaGEtkqNK<DOH<ux
z)AD;PGtSeJlADnhTF$+QDM6kVHg+bYO-bhv@-z>tQZ3A~3OWRDMm|oK969!Qco&P}
z8J1D4W|mUcOOnnOQA6$0n5Lx`HjxR+o&GjAwNs@~tKBAr;p+SJG=lgxpX-t_GE)+a
z)KM9#^OQNFiq}RYsl_9-N$CVZ+utf`BuadLW4b=cq5o(`(yMPD=+7mx8L&TLIM+Vk
zKa%jo#<*#D)Qwk^EL5BSxifrSx<|2_B~X|jd#@t@g`*>ItPvz2k*y<%1Ay!5;mdWo
z&K`d3KD=DLynVVQatHJDMS>0tpNzn0{$YzRH8eJs-KfJ$Slrf@xnj2=LK^?8t?dBg
zfdmE|h?_H+$N^$!5oYIejE(zxfzZX0YJss<=R`ZgljCC`(}y`c=<j4)<m{{}ehgg-
zgHC{Mbr<qFz<LC(2jiD}8z8nR$MgIXKr;s-uhQn{U4RcUNy<dvVZe6`R<|SyC_uX)
zmDQiUrq+!hs_5QcGVEw^cH1AZ{f2q<Y~k|7MU|BV)vntJ9H{vQ6BL6y=1G%m!PGik
z$ew+pi47us4vv>GA*Bpo#k_qW6L)t`{ax6S_f#XmLITi$hUKW^{Tk<y?NVl0Bl1GP
zeGq)At=v}^d(T9(pKy9^!dX%DD<7YPO+3~L-%0`tOa?P~NqS(%fX(D4eS^%P&7HMh
zqvpc7k0;Dg(eUV=XNZMH2xf}gN$H8g>&WiTD%71`P6kPkG_&UmN}XL)o}+hEr7YF7
zSB&5Q&AE0eb3;+C52+|ZuytEOSX-x(=YPEb&j%|wu;P<53JR`4qt^~m@g(E-&x#al
zaqO{cnUS#d1UvN~85FMLVRD>Y?k;KhMQ~ic5u)NN7C2$b9du{HMS4~&c`WraKQjMj
zkunp0i>C}XHm0@rd@aFm2?9hyQ#`EP{djvSn|N{{L^RF}ab^j#cjuIZ-&C+%PUu3C
znEY;G44W<eR9i_p!KvS;N6q4<rQwu4=hP#B-<Oe@Bt*%m$6(yR?!X8h`m@q6V#<N9
ze=u3F@r{8;%#&F^obED(^8^Xd!h^_QPz%xN^|i#n_DlPrWe7*uZH;gCgN?sCSL4pE
zgyx){%F03`SXRPjGXx<kAd{iB;V9xt$tA{-gGnVQTB<OcgQ?-Br}J-LMFuDzfkwuu
z6ppqYBZ12Ok50yK(193%La;s5^bI0m+7OhPn8_XvEo(VKWou!|CcTC&bgz-AnJPmw
zi#3|6`hPYRVfz?PRIDe960%L3uN^=VoJn$JXL}*P=qtnA8?ou{B)C`)bMHMFP$9vT
z(3xWq6Q|}vy7kEwJ7AhPmrVAO`2~n#2Z0QdeseY;_Lw0`tTjPYm^fY5ut8g~e9qX^
zRQ5X+VQhhNDG+B@ShzR@0#TBY0TC6$pv2<C3!-Mj#e8bg2vp2Ss3K)4%Q6BWU}DLd
zl!GG_NaC29pO@#v2VxU{m=k9e&mB)5If)i1Sz4V;S~dUyWwXZ|>wkUcc5KE?I>*I6
zCaDR(Cc{TQ?U{fenQt}Wwd?oUB5GBbmv6L^%S|>{VUJfnhkem!GLB2M5p#d{LQjN|
znly($bk?>`7em)fBu}7;;wI1z#F0?H&tz8$K#uiG3fc1te!jlO^QpY%ZIYFVMIW5m
zh#esiFKW<lgaBc8`^7u0?Ad)%re=^ADVj?xQ<Ha*NI?C?qCaajwH#R@t3i4gwRq}M
zPs3DbGL~G*S%veJZJ?!7DJob7lu3iSEh<3e@rtfWPqmY8J&&tGU6rMv9)Vc7B$&Yx
zSm+v+Mw*&@!ckklEU*7F?vi5R3Pz|Y@!q&Q__AB}YFf>D{fCH!IR3*VGBZlL>^-H5
zsj=}N`<(%qcRGcyX0Au{yc-Yr*=PjR7}D|x3utIYPM&Q{C35J?5pFdfOrcqy!@`+c
z+p}kPE4r~Ve_7OT;ZhuYLD!zHtCWT3wD$l{#GpIc^C<!q22sPzy!(2>G4r33p-Lem
zy1{;SWJ<qHQ+sRX1sC=T=oVsC&jwPQqS2kVkiNi7`9MF*P~o9w5zYMSOG^-wmx8@f
z7K83c>;;M)VZ^%Xfk;u4l~P4zbe|)|5l96C^d|#1h$O-gnOT%0*6_RC+tcs2%9TW<
zHZ3lG70rpMrb|g%as-|)2Jcid7bd7^i))3XF#B3BUvPa*@8dnD>}bPC?hdQ=fF+|w
z(c38q1L8hXN5&qTA9^Yzf$9%Rva}d4;ouGqgkbSZzIr*3Swn#uKDltvXU-;j#ve`{
z0W7bxmKZK=*%1*Az`8tYiUwq88ygp*!9ZTR4FFY6>Tpr;U}1mi2p~qOUK@dM0{hA>
zE01I$_!7u*{mF@sWoli!WDR^iKnkVZPYntZ2WeOSE-b_@QVyn;=FEn`=V9j(^tf2}
z2I}*UH#o~HJXL8_8r<uk-QAW<+2cRSxIc4_DDyP(X7p|~bxH>-zQnO%0V;;Pm@T(Q
zWqMYvLj(bPpJvh`h8eVCzS>toBl7A$NfCf))<zt@jMH9jz*sYQY4O%lksJl;s!TVv
z9-8*c-KU(ZN8=m^t*7j`D$SgOGK(XWohZmaa~6kEx&{&`u;BRF+fNX^op1=J8o1iO
zl#ixLNamEJ#l*OROU3I^1juZ$x=Z|g|9oEfp3!3^gGzNHgIYW+N-8R$M+7TEc!RGF
zDJ1&Ev*heDE?Gm!UyU!hOGrDn!bDI}|BtEi52XN;LbRH8@N#Uxs)G^>7p9DEBI~(?
z2S$INuvamgo7X82K}{hgzh)OC8epyp=`=dEWyrF$5>ZxIxN8L%uqt#rGK{KHF9+Xt
zwXdn=zZP*81!ZJ?!~Q0|Jh*}#8CBDw52t3K#=Fs!n^IF3gi1ok4R><Go?tY-<$1Ru
z6h~=CXH~mkIHPmo-}PnoH+qCaQWm#Q)=YF|RJp9`Xmwu|*3X8Vyq3!7loa*>bR2Y(
zj8E~GZZIYp*{U=6o~$fZQieic%)3RUMPS8L@_QM{#)o&t6LmA!yN+4b1OuwfvBR%y
zQIv4Fd)Rl3^V74KMprggAZ4pbO9@WTXEyJ`yr173HqFVrjkGFszlUiN^es7OsGynB
zu-1!`ez9IEJ&Y>d<Q<)ww^ubqJ8DT!>!;2+w|^ewwy-`fcpb3W#SsIyB}(9fD%W;%
z_cQ^|<%wwEgey=c-b6H{cKL-6ioync`v184$M`zFaD5zZ(%80br?G9@+Oh4%wr#6%
z<3^30G-_-&cJj>k{LXn^{NE+}vuE~bty%YVU#QX*9bCw$c?T<oBc!eJn4d>*1E389
zIeb*LZ<1R&`TS1FMD0P(F>o?STn0Dm4w-tl=ai6Oyt;Fo1(GTGQJ(jbk|lHSAzeXL
z)UOMC$v_BntR%Wr0+B?KG7B%Stoywm5d8<>UoS6enevhV^~Bq|1L$yFpEu9UZElJH
ziBe%<&%nUo&j(6w19J-tAn|GtNUsC3?tyWJ5<|)D_3uAE9gqYE0}P_&GfHY|{E|6o
zgAH1MsimQDYh`Cw-q;8WM9UXT!2f&w%KyB-fPjFYs;w=psfh(dlo9aWfip7J;UN6S
z<|<N-{QH-H0gJt-r>CX0)$CgikXc2QC65Ld=FHs$yi^KXS~9P%VL9-oFuTaSeb1PA
zHm#`;H<Q$I38a%#vce@wK$FW6w7y_GScszh{N93x`^xGT6|SFPift`(y9&uAk4;xu
z<Zg%*P8jKlIaO3n&(FHaQi|o6&b2w6Yf#g72x#osKYs9r2v5+)=L(9&OeFN@OT}-D
zC7~M(O|I^1H%z-ro3WUn3~~_rM=dbgA2-GPb6@Jw56~)Ob$ck@vb~+_^Z2nm*>Z=$
zKqiG<-rkY^VuC5rK~<~6!|lQ@&9UVA&kda^5WyD&RZJaO#El<wb8yx}PL9~)lN+9K
zYGq?RSVwQ{oJ9C&%;!#xPudl%pXPDu<5s8kdC2GK#e6^Q+--=Rw+AQ7){4-=F~7d9
z<ZFvB@Re!y<llYk{KAr$oZPzlM>#tFKe93k?$<?;Q{A<wqgpdFcf52C`neq@#awQM
z&rspvQ_NB$uc!hFtXMMKT8tT^(9iQ*ZuD<<^0R;b%DdV!!^)AuV`muW?+JvU!-8Ef
zlouj}4Q49*w#9?K3MKX(pFDFqq8?YSvHatxf<e6%i+{)72bUM<<q=RE$3=&i=jX+d
z!wnN02_%5Cbc}dkS4hrYPUbrQbc7rp5&F2^|5K;83Pi?!x}WlKhxltz8n=p0jimb}
z^UG~&z$!gFU=UoEx6+EAAi_7!T*rLqCmx@YxNPf`V(_8gVzJ=$+}THn7X8|BO3Zj%
zJGe}e3Ye*Dzx-|Yt?#gNx#nMJ3egd__iehdZ<hcw!1!~_R^|>K<GvNS94sMfVw-QU
zCg}8irKKG&xPB<6NtT8=zGpuhCrqG*2^(y-sT`{4+?sA_+|SD-u;k0(1~4ZMy!)1L
z*?(r1ET50TxDl_Kz$(Z_zI-f;T~=+pUYDG7BsE6F|9ujiG^Ga<0}t_Z-X(P@=7&v7
zCi7Wvi82jSs^zL(WNgdQJV$A{`3^XP%U2Pqlri&V-22f{m<S|7(}VFeyQwWn64XNb
z?yySk<^-ZD47wYJ=f8>VX&4&>`Fu*>sBso<Dh<htdXNk^j1a54{Sb6k3zv!uEGquy
zIBrJGad$1RIvUBU-W)X#6!n{Pk`&~#0pz2u%}O9|@8|o&Iv9&hY<I;?&oZ}c(^-^i
z-7v|chxZ1CEaq{&VqfiqC1SFHe9#{&v49{T{L;MXEX~E#G`PIH+?Eg-*bnd#u)A+k
zl{0}o8bUEi>;LWizwgNe(@a$xmMxru448B534j4w+lDQHTE@n@$ATeC8B8-xhje6#
zF>hn}V5nV@bM7OQ!%dM=xMWj~s2^Xc?%I1<_h0Pa(R5zJiwWib{?h7MiAs&7MuX~%
z%WDS;G_Z_h8;c!GOdiu2*?GI$W=Gt6yQ^!C3L%NK;#k0kl`V6bUhYZz`t|GEiBe$a
zpKIW*|L^g+hR%bU8=ZCr{y(SDs>|r{I$=ciE;E?GBj6Wp%+J#NCC#RK{?K)~+fyx9
z)unZp_WzwUaGINBr$B;nzW;-cX1)HH5`MsB22x<WGFfP9s-f?hma#HE_Yu!Tdz&AA
z#F#T?34O}gG{Cq5)&*=?;zi2x_nz`(kYY5cvKW%&(K-Qg|8w`*R_|r<a&jj4Hm{Ms
z1bAMyLx7&3c1%%82YTzNpYEz>rt>6&QA!)?B9fd4T0l4O@MH2x$2GS7?5(ms*Zd4Y
znm+u@j^*>Xql{G~6ScimbNOAS2nDW1^fOWdtdh!$N%_`djDhZ_j7joT*IxV_-@XA>
zQ}IhS7OPDb+9Lw7@7sJh>C1^Xo4(4EPLdI4YMK4bM0RPA-3Bqf{QDMm+(A7o=<l&Y
z?Mc7Z`9ufuCCe}HW+q90<GLfFs;`d~c)ZK7tjrH7qu8xNVp!zh@ZaCu6UnDCXGx8R
zIfpDH1oH0}J?azpR&l~rXQa3j`UfgUz8qhj520su3t?3ly>LDzR%Eyem>El#6f+>B
z7ggQYVaza7Y`9(DOHCVPojb;@T|K%2l+)_!YQvB5rlQm-2|jJmH}pHMe{So|++eEh
zx56iAEs<U$1v||Cmj6sEI{zU#$SX)l<d)ZentpOapanq4{7kp@&DyRw_y^xT%l*AF
zOj>=USb^?ao$|kohButghXZq~qIrGcOJl}XdzIXg^+>GLAB9aY8j@9ir87aXgIh0z
ztDpC<=TZr?1mh-01~rXiaiQ&@TUSTJ*hb<Q5>3s`s1QNMmVDPn?->1i{Y`xMrCEK5
zew0P5)B<&=a5i*}y+vfGh3;FplX}aET$Yx8uGfoP#1)4l1;#tM&%cbom3+BMVBsk>
zc#c6jy;LtbE1wZ6-vTcc;g~qk$hZ^QQE6$)*hX%6MPX;3Uj1VoSBLp*Za~;YIyU_O
zpz{S;L#w>due#n|_Ad5@s&w`1S;NtWyU#Y`V*mV^heA0%8Ux`2?i<)#XywG8Q6CHZ
zFB{ZF<={0e(*7}7kG*}#%PDe>&_zYAxs{H_lS@h&rtHW84t&_W)Kbu1FIt~c&Euky
znv^=;I<c)ngI7ooCb2Z${p~SIjy0V$96W&*O?{lF03~*1Wkn)TM<@H%aMN+w`R~Fn
zKCKUWz;I^kMDb{^hIseeLEr7yM>Gec7mJ(jf<tV-(1t(i3k$b=HusM=+cuPQ)O~|7
z%M3CZ=+`X(pfxhn>A5vHU`NhckU&#75W?vX#uf1NkwFis@u=2npDPw5G!>OoDDOe(
z@9!@!FYmM$$F=Lk9qH!2{W|9vg^ZN=+yTv6Z<uhS;1z>{cd@hRTB4oNcP%M;ON^Y<
z<G{p)c1RYq<7#P*bdadV^C~%z1_&e6C!W&pmRF@S`51J(Q!QjE>RXia2Esizl_Xsf
zrHFlk^GPw={qF+W!1kyEgV<|g(FScVMC2R}c)tbiE5fB(K7B+WvEJ3o(lw;Gt|S^x
zD_rS_$ri_5`R_Y-imvOh-m~=^ywb0+IUJ&2>F_Si!d)+3mrSnTDmW<cOfB~<h5G7Y
zHf&~WzW`&vjw+szZ~EgMde&<3e6=~}`S$2`koY6CsEG7%s>U1urW$n{rd##%($a>3
zZ!*2mOV`{=`EBBw@S_&n42xk{Y8Gi~0HmAH0{$CI-doktrodE*8lqim2P)TnPkuKW
zVoE?Wpz*<{WAs62c>a^_I!Az1d^ro&%^IF|jE!tabO=MXh4<@tDYQ&1I^5H>oXs7T
zfbP=zn~E1(Q1gd%z<poB+@99WX5cff9)ZZTz<b}D%NDJ;FX`3m{#TfU%Mt~bIo{6D
z1pm(1ls>;u|9b%|Hp0L6bI>|!PIc%m@5vOf4a)$b1;`jBy>ml-+VPCav!-}2x+*g2
zKF7p^A@d*k{FM;khqAPGfRP{HbguVk!WR(PCXJQjU?|&t1rbqzQ*qc!XzrD89XIn4
z9Z)7djCi%X$yfa}u^c$5wg*1rNBnUS(xu!Y)SnR8RIp<XdG&tNQ_I2qeE%3YT7^n^
zCip}t*wPF0&-q3xaQk|*M7;NnB#+qZ`(rx7?Z5VbDV6q&b4bp={xdz3a`)F=4T0fD
zFB%Lh?p%+XO+lqK@j`ERM~jH`+K>Ir&U$S*)Pa79n*QIq1Y4l<Um5zi>5^!Af*Rfi
zFVxL@8++kY{c=+hKJ&LO1r%Nwb7XSfGKQX3Pp;*(b}xGP1qaohTT}E6hy$M%Zy7$8
zJz1{L{1Gy=tzS0GLTj{K*4MxvKAvUxgToAnU%=ZLILLq6K0nTU2KSnOTfgkhS#Ljm
zI(rWUs0LG2dkWkRe&3bqkZ&_0h>H{#|CwuVFB2OIO*B(Sp`0I$5etf&?eSpw@8{Ub
z5gax|<d+^Ts$_rc(05DO98GRRgw!FnMA<m>xFY94e@Gjg!PE%e5DGRqT*kW6>a=7_
zw`UyAVq)pekC8;-M3Wy_s%7)KJ4o+w5Tj}vstw(#CW~W8dY@SK?L0=O`FV?>k)l!_
zKkgA<6rPpeV;D&ZJhl9BYoS0DMa?u4*$EqeWuXqBPU@_WR|+_OP2K$GAow=TrjPi{
z@cA7qj;G++OzQYLWYEVs(&b~^ycr<O05s3;EC0X|0DJ`!c3`7pV!V&|u6|uT-p<R7
zSUX-B4@X=4GGE>Yct?P=BuN@rbpY*7fF7#N04PdK8*zB^kn$9{>;q2$hSLY2vH2IJ
z<Y#)y93?#<1~Cc<J9q^6_g>KP*?Q%C{4%e1eV>*kJW&uI&=zRCBgWz<N_Gwrw=C1j
zB3@=)I9#g}sZkZ6<f+*4DBTF2a6h*i__F$5{r!&X5xOsR10E2$KpK=+?{@}^7^V7W
zFnt}Sg4q%N`xnmy1slrYhL{!a+vo&0`?sr(fe;Zgkg=|#%frHRL?3$}Y<M3xe1CtZ
zy{#+tNf+PC5WOxCk9IhowJMIePn@{zy77Xoyw*9Yy4_ID%b@jrrkvYf7&lIj?heOP
zIA&S_n!bf`gRKu9ji7hGla8<Ia)8{0rARrJf`qadf?70ERH*Os@0HFgvCvU@ENcGq
z*yoKOU=jOt`kwj0GJXhsSt{%v@A!FSogQ#bOoPOEJrXVgcQeh>dnef2Wr<50e*3Ot
zSNrtK;I9jf>;qTtFP{mR8U7aRsfX9611f05*N>s=5ay&Fzu=IQz>z>d5Ij;09@^Kp
zU8%t0JL0|>AuJ`}!LqiuP@i=kk=h!(v12R}Fxvi*=;8S2jb9MhbNrdaGVh(c!003R
z@m`Q}@`hOOMsJtMPw+H`x!c|qI#NDd;mFpP`~^HnMJTFNd~tageFW=zqu_&@`Ta10
z--RJ&#{a@M^o(dHX+-Hqc~fwO(4!XI<ukO0j6cE61N-ho|H&m~-J#L*0bkei-H4&-
zem%1DtH4!}T{Yj<xx|UHx6b=V;O^j+;M7+mzx=$umEruU-YTvN>s<eH6e-1#jz>8o
zlcUuL@8d_QpI5G*zlCI>pGTMl!<s?M8j&M<VzJOxUd}?!Bm8j9)D0I=I}WFlw089J
zd2PKY@UwS-3Ri;u`KA1x!Y9AHVuv>VbAt9xmB(|lFE$fmt^K*VG=?%*jKJ>=*QLED
zCl=mdQG6E_vHEo5?dA{uGZ4wnEyagGBLV%5yimPJB{DMO-YfN1i^}&MEec}J3L)m;
z;G@B(C;#;H`OD7@a_u@z3^+)<xNP7(;>OE}i+h6oCqbNy|LvcnG7Sb`shU1M4o8**
z!`$N*YxK5T!T);hO2GdF4kXRf^LcJ^Fe2H+42uGWZ7q<H@pXOygUdJ$AKzYIzNg}V
zxx*6YW%NHL_@#C4&fJjl<;wjs@UHpy_?C|>ur#c6r5kYXYV!?4)~$%9KQZAaga+T$
zo?2rC6Ti3Ygux6oakiX);+8ocJG_f)RBQfW!l!34=DOqWGtDG>=3!8$7ez@(^)5#u
z#DEI@LyLYaFG*MG{BE<j=`tNylQe}W1zP{NMWyGM8Ixvh*i9d3OETx;MjXL{t8x7B
zZgDyf<s_u=JDWe5ioG|A#bA=9Wwld~qvri&QAa6r);ow2W|2pVtVpRqq|Ri{%x=8q
z!^K6p)AP4K8D$E{e5e*=D$*wLnCet;vKMa13W?GNLv_Eu68JqdlwvcQ+V8;B&*3@H
z_5M_f3_|b)!}VYkhhF*o$KH}dK^Dbwi0k*0jg*!pwSjB|?gi4ea_3qrzM$_4?%JCY
z$>UN#NXL?dt1g@K#ZDdy^E4Mc8m&b9D{zPu*KKM;U*rM=CF6oQY#})@t&QVVN)UGu
zuU&}$)eH!GKa*jlTNck}G|JnMqV)I?^3~Y5&`7hRPE>(H{N334jGW&)Yzv=JBK)lZ
z_ZaXlB$A*S*Y6x9VdtqQh~xjf;P2}|oA>p@EAA}8K{sjbw>vW%G9pcJ#C{)^$HZ@3
znq^AsclI*TFV>a5LFuV@AN6#WYd6)q1V1FJXtkWEs)aL7v-pKtQ2mmVnkI<k$Dip_
zdf585X8J=SvNAKjP;KEo`3X97U1-wLjWg@Cx3(g-uMZP?{tY*57`gCRSX+w+8U$Ef
zEx`a$RHqrlNw39plqB`>>qzL+8j$J%`V$~?7mFcN$=dn^bFXyq>>fa*3~np~4ne&?
zF5$J;11Gg|h)eK~FFn}Ap+fSIuhoYXbu^KV*2r^tm&I6i_M_6b_L#hwDWeppc^VAM
zr?SVTO)Pld0X;hV6I@!wS(L2OPBP=)PFKR`Mk=I-$Wl-=g3{eCa@4}IRyFG>_|tC#
zano&LqF(~)lH>RDmDwe#P_HxsK3|%3;~o#!xoiHQNB2lhOl1c5-hRn5LpiZ6W5W2N
zeq0BOW<v}M27c+V2IdBuPn&Y<`u;@2XsIU6l~oazVX@d~xBS}i`H|rCiI`Tp->{v|
z=c!tTzdQVmtGB<I;HA$j_BZpIV#!GPWyKcZ7uF$TB?8)0VMj*OM}aGl1}N3lz(AhH
zbyT*dqKw();4YTqUu;7za^OEjO)Sge^j`~x6YPOJawNO^4O_lM892rF|5W0a9$d@*
zs?ev1cPi~530EFa+<!UMiZqh_XnCr1*IjJe8Ix?T@pXA!7PF)>z%8-5-AM6n?+TyH
zYH=~oh}x+Ym`)nJ2!%?xp3seA<T@|*u^A`$4^f?n`&NRnj#=)>R*KQ&Awu+((A|rv
zk4Toa*qL!0cLwgh-SWo)3p2t|ljXu%)$Og{M|^U}%9LxI0}O)NR^5mVErq8{?8~$>
z*?S&`dYVdrj<4>2g<x9Mk_}L0(a?;R6p|!#cN4I&v$JL~<!PLkWyH^IY)k-j17~h3
zsds>B;e6F|C4~Z^m|rV}VwlfO!^xQjAd3LR2x>kN^#Z2AgNcl!yu2y)ne#rE+`kT7
zN&@#76X}dOFB2l}r~b7Qm7=yE41xMjD!GKnw^QiIw~y|(t9{k+W)qZ9_yNB{2rt}D
zg$6z0KT;abYxiPof`mRULHRI7?~^$^nAS_S&xKeYd%i{o6=oun&oNJ3kB)=Ic;qb3
zv5o<E47!x5m(Scrz4l;*!e+B2{(RF%xc4r{*Ad=NW6jcYHiEZA_!f4i$47lTMjzKR
zm9B8!ThyN?2Y2*{lUtoh^`G9M%zpciI88P`kI+5$W)%xL@s;U?%a)2U7bo-b#~qDi
zREE~pC06IeRv55;3px>7N(#WtEnj<S>GdP>-Mzh13(8+-hn2?Q35)FnKl9%U^&O}^
zzQf)&9n^j+EmA+ao^IHf<Py9!38-lMVLrm=?bz|PL@1KxEtUEG%g6nd<8IsQ#Pb~X
zXVjKW+K)@3UfzbhhR<Cb)$T}7f7(2+pD8<=tmCC=4BghT4MI;Od5>@CS0AtV&jpU3
ziN1lDWmeZnmmj~-CdFp1`y1YZN)!9b`?_*cQ^6fNeL4Gb;Q}|H`-buvIC(>`D1{&3
z(nkabsRgy<t5$!jmLzNc3Yo|fMoB|x>D0-@xK4;=vynfNbrrBUWG1D)IPyCwAue}W
zmQ2sSuT;0T>s5WbJGH)kek~hbshzoDM)t$G$R4g`JyS98c>np+k49bXx?p^(A@JJl
ze3{;)AF0>pO4JP`km62pp#D6cq+w%x)etZ=G60S|<03y0_O5hc<`Vc#RulH^AOm?G
z;|cAmSJjVMC-73AnE3Vg*SAI<QBl=v^+y&VFDE{e1OU0oeR-g++`1~R@F12o{_o5T
z$^?IlnT(I$*0aFqNAZ+Imf^RjaNpVXUYC3!?wgWV8Z1~1++wdx>FEBnEig^Gz@xas
z<M}vt<XNjfm}Ku00qcG=`pDr2cK#zoA@Ea%xA)om@?8QB-oXdEE8120giHBxFz`_D
zz~%CNw5#K;F*vu6ZM6o7YzGYPZw4G?l2IVQE+otlsL#%4=I7?dh(DjG0fhiiKdwIm
z@LBxCL3wv~tna1w)}m($)rlK#sw^d48eEBU(?R|6S(G>#pbHGeHuOs3yYd?WX5*)~
zBcX!s*UPS|`g+iYqfMpjvSHegO)6e|_Yt=o;e?^D1<=Xt6j#hMf7&7v=g9oG`Ano3
z=k=Q0(*1v204AM!Wg6`t#$dL8YQ5tU!=Lt`iKt%tdVjsNKeG@3sGx5RWC-abXu-DO
z2~PhKwTUomnFHw}Pe45tbH>Y81tV$%9S4Cur=AWg0UaS+S}xDKV#3SshO^Bhy!kyY
z&KC@X838z1Xx4O;?jmhJN2HW~+6~7J`qNj2^pg-05~^!yH8a>gYuHju%H>5!So&XJ
zTClC2Q7Oij)KrSPuJgg3A}~x%^_h17>37UZ`E6SN1BtGlSC-7t05Xd2frj_EKJPsQ
zgkDL;WZcbPFaa#F=6NwXkZL*EmvfIU#o#ga>|1e-VJI(oirax*r|*i+>o3092_vqw
z|3ND~AZ;QB`|Ax-)YdRpqo41m`Y3Z@$SE6GBJbUSMcJ`BuU#ibOEnKCcD4Kimr~V2
z$)wNy40?|sluL$E!P5Uiv<Sii92`TNJ-*)CLC{}*oL#KkYkAHK{QnW$hjuZ0wk*IU
zKJ!~c_5W^j9+&f$T<3GRPQBY)pE30fDN5W_(-}n~NnBL8DtGm7#I@J%0CtciUB2{}
z6eFU19Rez7!5KQFYwod94*TsUn&|&fTp%FK+}vD$6z2ah+{s~dSdQjOEudedvpfO9
zhZaErZbHqRbf&#vt|JA@proj&&8yDv4kjKTO__S`Y@fgIxV>9YGCK1b{XhJ+iAOXG
zn60Pt_!`{#|L;S#RRTaRGI;m+`2YDTE>eUuZ{9L55nI+dolG>Dj5A0LXX5{D23FaS
zb15m4q)j0!DjO^srdc6YqKF1BiBhhqa)wYj1jD*hd;u&=eZJx?O+sx>3djtFe$P}d
zoU2rMtD~mR5|^e(RPGg+zvZP&k%18T1&CIWBIeHS0U=JyHBRH6lrjiZqVp5yz~XBe
zkl+yoE=m@qTd~tRvKuut2u>Fg-#&G)XD>!Zq*%yttlSr5cKbX2b6r@K3{{DQPW_m}
z1|ymxIgtb_hMY>omZe045*3}bnAPMaz?!AF_g~%!g(;aLL%OhN*rvH~T_Q@cvuW@b
zS;QB)@sYGkCFX_#h*Ua6pq^c_>!k#MYeQ@eKzOnI79cc{!Cde{FT;OsxxzUVIU>ix
zP{fW;&0MOQS#%j7*$oDf2K62WZ;@O%GnhypyXjzhi0;rMlML&3X_$SA$Fsa7rnLnl
zo06ukmrCBZBDb&t^A#XFD(FN`WAkh-POWxI>b^kzi&ZF;n<5|n=g~n18=Gre;!B|@
zn~pdY>);07uR^R+hL?AEBU>rHNKAr*IoFNJHE#}{(2Siy%s^Yt<u>@Kmo80le0~n}
zl#A!A00RaqG*}GkcoJ@36^o;y@lkQ-G<?`OtJuay=UB<F(OfHJ_&7MftT>8umokdC
z^hG&B9r>fjy$+)Nty(gg-2uvqPWJ<4zuX2f{@f(<ubc3c++SSV5*+w*hSW!Qdi27#
z1*efy^aCGI(WCf$AAzsk^&WtTsYs}jihd%Tik5BXa(D4$p(W2Aq^=@dPbld-<J7dn
zke6p5e(!BwYNZ06i~`y|mr=~OD`{@ckd`FEwIRM}z30fiC~V(?$IU9mU6jv1)an$H
zI1bpMk;>LRxw<ObI*_oO7_#B*oBEq74P(Wbjq6AZ7r~kn3ruC3utE<Q>#^-hR4IUQ
zkgh<pZV`<vV3vX4^B%iLB8&E`B`eIZ!OG1gdVY0C1Vvn(?x4lj=&cI1g^%5m;8IVd
zZutyonRe_eu@d!j35E+nMGe6i=0y$TZrfCsEZGj3C*pWSNHfuqP85qp%g?SY$y??{
zPZ6(%UgA+6j#H3rigfz)?EkSdRfx_PXBpi$Nib<iQ*N~tSUsBA3C^L_#w)*lW*0xy
zXkzg^gxtTNkMFA5!@Q9)FP@2#YhiO#0So7PrK?{|)=};M%LLLY09wLmQOZEOS5_8o
zAQZdtl3TP7eIuOF>PVv`b9ibe4JZjt&y)ZOIabRW*Mr3UU_^q`SzQ-v&Xksbp_bet
zm%Nr*<UBrBaZfq33E5q@7u0%Q4s9<IT}e|1Edf9<#%0a>B!_)j<k#F_K^AL?@K0!E
zRq2Fr?FVm5ts|LT7dMMYaT3AQq#Sxfe+q%pJlB`9Y00xp@L=hdR^k`Ph^wKGjN_R5
zBP5cw(O-kjMW1%2*~B6>Bdms>U&Pup?=m-GA?A_i^LFi@nwF-bp#du_Cx_qK+KLlL
z^{2Lql$rUl`dsS2>6D47_m~+n90OWIj^Imd8(|q&_4ipAcP7NM_WB_#{N2$YGq<1Y
zqinP|7x<#ephUA+!!&6F4b|XlQes(GtgJs{>4m@TKus?k?Lwy4<<ar355jzd$>GoR
zf7EiT!A&yBl@@own#Qr4N}FIp*ruIkbntSZVZOq*pAQLcD%d*W8_s+&qi<{a%P(oR
zr^9bTW7L*87fzL1d4h_0&Y-fe!SV|*g2+e}n{mx`0oSpx(^ZWZ<dqG4#bJHpHZa4e
z-y8!UnNT7mGhC*eII8Od%cZvVc#$b+55K=v#ks=$MUOj1|7YP=AA08wLh<N$@#ao0
znGoKGTh9}r0vRsdzK~bLYiVl&PIYsB@Ww}i43$ELjFS^nR3w=?fxIPe@W&pA!y>I=
zMcAq1Z!WiroQMb!K)AvGRy%S*L`9UKrl}ddMO@h2978FG1+Xq>_?wBp&(Es@Aouoc
z(ID`s*w`vH`8}&rQTYLz!~ZH*z@M{P1pbsFK{3AG9R~afz@1*)zM`WGA`$UV-W<pQ
z1h#|GqzpiS1JsI*=3}OS%RvNk-{^l7q7T!?70R8m(iJN!t1rf5$&`0@ciV0cDOp)#
z^JOutI0%k3#LNQ3=_CXVC6pxdzM^B`Q!Lrh_%y9xq}UO>rVL}fR(tRV!?Mr266#ob
z$NV=Kb@{gkBjBdFNq-hNJdup~0|lc+eDHF(j|jvHn!YdP#IQ!~<?{GND;9`*s6b}@
zv?@53_#23_m5i6ZKY1CoJ3#?H*n>TOHo(Z0g6rS9)h=HiINixF(4I*DFu$@AI{{4#
z&(+wm2p9QthG565f>_T3ECyR?^Hwy(p1{s3BG^V#6Winu47S15&5FyqsvS0ALrd~*
z$7A`MLaJUHj7BNq@-wL-9VgVFI=NBJ?`((u$~m*_>e`w~UOU<sc*nk>P#9J|st~la
zqAwArIs?^dq2UqN<_WuwL=^TX{Pp_FkFKXDKB+NSE<26EG-&g5DmV)4e(+2w?-#!3
zjXiItTAvd=kDFo|8k9RunOkp%s6hOnf}<nx)Rd*Pl9E(I!8^Hc>%@WSR<HkLfzW4h
zU&-Ba{=WG#$!T$!9gRAv7j@)H^DHzp_tF~NH%huV@zFC7%Qks~jbMFxy^hrJsT|oc
zN2WT6Ll%d}^}D6gGmU;*GUhEJM34RXXP*cM95m;;mTIy`tHW<soyg06nwoF56heQ+
zv|Qn7ZQ+jt-)Ih(xEd$%3Y+Wmkh=2F?pZeM!K%MGP%2y$U;cu<<HJ(UJ|frGlyr|z
zCaNIzAERrt(3?HQ&_0?OB+;d5n3lYAXXPa-G#(UnvC|zIcds2ooHyE<`OYQYIIbk+
z%lwS5+1xQMwQ)=BbYUp<piZb2Yt+KuS0Gj39dD7--YDSso%+#pw@N=A1z{IEO}Y}t
z$)kvN_4h;TL)F1aX+d8WMM+D(7p%eoR$E22vi1E_1Rjo@!2lxt!t6=KXMT*IAhBQp
z@^{yfgrv3gs4>3q77ucJ=Ol)PjLFH9D@_WS-7suJfO#4Rw4TMp(2Hr%5TK&Osqmiw
zX4{+!axqP#<Ca>jp+U&7X?a^45#Ymdx>N%&bwJm<VOFiWNCezN;fS_T!)OSI9V|Eq
zKubnNtr_iI;#tc;prF}u21rPv15!u0UG^#G7Z$3jbbvvIUsq?pzi(cmX}i+tK4D&9
zmKF^-gRlk$U|L&O0XzNu`!_BSb21oy5AZty$2Nd#y#uhFf9emWregFIb>AiMG4pQf
z4sgbxOo!3xswzqfZ(mQVL}yg&xVoFQr4^BJbePO5Eo{1K+t|=xci)A-K^TZ})EqYZ
z+kW%O!-^Z1Vque)R~hbXmCRq;imI3oU->gAy*OT+&4#uq*r{)w$;%64nch?`Iu^++
z&Q=#{Y6~;*!kLP*wSu3cg`V#;_h(LNI+>Ayoqn#8+^y4ysgHNcqHjw(=ZukyQNTx8
zZgo=6w4CO$>|j<W9A1m0ji5q}&h+&GA(f63&mIG($h9RCpNnZ{NG{~=2$bA_Q{FD1
zXsv(<3gl7{5TPrsiyf-1?KOcdddRS%h?%#|`)4aJ8+u$T6z4xSg;QbkWcpW1Ne}t~
zl5bAI0EkrPWVNRb2|3|TeQ$Y!(8N36FKbcj>qAV%tZk)}8K5+X7!r@XTmx~$w5w*3
zZrf!QK6m)%&f!SU)PZ#3tQ9>ulgH|zfHF@4b;<(yz0GkeeASK$1MX|#@)^*m78f-(
z9gqTE%Dfx<{^SJl|Ax?&0EHd5e`nBAjn>?jkg%E>JHXO;^nO@9@_sq7T<LiQd0gcS
z3hG=qaj*3Hzgh&IjTyD(ziDir*cTRACu)SzsZt<c<T<k)+W%&A$*eC8t*HwVtb(ZV
z{i0-tkmmZu--GC|W^2LZcWM!Yi3J8JY!>{tAB?Nva+&89pf)X+y)LFx2R8#271eA7
zpH4)JS&``WFl`D#BaVb>0&>Y%jKhKwa#S(t-zvv0GIE@zrK!c;DdM|wbCsM{_da*L
zgx0!{K_*;Z$57&f{Wx?PbyXrnY9z%&R<lKHvWYlJeX^&WKn&lHLrfvRhcif#K0lvi
zs#opvD5AlA#m+Z0?A8|vBNNw6hb`gFVJnOI{snv4Z1Oh&T#PDf^%8b^MS&1DB$pYv
z1jClij-T4BRa4?Td!(y0$zW2*bJLd2pN=JOBq||ffHplB9$ea9FiZkBlu>UkT>|i=
z(UBq=ej0ES{N5QI+8G7^S&9}|<}Xe*2j9Q|<Oe2E06r!Fu~ZVUB#9CSY@z^~#@EDx
z0x4Zxy7zZQKq&w)QGtQ;rA3!fwhRrRQd7u;4x(099|AUyV>2^@XL-U6%jgJDfL$k<
zfPj6q3X5bwz$T?LJD$SI1+8?lxR@BZVFPx-3|EQBSq5=nieZE%qw%cu9FAP3)>iy_
z!vOJEc?%1qgn4xmN!YbSPi(HBjF7M-Z5pIUG`Rg8Qq`Drdx|x5Gf%<5xShPs!v1q6
z3Sh#4a39Oce<@zYu3`+27VUlFhzk;LfRe+4Z%(sHwQ++ISgY}}kB;buY2+u!Iviso
z8D?hed-l#qJ3>=EG(`Prtm~6Zw!_M$V9EPWvRFDBgMv16#Gb-a1y6GCIhh!eO{cpQ
zC51{ps9!=H1vL{Ww|22cl)>7nVN~+6VhNx<YETUVU{^P1IpwbO6JsMz&g=@;TT*k_
zsQF-JREX;=6VOLDhLTe=_>gcJPB8Y%=KNl0?0I<9S6xx~H$oRX&izKP$fl|A&bn9|
z8irho-l%Y{xZ@i9(61){d^I5n3tP9*&aAWiQI<8WNez#kZkl>zk=PRlgrDl_tuv{A
zkSDR2neEwacB4iNI-3ZT%tfx+b^b5}yu1!`9C!co2q7oW8yEl=R}rntXCML0>Hy^N
zziAQBi&+YOTtj=^&GpUB%>m1Lv!iLkTJ6U3XlGT)5|zIWkMxqdRq91(5I@#b+>wiF
z)`XLICWVy~Br1jzv&HJajXym>n2?GZ4=DLDP4!kT8aLy^#>Q{M8jftlFrkuTM3cav
zU0Wq8{|Z;E5Q%{{f=MLv8gV~f!UK>TRli^>0+gOuZ#EV~eQRZ#F$TlQ=u^Ahkhm6a
z!-MUJ7ap$s;dlc*ZPi@Cds$r)JVdlOS(5lY+^JucG3<Yq64pIAI~$@P{thB^P)XC&
z;>41BePte*OVymd(a<>dC(I@Mr}QeIbpkHP#;Z3wVkTal!D7dOjvcDCAz8Dud#{KB
zJm;@pu?4vTd%>Ay(E7K{A*Jgy0iGP7<OO7&BkbXtHCb5sW9H@y<#B*ac8Wo#2`7R)
z2C$;qy?q9_pd>g5^MC#XIjQ_rr^BR6OEzW!6vhDmgCv>^)EvdA_l-Q73=rgf2M0#W
z79dCPdHeWcc*V`<d0ALk_$5rVe`SRmDCL0a3aI(SlPLgi3#YD#2AtnD#E$UsHc^-5
zhGe{yHt!NjcbIrB$5pyrAsbzuja}u*7_Lb4ZMCFo1f@MQ5;XbU14NjW7*I{o*iG&u
z3g^Af=rK)<Rv1t>Sb$x-?Sc#3g`?t>e%V}zORlo{4e40R4PUv>`++x7W;zk8Y@aWJ
z?FX92g?^maSDF?a1=QbP8~g=S=nzXmSdAWM3g??WF^H6!>enz}1qnZ4RIX??xwvFp
zb8rR)7DrH?Yin}xCUB7Fmo{Q%+F}uh)2lE`e&TiEDk<h0>3Lom*G(nNq(-n(CB`aF
z>z8us{isQ%V5X%*`yHz~KWykT!YuTOL8AeO0e8}5Z{F1Be`Lwv#{25b`-+Q$GXyNk
zQDAycJwign2!sqLwEA8w0OnhNl^UWrSrYsNJ3Biq6Vrext68J=4{fT6syQowpbG&I
z^cR$be20H50Vx3W^)X$ru(6LV3$wFP$SnJsO>QPcP$AJ8Ifu@EpN6_AKJ&7NjSPd~
zJeH6tGkEVDZtjWgR)ST{qop-UVU*Mf-lo_i$U;)~{6V?}<K4r<R+uWH=my<CJPGo^
z*{c62o(+CHl!_7%Z*Mo}2`4v6a2YRISFI&ZCHebW&sWM!)fua-?V~@or251Q_F{{e
zl$u%~;`$JxXQ^b{77VKO1Qm;8BVLqHkbg_Bn>WTVxm#rIgoNa~VP30ag0Hz_P`bJr
zp`aPn1ec*4+lHu!=a=H&r}X?_V!H$P?_nbIp+D3Sl1PJzi)UldZL~a4QME5K+1ktf
zLrbgOmW0;H<7V^Sd{TIfmvOUKE#oae#;yO$$;JiF9&2$8*RCdw425A&(3h07m)KZ&
zqCh`lfPfvTwxv2cI$GGiJhs`Rl$f3l^<A<Abg_n)@3kw#fyb$%qXR6-02iUCk-c1Q
z=VVO!bRY)9@8QB1uE$VHL`1Nvs)~by11;{ic3%Kvk#c5zy+d>iJS`2)_I5wEcCQ~T
zJT_Bd+XF>hT%0z8rsmrxASF#Cd^Fj9c8til&3W9M%!!rB#DyRWq4)xolqycAh9M(E
zrLgX9XuniHQ=hD5d8Xg4(YwAuzpT?*UE?xl>3h&hxk_dsOGcSY0t*TiQAB6q8R23c
z3E1!tl>dbQ5=Q%)$WHO1CV|vx6k$|h^cm3i5hlPVLy_G5e5ePQlE}%iBSqxdueUKd
z1dhb>rKduK4DJ+atMm9habidmwY7z7Gfe;bMS_WWpx^11b+O(Kuo#1;Kn8EYO^dC|
z>*kHBql9~{duYg)HUo!@Vi$d}$CZN)XxgzJ63C1Z1KsyG@NTQ(H1^xQKIo1_7vGDs
zoa7J>mwL<LsDfKfey=lKpEfX;tQ#zDHfEGxn5=E`{k|pS^%`GkEJ+jANNq<ZCT(*S
z;+r6>u{;00>Ti+i^V{0@8WOJQ*Ml#yD1ngY{qsg%$>&WN<;?Bx`o+dwuggH3cD2IC
zdA&P@P*~kB{gO*<HlQu5z2_JS{vAAz^A)$ei!j(1CD*%Q>{)ZLs|VkoeA{KOixP(C
zJWAFQwD`?vyHX{$8(U7jLJEc%CvqTfuZRqKY(>`0HufcVq?!o@e|uWO45nnvm&|S&
z#{8|Z2hzy{D&8%D2@1UUCa8zY>R<Jd;)(``{~DM`GbQla0zr)mg9PyL#)+M}Ne|5p
z?0O5)Dcpot-I58z!3$b46Y;XOzqKF+D`F=Oc9W#)HJ@p*V)k7+F!TKRKi?X>?w6Lw
zj^NFkafy76MQSza10O5{gGgZb?`AE3tO|0M%T}-hc0mhE?qOY*Ykj9|1CM>5{_i<k
zZ&XW5OSjE#Ezg|_cqHSXNh<65kAnK-8O&LfBn2vS9eq`1<&w;06vi8OJkmOL1xh33
zSvNwFR6trLU93o&V_8%ynmToB;B#6TM%GC|8hFZU_Gd1duDZJD6pzH1ZRU}u$IBm*
zA_X>ug0!%t8ZNrHnB25&+8wc~3J$U?k|Za|4YRBg91SZ6SA@Qp2Cmw8fDXP=f^d$4
z9FA93P)Y?GRRT;VeM|v))|pA<z6kU;IDX7j467j@h$G6VZ7AB;pa3l8W@!#JAev(f
zt6fdfA)><`Uk$)}q)8hJ!IH;<<&(PEusu?SLn~<~$`26#;CAw|D;%a_t2E=@_+Em3
zRu&tnm14n#RRRT*`+J8^3|uk=AhhV)rZ;*<Ly8Cy6%koj9Cp~nIu$;2aV=|TveSKY
z)1hJT3t}m<#tRb#%X}3jiQe7C27Hs6ek!Gmg;rO`d}2u?`SpIUx0Rm%ds}(N`|KCs
z{Mp1%V!hVnT};E|Acg{Q^UU8_PY$zd`h<1rba)F1$Sa16qe*_)aQW#r{y6uM3s!7Q
ztcZ*qW+<VcK0dqZ-||XS?fPC{rFA8Rm5(i)+BgdS91I?FO^%`Uw}Fa+BRX1CWoZ@?
zb<BK#AsEP(UUbV$C=jm`?j8Mm)y_bqOVpRcpsAczGt;mXs;QnXFoVwrrSHvOTn}9L
zY!cEK@Z@f%4@?Y&9NQG*<XO1eWif5Et(bx2CnWI{$QTGn2@#Ff4*QeI5veo(b7BF?
z0*hV{QzV-vz@)}dK00FL_q0(F$w>%L&o?p$Yb|6br(~dly{mZ?3ii5x|H=SRN_TCO
zij$?V6{NGHi>r@eZr8V-j)d~Yc7HVVsLbr&A(Zv!(6@B2vfLpesA=il_rn`TR9Dl;
zI!_^5!9#|*QMsyZ{J_HM&1`6F@3XS2f8Ux4CC!*HV;iIIG*Nh;qs$fh>od4M0cwu9
zGD(s74QfuBn9>l#k^rk&3LoQm>|Jor6D|7Akn_F6^L*e2Fx@Ps!GMGNmgBxe1b;9#
zCS&M-uko>s`~ie*e5c9zF<O(WEHdD-u_-2QJ^>Yw7m1dRDS-@rsF5HgtlNt7Z$Vf%
z)j}LslVGQMSxY#1N_x4pL3}J!Lw6;3sA*jyl~u34o`iYR6Pzl>emDS9Wx%e{G^L)Y
z{qD`kz-Xw&D}k|NYrh6}0r`bc<hQAw4+|(=A$nQfk>zv<M;dLuXeN<U@7j>P325PX
z^(<c{$)q0)58VuKLj$fz-&{KX*z`a$8fYe$VovDi;21@ErC;J}U+<zom<+c8Ztbj&
z*5Yl6s5&;JWQHprqUIfVD!mUr%zsPpPO?y;s4P)8^k7$xX3}UFJBYOencHj~@-?oc
z#Lb&6B$#=qV%(pbNp%H-eVx;u3^<s510)u1PU7raAJbwF)?naKSDW5)V`7%gy#1PW
z{VcyZZinL%7cK4D-qO{rYE67+&NX^o%hA?;|0wu$zI<=_5qK+VU3mv)P>P4pj+`^q
zw%cRXtfTer!}1u=hZo4`7iXlcnSStifvMc4%>WV7uZQXovJ;+_CrF2vtIg$AH7jXi
zV<QQtMe_3Y)4@f#iSKeHV9<>4uVShl8m15@ryzNdDS(NDug9Oe@mky1=w3PDIH-Xl
z68@bog&@|TOk$NL>#0?DSje40GrO&$U2C>Bu}X~rn?Qx^lu*aV#a$1;GuL`@1ON9M
z|M&4A6lq3!`pv8ua-D9)?Eice((hzuF4I}%mgbgn<`~KXtcU9xHbDo2vmEc`gN0L`
z=Z$~t8CsmXvRFo&q8S?Oeloxege&<@L>B{^ht2k#;QzPlkexvljf?Vm4b7PQ@&yYn
ze5mGCKee-(>SzjD;%dI>rO0G)6%9u^1bEon*0#Q8(2>tmSBEXB@wXgJbF_KlSt~Dc
z6!a*F3}z!k;5`5Dki!v0Ybz_a&x}5SsE)z^Jz`@PJCxo1eToR=20?=6|NZ@b<Dc)5
zpk~~HnfA>c;AGn)>#L8K5t=~%b3OW<$juW*2HWh{|L_q2mj~*kb9!h9;=~OPaO$MN
z5dFi^WT{B@`ktAii{jw&i}U|ouy$&1bMw%neBb!1z|U9Ut+FVZ??x3;9896-{PQ9c
zLYO%*Q=y2&)S3H-o;;8$#4F3B!24CP@0NG4u7TWu5}SBu@r?RJ!D6nK^1RnoNYBR2
z@N<V5CtIlZ<twt<w`;Vkdm9W@Z;lw;46a}GzJf}wR)d5~BQ@MpRa+ivFaI{wv;Oe0
zgqt=>TUt^BZlaCwoGNP0?uKN(JNys+o+fMR3C22l4QN;GTl!29gFC_~4tAO6Ui?5y
zUG(oST<kh+l7a;mzFO%T0z;RkjQD12JfgMu&#w_C`8&r#?#i$Gq~nfHSt?`0u8|JV
zATy4e1mWD~Pd~iJy@H|-C|-e%#<S0{jHo!56Mbx0PXTC;MPt32<I_2$kD~^qAk87Q
z_88v>$3aI-WwpKT>r={H-v?*QrTHBgEkWe5F{DkeO92o876nGz-%m!5KiYAq(}=(=
zMG6z;f7%X_1D%+g?df;UDpJ4)AA=S)URQTRq#tSH4I+zkBnWyH{x){yqjkYD7L?x3
zt4#M&Uypvipai+|<t`oXr<CaPvWxzk9V9^D8I>wA|3Zk_Wy_8n_$o+;Ww?u|H0Q)J
zn3ge<1du$XT3HY2zPKa`7$Mbr(jeS|Kp<(6Bp0v$;{s5n07k-qA+RzU>>_-{SiVrq
z-mjh&k1Mg3zu8Gy4I7Sn>$HM^DgLHYhRcl5Cq0qxxk|3$5hL<)U2;C|?#&5?Lcvy}
z51I1voce{BqY{1fu`QfT^u$d2&hsBJj@r5-!9i&4h1QP#vrieWzrI<kwQ1j753*DZ
z;@&Q=+JpnbJ&i6wZ0Wy=n^*|rim5T=YjXc$9_e$TR&NaU&s_a#K)!wv%A<bidWE(7
z>jDvCGU&Az*KTERU*6J^T2*B|oi7kEb7VG=jzxep!nhit&wwREl_Cd<;ug#K`P#4W
zMc$)V%(ENjk50UyY~w4uQK3BH-i4OpEDzSUa9+-8k;!hVvW1$1OrZm<f(gh9@6AFz
zV=|8;W@beH#u_9b)~2YwTY^r}1ev^mF3vKys4ivxBj(yF{`4Zk3Sr#5f|qBRQxS^q
zAg6r+EqJGmw6+V6MM*I+vH~Z5s?fN+*Bv`;vy7H5ZbcgVq6fVagq;Jttq8pPP6R@~
zd1&?HE7GMEYcl{#6Vi+_ryYcj1!`x4PU7`xf#oR@d8HXZO$9ly!&W?p<NI(9+&w4>
zmCE{hsx(<v4i0(OTadcO!G`D7*TT-uY=s;fYg@kIp&<%vW<%47bj{>33~VO-FUD7!
z0;5fnVer_1*FOgr7a)s95|)ypjuKBsr&eqNY^A#mku9flDf1Koy|$IZX6&%(&SKTV
z(BAd%J0@J1#ky-puA7@1*z9~$R#KA6?~xH8)1j!tn_crP&Yy?Yy!cL^f5Z!Jpl%97
z(V)~)mDFWZq=qM0-<MiLk%UiS-8krXnFZ7^?sT_%5{;rwHdk}i!1QdE=W&dvi~{;p
z4nrZ`<bx**H-Av819aE7XJI+^dGD~zYaEakIp2eZsoI`ffsG|i87f0VS*SBqa*5M9
zgzSWeuEXlr;-vUkz^Pq~176|vYPcY<@8tC;K5($}!RxX@ov^I=7H)yoiSxq3C=FdX
z>r!wqtO#S;IP>t3<;!YSfTA0s@XsK7M9h@~vZOh~&U*R8ve<XzmTi5P9{n=>8kqBv
zHt)yxpVO`En)=V{1K9F)m1JJ$KPTD3<ipk3A~yDX3L3y5%XvGG=f9r+W6sK(NP+SF
zQO~Gl&+=Du)91`S;B`F%Sn{^74<-OX=fF?{ym4jd()KrfuXI?M26ZDFwPOLO-;kYe
z<X@EMQ+vT%>>hhSTX-FYS@UG-BFZdzu25in3uTCCk{{i`=p98`D<}69exq!>tO(nl
zn-_~5i}$sAz)gygYprW;H&m4sb;Rer<=Z~B<2?3Qt*5aN6yC`RSbYAuBn|A|(g?T9
zIe2zgcDZWa-ql?)xcs!L%^6<R-MBatQlc`vd@C~Q^5L>0@rZ$Lm{Nkv#{f+S<+@U=
zL6JYVWMOrF=Net^DFb%-?av9WIzfANqfXK>^JH-{FyYq@*Vx`0JvPg=M9)i;*-Nr)
zSp&}va;eS2+FAse<eyGEWYNVe0s?~onATYjNd1n1*w+OPV1>phD+_M7))G};PY;-;
zs~H%CO&pLBAOYsmjy?BSoSaKQH<*=`70~%}BvZ^SE=G>*8UA&&wuY%$5>-&(1XR@v
z6-p&E5~pWpQIaKY9v-X$0)P!;NmW&nG0S(FXuz%xE@H6pXAj^F@9D`USbz<@^#H}S
zf<jzuY%B{8kNe;en;`pWAx=54iE4X}uzTirK52Uil)j!{jX^kHs-iqpv&Wafr!fo~
z+(_(fN~_jHIB=;sSEr)f5&;V-I&dO$Fx4zp$vw2C15GBP3o6N^tb9@`Q#1|4xKwRw
z+)PDQzov*xPGL0z+F&edGx~nVYh@KZu@go=;hQn*PTsCWAWg-_W{BPbH#<JjRijSO
z|4c}7LzO;$1Uz|3%v*t6K|y49WG#!STHeTdPjyIEO^uxMpiG<cyUZH2i@N%2)-j$b
zMLs2Zo^c_$h>RGyLK!Smf_c`-iLo)6l(zx-pC=fzGyhrq@PXmwfZs|+ec$yBWJc{t
z1^I>|QGPl080rdB^K1?(vx#wUL*2+f<n4wPyvG*sbBUh?`Q%C(kcaH}-#<L4(9i(7
zDERMyJQL`C<!t%o0w&79C6u6I&&Z!TZ~-C`OaeY`js6vUEEL?1(?>j?^?h9V9r1kz
znKq72FcTm0Z9OA*KkYhdZe-}|=#1?p2n9^p!DISf7TaY^-)-xxF~z%7>Q_vrAMB2m
ztYj0J2Hq)jO61K=FCb*=R*?)Evnuf84IVt0t_mY}CekKi!dLP_=v^Hu7ln+e6Ct7_
zIg%H6-w}-~0M$<r_<NGek19cV1%E^dt)P9oY$tmj%H*i*l#3yF#L;Wocp?EMGL>I?
zjcL2BXIfoVHFd>SGbz06a*}oVzSlP1WM<ZOe{5{H(#)_Rl+7TLQ)1~I#=l}wuuoLP
z@h)>wq{${JRwTheo@37#%TSvv8$#t8&R=s=<-{c?Cu=O<K6kgYwH2Q60!{#Zmx4{J
z1Su(Memn39UjT;L=m8w?V6y>pIEk_-Q-+zbailpL;_>40vYMLO6flJY{uvYDIgh}y
zYzj0!E}LW#b#emZ`*>Ln3@DxSC`mJ5Mh9YuFfmO4x-AfB_wfOSM9BMdC7v`*cE_ay
zD1Rl>Eqr`^0c8$Jp5ns%yvyLxJi}Qg)yP<?{UE&#O9s@x_)-|?ZZA>2)jK=p1;jY<
zU6ZyY2-MpWS9&b;ScUrqK7sGBgpA!`3T1+_znM^~xJQ8^i{$&Dj>N?S6x}4LZbE@M
zRRam7<T1C9{B<;J|7mGIDg`#~?yh#OM@r;j&3=8Cu5(VF*MP;7qltToyj!qcQ#7Hc
z)*cGPtP6LHjHIR8-FIxA<a8y+f0nMp*T;Ba+NtoCitHSOQfRg`mCofp;^Ur+${M|i
z&T(HwJkB@e!#L%tj|j4AMe|uZd6|>(yMjZ*EtN1yo70RH7~NaJ=F<@xo7=@q<|9ZK
ze74LujzC^V_llkN3K&fBXe$TQ8)>B2N-DEG6Kjk6dwcqqnk&8Uf~Q|Wswtr0R=$q2
zqKl^%mCcu>b#5a6@8gL1Je)X!0M5a}$th99qD2pQk^eUveZ0Rqbs2n#jf(>$v)*`k
zr~l!&pyvNBd_x2DFNb#0XtL~YT8yd-9MicxgY^b{&+{2XJ3yo2Na<r==;pln^Y`<u
zD`c1w|1}Pu$KOVq%>Ak=TEoD1_khpGz`!v(FqLFf${)IR+Gy6%Qvo;Fn*(=oTSZGr
zyYrEdF>1w?(b7qZit<>IrRWqhSwF6Xq*p%X@<#E=vYoo@5E3R)CLmyW%*|y)GnMY)
zf<x`tOgigin;`*zRul-pa|9!_L$WfMTT4|p_h51ug74BlRJHu%IJ$4<Wx(&mCP;b)
zp?nQZ9z?V6ilM<W@qNGSK7VG8W3^w-u~%`UsG;#-&AJ5b6p__}9pa^aTb@$6{mOj&
z;{V6hI{-%#wQa-MO|rqp$;P%ewrx8b+qSc@ZJQh0wl=nHTVL<{KF|AC{arOvQ(aR%
z)6+fYocp>j5M_>aU2@b?$IBE`pg@<Dg>h_3myHm|197F4%a*7h;%RN2Et+t3%Tj<u
z3<0uoDQod`JCqE!&${cSJ2~Nr-Mc*!h`|PiH%3Op(XqbpuJF7N@<}U??bE=@(`HQ>
zT(7(G3~rC8>uVr;#cIHkC4Ou_$;xNDGkj<lFub^$Q`|f_nl=DBafGqNsZt8RF!Q>6
zd|zHT0DVmy@%H0uOH#2XS+<?5!$rSFI~H$uPho>N7;i%x#}U_T_{1rxWm9)A1*)MG
zSj*$|)XY)mujZ@nMTukHN4QIA6+2y3(~IXcAC&_RHv|N5w0oDeu_!O6C8IC(=xA-8
zmR*;)4@@+M7dsi=o8>Jmd6wS@ge!A)t0D1!m9DslT%F-xcD^I!zjh92Ey^U@to6qK
z7|J(3zD)1da6o(XoE$&i%z%(ms{mJ6wkYol3x8~3+l16XI8RtS(_cMwQP4-^AERGl
z=dX+@ED<Eg#P7InJ$OPlGc-0ZbYjHqLRM!XP`7f$M)K3<`|I>gCYyl%M_~%<?2>}`
zaoj9Pni4e&ZH;SyUsXhl%g7riI=r(+gx^>6RIolP@>sDOpwJ8|Ep4gV0m@sM;_Fwh
zvkmUYdHl>tIxep9Ldj$dTwK6pBo;8%rO~lIEZ2Ew06cw^m6Mg3%l`Z^JYD!%Fl*MX
zJ*6odQX2uF%I#d^T_2|oFmQ0U<BYNukn!!+TgsHzvECe<!D3(Y$i(g!B(P<WUCu4`
zXRe9bi@rmWB%Z-NU#sv6AcNS+d^U%ty*^r12aPdooJdeIu4iyOz!@{m<)xNQFsB3%
zF7F|bZL$djLw2@io>ejqMu1Sk*`!_6i5>%v@h5miU!;<LyU^~IddG~WDUixK-E3HZ
zs;yb4H8CM#9O*l91g<OmmW-IS>wG<_mxsC!N?|aQcxkgMj3;lqG@RKczWss$22wsB
zHeq}?uc||a4wFOf5L%>52eb^(MWfn7jXHn&FIyS|%|vXTH%yiCAL>bFz{)v~oII=-
zSZOb}I(>I?N(Y>em@{M`wr$a)hZuFp7^Jn<Nq4T<Os2B(>zRbuun+n}S%AWP@L(dn
z#j!N=;zAqj3t*-#K^mW1aZcQ#8C6zBi46y#(O6Gy-PO{9;Wo0!Q@89#JqA#~J%kAB
zlvGq0fRRa77T)`lyZ!xvb8g-$uecZufMm=`3FtTYSNHAe6N2M7zSp+!(Qgh+JIGut
z*Ft%Sr+C_MMukyg8^@mYhQ$RBt~*Bo;YvBGesSgx%IT(@tKnJk*GesF_k9Jrc?C05
zSEbe)ZdM>87dd%u-xJf<>$TVhEc{7r!GUrifHq@me0rC>!ey<N1q#jwz3Y{!Sclus
z^v81hqjkWs3<rd<e7I49`c1^0W(!A$JhVrp!NBzyr|-ZQFRx-ZqH|HDFVp<<3m_)4
z8QTrsP^#4#2T50Dt92g!&F6<~*C&YMD!{^&+wxki#r}gZ-(cMphY+(#)&$k7cynhb
z5z=hE7ls4=qLi40X<Thu{8)-ptB-<wvJm3rz0kGm36isU<y!QZ`I#cVBdnr=+0cX2
z^-&;6Yae?0nkEW2b)XE&_2dETd={puA`Xf7^JDXKd&iL{JDWDAEv?Mj^3P%P=To$A
z%Ou~MH=fH{O{8M6%&kA(TZ|{q6!#-`zwzr91V#D;Ae8Z+JyxS;YEQa)zx2`ZIY>1D
zIBzN0g|V_Ep#uu!z9YwEM-3B+s21u)L&hTpz5Dk&_G!s;(?8?V08e|T8iMcp;p(I&
z`sBi}Bl<&!_xj8~QgKrCw|8p>aPW^BK`56^G3Tv^%%1BLG0pZA)uCGh*3E=fYYi1C
zRkD%eb7E4)b)i}bkPTR3`J32mQ;q(}P(7ETDEy5~QMY;z-0WBqO0;AWVafzu4w$+m
zNsfq}q#P!N|LvP<XgT6|{Oydz#a|&FN}skv%+3(u0Y{gQMNuv(IVLGFaT@SHKY1xt
z0+MsX)~yep4}AuZWj#UDCq$SqiIT_kn>hbE;bc#RSDpgPc+f3p58IZEk}?Tu7FF5J
z(|Z7ujuis~X<Ppduwziezal^ff{2kGmazk2n@FL=z^c4%xqV4@#kg+w)(z-}gp-hK
zSHTz0^5@E_u}oRA{Ir%u5eEEG<mU;2Fq-7Cz#_9fdv-uj1T;qj1PFjaH;;VBv2y$j
zbw<7@52M?Uc5i^xoY-KN*beEY6`@wK-X(Kj_D<b=9ciF9?oe1K1pZ5Te2k)@Iq8@1
z>wG?Q27i*H9zs-c=}BHZKLJWiqF9QYPzE?1D4uW$OAZMM`5r?cvPoZ_P>S@fWz(wZ
z1WFyZ%R{g*CK6Hd;0tHrKF!St$!ub0Bt|xE>w&ApSWStU{xqB0a?~oZz8!7GCnSAg
z$cXXKsZ;Jab!64Su16goO(H(#RL)mYn-GO`8AqPZ+7|AV@bRJKe|UUH{~dW~;0;80
z_)QKnNn&A6c!+tNsl@Le$G=ZIc&psH_DH_cggz-*c1owG#@%eyTC4DmD3son*-ZvB
z^Uw=7Q`cAOHP6h-D)tpIS5=i!2KxC;G&WW?-v)jKE+W$B%6FUZfnU2}9|xQ888{z#
z0FcwLjvWwvga!38`nZO5Z(gY1Iupf0)>7|Yy8<lnyGM9H`&{zpj~exrqH>}bF_87g
zudEC}5`ZbbuD(7})2c8vs(~|$h>;OE2?^ErnSkPK?_V|mc*`IIAf-zs3}n%<?DIj@
zk2Hf5DmROU2*l!f?d_iQzdyXa-*!J+w%drccsZVxBCbaW^q(0T8xyRbUY@`C>+RAp
zK#7p*^@t{lKd}u1?QS67%BQ0b2!-JRJ6*sT-d9c;?M+xSd^EnlzaPUy!%9*Jh1Dst
zDGJDR!N36EXI;8AZ$s(X;X~eL9so2|^U0eCYRS7vvON8p3)*DJnJvRm_!ELc>r(fB
z(AR32{^ezLz`&{If8f;rK*(4Dn##%p@v=N_>q!&;*%t&aJ71qJp=!q3@*x7*C!o4&
zu^v=b)<g^=`MFUNc>T@#gX@oFY}M(@Kin8t+PA;mjCsDUZfajv{|D&x+%c+Key)*U
zX!fRQV>7>S4<yK*+EhYUQWlmEJ8Eq#x(AviK>X`q?E@kSn)@deYB*l?-U;d;Tx%-b
zp1oU`u#s@zzJ87C?BY{CDYT#O@dKv#`2WGh?eNo=k99Vio%e9T{|6T@;rGYJA_5jW
z%m2T<eE8kJK}A$UzV#IcR+=5Htep5_{x2R4GTd58+ql7k4wH@DT8SJf0wiJqI?;A*
z+PI=z&0{{>*4T0jg#JWs%Kx4h0y|Qu?b#ANKoM@<(0exhVZ!#hV#kqhW*e~7-~u11
zm)MxyA@KP|nEt>kWO|}3bEl&<aduVvegR<$<0&ChG=CG};(|=TCuo4N5KRPIN_xyQ
z@84J&J3(+bRNW`btF?%MiK*}Gj4C|~(C?&YucF}MIA2)oPjSSy?ZLv0I{aTs<1$(W
zT)M>ivc9dcvTohCJ{1IiE{IRxOQ20cI4jzL%6pJ9^zz9@jjry!84-fk<M8ybX*Y7)
z`c@EdhF;^!WO0otVLEfafu(n-CP&|8Ze4|b(Y^3$U5`)E@rmkdN^Qei>*+mp+<&HK
zn4O&*iO1;&DPJAR^5B(Byk4P0UkLnn(;&-o)fgr<QX|o=QY}AkWzMnVGtT$Ks?x+K
zP$w{38!SHj;zEV$NGbfB@xbP$xp}_y4C-#hnKX9{Tc)7Fxg2P{)y{%p9}zYdSL6eA
ztjO@d;}vt=7OpwAV>zGtd;3CCJCVMuvaLSVz3J|kT3hrp!7JkxC6noF_1f;@TG7VY
zISII<zy1aR%1ZK77=2l^WtWZadIR_cPI&y3#SNp?dnp+iyO%q4T+b`-an|Pd?2kL0
zC%~m4WcGQ)YnRhM@(f1%^Ks4)FB5HM&c_(56E-E}E$3>JhpDzseEbErQkgnN+;Dk`
zJAIQQU#l33`^o8|0q!<D9@Mh47!lk1!F$_IGgQX=W3?`zvjut6lQsRKOpB|<46h*Q
zX`i<0?-l23%i{<pNZ78`n>~4m%ZNkGjpqqU-Pghi7yHWxb7<sqN8HQ7M?Rv-^rO*P
z-6tjh@*zWk1_mYZLIt1_1M+jof5NU`y*|6|i4pe#;hKO(E-Ra42aff+J2@RB<%C`4
z(J_mIeIzu$&+>HzSxH>69*14_G9X)Po(9Q)D8kITjG7gF*BkqeFm@ZNo-ZWL_d7p`
zMh+Mj2qNH1`Dt@3(Hr-iU7I5QcUWY{&=Gk93C-L+EVq;5%FV8zdl_9MO6y%~iZo<`
zvUyCh`bhj`1w!0_L`_?!TF9`~AncEFfIg~4NnG{NH;JI>Woqpzh+yGwxnlZE64R$W
zL};n#CuC`$Hb&gaJge^6v9bj~z;=xqF@T2s2tVS^+>7oAcN@Z`cERPVukhC1y#@mq
zLr<RCGsjH#_fEt};!4US2;@Ku6_^wS0T0TwqUq^ra6URU0df>zQJ4&b32klBH8jiv
z>6if55d=u^6^#=DUJdWx0O^>D3Of+GhK32iSmWYC>*Uk`h^<~;UPktBn%8LnEjfLv
z)ZN2FoQw(DqbfCOvelGv9X$sJG#`*M6e|{B$`mxVUr<}S6V&EFb1=KO7OiqKqAnsj
zBu<53ZQ7F+8ZTB5av-I&z&8Bv*U=*OFv$@j<zN8sq?<Q#CUp-=2|1955VS0Z)@8N&
zlOaH#p5c?_wMX`Xo-v~5^jbn*)%_8A2Bpz;;BGaLKgGbpAwmezXbTGag~}mNM>2X5
zoR5b``0=Ut{?;H?DJ~XDWXOQr)oe*6RC){<1IvIRZP4&F!BhSthPWldlBLIzMYwb}
zlw8BM?XXLxwL^)6ln?a4<~Hz97TKts2SSBk_X75pZoNg^%TJFORj)$Slw%CygP(mZ
zd++5$c^pPyI8O;3VX#5zcF*>lpI7jo!}-zPi5(s;w`&fcBGjk=x(VPh-9Y898ZlOl
zN;s~prA9NQG$g#?eGZ9JCzB?iGUpm_&BwiO@3%pvMLeLc5GDYu+#K!G!fwnp7hutU
zpHQ+(59;0PCE>5QBrR%lO1#<z_<dX6qbuHE>$RxLG@y>vxcuu4!-QmD023`Fe}~dB
zBtxJ1<L^#>Jx-gDkj4w=UOXd=b42WP9ZdEaVICBuL|kkt`m#V6m8dL3+FyLuzL6ev
zkeg6?5i-#cT2y!nDHk|8)UntQm>KrWbPuRMtL#sPfTRzo>_9{yzNK(~;6BF><yOX)
zj$$g@`Vaj%SgP3tVqbug1^~H;vZokXSOx+95CHK?kpcw`j<Pbl-45}HA+)q&i)&5h
z@p0<FIrUy<G@ebqS_amYvA%xX@i9gbz_S8NGHOTr;C@ui$<L3Up0;z;GSStI`S{@1
zuVL%!%MTv|`Nv3d<?3wiFk}GS08P*!a@2>nTpS&Le%}msI)7Ii7COGX&}Au~`hft6
z8Ct`$!YMLL`4V?XFf=@ZT<;|M_l`OrXB}q9eyq0>QIR=(drHc|o{mEYlF+h0$D5q0
zYOyyL5>g<ovk_OnOvJLlKWcl4c?hepiq@})__+P=m@A<u4Sk>P5V^V*pIGD!V>}tp
zim1&cx|=jURw{mN7Q`DpRVgH$`wMJTJG*b)+|WF6@Tin5MA5=GF3A~!K&gZWWKq}m
zHT10WL~3uwajDr@8WTm`2BB~f&jSCC<GnY>sG3TLcCS516VBU&Mx`yfG!+b(@PDL!
zOw5Bx&ch&DbUZv0bMqkHkKNB<AoB+u9i1>aU2+N_oUE+KTmxj-+kCut95UIgon;fB
z9(eC7!LR{BE34us;Pr5g0ogy`RY9Rv;!=c^>&JY3QUYQ+PLYXW(mG9I%#grks>^WD
zrC~V5P{|)gPR~G!L(&Vlm5^hO*2;VbtC2SjsuC2EA6gKiW<P!)dle9x%|SyLTR`1w
zG$F=FHCtv6=hZh*O)?0=wz=5!R7x{?T<qK@D<i|)JIzLh2-C*FnkuO(5o&lUu_()1
zIWs($>*cKqkeh|Zl*2rl7h3;H6OiiZGJ@D%%~iM*1z^i5k~k-v^Evu}Q2M(fW708;
zkRg3tmR;vI)j!E2h6rH40jg-883*8J2mq}1EnM&K|Ey10V~Byo2bq~;7+BbQASt4(
zyj%_?kV>_Zj{972k-2u6T3K1SU*8TWv}>thfXt*NOBMja1B{A*j=;Zm7%<oYU<X00
zm;yB_V5yPQ&=9Cg$H$Uh2n?cCUgza06An`cdwVDVod{?E01Pc4P7pAh2k!Z$_Lk#8
z*#qWB1ATp8wK>_bj-RfEU*uA{0~n7?94yeeYYTdi;L6fYOmD=V3S7`7(99~UNYi8#
zbri$Ah&DIk!su8cYCk0{DK>orIq?)&cv{}vk2&0BOzb@aHl`tu<_+gjrY}$`2aSJG
zp_N6Gs<nGxd>vR^6*vCwicwhJ;s0kWsKA(>)BI{liEBS{jx|6GzA0X6jV3WtSHC3v
zWc?9=TE_m)lDA==^Exjclc#TWR-4DCIK1%KW%(?p)(-2pR6ygbpk>+3HnL+}{Cmg`
zWmi?=2PcJiuW|RXrH$51hUo78W+)xEpURPXb5H2AIR3;BEE}j+d@xO{UOccJz;$B;
zFbOTKb^VGYHEJ23Pmng}^PGxv6ku48C|s^B1P5tajU*N?Inm@QETg<^zoh}@uYf+I
z_N!i-=M&W~Ar}{T)k;TT8tdEWXandUYRpnMfP3fvq_bbRbhQ~ah76qd?0i$SOG>RY
zt8H5+*6~2w58pH{<`mroM~>4nF}4f{Ek*n|Z<ioV%Rg+Azi5BVjhy`}mSC4+L~1}k
z6NuCf2hWq7x#r15n|k)-$1jk*_P8fK!=L@i_{j?<FRFUTpln{l=WR_R*8{4<E0$<P
zP!#=_ER8-DEavTOd2xdCkhFx3tXf%x=)15&EVc3H{L;EkOY&3;8JpXg8;$Dvm|#V2
z^sO%M4@WK0@k1yW$K5P!QH$kKAtTsOP4DZ+NA=gVE`VybR_vuE+H#=^iDQ?T73axA
z8_oN1Z*-66--85nC`5#XNhT(wfrQmH>qTY@?VrLxoh^@Y<+|p1U#@ci#9ab%y1z3s
zwS<2hwiAInUx;v|#o^HHX2iz!WX?>AG*y#EmM42?-=2XJtFWM7mLroN1^mqUHS9{q
zgZCmhVF~lUT!7meiPF{pAVxZ@H>~x@I3DUZVGFkIrWgqkUbp6?39eT{>G!JZ@-gFw
zb+3>cS@Xin!kz{DI~Q1kpUWc#5IFUC>}^l8qldi_vwHjp%G7y$s36~s8eW%Px;={o
z$LQxc8PSAKa_YX6&v^&zc=4Y+e1j5``Sl_bPhOT{48g{Hda$8&zhnfGsL2&>T&K^P
zZujcnyiQ0G2scCNDo!_NQ&cw4Gzw8pC`TNdJd7Id3l&q{vIZu=&!m`^nyObDD6%Fp
zq10Erc6SHk^XT%~y|kpM=p#J~(89QG$7iv*Tv<U34Dx{*OfPSm9FS=Nfn*M!?~gPp
zuR!<8G_CPZnWvi@JJ4kz$dqva=Jkn*32fetAb^4gh~;HJ?~r+3mm!{C7P}e=rD``X
zO+G)~C5=lB31=fBWa2?ly64p*$E=KH6&ZIy(OQ6}w!-9t9Ms3k&@wcip9Ak{(arZZ
zR<%AO3?F5T*&1yc&E}BBugk7qRM{fHP-+?PG~0*k!J=lQ=aQ`x)+UDd9(ENcZQtr;
z0*U&R_;X>POu7XcO+v=d6iOr_P_$z-l@_V$yIg*o@_ki}2^!rO=Cp7A(((>AyZ`gd
zR-7+DvdU5J_Nj8fh@p<?E<&hgX$-NVX*oGvLum5w3jSd$EmOp7eNsvty!08!+o@SS
zClC9VDqzQf1Nc>eNJzX15jsqX=F%mLSO5Asw`j8iB=Ibrvj7uvn~SOfby-Q-=++iO
zLj!X{LW1p^D;91vMAv5Uy0wX+Aqs%>tXl(MK<m=qWgt3vUjqLAKE0%@42b#&PEEzw
zxh8gUVkVb}y|};?k_R%WcWjqR%gZ&^yp4<oYIgzSxZu9Y{jqbPC;=QMfE?KTuC8oQ
zh>M-VWbJJYpvHN4fU{(wT5Yt_vXiOoz_ZFy(0usrn4wU;Dm;pnzIhm7jNUdQsT~q0
z=2}|rLRf~OWE4^P8-KOjAFEQjH%}nrBA1>aEL)-r{22MyvB$uDnFUz62onRJM|7Ym
zra)eNm<SeHYn3g5dEd!{qPm?`kyF(CaUY?x?Qw$N7>zzv6_H`kM;2UMsW^3@y22Ch
zLB1yMWG$ptF)f3!41&DmQfYtl+O&8QJ`p#L)Gte=xg<ufvP??{(A+ihcr*NTqGdT@
zwS^IB8G1d`<&)~lS_tIH8RIle>4*m%k&(sig6=al-g5!$+?c+-ias;P8RKTy00I7w
zx2disw#*6R&;k8`zU=|Gii(Q2eVq?HkbiB}>u1lil?H!gvK80uz)d<H9)3l|zOq2T
zt_uYkSk1#>H>ayncb+iJ0L}UP?DVuW;EMdpDBWWym|<s<@5lI)TeB7!0wvoLF2EFO
zZZD^(B_6t3yVpRdTf1uea&B2K;MQYFgl;cCemL#;MN~XybQGT@KEMzWkpvM)x8Q7l
z!ZzmPYy4&Bp1CxRbw|==j9FUB25y;<kht}|S?&g$num2p7kX%cL%^A{mOQ5n^Ff#7
zo&ZlK8Oui)fg)Yxhp<pMPS;pq-+uSGi#`&_bQFUF^ys7Fc^n90-Fnoj4FNlXAkZX8
zf$pCLC)V&?IYF|-Vn%G>H%o<CDcwyO6#1)hsasp0ap$S;^Sf`m`}DrD)f*onM{aoc
z_8E|E<rNlU0PrDkD1c3pqYwh+<8?CyEj-Y$iX{$hZIyQI&<4T_fP7c+IH187(5D9-
z_%)AQw{Dp(eWGBN!jc6Gpd_eKqiWIsYP0mXI3$$9>gJ7J^q<?d82kJCrfCw?sD1kM
zirTn?fLGw->+rsPNS-ecyMF9QZ4FdRgyIZ-_;T`yr06ifL2cXCfxtq|%9i_|sEokh
zXO%LtCA?*?_I28<X}5k={<|A+RrcYx>r-u;GndYdiN$+Q00+X*am^@w3rc<!a6jh6
zgl8?FMLf;|t6ab9!cHfSMYXLZHWVt&pR*uL8q1G7{v`M#AxZ6-SmiZIDrtX_N=~HP
zjDHSADL#8;1;o1ed-71QB2~<^qm&93O%qnzY}D9gFQI1SrIrkubrr0%{9j?j`0@bT
zzro8~Xn5{aCnc12yN=71vmlSN16n7-V}{ejSo;&{R~p2`{N3jCfF3B$viJOc*LFOU
zmm8U^JS@Z*Zz6JkNtzII%l`GMC3AS^CZ2ShSGKbj+EXFx&ld4k9g+A~KXQGC#G2BY
zH{@kfe)+Hzn`)xK!kJVUnIQ<WJDyQGAp<DSjfTgoW`v=!n-zsXfRs)S&2nmvF%I7n
z`E1v{uoA+`ssrM}s7H>fT=498pb(f`M91s-zJl@(GPbz?3JdgefQt^?6AC&yLvwRX
zfV;rinj1vY^TiZ6fA_H?0%U@2(<<aQe_$;FTo-{{1Ry?Fdbatr^4*n-s+{6GU|*G>
zt)VfN{c)UKd3^<&P23t+l)notYyzus@+OIyN5b}4$LNaN7A?(+T|C|-EI(+*iDB}Z
zSrL9;<D?}zC3AbxVt103UUG-E8DS%`?TEFEe3{o^X|jf=8OsmM-ZwY)qoX{{@vE|e
z$pxo}##zbtjLR-TQEC_v&WX2B(2YG;q4AT3c1w#No#`1_xlFDO_YBr7FkYa2bMq=*
zSAL7RWt3qG{pMP$#W@1}1eqgiN+N}HI)Tm=sf9<LP_O^Tv1Xtw<!ueP4W{a{wIQ}Q
zG9uZ08e|0+qCwH(Or?ye4mBQQGcaF$13LO6jid_=Fk?4nvAbgQ7LzYE7IYbz*C(f0
zzg?ZLNq1WqB3LJ_)q;D7Bec{)Z(>TCpaK)ZpOwk(E4{Vx+gfiKXUVl^-&p>CY}7so
z<j&j7%OhZpWoCf>4~p94dri{w6j!|!|3<>EY07Q&zr59d7$%55J_2A$Ab5%c_a7p!
ze@`|4aDIM1Fy4iGUV3c4ZWY4*=Qx*=2tpY$Lure$g>{#!brVB~-_j?X74_tl69xEt
zj5Rc-_?`v6fNdA`R?8_GAFJ`O)^{p^HJ1YPUZ}d|`n*>84^Vg76-Xh#O%HJOS^P(r
z{SRiU=Z`=l{;T2B_P;xNY(3(Ni;4hsHl)kC6B-D3gKt^6gTddbP+#(p6k_B&UB@cZ
z|DK2E%rFr1%JuQI3lc#r`ZP)9Gh9>iD29q;v6_SLAZkjsx%%04ZTpT^EjH+S^4|~b
zBMU_bw89<_)F7Xa*Pp%Hzmpd-_(8}|gk@y2_PTICyU?Yh*2-)#VS+47TL<?;XWM6A
zH70>VTtlqNCFXx|TThAM0sUA*2B_}k5ymFVzGM@c*x@SfI=iv`py!EC*R1$>JKM#p
zzW`r%k~C50^RR3G?TygIb05A0=gGs{>Epy4RhHtB_kqbprJ};=qgXKP0nK~NM#e?&
zyHAwOdz;sj3Fp(atd7X)9k}=9cj>jO<q_E1^iR9EpyIBO{9Ud-H=t*>Ty0r@@QAkG
zAMHITxr+#L0?*T%e7PKHA`I~jTtueA=+#0=%=YBG3B9r5s)Z8^e_@R3^G0z}cJVfm
z_l=NqY`w{lHLd9SZJ_ILI8g^_O4v4Ky*}aMAo}8&7jfVeZ*OT?=eo(J|M2n?1n>U1
znOUK}>#s<w5rhZU+bL$;lNtnC6q(uc4$oJ-CG|mKo3{ydOz+FvwT0;+QF0#VD4oyq
z?{`Rh8=Gz!uUKy}nEQcfK7GfN;u4QM^k}>j&R6-_+IyXw9vKs393B*O_9r;np4a5a
zzC?0^Xr1?6e_K17mIv*mJsYV%q;$3~KTdQozCc<Zx@38V*bg4=wzWJSFFGgDs_X6F
z+TOoRxIJ<D^nBDt?+*k2iD3L70RjMc1u!Qn5Z^ry-%GC#y36-TaqW*c9s4HouM9~+
zU3=@B+#@x`GFNY2n>o{@_)BR|FYmNh8;WG?4|jGS4*uX6<fh<ES{hwM8U^`AB2VX=
z9YTLT5T>|pnW8VO`&B}6V)Sc4FTCH(vfK{L;Fy#Cx*q*|NmWm&Lh0<4WaG0dS^sTu
z!D*pbgiEHTGmw_&o+&ZV=U|;`xBh+Xwm|aXCXtg$eRi2XxcQdqQO-y?G#ZAHjn3a*
za6-2h1}9zrbus%ORTQ4#?Lb`yPFO|Vx?)u)UC1(}E;yQjht5Cm)DicUS(Os+Ak?43
z=W|8adLG%%EH5&7#Yd!}Dwg+s9DYgB3vUPl-l6_8@lo9(q|e78<>#YUR7D~#k7sx;
zx|Pps6*dpA-R&2h_^*n|9_UeqJU86JFDp;`pRdpc&+|T0WJO0^Y(fWuTEi$EqMV^W
z7Lnp8v_I<b?A$>U>nw$mqu<caw^&^dV(qg$aw7xcw(cx!=+esfYl^cDs9b_lxjz{`
z&s%&9&Ju5>s1~jHNqCU>;nsQ#WVt82?pXk<SSWowLCtINtFf-<?;p<%aB<}W*`rK-
zT7_IU<Fj11^Po!2Kgp-fdb2&!OggIrRX+}tza>@F-M9^t<r&TvdR}PA-i{lry1rIv
z>-Vxf0z=~HzCB8~DqayVUZ+pQO&Fsc(n<kYuAPI<ga69jDV|sA&G!qJLld{<-4;;E
zL!a&G->lE&#oyd4ALXO8D-U!o+|%mFES|LY%HOtfKSzD=&bMUmKZAK`Ee?Hro+UFV
zKSfvIxQ33FlAk_#ULK!=^U;pDAK>bY8n`?!WhAKitR~)~JBE$Xwofq4GjHoMnAt$<
zV4Asx3x6o71}#!-;gLkn&C!AG$~(qKmc5AVJ-tfX7<*WN8x|kaS_y|r&#V>`o{#5@
z8qeLqfC>`z*XmXQ=_X>NEKHF*F{wD%g11R>+f57{3^HFBw^u|2WaN-B68;L*WJ!27
z$9t+X5@5wdPqWJx<dXV&uJ2YO0JHmoo?dxzv}UdEuVFfR$G{OVIfa2r%-33G`%t?V
zchZsVyrLFp_UY)0)riG7lQIV+NkL{@`XQx1zN^Rh+}^Q0kXvxmnmG8}8_}hfjN0F0
zGK&l(x@}Z@&ItBmF@V7#&yw*6$oOBc`fO02JF@!bOv?`!o0;bd(0c7pU!HnJK^r7-
z*{emJS8(g{=%8*Dnh4z8j*mX<KR$+u%r{a_&Wv2A`*0Kxc>!_}vrf6$Oj|S+1+CN|
z&**hT^hkSRh4x^l8rM|FciJtz34NkYOVSl21Anj3OTQ{Lo~5*DFJ|ZfXCKFLaj?>I
z%g!|ZWQjS=;lk_ovzzjjHC%W{F&`#cgg#KxZb(T}TCy_N`~LAO&oAeB{9+SyZRby~
zr;U}c7O;FTFClw51x4XI0nW^I`ac~S2!EgoY)@-n?Ur3~2Cg(HDk^tAO)lH}b~#1^
zKUX(Y%WKWcH;g1}*F3^C&X|U3G9|3P5+LAf;^%78T<wE?UGw-&yN!f|9}u&V+27%!
zkz_d17*?_2p<}hl-L{EmAFE+0>8#Ir_{imcm%+H-emi-V;WpLIh>TH`SC^N-@>+TW
z;*AG-4x8}nRZfT`7S{;u6T^$V9yN|}yabF>I`Q6-9W<qvkp3^9vh)ueqak_|Qbd#1
zUlwg&o4>UN&2O?rWKFg5w0T#tW_X3Fkp}1H2O74Zh?;=TPdEwf9c*D73F6|&J~eAR
zk2GU%@LpAg?-7dPVar&0a<L3SMJd5-J%H@kb0mGfUW&cD5|N-?8=`<gymzn=QumPW
zF)&fgrhLQimU<?;TfyIMx7zBsV`Z5}fh7D&VeSQ))a<(R)k`c830sEG18}Do5|RNd
zf%<1<KT`n9j#~SDXb^3n^#(vQfOLNQ<Mq<u;VhNwelGgT-D_`e?+VCrM?gVw9GHqr
zmm(dE#9}>D;SE0d9L|kbW$B69x+G+Af)l@i{!qi3G?=v}QpF0TQ523MjCwhUwxdA3
z8LKz+gl3ix<y$Vfw{=3$bc8LT$3&Vg&#5rP;*`WDj>_&2aIC&puG#l@A(t1SKVKLj
z4qQ9yh{v;##As_g&+EjiCqOh6@F!obi##F4;y)<4t@?SKLcET0wkGTG^cB@S&9$u~
z^*{lfdDaBE>^!I4mA7I2y?&MR`9m?D+~ZW**&<I2X{E$riJr`u5a%qvxbVw+!O+u-
z7N+y(g>+ulD{c?S(-Hd{u0~$hShPtB`k$5lbm3{P7V-eU%tDxq1Z^uA<C`^-eoT<o
z<@aq)rnEg6<B0tQA!P4I{QJUJ_BGS|S^~cilS>?x2XE+&my|vhUWMoYh0RREhOUXw
zf?_xP&fg8RUcTF$yvD*}KQ(UMQy86~{S#l8UDkvZ+oJZPH$oq`R_kS4(;#Z^?tXNQ
zw!u1#C@JlPXZ3ZYdRbP68oS43ig5b=@Nj@b?C~8HEBF(wjirmJ1>1PHsJN~bgn+Ol
z-*Ry0m2ToYT4|7RL{3Vi|HfXbS^pzDXL=*3tUWG+jp)2>W9J=-?xce4eR{3d9sD{<
zK>PC@_f0YGZ#~+qsdtLaBxcUFp;xd;0i)@diKsUbW@1{4T^~g3`(DR21H`Lfh*XTi
zwKrqa=Pd@q%FR-BV~lpox6rEKl8jfiM!K}d2Ahn>x0*Ck=gxNuWn9qM5rrt9vx5&b
zJRU7*Q5T%ojZKlnq3Ai%LdG5WgW*XlpYWEok*=r}(Dv&Qc^D4v)DsO`Y<7OHV&u(h
zOsUE9Gntnj_4Df*isA<8ffZ@Xk_!>kn4GDsksGKl9}geyevza*q?1HIc{j1Z_mRbQ
zcRsMkZgLZ>vmPdA7lESzeGCaEr#p!1P9;wUS4+`_SqwI}ecgXt!L-Z~i}C}}Aga?`
z#BX(FH1}t={ht9rK1vg7M`+j|ZBuiHEsJ)GO|da7BuQN#=j@H}t{At&O)+o_vs#rs
zRaAMUr71l(Hn;1+Gf!u`Vc2u1m)m`O+)G~rHajCAr}>(h%1rHzDahBaDBMO-RCwI6
z490(Ti>j*Azblf(x@GOWt7%9s)fw>KUw?0*ot6~ULOk8L)jMg*^Kt|iXMLqo6EaKy
z1QzirDYhH$*>8KIK>T24=f{}NM{;AM^Fq~T$od7F6){lPMd48KBY**|O~A(@Coc~v
zQDVdoyh%<@PRz>-bhb!Z7_djBG1rnBd%-|{FMP{wjY*h;h;2JtqyLN!cphEesE-D>
zAC4`DVwT-qJvcrq^~?%6JH6U7_VM-Q<pYs3Wo|x+mdAuNkt)MT)r=${j-YEmOnj59
zHY(rBDawh^-Uq|=*5@ECM6qN(vpyC{Y{?i(D$4eO{i{BJB~9=)#i}5WG&=XkT;Y(`
zKs)Z6B|qfw@x_HnPGyGk6fs}WLdIlA`DMXTN7FJZ`Y2KKpI-IaDCYFhV-1dL>>efJ
zpK4)yK5(;~29yoVlu6Rkg2mOy*r-7}hSY_VzBlzcf3OXzlQvdZ^6beFjAnAKT;)&v
zjxMZwRc6F9PdDO4Ah$~N-h(~tq-2Kn&JAt!Sn7%o^()K@`;^1=tikR*s7y!B1&I=K
zrzv8B)Q>J`bgOFv)3kYS9ZVE_s=sQeAO%MU*j|FNqyI7X#KcifUqEIuA>sew5ln^s
zshq{byCL5n8kIZoO`H8r$0SgkMB!n<ve`Elp}+an#^~f5t>UGckY9UT*B=bs>+ifq
zVGEP7@Hzzrh5k>vP|%7nw-=q+n)hnn@8<#mqN<sdmp_(&IeV`BMqv33ahWN2bcNR1
z5!5FQ#dURCg>cS3n!Q-@r7OHOUwYWU_;%@}^a<;Bsn!C^s^j~z_wPedtkYw8>8)5Y
zRAj^TJ{w*hk4z;UMS4zWY>_yO@P%n1>FM(|j?c9RmuC^bp_R!T;v{AGKg2(!#1FkB
zF<hI%-I^vyCd-YcY^JQE)A$DD1XQV37teASI4~Ji!bSyBacpw(H3oKqy9V#rnC1MX
z^oXqvkdpcjcAf@*nSVP(WOEMz4KDvX5WI8Gf}dc5&}4!qe2zO#im?u`bsslI_pZL>
z;d;iX=QB|T&uZw50<4mYr%7leM~17r+I$N&uThA}&dx~)%gFg3d*j)`BMT!NS|odw
zO!$l<HCq>nhQmeIpWg2rKRUYfXG|*TWAYX-AfaFdC1@-2$&RAZ-7zd=I`>SG)Zf=_
zh?Y*DuUt+`EKf`#)Wa)ULf~((hBG<C9nK|1b+9~KF7@8#+FZFf5iOjkpcCN~P0{s4
zepGsaGZoR2$Y}Nv5&8kdUjX54ZeFUppjD&8OOz1z@EB5m)NwzyDJU$oKg?_bVpR2p
zqO{Ido3hPji;$4)-H8Ln_78ME-7{BhbVS6(p@5DYkb7J0aEPmKU;uD9dI0ZHh0>r&
zalrG8O0g*8LnufoPgmtZ_GzNot-bxgdtn8n?MJyCYUocz*Szk2e?>H;zu-$-2(D~W
zTNPfs`|~dtYwYsfWSWW{S{MxABH(i_F1uF+2uWPOOAR<=WV6j-YNT3>2$o-_CJ%}T
z@VD&6{+30Oqr~NUtgxy0?8~=GEMD)yZ*KKme!iWZ;=RS0=`C|Ltod}FMo*1yTLmr^
zs(bERFb=;SKE!`<kBiTj8~uS^Z>;~-9mSr)TM&WN2d(q_vm@4E4gMoZqjfxiSmZb~
zJih@Xfr~M5@!&F2D9ZjZT1e>itql<;RkYL4=@EK}uu5C@4iUj0_p{9s$Up3yK2=!C
zOO{Ez-dNKLP)7oEOJyN6?A<aNT<nafIj>tF1MAyIuo&oC!f(&nG8djNi0+G4L%Bh=
zl9ckfZYY-Jg-q^9xBdxFPN*?=)f80`kyziC+1Pv)Lqub5(e=#W&a8RA*w9L+V3<H|
z-M&EL3)8@F*@4TI7(wFVA+=z)oXrH4O<fKg@=mp2qOV*J6E)MMLh!&(@}cu{@Z}M(
zf$@Rt1)vK+(wFez2pD#Unl#IH5QZFw<KaI)S$`ao$<n>P(%M->_~Jp?S7BvSe2d52
zE7JuaSjeYqXo&<&9D59~dJ#Vu8L+9q>`QoGE^_QlB$S@*fNTw(s)!@NIl<Etvj+)E
z!au2R^&xBh<EwZ31%a0Y=vN+oKLLuKT_icv1e1z^5DXXA&1vlIJ~Z$Cg_cdD`77ag
zG}GvelX!68%YG~GvV$LCfGU9%N=tZq)1W_l-}|hkvP9I;!3`EMAYITGD00$`b+dDO
z69Y$HcDT@=%wuA*XLqlNw`$W*Y0`(DRqW#{B_LiJ8I)Yy`LXNJf{)-QL)0PW^%1Bs
z+4@1%L?Ti7n^-k?C3AcHX-puFB(vH3_S^k~xM{t;k_1UfUkE%yUUPv*cG^)r@d0gG
z(BJB0C#;(z`w)@dKq!1yFTpOC7VJ>40`s2)zu5hGc(Xo?Sq_ttS+=c<NV9BGpVyR?
zoQdO0|K=xvgRm1!ACLd4j$w1dvf;X=hJBX|*1Q)I>NKoxa=AL>?Glre{0`!Wzjf{U
zbimp*+3x1R%Xm%{?HSm^kJaXKY4rI43YcMKgdz_i;{k;?o~Mw40wD#(Nb#<ykPw~E
z2iKW1#QQs7x6hi~+p9lk&g}2+Z@F9>+21d;hZ-K}anWM3l&>Tu_58&r<pVMiA}g1Y
zE#5LPDs1;vJ=+~?Hvp`b!GykNZX%tN(=4HCd+X;mSSdql%@$UP#fDyAs6}okBH?r#
z%TNEls*8j8^RLUs>=hl(O55&+NVc3D>~;l^O_+Dn6o#6G>n4>Nd+u@eQrPQ>Lp|Iy
z<GJoTNITUJ_kYXBj7HEQ-u+eFar!DVX>1p7H)fxyecsCpV5Vddj^bC2zLy-GpXBG_
z*o#tUAdNM{uynR&*K632^_LX1*%SGA=ph#;jbjq(w>+lH##`{X2@(>K|4ffq$E()U
z0lyWjs5jeN;z%zwb9Pv0BPGvDxOya~9@hJn_f4Qdt(WkxoB0S)7Q3Ffp<(BgM}sly
zrWwCkRUysXSi+>riADuFcMd}!A|mDCthBO}n;{E>D!j#_%^)s!ut)YMqsvyo*=}UQ
zQoedhs<^?c^+M%pP~VcwVdk#*cQy8_4dY)<>T4`j(Y&5jk)k?jy`7&cg8dzmDam>V
z4*Yl{{deZh6;%3v*F?qT36~#5d~XH9<M&M?csNc*$)g!{QQeQ@2I?FIIGh@BW&UhR
zhH|GBJ`N(UzG(0)A@#EvQFv6S;3kxYMkUJqQX0<n$9=Q<ao38<z<tm=wfM?;ouxJv
zq~(po^~w%2&LMdw#`HTgA$cGcvr>pEl&kCAE$#UzTJS{*cWLS+v**u;40#ooo%ieg
z(yQqsW(pvNBl1H_)eN0l6*80lo?9Aw*dRJYLBx*+d~X)#%gUjc!eD%gTuQW`DtI*`
z@zqfFp1t7IdZ}qgYCk+&l&9rvi2Dmj5YS|P#f8I%nTx?e`?5erY6tVoi94xd)EFp)
zz;|+d(Dp!4q}-h@HHeGqH^((cP!lic-J3P&E5cOiaN53m^UJ4=4zJ1!Wp_sHF1V_Y
z4a8?xCugoY&kB;fzQ?B`9p&t(QA9rAo{<!)%^-z@|Ak*&9ut&xU>1d=x@KP7p5M2h
z^~lsFtsslCZO;uQP4*}HVK<JBCAI$oT)4he9`1xnZ8HCgA+{XlY6gDU=_0hH^^Bf(
zyLD{N8R@kYr~vrcLV9~#9SxqO!I_4$I~qboBD$qnAu~I~%q))9gxeImed}35LRN5q
zm;3gzIF-@axNHfiSz{9tTvghGg1(iNTO6IA=QlP+Jw36ltgP6vWw>_W`uV}IWy+M5
znFGx#pmlZjP=|(EszeD`wK1f^08zn89CzDd2w<o*ntYCqOzEsP!YtNn%_fc<I0l0e
z+xrGS`4c>1B@D1;Kol}53s<`Cf4KmD30p$FZz`DXO*dvXtOk9D@s)Rq?srw~^NPa3
zRVn7#fos)v1#Z!|qgk5{z*HK&&n1(<lSYe~x*=ul44EhzC8xd~pev*Mk`WqUL88`{
zW@>6v1SnX7d7|j)`T0P4KnI^p3H_`a-DD|Vu1x)`ys&>#<u`1~2uYzhkvyX$tN59*
zd?&S_pu%?M=KSW=9lcmgNVDqa-0Uq730D5qaWO$|JH+r|Pc08==kLqZ(bVPl95^T<
z!kGZ<FQqeLhH)4zvSDtPgd!uUwocumqMP)D4wv{pg?>+FjAj@~sypA<Y_DMmoq)VO
ztqC6c|DY7eH=1*XiWrj?+|ZN~kIu@-VB48h4BE$&v{!+g5FSusW}T(}Zu09Fv4R;w
zQi&RpzQ4RVEcqM1VH6<83?r=P%+5OinH~u(h!!XAH5D?eIuPO;__IclGKb{(g@Qm2
zQ7DZ`Z2QkmP(a{r)nxu${IFm+8A3rE0VLAhfi2-UCRkTie~q!B-&Y5HVuDslpUyuQ
zH2FDphr)e|y@L!SA4%T>z^+<<2pJ6+#8`-@WJtVMW@eZ=#B;fPL!KoJo|gr~TrPbw
z6AD}{ttu<fKnrec%J2d=a&)g%vlHbo-^6$yBraofe<FwT|6&xHU7uJt5)mzOaFkPU
z#|EcN`CUbZ;EZsJ6<t8yTq^;Wst?V8Uy(Yd#gnTj{+xn`6xD%z4x6g5u?0iJ6}4`6
z-lVDPNo@}1c(x#xa5exbUe|B-{1N6IHW-loke@q$|86}(K)K{Qq$TcFl~n&k9&yvH
zeonu|TbM1xP+U0UWy(vImTvZ7D$dBh+)fnT$OF5)OBRQeKjiakvv|dVGW2@-C=hpa
zl*;s|mN0r)_vOyf@aGQ{<@p(|qao}}8TU~T;h~inQQ|msd#4wILYfr13cuDJeaiH7
zDDjbAc^%gxp5(z<JI|$i9E`<#%$bx3-7@$C#d&KolYIr!K@oi@h8genlwGaue}yo5
zTlomDD~yRFT~?=d0e?-uXcj%v8xKTg1?*fC%dSeH!)b>@2+ATch+L<|&qO#3=)!cF
z<g7J%)Phcs69nYnQAB4yEAu4j>b4dNBN|G~IPJbRjmc=-{^evQ_F{-F#`-Jg6Gvx(
z33aEqc0@w-GvW6GDdtL=*#(pBq00rqF^4v&1Bok_cEV_^tmv|(GmQIsA(6HqhP(SK
z6wX1BeIeM_FJL{<J2SIxXog3Ut!Nd-dyxVNL7EzdwXs%D=Z<g_<Iy7DvZsQjNIAI?
zcmoZrs8%>>ctlhv6Tu*|t3Cm@uKN0VAaOWGjJSVjsK(<_o05tO@PHrB<b;SG8atTK
zFtxEUSpjCdgM*BPp1(4pUyzWH0Hf4^fB=LaKLV!?iKwX)42C2C{|%%2lSDCGpSSDo
z{!nDMx9g$Q*99rKMqKn7!k~P`qa~+Jo<ZS=x3=2h<mjj}?lT%^hcT^i$TnP^IY0k6
zP);-Gema*yHb+&x;yfxN)%joAOOM~zI&Yg0;>A&@r`!U3(r@m-6}-LgaJ@1=--44(
zKJJtWMN&jxvvh99DrTSVt59!*tu%brn?w;2V^5LNC!)Mj{NEZMWbz27oSNS}jY|Gh
z68$8(o<XGTE#>bNVXTeDwvL8KpWvYs9N4`EiC9uQIX(wPPlS;v{$8kHmz6v7?n<EN
zlg_l`$qT=Sh=>T7VS@S2V?5e5=Oa{Yk#rBMYK)TD9}G^CNxibS$b_tcv9aB}d|sKC
z2KsoMPz7ivgVyr%d`!r)>1+1(MMv<zf}=x%C?ccAV6cDq9ma^o<S2T{(>VBhzMbM-
ziNii+pkBOe=Lcx%!8JZ8Y&>rsE^k6>Yd~+lu8vPze`6M~4tO9Z{I}Vk$Nb}a!2<T`
z*MPPR&lfH1BUvkw6a5Y*id?WoiNBFOShTK7;+NRVe1b9nTK?PiT17L&5loO|>;$KX
z(IZ6wftu{*(?bB+Gw|r*i~Zks{m*l!SOE|_Q>X6u9SJz^|7;in8yg#wD-9OnvS?cV
z|9!Ve`1*E${GW42Hk>8PMp~tB$z#lv7(htJ={*5tr=(~n2yQf5aNlo<(q_lK@cnHZ
z4n;Ayne!tFJGLMeNQ_7rnq;hKQSYp_*u_&ECmbD%J$ZR~YK_LTy(F>!IoE$r`2+}%
z5$2lAD|d_jFvT;`OwrRtyop0spV=-iie|JbQGiJ5XT!235GCU?Z6EUgZJK0aS~O@t
z$V_q+#s3|R@TQOt7XxDi$Y}gx`-1(pvo*!G<Q&6)**(mVKt9ZTcr%f4Nk{MbRbgaU
zrUe-h5d)(bC~1*K(5OTI_t3+UVeJhKiT9_Aw}6H7Rh!@Djl}a<{$#y(b(D$+mQTn2
zWk1Glm0t?*nTJS_U;xzwuus8H9s~UC#ooO=o#w^rISiB3rsWX+_s7LRwC~WC)!+H6
zhv9Y(QT1zZs5P2l3uD=`pSfpNx@Q4Y>0y;0Yw`(*7pZ<?zdH~ymlY9DYYe!WoY<Ss
z5-<9jI-J_#3v)XRw}bG-cG|VX*YCtQfdDzRi3zsv@Y&S_KKO>}Z-H?v^1E=Rh^G@2
z*jK=<Qcg=Na@|_ALM1*kQ(Da6XQNMhfcc=M<0Ef`P{nhea7V7~9h1E3Homh2#~|&I
z(78ou!#x<;35nqv_5LkK3FXg1jURmh5vEErdU{@o?Je7MAo2#Jj6bQsEBvO8;9uAq
zJJU*p<JsodIo-`iRvkkc<LOC%wq@VNW|unlZ8tW`*PEa<fC^p?9gh(73B0mD0cSQH
z$K=g`%#df-lw56MLIDb*r@mH4jCEiAl;<rhWKCUQ-jH^5WD3&$5w9VHL3%xW=5uwS
zW#MXez&gV5WA#Q_yUPL=Z|xp;N;XY40ZC4T(Pk#w?X{BG({@pV%Nir>1StdZ%l0kJ
zRN%E^{NptoWhN#PGBTYc*q@F}*P~*?U8p^IC}L7l1Y<dUNJ)J9>4>_5DC-_RPfMo3
zx(zNu-Tb#%92BKTP2)Qr_gEy<WZ#AzD>&Ac$_m#$+q=2ygLu{Dr)i!t4N>Q?r+YCQ
z8(JeHBS14Z7=a;ZX-QrCg89eig&QdmbH2Y+%uv+_#ng&@aw8gU+H0U~<UaOsQMR;N
z$0@dwGAc_%;vuV>qZa42(gOvPNKa1$g>?jMcJlF~Km8P5L2Jct)yFwyos5j0{<lz_
zpWn~tgnErHzb_sr;k@b-cMtC&-m#zSeQ7RJIis>j%fdM6)sVzs9;<ifJ>9FUVk@cE
z)}`6{fL-=C+g%R~p7;8q8y0&Wy*22XXUoZ=R#pBz3z#wqZ1iNLG3Z7+mL>-BUBI#w
z^XB2fnZ;%U;Nwk>X!MLnH2aTuwLM+#WWL>9oUgH}4Ni^bxtT3Q%P~!CmnH?i4c+Nn
zuP#(wV|n*fF=VvcUx`1Ye1yvycSvnQL|=b6>Ej_>&3K;S!zys!Ca($9^9OC73WMy`
z1Ei%mZYK@{nM_Swp3JK`@3-flchMMAbD>GJ3NIFdeBJxDTaK@CrcA?cIdha(aIG;X
zPUMB&T+Xat3Xu{u{T9#rFRr#A>P3+4dql>rDcxaX3x-nY%Gffql`8pdln`y%t2w#B
z3K;5tF&dDi#ct>4bj4T)!$+@+^U?HJx$LL{z1s$IL!`9_o-=E{+`$q>qGT``UY#=e
z{3|dy^!nlJZS?wGq_5=4zv?+i&=~gTiquePG(tPgI4`EuCl=$9l28Gf7~tw7QZ>EB
z^Q(U=e2NGbuIFh^Hg<>%KQ^lhvFEg(<>+f7QXu@^x1S1CSUP+`3WXp&{;TJrrOHI#
zU8^5fufISypk|6p@@Wt$7j^`UkMyAAi4Znd!?q&C<TIYgiGgLXU|D}r;sE7H#J<|C
zd6ZXGin2P^)JJi5yy+f!xg}DioHlnFF7(b=_>$47IHmY|(?i7@GZinnFA|=Dvf|qs
zs7Vvrabl#m77JC$eF+KApGpvK^cSzCZ^sT#a>xFEDSHfDcKd==<#JfdTI5xKmBQ!=
zQM?Ow*#*O<OS0wrOjg*;%X!$kUlVNILzmq1n<g$!iN;g7a7B2p5~R`S&Y1}yr|8KL
zZk_l4$_nn^AGkcq@2mq)$Tg)+E60mo{|yR`Pd1#<9bh&a8XG9r`hd?pNfF>FPI`1#
zvzwBcBe2~h*YvFT%Mu|M;;7)NLl^bknJ2YY`zvi_VNp%=^)Nk{hS&tUWC8zBpi2_k
zJryB?0-$hPZMxL&3yK6nt&(WF;sJ;;CfOyVX~R9axmoR^&E|)lot^5y`qLddl*y5S
z&;6&~=SaCRo~g2gz|bC@u<DR3^dG4)X(BRmh*?AK(izQfm^ynxRiVLk`cTj_aj9Pt
zB7ZfO`)?IAVBGPW1)v+=u_h}{?+{S4{B9Nl5DRjuaT7Tlp_wJ2@t5@V3bF15`-uBX
ztHSzKen*k7agDmxpO2*|A497`9|x~e-)rMOA7ziS8)62`cpln65k06N5QyIkXrEZ(
z@09m&sB%hMqHyoKZfz}THF#D=r=z^<gRa-EpQKtJ{^%`kizw;QuU#kF&@Hfud2L(6
zRi}{6c<u2%{=cTqF}jj)+uE`1bZl$Kw$(w$PCB-2+vwP~opjQ%ZQC}!ea^VwxOe}l
zQU7+;sJCjpYp(fBsN#G>7^>Kr0wlA3-Dg&+ak<r!<JYnk*<8MmTAxS4205S7Tp|Gq
z&}A$t+3y#egqV=3S9@$1BtfN9r?&GPd~cUWXRfJqFhgl$X^wMdoS+$M6V8}~9mjT{
z<)rWW*_a4Nh?*<>x;YKDOtr1|mY;7Im739j*gwcP_bW;9t^;N)a<IEP%)`$YZoa2F
zd7VtN%?OaQ2G&x2YP!_RB<$-<;OEtf5w{jW;tDr@ogp@xdA_=QmG5nXv#rmPadk=5
z*po*X#uN;zuQ^Gti&@*<6X2U0c}v$b#g9(!A5>B8HF8Zo{Jv(ja@CW@J*T|oaST!|
zfPH2F4}aB_E1fFv^|TS??9o&-)t-ImG(78L!k^iKJ$Dq(>icgwqq254=-6#8JAI(x
z-7cW3aeX|vjn0XR_hkznfn0FJ*@vMvna17VkAHJ#?B#Wju;e6*39Ij*3SAJ*$2%{6
z!V3yU(#u$D39931RjsV2;^H+YWzzFiFW1}wCBTW)ZL=Ll{I&hj2QUBg)%zjs^MU-V
zjuT&!bzhRN8Q_o(KZ+eSg%c=D5eAO~t6VPRq)FnEt-DYZ5>H^8*I6AMdxbJb-vUno
z`oZ;Xwc2e94jp86U#$ab<whn6e`j%8X?XKhT;v-8Ri_*RM*|iV(~O{iquBAK8V*tZ
zH*}GNcy>JgN2WQRGc$u$n3p}L-}|92sVU>}?Jx8xmLajv$na>W)^$-+Q=UfC$vhy{
z16fcf=pWi-X>QGN2zHLy@zmzQAtta?2>_-gpg^px^8z;kK%)T)F0M4t&k+=)MyZ%1
zrmg*-33=c&tdY^wH#MUh7e27WU<NXm@4<#C`Q;cfyOevgvtHI`+p428-g4zwPN^*f
zC)@=m=FAg*sCbwhkymwFQFHLUc$}P$#gE627E6J6xiP#irx;D>^_^jwc3t`YsHAnE
zErgMeZGVk{=#Bc*pmk6V|1z6|Zn1q&o7<wx=V>?K@Po(L{UYNUozCJY8@QhEx7`z8
zlRE#itk$V;xZg%i$F#c#O-puhs}6ojBRt5A=sa;%mJa7kltucbLfsO@6&IEe+0XAs
z9F*&SMIUvab8TqEGWamww#!Pn;g_=1ru>-_E+9y0@>G63Vr@m==uX+TEr-%4f;@ko
zFMy%s|D_OBnObp#)8x{u`jC5Tb{k5WWWA)@X28juot+Kr<AB+J5cyF&X^;$2!xKc=
zl{PdaI!6RwTwL6vcUX?nDc{X7<&aEij@)~3=zoxGakO-4^%rdO4LE;-yk7TC&4z+R
z<+Nds{;@UP?1h0Z?)sWiT4+_ET-qF&O2ir}W#5m@m`1O(>vvo`)tz2BYz4Sk5hW-J
zjI!xcW#;k<KQ#F5YbQ;pUh59;juo1ed9#f;y}qH-jk3}8{~CC=vds7h(r2ddK5<*s
z)r_ELOT~=a>QDd4&41jKM#qj;La#kK6~}Rh%2K7nf}G7x6qaW+W`lgAG2@=|dFrcm
z-9=zuDA%1boVH0$HP5?7eTuj@W#=11!$NzraJ%O5xH-ij42I+a6Qw3NLH?-Z`#S&G
zw3Q(>hd3(_=gg_9Y&LHYAG&k5YQgP^L1G3f{43@`pEzWAX?)kGGaRosymZKHuWCJF
zU`<;07X`lnX8D{%2Op|;Mrs*%XXT1>Ui5+m9rt`DRDw%pvbvsUP=zXDc8u?Yh2vl4
z<NHnh>EU_ct^Kag{liNc&p%Os0sL4vz5=CSpm)o(@%4efSu5lpph+yRy*=dNfen~Z
zkj~=*00Hhx9v&Zo_=BI7i*1{P0-r!wwy*>b4-c@I9vm8~Gn*Em$AS+D3A3=Zl_*x$
z-;gr5wH06@<~_9fa9vYht>4eAItL>qK(M4_R(uB&+*}{;a+n76L%j3dU^Zra0%sW&
zf0J~jIoc!xV^$p50+A)JK?p(V+u=2r(DFVGG?a<5rn=sVc=m0b8~3s8>4IQjV}%O+
zcUDyCLpHHv7~4Lj*k@1JL0HJCT=((6q_>!Mo@im}$&>z=3YX)^ih1+>hzebL^Z9tr
zg=+uIAI2?|-^nSSGt?)RkH$+2*D4V*OD1dPMX|=T_HN9X5w|Go&@D?QexaZyOM+eU
z=ix{(sSv)|L5UtLCQn4OAs5G%OQZ-5j}|M@WBsQq=*~yTM~IC2>yZ5LQX(l`?me(r
z)yv!6M*v6&4MFd?bT4oW2l&sSh*rGqAz{1gYJA>`L4iY{;$i1uU9UQ9xOV*!a|9&8
zQf6Q{;U^fPZd}nD?w?Pi1NME@qT>Vv?Q)^RP(ryizya*}2|Ug=NWnW6-Y!#0GMpbN
zfn~V#MAwCW%pof`yJS!lP`cC-+4vWj)?B6CIy4=52ILi{f5OSQq@dL(5)$eM#i2-4
zBolX-h7KW_Q3+~#Rb4rehS&%-DU07`D!b~a{&Zp$=@j*(h(SrB;Z_CgzO|7+IaoTT
z@r%`z2Dwbz9f(B+9>uvj71aH(Glt&M!lTL9rZm>rzOGkSTEY?DgMjCO<WfS*V2jX1
z-Vl$j3Te&xgl|@%V!g=rQ6b~<<kC=$=H10Ap$tKUS3=|UVSlEk#Z`5WSM`d36`0SZ
z9x#^r^&YAN8ioh^pG^;=43Oz#*ZaA38s`s8p(>sK2adckd@zbW9hsSy76<em%@#ni
zv9s?#>Uoo7E0>dVa$@)D?+hR>*O||XbaZsoYe6f~rU2i^EH;0h11%1L&+>-ndG&zp
zC=U8J$~WshFB{PZcP+9r*{>fsTzY>K>TNM}e?Nq5$*2VfGI+h0Clcg&wxon^aXcjX
zim6W9KB-1)sjDNGhZfypJ@xvg`xH}_43^<IzsJ)aNP+Yp8|Afz5n_+9Lj^W*4zAlB
zj3lRf>n5l0L^PY9@N~sF0@6bWW!2~wANh+y*_0NR#i{c+EE3k@l(f@?10Nv5#<3lb
zX<-rJ1QoOeKeveGC~0!A1b1g6g7K0?d-g2h9eBXey_SA%@0&z-g#A5^Gkm-D3J1^2
zp4*p~i^D9dc*utx6+1WIspAN~rusv(0Ue0Epv7H--bq~zPgUVW<<Uo8oFkQ#Et8qb
zJEtsEA|^?8E$|NIr8Q0OmHC!Z>Td2x9@{SfrK@1Q<okAM8jY>lYhOr=>MfQ+M3H|(
zeAfo?18ZA2NFAsn8<E75fT=ugb8960)o>H=f`--z!54K0QTL%EAmB$>A}qycZCvOl
zhf4eDT?K8d!5J1w8ln3mI@tEZm)9o<i8P2Te6D3LSv&nt5oiiw$VL=}F(0iR=enoO
z!_q;cnK6?JVWz*%>A>1{9D}lGQF%z@m?<DAD~sBJ3~@@Vauy{+0IGUyw1<(`brDAy
zO@O4<0L&?ot;|{2EUonV<7v*-nYZ&EapTQ^nTX#hI;2-qZaVbS*<2;PlR3nRA9u}?
z_QtD}1v~(q-KJ>sK=6BBN)=+5D!%;AkXKJKEw9(W2mR!ufZ|gQAG{l3<zj^(=mCjq
zCQPP$vn5O>ETh~By`RdzIcBI1rQ}ai`uoSVgE7VTbtY~)Ah*crM=*yq7~K)&oP#$s
z<{Ko${_uIr{PKLzkh)|VT?DK?sayKkhYS9N++Ki#=|+pM34nm1^Xca)%<sDilY%+4
zgxr-lR?75bYV_Cu!YqLkM$Pdr4ft<|DGt~Fu8r7HlfC?~D@m?b#@6*qAelBH+h=GV
z7(~8Qy&f(`%5A|V6&@LR0OWQV@BV#NRaFHxWPVmoYaDA;r{)opxpop1D@Q1EKNKX5
z7RhA*a0_Sq@<k!#{y4XgfDy~e%lnub82Cq^Q{C0>e-{nxlCi%q4nfet02ECe?$9Wy
zA<ne6cT)t2Du=z}I!TQz7~kx2tOR6|c;h_(;S<XF`SU*bbiItaY4|y&Mfi<-Crjxr
zG|!*>CK1NPm@J}aUtg~Iy*|aawum~R#TRAX9oI4NzN(%)oiF;zHj18}jsx~m=YmiW
z9n+8YhO4h_Cr0ilLKbLI#XV6n$qP$<fVOx^WvhjhAuZhXUE9jFu%_6wB{Gzbp`3^|
zGsd&}v8(lj;nlW3{G3pxTsYrqZ*c4yTRB{B;t3ZJ3*fNc82Xy~p`!)v0?m1ITjH~V
zd9Tqz4jQwQSG**>ku3JmW<Fsh{IjXi@SG4o-(f<RE9j>a#@O7mhRRF`32D{=3r&&O
z@Xq5><5SY4bMP>_%U{X6hSTWrnEdY<SAn&jFj1H8bTrDkm#2RluIqF2e)~p&_D_qA
zuanC(_;vHY^cAW(Lmey(H_AE%-*gqgcsUB!J3tp#tk2Z>gnYh1zLKEJK!FWsgL)l<
z{_eqGZln0PEp6GLIWgyypc`#7R!SYo%Ue<on=NT4b4n+Xzu{vO#BOT%Uf*AE+VFT0
z>Fi;v-1TKCfDY}}hI%a#2}i6|Zd#<S-cW+DJ8lzXk^%JzeLZbOX;GLcyKJfSjemXq
z!6VVm1tQ}w_5-LoNBCG%iuoSLlmcs-!6+&J7^0kD&v+-Uzdn#8X?>yz1XH4mr1sE6
zxSp1Ht@)DU`00Zd3}9Wu0?VRtn);7h)0oTAD@7%BDq?H2s2q}vZ=VrRW<pSGSaQ^_
zDLOgwGJ69v%GEfqo|6)+|9a8-Wn|_^KqsK1Cp>Q8e+CqmhJ?y6!Djdhc%S>Y5?J<`
zWQ<0S(wN8#_V!Q^uH*XTE=ke~FOMRdUt9T^Sji)If5u?6Qm6B<An7X#+7|y9jzH|{
zFr^BMePH!{(xEgH%UiUZwZwGy$~sX=42Q?l%eD2)<SSy-70PN(RD^rO$+Rsh*&%*M
zI(8~s8Q&EA{DI3k8$4G`y?O`B?&5+^Lk;xbct>kG!8knct3jD>_C&a~Ugryk8;shF
zOCj9WKQ|w{yIBcPw2=3_jVK#A52P{rwLhFczWt<S>~em0tmDN$aVb2vlb{pT?@cjK
znE{F#!9v9T+t-qeM4DkFFup*D(2vIvbz#`Ae9&?^j)qH0of?wmh}n^yd)N^Z<)Y&%
z%m?cWv39nPhh`uWrCKfbqO=JD64J0<EVsCKYfCQ{pIZiK+yz!*xcK<wbXN2c5wJj)
zFI|}`Z7RKHct5jA{`;pd6ckiMY^)e<ic#zO=!9HpZ7p&mcl5^x*weM(NFe0B;<Ymm
z{wNUegqzOoL^+<yU_AN<62lcMS_5@JX?($2t0(57E=yJt>-FzR&4!<2{+aHew{0cr
z`|9(?%-@rBD&4OuB_GMJDZg6jy|8b$e#B0D=O!*UJF^?6>0MccVv8YfOK2Oj5Fk+l
z*otTBqTHoldwp3i#j4Qf^$@RN`Vj@Zc5{eI!t9&ZoL{7*cN7Cx_675?-daV4m6<S3
zEB}>)EYCBgWWos?JcUt!BgbWn6{y9}?p-Scu}ZkKgxB$SlTtCoJzx3vJrZO=<A)OY
z@wsMYD?QqgUy7R8(k?#4%a+Z9!^qi@d`w5Fm8Rqe!7*e~G_e9=5@Cg%e`4&qg&nB4
zcn;`4r&BiYhx20)G|O-&3yMdu8O2MMP@H0;5>#pbnSTJRqz)Qijyoui`1sCQTfQ;=
z+t$VvxdYBWL+|bHzwRRYh0h-!tbXV>6k6~G>79oIl4TDLf)DR59TWj9-e`c+n2y4i
zH1J-#m7bT}bx_%XDZ`)MuJ1SSqVES@A0RRa{T+BqzfLSsZr=NkAwwLKy@=u4s!FH+
zwOQ^(=o}+x5S%<{9T;Uth3lvcW<WSl?qYC0$cdPkkjtjhg9?!?-<;$2eg=x8z49fD
zrI-smE45-2U)}LAmdgb&Ql~SGAE~5?kfsDWnh;#Aj{L+Eul8Mq0Yo~%{8u+f3tKpJ
zoLzdt#XF4pUwOg1JboQWi@*a0;tu)@|F0yXT#kPm%LAg6fLfMZxSr(E1R$pz%%6?+
zmP`^N79vLBvQy6;7cW-MU$j)BmbkjgQ%0QZic^JZ_a2eAXaY!)C*JD>V{T6i`DVv;
zAO6o|&7ijG|Hu#F;aP@F#ub$;lz29T5(DxLM3>ZUsm8`03ODTI@0KL8ii&>!0pbHw
zJ|m-|fTSLLLEo&F77pOR4m2_$L|$*NJ_7c@KYJFP|8uy><ntZ~MI71R-_Kw%h61wF
z@VOn6%gU$~vT%AI7%2Y#EC3HcvvAh_U}Sb)h1{qOhcPs2_x~QYH+I922t<HP+i9pA
z@4L?by;<SmAS1=fr>7wV|Et81lI|t&{GU@Zk{__kFhEN#F^ql_4k;%n&;}bfd@mm5
zpaS$__L`%L+1d|o7lwzAq=nFJt(P}8CK2$u{?yZp9oQviVMzdfrV}$WqZWCby<Obh
z2Y+EdNDN%Mk|m8wG)vZp!}w1hgNHtKVc`-el%i)V@L6b?9EA7&M=)f9PqQI+=d%Uo
z<_?qqA$q)SjN5NZtuyD3K&WP&)v~SY^;XOxPoEpmuLu+e-0>e=9(N@INZ23>-zWuo
z`>hI#okr@EPvs%-oMEN}I^GCC%;il3T^#wIgx}t6+?CMKEF@Sbu28%gtN8i3ylLf*
z3>MlH&+K3_bR_FmgU=4K5_TN+>R`*{Ju#F)zUw}OY1Iwj5G9>Rn!RkCJAMV9W0u-^
zTu>dRSivH)7o?60-OMc8NtUo?srZ2{)wn|On~g<%Tfe%D?MkFzTZIuUYTofq(dz$f
z3o+c&!Gl2d`TXSiuq2(7sL5G%^L>?wMb<V$2H(mDjn=UuS7jPrJC|^)|A55q-mHNS
zhUa>}cwyIaPJ8*xuJKc7RzKw41xIN6ovzT*I{Ve*!-&TLeWK91T`3RJI<A;Tu;H%?
zm(N2ZaX2b!{{dR1AgEiDFPo$wQ>F#>QjY{tM@v3vJ`IC)1C3MnemhbWY)L*s4%&4E
zqWc6g1wx&IR)9EHxl+Pd8;z7nBuw%zFHwi*&1~oXwVqeFGD)vMAoPK3d3+{ETZ2R>
zZ8L5W-Vrzb5zusZKX6D((@#5U@!GlJFDDYL6!|8z7AWYyctw?$jK(&0r;kr0(g<D9
z6n()vYgY99iqf$OaRK^MDnY^`fp!Y<8HKqb-s-o%fA4tEHm$#MD4SN$gN(`DJ4y#1
z1VZT@Q2TprAl3!~>sjYZr(Y(de0y_&keNomZ&2TxAR=bp5BCR8ewpILn;RHlN+{3_
zPOsmSm7Khw1go0)Ok3OAmGtq~DE9SldKTd=cdt;GapBWX{<-j<3+mLt>z5MwKfRCS
z0!y3ykM+O83%WM(%j{7yQ==OlS!=sGQwlz{SC-9<lv8sOW9;tw!``X6G{m$AHHO42
z6wix_ozcI&Kgp0T=u7)C<U1;29{v2%Y{dUzTxyPMdu__)<ci*mrr9dG5aNCP5*b;L
z3p@Z}q~-7#+}8hy4?Kg5#l+lEjiV4ybBl#MEwuWE{^;ia!+lGItZ@r$FEsnz)m}m-
zv$so0#ZZaS4Q#na_kJ&*vt`WVn6V(Kn6c*e+|y>ZWYg4Ex~^;9DMrHMK21PvdES99
zM3tb`@KJm-em(7DuWtSnmHtxG+FUIv#pQmCb=K@OFTWB+wG{LC1(hHGRe^MOW_4ZP
zs5BWJ0jtw}9d?{{Edm@}0>Ox!ZU|k|eDB`A^Yoygip7}USxWe^YkC~^JKa%PVd;^Z
z%S!WGVHEN&G}p`Du;fazQ5-!#?7Y*Fx5l;e1!)JW$Nj>}J?2bTTZ8>n1!drKtI~T7
z1k^jqF|$&T-9`Ft`#g%{^DT&L$V|KjO^)~N+5<}?L9)=?22#A<jU7X$%)TBi?8R;a
z*d)|~#F_Q8z3)rMyIORx_wx-lS6Gd|hY*!~UGK=ey{C7L^qL-P@vYD8ubbK&9Bn2O
zktt*=d>6F#_fx1uxl3=<myd~<)r0P<p*6WZghVz$d5ba$`cF9Oc{O8U>%^i=_w~j>
z*#jfE*a96y(q@@4&#iF5#k8=m6M#04XCkDxxdrlEr@k_~nQ?7M1w97L6}`Ozbs8#O
zkw|>|PH%kft35Y7f~Ip!Iry}_9}hRT3eexwkM)IfG1I^f9C^3*wk8mmX2i(=z?BwO
z*ZUfO=XafLZG)MPBxlQA+>nUh+TIS@U!=vibK70}8MclQ3HTL)akFmShO}hxDUItY
z%k^f$E~3u~?ngiHYRmb=fCyte@H3RI{i0+ackosXoUt3#Ww7~+lngkW!0(`$d&|Si
z3ycLZJ{<Fcfrl>yzNZ5N!GMj3|D3e9XZ(vwV>X>wQHtCzx{@Me&H3VXS{!~^75rnr
zk>G;~{aamzs#AY$;^14J?BvHojPE^pj)y5FSEk1^vX{j<u8GzZ0Uzscij}dK&m_o&
zZA_U`*~v}8FI;23&X>{7Uv0x-+4vZ|<s<8|dlk7(t>x)vdbIuTm4|!}a*QDSUcSRb
zJf-MPqbiFr2aFxL|0+xNsJF+C9zz}!v}6`;0G;bAdcO}6pr_jsAMt%MNxz+a-=Z?o
z6@Miz%h*VwgO&!~XYq6aO!?lN)FA%=|5Rp~wXJB9U&{wSYZC0IoaQ3=0lneVJY9HU
zu%Mq69YYi$#o+;zZjcStN^42fs^V~7C+$*kT56=J2)&cX%UAnx+Gz)A=)2zwndzUY
z4b4}vqko0xa~iK=Yh^h&9tvKUFGuvDa#Dgh{d#<%)XECOSNYaaDItQNibD?3$G%?S
zYya9!fGMF5`84xw3_ymqL|E|aD(zKHwa9O`yX6>L*CpwQ$%F*Akr;r)cqMu=1fBQA
zhPCk}Dz1Jee=_6xsj+@(Uv_H#(DK~w-VN~tRG;LRFSd*DkDMND>9!wP9LV76P0lnz
z#xY4EnUS4l<243QW8+&4j0J>0(LAy}Mj@IUdwA5RIWd$g#-ZG+$AZ|kV}IPhgL;C;
z=O;2hy!I)YB_US|_uFAvoYdCcnKq|&q8+3OGSnnHjZfgYhprijJa_9Lz(M9rzCQTO
zh1#9YAei+X|GS1}i%gR6$DaPO4IYG%6O<k7WX<h&`*Zt63ch&Q0{#o-jViF_mj$r;
zsxfA)EUl`n9I!^BU}Ve*2ms}<Uh$VHCg<nRb~#^SwbemdU0qe*eX)c*f`RXJ!zL~b
zKPloqMi|r5e7|>T(s0&jFspaZioy1Kf_oHeG+30unvNd!^k)5jImb%pi&*#(@IX{0
z;@9NwsZO1A^n1^WTVnUhw*hE02`-I3o{}LNi#`5A&O9eE?)Rpr2(h;0p9&B>cR%Ri
zg`xexX1cZ-NaT+Jrs*c!tP*K5MQHJ}H9Tr~oklM)FXajNOs_6VymP@z^N__T;Ql@w
zHs9<Gs(KYXnn>@BY9Z@4sa`5v7-nf{j@QBqJ_GHb(YZE#a@%JzGI41<er8G%{LMZd
zy6NIxtuY)+@{-jb)>w|AD1n9bnDG!o^neEFNj39(K*fbubKKrL9eMY}HNQ2k56<3h
z3uMp$;Zt6iH6=w<Q_xZU+C^}D*nEENOZ(?Wh1j_{udumLFP6grrKR~DroOsy`umzr
z#`jGKOY*z!(AjGgP2k`)POA@)3cN5DR>Sz1-VHgshP1#yIJ8vn>}v-3;zDoa>)y`T
zGf)ltZ(7>z^+7{naWi)eMW%>+?`5qmUF{K&q?cVMx|=JeyrQD44=h$>Z=jpa?Vc+j
zg8P|~a@056@52TDvdHNhS&em(R4`k>ebl<<va&TL<~v1l$PL@@)0v5*F!P!7nvD84
z*mI56zx42{q}^80ulKH=d>Fg<kd!z<p5%}3R=1;oU%$-2zBeBkJg!5kOhO>gz{H+>
zh9T<;s3T=gSF{MK+MI<hS6nZgt%nO#B#R40wXrK)KqF=7G*q!b5rY{TME@q%L5}2f
zUG{WN5;2j$gF49Y%K7wuZ@o|5Z2}Wcw!EfZr`i$U-3_nq7n+MWBRx!2NVjv9q{h8`
zB>g72)LUhaW@zNN+Y;j3)Y!k5{l8U_0V=AhcesBZpnjX>yFGKfeHk&R2}UqjwalJp
zI^pWP3-rF01vVLSI$lriD_K5sJa6~le=J&pg>Vr0G+k!#{d4kICfN6;Rk=gtP+MhF
zwX)_R12|R41cDK(tf(JpL`XMdRdd#sN^NezxMVP*h5zh;9(qcVg1)h;szgcmh?B9r
z&s#cFDr`Zp?0Pecp?LfmpU>=86_vBaeBD#1k^ojoYaRNa^eWubpP8Fk1rY7R<j~+<
z9gWo`!AjOpCB4b?+SGU-z~{55ZyTfZAL%y!txQWwU(s^IWM_0uouCjFD^KFvq-w^?
zT+*5p7FN;5mejfosiBN$BIW%N!!_e9@IgOOdB`E^xkBDFvD-T8cWh48_*S++^mxCZ
z1>Yg+Izp+-s@cvC%-!mL$bkdoHASfKtQ(+9W0~U~ckE$MB13d;?9hXN(ZG^ob~?MV
zg_OO4g+?n`@DHvZ+<Y-ZTiJ{$Xkn1u^$Dd+KZj|mAiBjmIodHLROg12k2|w06xP>|
z6v1RBI^B))MZzu5HtnG7vKx-v?f6=vIGmz<v<3Gy`cmKQWn_j`mO}fi+n)L9ATL*H
z$WvNUOPvf1oLFeOLrF3)v}pE+zYL5NvQAYO$>^emcdGSyx5nBx@?XsVAk{_DL6gx#
zvxYO`5O(IE6mdU96)9z(m=0Rua^Oo3XnsKPj0pLN`XWb2f$6eO9$C@p>S1N;(TWO*
z&@JE`HwF*KVfoT)FY+k+^XG?m4^VYMW@l#yM#ZqYoW(@rb0=!sWh?ypwfGx{7zCyW
z>%b_5elI1(g}m6KetmQxIqj0vWkR(C0sUp?Rxt4q`I*H`&5ssO!M?>+aQypRWoYUX
zw-MSyv)4}$F;u*+<WEIoOghI&KGc+|Y9TI_d#mx+rqxTI6TINtY0?J&Z-C0t<|w-b
zS=%*{8Ow%4xVk0WOF#Pekm&XwiI!)84x{ha!vj8{w*`ASB=||r!Cya9rYK*Yu<P$#
za!y^{@V&c)Wl_WHhLEmDaZ>)yW*=ApaiVM@c06WQfx@79D*pQDSTafU7+0@uv^~Lk
zTCezFxVF2!s(M~*!EYuOyD5C%PPrXmI-Qiwa>1l!OI6FseDmc1abWLKNCt0cDPb?(
zeUvT+L3z=h0jigZXOcs}3;j5ZL!LW9KL&C;ZVvpJUyUW1*y%AHwTqe4w|r1UO%DAb
z74oFv6FPMJerOj*{fvd2=T4>^p*#4!sXXPpS<Bn_4Sfe<cNDi%&p<LnLgXGe8;;@A
z#fFIJ6{ox96<aLhrj%w~ECE%8bla70&d2L?I-n@t%C0A&eEA#PO@5-Z!WvK93S7r)
z+GnDjrf+ZB5@sp6zLzE7T7WNPaK>Uxft~rN#g3W0JM#J47#h6CiO9*6yv}smq5{dz
zXubr>*6+j2o31^zpmv8re+pI4(*y}PV`SLhiB}FE#_h914mzK=`{Ig~;^A?oe+&>q
zY;1o)5$@E~)1De*wXNxSSMp`ELkfuiDSa{U0pGUF+=5X$jx%U3uP^M?`@rPKRjfs<
zdTol{e{ZSZGiNYe{|nP-HeZU51`V*gVr%()+FnWLiYg`iMyd%4TlFm{Y5vwitw8b}
z<M9&Xef=$oVP63j2x~eh;RIjQ{8|-V6{&%04kzwcO4)frnvP%I0IP0kjZHyW#NkG{
z|M3B$!BeRgW-jAYf1$wH+D2NnlCOkUPc2|+&!J?viV=~>U>Hlq)$Y?a^8CiD$u=wh
z`*%Ehr;VwcGiz_|<X@W~Uz9=mY3qFM)tN*lcprwH(nIzknh{`%8#}vRui5lbT84Ul
zymGr@Oyj0pqtytvz8L|m3I`<)wnzF*EG=w**LaecTp?-T$N6{%#$h)1<~d%SBs((%
zwMy=)hRQt!?W)-_IR1?*I+8DG`rDwxXtSNVW_oq@_ZSJbew73qe*&H#opc`h$tk3C
z(m<FPqUyx3rLJk4ycAwC5q!HRi?%g5x2G38j6~D(MRn*isc+ude}pk)u309rf@Vq$
z?kFBo`oOP`F)ea4g7S*pTq*|!h<W!CMc9=H&-(!x70}Bk-h0Q-e-ki@EW{Z++p^Y%
z&+I=e$$KIQ>Bk|~|H7GTXlWP!g-E?VxHxbL2Df;rQRqo;M)9IMDOE&YRg2SeM!Rp5
zx1wsPqdkXZuoCh4p!G&Sw|IMYJh*<MI>qJ8mmGun17&If?wK8N`CMoNyPK{#BjJmC
ziAg3AO&2NqgM6eR`gaC!#y0;P)q1_8r3FiO8ED8Hx^z|1){X*-kU;XPUjPp~2E*21
zFj4Q{0&Kg(o4H@fu3Q#V`?PuLw%7|&=v0A=*01)svT8QF&e@>{g0jZ776=m&WaZZe
zl<tgx0aVK!_u&s3kLa51{x^p8Rehz9Vy>U!kZJP~iSd&doq;Yu<G0HV_1Cx~SWcLQ
zpw7tnH(z!iRi&oBR!pfC$+nLp3Dfk|^q=D`v-R1}i*>u3%Thhl*}&PVk3%Bvn$`pL
z1b#CM1W?al|HHv@5HYSUAENGem(p>MfKn(kq^jfnKL`>SPkqJVLSThvd?WYp*Z{lK
zr0Fe<sd@uh&#1mz<Ao;$65eXr;lC&9GeK<+-jols=4G`8$9LIfi@z@fG^YX*eN<iQ
z<}>Exs{{;ZtY%;mbz|XAwq^1>1<zg866f47G6Zn#m6-34e82WY^n7nEBmG{{ynbjY
zN0l)`S5)meR!9}hoH4N}J&p3{>E1~$UDFl{M+i_P`S{>&3ud-uc@K@dWFdBr-g7a>
zCf~ds87YLBsD?#W@HkYUX&F<z$6XxlAC7C>z2bUTAd$ysZpIXeags82P(G5=4MgFW
zl{8fZA`;OXkFF^r=!(uWWOWZGq!CJv)kWxdbIzz2Ln8~c#;57)jR<aX#0I+>n&NJ-
zHE=paj2;wI>>{$|xAJXlN+YOz;BR#%*-&(rBXXAJazrM#oU&5gKWqlARbRP0EXRH}
z^9u|q<N_lYm9cF2#RKe^SIaL|$f=Nb)?j^35|gGi>iz!)n(WOr;vS&S{NTmf3d)px
zZVUea7_36D!N;lhoub}A*eF0~|4XO~DfX}Bw&k0mnq%K8_>c<6%w!mfc0u2sgHE}>
z2hUs0LJ*q92hEi%(N7H$o24`oG0hr`_NMBgkI;n6vO%+_24mFhtAq*0qmQhV_w8kv
z2rMK;5(j~>z1k%f6sDw7H*$yhTjDQ@VP6TxSe|75;4iPKVPK#Ls^2x%b_GaahOlHy
zwS~EvFr&m9;3xrY?htymj;L}E_$!hT0V15)P&XN=YlwBxKpi6$E|0Yci$>M}(Qza_
z5sBVM<21}g^!%V^ad`&tk#u*z_PYI`Up){MgmoC=(*Jh^wL9V)11EkM<9G2Fe!$?g
zBlZ;1_UhQgAW}{tb|TvWjawS7CkJhwjKd}1$s(+THhQuv{)1_-sqS2U(3Y{N22S~#
zFS(z`6$rn)B|O7QM&H8t>k9Vhdr=?3<weH@MO8Rdx$}hL_WrXY8d&MoxuUs^gD4C3
z{LuGiXICWTwsS1TggmaZ<{L%J*H1x3;EKQh73^BMgLYSB+ve$xS}P-RwI^|QB#xgP
zje%hNw!SVq0UVK)YV+rlh{o|#)R}wmv$Uk8prBAy#V5l@ny*x65_s7GH_?>95Py5V
zRs?e4fz7jO?N)$6pZ|9;kM9pfo_O&-T*Vrd=pl92(6fRl9Lcy&LPITEryEgs&a+#W
zEOQ)j+qyuXU1E#yGD2{&i(zIG4W~$B>g;1^Q~r&Z?%bY%GXlf8InSo0<B2`IXTiZy
zo9o-Jr6B5Ze%vhI3=Z0GQp*Os);Uu`wPgFEVUn;Q2rCRR)EkNfI-<V~LHK&DnLnrT
z0Z?c7T(d$w@6=w7ju}|cp1R!Pl6xoo?A1wQ*4{u?O|+fKdOrFAf}$N2rWmX=TfKJ4
z(FBU*6kEd16lwmZf)WufBs)ghB6A7zL;$)#2L@$4mm~FI0FOeLC<NZ87=8A<7m_{Y
zUZJ`gnJmXLv-O<~!ss1{OzB(ezg_tvMFmPKx~HGHELB09CK#)2b#YC_`O(`xNcTsA
zB99e<jq1HZJhSp&Cl=1vzlO%#d?$xTgVwr7WEF@uJrR!H!pxN7^FT<RK2O1ZbMqHb
zI2$iqNa7|DW((A8=tsEjfw1?BpiN)wgD_u!t$v$QOGl^sVuLHB-n=C~o6GpbD`1aR
zxHYydj6d9RXFbO}nfkf3*j$5dPScUn*zq}ixnQKMZ|y&h^*nSnaC*#W)G8^H#B*qn
zPEoM{2Bz<HQ9YwoZ_x~m>xG%uS39UA(N41|8(p2!H(SQ6bn}}EG_j5iOeE0lte4!K
zi-Hh1A@JBN5dz$Crxrjguu@#0Ldo~rf;$^80()rCT6M3DguEZfGqbfB4>qv-K}QQF
zBYw1vnf+9=LE8qWjSb;0sQ7sDO!!xuNgSx%#I0kDI+K4FJW41=Ss70cw;`}Ls=GQF
z83x{lCr|r>SkS0msj*#@bJd$T@JIzGlO(tGj=W+=1rkHsY~2iMtDg~$$9iMY8GZmv
z7$i=e7+LFDUHwM^2oh-JP~TEt4PTUI4Y7F&o43iHCLhk$7D2YNBd>uXq9<Z_KV@`Y
zKt;%Gq@p2Z$<D-cj$q|+5=tXEL}x@_;5^Lp(f$=q&1v|s(>Dm;Gjy4sEuDlZ*q8?a
z7jVI_OdTge!FqUjRO$ET*mDIFhXWm0thO5()_A0-*-~Ny=Z_6W|KP6L_G5d6vYkVL
z+M9OV^gbqy(9zB6hGV{bt0koh5zffG;umYB58$Cabhw8*cK6dV<NJm04(gCp0z!zQ
zQr%5}-ZzelP&c4bUGZ)dq&VpVU5D@eJVoPA8Ym|c)l`bK(UL^YcS@#L$wP_xq9i_V
z`RrwdbP?r8as4!j>-Vm>-j+LDG$V!%YYI&}Es+3c+^<6+@|b+LNduvpKtW@8CIOB(
zm>7i4PuvF8n7vD{vZ!&Bm+;D!rjublQfy|6BOB-byt@+t%e|~<#Z&9~;6To1|J~)s
zLg&|O_YR{UN%)T#-wu%Hkyz;0my?LwJt<X3X`Ey$Qc`y0EVbHU@`;2{zGjsFrUm<-
zv0q1U_%bv#+nvu<`e4ph>^JgkKkNb7Q17w$C2TX0-&2D3$C!)BK1zdV1}{~TZBKlk
z>-*v3hC9Kx5S;W!8R}<kb`4^yOV6t}D-!r>q)FH9aXmJG_AXe6&)&mwb<D3rZ5!7%
z&qmzTCq}{VU2Jmq4#&lti>XZU%Hw-M2Gh~AAe<Q7jZYig*w)A-$9JWfOB<}O8y5O7
zsV5s`)<+Tfi;4X;ZN9GCL1Yays{We{KHg?*y<K+Sw-qyt=-$W$_`^@_Io19>w^Ze(
zcZ{62-Dt(M@-uqf0TO|=grS&YC|ID>e?@iq!9Hy9b8T*`Sk}Yo(f6#vnS&3{^vY*o
z<?H%;5{tfzAIf1oJ89bLnhrb@xi!pvKVD*)E1jP<R%sOtj@dFfiYYQs&WAfXa`f`E
zd*Xy@ujDtTXr63`h+wNiD_F{tSHo|K7#F85t_jgvE4H!tY?8aE^9>zV5yO5e(3*9G
z_6W3R#h<q{y9&<e(hp<SjIg&7BRfeEjBiv>9((eFB)^FL{3kxM^^#<KP1D1+?2Mkx
z6GW`_v<Y?|<-tyA(C&vSCbGzH2U_tw)S7V?bPogQVI@<d{hWeNe=$?T@QWu@HvCGh
zL1evbhB0O(Mig>kc1oNQWVpZYkPOC1`0g3%Pq+2G@@-5oT;*USV77izHD@c4jh+I5
zjY<j|=(@=emWJpk14CADiZ42;S}Wg`#@48p-mgZd^z`tLNr4k2Dl@8Ce0j1$HTic`
zGi$3{yCCW&Mm2>8fDzPHxO_Z>86`c7gWey3AwDK3JYA}WIpvX)0$1nQGQ?!t>H@vx
z0hI+uAn=){A#Eqjde!R_T31q-zb~z$4G@E_f!zF=BJpvJKhQBGZ*EAFkOEq@N3fu3
zl2?Lj&a_})Xt2E=G>E6t`v#q8G#Gx{ES4_l*Yi0-7N^ISV6zTnkNb;9`3^svw&qpP
zHaEMRbSQ|3fH9^;6%>#H0CO1}m!=TAetu6)zt6--p*V<XkW||*Gs{B=ITG;U3P4sU
zwc5BPw2~dl0O>q`Y)PfeUUvfinee@H8SgQWcEnm}!6Zfp`xlLbJC88Io1+q-Bw#{n
zVzH2sIBSa}h|6vEH3%nSVEpPhL3Kn#L~Pr<io#DaGd@El4P7h|02PHd7Q+;AC>sGw
zFNv+mT=CztCyC-P9qdajI%z*ofT2XIs349o$e;{`Y@jD`HHTjZupr<`C)CBj@BktW
z3{Rl2h+zPhQ9c13^Uy1;vL-i0dTmpk26(#g#u{LW1$k0vvu9e5Nfe;t%=IXJfSGlO
zeA!5F@ryFQbs4QS-nAN36`@}c&hmCh<(Nv@v99k_*6V-6&2dJ!RAE)DZqu;5*24?J
z$z`Pq{2df*4b7nf3k)(*od>|k@fjiiJf@Muf_Cb9g-3!zo@e2A{_6o0%xvW*VCX~!
z8;Xr%|HhpyTFhk<VqW<m2xQ{!n4dv__+?`pr}n=3_IgPoJ-J*fgCpVeje@Bf7Q89_
zE?p2~3dfa<wQXnd&GAJ$=ivp9RzYiMi7F=)kD*#z=+|<&ZBtv?-79%XXCywAHa_={
zjJmS2Qd*%$1EDp>LSzsk=_tZL)4aLdrZ95`jr1x$<cObzqXQ|Bo)~_%4qq%@jCs-=
zri~s>>APOY>Pr!$uNVfVMSjpfoM3W49zu*ntAv`<Pd~rid$eU1ff?dvq=`-uO&C+_
zyz$c9Cz0IW0=-EnL8yVmsQGLsUI>l2NU~NtK?oDld`T7>Lr59Iq~P&<*7yw-=$kUi
z9gh=w(5Ax*o-kicL@YpPfouVZ#AputE%9hq->MRkT6|6{Wx%nl7}agUJW=)qmNy1g
z9S7T)-My~oSIN&4k@5}#0%|BNCal`e(T^n@sBi$5EpF0mU;a6Zw7L_$#cDk{$Weez
z+(r)3KNOq3!b2QD%*h*ts_P&W(UP{4TmTnI%C<hv!C+nOp&ziIqbKWA;ZIQi2M_co
zBt(F@r;xo2%;nT%bNyo~6ly=9-UYOugcJr#ZAQXALdjT`6h@8eF7@3P_!a%yYAX^J
zfj>ONq}S~TE-E4i008m7Gk|~uV9u_nf&!AdzRzRY2G@6L*52l-UE<UDlcdO3^~;^G
zNHk8`P;qziU>in{cRUK&(7%YsVH_4CZ`~J}%2;mhO<<-B5;PKi9&j`tII^Lvj3ye2
zKte}%4(CPkH2WQE75+YF2l3%z&yyX#r+=7K$I@e}qxpIE%?UIVy-<eZsPol-;Q%+z
zweb4fY_sQ&0PB_SSG953KGCnDeEpn@+Ta1ybzH3dI}|UPcJDSF{oQdJ79(eoD#4*}
e)cy0n-yp3|igxCgjGQ3ApR~AwSdEB5;Qs@i$6IRv

literal 0
HcmV?d00001

diff --git a/docs/topics/_images/network_03.png b/docs/topics/_images/network_03.png
new file mode 100644
index 0000000000000000000000000000000000000000..472fca95867e2a8d04aa0336d0a0cd33020a262e
GIT binary patch
literal 45506
zcmb5V1yCJ9w=RkWcM>GHySoQ>4;I`ZxVt+Whv4q+?(R;41b27c=mYYfb5GTMuim{=
zwTqrLYo_g6(qA{B^0HzGuwP+8KtK>A#Dx_>K)_)@KtP?LA>SmYG*F-4{=nFXtJ{Ns
zAojigKz~sqV!sKY93*5!pf;h=;jp-`A)cH*L?UVq0*(%*)^;Gp6=ULWB6x>4k+6}y
zft{I+gPFAz2pS9_@tYj>T`pv8<7#JSV(I`=hBU_YCPn;^+8OD+s}eYvSsH<`-XXoU
z2L$H*iITOkgR`EU5r~$}>$^JK$H&d=4J`EDRP8_-yXvCe#L(|zRXsZ^Gb<C2F6`al
zHyPBw_Z{sFj6iN){g3`jO~~5P(#XmI<aR8f^Gyc%uJ+x`!NLfn=yZl01cU%YLRjFt
zOWN^@vyYB)`s?eSWhP%cX`HkhK|5%6s<^6fm{ry0p)<x(OUo@SRZB(d;o)J)(WGC<
z7m;YHpnz`#fdL88tK`XsKME!b*KQvTVxxT_#ED_r@VpUBCffHrC-;~)I?g!v@U|L9
zmxN?dwx|R}<I|ZoLf)@7Laflr;KDI2pz7ZJ)7J62VK5=l+jSXaT=rLq$0#=7FT{6J
zS*OqMLLzGm(QKPT5}1!0yUS#9ZV;6Bi?g0W0{OiVBXFGf>L~6m<y-EFZW7Vp;w6*-
zlmT`8g|e%KoJ#|(UQ0OI&h3#_G;G9-mZof3)mYxen<`YnPcG^>2Ev1<bI9~^%psIJ
zh-!rq^?Zz|;#3Hl^tkG+f6K5(F)=$jvbNf9;LZht>3!hd{8FYjh`OLaG1c^B#x*i%
zd)vnaV#mAWVk|+fRFNhFEbX9)+TF{{A1>c2Qzx#DKtr$k=(0fDrnsQHxvx>GX3AIQ
z<xal!v`~eCe$A9tK`!>2tB4&sv;kqxQm*Y)k!Op2!~@gGtNCDe;5cNI?<XWYIhxUF
zMV>jER+rsVlcEgH#O7SBj@Qh)8N2V)RBo=P6Z!GdnWM|iTds$#Iy+)fC!6_&hx*fv
zuTg^yH@N2O!SZ<*!}&b5+0Wl3DonVg&b-64@5CpzX4`ftRrqSrbV{X!->z*Am%x|+
zjBPtUnV|mPdMJm3qTRHYajju(uYZ58{(y5~>*w_>TR5?!gWQt|sd2VUF&T`vRav}%
zTL@KQu${9o=B5)m(-Po<!*$0qSscJC!W$)33Z8CuM*7uDE-*>=gUeDhT|w+_Rb@;K
zfprI52&>?i{?pjvSVdq=CNuB=7tW0DRcL#jr>kYxXDGxhO?74DauA<bgDpH|+X#NN
zV)vpRSDrPDJ(D#wcA@mNgLrPKE#sG*b^bn!&4q@s{;^kNerYO2L6F^Q*}-oz52rHM
z$Ou~8Ylm$@#QRSvbCvqzw@#A5(pzPVzmDB3cyS{o#upTtQZo!StcOCdtn~*%F453i
z-={2XRr^NEY9$FmmBsjea?RQPS^0}*?DZn*f~HpMt2AV9Dg2ysusmpAt|t53&V~r5
zsJ10;qtEP$Ik3CaGmJkRZ!jOf(-%czpYD2gcDAqfu8iR3!nxY<2CP7bE$m*Yb33?n
z;BrS!yI%=#mi=?!5s3r_3NZgWHjE~p9zm}mI3vR;qs^YxDAOGcsP8M?lG%2&g)w`5
zeY^pLQbnc(lBmj-@(Krt<)n&qKCofqRYGJ<tp%CuK(Jh`JA2DLHeJNuTCI08Zn>1k
z<u*1()B{we&Jyeoq7jyDALG7Df9=hPFxTDXbU5=!f%2-wwAbWaoe1PWsCC_5>lAoM
zXTP#)4Ly5xM6*ue2v5oJ4z~rojmyx{VyP^uzHP9xw(>x#My<!msZ|g4Rv#>=mIx*u
zXiv1CxqO0)p5oJjtpr;`zN>|*xJ_C?PoLC)-4>L!;j<iu!bSFv9IwdZQ{xwhHL!=q
zFPz*@$c`$UCzj($5i8$CJsVCn<xFyi!%L+QU1iq-T^v}7#e>GN+3fH9l&{YuH|M;Q
z%nZnNwY*qO{)m~n=0tFxxhGi|_%c7h4Q~9D<k&k*b(z;71DTqw3uyIPgh@|rKI;Co
zO=d-4<szNx#H*^(Xul=|x7gZ_w72ZcFnp4Ax$JB+yC%J<UK{*{&%r`{xc%x?(m{$=
z0B935nJ!Jk7zVIv3M<}a%CQ7UwB1;SS4iLmFNnO`%+2A>JW0S*91OG~q*v7F>XhNo
z5J&BZC)*d9RW`cq337m;k3;1sajIvKN~et^v%YP=OR6KaU72%~T!QYR?a#rCNq!G6
z_^2lj6!$xxQFiatt;fz+tk?6ogT5R}<M5CgGvG;?TwCy+G~+^pF=;x2eUTjQp<S!%
zZARxNtERNk9+=n7Z(*;?)YqN#*<kpJx#l46S2OIj$LDg7!v}kDyu@TtHd19rcznLJ
zjox4-{<<*CZ!7~L3ikJxa$db!&MUpD6G-=KI#wI_9XFmG*WIJVVLq14hFshfpLw{r
zmxe%8>e9^Y`MaXL9HH*k9(XI#e*~I|hH#+&)*QQg*@L?ueMz<Euex=Cpnvz%l}hw+
zBu&d#g;rj5c*b&(n>zrH=U1+#UGG*k%m~|+BIw!)`fLkleD)`6DDTHduad&Trlyzm
zVi8D31el5n=0%q@aC*+IuLJq`n@FXA@Jh7_{*|Z7CnHZdUp;1v8IBe_%e;-618I)Z
z3|hEbh8@2P9G>YRT8j;StBk=m*Gwa&agnES?t<vDuW};u^!NMZouhv%yt2X-W^10F
zG+xK7*|5eiJAx*kR`%|%N>(7qFKGpfDK#AfN4;0B@pjwy$U5%6Ckzv(+~1n;^_E9j
z_1Ef-B`fV~dSbtxo$>eda%<vbd5u_zR>jM8+-Z$H_9eQw7|g@%EjLFad2G)vUla{D
zDg1~ccps18J0@<r-~hq^A79(`PN+z*Sq1`ln2m4EdYG640yhkpMd)}OQMS_T^MrcH
zetM_e*55*WtIwXG^g;X>`WU3)doaA%ATv#-7zesoA)&?rOz>#X(7nD~ZB6M0ElD==
zo!#H~5gdNMyzt3`Zi(F7>1@GOp8|G77!e^}1^IC@{{z)yqgaUdkcr|-JQF$k-ZCPl
zkpGCVgeCmm{ay&XJ_eva4%h$VCQa~IaKps{mh3~h5`*5gRRI~|GvG8!)fZ0*9CD!{
z!cxOu`{tkpFJliskxhH7ffi?Q&YbVvHiRR4q;3)TP}t7|*7cX@Eoycj`TBF*HJ3Ql
zsxfs_cC*+%w6_1a4)^RZ@VqRMrG?msz1KSwn%#&U`cWc~M1B91uwis!nFL2t>ybpq
z>!4AWe8o$vd9qpthk)L7^?E~QZ425aLT&TG0Dyv1fcD?3qud^`Q?)Tm-kovIUbk^9
zdCoQYlL41oD<0n}91)z?fS3lA&k(OHV|^#9V;O7|#oViw|K2RMy(j5zs7{mpp#dBI
zl`ja!DB5bJr{>q>%$ImZ$J@oeJ03ZPThsw%!Mn&m7Pi4mwS0rUNIVk!%41%mf#H+@
z{0_&?u7o`{v`iqnF*`>Rq}m@eEVyV(;|-zFeH+ii!touMI7u@PH(XCf|ABf9MzL|j
ziF@d!6sGW0mgM;P;V?&cOm0V_Ka6`#&hZ}uf!)6`Pm(8Uw-RB$a9en0d&DcXKX%_I
zS&HSp`#DJ1-NkDW9oo%Bf1D1?qk*sU^E_S6wlka!HW_0v+xSd9E&ZC?dfZL!@9)2u
zOr`v*Qone21Xqj{Ycd#+L|mPj$f(BbI*z*9NKm60N94N$iP#)IS*$htH>|?*S}rZF
zOh*{(*B5@kU~tn2osRoEeCn7x|1D!dOKFoikgcWBa=n^7;qvKW<r)f))MPBD!4_f3
znO`W^n~&KX?)ayf$<f~(nzfn8gZ3V6=pczPimAmo<0%bAdn~8M1jdcynD8VUyuFM?
z87cN}lzW;UcqVC#>9vDKW_KmYv2}~fzonHl$caT|^GHhS7yi?bbLZc@<rXWcLE?TT
zho6Bj?PW`KfS{|jfg5}B7>Kle<inssb+<dNpObhJL|^m!Y?JI3TY=}n*1PTY6CS|1
z6U;5Lxmx!f9koASLDp|Aj7&0xW)`J1N}-xn8m#CG8_=^dcN_4AoNL+{MJz4Dr<INt
zuAaU&93K~WnC5HGlf1XXE*!CwEc<s3u+e+{G(om4CDdd8!K|tEl851S-hu|dH+d^q
zpBoVYq1UCLJPM1IbUprR4oSOb()y8ctp-7|K&xM@GUb73JT*@=ee|`JvHPU+i4JZ&
zdZFqK(OULR*DHw^sXEW$)~;-Ebj=CHq*Z<-fhTbHb4u;-$+oPCsvI2B2F3?qD#~j^
zNtcP9>T&Og32B2B&6dZh!?okn>AHH_T9s$Q8eC{R@4Di-1a6{aWG5Omt_j^9H`*?4
zmtcg!rPDlJfy1|Nf=gF;WOJYc^p}2Qez-5YJlRsQvjQVMyt=ODN=sg(=L^E5Tc_Rc
zcrP6P+{uk>ZcG4iZp?>wiOIc+b?j)cMDW5rusz>Qb)02k97k*HTd0wizT(gc_=o0z
zm|??S42&Q>j{9GEgMUN1eH8-G{tQsplYdNYWn}u~o~#r6$Bi8<W6!xT?<B^1Z85Vk
zw|HnDC4qR7kB$xzh$WE(Y}<jJQZfrs=E*}-+!spnRGJWGB$Dqn7d5`S8SS-528r|g
z6WnJW@25ru4oOe|p3K#Gh7wuUJY(j~(&?_;QD(MHjnK!J(MDQ!5;Uw_Z~lO1XzkSG
z%5Ou-Z67#U7%Qh9FUIcJ!__`*0Kwd=I2tGXEMoAVNXL;fXwMDtHF?H*yoOglfnu}o
zy8g7!sN{{=3yqA%60%T~?V2&d>bMlnFmV?ooBL~_ow{-Al%!^SK2^hlO#{#>o$)ZQ
zAFbh~gG=Qpp|TVul^JS%vQj|rWE+Xzzc{X7uaLp+9O>k_6H92IHLx93+SX@IR!Pyn
zX~DmoPmj9vF10RX#2manBtok6h^^5x2nBMx>@WWLJ|ERH$}@h4?vasc*_<l8_$OQ{
zT_x4?J(+9E#_3N_8LTA%W{wJWv{Q&31)v@t61Twt^aplsMX>0Se}uhyWH=}X&}(C6
z^BJM?jwOjau7a&C8eOC|_5rtQcr+eyTN-v;3v|yXYI7{qQun4;*x19v&e!cf;jMnp
zVl}XRmIquOlB~3=w=r*NF6zdGok|J%MK@SwqI}imuo?&lLS)h*8fn>*SZA`NUW#LZ
zj5fZ;o-*?9NMUGB&T72&Rg5`WEY~N))K+FO9qS9tvKf!@_(NqhCF;ekPD+1ubvT&h
zy!so(EQziE-l??MNFs?UVDO^CPm4s9XOR}@ea-ZpIb&1t00$Fr59xu(2vl{g?e0Au
zKgnRZ$q-cDZJF%hwaG9ARWqiq`}9G1+?pB7mNRK^jGJt6TV&I)fT%wh&A<yD>-M@>
zw<VCJJR1G=ewm-BP)3BI(60^#+Z3(Bi2b4QN+`7X0k9=#%6+%L=1EGE7!1)7cBX7@
zJ3)?g+-#hn0IiKmtJ^40KSk4}`k?-S_pP<PX;D6V&OlAp*7)Yn_xE4%p+l8L@9+B&
zTT|>jJdJ#Bixs|VkIQ6(8cD=m)T8jRvWqT*4T5#9?KCt*r_zU28!IJ@vq}sk`shK_
z-9T&~3E{xe^!ixDygr&oswXDcYA;snWFIFMFwY7!sC6~qT3pHZ3PENx0wNtU#%?pK
z((GNG+tL_ihZfN~dX^;_;Gx7|-jjhL$|GnDI_z-MDoSQZFFe*BvcPc>W}+t}34Wc`
zbz5K)bUs9x2iJJ8*ZiPKrVXd{4?n_Nz&tU0QCVHF9KusY#Y6#{vE54@M4WUY(sM<k
ztMhGb8S(l-#HW#;Ib-SZ>*WUPQIqI4;BkP0{`ZButM{pif<_g+w7**4I)luJLN%ws
zKJdPo8`uiO-@08X2Z1l=i=x_FcSCD-i0$>JO&YWew0B*LM(H~q_SoLMvpJh_ZF_ui
zeOj`o9!iThf2RmyRJT}{Y@2JDCoQ8e7H#F#SN1V=GP2DCKAcNo>^zNx&5BRMs?U>v
zk?feEq2y!pM;eeOsa;DV@R5DUBDsZsq_#x<|4lpmxAXb03<a6=A4~C`Q@Q>E`+G;X
zjSXkDS+SB%&W-u*Ryci+iA(Wj7O|q)g>H?76e$6=Mr6S7$%Iu9{(wtVfI>dmJ!k|j
zAMa{cjPec}!f^C$JLq_S^mn_g3`;QWZwAidTKOW&T&MFCgT6-%)pn0xf9~g2(}RCc
zKaPHH=Qadq4@vjd<lW7cT3|l|d0y-Mrj9iQ;i|%R#SLDwpPbn0F8|uEEECTJ9Sc}!
zn5M$Ty#(_S7yCY=7p}w`tB^$IW?%`4NrCkK=+&gR4RCWUfsxlypYD5P#1{OH<=*NT
z+NjEToo!9-KR0;*9R~s8Z%!v|YalaZ?X@&GG7;@8tmm03X3eK7RVznr+hBQz$xk`_
zSnAopyJP2w`*ah7(|8)-^7-$QmhptFjWH}m=w;=+2qqTy>HI(6FSbt+3lV4gg>0Mm
zuVCEOh{vf5!6f&p=D-rQ*30uP%{^rOAF|FLtCN`sfV(zMff-QxpBL5CsBK5^Cg+qw
z6fJ~*AJjjFW1BmSq<e4a?nWsNPMk$JDT(QV8or}3FLP%&dY}WjC%O!&-UuCKNf&s?
zgGPm>$uY9YEoZFWvg(u6Z?fBm^@W{KYkdjnXUlZ)*wx==^ZvwU^TFCWu}+C(M`rd6
z#`NuBMCtdMBDsm-dx>3q&MvQ(n7(WMGOV#wRJBsv#r*t6rW_#8GKxdW<00Ii_P~d3
zD3IkzJ6Vm)@RIdokg3<*Kwi7~m`%xBp*F&wga3FCq#C8kaa)_Aq=X05-LcJJgCynY
zJN_n>oZ_;72lIR;u{5ej0uA+yja7Ke+-SKrf{T@7q>lB?FxYqQr%91&-+*uLZ4#C>
zdkc&I0_0IT+1<a=f9KQx6Rq!)iSoZz0GD6MVL)-y`GE5c3LlR+!JwQpZ+W5$3dlUx
zye0B$YM4q&N*Kls#wmGTY@8#Ty>$k*4S4>g9ch?Qw-OZS0G_KcW(Nlc_!8YEwm34G
zoa}7SSr%wFIFHQ<91S6HVapC@CBO2h-6CH0?%%>)&q%w;9*(ioW{H(+nMM*^+i8?A
ze9Yb3b7Bwo{V6_S8*Iftf_1c#v)I0#Pd&lEE`4S&vk3R@U=VfOzgQBheX?@?wi6BA
zI2OTN(HG$0dqziN2Q(s;mPwC)L4EIq>k2nREOMxD5}zm6(R^iLe*PzrLE9u2Gt}N7
z1Z+$Z3UuHk)MeWl?oOJ{#$|8j&Jgm<+%DP;sNgNjW4ME-l_$$+{FS1Indwl}+36O2
zFV+1qw^WETK1(bz1Wz;1s6At<oTg!ool%AeE#lr6z%jf`hm$9ai7Gyeo1TxQZ*fHG
z$AU*TN#H6NH53AkTsE(z<?Em=Cc70*Z!pqeO_?vOCn)s@jq$IJ9(8*{b0RV{Dwz<@
zHnRYc$!v5Y2*OroP}+7-4K6{dL1C;^CafR$Mt_k3q?38T?Hn472S4A3VTxB58JM`Z
zL9dN}xs&NF04{czWVS$tICWf6Tbs`XS8&YGIB{^3E^7al=gtvI6Ra=I2cgvy)ywyh
zi<d5>D=%%5I07&9ug{}1q%;n<O;{0bl~bZz$+E5*?4cC=2zaVFxfp{?5iKJ$Z=8B=
zZg?@n{nlS1nFHu8DVhDrs~vYZO?OKylV5G3ydN3zC8WKsOi$^b`S>k3YAsPDb_yQc
z^T?>g>8Z3mYH9$75m_7@S)%4IT^DCxdV1Qalwfulz*U7UVgrojQjNy}ClZzA%T*7i
z^8Io>&E(SrvOZa68XEbc`Z|W3dmWgPs7SM!Tf80A5IXP%Ht}jghKbKspX<X2wj)kM
z;sP)}12-n^G%xRrq`5j6|1I()9N;Ac#iu_lG1d{Mz8lBT(GcT*h~`VTJb`~a^Z&An
zod3iENQKyswD^C70_~cu%hDT3dJ!gg|0yJw%Vw4}&k1t%gL8Uo2_`;0T^!5Yf0NOJ
zdqXXcjg76|XoV|)3{TZ)nF^rqNcv9Izq-q!qmFqg#~wdb-x93jC~|+Of%+XAl=}y7
z8f@?eO#Z)AxG*+1!|J=Ls+gzea~9JZbhw>pLWRM68pxu?OoP?jBQ+ZDKR@|6fKog>
zndzkj(smP0(0tTQKC2+d9yYiooyHME0Bve!*4<z^5P>~76hq87?z`N3M`%iaJ{vwB
zOecQ+J(u6c=J&4ue1aZ%9f{!kKyH7x|1AObjRxJ|a(h@fS0Q$II|~9dtKe^*GjH4D
zC5fz5%54{c6}W)7eXEp(c)(>y!M_!Jnq;J#Oi#Dwe1A=u01VhS#xuLYCl(|pY++0P
zyLQt4d}VZ%iJc|y<HFq>rWdC&$FlMFEG}fo)<H8ipcht16aP{>MDHsJ(Lfhb2t(ew
zpPo?PZ}AVZVyJk#kL*8N1AnJS^gf)`|Iv*7cknJWi$$(7iITFedOay8^z_6VZwKv`
zi1<f{Uz>EQAXqNe1nO7c$6m(9q8=U{Ri6;U^qai+(af$YjV#9a##%1fm)R08yU51x
zU%>o}I|v7?QmB_eo>r$5B4}-wIqM}hlMGj~AjYaAyQ?{x*Oszx_I%r8jJ6$Y<Rd{Q
zwT6eensd5Ux_XgAd6?KLw->zwKCw$~6y*)TmmL@a!}(pBt2yJMlCaBuc5+Nh*C$)s
z1nv|tsuGL0WU$|5bgP;bE8!%l(4O#WLhjNR28D<-@oXhi@2`=OreeNU*1Ox;jyQU2
z)CA9!wDhdu-bDI+N0jAKI6i-dLawM@3V3bC;Sq70XN!Py=iu`jJ;}A!JqmGsg6*?^
zghb&i&1a`Y8I$o$3<HULi9>iq6RffJ#o!=LeXid9I=}W<LKyfAMO@}dHvVi5=i!jk
zAG<=Rs--g0sRk<70QH$SL=Cdn+>$D7zC`T9DrIFV)!%*byXVuxtTCUb6OF<fe}f_t
z2{iKgn7w+2w8mHoGZs2?q)m6gu*Lvae4osSwCP<IGn)RqdLB+(3@hTWF|;q^A4Fm_
zbW#bxDD@q__0<G>VdU1A5B#KMXwL?YPMgNVppn^hf5Kp}T5iH97lJ;Jg(5K^Q`Qq#
z;_8BmRZyGVuQrv5;XgUwhw94WK{>y7OZ-BMXU5KB=-!J;=Fav-t*N|AC>4&w7VXG}
zIh0AH!_6E6>6G`n`J*SR6lEG#%{vM6H3G9Nq5b+v1+KcZJsoTE87vD6r_%x5q{PJ1
zW;U1_RyFOt22v<=U$OKCY*NEQM1+2IZ=TSl))2;-TS{8`e;?5oFZKnEPh675LMNoS
zr&&2s`hl^dHJd|1TV>(@Aeq|aXk$odWy(;qnoX9&`T_gP0iyuDBC)-H-lfZ|MeZMo
zmqzjbmX!W~n}-e6XzOx2N$>0mIX7~QD&>;Y8WMYqIlr%ElZf4&RrcBN7>vaSh-4(d
z=sG*7$0xR&_@X-bmM3RDyGzUPXXeuO>^l!PPdfJtZKsMoRIEm7w~S5$Yp@bq(Tz+;
z@ICt~!}emHhwUAnUD>r7%d)J;yAdF_Pqn1Z4vFE2_APU-uDETG;;(0~VjK;lD5sFE
z&9SwdPkJWt51G%0p3SJ+;aWHo5CjF+@?RUxhuF|zt~i<ay&@B+F=ih&SPkzU0{X#{
z&nokPBmFFs8lr1q@Oqf?YHtjdImXO|{A&j@a2sDuYSai|=6YoJ;M^1(K|S_mzugGy
zq|=P}Trf-gVQ_MW=gyYPdnpp>TTq8wC3)&a)aA&-K5)p*u2OzfYfbR+5q;WjNB;+3
zN05h--a1Ag4lr1Q;ug=Wl@AEjFm3NmUU{Fm<=!OM7`vqCviNhg2BFT|h`RlCjoE?E
zqYv~g^0Z|yl5_hQO#e|>kHH1SB0y(vHkA&j$r<g{@ObdOBJbm@g}8jC^IqJah8p{d
zVeli+H__2mH(l&)AFvK04a%VreQ_t)!V1wQ4xL>(47t|?SVEMliK&dQcgg_hh$Oml
z1{!)`^7XylVlUmPH7pv~&l&WqL}5>WYGxKTwv@>P=Z(-*V4lbywz;sI__sV@u5}D=
zyYr7KEvu%CyU;9sWinC;8x1<69&NE$JgDtO7}mFWNrJA|6Ia#w#u%X0dTD`3LuYCu
z9sp|Y+#>uFXM%#4uUU8f3ru=+6|WQ~pk2PB@z+b|kN8)Ytg<Q#^8RtqB*++ZQIztb
zIByPrPmwQdoOVci)no#CFL;*#xTMa2q3fTq%-~PBoMF@{=j<+O)?Y>Sr6KIhn4F!F
zn+SKCg(PIdH&~svzGyD*U{T0LySURPm?6920ZGe~9Eln9$v)d}qdTqFyEBx8Zv!cN
zD!{Dlxu0oylX6ExGc|mTK2y+~WG$%mVX8|WKT-K<oycoR<js)nkIS20g+@|#*C9Nm
za}Sr(<zKd=wEamPJ}oaWevy6}WS{YtgPTA=Klp<9%bN$z{>0erWkx{XiMBt2E(W03
z)bH8Yvoh}ROd<q4+goR;$UoUSvUy?gSKCB>URw&>6N7$qyRcIhmL$sK<3e-qBV$@>
zd#*os-Z{@)jp!uqO4F9HGlq#KcQ?WIBJ1KQjqHnu@KQq3?(Ww}A)}E(PSpr%Z3|tF
zb!yWYxCqJ<pD6DM$K8C4b7ri-lsSpWt<C`<d|BnYjM@$9v_`$S0RQvU?)h2O<lG%D
zKsJ6M^{1_5x2G39P??h=0ySL;P~93h=2`w(!l0D0qTs*I0vOf^)0dIXA$OY}61|G!
z>y43JCP3yG(KU<xWgLhscp8a)?v8fPN+eJwFuks(#}#cO1PcW7`b}sti7Xr0e*N{`
zHB`xN#XuzT(~nIbW4WBTdew~g_U&{8synqn#(4PR^QIY@tZvaq87`=>grJ8#IS2%u
za7KNKGG0hx5BWRbD{XdHK%=9h-c^g>R54kVN38o?!V`ffY?;{~WwZ!`Ss>8I76Qq4
z$CF}IdQ^J|q=`Dc4E+lk2?I$e6XA-twu4fQKvd8TnDtLIz+6}317Mh#t*Xu~sjl<m
zjZ_ih*0fSi=HTKzS|dC<837-{!1|w0T==dMja+h0I4p}SQ-ta9rG6TdlH#Wcdknqk
zxO~Pt=xeG-7<yT-gd_ebu%_+SQHvb~e{jZfg3Y*J_{!~mE)7ns?_&o4yZ-i07QeaQ
z6VnR+xpUQDEb<zOE$#rG@rmX6wc6K$zDbUC=BAbxLIJQjGoI%Rzw8jaNE5ii7gH<6
zctJ_CB;=-rPW)4k0}?zvj=V7s&X1`1Q|s{Agp%g8@S4y{K#<%^M)@`Gs@O%bXw4D#
z{^O)5tR-)r>J9P(h=`rD)4E5I*Lv%$DhV{?1E<CCCfgV2++QS>p)+?t@&nhAyz6$>
zB+<0kRMD+IoqZK+Poxc?qZ+M@!)pnWy$59H-W0xXV>QCo45RSU?DlQPvbca-7E-S3
zLuYfcMWxtj$I53rP|-T^-A-qDXu&pR8HeXJ*{32A+kUo(Mi<OT*L7w_BKp^eo71I%
z)zpOAJ8TU~rw3h3%ov=5vr`@hu5D}Uf-CdXix%GHJ*PNVwurSwsaMbOdo@f2v|g4j
zCMZ7qrM{3POtJv4$ZPzRiyU+T9gJsw>wfU?5V@C^6~o38{AS77#~s!@RTIX`^y?Qd
z-S7QrEU}m7N8-y2zOl@5M|*^CL9nuU_Wm`Euw09f*jSAPf{a-a@n(gtQ;KWZ-3y*N
z1bKO0YD=Z*Peb$G6DFbp5z=*)YifU#2Y_D(vzHN2$Cu7;GXY0l&}8cu%1<gNe+^ts
z;XyKn-%JpwhE}}|jCcCdd{PX?iJy`pw3-g1_WKKyvr0VRsB8If1H?!IFfc*WY?n%{
zH~@FoT`9<;$87R5fK?8$1B=I6I{YpjBiFuma^}{jhr-`+$<D*IbUf3V&cvyhGa8u8
zhM%e$X@)QH^O8v#Z7=-LH#3+N=`&P$yCJuX<nFO|$<Ze5n$X#^>sZydXlYbr@8N^8
z2RnV$`DjlDq0JTITT;ySZ(*_CY2s8fGN*@HP31650s$pRY`Ww!JsspjXF(i%6GmH<
z?IrM5n3AT+-+UVDVr(wOH~*j`>;LJ8o<GTAFs2!%GSP`eCbd1WQ9PpythDU597Z`k
zB)x2~jo17xsh1>XEf#bi4YArF3#--sIiqN>44|vFUTIB;R$6chZ?gWQAcCw#_ltYw
zGAMK0!XL;ycZsz%kxM0|b9^1CwH}CH2B=B&4*<A!5Vw|nPYTgtep$k+N7mp|Z+e;C
zyJs8E5apd%^<|;qq?TSoAdDm*m+$uoK&u65Ag)=DyK~$Mq6$Qh7j`9-1=ulXE@gWK
ztW^;?3$3&X#bUPl<{4f$AtW=TcTi$f-qQ(jl_I+FOb>&UxF&nRaTHQUn+ZyLmG3r`
z*yf57Nh>MTUcA1Wx}Ce-cWL_mD#2q)p^(fqDNL2PmTYu1=sSzTS26dhiTxgE!7+O!
z2q0k}9ez+NNpH6|ma*~|QhZ+jrWM&;%XF7aeA&7RKNs9@?-`k$q3l{rK$@>AdQD4p
z*OqHAy=;^Z!bOJ99FhF_LPn4&yrL)Q_~S^!tq>2SktgAb@lXXWUFj7)k7d+fNV0Vf
zvT5kI^fc2`1MBLw!%vJ~yW2G%G^*Ub`Lsvz<$8id^?Py>SsrZkZFcMIR<Tni7-aqp
z<I5fIuQuAl%bnoavLny>qtA$g1V6DB3VijB@sk;nG|$dMFVK<7>1P@{OYp1Qlzb7|
z2hB(}7?vY&11=PgQ=Ye=d9T;Ncbq4(FV~KcJi2?jNQeAdOVB3Q@Rh{X9AKbiXxX6(
zV3Hw4ySn0bRyf1VvXc6jU<W$VdP9C-`k**M#%62MdwSnUXetEifnYA61uI<v;3SgV
z4<2?!VNd9)&Lo=6bOXLYZ-ck-iU>}UFQh|Fc2Ru&jBZl3rXKNntSvYhP$h0<hCf|f
z!g#Bdrk;@ZWRAh)@B$<W4p8frVr*@Nx!MsfNR0DXvgwiGy$@Ei*pQ|OCdJx3%M|{7
zFmOcIEeTp7<@Lh$kZ_F_w0?G+g(<8j!EA*Mad*aRN;?QRIqJ=z(bj->&x6TlR9xeY
z%D&oD>^vq?R2~34Obsmaf1z`LnW=nQ#J0sQWeQW)M+%3;X+aM%&{G3nQx{~AM2+`8
zA^N$SzyEl^^?-k}(umkgK6H<x79U+@Gl46)lWvdMqD$wfgD<tsP#XkquL}V-+r{yB
z2lE{mT?gE;y2-*%K6@#e7FnDam;IcKRuu;ZtClPA0fc9s_LNFuNggH5nb_7x1>rtN
z^fY>N(5X=;ROgYIj?6|O63Lc`%M{5B``xVu%x4zClgn$PQ5N8}W4q2*is8k~z~fxc
zWmPNrO}{VoO~wKJ{F7%}^S1u5y(EHNDXi)p&v{ZB$V?4{BU63Zb~Vuh12k)z(Iyh+
zLo1z!J8+3)$$IF(U2sl3bA<cellG^rGzg5ot_H|RCP-t`w-b*eyIL6JYRx%O^fd6z
zJ2@OeFlyCoV(LUa;Uj3*SOK8Tf6|5Nup=DTQ@=+t7X+U=(+#u{_<f;;k_;WEo$?&l
zFS{a6it&_lGWf-zoVwrC5;r7cqG+##lLK|5@Z0Coj%xL5;R(3Z!HrNE_J9H>&a(3E
z`e$}{$}&^JG~Vz9Uth$;W&Fk|_ap5dkg?|8>JeO?B_Ge{5Cb1e6n<-pW^2o=n>l2A
z{V(~M#XY7{TR<#HYI7TaOcE)IUpp48mAL*BoLu4a1f3gSYmk0z)YbGsypBrbfmrDG
z@~o{x*>JnKKJtkkS>uC;bIF>~S%?^ug9RtZolHDQAQacW=d1pMKXmbuF)Hn+8oa<y
zOA_;B6wL&(8^u;*r2~q<NUE;<y@WzaHWLaDZ{BAB*Toh3=_=2kS8ebZc;0#X?6rCu
zKBh=vk=XP0KgZ9Tl6C<MB|#<layUtwH?ft5VfWndEvp?#FSSQSz(m`Ho#1<u9)|PN
zzk*BFPI}>$d7A(e-G<YG$F;ANWwi1xiE;-R<jAZMzI;Kxw-)ElXW3V;mbgh&s)<C5
z^$-lz<cS;J2E^q6r<jLpVM${oLeARc3Udqt651Smj<}jm+x=!M)@B*39<i2l>kM(A
z%v_iA&iYxz8}ovL_C{u-Jr)6l%_eRBc;cOLvrGO)Qg0mGNx4!=Q+7Q>jBKhH9Ere#
z>cpL)scv$Lx1Z7HBIC7mc;(Ey_7_52w$Ee<#7XX~)63wFck#RqVBM-%67iA?+y3&a
z@Pc8v6eDkT8{-<@YinkRDYlj+@QoDMzB#^@K|Oah6!E16)$LkPP@u!No3U2bQK{aq
zS;tB&@E<9&OYO|W<UD31B3GJPlDfUa_1BjADP5FjXw@$Pka){MW!q*s{&Ol6Z6kug
z;Dfmmj`aE8&ba(fF0ax(`)tD*9Y$`OOhe$?lokQSAuq_g#Xcy-5>(0}uUGseZ*0zd
zszHZ!Z};7_$XNE{-A)BOQnhUFp5?r%wo=xe(o;w4OtDtE&M6GXXiSY$uWUWGwo1HK
z$GqsU&U9A!cdguQP9&t)X}0@WjK(w^DR{I1^yfG3*!T!mWacBHrf!Z$g|UKM%$J7C
z&xl_Q;R5GWZMEOl>oIZPB>dXA1io#=SmvoZPkTQ?+LeR%zueL>-C1Wl8=ZB8K?dz8
zxahJcM>KmHna#Ym19Ce_PYYi=ojW%#PD{uhGAB=$B<{1coT<%=pAM6Sqg?jP5})u>
zhFHJoH2>rgIfR>)h;Rq;7jGJ99Ox`cJj@>rH<+5}zSYjj`o31pRb5~6j@H{l_FvHg
zwV$Njv1i6X=l|Qw-Wat0Nz+|{(_2+(eM|21!V=1iyzx}LKT>CXn*A}(T1#QqPr+Sj
z*^2`>qCTI!%%h!1mEcLugselB6=<L^Beo)kP)<OuWYq{{2bZ=6MhtaOoOfMB&mUG{
z+|&i5-g6aW$d%tvo@Q>H=McQ5VQN<e^3T3spWZT8&ypArq(^WQg_{gJyH1ZFheQdE
z&L~}BtbF`R;yeHP?F`CI?ZrUH1MwgauiqO6I4LAd*$^e0m@41Ark&ip7eeqYvkpgo
zSn$*|S{J+~)A`ZTbq`6FbnwK4%*suATM~5+^rtCH^_vXDy$;7+KU0EBaSzK@<UL)P
zpqfLAzgjRSLp?wTlTreg9X9KquRWaJ#tMfj1E0yJeZ}3>dk_&51Gz4cgpV0e15m>3
zm9$Z048K(@yPUnBlzW4!_Pp#-YdqcB`XNB7d3pG1Eeic0d4s{xBc4;>TN3SMB6+Q*
zeSQC}-!OmrC#e=3E2l6PXN>#tFrGk0h28aw<q;Ze6Okko|HBk$3}IURuLf9S4H9Fi
zl3?qbgpaUr+nEy^PbXM?Y`judQZf5^y~lW#!F{XV=sRm;vrf;DHHo4_uS>GwoUDCA
zvCl8Ym-CZB9w}9JRIFFj7YQc>m~8PW8nl0%8x^>`sj5|>XWa`T#xnTAYk%4vnulmH
zMee7@9=A6<21>py1b;#cz3{}pye^7n*ZW9(ootlwG~miNAqNe4S#rf}PlZ`ykE}TW
zH#)r>&%@XW87n4Sn3CA_W!ucell^09qNmxo6UZi**=0}hIs<>)AnNw|H{!Jbb#q>h
z6wwAaMf?i?He$ls#D$Kr3@m2M_921EtP(q(VV|3=6*9t-CCK^&ZyOM4i0VseenKa6
z^$i?Q3Nq>znlYK=3H{n&a8B#q_vB^F;~V%T)U6VG9@aDA#Kdpr|7n!*VMZ-R_DMph
z8c|_hIvzPa)~9O_M_=#$)00cu{ii=1kbU%k&4p<8u4MJlPih$9UfJagB=EkfN!e(w
zgDzB+6)u58#C^uU4Ekusc<uv{O$OvgUfCq9hPi%tq33UAPLgoJ5^S%_4EK6~+B<xz
zhAUy9FYCrpW->LN6;KV=(urI_?%%h~dK9imL4UWn4sDah9>``vac@4|L6yXl;8GR+
zi``3~Lg!D{+u%&!Jal5K9+I{B9=}CS<mwnbLT<#pj#?4%w+3v$?J;b3*H*JRZ|=Zt
z<NwUB>d4%jak4wAh4_{<9NX!Pl#FlHU0FuV#3oY4CFfNxHMNDB1UaBr=q$n@^)U2e
zg_Laapv^{g*6hS|cS=~>mfCf5(clo%UWry&9<}U;*mj(4H7Dd(L9;pmc02VWCs%se
z0wgI(2>kL&W0<D~qjInpN5yYCb->aIh6yyt0C|^b`_!`6a#;S}96M0U^*WOHy^HDU
zjmvhUwB+0&xdzc8sqm%MA^P&MRYjFfn2NBik3y2zM9U0gdM@cAmNUk7^&-m6{ZmYU
z7Q5i90d5C}`#KCk%~3<eh(LseNCB<<@gY07J;ww=Tpdt>0V1wqRBkg}4#qM_@Kz?j
zM?-vg8QnpIqVO&FC<9T#WepMrDauD_@W?NuZdQ~VEqqU2NT>>F)wW|z{=9XcxovGU
z7}FS$E$ETSh=m795OEdzdX8x}on0X!g&Zu>be;M8BS0s{AcCyd(_(t40J`iXRn)lQ
zdH%dqp)unMn!g4n12~uu#m8v+>nzF`8N=szyztUuB+~TqIqD(33^E8X^jS$54Cwmv
z)yuF*llqZs`Fyp|`pd$%!Xpe8$W&y&OoXq0L4E`-Sr}25b4}Q{A$!}r?47N(`zZb2
zZEr}TZadN+|I9r~klAjP)g^BBdRx?oy_Ek|f8VK)^{%Xa+aRBKY;8XGZn-~Dxif^i
zdf0R|>}ZXR<aV05I3VA5rQ?@xEA@{=;uRevZhxfOJpeAF-j1nB-_1O?5<5?GTGKY?
zT6xgB@0F9k!q#6RRT@lc{9{R99GTx1@4r@vPH*8iP`pX)uGeZJI~<c$D-IuBBFx&3
zM=iKfgABdfV%r<pW`MxK18tzB>idqNo!!C*NxB-lCx`7ZL#D|<IU!Py?rkrF#R{0M
z!*|I#y1J%HoMzVlzEQ3JYciQRCvQf?t9d5h-&ZSPPddz2_Lqc)RL0&TEgjBLbd~I8
zf4X0fX-Etc%oUy}6{JsRA;x1G>Sbx`IX!JJktncnvioJgSkw6x!CrhSgV=iw9rlPf
z?MN(+hgme~DEIZz9U5fMdLRbpNZzC3Q{(y;-yD(wm4xPJ(${aqfy!q0^!`uVKSwHy
zIURgNGKcub@^%lyJU@Y+wy@P86^80CgB4ooESAWIm18Zla$;)99@084$i^%SjelnQ
ztZPU)TpGO2R}(mvgSdYST@WAFWK0o5NY-yuZis&ttVfAD&#D_ocGZd=9@8r!jWg8R
ztBIe??}V1*>Zc7@xFHVxv!n}!NlLcO6F##5^HlflE+bdXcw*OUU5qkJ^T#<65PMOB
zD;tlC!QAhjpYD}JS}vNby%Gcidz)>exHUY2`rtu++-|#?=1Q@-SXOM->812lq>CT8
zvc`MaK@RJI5T2gml>x-ISRA8|_@0!m31$IH`Y62fM9=lK+mBay-Yo!K<4-ZkWXZ3l
zdB}dP`qb@rg|-vrL<>z3n$Gb<*NS5`95ME#A(0W_$dI6V)8sS|0RlrW30G^b3GXES
zZMx`_M~Hel0TWtHr-ccDHN1`LY0eeK2}j(P%VcBBT6%6%L&VrF*ln3)4w_OD3b?z+
zr`sux+uwFMvyzN)WUCc)Omg|vCJ6=(4l!O;aSWTt$#}Uo-qih`p$bVy*?Srq7bhIX
zb&1zKbRGn5H^P#8bBYh4$t2ja%Y&rvsH1tOFaZx=;~pEoghbj?lviPTLWr5vf~NOJ
z*O@bT1_%k+5lAH2I&m_~N_Q2q&-USIbDi7YbdawH%Fc93qq%zHF=vq8nIa8iQ!yag
zHYuyiT=p7-#F2RpQZ1+R8lIYwiaW+Zh3ox-FzbR2nMnO+$3?QH*?P7Qdptrk%|2iA
zy4c#89a>?{f^sbq>TL#=Un>O?;%Pi5Kn(M(EI>_}^Lg!*k0J%L*wv9_HHAoRI(U8u
z4PPtl&$P4?K#1ulS_^0l<sV^D2b2sY84ov9pUoNGJn!jHUVVB<^a*#5svP0T(w<^Y
zmuY_q!$X_cGd$nwf!y;^nruDY`9wX?-#<UUA9<41DzuFS!O)f;QE#Vn=1Je(J7EA`
zas;>BT>ATT(A3p^cAbNBHQ~#nH`;XJ<EDM*IdF=W>X=8vv{OsT#Kg^7rbjR5>QKYW
z`VNPX<H=yqoi|qGeXs1GSe>^VNvp*c)Rb*w`J19+v+186Y-(SH8pWeO-9}(8#)&CD
zF2xM)34)l5fBLe<lesy|z2EGKEif#sSZmSMf5vU9R?H*09ftDV>jplpH&Xk3KR<0%
zc8er?h~0X$YREqX^!2`h0wwHezY*(Y3TwUcMy;B?d!8=LZbLY3@}(tJp%=a&kH%s&
zZv?T)-qRPF!P~&syCXXsjtc(4t^GB#og1k(mkN#lrTpep<7`EOJl&HtU>uo%Kmp=6
z%AhUI{IsozfC%pQzD$A;zg8x$t*o4o$PmOZYNo6*6mg$#Lo){s39@cy9gpqGjSXw~
zE1)S3!Tr%w=Uy3wnX)CNR_R>Ici-DhihFlK1M8(gU{Dcrf*j>ShoEE$9P5H~DB?Q3
zptRY<HlCyOPa2@Du)5v%^qI~OK{;zoRs|~1iC&&_VMt}J<yaV==C~=6v(oueTjZRx
zNP3EPmF|?qw!>v>_Lc=U!WZj2Y7AyRQ!1eC9U;x$S$>JM&Og{({O#4~|Ah2l>&`3s
zGEQi}0~R0P`&9|_6RAN?(C1_B#<<oT41RfeIg9=@>`$M2_VO@+fBmnu5{aINMFJAX
zN$if$gC?876;br@HG~~xlaD(sdr$3)I&Qp{uQ1T~wS_FdA|*=W)?3xwejyA%77W24
zBZc<!Z;RX!Dbc;iz^118fl3B?4gbewr-Z~@qu>G=;av@y5HTjvINC=}wXwJ0VqSUR
z$U%FNEdpEl<5>VlfOd|6-@94>vOd*Jr|5e@5Sq~!0_H;|7y?5g{_nA%G{Jvsg2ABv
zdmkCCt;-K03IW?vG}3|TpUee@+aeB}=Xy#=Ey1ztvZaM<cP!ao5A)u4RO?QeF)@|o
z%hVuw@r?&GhzO{Rf;{v&*!iH&YDswI5!Cvp$9ZrO1p?@SM9LzjgYth>%N<pHf+_wU
z=IaZJGwAA}PJ)`9n+vWrPr0Mn@tBA@fAg?>or`l}9G97n)qh&w{FFhpvg0G%vwb-)
zT1@}f<B<(;^UKFlJ*vJG&BL>~$NO0)u=8_P#p3qnED?<)m3$rNYYUaNmwV9cOz@Fu
z;76@V9f%sy4CZq*9&3fArEr6`3^H*d0QBkQ<?xac5+<rlHzB`$@n5PP<o&VF*mK1+
zrgI8*cu3BR79~Muf0c<jxkoCa6q+5he$CS|Q(%~1x4`bDeckhf%XCm9OlL0-WuUif
ziv0dD;2{SmRS^<Xj_I8CyH%#sMEcd3Qi*i=<>d&<=}7?u(C2N>uUE~^2ESshY)&Qg
zCae@fA*Q0In$D_GD__iD`1Kj+%V!vf-`JK&=G?|UO8#N{`L6q5Oe*SKOctC58}a|H
zg^$ZTW#D+R*38O^3<nV(nZQTVo@)IsHx-S=%>6tmWJ^Jnb=Sr%2p-O@IywZr1}K1A
z`iIh2*ed2Dv~{~Xr>L@Qd(rf$2Ja_<;sKNV)|h`YG?|17Az+h-c463@{k8G#5&4gc
zOmL*?ty<$`ehDQ@DQv9<%ju%LetV_8h5x{m6<~arA+dZ)ZEZ^dBxQzcu>L@prR&1Q
zG-`0nDNgiC96$Zvcj-Oon+P_{KJg)0Q(gUWq!WULFiWrOU>XSvcgj;D`v6=YpywLy
zzwkI&im_n$^FP=nmhpr5X9Rr^s3iZOO-L3^An?DVyx@kXPXHMi0vV(vCyPR^HPoeI
z;Oc8&HN5J#WR_lVFnswh8S}Fj;j_dab!L>CEyu236l&N%^?L8PJhQ5M%^m3D9+0#r
zx#I0m&of@fX~BE1JK13$Q>x$?-ye%5A65oS?;*OqZV1@1^9r8*T3P2`<No~KQkR3I
z4q6yuHvYebm-WsCAtysu*3R*q#>Qd!dYQouYVdv16DiD|NN1lY^RxdPKm>)Qz=E$M
zpi@5Q!I^v`zNWXAz=84_&Ux||qqMvY@z5u&W-D54OTIiY$QHWy`_jj4A*qVA!HRDC
zva9<v>Y-SQ1`()NlX>*^e=bq`!mG8>3r}BxfEL>lz6X9<f_5@QA13mEpkNhzfs~k!
z?joj}EyXg-9L1Wm+Bhcwy;H$tf5KDz^}(-dT+<sZ`c8|lm?C;jOD&#0C-`7bPt1qi
zMeKbebZ`Gdyu%$a258{xuS4~%#kY@>EdT6o9c(wuILD+BRjXXFtqAay?GJWqRXWQs
z`(V-PLa<QGvRFee|2VO$6ST%O^xNU9&JH+lNAT%dv;|em!b4EpTGOXv5B*9_;oj7*
zu3nF9Fz<8RRwunw;v@v_T{M8-LrCZD7_MPO&;Jta?P7E|M0VOQt{{IzWT0TwgQ>$I
zeR9fC4EVvAD_2(1$b#}%s3Hv)Uuk?%U8L2SE?5|ZQgi+oU#$lb17Gbt)@O)$606O_
z5+5ye(xFGQG{DjFiMB|epCjy1{L8$&yh(62Jc&$rLT?7#2khu!W*bNeIhV3{;mji=
zqbHV@OZUgZ!7B!NkEEXTQ|<HO?op!!b$)bXm+_9f8I^d$h<j~8#9B-Eu>JXosI3(*
zAMD#w^y^wZ0(nl}@z>O#%8bi642#zu1<|mOC(cJ5o5KZ`Qef2#nI3_oM(oCvsb5t)
zcm+}so8;>~!m^3eyAu5vhMigBJlD9wP|}A!jfz+PnNAcPbbF<MeAtABbrjY{3T#~V
zby5jBpTUIv`Y^NL63^8Ufi5Vg5npj2`l-#@niM#zG~{81Cxm~G?djxG)0F&RO*_D$
zR`bv3)<j_Z2sma!k->bPkp0hA{c73}5o|mF+wLmJ2MkdTBOhwoif`eAo`7y50abey
z{F_-Q>R|}Z2Ft%0yB3co_|OVS(*295iT=-OdHDZ-BF_JK>jxA}phtwc73Jj>?rMze
z2q1G$@70XF-;p@pYCo^We3NF>i%{yo#Gi-XBuD(h-IIqgxIyM-pk0kP_@{~`{@w=Y
z$Pw&4S^~Ek`&n+wB;WmRr2oxPdORL(zKbXMpLV9D#&<_l(0IVT#f0a)Vy&ml>-ON8
zR2V^v2u7viQ%Xf5G!i4T36MZux4`bo$VYiHq8|tY3cEpI)cn_30N*T&tq3<4A>!{=
z7clMhF!j6Z2pm~He{n=u32e05I;y(_&BC$*!P*3%95S<TvJ3@HZBM#Z{?ci)NdZlI
ziz;Qa`{1v+68~BXRZ&E0GZKCglrp984Q5v(wgD4QF_nnO+V?BY4(gt4MxIru2eHl*
z)s5(UW-s2d!q4YI9!$#54RWlj3(cffd=|nN;0Vb6ROCoj0_xQfq;uW<UExTus6x#0
zioj~5tGzw({17;FjQBnmNfQOO(ODNM%$sGMg+rMcIB-pTozI<M8rb@pH9w3Gc?~~*
zdM+aCxv}+A`a5;_Jxg+9B;vcXS&{aqNz+d_RGp`J9#Wr*ax&~bCwoijz7_*!WJErf
z*eoj^^5ro;42apvu}(kYv<;{n^7G>pv)DGScF+XkALlgWN?s^NjU2LlT}y?upyKxm
z{~H~}><%fDIM&l?LcWL}k@J8kOCz#719X{p+Q3<p-Tgw=FU7Ute-)}uZ1`H6nYF2S
zM2EuV{OO`f>ZFo;*m4<w5gOg5OZh^(RJ5$*0}5rWp3>*Or#n!(-B};;BP!@`48cW*
zYXCNDxB>RGaUX-MWxu%G5hT~UZwWcTA(0kPQuEXd-`Vx8pV>UiG7FfWz<s=%b~Na8
zKFi-^o4)sNrpV<D_bP!6M1fU3B~8lwcBNUCc{boGgWXSkw_e{reQx8`0{F|l=bMe@
zc_yid%O+&KB<`roH)4Z9tcnPxu&JXZXkkU{s0z_^1SS5mPSFDr5BllqnDlpM0mD;d
z`tPut1*G^8&rOO=fJ2y$j$HE+md@+2$Im+x7?p{Fjd2A#_G6+JGMV!(hQk@G-`)ig
zhpbw^Td4E+n_{zC<JM2EBye%TyOuP1PD-=Nv_!W$+4x_IV`>MkjP-VbT;LxD7(5Q3
z7ym~1RYAe=;OBK)X;U<GZVKMSDJ&nX3*v}Ed0uB*fIQ}u@@Er}UDnv<pPbpyS(Fsh
zL`J;f{|{wv85UR5ZHuD8-Q7L726uM}1cJLmaCesu7QAuy;O-FI-Q67;hs&Gy+u#0k
zo_p^7-MxBMtu<@Tnxn=TJ%tu(h1Fv?OKDL-#G%Jd07muJw}<yY959klsfA&zhZ&7;
z^@Y^I`-8HIrL2({-m?e>nU#&bsrkK46p2$=<v{$hqdg)Zojk=J7$rE;=OH|30mupF
z#8e_HKFbPU9jp>yrE6aPs`T18zxh&;;8S%%`F6n|4NMcQ<oa6#Qsj3?yAK+?MqZ4{
zMqEql1|iq0dnmUaS6$syS_p`c&lnHWNsM=IC>}!C1Y>@dP^|tA>q+KiWzWv!4|-sj
z5#K+je(zTly~`%r5PjjK7pa#%x&auXB*T5JLzfK-jE(m^j?F!^^?=XvLA2JONcD>*
zO8I4xB3&zcF4UkR2Q@3px^0h`13F&pJg0sJ)6yyHdjZ?Hp29YnR5MHj^1GT_MWVg%
zKB0$P0~!+Ag%nOn%(LT)oX1FgQBk$ArDTqcyN7bdb^eAOh4e>`b$^=E3x{yM=RGCx
z8&ae%tjSjCy@d>QBGwQq5FqpE&9XnQ{A^bh#>N+@tulozPO6jJ&<`P;iZ+-*6e^16
zrRHMuO%Bnivj38y!SK6a77Oo_qs;B(?7_&{&^E|@5|^rtG`KYno;~-_ki<MHq=h#0
zYi$j8GTdinzHQtJvSO`S-VvzCNTWjwc9(L32Icw$ahw51;^S^B%Yv(Vmj__%dil=1
z_5&J*KpqAr1J6n+R9b8eX>4H3jkM`!7U_^Foc^go$705Ray@IRB_U<_QHA){wTS4u
zJyTAR{v+yjwf!0Odpj9~e~60-Q%cklWrgD3SbUcGZ-o$k+WCLo!2O?!MxZ4TB@buT
zTbq>r&>-J*ghvem#7kg|rt8%YF5fW{9h+8nDgcdM+}>6U58>tT=OW2HuM@n7b+GP4
z8iuZkg-e=8uQO)A)O+d~Pl$P{-9j(H?<;F2y>+t7JyX5EBUCcShk+BkU)R~47E0oI
zPli6ny8}1}>aFqvhTXVWJk?P0vf9#LsA2q<b@~3hnUg5(zR~8{uLZu&o?$`PGsZ5N
z5u11#rEl*M&>|B(;7}3uXJCkUG+-p(fmk0vVYGIKW|yV5O;rEsxPaqzK!yfmy~L{a
zRfA{UZyl{`d$StUO@jA!<&){KHv=IvPnUY`@%UESG?N;$;;_%}4J@7#D2iWu*JkB*
zeWGMCA$)U0udGjogbvZ4rz85zmoPi(BaR{A;{Yc+IhGMU2hW$<2(xxoh8=!%sr5?o
zW&5t@6!*P^H4~pHn@suo(`8c*MnxhZwA4E`q{jXjzt==Ql_KM>&jqRu#kD%afYxik
z2Ke3ZQk%O*QSq~>6)=792mAlcEkhi>kzZ5SDz~+XSE4RU^J1>{!1p4R<`eOE#AIi}
z+Yfkh)8MUjltsJBAM=H_)jTOBk11>qS&6EzD&Y578D!ebg5%2E(4S;su24RU81mKo
z)L)@ESKDtulIV-{CSiqku1eY3!axz%ueccs{h|V$hXn8Nwz+~^6{c?!VFmhuUpmZR
z6trY#7FN}>{8$15t*Fg*EQi#3A|s91hT0QSxvE34Ii_{41){5l6ZdMh{Otz&L=<*N
z!B<NM?Jt;X1&O!c$8axtbFDLKs69+DKt}nrw1z(AML${f2&(Xk;fSb8^GrtvpDT|P
zki_TbbG~s6H+%+%w^|jBvBL1gr`tlDa>Ad&m0XJ>x%j=S@~Vfk<@TATjfJbhw@+BX
zLg}aglLi!uU5CectB1opsZ5}ep>wa?q%U$Loj1U#tTDT&H4Jpor23mAl8Y@@@5who
zE)sT59P55IxMOew#B92Nm2;-c-YdbIZI42C_b&v@2k1>|_3Ka%SU4LXxX<>d_la5N
z=$+1R$4@?qjfrqC1~%!XQpyDH0!yx(I3z)rLwCXR<D@QT2&IC1n=t>}3f2*Irpa#3
z)QRM_1sUgUc&j)T6(<9m3^G`ho^{7kNSF}fX*f0be5vOUCM?3+fwe?UVKgDZc&Z4W
zx{fzMG@>U&r}~L7Te{&e-op?5W|spW?72~8DtRHNDnrDkX8e-7LFMredLRKW-I^K%
z>9Yr*g<)k%vrsvgKHIolF|UMIWVN@&gPT*S_kzyQcgF-@F@mI|9POXjN@++Wh0VYD
zIPO11Z<Zs;X;G6<I&v>mdRaB?OYV>T&wjL!2oAk3Dj1Mev9&iy^DX@FForskUN`pd
zHWbMCzp#e<PQb|DkQEI-2>SmFhRE*>&GP%cJ)?6UbaAM^z0Se?8fp7BK{C<XQi9OR
z6f?~k{B#Z7hZFi1i&C!gbitVTevJ0k#gAT807S3(MeA#sGATin;^l?+SR-M9#M8`^
zy1@CW8%F+je8_hAjg-Z0cD8|K`dORoH;|R<-wAY1VWRiHd&<4VDXd2+pl*Up9c8&d
zVnIy-5n0STy94`Fu>2%$#nI*;nol+mWZoiyq$7iIx)-EGhJeH=jAY3N7{(Ib;X-yb
zgOp)+60ybuT5Lc;RD0ZkB^K_GuAk^xN$npedK1I>=W)1~qz*8`!8d5^?86Z8-hfvL
z$b5|8an(~Zt^$db>6<O_5@kmUE&Ko+yYdVi1n=ZA){9gl`}B-xp-gucL4Su6-@wt0
zW7SfqE?i#om(``4627XWV-#-X5;GQltOX}&-zP0{OAOAO9QFH)zFf~UF8FkfQ7x=S
zmX4;J*uEw3oUbqnNqC8WS0&JXhDX}pLCbiLFr*=XzR{Jh+;%?PlHOV=(kVBG=xDdS
z$pSwIV`cEIoksxV<x030fIc59QXU7L5*wRFkefR{<-EtQ-a2}T!=*aXfPM(RzZ5l8
zj=TlhRqjbRH71oa3={2528eP?;pugH*?2fnlq?Yb%KO{3qspJ0OQJ>`60$Gs=0o$I
zBZqBW5lgu;W*-wT)wNSn%D?D1xzqF+8i2VqM&>>$-@rk1-FJawDkEZ52nIvoo>|@w
z1wsjtR)fzBo-u9xYVY;r`5SRYqa1cJ4o#p1jq`h9vducRyt+`Z!2{r(>5-i?PyUYb
zQUJbw=(puiqSOe%#(?5`Sg`}_Ngss0=#XGG<u3+1Z2b&w${n_-3q;@>6t5t}t5D)l
z!6xwf@%}RToe{Nv%BT^(S|BNGtTQ<<-M?D}pbA!6ISo{u0t@t4?H`;{XEWTBrn#0F
zA3SSRRyD+289+g-i$<0}aQg!-Az=;3J>EPhLygE5ByRL@f41@Vvhi<5=blQFFJ5eV
zMV3_!OAB1aPFo+-{X3tTNe<-y!ZR=>`B?w1tN$~c_`l7a@c#+BaSwoo6AbDlo;#?s
z?K&)$no8hq5o8Vot7;5px7j&m+Wa-J&wU9KISX2P*rTtvrE2?*^}O{N6k3zD3}Wjz
z*tV<MBzY6x0er3IHY;(_pjqk!Ea-GIwojV5x;n;6`@Xh&x3y!n9kVMn>(@CvTgyLI
zRA3HezH5H+)Acq=j}KXK`)LbuA{yTKCbfa%b(v|Nk)Iq+(?I|#gs{7E@PPqpV?Ve<
zYOJ<-Z9WpN%WZsfQ(5J_D}qkKKhIXUoRzC&8J!ebl?By^C7wb)G_5_-3^xZXYk9We
zD>mLiJf3^Hw=%ecy45h$ZB~6iGk#e>)&rJgePTOJyo;rdF&_&QAM<qu%ZUU*b2sQJ
zP<~mh!4@)q&~GzU@HY!Aw>}T#6(ON}u!BmeBcJ;`t-P>@Ae0C&KNBNg=#C)vrS=ti
zwWdSGPnCR&0S`?Aru_FMQ9kqwYf>BLt&UkMz59LY$&6O5^bfI7Ga6lt&&p+6HjG=p
z7&CzI9eh4naws}Kt3R|w4fY$pT}t-$%$2S0pIcTP)FN~zEK!L4I_mqx3XZ(k^d|WM
z$4bOx_XmPMH2H;VMxci=nKnd^rU;V=h^f4GQ1$GA!}hKTuMa`d-tAGtnP2#6l+Iri
zPkVtDByLKv!wNdMZ-<J=-iqZak9s%-QX^pkrZCAAZf_uppZb2x>DLQ<bS^p*?g-|u
zc0YvhGoH+f34#^VMNBHM9?0E{QJUY+4=hQnv9NZEv2zCp^&RHOq`p^3KDw>^d3T`M
z6T00&B&wt>m417v;CBpZ(h^z`*;?NYC=1;Y*Ix*Y%XpeuJ$v2t?ZCM&8RG60*IQ96
zKA+6@j2AGjT9E>6hv9bQCRiOT&PXcblRqEn^4~S@{;8(*aU|I5SQc6M(RfDX=Cbk6
z+p8gLzG>xo&_gL;3qfcw5Lkh!kq$ikQh^)sitt%B**1^uvaf&l#TMr=3sGP&%!TAx
ziErrL_4W%CobkQCSZ0`&jzEp(Y+eZR6$}Zw;5><|>Vq*T;^jx!kgv@_zy{4&7k^32
zpY0%`aEYrEfa=i4Cep`|eF{U3_K8gTu2L^_d_{bWN#)@b(&VDnz#(M|fh4cZlDT)|
ztcS6Kr4$JmCH7NdLY^z#^kTgo<5x-Pa3C~-g_Tt<kM_>-O4N%<)dbRlNd#y5s>9dB
z{PU}mjpRqY&Iy$p3U9Nd?)H-r=fa+hm7MEWKKjuwOrox0aaAs!!IDlU_|BDD1UnC&
zNW)zEodpBS?C9n4@^L&@qxsIWb&?<8)(*AvMQ$CVXzv`EY}JUdo(IIfD)db}B2^y7
zLH@UVeISG+>@apeb0vjan8I#Z(NA7kia*F~FZ&hl!VmEp`AYCNvtNF<1Ige~R`T`c
zB6CZ4?rrn%(-eLv#>Ju758^paZ8Ri4HF9pQE{Jhz_Poub`s##^awntfErr&6@BTu=
z_Z9&z!m9KZsaT4V<k$*fWbPvG*|fg?xtW0^r-5iTDJGxkyC|)pTC*!ce>yx1x5BlW
zT)1lrjEzpKHNxB<n|P@(IC|`YR!_L9V}GL7Kf1qBm@Uyet+1||kHH5KQ4>ostF)xf
z71IwFl*pRpi=JwccKyCXkiI-><O53MTblrXxPPG!<X%=V7QffUYgzw_;}#56yJHn}
z=eVm<QtCjNDEhJX30|nG3$B90$CY@!(6=-Xy1WW$yD?wJ&jW5N6u*C@LG?`s?+&ZB
z<ayF2v4iY~{!(Bq!ED0HA8%*kIs4a`p7&o{9iNj-@usuB%ez;~-aRS83lRkr+~qTA
zOY*4Y{doNfFEp_Qn4f&N9Lj$raW|Em&{wjRO6)^f0k3|0!KFw(?<P_&5vUIPBbA)+
zg+gFx9y_FpU-Xd+a9P9_tuz?hX-^_{u^xbuLZYqs`sFU+eC2KA>+cOh(rRJ9gCF?o
z+CnnK^TjVsFs(oPpLFr|W>Y!7Pl{9<hYt&5pAZB+xX7sLe;3WKl-y!iDSoL4^c+TH
zBS{U5`)He1hJ0F4I$0H-5~5(;Xw|Du@-Kz*Xh#OF84^oX+I<lo39e~t(J7g<6`g+t
z-i<KsR*vOzNJZ$k1wP~s*`<JkY<2$ldW4gfV>~Zgp*wX=X!B5oXEg>(OTk7D<raha
zxuJ+%Y($_03pzohszFBLob@+(gQMk%pZ9>1>wg1lJkM6GtLfwxXDJE@y<p<oq-cNR
zZ)O&d_7Dp>PL;SG<d|{3vLWqiw<mbr_^mQ=n*9cO_5e5|yH8q!O8weH0&!V$u`c$=
zoHt9%*geY2hQs9fk^%MggOx|%$jd7Ygnf+SR$ECYy$QJTK|%i6T8_g_yMrT-p7l+{
zCNAJLeQfv5I<X5yIo6K+W(Ybs4G{>GIkiQ|!8G9JNcf(R3+iI}t?6R4cD9}I#sBe_
z1V=}i>2w9;MA1WxOQOAtiei?;NYR18Mt$Zfw<p|icBc#6oxPHw?n;oxn9+M>V2dj@
z5{Yf^+nwHN&Xx7~E*{5s78nJ(xc6zUQi9TAW_y~Ow{PklE+qXz>IQPL*VmqGS>L6b
zDEN;&Kn?e7FXzM%0*coOauMTdHNGnnZ+t;43?kRl#^IyN)QnQAnZ3K6*SPgQym1J>
z5(J4}zNfw%oKv-SQ-Q`fDY0qQE1Lm5%#5VCoiOS{eswVRUxCY6)+r=<B5T$&P_ONX
z?X?I(u&}wGu0{Y<HTlDTgoHxXu2=;fIPUDFyZ@l}9=L7CF7u!q%e&q^Ke83HKEY=4
z5U`hQxt!;)pweJ7#=k!i2kzsEv!N~`KQgaMZT*6m)r_6~XBOJwqW-NOC_(mX0lZ~#
zw;@riJJ?AiHU$sq5T7e4T&8{?HfBx72wnlky8N2LhV#gr+rw6qARoE~PlYJM{cPxl
z<`SgZ&N~Q0%D9X!l;Qk#Y1zzU*C4O;o03m;IdeC(#D0`W?iCTxe&=>Y3IE;@IZL11
zL*waSeo*|{5&8x+<!CD*4t>J&ym#g2Bc0SR(k$;=>`3nc{64G#0?;9=>HHdDRtWmv
zbg}NDk?{prW3|WP+nX?2%Op3Ug9W{3Dfdi-mFZO<j=AVh=!<D4NN?hrp;kPrL4D!x
zPWT6I0<%VVxlx>@_h)@iKG57{sO<)<LkViMO3f+53qs<ck)VfR!SeKkKB1b#K2~sF
zzu42x4YA2Q^v+{(BO>f(gau<;wg{hATkjLXB_}V3mB^<1Sy;bwsA(F&8R;dY6^e6Q
zYlq}D!i}R}Tg0}#jpzZRC@a`UwO@~#Nh<*95WH4IS-P8JAfm9nbStj5DWO7%cRsL9
z(}>gTi=t+y(aSFT{|rr&&*cP=EFxkkXecL;Vd|#ewkg4cF-mK-IMbySPR+~&T|8T)
z4YYaOq;T64rxnH(2TioGijeE<C0)qXKy}?W-<-pzC-5GOX*Av2W1VZIY?Ek8HtoLA
z3o#29e(7T{!xcBDWGypY9+!2nI+k&xksw1?oupw{>tioE>6`GubM)721Tfo>w-_;Z
zI|NUDi*fITCgnWZ@6hb$IpawaOf=u@A!^#NEV!JqyS>@R$P?aD1E{p0@#HFbtj~*r
z*H@C)O9%82p^?i)J}?me&|i+ZIx@CQxnS`;58BnNB4bO-yVeXE&{6DtKjAv{5!@=G
z9(1DJsK&70BTVZ|$$ohpr?Z}{E98lAV;nfD`B+I}R};O(G@k0N&wZ7=z`o(d#HE6M
z@6)j7AwNIdcOoYhS*C!GEzuro&dq)v5{lZJH%$)7=Yfn}okK4TQENcnQ5)eajw?pH
z8Ce|rK3d<%PZDysEo^c(MlpLP<+XF8(5I4p^x4L*fBbCthF5r{L+LS@H$)##=lW2Z
zf;`o$R-`*7rB@h<0L(5<9<rTx7A<=mFJ7(Hp0<HHuErrdl;hT2EggVuF_+oQpX68P
z5>3yg>b~o8TOFV91>MMH?FhQPu;aD4zNXBdUlcz0%iXqlJe+vay|tmZD8x<A^B{4P
z#Q0rsMcyj_!y>Ws-a3^?-#^HsS4{F-nv8j@W~vXJs7G>UMEPI{2`{%bhZQT;pY9hB
z=V@PyVa|d^iuKb)loWLx!eX;zu2(5Vn3UcSbc);VFp>FJz}L^o=<6lTR(~gcNKWJZ
zszab8WZ$i)9@054FNsTEvAc;{9rj0aAK5MGKUn3fB&zub$#{=Gsi7BH(W!WR@yjhl
ziNx38KyH+ab018*Q;HiD-lPiT8irU3$M1*|Dwnn2hEdeRz2CG{Y5G*^=-``r^nP!+
z4ZR$5+~x=%>Vi;*TTe>7owz}<=yUowbVLiX$F&Hs0(W2xz0uz}gttRy!b^J*+wVnW
z5i~dL=yRaKn9eB*YP9FO4NKP;Io5t--UHIBLH<*$S37mTFz+^au^Y}Qe~i>b>y!CQ
zFeV!g*qi<Aw1SxfN}|&^Pu@o1ikX81rtS<Vl}ytWZ~TY=uOm#Q>2l%(-ffj187j>1
zjgZaWMjR5~w=F)7y-`R7(nhnIxH7)^+`SHH7ugttZin*6XM*Z(WM=Z*D@8Qt5$_Xh
z0=v>-_7%*h-V28Kw|*n!a^A&>g4-CF;=0kIyI=c*$)T~96drcb1{_LT!)NO2&WPRP
zWaZKbww>cFKd$V`MJc-1(T062&DPt(PqEh2pGRoj+;)gK!}jIk6Fc8nn)#+VbQ|DJ
zvoZE^sVHfXkU=*N44>{aNnil3aqr)lpM(f7?dUNC>;=1j2}Zs*6YSg&;lg;EewJkR
z3!Z!|D;9Enhvo>r&=S{ogBU_VVtoj)BY$TF{ISLsb%Op8w9B+8zuVxN5%Wr0B@{b<
z{|om)Txi@{Bl6;u+I!71KXUDMt^DWr?SyA>oZ)oyglH{H<`wzJq!GuCCW4#o9n0@}
z#<%sE3B;+T&=0*DZ9rW_-){12VkDs%X$u!f1?~Y{ekXr0WzkPZBbp-iG|L5YGIao~
z6wp_kM%<jMRtCqnrp)1u-#<@__%m|0_VXO=eG^xeJ~W$1JSzeaq<=IW?}fLwlPtQq
z=<7Y%F02HljSCgduVJP(?%1@Iq5;eV18!x(dX9vtH%_<uO+km82oXS56!hl{a2>df
zFEDP#B;Bcl(@&N3K{&T+UTWdPFs=r^H<Z&>%s!pvalF*vgP#Yr8WL9T?;Cd>-PeM&
z#@H=d>4yFkxf*K{Tz1rAT<k?zyn1(CPCQ8i`DF}tywv?>V%WW~$8ZWtyzkL|`{7b2
zm296AJR?88Vy&7$?7OSmF`_=tUIo{q@14VUB#3$q<4}S?#>#=VTC$Kwjodj9A{F79
zPBL#RSK@)~CVq8cmj$&$-ZxN@Stos7ZRz<@ubV7eZbV8uVNyr$>dfau?6Bqd^Aioq
zMAtHYmT)=6pAJZZ1f2->)?d#!h|df4+AzLOJ~1SBqo5dcvWMP95FO%lMh~<YKKXv-
zB^3sPy*7>Hjz$-76FleCaZs`;vJ-(Cg{^q!#O~@-O1bV+Lii{qupy$Kzc>9Pg<Wj4
zC#f}`FdNVKVzDej{+Zc8Y00=x^N1C5IXO5LM}ySTK`*brmBiV7fnl4_{TN7Tb=|!n
zulqf!Q!1^cDrR9D*m#|7Z8d}#myqBe@|y;rU%B}UG8#zF^r^;4XNtx;F6z*NDr;F8
z&#y4FeY~50x!Vp956ltc5PeXOId->)-nqp_^iO6^t22We9ONdgql|pw&{fVDd;X)#
z7g$3Zdl5VLU|oT?UG%kP-i6gWLFQDSFMKqLu(~3i!%z<HT??>GA%O^!Xa(MwZvaxi
zia_(SHhVF&j(?GzL(|KIDlPVce^96OF=d#0`XqW~&?Y%SdCh!h;<unwUlZx*%HTQI
z!{|ZlRn}A}x$9$_k{~I<xjKgG(Hp>dG;lO#?BPD`EgGwER0-d$3{@pH$WsBGx0dbd
zhw`Y8#rh$^=QkZDF!;&133z%Cz#bm;G%7=9;RtUiC8GFnNl?Wi2mNY(Xhe*AKDL`p
zsx9^s+o3#JQm<T7WF;V5-4u1G#t)^fsl^8;s!-^i0z-+ONn_=}+6zV=O%roWSF-Rx
z<izaJN)0Xt2x)a&SO_Z!svyEFk*~Y^medl2XQl{S!=iaf;92`{_jXQ&w;MCXeUaI&
zvMYSLWD4~|m}(rQXz-4YcW{f?oc$!~2%Bg&y7xS}?>6bKAkeITM(PP->tQc)XCW@%
zu9U}6faE~cKHEwz9EWqVW#`S#YR?D~h$p5My&WyANoyj10I6rP%ahX-OdTU)YW6cf
zO@fKI2^Bo!TXs}Mlw45b@8?QJHyZ|uV0_Yr;Ec)rNIzFd7Er;00xpw&7T=msyqnpI
z|Az};C0;qqZ}~0e_-=*nHZ+{0l^<<QJOa1`h6ZgrJLo1tCVE<4^D-R??mHwlH84`A
z412T4GiT1q1nVp+#hu%j`p_y$o9b`C0Z2x<^eUyjLaa%cQS%<WfoeTrNmdx;Yp(%L
zUO5(HD|=o;CDQ8EV|htZ<Y;xhalug~QhR#B7w3R@9)hPd9})%I!9PfC>s%zITy|3u
zOXgdl>f{&c0jU)#nOIb7g}vRpd${<eRhxoZr0&fDE=3`4h30XbxWSrpIh^=S9g@nI
zun-8Z4_|M8%2qdlSExf@TzDi7@3@$TS4Jyol@^?Y%QwL!4xWP}eWa?Lz4%|`Po*52
zIp<DSa5>CjJ*Fr-PfSk)`-CPNtVZUX!n0CDpeq3++Xj)dWF5S>gjhd$3R#<Y_6z8T
z?u3y#sNu`mg$!49{94v%AbbL^)(_-tgHbEey3g=@Vw2o=56FyaQV)Gl%93jWdLAle
zhOcJ+hf<mbk|<x(vnXA^e(Ld<)x#md@QILpvnIGo1iKpr$uz3YII3m|xSk4G4=||c
zn;=^&tvT?F##71hyglrEt2O`+b=({qn)K#6cRd>Ra>pN)rSDV7WXDO%Xy)k)3>NKJ
z*uEwUVFu#onmdI^Ne(cLhacT65+_%dOPTt;_vMZq_{JY}^lb5~Ue(1LMi(Q<X@>)s
zM0b;ra1#W+^&!p@X_RYc{IPL(61L`M0b$aWkL>VI%RMsFd+y{DSq-#koEC2<+pex@
zya<ah2D~c&I@W!reKT|)ek=5?jI5G^tFk1m9x)C{$jxT9ploPC5>V*|TKzPlk9PCJ
zbD2Y|1A_DSi>IQA0Uj&{8&CzxLjH`X7Ed&7Pj|1M8UWV=^)9vmDAO}$E(Vybbhq?<
z`{HC~xr6+H9tz?UHe^p}X3;Wvhp79O)b%i7jxDl0?brcAGFN3~#V(&_slqd+L~rZI
zf>#jRM50JjN$cnpl*?jqR^eqwyFJRoYl{NypJ`)MX7dA@3quaebYPhya;baV@1vE^
z@X7sLCK4IHh-}6pNslgEdeS#7U3FLRx>HbPFjB4dsJC|ttOD82g>7vNP7PwfP(&Sb
ziyG0rz6CFbzF_(c>RRbA*6rZ4?&CkeIJ^6?k=W})oA<KJDZuYX%h-vR2JS<tzwt)5
zdqACxLzEikjwxU?<?*Cn^9an<yjX>2Q;*SHTdQD7KpN{&)FdpkoP0h#(}3ug%36~?
z-Q^NxCp;ZfL;PVAnfFqVJUxJ3|Lmr+k}B>~==NJ3%6JF$5s50}qx9N_=PuCE(jj<>
zMO@p143YR(;{53QN*4(Rs+wi=InXw7%4@)*KCs`)AfyhBDSbgeq0+z~{Y4p<KC)^n
z9$l=q$e`BRXw7SCzv@3&1|kSG+~cI6U&uX<E;eA!9l{`EKhsOi#~{Kmz=D1^QEANU
zc#OVGq^vT7npa&gAKRP@2GuP-(sRSr#kxO|`)q^GAp<@OPB3sk%?n;ZqLzQy?L!eY
z$2VMTNTjU1ja)AkX}q9mTgp$!Ow)){bAc-%BcWp6H)TZs6u~GyVMq3xJeK!}{j0!$
zjKnFQlw*q=3?A5p^2sgBLPIuH=HN&bWMWy9RYqUBIjYa#ck~<b2SW^jbBs}2?_7Cn
zmMOQdboN6kZ+mw1>4mtOW~H>>vaY{|ET&QVx<NBmY#wAe`<zuqAH)R47aIzx@>=-P
z8A2X>xNUOdR9_s^6ibrkZzjvm5;Leg^#f^!>y-T$)CP{|can;R(|JdelWX4D0&8~+
zOw)qe_lEBM>Bi0bd&p!@vveXle2^phS-lqq3CcImYc8Fz$BCn4lmd3Pm+0fzKR&kP
z8cI0k$<{~pFU=NgmT%4TH%c-1)bvVag|bKcT5`VxaDsbmYL(;RL;j`DWaoxwUl-cg
z7|6%#vYAb3lwYZZ*tsF$D2KLjbqUgNc(!TFp|{(>ganmw#gdSN0Jh>A^CDIf+gIR|
z=VyTGi*|rP4F5_{I<QWTx&3y!w?#~W0y`?l5`P#RCreU}fAcBKb{=bUj~xlS;Slj|
z{^5};%mq?#M2E8l)HDjj*v?;~rd!$RAS7HdVqq`1Tp8o?Bf9>!m_LL2wF~8txu?xF
z=;>fR{xdKMtn)N)EYf(Ez#ovM#`3jb^76_Fe;^`8TnS#~bs|&iMABgxKgSpyMzSIj
zJ`_6D%;=211C7d4m3=CSdY&0%w5HbWeSsc7(xdD|M&zfO436?#*`DZ|Az_Q?z{gl;
zd;-fDxxKn3Rk1?zQkok4Yk-vvMvqXIQhbur|E>|@Db2Mf3cXl1U|ZAG$o7wz6Ge@m
zjY!sR3%q(vmm4f|dO*%sWxh1ni3%Eyg@UJqb}f|M%Wbq#k2sm9$ESxcSjHOH5ii<v
z;c+Dfuel^uj0z^Mk(0-i`1GZwXeoH)gZavLs9T(lr`B#N?OwR83pCEtCiVXCPd&KZ
z6|af!{NV!VsNYhdHpFE5VC$nBduj0p0-<b(^IuxQPb=oNDwO45?bU=ph4cdvw&Jm}
zMjVopd>mGt7yZlMC3JpLaxe#dWl(LdLGdz1GaG<oER({+&3-ETJixb)VHaUAo<&LW
zLLM@4*2k?8Sc$G@^tp4^1nMcY_{|ZjlS;;7-^5l@WB9!$K^qXprrlkSIGX2DDzqr^
z2oU30!OA5`lWVg3jW3K*Ka}U{L}F)vuBWbaST{;JR!LmX=UMWyL3{3+JlQ!;(Ls(J
z3p2%HX6=6ZR$7Ydgz-VESAG=3-fMDIt+MX2fXe}C(0<s<cpbR=jw$7RgQo91?e)!v
zs@*>Pc=j56HS-5>Y<GPdvyoPNwBCj=^G}2wanx_)sq(4^5LcLH#u3*p?X#}OlpWQH
z(njT3cAKAY+2bwZ=#C@FS$hU)6FX}Oo<+j`m?vY{M;Uz%I7adMcunLt$1F-RdD_Ub
zYWTJ1M^EdatozOCeoZ)DlQiDW=ow%xl7x9OG_=qb2u72f!dG6sv}5q+`}E84l%CQ7
zCYX5?P*^1OoS`aJBDC`EwtR)$(e-j-O6s(}E+ib*4=6Y}oS{?Y7(CSbk+_jNfoKC3
zD#b4~FjXss4TizfK6xo+X_I2$>px?78v`7k)wz+vhTl)K8`dQ7N;eCPf)6BxqjK3q
z@BCmKMGeWLsG}yaqOssIvOvo1PPnbP=mLev^;1QlFhN2cn!37xNjgh)2I0)~F6pud
z=*eUnWuHm20akpMmv}$}rNV6r=9?I#`g@AlH-V_zvJ(|fNFQL}YiGMwCf8FBrKeH+
z&Ca~TIbIH^Q6+^lVb)8B72WZ6QA<<wIfq}w7%@+PL*`pSZll$r(0Wuonbc$&C5PDH
zgk?%ud~tx3G8+rPUL1g3?Z@fQ-56`(h3=$5NHrMz9O&VUY3{=e3WZ;9fGBuB>Jf4z
zK5~lx_z{V_O2gB3!JK4n3?Rx})_HHtKzMoaAU|D5r65&EWMD4eLk?7gsf;%Y`JmY)
z_nlgs8B{&qDkJvsHKrtaG^<kg+n!)K6M?ZK`WzxD_J{H$urzEMmBQPb;`k6YNgru@
zfjiz?iBFO~P?fm*>5g>ys20|ZlI1{Zia@??3{w4KRKyFt*5kJx61`+=fR8Q8{A-@E
zsdj5XmKtl;50uFntW*8loiS;Q)FkpcUT}<S<!&D8*k@(t`KOP`knd&L4eHE6d=Mmm
z=-|ycIJD@B78T|YW;03~P-+B<`Zw?^PIAh-=pt9u%I^)>mH*;3LI6%{SC-=5phKm@
zl)U(2n(oP~R@sZ&!vb95T3h`)%&52eWunb|PN~q(B8o~d)u!+j+N5BL*E2~*s;FbK
zi;aOlg|GM_Ci?i{Ws_ZYTDs23?RWy4S>!L>eLZ#_v83pracLS8_`l`7M2XB6I9ds<
zgw)T8(CN%xA8SQv65sB`AL2iJVTqO5v^^bc=?wnxM)Y3eT1_4!pUy{1ENxN@zpOdJ
zIRv{DlY<L1e_m(#I8HZEYqKF(Uq+rTBHYE|--aH#I4iurWvc5}VNt(b7|%XeII~dk
zz?w%>!!tY_BXGpTz`9JPw0O8TO+gB)Fu%PuH^uW^khG#g(onlR7$|-pRSCw=5+UZ*
zxYiUDKLX)vs}5ST$oP5F)qK#4V>w_9_wWSlDdzMh<*}+5<$DVl;5a*PE5-iduWWz8
zD~plRPzGuT(dk_Vl7g-HwY3Hq6|13u;e^QuE8mc0-HT3PCdzEIQ5AyjcemVA_dgkL
zAZUoNCdzpFv@JqI#VS>I4knalr!kM6DF5#<JoDJ(zd{mN*8icS1*2m8-%jkuMMP1d
z9y)}MCf;zUp+&mRiLeCSZGREC3Z+|a|0}gYl^S*e;u%c<1^7XW#_sEVCABWP=3{A`
zeLIfjr6coXSqTZ?L31<}yN{=bF03br%f`(0_ZRSgNl8~dpbQ=Kw6rt}TieX)Q5MAg
zi7X`!UM?&jN-Kx`2o-H0uZzprj-NfVqfTcX)4wFbE76;ei^$_(<$ir8gN5+)@|tqV
zghs&rNAIJ4XK4iZIqs#GB^veT=fB@4)3X3N{RBfr4*T|jNarYrYkdZdLO_9Sm@9RP
zHT6@P%DPef9SxbD^jjq<_y9ym2m(&7<uW+9X#Mu(#)tmxIkvW}{6qit<qpKxPMLbE
z2TP_-hKq)Zb(-B&`DCo??8wOQYi<t9rrfAk_arjg;9E^L>~sWK$WLKXGJ#aVR6(cb
z_5<M4voST9F*@wep5Tk;yWyx2_K7bOPC&2m_KAD`iRK9bk8=yxp+-}<a##{Abn^WN
z{-#RlHMSrREav)8|1vMa^JL&aub5M;s8HYy2<sMbX`c!UKdfhCVItvM=qdiA{A$Y}
zocXU(=Iu{H?|&|4{V&rpn(V)VFvR~dr;Ps_54d9g4ISJa4(ZFI(;w0w|4h`*Nt)nt
zB<tl)Q}-{0QZ(9+*`t4tqW*(dV6!4fD-b8mVmVFe{2HR(ztbHS`S~r3(maKNj**cH
z=qSJLQW)y$T>AMhBdU*K+7SWWW~H#booM6!z6dY0JL8L!a$=lDa_c?j<I6JJ+SjgA
zOSFz3BH?FB*h5N)S1ok#_d;6abu6Q?ci%cT_7j>o6@Gs`@WPP_@NdwSsB|wBR)`rg
zXAWl-b9+47h<wpcR??^+AJoS%uiU@B+=n=6*%~tH+N+9mob$%0HMuzx!qK{Jk!326
z)5!jpE6A|$F{pnuZ>6Pm_4QjG#4VNVA8}QvkRDnZJL;LnGqpRdh8gO_bZ<jjbHoqX
zc$nZO35uI|J`5f1Y~zebUFVL^1_M_@t5)M~+B-9xuVnab+eMYmJWYghwNfVEz@K^m
ztS~JcU2rew+q}b}dTiUT_3avT?VeRHRAMP+jn$0$PncgjuLeTRaHM{5q834&oSN#6
z#$!+6bvb?m1Lx<m`FtrpwK)n$KSK_Wze(%}N1r^(A}y{SPk-CW@*Q^agwrT=IFUC<
ziyNvdT05vjB}QC+r{~yb<)rPE^D574^52Mv78_)g2KxZlM$@_f@iX7PdH@IS372Oj
zwW05`$(qjqAoPh<m-dqZQJV7l_tr+hFiVXD85$O~knoG<NuC~03R8w(`G!}^o}AC<
zjOV4Mkhv3z4V>1hoH8Egt(eLZjwJuyX*)@-_3i1TN(cyDS}My!D5){`Izb-&7BXDi
z10+k<)7a~=B*W&!8!ewbly0tThOYu*8x2ur(yJNE#tbjJhM6%K7-lN=C`<Qo;>XjM
z<|=>7sd0#p*mf_!xGJ_nyc$;%?c<zONQ%(FkL&d6S<=KRRN^S+r23Bnvg*4mDpdE4
zImW;9Zc~k(arqx&(_rL(NrL}>7D#Y1`^eUP*|iO>;1m|xHV-Wh-Rkem7sUhd1@Vtp
z=-!_|3ZAb2!ld-9vXV8@e!sW)^9sP?C#>JtUt~|P|2xanlAx9WPy|}6!$IkEsWG@|
z4MT9&V(u7_>T{ucIiw4HTkOBYS%_QnHV7at5E2wKn?WcG&N??eOsH6VR{N1(wcg`I
zc#a0^;C9D$`;5$gqI?k`cN8m-f1z}xiVX|*J^j=+ovv;P3Mh88o_VYH0#>4M{%0eT
zEhNh>{~$N%w!@qv1R1BZIR+^Ab=ql2wuM={yQ`wz-8S>K<8CVM7J;Pj9cIm4%cRoF
zhVrbw>`xt!Kj3+m*pvC2%zbNC{v!9Zl}L{K77f8+hY6L>`%<9SpJiUmi$&QBQ-&ko
zvEJKPZ-23>SR=KEAtpA0A5FHei0joM)BZl;-hv=w#@mh~v=(MPnGF{&+Jo3zZy0-n
z@SL)@f%O@ZA`6o0OlxDId8y_JzHOhgs>k+;$@ekr=?%~5oVgI3w)TI-Jp<E9&d_?1
zIRR`ab?%t{4&ug-@(J)OTkE<^UpCtwRrYCe5^I~i(&aw!7%K)-X~5e}Z7EIg<X>sc
zoH#MDtQyapL!U05Z1sKcT};rsjmIR@&lJ0EWyqLprvD%=ztoQvY<BNTCLqw999E8$
zW+L-N5fh~8G6@kl@Dp#?-pDhNs^r`};eEn_sn{={WMLiU-YEJpqg3%qv~SdHI>(f?
z6qWaVkt6M^^WxI+)ae7J$`|`W8H%6dk38UYjXI~7oa0$Ws7T&ijmyEu3j1FPGqqSQ
zZCM~+y3s`_+l1CGE~xISMfO&triaUxX)kU?G*~uxGac$fT<pPLkvI<%Y1ec_b)fdV
z?u`VrCELGr30A~D90}@$Nj%mduU0TUS*L}Mv*jdG_zjYFU}R_V1uO=_+R4Eeoo-eR
zVW>++w6}vRXuy8Qv9=}A-oZ?)hIfsa_JQPiWg3o#H#(q#oi<$s`u^}zD`qbXZ!3jY
zpfj$(@3N~=DWr<{T-I-Y64Zh}icch@mc6=zo|Ys&ht<}ATwal+-Z?=jZSF_}d7}>5
z$-hiZL}c>SLjB$MOSjL7j86sLj+axa!#hV;!9$dS){L>m3_8b~CEKRa>&A!2i6cjk
zy7!TWI@o&47-G4i8fzYems|MpbMnr{BUDKzGDdspuk9*ejuv}N=<~@#D|8=<0jke|
zXzoSu^{}pqCkm0QurQ1Iz(f*l<;6go19cAX&E%IT;wQcXWsh%EOI5ao)g`f$U(klF
zpXn}stE8ce|5`wE24Sdc7|$O)G?VYObLHS^%&xX1ZgHQjlN3XnMs4edO^pL3P7#f7
zy%|(n*u_wpc90lSaFi%lEjH_smA!eSz-a3Yd#WtK2B~2OVR@raqlmZg{t-@u_u>sw
zz#;p27x$skKRKl8EPr>N+nPYZIzm#9uk1x3p`FzG(R7eQp4lNAa&S2V1J9BC{oJof
zII2t)?JI0dL(jS{bVozCBtqHk`R8iSF7p#LD&Bxdhpx0FiDhsNHR(_ddiAVUP3*-p
zcmDe3Kc(%0ScW0R$nYN+J+^g=cf4Uza7p&7kM|`ZCYBLd*5FqiXF#FpZ&SObDq%Ue
zPc4jZ^ZUQ;jU)@(|7gx9cPlGnb=|o;=*G>xN?D57YEXF(ecqO{mVUVr&5e0}+cXm3
zkG}rp338!HDh|*#%w85V>6nWU3wTk;%ySuwYb_zrTu?^Gk1WYi$1Q$dhx)Bmc2lek
zTxD)`po{CY<Su9nEX5h}apCJn<WXvfZ}DIj2?=QR<!=5J$2TofBi%eh<Lrr!1-L$D
zx^AtAdlKhV#_Y<i;|(!t(gHXexg^n6osixOe<C;7Xb&45Mg$GGMy|m8@vX-yV^;8W
z3hjcSReO0Q9S~9whnXFwjNn!hMG;u}J`UnIe!QzT<vx05Oi?CZ<QAWvQL1E@-9GKB
zCAdY2TQXH`{tV~8u-3bYWqtI1L4Jim50-GqM`H{TK$1Ndn)9=u?pxTeHAKCdUpcLS
z@iq=OV}v~|$eMzPgLi*e(&9<hT>SAFU6XTJ5%F%{;?xhHtPBOBU2c@0&sOa72C+Sb
zhTaFLIDxs@)coZuyfr`aDB$!$XU!Xxj~&Q3h0lS8{A?y2^O1Q3w66jtS+u#u*;%YX
zkqZkL&w~<uMMOzv|KvQqsevnLQDHw`169$2p|4*mdJKAg_S&O=a?2!=M3B#DWve?^
zY3iSCKFMI=C9BKfXuY?1VVQimX*P83FSLHGoiWjCCmq0Y(r61I3;aFO*qGSdn2kD*
z`CV!&-bUEFYu|4nsRrWkIFtcF5RebSq=j$vP2i#RJ7P9{&J?S9Zc6y&GOuaApJy;R
zTBa$20#J|7^|;@Iy|?Y)lqIN@fByD)aR#%2EBT?vF0=DRj>?=`9n=5kxKw4i=$DtX
zJse4mDf7AL3>J*6cD0jxGg_oFO6G7tArB7&pUj4uq}uSq6fHQ+#?l<MI%-G$uk9=Z
z@v0Pld;=4Io(RAMg<U>vdktg}x1<P?%FP?~ZHnyAZy|HgXx`BaV`Ppp=<93&!fN0s
z<{AXu+a<x)oTS3(m{d*eA=im`edrXHsN_!#Lw)O+q)oouij5MJsr_v>ToE_ui3GN6
zkP^8_Dyss$iB5RsQZ%`oX|(k2I$pGrCl)C_4<k}nkZqZ3CTp^#LSF!<ka;;k<G9v5
zgtO6R+~+{}1!_M{oZ-L{+&oAQD>2A9p|X%zY)^Ze-j{Pi#Qs6O4Mo>>k$!XI&B9Ba
zPpnnn6tDGvBl<bOG_#LWO}YP}C(0q8w-1H$F`?!^csy2lJK}os%`s{-W}5Oq8+Jo6
zZu7;CKwzjY^2pb5qE#cO<x@?(>3v`*%As;$L0+#h3Jq`bh1k>I8^A1^$PoVUwmZx`
zOY8*pzWWEPs0?J_ilF4CAk7`hAjNA$bLhTg`yhODJ=-A!ofL2q4)N7_BWCT^*B4O=
zESR~y!~&+-yVZOjL~&Zm8Dd>*gD<DU!j!qnNq2av-plE>KtY4^mK-`-|4Ig2YRVq^
z1wHNVS53{U%P@v7d`CBju3?Rga@VbmQq9GlyOn-zgg;#%Lt%7@FK()prR8CrW?IYK
zoo2T}!w_r~NFLkH#MD?gpg;ISn5K2?1=SAmL-jWt=gX?p?aXB-H@gk!I;v9tIJBM;
zL86ujZwN6`DZDYmYjo->-rZq<+#Vn%x|svp9Z?rn7LdU^&Q7y(*+nH-KYk9Aghf3u
z7Glo^>?@i%OmQA-m{Ba0cw|YQ6?`JwF7ai!p1J`3Pr5@V8Wpz@HlrM}(9ZU#ADxi2
z6f5WUV#>0_ibWEeE$&2A4jPy`-P+&8{iGPX)m}GCHk0Wp$I!L4qpk@x0fMc82CYil
zBxboLVOJxpv(o-6T+)_2@~3r+Z*6)?k*Z;`FQdC}91?0O37@Po;UKvJ5XK9U+3rmv
z^aYZYlfZLOmrrG*OFmh#1#0hY0me0*xa#8G)PyRb4b<#Lc9MzOfNVSyE=y7oKE|vA
zg^E7_dps|AWoRv);4T}fF^w%jMqYS?B#pg2AAdhmNghy;3x&Uz1byo^i(zw0Mr+MX
z>!87bC7~%P=?_6#l|8C_H4iep`>OUha7?k0`cG1cU3?&tndB4&4wkx3I$BJ4d4o97
zDlO}puMg<9MY^7u<bPX44%fGHUA##|nru2f{-!Kanqq1F%=B)Vrbb;ni%(CVZDl~X
z=*Q;#(dG@&Tnt@`DTq`|KHl-|j9RHb&PIOrD^V}L#Ekwqpi_9{ef~z+8BIHTOmFB7
z^eFtpz3}B08oZV!ECrKK{or$UMa3U(;?K2(8wcYnY~K4lxN_?0_N%rvzc&tI^-UfQ
zuW1h?x+bOUE=%cgm;YE(R66$mrX}Dud0Z89So;l&aq02HD>c$Rynu7?P~h@I-xbEH
z&yC^YH&}&k>>4_|$DooNGD)5!9DganRl;2%&c_{$kC$nNQ8`e)(LY%rmU59UxAsg<
zJLO}WAKg}2o_jp@daGams~lz-tG3t<;=t0y#+)6NR%l+%VVtt(`FR7mfX_AIaS1E{
z<=C1gPXn)Mz+w=Clw<nm)(y2vn<GqnBUEvem5iQF%yRl*k#tiF^1_*@LhvO_)MBit
zGpN?BygkQoI%`?}=!(C4gIMZu;fs2kG276B9ob!FQn7{%`g<*^0OI@%WU&*C!92g$
zQM1mI!4IMwnP-R!#NLY)+2)CW7TlpKrY^(-X&+;hf{G8nEsjS#jFP7*`S=S3w#)q_
z6C|9oim2ig7VRPh-CbZgnWu<AFz4cm*gFkqn`Vbn=*EWS2dbI`98I_%nCix=h~t3K
zc7iC(!Hhblm`LV~u#F{i!63pcP$+dfruFc7OmX&$2Bn}bK>PBtp2JK7GIk~+EhBcL
zHhBx>v~Sr`#e2as129feQ#X3q+F*D_m-Aht=lPN!J(+d_h!V+m9UVa|>WGj?;-rGx
zr}$0X*AY4A0Rgx0pEj-UMm2DAveW7M_k#ZBou6y-bvY^-vw;+#S4R)a-12V3&7sfn
zJMqf?w3~2R+N}Ga^G@*G5tYn++!H(H#>OQf$`~=t9(u_0lgfk*CEFEh=5k)&(tJcc
zx8mt`sUtrkjGlhlb)K;l{Hz40_yHgjfB)b{>%MScSFKPJm2c+@M2vAsUTQlD5%m=y
zrnpjxWzPfqf4Bf8PCp;KurX3O6sySoNu#D#FR_%fVUMTi=>i|t0zKF4M$9K*;H0qG
z7SJZh(lb3f?2?e#zXM9|vkqmW;U&V`O^;cb2eer5jJsKx_ZOW6UZU9>ZuABFcF8w{
zc=V476w|%n3gz>!&*=3Ax)(6F?R81P|8e_qM2y~u`yXo*)~oA|JEp&QN-)p){~g`Y
zOXbI@>Wsbs9!CB}8QNZ=H?oAW29qy#_yzn4qPm2MT@(d!p8U77cpbyQ)o7fTFh!HX
zmKS?ks{G%c@DtmEv@)sVoZ_W$b37Lbo@=<0NGDnJR-{dq<0^wTceskGiARKeu;Lap
z1X_W;lX{*Tv!d6HI6qTdgIggT2!{bE81ewL)A5`v?JyOp<i8d)icD<rq_NQ#YI;+m
zzD|)ZXMD7G07?clBF;iJz4`>zi7hUspYL`wimX!S$-3^$7LMO>PAb33ehJC;`#B5H
z_7LHIcfsoM__1+<*Q)#bS7{X!>Gyk4*H2oek#3EuRtj<Lw%mJl<+lv~Y{nBNcsii9
z_n$7X4pXwL-4@Lsm*C!BITDf>roD}oz$zz9558V^Kd~+FYnwUHW|u&;$X@2iF?^A$
zZ@Q{vONZm}^UY9VWQT)H=PrEI01TObNL(c6E%3!M)4FUBF-FUUm0~k^<Kzm#P;VQz
z1qK@ePE^}E5lpuz<ISn{44g|bJdH6Tie-+{h`;l@w*=*!v|J)^WpD?j`IVA3<^mp0
z3bU~_Ue3*ahfp7{D@)kKOI*#*9(M4bx)9}e&IBe@6oIxsbrszfWtv>z_mTX7D@{;;
zJ0S15;!h=TQ(0CYAYsaBw<W$KDUnLg6t93s8&%BhSA5W(`B+oIs8VY@$mpa5v~8vy
z@dz$a_7x7&S!Ta2zJ}j?2V@^QY&?<Mx2xx^sWgKSwKCb7<QtD<mR^2jN;swsdARf6
ziN_I{r+kJJ(-jNfLyM`7u8A%<5ogxtxc2<ki?Gg*E3c-#rRH-cuYMP@?&C)LY5=1b
z6ZZHf6O7;y>X$|Cx|7;h<e4v28}QPFjyW-Ce?saKrL<cwxZLxH3{+qUWY2u!GrHCP
zSkh?7dbzPr??BO`125Xf<jFQYo4mLuJ{px4Cx+ob-nwDjA(UvQ=C<lL=*FObN?4ye
zOJz{-K+1Rt626rRm7T}p;>#mi)BwBDD1u0-p9!mFyJX%ke+eXB$>mQgMQUoz?u9a$
z@KsqwEXH+I#{|9K7r_dl;H0`}9I!Bn0wE8sp}<9!C_W^+Q=#hg6e$?ak=E?NMYFL`
z@FKznUQorfg^@oN7eyq5DB#L;YIZoGByg&h`^vwPVfLg{8GWJGo-cZF1~>J{xGp6h
zIAY$_LKKanSJ~k$EkjR)n3a)vDQcX0pGU4Mh*9r2Qx*yPJBgtK$`g17EyXV7b@Kze
zu=!q7Ig9Vgd$pBwMcgOsyP_5x(u-}Zz2@h1tdyOePEfy1{FY?jQ21zVrP0%P(lVHT
zz8#9)zPLO1DN+$qTwN_4T5HP)We}&P?Y3ZcGz7Nm+YB~ll@TKCoJOeodVcCYR+JS<
zambNA1~FZCuFAY!(dK;nWVN2Nh(p#v8n|#`Gz;Ct7e@Qyh}uXacGLmB8W^?0CclLO
z+2r$0_8zojg7Rwid_?~AMS-$1Rp#`R#}}Z0N?7nWg$KLi%~W=N;C@QZRC<IV?&_wh
znUY#6(tRA%^SB(;IaRpI0#tn~G)MAQlc}f+T3Tf)IytO4s(CfTo2TDCr6vwmW1!;F
z-&k=vcd=kn&*I8gs7~#r&K{siGVVDnV~Sp6h}xumi7%%u=2RJl)=0Vgl$|>t%!;j!
z#>**QjT8MvMO{Kf=HuK`!;mXP85!?4J#7T)(=N|IFzo-+*jYx!(fnyUXo9=D1b26L
zf($OfAz08PxVt;SZE%7FcXxMpcXu1!d6MV9yB~JXdFRZBIXzw7UENi8RsZBFsS%MB
z9r=@Vo|D=OdA`{D5mLU70U8y#-0B!9#M0Jb7@1hq2aFqB?~~1Ylp(P^oiIPD_bhIj
zWa5#?rlCn!a7|M8P8=g9B!>!pgw^v;&?T&i32OHA5i1nGxsg-jW&KB$9pY5|M9w1N
z9cXsdm+?T+`XMFINUU)rS>j(UA$KpXfq$f15&pjjSA9H#nX>;h1n?g%YX80Yzrj~Q
z|A(=X=MYBR6Y+H)xikku596QmA(xw!2<QS{_eq%hUgjZLCr*mr%@mo+<w48Q><?*9
zuy3b;DJ5-@*iGxd*1s`tpOLX{8-bAt{>+@6#)Pw#kq(smsL}=x)bJA=;{4-U`a_{}
zG*}Q%fcFkoPm}rk)+E7C<H2_<Um(oqJ5TA*S|-$MRPlm@TQX--k)MR?qKuo<RsX4t
zi`m2<OI@37_`2=4!C&yh*pe<kTKll<O0sRuGC{Ac^3k^hL|+(?RKtf&2n6>Skzxmy
zS6artpiUqKB4A7LG;8G7FB_{WZog>50#2ccSB3$&2?273?lhMy?`DN{kVVeCHMWTA
zq)AYB^i~*G6Ld|~yi6g3t(Q93WL}=sd)OIoJj<c>PdNkNE!l(*I_sfoz(i#s_Usue
zG;OU&e3i>o_unsc#z|wjXQe#iSu{~+qy39hN}Fp8oARkF_)^nc$VT^jzqz$(*y%zI
zmO^Zo63)QR;xa@Z$1fCE=0(tnbWJ%M+1UY+xic|;9wD>cwhLva$*x<tRz_#lA*`xN
zurAEJ$!feAth7Oe*A!LcaMl^USjOO0z<Ab^XDX0l`TkBzpkX=3WvBkP0Uk%>!17+a
zhq@rve5O?C;e|F-e>Zc(gJiTP{Vj}Z_*RgS7!Eg(fTB=uBWjP{1sY^AMoLQ4^nizB
z=7qV+9MfuqGB!ikRd}3WyCPTF4{RtvaqrhxSzncR)#T3K6werfxAIJ%_@yRSj}j53
z|G=a4o#iH6RIIT7qVCS*lo8-nld&R&a~%P#FBEA^6B7~tvhf*seO&zV;4xB?JD>iy
zmgeAEislxJ?iP-k-5kPh=Jm}i#I8Tp%LxfeW(Bb^q!_bUa>!^ox_y+7Qcvt>mlRD)
zCd7j%f)g~?I93{udHewpL8<NxKH9~#b~+Y(C+#@Xg-ykx!chSksnpe8poI0ip`8kC
zaYfbQ7B4s*gG?3mgeC6T97%#p9ZlnXBo4{m<>_EE$x97aToR#~;?nHdemG6d7>XYv
zZpjZ=ND}%u{krj;L{5`7B*N-_By51A_~dUunxH6FM%F6Xt;T5We|p@ODO;2@`o@ap
z$H&V+w3(XTG6I{@?wxCotgmQI_X`4>u!b<E#&ixIPhzldUL}pfcG9}B>$X*cByLkJ
zxD@h-cnCBbUsV9c6veUhKPO!FT{)s)KfKd1AsVzMpj}^5TqMRTfMd+!jyoUwC}AyC
zctvA`F6@rQnIt^`sv}^|1D=ei;NRsW_pZIOiVU0jT(FRf#wXstYLrW*-(~awVF7Pd
z>z9S_AxjXZTGkKpk4QL1f+?-wcO#t*G@MWBo9k<sT|Im6+v)84e@EBP?q2%R80cef
zy#&%_wFvgsPz6rM@impGm^{3A$>U)(xXY)E>+5$`(FJ<Unce&Nopu{hkA(wRSAw}G
zu8Yx+2+UP^5A7e-W@OPurl~O8rk$qV7K7Zj5l8&5Xp3r%KBP#1jQH}com59gfqPlg
zH=Y)Zyt=&)8<2E`fi$BCG|jV=zaUSZK75%euphRqYRBs!xWao8Nl85)>uawF92s|W
zc=iY?cPx5(`kDZb!C4LoFBch#KdCH0WM_@jdu!=u;H>;GA2_222nQ0|;-~!`CBgoC
zQWwxy>Xg!6;kWVua11ZH$b8J`@wlv`Xm<m#78>)<eU*n^RZ|k~5QebL{OjM@zDM;6
z|H^2EOvMN(MAlEPB%9t?@o06+&n!lSC?z3DG#$rdDr;J}t`kk<5-zeIb;N?={*ma*
zp&NLl|M-i7DXy*m=ym|=|L!sU-(<UQm_g=Z;Ub}TBeXo|ufSIr_S-o4L=teT&?O?v
zRTLK+^+n*}2KjC`mg?9UF~Fm#l9r01`fU566des54Z(w*yk`f|^(AxxYW9Nmkjj`W
zoXS`*ez7ej1=)j5T7L%(fr@QxN;M4)HWn5io8{+e;3N$e7S=(NBH8KHb~{!>S^svX
zftkz<!OPYb&2A<ecWJE!{ey%F)w}_AK9%mCw}zI09{hsP$>oyf$A>w{KoVgUQ+55E
zrnQaged*+QHB!wzllsJlE0SMY`mQZe8h?hWkjvu+7&z5}Phf|<)A?HDHcx;U+1^-&
zytp`2NeOy|%Hz!PRsgL0=bJaW<aWDmy2N5m{5sgQwBfIefyH+9vWlTs&(mt<-laWh
zIJlVCTzXtBn=7~^sn!Cz_FDTxyJ%k|qvm7(Mh&pG5!IWcf7df#s<HgXZ|Nd122OiU
zW-)>+;j4oeMcr_F292AKPJb+o>1=Q1>oUVoBZruw;vioCxgXg+BtB!PkuNuz$MWW7
z8CTp2GuF)5J=8XtaX@!XO`t|n{~zM!<!OUc2B(6_Py+1?*aV9li~^4;l3z)_v4<tO
z$6@g@4GE>Hd2ry^Fds<$-V+-ZLVVY!QT5YN96lX{dyr?rc9n5b|8L+oKxA~|`J;c!
z=#Tb<f4!k%gtTA)?1qHqmPm?dTB0X}ggR3Sa2rct?%qWUf94lm{IR-*V9Gx2_pzec
zLp%D*Pvj$-in9v~yYWy*beumqj{ZSzF0AabRu1OPxwK9F8;il*OuFd`Ba~K;{Wzx+
zECl06mvR0X(+rUB1hoDaxG8J&0`ng}tjUpCU2^K*#<(RB|6dUU|JM}b@1DjH4n&j>
zUOuO=39Ii6ymAj0;2|OBsV}77+!tU`J`iU4gN~LGY`s%`m*0PAne-@EQYK#AhuBRx
zU+?O0=C_w0Z&h;s#t!w?h`@hM9@UG-8`N<dOY(sJPXoraCVUS^d@q>1N5oy&cr}OF
zzIJE~el!^)0?}QO$G8vb1pf1-q^}8{$r1jD`gMU<qKELwd?k(N3DvYO_7to8<%Sz|
zph^gz@|+;`g?cM4D8$cow;P^TbaGn!r0Zx$hq-TM(ebV=fotOek%9;99YlXxaQey(
zhMq5uP0`4>J4f4X8)d8@Ub$OMhh;py7Ve#^W16JF$WA1C-}z=MgSv&qET3-<kV^dB
zj@??wEeGmssD5*@1;xW7`s3T`C$Dh%0|V?ht?ZcgAJlVHffKW26Ys=&gq`qpUKPWh
zr4b3`oViq8dOAIH2{{pMmq{C!od9e4<gaM&Uq_^@)(w;%H=P{42Z3430Y6vjJ!4nq
z(`cP}Eh^IFdI_wRGYA_=>HxMslz+~ty6*PGZWSh5;1`9WhS4Hi{-HAxTCCgwgPZi&
zBKhVrA-TUI@w5|PnROr@96DPR<JSk@UFBIytbe0J6ZfyS%66pWtto~3Q5C&jmvl0(
z%W508y?>2+s~YNE)N`TfWcad!NuH$8cWm2hOWR$1*^TAr3Nn3BW6xKB@k5}W#HSBw
zuL!+@67LZfN>lEF=7`A02!%mz6gInD*ACuW#<aETx0OQ`WX!hzMju#hj7!i=dx#dN
zsU-RfB{vAOs3Lz)KCwtPabV=#sr~F99*|2Qe!ZZj>idfIJztrSiWagy&uC03kfFtA
zC!^lbvXwzF5R|@~qQa!l!Z@}I#Yu%uYO)1VH0GU%T`9%WoD?Fzzs_v9d2s-B&%`lO
z%RYmI<gaj`5*t7&i(R=3wy%c9iXm8tzq2iFw&X)!0?SaF;7^-$o9=}SKCeLJTB?#?
zMhxYB;6*u%GQuadWKXvEYKo%tg&mK6ZRRfgjc1?^b$I?HW?a%)!@;0Sn8T*}rL1Hc
z^Zn=O+2iY%b>w{s%}XrO;9A;Gz0Qi%PQI=e-9wB}4c!hmB}w`!8wr<J<FR0RqZnOZ
z^B&+nce?m_(%4u5R!5a6(Lf+_M<srWVl^Rnr{zzNa}nBhqV%;$9{;>^mA_s%K!CsQ
zVdoc$zu|WZov?zT*Qx2})-_xo@47EHGi(()5}9q79mJl;qz$JNSHiE<%TbkAzT`J+
ziNVriIL}ZboJx6LX_HWw_yO1OcZI~Q`7w=`EkPhPJhT@xlsI;+L0YIH(WSQq#dB2R
z4pX|^wmClS%e|tbbK!*^?n}1hD9V2POAkYkaBX5H3Eza8!mYFU_WHcoo;eh>#ld-M
z=pruPWaJ;YxvVOLzIu-SJ2HhpMJ)eE|3XauM}X1)HeCK+62#*GNmB6x*V;XHqcO?&
zFLtzlvH=;r$}){)J#BZsIX~ES1kCYmuZz*;mpKEYM(UP=2a~}Nv{u5&roJ|%$1@wm
z*A|pF3kEN49*I5vGR*q>n8si4*YBdc6#wX`zH6UD%tqxA9K!hJ!lJi`Im-tcOhJ{O
zrD)D-Fc4#BUPG^vrRwgLWwE_Y;0gID(?$)>#TBt3;A#7vlL1;`nGZL<JoL}VX`K#B
z#tm(VjcugmgW)~ZXJ!oLNY7qqmsg6%l^oBUpx|Yp)SpC19|UpguNOT;QAN0<jlSX6
zvr9T?hLS2rnIrt^&FLI}lu2oODtt?zlB(bH{T+;6Rx`<1RW4WGj|w&=ty4w5Fh4IV
zsLdmubYZwMH_A4KwQBz^e>_Z1v4hrXnM@dvNZ5*k?nD#9#ZUGk*JjZ}>g55!-CP*n
zm+3u#rvlPl>*$Uxft35WfK@qs?z^!%TNPnTQATNH=HySKiM0z#-{T%1NqM;Rt722_
z6a#;hA+D!N&0@~Xe=2y=yNwCKXqD;yI`x4AL-OBIWUCjqU$o1}f^I(p&7{)xif9Qg
z-!i~}-E-%5>u9ov<57322lULaH7ykk`CFL|*dx>ypp_R5LUG>C#2}j|%D<z&p74kC
z&sq-Q<nir!#?DUVr~P?{rILLy`sF4%LvnQLLPmvF{H#FZq|~>ddBNMJ`>Bf~q7%J*
zTlI4AW6yrD)YaUq%40MIb2mbiXZ&B0FI61M{63&|5xGB3txreN8<IV1Wuh`G1Km0a
zEiXOjYS;)J<OhWGg{86|=!b!X<PTuI^_gb7fTP<6M<ieZBPkB$)B=aGCtdVoS5Sxx
za~u6+h52Ddar>+H|4a$>rT%F<mmA=fW9wIQv}xH`@%O?$lA&jgQ&?ZKtsXQKQC;<A
zm{zPx6w?Mg{w(fM5|pUx(*cfYzzRJheo-b(X3MbG3+;+trWKe7-ZmZVA$=T!-O7q!
zzll|7sQxtbjY`?o(r-Z<|FGh`$5suHS|a07M0}sgBu4-by-Ty#0_{tu8vf^6qOie}
z42P!xI}z}T?&Am5WrIaS5jsfID7Zr|T0uj34J+D0b1$FJ%*{hZ)F%u2S*<!253@%j
zfe&0~#3(k3&?>Gct7amS!nNpvJDXoex3Q#%B{ZzaaoKZed?TyLf>CIYSjAMkHxg*=
zm{46H<OBjLN&VI}I*y$`C<6+?$3wyl#rS(EVGe}<fp|y3G|v8Ovi_f8hX3CxaLI;%
z4(aGhvM*_?OY@-B_5wU+MT34gWYb^kB&fO=qL8P0)<)poMNJ@D(MJF$eLRKnoWi6>
zLF#*%5(RWPTm}(k-mk(zmI@0>t|#d@g!SLW=tpFqY~-)*3nTl8{k&-}`*Hv{*3vxI
zZVOJTf_r&)aT&}g@ce+~{rjD;$+qq|Q>ACmJ|u+bVCOKgk$d#!=IwH@{=#z+@52J}
zct1(aWvj{KBPJ>)QA_$g`O<Jd0mEhu&Luqam*?%?egfQAkHdOAlJ*TjM<yX%Rg&}M
zECgwTjb<09;0D_=$VKLIvyJi-3qRo}BIPdrizWN7@WLx1>(S5P*_5Xn)7r-RDf@@t
zXC7f~%=JWx=RO@y_V0(GGlRJeBWDJ0FWiP*%--8Id}0|}Lm-Pv0>2KNtTrdz-QD?v
zkPs0!TywK<d0t+)dGA_^&;>w_&QZ6#jSJCm<GByp)Oa9OobD`Lai?EUn1g!tp*LZo
zVcrWp8wwdbwBcQ+%eW05NS-8%5}?DbfjuHHCcO>n{zI6`F-CGVIn5HQAP}c_bxQQ8
zuhnHd3yXuuQy)ue)*+%7!BWLwRdn&nmu-=`;$!nacakNo>)t8Gle}HH*ZS1>p=peQ
z4`(QvBOc`X7JhJIF4O8vzl+ndir{cjNu(cpLS3hf>)}u@5~cRoDVS$>?2VS;5j;iW
z%=)l~ly@de<ey6P@z5x!ts(YCvp&8p*E#+B87!XLxou|TY`Cd#U7+!kOu0`|H0g5x
zL^~IW2Zqw@gt@x$+;J`(SKx9zoBgBP)bY3RJypyaTG^37st6_zygYZjC#_-8XEg%!
zWcKJWXP@<!q_($vj$M_^8|pc>mwNIW6qbw5!U4fq(Kn>!%IW?)CqLBfM5ou=c>`|&
z_@!EFoYB$Io7>ydH*OXd5gJTjQ&XyO-}W=g&f}wPk~$Hx_t{;8X{xB!b;CYL;H|Q*
zr_Qg-vKF2`i{>|ZVHGK%{qj0IVvky;R>pp_E|G>a&&ui{TcKznD^P55&zpn}Ez?59
zjyRe^Q1(&ij=l9E5^Ew|%s?pIzAZ8*t5|JSS$q>=G&zee{(&QYqLGuvpKFQVT=nq?
ze&|?7#?umI?#O89*ONUtfc#r-FilT^{W4co`#?^Xp1g(VcgnC)L>8CZFOKbJJ&p>%
z!X6a8evh!3c46_mJtqcg35zz_$8x5jDBnd?<%WEWLZPs!_=1Mk@RNO~SZ|t0H!B#a
zK(7J4C8ZkHmq(i&&AiDyN<bN5Z~L99&Z4A)0<w)PQPZLF(>8KYdqeP66UWgMEXPOb
zr|PsyE=vhe4lxDamCgRCLM&)V)zt2|@0SvKCVJQ6?BGRkU0nM&=1y&e0J@IVxv>q-
z<$?9dQ0}tSi#!Se*HiS~_FWbO-Dq1Et@|IHhtmM(bEAiA{(-lq_$oANHfF!;9({Oz
z_dzc=>PJrA2u)3llY_#B=HQcB{z2hXusz@Rn;pN#T1R>wz^f?GBU;o&?jWQj{<#yD
zO_(J6<pFgL18)Wo%L1IcL-JI~^&Q~C66vNRm}(XO+P7^AYL4$3BAb~exYV#Qd$7((
zm`737XAhZACKY&`*lp6)!F##5i@i$)h`RvgH_iPQ9&N9<>$XSjC|9!45N}u>K<59{
zhCd&hLMDDSraK*99DsGw@2pA~g>J>tcd||}5@W#cqd_9gsh&PswzG_tAjo^N=5@7h
z(@lpTu-AX-bQ{xjhUZxCq=K)(VjeW;?GDulHsMzg0BdNYD``SAc^=Z~1*;I2qfIo9
zT|8+WkAqmL0xHk#zB~v!sko4bH{}siIN1{<oGpqSwM#7x>+pxHCsT6pN^FiXViKQx
ze9pj5Cyi*Fptw)epV>_;@(Medhn>{dQ`ouD;b;%Q3B%CFm88uZ^r*$y^~3+p;3&N6
z%U?;%?qI*TEA7mB)0lzjH7&~<N6N<?(OMq=5+&rAwJ7VQ0C8QbZs8U?vY$zIz0Jii
zOvxb}^#sZ1cgB8-IReMw;lNZ`O?UBJZQ9lpdlJo0YeNsr(g|0g1H3GQNpd#S_rt_5
zLAZT8EZ*&;&FzI5Ia%Lh+D%a1HR&IvZ6bQrT?m|5zn>~z-QQ%V^TM&i^)}bV8ca&x
z9lO&kQ|0tJ_<@9i*4tB7h%z3+sZmP=AHMih*QU+<Sf(wP0eH2L(C4D_3r5@qlbwh$
zKUUNOqNqd1kX5}(U3c(b)v3)m<xI<?ZoM-wGc72tYq8~CQH3~tOGu0@A624Gxa;FU
z`t<p2@4f*bowxPnmIWeGV0Yb{Xw1~XVObY0^pEE?G0)<{oios1a$W70cNo~fnDD*l
z3``RF93~r)dB+{Qv)q*x1rWC<OFeA$9NBG3Ax)X)ht^!;YL)lj#rEgAL}CrCU}B4n
zvOnGFrVXxd8@7~3DXV@kH{mZnb^_=mcIi(?69t4YvC+I3uRDtn<-o*F@~7Yhtv_=t
z8aNwjs-dP}`JJGOldOI^!y!s=#Gm}=fVsN?hGV<Jqu_+Tm{4?vAr39xO3;11B0va8
z3UXM6A|J+|h&rKA2+>RszmJIW{U>*#gicUU@S!cbq~ucuU*cl5d9EdXC49pk5^KvE
zI!+7y;Acs)0+>==U1LqL{UghZg(mV}h188MwjtxfmY4G$KKSgs&7#zfM(HXg9RcB6
z>6*o8i_WH^A_W^J!;yYTQqv)Np0KRyzTs3paL}^>1>d)BgIUIkJJ!>J9uwc&SY-+&
zmK;M)tzQ>4Sn^37*tbIZA0%D_Fq&yk*0?46sl0XhJI$tx-F{%X)jAv~tdggk*P^x<
zas7}deGM!##m)?Cqz{iTeTTWCfX7o0L6J!IJdDV>t_!aVMi(Z8Dk58B35LFVhYFCD
z5LF|okvBiq4?4%qyK0(d0lS2L2R+yLvqN(l=tx|sA@oYaD&v?9md_QxOr}6vJH>zi
z?6X;!qP<8610F?q&ff<FKMWzRUvTU-E9G9z;0FBoJ#7m5^?v&Z4|fi=QOTzJWfW1a
z{Y2_*Lm*CmyC^Z3cdy%$#PneF7l4zbkWp#NibW|!WUGWnluBVQhzN7t16gOBw6Ps;
zX4Q>qEi6OJhIkWXROMYGJvGN=^20R%gDVh;*0>s~6e9ZKQ5s}}|JGsra<=*~T&V1z
zkN&ZAlheJSn>dl1L27kc?y|x&`Miw6?U68oU2lbYdC;ppBb9k9zbae#tE|f9CY(jE
zpdG_i`;q9|gg{*3sWo$W$7t@@=TED;2#7D5^s#pHF|QKTvFGWsY6Qj?7)pSb7sOHp
z%Py<1l^~wYV%nAIFGOm7C67GK)jYD{GpDF+MLztE*zx4MAUY~gQPfR6e<kd%5XE2Q
zm-<+C*)p4Rity%|aV<`)pr3+82D&1^Nr_OxJuNwylZ;iZH`a(t5)6lH;xlmAGD=d&
zXF;g9!`2<IwZO|ks+;!X_#oy?X*0!~hI+pta8RiJTuCeRex~B+eJ&>P*IqoU4*d+H
z9X$qB{fym>YLbv3Km_sCaUpR24#4%R$dc!K?dV4gSuavvM=wv;_r2U1{!4-)UV7hm
zKD$})=YJu1p43QBnWfavRelg92sFc@G$*XOVTBy0c*{lFu_fuM{N&~}9rkz$qh^3h
z?X=4vCWt&|uspcBs<azO71(}*(P%|Gm@L`$G2Bi)8|29fg;y#lL^hQWIhdS-xwhuS
zw$2Q=d}W_>=f{1T`Z8n3SkPk2eAcB}{J2O6?|j;E7OLD*PV{;NUKcU`wcaM#nVIUv
zdA;aqMXID7Zv&jU?|SSZ76gmw%aAFQ_Vfn8dokwU<!0*pJR`GJf-A}=zSpOR$t0L)
zXw40i9|b4;mHalcHK7_w;uv%m<a@6CpoieJ4Q=I>p8fDzmPZ^GYyWM*pO1CRWmG$z
zR8e_T`JtQiGCD)5C81y;so#0a!uuXf<3lpm9O)D2?{+PXJQ>YU^Tu5%74KdCX-M~)
z_g09veSBT~a7DW#RR5m)`t$ZLs?caAiN{IXlgwCFEE5L|b#9-xK}f9nkv<Mi5i+-3
zkAxlDFn6hTs21`R<`dsx>i7>y1K?=lK!l&bmvW4^7OE1aa)L*F9*L73b9hs`!@ifo
z#RlO83D-W#5~WZv9<f)2A2dN?H#6@fZAsxBWWYHvArj<ZQ!<Lr+L}H%9W9yN9JxtI
z2L^_`Kuk+LBv1|gUDSAH&G4MDNL@DY6Gt&8=7?3i(tMKZIcyS&K6;^wdWTww1^i@;
zT$jMcG&%OxbkZ=RU(zRl^2lV=C}>R<sg`>_qUB{pJ9vF&LfHgA49=>{C5qP;UQR|j
zvgEzTTABotP1j>LX`=yBwqix$5mE3#@eW}NUHpO{jg@D}fTO<{=t31OVmC5boor@)
zgzx)%xC{5HqD#J;!C0s>C;WhMi=JeR?o}hy?@iExt}|u|0Y``{s`!)7?>hj>@qA$p
z8<lw%&S1}Y?$si)yf<uf8RBu!2cLnoUk`7mtDX^ryPh4F=K|MFcGb=G`?sw|S)M`!
z&k?5!ie|sbtjI)4;1k3o5^Ma5!hk?J<dUV5QxXNbTAI9K<+Pd@95T^-bTst57|h#9
z__zz>G<0KdR7)HGI=yrKTI=fK>SBs(f%gh=U87)!LWbD22MEtHzj4uZk9++1yb&u$
z{A!NaYFrV(i*5osa$E5{E4i=gda{K_iQb^pO%Q$~WMxc^&H#F-bq@yi>|RGU-cAaY
zM!?m7rG|KzHe8zkofOc^zGJ|iY!{aGR1+Vflp1%p190|>Eco$QlBx{PMn=_Ezqi)K
z=e0&_$0q${hTZOnt~~-wB<J9ksz#I$lm7NyV#y`A!9iwiO8m56RSmgUiyJZS5ev0g
zpg8&mMF?6U-M}JWr#f%be2NIl7$<d1PUxnr-y<DBs^6QntI5mEPW&69#D~sfEMN}w
zsJU)(dm)7ERQ(<K+`+*<C4ipt;Sze6T3`f7ni|bwf;yP5g?flEZ1gd<E{2%_{T!i4
z_|)iT@wa`{h6Gfsu1HJ=Z;aUY52x3=W~5Vnx(?KHe7GcMeOge9N>T5w)ig8L4mV>#
zr$T$ggt>f}h6e3uW!vHxmZ0h51f>I>#dq4g!W1T}3sL#e=cJQ2MFKh$bb$}?*nOFV
zLDy?P(Kz&|ub-doom%BAfoZkvwQBDJ`o~pV4dKg@B^2droStG?<GP6*S(`s!Eo(gH
z1_e2g2H$!hKI`q|u%$C$c<0Fg`y~pVD(gTA4G=&>qjP#=Q?h4%enX?Up$4TDFT0H8
zs{=B69ko$Z3rncciFJ+p#ynwt2~2^TOP>&Ikh7BNV+Afh30jt}m$Yi0?Wrgx=M9YP
z46Sb2$57+Y>gC=8MI(cZf<>*?Zrc+kLzFxgc3KT<JhS4St=GaBNgNw0F<l%ezBzj|
z_vmokvYa2Bq*JKcP@``3335n`Mx^Q2w>e*GmSL{9swN4cVjHs8O1a?*@OzklUNHLJ
zRzgY{8m~+QzdaDnN{{E<I>oo=n)ix`nyT)!{a)eH8-3)oI4$FQcuk%wp*hECpNJeD
zG>t3Mt*`pFugGP$7Gdd>N|qQgE<fZIF_V)#rPbcIx8Y0qzCz)5p?tT&#p4y>%cEC#
z<u;Pka)zk5%-ERhkb>;*7T?9|$0?KWmYYl+$=!V%3TuceJVv6VBzuk+eSiCcz#C27
zIm@hL$(~62Lz#%WIv@9W)WcQm(ZYQ`yRY@%djh0Si=;VJftR0rWULJX0#t>{a?S9W
zvzqfh+f1gxREu#el9mMZ80}Zr2FN~)5RRr`Zz`F8Ft=f$tRf`UMRQ~od5TMBida?m
z+a?C&>v6fh7yS799Jf!9=hNO@EpA=U0-wm8s(+po<!zyyygR8M>Uk$l%Igv)z@F9_
zeS@ULWR*?ks*j=4HMDWUHmb)yW64{C#pyPi**TzZjLm%>VWw|=w;7kLGf$}ow^S$S
zfw(hW{OHUm0_Y9mg_G5~3dLp$$;YDX8^AIZZp@r<3ca6c9CMeN!hVW%Ey-mBURtJ(
zYae*RE>*%Rr1Z6kp(|P*(BpJ>%%Nd0;;T}H3UW=RV7s+lJHsiu{Vof>A!@Z4gCWou
z#qM;mrv0fgoR0%rCN8w6&}vc93RmDecS`!CLMbd5@%(5sHUCBPeBpy^!-ef`K3YFg
zS2g2Nbad^v-H%oHq{nELI7~zkZqKYO+!k9~rKuHo;AP%A+B_)+kM&qILYd_m6yBZS
zbqMlAxR0ZTW;(HCc=;bT>jQ;6E!`1;ywL-Lv7iyD8@22Gb^Fy_RAE7J&nn5AC(1Gm
zIElg$nRzKCpC8Lp?#EswC??=^1th#VIBMMib^dcQ&?etr1xj2C@B5^ys3Ux1)@-i^
zv4<~-d*3)Ya+rI&JmNP#MLw}tC)HF?-wAZ}QXV!3PhJo2;$zg4n>I?iSRmsSq8&O3
z?n}#GPABxpnC|*<Wlw^>vWjq#8&`OKHIV08Iu%o$szj2(KhTKqnwDDe?=}_Y4oM6{
zvu?6KCnRs3@AP)c%=Yx3MCeZetpHLHlI**~fZuNrmg9eqa43A9?`4TVe)Q+F#y~Uh
zD(H;=#>$PN^YxgW$FpkiRXSCG7r(OFU_UXQG0Y!MsA-TKfaGGOv3=wf4}sHmDdvA3
zA^la`dMbW`G=VTJ_A6nH%QIb>Bku#lntQ_DCH9p?s6}LWbteQr1Uq<54<XSQ+4zO@
z1AANlCC~n1!Bezx4a+FGbZa7fZ*RY$`SB@%sjkNpmSSIIl&LEFI$aK8H~%GPxr3|)
zu`?0mT!Rf^hoC-sq>(bo6Hi`53Nd`HJFG$VFH%Vdk_0w2pKh&9W}?+t>r5vx(DA?!
zh>tl(;^d*<6j1uQIKsWT6(>ewp0|xgTzsd@718BnAL3!ixkFoN1Jxrg&jcgynX7wR
zneGfB+`oz3GAjYzIyoSa@0?_BQKD2{&k#XikZUh}Wjww`+{T1<izZA80Yb*Ncr%Fx
zR1^I~Np8|qOY%D8jjtD$9@d!ch<8l_cabm>WpK8z&G#OzeVkc5u|0SVzxydBu74Z}
z*Ek5Tj_SUuZdN|bV;sz*Y}}M{@!J~SSU*H(<lK>9^%O8jQAxQ`%>_?c=!a|ZEKun6
zY?~Jg3m}o4bsx%`oXVK&Fao=`XNWV%&J2bs4T|M<GjrA2xx?Ssk?xtxdlY55972ES
z5od&02=uiSXWcN#?kXv%ylF=F;5f%E8j325?mfxe_d&;syLrzKEn+D64^XgOo7V9_
zbBbPxK%9OK!5uODaXfH{T1gSTnmhfpi5r<DFmh_`aJR>DEgO;Z;T$wQV^D5lGY&N8
zySd?%J&;vtem^l}77?qDzKnni;w|y!mwR2`4|QkLIzB;O#KI@HLnFd3ygSw20x{hA
zDAXwTFEs4=c_PYwua~%`HwidymT}6jEa$uc?chdgrVXf?y+}g2+rjz<rI(82$T(`2
zs7k#zB!gEtLW$U#=REOs@?hj5-OpTK&q-Nmo`R0f%6W>$m_cz(er{$VgaFFzuE~z9
z<96s*Kh4;FzK{KaZoaDGVO1x2GsSaz-Ws|f>{CtaKWj&xhW<?)`IYGCrC}2z6#8db
zZnP<?%!rLYW%D?o(Jo<ccmp(wjLtL=$Y*Siw0EUeli9PF3wZ5<J5o~Ic849y9HFbW
zrG;jnY2y>EwvAcJAW!|k0=`ct$rb`#E#)DLv%t#<8jwc>r&dQBS+v#%RB<MaBt;H9
zpf^o$ip#S4cOvU=VH@jxMM44y?(^}_qvT&E$=$pN&$HwY+@I~Gs*+)R5jjh5nYJ>j
zaFK!!ZnTf0KXr40Gde9x8DYyvK25tgre?=cyycTI0CK+XTSMzhe0TAkkTbN4q*;HZ
zD@%FzC}&x$*i2XO6x?r-c@qm}IUP~mogsF~0A~w|Ub~?gyUBRTi&mWp?Ha)-G@kjg
zkGHXuYX(I9?#5w#mGAcFRUS|>@~&oCQ-eKD_=9C^zE5yu*Mta@+23KU8Eq8M47um@
zl0>|b71Ffd=m_(Q9ui!Gv5pk(-#~F(9w4Q`Af?w>lNm+%m2q2&#W9j3yPw)Gd)6Fu
zF!0`WFvsXsISD917GHl3L79hg?GCPTg`_kKK;S5hR6lGuc>K!h5+oRN?!l4-B)q9w
z#bncvpjPf<xRN%Nj{W+Ow4JI#J}gu@Gl{UL`^(z_mrR~fM*Dy^apw6Au%h!Mh`{*G
zHOql8U_fO;anm*H^?NjsW{8=UQ$`t65{w)JVD6GzElK_P1wKJ|k*FV#ChX4_2jCC=
zcNa+j6A%{e@76HjX9*<qf6GcX4(t+9$_#EMS-+f7n1K$Y6<UVfSpQQB#QnlesXkQ>
Tyui452mZ)PDoT`#8T$P{6=#Uk

literal 0
HcmV?d00001

diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
new file mode 100644
index 000000000..dd1dd3a62
--- /dev/null
+++ b/docs/topics/developer-tools.rst
@@ -0,0 +1,248 @@
+.. _topics-developer-tools:
+
+==========================
+Using your browser's Developer Tools for scraping
+==========================
+
+Here is a general guide on how to use your browser's Developer Tools
+to ease the scraping process. Today almost all browsers come with 
+built in `Developer Tools`_ and although we will use Firefox in this
+guide, the concepts are applicable to any other browser. 
+
+In this guide we'll introduce the basic tools to use from a browser's
+Developer Tools by scraping `quotes.toscrape.com`_.
+
+.. _topics-livedom:
+
+Caveats with inspecting the live browser DOM
+============================================
+
+Since Developer Tools operate on a live browser DOM, what you'll actually see
+when inspecting the page source is not the original HTML, but a modified one
+after applying some browser clean up and executing Javascript code.  Firefox,
+in particular, is known for adding ``<tbody>`` elements to tables.  Scrapy, on
+the other hand, does not modify the original page HTML, so you won't be able to
+extract any data if you use ``<tbody>`` in your XPath expressions.
+
+Therefore, you should keep in mind the following things:
+
+* Disable Javascript while inspecting the DOM looking for XPaths to be
+  used in Scrapy
+
+* Never use full XPath paths, use relative and clever ones based on attributes
+  (such as ``id``, ``class``, ``width``, etc) or any identifying features like
+  ``contains(@href, 'image')``.
+
+* Never include ``<tbody>`` elements in your XPath expressions unless you
+  really know what you're doing
+
+.. _topics-inspector:
+
+Inspecting a website
+===================================
+
+By far the most handy feature of the Developer Tools is the `Inspector` 
+feature, which allows you to inspect the underlying HTML code of 
+any webpage. To demonstrate the Inspector, let's take a 
+look at the `quotes.toscrape.com`_-site.
+
+On the site we have a total of ten quotes from various authors with specific
+tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes 
+on this page, without any meta-information about authors, tags, etc. 
+
+Instead of viewing the whole source code for the page, we can simply right click 
+on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`.
+In it you should see something like this:
+
+.. image:: _images/inspector_01.png
+   :width: 777
+   :height: 469
+   :alt: Firefox's Inspector-tool
+
+The interesting part for us is this:
+
+.. code-block:: html
+
+    <div class="quote" itemscope="" itemtype="http://schema.org/CreativeWork">
+      <span class="text" itemprop="text">(...)</span>
+      <span>(...)</span>
+      <div class="tags">(...)</div>
+    </div>
+
+If you hover over the first ``div`` directly above the ``span``-tag highlighted
+in the screenshot, you'll see that the corresponding section of the webpage gets
+highlighted as well. So now we have a section, but we can't find our quote text
+anywhere.
+
+The advantage of the `Inspector` is that it automatically expands and collapses
+sections and tags of a webpage, which greatly improves readability. You can
+expand and collapse a tag by clicking on the arrow in front of it or by double
+clicking directly on the tag. If we expand the ``span``-tag with the ``class=
+"text"`` we will see the quote-text we clicked on. The `Inspector` lets you
+copy XPaths to selected elements. Let's try it out: Right-click on the ``span``-
+tag, select ``Copy > XPath`` and paste it in the scrapy shell like so::
+
+    >>> scrapy shell "http://quotes.toscrape.com/"
+    (...)
+    >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').extract()
+    ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”]
+
+Adding ``text()`` at the end we are able to extract the first quote with this 
+basic selector. But this XPath is not really that clever. All it does is
+go down a desired path in the source code starting from ``html``. So let's 
+see if we can refine our XPath a bit: 
+
+If we check the `Inspector` again we'll see that directly beneath our 
+expanded ``div``-tag we have eight identical ``div``-tags, each with the 
+same attributes as our first. If we expand any of them, we'll see the same 
+structure as with our first quote: Two ``span``-tags and one ``div``-tag. We can
+expand each ``span``-tag with the ``class="text"`` inside our ``div``-tags and 
+see each quote. With this knowledge we can refine our XPath: Instead of a path
+to follow, we'll simply select all ``span``-tags with the ``class="text"``:: 
+
+    >>> response.xpath('//span[@class="text"]/text()').extract()
+   ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”,
+    '“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
+    '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
+    (...)]
+
+And with one simple, cleverer XPath we are able to extract all quotes from 
+the page. We could have constructed a loop over our first XPath to increase 
+the number of the last ``div``, but this would have been unnecessarily 
+complex and by simply constructing an XPath with the ``class="text"`` we 
+were able to extract all quotes in one line. 
+
+The `Inspector` has a lot of other helpful features, such as searching in the 
+source code or directly scrolling to an element you selected. Let's demonstrate
+a use case: 
+
+Say you want to find the ``Next``-button on the page. Type ``Next`` into the 
+search bar on the top right of the `Inspector`. You should get two results. 
+The first is a ``li``-tag with the ``class="text"``, the second the text 
+of an ``a``-tag. Right click on the ``a``-tag and select ``Scroll into View``.
+If you hover over the tag, you'll see the button highlighted. From here
+we could easily create a :ref:`Link Extract <topics-link-extractors>` to 
+follow the pagination. On a simple site such as this, there may not be 
+the need to find an element visually but the ``Scroll into View`` function
+can be quite useful on complex sites. 
+
+.. _topics-network-tool:
+
+The Network-tool
+================
+While scraping you may come across dynamic webpages where some parts
+of the page is loaded dynamically through multiple requests. While 
+this can be quite tricky, the `Network`-tool in the Developer Tools 
+greatly facilitates this task. To demonstrate the Network-tool, let's
+take a look at the page `quotes.toscrape.com/scroll`_. 
+
+The page is quite similar to the basic `quotes.toscrape.com`_-page, 
+but instead of the above-mentioned ``Next``-button, the page 
+automatically loads new quotes when you scroll to the bottom. We 
+could go ahead and try out different XPaths directly, but instead 
+we'll check another quite useful command from the scrapy shell::
+
+  >>> scrapy shell "quotes.toscrape.com/scroll"
+  (...)
+  >>> view(response)
+
+A browser window should open with the webpage but with one 
+crucial difference: Instead of the quotes we just see a greenish 
+bar with the word ``Loading...``. 
+
+.. image:: _images/network_01.png
+   :width: 777
+   :height: 296
+   :alt: Response from quotes.toscrape.com/scroll
+
+The ``view(response)``-command let's us view the response our
+shell or later our spider receives from the server. Here we see 
+that some basic template is loaded which includes the title, 
+the login-button and the footer, but the quotes are missing. This
+tells us that the quotes are being loaded from a different request
+than ``quotes.toscrape/scroll``. 
+
+If you click on the ``Network``-tab, you will probably only see 
+two entries. The first thing we do is enable persistent logs by 
+clicking on ``Persist Logs``. If this option is disabled, the 
+log is automatically cleared each time you navigate to a different
+page. Enabling this option is a good default, since it gives us 
+control on when to clear the logs. 
+
+If we reload the page now, you'll see the log get populated with six
+new requests. 
+
+.. image:: _images/network_02.png
+   :width: 777
+   :height: 241
+   :alt: Network tab with persistent logs and requests
+
+Here we see every request that has been made when reloading the page
+and can inspect each request and its response. So let's find out
+where our quotes are coming from: 
+
+First click on the request with the name ``scroll``. On the right 
+you can now inspect the request. In ``Headers`` you'll find details
+about the request headers, such as the URL, the method, the IP-address,
+and so on. We'll ignore the other tabs and click directly on ``Reponse``.
+
+What you should see in the ``Preview``-pane is the rendered HTML-code, 
+that is exactly what we saw when we called ``view(response`` in the 
+shell. Accordingly the ``type`` of the request in the log is ``html``. 
+The other requests have types like ``css`` or ``js``, but what 
+interests us is the one request called ``quotes?page=1`` with the 
+type ``json``. 
+
+If we click on this request, we see that the request URL is 
+``http://quotes.toscrape.com/api/quotes?page=1`` and the response
+is a JSON-object that contains our quotes. We can also right-click
+on the request and open ``Open in new tab`` to get a better overview. 
+
+.. image:: _images/network_03.png
+   :width: 777
+   :height: 375
+   :alt: JSON-object returned from the quotes.toscrape API
+
+With this response we can now easily parse the JSON-object and 
+also request each page to get every quote on the site::
+
+    import scrapy
+    import json
+
+
+    class QuoteSpider(scrapy.Spider):
+        name = 'quote'
+        allowed_domains = ['quotes.toscrape.com']
+        page = 1
+        start_urls = ['http://quotes.toscrape.com/api/quotes?page=1]
+
+        def parse(self, response):
+            data = json.loads(response.text)
+            for quote in data["quotes"]:
+                quote = quote["text"]
+                print(quote)
+            if data["has_next"]:
+                self.page += 1
+                url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page)            
+                yield scrapy.Request(url=url, callback=self.parse)
+
+This spider starts at the first page of the quotes-API. With each 
+response, we parse the ``response.text`` and assign it to ``data``. 
+This lets us operate on the JSON-object like on a Python dictionary. 
+We iterate through the ``quotes`` and print out the ``quote["text"]``.
+If the handy ``has_next``-element is ``true`` (try loading 
+`http://quotes.toscrape.com/api/quotes?page=10`_ in your browser or a
+page-number greater than 10), we increment the ``page``-attribute 
+and ``yield`` a new request, inserting the incremented page-number 
+into our ``url``. 
+
+You can see that with a few inspections in the `Network`-tool we 
+were able to easily replicate the dynamic requests of the scrolling 
+functionality of the page. Crawling dynamic pages can be quite
+daunting and pages can be very complex, but it (mostly) boils down
+to identifying the correct request and replicating it in your spider.
+
+.. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools
+.. _quotes.toscrape.com: http://quotes.toscrape.com
+.. _quotes.toscrape.com/scroll: quotes.toscrape.com/scroll/
+
diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst
deleted file mode 100644
index 4ea8d3bd0..000000000
--- a/docs/topics/firebug.rst
+++ /dev/null
@@ -1,167 +0,0 @@
-.. _topics-firebug:
-
-==========================
-Using Firebug for scraping
-==========================
-
-.. note:: Google Directory, the example website used in this guide is no longer
-   available as it `has been shut down by Google`_. The concepts in this guide
-   are still valid though. If you want to update this guide to use a new
-   (working) site, your contribution will be more than welcome!. See :ref:`topics-contributing`
-   for information on how to do so.
-
-Introduction
-============
-
-This document explains how to use `Firebug`_ (a Firefox add-on) to make the
-scraping process easier and more fun. For other useful Firefox add-ons see
-:ref:`topics-firefox-addons`. There are some caveats with using Firefox add-ons
-to inspect pages, see :ref:`topics-firefox-livedom`.
-
-In this example, we'll show how to use `Firebug`_ to scrape data from the
-`Google Directory`_, which contains the same data as the `Open Directory
-Project`_ used in the :ref:`tutorial <intro-tutorial>` but with a different
-face.
-
-.. _Firebug: https://getfirebug.com/
-.. _Google Directory: http://directory.google.com/
-.. _Open Directory Project: http://www.dmoz.org
-
-Firebug comes with a very useful feature called `Inspect Element`_ which allows
-you to inspect the HTML code of the different page elements just by hovering
-your mouse over them. Otherwise you would have to search for the tags manually
-through the HTML body which can be a very tedious task.
-
-.. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA
-
-In the following screenshot you can see the `Inspect Element`_ tool in action.
-
-.. image:: _images/firebug1.png
-   :width: 913
-   :height: 600
-   :alt: Inspecting elements with Firebug
-
-At first sight, we can see that the directory is divided in categories, which
-are also divided in subcategories.
-
-However, it seems that there are more subcategories than the ones being shown
-in this page, so we'll keep looking:
-
-.. image:: _images/firebug2.png
-   :width: 819
-   :height: 629
-   :alt: Inspecting elements with Firebug
-
-As expected, the subcategories contain links to other subcategories, and also
-links to actual websites, which is the purpose of the directory.
-
-Getting links to follow
-=======================
-
-By looking at the category URLs we can see they share a pattern:
-
-    http://directory.google.com/Category/Subcategory/Another_Subcategory
-
-Once we know that, we are able to construct a regular expression to follow
-those links. For example, the following one::
-
-    directory\.google\.com/[A-Z][a-zA-Z_/]+$
-
-So, based on that regular expression we can create the first crawling rule::
-
-    Rule(LinkExtractor(allow='directory.google.com/[A-Z][a-zA-Z_/]+$', ),
-        'parse_category',
-        follow=True,
-    ),
-
-The :class:`~scrapy.spiders.Rule` object instructs
-:class:`~scrapy.spiders.CrawlSpider` based spiders how to follow the
-category links. ``parse_category`` will be a method of the spider which will
-process and extract data from those pages.
-
-This is how the spider would look so far::
-
-   from scrapy.linkextractors import LinkExtractor
-   from scrapy.spiders import CrawlSpider, Rule
-
-   class GoogleDirectorySpider(CrawlSpider):
-       name = 'directory.google.com'
-       allowed_domains = ['directory.google.com']
-       start_urls = ['http://directory.google.com/']
-
-       rules = (
-           Rule(LinkExtractor(allow='directory\.google\.com/[A-Z][a-zA-Z_/]+$'),
-               'parse_category', follow=True,
-           ),
-       )
-
-       def parse_category(self, response):
-           # write the category page data extraction code here
-           pass
-
-
-Extracting the data
-===================
-
-Now we're going to write the code to extract data from those pages.
-
-With the help of Firebug, we'll take a look at some page containing links to
-websites (say http://directory.google.com/Top/Arts/Awards/) and find out how we can
-extract those links using :ref:`Selectors <topics-selectors>`. We'll also
-use the :ref:`Scrapy shell <topics-shell>` to test those XPath's and make sure
-they work as we expect.
-
-.. image:: _images/firebug3.png
-   :width: 965
-   :height: 751
-   :alt: Inspecting elements with Firebug
-
-As you can see, the page markup is not very descriptive: the elements don't
-contain ``id``, ``class`` or any attribute that clearly identifies them, so
-we'll use the ranking bars as a reference point to select the data to extract
-when we construct our XPaths.
-
-After using FireBug, we can see that each link is inside a ``td`` tag, which is
-itself inside a ``tr`` tag that also contains the link's ranking bar (in
-another ``td``).
-
-So we can select the ranking bar, then find its parent (the ``tr``), and then
-finally, the link's ``td`` (which contains the data we want to scrape).
-
-This results in the following XPath::
-
-    //td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td//a
-
-It's important to use the :ref:`Scrapy shell <topics-shell>` to test these
-complex XPath expressions and make sure they work as expected.
-
-Basically, that expression will look for the ranking bar's ``td`` element, and
-then select any ``td`` element who has a descendant ``a`` element whose
-``href`` attribute contains the string ``#pagerank``"
-
-Of course, this is not the only XPath, and maybe not the simpler one to select
-that data. Another approach could be, for example, to find any ``font`` tags
-that have that grey colour of the links,
-
-Finally, we can write our ``parse_category()`` method::
-
-    def parse_category(self, response):
-        # The path to website links in directory page
-        links = response.xpath('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font')
-
-        for link in links:
-            item = DirectoryItem()
-            item['name'] = link.xpath('a/text()').extract()
-            item['url'] = link.xpath('a/@href').extract()
-            item['description'] = link.xpath('font[2]/text()').extract()
-            yield item
-
-
-Be aware that you may find some elements which appear in Firebug but
-not in the original HTML, such as the typical case of ``<tbody>``
-elements.
-
-or tags which Therefer   in page HTML
-sources may on Firebug inspects the live DOM
-
-.. _has been shut down by Google: https://searchenginewatch.com/sew/news/2096661/google-directory-shut
diff --git a/docs/topics/firefox.rst b/docs/topics/firefox.rst
deleted file mode 100644
index 2c85848be..000000000
--- a/docs/topics/firefox.rst
+++ /dev/null
@@ -1,82 +0,0 @@
-.. _topics-firefox:
-
-==========================
-Using Firefox for scraping
-==========================
-
-Here is a list of tips and advice on using Firefox for scraping, along with a
-list of useful Firefox add-ons to ease the scraping process.
-
-.. _topics-firefox-livedom:
-
-Caveats with inspecting the live browser DOM
-============================================
-
-Since Firefox add-ons operate on a live browser DOM, what you'll actually see
-when inspecting the page source is not the original HTML, but a modified one
-after applying some browser clean up and executing Javascript code.  Firefox,
-in particular, is known for adding ``<tbody>`` elements to tables.  Scrapy, on
-the other hand, does not modify the original page HTML, so you won't be able to
-extract any data if you use ``<tbody>`` in your XPath expressions.
-
-Therefore, you should keep in mind the following things when working with
-Firefox and XPath:
-
-* Disable Firefox Javascript while inspecting the DOM looking for XPaths to be
-  used in Scrapy
-
-* Never use full XPath paths, use relative and clever ones based on attributes
-  (such as ``id``, ``class``, ``width``, etc) or any identifying features like
-  ``contains(@href, 'image')``.
-
-* Never include ``<tbody>`` elements in your XPath expressions unless you
-  really know what you're doing
-
-.. _topics-firefox-addons:
-
-Useful Firefox add-ons for scraping
-===================================
-
-Firebug
--------
-
-`Firebug`_ is a widely known tool among web developers and it's also very
-useful for scraping. In particular, its `Inspect Element`_ feature comes very
-handy when you need to construct the XPaths for extracting data because it
-allows you to view the HTML code of each page element while moving your mouse
-over it.
-
-See :ref:`topics-firebug` for a detailed guide on how to use Firebug with
-Scrapy.
-
-XPather
--------
-
-`XPather`_ allows you to test XPath expressions directly on the pages.
-
-XPath Checker
--------------
-
-`XPath Checker`_ is another Firefox add-on for testing XPaths on your pages.
-
-Tamper Data
------------
-
-`Tamper Data`_ is a Firefox add-on which allows you to view and modify the HTTP
-request headers sent by Firefox. Firebug also allows to view HTTP headers, but
-not to modify them.
-
-Firecookie
-----------
-
-`Firecookie`_ makes it easier to view and manage cookies. You can use this
-extension to create a new cookie, delete existing cookies, see a list of cookies
-for the current site, manage cookies permissions and a lot more.
-
-.. _Firebug: https://getfirebug.com/
-.. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA
-.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/
-.. _XPath Checker: https://addons.mozilla.org/en-US/firefox/addon/xpath-checker/
-.. _Tamper Data: https://addons.mozilla.org/en-US/firefox/addon/tamper-data/
-.. _Firecookie: https://addons.mozilla.org/en-US/firefox/addon/firecookie/
-

From 3a71e7dbce94e6056a5258c59df0150a94ddc187 Mon Sep 17 00:00:00 2001
From: testingcan <raphael.wuillemier@protonmail.com>
Date: Wed, 22 Aug 2018 16:57:51 +0200
Subject: [PATCH 1795/2974] Increased length of "="

---
 docs/topics/developer-tools.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index dd1dd3a62..aa4050948 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -1,8 +1,8 @@
 .. _topics-developer-tools:
 
-==========================
+=================================================
 Using your browser's Developer Tools for scraping
-==========================
+=================================================
 
 Here is a general guide on how to use your browser's Developer Tools
 to ease the scraping process. Today almost all browsers come with 

From 4d3aaabbcaa3a39ba56f443ccfe8a60888223013 Mon Sep 17 00:00:00 2001
From: Raphael Wuillemier <wuillemr@essilor.de>
Date: Thu, 23 Aug 2018 12:40:31 +0200
Subject: [PATCH 1796/2974] Updated code, added code snippets and improved
 readability

---
 docs/topics/developer-tools.rst | 84 ++++++++++++++++++++-------------
 1 file changed, 52 insertions(+), 32 deletions(-)

diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index aa4050948..6bd2c052a 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -27,7 +27,7 @@ extract any data if you use ``<tbody>`` in your XPath expressions.
 Therefore, you should keep in mind the following things:
 
 * Disable Javascript while inspecting the DOM looking for XPaths to be
-  used in Scrapy
+  used in Scrapy (in the Developer Tools settings click `Disable JavaScript`)
 
 * Never use full XPath paths, use relative and clever ones based on attributes
   (such as ``id``, ``class``, ``width``, etc) or any identifying features like
@@ -43,8 +43,8 @@ Inspecting a website
 
 By far the most handy feature of the Developer Tools is the `Inspector` 
 feature, which allows you to inspect the underlying HTML code of 
-any webpage. To demonstrate the Inspector, let's take a 
-look at the `quotes.toscrape.com`_-site.
+any webpage. To demonstrate the Inspector, let's look at the 
+`quotes.toscrape.com`_-site.
 
 On the site we have a total of ten quotes from various authors with specific
 tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes 
@@ -69,7 +69,7 @@ The interesting part for us is this:
       <div class="tags">(...)</div>
     </div>
 
-If you hover over the first ``div`` directly above the ``span``-tag highlighted
+If you hover over the first ``div`` directly above the ``span`` tag highlighted
 in the screenshot, you'll see that the corresponding section of the webpage gets
 highlighted as well. So now we have a section, but we can't find our quote text
 anywhere.
@@ -77,14 +77,14 @@ anywhere.
 The advantage of the `Inspector` is that it automatically expands and collapses
 sections and tags of a webpage, which greatly improves readability. You can
 expand and collapse a tag by clicking on the arrow in front of it or by double
-clicking directly on the tag. If we expand the ``span``-tag with the ``class=
+clicking directly on the tag. If we expand the ``span`` tag with the ``class=
 "text"`` we will see the quote-text we clicked on. The `Inspector` lets you
-copy XPaths to selected elements. Let's try it out: Right-click on the ``span``-
+copy XPaths to selected elements. Let's try it out: Right-click on the ``span`` 
 tag, select ``Copy > XPath`` and paste it in the scrapy shell like so::
 
-    >>> scrapy shell "http://quotes.toscrape.com/"
+    $ scrapy shell "http://quotes.toscrape.com/"
     (...)
-    >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').extract()
+    >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
     ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”]
 
 Adding ``text()`` at the end we are able to extract the first quote with this 
@@ -93,14 +93,28 @@ go down a desired path in the source code starting from ``html``. So let's
 see if we can refine our XPath a bit: 
 
 If we check the `Inspector` again we'll see that directly beneath our 
-expanded ``div``-tag we have eight identical ``div``-tags, each with the 
+expanded ``div`` tag we have nine identical ``div`` tags, each with the 
 same attributes as our first. If we expand any of them, we'll see the same 
-structure as with our first quote: Two ``span``-tags and one ``div``-tag. We can
-expand each ``span``-tag with the ``class="text"`` inside our ``div``-tags and 
-see each quote. With this knowledge we can refine our XPath: Instead of a path
-to follow, we'll simply select all ``span``-tags with the ``class="text"``:: 
+structure as with our first quote: Two ``span`` tags and one ``div`` tag. We can
+expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and 
+see each quote:
 
-    >>> response.xpath('//span[@class="text"]/text()').extract()
+.. code-block:: html
+
+    <div class="quote" itemscope="" itemtype="http://schema.org/CreativeWork">
+      <span class="text" itemprop="text">
+        “The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”
+      </span>
+      <span>(...)</span>
+      <div class="tags">(...)</div>
+    </div>
+
+
+With this knowledge we can refine our XPath: Instead of a path to follow,
+we'll simply select all ``span`` tags with the ``class="text"`` by using 
+the `has-class-extension`_:: 
+
+    >>> response.xpath('//span[has-class("text")]/text()').getall()
    ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”,
     '“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
     '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
@@ -109,40 +123,45 @@ to follow, we'll simply select all ``span``-tags with the ``class="text"``::
 And with one simple, cleverer XPath we are able to extract all quotes from 
 the page. We could have constructed a loop over our first XPath to increase 
 the number of the last ``div``, but this would have been unnecessarily 
-complex and by simply constructing an XPath with the ``class="text"`` we 
-were able to extract all quotes in one line. 
+complex and by simply constructing an XPath with ``has-class("text")``
+we were able to extract all quotes in one line. 
 
 The `Inspector` has a lot of other helpful features, such as searching in the 
 source code or directly scrolling to an element you selected. Let's demonstrate
 a use case: 
 
-Say you want to find the ``Next``-button on the page. Type ``Next`` into the 
+Say you want to find the ``Next`` button on the page. Type ``Next`` into the 
 search bar on the top right of the `Inspector`. You should get two results. 
-The first is a ``li``-tag with the ``class="text"``, the second the text 
-of an ``a``-tag. Right click on the ``a``-tag and select ``Scroll into View``.
+The first is a ``li`` tag with the ``class="text"``, the second the text 
+of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``.
 If you hover over the tag, you'll see the button highlighted. From here
-we could easily create a :ref:`Link Extract <topics-link-extractors>` to 
+we could easily create a :ref:`Link Extractor <topics-link-extractors>` to 
 follow the pagination. On a simple site such as this, there may not be 
 the need to find an element visually but the ``Scroll into View`` function
 can be quite useful on complex sites. 
 
+Note that the search bar can also be used to search for and test CSS
+selectors. For example, you could search for ``span.text`` to find 
+all quote texts. Instead of a full text search, this searches for 
+exactly the ``span`` tag with the ``class="text"`` in the page. 
+
 .. _topics-network-tool:
 
 The Network-tool
 ================
 While scraping you may come across dynamic webpages where some parts
-of the page is loaded dynamically through multiple requests. While 
+of the page are loaded dynamically through multiple requests. While 
 this can be quite tricky, the `Network`-tool in the Developer Tools 
 greatly facilitates this task. To demonstrate the Network-tool, let's
 take a look at the page `quotes.toscrape.com/scroll`_. 
 
 The page is quite similar to the basic `quotes.toscrape.com`_-page, 
-but instead of the above-mentioned ``Next``-button, the page 
+but instead of the above-mentioned ``Next`` button, the page 
 automatically loads new quotes when you scroll to the bottom. We 
 could go ahead and try out different XPaths directly, but instead 
 we'll check another quite useful command from the scrapy shell::
 
-  >>> scrapy shell "quotes.toscrape.com/scroll"
+  $ scrapy shell "quotes.toscrape.com/scroll"
   (...)
   >>> view(response)
 
@@ -155,14 +174,14 @@ bar with the word ``Loading...``.
    :height: 296
    :alt: Response from quotes.toscrape.com/scroll
 
-The ``view(response)``-command let's us view the response our
+The ``view(response)`` command let's us view the response our
 shell or later our spider receives from the server. Here we see 
 that some basic template is loaded which includes the title, 
 the login-button and the footer, but the quotes are missing. This
 tells us that the quotes are being loaded from a different request
 than ``quotes.toscrape/scroll``. 
 
-If you click on the ``Network``-tab, you will probably only see 
+If you click on the ``Network`` tab, you will probably only see 
 two entries. The first thing we do is enable persistent logs by 
 clicking on ``Persist Logs``. If this option is disabled, the 
 log is automatically cleared each time you navigate to a different
@@ -186,8 +205,8 @@ you can now inspect the request. In ``Headers`` you'll find details
 about the request headers, such as the URL, the method, the IP-address,
 and so on. We'll ignore the other tabs and click directly on ``Reponse``.
 
-What you should see in the ``Preview``-pane is the rendered HTML-code, 
-that is exactly what we saw when we called ``view(response`` in the 
+What you should see in the ``Preview`` pane is the rendered HTML-code, 
+that is exactly what we saw when we called ``view(response)`` in the 
 shell. Accordingly the ``type`` of the request in the log is ``html``. 
 The other requests have types like ``css`` or ``js``, but what 
 interests us is the one request called ``quotes?page=1`` with the 
@@ -219,8 +238,7 @@ also request each page to get every quote on the site::
         def parse(self, response):
             data = json.loads(response.text)
             for quote in data["quotes"]:
-                quote = quote["text"]
-                print(quote)
+                yield {"quote": quote["text"]
             if data["has_next"]:
                 self.page += 1
                 url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page)            
@@ -230,9 +248,9 @@ This spider starts at the first page of the quotes-API. With each
 response, we parse the ``response.text`` and assign it to ``data``. 
 This lets us operate on the JSON-object like on a Python dictionary. 
 We iterate through the ``quotes`` and print out the ``quote["text"]``.
-If the handy ``has_next``-element is ``true`` (try loading 
-`http://quotes.toscrape.com/api/quotes?page=10`_ in your browser or a
-page-number greater than 10), we increment the ``page``-attribute 
+If the handy ``has_next`` element is ``true`` (try loading 
+`quotes.toscrape.com/api/quotes?page=10`_ in your browser or a
+page-number greater than 10), we increment the ``page`` attribute 
 and ``yield`` a new request, inserting the incremented page-number 
 into our ``url``. 
 
@@ -245,4 +263,6 @@ to identifying the correct request and replicating it in your spider.
 .. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools
 .. _quotes.toscrape.com: http://quotes.toscrape.com
 .. _quotes.toscrape.com/scroll: quotes.toscrape.com/scroll/
+.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10
+.. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions
 

From e98e7f8506b401f80fa193faa741c2094d7b62d7 Mon Sep 17 00:00:00 2001
From: testingcan <raphael.wuillemier@protonmail.com>
Date: Thu, 23 Aug 2018 14:50:49 +0200
Subject: [PATCH 1797/2974] Added missing curly brace

---
 docs/topics/developer-tools.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index 6bd2c052a..c1976258d 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -238,7 +238,7 @@ also request each page to get every quote on the site::
         def parse(self, response):
             data = json.loads(response.text)
             for quote in data["quotes"]:
-                yield {"quote": quote["text"]
+                yield {"quote": quote["text"]}
             if data["has_next"]:
                 self.page += 1
                 url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page)            

From 79de3d569a3b50b3a07ced99d8e482862a979e88 Mon Sep 17 00:00:00 2001
From: Raphael Wuillemier <wuillemr@essilor.de>
Date: Thu, 23 Aug 2018 16:19:13 +0200
Subject: [PATCH 1798/2974] Removed obsolete firebug-images

---
 docs/topics/_images/firebug1.png | Bin 44391 -> 0 bytes
 docs/topics/_images/firebug2.png | Bin 69392 -> 0 bytes
 docs/topics/_images/firebug3.png | Bin 89644 -> 0 bytes
 3 files changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 docs/topics/_images/firebug1.png
 delete mode 100644 docs/topics/_images/firebug2.png
 delete mode 100644 docs/topics/_images/firebug3.png

diff --git a/docs/topics/_images/firebug1.png b/docs/topics/_images/firebug1.png
deleted file mode 100644
index e2eaefa838e627c3784ee76c5c8b13dd60783e59..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 44391
zcmb@u1yo$y)+G!f0RkaN2m}uSg1dWy1QIN`OVB{!u0;qIAV6?;3oeCgfS^TTg}b}E
z74#wZe%*cFd;iznWBg|nqsBRPZ127HT64`cH$flerLZuFF_4guu-;2cC?O%CL?9tO
z&_I6x9C3Vk`V#m;c2JTMLn`Sf-2`?X+e&LXAR%EsyZ=K*N=$wM97J<`FDHpM@dWqD
zODsgYv>_7GOQiP_Z&h5Uc4pl?pPAgE9ni`plAF_8y=VQv8Wvkknp5fDMjjlT%o38U
zgc~jX`;jJ&=9{;;pWa^32cp-e>IAcpqgQMrEAdmCyQiIGOATLl59u6V8%<DV_ZSm@
zAXzvDpN1Q+C77kUrS8i%yK%gCgDc0T-56sd1Mh=mt0KQ~-vXpJ{r62n`jY(o{_6+i
z&%u{wkM8&W{;gZ{&n18VX0LYt6Vgi2pRd1uTM(gsLkR3s`qd<KRm8f78uku`lU>r0
z*7-lRXWEIG7oT;dm1|1NTz$*8$u@BR#My|+vKH1hN+CXBbBezE$5PfPL>b)n4841}
z_l`9qaKHFXhY%ys$NdSJoKa!&M~J-AW0L;GeZQ8=?WA|T-9^Rbg__RpniF^akLPiM
zUA~!lvQyQt#N||fb9v>O<hFU_c(qwxWYh?2G}7$a$OHE0O!a;X;-(bb8Ta>;-w`uT
zF&s^7v2Si<H`WWCQ8C!C=pUSZZEm>2n8!hFB0K5JH|tG=%5u621c@>UC)?7s8Wo~(
zLSyvPL{}&9!;8r?i!*`ScyO#|Pm)d?7A>FPjWd_9UmI2DS@V8Gg<EI}z$+Q9?7Bq5
zydr^%E;o$f&h|u~L9_9Wd_gq{=WnOba{V`{$lPqRI|>BIrSk;ROy0mOls8qhy=RHN
z+2)R!6@WV$-Ep5K-Pjb78FZ1GQartW+WP9uxgs#M%GCBT%9(A|zQl$KCoy*@<3P#x
z8IsMj+$1PLTz69htLI(YXm6p;gGgRslgl23vjw~-NJyqc@47ahfzqU{-CUtpqD8cR
zDphJU<R6MGi@9{SDNm$z;{`^%e|A_AdNpj6ogB2%qY;K`NT4U2k2X>zNUc3h@K`C8
zLZ=MO4kYkya2@(4iIzvYqSVqsa8-Z#Ay!(fNsO5%y{L}q@MES}KAY|a?NLMh21C<h
z-SrATlWXsJ8-W^?F&%<CuxC?!s>z!9$fBY;Luq`8J0fG8_tuUouAhtSQHa04=cnrc
z=OnMToD?0@#Ldq8XM1t8mfI}FtXq>!EvE_Jwx3eO+f`~(Hi5!Y1~}4ln?};DFW^$%
zy-LPTTWp%{wac~>cIaf2mS%Zv0drgZ?Z5V(wN{c?@fh8hEf465b_wIleM$u^Ux0Rm
z@uW}LOl4aY3q&$TT}tUk8cCX|-gI?b7*iax-C4q@shBzYv)H3<@7D^7)l{{UXbD7Z
zy7s`%563;TLz-xSAm!BlP(a%;HdP}tE<EJLc>Yz?HH{fFbl7^tlocw!7pGHNGJgVn
z4Z?EWT_4B)AZg#SvSTvY_ZlN-sVYsMlle@12-|EuMfFzvebc+4($XaGq)KUt!cJ2B
zrDv{l7SHizpI`bzNH43XO9D&j?9Zl?9U8H2sfutu=iEs~f#dVm5o}k#^){DF`?WZo
zD?jVJN#sA)#>Z)D`bFv&U<!RZt|p*W@8G*-`jg~mf?bBHvhUhRB>}rz{?dE~1!)Es
zq3?f#J1EqYfK-ds5khzC^uD={?=9ySDIE(^m_8X56Op!mq0!kD!Ihn~>>QvGeI~}j
zRBPZmntl_gRIO4Cx+%f+q2g<b>%5=5WqqPfI=9<j^}7PT=fqW3*0@?7FCAZ8sh|4&
zNU!lNm@W`h+n_(O)Eqao)5pkpWnlIT=Z<ny2JT&WVb#~tx#MZMzU5H3+4Mw^jY&C`
zUZ0AvA*60Zp@wbUKe^Ql6<%m>;b3^y_0avH8ltJ!5aq%A(<1Z2Pm_Kn$43rvQCF{>
zX4rs!(`eCfe;@k_)a@Z2U!uGf4Rq~^>Yn;@*mHrbFcvCXybYeMTF2ZD#S3ag)FXXX
zoFjjI*p)@wm&ZC+zo(F=lD`;lM#0v3ZLUn<$5d(aqUNHM#?u{E+hCm<YH*KF51PPp
z6;i=@Q{?)o1>IJ_WXPm~Wuh#f2F?Z!;(7>9ws&fA@&zEv)J}Q5im&c6<;*;Lw>rM=
zfgzmLG31l#^WH1Uw4)zmpRC`c@sEQ{;jEwE&}NlnN%g~7Gi}gPE$)!hswy6{IDU57
zan)hUSfMH~dhS1eU3=LWe~}rUG_ctQXGmetju8Isjh<3wU+SGc@Jl<~3$4OZ8a-cm
z)Naz)cu+co<lSkIVXVYi5EpaV&Phc|%Px~OlPL$b*tfZ#>k|oNWKe<Xh3!Wyo^hpS
z-g2}>I#=0hnxI1@E5psG9I7~s%Qj#hMhnnTt}LH~Kc<v<$Fgb3WVhoPhQgrU`<RlH
zc2|I8+Qk)pX1a&QQlNqem8X4l?^%X2?80@bOB@kcZJ}MNqQ@WOVlf-Ojk-%@e75Mz
zeK89}gbh-WaBons|3M$#V8K|y((?{dTKUh*uR2W_IA}&A;r@9PL9P<rgd$^i<@*@T
zNnEy(J{*I**p3CaZ2@AYB|poWf8ewkH<6v%?JL98;C4|WdtOCk+;TfCCDw&37IcP3
zUvBC>;SchT<Z;Eq<V1OB?1Q^CM{=!>J)uz^W{H=t)<dF8m2@7J%=ca9Dto;m2x2Br
zmt&gUU_DBE&bbZ$t`KfU7LMDsGP)Z2gqVmBXM9p)Y2uO;ZA5=Xe^M(!EhTn>w;XD8
z{S@2BL$J;-wBsDi)Amd4e$X8Yh=4I3j>}cAQhUygKUz7O%qrM&ca9(dPaX6fm|2oj
ztpRa19V$!Cm4Fj@q#9iGE|+&}y3)#=+q{8wzxvGSD2X3*!FQFTrAC#NhfVt={zp!C
zX3JdvOkZ!)t>ipYZ$4~@<h0JIE~G_Q7M)1cps}h+bnK4ZJ!+<22QIg<%CsRs=-t3q
z%A#O1Vn$z)idfTxKS0g$!Fh!joSc#zLL5xs+4O5K4B;}@vJU1SS6y&--!q5%#mJ<6
z-xr<RK>{MeH>mt%!=cT4Zz{h=?_DqVNXz$)q3_iw^`+4xM!R=507DS4I^YuPPG((C
zrWeQ>q33po**tHNyYqBc_dib|eQ@qP^@Jn@Rz`+e-S(GhNZ-D^OkXjO@sKf`|4o~j
z8qa6@iajC4<Ze@Q<{m7N>5*vcn{jJf*MBUD4i^!s&(6;_N@1GEv>vU#vOKqQwpZI8
zRrM~#I6;qQT=fw|(S69`p4I%x$~W&|O(+U>#oWB$*5ED^C*AHIs5#?du58|x@$kx5
z5ze1CPl%AWF)Ze_1mh;%Vm&;|t#HiUG+HDPgDV8f6;@2r(#X*q?sxH85Z7U2bXm&+
zuR0tmQ}RHaQeWfroFM}8jpk<n#fyGVvye&du6{G~_v5$LNKo4eFLbi<nkZq*BJOt6
z7ZXp^8x;KHb&;8#ZmhhGYDqa}E<c41#sfI(3(GR=QA_p36x2vBZ0p%-n~U9IgT&Q4
zmLPpG@AFAwg$y&Xk6*P)$=5`g2C06Om`N<4hw4R5@J77Xs2jh#{5}A9xw^+QGbyux
zN!Z#gN`fX)D5;BFCiX+X-tH4{WOC#Ov~Ryiz3CmqkM;A#-FlvQWx18@6D86Aiq}V5
zX+B#a&+7v^F5hh9{$5khK8%8=B~FHwuo-&rW$~ocWwOh#t+!mE%_A^RBuT(-K_WLL
zZQvpqDsSaXB|KU%1OgBNG_?#*tXi~QxK_Q)D`mj$jn|B4iI9s4v|W4Y!4mKw!e$vD
z+$lp8oK#)sdnj#f@?d4<6Hk&p8F1dm5`oe0G6Q;AT!xC?`?vk_{U=kSJ%#+K#P=P9
zR0h8o=q#1iV&^d~lBO&8T;h5?Jnd1sN4CUEy?pr{ZSg=R>BDu)^t?0?JT@B(`vsWS
z*~hylz}93zPy=u*&H^ro<Voyo%;bkQ!bf9X-U~7<+E>jhURQ_FZ<Y6y+Az@7m1}u{
zGk?j?i=HBYp#I7%0utr$O}#sLf<G>`iOYN;8*Sxa_Mx>m+08ii-4W$0<i#m0TN~gS
zBqTn8Kll|1=^xPeZ$yp64}?Kr`|G#=XSDq{68{I<{^u?KM&ke7;|8BU1i*eR$vc*a
zbRaFczn-rWwBMBl?4%bi%>8-6|L0fxyR}_@cfOC1o-X=&-<$`AoA~`WJPs6XIK_2D
zcSIpq5JS)S=^6$_-plJYb9Lpzda)n?ianpd+uPsiYnWb-o4pTq!V2RhEf@Pbgf3=;
zF6$wAM~=JhVI=)K4GyOBDI0H)kQ%D0h)g&}>z!g)n^YFNcO`(NMR>pW+`l*W>~|L$
z^>>PN%P}2c<!>U$NVSD@45^t*r&qYopNo$bW4`nw6`mSgJ*6^x^yRvDpbLUd>3sn@
zp|~ZML%-bAh~3mO<v8uiBHDiK8Ow1%c16$G{(;aSl_80nqUr27Sk&`C<@X&7Yr1I(
zo5J%T{Ei?;H7l)EWOa4w5$1>MGZ+p=&fB*{147DLYPlw3N3M9}3ieA9Mq*+mg@q&2
z+B`<@?9B7eM)F^~2WceV*0iWr)+qPA=H{+9f%dkyarlenYb1*RCs~)EBAy6;9aw*a
zhN<tx{R1a+=woi7C8`Cu*|azM6?QmR1fA8+#7z_}$tJw@WMpH+&2dfkIQzY1bMt0@
z|M0;<AMbn-53$Ye({sp!H`s;6pZh;BALBJ(BaB{1D(ZR7K$M^sHkIFeJrW3vtX-=%
z?^5qr5T5kMmG!md^|gn<;H9aTyt5HEi-|FR_RQgVaMF39iu$(5ZS8Ge!djs^T3`T1
zf+56=5RN@HjYX@#b9>_e@@l&EmO1e_qnnvYuD02iLC-H3rg^=lrna?F<>Yh)yzM}$
zmAqUivA_rx+0)rRgY<$zr|Yv(Jh7#vnS@myl&T_GK~q_gXY2X-R=~%_m0T`a<F^nC
z^QiU2M@e+e7Lgx%U)|J@Ha>}_b!XB=3UT)a4J>0ON9Mg##J^mXh}C4DWX;(Q7rHbb
z3LaoohfmT*`3}p(=V|7z`u%|E!$cGh&lN({b}j~PE|bVHD3-!YP(wBzlfy;fRi#a8
zIUhHoqJ%3TMwL^zlauT{J>(evw^ic>vLT`5%)QW%Aq%OUX+kmd{L=AoVjAFpV*b*c
zTOI}b<;dx%dLaX&)Arb-=;$ZFc_X7(g0?TSi($3ax-^{+>P3$pJuq(pCrnq5o@`RN
zUhf*17Q^1mR^}Hp>!roh_4a)qvyd`2eMd|Et!Ou7*$tnVLCC>0{loL|K{ihqvA0-v
zhPYD=vCv=1o52cFy41&aTmFiHTMXKB0>Y(t2S{Slidv_luPJcR&Zj)gaR-8=Ze6f{
zCCv+Sv#|_XI=IJ7KHTaRy)E2w_e~09b4|Cg@gDkB`)eg%5lpGKVYP9Q2qQ>V5u&7K
zW6R2Y7Z>%6)3uDTNr+bJ*|R9J7aKaN+|l{eR8;QWIJ{!g+DSuj&QP3JQIgmsYwys9
zw&^oI_AcZ4Ql{eI;YTa#7H`)IDky+uptesiPE=N|uQ0&Z%=UxnNp{wpA2*e$I5`D)
z&eqmZG&Q#<#qHxAx=vhB_)?`>Zi)Gwi_mn_b2=7QX_k-O+!5$v-;~UMHz6YpMsa|d
z3kxkN{fd%K5;l*fNSG4{p@FyT(VD~FpOLx*^j*!`TDjMI?LnCEg2$9CY0LuUR8Bo%
zUAID<1Y}Rg?3F>K8%|E{V%@)dSo-j;?5uXJTC8!OIUJizw=b`u#g~`!>L)#VKu+mb
z$2?#u{ti@#Y=k^ZdWkQ&wRIwTGk7#C^Bgc_-npN*Cve*^pATuoii#zD1>Mr>pz9@9
za==PB<iZ|PqPD}dE#sIOrS5%G(Ct}FjA}wIf8n#V?NZdI-!W$n=PyH#suLq~bG1zR
zbAl;EV#(&gxcrP08446{<w8r!%2E<27?>>c;*tm)wrmT@=BW+oqim`#E*}>ZICgZD
zFD|ld=xt~Uk>iFuI$pax+24;$PIlQ+iBgk!>@URP!$5!uOf5;tUD>I`w9>lv<L;B9
z#aVEVs8hk?dyf!q4jR4zSO4gu@&If9o{740Tip_FVO~1y9_94!H6_u{CEz=LOqnx@
z0rw}#B}eR=5f!NIeyRNuv5fXRG~}gU<w)VsqCsF6`8F^uD}OTas%MQW*VQF<gh1n0
zo#TZjLvSJ^l^CM(PuvTNVezrKQ%%|;_QovkB<{T%Lzx-kvr|*w*9U6spNt$Xd*F%@
zu(Cj*@QqUywb-jkGw9x&n=#u*AmV=^Ki*d77byNTk=Ax2ZqqP1B02)<rV!&~iAG$h
z?k!_3X;^Fs4(2iCovn6>WTXxARuf1UU{I@?(gY&tpO8Iuo@nIXxZY1vR-Vqxf+YBR
zU7_>XJgq2g8`sCa8t~QvjxR0!$o`6wz}3S@C5}Lx!@@<TLS)jdr|?g9YhW$6Q#`J=
z&b-M|-j^~qKyZH(o?-LEbtON8OkG&O<^|=-`f|b!cc1nh?fe@Bh`$Z^dZacl;41KJ
z4Kq2}|0exbL^<~LjnzK2Bi{rilkCOyzHb_MiudI&&6f+09`fa7U)<haFO;!mcs;+m
z7SwMbI6Z<Q16IYtB1A~2XPmtqNwXvqmpM2+=(`a#3eQX8N45C;lZ~6(ewdmyqPqtM
zqo;o_F^?%T|7&)2eaK_qf58RiK8mNqJHuIZ%G+#qG_WJO^E1WH&!&%>Jf4dO)S4Je
z%!ke{0>OTRz0bR3i|3-|!7qcmjKb96s=IBx1#79>ItOM`84k`-m(3Nidn-mIqJAm(
zu-1j@4)Fgq^RfEPMI#pZzuv5~8wsfuA5riRBUq|P{{lFUQ+|*2p1pRQZvnNsl-<9}
zuOZ66XvBN)GRRDNEF+1;$A}D^{!94yXrau?$mWHf9rBL?f$l{B=6oS<#_#-XoOH|U
z9Q<l%X!1<=Rl`}*g76#<xx(;aDKzj))2DMp-1hMi7d%h@{8Y}xF5)%yN~+N01=5-l
z_Jv?ngI)0KGur+zGJBsl^+tg=dt~zCdaVF$`3Bp;#pP<R!E|kNT%R{cLjE%eLg4LP
zf^a~IlE%dCp!lP3q2a>xm$^AK*86immb*xXn6Y23-MHJzvu8y#p6l)I&9<UF9U%^#
z9jS4dXaY#j^uVSY`(J0f+<oxfY0y{+mO-ioSEpo1Z)2meo?ZnoCAf%<#C`Rc<ZUdu
z*i49tPi}8FLqk#5<QiMw6l<$*uE*`|rSf<<+kv)WZD##kU7c&_{=;QueeHqUdu3Lp
z4yb*~As(N`wj4(aRC#|dIojAvPTN3tH`qZ^T}oeM^2&J|mfx^FdP@2e3loZukz;39
z$C0o_9^53CQ~<n>>#bXPY3Z*H|F~#s0XL4e=4F$8VMv}(>PpkO1=Xtx9i8m*a(+VA
zYOfPTMEYAz$CQKwi!-0KL+S<>==CYo(LQnlca%vZ>`I>4vU~~|rJ@32V|{Vhb$a5v
zV|n@oS(6DdW96l!Gb7?;5s_z>SXk)T<|pPaHlJU@Msg*3%YJuph`MI(R>wm}msK}8
z=-cYOCJU+Iz3U)EczQiD8{kP|F$*#V%?@ia<ppf=B}zU85aq@vc%BC%e*HWcMCEsL
zBy*21O1)1_#{e*(s+z;d^qraR8$Ii|^^C^qGL`Gm`+Z+DPyz=U_82T>dwCaT7Hc`H
zMHTTgd$`s9@*^*AUKNOOb5%i9x8BXc00f%x$Bc+$W!x{b8Y|m3f#(+NSUPD<Eks6A
z@>+lX*S;+TU*>Rga~G7`T~wMk5wfzh1RCylQD~dNDumKggy;GL#=P&Q?Qd_=J3AL0
z)~{rfHBJCmB6`dBGvEB;JJDPSVcaC?2BUhk(xyQc`&^hV#DITG(7dd}qAPsgX;F-X
za-|b%H-uGfaB}_f*eYCfE+i+dp;#eTmR_rFSU=;%>p1L??~b}v1)0japG*-~zsB`T
zO>;T*y_?)SFhp5d!PuyGw}>ie+%>LPEc3JdX?5tKz^`Fu-3~`m*sDNU1x<Nmq!S#m
z5=H#e+v_UjJOV|0Zm?%-t8Z1GeQ@U2ufv!aTqnyno=)@4oA{EeYX#zB9F781l8OQ;
z>`Y8qMMblYg@c3N8-=(LwYl%M_tWG>_r_U`2FTqcB|YLQL){F{MwayAt6E!$E-x(v
z-Ge^uZcJd$n*!E3z4T`A(bl%piBZn*OC0IFjDu(B)Q*9UBa?qwqMuG-VS{SWq;jr&
ztWQbN2*;>1PXY7oq|jeI)N)&IH&(QbFKIQc#!LWxN>-3VXR+3|J?{zsJv=ZwNK1xE
z7lT@^F_vGl;Tj_?-7RdNotpXc>jODk+W;l=jn?iKur3t=bi@KhQqiL7EmQ1sLQt)g
zeV6qjxebp37Tk_2gcR^hgU9@Z?~I;2krj2X6k%i2&0<f~YrF6a((~pl9NhfgC)4Kl
z*uN{B_ana3js{|;uEUg2#OIBnd(6tAHA{<ol~r%Fn7+QIgv4vauq&+<#Cfu+mJm)=
z5ld^=r&&yv0&6$h=SqTA^WlsMET%5(r|GHYRMi$hx}b9&V@EH}@_5ju_I_4Z4>l`S
zPa1@bR#y{?iE+>c6LU~sH|!@yMjg`yV?6Hp4C3SyOUur!4kJsV#d|;LV(joO0WD9S
z``tU!&C^bPhv0cH^*KmHi@TG*CPSU6+@m<@>M`b&E$zL$0L_(q|C&0WNgwRKU2Ng=
zrou|wscW;TtxbC$;``NFzVPNi{CH!i4qq~qIEFEXL9pL(t~R5pS~)91z^k7A<x9lV
z>DM^Y@ja3C0x6TDW0tJCd;=D{8<ylE9`{2%^=1anzYTQGej`;4&wxeOVd80O75ZZQ
z`2^{PV30)!=B}Vc5>~a^ZrWY4_mYITp&^T;Vo*>W&;XH>kIlX(?*Rh@dYOxRGxS|S
zo#y;x`S!)H&)ES_dh{T<uK-hQnw{|dq@Yj~8$FZwo|&bYd$$)%RtB^XrU~!_rV+l^
zn#jK>8;?GGk>_nYA(5}tgm|iz6n&&S!SDxn1P4z-yf#+Dg~tkWI%-`?+uD682-mHv
zM~8Ek0AF3AZ!^A~-zR&zGud%+#P9II-kwW<Oj1#zG*7k}`4=S1sy(1?cZOa)|9ebK
zeU6OHY?rweYvtv!m%eBDQYV&`a@1l@90Omq4|Hgf>DQe34y>97r$`^NFsWm`Yv<yH
z#`t-<I?)$x8wdouyCb88y|ige)%&i@XFxY?7Wgzr{jBZ|Q(PN10F8>=x|XUIGOWRl
zDgA^_qX&*wpZCFvgy(22xL3D&9h{sP=*?@K<1A=foFv9dv1iJ~n?H%Cf9&3Fg|A#3
zc4kv6YQ+E0#3SG1u+QH39rP|~968o{Q>C5nPd5RXl<hY+H4s#lp}oyNCpFa^_+S#o
z7O2#iJ^*H!lYRYJJ2s50hU3N0V(mh4an9rvn)33=6SSohv!#d#%?CczuU}v9OeT@&
zHM-jM#j`tC$}y;RcC4a5j09W-;`(`?{e9rFsOo&q*z4tKJU3_F;5=GZ(`qPF^!I!?
zGu0+R2OY~YNlBoWU)D~qto+QAW89h*l%tELrd|ObO8R<rfy&Q|B+<L-1d>o@RTZty
z=^QM}sx08c(GPqiG6og#Y2vmYJ_wS{wf`Y<dEP=o^@`)t^S!imXTyH)zGrGo=1<n<
z&oB7c;3s40<ot5)!ul?6J&vzwnlE$dDJjR-oPKH5l${iJT>ou9KmnB85nK3VP(c*o
z$FuNl@NOfj!D4G9GfQfFANpx46=hLQK@)rwC48ZShg6?Nb#qPzYO$M5v2yU^aU8&<
zyunf2h$0u2tH1ftP;B$ab@U;UF;Cbe&&@caBkR-4+l^=B3U~ysh3sCvVplv_@OvA#
zkNy{Snq%e@xUs&+Oi2G-<@Ns;F6h52^qylQ*CQdh<Al?)dokc20Z|$-d*Nf*G)+ij
zgDoNc^YA&;r1FTQLbr>>uZhI7BsLGNCU8D;x{@;2YfKqAmn5`4K1#nlf_CaQ;BRgg
zoH_SSC(XCCK)>PcU7z{$9Z5dFr=D<%V+%^gRMofpQ?2M(vGpLG6kZQzUkhp@4%#09
zv$ZR1VYCp}!*>2^2NDn+jkjiQRoq;y=6zWaY1BWZd=KabjOI#7o~XFhIj_rK0ud14
zAL^;iIv(QLMkM5u^OmHS$lvWXAoM(k_HQxC;YO{2*x0pJ(-!EI`_0ofx1Rdy>Uk9$
zy9%OJo)@2*eDziAMDJfrB#n@Kufj-*jZ%VToV~P^OmvJ(cVgnRMqR}#4vuLUGps|B
zm?#0!hPkR<${-k<O_9bA7cy6hD7C4FK)8g2j$yoop`F^?<0Bif%#7H7fSIA))pw$f
zx;kQ0=IiE(@vrT@-5*77Y~Sjp+#;%%B(ncY3ve!;;8$g|v$_s?t-x}0ApI<8xHAkU
z>Q*sSi6{8(9!<FumXr*~$L|1$^9gFOM^Qyhm&nec4de^VbeYud;iE^y0PW5%(&T>V
zFE$6HyE3ugSoOm=n6s3KYOITc!Us-IWF4GhfDm#A<CP;A2oZep_{=L|Y#=GM;%0@`
zgAI%{NY%II@dS0N6KfVgo98=<Pl%RcBXAKdGUV*U%f(iFV9!M$O&EKPA}gcSkYfXl
zO+Vd6c}%IQTL1g^Oz(%3O)YHfypfTgK(=SO7vjatXL^y7g9ToRXzP53Xt{mP<K0|Z
z#m2}8@9bJ18{clQJ&_9SZLnXhcHBK3_vNvEvVvF0Gu@78G@AJCo%G0(hh+U5)wd96
zOpgi*YHeTgrm(e6X&}WJ7vtby`W-wCqoQJjhqb_k9B=2kzW8>N9!ih&g<PzgJH*@C
zdid@pFM+;&U53e8TGZJ&7RP?K&0o#eM~MN8Bf(<4BoUX&UH}-yzyQ-^et3}*5#XLd
zV=99(8xICVv_EuLIc-}<M^}QXI_m{yeNZsON&s%E+eCi{)XLt@`toJU4^U~NeY+O&
z)gk9VG>$zpKR>IWcJU}N1Txyw<2JMWBSj6==$Ie+1B8p4Kx#B<>S0Ol+O^8F3{(n}
zV4eI+=-@lu&1EX1k{Vw$GpkuGujPPP`Z7;9ezMR!1>UM^Vikt=0nr-S1f>af@SYI$
z#JP!<C{QR$U#G2S67b|ZnKZi|G`a9h)e~)-l2BApaRxVc!lNk2c8OnXBu7&xazh1$
zW^#<)UmcdVybB}Sep_l~&hF~g6MyCVl>^`l$Viqqdu+^!tR4Pp)qY{=)j+c5&Q9O;
zdCPS;#XM9i{t>F_$jFAhy&)>LbI$nM%4b8v3jWSdQXg{c00>>$Y*FxYaLbX`<}nVs
ztH3NeOSuI^*trif^*;yD_h;$2_r)hfs&Md;J#*9V+1bsJ{M-AS8wZaxSy3~;!IPhX
z{{3%q(LdwJQA`2&i4)J^UyaU6`1vJcyK~I*b;=^HWgvO^C%T<IK`)-v=;>pjet7wV
zgG8?`Q>U?(iE4RFO6CFb{$R$SS?Xa`JY?&X>cW#8B1N00{U|vl$6eT@JYVOmqnUt|
z=cf50)2=@mPb4jw5BD&x<)r#W+5?{vgLKDw+y||&g`rKlj2|FUXD)B>CT-e?b_iF}
zpEj)&{jZVx$VI%|a;ti`U1|lSZ@j!Ds95%}f$&eLT7_hbJAFpsX@b~)W>@&W0rD5|
zRUB%%?n}VL(J9AEYvvo}`Ej!%=wfR2CR#i%tF@7$A|*CAhhk#H2hzfU5MemUQLd+8
zHSrO^8GSiXb6NDFqR5olwS2LeOR(G8ok9pRG8(&4z<FL;z~<=);9#V}c{G@)2r$tp
z{rz|Q@Zep}-*5Ak<@|mab+t`TEzvPFS8<mp|7k`U(H#%wge6ZuZs5|^R;6Xa#YFi*
zU*BgjY4c4O&e1s`j@a<9iVHa>kN8(F{oH%%90^TKF}vVHjh)eR%HPx1YvG3gP5qcs
zyIKWk`H00b;1lTSb8@w5XxGVvG{%PwrDYwxmxAV=(YoV`S%6NRxeShsS;eG3e8`v0
z9p28b<}uPrv^clE?2f3@IUDt7_Oa$M3mXWe2p|eo`S77<vE7LbbFc=BNlPU*sc>jJ
ze?|wEDQV(@7kHxZ@SB9EnVFuV;`P@@IVl!pu(O%aiyF;eual|53^oRnVM&DpX*R{h
zqd<fJfLUQ_B08ln>TKhH@f4TMy@#X$28=Q0U1;aUw2c)&Ce%=wS{lyBnIrCtK<;=@
zL=k8l)pB|)4yVG+;WSrHRn1<fumzRPtzx`p{^`ruJuJ2)BsN{Txb1i0{jBunEm2;Y
z+L&QP;M~K5QtwgWa~&c|<MTZ*quNCR0Uf=0c=&*oCD#NY2^lO&-F`(H#lwo}ejri=
zY!hJ@>#kqRxbD52zH2x+RvKqlpyQeg%KtXkf0ftP@nex~RU1hiB-x-2$?#aB0}K}(
zlPn$1f&2NI=$S?-JkQ3%t-+DrVklDtaoxZtc&wx28WB-asD4esk7wOAhd`v1>V~*q
zu73GRr%o7`RK=|Ms@ah~7}{K4Ml(o7z_R18@P>}kc(pgUSQ{j(?jP8Yo%>EMd4mRz
zzpjp-h9*elPH0Jf>7eOcY;E9tcQj;wsf|QrrU_za2oZLny0s!Gdki$<V!?M1z}moM
zaB^|6w|^xf@>Pmr_jI=gWy3E9XK9(L)}F=!h#IwgE&!1Q&=bi!c3Udmf`*;1??X{I
zfKP$V*70J#7-`T*!}>|>-2yEf7yQNHVMI89(GKt)eK|m>O{1a!IL-T0|E=iuOL4)2
zbHnP(^{H$fZGSDfB{gu$ukFwkk%>o0@&dIjfvA9YnYkO6I%{&=t@-RG6tp=%JMri!
zfF(LEFF^!R4@C2j5H3o}^Oc^?>x<B+j)k46*LE`ndHv{NFpxD*k&Y%VdFtj4)9GOZ
zVwH8=`=<O<q~U<Wdv3169A3hC*WLS;MNRhD{X7<ytSNivqJK|;>3+{#N)SB>pkidQ
z^xnw$Dn-7j`Vj{{j924g(|x!ciY;yYichmIQMCxJD2m{c&I*DKgr^9OuZQv$>_ZLM
zh?kF#jk2<I(>yCC*tIo2;%qOqX;)-FxrbDP*S9EW7Jte~<~lkhvbmXYH8skZ1ku3~
z=E%rIg3jG?&?GB)Q9g%UejXkvy66CZ6LJxqt|E>UjivvM@`<+wv3=YgQ@y&(X=uoU
zrRvU1S(K^ekx6xd<2yQb0Rq9zJ-4p@1FbBb3f;-hs*`}lXaR|#U42c>&Th>r@b^B8
zokdG!ZI$NuD3V!ECwJ&Yg6iFkS!j!Qy>xW7?I&{&!BJJRrI$Qb1zHU}3zzF1&CL%0
z(o-pNW2FcEOl0wpdcR4X1UF)J=~ZYbtj@K3H!FM#4XZa3jgC_ph7Xx=B3Q0A`ldzl
z0W!E7=~MQ?=!`pmS&hjDM!raTosgrd^RzcH&D(mmS(r~~#(!WvsN-t_XOs7(2@W)0
z3ccJFe=PIc>i6w4OxDBWXO=TB;!}!wNQjd{E7jf<^Yo{AA7>&4yvGKg-yViv0x2om
zbZi_5h?R*+R{B{tHxKt4;rlan1i*+cy!lsxD31`}r8Kn*S?P%)Q$;TF(zGnqj!xh?
zNxW#))t$PvIBo!jz1h!Vfdu9JO$!}X=jQ?wjv6mUML>gvVry-!(%CAfN}%C}E$`oG
zps!6gfH#_)+^Vc>rJ)hGo*AsDna{-{DF7-2{{H2U9<c*u$xxEQcvV7{N)yvgpez*u
zJuPk4a^e-pMnhZ}Z5L_IrR+xC-PWw$T)qtrZC~#{P2elPC(FHW>^>^P4~RG$0jA4n
zUjTIc-juSJBRV%@S`}r8y>kbo)K#qUAW685|BO?nu$DBM+o;o(Nl(XlGg7IT$JsLt
zemDYgzVBU{`g(n<V(|7@qGFa%ZQ6sKM?BU=7S=q`u}^#YzBjv;hw9aWy%x4h)k*Hg
zQ%YO(nhgEZ;t12U4PgzJpWS=yF_6*&aqL6gKI)fdQ831!`n(Yn-J9lZD*Grd+CQM4
z6kDUA&;lm%IO}F)m|`w5NmWy`5G=vMf^|cc;o%^QN`P!-a;hMW*re!c=gLtb{%uI)
zak;SY{l>FuGGJ`V26F6s&wW`t*#o?TEOPEKz=HuCmb2qGDuAUDNEupUh!x2%Y~X%n
zs*r$OG3GS3G-g>>mBEEMD;Ol3#BQ!1)mLr!*fPmueRzPWc**|b7#Gi$zT4_MS=s8s
zLN01*qmJOC090XUzL}NFUtO}>2D)0`O^J=kSpsVsunyMPT%B(7#t{-&#2-C=TqhMa
z!fEEGWWHeUq(tX<ydEcXD1j@fc&~tvT3mKGZh7C(Ah>zTD=Jf*KNJ{<x!Hw_%fiD5
zWLw`H*0ln#2X^O0x2J8Ail1X**zeVrRVWcYkCF3se1Y>L<Ku>aYTjlg5tP}N0Q<OR
ze(Lrh&=aUAlPkQ!%NZTroP?^YZws6SRGm#z$8A%+vUaM_0fY8s*MaANEOvHq0#~;g
z(_5y8@<k3$oBn5~kd{BmG2fVbZ=8sl*YN^U`rcII!Svc(-O1nf(d2PF?_Ox5rdIjk
z`FT4wO3a7lzJv~@yws{$&5&2I8(TcGFSxj#=+_P3*?1Q&fHBGle*$d_nA0IAcx~KU
zs!wrnwLOZ8aykW4Wo_9}z-HV&De&uxdh+7O=$NVJ9Rg>D1PLjO<>i}?O7_Pnd~3**
z5-%6>Rd_|s3QAj?*yW^gypCaCrlvl5oSkqY(qlt_+64)307sntTECBAepFThXGk3D
zd0`PLK;HB6V&~CK%Q3T9uTOxJZV)z3_I7p+X=z_HJD6CRN3X6i>x;$fyK3!H^*x6s
zCPfyz4aYS@w66my1ku`>H?}MdToejxmk*^?qPGS2Pa149Vpd1CcM_b|herJyn=S!N
zadVbb0m(yx0yu9L1Dty^lHUYmYJjXEj^#ccuf?kZFChxB_4D&Y_inwO-@#L@xZr2;
zC?oz=45JO~;S-XzYKJ_X91fts9?QbaORpapl?A}n;8%T?Qu}i^H=dy|FJ@f^y~f&|
zt5fHu)ZRXofmGc-C0h0-=vG+%Xmlat!02dxc6N%)5xaNuKEOWD)Mi*&a5SDMJ@UA=
z;&B>dN{eA+#K&k7%g`^+G(5AUFnvCISygovr&ka!INsI#lwB7fwZbPV>Hme)`mi<)
zXYy)`=qP~@)ckQCHECon>>RtG$Vs&nteyU)2?h`hGN$f*>(IKM#@o9p;OhhA4$Vg&
zR(~qKid8O!m=VNnY*j<@>n6(_%;b!-G0&k7JbVB^aeI4C0+eCv1)4U=`h>^<S%7WV
z)5`!XAF&Kw<}L)_jbENav~)_;NK(P|zhQD)>$KPFqM-OyrF;$+7O9s%`~pmO=1UD?
zDFoFhh!SXUq=6&`kS633T2CsiE>c$2vHJQ<)mI5{akWos;*lhI-MFI2q!6;|k#d?o
zf#L#s;yc8C=eKXNBKmlEk%xN?_DQQ%TFb%DS4BiPZ1yX48!RTKZ6Ns!F`pLOloSTI
z6b35&F=bR4P?Qa>&n!em6B-&EZ*TCbrVOM4KjKT?6JL4%f%t-nd}>imY}zjBDAxQF
z<z}6DI3{hXc|0ucolXVO1gXuz?hZfnFumw>G+eY=Hc4$|H>Go3=BqC!0Sgo09V(LD
z$DGZWgDu~U&x$0;w!fi{45QHhHAB$1TJcNoO+Hs&lDe#Z<-8|kYVvYS_ZqNgHd}+-
zO}Uknh99p0hf~%r4lrSG1<~0CIV&sfZxX|35b^qB*o#-s%np5pzSY%j9URE60ZE*s
zRQTAmS9T+icPoG^(9%R7yg6@hxx<dCn(wvyMO87O;_HKlo;^EAi7qPf`l@%R|8>{n
zfc~dGNQ#(9=i50%QpTYl=|Y3-u2-VUaH(c+!7lX7wm%Jic{hpaJFO!(ABP!2V=Y2E
z`J62F&*!mE4{QOv^669CGul^p&m_;*EB;8tfE5;j^H}$A@Sh74{x76b|76_$J9_N@
zybj|2ac{=Wfys2L2HO~No15b>Gs{oYzq&Z|-*3?mN%d(6?ut5ajRYe&R2huARyQbW
z!@Vv|mgCtk$;r)6Xk1P<8VU92Tz)F28<DvFy<h_s8_7LbrNDhDXd9?f36^C3bd{2K
zA#L>kWry>avb2ey)?66}m5>}K&1>vOq*9xGkCWg1^?#&Q`PFKoc7{WtBvSxsa<dxu
z*2>C`81}8LZ9+Uy3=qByiVg}p@BC0#`8<Su@MWU9H~{AeV9pYg0&8l*>FB-*+rFfx
zf4}lEH2bG3B>Ubgon-4atk~D>MM$|L)<u19PiPhXAbDPcFHnbp)Aw?r#=$XhAVMa(
z-DzHNx@go#CPK%u=9&p(C%b&%wET9N>;V5ds=BqyoW!10TdCr)rShyvf=T{vdY)lw
znjU@xNQ(fDiWM)%%{5ytQ3XKd?1g2^Nnq2AO%zDmf09o5Ct_2S$Vhje9!O`U@$e7>
z%>TFVea;Rt4t5bhg{0*`>~gg5gM>7%>lQslOmD5T6^#th*{9HN-GDW1Zdw2fMU1e3
z+UP_%>Xo^BePUOXgoN>s0{htDG6n{h-w(hKyPqy$`ZxNZ=*BYP>Nk^`sO<nn@%K@|
zo?h^jl$^!I1e<0}E3Uer0I132nn~=k)5Tg0!8t0@8|>hqp8lSpn=24-do?pvXm2G@
z`M~g>bIeFfTi6<H&BzdgKN*fLElqE}t(MaR^8@~(nQ4_KQv3Ol>+jFsd7+v;cOpNx
zG*Frv3qHif$7^fmbLy@xE=DI3b36Vdfh<;-EFrEzay4Y4%?AKB4dKK>k8B7e59{sC
z02q5KtIm-TX?N%O9r5=`?e(7(CxPr54Lg8EYa|ed!$z+sf(i>LUy}0U$)oc-{<J<^
zs*nu1A{Bb^0lig2JUryz$+t>Z3&%<mnZc`59~XN~o~Ax~dIR^K`{g<(*^zk6l++?~
zDLH+hQdN5Vs}<w$ot+oO&Jxbfvi22$Vj14MRSa;$EUA#ABSS!Q1rWnHVq4p%S?_{?
z11bfzE_0hw==|=Lt(Bhk6;f(yT*{xkn!xZ}?dJmghZ>2$vk6b~)cfG-`o#rBBgNW;
zAC)*H3dt1Jd3S#K{HW$xu6L?|Eg}xYp_34lm|i~9X&@P(@zbfrF=zYD7-Pqwc~YX=
zO7*UmO053v#~;T)2=V&?#MY0|{ttKpoo4s|&7!cn1y>}*OuMo1!fc@W#S?sp=S;1h
z<EQW3C$+$|@$!5K$ZWK0#=euN<B+8w#<aOj_j}PPGcwvZ%}@(e?69!f$BPd0=D#>?
zRrG-qR+Sd2Wr>f0Vx#ZOEPTWw>%;T)HPcgOM+1X*a-l4=w5A6J>hAC5!(P9BEyy-l
zS71SROd#}sQe7>MX*-``T;5ZEnajUXu3hlI{>A8Fx5A_5hdGibw~kinAlhA-nH9&!
z+^MNI8=uDk0F+HV1Y`k#n&{h)hF~Iz;@IF&7z>l--9}a!P|$**j<1z5;9q_DazZIS
zGYrr2zAm=vi@!+`^i?lye(U{(%~C_&G*^p-;xz##+t|WOVCX2o8ZBnwnT<NvErsRq
zd}RcMS0$l?MLhVy*s$TjQ6xLzyRf)bLDnz##V7p5MKQ5&5}Ahd%IbNveoshUw<S}+
zI@%u7T}tfi)+Z>~;YsS%6B9{sVm@D_(AnqC9NzITWLu}%X3_Rpa)9u%szwd299N`|
z-&ci;J|z(tDXto|T@5}O$<KA#W&kq9+l?%D3F&hj47a;;Zv;9ekk3|E5eW(wfx%xV
zG-`G%LqC|@++Gt<P_%}yUR_?G%cohZ<$s@?OaxlI;rHAK>uKA#dYOBbHyGU4>_rJN
zyLNRuRApEODsj0c@4^WZn#DPr@a!9z=G~B$@5LaratXsp_heDk(B~@IG?52l(w{yF
zLj1t8tSwIb+qS}hsO>fX&&#Q>=Ny{;QP0?kgOuC~<>FW!u7bY<2opz1hZH))Y2bYP
zrYm45^8bSh3_{=J0r)hz)E!mbVh^B@D;RKZuD(8z(kcL1EPC`U?_;;ePPyK^*Emy5
z0Z<~SV@CoS(=j?~KFM)dZnT(?Fy!}Rc-R@F&Oknp3gv*B#2O*rXR>sFoag-b+0A(i
zbGV3>?X72_k~tvtSz0;^7b&3Ne4WN}JYm#?P8mWFv$|$+;?BUU4so?3g3MUoA6c1E
zTf8{rWqk6K08oKj=386y{2c8BvJxew4=^4=3^zZVgq>=lrJg9@7647ik?^53572^{
zn-|PBtWrs>Rq(#%=igswt&b%TH_W_#rf`zVHx%O4jJy{x+i+c1d*HmQ6wkcQQRJxR
z4hY_4a}(KGLvT1giIo;Ho%Te9u6Xk{8Wy#_eL|K90T{c><Kn6+cV8b13l7gy)n}B(
zgD=&?xNRGmAUdY}1Z@65sW?4Qw;>mZsF?)}nCF<o#~L{~Q+|FrS6Os$m&=wr5k98A
zo9&#tBs>Fc!^7)>xjD~#uEY}C<24J=Jc_{<sfbYEr4^NFolXgfus}<8_Uu%^7&w30
zFzU6!+11%JGBCIDmThmADPnAzRE>DXcmiW^hNmnJ4&S>tujP++78GRo`1IS?v9PhV
zcXYfj%_Alb7@{Z!8e#XUB`Zs+eZB9n+&PCsxBIZ7&pu&Q0|}tJzMv5C4+}FguEBV(
zU%w6pBGU1tP`SpzMpn~PQuGW=KmW(xcU}i@g=^G@e3%a#zaXtg<Lj^Wey$l14VV6a
z2k>}x4T5s<3r>fI_{xg`SjYe|;qU+W(E~jobpUjIPFIy~BIRBlY{F!q-ud~<sj8oG
z`cJ<F+dJ7U0;BMXN;eJ&e)aerbMb&YjevOz-qI2RIffOhm7e8!(?70<0};XNT40*J
z2yi!k4_&IJ_$=eK4bnNAMu8e2aVR^x;&1Ar7a-e*6rMN&9@PWsZ%g*}<mKgmcS+qh
zqZ#3h1gu?Z{kzc%)!9UV+dMGnXJ1%nLq>An;sA9yX?(E;)C^5Oh2uTepVpoFK0|r|
zsvUSWzt$`;`U0@#gI`TzMybznkiK;9PePgQm(~0L_&2^Qld-UEwq`)B4s2mh0Vo4(
z_wa8+@4Ppza8DA@CI<K!X9nWhBh7lt|52}%j1}e2G}KU7qrW@C7(@WI$1m+oo%0Q>
zH@esDEG~i$0Il&K=)f4Ss6JV4MELvt@SE>lQNMp1D~}Y@;7bB>#9a}t28bKE!qLUD
ze=f5d66Zpr5bR`g+{+8DtCI~70Vgd@PJU5wT~YCfS&DrJ0Gwey93~T75uWIm#}WXz
z%r9soMVcJ~WT$WiU#`U3o12L`2d_ETtVFM8p`9Cmu0Bnm6{xTP>XC#zeeYRnywg6$
ztnhGyd65njl2D*JfGw<Y&`s5ZfnlhnwRE<J#Sd;AvP+awJk0eze;Y^=dM^=k*?%fx
zn?Clrkr$QhJ{2tid_lf)h>$Q`M>pr$YkUTL_cgM+z1brG%BCb!`~5g`wrKsT6hX)Q
z@}+MA=L@!%jDp&7qPG{z?pJS}tKENnVcO#+c`^bdyI}C<-kv$2Bd@6@=BrNbiX!Fv
zNJ0=xkN;tL$=-zHIj7<$0R^LCFvL`&2sX7wc5Vm*N?_LW<fP<#Vs|Ywf3tZ}N&iAL
z!Yp{EWd_d9Qd|L+8>o@S=S|m)7wMAx5~>=b4xyBiUCS&Y;|M`qi=XWk!=P|S9g}jR
z9WU^jtb?Ux*he}*M<?W28)iNydZ)R(pT>fh!>pq*H77qQG{h9U{-TCp+5iU7Z$KU|
zbk{gBuD-C;Ci4L=MTqFrSfQzcVCrj~wxH#@o|25WtOR+yJjAkb7uWaSd$uhpTa|)#
z{~~{L+0ldNkxiQ);tl!xT3?AIOyUBAYpbK2i(jm>S^Duw6f{usU44m`YHLxwy(E3i
zQy${w$CM<zYGY$tG9Lo6^RRC>^9bsHT^u&C=zON~1M+yU!w!tAnB|+x>Mn}?j;6JL
zV-{)7JMREhorQ&p(ei;g+?`cVNB;J_wR}?uO9qh(E7M4-f(h<*BmxRgK%U#+_&#4P
zNdaQ{Z{j->HOQo%dt=Qmk5_$-SjP2k4Nra4_V%eaqL3FdrWM5$K-x7}mFr@2>w^Ng
zd>R2Vlj>eW&WY=r%{&=io2Qmmp$)c_M@J?!%A@|6RtX900H3*Hk0>lvo3t$MI3ish
z(;{KII_w$)v<HHMo~C(^yKI;2tt*ZXafL3k&+ip%k$|3Nt^f4!Y;ArKEI4ZJVt98u
zwF(6yJ~TGye7CTQA7(iw9}@Sifqf%a@D-3vDI*~X`~3hB+zG_0Fkl&$f1fO<P=Xe1
z{V!?}p5%>iS}xKYrw^X0F4kg-B@<AQQ5@+@7;m}E4f23xWDvQzSzOu4g^taO&aN%8
zVFMC3S;PaM-$h6s2vb;@8VPhwN1&x#@vfJV7oZq)b#o{wi5S(QdAT+=Ro}kdeDtWM
zzS2e?lRHUdFJf*l@*s<D3vmw0ldn5ow)-zF0COgt#m&W6K-(yX4ZH8n<f>F&VdJC-
zh^bshy%TRAftDqzN80&zT)A0zV`k>8C95ptFAkGOfh%N{zq7c5M}rDS8W8h&tUC%p
z`{bXoEdD;6F>aAn#E(fTeEPU2eYyxv1e6SVd#Cn20So17&}}Kf4KcB(^2wyDntDk9
z4G!Vv@tAh3tCEq3!>->9`y-`YFJi(}R@UwcgFOq%KG}X4Zg#oVvo3RTl|(3#(l1>K
z@CAN;#vk#yjit|A=DC4_pd$WryvU(3(*-@qo0sff8HT}jJ(_h)HEJzYFj%Vn%*^Yo
zh+deg=()}&RXD!Jj#5ZzX>Pbjc1;7r4WObmFt{4ZI`#3{Bo2*`!Pb7l&yoS+w4a&|
zJv`(Gl-d@M`#F4SF&$0s-`rdQ5HxEkD3s$&PZ^;|noI-i2Tawi#NKx&1q@bZ4r;N6
zDZ+IV4r(D#poAd=ZuH6gY1Fl~-p{-jiEu7}?^NIG=Lnij<s|c_%y;pIR(-AOsqPU{
zEkEd=5;dFMEpq{&f|LGf?Dlftl2)FJ`t_O;eoTBkkk2lQW3RdttO33e;OM99_()5y
z$F|nvA@(!LRDc{3|3iEo7ToQ3`b(1IBH`g_-4|d0CNSejS6daXO|*}R^_y<k^W#=P
zMgdhoRpo{ik8jREf(sRZ@*X|Rc1h-CW*T90Fk(V6F89oPFi?aw)|aAv(=TWbg4lh?
z@d4|o=Pk!WoX=J&sLT+Hfonrq4)npq+zjdrs?LjGe&<9lKvo_ZsjZyR(NUAD%YRl4
z46!;^{iYBjgzA3DWtdEHu8yu-Nr^lS%`PBg)nvS{&K9UX2XK|YSd0B?vGyUkcjGv4
z1XzlIBbMskYj$~raiu1@$Ibe0?4#w;=tGZYk@*H*QBnQ&9573fm31{Cs2FjahADE_
zsK_!-^h8C{|13tGK`In)sL&L4q*+%XmN5vhp3fN>Ob5=&-I#!G%7qpSuPw!>KLE<~
zx%172Hj!07mhtL&rU}S)>mBniItUpp?|W*!kG`j8Xu}a-61VAg46DKs0s@@^E63Pf
zO2T1dT7Xel7gx0!Vp~q(S6XSqT78|cu&&OWx3*MQqZH<foF~|7QUq5SV#+I&Y8=fk
z$x_Kk2jp1DJ&M$2LhSbS`~%TMS^Aq~&ZhI_A^Z4z;6`@(_xJ!~9qSj^@ST)iz%DZ)
zrv~=NepG#}?WFE0s;(3>#J{N2ikp_z3$Xxg^3vMt)8i<hH@N$zD!Fpryi}wE-o0m4
zz~{}YsC)rcNb4Iw8bU^#xVXa<6bj1~Ma!n!0(6T;o!`riQk|f&4A|K8$0J$@(sj+^
zzq|IZE7lwy+}H%a@O*D;+X+bhxVd@P0|7b7KUIGLVa)8)GUw+!l8SyfN#TMdU>)}f
zt?tPux=9i3vdW<p`UVgpCNJ=ghL^E%?3<TCK|LZK)d`c(v)xfJn8U?o^X8J~q>=dg
z&t$V^uJ$qEa<CtP=RQB(OL>j|0T0i<<zmUS>Yp_10^h*YWE7Ms71pcKuy{L|p+%bq
z0v6lMs9x_{=1tmRd?!sfMx3vuO?e;7DT|4*$k-%R!KY|43sGm6?)?yw<J8|2Wk%KK
zyCx^5lqU1v%}KK4$wR;C2VuMG|E1MK2ef+g!dEunA_Q@m5qW>psMg+Y3t{`d9tGyf
zFN7V2ql1mhVmc-c7Lx5<4i#TATJiDzCizBnEI`xA?-oK<aYMh|G)z75M8gE&Vh@JW
zdt1l9nZUmg<bPb+`8V<SPjc~}1mb^`^W8TM*#51keSh#DkE``YAAE!~xXi#%+2Fzm
zzR0MnR4Ifj-rn}x*;y;+oq_l2L^eokJzRn6wlW~eLPpUsgB{&o&m&S&+|KuK6;*^#
z2`BEk#}eJ(6Gv4K!PQ;Z{`|bjZ}fAoYcW#(r~mc;V(#h3UO?Z$X^LM{`9h$iO@`Ic
ztiRWF@zLR26Xm(YGOQnWrRZ0OSsHhCcJxe>jJ4VF4<onx(u{C^kt{&$b}%ys09Zr6
zi8x2CD<Akz1e5F@=K<8Og8}jp>p!g4v{j(M?8Kh{l;7g$Zs)UL7_5DIS_&WkH#r|p
zWHhz?0pFhmpyuN&ZlJ<NR_{&3s5;^qo&?~ofX~oS32WoaW_z06%9X=qfYk#kMz!@H
zj0`JkY)aW*(5(pz{9mw;xz&KcvBG5a)29Tn3>7B%H1;e$UiJ1OK*({A_lXjA*J9Uz
z67}~!@S^`wp?b`V((4y#1DH<<ny)NgQc7fH$f}o1VNy!ua9Gc3J)=n5-b`5AVjvUb
zxl6K1l=Y3mvA?j@1y+d$N)dC`fgoCE@w~<JJM_xpfT*$c>u|Fdr=OVl$B&(Wuq-yl
zJfX^Qk@J7B_timhciWaE`AC2eEVu@D3l=<q;1=8=K+wiDG!_98NN^g0OK@l)xRc=S
zZjDQ0!5!w1d+&SizPU5?X6jYVKSOmD-OzpdC;RNP_u6Z(V;Rq=#Ig@MU)*B?KUp@M
zS(FcccC6|rz_&u9c3C|=M7Dxl``&{ZHbCtFq#^|luiuWCUL0xm+~1-i_uq<iKA;+j
z+rhmBB~^lhyZZXa4FHgNz<!;h>Zx5g6T{bV4IO9;dEe5-Sl7_-;T&4O5ef3YiH3Sk
zYC!(^2lBZ2Z^$F{o6FNCkms{0Q@-xlVa!UACB@WmE4N29OaZcGWR$3yI_<_&{_mM-
zFK)8M3Er6JvP(a#|6<<Va&a;qUs$4RXK6O)>T$SK^3=}Mv{1rnT6cbpu?p(uXq6ML
z)5r&)|66(?F6GIOsw#sxMLqyA_r%RDwH-lEO;k$n@_+oKFfklv7g*E!TFg1aWUel<
z<>jDV6w%*fy_Qi;`3^o<*H53v@UgcWYGl;oG&D~)02)ZUJO%jfAe4iTRXL=5FnVgE
zVQ-YIm-)2rAT%)?0&2moXO=1K^vhEq+LD)_D%UeJOxpQ*2bFr9L76S%;WJ?cv}*SS
ziTq#BnPK<i6w4vny`OOffe~O*Fk1NeoOf^-+L(1se2rP@?%uEx@u|rzA5B#w@x87#
z2OR^$hKIW;p2xPOpx$f5?$yJ?r!PWtz6V^Xzkz~sUw5At;2eV96lZ5jKvnMN<Yp%)
zjlWGG<jt+D28wjmUvR##ok9GaJZS3sd%ggWwH;;#o}${`$Br#tqeN1C)XnF+|5wIi
zXC(&pK?M+pm;$96Xk@Kdyq=vc<jdorL?K^|mO6E9mXHK~O`ju8b*TWY?Ao)voaaA>
zH>9~%c%G(z!gQkbFDoYz6&*&S2FPi#iM>BgeKH<KrU)fB@gh1kXnh_XtqBd;J6x_T
z!?d*U6BRuJ6A)k<a&qY)bZ>4p<T5&05rMRmoFja{kn?uHd~JC2K&c8e&W?$(rg{uI
z!Fz2c&5|8Yv9$gj51mhF2Zc?r*kED07Uq$5;a?w(cl<CDUBAoDexH+*B&C0NQ|5`t
zj>FN>IktEKqEtrJsPGCgI`I!;p~7!RbT1(+U>fDG=jy7VfT*~ZLbQvZl0ciRfZqpb
z%5xQrhmY|jZtoLkg{|&cU(){KqWfxVLf=cJ?v|D+?i8UxY*RDCIBlo|noF#a9ei%l
zNKfD4+Wl8M)hEEm8k@4Ia+dN$Y7B#<_TznXze82gw#|l@s(aFAbrRfL!z#@ehqxo-
zn+TAb>q2<+mYqeZ-?X54#29`hw{94-C^s-oHZ)91#(C;E4?od6G%Ga>yX_{WMWgO9
zEOn$hI3g%8+D=KKDs?dK*qRhGGV%ts3vW-<_W1?8V{*ya_)LWhw17HV8PaHZTA;`h
zwi$7IUh%x0UvT!jSYd~nL7mh+bB-xOqc70}-P!8k?4jw4`ua;kS}P+9(6+S$MPyz1
zOux!=G-i&_OpiC`GDH9;%FLG0*1{6?6WebI`V1y3RJ7bglD=)zw{;2khYpuRa=!oq
zGj&NRKl7RNm&V3-IFT_~rnv}Sd_k>l`qjy!4caIE_=FWa3hV8M&R^`7_HZ6z69UFH
z7^*!qz|uncv4p(BwBR2@B}R^&UI_QcuM7{Ff=Fs%-`w0N`Iew{n4y_}36OTYAQmyP
z6L4zX-Q_|Qnd9mf0$an6#c6O6lX2P_BMq-#Y$R>D{x8XeuJ4zZ>p-G-ol<`h>X3)L
zLGiVZEOf1O9U79@uyB1<k&a2iBKbV<!v_h~{JR)ppHokt6TEEI#3#5q`~2=hY7W6<
zM^>#X=hTpEh8LZ6)Xc@j(irj3GCz7i*7n-fVrFM&^;)vaj=+XM_Q@PSKZ{H#eqa0w
zmsy}_yUm?L2D3IDY%+_{8O0Rg@eIqdp1391+vZs&v4*tFfSAs8h1htIeN;6++2UOV
zZJmH28Qhn#koIW{jha;p-_pDjAA^Bap~BhxM;`fkiAnR*N?T@ek3u;%Ehd<vPyPN{
z$|gE>p1P^fBg(7hhT&IhXsfYC&h-A?ds2KC5ROC;%+GEuwysW=KAbR?a%=G=^4srB
z_qniftDo94VA_f7l0W;#Ehr@gpLyhdxh(2#%N45akbcBgGj7c3wU!h-nr{dKGyN2+
z!JE4_Zb|&z6bJs{pigl`gSmaje$MtwaM#dV%cD)kDqqoWqKpsxV5wm;zB1yMT#kb@
zj6NS^Zi;t#N8C_LZBQ5XR|pjplB)#++(4(F#ZnuPB=T?A3zX6MHqaK>zNX&S_kr82
zz}Tx-l$rS(ywvODU3Ri_Md=b?yMdCg&&B<@PRiMt9z47cV?`d=7$s%JscdXHKc9E6
zTg7(APCMi|pE(Z%6I@+hvi0;#83UhIDLlh`h`&`bu-hoGPQtFN{2^caV}<<@CN`0t
z{sH*#4>P+^%A?vp*T2%?WPktZ+C)H6(b>`FFNIj4+S&y4zoKhvCnM?m@)GZzmlQP~
z>sAdgKl!P~w7mDWrBjZ}Jb3np*~dwL&6N2;@h#y;Zac3nV)e|QLibD=3;slN@0e>m
zOMCk{ld|NTor_V1@1V>|jqheFlK5H%|K6m<L4Nj*!lO(MxMep=;^O$CeD)qLvC-@#
zGX4xNcsrauy*;_Kw8i#!fsU+|gZ>BJ*Jmh4cm{uiGPC{zl*x6+(0B6v;G^Ri1wY5Z
z$j&~sT`^uXk=xI{qneh*{S!sLquhu<rhGT+N)$xZyD{jh-wJ};?Jsv$$HhI3iqGMw
z8xauDUR{V_h#DkaIXdbBR=0_a_F4M?J^Ec68+RT;Hbla+Z)X<;9+o{FDk`zZRh)RF
z>K<W?IMB%xHBB_ZXBi^bZ_^6JqeH-1)rgaseUpt3?dc@J!!s3hFfutgULJ#+V2ixI
zXn8DE@78qhE?aw8<=JWiX~=%RI8hj>`SyCp?#G4p>skS?o_>cUUS^Lmwu_Py6La%s
zi{6qH5kUd*#_~^n32kQYgoAQsW~d1XHC$b}LEy}_vU>#<{$lGW8ENT@_$Wo`A{|v*
zhZIwXloBwFYz&Kwi``axy!EHwD0F9R3?Jm>;paIZK@n?XJE?Dp9#29~KhC_{UQeG9
za3JeXj54o9e^jm&>8~ZnP09&qM9W(=2R_vv*GS{?Txe>_V#v%B7k1aEs?G$T;57Cv
zEr$(GyxM%e(<r|7^S$3qm$#f;Lbj!z<VUq~4hhwu6hDsRBP9U=6%6ctdAW$<6*cCo
z5%uk**&p9Ocu|-)-#@_?%gGoG92#r$K8|}kD<p8lM{MYl1L;WW(+o?6;Ma6L3Ms3c
z!>iv9)^{!Yl0<>%3k8IK;PCDBD?V$qpB$e^rJwr=i}D7#8!Z_=rb2MNk<F_1j4X0E
z*EZH9JCbgI{|bF!q@VBTBnQt|&*RU9Yl6cc8)bHDC72P_3#%HFfJb!hS-&=5tFFNi
zJMXwCY9uY^4+Zy9F+iu_a|oPRbSlv2yp4<|LN+;8fKEW5EF;5iIO~`S873AF+9LHe
zM}K#9Exdmp0~wFW)2o7((}O9UMT<_Izz+**4Ngx_%gy}-@C+m(B=G)9C<TFuiNub5
zPD`5>9fN#i<Of#PPrAB$MqY5mqoK2d-i?hkDJe--R)Pcph<%RR6EXp4-IaSaOM>^s
z_I|9fdwFG}-S)xOWY!m=k4R+GYL64>+3{zeDU%x+NUig9v4TQ70v`6xn0@_>gGKtr
z>C`bP?+t(pBWMHe+=)Ssy7s0OQhNw#i+(6nU1t&C9a2uVg)X979z`-HJBoRh1bLy$
zm*ce7t4j(O?G@_Q9InQ&Qz|Jfto1)Isva_(`72Lgg1aLr@vnnb!JSE3tx-g2_N@ny
zkr8XxN8iDN%gm&54Y-Yb4W$oGcjZ%F<p?rCL8(EfMn~R|Q$#t;gcoxHYVGQ3*F?LZ
z;L!Dzy0HD+P`ay=oGPED)>sxa*Y9SeuFg+@k?zG!^4qr=gaqZz9+IP}VJdp=712Hh
zs)MVm#}25P51d`hjf|{G$&jS78X98_K74s_J-uk8K5pbIby{sZdK|jEHvLxv4YhS5
zy4g#jE?L|xEDA@i>jPi&wG;TDIS?$jN4hDJ7F`SNq8NrNfzqTYDNM1$NP1^MxKJo*
zLOzwpk+Y#NU)cfaE-BzWsfgvWvD@)kVeuvB3FPo~l$K=NyPxqR>PhTyLxXi|D}2U*
z-dn@nbJ>O52_8^Gqdu<6!xW<d*VODxf#z9_6?nr#%#whUh(o@qnwlye&7U=b=K9Q*
zm8fxBg$rK19eDDktLxYW1q5mhyA9-I32%bs%PK&ocYR{(qoTqyx)hAj`t$vbeW9g_
zmC#dhKiTK+%$-a8Yspy5&G%DNF^^9wK)pdz(|J@O4-1pr%1D)ZeB#ykbAEeQzl&|>
z%t+4zrNQ$fdx^Bw2jx-f7b#r$7@ioY-${$YR4#VBeR_o{w(en1?4K?7#4$+RQnkT4
z6+sK<L$1e58cOcC7Agu}YAAfGkxt8>8IW~xUxoB2nrEgPi#K1hWfmrpPI-oS^e!}i
zygttdUDe(FyKR7N(x#LP{iyd*gsI~x?+Ehu0#GOcYA<H(Q<%iB`>wP;)V|!r6=1@2
zrQnk^3$io$Jx|u#dq`xAIWYBX479q^HSCnWc{(-pRIdDKMnmNC#@TMk*!z3sM@Ja(
z7Nn&3=WH)mR=W;$Ej2|s*zMdrRP1au99<=S(iX?y{P0J4{5{=KZU(8Nd^qtIfmppd
zm4zeY0`PL!s*!`b@oOHdB$&xJW8d5ae)d?CZ=F#tOG{)z6}f+mW8B~<`u7qgs?2jz
zeCscS1ojA{LZcENud~N0_$6VeA6~kfCz*Uh+Ek}4*jcTWAID_HA^eIwJXH8hv4oG+
zA>mZwb6ggRgw0&vWi09G%U#d4nq*cCL%iWK2H#(4zd;Jw+;5F6lq?whbEX=+yi6WD
z1!$PI#=LsPKdq0DLwPJxny|%{SOp>uSJUoHwYH(ilCot2g7QKG-GcmbsV|9Rh3o`h
z2sAZ8_@dyJ#F+_;jXgWv;}^et)jjwKrkQV`RmjG{WTnaU?a+0jDd1-A<lXi=odTWG
zEZ05L-bZea{L?$yLMQ<h{j80Qa&<L2xwO(TZJ4!aCa+LSLz{J^e{Y3&_j{PoObz`F
z;>M=M^sLO<{k$J|5tklVp;%7rCe%A@gEXMfS(0|US3uN#r{Hq&p_AG7k{_zomMTYm
zNw5R~ui_#sD*JL5%gmP&v9L~UIE$*?%KXp%&ImWC73Q}0Ct9QEIOeEsl0fP_`b%L!
z@|q8qAT%T;+fWgQm;6`&^ju*d)}xl^NZc~Yf6T7UNSz@o^34L2*DQ?qU`bL+iq9AF
zlA}@OF!qhA|Fp8dB0wK`TyzQu$B#&(`_>q-h{KQf+F><>=^1J(Lzz-qT5^Sj%wS2&
zF{*c5XsU8r61qzFyL>ANn%rdsG>%~M4Ez-{?To!WY)&2FUqR!6CP9S*r-z#oE=E)`
zG*6agbOuZ>;=r?wY(2jl*P~KOVM9whYgMtcH`Ky&cGF$GJ$Tja-RI!C{q><`x3Oq&
zDyvX%gzt@^OY7hR3EAwh-94fFzdK^$KshCtVnPhWE~%x>TR!;Zk(!#C!rQl5&%@W3
zq6mq!Gi~fPwIV{4)UrRd4fH{JeqLYky?-wNMh5N3;y1PfjQ&;Qx`H1ryE$7$jGFeA
zZa99s)WH-*!l64t`D3(&<^OJTZdxI2w<#22fBqwOvY1az?LM_=Z(`g{Gd7>JLo$q>
z;u+T!W!O0WYMq0$9)d&`fk{`#tdUc#V2b5)f*#X;XMV2s#P5sLxLv>3X3B5wfW7!F
z`06f_?l4}KC+YpX1L5He_FQ2w{T(byly6gE&2xi~gXMYCb%W&(kNCvjI=*VpKNUV2
zl+o0$jOl)J*b|3k?7d)S*4!NCr~a_fi01%}n!MHvLG}00GPl|1kMPSsgM<I9<@Vo(
zMGla?(x8!|FTsYk_yMU(VLzK5Lp+!DS`{u_6vthcF^7^X=u7QbR01wVIxG^D=K@Va
ztSGKwl+ma(Ca8bZq<u87&jW<}08{h!5Pk4#-^P7ol2eR!_14s`#yzwSuEiSD_1Yt1
zJo2lk=v-LP-EGjPj^(*u?y~>pdkW$aC8w%e4F9_zVecyYnX22hfTxsBdwhi3ti<?j
z+Fu}Te0S`crhg21$^VrtBh1P^OI?kQj6WK#t8Oz8$jzM@5CF&g$n2lIaIp(n>tzMK
znbnyAT1>=0bevY?6$}zjjeNELfQ$G0EK!FA8!#W|zIqAz7G{foCE@HhW0&7A507P}
z_1DnCvP@`-{t)DKGK(S6(N5(h!0TULA-%kcb#{4h=ex9S!rRJQOFH3N_o^`H16`z<
z*$PpSbh){aO-)*HU$u%}>F2*DdIkPOLNXMLQ9UNbIf}umuWR<JJncnX3L-w9MNU34
zBU_=ndz$Q*5XAWBn!1tEDsUP&5^qYT)3ehht}hShf%x0%svHML?!g+7g?Z-)ZCn;Q
zbxSY1rl^L{Y%u(CRC`oaRhOT?Gtcg5l1r>%Vd2w3Qj&UaPvm_pMMMQWQOiqGmd@4I
zRtmuTaJ!OB^P$t*x|&{*WqT@TM=uQMG_gCElrAMCs9#?b4i0i?7yf1KKMB^VrS>%)
zea7htI?&DR>wawKq^34hVLRD-eKp+E8wHpG$-d_nPEPvSxP`$kqk@84!!<%H$kELA
z?@6?@ItdBoz!T28SI*md-FD)9jzz2}>l}#Hky2f=HyN)W&YdycD~eh%ic9#Clwp6N
z>gws1`|b7h#gv!L#I!=8@t$wLS@VwW6`&ZdAVWkb*57>`{nGgC2kUovdUYd?n=PKb
z+Fh-$6cCp^(<k~ebdh1?ahh5T+6a(pdhD4F6q{&KG!7NRH>yVu`fGD;Bw|cx9?d3o
zjlq*aa9NB%FerA^J4xoLzUU8qVT?nW42z^85t&%rYo}mTDoJ=YsFlKpYH9hx)U-G(
zOc1nYR@p6g6y&i4KvA`|f0aYF30$S#w#Q6%Bnt8Eg78^*N8^f!NCyaU>?jgxkmR+A
zAH~x|7;sq5%%VyL>PWpr`-ELd%JnlbE9ks`HuXt&hpfq)3@#c8V*EF9Xt}wnt~0qT
zf)=ch<*wD$k@;VQ3kzD!pY*G%(b1?!M=?GCGz}*hAI5}3=joN*cwsNq==<RWaiUfa
zlFyt1#Rij)UaErw6&0V6$b}R?vDE6gx#Bd%CCj|?pgT$StwO$ad;Q{9`g%>8)5z+y
zurN>Wy2h7yM*3tG_DMjfrkgW=GIq|p^s?>IM(PVfK#{7-Wj3He7k6QpAC%dCgd{$+
zH}>6FTBhshcL)iAf8%8qprcz#@?ha@=){p|3oWm9qRoJ_)6mkoZs{b%leX$R1YM)^
zsA|GY!RxPIUug#9v9u;Q1Xs}-EM28Tp{t6Ish2O$o@YKS$)p#dr(gbvT}MM>fAP8P
z)~k?>3gvUyD#yhzj6x{{QdF9n{QW^F^ODBXr@jOpqMC&FT-E=oZdj*lw5Ft+@cw<h
ziHYBnc$txMauFpVuan9-y`9vI6>D6Xxt{!t^g(XKX&G_s>hzAqASQ^j=0eXuyvo-9
zkzi1bm*Iq$mP~ei<uMNeh%JSX5P7c)e6bWYCK@zq5NU%LGn9Q(oP7PEjSW7+Tm7Ju
zQ4<tR8wUUuFB5jI6la=^QQFfpFh74RAYqZk2Koz{n-%FurP20&%jV~VyJ!p$Ea)LP
zfgpD3*F^F-IJ-Dq%Txft<UhCoL(R>$BTSp956K^3mAxwZc#EA}yS^w#=DMA!rw#au
zR;W#0r)FfpOVkFZx^hyQZCa#e#>TwM{lBU40kfD3F`=;*D#^1sy}1@BHWmkBbBm2P
z?ArA#gsf%uqazv1R_gt5>907Xl?&gv$B-Kg7wlhr^mKEd_*=2Pnq6>A1q!=_b?15d
z%F64ODUWoj20G`GIU@$RxhrS6n-K_!r5`N56Nnd1w`WpwiMTn#0*Kd0Mjf<#Wqt7c
z5n0#WyH6XgDx~W}Q9eCFsbN(;<<B-siNAY?K9&cVzMhWNO}wFT5gj0dTJ6mLT-#%E
z4;sng)w9%gs)Jq#FfA_90Hr5Jaq%xd%u2&3;SLb99sren&d&#(4Mc=kgF5>QS!-Hb
z+TVp|UeVgm&in;#%JQ4ej)*a756p8k1%){<#Q|wY!P<KI-JJ~qhl;$$_}KA0J?Y+V
z2)%z*RoPm|)j2u~%Q6M);U?pJsH35DNQtu2Bix|nvokmW!M@8{{NiGZL!IjJ;c#D`
z(4T4Cg?DIJS5~?9SOUOcQL<Kn)bjMS5d$!o^Jr_|mJL&wj4J-<lzhH%Z5}H(Om@9?
zP;-3)lx!*ad-AH|#5@Xv*gcnVy(Bs!=>@<tg@HOeqsYjpAa+^o-13<%)cxx;Qkh>P
zj;B2Wr^HHC)!SQA;URm}U@Qk;f0F)y=C&j!;NpTjbY-oNFQ^qGGcvbG`})#Fl=`Kc
zJSL<yc(F$@PU6=a^G*=!GWaVd*|awt4oJG&1{4%bvlOq)OgMaGX0A*k=6AO}P8J;v
z3F!jaaH)l0&o>hlp@J#X+obH%^(LV5Stu!Wty<`fQ+Se-pO%+*efXetHAcXc8?DMW
zx)+10R~x)E>LGGj?4uWd7$#0S@%UoSclw|u2xQ|4&CQQGkbgUs$&%J3kfM_bWbjxl
zBSMbTh-6h_v$7#)<|CU$wfW-pZ+{5#@=6|!3`X3nCM4hZ$E{KM@fy@kkmKz;1o&S4
zrC;&Zz2;>8b@zxyF3a6X?1}DJeGX&$!JOQPJAZP>PuE{!Ei5!%g=Z$z{v3)-XOd8D
z8Hz-%(Y)TpevU@M^>b!2u7B-uFpUJY!~AoX(6PF@o=Jx@kMH>d4~tnN8gJ%)in+P)
zD9EDeHO`4LbqqCqC@D_8dygIT-EtUXEn2ly!FS$B5->9U8qI&9UkMHT^bijBBqSV}
z_`wfmyPM|4z(en=^8Tg(Wq>P4P=~eC0?d(FeJhlLtj;PT3R+I(N6u>#T@E8qVOmy7
zvG-e$2rqKD$j<+lnB#cSgM1dpiH6zkZiQ07ZF<3~b$D=4%Ebf0?$$PTo33=2x1(yU
z0y@O5M55!mwG=y`>gwaD^lQYo)!*eFO6s)OW3yw=!_S2r4_SV0+^IjxewrhhNV~7}
zTQnU*0T5dT?5HF~&65bkJ0eo<9OiZi^tx%?g2q|bhE#^AC{^vAaCG0?=3AGWkbRGW
zi6!Fd;Yik{F*%Ug*fAFS$#<);jPiQX5$M-^Lu%WwIWC5q7!W@^9CCc>svj(W8%zy|
zicfht9sd;N%}@f0=sQD3j!X+iw-ST1yY(k5s2?&;=JI)ysE~$US9hf7^KOPGq$+Rb
zvvWWMk9n*ze?2S!TCPl7+p8@5T?@Ado^jMa8FCF6HjF>u@Jn#DBmW>4vAh25sipD@
zRn<+=Q19FGIvC)ctk=I>_+$hCx2z2#VZ~!1gW)>TSR5MsecucCi=)F>V<a4Y(SnV3
zwDh6Ym>O)X!9Q;;^pw$3R`&Bsoo7H6!qOiw4N6Z>@5{(sf8neV5`K7nS^8pA+vSLl
zD>HJkIIU-tPm@W}sH(U)S1vd2Y~d4ZZf--w=z`%tqkN+n5A`kkdMeA6D#{e&#YBtp
z<kQ?Z@nJguhQbM{gQqSE0FXl>KjP+N>3P7**J|8J{}X)Yg72);IOlV5j4=wN2U(50
zwh@gYg^+PLyr;xm01l6u1uX`#otkD5-Erj*hw7>fA)&0g8{f>f{J_w|<PBGYb4>Lc
zSu?7RH;2Ilt;fgA;K8a**!_n0gpCB(wtzyIWoAYV3>3@RI62j2<#S=RT{q?thx^Og
zmE8GIelrzw){E+;>OU~nz-X_?YDOMz)x{OqF-Tl$KBSo~Z(>LpQryP)vz#2t<d2u+
z(LAp`WBbJ(dl+CosYI}G@mBy9q{wJ|16{$MuHcg?h<RR@c2NZ#J?gNlS8u6h0*4W1
zUjD-bo_3R!F1(80gE9x&Meg}|q(-`eB3uK`+f!<AFKYj1g6ONgE;FTT2#)PZb>Kjv
zzSE|B$CfT|cbzF8OWvH?7|Il=wB56^!4UTo``ra243)rgSrgLI?Bq1Gu{beO|NeH1
zTV^!B#=YKJJx-^9iG?&ip49u?)vVnD0ab{OpjJ}G;$R_#ZffbbW84q<4_B~$FHn{)
za&6^NXYV6+X6xxPxfW3cX<@$xK~d4N94jmHOw$94h_TABu|9}TED3I+bUU?7Q5ja(
zK>dl5=?@zI3e!U?vK_hAZ#_Wn@SB+EZfXDhy33OM!q)?&XL@0z?Wzneu`OI+vz(07
z=c|j6G%yyDzzBI52M2RNz|&)T2|QlPVsG!ElG4nrrQl$Gfl3X{@{$sMO+E=c8=K{=
zr5M7@=iw7hyX@@pM6vjAFOw7tbGZkYvQgdEfhNn&QvmU)s)BvTuEB{+y$<twR|74C
zWCOjf6mK<My{Wvp&tnpyJKUHDtwf7)JO?C!aYB3^{ggkT?XKH3fAGlTC0G>g_A?sJ
zqOwHt+_6~_?jeqBCj$+N&08}YPL(wyZX4#|6F5O69q*>(`O!h_UX3T_2gW3@9_-5x
zpC2!**R|F5e4ghO!V-jLy1lx;U9U2T<|Wg<OeR1#6II~$6kYE0aF|7<e*`ife{}YF
zY(i^Ob?I5xpB1>wt5SzeOGlSpT_YW)S)x-gQLo=qFQ*$P!o>9PiXy7kOHUJKa1@!H
z9S2on%sv)R0COKd_}Ey)ux7*?;RCb~I`Lsvc1Z??@c9YtSK;PXIim~BU~>}5Pe>55
zf(0Y77MpxC_dJC+_jWQuPk*O2UfW<k?A|}9b=uOoCqrEcN>mNL_2vLfF5?B()_30H
zDHUJ}R+r;F5GYejvvI~k*ahq}JU)&c=n7NBiZ$~x4WUuPV8JgwUe8*aXMXUC`I<#K
z_4EBI0l^ePj67-l?SHhE&-Eu1dEbiQkM!%Owlibps~L62-}wH1v7)XbC?lo2d4LO7
zVREi_D!!d<0fsv4EaxXcfNi0%yRe3^5-`B*XKglJJ>|xS0ji8mo&zt^9^f4wNL_bD
z-#*<71gSG|Ac?3DD|2o@Ew(hL&&IIJPSOzX!Nu;;&Q5L;q4VK#l1CTnhZYY6`?$+E
zax<d3`YG+0ZV0jUcp<+v`#2xR`zJp#r8>n53u9cchX-U24(sfk#)o+{Q+`)PxsBC{
z#GTvhB)QIk+f4Isx48&jnkNqHv{LA%+mRa+iu0$7j;X++Ol&Zd3BaWq{A2!qBK?GW
z50D@lJ3ivmzL>J(HuXOOggxXah(rVhpN<ayPgq`{;SLMzX-rIXCVf#(j@wF?siBvu
z=Q?$5oAmjxO1`b_>B2&j^2Kx2=(K}W$T+F%w(DSeI*64F`nWJv*_*7{x5~V3&cbR9
zDARmL6)bDr^=d0L*t9e>nYD^IMr$mN7r$gy+Lw=xvyK-qkGe>F{bmFZ@vx(8hyh5*
zwr6yiBL+YKs+lie^~u^QgI#C74C-dB2}{qD-`nrbd`KkZHl?y8-HxTird6P;UIU!g
z3x_BUbmi|$;or6O9+<Ds)$#{JgF#)SFNxs&dqOY|AW!S-dis+dUY>6vJ?3)fu>wOQ
zijH!eS}6e-oLc*pIO43G>WDM|B<@rMu*~aB??0GWBr;d6v!yNO;xr>Jjvk97(rNxV
z{6#+Of5QHb(;LoWje|Yw{DJzd27S#r0R94;mfA>a4qJafj<RdqYyA9_Ue7;z|Ad>D
zcd1{T>>0)*WU^Roj66&smQYuhzPcuBYeIF<3C+&#u$wdXnPBEKAqKzSF9MP`3j&on
zo4NeMm8m?SS}A$=YT97YBcS5Rj6-*Rn$g;->2zx~QL@q9v4}FTK$a^RHC$%BOkYl>
z5B)yr^a@!!-B3R>$|uF72u0@79D9&boTTR;7~=h#M@_9KqmUB4^45ZEX5^~7Xj-Ns
z<KMuea-rpQb?nbh+om19JiA?%Afh(pGI8;>bgKg%DX;4vpv?CvB|ZZxR!Z1nr%yn%
zYsSB$+t19RdgHZ86Ab0J4J3T1p4V^oPp_ZnTGKT1?(LAw-ZYT-ik?`xJM`~a(n4LA
zA3FX<NgaPfP0SD~K}GeppJF8Ga3<AFYrWbf0P;r*S&sPsm<jk#Y`_1)CJ+Dh&gFka
zJWS=pd5wWGZ%>Sy^jQuBbl}8a58*X@ZlMEEo%XNlKRdwv^Bxobd`AE3>)$oV17b->
zcgxG$81Y@_EgXn~LJS1WTKdJ8DMe8KhEuk&c65`I<3O>K@BVE}x`NlpTS&FS0g45-
z&;vgV3eVAgG1J~YeT@`k{RRI6dj9lZ@cttg^v`(b-#L_jKfVE_*E1IEy-H(*BB<u7
z#)yF_hFE!0_*;0o9%}Qzf=P2vPY+k=?Z^X~UkxnTd;04<w;c`b;PoGU#4Q<Bu(8w3
z{KUfKWo0#(qQCw5y7lWsKCU-!KXP%=Rt_jWDK6$B(wkmgBcc%Zlz(mEyEn8lHpZWc
z7x$;Z$n_Cz|06De%IJ!{2N!Q~BziU#2wItqeS9bS%)(&-=Jxi!FJ7cI-pmJKEC0N_
zWq<xm;(48<IW*MsU0+k>p+a{ezUkV?J!OqAYwNiW+yBB}ThaOqk(XDXZECi@?~*?|
z|L#@D1JV5H`Yooz+&TA)96_Oh?z@<S=3Tp5e~~@k?bmg2FtG0t9Gqh5=47dM+>?LR
z{ir`Ng_+e$Lj5I&aS1X&Q~g7?0)})*C@DYN$aE0mySO|ivUC33K<LfyJ5!tLLzIV-
zq=-9QA>X)B);BPp3H;=wLxzc~Oc11a{8+{2xMrQ`t$V-LlFtTq6(N2oX1-f}n5q5g
z>}vk)4JMGdkY+m{5m{ZcnE9x5d!^4A?u~)AWD3d@@td>@nGn?sY}fK$+p-S4QNEEc
z=(L%v`!JZmMh)q1DwkLa*zMsjLU)hT?Xz*S#{NW6&Ns4eby{-15owQ)jg@c`;Ny#x
znrec-%nuanx~$$@%hcIKT|Mwk<TZ!B>iGSDQAzajK=tUpN|MmQdO`KCG!wHnrHQla
zPsrHMNkVQG$7W%lFwq-t-gk;$x#}Fv&VD9PhhtHQ?aycAzMyUf&7BQ5Kf_2_|GK}u
zc{v%i)}OrP8iw$_X#902?am9y_OA)Wdm#b|2?<9V*@_GbjS-&fYsPMx^WTiVC#;Cs
zwDIM;$-7YmaLjar3+m8tWq*0~`0Cy#SI-+k3+tQrRE)~Wg!oUr7niLc_=&Vocm)JV
znYD{gjE!aG(MHHhhc*;7q@_I+Hqe7YR`Q4f1Ec~l(T%TdU6qvJFH~Fh?D=wKhmY<w
zfod(4y<V8G46PxaZ5EXwo2aF&<MZ_fDl9QBCmTnriRQ2#Ib~&0=;3%Lb}t&%qlANr
zC4nzr_Gqb6Wq3|W*%C6SNXa&b6AF-<r1u5U>kRDV_aCwaV1>#D8^<$|l!Vi?0Au$k
zZfs88o;(<hVNU=oxS^4evZZ=#{)WA=ohq;)SbyX1p(quO>-FX&LPD@EOd4;UAs0x7
zCzTp5ZFUQ>=N@8Vb@lY<)w?Ib;k}aT438y%7XD&;N}Juqu0A#OFPNsLeueGy(O`B_
zflnNc^XgW$K;@;!MeTID^vjr7ZSVcbowI{AU`&`;Xxou)1C7dwU#G8HWvF&A!O$HY
z2KDas(x{Rr<+iN~sGe~%ZW&KEQv5EF+uKK*KM(?S4RUF(Z2deE3cGlfVxwbLR##y`
zEZ#nK?V*&p`B{Y<eB~Cx%GTL3X$r>@XlEBqt+b!~T%9(?SJ*e>Gr!CDT`hhoa`!nk
zva;$3A&bi&Z?G{p=Q|ku<Q4)QNPRCUA?bUzGI4q4G8uF8SzOF|=p)wgvEjr5!C-%5
zwdYcM?Obg43YE{L?w0U00#R+)-`ops;V+c2^E*QuOl7`RZi&)6(lvAwisChx@!X)^
z+<1L{U0ct=aV2)M*lnrx5}V@b@^rm%i3$P=*{no(2c?{&h!XL6$n~V{if!;dvVlOp
zh<l1XSLMp!V{PdBURcQA1q;G(ZB(7o8BfRDcin1tXxP|$9o3wgU`{~oJ6D0FeE0n?
z^dHE3R={BPPR)FMKThli2Da^;s3@u^J@NO)3VG(UuOc8Y9y;w?d(yqP)Ly|Um)!V!
zFWc#p@8x@dPwfo}2smJ3V7xKTD=dr?^qR|jXo*IBKqRs^lLw=4aC-fUGS|%~de(bx
z>5FzF{`a~3m44~`ww%3Gbughu7%eqbSzzyfHIq|N6dBhK?2S%M)g;bb)?ME960ZIE
z+^_LP1GyjO2Q#QrsqPW;$}<eZs#q}p{I1TXQ(xRG*J@~vaqAu0YNs?VJ#R%nL9{ox
z9?zVGiC=H6H9ruP_Sx2J%IbBvBJukQK%tIB#+13sw)>;C-xY+srv;dDZJR!-7aE==
zcs<D@d^QJXAhxa%d;U!N*WN-FcgReLt4VR3uj_`O*~fbl{mIwe-TC6w8JU^pZ%<{9
zvcI>_r6k8&t+xxK9M-vwl@b*`tB83pe8xyWH)#$V7RiWcu<tyBogHbs%MW=x>L_O$
zF*Y-A6}yXEe8T?B9+%hfHDXs8nsx1D>)^l$lb>u?VKd@#Ii9grry>>eo|xeNK07<B
zF)ID_^DEKNVA6=Bq-L>(hzM>-pSJ|s0rv4hG)^Q8CTek*Bb*{*nDTCNQjoO1W?JSD
zRA07)h4CIok&-142U7cwjE{R1xSw`#=V}nn?+QRxF};NAm6r$j>a!*baAg*-P9)Iw
zj*hAoZcV+e?oyTZy9Q?nso+ya55AKenNV^QdVEk7bsx%jprGJ@8@TaJZN}u2g1kIp
zjx-@Wyd^m{9a_8LQYHA0D_A9yw%Od9DqX{MYtDy>+!AOBX?PL^a5!JGeCE^3jtA?#
zf)FF=#U}y@WhMN{C2lRLzNi8PMQfT$9=X^DPW3U`FbxQI$FOHOEpvf2zL$!p?H2K!
z6cx12?kqd>8!WG^qXz`k%SVrjdgIU5GrP6>lZC0sYJFORR|Z*AR_ueb;vm`_6Vua2
z0kkz1y)}nqBo6j-Rr7nEci%sGV#SnD4I;(hSB7(+l1FWcb^_X$+`Mj%_O^Ib{LV(_
z<f14q&pHMfQ*=gFD2xue&e2c8`~=+dqoQb=xq!7EBF0iZ@;jwTPncYNsz^pYcix|#
z{mdb1iHQQOR4)9L&TOP+1C`TlJ>~w6FEt*T9X(^IkGlZz;HzY5=xLhEMK}tRAey$a
z+VAxh?$|cTqwQg1RF-?xc_m>zdZfFzRTAb=?|0atlHyyTubCVko&oi~f_^|=T!jyW
zM`bHR>{y;XTkjgp*Au4^R1>{IMMbqaMDA?o`z$aD191<(IkVN(Rj{Dc<n<;UtbetR
zBEUEsH1@k$(eYNcPd4CKv)3IzjDKB9s-D-cT6IwGSaksAQ6vaQAmi5*edU|u_wHQ^
zQyd*yt9P@qL`JG}7}h*>Lq>D#{th7{0ati?yg#meDKi_vweILrr?12{<m&ToT@Yi(
znzt~5Ba!5BP$7pW_TAek<ep@cjAEiNqzzw8l@O+Jf1*6#?Q~Kn{tdG`G5^4oiRF)w
zdb93zO-;DPN1vHprLb6WGM|M-`dkf`2w4uJnyr<UOz1{0`;C%<f;JO=ZmzY1{TaYa
z*vz_(eFayOJP;RGVQ;~^Ve*`Y4f|VNYZI}_UyOd7^;U8kSF^-+fBB+dWZ3wg-NVJD
z9(r`1<VPx2due@SWaqotNU^!zSY*y>ge(hfym>9rVa%w^$IoA1YsM&}#K_pr{ar?^
z`gCuhS;AqCZMO0zo5zy#^Gkvn<Yv6lV_PjV`6EFLj|L)pkBO2(T_|)T3LKh)jg8)g
zf`gsU`9@|^Xng;g_|-_H_W0FW!$sS{*(JDyKi@ZqZ;4M#O~uE?-n3;^VhYWW?`l2%
z%4<mV<UH(y8EFV4<Dq5S%XEbsl6&vw;0Fg_J?(mBhxedV%f#<fzGt}<JK@8o9#Oar
z-gI3vL(A{C-t<y@M2T)#i7jXaJMLkz9G*1xC&|&#8ZIseIk*u{DLxs{jkpz+BBR>E
z0>$Npg}tVO!Qtz(U~%`J^(qzd^V8v~8{3H%uAbiu&B;Ei*2kGakEz00@qg@Yb@X#O
zy14l)1Zdix)o72^mF$RXYHDIOue((Ews#j#z_XORa&mJwGGmQ%a|*T)2p;GPPXfOK
zz~^ro4y(^?*1UEbj#EN5#~d78TtbYh7q=!4lDr`cd)_|xPtP&KyWX662E=k0Ckc4O
zjK$YvjsKM}`c2Izv?*y<s%ZCfTSWmR#-cx&IYNHCuB4$raip<v%_U!}IM&UB)GMq%
z<zk^wg^=T$pMX%^(yU#P-P@(vZQ<D&gA^Z&$?2Pr(VKraQDPws_^_1$t>H5s20X*_
zeq!JBdx!Sta{}*T_{S1Y3##cVmF)?M=MhLz8tOHzF9T*{WkDihMVXn^T#SQW1C9?K
zeA*#J=Rn;KavZ7Y-r!lg{(R%I7V3dx{cip4JIWV`i~GL|wh|fP&e8lQ0;L*g1ssXB
zG_HF-Z?~8zQ-Ba<MFj~d_nlJo0a^{sAdiNd1#K^PGZvIVRFrylOM3s`1r(lRp+LU3
zvF+3O#eLKVX6Go-M6l#(X=zdUR1FUg-z&ZInFD1#K#=u|2{Bc|>T3a08^-!2B0o{w
zP|{WM?`<OYP}V<MuHVO(5<J6X&S`UtzYV?}!_dA?ScFpnxCs2b^ZN9O`S!^_3umQj
zL&Q^#K(zbqw(AXrYc>&f1t>+gKUoQ-*UrmpMj}l$=jRU?mCMS?X#9(-Jq1?RF`IFQ
zd?EUAiqO?V>-L=sNagJZM42tO5ePU@*ef~K0yg&3yOareY8eScFp{R?&U{#AKvI&@
zZn8Ic=R1yWT>=bqPD*N}VP474YMmDO_)-z}B19idu61q9tpW#lXKNX$z&lgBok3wU
z%US|Wk_Xd-DcDrKZAVU@L>g9=ck)LEOsHSi>x54SQz^%<L^3!Xykp8w9Px-rg8PQ8
z9P~m3wxK%S9^Cm#K^EurDYHb!VQ$pdi5LqjXRp34hOpTPvB!a;g7Ybfux)*Zqw7y8
zI1)y>2Q+gQ;6si@E@P{!F_$0G<EMnG4LESnwruUBrKP?d;=~<8k|%D|@Tu{ot~&`X
zygVOEblJmfM9isNGoaQZIT7MiVm*X?$p;fho8y|>R%K_L!J;Xer4BPTHa5UbX+bwZ
zZ+dBQv7_6DfZrUsgsi7$t1+X>_{a#xU$?PPb_|Eb<|d1P@_2=N?7m(FZg4sq8{3Ql
zs8|N?A0vyPM;1<<u>5gW)_gQ-d;7xgGE!&fn6D7Uet!J{Vg2R`SmYua3@nY!96x%c
zLq1pCG&n&+anD4-X;EiXXRhQI9VZLA6xL*ArIdma9=6Cuk23^9eYQ4(;zMR*Yg^;Z
zH#jrn{j!BiSXe*kWxVk#*M8q>yO~fcJvlVhA^G5RwC2m&5vV>=79vuwsn;WRynjsX
zpOc(tV^j}ir=_K(qkASQnxv$|H)SGJn08&-CC?OWLd;_-JuJWTLQ^XUn_PqqU#~a$
z=yO88f{#x#IGIVM&>OdstcsnW*0Aa6X~P;<&C$?8=cO4rNKv8BULmFKBZzk6D{bwY
zX{vTVA=fw8Gn4o3zx{!}*)QX}y}R*aF)q_T+&HZD#(R0ZI;>YC6Yd{g@>JE$t^A8Z
zV^<dozsYtahxr?e5HexCC-a+#Cw%~Tc-NrF05Lo=P<-|Jr)A2Hvxf%&6i*v>8~k?8
zh#JqI_BAy%6(8u9x*WjdR|cqGgoa}I<LK)r{4P%0QgE&oxsrB=RmI~hB8ME-!=Lc*
z@JMv*?%iAmu=DVghR>rET~qqwsCt?!+c&(VF*k<3J%1A_eSH=Lb|=Qikdn4gmdgB=
zWo>yX@#OL!TmZ)QG|3O}0ye?(9-Ps;jLj9#c`!5MN}rOel*fGi{PhyLJD0S(eTvMl
zr=_8NjQ5x?&)Rx9MIDd$c!)KaD@o}Z;b$D#y`s+0zrMTi0&Qcw|19ED_7Bd{Q2`tY
zHntvkKofHmm4;n<{>-x)JqQFM>~g$YS9dp8Q5sG6q)Wj+{ABJ%N7H1MmGg&+aKrNA
zki&7oCyK3Y%>ZHX)0djVRw9)Blatm%E$A26<Tb8md0}A!)XV!Y2xha$@1H*W{QUmm
z4;a28zzWK$U5I^nq-wXmu@MoSC}f9R&REt^QNfPLNpx+Mn@gso6s-1?l=wLMy|TP~
zdt}4q3>8&Ab7;DIGmZK2_Lgm}J0-ALXpWEg*2&S)WZj0_@y5G5a|v@3#U@LuRKCxu
zywB7FR|?z!nG?s7F*`IN!}3k6wuZDcrWtW$bfPp99*LOZ5%PB$_3^R$8mB>LSKra`
zKArzeM@PCfbn6=vqs#+QG7}FdEp4r$1~>E#z)<IRzi3CPJA5U3EcGl(D;S3|rAY71
zcGU<{^r!Y!?(<a7u9xkRM5f8c<;b%&E%^D-#1By~G2@CYk`NmT0h5t3?7)pT`;+F4
zcjk;J@;#4E*qG~&wr{THHx_{0io?rO$A=GHLY<e^ZMIuFOXGAW&7{M~h28N}+SL(n
zkdje>j~;2d>C~^4TJ*iPS{+9$s-%dCpZtj1H1_dq?+XKBSH?bPefN@PBR<uVQD>x=
z1Op}qGlPFsbxpAfSP4gs%|l2@Jhv)Ob8<Si5@+JV`kpx-R&sRe($Y5dUHLezhvR_8
zZ4oyH9#+d=mx6>bF^DS%AkRwfXF7OUnDhWRX-edBE>8ktB6W}UZRg`J*R|WTmdT!a
zKd+zh@d~e3$!TixQMZha!VD)1^76EbmL?HV?cdR;?d+T@64T$xXVM6+1ZICq2rH@_
z=*k<Go@gh4nM!oPAVIdeK0fz6Zb&35jlEmVTr~5R*!5UhUT)n3++hhcH!r-b@;6gu
z)^l@nDjd4Vr6nG_sj|?~GTx!&!o0$!bYh9k3*jZKQHy@xLuC2p*;znk*GuqdrKf*L
z)U(7S5L!64zYhfRRcUF@yc^mW($aoah8V)vZqwa}*yuUr+c&!+)|ywbLssg1@AD+e
zwI5&~msXaFyPgs~@cQ)Y<5f+bf>^^ne9JzVwnm=7w^*?2wczP6ZiONp%kH@+Fvs;c
zc?E~F2d9ur-b3u)ob2^dOkKs0VvyD5<wI;Bn+67nkEX)=#}((LL%()(v*zUGaWK{C
z(ibK>-spUM8kLJJn&Uf#qTP4CYp7cLls=US*injp?J*2n>@8w|PjFQCo}%UW>!aOW
zrvnQ)YwO9M@5M+tb@lWVtNbqgFWO`0Sf(3qvh^!p&aAX?r<aF5>^V`BlA;Hk!1}AA
zFV)r6F|eS#ycSBz9slq=7H(>=8H481D`_dIOV7*YvfQZJ<1sT_Tobv@T+#~~|GL9P
z76k<bZ+-y7u7ftg1yvDI$fx@cjH+DuuB0vAoH}}WT@VJ*32=%(^?nAvWK>GSh?-El
z)cRXO{ugK|1(mSiv#-Iywl+3mE;|ws2$_D43hj=)J-9Cc0j=-vTyIU;WpB`@p@pRr
zH(eWdYwFZZryK~;ap)-|dz<b>sj!V{!j~8@(<U~HnS!<0?>nnp-#bB67H!Rg5WU~q
zHLx~8-^b^wwOiXl?CD1w=k+xppA~ZG=&^(wj5Ab}!ZRHW7K(Kn_Md)^(iU2#68lv@
zzI#>{F+sMJudGcLl=ziSj*m^dQSSmNSu5*^iK&U1jY>3$(XlZA4u+=ZT(pIlOjC*5
z_k@XGdGE{w=<-l8_$LZ`0O=4Xvv#W%&57S8Ce5FmzBZwW1Pk!;E)Qk0A|IV&A9r{3
z)VQ<nHS8^oG<LWxhye3Ae08s`{^9;ZNlD3(hv4^Q*^B3BgPaGn{x~u+7a+Op?d+_D
zisk}ntVi*+gG0MLCBHYCRK#+$tHkH*Y_Ex8YGkjwxA)=r+vD8waz|OK@aqu&N7w{t
zr>*pr6UL=@5*Gc5eS?<qT5A5FMGa8m7FF?Em#kN#<TF1k<tEbuM!G#^Ui(VAngm)i
zPJ9+gpos(XYYPiyu=sYK9J(4Gk3{(yS9zbVeOJrvX_}eo?S;+yYYoZx362fXEF7H8
z&0S@S@dnl2E)WS~b|rCfPq>*^7BgA>!^dw9g2&9@dxz_sjkpp92IopXRy6uz_FFlu
z+C_eZl``LPX}2b-za*u!ep$W1v})9hLp(vChKai`EW&1uy_L|MJUsM&pPGHVe~v^_
zd!(5NvWArub74P1Z-BDD7@a?~j_U309U8JXsGjmILU$iAe1;9Ks^~~Gb5sMO((j^H
zfAnD^cmh17r${iU^H>~AW!AsFl<Q;;6A{YSWKh>P$$Tp^SHFv{G`zKevV{%@9<Gt%
zf}TdCfb-=-g@-$y1i$@T0tZ<z?tnZCjKc>R2Xbx8Sn=(J1tTM)y`wpY(h`@3i(}T0
zyKbY8e+Z@{)u<`M+L!if%sWIV!?LX7v!-`RQr&OfiQV<fAq3;N;$j3KwyvB{*1VE+
zf8^EFw6$g;qC2CGeXmfxaM2P<_n`-Warx8W8$Wq{T|G_^hD=T@7nNGsoAQ!iYC1gb
zylKx0DO!M|fw5+`wziD)Z;+79@=H^#=VgW?h*~u*cRQG=3ca}scGW>CLCEuA%&_0Y
zO|L}q>c#7Q_HQdFHGg?M(sEEv@YjmT%xt~q?Tvc(#Yw1!8>4!B7ObqKtW3wseDXhp
z);=*NFJJ0NNZ^@#{sgM$ak|-+FYBk7f<f@GLZd3(7J^GlgYy9li~?OLsGs|ci;EuA
zsa<5qlf*GOp+gJk-XLjsV!8cL+eL=@#b4mn`k>%uz(F=QXh8%;oFT!@5&}*Wn%fsY
z-z=oN-9Gu>mvP=Pqu6KT?c*K1r0Ax-5@ihe-45cSTdj<B-yqNQqc04uY=|E><bSj?
zsssQb$u%?!wj~~%4kFXx;4v-q6NZ3*qaQzFo0}S_{a@jGUb)fWM}9_&j2$<kX4^`b
znCJ%o|7P^JBQbnFR|xhxj;)lVQ+iZx3g?qbUp1`7YYVyX9nD+n>(gRXW_#UjXDe$+
z5hoDV(o!}q;$-^1v{cjeaBpwvWP1ws1dq=w8C?A5M2Tc5NoK?WFc|yA#E_aSTD4~4
z_wuZ;{+$R~(c_&YT;lZ%Bg)5;Y;3ZMn4mkozNJ-)n1X_kf+8l#u+-c0vc}DFdEf1u
z&EJkpT8(8RD@!O-ZDD?1O<#SGA*!NFuWXEs!*^%?SAlu=b@31G1OZORzC;uvqJ`}Z
zhl_hZbilSFJ86=F?Ckqz2luck;zK;|gVB}?Ue~R*P4UMmrU3!5k)Un@3_*A_1irCh
z4V0!v;`vljQI8&Ay-|vD=At(zp{C;qEihDAT<oZF+UCnv6ny6AJFqjWesxLC)eMFM
zMZg^mNjq3Z^$Y4)<C}|0JtvZ8E)mKU9O`xkUtcUTfzZh+C(Mho!qAE7sGraGm%_FQ
zCJb)p2xwApeJ|_R|J|BKq6eLOoN2r<Vtcytc>9|I_$ROokM<1Q>niNqnwl%;%qGT;
z(5RdC?Favr7|TD`OH=a~jI;-W)X_bGorJDLTqjpOxcyF0A&d(D#;Dh5CDFc5anVl=
zbb3HSd~fak^EE5I&g<_?!C5d~xRo8u(Lqf|hl<QgkZp@h0{G)`V@3(;pwCY{J+}#(
zni_&a>J4;s!ZvQLez{-(b$NRVjoR5W;pNcCE7mbRbvm25a#oX>s$Ffk&<>DwSZujl
zG1C?}G8*A|Agie8)%%!{@R_i_lnfP>xG$p65S>PqXO#pD<_apW1A>97I=Yi3VeISy
z{^9zqf_jki(=AHDA+y~%lK1bGkt^KXElnR<TEgw7rxXkf{09e3z+-ZfuRFHANZ&)6
z!}wF~-)}ZuW~9AMo;nKw78XZ2AA#Ld>-+*KNblm}8Wv@bUd0wR8Cj|4!QHhUbVn!L
zx#a5#ahxyoGNH?qLeaxRG%4cZzX#GLzgdDK19zkiVqahv)rlX>FCm*$Wsnv=EMGkq
zpPbAO8G8xWeXRNFmEZjHv1sN8>S{5#V7vE@sd8XHU&+ReT)Lt^9;`jso)aDI@4v%?
z3C8QtT~Ov5(UOZN!*@PAYV^w|h~>s?UfM&)b+%nxTPt>2`W8agUaYc-{9G@Jj!7qe
zJz?~|*ZkJEg_lYYeLg%9iD#f`NyKppalRs(Y)G<2gjV1Bo#g^Y4@J_2u!~|)TYY}g
zR9`q>sBY+Sd#nCc&H5is=<illL%QRJn)D179{Y_EsYAK38|3x+_B)-mxX}N<)A|2S
z=TC|MU&=N8-#H!63vnq(zQ0#gQ`0$Ek4#JuI1)_3b=N|;7;-QF>g<LT;Se4x*>0^g
zH9-|QfgwJz4B&X;Fy3|@B^ys_)2asve%V{=PXCqc<SI|IJt+yUoT&3NW_lvLM(C*n
zWl)fHktgszmX`VJ>q|@RyVK>YFX(J;kwO5zJrh6?YUjdWWz|25{BQCzn7v$>v;R#C
zlo@d#*H;PZY?w2gPgK-ytRP6Qlp!xQ_4?=g0%mortPD0<|CZK>a)_J(If+1F{VH%h
zYyH?$^<H@?DfcXh!PN>cZA!Aqfk*NRFfBVv5Q7l-|Ly8DZ=cT8a~mQU`+FJws&`?b
z=Hylx`5`idcw9Mr%LM@oG#-d}Hi?whbsEfW1OYDx1Wb3<?*9~(8t$6DVb;^9pGm+c
z9BXeg!g_cS=u=~I515J$ac8gHe>N3ot#|(wQ<(yr`~q=WI)<$J)u6`@6O-WSKs}bK
z(P6ysuZSFy1K_O(y60z5s<MftfgD4=k#qht8Hf^9$}Fu6bZb_amTN$+UW6e~M<@In
zG1zI?E9id|b{$YnHQPG;7DPZ)M5IPgKtVvI2?&v9Kzi@(M+gvll^Rs4G-=X_fHdhu
zP`dOY(m^^%FQG$1&l~XG``&u*uKU)?O4gh+nPldh`SzZD_Us)T{7^{f0eA^ln(c4x
zIj?lK&aFHg06hUL8}do5S6<#3Y)?{Y{8L9?_G)Mc4<<A<bum1RE43Pi^7ET*&tK?_
zr<|H*?$=R$@PS@1(qmCcK{nX)<mkIZfZ3Uj1$=&P6_yv5fmlcPM!=&^vUH_Ziwr%a
zyrN<vA2JQh7501CFuU7GAbIZJ|8a@Q-u~sP&UI>dbGU7xplN&E{`Om!^_1m>1s@t9
zD}i*g%c8Z+z5A2=a(U8K|Eu*6$--OMBwLGv(CCKdP!oO|`UcVb+&ehj-G}dw_Dno3
z@W=j<{t*4!y_4JK34vD~#XI=gcdcF<Ut)TB=+eOg`U@baH0iyB8VjHTr4Szv{ZnD$
zbLwBeBYxR$D;OPyu816D)imM<*FmS^(zq$mfdpZaZBXt;f?;3uQ_y=+2i&qw)t84B
zPfnW!$`d%UsyjyKI??)N^ICwY-Xx2B{QQk#!rHLs-(cGS#`)t4r}y5;?mA)ePM7Tf
z8%=M*^$X-XBjq;n?k*W8UEd6HMzG&h-;Q|B?&nwdo0(7cS2l_;C9maH`hO%Wf+0bD
zyGS}^VD|-cze?SWjk6=3cX=^~!^$P2Jwr(T&8}6SQ=C!kL!r|-3&BTwz*-LBzGKwa
zw*JP(VyELHGi4>G3sIu+k@}6rzwc0{rM#N;7MC7-y;n=gqkexTwU$d{>sfU6XI#b*
zHwlvhjvx7<lYV(!Ref}06JI;t%FSe%>x4ZwQz9pT2oRfC8^xb=D7XChxj$fv@j9#=
ze~>%2|FcIoh7YTN67semY*fym8W^oT(N<GODkak2S$0}}a<RZ!bQjypR#><ve1F?h
z+)T<jSrmjV3HZo2^d(FI2_<7-I0I>-p=|<J)B1k%T3yLtne6>xThA)R7}Ulqwf7*C
zrACK-g)udplwbUZDIF|~cDdH7^KR&Q(>9c-o+#q_;oiPqpQ3?`hX=kcm8gbWWWTlj
z>$IA%iXr9dUZvT9EcR`f+iC}Kx;HEqY}MdwXx%5rDlc#q$&;PQk8$OGeFEM)BD!4G
z*18+*9*(Tx2Bk(L^NQY6<`|QfC(rwr`km=tWo-`^%QCRdhKA!q$tm7xKW!M3vwI|X
zxZc<?;?LSg^JdVfPe>@=Yl`t7J;EkhbI(uQ@Y<<0!S>!kN-j<z<`ve~mpf*Z$@*kS
z;fneZrET1-h<wA7IQRWJ7`L>kHF`B1f%Ye+pcQ%g{z0PrrQha#-5DCGAcc0|)WXC`
zQl-wu?LuqAqq~~-!jr3UdtdAcnf3_E*~@db2+EssbY0__?kz*JiYtL+2|~vUs$ItX
zyt2~-g?9wuwjV#;b|c}jFn$XSxx64UG0h?ihU7s}yYMS#)7iGnYK9FD_s=uf%KlQi
z`nLmaE*k4^*0#(N?(H%az9O*o^ZXH7RI6+JO`g2MrBn<5G#ifC_K|)xa9tkyvF_rY
z#Tn+Tk<Um=?ag#w*2M7n$A0%hM4}G7@saZlsT~=j($1_D{gSWO!bnJIsTp5d7=wBo
zxE~FfJm>Z6GM$}t4viMUya$VLhq07g9<FF8;(qWsAb93DB30MWeY$Cg`)9{_a{22#
zM5y08hZUCa`@>R-q6gn8YSFQ!4lD*2^D$em`Wx`@&)z*<Oj+63-y{)Wn7v+#tGV=W
zs_!>YGKVKs9BlZucDzxM80{Bg)>qXW?Y^<Nl2L=?|4KLYTSbLy?X$6s*HRFvjJ7@T
z^p7Uz=g{cNN9V)-fcwzN>1=Jgr#je=1IXx)w*O+9U1?zB&v#Es%l4{GUiplJjS}D;
z(=;%w7Se|GcC&#^hOyo&J249>b!&+%zL_hHwG~#lsBlp`OrGU}09<{JUpyB&X{f28
zy}hOx0CJnJ1Uku&yVj<RAN2k$OX|4$n&5D-TtIkgV#7L>LVjHNsp8X;!j<;%TYtg-
zV0cSN2s}G^`%gDp=8pnPkuQ5Bu}1{ln&^Yik8P*+Y<(X|J&Tg7D`?@2B9dWHR{HCU
zx-w>5<HnAA%7Dgu$tZn^KBu<E*oVASk;DZZO@j_pjio;rr6foDqsdfm7(zm+WIdTA
zWayoJ+)i%Mwh&#6{`&gZn~ST^adJTvFSsp?hr-s&yW;o7=u~lYCx=@z3<fjX66Dyv
zzhGvgH9-)FaSBzDoo)U^Kl_1|M{Hx(IY@B*x=|{*3j2PSR!kZ19N8(ujznKeIlcr$
zu{BAbviVyxc}AqQb&NM}W#=yiz}*fSAN~AGO9%4WOc3&nviBemb2=?>2J7rjQ2fzV
zVD12|JP?20Y|34z7zFgJ**u>)kowSZO^;&wjJl1kUA}DdT*@?ifMTNEdviLUlC3~K
z?&IVB2Re88`O`FHy}i{#b*k^Pv!{QYVkJ6%x^FROmy-GlLRwZ;!x9B~e%7U$xF5W1
zC>R>b8^(<1=P~A)lwzVHugCMs)YpHu?CoO{e&R$%F2B7w47`EV)tnOF{{`uGzt5^b
zE<0S6^6Y!>y_j4P5e7=imK-(x&=pzP_44xLRqXIp;YytPTP>!qHbK?Z(I4KWr$@(#
z^eft|g-%WNcXV**ta_$hhCJf{Y1EBVFO!0B9?7JaM@*2pWS5lo4~1P<%<eGL38Y*m
zb9!_)uk=1w4aO1S{o;jzc0B(>rI^UaDQEwqm>+Vpy{(P^>DbI-&wUaS6A*dY+gpZN
zML9J%943nBG>U58+;kw+G`0kXE2@X04OQ%mS(%JGBJQZxH;5!9S&@Ke*!k4kc|}gu
z9<D)w8}9-)u17EkZ;g$6>@1tw_GW&P6mq>8^CB`T*Rllj_7Wh)*}P|@T3@*w$15UO
zV}z0erJZ<xgJ*t8R7DUP8dVYiIpMVyI0<(OC3yR@Sv=Ff{sdHb*7>eXEcG_f763(9
zEBpt5yM6x&aN<srRB&VS^M1>=E>!pP)c1a*O@kU2uXrvF1ykE5Ke4y2KOa+*tziqo
zbKbl6O-7CAboS7SDfg_##XYcH?=tI@bF^EGOA*yJv_-J7uspVi^!1BpbeMZb#&Dn{
zZ)G)z9bVJWQ2$}n_kS^eat_kXe(&F74h_-t1F=kSI29fCurL)3+{c<PYFI4ea_X0z
zl*G}=&J^-u4AQpO?SDcqP7d*z!?2z&lJoP!L`HUAxUgDWs($f&_R(>2r8DMH-^5*G
zs)$fJ#S0fJZn5PxHMzR1btuZTf%M-OFMd{0X{H3G1f2_m7r%=O3J&`Rc+iUou`n6S
z*%vo=amyG&q1^a&S<kAyy*S~jG;wMu%Cj@6t{Ef{kZEd4@b!HJ50fLIz}q>LuyjHo
zUoLlG>U($B##xx_IhgBZ4IkW(=uDQ%h@5@t2wxqFSXi4#*W~t?X~$x3Pm`PqneH!6
zLf0Y|7ZX-O^L1m(ZN{aU0h)pwM&*wmXI=-!gAua9(+F;Vpa(#lEm!dN$58nmlg2q*
zB!<OGrkclNIBxWefw~Da>lNbLg72MADz~KuR)}})X$8M29v=pUL^lLT<$G6!VEkjt
zMXx8Xtv*+L<6QA|moR9*RBKQ-Lw_dtl%SKz+&UYR4Y|JxN8C3e90qeWp*+Uc<2yN)
zvhv&j!89#8A2W4r?Noy5y&EwxR53Ag__gszC~2{}pV`@DrS4Ip#(j#>%K|%FTQ4_^
zEG7*uQhcZanWOxG4W_Y9EX5tf*J`xO+w=!{VP|(&E0uHf@|7g%Zk}8GDnEGf7CY5D
zbB<i~gLV#0n{dYDokmLN6JgsYPKyC%S4#dxLo?)%ZnKA7@60j=ETtqR*_d@!z4C!A
z)9?k-N?XlISWwXQ5I3ff98Jmg8PHE~k6QZXuNm7aCQ8=Uh&v^gF;$!XAn${^p$c;>
zSKHcniP3Ah6us@@Qj#kQ<JH1Oi%M+B6=FhRA8Oww60^(!Vw;lDlz@<_u3mF_+J~9)
zEBGdAB113d`*-j1!BEe4)r<TKw|42mkAHgaHh{@b*hcTCrdrU-yFK%<?7d6P#-`iM
z2>Gb4Dwjn2p~Ow(^)nu=(%mV;BVuQ)IYMvW#CYPv9nH^$G)nB69_~V{tYNA-6$xP@
z0ow|6bmi{q*wwi~PJ5?lVb{+iV>O(~LNZc*_hMAEwZVU;5p);@$?|9fw**pB!U}5+
zszO717qOWf@b$^UTYQ$fgcHS<ru`+Pkh6}*LfD{Pb9+0y3y##TK9DALaCEdiPn>0D
zX3%4~zynO$`5;x4xds(gOzn}1in8)VhU{0gtzI=s({bI+2F3@0kX6e_imBOXhjNns
zJH}Nt`7xsMO(9uQGB1yDAvL$!gKc#ZwYj&6gnC6YP`e(j6cWrcH$!MVR#(x1p$w+k
zAkeuv@3xRr5r3ysu726DWEqj-J1XkQ;c^}r3rk_19Nn$#7cw+m-_M<LUHFvCQWt)G
zf|I+Jo0+9av;7A+%{)S@D|EJmeFHo|X3n41qn-QvET#zJfgy)QrJ(TG4ZEP=s8orS
zZ$1jivmG?{j=AaaD#L4Yv1mGx18pc8(HuT$oH^GXLzAUI1r^$XX=qT~Yzw2^O>7FL
z;MShDMvm4P&{(E#ZOZ~_A|rjPuhNy;pC0nj+Q=G`(DgZ=xCOudY5|w;kgrkuNj<Bt
z6p3)2ij)ssr_6Tl7d_hDD|g&#fxS=&11J1JoV?sz3%Z$A*9l-RkV?-nVN`{2TAF4o
z_GKZ}`X8cDh#XTEWTi__v;GoCp<ykLNi#Tedy?N1Eud3xxizJA&Ttsefgp!cdb2wW
zSb_Ew8%7|!=!mz6?-T!m__TZnNrwKLDE=3ToEaAZiS%>cnJJ0=mDLEU0@H*;zJ8*h
zYXLDx0|BH{(WMI)1k%$U_b<!)C;U(8pwwzeVCUzbUgKKm(kxZZ6_AoLy>VmguFn}$
z$!BCQy2#0W_=-pHATkA?ILRrZt%j)R68SvJDq~{ml}wNvEG*Zr-ZnN?=UZBeVxSlD
z_s`tlvjV|(+1V@PrGX3#{FjjUsQjN4gFA}<k75AHBO#(yHvXOW>64?Fn1_{h3=t7K
z3bkM}ZYBA0rdjp%Yc7A^0bd2s4Lwi9Lg!I*bQOpNZS;|y#q=o?7~py^s31Yw=!RW*
zIy~#{-H`iSgM0fdLDEuwT%sp?NrG?XX<J%;93NhXa!*XCpK1;NTQqzEFvKCGD3``7
zWhH$zC4FN9vHy|=QT!|{z2%r}wa6qr^{8Y9$<v*#a|b4^Enio2o1-8tT`D~SYh~}k
zDh=!|^z+(1v;U9{-B&>^(_1N(0NXRs@GKX>$&A$CWV;IBD_Tramz?L+(nAkk_M2Xj
zAf`QLuJVH0v4ML!5v(srS6a2DW)HU!AT-#{z)5Chm{1qse+X&MCkx}yKWc}pCbCmU
zTf2O^8c%ZC<^x_twFiC;O}Kv2HVI>S1}>S~^BrYAlaXm#NyT6=ir0Z!1JQazJFmU)
z(@?tpRCfvP)c%#{ZJ5va%p8=A$^}=V6h-%k=No}X>$S7$=AG=!3_?y?gx?hwsDu|j
zr`1vga|6F1f2}14`klW7E__Hli)4Cpx~{SyfgtcLq!c?0=4|(62Ms^*o;TDzB?w|L
z`}|JHwT#oA^ELR6PgBXw2t@&wtEUZUIe}lP<C(uo`p*LOR@c?j9&6p6a6T2RXG+#b
z8OOne<ncgd)!Rk#)DnM_LcXK)I&}yksjq|Tb5sRQTB>z)ugS|pH$W}RL*!m^esR?E
zHg0Gs<#TGloMEMYdMzpaUt(Lr3i1Qzg9u5rzBFuaLbtS1E=;^QtD-QzomW{ibiBOh
zQ^QTBR!Ko}xSa?i9AS=xFs6?w5GB>lx?9q<vHAyB_EuXT37_nj?_;p?vQS}{5Sh*P
zm|O`%db8{#{NU&Td9uQSU_X9&07-6Z<UlhTSysC<+Qa2V9J1r#wo>+ThNLQgG!h;{
zyVC3WJy6*sDpEBh{4wT-9B`nxa<qa+#Vsyp;<gx1>Jpr(sHph-7eTzxQ1ru`b<%SI
z)x4I6I0omGV>iTCPJ*<%MdK|^fnAQ;ZJ7P^>;ih7md`yYH*o}2|0&qd&urQ-IUW;N
z+vAMWL(qHg{cfS*LEtq#cD#&@?4%b%>0oqFbua7yzd3X!63p=2Hf;f`p>MO}fl+KV
z#}25^`T5q3!Vz05!Npv&=toC1)VstWt%dz~g1R6Z=I_CRu)|4TucbrzEwZ<Lh|V&Q
zNTkzC?p;`zh>+`AQY^{<4blz&jo)2qVs<##iDyG&d(+`~e}RWJA>$q@u!*=mV{AfF
zl=jMC-oY!AG~-{O0|$%gp-5O@r(k@knX{8;JshB-zre-AKua&`Iot4U5B~u^pVaYU
zxP}i#5SO#k+chH`EKfyd{8*@^O7S`x%0ekV{SdFdFf4V~da|aApUn&k<9FX_T`CM%
zJM#R+76e5=VYpt|2If1O{Wl5A9q6Wx&;rcDLF4b;Zh<$M)worGfN)cK_qAa}y2f)l
zsMgF-u6?)YG7F%<!Dvi#a~{pCs8Wua%j$@8PyGc$sGv?)L9A5GXe|nsp(1GXjF2IF
z({}GkaoLBQ9FG=SFHNN0&hDnDJFkQ!4!Wn3szm?vm9*p)Mxm!2zp{#HZ{1}*@v@lx
z`<M^EK545xTj{TlwmPv|>TR;zg@6X_rhGCYf?@kF9g{0NoXoGHH<WkUsg4ivn6k39
zrP&+16D70@pJpieALv#z9!eGBR!8y6yN1v_9qrRWuxu(pq%7?>QY!k&QLu9!&s9^&
zk~gh~K?^xg2I67C@pRU`n0V6?v*{w&F0R5<2_dINBLqUgb}ccWR&?aOnC?OVSc&<M
z5ngNM2=dWn;ZY=Sg1d{ztD;Kd(FPMc8=I6=^AaToTb+u{1M!)(!a{sc`edN=!3j5;
z!VY11is{<^@{}<YO-+~kTFl1aWJ-*p!0mkxZhxEF|CRf#cM?xUb{)UYyT&hdPwm9>
zpf)Jf%5knPOTXZ|_1)9~RMga(wk~UW1;4tP23CG1=}V@#WoCJ&t7a*%N`=nbSZ|{G
zF>&QRtRi%Z9dwGp3E>SHvM}F{xC|HQRux9Kmgwz^7cXcZo39*!Q@7v$8aO8V0DO9S
zIi9$3X-ewp>svl>cYjsIhw1^!T5Fr&v2jO&eJ!1#-VunbmXZ|!H{mnaj)|$Tt=+G1
zL>zvofJFIxRqHaFzA9>4w_z;04$og4*_BCF$Z}XE481K|5`6$DSS6fLz~LvJ+P7rM
z@lIM}@U1Nx_;>2*sqx`VPC|(ogk9{M)ted%ptGVIIa&02`fA0{W^(%MwV`4O6TXLb
z%7Q4@Bi~f>qg6MwXcy|x`a|lTwOOXI*r#}XGa=qrb@}Tj=L`pFtTi=bZNnIH4Q%JJ
zyK3lw#h>VmGP+~w@P)*nhNV7)IKuboZnQ+D%c_He!Y{!abbH1`kY|bQ+Lf-TgP~$M
zi2@^)8xbtBv1p+f&|mMZ*&I*^kfV5veQeMghue@CvlvKO!foOdCTdLf><q8bh>i)k
z9bPenqMutCeT$o#f^i1vr8pkQxw*M@F7ZUjLZ4>vu9n`H{lEot+fo~73DSFHJ%V&z
zd)${=mn3M+F=<%sq4@cEdXG6ToTr8){!#8h>Cekv$KxzN9@hwF4p@?sa#xocDJLTe
zv%a`Lm9#y1P&&T&1@88AQjR{w3nxGrN^n}AsNP=SuEIPi=}aB^P5}hSgYKDHwDU>T
zhTAr^L9MrbR-b+=%mj39sjhD#Ntm!T#|($J4BPsKG0bRIM^znd<4Z@V#|{Ry`swCZ
zsx5aDem8hZz1({^nH`hs2{m{(vT@P9FvO$$%T8<&tk!LHWH&`5`II0G)CTS>&$QIK
z!_!I7U#GU_(!dR~pPt79leH>mTvfHm$1BQ(19asVd~?OAg6P~qfE-%mc;tD=D;Zq_
z#!?56-)s^fur!Q847%YCvL8)>#V8vpP>~|?i|tD81ZsU%YP|}2EQViG1Ze1#J#9Zy
zvYI+wiZ`Vn*;@{A)Ofu?bXY`Fx6<`*hweNoURfWnB;ek-@!ixNjOPJrlkWR6JmEoE
zgFo`scVuPHO=k{H+$qhAlp=@wvew2CRW;q8hK}Zc8onwnDoRV69xERUY>FTvqU6kP
zU=2m(E|zoZUxz@lvu>WO4u0w9?zoYN#s?epNtl5Ieshc>6~q_i;uw$EWLJu0W+vI)
zZDeD?_INobXYkaq01}$Pz;ya~c4qx-bL_*Pzf6<oeiCx3xO5-gpv>~Wb}HqyN(jUj
X`n*5%RwE@&zab-~AXy-8^ya?+Th!kC

diff --git a/docs/topics/_images/firebug2.png b/docs/topics/_images/firebug2.png
deleted file mode 100644
index 4cab634311cdc6f7349c70d5f30613484a770009..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 69392
zcmbTe1ymeS)-_59!9s9C2pZfWxJz(?ySoJq?vOx$;O_1Y!5u<ycXzko?(dS!%s+4D
z`@gr=tCcLOyQ{nFR-Jp!K6~%m{xVV`ui)RpLqI^h5)&1agMfJM3IPFW4+{yNIUM!)
z4E{W`l@s{@Q8I|L10KMA7FD%{fOv`a^#2(|Led-XB#fPyq!7$B0tx~J=URhNIRpd&
zgqR?|g468pWM>t$uG^oi8gQ4vI5tWO-wN_N3K#QKkdA0C0@aGEtCeJq^Cq986jxCy
zsjI)Fq;#T6Gr^Ggah|s!EQ<{HmEaqM7_3-#T?=o_5~@HswXy8dP+MCCCnM)+^I?C(
zsnJD7|0^~3>ob%r#y7$}3W+IS5%+bfaEn%GK17y732&#l*OwTj&2=I)Tj*q+e~o=;
zRGb{cRCgHac8hI9;13HFDWt>IXAJCM5-5JBAB^c?w#L|V)lvmfyE6A(B4F^;$szxz
zymF`)8H2X}^Bb<FtNt+yh*#24-$!;roH0{e)TX!X+}_wN*g-(xiWJ%w_VM{u-+6yo
z<_>3MXJ_Z&sJG1y2nY}t6MH5?vg61a|2)>b{NCbr4}5Ve%51Mcw_*<_J4#uJC$m!*
z*MW4lZ1V+;i*lEQNFzpLxuTsN1->MM$LGl0q&WMQuSRLnr<mI8$A{P&()a?apHJ_P
z`Qro$R__jt<+pQFLInxMq@^z}FAwKU@9*!&Mn|*cC{*GQgmY%sp7&oQIR<hoyqYwj
ziuK4}wC}7s)On7p86dAcn{sNYy<hH0CjABXOzB9RK|<ZYnyOlgo#dA8A&IWI;((Q$
zY>`X-vgeUn2k)EMdearjWHJ@bqp06qSs}x_df&E~nY!&4C1t8VW^;=2tk)VNYLlL+
zSqZnmzc|_X>Gg`2vm`;`8~Yr+_@;ZaqX9YN25N4zl&|aZ{azy)N_59HAr$|OTi#;_
zOG9XGA}VXZ)?z8X+h<(Hb9j+=B1SY72s2CS^WX7cDWvP#(`CY>y{M<~cZ-Y1>{N-V
zlJV=(GuWg}J{8_}uy8(ZC>2+G-RHD>s;)9jm1T9cQI@6_Fw;+-Bpc&3wy+MQp=EeV
zqE^plkg{(uC`6tsbPo*J+uBx?o0^uTr4cYQN4M&1@9doKP0<k${2U)g=MM`DL*B?F
zBqAz%l`i9SyxQUA1*xgI@Ls?3>iSxDXL53qmX`MV`dYvj;h7_e*yB6sA{((&0;|j^
z0@#z?{Ou!Xqse<{T+QAr=gmDL+-N`Xz8-1zvWR!rAP%}5clY)TXzxFzT9qZ14{Pj0
z6mE5gS`=RGafIHhi8BtLoV`atsK#P!Qz({8fTk}CMcd1O^rNY$oKTdy)=Nx2p)KP?
zPW^r`g|vvT;c*GCOe^3Q#0-C(Uz3>2)1z?*C0g`Jw&dv~4!of)aGy>hJ-n$aVLdNc
zL7@?YKMaItd%GY<a!F|^uf=R73X!nRPHo$23A~mV>S0;u3e_7gm%4O+hsYUxSdW&e
z%fR<OPVp`wTw8~}-`S>=hE{rMh37{Sg!6MExtw8z*}l?6I-Vj*5mN3jz1{c2_xQ3h
zTM>s{`F5Y!mk_JHPPb1ylZdjAidfE{@X@;Mce8XUo^d95z3xHbUaqM^FIh^FD#uBk
zO_e}+5bnqAPscdcnUHiB#8rz^9Aq>pMCM^HvDY0-ZQP1&8OC<iEztWGUJJIkH%k$R
zDM$M&2T^gA<=V_WivK`Ys&ne7{T1m-Rrbqg22y)3sX>{2Rai*2g#IV@;~_C^kCK{N
z+t9nqoBX$BqJw(k)%T_AT~>YHEkt$HxdQn<Fwj(+d#TW2NTqOHAGSQMHi(4c4Gaum
z!21LT2Rk0jv)gSW$Kc`M6kQz2=Sp>ScEUqJe&zeT*&p8(iod3cg@yJ0{d;o0*~1fj
z)_$8ltD31OOjfC~fu;+XhOcC1%RMD`N=?qKR9@B@Nh&?ax9saF*ZBo8Bp9reHux+>
zNaT}t%dGMN9*gY;?@4Upy_Bh%Jd<w+q6j5TQ4w^!WW|dD;XnHFww$Qja}MelNC_rl
zHva0UbxGpxV2Qmh2sfa`diKUAqo>IRO=?^xnpm)+MLkbe9>c1*q-3$q*4WarM1{Jf
zurM$%P@ZCG)S{KD2N_E(<#Wa-Q~j10H<m02p^*-rA*yXX>f>Ik2~OVJwGDq`EC$?#
zke=`Bf&yH>;N0RH6CFRJ%5E+;xgH-M(`&fCO~$b<xNoyP-hQ=~8g=ut^#yqV&#f|j
z8;LZbWeQ41Z~)ezb6f?*)UXZDmBZ}E`Rw;4_pOL259JfWWf~h-J2Nd_Bhac=SzMpu
z4zCgB7MUI=e2A#I_F8EcDVGQ%ijn6eW#|%3W9LlU#g3*5z8R<~U|XYZXMa#pd6QVo
z^Cnu9I6XbRoh&dIn-w0)i}V=EtIZE-O_iCMd1Yl~xzK>rA3BAaJDhWjnU^+V!m@Pc
zq9uZnSCL|?^LHUJmi2*#cxXr-F?;;0sR8uB^?eytLkP4WIXz7U@_v-T&pj`1wy#HM
z`m@HEgFc`*JDQveW)XJo+$+v`Z4=w}!2Yn8?Wt>{-k6{ER!m@0Jf4h;ivw|k@}@C+
z&PzFBMJPsRBI|?s?ZtI{s#o6+dZrz_F&R~sn~unFmqxB*YJr?gPBttQbOO8;sAh@9
z0Zumm<!*!WA3x}6V_&va<&G3fBMxZXbe>i`7kL}gY$Awk@RR;|7gAnqXxVX5U)MD0
zYmQ7_SxqN=#xV(!jU8Udjl!?6UO~y95*}5CO*7x18L=u)cLh$8D>R0mr1MnN=6?p+
zX=Adf>kao1!CU~mUij*f@TSDn-AcV`8+p9Gq|L6%=*5iEOd}L(b!i97PRG`%q=W?1
zLc-d?K}fFoOu6Cx%{dAR3IzoPybt6@L0N_Cyxv}Mx4HiQpsuyn4cJw6r60V!w~gBi
z(c0z}FA=Ig)l6|oZA!m#TkB+K8S3V_e%N-n59)YL{IbD$_YwV9QHhB=IVuu}@mW&M
z8^qBTrS)^exa1JTjZ*AJ!_1e;A#`%DL(|fgqy;JUBwkfIMaW4r#0Eh~%Uw604n*6M
z(Tz`|P%hq_Y<%QNZE?T70I`t5VJ}XOUOh{+9Br_@Wg#O^0)kwR)fy4YIEbKo`MvOT
zaZ>5dY5_yf@NuWX;0jx@gIyvi(Ft<sXGm?*!%f^I!34>(b^nG2P6s$V-#u3sd@?Tr
z%1hA^gi7LINqxf}qq#%Ok^5M6tT&(Fu1_LeDO)%<$4y_sdNCcX8^z4_Qsl&NxRoSH
zWWsvIsWPkch2~5!2oYKad~>QQkEgxvRg@)%?NL&gZpmpZQM(wMKuWq9J$qHP&R|z-
zsnj*_kwCXW-8@ojc6OGMVmg>vSXh|S(VGH2prD|@Y^vn=`fjGem<V=tWhFc;?8^dU
zN>~`d>%dm_xwDU<Ze-QF%=2&K4NF-E5hxod$sbiroT-SP@oLy{)>d;Zd%L+R?|unI
z=(<e|Nwp}+o-m4i_a@t7LP#W*!6jj5`%@0Amng}0`g#A+Mz+X<omEx(q_iMS`|=M%
zVp}oOf|EwZ<F%pQMM`8?Q!_JGtHmG!Q8%}{?2oZA)T)q{6BH7vCLUXE3&wjc8+Qvw
z8&a-N{EfUU9QAdO{9)>4_8x)MQ7})@U!FN<H%f|h3`=%ocw)KODh8$9SnGOufT`{F
zOurOnR6a!!|G-RdugLKN;w66=BaDiX5(nC5Tp>!0Pb0uN^J;bC%ZBDT3p=Q96F)5#
zV;42nUX^AfBO&EzZJGz#9Bf&nS>^c<h$_Zs6_WhUc)et@Fdf&oQ9ukX=fhDOl7459
ze_)_%1%B1<{e5pQuM{q4uWV|SAAH29`_pCmhl@?GU%v*&4j8M@D(@p{e<_Nr88PcG
zvNKh+`o`JY;Gz(qa@Gl@q)wvv0GDe+{Y>g$yc^qn)M?Rmu;JCi^!Ay@<RHn}pgoqJ
z=J%0_%%a7;u=q|g`PvDzaPbEF2?kp;+=x12=Hf!7;%`|syE{7r3Cw*zpEd#lf;6yA
zZ2qJYTdnC{9jdK&3DJ0Gr8Y=kHas$!KCakj*-^lgEoBt*NnXT}u$?IBXKYV{GYlC@
zP*M^OHg=*`lS}JopYM_1vlGpS@N`B}Sv&EH9^3*o`<5I`kKbBeIgVl%Zg`R%H~L~b
z)H}72zR4x~Jg2W`5%A%mlSJSDjiLw5QE=RBNlchTAkBt3nv|3j0(7P!N3d`ro3);*
zYJ9z@=*yNkOB0h?kQeT6Fa7-d5)%?AwfQJ$XzU#vptbh}zc(4R8?3Frv}&B#xTxFI
z+<IWz2<(9MB2F>Klghl4h>)6!u!G|Iw$CZ*IVfJJeNdaLVfhH5^s<bW5=Wy;A^b-O
z33Ju>P83>rz#GTGsq3qRuznKnA>y?LWwu@lidZ({risPo0lKe{rNm-~NO`jyQCE2h
zt@x~u0eE+1vcLIb@Fe%1)7@7OGFa?xOL6r?#+!ml7W7ofK9viL2b7l5At--mQ`h~4
zV@t)kfT90#pxFKnazG64e1;>DC^2paP9vP}9V+=KTCE6+GL+B7uHCWH!H?VX+W<~X
zHWdMohg!!1dvLawb1t_Xn#$#aP8iv1LwX9ML1F+odb*FaUf~H}L{&&Sxwse^8?QAO
z52YBEslfYWG&l25Q1o<ky!goLeq(27nEfF=FFzjyB{q`@a*W5roilALRkZl&>8aIH
zGq?S&!p2fjN=nLdVKQvlRME69{wWdFv0!i*9p=f#u`Kh8M;5=h)0XO)OG(w%7RY#X
z!<HD~!xDpL)8-eeee}11EefgJKUaSk%E5sOfy{9mo%5_(&wOQ|!hS-+|3iq4nSn7<
zxvwO>64mDhO<xsEj8(+4m>?(XM-FLX8PT^G?4GP{H9rhiYFLtO2wiSMAux%K1qXKS
zi!kQ)MK>et9=E?2V85?YA(@h%D7d31I8wsLPyO}{DqV&g9e!<Xt)~{-iNrYvT1{}@
zh1<Z~ww>LeNOu0+<QYMrL;pYSP>s##@bA9$g+}j4STBMXFJ8E~xRjKX0K7q(?1rB^
zEB!Xi?Jzdxb-9y3=>pZ8rQg=Wqd8&b*0rTQlWT(`D2nQkJFs4E%h`|aQ58AQ7EtE7
z&XJb3ZSDH@z~MGnAUTE^XfzmUA2t>Cs%)U5TQGEhH@Y?(NI3RRO=Jck(a%|rZY)+M
z(MJCx^u3{9iHnMYJJY(Dv$?T>frcipsA#C8L$CAm3&?wV9ngq4P5=uQ6j18x>swhJ
z?u_N??|@Fh4-psDUl5#qUo2fs*{6if8FQ9oDeXq5_gAK5AAU23xaWyjHQ+D)ygQz=
z;ZB%rXE&P5NQ$ImDbDOy{X`N;UhN@VN*%2-yE}L@Z?ktO<xgh%LqnacX}jeX|F)UX
zNE;)S%JFpk@s3kn^7-#V%?r$sa-?rjr*$^iFbl`ej9drn3NUIHXaxX%B*Qyvc&%9R
zI9I+4Poby{rp4@?mAR3-<qQQ`MsZg%QjsAZyrFnU-sPK2FcUd1(j&lJ13hE7x;r7O
z!f#E*oXn5$Y*Xf^bRnOn*bEx&+rOjWAd(RhexQtYadM*8s70ppU*ht!(5V^zs$A_)
z@p1sE*w`i10uRBv>kUY}p@HE^hO^@%`)oZmN$q73|CY7PzuK<0`%_N*`n=CB!*SfO
z`_olQTpZrJRi)X@nQRgZ7kB^n@A(r-IEY3eWY3cNKpvyyQ^&UE3BxTLXuJ9vFXapJ
z@0@Q*QQnjs>a<^1yK&=eqc4wTa+5_pBZ62iUXG)u{+bes7=#D$rGjO6;#h>W$a=Bq
z7kgExViMW-XbuRNmLqj+2K?X9iyU3wLZBDrmJ2ul<pH+gDwD5pe|I6ivpxH}_phBa
zY{Eg^sQwsFr`OYOA?hIw9kI=l?ytKs6Pj-XQaHEva;7&a8b0UtA>hI^wsmKj@TKO;
z^<dMf8#kjOWg5hwGf|R>w{IIH&L3RM8Dd_Ow!XtXJ1Z%~x{}^rXZ=}o^F<4@$+?7x
zn2anoI@<o=<<E?a3=Iv9<rg}9mtSrbo=c}#hk6oZX~@uNgQk1|L)bF!YNufCr>yW{
zkbnKJf!6zH!Hk8p8<}FANR}s^FmWsXs+ZU%&}!_S#|zNIN9`?EpSj%;Xw2<?nrOad
z|0%eHww+VB%YA`Mi|S9z&AqU+)M8tBet9YV@nfWTHvO+J2kqSou5Mpu0Gm+eSHLNU
z)bQbCMKsfK^V_hHCE@`RGQCB3NAWbrM%3??E8FJE!TP1NB&Snz+{QUr(wobTSQW<y
zzSfx5<BreflUTw%cOo3?p_<WfBg{C}{ewbupM!%>P9&5#cx?oPtW$<Zl`aD<93J|L
zf1J65+)Y;pw3M7qmu53Pr;)~Q`>B)oo>9WT;vda@TXVh-!FL(*MyF#ve*Ui|B?ocu
z_3!TPjv_EHF#P@f?Km{GwLNDmKM@HB{Tw|m{i(a;+4iHl+QHhoXJ`nY!2lM4AuVx-
zxBZJEZL8)X&k`p?RJ@J9yf#^zK-2A%GZ`Ue8B)~cwag<pC46bt4kUi$b)sU4jo}}*
z3o_0fUMi?wkpc}18j*9RHnx(vK3hYnOHHmdm6Zz0%2RD^`WL@v0H#=efbhLkSv$f&
z;BT5A7Hs&0!-IV1QF@4}4}Z*Xj^jMI^a{m#K--<z!SD9m8>vJ;M{9+Ay}QUT=H0Dp
zAoj@fcN?Jkp~Ihug>BpjG*#J?xAV;HCe0~568e)mBSYoui-r0rYVj(X=@e~n4skho
zS|xmqvAG*&u;D`%q!mz~_^^)!;pmC*Hu$0qi5FS=4Au)4$_tcn+d7TAtY;~UMw`&Z
z!kqr;qQZr$G<^s{48LNcg+&yUltr^pg&x*?XWUb@xCc9JURJzlo@57IdV6p{hKBAJ
z*M}uB4eGr_3L#!Az4AEJI*Hc8=*5=7dn0k#LRHEnF6Waa&&Li&>0}NJY-}oISSZNW
zct&G9UXP$`Wo6~V`8q~cR%PnfQxg*%9Ub}OXrSwkhzJiyK|}L~<ii{r&HdYBciB2p
zr{{Lqwg<yE{sbKrX!<LiLVobmnujel^&VFZ_DB>?DH!KAywl<>BW@@2b7-g|t(jQg
zOpton@^E9u&(E)~Pn0^;Vz)Cok|_cvi6Nmt-&6ZM_j@X0$QM10yncGi`GsZ>iPm+o
z%hr}%L<l>2&-DKn1EGGds#o*t-J!NYjt8W(4EnikcaukfFm*&lT!h6V)%EdQIifHk
z@el-gz+!_V9S=`Rzo6&NX!iQLL3AXvlS|_E`gi$_P=j`VPPk59iu14DFMsSyo@#5d
z{rXjbH@{kZV)6E@&7QG?uue@3wp;=B<xBmF#IEhmVC)x=&#GyVVG;3pf-Jh)+qqoM
zzNe+cpcqR^N;)~Q1<|K*5)J-aL+rB!2Bx+VW|iC2PWB<l%=3qLZp%26k6k_X>N(oS
z4{kOlHHl_6>XMcvJC$icD%4sn9^AFOY5jrJtp#E}(--@v_Pj4*b#+ZfcXHmhn4blb
z<)0Jd+LA}4<H&LYB4CzaFF1<q)hTys)ow+*<wVG*%8l_Q1_XjIkr(_;<^`ITFhnpB
z#t`um0BCn^*r+FO7Zw!!YHK4w_3ssX&ebGocjBIewJo#38zC~FVVd8_`8i8>Com`X
zy8M7OD1OFms|pmWEi6d$xdHjdTx27Ap9FPsa8Z^|uSrQsuV@3Bk4<z94N0nx|A+oi
zts*`iE40g7$S%$GK&|!R?)Pbiqq0ZY04;s(^=?Etj4s!`$QSK`wl|z(Opj%JYa2gk
zZQGPc-m$S)&J8f9_Y5#6zmB4yrna-Tes&2Hwe!zblMpuI9@Dd%O&^LMI`sW7%MCY4
zOyy`X-w;;PLqyP&H|XNisb$I>3-|in^si3lO6k>@b}`M`>{?+xNA4ziO@U}-FlaY`
zUy{fi6u~}8oEsukAj_bJocav3D60ntmZqk&yMn1<Ve}TWOq!aSaobB?w`~4pmF}6C
zpP<zugkMw-6h;r&^(YT!Gc9C#-F+U}U!Fbrf^@R4ng5)J0q4+)!=t(`Mu|$5W@K`*
zvxb|C3l$ah$88`QDr&pldynTL78d)P1Bob5{-L2y6C{G<qs|!wH`J=OBjp2Vsh&$)
z8s!%!>6zxYZS^uw4BJ~y7Z5}JonuUObva{@QbIx<Rc2F4#cD4EP-Jv_Kb(R}$fG_*
z1-L{y$63(4-j|Jx3}C=|%?A{psvD$p>8abpFcxl7sQ+!2h5t^ZPm!Kg@L#!Zr=q45
z`#kDzzRGJSG^fXC_##dX?CPxZZ6bWh*j{LGY`IzuR2qWt@$q@}>ec6pO-@v6R2}mi
z_p0u`c8vk^m367$#t{)h;g8x(Joh4>U3jTzQX2MxsH|g5?>jp+Xs1TYcXCI)Uc98$
zsQsbeg$VNEaTmV+&@%$qjNDvSvq|db&!0o_LqbCS`t?gKI;n3XLRIP6my(<fVqJrR
zf#PVYmM|Y>Dn^#E0}od7LSo`XQU_Z}lcy5!B4~AWb$WUlKt(lmb?@8flvgz6SC+ax
z{X@m*Cpp{0?H@$c{p6Hs8Zgvr4kOf!>z+HOlE67X?xu0&$FeI2Hue`MR+dmk+IbM9
zA0UK9Mgo*MR;XMG#tWFZ)2*Rg@~+>@JNt=2^$pyhfeVSC<2CoPc7cpFiAa+Jq)KYV
z=x!|M0Yz*baYaIMZ;yYg&T@mVuWw%r&0LG8r;}4{ette+J7O%Ytd^7{KYsj}oSdxt
z?HM<6>58u>S(7!%_91b~)aSMX$F^$m(VQISx;oOo_t}eJW@g5CFiExAJU*ZT4DL-6
zZ<^_z%xb17wCDVtfZ<V>a2brM3sW&t;~D1^ZuhM@*^M&HM)-GG&8AJA17Qh^E+QBP
z<1bNB=+VlbOgWw_C@DSMoesTw_ikF?pf|MMJNI(?`&g^w;mE#A)LFM`UE9IbA=QD#
z@^>isaK2;WXnd}wH+^dFtB*Ek>P>S5--*;Qmsh~3f|j->A=@iA*Vw^<rP=krn<?sr
zt}e27@AyG);`8h3%1K&!NQq>qr{|qK0z`f4(BurxOuK~ivYush%1`d{IzsYF<;w<L
zbrRW}1<Mk+KUpy>CMG5>?&^F>o0EgX>-Kp<+*=crs=f2DvH_H>urgt5Kb0|WYK{>w
zY-+LYTQF{{4P%?nxRJ_KXeS$h$17tMu&1{%IY*F@kx9$V-P+w1-$X#phdSA$n)o?3
zC+MiHs9jNdxu|GZWCIrv;qtOFG528w+dah6xyki~0IfiiFZSxe<pk~ROOY9v553nK
zb-~^TR}dB4TN>&Ea=OE5{BUq^fDoNdtOuA85toA))n7wXGnP)<Qh)61s^dEf5gU$_
zpx`$J%JZWmmD<nmw4OiM!p!~bBupI08QTfOr1M3AKI8MfO)%#^z`>QRA_)s2=!Vg{
zSrTDKlOWUF^0kN-jQ_Xd3PKXr>-Rzfrj^N3v%3N%#yq}^5t#o1d-03<i6ZCA?>lYS
z8JLGN&=$Tn4n=%x^t6{r0sY6NTTk3URB53I7QRmPT2^0^89QzejvDv#tgI}FBsSUX
zH?4xDuaC~EKg45K-kj_Z9ag}}$;rJz5Z>~*qvYV2oteSn@pu7o_#+X7A%w))^3EfF
zyT2z=8rhgWZ#5ivVd-n)^W`{*q-PMg+xie+Y^fk1z{BC}{|(+l5K8?8{L96mY;Y-q
zd&}Ogn##7ABlb5>(kTQ);}nYVQ0V$1e*G_*o}QkOk&y)=PXz@9(7v1zJsrT;4%a9m
zN*r-^X6Nboc<^a(X^EGHMv^kxYvZYxSjOecy-n;F-x9IRBOgRlo81K^Xwb6`BSo<*
z0uQOM(W7AUr%In=iu2LKB9r%K+#B_pHs!-3%#<F(ZTtpGv<x#g^Z4XIt>w3gFA6P=
zGAXelma1Dk+;W?nr8Y-XFRouVfL6>u5nH&Yap*Dx!7xW-b$qJl$QPEFOy>>BDgF*J
zB%C%k2JYg|egM=#lgBjV=AM;#Qbx11=SwzeyWUl;cxRv7<w28;W#WAE1{!)UC|<W<
zQidj4jbpU~bM7}!ie-U@vup{u(@h_CHa?jDQDe5YwhQy~N7CQk!OF?XhR4N;=8m18
zpXW*@vgozLLh-}H!^_FbV<GsaqyS)X4>GHSq$CQIZp91;P7rypUbXf0gNdv|v$Jpc
zk<o>$=4+3SRs`UE7);0WK!+_)jt<3NR9vj7p>cA0>W6>@sr968Ls)U!dMjlnx62S`
zP89tz|5|E4xKId`r3`ZAH_IuHj;NS)n_FS7_X@!(#K!MQ<{p$!)@lqX{c6nh$Rkjy
zi<mev%IiN6Mw36`%!AvrU2B#Y_5Vq;Asw3!Avpt}vsU&iCTUF4PxN-wnEz4sfpm{X
zYl)XTJ0&FzqzLarZ&cg4-<!9}K}UHbE|kx^W~zMsIv;(ms*#EE&c07r`kyM)#KZ)r
z<3T?Vq5?bY?d`3ttyOLSRs~nq=Yy`k{#r7pBbd?OvLCP3LTfA+P~m+(q|;MS>~!IK
zwvXa-Iek7_=BJJA+ps~x<GKRZuF*dzC@3!O4f@k{`cjiB6~YGh@Pi?7v!(v?Hy`|y
z<=3T#!`|=mVkN8kNYJcP=GF6)PNc=%mzN>UUAN?1>;y@$RumH&PSmT!pPfY11hK&p
z&p&A@eYUYD|MUX<AAJRsXdtT)XFkS=i;=+d`pqj#`#8bZ2y-k_H^h)wsOmoHI0Kfi
z8Avz5sM<;Y>N(R6CMJ0blos(G+r#M@8KjJ_F))OLg;)Ki#>c<;`FRIEO}n3*d}K>q
z@m~=YZ>QYE?mgP#;*G9K{s9KnspW7MyOVYjcr{qf-jLW9iAmuGj$61f#;Q>sBI0~f
zxyeA8^RR31=+t{zC4$15OKngX;VAg4tLHBNtCi^XWjslb=>pfeC%mqba#+)htPll|
zbni+nRtXD_p9seHj04o><MU>%f8>#)1rV?&*LVAKHBT`*o-h9)T_&5$VJR1s)M`#Z
z5{-|KcXkSgi$*0RaOnJmpl&KC=p8BBp@kF?=>nutBQzr;BVH(s^1G+|vW%P7aJaTU
z(Q8z{PL(V3@aWKbZ`Qk4V92X^|1+OXa!KYR(0%ue!t=*Qm6V-hjnYUyDd^1MR7;Rx
z`qkFOh@glrqQKQLGxTX$oq6?^z?085-nR+Wc$@V-=jpC-%tpNG{@mEIb2YQ6re@H=
zhim|-SN8jq;>A53qwLJE%vcY@!-n;oozE5)7H;YU$m5ffmWz#8{9#vDS07_(X;jOh
zhF-zJ@gui)cf-y-y+4UPJUko(;0NUP`FR{^%)kzi_UCfs^7YEzxS>7`ieU9Od8g4V
zz%mm3Rq96BTALa)Ck!=~sQCpFqsN<nNOcClSyi)4F<26XHLjtm&9Qz|>5EvId!pI1
z(so%t%BxQ_X6*c5fLvitwHl?NQ8uG>$jrpjOkeeEHpvmSg*_lPCLEK>-pBS;Rhp}L
zPK=EuGMmMG{rb#{;Muchy1Kf(K14)B*x1-(V`FQptCK~l-UOoN=0%~QLs!RZC{T5d
zhg#<5zo)c4gGJ(*-b#|(2qJ&$+n_8h_>=vQE=CuuMBcKrvT=VcWcf%aY=52*`^u_)
zyZ}WxF(WK<JrR4uIKr;JdSM_us*xxt&OFI2U5fo5Y<T5emz73?G*>=lE?)GgW7tDd
zv{K8wwRBy-MPWLmw*@V(eqYwM57ETSmZ}kj{+_?U?c(C1ePYz4(jSd1COTTTfrNx4
zl1zGX>S;^ma<&l@gNv~=GgH^n3bBZdjh&mFU1g4niQ(Yn{5y9b1d}X`HV_-MTS{wf
zzY?L63qp28DtH{7>2Yyz{Tp@Nah(z4$633>U9TAmdfyg=ik7R7=QGc#5!Fef)2>5b
zq*rK%TZLq;<LY)G-p1jp0(SD7TU{7kXlMb&RSkQ4(qq*?vsEwr14efi`E3=GizCUY
z;;H=>VA-uwiVKaz(QTsCi8x)um;qW<e+4`{&1Rmnhn!ktsjO03IwkpW{{U}$Yip4}
z=;C(8!^c0|+40X#<#vtf&;x?l&CN|#7BMC!CUui?sTPnW3ByH+QM0qMre|h!cktcM
zWaZ>gL%Y_Foy5e%czAf6(12CPsp=^tAsQ#_E*0%rrSIEVZ*Ue)TrS~#HGExOu_D0l
z`xshqA2e=#?6k{u)8y_2cnX(H^FXstmfdZjdC^rlx~!LkJfPA|*6^cw@>#~?f#>hr
zfH-$Ww~~3#h@Kvr_ppk{orEk;`?;J|->$ss$8VvT%KRjmq+d<b%YID=)0bQ^Do!eE
zejlWzN?BGpk$hgaF}akCDm|`K2MjtbpQGFfvXX|P=h(JG03mHVScU6*!&Ey}uiDYq
zQX{W*ruc8rj*49*=uPX&XF@{4EV+C?zph_CFU$0_w6uUJK~Yf=8XEdJG=fUK9Ystt
z@U#FfIyWbW5Ee8gsOae9=$D~y71h;m0SlxvH~o?OPX)H)a0lj><&3AxbcD$F55T4K
z{Sqr53fDLRP5XEDg|BiF!0gc8Wp39emdF%;0NBjb6s8Nz)4K>(&7qr}9%%WE?w^cf
zpxh8AJ1FAnMb(7sMoA=mo$Ssh<&>^_lVIY_|L67X5fri`7RVOhHv`)?bmdQ|KYI`$
zT>o<i0)^}!PyP1}KwE!b`|lkHWb40>?f>lj4<G)oulwJ=<qm*8QBhH4(HWtMe|)Bu
zb~_xO8^WJ2PxC_TS@svt26JvGp0R6kvf6t@#lXP8SZqkIEFvNzATY2)udJ!*K1(bT
z8VPUSqS!10JK^tnJ2?_cae0`Un>TtsJ^*Hn#e8~LK+w2irseLW4-^+nS`E=`IVwuZ
zgT<zgIR6|G_xsiMv=I|DGO0%QTbAlV8FG-budj~v%T#`EY@F}prpo0jY@Ae8Re^az
zLrWXew{dWAkm`OB52&?jy@2~JNL9&mU_StWe{+3ZCVw0bjWiEqXNdPPiRshm_DE)#
zrm^uBc-Sbf7Au~ONv8#HNw!@6T30AQZyZWml+jvsHY8+ZF#O0s0%5n^+}_>QvxkO)
z%F50@c4`#xLtLTCPuTSYjo60|kck5<@88EW81{CD6TL5MFot{m_rm!JCjraN&dv@Z
zBI2~Ajg5`7vopX0&CL(dRLZDmXn;;F)p!7@`E+wYz1}WggSN$fZxS#KUS3|nx25Ps
zDS1BM=v<MWu6uj2Pae<-I7E#6yu9wEC2b3(E_Ew&^Z90XPA`J3!Q{5SC2%D8=-a~<
z5m_b{mJ7gn)IuOY6EQK3cXx}bB1oQtOykh%^#Z7FZaj|{`*TkJ2(Kfl(VIvC(4>Mb
zk=4rMXt{MX`(s%<@B<Acb4~$2gm^Zni;Q%1(i?e)&9{RK3!2KxvBH^>sXR?Te#lsC
zB<%L|_a`wLhk)8yR(71kZaa<#=uid1#z9kVDk>^qN$Y;vOaNo-bX`<b1T=qrxq)O(
zRw}BOL%(_>i?o%MYnq#z)6-k$=kLa(c{}>rApcH`L*lH1H9^_R>gtR6I$H_Sv0uL+
z`H}HLYmA5R8WDUx$g&4W<!ZNhB=;DI6QfqWpl6zB+d3`tMTRN^M73)XxOxHt-v0h#
zeMKQPmX?+w7HfLIDn<|5T{j;5AqHk<679!w!#>NU=KHNwPf(1&$+<c^xAOg&w98E)
zY-KV%y}gv=<lD=w-oP}HDVGlzIbdE;#5U4R;;^R#!)t5v7S9goSRF`YEu_iG#{-DA
zzMcaN9f}`V3nuUYo?1#uN-ERuauC|XL4<ipySKZ`BxY!CE@93pdCtksUaV3EDKr9N
zB9+$@?;nlYqQ&eHj}$;ufN$^bmw@6I4#jg{`Q?+4@D>bQRTT>jEmKZ~GJ1FWX&axI
z(5(IZOd8bA(vp%+N9GAegYGbJp3F>7n?{3!WHFbAhX*JvNccQ}DPV{XYSjTOcDcOT
z<;6uvXecINu)p%v*@K`5oQ$5FW|Par++5<cUc&B^s;j6N{UM!CR)IElZDU0!<xiA1
zMs8Q;pMl5+c=5gIIJvg=5`c)>9@p8;&5NC#M!iuKlq7&}0+K1{1wNUYa_Ij0_WQRv
zwL1{WU=VST13RX<0~2<y*CVBAwHn_5n?kl6nN*UBjLfjy8#A*#tLAG`&<EU{@1gp)
zx3=nnw<S;C3?_5tE9}-<uW2<pea)6m<+(rE=mSnEST6)TE~lHbo%YR9g_hXZDS#|Y
zO+f}S2a`$%{V&k56!>QqfR+5AQUo}BTvtonzbVTE<|z8(`O~!`M+Z2BjF=b!hYcCw
zZ(<y_wEGEVTK40|_wV1sdJI5W8q1M<?)~?jeC;zmLD;O8^R?DGf9eLrmyqYm6aT5P
z{Yz~7S2zB&<75W3BdFl|S%UpMMyh1q1&YUoJzTwr$jHphOwO9Yyu3V6p^kuGcX8wU
zse2sF;w#*x8s~fNvYxp-NLt%<-+*8dT*l$_!%%BtR52Hq`j!^YEv3D~!_=4<(rEG4
z{AUm*_?lM^#2dC{W$_olEwqPtQSF#rU6oRjNVBMpi|ur><k%?naaoBm8@f|Xpz|%=
z-DIb6gf&dz{yT2Pn~dYBVhpTLuQH&WQ@`GPWSx;pFkVa+_%MUGU=<1K;MB9qKxuA@
zs2p^3q1R44nY>o!sr1}4&(z;)2yL#qjQ}@}=GI&8VYa%uIxr2Dl$53|fpB4JO6lD?
zJT|sBRjSR$_a#4{!i!)imAA$9((==%e4sZ24DfrVA~-1Mhrovq9}-s2yG0w-%*@O%
zF=K%i_D;a}CrD#_oZGX^g9vse&>Uyl9m7;US|!g-u(zVm+ybExX}<>+l@!J_wQ+dV
z{17A+@@$#Al+C=H_(0bp^jdZ$`cODDstMhbJ-xeYYF#bd*rn+*CsV3s$ac|z&SEbv
z3(Mr*-rMgmhwSqw-@~1I#t31b5ZVn*`&U(5t``#5jGrzK7QlViaB{i|!Q})791!UK
z*}2)-l1XgV;5GGX^O32kP=EhcB+#@fgfLq-HZ`U4xWiYFLO}c}b~4B>v6C_8ev7C0
z9~r_$E5e(v0plQKsTYciDx>DfTBfOjjh`$->0M{aP6Y1<I<xT0(+r$<F(qM{A}xh+
z=8>GU1g?F#Bp;fS(pakedyBZ2uLwV~<sOdNR|FhttpF2LL#}=&_e0IC%Upw8RVBsd
zBu6A>9KQ74$N6D;aU)<{APcz$zPaYfx^o<MEp_)~bDPq6mk^w(52Qjsv%68-lA<4C
zcl|Pfh%viB)aBT$Ro>&UNNFbj5to~?!>o2CG<M-=S^pBglWi=?I)@#F3JZ|;Wdr`M
z0XmLvNaX7T{MAtc(K%zHJj~;O)${NyRME6_wAx;RcE$v|q7;59ct*L(E|R9XFrSE6
zkLma6W2^yhVRndVLsf=gqovZZ%Gz()$|B}5EmXgb1b8!s5Ylk97l*JL2g6xxVpB4O
zJeD|+U9yF0TQ}X<y!E}kfFnFw0NKN6`R7-FtraNk@2-x4a;kDupjgPo#g(T}5FS2K
zq*}3U=TfKy7zqk0DsNRZh%YJF)`YOXbo-=!=byaQ71S{AS{*I@_50wo>9KZ+?y)S*
z@M}Ml$C>xTC7U-{Kg(QG`Odtfp7g}2gsypv2!j~`a3pL=Vn9_@&3*&^9?e`$O9mLJ
zIDL;5^0^@%t0h)L=|_~tj~s|^Ezrv%!Y@S$UQ;NPGckW7=pOTgDpSS7+8!1=?JvbO
zUfWR)odOIy{rjm-d5@**`t^8n+Bu@|hj)P#pZwtW55mdv?<ihQXbU`^-%kCW8a1P9
zD#378iWUp8aFsMaReB+hR+=Ud{-}LHI#bbA-DJ7kQyS~x^v;tHsIsPoAi9&-rhsoR
z;qu2bvnX|{MEb7A30j`&Vz0Isl2Kl+)V*-)nr6c)(*|Y(BWoSe()C?_M-}g++Y!WK
zDzYC){JO6?KM7^A*u%6=ldiorogiCdGgaKvz?bzM+DeRz%Tqmw79AZMTUc07H^O-R
zni3{Ifu4~O^f2^hkPyRCPr_`2Z@Q@O$&%CQ?<Y<L@GItPXkCpjMO2Dx%4X-&MWIWT
z0xSZdZ19sM%w2tLwRT>jRBNh}#LrX3J_S3J!NQjGUUDws(Oii>w}^uQqiyq6MR;Y=
zj+8Ty^p1B8+F!jAVkn>@o@TkPGpUzPkthH)j8w7zb5Ehc_vos)NzPDNa=A~Hl9ZJV
zB5rmJ?_1SYjZ^!`P1~!nd<c2{H1rbDE*1Jh%&NZ3D&67O#Fr*yQM%6LiIuVhi=hjZ
z-VJJT7QPW0nUD`G!84}%n2}Eej_T85naj-<Hf>PE3g<g8OU|4`EPg#{fKx??at6!q
zXd;)CWtOMdqF87uSG?y&%*3QX8ktWF4fC{FE!0O37!?}4kMIBS<HzpqZj4`E767!v
z!!m&HO4GF4cbqtmP5};rU*WreQ3=hyAR%vg3{Y2a+BoE-3YMsv65$9*&Ufi+@fV8x
z3ETyBMLV;4_C9u&kH)>2wQ#0pRd5Kp-m3=~gTNRU>zvW^vfr4x%q`ZAiDgWyX-fB8
z3kF^2jZd&K(hn9o@@wXq<g|ByqTIJnq%Aqmt#jVZ)mo8KHA!)aXEcclFOE#Hxo3`+
zgJGTM%AL*n(BRrWF6Y-4qdK!w-CtnX<I=#~v$HH*tl#t9LQ0PQB+nwdTHI)Bo;bQG
z5U+J*y^Gk3fH6h|Z|VBxqOKmC&T}~TZaYDX=>FEK86KKFg}Gv9By5jG1=%<nbU1X%
z=Q$)$KFnBtDp@T2^wmAJzG~+^3MEQaB}d_;7(u~x8YS5q*lk1%4CQ5I@430TX=yEi
zAJNGLu#SKL0pPZ{xLBQ;Ahu7j!37YlM@yyFl!pP>7(|~`uRcVH5_DD9G0R0qz`wY0
zM0E_XpzWE+l*Tz#k?<#duV6|f+I~fr?QyGj!#4Ii42w>lcETSU<1i(JMmbS<+YpsC
zi*Pz~t}&Y^6|4`iYd^(Qg=7fS-Iqzn<u3ngvz9bV>V0YCaZ%UKqD&Q+V@Xocna{E?
z)O^l7coEzKtpDzI4AS>9Vm8!GWoatoqBg-}!0>-@&B3FbBa`-uE1uMSPN8;&?X95c
z(p$1)Sn~<y?^A4I98#Fq-x<iHQ~LY+6%t9pMJG$Oml{=4qN{+HxV)U9%16D#+Q`75
zy}kV>$14bk{mjIJHS^`3p$CPH`O+O$_a*)tV-G!Mc#H{*QEqMKrGRFVFfF-Z^2iMG
zfaSnv#V`Zfsv<=O^xFO#43%u!l^bKIUbAWztHU=Z9^U7rubpx07mY``Y`G~n7RIbM
zt}@mGlQ?1&-ifGgL=YuXH44807w6Y^3Q{kZj*iTU8{yge##d^zKklzAoSEf+ZS^7*
z(OCUSYtfwh-~!m6J4w1HOArdDa-%F;A;mQgukwjHYgQEjqp|$w>nLfO?-nIW^pXn&
zjiR08=ER3@<$0>uDpPuj!>!E>5-h#Um*O<)XD;w}!brr25KtVyX{U9h+i&1hgHLb(
zHlzLePkojcfijfA86smYl9*s{$bsWIzYGLT=Ss~`^%H+Gj~6WE^fnBNL>-@<^$!kq
zIu509_vw|P!wb2)y8}YTqs2%~O-)a470_z?+uNd}UjQouTC5)u&ms73w`U7?ue_6P
zeAT>pS#DN7jLEkxJ+3sGu@g}?j&|}p_K?nv_ema>VjWu5>kd+FQkB{As$HonM-)>p
zn1iZlI*~<qVU};pJ9~1t4@}Bc{<iGT7oBmH&1)K*DqV(@f#>E|uasjudZat%!gp-b
zo($wfZfypS=Il-WZ_%8;5v_DQIJVgXGN`I&sswx-F26@|o~xKHY1b$LNzfdfavlbY
zdS?1%-wo+6kEPv{FFevHP{+wu59gMys|DlmIr-|=ULRavymF;AmkD!|Un}^03qC)G
za|8R0*$e1Pmn&oUt}cVy&Qb8fce*|!wymcZd}euSdx*x0=Q5K4cu*RoDYG7a1;1~<
z1v%1?H>(gABudHiJ-z3|pp)h6Qx=qrc1#W>XEYg3`}XY{=x%;TC;bncj}kQx4<3J5
zlk4Rw%)wkusCYJ^_~BYR%MVtPvKe!ZWIze&#W6nt-KaYLLb_BQNMDFAk4dAL!Dc)Z
zQq;$g@({T1NJxBIb*QzQ8`1WHLP8uI94^k!Uz~zsR6!_pwG3!$KFpCeUZ@wqD;H5x
zUw}v^2tfeWMof$}P>}=&lvVUs-yN9iWW8W{0`>%}P!MU1a0u1l-!smqKH!l4N5Vf?
zl>d#jIdpHIv7l4%Vr3046Xq<60#0XK+y!nu5g{SK`0-u-=^6x+kdK=F7r8BMYs?2Q
zw!%H_OTDT|bZXq^8;_HhH%m8K+((MZrryhVx8kyfZKz}pjcO<Bo167@b-MVIZ9iM0
z{oF}PC(izR4HW^bSH?D2c7Vs@CYmlI9Q>9Uw0-bCUIgC(+Y%V~(Zk~*?47{R;HT4D
zz7J-zKEZ?gVBDhS_@ictFfirQ3$D>!r};@9NEpn14bzR9?q9z)E!+imD~yMLu{1Bq
z>qV54n^@AIQAK7(hS5+;liRh;SH2_Ack7{SWr~CXGU<;EzZ<k|z(&Povo8AkiGs<M
zPW|wgt>8|jZ7Fz=wz>CZFa0%Czq~@)oJNk4shb-6Ph9Eo+N0ZBz43+C3!<}fAUt%5
zOjH(C{0D}dRO~xS_)k1-El>=2?*By8Y?_e}#io@PU)BPtSCX2grv@mSU44k6)Am_v
zNoSa_B<tjTWf`JlT_<)k{6U%pjfq2```%=6zI-ks4x0?n;^*hN=;@I^dY-KJ1Yt1(
zZ4hk6{c*?ZJ!p5x;CS$^Bi*Oyaatl81b*;_7)C_0bF2fiQHIOuYIsET7}3Wgw-S-g
z1s5*&|3sr3U*9nXJ{NEyQAVn7d%Hd#>R_VDo)R738R{|oh-ZBCZN{iUIR=GwoRaK9
zfqMwO)5w{)0$T383e;2nq!O_&O3Wwm290}_eUh6$;cWZqa13u}w)Ssxe(&&lIZSIr
znnMW-Ja|U4KF2L+Y%f&8lsvk2Lqm&nK#Kk^u}D|`i@KuX#`gAi3*aJ9QdC^=i;Isx
zcH;WS@Lv8|Quh`tspH{@&sS!n87q4Q#I*9*;AxgjnQytCt8tDiuz`xSq+<c11eaBV
zt!A4O;>UIX`>pzp_ZBsBwy~F<`M}PV6h}xLM1OobeU?w(GE64jA1W7}6V`D84D#-Y
zVuF8~nA%Te1(JRc5HMs=?K@ju)~VW19GxoBbOlS`zVZ<u!|Lhl1LK}gw%@mJz&2s7
zo-R|Q+3=cIkTlZA*Ei$oZa|zl)nAmf7PoNjYqio>jAAiN<{4T(p%O);Ifu`gt2?))
zwL(7156u;$z(wkwwKg4}LkY9VZGPn}FQi*7Lo7o8`yrjlI9`S6lOW@8M4!y6uUdBf
z2acLYMmJ0=QTck(XKE9$0@G==u0FOzITwa0CMS@mSRbjG;c_l|-s#Q22Iat2)tmg9
zep>DU3Qe_D9$4o2cX0&-f7q8VU%(nG>f|TxN$+j2FTr;e>KwVmhS0tZ2LnZnr!jo7
z%rqk!RP8I0D7Kq~K|kc$V6cpd7ca>!mU^MNI_(!gAo@bnD;Mdi`$$I*A6*D<adls9
z@QpgV%*Id2m*SInpK&&f^-MBo*895RH<1I=^V5~e%E!2`^IC$vX?yG>D+@+|<TAQ5
zdj|EST3&0bhsJ-(5y`F`+DtGoD;${(h>S$@ZZ&ai|72xV%CZ0$<i^JNg#}#xu#*!T
zb#-;F!ph2V;COKR!>+sVXg;hD*Qy%9pxyjd>#X%CP$KE08(tl*w?R}_Ql{ZV9(7Y+
ziQ#h*<Ij``$RN*f_#{dFA)hG#ERSH;#3_pyDL_bgYZ_hmUsz@J&$*_bZ`?!gY15o=
z*fkA-Wu^OwtZ)Wq6DB~?3;NaRP_wtQ6Q7*CaM0%N<_4T(-+%y09uyjSd3M%5%6qe&
z2{anKClv-z-)?ShM*@J?J<@x+&=6+^^=q13i~2zx4RekIm9aS0=EuHTPB;Hm8E4MW
zyPZ78UfF8B*Y;1o9RYc)y6CfO<!P&Ze{UTL3t~NgdJ}L3A-_xm7hAP{hQ&yGqCiaO
z_mte0N9D%S97?WPhGOvG2EsPAnOQ`!V&)5<3;4Ge8q;YeOSkFJuii$5g~`avd$&ZO
zc|M|4e{FG>x48LldyJCNc%Cds?7VkJ&#Qx*7=RASk|>!w1_*Hko<HnAH-7JY+C;20
z{xbmxT(W<aPVCu$f0fRP=V0i3L%D^MY%B6Hhs5oDsS8+-_R<<JpRNgCXb79@Z!*F^
z^$f(K@JorOr@{8WRmXo9H~8jumJ7?faZ>)4$mSbRIe=FtE%DE%5Dwy@+(2-TJ^ahw
z?e2Gbdke@Hv23|*f~RA)3G0ujVx63Hb+R^?=jqRk$!eZe9w1pqEg{U_Tfr%!d?6?^
zhW-u+Bg|A6xrax_L-GLXBR65s#c6R0jSx`e|NJfh&vk?7-u%1<D=X{WUc&!{qAL1-
zmsF<TiShrw(E2OmA826rLbIV02>t_nev1Z(I2?;tw?o^9zCEC7whJi%bKoM(3NSx<
zE|@$ZxC85((e&D62Vn16J}>jRvC`0301mXl@o=%+FtEAVLqY-}=5Ij->@dJa2Vmhh
zN0T$3i>oUjZPBIvaFIt!!bxwyN;7_J^PLxj=F0CxLOqn<&yYG+@?XSAfBfdg9zz?M
z0W<a9z1Ws4ZYzoV=Py*iIISU;@mTjiic&=`PY0kL1IbQ5xI$#4d{V;IUr1q^aF1S`
zw0Snb3M{B;P^RyU-wa|L@K@;xW1>8QRQAvxvg^T5kFDyyb(RhJk<DR-#Ma!6I=FLd
zw@|bS%A6eBtYu%GhM`i->B;*G>Y7}leASnY@HP*YVAeYT(?Og95N?&KWLVThzG;__
zfN6ly{Dg{kNY&kx<sC)aO{TZ9@v)7h%LX``QYYm2tK5sL(RyT*lr~pay$(y%s(Swr
z^KY@RK+1d>N(~xoX8Yaoug_t?61DimL>Or3RlnNW+NkL0)~8>J0QBnW_lmD%y!ZNw
zORdn;2jG_uh8TSSC(~2(Rl&VDl~qw^r{&O1^F#tYc0XM7F{~qM*^l>L1UYfzxKxk4
zwd{YW#V3zbpUdo(Fv30VADC~e&SpQBnhB*S8b;{PX|1Wof4bX-xj(dqmX5QicFj&8
zZ#RxOE*YZ*Ke|BH{~oBP`^Kp?hl%C6kFXq5Cem`dkD`aJp^Vd3;T;Lm0r>Ol_c61q
z!JN~M&Cc!O!i8$&*u_M@q?63FlxYTBr@yHB{$X`+h9<^wf%ob2bh+S--<B;m!@*=t
zk&=aW;l3clpL&i~r3xIUPW1o1{M7t`UDwE{bM1H;(9dAg>;O5zWQrcW=jUV2?1s{C
zhsq7YR^uopp?O2KI76K>Z8dkXBVMM=czMJwJX~|(j*6imWL>E}qDQT`#k6m#J8fnj
z%A65;0H8^ZAsf{wv_eCrj0#PS{xUhEVQNI=k0^nx=snAmOo8d;fggp^e9A{S2)P0>
zls$|~lNEz9ws-S|ltU|`H^#?tJrL^*=@MU<<(JMjlg{|lz`c4E-D5yA7B69>qM~AN
z&s^Q$us>_NH3;`4Zy;l1yY7xtG-gSVz68wBpUkxV)~=o@3Cza~ayeN-uwV)h4V8`~
z0?1i^i*+NygMzqZ_IPXXY&oi#tEQX!SlPcb61_VKc~3OQj_w?y<v54Z=?e|CK+?ov
zgvxb>V9aEnR{<d0S@le{H~mTJ=u^wuBYwhCQW)UPuvD497BIw&4d8oem}xKQ1jTPe
zUB=aXdcJRKPj@N^gL*@5{Ao92mNMSktU~9B37N9k7sLUsS=v~$PuIS$!NP*FvNGUh
zsj9B-lsh{+Ga5($>$$!Vi1P9A0aKC|zzwEqB%qxE#L?Qeo&9rk_|e<vy9X7t{P<08
z4$RDTiYyFsT~A~^v0)~{x_hN+G6a8*$8I}2NOoDO$&>@!OU{cXfM)Sdn`e;8Pld8i
zivz>XUmu^+ivWW}?`0y(^okfI?~a{?3*?k*(8yhBaaTXnSf<>pF^&bUc&5zPj;O|2
z7T$nyM4H)mjd;-~J8u*B13$JbT9nGE$UCacQo;$x=IwXIxGzjfwOiZR=+VFFMFCD%
z2e1|wte{;1Xp(GpZqBSA{Vz`Vo6&}a?8a=nSenkEJ<ZRIHE_KMY+K4hRC;upQTq%{
zteZ^$AFdJ)pp@rbbz=g>itPr>WI>Cc^jMi5ig>o{nTcBIj@})M2o!(NKqZz>FwI{4
zfAb67dp0KS&MmDs!0m&mdC2%F#krwP68!83;wbMaTA^%SUP;)&b-{piNu@yG1zd|=
z#Xj%oyN^*RScC_NEqs9jd%f)Ds+v{5`MxKVfP0s~QKrbn&0SGZalBAjSqYe!(v}A`
z_IW|%_Wv~SN^?<>k%>5CKHa!4H4Lj@$@~R;VPTiqkE16eav>>3FiZDh{dXP@hp|X<
zD#z(Mmth{<Gq)J^=H$qfoMDmuvcKIo=!@<*zGd)d1|OXOT;b)*ZRLz{n+WR~s8#>W
zJq7yA+0?M~67>5g0ZDyWKTIr2S!e=HS8sWf##ZarBy3|sJ&s2N*Q)Ur5otJNoxBcI
zFf6UiOBmbOJy-mOd_$BoV`rNu$Nh^LOoj<KlC=Qt*zfG>vSmvG#mM)L7Z|7b!@%!z
zfJQ(_Q1JJBojmpHp+r_Bsv)C_9lQFI+vukuLTqc01GA$xk2@5?v&IQvl%;<Yw9yw^
zvo^iGZDMR}tgWq0|Hp%BhoWhZPv$0#x~=iOMR`2jsm$G)^3O*+y0>2|xpkoYdGP<o
zcQJhaGcfy$qfYHF`U9!I+M9o$;GdTKOEI_&KzWPLs27opJ6w8ABp}Tn>WJ(w7?F~4
zaCj)K(f>~`><3;*h@%prq*q+mU&Bt~ORn4OD@aw=i;$eoiKC@un*CaSRTS^ql_VU-
zy=Rua$*^9VS4N|;6_Km(;&B0t>R4p9MQXSrWXQoGMHPy4S6kY}1O)fu|3leVhE>&d
zYg-5c(gKnaQc@z_(hbs$bayvMx3Hy4q`SMjTe`cu`<v)n*LlzR@$KumaKYZsUTe)c
z*BE1tao_i_wEBZW1=aJSIZ(_b5)AeB^);I+c~3&(+qv=|iY~N!Pyd{Mfr>I*czEI9
z-1YaSYxz*Gc{snUgKn5Kr-j95RUKQ9N8899Jqekzgp2-P%U@0WY;yw~(3?jROQ0nF
z92%j`{$Q^5vA}tFc!<kpH#su{Fj^>s$LHs6*Qex)@F3aZ3lcBG1)ARK86GqW@$Vza
z5b~&!bs|5_Q{MDR$|){O5kH^#-is#4_z46W0$cj7m4}f()qOFlT=JZJt>-P~`hfNZ
z&^`G!O)vMW4{(NnK*fq!>BN9btW&iFp?sMnZm|&FJ^z*>WVe`Ps#f7gAiouz5uFA@
zLE@EF@RxBRqxDKR{&LD}5zi#^zD9Ga4QEd5f_RozmV;^hT?A!^onH}?ym^dD4rCFQ
zTNB%0dWcTxz=wM8gOVLeNSZkCWwv#bit^ikV7J>dOu^Fp95*n!u_uf0e8`sN`I$OB
zK4Yq1{ycB&DKER~*!Eb?<j~6AzNIeUQcRjuOj8E%p<rPJ9*qA%c&37-_yDC=)xly=
z_64}-c_L56G^y@u*NvSB{a;lkVZa$vX7IhgXiLi3h;5mq4AU9s&S@AQjHfuv`yWN7
z;Gc}&L@ZoL-X5^-SwUDVT5wJPP~Pp_@uq>?`8w=p{`T$lo$<NxR(Z7gL?C-^4>2hC
zkfiE$^({AjSMMJ4+An&^a*>LwtZ+a6*7|>D5xmgdB6^a6l#gi?2^<m3F>(5R*l7c(
zaNVIR<z1y+HjB@jXS<G8dAV-HgEVPbc?`+#>A)(u-{&-T{0z@=r;kS{JVV>^E5`=f
z47QDzmvZC_<=cJn%<aQI06Yee-cZiazOb*Utz9A4)6=`;w*$|Uy?#h(5)oTTEX%^l
zl#uecO=caC7jdu^-+4dkwDA#^KTci=WE3MC40`haZ1{sP7RMO#^Zfr*gC5ri5BJ}l
zzmbB6|JYIKpe`DN)=V%@crq{bTZTyFSF3=~_X5f(<0F@4U#*3AI|T*=2c-@De7O6E
z@EFa2VacHsv6sqdo=*>T0os*ToP6Ey=|59We1oy@e&~zo&R#|7agIvw8)mxDRQbdx
zr*aC$Q-{n`2Y}=e%W!&4%KZ3p0O_8d-ekI1F~@Fz`ufQ>yj7z(J}KsWrtUZ9j4Oqv
ziN=`3(KKE6b9F+;9$-Q!(h<yL#i0QOapT<G_duGgUx@@={MqPZz<@R*uY&#gJp~Ll
znth>#S=;Z1K2&k_ubkQex+-JYf5VJ+oY^c}%D0|hng25kX+t?z)ZTAgMt`Gzbu%DB
zB(rxG9;&S$XV=yOR6GD76?k`DFu&S@OUhZh%Oc@4AiGe}^2NA}@r<VU@9g>4_$eG_
zs?eX=s*Dj3fqR8^uG?DuYSOo6MMbr+xes7G9UUDY&;VgGgYo#+09BwUVMGQtbWA@-
zUz<vi`HQ+MZ)$eFT4zxVj7&7+510f<UsFdQNdO7E1|YzQJPlJ*9SX;Y9!XZU_@t&Y
zJZ<_1;~fA*KF#`Jm!&-=n)!{|a+jAMDeV`ZiDoLVK>vT|A?j>9a1O(ln%g+Z0@>@@
zXUY>kR#<%%q}=_~&Af)PG0MP5RZ6c^Weo%-=waU|dbH%4w#qOYg^Z8i@A-%BVRxHi
z(e7avv~h>o2ZsL9gW!D+T&y<$H6<{AWMO$L`B(bE*ANj{HE-&=Fzf%Vo}r}#%2xOP
zE?>yiyIbxnjDzC$L+5;KC({>G#9KX$xHR!f8*}m>=*pfYsgxyNis2>b?l9ku(WGEo
zP@nSa2y!#JO8R8}^mr%Br<?BqC&_IHr~BbYg<(y)aa@^j>A*qQ(rbd)Sg{cw>46J?
zj<Vmcw?1ImFw@_sB(KpgQ!bh{X&oVVJbe=$)sgl+Y(c)=4mEvF`0}z7ftR#v3&Rl0
z)>ra;E+Hp}x>7b;g|S~DBxbpnPdxQ0DvXi~K`rsZ=5ce$xM5#9`IRFRHl-eXE7F!k
zurvDoyVcw?UBhGrqg*NyBD)o<J1)}IVU;Swtx37lGZI^`d3kjc^&2OG6I9_4fw2Z@
z{7|~g=IZYqo10OITPa4y(Q}a^GzLfO2A0mtD=PrW6wnGv@&fttGbW7r@;4`gTp;6(
zEUcuY1h_UuyHB4Cjk6C-%p-E%nVwxrFkQpaX*jZWN}11CkX_%4loD2{kuLoxs8shK
zKnPuAOJLR(ma4puVDg=J6iBExF}g2ZS*0GQDjA7EvwD|%@*4`)R|SrKyH5>uId`#s
za_Kq<x8TO_a3c>v^Vj-KEm0*U<@Q13Ramoq4JVN!PRP}+;z4`&#kI4pnR}+g7}Fk>
z;gUaQbEDhV<wo~(@m?WfgsxdkCKe07g!xj2?iEJ(Mwrn{NBw@Mb-O|WX(o13y3{Y_
z2W9+$h8*TB?)4=fS!;ClowBB<b?wtf;FRlFlxLF))N0o^HWcLLm;C^t$@zR&smx#)
za9+=x?m%|<@Dk(@Zf{*~4#>dPq>MQ4XmlNSzV<mh`0X$T1RDji6#D&f{1ANrEdhAD
zR^Qjpt%1WAGAP(!UaGtS9w!d=T7yDsS#ad$(f@?yc3#EY_YnS#iiTE;n*WJLb@(S5
z^<M{1@cx4Cp8UswKI+p2iQ_&_yve=3*iRtuVD`4yes4N5DG8zfPi}!b_mH_c)$}dE
z=7DII)oyn(D%)&#vKV05nBx1tI{rEF?h>QMux<JHp!|=7htD$VI_RQPbI>BBjc+HN
zv&x{#jIIhj&vn!+3x3d-9JSMJY>(vv`qBOU{iUXnQI2bl6qzcZ?DA{^SR(+B21#eH
zo?L;A4^6}6ugXd$9-+bsd{&#sciHjGn>?PT3WN|2ybqkNvHgJj76}RI&6`M&)T?*4
z$5j4vIc46RSNp4VMd#lJ2khVj7boOZ9yl{+%5%ze*#}4n(r5b9-rlEXTr+kL%5kB9
zkVo_3Q_uPQ($&cZz#bhPR$Fg?G#eT!YO7wU`^{b@P2(+!Jz3+;J{xE9Lo+n~JMWQ^
z5o)!X-<gv5H@|RH6}_cEjS$H5HLv^`{iY<ny4?E<mbrNB7M|%>lQ>cA9&9cVw<jmK
z0l$eJ3WNfC_In3o;v9A)qVU4#87Ac$GjLWD8M@ev`%Tibn|(VkbkwG5i%qJQjob_S
zB0s^23H0f*Bfrb<GAPQAQ6F0PV9hsFUXR@?bkIUGiQ~q`n>>4T-GX5?h?o5_o;FNe
zvs8<1feH`L;q0yDIxrqiiEqMa20l@{5ZM2uKQYwoP)R+Ik5<_oY)RkluF;~w%Vw2D
z1L!<{ewI#^{P6|1o4qOh<aR^=KAlMy#Q<1!B$y=E^Qq-!E%tjb505jWNQoZQ4v1vY
z)M{4QhJZ+>sYwtFm}=mq+_kcM9jw!DvR2Nu6Y?_NdOPx@W)5R{V9W-aeL(+g_lC14
zvsaWt1ccS%m-iAbN-0|!2a!lupWJfUVsPClAF_?g*A@<HLo*9E!5hEFl-(&Ew!hkK
z<Z*2atsQX4UN6mOt?;Y8B<)4q&%=CUW!gI1vJ<_{LEG*=9yM3<Yax1gE4U_zyKVk{
zz%`nhft=~RoMlGcj-l;Dm*P?<oE&2Ht|H=X<6VfYv~AYVmJMBOAIm*}83HK!WxloZ
z;RdI+mewIJymMDKHP8vl&JGJhEVlha+<G#!i?<<YxaUQ5m8&6xa9)4yqI1438(GmI
zl16<+Pe}WN^jJ$$C9SKwu2I2$oBM1=rPcL$bptF&zAup?%A#H!bV6G#%wxG(HSRcI
zym3{jw-AkC_xOOgNIi)nk?Nz^A`!QAF;yKJ-(dc*Q#}3F)m&z+jfrUQ!9NY(PT}5K
z(BUhl2IX2^HRq{G+{>_seE}C>pgjGl07FF$+v%umJY7zBEXA{RiMj17rX2w00w^7G
z8-UZ_pCmu9yWhK5S<&H*f*KaS+vVrNiJh}+aL{Z(-{SEnW770I?&uYNHlqG~&5+{`
z&qsK2YpjN;JaszqgV9eH7lS$t7rE$xQ}Vrc^<@<0?<wQL{CHEL#<rpDV|ize0=VU3
zr*zq$I`ZpAEq2|I`tltamJLlz4d7*88A^`L8T&3lRtF_7QhS82*OBnZXc)u7YCpG?
zDqcLFxL2vXOI%J6t52EN!<T%X&H6+9U8E5`7FVO6&OlEOEIhm{85%+?;P6Zj4IS<j
zHtsJpKIB)<^T3G5(Qg3=?+5ujBv>Dy4-qXyOx-?eK%Z8d^Jh(ZbWYV3DUwl0R2isT
zesx55mEeM^WL`j<|LlQ8G4N@+v4poR8ZE}<O~Hzqylf7=kH7*~IQ`(7h7H$2{`DEO
z`0|tPHkPN)BU~<@Md_m-BCfk$>8pMv8zi_-6n2DT^yA_XS%1H>X*>RTUP;{e%HJO*
zU3zQLG9xonGKSWZzEO${hfb#p0Ob#PX=n<;zspMi!;WG<V@O6u<{upFZTTmU<H6DI
zpx}C5YCnLJlPw=_Dle8#x2GtbFf<W#|6>W_%9NRt(@QnB7A1ZnN0f1iB8*QXJ0`1W
zL_w*7o+Th|iz_+kPu_svsFs#pW<WuOASua7c4aDO!t`Z2(L2n2m)7RnthPNW0sZ}%
z2SaHj|3$DoTvWWNBI|p^FeWTo&+qLRC8Eb6#(EzTC4XF7{9t*FH-Qc0kz!*F@vTqR
z#W|A~3e?zcy8ef!N2S4b1*Q@!q{D;}Z0M6&ZH-pD!l{k>dU^uG!u<YqZ&jUF(>Kk2
zc2R|{YD$IWSI{Tdq2EoJ(Zt<*W*G>u*R$1kEpQ3wO@6ryY$Fk7tGIWkd1L#s&J+b4
zGjZC0o5RlK8NcUcAt6+u)XnTuxV<3me*Ocxgq10hT#xeAbZN?NNQynMEaAm1O1)l3
zmlyVeW=e{(teiBoKbTCzmrt9q{7^Hgq{Vg|MK$zKc5@TdYVGcp{O~D0d<c-V0@KCD
z)g1f%dl68-0;StxMF33Sc>76j@#xE!FZk}a#m!$0CJN>NwD8&U=K(VQ2ZT#cZ?%O1
z_7Lc;0#8Br4;#3v8IsI+a5#^fuPTy*n8I|Ax&xE)ZRWK(N3=5~O^P_91;Fu2XD2Wi
z%ftOH2bp~kduz7(Hdp<rBK&;ME^(RIjh?#Z{`D_Uq=_nSr@(2n=5v2>r{h<ZtNpR`
zjF&S_7%ow6=;TpFDNKJ>`4f8Ruue7GzsHM5P>kn)51$<VQxE802T$1lK^O)HKhHj&
zMSehRaI*#SP(Y=iGI$bXRYfEv{nj5VK;Zwz-Z^~|hD-w40#F!`9x&;ZWdgAP;Pl7$
z>lLv-ex~eY+>rwHj!}McJ{ik{S4LeIWmHKm;tvV8Pem6}*|lp`x668v+v6c3^1AjH
zW_9QxluLgDfwnNy`5N0DfCL6ITI_g*K)Q6LBLpHu+_m5N$?_0?kPl|->>3a==IR_F
zohz>~X}^Nn*6G@d7cW5mMps`SluK}5vi!LS-LR9nZb!{MsN}>wOtj|0{EB;5iArO+
zY-l`v2xMeWZv_6~i;OF<$6@?L9Cpvxtih>(FGunN5e<;l5wfu<n=;{aJYG%Wa9$rs
z+*#C}KAE#$09uBKoR?1s071<TsGxAVUJpe>VqzTbu5JI0=ud{BVEZu;XE<B)DhjbZ
zjc<*Bw(ygTaCd7UO42bZ%6AiI0S^AV{dnAzh>X|ku*#TXD!3qDC{<j_!E3!ztT5vJ
zoSD4I0sA07XmwW*@!E(asq4R&b}lJc^qhFPo^W3g_dO+Vh@N6qSuKA|y$CA4)m(oQ
z!ltfq$+JI2#Gr@T!Iua}YDD^ecDHvB1R0z)xe+8IOCO&&U1h@>)-{3Tgj9mQD)=<F
z;reJxaHYUPK6NE?E;r`ejjqNI5x+`qjz49P%-2BDGYcz0?U{liklAG=go)L58w3Ev
zDBW;0b>EYdCEc$96+*-1h`;zT2!KE!G-@@gB?$n>-qON@x%4;@LmeCg1%pE4ee&uz
zrSk?wEX1xgb!oraZtv#24-VR*-^AUYslUQ|n1X)q$OM1$i4C{cT|`aCig}02xYtZ=
z_@H;wMqrNrotC+KCg$Bi(#?V*B0H_M(jf+Z)bzc=68I;h5AlbC^=|Lh{}~Q!jVT>j
zHSc`zavy!~cBEo*-tH79L?N`MFYT}YtoA*xk+hje&a#-}#OQVbuoqz%Ie0Y=4wGHX
z3IjWK8vbkwqo#+<$z!DeTF;s&K>`UUSzBG5WJ?7Z`A1_nk*5SnN;13A;MJPsm9PVl
z_FgMcHU^v!P4CCYhCBD`ZJDFu$<O@aT}5g7kM{yxnjJCz#_{VI<MJwpX<rBwBJHsb
zi|9IODNZTy@Nn(KW3?^FGNG@r5!vw_^B2$We&^7QTC~gg!84CK&9N32!nV?`CHf9)
zPQahZJF$Fn8x29IU6=B=-W&AMV5{QImw&q_BTR%FXhqItNDzV?a~h`p(i#42T=S_C
zLJQjmITc-!<BN6mlxWA{U;H3`V`<3BGFa=0Tw7Z^50}p^uc~qYF`uEKp+lcLh!ubz
zePnA+OdKpUx&wL<buXClAatsh$0>ed07buI?J12vP-mNRGVY^=DZNhbQmRz1BR!v~
zuZ6@TXC$gfk3WcxMU^GL(CDY4WP}icF7-xH-2@Nc-iukMxT~~Z8NCcRd}PQPuo|)V
zl0H&m4Pm*3rHLHUC!<L@wG;5Iq}|EM`+|i7f`Y&7^C|9jy`%$}HumzDo<r)g3Fjh{
zH3|BlNL;P`)qRa0@@MWPEG07LrOZ={^Ev1iZW0B_^i-&*%}zzrr9%W(HZH#R0g3`(
zT*ZIX7U1S?;>A8Nu@e&NDl;4%&sPEIPiF5Sz;#2wW3SR`hfR^11hONyXYa19=&@dp
z75vRaQ;CVz1+-|TWHB%OGF|doQ*sDS{-%;(4twy98|KZ%+W-u$TQA3U#}5Bco)If>
zsB!-m1G6W!&}<Qo^TDZ5r?a+w;y&+YF_&xuKH7cwuC8)>0-rwMIWg+S3wWX9B{)@Q
z#YAUEQ<m|ktwZjTzulNcS&GWw<cR%xjfQ3{Br7X=mZjZBc(7=SR2W_KrRTr(FG+?)
zICe8TCE9+smmfX%{1bf`9$12G93F?$>6n7XYeYm+4xfHKRHo7Tx;njqgaAJpK*>Qx
zT`5Tb8D~Hv`^STeWzey~tk31V(U%fuR6eEG{+nVn^nLp)*ZRY>|BgSFk3b|hbD5D-
z*OfM^$V4N>g2w#6L>AG1B8hW^{XX4h4fixRTSwY3$>mtr+X{+`c(Of-mQF4^OxDi>
zL(VF35y|$Y%~~C_@W?7oY2moFferI%W<xVQLH))WG&X-0Hbx8@#oOSQV()a0e0C#T
zR)9&}K*bH-r3Q(H*x1;%$E{o#E*qo%01h;hB~VCBPfurz&o3#ldbqy>{=R*s@$OtD
zL_nJ{9+Yb&6L)x(b^eJ3U}XTnv}-FQp1*ku+Lp(V_|dvMCluD>yBO$SO;1FgDaX6h
z79PM%`SK1sjQm&R^8H3)J8;o7C@90G-bZuN_yCFx4Wu)Ml3weY!`I7v(FzTIGob9o
z)%rJ?u(e~-Hm8FC?b8X9wa<p%wY}@fIv9GUp5Rirne>rPzG^$(jy}A%(f`&W%4#BK
zq)BT2)c%_riz+Y8<7573H0Bp@t93QMY+crYcfr?=LL1{q*f5r;*z8T80nPKQyu3E9
zZpYKjFWy7I+4{p{GnxQVGqoC<tLvS@-kzQ`X+yA!1uArhzkwkS<GP>xN?T?;VFsw?
zo}Qi#>~bvs5#v0zVR&Njx~M}k-rnILlrdwhrT9WXB0Nm&$mgXQM_Oj)hsYkKQoV9+
zHIP&Y4GzX(G{KDdL&KUb>3qc{e)Zzm;g0z)iIz&hdQQ$=c0f>aJaUoj{@QE!sJ{NE
z?Vqj46Y%k?EDux0#}hvB7=MZVPmGC2=ik8S$-fSsEdG;a`rijn{*?y#?}Puyhx{``
z@~`jw?}PupvoxVXMU#Y-M?k&&6W2X?g7s?m|Gvs5qI2frH<%UAo?AaUkT44<!_fB4
zkIIWltL0VgHqd^4J!q=evsWi4sV={2+{ZaeDNzZ~2w#({D+VJOHP*YTi{tTR<^D1r
z-%)~!`(`&Y&5+{HH>p*cRhIL73Yzowj5l=*W)Ea`hN2crsvS>GT6CA5H{cu!F5&aJ
zJow)?HXhfC<Oa^8&&k_H!>x%zQ{rMXF^;gI5Tq3-EaX+OY{!4eA2K&s;*ZTj=2Z+y
zSvR?#`VGl$uhGJ!2|Meu1;Huf10|1vP%XjblJ&>ICC}nw?W3lqE>FIS44?9dcI9r^
zc^sF9#oLjmqW0{S&#?e7>}QB(u~LP`bSRQj{l?Duk@>01bAqzHWyYZXZ#2;2RVDPD
zMr->@PnSI!^&ZZg1}7AfG`1>-2i;R2cJcKq*Q1QC_+!f>-YZj0pDu`_vPxbax6^1A
zvF{U2aD-Wl^~F%|$|<Q$O;+9DZFqD&<zqzGqS~#by~tM`b<Svk@;P8XL4f0+htb%-
zJ`Q1F3hoW*kzKpxJc-Sc%`SAg`8@-Z(tQL8==$A|Jqg=)-B5nIH!=v}jDDrVH=^P?
zPa#A_Td!a9aigPGO`~FAI33wsYV{|V{+&`|nR=s``}TuGhzS=8D_HH^y@&0QG&Ur2
z-UFz&d~OcGjg#&!8%L~={h(l)q1KD_EWP8|N~_{GiWiIfFeyL0(M+JCBEEk^+D4vz
zuHrf8dP411<NC$HL7?)j{d}Fa+n|f~K!|>7l5R;Hb5os|GbhfirnBRX{m8;pbbaA~
zZ04seiS?wig+-mx$%AwE%6H{Ww@FH|IjXE*$>ZDYV%t=$qD9Y%RDZk*mS3q5ok8Mx
zPWW=NQ)aAd=)OBKBBVDY>VoRTY0z3C{UeR~eck*CEC$-N{N!O5lq<&MPS;RZ8h`m-
zcyHGV4kpX@-t}XaTg~BCx4x?*#ew2`<hvu-LkSOkmDAH-{X~opUoQOyosu7P?)C4A
z&fspS%4keYb|*h;yoQ~^IkxX!P!TjuoG~uY7YTG+zLZ?4A9tc+tKBW%FEqt2e;t-Q
z;XGgQWu~O3gonwG^Jc6R%ybb~kU-tt%EiU&9k#PB*~Zsy#%>xXddW<hfCTZx;H^<e
zdF>}VIOcB(C>qV$>Rac59z>Io#iGW)S#>hvx+P~hJn!viOu-zqxx1*W^6kIMnTF<r
z-@I44==LMEN7$^}@lg|#M~(g-wbcYyOapR%X6awvl%zdGl=DvD)mREuNZ4JjFEQ|r
z*PZmSQA1)x_pQ3!yn{--MtzY+;?Km4bhI7})iQA79I~26X!RCi5~J;FbR84Co4f~-
zZS)FtR;bUS!c_`>sJ@!Qq1(*;Ow&=n85MfYUw#$7!_z%dGolh%N$etgFf{+12wLPa
zBDE4g!CZf|E(dQhXQXO$z7rnW>zPTnO@~Qgsb7ERE&ouVY(8#YU-gE7zVd>XbM4n?
z5XD)foNrFMK01*%`l-sIG(K1OLI!d?5)ZUF4l&`xeZ#q4d;d$()}%ZRn3N4RPD$0%
zci!kp%8Dt4Swjkx>}OK7BmSgsghJ%C!z;Rcr`%-j&u{NH{toiT3bkpy_PkQ(K0N|-
zxMIDF7}NH=8EP}@0bvG6avw*4a{^(3fvn$`H`jr~xB@|rS}rOtB;T*ySszTqZiBUg
zOn^oerdJYI<y|HQ{jBa863?5nHwez@ZAn91PPf1mtPN7V4AMMFjz{Ks*t|lF7Vdj*
zU#-K{!XN7~xYRhdLFKn?U}R(Xz~t0WtUrsFE-+<o-zak-5xy;?t)8%^9}UiPNbA;$
zZf_1oB1nS9N>uRX*Ys`IS`FnsuwWARiN0mn%cRGn@n@L_i4|J%iNZ{bc9dcLyfBF$
znEf{3AwC4&{_z+v1h<9;-&&@5Zn(YeUx?RVW|eBm%jFx*;qS;NX$Y&<=Dn@SP?2JK
zrSVPU`ZC~dH&@2!$EBucMpFYf*3mrM&3ujK&VABd<(pEyD7}W8-r6n4t)iD;sg}V5
zgH6N~CKi}XPwzwqSIl~8$859my+nR10b)eLrC>2&z;GV6$(7SeMw2wr#)}FR4{xhu
zHm(Iv&5P(0dbtGU816X(r}RiFMpX+)s0pY>JxgjzER-CI=&OWOY}`MYzwf<2xA&)i
zK_IKH5sD-|k&VogG9vZw(VS`!xNU3C#F#PHlpfn0xyKK=^!HH|9f_XJs}fEp4{@{v
zqsNDNp@TDvz0Nv~F}5y9O%!HrR#?qD^ZIqTZ$cl3VG$-059xjq=BOq7d_tQsFGYS%
zqGat)@z2{%42EBj=aH}CTsF`NeYQXEdZ8k|6#WK4AWrPzXxFtq>pkUjmma5>>7DWG
zZm}>0E^b#c4cUAKASN`fZpqtayi#8l;#at79CNmlK;=Eg2UHzVT$NzRPJvT6BF9}&
zvGO${nTS7YBzMQg<G7uBdZ12O<3M=Pmdx)?z12uO^x^Qde)O`Z9p`#1O<1TllCg2{
z_U)Wr-n}V9q$m~%9NZ}^W$p*?lNg8mns_~;R<#6w>6h5=%@4<FTqbwMa~kp2zh!TS
zxv#O*BFeb7GPAi{6K_G7%;|KKRU2iWQB*;zjJLNq?_jc8!C|Gw#cI*8Soga0wLi|#
zaN){DWn(8%*gWTU`$^oSBcU(t@-*^-;2f=sSJ;qXPju)$?e*^mjxybP!!{E4;>P5z
zgVZH_<f}QK*5!BJM%YF^s!P);#(R)Ms-4%ZOqYQ!ckTBo0%nd~`Dy4hrdHE3ftkQO
z9^OsA6n)-z|2RP1>C2)RvKSp5p{@kyK3uR5RjMwr^w|Akp<F#vAJV|yu0a6C)%9!#
zubBgBk2gVfsE|aw1$u+4Dx;x7Yba3nd6bU`FAJpvTe2S~UDd_5y7y3v8yvWh8Og>F
zr0eA#jDH(Fy3ge)PWJ&<YiDaae)$Rx<f`MF5!O%}5eURnv@=V>e)=Q>#QnV5>iTx7
z^PaU#ZiHMd^@dhNl_B9fPGP0fQi*a=V;ohr?Rr;R1CkD_gZA!>O5!dE4>{LHkty45
zmeCb&DSfN!*K;tku_<4J`@&J3(`M<skGT>oW3S08*xeg7U)fuwrb(|ab3Hw`{xtB$
zIB4Rbc%IX6V=WN}U0<`&M>nYF*yxy1eKO|v$L>HSwZO^`+7rH-*w?;i?MUYuW9~<t
zVqk4~MCqKkGi8cPwAzc4-4Soc-tz4SP!Uzen67(GAQE~v)Zg!oe^I%@;q3p%sC~=$
zJPPcN&mFo%U~-wsA|0n+A%jt2PNCipJtS&!zy7O6>W9jrg%%XG?wgHyjf+m!|9JNB
zyxNP-mdAltaid#@yz(O%n&|Q_z0%iUK5Qxep<Synxkq|@v)PExq5&@z8$BDYWxcR{
zBER)(q%N+`wy{uj|HtgAfEhMXG=tl8U8{39M<dCDffC&4aG~}<T|z=mLRF$AZRBHI
zDLHL4l0)prfAz|5rsI@_ZmVm0VM1L-0}kS2ieHaQP<DP%KGX+3*o0aVS4PpT`NgcY
z>P9=sw0~tbx&EOvZLo%{2D8sMcqG^oy7LPM)*n2%sX2q8Ttw@ySzpTw2?v#iTm<k9
z;8pf=^6Ni>-=9h;wYm3eKd@+#{=PRP^fm17CICdc|8<bH`0Z6z6YUTCRwP^+t*QDF
zbGqq$HFN)cN;H2w=AD?=YkE5~w&<3QHs%)#HNOsi^NK+zAl5voPy@p4jsm73Z@4bT
z-*Dh{YY`woANiOR2d}Jjt{pA4{=J+s>91iB6m+<N^n{veKQ|n)!qVa?Ar1Y~u)N%I
zn0n^ZeG*i7r}(MOOAfhG<WKQ<fxEwXXS4m78NcA*(!j#3P30-xQ+ZW%EPoCGIvuaf
z*D}hVO#ZoxrWG4(a}5d8Gv1_xhJkI=rm+c*RbP<xsdr@wDU{eB%%^5z<CBk$Uwf)5
z?HALk8}J^7ACrqvKvCX+4$EW+>_)-M0~B`&qO5|Q;`^_^1`SJ@pU50yZR20s$-`w7
zmWsIGIV3nGmM_}1?!mj7ReYAY`rc|(s`VbKBZ1OYB17G_vbkCC^k~D#I8VNIJ5N5Z
zV4MO{>u~v*Oz(!m0hACjq-Qk?2TJqVN~ctnmA7$T#=W2=e>^ds{><m?*5UElh2gT8
zkBsr{YsJi}=$1DsD%z0_)|0z8mA|o1E0)<)HIQg3;+gZje&+O<xN9`X8ls3Yf0PW+
zs_{>=GFwth?Y?Z<TD4FD(1DC)&y}p~BB3=L^!V_{%a)5RNJ#;rrmTLRBuq4qXDzc@
z>?gqJa;aY9C&72826f-k!1pHJ-xXWNCzR@;RK34fOfTcN5FIN-#@=~3O5-E&2$5-h
zXt^3&(+jw`!<9<Gz@~vESX~(`S1#_6wif#BRq3<psXC^XTKKI~ivXUN;Z0PRqJTP7
z@?c5KM?|!|+}zRaGREAH-Zda1aD3(BB8d8ckB7H!;qO;pUvF#6+UFb4u2ro%aJ?`i
zM-d4-=vB)Hlu9@sN9*8VaODVz2S`N%HX{rH7(ykb0Hou0<jl;Mo=lQz4i*-)zM7z;
zU_k>ysV30-qK+<|y1;eJm*2??5=oEfU1LogFe<mIR;^YwD^@I1rgGcd*zi;E)>IT1
z4^b}!N=nk%KYq7h`i=pb!NdfSw+`Lgq}N=-UuoZYTR6_<Of=U$j@nQL`@pxjZ+ej=
zLTG$g4=@*2-dYvy-<0|=qdP|?rs{NXyuS;zWapMU-4K)x>oIC-1<X;Mv8sH<OlC$C
zoqB(TX!o(+6`!t3Y=BCNX|fZ>+r;yGdVYn&jb-^5dJ0DTlIUhkCM;Hn0RPW#{dzk#
zHGdRpJe^~`S2it#65gc2@W99c5u5Sx>&r{#S|t*Q@H?bQKcMkAY_NLd0J4e#GDIi=
z06;fv&^9wsN_->10|88wSh0+E{6HdR#DE^~RII8K)T4pcZ&MTZEZv*PH-stv%y|U`
z@?>Z*Dbv%+xnqHB4M1ly4Sn3GT(6V`l+4<-49k_-QMf*mb+tO}_5Y!y<l|T6?qkI!
z<mOkr^M<!YJZFAwEAK63S}wQVTm0Y>G4^du4;!SHSKL`*U*^u$n;~2fT!i;(D^Z0u
z{9^yEkk3fuoinDl@FXMpg&xuRRPQmkuGXa@MapbhBf9um)}Yp<t~INAb;&l=(CluV
zWvl8-PD134L>_M`S67ap_7=&uM4rKj1P+ND6WcReAmxUG;McF`?CiV}A9PxCd3}u%
z)og;9opS0_f47xZkyBXMc~UfG(vok(JUsWGw=vMPa=M1L5nFdxHTj?_fDrAY*rnA`
zHiqs;V^G49<yzO(Tv~P^xOc~s3uQe(ezg7rBjs2N)%Uw%&*>WnwfsnmPe=1%t(<GF
zo(~)4;11T=(!sUBD4G%vhDC6;k%Pua^bA#vYacGgwrs$vHlPC!ffnZa=LpvjdLJV1
zJb7c#K1Zd7UFR(<uac?!&1g6F95te08uVEU?;muDLxSn+29JI%0`c_r`~7OgmTYRK
z(n-o2$z~$&x-=Y6rRWTsEt)n8D*UGJzWJVLDkB2=xCd7JGqQ^>3gf0JgOT-%cGTpw
zP(7k%{10u<aTxu6uaq^42m65rB+;#7J7<*WqQod*n!TEv)-04B5eLWbx|?k?x;r$j
z(O{vnWj3WTSV0Y{m}^cKF9&rle`5acLjjORWRAZfh9t2xnZG7+a11XzGQ8EKzrw<=
zv8<jVuD_XQC{coWP>)C%kXJK%bztF+|69s@R90EX3Ro2t?R;q!LTzEZb_le^TETv%
zHfD*^M{Tcs3GMv+k{A4kP%R|2ojrP`{d%8{A_7|pc!W}i7n#BtutF4X7yW)5l5&~?
ztEshgas@+hLkr9l8Yp7l?!7HmRF#q8t-D`VLMd@p3m}Q-BLq4)9S2e?2g{&!<M2+)
zi(*Bw1c0;FPUaH=S$R1bnU$TLAg)fJwNmA9d2bACgUT6e%pYmg>A23THbRJXZ%`j$
zEr$d<<%uo*PwqB^gLheEAEVLZkbJ$>O}J{s`xk{8YPwX_by{eNBZ`Q$UbuM9m=WId
zMxQrYy&V(&UDjwt9w{=mX4L1q#0dd+<3G3&o!xtQQ46dfu7tUP0T8OcT>s^YX%A~$
zUS8$(9zBSm9BX;MNMHDTNC(ob1I%3AFC6y|8YB>8{ox&+u*yn%z=Jh3BoFG35Y`1+
z=fsJEB)R$d?FU<^XY*Xed1wU0=+z2^Uc!IUt7olPT&D9&^=v|;4L!pio=@}>G}P>W
zW@PyGt}Q!<oNN9Regb*)+F`gwox-mgA}=S0v;8i^dxwUG77q>%fa{w5Gx1OGHde%q
zE#~lKcDee@@c;SeL&YZxC(maySk&GAz?%MD{QQrwqcq<NltT2VGP8T!DFG$~D27Y^
z=*h3(eU_hZoAf=(R-vC!FN#DqGg30M$cEDp?x{lp_s*}XDl&HVI?z$?YoI)voTnh%
z6WYX<Sr`B~I0e)rT^${%I|`=Lt&H}s1{N2ecDW(EjeO0|UK*2%>4x#MaT<;q!kse3
z+%>J32++dk=H-D-3E*cNEj;|l?4$|fCBtLdG)BUc@uf@HD3o%3RO=FuN|aYn0MA9v
zPCG!9fjV{R6eY?lBBInoeC0OUJb<t-DH+i#b#!(X6cDgt1}<ROKsH;JH4&keHd_|!
z%^TeZ$l9eG)_rnS0wLAH(lj+CWnP`CDP64d>E>rSTOg@wY59fME4*_B3L1I^AEk22
z1o*!PC5viRJUl!=Njsw3E8Ots-KYr+49dyP`3Mh(fB=j&eGLSbJDn}$>-jTgRRU%>
zUYSSH?ov%*pR{5DaHOoJM;GhE^O=t?ZP<X4ii#mtj4+&*A}3LZ_}52&FY42COS4lT
zg;K6;S~|6A<;chPq5#9t58#Ym!N4@RcjFT1(8U5dH(Y#tAS)-BN}0_OAe)o48>MMj
zMnvVUSxKDVTWYrDQL8*-QUPGwz*iyo;bLa}tSNc_CU{$wg6Mk%Mt0M{34H6)dQ`q@
z1xa=j^;jBRQ0JZ~QIfSMP^x<DVZ~VhIy?XgDkPqlUQ3{pB*5~SP5Ftqj=W0NWMi~q
zHC^S~FuQPA70u+;prEEEIZZElj%+c1LmYBirNTHWdjb<JbjTdudj#NV9!Rh-FnL*7
zKaZJhS=CHTD1kId?wF5HPgRu!ObVF0di8~D8Bzp4pgjjT@<fD$NIalO1=<$^keq`c
zB?X1iL_wk}zO4aE&M|IQVd25=>`&3X>De`)IV>Sz%goG-RxzkcFXIArfi@Gy&d$!2
zw<01U;O;m3wKNofqNrEO)56@ev=@VmQBP!RMq%`5JUG%?hJNrGDLN4d(}=R#1+Q6E
zZ(G@d2ApCfJCnskZa~@?=*$|xFaHdxl><#qZ1-p15~Rseh>D1|@Q|Oug4xcNwW`Kh
zI3c5=dJPZ1YQcQU5HSt$$RK43(2TWl&M^~Kj@F+m30IE(@k5a^3iQ{iBOIa2exYqu
z(0fk!X3{Z9lX-U5l<#^x?cgiDQAAw&$Gu(`gy?P^pdn2d-lapAElYuqkB{Idqo6<v
z>7Fe|jJ!^3p*86wNs!=(*^JW(u3R12+DEcJBthnhjEn>vX_Ao@vnd>z{!}%>r2^Jp
z3w>a?IzbR<<>=~KKWsqU{}ve;s3A|8RPdw>?Y!nohC@W$IJpAtp@2qDYIZYFvRg@)
zCXes$SSeS|7Zn!jwSULXo)ANqm7`ZW1^&}2KD@dG#WMC1AOT283A#xF%}79**xojZ
zKAxPAQ&Z#r&F_LUPxUnGCvu;<ho2+?vTp{CbZgCfVjeO&x??v2PuSeC>z{qxKI(;;
zOkm0H%j4qVh3)_aC3)W5Xfcu%i^)ql6QFAdkm8t_n<WC$&L~5tt0W&OA!d?eRKhz(
z5oTKj;~*H~`}F0`=~?7gu>spDb$C)7-`4&)0UKSM8U%q)pT7TlJi%j+;RU>rwn*ma
z5?*G!3{Eu|NQWW|dMnnmufL2VKF&voX4dldO-1KqWfj@GBocA#NA-UkW1oJHa?Y$~
zVxdeK1vCn8Fh}mVZnk0^W=BmJ7pg41Y&3GJsuT++0N^)6lIwfqfydn)R89_%wFA8w
zz>5V(LQydWK<9Y6w2FcLAiEd$I}nLbuQ^r~KxaCWJb6R^Z3^ztqMrEgVyyJGA#0l(
zSbSV_Cs!^!G^JBM>YqP110tMNbsdm=Oo3@hPcIDroh0UD1;LK$mibLhHAOr(qLzQ;
z7;8Wh=q5^y39(kIz-WEtw`5^miL53bfllJ|kzQ1LI{K<8QY0N%F^x<>GY0KI@9yq)
zc6Pw=KUUWNG9Y+f%~Fbrpv$7*Ej5c=TIXDeV@8}{HQ%>R;Q%1wl#>G*uDw%kF`%CZ
zVx7dq-VVwGYu6aq*rKOTzSY&$0Stw?xjBNLcDZtCNlBclLrxATz5*<(2U1RcKG;z>
z2rgMwlR#c60Hhj9BA6o(_^2vjG5zRV0ReZ4)MoJ$`neynw94(m-j<a((gbgnTj$UX
zgI$q$ywyn{-RbHe!UV#~V1U3wbGiLzwK&Z}=Eg$Nd|!W7R)E-u+LzmHx|!J`=2&M3
zb5E<oL+LS##ib#?wWFnA$nF~<-rFfrW5x;Peg`_o0ag3?`FT(+1!sSUv}jc$hf#km
zUTZ;XvLGN~BR;~RY_(-$??0t6zq+=@&(A;AD~z>eS!t4#z5azd+(IES0vKEx8k(Ej
zr=Hx4YwtS_QfQmGni`9Wh79PnF!g9LkB^TJ9pGDmV+(4h!wL~eAGzW~@EZkw9CIl<
zXJByf6DIs+B&=mU;YDPwEJZ08ka#(YH%C?hLc}c#A|GSL0FzbL_UifySO*Y!lq(;J
zy4yqj{3b$+CFJw(G18P~_;tVSCz4+Bh*9xLaRo*0-$u4zQ+D+7gKN<DX-Jc<q*<C>
z8m$S;NX_hCRNA-d+;Y%oab8TXv>FJp=I?*DrH(`vnh6s5>gVXYL~9r!pflM_HkL*!
z3K3%yo8||x4$1M^nOrvPk;?Y|w4b`GFK*b9Q*)6kHp?)B@P$El`sf^ifCZ>O14|vi
zw@mkM>!cY5i2y?Z=-i#*i}--l9N3t!>13<$B0Kz1|5C41i45%}6x7lYK!JmCCV>QW
zu8^a_>SM^KNX^#R_5$C2yF~+*Q@tzrnqYxcDYAD-bgy2&db}D__v%@{Ayzs%prs3>
zP>&tz!1*Z@*oc|ava&V-(BZPy91n&YD)KA+ci0s(mNXu^GT&ONmb7hP)cnYUxe3F8
z(B*S?g~R|?V7b9Fd(d=B#bYG-ESjS_d^)_Ph(Kt^Dpy7N)q24=5S9fq0%E1hQYo;H
z>#KzxtCkgbXG%liiM#&Rx3w*wV0X&Hc4}N!&oIK!Pq-Fnokc{TEF(Ys`bF5XVv|cV
zoI6;w@==urv%;K+w1(vE3RSgD4O<%wXhRNO0T?!iSi=7{ND+V2TlU}4ujN8t-U4~e
zRRGZgR2`s|%GxGVpDNN??_6zFO#_)55-h-ld0BQv0KZ+y$Q~p6biD~+;LGeDVeFSY
zAlS;uF*2a-Sv#)CoI`z&d8mV+DTVLTVfZgBz+-Ue(diK!49}U2kBj@d@$+L0`llu=
zGh1`>yfP5{`ZdYLyQy{?9Dx`22~TAw3(+_<{ENSTphubfiwXzXwEsE)z&ef;(w#tP
zVJeMd0F2d@IT7?x-RT{Cgftt9+$ZMxMiqP2uE8VL#kuFE#O7>v*DT*oOgl-yO}>5n
ze(;rXlikeLS$YG9Y_DAF&Q$|%QaLRT)~cxQ>efT*oY$rLQ&9qZS&Vrm`s8a)dOP-C
zoimNME)~9JawQTL7dI8<&2Il~sBQPB-i&GyLNQTgslL?cz7<2qC51e&ajfX6c}s2%
zb^n1wuXUD3uAsR575hGeJ9U_?jl%mf-0WI-fqz^{SNEmdh1J1~B-|%}C3By=`%WX5
z&sdl&a~aHRUA4UJTkZ4ODc+T$p?A&YVZx~S{?4s?bI92B>g@`vI#*d~LeBObelF?2
zYm?-6R{rM6@8albEFL<Iei8cWI+37+KqV;M^RI4v5CuE*%3b02nAUDL#dZP<Gk5iL
zH_o~Po6a|3$;P;4Bn(ifJ9D*W4#uZ*_v*2O*E;!5a@mEY)w|ErwkLx=(@>q@Ek8wY
zc1SYa0%1cwP!eF7t#GuWe+1x$m8Mri(%tp7=06;~Oe_?iwaRN%!0w^OabLu#`@JI1
zgI{PprgougQ!t@yPf1?Eg|RSd)$mZej_>0_;C!Hwc%LPydS+W|ZZGTd09B1`Pp-yf
z^6W}q#xL}u`bU9vr5GaLuS$<k0HXL%DOK$FIb7vD4JDYsJ#j2QPVTU$_OpIZZ4&tb
z>s{i{$U-!o+Z=5N){tv})3x?pz@<}n%PH#}^pR}8Kq-uq`|3_@i2qZeGrg?Oxhs{B
zl$4a3df;F8Vu5iPDT+}wD{(V8T>^MFc<ckv1ZboDWe(15Oz~rb>>DAONo3pKpr7$h
ziE@Srh1M54I9K(GfDPiOs2591-kj>-&yzJ-2D8TZ#Lg5x8Aa#MC^{wb=^UjpY6*u`
zKr<xS_P$K>2S<?^^RzxH=b9OH?@t$JBxiv6@G_+SAQ2I7Lq<h+>L$GA=A9wY*u$+?
zgkcaGO@q@yME3(sR&Mz;7vIijd<$|@A{EFY(1Qnx+RFKiWMl^0zq3Kp^vh6^dw0OU
zL-2E8213E$K_`KbfBa|(L^4$O?+l3<7~Twe-KYZ%ymG`s_4?~apc5=C4~$xa3+RY2
z{}^7E0K5-@H@lEB_>-tD5|3W#6;Lz=;2xM1K+y*6?C`?;MA<Bt`8ICP74ydd#P<j=
zT}q}*K<7nFb_ZH<egx&mha<^3VYD%RK5v9_rzV)bLAx_FtcP26#?V_Ia-=eM8`k1~
z+4eL-m)y}|(xB+hnP2$Yo?(q&FBggzrZ{$woV5S2lxufnFn7_8m*|b5PAq-qsG<if
zBX)h|hvQ__<8xgblt6sCK<M<3hJ}F2z;#K{!>RW*qOvurRDWB8Jr&?eV_dhhqHil2
zWoBbI7p12&wh6#K0kp&-4BrGJ1C5BM3i=}L>!O6Fwh{~gf;Q-M{S9fl(ft7^ZUIS~
z1l)2ElNgR>O!<M(ejl_xLPG<)uuvS9k#b`NPwwyW@j>RWPEo7UxZXoYE1l)adyk9=
z_(y$f@M~u%t|(+73G{FPJ_X1*E51Vl`qB{%gwFYod-%?De6b-Rqg+biFH>#L9iJ1&
z+Lp_+Z>3hs_Ta2!l$~!d%-wf5snXe3PGLrFVYHQUp#@H5ygdHoMR>`7(WQY%QKK*G
zeZoODsupfQ$<2H7V%VSjcu*mVu2Nqctd#3Z%Ux}d3}bz*+l^p8I+|6OU3N*1S7TMm
zb()7*^`jy@Uziq5BM7)SleruY7lC{6)qhV)3NlCmd~|^3^Faa6gaS3x(9NxZCk2qZ
z&zAtr5vaa+4*|_~2X=J<ftUL=Ad+&7YLiL?9cV09I)sT)z^>Wt)}ii&@Ad=epM^>y
zc6fAz9KZ))!>L@u>c-&Co1RwX&MT&ICX}(gqGRb~*@|vuFtkTmeR>h1DsNzPJnf7a
zd2hvsiYr;tb!WvlsvmPYtn85#D+*eMRo~4%T-iB@{JtF740o`mmy=)ME~q#Da8|7W
zr_lsT0ndg(W5pc0PpuKIx77eq$=&rFqe#TmPzV2K^f;COg&WQ}xn-Nykt^Kc6(hDx
z^4P@0MZ?O@M&_7N;-}Ts)wYffR&T&%0kixK=^7}=%ScND(bk)@olIGZOo_xm%@}B5
z__5(F8=xNU-Nc$0T73<249pX{{O5nxJwg+nzV;T2;`I(9)(9~KdF4o{ZAsEOM+nQK
z6(q&NjFaQA94rK{>UTmHoEz=uEDZD=MRHmooJ$!(^yokRI`ygU?AxX%;Pxh*;zqEv
zbAeeoFLWmIzQ3$yObNrl3k^oq)5xoo<T%%T(;9Ju5S_%LXo_#W&CLMe<W5DXFEcAX
zbEnr#2D;}Y&p=8%r4yts!QPzi(<euEbr+YqF{A#0f%A(Cu#ro8|FNDhLgbYvaP2}+
z89XEqIk}H}um0e{h8E}XHj)CIn{*KR)REckSJgcIITjtwy`|3EuU%Ds&$#V-AUQex
zFzIdib%o9A=rVp}o?~4q>;_FnANR09zBig$uS6+n6t+pGTE}sFmSV3*1&0^0yOO&o
z7p*doKDMi@R^#C8z!%))MK|{*pyQbsPBXY!zX!SIw6c!L?Q|c{-pqC98!IP72!S_s
zrZMmz>h`o)l8MZu7J$GM>HLe;4s}9SG4w98ggJT3_tf|Uk2>ymqN8>AcrrnlL{3HP
zlS1f$590Vgzxb(7uL}^PL`Fvgotp?eP>+1LzX3Ah>_e%YUeNkSq?~6?uAnJ_tSlN&
z7iTaMu&1M-dl6{$9Eh|%Rk{&QB=`~@b0~)R^l(QcjhKWaBRze|`4@oxg6^}_RNOzN
zfJ%I;A7W-&+Tx()^n#*&(`T8-#cnwR1D^9=EBrHV+lK(78{RwtK<sn%H8#r$PuLBo
zHo|SOdSm=|KO#>tQBM?JM(8j&8S_roFu^)ovp;Zjgp)TUCj?fXFVq;;n_XP=+&1D>
z7N}><IMV6q1QzN;)qDV+K89A`*8h%_-B83hIR6?RLUqG#brDyh`BcI0@mv`uR|Q>3
zUI*s(?+4G5nMuD+iC$fH(opG{Oz9)Ti-H8`q$9=pX>%O%=ZU1S8n-*?JLg)$0NJ>|
z3#z{#2puw-&x_55+B5UP@CynSYBb)1&LXGB$5<RrWvyGlcZ7t7u3Io8XnISjswPNe
z?#$P-@niv>26#9ea7IgsNdTWyfFcs;5E+f-$kD}`^u>Ja8^mQWk_GDafbpr2ulzo`
z*U*a902T}zbJIT(oK2weN!z@Zw4B20hX&jJ^RpLDi)lQm;N#d1dNBEsIj(X!zkaem
zg5^f~iIh|y2wV>=RrEtr9}9CJ;cjvC(xWr<V%|B7{&v;i5#+@Dd=MZC7hj6sO#e}-
z{1?shKe3BQMeH}LdXT6mc4n;+m9hK7&jLNNXI?&bm<1^|S8IAfZH5f7^$qDYlkxqD
zxEs3<I2CESjB@<_(@$f9aSxd_>tFnYdn%cgjUNCv5~l?E7K3fdM|iIW=mOHkr~tg~
z$)_MYsjI6Cq<Bmx3UHGLFV4<z0rnVpjx9+?u}rR?XHV2%#Kdb%kVd52xeW2R?d&Q}
z%#pYkj8B$13_qnouk@w|D7)=~1i>Nm78y15K1jxW{kjPFH()0KHc_C**eqyX<)sU5
z8jl6!l-cnRcR|F~Ir)~$6z`=Jt@U~zV697~FGnK}`0@P&-61-H-)a>rrbwC0R)zal
zDpHC@QGz@ou>Fsnh>Xe|Zo$=oln<~YIy$RBkLI(?3h<l2R|22G{SK4C=!bHC2dxPp
zoq~1-@8|%i6^IQ2Ah}eF?`v#80BYAd`cPOmGBm_ZgTi9~`c#5WO<=nz!sFF^xSd<r
z1Co_u=hMc8Z{c4q&${33Xn^e;V7RDMngN(H{21|AXa+Y08ylH~Q$cMD8^#Ujf3mEy
z2vE$8;AxK5a05*sypnDArWu@0q=hA302VH2{ds$L2lVS0O{Z5!IQyK*4L{IlV3%>j
zRk+DO5U8PdzC><OdbEZ@eW8)7gTAX4R-RSDqxcwCq0f4-IV9OdHN?LMkk2Y4CoHaE
zzsxzEXZKJBmaAtR@d`C(B6?p(#+^QcgxW{<qPpt&(HXJ^xB`u`xU_AL&aJq`#Ye9i
z#xc+ygb@aof4)3u^$mnX1N|QkTM_*b@e8E0%ar%PlQHkG@|t~qVd3F$sTItWNV*lk
z5z{l+?ay9;+Zx_E!wS*6zrCWVKSlA?1hU4vzq27@M(rnkaSX<w4adOyMZ)Z{?TsYo
z{p5#R(t06)z=V*3E>WXGm*+F)To<#}RdGNiMYYTz{H_{okU{q2gF?aXSS|)HgK&8&
z`S+dygg{8rPQl?oI|(T%20A(@p%yBYf+toYfjOLccczl2-Z7h)^cf&?;C|Hd7D>;^
z&ITJITSNTy`X6O)utr}i|AbO$%oc`@_BzYzA%5eLg29hdGm{*j(~i5Ef|APOL~`<~
zT9@Q)ivhK6M)Kg6Gn(rgvFUFjGSXqA`2!~ouV<oE&9Z*mkQZ4m*j2Mvn{eLHZLz%?
zPJdfL62iJ@Inuml)vs52lFr`xikhDO6hveW%%r4#{$atvkKJ+bs6TuF-2)w%+1Xt{
z@I?_Bm=YEwqoIKha>{<rXWQlG3!Lz+`?J-kAa$sraiIE)042#!o%nY6*|LrFE6`E_
zyvLjM77I%x0VsF!!*s7%z0DC$$CdMxwX>5-I90Xxv8S#mDT(M>2ZdaIU3H4?0H7<S
zC?SDheM74;A*_<}aqcISa?x3QoQ(yGVpxut^wZfh=DEE=@v&uuW2t~)9FEaXVWqJ(
zS@pxJJ-b$%nOk2@<>>X?NWiDI*~2D^OuN@W4P6MzZhwY4nl!p!UxP%PwD@2IUIiim
z%*qxFEUeZ1iq5_zs8R&Ae*?ABPsoF7R=9YX?JG1I4WM_^>c$2JF0SkD<i|Kp8_*!-
z{_X~}aAEmjak@D?IjI=b4j$JM*BiSLlwE$p8kk5S1l9<V&7UNOeid9c*(8y6vQW}f
zvE9e(1&8p3tT+hSeWLHH>mAM~*v+Ooz2ptVm>guOCUeC4(`dd#3R8I0;$_y}Rc9;D
z5^zOg_D_GBJ7!k-8C3IQYRzT8MDPo6=2WV2y}F>ujq?tE1r>|Zm+UR+m%e(ZeP51#
zLWb~1vYz9S;*MrY+Pj60He4XZ`-4s9sdECbUd4_6nbk_>rBiPAR~sNF6UT{$g>}7G
zxd5tz@%{2DDxz(mY>}DdWCYq;0luL?YPcKrI#?HN>F|+4X=+oSTR_I>#<&6eB&ml8
zh*Ruk!KRgbf)p>s9`1*7Q#@J;cc{T3&eMOaVLUmw3OpH^*VKM}vbv08c$q4HZp|?a
zA{(r?7fVgLioXAgTGx&xxpw!<sc3+{+PCV??zHo9E)x~2l3!mS>b~pe%p5AvhGKl_
z2z9v<ojs8sxIjY}5HbUy0~N}snwlCw*h9_++pM`VLx_4I4K=mf-PuH<q@=L0Qk8`!
zcsfBG0(yZUKLDlykl4Nf1r_w5_iSv=pn(kEJ!ng`tY89ibON+q&F?_#8Xip`MGV-0
zdxf!4R4V7gg7{QmAq;Lnl7fO{zkDhF+WZYrmq&tD^TJELfgzU=7pDR_M;aQ?U59h5
z`qYPZ7?@RQwolD!OB`Mq?OX3bLyqfs^9ChZS+L(hr&L_@bj@=5YdG0+&`mfpGO$$P
zLwG4XK&skpW8i6ygZ+)Q3@CK6i?taXCv92rm@QHF3{{-A3|)|s$k8vLb*eUxv;P?)
z>Ctp}*_HZigJ}t#bkJu9ao4hjJ6ra#sS-#C17yk7701Kv2~auifY8y=f!<I6?*Y6`
zX0n^e_55ZpO(P{r0B|LH8b+nc`Jkgu`vR`0&>u5*bonUE)X`y8Jqvmof;$~iEq8gi
z`+t<Z1yq%7*Di`(AR-_Q3eqLrU{KOs64DLQ4Jy*1bc29&cY{i^=#-XjkZw40`M%%x
z?>+WD<BYS$a7e9Zt#v>5J>!~lUe_Vig`i?VN_zED4Krk^W~)BH&(Es9?|Jz1+A~fd
zb)Jo2r3oQq&=DmcT-#?c><$+L0W4b>?R2h7ytvwy7OOBOi!G{eNTIB|d3S3Rn>PY1
zVTA{#4OnClYO=0!1y2U`uKK9Z@|U{RKT=J84C|U?|5*4EgR}rjpv)B_8o!q2?@%GY
zp%}o^_KIA%<Uk?{)B2u+Y!q`%e2>NfRW{AKRfYJo>9_y;ImzChQgDe+%=1+6QCQ8@
ztc(2@&iig}5T^~q67uO1P_TL;NnURSXxhM4Pw4A-f0H5lYuDS7QTwf_gt#~!ryae&
zJ+r`5ZFN-=-KWvd-=Ev@&*M0>&rAQbb`ewdqCTriz2$I@@*Cs+bGQWMqhF}VG#(!R
zR~!zR!ouS}<FW6{Z^LGXi4^S9S4E$Q@Mv8vVyEXhu5F_~sOo$V^L8f%0yKzfnT))0
z=Toa1zB#swp!fWZdf(vx{jU$bY5bBIx)hlIxhP*@>&nN5BI2WIt#V>Y|C(<WcX<G^
zv&r^%@-FS66^NF7#@u|^mUndTq$y9p?m6A?#EiomKZ40mtUEYGZwM4tONrSu#QGId
zz{evm`P-@-Lxpahd~VNCse1d}-e)aErp|X;)iVgYw9BdoJZ35Ci-?6Cm?HD%AP!Ef
zVaJj{2BF9R5`~=b{s8K&@(AfSSo3pRM|(d3=VB9nAE;%=cvyd4`izX4=G130;^6g=
z$?gf4SC21?4{g3h^qXH_D@HSB3F7%wmH!BkPW_+oUG1H&g+k25(H0d|)lraPiEl)y
z%Zq)qXnxBaiG@P^@WeTRZLXr)fpQ5=Dw3u1&mosaLr8Ht_MBAYMS`DGK`nu8J-Zd6
z=)RUYRn5EFNKv<});!3dma=O$5zFcYWt%E;W*!U|wG`V=Q7{_*uaKLNeh#xckp4i>
zZiD-^VknFbdDnZLA+%QieKx48m-}u_S{s!m948KY=1h2a`8zO1)E$5@nic=;<Y<PJ
z=5aFMI&fd)vgF+tl*QqFfZL(DV=QRB9@ISEsG)?9K^9#!e^G7<Kp=UNu$hJeE1;nc
zU3TD^Ksh;2i2w_W6=+DIhV811dI_p;N<Oc6!HwAhH+5Khcyu%;fD{ae?iD~4-n{#p
zSOV0iHyR<?a^D|P_Wbzq!AoG?29wY65=OSnfYkc-6ZFtX_|eyH7%zWGgKG>4?>UIR
zgEeIYc{-gY-yfd0c*2Eaf0(h8a9KYE*|jI|GSTtzopalSQl%B1yU?GKCbuzGX4>}U
ziRVo;>%s_|fXAQA*vBFKg(*NIR(#Bd57Q-LNFUsWu}wuq$N~XbQp#bYpjb?c(hL=S
zda%!3N~Cyv|L=}|??Sw^x0TM3!-}Y8APd_1RWz5%{N-HuYx0ymq+036*#*N<80b3H
zoxRcculf1{r&kv}_P#Gj5<XUq4E6B|x;fsU9RAHi8K`2))t-HN5I4$T-;XMA+6bui
zT?-D!$F86v1=KL2Pr1Qs!p8$KL0@P%^PGL>YXg}8xp?H`PX}Am9)JS^XS1Oe*ni8h
zQscjf4&-PsM{dlDj1OV>P4V#qpvRwmW12|L^_(2n_zVI`C`|$nGTnpQQi;4P>iEXq
zpbltSHU^Z0x=@6eu7_`7d~ldP4vL&}-#=Uf)9b#Pl|g7GG)8(m87wO33kd~0@)Jyf
z*C2#(cz7STlR<RQ_2;KMEC!v0YIz`Yb_9Zbe%6^|!KJRZ{-X6Lo7puV+5#p`eQkk$
zNXqR^<Fi+9sigCm+46A96eGW7n$4<qUJ322R-;Ya>w80^A8aMBcDfsD;i1dN5K}X5
z7TB?O&)G;zlFA?s-;B78{r>~*%Kufv>G)VcKr30sSCI7d9jGQ*STMX?1{cmt%t%Id
z&EY+$Yx0WgFu<~Cd7QIp|GW+b_YhH57=&Hw7i9q{NKieKg674veq^S!^K9vG+C#v0
zxfk5CmBYXfB?WqG2vy*Br$<M(uYZKb6YyX&+n^+90o=u(TQTo?>9zzy`^I)|GZEBP
z-~k<a?0pGd*v%B)c+U4DHF@*UwjT6;x+?0%38<N#b?8VGj5dbZQ@$X6*Px2HD=C@J
zzrE?(sEHG-Lf3N5m4DhmceMn1)utR=&(BKL@@%|8eGei*pLY@xoq%gRc<FI@?f@MS
z&d$!D`4}#?0CJQ}sJoB-s@{IvR?8di>G>=y*^dfXm+#*0?!bzz1zRgjj6z6_je+sP
zgf{z87U*@?*wcsjH0xj_&cZOHih=^<a_e)(kLjmhAm=q8?;J4SvA~7-UQ`wdC)#&2
zzg);D5OwFo%==Mm9lY)ubxUto3IvP@H$CT~193lJeiuJ7MsHV+|3w(t<zK`_*JDCy
z$4c<p&aSMXL4f#+SlTyYN~?mH+xM<t6%c5Eay#*791;=_0B%51&%?umKk2iJCGk@x
ztUy*M{lY1|aa~4J<BI`2Q_vW`{k96l(y;bV;woBNmPvbyah4GN10N?k9&^*E5^E@(
zO5u0t6?t-zQ&8``-HgNYcd9>7nCxQy{jFyqe;puHCHXK9dD~lH&^TW!`*N9HvE%Hy
zHy$@HdF*zuj?!g3hmzMGJ`JY$FNb{mdNnQk1JR&X;-z)$A2uRW*uKB6B_0aj*4N6N
zt`Mr*Sh{~ZIYXqqoQ^KGfhoMzkmxd<n`8X9tFsa7Wsm$=hKNgv3cVgjh^T_5CNVCq
zsI%{b6F4(~Q2u%TnHZf|w>b$PA{c0cgIM@)BV8thn$T%Q#BEFUG7}g&;Pl}=d;45B
zw6}3<LFSGy{`CK#R-cU@9(MU%ea-SOxW$wE54hDG3f!8BJPsD1g?ZES2|az+R5?`9
zfzV`;GRak^WE4{S#kV9hiB5|#p105;dptck$sSsTh#s^Ep#h0Ku1X!`%JcH`FCbr6
zXD6pdMM4tDxPtiY<szq6Tks~GtMQB9_gKzj3`H+Qp8I47x(LWYB`h<q>XJ4pDc$9Q
zLBY1xXk41e9)zmTWBW~hJUnrfeDm#lSfm?q)d5Kdz-PStYoj=BO0IZzo2PU>1Jh#<
z$~S{vmUVp6&eV6Gum)QWp))&vQ03+PJKqPu+79~kb?thq!+GPNgGwZY)Y-GOW%X&a
zx=r>84mvXF0LMc1soG)79R`7Ctt5Iv2_+?;!0FGFO$B|sa;rJvPrX1UA1_YU^~|6%
z8UVn#r2KWbA`Aw{O9S4&1N_=j3ET@C8+>NJ3SqU!GJuR6IA6EIr1^`~@<8_Q;2>G&
zSD`ji2No7IWyl01BJXGeuX`{t-^oIh7W_TbqBvPt!s@ACTN#ld*Y5f4qJL(I??1c#
zC-a^A`RE5A{D6x0S&hJf_WLO%>0;!N!E@7V2b|Q#_0uFCfkrWsGS0xz58@-sgg2MX
z_n#t=S1M(H`x(=*sdD0<YU^*STlKAXcy+!{{w>J<C%!}ZpAJCaL2;ZfRD<GRUcrG<
z%1Z?9j1{XL!7}k>&y-s$n_Zm;iid?tv8*1An4+Ni`;S-E@~)}d1=Qrd$r=8bqgM8Y
zHn^JV*RYAl^zet-%wferlgV-JQUAxZ>)2OcZ*F}KU^1Bal;)ds7!XB4xfJg*ESWeF
z7gtLS_!%%ru%J59=SfH&Yn{FyNx&<}imdfB#Ra+G#9Pz;%eP(#yWyYFC|4Twr9eOt
zsn0~M;_vAGyh*hpG93h1lQY=z@7M1Pjo{2>?~Au8cInG^)Y@;Txw)N#)NFmn677fQ
zZ;g$is8*%d{xDhQh~`kg%u_}@AiD#5s>VTj-;m-D*5M=m1)45{NvEltRs<?W#=#J^
zPag^jQM+C-G&Z5ZocLt5B1KqT@e*x}L}YAiHRReMqVqE|NQsk9uay3NIk>@r*HB2i
zIo>PKz(hG%&pbcBc4(Wqckf=+9+cUDe=0gIj>D)IROYV$^yP1PvaqEVK2=YZZuzV5
zM|(T8jQ=rP0KHrQ_&8WuZSC!+%FW|Ub~_LVAZD@D)g36*ZUl-oWs_9y@(<e4d2cE}
zw3q-NA8h<om5qqGbVPZkn&1Ehz_I(ke_Uf|ygZ-2wBi84(B5oCS`Fd%+R=awg4}jY
zY^+oavk={*-3HI_=^?d}F={0ujxzb@f+iyHN1#MNKoAF2A*b&6cv9T!APfJGMgSoX
zPtC-{1P$XgA*i>de&z>t*|LfXtrCME2Jw)ZCyxS<>Ri|gyfVrON?B5ppNaW^c#Ev(
z&)$l>h)652{x!>)m<|uRm%Wkg{F-&1joI3eeM}_aUJJB8Nv=D7Wo7$Nr#m_&bpXxa
zbjdi2vC`MAWf_rB;LFHZf*+(yB~$||h0`~raL?cVDg+P{LFeD<l&PeCgi<|&k2xzJ
zHWfYlq;i%HL1{jDT7KOWrH!O*jA>a;rrc!KA|{SB*>RG)9UfX4{fz3T>RhTJwy(zQ
zv^7uidR$`FQs$U*)@4jPw6fS5OUQhDd@79kezvt`LVXn?up<B})G92xg4zKiEmh#z
z@vNdV`f3{(w%f%qeRO&X<!7iUN&pep+-HHs{_%2i7Z7U&bOx%9(gNqU5F44$_Sr`H
z0xBC86Vu)h0WdlAr$RZAf}C73fm^Z6Daqr)3CPXiSVIITBO{Z>o~=tzX$G>k03$$%
z0xQnZ^KKhQw#E;5i+#Mjis<6v<IOBAG-YJI7q5e-<^<9QxZS=;r3^q@LqeF|{r(Pk
zWzK=xhLKTTjG;evtK<OIAJj46a6+cC)4dSW^kC=iFw2wzv;ep~ux7vmNI+E9INL1g
z)JlB`+JO`mfG5O2&IC=Fyv=OW9ngx%Qs`KS;kL`3mH9K+IpBVA18!^eI?q`Twto!Z
z#g%YSBJ~{rxB_hljT7tnHVoKWVAfXZ4~~)wU|~Q{bOG@eApQxngS8P$NFk~Z)#O!&
zC?80YrSHY3R+Aa2nW-Ufrmd>DKX(29+|0X^Svznu6BNf!OY8l&8Gq8`Qb`UdkD?j7
zO$`h_B#Khi_feC-5X-|$q^t&lZeF*OQXqGM{^rHSMc{MTsR5U#_VJcr{F!Htvv-}-
z?joQH6C04i@bK^qeS|V7J3D)XTd4EFdJM0|fD(jQ<z}Oy@$tD?U6OI^L4JNS9@PD%
zvP9$01)=$8ClT4L^Ux}rKE)o>0~`(b3zbOF7%mG6>VmxGXYo4tB#<V~McjTMtFBJy
zz{Ad74z*}*{=Wb1rhrfsK)_0xn%%`_PEJ)NB{twV>>@v;P)No;efsoW1(dFBJ?Jg_
zzxla7g}P?x=2gW+e2|e!;Bn}{3<wB_xSO39lW~4_@45V`-Kc{t$fdm1)rA2Rl}Kzq
zLHJNm-#T*x838+6TcoW_8z%OU2mT9GRMg7x;NV~&p4=HVgUfH$4J#-aFRblsY77K_
zH7;6tXhl>0Vogmh_s>3mjPik#@EWnK*QUa;UqD88_J*69R<q+H?IES`^p%`-?^6CP
zV-vq%L~ik`JZg`Kk-&Z`D2}w#Dpw;eJA!>nXj(a8bSUCJv&{;mBI{FiHE_=5syC(O
zs?8Q4Q-jE(yu93^FXaZ#2kO3ofw`+uG-e8|?8_=~3}`Yne$d?uY5+xa<FH?$ii3cd
zR(}+J0wf(P%vS}UhKN><L*SHsr6|3{C{MMOI)4lc6_~RCf1^WyB)ba;rVgp!6x)y`
zd@!knV8hO}&S918f~%=(Xplc_hr|aeWVvcG=3TrSOIj$gV+Dw&6(iW9Vq3~M^lDAh
zQIF`==rpPo<h^2t6Nh!<K?&W&P@bQzdJnen<Smi4?3^7M#g4Cb^wf(Ub<&GNf2~mL
zWY*w+Hg36Kt5D{te}B=}VYHQ<l-zQ1?r!ma_iao^efSzD!k3el$cmeb3tF2q-AF)c
zEU~thH%6kx&02Jz<nx)igOXBYx>;=b&yaRK2n-{V7za)^Wm<z)e|h5va<ZAv)*Vg-
z`1&sQr`<yogKOflU6z4H5nIA5%?|D$8R@?zb1d@8w2NEHN*|rYh69-puB{=8b3cFX
z0UwiHg&H1=mdV$WWv570&-OVeXo%Ys_FIg^Fsu}$fLuI00nIu@ri;b`msfiSFJie{
z+KC2T&yI?C$(fjB-P`2NwS1|#xT+L$luU<n`)s2HH^$~HSN*YR`Hy~IJ7$*Q?VVj-
zR>{(hH$e=)ZOSq(GS*1TK4rXAlk3#NN$kvFA2CsuWvh|TqGU+q&M7}!u@01v63Vm_
zf-g$s2>KViUYU#A#z%GBXS+P<v+7b{nH^N1&nL_-1^$(;XeP2JPhLPjUyXw$plds8
zEcmRT#P`AzDF7eM^eS;02>N)wbXz$%XjNEfynfvdYYf{+aWoGyzs;FOpNUfe$aB^H
zfX6!l9d9^|&~W(C)04Ig&i+?FIq94dKd_7-nuE;71zAA?94dC!Bp!YG*L{ruQACO?
zVS)DPXA%L;6<NvD<T3+E`CJ5+5%z-zVBsI&<C_>5EUv7mmg#>zxEcjA>I^D{k3n9Y
zmev%WUd`AHjTlyB5H;XoVJVz9XBnH|daHslf>Q!ei3N~#my>6+?Up8aTro0f81s|Y
z7*Z=B9V;=E2J|E$Az^`ft9CayDoPnDojp`+=3{%%Edys}4eY+wLRl*~mntyTP_~7}
zpA+z9@BAQVy>s{OEPpq$BBzu+65BESSTZ25xJN^6L~hcFd89P*AE#P8nbrN@?fk7c
zKxV+d|Ky{h7uvSK|9bw@hEWx_#A8{N0(9de_*!7v2Fk-jwN5tIu6Z78jN2H#RE^|w
z{PPyLwZPGU0W5dgHE<e*G`brSLH9%G*#zB5$a#Ty4gKD|fxU9!0VO~~H&$0+f(#fL
zYcMILGT94qbJu}F+8pTdfI8UYXvRm(9BhUWP1>Nm#?&3-u_g$w!0c3gE;ZECTV7k6
zhR~@M<8cbq6E<n8;0}wrzkR)i1qNdPi*{dX18dp(?GY60Fm7Lm=q7iJx$@o2OJLs|
zbPqR#yA5RFV5Fe%7Vc3fy_OAiRp1eQqWlg7dDIN^I^-LR*X!L*0d`Q3PTT&ki!`B>
zGFCLj*As%lNvLbdi}AW1J#)zVFjn8TCM_;*V`Fm)$*Zlpi&!I~P70(Ys%mNgrQ#$b
zdv#$1J2aeHzOxJY$dHJE?U>?Hz!s7BLpMJ5QQWX%_SnsfvTxCsV4_Xc#H%~Y#uODs
zHx-utW2E1t%#jXOr?7Sz=quN3G7|o8K~Ji&)v9BlynI<_8~UQ~h2_0q^_O)&-1O$@
z*Bt?NQqEQB%TZ=b<ONp=$h?GvM6PKNlA0JI-Bzghi|-@!m?0L<QaHJL6HjwXQ+?cY
zMRBtbWgo7qt9ND92cRP(XDXNw4PdN?(Z_p_=y2K1R1!?eB`8jzy9i3rt$}#*E^lBz
zmP2cA-hJj;a?=&;b-0AJ73u8t%**(M1gIWssHt)5wLLh#b-H>RG;j?Kmw@prZ<<t-
zXtLVwO`S8F(#!J$li`Ob3TWNgNC5Qt#!rG5$n@x6X|8{j>i-qv0+IT^zd9bK0TKd>
zOBo8q%b+GJ2O&g@z~+|YNb4UZkJcK%9|%vAM43EsY~)+fw&w@yLY387kk2IT#{aWa
zpo-V7pK-X9#gA3S;ZW{jcAB4l=}^|?rPFh%VLH$^Vr$V*nORq5SHf6^qm_NWF;?MI
zd&<UkKX%m#(Fy?Bl;F`##|=GZe^i|OsqxvR0{L?h4h2^#F3nnl&4D~awF)!K&a_|b
zY=k^L!!;Ghotj1F+%cSG%s){8_5c3nek*6$>ESy~#6e5_Ih{n;{-!2_lB>~(ci?xf
z@|}0``T8z%eKzI}=53j26qDS_N7_+F4L2)p@=TxZj?gvxSTlDh_qdOB&6hV!dOWd7
zjEro<B3Zcq&Y6j$O2DaF;ittpsonFi8h-Bz?~3{c4%U*JZ*puU@=bh}p`!}PC^j^1
z=~-<fvujS^gk0(4)%B@xF8;Pi->XZ%*3V|r9`CGX{j5*T9gJLM>0gDf$Bd-c6GD5g
zVp1}e3B+w3H_j+Eczx62jb!w&eYqsEQ4@O+Hdnu^G(LNEC6c@c3o-SU$L7-7V`ndd
zB?+JX!P!>zIIORwhlgR$qnRIC#jAsKkZu8;o$x+;ZN4hpn_i;sY*P#eWrc9YjlqF<
z-zm50RiEGCgOvKg-QE+aE7FMAas`j6T4V9l!1-btGugEzx{3m^qzkWC1v5&5^Zk8}
zeTxPjxjoD5y`e69j$$I4-sYG;LJJMZ)Jf>Z*1y@McBuza&=)OPRBV}4Kg}I&@uF_b
z;K(<dgG1{G@YTnUS6y6A0VCuv?*GKY-+EpHFhg3+#0CJnm%tJaaS<3d2ue>*uo28K
z*^6jw=7#XpzXCY7Ymohl<KvipkkD4!uDle4WEL_DvJ@KLQV<wzuGcd(B;kDbn4e#h
zCZf8sQX=}<kIv3OUq~De0U%!LCL-P0+arVuKfJ)O)nn2a3F!VV0-NRAJPW25iSxF#
z`5M03c_D+(B)Lx;DwYPQQkDC8HE2lPzP$`B1=0w><)K6e@J~V#G8tuY-dhH(7T`&P
zbP+xq<ZaxXoSEIQ4WF9}#glBDX12U3>~xD6=7k`_UE^A*i~M4xw0YI|)hlR8Q;t20
zoyP*Kd3E$O#Lfx?ep4qE%L8SM_`i?yM~kieCd`XO%l8q3@~74;@*QapBTY*fZb}y`
z)sFX#?7g-%#>NMbmIMn+&&=!$FlRwQ!OkRsqyXlH<j{T%sh2N*hLVf>0!gg~_;?at
z$3l}cC<hz?aXHnL#X!2`%a<R)v}kHhf!7A@_MpH(VDGQ;e5dFl+6R~8H5tr&9Y*mw
ztshWuc={L9?_k~T-n|Poi0<NbKrN8WdJsnC%{><cTn@0^dpHcN?Cev}XbSNH*gq%s
zCKer&HDz0H5uk(wVG<r5UIM4pNAY=MC%|ZHop!-?!j+JE2T*9}ied4ose#xInrqUK
zdo~j*UU4ZbioD9b`s;M1@J@KLJORliSxx2hj>$4hgXqx5N^jW7UzjcFsSxCK^hBcC
z*Hd27E!G%R1nkg`mDp=q%LO`4euy;O<+UhGD2Q|jZMV%H7`sSIwMCP-uAf#+Q%e8r
zSufDI_x2Vl7idBRz#WMXdjx`vn3z%MS+Sg|&B8SU{6N6n4ET)A%yJ!TAnk$996?)P
zK?b)CR@f?8C63)Z`{0XQP9%fIDzKx!eT$5(UU=SYZfa5xJgcxkBjKL_$?k~>SD-^y
zN2?BOgu=OSFrhJQdObZ>Xj27#ydBF-?m1Uzdy!2QhQ?$(0s=^s)J+lK3v}V{jEsIk
zEK{UHh>J_r0NHLXH*(fJ;$z9G>;1E*65=$wY;yOh@0%lHpG^n*R_`$h)Cm*_@vs&A
z0BdZwVX8@x$6V3R^I&``B{?xRNO+Oh*L=D+T}N#oDRSs&ctN>{ipXvbDn{=(P|?uz
znKg8Wg@b5xoJ~#FVA~I&0w;QYe!f_RhPpb`T%-koCKDI_K@Hy3%`Gf6v;{)RVl!-F
zxv^zr;e?P2GFv$%C1F9-#&;_)8v^tNJ2#;NXO;6la3tp_<%KSVC}$eK;w=rfdGX>y
zZHP55JqH_GgmR{>vS!099elXQv$=So*7ldusy-wRUp!30!a^S>Vm{#5lz;Pqn1#R1
zUaTT|iYzyrL0P(zaf~3g>Rc{QU3X>(aW;|UB)0T}OJ8UB`yZj&osmoZoJEXfCT42d
zUft|+yRr`kY(U})1&-(%ZJ2gspH<+!L`<@SoP2pX#46<<KD+?xV@Ly>z~KWvVZ$Up
zL@c*AZrK1x7&7dD;Xk@ZHcRJzZ2%HY{RV8l3{45~^Cu}M=P!W5TG12%l@Nm7Hb{KM
zw;&#|9{OiX8_eIbd8OnU_yIm2%6qks$gVw<=ZdAa#^e2I8?=@8{`%zK8xao<3l-Pw
z3w|>o6{t~AKE>VU_<+v;58Yvzc#`iQ*}u$(nUlU@z7J=Kb&v~&@|FF!&EKK?_r93O
z@Bj7e|6Bq^@b9l=NLG~dRIFr^vqGN##D(lI0MQ{texv28l}Yd?Fv7${Fv$6=82R+?
zS+h*tO-DY``5Qf5?M0=<W@eLZeLsGD)5hex0-VW`euDnm$a#tPtHSa5=99aU>Lg7R
z_{(bqczChVv8}BF{7$Q-^&e)M#iN>MnX|C0{!%MGP(049PS(CA8>5-CcwbO2I*RUf
zb&2_!`#qmWEy8Fzx^qy7)KFL3-yFi6L+)~v2+kH&IrXPs{1zNA!;;uSZ|7Z(5BO5s
z@2zkQSJ=s^n^iwMHMX{<l$OQ|C{iqdg%jhLy6r*4w>UR4CuzJJ$5zfZG&VA2ns{^2
zetdQPB=<Dz%^X{UstXCqV6!jF!NtPIXsV)0){E_ddpHH9)b(4qHt(6rpW?k*SRm<)
zWkF2SZy3K+CfN!m8s}o`cS%jkrWp!V${WwR7%HJ$9(uC&(|v-F<n${;?ftq~3>Qz$
zIgB659&T54L{KsnBRF%Wit`SOFl8UTz8s5QEfYH#@9KW0)hqfXWLV{Up|Epc+MLhs
zBYU%|9PQ5^9=>+M!>d!AQ}K$b=2#nIA|o3ZXe8o6ed~4=<vF&(EN(_s)zI?y@9&8<
z+{h27ir73<-tJMcaR^9@=88rVGj=(WBD(BN=8`iqD0>G+IEphDUDg^!g74$8XPdi7
zf0oi%a(@^e7S_lIun4E!2_5`oa?A;ph}d!Omz7h<q|3p<Hc4-Ko=XA$`2h*%H<g){
zflE|k?*6<4AIplwwP8z{+QOB+6%X$q+{yAf!_S8&pw_N+(OrL6gm%b(!lO_*$=!Uc
z`f!`1(4|K+ZSGdY*l>fP^tsDH_%Qv&!T465T7~Ikx+h!=5$S>K$PwoOEOTqp1J~(y
z;n>Ps)#2u{x>~QA!}X2H3Xiz*&AQWr&%X$=>}F|rzMTch=7|kF9{H`LR%TH#S8-{#
z)i>T;<`z_uWVm92Z&Fq#CE>b0Ql|{e6}-@Y^cpA0VRvWO6bUg?vlw)(EYHBZUJ5s6
z0bmL1O->eB^?G2zTQ?dhot)tf`%Qnx0tbN#dt%Qi68F;s59pTOBzV@Fm(cnLBQ@zM
zxnha7org6S_htzjvpUM3;q1MLR|XaK*jV$Ew5;8TF3UdAhOE5Nfum11DU{IC0yTHK
zLJxHDNUyoMiPIuX%GZ~YD%{TZafo<I8R_WMOAdVW5e;|I$0Y<ymOA4&t<KWX)?n~}
zn9u)WWy#$0=KbIPXT|2HEhNL$Rb@^`e`I^lgccVKu3WhJEuS3+#Jy@%az9lZFSo#p
ziP7??vz|-uy|F7mp5oD+c)GJ@`AVr^?_f1PGO~W5_0RY3A=^dhux!d?t&cB*xv?Kz
z48Y4xOpNh5{&{9}p#v>ulf&~XyI@jwmM|jxr3iV}=Lh%xgrtfKXgvuLeW_G17!K?%
ze3I^u2nY`8*e0v0!jhzMhIqw-cx{*eK;AiCscl|kZw_%6E~`g!QPK97AD)gBQmqzB
z$xPL#)<I@ozTH}tj_)ot3F2DVG)vOc5G;cxE|V$EJR9R>=es>*A3l&(#eOCd7|oDo
zq!mZtoBi2~DQP&<8C?uoLm&_w<`-5&hfmf<y#6@MH{+U2;c0c9D8x2}LB+TAvV8m(
z>(t2rqkKlsdT;%?<+#VpCUjLeG;AEDaiitH!5?)i?`XV%@;&hEt;PIhxe=O9M@L6+
z(n&?x)8x9NzHN1%i3vi>H^bvY7w*<CEQ(5lbQ@cog`^3cvDL~Pcj)TO#VKuagilT)
z?i(&hzk|KZ<8&qp-BOi%XJ;Qb`JAUuw{BvTt@W;SW?_O4$kz85zYss%bjS9&c3D@*
z#Y$(r#XySBanZ=wpfa2uM0*IMp9f#GQyDale0;CAN*p;3qYfcfX|Zo)^BJ{bldw6v
z_SRPU=fyIcr`LWK<0w5jT&LC`mEy`$SX~`1g3`fAfu@WbD<vfd1v@_j|BwEkKZl0W
z^zRua7Hle;%@<;%dV2D&jLvCnZjRbqT1=hIeK^NB!)%@IqT%rN_+t8w7Cyx3X_FSc
zTCM!t#*%6^@2kA55s)}g*=|;*3?|XW_4aL%j^AtEN!w}GB13au*$B!kHDtUrRqoz(
z5Y)WZ(|u$zoH6^f++=cdJ*EL7qL+dIA8#gVU~Qco?N=U7cdc^dZc66|pXLTPD#}}q
zT=`AU*Wz^WGs1h8l|%I`>tV*SA>KKWU_!!%MXO=c5>uXe7e3Q*)F)wwfE@z0rtvWq
zzlT!zWq4mLM=QDM<z0;{N=pH3nAo^8*NTeAUhL6;;&2%3UeAAUV83IgshKce0n8;S
zsk)Wpha|kX+`M0We=SumnUXs1QH_mcnC`dA4zQTn+6Kzyn79bT!OlBz@9K)HfGAfr
zk~{?<#v!v)bYcuYu{SlPgxq>x(}WiIuVUcfOqT^zoLP)8Xz1v8mP!$RUS$<Ne38GX
z<l8n{<wQS}YrL|9T9&TqurjM}renR>Im++mOir2c0Ff>Hbja+_Kt}ttYsG3y(3^={
z?DH;|(jxPY_ASaK{1Fr(_o98<1_u>Z@<kQY$dH>g=am1==*(4=Ubfjlmv<#z38RyS
z`d71l<p|AvI!zy$W4`rk-ifuM?opHzZEL1is$GD0c(lf2+Z~dbrJRBc5kadyW}E&T
z1H@qceT=>(aRCBuy+3IP<(|Y-hoS5*D+3vw@k~JtG%<m7_6r}=(wfJgrT+L4usP}R
zX8{M%5=fi|$WpcBEr!iO);3fows90m%nb1#_?8k~7DA67)E*qH>5L7&ZZI34%8)L&
zb_-oH{#kwt0b%W1s~+M><${xZ*;H>jM0K^@V4Yc-eajfG+{No$dDHy}NsEX>Rh0K)
zyyqrPPH(SQT+|%ysX9AM=ZChANKHAN<U2W28V2MP#k3~jsj{1_TgYY`>iub&;WK^X
z!Z7iaG>ScOF%r$RpbuJ_68Oz`z&4;0P2im9R`Q<1+U|^x6ik0<IJKP4-^dy;98_9Z
z9A4}g$}vg$=ya8BvdnlcdoX)!{LzxEEh@&1V0>*OBgl_b<jX@EZ=tVGE0a<S*d8q>
zi3GPcwZMGMTIC&_D5a77y5+XAwGoXsHOo8M=~;0R=7DmGVAS?2ex{zVnX19L=^9wO
za(<eYmN8xJw$z_Sdwy~};^+z1@R?0ZX-iPvjjKPPa%Q*UO9*Gic_bq^ySI>=zlyCG
zA{_;eSzGh9l_8h;-VYbfPu<Q~Q_F-@-%tj45Z{%)oyT3TE6}E9WH4Iz%TOu(*H=ZG
zQ{{nK>H!1IN=Fz{iwF-_ueGg$TzSW*{_JENl8V<)LoY9CP8UX|rQ*0_#4q;-s-_Dx
zUBJzQoYO-;290`V^On|$S0p|07tK-A>h;zz9mq?)vhuT8yb6jy<51gIz8tk53xDmO
zcQTt}skpmbpVvOHw6bF@F6beWz(7T%US~ig-7Or1e{y0Q!|7h?(rPcA_;hQcZY!w-
zM^1jA%tEK%2eSd*5WJN)OoLi(00H=@iElfU$OXw}i$%T2zfd|-EHRK2k21`3a?Z=h
z;8;mTf5f1+vsp8_zqe;n`(vwiuGZAylL}pMK!8^LENydqEH`Ou2}}LdepXEfdv37w
zzq_gLf=g=i(>-}59sS4b#yWOv3`<r;eYohaMCR7x^reXTmhN76mh+|-<22nE^BLmE
z-+1J4^y5?EpQNpc;~pR&7|*f!^*L@HT{3~0+UO$1ut$!FkZ`dxarE-|3W|VMQem$o
zc-%-jjUUCiH7>2j@|0HS`<v&QRf5H<r)=BXckIi{VFYEiABX<7W}(q*0|N>~akcAR
z&HdRY`3jd8S0ESK<=6O<7}EF6Z>|$hUfFfKOipUDA`QXzYHc|G!mPbmC?INVbhJC3
z>3n_EqpWu6!R`1s4jYRp2?aHLk|@WOogrB=pB)@Lb~8~wQLNgx+a-oslChlbyr$J>
zFW(YZ+se1LP${M%9xHp$a@T#xG3knpWayFEJ#@DH-G)X`5lDooLV%`O;=Eue9#wE(
z6>?i1^VNCFdW|$YlJL1vJgu#h+`Txv1Ola2%8J-iI+XXpwslGx6f|wYCq>J~W)@W+
zOigi$!bXCk!*wuylFwc4?Oktf7V=}_NKmVMqpDh$pZNRgYm_RM%Xl99v!-gcX<e#k
zRVXNSAM!`C?*-E5o_3m0(U|D;K2Nc9otePKBv^Xku5rWpDk&7y*Lt%kg)zS}v~^<T
zziH4O@0r29E5U&Bo#2tx!;tT#$=^$#U=VS+)+Pkl5O8pyl04TK$hPCr)t!?h80O;U
ze#?q{B}>hwc5T)2D%FrUqM<VMu(@z<6#MINndg%VD+TGgxaqRH2X|B}o&Smn+oouF
zG1IHfx^g4ajb*X-Pmurrn=$>HOg;DX@7ZVRb^1>)fYpV|0anZ3R$~*|*?IR?=n{Yw
ztemu40G&9qm9Mi4u{KoW;CZt}ut{jRfa#}!YGmeYj)b_l&}BT=z@Pnd1`yRm`JNNN
z8OK_b)5;g#@w5dkaBuI1`q%beQ~s}rhX}7k_z4|d=j!2aB!H_R82`|(l=x5vSU*%q
zv%&+|@Vt{pj;OCjy17S(MHD>^YybBzWrpgnTm%Jh`uRo_(G@`ser0;?4!5cju{}*W
za$fA~g-HDVX~z>K;m+M(!QkGtfBCbF7hHLTdV2OO<N4#a`juwL)N2Z=l$$*6JFFOJ
z%Oe?g_4RWDHG?^;dbjMZH#Avi3?+%gtB<ed>=Tqf===Cvx-jm_-@H9OC1v*LQ|l;K
z6(pEYv3Rs-pi#>+w8Bt~t-Si~k{K1Fv3FOr;I%v@sdHp#u5DCJve<<TpjG$Rr+din
z6Y$TT?62)Ohp)PRnBi~DnqYSy9p#EDP#2ea4gMI=Dl>EZTfz8Jg3#CEk;EOAkdRtX
zU}>zq-RCn`br##7lWt&|la?8t^GO?*jEs!uO<cZ4T|z8-h``Cym6eE`5f)}<zqan~
z;f$OLn_UC{8*!{wggnkqqlzg>cT`Q(*y5TL*EWW-T5Be4*U3_B0FAoD$k<%6yLb04
zqgI`<k<s!{R;rhm(`!E}{+VrB9(wgMw01o?($J1?Agzh0=j=R|`%Nid9TytkG-<Iq
z81Fmpu)qaH#{<wOD>t8n7OYmUII;c0>{iAkl9{L8+N#6vQT3RqJu;Y?y<A8S;-YcU
z2~O%0@+37iFW?PVky2jrM<j&tXrCF0iwB5D<#(+0g=?}6Wtq>MNt@UiM0BFZhQ-Al
zjueHRlA_-XCWKild4>tzcYoetiukO^3p`DM9FT<5(Q)4Q$<Eg>-$Xv~u8xl4BIEdw
zkan*)w*GIo&`Ioecl$<02I^kA{)C;q>d=(6c2C3BtX6?$qqknN(OXGS&%$DHrSF4F
zW^L`+)cLZ6_9KkZG1krA`k)UVu3!-HUCfRILK%Imy3As%>4UG_0JDzHoVb|a#4FF6
zdkb0f$7@WlfIoBoM`QFJYO(}HiN#>yD;EykxtqM^2Cv=Ug)EqSDWXdyv?}Ix7)p1v
z+dNx4*Z{*gT5evWywuHMF{RLvo*ueHxvRSQ@rk#Jxu?g)S=!Ey^=19FI8N>dvU4rN
zTj;AjeBp6X(t^Um-suz+7j7=?0Y+j4_>yHXhI4Ln;Hs|vF|#z>_Yr-xh@k0+Wu&1>
z>04q4<^3m+lQeqYFD@<(r_4Xq4&XJ6xO4rg>6);c^Wj7t<xc-XPFEZcM&APMh)Sf}
zR-HpvXQ%ISUE-v}=9I%i$G$^+9O51~H}^#Kxz?q-o8{z1PhZ~?Hnt#>*~YHm`}diO
ztTx80sPi?8D%ba7Y_{**k;@v)G9Ey=oO<+Vk&OpQmuu<T6cZOGEjS#6pY^Ef@G&<%
zU#d+;Fr9dX#WcA$w~Kwa!0J?T9t3L*IQhK(!FDFAqJj|OhPLau*G~oEtF5TqspvaJ
zPf1EkKeb&kd;8VnVs>c!a&l{NO|te0<*TQ?17Re;75F+=20M?_Ui2wn6n-%2h_5OE
z+0pauTveaXgvZBAdC>5z*paUGNl~T`PYLnc#%Cgyi%qd>xYTcMU_N~M!ZIq=^PFdk
z(_zP!gPD2$HF9}{61Y$Q$d(pJ;|Mh5J74{v6XJfwjoaRc(y~wo;d39at>agt2a$ly
z;&cd^H=X9uN~~R+uhI5T>70h}Dm1QN;~28e!W0f1WUoX8?aSFGG<bO4G>dR))Y$Ux
zj96K2b@wq9y}F6c>$qHi@5jmGu*2zaq2qI4jveQ)x+j26(j0h)&|>iGk4yLIb2tf@
zEjNbOIblr)`+Q6KDR&3m()0}skUMtFS_GSgr37o2lidpIHZCbBzb*CS2|tbyvzV^h
zT7Ek+G1}hrOU0BN+F?+iU}#j?hlxY(xnwCIUhFUKeO>)qd!{?Np|dlML*vv`h;MY%
zE2=VT&`NT!wvIje;R7{V&`WACVP}g_xE{A->Vq4pSDk3C*v@YD>vzsberqdbI<L5-
zQ)KMk-pVhgRLA}tVR>__?K^_=lW8ST3Xu!3-<Z%`t$B%fQ*QQS|IFChf3;IdU_Sg3
zK{Z58^4uxqXDYCkl$I6-PF!Y0q(N^1y=nIAu7tSP7hF-1iODM3>aL_1LhC$jX=&-{
zLudakgS;Ucff@kHOY`$%zf_d7za}PH9}T66XH-^x>g>$@)I=HcGdGd%j0RhI@7{8n
z<Ie=fS=Vj)Xtv{z-OoDLM@f2;8h-nb3@aZvMxVLz<ZN^#Q%Dj5R2mz9ww~Y+OUpO-
z^@A(~|1Jm1iSavThnsHH)LCaMit=7#3h0~lVk;|nf_jEp+u~A?8}}rp85rCpB3$>Q
zVv!dWJUA#9lV91GTxn{TKR?~i8)eK{9c-T)-na}e7ZTj%j~V*<;u<eV;C>&+wz`Y^
zA-HP1s!Frcc7N*JCxLI0o}NBHTuf(fmdB+UGX!j>iCVQxbn^{a*h>CG*;5vJ`a?rC
zHrm>K0!s|O&HCyo3=zM^N7LX&NJv<lZ~P?U7{Vuf`0A>+w%@5%SJ|tNQCs&Ee%>#m
z#-t2P;?%B9H_H}M(W%Vw!4;+uc+e_vw6*W(uvFsPklKxk$J;yA&A`aGHM>CjYKlZu
zFyz+#W800{x0aNTy?yaH$``Zy!C=Rlm~Qm1{%928u$<UjTq1KkHp=f`#M+{u;Gg+u
zMwyYWUJc_{f~3?83``s>Ih!shI?xCQRj1Zxoi&rLRVOtLyOT9r->YnJyzk=)3NCeP
zyv0qAODgEE-g&pRJTn!Nfus2N?RBEDvO3!pe;qG|JT-pDqvf%(xGnpw^dt7%o2}ra
zzvc_aU)A`$#%g9XTH>Q6Au%zuRslE&D(W7rV&yFH<_4ZO90d`<;NSydqH$WK9Ou*C
znYjrKdHJw|jj?4-KOYw!Ak_PoK|OItj1-?;4@EjopPBkqprD+TAhYoIXMxYZ62j2$
z!^9fybSm<(F|Ibtj5G>N3`D2|g|2F3P2lAB?-0rCpx39Sp3i@#NNjlLbot>GsccqA
za&kffy|N%h1kuT^sFjr;oQE*^Ffb)Y1IS}??c|W{S#N{WT6eR!vhVUP`MtsJvi`|e
z9v4E;+!b_sV{fs)=Gd;zdjI6Jwg9c7+zr!V^Z6FTv&!#_T{QI*+4}dEm{LX5D;>$t
zGPL~Krlu@9moez4tcC|)UNm(G#b0e{Yb(@vQ;{=b(MC(y+fb04{A04_us}WU$o*RW
z2pcETPTM_Tf$m4G&dWmL!#~3tJX`3*Gz|2LoYn0x1fe|r5IU3+M3?y6jKLmhiCnDD
zuWYmDmPAY!kfrF0<LE>|MCW*~?nZ-k`$I(W&OP<(S3f*|>`1Ti==OD(7>DkD^^E*?
zT_@u^7dMq#;Pq;oF*3gLJA{yX<MO$=9UNDX-H9H-v{Jeref5QqxbKtldh`n$+DJF=
z1c>l{=LK^rGCnQPx<cuo9yY`OvOPhi;38u$`#FW~ji+z@^sh?@xvZZ44J&(hX?<30
zdUbUs9&>SJsoEGq>+s09{4s|z%&uoi#Ky<#=}dfeN9Tt(fl`N-G`9|C5_0UOP|ws<
zWJR^$d#*WZV|Y|lMp`<*<GX1@4rb(I<U7X3Fsxfz7+PuR8domR$l;L$z7C;EWaYwA
zw&|E%{l$9FALLU0W)qIbx#08srz|qY4C}i7hFWh#HA`V<N6X3B*s3kRgg5QsLu=*y
z+WF1;1YgCou({3EwUW(EGlju`CUS8VUESfygdjgB>XOhwh2iNWHdZ|i^<OqigB35R
z=!%L)p#z0N8ctP7*(4oS<b%A$3#{GNCOsENGY)pm3cYsK%_mOmX`O;O6=qeVnSH{h
zYmzPpyz^jb6|>Q6ZAXS8&bJ!;fd{9=d<^KF3g5G_=||CNt42;1d!eBX)h9<SEJR4f
z+LmfJ2Kgt5Nz(5fPU{*~4mU4k*eoT|3}uyz_>+>QPENS@jcBE+Q1-2+iTB_W6zuLz
zU=YNSNk3J~D`;q$baPP{)J>$wN+2sEBO!=ul$_{KGc2>$>Y~abIv(LptVT~*O&JP5
zT5XYUl**ZNe3++MJfB;%&Fyfw!NfGGu48L!ztr7;hnu@Haa^&yY}VYF<YqrSXj|vg
zYM)rYE8!!zJ*L9Gxj7C?2?YSeRg`7qW`VRMEe-m(A8r;DKBFYt_Z#!w9WD8ZMv#Ac
ze7?EWV7N9s9uw0X`%*Yq@TX?wcW%yF#}KuuP<ncO<$S&K^JCKP<X58!Jfr<`2TgZ|
zn}u}eL_YIF&FpaA!C}3U$DlJQR&(X_V6wzyD6L<vonu;E!Jko4%N<58bYx{!1)EYA
zyBZaA<?|SGOq}(N+Wed85XF-<B|1!|df%8r^ha~)?8jISn*3-amXEf5D5d#nl46(U
zs-V1oD=jS)lM-r1?%8Ia^t6j+&W`7Ith40GBWM{TBE^*QYq1_Y*xso{FsO^#zq`F)
z96Dy670in$zH$X)+SR70RH4{ph`h`ayHvs_{e`>RT-IrCmPTo;(he=Y^o5(IOBIgW
z;o)>6`N3vgocP&dTtJ=6meO6kdv}{k3=-y7czSH99u=hNhsCJY+O2KWyB*rD9khC*
zjU@5WHr=CvN~QvdmA{~un24)<sLr5p{K<*w*_pdC@9pH3p`~Tn;jW;e0bM^8Z>JD?
zfh2uZc7Y@|rWslbot@drtlCQ8<7{<zZ^+2NxpV!=OKrb`t*sr;SJ?&iEiG$1yW&-;
zN@`@?DJjgG3rX5a%65{*zMQH`l1*9Bjt6w7Gd^Z&YD4U%1_!S!EV4Qr^P?I%XbwbM
zZ)Z7dO%m~3;1jw!A8z{Wcy<WpRBd<mFtRbJUb;@rA@}LPJ=uvbV-&{zUlc^9emQDr
zYBqb@-t}hny6^9Krq(Kdm@Hpo#FjU&%$hg^=H@j27n$yyoqr6hi`t=1;))E7W)5Pp
zFO^PfdU}6<<o6A16vyR;t|Y22V)|ZQRdQLMAUlN?KWsnofT2YRu@g?x*}nPq*3M2;
z;UH@w{#s}p_8W_rPnU7zM8g-<VNz8wvb=@}o%MR-?ELhhtmjQNvHg`fUfmWyXq9i%
zTW<-1%+CLG&niEJiHVKvF)1IP_vs^Q?>n-_@5nv)mJ=MPxMR8PpOQKseVK9+aht7o
z!y=L9vG46q!@?JX_UFTC<pv&S=Rna?2sGUC$O_oEoBaARr1+}rXy-DLHa~e!Rln2z
zDCT}}TuXwEj*iQknWjQlR`^TiJVZIidLY?;UTe*9xWBh9Wjpr<uQ>|`M{f5&n}ZB1
zl8_e{N~7}X>cqIq_;_5XNpjavh%NU=KI34|&)*%eVV1Y~lvVY!^h{m*wQT)Zd~mSA
zj~`P}43enmTa;M-XXj*z4cUF3w-^`b=*bBP9tNz8TWX1+-%BlKzI!iqa}qlzjaFH+
zW4MAbPvwP>keXWeGgiCo)3LIVmd}<153OM&HJEpr<M-0YQ{FRs)fnKLi0l{N@w}xv
zrimdSf0DCD^_f^Q$wNw2^(j_=e;TWO?nzS!nemF;!`sWPBr97D#)!xc3&XZAO7|Yc
zw|6|O<4x?QRf=b%p^;DMrBzNemSlbQ02qaI+8-tr)URIr{ve~xZW$Hvy78xl!`krP
zajm$SnM9M$O2xG1tYzrtl-mfvl>|CPtaR)4is`R5+8Tzp2i=wj8Pp_ckFkRc%i57d
z0YcMmu7q$2&<W`6eZ$Wm%&R|ipfO;pV9Ge`G<tL@a@#BA&mVUjMk&<$2llqCjsKiE
zMgd9|9u_VhGIB~5a*-oC0m{_MXGslEs!>8j9#2)V#MBUA2~JHh5;dzfBzoUZTkL9Y
zUrGB|Ri$VA+U1}YOU&(%=*IP01=lqt{{XT#Z**{HFZ;9G!+3D<R?L6(w*FG_=>K+q
z#}jX28}%5Cbga19V<g`vGLj%dGv*_3zt&w!?;nW_wG-t1I?+e8{rI3({!J;bt=%t@
zK{1KyawAQ(BvVTOI%+!_;%|wjG2=ufVV&07nKo2bs=pbh_xBUrn{?p2di8*2UxO`#
zR$s5wemFO5yaHVzx=T(R5d{9}WLxBNX?MzHx%a#)&WDF9W|!mj24NO9{;J2Hx498R
zEh#C**=GznV<8@S|C%)uOsz@BY>laiCtIhH^ll5DwGNi0F{9|GhplHvx+v1BWo{#^
zL_;aN#Z@+jJH`6-^l#K=?66R5ogWe%|CngIG<Uk^;(j7L(N(E1wAalL>{hE@$o$;q
zH^vT4PhE%J`b^`tcvPaxo)NzH{kc0Rf4rSt1X@R15jR4}2hSe_X=e)E%P}9n2|!&U
z3JNqlOg$>tp_()mAXVw9mhWl7MEQ+Iilj(_$A6j<|28}Px2xf&I22ty6zPz?2Qn`2
z@|@8!LkbS`aO;z5|ModH9V^w~MO8bhOAqa_)%48J3P8k*@=mGFpiQqo>hh_M)g6zJ
zrlwlO90nK1_20+awQ1t*O`mvqso2X&$tk>RYukFi)Yei_q?0nSNAmKuH_bOQ_TTZ+
zGnZ^`78J~omS5_|ULhvtQOc{1kB;`CM!u?Gyk+#xUHu8I*RQqRgJsOS2M1Gv2rQAF
zAXk+#FE1!HrKQlU`PlXtjmyTSqbsd1)p&ElKYX;rxal?x-M8dd#s};EO<Xocdqy2y
z<d|XpSOf&CQwgM`zP|3qUIv|+qM!Zob<NFrZ5G=%Cmk#%9Ld<qZrdQa?8`Pb&NcS&
zOAQXIoEmvxz=W0}{b}=hZFIVG`Q-Pv=1~myVb|E&neX*WWV!~2X;TR#(a~joeNRY8
zb6+JkTE>}MRYk^~pNmuGwV!oLgrX4NE%g4eGTWnPRn`4rlzJC(YgF9az|}D~Qjif5
z#rP8y*|(DSbBWW`TY-j_t}{Z6JU37u8ke#})zxy@4dwb<F)%(;Nb|!G%`=nmwLdd?
z=4KMM(_c1~aq2GlyP{tN;0*kkVf-nXTsZ&x$FH`w=-Jtw@6Sg-0X-T?VU=mlnWM?#
zf7jaDYlSN&i|gm0y0)5oe&IJNEpsUmk&@qUx5T3;ynPNa?+RU9V5nDTQS&)|jpw2Z
z?C7AP3#@i15Z&HRE`0VdfSNVj)s^Pq`56^&Jp7JzPe}FJHGF|Y`}MEwVOmd5{!jBU
z0o&!dc_UMIY)Dd=s$4iDH6x{tpRc~^hv%Ptw|?vYV?Y?*EFvb+?vh$sKJ?S%_K%4Z
zkBppO6>~YP_kBSr@fDjk^VO}fsuEHw#3Ov3$odoij~_M11fQc2@rg}V*NMX6Vx~Z1
zteQ-&!6i2b+YYUUIsHCSNPRolFg~`HKmyaXQqcKw?ZmNa@W<99>$As<jAPZ<yU5+<
z%b9Hd8z&IH`WpN_;uZXl^55g}7nA;P)6bv9x6uzar3X@RN#T#aG?uN}i3hosWvoh<
zj&Mev2T~T#6X%VhMCl96UF2oCXBQw_haOlByKv0O;e1}pimiuT#eEc!gnW(a@2dOU
zA{1fuXOtKL;zt2bDHik8>_!`PZ#6z>e1Oqz*~WxQulmx!V1A`tIU;VnoX3ILZ~0&Y
zm7~JE2jTrp!uP`uHT2I!;8AC&^AF5i)YbDdmhNMQ?I(G-OI<u@`~)7hxO{4`a(mI&
zS0sXfO!7R??+GI#GwV}EMt?u2kmKDRII7%sDV3awYJ9x*xoUP?OibtjNPB@{$fvBC
z9C%7gP2Jgpk=+#&t5#|?7i2Mc5zV&tt*p#SVGuVmkF$raA^9nNkgMxaIlqqJnr`bu
zU+CU$!XQ(#@uNG%L%1ucRsU%vITtwlLnUp+kRdlaQ6)<zT)DV-GbWCJCVgo!lHZM+
zhmP)dNl!P8a89;1t@2VpEZe@#ff+v2$Gi88mt|u4PKByt@D>*PGi6yF@>{t5Qc_w}
z-VpS}I~BVeCJ=B;iAvJ{G=bi(Ogu7yXvE6|jN-LiQH4xa5$e}WPinfe4Thl?eCg1|
z*0x-ZJ0v#N@7Kqx8)%hQWYA42pc1*idkS?<R)+26K1YrUt-ZaMzDg{eCSKgiIeCKA
zlYgT%YDDqzaoUcLuM!S6GSUiZ!&=L|KID#1P;z_>(Y2EkBn|;)>N@WHraCF&&7X<&
zk54Z03mR_TgJQ;4Vdy5a3;zFcyd6#0IY#RDyAw7B1_B;F-02?(;BhEldYB=>1%eUJ
zjAX=%R7@Z|U+|3W$k?(oG%<ND=<JN@5G}XQ{Y^}3Aa(yG=@jZq?e-s@MKH71+`RDN
zLw-m|V<A2qNR#W`*&|#+H@K>P?!|Ww5^9^|Kcg2=R*?%YV9!WRRee`@m?n<d*m!!!
zi`3)NcPWIB&rxRoCMxlS$;H{gEAP_KjPOXY|ArrOCf_4ydrFm#HfyBf*e4$Zq-kqc
zIHmsR3BM>6&h0qg`;YP1OOr<b<Rz<$=jA0^VZpn9{D0buzqb(sj_T6<l3pa%f_MkQ
zPlZ%RRq1gr*({IaXmQ#QxKW5XtQ|Z*wvERLrqH-Mz=(A|pY!_J;q;xI@clIxbfW4c
z{gt(X8rsFP%^Kn>S6l-8JLdb6XT4tRojuk!%Di0e^8td#j*mB{u3nMXB}*IYDSQ_D
z?1NJ7*ZV|9OWio&R@(bR)isQ_G!suYCt?B@z&FScM2C=c2AxemLKP&EC7W(l<!Jw3
zT=ZCu#$4Iu{x>*1-|w|@u0`@|GVJfTaNtqEg!#XIIuceNS_ubhM~-XwIlqH1{!=~E
z@dRlP{$C~u90wGRN4yn0G)1F)O8JqrGj^EyiUk!W^ErH`D>Za>wYCY`FSBhzc$C$v
zLeEdBpMKh$OzoE=O=(7&>&5Q0FAoy@k#>$$>dXJ?%7+jIXM^n*=d2D4=HtS3E@4GX
z7$Gru=jR+Zu0wTfVL|Wh`(}8aObK{<{;>9M6C{n#m}3+u!hD;_&x9Ug?t;r+pDFUx
zrjd$8SiaNIZ664ekT6=23yRRw_cJrE2oDcO$cu?}Y^;~V4{#|yoyvDY9zM5H)!3wW
z-k-KqiRDB*)R~-Ik(<ZCW>L#UIxCwd0xVB2Um5p)^?w)UYtWVAer`0yn{RS*Y?#>p
zKVd=|#4BK=X~LDM$WnH8ibF$xo)IRG#b;#+fjw9r-uSFqBBl0*Ktw1=d(?k-WrraC
zztKQf!`@V7kyU|a*TF_WNXGgk!DNo|Zc9**>H*78*r3g#U=Z;n1qVl4)3@296y7+K
zi;0Q;3~88-z2SN=s_G<~eMH$qT`(gw<8nyF6+=ka_AA=?Fg+Sqts>ap-{}8o>?;GR
z>eh7;Q3M5~B?Rg24gpDN2|-%vM!H!lk`j`O?vNHK$whaAAl=>FtUH$9clN&f?0xRJ
z{9vsaOy-<pj5nU~j^|krPY_Ag)iv)YO<7b?F)AzyHR~6isK4q55lmJN!Mjk<1=fv0
z@v7FhaB-Jp{2=t093dJ8S^&}f>isL54G-(rkfT%Wx3KPAT`ObmSzXC<S7&A1EKQx;
z+t`poei8WU&EnP-0^abm)P(7-)m&ez#+n=*%Hfl%cDHBa{cWP2m^x>pwhey0*<|9S
z3t*vt3xlWZOvjsOoo;KLwqQ+^bI%mGAJg}YpSgP9@VqevousC5jem<PPhI0p98-)O
zV9as&G14O^wK!~8ArKWQ5*Fyj?S9C?#pSD#sT+W0XJcffnyh&CKO~My@d?C_BT~p(
zp1QxT_ekCoC#R#+{_!~^Brx!k@5UE4^Er{=@Z6_DpwFOVWV;`&U+wY45TMIrl7)bY
zNdPcmJ#q>SA@%103?XR`lU|n${3D2wjAl~k8Nde?q5xDOniG(2dv6T4g3Lucy>m`<
z(cMi3i>xksWFWk`C9g0<T;hVa?>{7P2HDI1cLJyS{$l<G0ffAIWH%V%RSno$_h7ko
z!XTo=AnBY*mJZ9xmY02&Mh?k>!E;!2+~ltQ-GC8;Ap@4GRQ3#X_4X_D_rI+6xh)}G
zD7#JyS#0*Yr34uOi_WU&TEFqG?W@P`o-n7C!C01ZD?%-8)rd+g4yBaorp}jkE&d|y
z?Uzxs&TeEu`PJ0sJp^#QS~6{ITcBT~R!BkL*)w$ow>%W$hg}Az2veAu{U}YxrCSdd
zeYx*NIL}IPxs01K?7FL<U)ft_BG1Ik0>+EnJ82(wzC(x(`{NVkP%m;s39;D8f@9K&
z200s{WGvWxML1vhXX5@JF%Q`JC$m6&03HSK0}j1vJ<8rie0Ml^$?_<WZU~Qh7kI@X
zi`XTZl(ucEySZrp{PoEAwn)I<2Jp-uG||j{ZE?jbTpp%MGP5qQMaIQdS^hEs@yjDM
z`|HzE|Np857}oXi#U+tXNijAQmxngIJl_9ZYWZuWwN*Go09^gqqE{^#^h>vLa_C2n
zpDi78atLgu-7<kAH|-O&u(2V<=kYgT>dyrhX|*?fD3gH?s-U1xEiDyVTFILtaAbV8
zE5v787m=X-xd{>P3n04H8x2Ei>*Bb`NW4s5v!m4oy&8V&Ig!C%71)quLbl=C8)IWh
z0BH~!s8dh=j_Xa@;Bd^5adJA1X6PVfxy-He!fq!A*8-s)L^ZGQJG&-h%y-UMFg27-
zX>U<FnItPqE+Z+NAbbH<-!*+xDQ*d<sx0Lg@|O5_2hDyCF7zSrsOXjYS2Yc7#)#-r
zW_b{AME7K^SUL&{Cfq~yL`C(aqh8*bI6`#w{^Tx6V@o+KvRAb(Yz~(fDsKGuP}R)W
zzeobt-Pp*xuyCVYS>F8pj|=Fpdg-HN3J{U3x}-oF%2<<o$j>Poe4&5!%3AAqHdj9S
zz`A-_YL5ezeij|`?KhX3Oz7wyD!~txf#E_LHT;i#=!seVuRmES;*@*mF4W-jfAVO5
zAA0pv6t1wk+R9$?R>HNnkB?p9r5-tUkx>&nKI9~PQmOt24T!!=2pXQEl^5idoL+A-
z2|`kYh4Y`JcEbGh12FW2AwHi10&G|U6evKfd1i-SFgQuWDp4QOIC`pQ5;WTQ3_LTX
zk{i`@h`gR~ajxZM!ps^YzI{%vCdYJPArMiY#N<nlHQmK$0U3f@#e0pXPLxry2Dks+
z!=<E>nodBKusd46LJ5h8BcX(Ct?JaAkNo^2D|mQp6|k|X*$UizQCmK<D?I4Yn0}?e
z&HiR<p>%09j}AnH4Q_|aMxn$us=e{C8UQ|hXIx(Fyp;;S;j12QY2_Je{2(m6KAHy|
zhB8xdaUC9eG32U3xbeq_Zf`~_S%{w0sh3R(zLwEav!CIMj+MT12lU1woe>yZqjXPB
zUh*jzL<JgpN(#L#`T!!h-^8IbC<_=pvF%|4N|Gxp*))Fl{*05I8IpQCgYvw*rp}0B
z;F?}8EPr3j8~u|t*SJND6{b&=fm-z7<7r)w%jht&?6xl!y-!X|cozEW8IwP+*ve;b
zgFp}YE?hw|SwgbR3~pnbMTpBa)4tk<)dz_F=YYN9cQ6K5RL)ftA|nx!IB7{cOsFPd
zOWs*YppO}jVZwXzGLH>{N5K+Gz~%#<)4a_<w}3%yeO=}7fWq5*$C)woN6uo6-TG3v
z#W1y?!{`2#Y>N>9CNO$y#+&|n5f)7BI4`!{8&4-i*#$;Yrc9MRK&i;kkzREyxc+)k
z3qXoKlG%yhzjZk{Xc$b_+PHs}?cBd_RqJH`yTLoHz=~?+*Qo>lJ45k}4dZt&E-sem
z#b{{o-GhTWN29Y;IoY4ZD|l<M^jxk49{n!;?Dr5J6xtUdr{zx(zs*I-$v5{kNqDNe
z`(<fb-Ub#8&B`H*g8<XQZw~FBUk#098CSYt=H4&v#3$my1NZU9vyZ$lUuh7WY)I(U
z41T3BHq0nhs>m#?{Ap-TZ|6Iil#_$6cJ8&!IDXcx?D&^KDKSuHVl()hj0e)u7SNPI
z|G!xSJ{Be8FrIc?0uJ`Z#%s#fWj>ScgZ=#vFMPFw+f*NZ!4MWWIerph4wh401~(PR
zm5DA*;P?fMq`jafi1KTb$IA29jbZFhwq12yL`fEus<ItY7F98)*ZbM1nY&3}z*6jz
zGP*iCLKP86%>8PFUGK7(>)#QF-_P#=J1G8K<^SJ&Svp>xzk2foVYTQl&dn`8An*q<
zKsA?rke!V~LV{cR29UZ!mxDm&q5XYc;bo_1C<#dq?$xQYe(B;h)WC&{i))n3Ab$#H
z;`i4#3UfX>75@2byX!ACi@%nXIQV2!eY$_2`z;b+_&VF=wtkbHm{s|6SK#V28=na<
zhISRY4A=u#j&i>e4Qp@wM>f8T0@4S*d&Q*4$sh|cHKR2%%UU+hS$>eZvSy4&xOzlr
zF)?g8!D*nlK#T3?H#A#sY$|<qHWDssX(5Ji>Y4{upT<h|o;$7If|;1)q+a|=vS%^f
zy1G89bq?lRV}L55y0TIS50|NehFCB<fe&!Kun7Fp$I8w-cb!ua`{1y~rk@4Yg^yYo
zOO}}qo$kE0j#RauakQP0v$r?4D!6IRl+$~dgE0Oe7owa-L2@etDQ23UqBl9+Xx41K
zMH~9&uZL8?WIL1h=li+<VDYimn1EUTTfqA?ulTX-kHF5W+V>&2o1paja-<<Dw`7`y
z3Hp{B*ylefiovPOQ&GsnC9wT!0E7jj6e`f&w+L_mahNhm0k}_)wCvMfFj>2wFm24$
zt1bJtz5bRjT^Y9VO}QnI<_-dq6ste*^<dVAL-q!4m7s6Mp<l&qQiy}3ChJC=W-_{J
zsFC6H8zPC=TZHk64d;D~YD@QP$63EwKM@>;EskeNUtV?d-Dv(k^cH2-FPqP_O|i+Z
z2Js-`9-J{)dU4b^X9IIE_eM?zH<)Uu#T7$UrCm}gVt|sz!AC`T=3WJ2?YJB?M<yLv
z^3*U;N0TBp*wLYva(=tLCvxqz)DOE&=0C4@6{;%Q2?NfHSGNGu4(X~L!_%$8yGm_J
zS>vYpDN1Cwv9g*UjEvuz?vg{f>CDqZ*$sw@?YAS~bGs_RdQkFxB?4mL0e`+X+MDEF
z;6sri9c$V|j=3>EEX+j*IU<M5ghS-e1AhJ_(Z+JLoT-^a=doD<e|4p#PdZs^<=~;w
zS%Rv{<+%Lo7Qa%rg2QD@gC%KOqLTboqWL>hoZGfShMuaxp6=>^kVhDle9W|l()R+)
z6|YaGs>@(Ed1>DyOC2hl9RyBJQ1X{#CCM>%o|n5q&4?xns&*i;@hvAbtPCuQaU+ux
z2R_pyp)hia4BE<2kwo4fdC4cKX2l2L-}6d`j#ci6>=v8SVk~ElkM?^kEWhhl+)feh
zc~iCYN=~{m2jk#TawJQtvM6M00mes`W46L5mBfp|q9Y*iGW&<CP=%sg-?FLPPOz#F
zvZT|Nis6}(Z7X{fSv#MjIdh4wy7s9-?-GkpxDq-3GlP0dSM-XfDz&-hn-Bi@X-}RX
z+>k2Asomshxi%B%t|Z%Bof_He2Pg_8YdP{c;n(jsq*I7NgMz4bqSm)(dBHnczfAPa
zk9XbsBGWq)MyN?O4rht`ftW?fYl>cwcFlLe*y)CN-$>O&Ac3{%cn=tD^eV(FblkDu
z2llixzY(e{87Snqf&KpR1ybmLd?@0v>#xwBugFZ#SWR-2<w;jHqPgpURX>%tbD<Od
zgi;S@ubq~`FHth9Qi*l#F#aRrRa2I}na#u>@WiWJJNF;R=`|?3NvvMlIHQt0KKOlS
z%}zwv>9YGJMD$jFfJ`V&mooX{^_vFqBM~y8`QhYssVyIpjlDB)W)JU8x`jl7zmhXA
z)NXcTZNhZ-{G<Q<OF}!DRM-VO9qj8DeiFX&eaktAfrF#x^;1u8bNOG#Wm-sC5PuXM
z(7m$Wd_bvPZv+1>A|mG<0!bK~9`c9ezM8_W@C?oYUHjSM=c@6rnABF03;5}-2y4FO
z9QnDcn`%lSwX$sT)`f}q<ZF)mL{IKW$*Mh6VMyh-aL4_eXxP!p?~;}nmy_`^jnboA
z_<{IXp1N3FuGq^JB+I}|sgL6UDDI?B-hQ~bG8*CUEu1Jrl~fVk8?STgV;-BeD%#Wc
zJTe)W<r=TtgC~e8wnd4+w-X(6gilaWxwz0eh_qUu0;1*RdCCOP`m1*9ox@Qh_x#w(
zrwlR|&?RXRDv~V)c~k6&MnT2reHP!GBN{F=(`)LN5+V2vyf&ZS^I6}={0d{|T=q>y
z_KT<cLfC(NapsqLX7{H0z2bH`+L8!-RfrPD;M>5UAVyo1Kt?KK3GUEJ@K)v6-HS+-
z7Wk7gA-$Q|gA<q%o5zIAGs#_}Bz?$U6z%ITE(@JzDf+Oj$FTE)k!55IBJ|`250j&z
z>5R;3DP_@x@0{x^e5T5$SYIeI600`h=UKiZ>>U`gM7IsQN)V6y>uwA+s#mYo;5yDv
zu{HIk2FsE^2|Qj+F*JXMgw)BuuuwtG*+1nx+MNb{U1US|g^eIuw@{O}<dKJaM-!Eq
z&vn;u^Y!4EtKazGa1L!ozmgK=hDNca#R#b2?dsD%h!`3$#o_O;P~FOje4=wJb@8(<
zgERTX242eR>%+N+X6MSpY!NT7y*oq&J5LK23IBczQteS4_N4GKsLH06MDO5+XKR0Z
z%Zk9o&$auQ-@BO@zsPpa_t>eA_oePS@2W}x$=i1u%4#dDdmY(v#CIq!-R9m&z&uXF
zB}%5h5zI2g-F>&q;C~;is~La4_W0O<=Ool1h5rQF2T|sCJH{4D#G2E8DH$6}a1z=V
zOR##Re^{f25u<x;GSq{MiuoO<nAh<lKro!F{MT^KZFsswD_;M;eZ_b7)P*IFg*7JR
zppy7_WB;lx(OQGUZw8(A(at}ND^ojM&3uxsGR!o5&^qE@bz0B7Qj|Ct4f$4n73dUw
z<7HA)Rp94$p+L^JkOd#DmX}{sATl<~)cbb6viXQyQ~u#myofC!<U%yPg`CMrfxMx9
zAKd9@Ul~n1pc4W3RAz%3+za|=$7<G_2fWOzi%<{u^3OPt>ptEsZ0EmWuwFQ33xhwa
zQ)5SMo>qH=fr6A)vOr2VjasbvD~GH{{&}p7WEauI)+d9Qm(-|_>V8e#6;ciR%o2AS
zu2@jW%<@f!4=w2ISCHBi<!t$n@<CfaEps!Y<Vq7B2sEe>ePv1d)@!A`FTo?zZ{nWW
z-Q2FI;j)f!7eW2^{iEWvrr=re1TvB>;;m+r>ficT4G9I{V3K(=y|+|7cDN07Xh;W-
z0{>w=mI0-^^*C8w^)^T#7NqT)5fHSU&Zo+NKyMW!+e}t|Ib(z>=FiG}0NhO4Q$E!m
zTq`e&Z*3$5h3+{%>wVrfD29Y|nP9B+>dmE@{R5O^RPTh%8H2bp91a`(dShjeu{MKZ
zVipvR^cKX4$o=-n({s0n2O<%E`)Sf>i6lsuqPiVuH-XtSxA!(bjnsq0HUu|K`8<p~
z6bu0*VYl~kXqORR$8)sa#F?!@1L1Foz;3a2w71<(<?LAQHrkz5BaZ>*`2iJe_BA3~
zM>0-A@^IOs_4M4_vdiO;q?|ZM>d>H^=mfkf7;}U?Al@OOn<jKXXS0*LP=ZIOxlLr9
zNJ^>LvJ&0sJKrUGZ70)YZ*@uO$CI)wORX24!oup|<m&Z7{HE^xM=$T7K!EW~P8K(Z
zJOZ;?g+<OC1+ayywAJKLfP6eIUiw_SJqInR7n+lSd8u`~MrBjAmaMg7IM}<NH+Psx
zVf~)?Y1fb*&jlbS|3~Q^WLt6=EzRZwWQQj-R#xeM-C!ElJm@Jl$;nTzB`6~RGozDB
zWmP5bLpD`?eL`Hd`NbdMKB9dc_A^TA40zNrkumtP*;%ig@a3pMeFY}w@%d78f?R@0
zMe2jj&Q~GC6&=HeMmtq;@q7GM=roa`tE-9U&0FI!Yz=XOZyQbNa%biskZe&&PzmYW
z6-nibYBx^yNnC|0#>dj+q*VPf4}AarGd%&Pt35DqZ+p8kjKpkl?JK$POB}Q!Es4E4
ziA-q~TibDsiJuXROBU@Rws{neEoV}S1cXp;4vu>(E6%?mcaXl?H{ZON$qo%1-K<<)
zZNE%lH0`q*kDjW;?_NnoMKu%Te`96_la?eMcg|5h`WzU`mOD|FR~m7BV{vgfW%aEn
zyIW4~Wg3rF=im?jx`XzincCbu^*wV~Pp?-;JX>M0kLUS5DD+HeCG`Jn%%U$`lTIPb
z+HbcpoGBQTr!?7JQr59&@<rF{;*dG$GhQ%!n5^w^MXStAh=;fM@Ql3F%RP2~5t*mN
z04u61h0`YPXjMf9_4x^2-Y+~n$@q9Zot10%8V5grOwdw?Cp(wqpQ;ZoGiaGoCMHS*
z_}{SU#0tD?{%A9WUZSUFD03f)y)P+^9dq7x;yfIZryeR8wGU^+BY_{V-E<QWBnm%2
zGpKWuhigjC=xggZo}7?&*gHEXRh;Yy%F%S#>v{^&MLf|Rdu%Wl&0!ST(gIldSh;yB
zob7I?lDmtFcJ%dA1N==bz#L2EWA9c`>d0^p9OjJTfc<%?va%^{ov92pbDR3=h6cZ-
zpEWfER$G0@MtI*Q1_viw&{Wi072})jN%;&>#s>#qo13&rN!j!8=t|tJdUtni)AIec
zw~E>;Ev<@x4xfi#6n{hSASry>;$N`c0wr$Ng*)r!kYiA2Uu9cf@l9JNA&!TMQCwB6
zCze^j(9BH0wb)VY!^l?(-up_2rBgpDXh2T9*Y#YwBdp%z`n*I+3I-B49CCwbXw|E<
zLmj)PB;4kWr#ougG^?vPF`LOB@7nDdUb<#H5n!4qBgs?{5oJ7P+NnyoXE`~gb7CBq
zP;sPuA0s0gFM%)o_FN`7FzwFWz`*8~jhizJ-f2P}t^E81Eb@lvlwxyBRn+@}xw!+)
zVULjB;?xyjx+eZe&#By$?(4IXRM&B?Z>Y@I(i$iP_%I>aZfq)yp2ObqbxijF+pj@E
z6Hd_G#x#P%>Ee4|qFUSJm5;iK0`g*gkB#`i2U_Z*pNEGTKPHgy;w1@bI`7Xjt*%z#
zmNTf`mfPwTt4`!3yKM;XIxHKwuS~ek3aIE6ZITIgeEz&RHWqe%P8r{;B6Kc#{r<Ir
z;?1?{HoHI=GBQB7H%62<*dP>PneIn{fyl__E%);aGTWttL$gbbFpAUJEPtF|WNW<6
z!@<o>5w4~ep6(kO!W7~}`XEc~_^EkV^ozu^`Q6{qhcks7V}0G1E)@xZRFq*$y}iAT
zj5cv+-${~wM%w01cXxwX2mqEzhv(cyHD7)?BRDw#zto(piBV^yT6=uq=)AfbiA{CC
zqty)+g+F2zO;{wElWXc*J!8ZEYTs(rtYB=+$rrg_O0JuCW1MzvIS#1Jy}h>p(<kh5
z(?L(KAV=QBq>6y``ttfBe<(A6lz;!QGXm3V#rvus@HW|J|4<xpl;8o@=sBsL4vBye
z;H4S5zQDr~kE&t)n4Lol(0(>pRxCkem-#rWgNv!%dBZm_7Y2eNAE=t!1+cLj(Au!~
z+#4eiz~;}3Ai8I90`t?4y<jqR{+;g@KLU#crXKP8*<Z}Wz-=J5I{)iX<MZ63h-k0}
z_?-D!Ci<V73#H`tU>lq`s99<Iyv#C&Wbv=#jcJF6HF0ssz@%OGR9JPj`|Iigj>9Qu
zYMkJ}XF@{6e8p2gV7J-^y4O?N6M6;~>x)0wXmir)>M&KnOcc1H*myBRl?_vLk0=&q
zztj8g{rx_nY*kgcliD69uhAcl=q+D5M|Rc3SkLJ{A-?=n?LNMe`X4L{Wk0wfZ;ZJD
z%o}L-O2<+M)=aHyP1(`>l$wV4$qBdlV;=6o!SvBlr`o#okJ~%7ML^Z_<YFy~P4|tZ
zon3a>;TJ-?dSE`e%!&(s6<}L}#~1P2dmV6FkU>e!Q(+BUfQw74<C-97cnl&#s#gYd
zGt?Q+J^Sz77s*Yw)p*TAF5<=TTyJPJ3-ghal5(maKLAmWdBEtpfo5jvx&74fxy@*8
z&3pZtQ52@$AWZRba+03nBdx+??Yc}~`Iq$CrQ-Q|>uXT$*V%FkNgjvG8T;?~WMqU|
zfp7miO7h2oaB~h*aSr=Jv~=Xdg~j{(qho2y)usIx{RmS?d9A;?yp!$1X+@Mz5Yc^j
z97aVs`7=VVn1%OL!Ft<R5<nPWznB37{R43X9&vdWgc;!TpN)^$K;l$F{QhUZxD1-9
z?V3%Yj`SR<k^FIx9*S6t)%T;W!FIiAzm3qq&C`<#^UP)V^exSA<D2(XhM93r=R28$
z6kvh4syg1vDyN9Anr_4IsWAatMT4yG?YLS&SzAM8ZJ`%OKvoN=@tn<QFfk*3TIcQ#
z!8LYf%gn;l1Do_{eI(vSQJn2;6_%n_XErXJ6*sNfH+ygV6*~C$<^J||w5-a>Sp`x8
z*HM7S%P4Hd5%2<n<qG1YNMcv1M0x5SskfTGj!Eyy>DfxY1Tnm~pOAs2t}gwwrMtpI
zkCf}`4$e|lrLRG!!2vc671iU~UQUsw@5iJvim@@hbm*j!#YL&)4;9DDuaII#UKKU^
z2y1S>xmH4bSh3eZ+vuIUyg~vyx3^bJt^$UvtXGDH=-oYAt!$A{C|y+7^vsNMiZFvN
z7qF1y<F1FxU7$Y63y|$Q1G>n702#nBtfCTe3R~LU-91_x5Ov;+-iPdf|4oeZMiyv?
zK%wJ6X$=L48ny;CHj25ry6#jmGBN@Yabr#`z*;FK<#%iGD0xi~gCVASeSIA;W?7V|
zd2JzcEb#Z6i^c<jfYXt|-zQW9+rnIOs(IY>^y~F!vqxuV;bCC}4<3{qEoK_iI|;dL
zf9+WT{8@lajGG&3X=UZ@?G5D4v6BIDz)*%%bIa)7Bd(O*gx;qwyuK<D*WL=Y4H%nl
z)hBf!kLW0ffu&1?%x|7SdjNBw-+0$Houj>}7)oW1izvp^!q&aS`bWv<JN={)>1`*e
zToU6~EyR6j=u1eK*#(*;Mn(f}9=m5pLO&hN4*C)m6AZAO3pTcTiVOxS%HNZFHC|lK
z79TaCCaHF1=Qy<Z+xy4z7!RiXo}Om1bPl*J2^1LyVU8`Ch}YU!QSJrw_Lc}0GrF?8
zoLopl#ITjshap)8Yl+ej36Rw9Olt|7bib&+KKlKeOS6QRSkL%m^Vgiu)ehgBSDD9&
z<wr|EF$MTq0KpPyC_t0A{17l?Is=I>z#av}J^`1h#rWR(Iz0mef^qqjIIaMH<f~YT
z2#})6$;pX}d-9Rh-P4nhkg%$%%8HGIloSvaPft!J07*<z(!x1%L{mvgOG`^h{TdxL
zM>JKlFXyDPDKswbhMU{vrjtu(ASAM&z{<iRub^Nn91uwZb&Un0j5&9;%q((lA4x!T
z25c8H6*?9c9BBLThv@*S6R^l|Xv~=xl`93AOV>>1F}ZP6dofJufGd9D@@agRf*~>v
z4$cqPJ$)Y^&P>K><<-UB4-c#rIb_%mu2!F-B`cxErE_spu109U1-P8*lOBQB(8-?;
z|MC6z@<ug$u}w!hiq*=hxFKbUzqmN8tg=B*CU2QP%*t+!8YhZN>A%HeXl6<O8jP89
z$_Hi1bkV4cw;YG4Wfm4rwiv<u+GrOnEh{`g-v)~9;0KV0su%o1mI1-I>8)6AYQPe|
z5-YU!R!J=m3gvU%)dghNJK|BCN;m!}9^n0D={0YwzJI?Dh};mQ(DCtks(I()%F4>w
zs$KTFMn;*rxw%8I5kU+f{Kdn~y?!j?wY3N@X20s%#=$L1^=~gGcQ;+bFiCp)^y#(o
z0#WmJU7pG?7O^O0wkiX(=bkiT*a6P&@v$+b1a8?8)Q^9?<9{0muEw-mkI5hJTrecr
zb6$1nCJ{Ye9m9j5;KJ-qIi38$4G$6}q*8JznXY>X5CLXm`=@lr2Ke|Cd3h%Cv81JC
zWl2Kg0?!&PIv<mK+#)%dn8d-3Pdr;D%QWiHf9`WVo{=%<vcnh@&eq+dL;vccs45JD
znEUP3DMJE|qhpZVNq^%<f=RoEEoaZrg92X9to;0o{RR1rjo=(w=eKE&9+%7umw>rE
zG&B?lTMgBb0=oC~ZqTJAjZID4Z)PRHIz+$>59;7|ECHz~&`wEafA>kd==2l-UYtsl
zi<au(mH+<jlt|Lc3P@Fw1)b&6UjbQ`tehMmOmUPvWRNIHxv{;?&Be7e=Z0?{=v^JT
zKGuf&;8PrHe7O>$TiemgfkI3}Q($xILn70MHo`l5dJrOuU_DDJw?$+Qmg{%tOrcMm
z&k`je$%|&3WhV`BU5j@Oo9}({r;9t+(=itHEVDapL*e3a5)3dnxU+F%+~}M%2oVsR
z^b4q6Sz`l!6L?Rs9)U+)jW+W$ilt5Vxm#pcNa5I6tekvkm92%v>9G8Oq-1AC25oe#
z2HNQK_w@Pc>U>{R0}eh|J&AJsVXpI*LrgdC&b8{=kZbzUk$h3UDh;jDb9V<7x1Wib
znaS^O0R29Y=sDu!V%4c+W@8)3S_Nhv7Z*1%F|p#|)Z`?Y=Qc-$c5-xdmY#@l=T~6Y
zx5zob%FVsEC~<l}zJCX%_6s1}uC$KNsRmlOfVuTsQ4v`3W&_NO*8;Axa70wp$nY?r
zucMs)T2<w0ZEdZmr#GNY7(StV5O;$IQv%a-GC@R-4=^?i-qX$xrb}2>1eQ;IM?%6B
zmz8{>ehKyfVQ_p~b09_p*y>LZ;-J1Zts!(#=}*(y+KZL1c?~OVHx&p*ln?F@0pTDI
zj~eYFC3*R+CKMyq{znfSt%x^sZ=3R=S1N3LrLsqA@co`g4}fJ5Ekt<r#{GqGX=#t>
zXl4#|3W|@z!=3$MoSfzzm{e9e)sfTFA#(z)b7<Ue8#|hs;yurl##u*w#3dsl<_HPl
zrgr27dOyLUs2->ld_REyMkv%n>9|OlhDM%}>NyB;)R%*g4iA?ds#m9#0X3fNjW#fF
zpd=tQHTAS@X<=ajQ1>Np7_|eTk32mAK*r9@tgyB(2-aYyrKJhE^_sW@L0F@@3=Iuu
zA#m>oc1lW0KqJ!A!vth(`x1&FcL8TPSOUuq)C)X4Js14JaiHOF7a6&n1ZS?st=A`^
z{HKp-TS?S}`+<t#eMv6~NheVFv+zhY6Rg1lt7gHh86~Co&m09>&&!S63e6ttWOvvD
zgnRUN`P}fNR^(tRaH2+*OUqca4He;F0K4$f!2!o5481J1w=Mq8J6lDGn^Dm>Ck~=G
z%JU-f99Kd||9HQ$t0;ci5Q2atFj;e11f)*0<)Oqksgl>I9ZpV8g@Etze18F<664F^
za*TT^0FWJ|{SS{*?d7tEf$9(~HT9F{7pO?CD7&P{002U1AbOBC*X;ZkDPPDRQogVD
z$j8ZS4d=~MQ?;T7p|gn%958|sHsTshD``9I>?_-roms`xQ|d)rqdOpA6cz&%&-H*8
z6ZIs6>6VIp?;_PqcjW#J1w$QO5h0%L$^P#WJsf<g>gwm$_wO&~{mDn1x5ik&>|E;|
zq|4^9A~TI5hY{9ldtY7M$4>wlwLdsNXS>R;bmhGk92^WJDB~3f0IRl&O1ho^D{DV6
z3{9WzCE)DLS*D=ey9Xi`&)eHuV41uY0LLkyKm~-MZLp>ys{eOb6$r*9knI54MGOlc
zakq?oPDl*FN~Sm8-+t6(ohvm8IQ<=eXUI+}U+Tyi+AF@w-qsOC^f*sFU$nQoTh2!k
z&}=qOiRY<H%g87vaCfbjzvFG)t~;-81p~x~7`+on)hs()H2XaShVveQSy&(ujp^D+
zCiQJg_B~|au_g3|CwBd{_f}9wRQ?O3QdT?MDmiIugY~8C^um^5+8>u{ygNRLzffh+
z(uz=*L%(NX+1?T5d-wS*e|FBSe(Z5WYEzpDSRVVr-Tf)%#{Tz2T)g%>$40<IPqO*?
z`bI@X>FDSH{!2iHUQytE*hQzPs0g?+FCSW_)YjAhRSO`Ed3(MT4!rB~L%>G`#0h}@
zlIL^_P{{%+3?Sr-=Q2C=1EWgN<JjEF$_mI{0!@nYEmtQeP>QIKEck2i2?jnVi6J@~
z+K{BI=eEB_|F)Q(UNQ^@2QoY;=<zMwJSl>A-eCf%Nn_Ae1gQ*z0e@U@FwnB)5VVQI
z;1FW-VI`W}y*wHj)Lp!a2XwW>#Ejqh9VL6|IS>I)di*nCfeMo0y}Cw!+%l@^2sD=y
zOC#%TBISV_A0JfLUFHV&GTEbo7^ZN$84vAT5tS-vL&NmtiLU&^GY}h=?Cl7GP>~@<
z*OHOXI<Cn?AgtEC%wny7kwFa;AZZ=#Db*eo*fP#;{0RHSyaedY)fr<*J9v0`fpm+d
zrj?~7&XG)MW~MK^D^+X-?gpfU>;Rz};9+Hq$wz$-q&-&QpDf2fSfB^VW$5kcNfdG|
z1{U8XU6v@<M?zr|ObfJ6jEz+~Z44s_%zu6rP&ha+zW-T3VKp!gNRJQ`6Qdw!#Pjz0
z$0ZYrLgVhd>sOX=4BE>DTJk+B$X~c*sC{a=GDojM)L=)wtU+_w9s*5n|HQ;{>lzwX
zHga9Lxy#YblE%h#y`RJP2+VA}Yr{aaLpna&FH0>>lna<!{|2b|^IKZbQjxIEi2C<W
zu7TpjBpVHmteIIZcqX~*O&5lscp|u3S^m$TKY^ST10JHuMn^}NmyOjtO>At6%9Q@+
zs9=l;D%q-tdWjKaM*mD<YrLeYq5`3^@t(pl4xpX%&+3VCWJjc76u?sr(3pzF9gU2N
zY%#NnaMIeG2~{X9?4oTys3G(@vHf)8b5H!GrR78<jm-8o^IE?%=7aadoUQ(Wr`rOA
zPtKF->tjkWnpms;AZC0H;VQ3RA1$;%hez-g6*n8j){c+0_(Q;xj)TO|6nIcTmI(3}
zz~x8+B-F+F=u+G$>F9E^vuS8(q}-AT2nc|#DA0-nZOg>Q#^+R@Ap5_1Z#oIVWNbSi
ztPZZNTJQ_#cRhd530PWL4H`&saa+vPk^Oq#p|`39q<>f^`OBO3gEp_PH!?EX+T8Tu
zwE;~|E@&#I&}{b;2(N?!vggsK9}(6fUIj;V>(M<)M3W;1NKv9-Q=CxYzas{eo9pv9
zIKdi)><zv>#?fEJii!ZP%^4rjU0JEFJz3N`z*dnj7Zb;SvfL%LuwWY&=E5Ixms9D(
zl^9@V20RBqctA<1A8`7kSo-_<*#KoMpo$|6<BNyE#b~1dQE(m5`c60mx(n#=UxeRX
zb|!%h6crWSNhE1*X#rUf4^SfjLRzY|4liH6eDfx*pS%aC)uG<Icf!2leKz9?+#=8e
z*Vfkle7CvI2%-0o{%W7xmr=KqG$-9{lMC;$H{kLTaoLUw4-X$QW~HReC@bT7_Drl+
zelZsVWTI<0T+e6$vCaO$K^KBMJCNO3=B;$lgu7{U>wuudouNJ9GI{^XS|5;iz7_ln
z49UzgM#DBSrq_2{=z7<3pGc4KU&4_X&0=c4DqB~~m}uspkmDyJ*D9p-3lwzmxOlO*
zTBOK#(%AU<-8P1^&#u-&<+D}2rfGQz1WQ_4j359oiSNZO!<^p*K_=4u9iVUh0?F+e
zJ{}6i10+{mFt6g)xPU?jPRIIR@RMD?Klq8H*AyneRRHYvI6ZAXAE3xPk>BXSDy$T5
zD8->VI0RgI&0hdpqDxmO$@1c-#=ri6UC-TJPeI}1mmU$tt`gW72%~sCE4kpRloUaq
z`HQ#El+&N(MHd#nWTf%?#Lcs`)YbHn43ea(X2(`=^LgzbpxXhW^rRgPtMK2a7;-Lt
zo$mOhr?bT+x|#%%j-Qy;tfFDF7$}vQfkOX_tC7Q%Dovma9TNlk-ZkG#E+|dL_VzeP
zYN0GWN$d-al{S6=%PW-vcf2d$RU$b3E;g1Nh;2CS&2IGf`&RUj5NoGGM1Z@@K0Vc#
z$w|x7={7f#;Ijo()#?Vy5&^1?j+#%iBLSqG>C4QLR~A)YSotrgT&W7;+AM#r{hxCu
zptiB5`*z+@a3e=Vf+(wq1v{d;I>IyLKVaVk015rMhJc4n-9p8lfh!}Wg*zoLc`X&;
z2O?JWAsxSKmz5ONW0XSBmLScCwbcKq{Q<BjVjILzh>s_R%Oepf&K@ZcPN1%Y8qh=~
zBf{s{F3*(%qi!P9Mih@WKJDV)I1>sUwRCnqDT<6N>$(!@J6|l&3$oYH(0G6-fJJ&j
zbE@YoFW-|ck?OUPotMXORo&lTe<kwknsMcA!d|q+Kq&E+kcl>66Un*O%3XtBoK%(c
zCJ1ecc2w8Rlby~n0SCqn>+1p-@XcC?*KC`_ZkfBhwTuiZFNh<N8YPnLErIk|iJQ>&
zR!R6ikU~041q9O*HSS9nWqvv$vvFU4ZlxMP3i9ea&r)FZ<nEQ{$3aUnJKNioaN;)T
z+G@%*waB|0xMZ@ZPMi<@%5ZjSrA>H8JL2xe+9y`3*OHhRnJ(5K@a?1Zb!S#qBaqPm
zH42ia6}x*7xPqGFRV7O?TS!@XIVoUzoeXq6Qd&J)K?|zowLO2|8#e?jv$B9-dm;SU
z(igT;4#aiIUOvCpckT<7fU?1tMP?P%c13*Z7Ki2fddyNTAPH;X!kj=*_Fw?Q%yNSD
zfSPjG^(X!xJd<e6#)OdV*5Wt2eGr3C`1ba)68!5~CzbE_KlnG98!fD0c4!EjF!lVf
zhwB^)gsD6)ozhpUatD&^U5MW~(ZDaRzby&SJ@!W2hjCI>MO792bgIuNe&QiPgOcPI
z1FXd!Lb&_TTn2n?d%|&NX9vW`S308sUkKq7>ysLV52p5qI_Ag(@>TQH8DIM}Vad;-
zH~o2AZqJAyn#u6T=XdVj-%mw?nGb!6=dy($Ul(eXa2KdA?9~AU(2Cp?hev3)C7M2>
zHw=v<60whAH&N%G11q-p!+?wF9p)o6GD6&&@Ylclup8*O*L3xW9)~oob*}G>`0`H>
x7qRtspZynn_icXhz|t~5W+~!?4-Nb9JHf4=PwqE<XGH9fl~R%{7JvKc{{Yo)ka7S3

diff --git a/docs/topics/_images/firebug3.png b/docs/topics/_images/firebug3.png
deleted file mode 100644
index affbe14bc26c847c82c08b2243c3da31712d8240..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 89644
zcmaI81ymeOw>C<QU<nWi5HvW!-7S#d7Tn$4-Q5Wg+(MAS-JM{;-QC^YZ;|)?&N=_R
z=l;DGi=LkDnW|me>e<hpASnsKH*lD6P*6~BgoVCHLqWahfP#8{2=g3VX+t_%1pl7d
zNDJ~o6%Al-gBP%tLP|DJQ1B>Ef6t&2lQF=>*S5mqUtdolAitq_kC34A2MX#Vl<=3&
zvW_!*DNe2^UvFDTFNcZDoQWD$$o=>zq|8yjm?KeGtK8wk1e1}yT7LiR>sl;7wT0B@
zXP<Kdd#D0>D99@Otlx`>CgN_#WE&+hZazFHE)WydCy6J<@}(UHAg0YvPbX|TZ6qWZ
zK+3Tuwt^Tg_oqU^SpFVI*v#?&Imaz?tfSZ^h+lxeb=0a}?TWCNttvEXU#PR&4MHLP
zdHwf3Psf|cFhLbe{qpO+1O`HZP{YI>flv&qn=_-3j_0wjpKkiO-eT79%Ou}SgOZy2
zzPzYlh#<C%gNGRBG5gmolo5)d-Lu`1Gvo%7tjwS{G_h=P)M~*YA@FbCc1n(pj9?eT
z`Tds5m&f-lQmenXJ#5aGCFf3CtTLP3O!HV+T*SlIrNy{7n0MKqVZx1=gh-~P^9Q`U
zeD~FCs>DkO0TwnWEUbTE;9#zn!ItXpm_MePlO2W+hPa9yb#ybQ58oLV?*GnBEEE+@
zBuTx1%}puF5_^lwj!!Gq>L#Rexig-R!*2Ug0!L?Y6e6jhps?Bnkx1E@t+sG=<xJ{-
zSdHLmy57LzRDC$^!n<~Tc#Y@ki6kN^$>Z_h8iC7&9byePgv{67-K|ih>g9;}Gz8CB
ze!_#x8q2(m{_)7=!k4CRMR9-QFiJeEZ!@=#->1bv+VwBj48(+*+!=a3NfUSXd)4L6
z&j)vQ(<)sPdsHpDM(fL{53Yn5gq(7#H8!F?IrNeAnXy~(Do1u3YH)5#STy&wrpGzB
zX5!D|(YvUPg(0-%Fe_|u#<^H~#B_u@nZeb+nLRNd{@9U4+K~XS{)b^~>genj=V<b#
zby}1@H-4X^$rP1Y!uT<@YF<%6Ncc)*9{<<RpFf+L7MRra>)(P|r@G%5-Pv}AegNV@
z3i|TgYiS#jMomusd$bs=)ZOV|NQZW*BB$|K&gtf0{Wv2tbJfqEP&e=YE&`I|t!KET
zNUWrc+C%+-ZbiL%qhp=^?6j|pT*^%tf+F+2`OAy#Un?ity^_^Y?sW+{pJ+WU)afF5
zRwHJ(rh{Ie;!VaX^LQ+}vAcauYgABGF}qzKSam6zHMo}(CHJ^|HiGETj!V8bWbwmw
zF!m&UIZk1atj4-&0A9wmk<aG-mzIG*{c0A@OM;7RhXd#WqffQDMm?)~I1w@Sp+*#U
zj^Q^=uiq8GKQyn#B1+rk<mBc?cIy^xKi-{6f`xC&Z9{f#wgYGG!CIGGd_R8ts2OMc
z_|a4G0{ZU+Jss;l7YKV$Qu;UV$;mahhz@q@7K_U4=9{*UU#AIB5I7R>T(sBrB=^<t
z76>|+EG{6u&sl;dcRFSMiZfkEe6lF6U<oeT>BimKvb?|lnNrRzjr%($z|x>N;@2=A
zr4~f&?ahj8QY8<U{f6fGS}IG9^$L7<_+y|bk465~;0D=OA`Qy*hFbC!-MY7ukH^N^
z)eq8MfmFwveTv9d2k{Gqa>C>3UTw>)S(+I3Cu`jg_qQBbkgTlxO`b<Kx0|z^Zy85F
z?DuEx!CFH5v})4cffJHwSayVeW{472!=Q-Sl2A*pz0m`Mz@L|Vb2u3@fERbUeOzyn
zR@PE42j5|ubbi2YsF%w`A0B}x*bZBfb$P+_rD6r;bqr?6tnKrYV8)4(_<9!qt`nmQ
z+OZkQgW?-Im-WKl&d;lBgA#DRdW3Jd3EZXRbw8w7#t0dd9^k4zSnOvz1Y6U;>140h
zVSkgkBgO0`n>V`c?R^l<MnNH(E=3k20_nqIHq9w6=CRr61FmDid6&v}sK#pbVsE<K
z@mM=uYO3DBBIwJCJZFl8U>(8RzjJ~*uDC2wBxC-}>YyXb<Amlm@R7KhY!Q20*%OJp
zhk>mLMko_~k?uA5Jfyxb=Zju=V7&{WQL2Y~@wJ)YdP16Aw;PepBFuL5yB744hH_p@
zjKid66jWqR$eBaudcQjQdAn3XTBuG=DKO`g!u5=d3_0=`8BVvA_8>Z)&hH!OaDMl<
zmptxwtnNFTn=E=gsCQR$wKnbCNSMHdp`f6=xPJ;d|GJR)i?jY!m;8z>wE8y`a!D+M
zG$K3Of*!p;kG*zB#%AdAtNQBfW1WxSz1p)PHgocO3gy-(&dbO7Noyn_2H-MYvszAm
z$qspeM1AaG%w1KT8R@s1{>%suBQDL*xm`_6d`4&IKPN_czh_C3-uL!1Q#;}#O0s;Q
z^ZMR_UKVmH-V4c9wfF&TDa&@QKA99BAD@u09d8YMG4RoOaui+dhs}>o6%}LV)w}06
zi|)5!j(*1hc<w{nwrC;592~b>!x{Yk2v#aO^78W9+5~45f0se|ZCDNW{m!kHOZAV!
z_@cbgS)VpXuGhWthhZ7bdYvan;|$}2)lJ?=n5^;-on@9Q(vXEHD`ADO$_p=@^<s}L
z?v&KYIR9ctaVAc76-qtGfb_#9On2w{ywbj#UU2s*e|7Oh1NV}?Qz>zFq2<mmTF(1>
zGrHYP)Y#AgnJ-5q19G^tGV8Xhz(9l4im{CM7u@{nbh^TM9`9_vetmTeyMsb1QFD+i
zAtEx<<W?_`fgIAYKU<x~=|rE3i-mOyLIGA-r+lHZcnZ5*x)f41l15J(;?dw{OkCXN
z=4SKRi0Dx9->Dk1l?X^C*IIOK=u;Dq4<<6+y_P}Zb1)xkJ{LP^ZO9ZX^O8(vO_$Ei
zmPna-X=gTDMMXlQztrOO@AICHGp~_pF{+IWZ}5Cx{{2WO7)+Dq01KanPQ|o^#!}PS
zvUfI(_qm(T(Oi)UV@q1G{<#ZE(uwuL<==DO)0b#^&;EU}V)XBc=RffZy!x2>k6VL+
z3WPlKyr~aRcSStDI$oWvGBd#}tu~)`UTKFl<o^5I$J7gY+ZTyg|JS6T)Tn(CtGKYr
zt+eLqY_P73FGR8D;O>X*T~hGDi_Fy6)eHDh?(3NL*V2yBuydAFn2Nu%MB`RgRsz$w
zn%(bjx}!-C=W5$lk7p`PSghBCg1&fTyLV#0e*9qL`@K~Y824eN$)qEvdK$Nz|63dp
z4GAG3AyZRRdwcuMft2*<7f(YyD(M{k-&28xK!}@!F~sCkv`>Hp41%mBUa)}k)=T^K
z|D*$wY25S89*;FOtRWrNYkwla>JP@#wSDrpU+cp2SUbJOb3J;-N5;i<e|LS#<$U&&
zhLnx%x+jJLN%HSk4^1(yxJST4j2BdKJ}ii|Y?Q3w&jFXGk!tRvK|eVg*O&*w{rDu0
z$u(n4?Ls(2g=*@C&F=+2^?9^`TXPYrB+-S-l-X3QLkRRIYOYTMsz-dO>74?q`#p7m
zcnvqQ1EqKsz6goWPlt|rE614yTlI@W>LRr;#y5IEkOywIES`>JCz(r?i+WpG(p^Br
z^unv~J!6mrmh9F#ggkTkc{Cp}|F=}50h?M$VTLp${RXDJ*s8vryysfF$^Wk;g;&Gk
zZ*o9sb#t>@Tx~Kb@exUgC~7m+g_?&a?cu^?vWN>rPLK~!qD5db?EsmW_kzrfij2$?
z_pj@1;m4MAaFs4|P24$Iv?Ixw-yVU41owte@yv_dZ0rBZZn>Lw>2SP3$3xvzV>;+D
zb)bDb9^oq=RnsUQQf)SzN*F$HG<xk7*tnfSaMnEk5xs=^s`ZDTmWG?6_M?U$_!q@4
zihsPx8awLddDezg8UV*}hDE$sp5<zIwLZ=OSCm|wp<r+<>X1zXdz(RWozF9)pO&8?
zoars=PU7cCS|UiFnYEUk{V5{Oj#GYyF1zq(m2Kx1H|&6gexPHNX?B1lId5moLbs*N
z$eds@F%dICOu+dQDQOqon0kq3>#Rj{qwDnva2!*QlixAy7YV-z3kS_wKY#u_FF!v$
z;2*Wv3Qc82F~L5Qo9I7<?;=s(C+@?*b#vM;@HQ@d-^|E(7qz3$<zCQ32VA5t9%4uX
zIAxC+eWlidm>O2=y4g9OAA@=Nah(`~z=~3@#?w&#u8nrQ|3)h^F3gVEnT_Eg=L4G4
zbS~25Oj=M~gMp1v?#!BF4C8PvS@lUY+6a=D@Z)grQef!-Gc8fCu%kj&9tG8xhl9D0
zGy1_4j!EOnx@>ebG-{=?%z}bGx?%nD_wV1EO_vdKap6c_gT;5)pOFJuvVsE8f}-hi
zgL<ki<H~^)4)nkCgEG{`wBjjZ^klHnYwwFe#_$J`W3iYbg@^iZboCe}y+_(a7aMO7
z4ViQ(c41yk>YH{@Ad1v`#ed>It6u4!a&a|`y6u!-t*h%`N0EAlmn9|E1Z|O1nKq|!
zzI)N55*Oyv*`1bx`lWav@!m=$F4pz<b5MKQ>4kp|MEWlG9al`67JlC=5;wd^b^qSS
z!ud0aZxY$9iUyF<LPC(W3=pE0o5{1NNM%arN<iS@>LfSj=H^m393)aX*@%eNns^dR
zrk9tO!Ah#uTCe|!Bt+(0?F^eOHyB*$2=U|7I$UTxj(fY^JG^ZxOR@6TCqi*&ep;~5
zDU~wywmj&}aX+EtBo^wX>R}PAMEmF)d?MCvV&AIO@QYK%77O2q4V+p$NJxD-+RAs?
zsr4LOpjfywA6{=Z;kja-U~1!Op1iBdIO!X#TjP!@weUR?hFyZ6>x$T~zb@pN!fYGz
zKveKxl{=Reex3K_ZJeA%$*PsBUDkNvU8WzSs%C`g)j&<d^zN@z4D!=xD+H#hrU#{_
zbFZx}!x)iu3`qlheOy+{r5FYEpx|JFM!$Z2lv&QZySr?$1aG`F4hL#(?gcY8hl9DG
zpdhb*%$HYZH1HMz;c`2@S|B&e?J#c>8Q<tX4T*BTS&oAaLsX5+o`_Yd)SIX*AM??G
zX&(jab2gl`Q^sc#Hc(|=HIv&iJjxHY1Me4@rwbI#tE^h#ogC{JJu`Z89S{~FlnA%x
z;@cD<za3qpYED<H$IUo`Ml49&?nHN=>8v5>jK0dfcADHNL_=spD#aE}oXXDFs;|sC
zhjvkl)ay8V{#Umxy@b=FT6NoB@}X(`p&nqNO3Sux9Pa*;8ud4E5l&9!cDq?>^|5F;
z_Lo6w0P#L5>RtW3`nPYTK$pXDI99fK?v8vUAmxgb)6>&}5x6+?x-f2RFzE#a1?}x$
zwwC{{H6!!?Z}!qg)31oz<eYtTJ)wZa_oAz3Q4!6Z<)sF8m6iH3JWjC=#>I|{mc%QG
zRJ=O3%8m19EWiWOnfb|}&-c?>6<KRf?)ocpln#2eeg1a*JP!xCzI?_dePEREZZoV-
z&2biWRKDiyR-Q2^q$&8?GS7H-<lWbo79$+#&4>BF;?gR<`@@UD#ULD7gedVqN_1^g
z3A|aXicY3d$st&3VSC6p9QJqDn`so}<VTWHc&wIC_!ZXcA_Q$9>b-jYIp8f$)9nFA
zm;hmnNOW{`03u$vMp#(b{nctjn1D{cd?CnD@}$U)Z1b#pdV5b#Pc@vkiT_I|1;Z!?
z2eYNu$fL#sfwTPT-w}o1_(Gd=C$n1dWl4iXsS<W)rpio3OsucP>qYC{Us-Y%95dbt
zIl`|k-{1@XV@YX4inm<2vWDX^c65}n&Bl_d+73L1YX7{sq!k9iLoNC8H%W$qVk9=D
z{ZCfI%lYP?d!YUe!QfT<=)aesjv4=R_J4i%zdz)Ed-lKI{og+1|Nid(G5}A>r4dyf
zDlIRsf5+wtt(f*d^00re{*QDm@uex9KUHVT9C)kcMe=7%9eU?LlA2gfbe4nzq$5VJ
zPp<0pr_Xb(t+y?D(_-+jkY=w#1`-m*e5Gm_8w@3o`L;^MBi7~C@qO9JIrPN`>oV#`
zjddvSjK5;5?8{fGH*&aK1{iQNPKzz2fbWQSr^?J?A+P>}GH;N@?ScF%MCcG_{&q0#
zQFeFS3TqWtZhoEb+M63*fG9NZIN~!*%tVYjMKkl+ZF)Z~x$%l{LJl&&Vod3d?n8G4
zq0v{SuR|I?H<?xD3MA9|c!H327V~)AZ^Z5xu37`D2#<Uq+NlI(*Ju;(&`e+{Ox32g
zIUwj~N+s6`{Yy<x!@-cRw;R0{he_X7gK%UwFPz)?4t3gK71~p9<6K$l8--5yjMnqD
z<3CE=Iyg^mi%QVI-jI=9Wh@yri<t~ijh{u+o4|@zh*exPn{-v+o%WwD@){U*b5+B#
zTSs?}wyg#zU{WwuCL+T+tZsADxdob1wkM97NF1otZd~@gDu-WiI>AA!lbnrXZ?ZiP
z%g_EwB9~|^AE6|+mQO3&;C5!_J9O24{6$-NkkOP*-(^w$#J)T3lSa==`jrtCu7$d2
z`>wmx>m4iwjh(Bg)iTa7M>n0D2d?wOx>t7oe*PJc4z_=E_<!bu$uhptP`k_PRAtD%
zkN@}$DfJ!YTikhf+pyY={Ltc|dk@Vl%x;^-o(?pfls0xkU+HM+(hfyqX>z_CQAKo9
zI{V#zm8<>mughK09O3}M67O}1&9_i>W4|yWDGx&F>$P2LbI0Q*@z_g2Io#`8P%<eh
zq?^R<=31#??bk0|+`m6aso;pcq|9R+2Mat!Sv4!#N9@3%$I~L5(4xy=#@URFKSo*?
zp~Eg0hOp0*H%~T75Pj~p+jqbq{G8DEbY*>TkGovc-pj2NM>2WdrYa*GAxume^-MJ{
zbvIf<@GxJ(Z#1BwXq+w>k{0X~+`DnYcIRdF!lV0oU2~vyVYt!ibeTHbAnCS!&dz7@
zUhYVtCC{^%p;?{Day(hx23r(s*#66+#jSrae6y6RR@=}UhPwWvBG?X2s})F)JH;JZ
zs_K>TrUJ{``9KI5i_Pj#m{MXvpdGTL$L(4Cg%0-NFJz0!Ej2De7l|nzd_%jT6P~!{
z=~eOgCr6*?Z!B|O6>9c8d})0x?)XO>M<eADOqIUrc+#0fLax9`TlL$F$NO$u+uPyx
z$E#n1%X4kAm6t&Wqrl`=FT*VGM%;I8R1*3~%<-v4gEnWl1CT_TQzhj7+%23cASjkz
z?x;_TOgEX_#$IYX3`8H+G`6c+SIb<yNdA$+Ck=Kv+M3dYD7TV(v{r{xjpq|L%Nzj~
z*9IYiv0cK_M3s-3TrRL!9-4|gy7wI`A<MhJZ{24vuYEVUMutntwrX+DRcY8$R0g$h
z%j)DevtdPzMT-<v<Bo{u3F~;?pZ@TrVY6Iq@xpq|N;z=E+&25N5;n$nA*be>w2C*9
zbH*$dOuC*2C4(XY)_@r=O#ha<#<@*Ie<7``EGES!PFes`;=D<f%8OS)-V<-tLy%@{
z><}YsRtDcs_7sn+>{c6?3BqvAZYZskzQAYP+)B3QcdAWy9T>iCh!x<><uxu#fIIiI
z?|TT?i7;^Bf@i>ng5r%XJ-!pb(odSu!-OwY3}h<oWyI2-bqv?d%fRf8Si@~K&_h;W
ztT-RC=D#kPXlv)BYufVBAYQ+hIh;&10gCeqAC)vhhd!ysuR@HJ?~V_NZ{2QPB44{b
zZ`MRjogb#~(WP<NSoQLf-FL`DO~b||sCLa9viF+x6Uq_WUA1$=x-5=(nEC#r^W<c8
z#9YxxuCv*$-Py%AX&j@zkn*jyU&=OA?xDr1E6d`3>DmXSwYb|`ia@0!oQu#?1`0Q6
z5tY>PSL@y%1BYkdx_?u>gGU5MuZYlPj#sW#g=y0d+?=B+F5lKzij3ockP;E!A1f<d
zk1p?@KkHhKKhE$ZW~;l3m1;5DRXVS0P{bvx^Kqh3;*~VW7?Q$jc=J?22l-&<%WTVD
zZR5=2bUim}cR5^hp?F=?K)dh#rd%&px9Rq_p7cbb1p$EbLyU97^V1$?k2UR9FIJ#S
zkn-&`cN-Nh^YY}1*nkGhI7lRzW&=wLsKVXpI>oAY*h@!8<U)BAJKS0T3MX?7$;s<n
zv@V~bYM0KLoo=Vf)MHQ29skn(>vyJ>-whw9U#0JQ4IQB>W;hagJ_SGU|FHzV1ewU@
zG<0!u`#<iXNcFlfkFhp!k~V7?h2%0+Zi_!O`1>iIO7=C&yN8L$V4kkXVvq;wa7J0d
zS;X)De}$=oDml2ttizlCY<c8}uvRcso{(?rsIaYV7NHBCKwSU?qQd9OenoGn3Mq)G
z$Y*%iHR29;JTj&B85bGsCF6-G^&EPJ=Q328CrgHSF(l*)oy8s$Gb!I$^iFvyc3~N{
z&pNuV#-TU6s$M%bPr2tq7tOF4L#zwm>9YbHjzs5vakq;&JZZs>ojY@N*Skn6Yz;bO
z8F3uXX9c=(c5~jwZ}aUY?nC|2;dSFC#T_bG>NW%FjiEsjZv1=an*!W&5{JZwy$C(o
zXAYT^(#w#cF=JGBIBhsc^MBcU5ax@oUf`RZD^K%LaML_3y0eOhsXo<9yQduOI%G6~
z1$b?aeKdBG`As32bKq0M@ywqg8>}gt?c+Z-oN!9W(T9?0O*S*()3@zsMkr$&5G<bM
zpr=PxufxoE(D*Tq&peBdig7F0c`-h&iw0EU+Mhih-0Jkqw3_&Iw0{2elgVcS>6**0
zZRMkH-fq61>T>Wi3T&EXkI#x+hIT*6Xce&=-2dF%dg~h>k90wJid;EGqy362CslT}
z%)!>rP-Fl-g(shz<JW#5aCMgots0N0_xy=r`b1qU+V=2=7xU|fNPLTGH$@;kt#zY`
z`IiStUM-!yjuWM!IjDEkO7lY=fq5G;O1{)IIB-x?0o(>;5*uIP>H@`r(6_1XD&~I#
zAYPNP0d~&(f+ExL7C8}~tXa%?>z#qzAM>B=`<g7~nm%>Q2%QeaPsa2=ns&A&f%u8M
za5V>H=H@M!x@z=i$K6fYWK7?vj6J}vYNoJx;qJZU-Ts7yep15VvIcphtWiOONx>;r
znEYy}IfKY%?1akhVBR1*S2lJZ+tuWc=9M+8MCJb3u7%FVAsKZs53A7<aR^Bo`3QR9
zjY-UE7S*HaatW)AbN_JP_?q0ho9#u$59UwIeH-!(g&$ND5LM?H%1Hf79`hbU4r88D
z*u*HiN3}R^ZXAu{U!l))1y9;)111rbEJ{@_RHuEAZkBrxvHZ=KY_9sMHR+SZUA>Fc
zCrwm}*%~)OMX{@oa{XN2YFLbbq5t?6_4(x-HuL5q``sdllffpzi#+EKJ<alU{~cg(
z=#;~NUst5l>>zF9`B3hF;1(vvn&5F#J3ojc##OUlYxblQk7}zoIsAI+Beapd8%^V@
zg;BM{PI68ZkE(1S-lt)Ga4wQO0vh;6NhyPLV6NJSQ^w}DDh*NrYaGqMKnkHJ)e?O!
zCJUy=D++3qF>AD+ULd}@>U5LT(L5m3_q$)cD891kSRd()Ec+wg#*FVPe^>iF`3Xue
z#4i^vkHRKd7vw&8xsTiL=&W#2L?hj_^>a_N0)s(>Uf|$^@cHk***~ZAf9{PBa6^CL
zMzo)6A93Q22tMZNFf^Ugd@9^N9Y>gT_qdA<Qm>%@a|Kh=?Qhi%y!n5ipnq?NT3Yso
zC;9ojO=>|-NA*|W`cVQ(dAIwmh?}ZQI<M{Fx0mG^bEf?H7X;K12O(sWG?+7X$fK$;
z-E7EK&nHE{I+3`=ylK9?`T8RDk2e0>*T;UQrJ*QD?d!+OI+Xu>10s}STDETopY|t6
z)lE~H5!2Yzw5e`RB?rlbyo#SY_KjvDtbotFw)KO1O^2^i-z9O?E`j;v*Ewb->^L&G
zprvUu#Rt2Lrk<UzVF11Ht1h#kg1Yz%6$e7RZjTVxWK9l3WZqO(c_J01Uo?HJS2j6C
zTw`otccN8L(YF#Ai&a{~R~@lTYtsuN#?~X^VOIy$xTH{fL;8P47zjy1*8bF`TQX~Q
z85<C0ckz+9xamNB&A43KUpa|rNuARyr*yvdn+IXr_aw71xjFRkSu@3B92tpNeA3aM
zO>&2*&tAW@IhSKj;<sbQl2<W{?w&X8MGl_!5=jM!BSlBbgi@o5eWfBLF)?vbm!~9D
z^xxHKA4PmU0u|o0b6{LC?p-=fNgdI#ToxGi;f6e^J*`cDaenWcMG5+Pf;XQN01OCK
z;K`ZLx^M`I$;%ag2YINXE`BD8v4J;F=*xD&XA0<zg0{3>zx=JORx3xoSuz#6#yB>W
z7_+^zyX(i7zDu`cbNcsdkD$CRk>@|cuf@JkeBoUSgU=r(PN_xc+y1U9ePV-ViNN<v
z*~F4H;*Umd>C1rQ%xBoX-=*5+1&%BA(1IT4$4%1luxQ}$QIYy8r%Qt0i8}5zIv}mX
z72U*slIqqeMVii77WfYrz&}0D+C)r6fDR>XuuPej2DPBDaC~xWpP3`B_YDF90Gh9i
z86P}kWq*u*a^E{V)R0l}nWh6cYh16=ql<w|q5P`Tk*6d<Br9&*j~_2zzdm*3^p#Rm
zOG!(^<NKpsDl04d(YK)n6%Eb)P|2_EsHB7zq3z7s%}k9-zv~M@q$;&R`5|}q*15BS
z+L&>rDs^#gZt1kCDzyT2oJyg5qk2Rd^eaF`(87?HlhdQcm@=)BB`2<=_1797AGfo!
zgTYS_^EYU-V8jfA!B@qN*s-Ih`KvXE;+$@f9f=GP6mmFXaSzX3OFp_;<BkHf&|aMn
z#{2H{Z~26XmdE60?|i;@=vK$mm9}e@3l^#&cautVNZVUIG`=@{Y`I0Cj<z}cF~N}y
z<7?{mJ!?|=-lxX7P;RSB|7!VdpUNw7l4$94-STNBCMG#KIgzY)5$QoCQ`<jkC}ckb
zBaIkWf?+;;_N?f0=Cmmq%**gDzVsno`eR2>+LzDDe;4${lzCQ{etCa)o>-7Hy<1*J
zMh9qVOgbf+C}b1>oxSVU&T2qG-ow2zPdEv-j+FY6Y+R|FB^`v6S5#CrZR)64JZ>yZ
zRWP|rg(hlfNEXsDvU~1G;I3ns6L-#@i2fuD&)4vKmzEJ?1LG!zF!(zAjps%>g>Ud>
z3oE;yvxVc0l`ECbnXaCPK;LOdWDjqY!27f{(y(rvC^F~U5K$@|fdZiK_o}gX7{AZF
zy0m{c+_nU&Qg2$af<cnUBvR=|9IRNfTG!5L(jsktnKENes#T?Kf)>MG#i!xRDJlx@
z&|YR)!7p}*Q(ZO6Gp)3%1QNl18vyG3x3{bCZD(~ap`qW!w(eo%rXxW6tnO!40mQoP
zv}D?pz|g*Kc+?rEAO2|`Ek3U<TxbI!FWA01=|6s?zG(K0Z_~p^1B%-?v2)g8$o!fs
zmdrXh2`9s>D`!%_L)Av)j!n-mr{GU0K2a?Fy{V9RWf(cbF?0GT=Bb+3BYHH@<c<6}
zFc{X7Y2gdAX!BICN^*7NoQViOUQdjT2@$=Eo<_yM2-J#4kHa>}{Dh4URvT!WVBMU-
z2RQ!eJ?5_2ZmfBHM5ABm=f5}pmbL(GMa!E;vs_U~LCbD#5<k|mLJtisY5l5=8U~-|
z6C`^E4e5)yV&ep6&Kg_lO}9`i#yVVTb_+IUzX6$6W2?HHaZvV@kvOIR3z91bjeCoh
zjnJx5*BgW@x}^RM>l$buuMX{?F9eP1K<{g@ssAz|)K#sWX@7QiYT5h{AvXH6E012c
z3CG-ZY+vn^V}4elF8Hr=q3swgJ(-F%B2oGXY54OFZMB3>`8Rcwj{KLSWNYYvV>t7K
zDN0|D&x494ld(z}%8P1k%oEdg@8uJceUL^JrpZMM>CA+yT6*|vtQ|PGXc4B(;&T_&
z^ULKFl=(BRpQjH#+zjfMQ=!2h?<9?W;uC_5F3)1wvZpAz9tz?er%c$_&l^{sT)X;y
zU!~&{B17G=Wxq?|ELBvMll$5ryE#w1cJz)&oH^*ruW{q!MwBZocToE|tE2r)iH1_G
zAScJBPLO$WdYYe?2Zw;rekr+XMN5s_WUnqKxBjhBg5=ecrYva%LsTf~*{T5>$XYS2
zGeXa602(!L>RaKf^^Vx6=M2BU4@Katm3+iOYonp6`0+lG4k3tWZ_DwS#*!6M=;K<=
z>S0({70UI+WoMSOk&)6T{jQ_Hp>0(B_RUkKEX=zkjE9E@iR#J8$<u3BSFNdI3`D2=
z^Kr*3GPDr8adz!RZ-+SS^lp;qZs6>coIX4dqkwYF+}C1wF7P056(G`3GK-SLj89HR
zvqp%?tEq(@GUM+wx@2U(#3w>?k}r-H{+nDlmi$}6)gLig+}LX2^`t0DyS4n4mlxc-
zzOvgM{$KHmxBU48<Wp;$7alvv0(dMx0Kehm<D(YwEG_A;9{|Nwd|J1cR(A>SyL4JM
zH~tZo*NesCSDe{&+!+6+B>x2!{x_ls&`9y_ju3RLN^FgoM=2>8ndp?1`Cqc;Rc~P#
zt0#=`sdjS{BjS}@b0iq>B(!#XgO<M0qqmRpp8m?op(Q8J6OE(R4A$o;YoUDuMVtJ4
z3I?F0F8%@;-GBbHctpH#Hr3H#oN&3xAom<hPxI7r_WuLrUl#BSM`gxx=H&x)1|ILa
zHIiigv$m4p;FCXp{safZA+)u83=5k#572l337h9KDikik((S1E(C#_*TZMEf!^#-v
z#?&39=7t8<jA2k2j1<B^K?!+WWAfMP=;#1-C7_9mi##MG{%y-2eZ@$kl?#h<b6aV}
z$YVM?J5L-rAze6k9xLDCD_kJ~ncz#y%B0JuN3ulG^MHTK4+)0b-4*A=rJ<xmcvM2F
zpD&Wn2TF;=XI;xlP9EH^zpBU19fATWvU`vA-k72i!{5)(?^BEj=DhH$pCc&(w1Ur)
zQ$1Au(m#>1v(Jo|Wb%zD60)<`RmkcvtN8MLwzUoI)+PV+3D`VABrS7Qj1bW;y1;1a
zvkNP;v%tW>krC<odFr@cK=QC^6v81PB_$;yBBG^58q{P?vY4xhjE`4TQ4!6c%;gHo
zmrdP^)H13dr1DKyiJDkXQDI-s1(w+O^XE^XNNrjas3Lir_R_662U+ybja(w(_Sgq#
z_snspd(2Ft;P&Lj$;%z*n;2JW(Hz%%V!Ci$5C9h^CI(n041UJ2<L$+MW+qX@9)A}1
zw&!384i;-Nv*pjzF)rMzvF$a{uV26VHp$w5e2d+iE_^mag=LiElmihWVfZ;pB|F39
zYHMfB76^%)UBG?!&Kn61_S=uRI2lm$WMTq@gmvrMtgNh8ukw6-?P2giC4d$K6y>{?
zEys-QzJ2><Ys<(-#=^qF&dyFv{np{<@87=xF@o)a7qXE_z8F+jR|hyoHnpmWd7z+=
zf`hYvcxX|nEGi;$WG<zuYFeuJ4h7{H0VqUdWaOE?RY87DP0fO0#H5_6YN)}H5ES{m
z(MucD7G!BH!ybGl%Es@Rh-zjt6;Gqm%c`xdrB5gjPh!5x70D7268ibHK&kR-Dk`&0
zZZ5vxMY71LKZS?>9HXG56f2Ym(cHB$FEjH?x|Ag=_6?@+zA*1<H#Z$6N;Sk<bfm7q
zrWe{+4UDQ%IXWupC|IaB^Id3nqxo!ztmdSZzm~&k1@g8^^g^rQ7!i{w|4iI926c3|
zv5Co7g2-u8`TVC;1N$wcs!E<D+Q}G?{9M>iYbu-@TcQzT%XOqH$$ICkB807@2PPfZ
z58x$5Z&j)=cS@a>NJb(U@IsLMWb6GSiX^CC-*O?{^U12I;WTBE#`W&*?1+(|7!7~|
z9(Nk>?J*)*t(rv6Ah$zJ8w7a(GBPr|+WXYh_;^5!nl@!lEt=e&7vWQGLILaJ+xAVa
z|0y1Xj2o|?UenRhIsTH*4+{-l16dF*4LP|;{&-zXS1EO2Ra8#MRW5qO%b(s)#&OW>
z_XQ)-TIkJg<ad~KP(rKsBm2Rew6C532fqF%%U?<3ynpYv`~JfRrLXyMVkE1UQ&$Ym
zZf=e08ii71a}z?wlFu<EFM_lZYPe#a^d6=9OxC1OUX&4K+yeg_Z0DMK^?rwCi`LXU
zr`Z>W&U72v#7M6($~Q(D8#*;&B>BJ{bZC|;I;Q1;vsEL+09I=t(_`%w!zIg|4&KdZ
z?u8#R5QthIlQbm_jRV528cL&mZzqN^?!~95Wdui$K)#|BNdbka!{yeMigDxqzCK$q
ze6@PJWYa*i3AB(WzDK5{EU*v2fHSXVn2+O^Z$b(8xc-#~w7Dqr>F{k^{vsd%{qp5U
z-var3-j6K|FfX6ykMGE*-tN{p)H^(v`b-#~0d+YkN`N5Fmg#>kBn08(w1Xy9<19=Z
z+(dC!qBK8)d-DcV-J*KeLEI71BcrBr3(LIFUFqg#p^UCZpHm(oHQ&yixGLahMDbgU
zg8zRB#wRl-NFAYn<oku*f?1Yyu5w}L=IYdxVpuj2D{D<r5j8XOgv_l}t|WOVI*Q<n
z02vx+_O!F(>|}PXeeH0JuugApZ)UTZB@UFFcTe*9V2pe4B%duu_!xWc4{G?S`o8n#
z#RUaoA|eEohV%*Mb2Z4gxDtEPb{zH9X0x$|wB8L34S)U!d3bod4bUl{E-Nc@rRJ0y
zcXxL$RH+t#0jV-z?lxdq)~c3Ihvpm6C%|YJlq%ZS<0!<E{Mk6JA&C|yLv^Nt5+@~k
zA1ZJ4PK%Q&Kvu|q;;7;)rZ%20QhMC9seQfzFE6i^l@(B1Gczjg%lg8|u~c=JGRREi
z?-G2W-IE{{=e)}3Tc<T#*-PmUUKNcPaD637SDDBk*fC-fW%ZARZ|ls^-v$sO!Q|!T
zfz3*Dvj<>dNI+0vUe+Yzj(;s~vLJoyzdzgJH9&eW{EzM&FH_Ob&;SvBlNk9@JYQzz
zpBUg0e5MVZIWj<+kxy8%=o6KfN6y2;)76FS#k;gg+WA=5<-7EK$=myRbhoco{IFqP
zZ*SWu9RiRHC0eg8eXcC|Bo-UCgqxcizb_<@orh=8ka0!Qx7{Y4Qwqc>V2vHx-}1-X
zDNG0=VHUS7n3Le);Bt205a6Wq$Mwsn^~;xgnN#6-U&UxLeurB%V$0;CukeRZ{mwN0
za*sn+sccrMTqsY$f9vb-9~&DBLL0Kh{b)5zE*I>av-OKgLO(73?-T2}Dp7D7xf5Pq
zUXBb(l(D~m^F_@0?5L9yey>80X^~h(Xft1l7SP?&{9!B|51^p8(e{7_>l^g^#f$Ns
z=dR@ekc)JMwDeWFWEzYrix;oi(`p1|BvMlC>AV3$I>6swgy>y$kb1qI4gwS6E$Vg$
zf>73zy$QzooEwSs^z{60YfDMuWc`xysa0dk9<QE^U-a8{e-aB2fD{?Rn>QV!I@;QB
z@bLZm3=v&2<S|9h>ZEfU-0$n=%`b0c;$-D+Z(pyiuP2C+q)YMc9R$NfNae#H@l^g<
zf95*@d>MHRaOb{VtA76e(p*Bm0e*nIqTcL|<FP#s|19U{yHIH9|DkE|vLG_-GV-DT
z@>0G+@rnftLn&fN$2+hwd@Hs{ka<C5x!e6_EF9JU&1dB}L33f^c!+X9Kma-_s;+^-
zstXcGI}++&k2lH+d^DpW_e$yf;6LA<LA|FIU^hDf)UUgwEmKXGUdhUe?$}sdOA8-a
z(q&EbY=^;?)Bp5U+lATEfBUS^pG8kcxMRed;z-6}lWNn@^bM^vg2wn~xfQ3~TRU6U
z|JKwKWCJa1Y?0(%w7l5d=ugtp(vr>{`;@ZN=#FW=I1tnOBB4N+$8w*Sb}91k_fL15
zwvf;7Jv)DUof|<J9#j~OW-t3`V#q5gDQQ>$4-}DVPbEV8Q_o&Y3J)rg0hF?NLp~A`
z`V4M%b#$~IX8p~2pDO1c*G_B<x%-~zz&_+qVIA04O@A5x;I$*-{LR@~)iceH;KkGN
z^VtOC?}Om{e-y0$8(DL=-Rh~^>d4Bbxt`El{SAA~-$nyS@Ca|38$L2UZV>B?PPL9&
z1FqY;MzKXEd0iQG-GhmF>gY@Re=6cDMljP{OIHhdEYR~}JImGguD><xx;Y+%w}yHx
zHomKLUCMKjMH)KlIJKV`kt3K;wo+@VB|mWe7GLBY85zONDjL(>;ncQ&!WKb`z;=#}
zI9@;H)!ij?ZaHcX8FyTl^p_J2N73Wypew)@8>`LUAUXP@;8V*v0eEvm_L+z;H2;EX
zfcZ@vWtF{=b-<E}q@UTY|JbnKB=k+iSm|7#HF~A*bB}1g0qPjmUbw>Xdy&Z$SZ0cg
z>)Vbt0%|bbP*WSd!&v&fJO?*bc6<MoQldzi8y(?aRt)d!xv7cCwH31+2!<6~V!mp0
zxYlMWsu&;tNtN}c+l<i2BfV@Go~0=V4FP}*{%}(`BWxlqLtDDAPtjTJKfkjdcWui%
zF`^*ruUxN!_ff9v8-lY@f=eJeYhkl<HFo~6VJx-Uj`N-2s5Rr$#skneiCM}K4nzB0
zuIGnF$GdkCs~+qtH}=w9(g#D-ib&6oII{L@7&6>a7OI@wBUc-*oHKU^eAkQLANlR!
zJXYQ2Q#)-t6uM8TK8z}?ecri7VS3T^-GjunWy|xF)R>cM6h3l`YDr{G$NMoULzSg|
zh|onEw)4A(<OF4k^X9q({}a<reZmUxS47pDqOG$1m-iWW&=;5+>9*t_hfgwyaoi~!
zIZKlqx5qfbFQaat8rjXW^bYV5gOih!eSKd}t_mi_W_$vZS%SG+sKJ_}_H%FKI|anz
zdjaT=zAGGo8)m@|?rPi7gIB8xi0Jk$h^WYWfSw;=>|(c&O;#Nv&;?I2h}aPk?R?e$
z$mrZQ_dBu}x_)B}HJ%U;Mar)*%eWj}&zbyP<0~OyaTy7xmCi-M^G5$aoMjNm{#W0;
z#X>4K8#~=aWQQ~^(ASVtRB@mvGgDh!;kDy&{rMR1i#IiDO}f7o`vIg2o2A%g`Gt9w
znWo9RWG<JB_8=7JMs7w%+uy#hAccDZ`X2q?U?oc&uY!I98Hhwe|Hj(dTB%N#b*)1{
zWJLudCnByAAh?enJ5HDB6*ruS0KkcFE6w9!d!dnoPbg#9biVFA{#r-Y*ATQ?x!3r*
zrHU7ai)k@2en=t`61Ysps1aRlS%3vpVB#(akT06%yA#0Rp4q=>^?7}{I~kEpKw9`@
z#(|LTgtOI5ff#+QF2BrekZ(IlLH|;QBVMTr<uNuW3?dqdz7;~t-dOG;$+$o5=pc13
zAg8F9D!||i?B6?l26(mDk{o9z^)?wzUcrndVu`axl)SH!!WL#RPt)j}I2dMNJ1#$T
zgp9;jOvaqr;kjb%#-GRrH;<;lH^@BB9VfCV2(qi=+-MJhRshTuAhq+DfVGW{?5r#R
zOaR&J1McBKGMgiUxNnC9gtgjYF_p)Io6m5vNDUB|B!q>n-teEEoRCN)Z#gD^|85C(
zNJB$XDH7Elj}L(A1^3c>Zcf$p^JKlK`S@oT(BmRhJV~Qm0n=6jn1HzKwubG2$c)!G
zY=C<#EZp7Tbo%mnYb2rI7KQcLlT$PN*5V%FFRf-I9@J05{QDwOcS3&lNYrFOI^SB%
zWE791aM$+PuJyvmw7G$-z7*Nuu`HQ>(r))5Lf8iS0*S|V$x)T?hmOl%Vv=8fl|sFV
zAP$0V{`hDu3~S@U>^ozu!i&^J*TEC0l953<y$w&qxU)-M@VVzURTM|E@$^~s2y+P&
zY-c#m?&0C!=IQPEu7aZCpQEC083MvY0M(=ioaGSo!~4&kF!g|_1vU&qMAn>uBmJkp
z|0|4IgJWdZs-CXyl9v%5A3#bl@}eUn>HgBJ31UP5@hvQ_Me5$;@!p{?o(}A32qFyu
z2l|YK-S~-PRY9iNEXW@TZ3!59-D}vBqJVuS(wsnqSZksy?lrwx>45j~-mRIMI#L(z
z5pbEic(;8jp}1twe8qRHRudlKP48=Rma~Qlk2kl-&Ai}>be5>!)TqE~w9tRpe?HvU
z)hiO_r`7$ebq=sag{~T*s>{Y^c=3M5O%MiF!=_BcTeWK+tHB>#+jEQta?;WsnJeO<
zW=`&^F#v0+Fq@s&X65E~e|)%yNry+L&dknEWwrXHP^?ZKV+<?}!{5m&D4iYbZNcW*
zw*~y~8=u@gZ*MgJ@1?!5<=>u>blxz)*(NTBT|HK6-!7h|5XSUuE1RV)Hc9)QJyl-2
zckzBx!=l!$qsBp0$vANYEA2>%2orv&ZM7WD6}M5_*-M9rUq+eVG-j-A@EGL-OnS3>
zzQoRQ0A3dnO!?qAn>yE(IB{E)%iAd2#SA+!wmOyG_Sc4E>0qABcZhg54f6TKZ!@Jo
zwK*z(M0)w^m7c!-iDLsGA}006rljCtWA|;IHrBVFIZ2Wezbq;!AbKAUmmnsu)IEms
zw~}R8>R83y=(cB1n}9Nk7SNAjE&5UW(rE$@|A$+V`R2_?SaYQS3>TejmAk}pK?H%L
z>qPotW$iRkvl0e!*W+!@Ly*=3Wh_gjU^Vi<EGm;*OMfnW+f4)<jO<B;-h|AyoJw3w
z-z@5T9Gatl^*EZAB{yKPX|N~Z#{`l}BdcJkAzzcnN60Pj2jHhrZUALe{vfEWINXiD
z)LEc4YQNNsJqYxzylyTc>oK_L@>M_&XdWRITp9epDj$x-UIp2iX};vM!NmLdEkJ#y
zQ-J6lKs`R8!i~uI4oULd|42*>#<ShFwrFE-8lBOk{Sgz>(b-AhiyeW?w?9+KVY$@e
z`RoM}pIW^=p!0ixW&uoV*!jnIM4v#%-uCwPm9xFGv$M5znF>6Tp8f8G*<zDhZf@??
zgHab1o&>AW6VDK+yIWm)J8>}c!_3X(ehmL_YD9$E3NxkdDHgsXQTXQ2+;OD<K?B5y
zfu<YD`4tPHjhSkB!6MhjpB!Dc8lAXNm{;AySAdC}GPN%NTK67zt?x&sXQ%{M0y3eU
z6@EfM#P@`?poDk0(`6DjdQ&PF;Z^3R(P^qu^Z~=#Nb^sYSuayaRj2K~gF~+MSj7^c
z1XH@w;XO{3k9$+y5hQAL16)-lyc2I$hKcp;q`XuCklkG@Oc1PMI{zwERq6I(^CM{O
z3=AWXfWV{!hz0aqx?L>?fc;yW{~MdS`-wsou<^ov+qI2>|Kt02fH02M+8Fpsfo_U(
zXRgvWGxefk7~HNkvgB%Yw%Y)N)1#k0T5d~N<gncu0!IY*jUl*fU7Jh-iOvRM;|=j<
za$f0k<6!o|hl0}do;_b`bh#vt>1lGiWyj%}slXP0iZlvKZeK(i)M=qTUw^f^>No+B
z9g^zYbAC0Py>9p<eqPS|?|vjuM?=R8E{Bo0)!0t~$}<*v4^SmrUbx=oB7vKr{<l^Z
zsFiK~x4rd}Vz(79OHK(XuL)Oi#ec%Y-wZE-Zr0)9;j{i>jIi<V^a+ciG3DRpi`j1N
z8TP6&^=|Th3D`lAxV=8gG5VP5@Awu9{kaU!+(?yTKmv=!LM)XEEy9aMpBBucKVJ|V
z4mnZ7wj#d+bo6t^Fz}ZdK(wyhJhZfyQzcq}Q~RrV9Uo`TYvV-?QhcH<BWY%qh=q}i
zj=g==E)AU5A=k-DWd-s`$iw=?7x6_0OqVh3`v^}@Ppj2V&`rcr3i-?_E(kWn#p;cy
za2>fake!_!^Z7a^oz75zdwJ=B$>Xw~pPzH$A_7kBki*dtdRs{niv@^qlj<&ox%qfp
zRwU3;w0b>1)_db*)QB+$w|}sL&N<M{RHjHNPacEkc7_~jUa7n}n1=W69l^Vx&Ir8X
z$=%~K{(!f{BqYG+0Pv@{=_WY46Kwh*nVFy{<Hbu0)*V2g2yHiK+vP5o<^<&;SzseK
zx@{{#T%g{loGumEu@a}@Ufa}^s!rrPbaTE-NJywei5BB4kZ}yi=U}5rpRhNspuvP0
z*4~)rcA+FDh7i%UJC-Y>uTR3yBwU&C>3h$BL~!)NZthW0hx^So<SLB@z=fdIWYD7N
znuMDhw|Vzuz4vS*D=L|lHmM&_^xuL521c>ozEI_s_ELI8-Y^i0@iRQLhfK6RHR<lR
zm~C>;UC!k?^%bHB$irSVnnO&02*RRs_AdRqd(XL-)H{ZZn6vaB2DiC_o362s$B0!s
zPRE>Zi_B+#jJ}CUJbNB};ZQw_;-(LwH%e|F&iYVAlFQDGT^aAI3tSVhr%dG<c?Bh<
z57^iR<Hk4^3S_9tg%cT;Z&!@i0X7Om)EExg1Y2O&t0N%4FaZYJEh&QgHbfq5M#Jym
z*4^`HQc3sw^U340rKKge+Y3;l$d}CnD?c(e1&n9W<8fY-cFVR-sa(JDY9$22A7m++
zz{4>oy%}V%+ZhA;R~zdYBCD)?)5kD)2iiT;(P8l!-S4i-%IGm+L7U#bL8=q9g3X`r
z_%@pZb`8}fB_&l=_T5p$^L2JQIV8FEjIX4pr-iLXT)5sZR8E1c;yo(r*!Vb=Mw2Re
z3`n11Swa3q+US%^MEVwuQjym2NV8H|O-?Q*Hg;ozyRY#_#e2*ZXz?NM{>8D(l-_P<
zS-mK@7Fq!!39fSkjmny6PFZb6e*=2F?@9XK@k2M{^>2=o7c#c;HAm276QepuaoGmj
z#?6wf>(Zr13tNhs_U)amXI;bfMmS)XM-Wt^)9)K`pF|GiMVD$n=3P;@JqqFI6s1ro
zYM<}^N7^;2MIG8!YW`ZRu7Y2_%+=Y!^9*!%<BtHu0pMF+cqWXPppJ_t`1PyVLW5GF
zJVWS`6)-M7KE9(>U}5_Ju6}e8)Ya7m<#ZERUm-_a=g1>*n)dz7cm4qZiw#cEQVawH
z-Zj-_Wl>0AV@)p`Ey9uaJa=y|lXbf>74A^k6f_e?_JeA4NKb=o6v}G=akH_p_3gKD
z8snS!`PY0#VMclfHe>(%E<lCmDGQU)rga&vTten^M@&hVoxvt!D^jPNc0+{0GwM`x
zB6K^(M3sd3@J&YCuNq(ZI}Ju0X9zc)YU<2nBv`~K1;|VCE$UNJ=a|!U5*Q^`w*E_0
zbK!h?dV0|$;+S;WFLTi{$JLtMx!Bn;BQ8M^8FYP%WSuZ3`#2_sgCu$K+-T#NF`%tf
zx$vsALmPjjR<&r9#F_IRY&3+0g<;{~Km$LA<I%^kda#YDayVQ7S}#G2&1(4*lndnZ
zK}uyenk}A5%o_ZKl`KX`f;~!PE0e^?!H&}|pCra!7ARLj_u78$5kgvSvP5p5C~<9u
zW4NR(?sQkklf820pqa{N>#vADY0Z)m=YB5FZC*#piA>ONPU`kI&Us*^^7_@QwJH(6
z^1OvrYMA0w40NPjP7kZZ*%ZI!Zp%aa>z`3e9sd^IlEgBU_OFc23nh&17Kp$t95&>{
zOaZ&B3t(`X7*#Hp2e2q4jwl1b%eG|-N2^{+a?C_3LKWrY{M1NmayrdkeU8x24BA8<
zK!X=RnMTgOgW3<C`_*U2==A=@&W<s`ym>Xx!`15PMt?|H7*m4SnW!P?m?Xe`nfry{
zT$2>Gt*s3wZ5AM53AWIfOt}>q<iM?oIX5}{c42RlN{AD&z8OdNzexYrTRqL5J3D(v
z6|jtZ&c<&l1RwVD(Hzv$NZx0B;~L%foNNE@>8FV_*ua9-ik?aGs5a|q4Hgi0{(#Am
zdSQ;6Nf#+dVzxxNx6LRnH$R*q=yJZ}r=>WU%EdrW-vG0@bc<_CsZ_QWEy=A^rYr33
z-ZXApSXc;}8Of|FL30e~iDAM(0MI!2T?B_lF0iOzzqP);4)n;#$mk1j`=b!?B>US7
zv;6VIl$875@H9x*;I{$%dCUNF1hX|%AVZ1_70&Pa`Wk41jq8G1jpcG8lL;D?GAQB-
zA9j2tr=dv$`(R&Kl&!DU<A3CSAA=uX$jSn46de7DOd&h!H+=Dif-)a-QxcYPj}|_U
zuAD_lCNggKvn*0lQjTrg0*`<3n_E_0WCreymCx&O@BXea*_4oUE1_2~Ffc8TRa<~f
z0Q@`<U{pX52KS{{^$Tkh07K4>!1Et|6U#Gfzz?tW?ysF?Uf2AZm96q2=cc~+s4!Y8
zcQyYTJk#Eb6xs1NxSWc`lNQrupv*%=O6r0<(0WsWB}+d3<C6?n)>>6_DXF3IL2j45
zDY`pp0RaJPYx=m}u%Mvpvr$RmMP;|`tT@26;Y+u4#UIlzFH|gj?qS08=@aZD%8LgD
zfAC`$fV|P(-cBeOKE$bi4<7oPih;5u=A%1a<R<7i4ULF!xJSYG_Wbzw6;RVabfera
z*3Kn_F+-ZPVhQwL$-x$H<daM`1_Ja=fM~U`w2`3P$C#7^PGJ}F28VnaSr8)Uj`z<&
z%y<P2-LrmjzB}oni|0&3{N#sPeT-W-bd2)`)_)}AiAg^L84gLLkm_!h$mLj?+xUva
ziPT@yv*$I}Mz^CVpFN%{6#OQ)90;Lqb+!5-yx-saf5>kF(wOu*S0Z=pcs!@W(uPou
zy?^TrMEcU{y$rz!dJBgxX1%afR-47}{u0#tKeXEIf#~Xv_)JVOJGpOt7U7<G3P7=g
zk~wdPA3TX8AfD}mM&#c9ejFZmPQ@n#4;(8-Eex#K5~ikmsa!5orc-GD(*2GZm+-|N
zCJ;O#KuS6F>?v6ke5d&5-`?q`{^oz4|KDNC|Fqw<;2gOjvI7*7JvEqkFTU!x0SR=N
zhm4m;)PNaua<9jvTeU#6U6p)s<@IYT4z6vMoAd9uFZ42*)LZJLVEOHW(y7B_|KZQ^
zib797<}IJ?dOzMyF05OA3#h1!1aM`cs0_p4_XoX>*U^Uv!f)n0xQ+|(o!W{Ekwcht
zh6?phzAy1(dyG9Gdy1rNWMr!dyAm6pvi#)?fSeeqMH`>z1EhC!B;umu{=|&94WYUi
zqghGm<b1jcbe`#XsZvLbH|1YtxLGmCfARc>%<u$#rdzU#G3+fUGFzD&@|K)7CB`kj
zF$;wHU|_7i<Gu>3_0%dnI7^OAV<bwIzd+_Foy;cWWON?t?d&suOy0ygA7h97D4020
zFtvGzju=ui7oA2|)*sWT(&K7K4UrT3e;9iUs4Ba!Tl^pjsFX#gfJ!6XNQi`hfP%EN
zba#s&A>G|l(%m8=-Q6hN(hYav`~JRr?>D}GoWXFMA%}C$v!A`!o@=hT=H9|#V<YH-
z3;~1>jw;7EizcWhSECPVhTwNuMY?CQr<+H$u4(h=EUKJu28?_<E3_s%nQK|tw}2Cr
zQG8q=FnTgE6kGBs$2XVJwkmP3X)C;ZuKV+WjpM}P@$<mL->2!md^92_J0rJ_gbo7T
zCeGhP)$Y|O2pBV5uH^dZ>7M<|4{EzlE6^+by{GR9AkpBc+++_lpq>0<MA?L{`y5C2
zeuD=a_xAp+uH3Cp`-J%J*nV4Nu@%%Or#2Zf{BuDXe5#{J0<T00m)r>WayEt>;!-CS
z2#6V@c`r9Z9ptbYfJ-!sR^i+kdMDFm1I!%sj4tn=Ksp4Qiin8FEN$%3CHPdBb2kUc
z%gIS6aO<+iL2F%0AaTdSv`xo9lwQ*Q|4&MfD|;6la>8zj#+sIlxgIY>0YfulT7gBv
zTdC3DDxDFcSNN9h^l+=Ey<G^6>hWV}#rA8N<C<E2k2)g%5wIn=`xpT)0$kXd4($kz
zAR6po0P=zH>egX~%4FT2@3*gcAjn@UDa9lvCdS1Xm5lwKo6~UKehyX|ZwurnWFxjH
z|1Q91ye>iF0@8-B4RCB##bT$wNX4P*lC?Zn)lN8XKdy#8(3$*xY{KLzz4u}P=@zy>
zNy|!xCvPwRztDz^*#8G@IDPw<m6a7Z1c^NC1LzB5%c9>+#OwX&J7cDNJmeugNdlzg
zrLajgeZqpAq*651!NCE>-};K?s&qNEeg8g7I&%WDei&`&ZZtJzsHa<<qI#TlokB|B
zztMWCk`!I$%e~}v>J9Jv9-=qdZcaKZ_om#T5VhCYg-X%pbe+@L(RPr)MP*o?n96l>
z>$4+^GG#uyP1O*wQlJz)QLz9P<B;zyL}>)#v*>}}3Y5wrK^8dQyC;kJW+Ei=-+|*D
z|K^H?g?(I9vmeE9UO&`FnuDhYt-umPdNY2q{}74?&kp}T2*qU=qf`)7s+t`Nl><S1
za(cRZd<MP=4{&gpYmEl8WPt>H81k8jYX^Fk1CfYagOid9Uemw^HfI3<7Y0LU8GZu-
z0166ZtgX9u@7C7*P-W`h`|qS*KFlhR-rzxs0dFXg0rC6if@U{B!72cLSGA?a(y+WN
z<$16%f!{&yg^hNdnwmO_QRC;pjLYHF(UCn#(Bu2JfR#-Q3!78(=V|*JfP>fzXPUHs
zkKgX_??h*@-gwMs_mXso`^TqiV#u5;9AP?^p-~UrW)ve_S;R>!E0ox7K)b$7ZOCG&
zv@ZpY|28szl34{yKr1pb@|O3$prB*jG@G!rv^4A#34wESbB9|qLD<#izl018SHaQB
zaeoyG=Z{`SyI&~@qi>7Gf=I&i!v|1ubOG5RObIgAp}|4>K_c*&+5|QPbS>F%UFp@;
zj?I1#L`ZP_pZ{nCzFbmMGX+Z3FEc$#ogd@>y_DYuv-~^x1Ylx<&Qkj)QVaTDJzF}?
zJvUkQIcV9`-F~MT39Nv7j5H6cdFP|gEe*KH=hX@GHLk{(9i~mAM3Hu7VcvZhBDq5-
zsHk(Oa@pamiH~drv0JWrkUe?wgoP!nWeyT?5<W*(9v&V}&gh<{xuR^`C_IP+pg*CZ
zr4<%>l9T{`Q5kZnrS0t!a{FV(piTWr^))xws&;QN&de0R3mp!O$;^-fYiMU8-^9$!
z4D~r+T9=FCrHL}5$Jl6~F{#gNz8wulW$>V9O`PVpb&_OPsB3gxaGr{-7=%h0I!mdm
zD)8b9gVS6Ej2K(&&mKh<nyE9p)<{@^_Y~x;rOlcw-jjEIMqLv3U!dtG<xW)9ThE#0
z-4T*OkH|}s=UhueBT2GZb}}(7@7q7{;k<y7iVXU$eo<_z>BEpL5Fw<z3Qm0Z@haaR
z9DIG2EE(lA(Xu^Njp7N7*mUgdIOOy8pRo-`@~1qIQBlzG+_6{3%hn*QodP5cmR!E5
zIKNusTf=!E0y>qGm*3v#`w@tl`m7)QU)FEc{vXyamlEnOS9V_`1xbDHU&_eXm6|+z
z_AL&`g|^2@qZ<eC0o-Gs{2nJcTLNxQ6Ab{#BON_GpY8f6pfGG99Dgs+6jiC_DV(kq
zG*p{N$l<mBXY@=xI9*LV8FgC9aO*i$wkN~iE#iyT;q;$a^aZkIz|PvMICWmPEhA%p
z>8-1p`hKn;uY&H0`aw>+UUR79kyL!{4@UYkX1!*koL_xoy!>k`OUO8G_ncNU=`tr~
zl!>c$_CLs#;Wxb+IP0{Bs6LXXqR$-jXZcDcc^?<I!f=SDWNZ>*r)HBk*Ocqpa9(94
zC+4R)>(AJrlsej|nBlhD{EWRiS+xt7>0{D6t!By2=JhUp1-1k}`#-ZUNL@Z+6Y$y>
z8X2VHdt;H-o6Yc_$#*4i+YkT|xE27WaWg>aL1uA&evXPsWWUl+)shSHN{G~FkX9J8
zt3V7+bIHO3w*e?KUBH-v(A-N%5G^b=))7o@V4Q!>%rwB}6%!M)CnfGDPk4pSzENNq
z?fr;3wKYZ&Mqu3<krCxAvfEOJWZms%_cmT~#T&`UzbG#DF7!_P*sx1hxh7TOE+r!^
z-7PP}x8R80JCQ@x@%KhZ-TW$pjD=UR_HQeA7yi)&>mPTIcmI^8{|7kuw+%$@t<*u|
z#-gGVi)@O)=RZubCwI()K6$~FiM)8>*+|&NlD#8f?#Ypaa~MH6+tk&ib=4C7d52yr
z+IwY5mbrf9MmLaf9g*~S-hyfFwBL^a|4dmE_uPqY7L-pKKwsK_!8Z6un$cT5K;Npp
zR<DbIA*x2NEA|Uf!9TJUOUoUwQR(RDh%06cBS;B}Z(qAsQc|Mk#(K-U5{Gl1yG(Bv
z|L|KES)!}XDGAaQEcS*O+rxjl4(@FiK#^_Oo?e#NO<PDMs1XU=yy@g;jQ=nhjpj*Z
z?t>d6YdENzN)z1XZ~6>XI4U2!7~&qMd8byEr$zk0fA}WN^D)eWX-cHf(+i8I0c~|0
z97rPD&H~QkPD#9TybV^CTWxb;adx(Kp<<MvD}aj81GMWTBqWR)^&%Rr5AWYPcIIDo
zCqr8tkox?V4)fM6W)6;#p`ipOO(6#dE~UiCNOtEVa|qogCL2(CTv}N0==>0{cQ14D
zh0NFv;F0n1aadN~vJ4va>zkV(3tgF=HJ@+2^L*FP&@hrtNlQ;pPfM#9w)qGYT?6D#
zO<eYWl0j_y<;$1M%q5ta)cQ0Wsap4{$pH!iglW6=SUR2z3!Wv!$B#bw_RAc0K}aF@
z_V)C;<EiiZMhl!v%0wKzT+w<y^#t4o!f%;##;2w}WML`!{aeeR3!dJ-?GI1Onx5eR
z`)M-H%E!kiDcKPmjLn`RM&cuq3fq1#hAytsns!905v0fWNJtp+{+#R^85yks(>JCN
z&Nnb7!{ujugZuOWaSPE<r)s_P*pDANjUR5n_XcvP(<)>KEVDpAxiMWAr@Jnj3Kz4o
zQUxppByM?1C8Dt}x?@;cU_)+geGiUdLMg%LUsuN4pYz92aa2oss1_Cl$te+r{|ucR
zj6KkjMkUmc7Y&gZ$X@dDZxmbXU?gOFLa&J?#C|6O57~P9wGCOu2p_lPTkn`0xq%@?
zYCXK^R#?gA*4Bgl{g!riL4kq%u4k3t*O)D?sv1vlh}JGtt<*c!{Z{&;+*9tL66ZOw
z^V`~AIv-;b%nxUZ!@jw+wicI|sIH*E4jywo0+;c0W;6BYkZyuDS3*JpvVT^Ce#&;~
zWoe{$0I2}9b*7aXkL>^@FO>3UC|9vWuS<^AY$)dquu4#(0c9x2muN8B`%#LOA-8`1
zyxDHXWocuhiwxK5UORa`P@v$ccJF#fVxsB0cdrx_M6rKB+^>hNn*A2yTW^Xm07-a!
zLdV~M9F)2{Rt_VBkEQah0tYuPlMlPubS=F~xftqrtpm&0s|}`LU8rM#*JUfLy^an*
z@6mF^>8Kr1jE^Ha+uHDfq%Bo7$$g@B4<CPJb(IFkzl|Hda%jYd6n&J_7-aWuSN1L9
zMI$fc1#)3;B0k4(G9O@5Fz}f_cBumO$^kP9%94dae2Z&vwFR)9WN0$!v|{(iA){id
zmK)=rDh-sqsdewAeq`4r>rBymBd-5RM45*aU7PJ_DXtaw=us@Q-(Ym$I4*-X|B%xm
zb)ElH)D(g{&zJb$pvpN;rImctjn1ZFQ*(*X^8O{n;kdUvQlO!zpa86A-?wkyW@a=T
z$LpNz*f#Hkds&wx*CZ6hpe1NoljtA+3^~EwHK0mKt2}I)p5`Yf_uvJ}w5YJ~mGN68
zB?CRZFRN6XEG+WM(j0zyc`!+x1_lOJj7{NgfxX*qK&0ubNOr_CfJoL~cu9h!_@%}^
zTKZ_Xo!h{zzAk7yR@~g#xel=Sy5MI*j!i(I#`h8K^Yc18oYd6WKwQ%?rb#&InHPOE
zwPtvSy>Yy|1dd)nC|(yF93152<g9hv&*9f31ENjJ(2&Lf7I!DVd{I$R2u)0Mbk-hy
za=B4$U7buKFEu@wwuBprjj?#1cOpI07l=E=lmsjQ3efA<kOZ->d3@{ags~5BU`R<x
z86F-6Y@|qi5G|FLpTD?!JYQ5+W+NvTIvB^!&c3cMHKM?<Gxk-gx9-mJhX5&ldLz&N
zx1k$_28@e2>iXgJ6#wmZTu*WQA8tpPwy1OlXa<4f&Fr+Z<t7Zj^@Qhk^#p#LZ!fjz
zl0~E$QMTLDBn1bT*p|7r^IcRWS4h=hkk?jL+9FCCu9x610L+1c9Sg8=*4C7}l3c8;
ztgWRl0|b@EzZG+^qrAeDRaZCF*MEU60;MO{%kx8ENA6zN)Yn&(m2DJ7LS*ITQ3V+p
z8r~mp;nog6=<IJ^t$O!owh;`Et_yzj^15;T`XCGX{CdcPk1k(?MqZiZE~{Go`tAiJ
z6%3LzxVZoj^24BZ(ko2jLM*TA&v8c5sg%Fm95{kXOw7{n<z>A4_eDm?)z+!ilFEyV
zQ}@2S#$hk{V$`8dlp&jG8{Yor6@g^&e%yhUIw?daS-#A)elPzNnf+-O33^TZkLwhE
zy22UAMOD=D%HR7Lb0nk`H8P@Q_Z7PPJHpvt;q-S24b|L@Bg9T<8Edd;U5u8=jhC4i
zAJ@~-dF6_NfrZ6LO&uaeIUmSZZIo->mY$sKo=-6K1Y{BOurCH*Napv;9MGZa8VDmV
z4p<921Iz>z2=x;xv{XnZaT!!$xsXcU&=K$iO@Bu|15<8FseUAb!hRB3t0xo~_LloR
z$$U1aYFd8$xEiZrn<oOjv;nuqbR$57pP?>4!FJa-9Y7Ss3_EjkFi)lX&uE!t?HgDM
zo<Ju@e<&pXE0$~vst7VZzNxl0YD&tFroR5nFa^~ho&dF&u1QuSre|W3*MGvQJAu-f
zx$;5EAe^S&Ek;|s{NmL7sbbGaxa{kml`QLDTFdfvcBFCp2JC`atf`}4WpY`u;|NhR
zsOZj9a|S!No0#=hV`OqIf`Vz&<8DlCeC`rizMr<s>4X~6wSDSC!vayT%4!L=A-Kyw
zo13MgnFOn))K#t?r|@;5P82cB!Os5Z()Uw3$ZsIaf_UEE*%{AamY9?FGM;lA1Ud8i
zhvVbQ&ECNGsuaExglG-g8Cc|R2*NWktXsQE`GwI5Sl>gT2})cj?&4vOx{?HvCT)Mh
z#C-dggNB%Y0e^se?b^J0hP<#rk2Qnd+_j9O<71$NfqsXU8Yf4`99)!ejkq`_&F?@n
zfp8WcfC;|y--k_Jj3y>33KtLW?d5E`6~?C~r8>ub*p?bDPK>0aq$DKo|L0z0TE}a>
zrad>+vbVPfzMhDPD4hzl%m+I=em*|$UR^~aD7%6qR1;_}hXVOP1DrEDItl?4*ndr^
zl<E8eheW_IZ7!euap>g?hOJEqv8&(7?~iA&MouUBbE(l4ZL{g#&*n}zQLV%~dA&82
zIe7PtXD~x*_;leQp3iah3;mlNO!99Gt50Z}(FeDRTxg~!p4?v16SU*ej>&&T^Piw#
z;i9~-Z&38M)?na~lamvF($?mt^aUZGAT|_j?O?a=?JX=R0gJMr(hVqX?CtMoWP&;t
zI2>M*p6Tf;F^qMR{j%uI+qaFtVlt=&_Cx>ApI@^UKziGk`kY?5l<Dcyu_EnH5}w7S
z9{CZIFUOqyG6)~Cvq!efg`A$l>m7Kf0aq!}AP|j>jq``jK>j~cyZ?IiS_a=h)mu~B
zaE5m!S#C#vG#R3t7TO=|e<y=x8j#(D6E-}ON&K3Yrfq6kZ!ssj;;oz^OOA~OZc-BL
z;NUE~%2uW(jQs<|XusFjza9nEqVutg_7EYg59GO}8FE%cBAYup*5FQCdf_svQ#O1P
z;leez1cxqA&q7wCjqhZ+6KW^u;&KiIo6G)?B6NlHE5h^yWM}a1>cIpiv7q2SfG9NK
zzBXXp_=Q#Cfyui-s<9I(r>B|zRP6O%K0kbuDW7y3S2$_4?lfHaJ2P5@==}taHvzT>
z9GL$nL|Xn3ws2A^@n2AA^v8q$+B~CJ^xDr@1U8%B=%<f8GqpTHd%QU}2g^AYxS6Me
z;$mWW%sOJq%HyYcXFhi*fR13Y`29n4RpVjEpJV5uw`Z6lqOk{RlcPm@3^Rpj9~zHS
zCcEwNIMDie!P^#PO41)j0BYM}l**vO0BgD#+7ha}ZCC<l!J(ndP~rVaNCiUn{{CXw
z=c4?fZyiz1-d=B<J)1SvkCvej+iihc*WCR4$$d;xe%0!f13p1v;ikq$@`n$FR?I#O
zep0V<1hc;z6e$k7rdl@zv1=2kKR_?ab-}A25tl#0ydTKWu72L}2mJGd4Bq$e|6GCn
zf1i9fX8reN@LLYF2N}ODHhU<wR%dX<Txs4C|IrHGAJ~k2>iIP(KlbS1itn@X54cNr
zIy~?Ui?`;94?q9&&d02&A2r;bv`Q7!yOxV=XXIx3H~f*Z-1=hoTfJskqK9;=Vr^8|
z?Bd%z{AQ-7idFr92oLY*DXOpa2}TC8ngD54_th8LsrP7DN+t5W-c8gxV>!OvYo}B{
z_fkhgLgtQFr<=<kZ%INs;^H5*5sM$a7N_}9lJ+GHN0df}#c$sb`FrICKW9%$Elt(v
z4Ce9&w3Aa3J$??j(dcS$m$8goN?I8((`N+KbXOiPX4)>Yhl)RyU68MNv^N-tzleRf
zhIONu@xj8d@$qjP#;La&{*N0q>Twjrbu_4OGwYy-T)?2JGLo~B-KenZYE05pFZhUK
zN8E?o^D=WPYGcE~N?gw!o9BW&4)q03wB<>z?gUekDc*P6r5o2{k=-w&^d8&QMm(IU
zQQ&{~^{H;|h4$wyHCi)tr%)2bygxE=cXS*|2CZ72XU;WpRWL?KXe10SR5@|fHTZES
zd^XTfdNf_bEMLAMoO=6MV@F20e!FBOc6^eQv{F>jG5<2#=4kqC_tWiZT1Lu8kIKRM
za~scqhVkzm5`w7_9(?xtl`PHX<Ol-6f1I=(*o?2ln43q;+NNlwu2^9xTn84&gn5&D
zhHRp*s~e7&zr=GH<(&jDqac<7xjPO=&ujTAMzAS%hC1+x+mNjYH`~u`(<Xj0&S>Xa
zzv9u)TcM?I(>^qHq&=M2P>y_RC@uLWpaDb?;Qk8j1pwe+s{r5E*0MJ{4K=nKp*&PU
zpP;%sqTvQDDX>*sF{Ac2n)rBlV#DMtRj_M!keGom2MA^8n?RUDWCZ)@YtMi>0C^|k
zSwK`h>=a{F&@rC~R)|zoK$`*S21Q444GmHRWS`xMe5Xg-+K4}Udw}j)lX?_$kP*;O
z04<hNbxw+ucdKe@WMKQ0l=Rmm%SN;G>)U7ae@k}(gbN9;Jz0ntSS^AWrsbW(yLZVA
z4VRD_NlHorE)`O!0#f21Ex^wJLd93F%=cG^An}AO6Z~*hl$59)0lC6TwAop?ax^o)
ztzf5|R~qQ=FDZZXemRfX?}t`9(H&m0R?6O6Z7RiSZ{`A?)EK{v(Qlc|d0V<KfXpk<
zr@{9$fEVW`C2B)k^?{^?x@bUUB$<g`Q{l>&;!I+v<+_*HIhxLpslm4k4D1H%{@~7?
zBAvFo%%373eXz`=1brNUpzH2{0@UiA9bhc|#xx<M0b<b!0wyskHr5b2cID(OVT>Un
zL$Np`Jw5n#)@mJP2e|ixYL&<3#JcD$xTP+DcYGL%YPEIt*RSiqpgltSJ>M1vM+AUO
z;iDWeVix+dAO{2*WuUK*%jsYppa_sn_$XShJ{P+wJ*1!jN(89p{EQ4}LKDYfehW=-
zP-YfVQ=7`l$^z19dT40BgRy~92CDE7JLzE6Tn%CRtSi<diCc!Gs*Hr3MP8*kKfXpr
zxiNgDyBC%jX&Z~5{jj|p{<*Ev^|LUiw9qzMINlm-@9{~|gAAJkW3}iD8rjkf)wnn+
z5-eYaLql#!%C9G^WzI%kpovtW56{w!YiMYIjQ-0PJ+L!oW@dg{PQd3_T7UJp0Oxh!
zaw+2^LQxsjYe9U|+t(MuXA06~35i<>UdKHj=*9T)BiL6DX7r$wcy~^6GMT=<K7UdH
z)aOBW1-0Vp9+sz1fspJU8R1}J!eh}xA44W2hB^h0UMHTmwl+O|9`JsAE+^p*$tTNa
zV0#H=0s=z9wdLi{1Z)!73$T|1o`V+q^@)lCG0L68!>B~yPH6v~V$_v0p1xKyOZp8t
zn<-RDOlq8ROa{I;5Y`)J=I6$~X@t+cEzXU_E09nEG$TBUepXT13E#u;6YRO4Bx%X<
zG}p?gdu<hkRPAJnv+9BTT{SheooDj$@((rRLMq<N9{M&94weGdTO_2Vr3GMeAuu5!
z0Wyo*2rSYo%gEobz9DbjhWqdCj`%<}XfJNs3-|BrV8RR^A0J#D5WwNI?(FPX!n<ZS
zf-Zs1<_SIhRH;E&X<(^vwFlm-94jbe4T@2tM?(H2&J)fPalp9t&+TVQQjj*musAsV
z;Lpe{W+xe@lkQi)Cz)8;8lT&E6OA=#jJ8j|v}lroU+(fjJZ2Y<W2S6<)xuB@+3LvM
zDq;f4p@$y|jy2d1PJM-%xm7U&MP8b-ajphP$B2%ErKYAr0X-BI1qCJixOBby`p#Qe
z!GnWvL_%cGQcYErT9qXgbQkJ&MCyoUO-mLN3Aj3gz8c_+va<5S5TD7AYY6ybInUd-
zZ&PD@`O({JzWuuy&@>%t4wkp~RUb3ZUuG_M<wq=^{}YQ-_4r-BU;b#J=-Ov#HsQ0L
z4~5>JA<^8j+=BosD&#Y%b=A_St)hex0fS*f+N8dp^mIqh9m?DiX!(oDosLOVc$P#R
zj`k+azSa{(f6Zt*pLY{QVB1yf%m+?Rxp5*r>k;(kiPWLHRrJfqlHpu6{O;pd#-?*w
z$2}3niJk$R@pkFyfq~NM>WNiVTnp5QcmX$8Fuk7_jAFPnA$@#^0mcDvkGMEF5$@|x
z5tY@|>XqgxP&h-(E-YMCPG$Iy)A<=aI-|%SIfcg5pcd$tJUKs{$#}ra#|PHLFY@)l
zo3?vkAS?qDAU$AoJU&Nj0N$2XR#$>RC`CO*7kNyEEH_tISJxBI#bWVW#PX80YC+hf
z<3YrWtGFq0YcuH!m9gD)<?$eX=h2PDe$*w5_))Ey2TPfDG+*o=?+yDWOfDRUO*}{m
z=`OCx7hDYK@#}F_c<_QFVXA@W+%?rad1wGhPsVjLc;oW<6!ZM5@WBJBGU1Ki-s8>9
zGX-ahc+U??m@0=Bdo=8Z?Dx7|^(I?d^dXJ!4|r&qiI`npu7vw;vo;K2?MhdY?*Qs!
zXd#E&I{6fFRnLS*OUSkmy^xN9UT=l{{l0prE{r84rbnx}1wz8YDix;m^rYO@SsosZ
zu#*6}WA^?%C&&mRxzX{MzV-E4fQ%WE&r9&B=doRH<;6~LH^u|vDkDP|S^(frq+Y&U
zHK<D6`3u;@etnK%zU&Ngl0{3;e&uZMbyWf@FDzV!z%coAhe2@R4vgJL%t-;M4jz?y
zm!uRY`<WWbr%#K(CEc$@Oy1$2U8yruAo}%PBzHEW;a4OSQVahqHJ!%u0kL}ncv$Fv
zx324r&IJEF0T?5CbyMJ{Gc)AmTXu_8BG=YBejlBWpI>oFiBrK#N14e%^!SO7R>pUU
zW9^u<eIEWro`=z2ibM>|;0m!YFsRV3MYK4$EVrWUI%No;Am?~oA<?csh77lrMnB%n
zAi^;6?Vko``TK6h3u@C+Go%ZI?8rxCi$k5k7<$<kQlWJ&K7tpUj26*vDbRKjJ$pKu
zR6o%Jv)Z$ihKYv4as_2uKdcSVP^eM=GUr6aBC$3%?;RWj>e%`0=&2IJU0=P<=#<P%
zeFK9kn>B@YY3MhDatV-;$q=rgkWyRATU`!9Dp<6@X7*n_#lgXWr*e}2oy{(OMTdiW
z<3>ze95j7PM>D0sY~bI0s1cDraNFxUm(?O3L_d7|t^^*?l}bzi1w1f39NDoDn)yG|
zwqsVRszjaymK5#EtxVh*FGinWA;`(e;p3qK2GxxFz9JGWa<66j^puvb2`j^n+_Aa2
z85<i5GTIk`M4AT_QzEZdKNd3+R;}JUL+5V2fu-vV(qdR`Azi{kLeWFoXg1k#%g!%2
zUPS!RDfc69<s!n=eO8>LM^Pn1rSw0s@10)VY{2e0G|Gw#|GoKy2~aAThJzL=EA63l
z4YQx>_Q+SIu%x6*-~t}YBu-XZcozRE)NF#jRA}mf6%SiR4PcV$a>!iqV2%T>NH@VE
z=Dv@O{SKH&`eWE9pa={bp$0U&goFs<n2wjOK&c9{Vn`3ctbkVGYgSKftBvCavR2Q6
zh6`?5T3QfiQd6(MiYgX3umGFy=x97&5urrLxah${1IlM)WbZ&U3;3iOP&qUf!zTaw
zsRY}+Xt<ZAwM0;%EC~XXqYrR#58;c0t2pdpG{lm3P~yg}{{&rg!|7%XY+0~=0Vzg}
z1zF#d&AtaHc0rEk2!9pER&5vN&Q<9glC&2uSaMsYrNX_!tzL`2)XR-V_GTs<lPL+A
z%nYRis<_r>4F>`0RJyl??k}CwQd3bd2!6OS_|ZI7og7s)f8CD_i}t}k-28pUHA}?<
zX#d@*1hFb)^esSZLNRpSLQ5;Pl%E*L6_}jpCP@NrKprYWJsev^N-Dt9vk4e#9eRVm
z{BEjh#lgYB&6_vpEu^IbVH4_aWA2f(eF%&Z$U^0KOpJ}wZJ1eDbWiD?JPEd?vZL9E
zk;dR!NzX)2lUME*bwC8uB!!0$!oeXK+I`tCIEd8ViD4O=m>h0SLCGLLhgA1*s@&u7
z2&MK6+dnMJWVpQHw<bnMp@S#11-@+ODs_M@%1mFEEY;Hp<z{TLaHQ;hR>CG1nzkLl
zj)9~CiDW)N<-;K*#Qe_DWIiCFhgK(9DvNo#6f2~T+kxW*5I}m>3z(Ah90qi)4ymw!
zJ_Y-Rwz=3?f_7=snffU>H3sTBD+2P&6~w&u=JxiSg4ob-3r$dd6rp3~bi5=on;Oz%
zO9K)e3+w5fRbxs0*^P3Vk!iHV{>3tHTC`d-SK?{;Dq(Ab`T9~yikO(V?W1So|8=h;
zR+Fc(C6-b}`K26kLZc<}w9qKqzsVcK2K?fW(4l5ztSBj=KuZM*5vrz60E=#Z0D31k
zE9;dwQI+6|>bHJR8Ur7QOUKp&^8>|3yfFUOlGecCf`|vW$I{Z*1{42{sca1ptKv5?
zGKvccN%^TWrsD^<t$6J%{JY<lxia|fZIXhvb%wV1=}T$p$b8{}fabYvP!_b!0b%`S
zBvUtmb_4_<XRxI}-qh)u9_J&G@Jd0CTG8c0QUIp)Eg%Zz^JQogB!w3S41L<(JpeLt
z%>DJ>YkH%5rG%t-_z=86B(OyQ=rXES_7-ReylD<03^%2V$QPhvgJfG1c{9n>9(wjF
zb=pE9O@iR}?c1|<?rR>;{)>q$o^W0nKl#7;!1jG`Hz>Vv8wZvHv`*5g*GAI;F*yyI
z6hMbif1CUL8@QI_qalVee9#b9F|sf+ngQ=soREu@Kmcwt^8(x)v&+`nNQi9b|DqtH
zTUH0N2Y~qo<`3pIi1srF_1Hb00%fD$lNfS2_)0~kWBr9Uw${u|UEN5_Y~nNHRi-Dr
zn%_UdVJ0~DuO6+YXkX%PpT5`slR!t~oBdkTTaX%AF2jJ``}B2$Zf~-nWF%b-oGXKP
z8Ov&LadtevXu$rvH~WXVg|>rlzJ3<m<k^cAQH><<#e$4$sITuS2M5;8n<?w_P#z8p
z3?v{RfD#F$Mw$Z$P*_oAID}J3{3sL@6``Fm3mQQmJ$eKIlI(V~);5T=);Bheo$G6A
zc&gMslKCtzF9YQ>WaUzwlViBp83U*j9<VQbg-h-^VND=_`d9DBh$8E=y1tl}YwokN
z?{<8VfZqi}g--3}A3s1z0Zu(mPNz_V8yKUzonV!L3hU*{H)h-3)YSB<s`<YsOVoH{
z!E01KFNn1e)?c#ER9a9Z_2@_Y_rH(g`{K`0B_HwQbq>>1vh}}9JE#3FAz{bdHbhO>
zg}b`oO@l}U^r$ftjFx|DIW(paDz4b60ftzf{TIY+TQ#?M1_kwi(iniwCpj`wZU|X$
z3X!;^q>*c%^ybZ<9BdHtQIReZJ6@nRX%}M8d%gpF4bba>uLxxE&`v=q%2S^Ws3#Vu
zJDPgebHq7N=r85*fy{?s_VCUX9=b03iM@Mcg0sR6H4_%%Ka5vb*x<arG=u5Z?EU|O
zH(>@Zzl1E_-25-Jv;PYTVJa*1FZS8rH9&|-xq2)A=L2$PFAV;HMgI?2>F=k*7ufyJ
z%LKz4<qMQ_pI0T~R79q|k&~mz$-nbDV#WNq((|(G<lB48txLPIO8>m+y!NxM{gu*i
zea5XMdIlP1W_Ck^@$cj95+grEJv$tZ{>Yl-;psmA=i5P`Of@-`&Zst-3+q4Pg}VKh
z{6s;Cc7NKD{N1D2imfw>yKnrPqu>0?0d|qH9Xh=?G`yX|7Qdc{cveDLcQ7<Zf!af$
z%(eR7;4xJO1amGPp5whW{}YJCSJ&J{_+U6Qmxi|TZpa=hJzvRlllqmgKREFcYn#ZA
z_skt4;={u<j=1^iN-{G)J~PR-u(Sx{d4HEM)_m{8=OA%!01t(=+@#^sBHO8le=B8X
z#%+n+zK3+U1`Ec>pw*dsWyY=jZh@c3_i)^TG_x5K#p42--&}!#cf>ef$+PHAN9{VR
zm1tzzu5C^grNx=t4!}BLPYb2eailpMd0pZfRkec}-soH@$9iAoL&e5S#bYXJk^;wQ
z{Htrm@V|A+H(q*Jf8Sx~&k#KeA4aC4Teb50NJ|xSs}dDUu8)x@Ev=5L-*t&6;+a=i
z(#y-g5R~%M5D$sTGnEKVQ&m+tB$6b&LcO{<2IjBdCb2Uykt}U(jby3>jaLpYqV%+v
zCh9+kikwF#51vo=vC@$6yKOSz@c#XKb2fWk`lgfUh56sM+Z6)dK|~YNQv+#M700))
z`TO~`M#;$!=yk;`^;N~?#<lp!Ubv}N)0y1GajkCHnzh~F*<Ta*vtD((m{`5xVx;hn
zw(1m6534nBzsCwBul}#KZ*IKC>EPJ<8)HMf7l)Cw;a669%f<C$*K^czqnT`Yo5DW>
z#a6FvzN&B(OjSDsC~>TK7@I++vWL%<rFJUq>5VSn_Dk{LCxF*Vx1<CvyAV@4rH%yE
z$w1=EB0EyjrSeUT2^ItsCkNdbD;G}^lIKI|BuyLv`i<v3)8}CGm0<&ssM_{29X2&2
z3~8C3Xr8T(K$7W=DL(z~i?4-+V#CVSzJ)gx%jhZ5zE|bdd`%2bZR6@|Yu=d8_K|Tx
zsjhZFKcC4O9eoU6i)fbP$55B{;IajyMqN&(=2nB1_!Gt#;btBu9bPaQoz1h(Ttv>J
z<#lzbEEc$_nll3F?ejYKH8Rr;*9hW#xCmB)awZ>}BVa9r6v^_@L-%8=-AvxbXk#6q
ze4VwxAq_XDcb6oHqu(7~VUJPQr>mqSCD)F3KW<?W<8$jUjm`fFNvp9nwy`leTp!#0
zMg=yXd*{xmh;K7)Gt>8~-R4frCMxW@x;g+W2GrPoVo)naLtIqj?|0d)c~%(j9jt~X
zB$U|AAh}&Wr54|YV1rTYdbTuVbc*HN;Izc9IAXiK-E%F<nMzS`s7S9<DLVG-^X8jk
z4tuBV)A2IGj`~}x({(B$qdf@hN}Z~HS<1U@YkQsP>GPdmiBO#;5~phV1E}<mXZk97
zlK4#YhKn1WzF2K~-Npqh(wd*3GBr7VdAd~s-`v{T+8c{d;&FYNwvk=M0hHL{8ZPmQ
zmn5SK_O@nvMq9IxFcNpiaA&@%)#;BMKvps3bYL=0RO_O`v@3Ek5X`Si*e-<9cR#^?
zp$87s+|G(o>hPQ)O=Mdt!Q9+z2_ktU({OIvw5|PVO?&ioOVzvMj&5=eJ?<5&|LZ$G
zi|$1X=^=CadrIJgq6(%gv!!fs`5IJ>!8>f;MB2Wh?2M!CYo**|_jr1SXtQLl>%z}K
zMi#5_iXt{9)5f{%_^*;?(lCOG441=S0w|Q!m6eBUf@CJcJ|Sj@Q+5N4LU3>v{Y-*L
z`6g@ZWOnw3^OpHiPJ80{Zy@d=e{n6bwW+Z+X^VpL<EJ1Zr^hFZh(DwPE4vO>4@jS<
z$^}YR<-cSn<zXN*|K+TdtJi(;lKVv!a>gVq6t1L50*Z;0%OmlkqM)86o~gzu48-q?
z08%zbqF!T8Y8o2pmlERA;%v;U#uLL;0cprh1-q?EoAbX9^yjw%1M{e9M@sa$mja3S
zOf6<V)VuJLYgFUg81*E&EOm=A`2hTEVl!$8s68Y6)P&^j-Zn7eN=%6ng!1$)sFy}7
zlD5uk+*o8p#*u|9*~7ii|6MYR0Rk#z#YpZztxG4W>B`g`uHQSHmp#XDx=fI?<B~&v
z&2Gsy*kS|GYM#%-q}_?-B;WVR#G6pdymoeS7Q5hxWPi`+#uBCyH(Q!j8*XIoN!Je#
z_{rUCx^KmzEx7t1&(_mHlzvS+##kBZ^-r+TTIa-(=3w+#-Y08hN)tOBESnk_kmoHf
z3@JGRR*47>79+VNBo)vK7eHW}k!_VB{+u}o1#M2P7Enx0JtZfX?ar~=np^6*he*!z
zXK|q6do{m)Jahe8ioaFdou*HC_&FuoQ1+@1ar|OsYH2A^kw4uZRc*5o8-2f(&UGYT
zox8f4!*p^TWQdNADA*sh>fgwCYBWe|RA>$~eR>Kug$S|$4sTyEO|%j3;$T76=Z`~I
z$PmjQ$w;_${X5Fj^>aIhsRgk+Zx*h$^jJ6k1ft8<KA}qkJvK$U9ZTbwlY(i_`PxQ3
zDw2c6!?0Z~zfz(+DPk$Iyh6H}VLR&{5?%}pgqU<Np1UX?eOf0u;@z8GKltiX$4k>)
zSt@btH+o`Nc1x;CVroLgd`|7s0zRYn&Co?GAzsv#t1?uCIGVoI>xp$eYV8AcNCJ<W
z<_;XSYx?JRhQ|KJfK$g840`2g?CX8QWfZ67G+sv9V(sfc?69A2rC6;nytvwguSqln
zJ^5v>o%o}!yK`kKO=ZLXRFS0N;s;q~iRZ7n^xBRXWl?%%Tju+Q@3%FST?`FP#UBkI
zW+%G`zO|-rf3~$U63{>R%vV#c!FegUT#wwp^9f5ky#+cG!t*7oYRVdH&rx(0^Yb@0
zr)yp~oKpIr*n8W;+FAH&SX;DgZexOq@D*>H&)$0$@-)pG1D`dyynOL{vw-XQhNYIA
zrIZwgRcK@X&dK4+B~^X>%+te5`F+sb_usHb*+#`8h)xK*cklG5JD~;{^@P2$@$!Q3
za%^&pUL_V9t12tq#$Jlt60RI>5mLev$rUifJ4ocI5S+dSm)_bECsSdvwX(pX_XAr4
z1G~XKVNL7p)b#M50{!EV<DbMTDwu3tBOJ`x>->x|KkSMtoH=lM;?I@#Hh8XifOq5S
zL4uodnIV<SO5E14fD3^l=q2Xc2F&VJ1e}J^i=F#YkK-zg&*%DeJ)V(aqivp9*&R&w
zv_<h>8h4MI4WC;bs!K_AL6#lm@<1d~<!$s#&0+F|UHceIW`}aw!Nk6w&1evtcDGQ5
zjdJOGNKdj2`a`)&^bya@TMs^b_zpj_!64_5a<ty!=6a`^8EDF=@g)iUbfF-wPf3X@
zJmQU9b|T*Zm)*%=+l+I?MO#>=4Wmnq=E+{RMDs{*Z#2J2d+`FTz;Swox3{P#;be7L
zQ+6R*h*)@h{LX%?nXXa4)c+nmBkb2~>wI<$$QU*$&tSUqzYpI^dUTl*Ca<R3iMuk6
zHzUd5D4WeTEj*x{`A4=a@5RiqYQA@LQ?^*0!Fahq<A8y&KRDZzUVcky2}%N}=;&lK
z)?ik#m!ZUxiOQgq7!v~xhWiV+Nf?eLtV1O)4$o6+4nJ;c;Fo%n`9L|TJCQBHxvgbs
zAa;CQDo$8VPL95_tKrO&M5*3+<#$Gh;^G4Em5qyW!hRa!lqwhm^NQ-~_2zy3@sp}h
z{^(ikRxUZ<sft~1kvl1qMWCs&LW9T@GAdTs`0Nr7nSIC4UKdoy!|kGyvIGvZ?x~u?
zaOck7mc;8DMC%*Zg)}8B#}nK(1Bh9KiehtUhBE&KVXj&UvbVK+xgaUK`i!Nn5dO>$
zN@o@q+P0hHJesv`+IU38x^sTX1r^uLN(z~h_>6v5@5hCUX5aTV&kOP@X7U*hCLLBH
zgqazI#P)Y;itT!C?DA25DE^*%N~8Yx!N{M9YEFOSI+90vuUC}=@aCI_X5m=XbSnO0
zn{^#y<H^@7jRl<)G!}COS&~r{6<)@o)V)<#%Uh0glz+xESNr|ATcU6BckcqVjsf4m
z>Rh8mrc!;5d=y*+aG5;ToBoCmNpCqg98Z*Q@mek;Bs{vW(tu#QaF8}D_I%9MzlLZ~
zt6g7Q@9iFXuprDY60bT57{GRG5>!fFhRIe}naTXr!T{xU!O=CvO66~-2RDc&*FD~4
zku9tK{axCQryls?-F7-0ymrRV^EUxN`;b-7kAVm@G0#N(_dgI>MgJu*_@8tItkgEA
znPyY9|2u2qK4v}k|9y3}S3g-{GMA1Qsw|0`7UJRpq@*S$#0WA04%S~w@Ck@jk}3N8
zw=SwJn$lNXQ-}e1UxDvv=kcFEQ}KIV(Pa<y4foGlw6Vq_OV#4ZAOs%`bB`-~d!KZv
zIA6U~FgWO8Qhwe8jyZhrzq^S<h7%S8{n8$fCTE1O%_6gI-b4Zt#=$@6@0Zz;oge5T
zgcGz8A3u`0&zf`Ua8IyOvk8iY87fWu{b<2pES@CO>6sUll;0e}sw_Fd4_kO}qw4)N
z&Tr(?bh6-{D)d1E<)57FxbT=MQNZQ6eZNC7KX*hyNX&h5e;Y+eh!x|m&@g(oNms-M
z&!Oo|74D4o-?6CGuHHYISHsNQaSihGYY!j9^?Ke6l1fkp^*(FJwOcw=mQCF}b(?qN
zB0^xIPR|#y=NTiBu~4XOhV#A>IgzU{RqILMlf(%mVmkq`cE9WV$r%=Am7DT3yWXwa
z{ufnF0(=z0WY|#kjfpGUBjve;C@!(OsI^TiEc9ua!^bC(jw5(BoJp$~<h>EVyoBPO
z;g37ma1ZlS8WbcnO^5U1DokhzS8@T9*E?-Nrxpp~?b~R(t;LOZW{)R&fBtm*Uinl<
z-2O*Q=ht<|SP|Q`jiIWZ$8~SJiv+E&qEIkaf?s<(w9j7bE4~)g)RdH$Cry&k?}^RM
z%<OpNQGeJoBc;n;HREZrQCz&^w#nBI^!@5&UvF=6jzn<i$?pV{x3A2|d_~X^TE5Xu
zzYv1KD*X55OnoN}L{DgUDkJjv5MnYJY+F+`q@<)X^>s$Ts>i(S*cX11&v)-t#h7Ok
zIoby-XJ3&{-t5WRvp+s1)z#b-FV1?tPkv2L-`lK1)d{0@Q>Ue)NxaY0-P-!6JHe^n
zb-8^gjlYgwr8rrZkG3WB8zU6*T^p<yqF9!;y=e~qqXjr``CM9J^!(AlQnyBFQF$zj
z5e-81Y-6Wt7q{gz34>HN+$@&d<m7LiUv**ZVqS9ih;R2K9<Gihv<$qPA>TZ|nzQ_R
zABFmAdzLDU)`}UeqhRDg^59GAc5&u^;|1o{2q@H7*c~r7dAjo6Ob8?dhd;#e=c%lK
zDl-ZK8QDc92y<Q%W{t(=qxl6~gnN}z2j=F+kuyKy%;LB~Lmf^k*ichb9(#-QOu^Pv
zOHSfVguAoB@+=a1LT!jIGNP=kY`f>KmE|t*lLoiLFgNqTbQzuf9+?{6qgE2i>y05j
znK!-VAyH>gbLDijo)`{>`{`)2vh$~pqRQ}oLahUWUC<RH1o<QL_T|N?G|<yq?&R#l
z3W(m~jFn4o%!eA_^?L(BPcOx%rZ$6go-PIB>zQ8xmJSW37a;qm!M=s`?zrCzQD}2Y
zV&JtPi5r^f#~2e+M#QC+RTL6ZYCaKI*B@)xv;crZW$$o>5XW_~Ti|eg!vtwIspjzJ
zha+GyL0gA3J5XOI<9%*<xam2JdC>1YJgfl4Fcb_T?0cs_#>as+B_#MQd1F{X0Z`4R
zX?}4Z?cFyT4R<0!V?i%&qEfcAyZ@Ax_5EAotMxyRyfB~5go=7vW~3033BE8!^_aJm
z>kbC9ePUSPIA*(wX+hd3IGXyc?G^2Qysk8G$zhGAgo#kA8QIvjH)kBLQQ1O0Ol=#m
zT5>yIS(4}Cs)~%n8TcYi!li|$v^kY`cw#ei=FEI0pKz|%0|*Rnh)+m}{H-IZz;<)`
zbY5H)4s#%E8J5p{xwF5)sJ~HvjvAD7q?F)XU0Hb_@lvJqmGNENq~L_Gi43b3AKary
zGrxH-Bob#7*sk_Y!-Q7Yn_9XlUY_?AX?Esl3BsQ~))vmApL9IrsOKE~<SewAP9qG?
z{99WShgwNz6S<!_m>vxhA@aSwtu2%-e8-2of8}A}^B$Wp`B7<fUusR)GV&*JZ2dNC
z0B<()pP!X1hwjI<nr7t3p{IyATUHHrmq%!oDX<S046p<yaq61rj#ZZl>-R`2E61WQ
ztO|}y3>k4G@yRUr)j?7;#ad~$@tt&eq(K{%BfGIUG6;|VV=y+ZTZ8l5&%CE;pWMof
zWn0Uq5Pyb>pYOZf3q@8^Qbs(JHYIV0jf=xV;4JFRXM1^7t>;osRofnK9Cl1ktKGdD
z2I;Tm#rG|YP#L?G^*j5A8?YvHyJ~FD3-o*LwVVd^_YYreXBN&ds>QJOHE7!W6Rg6G
zMlO|4ZGtE1?c39R*EmdL*o?RxP6m21+4C=lo13|wQakLA%q*|19iEnMHJw(}RUbIo
zUzTfWnXkyr``!~Un><phDQCEO(>J&NLiEK8{oZF9#oa*yiK<mvN>FR2p`nGEN`r+}
zYk6=NqR_cH2apK6nQCTsc1(D<^Y4~0S6AeD`9{XYg~dg+z_SvDB!%Oom~4_?@llEh
z#Ny)Oyq~$cx`#2b5up)76f$CHh)sv<Q_}-~BW==4BNJWCt#`S(X=$bRb~qn%kRmpx
z-9#cI!o!&-WZcs?8%b1|gBG9gf8c)n*q^tsv=jw_sd!-&>qq5@f>B&nmcXl<Q&)GM
zEa-VZSJqlCN<uQ}Pb2xW@3y4wLS-QopQdUq$*8F4DOhMJSk{KtM()A!0Z$aMs;5Kc
z<&eJB7~x`IU{pOaHO)8cQv12XZbH-+(;(gRYR~xmp5clcm!*rujc}ay+SU2NpRK*i
zonvr_>F0#_an^A015Dd8x97S`^X<0L@u4V)I}P!VJHX1AQ2j6ecFZI$(_?OH5m`gX
zUBj8#k&Z#DNM~ndhm5eZt9U>g300+1jRu))vos=1#K-4n#g+@jn8cGv$-*s|@XE`J
z-Vj00vsRio_5!1gSp?!yY%J(CiI}Y^#~>usXf)J2o}3_N7dLV47W>j{&3;v@sZK8Z
z)9Cpk$Az@-$B)Y7<if11(az4-8Qug0J8iln`3YfR^1C~?<Ob9i7jXAB2IMoo8YF87
zjOCHNJl=DYaWQSR_G<d&yxqJ6b%d!R?V22;nEnK^kHuRQ<D+lkKx%G2z7iWFr>dyZ
zva*wdm8lAut-a5kg@qc-%*D|iL&5>m_i-0zs32k>-f1cw=7s^|5fh^XC{RHm5)m8~
zg@pi5AIto`gH0lbh4Ew$WP*cjuLK^aA3ykI&%V~0H}aa8nADnHk-0uO_KvnEThOT(
z5+2^ED!|5;CmD5_7~hxSuP-5yvp3jy(Qwud6&<pyiBG(?>&xO{nWsm4R(57HLlYg7
zlkvq?{7DULIvr;>Ja`eamZSV^JzZrcdkIb!m;7tL$KbGntl}IRYA$%c$B05Ej*j;3
zwjM?@d6C;zEMuk*f>iJb?RGci)YZ2YUzdu#mcl7s`O*LVA<QKDjW;T|`j%ZP2CKJ;
zW-U3d2VZ6$i9S5su*w&bd#=RnAop!v9LXAD4E8tjhOpLOV(I=_oxk12m%_qg?zpMf
zJ#`+%mb~pB8X7urIZ1rcK#nk9ob=k9s#aD~HupeAK&~08))$eBs%FN}Ee+eRfFs^S
zuD;#o1nstEQ(NMfj5<mIq0#a3haC>Q>M&%wx)C1rt}X(<5Sdw7M<@GFFhA{z7m(;&
z);aBSo_ADbWi0~-aP8Ua*8nY7hDtuEsybM&F85_qG|>&Zcl@$7H*#C*RvW`iNErVi
z-~<xyG#SUk(DUFickV0P*^26ykY{2j5gVs+T5<PxIIF?<vuPRKAYHRDYkV}&YUO+X
z-XKIo(%(Fr32r<j<%OR4qy(OWmkHbsKlif3*$D~ofW1sWMXVRgArSZOrPaGQLM>^=
z30q!Gt!rub^Jnz7MUsR`E+QWnB2)y(5J-rpbZg%5)|+oLJLF_$u5Atnx;%zg*z*P`
z7O=FsaihA2G@-JL(uMeNeap@RBe$EMI)0f~svfH<(&g)N>MA`{d~d2r#n;^}fr|Km
z@crm~P<9=0<SYfiimtw_Y~iy(*k$0w60p|SJYGr44}AX~Z~X9%B*WQ>3XCUcX!7#9
zo;6%_{@nm(1(A=Or_-WaDz+wXxWA!|k~>k}-1}b8fb0%DqlYWMjzdA>)aJBPq1h#Q
zmGmH)&IE&8lDqH4fvrstkMZ0xlk4R2@`N3h^CdFkt+MiGQjYPWGUNoWPOqGW0o6b#
zhNw7=I31Eo@DTC7cmML}a%I46YdY8I5jxiz%HUs74F4-<m(xpUW;**0l(FwgGTs<_
zgk*J$0<LH2FM>?xE!O8+`iI`Ed{M7bsgfsbi5?wQ=W`17dRkP}-7Vceec$Y`KlET@
zqReat6Tw_k{PS}Hr_mxScL}sIa6)NcqejP&<v{A?`rIW3A^F`~x(Z*oR88-l1qX;c
zB(Ja!Cd?`;Epxlv80k(VA#LK(nOihL6mq}ESEu`%c%+DsVZTgZ&#hBA1-=0Urh58I
zXXblHyB~j$29c5{MUAwT<R;{8EK04xkq@W5<QBTUBmT&^=*wHk2v^UG3X5|)b@ha{
z*9BqvBae#?k!BAC7>GIT?K>7R@;Ao$WyDmAEceeC1xT6glZ}a~`$HKQ8Dp8bzNN{(
z_&~<+;=*C)^-&eeFZ4*a>kS;z2tn-rf-Va<y(nseQhw30YMzl|m~TROk&Be{=F!~h
z-*l!oB{$b;efbw;>|L=e1<A?1onJGy+%I0Z9jPBkz5vjpTxBSwpirKX(LBjj;*2`T
z_tUiR`!Vd4j|2pg$b2$`NX1G5H7^Ni?xT4s<h8Z8mzS5vLPFk4?uo~IubA@P?<0Vo
zd;zD5VLe-uV^$-9d7WJ7`kmp&GYoNkpIvrm8`CtKUIhDpkFQ>GW`!oVi{G5vi{hh1
zvuC#3ie_63M;HI(l(G_^q}6^W@;YK+%pnamam3yenTlooG+F)9^f>Es^DSGj7#f55
zO)^eZ=>)zmwSSXKl=<_Q@vU1F>`UR&iZ6Kh&La})kb_!`#$zOs2I)e?e%!TiJfT%1
z-dLpP|C{fMA@8_8lJ0W5v|j2a!6N;MKvEbUcIMYup-3Vm#QX#$@UVo42c(w;QHuJ`
z+W%z8h2-1E1BU}l(=EyMQHo2Sg0!}#fZ&f#FDUrb_D?ZWj3xQFKqkyyd&lpezhRZ*
zCw5swxdjTVzn^QkIKZ+<Cqn*rE`hoZ<Ik_VTi2Q~*odU3H0IE8zy6ez&@f@BM|*S`
zj>BG#pQZ0oDo3<^j(f`!6$WT`W%1|!r#5m;SwFWC(nlkYW%X_fKAYov|B+0g*0Y9r
z>?}|Ei2#+42$LNH9==D&9p?l5*NXDE`EQXN9SgN?<a(?=4ou|__X?2k#Cnl(H)AJj
z=`>G2Oy{`EH5R@h^d8^@g_hLEjw)_%$y(xgKNohD7_Gd+%uGggLr!jXL2|S<rrLal
z|7ceR4I|I!jl8GjFs{VHoxyvVzHo`h*2o~nY!iUv1VVhMLX|_Z{Hny2mD|2ZUE13d
zh2=N3)3=6ZI`iAxe=9S(j6l8Nk<Van|4i<#JT#}wE}<$b&;GtvVJbU9{RBHYZYwY6
zgmDh)>P3a4DzC_r&+HD;(;v$8XjfNHcz#Ho#>1ymGYQQlA|TK<tgg}R;Jb6Dhfs83
zAXBpghlvH5g#``M`;YnRcZc0~v(MgE2D4tnB$l&@@0aWQj31?3I!ez2#&fp>8qnzT
zOc)qGnNCS4mi3hA-<zlq_`YNou@TmEQ@xBvLeb7?sk<dC%wLykVYkB3o(LqyRG#A2
zgp~IqWTY7$5my+#3Flt>5Z=y_ow%MMkFH$(9qYLa0_~Px|IpOtFAF;AY#%!DmvlQj
za1?CSP{*2xmR88ub6Q2wm9?!ud%QoH?6~cou$z50fH$8gT}dh}y;~au@ZcL%(tU$M
zXXQD(ggjn7!M>G?c=qHgc!0HcIpRC7e$iU!?1a{D(!KTLumZCq5G*@5MCfoKY?jK(
zM66jgb)SpnnoNqJdHwhq@UdXa4fg6FJi&NgaWpRx9m-$pFOAMBEpP)#gTfL?cn-jz
zMMJlG>Y<9mH}^kMahqsfWoyGzkuKu-1WLuGQyYGMt3EzU2M4EZcdG;z+S|J%<ioIi
zVb(YvJ`#B-B<WusOy=|9n%05wLuNUQPbk+_<Dbr9WgAT|I~>fn*{2k;Gc$LM9Xokf
z$>R_ZY@VHN?Zpxlsg?y0Zox+pa!haU`rA)zg)T2!)8@WH33~!6yF@ZHfd(I&HPhQ(
zWpV6N<ZHQ$nA{<8(lQ7AvMBUwly2CL?cGx&9PxB}+czn0kL1;C%-wsW;?V3Dr>mRl
zu$#}+nb1gF+(l>*3qym7^%D~hdpdWdhsTb;wUh4gRg$m3zv`?^_9W`mI%JzBggI$(
zTgdU?jG8<pn9GI|?RSs>nxdkRuqT3yGyXCsBJ%Ea!SKkt)zvS%Y;112R|^<eJ<nHn
z=_^^1;o^FrYHIH4`X-1(K&|fN14Bh`-<X0j4JBG?x=0pl5*Xw?3M0hS_pg5XJWS@^
z_2>NbD)zgNp4!laB;-vOo}BRb`Fq`RG>uxsE@sh+xja7lXtRFQ^t*Ya>3iZE1qI36
z5XZ>KTQ)Wo4cktBxoesj0q@2<o=By3T;7iufsc}t<G{|9VYc=S`PwR#^Vn(SiDx83
zr~n!t^%4UQ1#yB&aYRSEh{Q}Y`ev}vbk>0LMQ!qnpd|ZR?AnxvXghu1PoqZ`o3+Fp
z9N=g(BV%KQ*U|Fy_kBJ_gtKYXo$x;VeGKbnZ~s`krJR_py}9#cR51u(%x4kk#J3*a
z!6Trl@F1HWXGl#!x`y=Lh}K}Jp`mHADx}6vAY9Bxt%hWNQQ=@~tCuWxd1d>5vG<-)
zQFYs!uNhDj1Ti5ZSp-2ra#Dhz0!7Y2$vNk!0+KUGE^<yHIZMu<k|jybIToSlS-$5z
z|32NP&+R*I-!blo%Wwz<Rl9brz1Et~oNKS=clNn}n}7+xOLfh|YY+M>SSTr<w*;Iy
z(}R4mJkaM+dwh<T)@<eWg^Mrr9@47}Js@UddB%ROPRiIADK$%bg@U3*wQ`S%ntK0e
zoz(MFv!F0SDg#lqc{F<->-f_(V+sQAt5<CY)a@%y2p*NWNuj9w3xiXR%yzZDbbWhX
z$h#I~JTB7P*^|%d%DdPZ)!5#C*Y&hSx7)ALbzKO5CV4h_oKIC!P9`>9IxOPinI_b5
zY7<J9C^d{FH+T0I<6e!W?$!smvvT`lP0e-FwPAeHQ#r9j{LaqjlplAOCmv1Il4fPU
zJ9SwYQ!Y_1bKK80IDhF|G>e2}Vxz@1ES;{UBJ1Uo_wV=W>h|G5>qbg2lE?VkJ+Iun
zoA7B`sqSrVUTk06r<*~`$k~swD-^?m=>tB}k00OGUs$Sts`=m{cG+kJBlX%?_7Z!8
zC)Z<DhvbCk8zsnVh6Bf{l9IU#-z#mI!*6b(M(f;2<nvZwFHQCI^tN`@9c9(W<F$Mx
z^7(S6$7+T;X><hMgZZ@$Mg)$$s79OpHBa(j@J3Ov9(|GN6uQ)=#@U{b>N|tSj~FiB
zUsGj+EID_7J&?$|F){J!Sfw&^QEDt#9M{q=gAR|F`h?f9Qynx6c*|GW6_gei^Nvfv
z(t^{WW=he+{3ndnPuQmGCTefZ>VCdCEB85qiA}$^*=~2~ejsVj1H}3Bbt6Z)L}jZZ
z?;5eclwv85l-|90%Psi2yJv0qLzAI1E-t1le0KODp>?AwsNPK7&V_{3)B;OgR}zx(
zokfzAE0R9y&qgZ;_xmz+U%srIS*!f;L2Z1T=I+*6g)sG>D%E#$auURd%`DC-D`Q_y
zVwkDH5!&+gzH&1InRDRXM4&sru08Lv@%!7%jE3cQwS~Ik$d|;x_zC^;a=M!#xkW`K
z-7&Va3qQ!1#&6ucvw9iNwZ=4low)YM@3#5CIQl&Z^ukxT>X*VPp7re?qbN0}Rf&lD
zRq~bsby8$%UhQfzl4yUIy6N%|W-aIS@v~CFh<NeugA?X^MpHG;F1W_D^WyUX4s{OE
z&>jla6~ZyB&32?v#s2$#bH4g0b=OEylwK7>_rOVY>{BNEL&Vj(^{vTv0vz*Ajn@(q
zecSVelAL{nAQ$ZMP<H!>{e6q(syp_lgc8@V?veNh-n-*xkbuPK5!inC^PXJ<U5;2y
z^hjq~Z|4qX``$QV*~7Dc)XM+$rp2FcAM(D6c6!e2*(=g;F?%!Qr+kbr<3}RWfj5dn
zLplk_-5;;b8~4>8;@*d{U1xd>F~}BAcOhD|s%xZ5ykHd7g}NQ-hF$kr0o)~f`Oj#t
zPay~#^6Q;(S6;q+t92pX`BHy>p9h`Xd*uBbg*t<TL+|8_z-Ydcp#$Em2(E|W!aT&R
zRm4g?)NNZ6&O;zUo-y-|&(HF_?mjfoP;c4_`ts0;q4+SK;0B+T(J1zvJ3%4m&OEVa
zG8)I7a1xLdL3ftLlU~3&f2ObROnjivz?Ijt!Jn?0;u2%3j1&{=WWin&=xieItjL&@
z)NrF;JAs}pPut@MJ|nkKzbDTQ=AFpjzj2|W0)Bq=O7uim*Dan@Pyd7OmC*_z)%1kt
zZ%t}Sv$9?)Ny*A;zAGqAc>Wx<zVhmo5E9v9WnGGOQK<T9$R16!fPki`m~E+1ISu)g
zeM5%R;WMM*b2>V2buQJ_;olcmh4Vb9_ei;Jwh)vx9roErr?adtsmWycmA0<@0ZK?u
z-`LWe!)jTsBFh=c6EjqEa-G*Tdlc_ez7kPtc4f8Y{B&De)7OwKlrj%@0L+{rb(9rX
z34GdVXl{iDD1*p}vwrb<Hbw^GDlNp-$;#fs@e3D)7MD!+H`qUi$q0U?q`qlyANRd)
zbZOR_{Aq#606Ls5d1u$w&f&ghBuoG|V%S~|fyn#M`lYpYYb$9qC!#Lh<dIx=C;deE
z_6tb-QHP&Mxo^ir$iMo0Ucy#Q`Hi43p`4t1(J#;~5+X*UA_s?Pd9ebN*Y(D>Q234D
zOM>q1qUJziTi8&}zPMPRiD(Htl}#IY{52d47ki>;F+f70(qzndIASI%m<nVJ1m6HJ
zI}_8|c>3TY>AalLoM$XJ!Ei%xlC3E}C3#Vi-W8mW4<A0z>7gd4=r_}Q{;lZ_GdieZ
z&L63ZK>FRfY)C=T8yi=v8_A`em^j&8dn%rvdA&n+q-+!OR_4<rpi@du7`y|-OTS*$
ze&TIGFEoquabw$8p>(YTo}u4M^0mfbErGCnap%u8@Kb1u>t`s6C%<aU)pCFLD$NyV
zr~1}<&^kv4?rm~v1yeeTQF`N5bhwtfW}ZgdFE7VbRkgceX~G9w?_Zt=-KVAVnGnV?
zDIKXb?_>P14(EziyKs>x#I_w7k0UxPNFrlr{IG@8J)15u@x>YZd)==6uixN&d;tHc
zs;13VR9tvI7GG3OE}vhsFua`}8+)07jh(MH3q%2Enimnu3kybckBJB_NJ^2*L?h&7
zWJ24ndKL<eMJBbS<9sBK|NIy%og+EJTf5~<6`W5<nwt18@|3=@#2&s=(0S_QRL^O1
z{BCn<`Q5voM|9b#{UTuh;{ob@;(({AfA`C$mtR(%hf)8Sb@gdqhPR^4B=(Zc5{|gt
z<};e&@HHZUNpF!HMY5>4?xbYC_LcePk6~)%@*L5W9o5d?*UIPo@3Qv9--11Fgn_p{
zvdMs$Q{r8kAim{GH9HGF){n`^u=%C_-&8Q$I)!>)KGr>Q+03>!6cKqJKyvmBlE-Zz
zH}i@s3X?DmJ;lEK@w<dV89u9N-l^5I|K}RUQP_d?9B#>-+j;{n7-36mw6&ex9gpC&
zq%QdSwRxg;HHw*HbHefZ1(Y5tV;><OMnp6}e`8jgXjYeKUN_?Ayu56f)Id?mfO^%?
zQFmM$)-~MDo?8cpFRRLR1)PqRSv>e5c?!lJ$al?lI(5m&`*d=YXlpubGPyQZnsxQ|
z8N-3+LLLMi9QPtwvU>+|_s^gEg@rW{!;($WP@3YfSHxlVprk?xvNDPYv&NQN4=X3~
zgN~#XXU(VI_CI>~@cpq9v<VJj#+4So4+#%`TP)W^mf7ZKP2-ZT6<IC%e>_(=;=q}s
zxNR#WRGcKRKHepx)H-j8g=eEu{`IJIVlN=DCt_2_$hsJ0$`IJ*S8qDzMn&a`P{IwE
z*9u>)pk@e1VAat_aO&hpv)>#mb?d*x@I)|j_a*24c=ht!cy;A2`lJ9j;Lo0=IocY2
zDhabpw-l}(Z^~(C9%LiuQwJ2rw~<mBMzs}i6bZ!iuP5>7+q93{={nyqc6L2%?|9hW
zL;tuvm!47E<Y$^cY`x^U&xd@0ymaYf>Ao_Z*+^-F*Y&ricqJvKEc6VQC@8S;T<Ii7
zznncj9yf5=ctA<xR$ji9F5MIP1NJ^^&Ud3ZIja_XxkPRy-Kf=rMp@64?(y@pX&bXW
zf4;h^8u4uAR^h|ilyV0a1;rOJtiNY2l$7A#5L9h-CiI!PMR<!|f=6+a(3h&ad!nQJ
z<agZ%nDyu89A|63mb?=n<*V(ru#~5#RMyuwj$>PI|B)=9RYm)Z=@-#`w9^Fd1e?qL
zou^Yd)hB0_mCx&h-mbxYB&?UQj@o|J+*}E}dCFD#N$LJORY>n->+sVVD~jc3`T7{w
z^G92Yy*QA?bR)$^OM3znAJ2|1j!H5rcy>*<Fw5ny{YdkFIxY-1JYdRjeB0CGfo5}{
z2KZigOVO`S8Tt2g2;A<?`fXcI_ne%JK}{6rvX2AWiS@Oo1Ex~GPDZ+pUPP;PN3v0@
zyPj*Ge4m`W8BF0OdG(Lf!R(QXKEH0^Q4HJYV}`Awqc94m)$!adD$EKJ=3i&}|8yJp
zzwE4wzFmCZJX?qEuV)MLY@qI6eeTBH9^LTnaP!??zl92nPDk=gPCoUXEIl5MZ@sVz
z=ALGd9j0cr!VM4pom%)5|Mc4BbE>Kv8}Cz6Dnc3sH@41+iEV6sge}SSDU=&K7gWbi
zU5MHFNJYgGzY?7JeQ3S#Gc|9tVR{JaCZIYJI{x|uPvTtyGcuE-kF&ELF+Ofj&xs0`
zEifExP^)%9vaS1MW_I^yPvaz+q<ii>_xW)4#;t}j^Czw@s%^K_zb*WbP`gI)^4l%-
zJ|Y(sl^-!yQ&U2=vb=`QrD{h3!3(fAxZ#Pdi7E;XAtq)GmUyq%$7yU=9U{YJKhy~P
zIe|>P*r(ES@hEe~%aWx0^+`7amXeY+A1Fs|KH8jFHEMEx;DjE?Q_T&yPx7<G*te8B
z@r!G8!-t<zPoBYcxG!I`l#K@ZJ!0>{!=F-LzJAPlgyPqnu5js&_smtEoD$L-xv5rj
z!L2mM&E35|ikXqmE=X0n@E#9l+YSV3)-u<McliHXbNijp*~!1}1^6pdJ$b%TTX;}=
zS~pt0(J%kRpwG3<%EhH0`nGO2B|5&^G0-<l^cV|`%Wgr;aGAoVVu)PhF#KXK!855g
zrQR;Vr(e8;BKTzlg<dRtEx8nR)8|5_-H3)NzI$JIytqcjbBj%Wi{b(n)vAOCUgQsA
zCrDeGH#=chh>Fa&o}<~0Tio39-21vnj_pnk+qI&j!wM~}DvuIyAUyLOYtQlaOG{~I
zfwBI><>FoI;;Yv3cZrwQSlOgBIoU-%Gmt1tY?xSVEn8bZmQG74n7ny2r7D(~gVq~d
zu~R6S-qpJWnNN#I5$GFJd3$S(mSKHBA}Kkd;IEC1o-IC2T5Bp_HL-9mJKSd1eyyac
z`UZ}LLaj5Xat_qJ`1p^L$U_jn)I8a(?DF3q8NqpnO@`Y58z%|tZ6}A@+Ni0UDX5#>
zPFT2O{Yc&@DcvtWH}?H`POi7_l~)MYvU-<!lC2-YT73^gq7NUK^-@pG3!~At1ozP*
z1o_o3(c7e8w9*j7to!l10>)zUlKV;TrME#x2;$L`qnR<=w$l?*)P}&b>+rX4)39;$
z(r=#T@+;%IO7zs4%|WI$FOD!Td&_h6S3~bjjE^Pqj1wv5*=Noy>;L(d^Os5a7vvx7
z>TeErpYsIkJw8H=sY?pM6E^(5o$yrLbQ^6~qI6hc+kYD=g0yBnl5b#`xI-OsG9fXW
z5>NQ&aEm6nJ5H;wPvvDAYKP(mFPfTT0A6Q?)_=@e+1PL&uDcj6tD*%MRLY2ml^5np
z1SrYLuamhybdkXK7LmCTEaFwkHIa#Q<(=(}T6PcWP0!~!b!uO4ft@k~K26J`qc&iV
z<Z6E{eeFDlYfaslvu8Of;>QnrA3kqELqbBt@Gq01!Z#vsvitG^`oaq<qW$g&K!FXK
zVQI-TI62w7oZ}8pbSz(wr5sx+NkM_fErwBk8jM?bPI_g*9@eiRS8y3N$;nS{&dD_%
z|B|S6e6Y^Xe{J|zbeonbbEg{#z7HvTo(kD?^JePWH1o>u7aU|Wzw55>@-_@T|KzO|
z+3L7YUW0aMcU(X5oHt{orZM|L9u*k)MnIq}RoirS<I^X{+LN2nVLM7%GlA`f=KVq(
zj@_QRa`N5ZC8nkYOeUzo)$#jx&q)FteZ2>Tx}ie8eaKM!s1;*n<w87O_Gj<-mp)yk
zDlYC?@Q2e*2-cJPmgMJZ!Y^cXU45ndN(|~_R8*?P?^)meOq9IL{s*t7Wi==BZ+okz
zWo0|gDU$j}K0@R_`vmD9vuGt0?JR{*35&<t>B8qEAW(Cj^gz1@xna#-zb<TOu*AiA
zJJ`$0<|W!i{BTZFybKy!K9Bxu39G(=L2=1=!jR+GXjNxD|LBi8uO{_}-I-$Zvb5>t
z9OVw-3W~waFPYw+DR<m|QPliWLt7|1y7m3pnQjt)n!KzE4pTL8^Ud(RJ)(tB_!!^t
zj?yb*!Tw{Ul$euqjX4gt!K!4;P`_GO{hV4=*@MTqOb-_l1j)fiSbMG$aHJePQ7jB;
zIAn}f-%pqNb;^$i`m=3^nx2tqjaS9a%nSkm!oYa-%U2)yfSxC#{#5mf3SKy5fa<c5
z%{g3snOjSnvAOJBb$w}RO-H+5OG{>=ymxJRIV5=VrUM0>ke>U=eO2-Q{dwi2MjT{w
zh3;=g*E<zP!@+D5_p-3TvARaaVEosw%|O>1W=tEsh|^xX`Uv&=Rpy$n3=M^)e(H}G
zn^4ZjH}A5S#!ffGeDw3zYXU;j;lYqF!C${%R=UH7oOgnGtG~aPlBO0Jf4cTFRg>f%
z<zE;7dLF`-!+&>hot16e)Re}Z%>4<jrk0i{_JoHn4NXks?ZC*}czc5zW=uBp`uaaM
zO)PkM8PwD=TLP++u~AzcZOl4aOt|1!_R*q~#jr}v@o{x|xmdxTEV_aWb#l&9V=9cp
z^6T`Ji&BkKPH=GRTb5L%?i9+cZmyZ+z}5^&N|xPsou4jg7>lW>h&XWept@sm`1djM
zvhll9t3eHzq-@3F;yP=bYjc#s<rPt-G$Spfr>kW!3;n+HX}MR`cmMYMefxO*V7*)s
zLG*!eOjuZ^9)>Z6nS{6y+jrc{*fCRx0>Z6stahTnz9*4_FTA*TLf9C7`)5)cj;8J4
zaJjsTd7MBq%D8@qijLjwHRd54#LC_HKITUSM1!==gGB*1(pFdTIBCp~w-f#1TG7JP
znwx`1>CDb!<zKf=y%Fn>)_uR5qs?xAvQT~7%4%UydDIs_gqAKA-5z<~o{H;h05;sW
z2w%&Kx<ZSnU*YKdj8CowT=~M|Q%YW&YYhV9Xv=!rQx4@5y2@!PO@SF<oYO<$Ar%{*
zv5uhy7hxJs3^@X(V|V<Y!^4|;XL`rSn_Wd*D|yv4UsiEDG+Mf?4CCAQdQ9D2{pR9K
zams2KRK2=iF0>@BLG=`YX4e|`j8Pi-HH;rRK4pBWaN{tr>-13L4snvx5Bc3#A7_%-
zJ{+=M^*cx1R*d{2KWi-mBINXB$>8H0Q?H&9#dGCjw{gfF>JN=N3;PZ;GrOY@YTgwM
zY(iM1e6r>_DN{@Smv$K79KShK%p02OMih3g*N#^@R(jtp%rBnpVDyW8a)@%wHNL2u
z#VaaeY*h0`kBi~vff$dPQn2$(x}r39f9gn|Q)$j&E#XT(*O_!C)|uHK6zv8KtSl4S
zJ<Y?Havj8lp7mGXB(UsFh;%PVj&;*<AZ5&z7?aFpcRzW7!VS1~vT~^4dSqYb5vAXs
z9}#LZZ!~h$P*TmOpcMJoMOO*Em?ceQ(QO}wclv~^zt{O7soZqu=6*Y$s&!RSQq3l7
z1;#6Vp!d7x(@tk*<~OrJ9k;SGpV~T=l+D!^6W5NpqJD{?Ed$K%-m9caK5=P5llqRI
zbkHp&k+k!jbnnQ&rkj~)_NXPib2^W+HbK(byB(XeLcup?XxPIChAK$B>s3TC+LQ!}
zkx}#qJ23&>-HZ8t-q;Rw$J@rI^D76@^YJ!Xv#ifpwn@i_oY>|MGGnLHFAa};!aCMQ
z2JZU_6)fn$87d$I0%4P_{gI7;aEo(V=)1`763556`)d-Kmzs>W9Nr1qeN$Ib8red~
z6CpD-@)64`M!WgG6H%<1KYf9d**tU0IETJl?$jWNAI^=&-O7duLTVA0%SO(Z<-5^)
z_u5TsCthH;@mg%fEfUiBxwyP9GCk(oKi$o3gHBSouD6-DZrM*_^7!e#88Pp;b6MuA
zb;9&Vnx#5Pff%(g(95F;6X6J0Z?Bg=#TB?chd$Zs4zMxkJZv5rE~hE9L|hBn+jMw0
z6aB^ZW{ZjTt$f$p4^CEJ$C2x#9{(g(W7L{Q!st;zf~)tvZa!w)3VF57^5J~@wEL}^
zw~K@O<D^-vWtCLp8Mu{Kj}8ykEqytoFs+H&-6}rrmBnqq=TEC<noD+fb-v3Qq03WE
zy%Gm!7>b6?zt@ibXcria&;KdFh9CF+rN1<~EnCAYJ{6tXlCPe|y>&g|-JkiCfvRq*
zzR@vY`V&v6dq0Y+%Xj|Bi(YY3ae^;!BAU$6yzdL?Wm2P*G?yO|V;r*5gT1xaW*%?E
zR5*7{jZdld*Q%i{bXm8LnBq?~rqIbv+@6K&PobU8gn8>+q;Ef%r<M`yX>dK&AC<;*
z!EX=DoGV(k?oS_gj_YDfz<+z!Z8yhDYCIi(jcHz(2r8nFryk0jg9q=?X2Dx`ihr;2
z?JsH5)3G=3+7QF9&ww&BzFQNq`LXo03jUZ3rif?G?0h)+Gqqv*5TcHE7Xv0ts07->
zcfV7)fecc32ovCY#D6zQ;r$VOcFpf<v9K|d&qD1{Zmul~uiR`>&d4Y+X5<>bmxv$%
zp?B`%f<g*?eLQj*`Ov4DBJ1N?owub_Pd2{EW_y)$m|Ie?u;85+A}1gcq9mAFLqBNj
zG8Q&6n)J9*`kfQr_x<iOE9+IfqJmR!hyoirVSh#Ck*aqBZ3mkZ9`5(P7PEx5jq5ZJ
z-)%Lmg{Vhx2lYTl@8RYqS*dC09cB@|g$8B?rI1gTm~^ix7I`^t(aE=#nd7;9ZW}Kt
zW;y!pCO}A?VlpHlr99%N1MT*n2JwvUtz<JFTO#@UmRptE^*S|gb#yZC*`YM74bVjW
z&JQ#;D{iUZ`{H~=%)sDGy2pH7--H@cfn+zYrj$Z(*6-|U>(6d%_j$$|a`t@dSS6kY
zD=PsV9SEgMxDRaCxVZ(S2@8r+wDPl-=Bn&DnwsQ)jmReyz-Rr9(}32;Pf1SR-dp_?
z5O5?))9@tDcwb3reJruL%Fg_Sl9IN>_ZQxR+>Rw3oly-bQkMK)9v8{nZxa)Kvd5R+
zbbB7#WTiSbmXj0h=eGo2<j&5jOibha%+Ni)g?gUce<1R|FGMoA=F7|}A-gFFf&8VC
z9h>actCU=SN~&NS!h^5R{!I$=R|Db=roRYkK3ukqAzm*nC9KEizy0Yc6BBh~>^%NI
zrG)kMkdRic^`Jmr^5mDneSMyjSFbkyiY_Y)+OevkkK-V8x50Gjob*IuySibr%AC(Z
z<L*0i?pL_owg#>>_4M!|#N(1$R4PBOV<_*Kj%z!`(|l(?3X^yw)T93w=?Lbzf9ERD
zrICu{Ka@@oU$r~#yy>u06{jLd#IZ6g!8)zb8LyD^W~aT}_WqkFm|XG%u72gYz<eI&
zurr8n!6DnAkfl#-Q)Zk7J2xgyk*>!FT)2d^TB{6t!(z@XEUj;h-jQu?;NKVue<~_p
zK3rzdkJEWOorUGINDnK4UD;W~-Ob|Qpby_d9fU4MaYy0vBVCaI)Na{7y;Oer%o)V}
zHNL9|ng4FU<r|Jq9fCk*D|H!5wx~Vyp;xAUh;NnG!w`udtuXoRER!>5W)7tk?P4$1
z?%6X4+&n}>=)mxB9h%Tv5TLtpar?^3>szE2ll8enL;R4m%<SViHfc=Fy#yqjfReX&
zx{)#4uVJc`RM|)8gp!ISCG82?<k!}?hJxtiqtzWBcQtCAe0irFKvJ3FMNN>a9f`%a
zAl26>sA2eKWkp2kuB4VK3d{P#O8$=@jia%C_r<#T95#^-SHa7UDjh*cO6u_2pEyOG
z&q6*Exjs(S+}zkIB9TzdQ*Q32*1SFMUCH&7jpoM>JZ3pnv1UKAYDdg`Va=lcVAm`Q
zhOJ@#bwWZWquU%devCSr6gZd3osW2>Qv3RZ$U&5D?$d~iYmIZ)9Hy*2#rM9z@{~`U
zzfU8JI%RDmIvQ)Y+@PNVl`<zxNiHb~cbKiM8F1ywvGnxnTN9HU8>g&HoKllt_x7Xg
zhzPw-(YiT}*OL}awM5Ck>a*0Ir)1q{x%CV@aT>?sj7FNJ1x!r1G&HY(6^MuYRp_K=
z%YKpSQMebCmV!G@WkF5A-NKR*`-V>;o71&da5%IK4XHz`#avFm?2q$o?GDaV?>|fF
z|8BNkI>8?rqUP&&EU9f@q6cKhm7Xg%@1d8~g;`iIhH&i@=+6z8m_#Wny974RSALO0
zr?bb9Q`6XQjhMp7gyol{9ggBHtbxAgp9T5v-fdkO*7`x-)n8#wp;#7lo4EJ<>Pl<M
zE3F=+r|WA3O-)VXpuEKIlvj=20m!u^p~e=^XV388ba;M9OoX3DYs7RqA0#F^edue*
zyh;DS^@L8}fINmZxc0=K5hm5^-Xhk!6tb_y;_xA<K%ba?b!2Pnp36p9JL0-OE0DXI
zo);PG-bYenMi#}4IkKf?QXdzJ$#VAc9ku{QvPqVF*>oZ`>!n{JQSqdU3lGj?h3wf7
z;6J}c;$TP6PcKPx-9UbZvC^VHGbS?Qs_i1d<{ou$uol`0$stJX2rk|$Zg|(kB7-X4
z3A{2kW>(`pQ<0~5*7x>%8tv6u7bE@hu0{6<M!Ecx!N9Cw$KAvF-fb{IiE00Eio)AI
z-u5eaDu7Kvu$d~<I^Wf0jN33;J3&j}oOf@$uP^If@96OxeqP;E+7>N}Wd#}j$T^))
z?;?-(N!uQh?X+f*`TOHGJc0YM%ga9%-6f8C;TD-jUGqH3Z*xa0Pfgp;FS3HOxxp>C
ztq5DDrPB}Wv|d>FNJU2Rv>xXw`$a{4dfwlVNu*)YjlcDq)V1LarKrGps`#u-!g?z!
z?gMpHY~P_wS2$8kRLo0B)EX!Aex8bG|7ezuxAaQ`{v`vocyYR_rM}`|gHGx6NO^Vb
z+n>yBTof1#K4*kt7t?Kp#YsaSmGXUsJ-yIM|8pw5^y}6q&z90^2j>s+`0p969Hmdb
z0tdw^=gP$7=^B6kBJ^DU-}3O^=EbNgdnjIs#=KNg_6`sKR76kT=#H<cd42w41g87|
zx26j{e%v7@#)6C0uS$BiCSC7c9a43I(Nv$Co66(h(1LN(;#_rg+tGTcKk+MFn&B@W
zR4QLqRpXE;U08~GXH56u-F?*P#86~W5x%ta`N72CA8x1N6cWqQr_^&{UeQ4n8d8J=
zNQAXapsbiz`kd?=%Bpe?d6Hs(_DoBcFJH+kZ3+4K96q}v$ho!WW#TH})88TBFEOUj
z?Fr#K@uu0VsHjv_WC6>?|Elyw;9;7A=l3f^lPuGWN+s9o&-gzjY*Pyo*Gxc(q`A3i
zE&eRHcyXPYrdCaUkV*UXU9y>R9`f*z0lBR}*W2`O@@qIDV+~+qE7e!JCS-wL42kr(
zNXvNZM&y6x*@b!YRWBe~g3Z^}59HUQ_~LVvh3C4d3<pC{+oUT8H*RMz>_(_l6SKi#
zL&A2Rn2oZ-h*DC1r7_ij*O&?84~Oj9wVCvIc7H+ML$7CS(i02ZTgx$~A+sIj#5I|j
z+4c>>nTV4Zg<`=zcRL?i%J_el8U@c^!*lp71Fh9-AZu-)<0&kqdz;CU8m7&aUqa;M
z*bUq&qS-p$oZb?ZNgH$D4K#dw$hmBSY{$uL*ESlXnW%sxK*fRDJwCPtN15f{59&Iw
zo)3DQ+~08Dca(|evOi33Xlgl?b~=8(xp`iwWiKL@U1V(iMR-X4&T7ZbVgn`RcRv1f
zlYWVPX1Mv53n~NAQMtOMPPdI}v3BPpZ#Q@P#aQ=0=Z_4hyCW?<hj7)%%0}^)A>L>@
zIhSWjpFiE_Yp(xTZsT82q=nw+%RQf&IoS8a#KbR?mXHvtp*+|*8E;3&qSUHy+uEij
zePj%BU~hObQp!xwMxn1-+1weqneh3*mT9!2>B1!@R@T7Al(cTNEXvhdHvQl`3^gT|
z0t4?197*3-VqOk*LKml(+9RWxBqamy<we_D$I!{R%E@nMQ4x&m|LJcs(x%4RpK#x;
zP3Sr~M~$jnRZJ2m@5{@`Tgj3Y=AqfeZ^AP-?{r+B6gr!ri{EuwA#Afp%uf9*q8W5T
z>b#+BgSwf`LiB!zqk$lKA~7X5w}=z>el#7+#^&M*<<VZSY_^E@tTg-R7o<Y}V$^p+
z6O9I&Ukov+4E;oB6OO*2_3e_XTG%zmjNW>9@Si802oa(rcRi+f^~(OTCtuXz%10cY
zHfk4}@kX1>JZz9c=ijSU{IafTiRd?xky@}3%ar~6o#^ay`2$|^Kl%qW4CV`-i1`L-
z1LhkD?dK6>8;XCR<RGW;<P{G1@B&RGF*OGG<9mjh!z{buuLT)c@C!^)#qG*1lcwgI
z73HfE?#KJJ{^<zy?M?G{u0-A<8sr7VR5mu0k5^yBE*UWC;4Y5EKmj!?n-po>|473`
zd~{SqtgGR9u$Q3Fb#M2wN<M6J%aQBE%660WkNf!G4dATC;4ZUms8CL~xZWbLXdM_3
zrFK-_O!p^#bCcnovC**^o0Y(M)G8Yc9dZg%clS>Oq@)%{tWi(h+&}Sh=cZCY+0$|^
zNXWO0R;_$%Be8>Nc%-C6ht$bTuim}yD)DOr!<ZkK7THkvXTEjewvR&9TGK5oqY3mU
z-#6;B8^9F^rYMz_Iqv%Tb>FR<hP<n*vHqi#6@Edal??$u<S_q3jk5D!YLrRr|B(C1
z|04G#;cb6!Li$$sjX_N%=(F6N8&(qJ*L#qwOqO#`GW%ljLqeqHmbp|4B6SoM@cF$8
zHXE=`42~MryEQ8E)=cE&#9qGKjpJIasbRQt$21XBT>o8A?}GOPw&S%yW$()6aWQe1
z>byLlr$6Ht5ylD)vodH4d3qO*lq~=>JWM>12WGdh$S?~3ul0*_6O;d@KB#vc4sDlX
z=6v`5t$)gQei0p=-Gv_~BqX`d?uUm7j@x@W?OeaZ;**lbi)*U)U|^uJ$s;9wo5W>K
z6yE@oTYf=)LQ?bAu=9?`5N1>r3v6z0_k>EPW3$ZtDt4d$Tf+y+D%^gLDY(7AkMmbL
zj!9VIF{Z;|zWuG;|FO;h5AEs&$@~71{wAb+*>Jsoe?p@Pb3o{{3sc$oJ`s<JeH}2b
zg=0Jei{*dhr$Vi+rz3WUJ1nH6w;)&D`8`!Mg(W9;)!de5ZceK|%k#s3sXQeNycCiV
z&+PxXmQRN+GS_UY_eh3_=#Kx%e#6VO2`|UvN?R;PO5%se7(U#vf0JxFz^8hT6#i$j
zi8TM0yfVBYCR#~2U{p?WlkH8%I3Ab(LNL^%L079(zs|`K(WEFSm_!%<OP|VuR!Hz%
z^mufJzqQHt%FmyV+&*4=O*u7OsH)R_k2*Y>kY7rw-O1ULjxo(6JG(L_TsAaV(s^FI
zGC%*#%Vy;aUHFg-_FpE>W0N)3z4&UiIP~$OZD^=^ezsv06O}*y&mVP^RB=jQwQy;^
zVuGG4Wn*tBB~54vEA*u&X6<1`6&0VMWWK#Up;}>;<K}i@7jqpjlga#y79ruZIrIiH
zO6EfFM@9z($i=cxPHL?T3D*Rc=@X&`YrTqD^yPk0IM0bni28YkhF&!wf|olyS&6??
z%*|cnAGm*?t^2Gj=3*yY+e5f1d3c=7z|>#QxD2ghev7%}m~WUY4fFM{GaNi@m`--O
zVgP@6uS>WVF{nvG%c1U2Bgb-lQ5S3dX?MMRC$3us=7`@M4u_>3YHU(Z>9Qsv*CVX?
zs~f@_{Rp+TvH5Z;NG=Jm8r}Jmfq}|nPn9w=I@Xf!UB;dwJ8!u>|Cb$!6Ek^2N1E4j
zO@Ia+ABQ_VSDOzx4LX1Qnu95+Iq;N$awaG@H%fGmwSjwRG4bF-@jvzSw&#a`Glxl{
z;Fz$o<`RhXd$$%7rTCRECK@g_gL+O{vz3*o%VyKrk<!}9sh<(2u_zQJb7*8_(3dA$
zn=N?fQl+->c+HM}pNG5SVz%dIa}>;rs+}(`VM;zQWY=>4S&U$1{dpK8KeAR_G+rs-
zZ@)S9v8bp$A;FEn+iuS&0tEl7)VX@Xtbfz3oZwh?FW=S*BL+rgCM0Z&(GHKvQWDo*
zYYL2w&Al`b{Wmif>uCMQun0Z^FirlC9V{FZoqDiU;@8^lF6U1en^aAVtAR3c9y|T3
z>fpX!Ul<xHCh8|F>W6h?Xvcns>vK^N_b7k0KH=szh3hF74xX!BUlj9;49q%!9qT_s
zfnbj&tUCM&k$iCcf<GY`)1NoW-ktu3ZuWN{{I9sGm;2YW^vH$CTY?IX06o#|t-7;p
zmC>+5-=B9F=uN1Dk{2YEW1}_?`O!xc&9G#NIw-^E7fiysl}^O>&`Xf=k&g&v<Jidy
zub7d5p`m(+zlm~~e*XozHVMIu&FJABK6GVJkN|`^=qNsX#-j>f%_j9gwv)NP!*0Nz
z{l}6c&w=8<NvM`ERmS(8|FsS%{2%8eWhFsHQD4E7Om77xDRDJfYfD%$o0_6qU8R%g
z$9&4A9eyG!^;?V#u#Ae^9_7>W_q_m@IUfdl>Gs$}aacym7>HxPdPTdvOOjU=IajAB
zI@mwf)t=SXno(n!J~lW=MKS%jj~R0~Jh%!tGh521XxmvNkF0L@1#7D(3=GNv_cm><
zilhQ4f`dY7+FFAHS!>&c457eLQ`2yAjI_1*`l1O_64i5G?&bOL&(84oL=+42D&;>G
z)EAU9h%hTp&Y0(8d#+RLJ=@&e8JWpd6j&RdWk^E}4q(nC0hY`5pZz`iyr;P|_&+Xt
zW1jVIV%0stq~aAmYgZv6xNP4FzcNG<ebM{X(Vc<)=LIC(Wv4q;`w<D4HIp-EtW*$$
z{4SZa!q$Ht>RtS|Au+*mvFUU^vEk)czQ|~>me~U?>LdW}<Mx_mf<@db{`9Yf)Tft@
zc$&Zkmzd22$zf?T5}+F8SUVA0l?lt#8N8S`uxrK{TPM5ei-o(uXT7wnUs<co$yK^O
zlzPy+=9-Rh+3jf%KGxsLb`^iqBjJum!fG*rKCMCxtXw~xUt>K?74mIZUS1}0<*}Gq
zP}k{n0Q1b21Ni~hI}<;Cd}zkJ!~6S{NVzAhtZD=$dO#jt$sZm*Vl@><8Jzrz@A>Rh
zZelF8H3z#6lfA06-?Y<^w#4$Nh8+n62$FlFHr9;*T7<&!eieT*z0Mc^xNWGgGHov6
zDly@Y+n9rY-yZCB51MRHWkBx{A8mKLx|BNX3Y(a?68JW)t+^_l><8Mc9jBAeHD6V)
zdGbpYpt7qYwz7Tggs6)~cJRi<=7uV&!T_!?OmGue(T&$x>w2k)Ir8(;-Z!NUH`Il>
zUC|K{YWK3ST^E~fWgFHL)uJ~8duUx?8y;AQymRH1SL4-8V<dA&k<|SDb4h#J9V)(N
z8F(cpf6-=h4j&zEyK!Pw|LfP~Kbrf@SoFXV4_G?0Qc<7IGWHIs$HjTxBnp3sR+3R!
zVd_+zeuXxj@gfbFN!r7V^WTb!h>2R)Q-H>vt_K;daeZ(Ps=ajS%cQIG`WQm)bb4j@
z>(TytJdg5T+r;4UZvMJH_ecnp?-Z~4e!LsU<3C>)J@`&mPcG{H(Oe5Q&(`<fk}d+v
zpq{s9cFcBOJbrJ(p24(wcW&(SXD}r~ue%llOO_rR$7Or}%^){7Pcs#jiKvLEkJ9e-
zh9|Cvq019<r$qjL;_p}hX7bQy<mz#B#W@<iiD|tv_}$pDe}4FTX8d18=RZA_n-y=3
zuj>i!5@raVd9IuU6&{Hze_sEe{zLe|-ph_z!~^e^xXUpTrLF1L0IU@f6!eXh25iIW
z=~wi3lhTn75hFZftgBKyON)DEC!quE__$J>e0;U>K!r9Gzo?pGol$tT*jB%P*1eD0
zI*GM)<)l$bg=Jf*`o6UX5|!hB?M#E;Wn>+rlZ3ej#>2d)C-B4`egIo<Y2*tati%A$
z$Zq59i7pi}aZFiR8Thw@4<wJ{o}Hs3-<nW)-AHS5b6QGD<M=p|FS@$g8O*qW33$t%
z=Q~sY%`3q?25*L=aD&^bcXwyH)y9@1z59IKgD6^h`g?#D;Vu3_B2Rp5D$L<3%Qm7{
ziJtbeorJZ}ThX6LUZ~ovC81xnhF$y4Y?U}q0bbk}v9O4ViG7=<e*gXr?)G0Mfin$X
z<KyFi-x{<NlJhz^H~=O&B8IgKtP%jLF*sN{!V&{Ev|vmQe3@p_RoFm_L11%O>e$i_
zhETm>H5q+KiT|Dx|7GC)xbSdUU{!qnJWy(?E<i;Y{u<y1xRKlNVj!{+@M|$e0gez)
z?}|VdYwV`UMl_b_;u(ZJkFkaZ5KTkF!<)AEWX#9-H|1m*B5n~8fx&!gdOD+WiQoJ&
zSR=wiAnm5*=jV%3AMFmRho3r3H~ae&v+)8r!9D1O+a)-=Y=8{_RS+DBxibm*!F>5O
z0Bn4Hi3StF4;!r1^VMs|2L^0_YYN0uelJ1tSK{JuA9`zdw~d+E?=~71oqV-w7yv+y
z5s&BPF&*bd(hrY}q^G9?`T@>{PF+5<V;1;Rz-VI!Zn~M-6IW$0J*K6h5#FRL=2rmo
z^-yZDDZUY)wE^P=hYU!<;FK;qLqkrU1m>E+%>+;|Hx5p|y$hT^h5IqMu`HeI@7D$E
za=?fR3SKxxWj@W!T?C>85Dmfi1-P4C(3jG^Gd%&*-2YWtT3$Xr%QHJ$>w}#GHdb+D
zpEw<NJonhkU5bnSmQpcpocQ-hNJ2xwO^$GEY;4xTUQ3G-?i436Us950i0~C8Z!qx#
zw`L0ylQrOq0yA%TXeguKEA(|LnDPVP0^EaTwO)<krYVU|R*RESQqCN$*Oq?y@|!U%
zX!zv;<?~#7>+SUCS)QjG4S3z%-9FeF1!aF4QQYn{g%yjam?nFqsm|sTEbbZ{A2j_&
z-9R69^PL#P{>)T;N=Mhx(gI+Z3(L$4dosem0O|s^@8U@Aa{Fx^n~EDX69NnoM@L76
zrvTL5r0Ii8wu1RG0=|&=Jua@w)>aW47T28xNd%$?u#!M1aoHcU<(}74PwMRKbaHkE
z_)6y`Xif}!8@#34p8~(F8ra0r(!JoWZS<?idp@==^7{4b(Ai5%OCv-m!DW1g158x>
zft3e+^~aAN;213tPy&z(ph_z#D_dSp1?PMPl%*{4GJra185rjHM&Qh~nSl8mB}y&5
zk`df7Ys5=IK>;9(d~kGR=`n?USt~Lfq^++W^78UBF@@wJ&?oo%BKrpD85oe<_Fom~
z0j0E2VZCe@9qGVcywcZ~x!WdfUC;Mp?!skLc7vyxadLLNyDO}Ha|zenfr;$r?AW=t
z1S}<_Q&s@`WaxX|G(<dud}6;BbTL&ct0Eu_gKsv<k*PSc-jT%7V(*-AJg@V!oBo_5
zdc{`-rtRd}TrZ1?>~4T%_V@1+LhiKEX<y=rvDf4K<|NomvzOblaA-Lj<kg*1jh_~G
zrFRov>Di&vdYx|51einP@hTL6ji8P0+S;A7O5fc9S8>1?2nYxeMTmw?5N6kK$dP0~
z%decmfW7Ov3<TQ}(}}!j2}Q+WFaY52eM?9f3g`p0zQ+l@Ed9;FzP^X!X|8+2x)(2A
z1cdn^AW*<nVz{(pMf(ikX28w~X(^>SH$QK=mA1AK5<-wE(_Ul$kd{_5fcn_t4cI3I
z2VdW1z98;#o0!-#EAA0DH#dxbU^BL(-fh!{zaF;sM#IvMvAb#EbS46Q3&%jzzz_*&
z9)Pmg+SuImZSrZz6cgo%RkA@kSuBM*lpz@%>xl!DL=DpWS>rivi480zu<C)i$*$EJ
z1n7#e@Nj_3%|@uZOtz&jc@gsBEF2Xi1WUen@u8q#6U@5RT~OM_!YY!I!Q|Agh`D%$
zXM3FGDr>8&aS;(Mr@<trMif=923+Ie^N0^n+eiO=!?-$DIS7FuEv>n|{Shs#KEx2E
zT?D)_EGYE5-<Y4N61rZ79C#GZiJ<51jh5p|m(~3QEIvR<g3Ha<iIj5-hYmP?El*%s
zqMZ!p^mFP)0O1)rb=>_KH0q8+C@LxnEDv6b8Ih(1DRD0r@P3CL0AyciHN#GhBSiy8
z$4YVw;2VjG;&SC)z`~-TrUvL3sG@yPTI9h%YW9+rx@I7nDdT|J(k5fkH#T<Oo)hD5
z?TTT`1xD>@iaB4A1)%M703DH9zz<9_NIah*%_eG`!9E5JGe!9MVJ8SFbPQ*g6t}a)
zG8490xn>Xh$f%21pa%hl1GpG)C=E@`;ag`62(y=2v$WIG^743P@#bIT%8Dm!d`iA4
zt(s00iQuWcTFjAg9)sQi&O~=-CoVQN5K4Uz75%fJ%v0`&a}GdvKL8lx&mTV~*VZu5
zc0H~snD&#sXeqa2H4OTJxu_fgh-YPGx)hajy2QEEXD7GQO?pis1}7&^6Ps4Oz}^In
zytOqY>UQ^trkEp4{U(uHa5_!4RIGIA>Tdlut=Y97Hitw^soOocf!4U^ZpY4@^a2YE
z4;QznpnwDRDzHsBVE{h<o)Bq17+Q5dRDz{@V0KGM{RBpd{WTTXw+>wf7$pE<&azxu
zspsV7bpbT1dk|=I;0tjAwT*-0DhX#8O?+P@IJ^RdJ|XBb+$pJ1ORa5an7wU?kd^HR
z7MHt>S9Z3cwsvYWAvlQW=<33XWKdz%Titgy1_QNo{i^W;WnBR@i7->pE`{=0W5&Vf
zKQvN6PVmBVbCVAdw^%2Pl$37u_(NQV+)udaf>}*{iM@~Jr9CwsC9`5#KY$*lmbbRD
z(l^<pU}v}lGxa)%P{1Nvlsf@a1WYe`(-DIPjV#Ze4Gj;|P8r77=SFKb``uxz-U+F!
ztPD!Ri8ao!Yp(p3mvQ(p>28LDov?5VK)veg>jUodRA@B$03RTefej@Y<93V|bC{K-
zMIh4D%HDCgO|6n6o&x5f#9g%ZT)LHkJch6ZEnMgdx9Ste0MZmd%bJOoc(i!Lu*<C$
zrGWJv5fLFueFYD1u*^Wx#beqW&@fLmn;*LDZfq>TnZi&3r!9z-m7S?fPoHMy=Yu&O
zhI(Qx!b3%+y)sk)UNJxbS1dClJDS}Dl*ME9no&5{y1Ewy_QBBzgHGu2FgcV#PzM}M
zEZg~5`q}yUo*1?sK%xT)0p_&|7G;RE08a6cX@&n?k2i(Uh5omnY5*k$0*{rowRY`h
zo5uE?8#iFTzqwmqRp@0B260OQPM2dl35g(YZ||uN%Owd-DP{1n`vFiD`kQYkL%^;L
zNj<m7$}csaiI{YI)7tX2o-paA^oV!|v{BPc0_J-44Gi$^k6Hp10K8UIzr6iWf`el!
zhKwF;ZEITu&;pQZAr8{g(n3oBd*{1BBgB6YiZFHop|BJLS8p1arNBY}bJx!zly@nv
zL>cdtvv*ApaS*bw1UVc4Z!kq_5r~AVtE=FYn_=9Y*_CLFkV{>_Z3!UhRS@Obrgf_I
z{I)3nJ4}4l2k0J5QJozfbPNn|)9IVTK=$|PFMEy<j0#_V1L&fCfB%8Onmi}8IVd@y
z-*8;r&d56S%JOoD3$1%JR0-z7K7U4XS}#@ak53CGq;;cEz(WV{Aed5c0zd+W_34Hu
zALia0FjU>@05y+l^d<y)=@iJjG(Z?0Ob3FL0Jc^`ITseqoAZ}&VRp;N%NsA$MF2?A
zI);|LbWB#B?Br;l@$ut+yVFI$Di>Ug0^Qyn$3=wl_VddDhza^*Ig%xRts+d0?cpq?
zmZ@niWY0c6K3-nfXt58NFdxfm94$(notepDw~>&ZPJ<wn#!LSk7tLABYo8S$Z8Mqg
z!F{y_6^@BDM6kNqBy*DT(LwxAz@dqEA|N5@k|sJH3(+Nz@IS7<!k7YXL{I8`nx;TU
zYHCi4#jx7j**&BoGi!X6Dz%q(8LraGeRE`I_MD1czBH1VrKV1TZ3KHUo_jiuP<re*
zlT%Xqe5NNj<bcZ=>?EL_SO2G8d!sko9u79q?QD*TpIaZGZqy)eGM!wG|7kx*CCiDq
zO65q*sAZ(5_p`imZfO=jE<W&}@$mh(5r+gC%__rt*f3dwMhC_W#H2v3vA#Z;+5U~t
z@}RnR#>QQ!@fm5sIO7Yvq(2@^pJXmeb#30*Mn$Gsxdj4Dji#uRpXzxjv(eBO)7q1g
zswx%g(8ZF$DrLVi02l*Ds`eCp1Q17-yYNzmt`8>XHx8hnbmn=Wq_i|4KK}P<Uqy9w
z*R7d0h&4-J#O@)7=bo4TNkjim6i+gF@2_uC^*!FobfbFnnUa!nqXkH|-%&?ih|;;M
zA^#y8L@6FgeUO_??Pfh!8<?TnB~4@C);W&IX}UOGB&_OP!Qay;Z*m5g`@jS5k$7hu
zmo0!@3iyEb7UA5Up>YX_k4CRww}V>TjSmOPCuD&>zP>>A1z=pVo!75v<uKwZG)F0p
zbj$d7O-hR3^vM=-X!*lSev_g`OPxvkg|y|xmf+Pj3-LItk1v>{6%^Pt8nK-XlDYC&
z4<-V##sfn@-~CVN9|0QI9yvmB&Px5IDh-{vxjCdQ0Ey8k00O;TS2T+e`_XSaQ`2>b
zScX;Pbb?M!$Ezcy07RZ!TEZ|EmzPIYqq|yK<{?SA8;@{-^s2!V55`SlW!Rhf^nixK
zSJn^w?(+b13X^t$Wo6I&H(JOdR@~xowHLl8t~}lYeQbgkwq}6KGGf+TS!a0vN$Pp-
z+1(<T(*gq#a&k9&Mx>jpugns6{NG6Uz9QC%pXq&vj1&y%z!wM$kWrzWWQ{#N`9(!V
zAk)oBe)aY;^YWr0wt}1l5ZpB!95Q#no6gM4zz~IG0t*WZ=_IAOx71HZO${md%+6x3
z4~>vyAq-?#Kv$EM4F{N$ld2d)1Uxk!A`Q$=u{8A=0JxC?aMYB%fyWmM3kzzo=tkEP
zAZ~+Nf`s$r1gKb9tq4IuL3{i1t@9Uma_n91l1gUh@eX_5fZuM!$KMV86|JVG1_-ts
z5H~g}Lo36D_K*RTsW8Yt=3=q7-pzC<t9X6WE8Wvf!uQo`QT=vXudxiDlmri`kc{(n
zK+l0R6GFk*Se!HW<^P!4wehN|9Q>b8?JmuF9&J>u++~fgSD!$otNTlH7SriyYoiVh
zA|oT;I$z@X{vY$YOfH}P|I>+`88Wn<12T9)!A8h6ARGZx8nPk#YryBxl#vMo({U@S
z9RQ1mC4++3{{8I(V54IBGaoT9$e~rj)A?a+!oXxHfdT_FGjnL0mJbPwMR{zOu9jBH
z)DJy}0dBqzh_k;bZ-9O7ojYv<G^Z|j7#Vf*)-42*Tao@I#6LSgs&oI9PZ%uq9nNHO
zPR8q5p7UV4_m4c?!RD`wLO0&IxrRr_(OOdSI)YbLICkUJb5f8b5!n!gp+7Nzvut0V
z70k-VJW-fLJb;J8@k27e3h(;9j599!^(oQ3Dm#YnB6W&`9r#!4>1B}D1HE>r=c@ZY
z<PI4zrTx2@>>+M4JHO^<N#w6N>3MGtU5s9puAEAF70PNl?gTStM#e*bF7C4xp%~iH
z+#HC<M|`_shYkrQZ)2jO3Qpza<qhxUfgk4xu<Z=^LE=H$JWu}-=!@+g9jJ7oy+Ro<
zr;(OUwu?az>&6q^xDgBCdAo-XyhgMq7z!iKLNdV76EGDUdY2i@%R<?#JtM-xL?C8$
z34q$Rcs3cK3<yC#KfjE4GP6gGLqn0$t1_Sz0MZMFkdobaF$l=}^4{#^+apaVoMK22
z&Rex27(uiRp&jlitO0<o0=?Wlabu08$-b}KGojM)eGR)#!-V5zeHv0+xh1H+@(~G4
zX~YHKUy%58f+=lIju9|<R|LZQXQizhapEuRoG5&6%>WPLp&`4I!(B)pzLroxc}V1X
zXf4e)g8r^!9FVE@_PRSdrcYY~TuZKx^A(e<w{NF^bC<z$T}+MGTN^Xf)Py-n6P*0Q
z!k$5iDEAOEYV91p8u0sAT8t-Qs~`-+5pgs(k@{lUalKj00>IV<9wxx)hfSr<S~>H_
z&)5I|ncV+ZGdXOhPR52Ov64l0n^W%Y?kOoLvmFuJ_sVQme*tna@E*s{{(jl{zF`k*
zh^Grbl5xl&Ycm61V&I!uSls2bdWhi7d}=yc_SEe6b21|E@NRSwYX>nuKZu0luw?Q}
zoJ@RqdEkP-_NT$UdbO*&8z8e}y7qP5^(%cju3g!cyt(Eu1o$D+NcGXUF8!$}(a~MG
zdX<FF#Ys<Z!H0+rDoW`i*stPhd3W!qC|$iH?k(e+DUdGPoqnxJ=B4qqrRQ&Nf2vj)
z8y?PV@GaOF=<`%zz^o=fvOjBq3L+B#b_&#%^7_3D_4I(+C*3&6x~;_0;&+Ex&0&$s
z*4CCxh>(3mV$;v|gR122MIjc6-3_R<L9G&yiX_0ag>(`MI#Q}8TaY-SPGWZ2v!#$o
z(Q2~>n@de2LP(9JP=ql~l*>i&#lAcUuZ>XL)dBepNXHK!K7b}^;lJw8>JW`@vG_9;
zpp}e_j8URD@88ce?@v0u+%gUMGvt+$$4irwKOn#ZqO+*z84V4MP1GB04Gk%E^$94S
zgH!Z!kOPRXp#n{quOR;~1g!R6$F%1)64-rK4(ICT=BBYt&H~xBqM{<H>#;eEMtXXB
zcGFAjitRl;@oc6@Z|^ps)LprJ8Eyv$9A>6;`sK_Z_#cx=9P1K4)o6tLTP{xpa`<+a
z>CmP*=7$d#pd~M5+S3qms>IGwLSDBC^lK={Hbu<$B{rv&=m4@33eyVokW}U7=Kh^|
zOUKt^MFa(Ed(a3zPAiG~3~l7w_5yRgGIDdB*UIN%GdJXBSq+5Y9;MWXE<+GBRtFpJ
zVA;aU%PTDG1`SO*P&M^frhql~(ajCAzVqkLTenupb(upu0;Rh@TmB6tfq$?^C+i`c
z+PY7ycl2Emp5xYF3aLyhC|QDx2WisBI^Q?<y#$IKR-fRa>eHtoAt#s+_`O4ChA9Vb
zY?wuf0;B>cp|*=ZsZNCNJ&$NEJTj>JcyYcvPP(QROVYCT=VEVqM0hyuImnE7G@Ehv
zbk)_>_ui28ZNfI}Sb&5=t`BLPV=Q)G9|vFU;ZimfQ)OjkflCNsbeR;rfdw~J!TwOF
zsX;se6{azeqsWa)X};mxgt=Dw`ioJ_y6m>|zqRzgy*~@(Yp6Ry61X@~TWi96alWK-
z=}`(4iBeNj@2>Mz8xAs*!0y+c@p8*_az23iVG<toCxeAh9*Ow5bSX8~9cVq!2FehW
zuV1gLr47P}H2?*I7y#RNXcu5i%;^6KGSbrN12l+!^LB>g;-K_N_FA~(7NgZN^CGk2
zWh8oDVXoT2Oei8LIoUqzW3BR*=(((5b-PKAogFI@YD9qoR3~(ejH>i|sqygeW+^WO
zutSs^mw^Q;zcGsLQ2YMaE9=(nmTApwx0iH?$xcFbt1LCLV)$ff3=H5#c#z#t5QU2K
z?C}*$0XHFOPs<u77c4oFlK4?NjC}-mDC6q~d->DUC#5-is#0FhUk5@L&A(=Fc^9y&
zay?CizU}(mSAWxfz7BE_kOU7Nd>b7-Hc!WNwyL|V2SS(0a&mJyCSsg?G79~_%~Raf
zs^YU&&Ei06@RIh1nQXwJH{3rfW?WdX0g1J@S~~GLBBDc!X>2dr*7tg;LH{F5%kBLM
z*HeI;zD(w`9!ur;ZhYo`yiP)|<=^)LsAZfxv+xw8z`1kh9Gsp)1_HVYZZ8ocDe-P#
z9=W%``gb;m`JTYn-!jcp<6a@eYyAaKo9^xx(Ck1-<a8z7TpXx+K`^UdnLd)HnkCFL
zc0bSc=UqsiUeC|Z??YGOGgN={J*%p3_$7c4cXRBQ1dW4+kGzY^zQyci&`HWKL>;tW
zeg6TgcH>2pH!!Tl`S|$u<}om9#2wkr2hc&qUcI`*ZTAIT7y>JHDk>_FUqK-b$>Vqf
zPtwuJ*TQX(Z=Utt($L5VXdS&O^>IQ+p6?>9{-5o->ACU7tawh?zmBPPXKp`#A_4%4
zE5d%a6A{K%E&Z2HM<n}?B&BqTjR#$0HyNttDp&Y>zffd!bgS0Kll53S3cg?K$XM0_
z`Z@zU`_Ryk0??MRu<{iPKV8)&J)T@$bw(fC`^#&2cm)R5Iv?#7>G!IrsscsW+1a^L
zwGcG=Jx(i;>Fu_BC~O>TOu`VdiLh8a?lZSz&AjFgd|_Q}ZE!bCOHExw-DB65$_#Vf
z2u;L=Dr9w2jVN{R>W81I{)sxo-F9qtUFXxR-lta~S)V@L+}s2o+9`X-tTK~CCrwmR
zQc^}n28<Y3M%j$5t*EE~JLoR934jlTw`l=rEqhBH0%3vq^vk_PXRuX=)kS&Gmj(t)
zV5@E?MdN+|gj8Sv+`2`VC*bJd(A?4jb++o<+*QcdU?tZbYQ7<{v6xzWT-@sK-`_%^
zAO|Q=AbJ8oHI~;|X*1=a!DkrLy}iAVaR2P>4bRVRo85vHP(oP2<FXCd7W{AQy?Hp-
zdHXk9O^e7bTXsUoF4;@6MaoVRSxSj)S)+}zg^-;{BqV!esmM~KP}azjwd}Hm=k=|b
zxn}0Me$Vs!9mjni_x;D^kD0mVn(+NB@AEug+exnn!tQl_{qY~_*%bO%CxO;@v8f0<
zuzh@d6mL~lt^-m6>T~NBuaJ;|_q{8L5=~kIoaa7#;8_X1MnHRJb<l~nym>Q@kq74w
zV6Qki&wT{62Q2iT7`A~y{if+T3bNm69rx1UF%{xR!{EzY*!D|HbE9@~bC*7QmLH<h
zHyZGW^XSR*ufI5ksam!Ni-hxNy=`p74uKKC?H%cXB&-hF8RDFr_shz>aI6OiWW>GD
z4ct@Kwg)YV*ZS%Lu*e_2RH9u(Nt+)^<k+pzfb(JXUVPEfVMKXQfEL4cM@Pr@KTnNk
z``h2%`*D#qG3sR1kj~<`%=%oW1L2s@$`Zh^gSwT~bZ;9YRtf9sw$_L9U@L9I(WsXT
zo^@Xwah1NW1Cer3$;G%Gl|cBqU}ID7Z*k@f9mb@`Lm@1n*hf21kAcxnR;=6}j{kZj
z0!KaI*1oEn9nV4AX(_C>zJGsQYG~|3Cbni<oI6))_F~gpC++>isWm~jZnb0hC@(Kp
z;xH<4+}+d%bUY{^090@A?b|(1%&$8ZU}f?Srv2|aJ7aZt^{|!vbyDVx?k&8V4i33B
z-|@VGmJ*H8$i1&D7?r<-SwmNrQL`y_m}hFwhWHyGpmC6PED!ch9t>8y&<KnM9}`?T
zR>kC;D85Y~Uv=Ib)l4u>lGWq+L4DyD8xh`V<2$U=WOwN2S^qV^slB+?Hb;<;um2Bj
z$O1iw<4)t?r!M0ku?!o>>0^_UJC!^-gu9i?b4KVQ9%U7a_c&DbyM)DWyMkFI3=<)@
zZ<CRd>M2EwS`n|5T%B4&IdPdSDoxCa4NVdAn*U(X38>d=Og48)qf#-AfQF0PG!He#
z;lyOvx$}X;O`Zd;c^K)?uz>0NlAfr2ioBK77K82g^|7&Y!Bs9`W-!SaXg<Uogz+x^
z@DqQ31dU3q7rgYoiwz&=&z+;OZ&6tJnt$!uHB@S7iH4zW`PyamUG&O>2fK$mj>rFB
zNzVCRewbJKJ40H*O2Wv1krYH1@P;+;b{{WQqwMcFJ+olB=6~2C2Yfyx5TM?Fuy#y0
z?R&YBUNmsheT^nMbgU1oMKdU7>;y+q0@9D*{GqoAnR-82ZY?c+O;v}ZZ%@!%9k<Dz
z$(XHvDVA+h=w$3-JpvLKyG`xv1Xncqt~}NnnKhyJGpARUs;;kB`y8lw<Os=%PmU?$
zoj76hYiKWl#DfFMpg;L~doC77(=#*6S=PZ}V-YNMM&ZZ>dwT<=$C;Ud{{H6akvO}m
ztCf6w)<LeaD0rK|=gPp~p#J(`Fz8O9QxyWK+-NAl>Z=ifQBT8C^qC~_iJSU@U@kuZ
zSP<90g|#_I)2^YO!2QS0U>pp|j%;yF%gY|Gb{~*CZ(6u#r*n^Wx=E6BH9A#%5LH!G
zGqZcpci7qrtZX`P;k3&qWddLQ;Y0dZpX|fJ*V}0bgZ88Gf=#m-XNr>vXBw7guA@oE
zYYs@5BTRK#t+>?7QJH{qXaPrD$E|?csr%C>1*C(4Fa=RfBjRhBsH2hXLFoQ)SYzk4
zZ^V($w)a19A;+lo!t&lddmNmdy#|6MiP&+|fTG3Q9J9dPxH<DC<+Sk3oTXlX(^yft
zshj+kx9puq32(+)OkeKpgBz~5l=*mokil?)5d&Pu;!vDDuJyxdimM?@@KA{b&<dHx
zhlYkmL=<=}IVS#w1>es=vV!11<r<5Ze)i;c2$pDih_esY1n+wM_%WmppiMvueF3vD
z1SQH=GA7<sMQjLd&r$2>>>T^isiUVSajun@eP{pJn6Z&j#`KNk<YXMZBy@s<0z&OZ
z(Fw8hIqoiP=@NGN18O6vGy%(lS_y=%6NM1}?Lch^IVtJM+|ygMsArvbv1iP-rORJG
zlF44RC+wVv%SgfNrPk!=*AE*HG865ndJ>fh!qCM{gtjMkIrPmJ2*6ksyvs}kSw$^X
z!NKFKDP(lO<`E2|0gIj1rkO+0;KPmxl@Cr#2&SF|56YJ~E%=20QN_K7TOYCXynpWi
z5)5O*eU*^MhPnOVuO7Y8^llwutMmysb=P~{5}rY4hvMP(tWHPc@=V5F_LiLIdlEW#
zqP#Am)?*`OZ%qxiu<%7(d&7qZ$&H|66g*i<Lq`XB$L{d@msM2~kBuWv^|_f;c)2%3
ziv*mKla<A(2?0wumUavdRtq|1fWRIbANO3CD0iDa&8)43=1b;M7S1a?HMD{zb>8kj
zfKtDS5`s+S;>C-{Y)w;#fHv{-j}L?>Otj@B`TA}dZrs3gWNB$>cBm0NT8x5^H#mE{
z#=CKGGA?7f>^ntW##%8{gn^(}4x-0x19%8Vyu;oruib0wmMxgByV)crQx$Ipw#456
zYsx}HV~H_taVB1xFoWZFbarr{419&XL4}2p>HaA3p{?#?I*XsFW^%KpZajR|TGZk=
zlW{>r>gWB!HmN4Lg>G~`0E}GRQu3xg4raIqW0WOhmH#?COh-u>cIQq%CcMwjH)boF
z;Bmir@d;o|yo+QpZI7e~*O8XC$YcJTzfy`w9>h6Gfh|8yhKc>;d03?!mG~lImGA25
zSwm~mITYW?)cGB+$lHrw+eLz3wTdO7gcHU8n!)K=SOmlCv8*1o58pU-{nl?I&2E=Y
z?fq{tbJE_(oz^in_Q0}vOmrdCS7W7zhan`9fZ_=YCV_E-d0wqzT{}R8-S6u&Z-hY1
z4WkQdh?NK=klUwoF-|K8?%gZrG_+%Qf+=2rlwOz(Jx*z<M^mC4*toU}X`j0<8JzUR
z(NbAZpgXXd?)t6s6|E(R6dBhE-9a9#gCWd5Btsq_%Sy+!jX&Ls2oJ@U9-QIOqH%C=
zEJ|+CMziMBSzsd?la-b>gzJ)+nD}?Z_GpK1CGCtj)3Cwsz1oMJI`iSgV^SG}w=BtP
zM$Oq-TuX1t^+=-H0f`wBp|VhL1yzH4Xai?wW`cr)B`<Ui_Vp#l-*`^R0|`nT-wj{i
zT+eEE(VSQ(D5~7wwk~cx^vRYp)n_Ki^{56){+N;`X)22y+HP&_(}0`fefn0zJDGG_
z6SD-9Bl!*Vb#-}rzxo<;Tp!d~^f0cQ<M&b@j)~G8mF?fKVG-Zn!S3LJa;B8oS<0mL
z$PHJE=ig-$JxzQ+a9+)w4KoNDzAc+KN5{n}OgQ6Vi!`X(TB*u@qp(n7*fU&<(xr%U
z(M#!i&_ep-BNDYb4Z{|(&7|`9)`{;#Ng_9Vdi%t@s$%tf7o<Be8fjgovvwl=`q{=<
z)V8*^GU>BA-*$E`{qZyLZKb5o2U<uV@sv{r^l;$7fe<0<%a>QtO9`2lJ*?X2;AS27
z+ozu{o<Bypx%6eXvf8MB$}|rSPjbqCij`DNk-f0s+*NR7{$YAo)x4JR00yQ}DEqwe
z_?-E6WQ%cRlyn!-5BZct39W>Oq@tb*xwLd{Ei5j($B5;qPnB#Gp88BD|8T#E>qPhH
z@Nn&#EI`@sd+rA?4C0{#^?iLC`9~l9yxd&>z(CN_j~_ki%s&&-&wdVuH@m6j;cjjX
zW>3!*%-X^5?VZlq#<;!YJylbT?{IS5<8)S)qGxZUTv&*<)@zuX_uZo8oXZHFc-r~d
zN9K`-g8G>4{cp=5Vm-cXUQs8{GG5y3(ou3`QEbrNE`;daWd@gotm=QG%=~&?p{%^T
zDM=w`Iv|Szb5$|YyTK+y;bCi=m6Ky%e10D*YZ6M2jhFmV;rTWk7_v@74HMde2}#mq
zP23-@3d2}~js-X#a51^0-_kYsBJ;?-MI#``%qwn>s<$kaJ64)0(rR19Fu66y{dol0
zR76_~^bQcK0Sc3~*V@^(H^N@vl&&t#&YfVxDr`GM6K6pO;9)9X8j7o^s1S|$+IGrE
zZ*3gK4^q%N`Eq+mFCl7ql$E8SsmUyUmO^Xn1_jV5fIa>Fr(e08g=h~l)A6<(umcJ5
zm!-M5Zcv~*xDy_ZH#9FlpP7ltZALA)19}56NocshiSQ@V9B#Y=eXNIv2jC$OkJ1?z
zi4qeQCMK``YJaEUw+@L!_s=z_s(}^@IF+&UCC0fJaT}`jjv~8HW0R9wMAHiky`?Uh
z914JSpm2CiPV+8*XvxM)DZ6ypl2w}mV2_VCfy?o5cW1^}Us}2h$PiG`5CQCQEHU}B
z+Zp3CVoPLsd2laxVGD5+k5=*VRn>#@@dSz9W=L6p$Dum|KKgx{SN7y_m+Fl3npg!x
z<-g;Zh&9-jl9CeKwQ~1a?iFW;KF*_jk=P&|5fvqLvQ#&0w~JWxIT7ClX<xJ<pf!44
zzPwg>OwoE*U-*W?p%0Boo1hn~B3IZiBg4;e>s3FLfGGEWo)mU;+qxJcSSPAx^Uk+>
z5&K*oPV$x@z^l=bkzWP|EKZ-kHqS730MGmFfB<RtnL&&T9i!Td-sI4yO-?qgB}GN0
zPw?sk0mY5L7c}Bh4v=1QU^M#h1GoT0w5Y+%^z@6LTzjYE!!P@biiyq6&&zo&3h#=J
ziCG#=PeN12S8G!4F&`^&eoeu1l4a+Iu)S-nESj^njHxz*BB5KjZr|2;$F(_lvl5Si
z)vH}Q(&l#!a?s@zD1>g`Yc?JleO{4>TmF9k_?V=lnz6eQGxOOk?#i)t2Uti8Mq@;t
z#E)+#x-&VLd}wjip~9&`ZdljAKtcEUs*xaVgkGWRxAOBzV@{8>q-113+&y`#W<Kwq
zwI{qDTX%QU20{=EqD7=}VX`+VDQWWSSAnQEP`RBk{D%9DMis>%>!5?p`SW64j{`(*
z9pkmdKaQSypgr2(pQ6hb!z|%Zp7>ErNGL>#u~Gfa0K4p#E4$B5_LO6QD3F_8kqeaw
z>%pHnQR=)Wq!ySl*u|Qs^c_r6mr|Wm<wVjreNr0vxLlbWqwDVNZfkk{<Y4J<Qj|TO
zlXA-gBFrE7B8l(wxYmU6Y(#jm>N_Xl^EH52q5C~^X1KfC#EXmbrW%Wj^Y9!g84&UC
zLx-$)<rkSp$4}PG@FcU@Vacjsq<Rc7I-&HWU7@u+&CE43MY*a&#HX+Ip}g*MjUF1a
zaX9@V_0*k++dE!Cl#W*nJm`-fUg2N?&7>Xhkal(&XXu`D8{0GEVP<>!+0&=*6ri%z
zINFA`72I9`|HI78akPS2hlP`xCck}qmA-S?h%cnJWz@N$b`Iq~2Y@Te?sancXo>T*
zs}3<0q@VlxUPy!W<;k;lauPctY0>!MT%jsgih}IJ59g}R4HlnpmQE-ymz!bO^oEU$
zj7%V(v;OT{v;&w0GLz*C60ya`G|JH4U86nJKEJA++9&M5M<tJZv_rsWK^DrggxK76
z7Ri%x{7Nrp^j&FR@l2k|#Qs6yu!H9xC(_;|ZDy|XzX)0n?-owXOD-<VGM9qnbvvRH
zyBSmxH)mZIIBVaFW9swgC&2fh5s8)=(B<2yL(N*hSbbV_Y|i7dmlx#4a%*AC@?D{|
z=_RwDgng)p`}biZ?W2beDTsZg6OOV`Fv)un*hA}AxcI8VVGTR0=jxYlDLzfycSI|Y
znuI$$rqC%ijsf54fi`56dsj92HbgInCJu|@_>z*cF{EN-RCmi=>buE0l63nvcQuC4
zBVO`8Z{wxVx>{IAC9}l>^uzf_MTKA1OxtPMXbdlP^y^-)*~Z(<G9v|d5)-{4zyRjp
z?&&GHZ(k=yj;^kJ<=2~xbhxO{|A!f3ymGR%ypi${0@|ww%oa~^FC5xMrp`uz{v}5I
z*3Fx>i;GX|$p|XKm>2>ATBT^*1cYXY_Fm-O;nF)V`Y&Aa`(AL5yEzWL0e&%5(oI)C
zLspwEv_X|_7s;-P_ORyQj_@v~-<ls~DLU4&-?R}pyM4Yu)HZbDz|qYp3Yc#|Qw$DT
z;uITFD;AM661y$C-4JeBrwWh(5Q#vURov^{)y;rm;Z-Zw@k9xY{FKL#cqAW#9^<B^
z$&HShEUp8I3VV-Ko-34wumyxU&EZ!+73NQx@6|6Vl~zsV)oCbea5<ST7ub5rl_cJf
zs_vdUa;JGZrZ4&J!3|HO(iXJR@x-R468XM__#O0>INT4G)GsZ(fyx=>^Wu<ue)Tsv
zAKup2zYrJT<3oqgROUX5j#A+6QK2kQ2G0Oj1*G`hyvfPIftdz^)~c5;g*`aVlaZ5e
z+q^lH-qOn}el4~&%P3!<;``jf0;+pbH5=*?P+4x72=xc3Y$u9r&G!n82xz2RkMFhr
zx>its;S&8p-~d5#)6HQxS-y^q-BHKlVWGR~4`4H?dP+~dkE?n5dx5oMjD(dTXFplX
zwR;k+E!WH$uQ_jX<j^>Jw7EDFo%R_L!J5IIg01DIKA89kyx9CQUIa|BfX(x*yK$Ar
z2L`HIB@Nv3yK_C}$AH{IXn!Y}(s~zH^wLrKfNm1v;?OQH3_Qun?BE_AKeoef0GUPi
z7jBkRnTI}J;XN1i0{krpgLg^OM~4JPn;YLxF9}UO^^$PX0H?+bX<FuXoR<cAo74vn
zUMP}Avya;Zq-@=`jf8{*<takl3X5aw^#c2Ll2yHX_s*$FuJ#85GxIliA#h)t6xL<M
z#Uo6IIQnLn?%%(U;Tz}zt*QdpLty$%!?WdOWkt3fiazdTkk0b()HXIY_Vy-ql-+8j
zHp<6}Oi=r6ZEcN>!Sm)<9%zG*z#zZm<qfSgY$8^6nc3NdN!$ORKD@81%PwB_=q(a3
zDB9WC!I-fcti@>D)Y{rwQ==SbC?9csi)6adHAtn<Q7?JP&wi1zU^~xI?v6L_OrpDA
z?Lf)1XX~(c(9>DG^iz_qNgNCh9LyCcVH*@~BsIR9I7Ak{<>#S<ACjKvfg|K$K|$*^
z>d5+X|J*V)XaWGiaZ&keaLYhM3}W}ek*J1Phxz&W(a}+W4|wBfdF}7Y;QHaifq?!p
zF=3Y{s#cUcG$bxhW8}|hbjud9Er~EyU44CS+CZQ~wvUHYU88r^n?#!d*AbS|;GgFL
z<{9n1T}N7!1~orY-3o&&I@!;k@6C&mZAcT3{$v1z38mj}a^YT3S&le6JGR#a?h)YU
zKT+->vFfR<O||-fV(^F2vrE~^_igO$d!Ly9mbpn=dVB%Me`HkD0=h4}Eucgz7hehS
z1~|I7td86d0oV;Z6pY^W_tZw=wF7|Z&WgtZRQcE?WylB5%L-^$(3iM8v$M1L0y<Aj
zpPg@DgJzX6xMpxA9a6MI<b$vRDO_F_zprI#ny8;C@#9)a<Xy>2r(S2#MdWSvf7Krl
zK%A}5A6`Edw~6ounG~Lj)Xr!fb%t=<wBRgd&8xbhx?Ii&oZtLKn9qM5Y*P#xF!bTL
z>A%C+Q7)J?d?j^L2M+Q&|Ecd?_Nc=_d#23&)i!m%MlICkrJ?20VhJ%gb!zhS=ZmoG
zjEtN;M0rE(C>PaxA-%fSuTvIoSwsw<JIdnXC)F~zZHGmNAD6tyY=T~k;o2bgIZO7u
zk^^kVGCzQH^;-@nlxx+{Fk6jd*b9r#;?+UosI>2WxR;#f9q)P_X>K8hT(=s?8b9M(
zpDw?)!Cj`hf0mN9BVohKHV?yno1Z3$3}jpGr>s7mtNuX!=us58%k7R(yOJ7I?W<Hf
zcnvDlanJA&O3Na4S=RU5v#Ysc54xx6=e<<-d>X>t-td`nZ~k+l=LG}k<ks-)dU|?B
zsz0#uY)pR@L=XU>@TY0U$pf;xxNLEz^yzOew{BozxorlAp7jpt1VJe&_Zo61SWyZK
z=OG!QwWZ#cXy|&N{f}wyFMt17CC4(^w>_5D!}vLXFLI{+S8|+ybOct0EV}Krr%tBt
zl|$B1#x0LBO1tv2;nXj<?}w{$p@Y-vj3RDe6cX*C@Me9l{)UsfP&03Vb9R5<5znCD
zvwTY2USGCuu$5h^=h>M#1P*#P)%MjDkQ1Xi$}u|}QL6wuY~HfPC@J!WB1E(1Cr+FI
zo{AzDeW;bCC#8Gn`<Zg?2Kv*vhS~aYUZ;9EpRq*W7(9F=I<ej*8e+_w$={tvTl)RY
zsI4j>F~%P>wg?`5+WFjv=}ScjRm_Eklk2%dG9_Qq!Q2&zoqatdI3qSFlB2Tu%4pzf
z>v8b#!g6w-Uc9(kI{;G|y2m)dbn-O<UFR}>w_YZX>nmjF&n7EQ6rR(AoDKG?B#4Q=
zb>EGSHavOq!i?7p$dr$*t!3Xk&$n2a&|>D3jk(D4^4b;>NFrC3m*I(X;%~Ytri2G^
zn?gqbe&F)X74-+rdA~hKXDoWXFKfxp>CJDU%^N?9<fP*Wgy+r5YWL8<9;e=yQ?F+D
zBKZ^)P9+~JoBwzsdJWB=F5d^p$YLWRQg!*(=h77z)+VQ>rq)(hMq5_k+rX>X($;2R
zY|Kbce=F?ham28^KPwLHWy9r>!-2FgxGd!Otd0Z3k%7b2zDMJD#It8BfL05PFF<Np
zS*f6`WvPkz2|d}v4`U!dhEjP`)o}-skD*rb(-2p!obxD`my;`pC@ekwrVGc8C^WCo
z&kOr-kSaQ%JU}GkWnUY@daBrt`S5Z0J7m0n?7ttP!9D)%n}qDK=jhf&+?BFLJezOQ
z)LDvpt=Gk>+SgQ96L3c$YFHtIqofwSx=Bu(Q|;{tyN1N>p_(8KB5p-PwqNIgh<VlJ
z)aT!*N67}2h{(ubsHwv^jqzx7;J9#dU4lbRSw`FVH|R?-?SYsB!iUET%+lOw{~+O1
zKryK(E?y|ET-|&BPVSBxW^VUZ5nK4G0);9rIyqI52XJ+T@@!1~68uF=@o6J`Ym6)`
zZ*@OUyYWT7uC2W(D<3v-;98KS#k2h*X2P=F+nhQIvl9o~T)mbhVw%H}4Pn5Hj;<S?
zHNU*I;Q)A>nb}!9-AB~bN1W9l-^ghruvu#mOWSAd3<i*ZrUTdi=FM%H0Jm|3vXsZ>
zWM-ZyI2(qY%@EFX<ew3_+#JPAM;C>n9UIG7`lPJv2c|z#GO~SHB`_9zdghAu?SILr
zwJBkI@eeJ)f5xu=j9vd3yZ$qF{b%g@&)D^!vFkr$*MG*Y|Le!Dp{kD?D^8KgQo7&$
z@#SpvQ6y;W4O1QIg&0OFN%Xa4XdJ}U`yDuVdFz(5xM(1HB<bqxgmDU1xmu=_?^Cp2
z_7y9POPJSC1im^~*g3c|!_qPToQq4y_~o3&<0|o!S_(S~(~l(4YHg0=orQn|e#|z;
z^BT1eHmnWsE?kbNv$dF54w2}s->ZNWl%j-#I{RG1lGsDs<@&`-sW)yA_?mv352A12
zCxDn`S%l=|E(5!BmbyHp0=YiaF&aH#yUj6>^<G~1^hhrg0AG*K`YKvqo8hAB>S`#b
z!iU3q%RP(19T&El#t9DOEw(y*uD>BeNh@GT$T?_hTUZ)SI^j+h&ko31ez9I3^z8eb
zb_2f1yUe_DAD-wHnK!z|onW~VLAC+ES>y05^mY9F{31!rF)OrwLU#=^bRkn9_U+Fu
z?s++<AS`@OJHzzy{D;(3t_Rvn=?bg$Q$9Cu-UNlAmDEE=Pwxim9b_CRw&%|e1UUz!
z05bv(bN;-=#33Q$0s=dM+3OTYbEGfjB4r~W1+p7h6;eK&gDMiHH4)&H;B6+_a}Q<f
zyE;1y^BiW^Lsk;8Z%Sn)#z6WWgO+gu>I703Ku{L7`BTuASXkk^L~TP6izy8Mv(FO)
z*CLcVyaC4WCQJ|1GF-M4HL|zQ>ASi~-<Tz&p7R{=VENPo+~IvIDgf`rxVeWQ5dy>m
zNIIu-O_~74fM1aTS7PyfaG#{LrJ^w9%S}=mmhbM2`)}Vh!MVjQN!68&NAcptDNK)K
z%+fRr3~H<)*R~&?2of98EH|}3bLLD}Z6jb@sGvwmNOmzWBx}*}Bsp+|jQ~!9hOlz=
zn>|Bhkd^R+aQ*$;=70lEc}@Gq#qIJw#?|2`yPa*Xy?_tZ%M0F!e}c?CZBt0CPKGX6
z(ZOkd=WSG$T)Vnn9o%-kuTMVu&EpI6T$L3AEjjfBj1y!l$Upfc@^LY<tyDBL5lKnC
z;8L`;71NK*tg?8NdxOlFBj~Iwvd;ql$Hm7NA}b7G7V!cUKJvlq<jGGdhXbJj(bm>R
zgizpUc!g5Q)2Cd!cYkPYedi1E5(-K?=s?UHq2B!lbw@{s6es6(m!rg)7@=b{E{(PB
z$u;~sF_9P-7id!f^*gv7T$B=%3j1F9s&kO1k`NJ1&COYo1{^tYBCgL#FX$j}5d5Rs
zI$xwZ9Ua{R?fS0|-;d81B!bmycz+f*kGyZs27!Yh{~F)Cx&O_-ck*hM|D5%S6J0QI
zyUUs$J9f;$LF8{(+7k5zefE-p&?=>^drK1a{eJQ<Qy{wDvxba5)X&hRmn-=R_aph?
zTH<vhF#@bF7=>AKcxb4flR-*j=HY#rbo+~L?(Q*6jG|r9g``!3UXphewr_EW0}=<Z
zyslw|m(*(a6~b67e789Gu&4nA^$v)5a!abj4SupMt9MeJEqM`t1UptiQ4yj<@-Mj|
z9M#Oj>Az;W-Uk1N>H6QpbRmXQhqZaugdlh3;UPT&kbC?6A2(UO@A*he$5JdH4P|=i
z2VDLQagrf1C;hISMIHzuj8s)`!KtIxIC>@nE=w~$pV>x9nM+^qwWxb|ECYi9fH?f^
z+o$&Svv}0)?VA|)kXUMlsfvq>E1gz@<^U{LK#H=m^5#uf`+Q=$$uI2Zy?OxgbJopl
zEG#TYhT@hfgEh$78v0HdQW6_E6&0136Ib+&6UWuJ1{QZPbhcarTm%FS3hZ<+MVqg|
z%#c*Ef$gNNfbCNY6?I8#Ejy<js+oDj^~^V!(sJ+ZZvBJZg<1u+O1LE<X#+SE&GV^d
z&a!d;442t%e9)z_%pAxrfFkvJuUv3C16v54PPmf$N8!eRQivve6&;4(GT(6k6O(^u
zgZo;6!0Mw9V~~MD7o<(I^QS($O8q+b0AQ)AHymONB4G`_qb+&g<_A5fhA5m`gw)h)
z&-mSx-nQ}iP;`$%!ri-f@7&>_A%^C_8CfIX62mo#LY5NZ<1HHr5lS!+fy3O-!?UgB
z$-{?dA-=*H8F=f~6GQ3pCp|zO0W<Oj0GPTB^*gL9IJ`+nsz&^?q-158C@2n`4xOHM
zSU#_f%!Rzg&X(R@1)nu9s_Sq+L-`-`^%!5IzMdW|xW0vD_wt1kVzZ0Oa#vcoi#yCW
z2L}g>e6K!2C=U6VnMe2{1x?Eg`69tIkq(>&^Q&6h)e>88^)bRc_?C<Y$+eHiv`GAH
zhzR(+u#}Wq?_-F9lJ-8O5BF?goby-(&qXI>n$<RrGX^3~Dk>`QC%DSlAtB^Cy(V88
z9c!UCNBk5nzM7wM^o3BfAsKUCM<L#BMY~z>kGO3R(cq6lNnKc6>=YH}7CLcnZCcZ#
zN^cMMoiIte5i>J+!P9;=!HGkPcYd)b;OiiH_keFD`4~aLa`*055(zWEtn@trj7&^0
zmj(IzALok%C(CWH*Uw=84v>Wq0y52>0d5`@)rR{@$TWj&+ao6MM$<5~;L=i3R@T<W
zY^(6Q5@@ibriGcA&O9rU)i2YC2#b$rl9ZH$65o23LtD$T5ryd5wM{Tv;op9|x$&+#
zh%iXz2WOXTY;2(FxLcB4RCEazJ+7@rxu;*jxHw(jh~NTfG$Bv{Cpjl6Z*x`-ycQDU
zE}ULEMn>BmZ@vATtu0hNUuKsYqK1PA1z-qoUZVRD153y-LUvlT6wFP6w(p`ErT4s}
zC3vScZ}3`ZmAK?tBac`a4O&9oeu*EF=rNW7YY4KPNRH}qFL2k)aBn9i&CbXOQ{{r7
z2%iwVaFpOl(epJQ+S(AkhLV6c&hTED(F8pcQ^V`mP)tT4Os4)dS4T~3T+8Ay%iRPj
zg&>4N6fYo9LxBpfrM!|@&QIoc;C-**8@?D6&Tb9&s@!J2yjx?m?pVXyxB7gMA|fJa
z2XV4IQUA?)|9_Y1iZK1(%XEG1Z1DK@SmZkFFE-+g;gd=7kVMLbLTa@N1#gf%s<LtR
z$0?E_edC_?%X{)okG-biq78(As=zo~gS*O}SP3tX(}zcT>JPNjTTTMtexGufo{lq)
z6S(&60-hS@tlaQZBqFyhR{fc{MbFmHi|^Bz2q(7u*<8Ksvaj45^+qJ^`OV$j+b{A%
z18^+)m|>_a3jO8F<=3xQ!4(1imGm&h+x=mxN|n0cg9CdwZy!(0lyYxn7}5x56%Ml6
zb!V}<!Bt<=VktcCVVG~-(59;gvh};Vx^|6eu@HXHLFGJ=gLvKupT$b3j<Q2ZYi3?t
zZF#rWXmpunmDUrGMf<_p8s@Ma=poBp^r}|5TS9bd+`NN=ZT+nqKf}?@xrSNV8PJYF
zr^{N@jYq2L!;?mP?a<GoqxH)@gBF`(E)X@T6x4h9eAU?FDyuBVs;z$0i)_Pl=abe~
zDX-Yn;+b6q)dl_yt<Rh9v08x)#g|tu!GMc+mCG~rJfG)k-#R;NyFt1EhuaI}K{wDl
zOP_RfoC#tb0^4`~{5V20_O^4A$>vU6dGfc&k-u<V|HuB3MMy=^R?5*lF#8cZI=T?I
zW02v>A)4xb*ek{a(;UftJu3D$GdL468!TFDJ>GNw*6;_*rt{iCr%?vA0AwSFzhJ)N
z?dq9jAG5rRuaD|Ip1DyKFTLlB`S_I`Pe!-!8#P{`-1xl8m{@FGV4`}eX^6FG*IHF;
zd>+44^09~dnXgWZB$EkMES@$pVyK{Ki64v=&u(iWKlE+1naBN#+>XETXtcFHOjfHo
zF!X1BP)rU@rJmtBV7IjA*}H;oJ7V)nNS01<?MZkdX|ZwNNh&r;Be_)k=z^-AsgNK8
zg?hJn>Z((=uTDoPF84YY90#w3Kn-vd^$HajfRa)vHiT|0GJA0iM<iw@oTc4(BK5B+
zafIn-7A6mYu7MJS<}kbz=o%P42mXs_)Bj!-JGrk|c8WLGT)kNDie`eP#3z-_8_2EX
zf(Pq-_Z~5gDNYy9UTg`b6uO~7S-$)CDJ(D~B>f2F;uUtEqytjWsJJ5KsZ{u%vrpO~
zOD8P7B2ZcseBb*6g|um025Lwxd>itQR^3cQ#3HoW#l;4{h9_rNR|UtxTl$&Vj~>k|
zEYt>lf>a-^{kg)@`}Zx7WEf2tq|RHW&cSm9w=KbQhdQ)%>*-j|+M{IJ6jX!4uTJMG
zCXY-`8t~FItnMVR-PB7#iCsM0G)5a~p{*RO3xihiu(I;l{4xw)79v`2a8c}G&l?>@
zPgU!tlhcOjqvMiA1Ev6&HK2Gwiv=x=3dx;%ON?5WR&bH&TOdz|qZ`QyYu}42t#lRn
zLYV$Ze2qbE4-e(XXkVFLqq{1898%pw$ZaGgCqKhC<t7`c4h7?(KaP#ulT6OSy1Kg2
zUO!Gd$%$NRP_N|Hk#;xbAQv7~I$)X_v|I1xp6eQs(MrK(jK&I98_cX<cKSICzP$Tk
zm#K5VjG*!NH^mA@_I6GlaVp<NiejJH*B{X@c`Cz@>Az=iND0>F{{A}=5xP1$hv*>v
z$J6k0EJ8$=5?xVY;og&_5w!!z_JTIgVp(b58EJN(XgDU7(~Z5@`PD#oN(u=z!E7fO
zHL&2xc76gC3pfKb8&FcfMO+<pk~=&>+C^P}7Ih5s4(>{DP=u+0LGju59Pn_%z6ga8
z{;i6V=`ETYS^iAekGKLAgv}R%TJdq>U~|~z(~4m`%_d=>hK@7li7sKHn)g^mm>TS}
zvT|}Z*`Ynv^c|w;dghqUQWk_koDQY@b&->>#F+>p{ScnQS1O{8(SyfTToZp%yX9qN
z8f2QhH29206iI`#%nSj8*Pb?haV*j0v*HstRT1wE9VX%Mexda5XDtnDHgaLwI{zyr
zs_Y93;TjB<kE)}|+$FdeD3PQeD~+QlQzj{eS4HKf)}z#+ee0D;MREzQOea44sQS)a
ztKWh2HU*QRX*cPF+k2Y+Pyq7lASeMo)-(*{EFov|PS`myCLBehuXS2_=E9{c-3R)a
zhdC$@AIIai7`Qo);Kmv0Qb23Z7`#>NI})iNl)EchH>ApGF_+34&@_yPvH!AVqZRoc
zA|q-kjYD4`N3`;|qpI@VfBRwmD<><x<+!*kk$HsDg7<0#O}enh*u2Nb1cg+|3l)iJ
zyszPue(>Pcix=kTkdaejuJlfs)?aCkXQM?L00Rxp-^H4st>xwMd-Y8DNkXb~^Xk=&
zfdehwOK)#*l(5;&LiGcQ3QozE(aoD)zz+$y%wFG%<I~m6)b%C@HIlOCY5P1n1?2V~
z6^Jgd{`l6$j&O<Z#0TLDx3zHTY>nX)Njqdps;0bogBVMSRsX{JFi$|r+S<Z)PfwXQ
z)jyIv6<8WFAMo;hKQ;XMJk>;38mpj!+thRD5B6Z_c?s?ryH>WO!7YL%^Z7_qu(P*M
zPl_=Ayzh+V_RX89vITmqhet=HL`4;k)2z01$HIGsP8}f6*w?R?MY$Zc5ICb%HoVLc
zM|h4|cI`TWJeZbIv?KZXVm&G~M5Lq=MrSoB=mv#JBPE@AlmGPqL(YT#7cEVFeLWO*
zMfv$v<Mo{#_6>3S5E3AD3cwdY6-!G?d>EahyWE|fBT7Pi#gV4@*D0wV$L23Z;IDCx
zHttpT($VM>Q;?au04&1kvL_%QBqYS&e+ecOGr|Qvn0)DUbfj*K{9cm7j&j?bZk3Ei
zaXpBOWFY#jp<PM%QkbHms8whN)g+}|wXu{Hb}|}kb#3<0n<5y<Whu2Y3LZT=_vY@p
z>rKdF{yM8Pb`Oao{1-<aPl=SKd2xrsj+u4&cql91pOT4#BG!~RZwOB^CL%XCH|Veo
z4e71FR#jC&fSe^C)>!`l68mqU-h0a2(TrEWFst7t=6JBq92~vaYietsLvnlR)7Ybr
zS#D_`9OSyZZDcsfCow)=+<>A)GW%do+bP6MLgt4jkBcicGn1d4{i+f1yKxo|5z7YW
z|8oAJrKEh{)^;YwNO(3u7)cU5B}wCkzTHEBCo$r|6Mz&GNdC^-+Ybv$PP2VFn4@AN
zaO9DOFhMfPlqTU3>mw(cfElQ?B<3pD^B~R57Ib^=qz1H7;|aaYk>mGKgMqM?V^7Jp
z^;c(#o@L%_sidnuMx!?y<1bWl;s;eZlr(RX6cjMEa4b|Te*Z%Yz|kL7`D6D<w|@%j
zvbJ2U9}<Vif?q%D$=8aA+&DeVUVcUyxw#?|5|K^AV!cEnTl1{(dn2+d4l#0-3i&oW
zrM~}VJ}SoSq~YW^3W2S&b8CJ59wa|i>ty9}e!42aMR$V<YURJF1WSI-GCwc<UuT)-
zpr9R2Tf#i4E!@Lz`1wJbqsm1EFb@6%2pNG!zF$AQOG-kbFgKUdqI7wA86gmmbR)5Y
zBtr5HGbRvRU*LI;uC9_s?wHJQ3KR>FP*%)tqz?FfKFpNQr0Dp8Qo<1i&#s05?Wqc{
z4wFVeUy^r9gkRw2m^nu4k3r{gU3ir$&DlxpH__*Y9vnQIhB$mR9{oBg{<sg(G2=Ev
zg)JY;d}4h|Cz@D&rfXOyeq2&hT<nlL#Zp(k9IAXLD#{h^Ka9ah01?Sidg}P)H3mKp
zPfxgrjq0zZN`*Uk&&KtBu1vC<Xv7-eqdyjW)+NIa1qK8l^a4sBoI#awWwp<<jZ947
z!oQ4u##eC~!2RhOfRPn|e3$WJVmNIG7O})lfak$mO-;?hg0pYd`D1%MJK+XEiwKL=
zJ1i?l+Xm33*F-L?xxUvBn6Lg71dT#btDJlc?-F!0F@skNv84^J|Gt;LS#lq5Ou3ce
zp(-ja2G|kdvZpXVADi!P-M$Tv_RHuAcqvQ}im80R#AjXJX;-3&yVHGOt5XpBPL|D*
zRm1|owZ})q0Zn47=;p-88h~^WDd>v$?;2vF!D~UlY@qdRZ>P`ixK6+!*+MLX?jZ_?
z4yZD|e|6A^`G}f)VVF&@?1Q+S0XH`K#S=vb(x_=49`i?sB41)~=zJOi__^(b6~02u
zCx-_i%V75tHm~fPTV#8}*V9N^KB@a{gnm<xPtVaY4-&kpyC)#EVrIT@LM6JZ8sP?c
z60238;eez~3OO6TwALP@J>eGQA)g9V`pIZqmUYc7OdwZc>i;?JDAbVe8;pdzXku)v
z6y;S*era<&{<z@JS;maGpNM<Il?=b5=|_$pO`NVmt~fMZ<w$kj(p7zx=OZ)4-x4`~
zpK_)`A9hug+pSzWq4zV!A3r&xLE&3cQE~pt`dy**#vJd%$8}v?7m_m#riHj3bydhc
zeiC}Z&T!G^?OMwT#ZSGmRjpTlU*EGq`2NS1oXRQ1(Bg$Sl@M*3!hsJ4)JdL?%d#7}
zPe)t*+P;FZjB9OFHtIp;k1rX1ez}vCtSf6P>*s7u_PGikKJMol7SkK^iSfyW-+v*u
zJ4@*+EeG?hSKZyB*H)XLoDVe050bd=>P+7?`Ql5Qj{Vx@{^3VOU3u%DmWx>(R5sjM
z*#7(H(>X8q?d8C#+q(Sf3OPB;)BRFLLRWf5`$orf=Z0qMMa(bJ6fPNxwKM<v@1q%h
zw0MWGvdUgut7Yu;tzGG3#nf1`u)6wfJv)te?9?funzkEpzt$8HCQ~i)+qeA|7vEKG
zTUqpyL(Oe%3J;gvwk;s##^cA`-v?FLTO@eA6F1-g{lkbk$(?L#iz>DBfA>LeWGuGS
zQtiSi*2BjS-M|0Fd!E1IzC|;M&=W?Vg(~u<hQwgHCq5Q)wn7H3XL=@0LsEIw-_P)P
zU0%Gz`Rmse4qi-OZ5h_(Q_yMx*tsX*(dcOMsk^ROkB=5Pei?fDbf7IKpV6nMvlPZZ
zA^DJ-9jOlG3qp}ycN@1;^pr1Dsn9*^<)8fXx4UxuvzOBESMLA!Z*H63jn8|OujSSi
zR$y0c^QTW24jP15)#=FYV2PqmA8r3RV7rOts@z4takRy%7V}iE+W=|!a!V}rSV)$2
zcy8rle;~}$T)rD>)VT<Xd7P<d)-W(lj*g1Bh5_c(Sg$az+N%7}GzSJCS;TV!ca!kG
z>HBl_NyEeWxj8JLwkhl!P5FjzPvF_Ki41KB&z2B8xSOsb%vFVr0=z^pc75{Cfd{<r
zv()CZYEVU<1Vjl96d~^#up*1!%+HUkF*#q*w^!t4mxY(Sf`~|NUf!LQhyNJN`gf@#
zmr1$h<40OnR(oFYKctR4a&ieNysoYboGQ1^7Ir>*#0Q)&E!H%r7;0^|N@4@)mPqrY
z8e%H~)n<>>$($S^|ED;8!crar^J63VT|^i$OPHcQbabd0j?9KQ4UiuEJ~}XPy#S$?
zbGP{T_^MBfh-pNsM<ylZ9OLcr7~flZOn?^o$!L^cTZ;Y&WoBkZHAvRaBq%uH^%KX~
zh$e^Fcasp2z4RgIPSGbo2Cw4?y|z)lbvcx=NT?x%=oJ@BmM3zv${aqfu(oJnNcd{U
zZh#%Hf<r=1!6~M&KC7@kps-feu5^GeB%ayi%g@^!-7{{pqx-_ng9i_S4<%59Wizjl
z$IlzE3}rGjjox525h3q2^<vv18Cb+q9-|M)6@Ca~_ImOIK(G&j0$}RLFCwHvQwRtb
zkx$qF0Kb_1dTp{WP{9KE1|e>VGkOE+D&K81LJ|D^(w79sEcDm8<XnEA?T@y7{3wXA
zXHQ6Qa5==eipgvOOe`!z>Pp{%b_W&kh@~LA?T{-3yOOwLd3lat4V~4zy(`oo7(cz3
zjBM$iub7eXvohhB&9mH|-;Y!n*d~zYZ+qrUdb0ClHjA_Y4}!r2pHCgN9v$(qV-)it
zD=mm893LGu0?Fm%)0vff3PErpA`ZxI#bDk2K127>srBWy8tzTn?t@=H^M%AX{Ce6B
z8;&c?thDfGB_gv4^6k1^1zz%%#l>_&Z7nU~DZDHp=`Eu>-jGgG{=yhat6^BV|Ne92
z8S}@^0k-P}ra5%}N;HT4l{jDr$SbK&XhG;dQZp2jkEOS4-+T@E4M@{SETfQ8wvY(>
zF3xoP#ED%xBmMn~>rC|Y_eegRlQ;=IFdZv9YCB?QvlOd_&J{9p5?M%dVG=^;TvYTO
zs{~-*urHfd2G@dQP7o`EH`!zL+{bobMRV9<TwElQnuhH<v76%_=Qc6Qpgm0q%zm#A
zX+v#}ni6!s(Gx@|zAV?!f=0gif=nRr`rKX3A@5mb-JU=4T9`Olr2rrTFgA!6v>>Eb
z4kF{;i6Wyn-Yv-B=lUupt1)X#!~<%=?};HIm6s3W8aKB!o??vboB~ScZhfV;(6Yv0
zM;eQ$jH_#-Ba1%QYjGtX+c>J^ArzaD@$^o{eRM>d=?~Jkn5TrTeG&3mdIR~u+HzZ^
z_tKXrf|{W6$D2w5{~_}o3nU0$_@?zAO+X*3Bfs?ZO+!)35J#l?Wjg;i)~;W7F>ZdY
zy81?Vcneh1_z2SyEzQkP^!C8yg?`cnIT+ZRfHqNxk8k<Yq&JMY!-Io}<glWC0CfNW
z^ko?SVzcVYPoC?bb^?e#(eI}>^6eYq6~4kv)oJo(OQln+h$o&neSLlK&e-PA+0})M
zvL}E;NU4;{s3lr1h7yCGxu4j+eada|66f!#)xt;a1;e+WFGw?uSui~$8nwIk?bGK7
zQ&Lt2u>%{`4l*+EHj59UOqX-kmxTgSV6ZXu{8pHj#!XVQHo3kwS#yKHa3DrVd>>`#
zYtA#<4BYE<2p?NGQl#8x8O{Ote-Tn~^vDr0BX_Xf5U_J|D`)5;*j|U`)$lBSOC02I
zpI&%?0QEoH`*j3x2}>N3GGwzunk(oGZW6>VLJ$3m%9uXy0iy&`_cghw5R3wM2#k_(
zCvHXD&a!DpY`BFxDg52$PnN@AN3dEBB9M4_r{?EfoSfd&*C&%;jIcTQi$-xfMbQ%v
zQ0Gq~Va{U?cRmnKW!MDE6VZ02PaZfSb%~LjJOt()>~+Y;wx2&wxyIFZKQ}`?xA1du
zH!ZR?0tc^rA2bN9NeR+NGT)!Y{fII6ey8r0eJ8vee>2Tkgcoj6J`KE0ON+eXSWX)p
zayEs=A9t}oEf1@7GG?QIuM+N!VCAZ>?Ce5H>fqEm0oVa-I#+01C3%<Lj?vl!`!DcX
z8)xfthnw%~#g!eHCMG5pXVg4%hG+E=Z9s~3QzFvD2&pL;&2S2vZ#J5>wu2fRR-@%z
z;SlKOv<VsIRbR`^_fxiiUFkkMgt?V&uknrJ8!}{<z6#giK;22a#d0s<?nXO~+dLhx
z`f`(E2P*WOlTWzsbB(;_L)sMu1ra}(2s`HpZR+oPK4B@L`c80`{QIxxqPBTkj5xwj
z*-yOtK%KX7IgHeaxU*%DAx1}OiRYqFg{JA7pDj`dN#=JgtrvkEe>4k|F+(i07bb)+
zh+dj;9d4A}aZ_!C?8lo$jce{3OJ4OjU%coxoZw1Wz91t54=>IQ))m?2@p)uFY<C~N
z!76O15hhV=Wgk)OX7`fn=Yqsl4-aAvD*poGZ_e-UPMJk9)Hl#u11i<gk|R*G_+n%%
zHLvo^xc4E)dp;9>uh&JFjeP{Es(JA7rUoCUPpf3I!Ptgz3y&tv;VoOYV$Tv`-w_P_
z!on22$2h`Zk3s^|;-cH3<UhqYbL?yr6BCFJdVj7ET7K*s(vbKigPC3xd+eOXUnL*B
z^R;DOPmgI8^?|9+TFjLQ3~_VYuk&_bT9;1`ciKD|eCX2O@L+Rsk8+k#GC>S7pxNP2
zLizVo;W`}~C%dl<Y2P9uVNJtv;+&%Nb@$0eHOprFkf@F#=n9|)U&xlq!H!^~G|s5o
zOlcxef|VoYFkD}hJAuN&yh@W!^*a3PP`u!`iqWVMKPXKmMrw5K1Vz1HJuyR)YGmC2
zHQ}x|m}+g?vIQFzii?WwMnphmbeu--A1G=6mU>IkKm7?+GZ<EJM!9)-G|PSc2kt7K
zmR(Y%q@G^)-E(Aq`KK;s7|8ZV%8?q~m#kkoqTW2v-lGHVa0XP;jAFhY=0a~{_g~VP
z@U7<Tf47mV;c9eqAu<B`Fhpq}BM{QI=*HkJzj2Qdm8<Rl{`j#Vj^$-ntO$qpE%rma
zi8S|6NqI?WY0p!K>!!y}XDP{Bg}Ma!k~+#YGQUSm2{77`Aq`D_<}><SPT@O6<-_>c
ze_aaOCyDazGnN-FWb5)B?zsmR7Dd7BAPnm?ho{|Sfrc8sm3|Rt#7}^f#2J4y-?-SD
zE;2gEFAR$Ac!VmKc7`r@TwJ#l^Uj@m$B(nq5W7ex9AMk;wKxT9R6R)}W14YxTI9m~
zyk6!9Zqk=2B_&e(_eXap4$A1@IQ!{U{)QlrV0UWj9d!>m=eYaY?LU=TVe{!a@Is&Q
zk9+fD!;jRGhj*O-+F~MjW{oRJv}SqGxW}-at*vG8;lP+#%s?F{LwHuuD;|U0`d)-r
zTT9^fzDWlpsXIE}Swfl77mT`j;|3z?qisEr8XAy-Ozr=wke|Pg4|oGvZpUaOkUBs^
ztT|@pI1JDcO@JwK;tn52bOsu7gu|4%PO_LM8&x5VFf|hC-IPR|hZy0GLRT5q)smHv
zA+XNK7;bum{rZcnhxy#}h>2FgGX+2*F@<-CZrsAZD_(ZU1`qP;O@$v<mrkdYG^(ZD
z$$gT`HYRX{&ZHsOQa?|zhR1aGn;^oRDC;mj>hHfDP(Cu4a10NeN17rC143{HO7qyr
z$Rgt69-pc-)zzJQ*1JlGld7-J$jp4htqw^A(2BWQRv&Sra8Io_c{$qXARUC_vlJ26
z-`8gj=l^nl@W%g->dkcqZex9Kcq>t*X`gbkv3*AFCU=Id<FI)CEmRnE!Z<A8AO-Sd
zO?X}*$U*c5bSSDOws#b;%Mh&z5GqUu@Yd7XON#DEkR6KAaz(%)l6tw-oZi&j-mz=O
zS!Cb78gU*6CjZ32upkZnx<#c&N-RmZ0?)&*j81a)R9yB%M~otglF7|aPDX}kLYS_2
zBVWTM6_B#%wKg*H5MCo|>$L2MUsu6_kQBOb=7Rpv<HT$qp%<_YL%F9(YuEi0hfaE0
z8Unof+vaZM%p-L<R+Wn|dIY|LpL1XFyJ%-Yw44PqzY6q_w|d6k>-US|-DOfpD8x$y
z!!abN(}6oJh0RjzV6QD1PULhD*^b-OiOd0P27^-z6$AIeGIK-z{vigCctBZg^GaS>
zsQ8h(m!AEjQ}F9Gf3^R%l2$%POk7szw7;&utbBbsJFbFoxY6J0`c4G+{;KPew(VE!
z?QHMwD~<%Bqr!T*70yPG99UEqXHqOoB_=A0xRWVgfVe_JTA3dZ0rcdOXFlVde_Pzi
zoIvehj~^>NJ@(SYBilml`jfT6X*WD`jKOL7`QhFh19OM;D~xw#Y!#&!J{}bs3P{bW
zIqnAK7bG6Q$b}OMc=K$6Yfp0V=I9l5CaTfs=vu>3z)JM|<}c$crEcxmWm$UQwyFTF
zfv)b^LSvlVG^d$r3#k@7DTkCr>&3a6%xVlV^(ZD^Ir0-Ni7sxfy5f^v+c319fqTQW
zjVkU7!jq!uaA`K!7X4bJ{6DSpv4f!{qdRx#^V!dQZr~ACe9lJrZf3Ol-;xrq(X~Rw
zTQOa;i@R4$^Ze0=>s2SN->H-#cT?4WSr*xCLbvOY61`bNe<A#bh_IN;{Z^+K069tu
z4Y99&CW0&Ra@^=UH^FOrmZD3V?CvDK_45W3Ev<t;DdGgdPAZnKn<!%Kq#r5eoWhKc
zci0=QRSRDr^~jyX>rU8bhQ7Q>?#@=GZG>M<$5rmaL>~>#>7^eK5hL0=AmyK?Nqb(X
z)fhEl*7NczOMZuQ3j4qk6Iqs@QvCbbpVP}&lOtjD^%E@%Uu0R~VJ$1`1`MRcwAMO}
z?io=8Vqaugv>s?9dYp=Od2x|G{K$B{w5#6<?EZRcQA<{64AR)#?HV(kr6z<F$9N&d
zMyob{Wmo<t+keSoYn)AL**R~>(u4`_qfMf>XVm_~WEi%-g1iU9$Hie$kgJ%XyNu9^
zeUg%dDDftrwR|j6;4fk=Ke!`&L$kW-s2e|Ft|uT|7N6zm$cTv%ux)?#tlO7JgOQbU
zN#4sgYHIZapC}^q$mJI|^qQNQeMUIl9KI{ErWx)kU}(U=Sc45sh=jHZNWsXm520TG
zhM|Ij5JT+syXDY${<p2rko~^~`uP<0#kdyG1!(%uyarQ%^7Y^tFA)jGC6PD1z9e*L
zt{53(PDCdpFr!;L_u&yzK+N=qYtFs?G6RVSb~b>X3ZD)ufgCs6D1WGT?OQK_(l*zq
zbzu@!tUL5RIxv_}+w86|@4xst(!kW_?Af}eCURop(-N*uPH<uq%&7W{-A+x&lQxpo
zV|{S&soDmc-Kow&ks3z{>&Q1mmxlHx{iD=pec5CX3iI~i<G7>gEev<_?Jg2p2*K&8
zsds0^qhDnvUN33hlegO9QEtjbb({v`16{tz`XO%Dqbi3F>*?we`6?zH<)I)#QdF7u
zs>h6CgD`^}<oCk91o-{!N!hyCbA_=J1Q#z%&%4b2U@a3%V$L-<tJa!xY-R|!fhT}{
z(DT|~`vIfrjQ3{=A}|ZYpWK#jh_`&{Xt5T?usM8lkdZ%sg4NG0*U|m^JIiW8@gV?n
zsOgXR62&~mrzp&aj}tVVdKI1vkNc8-af_J!aKZiZo<_T9y@_^K#GC`kAv5z%Vqb0-
z;Swi|EQHKsWT~b+M4l9)qQJ&Ln;7yWR7ndoSJts#wS5PZ5#aIg31popu~8VBHT@~5
z3{M-7CSzc?_DwOOJpD1zOuxFhuWx9m2t7K!70Mwe2~tEbEMsSmsuH~)Q#p}$^;YxU
z8&e~vnx|(O3AZYZCT+nJaS8jSq~1qadNpGG3>cl{V{Y*NBJBcyQkw?Xz5sbL7>o`p
zDYdJQqMRbC9p-89HJEBOG=bE(0h`dEZT>zzGm}EcKtCmu&X6xDE^cXMg>nnrDjO3e
zc+WBl&V=_Z7aeN$0)iqVB4TsiRy?%0xc}2II1q#*?<FF{yW@UCvk@!KY<|N`5uFuP
z$)iKd3Dt2XP}2TGmuS6M>9f++i(RI<5WiaZG6oM}XwUl=;ZDExAU8f~X!ErGPq1aQ
zG!c2DSeKKD;;Hud0@?-$=g?cJKM<9XiA&*?Pg0PU9vd7)uCERMgUrlKCr8I_M2N4h
zE<kgT!SMRUCI`H7;6x>jo+9yCJ}C(ss@vP^j9c<MsVY7W{Dar(R<vklklTfU+!x~z
zsu*x1#Bd-WCt5yC0z>$yJn#?z3&QBP+rjSZ>!Y+jgRx;cjWdWsgBS6+4!zM==Va&<
zBN1oyms|_Bai5bW$W7fozOT}}4>khKA%F=R2Brb8U_DS)k|=pSTQ}F00xE6k9`!o%
zhR|vpTvu$K=UKh!?Ccb03|8XE{XAB9>6<YCTgU)_CVM$q>?$nDM$opx^2mKmWH`RD
zWw(8BIUp=LEbM(t%XOjF@%f+9y}R>0JM8QSC*x^}HpAzF8R-6dVNcaMlCb7^kITPj
zKag+F3;jzb#&RKF&9c(Rcx)lVnmai1p;Jbf`AwVKJA{X#kh38&u7LY>`;ambHw`rv
z6*Q|FrP2I#uiMPTLNw7yfb)d5`FvJr%0rz0xDPFpNv5#M_fdtf&Ej?4-mE?t^Md|Y
ztVL)F;iNY6J5a&K%g=v5>2jRF=Cq2)=MEw7)uE5Vf>bk8!VBQm|N8>x4bL@RvS<+g
z)xZ5R^wTV2!_qpS1|+4|NZ*-Uzu4P;;X!0w?<TT|4V*ux^sFnaTJ|{Bch7o@gy#{R
zT0HFdJ<lq%QpP^uX*>6QZrL}>@5w*Do9QNcK?oBC9wFj`#GkU+w=#sx>4XG4Bsepy
zgX_4{+}xCl_RE_uA<<$(gRXVw0_9<X!0Y$wEvxJrUESFDE#^!jvd=*b?>c!WHg>sH
zM~e^~1e?8|pI!Ttjag9pVL+Mli2c#Jbt@iNq%Rm5sU6ovRl=VGiHq&RfSspF|04C0
zq=`*T>_JFOOUofvy8ej?@4-6CZ2c2@dS4K+rVASljyBpP{<o2Dovqr_a}{GWK6Z7T
zcjF>v`H?a3yW;7!yKJKR1BffQ1m>ov4-y|PJtRQOPZO}V=IyKaLORH=T3yvObcNYQ
z%i_iJSN~LgQI5`g{1wQja;5*XWE*_}lKZRC8v?6(AnM<Z?eo<0?{{@vq~*-MffNLF
z!c~}3viRsB&CAP+*d|m^Sdjz;Z>K!OQ-*j4yp%xw@MzKssd9(^=~kh?%PMn80Gk8|
z2e3IIPS<r3AO3A!-F_^@6S>gH%&jeb_{WH>f<iA$EXY)I*<30TIkoK>V#VR^h(KOV
z8BWx)>+`PK8#XX2t0^7Seb+7=Hql<`qmYeQ+umY(R_|yNe)dmSr)84mf=jOZA}q&c
zEW%LWlB;VI<V``U|B5`&>L)vda8vAd#ztRgsjgqU7CnQr9-@AvhlA<9E{GK#2&Z}A
zgIfIz8GTms-XSVBmJ<TX#@qLNijE{)i47lHlrldcksYaNkk<~98p|j3c69~P4MG)n
zoaSc=n%|eKxHJHd_m05aBhE<b{p$HklhMH=)c?IvT(k7UJ1F^L^?QC+<AIy?g;X`n
zs+H88EzvTD;*ydmefh*;oIs?&u*@|<Q4~ILpnrUvKT;hsJVb!2MVSD_z&0i@R6kZH
zeYB#wOI_Z)eS3TD6T)CI)C@gNFFCkU7ZeopswS&C@@*j%;iEN}F<s~xK!zbyC9~K~
zk!2!CzJ0p`wp@372RK1<Sk7(girx|wettfX0sNUPw(E62A++pq$^UE|7ug*)0VC3%
z1G(|`QL_9WcEK`84|(^FBJ*AQE$_v?ZOiEp8>_jRal|iO#zYH*3zHH6GJrguo)E_q
z_OReOV7G+a;o}WL*5>qLa~zA5yJ92)!}MPssnz;<F1h7>Yym@s&?pdkAhl}RG!Vm0
zOzZ(C2}V4m&rjoeoy2~DhYv?wD_5ZVO2AxdU=XDl0Zox<sq-ineV)zZUJy($;<w#O
z0MwVS{0mS&oO4|Gv+GeYYjm_w%s$G@gjVJRf<s3}cn=>R^i>U=pl4-Gk4!>UK+Qg{
zH~9Ipu!xBHV;Ny@$&WnEbzCnfC@7|oL9&bXw6f9&*`NxgzY@Z|jS$d*Me_Uis~*<{
zQHiV$Cnrg1>E(sW^;OpO#oF{F?^1N7DG#A2iYsgD4`X1E!UtDWS26xMgIVuOs|;~N
z!aI5c{L)A|i^-DRzaJzVJvH@xT|Ur=djk6ZlBiXrjo12L@mYmNG0!}#&96QMZ-LSi
zCxF%ToJmbunwrQ(ug)3Wernv|A0bnA?Ap@J%zqig{pN9|CVEatk#yrt&C^N^Q<LX}
zV3L-GW^sABw4~(A;9xWpCEd%o7C;^7DX^69$&*kcZb9RM5pW|94vpilCz025`1rq>
z&*wr<udy@4$o&4X>$9}V2#>SyZ+{=YfBg4`??jRR;qYB*@z3Ggd=-lrfc{mPJ9;XY
z{x;yMpH%bJ#+E2c0UCP5PAlIZ{rWZa)cR>#CBK{qYw6sX3j~`iISI*qAHrYkzo4=K
zy$tjot(BQ9J7j-T_&mSDLX(QE<();Yq|QI4Y$g)I=e*)R?)Kxn+!d7_hHE0m8NiFB
zr5PLk_`hrj{|SmYw>dfYuYdeEe-)_WJ`uW`O3l@^c*{w-<Gb!%kTGl^;h)MGdXjvP
zrz4K`*+Ad+O|FDL`P|~fU8CN5Eaj0xlCGlBjY$$uw<FX5(lsyBDdf3);52-CVp7xY
zN?%m!zEjc0@8YbKJ>4drxN$t6AxI+Js`tNrGL4PM+4?dk3wF!4Z;7txsXoLioZ2?N
zSH_0u>(8fFXJgCc0-M6z)vHazSd0ON3SOswcwM~?L0Aluq_Y$I8%TZA($gg+B*5Ok
z+ji;cvuA|-YrrOYdP^%SVjt!_$}!dTWy@2IY8d!kX5^3fi$a;Je=7Ypm}m)E+4w!f
zgs`1Qc%z=YLoX05S@Ipi`5P80#)&2r&2@G1^;KIl(sK<Jz3w?XcjsE%G|rwF8EMBm
zagvuda3}F$AD>FaWV55g_)~w4QR3<mPyGWp|9@J$(y*BKw{JO|2&a_Rky9z6MN(%<
zilSXp3hn!%q+P4zFlZl#R9b0ol(dkv(4Hk}lWEhUG9`&<(KP?ha6QZaxt<r#i)Y@r
zx-!$uZ@$0#dw)OoXSwn6k>=PYY0ALXEA&+^q)cD$U{;s?bfhW&n=iDKm;j@Wg9&9}
zK{#TF`Bo&46aZ2kZEb{!N?Kjh6Zi4*($>+T=H<yss``C2Ie72_PKW(<*kk>s2QcbP
z{^uXB&x42Td!W3@^o^Jg_rplKq{IgfKa*^}_Pd7u@Izodl3gS~A`5hJkq8tnC$$Jv
zl*jGhB!A@MQs7}%6Zv<vn_i1SWXC`(Y~{aC*)Lk4cVv`QSEtq|T>&otpYlrJ^+!4V
zJ(=fXKMiKHXM(FG`<>NDU2v)VzW~@jV9DnjjuW{?MvZmzvp)hgbaq(FVzor11Aswg
zp!L0fMgQMOvm~oipMnxSKfaIK!Aqud@v}ZcdtV=cu*59$pF3no2oZW(;T&%{U}42=
zAljyDmzA0<E_*6a;E2wBy}VX?5ku`^?b!Csl8VP7JPaqj&vG8ktPBv?ebYpH7bWL%
zq4t7-bJ*<*e73yT{yI|E#V;8hA^7wK*BZaYWj?~yz+RdoZN+)jPy@H=m$Sai$H75K
zwtl6OZSH04ZBa2X4|}JEWzlhv(=bqV&&b;ay*13u@84&U%^1Rj=L17!o*y>wntjWA
zY570}?f25lxjB+tBcJ{73ltMf$0_`BjShRu`^yRnqB+_TuiZP3L1}wDuv{@QvDjlV
zBbk8`fxhO{6Q9;%5<}q__EiF@v_JJy@BW-AzaL1W8-ICAolRR$FN#8WSM11=|JEEP
zkp^sN(cP6ZQN|RyVF~?cIM6U)t-`t+*x9+#1A)RPwTNdUfp@x=R6_fxkf^jZlSx+Z
zAzs$LpH?R<C~Z7xr0PGto@F1`y-@mMwAe%v<08DRMRTCu4ULV`rp508wgGVl!&e0q
zIvifW9yT*Jb|^AIw8lB_cBs-IQ#ki|@b6x!n~}}~j1k@}<3L)R7*75VOG`oTV>9}F
z>!QDYWS13!y>HzzMWkm+ilwP3-B)Rya3fd95i`A%MCYhmdg35h+&DF{Pp}}$*xq6L
z9A<43AcMq-V;;-ihU}wLNW154GuDB_uD15AJ+`$NYhXZ(LuJ}$&)iMC)Ngr-6@?R+
zK;53}W2z2AANLDdO+X-l-w4>`;2Z#Tz;O}ivRW7@#!4wXk&Yl{R4c)6j)Y((1+d2<
zpeRndd9%csU#9r<nKK9wsO+srT%dWu8<sI*Z0v_t`bL#EBM75JSB^}&2xNh@w$AM{
z+&woCYnd~~iV1}EqxBJ^99phZrhusEwj$PH9gqOdaKDUf*N5Z1@uuI3i$jgf4n24&
zvI7i4oP{v^Kx1x2T3%~PjIF!u8bTfW<PtJ09E5IO6HPGuCOwG+i~y`GZtzH%z4s&n
zNXO<%eNyozy61Tn;^SbIj5-9=3c}S8@2<{gvf#MfyQi;O1l!%ZD;S<^gyC9z|CCg3
z+q_)P8HJ>N_i^H@A<wY6YkVZ*7Rzt$-Xo6Pc?&>4AYX4zy=b>xw96h}jj4v&(CfUF
z^cNQ{T)?MMB*eg4VTdkEzQ!r4K{yapT2wTHNIDQr;y>*SzV9oujb|XVR=9fBhZyDi
z+lYNw9HcTB-E9~{X<;PJyr^8ZKVV~XXmIV}5KN3bpT|n%;)HtP(B3qb+x&$ZvicVM
z`)`j6{^z{ey>TTlQ{%lF)b4)adv4`&A*t-0Gc9d(cc^||T5>o^zs|#PCTIgagDSXl
zYt3Nbsuc&{#SE$__5d%m3K&{*DUknMRQx|4#@}ORoX0mlIyl$Qt;1aKU!DFqQ}~3!
zk+hfh79V?Kkwfa-5Dy*@%6XM)GyothBA^Z6oDdYfw<FG%xn>{nN&xSwY5hmjN*B8`
zj=qbISg~Sto6V2VdK5em-v0J}xc150HEa0`)jxjo-@MQVtQ!^;ZVKjTR$H8)(dDSV
zQx%UbAOum0Aih?E3x|*w7ICQY+A@uM<%*{z3#BvDb{wV!Z$O1gCVE}u*z4Mv%cA1K
ziUHabg1l?&atxNmrmH}=)gDXZiz~Hjh=9EWx<Vu`;d7vae)sydCj|6BPa={4Wg6;K
zEth2M^`9{)_NXy9ip{|izW>mofCwDzki#q(QC<362aoNBS$yLFT<~>Y3da)aSeKh0
zVH^rxz9RN9JO>66nMTn-&)#%*KY^F@a2$2Wpn%jR6<x;BmDug7paawiFv`h$7SEul
zK0h8){C6HtN8~^e@LHsY<HoB$)0f4%O<>7;w`;=k-N%r@s7F|Wm}DSLo|Bt<AEG1S
zZfmPja8~b*|IQ~<T29EV!11&iM#rF*Aju8vov=n~rK`O`%K;FAaO_3<`{)rX)5N#;
zse^skjFWvbbN@wOy%T35sO5ql_^J{}!K89y9znqhuaQQaG{J<Jrj=af_<|>~$U3Yz
zA>idtfW0$T#|X%~B8FmH$ZQ(|^D^m3Av3ECn1Rcp5~32-v;csDBWD#jdJ_f3qo-8+
zp|7*UMpEA~RsSw~S4Rhhq><raXv)zx7RuATD)w#{$7t<UqP(siNlOcpF*6vV*3=_D
zEu~n{bOcSiJ+2x`XM-Il7oKDdDIP>k92|d&Q?IB$P&r<<Z$Pco)#@&-8AR&x;^Hsh
zX=5N4mzwkQ0viDtf}QaBxqko9A5(v&JldT!47LpjA^_7W@qq#AlD<yiQlInYd6RSF
z`gK6^PA!RI_eiI=8hzLbA$BLl5HutBhiXuSrXDs>71nn3p~N7>LmvB$|3IKKR#d9O
zo{^SDVyNG5qY3#v+{b;oT_PCor?=xyxH9vN4skj%Phgs#EBzY6QFv)EqH1ev$y?rF
z=AqN!c-jmc!;EWtk)L$6?vC6ar;dHz=WjPZ)__n$TVa?N_Dy|qpH5Cy4*UX#5^Rs4
zN4l2KR2Z`zET`#!^lsZNp=G^OYT;P94fRO8C`zzPcO_zv+u=!-s<&x5y+&WdOr)a!
zUdL18NS8_hc-U^{;(~`wj>W9}`3hHrF=NjHYaLX@(!EoWPx+Hc2<LNxAOjnqQMO*W
zFTEWTjLX^yC+s<V3UhN`E4m?94JHUCDB-52<zO(dc=nxq#y&DqQm|PW#f`J^ywdfp
z9RVe6N{QZVU(7XM^WS)on7-FPB|A415DjTlNh_t9&pt{!UKjX|veQex8_(YRy?gjj
zK}`!hL$?vs0bOgnGw>)3bQEK5jAcE(X210Ef%jipCTTuODFCM3TWh9LI{Yd^U<&|e
z0%7uYadF_v9<<`Df`S$syXg@=9v<;ntsv|CM;oy3_FqvqO{1Czb*cq%5naOF6q58;
z8%bqrs^CZtwIU{p0^t?<Cl}RMIcJUlfjA8Y6k<5aAv!`YX$?vkMlq8FqnDNmUtP-K
zE&``h8NK|I`6fk(3@sZ7dJ#Z7?$g@bYyu-xS66C_6U&1(hk4$)lE%k>narP)w*R6V
z>keEj5AH^F+mm+w<*2u*WE^-axWO4xz=#2R$4i=e;|7P>3O1Cq1HpbE_kn^Zg~SOH
zNmMTtMLJ9ofa$3~7S)j@K0K8sA(8cbcZ8%Yrc1~=6A@8+M5VvR=r#X4Zfuw&php5|
zk5QhjxPx*Vv5Nx?mGxrR?{O|KP8OoBRrrj@`09=f8T22^xxA-QPaHX8dpB<?1EZ^9
zGpqzA-SW)zC_zn_Yy%*dv#lkA*~@seTMUYw%uKz6JwxC}!s`@==yrefjrHqi%?FLg
zJ79MI1ANwX!XQwwxRWj}d9P#}2HJXhT;I>*qV-n=`=34gKr_Thehy<a$WqjrAsT0e
zY>~!08k3I7Xx^{>z(%z%b~VBV{X}BbiVMvQMh7ZF8l0|eY;4TT5__Z33jx5#u?h$Y
zB-m`iAV|`E%_QSk=*2`PV#uptIE@Yj8vxXex}lC*jgcRLtDzs5R`6Q*GJ4|fS){C^
zF<tZtx<vU#^cmB`BFDOhBp0rv2aa|=@3#p9i#JZn-06WC#`n_Vsr&uip?v|{44>SJ
ztbSHEg~uCW)^M&J0_(2+Lo#BuI(D(YTAh*%SJ~r~ysrI;Nl6f-sLl_rrmK%`kk&_f
zC9I_Do0`Cf43kU16aseKjcy81G_mil2@xQQ&XIfwSci(XDlJ~g_rva<)hqHxBGn|}
zJG5{K`&_PfxkiVDnN$Tm=pA=!7P#d432T#;f46$oDr|q-RL*^Hnfq(NElBa=H$zQ>
zm}k_ZhVuSh*<kYxX#Mxjs!xa|=)777Qv%%e$EJmD-tOBCatjDSm^I>{uBb@spl4{7
zbE(0Qe{y1CmJ8)X!R;-EL$}Xe{M$`)q&4{=&CRbOWrTROPIy4wPTBB@S|*(;9D^P6
z{89k4C)hFT(ID&`qH%&ytIC+Yq0cH;tc$l7BYl{*mdO_$H=(`wpRHAQo=BG+)ji+r
z?63?Do5OB-@susBG#@x~fiX);QDW8QYWC!s^W5Cr#^&a`mtNRo++}3cfhBcTpbo?#
zOlbO47~&EVbkr#9$*kgrW}D?5&D9{)tQ37neh!TYfF5LmfFgyw>b%tR8{`Juq6kL=
z=;JXq4yCdx27)q@D^?t|w$y^9)ApsJt}}M;WRqHe8BexwqP)j4HRb2$?<bt!G{YrC
z{2XdjKPQTX%N*-q)F3SYoW;ssf7}|sIJ|ujwEVg0!{F{QuFXWWd`!8F^{G$uo^;UR
z`taS|Kc@qsvJJ?ffw{yV!)5~K8qTS$`Z=lpxBl?|<%d1<uUGmv!Z+)AJ;uj8ML%Z>
zpPf_+l<I!QAbno_YFx9A*+0Q}SMa^d>F3{M+%i>Yx>>Eqe4#SEX;-h;vmObL?54|C
zOrP`^e(&yMA67^pu|_6O4>mm;tEn|MS<0FlTlyMFXQmo>PxPcUeU$y(GxsMMoKNqi
zd)ZsOJhR8&Z@E3xPWi`}D*Nf@%<ha0%S)eh$=5vmEaw?zuL}K=HKTvNsEysJ8sc83
z=CM)hwE4wa!91e_ErYbsC=v3~<jGLQ?&_h!m@Tc*6u<o;n#Qd6Yh%!o!ya?;tn$ZQ
zs=@M$d7dOWHUT?fGJ)pIEE|2tsAI2tv)trnti<}y8w8w01sM#c&}KNYSP;f%zEZa>
zSIO<{Jof#as-ez0;m`=dJ5Q<#KlT2)JeO3Mzt@aqTW87a-*1uY%X*2Y=X!E$Gve0?
z8(9Z`>QZdwRFhu*th(4cviuY3;yc)xrYf~Ge8XI3Q;4_hVxI8iTon%=x974)w>NWi
z->#p!oP$Qzp{gR)9)zE-82syk9%R}Ih3$x5_aCAn*6;r75%72QQte?iwn^=C@f<g0
z4m*%#YuRw_9y0%<qGQ|C&ZDtayWKOEj+~V#^T-oQ=1^m6;`UglDk1MexaK9UCN>&W
zdf;Y?1|=$?GA1Z>E$7{nejIY6nK{A>DdE=Y61^8V8}4Kki4yLVI{#Mj!cP3~-Svv?
zTeTN%8R?v&^amLWh>@w?l=9%#CW+y-gejHO1nFwWVWB6nj97WOXXfNt$@pHs^sal`
z*NiJ42(x;lm%*-`NG`okAzkWt8@(r;Q>Q2-FLiSkWiFlUdA*FR`T8sQv$D*yy2EV3
zo%LRXtXOaH=U!y*SUE3(ree&vWGsh{M3Wc6IlXhco0f#*^(#4Ub=0k74z<e~t(ixk
zRD2;m&*zB*+5gtjsF@T?;A>lfnv<`=o9OoK+t?i7N2lWgnIks)P|)C645QNv#~*xM
z|GQJfTiHgkF(cqv&l!J?<GP*w{FX{m9fQqpN+tqQoV3_NO4()l+Z@*aN%o{%>>L>+
zIJ&Op6(c{gC>GX~zo@1ycJlX|>Ce`S2-^1#l~Q%fD;zv+1dcBfo1b$YRP5B!b>6OV
zAher)e{oIsqxp%cE>Rhpydh(FbUDFnb%NhUcF`4L0^K?~+LT4^P=C(s0U7e{{O4j;
z#FgUpTBmxG4+Bw~{UZGEFAcUk{oOjL;}wq|lkZSQh+Iu;5`y=&33W9S8xvlPqQ|vk
zK@KG}J&Do4fzGe4(wlgriV$J|5W>6samgP1(rE6V1^uoMdm5F5Mn(L3H63%J4Ctfk
zX6Hu9SH}ZW$1ivMNzrig;w_;`av%D$Qv)*x&xf(c@MG%zVnsyy-0>~eyL2gEjV<=;
zPF@muy>GdNaw~g|?cQ6&7*S=TGuPC7Hs>g+dDZKT1zHhTO(`ziiS<r4HM62G#mULW
zWGJ`7)K7Vy0d@ISwKH-WkM~J;s2R={Zd(0?$@zn(xq0a7%;`&aM5+2*5_h8Zd}B{K
zS#ogZrDKNRx9_Lc$~M-2ol}x&)Y_Jot*Ui5T{YEW?(*RTaaoZlYx162xuO&{F`NE6
z9_EMjIZLIt=VpG)%u;luX$0XY&H=~0Gs+~(*l|l7CGfz78QRs%H$v2(mx!7k{`RzK
ztL2%C%|b1&97?3MYkueoek;w8jc#4Hc1K{l=jjJ$Orz<(D+_4l-BRYkiCT_nXRAKE
zsM<!W4<~CHxtd;C=06sdAI2Z{dk6KIifb>G%QxF(^>w}MaH2o&z?ShyvTWL%w^smt
z#`dPH&bqAR^T+w3DkjWs@{UXBd0t*uN(|whcMuV-S643HbA(6bE!z!m*_jzlalmeb
zoUXV8Z^iaY8W|hE(w7gMj4vSluzNqju=$O&A$5<fV-i-d{3;!ZR=$zIF9B6o!zWex
zzq&UXYr4`#4^2_uRo!)Huj8XzUFA4oU_oKK%T8!AmE^JIJfc*)ct=-7C~Ql&4da#L
z((H0`^)tS&(>W^{wE}57XP5W+@(eOg{75Y%1zX%ZzbHEpxkidsd!yGS-A-0x!S=0|
z`qc7lSJO6QZaFuRmljo)!nV}{$LqJ)eK|FnG#+Ssv~q6J@p<;{{OfFywAw#qQYv(1
jl!O{G=T>r4J~Qu^u+32<BOy6mu|oTZ{^5ISHW&W|pcxF@


From 6e9fa3a41ffbb97af226b72487ddf8b32af19a9d Mon Sep 17 00:00:00 2001
From: Steven Almeroth <sroth77@gmail.com>
Date: Fri, 24 Aug 2018 15:18:16 -0400
Subject: [PATCH 1799/2974] PEP8 ofsite middleware

---
 scrapy/spidermiddlewares/offsite.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 310166cad..232e96cbb 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -3,7 +3,6 @@ Offsite Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
-
 import re
 import logging
 import warnings
@@ -35,8 +34,9 @@ class OffsiteMiddleware(object):
                     domain = urlparse_cached(x).hostname
                     if domain and domain not in self.domains_seen:
                         self.domains_seen.add(domain)
-                        logger.debug("Filtered offsite request to %(domain)r: %(request)s",
-                                     {'domain': domain, 'request': x}, extra={'spider': spider})
+                        logger.debug(
+                            "Filtered offsite request to %(domain)r: %(request)s",
+                            {'domain': domain, 'request': x}, extra={'spider': spider})
                         self.stats.inc_value('offsite/domains', spider=spider)
                     self.stats.inc_value('offsite/filtered', spider=spider)
             else:
@@ -52,13 +52,15 @@ class OffsiteMiddleware(object):
         """Override this method to implement a different offsite policy"""
         allowed_domains = getattr(spider, 'allowed_domains', None)
         if not allowed_domains:
-            return re.compile('') # allow all by default
+            return re.compile('')  # allow all by default
         url_pattern = re.compile("^https?://.*$")
         for domain in allowed_domains:
             if url_pattern.match(domain):
-                warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, URLWarning)
-                
-        regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None)
+                message = ("allowed_domains accepts only domains, not URLs. "
+                           "Ignoring URL entry %s in allowed_domains." % domain)
+                warnings.warn(message, URLWarning)
+        domains = [re.escape(d) for d in allowed_domains if d is not None]
+        regex = r'^(.*\.)?(%s)$' % '|'.join(domains)
         return re.compile(regex)
 
     def spider_opened(self, spider):

From c02cfa574cc47d6b086cc66025ddef9f3174ac02 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Wed, 29 Aug 2018 11:21:55 +0000
Subject: [PATCH 1800/2974] remove comma

---
 docs/topics/signals.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index cf7b8db2f..ff07b9d55 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -285,7 +285,7 @@ request_reached_downloader
 .. signal:: request_reached_downloader
 .. function:: request_reached_downloader(request, spider)
 
-    Sent when a :class:`~scrapy.http.Request`, reached downloader.
+    Sent when a :class:`~scrapy.http.Request` reached downloader.
 
     The signal does not support returning deferreds from their handlers.
 

From 8dbbbd13950dcb21dda759b073c64ffdca85c2d6 Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Mon, 3 Sep 2018 20:07:37 +0300
Subject: [PATCH 1801/2974] Use request_cls attribute in contract definition

---
 docs/topics/contracts.rst    |  5 +++--
 scrapy/contracts/__init__.py | 15 +++++++++------
 tests/test_contracts.py      |  2 +-
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index ada6fd227..70f20d4ed 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -86,8 +86,9 @@ override three methods:
     .. method:: Contract.adjust_request_args(args)
 
         This receives a ``dict`` as an argument containing default arguments
-        for request object. :class:`~scrapy.http.Request` is used
-        if ``request_cls`` is not set on ``args``.
+        for request object. :class:`~scrapy.http.Request` is used by default,
+        but this can be changed with the ``request_cls`` attribute.
+        If multiple contracts in chain have this attribute defined, the last one is used.
 
         Must return the same or a modified version of it.
 
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 801c18e73..851a26a8e 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -4,7 +4,6 @@ from functools import wraps
 from inspect import getmembers
 from unittest import TestCase
 
-from scrapy import FormRequest
 from scrapy.http import Request
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.python import get_spec
@@ -50,14 +49,17 @@ class ContractsManager(object):
     def from_method(self, method, results):
         contracts = self.extract_contracts(method)
         if contracts:
-            # prepare request arguments
-            kwargs = {'callback': method}
+            request_cls = Request
+            for contract in contracts:
+                if contract.request_cls is not None:
+                    request_cls = contract.request_cls
+
+            # calculate request args
+            args, kwargs = get_spec(request_cls.__init__)
+            kwargs['callback'] = method
             for contract in contracts:
                 kwargs = contract.adjust_request_args(kwargs)
 
-            request_cls = kwargs.pop('request_cls', Request)
-
-            args, _ = get_spec(request_cls.__init__)
             args.remove('self')
 
             # check if all positional arguments are defined in kwargs
@@ -98,6 +100,7 @@ class ContractsManager(object):
 
 class Contract(object):
     """ Abstract class for contracts """
+    request_cls = None
 
     def __init__(self, method, *args):
         self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name)
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index c35b068a4..fc5c94771 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -27,9 +27,9 @@ class ResponseMock(object):
 
 class CustomFormContract(Contract):
     name = 'custom_form'
+    request_cls = FormRequest
 
     def adjust_request_args(self, args):
-        args['request_cls'] = FormRequest
         args['formdata'] = {'name': 'scrapy'}
         return args
 

From e65f7e0c91ccb16525bb318cb50339979387fdbf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 5 Sep 2018 10:49:46 -0300
Subject: [PATCH 1802/2974] Working POC for authenticating telnet console

---
 scrapy/extensions/telnet.py | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 3024ddfaa..5e9fce7c5 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -22,6 +22,7 @@ from scrapy import signals
 from scrapy.utils.trackref import print_live_refs
 from scrapy.utils.engine import print_engine_status
 from scrapy.utils.reactor import listen_tcp
+from scrapy.utils.decorators import defers
 
 try:
     import guppy
@@ -49,6 +50,8 @@ class TelnetConsole(protocol.ServerFactory):
         self.noisy = False
         self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]
         self.host = crawler.settings['TELNETCONSOLE_HOST']
+        self.username = crawler.settings.get('TELNETCONSOLE_USERNAME', 'scrapy')
+        self.password = crawler.settings.get('TELNETCONSOLE_PASSWORD', 'scrapy')
         self.crawler.signals.connect(self.start_listening, signals.engine_started)
         self.crawler.signals.connect(self.stop_listening, signals.engine_stopped)
 
@@ -67,9 +70,25 @@ class TelnetConsole(protocol.ServerFactory):
         self.port.stopListening()
 
     def protocol(self):
-        telnet_vars = self._get_telnet_vars()
-        return telnet.TelnetTransport(telnet.TelnetBootstrapProtocol,
-            insults.ServerProtocol, manhole.Manhole, telnet_vars)
+        class Portal:
+            """An implementation of IPortal"""
+            @defers
+            def login(self_, credentials, mind, *interfaces):
+                if not (credentials.username == self.username
+                        and credentials.checkPassword(self.password)):
+                    raise ValueError("Invalid credentials")
+
+                protocol = telnet.TelnetBootstrapProtocol(
+                    insults.ServerProtocol,
+                    manhole.Manhole,
+                    self._get_telnet_vars()
+                )
+                return (interfaces[0], protocol, lambda: None)
+
+        return telnet.TelnetTransport(
+            telnet.AuthenticatingTelnetProtocol,
+            Portal()
+        )
 
     def _get_telnet_vars(self):
         # Note: if you add entries here also update topics/telnetconsole.rst

From eb64214c8a0053627e625debdb55373c8e17ef1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 5 Sep 2018 10:54:40 -0300
Subject: [PATCH 1803/2974] Move telnetconsole settings defaults to scrapy
 defaults

---
 scrapy/extensions/telnet.py         | 4 ++--
 scrapy/settings/default_settings.py | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 5e9fce7c5..93342f225 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -50,8 +50,8 @@ class TelnetConsole(protocol.ServerFactory):
         self.noisy = False
         self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]
         self.host = crawler.settings['TELNETCONSOLE_HOST']
-        self.username = crawler.settings.get('TELNETCONSOLE_USERNAME', 'scrapy')
-        self.password = crawler.settings.get('TELNETCONSOLE_PASSWORD', 'scrapy')
+        self.username = crawler.settings['TELNETCONSOLE_USERNAME']
+        self.password = crawler.settings['TELNETCONSOLE_PASSWORD']
         self.crawler.signals.connect(self.start_listening, signals.engine_started)
         self.crawler.signals.connect(self.stop_listening, signals.engine_stopped)
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index ca004aedd..2b7bc173c 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -277,6 +277,8 @@ USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__versi
 TELNETCONSOLE_ENABLED = 1
 TELNETCONSOLE_PORT = [6023, 6073]
 TELNETCONSOLE_HOST = '127.0.0.1'
+TELNETCONSOLE_USERNAME = 'scrapy'
+TELNETCONSOLE_PASSWORD = 'scrapy'
 
 SPIDER_CONTRACTS = {}
 SPIDER_CONTRACTS_BASE = {

From 25ac4691b414e7c18a4e9dec3bb6a85563d2488d Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 9 Aug 2018 03:32:46 +0500
Subject: [PATCH 1804/2974] require parsel 1.5+

---
 requirements-py2.txt | 2 +-
 requirements-py3.txt | 2 +-
 setup.py             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements-py2.txt b/requirements-py2.txt
index 03b33d02d..0771aae3a 100644
--- a/requirements-py2.txt
+++ b/requirements-py2.txt
@@ -6,5 +6,5 @@ queuelib
 w3lib>=1.17.0
 six>=1.5.2
 PyDispatcher>=2.0.5
-parsel>=1.4
+parsel>=1.5
 service_identity
diff --git a/requirements-py3.txt b/requirements-py3.txt
index b38c4cc09..5a5d4c95a 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -6,5 +6,5 @@ queuelib>=1.1.1
 w3lib>=1.17.0
 six>=1.5.2
 PyDispatcher>=2.0.5
-parsel>=1.4
+parsel>=1.5
 service_identity
diff --git a/setup.py b/setup.py
index c37919cda..8c47f67ce 100644
--- a/setup.py
+++ b/setup.py
@@ -71,7 +71,7 @@ setup(
         'pyOpenSSL',
         'cssselect>=0.9',
         'six>=1.5.2',
-        'parsel>=1.4',
+        'parsel>=1.5',
         'PyDispatcher>=2.0.5',
         'service_identity',
     ],

From 0ccead6681c9e2bf1902cbc1d4bde543be7d73e7 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Aug 2018 16:16:36 +0500
Subject: [PATCH 1805/2974] DOC more Python 3 in examples

---
 docs/topics/commands.rst  |  6 ++---
 docs/topics/items.rst     |  6 ++---
 docs/topics/jobs.rst      |  4 ++--
 docs/topics/loaders.rst   | 14 ++++++------
 docs/topics/selectors.rst | 48 +++++++++++++++++++--------------------
 docs/topics/settings.rst  |  2 +-
 6 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 3088017cb..ef9c45196 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -458,9 +458,9 @@ Usage example::
 
     >>> STATUS DEPTH LEVEL 1 <<<
     # Scraped Items  ------------------------------------------------------------
-    [{'name': u'Example item',
-     'category': u'Furniture',
-     'length': u'12 cm'}]
+    [{'name': 'Example item',
+     'category': 'Furniture',
+     'length': '12 cm'}]
 
     # Requests  -----------------------------------------------------------------
     []
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 4423bbda2..ae44aecd3 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -86,7 +86,7 @@ Creating items
 ::
 
     >>> product = Product(name='Desktop PC', price=1000)
-    >>> print product
+    >>> print(product)
     Product(name='Desktop PC', price=1000)
 
 Getting field values
@@ -161,11 +161,11 @@ Other common tasks
 Copying items::
 
     >>> product2 = Product(product)
-    >>> print product2
+    >>> print(product2)
     Product(name='Desktop PC', price=1000)
 
     >>> product3 = product2.copy()
-    >>> print product3
+    >>> print(product3)
     Product(name='Desktop PC', price=1000)
 
 Creating dicts from items::
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 06c7fff3d..8e1574376 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -84,7 +84,7 @@ So, for example, this won't work::
         return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg))
 
     def other_callback(self, response, somearg):
-        print "the argument passed is:", somearg
+        print("the argument passed is: %s" % somearg)
 
 But this will::
 
@@ -94,7 +94,7 @@ But this will::
 
     def other_callback(self, response):
         somearg = response.meta['somearg']
-        print "the argument passed is:", somearg
+        print("the argument passed is: %s" % somearg)
 
 If you wish to log the requests that couldn't be serialized, you can set the
 :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index a895b535c..f3b6aa4a1 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -678,10 +678,10 @@ Here is a list of all built-in processors:
         >>> from scrapy.loader.processors import Join
         >>> proc = Join()
         >>> proc(['one', 'two', 'three'])
-        u'one two three'
+        'one two three'
         >>> proc = Join('<br>')
         >>> proc(['one', 'two', 'three'])
-        u'one<br>two<br>three'
+        'one<br>two<br>three'
 
 .. class:: Compose(\*functions, \**default_loader_context)
 
@@ -744,9 +744,9 @@ Here is a list of all built-in processors:
         ...     return None if x == 'world' else x
         ...
         >>> from scrapy.loader.processors import MapCompose
-        >>> proc = MapCompose(filter_world, unicode.upper)
-        >>> proc([u'hello', u'world', u'this', u'is', u'scrapy'])
-        [u'HELLO, u'THIS', u'IS', u'SCRAPY']
+        >>> proc = MapCompose(filter_world, str.upper)
+        >>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
+        ['HELLO, 'THIS', 'IS', 'SCRAPY']
 
     As with the Compose processor, functions can receive Loader contexts, and
     constructor keyword arguments are used as default context values. See
@@ -772,7 +772,7 @@ Here is a list of all built-in processors:
         >>> import json
         >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
         >>> proc_single_json_str('{"foo": "bar"}')
-        u'bar'
+        'bar'
         >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
         >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
-        [u'bar']
+        ['bar']
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 8ac40c3cc..25c1f0aab 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -235,17 +235,17 @@ Here's an example used to extract image names from the :ref:`HTML code
 <topics-selectors-htmlcode>` above::
 
     >>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
-    [u'My image 1',
-     u'My image 2',
-     u'My image 3',
-     u'My image 4',
-     u'My image 5']
+    ['My image 1',
+     'My image 2',
+     'My image 3',
+     'My image 4',
+     'My image 5']
 
 There's an additional helper reciprocating ``.extract_first()`` for ``.re()``,
 named ``.re_first()``. Use it to extract just the first matching string::
 
     >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
-    u'My image 1'
+    'My image 1'
 
 .. _topics-selectors-relative-xpaths:
 
@@ -431,26 +431,26 @@ with groups of itemscopes and corresponding itemprops::
     ...     print "    properties:", props.extract()
     ...     print
 
-    current scope: [u'http://schema.org/Product']
-        properties: [u'name', u'aggregateRating', u'offers', u'description', u'review', u'review']
+    current scope: ['http://schema.org/Product']
+        properties: ['name', 'aggregateRating', 'offers', 'description', 'review', 'review']
 
-    current scope: [u'http://schema.org/AggregateRating']
-        properties: [u'ratingValue', u'reviewCount']
+    current scope: ['http://schema.org/AggregateRating']
+        properties: ['ratingValue', 'reviewCount']
 
-    current scope: [u'http://schema.org/Offer']
-        properties: [u'price', u'availability']
+    current scope: ['http://schema.org/Offer']
+        properties: ['price', 'availability']
 
-    current scope: [u'http://schema.org/Review']
-        properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description']
+    current scope: ['http://schema.org/Review']
+        properties: ['name', 'author', 'datePublished', 'reviewRating', 'description']
 
-    current scope: [u'http://schema.org/Rating']
-        properties: [u'worstRating', u'ratingValue', u'bestRating']
+    current scope: ['http://schema.org/Rating']
+        properties: ['worstRating', 'ratingValue', 'bestRating']
 
-    current scope: [u'http://schema.org/Review']
-        properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description']
+    current scope: ['http://schema.org/Review']
+        properties: ['name', 'author', 'datePublished', 'reviewRating', 'description']
 
-    current scope: [u'http://schema.org/Rating']
-        properties: [u'worstRating', u'ratingValue', u'bestRating']
+    current scope: ['http://schema.org/Rating']
+        properties: ['worstRating', 'ratingValue', 'bestRating']
 
     >>>
 
@@ -543,22 +543,22 @@ Example::
 This gets all first ``<li>``  elements under whatever it is its parent::
 
     >>> xp("//li[1]")
-    [u'<li>1</li>', u'<li>4</li>']
+    ['<li>1</li>', '<li>4</li>']
 
 And this gets the first ``<li>``  element in the whole document::
 
     >>> xp("(//li)[1]")
-    [u'<li>1</li>']
+    ['<li>1</li>']
 
 This gets all first ``<li>``  elements under an ``<ul>``  parent::
 
     >>> xp("//ul/li[1]")
-    [u'<li>1</li>', u'<li>4</li>']
+    ['<li>1</li>', '<li>4</li>']
 
 And this gets the first ``<li>``  element under an ``<ul>``  parent in the whole document::
 
     >>> xp("(//ul/li)[1]")
-    [u'<li>1</li>']
+    ['<li>1</li>']
 
 When querying by class, consider using CSS
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 1f1217770..47b6cf13d 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -871,7 +871,7 @@ LOG_STDOUT
 Default: ``False``
 
 If ``True``, all standard output (and error) of your process will be redirected
-to the log. For example if you ``print 'hello'`` it will appear in the Scrapy
+to the log. For example if you ``print('hello')`` it will appear in the Scrapy
 log.
 
 .. setting:: LOG_SHORT_NAMES

From 395d9d033a39ae9e82337ce50cffabd1ecedb702 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Aug 2018 16:16:57 +0500
Subject: [PATCH 1806/2974] add pytest temp files to gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index ff6e2ea65..7392ed31e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,7 @@ htmlcov/
 .pytest_cache/
 .coverage.*
 .cache/
+.pytest_cache/
 
 # Windows
 Thumbs.db

From ca27010d4f8c35d1c98259e56be797a18b044304 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Aug 2018 16:22:56 +0500
Subject: [PATCH 1807/2974] DOC .extract_first() -> .get()

---
 docs/intro/overview.rst |  6 ++---
 docs/intro/tutorial.rst | 49 +++++++++++++++++++++--------------------
 docs/topics/shell.rst   |  2 +-
 docs/topics/spiders.rst | 10 ++++-----
 4 files changed, 34 insertions(+), 33 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 6f1c2c43f..9d7c94d39 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -34,11 +34,11 @@ http://quotes.toscrape.com, following the pagination::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.xpath('span/small/text()').extract_first(),
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.xpath('span/small/text()').get(),
                 }
 
-            next_page = response.css('li.next a::attr("href")').extract_first()
+            next_page = response.css('li.next a::attr("href")').get()
             if next_page is not None:
                 yield response.follow(next_page, self.parse)
 
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index fa6dc274d..a24cf0f5b 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -269,23 +269,24 @@ The other thing is that the result of calling ``.extract()`` is a list, because
 we're dealing with an instance of :class:`~scrapy.selector.SelectorList`.  When
 you know you just want the first result, as in this case, you can do::
 
-    >>> response.css('title::text').extract_first()
+    >>> response.css('title::text').get()
     'Quotes to Scrape'
 
 As an alternative, you could've written::
 
-    >>> response.css('title::text')[0].extract()
+    >>> response.css('title::text')[0].get()
     'Quotes to Scrape'
 
-However, using ``.extract_first()`` avoids an ``IndexError`` and returns
-``None`` when it doesn't find any element matching the selection.
+However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
+instance avoids an ``IndexError`` and returns ``None`` when it doesn't
+find any element matching the selection.
 
 There's a lesson here: for most scraping code, you want it to be resilient to
 errors due to things not being found on a page, so that even if some parts fail
 to be scraped, you can at least get **some** data.
 
 Besides the :meth:`~scrapy.selector.Selector.extract` and
-:meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use
+:meth:`~scrapy.selector.SelectorList.get` methods, you can also use
 the :meth:`~scrapy.selector.Selector.re` method to extract using `regular
 expressions`::
 
@@ -314,7 +315,7 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions::
 
     >>> response.xpath('//title')
     [<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
-    >>> response.xpath('//title/text()').extract_first()
+    >>> response.xpath('//title/text()').get()
     'Quotes to Scrape'
 
 XPath expressions are very powerful, and are the foundation of Scrapy
@@ -383,10 +384,10 @@ variable, so that we can run our CSS selectors directly on a particular quote::
 Now, let's extract ``title``, ``author`` and the ``tags`` from that quote
 using the ``quote`` object we just created::
 
-    >>> title = quote.css("span.text::text").extract_first()
+    >>> title = quote.css("span.text::text").get()
     >>> title
     '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
-    >>> author = quote.css("small.author::text").extract_first()
+    >>> author = quote.css("small.author::text").get()
     >>> author
     'Albert Einstein'
 
@@ -401,8 +402,8 @@ Having figured out how to extract each bit, we can now iterate over all the
 quotes elements and put them together into a Python dictionary::
 
     >>> for quote in response.css("div.quote"):
-    ...     text = quote.css("span.text::text").extract_first()
-    ...     author = quote.css("small.author::text").extract_first()
+    ...     text = quote.css("span.text::text").get()
+    ...     author = quote.css("small.author::text").get()
     ...     tags = quote.css("div.tags a.tag::text").extract()
     ...     print(dict(text=text, author=author, tags=tags))
     {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
@@ -434,8 +435,8 @@ in the callback, as you can see below::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.css('small.author::text').extract_first(),
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.css('small.author::text').get(),
                     'tags': quote.css('div.tags a.tag::text').extract(),
                 }
 
@@ -508,14 +509,14 @@ markup:
 
 We can try extracting it in the shell::
 
-    >>> response.css('li.next a').extract_first()
+    >>> response.css('li.next a').get()
     '<a href="/page/2/">Next <span aria-hidden="true">→</span></a>'
 
 This gets the anchor element, but we want the attribute ``href``. For that,
 Scrapy supports a CSS extension that let's you select the attribute contents,
 like this::
 
-    >>> response.css('li.next a::attr(href)').extract_first()
+    >>> response.css('li.next a::attr(href)').get()
     '/page/2/'
 
 Let's see now our spider modified to recursively follow the link to the next
@@ -533,12 +534,12 @@ page, extracting data from it::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.css('small.author::text').extract_first(),
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.css('small.author::text').get(),
                     'tags': quote.css('div.tags a.tag::text').extract(),
                 }
 
-            next_page = response.css('li.next a::attr(href)').extract_first()
+            next_page = response.css('li.next a::attr(href)').get()
             if next_page is not None:
                 next_page = response.urljoin(next_page)
                 yield scrapy.Request(next_page, callback=self.parse)
@@ -584,12 +585,12 @@ As a shortcut for creating Request objects you can use
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.css('span small::text').extract_first(),
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.css('span small::text').get(),
                     'tags': quote.css('div.tags a.tag::text').extract(),
                 }
 
-            next_page = response.css('li.next a::attr(href)').extract_first()
+            next_page = response.css('li.next a::attr(href)').get()
             if next_page is not None:
                 yield response.follow(next_page, callback=self.parse)
 
@@ -641,7 +642,7 @@ this time for scraping author information::
 
         def parse_author(self, response):
             def extract_with_css(query):
-                return response.css(query).extract_first().strip()
+                return response.css(query).get().strip()
 
             yield {
                 'name': extract_with_css('h3.author-title::text'),
@@ -710,11 +711,11 @@ with a specific tag, building the URL based on the argument::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').extract_first(),
-                    'author': quote.css('small.author::text').extract_first(),
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.css('small.author::text').get(),
                 }
 
-            next_page = response.css('li.next a::attr(href)').extract_first()
+            next_page = response.css('li.next a::attr(href)').get()
             if next_page is not None:
                 yield response.follow(next_page, self.parse)
 
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 11ab199f2..9de6abef7 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -179,7 +179,7 @@ all start with the ``[s]`` prefix)::
 
 After that, we can start playing with the objects::
 
-    >>> response.xpath('//title/text()').extract_first()
+    >>> response.xpath('//title/text()').get()
     'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
 
     >>> fetch("https://reddit.com")
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 697732b47..4505b7315 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -434,8 +434,8 @@ Let's now take a look at an example CrawlSpider with rules::
             self.logger.info('Hi, this is an item page! %s', response.url)
             item = scrapy.Item()
             item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
-            item['name'] = response.xpath('//td[@id="item_name"]/text()').extract()
-            item['description'] = response.xpath('//td[@id="item_description"]/text()').extract()
+            item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
+            item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
             return item
 
 
@@ -548,9 +548,9 @@ These spiders are pretty easy to use, let's have a look at one example::
             self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract()))
 
             item = TestItem()
-            item['id'] = node.xpath('@id').extract()
-            item['name'] = node.xpath('name').extract()
-            item['description'] = node.xpath('description').extract()
+            item['id'] = node.xpath('@id').get()
+            item['name'] = node.xpath('name').get()
+            item['description'] = node.xpath('description').get()
             return item
 
 Basically what we did up there was to create a spider that downloads a feed from

From d32c4deaa99bf0155542d1b508b5535065b5d7a9 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 15 Aug 2018 16:23:31 +0500
Subject: [PATCH 1808/2974] DOC update Scrapy selectors tutorial to match
 parsel's tutorial better

---
 docs/topics/selectors.rst | 471 +++++++++++++++++++++++++++-----------
 1 file changed, 340 insertions(+), 131 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 25c1f0aab..00158ecf1 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -6,7 +6,7 @@ Selectors
 
 When you're scraping web pages, the most common task you need to perform is
 to extract data from the HTML source. There are several libraries available to
-achieve this:
+achieve this, such as:
 
  * `BeautifulSoup`_ is a very popular web scraping library among Python
    programmers which constructs a Python object based on the structure of the
@@ -25,8 +25,9 @@ either by `XPath`_ or `CSS`_ expressions.
 used with HTML. `CSS`_ is a language for applying styles to HTML documents. It
 defines selectors to associate those styles with specific HTML elements.
 
-Scrapy selectors are built over the `lxml`_ library, which means they're very
-similar in speed and parsing accuracy.
+Scrapy selectors are powered by `parsel`_ library, which uses `lxml`_ library
+under the hood. It means Scrapy selectors are very similar in speed and
+parsing accuracy to lxml.
 
 This page explains how selectors work and describes their API which is very
 small and simple, unlike the `lxml`_ API which is much bigger because the
@@ -42,7 +43,7 @@ For a complete reference of the selectors API see
 .. _cssselect: https://pypi.python.org/pypi/cssselect/
 .. _XPath: https://www.w3.org/TR/xpath
 .. _CSS: https://www.w3.org/TR/selectors
-
+.. _parsel: https://parsel.readthedocs.io/
 
 Using selectors
 ===============
@@ -63,21 +64,32 @@ input type::
 Constructing from text::
 
     >>> body = '<html><body><span>good</span></body></html>'
-    >>> Selector(text=body).xpath('//span/text()').extract()
-    [u'good']
+    >>> Selector(text=body).xpath('//span/text()').get()
+    'good'
 
 Constructing from response::
 
     >>> response = HtmlResponse(url='http://example.com', body=body)
-    >>> Selector(response=response).xpath('//span/text()').extract()
-    [u'good']
+    >>> Selector(response=response).xpath('//span/text()').get()
+    'good'
 
 For convenience, response objects expose a selector on `.selector` attribute,
-it's totally OK to use this shortcut when possible::
+it's totally OK to use this shortcut when possible. By using it you can
+ensure the response body is parsed only once::
 
-    >>> response.selector.xpath('//span/text()').extract()
-    [u'good']
+    >>> response.selector.xpath('//span/text()').get()
+    'good'
 
+Querying responses using XPath and CSS is so common that responses include two
+more shortcuts: ``response.xpath()`` and ``response.css()``::
+
+    >>> response.xpath('//span/text()').get()
+    'good'
+    >>> response.css('span::text').get()
+    'good'
+
+Usually there is no need to construct Scrapy selectors manually because of
+these shortcuts.
 
 Using selectors
 ---------------
@@ -90,7 +102,7 @@ documentation server:
 
 .. _topics-selectors-htmlcode:
 
-Here's its HTML code:
+For the sake of completeness, here's its full HTML code:
 
 .. literalinclude:: ../_static/selectors-sample1.html
    :language: html
@@ -111,90 +123,179 @@ Since we're dealing with HTML, the selector will automatically use an HTML parse
 So, by looking at the :ref:`HTML code <topics-selectors-htmlcode>` of that
 page, let's construct an XPath for selecting the text inside the title tag::
 
-    >>> response.selector.xpath('//title/text()')
-    [<Selector (text) xpath=//title/text()>]
-
-Querying responses using XPath and CSS is so common that responses include two
-convenience shortcuts: ``response.xpath()`` and ``response.css()``::
-
     >>> response.xpath('//title/text()')
-    [<Selector (text) xpath=//title/text()>]
-    >>> response.css('title::text')
-    [<Selector (text) xpath=//title/text()>]
+    [<Selector xpath='//title/text()' data='Example website'>]
+
+To actually extract the textual data, you must call the selector ``.get()``
+or ``.getall()`` methods, as follows::
+
+    >>> response.xpath('//title/text()').getall()
+    ['Example website']
+    >>> response.xpath('//title/text()').get()
+    'Example website'
+
+``.get()`` always returns a single result; if there are several matches,
+content of a first match is returned; if there are no matches, None
+is returned. ``.getall()`` returns a list with all results.
+
+Notice that CSS selectors can select text or attribute nodes using CSS3
+pseudo-elements::
+
+    >>> selector.css('title::text').get()
+    'Example website'
 
 As you can see, ``.xpath()`` and ``.css()`` methods return a
 :class:`~scrapy.selector.SelectorList` instance, which is a list of new
 selectors. This API can be used for quickly selecting nested data::
 
-    >>> response.css('img').xpath('@src').extract()
-    [u'image1_thumb.jpg',
-     u'image2_thumb.jpg',
-     u'image3_thumb.jpg',
-     u'image4_thumb.jpg',
-     u'image5_thumb.jpg']
+    >>> response.css('img').xpath('@src').getall()
+    ['image1_thumb.jpg',
+     'image2_thumb.jpg',
+     'image3_thumb.jpg',
+     'image4_thumb.jpg',
+     'image5_thumb.jpg']
 
-To actually extract the textual data, you must call the selector ``.extract()``
-method, as follows::
+If you want to extract only the first matched element, you can call the
+selector ``.get()`` (or its alias ``.extract_first()`` commonly used in
+previous Scrapy versions)::
 
-    >>> response.xpath('//title/text()').extract()
-    [u'Example website']
+    >>> response.xpath('//div[@id="images"]/a/text()').get()
+    'Name: My image 1 '
 
-If you want to extract only first matched element, you can call the selector ``.extract_first()``
+It returns ``None`` if no element was found::
 
-    >>> response.xpath('//div[@id="images"]/a/text()').extract_first()
-    u'Name: My image 1 '
-
-It returns ``None`` if no element was found:
-
-    >>> response.xpath('//div[@id="not-exists"]/text()').extract_first() is None
+    >>> response.xpath('//div[@id="not-exists"]/text()').get() is None
     True
 
-A default return value can be provided as an argument, to be used instead of ``None``:
+A default return value can be provided as an argument, to be used instead
+of ``None``:
 
-    >>> response.xpath('//div[@id="not-exists"]/text()').extract_first(default='not-found')
+    >>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found')
     'not-found'
 
-Notice that CSS selectors can select text or attribute nodes using CSS3
-pseudo-elements::
+Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes
+using ``.attrib`` property of a :class:`~scrapy.selector.Selector`::
 
-    >>> response.css('title::text').extract()
-    [u'Example website']
+    >>> [img.attrib['src'] for img in response.css('img')]
+    ['image1_thumb.jpg',
+     'image2_thumb.jpg',
+     'image3_thumb.jpg',
+     'image4_thumb.jpg',
+     'image5_thumb.jpg']
+
+As a shortcut, ``.attrib`` is also available on SelectorList directly;
+it returns attributes for the first matching element::
+
+    >>> response.css('img').attrib['src']
+    'image1_thumb.jpg'
+
+This is most useful when only a single result is expected, e.g. when selecting
+by id, or selecting unique elements on a web page::
+
+    >>> response.css('base').attrib['href']
+    'http://example.com/'
 
 Now we're going to get the base URL and some image links::
 
-    >>> response.xpath('//base/@href').extract()
-    [u'http://example.com/']
+    >>> response.xpath('//base/@href').get()
+    'http://example.com/'
 
-    >>> response.css('base::attr(href)').extract()
-    [u'http://example.com/']
+    >>> response.css('base::attr(href)').get()
+    'http://example.com/'
 
-    >>> response.xpath('//a[contains(@href, "image")]/@href').extract()
-    [u'image1.html',
-     u'image2.html',
-     u'image3.html',
-     u'image4.html',
-     u'image5.html']
+    >>> response.css('base').attrib['href']
+    'http://example.com/'
 
-    >>> response.css('a[href*=image]::attr(href)').extract()
-    [u'image1.html',
-     u'image2.html',
-     u'image3.html',
-     u'image4.html',
-     u'image5.html']
+    >>> response.xpath('//a[contains(@href, "image")]/@href').getall()
+    ['image1.html',
+     'image2.html',
+     'image3.html',
+     'image4.html',
+     'image5.html']
 
-    >>> response.xpath('//a[contains(@href, "image")]/img/@src').extract()
-    [u'image1_thumb.jpg',
-     u'image2_thumb.jpg',
-     u'image3_thumb.jpg',
-     u'image4_thumb.jpg',
-     u'image5_thumb.jpg']
+    >>> response.css('a[href*=image]::attr(href)').getall()
+    ['image1.html',
+     'image2.html',
+     'image3.html',
+     'image4.html',
+     'image5.html']
 
-    >>> response.css('a[href*=image] img::attr(src)').extract()
-    [u'image1_thumb.jpg',
-     u'image2_thumb.jpg',
-     u'image3_thumb.jpg',
-     u'image4_thumb.jpg',
-     u'image5_thumb.jpg']
+    >>> response.xpath('//a[contains(@href, "image")]/img/@src').getall()
+    ['image1_thumb.jpg',
+     'image2_thumb.jpg',
+     'image3_thumb.jpg',
+     'image4_thumb.jpg',
+     'image5_thumb.jpg']
+
+    >>> response.css('a[href*=image] img::attr(src)').getall()
+    ['image1_thumb.jpg',
+     'image2_thumb.jpg',
+     'image3_thumb.jpg',
+     'image4_thumb.jpg',
+     'image5_thumb.jpg']
+
+.. _topics-selectors-css-extensions:
+
+Extensions to CSS Selectors
+---------------------------
+
+Per W3C standards, `CSS selectors`_ do not support selecting text nodes
+or attribute values.
+But selecting these is so essential in a web scraping context
+that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**:
+
+* to select text nodes, use ``::text``
+* to select attribute values, use ``::attr(name)`` where *name* is the
+  name of the attribute that you want the value of
+
+.. warning::
+    These pseudo-elements are Scrapy-/Parsel-specific.
+    They will most probably not work with other libraries like
+   `lxml`_ or `PyQuery`_.
+
+.. _PyQuery: https://pypi.python.org/pypi/pyquery
+
+Examples:
+
+* ``title::text`` selects children text nodes of a descendant ``<title>`` element::
+
+    >>> response.css('title::text').get()
+    'Example website'
+
+* ``*::text`` selects all descendant text nodes of the current selector context::
+
+    >>> response.css('#images *::text').getall()
+    ['\n   ',
+     'Name: My image 1 ',
+     '\n   ',
+     'Name: My image 2 ',
+     '\n   ',
+     'Name: My image 3 ',
+     '\n   ',
+     'Name: My image 4 ',
+     '\n   ',
+     'Name: My image 5 ',
+     '\n  ']
+
+* ``a::attr(href)`` selects the *href* attribute value of descendant links::
+
+    >>> response.css('a::attr(href)').getall()
+    ['image1.html',
+     'image2.html',
+     'image3.html',
+     'image4.html',
+     'image5.html']
+
+.. note::
+    You cannot chain these pseudo-elements. But in practice it would not
+    make much sense: text nodes do not have attributes, and attribute values
+    are string values already and do not have children nodes.
+
+.. note::
+    See also: :ref:`selecting-attributes`.
+
+
+.. _CSS Selectors: https://www.w3.org/TR/css3-selectors/#selectors
 
 .. _topics-selectors-nesting-selectors:
 
@@ -206,22 +307,65 @@ of the same type, so you can call the selection methods for those selectors
 too. Here's an example::
 
     >>> links = response.xpath('//a[contains(@href, "image")]')
-    >>> links.extract()
-    [u'<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
-     u'<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
-     u'<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
-     u'<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
-     u'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
+    >>> links.getall()
+    ['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
+     '<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
+     '<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
+     '<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
+     '<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
 
     >>> for index, link in enumerate(links):
-    ...     args = (index, link.xpath('@href').extract(), link.xpath('img/@src').extract())
-    ...     print 'Link number %d points to url %s and image %s' % args
+    ...     args = (index, link.xpath('@href').get(), link.xpath('img/@src').get())
+    ...     print('Link number %d points to url %r and image %r' % args)
 
-    Link number 0 points to url [u'image1.html'] and image [u'image1_thumb.jpg']
-    Link number 1 points to url [u'image2.html'] and image [u'image2_thumb.jpg']
-    Link number 2 points to url [u'image3.html'] and image [u'image3_thumb.jpg']
-    Link number 3 points to url [u'image4.html'] and image [u'image4_thumb.jpg']
-    Link number 4 points to url [u'image5.html'] and image [u'image5_thumb.jpg']
+    Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
+    Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
+    Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
+    Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg'
+    Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg'
+
+.. _selecting-attributes:
+
+Selecting element attributes
+----------------------------
+
+There are several ways to get a value of an attribute. First, one can use
+XPath syntax::
+
+    >>> response.xpath("//a/@href").getall()
+    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+XPath syntax has a few advantages: it is a standard XPath feature, and
+``@attributes`` can be used in other parts of an XPath expression - e.g.
+it is possible to filter by attribute value.
+
+Scrapy also provides an extension to CSS selectors (``::attr(...)``)
+which allows to get attribute values::
+
+    >>> response.css('a::attr(href)').getall()
+    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+In addition to that, there is a ``.attrib`` property of Selector.
+You can use it if you prefer to lookup attributes in Python
+code, without using XPaths or CSS extensions::
+
+    >>> [a.attrib['href'] for a in response.css('a')]
+    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+This property is also available on SelectorList; it returns a dictionary
+with attributes of a first matching element. It is convenient to use when
+a selector is expected to give a single result (e.g. when selecting by element
+ID, or when selecting an unique element on a page)::
+
+    >>> response.css('base').attrib
+    {'href': 'http://example.com/'}
+    >>> response.css('base').attrib['href']
+    'http://example.com/'
+
+``.attrib`` property of an empty SelectorList is empty::
+
+    >>> response.css('foo').attrib
+    {}
 
 Using selectors with regular expressions
 ----------------------------------------
@@ -241,8 +385,9 @@ Here's an example used to extract image names from the :ref:`HTML code
      'My image 4',
      'My image 5']
 
-There's an additional helper reciprocating ``.extract_first()`` for ``.re()``,
-named ``.re_first()``. Use it to extract just the first matching string::
+There's an additional helper reciprocating ``.get()`` (and its
+alias ``.extract_first()``) for ``.re()``, named ``.re_first()``.
+Use it to extract just the first matching string::
 
     >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
     'My image 1'
@@ -266,17 +411,17 @@ it actually extracts all ``<p>`` elements from the document, not only those
 inside ``<div>`` elements::
 
     >>> for p in divs.xpath('//p'):  # this is wrong - gets all <p> from the whole document
-    ...     print p.extract()
+    ...     print(p.get())
 
 This is the proper way to do it (note the dot prefixing the ``.//p`` XPath)::
 
     >>> for p in divs.xpath('.//p'):  # extracts all <p> inside
-    ...     print p.extract()
+    ...     print(p.get())
 
 Another common case would be to extract all direct ``<p>`` children::
 
     >>> for p in divs.xpath('p'):
-    ...     print p.extract()
+    ...     print(p.get())
 
 For more details about relative XPaths see the `Location Paths`_ section in the
 XPath specification.
@@ -298,14 +443,14 @@ Here's an example to match an element based on its "id" attribute value,
 without hard-coding it (that was shown previously)::
 
     >>> # `$val` used in the expression, a `val` argument needs to be passed
-    >>> response.xpath('//div[@id=$val]/a/text()', val='images').extract_first()
-    u'Name: My image 1 '
+    >>> response.xpath('//div[@id=$val]/a/text()', val='images').get()
+    'Name: My image 1 '
 
 Here's another example, to find the "id" attribute of a ``<div>`` tag containing
 five ``<a>`` children (here we pass the value ``5`` as an integer)::
 
-    >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).extract_first()
-    u'images'
+    >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get()
+    'images'
 
 All variable references must have a binding value when calling ``.xpath()``
 (otherwise you'll get a ``ValueError: XPath error:`` exception).
@@ -314,13 +459,12 @@ This is done by passing as many named arguments as necessary.
 `parsel`_, the library powering Scrapy selectors, has more details and examples
 on `XPath variables`_.
 
-.. _parsel: https://parsel.readthedocs.io/
 .. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions
 
 Using EXSLT extensions
 ----------------------
 
-Being built atop `lxml`_, Scrapy selectors also support some `EXSLT`_ extensions
+Being built atop `lxml`_, Scrapy selectors support some `EXSLT`_ extensions
 and come with these pre-registered namespaces to use in XPath expressions:
 
 
@@ -340,7 +484,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's
 Example selecting links in list item with a "class" attribute ending with a digit::
 
     >>> from scrapy import Selector
-    >>> doc = """
+    >>> doc = u"""
     ... <div>
     ...     <ul>
     ...         <li class="item-0"><a href="link1.html">first item</a></li>
@@ -352,10 +496,10 @@ Example selecting links in list item with a "class" attribute ending with a digi
     ... </div>
     ... """
     >>> sel = Selector(text=doc, type="html")
-    >>> sel.xpath('//li//@href').extract()
-    [u'link1.html', u'link2.html', u'link3.html', u'link4.html', u'link5.html']
-    >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').extract()
-    [u'link1.html', u'link2.html', u'link4.html', u'link5.html']
+    >>> sel.xpath('//li//@href').getall()
+    ['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html']
+    >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall()
+    ['link1.html', 'link2.html', 'link4.html', 'link5.html']
     >>>
 
 .. warning:: C library ``libxslt`` doesn't natively support EXSLT regular
@@ -372,7 +516,7 @@ extracting text elements for example.
 Example extracting microdata (sample content taken from http://schema.org/Product)
 with groups of itemscopes and corresponding itemprops::
 
-    >>> doc = """
+    >>> doc = u"""
     ... <div itemscope itemtype="http://schema.org/Product">
     ...   <span itemprop="name">Kenmore White 17" Microwave</span>
     ...   <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
@@ -424,12 +568,12 @@ with groups of itemscopes and corresponding itemprops::
     ... """
     >>> sel = Selector(text=doc, type="html")
     >>> for scope in sel.xpath('//div[@itemscope]'):
-    ...     print "current scope:", scope.xpath('@itemtype').extract()
+    ...     print("current scope:", scope.xpath('@itemtype').getall())
     ...     props = scope.xpath('''
     ...                 set:difference(./descendant::*/@itemprop,
     ...                                .//*[@itemscope]/*/@itemprop)''')
-    ...     print "    properties:", props.extract()
-    ...     print
+    ...     print("    properties: %s" % (props.getall()))
+    ...     print("")
 
     current scope: ['http://schema.org/Product']
         properties: ['name', 'aggregateRating', 'offers', 'description', 'review', 'review']
@@ -493,27 +637,27 @@ Example::
 
 Converting a *node-set* to string::
 
-    >>> sel.xpath('//a//text()').extract() # take a peek at the node-set
-    [u'Click here to go to the ', u'Next Page']
-    >>> sel.xpath("string(//a[1]//text())").extract() # convert it to string
-    [u'Click here to go to the ']
+    >>> sel.xpath('//a//text()').getall() # take a peek at the node-set
+    ['Click here to go to the ', 'Next Page']
+    >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
+    ['Click here to go to the ']
 
 A *node* converted to a string, however, puts together the text of itself plus of all its descendants::
 
-    >>> sel.xpath("//a[1]").extract() # select the first node
-    [u'<a href="#">Click here to go to the <strong>Next Page</strong></a>']
-    >>> sel.xpath("string(//a[1])").extract() # convert it to string
-    [u'Click here to go to the Next Page']
+    >>> sel.xpath("//a[1]").getall() # select the first node
+    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+    >>> sel.xpath("string(//a[1])").getall() # convert it to string
+    ['Click here to go to the Next Page']
 
 So, using the ``.//text()`` node-set won't select anything in this case::
 
-    >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").extract()
+    >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
     []
 
 But using the ``.`` to mean the node, works::
 
-    >>> sel.xpath("//a[contains(., 'Next Page')]").extract()
-    [u'<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+    >>> sel.xpath("//a[contains(., 'Next Page')]").getall()
+    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
 
 .. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
 
@@ -538,7 +682,7 @@ Example::
     ....:         <li>5</li>
     ....:         <li>6</li>
     ....:     </ul>""")
-    >>> xp = lambda x: sel.xpath(x).extract()
+    >>> xp = lambda x: sel.xpath(x).getall()
 
 This gets all first ``<li>``  elements under whatever it is its parent::
 
@@ -578,12 +722,59 @@ you can just select by class using CSS and then switch to XPath when needed::
 
     >>> from scrapy import Selector
     >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
-    >>> sel.css('.shout').xpath('./time/@datetime').extract()
-    [u'2014-07-23 19:00']
+    >>> sel.css('.shout').xpath('./time/@datetime').getall()
+    ['2014-07-23 19:00']
 
 This is cleaner than using the verbose XPath trick shown above. Just remember
 to use the ``.`` in the XPath expressions that will follow.
 
+.. _old-extraction-api:
+
+extract() and extract_first()
+-----------------------------
+
+If you're a long-time Scrapy user, you're probably familiar
+with ``.extract()`` and ``.extract_first()`` selector methods. These methods
+are still supported by Scrapy, there are no plans to deprecate them.
+
+However, Scrapy usage docs are now written using ``.get()`` and
+``.getall()`` methods. We feel that these new methods result in a more concise
+and readable code.
+
+The following examples show how these methods map to each other.
+
+1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``::
+
+     >>> response.css('a::attr(href)').get()
+     'image1.html'
+     >>> response.css('a::attr(href)').extract_first()
+     'image1.html'
+
+2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``::
+
+     >>> response.css('a::attr(href)').getall()
+     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+     >>> response.css('a::attr(href)').extract()
+     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+2. ``Selector.get()`` is the same as ``Selector.extract()``::
+
+     >>> response.css('a::attr(href)')[0].get()
+     'image1.html'
+     >>> response.css('a::attr(href)')[0].extract()
+     'image1.html'
+
+4. For consistency, there is also ``Selector.getall()``, which returns a list::
+
+    >>> response.css('a::attr(href)')[0].getall()
+    ['image1.html']
+
+So, the main difference is that output of ``.get()`` and ``.getall()`` methods
+is more predictable: ``.get()`` always returns a single result, ``.getall()``
+always returns a list of all extracted results. With ``.extract()`` method
+it was not always obvious if a result is a list or not; to get a single
+result either ``.extract()`` or ``.extract_first()`` should be called.
+
 
 .. _topics-selectors-ref:
 
@@ -718,10 +909,12 @@ SelectorList objects
        their results flattened, as a list of unicode strings.
 
 
+.. _selector-examples-html:
+
 Selector examples on HTML response
 ----------------------------------
 
-Here's a couple of :class:`Selector` examples to illustrate several concepts.
+Here are some :class:`Selector` examples to illustrate several concepts.
 In all cases, we assume there is already a :class:`Selector` instantiated with
 a :class:`~scrapy.http.HtmlResponse` object like this::
 
@@ -735,20 +928,22 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
 2. Extract the text of all ``<h1>`` elements from an HTML response body,
    returning a list of unicode strings::
 
-      sel.xpath("//h1").extract()         # this includes the h1 tag
-      sel.xpath("//h1/text()").extract()  # this excludes the h1 tag
+      sel.xpath("//h1").getall()         # this includes the h1 tag
+      sel.xpath("//h1/text()").getall()  # this excludes the h1 tag
 
 3. Iterate over all ``<p>`` tags and print their class attribute::
 
       for node in sel.xpath("//p"):
-          print node.xpath("@class").extract()
+          print(node.attrib['class'])
+
+
+.. _selector-examples-xml:
 
 Selector examples on XML response
 ---------------------------------
 
-Here's a couple of examples to illustrate several concepts. In both cases we
-assume there is already a :class:`Selector` instantiated with an
-:class:`~scrapy.http.XmlResponse` object like this::
+Here are some examples to illustrate concepts for :class:`Selector` objects
+instantiated with an :class:`~scrapy.http.XmlResponse` object::
 
       sel = Selector(xml_response)
 
@@ -761,7 +956,7 @@ assume there is already a :class:`Selector` instantiated with an
    a namespace::
 
       sel.register_namespace("g", "http://base.google.com/ns/1.0")
-      sel.xpath("//g:price").extract()
+      sel.xpath("//g:price").getall()
 
 .. _removing-namespaces:
 
@@ -781,6 +976,20 @@ First, we open the shell with the url we want to scrape::
 
     $ scrapy shell https://github.com/blog.atom
 
+.. highlight:: xml
+
+This is how the file starts::
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <feed xml:lang="en-US"
+          xmlns="http://www.w3.org/2005/Atom"
+          xmlns:media="http://search.yahoo.com/mrss/">
+      <id>tag:github.com,2008:/blog</id>
+      ...
+
+You can see two namespace declarations: a default "http://www.w3.org/2005/Atom"
+and another one using the "media:" prefix for "http://search.yahoo.com/mrss/".
+
 .. highlight:: python
 
 Once in the shell we can try selecting all ``<link>`` objects and see that it
@@ -794,8 +1003,8 @@ nodes can be accessed directly by their names::
 
     >>> response.selector.remove_namespaces()
     >>> response.xpath("//link")
-    [<Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>,
-     <Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>,
+    [<Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
+     <Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
      ...
 
 If you wonder why the namespace removal procedure isn't always called by default
@@ -803,8 +1012,8 @@ instead of having to call it manually, this is because of two reasons, which, in
 of relevance, are:
 
 1. Removing namespaces requires to iterate and modify all nodes in the
-   document, which is a reasonably expensive operation to perform for all
-   documents crawled by Scrapy
+   document, which is a reasonably expensive operation to perform by default
+   for all documents crawled by Scrapy
 
 2. There could be some cases where using namespaces is actually required, in
    case some element names clash between namespaces. These cases are very rare

From 09fd6c2a816ebec175ff3ffd945ce3263c9f1570 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 03:59:58 +0500
Subject: [PATCH 1809/2974] DOC unlink Firefox & Firebug sections from the
 tutorial for now.

See https://github.com/scrapy/scrapy/issues/3373 and https://github.com/scrapy/scrapy/issues/3372 for motivation.
---
 docs/intro/tutorial.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index a24cf0f5b..453f5114f 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -299,7 +299,8 @@ expressions`::
 
 In order to find the proper CSS selectors to use, you might find useful opening
 the response page from the shell in your web browser using ``view(response)``.
-You can use your browser developer tools (see section about :ref:`topics-developer-tools`).
+You can use your browser developer tools to inspect the HTML and come up
+with a selector (see section about :ref:`topics-developer-tools`).
 
 `Selector Gadget`_ is also a nice tool to quickly find CSS selector for
 visually selected elements, which works in many browsers.

From 2c48d156db7d7846d80fa379dfb33552ac5b5a85 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:01:17 +0500
Subject: [PATCH 1810/2974] DOC cleanup references in tutorials:

* remove unused link
* fix ReST syntax
* fix a link to regular expression docs
---
 docs/intro/tutorial.rst   | 3 +--
 docs/topics/selectors.rst | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 453f5114f..92d1065af 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -288,7 +288,7 @@ to be scraped, you can at least get **some** data.
 Besides the :meth:`~scrapy.selector.Selector.extract` and
 :meth:`~scrapy.selector.SelectorList.get` methods, you can also use
 the :meth:`~scrapy.selector.Selector.re` method to extract using `regular
-expressions`::
+expressions`_::
 
     >>> response.css('title::text').re(r'Quotes.*')
     ['Quotes to Scrape']
@@ -740,4 +740,3 @@ modeling the scraped data. If you prefer to play with an example project, check
 the :ref:`intro-examples` section.
 
 .. _JSON: https://en.wikipedia.org/wiki/JSON
-.. _dirbot: https://github.com/scrapy/dirbot
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 00158ecf1..cf1b0ba22 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -251,7 +251,7 @@ that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**:
 .. warning::
     These pseudo-elements are Scrapy-/Parsel-specific.
     They will most probably not work with other libraries like
-   `lxml`_ or `PyQuery`_.
+    `lxml`_ or `PyQuery`_.
 
 .. _PyQuery: https://pypi.python.org/pypi/pyquery
 

From 53da56c8dcd6edba12284ed5d125ddd1dcf6f99c Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:17:55 +0500
Subject: [PATCH 1811/2974] TST update tests to use get/getall/attrib instead
 of extract

---
 tests/test_command_shell.py   |  2 +-
 tests/test_http_response.py   | 34 +++++++++++-----------
 tests/test_loader.py          |  4 +--
 tests/test_pipeline_crawl.py  |  2 +-
 tests/test_selector.py        | 24 +++++++--------
 tests/test_spider.py          |  8 ++---
 tests/test_utils_iterators.py | 55 ++++++++++++++++++-----------------
 7 files changed, 66 insertions(+), 63 deletions(-)

diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 3e27d6abd..36baacfbd 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -35,7 +35,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_response_selector_html(self):
-        xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]'
+        xpath = 'response.xpath("//p[@class=\'one\']/text()").get()'
         _, out, _ = yield self.execute([self.url('/html'), '-c', xpath])
         self.assertEqual(out.strip(), b'Works')
 
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 820758dc9..3b90e3dac 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -336,11 +336,11 @@ class TextResponseTest(BaseResponseTest):
         self.assertIs(response.selector.response, response)
 
         self.assertEqual(
-            response.selector.xpath("//title/text()").extract(),
+            response.selector.xpath("//title/text()").getall(),
             [u'Some page']
         )
         self.assertEqual(
-            response.selector.css("title::text").extract(),
+            response.selector.css("title::text").getall(),
             [u'Some page']
         )
         self.assertEqual(
@@ -353,12 +353,12 @@ class TextResponseTest(BaseResponseTest):
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
-            response.xpath("//title/text()").extract(),
-            response.selector.xpath("//title/text()").extract(),
+            response.xpath("//title/text()").getall(),
+            response.selector.xpath("//title/text()").getall(),
         )
         self.assertEqual(
-            response.css("title::text").extract(),
-            response.selector.css("title::text").extract(),
+            response.css("title::text").getall(),
+            response.selector.css("title::text").getall(),
         )
 
     def test_selector_shortcuts_kwargs(self):
@@ -366,13 +366,13 @@ class TextResponseTest(BaseResponseTest):
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
-            response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(),
-            response.xpath("normalize-space(//p[@class=\"content\"])").extract(),
+            response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").getall(),
+            response.xpath("normalize-space(//p[@class=\"content\"])").getall(),
         )
         self.assertEqual(
             response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()",
-                pclass="content", pcount=1).extract(),
-            response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(),
+                pclass="content", pcount=1).getall(),
+            response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(),
         )
 
     def test_urljoin_with_base_url(self):
@@ -562,7 +562,7 @@ class XmlResponseTest(TextResponseTest):
         self.assertIs(response.selector.response, response)
 
         self.assertEqual(
-            response.selector.xpath("//elem/text()").extract(),
+            response.selector.xpath("//elem/text()").getall(),
             [u'value']
         )
 
@@ -571,8 +571,8 @@ class XmlResponseTest(TextResponseTest):
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
-            response.xpath("//elem/text()").extract(),
-            response.selector.xpath("//elem/text()").extract(),
+            response.xpath("//elem/text()").getall(),
+            response.selector.xpath("//elem/text()").getall(),
         )
 
     def test_selector_shortcuts_kwargs(self):
@@ -583,12 +583,12 @@ class XmlResponseTest(TextResponseTest):
         response = self.response_class("http://www.example.com", body=body)
 
         self.assertEqual(
-            response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
-            response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
+            response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),
+            response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),
         )
 
         response.selector.register_namespace('s2', 'http://scrapy.org')
         self.assertEqual(
-            response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(),
-            response.selector.xpath("//s2:elem/text()").extract(),
+            response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(),
+            response.selector.xpath("//s2:elem/text()").getall(),
         )
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 3b5714058..8b58e4dbd 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -634,7 +634,7 @@ class SubselectorLoaderTest(unittest.TestCase):
         nl = l.nested_xpath("//header")
         nl.add_xpath('name', 'div/text()')
         nl.add_css('name_div', '#id')
-        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
+        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
 
         self.assertEqual(l.get_output_value('name'), [u'marta'])
         self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
@@ -649,7 +649,7 @@ class SubselectorLoaderTest(unittest.TestCase):
         nl = l.nested_css("header")
         nl.add_xpath('name', 'div/text()')
         nl.add_css('name_div', '#id')
-        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
+        nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
 
         self.assertEqual(l.get_output_value('name'), [u'marta'])
         self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 5985a6f3e..fb72c9d6d 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -29,7 +29,7 @@ class MediaDownloadSpider(SimpleSpider):
                     for href in response.xpath('''
                         //table[thead/tr/th="Filename"]
                             /tbody//a/@href
-                        ''').extract()],
+                        ''').getall()],
         }
         yield item
 
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 526660cc8..bc4baf7ea 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -20,17 +20,17 @@ class SelectorTestCase(unittest.TestCase):
         for x in xl:
             assert isinstance(x, Selector)
 
-        self.assertEqual(sel.xpath('//input').extract(),
-                         [x.extract() for x in sel.xpath('//input')])
+        self.assertEqual(sel.xpath('//input').getall(),
+                         [x.get() for x in sel.xpath('//input')])
 
-        self.assertEqual([x.extract() for x in sel.xpath("//input[@name='a']/@name")],
+        self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")],
                          [u'a'])
-        self.assertEqual([x.extract() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
+        self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
                          [u'12.0'])
 
-        self.assertEqual(sel.xpath("concat('xpath', 'rules')").extract(),
+        self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(),
                          [u'xpathrules'])
-        self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
+        self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
                          [u'12'])
 
     def test_root_base_url(self):
@@ -60,12 +60,12 @@ class SelectorTestCase(unittest.TestCase):
         text = b'<div><img src="a.jpg"><p>Hello</div>'
         sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
         self.assertEqual(sel.type, 'xml')
-        self.assertEqual(sel.xpath("//div").extract(),
+        self.assertEqual(sel.xpath("//div").getall(),
                          [u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
 
         sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
         self.assertEqual(sel.type, 'html')
-        self.assertEqual(sel.xpath("//div").extract(),
+        self.assertEqual(sel.xpath("//div").getall(),
                          [u'<div><img src="a.jpg"><p>Hello</p></div>'])
 
     def test_http_header_encoding_precedence(self):
@@ -84,15 +84,15 @@ class SelectorTestCase(unittest.TestCase):
         headers = {'Content-Type': ['text/html; charset=utf-8']}
         response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
         x = Selector(response)
-        self.assertEqual(x.xpath("//span[@id='blank']/text()").extract(),
+        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),
                           [u'\xa3'])
 
     def test_badly_encoded_body(self):
         # \xe9 alone isn't valid utf8 sequence
-        r1 = TextResponse('http://www.example.com', \
-                          body=b'<html><p>an Jos\xe9 de</p><html>', \
+        r1 = TextResponse('http://www.example.com',
+                          body=b'<html><p>an Jos\xe9 de</p><html>',
                           encoding='utf-8')
-        Selector(r1).xpath('//text()').extract()
+        Selector(r1).xpath('//text()').getall()
 
     def test_weakref_slots(self):
         """Check that classes are using slots and are weak-referenceable"""
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 929e0fea8..f26da2334 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -147,10 +147,10 @@ class XMLFeedSpiderTest(SpiderTest):
 
             def parse_node(self, response, selector):
                 yield {
-                    'loc': selector.xpath('a:loc/text()').extract(),
-                    'updated': selector.xpath('b:updated/text()').extract(),
-                    'other': selector.xpath('other/@value').extract(),
-                    'custom': selector.xpath('other/@b:custom').extract(),
+                    'loc': selector.xpath('a:loc/text()').getall(),
+                    'updated': selector.xpath('b:updated/text()').getall(),
+                    'other': selector.xpath('other/@value').getall(),
+                    'custom': selector.xpath('other/@b:custom').getall(),
                 }
 
         for iterator in ('iternodes', 'xml'):
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index f953076b8..00eb78068 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -30,10 +30,13 @@ class XmliterTestCase(unittest.TestCase):
         response = XmlResponse(url="http://example.com", body=body)
         attrs = []
         for x in self.xmliter(response, 'product'):
-            attrs.append((x.xpath("@id").extract(), x.xpath("name/text()").extract(), x.xpath("./type/text()").extract()))
+            attrs.append((
+                x.attrib['id'],
+                x.xpath("name/text()").extract(),
+                x.xpath("./type/text()").extract()))
 
         self.assertEqual(attrs,
-                         [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])])
+                         [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])])
 
     def test_xmliter_unusual_node(self):
         body = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -43,7 +46,7 @@ class XmliterTestCase(unittest.TestCase):
             </root>
         """
         response = XmlResponse(url="http://example.com", body=body)
-        nodenames = [e.xpath('name()').extract()
+        nodenames = [e.xpath('name()').getall()
                  for e in self.xmliter(response, 'matchme...')]
         self.assertEqual(nodenames, [['matchme...']])
 
@@ -93,19 +96,19 @@ class XmliterTestCase(unittest.TestCase):
 
             attrs = []
             for x in self.xmliter(r, u'þingflokkur'):
-                attrs.append((x.xpath('@id').extract(),
-                              x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),
-                              x.xpath(u'./tímabil/fyrstaþing/text()').extract()))
+                attrs.append((x.attrib['id'],
+                              x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(),
+                              x.xpath(u'./tímabil/fyrstaþing/text()').getall()))
 
             self.assertEqual(attrs,
-                             [([u'26'], [u'-'], [u'80']),
-                              ([u'21'], [u'Ab'], [u'76']),
-                              ([u'27'], [u'A'], [u'27'])])
+                             [(u'26', [u'-'], [u'80']),
+                              (u'21', [u'Ab'], [u'76']),
+                              (u'27', [u'A'], [u'27'])])
 
     def test_xmliter_text(self):
         body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
 
-        self.assertEqual([x.xpath("text()").extract() for x in self.xmliter(body, 'product')],
+        self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
                          [[u'one'], [u'two']])
 
     def test_xmliter_namespaces(self):
@@ -132,15 +135,15 @@ class XmliterTestCase(unittest.TestCase):
 
         node = next(my_iter)
         node.register_namespace('g', 'http://base.google.com/ns/1.0')
-        self.assertEqual(node.xpath('title/text()').extract(), ['Item 1'])
-        self.assertEqual(node.xpath('description/text()').extract(), ['This is item 1'])
-        self.assertEqual(node.xpath('link/text()').extract(), ['http://www.mydummycompany.com/items/1'])
-        self.assertEqual(node.xpath('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
-        self.assertEqual(node.xpath('g:id/text()').extract(), ['ITEM_1'])
-        self.assertEqual(node.xpath('g:price/text()').extract(), ['400'])
-        self.assertEqual(node.xpath('image_link/text()').extract(), [])
-        self.assertEqual(node.xpath('id/text()').extract(), [])
-        self.assertEqual(node.xpath('price/text()').extract(), [])
+        self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])
+        self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1'])
+        self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1'])
+        self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
+        self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1'])
+        self.assertEqual(node.xpath('g:price/text()').getall(), ['400'])
+        self.assertEqual(node.xpath('image_link/text()').getall(), [])
+        self.assertEqual(node.xpath('id/text()').getall(), [])
+        self.assertEqual(node.xpath('price/text()').getall(), [])
 
     def test_xmliter_exception(self):
         body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
@@ -159,7 +162,7 @@ class XmliterTestCase(unittest.TestCase):
         body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
         response = XmlResponse('http://www.example.com', body=body)
         self.assertEqual(
-            next(self.xmliter(response, 'item')).extract(),
+            next(self.xmliter(response, 'item')).get(),
             u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
         )
 
@@ -192,9 +195,9 @@ class LxmlXmliterTestCase(XmliterTestCase):
 
         namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0')
         node = next(namespace_iter)
-        self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
+        self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
         node = next(namespace_iter)
-        self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg'])
+        self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg'])
 
     def test_xmliter_namespaces_prefix(self):
         body = b"""\
@@ -219,14 +222,14 @@ class LxmlXmliterTestCase(XmliterTestCase):
         my_iter = self.xmliter(response, 'table', 'http://www.w3.org/TR/html4/', 'h')
 
         node = next(my_iter)
-        self.assertEqual(len(node.xpath('h:tr/h:td').extract()), 2)
-        self.assertEqual(node.xpath('h:tr/h:td[1]/text()').extract(), ['Apples'])
-        self.assertEqual(node.xpath('h:tr/h:td[2]/text()').extract(), ['Bananas'])
+        self.assertEqual(len(node.xpath('h:tr/h:td').getall()), 2)
+        self.assertEqual(node.xpath('h:tr/h:td[1]/text()').getall(), ['Apples'])
+        self.assertEqual(node.xpath('h:tr/h:td[2]/text()').getall(), ['Bananas'])
 
         my_iter = self.xmliter(response, 'table', 'http://www.w3schools.com/furniture', 'f')
 
         node = next(my_iter)
-        self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table'])
+        self.assertEqual(node.xpath('f:name/text()').getall(), ['African Coffee Table'])
 
     def test_xmliter_objtype_exception(self):
         i = self.xmliter(42, 'product')

From 8c29be606c1be71059b454e41b87354e22569423 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:18:29 +0500
Subject: [PATCH 1812/2974] update spider templates to use .get

---
 scrapy/templates/spiders/crawl.tmpl   | 10 +++++-----
 scrapy/templates/spiders/xmlfeed.tmpl | 10 +++++-----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 802cb88a1..878425125 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -14,8 +14,8 @@ class $classname(CrawlSpider):
     )
 
     def parse_item(self, response):
-        i = {}
-        #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
-        #i['name'] = response.xpath('//div[@id="name"]').extract()
-        #i['description'] = response.xpath('//div[@id="description"]').extract()
-        return i
+        item = {}
+        #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
+        #item['name'] = response.xpath('//div[@id="name"]').get()
+        #item['description'] = response.xpath('//div[@id="description"]').get()
+        return item
diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl
index 7c2ff8850..863c9772f 100644
--- a/scrapy/templates/spiders/xmlfeed.tmpl
+++ b/scrapy/templates/spiders/xmlfeed.tmpl
@@ -10,8 +10,8 @@ class $classname(XMLFeedSpider):
     itertag = 'item' # change it accordingly
 
     def parse_node(self, response, selector):
-        i = {}
-        #i['url'] = selector.select('url').extract()
-        #i['name'] = selector.select('name').extract()
-        #i['description'] = selector.select('description').extract()
-        return i
+        item = {}
+        #item['url'] = selector.select('url').get()
+        #item['name'] = selector.select('name').get()
+        #item['description'] = selector.select('description').get()
+        return item

From 460f0f045141f73dcc6d7a2309b7e2f0c2a492b4 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:20:32 +0500
Subject: [PATCH 1813/2974] [backwards incompatible] switch ItemLoader from
 .extract to .getall.

This change is backwards incompatible if ItemLoader is used with a custom Selector
subclass which overrides .extract without overriding .getall.
---
 scrapy/loader/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index e73413318..a7c75a46a 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -181,7 +181,7 @@ class ItemLoader(object):
     def _get_xpathvalues(self, xpaths, **kw):
         self._check_selector_method()
         xpaths = arg_to_iter(xpaths)
-        return flatten(self.selector.xpath(xpath).extract() for xpath in xpaths)
+        return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths)
 
     def add_css(self, field_name, css, *processors, **kw):
         values = self._get_cssvalues(css, **kw)
@@ -198,6 +198,6 @@ class ItemLoader(object):
     def _get_cssvalues(self, csss, **kw):
         self._check_selector_method()
         csss = arg_to_iter(csss)
-        return flatten(self.selector.css(css).extract() for css in csss)
+        return flatten(self.selector.css(css).getall() for css in csss)
 
 XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)

From 12e42bbe06bea5a0f86e95630b184e86b41c95c7 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:20:55 +0500
Subject: [PATCH 1814/2974] switch SgmlLinkExtractor to .getall

---
 scrapy/linkextractors/sgml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index f4ca4262a..5fa6b771c 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -141,7 +141,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
             base_url = get_base_url(response)
             body = u''.join(f
                             for x in self.restrict_xpaths
-                            for f in response.xpath(x).extract()
+                            for f in response.xpath(x).getall()
                             ).encode(response.encoding, errors='xmlcharrefreplace')
         else:
             body = response.body

From afce9716fabff6169dc9cc146a2589ebf4c1ed13 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:25:43 +0500
Subject: [PATCH 1815/2974] DOC mention .attrib in the tutorial

---
 docs/intro/tutorial.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 92d1065af..07fb4807f 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -520,6 +520,12 @@ like this::
     >>> response.css('li.next a::attr(href)').get()
     '/page/2/'
 
+There is also an ``attrib`` property available
+(see :ref:`selecting-attributes` for more)::
+
+    >>> response.css('li.next a').attrib['href']
+    '/page/2'
+
 Let's see now our spider modified to recursively follow the link to the next
 page, extracting data from it::
 

From bdcc045f62a34d1655a4ead4e0b38d6fe97a08fc Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 04:27:21 +0500
Subject: [PATCH 1816/2974] DOC switch from .extract to get/getall API in docs

Also, response.urljoin is added in a few places, for robustness.
---
 docs/intro/tutorial.rst | 22 +++++++++++-----------
 docs/topics/shell.rst   |  4 ++--
 docs/topics/spiders.rst | 14 +++++++-------
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 07fb4807f..46e84b21c 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -254,7 +254,7 @@ data.
 
 To extract the text from the title above, you can do::
 
-    >>> response.css('title::text').extract()
+    >>> response.css('title::text').getall()
     ['Quotes to Scrape']
 
 There are two things to note here: one is that we've added ``::text`` to the
@@ -262,12 +262,12 @@ CSS query, to mean we want to select only the text elements directly inside
 ``<title>`` element.  If we don't specify ``::text``, we'd get the full title
 element, including its tags::
 
-    >>> response.css('title').extract()
+    >>> response.css('title').getall()
     ['<title>Quotes to Scrape</title>']
 
-The other thing is that the result of calling ``.extract()`` is a list, because
-we're dealing with an instance of :class:`~scrapy.selector.SelectorList`.  When
-you know you just want the first result, as in this case, you can do::
+The other thing is that the result of calling ``.getall()`` is a list: it is
+possible that a selector returns more than one result, so we extract them all.
+When you know you just want the first result, as in this case, you can do::
 
     >>> response.css('title::text').get()
     'Quotes to Scrape'
@@ -392,10 +392,10 @@ using the ``quote`` object we just created::
     >>> author
     'Albert Einstein'
 
-Given that the tags are a list of strings, we can use the ``.extract()`` method
+Given that the tags are a list of strings, we can use the ``.getall()`` method
 to get all of them::
 
-    >>> tags = quote.css("div.tags a.tag::text").extract()
+    >>> tags = quote.css("div.tags a.tag::text").getall()
     >>> tags
     ['change', 'deep-thoughts', 'thinking', 'world']
 
@@ -405,7 +405,7 @@ quotes elements and put them together into a Python dictionary::
     >>> for quote in response.css("div.quote"):
     ...     text = quote.css("span.text::text").get()
     ...     author = quote.css("small.author::text").get()
-    ...     tags = quote.css("div.tags a.tag::text").extract()
+    ...     tags = quote.css("div.tags a.tag::text").getall()
     ...     print(dict(text=text, author=author, tags=tags))
     {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
     {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
@@ -438,7 +438,7 @@ in the callback, as you can see below::
                 yield {
                     'text': quote.css('span.text::text').get(),
                     'author': quote.css('small.author::text').get(),
-                    'tags': quote.css('div.tags a.tag::text').extract(),
+                    'tags': quote.css('div.tags a.tag::text').getall(),
                 }
 
 If you run this spider, it will output the extracted data with the log::
@@ -543,7 +543,7 @@ page, extracting data from it::
                 yield {
                     'text': quote.css('span.text::text').get(),
                     'author': quote.css('small.author::text').get(),
-                    'tags': quote.css('div.tags a.tag::text').extract(),
+                    'tags': quote.css('div.tags a.tag::text').getall(),
                 }
 
             next_page = response.css('li.next a::attr(href)').get()
@@ -594,7 +594,7 @@ As a shortcut for creating Request objects you can use
                 yield {
                     'text': quote.css('span.text::text').get(),
                     'author': quote.css('span small::text').get(),
-                    'tags': quote.css('div.tags a.tag::text').extract(),
+                    'tags': quote.css('div.tags a.tag::text').getall(),
                 }
 
             next_page = response.css('li.next a::attr(href)').get()
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 9de6abef7..68a0b19b5 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -184,8 +184,8 @@ After that, we can start playing with the objects::
 
     >>> fetch("https://reddit.com")
 
-    >>> response.xpath('//title/text()').extract()
-    ['reddit: the front page of the internet']
+    >>> response.xpath('//title/text()').get()
+    'reddit: the front page of the internet'
 
     >>> request = request.replace(method="POST")
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 4505b7315..a08dc30f2 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -229,11 +229,11 @@ Return multiple Requests and items from a single callback::
         ]
 
         def parse(self, response):
-            for h3 in response.xpath('//h3').extract():
+            for h3 in response.xpath('//h3').getall():
                 yield {"title": h3}
 
-            for url in response.xpath('//a/@href').extract():
-                yield scrapy.Request(url, callback=self.parse)
+            for href in response.xpath('//a/@href').getall():
+                yield scrapy.Request(response.urljoin(href), self.parse)
 
 Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
 to give data more structure you can use :ref:`topics-items`::
@@ -251,11 +251,11 @@ to give data more structure you can use :ref:`topics-items`::
             yield scrapy.Request('http://www.example.com/3.html', self.parse)
 
         def parse(self, response):
-            for h3 in response.xpath('//h3').extract():
+            for h3 in response.xpath('//h3').getall():
                 yield MyItem(title=h3)
 
-            for url in response.xpath('//a/@href').extract():
-                yield scrapy.Request(url, callback=self.parse)
+            for href in response.xpath('//a/@href').getall():
+                yield scrapy.Request(response.urljoin(href), self.parse)
 
 .. _spiderargs:
 
@@ -545,7 +545,7 @@ These spiders are pretty easy to use, let's have a look at one example::
         itertag = 'item'
 
         def parse_node(self, response, node):
-            self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract()))
+            self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.getall()))
 
             item = TestItem()
             item['id'] = node.xpath('@id').get()

From 7fdfdb7fa244ed44274aaa9750efec1c333eedd1 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 22 Aug 2018 17:34:44 +0500
Subject: [PATCH 1817/2974] DOC reorganize selectors tutorial, port more topics
 from parsel docs, adjust wording in the introduction

---
 docs/topics/selectors.rst | 514 +++++++++++++++++++++-----------------
 1 file changed, 281 insertions(+), 233 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index cf1b0ba22..2db982b37 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -25,17 +25,14 @@ either by `XPath`_ or `CSS`_ expressions.
 used with HTML. `CSS`_ is a language for applying styles to HTML documents. It
 defines selectors to associate those styles with specific HTML elements.
 
-Scrapy selectors are powered by `parsel`_ library, which uses `lxml`_ library
-under the hood. It means Scrapy selectors are very similar in speed and
-parsing accuracy to lxml.
+.. note::
+    Scrapy Selectors is a thin wrapper around `parsel`_ library; the purpose of
+    this wrapper is to provide better integration with Scrapy Response objects.
 
-This page explains how selectors work and describes their API which is very
-small and simple, unlike the `lxml`_ API which is much bigger because the
-`lxml`_ library can be used for many other tasks, besides selecting markup
-documents.
-
-For a complete reference of the selectors API see
-:ref:`Selector reference <topics-selectors-ref>`
+    `parsel`_ is a stand-alone web scraping library which can be used without
+    Scrapy. It uses `lxml`_ library under the hood, and implements an
+    easy API on top of lxml API. It means Scrapy selectors are very similar
+    in speed and parsing accuracy to lxml.
 
 .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
 .. _lxml: http://lxml.de/
@@ -73,9 +70,8 @@ Constructing from response::
     >>> Selector(response=response).xpath('//span/text()').get()
     'good'
 
-For convenience, response objects expose a selector on `.selector` attribute,
-it's totally OK to use this shortcut when possible. By using it you can
-ensure the response body is parsed only once::
+For convenience, response objects expose a selector on `.selector` attribute.
+By using it you can ensure the response body is parsed only once::
 
     >>> response.selector.xpath('//span/text()').get()
     'good'
@@ -88,8 +84,10 @@ more shortcuts: ``response.xpath()`` and ``response.css()``::
     >>> response.css('span::text').get()
     'good'
 
-Usually there is no need to construct Scrapy selectors manually because of
-these shortcuts.
+Usually there is no need to construct Scrapy selectors manually:
+``response`` object is available in Spider callbacks, so in most cases
+it is more convenient to use ``response.css()`` and ``response.xpath()``
+shortcuts.
 
 Using selectors
 ---------------
@@ -392,6 +390,71 @@ Use it to extract just the first matching string::
     >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
     'My image 1'
 
+.. _old-extraction-api:
+
+extract() and extract_first()
+-----------------------------
+
+If you're a long-time Scrapy user, you're probably familiar
+with ``.extract()`` and ``.extract_first()`` selector methods. Many blog posts
+and tutorials are using them as well. These methods are still supported
+by Scrapy, there are **no plans** to deprecate them.
+
+However, Scrapy usage docs are now written using ``.get()`` and
+``.getall()`` methods. We feel that these new methods result in a more concise
+and readable code.
+
+The following examples show how these methods map to each other.
+
+1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``::
+
+     >>> response.css('a::attr(href)').get()
+     'image1.html'
+     >>> response.css('a::attr(href)').extract_first()
+     'image1.html'
+
+2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``::
+
+     >>> response.css('a::attr(href)').getall()
+     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+     >>> response.css('a::attr(href)').extract()
+     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+2. ``Selector.get()`` is the same as ``Selector.extract()``::
+
+     >>> response.css('a::attr(href)')[0].get()
+     'image1.html'
+     >>> response.css('a::attr(href)')[0].extract()
+     'image1.html'
+
+4. For consistency, there is also ``Selector.getall()``, which returns a list::
+
+    >>> response.css('a::attr(href)')[0].getall()
+    ['image1.html']
+
+So, the main difference is that output of ``.get()`` and ``.getall()`` methods
+is more predictable: ``.get()`` always returns a single result, ``.getall()``
+always returns a list of all extracted results. With ``.extract()`` method
+it was not always obvious if a result is a list or not; to get a single
+result either ``.extract()`` or ``.extract_first()`` should be called.
+
+
+.. _topics-selectors-xpaths:
+
+Working with XPaths
+===================
+
+Here are some tips which may help you to use XPath with Scrapy selectors
+effectively. If you are not much familiar with XPath yet,
+you may want to take a look first at this `XPath tutorial`_.
+
+.. note::
+    Some of the tips are based on `this post from ScrapingHub's blog`_.
+
+.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
+.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
+
+
 .. _topics-selectors-relative-xpaths:
 
 Working with relative XPaths
@@ -428,6 +491,114 @@ XPath specification.
 
 .. _Location Paths: https://www.w3.org/TR/xpath#location-paths
 
+When querying by class, consider using CSS
+------------------------------------------
+
+Because an element can contain multiple CSS classes, the XPath way to select elements
+by class is the rather verbose::
+
+    *[contains(concat(' ', normalize-space(@class), ' '), ' someclass ')]
+
+If you use ``@class='someclass'`` you may end up missing elements that have
+other classes, and if you just use ``contains(@class, 'someclass')`` to make up
+for that you may end up with more elements that you want, if they have a different
+class name that shares the string ``someclass``.
+
+As it turns out, Scrapy selectors allow you to chain selectors, so most of the time
+you can just select by class using CSS and then switch to XPath when needed::
+
+    >>> from scrapy import Selector
+    >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
+    >>> sel.css('.shout').xpath('./time/@datetime').getall()
+    ['2014-07-23 19:00']
+
+This is cleaner than using the verbose XPath trick shown above. Just remember
+to use the ``.`` in the XPath expressions that will follow.
+
+Beware of the difference between //node[1] and (//node)[1]
+----------------------------------------------------------
+
+``//node[1]`` selects all the nodes occurring first under their respective parents.
+
+``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them.
+
+Example::
+
+    >>> from scrapy import Selector
+    >>> sel = Selector(text="""
+    ....:     <ul class="list">
+    ....:         <li>1</li>
+    ....:         <li>2</li>
+    ....:         <li>3</li>
+    ....:     </ul>
+    ....:     <ul class="list">
+    ....:         <li>4</li>
+    ....:         <li>5</li>
+    ....:         <li>6</li>
+    ....:     </ul>""")
+    >>> xp = lambda x: sel.xpath(x).getall()
+
+This gets all first ``<li>``  elements under whatever it is its parent::
+
+    >>> xp("//li[1]")
+    ['<li>1</li>', '<li>4</li>']
+
+And this gets the first ``<li>``  element in the whole document::
+
+    >>> xp("(//li)[1]")
+    ['<li>1</li>']
+
+This gets all first ``<li>``  elements under an ``<ul>``  parent::
+
+    >>> xp("//ul/li[1]")
+    ['<li>1</li>', '<li>4</li>']
+
+And this gets the first ``<li>``  element under an ``<ul>``  parent in the whole document::
+
+    >>> xp("(//ul/li)[1]")
+    ['<li>1</li>']
+
+Using text nodes in a condition
+-------------------------------
+
+When you need to use the text content as argument to an `XPath string function`_,
+avoid using ``.//text()`` and use just ``.`` instead.
+
+This is because the expression ``.//text()`` yields a collection of text elements -- a *node-set*.
+And when a node-set is converted to a string, which happens when it is passed as argument to
+a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only.
+
+Example::
+
+    >>> from scrapy import Selector
+    >>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
+
+Converting a *node-set* to string::
+
+    >>> sel.xpath('//a//text()').getall() # take a peek at the node-set
+    ['Click here to go to the ', 'Next Page']
+    >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
+    ['Click here to go to the ']
+
+A *node* converted to a string, however, puts together the text of itself plus of all its descendants::
+
+    >>> sel.xpath("//a[1]").getall() # select the first node
+    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+    >>> sel.xpath("string(//a[1])").getall() # convert it to string
+    ['Click here to go to the Next Page']
+
+So, using the ``.//text()`` node-set won't select anything in this case::
+
+    >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
+    []
+
+But using the ``.`` to mean the node, works::
+
+    >>> sel.xpath("//a[contains(., 'Next Page')]").getall()
+    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+
+.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
+
 .. _topics-selectors-xpath-variables:
 
 Variables in XPath expressions
@@ -461,6 +632,69 @@ on `XPath variables`_.
 
 .. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions
 
+
+.. _removing-namespaces:
+
+Removing namespaces
+-------------------
+
+When dealing with scraping projects, it is often quite convenient to get rid of
+namespaces altogether and just work with element names, to write more
+simple/convenient XPaths. You can use the
+:meth:`Selector.remove_namespaces` method for that.
+
+Let's show an example that illustrates this with GitHub blog atom feed.
+
+.. highlight:: sh
+
+First, we open the shell with the url we want to scrape::
+
+    $ scrapy shell https://github.com/blog.atom
+
+.. highlight:: xml
+
+This is how the file starts::
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <feed xml:lang="en-US"
+          xmlns="http://www.w3.org/2005/Atom"
+          xmlns:media="http://search.yahoo.com/mrss/">
+      <id>tag:github.com,2008:/blog</id>
+      ...
+
+You can see two namespace declarations: a default "http://www.w3.org/2005/Atom"
+and another one using the "media:" prefix for "http://search.yahoo.com/mrss/".
+
+.. highlight:: python
+
+Once in the shell we can try selecting all ``<link>`` objects and see that it
+doesn't work (because the Atom XML namespace is obfuscating those nodes)::
+
+    >>> response.xpath("//link")
+    []
+
+But once we call the :meth:`Selector.remove_namespaces` method, all
+nodes can be accessed directly by their names::
+
+    >>> response.selector.remove_namespaces()
+    >>> response.xpath("//link")
+    [<Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
+     <Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
+     ...
+
+If you wonder why the namespace removal procedure isn't always called by default
+instead of having to call it manually, this is because of two reasons, which, in order
+of relevance, are:
+
+1. Removing namespaces requires to iterate and modify all nodes in the
+   document, which is a reasonably expensive operation to perform by default
+   for all documents crawled by Scrapy
+
+2. There could be some cases where using namespaces is actually required, in
+   case some element names clash between namespaces. These cases are very rare
+   though.
+
+
 Using EXSLT extensions
 ----------------------
 
@@ -606,174 +840,44 @@ inside another ``itemscope``.
 .. _regular expressions: http://exslt.org/regexp/index.html
 .. _set manipulation: http://exslt.org/set/index.html
 
+Other XPath extensions
+----------------------
 
-Some XPath tips
----------------
+Scrapy selectors also provide a sorely missed XPath extension function
+``has-class`` that returns ``True`` for nodes that have all of the specified
+HTML classes.
 
-Here are some tips that you may find useful when using XPath
-with Scrapy selectors, based on `this post from ScrapingHub's blog`_.
-If you are not much familiar with XPath yet,
-you may want to take a look first at this `XPath tutorial`_.
+.. highlight:: html
 
+For the following HTML::
 
-.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
-.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
+    <p class="foo bar-baz">First</p>
+    <p class="foo">Second</p>
+    <p class="bar">Third</p>
+    <p>Fourth</p>
 
+.. highlight:: python
 
-Using text nodes in a condition
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+You can use it like this::
 
-When you need to use the text content as argument to an `XPath string function`_,
-avoid using ``.//text()`` and use just ``.`` instead.
-
-This is because the expression ``.//text()`` yields a collection of text elements -- a *node-set*.
-And when a node-set is converted to a string, which happens when it is passed as argument to
-a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only.
-
-Example::
-
-    >>> from scrapy import Selector
-    >>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
-
-Converting a *node-set* to string::
-
-    >>> sel.xpath('//a//text()').getall() # take a peek at the node-set
-    ['Click here to go to the ', 'Next Page']
-    >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
-    ['Click here to go to the ']
-
-A *node* converted to a string, however, puts together the text of itself plus of all its descendants::
-
-    >>> sel.xpath("//a[1]").getall() # select the first node
-    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
-    >>> sel.xpath("string(//a[1])").getall() # convert it to string
-    ['Click here to go to the Next Page']
-
-So, using the ``.//text()`` node-set won't select anything in this case::
-
-    >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
+    >>> response.xpath('//p[has-class("foo")]')
+    [<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>,
+     <Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>]
+    >>> response.xpath('//p[has-class("foo", "bar-baz")]')
+    [<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>]
+    >>> response.xpath('//p[has-class("foo", "bar")]')
     []
 
-But using the ``.`` to mean the node, works::
+So XPath ``//p[has-class("foo", "bar-baz")]`` is roughly equivalent to CSS
+``p.foo.bar-baz``.  Please note, that it is slower in most of the cases,
+because it's a pure-Python function that's invoked for every node in question
+whereas the CSS lookup is translated into XPath and thus runs more efficiently,
+so performance-wise its uses are limited to situations that are not easily
+described with CSS selectors.
 
-    >>> sel.xpath("//a[contains(., 'Next Page')]").getall()
-    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+Parsel also simplifies adding your own XPath extensions.
 
-.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
-
-Beware of the difference between //node[1] and (//node)[1]
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-``//node[1]`` selects all the nodes occurring first under their respective parents.
-
-``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them.
-
-Example::
-
-    >>> from scrapy import Selector
-    >>> sel = Selector(text="""
-    ....:     <ul class="list">
-    ....:         <li>1</li>
-    ....:         <li>2</li>
-    ....:         <li>3</li>
-    ....:     </ul>
-    ....:     <ul class="list">
-    ....:         <li>4</li>
-    ....:         <li>5</li>
-    ....:         <li>6</li>
-    ....:     </ul>""")
-    >>> xp = lambda x: sel.xpath(x).getall()
-
-This gets all first ``<li>``  elements under whatever it is its parent::
-
-    >>> xp("//li[1]")
-    ['<li>1</li>', '<li>4</li>']
-
-And this gets the first ``<li>``  element in the whole document::
-
-    >>> xp("(//li)[1]")
-    ['<li>1</li>']
-
-This gets all first ``<li>``  elements under an ``<ul>``  parent::
-
-    >>> xp("//ul/li[1]")
-    ['<li>1</li>', '<li>4</li>']
-
-And this gets the first ``<li>``  element under an ``<ul>``  parent in the whole document::
-
-    >>> xp("(//ul/li)[1]")
-    ['<li>1</li>']
-
-When querying by class, consider using CSS
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Because an element can contain multiple CSS classes, the XPath way to select elements
-by class is the rather verbose::
-
-    *[contains(concat(' ', normalize-space(@class), ' '), ' someclass ')]
-
-If you use ``@class='someclass'`` you may end up missing elements that have
-other classes, and if you just use ``contains(@class, 'someclass')`` to make up
-for that you may end up with more elements that you want, if they have a different
-class name that shares the string ``someclass``.
-
-As it turns out, Scrapy selectors allow you to chain selectors, so most of the time
-you can just select by class using CSS and then switch to XPath when needed::
-
-    >>> from scrapy import Selector
-    >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
-    >>> sel.css('.shout').xpath('./time/@datetime').getall()
-    ['2014-07-23 19:00']
-
-This is cleaner than using the verbose XPath trick shown above. Just remember
-to use the ``.`` in the XPath expressions that will follow.
-
-.. _old-extraction-api:
-
-extract() and extract_first()
------------------------------
-
-If you're a long-time Scrapy user, you're probably familiar
-with ``.extract()`` and ``.extract_first()`` selector methods. These methods
-are still supported by Scrapy, there are no plans to deprecate them.
-
-However, Scrapy usage docs are now written using ``.get()`` and
-``.getall()`` methods. We feel that these new methods result in a more concise
-and readable code.
-
-The following examples show how these methods map to each other.
-
-1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``::
-
-     >>> response.css('a::attr(href)').get()
-     'image1.html'
-     >>> response.css('a::attr(href)').extract_first()
-     'image1.html'
-
-2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``::
-
-     >>> response.css('a::attr(href)').getall()
-     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
-     >>> response.css('a::attr(href)').extract()
-     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
-
-2. ``Selector.get()`` is the same as ``Selector.extract()``::
-
-     >>> response.css('a::attr(href)')[0].get()
-     'image1.html'
-     >>> response.css('a::attr(href)')[0].extract()
-     'image1.html'
-
-4. For consistency, there is also ``Selector.getall()``, which returns a list::
-
-    >>> response.css('a::attr(href)')[0].getall()
-    ['image1.html']
-
-So, the main difference is that output of ``.get()`` and ``.getall()`` methods
-is more predictable: ``.get()`` always returns a single result, ``.getall()``
-always returns a list of all extracted results. With ``.extract()`` method
-it was not always obvious if a result is a list or not; to get a single
-result either ``.extract()`` or ``.extract_first()`` should be called.
+.. autofunction:: parsel.xpathfuncs.set_xpathfunc
 
 
 .. _topics-selectors-ref:
@@ -909,6 +1013,11 @@ SelectorList objects
        their results flattened, as a list of unicode strings.
 
 
+.. _selector-examples:
+
+Examples
+========
+
 .. _selector-examples-html:
 
 Selector examples on HTML response
@@ -958,65 +1067,4 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object::
       sel.register_namespace("g", "http://base.google.com/ns/1.0")
       sel.xpath("//g:price").getall()
 
-.. _removing-namespaces:
-
-Removing namespaces
--------------------
-
-When dealing with scraping projects, it is often quite convenient to get rid of
-namespaces altogether and just work with element names, to write more
-simple/convenient XPaths. You can use the
-:meth:`Selector.remove_namespaces` method for that.
-
-Let's show an example that illustrates this with GitHub blog atom feed.
-
-.. highlight:: sh
-
-First, we open the shell with the url we want to scrape::
-
-    $ scrapy shell https://github.com/blog.atom
-
-.. highlight:: xml
-
-This is how the file starts::
-
-    <?xml version="1.0" encoding="UTF-8"?>
-    <feed xml:lang="en-US"
-          xmlns="http://www.w3.org/2005/Atom"
-          xmlns:media="http://search.yahoo.com/mrss/">
-      <id>tag:github.com,2008:/blog</id>
-      ...
-
-You can see two namespace declarations: a default "http://www.w3.org/2005/Atom"
-and another one using the "media:" prefix for "http://search.yahoo.com/mrss/".
-
-.. highlight:: python
-
-Once in the shell we can try selecting all ``<link>`` objects and see that it
-doesn't work (because the Atom XML namespace is obfuscating those nodes)::
-
-    >>> response.xpath("//link")
-    []
-
-But once we call the :meth:`Selector.remove_namespaces` method, all
-nodes can be accessed directly by their names::
-
-    >>> response.selector.remove_namespaces()
-    >>> response.xpath("//link")
-    [<Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
-     <Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
-     ...
-
-If you wonder why the namespace removal procedure isn't always called by default
-instead of having to call it manually, this is because of two reasons, which, in order
-of relevance, are:
-
-1. Removing namespaces requires to iterate and modify all nodes in the
-   document, which is a reasonably expensive operation to perform by default
-   for all documents crawled by Scrapy
-
-2. There could be some cases where using namespaces is actually required, in
-   case some element names clash between namespaces. These cases are very rare
-   though.
-
 .. _Google Base XML feed: https://support.google.com/merchants/answer/160589?hl=en&ref_topic=2473799

From dc95ecbe25a5921f902fbccc93d50f7682cefda2 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 12 Sep 2018 18:36:25 +0500
Subject: [PATCH 1818/2974] DOC use autodocs for selectors; document more
 methods and attributes; suggest get/getall

---
 docs/topics/selectors.rst  | 114 +++++++++----------------------------
 scrapy/selector/unified.py |  33 +++++++++++
 2 files changed, 60 insertions(+), 87 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 2db982b37..95d6a1a04 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -891,127 +891,67 @@ Built-in Selectors reference
 Selector objects
 ----------------
 
-.. class:: Selector(response=None, text=None, type=None)
+.. autoclass:: Selector
 
-  An instance of :class:`Selector` is a wrapper over response to select
-  certain parts of its content.
-
-  ``response`` is an :class:`~scrapy.http.HtmlResponse` or an
-  :class:`~scrapy.http.XmlResponse` object that will be used for selecting and
-  extracting data.
-
-  ``text`` is a unicode string or utf-8 encoded text for cases when a
-  ``response`` isn't available. Using ``text`` and ``response`` together is
-  undefined behavior.
-
-  ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` or ``None`` (default).
-
-    If ``type`` is ``None``, the selector automatically chooses the best type
-    based on ``response`` type (see below), or defaults to ``"html"`` in case it
-    is used together with ``text``.
-
-    If ``type`` is ``None`` and a ``response`` is passed, the selector type is
-    inferred from the response type as follows:
-
-        * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
-        * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
-        * ``"html"`` for anything else
-
-   Otherwise, if ``type`` is set, the selector type will be forced and no
-   detection will occur.
-
-  .. method:: xpath(query)
-
-      Find nodes matching the xpath ``query`` and return the result as a
-      :class:`SelectorList` instance with all elements flattened. List
-      elements implement :class:`Selector` interface too.
-
-      ``query`` is a string containing the XPATH query to apply.
+  .. automethod:: xpath
 
       .. note::
 
           For convenience, this method can be called as ``response.xpath()``
 
-  .. method:: css(query)
-
-      Apply the given CSS selector and return a :class:`SelectorList` instance.
-
-      ``query`` is a string containing the CSS selector to apply.
-
-      In the background, CSS queries are translated into XPath queries using
-      `cssselect`_ library and run ``.xpath()`` method.
+  .. automethod:: css
 
       .. note::
 
-          For convenience this method can be called as ``response.css()``
+          For convenience, this method can be called as ``response.css()``
 
-  .. method:: extract()
+  .. automethod:: get
 
-     Serialize and return the matched nodes as a list of unicode strings.
-     Percent encoded content is unquoted.
+     See also: :ref:`old-extraction-api`
 
-  .. method:: re(regex)
+  .. autoattribute:: attrib
 
-     Apply the given regex and return a list of unicode strings with the
-     matches.
+     See also: :ref:`selecting-attributes`.
 
-     ``regex`` can be either a compiled regular expression or a string which
-     will be compiled to a regular expression using ``re.compile(regex)``
+  .. automethod:: re
 
-    .. note::
+  .. automethod:: re_first
 
-        Note that ``re()`` and ``re_first()`` both decode HTML entities (except ``&lt;`` and ``&amp;``).
+  .. automethod:: register_namespace
 
-  .. method:: register_namespace(prefix, uri)
+  .. automethod:: remove_namespaces
 
-     Register the given namespace to be used in this :class:`Selector`.
-     Without registering namespaces you can't select or extract data from
-     non-standard namespaces. See examples below.
+  .. automethod:: __bool__
 
-  .. method:: remove_namespaces()
-
-     Remove all namespaces, allowing to traverse the document using
-     namespace-less xpaths. See example below.
-
-  .. method:: __nonzero__()
-
-     Returns ``True`` if there is any real content selected or ``False``
-     otherwise.  In other words, the boolean value of a :class:`Selector` is
-     given by the contents it selects.
+  .. automethod:: getall
 
+     This method is added to Selector for consistency; it is more useful
+     with SelectorList. See also: :ref:`old-extraction-api`
 
 SelectorList objects
 --------------------
 
-.. class:: SelectorList
+.. autoclass:: SelectorList
 
-   The :class:`SelectorList` class is a subclass of the builtin ``list``
-   class, which provides a few additional methods.
+   .. automethod:: xpath
 
-   .. method:: xpath(query)
+   .. automethod:: css
 
-       Call the ``.xpath()`` method for each element in this list and return
-       their results flattened as another :class:`SelectorList`.
+   .. automethod:: getall
 
-       ``query`` is the same argument as the one in :meth:`Selector.xpath`
+      See also: :ref:`old-extraction-api`
 
-   .. method:: css(query)
+   .. automethod:: get
 
-       Call the ``.css()`` method for each element in this list and return
-       their results flattened as another :class:`SelectorList`.
+      See also: :ref:`old-extraction-api`
 
-       ``query`` is the same argument as the one in :meth:`Selector.css`
+   .. automethod:: re
 
-   .. method:: extract()
+   .. automethod:: re_first
 
-       Call the ``.extract()`` method for each element in this list and return
-       their results flattened, as a list of unicode strings.
-
-   .. method:: re()
-
-       Call the ``.re()`` method for each element in this list and return
-       their results flattened, as a list of unicode strings.
+   .. autoattribute:: attrib
 
+      See also: :ref:`selecting-attributes`.
 
 .. _selector-examples:
 
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 64cb0232c..8f6cb1d79 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -27,6 +27,10 @@ def _response_from_text(text, st):
 
 
 class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
+    """
+    The :class:`SelectorList` class is a subclass of the builtin ``list``
+    class, which provides a few additional methods.
+    """
     @deprecated(use_instead='.extract()')
     def extract_unquoted(self):
         return [x.extract_unquoted() for x in self]
@@ -41,6 +45,35 @@ class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
 
 
 class Selector(_ParselSelector, object_ref):
+    """
+    An instance of :class:`Selector` is a wrapper over response to select
+    certain parts of its content.
+
+    ``response`` is an :class:`~scrapy.http.HtmlResponse` or an
+    :class:`~scrapy.http.XmlResponse` object that will be used for selecting
+    and extracting data.
+
+    ``text`` is a unicode string or utf-8 encoded text for cases when a
+    ``response`` isn't available. Using ``text`` and ``response`` together is
+    undefined behavior.
+
+    ``type`` defines the selector type, it can be ``"html"``, ``"xml"``
+    or ``None`` (default).
+
+    If ``type`` is ``None``, the selector automatically chooses the best type
+    based on ``response`` type (see below), or defaults to ``"html"`` in case it
+    is used together with ``text``.
+
+    If ``type`` is ``None`` and a ``response`` is passed, the selector type is
+    inferred from the response type as follows:
+
+    * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
+    * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
+    * ``"html"`` for anything else
+
+    Otherwise, if ``type`` is set, the selector type will be forced and no
+    detection will occur.
+    """
 
     __slots__ = ['response']
     selectorlist_cls = SelectorList

From 9db21e55028feec932136e74dad0c0f0dbe7f436 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 15 Sep 2018 02:43:37 +0500
Subject: [PATCH 1819/2974] DOC fix remove_namespaces example

See https://github.com/scrapy/parsel/pull/119
---
 docs/topics/selectors.rst | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 95d6a1a04..20c3fff3c 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -643,27 +643,30 @@ namespaces altogether and just work with element names, to write more
 simple/convenient XPaths. You can use the
 :meth:`Selector.remove_namespaces` method for that.
 
-Let's show an example that illustrates this with GitHub blog atom feed.
+Let's show an example that illustrates this with the Python Insider blog atom feed.
 
 .. highlight:: sh
 
 First, we open the shell with the url we want to scrape::
 
-    $ scrapy shell https://github.com/blog.atom
-
-.. highlight:: xml
+    $ scrapy shell https://feeds.feedburner.com/PythonInsider
 
 This is how the file starts::
 
     <?xml version="1.0" encoding="UTF-8"?>
-    <feed xml:lang="en-US"
-          xmlns="http://www.w3.org/2005/Atom"
-          xmlns:media="http://search.yahoo.com/mrss/">
-      <id>tag:github.com,2008:/blog</id>
+    <?xml-stylesheet ...
+    <feed xmlns="http://www.w3.org/2005/Atom"
+          xmlns:openSearch="http://a9.com/-/spec/opensearchrss/1.0/"
+          xmlns:blogger="http://schemas.google.com/blogger/2008"
+          xmlns:georss="http://www.georss.org/georss"
+          xmlns:gd="http://schemas.google.com/g/2005"
+          xmlns:thr="http://purl.org/syndication/thread/1.0"
+          xmlns:feedburner="http://rssnamespace.org/feedburner/ext/1.0">
       ...
 
-You can see two namespace declarations: a default "http://www.w3.org/2005/Atom"
-and another one using the "media:" prefix for "http://search.yahoo.com/mrss/".
+You can see several namespace declarations including a default
+"http://www.w3.org/2005/Atom" and another one using the "gd:" prefix for
+"http://schemas.google.com/g/2005".
 
 .. highlight:: python
 
@@ -678,8 +681,8 @@ nodes can be accessed directly by their names::
 
     >>> response.selector.remove_namespaces()
     >>> response.xpath("//link")
-    [<Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
-     <Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>,
+    [<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>,
+     <Selector xpath='//link' data='<link rel="next" type="application/atom+'>,
      ...
 
 If you wonder why the namespace removal procedure isn't always called by default

From 2c3b2158c99953823500f275ccc5206c11c9a811 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 18 Sep 2018 05:02:17 +0500
Subject: [PATCH 1820/2974] DOC address @stummjr's review comments

* fixed several small issues
* re-written "Creating Selectors" section
* fixed remaining .extract usage in tests
---
 docs/intro/tutorial.rst       |  6 ++--
 docs/topics/selectors.rst     | 52 +++++++++++++++++++----------------
 tests/test_utils_iterators.py |  4 +--
 3 files changed, 33 insertions(+), 29 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 46e84b21c..ad17ef096 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -285,9 +285,9 @@ There's a lesson here: for most scraping code, you want it to be resilient to
 errors due to things not being found on a page, so that even if some parts fail
 to be scraped, you can at least get **some** data.
 
-Besides the :meth:`~scrapy.selector.Selector.extract` and
+Besides the :meth:`~scrapy.selector.SelectorList.getall` and
 :meth:`~scrapy.selector.SelectorList.get` methods, you can also use
-the :meth:`~scrapy.selector.Selector.re` method to extract using `regular
+the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
 expressions`_::
 
     >>> response.css('title::text').re(r'Quotes.*')
@@ -649,7 +649,7 @@ this time for scraping author information::
 
         def parse_author(self, response):
             def extract_with_css(query):
-                return response.css(query).get().strip()
+                return response.css(query).get(default='').strip()
 
             yield {
                 'name': extract_with_css('h3.author-title::text'),
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 20c3fff3c..68913c697 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -50,28 +50,8 @@ Constructing selectors
 
 .. highlight:: python
 
-Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
-constructed by passing **text** or :class:`~scrapy.http.TextResponse`
-object. It automatically chooses the best parsing rules (XML vs HTML) based on
-input type::
-
-    >>> from scrapy.selector import Selector
-    >>> from scrapy.http import HtmlResponse
-
-Constructing from text::
-
-    >>> body = '<html><body><span>good</span></body></html>'
-    >>> Selector(text=body).xpath('//span/text()').get()
-    'good'
-
-Constructing from response::
-
-    >>> response = HtmlResponse(url='http://example.com', body=body)
-    >>> Selector(response=response).xpath('//span/text()').get()
-    'good'
-
-For convenience, response objects expose a selector on `.selector` attribute.
-By using it you can ensure the response body is parsed only once::
+Response objects expose a :class:`~scrapy.selector.Selector` instance
+on ``.selector`` attribute::
 
     >>> response.selector.xpath('//span/text()').get()
     'good'
@@ -84,10 +64,34 @@ more shortcuts: ``response.xpath()`` and ``response.css()``::
     >>> response.css('span::text').get()
     'good'
 
+Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
+constructed by passing either :class:`~scrapy.http.TextResponse` object or
+markup as an unicode string (in ``text`` argument).
 Usually there is no need to construct Scrapy selectors manually:
 ``response`` object is available in Spider callbacks, so in most cases
 it is more convenient to use ``response.css()`` and ``response.xpath()``
-shortcuts.
+shortcuts. By using ``response.selector`` or one of these shortcuts
+you can also ensure the response body is parsed only once.
+
+But if required, it is possible to use ``Selector`` directly.
+Constructing from text::
+
+    >>> from scrapy.selector import Selector
+    >>> body = '<html><body><span>good</span></body></html>'
+    >>> Selector(text=body).xpath('//span/text()').get()
+    'good'
+
+Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of
+:class:`~scrapy.http.TextResponse` subclasses::
+
+    >>> from scrapy.selector import Selector
+    >>> from scrapy.http import HtmlResponse
+    >>> response = HtmlResponse(url='http://example.com', body=body)
+    >>> Selector(response=response).xpath('//span/text()').get()
+    'good'
+
+``Selector`` automatically chooses the best parsing rules
+(XML vs HTML) based on input type.
 
 Using selectors
 ---------------
@@ -139,7 +143,7 @@ is returned. ``.getall()`` returns a list with all results.
 Notice that CSS selectors can select text or attribute nodes using CSS3
 pseudo-elements::
 
-    >>> selector.css('title::text').get()
+    >>> response.css('title::text').get()
     'Example website'
 
 As you can see, ``.xpath()`` and ``.css()`` methods return a
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 00eb78068..2d845697e 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -32,8 +32,8 @@ class XmliterTestCase(unittest.TestCase):
         for x in self.xmliter(response, 'product'):
             attrs.append((
                 x.attrib['id'],
-                x.xpath("name/text()").extract(),
-                x.xpath("./type/text()").extract()))
+                x.xpath("name/text()").getall(),
+                x.xpath("./type/text()").getall()))
 
         self.assertEqual(attrs,
                          [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])])

From ffbd33edac0367e9f975b9863e0c31e1c2b72ebc Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 18 Sep 2018 05:03:35 +0500
Subject: [PATCH 1821/2974] DOC mention gotcha with `foo::text` selector and
 empty `foo` elements

also, move "Selecting attributes" reference closer to `a::atr(href)` example
---
 docs/topics/selectors.rst | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 68913c697..9dced7473 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -279,6 +279,19 @@ Examples:
      'Name: My image 5 ',
      '\n  ']
 
+* ``foo::text`` returns no results if ``foo`` element exists, but contains
+  no text (i.e. text is empty)::
+
+    >>> response.css('img::text').getall()
+    []
+
+  This means ``.css('foo::text').get()`` could return None even if an element
+  exists. Use ``default=''`` if you always want a string::
+
+    >>> response.css('img::text').get()
+    >>> response.css('img::text').get(default='')
+    ''
+
 * ``a::attr(href)`` selects the *href* attribute value of descendant links::
 
     >>> response.css('a::attr(href)').getall()
@@ -288,15 +301,14 @@ Examples:
      'image4.html',
      'image5.html']
 
+.. note::
+    See also: :ref:`selecting-attributes`.
+
 .. note::
     You cannot chain these pseudo-elements. But in practice it would not
     make much sense: text nodes do not have attributes, and attribute values
     are string values already and do not have children nodes.
 
-.. note::
-    See also: :ref:`selecting-attributes`.
-
-
 .. _CSS Selectors: https://www.w3.org/TR/css3-selectors/#selectors
 
 .. _topics-selectors-nesting-selectors:

From 37cfb49805c86168af7a831fc33ec4aeb83e53da Mon Sep 17 00:00:00 2001
From: Henrique Coura <coura.henrique@gmail.com>
Date: Mon, 24 Sep 2018 16:42:49 -0300
Subject: [PATCH 1822/2974] Randomly generate telnet credentials by default

---
 scrapy/extensions/telnet.py         | 25 +++++++++---
 scrapy/settings/default_settings.py |  2 -
 tests/test_extension_telnet.py      | 59 +++++++++++++++++++++++++++++
 3 files changed, 79 insertions(+), 7 deletions(-)
 create mode 100644 tests/test_extension_telnet.py

diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 93342f225..3d0afeffb 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -7,6 +7,8 @@ See documentation in docs/topics/telnetconsole.rst
 import pprint
 import logging
 import traceback
+import binascii
+import os
 
 from twisted.internet import protocol
 try:
@@ -50,8 +52,21 @@ class TelnetConsole(protocol.ServerFactory):
         self.noisy = False
         self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]
         self.host = crawler.settings['TELNETCONSOLE_HOST']
-        self.username = crawler.settings['TELNETCONSOLE_USERNAME']
-        self.password = crawler.settings['TELNETCONSOLE_PASSWORD']
+
+        username = crawler.settings.get('TELNETCONSOLE_USERNAME', None)
+        if username:
+            self.username = username.encode('utf8')
+        else:
+            self.username = binascii.hexlify(os.urandom(8))
+
+        password = crawler.settings.get('TELNETCONSOLE_PASSWORD', None)
+        if password:
+            self.password = password.encode('utf8')
+        else:
+            self.password = binascii.hexlify(os.urandom(8))
+
+        logger.info('Telnet Username: %s' % self.username)
+        logger.info('Telnet Password: %s' % self.password)
         self.crawler.signals.connect(self.start_listening, signals.engine_started)
         self.crawler.signals.connect(self.stop_listening, signals.engine_stopped)
 
@@ -74,8 +89,8 @@ class TelnetConsole(protocol.ServerFactory):
             """An implementation of IPortal"""
             @defers
             def login(self_, credentials, mind, *interfaces):
-                if not (credentials.username == self.username
-                        and credentials.checkPassword(self.password)):
+                if not (credentials.username == self.username and
+                        credentials.checkPassword(self.password)):
                     raise ValueError("Invalid credentials")
 
                 protocol = telnet.TelnetBootstrapProtocol(
@@ -104,7 +119,7 @@ class TelnetConsole(protocol.ServerFactory):
             'p': pprint.pprint,
             'prefs': print_live_refs,
             'hpy': hpy,
-            'help': "This is Scrapy telnet console. For more info see: " \
+            'help': "This is Scrapy telnet console. For more info see: "
                 "https://doc.scrapy.org/en/latest/topics/telnetconsole.html",
         }
         self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 2b7bc173c..ca004aedd 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -277,8 +277,6 @@ USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__versi
 TELNETCONSOLE_ENABLED = 1
 TELNETCONSOLE_PORT = [6023, 6073]
 TELNETCONSOLE_HOST = '127.0.0.1'
-TELNETCONSOLE_USERNAME = 'scrapy'
-TELNETCONSOLE_PASSWORD = 'scrapy'
 
 SPIDER_CONTRACTS = {}
 SPIDER_CONTRACTS_BASE = {
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
new file mode 100644
index 000000000..ffea1c463
--- /dev/null
+++ b/tests/test_extension_telnet.py
@@ -0,0 +1,59 @@
+try:
+    import unittest.mock as mock
+except ImportError:
+    import mock
+
+from twisted.trial import unittest
+from twisted.conch.telnet import ITelnetProtocol
+from twisted.cred import credentials
+from twisted.internet import defer
+
+from scrapy.extensions.telnet import TelnetConsole, logger
+from scrapy.utils.test import get_crawler
+
+
+class TelnetExtensionTest(unittest.TestCase):    
+    def _get_console_and_portal(self, settings=None):
+        crawler = get_crawler(settings_dict=settings)
+        console = TelnetConsole(crawler)
+        username = console.username
+        password = console.password
+
+        def _get_telnet_vars():
+            # This function has some side effects we don't need for this test
+            return {}
+        console._get_telnet_vars = _get_telnet_vars
+
+        console.start_listening()
+        protocol = console.protocol()
+        portal = protocol.protocolArgs[0]
+
+        return console, portal
+
+    @defer.inlineCallbacks
+    def test_bad_credentials(self):
+        console, portal = self._get_console_and_portal()
+        creds = credentials.UsernamePassword(b'username', b'password')
+        d = portal.login(creds, None, ITelnetProtocol)
+        yield self.assertFailure(d, ValueError)
+        console.stop_listening()
+
+    @defer.inlineCallbacks
+    def test_good_credentials(self):
+        console, portal = self._get_console_and_portal()
+        creds = credentials.UsernamePassword(console.username, console.password)
+        d = portal.login(creds, None, ITelnetProtocol)
+        yield d
+        console.stop_listening()
+
+    @defer.inlineCallbacks
+    def test_custom_credentials(self):
+        settings = {
+            'TELNETCONSOLE_USERNAME': 'user',
+            'TELNETCONSOLE_PASSWORD': 'pass',
+        }
+        console, portal = self._get_console_and_portal(settings=settings)
+        creds = credentials.UsernamePassword(b'user', b'pass')
+        d = portal.login(creds, None, ITelnetProtocol)
+        yield d
+        console.stop_listening()

From e57a629efc0846ed396247baf22d7846689b82e4 Mon Sep 17 00:00:00 2001
From: Henrique Coura <coura.henrique@gmail.com>
Date: Wed, 26 Sep 2018 11:54:57 -0300
Subject: [PATCH 1823/2974] Generate only password, encode username/password
 only on login

---
 scrapy/extensions/telnet.py         | 22 ++++++++--------------
 scrapy/settings/default_settings.py |  2 ++
 tests/test_extension_telnet.py      |  5 ++++-
 3 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 3d0afeffb..6df435cef 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -53,20 +53,14 @@ class TelnetConsole(protocol.ServerFactory):
         self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]
         self.host = crawler.settings['TELNETCONSOLE_HOST']
 
-        username = crawler.settings.get('TELNETCONSOLE_USERNAME', None)
-        if username:
-            self.username = username.encode('utf8')
-        else:
-            self.username = binascii.hexlify(os.urandom(8))
+        self.username = crawler.settings['TELNETCONSOLE_USERNAME']
+        self.password = crawler.settings['TELNETCONSOLE_PASSWORD']
 
-        password = crawler.settings.get('TELNETCONSOLE_PASSWORD', None)
-        if password:
-            self.password = password.encode('utf8')
-        else:
-            self.password = binascii.hexlify(os.urandom(8))
+        if not self.password:
+            self.password = binascii.hexlify(os.urandom(8)).decode('utf8')
+            logger.info('Telnet Username: %s', self.username)
+            logger.info('Telnet Password: %s', self.password)
 
-        logger.info('Telnet Username: %s' % self.username)
-        logger.info('Telnet Password: %s' % self.password)
         self.crawler.signals.connect(self.start_listening, signals.engine_started)
         self.crawler.signals.connect(self.stop_listening, signals.engine_stopped)
 
@@ -89,8 +83,8 @@ class TelnetConsole(protocol.ServerFactory):
             """An implementation of IPortal"""
             @defers
             def login(self_, credentials, mind, *interfaces):
-                if not (credentials.username == self.username and
-                        credentials.checkPassword(self.password)):
+                if not (credentials.username == self.username.encode('utf8') and
+                        credentials.checkPassword(self.password.encode('utf8'))):
                     raise ValueError("Invalid credentials")
 
                 protocol = telnet.TelnetBootstrapProtocol(
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index ca004aedd..3734a0a58 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -277,6 +277,8 @@ USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__versi
 TELNETCONSOLE_ENABLED = 1
 TELNETCONSOLE_PORT = [6023, 6073]
 TELNETCONSOLE_HOST = '127.0.0.1'
+TELNETCONSOLE_USERNAME = 'scrapy'
+TELNETCONSOLE_PASSWORD = None
 
 SPIDER_CONTRACTS = {}
 SPIDER_CONTRACTS_BASE = {
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index ffea1c463..487c7c29f 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -41,7 +41,10 @@ class TelnetExtensionTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_good_credentials(self):
         console, portal = self._get_console_and_portal()
-        creds = credentials.UsernamePassword(console.username, console.password)
+        creds = credentials.UsernamePassword(
+            console.username.encode('utf8'),
+            console.password.encode('utf8')
+        )
         d = portal.login(creds, None, ITelnetProtocol)
         yield d
         console.stop_listening()

From 5f9931d2ada7a2a05df77b1c061eeb482fcda347 Mon Sep 17 00:00:00 2001
From: Henrique Coura <coura.henrique@gmail.com>
Date: Wed, 26 Sep 2018 13:07:04 -0300
Subject: [PATCH 1824/2974] do not log username

---
 scrapy/extensions/telnet.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 6df435cef..a3d55f3c6 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -58,7 +58,6 @@ class TelnetConsole(protocol.ServerFactory):
 
         if not self.password:
             self.password = binascii.hexlify(os.urandom(8)).decode('utf8')
-            logger.info('Telnet Username: %s', self.username)
             logger.info('Telnet Password: %s', self.password)
 
         self.crawler.signals.connect(self.start_listening, signals.engine_started)

From 441e1e750fe7ad970adafc4c1f42834f7db86d1d Mon Sep 17 00:00:00 2001
From: Henrique Coura <coura.henrique@gmail.com>
Date: Wed, 26 Sep 2018 13:28:34 -0300
Subject: [PATCH 1825/2974] Style changes

---
 scrapy/extensions/telnet.py    | 3 +--
 tests/test_extension_telnet.py | 8 +++-----
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index a3d55f3c6..dcf73eb88 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -52,7 +52,6 @@ class TelnetConsole(protocol.ServerFactory):
         self.noisy = False
         self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]
         self.host = crawler.settings['TELNETCONSOLE_HOST']
-
         self.username = crawler.settings['TELNETCONSOLE_USERNAME']
         self.password = crawler.settings['TELNETCONSOLE_PASSWORD']
 
@@ -113,7 +112,7 @@ class TelnetConsole(protocol.ServerFactory):
             'prefs': print_live_refs,
             'hpy': hpy,
             'help': "This is Scrapy telnet console. For more info see: "
-                "https://doc.scrapy.org/en/latest/topics/telnetconsole.html",
+                    "https://doc.scrapy.org/en/latest/topics/telnetconsole.html",
         }
         self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars)
         return telnet_vars
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index 487c7c29f..4f389e5cb 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -12,17 +12,15 @@ from scrapy.extensions.telnet import TelnetConsole, logger
 from scrapy.utils.test import get_crawler
 
 
-class TelnetExtensionTest(unittest.TestCase):    
+class TelnetExtensionTest(unittest.TestCase):
     def _get_console_and_portal(self, settings=None):
         crawler = get_crawler(settings_dict=settings)
         console = TelnetConsole(crawler)
         username = console.username
         password = console.password
 
-        def _get_telnet_vars():
-            # This function has some side effects we don't need for this test
-            return {}
-        console._get_telnet_vars = _get_telnet_vars
+        # This function has some side effects we don't need for this test
+        console._get_telnet_vars = lambda: {}
 
         console.start_listening()
         protocol = console.protocol()

From edaf74bfaeef7d995676ad8f6bfb8056a8e6966d Mon Sep 17 00:00:00 2001
From: jfflisikowski <jfflisikowski@gmail.com>
Date: Tue, 2 Oct 2018 19:48:48 +0200
Subject: [PATCH 1826/2974] Correct the unclear comments by adding <# <
 processing code not shown >

---
 docs/topics/debug.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst
index d1991c02f..f93aa2c72 100644
--- a/docs/topics/debug.rst
+++ b/docs/topics/debug.rst
@@ -18,11 +18,13 @@ Consider the following scrapy spider below::
             )
 
         def parse(self, response):
-            # collect `item_urls`
+            # <processing code not shown>
+            # collect `item_urls` 
             for item_url in item_urls:
                 yield scrapy.Request(item_url, self.parse_item)
 
         def parse_item(self, response):
+            # <processing code not shown>
             item = MyItem()
             # populate `item` fields
             # and extract item_details_url

From 58f5565357ed532970772cd55c2d17d1e00198a9 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 11 Oct 2018 11:23:12 -0300
Subject: [PATCH 1827/2974] Move MutableChain to scrapy.utils.python

---
 scrapy/core/spidermw.py | 21 +--------------------
 scrapy/utils/python.py  | 21 +++++++++++++++++++++
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 4268c91d6..d776430e5 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -11,32 +11,13 @@ from scrapy.exceptions import _InvalidOutput
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.conf import build_component_list
+from scrapy.utils.python import MutableChain
 
 
 def _isiterable(possible_iterator):
     return hasattr(possible_iterator, '__iter__')
 
 
-class MutableChain:
-    """
-    Thin wrapper around itertools.chain, allowing to add iterables "in-place"
-    """
-    def __init__(self, *args):
-        self.data = chain(*args)
-
-    def extend(self, *iterables):
-        self.data = chain(self.data, *iterables)
-
-    def __iter__(self):
-        return self.data.__iter__()
-
-    def __next__(self):  # py3
-        return self.data.__next__()
-
-    def next(self):  # py2
-        return self.data.next()
-
-
 class SpiderMiddlewareManager(MiddlewareManager):
 
     component_name = 'spider middleware'
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 732ca13a0..7971b4dde 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -9,6 +9,7 @@ import weakref
 import errno
 import six
 from functools import partial, wraps
+from itertools import chain
 import sys
 
 from scrapy.utils.decorators import deprecated
@@ -387,3 +388,23 @@ if hasattr(sys, "pypy_version_info"):
 else:
     def garbage_collect():
         gc.collect()
+
+
+class MutableChain(object):
+    """
+    Thin wrapper around itertools.chain, allowing to add iterables "in-place"
+    """
+    def __init__(self, *args):
+        self.data = chain(*args)
+
+    def extend(self, *iterables):
+        self.data = chain(self.data, *iterables)
+
+    def __iter__(self):
+        return self.data.__iter__()
+
+    def __next__(self):  # py3
+        return self.data.__next__()
+
+    def next(self):  # py2
+        return self.data.next()

From a05eaeed73a469493e78b5a1c5f0b4de2adf41c2 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 11 Oct 2018 11:31:51 -0300
Subject: [PATCH 1828/2974] Simplify MutableChain

---
 scrapy/utils/python.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 7971b4dde..1a6bab990 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -403,8 +403,7 @@ class MutableChain(object):
     def __iter__(self):
         return self.data.__iter__()
 
-    def __next__(self):  # py3
-        return self.data.__next__()
+    def __next__(self):
+        return next(self.data)
 
-    def next(self):  # py2
-        return self.data.next()
+    next = __next__

From 15f0a890ee9f059111333fdeb6c6c3b5a8dadc07 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 11 Oct 2018 11:34:59 -0300
Subject: [PATCH 1829/2974] Assign processing methods to a variable before
 iterating

---
 scrapy/core/spidermw.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index d776430e5..3fae770a9 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -58,7 +58,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
             # don't handle _InvalidOutput exception
             if isinstance(exception, _InvalidOutput):
                 return _failure
-            for method_index, method in enumerate(self.methods['process_spider_exception'][start_index:], start=start_index):
+            method_list = self.methods['process_spider_exception'][start_index:]
+            for method_index, method in enumerate(method_list, start=start_index):
                 if method is None:
                     continue
                 result = method(response=response, exception=exception, spider=spider)
@@ -88,7 +89,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
                         raise
                     recovered.extend(exception_result)
 
-            for method_index, method in enumerate(self.methods['process_spider_output'][start_index:], start=start_index):
+            method_list = self.methods['process_spider_output'][start_index:]
+            for method_index, method in enumerate(method_list, start=start_index):
                 if method is None:
                     continue
                 # the following might fail directly if the output value is not a generator

From e0360e5223b618934ee006b4c9ed63012e7e621f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 11 Oct 2018 11:55:13 -0300
Subject: [PATCH 1830/2974] Add tests for MutableChain

---
 tests/test_utils_python.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index f6133657b..3e1148354 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -9,11 +9,23 @@ import six
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
     WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode,
-    without_none_values)
+    without_none_values, MutableChain)
 
 __doctests__ = ['scrapy.utils.python']
 
 
+class MutableChainTest(unittest.TestCase):
+    def test_mutablechain(self):
+        m = MutableChain(range(2), [2, 3], (4, 5))
+        m.extend(range(6, 7))
+        m.extend([7, 8])
+        m.extend([9, 10], (11, 12))
+        self.assertEqual(next(m), 0)
+        self.assertEqual(m.next(), 1)
+        self.assertEqual(m.__next__(), 2)
+        self.assertEqual(list(m), list(range(3, 13)))
+
+
 class ToUnicodeTest(unittest.TestCase):
     def test_converting_an_utf8_encoded_string_to_unicode(self):
         self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e')

From 92b7955d75eba3ddad1e4815cb80cf60c7a9a7a9 Mon Sep 17 00:00:00 2001
From: Henrique Coura <coura.henrique@gmail.com>
Date: Tue, 16 Oct 2018 14:50:00 -0300
Subject: [PATCH 1832/2974] Add Telnet console authentication docs

---
 docs/topics/telnetconsole.rst | 36 ++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index ce79c9f35..49c372598 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -26,8 +26,21 @@ The telnet console listens in the TCP port defined in the
 the console you need to type::
 
     telnet localhost 6023
+    Trying localhost...
+    Connected to localhost.
+    Escape character is '^]'.
+    Username:
+    Password:  
     >>>
-    
+
+By default Username is ``scrapy`` and Password is autogenerated. The 
+autogenerated Password can be seen on scrapy logs like the example bellow::
+
+    2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
+
+Default Username and Password can be overriden by the settings 
+:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`
+
 You need the telnet program which comes installed by default in Windows, and
 most Linux distros.
 
@@ -160,3 +173,24 @@ Default: ``'127.0.0.1'``
 
 The interface the telnet console should listen on
 
+
+.. setting:: TELNETCONSOLE_USERNAME
+
+TELNETCONSOLE_USERNAME
+------------------
+
+Default: ``'scrapy'``
+
+The username used for the telnet console
+
+
+.. setting:: TELNETCONSOLE_PASSWORD
+
+TELNETCONSOLE_PASSWORD
+------------------
+
+Default: ``None``
+
+The password used for the telnet console, default behaviour is to have it
+autogenerated
+

From 44f8e28b3c8608f65dbc7836b36bc231e38393b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Tue, 16 Oct 2018 19:53:20 -0300
Subject: [PATCH 1833/2974] Fix headings' underlines

---
 docs/topics/telnetconsole.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index 49c372598..4db9cafb2 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -177,7 +177,7 @@ The interface the telnet console should listen on
 .. setting:: TELNETCONSOLE_USERNAME
 
 TELNETCONSOLE_USERNAME
-------------------
+----------------------
 
 Default: ``'scrapy'``
 
@@ -187,7 +187,7 @@ The username used for the telnet console
 .. setting:: TELNETCONSOLE_PASSWORD
 
 TELNETCONSOLE_PASSWORD
-------------------
+----------------------
 
 Default: ``None``
 

From c9b5bd6ad7728274b0f82fc3211c5ade5cd0d389 Mon Sep 17 00:00:00 2001
From: Immanuella Lim <immanuella@live.com>
Date: Thu, 18 Oct 2018 02:22:32 +0800
Subject: [PATCH 1834/2974] Remove ad link Dive Into Python3 from tutorial docs

---
 docs/intro/tutorial.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index ad17ef096..143e018ac 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -23,7 +23,7 @@ start by getting an idea of what the language is like, to get the most out of
 Scrapy.
 
 If you're already familiar with other languages, and want to learn Python
-quickly, we recommend reading through `Dive Into Python 3`_.  Alternatively,
+quickly, we recommend reading through `Crash into Python`_.  Alternatively,
 you can follow the `Python Tutorial`_.
 
 If you're new to programming and want to start with Python, the following books
@@ -40,7 +40,7 @@ as well as the `suggested resources in the learnpython-subreddit`_.
 
 .. _Python: https://www.python.org/
 .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
-.. _Dive Into Python 3: http://www.diveintopython3.net
+.. _Crash into Python: https://stephensugden.com/crash_into_python/
 .. _Python Tutorial: https://docs.python.org/3/tutorial
 .. _Automate the Boring Stuff With Python: https://automatetheboringstuff.com/
 .. _How To Think Like a Computer Scientist: http://openbookproject.net/thinkcs/python/english3e/

From f97e3e90f25c5077b47d9ec11a4cf84ea777227e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 29 Oct 2018 12:40:20 -0300
Subject: [PATCH 1835/2974] Use collections.deque instead of list to store
 methods

---
 scrapy/core/downloader/middleware.py | 4 ++--
 scrapy/core/spidermw.py              | 6 +++---
 scrapy/middleware.py                 | 6 +++---
 tests/test_middleware.py             | 6 +++---
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index c3b23e284..f5e2fca63 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -26,9 +26,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         if hasattr(mw, 'process_request'):
             self.methods['process_request'].append(mw.process_request)
         if hasattr(mw, 'process_response'):
-            self.methods['process_response'].insert(0, mw.process_response)
+            self.methods['process_response'].appendleft(mw.process_response)
         if hasattr(mw, 'process_exception'):
-            self.methods['process_exception'].insert(0, mw.process_exception)
+            self.methods['process_exception'].appendleft(mw.process_exception)
 
     def download(self, download_func, request, spider):
         @defer.inlineCallbacks
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index a206e4b0c..16b8435ab 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -25,11 +25,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
         if hasattr(mw, 'process_spider_input'):
             self.methods['process_spider_input'].append(mw.process_spider_input)
         if hasattr(mw, 'process_spider_output'):
-            self.methods['process_spider_output'].insert(0, mw.process_spider_output)
+            self.methods['process_spider_output'].appendleft(mw.process_spider_output)
         if hasattr(mw, 'process_spider_exception'):
-            self.methods['process_spider_exception'].insert(0, mw.process_spider_exception)
+            self.methods['process_spider_exception'].appendleft(mw.process_spider_exception)
         if hasattr(mw, 'process_start_requests'):
-            self.methods['process_start_requests'].insert(0, mw.process_start_requests)
+            self.methods['process_start_requests'].appendleft(mw.process_start_requests)
 
     def scrape_response(self, scrape_func, response, request, spider):
         fname = lambda f:'%s.%s' % (
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index f2240984c..1cfd8a782 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,4 +1,4 @@
-from collections import defaultdict
+from collections import defaultdict, deque
 import logging
 import pprint
 
@@ -16,7 +16,7 @@ class MiddlewareManager(object):
 
     def __init__(self, *middlewares):
         self.middlewares = middlewares
-        self.methods = defaultdict(list)
+        self.methods = defaultdict(deque)
         for mw in middlewares:
             self._add_middleware(mw)
 
@@ -56,7 +56,7 @@ class MiddlewareManager(object):
         if hasattr(mw, 'open_spider'):
             self.methods['open_spider'].append(mw.open_spider)
         if hasattr(mw, 'close_spider'):
-            self.methods['close_spider'].insert(0, mw.close_spider)
+            self.methods['close_spider'].appendleft(mw.close_spider)
 
     def _process_parallel(self, methodname, obj, *args):
         return process_parallel(self.methods[methodname], obj, *args)
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index b6d885330..aea0be825 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -60,9 +60,9 @@ class MiddlewareManagerTest(unittest.TestCase):
     def test_init(self):
         m1, m2, m3 = M1(), M2(), M3()
         mwman = TestMiddlewareManager(m1, m2, m3)
-        self.assertEqual(mwman.methods['open_spider'], [m1.open_spider, m2.open_spider])
-        self.assertEqual(mwman.methods['close_spider'], [m2.close_spider, m1.close_spider])
-        self.assertEqual(mwman.methods['process'], [m1.process, m3.process])
+        self.assertEqual(list(mwman.methods['open_spider']), [m1.open_spider, m2.open_spider])
+        self.assertEqual(list(mwman.methods['close_spider']), [m2.close_spider, m1.close_spider])
+        self.assertEqual(list(mwman.methods['process']), [m1.process, m3.process])
 
     def test_methods(self):
         mwman = TestMiddlewareManager(M1(), M2(), M3())

From 6c98010f110c432a2311c1aef1d463dc5a6ccba4 Mon Sep 17 00:00:00 2001
From: Immanuella Lim <immanuella@live.com>
Date: Sun, 4 Nov 2018 16:04:45 +0800
Subject: [PATCH 1836/2974] Remove 'Dive into Python3' reference

---
 docs/intro/tutorial.rst | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 143e018ac..41e61542a 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -22,9 +22,7 @@ Scrapy is written in Python_. If you're new to the language you might want to
 start by getting an idea of what the language is like, to get the most out of
 Scrapy.
 
-If you're already familiar with other languages, and want to learn Python
-quickly, we recommend reading through `Crash into Python`_.  Alternatively,
-you can follow the `Python Tutorial`_.
+If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource.
 
 If you're new to programming and want to start with Python, the following books
 may be useful to you: 
@@ -40,7 +38,6 @@ as well as the `suggested resources in the learnpython-subreddit`_.
 
 .. _Python: https://www.python.org/
 .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
-.. _Crash into Python: https://stephensugden.com/crash_into_python/
 .. _Python Tutorial: https://docs.python.org/3/tutorial
 .. _Automate the Boring Stuff With Python: https://automatetheboringstuff.com/
 .. _How To Think Like a Computer Scientist: http://openbookproject.net/thinkcs/python/english3e/

From 491929c212999aa816e561aeed19a902664d01e5 Mon Sep 17 00:00:00 2001
From: Todd <toddrme2178@gmail.com>
Date: Fri, 16 Nov 2018 13:38:19 -0500
Subject: [PATCH 1837/2974] Include additional files in sdists

In particular this includes files needed for running the tests, as well as the changelog.
---
 MANIFEST.in | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index 94de4f3bf..ae7db51fa 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -3,13 +3,24 @@ include AUTHORS
 include INSTALL
 include LICENSE
 include MANIFEST.in
+include NEWS
+
 include scrapy/VERSION
 include scrapy/mime.types
+
+include codecov.yml
+include conftest.py
+include pytest.ini
+include requirements-*.txt
+include tox.ini
+
 recursive-include scrapy/templates *
 recursive-include scrapy license.txt
 recursive-include docs *
 prune docs/build
+
 recursive-include extras *
 recursive-include bin *
 recursive-include tests *
+
 global-exclude __pycache__ *.py[cod]

From 127bf499f1d6b4a924d87e39ff89b528586c78c7 Mon Sep 17 00:00:00 2001
From: Frederik Elwert <frederik.elwert@web.de>
Date: Fri, 16 Nov 2018 22:15:03 +0100
Subject: [PATCH 1838/2974] Add documentation to `scrapy shell` command.

The special syntax required for local files (`./file.html`) is not documented as part of the `scrapy shell --help` output. This patch adds that.
---
 scrapy/commands/shell.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 40a58d94a..e05084272 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -28,7 +28,8 @@ class Command(ScrapyCommand):
         return "Interactive scraping console"
 
     def long_desc(self):
-        return "Interactive console for scraping the given url"
+        return ("Interactive console for scraping the given url or file. "
+                "Use ./file.html syntax or full path for local file.")
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)

From a25cf5c82f99f7ae11346a2e565d6255835c3814 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Tue, 20 Nov 2018 16:13:09 +0000
Subject: [PATCH 1839/2974] function to get unique file queues for any type of
 base queue

---
 scrapy/core/queues.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 scrapy/core/queues.py

diff --git a/scrapy/core/queues.py b/scrapy/core/queues.py
new file mode 100644
index 000000000..96d582fc7
--- /dev/null
+++ b/scrapy/core/queues.py
@@ -0,0 +1,15 @@
+import uuid
+import os.path
+
+
+def unique_files_queue(queue_class):
+
+    class UniqueFilesQueue(queue_class):
+        def __init__(self, path):
+            path = path + "-" + uuid.uuid4().hex
+            while os.path.exists(path):
+                path = path + "-" + uuid.uuid4().hex
+
+            super().__init__(path)
+
+    return UniqueFilesQueue

From 1ce6662a9d7115348788972afce62a5c45199021 Mon Sep 17 00:00:00 2001
From: kasun Herath <kasunh01@gmail.com>
Date: Sat, 24 Nov 2018 20:02:00 +0530
Subject: [PATCH 1840/2974] Implement Request subclass for json requests

---
 scrapy/http/__init__.py             |  1 +
 scrapy/http/request/json_request.py | 28 +++++++++++++++
 tests/test_http_request.py          | 55 ++++++++++++++++++++++++++++-
 3 files changed, 83 insertions(+), 1 deletion(-)
 create mode 100644 scrapy/http/request/json_request.py

diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py
index f04a9d3e5..4b2f7b33f 100644
--- a/scrapy/http/__init__.py
+++ b/scrapy/http/__init__.py
@@ -10,6 +10,7 @@ from scrapy.http.headers import Headers
 from scrapy.http.request import Request
 from scrapy.http.request.form import FormRequest
 from scrapy.http.request.rpc import XmlRpcRequest
+from scrapy.http.request.json_request import JSONRequest
 
 from scrapy.http.response import Response
 from scrapy.http.response.html import HtmlResponse
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
new file mode 100644
index 000000000..0fdd2ddf1
--- /dev/null
+++ b/scrapy/http/request/json_request.py
@@ -0,0 +1,28 @@
+"""
+This module implements the JSONRequest class which is a more convenient class
+(than Request) to generate JSON Requests.
+
+See documentation in docs/topics/request-response.rst
+"""
+
+import json
+
+from scrapy.http.request import Request
+
+
+class JSONRequest(Request):
+    def __init__(self, *args, **kwargs):
+        if 'method' not in kwargs:
+            kwargs['method'] = 'POST'
+
+        data = kwargs.pop('data', {})
+        kwargs['body'] = json.dumps(data)
+        super(JSONRequest, self).__init__(*args, **kwargs)
+        self.headers.setdefault(b'Content-Type', b'application/json')
+
+    def replace(self, *args, **kwargs):
+        """ Create a new Request with the same attributes except for those
+            given new values. """
+
+        kwargs.pop('body', None)
+        return super(JSONRequest, self).replace(*args, **kwargs)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 58326a384..3f2e4f521 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -2,6 +2,7 @@
 import cgi
 import unittest
 import re
+import json
 
 import six
 from six.moves import xmlrpc_client as xmlrpclib
@@ -9,7 +10,7 @@ from six.moves.urllib.parse import urlparse, parse_qs, unquote
 if six.PY3:
     from urllib.parse import unquote_to_bytes
 
-from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, HtmlResponse
+from scrapy.http import Request, FormRequest, XmlRpcRequest, JSONRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_native_str
 
 
@@ -1147,5 +1148,57 @@ class XmlRpcRequestTest(RequestTest):
         self._test_request(params=(u'pas£',), encoding='latin1')
 
 
+class JSONRequestTest(RequestTest):
+    request_class = JSONRequest
+    default_method = 'POST'
+    default_headers = {b'Content-Type': [b'application/json']}
+
+    def test_body(self):
+        r1 = self.request_class(url="http://www.example.com/")
+        self.assertEqual(r1.body, '{}')
+
+        r2 = self.request_class(url="http://www.example.com/", body=b"")
+        self.assertEqual(r2.body, '{}')
+
+        data = {
+            'name': 'value',
+        }
+        r3 = self.request_class(url="http://www.example.com/", data=data)
+        self.assertEqual(r3.body, json.dumps(data))
+
+        r4 = self.request_class(url="http://www.example.com/", body='body1', data=data)
+        self.assertEqual(r3.body, json.dumps(data))
+
+    def test_replace(self):
+        """Test Request.replace() method"""
+        r1 = self.request_class("http://www.example.com")
+        hdrs = Headers(r1.headers)
+        hdrs[b'key'] = b'value'
+        r2 = r1.replace(body="New body", headers=hdrs)
+
+        # body will not be replaced
+        self.assertEqual(r1.body, r2.body)
+        self.assertEqual(r1.url, r2.url)
+        self.assertEqual((r1.headers, r2.headers), (self.default_headers, hdrs))
+
+        # Empty attributes (which may fail if not compared properly)
+        r3 = self.request_class("http://www.example.com", meta={'a': 1}, dont_filter=True)
+        r4 = r3.replace(url="http://www.example.com/2", meta={}, dont_filter=False)
+        self.assertEqual(r4.url, "http://www.example.com/2")
+        self.assertEqual(r4.meta, {})
+        assert r4.dont_filter is False
+
+        data1 = {
+            'name': 'value1',
+        }
+        data2 = {
+            'name': 'value2',
+        }
+        r5 = self.request_class("http://www.example.com", data=data1)
+        r6 = r5.replace(url="http://www.example.com/2", data=data2)
+        self.assertNotEqual(r5.body, r6.body)
+        self.assertEqual((r5.body, r6.body), (json.dumps(data1), json.dumps(data2)))
+
+
 if __name__ == "__main__":
     unittest.main()

From 1b2b8b4bf0c73b4ad143f943584545702d66cbb7 Mon Sep 17 00:00:00 2001
From: kasun Herath <kasunh01@gmail.com>
Date: Tue, 27 Nov 2018 08:57:44 +0530
Subject: [PATCH 1841/2974] fix tests under py3

---
 tests/test_http_request.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 3f2e4f521..a2021bd65 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1155,19 +1155,19 @@ class JSONRequestTest(RequestTest):
 
     def test_body(self):
         r1 = self.request_class(url="http://www.example.com/")
-        self.assertEqual(r1.body, '{}')
+        self.assertEqual(r1.body, b'{}')
 
         r2 = self.request_class(url="http://www.example.com/", body=b"")
-        self.assertEqual(r2.body, '{}')
+        self.assertEqual(r2.body, b'{}')
 
         data = {
             'name': 'value',
         }
         r3 = self.request_class(url="http://www.example.com/", data=data)
-        self.assertEqual(r3.body, json.dumps(data))
+        self.assertEqual(r3.body, to_bytes(json.dumps(data)))
 
         r4 = self.request_class(url="http://www.example.com/", body='body1', data=data)
-        self.assertEqual(r3.body, json.dumps(data))
+        self.assertEqual(r3.body, to_bytes(json.dumps(data)))
 
     def test_replace(self):
         """Test Request.replace() method"""
@@ -1197,7 +1197,7 @@ class JSONRequestTest(RequestTest):
         r5 = self.request_class("http://www.example.com", data=data1)
         r6 = r5.replace(url="http://www.example.com/2", data=data2)
         self.assertNotEqual(r5.body, r6.body)
-        self.assertEqual((r5.body, r6.body), (json.dumps(data1), json.dumps(data2)))
+        self.assertEqual((r5.body, r6.body), (to_bytes(json.dumps(data1)), to_bytes(json.dumps(data2))))
 
 
 if __name__ == "__main__":

From 274b65dff4dc8b8300d872171679f173fbe0a746 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 3 Dec 2018 16:36:05 +0100
Subject: [PATCH 1842/2974] Add a troubleshooting section to the installation
 instructions

Its initial content covers the workaround for #2473.
---
 docs/intro/install.rst | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 4a9aa3cfb..daec7fcb7 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -30,7 +30,8 @@ dependencies depending on your operating system, so be sure to check the
 We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
 to avoid conflicting with your system packages.
 
-For more detailed and platform specifics instructions, read on.
+For more detailed and platform specifics instructions, as well as
+troubleshooting information, read on.
 
 
 Things that are good to know
@@ -247,6 +248,34 @@ that setuptools was unable to pick up one PyPy-specific dependency.
 To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``.
 
 
+.. _intro-install-troubleshooting:
+
+Troubleshooting
+===============
+
+AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1'
+----------------------------------------------------------------
+
+After you install or upgrade Scrapy, Twisted or pyOpenSSL, you may get an
+exception with the following traceback::
+
+    […]
+      File "[…]/site-packages/twisted/protocols/tls.py", line 63, in <module>
+        from twisted.internet._sslverify import _setAcceptableProtocols
+      File "[…]/site-packages/twisted/internet/_sslverify.py", line 38, in <module>
+        TLSVersion.TLSv1_1: SSL.OP_NO_TLSv1_1,
+    AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1'
+
+The reason you get this exception is that your system or virtual environment
+has a version of pyOpenSSL that your version of Twisted does not support.
+
+To install a version of pyOpenSSL that your version of Twisted supports,
+reinstall Twisted with the :code:`tls` extra option::
+
+    pip install twisted[tls]
+
+For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
+
 .. _Python: https://www.python.org/
 .. _pip: https://pip.pypa.io/en/latest/installing/
 .. _lxml: http://lxml.de/

From 62f3349c1aee54599ab7ee8755d2b31090639105 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 3 Dec 2018 17:14:10 +0100
Subject: [PATCH 1843/2974] Document the SCRAPY_PROJECT environment variable

Fixes #1109
---
 docs/topics/commands.rst | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index ef9c45196..97f8311de 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -37,7 +37,7 @@ Scrapy also understands, and can be configured through, a number of environment
 variables. Currently these are:
 
 * ``SCRAPY_SETTINGS_MODULE`` (see :ref:`topics-settings-module-envvar`)
-* ``SCRAPY_PROJECT``
+* ``SCRAPY_PROJECT`` (see :ref:`topics-project-envvar`)
 * ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`)
 
 .. _topics-project-structure:
@@ -71,6 +71,33 @@ the project settings. Here is an example::
     [settings]
     default = myproject.settings
 
+.. _topics-project-envvar:
+
+Sharing the root directory between projects
+===========================================
+
+A project root directory, the one that contains the ``scrapy.cfg``, may be
+shared by multiple Scrapy projects, each with its own settings module.
+
+In that case, you must define one or more aliases for those settings modules
+under ``[settings]`` in your ``scrapy.cfg`` file::
+
+    [settings]
+    default = myproject1.settings
+    project1 = myproject1.settings
+    project2 = myproject2.settings
+
+By default, the ``scrapy`` command-line tool will use the ``default`` settings.
+Use the ``SCRAPY_PROJECT`` environment variable to specify a different project
+for ``scrapy`` to use::
+
+    $ scrapy settings --get BOT_NAME
+    Project 1 Bot
+    $ export SCRAPY_PROJECT=project2
+    $ scrapy settings --get BOT_NAME
+    Project 2 Bot
+
+
 Using the ``scrapy`` tool
 =========================
 

From d7c8eee2fc918d07feb708c41089da21b2b9aea5 Mon Sep 17 00:00:00 2001
From: fpghost <leehodg@gmail.com>
Date: Tue, 4 Dec 2018 10:57:51 +0100
Subject: [PATCH 1844/2974] the strip() isnt needed

---
 scrapy/downloadermiddlewares/httpproxy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 1dd47359f..2c35d1b90 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -30,7 +30,7 @@ class HttpProxyMiddleware(object):
         user_pass = to_bytes(
             '%s:%s' % (unquote(username), unquote(password)),
             encoding=self.auth_encoding)
-        return base64.b64encode(user_pass).strip()
+        return base64.b64encode(user_pass)
 
     def _get_proxy(self, url, orig_type):
         proxy_type, user, password, hostport = _parse_proxy(url)

From cd619c1d4f3810c96af0ff5c5735c1856dfac95a Mon Sep 17 00:00:00 2001
From: kasun Herath <kasunh01@gmail.com>
Date: Sat, 8 Dec 2018 22:10:45 +0530
Subject: [PATCH 1845/2974] removed overriden replace method

---
 scrapy/http/request/json_request.py | 20 +++++------
 tests/test_http_request.py          | 51 ++++++++---------------------
 2 files changed, 21 insertions(+), 50 deletions(-)

diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 0fdd2ddf1..03a0ab061 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -12,17 +12,13 @@ from scrapy.http.request import Request
 
 class JSONRequest(Request):
     def __init__(self, *args, **kwargs):
-        if 'method' not in kwargs:
-            kwargs['method'] = 'POST'
+        data = kwargs.pop('data', None)
+        if data:
+            kwargs['body'] = json.dumps(data)
+
+            if 'method' not in kwargs:
+                kwargs['method'] = 'POST'
 
-        data = kwargs.pop('data', {})
-        kwargs['body'] = json.dumps(data)
         super(JSONRequest, self).__init__(*args, **kwargs)
-        self.headers.setdefault(b'Content-Type', b'application/json')
-
-    def replace(self, *args, **kwargs):
-        """ Create a new Request with the same attributes except for those
-            given new values. """
-
-        kwargs.pop('body', None)
-        return super(JSONRequest, self).replace(*args, **kwargs)
+        self.headers.setdefault('Content-Type', 'application/json')
+        self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index a2021bd65..793a583bc 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1150,54 +1150,29 @@ class XmlRpcRequestTest(RequestTest):
 
 class JSONRequestTest(RequestTest):
     request_class = JSONRequest
-    default_method = 'POST'
-    default_headers = {b'Content-Type': [b'application/json']}
+    default_method = 'GET'
+    default_headers = {b'Content-Type': [b'application/json'], b'Accept': [b'application/json, text/javascript, */*; q=0.01']}
 
-    def test_body(self):
+    def test_data(self):
         r1 = self.request_class(url="http://www.example.com/")
-        self.assertEqual(r1.body, b'{}')
+        self.assertEqual(r1.body, b'')
+        self.assertEqual(r1.method, 'GET')
 
-        r2 = self.request_class(url="http://www.example.com/", body=b"")
-        self.assertEqual(r2.body, b'{}')
+        body = b'body'
+        r2 = self.request_class(url="http://www.example.com/", body=body)
+        self.assertEqual(r2.body, body)
+        self.assertEqual(r2.method, 'GET')
 
         data = {
             'name': 'value',
         }
         r3 = self.request_class(url="http://www.example.com/", data=data)
         self.assertEqual(r3.body, to_bytes(json.dumps(data)))
+        self.assertEqual(r3.method, 'POST')
 
-        r4 = self.request_class(url="http://www.example.com/", body='body1', data=data)
-        self.assertEqual(r3.body, to_bytes(json.dumps(data)))
-
-    def test_replace(self):
-        """Test Request.replace() method"""
-        r1 = self.request_class("http://www.example.com")
-        hdrs = Headers(r1.headers)
-        hdrs[b'key'] = b'value'
-        r2 = r1.replace(body="New body", headers=hdrs)
-
-        # body will not be replaced
-        self.assertEqual(r1.body, r2.body)
-        self.assertEqual(r1.url, r2.url)
-        self.assertEqual((r1.headers, r2.headers), (self.default_headers, hdrs))
-
-        # Empty attributes (which may fail if not compared properly)
-        r3 = self.request_class("http://www.example.com", meta={'a': 1}, dont_filter=True)
-        r4 = r3.replace(url="http://www.example.com/2", meta={}, dont_filter=False)
-        self.assertEqual(r4.url, "http://www.example.com/2")
-        self.assertEqual(r4.meta, {})
-        assert r4.dont_filter is False
-
-        data1 = {
-            'name': 'value1',
-        }
-        data2 = {
-            'name': 'value2',
-        }
-        r5 = self.request_class("http://www.example.com", data=data1)
-        r6 = r5.replace(url="http://www.example.com/2", data=data2)
-        self.assertNotEqual(r5.body, r6.body)
-        self.assertEqual((r5.body, r6.body), (to_bytes(json.dumps(data1)), to_bytes(json.dumps(data2))))
+        r4 = self.request_class(url="http://www.example.com/", body=body, data=data)
+        self.assertEqual(r4.body, to_bytes(json.dumps(data)))
+        self.assertEqual(r4.method, 'POST')
 
 
 if __name__ == "__main__":

From c347acbff6545c428aa2c965cd03f03db6bae1bf Mon Sep 17 00:00:00 2001
From: kasun Herath <kasunh01@gmail.com>
Date: Sun, 9 Dec 2018 11:27:09 +0530
Subject: [PATCH 1846/2974] warning if body and data are provided

---
 scrapy/http/request/json_request.py |  7 ++++++-
 tests/test_http_request.py          | 25 ++++++++++++++++++++++---
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 03a0ab061..3b791eda3 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -6,14 +6,19 @@ See documentation in docs/topics/request-response.rst
 """
 
 import json
+import warnings
 
 from scrapy.http.request import Request
 
 
 class JSONRequest(Request):
     def __init__(self, *args, **kwargs):
+        body_passed = 'body' in kwargs
         data = kwargs.pop('data', None)
-        if data:
+        if body_passed and data:
+            warnings.warn('Both body and data passed. data will be ignored')
+
+        elif not body_passed and data:
             kwargs['body'] = json.dumps(data)
 
             if 'method' not in kwargs:
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 793a583bc..e5a85e6fc 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -3,6 +3,7 @@ import cgi
 import unittest
 import re
 import json
+import warnings
 
 import six
 from six.moves import xmlrpc_client as xmlrpclib
@@ -1153,6 +1154,10 @@ class JSONRequestTest(RequestTest):
     default_method = 'GET'
     default_headers = {b'Content-Type': [b'application/json'], b'Accept': [b'application/json, text/javascript, */*; q=0.01']}
 
+    def setUp(self):
+        warnings.simplefilter("always")
+        super(JSONRequestTest, self).setUp()
+
     def test_data(self):
         r1 = self.request_class(url="http://www.example.com/")
         self.assertEqual(r1.body, b'')
@@ -1170,9 +1175,23 @@ class JSONRequestTest(RequestTest):
         self.assertEqual(r3.body, to_bytes(json.dumps(data)))
         self.assertEqual(r3.method, 'POST')
 
-        r4 = self.request_class(url="http://www.example.com/", body=body, data=data)
-        self.assertEqual(r4.body, to_bytes(json.dumps(data)))
-        self.assertEqual(r4.method, 'POST')
+        with warnings.catch_warnings(record=True) as _warnings:
+            r4 = self.request_class(url="http://www.example.com/", body=body, data=data)
+            self.assertEqual(r4.body, body)
+            self.assertEqual(r4.method, 'GET')
+            self.assertEqual(len(_warnings), 1)
+            self.assertIn('data will be ignored', str(_warnings[0].message))
+
+        with warnings.catch_warnings(record=True) as _warnings:
+            r5 = self.request_class(url="http://www.example.com/", body=b'', data=data)
+            self.assertEqual(r5.body, b'')
+            self.assertEqual(r5.method, 'GET')
+            self.assertEqual(len(_warnings), 1)
+            self.assertIn('data will be ignored', str(_warnings[0].message))
+
+    def tearDown(self):
+        warnings.resetwarnings()
+        super(JSONRequestTest, self).tearDown()
 
 
 if __name__ == "__main__":

From 3c981bf204c739fa77e205b9747d2aff446c99d5 Mon Sep 17 00:00:00 2001
From: kasun Herath <kasunh01@gmail.com>
Date: Sun, 9 Dec 2018 12:56:12 +0530
Subject: [PATCH 1847/2974] add documentation

---
 docs/topics/request-response.rst | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index e29914dbf..d957915e7 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -508,6 +508,38 @@ method for this job. Here's an example spider which uses it::
 
             # continue scraping with authenticated session...
 
+JSONRequest
+-----------
+
+The JSONRequest class extends the base :class:`Request` class with functionality for
+dealing with JSON requests.
+
+.. class:: JSONRequest(url, [data, ...])
+
+   The :class:`JSONRequest` class adds a new argument to the constructor called data. The
+   remaining arguments are the same as for the :class:`Request` class and are
+   not documented here.
+
+   Using the :class:`JSONRequest` will set the `Content-Type` header to `application/json`
+   and `Accept` header to `application/json, text/javascript, */*; q=0.01` 
+
+   :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body.
+      if :attr:`Request.body` argument is provided this parameter will be ignored.
+      if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be 
+      set to POST automatically.
+   :type data: JSON serializable object
+
+JSONRequest usage example
+-------------------------
+
+Sending a JSON POST request with a JSON payload::
+
+   data = {
+       'name1': 'value1',
+       'name2': 'value2',
+   }
+   yield JSONRequest(url='http://www.example.com/post/action', data=data)
+
 
 Response objects
 ================

From 4d48759978ac2405bc2cb30f84af948693e4cad3 Mon Sep 17 00:00:00 2001
From: Lucy Wang <wxitb2017@gmail.com>
Date: Mon, 10 Dec 2018 14:44:15 +0800
Subject: [PATCH 1848/2974] remove "sudo: false" now that travis no longer
 supports it

https://changelog.travis-ci.com/deprecation-container-based-linux-build-environment-82037
---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 4218d13bf..08b0bf119 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,4 @@
 language: python
-sudo: false
 branches:
   only:
     - master

From ecda69130e97629b15d3b09b1e588cb6777ee94d Mon Sep 17 00:00:00 2001
From: kasun Herath <kasunh01@gmail.com>
Date: Mon, 10 Dec 2018 22:34:49 +0530
Subject: [PATCH 1849/2974] allow to send empty data values and docs changes

---
 docs/topics/request-response.rst    |  6 +++---
 scrapy/http/request/json_request.py |  8 +++++---
 tests/test_http_request.py          | 27 +++++++++++++++++++++------
 3 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index d957915e7..02b853fc0 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -520,13 +520,13 @@ dealing with JSON requests.
    remaining arguments are the same as for the :class:`Request` class and are
    not documented here.
 
-   Using the :class:`JSONRequest` will set the `Content-Type` header to `application/json`
-   and `Accept` header to `application/json, text/javascript, */*; q=0.01` 
+   Using the :class:`JSONRequest` will set the ``Content-Type`` header to ``application/json``
+   and ``Accept`` header to ``application/json, text/javascript, */*; q=0.01``
 
    :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body.
       if :attr:`Request.body` argument is provided this parameter will be ignored.
       if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be 
-      set to POST automatically.
+      set to ``'POST'`` automatically.
    :type data: JSON serializable object
 
 JSONRequest usage example
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 3b791eda3..593dfdcb0 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -13,12 +13,14 @@ from scrapy.http.request import Request
 
 class JSONRequest(Request):
     def __init__(self, *args, **kwargs):
-        body_passed = 'body' in kwargs
+        body_passed = kwargs.get('body', None) is not None
         data = kwargs.pop('data', None)
-        if body_passed and data:
+        data_passed = data is not None
+
+        if body_passed and data_passed:
             warnings.warn('Both body and data passed. data will be ignored')
 
-        elif not body_passed and data:
+        elif not body_passed and data_passed:
             kwargs['body'] = json.dumps(data)
 
             if 'method' not in kwargs:
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index e5a85e6fc..5eb655c12 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1175,20 +1175,35 @@ class JSONRequestTest(RequestTest):
         self.assertEqual(r3.body, to_bytes(json.dumps(data)))
         self.assertEqual(r3.method, 'POST')
 
+        r4 = self.request_class(url="http://www.example.com/", data=[])
+        self.assertEqual(r4.body, to_bytes(json.dumps([])))
+        self.assertEqual(r4.method, 'POST')
+
         with warnings.catch_warnings(record=True) as _warnings:
-            r4 = self.request_class(url="http://www.example.com/", body=body, data=data)
-            self.assertEqual(r4.body, body)
-            self.assertEqual(r4.method, 'GET')
+            r5 = self.request_class(url="http://www.example.com/", body=body, data=data)
+            self.assertEqual(r5.body, body)
+            self.assertEqual(r5.method, 'GET')
             self.assertEqual(len(_warnings), 1)
             self.assertIn('data will be ignored', str(_warnings[0].message))
 
         with warnings.catch_warnings(record=True) as _warnings:
-            r5 = self.request_class(url="http://www.example.com/", body=b'', data=data)
-            self.assertEqual(r5.body, b'')
-            self.assertEqual(r5.method, 'GET')
+            r6 = self.request_class(url="http://www.example.com/", body=b'', data=data)
+            self.assertEqual(r6.body, b'')
+            self.assertEqual(r6.method, 'GET')
             self.assertEqual(len(_warnings), 1)
             self.assertIn('data will be ignored', str(_warnings[0].message))
 
+        with warnings.catch_warnings(record=True) as _warnings:
+            r7 = self.request_class(url="http://www.example.com/", body=None, data=data)
+            self.assertEqual(r7.body, to_bytes(json.dumps(data)))
+            self.assertEqual(r7.method, 'POST')
+            self.assertEqual(len(_warnings), 0)
+
+        with warnings.catch_warnings(record=True) as _warnings:
+            r8 = self.request_class(url="http://www.example.com/", body=None, data=None)
+            self.assertEqual(r8.method, 'GET')
+            self.assertEqual(len(_warnings), 0)
+
     def tearDown(self):
         warnings.resetwarnings()
         super(JSONRequestTest, self).tearDown()

From cd9d8e28cdf49ff63e1b3f9126e6651fcd77e0fa Mon Sep 17 00:00:00 2001
From: hsiao yi <hsiaoyi0504@gmail.com>
Date: Tue, 11 Dec 2018 19:21:07 +0800
Subject: [PATCH 1850/2974] unify the quote style

---
 docs/intro/overview.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 9d7c94d39..8b2fef065 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -26,7 +26,7 @@ http://quotes.toscrape.com, following the pagination::
 
 
     class QuotesSpider(scrapy.Spider):
-        name = "quotes"
+        name = 'quotes'
         start_urls = [
             'http://quotes.toscrape.com/tag/humor/',
         ]

From 71ef321b68d2fd202de145d0c580387ee59cd2e2 Mon Sep 17 00:00:00 2001
From: kasun Herath <kasunh01@gmail.com>
Date: Wed, 12 Dec 2018 11:12:48 +0530
Subject: [PATCH 1851/2974] sort_keys while serializing to json

---
 scrapy/http/request/json_request.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 593dfdcb0..afc4356a3 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -21,7 +21,7 @@ class JSONRequest(Request):
             warnings.warn('Both body and data passed. data will be ignored')
 
         elif not body_passed and data_passed:
-            kwargs['body'] = json.dumps(data)
+            kwargs['body'] = json.dumps(data, sort_keys=True)
 
             if 'method' not in kwargs:
                 kwargs['method'] = 'POST'

From 8f1507a4a5de2ed55cb0fda198265845a047fedb Mon Sep 17 00:00:00 2001
From: kasun Herath <kasunh01@gmail.com>
Date: Mon, 17 Dec 2018 23:14:06 +0530
Subject: [PATCH 1852/2974] dumps_kwargs

---
 docs/topics/request-response.rst    |  10 ++-
 scrapy/http/request/json_request.py |  21 ++++-
 tests/test_http_request.py          | 114 +++++++++++++++++++++++++++-
 3 files changed, 138 insertions(+), 7 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 02b853fc0..4e6f00bb0 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -514,9 +514,9 @@ JSONRequest
 The JSONRequest class extends the base :class:`Request` class with functionality for
 dealing with JSON requests.
 
-.. class:: JSONRequest(url, [data, ...])
+.. class:: JSONRequest(url, [... data])
 
-   The :class:`JSONRequest` class adds a new argument to the constructor called data. The
+   The :class:`JSONRequest` class adds two new argument to the constructor. The
    remaining arguments are the same as for the :class:`Request` class and are
    not documented here.
 
@@ -529,6 +529,12 @@ dealing with JSON requests.
       set to ``'POST'`` automatically.
    :type data: JSON serializable object
 
+   :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize data
+       into JSON format.
+   :type dumps_kwargs: dict
+
+.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps
+
 JSONRequest usage example
 -------------------------
 
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index afc4356a3..7499610b9 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -13,6 +13,7 @@ from scrapy.http.request import Request
 
 class JSONRequest(Request):
     def __init__(self, *args, **kwargs):
+        dumps_kwargs = kwargs.pop('dumps_kwargs', {})
         body_passed = kwargs.get('body', None) is not None
         data = kwargs.pop('data', None)
         data_passed = data is not None
@@ -21,7 +22,7 @@ class JSONRequest(Request):
             warnings.warn('Both body and data passed. data will be ignored')
 
         elif not body_passed and data_passed:
-            kwargs['body'] = json.dumps(data, sort_keys=True)
+            kwargs['body'] = self.dump(data, **dumps_kwargs)
 
             if 'method' not in kwargs:
                 kwargs['method'] = 'POST'
@@ -29,3 +30,21 @@ class JSONRequest(Request):
         super(JSONRequest, self).__init__(*args, **kwargs)
         self.headers.setdefault('Content-Type', 'application/json')
         self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
+        self._dumps_kwargs = dumps_kwargs
+
+    def replace(self, *args, **kwargs):
+        body_passed = kwargs.get('body', None) is not None
+        data = kwargs.pop('data', None)
+        data_passed = data is not None
+
+        if body_passed and data_passed:
+            warnings.warn('Both body and data passed. data will be ignored')
+
+        elif not body_passed and data_passed:
+            kwargs['body'] = self.dump(data, **self._dumps_kwargs)
+
+        return super(JSONRequest, self).replace(*args, **kwargs)
+
+    def dump(self, data, **kwargs):
+        """Convert to JSON """
+        return json.dumps(data, sort_keys=True, **kwargs)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 5eb655c12..6dcfa25da 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -14,6 +14,8 @@ if six.PY3:
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JSONRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_native_str
 
+from tests import mock
+
 
 class RequestTest(unittest.TestCase):
 
@@ -1161,24 +1163,49 @@ class JSONRequestTest(RequestTest):
     def test_data(self):
         r1 = self.request_class(url="http://www.example.com/")
         self.assertEqual(r1.body, b'')
-        self.assertEqual(r1.method, 'GET')
 
         body = b'body'
         r2 = self.request_class(url="http://www.example.com/", body=body)
         self.assertEqual(r2.body, body)
-        self.assertEqual(r2.method, 'GET')
 
         data = {
             'name': 'value',
         }
         r3 = self.request_class(url="http://www.example.com/", data=data)
         self.assertEqual(r3.body, to_bytes(json.dumps(data)))
-        self.assertEqual(r3.method, 'POST')
 
+        # empty data
         r4 = self.request_class(url="http://www.example.com/", data=[])
         self.assertEqual(r4.body, to_bytes(json.dumps([])))
-        self.assertEqual(r4.method, 'POST')
 
+    def test_data_method(self):
+        # data is not passed
+        r1 = self.request_class(url="http://www.example.com/")
+        self.assertEqual(r1.method, 'GET')
+
+        body = b'body'
+        r2 = self.request_class(url="http://www.example.com/", body=body)
+        self.assertEqual(r2.method, 'GET')
+
+        data = {
+            'name': 'value',
+        }
+        r3 = self.request_class(url="http://www.example.com/", data=data)
+        self.assertEqual(r3.method, 'POST')
+
+        # method passed explicitly
+        r4 = self.request_class(url="http://www.example.com/", data=data, method='GET')
+        self.assertEqual(r4.method, 'GET')
+
+        r5 = self.request_class(url="http://www.example.com/", data=[])
+        self.assertEqual(r5.method, 'POST')
+
+    def test_body_data(self):
+        """ passing both body and data should result a warning """
+        body = b'body'
+        data = {
+            'name': 'value',
+        }
         with warnings.catch_warnings(record=True) as _warnings:
             r5 = self.request_class(url="http://www.example.com/", body=body, data=data)
             self.assertEqual(r5.body, body)
@@ -1186,6 +1213,11 @@ class JSONRequestTest(RequestTest):
             self.assertEqual(len(_warnings), 1)
             self.assertIn('data will be ignored', str(_warnings[0].message))
 
+    def test_empty_body_data(self):
+        """ passing any body value and data should result a warning """
+        data = {
+            'name': 'value',
+        }
         with warnings.catch_warnings(record=True) as _warnings:
             r6 = self.request_class(url="http://www.example.com/", body=b'', data=data)
             self.assertEqual(r6.body, b'')
@@ -1193,17 +1225,91 @@ class JSONRequestTest(RequestTest):
             self.assertEqual(len(_warnings), 1)
             self.assertIn('data will be ignored', str(_warnings[0].message))
 
+    def test_body_none_data(self):
+        data = {
+            'name': 'value',
+        }
         with warnings.catch_warnings(record=True) as _warnings:
             r7 = self.request_class(url="http://www.example.com/", body=None, data=data)
             self.assertEqual(r7.body, to_bytes(json.dumps(data)))
             self.assertEqual(r7.method, 'POST')
             self.assertEqual(len(_warnings), 0)
 
+    def test_body_data_none(self):
         with warnings.catch_warnings(record=True) as _warnings:
             r8 = self.request_class(url="http://www.example.com/", body=None, data=None)
             self.assertEqual(r8.method, 'GET')
             self.assertEqual(len(_warnings), 0)
 
+    def test_dumps_sort_keys(self):
+        """ Test that sort_keys=True is passed to json.dumps by default """
+        data = {
+            'name': 'value',
+        }
+        with mock.patch('json.dumps', return_value=b'') as mock_dumps:
+            self.request_class(url="http://www.example.com/", data=data)
+            kwargs = mock_dumps.call_args[1]
+            self.assertEqual(kwargs['sort_keys'], True)
+
+    def test_dumps_kwargs(self):
+        """ Test that dumps_kwargs are passed to json.dumps """
+        data = {
+            'name': 'value',
+        }
+        dumps_kwargs = {
+            'ensure_ascii': True,
+            'allow_nan': True,
+        }
+        with mock.patch('json.dumps', return_value=b'') as mock_dumps:
+            self.request_class(url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs)
+            kwargs = mock_dumps.call_args[1]
+            self.assertEqual(kwargs['ensure_ascii'], True)
+            self.assertEqual(kwargs['allow_nan'], True)
+
+    def test_replace_data(self):
+        data1 = {
+            'name1': 'value1',
+        }
+        data2 = {
+            'name2': 'value2',
+        }
+        r1 = self.request_class(url="http://www.example.com/", data=data1)
+        r2 = r1.replace(data=data2)
+        self.assertEqual(r2.body, to_bytes(json.dumps(data2)))
+
+    def test_replace_sort_keys(self):
+        """ Test that replace provides sort_keys=True to json.dumps """
+        data1 = {
+            'name1': 'value1',
+        }
+        data2 = {
+            'name2': 'value2',
+        }
+        r1 = self.request_class(url="http://www.example.com/", data=data1)
+        with mock.patch('json.dumps', return_value=b'') as mock_dumps:
+            r1.replace(data=data2)
+            kwargs = mock_dumps.call_args[1]
+            self.assertEqual(kwargs['sort_keys'], True)
+
+    def test_replace_dumps_kwargs(self):
+        """ Test that dumps_kwargs are provided json.dumps when replace is called """
+        data1 = {
+            'name1': 'value1',
+        }
+        data2 = {
+            'name2': 'value2',
+        }
+        dumps_kwargs = {
+            'ensure_ascii': True,
+            'allow_nan': True,
+        }
+        r1 = self.request_class(url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs)
+        with mock.patch('json.dumps', return_value=b'') as mock_dumps:
+            r1.replace(data=data2)
+            kwargs = mock_dumps.call_args[1]
+            self.assertEqual(kwargs['ensure_ascii'], True)
+            self.assertEqual(kwargs['allow_nan'], True)
+
     def tearDown(self):
         warnings.resetwarnings()
         super(JSONRequestTest, self).tearDown()

From 12ad06b7ac57dd022a4add16259ee8fd64d5ede2 Mon Sep 17 00:00:00 2001
From: kasun Herath <kasunh01@gmail.com>
Date: Mon, 17 Dec 2018 23:17:13 +0530
Subject: [PATCH 1853/2974] docs change

---
 docs/topics/request-response.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 4e6f00bb0..6758269b1 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -529,8 +529,8 @@ dealing with JSON requests.
       set to ``'POST'`` automatically.
    :type data: JSON serializable object
 
-   :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize data
-       into JSON format.
+   :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize
+       data into JSON format.
    :type dumps_kwargs: dict
 
 .. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps

From 24acc50d1894b6566e427f1dfea14e2aa647077e Mon Sep 17 00:00:00 2001
From: kasun Herath <kasunh01@gmail.com>
Date: Tue, 18 Dec 2018 23:16:14 +0530
Subject: [PATCH 1854/2974] dumps_kwargs parameter in docs

---
 docs/topics/request-response.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 6758269b1..37b73edd1 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -514,7 +514,7 @@ JSONRequest
 The JSONRequest class extends the base :class:`Request` class with functionality for
 dealing with JSON requests.
 
-.. class:: JSONRequest(url, [... data])
+.. class:: JSONRequest(url, [... data, dumps_kwargs])
 
    The :class:`JSONRequest` class adds two new argument to the constructor. The
    remaining arguments are the same as for the :class:`Request` class and are

From f6dfc5f3dd56b7c823e3f53f7f9f63515ca7c3e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 20 Dec 2018 19:23:23 -0300
Subject: [PATCH 1855/2974] Fix boto import error under Jessie testing
 environment

---
 .travis.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 4218d13bf..252c783d7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -43,6 +43,11 @@ install:
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
       fi
+      if [ "$TOXENV" = "jessie" ]; then
+        # Not used directly but allows boto GCE plugins to load.
+        # https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262
+        pip install google-compute-engine
+      fi
   - pip install -U tox twine wheel codecov
 
 script: tox

From 8ed6beb7f9199e8924cd03bd34a46194c3d82e32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 20 Dec 2018 19:39:29 -0300
Subject: [PATCH 1856/2974] Needs to be installed within tox env

---
 .travis.yml | 5 -----
 tox.ini     | 3 +++
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 252c783d7..4218d13bf 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -43,11 +43,6 @@ install:
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
       fi
-      if [ "$TOXENV" = "jessie" ]; then
-        # Not used directly but allows boto GCE plugins to load.
-        # https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262
-        pip install google-compute-engine
-      fi
   - pip install -U tox twine wheel codecov
 
 script: tox
diff --git a/tox.ini b/tox.ini
index e5543fe2a..0c0f8f7b7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -51,6 +51,9 @@ deps =
     cssselect==0.9.1
     zope.interface==4.1.1
     -rtests/requirements-py2.txt
+# Not used directly but allows boto GCE plugins to load.
+# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262
+    google-compute-engine==2.8.12
 
 [testenv:trunk]
 basepython = python2.7

From f85c915872cf70bb87a05cecd6ef5a6534d2c4ed Mon Sep 17 00:00:00 2001
From: Joaquin Garmendia Cabrera <joaquingc123@gmail.com>
Date: Sun, 23 Dec 2018 00:26:58 -0500
Subject: [PATCH 1857/2974] Update item-pipeline example

---
 docs/topics/item-pipeline.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 38265b474..1c2c51e05 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -87,8 +87,8 @@ contain a price::
         vat_factor = 1.15
 
         def process_item(self, item, spider):
-            if item['price']:
-                if item['price_excludes_vat']:
+            if 'price' in item and item['price']:
+                if 'price_excludes_vat' in item and item['price_excludes_vat']:
                     item['price'] = item['price'] * self.vat_factor
                 return item
             else:

From 7c26701012c8e41a3e2c2644e05ce852d7472bc3 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 26 Dec 2018 01:33:58 +0500
Subject: [PATCH 1858/2974] DOC warn about telnet console being insecure

---
 docs/topics/telnetconsole.rst | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index 4db9cafb2..bf2ffa443 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -16,6 +16,17 @@ The telnet console is a :ref:`built-in Scrapy extension
 disable it if you want. For more information about the extension itself see
 :ref:`topics-extensions-ref-telnetconsole`.
 
+.. warning::
+    It is not secure to use telnet console via public networks, as telnet
+    doesn't provide any transport-layer security. Having username/password
+    authentication doesn't change that.
+
+    Intended usage is connecting to a running Scrapy spider locally
+    (spider process and telnet client are on the same machine)
+    or over a secure connection (VPN, SSH tunnel).
+    Please avoid using telnet console over insecure connections,
+    or disable it completely using :setting:`TELNETCONSOLE_ENABLED` option.
+
 .. highlight:: none
 
 How to access the telnet console
@@ -39,7 +50,12 @@ autogenerated Password can be seen on scrapy logs like the example bellow::
     2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
 
 Default Username and Password can be overriden by the settings 
-:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`
+:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`.
+
+.. warning::
+    Username and password provide only a limited protection, as telnet
+    is not using secure transport - by default traffic is not encrypted
+    even if username and password are set.
 
 You need the telnet program which comes installed by default in Windows, and
 most Linux distros.

From cdd04dfb1d9a2e6fd8c188dccae26bbdd3454ebd Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 26 Dec 2018 13:13:49 +0500
Subject: [PATCH 1859/2974] declare Python 3.7 support in setup.py

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index 8c47f67ce..bd666e93c 100644
--- a/setup.py
+++ b/setup.py
@@ -56,6 +56,7 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
         'Topic :: Internet :: WWW/HTTP',

From 71e47629b1cb65a61d8e4809177817c1a833f73c Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 26 Dec 2018 16:35:05 +0500
Subject: [PATCH 1860/2974] DOC fix docs for AWS_... settings. A follow-up to
 GH-2609.

---
 docs/topics/settings.rst | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 47b6cf13d..0ac26a9bd 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -178,35 +178,48 @@ Default: ``None``
 The AWS secret key used by code that requires access to `Amazon Web services`_,
 such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
 
-.. setting:: BOT_NAME
+.. setting:: AWS_ENDPOINT_URL
 
 AWS_ENDPOINT_URL
 ----------------
 
 Default: ``None``
 
-Endpoint URL used for S3-like self-hosted storage. Storage like Minio or s3.scality.
+Endpoint URL used for S3-like storage, for example Minio or s3.scality.
+Only supported with ``botocore`` library.
 
-.. setting:: AWS_ENDPOINT_URL
+.. setting:: AWS_USE_SSL
 
 AWS_USE_SSL
 -----------
 
 Default: ``None``
 
-Use this option if you want to disable SSL connection for communication with S3 or S3-like storage.
-By default SSL will be used.
+Use this option if you want to disable SSL connection for communication with
+S3 or S3-like storage. By default SSL will be used.
+Only supported with ``botocore`` library.
 
-.. setting:: AWS_USE_SSL
+.. setting:: AWS_VERIFY
 
 AWS_VERIFY
 ----------
 
 Default: ``None``
 
-Verify SSL connection between Scrapy and S3 or S3-like storage. By default SSL verification will occur.
+Verify SSL connection between Scrapy and S3 or S3-like storage. By default
+SSL verification will occur. Only supported with ``botocore`` library.
 
-.. setting:: AWS_VERIFY
+.. setting:: AWS_REGION_NAME
+
+AWS_REGION_NAME
+---------------
+
+Default: ``None``
+
+The name of the region associated with the AWS client.
+Only supported with ``botocore`` library.
+
+.. setting:: BOT_NAME
 
 BOT_NAME
 --------

From a5e1b7bb4724bafa26b476a87a9f12b4d6479661 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Thu, 29 Nov 2018 18:19:14 -0300
Subject: [PATCH 1861/2974] add sitemap_filter attribute to SitemapSpider class

it makes it possible to filter sitemap urls by any available attribute

for example, you can filter urls with lastmod greater than a given datetime

it can be helpful when the url loc itself does not aggregate that information
---
 docs/topics/spiders.rst   | 26 ++++++++++++++++++++++++++
 scrapy/spiders/sitemap.py | 10 ++++++++--
 2 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index a08dc30f2..b0b9e0483 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -680,6 +680,32 @@ SitemapSpider
 
         Default is ``sitemap_alternate_links`` disabled.
 
+    .. attribute:: sitemap_filter
+
+        Specifies a function to filter sitemap entries and their attributes.
+
+        For example::
+
+            <url>
+                <loc>http://example.com/</loc>
+                <lastmod>2005-01-01</lastmod>
+            </url>
+
+        We can define a ``sitemap_filter`` function to filter ``urls`` by date::
+
+            def sitemap_filter(urls):
+                from datetime import datetime
+                for url in urls:
+                    date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')
+                    if date_time.year >= 2005:
+                        yield url
+
+        This would retrieve only ``urls`` modified on 2005 and the following
+        years.
+
+        If you omit this attribute, all urls found in sitemaps will be
+        processed, observing other attributes and their settings.
+
 
 SitemapSpider examples
 ~~~~~~~~~~~~~~~~~~~~~~
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 0ee8ba5e7..907aba243 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -17,6 +17,7 @@ class SitemapSpider(Spider):
     sitemap_rules = [('', 'parse')]
     sitemap_follow = ['']
     sitemap_alternate_links = False
+    sitemap_filter = None
 
     def __init__(self, *a, **kw):
         super(SitemapSpider, self).__init__(*a, **kw)
@@ -43,12 +44,17 @@ class SitemapSpider(Spider):
                 return
 
             s = Sitemap(body)
+            if callable(self.sitemap_filter):
+                it = self.sitemap_filter(s)
+            else:
+                it = s
+
             if s.type == 'sitemapindex':
-                for loc in iterloc(s, self.sitemap_alternate_links):
+                for loc in iterloc(it, self.sitemap_alternate_links):
                     if any(x.search(loc) for x in self._follow):
                         yield Request(loc, callback=self._parse_sitemap)
             elif s.type == 'urlset':
-                for loc in iterloc(s, self.sitemap_alternate_links):
+                for loc in iterloc(it, self.sitemap_alternate_links):
                     for r, c in self._cbs:
                         if r.search(loc):
                             yield Request(loc, callback=c)

From 672385a371453c84faa2f31425e3701b25260629 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Thu, 29 Nov 2018 18:33:20 -0300
Subject: [PATCH 1862/2974] using a method definition instead of a None
 attribute

---
 docs/topics/spiders.rst   |  4 ++--
 scrapy/spiders/sitemap.py | 14 +++++++++-----
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index b0b9e0483..127c8d03e 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -680,7 +680,7 @@ SitemapSpider
 
         Default is ``sitemap_alternate_links`` disabled.
 
-    .. attribute:: sitemap_filter
+    .. method:: sitemap_filter(urls)
 
         Specifies a function to filter sitemap entries and their attributes.
 
@@ -703,7 +703,7 @@ SitemapSpider
         This would retrieve only ``urls`` modified on 2005 and the following
         years.
 
-        If you omit this attribute, all urls found in sitemaps will be
+        If you omit this method, all urls found in sitemaps will be
         processed, observing other attributes and their settings.
 
 
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 907aba243..c86e986db 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -17,7 +17,6 @@ class SitemapSpider(Spider):
     sitemap_rules = [('', 'parse')]
     sitemap_follow = ['']
     sitemap_alternate_links = False
-    sitemap_filter = None
 
     def __init__(self, *a, **kw):
         super(SitemapSpider, self).__init__(*a, **kw)
@@ -32,6 +31,14 @@ class SitemapSpider(Spider):
         for url in self.sitemap_urls:
             yield Request(url, self._parse_sitemap)
 
+    def sitemap_filter(self, urls):
+        """This method can be used to filter sitemap entries by their
+        attributes, for example, you can filter locs with lastmod greater
+        than a given date (see docs).
+        """
+        for url in urls:
+            yield url
+
     def _parse_sitemap(self, response):
         if response.url.endswith('/robots.txt'):
             for url in sitemap_urls_from_robots(response.text, base_url=response.url):
@@ -44,10 +51,7 @@ class SitemapSpider(Spider):
                 return
 
             s = Sitemap(body)
-            if callable(self.sitemap_filter):
-                it = self.sitemap_filter(s)
-            else:
-                it = s
+            it = self.sitemap_filter(s)
 
             if s.type == 'sitemapindex':
                 for loc in iterloc(it, self.sitemap_alternate_links):

From d7d5917ff12ecb8db7cd04592f7cc18b0ab1a996 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Fri, 30 Nov 2018 11:20:12 -0300
Subject: [PATCH 1863/2974] add tests for the sitemap_filter method in the
 SitemapSpider class

---
 tests/test_spider.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/tests/test_spider.py b/tests/test_spider.py
index f26da2334..871852ab2 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -375,6 +375,38 @@ Sitemap: /sitemap-relative-url.xml
                           'http://www.example.com/schweiz-deutsch/',
                           'http://www.example.com/italiano/'])
 
+    def test_sitemap_filter(self):
+        sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:xhtml="http://www.w3.org/1999/xhtml">
+        <url>
+            <loc>http://www.example.com/english/</loc>
+            <lastmod>2010-01-01</lastmod>
+        </url>
+        <url>
+            <loc>http://www.example.com/portuguese/</loc>
+            <lastmod>2005-01-01</lastmod>
+        </url>
+    </urlset>"""
+
+        class FilteredSitemapSpider(self.spider_class):
+            def sitemap_filter(self, urls):
+                from datetime import datetime
+                for url in urls:
+                    date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')
+                    if date_time.year > 2008:
+                        yield url
+
+        r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
+        spider = self.spider_class("example.com")
+        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
+                         ['http://www.example.com/english/',
+                          'http://www.example.com/portuguese/'])
+
+        spider = FilteredSitemapSpider("example.com")
+        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
+                         ['http://www.example.com/english/'])
+
 
 class DeprecationTest(unittest.TestCase):
 

From 657f0663b3cb97ca1c1a498c066de444bd30fa82 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Thu, 20 Dec 2018 13:35:52 -0300
Subject: [PATCH 1864/2974] rename param from urls to entries

---
 docs/topics/spiders.rst   | 16 ++++++++--------
 scrapy/spiders/sitemap.py |  6 +++---
 tests/test_spider.py      |  8 ++++----
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 127c8d03e..918f1cc36 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -680,7 +680,7 @@ SitemapSpider
 
         Default is ``sitemap_alternate_links`` disabled.
 
-    .. method:: sitemap_filter(urls)
+    .. method:: sitemap_filter(entries)
 
         Specifies a function to filter sitemap entries and their attributes.
 
@@ -691,19 +691,19 @@ SitemapSpider
                 <lastmod>2005-01-01</lastmod>
             </url>
 
-        We can define a ``sitemap_filter`` function to filter ``urls`` by date::
+        We can define a ``sitemap_filter`` function to filter ``entries`` by date::
 
-            def sitemap_filter(urls):
+            def sitemap_filter(entries):
                 from datetime import datetime
-                for url in urls:
-                    date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')
+                for entry in entries:
+                    date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
                     if date_time.year >= 2005:
-                        yield url
+                        yield entry
 
-        This would retrieve only ``urls`` modified on 2005 and the following
+        This would retrieve only ``entries`` modified on 2005 and the following
         years.
 
-        If you omit this method, all urls found in sitemaps will be
+        If you omit this method, all entries found in sitemaps will be
         processed, observing other attributes and their settings.
 
 
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index c86e986db..534c45c70 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -31,13 +31,13 @@ class SitemapSpider(Spider):
         for url in self.sitemap_urls:
             yield Request(url, self._parse_sitemap)
 
-    def sitemap_filter(self, urls):
+    def sitemap_filter(self, entries):
         """This method can be used to filter sitemap entries by their
         attributes, for example, you can filter locs with lastmod greater
         than a given date (see docs).
         """
-        for url in urls:
-            yield url
+        for entry in entries:
+            yield entry
 
     def _parse_sitemap(self, response):
         if response.url.endswith('/robots.txt'):
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 871852ab2..d5d10c9ea 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -390,12 +390,12 @@ Sitemap: /sitemap-relative-url.xml
     </urlset>"""
 
         class FilteredSitemapSpider(self.spider_class):
-            def sitemap_filter(self, urls):
+            def sitemap_filter(self, entries):
                 from datetime import datetime
-                for url in urls:
-                    date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')
+                for entry in entries:
+                    date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
                     if date_time.year > 2008:
-                        yield url
+                        yield entry
 
         r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
         spider = self.spider_class("example.com")

From 5e7ecf9dc1954060fd0445dce5fb54e020dd3e59 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Fri, 21 Dec 2018 17:31:52 -0300
Subject: [PATCH 1865/2974] add tests for sitemapindex

---
 tests/test_spider.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/tests/test_spider.py b/tests/test_spider.py
index d5d10c9ea..8b56cfec1 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -407,6 +407,37 @@ Sitemap: /sitemap-relative-url.xml
         self.assertEqual([req.url for req in spider._parse_sitemap(r)],
                          ['http://www.example.com/english/'])
 
+    def test_sitemapindex_filter(self):
+        sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
+    <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        <sitemap>
+            <loc>http://www.example.com/sitemap1.xml</loc>
+            <lastmod>2004-01-01T20:00:00+00:00</lastmod>
+        </sitemap>
+        <sitemap>
+            <loc>http://www.example.com/sitemap2.xml</loc>
+            <lastmod>2005-01-01</lastmod>
+        </sitemap>
+    </sitemapindex>"""
+
+        class FilteredSitemapSpider(self.spider_class):
+            def sitemap_filter(self, entries):
+                from datetime import datetime
+                for entry in entries:
+                    date_time = datetime.strptime(entry['lastmod'].split('T')[0], '%Y-%m-%d')
+                    if date_time.year > 2004:
+                        yield entry
+
+        r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
+        spider = self.spider_class("example.com")
+        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
+                         ['http://www.example.com/sitemap1.xml',
+                          'http://www.example.com/sitemap2.xml'])
+
+        spider = FilteredSitemapSpider("example.com")
+        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
+                         ['http://www.example.com/sitemap2.xml'])
+
 
 class DeprecationTest(unittest.TestCase):
 

From 10f46bca54b2879da02641159e53453fe0cc97dc Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Wed, 26 Dec 2018 11:20:18 -0300
Subject: [PATCH 1866/2974] documenting sitemap entries as suggested by @kmike

---
 docs/topics/spiders.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 918f1cc36..9d4ed6ca6 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -703,6 +703,16 @@ SitemapSpider
         This would retrieve only ``entries`` modified on 2005 and the following
         years.
 
+        Entries are dict objects extracted from the sitemap document.
+        Usually, the key is the tag name and the value is the text inside it.
+
+        It's important to notice that:
+
+        - as the loc attribute is required, entries without this tag are discarded
+        - alternate links are stored in a list with the key ``alternate``
+          (see ``sitemap_alternate_links``)
+        - namespaces are removed, so lxml tags named as ``{foo}bar`` become only ``bar``
+
         If you omit this method, all entries found in sitemaps will be
         processed, observing other attributes and their settings.
 

From fe283bcd058734f88977a2033dfa36664e7ee619 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Wed, 26 Dec 2018 12:32:22 -0300
Subject: [PATCH 1867/2974] add test case for sitemap filter with alternate
 links

---
 tests/test_spider.py | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/tests/test_spider.py b/tests/test_spider.py
index 8b56cfec1..fefdaa403 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -407,6 +407,41 @@ Sitemap: /sitemap-relative-url.xml
         self.assertEqual([req.url for req in spider._parse_sitemap(r)],
                          ['http://www.example.com/english/'])
 
+    def test_sitemap_filter_with_alternate_links(self):
+        sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:xhtml="http://www.w3.org/1999/xhtml">
+        <url>
+            <loc>http://www.example.com/english/article_1/</loc>
+            <lastmod>2010-01-01</lastmod>
+            <xhtml:link rel="alternate" hreflang="de"
+                href="http://www.example.com/deutsch/article_1/"/>
+        </url>
+        <url>
+            <loc>http://www.example.com/english/article_2/</loc>
+            <lastmod>2015-01-01</lastmod>
+        </url>
+    </urlset>"""
+
+        class FilteredSitemapSpider(self.spider_class):
+            def sitemap_filter(self, entries):
+                for entry in entries:
+                    alternate_links = entry.get('alternate', tuple())
+                    for link in alternate_links:
+                        if '/deutsch/' in link:
+                            entry['loc'] = link
+                            yield entry
+
+        r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
+        spider = self.spider_class("example.com")
+        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
+                         ['http://www.example.com/english/article_1/',
+                          'http://www.example.com/english/article_2/'])
+
+        spider = FilteredSitemapSpider("example.com")
+        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
+                         ['http://www.example.com/deutsch/article_1/'])
+
     def test_sitemapindex_filter(self):
         sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
     <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">

From e1597f7c420ead9a563677aab61f18f9b89640a9 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Wed, 26 Dec 2018 15:05:21 -0300
Subject: [PATCH 1868/2974] improve readability

---
 docs/topics/spiders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 9d4ed6ca6..c47a2fca0 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -711,7 +711,7 @@ SitemapSpider
         - as the loc attribute is required, entries without this tag are discarded
         - alternate links are stored in a list with the key ``alternate``
           (see ``sitemap_alternate_links``)
-        - namespaces are removed, so lxml tags named as ``{foo}bar`` become only ``bar``
+        - namespaces are removed, so lxml tags named as ``{namespace}tagname`` become only ``tagname``
 
         If you omit this method, all entries found in sitemaps will be
         processed, observing other attributes and their settings.

From b68308779a6d2ce7deda3675d0bcdf671a4fb935 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Thu, 27 Dec 2018 17:37:59 -0300
Subject: [PATCH 1869/2974] improving docs

---
 docs/topics/spiders.rst | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index c47a2fca0..4f7135309 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -682,7 +682,8 @@ SitemapSpider
 
     .. method:: sitemap_filter(entries)
 
-        Specifies a function to filter sitemap entries and their attributes.
+        This is a filter funtion that could be overridden to select sitemap entries
+        based on their attributes.
 
         For example::
 
@@ -693,12 +694,17 @@ SitemapSpider
 
         We can define a ``sitemap_filter`` function to filter ``entries`` by date::
 
-            def sitemap_filter(entries):
-                from datetime import datetime
-                for entry in entries:
-                    date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
-                    if date_time.year >= 2005:
-                        yield entry
+            class FilteredSitemapSpider(scrapy.SitemapSpider):
+                name = 'filtered_sitemap_spider'
+                allowed_domains = ['example.com']
+                sitemap_urls = ['http://example.com/sitemap.xml']
+
+                def sitemap_filter(self, entries):
+                    from datetime import datetime
+                    for entry in entries:
+                        date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
+                        if date_time.year >= 2005:
+                            yield entry
 
         This would retrieve only ``entries`` modified on 2005 and the following
         years.

From bfbcf52e9df77af7a7c9a8a7a711e06612be4763 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Thu, 27 Dec 2018 18:12:31 -0300
Subject: [PATCH 1870/2974] fix SitemapSpider import

---
 docs/topics/spiders.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 4f7135309..39410d66e 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -694,7 +694,9 @@ SitemapSpider
 
         We can define a ``sitemap_filter`` function to filter ``entries`` by date::
 
-            class FilteredSitemapSpider(scrapy.SitemapSpider):
+            from scrapy.spiders.sitemap import SitemapSpider
+
+            class FilteredSitemapSpider(SitemapSpider):
                 name = 'filtered_sitemap_spider'
                 allowed_domains = ['example.com']
                 sitemap_urls = ['http://example.com/sitemap.xml']

From 5a824c906c501a204624ea7b4fb99904807c8b81 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Thu, 27 Dec 2018 18:34:41 -0300
Subject: [PATCH 1871/2974] using shorter import version and moving datetime
 import to the beginning of the code snippet

---
 docs/topics/spiders.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 39410d66e..742a88659 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -694,7 +694,8 @@ SitemapSpider
 
         We can define a ``sitemap_filter`` function to filter ``entries`` by date::
 
-            from scrapy.spiders.sitemap import SitemapSpider
+            from datetime import datetime
+            from scrapy.spiders import SitemapSpider
 
             class FilteredSitemapSpider(SitemapSpider):
                 name = 'filtered_sitemap_spider'
@@ -702,7 +703,6 @@ SitemapSpider
                 sitemap_urls = ['http://example.com/sitemap.xml']
 
                 def sitemap_filter(self, entries):
-                    from datetime import datetime
                     for entry in entries:
                         date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
                         if date_time.year >= 2005:

From e1f8b55ba0a132ed28c71661e2df3c5bc27feb75 Mon Sep 17 00:00:00 2001
From: Joaquin Garmendia Cabrera <joaquingc123@gmail.com>
Date: Fri, 28 Dec 2018 16:53:12 -0500
Subject: [PATCH 1872/2974] Improve syntax for readability

---
 docs/topics/item-pipeline.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 1c2c51e05..fae18200a 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -87,8 +87,8 @@ contain a price::
         vat_factor = 1.15
 
         def process_item(self, item, spider):
-            if 'price' in item and item['price']:
-                if 'price_excludes_vat' in item and item['price_excludes_vat']:
+            if item.get('price'):
+                if item.get('price_excludes_vat'):
                     item['price'] = item['price'] * self.vat_factor
                 return item
             else:

From 6c78b3d5ef94791b11c2ce3dfd5cebd757a68b2a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 3 Jan 2019 13:15:58 -0300
Subject: [PATCH 1873/2974] Deques can't be sliced, use itertools.islice
 instead

---
 scrapy/core/spidermw.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 58bd7c2c8..e07f76bdf 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,7 +3,7 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
-from itertools import chain
+from itertools import chain, islice
 
 import six
 from twisted.python.failure import Failure
@@ -58,7 +58,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             # don't handle _InvalidOutput exception
             if isinstance(exception, _InvalidOutput):
                 return _failure
-            method_list = self.methods['process_spider_exception'][start_index:]
+            method_list = islice(self.methods['process_spider_exception'], start_index, None)
             for method_index, method in enumerate(method_list, start=start_index):
                 if method is None:
                     continue
@@ -89,7 +89,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                         raise
                     recovered.extend(exception_result)
 
-            method_list = self.methods['process_spider_output'][start_index:]
+            method_list = islice(self.methods['process_spider_output'], start_index, None)
             for method_index, method in enumerate(method_list, start=start_index):
                 if method is None:
                     continue

From 50a0d87d1e472fcc514f3dc2b028b653b7826a9c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 3 Jan 2019 17:20:08 -0300
Subject: [PATCH 1874/2974] Passing keyword arguments to callbacks

---
 scrapy/core/scraper.py          | 2 +-
 scrapy/http/request/__init__.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index ee1e95a0c..7981ce231 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -143,7 +143,7 @@ class Scraper(object):
     def call_spider(self, result, request, spider):
         result.request = request
         dfd = defer_result(result)
-        dfd.addCallbacks(request.callback or spider.parse, request.errback)
+        dfd.addCallbacks(request.callback or spider.parse, request.errback, callbackKeywords=request.kwargs)
         return dfd.addCallback(iterate_spider_output)
 
     def handle_spider_error(self, _failure, request, response, spider):
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index cd4360483..7d5cc9dae 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -18,7 +18,7 @@ class Request(object_ref):
 
     def __init__(self, url, callback=None, method='GET', headers=None, body=None,
                  cookies=None, meta=None, encoding='utf-8', priority=0,
-                 dont_filter=False, errback=None, flags=None):
+                 dont_filter=False, errback=None, flags=None, kwargs=None):
 
         self._encoding = encoding  # this one has to be set first
         self.method = str(method).upper()
@@ -41,6 +41,7 @@ class Request(object_ref):
 
         self._meta = dict(meta) if meta else None
         self.flags = [] if flags is None else list(flags)
+        self.kwargs = dict(kwargs) if kwargs else None
 
     @property
     def meta(self):

From a2b509a42266a2ab3389de64b608e616f88f77e5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 3 Jan 2019 17:38:06 -0300
Subject: [PATCH 1875/2974] Pass callback kwargs with response.follow

---
 scrapy/http/response/__init__.py | 5 +++--
 scrapy/http/response/text.py     | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 1974259b5..99b04a26e 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -106,7 +106,7 @@ class Response(object_ref):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding='utf-8', priority=0,
-               dont_filter=False, errback=None):
+               dont_filter=False, errback=None, kwargs=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -132,4 +132,5 @@ class Response(object_ref):
                        encoding=encoding,
                        priority=priority,
                        dont_filter=dont_filter,
-                       errback=errback)
+                       errback=errback,
+                       kwargs=kwargs)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 74a042f2c..2039621b3 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -123,7 +123,7 @@ class TextResponse(Response):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding=None, priority=0,
-               dont_filter=False, errback=None):
+               dont_filter=False, errback=None, kwargs=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -154,7 +154,8 @@ class TextResponse(Response):
             encoding=encoding,
             priority=priority,
             dont_filter=dont_filter,
-            errback=errback
+            errback=errback,
+            kwargs=kwargs,
         )
 
 

From 69a1ee79aa43bb1444e5b2a800a2e1702db6866e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 3 Jan 2019 17:38:29 -0300
Subject: [PATCH 1876/2974] Copy request.kwargs

---
 scrapy/http/request/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 7d5cc9dae..9a155f415 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -93,7 +93,7 @@ class Request(object_ref):
         given new values.
         """
         for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',
-                  'encoding', 'priority', 'dont_filter', 'callback', 'errback']:
+                  'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'kwargs']:
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)

From a67f1ce512ffa111a400e2718ebbf3e7bd32d0ae Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 3 Jan 2019 17:49:41 -0300
Subject: [PATCH 1877/2974] Serialize Request kwargs

---
 scrapy/utils/reqser.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 959dddbd5..d537057b1 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -32,7 +32,8 @@ def request_to_dict(request, spider=None):
         '_encoding': request._encoding,
         'priority': request.priority,
         'dont_filter': request.dont_filter,
-        'flags': request.flags
+        'flags': request.flags,
+        'kwargs': request.kwargs,
     }
     if type(request) is not Request:
         d['_class'] = request.__module__ + '.' + request.__class__.__name__
@@ -64,7 +65,9 @@ def request_from_dict(d, spider=None):
         encoding=d['_encoding'],
         priority=d['priority'],
         dont_filter=d['dont_filter'],
-        flags=d.get('flags'))
+        flags=d.get('flags'),
+        kwargs=d.get('kwargs'),
+    )
 
 
 def _find_method(obj, func):

From 770a501fb32b7582acaa5900ac2f41ea46a321cd Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 9 Jan 2019 10:40:03 -0300
Subject: [PATCH 1878/2974] Test request kwargs (copy, serialization)

---
 scrapy/http/request/__init__.py | 8 +++++++-
 tests/test_http_request.py      | 5 +++++
 tests/test_utils_reqser.py      | 2 ++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 9a155f415..c016eb727 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -40,8 +40,14 @@ class Request(object_ref):
         self.dont_filter = dont_filter
 
         self._meta = dict(meta) if meta else None
+        self._kwargs = dict(kwargs) if kwargs else None
         self.flags = [] if flags is None else list(flags)
-        self.kwargs = dict(kwargs) if kwargs else None
+
+    @property
+    def kwargs(self):
+        if self._kwargs is None:
+            self._kwargs = {}
+        return self._kwargs
 
     @property
     def meta(self):
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 58326a384..610893d8a 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -177,6 +177,7 @@ class RequestTest(unittest.TestCase):
         r1 = self.request_class("http://www.example.com", flags=['f1', 'f2'],
                                 callback=somecallback, errback=somecallback)
         r1.meta['foo'] = 'bar'
+        r1.kwargs['key'] = 'value'
         r2 = r1.copy()
 
         # make sure copy does not propagate callbacks
@@ -189,6 +190,10 @@ class RequestTest(unittest.TestCase):
         assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
         self.assertEqual(r1.flags, r2.flags)
 
+        # make sure kwargs dict is shallow copied
+        assert r1.kwargs is not r2.kwargs, "kwargs must be a shallow copy, not identical"
+        self.assertEqual(r1.kwargs, r2.kwargs)
+
         # make sure meta dict is shallow copied
         assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical"
         self.assertEqual(r1.meta, r2.meta)
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index dcc070b8f..76de20f22 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -26,6 +26,7 @@ class RequestSerializationTest(unittest.TestCase):
             encoding='latin-1',
             priority=20,
             meta={'a': 'b'},
+            kwargs={'k': 'v'},
             flags=['testFlag'])
         self._assert_serializes_ok(r, spider=self.spider)
 
@@ -52,6 +53,7 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertEqual(r1.headers, r2.headers)
         self.assertEqual(r1.cookies, r2.cookies)
         self.assertEqual(r1.meta, r2.meta)
+        self.assertEqual(r1.kwargs, r2.kwargs)
         self.assertEqual(r1._encoding, r2._encoding)
         self.assertEqual(r1.priority, r2.priority)
         self.assertEqual(r1.dont_filter, r2.dont_filter)

From 57e7c769779b9d37058d1d8839215a9c269b8c5b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 9 Jan 2019 10:40:44 -0300
Subject: [PATCH 1879/2974] Test callback kwargs

---
 tests/spiders.py    | 34 ++++++++++++++++++++++++++++++++++
 tests/test_crawl.py |  8 +++++++-
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index 7816bf7c7..5a1471072 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -28,6 +28,40 @@ class MetaSpider(MockServerSpider):
         self.meta['close_reason'] = reason
 
 
+class KeywordArgumentsSpider(MockServerSpider):
+
+    name = 'kwargs'
+    checks = set()
+
+    def start_requests(self):
+        data = {'key': 'value', 'number': 123}
+        yield Request(self.mockserver.url('/first'), self.parse_first, kwargs=data)
+        yield Request(self.mockserver.url('/general_with'), self.parse_general, kwargs=data)
+        yield Request(self.mockserver.url('/general_without'), self.parse_general)
+        yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
+
+    def parse_first(self, response, key, number):
+        self.checks.add(key == 'value')
+        self.checks.add(number == 123)
+        yield response.follow(
+            self.mockserver.url('/two'),
+            self.parse_second,
+            kwargs={'new_key': 'new_value'})
+
+    def parse_second(self, response, new_key):
+        self.checks.add(new_key == 'new_value')
+
+    def parse_general(self, response, **kwargs):
+        if response.url.endswith('/general_with'):
+            self.checks.add(kwargs['key'] == 'value')
+            self.checks.add(kwargs['number'] == 123)
+        elif response.url.endswith('/general_without'):
+            self.checks.add(kwargs == {})
+
+    def parse_no_kwargs(self, response):
+        pass
+
+
 class FollowAllSpider(MetaSpider):
 
     name = 'follow'
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3fc13eeb7..3879a017c 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -8,7 +8,7 @@ from twisted.trial.unittest import TestCase
 from scrapy.http import Request
 from scrapy.crawler import CrawlerRunner
 from scrapy.utils.python import to_unicode
-from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
+from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, KeywordArgumentsSpider, \
     BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
 from tests.mockserver import MockServer
 
@@ -23,6 +23,12 @@ class CrawlTestCase(TestCase):
     def tearDown(self):
         self.mockserver.__exit__(None, None, None)
 
+    @defer.inlineCallbacks
+    def test_callback_kwargs(self):
+        crawler = self.runner.create_crawler(KeywordArgumentsSpider)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertEqual(crawler.spider.checks, set([True]))
+
     @defer.inlineCallbacks
     def test_follow_all(self):
         crawler = self.runner.create_crawler(FollowAllSpider)

From 3f914f6d8c369a18e1f856c01b7d1ad2a63f6e49 Mon Sep 17 00:00:00 2001
From: kasun Herath <kasunh01@gmail.com>
Date: Mon, 14 Jan 2019 23:03:14 +0530
Subject: [PATCH 1880/2974] made jsonrequest dump into private method

---
 scrapy/http/request/json_request.py | 15 +++++++++------
 tests/test_http_request.py          |  2 +-
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 7499610b9..1e2c6b0c6 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -5,6 +5,7 @@ This module implements the JSONRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 
+import copy
 import json
 import warnings
 
@@ -13,7 +14,10 @@ from scrapy.http.request import Request
 
 class JSONRequest(Request):
     def __init__(self, *args, **kwargs):
-        dumps_kwargs = kwargs.pop('dumps_kwargs', {})
+        dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {}))
+        dumps_kwargs['sort_keys'] = True
+        self._dumps_kwargs = dumps_kwargs
+
         body_passed = kwargs.get('body', None) is not None
         data = kwargs.pop('data', None)
         data_passed = data is not None
@@ -22,7 +26,7 @@ class JSONRequest(Request):
             warnings.warn('Both body and data passed. data will be ignored')
 
         elif not body_passed and data_passed:
-            kwargs['body'] = self.dump(data, **dumps_kwargs)
+            kwargs['body'] = self._dumps(data)
 
             if 'method' not in kwargs:
                 kwargs['method'] = 'POST'
@@ -30,7 +34,6 @@ class JSONRequest(Request):
         super(JSONRequest, self).__init__(*args, **kwargs)
         self.headers.setdefault('Content-Type', 'application/json')
         self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
-        self._dumps_kwargs = dumps_kwargs
 
     def replace(self, *args, **kwargs):
         body_passed = kwargs.get('body', None) is not None
@@ -41,10 +44,10 @@ class JSONRequest(Request):
             warnings.warn('Both body and data passed. data will be ignored')
 
         elif not body_passed and data_passed:
-            kwargs['body'] = self.dump(data, **self._dumps_kwargs)
+            kwargs['body'] = self._dumps(data)
 
         return super(JSONRequest, self).replace(*args, **kwargs)
 
-    def dump(self, data, **kwargs):
+    def _dumps(self, data):
         """Convert to JSON """
-        return json.dumps(data, sort_keys=True, **kwargs)
+        return json.dumps(data, **self._dumps_kwargs)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 6dcfa25da..49f148016 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1292,7 +1292,7 @@ class JSONRequestTest(RequestTest):
             self.assertEqual(kwargs['sort_keys'], True)
 
     def test_replace_dumps_kwargs(self):
-        """ Test that dumps_kwargs are provided json.dumps when replace is called """
+        """ Test that dumps_kwargs are provided to json.dumps when replace is called """
         data1 = {
             'name1': 'value1',
         }

From bddfeaba4c17040b2986403f8b2ba25d4252e1b5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 15 Jan 2019 15:35:46 -0300
Subject: [PATCH 1881/2974] Add Request.kwargs docs

---
 docs/topics/request-response.rst | 40 +++++++++++++++++++++++++++-----
 1 file changed, 34 insertions(+), 6 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index e29914dbf..d12766676 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
 Request objects
 ===============
 
-.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags])
+.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, kwargs])
 
     A :class:`Request` object represents an HTTP request, which is usually
     generated in the Spider and executed by the Downloader, and thus generating
@@ -126,6 +126,9 @@ Request objects
     :param flags:  Flags sent to the request, can be used for logging or similar purposes.
     :type flags: list
 
+    :param kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback.
+    :type kwargs: dict
+
     .. attribute:: Request.url
 
         A string containing the URL of this request. Keep in mind that this
@@ -165,6 +168,17 @@ Request objects
         ``copy()`` or ``replace()`` methods, and can also be accessed, in your
         spider, from the ``response.meta`` attribute.
 
+    .. attribute:: Request.kwargs
+
+        A dictionary that contains arbitrary metadata for this request. Its contents
+        will be passed to the Request's callback as keyword arguments. It is empty
+        for new Requests, which means by default callbacks only get a :class:`Response`
+        object as argument.
+
+        This dict is `shallow copied`_ when the request is cloned using the
+        ``copy()`` or ``replace()`` methods, and can also be accessed, in your
+        spider, from the ``response.kwargs`` attribute.
+
     .. _shallow copied: https://docs.python.org/2/library/copy.html
 
     .. method:: Request.copy()
@@ -200,11 +214,9 @@ Example::
         self.logger.info("Visited %s", response.url)
 
 In some cases you may be interested in passing arguments to those callback
-functions so you can receive the arguments later, in the second callback. You
-can use the :attr:`Request.meta` attribute for that.
-
-Here's an example of how to pass an item using this mechanism, to populate
-different fields from different pages::
+functions so you can receive the arguments later, in the second callback.
+The following two examples show how to achieve this by using the 
+:attr:`Request.meta` and :attr:`Request.kwargs` attributes respectively::
 
     def parse_page1(self, response):
         item = MyItem()
@@ -219,6 +231,22 @@ different fields from different pages::
         item['other_url'] = response.url
         yield item
 
+::
+
+    def parse_page1(self, response):
+        item = MyItem()
+        item['main_url'] = response.url
+        request = scrapy.Request("http://www.example.com/some_page.html",
+                                 callback=self.parse_page2)
+        request.kwargs['item'] = item
+        request.kwargs['foo'] = 'bar'
+        yield request
+
+    def parse_page2(self, response, item, foo):
+        item['other_url'] = response.url
+        item['foo'] = foo
+        yield item
+
 
 .. _topics-request-response-ref-errbacks:
 

From bdf12f775062fda8aa8bf03f7b4faade4faac16d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Fri, 18 Jan 2019 11:38:59 -0200
Subject: [PATCH 1882/2974] Logging the request referer when DUPEFILTER_DEBUG
 is active

---
 scrapy/dupefilters.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 9d8966b9c..0bcdd3495 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -3,8 +3,7 @@ import os
 import logging
 
 from scrapy.utils.job import job_dir
-from scrapy.utils.request import request_fingerprint
-
+from scrapy.utils.request import referer_str, request_fingerprint
 
 class BaseDupeFilter(object):
 
@@ -61,8 +60,9 @@ class RFPDupeFilter(BaseDupeFilter):
 
     def log(self, request, spider):
         if self.debug:
-            msg = "Filtered duplicate request: %(request)s"
-            self.logger.debug(msg, {'request': request}, extra={'spider': spider})
+            msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)"
+            args = {'request': request, 'referer': referer_str(request) }
+            self.logger.debug(msg, args, extra={'spider': spider})
         elif self.logdupes:
             msg = ("Filtered duplicate request: %(request)s"
                    " - no more duplicates will be shown"

From 8eade7d8640e112faf8677f4666bbe3ab10c7234 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Fri, 18 Jan 2019 11:39:35 -0200
Subject: [PATCH 1883/2974] Testing stats and log messages from RFPDupeFilter

---
 tests/test_dupefilters.py | 57 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 56 insertions(+), 1 deletion(-)

diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index db69597a2..d7eb98c97 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -2,6 +2,7 @@ import hashlib
 import tempfile
 import unittest
 import shutil
+from testfixtures import LogCapture
 
 from scrapy.dupefilters import RFPDupeFilter
 from scrapy.http import Request
@@ -9,7 +10,7 @@ from scrapy.core.scheduler import Scheduler
 from scrapy.utils.python import to_bytes
 from scrapy.utils.job import job_dir
 from scrapy.utils.test import get_crawler
-
+from tests.spiders import SimpleSpider
 
 class FromCrawlerRFPDupeFilter(RFPDupeFilter):
 
@@ -126,3 +127,57 @@ class RFPDupeFilterTest(unittest.TestCase):
         assert case_insensitive_dupefilter.request_seen(r2)
 
         case_insensitive_dupefilter.close('finished')
+
+    def test_log(self):
+        with LogCapture() as l:
+            settings = {'DUPEFILTER_DEBUG': False,
+                        'DUPEFILTER_CLASS': __name__  + '.FromCrawlerRFPDupeFilter'}
+            crawler = get_crawler(SimpleSpider, settings_dict=settings)
+            scheduler = Scheduler.from_crawler(crawler)
+            spider = SimpleSpider.from_crawler(crawler)
+
+            dupefilter = scheduler.df
+            dupefilter.open()
+
+            r1 = Request('http://scrapytest.org/index.html')
+            r2 = Request('http://scrapytest.org/index.html')
+            
+            dupefilter.log(r1, spider)
+            dupefilter.log(r2, spider)
+
+            assert crawler.stats.get_value('dupefilter/filtered') == 2
+            l.check_present(('scrapy.dupefilters', 'DEBUG', 
+                ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
+                ' - no more duplicates will be shown'
+                ' (see DUPEFILTER_DEBUG to show all duplicates)')))
+
+            dupefilter.close('finished')
+
+    def test_log_debug(self):
+        with LogCapture() as l:
+            settings = {'DUPEFILTER_DEBUG': True,
+                        'DUPEFILTER_CLASS': __name__  + '.FromCrawlerRFPDupeFilter'}
+            crawler = get_crawler(SimpleSpider, settings_dict=settings)
+            scheduler = Scheduler.from_crawler(crawler)
+            spider = SimpleSpider.from_crawler(crawler)
+
+            dupefilter = scheduler.df
+            dupefilter.open()
+
+            r1 = Request('http://scrapytest.org/index.html')
+            r2 = Request('http://scrapytest.org/index.html',
+                headers={'Referer': 'http://scrapytest.org/INDEX.html'}
+            )
+            
+            dupefilter.log(r1, spider)
+            dupefilter.log(r2, spider)
+
+            assert crawler.stats.get_value('dupefilter/filtered') == 2
+            l.check_present(('scrapy.dupefilters', 'DEBUG',
+                ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
+                ' (referer: None)')))
+            l.check_present(('scrapy.dupefilters', 'DEBUG',
+                ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
+                ' (referer: http://scrapytest.org/INDEX.html)')))
+
+            dupefilter.close('finished')

From 71743a6546e96b5d99bd3c068a7ec5b71dca1659 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Sat, 19 Jan 2019 18:43:58 +0000
Subject: [PATCH 1884/2974] Add release notes for v1.5.2

---
 docs/news.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 01016e2e6..adf679ded 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -12,6 +12,22 @@ Cleanups
 * Remove deprecated ``CrawlerSettings`` class.
 * Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes.
 
+Scrapy 1.5.2 (2019-01-22)
+-------------------------
+
+* *Security bugfix*: Telnet console extension can be easily exploited by rogue
+  websites POSTing content to http://localhost:6023, we haven't found a way to
+  exploit it from Scrapy, but it is very easy to trick a browser to do so and
+  elevates the risk for local development environment.
+
+  *The fix is backwards incompatible*, it enables telnet user-password
+  authentication by default with a random generated password. If you can't
+  upgrade right away, please consider setting :setting:`TELNET_CONSOLE_PORT`
+  out of its default value.
+
+  See :ref:`telnet console <topics-telnetconsole>` documentation for more info
+
+* Backport CI build failure under GCE environemnt due to boto import error.
 
 Scrapy 1.5.1 (2018-07-12)
 -------------------------

From d9aa5391327dd34f8d840e7ce2bca1eb8583d932 Mon Sep 17 00:00:00 2001
From: kasun Herath <kasunh01@gmail.com>
Date: Fri, 25 Jan 2019 21:26:28 +0530
Subject: [PATCH 1885/2974] enabled sort keys only if not provided

---
 scrapy/http/request/json_request.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 1e2c6b0c6..8f7a61a6d 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -15,7 +15,7 @@ from scrapy.http.request import Request
 class JSONRequest(Request):
     def __init__(self, *args, **kwargs):
         dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {}))
-        dumps_kwargs['sort_keys'] = True
+        dumps_kwargs.setdefault('sort_keys', True)
         self._dumps_kwargs = dumps_kwargs
 
         body_passed = kwargs.get('body', None) is not None

From b828b5f8c8650a30aef382af661b6c7b9ea57186 Mon Sep 17 00:00:00 2001
From: Harry Moreno <morenoh149@gmail.com>
Date: Sat, 26 Jan 2019 18:39:05 -0500
Subject: [PATCH 1886/2974] fix grammar

---
 docs/topics/jobs.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 8e1574376..ea684b4cf 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -30,7 +30,7 @@ a *single* job.
 How to use it
 =============
 
-To start a spider with persistence supported enabled, run it like this::
+To start a spider with persistence support enabled, run it like this::
 
     scrapy crawl somespider -s JOBDIR=crawls/somespider-1
 

From 8fca98616a90d8452ff2e488bfea93e5c89caf08 Mon Sep 17 00:00:00 2001
From: Harry Moreno <morenoh149@gmail.com>
Date: Sat, 26 Jan 2019 16:47:10 -0500
Subject: [PATCH 1887/2974] fix grammar

---
 docs/topics/media-pipeline.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index a1f518cbd..c60b55391 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -310,7 +310,7 @@ images.
 
 .. setting:: IMAGES_THUMBS
 
-In order use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary
+In order to use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary
 where the keys are the thumbnail names and the values are their dimensions.
 
 For example::

From e3e804cfb0fc05ef3fc569ec6e0af247ce504d06 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 28 Jan 2019 15:10:34 -0300
Subject: [PATCH 1888/2974] Styling nitpick :-)

---
 scrapy/core/downloader/middleware.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index a8e6f93a3..7a6a4dfac 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -41,7 +41,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                                          (six.get_method_self(method).__class__.__name__, response.__class__.__name__))
                 if response:
                     defer.returnValue(response)
-            defer.returnValue((yield download_func(request=request,spider=spider)))
+            defer.returnValue((yield download_func(request=request, spider=spider)))
 
         @defer.inlineCallbacks
         def process_response(response):

From 706910790b6ee755bafa828606e215e668af3eee Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 26 Dec 2018 18:28:24 +0500
Subject: [PATCH 1889/2974] [wip] draft 1.6 release notes

---
 docs/news.rst | 153 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 150 insertions(+), 3 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index adf679ded..99a339cea 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -6,11 +6,156 @@ Release notes
 Scrapy 1.6.0 (unreleased)
 -------------------------
 
+Highlights for this release:
+
+* better Windows compatibility;
+* Python 3.7 compatibility;
+* big documentation improvements, including a switch
+  from ``.extract() / .extract_first()`` API to ``.get() / .getall()`` API;
+* Feed exports, FilePipeline and MediaPipeline improvements;
+* ``scrapy.contracts`` fixes and new features;
+* large clean-up of deprecated code
+* TODO
+
+parsel 1.5
+~~~~~~~~~~
+
+TODO
+While this is not a change in Scrapy itself, a new version of ``parsel``
+is released; Scrapy now depends on ``parsel >= 1.5``.
+
+Feed export improvements
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+* ``from_crawler`` support is added to feed exporters and feed storages. This,
+  among other things, allow to access Scrapy settings from custom storages
+  and exporters (:issue:`1605`, :issue:`3348`).
+* fixed issue with extra blank lines in .csv exports under Windows
+  (:issue:`3039`);
+* better error message when an exporter is disabled (:issue:`3358`);
+
+FilePipeline and MediaPipeline improvements
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Expose more options for S3FilesStore: :setting:`AWS_ENDPOINT_URL`,
+  :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY`, :setting:`AWS_REGION_NAME`.
+  For example, this allows to use alternative or self-hosted
+  AWS-compatible providers (:issue:`2609`).
+* ACL support for Google Cloud Storage: :setting:`FILES_STORE_GCS_ACL` and
+  :setting:`IMAGES_STORE_GCS_ACL` (:issue:`3199`).
+
+``scrapy.contracts`` improvements
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Exceptions in contracts code are handled better (:issue:`3377`);
+* ``dont_filter=True`` is used for contract requests, which allows to test
+  different callbacks with the same URL (:issue:`3381`);
+* ``request_cls`` attribute in Contract subclasses allow to use different
+  Request classes in contracts, for example FormRequest (:issue:`3383`).
+* Fixed errback handling in contracts, e.g. for cases where a contract
+  is executed for URL which returns non-200 response (:issue:`3371`).
+
+Documentation improvements
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Docs are re-written to suggest .get/.getall API instead of
+  .extract/.extract_first. Also, :ref:`topics-selectors` docs are updated
+  and re-structured to match latest parsel docs; they now contain more topics,
+  such as :ref:`selecting-attributes` or :ref:`topics-selectors-css-extensions`
+  (:issue:`3390`).
+* :ref:`topics-developer-tools` is a new tutorial which replaces
+  old Firefox and Firebug tutorials (:issue:`3400`).
+* SCRAPY_PROJECT environment variable is documented (:issue:`3518`);
+* troubleshooting section is added to install instructions (:issue:`3517`);
+* improved links to beginner resources in the tutorial
+  (:issue:`3367`, :issue:`3468`);
+* fixed :setting:`RETRY_HTTP_CODES` default values in docs (:issue:`3335`);
+* remove unused `DEPTH_STATS` option from docs (:issue:`3245`);
+* other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`).
+
+Better Windows support
+~~~~~~~~~~~~~~~~~~~~~~
+
+* All Scrapy tests now pass on Windows; Scrapy testing suite is executed
+  in a Windows environment on CI (:issue:`3315`).
+* Scrapy used to produce unnecessary blank lines in .csv exports on Windows,
+  this is fixed (:issue:`3039`).
+
+Testing fixes
+~~~~~~~~~~~~~
+
+* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`)
+* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`,
+  :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`)
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+Compatibility shims for pre-1.0 Scrapy module names are removed
+(:issue:`3318`):
+
+* ``scrapy.command``
+* ``scrapy.contrib`` (with all submodules)
+* ``scrapy.contrib_exp`` (with all submodules)
+* ``scrapy.dupefilter``
+* ``scrapy.linkextractor``
+* ``scrapy.project``
+* ``scrapy.spider``
+* ``scrapy.spidermanager``
+* ``scrapy.squeue``
+* ``scrapy.stats``
+* ``scrapy.statscol``
+* ``scrapy.utils.decorator``
+
+See :ref:`module_relocations` for more information, or use suggestions
+from Scrapy 1.5.x deprecation warnings to update your code.
+
+Other deprecation removals:
+
+* Deprecated scrapy.interfaces.ISpiderManager is removed; please use
+  scrapy.interfaces.ISpiderLoader.
+* Deprecated ``CrawlerSettings`` class is removed (:issue:`3327`).
+* Deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes
+  are removed (:issue:`3327`, :issue:`3359`).
+
+Internal improvements
+~~~~~~~~~~~~~~~~~~~~~
+
+* ``from_crawler`` support is added to dupefilters (:issue:`2956`); this allows
+  to access e.g. settings or a spider from a dupefilter.
+* :signal:`item_error` is fired when an error happens in a pipeline
+  (:issue:`3256`);
+* :signal:`request_reached_downloader` is fired when Downloader gets
+  a new Request; this signal can be useful e.g. for custom Schedulers
+  (:issue:`3393`).
+* ``scrapy.http.cookies.CookieJar.clear`` accepts "domain", "path" and "name"
+  optional arguments (:issue:`3231`).
+
+Usability improvements
+~~~~~~~~~~~~~~~~~~~~~~
+
+* more stats for RobotsTxtMiddleware (:issue:`3100`)
+* INFO log level is used to show telnet host/port (:issue:`3115`)
+* a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`)
+* better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`)
+* non-zero exit code is returned from Scrapy commands when error happens
+  on spider inititalization (:issue:`3226`)
+* Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`);
+  "flv" is added to common video extensions (:issue:`3165`)
+
+Bug fixes
+~~~~~~~~~
+* proper handling of pickling errors in Python 3 when serializing objects
+  for disk queues (:issue:`3082`)
+* flags are now preserved when copying Requests (:issue:`3342`);
+* FormRequest.from_response clickdata shouldn't ignore elements with
+  ``input[type=image]`` (:issue:`3153`).
+* FormRequest.from_response should preserve duplicate keys (:issue:`3247`)
+
 Cleanups
 ~~~~~~~~
-
-* Remove deprecated ``CrawlerSettings`` class.
-* Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes.
+* additional files are included to sdist (:issue:`3495`);
+* code style fixes (:issue:`3405`, :issue:`3304`)
 
 Scrapy 1.5.2 (2019-01-22)
 -------------------------
@@ -1080,6 +1225,8 @@ until it reaches a stable status.
 
 See more examples for scripts running Scrapy: :ref:`topics-practices`
 
+.. _module_relocations:
+
 Module Relocations
 ~~~~~~~~~~~~~~~~~~
 

From e479f5aa15809e7f75a7dbc20d0629f57be46b5d Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 27 Dec 2018 00:48:10 +0500
Subject: [PATCH 1890/2974] DOC update changelog

* changes from recently merged pull requests
* more highlights
* re-organized headers
* Selector API changes
---
 docs/news.rst | 142 +++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 105 insertions(+), 37 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 99a339cea..bf469a350 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -6,41 +6,83 @@ Release notes
 Scrapy 1.6.0 (unreleased)
 -------------------------
 
-Highlights for this release:
+Highlights:
 
-* better Windows compatibility;
+* better Windows support;
 * Python 3.7 compatibility;
 * big documentation improvements, including a switch
-  from ``.extract() / .extract_first()`` API to ``.get() / .getall()`` API;
-* Feed exports, FilePipeline and MediaPipeline improvements;
+  from ``.extract()`` + ``.extract_first()`` API to ``.get()`` + ``.getall()``
+  API;
+* feed exports, FilePipeline and MediaPipeline improvements;
+* better extensibility: :signal:`item_error` and
+  :signal:`request_reached_downloader` signals; ``from_crawler`` support
+  for feed exporters, feed storages and dupefilters.
 * ``scrapy.contracts`` fixes and new features;
-* large clean-up of deprecated code
-* TODO
+* telnet console security improvements;
+* clean-up of the deprecated code;
+* various bug fixes, small new features and usability improvements across
+  the codebase.
 
-parsel 1.5
-~~~~~~~~~~
+Selector API changes
+~~~~~~~~~~~~~~~~~~~~
 
-TODO
-While this is not a change in Scrapy itself, a new version of ``parsel``
-is released; Scrapy now depends on ``parsel >= 1.5``.
+While these are not changes in Scrapy itself, but rather in the parsel_
+library which Scrapy uses for xpath/css selectors, these changes are
+worth mentioning here. Scrapy now depends on parsel >= 1.5, and
+Scrapy documentation is updated to follow recent ``parsel`` API conventions.
 
-Feed export improvements
-~~~~~~~~~~~~~~~~~~~~~~~~
+Most visible change is that ``.get()`` and ``.getall()`` selector
+methods are now preferred over ``.extract()`` and ``.extract_first()``.
+We feel that these new methods result in a more concise and readable code.
+See :ref:`old-extraction-api` for more details.
+
+.. note::
+    There are currently **no plans** to deprecate ``.extract()``
+    and ``.extract_first()`` methods.
+
+Another useful new feature is the introduction of ``Selector.attrib`` and
+``SelectorList.attrib`` properties, which make it easier to get
+attributes of HTML elements. See :ref:`selecting-attributes`.
+
+CSS selectors are cached in parsel >= 1.5, which makes them faster
+when the same CSS path is used many times. This is very common in
+case of Scrapy spiders: callbacks are usually called several times,
+on different pages.
+
+If you're using custom ``Selector`` or ``SelectorList`` subclasses,
+a **backwards incompatible** change in parsel may affect your code.
+See `parsel changelog`_ for a detailed description, as well as for the
+full list of improvements.
+
+.. _parsel changelog: https://parsel.readthedocs.io/en/latest/history.html
+
+Telnet console
+~~~~~~~~~~~~~~
+
+**Backwards incompatible**: Scrapy's telnet console now requires username
+and password. See :ref:`topics-telnetconsole` for more details.
+
+New extensibility features
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 * ``from_crawler`` support is added to feed exporters and feed storages. This,
-  among other things, allow to access Scrapy settings from custom storages
-  and exporters (:issue:`1605`, :issue:`3348`).
-* fixed issue with extra blank lines in .csv exports under Windows
-  (:issue:`3039`);
-* better error message when an exporter is disabled (:issue:`3358`);
+  among other things, allows to access Scrapy settings from custom feed
+  storages and exporters (:issue:`1605`, :issue:`3348`).
+* ``from_crawler`` support is added to dupefilters (:issue:`2956`); this allows
+  to access e.g. settings or a spider from a dupefilter.
+* :signal:`item_error` is fired when an error happens in a pipeline
+  (:issue:`3256`);
+* :signal:`request_reached_downloader` is fired when Downloader gets
+  a new Request; this signal can be useful e.g. for custom Schedulers
+  (:issue:`3393`).
 
-FilePipeline and MediaPipeline improvements
+New FilePipeline and MediaPipeline features
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 * Expose more options for S3FilesStore: :setting:`AWS_ENDPOINT_URL`,
   :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY`, :setting:`AWS_REGION_NAME`.
   For example, this allows to use alternative or self-hosted
-  AWS-compatible providers (:issue:`2609`).
+  AWS-compatible providers (:issue:`2609`, :issue:`3548`).
 * ACL support for Google Cloud Storage: :setting:`FILES_STORE_GCS_ACL` and
   :setting:`IMAGES_STORE_GCS_ACL` (:issue:`3199`).
 
@@ -55,6 +97,47 @@ FilePipeline and MediaPipeline improvements
 * Fixed errback handling in contracts, e.g. for cases where a contract
   is executed for URL which returns non-200 response (:issue:`3371`).
 
+Usability and other improvements, cleanups
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* All Scrapy tests now pass on Windows; Scrapy testing suite is executed
+  in a Windows environment on CI (:issue:`3315`).
+* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`).
+* Lazy loading of Downloader Handlers is now optional; this enables better
+  initialization error handling in custom Downloader Handlers (:issue:`3394`).
+* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`,
+  :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`)
+* better error message when an exporter is disabled (:issue:`3358`);
+* ``scrapy.http.cookies.CookieJar.clear`` accepts "domain", "path" and "name"
+  optional arguments (:issue:`3231`).
+* more stats for RobotsTxtMiddleware (:issue:`3100`)
+* INFO log level is used to show telnet host/port (:issue:`3115`)
+* a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`)
+* better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`)
+* non-zero exit code is returned from Scrapy commands when error happens
+  on spider inititalization (:issue:`3226`);
+* link extraction improvements: "ftp" is added to scheme list (:issue:`3152`);
+  "flv" is added to common video extensions (:issue:`3165`)
+* `scrapy shell --help` mentions syntax required for local files
+  (``./file.html``) - :issue:`3496`.
+* additional files are included to sdist (:issue:`3495`);
+* code style fixes (:issue:`3405`, :issue:`3304`);
+* unneeded .strip() call is removed (:issue:`3519`);
+* collections.deque is used to store MiddlewareManager methods instead
+  of a list (:issue:`3476`)
+
+Bug fixes
+~~~~~~~~~
+
+* fixed issue with extra blank lines in .csv exports under Windows
+  (:issue:`3039`);
+* proper handling of pickling errors in Python 3 when serializing objects
+  for disk queues (:issue:`3082`)
+* flags are now preserved when copying Requests (:issue:`3342`);
+* FormRequest.from_response clickdata shouldn't ignore elements with
+  ``input[type=image]`` (:issue:`3153`).
+* FormRequest.from_response should preserve duplicate keys (:issue:`3247`)
+
 Documentation improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -73,21 +156,6 @@ Documentation improvements
 * remove unused `DEPTH_STATS` option from docs (:issue:`3245`);
 * other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`).
 
-Better Windows support
-~~~~~~~~~~~~~~~~~~~~~~
-
-* All Scrapy tests now pass on Windows; Scrapy testing suite is executed
-  in a Windows environment on CI (:issue:`3315`).
-* Scrapy used to produce unnecessary blank lines in .csv exports on Windows,
-  this is fixed (:issue:`3039`).
-
-Testing fixes
-~~~~~~~~~~~~~
-
-* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`)
-* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`,
-  :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`)
-
 Deprecation removals
 ~~~~~~~~~~~~~~~~~~~~
 
@@ -107,7 +175,7 @@ Compatibility shims for pre-1.0 Scrapy module names are removed
 * ``scrapy.statscol``
 * ``scrapy.utils.decorator``
 
-See :ref:`module_relocations` for more information, or use suggestions
+See :ref:`module-relocations` for more information, or use suggestions
 from Scrapy 1.5.x deprecation warnings to update your code.
 
 Other deprecation removals:
@@ -1225,7 +1293,7 @@ until it reaches a stable status.
 
 See more examples for scripts running Scrapy: :ref:`topics-practices`
 
-.. _module_relocations:
+.. _module-relocations:
 
 Module Relocations
 ~~~~~~~~~~~~~~~~~~

From 638469f9efdcc104f7b1a1c1a9890694e0d41c68 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 28 Dec 2018 01:13:01 +0500
Subject: [PATCH 1891/2974] DOC extract_first/extract matches get/getall better

Thanks @Gallaecio!
---
 docs/news.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index bf469a350..4a236f1b9 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -11,7 +11,7 @@ Highlights:
 * better Windows support;
 * Python 3.7 compatibility;
 * big documentation improvements, including a switch
-  from ``.extract()`` + ``.extract_first()`` API to ``.get()`` + ``.getall()``
+  from ``.extract_first()`` + ``.extract()`` API to ``.get()`` + ``.getall()``
   API;
 * feed exports, FilePipeline and MediaPipeline improvements;
 * better extensibility: :signal:`item_error` and
@@ -32,7 +32,7 @@ worth mentioning here. Scrapy now depends on parsel >= 1.5, and
 Scrapy documentation is updated to follow recent ``parsel`` API conventions.
 
 Most visible change is that ``.get()`` and ``.getall()`` selector
-methods are now preferred over ``.extract()`` and ``.extract_first()``.
+methods are now preferred over ``.extract_first()`` and ``.extract()``.
 We feel that these new methods result in a more concise and readable code.
 See :ref:`old-extraction-api` for more details.
 

From 4cf4dd1d3e068e0df32f700c89d833cc7cd79b85 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 30 Jan 2019 03:08:17 +0500
Subject: [PATCH 1892/2974] DOC add recent changes to changelog

---
 docs/news.rst | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 4a236f1b9..1a08f93ec 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -75,6 +75,9 @@ New extensibility features
 * :signal:`request_reached_downloader` is fired when Downloader gets
   a new Request; this signal can be useful e.g. for custom Schedulers
   (:issue:`3393`).
+* new SitemapSpider :meth:`~.SitemapSpider.sitemap_filter` method which allows
+  to select sitemap entries based on their attributes in SitemapSpider
+  subclasses (:issue:`3512`).
 
 New FilePipeline and MediaPipeline features
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -125,6 +128,7 @@ Usability and other improvements, cleanups
 * unneeded .strip() call is removed (:issue:`3519`);
 * collections.deque is used to store MiddlewareManager methods instead
   of a list (:issue:`3476`)
+* Referer header value is added to RFPDupeFilter log messages (:issue:`3588`)
 
 Bug fixes
 ~~~~~~~~~
@@ -154,7 +158,8 @@ Documentation improvements
   (:issue:`3367`, :issue:`3468`);
 * fixed :setting:`RETRY_HTTP_CODES` default values in docs (:issue:`3335`);
 * remove unused `DEPTH_STATS` option from docs (:issue:`3245`);
-* other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`).
+* other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`, :issue:`3544`,
+  :issue:`3605`).
 
 Deprecation removals
 ~~~~~~~~~~~~~~~~~~~~

From 0fc9d705c271f5d87174143c09f95993e5a45797 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 30 Jan 2019 03:28:19 +0500
Subject: [PATCH 1893/2974] DOC mention that telnet security improvements
 happened in 1.5.2

---
 docs/news.rst | 45 ++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 1a08f93ec..a4f07efad 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,8 @@
 Release notes
 =============
 
+.. _release-1.6.0:
+
 Scrapy 1.6.0 (unreleased)
 -------------------------
 
@@ -18,11 +20,13 @@ Highlights:
   :signal:`request_reached_downloader` signals; ``from_crawler`` support
   for feed exporters, feed storages and dupefilters.
 * ``scrapy.contracts`` fixes and new features;
-* telnet console security improvements;
+* telnet console security improvements, first released as a
+  backport in :ref:`release-1.5.2`;
 * clean-up of the deprecated code;
 * various bug fixes, small new features and usability improvements across
   the codebase.
 
+
 Selector API changes
 ~~~~~~~~~~~~~~~~~~~~
 
@@ -230,6 +234,8 @@ Cleanups
 * additional files are included to sdist (:issue:`3495`);
 * code style fixes (:issue:`3405`, :issue:`3304`)
 
+.. _release-1.5.2:
+
 Scrapy 1.5.2 (2019-01-22)
 -------------------------
 
@@ -247,6 +253,8 @@ Scrapy 1.5.2 (2019-01-22)
 
 * Backport CI build failure under GCE environemnt due to boto import error.
 
+.. _release-1.5.1:
+
 Scrapy 1.5.1 (2018-07-12)
 -------------------------
 
@@ -262,6 +270,9 @@ This is a maintenance release with important bug fixes, but no new features:
   :issue:`3279`, :issue:`3201`, :issue:`3260`, :issue:`3284`, :issue:`3298`,
   :issue:`3294`).
 
+
+.. _release-1.5.0:
+
 Scrapy 1.5.0 (2017-12-29)
 -------------------------
 
@@ -373,6 +384,7 @@ Docs
 - Document ``from_crawler`` methods for spider and downloader middlewares
   (:issue:`3019`)
 
+.. _release-1.4.0:
 
 Scrapy 1.4.0 (2017-05-18)
 -------------------------
@@ -559,6 +571,8 @@ Documentation
 - Clarify ``allowed_domains`` example (:issue:`2670`)
 
 
+.. _release-1.3.3:
+
 Scrapy 1.3.3 (2017-03-10)
 -------------------------
 
@@ -571,6 +585,7 @@ Bug fixes
   A new setting is introduced to toggle between warning or exception if needed ;
   see :setting:`SPIDER_LOADER_WARN_ONLY` for details.
 
+.. _release-1.3.2:
 
 Scrapy 1.3.2 (2017-02-13)
 -------------------------
@@ -582,6 +597,8 @@ Bug fixes
 - Use consistent selectors for author field in tutorial (:issue:`2551`).
 - Fix TLS compatibility in Twisted 17+ (:issue:`2558`)
 
+.. _release-1.3.1:
+
 Scrapy 1.3.1 (2017-02-08)
 -------------------------
 
@@ -630,6 +647,8 @@ Cleanups
 - Remove dead code supporting old Twisted versions (:issue:`2544`).
 
 
+.. _release-1.3.0:
+
 Scrapy 1.3.0 (2016-12-21)
 -------------------------
 
@@ -669,6 +688,7 @@ Dependencies & Cleanups
 - ``ChunkedTransferMiddleware`` is deprecated and removed from the default
   downloader middlewares.
 
+.. _release-1.2.3:
 
 Scrapy 1.2.3 (2017-03-03)
 -------------------------
@@ -676,6 +696,8 @@ Scrapy 1.2.3 (2017-03-03)
 - Packaging fix: disallow unsupported Twisted versions in setup.py
 
 
+.. _release-1.2.2:
+
 Scrapy 1.2.2 (2016-12-06)
 -------------------------
 
@@ -711,6 +733,8 @@ Other changes
 .. _conda-forge: https://anaconda.org/conda-forge/scrapy
 
 
+.. _release-1.2.1:
+
 Scrapy 1.2.1 (2016-10-21)
 -------------------------
 
@@ -735,6 +759,8 @@ Other changes
 - Removed ``www.`` from ``start_urls`` in built-in spider templates (:issue:`2299`).
 
 
+.. _release-1.2.0:
+
 Scrapy 1.2.0 (2016-10-03)
 -------------------------
 
@@ -803,12 +829,14 @@ Documentation
 - Reworded misleading :setting:`RANDOMIZE_DOWNLOAD_DELAY` description (:issue:`2190`).
 - Add StackOverflow as a support channel (:issue:`2257`).
 
+.. _release-1.1.4:
 
 Scrapy 1.1.4 (2017-03-03)
 -------------------------
 
 - Packaging fix: disallow unsupported Twisted versions in setup.py
 
+.. _release-1.1.3:
 
 Scrapy 1.1.3 (2016-09-22)
 -------------------------
@@ -826,6 +854,7 @@ Documentation
   rewritten to use http://toscrape.com websites
   (:issue:`2236`, :issue:`2249`, :issue:`2252`).
 
+.. _release-1.1.2:
 
 Scrapy 1.1.2 (2016-08-18)
 -------------------------
@@ -840,6 +869,7 @@ Bug fixes
 - :setting:`IMAGES_EXPIRES` default value set back to 90
   (the regression was introduced in 1.1.1)
 
+.. _release-1.1.1:
 
 Scrapy 1.1.1 (2016-07-13)
 -------------------------
@@ -892,6 +922,7 @@ Tests
 
 - Upgrade py.test requirement on Travis CI and Pin pytest-cov to 2.2.1 (:issue:`2095`)
 
+.. _release-1.1.0:
 
 Scrapy 1.1.0 (2016-05-11)
 -------------------------
@@ -1081,12 +1112,14 @@ Bugfixes
 - HTTPS+CONNECT tunnels could get mixed up when using multiple proxies
   to same remote host (:issue:`1912`).
 
+.. _release-1.0.7:
 
 Scrapy 1.0.7 (2017-03-03)
 -------------------------
 
 - Packaging fix: disallow unsupported Twisted versions in setup.py
 
+.. _release-1.0.6:
 
 Scrapy 1.0.6 (2016-05-04)
 -------------------------
@@ -1096,6 +1129,7 @@ Scrapy 1.0.6 (2016-05-04)
 - DOC: Support for Sphinx 1.4+ (:issue:`1893`)
 - DOC: Consistency in selectors examples (:issue:`1869`)
 
+.. _release-1.0.5:
 
 Scrapy 1.0.5 (2016-02-04)
 -------------------------
@@ -1105,6 +1139,7 @@ Scrapy 1.0.5 (2016-02-04)
 - DOC: Fixed typos in tutorial and media-pipeline (:commit:`808a9ea` and :commit:`803bd87`)
 - DOC: Add AjaxCrawlMiddleware to DOWNLOADER_MIDDLEWARES_BASE in settings docs (:commit:`aa94121`)
 
+.. _release-1.0.4:
 
 Scrapy 1.0.4 (2015-12-30)
 -------------------------
@@ -1158,12 +1193,16 @@ Scrapy 1.0.4 (2015-12-30)
 - Small grammatical change (:commit:`8752294`)
 - Add openssl version to version command (:commit:`13c45ac`)
 
+.. _release-1.0.3:
+
 Scrapy 1.0.3 (2015-08-11)
 -------------------------
 
 - add service_identity to scrapy install_requires (:commit:`cbc2501`)
 - Workaround for travis#296 (:commit:`66af9cd`)
 
+.. _release-1.0.2:
+
 Scrapy 1.0.2 (2015-08-06)
 -------------------------
 
@@ -1174,6 +1213,8 @@ Scrapy 1.0.2 (2015-08-06)
 - Fixed typos (:commit:`a9ae7b0`)
 - Fix doc reference. (:commit:`7c8a4fe`)
 
+.. _release-1.0.1:
+
 Scrapy 1.0.1 (2015-07-01)
 -------------------------
 
@@ -1184,6 +1225,8 @@ Scrapy 1.0.1 (2015-07-01)
 - DOC remove version suffix from ubuntu package (:commit:`5303c66`)
 - DOC Update release date for 1.0 (:commit:`c89fa29`)
 
+.. _release-1.0.0:
+
 Scrapy 1.0.0 (2015-06-19)
 -------------------------
 

From 2c8c8b2dd8683787826713ed1d0fbfb2ec1af04a Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 30 Jan 2019 17:30:13 +0500
Subject: [PATCH 1894/2974] DOC fix after bad merge - remove duplicate entries
 in changelog

---
 docs/news.rst | 72 +++++++++++++--------------------------------------
 1 file changed, 18 insertions(+), 54 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index a4f07efad..4711d2f35 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -26,7 +26,6 @@ Highlights:
 * various bug fixes, small new features and usability improvements across
   the codebase.
 
-
 Selector API changes
 ~~~~~~~~~~~~~~~~~~~~
 
@@ -82,6 +81,8 @@ New extensibility features
 * new SitemapSpider :meth:`~.SitemapSpider.sitemap_filter` method which allows
   to select sitemap entries based on their attributes in SitemapSpider
   subclasses (:issue:`3512`).
+* Lazy loading of Downloader Handlers is now optional; this enables better
+  initialization error handling in custom Downloader Handlers (:issue:`3394`).
 
 New FilePipeline and MediaPipeline features
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -104,34 +105,20 @@ New FilePipeline and MediaPipeline features
 * Fixed errback handling in contracts, e.g. for cases where a contract
   is executed for URL which returns non-200 response (:issue:`3371`).
 
-Usability and other improvements, cleanups
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Usability improvements
+~~~~~~~~~~~~~~~~~~~~~~
 
-* All Scrapy tests now pass on Windows; Scrapy testing suite is executed
-  in a Windows environment on CI (:issue:`3315`).
-* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`).
-* Lazy loading of Downloader Handlers is now optional; this enables better
-  initialization error handling in custom Downloader Handlers (:issue:`3394`).
-* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`,
-  :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`)
-* better error message when an exporter is disabled (:issue:`3358`);
-* ``scrapy.http.cookies.CookieJar.clear`` accepts "domain", "path" and "name"
-  optional arguments (:issue:`3231`).
 * more stats for RobotsTxtMiddleware (:issue:`3100`)
 * INFO log level is used to show telnet host/port (:issue:`3115`)
 * a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`)
 * better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`)
 * non-zero exit code is returned from Scrapy commands when error happens
-  on spider inititalization (:issue:`3226`);
-* link extraction improvements: "ftp" is added to scheme list (:issue:`3152`);
+  on spider inititalization (:issue:`3226`)
+* Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`);
   "flv" is added to common video extensions (:issue:`3165`)
+* better error message when an exporter is disabled (:issue:`3358`);
 * `scrapy shell --help` mentions syntax required for local files
   (``./file.html``) - :issue:`3496`.
-* additional files are included to sdist (:issue:`3495`);
-* code style fixes (:issue:`3405`, :issue:`3304`);
-* unneeded .strip() call is removed (:issue:`3519`);
-* collections.deque is used to store MiddlewareManager methods instead
-  of a list (:issue:`3476`)
 * Referer header value is added to RFPDupeFilter log messages (:issue:`3588`)
 
 Bug fixes
@@ -195,44 +182,21 @@ Other deprecation removals:
 * Deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes
   are removed (:issue:`3327`, :issue:`3359`).
 
-Internal improvements
-~~~~~~~~~~~~~~~~~~~~~
+Other improvements, cleanups
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-* ``from_crawler`` support is added to dupefilters (:issue:`2956`); this allows
-  to access e.g. settings or a spider from a dupefilter.
-* :signal:`item_error` is fired when an error happens in a pipeline
-  (:issue:`3256`);
-* :signal:`request_reached_downloader` is fired when Downloader gets
-  a new Request; this signal can be useful e.g. for custom Schedulers
-  (:issue:`3393`).
+* All Scrapy tests now pass on Windows; Scrapy testing suite is executed
+  in a Windows environment on CI (:issue:`3315`).
+* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`).
+* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`,
+  :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`)
 * ``scrapy.http.cookies.CookieJar.clear`` accepts "domain", "path" and "name"
   optional arguments (:issue:`3231`).
-
-Usability improvements
-~~~~~~~~~~~~~~~~~~~~~~
-
-* more stats for RobotsTxtMiddleware (:issue:`3100`)
-* INFO log level is used to show telnet host/port (:issue:`3115`)
-* a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`)
-* better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`)
-* non-zero exit code is returned from Scrapy commands when error happens
-  on spider inititalization (:issue:`3226`)
-* Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`);
-  "flv" is added to common video extensions (:issue:`3165`)
-
-Bug fixes
-~~~~~~~~~
-* proper handling of pickling errors in Python 3 when serializing objects
-  for disk queues (:issue:`3082`)
-* flags are now preserved when copying Requests (:issue:`3342`);
-* FormRequest.from_response clickdata shouldn't ignore elements with
-  ``input[type=image]`` (:issue:`3153`).
-* FormRequest.from_response should preserve duplicate keys (:issue:`3247`)
-
-Cleanups
-~~~~~~~~
 * additional files are included to sdist (:issue:`3495`);
-* code style fixes (:issue:`3405`, :issue:`3304`)
+* code style fixes (:issue:`3405`, :issue:`3304`);
+* unneeded .strip() call is removed (:issue:`3519`);
+* collections.deque is used to store MiddlewareManager methods instead
+  of a list (:issue:`3476`)
 
 .. _release-1.5.2:
 

From 91791cd329936ee6ac53523460f9b72c20c66afb Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 30 Jan 2019 17:53:58 +0500
Subject: [PATCH 1895/2974] DOC final changelog cleanups

---
 docs/news.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 4711d2f35..543901809 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -63,7 +63,8 @@ Telnet console
 ~~~~~~~~~~~~~~
 
 **Backwards incompatible**: Scrapy's telnet console now requires username
-and password. See :ref:`topics-telnetconsole` for more details.
+and password. See :ref:`topics-telnetconsole` for more details. This change
+fixes a **security issue**; see :ref:`release-1.5.2` release notes for details.
 
 New extensibility features
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -117,7 +118,7 @@ Usability improvements
 * Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`);
   "flv" is added to common video extensions (:issue:`3165`)
 * better error message when an exporter is disabled (:issue:`3358`);
-* `scrapy shell --help` mentions syntax required for local files
+* ``scrapy shell --help`` mentions syntax required for local files
   (``./file.html``) - :issue:`3496`.
 * Referer header value is added to RFPDupeFilter log messages (:issue:`3588`)
 

From b8594353d03be5574f51766c35566b713584302b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 30 Jan 2019 18:00:40 -0300
Subject: [PATCH 1896/2974] =?UTF-8?q?Bump=20version:=201.5.0=20=E2=86=92?=
 =?UTF-8?q?=201.6.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 6e7be142e..8cecb7ad4 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.5.0
+current_version = 1.6.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index bc80560fa..dc1e644a1 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.5.0
+1.6.0

From 88326cd8be7f9c9f09924144a4d3a9666fdcf0b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 31 Jan 2019 01:16:28 -0300
Subject: [PATCH 1897/2974] Set release date to 1.6.0

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 543901809..668473887 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -5,7 +5,7 @@ Release notes
 
 .. _release-1.6.0:
 
-Scrapy 1.6.0 (unreleased)
+Scrapy 1.6.0 (2019-01-30)
 -------------------------
 
 Highlights:

From 65d631329a1434ec013f24341e4b8520241aec70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 31 Jan 2019 01:28:53 -0300
Subject: [PATCH 1898/2974] Be consistent with domain used for links to
 documentation website

---
 CONTRIBUTING.md                               |  2 +-
 INSTALL                                       |  2 +-
 README.rst                                    |  8 ++++----
 docs/contributing.rst                         |  2 +-
 docs/topics/selectors.rst                     |  4 ++--
 scrapy/extensions/telnet.py                   |  2 +-
 scrapy/templates/project/module/items.py.tmpl |  2 +-
 .../project/module/middlewares.py.tmpl        |  2 +-
 .../project/module/pipelines.py.tmpl          |  2 +-
 .../templates/project/module/settings.py.tmpl | 20 +++++++++----------
 sep/sep-001.rst                               |  2 +-
 sep/sep-006.rst                               |  4 ++--
 tests/__init__.py                             |  2 +-
 13 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 0a11b05d2..a05d07aee 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,5 +1,5 @@
 The guidelines for contributing are available here:
-https://doc.scrapy.org/en/master/contributing.html
+https://docs.scrapy.org/en/master/contributing.html
 
 Please do not abuse the issue tracker for support questions.
 If your issue topic can be rephrased to "How to ...?", please use the
diff --git a/INSTALL b/INSTALL
index a3c7899c6..06e812936 100644
--- a/INSTALL
+++ b/INSTALL
@@ -1,4 +1,4 @@
 For information about installing Scrapy see:
 
 * docs/intro/install.rst (local file)
-* https://doc.scrapy.org/en/latest/intro/install.html (online version)
+* https://docs.scrapy.org/en/latest/intro/install.html (online version)
diff --git a/README.rst b/README.rst
index 1361eac26..c28d217ff 100644
--- a/README.rst
+++ b/README.rst
@@ -51,18 +51,18 @@ The quick way::
     pip install scrapy
 
 For more details see the install section in the documentation:
-https://doc.scrapy.org/en/latest/intro/install.html
+https://docs.scrapy.org/en/latest/intro/install.html
 
 Documentation
 =============
 
-Documentation is available online at https://doc.scrapy.org/ and in the ``docs``
+Documentation is available online at https://docs.scrapy.org/ and in the ``docs``
 directory.
 
 Releases
 ========
 
-You can find release notes at https://doc.scrapy.org/en/latest/news.html
+You can find release notes at https://docs.scrapy.org/en/latest/news.html
 
 Community (blog, twitter, mail list, IRC)
 =========================================
@@ -72,7 +72,7 @@ See https://scrapy.org/community/
 Contributing
 ============
 
-See https://doc.scrapy.org/en/master/contributing.html
+See https://docs.scrapy.org/en/master/contributing.html
 
 Code of Conduct
 ---------------
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 2369c3436..cf27337c8 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -7,7 +7,7 @@ Contributing to Scrapy
 .. important::
 
     Double check that you are reading the most recent version of this document at
-    https://doc.scrapy.org/en/master/contributing.html
+    https://docs.scrapy.org/en/master/contributing.html
 
 There are many ways to contribute to Scrapy. Here are some of them:
 
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 9dced7473..df1d67ae8 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -100,7 +100,7 @@ To explain how to use the selectors we'll use the `Scrapy shell` (which
 provides interactive testing) and an example page located in the Scrapy
 documentation server:
 
-    https://doc.scrapy.org/en/latest/_static/selectors-sample1.html
+    https://docs.scrapy.org/en/latest/_static/selectors-sample1.html
 
 .. _topics-selectors-htmlcode:
 
@@ -113,7 +113,7 @@ For the sake of completeness, here's its full HTML code:
 
 First, let's open the shell::
 
-    scrapy shell https://doc.scrapy.org/en/latest/_static/selectors-sample1.html
+    scrapy shell https://docs.scrapy.org/en/latest/_static/selectors-sample1.html
 
 Then, after the shell loads, you'll have the response available as ``response``
 shell variable, and its attached selector in ``response.selector`` attribute.
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index dcf73eb88..26b214ee2 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -112,7 +112,7 @@ class TelnetConsole(protocol.ServerFactory):
             'prefs': print_live_refs,
             'hpy': hpy,
             'help': "This is Scrapy telnet console. For more info see: "
-                    "https://doc.scrapy.org/en/latest/topics/telnetconsole.html",
+                    "https://docs.scrapy.org/en/latest/topics/telnetconsole.html",
         }
         self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars)
         return telnet_vars
diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl
index 7d766f4fc..a12d08414 100644
--- a/scrapy/templates/project/module/items.py.tmpl
+++ b/scrapy/templates/project/module/items.py.tmpl
@@ -3,7 +3,7 @@
 # Define here the models for your scraped items
 #
 # See documentation in:
-# https://doc.scrapy.org/en/latest/topics/items.html
+# https://docs.scrapy.org/en/latest/topics/items.html
 
 import scrapy
 
diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
index c5b542bd6..5debe1cd2 100644
--- a/scrapy/templates/project/module/middlewares.py.tmpl
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -3,7 +3,7 @@
 # Define here the models for your spider middleware
 #
 # See documentation in:
-# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
 
 from scrapy import signals
 
diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl
index e58dab089..fb641d447 100644
--- a/scrapy/templates/project/module/pipelines.py.tmpl
+++ b/scrapy/templates/project/module/pipelines.py.tmpl
@@ -3,7 +3,7 @@
 # Define your item pipelines here
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
-# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
 
 
 class ${ProjectName}Pipeline(object):
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index a0557473e..cb220eafc 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -5,9 +5,9 @@
 # For simplicity, this file contains only settings considered important or
 # commonly used. You can find more settings consulting the documentation:
 #
-#     https://doc.scrapy.org/en/latest/topics/settings.html
-#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
-#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+#     https://docs.scrapy.org/en/latest/topics/settings.html
+#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
 
 BOT_NAME = '$project_name'
 
@@ -25,7 +25,7 @@ ROBOTSTXT_OBEY = True
 #CONCURRENT_REQUESTS = 32
 
 # Configure a delay for requests for the same website (default: 0)
-# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
+# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
 # See also autothrottle settings and docs
 #DOWNLOAD_DELAY = 3
 # The download delay setting will honor only one of:
@@ -45,31 +45,31 @@ ROBOTSTXT_OBEY = True
 #}
 
 # Enable or disable spider middlewares
-# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
 #SPIDER_MIDDLEWARES = {
 #    '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543,
 #}
 
 # Enable or disable downloader middlewares
-# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
 #DOWNLOADER_MIDDLEWARES = {
 #    '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543,
 #}
 
 # Enable or disable extensions
-# See https://doc.scrapy.org/en/latest/topics/extensions.html
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
 #EXTENSIONS = {
 #    'scrapy.extensions.telnet.TelnetConsole': None,
 #}
 
 # Configure item pipelines
-# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
 #ITEM_PIPELINES = {
 #    '$project_name.pipelines.${ProjectName}Pipeline': 300,
 #}
 
 # Enable and configure the AutoThrottle extension (disabled by default)
-# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
 #AUTOTHROTTLE_ENABLED = True
 # The initial download delay
 #AUTOTHROTTLE_START_DELAY = 5
@@ -82,7 +82,7 @@ ROBOTSTXT_OBEY = True
 #AUTOTHROTTLE_DEBUG = False
 
 # Enable and configure HTTP caching (disabled by default)
-# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
 #HTTPCACHE_ENABLED = True
 #HTTPCACHE_EXPIRATION_SECS = 0
 #HTTPCACHE_DIR = 'httpcache'
diff --git a/sep/sep-001.rst b/sep/sep-001.rst
index 3766f38fc..2a66f9802 100644
--- a/sep/sep-001.rst
+++ b/sep/sep-001.rst
@@ -61,7 +61,7 @@ ItemForm
 --------
 
 Pros:
-- same API used for Items (see https://doc.scrapy.org/en/latest/topics/items.html)
+- same API used for Items (see https://docs.scrapy.org/en/latest/topics/items.html)
 - some people consider setitem API more elegant than methods API
 
 Cons:
diff --git a/sep/sep-006.rst b/sep/sep-006.rst
index 7425c0930..366fcf033 100644
--- a/sep/sep-006.rst
+++ b/sep/sep-006.rst
@@ -16,7 +16,7 @@ Motivation
 ==========
 
 When you use Selectors in Scrapy, your final goal is to "extract" the data that
-you've selected, as the [https://doc.scrapy.org/en/latest/topics/selectors.html
+you've selected, as the [https://docs.scrapy.org/en/latest/topics/selectors.html
 XPath Selectors documentation] says (bolding by me):
 
    When you’re scraping web pages, the most common task you need to perform is
@@ -71,5 +71,5 @@ webpage or set of pages.
 References
 ==========
 
- 1. XPath Selectors (https://doc.scrapy.org/topics/selectors.html)
+ 1. XPath Selectors (https://docs.scrapy.org/topics/selectors.html)
  2. XPath and XSLT with lxml (http://lxml.de/xpathxslt.html)
diff --git a/tests/__init__.py b/tests/__init__.py
index 55b1ecde8..a54367f8c 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -1,7 +1,7 @@
 """
 tests: this package contains all Scrapy unittests
 
-see https://doc.scrapy.org/en/latest/contributing.html#running-tests
+see https://docs.scrapy.org/en/latest/contributing.html#running-tests
 """
 
 import os

From 5a55c4269d3389df4e486657caa58985f8e37e5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Thu, 31 Jan 2019 17:20:29 -0200
Subject: [PATCH 1899/2974] Adding GCSFeedStorage

---
 scrapy/extensions/feedexport.py | 26 ++++++++++++++++++
 tests/test_feedexport.py        | 48 ++++++++++++++++++++++++++++++++-
 2 files changed, 73 insertions(+), 1 deletion(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 22ebf3b3f..00d5d8025 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -146,6 +146,32 @@ class S3FeedStorage(BlockingFeedStorage):
             key.close()
 
 
+
+class GCSFeedStorage(BlockingFeedStorage):
+
+    project_id = None
+    bucket_name = None
+    blob_name = None
+
+    def __init__(self, uri, project_id):
+        self.project_id = project_id
+        u = urlparse(uri)
+        self.bucket_name = u.hostname
+        self.blob_name = u.path[1:]  # remove first "/"
+
+    @classmethod
+    def from_crawler(cls, crawler, uri):
+        return cls(uri, crawler.settings['GCS_PROJECT_ID'])
+
+    def _store_in_thread(self, file):
+        file.seek(0)
+        from google.cloud.storage import Client
+        client = Client(project=self.project_id)
+        bucket = client.get_bucket(self.bucket_name)
+        blob = bucket.blob(self.blob_name)
+        blob.upload_from_file(file)
+
+
 class FTPFeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e46c8c14e..f3c499b3f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -21,7 +21,7 @@ from w3lib.url import path_to_file_uri
 import scrapy
 from scrapy.exporters import CsvItemExporter
 from scrapy.extensions.feedexport import (
-    IFeedStorage, FileFeedStorage, FTPFeedStorage,
+    IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage,
     S3FeedStorage, StdoutFeedStorage,
     BlockingFeedStorage)
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
@@ -187,6 +187,52 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(content, expected_content)
 
 
+class GCSFeedStorageTest(unittest.TestCase):
+
+    @mock.patch('scrapy.conf.settings',
+                new={'GCS_PROJECT_ID': 'conf_id' }, create=True)
+    def test_parse_settings(self):
+        try:
+            from google.cloud.storage import Client
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        settings = {'GCS_PROJECT_ID': '123' }
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv')
+        assert storage.project_id == '123'
+        assert storage.bucket_name == 'mybucket'
+        assert storage.blob_name == 'export.csv'
+
+    @defer.inlineCallbacks
+    def test_store(self):
+        try:
+            from google.cloud.storage import Client, Bucket, Blob
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        uri = 'gcs://mybucket/export.csv'
+        project_id = 'myproject-123'
+        with mock.patch('google.cloud.storage.Client') as m:
+            client_mock = mock.create_autospec(Client)
+            m.return_value = client_mock
+
+            bucket_mock = mock.create_autospec(Bucket)
+            client_mock.get_bucket.return_value = bucket_mock
+
+            blob_mock = mock.create_autospec(Blob)
+            bucket_mock.blob.return_value = blob_mock
+
+            f = mock.Mock()
+            storage = GCSFeedStorage(uri, project_id)
+            yield storage.store(f)
+
+            f.seek.assert_called_once_with(0)
+            m.assert_called_once_with(project=project_id)
+            client_mock.get_bucket.assert_called_once_with('mybucket')
+            bucket_mock.blob.assert_called_once_with('export.csv')
+            blob_mock.upload_from_file.assert_called_once_with(f)
+
 class StdoutFeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks

From a4059851e7b6c8d712b2bc73dbff99be2d569d21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Thu, 31 Jan 2019 18:29:15 -0200
Subject: [PATCH 1900/2974] Refactoring tests

---
 scrapy/utils/test.py     | 16 ++++++++++++++++
 tests/test_feedexport.py | 15 +++++----------
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 4b935c51b..84eae97a7 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -7,6 +7,7 @@ import os
 
 from importlib import import_module
 from twisted.trial.unittest import SkipTest
+from tests import mock
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.boto import is_botocore
@@ -91,3 +92,18 @@ def assert_samelines(testcase, text1, text2, msg=None):
     line endings between platforms
     """
     testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
+
+def mock_google_cloud_storage():
+    """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
+    classes and set their proper return values.
+    """
+    from google.cloud.storage import Client, Bucket, Blob
+    client_mock = mock.create_autospec(Client)
+
+    bucket_mock = mock.create_autospec(Bucket)
+    client_mock.get_bucket.return_value = bucket_mock
+
+    blob_mock = mock.create_autospec(Blob)
+    bucket_mock.blob.return_value = blob_mock
+
+    return (client_mock, bucket_mock, blob_mock)
\ No newline at end of file
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index f3c499b3f..6d23c68c3 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -24,7 +24,8 @@ from scrapy.extensions.feedexport import (
     IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage,
     S3FeedStorage, StdoutFeedStorage,
     BlockingFeedStorage)
-from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
+from scrapy.utils.test import (assert_aws_environ, get_s3_content_and_delete,
+    get_crawler, mock_google_cloud_storage)
 from scrapy.utils.python import to_native_str
 
 
@@ -207,22 +208,16 @@ class GCSFeedStorageTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_store(self):
         try:
-            from google.cloud.storage import Client, Bucket, Blob
+            from google.cloud.storage import Client
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 
         uri = 'gcs://mybucket/export.csv'
         project_id = 'myproject-123'
-        with mock.patch('google.cloud.storage.Client') as m:
-            client_mock = mock.create_autospec(Client)
+        (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
+        with mock.patch('google.cloud.storage.Client') as m:    
             m.return_value = client_mock
 
-            bucket_mock = mock.create_autospec(Bucket)
-            client_mock.get_bucket.return_value = bucket_mock
-
-            blob_mock = mock.create_autospec(Blob)
-            bucket_mock.blob.return_value = blob_mock
-
             f = mock.Mock()
             storage = GCSFeedStorage(uri, project_id)
             yield storage.store(f)

From 38af090f4d6799a0499b116a62c12509f59f561b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 4 Feb 2019 11:17:58 +0100
Subject: [PATCH 1901/2974] Indicate that users must implement their own
 authentication result check

The example of form-based login could lead some users to think its authentication result
check was final. See https://stackoverflow.com/a/54410966/939364

This change should make it more obvious that users are expected to implement their
own logic to check whether authentication worked or not.
---
 docs/topics/request-response.rst | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index e29914dbf..76360b15f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -489,6 +489,11 @@ method for this job. Here's an example spider which uses it::
 
     import scrapy
 
+    def authentication_failed(response):
+        # TODO: Check the contents of the response and return True if it failed
+        # or False if it succeeded.
+        pass
+
     class LoginSpider(scrapy.Spider):
         name = 'example.com'
         start_urls = ['http://www.example.com/users/login.php']
@@ -501,8 +506,7 @@ method for this job. Here's an example spider which uses it::
             )
 
         def after_login(self, response):
-            # check login succeed before going on
-            if "authentication failed" in response.body:
+            if authentication_failed(response):
                 self.logger.error("Login failed")
                 return
 

From 1bb6c4154c43d03d00147d810147c4f3c807505e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Fri, 8 Feb 2019 09:04:01 -0200
Subject: [PATCH 1902/2974] Turning into instance attributes

---
 scrapy/extensions/feedexport.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 00d5d8025..a81f44045 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -149,10 +149,6 @@ class S3FeedStorage(BlockingFeedStorage):
 
 class GCSFeedStorage(BlockingFeedStorage):
 
-    project_id = None
-    bucket_name = None
-    blob_name = None
-
     def __init__(self, uri, project_id):
         self.project_id = project_id
         u = urlparse(uri)

From fc6809b024dc25663432b6c0c5780021a827ea20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Fri, 8 Feb 2019 09:08:54 -0200
Subject: [PATCH 1903/2974] Add gcs schema to FEED_STORAGES_BASE

---
 scrapy/settings/default_settings.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 3734a0a58..1a12f35a3 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -145,6 +145,7 @@ FEED_STORAGES_BASE = {
     'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
+    'gcs': 'scrapy.extensions.feedexport.GCSFeedStorage',
 }
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {

From 4a53de165a53433a9a2a8cc4db34e5507c47fbd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Fri, 8 Feb 2019 09:09:56 -0200
Subject: [PATCH 1904/2974] Sorted schemas alphabetically

---
 scrapy/extensions/feedexport.py     | 1 -
 scrapy/settings/default_settings.py | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index a81f44045..8347b42ca 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -146,7 +146,6 @@ class S3FeedStorage(BlockingFeedStorage):
             key.close()
 
 
-
 class GCSFeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri, project_id):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 1a12f35a3..8769c01ba 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -142,10 +142,10 @@ FEED_STORAGES = {}
 FEED_STORAGES_BASE = {
     '': 'scrapy.extensions.feedexport.FileFeedStorage',
     'file': 'scrapy.extensions.feedexport.FileFeedStorage',
-    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
-    's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
     'gcs': 'scrapy.extensions.feedexport.GCSFeedStorage',
+    's3': 'scrapy.extensions.feedexport.S3FeedStorage',
+    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
 }
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {

From 2bbbd02bda368c2f052f3aa38f99498a632328bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Fri, 8 Feb 2019 09:45:10 -0200
Subject: [PATCH 1905/2974] Adding an option to set ACL while uploading the
 blob to GCS

---
 scrapy/extensions/feedexport.py     | 11 ++++++++---
 scrapy/settings/default_settings.py |  2 ++
 tests/test_feedexport.py            | 10 ++++++----
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 8347b42ca..fbbf9bb97 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -148,15 +148,20 @@ class S3FeedStorage(BlockingFeedStorage):
 
 class GCSFeedStorage(BlockingFeedStorage):
 
-    def __init__(self, uri, project_id):
+    def __init__(self, uri, project_id, acl):
         self.project_id = project_id
+        self.acl = acl
         u = urlparse(uri)
         self.bucket_name = u.hostname
         self.blob_name = u.path[1:]  # remove first "/"
 
     @classmethod
     def from_crawler(cls, crawler, uri):
-        return cls(uri, crawler.settings['GCS_PROJECT_ID'])
+        return cls(
+            uri,
+            crawler.settings['GCS_PROJECT_ID'],
+            crawler.settings['FEED_STORAGE_GCS_ACL']
+        )
 
     def _store_in_thread(self, file):
         file.seek(0)
@@ -164,7 +169,7 @@ class GCSFeedStorage(BlockingFeedStorage):
         client = Client(project=self.project_id)
         bucket = client.get_bucket(self.bucket_name)
         blob = bucket.blob(self.blob_name)
-        blob.upload_from_file(file)
+        blob.upload_from_file(file, predefined_acl=self.acl)
 
 
 class FTPFeedStorage(BlockingFeedStorage):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8769c01ba..5d2862980 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -159,6 +159,8 @@ FEED_EXPORTERS_BASE = {
 }
 FEED_EXPORT_INDENT = 0
 
+FEED_STORAGE_GCS_ACL = None
+
 FILES_STORE_S3_ACL = 'private'
 FILES_STORE_GCS_ACL = ''
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 6d23c68c3..5cbca6d28 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -191,17 +191,18 @@ class S3FeedStorageTest(unittest.TestCase):
 class GCSFeedStorageTest(unittest.TestCase):
 
     @mock.patch('scrapy.conf.settings',
-                new={'GCS_PROJECT_ID': 'conf_id' }, create=True)
+                new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': None }, create=True)
     def test_parse_settings(self):
         try:
             from google.cloud.storage import Client
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 
-        settings = {'GCS_PROJECT_ID': '123' }
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' }
         crawler = get_crawler(settings_dict=settings)
         storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv')
         assert storage.project_id == '123'
+        assert storage.acl == 'publicRead'
         assert storage.bucket_name == 'mybucket'
         assert storage.blob_name == 'export.csv'
 
@@ -214,19 +215,20 @@ class GCSFeedStorageTest(unittest.TestCase):
 
         uri = 'gcs://mybucket/export.csv'
         project_id = 'myproject-123'
+        acl = 'publicRead'
         (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
         with mock.patch('google.cloud.storage.Client') as m:    
             m.return_value = client_mock
 
             f = mock.Mock()
-            storage = GCSFeedStorage(uri, project_id)
+            storage = GCSFeedStorage(uri, project_id, acl)
             yield storage.store(f)
 
             f.seek.assert_called_once_with(0)
             m.assert_called_once_with(project=project_id)
             client_mock.get_bucket.assert_called_once_with('mybucket')
             bucket_mock.blob.assert_called_once_with('export.csv')
-            blob_mock.upload_from_file.assert_called_once_with(f)
+            blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
 
 class StdoutFeedStorageTest(unittest.TestCase):
 

From cb5f800b0f7029d2bf2e09e9bc1391078c133a92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Fri, 8 Feb 2019 11:26:33 -0200
Subject: [PATCH 1906/2974] Adding documentation about Google Cloud Storage
 Feed Export

---
 docs/topics/feed-exports.rst        | 22 ++++++++++++++++++++++
 docs/topics/settings.rst            | 18 ++++++++++++++++++
 scrapy/settings/default_settings.py |  2 ++
 3 files changed, 42 insertions(+)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index b64dbfbfd..efb63b0ba 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -185,6 +185,27 @@ passed through the following settings:
  * :setting:`AWS_ACCESS_KEY_ID`
  * :setting:`AWS_SECRET_ACCESS_KEY`
 
+.. _topics-feed-storage-gcs:
+
+Google Cloud Storage (GCS)
+--------------------------
+
+The feeds are stored on `Google Cloud Storage`_.
+
+ * URI scheme: ``gcs``
+ * Example URIs:
+
+   * ``gcs://mybucket/path/to/export.csv``
+
+ * Required external libraries: `google-cloud-storage <https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python>`_.
+
+For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
+
+You can set a *Project ID* and *Access Control List (ACL)* through the following settings:
+
+ * :setting:`FEED_STORAGE_GCS_ACL`
+ * :setting:`GCS_PROJECT_ID`
+
 .. _topics-feed-storage-stdout:
 
 Standard output
@@ -366,3 +387,4 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. _Amazon S3: https://aws.amazon.com/s3/
 .. _boto: https://github.com/boto/boto
 .. _botocore: https://github.com/boto/botocore
+.. _Google Cloud Storage: https://cloud.google.com/storage/
\ No newline at end of file
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0ac26a9bd..90ae8fd93 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -749,6 +749,14 @@ The Feed Temp dir allows you to set a custom folder to save crawler
 temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
 :ref:`Amazon S3 <topics-feed-storage-s3>`.
 
+.. setting:: FEED_STORAGE_GCS_ACL
+
+FEED_STORAGE_GCS_ACL
+--------------------
+
+The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
+For more information on how to set this value, please refer to `Google Cloud documentation <https://cloud.google.com/storage/docs/access-control/lists>`_.
+
 .. setting:: FTP_PASSIVE_MODE
 
 FTP_PASSIVE_MODE
@@ -786,6 +794,15 @@ Default: ``"anonymous"``
 The username to use for FTP connections when there is no ``"ftp_user"``
 in ``Request`` meta.
 
+.. setting:: GCS_PROJECT_ID
+
+GCS_PROJECT_ID
+-----------------
+
+Default: ``None``
+
+The Project ID that will be used when storing data on `Google Cloud Storage`_.
+
 .. setting:: ITEM_PIPELINES
 
 ITEM_PIPELINES
@@ -1371,3 +1388,4 @@ case to see how to enable and use them.
 .. _Amazon web services: https://aws.amazon.com/
 .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
 .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search
+.. _Google Cloud Storage: https://cloud.google.com/storage/
\ No newline at end of file
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 5d2862980..c17e94a64 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -168,6 +168,8 @@ FTP_USER = 'anonymous'
 FTP_PASSWORD = 'guest'
 FTP_PASSIVE_MODE = True
 
+GCS_PROJECT_ID = None
+
 HTTPCACHE_ENABLED = False
 HTTPCACHE_DIR = 'httpcache'
 HTTPCACHE_IGNORE_MISSING = False

From 013568097db04396d780d1c91d37027115af7fe2 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Tue, 29 Jan 2019 11:10:06 -0300
Subject: [PATCH 1907/2974] add FEED_STORAGE_S3_ACL setting

---
 scrapy/extensions/feedexport.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 22ebf3b3f..eb0802261 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -118,6 +118,7 @@ class S3FeedStorage(BlockingFeedStorage):
         self.secret_key = u.password or secret_key
         self.is_botocore = is_botocore()
         self.keyname = u.path[1:]  # remove first "/"
+        self.policy = settings.get('FEED_STORAGE_S3_ACL', 'private')
         if self.is_botocore:
             import botocore.session
             session = botocore.session.get_session()
@@ -137,12 +138,13 @@ class S3FeedStorage(BlockingFeedStorage):
         file.seek(0)
         if self.is_botocore:
             self.s3_client.put_object(
-                Bucket=self.bucketname, Key=self.keyname, Body=file)
+                Bucket=self.bucketname, Key=self.keyname, Body=file,
+                ACL=self.policy)
         else:
             conn = self.connect_s3(self.access_key, self.secret_key)
             bucket = conn.get_bucket(self.bucketname, validate=False)
             key = bucket.new_key(self.keyname)
-            key.set_contents_from_file(file)
+            key.set_contents_from_file(file, policy=self.policy)
             key.close()
 
 

From ad83ffdf1f4d69ffb62b243429e7b59d0930524c Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Wed, 6 Feb 2019 18:32:46 -0200
Subject: [PATCH 1908/2974] refactoring

---
 scrapy/extensions/feedexport.py | 19 ++++++--
 tests/test_feedexport.py        | 84 +++++++++++++++++++++++++++++++++
 2 files changed, 98 insertions(+), 5 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index eb0802261..ca30322be 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -93,7 +93,7 @@ class FileFeedStorage(object):
 
 class S3FeedStorage(BlockingFeedStorage):
 
-    def __init__(self, uri, access_key=None, secret_key=None):
+    def __init__(self, uri, access_key=None, secret_key=None, acl=None):
         # BEGIN Backwards compatibility for initialising without keys (and
         # without using from_crawler)
         no_defaults = access_key is None and secret_key is None
@@ -118,7 +118,7 @@ class S3FeedStorage(BlockingFeedStorage):
         self.secret_key = u.password or secret_key
         self.is_botocore = is_botocore()
         self.keyname = u.path[1:]  # remove first "/"
-        self.policy = settings.get('FEED_STORAGE_S3_ACL', 'private')
+        self.acl = acl
         if self.is_botocore:
             import botocore.session
             session = botocore.session.get_session()
@@ -132,19 +132,28 @@ class S3FeedStorage(BlockingFeedStorage):
     @classmethod
     def from_crawler(cls, crawler, uri):
         return cls(uri, crawler.settings['AWS_ACCESS_KEY_ID'],
-                   crawler.settings['AWS_SECRET_ACCESS_KEY'])
+                   crawler.settings['AWS_SECRET_ACCESS_KEY'],
+                   crawler.settings.get('FEED_STORAGE_S3_ACL'))
 
     def _store_in_thread(self, file):
         file.seek(0)
         if self.is_botocore:
+            kwargs = dict()
+            if self.acl:
+                kwargs.update(dict(ACL=self.acl))
+
             self.s3_client.put_object(
                 Bucket=self.bucketname, Key=self.keyname, Body=file,
-                ACL=self.policy)
+                **kwargs)
         else:
             conn = self.connect_s3(self.access_key, self.secret_key)
             bucket = conn.get_bucket(self.bucketname, validate=False)
             key = bucket.new_key(self.keyname)
-            key.set_contents_from_file(file, policy=self.policy)
+            kwargs = dict()
+            if self.acl:
+                kwargs.update(dict(policy=self.acl))
+
+            key.set_contents_from_file(file, **kwargs)
             key.close()
 
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e46c8c14e..b07635cb0 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -18,6 +18,7 @@ from tests import mock
 from tests.mockserver import MockServer
 from w3lib.url import path_to_file_uri
 
+import botocore.client
 import scrapy
 from scrapy.exporters import CsvItemExporter
 from scrapy.extensions.feedexport import (
@@ -186,6 +187,89 @@ class S3FeedStorageTest(unittest.TestCase):
         content = get_s3_content_and_delete(u.hostname, u.path[1:])
         self.assertEqual(content, expected_content)
 
+    def test_init_without_acl(self):
+        storage = S3FeedStorage(
+            's3://mybucket/export.csv',
+            'access_key',
+            'secret_key'
+        )
+        self.assertEqual(storage.access_key, 'access_key')
+        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.acl, None)
+
+    def test_init_with_acl(self):
+        storage = S3FeedStorage(
+            's3://mybucket/export.csv',
+            'access_key',
+            'secret_key',
+            'custom-acl'
+        )
+        self.assertEqual(storage.access_key, 'access_key')
+        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.acl, 'custom-acl')
+
+    def test_from_crawler_without_acl(self):
+        settings = {
+            'AWS_ACCESS_KEY_ID': 'access_key',
+            'AWS_SECRET_ACCESS_KEY': 'secret_key',
+        }
+        crawler = get_crawler(settings_dict=settings)
+        storage = S3FeedStorage.from_crawler(
+            crawler,
+            's3://mybucket/export.csv'
+        )
+        self.assertEqual(storage.access_key, 'access_key')
+        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.acl, None)
+
+    def test_from_crawler_with_acl(self):
+        settings = {
+            'AWS_ACCESS_KEY_ID': 'access_key',
+            'AWS_SECRET_ACCESS_KEY': 'secret_key',
+            'FEED_STORAGE_S3_ACL': 'custom-acl',
+        }
+        crawler = get_crawler(settings_dict=settings)
+        storage = S3FeedStorage.from_crawler(
+            crawler,
+            's3://mybucket/export.csv'
+        )
+        self.assertEqual(storage.access_key, 'access_key')
+        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.acl, 'custom-acl')
+
+    def test_store_in_thread_without_acl(self):
+        storage = S3FeedStorage(
+            's3://mybucket/export.csv',
+            'access_key',
+            'secret_key',
+        )
+        self.assertEqual(storage.access_key, 'access_key')
+        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.acl, None)
+
+        with mock.patch('botocore.client.BaseClient._make_api_call') as _make_api_call_mock:
+            storage._store_in_thread(BytesIO(b'test file'))
+            operation_name, api_params = _make_api_call_mock.call_args[0]
+            self.assertEqual(operation_name, 'PutObject')
+            self.assertNotIn('ACL', api_params)
+
+    def test_store_in_thread_with_acl(self):
+        storage = S3FeedStorage(
+            's3://mybucket/export.csv',
+            'access_key',
+            'secret_key',
+            'custom-acl'
+        )
+        self.assertEqual(storage.access_key, 'access_key')
+        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.acl, 'custom-acl')
+
+        with mock.patch('botocore.client.BaseClient._make_api_call') as _make_api_call_mock:
+            storage._store_in_thread(BytesIO(b'test file'))
+            operation_name, api_params = _make_api_call_mock.call_args[0]
+            self.assertEqual(operation_name, 'PutObject')
+            self.assertEqual(api_params.get('ACL'), 'custom-acl')
+
 
 class StdoutFeedStorageTest(unittest.TestCase):
 

From 126207fb7bca21d3d95ed9c66028e82771180370 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Wed, 6 Feb 2019 18:38:17 -0200
Subject: [PATCH 1909/2974] PEP8: use short name for mock method

---
 tests/test_feedexport.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index b07635cb0..bfac06efc 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -247,9 +247,9 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, None)
 
-        with mock.patch('botocore.client.BaseClient._make_api_call') as _make_api_call_mock:
+        with mock.patch('botocore.client.BaseClient._make_api_call') as m:
             storage._store_in_thread(BytesIO(b'test file'))
-            operation_name, api_params = _make_api_call_mock.call_args[0]
+            operation_name, api_params = m.call_args[0]
             self.assertEqual(operation_name, 'PutObject')
             self.assertNotIn('ACL', api_params)
 
@@ -264,9 +264,9 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, 'custom-acl')
 
-        with mock.patch('botocore.client.BaseClient._make_api_call') as _make_api_call_mock:
+        with mock.patch('botocore.client.BaseClient._make_api_call') as m:
             storage._store_in_thread(BytesIO(b'test file'))
-            operation_name, api_params = _make_api_call_mock.call_args[0]
+            operation_name, api_params = m.call_args[0]
             self.assertEqual(operation_name, 'PutObject')
             self.assertEqual(api_params.get('ACL'), 'custom-acl')
 

From e0f34be383e361c75b22da59c97dee1db189937e Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Wed, 6 Feb 2019 18:50:19 -0200
Subject: [PATCH 1910/2974] update docs

---
 docs/topics/feed-exports.rst | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index b64dbfbfd..661751ed9 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -185,6 +185,10 @@ passed through the following settings:
  * :setting:`AWS_ACCESS_KEY_ID`
  * :setting:`AWS_SECRET_ACCESS_KEY`
 
+You can also define a custom ACL for exported objects using this setting:
+
+ * :setting:`FEED_STORAGE_S3_ACL`
+
 .. _topics-feed-storage-stdout:
 
 Standard output
@@ -205,6 +209,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_URI` (mandatory)
  * :setting:`FEED_FORMAT`
  * :setting:`FEED_STORAGES`
+ * :setting:`FEED_STORAGE_S3_ACL`
  * :setting:`FEED_EXPORTERS`
  * :setting:`FEED_STORE_EMPTY`
  * :setting:`FEED_EXPORT_ENCODING`
@@ -302,11 +307,22 @@ Default: ``{}``
 A dict containing additional feed storage backends supported by your project.
 The keys are URI schemes and the values are paths to storage classes.
 
+.. setting:: FEED_STORAGE_S3_ACL
+
+FEED_STORAGE_S3_ACL
+-------------------
+
+Default: ``None``
+
+A string containing a custom ACL for feeds exported to Amazon S3 by your project.
+
+For a complete list of available values, access the `Canned ACL`_ section on Amazon S3 docs.
+
 .. setting:: FEED_STORAGES_BASE
 
 FEED_STORAGES_BASE
 ------------------
-
+`
 Default::
 
     {
@@ -366,3 +382,4 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. _Amazon S3: https://aws.amazon.com/s3/
 .. _boto: https://github.com/boto/boto
 .. _botocore: https://github.com/boto/botocore
+.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl

From 7b83ed7c5e1fcd81baf50db3a76f10ade7aa226e Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Wed, 6 Feb 2019 18:52:24 -0200
Subject: [PATCH 1911/2974] remove typo

---
 docs/topics/feed-exports.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 661751ed9..25979dfef 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -322,7 +322,7 @@ For a complete list of available values, access the `Canned ACL`_ section on Ama
 
 FEED_STORAGES_BASE
 ------------------
-`
+
 Default::
 
     {

From e25b9a2323c169a4032ff07f912299b32de4b2e0 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Wed, 6 Feb 2019 18:52:39 -0200
Subject: [PATCH 1912/2974] calling it feeds instead of objects

---
 docs/topics/feed-exports.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 25979dfef..dee0c3ffa 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -185,7 +185,7 @@ passed through the following settings:
  * :setting:`AWS_ACCESS_KEY_ID`
  * :setting:`AWS_SECRET_ACCESS_KEY`
 
-You can also define a custom ACL for exported objects using this setting:
+You can also define a custom ACL for exported feeds using this setting:
 
  * :setting:`FEED_STORAGE_S3_ACL`
 

From dbeb088eea1713ac43f3d23579c36ece5f67563f Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Thu, 7 Feb 2019 09:29:16 -0200
Subject: [PATCH 1913/2974] trying to fix jessie testenv by adding botocore to
 requirements and fixing its version

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 0c0f8f7b7..f2f3e1293 100644
--- a/tox.ini
+++ b/tox.ini
@@ -47,6 +47,7 @@ deps =
     lxml==3.4.0
     Twisted==14.0.2
     boto==2.34.0
+    botocore==1.12.89
     Pillow==2.6.1
     cssselect==0.9.1
     zope.interface==4.1.1

From 079af889e7d010a79640e6874c3e6dc394b936ae Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Thu, 7 Feb 2019 10:42:59 -0200
Subject: [PATCH 1914/2974] also testing without botocore

---
 tests/test_feedexport.py | 53 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 51 insertions(+), 2 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index bfac06efc..520ca4a8f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -237,7 +237,7 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, 'custom-acl')
 
-    def test_store_in_thread_without_acl(self):
+    def test_store_in_thread_botocore_without_acl(self):
         storage = S3FeedStorage(
             's3://mybucket/export.csv',
             'access_key',
@@ -253,7 +253,7 @@ class S3FeedStorageTest(unittest.TestCase):
             self.assertEqual(operation_name, 'PutObject')
             self.assertNotIn('ACL', api_params)
 
-    def test_store_in_thread_with_acl(self):
+    def test_store_in_thread_botocore_with_acl(self):
         storage = S3FeedStorage(
             's3://mybucket/export.csv',
             'access_key',
@@ -270,6 +270,55 @@ class S3FeedStorageTest(unittest.TestCase):
             self.assertEqual(operation_name, 'PutObject')
             self.assertEqual(api_params.get('ACL'), 'custom-acl')
 
+    def test_store_in_thread_not_botocore_without_acl(self):
+        storage = S3FeedStorage(
+            's3://mybucket/export.csv',
+            'access_key',
+            'secret_key',
+        )
+        self.assertEqual(storage.access_key, 'access_key')
+        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.acl, None)
+
+        storage.is_botocore = False
+        storage.connect_s3 = mock.MagicMock()
+        self.assertFalse(storage.is_botocore)
+
+        storage._store_in_thread(BytesIO(b'test file'))
+
+        conn = storage.connect_s3(*storage.connect_s3.call_args)
+        bucket = conn.get_bucket(*conn.get_bucket.call_args)
+        key = bucket.new_key(*bucket.new_key.call_args)
+        self.assertNotIn(
+            dict(policy='custom-acl'),
+            key.set_contents_from_file.call_args
+        )
+
+    def test_store_in_thread_not_botocore_with_acl(self):
+        storage = S3FeedStorage(
+            's3://mybucket/export.csv',
+            'access_key',
+            'secret_key',
+            'custom-acl'
+        )
+        self.assertEqual(storage.access_key, 'access_key')
+        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.acl, 'custom-acl')
+
+        storage.is_botocore = False
+        storage.connect_s3 = mock.MagicMock()
+        self.assertFalse(storage.is_botocore)
+
+        storage._store_in_thread(BytesIO(b'test file'))
+
+        conn = storage.connect_s3(*storage.connect_s3.call_args)
+        bucket = conn.get_bucket(*conn.get_bucket.call_args)
+        key = bucket.new_key(*bucket.new_key.call_args)
+        self.assertIn(
+            dict(policy='custom-acl'),
+            key.set_contents_from_file.call_args
+        )
+
 
 class StdoutFeedStorageTest(unittest.TestCase):
 

From ceae356e62dc2e56465a58b3d9fe00813e289dd6 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Fri, 8 Feb 2019 11:47:35 -0200
Subject: [PATCH 1915/2974] add FEED_STORAGE_S3_ACL to default_settings.py file

---
 scrapy/settings/default_settings.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 3734a0a58..776c5af23 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -158,6 +158,8 @@ FEED_EXPORTERS_BASE = {
 }
 FEED_EXPORT_INDENT = 0
 
+FEED_STORAGE_S3_ACL = None
+
 FILES_STORE_S3_ACL = 'private'
 FILES_STORE_GCS_ACL = ''
 

From cfd183a9d19563f09487af942c9a635d665a1905 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Fri, 8 Feb 2019 14:49:26 -0200
Subject: [PATCH 1916/2974] no need to use get here since we're defining a
 default value in default_settings.py

---
 scrapy/extensions/feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index ca30322be..2b4594ad8 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -133,7 +133,7 @@ class S3FeedStorage(BlockingFeedStorage):
     def from_crawler(cls, crawler, uri):
         return cls(uri, crawler.settings['AWS_ACCESS_KEY_ID'],
                    crawler.settings['AWS_SECRET_ACCESS_KEY'],
-                   crawler.settings.get('FEED_STORAGE_S3_ACL'))
+                   crawler.settings['FEED_STORAGE_S3_ACL'])
 
     def _store_in_thread(self, file):
         file.seek(0)

From f824f5b2d17b082dac04505ba27afdfa869a11c7 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Fri, 8 Feb 2019 15:19:57 -0200
Subject: [PATCH 1917/2974] testing public method store instead of private
 method _store_in_thread

need to mock deferToThread function
---
 tests/test_feedexport.py | 36 ++++++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 520ca4a8f..e8c32ea43 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -237,7 +237,7 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, 'custom-acl')
 
-    def test_store_in_thread_botocore_without_acl(self):
+    def test_store_botocore_without_acl(self):
         storage = S3FeedStorage(
             's3://mybucket/export.csv',
             'access_key',
@@ -247,13 +247,19 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, None)
 
+        def _defer(f, *args, **kwargs):
+            return f(*args, **kwargs)
+
         with mock.patch('botocore.client.BaseClient._make_api_call') as m:
-            storage._store_in_thread(BytesIO(b'test file'))
+            with mock.patch('twisted.internet.threads.deferToThread',
+                            new=_defer):
+                storage.store(BytesIO(b'test file'))
+
             operation_name, api_params = m.call_args[0]
             self.assertEqual(operation_name, 'PutObject')
             self.assertNotIn('ACL', api_params)
 
-    def test_store_in_thread_botocore_with_acl(self):
+    def test_store_botocore_with_acl(self):
         storage = S3FeedStorage(
             's3://mybucket/export.csv',
             'access_key',
@@ -264,13 +270,19 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, 'custom-acl')
 
+        def _defer(f, *args, **kwargs):
+            return f(*args, **kwargs)
+
         with mock.patch('botocore.client.BaseClient._make_api_call') as m:
-            storage._store_in_thread(BytesIO(b'test file'))
+            with mock.patch('twisted.internet.threads.deferToThread',
+                            new=_defer):
+                storage.store(BytesIO(b'test file'))
+
             operation_name, api_params = m.call_args[0]
             self.assertEqual(operation_name, 'PutObject')
             self.assertEqual(api_params.get('ACL'), 'custom-acl')
 
-    def test_store_in_thread_not_botocore_without_acl(self):
+    def test_store_not_botocore_without_acl(self):
         storage = S3FeedStorage(
             's3://mybucket/export.csv',
             'access_key',
@@ -284,7 +296,11 @@ class S3FeedStorageTest(unittest.TestCase):
         storage.connect_s3 = mock.MagicMock()
         self.assertFalse(storage.is_botocore)
 
-        storage._store_in_thread(BytesIO(b'test file'))
+        def _defer(f, *args, **kwargs):
+            return f(*args, **kwargs)
+
+        with mock.patch('twisted.internet.threads.deferToThread', new=_defer):
+            storage.store(BytesIO(b'test file'))
 
         conn = storage.connect_s3(*storage.connect_s3.call_args)
         bucket = conn.get_bucket(*conn.get_bucket.call_args)
@@ -294,7 +310,7 @@ class S3FeedStorageTest(unittest.TestCase):
             key.set_contents_from_file.call_args
         )
 
-    def test_store_in_thread_not_botocore_with_acl(self):
+    def test_store_not_botocore_with_acl(self):
         storage = S3FeedStorage(
             's3://mybucket/export.csv',
             'access_key',
@@ -309,7 +325,11 @@ class S3FeedStorageTest(unittest.TestCase):
         storage.connect_s3 = mock.MagicMock()
         self.assertFalse(storage.is_botocore)
 
-        storage._store_in_thread(BytesIO(b'test file'))
+        def _defer(f, *args, **kwargs):
+            return f(*args, **kwargs)
+
+        with mock.patch('twisted.internet.threads.deferToThread', new=_defer):
+            storage.store(BytesIO(b'test file'))
 
         conn = storage.connect_s3(*storage.connect_s3.call_args)
         bucket = conn.get_bucket(*conn.get_bucket.call_args)

From 1eac2a163c2c734594d4f1e7e026eab309b2b0b5 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Fri, 8 Feb 2019 16:50:39 -0200
Subject: [PATCH 1918/2974] simplifying how we deal with threads.deferToThread
 calls

---
 tests/test_feedexport.py | 30 ++++++++----------------------
 1 file changed, 8 insertions(+), 22 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e8c32ea43..0f31ef00e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -237,6 +237,7 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, 'custom-acl')
 
+    @defer.inlineCallbacks
     def test_store_botocore_without_acl(self):
         storage = S3FeedStorage(
             's3://mybucket/export.csv',
@@ -247,18 +248,14 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, None)
 
-        def _defer(f, *args, **kwargs):
-            return f(*args, **kwargs)
-
         with mock.patch('botocore.client.BaseClient._make_api_call') as m:
-            with mock.patch('twisted.internet.threads.deferToThread',
-                            new=_defer):
-                storage.store(BytesIO(b'test file'))
+            yield storage.store(BytesIO(b'test file'))
 
             operation_name, api_params = m.call_args[0]
             self.assertEqual(operation_name, 'PutObject')
             self.assertNotIn('ACL', api_params)
 
+    @defer.inlineCallbacks
     def test_store_botocore_with_acl(self):
         storage = S3FeedStorage(
             's3://mybucket/export.csv',
@@ -270,18 +267,14 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, 'custom-acl')
 
-        def _defer(f, *args, **kwargs):
-            return f(*args, **kwargs)
-
         with mock.patch('botocore.client.BaseClient._make_api_call') as m:
-            with mock.patch('twisted.internet.threads.deferToThread',
-                            new=_defer):
-                storage.store(BytesIO(b'test file'))
+            yield storage.store(BytesIO(b'test file'))
 
             operation_name, api_params = m.call_args[0]
             self.assertEqual(operation_name, 'PutObject')
             self.assertEqual(api_params.get('ACL'), 'custom-acl')
 
+    @defer.inlineCallbacks
     def test_store_not_botocore_without_acl(self):
         storage = S3FeedStorage(
             's3://mybucket/export.csv',
@@ -296,11 +289,7 @@ class S3FeedStorageTest(unittest.TestCase):
         storage.connect_s3 = mock.MagicMock()
         self.assertFalse(storage.is_botocore)
 
-        def _defer(f, *args, **kwargs):
-            return f(*args, **kwargs)
-
-        with mock.patch('twisted.internet.threads.deferToThread', new=_defer):
-            storage.store(BytesIO(b'test file'))
+        yield storage.store(BytesIO(b'test file'))
 
         conn = storage.connect_s3(*storage.connect_s3.call_args)
         bucket = conn.get_bucket(*conn.get_bucket.call_args)
@@ -310,6 +299,7 @@ class S3FeedStorageTest(unittest.TestCase):
             key.set_contents_from_file.call_args
         )
 
+    @defer.inlineCallbacks
     def test_store_not_botocore_with_acl(self):
         storage = S3FeedStorage(
             's3://mybucket/export.csv',
@@ -325,11 +315,7 @@ class S3FeedStorageTest(unittest.TestCase):
         storage.connect_s3 = mock.MagicMock()
         self.assertFalse(storage.is_botocore)
 
-        def _defer(f, *args, **kwargs):
-            return f(*args, **kwargs)
-
-        with mock.patch('twisted.internet.threads.deferToThread', new=_defer):
-            storage.store(BytesIO(b'test file'))
+        yield storage.store(BytesIO(b'test file'))
 
         conn = storage.connect_s3(*storage.connect_s3.call_args)
         bucket = conn.get_bucket(*conn.get_bucket.call_args)

From 03e61b9908733f085d87da6bd29152389961b81a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 1 Feb 2019 13:50:01 +0100
Subject: [PATCH 1919/2974] Check that spidercls arguments in scrapy.crawler
 classes are not spider objects

---
 scrapy/crawler.py                            | 13 +++++++++++++
 tests/test_crawler.py                        | 13 +++++++++++++
 tests/test_downloadermiddleware_httpproxy.py |  2 +-
 3 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 04aee18ed..ee00d27b4 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -7,6 +7,7 @@ import sys
 from twisted.internet import reactor, defer
 from zope.interface.verify import verifyClass, DoesNotImplement
 
+from scrapy import Spider
 from scrapy.core.engine import ExecutionEngine
 from scrapy.resolver import CachingThreadedResolver
 from scrapy.interfaces import ISpiderLoader
@@ -27,6 +28,10 @@ logger = logging.getLogger(__name__)
 class Crawler(object):
 
     def __init__(self, spidercls, settings=None):
+        if isinstance(spidercls, Spider):
+            raise ValueError(
+                'The spidercls argument must be a class, not an object')
+
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
 
@@ -168,6 +173,10 @@ class CrawlerRunner(object):
 
         :param dict kwargs: keyword arguments to initialize the spider
         """
+        if isinstance(crawler_or_spidercls, Spider):
+            raise ValueError(
+                'The crawler_or_spidercls argument cannot be a spider object, '
+                'it must be a spider class (or a Crawler object)')
         crawler = self.create_crawler(crawler_or_spidercls)
         return self._crawl(crawler, *args, **kwargs)
 
@@ -195,6 +204,10 @@ class CrawlerRunner(object):
           a spider with this name in a Scrapy project (using spider loader),
           then creates a Crawler instance for it.
         """
+        if isinstance(crawler_or_spidercls, Spider):
+            raise ValueError(
+                'The crawler_or_spidercls argument cannot be a spider object, '
+                'it must be a spider class (or a Crawler object)')
         if isinstance(crawler_or_spidercls, Crawler):
             return crawler_or_spidercls
         return self._create_crawler(crawler_or_spidercls)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 268948a70..37cea3ad3 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -4,6 +4,7 @@ import warnings
 
 from twisted.internet import defer
 from twisted.trial import unittest
+from pytest import raises
 
 import scrapy
 from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
@@ -66,6 +67,10 @@ class CrawlerTestCase(BaseCrawlerTest):
         crawler = Crawler(DefaultSpider)
         self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
 
+    def test_crawler_rejects_spider_objects(self):
+        with raises(ValueError):
+            Crawler(DefaultSpider())
+
 
 class SpiderSettingsTestCase(unittest.TestCase):
     def test_spider_custom_settings(self):
@@ -177,6 +182,14 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
             self.assertEqual(len(w), 1)
             self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message))
 
+    def test_crawl_rejects_spider_objects(self):
+        with raises(ValueError):
+            CrawlerRunner().crawl(DefaultSpider())
+
+    def test_create_crawler_rejects_spider_objects(self):
+        with raises(ValueError):
+            CrawlerRunner().create_crawler(DefaultSpider())
+
 
 class CrawlerProcessTest(BaseCrawlerTest):
     def test_crawler_process_accepts_dict(self):
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 537126613..30920b2da 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -25,7 +25,7 @@ class TestHttpProxyMiddleware(TestCase):
 
     def test_not_enabled(self):
         settings = Settings({'HTTPPROXY_ENABLED': False})
-        crawler = Crawler(spider, settings)
+        crawler = Crawler(Spider, settings)
         self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler))
 
     def test_no_environment_proxies(self):

From 7c9f0bd86c5f02ea803fa6bf1242d34d9c9f47d5 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Tue, 12 Feb 2019 12:19:30 -0200
Subject: [PATCH 1920/2974] using named params with optional amazon s3 params

---
 scrapy/extensions/feedexport.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 2b4594ad8..f6bc460ea 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -131,9 +131,12 @@ class S3FeedStorage(BlockingFeedStorage):
 
     @classmethod
     def from_crawler(cls, crawler, uri):
-        return cls(uri, crawler.settings['AWS_ACCESS_KEY_ID'],
-                   crawler.settings['AWS_SECRET_ACCESS_KEY'],
-                   crawler.settings['FEED_STORAGE_S3_ACL'])
+        return cls(
+            uri=uri,
+            access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
+            secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
+            acl=crawler.settings['FEED_STORAGE_S3_ACL']
+        )
 
     def _store_in_thread(self, file):
         file.seek(0)

From c2dede27bd56bd783c45fb7302ca06b7c2c025c0 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Tue, 12 Feb 2019 12:22:05 -0200
Subject: [PATCH 1921/2974] reduce code with simple ternary operator

---
 scrapy/extensions/feedexport.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index f6bc460ea..40f985f19 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -141,10 +141,7 @@ class S3FeedStorage(BlockingFeedStorage):
     def _store_in_thread(self, file):
         file.seek(0)
         if self.is_botocore:
-            kwargs = dict()
-            if self.acl:
-                kwargs.update(dict(ACL=self.acl))
-
+            kwargs = {'ACL': self.acl} if self.acl else {}
             self.s3_client.put_object(
                 Bucket=self.bucketname, Key=self.keyname, Body=file,
                 **kwargs)
@@ -152,10 +149,7 @@ class S3FeedStorage(BlockingFeedStorage):
             conn = self.connect_s3(self.access_key, self.secret_key)
             bucket = conn.get_bucket(self.bucketname, validate=False)
             key = bucket.new_key(self.keyname)
-            kwargs = dict()
-            if self.acl:
-                kwargs.update(dict(policy=self.acl))
-
+            kwargs = {'policy': self.acl} if self.acl else {}
             key.set_contents_from_file(file, **kwargs)
             key.close()
 

From 984e706fd2e06457bcdd1226366d08950ac101b0 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Tue, 12 Feb 2019 12:26:57 -0200
Subject: [PATCH 1922/2974] using blank string instead of None as default value
 as proposed by @kmike

---
 docs/topics/feed-exports.rst        | 2 +-
 scrapy/extensions/feedexport.py     | 2 +-
 scrapy/settings/default_settings.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index dee0c3ffa..cf70b8aca 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -312,7 +312,7 @@ The keys are URI schemes and the values are paths to storage classes.
 FEED_STORAGE_S3_ACL
 -------------------
 
-Default: ``None``
+Default: ``''`` (empty string)
 
 A string containing a custom ACL for feeds exported to Amazon S3 by your project.
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 40f985f19..975fa1229 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -135,7 +135,7 @@ class S3FeedStorage(BlockingFeedStorage):
             uri=uri,
             access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
             secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
-            acl=crawler.settings['FEED_STORAGE_S3_ACL']
+            acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None
         )
 
     def _store_in_thread(self, file):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 776c5af23..a800d39ab 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -158,7 +158,7 @@ FEED_EXPORTERS_BASE = {
 }
 FEED_EXPORT_INDENT = 0
 
-FEED_STORAGE_S3_ACL = None
+FEED_STORAGE_S3_ACL = ''
 
 FILES_STORE_S3_ACL = 'private'
 FILES_STORE_GCS_ACL = ''

From 04ccf79e38561a2175c18440b3b4a53ba2f4992f Mon Sep 17 00:00:00 2001
From: Pedro Sousa <pedrotorredesousa@gmail.com>
Date: Wed, 13 Feb 2019 15:39:45 +0000
Subject: [PATCH 1923/2974] A different S3 Endpoint URL is now possible when
 uploading images

---
 scrapy/pipelines/images.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 95323c613..8338a6281 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -89,6 +89,7 @@ class ImagesPipeline(FilesPipeline):
         s3store = cls.STORE_SCHEMES['s3']
         s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
+        s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']
         s3store.POLICY = settings['IMAGES_STORE_S3_ACL']
 
         gcs_store = cls.STORE_SCHEMES['gs']

From 50bf4c60c480a276651ff703cf9fed8e7f981d20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 13 Feb 2019 17:39:20 +0100
Subject: [PATCH 1924/2974] Document that the main entry point of downloader
 and spider middlewares is from_crawler()

---
 docs/topics/downloader-middleware.rst | 8 ++++++--
 docs/topics/spider-middleware.rst     | 8 ++++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 8dbe249fa..18a0639ce 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -55,8 +55,12 @@ particular setting. See each middleware documentation for more info.
 Writing your own downloader middleware
 ======================================
 
-Each middleware component is a Python class that defines one or
-more of the following methods:
+Each downloader middleware is a Python class that defines one or more of the
+methods defined below.
+
+The main entry point is the ``from_crawler`` class method, which receives a
+:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler`
+object gives you access, for example, to the :ref:`settings <topics-settings>`.
 
 .. module:: scrapy.downloadermiddlewares
 
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 2b7e42771..62b5ca0e8 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -57,8 +57,12 @@ particular setting. See each middleware documentation for more info.
 Writing your own spider middleware
 ==================================
 
-Each middleware component is a Python class that defines one or more of the
-following methods:
+Each spider middleware is a Python class that defines one or more of the
+methods defined below.
+
+The main entry point is the ``from_crawler`` class method, which receives a
+:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler`
+object gives you access, for example, to the :ref:`settings <topics-settings>`.
 
 .. module:: scrapy.spidermiddlewares
 

From 430e9392483b3992c16bd0314f1bcaed91a9d392 Mon Sep 17 00:00:00 2001
From: Pedro Sousa <pedrotorredesousa@gmail.com>
Date: Wed, 13 Feb 2019 19:59:40 +0000
Subject: [PATCH 1925/2974] Added missing AWS Settings for ImagesPipeline

---
 scrapy/pipelines/images.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 8338a6281..a1457c7e9 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -90,6 +90,9 @@ class ImagesPipeline(FilesPipeline):
         s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
         s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']
+        s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']
+        s3store.AWS_USE_SSL = settings['AWS_USE_SSL']
+        s3store.AWS_VERIFY = settings['AWS_VERIFY']
         s3store.POLICY = settings['IMAGES_STORE_S3_ACL']
 
         gcs_store = cls.STORE_SCHEMES['gs']

From b4d132b9f0824263d83331d0b36870f6f64918e4 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Wed, 13 Feb 2019 19:21:14 -0200
Subject: [PATCH 1926/2974] setting botocore version as described in debian
 jessie website

https://packages.debian.org/en/jessie/python-botocore
---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index f2f3e1293..584da2dcd 100644
--- a/tox.ini
+++ b/tox.ini
@@ -47,7 +47,7 @@ deps =
     lxml==3.4.0
     Twisted==14.0.2
     boto==2.34.0
-    botocore==1.12.89
+    botocore==0.62
     Pillow==2.6.1
     cssselect==0.9.1
     zope.interface==4.1.1

From dc0b643832e9f3400e432c2ef7a34e6c75ac8366 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Wed, 13 Feb 2019 19:44:50 -0200
Subject: [PATCH 1927/2974] refactoring tests to avoid mocking private method

---
 tests/test_feedexport.py | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 0f31ef00e..c103593f9 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -248,12 +248,9 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, None)
 
-        with mock.patch('botocore.client.BaseClient._make_api_call') as m:
-            yield storage.store(BytesIO(b'test file'))
-
-            operation_name, api_params = m.call_args[0]
-            self.assertEqual(operation_name, 'PutObject')
-            self.assertNotIn('ACL', api_params)
+        storage.s3_client = mock.MagicMock()
+        yield storage.store(BytesIO(b'test file'))
+        self.assertNotIn('ACL', storage.s3_client.put_object.call_args[1])
 
     @defer.inlineCallbacks
     def test_store_botocore_with_acl(self):
@@ -267,12 +264,12 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, 'custom-acl')
 
-        with mock.patch('botocore.client.BaseClient._make_api_call') as m:
-            yield storage.store(BytesIO(b'test file'))
-
-            operation_name, api_params = m.call_args[0]
-            self.assertEqual(operation_name, 'PutObject')
-            self.assertEqual(api_params.get('ACL'), 'custom-acl')
+        storage.s3_client = mock.MagicMock()
+        yield storage.store(BytesIO(b'test file'))
+        self.assertEqual(
+            storage.s3_client.put_object.call_args[1].get('ACL'),
+            'custom-acl'
+        )
 
     @defer.inlineCallbacks
     def test_store_not_botocore_without_acl(self):

From ea8be627d15aa6fe1beaf50fff666cbeb161d94d Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Wed, 13 Feb 2019 19:53:10 -0200
Subject: [PATCH 1928/2974] botocore is not supported on debian jessie

---
 tests/test_feedexport.py | 7 ++++++-
 tox.ini                  | 1 -
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c103593f9..2bf57e278 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -18,7 +18,6 @@ from tests import mock
 from tests.mockserver import MockServer
 from w3lib.url import path_to_file_uri
 
-import botocore.client
 import scrapy
 from scrapy.exporters import CsvItemExporter
 from scrapy.extensions.feedexport import (
@@ -239,6 +238,9 @@ class S3FeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_store_botocore_without_acl(self):
+        if os.getenv('TOX_ENV_NAME') == 'jessie':
+            raise unittest.SkipTest('botocore is not supported on jessie')
+
         storage = S3FeedStorage(
             's3://mybucket/export.csv',
             'access_key',
@@ -254,6 +256,9 @@ class S3FeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_store_botocore_with_acl(self):
+        if os.getenv('TOX_ENV_NAME') == 'jessie':
+            raise unittest.SkipTest('botocore is not supported on jessie')
+
         storage = S3FeedStorage(
             's3://mybucket/export.csv',
             'access_key',
diff --git a/tox.ini b/tox.ini
index 584da2dcd..0c0f8f7b7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -47,7 +47,6 @@ deps =
     lxml==3.4.0
     Twisted==14.0.2
     boto==2.34.0
-    botocore==0.62
     Pillow==2.6.1
     cssselect==0.9.1
     zope.interface==4.1.1

From 9b8ba4c383df0f3029d1b07ab9647a7d902600f4 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Thu, 14 Feb 2019 16:20:56 -0200
Subject: [PATCH 1929/2974] try to import botocore before runing some tests

---
 tests/test_feedexport.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 2bf57e278..3ff79c912 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -238,8 +238,10 @@ class S3FeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_store_botocore_without_acl(self):
-        if os.getenv('TOX_ENV_NAME') == 'jessie':
-            raise unittest.SkipTest('botocore is not supported on jessie')
+        try:
+            import botocore
+        except ImportError:
+            raise unittest.SkipTest('botocore is required')
 
         storage = S3FeedStorage(
             's3://mybucket/export.csv',
@@ -256,8 +258,10 @@ class S3FeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_store_botocore_with_acl(self):
-        if os.getenv('TOX_ENV_NAME') == 'jessie':
-            raise unittest.SkipTest('botocore is not supported on jessie')
+        try:
+            import botocore
+        except ImportError:
+            raise unittest.SkipTest('botocore is required')
 
         storage = S3FeedStorage(
             's3://mybucket/export.csv',

From b02d26fae8892775ad6ef306d80b02e6bc69d12e Mon Sep 17 00:00:00 2001
From: John de la Garza <john@jjdev.com>
Date: Fri, 15 Feb 2019 16:54:19 -0800
Subject: [PATCH 1931/2974] rel_has_nofollow: remove redundant if statement

---
 scrapy/utils/misc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 5ccfdcd72..6de36d45c 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -116,7 +116,7 @@ def md5sum(file):
 
 def rel_has_nofollow(rel):
     """Return True if link rel attribute has nofollow type"""
-    return True if rel is not None and 'nofollow' in rel.split() else False
+    return rel is not None and 'nofollow' in rel.split()
 
 
 def create_instance(objcls, settings, crawler, *args, **kwargs):

From 0bb3d8ca93cf68e0ef231ee734f8a1a3c4075a72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Wed, 27 Feb 2019 18:41:01 -0300
Subject: [PATCH 1932/2974] Updating Google Cloud Storage scheme to gs instead
 of gcs

---
 docs/topics/feed-exports.rst        | 4 ++--
 scrapy/settings/default_settings.py | 2 +-
 tests/test_feedexport.py            | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index efb63b0ba..0957a5997 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -192,10 +192,10 @@ Google Cloud Storage (GCS)
 
 The feeds are stored on `Google Cloud Storage`_.
 
- * URI scheme: ``gcs``
+ * URI scheme: ``gs``
  * Example URIs:
 
-   * ``gcs://mybucket/path/to/export.csv``
+   * ``gs://mybucket/path/to/export.csv``
 
  * Required external libraries: `google-cloud-storage <https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python>`_.
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index c17e94a64..50fcd1d0a 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -143,7 +143,7 @@ FEED_STORAGES_BASE = {
     '': 'scrapy.extensions.feedexport.FileFeedStorage',
     'file': 'scrapy.extensions.feedexport.FileFeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
-    'gcs': 'scrapy.extensions.feedexport.GCSFeedStorage',
+    'gs': 'scrapy.extensions.feedexport.GCSFeedStorage',
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
 }
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 5cbca6d28..41df7d7af 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -200,7 +200,7 @@ class GCSFeedStorageTest(unittest.TestCase):
 
         settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' }
         crawler = get_crawler(settings_dict=settings)
-        storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv')
+        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
         assert storage.project_id == '123'
         assert storage.acl == 'publicRead'
         assert storage.bucket_name == 'mybucket'

From e3b15252c80ca3d0872f3068c382a0a3e7cc9db6 Mon Sep 17 00:00:00 2001
From: Matthieu Grandrie <matthieu.cham@gmail.com>
Date: Thu, 21 Feb 2019 17:19:58 +0100
Subject: [PATCH 1933/2974] New constructor arg *restrict_text* for
 FilteringLinkExtractor.

Same as allow and deny args, it holds a string, a regex or an iterable of. Links whose text don't match one of the regex are filtered out.
DOC restrict_text in LxmlLinkExtractor
---
 docs/topics/link-extractors.rst   |  6 ++++++
 scrapy/linkextractors/__init__.py |  6 +++++-
 scrapy/linkextractors/lxmlhtml.py |  9 +++++----
 scrapy/linkextractors/sgml.py     | 13 +++++++------
 tests/test_linkextractors.py      | 24 ++++++++++++++++++++++++
 5 files changed, 47 insertions(+), 11 deletions(-)

diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index f40a36d31..713a94e10 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -93,6 +93,12 @@ LxmlLinkExtractor
         Has the same behaviour as ``restrict_xpaths``.
     :type restrict_css: str or list
 
+    :param restrict_text: a single regular expression (or list of regular expressions)
+        that the link's text must match in order to be extracted. If not
+        given (or empty), it will match all links. If a list of regular expressions is
+        given, the link will be extracted if it matches at least one.
+    :type restrict_text: a regular expression (or list of)
+
     :param tags: a tag or a list of tags to consider when extracting links.
         Defaults to ``('a', 'area')``.
     :type tags: str or list
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 97e8c0af1..ebf3cd7d8 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -50,7 +50,7 @@ class FilteringLinkExtractor(object):
     _csstranslator = HTMLTranslator()
 
     def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
-                 restrict_xpaths, canonicalize, deny_extensions, restrict_css):
+                 restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):
 
         self.link_extractor = link_extractor
 
@@ -70,6 +70,8 @@ class FilteringLinkExtractor(object):
         if deny_extensions is None:
             deny_extensions = IGNORED_EXTENSIONS
         self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)}
+        self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x)
+                              for x in arg_to_iter(restrict_text)]
 
     def _link_allowed(self, link):
         if not _is_valid_url(link.url):
@@ -85,6 +87,8 @@ class FilteringLinkExtractor(object):
             return False
         if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions):
             return False
+        if self.restrict_text and not _matches(link.text, self.restrict_text):
+            return False
         return True
 
     def matches(self, url):
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index a7092f9b8..8f6f93a44 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -97,7 +97,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
                  tags=('a', 'area'), attrs=('href',), canonicalize=False,
                  unique=True, process_value=None, deny_extensions=None, restrict_css=(),
-                 strip=True):
+                 strip=True, restrict_text=None):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
         tag_func = lambda x: x in tags
         attr_func = lambda x: x in attrs
@@ -111,9 +111,10 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
         )
 
         super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
-            allow_domains=allow_domains, deny_domains=deny_domains,
-            restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
-            canonicalize=canonicalize, deny_extensions=deny_extensions)
+                                                allow_domains=allow_domains, deny_domains=deny_domains,
+                                                restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
+                                                canonicalize=canonicalize, deny_extensions=deny_extensions,
+                                                restrict_text=restrict_text)
 
     def extract_links(self, response):
         base_url = get_base_url(response)
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 5fa6b771c..8940a4d77 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -113,7 +113,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
                  tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True,
                  process_value=None, deny_extensions=None, restrict_css=(),
-                 strip=True):
+                 strip=True, restrict_text=()):
         warnings.warn(
             "SgmlLinkExtractor is deprecated and will be removed in future releases. "
             "Please use scrapy.linkextractors.LinkExtractor",
@@ -127,13 +127,14 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
         with warnings.catch_warnings():
             warnings.simplefilter('ignore', ScrapyDeprecationWarning)
             lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
-                unique=unique, process_value=process_value, strip=strip,
-                canonicalized=canonicalize)
+                                       unique=unique, process_value=process_value, strip=strip,
+                                       canonicalized=canonicalize)
 
         super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
-            allow_domains=allow_domains, deny_domains=deny_domains,
-            restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
-            canonicalize=canonicalize, deny_extensions=deny_extensions)
+                                                allow_domains=allow_domains, deny_domains=deny_domains,
+                                                restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
+                                                canonicalize=canonicalize, deny_extensions=deny_extensions,
+                                                restrict_text=restrict_text)
 
     def extract_links(self, response):
         base_url = None
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 903032b52..c9cd629f4 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -479,6 +479,30 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
             Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
         ])
 
+    def test_link_restrict_text(self):
+        html = b"""
+        <a href="http://example.org/item1.html">Pic of a cat</a>
+        <a href="http://example.org/item2.html">Pic of a dog</a>
+        <a href="http://example.org/item3.html">Pic of a cow</a>
+        """
+        response = HtmlResponse("http://example.org/index.html", body=html)
+        # Simple text inclusion test
+        lx = self.extractor_cls(restrict_text='dog')
+        self.assertEqual([link for link in lx.extract_links(response)], [
+            Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+        ])
+        # Unique regex test
+        lx = self.extractor_cls(restrict_text=r'of.*dog')
+        self.assertEqual([link for link in lx.extract_links(response)], [
+            Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+        ])
+        # Multiple regex test
+        lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat'])
+        self.assertEqual([link for link in lx.extract_links(response)], [
+            Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False),
+            Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+        ])
+
     @pytest.mark.xfail
     def test_restrict_xpaths_with_html_entities(self):
         super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()

From 858f5be74728209d8ef71794296814abca4c1c93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 1 Mar 2019 16:10:23 +0100
Subject: [PATCH 1934/2974] =?UTF-8?q?backwards=20=E2=86=92=20backward=20(a?=
 =?UTF-8?q?dj.)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/news.rst                           | 64 ++++++++++++-------------
 docs/topics/request-response.rst        |  2 +-
 docs/topics/spiders.rst                 |  2 +-
 docs/versioning.rst                     |  2 +-
 scrapy/cmdline.py                       |  4 +-
 scrapy/conf.py                          |  2 +-
 scrapy/core/downloader/handlers/http.py |  2 +-
 scrapy/extensions/feedexport.py         |  4 +-
 scrapy/log.py                           |  2 +-
 scrapy/signals.py                       |  2 +-
 scrapy/utils/conf.py                    |  4 +-
 sep/sep-018.rst                         |  2 +-
 tests/test_downloader_handlers.py       |  2 +-
 tests/test_feedexport.py                |  2 +-
 tests/test_utils_conf.py                |  2 +-
 15 files changed, 49 insertions(+), 49 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 668473887..7ac1664fe 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -53,7 +53,7 @@ case of Scrapy spiders: callbacks are usually called several times,
 on different pages.
 
 If you're using custom ``Selector`` or ``SelectorList`` subclasses,
-a **backwards incompatible** change in parsel may affect your code.
+a **backward incompatible** change in parsel may affect your code.
 See `parsel changelog`_ for a detailed description, as well as for the
 full list of improvements.
 
@@ -62,7 +62,7 @@ full list of improvements.
 Telnet console
 ~~~~~~~~~~~~~~
 
-**Backwards incompatible**: Scrapy's telnet console now requires username
+**Backward incompatible**: Scrapy's telnet console now requires username
 and password. See :ref:`topics-telnetconsole` for more details. This change
 fixes a **security issue**; see :ref:`release-1.5.2` release notes for details.
 
@@ -209,7 +209,7 @@ Scrapy 1.5.2 (2019-01-22)
   exploit it from Scrapy, but it is very easy to trick a browser to do so and
   elevates the risk for local development environment.
 
-  *The fix is backwards incompatible*, it enables telnet user-password
+  *The fix is backward incompatible*, it enables telnet user-password
   authentication by default with a random generated password. If you can't
   upgrade right away, please consider setting :setting:`TELNET_CONSOLE_PORT`
   out of its default value.
@@ -256,15 +256,15 @@ Some highlights:
 * Better default handling of HTTP 308, 522 and 524 status codes.
 * Documentation is improved, as usual.
 
-Backwards Incompatible Changes
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Backward Incompatible Changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 * Scrapy 1.5 drops support for Python 3.3.
 * Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`).
-  **This is technically backwards-incompatible**; override
+  **This is technically backward-incompatible**; override
   :setting:`USER_AGENT` if you relied on old value.
 * Logging of settings overridden by ``custom_settings`` is fixed;
-  **this is technically backwards-incompatible** because the logger
+  **this is technically backward-incompatible** because the logger
   changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``. If you're
   parsing Scrapy logs, please update your log parsers (:issue:`1343`).
 * LinkExtractor now ignores ``m4v`` extension by default, this is change
@@ -301,11 +301,11 @@ Bug fixes
 ~~~~~~~~~
 
 - Fix logging of settings overridden by ``custom_settings``;
-  **this is technically backwards-incompatible** because the logger
+  **this is technically backward-incompatible** because the logger
   changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``, so please
   update your log parsers if needed (:issue:`1343`)
 - Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`).
-  **This is technically backwards-incompatible**; override
+  **This is technically backward-incompatible**; override
   :setting:`USER_AGENT` if you relied on old value.
 - Fix PyPy and PyPy3 test failures, support them officially
   (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`,
@@ -415,18 +415,18 @@ offset, using the new :setting:`FEED_EXPORT_INDENT` setting.
 
 Enjoy! (Or read on for the rest of changes in this release.)
 
-Deprecations and Backwards Incompatible Changes
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Deprecations and Backward Incompatible Changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 - Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor`
   (:issue:`2537`, fixes :issue:`1941` and :issue:`1982`):
-  **warning, this is technically backwards-incompatible**
+  **warning, this is technically backward-incompatible**
 - Enable memusage extension by default (:issue:`2539`, fixes :issue:`2187`);
-  **this is technically backwards-incompatible** so please check if you have
+  **this is technically backward-incompatible** so please check if you have
   any non-default ``MEMUSAGE_***`` options set.
 - ``EDITOR`` environment variable now takes precedence over ``EDITOR``
   option defined in settings.py (:issue:`1829`); Scrapy default settings
-  no longer depend on environment variables. **This is technically a backwards
+  no longer depend on environment variables. **This is technically a backward
   incompatible change**.
 - ``Spider.make_requests_from_url`` is deprecated
   (:issue:`1728`, fixes :issue:`1495`).
@@ -636,10 +636,10 @@ New Features
   scrapy shell now follow HTTP redirections by default (:issue:`2290`);
   See :command:`fetch` and :command:`shell` for details.
 - ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``;
-  this is technically **backwards incompatible** so please check your log parsers.
+  this is technically **backward incompatible** so please check your log parsers.
 - By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``,
   instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``);
-  this is **backwards incompatible** if you have log parsers expecting the short
+  this is **backward incompatible** if you have log parsers expecting the short
   logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES`
   set to ``True``.
 
@@ -750,11 +750,11 @@ Bug fixes
 ~~~~~~~~~
 
 - DefaultRequestHeaders middleware now runs before UserAgent middleware
-  (:issue:`2088`). **Warning: this is technically backwards incompatible**,
+  (:issue:`2088`). **Warning: this is technically backward incompatible**,
   though we consider this a bug fix.
 - HTTP cache extension and plugins that use the ``.scrapy`` data directory now
   work outside projects (:issue:`1581`).  **Warning: this is technically
-  backwards incompatible**, though we consider this a bug fix.
+  backward incompatible**, though we consider this a bug fix.
 - ``Selector`` does not allow passing both ``response`` and ``text`` anymore
   (:issue:`2153`).
 - Fixed logging of wrong callback name with ``scrapy parse`` (:issue:`2169`).
@@ -934,13 +934,13 @@ This 1.1 release brings a lot of interesting features and bug fixes:
   - Accept XML node names containing dots as valid (:issue:`1533`).
   - When uploading files or images to S3 (with ``FilesPipeline`` or
     ``ImagesPipeline``), the default ACL policy is now "private" instead
-    of "public" **Warning: backwards incompatible!**.
+    of "public" **Warning: backward incompatible!**.
     You can use :setting:`FILES_STORE_S3_ACL` to change it.
   - We've reimplemented ``canonicalize_url()`` for more correct output,
     especially for URLs with non-ASCII characters (:issue:`1947`).
     This could change link extractors output compared to previous scrapy versions.
     This may also invalidate some cache entries you could still have from pre-1.1 runs.
-    **Warning: backwards incompatible!**.
+    **Warning: backward incompatible!**.
 
 Keep reading for more details on other improvements and bug fixes.
 
@@ -973,7 +973,7 @@ Additional New Features and Enhancements
   - Support for bpython and configure preferred Python shell via
     ``SCRAPY_PYTHON_SHELL`` (:issue:`1100`, :issue:`1444`).
   - Support URLs without scheme (:issue:`1498`)
-    **Warning: backwards incompatible!**
+    **Warning: backward incompatible!**
   - Bring back support for relative file path (:issue:`1710`, :issue:`1550`).
 
 - Added :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` setting to change default check
@@ -1056,7 +1056,7 @@ Bugfixes
 ~~~~~~~~
 
 - Scrapy does not retry requests that got a ``HTTP 400 Bad Request``
-  response anymore (:issue:`1289`). **Warning: backwards incompatible!**
+  response anymore (:issue:`1289`). **Warning: backward incompatible!**
 - Support empty password for http_proxy config (:issue:`1274`).
 - Interpret ``application/x-json`` as ``TextResponse`` (:issue:`1333`).
 - Support link rel attribute with multiple values (:issue:`1201`).
@@ -1646,7 +1646,7 @@ Scrapy 0.24.2 (2014-07-08)
 Scrapy 0.24.1 (2014-06-27)
 --------------------------
 
-- Fix deprecated CrawlerSettings and increase backwards compatibility with
+- Fix deprecated CrawlerSettings and increase backward compatibility with
   .defaults attribute (:commit:`8e3f20a`)
 
 
@@ -1772,7 +1772,7 @@ Scrapy 0.22.0 (released 2014-01-17)
 Enhancements
 ~~~~~~~~~~~~
 
-- [**Backwards incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`)
+- [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`)
   To restore old backend set `HTTPCACHE_STORAGE` to `scrapy.contrib.httpcache.DbmCacheStorage`
 - Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`)
 - Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`)
@@ -2092,7 +2092,7 @@ Scrapy 0.16.1 (released 2012-10-26)
 -----------------------------------
 
 - fixed LogStats extension, which got broken after a wrong merge before the 0.16 release (:commit:`8c780fd`)
-- better backwards compatibility for scrapy.conf.settings (:commit:`3403089`)
+- better backward compatibility for scrapy.conf.settings (:commit:`3403089`)
 - extended documentation on how to access crawler stats from extensions (:commit:`c4da0b5`)
 - removed .hgtags (no longer needed now that scrapy uses git) (:commit:`d52c188`)
 - fix dashes under rst headers (:commit:`fa4f7f9`)
@@ -2107,7 +2107,7 @@ Scrapy changes:
 - added :ref:`topics-contracts`, a mechanism for testing spiders in a formal/reproducible way
 - added options ``-o`` and ``-t`` to the :command:`runspider` command
 - documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED`
-- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backwards compatibility is kept on the Stats Collector API and signals.
+- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals.
 - added :meth:`~scrapy.contrib.spidermiddleware.SpiderMiddleware.process_start_requests` method to spider middlewares
 - dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
 - dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
@@ -2259,7 +2259,7 @@ Code rearranged and removed
 - Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`)
 - removed ``CONCURRENT_SPIDERS`` setting (use scrapyd maxproc instead) (:rev:`2789`)
 - Renamed attributes of core components: downloader.sites -> downloader.slots, scraper.sites -> scraper.slots (:rev:`2717`, :rev:`2718`)
-- Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backwards compatibility kept.
+- Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backward compatibility kept.
 
 Scrapy 0.12
 -----------
@@ -2356,11 +2356,11 @@ API changes
    - ``scrapy.stats.collector.SimpledbStatsCollector`` to ``scrapy.contrib.statscol.SimpledbStatsCollector``
 - default per-command settings are now specified in the ``default_settings`` attribute of command object class (#201)
 - changed arguments of Item pipeline ``process_item()`` method from ``(spider, item)`` to ``(item, spider)``
-   - backwards compatibility kept (with deprecation warning)
+   - backward compatibility kept (with deprecation warning)
 - moved ``scrapy.core.signals`` module to ``scrapy.signals``
-   - backwards compatibility kept (with deprecation warning)
+   - backward compatibility kept (with deprecation warning)
 - moved ``scrapy.core.exceptions`` module to ``scrapy.exceptions``
-   - backwards compatibility kept (with deprecation warning)
+   - backward compatibility kept (with deprecation warning)
 - added ``handles_request()`` class method to ``BaseSpider``
 - dropped ``scrapy.log.exc()`` function (use ``scrapy.log.err()`` instead)
 - dropped ``component`` argument of ``scrapy.log.msg()`` function
@@ -2431,8 +2431,8 @@ New features
 - Added support for HTTP proxies (``HttpProxyMiddleware``) (:rev:`1781`, :rev:`1785`)
 - Offsite spider middleware now logs messages when filtering out requests (:rev:`1841`)
 
-Backwards-incompatible changes
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 - Changed ``scrapy.utils.response.get_meta_refresh()`` signature (:rev:`1804`)
 - Removed deprecated ``scrapy.item.ScrapedItem`` class - use ``scrapy.item.Item instead`` (:rev:`1838`)
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 76360b15f..4511f3469 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -728,7 +728,7 @@ TextResponse objects
     .. method:: TextResponse.body_as_unicode()
 
         The same as :attr:`text`, but available as a method. This method is
-        kept for backwards compatibility; please prefer ``response.text``.
+        kept for backward compatibility; please prefer ``response.text``.
 
 
 HtmlResponse objects
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 742a88659..e1d36aa24 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -190,7 +190,7 @@ scrapy.Spider
    .. method:: log(message, [level, component])
 
        Wrapper that sends a log message through the Spider's :attr:`logger`,
-       kept for backwards compatibility. For more information see
+       kept for backward compatibility. For more information see
        :ref:`topics-logging-from-spiders`.
 
    .. method:: closed(reason)
diff --git a/docs/versioning.rst b/docs/versioning.rst
index 0421ba544..227085f02 100644
--- a/docs/versioning.rst
+++ b/docs/versioning.rst
@@ -12,7 +12,7 @@ There are 3 numbers in a Scrapy version: *A.B.C*
 * *A* is the major version. This will rarely change and will signify very
   large changes.
 * *B* is the release number. This will include many changes including features
-  and things that possibly break backwards compatibility, although we strive to
+  and things that possibly break backward compatibility, although we strive to
   keep theses cases at a minimum.
 * *C* is the bugfix release number.
 
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index dc6b59fe0..fa2506eb0 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -99,7 +99,7 @@ def execute(argv=None, settings=None):
     if argv is None:
         argv = sys.argv
 
-    # --- backwards compatibility for scrapy.conf.settings singleton ---
+    # --- backward compatibility for scrapy.conf.settings singleton ---
     if settings is None and 'scrapy.conf' in sys.modules:
         from scrapy import conf
         if hasattr(conf, 'settings'):
@@ -116,7 +116,7 @@ def execute(argv=None, settings=None):
             settings['EDITOR'] = editor
     check_deprecated_settings(settings)
 
-    # --- backwards compatibility for scrapy.conf.settings singleton ---
+    # --- backward compatibility for scrapy.conf.settings singleton ---
     import warnings
     from scrapy.exceptions import ScrapyDeprecationWarning
     with warnings.catch_warnings():
diff --git a/scrapy/conf.py b/scrapy/conf.py
index 23efc6ffd..6c40edcdd 100644
--- a/scrapy/conf.py
+++ b/scrapy/conf.py
@@ -1,4 +1,4 @@
-# This module is kept for backwards compatibility, so users can import
+# This module is kept for backward compatibility, so users can import
 # scrapy.conf.settings and get the settings they expect
 
 import sys
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index e4a7d8564..e76823623 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -3,7 +3,7 @@ from .http10 import HTTP10DownloadHandler
 from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
 
 
-# backwards compatibility
+# backward compatibility
 class HttpDownloadHandler(HTTP10DownloadHandler):
 
     def __init__(self, *args, **kwargs):
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 22ebf3b3f..3b4d809e8 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -94,7 +94,7 @@ class FileFeedStorage(object):
 class S3FeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri, access_key=None, secret_key=None):
-        # BEGIN Backwards compatibility for initialising without keys (and
+        # BEGIN Backward compatibility for initialising without keys (and
         # without using from_crawler)
         no_defaults = access_key is None and secret_key is None
         if no_defaults:
@@ -111,7 +111,7 @@ class S3FeedStorage(BlockingFeedStorage):
                 )
                 access_key = settings['AWS_ACCESS_KEY_ID']
                 secret_key = settings['AWS_SECRET_ACCESS_KEY']
-        # END Backwards compatibility
+        # END Backward compatibility
         u = urlparse(uri)
         self.bucketname = u.hostname
         self.access_key = u.username or access_key
diff --git a/scrapy/log.py b/scrapy/log.py
index 719fceaad..777bd6dc4 100644
--- a/scrapy/log.py
+++ b/scrapy/log.py
@@ -17,7 +17,7 @@ warnings.warn("Module `scrapy.log` has been deprecated, Scrapy now relies on "
               ScrapyDeprecationWarning, stacklevel=2)
 
 
-# Imports and level_names variable kept for backwards-compatibility
+# Imports and level_names variable kept for backward-compatibility
 
 DEBUG = logging.DEBUG
 INFO = logging.INFO
diff --git a/scrapy/signals.py b/scrapy/signals.py
index c0e4bb74e..6b9125302 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -20,7 +20,7 @@ item_scraped = object()
 item_dropped = object()
 item_error = object()
 
-# for backwards compatibility
+# for backward compatibility
 stats_spider_opened = spider_opened
 stats_spider_closing = spider_closed
 stats_spider_closed = spider_closed
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 435e9a6b3..fbd297340 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -42,14 +42,14 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
                 raise ValueError('Invalid value {} for component {}, please provide ' \
                                  'a real number or None instead'.format(value, name))
 
-    # BEGIN Backwards compatibility for old (base, custom) call signature
+    # BEGIN Backward compatibility for old (base, custom) call signature
     if isinstance(custom, (list, tuple)):
         _check_components(custom)
         return type(custom)(convert(c) for c in custom)
 
     if custom is not None:
         compdict.update(custom)
-    # END Backwards compatibility
+    # END Backward compatibility
 
     _validate_values(compdict)
     compdict = without_none_values(_map_keys(compdict))
diff --git a/sep/sep-018.rst b/sep/sep-018.rst
index aca7ac342..fe707923a 100644
--- a/sep/sep-018.rst
+++ b/sep/sep-018.rst
@@ -211,7 +211,7 @@ spider methods on each event such as:
 - call additional spider middlewares defined in the ``Spider.middlewares``
   attribute
 - call ``Spider.next_request()`` and ``Spider.start_requests()`` on
-  ``next_request()`` middleware method (this would implicitly support backwards
+  ``next_request()`` middleware method (this would implicitly support backward
   compatibility)
 
 Differences with Spider middleware v1
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 0d0829793..81235a16f 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -50,7 +50,7 @@ class DummyDH(object):
 
 
 class DummyLazyDH(object):
-    # Default is lazy for backwards compatibility
+    # Default is lazy for backward compatibility
 
     def __init__(self, crawler):
         pass
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e46c8c14e..b254b9f38 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -161,7 +161,7 @@ class S3FeedStorageTest(unittest.TestCase):
                                 aws_credentials['AWS_SECRET_ACCESS_KEY'])
         self.assertEqual(storage.access_key, 'uri_key')
         self.assertEqual(storage.secret_key, 'uri_secret')
-        # Backwards compatibility for initialising without settings
+        # Backward compatibility for initialising without settings
         with warnings.catch_warnings(record=True) as w:
             storage = S3FeedStorage('s3://mybucket/export.csv')
             self.assertEqual(storage.access_key, 'conf_key')
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index f203c32ef..29937c189 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -11,7 +11,7 @@ class BuildComponentListTest(unittest.TestCase):
         self.assertEqual(build_component_list(d, convert=lambda x: x),
                          ['one', 'four', 'three'])
 
-    def test_backwards_compatible_build_dict(self):
+    def test_backward_compatible_build_dict(self):
         base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None}
         custom = {'two': None, 'three': 8, 'four': 4}
         self.assertEqual(build_component_list(base, custom,

From 75d6f56c8a731ea4e1c06814a59a0b51741d04a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 1 Mar 2019 16:56:58 +0100
Subject: [PATCH 1935/2974] Switch from ` to `` where inline code formatting is
 desired

---
 docs/contributing.rst                 |   2 +-
 docs/news.rst                         | 130 +++++++++++++-------------
 docs/topics/api.rst                   |   4 +-
 docs/topics/downloader-middleware.rst |  40 ++++----
 docs/topics/exporters.rst             |   2 +-
 docs/topics/extensions.rst            |   4 +-
 docs/topics/jobs.rst                  |   2 +-
 docs/topics/loaders.rst               |   2 +-
 docs/topics/logging.rst               |   2 +-
 docs/topics/media-pipeline.rst        |   2 +-
 docs/topics/practices.rst             |   2 +-
 docs/topics/request-response.rst      |   8 +-
 docs/topics/selectors.rst             |   2 +-
 docs/topics/settings.rst              |   4 +-
 docs/topics/spider-middleware.rst     |   4 +-
 docs/topics/spiders.rst               |   6 +-
 docs/topics/ubuntu.rst                |   4 +-
 scrapy/crawler.py                     |  10 +-
 scrapy/logformatter.py                |  18 ++--
 scrapy/pipelines/files.py             |   6 +-
 scrapy/utils/ftp.py                   |   2 +-
 scrapy/utils/log.py                   |   2 +-
 scrapy/utils/python.py                |  18 ++--
 scrapy/utils/url.py                   |   8 +-
 sep/sep-006.rst                       |   5 +-
 tests/mocks/dummydbm.py               |   2 +-
 tests/test_command_shell.py           |   4 +-
 27 files changed, 148 insertions(+), 147 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index cf27337c8..9b508e418 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -55,7 +55,7 @@ guidelines when you're going to report a new bug.
 
 * search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has
   been discussed there, or if you're not sure if what you're seeing is a bug.
-  You can also ask in the `#scrapy` IRC channel.
+  You can also ask in the ``#scrapy`` IRC channel.
 
 * write **complete, reproducible, specific bug reports**. The smaller the test
   case, the better. Remember that other developers won't have your project to
diff --git a/docs/news.rst b/docs/news.rst
index 668473887..1849a3ca8 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -149,7 +149,7 @@ Documentation improvements
 * improved links to beginner resources in the tutorial
   (:issue:`3367`, :issue:`3468`);
 * fixed :setting:`RETRY_HTTP_CODES` default values in docs (:issue:`3335`);
-* remove unused `DEPTH_STATS` option from docs (:issue:`3245`);
+* remove unused ``DEPTH_STATS`` option from docs (:issue:`3245`);
 * other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`, :issue:`3544`,
   :issue:`3605`).
 
@@ -1313,7 +1313,7 @@ Module Relocations
 
 There’s been a large rearrangement of modules trying to improve the general
 structure of Scrapy. Main changes were separating various subpackages into
-new projects and dissolving both `scrapy.contrib` and `scrapy.contrib_exp`
+new projects and dissolving both ``scrapy.contrib`` and ``scrapy.contrib_exp``
 into top level packages. Backward compatibility was kept among internal
 relocations, while importing deprecated modules expect warnings indicating
 their new place.
@@ -1344,7 +1344,7 @@ Outsourced packages
 |                                     | /scrapy-plugins/scrapy-jsonrpc>`_   |
 +-------------------------------------+-------------------------------------+
 
-`scrapy.contrib_exp` and `scrapy.contrib` dissolutions
+``scrapy.contrib_exp`` and ``scrapy.contrib`` dissolutions
 
 +-------------------------------------+-------------------------------------+
 | Old location                        | New location                        |
@@ -1556,7 +1556,7 @@ Code refactoring
   (:issue:`1078`)
 - Pydispatch pep8 (:issue:`992`)
 - Removed unused 'load=False' parameter from walk_modules() (:issue:`871`)
-- For consistency, use `job_dir` helper in `SpiderState` extension.
+- For consistency, use ``job_dir`` helper in ``SpiderState`` extension.
   (:issue:`805`)
 - rename "sflo" local variables to less cryptic "log_observer" (:issue:`775`)
 
@@ -1669,10 +1669,10 @@ Enhancements
   cache middleware (:issue:`541`, :issue:`500`, :issue:`571`)
 - Expose current crawler in Scrapy shell (:issue:`557`)
 - Improve testsuite comparing CSV and XML exporters (:issue:`570`)
-- New `offsite/filtered` and `offsite/domains` stats (:issue:`566`)
+- New ``offsite/filtered`` and ``offsite/domains`` stats (:issue:`566`)
 - Support process_links as generator in CrawlSpider (:issue:`555`)
 - Verbose logging and new stats counters for DupeFilter (:issue:`553`)
-- Add a mimetype parameter to `MailSender.send()` (:issue:`602`)
+- Add a mimetype parameter to ``MailSender.send()`` (:issue:`602`)
 - Generalize file pipeline log messages (:issue:`622`)
 - Replace unencodeable codepoints with html entities in SGMLLinkExtractor (:issue:`565`)
 - Converted SEP documents to rst format (:issue:`629`, :issue:`630`,
@@ -1691,20 +1691,20 @@ Enhancements
 - Make scrapy.version_info a tuple of integers (:issue:`681`, :issue:`692`)
 - Infer exporter's output format from filename extensions
   (:issue:`546`, :issue:`659`, :issue:`760`)
-- Support case-insensitive domains in `url_is_from_any_domain()` (:issue:`693`)
+- Support case-insensitive domains in ``url_is_from_any_domain()`` (:issue:`693`)
 - Remove pep8 warnings in project and spider templates (:issue:`698`)
-- Tests and docs for `request_fingerprint` function (:issue:`597`)
-- Update SEP-19 for GSoC project `per-spider settings` (:issue:`705`)
+- Tests and docs for ``request_fingerprint`` function (:issue:`597`)
+- Update SEP-19 for GSoC project ``per-spider settings`` (:issue:`705`)
 - Set exit code to non-zero when contracts fails (:issue:`727`)
 - Add a setting to control what class is instanciated as Downloader component
   (:issue:`738`)
-- Pass response in `item_dropped` signal (:issue:`724`)
-- Improve `scrapy check` contracts command (:issue:`733`, :issue:`752`)
-- Document `spider.closed()` shortcut (:issue:`719`)
-- Document `request_scheduled` signal (:issue:`746`)
+- Pass response in ``item_dropped`` signal (:issue:`724`)
+- Improve ``scrapy check`` contracts command (:issue:`733`, :issue:`752`)
+- Document ``spider.closed()`` shortcut (:issue:`719`)
+- Document ``request_scheduled`` signal (:issue:`746`)
 - Add a note about reporting security issues (:issue:`697`)
 - Add LevelDB http cache storage backend (:issue:`626`, :issue:`500`)
-- Sort spider list output of `scrapy list` command (:issue:`742`)
+- Sort spider list output of ``scrapy list`` command (:issue:`742`)
 - Multiple documentation enhancemens and fixes
   (:issue:`575`, :issue:`587`, :issue:`590`, :issue:`596`, :issue:`610`,
   :issue:`617`, :issue:`618`, :issue:`627`, :issue:`613`, :issue:`643`,
@@ -1772,23 +1772,23 @@ Scrapy 0.22.0 (released 2014-01-17)
 Enhancements
 ~~~~~~~~~~~~
 
-- [**Backwards incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`)
-  To restore old backend set `HTTPCACHE_STORAGE` to `scrapy.contrib.httpcache.DbmCacheStorage`
+- [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`)
+  To restore old backend set ``HTTPCACHE_STORAGE`` to ``scrapy.contrib.httpcache.DbmCacheStorage``
 - Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`)
 - Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`)
 - Rename scrapy.spider.BaseSpider to scrapy.spider.Spider (:issue:`510`, :issue:`519`)
 - Selectors register EXSLT namespaces by default (:issue:`472`)
 - Unify item loaders similar to selectors renaming (:issue:`461`)
-- Make `RFPDupeFilter` class easily subclassable (:issue:`533`)
+- Make ``RFPDupeFilter`` class easily subclassable (:issue:`533`)
 - Improve test coverage and forthcoming Python 3 support (:issue:`525`)
 - Promote startup info on settings and middleware to INFO level (:issue:`520`)
-- Support partials in `get_func_args` util (:issue:`506`, issue:`504`)
+- Support partials in ``get_func_args`` util (:issue:`506`, issue:`504`)
 - Allow running indiviual tests via tox (:issue:`503`)
 - Update extensions ignored by link extractors (:issue:`498`)
 - Add middleware methods to get files/images/thumbs paths (:issue:`490`)
 - Improve offsite middleware tests (:issue:`478`)
 - Add a way to skip default Referer header set by RefererMiddleware (:issue:`475`)
-- Do not send `x-gzip` in default `Accept-Encoding` header (:issue:`469`)
+- Do not send ``x-gzip`` in default ``Accept-Encoding`` header (:issue:`469`)
 - Support defining http error handling using settings (:issue:`466`)
 - Use modern python idioms wherever you find legacies (:issue:`497`)
 - Improve and correct documentation
@@ -1799,14 +1799,14 @@ Fixes
 ~~~~~
 
 - Update Selector class imports in CrawlSpider template (:issue:`484`)
-- Fix unexistent reference to `engine.slots` (:issue:`464`)
-- Do not try to call `body_as_unicode()` on a non-TextResponse instance (:issue:`462`)
+- Fix unexistent reference to ``engine.slots`` (:issue:`464`)
+- Do not try to call ``body_as_unicode()`` on a non-TextResponse instance (:issue:`462`)
 - Warn when subclassing XPathItemLoader, previously it only warned on
   instantiation. (:issue:`523`)
 - Warn when subclassing XPathSelector, previously it only warned on
   instantiation. (:issue:`537`)
 - Multiple fixes to memory stats (:issue:`531`, :issue:`530`, :issue:`529`)
-- Fix overriding url in `FormRequest.from_response()` (:issue:`507`)
+- Fix overriding url in ``FormRequest.from_response()`` (:issue:`507`)
 - Fix tests runner under pip 1.5 (:issue:`513`)
 - Fix logging error when spider name is unicode (:issue:`479`)
 
@@ -1833,7 +1833,7 @@ Enhancements
   (modifying them had been deprecated for a long time)
 - :setting:`ITEM_PIPELINES` is now defined as a dict (instead of a list)
 - Sitemap spider can fetch alternate URLs (:issue:`360`)
-- `Selector.remove_namespaces()` now remove namespaces from element's attributes. (:issue:`416`)
+- ``Selector.remove_namespaces()`` now remove namespaces from element's attributes. (:issue:`416`)
 - Paved the road for Python 3.3+ (:issue:`435`, :issue:`436`, :issue:`431`, :issue:`452`)
 - New item exporter using native python types with nesting support (:issue:`366`)
 - Tune HTTP1.1 pool size so it matches concurrency defined by settings (:commit:`b43b5f575`)
@@ -1844,13 +1844,13 @@ Enhancements
 - Mock server (used for tests) can listen for HTTPS requests (:issue:`410`)
 - Remove multi spider support from multiple core components
   (:issue:`422`, :issue:`421`, :issue:`420`, :issue:`419`, :issue:`423`, :issue:`418`)
-- Travis-CI now tests Scrapy changes against development versions of `w3lib` and `queuelib` python packages.
+- Travis-CI now tests Scrapy changes against development versions of ``w3lib`` and ``queuelib`` python packages.
 - Add pypy 2.1 to continuous integration tests (:commit:`ecfa7431`)
 - Pylinted, pep8 and removed old-style exceptions from source (:issue:`430`, :issue:`432`)
 - Use importlib for parametric imports (:issue:`445`)
 - Handle a regression introduced in Python 2.7.5 that affects XmlItemExporter (:issue:`372`)
 - Bugfix crawling shutdown on SIGINT (:issue:`450`)
-- Do not submit `reset` type inputs in FormRequest.from_response (:commit:`b326b87`)
+- Do not submit ``reset`` type inputs in FormRequest.from_response (:commit:`b326b87`)
 - Do not silence download errors when request errback raises an exception (:commit:`684cfc0`)
 
 Bugfixes
@@ -1865,8 +1865,8 @@ Bugfixes
 - Improve request-response docs (:issue:`391`)
 - Improve best practices docs (:issue:`399`, :issue:`400`, :issue:`401`, :issue:`402`)
 - Improve django integration docs (:issue:`404`)
-- Document `bindaddress` request meta (:commit:`37c24e01d7`)
-- Improve `Request` class documentation (:issue:`226`)
+- Document ``bindaddress`` request meta (:commit:`37c24e01d7`)
+- Improve ``Request`` class documentation (:issue:`226`)
 
 Other
 ~~~~~
@@ -1875,7 +1875,7 @@ Other
 - Add `cssselect`_ python package as install dependency
 - Drop libxml2 and multi selector's backend support, `lxml`_ is required from now on.
 - Minimum Twisted version increased to 10.0.0, dropped Twisted 8.0 support.
-- Running test suite now requires `mock` python library (:issue:`390`)
+- Running test suite now requires ``mock`` python library (:issue:`390`)
 
 
 Thanks
@@ -1929,7 +1929,7 @@ Scrapy 0.18.3 (released 2013-10-03)
 Scrapy 0.18.2 (released 2013-09-03)
 -----------------------------------
 
-- Backport `scrapy check` command fixes and backward compatible multi
+- Backport ``scrapy check`` command fixes and backward compatible multi
   crawler process(:issue:`339`)
 
 Scrapy 0.18.1 (released 2013-08-27)
@@ -1958,31 +1958,31 @@ Scrapy 0.18.0 (released 2013-08-09)
 - Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`)
 - Use lxml recover option to parse sitemaps (:issue:`347`)
 - Bugfix cookie merging by hostname and not by netloc (:issue:`352`)
-- Support disabling `HttpCompressionMiddleware` using a flag setting (:issue:`359`)
-- Support xml namespaces using `iternodes` parser in `XMLFeedSpider` (:issue:`12`)
-- Support `dont_cache` request meta flag (:issue:`19`)
-- Bugfix `scrapy.utils.gz.gunzip` broken by changes in python 2.7.4 (:commit:`4dc76e`)
-- Bugfix url encoding on `SgmlLinkExtractor` (:issue:`24`)
-- Bugfix `TakeFirst` processor shouldn't discard zero (0) value (:issue:`59`)
+- Support disabling ``HttpCompressionMiddleware`` using a flag setting (:issue:`359`)
+- Support xml namespaces using ``iternodes`` parser in ``XMLFeedSpider`` (:issue:`12`)
+- Support ``dont_cache`` request meta flag (:issue:`19`)
+- Bugfix ``scrapy.utils.gz.gunzip`` broken by changes in python 2.7.4 (:commit:`4dc76e`)
+- Bugfix url encoding on ``SgmlLinkExtractor`` (:issue:`24`)
+- Bugfix ``TakeFirst`` processor shouldn't discard zero (0) value (:issue:`59`)
 - Support nested items in xml exporter (:issue:`66`)
 - Improve cookies handling performance (:issue:`77`)
 - Log dupe filtered requests once (:issue:`105`)
 - Split redirection middleware into status and meta based middlewares (:issue:`78`)
 - Use HTTP1.1 as default downloader handler (:issue:`109` and :issue:`318`)
-- Support xpath form selection on `FormRequest.from_response` (:issue:`185`)
-- Bugfix unicode decoding error on `SgmlLinkExtractor` (:issue:`199`)
+- Support xpath form selection on ``FormRequest.from_response`` (:issue:`185`)
+- Bugfix unicode decoding error on ``SgmlLinkExtractor`` (:issue:`199`)
 - Bugfix signal dispatching on pypi interpreter (:issue:`205`)
 - Improve request delay and concurrency handling (:issue:`206`)
-- Add RFC2616 cache policy to `HttpCacheMiddleware` (:issue:`212`)
+- Add RFC2616 cache policy to ``HttpCacheMiddleware`` (:issue:`212`)
 - Allow customization of messages logged by engine (:issue:`214`)
-- Multiples improvements to `DjangoItem` (:issue:`217`, :issue:`218`, :issue:`221`)
+- Multiples improvements to ``DjangoItem`` (:issue:`217`, :issue:`218`, :issue:`221`)
 - Extend Scrapy commands using setuptools entry points (:issue:`260`)
-- Allow spider `allowed_domains` value to be set/tuple (:issue:`261`)
-- Support `settings.getdict` (:issue:`269`)
-- Simplify internal `scrapy.core.scraper` slot handling (:issue:`271`)
-- Added `Item.copy` (:issue:`290`)
+- Allow spider ``allowed_domains`` value to be set/tuple (:issue:`261`)
+- Support ``settings.getdict`` (:issue:`269`)
+- Simplify internal ``scrapy.core.scraper`` slot handling (:issue:`271`)
+- Added ``Item.copy`` (:issue:`290`)
 - Collect idle downloader slots (:issue:`297`)
-- Add `ftp://` scheme downloader handler (:issue:`329`)
+- Add ``ftp://`` scheme downloader handler (:issue:`329`)
 - Added downloader benchmark webserver and spider tools :ref:`benchmarking`
 - Moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on
 - Add scrapy commands using external libraries (:issue:`260`)
@@ -2113,7 +2113,7 @@ Scrapy changes:
 - dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
 - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info.
 - documented :ref:`topics-api`
-- `lxml` is now the default selectors backend instead of `libxml2`
+- ``lxml`` is now the default selectors backend instead of ``libxml2``
 - ported FormRequest.from_response() to use `lxml`_ instead of `ClientForm`_
 - removed modules: ``scrapy.xlib.BeautifulSoup`` and ``scrapy.xlib.ClientForm``
 - SitemapSpider: added support for sitemap urls ending in .xml and .xml.gz, even if they advertise a wrong content type (:commit:`10ed28b`)
@@ -2206,16 +2206,16 @@ New features and settings
 - New ``ChunkedTransferMiddleware`` (enabled by default) to support `chunked transfer encoding`_ (:rev:`2769`)
 - Add boto 2.0 support for S3 downloader handler (:rev:`2763`)
 - Added `marshal`_ to formats supported by feed exports (:rev:`2744`)
-- In request errbacks, offending requests are now received in `failure.request` attribute (:rev:`2738`)
+- In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`)
 - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`)
    - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by:
       - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP`
    - check the documentation for more details
 - Added builtin caching DNS resolver (:rev:`2728`)
 - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`)
-- Moved spider queues to scrapyd: `scrapy.spiderqueue` -> `scrapyd.spiderqueue` (:rev:`2708`)
-- Moved sqlite utils to scrapyd: `scrapy.utils.sqlite` -> `scrapyd.sqlite` (:rev:`2781`)
-- Real support for returning iterators on `start_requests()` method. The iterator is now consumed during the crawl when the spider is getting idle (:rev:`2704`)
+- Moved spider queues to scrapyd: ``scrapy.spiderqueue`` -> ``scrapyd.spiderqueue`` (:rev:`2708`)
+- Moved sqlite utils to scrapyd: ``scrapy.utils.sqlite`` -> ``scrapyd.sqlite`` (:rev:`2781`)
+- Real support for returning iterators on ``start_requests()`` method. The iterator is now consumed during the crawl when the spider is getting idle (:rev:`2704`)
 - Added :setting:`REDIRECT_ENABLED` setting to quickly enable/disable the redirect middleware (:rev:`2697`)
 - Added :setting:`RETRY_ENABLED` setting to quickly enable/disable the retry middleware (:rev:`2694`)
 - Added ``CloseSpider`` exception to manually close spiders (:rev:`2691`)
@@ -2223,19 +2223,19 @@ New features and settings
 - Refactored close spider behavior to wait for all downloads to finish and be processed by spiders, before closing the spider (:rev:`2688`)
 - Added ``SitemapSpider`` (see documentation in Spiders page) (:rev:`2658`)
 - Added ``LogStats`` extension for periodically logging basic stats (like crawled pages and scraped items) (:rev:`2657`)
-- Make handling of gzipped responses more robust (#319, :rev:`2643`). Now Scrapy will try and decompress as much as possible from a gzipped response, instead of failing with an `IOError`.
+- Make handling of gzipped responses more robust (#319, :rev:`2643`). Now Scrapy will try and decompress as much as possible from a gzipped response, instead of failing with an ``IOError``.
 - Simplified !MemoryDebugger extension to use stats for dumping memory debugging info (:rev:`2639`)
-- Added new command to edit spiders: ``scrapy edit`` (:rev:`2636`) and `-e` flag to `genspider` command that uses it (:rev:`2653`)
+- Added new command to edit spiders: ``scrapy edit`` (:rev:`2636`) and ``-e`` flag to ``genspider`` command that uses it (:rev:`2653`)
 - Changed default representation of items to pretty-printed dicts. (:rev:`2631`). This improves default logging by making log more readable in the default case, for both Scraped and Dropped lines.
 - Added :signal:`spider_error` signal (:rev:`2628`)
 - Added :setting:`COOKIES_ENABLED` setting (:rev:`2625`)
-- Stats are now dumped to Scrapy log (default value of :setting:`STATS_DUMP` setting has been changed to `True`). This is to make Scrapy users more aware of Scrapy stats and the data that is collected there.
+- Stats are now dumped to Scrapy log (default value of :setting:`STATS_DUMP` setting has been changed to ``True``). This is to make Scrapy users more aware of Scrapy stats and the data that is collected there.
 - Added support for dynamically adjusting download delay and maximum concurrent requests (:rev:`2599`)
 - Added new DBM HTTP cache storage backend (:rev:`2576`)
 - Added ``listjobs.json`` API to Scrapyd (:rev:`2571`)
 - ``CsvItemExporter``: added ``join_multivalued`` parameter (:rev:`2578`)
 - Added namespace support to ``xmliter_lxml`` (:rev:`2552`)
-- Improved cookies middleware by making `COOKIES_DEBUG` nicer and documenting it (:rev:`2579`)
+- Improved cookies middleware by making ``COOKIES_DEBUG`` nicer and documenting it (:rev:`2579`)
 - Several improvements to Scrapyd and Link extractors
 
 Code rearranged and removed
@@ -2249,11 +2249,11 @@ Code rearranged and removed
 - Reduced Scrapy codebase by striping part of Scrapy code into two new libraries:
    - `w3lib`_ (several functions from ``scrapy.utils.{http,markup,multipart,response,url}``, done in :rev:`2584`)
    - `scrapely`_ (was ``scrapy.contrib.ibl``, done in :rev:`2586`)
-- Removed unused function: `scrapy.utils.request.request_info()` (:rev:`2577`)
-- Removed googledir project from `examples/googledir`. There's now a new example project called `dirbot` available on github: https://github.com/scrapy/dirbot
+- Removed unused function: ``scrapy.utils.request.request_info()`` (:rev:`2577`)
+- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on github: https://github.com/scrapy/dirbot
 - Removed support for default field values in Scrapy items (:rev:`2616`)
 - Removed experimental crawlspider v2 (:rev:`2632`)
-- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (`DUPEFILTER_CLASS` setting) (:rev:`2640`)
+- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
 - Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`)
 - Removed deprecated Execution Queue (:rev:`2704`)
 - Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`)
@@ -2289,13 +2289,13 @@ Scrapyd changes
 - Scrapyd now uses one process per spider
 - It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default)
 - A minimal web ui was added, available at http://localhost:6800 by default
-- There is now a `scrapy server` command to start a Scrapyd server of the current project
+- There is now a ``scrapy server`` command to start a Scrapyd server of the current project
 
 Changes to settings
 ~~~~~~~~~~~~~~~~~~~
 
-- added `HTTPCACHE_ENABLED` setting (False by default) to enable HTTP cache middleware
-- changed `HTTPCACHE_EXPIRATION_SECS` semantics: now zero means "never expire".
+- added ``HTTPCACHE_ENABLED`` setting (False by default) to enable HTTP cache middleware
+- changed ``HTTPCACHE_EXPIRATION_SECS`` semantics: now zero means "never expire".
 
 Deprecated/obsoleted functionality
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -2326,17 +2326,17 @@ New features and improvements
 - Splitted Debian package into two packages - the library and the service (#187)
 - Scrapy log refactoring (#188)
 - New extension for keeping persistent spider contexts among different runs (#203)
-- Added `dont_redirect` request.meta key for avoiding redirects (#233)
-- Added `dont_retry` request.meta key for avoiding retries (#234)
+- Added ``dont_redirect`` request.meta key for avoiding redirects (#233)
+- Added ``dont_retry`` request.meta key for avoiding retries (#234)
 
 Command-line tool changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- New `scrapy` command which replaces the old `scrapy-ctl.py` (#199)
-  - there is only one global `scrapy` command now, instead of one `scrapy-ctl.py` per project
-  - Added `scrapy.bat` script for running more conveniently from Windows
+- New ``scrapy`` command which replaces the old ``scrapy-ctl.py`` (#199)
+  - there is only one global ``scrapy`` command now, instead of one ``scrapy-ctl.py`` per project
+  - Added ``scrapy.bat`` script for running more conveniently from Windows
 - Added bash completion to command-line tool (#210)
-- Renamed command `start` to `runserver` (#209)
+- Renamed command ``start`` to ``runserver`` (#209)
 
 API changes
 ~~~~~~~~~~~
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 985cc0433..ba832ab5d 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -94,7 +94,7 @@ how you :ref:`configure the downloader middlewares
     .. method:: crawl(\*args, \**kwargs)
 
         Starts the crawler by instantiating its spider class with the given
-        `args` and `kwargs` arguments, while setting the execution engine in
+        ``args`` and ``kwargs`` arguments, while setting the execution engine in
         motion.
 
         Returns a deferred that is fired when the crawl is finished.
@@ -180,7 +180,7 @@ SpiderLoader API
     .. method:: load(spider_name)
 
        Get the Spider class with the given name. It'll look into the previously
-       loaded spiders for a spider class with name `spider_name` and will raise
+       loaded spiders for a spider class with name ``spider_name`` and will raise
        a KeyError if not found.
 
        :param spider_name: spider class name
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 8dbe249fa..e6812eddd 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -41,7 +41,7 @@ previous (or subsequent) middleware being applied.
 
 If you want to disable a built-in middleware (the ones defined in
 :setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it
-in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign `None`
+in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None``
 as its value.  For example, if you want to disable the user-agent middleware::
 
     DOWNLOADER_MIDDLEWARES = {
@@ -357,7 +357,7 @@ HttpCacheMiddleware
 
     .. reqmeta:: dont_cache
 
-    You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals `True`.
+    You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals ``True``.
 
 .. _httpcache-policy-dummy:
 
@@ -390,17 +390,17 @@ runs to avoid downloading unmodified data (to save bandwidth and speed up crawls
 
 what is implemented:
 
-* Do not attempt to store responses/requests with `no-store` cache-control directive set
-* Do not serve responses from cache if `no-cache` cache-control directive is set even for fresh responses
-* Compute freshness lifetime from `max-age` cache-control directive
-* Compute freshness lifetime from `Expires` response header
-* Compute freshness lifetime from `Last-Modified` response header (heuristic used by Firefox)
-* Compute current age from `Age` response header
-* Compute current age from `Date` header
-* Revalidate stale responses based on `Last-Modified` response header
-* Revalidate stale responses based on `ETag` response header
-* Set `Date` header for any received response missing it
-* Support `max-stale` cache-control directive in requests
+* Do not attempt to store responses/requests with ``no-store`` cache-control directive set
+* Do not serve responses from cache if ``no-cache`` cache-control directive is set even for fresh responses
+* Compute freshness lifetime from ``max-age`` cache-control directive
+* Compute freshness lifetime from ``Expires`` response header
+* Compute freshness lifetime from ``Last-Modified`` response header (heuristic used by Firefox)
+* Compute current age from ``Age`` response header
+* Compute current age from ``Date`` header
+* Revalidate stale responses based on ``Last-Modified`` response header
+* Revalidate stale responses based on ``ETag`` response header
+* Set ``Date`` header for any received response missing it
+* Support ``max-stale`` cache-control directive in requests
 
   This allows spiders to be configured with the full RFC2616 cache policy,
   but avoid revalidation on a request-by-request basis, while remaining
@@ -408,15 +408,15 @@ what is implemented:
 
   Example:
 
-  Add `Cache-Control: max-stale=600` to Request headers to accept responses that
+  Add ``Cache-Control: max-stale=600`` to Request headers to accept responses that
   have exceeded their expiration time by no more than 600 seconds.
 
   See also: RFC2616, 14.9.3
 
 what is missing:
 
-* `Pragma: no-cache` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
-* `Vary` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6
+* ``Pragma: no-cache`` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
+* ``Vary`` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6
 * Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10
 * ... probably others ..
 
@@ -626,12 +626,12 @@ Default: ``False``
 If enabled, will cache pages unconditionally.
 
 A spider may wish to have all responses available in the cache, for
-future use with `Cache-Control: max-stale`, for instance. The
+future use with ``Cache-Control: max-stale``, for instance. The
 DummyPolicy caches all responses but never revalidates them, and
 sometimes a more nuanced policy is desirable.
 
-This setting still respects `Cache-Control: no-store` directives in responses.
-If you don't want that, filter `no-store` out of the Cache-Control headers in
+This setting still respects ``Cache-Control: no-store`` directives in responses.
+If you don't want that, filter ``no-store`` out of the Cache-Control headers in
 responses you feedto the cache middleware.
 
 .. setting:: HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS
@@ -940,7 +940,7 @@ UserAgentMiddleware
 
    Middleware that allows spiders to override the default user agent.
 
-   In order for a spider to override the default user agent, its `user_agent`
+   In order for a spider to override the default user agent, its ``user_agent``
    attribute must be set.
 
 .. _ajaxcrawl-middleware:
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 95f7920f8..f5048d2da 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -303,7 +303,7 @@ CsvItemExporter
 
    The additional keyword arguments of this constructor are passed to the
    :class:`BaseItemExporter` constructor, and the leftover arguments to the
-   `csv.writer`_ constructor, so you can use any `csv.writer` constructor
+   `csv.writer`_ constructor, so you can use any ``csv.writer`` constructor
    argument to customize this exporter.
 
    A typical output of this exporter would be::
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index c421a5e05..d6e7452a1 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -19,7 +19,7 @@ settings, just like any other Scrapy code.
 It is customary for extensions to prefix their settings with their own name, to
 avoid collision with existing (and future) extensions. For example, a
 hypothetic extension to handle `Google Sitemaps`_ would use settings like
-`GOOGLESITEMAP_ENABLED`, `GOOGLESITEMAP_DEPTH`, and so on.
+``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on.
 
 .. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps
 
@@ -368,7 +368,7 @@ Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_
 signal is received. After the debugger is exited, the Scrapy process continues
 running normally.
 
-For more info see `Debugging in Python`.
+For more info see `Debugging in Python`_.
 
 This extension only works on POSIX-compliant platforms (ie. not Windows).
 
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index ea684b4cf..1a5d52487 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -71,7 +71,7 @@ on cookies.
 Request serialization
 ---------------------
 
-Requests must be serializable by the `pickle` module, in order for persistence
+Requests must be serializable by the ``pickle`` module, in order for persistence
 to work, so you should make sure that your requests are serializable.
 
 The most common issue here is to use ``lambda`` functions on request callbacks that
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index f3b6aa4a1..1c2f1da4d 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -286,7 +286,7 @@ ItemLoader objects
     given, one is instantiated automatically using the class in
     :attr:`default_item_class`.
 
-    When instantiated with a `selector` or a `response` parameters
+    When instantiated with a ``selector`` or a ``response`` parameters
     the :class:`ItemLoader` class provides convenient mechanisms for extracting
     data from web pages using :ref:`selectors <topics-selectors>`.
 
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 0986929ad..8e280d929 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -243,7 +243,7 @@ scrapy.utils.log module
     case, its usage is not required but it's recommended.
 
     If you plan on configuring the handlers yourself is still recommended you
-    call this function, passing `install_root_handler=False`. Bear in mind
+    call this function, passing ``install_root_handler=False``. Bear in mind
     there won't be any log output set by default in that case.
 
     To get you started on manually configuring logging's output, you can use
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index c60b55391..381a2988a 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -132,7 +132,7 @@ For example, the following image URL::
 
     http://www.example.com/image.jpg
 
-Whose `SHA1 hash` is::
+Whose ``SHA1 hash`` is::
 
     3afec3b4765f8f0a07b78f98c07b83f013567a0a
 
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 02cfa9b05..298a078a7 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -80,7 +80,7 @@ returned by the :meth:`CrawlerRunner.crawl
 <scrapy.crawler.CrawlerRunner.crawl>` method.
 
 Here's an example of its usage, along with a callback to manually stop the
-reactor after `MySpider` has finished running.
+reactor after ``MySpider`` has finished running.
 
 ::
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 76360b15f..8b3ba4f2d 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -50,7 +50,7 @@ Request objects
     :type meta: dict
 
     :param body: the request body. If a ``unicode`` is passed, then it's encoded to
-      ``str`` using the `encoding` passed (which defaults to ``utf-8``). If
+      ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If
       ``body`` is not given, an empty string is stored. Regardless of the
       type of this argument, the final value stored will be a ``str`` (never
       ``unicode`` or ``None``).
@@ -610,7 +610,7 @@ Response objects
     .. attribute:: Response.flags
 
         A list that contains flags for this response. Flags are labels used for
-        tagging Responses. For example: `'cached'`, `'redirected`', etc. And
+        tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And
         they're shown on the string representation of the Response (`__str__`
         method) which is used by the engine for logging.
 
@@ -682,7 +682,7 @@ TextResponse objects
 
             ``unicode(response.body)`` is not a correct way to convert response
             body to unicode: you would be using the system default encoding
-            (typically `ascii`) instead of the response encoding.
+            (typically ``ascii``) instead of the response encoding.
 
 
     .. attribute:: TextResponse.encoding
@@ -690,7 +690,7 @@ TextResponse objects
        A string with the encoding of this response. The encoding is resolved by
        trying the following mechanisms, in order:
 
-       1. the encoding passed in the constructor `encoding` argument
+       1. the encoding passed in the constructor ``encoding`` argument
 
        2. the encoding declared in the Content-Type HTTP header. If this
           encoding is not valid (ie. unknown), it is ignored and the next
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index df1d67ae8..edc18f14d 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -96,7 +96,7 @@ Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of
 Using selectors
 ---------------
 
-To explain how to use the selectors we'll use the `Scrapy shell` (which
+To explain how to use the selectors we'll use the ``Scrapy shell`` (which
 provides interactive testing) and an example page located in the Scrapy
 documentation server:
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0ac26a9bd..1afa513c8 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -599,7 +599,7 @@ The amount of time (in secs) that the downloader will wait before timing out.
 DOWNLOAD_MAXSIZE
 ----------------
 
-Default: `1073741824` (1024MB)
+Default: ``1073741824`` (1024MB)
 
 The maximum response size (in bytes) that downloader will download.
 
@@ -620,7 +620,7 @@ If you want to disable it set to 0.
 DOWNLOAD_WARNSIZE
 -----------------
 
-Default: `33554432` (32MB)
+Default: ``33554432`` (32MB)
 
 The response size (in bytes) that downloader will start to warn.
 
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 2b7e42771..b551aa47d 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -43,7 +43,7 @@ previous (or subsequent) middleware being applied.
 
 If you want to disable a builtin middleware (the ones defined in
 :setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
-in your project :setting:`SPIDER_MIDDLEWARES` setting and assign `None` as its
+in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its
 value.  For example, if you want to disable the off-site middleware::
 
     SPIDER_MIDDLEWARES = {
@@ -200,7 +200,7 @@ DepthMiddleware
 .. class:: DepthMiddleware
 
    DepthMiddleware is used for tracking the depth of each Request inside the
-   site being scraped. It works by setting `request.meta['depth'] = 0` whenever
+   site being scraped. It works by setting ``request.meta['depth'] = 0`` whenever
    there is no value previously set (usually just the first Request) and
    incrementing it by 1 otherwise.
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 742a88659..09feedefc 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -129,7 +129,7 @@ scrapy.Spider
 
        You probably won't need to override this directly because the default
        implementation acts as a proxy to the :meth:`__init__` method, calling
-       it with the given arguments `args` and named arguments `kwargs`.
+       it with the given arguments ``args`` and named arguments ``kwargs``.
 
        Nonetheless, this method sets the :attr:`crawler` and :attr:`settings`
        attributes in the new instance so they can be accessed later inside the
@@ -298,13 +298,13 @@ The above example can also be written as follows::
 
 Keep in mind that spider arguments are only strings.
 The spider will not do any parsing on its own.
-If you were to set the `start_urls` attribute from the command line,
+If you were to set the ``start_urls`` attribute from the command line,
 you would have to parse it on your own into a list
 using something like
 `ast.literal_eval <https://docs.python.org/library/ast.html#ast.literal_eval>`_
 or `json.loads <https://docs.python.org/library/json.html#json.loads>`_
 and then set it as an attribute.
-Otherwise, you would cause iteration over a `start_urls` string
+Otherwise, you would cause iteration over a ``start_urls`` string
 (a very common python pitfall)
 resulting in each character being seen as a separate url.
 
diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst
index 81ce800aa..6c993a970 100644
--- a/docs/topics/ubuntu.rst
+++ b/docs/topics/ubuntu.rst
@@ -22,7 +22,7 @@ To use the packages:
 
     sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 627220E7
 
-2. Create `/etc/apt/sources.list.d/scrapy.list` file using the following command::
+2. Create ``/etc/apt/sources.list.d/scrapy.list`` file using the following command::
 
     echo 'deb http://archive.scrapy.org/ubuntu scrapy main' | sudo tee /etc/apt/sources.list.d/scrapy.list
 
@@ -34,7 +34,7 @@ To use the packages:
 
 .. note:: Repeat step 3 if you are trying to upgrade Scrapy.
 
-.. warning:: `python-scrapy` is a different package provided by official debian
+.. warning:: ``python-scrapy`` is a different package provided by official debian
    repositories, it's very outdated and it isn't supported by Scrapy team.
 
 .. _Scrapinghub: https://scrapinghub.com/
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 04aee18ed..2ecc4daad 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -153,7 +153,7 @@ class CrawlerRunner(object):
         It will call the given Crawler's :meth:`~Crawler.crawl` method, while
         keeping track of it so it can be stopped later.
 
-        If `crawler_or_spidercls` isn't a :class:`~scrapy.crawler.Crawler`
+        If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler`
         instance, this method will try to create one using this parameter as
         the spider class given to it.
 
@@ -188,10 +188,10 @@ class CrawlerRunner(object):
         """
         Return a :class:`~scrapy.crawler.Crawler` object.
 
-        * If `crawler_or_spidercls` is a Crawler, it is returned as-is.
-        * If `crawler_or_spidercls` is a Spider subclass, a new Crawler
+        * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is.
+        * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler
           is constructed for it.
-        * If `crawler_or_spidercls` is a string, this function finds
+        * If ``crawler_or_spidercls`` is a string, this function finds
           a spider with this name in a Scrapy project (using spider loader),
           then creates a Crawler instance for it.
         """
@@ -273,7 +273,7 @@ class CrawlerProcess(CrawlerRunner):
         :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based
         on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
 
-        If `stop_after_crawl` is True, the reactor will be stopped after all
+        If ``stop_after_crawl`` is True, the reactor will be stopped after all
         crawlers have finished, using :meth:`join`.
 
         :param boolean stop_after_crawl: stop or not the reactor when all
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 075a6d862..65f347dcf 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -13,21 +13,21 @@ CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(ref
 class LogFormatter(object):
     """Class for generating log messages for different actions.
 
-    All methods must return a dictionary listing the parameters `level`, `msg`
-    and `args` which are going to be used for constructing the log message when
-    calling logging.log.
+    All methods must return a dictionary listing the parameters ``level``,
+    ``msg`` and ``args`` which are going to be used for constructing the log
+    message when calling logging.log.
 
     Dictionary keys for the method outputs:
-        * `level` should be the log level for that action, you can use those
+        * ``level`` should be the log level for that action, you can use those
         from the python logging library: logging.DEBUG, logging.INFO,
         logging.WARNING, logging.ERROR and logging.CRITICAL.
 
-        * `msg` should be a string that can contain different formatting
-        placeholders. This string, formatted with the provided `args`, is going
-        to be the log message for that action.
+        * ``msg`` should be a string that can contain different formatting
+        placeholders. This string, formatted with the provided ``args``, is
+        going to be the log message for that action.
 
-        * `args` should be a tuple or dict with the formatting placeholders for
-        `msg`.  The final log message is computed as output['msg'] %
+        * ``args`` should be a tuple or dict with the formatting placeholders
+        for ``msg``.  The final log message is computed as output['msg'] %
         output['args'].
     """
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 510cc23c7..2d8091f5b 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -255,13 +255,13 @@ class FilesPipeline(MediaPipeline):
     doing stat of the files and determining if file is new, uptodate or
     expired.
 
-    `new` files are those that pipeline never processed and needs to be
+    ``new`` files are those that pipeline never processed and needs to be
         downloaded from supplier site the first time.
 
-    `uptodate` files are the ones that the pipeline processed and are still
+    ``uptodate`` files are the ones that the pipeline processed and are still
         valid files.
 
-    `expired` files are those that pipeline already processed but the last
+    ``expired`` files are those that pipeline already processed but the last
         modification was made long time ago, so a reprocessing is recommended to
         refresh it in case of change.
 
diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index f255d436f..9eca6a4da 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -2,7 +2,7 @@ from ftplib import error_perm
 from posixpath import dirname
 
 def ftp_makedirs_cwd(ftp, path, first_call=True):
-    """Set the current directory of the FTP connection given in the `ftp`
+    """Set the current directory of the FTP connection given in the ``ftp``
     argument (as a ftplib.FTP object), creating all parent directories if they
     don't exist. The ftplib.FTP object must be already connected and logged in.
     """
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 828880709..e07fb8698 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -32,7 +32,7 @@ class TopLevelFormatter(logging.Filter):
 
     Since it can't be set for just one logger (it won't propagate for its
     children), it's going to be set in the root handler, with a parametrized
-    `loggers` list where it should act.
+    ``loggers`` list where it should act.
     """
 
     def __init__(self, loggers=None):
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 732ca13a0..aade3d9ac 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -97,8 +97,8 @@ def unicode_to_str(text, encoding=None, errors='strict'):
 
 
 def to_unicode(text, encoding=None, errors='strict'):
-    """Return the unicode representation of a bytes object `text`. If `text`
-    is already an unicode object, return it as-is."""
+    """Return the unicode representation of a bytes object ``text``. If
+    ``text`` is already an unicode object, return it as-is."""
     if isinstance(text, six.text_type):
         return text
     if not isinstance(text, (bytes, six.text_type)):
@@ -110,7 +110,7 @@ def to_unicode(text, encoding=None, errors='strict'):
 
 
 def to_bytes(text, encoding=None, errors='strict'):
-    """Return the binary representation of `text`. If `text`
+    """Return the binary representation of ``text``. If ``text``
     is already a bytes object, return it as-is."""
     if isinstance(text, bytes):
         return text
@@ -123,7 +123,7 @@ def to_bytes(text, encoding=None, errors='strict'):
 
 
 def to_native_str(text, encoding=None, errors='strict'):
-    """ Return str representation of `text`
+    """ Return str representation of ``text``
     (bytes in Python 2.x and unicode in Python 3.x). """
     if six.PY2:
         return to_bytes(text, encoding, errors)
@@ -189,7 +189,7 @@ def isbinarytext(text):
 
 
 def binary_is_text(data):
-    """ Returns `True` if the given ``data`` argument (a ``bytes`` object)
+    """ Returns ``True`` if the given ``data`` argument (a ``bytes`` object)
     does not contain unprintable control characters.
     """
     if not isinstance(data, bytes):
@@ -314,7 +314,7 @@ class WeakKeyCache(object):
 @deprecated
 def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
     """Return a (new) dict with unicode keys (and values when "keys_only" is
-    False) of the given dict converted to strings. `dct_or_tuples` can be a
+    False) of the given dict converted to strings. ``dct_or_tuples`` can be a
     dict or a list of tuples, like any dict constructor supports.
     """
     d = {}
@@ -357,10 +357,10 @@ def retry_on_eintr(function, *args, **kw):
 
 
 def without_none_values(iterable):
-    """Return a copy of `iterable` with all `None` entries removed.
+    """Return a copy of ``iterable`` with all ``None`` entries removed.
 
-    If `iterable` is a mapping, return a dictionary where all pairs that have
-    value `None` have been removed.
+    If ``iterable`` is a mapping, return a dictionary where all pairs that have
+    value ``None`` have been removed.
     """
     try:
         return {k: v for k, v in six.iteritems(iterable) if v is not None}
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 657c53815..b3a4be007 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -109,12 +109,12 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=
 
     """Strip URL string from some of its components:
 
-    - `strip_credentials` removes "user:password@"
-    - `strip_default_port` removes ":80" (resp. ":443", ":21")
+    - ``strip_credentials`` removes "user:password@"
+    - ``strip_default_port`` removes ":80" (resp. ":443", ":21")
       from http:// (resp. https://, ftp://) URLs
-    - `origin_only` replaces path component with "/", also dropping
+    - ``origin_only`` replaces path component with "/", also dropping
       query and fragment components ; it also strips credentials
-    - `strip_fragment` drops any #fragment component
+    - ``strip_fragment`` drops any #fragment component
     """
 
     parsed_url = urlparse(url)
diff --git a/sep/sep-006.rst b/sep/sep-006.rst
index 366fcf033..eb362e945 100644
--- a/sep/sep-006.rst
+++ b/sep/sep-006.rst
@@ -10,7 +10,8 @@ Status   Obsolete (discarded)
 SEP-006: Rename of Selectors to Extractors
 ==========================================
 
-This SEP proposes a more meaningful naming of XPathSelectors or "Selectors" and their `x` method.
+This SEP proposes a more meaningful naming of XPathSelectors or "Selectors" and
+their ``x`` method.
 
 Motivation
 ==========
@@ -57,7 +58,7 @@ Additional changes
 As the name of the method for performing selection (the ``x`` method) is not
 descriptive nor mnemotechnic enough and clearly clashes with ``extract`` method
 (x sounds like a short for extract in english), we propose to rename it to
-`select`, `sel` (is shortness if required), or `xpath` after `lxml's
+``select``, ``sel`` (is shortness if required), or ``xpath`` after `lxml's
 <http://lxml.de/xpathxslt.html>`_ ``xpath`` method.
 
 Bonus (ItemBuilder)
diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py
index 40d9293b2..431428331 100644
--- a/tests/mocks/dummydbm.py
+++ b/tests/mocks/dummydbm.py
@@ -16,7 +16,7 @@ _DATABASES = collections.defaultdict(DummyDB)
 def open(file, flag='r', mode=0o666):
     """Open or create a dummy database compatible.
 
-    Arguments `flag` and `mode` are ignored.
+    Arguments ``flag`` and ``mode`` are ignored.
     """
     # return same instance for same file argument
     return _DATABASES[file]
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 36baacfbd..d664b6ade 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -61,7 +61,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_fetch_redirect_follow_302(self):
-        """Test that calling `fetch(url)` follows HTTP redirects by default."""
+        """Test that calling ``fetch(url)`` follows HTTP redirects by default."""
         url = self.url('/redirect-no-meta-refresh')
         code = "fetch('{0}')"
         errcode, out, errout = yield self.execute(['-c', code.format(url)])
@@ -71,7 +71,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_fetch_redirect_not_follow_302(self):
-        """Test that calling `fetch(url, redirect=False)` disables automatic redirects."""
+        """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects."""
         url = self.url('/redirect-no-meta-refresh')
         code = "fetch('{0}', redirect=False)"
         errcode, out, errout = yield self.execute(['-c', code.format(url)])

From 82d239f3b148d9ce69f67bd7a2cb00de7e934aa6 Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Wed, 6 Mar 2019 12:08:09 +0530
Subject: [PATCH 1936/2974] docs for scrapy.logformatter

---
 docs/topics/logging.rst | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 0986929ad..a5fecebba 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -193,6 +193,45 @@ to override some of the Scrapy settings regarding logging.
     Module `logging.handlers <https://docs.python.org/2/library/logging.handlers.html>`_
         Further documentation on available handlers
 
+Custom Log Formats
+-------------------
+
+Custom log format can be set for different actions by extending ``scrapy.logformatter.LogFormatter`` class.
+
+Each method of ``scrapy.logformatter.LogFormatter`` represents an action. All methods inherited from 
+``scrapy.logformatter.LogFormatter`` in your custom log formatting class must return a dictionary listing
+the parameters ``level``, ``msg`` and ``args`` which are going to be used for constructing the log message.
+Listed below is details of what each key represents :
+
+*   ``level`` is the log level for that action, you can use those from the python logging library:
+    :setting:`logging.DEBUG`, :setting:`logging.INFO`, :setting:`logging.WARNING`, :setting:`logging.ERROR`
+    and :setting:`logging.CRITICAL`.
+
+*   ``msg`` should be a string that can contain different formatting placeholders. This string, formatted
+    with the provided ``args``, is going to be the long message for that action.
+
+*   ``args`` should be a tuple or dict with the formatting placeholders for `msg`. The final log message is
+    computed as ``msg % args``.
+
+.. note:: To use custom log formatting class, you must mention it in ``settings.py``, by adding a line 
+   ``LOG_FORMATTER = '<path_to_your_class>’``
+ 
+.. class:: scrapy.logformatter.LogFormatter
+
+   The default log formatting class in Scrapy.
+
+   .. method:: crawled (request, response, spider)
+
+      ``crawled`` is called to log message when the crawler finds a webpage.
+
+   .. method:: scraped(item, response, spider)
+
+      ``scraped`` is called to log message when an item scraped by a spider.
+
+   .. method::  dropped(item, exception, response, spider)  
+
+      ``dropped`` is called to log message when an item is dropped while it is passing through the item pipeline. 
+
 Advanced customization
 ----------------------
 

From 184def1060f95767ef767948edeb3bbb4ed0e428 Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Thu, 7 Mar 2019 00:09:10 +0530
Subject: [PATCH 1937/2974] fix a link inside docs

---
 docs/topics/architecture.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index 4ac39ad2d..2effe94dc 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -172,5 +172,5 @@ links:
 
 .. _Twisted: https://twistedmatrix.com/trac/
 .. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
-.. _Twisted - hello, asynchronous programming: http://jessenoller.com/2009/02/11/twisted-hello-asynchronous-programming/
+.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
 .. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/

From 924b67437b92f14601816d02c5d153e7281da6d4 Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Thu, 7 Mar 2019 16:40:59 +0530
Subject: [PATCH 1938/2974] move api docs to source code

---
 docs/topics/logging.rst  | 38 ++++----------------------------------
 docs/topics/settings.rst |  9 +++++++++
 scrapy/logformatter.py   | 36 +++++++++++++++++++++++-------------
 3 files changed, 36 insertions(+), 47 deletions(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index a5fecebba..72f24bae6 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -196,41 +196,11 @@ to override some of the Scrapy settings regarding logging.
 Custom Log Formats
 -------------------
 
-Custom log format can be set for different actions by extending ``scrapy.logformatter.LogFormatter`` class.
-
-Each method of ``scrapy.logformatter.LogFormatter`` represents an action. All methods inherited from 
-``scrapy.logformatter.LogFormatter`` in your custom log formatting class must return a dictionary listing
-the parameters ``level``, ``msg`` and ``args`` which are going to be used for constructing the log message.
-Listed below is details of what each key represents :
-
-*   ``level`` is the log level for that action, you can use those from the python logging library:
-    :setting:`logging.DEBUG`, :setting:`logging.INFO`, :setting:`logging.WARNING`, :setting:`logging.ERROR`
-    and :setting:`logging.CRITICAL`.
-
-*   ``msg`` should be a string that can contain different formatting placeholders. This string, formatted
-    with the provided ``args``, is going to be the long message for that action.
-
-*   ``args`` should be a tuple or dict with the formatting placeholders for `msg`. The final log message is
-    computed as ``msg % args``.
-
-.. note:: To use custom log formatting class, you must mention it in ``settings.py``, by adding a line 
-   ``LOG_FORMATTER = '<path_to_your_class>’``
+Custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class
+and making :setting:`LOG_FORMATTER` inside ``settings.py`` point to your new class.
  
-.. class:: scrapy.logformatter.LogFormatter
-
-   The default log formatting class in Scrapy.
-
-   .. method:: crawled (request, response, spider)
-
-      ``crawled`` is called to log message when the crawler finds a webpage.
-
-   .. method:: scraped(item, response, spider)
-
-      ``scraped`` is called to log message when an item scraped by a spider.
-
-   .. method::  dropped(item, exception, response, spider)  
-
-      ``dropped`` is called to log message when an item is dropped while it is passing through the item pipeline. 
+.. autoclass:: scrapy.logformatter.LogFormatter
+   :members:
 
 Advanced customization
 ----------------------
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0ac26a9bd..1dfb5b8aa 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -866,6 +866,15 @@ directives.
 
 .. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior
 
+.. setting:: LOG_FORMATTER
+
+LOG_FORMATTER
+-------------
+
+Default: ``scrapy.logformatter.LogFormatter``
+
+The class to use for formatting log messages for different actions.
+
 .. setting:: LOG_LEVEL
 
 LOG_LEVEL
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 075a6d862..0bb8aee58 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -13,25 +13,29 @@ CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(ref
 class LogFormatter(object):
     """Class for generating log messages for different actions.
 
-    All methods must return a dictionary listing the parameters `level`, `msg`
-    and `args` which are going to be used for constructing the log message when
-    calling logging.log.
+    All methods must return a dictionary listing the parameters ``level``, ``msg``
+    and ``args`` which are going to be used for constructing the log message when
+    calling ``logging.log``.
 
     Dictionary keys for the method outputs:
-        * `level` should be the log level for that action, you can use those
-        from the python logging library: logging.DEBUG, logging.INFO,
-        logging.WARNING, logging.ERROR and logging.CRITICAL.
 
-        * `msg` should be a string that can contain different formatting
-        placeholders. This string, formatted with the provided `args`, is going
-        to be the log message for that action.
+        *   ``level`` is the log level for that action, you can use those from the 
+            `python logging library <https://docs.python.org/3/library/logging.html>`_ :
+            ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR``
+            and ``logging.CRITICAL``.
+
+        *   ``msg`` should be a string that can contain different formatting placeholders. This string, formatted
+            with the provided ``args``, is going to be the long message for that action.
+
+        *   ``args`` should be a tuple or dict with the formatting placeholders for ``msg``. The final log message is
+            computed as ``msg % args``.
 
-        * `args` should be a tuple or dict with the formatting placeholders for
-        `msg`.  The final log message is computed as output['msg'] %
-        output['args'].
     """
 
     def crawled(self, request, response, spider):
+        """
+        ``crawled`` is called to log message when the crawler finds a webpage.
+        """
         request_flags = ' %s' % str(request.flags) if request.flags else ''
         response_flags = ' %s' % str(response.flags) if response.flags else ''
         return {
@@ -40,7 +44,7 @@ class LogFormatter(object):
             'args': {
                 'status': response.status,
                 'request': request,
-                'request_flags' : request_flags,
+                'request_flags': request_flags,
                 'referer': referer_str(request),
                 'response_flags': response_flags,
                 # backward compatibility with Scrapy logformatter below 1.4 version
@@ -49,6 +53,9 @@ class LogFormatter(object):
         }
 
     def scraped(self, item, response, spider):
+        """
+        ``scraped`` is called to log message when an item is scraped by a spider.
+        """
         if isinstance(response, Failure):
             src = response.getErrorMessage()
         else:
@@ -63,6 +70,9 @@ class LogFormatter(object):
         }
 
     def dropped(self, item, exception, response, spider):
+        """
+        ``dropped`` is called to log message when an item is dropped while it is passing through the item pipeline. 
+        """
         return {
             'level': logging.WARNING,
             'msg': DROPPEDMSG,

From 4ef38d925e0ded380a7cebabe3aab2340d4f3d37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 8 Mar 2019 14:21:00 +0100
Subject: [PATCH 1939/2974] Remove the unexisting retry_complete signal from
 the documentation

---
 docs/topics/downloader-middleware.rst | 2 --
 scrapy/downloadermiddlewares/retry.py | 4 +---
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 8dbe249fa..9988ab18b 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -834,8 +834,6 @@ RetryMiddleware
 
 Failed pages are collected on the scraping process and rescheduled at the
 end, once the spider has finished crawling all regular (non failed) pages.
-Once there are no more failed pages to retry, this middleware sends a signal
-(retry_complete), so other extensions could connect to that signal.
 
 The :class:`RetryMiddleware` can be configured through the following
 settings (see the settings documentation for more info):
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 07e979628..dbc605a4c 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -7,9 +7,7 @@ RETRY_TIMES - how many times to retry a failed page
 RETRY_HTTP_CODES - which HTTP response codes to retry
 
 Failed pages are collected on the scraping process and rescheduled at the end,
-once the spider has finished crawling all regular (non failed) pages. Once
-there is no more failed pages to retry this middleware sends a signal
-(retry_complete), so other extensions could connect to that signal.
+once the spider has finished crawling all regular (non failed) pages.
 """
 import logging
 

From e108e3adbfa1a1f9bdd2a84180f18e5e43a39d01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 8 Mar 2019 15:13:11 +0100
Subject: [PATCH 1940/2974] Clarify the documentation of DEPTH_PRIORITY further

---
 docs/topics/settings.rst | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0ac26a9bd..229a9e956 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -331,16 +331,16 @@ Default: ``0``
 
 Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
 
-An integer that is used to adjust the request priority based on its depth:
+An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of
+a :class:`~scrapy.http.Request` based on its depth.
 
-- if zero (default), no priority adjustment is made from depth
-- **a positive value will decrease the priority, i.e. higher depth
-  requests will be processed later** ; this is commonly used when doing
-  breadth-first crawls (BFO)
-- a negative value will increase priority, i.e., higher depth requests
-  will be processed sooner (DFO)
+The priority of a request is adjusted as follows::
 
-See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO.
+    request.priority = request.priority - ( depth * DEPTH_PRIORITY )
+
+As depth increases, positive values of ``DEPTH_PRIORITY`` decrease request
+priority (BFO), while negative values increase request priority (DFO). See
+also :ref:`faq-bfo-dfo`.
 
 .. note::
 

From b1063d9b2ca1a6bfb947fdd8b0158633184114ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 8 Mar 2019 17:22:49 +0100
Subject: [PATCH 1941/2974] Use the description from README.rst on index.rst

---
 docs/index.rst | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/docs/index.rst b/docs/index.rst
index 0a96aa88e..cedde8f38 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -4,7 +4,13 @@
 Scrapy |version| documentation
 ==============================
 
-This documentation contains everything you need to know about Scrapy.
+Scrapy is a fast high-level `web crawling`_ and `web scraping`_ framework, used
+to crawl websites and extract structured data from their pages. It can be used
+for a wide range of purposes, from data mining to monitoring and automated
+testing.
+
+.. _web crawling: https://en.wikipedia.org/wiki/Web_crawler
+.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
 
 Getting help
 ============

From 91aec8b3bb805e9595f1b778fb14f703c6acf2e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 8 Mar 2019 18:19:30 +0100
Subject: [PATCH 1942/2974] Update developer-tools.rst

Fixes #3674
---
 docs/topics/developer-tools.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index c1976258d..82857c9da 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -233,7 +233,7 @@ also request each page to get every quote on the site::
         name = 'quote'
         allowed_domains = ['quotes.toscrape.com']
         page = 1
-        start_urls = ['http://quotes.toscrape.com/api/quotes?page=1]
+        start_urls = ['http://quotes.toscrape.com/api/quotes?page=1']
 
         def parse(self, response):
             data = json.loads(response.text)

From 82049e9c41f878d84f0fe10f827c6fe2a33f7ba6 Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Sun, 10 Mar 2019 20:14:55 +0530
Subject: [PATCH 1943/2974] make suggested changes.

---
 docs/topics/logging.rst  | 10 ++++++----
 docs/topics/settings.rst |  4 ++--
 scrapy/logformatter.py   | 26 +++++++++++++++++---------
 3 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 72f24bae6..006530a8c 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -193,11 +193,13 @@ to override some of the Scrapy settings regarding logging.
     Module `logging.handlers <https://docs.python.org/2/library/logging.handlers.html>`_
         Further documentation on available handlers
 
-Custom Log Formats
--------------------
+.. _custom-log-formats:
 
-Custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class
-and making :setting:`LOG_FORMATTER` inside ``settings.py`` point to your new class.
+Custom Log Formats
+------------------
+
+A custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class
+and making :setting:`LOG_FORMATTER` point to your new class.
  
 .. autoclass:: scrapy.logformatter.LogFormatter
    :members:
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 1dfb5b8aa..a36c0b34c 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -871,9 +871,9 @@ directives.
 LOG_FORMATTER
 -------------
 
-Default: ``scrapy.logformatter.LogFormatter``
+Default: :class:`scrapy.logformatter.LogFormatter`
 
-The class to use for formatting log messages for different actions.
+The class to use for :ref:`formatting log messages <custom-log-formats>` for different actions.
 
 .. setting:: LOG_LEVEL
 
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 0bb8aee58..17c69cba8 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -30,12 +30,24 @@ class LogFormatter(object):
         *   ``args`` should be a tuple or dict with the formatting placeholders for ``msg``. The final log message is
             computed as ``msg % args``.
 
+    Here is an example on how to create a custom log formatter to lower the severity level of the log message
+    when an item is dropped from the pipeline::
+
+            class PoliteLogFormatter(logformatter.LogFormatter):
+                def dropped(self, item, exception, response, spider):
+                    return {
+                        'level': logging.INFO, # lowering the level from logging.WARNING
+                        'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s",
+                        'args': {
+                            'exception': exception,
+                            'item': item,
+                        }
+                    }
+
     """
 
     def crawled(self, request, response, spider):
-        """
-        ``crawled`` is called to log message when the crawler finds a webpage.
-        """
+        """Logs a message when the crawler finds a webpage."""
         request_flags = ' %s' % str(request.flags) if request.flags else ''
         response_flags = ' %s' % str(response.flags) if response.flags else ''
         return {
@@ -53,9 +65,7 @@ class LogFormatter(object):
         }
 
     def scraped(self, item, response, spider):
-        """
-        ``scraped`` is called to log message when an item is scraped by a spider.
-        """
+        """Logs a message when an item is scraped by a spider."""
         if isinstance(response, Failure):
             src = response.getErrorMessage()
         else:
@@ -70,9 +80,7 @@ class LogFormatter(object):
         }
 
     def dropped(self, item, exception, response, spider):
-        """
-        ``dropped`` is called to log message when an item is dropped while it is passing through the item pipeline. 
-        """
+        """Logs a message when an item is dropped while it is passing through the item pipeline."""
         return {
             'level': logging.WARNING,
             'msg': DROPPEDMSG,

From e9cd4ee03aa41e27bea0408b10970ec5bedf35d3 Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Sun, 10 Mar 2019 20:37:56 +0530
Subject: [PATCH 1944/2974] fix list alignment and line width

---
 scrapy/logformatter.py | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 17c69cba8..717120242 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -19,19 +19,18 @@ class LogFormatter(object):
 
     Dictionary keys for the method outputs:
 
-        *   ``level`` is the log level for that action, you can use those from the 
-            `python logging library <https://docs.python.org/3/library/logging.html>`_ :
-            ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR``
-            and ``logging.CRITICAL``.
+    *   ``level`` is the log level for that action, you can use those from the
+        `python logging library <https://docs.python.org/3/library/logging.html>`_ :
+        ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR``
+        and ``logging.CRITICAL``.
+    *   ``msg`` should be a string that can contain different formatting placeholders.
+        This string, formatted with the provided ``args``, is going to be the long message
+        for that action.
+    *   ``args`` should be a tuple or dict with the formatting placeholders for ``msg``.
+        The final log message is computed as ``msg % args``.
 
-        *   ``msg`` should be a string that can contain different formatting placeholders. This string, formatted
-            with the provided ``args``, is going to be the long message for that action.
-
-        *   ``args`` should be a tuple or dict with the formatting placeholders for ``msg``. The final log message is
-            computed as ``msg % args``.
-
-    Here is an example on how to create a custom log formatter to lower the severity level of the log message
-    when an item is dropped from the pipeline::
+    Here is an example on how to create a custom log formatter to lower the severity level of
+    the log message when an item is dropped from the pipeline::
 
             class PoliteLogFormatter(logformatter.LogFormatter):
                 def dropped(self, item, exception, response, spider):

From 43fd6229684b3ccca564524fc92faf009a8c4c97 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 13 Mar 2019 10:21:50 +0000
Subject: [PATCH 1945/2974] Rule.process_request: optionally take a Response
 object

---
 scrapy/spiders/crawl.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index e5ac72e18..5aec0fd83 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -24,12 +24,23 @@ class Rule(object):
         self.callback = callback
         self.cb_kwargs = cb_kwargs or {}
         self.process_links = process_links
-        self.process_request = process_request
+        self.process_request_function = process_request
         if follow is None:
             self.follow = False if callback else True
         else:
             self.follow = follow
 
+    def process_request(self, request, response):
+        """
+        Wrapper around the request processing function to maintain backward compatibility
+        with functions that do not take a Response object as parameter.
+        """
+        argcount = self.process_request_function.__code__.co_argcount
+        if getattr(self.process_request_function, '__self__', None):
+            argcount = argcount - 1
+        args = [request] if argcount == 1 else [request, response]
+        return self.process_request_function(*args)
+
 
 class CrawlSpider(Spider):
 
@@ -65,7 +76,7 @@ class CrawlSpider(Spider):
             for link in links:
                 seen.add(link)
                 r = self._build_request(n, link)
-                yield rule.process_request(r)
+                yield rule.process_request(r, response)
 
     def _response_downloaded(self, response):
         rule = self._rules[response.meta['rule']]
@@ -93,7 +104,7 @@ class CrawlSpider(Spider):
         for rule in self._rules:
             rule.callback = get_method(rule.callback)
             rule.process_links = get_method(rule.process_links)
-            rule.process_request = get_method(rule.process_request)
+            rule.process_request_function = get_method(rule.process_request_function)
 
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):

From 22fda61d62a2b230b0e8588eabb0d71cb77141b7 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 13 Mar 2019 10:54:38 +0000
Subject: [PATCH 1946/2974] Rule.process_request: tests

---
 tests/test_spider.py | 98 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)

diff --git a/tests/test_spider.py b/tests/test_spider.py
index fefdaa403..5e20e0d99 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -263,6 +263,104 @@ class CrawlSpiderTest(SpiderTest):
                            'http://example.org/about.html',
                            'http://example.org/nofollow.html'])
 
+    def test_process_request(self):
+
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+
+        def process_request_change_domain(request):
+            return request.replace(url=request.url.replace('.org', '.com'))
+
+        class _CrawlSpider(self.spider_class):
+            name="test"
+            allowed_domains=['example.org']
+            rules = (
+                Rule(LinkExtractor(), process_request=process_request_change_domain),
+            )
+
+        spider = _CrawlSpider()
+        output = list(spider._requests_to_follow(response))
+        self.assertEqual(len(output), 3)
+        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertEqual([r.url for r in output],
+                         ['http://example.com/somepage/item/12.html',
+                          'http://example.com/about.html',
+                          'http://example.com/nofollow.html'])
+
+    def test_process_request_with_response(self):
+
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+
+        def process_request_meta_response_class(request, response):
+            request.meta['response_class'] = response.__class__.__name__
+            return request
+
+        class _CrawlSpider(self.spider_class):
+            name="test"
+            allowed_domains=['example.org']
+            rules = (
+                Rule(LinkExtractor(), process_request=process_request_meta_response_class),
+            )
+
+        spider = _CrawlSpider()
+        output = list(spider._requests_to_follow(response))
+        self.assertEqual(len(output), 3)
+        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertEqual([r.url for r in output],
+                         ['http://example.org/somepage/item/12.html',
+                          'http://example.org/about.html',
+                          'http://example.org/nofollow.html'])
+        self.assertEqual([r.meta['response_class'] for r in output],
+                         ['HtmlResponse', 'HtmlResponse', 'HtmlResponse'])
+
+    def test_process_request_instance_method(self):
+
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+
+        class _CrawlSpider(self.spider_class):
+            name="test"
+            allowed_domains=['example.org']
+            rules = (
+                Rule(LinkExtractor(), process_request='process_request_upper'),
+            )
+
+            def process_request_upper(self, request):
+                return request.replace(url=request.url.upper())
+
+        spider = _CrawlSpider()
+        output = list(spider._requests_to_follow(response))
+        self.assertEqual(len(output), 3)
+        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertEqual([r.url for r in output],
+                         ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
+                          'http://EXAMPLE.ORG/ABOUT.HTML',
+                          'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
+
+    def test_process_request_instance_method_with_response(self):
+
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+
+        class _CrawlSpider(self.spider_class):
+            name="test"
+            allowed_domains=['example.org']
+            rules = (
+                Rule(LinkExtractor(), process_request='process_request_meta_response_class'),
+            )
+
+            def process_request_meta_response_class(self, request, response):
+                request.meta['response_class'] = response.__class__.__name__
+                return request
+
+        spider = _CrawlSpider()
+        output = list(spider._requests_to_follow(response))
+        self.assertEqual(len(output), 3)
+        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertEqual([r.url for r in output],
+                         ['http://example.org/somepage/item/12.html',
+                          'http://example.org/about.html',
+                          'http://example.org/nofollow.html'])
+        self.assertEqual([r.meta['response_class'] for r in output],
+                         ['HtmlResponse', 'HtmlResponse', 'HtmlResponse'])
+
     def test_follow_links_attribute_population(self):
         crawler = get_crawler()
         spider = self.spider_class.from_crawler(crawler, 'example.com')

From b30ca379b6785c7ceb75e12285fe7865b4f607d1 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 13 Mar 2019 11:02:51 +0000
Subject: [PATCH 1947/2974] Rule.process_request: docs

---
 docs/topics/spiders.rst | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 742a88659..24b6f7ec9 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -402,10 +402,12 @@ Crawling rules
    of links extracted from each response using the specified ``link_extractor``.
    This is mainly used for filtering purposes.
 
-   ``process_request`` is a callable, or a string (in which case a method from
-   the spider object with that name will be used) which will be called with
-   every request extracted by this rule, and must return a request or None (to
-   filter out the request).
+   ``process_request`` is a callable (or a string, in which case a method from
+   the spider object with that name will be used) which will be called for
+   every request extracted by this rule. This callable should take a Request object
+   as first positional argument and, optionally, the Response object from which the
+   Request originated as second positional argument. It must return a request or None
+   (to filter out the request).
 
 CrawlSpider example
 ~~~~~~~~~~~~~~~~~~~

From 83ec947fe732035e147c21df352e199ce2cce5c8 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 13 Mar 2019 11:23:51 +0000
Subject: [PATCH 1948/2974] Rule.process_request defaults to None in the docs

---
 scrapy/spiders/crawl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 5aec0fd83..ad86fc19d 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -19,12 +19,12 @@ def identity(x):
 
 class Rule(object):
 
-    def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity):
+    def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None):
         self.link_extractor = link_extractor
         self.callback = callback
         self.cb_kwargs = cb_kwargs or {}
         self.process_links = process_links
-        self.process_request_function = process_request
+        self.process_request_function = process_request or identity
         if follow is None:
             self.follow = False if callback else True
         else:

From a2ff647aace899982bd494d73fea5a7c35ded722 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 15 Mar 2019 06:36:15 +0100
Subject: [PATCH 1949/2974] Remove docs/topics/ubuntu.rst

---
 docs/topics/ubuntu.rst | 41 -----------------------------------------
 1 file changed, 41 deletions(-)
 delete mode 100644 docs/topics/ubuntu.rst

diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst
deleted file mode 100644
index 6c993a970..000000000
--- a/docs/topics/ubuntu.rst
+++ /dev/null
@@ -1,41 +0,0 @@
-:orphan: Ubuntu packages are obsolete
-
-.. _topics-ubuntu:
-
-===============
-Ubuntu packages
-===============
-
-.. versionadded:: 0.10
-
-`Scrapinghub`_ publishes apt-gettable packages which are generally fresher than
-those in Ubuntu, and more stable too since they're continuously built from
-`GitHub repo`_ (master & stable branches) and so they contain the latest bug
-fixes.
-
-.. caution:: These packages are currently not updated and may not work on
-   Ubuntu 16.04 and above, see :issue:`2076` and :issue:`2137`.
-
-To use the packages:
-
-1. Import the GPG key used to sign Scrapy packages into APT keyring::
-
-    sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 627220E7
-
-2. Create ``/etc/apt/sources.list.d/scrapy.list`` file using the following command::
-
-    echo 'deb http://archive.scrapy.org/ubuntu scrapy main' | sudo tee /etc/apt/sources.list.d/scrapy.list
-
-3. Update package lists and install the scrapy package:
-
-   .. parsed-literal::
-
-      sudo apt-get update && sudo apt-get install scrapy
-
-.. note:: Repeat step 3 if you are trying to upgrade Scrapy.
-
-.. warning:: ``python-scrapy`` is a different package provided by official debian
-   repositories, it's very outdated and it isn't supported by Scrapy team.
-
-.. _Scrapinghub: https://scrapinghub.com/
-.. _GitHub repo: https://github.com/scrapy/scrapy

From 01ed605d02013b1d7955369562b2443d2a561599 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 15 Mar 2019 16:54:14 +0000
Subject: [PATCH 1950/2974] PEP8 changes to test_spider.py

---
 tests/test_spider.py | 60 +++++++++++++++++++++-----------------------
 1 file changed, 29 insertions(+), 31 deletions(-)

diff --git a/tests/test_spider.py b/tests/test_spider.py
index 5e20e0d99..c9af7a2d7 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -105,11 +105,11 @@ class SpiderTest(unittest.TestCase):
 
     def test_logger(self):
         spider = self.spider_class('example.com')
-        with LogCapture() as l:
+        with LogCapture() as lc:
             spider.logger.info('test log msg')
-        l.check(('example.com', 'INFO', 'test log msg'))
+        lc.check(('example.com', 'INFO', 'test log msg'))
 
-        record = l.records[0]
+        record = lc.records[0]
         self.assertIn('spider', record.__dict__)
         self.assertIs(record.spider, spider)
 
@@ -190,12 +190,11 @@ class CrawlSpiderTest(SpiderTest):
 
     def test_process_links(self):
 
-        response = HtmlResponse("http://example.org/somepage/index.html",
-            body=self.test_body)
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
 
         class _CrawlSpider(self.spider_class):
-            name="test"
-            allowed_domains=['example.org']
+            name = "test"
+            allowed_domains = ['example.org']
             rules = (
                 Rule(LinkExtractor(), process_links="dummy_process_links"),
             )
@@ -208,24 +207,24 @@ class CrawlSpiderTest(SpiderTest):
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
         self.assertEqual([r.url for r in output],
-                          ['http://example.org/somepage/item/12.html',
-                           'http://example.org/about.html',
-                           'http://example.org/nofollow.html'])
+                         ['http://example.org/somepage/item/12.html',
+                          'http://example.org/about.html',
+                          'http://example.org/nofollow.html'])
 
     def test_process_links_filter(self):
 
-        response = HtmlResponse("http://example.org/somepage/index.html",
-            body=self.test_body)
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
 
         class _CrawlSpider(self.spider_class):
             import re
 
-            name="test"
-            allowed_domains=['example.org']
+            name = "test"
+            allowed_domains = ['example.org']
             rules = (
                 Rule(LinkExtractor(), process_links="filter_process_links"),
             )
             _test_regex = re.compile('nofollow')
+
             def filter_process_links(self, links):
                 return [link for link in links
                         if not self._test_regex.search(link.url)]
@@ -235,17 +234,16 @@ class CrawlSpiderTest(SpiderTest):
         self.assertEqual(len(output), 2)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
         self.assertEqual([r.url for r in output],
-                          ['http://example.org/somepage/item/12.html',
-                           'http://example.org/about.html'])
+                         ['http://example.org/somepage/item/12.html',
+                          'http://example.org/about.html'])
 
     def test_process_links_generator(self):
 
-        response = HtmlResponse("http://example.org/somepage/index.html",
-            body=self.test_body)
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
 
         class _CrawlSpider(self.spider_class):
-            name="test"
-            allowed_domains=['example.org']
+            name = "test"
+            allowed_domains = ['example.org']
             rules = (
                 Rule(LinkExtractor(), process_links="dummy_process_links"),
             )
@@ -259,9 +257,9 @@ class CrawlSpiderTest(SpiderTest):
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
         self.assertEqual([r.url for r in output],
-                          ['http://example.org/somepage/item/12.html',
-                           'http://example.org/about.html',
-                           'http://example.org/nofollow.html'])
+                         ['http://example.org/somepage/item/12.html',
+                          'http://example.org/about.html',
+                          'http://example.org/nofollow.html'])
 
     def test_process_request(self):
 
@@ -271,8 +269,8 @@ class CrawlSpiderTest(SpiderTest):
             return request.replace(url=request.url.replace('.org', '.com'))
 
         class _CrawlSpider(self.spider_class):
-            name="test"
-            allowed_domains=['example.org']
+            name = "test"
+            allowed_domains = ['example.org']
             rules = (
                 Rule(LinkExtractor(), process_request=process_request_change_domain),
             )
@@ -295,8 +293,8 @@ class CrawlSpiderTest(SpiderTest):
             return request
 
         class _CrawlSpider(self.spider_class):
-            name="test"
-            allowed_domains=['example.org']
+            name = "test"
+            allowed_domains = ['example.org']
             rules = (
                 Rule(LinkExtractor(), process_request=process_request_meta_response_class),
             )
@@ -317,8 +315,8 @@ class CrawlSpiderTest(SpiderTest):
         response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
 
         class _CrawlSpider(self.spider_class):
-            name="test"
-            allowed_domains=['example.org']
+            name = "test"
+            allowed_domains = ['example.org']
             rules = (
                 Rule(LinkExtractor(), process_request='process_request_upper'),
             )
@@ -340,8 +338,8 @@ class CrawlSpiderTest(SpiderTest):
         response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
 
         class _CrawlSpider(self.spider_class):
-            name="test"
-            allowed_domains=['example.org']
+            name = "test"
+            allowed_domains = ['example.org']
             rules = (
                 Rule(LinkExtractor(), process_request='process_request_meta_response_class'),
             )

From 645e8d16a4c966b50bd39667aaef28dc1eeb43b8 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 15 Mar 2019 22:20:36 +0000
Subject: [PATCH 1951/2974] Count keyword argument checks

---
 tests/spiders.py    | 21 +++++++++++++--------
 tests/test_crawl.py |  3 ++-
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index 5a1471072..7b4707f62 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -31,7 +31,7 @@ class MetaSpider(MockServerSpider):
 class KeywordArgumentsSpider(MockServerSpider):
 
     name = 'kwargs'
-    checks = set()
+    checks = list()
 
     def start_requests(self):
         data = {'key': 'value', 'number': 123}
@@ -41,25 +41,30 @@ class KeywordArgumentsSpider(MockServerSpider):
         yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
 
     def parse_first(self, response, key, number):
-        self.checks.add(key == 'value')
-        self.checks.add(number == 123)
+        self.checks.append(key == 'value')
+        self.checks.append(number == 123)
+        self.crawler.stats.inc_value('boolean_checks', 2)
         yield response.follow(
             self.mockserver.url('/two'),
             self.parse_second,
             kwargs={'new_key': 'new_value'})
 
     def parse_second(self, response, new_key):
-        self.checks.add(new_key == 'new_value')
+        self.checks.append(new_key == 'new_value')
+        self.crawler.stats.inc_value('boolean_checks')
 
     def parse_general(self, response, **kwargs):
         if response.url.endswith('/general_with'):
-            self.checks.add(kwargs['key'] == 'value')
-            self.checks.add(kwargs['number'] == 123)
+            self.checks.append(kwargs['key'] == 'value')
+            self.checks.append(kwargs['number'] == 123)
+            self.crawler.stats.inc_value('boolean_checks', 2)
         elif response.url.endswith('/general_without'):
-            self.checks.add(kwargs == {})
+            self.checks.append(kwargs == {})
+            self.crawler.stats.inc_value('boolean_checks')
 
     def parse_no_kwargs(self, response):
-        pass
+        self.checks.append(response.url.endswith('/no_kwargs'))
+        self.crawler.stats.inc_value('boolean_checks')
 
 
 class FollowAllSpider(MetaSpider):
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3879a017c..9a39b8cb4 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -27,7 +27,8 @@ class CrawlTestCase(TestCase):
     def test_callback_kwargs(self):
         crawler = self.runner.create_crawler(KeywordArgumentsSpider)
         yield crawler.crawl(mockserver=self.mockserver)
-        self.assertEqual(crawler.spider.checks, set([True]))
+        self.assertTrue(all(crawler.spider.checks))
+        self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks'))
 
     @defer.inlineCallbacks
     def test_follow_all(self):

From 6760bca74b1f51ce83ed73318d5ddcef03c9d129 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 15 Mar 2019 22:32:45 +0000
Subject: [PATCH 1952/2974] Rename Request.kwargs to Request.cb_kwargs

---
 docs/topics/request-response.rst | 16 ++++++++--------
 scrapy/core/scraper.py           |  4 +++-
 scrapy/http/request/__init__.py  | 14 +++++++-------
 scrapy/http/response/__init__.py |  4 ++--
 scrapy/http/response/text.py     |  4 ++--
 scrapy/utils/reqser.py           |  4 ++--
 tests/spiders.py                 |  6 +++---
 tests/test_http_request.py       |  8 ++++----
 tests/test_utils_reqser.py       |  4 ++--
 9 files changed, 33 insertions(+), 31 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index d12766676..b3f849540 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
 Request objects
 ===============
 
-.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, kwargs])
+.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, cb_kwargs])
 
     A :class:`Request` object represents an HTTP request, which is usually
     generated in the Spider and executed by the Downloader, and thus generating
@@ -126,8 +126,8 @@ Request objects
     :param flags:  Flags sent to the request, can be used for logging or similar purposes.
     :type flags: list
 
-    :param kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback.
-    :type kwargs: dict
+    :param cb_kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback.
+    :type cb_kwargs: dict
 
     .. attribute:: Request.url
 
@@ -168,7 +168,7 @@ Request objects
         ``copy()`` or ``replace()`` methods, and can also be accessed, in your
         spider, from the ``response.meta`` attribute.
 
-    .. attribute:: Request.kwargs
+    .. attribute:: Request.cb_kwargs
 
         A dictionary that contains arbitrary metadata for this request. Its contents
         will be passed to the Request's callback as keyword arguments. It is empty
@@ -177,7 +177,7 @@ Request objects
 
         This dict is `shallow copied`_ when the request is cloned using the
         ``copy()`` or ``replace()`` methods, and can also be accessed, in your
-        spider, from the ``response.kwargs`` attribute.
+        spider, from the ``response.cb_kwargs`` attribute.
 
     .. _shallow copied: https://docs.python.org/2/library/copy.html
 
@@ -216,7 +216,7 @@ Example::
 In some cases you may be interested in passing arguments to those callback
 functions so you can receive the arguments later, in the second callback.
 The following two examples show how to achieve this by using the 
-:attr:`Request.meta` and :attr:`Request.kwargs` attributes respectively::
+:attr:`Request.meta` and :attr:`Request.cb_kwargs` attributes respectively::
 
     def parse_page1(self, response):
         item = MyItem()
@@ -238,8 +238,8 @@ The following two examples show how to achieve this by using the
         item['main_url'] = response.url
         request = scrapy.Request("http://www.example.com/some_page.html",
                                  callback=self.parse_page2)
-        request.kwargs['item'] = item
-        request.kwargs['foo'] = 'bar'
+        request.cb_kwargs['item'] = item
+        request.cb_kwargs['foo'] = 'bar'
         yield request
 
     def parse_page2(self, response, item, foo):
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 7981ce231..08dd1acc5 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -143,7 +143,9 @@ class Scraper(object):
     def call_spider(self, result, request, spider):
         result.request = request
         dfd = defer_result(result)
-        dfd.addCallbacks(request.callback or spider.parse, request.errback, callbackKeywords=request.kwargs)
+        dfd.addCallbacks(callback=request.callback or spider.parse,
+                         errback=request.errback,
+                         callbackKeywords=request.cb_kwargs)
         return dfd.addCallback(iterate_spider_output)
 
     def handle_spider_error(self, _failure, request, response, spider):
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index c016eb727..f5935c4ef 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -18,7 +18,7 @@ class Request(object_ref):
 
     def __init__(self, url, callback=None, method='GET', headers=None, body=None,
                  cookies=None, meta=None, encoding='utf-8', priority=0,
-                 dont_filter=False, errback=None, flags=None, kwargs=None):
+                 dont_filter=False, errback=None, flags=None, cb_kwargs=None):
 
         self._encoding = encoding  # this one has to be set first
         self.method = str(method).upper()
@@ -40,14 +40,14 @@ class Request(object_ref):
         self.dont_filter = dont_filter
 
         self._meta = dict(meta) if meta else None
-        self._kwargs = dict(kwargs) if kwargs else None
+        self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None
         self.flags = [] if flags is None else list(flags)
 
     @property
-    def kwargs(self):
-        if self._kwargs is None:
-            self._kwargs = {}
-        return self._kwargs
+    def cb_kwargs(self):
+        if self._cb_kwargs is None:
+            self._cb_kwargs = {}
+        return self._cb_kwargs
 
     @property
     def meta(self):
@@ -99,7 +99,7 @@ class Request(object_ref):
         given new values.
         """
         for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',
-                  'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'kwargs']:
+                  'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']:
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 99b04a26e..b0a526b72 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -106,7 +106,7 @@ class Response(object_ref):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding='utf-8', priority=0,
-               dont_filter=False, errback=None, kwargs=None):
+               dont_filter=False, errback=None, cb_kwargs=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -133,4 +133,4 @@ class Response(object_ref):
                        priority=priority,
                        dont_filter=dont_filter,
                        errback=errback,
-                       kwargs=kwargs)
+                       cb_kwargs=cb_kwargs)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 2039621b3..339913d4e 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -123,7 +123,7 @@ class TextResponse(Response):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding=None, priority=0,
-               dont_filter=False, errback=None, kwargs=None):
+               dont_filter=False, errback=None, cb_kwargs=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -155,7 +155,7 @@ class TextResponse(Response):
             priority=priority,
             dont_filter=dont_filter,
             errback=errback,
-            kwargs=kwargs,
+            cb_kwargs=cb_kwargs,
         )
 
 
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index d537057b1..e7016b92a 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -33,7 +33,7 @@ def request_to_dict(request, spider=None):
         'priority': request.priority,
         'dont_filter': request.dont_filter,
         'flags': request.flags,
-        'kwargs': request.kwargs,
+        'cb_kwargs': request.cb_kwargs,
     }
     if type(request) is not Request:
         d['_class'] = request.__module__ + '.' + request.__class__.__name__
@@ -66,7 +66,7 @@ def request_from_dict(d, spider=None):
         priority=d['priority'],
         dont_filter=d['dont_filter'],
         flags=d.get('flags'),
-        kwargs=d.get('kwargs'),
+        cb_kwargs=d.get('cb_kwargs'),
     )
 
 
diff --git a/tests/spiders.py b/tests/spiders.py
index 7b4707f62..a06985837 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -35,8 +35,8 @@ class KeywordArgumentsSpider(MockServerSpider):
 
     def start_requests(self):
         data = {'key': 'value', 'number': 123}
-        yield Request(self.mockserver.url('/first'), self.parse_first, kwargs=data)
-        yield Request(self.mockserver.url('/general_with'), self.parse_general, kwargs=data)
+        yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data)
+        yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
         yield Request(self.mockserver.url('/general_without'), self.parse_general)
         yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
 
@@ -47,7 +47,7 @@ class KeywordArgumentsSpider(MockServerSpider):
         yield response.follow(
             self.mockserver.url('/two'),
             self.parse_second,
-            kwargs={'new_key': 'new_value'})
+            cb_kwargs={'new_key': 'new_value'})
 
     def parse_second(self, response, new_key):
         self.checks.append(new_key == 'new_value')
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 610893d8a..c1949a28c 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -177,7 +177,7 @@ class RequestTest(unittest.TestCase):
         r1 = self.request_class("http://www.example.com", flags=['f1', 'f2'],
                                 callback=somecallback, errback=somecallback)
         r1.meta['foo'] = 'bar'
-        r1.kwargs['key'] = 'value'
+        r1.cb_kwargs['key'] = 'value'
         r2 = r1.copy()
 
         # make sure copy does not propagate callbacks
@@ -190,9 +190,9 @@ class RequestTest(unittest.TestCase):
         assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
         self.assertEqual(r1.flags, r2.flags)
 
-        # make sure kwargs dict is shallow copied
-        assert r1.kwargs is not r2.kwargs, "kwargs must be a shallow copy, not identical"
-        self.assertEqual(r1.kwargs, r2.kwargs)
+        # make sure cb_kwargs dict is shallow copied
+        assert r1.cb_kwargs is not r2.cb_kwargs, "cb_kwargs must be a shallow copy, not identical"
+        self.assertEqual(r1.cb_kwargs, r2.cb_kwargs)
 
         # make sure meta dict is shallow copied
         assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical"
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 76de20f22..e1601b76b 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -26,7 +26,7 @@ class RequestSerializationTest(unittest.TestCase):
             encoding='latin-1',
             priority=20,
             meta={'a': 'b'},
-            kwargs={'k': 'v'},
+            cb_kwargs={'k': 'v'},
             flags=['testFlag'])
         self._assert_serializes_ok(r, spider=self.spider)
 
@@ -53,7 +53,7 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertEqual(r1.headers, r2.headers)
         self.assertEqual(r1.cookies, r2.cookies)
         self.assertEqual(r1.meta, r2.meta)
-        self.assertEqual(r1.kwargs, r2.kwargs)
+        self.assertEqual(r1.cb_kwargs, r2.cb_kwargs)
         self.assertEqual(r1._encoding, r2._encoding)
         self.assertEqual(r1.priority, r2.priority)
         self.assertEqual(r1.dont_filter, r2.dont_filter)

From 92bbc5290d2b381ea60d68442a887d1ba020874e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 16 Mar 2019 05:41:40 +0000
Subject: [PATCH 1953/2974] Rule.process_request - Renaming

---
 scrapy/spiders/crawl.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index ad86fc19d..c01f75798 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -24,22 +24,22 @@ class Rule(object):
         self.callback = callback
         self.cb_kwargs = cb_kwargs or {}
         self.process_links = process_links
-        self.process_request_function = process_request or identity
+        self.process_request = process_request or identity
         if follow is None:
             self.follow = False if callback else True
         else:
             self.follow = follow
 
-    def process_request(self, request, response):
+    def _process_request(self, request, response):
         """
         Wrapper around the request processing function to maintain backward compatibility
         with functions that do not take a Response object as parameter.
         """
-        argcount = self.process_request_function.__code__.co_argcount
-        if getattr(self.process_request_function, '__self__', None):
+        argcount = self.process_request.__code__.co_argcount
+        if hasattr(self.process_request, '__self__'):
             argcount = argcount - 1
         args = [request] if argcount == 1 else [request, response]
-        return self.process_request_function(*args)
+        return self.process_request(*args)
 
 
 class CrawlSpider(Spider):
@@ -75,8 +75,8 @@ class CrawlSpider(Spider):
                 links = rule.process_links(links)
             for link in links:
                 seen.add(link)
-                r = self._build_request(n, link)
-                yield rule.process_request(r, response)
+                request = self._build_request(n, link)
+                yield rule._process_request(request, response)
 
     def _response_downloaded(self, response):
         rule = self._rules[response.meta['rule']]
@@ -104,7 +104,7 @@ class CrawlSpider(Spider):
         for rule in self._rules:
             rule.callback = get_method(rule.callback)
             rule.process_links = get_method(rule.process_links)
-            rule.process_request_function = get_method(rule.process_request_function)
+            rule.process_request = get_method(rule.process_request)
 
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):

From 044318920a463d2e04efe2a4c65d8f72d1b5ecb6 Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Sun, 17 Mar 2019 16:54:28 +0530
Subject: [PATCH 1954/2974] doc for creating custom cache storage backend.

---
 docs/topics/downloader-middleware.rst | 53 +++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 0d976077b..f913b059d 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -496,6 +496,59 @@ In order to use this storage backend:
 .. _LevelDB: https://github.com/google/leveldb
 .. _leveldb python bindings: https://pypi.python.org/pypi/leveldb
 
+.. _httpcache-storage-custom:
+
+Implementing custom cache storage backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can implement custom cache storage backend by creating a Python class that
+defines the methods described below.
+
+.. module:: scrapy.extensions.httpcache
+
+.. class:: CacheStorage
+
+    .. method:: open_spider(spider)
+
+      This method gets called after a spider has been opened for crawling.
+
+      :param spider: the spider which has been opened
+      :type spider: :class:`~scrapy.spiders.Spider` object
+
+    .. method:: close_spider(spider)
+
+      This method gets called after a spider has been closed.
+
+      :param spider: the spider which has been closed
+      :type spider: :class:`~scrapy.spiders.Spider` object
+
+    .. method:: retrieve_response(spider, request)
+
+      Returns response if present in cache, or ``None`` otherwise.
+
+      :param spider: the spider which generated the request
+      :type spider: :class:`~scrapy.spiders.Spider` object
+
+      :param request: the request to find cached reponse for
+      :type request: :class:`~scrapy.http.Request` object
+
+    .. method:: store_response(spider, request, response)
+
+      Stores the given response in the cache.
+
+      :param spider: the spider for which the response is intended
+      :type spider: :class:`~scrapy.spiders.Spider` object
+
+      :param request: corresponding request the spider generated
+      :type request: :class:`~scrapy.http.Request` object
+
+      :param response: the response to store in the cache
+      :type response: :class:`~scrapy.http.Response` object
+
+In order to use your storage backend, set:
+
+* :setting:`HTTPCACHE_STORAGE` to path of your custom storage class.
+
 
 HTTPCache middleware settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

From 70aa5b1333a981b3b5eae70e29144c4c25fcfd4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Wed, 20 Mar 2019 15:32:20 +0100
Subject: [PATCH 1955/2974] Fix numeration

---
 docs/topics/selectors.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index edc18f14d..282a585d4 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -436,7 +436,7 @@ The following examples show how these methods map to each other.
      >>> response.css('a::attr(href)').extract()
      ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
 
-2. ``Selector.get()`` is the same as ``Selector.extract()``::
+3. ``Selector.get()`` is the same as ``Selector.extract()``::
 
      >>> response.css('a::attr(href)')[0].get()
      'image1.html'

From 821f5bb26077d7f9a6b2b1a72f210f81779f5393 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Mon, 3 Dec 2018 11:00:03 +0000
Subject: [PATCH 1956/2974] First implementation handle exception use O(N)
 instead of O(NlogN) here we have request as struct additional check for
 meptiness small performance improvement do not consume another request test
 number of responses mark requests back to 3 slots test case raise exceptions
 in case of missed meta add marks to requests and work only with your own
 requests only disk queue should obtain signals separate functions for slot
 rasd/write use signlas without variable stop crawler get signals in correct
 place logic test for download-aware priority queue update comment for
 structure ensure text type transform slot name to path use implicit structure
 use unicode type implicitly use real crawler add signals more slot accounting
 simple implementation of pop small slot accounting code no need for custom
 len function ability to call super in py27 add slots generic tests for
 downloader aware queue dummy implementation of crawler aware priority queue
 move common logic to base class rename class pass crawler to pqclass
 constructor do not copy quelib.PriorityQueue code add comment about new class
 remove obsolete function modify behaviour of queuelib.PriorityQueue to dodge
 very complex priority better way to get name remove obsolete commentary check
 boundaries function for priority convertion with known limits correct import
 path move file do not switch on by deffault as ip concurrency not supported
 set scheduler slot in case of empty slot use constant single place for added
 urls single place for constants use as default queue correct format for error
 text test migration from old version with on disk queue in these tests we
 have only two inflection points - jobdir and priority_queue_cls we do not
 need separate mock spider, use usual one do not rely on order of dict
 elements, imply order of list test round robiness of priority queue add
 comments and requirements for our magick function remove debug logging put
 queues into slot as we fabricate priorities we do not need special types
 anymore fabricate priority for priority queue more versatile priorities
 Scheduler class is not inflection point wrap correct types check for
 emptinees before initialization tests for new priority queue correct default
 type for startprios use exact values put common settings to base class test
 priorities for disk scheduler test dequeue for disk scheduler test length for
 disk scheduler setUp/tearDown methods for on disk schedulers new methods
 remove excessive line base class to handle scheduler creation correct method
 names test priorities deque test close scheduler on test end enqueue some
 requests test template for scheduler use downloader slot I/O implementation
 for RoundRobin queue round-robin implementation without I/O and slot
 detection wrappers for every disk queue class

---
 scrapy/core/downloader/__init__.py |   8 +-
 scrapy/core/queues.py              |  15 --
 scrapy/core/scheduler.py           |  17 +-
 scrapy/pqueues.py                  | 246 ++++++++++++++++++++++
 tests/test_scheduler.py            | 315 +++++++++++++++++++++++++++++
 5 files changed, 578 insertions(+), 23 deletions(-)
 delete mode 100644 scrapy/core/queues.py
 create mode 100644 scrapy/pqueues.py
 create mode 100644 tests/test_scheduler.py

diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 59c3ad074..4695d75f4 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -75,6 +75,8 @@ def _get_concurrency_delay(concurrency, spider, settings):
 
 class Downloader(object):
 
+    DOWNLOAD_SLOT = 'download_slot'
+
     def __init__(self, crawler):
         self.settings = crawler.settings
         self.signals = crawler.signals
@@ -111,8 +113,8 @@ class Downloader(object):
         return key, self.slots[key]
 
     def _get_slot_key(self, request, spider):
-        if 'download_slot' in request.meta:
-            return request.meta['download_slot']
+        if self.DOWNLOAD_SLOT in request.meta:
+            return request.meta[self.DOWNLOAD_SLOT]
 
         key = urlparse_cached(request).hostname or ''
         if self.ip_concurrency:
@@ -122,7 +124,7 @@ class Downloader(object):
 
     def _enqueue_request(self, request, spider):
         key, slot = self._get_slot(request, spider)
-        request.meta['download_slot'] = key
+        request.meta[self.DOWNLOAD_SLOT] = key
 
         def _deactivate(response):
             slot.active.remove(request)
diff --git a/scrapy/core/queues.py b/scrapy/core/queues.py
deleted file mode 100644
index 96d582fc7..000000000
--- a/scrapy/core/queues.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import uuid
-import os.path
-
-
-def unique_files_queue(queue_class):
-
-    class UniqueFilesQueue(queue_class):
-        def __init__(self, path):
-            path = path + "-" + uuid.uuid4().hex
-            while os.path.exists(path):
-                path = path + "-" + uuid.uuid4().hex
-
-            super().__init__(path)
-
-    return UniqueFilesQueue
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index eb790a67e..d40f3aa0c 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
 class Scheduler(object):
 
     def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None,
-                 logunser=False, stats=None, pqclass=None):
+                 logunser=False, stats=None, pqclass=None, crawler=None):
         self.df = dupefilter
         self.dqdir = self._dqdir(jobdir)
         self.pqclass = pqclass
@@ -21,6 +21,7 @@ class Scheduler(object):
         self.mqclass = mqclass
         self.logunser = logunser
         self.stats = stats
+        self.crawler = crawler
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -32,14 +33,15 @@ class Scheduler(object):
         mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
         logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS', settings.getbool('SCHEDULER_DEBUG'))
         return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
-                   stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass)
+                   stats=crawler.stats, pqclass=pqclass, dqclass=dqclass,
+                   mqclass=mqclass, crawler=crawler)
 
     def has_pending_requests(self):
         return len(self) > 0
 
     def open(self, spider):
         self.spider = spider
-        self.mqs = self.pqclass(self._newmq)
+        self.mqs = create_instance(self.pqclass, None, self.crawler, self._newmq)
         self.dqs = self._dq() if self.dqdir else None
         return self.df.open()
 
@@ -111,7 +113,7 @@ class Scheduler(object):
         return self.mqclass()
 
     def _newdq(self, priority):
-        return self.dqclass(join(self.dqdir, 'p%s' % priority))
+        return self.dqclass(join(self.dqdir, 'p%s' % (priority, )))
 
     def _dq(self):
         activef = join(self.dqdir, 'active.json')
@@ -120,7 +122,12 @@ class Scheduler(object):
                 prios = json.load(f)
         else:
             prios = ()
-        q = self.pqclass(self._newdq, startprios=prios)
+
+        q = create_instance(self.pqclass,
+                            None,
+                            self.crawler,
+                            self._newdq,
+                            startprios=prios)
         if q:
             logger.info("Resuming crawl (%(queuesize)d requests scheduled)",
                         {'queuesize': len(q)}, extra={'spider': self.spider})
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
new file mode 100644
index 000000000..75073b7a4
--- /dev/null
+++ b/scrapy/pqueues.py
@@ -0,0 +1,246 @@
+from collections import deque
+import hashlib
+import logging
+from six import text_type
+from six.moves.urllib.parse import urlparse
+
+from queuelib import PriorityQueue
+
+from scrapy.core.downloader import Downloader
+from scrapy.http import Request
+from scrapy.signals import request_reached_downloader, response_downloaded
+
+
+logger = logging.getLogger(__name__)
+
+
+SCHEDULER_SLOT_META_KEY = Downloader.DOWNLOAD_SLOT
+
+
+def _get_from_request(request, key, default=None):
+    if isinstance(request, dict):
+        return request.get(key, default)
+
+    if isinstance(request, Request):
+        return getattr(request, key, default)
+
+    raise ValueError('Bad type of request "%s"' % (request.__class__, ))
+
+
+def _scheduler_slot_read(request, default=None):
+    meta = _get_from_request(request, 'meta', dict())
+    slot = meta.get(SCHEDULER_SLOT_META_KEY, default)
+    return slot
+
+
+def _scheduler_slot_write(request, slot):
+    meta = _get_from_request(request, 'meta', None)
+    if not isinstance(meta, dict):
+        raise ValueError('No meta attribute in %s' % (request, ))
+    meta[SCHEDULER_SLOT_META_KEY] = slot
+
+
+def _scheduler_slot(request):
+
+    slot = _scheduler_slot_read(request, None)
+    if slot is None:
+        url = _get_from_request(request, 'url')
+        slot = urlparse(url).hostname or ''
+        _scheduler_slot_write(request, slot)
+
+    return slot
+
+
+def _pathable(x):
+    pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in x])
+
+    """
+        as we replace some letters we can get collision for different slots
+        add we add unique part
+    """
+    unique_slot = hashlib.md5(x.encode('utf8')).hexdigest()
+
+    return '-'.join([pathable_slot, unique_slot])
+
+
+class PrioritySlot:
+    __slots__ = ('priority', 'slot')
+
+    def __init__(self, priority=0, slot=None):
+        self.priority = priority
+        self.slot = slot
+
+    def __hash__(self):
+       return hash((self.priority, self.slot))
+
+    def __eq__(self, other):
+       return (self.priority, self.slot) == (other.priority, other.slot)
+
+    def __lt__(self, other):
+       return (self.priority, self.slot) < (other.priority, other.slot)
+
+    def __str__(self):
+       return '_'.join([text_type(self.priority), _pathable(text_type(self.slot))])
+
+
+class PriorityAsTupleQueue(PriorityQueue):
+    """
+        Python structures is not directly (de)serialized (to)from json.
+        We need this modified queue to transform custom structure (from)to
+        json serializable structures
+    """
+    def __init__(self, qfactory, startprios=()):
+
+        super(PriorityAsTupleQueue, self).__init__(
+                qfactory,
+                [PrioritySlot(priority=p[0], slot=p[1]) for p in startprios]
+                )
+
+    def close(self):
+        startprios = super(PriorityAsTupleQueue, self).close()
+        return [(s.priority, s.slot) for s in startprios]
+
+    def is_empty(self):
+        return not self.queues or len(self) == 0
+
+
+class SlotBasedPriorityQueue(object):
+
+    def __init__(self, qfactory, startprios={}):
+        self.pqueues = dict()     # slot -> priority queue
+        self.qfactory = qfactory  # factory for creating new internal queues
+
+        if not startprios:
+            return
+
+        if not isinstance(startprios, dict):
+            raise ValueError("Looks like your priorities file malforfemed. "
+                             "Possible reason: You run scrapy with previous "
+                             "version. Interrupted it. Updated scrapy. And "
+                             "run again.")
+
+        for slot, prios in startprios.items():
+            self.pqueues[slot] = PriorityAsTupleQueue(self.qfactory, prios)
+
+    def pop_slot(self, slot):
+        queue = self.pqueues[slot]
+        request = queue.pop()
+        is_empty = queue.is_empty()
+        if is_empty:
+            del self.pqueues[slot]
+
+        return request, is_empty
+
+    def push_slot(self, request, priority):
+        slot = _scheduler_slot(request)
+        is_new = False
+        if slot not in self.pqueues:
+            is_new = True
+            self.pqueues[slot] = PriorityAsTupleQueue(self.qfactory)
+        self.pqueues[slot].push(request, PrioritySlot(priority=priority, slot=slot))
+        return slot, is_new
+
+    def close(self):
+        startprios = dict()
+        for slot, queue in self.pqueues.items():
+            prios = queue.close()
+            startprios[slot] = prios
+        self.pqueues.clear()
+        return startprios
+
+    def __len__(self):
+        return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
+
+
+class RoundRobinPriorityQueue(SlotBasedPriorityQueue):
+
+    def __init__(self, qfactory, startprios={}):
+        super(RoundRobinPriorityQueue, self).__init__(qfactory, startprios)
+        self._slots = deque()
+        for slot in self.pqueues:
+            self._slots.append(slot)
+
+    def push(self, request, priority):
+        slot, is_new = self.push_slot(request, priority)
+        if is_new:
+            self._slots.append(slot)
+
+    def pop(self):
+        if not self._slots:
+            return
+
+        slot = self._slots.popleft()
+        request, is_empty = self.pop_slot(slot)
+
+        if not is_empty:
+            self._slots.append(slot)
+
+        return request
+
+    def close(self):
+        self._slots.clear()
+        return super(RoundRobinPriorityQueue, self).close()
+
+
+class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue):
+
+    _DOWNLOADER_AWARE_PQ_ID = 'DOWNLOADER_AWARE_PQ_ID'
+
+    @classmethod
+    def from_crawler(cls, crawler, qfactory, startprios={}):
+        return cls(crawler, qfactory, startprios)
+
+    def __init__(self, crawler, qfactory, startprios={}):
+        super(DownloaderAwarePriorityQueue, self).__init__(qfactory, startprios)
+        self._slots = {slot: 0 for slot in self.pqueues}
+        crawler.signals.connect(self.on_response_download,
+                                signal=response_downloaded)
+        crawler.signals.connect(self.on_request_reached_downloader,
+                                signal=request_reached_downloader)
+
+    def mark(self, request):
+        meta = _get_from_request(request, 'meta', None)
+        if not isinstance(meta, dict):
+            raise ValueError('No meta attribute in %s' % (request, ))
+        meta[self._DOWNLOADER_AWARE_PQ_ID] = id(self)
+
+    def check_mark(self, request):
+        return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self)
+
+    def pop(self):
+        slots = [(d, s) for s,d in self._slots.items() if s in self.pqueues]
+
+        if not slots:
+            return
+
+        slot = min(slots)[1]
+        request, _ = self.pop_slot(slot)
+        self.mark(request)
+        return request
+
+    def push(self, request, priority):
+        slot, _ = self.push_slot(request, priority)
+        if slot not in self._slots:
+            self._slots[slot] = 0
+
+    def on_response_download(self, response, request, spider):
+        if not self.check_mark(request):
+            return
+
+        slot = _scheduler_slot_read(request)
+        if slot not in self._slots or self._slots[slot] <= 0:
+            raise ValueError('Get response for wrong slot "%s"' % (slot, ))
+        self._slots[slot] = self._slots[slot] - 1
+        if self._slots[slot] == 0 and slot not in self.pqueues:
+            del self._slots[slot]
+
+    def on_request_reached_downloader(self, request, spider):
+        if not self.check_mark(request):
+            return
+
+        slot = _scheduler_slot_read(request)
+        self._slots[slot] = self._slots.get(slot, 0) + 1
+
+    def close(self):
+        self._slots.clear()
+        return super(DownloaderAwarePriorityQueue, self).close()
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
new file mode 100644
index 000000000..fd86e8d8c
--- /dev/null
+++ b/tests/test_scheduler.py
@@ -0,0 +1,315 @@
+import contextlib
+import shutil
+import tempfile
+import unittest
+
+from scrapy.crawler import Crawler
+from scrapy.core.scheduler import Scheduler
+from scrapy.http import Request
+from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write
+from scrapy.signals import request_reached_downloader, response_downloaded
+from scrapy.spiders import Spider
+
+class MockCrawler(Crawler):
+    def __init__(self, priority_queue_cls, jobdir):
+
+        settings = dict(LOG_UNSERIALIZABLE_REQUESTS=False,
+                       SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
+                       SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
+                       SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
+                       JOBDIR=jobdir,
+                       DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter')
+        super(MockCrawler, self).__init__(Spider, settings)
+
+
+class SchedulerHandler:
+    priority_queue_cls = None
+    jobdir = None
+
+    def create_scheduler(self):
+        self.mock_crawler = MockCrawler(self.priority_queue_cls, self.jobdir)
+        self.scheduler = Scheduler.from_crawler(self.mock_crawler)
+        self.spider = Spider(name='spider')
+        self.scheduler.open(self.spider)
+
+    def close_scheduler(self):
+        self.scheduler.close('finished')
+        self.mock_crawler.stop()
+
+    def setUp(self):
+        self.create_scheduler()
+
+    def tearDown(self):
+        self.close_scheduler()
+
+
+_PRIORITIES = [("http://foo.com/a", -2),
+               ("http://foo.com/d", 1),
+               ("http://foo.com/b", -1),
+               ("http://foo.com/c", 0),
+               ("http://foo.com/e", 2)]
+
+
+_URLS = {"http://foo.com/a", "http://foo.com/b", "http://foo.com/c"}
+
+
+class BaseSchedulerInMemoryTester(SchedulerHandler):
+    def test_length(self):
+        self.assertFalse(self.scheduler.has_pending_requests())
+        self.assertEqual(len(self.scheduler), 0)
+
+        for url in _URLS:
+            self.scheduler.enqueue_request(Request(url))
+
+        self.assertTrue(self.scheduler.has_pending_requests())
+        self.assertEqual(len(self.scheduler), len(_URLS))
+
+    def test_dequeue(self):
+        for url in _URLS:
+            self.scheduler.enqueue_request(Request(url))
+
+        urls = set()
+        while self.scheduler.has_pending_requests():
+            urls.add(self.scheduler.next_request().url)
+
+        self.assertEqual(urls, _URLS)
+
+    def test_dequeue_priorities(self):
+        for url, priority in _PRIORITIES:
+            self.scheduler.enqueue_request(Request(url, priority=priority))
+
+        priorities = list()
+        while self.scheduler.has_pending_requests():
+            priorities.append(self.scheduler.next_request().priority)
+
+        self.assertEqual(priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x))
+
+
+class BaseSchedulerOnDiskTester(SchedulerHandler):
+
+    def setUp(self):
+        self.jobdir = tempfile.mkdtemp()
+        self.create_scheduler()
+
+    def tearDown(self):
+        self.close_scheduler()
+
+        shutil.rmtree(self.jobdir)
+        self.jobdir = None
+
+    def test_length(self):
+        self.assertFalse(self.scheduler.has_pending_requests())
+        self.assertEqual(len(self.scheduler), 0)
+
+        for url in _URLS:
+            self.scheduler.enqueue_request(Request(url))
+
+        self.close_scheduler()
+        self.create_scheduler()
+
+        self.assertTrue(self.scheduler.has_pending_requests())
+        self.assertEqual(len(self.scheduler), len(_URLS))
+
+    def test_dequeue(self):
+        for url in _URLS:
+            self.scheduler.enqueue_request(Request(url))
+
+        self.close_scheduler()
+        self.create_scheduler()
+
+        urls = set()
+        while self.scheduler.has_pending_requests():
+            urls.add(self.scheduler.next_request().url)
+
+        self.assertEqual(urls, _URLS)
+
+    def test_dequeue_priorities(self):
+        for url, priority in _PRIORITIES:
+            self.scheduler.enqueue_request(Request(url, priority=priority))
+
+        self.close_scheduler()
+        self.create_scheduler()
+
+        priorities = list()
+        while self.scheduler.has_pending_requests():
+            priorities.append(self.scheduler.next_request().priority)
+
+        self.assertEqual(priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x))
+
+
+class TestSchedulerInMemory(BaseSchedulerInMemoryTester, unittest.TestCase):
+    priority_queue_cls = 'queuelib.PriorityQueue'
+
+
+class TestSchedulerOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase):
+    priority_queue_cls = 'queuelib.PriorityQueue'
+
+
+_SLOTS = [("http://foo.com/a", 'a'),
+          ("http://foo.com/b", 'a'),
+          ("http://foo.com/c", 'b'),
+          ("http://foo.com/d", 'b'),
+          ("http://foo.com/e", 'c'),
+          ("http://foo.com/f", 'c')]
+
+
+class TestSchedulerWithRoundRobinInMemory(BaseSchedulerInMemoryTester, unittest.TestCase):
+    priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue'
+
+    def test_round_robin(self):
+        for url, slot in _SLOTS:
+            request = Request(url)
+            _scheduler_slot_write(request, slot)
+            self.scheduler.enqueue_request(request)
+
+        slots = list()
+        while self.scheduler.has_pending_requests():
+            slots.append(_scheduler_slot_read(self.scheduler.next_request()))
+
+        for i in range(0, len(_SLOTS), 2):
+            self.assertNotEqual(slots[i], slots[i+1])
+
+    def test_is_meta_set(self):
+        url = "http://foo.com/a"
+        request = Request(url)
+        if _scheduler_slot_read(request):
+            _scheduler_slot_write(request, None)
+        self.scheduler.enqueue_request(request)
+        self.assertIsNotNone(_scheduler_slot_read(request, None), None)
+
+
+class TestSchedulerWithRoundRobinOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase):
+    priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue'
+
+    def test_round_robin(self):
+        for url, slot in _SLOTS:
+            request = Request(url)
+            _scheduler_slot_write(request, slot)
+            self.scheduler.enqueue_request(request)
+
+        self.close_scheduler()
+        self.create_scheduler()
+
+        slots = list()
+        while self.scheduler.has_pending_requests():
+            slots.append(_scheduler_slot_read(self.scheduler.next_request()))
+
+        for i in range(0, len(_SLOTS), 2):
+            self.assertNotEqual(slots[i], slots[i+1])
+
+    def test_is_meta_set(self):
+        url = "http://foo.com/a"
+        request = Request(url)
+        if _scheduler_slot_read(request):
+            _scheduler_slot_write(request, None)
+        self.scheduler.enqueue_request(request)
+
+        self.close_scheduler()
+        self.create_scheduler()
+
+        self.assertIsNotNone(_scheduler_slot_read(request, None), None)
+
+
+@contextlib.contextmanager
+def mkdtemp():
+    dir = tempfile.mkdtemp()
+    try:
+        yield dir
+    finally:
+        shutil.rmtree(dir)
+
+
+def _migration():
+
+    with mkdtemp() as tmp_dir:
+        prev_scheduler_handler = SchedulerHandler()
+        prev_scheduler_handler.priority_queue_cls = 'queuelib.PriorityQueue'
+        prev_scheduler_handler.jobdir = tmp_dir
+
+        prev_scheduler_handler.create_scheduler()
+        for url in _URLS:
+            prev_scheduler_handler.scheduler.enqueue_request(Request(url))
+        prev_scheduler_handler.close_scheduler()
+
+        next_scheduler_handler = SchedulerHandler()
+        next_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue'
+        next_scheduler_handler.jobdir = tmp_dir
+
+        next_scheduler_handler.create_scheduler()
+
+
+class TestMigration(unittest.TestCase):
+    def test_migration(self):
+        self.assertRaises(ValueError, _migration)
+
+
+class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, unittest.TestCase):
+    priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
+
+    def test_logic(self):
+        for url, slot in _SLOTS:
+            request = Request(url)
+            _scheduler_slot_write(request, slot)
+            self.scheduler.enqueue_request(request)
+
+        slots = list()
+        requests = list()
+        while self.scheduler.has_pending_requests():
+            request = self.scheduler.next_request()
+            slots.append(_scheduler_slot_read(request))
+            self.mock_crawler.signals.send_catch_log(
+                    signal=request_reached_downloader,
+                    request=request,
+                    spider=self.spider
+                    )
+            requests.append(request)
+        self.assertEqual(len(slots), len(_SLOTS))
+
+        for request in requests:
+            self.mock_crawler.signals.send_catch_log(signal=response_downloaded,
+                                                     request=request,
+                                                     response=None,
+                                                     spider=self.spider)
+
+        unique_slots = len(set(s for _, s in _SLOTS))
+        for i in range(0, len(_SLOTS), unique_slots):
+            part = slots[i:i + unique_slots]
+            self.assertEqual(len(part), len(set(part)))
+
+
+class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase):
+    priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
+    def test_logic(self):
+        for url, slot in _SLOTS:
+            request = Request(url)
+            _scheduler_slot_write(request, slot)
+            self.scheduler.enqueue_request(request)
+
+        self.close_scheduler()
+        self.create_scheduler()
+
+        slots = list()
+        requests = list()
+        while self.scheduler.has_pending_requests():
+            request = self.scheduler.next_request()
+            slots.append(_scheduler_slot_read(request))
+            self.mock_crawler.signals.send_catch_log(
+                    signal=request_reached_downloader,
+                    request=request,
+                    spider=self.spider
+                    )
+            requests.append(request)
+
+        self.assertEqual(self.scheduler.mqs._slots, {})
+        self.assertEqual(len(slots), len(_SLOTS))
+
+        for request in requests:
+            self.mock_crawler.signals.send_catch_log(signal=response_downloaded,
+                                                     request=request,
+                                                     response=None,
+                                                     spider=self.spider)
+
+        unique_slots = len(set(s for _, s in _SLOTS))
+        for i in range(0, len(_SLOTS), unique_slots):
+            part = slots[i:i + unique_slots]
+            self.assertEqual(len(part), len(set(part)))

From afdb69ea6daac8bd4f580d6c20bf9e93b741957b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 3 Dec 2018 16:36:05 +0100
Subject: [PATCH 1957/2974] Add a troubleshooting section to the installation
 instructions

Its initial content covers the workaround for #2473.
---
 docs/intro/install.rst | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 4a9aa3cfb..daec7fcb7 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -30,7 +30,8 @@ dependencies depending on your operating system, so be sure to check the
 We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
 to avoid conflicting with your system packages.
 
-For more detailed and platform specifics instructions, read on.
+For more detailed and platform specifics instructions, as well as
+troubleshooting information, read on.
 
 
 Things that are good to know
@@ -247,6 +248,34 @@ that setuptools was unable to pick up one PyPy-specific dependency.
 To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``.
 
 
+.. _intro-install-troubleshooting:
+
+Troubleshooting
+===============
+
+AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1'
+----------------------------------------------------------------
+
+After you install or upgrade Scrapy, Twisted or pyOpenSSL, you may get an
+exception with the following traceback::
+
+    […]
+      File "[…]/site-packages/twisted/protocols/tls.py", line 63, in <module>
+        from twisted.internet._sslverify import _setAcceptableProtocols
+      File "[…]/site-packages/twisted/internet/_sslverify.py", line 38, in <module>
+        TLSVersion.TLSv1_1: SSL.OP_NO_TLSv1_1,
+    AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1'
+
+The reason you get this exception is that your system or virtual environment
+has a version of pyOpenSSL that your version of Twisted does not support.
+
+To install a version of pyOpenSSL that your version of Twisted supports,
+reinstall Twisted with the :code:`tls` extra option::
+
+    pip install twisted[tls]
+
+For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
+
 .. _Python: https://www.python.org/
 .. _pip: https://pip.pypa.io/en/latest/installing/
 .. _lxml: http://lxml.de/

From 9c314800e4b195df41e5c0aba0d9ffe4bcffec8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 3 Dec 2018 17:14:10 +0100
Subject: [PATCH 1958/2974] Document the SCRAPY_PROJECT environment variable

Fixes #1109
---
 docs/topics/commands.rst | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index ef9c45196..97f8311de 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -37,7 +37,7 @@ Scrapy also understands, and can be configured through, a number of environment
 variables. Currently these are:
 
 * ``SCRAPY_SETTINGS_MODULE`` (see :ref:`topics-settings-module-envvar`)
-* ``SCRAPY_PROJECT``
+* ``SCRAPY_PROJECT`` (see :ref:`topics-project-envvar`)
 * ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`)
 
 .. _topics-project-structure:
@@ -71,6 +71,33 @@ the project settings. Here is an example::
     [settings]
     default = myproject.settings
 
+.. _topics-project-envvar:
+
+Sharing the root directory between projects
+===========================================
+
+A project root directory, the one that contains the ``scrapy.cfg``, may be
+shared by multiple Scrapy projects, each with its own settings module.
+
+In that case, you must define one or more aliases for those settings modules
+under ``[settings]`` in your ``scrapy.cfg`` file::
+
+    [settings]
+    default = myproject1.settings
+    project1 = myproject1.settings
+    project2 = myproject2.settings
+
+By default, the ``scrapy`` command-line tool will use the ``default`` settings.
+Use the ``SCRAPY_PROJECT`` environment variable to specify a different project
+for ``scrapy`` to use::
+
+    $ scrapy settings --get BOT_NAME
+    Project 1 Bot
+    $ export SCRAPY_PROJECT=project2
+    $ scrapy settings --get BOT_NAME
+    Project 2 Bot
+
+
 Using the ``scrapy`` tool
 =========================
 

From f56079f6c71a77c1f70510cf291cd808617933cd Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Wed, 5 Dec 2018 10:02:42 +0000
Subject: [PATCH 1959/2974] Test cleanups PEP8 fixes no need to close
 implicitly do not use pytest need to put it into class remove round-robin
 queue additional check for empty queue use pytest tmpdir fixture

---
 scrapy/pqueues.py       |  50 ++++------------
 tests/test_scheduler.py | 128 ++++++++++++----------------------------
 2 files changed, 50 insertions(+), 128 deletions(-)

diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 75073b7a4..287a8de35 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -1,4 +1,3 @@
-from collections import deque
 import hashlib
 import logging
 from six import text_type
@@ -71,16 +70,17 @@ class PrioritySlot:
         self.slot = slot
 
     def __hash__(self):
-       return hash((self.priority, self.slot))
+        return hash((self.priority, self.slot))
 
     def __eq__(self, other):
-       return (self.priority, self.slot) == (other.priority, other.slot)
+        return (self.priority, self.slot) == (other.priority, other.slot)
 
     def __lt__(self, other):
-       return (self.priority, self.slot) < (other.priority, other.slot)
+        return (self.priority, self.slot) < (other.priority, other.slot)
 
     def __str__(self):
-       return '_'.join([text_type(self.priority), _pathable(text_type(self.slot))])
+        return '_'.join([text_type(self.priority),
+                         _pathable(text_type(self.slot))])
 
 
 class PriorityAsTupleQueue(PriorityQueue):
@@ -135,9 +135,10 @@ class SlotBasedPriorityQueue(object):
         slot = _scheduler_slot(request)
         is_new = False
         if slot not in self.pqueues:
-            is_new = True
             self.pqueues[slot] = PriorityAsTupleQueue(self.qfactory)
-        self.pqueues[slot].push(request, PrioritySlot(priority=priority, slot=slot))
+        queue = self.pqueues[slot]
+        is_new = queue.is_empty()
+        queue.push(request, PrioritySlot(priority=priority, slot=slot))
         return slot, is_new
 
     def close(self):
@@ -152,36 +153,6 @@ class SlotBasedPriorityQueue(object):
         return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
 
 
-class RoundRobinPriorityQueue(SlotBasedPriorityQueue):
-
-    def __init__(self, qfactory, startprios={}):
-        super(RoundRobinPriorityQueue, self).__init__(qfactory, startprios)
-        self._slots = deque()
-        for slot in self.pqueues:
-            self._slots.append(slot)
-
-    def push(self, request, priority):
-        slot, is_new = self.push_slot(request, priority)
-        if is_new:
-            self._slots.append(slot)
-
-    def pop(self):
-        if not self._slots:
-            return
-
-        slot = self._slots.popleft()
-        request, is_empty = self.pop_slot(slot)
-
-        if not is_empty:
-            self._slots.append(slot)
-
-        return request
-
-    def close(self):
-        self._slots.clear()
-        return super(RoundRobinPriorityQueue, self).close()
-
-
 class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue):
 
     _DOWNLOADER_AWARE_PQ_ID = 'DOWNLOADER_AWARE_PQ_ID'
@@ -191,7 +162,8 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue):
         return cls(crawler, qfactory, startprios)
 
     def __init__(self, crawler, qfactory, startprios={}):
-        super(DownloaderAwarePriorityQueue, self).__init__(qfactory, startprios)
+        super(DownloaderAwarePriorityQueue, self).__init__(qfactory,
+                                                           startprios)
         self._slots = {slot: 0 for slot in self.pqueues}
         crawler.signals.connect(self.on_response_download,
                                 signal=response_downloaded)
@@ -208,7 +180,7 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue):
         return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self)
 
     def pop(self):
-        slots = [(d, s) for s,d in self._slots.items() if s in self.pqueues]
+        slots = [(d, s) for s, d in self._slots.items() if s in self.pqueues]
 
         if not slots:
             return
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index fd86e8d8c..e1cf5842d 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -1,4 +1,3 @@
-import contextlib
 import shutil
 import tempfile
 import unittest
@@ -10,15 +9,18 @@ from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write
 from scrapy.signals import request_reached_downloader, response_downloaded
 from scrapy.spiders import Spider
 
+
 class MockCrawler(Crawler):
     def __init__(self, priority_queue_cls, jobdir):
 
-        settings = dict(LOG_UNSERIALIZABLE_REQUESTS=False,
-                       SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
-                       SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
-                       SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
-                       JOBDIR=jobdir,
-                       DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter')
+        settings = dict(
+                LOG_UNSERIALIZABLE_REQUESTS=False,
+                SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
+                SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
+                SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
+                JOBDIR=jobdir,
+                DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter'
+                )
         super(MockCrawler, self).__init__(Spider, settings)
 
 
@@ -82,7 +84,8 @@ class BaseSchedulerInMemoryTester(SchedulerHandler):
         while self.scheduler.has_pending_requests():
             priorities.append(self.scheduler.next_request().priority)
 
-        self.assertEqual(priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x))
+        self.assertEqual(priorities,
+                         sorted([x[1] for x in _PRIORITIES], key=lambda x: -x))
 
 
 class BaseSchedulerOnDiskTester(SchedulerHandler):
@@ -134,7 +137,8 @@ class BaseSchedulerOnDiskTester(SchedulerHandler):
         while self.scheduler.has_pending_requests():
             priorities.append(self.scheduler.next_request().priority)
 
-        self.assertEqual(priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x))
+        self.assertEqual(priorities,
+                         sorted([x[1] for x in _PRIORITIES], key=lambda x: -x))
 
 
 class TestSchedulerInMemory(BaseSchedulerInMemoryTester, unittest.TestCase):
@@ -153,75 +157,15 @@ _SLOTS = [("http://foo.com/a", 'a'),
           ("http://foo.com/f", 'c')]
 
 
-class TestSchedulerWithRoundRobinInMemory(BaseSchedulerInMemoryTester, unittest.TestCase):
-    priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue'
+class TestMigration(unittest.TestCase):
 
-    def test_round_robin(self):
-        for url, slot in _SLOTS:
-            request = Request(url)
-            _scheduler_slot_write(request, slot)
-            self.scheduler.enqueue_request(request)
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp()
 
-        slots = list()
-        while self.scheduler.has_pending_requests():
-            slots.append(_scheduler_slot_read(self.scheduler.next_request()))
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir)
 
-        for i in range(0, len(_SLOTS), 2):
-            self.assertNotEqual(slots[i], slots[i+1])
-
-    def test_is_meta_set(self):
-        url = "http://foo.com/a"
-        request = Request(url)
-        if _scheduler_slot_read(request):
-            _scheduler_slot_write(request, None)
-        self.scheduler.enqueue_request(request)
-        self.assertIsNotNone(_scheduler_slot_read(request, None), None)
-
-
-class TestSchedulerWithRoundRobinOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase):
-    priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue'
-
-    def test_round_robin(self):
-        for url, slot in _SLOTS:
-            request = Request(url)
-            _scheduler_slot_write(request, slot)
-            self.scheduler.enqueue_request(request)
-
-        self.close_scheduler()
-        self.create_scheduler()
-
-        slots = list()
-        while self.scheduler.has_pending_requests():
-            slots.append(_scheduler_slot_read(self.scheduler.next_request()))
-
-        for i in range(0, len(_SLOTS), 2):
-            self.assertNotEqual(slots[i], slots[i+1])
-
-    def test_is_meta_set(self):
-        url = "http://foo.com/a"
-        request = Request(url)
-        if _scheduler_slot_read(request):
-            _scheduler_slot_write(request, None)
-        self.scheduler.enqueue_request(request)
-
-        self.close_scheduler()
-        self.create_scheduler()
-
-        self.assertIsNotNone(_scheduler_slot_read(request, None), None)
-
-
-@contextlib.contextmanager
-def mkdtemp():
-    dir = tempfile.mkdtemp()
-    try:
-        yield dir
-    finally:
-        shutil.rmtree(dir)
-
-
-def _migration():
-
-    with mkdtemp() as tmp_dir:
+    def _migration(self, tmp_dir):
         prev_scheduler_handler = SchedulerHandler()
         prev_scheduler_handler.priority_queue_cls = 'queuelib.PriorityQueue'
         prev_scheduler_handler.jobdir = tmp_dir
@@ -232,18 +176,18 @@ def _migration():
         prev_scheduler_handler.close_scheduler()
 
         next_scheduler_handler = SchedulerHandler()
-        next_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue'
+        next_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
         next_scheduler_handler.jobdir = tmp_dir
 
         next_scheduler_handler.create_scheduler()
 
-
-class TestMigration(unittest.TestCase):
     def test_migration(self):
-        self.assertRaises(ValueError, _migration)
+        with self.assertRaises(ValueError):
+            self._migration(self.tmpdir)
 
 
-class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, unittest.TestCase):
+class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester,
+                                               unittest.TestCase):
     priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
 
     def test_logic(self):
@@ -266,10 +210,12 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, unit
         self.assertEqual(len(slots), len(_SLOTS))
 
         for request in requests:
-            self.mock_crawler.signals.send_catch_log(signal=response_downloaded,
-                                                     request=request,
-                                                     response=None,
-                                                     spider=self.spider)
+            self.mock_crawler.signals.send_catch_log(
+                    signal=response_downloaded,
+                    request=request,
+                    response=None,
+                    spider=self.spider
+                    )
 
         unique_slots = len(set(s for _, s in _SLOTS))
         for i in range(0, len(_SLOTS), unique_slots):
@@ -277,8 +223,10 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, unit
             self.assertEqual(len(part), len(set(part)))
 
 
-class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase):
+class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
+                                             unittest.TestCase):
     priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
+
     def test_logic(self):
         for url, slot in _SLOTS:
             request = Request(url)
@@ -304,10 +252,12 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, unittest
         self.assertEqual(len(slots), len(_SLOTS))
 
         for request in requests:
-            self.mock_crawler.signals.send_catch_log(signal=response_downloaded,
-                                                     request=request,
-                                                     response=None,
-                                                     spider=self.spider)
+            self.mock_crawler.signals.send_catch_log(
+                    signal=response_downloaded,
+                    request=request,
+                    response=None,
+                    spider=self.spider
+                    )
 
         unique_slots = len(set(s for _, s in _SLOTS))
         for i in range(0, len(_SLOTS), unique_slots):

From 7efba101946af93397ec3c2323b920644e20ce04 Mon Sep 17 00:00:00 2001
From: Lucy Wang <wxitb2017@gmail.com>
Date: Mon, 10 Dec 2018 14:44:15 +0800
Subject: [PATCH 1960/2974] remove "sudo: false" now that travis no longer
 supports it

https://changelog.travis-ci.com/deprecation-container-based-linux-build-environment-82037
---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 4218d13bf..08b0bf119 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,4 @@
 language: python
-sudo: false
 branches:
   only:
     - master

From 0e06b9a81672ec432d2fccc3cbacc823ea47b656 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 14 Dec 2018 14:35:18 +0000
Subject: [PATCH 1961/2974] use urlparse_cached where it is possible

---
 scrapy/pqueues.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 287a8de35..ff7ec8c8a 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -8,6 +8,7 @@ from queuelib import PriorityQueue
 from scrapy.core.downloader import Downloader
 from scrapy.http import Request
 from scrapy.signals import request_reached_downloader, response_downloaded
+from scrapy.utils.httpobj import urlparse_cached
 
 
 logger = logging.getLogger(__name__)
@@ -41,11 +42,26 @@ def _scheduler_slot_write(request, slot):
 
 def _scheduler_slot(request):
 
-    slot = _scheduler_slot_read(request, None)
-    if slot is None:
-        url = _get_from_request(request, 'url')
+    if isinstance(request, dict):
+        meta = request.get('meta', dict())
+    elif isinstance(request, Request):
+        meta = request.meta
+    else:
+        raise ValueError('Bad type of request "%s"' % (request.__class__, ))
+
+    slot = meta.get(SCHEDULER_SLOT_META_KEY, None)
+
+    if slot is not None:
+        return slot
+
+    if isinstance(request, dict):
+        url = request.get('url', None)
         slot = urlparse(url).hostname or ''
-        _scheduler_slot_write(request, slot)
+    elif isinstance(request, Request):
+        url = request.url
+        slot = urlparse_cached(request).hostname or ''
+
+    meta[SCHEDULER_SLOT_META_KEY] = slot
 
     return slot
 

From 484927b08caff66ea622f8553468c831154df30a Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 14 Dec 2018 14:38:28 +0000
Subject: [PATCH 1962/2974] less complex implementation

---
 scrapy/pqueues.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index ff7ec8c8a..538678345 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -28,16 +28,11 @@ def _get_from_request(request, key, default=None):
 
 
 def _scheduler_slot_read(request, default=None):
-    meta = _get_from_request(request, 'meta', dict())
-    slot = meta.get(SCHEDULER_SLOT_META_KEY, default)
-    return slot
+    return request.meta.get(SCHEDULER_SLOT_META_KEY, default)
 
 
 def _scheduler_slot_write(request, slot):
-    meta = _get_from_request(request, 'meta', None)
-    if not isinstance(meta, dict):
-        raise ValueError('No meta attribute in %s' % (request, ))
-    meta[SCHEDULER_SLOT_META_KEY] = slot
+    request.meta[SCHEDULER_SLOT_META_KEY] = slot
 
 
 def _scheduler_slot(request):

From 6af964cc0b47c570e035a3486b9f8aebd349bd84 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 14 Dec 2018 14:54:24 +0000
Subject: [PATCH 1963/2974] common indentation for comment

---
 scrapy/pqueues.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 538678345..31e90ff12 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -96,9 +96,9 @@ class PrioritySlot:
 
 class PriorityAsTupleQueue(PriorityQueue):
     """
-        Python structures is not directly (de)serialized (to)from json.
-        We need this modified queue to transform custom structure (from)to
-        json serializable structures
+    Python structures is not directly (de)serialized (to)from json.
+    We need this modified queue to transform custom structure (from)to
+    json serializable structures
     """
     def __init__(self, qfactory, startprios=()):
 

From a46613afa8acd136f4ba62df2ced2f3c87679512 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 14 Dec 2018 14:55:06 +0000
Subject: [PATCH 1964/2974] use regular comments

---
 scrapy/pqueues.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 31e90ff12..75fc198d0 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -64,10 +64,8 @@ def _scheduler_slot(request):
 def _pathable(x):
     pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in x])
 
-    """
-        as we replace some letters we can get collision for different slots
-        add we add unique part
-    """
+    # as we replace some letters we can get collision for different slots
+    # add we add unique part
     unique_slot = hashlib.md5(x.encode('utf8')).hexdigest()
 
     return '-'.join([pathable_slot, unique_slot])

From a23e1894b3a09e1daf49dd9592546b2d21bc9a72 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 14 Dec 2018 16:18:34 +0000
Subject: [PATCH 1965/2974] Fix boto problem another way to fix boto problem
 Revert "fix for travis ci based on https://github.com/boto/boto/issues/3717"

This reverts commit 150d2564ff0ea994652da7f5be333d72e0b38d93.
fix for travis ci based on https://github.com/boto/boto/issues/3717
---
 tests/requirements-py2.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txt
index 790f29d34..f5bcfda60 100644
--- a/tests/requirements-py2.txt
+++ b/tests/requirements-py2.txt
@@ -11,3 +11,4 @@ testfixtures
 # optional for shell wrapper tests
 bpython
 ipython<6.0
+google-compute-engine

From d970be64cc47c382bd615cd547e7e94c17e27b48 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Mon, 17 Dec 2018 13:52:11 +0000
Subject: [PATCH 1966/2974] Integration test integration testing only
 everything is working, not logic of PQ use method create slot attribute in
 constructor corect class for test case stop crawler in teardown method use
 class correct entity naming python 2 adaptation integration test with crawler
 and spider

---
 tests/test_scheduler.py | 46 +++++++++++++++++++++++++++++++++++++----
 1 file changed, 42 insertions(+), 4 deletions(-)

diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index e1cf5842d..9bdc82b30 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -2,12 +2,17 @@ import shutil
 import tempfile
 import unittest
 
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+
 from scrapy.crawler import Crawler
 from scrapy.core.scheduler import Scheduler
 from scrapy.http import Request
 from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write
 from scrapy.signals import request_reached_downloader, response_downloaded
 from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
+from tests.mockserver import MockServer
 
 
 class MockCrawler(Crawler):
@@ -223,6 +228,13 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester,
             self.assertEqual(len(part), len(set(part)))
 
 
+def _is_slots_unique(base_slots, result_slots):
+    unique_slots = len(set(s for _, s in base_slots))
+    for i in range(0, len(result_slots), unique_slots):
+        part = result_slots[i:i + unique_slots]
+        assert len(part) == len(set(part))
+
+
 class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
                                              unittest.TestCase):
     priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
@@ -259,7 +271,33 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
                     spider=self.spider
                     )
 
-        unique_slots = len(set(s for _, s in _SLOTS))
-        for i in range(0, len(_SLOTS), unique_slots):
-            part = slots[i:i + unique_slots]
-            self.assertEqual(len(part), len(set(part)))
+        _is_slots_unique(_SLOTS, slots)
+
+
+class StartUrlsSpider(Spider):
+
+    def __init__(self, start_urls):
+        self.start_urls = start_urls
+
+
+class TestIntegrationWithDownloaderAwareOnDisk(TestCase):
+    def setUp(self):
+        self.crawler = get_crawler(
+                    StartUrlsSpider,
+                    {'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
+                     'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'}
+                    )
+
+    @defer.inlineCallbacks
+    def tearDown(self):
+        yield self.crawler.stop()
+
+    @defer.inlineCallbacks
+    def test_integration_downloader_aware_priority_queue(self):
+        with MockServer() as mockserver:
+
+            url = mockserver.url("/status?n=200", is_secure=False)
+            slots = [url] * 6
+            yield self.crawler.crawl(slots)
+            self.assertEqual(self.crawler.stats.get_value('downloader/response_count'),
+                             len(slots))

From 7d3175ac8433f964ebbb80ebd67f9899cf059100 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 20 Dec 2018 19:23:23 -0300
Subject: [PATCH 1967/2974] Fix boto import error under Jessie testing
 environment

---
 .travis.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 08b0bf119..a201f97b1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,6 +42,11 @@ install:
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
       fi
+      if [ "$TOXENV" = "jessie" ]; then
+        # Not used directly but allows boto GCE plugins to load.
+        # https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262
+        pip install google-compute-engine
+      fi
   - pip install -U tox twine wheel codecov
 
 script: tox

From 6ff2574c277ba1eda31fb43f86f43d5b7b4bef09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 20 Dec 2018 19:39:29 -0300
Subject: [PATCH 1968/2974] Needs to be installed within tox env

---
 .travis.yml | 5 -----
 tox.ini     | 3 +++
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index a201f97b1..08b0bf119 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,11 +42,6 @@ install:
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
       fi
-      if [ "$TOXENV" = "jessie" ]; then
-        # Not used directly but allows boto GCE plugins to load.
-        # https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262
-        pip install google-compute-engine
-      fi
   - pip install -U tox twine wheel codecov
 
 script: tox
diff --git a/tox.ini b/tox.ini
index e5543fe2a..0c0f8f7b7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -51,6 +51,9 @@ deps =
     cssselect==0.9.1
     zope.interface==4.1.1
     -rtests/requirements-py2.txt
+# Not used directly but allows boto GCE plugins to load.
+# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262
+    google-compute-engine==2.8.12
 
 [testenv:trunk]
 basepython = python2.7

From 4163a7a1c7ac11c8d4db70f371c26181b90d8dfd Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 21 Dec 2018 09:10:32 +0000
Subject: [PATCH 1969/2974] no need for this

---
 tests/requirements-py2.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txt
index f5bcfda60..790f29d34 100644
--- a/tests/requirements-py2.txt
+++ b/tests/requirements-py2.txt
@@ -11,4 +11,3 @@ testfixtures
 # optional for shell wrapper tests
 bpython
 ipython<6.0
-google-compute-engine

From 987c2ae4a964e45120c245235c9b0c49dc36b71f Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Tue, 25 Dec 2018 09:13:09 +0000
Subject: [PATCH 1970/2974] test ip concurrency incompatibility with DAPQ

---
 tests/test_scheduler.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 9bdc82b30..17b706bd7 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -301,3 +301,20 @@ class TestIntegrationWithDownloaderAwareOnDisk(TestCase):
             yield self.crawler.crawl(slots)
             self.assertEqual(self.crawler.stats.get_value('downloader/response_count'),
                              len(slots))
+
+
+class TestIncompatibility(unittest.TestCase):
+
+    def _incompatible(self):
+        settings = dict(
+                SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
+                CONCURRENT_REQUESTS_PER_IP=1
+                )
+        crawler = Crawler(Spider, settings)
+        scheduler = Scheduler.from_crawler(crawler)
+        spider = Spider(name='spider')
+        scheduler.open(spider)
+
+    def test_incompatibility(self):
+        with self.assertRaises(ValueError):
+            self._incompatible()

From 8e8ce301b1a56e40f7e9c322a7b73b8dcfcefc43 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Tue, 25 Dec 2018 09:14:09 +0000
Subject: [PATCH 1971/2974] check CONCURRENT_REQUESTS_PER_IP is not set

---
 scrapy/pqueues.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 75fc198d0..d9effc9d1 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -171,6 +171,14 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue):
         return cls(crawler, qfactory, startprios)
 
     def __init__(self, crawler, qfactory, startprios={}):
+        ip_concurrency_key = 'CONCURRENT_REQUESTS_PER_IP'
+        ip_concurrency = crawler.settings.getint(ip_concurrency_key, 0)
+
+        if ip_concurrency > 0:
+            raise ValueError('"%s" does not support %s=%d' % (self.__class__,
+                                                              ip_concurrency_key,
+                                                              ip_concurrency))
+
         super(DownloaderAwarePriorityQueue, self).__init__(qfactory,
                                                            startprios)
         self._slots = {slot: 0 for slot in self.pqueues}

From 338b78d796de6c93af0f4bcb762f82f5a14b87cd Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Tue, 25 Dec 2018 09:44:20 +0000
Subject: [PATCH 1972/2974] Add documentation add section to broad-crawl topic
 reword in accord with broad-crawl topic add documentation for new priority
 queue

---
 docs/topics/broad-crawls.rst | 11 +++++++++++
 docs/topics/settings.rst     |  7 ++++++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index eb02086dc..37f7a8748 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -39,6 +39,17 @@ you need to keep in mind when using Scrapy for doing broad crawls, along with
 concrete suggestions of Scrapy settings to tune in order to achieve an
 efficient broad crawl.
 
+Use proper :setting:`SCHEDULER_PRIORITY_QUEUE`
+==============================================
+
+Default scrapy's scheduler priority queue is ``'queuelib.PriorityQueue'``.
+It works best during single domain crawl. And it does not work well with crawling
+many different domains in parallel
+
+To apply recommended priority queue use::
+
+    SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
+
 Increase concurrency
 ====================
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 47b6cf13d..7b9ff7e39 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1144,7 +1144,12 @@ SCHEDULER_PRIORITY_QUEUE
 ------------------------
 Default: ``'queuelib.PriorityQueue'``
 
-Type of priority queue used by scheduler.
+Type of priority queue used by scheduler. Another available type is
+``scrapy.pqueues.DownloaderAwarePriorityQueue``.
+``scrapy.pqueues.DownloaderAwarePriorityQueue`` is works better than
+``'queuelib.PriorityQueue'`` when you crawl many different domains in parallel.
+But ``scrapy.pqueues.DownloaderAwarePriorityQueue``
+does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`.
 
 .. setting:: SPIDER_CONTRACTS
 

From 2cb4dc32052c306568206f0997de4b4e53069efd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?=
 <julio.batista@outlook.com>
Date: Fri, 22 Mar 2019 09:50:11 -0300
Subject: [PATCH 1973/2974] Mentioning to use JSON API for ACLs

---
 docs/topics/settings.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 90ae8fd93..fcdf31cac 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -755,7 +755,7 @@ FEED_STORAGE_GCS_ACL
 --------------------
 
 The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
-For more information on how to set this value, please refer to `Google Cloud documentation <https://cloud.google.com/storage/docs/access-control/lists>`_.
+For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation <https://cloud.google.com/storage/docs/access-control/lists>`_.
 
 .. setting:: FTP_PASSIVE_MODE
 
@@ -1388,4 +1388,4 @@ case to see how to enable and use them.
 .. _Amazon web services: https://aws.amazon.com/
 .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
 .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search
-.. _Google Cloud Storage: https://cloud.google.com/storage/
\ No newline at end of file
+.. _Google Cloud Storage: https://cloud.google.com/storage/

From 7c148fce5acc100f5f01719db374578bfca2512a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 8 Mar 2019 15:40:16 +0100
Subject: [PATCH 1974/2974] Implement Item.deepcopy()

---
 docs/topics/items.rst | 44 +++++++++++++++++++++++++++++++++----------
 scrapy/item.py        |  8 ++++++++
 tests/test_item.py    |  8 ++++++++
 3 files changed, 50 insertions(+), 10 deletions(-)

diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index ae44aecd3..d744fd9ea 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -40,6 +40,7 @@ objects. Here is an example::
         name = scrapy.Field()
         price = scrapy.Field()
         stock = scrapy.Field()
+        tags = scrapy.Field()
         last_updated = scrapy.Field(serializer=str)
 
 .. note:: Those familiar with `Django`_ will notice that Scrapy Items are
@@ -155,19 +156,42 @@ To access all populated values, just use the typical `dict API`_::
     >>> product.items()
     [('price', 1000), ('name', 'Desktop PC')]
 
+
+Copying items
+-------------
+
+To copy an item, you must first decide whether you want a shallow copy or a
+deep copy.
+
+If your item contains mutable_ values like lists or dictionaries, a shallow
+copy will keep references to the same mutable values across all different
+copies.
+
+.. _mutable: https://docs.python.org/glossary.html#term-mutable
+
+For example, if you have an item with a list of tags, and you create a shallow
+copy of that item, both the original item and the copy have the same list of
+tags. Adding a tag to the list of one of the items will add the tag to the
+other item as well.
+
+If that is not the desired behavior, use a deep copy instead.
+
+See the `documentation of the copy module`_ for more information.
+
+.. _documentation of the copy module: https://docs.python.org/library/copy.html
+
+To create a shallow copy of an item, you can either call
+:meth:`~scrapy.item.Item.copy` on an existing item
+(``product2 = product.copy()``) or instantiate your item class from an existing
+item (``product2 = Product(product)``).
+
+To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
+(``product2 = product.deepcopy()``).
+
+
 Other common tasks
 ------------------
 
-Copying items::
-
-    >>> product2 = Product(product)
-    >>> print(product2)
-    Product(name='Desktop PC', price=1000)
-
-    >>> product3 = product2.copy()
-    >>> print(product3)
-    Product(name='Desktop PC', price=1000)
-
 Creating dicts from items::
 
     >>> dict(product) # create a dict from all populated values
diff --git a/scrapy/item.py b/scrapy/item.py
index aa05e9c69..031b80a2d 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/item.rst
 
 from pprint import pformat
 from collections import MutableMapping
+from copy import deepcopy
 
 from abc import ABCMeta
 import six
@@ -96,6 +97,13 @@ class DictItem(MutableMapping, BaseItem):
     def copy(self):
         return self.__class__(self)
 
+    def deepcopy(self):
+        """Return a `deep copy`_ of this item.
+
+        .. _deep copy: https://docs.python.org/library/copy.html#copy.deepcopy
+        """
+        return deepcopy(self)
+
 
 @six.add_metaclass(ItemMeta)
 class Item(DictItem):
diff --git a/tests/test_item.py b/tests/test_item.py
index 2c1eb0dd3..010d3b141 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -249,6 +249,14 @@ class ItemTest(unittest.TestCase):
         copied_item['name'] = copied_item['name'].upper()
         self.assertNotEqual(item['name'], copied_item['name'])
 
+    def test_deepcopy(self):
+        class TestItem(Item):
+            tags = Field()
+        item = TestItem({'tags': ['tag1']})
+        copied_item = item.deepcopy()
+        item['tags'].append('tag2')
+        assert item['tags'] != copied_item['tags']
+
 
 class ItemMetaTest(unittest.TestCase):
 

From 9a0fe8bf2dc108b1f2c50aaef211b39981e65c25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez=20Cabot?= <mhernandez@apsl.net>
Date: Wed, 20 Mar 2019 16:13:31 +0100
Subject: [PATCH 1975/2974] remove duplicated entry in gitignore

---
 .gitignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 7392ed31e..ff6e2ea65 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,7 +15,6 @@ htmlcov/
 .pytest_cache/
 .coverage.*
 .cache/
-.pytest_cache/
 
 # Windows
 Thumbs.db

From bbf24b7a1ce2e91eab57d1b8524d398822a1ddd1 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 22 Mar 2019 18:02:31 -0300
Subject: [PATCH 1976/2974] Rule.process_request: use
 scrapy.utils.python.get_func_args

---
 scrapy/spiders/crawl.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index c01f75798..f474b0a18 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -10,6 +10,7 @@ import six
 
 from scrapy.http import Request, HtmlResponse
 from scrapy.utils.spider import iterate_spider_output
+from scrapy.utils.python import get_func_args
 from scrapy.spiders import Spider
 
 
@@ -35,10 +36,8 @@ class Rule(object):
         Wrapper around the request processing function to maintain backward compatibility
         with functions that do not take a Response object as parameter.
         """
-        argcount = self.process_request.__code__.co_argcount
-        if hasattr(self.process_request, '__self__'):
-            argcount = argcount - 1
-        args = [request] if argcount == 1 else [request, response]
+        arg_count = len(get_func_args(self.process_request))
+        args = [request] if arg_count == 1 else [request, response]
         return self.process_request(*args)
 
 

From 56929e77d98391255b77ffd3350abb49da18009e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 22 Mar 2019 18:34:55 -0300
Subject: [PATCH 1977/2974] Rule.process_request: deprecate the use of
 functions taking only one argument

---
 scrapy/spiders/crawl.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index f474b0a18..f469891d0 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -6,16 +6,19 @@ See documentation in docs/topics/spiders.rst
 """
 
 import copy
+import warnings
+
 import six
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, HtmlResponse
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.python import get_func_args
 from scrapy.spiders import Spider
 
 
-def identity(x):
-    return x
+def _identity(request, response):
+    return request
 
 
 class Rule(object):
@@ -25,19 +28,21 @@ class Rule(object):
         self.callback = callback
         self.cb_kwargs = cb_kwargs or {}
         self.process_links = process_links
-        self.process_request = process_request or identity
-        if follow is None:
-            self.follow = False if callback else True
-        else:
-            self.follow = follow
+        self.process_request = process_request or _identity
+        self.follow = follow if follow is not None else not callback
 
     def _process_request(self, request, response):
         """
-        Wrapper around the request processing function to maintain backward compatibility
-        with functions that do not take a Response object as parameter.
+        Wrapper around the request processing function to maintain backward
+        compatibility with functions that do not take a Response object
         """
         arg_count = len(get_func_args(self.process_request))
-        args = [request] if arg_count == 1 else [request, response]
+        if arg_count == 1:
+            args = [request]
+            msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated'
+            warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
+        else:
+            args = [request, response]
         return self.process_request(*args)
 
 

From 174ba3cc5671cdc9e66cb29275986ff7481affc5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 22 Mar 2019 19:16:18 -0300
Subject: [PATCH 1978/2974] Rule.process_request: update docs

---
 docs/topics/spiders.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 24b6f7ec9..30e15906e 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -403,11 +403,11 @@ Crawling rules
    This is mainly used for filtering purposes.
 
    ``process_request`` is a callable (or a string, in which case a method from
-   the spider object with that name will be used) which will be called for
-   every request extracted by this rule. This callable should take a Request object
-   as first positional argument and, optionally, the Response object from which the
-   Request originated as second positional argument. It must return a request or None
-   (to filter out the request).
+   the spider object with that name will be used) which will be called for every
+   :class:`~scrapy.http.Request` extracted by this rule. This callable should
+   take said request as first argument and the :class:`~scrapy.http.Response`
+   from which the request originated as second argument. It must return a
+   ``Request`` object or ``None`` (to filter out the request).
 
 CrawlSpider example
 ~~~~~~~~~~~~~~~~~~~

From 1b4385b7e3f78694c0378455644b539d80d293a2 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 22 Mar 2019 19:46:17 -0300
Subject: [PATCH 1979/2974] Rule.process_request: move deprecation warnings and
 compiling code, update tests

---
 scrapy/spiders/crawl.py | 35 +++++++++++++++++++----------------
 tests/test_spider.py    | 38 ++++++++++++++++++++++----------------
 2 files changed, 41 insertions(+), 32 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index f469891d0..6db3a1e06 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -21,6 +21,13 @@ def _identity(request, response):
     return request
 
 
+def _get_method(method, spider):
+    if callable(method):
+        return method
+    elif isinstance(method, six.string_types):
+        return getattr(spider, method, None)
+
+
 class Rule(object):
 
     def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None):
@@ -29,20 +36,24 @@ class Rule(object):
         self.cb_kwargs = cb_kwargs or {}
         self.process_links = process_links
         self.process_request = process_request or _identity
+        self.process_request_argcount = None
         self.follow = follow if follow is not None else not callback
 
+    def _compile(self, spider):
+        self.callback = _get_method(self.callback, spider)
+        self.process_links = _get_method(self.process_links, spider)
+        self.process_request = _get_method(self.process_request, spider)
+        self.process_request_argcount = len(get_func_args(self.process_request))
+        if self.process_request_argcount == 1:
+            msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated'
+            warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
+
     def _process_request(self, request, response):
         """
         Wrapper around the request processing function to maintain backward
         compatibility with functions that do not take a Response object
         """
-        arg_count = len(get_func_args(self.process_request))
-        if arg_count == 1:
-            args = [request]
-            msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated'
-            warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
-        else:
-            args = [request, response]
+        args = [request] if self.process_request_argcount == 1 else [request, response]
         return self.process_request(*args)
 
 
@@ -98,17 +109,9 @@ class CrawlSpider(Spider):
                 yield request_or_item
 
     def _compile_rules(self):
-        def get_method(method):
-            if callable(method):
-                return method
-            elif isinstance(method, six.string_types):
-                return getattr(self, method, None)
-
         self._rules = [copy.copy(r) for r in self.rules]
         for rule in self._rules:
-            rule.callback = get_method(rule.callback)
-            rule.process_links = get_method(rule.process_links)
-            rule.process_request = get_method(rule.process_request)
+            rule._compile(self)
 
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):
diff --git a/tests/test_spider.py b/tests/test_spider.py
index c9af7a2d7..83fb68c2f 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -275,14 +275,17 @@ class CrawlSpiderTest(SpiderTest):
                 Rule(LinkExtractor(), process_request=process_request_change_domain),
             )
 
-        spider = _CrawlSpider()
-        output = list(spider._requests_to_follow(response))
-        self.assertEqual(len(output), 3)
-        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEqual([r.url for r in output],
-                         ['http://example.com/somepage/item/12.html',
-                          'http://example.com/about.html',
-                          'http://example.com/nofollow.html'])
+        with warnings.catch_warnings(record=True) as cw:
+            spider = _CrawlSpider()
+            output = list(spider._requests_to_follow(response))
+            self.assertEqual(len(output), 3)
+            self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+            self.assertEqual([r.url for r in output],
+                             ['http://example.com/somepage/item/12.html',
+                              'http://example.com/about.html',
+                              'http://example.com/nofollow.html'])
+            self.assertEqual(len(cw), 1)
+            self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
 
     def test_process_request_with_response(self):
 
@@ -324,14 +327,17 @@ class CrawlSpiderTest(SpiderTest):
             def process_request_upper(self, request):
                 return request.replace(url=request.url.upper())
 
-        spider = _CrawlSpider()
-        output = list(spider._requests_to_follow(response))
-        self.assertEqual(len(output), 3)
-        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEqual([r.url for r in output],
-                         ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
-                          'http://EXAMPLE.ORG/ABOUT.HTML',
-                          'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
+        with warnings.catch_warnings(record=True) as cw:
+            spider = _CrawlSpider()
+            output = list(spider._requests_to_follow(response))
+            self.assertEqual(len(output), 3)
+            self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+            self.assertEqual([r.url for r in output],
+                             ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
+                              'http://EXAMPLE.ORG/ABOUT.HTML',
+                              'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
+            self.assertEqual(len(cw), 1)
+            self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
 
     def test_process_request_instance_method_with_response(self):
 

From 90934959d07db881aec5933fd7b77bcd2dccfa4f Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 27 Dec 2018 17:12:24 +0500
Subject: [PATCH 1980/2974] actually apply __slots__ suggestion [wip]
 refactoring

* SlotPriorityQueues doesn't care about objects inside, it is now just
  a container for multiple priority queues
* assorted variable renames
* don't inherit DownloaderAwarePriorityQueue from SlotBasedPriorityQueue
* apply @whalebot-helmsman's suggestions for __slots__ and meta issues
more bike-shedding

* remove mutable default arguments
* more verbose variable names
remove unneeded code

* PriorityAsTupleQueue.is_empty does the same as len(self) == 0
* custom PriorityAsTupleQueue.close is not needed after a switch
  to namedtuples
* is_new and is_empty return values are unused
* "url" local variable is unused
PrioritySlot.__str__ shouldn't return unicode in Python 2

also, do some bike-shedding: _pathable -> _path_safe
use namedtuple for PrioritySlot
cleanup: _get_from_request does the same here

Request.meta is always a dict
---
 scrapy/pqueues.py       | 180 ++++++++++++++++++----------------------
 tests/test_scheduler.py |   2 +-
 2 files changed, 82 insertions(+), 100 deletions(-)

diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index d9effc9d1..3ef896b99 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -1,6 +1,6 @@
 import hashlib
 import logging
-from six import text_type
+from collections import namedtuple
 from six.moves.urllib.parse import urlparse
 
 from queuelib import PriorityQueue
@@ -17,12 +17,12 @@ logger = logging.getLogger(__name__)
 SCHEDULER_SLOT_META_KEY = Downloader.DOWNLOAD_SLOT
 
 
-def _get_from_request(request, key, default=None):
+def _get_request_meta(request):
     if isinstance(request, dict):
-        return request.get(key, default)
+        return request.setdefault('meta', {})
 
     if isinstance(request, Request):
-        return getattr(request, key, default)
+        return request.meta
 
     raise ValueError('Bad type of request "%s"' % (request.__class__, ))
 
@@ -35,15 +35,8 @@ def _scheduler_slot_write(request, slot):
     request.meta[SCHEDULER_SLOT_META_KEY] = slot
 
 
-def _scheduler_slot(request):
-
-    if isinstance(request, dict):
-        meta = request.get('meta', dict())
-    elif isinstance(request, Request):
-        meta = request.meta
-    else:
-        raise ValueError('Bad type of request "%s"' % (request.__class__, ))
-
+def _set_scheduler_slot(request):
+    meta = _get_request_meta(request)
     slot = meta.get(SCHEDULER_SLOT_META_KEY, None)
 
     if slot is not None:
@@ -53,43 +46,29 @@ def _scheduler_slot(request):
         url = request.get('url', None)
         slot = urlparse(url).hostname or ''
     elif isinstance(request, Request):
-        url = request.url
         slot = urlparse_cached(request).hostname or ''
 
     meta[SCHEDULER_SLOT_META_KEY] = slot
-
     return slot
 
 
-def _pathable(x):
-    pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in x])
-
+def _path_safe(text):
+    """ Return a filesystem-safe version of a string ``text`` """
+    pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_'
+                             for c in text])
     # as we replace some letters we can get collision for different slots
     # add we add unique part
-    unique_slot = hashlib.md5(x.encode('utf8')).hexdigest()
-
+    unique_slot = hashlib.md5(text.encode('utf8')).hexdigest()
     return '-'.join([pathable_slot, unique_slot])
 
 
-class PrioritySlot:
-    __slots__ = ('priority', 'slot')
-
-    def __init__(self, priority=0, slot=None):
-        self.priority = priority
-        self.slot = slot
-
-    def __hash__(self):
-        return hash((self.priority, self.slot))
-
-    def __eq__(self, other):
-        return (self.priority, self.slot) == (other.priority, other.slot)
-
-    def __lt__(self, other):
-        return (self.priority, self.slot) < (other.priority, other.slot)
+class PrioritySlot(namedtuple("PrioritySlot", ["priority", "slot"])):
+    """ ``(priority, slot)`` tuple which uses a path-safe slot name
+    when converting to str """
+    __slots__ = ()
 
     def __str__(self):
-        return '_'.join([text_type(self.priority),
-                         _pathable(text_type(self.slot))])
+        return '%s_%s' % (self.priority, _path_safe(str(self.slot)))
 
 
 class PriorityAsTupleQueue(PriorityQueue):
@@ -99,78 +78,65 @@ class PriorityAsTupleQueue(PriorityQueue):
     json serializable structures
     """
     def __init__(self, qfactory, startprios=()):
-
+        startprios = [PrioritySlot(priority=p[0], slot=p[1])
+                      for p in startprios]
         super(PriorityAsTupleQueue, self).__init__(
-                qfactory,
-                [PrioritySlot(priority=p[0], slot=p[1]) for p in startprios]
-                )
-
-    def close(self):
-        startprios = super(PriorityAsTupleQueue, self).close()
-        return [(s.priority, s.slot) for s in startprios]
-
-    def is_empty(self):
-        return not self.queues or len(self) == 0
+            qfactory=qfactory,
+            startprios=startprios)
 
 
-class SlotBasedPriorityQueue(object):
+class SlotPriorityQueues(object):
+    """ Container for multiple priority queues. """
+    def __init__(self, pqfactory, slot_startprios=None):
+        """
+        ``pqfactory`` is a factory for creating new PriorityQueues.
+        It must be a function which accepts a single optional ``startprios``
+        argument, with a list of priorities to create queues for.
 
-    def __init__(self, qfactory, startprios={}):
-        self.pqueues = dict()     # slot -> priority queue
-        self.qfactory = qfactory  # factory for creating new internal queues
-
-        if not startprios:
-            return
-
-        if not isinstance(startprios, dict):
-            raise ValueError("Looks like your priorities file malforfemed. "
-                             "Possible reason: You run scrapy with previous "
-                             "version. Interrupted it. Updated scrapy. And "
-                             "run again.")
-
-        for slot, prios in startprios.items():
-            self.pqueues[slot] = PriorityAsTupleQueue(self.qfactory, prios)
+        ``slot_startprios`` is a ``{slot: startprios}`` dict.
+        """
+        self.pqfactory = pqfactory
+        self.pqueues = {}  # slot -> priority queue
+        for slot, startprios in (slot_startprios or {}).items():
+            self.pqueues[slot] = self.pqfactory(startprios)
 
     def pop_slot(self, slot):
+        """ Pop an object from a priority queue for this slot """
         queue = self.pqueues[slot]
         request = queue.pop()
-        is_empty = queue.is_empty()
-        if is_empty:
+        if len(queue) == 0:
             del self.pqueues[slot]
+        return request
 
-        return request, is_empty
-
-    def push_slot(self, request, priority):
-        slot = _scheduler_slot(request)
-        is_new = False
+    def push_slot(self, slot, obj, priority):
+        """ Push an object to a priority queue for this slot """
         if slot not in self.pqueues:
-            self.pqueues[slot] = PriorityAsTupleQueue(self.qfactory)
+            self.pqueues[slot] = self.pqfactory()
         queue = self.pqueues[slot]
-        is_new = queue.is_empty()
-        queue.push(request, PrioritySlot(priority=priority, slot=slot))
-        return slot, is_new
+        queue.push(obj, priority)
 
     def close(self):
-        startprios = dict()
-        for slot, queue in self.pqueues.items():
-            prios = queue.close()
-            startprios[slot] = prios
+        active = {slot: queue.close()
+                  for slot, queue in self.pqueues.items()}
         self.pqueues.clear()
-        return startprios
+        return active
 
     def __len__(self):
         return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
 
+    def __contains__(self, slot):
+        return slot in self.pqueues
 
-class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue):
+
+class DownloaderAwarePriorityQueue(object):
 
     _DOWNLOADER_AWARE_PQ_ID = 'DOWNLOADER_AWARE_PQ_ID'
 
     @classmethod
-    def from_crawler(cls, crawler, qfactory, startprios={}):
+    def from_crawler(cls, crawler, qfactory, startprios=None):
         return cls(crawler, qfactory, startprios)
 
-    def __init__(self, crawler, qfactory, startprios={}):
+    def __init__(self, crawler, qfactory, startprios=None):
         ip_concurrency_key = 'CONCURRENT_REQUESTS_PER_IP'
         ip_concurrency = crawler.settings.getint(ip_concurrency_key, 0)
 
@@ -179,16 +145,25 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue):
                                                               ip_concurrency_key,
                                                               ip_concurrency))
 
-        super(DownloaderAwarePriorityQueue, self).__init__(qfactory,
-                                                           startprios)
-        self._slots = {slot: 0 for slot in self.pqueues}
+        def pqfactory(startprios=()):
+            return PriorityAsTupleQueue(qfactory, startprios)
+
+        if startprios and not isinstance(startprios, dict):
+            raise ValueError("DownloaderAwarePriorityQueue accepts "
+                             "``startprios`` as a dict; %r instance is passed."
+                             " Only a crawl started with the same priority "
+                             "queue class can be resumed." % startprios.__class__)
+        self._slot_pqueues = SlotPriorityQueues(pqfactory,
+                                                slot_startprios=startprios)
+
+        self._active_downloads = {slot: 0 for slot in self._slot_pqueues.pqueues}
         crawler.signals.connect(self.on_response_download,
                                 signal=response_downloaded)
         crawler.signals.connect(self.on_request_reached_downloader,
                                 signal=request_reached_downloader)
 
     def mark(self, request):
-        meta = _get_from_request(request, 'meta', None)
+        meta = _get_request_meta(request)
         if not isinstance(meta, dict):
             raise ValueError('No meta attribute in %s' % (request, ))
         meta[self._DOWNLOADER_AWARE_PQ_ID] = id(self)
@@ -197,39 +172,46 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue):
         return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self)
 
     def pop(self):
-        slots = [(d, s) for s, d in self._slots.items() if s in self.pqueues]
+        slots = [(active_downloads, slot)
+                 for slot, active_downloads in self._active_downloads.items()
+                 if slot in self._slot_pqueues]
 
         if not slots:
             return
 
         slot = min(slots)[1]
-        request, _ = self.pop_slot(slot)
+        request = self._slot_pqueues.pop_slot(slot)
         self.mark(request)
         return request
 
     def push(self, request, priority):
-        slot, _ = self.push_slot(request, priority)
-        if slot not in self._slots:
-            self._slots[slot] = 0
+        slot = _set_scheduler_slot(request)
+        priority_slot = PrioritySlot(priority=priority, slot=slot)
+        self._slot_pqueues.push_slot(slot, request, priority_slot)
+        if slot not in self._active_downloads:
+            self._active_downloads[slot] = 0
 
     def on_response_download(self, response, request, spider):
         if not self.check_mark(request):
             return
 
         slot = _scheduler_slot_read(request)
-        if slot not in self._slots or self._slots[slot] <= 0:
+        if slot not in self._active_downloads or self._active_downloads[slot] <= 0:
             raise ValueError('Get response for wrong slot "%s"' % (slot, ))
-        self._slots[slot] = self._slots[slot] - 1
-        if self._slots[slot] == 0 and slot not in self.pqueues:
-            del self._slots[slot]
+        self._active_downloads[slot] = self._active_downloads[slot] - 1
+        if self._active_downloads[slot] == 0 and slot not in self._slot_pqueues:
+            del self._active_downloads[slot]
 
     def on_request_reached_downloader(self, request, spider):
         if not self.check_mark(request):
             return
 
         slot = _scheduler_slot_read(request)
-        self._slots[slot] = self._slots.get(slot, 0) + 1
+        self._active_downloads[slot] = self._active_downloads.get(slot, 0) + 1
 
     def close(self):
-        self._slots.clear()
-        return super(DownloaderAwarePriorityQueue, self).close()
+        self._active_downloads.clear()
+        return self._slot_pqueues.close()
+
+    def __len__(self):
+        return len(self._slot_pqueues)
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 17b706bd7..5dd35f45c 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -260,7 +260,7 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
                     )
             requests.append(request)
 
-        self.assertEqual(self.scheduler.mqs._slots, {})
+        self.assertEqual(self.scheduler.mqs._active_downloads, {})
         self.assertEqual(len(slots), len(_SLOTS))
 
         for request in requests:

From 757f53a32461ef0c3d2fe4caf64197f67271b5f3 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Wed, 9 Jan 2019 10:00:13 +0000
Subject: [PATCH 1981/2974] Address Lucy's comments add tests to check
 correctness of slot setermination unmark requests after downloading shorter
 better exception message

---
 scrapy/pqueues.py       | 15 ++++++++++++---
 tests/test_scheduler.py |  4 ++--
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 3ef896b99..d8eed010f 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -36,6 +36,12 @@ def _scheduler_slot_write(request, slot):
 
 
 def _set_scheduler_slot(request):
+    """
+        >>> _set_scheduler_slot({'url':'http://foo.com'}) == _set_scheduler_slot({'url':'http://bar.com'})
+        False
+        >>> _set_scheduler_slot({'url':'http://foo.com'}) == _set_scheduler_slot({'url':'http://foo.com'})
+        True
+    """
     meta = _get_request_meta(request)
     slot = meta.get(SCHEDULER_SLOT_META_KEY, None)
 
@@ -141,9 +147,8 @@ class DownloaderAwarePriorityQueue(object):
         ip_concurrency = crawler.settings.getint(ip_concurrency_key, 0)
 
         if ip_concurrency > 0:
-            raise ValueError('"%s" does not support %s=%d' % (self.__class__,
-                                                              ip_concurrency_key,
-                                                              ip_concurrency))
+            raise ValueError('"%s" does not support setting %s' % (self.__class__,
+                                                                   ip_concurrency_key))
 
         def pqfactory(startprios=()):
             return PriorityAsTupleQueue(qfactory, startprios)
@@ -171,6 +176,9 @@ class DownloaderAwarePriorityQueue(object):
     def check_mark(self, request):
         return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self)
 
+    def unmark(self, request):
+        del request.meta[self._DOWNLOADER_AWARE_PQ_ID]
+
     def pop(self):
         slots = [(active_downloads, slot)
                  for slot, active_downloads in self._active_downloads.items()
@@ -194,6 +202,7 @@ class DownloaderAwarePriorityQueue(object):
     def on_response_download(self, response, request, spider):
         if not self.check_mark(request):
             return
+        self.unmark(request)
 
         slot = _scheduler_slot_read(request)
         if slot not in self._active_downloads or self._active_downloads[slot] <= 0:
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 5dd35f45c..3fb70a110 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -248,8 +248,8 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
         self.close_scheduler()
         self.create_scheduler()
 
-        slots = list()
-        requests = list()
+        slots = []
+        requests = []
         while self.scheduler.has_pending_requests():
             request = self.scheduler.next_request()
             slots.append(_scheduler_slot_read(request))

From 3b1db71dac8716878ff1b94ee0d1095e5c80795f Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Wed, 9 Jan 2019 12:14:40 +0000
Subject: [PATCH 1982/2974] New signal update signature documentation for new
 signal utilize new signal correct signal handler signature emit new signal
 test another signal new signal rename test file faster test rename test case
 tests for signal emitting in bad cases

---
 docs/topics/signals.rst            | 17 +++++++++
 scrapy/core/downloader/__init__.py |  3 ++
 scrapy/pqueues.py                  |  6 +--
 scrapy/signals.py                  |  1 +
 tests/test_request_left.py         | 59 ++++++++++++++++++++++++++++++
 tests/test_scheduler.py            |  8 ++--
 6 files changed, 86 insertions(+), 8 deletions(-)
 create mode 100644 tests/test_request_left.py

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index ff07b9d55..f13e8270c 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -295,6 +295,23 @@ request_reached_downloader
     :param spider: the spider that yielded the request
     :type spider: :class:`~scrapy.spiders.Spider` object
 
+request_left_downloader
+---------------------------
+
+.. signal:: request_left_downloader
+.. function:: request_left_downloader(request, spider)
+
+    Sent when a :class:`~scrapy.http.Request` left downloader even in case of
+    failure.
+
+    The signal does not support returning deferreds from their handlers.
+
+    :param request: the request that reached downloader
+    :type request: :class:`~scrapy.http.Request` object
+
+    :param spider: the spider that yielded the request
+    :type spider: :class:`~scrapy.spiders.Spider` object
+
 response_received
 -----------------
 
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 4695d75f4..d856a2f37 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -188,6 +188,9 @@ class Downloader(object):
         def finish_transferring(_):
             slot.transferring.remove(request)
             self._process_queue(spider, slot)
+            self.signals.send_catch_log(signal=signals.request_left_downloader,
+                                        request=request,
+                                        spider=spider)
             return _
 
         return dfd.addBoth(finish_transferring)
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index d8eed010f..6a9feb599 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -7,7 +7,7 @@ from queuelib import PriorityQueue
 
 from scrapy.core.downloader import Downloader
 from scrapy.http import Request
-from scrapy.signals import request_reached_downloader, response_downloaded
+from scrapy.signals import request_reached_downloader, request_left_downloader
 from scrapy.utils.httpobj import urlparse_cached
 
 
@@ -163,7 +163,7 @@ class DownloaderAwarePriorityQueue(object):
 
         self._active_downloads = {slot: 0 for slot in self._slot_pqueues.pqueues}
         crawler.signals.connect(self.on_response_download,
-                                signal=response_downloaded)
+                                signal=request_left_downloader)
         crawler.signals.connect(self.on_request_reached_downloader,
                                 signal=request_reached_downloader)
 
@@ -199,7 +199,7 @@ class DownloaderAwarePriorityQueue(object):
         if slot not in self._active_downloads:
             self._active_downloads[slot] = 0
 
-    def on_response_download(self, response, request, spider):
+    def on_response_download(self, request, spider):
         if not self.check_mark(request):
             return
         self.unmark(request)
diff --git a/scrapy/signals.py b/scrapy/signals.py
index c0e4bb74e..2ea986b8c 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -14,6 +14,7 @@ spider_error = object()
 request_scheduled = object()
 request_dropped = object()
 request_reached_downloader = object()
+request_left_downloader = object()
 response_received = object()
 response_downloaded = object()
 item_scraped = object()
diff --git a/tests/test_request_left.py b/tests/test_request_left.py
new file mode 100644
index 000000000..ddeca0499
--- /dev/null
+++ b/tests/test_request_left.py
@@ -0,0 +1,59 @@
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+from scrapy.signals import request_left_downloader
+from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
+from tests.mockserver import MockServer
+
+class SignalCatcherSpider(Spider):
+    name = 'signal_catcher'
+
+    def __init__(self, crawler, url, *args, **kwargs):
+        super(SignalCatcherSpider, self).__init__(*args, **kwargs)
+        crawler.signals.connect(self.on_response_download,
+                                signal=request_left_downloader)
+        self.catched_times = 0
+        self.start_urls = [url]
+
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        spider = cls(crawler, *args, **kwargs)
+        return spider
+
+    def on_response_download(self, request, spider):
+        self.catched_times = self.catched_times + 1
+
+
+class TestCatching(TestCase):
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_success(self):
+        crawler = get_crawler(SignalCatcherSpider)
+        yield crawler.crawl(self.mockserver.url("/status?n=200"))
+        self.assertEqual(crawler.spider.catched_times, 1)
+
+    @defer.inlineCallbacks
+    def test_timeout(self):
+        crawler = get_crawler(SignalCatcherSpider,
+                              {'DOWNLOAD_TIMEOUT': 0.1})
+        yield crawler.crawl(self.mockserver.url("/delay?n=0.2"))
+        self.assertEqual(crawler.spider.catched_times, 1)
+
+    @defer.inlineCallbacks
+    def test_disconnect(self):
+        crawler = get_crawler(SignalCatcherSpider)
+        yield crawler.crawl(self.mockserver.url("/drop"))
+        self.assertEqual(crawler.spider.catched_times, 1)
+
+    @defer.inlineCallbacks
+    def test_noconnect(self):
+        crawler = get_crawler(SignalCatcherSpider)
+        yield crawler.crawl('http://thereisdefinetelynosuchdomain.com')
+        self.assertEqual(crawler.spider.catched_times, 1)
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 3fb70a110..1bcc1e5a8 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -9,7 +9,7 @@ from scrapy.crawler import Crawler
 from scrapy.core.scheduler import Scheduler
 from scrapy.http import Request
 from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write
-from scrapy.signals import request_reached_downloader, response_downloaded
+from scrapy.signals import request_reached_downloader, request_left_downloader
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
@@ -216,9 +216,8 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester,
 
         for request in requests:
             self.mock_crawler.signals.send_catch_log(
-                    signal=response_downloaded,
+                    signal=request_left_downloader,
                     request=request,
-                    response=None,
                     spider=self.spider
                     )
 
@@ -265,9 +264,8 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
 
         for request in requests:
             self.mock_crawler.signals.send_catch_log(
-                    signal=response_downloaded,
+                    signal=request_left_downloader,
                     request=request,
-                    response=None,
                     spider=self.spider
                     )
 

From 83eb5376458ce1d444e8ad7911730ee0c58c8544 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 17 Jan 2019 07:38:15 +0500
Subject: [PATCH 1983/2974] assorted cleanups: comments, docstrings, etc
 scheduler cleanup

Scheduler no longer converts requests to dicts; PriorityQueue
instances always work with Request instances; converting Requests
to dicts is now Priority Queue responsibility.
minor cleanup
---
 docs/topics/settings.rst            |   6 +-
 scrapy/core/scheduler.py            |  99 +++++++++++++----
 scrapy/pqueues.py                   | 158 +++++++++++++++-------------
 scrapy/settings/default_settings.py |   2 +-
 scrapy/squeues.py                   |  11 +-
 5 files changed, 175 insertions(+), 101 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 7b9ff7e39..6e13e64d6 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1142,13 +1142,13 @@ Type of in-memory queue used by scheduler. Other available type is:
 
 SCHEDULER_PRIORITY_QUEUE
 ------------------------
-Default: ``'queuelib.PriorityQueue'``
+Default: ``'scrapy.pqueues.ScrapyPriorityQueue'``
 
 Type of priority queue used by scheduler. Another available type is
 ``scrapy.pqueues.DownloaderAwarePriorityQueue``.
 ``scrapy.pqueues.DownloaderAwarePriorityQueue`` is works better than
-``'queuelib.PriorityQueue'`` when you crawl many different domains in parallel.
-But ``scrapy.pqueues.DownloaderAwarePriorityQueue``
+``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different
+domains in parallel. But ``scrapy.pqueues.DownloaderAwarePriorityQueue``
 does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`.
 
 .. setting:: SPIDER_CONTRACTS
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index d40f3aa0c..c385fafe1 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -1,17 +1,44 @@
 import os
 import json
 import logging
+import warnings
 from os.path import join, exists
 
-from scrapy.utils.reqser import request_to_dict, request_from_dict
+from queuelib import PriorityQueue
+
 from scrapy.utils.misc import load_object, create_instance
 from scrapy.utils.job import job_dir
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+
 
 logger = logging.getLogger(__name__)
 
 
 class Scheduler(object):
+    """
+    Scrapy Scheduler. It allows to enqueue requests and then get
+    a next request to download. Scheduler is also handling duplication
+    filtering, via dupefilter.
 
+    Prioritization and queueing is not performed by the Scheduler.
+    User sets ``priority`` field for each Request, and a PriorityQueue
+    (defined by :setting:`SCHEDULER_PRIORITY_QUEUE`) uses these priorities
+    to dequeue requests in a desired order.
+
+    Scheduler uses two PriorityQueue instances, configured to work in-memory
+    and on-disk (optional). When on-disk queue is present, it is used by
+    default, and an in-memory queue is used as a fallback for cases where
+    a disk queue can't handle a request (can't serialize it).
+
+    :setting:`SCHEDULER_MEMORY_QUEUE` and
+    :setting:`SCHEDULER_DISK_QUEUE` allow to specify lower-level queue classes
+    which PriorityQueue instances would be instantiated with, to keep requests
+    on disk and in memory respectively.
+
+    Overall, Scheduler is an object which holds several PriorityQueue instances
+    (in-memory and on-disk) and implements fallback logic for them.
+    Also, it handles dupefilters.
+    """
     def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None,
                  logunser=False, stats=None, pqclass=None, crawler=None):
         self.df = dupefilter
@@ -29,9 +56,19 @@ class Scheduler(object):
         dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
         dupefilter = create_instance(dupefilter_cls, settings, crawler)
         pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
+        if pqclass is PriorityQueue:
+            # backwards compatibility
+            warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'"
+                          " is no longer supported because of API changes; "
+                          "please use 'scrapy.pqueues.ScrapyPriorityQueue'",
+                          ScrapyDeprecationWarning)
+            from scrapy.pqueues import ScrapyPriorityQueue
+            pqclass = ScrapyPriorityQueue
+
         dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
         mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
-        logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS', settings.getbool('SCHEDULER_DEBUG'))
+        logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS',
+                                    settings.getbool('SCHEDULER_DEBUG'))
         return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
                    stats=crawler.stats, pqclass=pqclass, dqclass=dqclass,
                    mqclass=mqclass, crawler=crawler)
@@ -41,15 +78,19 @@ class Scheduler(object):
 
     def open(self, spider):
         self.spider = spider
-        self.mqs = create_instance(self.pqclass, None, self.crawler, self._newmq)
+
+        # in-memory PriorityQueue instance
+        self.mqs = self._mq()
+
+        # on-disk PriorityQueue instance
         self.dqs = self._dq() if self.dqdir else None
+
         return self.df.open()
 
     def close(self, reason):
         if self.dqs:
-            prios = self.dqs.close()
-            with open(join(self.dqdir, 'active.json'), 'w') as f:
-                json.dump(prios, f)
+            state = self.dqs.close()
+            self._write_dqs_state(self.dqdir, state)
         return self.df.close(reason)
 
     def enqueue_request(self, request):
@@ -66,7 +107,7 @@ class Scheduler(object):
         return True
 
     def next_request(self):
-        request = self.mqs.pop()
+        request = self._mqpop()
         if request:
             self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider)
         else:
@@ -84,8 +125,7 @@ class Scheduler(object):
         if self.dqs is None:
             return
         try:
-            reqd = request_to_dict(request, self.spider)
-            self.dqs.push(reqd, -request.priority)
+            self.dqs.push(request, -request.priority)
         except ValueError as e:  # non serializable request
             if self.logunser:
                 msg = ("Unable to serialize request: %(request)s - reason:"
@@ -105,37 +145,54 @@ class Scheduler(object):
 
     def _dqpop(self):
         if self.dqs:
-            d = self.dqs.pop()
-            if d:
-                return request_from_dict(d, self.spider)
+            return self.dqs.pop()
+
+    def _mqpop(self):
+        return self.mqs.pop()
 
     def _newmq(self, priority):
+        """ Factory for creating memory queues. """
         return self.mqclass()
 
     def _newdq(self, priority):
-        return self.dqclass(join(self.dqdir, 'p%s' % (priority, )))
+        """ Factory for creating disk queues. """
+        path = join(self.dqdir, 'p%s' % (priority, ))
+        return self.dqclass(path)
+
+    def _mq(self):
+        """ Create a new priority queue instance, with in-memory storage """
+        return create_instance(self.pqclass, None, self.crawler, self._newmq,
+                               serialize=False)
 
     def _dq(self):
-        activef = join(self.dqdir, 'active.json')
-        if exists(activef):
-            with open(activef) as f:
-                prios = json.load(f)
-        else:
-            prios = ()
-
+        """ Create a new priority queue instance, with disk storage """
+        state = self._read_dqs_state(self.dqdir)
         q = create_instance(self.pqclass,
                             None,
                             self.crawler,
                             self._newdq,
-                            startprios=prios)
+                            state,
+                            serialize=True)
         if q:
             logger.info("Resuming crawl (%(queuesize)d requests scheduled)",
                         {'queuesize': len(q)}, extra={'spider': self.spider})
         return q
 
     def _dqdir(self, jobdir):
+        """ Return a folder name to keep disk queue state at """
         if jobdir:
             dqdir = join(jobdir, 'requests.queue')
             if not exists(dqdir):
                 os.makedirs(dqdir)
             return dqdir
+
+    def _read_dqs_state(self, dqdir):
+        path = join(dqdir, 'active.json')
+        if not exists(path):
+            return ()
+        with open(path) as f:
+            return json.load(f)
+
+    def _write_dqs_state(self, dqdir, state):
+        with open(join(dqdir, 'active.json'), 'w') as f:
+            json.dump(state, f)
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 6a9feb599..622f6bbc5 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -1,10 +1,10 @@
 import hashlib
 import logging
 from collections import namedtuple
-from six.moves.urllib.parse import urlparse
 
 from queuelib import PriorityQueue
 
+from scrapy.utils.reqser import request_to_dict, request_from_dict
 from scrapy.core.downloader import Downloader
 from scrapy.http import Request
 from scrapy.signals import request_reached_downloader, request_left_downloader
@@ -17,16 +17,6 @@ logger = logging.getLogger(__name__)
 SCHEDULER_SLOT_META_KEY = Downloader.DOWNLOAD_SLOT
 
 
-def _get_request_meta(request):
-    if isinstance(request, dict):
-        return request.setdefault('meta', {})
-
-    if isinstance(request, Request):
-        return request.meta
-
-    raise ValueError('Bad type of request "%s"' % (request.__class__, ))
-
-
 def _scheduler_slot_read(request, default=None):
     return request.meta.get(SCHEDULER_SLOT_META_KEY, default)
 
@@ -37,24 +27,17 @@ def _scheduler_slot_write(request, slot):
 
 def _set_scheduler_slot(request):
     """
-        >>> _set_scheduler_slot({'url':'http://foo.com'}) == _set_scheduler_slot({'url':'http://bar.com'})
-        False
-        >>> _set_scheduler_slot({'url':'http://foo.com'}) == _set_scheduler_slot({'url':'http://foo.com'})
-        True
+    >>> request = Request('http://example.com')
+    >>> _set_scheduler_slot(request)
+    'example.com'
+    >>> _scheduler_slot_read(request)
+    'example.com'
     """
-    meta = _get_request_meta(request)
-    slot = meta.get(SCHEDULER_SLOT_META_KEY, None)
-
+    slot = _scheduler_slot_read(request, None)
     if slot is not None:
         return slot
-
-    if isinstance(request, dict):
-        url = request.get('url', None)
-        slot = urlparse(url).hostname or ''
-    elif isinstance(request, Request):
-        slot = urlparse_cached(request).hostname or ''
-
-    meta[SCHEDULER_SLOT_META_KEY] = slot
+    slot = urlparse_cached(request).hostname or ''
+    _scheduler_slot_write(request, slot)
     return slot
 
 
@@ -68,30 +51,25 @@ def _path_safe(text):
     return '-'.join([pathable_slot, unique_slot])
 
 
-class PrioritySlot(namedtuple("PrioritySlot", ["priority", "slot"])):
-    """ ``(priority, slot)`` tuple which uses a path-safe slot name
-    when converting to str """
+class _Priority(namedtuple("_Priority", ["priority", "slot"])):
+    """ Slot-specific priority. It is a hack - ``(priority, slot)`` tuple
+    which can be used instead of int priorities in queues:
+
+    * they are ordered in the same way - order is still by priority value,
+      min(prios) works;
+    * str(p) representation is guaranteed to be different when slots
+      are different - this is important because str(p) is used to create
+      queue files on disk;
+    * they have readable str(p) representation which is safe
+      to use as a file name.
+    """
     __slots__ = ()
 
     def __str__(self):
         return '%s_%s' % (self.priority, _path_safe(str(self.slot)))
 
 
-class PriorityAsTupleQueue(PriorityQueue):
-    """
-    Python structures is not directly (de)serialized (to)from json.
-    We need this modified queue to transform custom structure (from)to
-    json serializable structures
-    """
-    def __init__(self, qfactory, startprios=()):
-        startprios = [PrioritySlot(priority=p[0], slot=p[1])
-                      for p in startprios]
-        super(PriorityAsTupleQueue, self).__init__(
-            qfactory=qfactory,
-            startprios=startprios)
-
-
-class SlotPriorityQueues(object):
+class _SlotPriorityQueues(object):
     """ Container for multiple priority queues. """
     def __init__(self, pqfactory, slot_startprios=None):
         """
@@ -134,44 +112,78 @@ class SlotPriorityQueues(object):
         return slot in self.pqueues
 
 
-class DownloaderAwarePriorityQueue(object):
-
-    _DOWNLOADER_AWARE_PQ_ID = 'DOWNLOADER_AWARE_PQ_ID'
+class ScrapyPriorityQueue(PriorityQueue):
+    """
+    PriorityQueue which works with scrapy.Request instances and
+    can optionally convert them to/from dicts before/after putting to a queue.
+    """
+    def __init__(self, crawler, qfactory, startprios=(), serialize=False):
+        super(ScrapyPriorityQueue, self).__init__(qfactory, startprios)
+        self.serialize = serialize
+        self.spider = crawler.spider
 
     @classmethod
-    def from_crawler(cls, crawler, qfactory, startprios=None):
-        return cls(crawler, qfactory, startprios)
+    def from_crawler(cls, crawler, qfactory, startprios=(), serialize=False):
+        return cls(crawler, qfactory, startprios, serialize)
 
-    def __init__(self, crawler, qfactory, startprios=None):
-        ip_concurrency_key = 'CONCURRENT_REQUESTS_PER_IP'
-        ip_concurrency = crawler.settings.getint(ip_concurrency_key, 0)
+    def push(self, request, priority=0):
+        if self.serialize:
+            request = request_to_dict(request, self.spider)
+        super(ScrapyPriorityQueue, self).push(request, priority)
 
-        if ip_concurrency > 0:
-            raise ValueError('"%s" does not support setting %s' % (self.__class__,
-                                                                   ip_concurrency_key))
+    def pop(self):
+        request = super(ScrapyPriorityQueue, self).pop()
+        if request and self.serialize:
+            request = request_from_dict(request, self.spider)
+        return request
+
+
+class DownloaderAwarePriorityQueue(object):
+    """ PriorityQueue which takes Downlaoder activity in account:
+    domains (slots) with the least amount of active downloads are dequeued
+    first.
+    """
+    _DOWNLOADER_AWARE_PQ_ID = '_DOWNLOADER_AWARE_PQ_ID'
+
+    @classmethod
+    def from_crawler(cls, crawler, qfactory, slot_startprios=None, serialize=False):
+        return cls(crawler, qfactory, slot_startprios, serialize)
+
+    def __init__(self, crawler, qfactory, slot_startprios=None, serialize=False):
+        if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0:
+            raise ValueError('"%s" does not support CONCURRENT_REQUESTS_PER_IP'
+                             % (self.__class__,))
+
+        if slot_startprios and not isinstance(slot_startprios, dict):
+            raise ValueError("DownloaderAwarePriorityQueue accepts "
+                             "``slot_startprios`` as a dict; %r instance "
+                             "is passed. Most likely, it means the state is"
+                             "created by an incompatible priority queue. "
+                             "Only a crawl started with the same priority "
+                             "queue class can be resumed." %
+                             slot_startprios.__class__)
+
+        slot_startprios = {
+            slot: [_Priority(p, slot) for p in startprios]
+            for slot, startprios in (slot_startprios or {}).items()}
 
         def pqfactory(startprios=()):
-            return PriorityAsTupleQueue(qfactory, startprios)
-
-        if startprios and not isinstance(startprios, dict):
-            raise ValueError("DownloaderAwarePriorityQueue accepts "
-                             "``startprios`` as a dict; %r instance is passed."
-                             " Only a crawl started with the same priority "
-                             "queue class can be resumed." % startprios.__class__)
-        self._slot_pqueues = SlotPriorityQueues(pqfactory,
-                                                slot_startprios=startprios)
+            return ScrapyPriorityQueue(crawler, qfactory, startprios, serialize)
+        self._slot_pqueues = _SlotPriorityQueues(pqfactory, slot_startprios)
 
         self._active_downloads = {slot: 0 for slot in self._slot_pqueues.pqueues}
         crawler.signals.connect(self.on_response_download,
                                 signal=request_left_downloader)
         crawler.signals.connect(self.on_request_reached_downloader,
                                 signal=request_reached_downloader)
+        self.serialize = serialize
 
+    # There are two PriorityQueues at the same time (memory and disk-based),
+    # and they both listen to Downloader signals. To filter out signals
+    # coming from the other queue, each queue keeps track of its own
+    # requests using mark / unmark / check_mark methods.
     def mark(self, request):
-        meta = _get_request_meta(request)
-        if not isinstance(meta, dict):
-            raise ValueError('No meta attribute in %s' % (request, ))
-        meta[self._DOWNLOADER_AWARE_PQ_ID] = id(self)
+        request.meta[self._DOWNLOADER_AWARE_PQ_ID] = id(self)
 
     def check_mark(self, request):
         return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self)
@@ -194,7 +206,7 @@ class DownloaderAwarePriorityQueue(object):
 
     def push(self, request, priority):
         slot = _set_scheduler_slot(request)
-        priority_slot = PrioritySlot(priority=priority, slot=slot)
+        priority_slot = _Priority(priority=priority, slot=slot)
         self._slot_pqueues.push_slot(slot, request, priority_slot)
         if slot not in self._active_downloads:
             self._active_downloads[slot] = 0
@@ -206,8 +218,8 @@ class DownloaderAwarePriorityQueue(object):
 
         slot = _scheduler_slot_read(request)
         if slot not in self._active_downloads or self._active_downloads[slot] <= 0:
-            raise ValueError('Get response for wrong slot "%s"' % (slot, ))
-        self._active_downloads[slot] = self._active_downloads[slot] - 1
+            raise ValueError('Got response for a wrong slot "%s"' % (slot, ))
+        self._active_downloads[slot] -= 1
         if self._active_downloads[slot] == 0 and slot not in self._slot_pqueues:
             del self._active_downloads[slot]
 
@@ -220,7 +232,9 @@ class DownloaderAwarePriorityQueue(object):
 
     def close(self):
         self._active_downloads.clear()
-        return self._slot_pqueues.close()
+        active = self._slot_pqueues.close()
+        return {slot: [p.priority for p in startprios]
+                for slot, startprios in active.items()}
 
     def __len__(self):
         return len(self._slot_pqueues)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index ca004aedd..365b405cb 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -244,7 +244,7 @@ ROBOTSTXT_OBEY = False
 SCHEDULER = 'scrapy.core.scheduler.Scheduler'
 SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
 SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
-SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'
+SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.ScrapyPriorityQueue'
 
 SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
 SPIDER_LOADER_WARN_ONLY = False
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index d2074a457..30cc926e5 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -7,6 +7,7 @@ from six.moves import cPickle as pickle
 
 from queuelib import queue
 
+
 def _serializable_queue(queue_class, serialize, deserialize):
 
     class SerializableQueue(queue_class):
@@ -22,6 +23,7 @@ def _serializable_queue(queue_class, serialize, deserialize):
 
     return SerializableQueue
 
+
 def _pickle_serialize(obj):
     try:
         return pickle.dumps(obj, protocol=2)
@@ -31,13 +33,14 @@ def _pickle_serialize(obj):
     except (pickle.PicklingError, AttributeError, TypeError) as e:
         raise ValueError(str(e))
 
-PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \
+
+PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue,
     _pickle_serialize, pickle.loads)
-PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, \
+PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue,
     _pickle_serialize, pickle.loads)
-MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \
+MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue,
     marshal.dumps, marshal.loads)
-MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, \
+MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue,
     marshal.dumps, marshal.loads)
 FifoMemoryQueue = queue.FifoMemoryQueue
 LifoMemoryQueue = queue.LifoMemoryQueue

From 443fb98a4776f4196662bb48918f1471758b7ae7 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Tue, 5 Mar 2019 12:44:07 +0000
Subject: [PATCH 1984/2974] Use downloader directly rename variable remove old
 write function remove unused imports remove old read function remove unused
 function use mock methods mock downloader close downloader add parse method
 use new PQ class create mock downloader use downloader directly remove
 mark/unmark mechanism

---
 scrapy/pqueues.py       | 103 ++++++++++------------------------------
 tests/test_scheduler.py |  87 +++++++++++++++++++++------------
 2 files changed, 81 insertions(+), 109 deletions(-)

diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 622f6bbc5..0681e6729 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -5,42 +5,11 @@ from collections import namedtuple
 from queuelib import PriorityQueue
 
 from scrapy.utils.reqser import request_to_dict, request_from_dict
-from scrapy.core.downloader import Downloader
-from scrapy.http import Request
-from scrapy.signals import request_reached_downloader, request_left_downloader
-from scrapy.utils.httpobj import urlparse_cached
 
 
 logger = logging.getLogger(__name__)
 
 
-SCHEDULER_SLOT_META_KEY = Downloader.DOWNLOAD_SLOT
-
-
-def _scheduler_slot_read(request, default=None):
-    return request.meta.get(SCHEDULER_SLOT_META_KEY, default)
-
-
-def _scheduler_slot_write(request, slot):
-    request.meta[SCHEDULER_SLOT_META_KEY] = slot
-
-
-def _set_scheduler_slot(request):
-    """
-    >>> request = Request('http://example.com')
-    >>> _set_scheduler_slot(request)
-    'example.com'
-    >>> _scheduler_slot_read(request)
-    'example.com'
-    """
-    slot = _scheduler_slot_read(request, None)
-    if slot is not None:
-        return slot
-    slot = urlparse_cached(request).hostname or ''
-    _scheduler_slot_write(request, slot)
-    return slot
-
-
 def _path_safe(text):
     """ Return a filesystem-safe version of a string ``text`` """
     pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_'
@@ -138,6 +107,25 @@ class ScrapyPriorityQueue(PriorityQueue):
         return request
 
 
+class DownloaderInterface(object):
+
+    def __init__(self, crawler):
+        self.downloader = crawler.engine.downloader
+
+    def stats(self, possible_slots):
+        return [(self._active_downloads(slot), slot)
+                for slot in possible_slots]
+
+    def get_slot_key(self, request):
+        return self.downloader._get_slot_key(request, None)
+
+    def _active_downloads(self, slot):
+        """ Return a number of requests in a Downloader for a given slot """
+        if slot not in self.downloader.slots:
+            return 0
+        return len(self.downloader.slots[slot].active)
+
+
 class DownloaderAwarePriorityQueue(object):
     """ PriorityQueue which takes Downlaoder activity in account:
     domains (slots) with the least amount of active downloads are dequeued
@@ -170,68 +158,25 @@ class DownloaderAwarePriorityQueue(object):
         def pqfactory(startprios=()):
             return ScrapyPriorityQueue(crawler, qfactory, startprios, serialize)
         self._slot_pqueues = _SlotPriorityQueues(pqfactory, slot_startprios)
-
-        self._active_downloads = {slot: 0 for slot in self._slot_pqueues.pqueues}
-        crawler.signals.connect(self.on_response_download,
-                                signal=request_left_downloader)
-        crawler.signals.connect(self.on_request_reached_downloader,
-                                signal=request_reached_downloader)
         self.serialize = serialize
-
-    # There are two PriorityQueues at the same time (memory and disk-based),
-    # and they both listen to Downloader signals. To filter out signals
-    # coming from the other queue, each queue keeps track of its own
-    # requests using mark / unmark / check_mark methods.
-    def mark(self, request):
-        request.meta[self._DOWNLOADER_AWARE_PQ_ID] = id(self)
-
-    def check_mark(self, request):
-        return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self)
-
-    def unmark(self, request):
-        del request.meta[self._DOWNLOADER_AWARE_PQ_ID]
+        self._downloader_interface = DownloaderInterface(crawler)
 
     def pop(self):
-        slots = [(active_downloads, slot)
-                 for slot, active_downloads in self._active_downloads.items()
-                 if slot in self._slot_pqueues]
+        stats = self._downloader_interface.stats(self._slot_pqueues.pqueues)
 
-        if not slots:
+        if not stats:
             return
 
-        slot = min(slots)[1]
+        slot = min(stats)[1]
         request = self._slot_pqueues.pop_slot(slot)
-        self.mark(request)
         return request
 
     def push(self, request, priority):
-        slot = _set_scheduler_slot(request)
+        slot = self._downloader_interface.get_slot_key(request)
         priority_slot = _Priority(priority=priority, slot=slot)
         self._slot_pqueues.push_slot(slot, request, priority_slot)
-        if slot not in self._active_downloads:
-            self._active_downloads[slot] = 0
-
-    def on_response_download(self, request, spider):
-        if not self.check_mark(request):
-            return
-        self.unmark(request)
-
-        slot = _scheduler_slot_read(request)
-        if slot not in self._active_downloads or self._active_downloads[slot] <= 0:
-            raise ValueError('Got response for a wrong slot "%s"' % (slot, ))
-        self._active_downloads[slot] -= 1
-        if self._active_downloads[slot] == 0 and slot not in self._slot_pqueues:
-            del self._active_downloads[slot]
-
-    def on_request_reached_downloader(self, request, spider):
-        if not self.check_mark(request):
-            return
-
-        slot = _scheduler_slot_read(request)
-        self._active_downloads[slot] = self._active_downloads.get(slot, 0) + 1
 
     def close(self):
-        self._active_downloads.clear()
         active = self._slot_pqueues.close()
         return {slot: [p.priority for p in startprios]
                 for slot, startprios in active.items()}
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 1bcc1e5a8..75c0b7530 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -1,20 +1,50 @@
 import shutil
 import tempfile
 import unittest
+import collections
 
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 
 from scrapy.crawler import Crawler
+from scrapy.core.downloader import Downloader
 from scrapy.core.scheduler import Scheduler
 from scrapy.http import Request
-from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write
-from scrapy.signals import request_reached_downloader, request_left_downloader
 from scrapy.spiders import Spider
+from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
 
 
+MockEngine = collections.namedtuple('MockEngine', ['downloader'])
+MockSlot = collections.namedtuple('MockSlot', ['active'])
+
+
+class MockDownloader:
+    def __init__(self):
+        self.slots = dict()
+
+    def _set_slot_key(self, slot, request, spider):
+        request.meta[Downloader.DOWNLOAD_SLOT] = slot
+
+    def _get_slot_key(self, request, spider):
+        if Downloader.DOWNLOAD_SLOT in request.meta:
+            return request.meta[Downloader.DOWNLOAD_SLOT]
+
+        return urlparse_cached(request).hostname or ''
+
+    def increment(self, slot_key):
+        slot = self.slots.setdefault(slot_key, MockSlot(active=list()))
+        slot.active.append(1)
+
+    def decrement(self, slot_key):
+        slot = self.slots.get(slot_key)
+        slot.active.pop()
+
+    def close(self):
+        pass
+
+
 class MockCrawler(Crawler):
     def __init__(self, priority_queue_cls, jobdir):
 
@@ -27,6 +57,7 @@ class MockCrawler(Crawler):
                 DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter'
                 )
         super(MockCrawler, self).__init__(Spider, settings)
+        self.engine = MockEngine(downloader=MockDownloader())
 
 
 class SchedulerHandler:
@@ -42,6 +73,7 @@ class SchedulerHandler:
     def close_scheduler(self):
         self.scheduler.close('finished')
         self.mock_crawler.stop()
+        self.mock_crawler.engine.downloader.close()
 
     def setUp(self):
         self.create_scheduler()
@@ -147,11 +179,11 @@ class BaseSchedulerOnDiskTester(SchedulerHandler):
 
 
 class TestSchedulerInMemory(BaseSchedulerInMemoryTester, unittest.TestCase):
-    priority_queue_cls = 'queuelib.PriorityQueue'
+    priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue'
 
 
 class TestSchedulerOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase):
-    priority_queue_cls = 'queuelib.PriorityQueue'
+    priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue'
 
 
 _SLOTS = [("http://foo.com/a", 'a'),
@@ -172,7 +204,7 @@ class TestMigration(unittest.TestCase):
 
     def _migration(self, tmp_dir):
         prev_scheduler_handler = SchedulerHandler()
-        prev_scheduler_handler.priority_queue_cls = 'queuelib.PriorityQueue'
+        prev_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue'
         prev_scheduler_handler.jobdir = tmp_dir
 
         prev_scheduler_handler.create_scheduler()
@@ -196,30 +228,25 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester,
     priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
 
     def test_logic(self):
+        downloader = self.mock_crawler.engine.downloader
         for url, slot in _SLOTS:
             request = Request(url)
-            _scheduler_slot_write(request, slot)
+            downloader._set_slot_key(slot, request, None)
             self.scheduler.enqueue_request(request)
 
         slots = list()
         requests = list()
         while self.scheduler.has_pending_requests():
             request = self.scheduler.next_request()
-            slots.append(_scheduler_slot_read(request))
-            self.mock_crawler.signals.send_catch_log(
-                    signal=request_reached_downloader,
-                    request=request,
-                    spider=self.spider
-                    )
+            slot = downloader._get_slot_key(request, None)
+            slots.append(slot)
+            downloader.increment(slot)
             requests.append(request)
         self.assertEqual(len(slots), len(_SLOTS))
 
         for request in requests:
-            self.mock_crawler.signals.send_catch_log(
-                    signal=request_left_downloader,
-                    request=request,
-                    spider=self.spider
-                    )
+            slot = downloader._get_slot_key(request, None)
+            self.mock_crawler.engine.downloader.decrement(slot)
 
         unique_slots = len(set(s for _, s in _SLOTS))
         for i in range(0, len(_SLOTS), unique_slots):
@@ -239,9 +266,11 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
     priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
 
     def test_logic(self):
+        downloader = self.mock_crawler.engine.downloader
+
         for url, slot in _SLOTS:
             request = Request(url)
-            _scheduler_slot_write(request, slot)
+            downloader._set_slot_key(slot, request, None)
             self.scheduler.enqueue_request(request)
 
         self.close_scheduler()
@@ -249,27 +278,22 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
 
         slots = []
         requests = []
+        downloader = self.mock_crawler.engine.downloader
         while self.scheduler.has_pending_requests():
             request = self.scheduler.next_request()
-            slots.append(_scheduler_slot_read(request))
-            self.mock_crawler.signals.send_catch_log(
-                    signal=request_reached_downloader,
-                    request=request,
-                    spider=self.spider
-                    )
+            slot = downloader._get_slot_key(request, None)
+            slots.append(slot)
+            downloader.increment(slot)
             requests.append(request)
 
-        self.assertEqual(self.scheduler.mqs._active_downloads, {})
         self.assertEqual(len(slots), len(_SLOTS))
 
         for request in requests:
-            self.mock_crawler.signals.send_catch_log(
-                    signal=request_left_downloader,
-                    request=request,
-                    spider=self.spider
-                    )
+            slot = downloader._get_slot_key(request, None)
+            downloader.decrement(slot)
 
         _is_slots_unique(_SLOTS, slots)
+        self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0)
 
 
 class StartUrlsSpider(Spider):
@@ -277,6 +301,9 @@ class StartUrlsSpider(Spider):
     def __init__(self, start_urls):
         self.start_urls = start_urls
 
+    def parse(self, response):
+        pass
+
 
 class TestIntegrationWithDownloaderAwareOnDisk(TestCase):
     def setUp(self):

From 989bba6cb340fcc1ddb32e75ade567864d8b3884 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Thu, 7 Mar 2019 09:00:14 +0000
Subject: [PATCH 1985/2974] Revert "new signal"

This reverts commit 646164fd7d6dd52061804d2df7424cff929bf739.
remove tests
Revert "emit new signal"

This reverts commit fcde0c6880678957a76af6083b6248f430a00fcf.
Revert "documentation for new signal"

This reverts commit 8aeb9f696ece95c16499a96767a7afa3d9c4abf4.
---
 docs/topics/signals.rst            | 17 ---------
 scrapy/core/downloader/__init__.py |  3 --
 scrapy/signals.py                  |  1 -
 tests/test_request_left.py         | 59 ------------------------------
 4 files changed, 80 deletions(-)
 delete mode 100644 tests/test_request_left.py

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index f13e8270c..ff07b9d55 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -295,23 +295,6 @@ request_reached_downloader
     :param spider: the spider that yielded the request
     :type spider: :class:`~scrapy.spiders.Spider` object
 
-request_left_downloader
----------------------------
-
-.. signal:: request_left_downloader
-.. function:: request_left_downloader(request, spider)
-
-    Sent when a :class:`~scrapy.http.Request` left downloader even in case of
-    failure.
-
-    The signal does not support returning deferreds from their handlers.
-
-    :param request: the request that reached downloader
-    :type request: :class:`~scrapy.http.Request` object
-
-    :param spider: the spider that yielded the request
-    :type spider: :class:`~scrapy.spiders.Spider` object
-
 response_received
 -----------------
 
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index d856a2f37..4695d75f4 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -188,9 +188,6 @@ class Downloader(object):
         def finish_transferring(_):
             slot.transferring.remove(request)
             self._process_queue(spider, slot)
-            self.signals.send_catch_log(signal=signals.request_left_downloader,
-                                        request=request,
-                                        spider=spider)
             return _
 
         return dfd.addBoth(finish_transferring)
diff --git a/scrapy/signals.py b/scrapy/signals.py
index 2ea986b8c..c0e4bb74e 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -14,7 +14,6 @@ spider_error = object()
 request_scheduled = object()
 request_dropped = object()
 request_reached_downloader = object()
-request_left_downloader = object()
 response_received = object()
 response_downloaded = object()
 item_scraped = object()
diff --git a/tests/test_request_left.py b/tests/test_request_left.py
deleted file mode 100644
index ddeca0499..000000000
--- a/tests/test_request_left.py
+++ /dev/null
@@ -1,59 +0,0 @@
-from twisted.internet import defer
-from twisted.trial.unittest import TestCase
-from scrapy.signals import request_left_downloader
-from scrapy.spiders import Spider
-from scrapy.utils.test import get_crawler
-from tests.mockserver import MockServer
-
-class SignalCatcherSpider(Spider):
-    name = 'signal_catcher'
-
-    def __init__(self, crawler, url, *args, **kwargs):
-        super(SignalCatcherSpider, self).__init__(*args, **kwargs)
-        crawler.signals.connect(self.on_response_download,
-                                signal=request_left_downloader)
-        self.catched_times = 0
-        self.start_urls = [url]
-
-    @classmethod
-    def from_crawler(cls, crawler, *args, **kwargs):
-        spider = cls(crawler, *args, **kwargs)
-        return spider
-
-    def on_response_download(self, request, spider):
-        self.catched_times = self.catched_times + 1
-
-
-class TestCatching(TestCase):
-
-    def setUp(self):
-        self.mockserver = MockServer()
-        self.mockserver.__enter__()
-
-    def tearDown(self):
-        self.mockserver.__exit__(None, None, None)
-
-    @defer.inlineCallbacks
-    def test_success(self):
-        crawler = get_crawler(SignalCatcherSpider)
-        yield crawler.crawl(self.mockserver.url("/status?n=200"))
-        self.assertEqual(crawler.spider.catched_times, 1)
-
-    @defer.inlineCallbacks
-    def test_timeout(self):
-        crawler = get_crawler(SignalCatcherSpider,
-                              {'DOWNLOAD_TIMEOUT': 0.1})
-        yield crawler.crawl(self.mockserver.url("/delay?n=0.2"))
-        self.assertEqual(crawler.spider.catched_times, 1)
-
-    @defer.inlineCallbacks
-    def test_disconnect(self):
-        crawler = get_crawler(SignalCatcherSpider)
-        yield crawler.crawl(self.mockserver.url("/drop"))
-        self.assertEqual(crawler.spider.catched_times, 1)
-
-    @defer.inlineCallbacks
-    def test_noconnect(self):
-        crawler = get_crawler(SignalCatcherSpider)
-        yield crawler.crawl('http://thereisdefinetelynosuchdomain.com')
-        self.assertEqual(crawler.spider.catched_times, 1)

From 8afffb7234b282dd8bd28eec2e4eb8e3f86b5723 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 22 Mar 2019 09:12:23 +0000
Subject: [PATCH 1986/2974] Tests Cleanup add doctest for function no need in
 this variables move common assertion inside function rename variable rename
 variables rename function use function this is not a method of public API
 correct name for test Update docs/topics/settings.rst

Co-Authored-By: whalebot-helmsman <whalebot.helmsman@gmail.com>
---
 docs/topics/settings.rst |  4 +-
 tests/test_scheduler.py  | 82 ++++++++++++++++++++++------------------
 2 files changed, 48 insertions(+), 38 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 6e13e64d6..cf454f4ec 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1146,9 +1146,9 @@ Default: ``'scrapy.pqueues.ScrapyPriorityQueue'``
 
 Type of priority queue used by scheduler. Another available type is
 ``scrapy.pqueues.DownloaderAwarePriorityQueue``.
-``scrapy.pqueues.DownloaderAwarePriorityQueue`` is works better than
+``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than
 ``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different
-domains in parallel. But ``scrapy.pqueues.DownloaderAwarePriorityQueue``
+domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue``
 does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`.
 
 .. setting:: SPIDER_CONTRACTS
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 75c0b7530..eaf748d35 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -24,9 +24,6 @@ class MockDownloader:
     def __init__(self):
         self.slots = dict()
 
-    def _set_slot_key(self, slot, request, spider):
-        request.meta[Downloader.DOWNLOAD_SLOT] = slot
-
     def _get_slot_key(self, request, spider):
         if Downloader.DOWNLOAD_SLOT in request.meta:
             return request.meta[Downloader.DOWNLOAD_SLOT]
@@ -186,12 +183,12 @@ class TestSchedulerOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase):
     priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue'
 
 
-_SLOTS = [("http://foo.com/a", 'a'),
-          ("http://foo.com/b", 'a'),
-          ("http://foo.com/c", 'b'),
-          ("http://foo.com/d", 'b'),
-          ("http://foo.com/e", 'c'),
-          ("http://foo.com/f", 'c')]
+_URLS_WITH_SLOTS = [("http://foo.com/a", 'a'),
+                    ("http://foo.com/b", 'a'),
+                    ("http://foo.com/c", 'b'),
+                    ("http://foo.com/d", 'b'),
+                    ("http://foo.com/e", 'c'),
+                    ("http://foo.com/f", 'c')]
 
 
 class TestMigration(unittest.TestCase):
@@ -228,37 +225,52 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester,
     priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
 
     def test_logic(self):
-        downloader = self.mock_crawler.engine.downloader
-        for url, slot in _SLOTS:
+        for url, slot in _URLS_WITH_SLOTS:
             request = Request(url)
-            downloader._set_slot_key(slot, request, None)
+            request.meta[Downloader.DOWNLOAD_SLOT] = slot
             self.scheduler.enqueue_request(request)
 
-        slots = list()
+        downloader = self.mock_crawler.engine.downloader
+        dequeued_slots = list()
         requests = list()
         while self.scheduler.has_pending_requests():
             request = self.scheduler.next_request()
             slot = downloader._get_slot_key(request, None)
-            slots.append(slot)
+            dequeued_slots.append(slot)
             downloader.increment(slot)
             requests.append(request)
-        self.assertEqual(len(slots), len(_SLOTS))
 
         for request in requests:
             slot = downloader._get_slot_key(request, None)
             self.mock_crawler.engine.downloader.decrement(slot)
 
-        unique_slots = len(set(s for _, s in _SLOTS))
-        for i in range(0, len(_SLOTS), unique_slots):
-            part = slots[i:i + unique_slots]
-            self.assertEqual(len(part), len(set(part)))
+        self.assertTrue(_is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS),
+                                            dequeued_slots))
 
 
-def _is_slots_unique(base_slots, result_slots):
-    unique_slots = len(set(s for _, s in base_slots))
-    for i in range(0, len(result_slots), unique_slots):
-        part = result_slots[i:i + unique_slots]
-        assert len(part) == len(set(part))
+def _is_scheduling_fair(enqueued_slots, dequeued_slots):
+    """
+    We enqueued same number of requests for every slot.
+    Assert correct order, e.g.
+
+    >>> enqueued = ['a', 'b', 'c'] * 2
+    >>> correct = ['a', 'c', 'b', 'b', 'a', 'c']
+    >>> incorrect = ['a', 'a', 'b', 'c', 'c', 'b']
+    >>> _is_scheduling_fair(enqueued, correct)
+    True
+    >>> _is_scheduling_fair(enqueued, incorrect)
+    False
+    """
+    if len(dequeued_slots) != len(enqueued_slots):
+        return False
+
+    slots_number = len(set(enqueued_slots))
+    for i in range(0, len(dequeued_slots), slots_number):
+        part = dequeued_slots[i:i + slots_number]
+        if len(part) != len(set(part)):
+            return False
+
+    return True
 
 
 class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
@@ -266,33 +278,31 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
     priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
 
     def test_logic(self):
-        downloader = self.mock_crawler.engine.downloader
 
-        for url, slot in _SLOTS:
+        for url, slot in _URLS_WITH_SLOTS:
             request = Request(url)
-            downloader._set_slot_key(slot, request, None)
+            request.meta[Downloader.DOWNLOAD_SLOT] = slot
             self.scheduler.enqueue_request(request)
 
         self.close_scheduler()
         self.create_scheduler()
 
-        slots = []
+        dequeued_slots = list()
         requests = []
         downloader = self.mock_crawler.engine.downloader
         while self.scheduler.has_pending_requests():
             request = self.scheduler.next_request()
             slot = downloader._get_slot_key(request, None)
-            slots.append(slot)
+            dequeued_slots.append(slot)
             downloader.increment(slot)
             requests.append(request)
 
-        self.assertEqual(len(slots), len(_SLOTS))
-
         for request in requests:
             slot = downloader._get_slot_key(request, None)
             downloader.decrement(slot)
 
-        _is_slots_unique(_SLOTS, slots)
+        self.assertTrue(_is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS),
+                                            dequeued_slots))
         self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0)
 
 
@@ -305,7 +315,7 @@ class StartUrlsSpider(Spider):
         pass
 
 
-class TestIntegrationWithDownloaderAwareOnDisk(TestCase):
+class TestIntegrationWithDownloaderAwareInMemory(TestCase):
     def setUp(self):
         self.crawler = get_crawler(
                     StartUrlsSpider,
@@ -322,10 +332,10 @@ class TestIntegrationWithDownloaderAwareOnDisk(TestCase):
         with MockServer() as mockserver:
 
             url = mockserver.url("/status?n=200", is_secure=False)
-            slots = [url] * 6
-            yield self.crawler.crawl(slots)
+            start_urls = [url] * 6
+            yield self.crawler.crawl(start_urls)
             self.assertEqual(self.crawler.stats.get_value('downloader/response_count'),
-                             len(slots))
+                             len(start_urls))
 
 
 class TestIncompatibility(unittest.TestCase):

From df574de8cc5c58618f6075ca3afb14059a9e30ed Mon Sep 17 00:00:00 2001
From: Lucy Wang <lucy@scrapinghub.com>
Date: Sat, 23 Mar 2019 00:54:39 +0800
Subject: [PATCH 1987/2974] improve tests and fix some lint warnings (#6)

* refactor downloader-aware test cases

* fix lint

* add doctest for _path_safe

* remove unused code

* better doctest
---
 scrapy/pqueues.py       | 12 +++++++--
 tests/test_scheduler.py | 57 ++++++++++++++++-------------------------
 2 files changed, 32 insertions(+), 37 deletions(-)

diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 0681e6729..6ecd1b51a 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -11,7 +11,16 @@ logger = logging.getLogger(__name__)
 
 
 def _path_safe(text):
-    """ Return a filesystem-safe version of a string ``text`` """
+    """
+    Return a filesystem-safe version of a string ``text``
+
+    >>> _path_safe('simple.org').startswith('simple.org')
+    True
+    >>> _path_safe('dash-underscore_.org').startswith('dash-underscore_.org')
+    True
+    >>> _path_safe('some@symbol?').startswith('some_symbol_')
+    True
+    """
     pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_'
                              for c in text])
     # as we replace some letters we can get collision for different slots
@@ -131,7 +140,6 @@ class DownloaderAwarePriorityQueue(object):
     domains (slots) with the least amount of active downloads are dequeued
     first.
     """
-    _DOWNLOADER_AWARE_PQ_ID = '_DOWNLOADER_AWARE_PQ_ID'
 
     @classmethod
     def from_crawler(cls, crawler, qfactory, slot_startprios=None, serialize=False):
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index eaf748d35..e0e3600e5 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -20,7 +20,7 @@ MockEngine = collections.namedtuple('MockEngine', ['downloader'])
 MockSlot = collections.namedtuple('MockSlot', ['active'])
 
 
-class MockDownloader:
+class MockDownloader(object):
     def __init__(self):
         self.slots = dict()
 
@@ -57,7 +57,7 @@ class MockCrawler(Crawler):
         self.engine = MockEngine(downloader=MockDownloader())
 
 
-class SchedulerHandler:
+class SchedulerHandler(object):
     priority_queue_cls = None
     jobdir = None
 
@@ -220,34 +220,6 @@ class TestMigration(unittest.TestCase):
             self._migration(self.tmpdir)
 
 
-class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester,
-                                               unittest.TestCase):
-    priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
-
-    def test_logic(self):
-        for url, slot in _URLS_WITH_SLOTS:
-            request = Request(url)
-            request.meta[Downloader.DOWNLOAD_SLOT] = slot
-            self.scheduler.enqueue_request(request)
-
-        downloader = self.mock_crawler.engine.downloader
-        dequeued_slots = list()
-        requests = list()
-        while self.scheduler.has_pending_requests():
-            request = self.scheduler.next_request()
-            slot = downloader._get_slot_key(request, None)
-            dequeued_slots.append(slot)
-            downloader.increment(slot)
-            requests.append(request)
-
-        for request in requests:
-            slot = downloader._get_slot_key(request, None)
-            self.mock_crawler.engine.downloader.decrement(slot)
-
-        self.assertTrue(_is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS),
-                                            dequeued_slots))
-
-
 def _is_scheduling_fair(enqueued_slots, dequeued_slots):
     """
     We enqueued same number of requests for every slot.
@@ -273,31 +245,33 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots):
     return True
 
 
-class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
-                                             unittest.TestCase):
+class DownloaderAwareSchedulerTestMixin(object):
     priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
+    reopen = False
 
     def test_logic(self):
-
         for url, slot in _URLS_WITH_SLOTS:
             request = Request(url)
             request.meta[Downloader.DOWNLOAD_SLOT] = slot
             self.scheduler.enqueue_request(request)
 
-        self.close_scheduler()
-        self.create_scheduler()
+        if self.reopen:
+            self.close_scheduler()
+            self.create_scheduler()
 
         dequeued_slots = list()
         requests = []
         downloader = self.mock_crawler.engine.downloader
         while self.scheduler.has_pending_requests():
             request = self.scheduler.next_request()
+            # pylint: disable=protected-access
             slot = downloader._get_slot_key(request, None)
             dequeued_slots.append(slot)
             downloader.increment(slot)
             requests.append(request)
 
         for request in requests:
+            # pylint: disable=protected-access
             slot = downloader._get_slot_key(request, None)
             downloader.decrement(slot)
 
@@ -306,10 +280,23 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
         self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0)
 
 
+class TestSchedulerWithDownloaderAwareInMemory(DownloaderAwareSchedulerTestMixin,
+                                               BaseSchedulerInMemoryTester,
+                                               unittest.TestCase):
+    pass
+
+
+class TestSchedulerWithDownloaderAwareOnDisk(DownloaderAwareSchedulerTestMixin,
+                                             BaseSchedulerOnDiskTester,
+                                             unittest.TestCase):
+    reopen = True
+
+
 class StartUrlsSpider(Spider):
 
     def __init__(self, start_urls):
         self.start_urls = start_urls
+        super(StartUrlsSpider, self).__init__(start_urls)
 
     def parse(self, response):
         pass

From 31b8a6b33aed9e77a4d37a5c83b1545202207cad Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Mon, 25 Mar 2019 08:53:15 +0000
Subject: [PATCH 1988/2974] report warnings

---
 tests/test_crawler.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 268948a70..d9ec9ee8d 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,5 +1,4 @@
 import logging
-import tempfile
 import warnings
 
 from twisted.internet import defer
@@ -37,7 +36,11 @@ class CrawlerTestCase(BaseCrawlerTest):
             self.assertIsInstance(spiders, sl_cls)
 
             self.crawler.spiders
-            self.assertEqual(len(w), 1, "Warn deprecated access only once")
+            is_one_warning = len(w) == 1
+            if not is_one_warning:
+                for warning in w:
+                    print(warning)
+            self.assertTrue(is_one_warning, "Warn deprecated access only once")
 
     def test_populate_spidercls_settings(self):
         spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}

From 73e4ff5304d273404a147d06726a8ae8cae1c925 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Mon, 25 Mar 2019 13:48:58 +0000
Subject: [PATCH 1989/2974] report warnings

---
 tests/test_crawler.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 8c4bbe0d9..e811c5757 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -182,8 +182,12 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
                 'SPIDER_MANAGER_CLASS': 'tests.test_crawler.CustomSpiderLoader'
             })
             self.assertIsInstance(runner.spider_loader, CustomSpiderLoader)
-            self.assertEqual(len(w), 1)
+            is_one_warning = len(w) == 1
+            if not is_one_warning:
+                for warning in w:
+                    print(warning)
             self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message))
+            self.assertTrue(is_one_warning)
 
     def test_crawl_rejects_spider_objects(self):
         with raises(ValueError):

From 431f18a9a1a87ac8b789e3c260016e510ab48ad3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 1 Feb 2019 13:22:38 +0100
Subject: [PATCH 1990/2974] Document FilesPipeline.file_path and
 ImagesPipeline.file_path

---
 docs/topics/media-pipeline.rst | 60 ++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index c60b55391..c97b4c3c2 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -392,6 +392,36 @@ See here the methods that you can override in your custom Files Pipeline:
 
 .. class:: FilesPipeline
 
+   .. method:: file_path(request, response, info)
+
+      This method is called once per downloaded item. It returns the
+      download path of the file originating from the specified
+      :class:`response <scrapy.http.Response>`.
+
+      In addition to ``response``, this method receives the original
+      :class:`request <scrapy.Request>` and
+      :class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
+
+      You can override this method to customize the download path of each file.
+
+      For example, if file URLs end like regular paths (e.g.
+      ``https://example.com/a/b/c/foo.png``), you can use the following
+      approach to download all files into the ``files`` folder with their
+      original filenames (e.g. ``files/foo.png``)::
+
+        import os
+        from urllib.parse import urlparse
+
+        from scrapy.pipelines.files import FilesPipeline
+
+        class MyFilesPipeline(FilesPipeline):
+
+            def file_path(self, request, response, info):
+                return 'files/' + os.path.basename(urlparse(request.url).path)
+
+      By default the :meth:`file_path` method returns
+      ``full/<request URL hash>.<extension>``.
+
    .. method:: FilesPipeline.get_media_requests(item, info)
 
       As seen on the workflow, the pipeline will get the URLs of the images to
@@ -475,6 +505,36 @@ See here the methods that you can override in your custom Images Pipeline:
     The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
     customizing the field names and adding custom behavior for images.
 
+   .. method:: file_path(request, response, info)
+
+      This method is called once per downloaded item. It returns the
+      download path of the file originating from the specified
+      :class:`response <scrapy.http.Response>`.
+
+      In addition to ``response``, this method receives the original
+      :class:`request <scrapy.Request>` and
+      :class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
+
+      You can override this method to customize the download path of each file.
+
+      For example, if file URLs end like regular paths (e.g.
+      ``https://example.com/a/b/c/foo.png``), you can use the following
+      approach to download all files into the ``files`` folder with their
+      original filenames (e.g. ``files/foo.png``)::
+
+        import os
+        from urllib.parse import urlparse
+
+        from scrapy.pipelines.images import ImagesPipeline
+
+        class MyImagesPipeline(ImagesPipeline):
+
+            def file_path(self, request, response, info):
+                return 'files/' + os.path.basename(urlparse(request.url).path)
+
+      By default the :meth:`file_path` method returns
+      ``full/<request URL hash>.<extension>``.
+
    .. method:: ImagesPipeline.get_media_requests(item, info)
 
       Works the same way as :meth:`FilesPipeline.get_media_requests` method,

From 2fd8b7c28c255148f6d7320cb97292ab2d569eff Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Wed, 27 Mar 2019 00:45:53 +0530
Subject: [PATCH 1991/2974] [MRG+1] redirect_reasons in Request.meta (#3687)

---
 docs/topics/downloader-middleware.rst       | 16 ++++++++++++++--
 docs/topics/request-response.rst            |  1 +
 scrapy/downloadermiddlewares/redirect.py    |  2 ++
 tests/test_downloadermiddleware_redirect.py | 19 +++++++++++++++++++
 4 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 0d976077b..f2f3ef466 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -733,6 +733,17 @@ RedirectMiddleware
 The urls which the request goes through (while being redirected) can be found
 in the ``redirect_urls`` :attr:`Request.meta <scrapy.http.Request.meta>` key.
 
+.. reqmeta:: redirect_reasons
+
+The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the
+``redirect_reasons`` :attr:`Request.meta <scrapy.http.Request.meta>` key. For
+example: ``[301, 302, 307, 'meta refresh']``.
+
+The format of a reason depends on the middleware that handled the corresponding
+redirect. For example, :class:`RedirectMiddleware` indicates the triggering
+response status code as an integer, while :class:`MetaRefreshMiddleware`
+always uses the ``'meta refresh'`` string as reason.
+
 The :class:`RedirectMiddleware` can be configured through the following
 settings (see the settings documentation for more info):
 
@@ -796,8 +807,9 @@ settings (see the settings documentation for more info):
 * :setting:`METAREFRESH_ENABLED`
 * :setting:`METAREFRESH_MAXDELAY`
 
-This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`
-and :reqmeta:`redirect_urls` request meta keys as described for :class:`RedirectMiddleware`
+This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`,
+:reqmeta:`redirect_urls` and :reqmeta:`redirect_reasons` request meta keys as described
+for :class:`RedirectMiddleware`
 
 
 MetaRefreshMiddleware settings
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index ca59b46d8..ac6fe6e3f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -299,6 +299,7 @@ Those are:
 * :reqmeta:`dont_merge_cookies`
 * :reqmeta:`cookiejar`
 * :reqmeta:`dont_cache`
+* :reqmeta:`redirect_reasons`
 * :reqmeta:`redirect_urls`
 * :reqmeta:`bindaddress`
 * :reqmeta:`dont_obey_robotstxt`
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 30cae3fee..cb59d3fd2 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -34,6 +34,8 @@ class BaseRedirectMiddleware(object):
             redirected.meta['redirect_ttl'] = ttl - 1
             redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \
                 [request.url]
+            redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \
+                [reason]
             redirected.dont_filter = request.dont_filter
             redirected.priority = request.priority + self.priority_adjust
             logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 74137b4cd..6c81c94ca 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -139,6 +139,16 @@ class RedirectMiddlewareTest(unittest.TestCase):
         self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
         self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])
 
+    def test_redirect_reasons(self):
+        req1 = Request('http://scrapytest.org/first')
+        rsp1 = Response('http://scrapytest.org/first', headers={'Location': '/redirected1'}, status=301)
+        req2 = self.mw.process_response(req1, rsp1, self.spider)
+        rsp2 = Response('http://scrapytest.org/redirected1', headers={'Location': '/redirected2'}, status=301)
+        req3 = self.mw.process_response(req2, rsp2, self.spider)
+
+        self.assertEqual(req2.meta['redirect_reasons'], [301])
+        self.assertEqual(req3.meta['redirect_reasons'], [301, 301])
+
     def test_spider_handling(self):
         smartspider = self.crawler._create_spider('smarty')
         smartspider.handle_httpstatus_list = [404, 301, 302]
@@ -259,6 +269,15 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
         self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
         self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])
 
+    def test_redirect_reasons(self):
+        req1 = Request('http://scrapytest.org/first')
+        rsp1 = HtmlResponse('http://scrapytest.org/first', body=self._body(url='/redirected'))
+        req2 = self.mw.process_response(req1, rsp1, self.spider)
+        rsp2 = HtmlResponse('http://scrapytest.org/redirected', body=self._body(url='/redirected1'))
+        req3 = self.mw.process_response(req2, rsp2, self.spider)
+
+        self.assertEqual(req2.meta['redirect_reasons'], ['meta refresh'])
+        self.assertEqual(req3.meta['redirect_reasons'], ['meta refresh', 'meta refresh'])
 
 if __name__ == "__main__":
     unittest.main()

From ce837b0f740e989520cce58d1606a528b598503e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 25 Mar 2019 18:04:04 +0100
Subject: [PATCH 1992/2974] Update the documentation policies: Ask to use
 docstrings to document API members

---
 docs/contributing.rst | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 9b508e418..aac0f4496 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -165,18 +165,18 @@ Scrapy:
 Documentation policies
 ======================
 
-* **Don't** use docstrings for documenting classes, or methods which are
-  already documented in the official (sphinx) documentation. Alternatively,
-  **do** provide a docstring, but make sure sphinx documentation uses
-  autodoc_ extension to pull the docstring. For example, the
-  :meth:`ItemLoader.add_value` method should be either
-  documented only in the sphinx documentation (not as a docstring), or
-  it should have a docstring which is pulled to sphinx documentation using
-  autodoc_ extension.
+For reference documentation of API members (classes, methods, etc.) use
+docstrings and make sure that the Sphinx documentation uses the autodoc_
+extension to pull the docstrings. API reference documentation should be
+IDE-friendly: short, to the point, and it may provide short examples.
 
-* **Do** use docstrings for documenting functions not present in the official
-  (sphinx) documentation, such as functions from ``scrapy.utils`` package and
-  its sub-modules.
+Other types of documentation, such as tutorials or topics, should be covered in
+files within the ``docs/`` directory. This includes documentation that is
+specific to an API member, but goes beyond API reference documentation.
+
+In any case, if something is covered in a docstring, use the autodoc_
+extension to pull the docstring into the documentation instead of duplicating
+the docstring in files within the ``docs/`` directory.
 
 .. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html
 

From 845bae6637239c859c9952c23f42902e36d10f6b Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 27 Mar 2019 08:49:19 +0000
Subject: [PATCH 1993/2974] Update docs/topics/broad-crawls.rst

Co-Authored-By: whalebot-helmsman <whalebot.helmsman@gmail.com>
---
 docs/topics/broad-crawls.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index 37f7a8748..64c8883b1 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -42,7 +42,7 @@ efficient broad crawl.
 Use proper :setting:`SCHEDULER_PRIORITY_QUEUE`
 ==============================================
 
-Default scrapy's scheduler priority queue is ``'queuelib.PriorityQueue'``.
+Default scrapy's scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``.
 It works best during single domain crawl. And it does not work well with crawling
 many different domains in parallel
 

From 9c9bca4e1c7984089c44f3a44e7594e06307b12f Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Wed, 27 Mar 2019 18:29:48 +0530
Subject: [PATCH 1994/2974] make suggested changes.

---
 docs/topics/downloader-middleware.rst | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index f913b059d..dfbcdb8fa 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -349,7 +349,7 @@ HttpCacheMiddleware
         * :ref:`httpcache-storage-leveldb`
 
     You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE`
-    setting. Or you can also implement your own storage backend.
+    setting. Or you can also :ref:`implement your own storage backend. <httpcache-storage-custom>`
 
     Scrapy ships with two HTTP cache policies:
 
@@ -498,10 +498,10 @@ In order to use this storage backend:
 
 .. _httpcache-storage-custom:
 
-Implementing custom cache storage backend
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Writing your own storage backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You can implement custom cache storage backend by creating a Python class that
+You can implement a cache storage backend by creating a Python class that
 defines the methods described below.
 
 .. module:: scrapy.extensions.httpcache
@@ -510,14 +510,16 @@ defines the methods described below.
 
     .. method:: open_spider(spider)
 
-      This method gets called after a spider has been opened for crawling.
+      This method gets called after a spider has been opened for crawling. It handles 
+      the :signal:`open_spider <spider_opened>` signal.
 
       :param spider: the spider which has been opened
       :type spider: :class:`~scrapy.spiders.Spider` object
 
     .. method:: close_spider(spider)
 
-      This method gets called after a spider has been closed.
+      This method gets called after a spider has been closed. It handles 
+      the :signal:`close_spider <spider_closed>` signal. 
 
       :param spider: the spider which has been closed
       :type spider: :class:`~scrapy.spiders.Spider` object
@@ -539,7 +541,7 @@ defines the methods described below.
       :param spider: the spider for which the response is intended
       :type spider: :class:`~scrapy.spiders.Spider` object
 
-      :param request: corresponding request the spider generated
+      :param request: the corresponding request the spider generated
       :type request: :class:`~scrapy.http.Request` object
 
       :param response: the response to store in the cache
@@ -547,7 +549,7 @@ defines the methods described below.
 
 In order to use your storage backend, set:
 
-* :setting:`HTTPCACHE_STORAGE` to path of your custom storage class.
+* :setting:`HTTPCACHE_STORAGE` to the Python import path of your custom storage class.
 
 
 HTTPCache middleware settings

From 8528f5065f99046b149b5e1901d6cbe5296f048a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 27 Mar 2019 14:42:26 -0300
Subject: [PATCH 1995/2974] [Doc] Update cb_kwargs example

---
 docs/topics/request-response.rst | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index b3f849540..61789be0f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -233,19 +233,19 @@ The following two examples show how to achieve this by using the
 
 ::
 
-    def parse_page1(self, response):
-        item = MyItem()
-        item['main_url'] = response.url
-        request = scrapy.Request("http://www.example.com/some_page.html",
-                                 callback=self.parse_page2)
-        request.cb_kwargs['item'] = item
-        request.cb_kwargs['foo'] = 'bar'
+    def parse(self, response):
+        request = scrapy.Request('http://www.example.com/index.html',
+                                 callback=self.parse_page2,
+                                 cb_kwargs=dict(main_url=response.url))
+        request.cb_kwargs['foo'] = 'bar'  # add more arguments for the callback
         yield request
 
-    def parse_page2(self, response, item, foo):
-        item['other_url'] = response.url
-        item['foo'] = foo
-        yield item
+    def parse_page2(self, response, main_url, foo):
+        yield dict(
+            main_url=main_url,
+            other_url=response.url,
+            foo=foo,
+        )
 
 
 .. _topics-request-response-ref-errbacks:

From 70a4d93aa324fb276e60d641b37bdc6eb707b1cb Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 28 Mar 2019 10:40:41 -0300
Subject: [PATCH 1996/2974] Callback kwargs: more tests

---
 tests/spiders.py    | 22 ++++++++++++++++++++++
 tests/test_crawl.py | 21 ++++++++++++++++++++-
 2 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index a06985837..8c8d50ff5 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -39,6 +39,9 @@ class KeywordArgumentsSpider(MockServerSpider):
         yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
         yield Request(self.mockserver.url('/general_without'), self.parse_general)
         yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
+        yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data)
+        yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)
+        yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)
 
     def parse_first(self, response, key, number):
         self.checks.append(key == 'value')
@@ -66,6 +69,25 @@ class KeywordArgumentsSpider(MockServerSpider):
         self.checks.append(response.url.endswith('/no_kwargs'))
         self.crawler.stats.inc_value('boolean_checks')
 
+    def parse_default(self, response, key, number=None, default=99):
+        self.checks.append(response.url.endswith('/default'))
+        self.checks.append(key == 'value')
+        self.checks.append(number == 123)
+        self.checks.append(default == 99)
+        self.crawler.stats.inc_value('boolean_checks', 4)
+
+    def parse_takes_less(self, response, key):
+        """
+        Should raise
+        TypeError: parse_takes_less() got an unexpected keyword argument 'number'
+        """
+
+    def parse_takes_more(self, response, key, number, other):
+        """
+        Should raise
+        TypeError: parse_takes_more() missing 1 required positional argument: 'other'
+        """
+
 
 class FollowAllSpider(MetaSpider):
 
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 9a39b8cb4..2b3e56ee9 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -4,6 +4,7 @@ import logging
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
+import six
 
 from scrapy.http import Request
 from scrapy.crawler import CrawlerRunner
@@ -15,6 +16,8 @@ from tests.mockserver import MockServer
 
 class CrawlTestCase(TestCase):
 
+    maxDiff = None
+
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
@@ -26,9 +29,25 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_callback_kwargs(self):
         crawler = self.runner.create_crawler(KeywordArgumentsSpider)
-        yield crawler.crawl(mockserver=self.mockserver)
+        with LogCapture() as log:
+            yield crawler.crawl(mockserver=self.mockserver)
         self.assertTrue(all(crawler.spider.checks))
         self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks'))
+        # check exceptions for argument mismatch
+        exceptions = {}
+        for line in log.records:
+            for key in ('takes_less', 'takes_more'):
+                if key in line.getMessage():
+                    exceptions[key] = line
+        self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
+        self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
+        self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
+        # py2 and py3 messages are different
+        exc_message = str(exceptions['takes_more'].exc_info[1])
+        if six.PY2:
+            self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
+        elif six.PY3:
+            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
 
     @defer.inlineCallbacks
     def test_follow_all(self):

From 3efe3bea1cbb5ae83c024fc6dc8e1776a47a345f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 28 Mar 2019 14:16:03 -0300
Subject: [PATCH 1997/2974] Update docs about cb_kwargs and meta

---
 docs/topics/request-response.rst | 41 ++++++++++++++++++++------------
 1 file changed, 26 insertions(+), 15 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index dd0db8156..05ca8d6c1 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -215,24 +215,12 @@ Example::
 
 In some cases you may be interested in passing arguments to those callback
 functions so you can receive the arguments later, in the second callback.
-The following two examples show how to achieve this by using the 
-:attr:`Request.meta` and :attr:`Request.cb_kwargs` attributes respectively::
-
-    def parse_page1(self, response):
-        item = MyItem()
-        item['main_url'] = response.url
-        request = scrapy.Request("http://www.example.com/some_page.html",
-                                 callback=self.parse_page2)
-        request.meta['item'] = item
-        yield request
-
-    def parse_page2(self, response):
-        item = response.meta['item']
-        item['other_url'] = response.url
-        yield item
+The following example shows how to achieve this by using the
+:attr:`Request.cb_kwargs` attribute:
 
 ::
 
+    # pass information to the next callback using the Request.cb_kwargs attribute
     def parse(self, response):
         request = scrapy.Request('http://www.example.com/index.html',
                                  callback=self.parse_page2,
@@ -247,6 +235,29 @@ The following two examples show how to achieve this by using the
             foo=foo,
         )
 
+.. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``.
+   Prior to that, :attr:`Request.meta` was the recommended option for passing
+   information around callbacks. However, after ``1.7`` :attr:`Request.cb_kwargs`
+   became the preferred way of passing user information, leaving :attr:`Request.meta`
+   to be used by internal components like spider or downloader middlewares.
+   The following example, which uses :attr:`Request.meta`, is only kept for historical
+   reasons.
+
+::
+
+    # pass information to the next callback using the Request.meta attribute
+    def parse_page1(self, response):
+        item = MyItem()
+        item['main_url'] = response.url
+        request = scrapy.Request("http://www.example.com/some_page.html",
+                                 callback=self.parse_page2)
+        request.meta['item'] = item
+        yield request
+
+    def parse_page2(self, response):
+        item = response.meta['item']
+        item['other_url'] = response.url
+        yield item
 
 .. _topics-request-response-ref-errbacks:
 

From e8af6331b5ff62d71ff80eddcc52b85c25482c0e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 28 Mar 2019 14:56:31 -0300
Subject: [PATCH 1998/2974] Add cb_kwargs option to the parse command

---
 docs/topics/commands.rst    |  3 +++
 scrapy/commands/parse.py    | 37 ++++++++++++++++++++++++-------------
 tests/test_command_parse.py | 14 ++++++++++++++
 3 files changed, 41 insertions(+), 13 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 97f8311de..6644d65e4 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -461,6 +461,9 @@ Supported options:
 * ``--meta`` or ``-m``: additional request meta that will be passed to the callback 
   request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
 
+* ``--cb_kwargs``: additional keyword arguments that will be passed to the callback.
+  This must be a valid json string. Example: --cb_kwargs='{"foo" : "bar"}'
+
 * ``--pipelines``: process items through pipelines
 
 * ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider`
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 69418a478..2486f3f23 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -51,12 +51,13 @@ class Command(ScrapyCommand):
             help="use this callback for parsing, instead looking for a callback")
         parser.add_option("-m", "--meta", dest="meta",
             help="inject extra meta into the Request, it must be a valid raw json string")
+        parser.add_option("--cb_kwargs", dest="cb_kwargs",
+            help="inject extra cb_kwargs into the Request, it must be a valid raw json string")
         parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
             help="maximum depth for parsing requests [default: %default]")
         parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
             help="print each depth level one by one")
 
-
     @property
     def max_level(self):
         levels = list(self.items.keys()) + list(self.requests.keys())
@@ -111,10 +112,11 @@ class Command(ScrapyCommand):
             if not opts.nolinks:
                 self.print_requests(colour=colour)
 
-    def run_callback(self, response, cb):
+    def run_callback(self, response, callback, cb_kwargs=None):
+        cb_kwargs = cb_kwargs or {}
         items, requests = [], []
 
-        for x in iterate_spider_output(cb(response)):
+        for x in iterate_spider_output(callback(response, **cb_kwargs)):
             if isinstance(x, (BaseItem, dict)):
                 items.append(x)
             elif isinstance(x, Request):
@@ -142,8 +144,7 @@ class Command(ScrapyCommand):
         else:
             self.spidercls = spidercls_for_request(spider_loader, Request(url))
             if not self.spidercls:
-                logger.error('Unable to find spider for: %(url)s',
-                             {'url': url})
+                logger.error('Unable to find spider for: %(url)s', {'url': url})
 
         # Request requires callback argument as callable or None, not string
         request = Request(url, None)
@@ -160,7 +161,7 @@ class Command(ScrapyCommand):
                          {'url': url})
 
     def prepare_request(self, spider, request, opts):
-        def callback(response):
+        def callback(response, **cb_kwargs):
             # memorize first request
             if not self.first_response:
                 self.first_response = response
@@ -175,7 +176,7 @@ class Command(ScrapyCommand):
 
                     if not cb:
                         logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',
-                                 {'url': response.url, 'spider': spider.name})
+                                     {'url': response.url, 'spider': spider.name})
                         return
                 else:
                     cb = 'parse'
@@ -192,7 +193,7 @@ class Command(ScrapyCommand):
             # parse items and requests
             depth = response.meta['_depth']
 
-            items, requests = self.run_callback(response, cb)
+            items, requests = self.run_callback(response, cb, cb_kwargs)
             if opts.pipelines:
                 itemproc = self.pcrawler.engine.scraper.itemproc
                 for item in items:
@@ -207,10 +208,14 @@ class Command(ScrapyCommand):
                     req.callback = callback
                 return requests
 
-        #update request meta if any extra meta was passed through the --meta/-m opts.
+        # update request meta if any extra meta was passed through the --meta/-m opts.
         if opts.meta:
             request.meta.update(opts.meta)
 
+        # update cb_kwargs if any extra cb_kwargs was passed through the --cb_kwargs option.
+        if opts.cb_kwargs:
+            request.cb_kwargs.update(opts.cb_kwargs)
+
         request.meta['_depth'] = 1
         request.meta['_callback'] = request.callback
         request.callback = callback
@@ -221,23 +226,29 @@ class Command(ScrapyCommand):
 
         self.process_spider_arguments(opts)
         self.process_request_meta(opts)
+        self.process_request_cb_kwargs(opts)
 
     def process_spider_arguments(self, opts):
-
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
 
     def process_request_meta(self, opts):
-
         if opts.meta:
             try:
                 opts.meta = json.loads(opts.meta)
             except ValueError:
-                raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \
-                                "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
+                raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. "
+                                 "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
 
+    def process_request_cb_kwargs(self, opts):
+        if opts.cb_kwargs:
+            try:
+                opts.cb_kwargs = json.loads(opts.cb_kwargs)
+            except ValueError:
+                raise UsageError("Invalid --cb_kwargs value, pass a valid json string to --cb_kwargs. "
+                                 "Example: --cb_kwargs='{\"foo\" : \"bar\"}'", print_help=False)
 
     def run(self, args, opts):
         # parse arguments
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 02037b866..1404005fb 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -43,6 +43,12 @@ class MySpider(scrapy.Spider):
         else:
             self.logger.debug('It Works!')
 
+    def parse_request_with_cb_kwargs(self, response, foo=None, key=None):
+        if foo == 'bar' and key == 'value':
+            self.logger.debug('It Works!')
+        else:
+            self.logger.debug('It Does Not Work :(')
+
     def parse_request_without_meta(self, response):
         foo = response.meta.get('foo', 'bar')
 
@@ -120,6 +126,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
+    @defer.inlineCallbacks
+    def test_request_with_cb_kwargs(self):
+        raw_json_string = '{"foo" : "bar", "key": "value"}'
+        _, _, stderr = yield self.execute(['--spider', self.spider_name,
+                                           '--cb_kwargs', raw_json_string,
+                                           '-c', 'parse_request_with_cb_kwargs',
+                                           self.url('/html')])
+        self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_request_without_meta(self):

From 8fb077694fcaa50a8625c8e2e8d0068add2b056d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 28 Mar 2019 15:18:00 -0300
Subject: [PATCH 1999/2974] Request.cb_kwargs: Update docs

---
 docs/topics/debug.rst            | 13 +++++--------
 docs/topics/jobs.rst             |  9 +++++----
 docs/topics/leaks.rst            | 14 ++++++++------
 docs/topics/request-response.rst | 13 ++++++-------
 4 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst
index f93aa2c72..0aaad0c77 100644
--- a/docs/topics/debug.rst
+++ b/docs/topics/debug.rst
@@ -28,16 +28,15 @@ Consider the following scrapy spider below::
             item = MyItem()
             # populate `item` fields
             # and extract item_details_url
-            yield scrapy.Request(item_details_url, self.parse_details, meta={'item': item})
+            yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item})
 
-        def parse_details(self, response):
-            item = response.meta['item']
+        def parse_details(self, response, item):
             # populate more `item` fields
             return item
 
 Basically this is a simple spider which parses two pages of items (the
 start_urls). Items also have a details page with additional information, so we
-use the ``meta`` functionality of :class:`~scrapy.http.Request` to pass a
+use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a
 partially populated item.
 
 
@@ -100,8 +99,7 @@ Fortunately, the :command:`shell` is your bread and butter in this case (see
 
     from scrapy.shell import inspect_response
 
-    def parse_details(self, response):
-        item = response.meta.get('item', None)
+    def parse_details(self, response, item=None):
         if item:
             # populate more `item` fields
             return item
@@ -134,8 +132,7 @@ Logging is another useful option for getting information about your spider run.
 Although not as convenient, it comes with the advantage that the logs will be
 available in all future runs should they be necessary again::
 
-    def parse_details(self, response):
-        item = response.meta.get('item', None)
+    def parse_details(self, response, item=None):
         if item:
             # populate more `item` fields
             return item
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 1a5d52487..9fd311c69 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -81,7 +81,8 @@ So, for example, this won't work::
 
     def some_callback(self, response):
         somearg = 'test'
-        return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg))
+        return scrapy.Request('http://www.example.com',
+                              callback=lambda r: self.other_callback(r, somearg))
 
     def other_callback(self, response, somearg):
         print("the argument passed is: %s" % somearg)
@@ -90,10 +91,10 @@ But this will::
 
     def some_callback(self, response):
         somearg = 'test'
-        return scrapy.Request('http://www.example.com', callback=self.other_callback, meta={'somearg': somearg})
+        return scrapy.Request('http://www.example.com',
+                              callback=self.other_callback, cb_kwargs={'somearg': somearg})
 
-    def other_callback(self, response):
-        somearg = response.meta['somearg']
+    def other_callback(self, response, somearg):
         print("the argument passed is: %s" % somearg)
 
 If you wish to log the requests that couldn't be serialized, you can set the
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index af14d14e8..8278e9849 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -27,10 +27,11 @@ Common causes of memory leaks
 
 It happens quite often (sometimes by accident, sometimes on purpose) that the
 Scrapy developer passes objects referenced in Requests (for example, using the
-:attr:`~scrapy.http.Request.meta` attribute or the request callback function)
-and that effectively bounds the lifetime of those referenced objects to the
-lifetime of the Request. This is, by far, the most common cause of memory leaks
-in Scrapy projects, and a quite difficult one to debug for newcomers.
+:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta`
+attributes or the request callback function) and that effectively bounds the
+lifetime of those referenced objects to the lifetime of the Request. This is,
+by far, the most common cause of memory leaks in Scrapy projects, and a quite
+difficult one to debug for newcomers.
 
 In big projects, the spiders are typically written by different people and some
 of those spiders could be "leaking" and thus affecting the rest of the other
@@ -48,7 +49,8 @@ Too Many Requests?
 
 By default Scrapy keeps the request queue in memory; it includes
 :class:`~scrapy.http.Request` objects and all objects
-referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.meta`).
+referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs`
+and :attr:`~scrapy.http.Request.meta`).
 While not necessarily a leak, this can take a lot of memory. Enabling
 :ref:`persistent job queue <topics-jobs>` could help keeping memory usage
 in control.
@@ -101,7 +103,7 @@ Let's see a concrete example of a hypothetical case of memory leaks.
 Suppose we have some spider with a line similar to this one::
 
     return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id,
-        callback=self.parse, meta={referer: response})
+                   callback=self.parse, cb_kwargs={'referer': response})
 
 That line is passing a response reference inside a request which effectively
 ties the response lifetime to the requests' one, and that would definitely
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 05ca8d6c1..f299c2cff 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -186,12 +186,12 @@ Request objects
        Return a new Request which is a copy of this Request. See also:
        :ref:`topics-request-response-ref-request-callback-arguments`.
 
-    .. method:: Request.replace([url, method, headers, body, cookies, meta, encoding, dont_filter, callback, errback])
+    .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs])
 
        Return a Request object with the same members, except for those members
        given new values by whichever keyword arguments are specified. The
-       attribute :attr:`Request.meta` is copied by default (unless a new value
-       is given in the ``meta`` argument). See also
+       :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are copied by default
+       (unless new values are given as arguments). See also
        :ref:`topics-request-response-ref-request-callback-arguments`.
 
 .. _topics-request-response-ref-request-callback-arguments:
@@ -237,11 +237,10 @@ The following example shows how to achieve this by using the
 
 .. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``.
    Prior to that, :attr:`Request.meta` was the recommended option for passing
-   information around callbacks. However, after ``1.7`` :attr:`Request.cb_kwargs`
+   information around callbacks. However, after ``1.7``, using :attr:`Request.cb_kwargs`
    became the preferred way of passing user information, leaving :attr:`Request.meta`
-   to be used by internal components like spider or downloader middlewares.
-   The following example, which uses :attr:`Request.meta`, is only kept for historical
-   reasons.
+   to be populated by internal components like spider or downloader middlewares.
+   The following :attr:`Request.meta` example is only kept for historical reasons.
 
 ::
 

From 46b9ab0c58354deb1045c20f3bc061526d69f356 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 29 Mar 2019 10:28:36 +0000
Subject: [PATCH 2000/2974] Update docs/topics/broad-crawls.rst

Co-Authored-By: whalebot-helmsman <whalebot.helmsman@gmail.com>
---
 docs/topics/broad-crawls.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index 64c8883b1..68a24a4d2 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -42,7 +42,7 @@ efficient broad crawl.
 Use proper :setting:`SCHEDULER_PRIORITY_QUEUE`
 ==============================================
 
-Default scrapy's scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``.
+Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``.
 It works best during single domain crawl. And it does not work well with crawling
 many different domains in parallel
 

From e3df6be360a58f016e31d5bfa2e04cd2e5d1965b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 29 Mar 2019 10:28:52 +0000
Subject: [PATCH 2001/2974] Update docs/topics/broad-crawls.rst

Co-Authored-By: whalebot-helmsman <whalebot.helmsman@gmail.com>
---
 docs/topics/broad-crawls.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index 68a24a4d2..b149d7f4a 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -43,7 +43,7 @@ Use proper :setting:`SCHEDULER_PRIORITY_QUEUE`
 ==============================================
 
 Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``.
-It works best during single domain crawl. And it does not work well with crawling
+It works best during single-domain crawl. It does not work well with crawling
 many different domains in parallel
 
 To apply recommended priority queue use::

From bd228f1d962c7f4759536d8cda278857de7d5234 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 29 Mar 2019 10:29:04 +0000
Subject: [PATCH 2002/2974] Update docs/topics/broad-crawls.rst

Co-Authored-By: whalebot-helmsman <whalebot.helmsman@gmail.com>
---
 docs/topics/broad-crawls.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index b149d7f4a..a01f28248 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -46,7 +46,7 @@ Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQ
 It works best during single-domain crawl. It does not work well with crawling
 many different domains in parallel
 
-To apply recommended priority queue use::
+To apply the recommended priority queue use::
 
     SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
 

From 1ee99e1f4240af6a7a72fe7c58b89d7bce1cd09e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 29 Mar 2019 10:29:15 +0000
Subject: [PATCH 2003/2974] Update docs/topics/settings.rst

Co-Authored-By: whalebot-helmsman <whalebot.helmsman@gmail.com>
---
 docs/topics/settings.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index ed94146f4..4a5439bfc 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1157,7 +1157,7 @@ SCHEDULER_PRIORITY_QUEUE
 ------------------------
 Default: ``'scrapy.pqueues.ScrapyPriorityQueue'``
 
-Type of priority queue used by scheduler. Another available type is
+Type of priority queue used by the scheduler. Another available type is
 ``scrapy.pqueues.DownloaderAwarePriorityQueue``.
 ``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than
 ``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different

From 2b4bcfaf494073520e84bbf301d5141a2e19a3e6 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 29 Mar 2019 10:30:26 +0000
Subject: [PATCH 2004/2974] remove comment

---
 scrapy/core/scheduler.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index c385fafe1..9d0258db2 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -57,7 +57,6 @@ class Scheduler(object):
         dupefilter = create_instance(dupefilter_cls, settings, crawler)
         pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
         if pqclass is PriorityQueue:
-            # backwards compatibility
             warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'"
                           " is no longer supported because of API changes; "
                           "please use 'scrapy.pqueues.ScrapyPriorityQueue'",

From 554d8728227a9ea96e5ea3a8a4fd782d42fdbd66 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 29 Mar 2019 10:31:15 +0000
Subject: [PATCH 2005/2974] remove spacing

---
 scrapy/core/scheduler.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 9d0258db2..d87d2ffdc 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -77,13 +77,8 @@ class Scheduler(object):
 
     def open(self, spider):
         self.spider = spider
-
-        # in-memory PriorityQueue instance
         self.mqs = self._mq()
-
-        # on-disk PriorityQueue instance
         self.dqs = self._dq() if self.dqdir else None
-
         return self.df.open()
 
     def close(self, reason):

From f08f841d0bebd097358889c0c98f83f051828f15 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 29 Mar 2019 10:35:49 +0000
Subject: [PATCH 2006/2974] remove small single use method

---
 scrapy/core/scheduler.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index d87d2ffdc..975aede0c 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -101,7 +101,7 @@ class Scheduler(object):
         return True
 
     def next_request(self):
-        request = self._mqpop()
+        request = self.mqs.pop()
         if request:
             self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider)
         else:
@@ -141,9 +141,6 @@ class Scheduler(object):
         if self.dqs:
             return self.dqs.pop()
 
-    def _mqpop(self):
-        return self.mqs.pop()
-
     def _newmq(self, priority):
         """ Factory for creating memory queues. """
         return self.mqclass()

From ef743983a98ae0891abf9aca4c9b19cb44861c49 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 29 Mar 2019 10:38:13 +0000
Subject: [PATCH 2007/2974] change wording

---
 docs/topics/broad-crawls.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index a01f28248..6e50c0bc7 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -39,7 +39,7 @@ you need to keep in mind when using Scrapy for doing broad crawls, along with
 concrete suggestions of Scrapy settings to tune in order to achieve an
 efficient broad crawl.
 
-Use proper :setting:`SCHEDULER_PRIORITY_QUEUE`
+Use the right :setting:`SCHEDULER_PRIORITY_QUEUE`
 ==============================================
 
 Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``.
@@ -96,7 +96,7 @@ When doing broad crawls you are often only interested in the crawl rates you
 get and any errors found. These stats are reported by Scrapy when using the
 ``INFO`` log level. In order to save CPU (and log storage requirements) you
 should not use ``DEBUG`` log level when preforming large broad crawls in
-production. Using ``DEBUG`` level when developing your (broad) crawler may be 
+production. Using ``DEBUG`` level when developing your (broad) crawler may be
 fine though.
 
 To set the log level use::

From 1c6733454e14a3c237ed602b65ae5e0a8a78dee5 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 29 Mar 2019 10:44:55 +0000
Subject: [PATCH 2008/2974] added underlines

---
 docs/topics/broad-crawls.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index 6e50c0bc7..b887b98af 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -40,7 +40,7 @@ concrete suggestions of Scrapy settings to tune in order to achieve an
 efficient broad crawl.
 
 Use the right :setting:`SCHEDULER_PRIORITY_QUEUE`
-==============================================
+=================================================
 
 Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``.
 It works best during single-domain crawl. It does not work well with crawling

From f5e0b6b89ace437af850e0225651329101a59862 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 29 Mar 2019 14:03:26 -0300
Subject: [PATCH 2009/2974] parse command: rename cb_kwargs option to cbkwargs

---
 scrapy/commands/parse.py    | 18 +++++++++---------
 tests/test_command_parse.py |  2 +-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 2486f3f23..e948d6406 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -51,8 +51,8 @@ class Command(ScrapyCommand):
             help="use this callback for parsing, instead looking for a callback")
         parser.add_option("-m", "--meta", dest="meta",
             help="inject extra meta into the Request, it must be a valid raw json string")
-        parser.add_option("--cb_kwargs", dest="cb_kwargs",
-            help="inject extra cb_kwargs into the Request, it must be a valid raw json string")
+        parser.add_option("--cbkwargs", dest="cbkwargs",
+            help="inject extra cbkwargs into the Request, it must be a valid raw json string")
         parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
             help="maximum depth for parsing requests [default: %default]")
         parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
@@ -212,9 +212,9 @@ class Command(ScrapyCommand):
         if opts.meta:
             request.meta.update(opts.meta)
 
-        # update cb_kwargs if any extra cb_kwargs was passed through the --cb_kwargs option.
-        if opts.cb_kwargs:
-            request.cb_kwargs.update(opts.cb_kwargs)
+        # update cb_kwargs if any extra values were was passed through the --cbkwargs option.
+        if opts.cbkwargs:
+            request.cb_kwargs.update(opts.cbkwargs)
 
         request.meta['_depth'] = 1
         request.meta['_callback'] = request.callback
@@ -243,12 +243,12 @@ class Command(ScrapyCommand):
                                  "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
 
     def process_request_cb_kwargs(self, opts):
-        if opts.cb_kwargs:
+        if opts.cbkwargs:
             try:
-                opts.cb_kwargs = json.loads(opts.cb_kwargs)
+                opts.cbkwargs = json.loads(opts.cbkwargs)
             except ValueError:
-                raise UsageError("Invalid --cb_kwargs value, pass a valid json string to --cb_kwargs. "
-                                 "Example: --cb_kwargs='{\"foo\" : \"bar\"}'", print_help=False)
+                raise UsageError("Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
+                                 "Example: --cbkwargs='{\"foo\" : \"bar\"}'", print_help=False)
 
     def run(self, args, opts):
         # parse arguments
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 1404005fb..c18a6ce9f 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -130,7 +130,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
     def test_request_with_cb_kwargs(self):
         raw_json_string = '{"foo" : "bar", "key": "value"}'
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
-                                           '--cb_kwargs', raw_json_string,
+                                           '--cbkwargs', raw_json_string,
                                            '-c', 'parse_request_with_cb_kwargs',
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))

From ccb56a317ee249978496e918ee5b74d83c2d7199 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 29 Mar 2019 14:12:26 -0300
Subject: [PATCH 2010/2974] Update docs about cb_kwargs and meta

---
 docs/topics/request-response.rst | 30 ++++++------------------------
 1 file changed, 6 insertions(+), 24 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index f299c2cff..4e81ce878 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -190,8 +190,8 @@ Request objects
 
        Return a Request object with the same members, except for those members
        given new values by whichever keyword arguments are specified. The
-       :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are copied by default
-       (unless new values are given as arguments). See also
+       :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are shallow
+       copied by default (unless new values are given as arguments). See also
        :ref:`topics-request-response-ref-request-callback-arguments`.
 
 .. _topics-request-response-ref-request-callback-arguments:
@@ -220,7 +220,6 @@ The following example shows how to achieve this by using the
 
 ::
 
-    # pass information to the next callback using the Request.cb_kwargs attribute
     def parse(self, response):
         request = scrapy.Request('http://www.example.com/index.html',
                                  callback=self.parse_page2,
@@ -236,27 +235,10 @@ The following example shows how to achieve this by using the
         )
 
 .. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``.
-   Prior to that, :attr:`Request.meta` was the recommended option for passing
-   information around callbacks. However, after ``1.7``, using :attr:`Request.cb_kwargs`
-   became the preferred way of passing user information, leaving :attr:`Request.meta`
-   to be populated by internal components like spider or downloader middlewares.
-   The following :attr:`Request.meta` example is only kept for historical reasons.
-
-::
-
-    # pass information to the next callback using the Request.meta attribute
-    def parse_page1(self, response):
-        item = MyItem()
-        item['main_url'] = response.url
-        request = scrapy.Request("http://www.example.com/some_page.html",
-                                 callback=self.parse_page2)
-        request.meta['item'] = item
-        yield request
-
-    def parse_page2(self, response):
-        item = response.meta['item']
-        item['other_url'] = response.url
-        yield item
+   Prior to that, using :attr:`Request.meta` was recommended for passing
+   information around callbacks. After ``1.7``, :attr:`Request.cb_kwargs`
+   became the preferred way for handling user information, leaving :attr:`Request.meta`
+   for communication with components like middlewares and extensions.
 
 .. _topics-request-response-ref-errbacks:
 

From 294ef51bb24782a0527892ea93bb4876daa7ca50 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 29 Mar 2019 16:12:55 -0300
Subject: [PATCH 2011/2974] parse command: update docs about passing callback
 keyword arguments

---
 docs/topics/commands.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 6644d65e4..a93bee06b 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -461,8 +461,8 @@ Supported options:
 * ``--meta`` or ``-m``: additional request meta that will be passed to the callback 
   request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
 
-* ``--cb_kwargs``: additional keyword arguments that will be passed to the callback.
-  This must be a valid json string. Example: --cb_kwargs='{"foo" : "bar"}'
+* ``--cbkwargs``: additional keyword arguments that will be passed to the callback.
+  This must be a valid json string. Example: --cbkwargs='{"foo" : "bar"}'
 
 * ``--pipelines``: process items through pipelines
 

From 0522fe35c334141e90741644fec368cdbd12044e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 29 Mar 2019 16:15:34 -0300
Subject: [PATCH 2012/2974] parse command: improve option description

---
 scrapy/commands/parse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index e948d6406..d4f2234b0 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -52,7 +52,7 @@ class Command(ScrapyCommand):
         parser.add_option("-m", "--meta", dest="meta",
             help="inject extra meta into the Request, it must be a valid raw json string")
         parser.add_option("--cbkwargs", dest="cbkwargs",
-            help="inject extra cbkwargs into the Request, it must be a valid raw json string")
+            help="inject extra callback kwargs into the Request, it must be a valid raw json string")
         parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
             help="maximum depth for parsing requests [default: %default]")
         parser.add_option("-v", "--verbose", dest="verbose", action="store_true",

From 07ff9248a5fd2eac4f53da92766dcb5a7ca48569 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 1 Apr 2019 12:31:26 -0300
Subject: [PATCH 2013/2974] [Docs] CrawlSpider: add note about link text

---
 docs/topics/spiders.rst | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 8c4049f85..5417ef129 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -377,7 +377,10 @@ Crawling rules
 .. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None)
 
    ``link_extractor`` is a :ref:`Link Extractor <topics-link-extractors>` object which
-   defines how links will be extracted from each crawled page.
+   defines how links will be extracted from each crawled page. Each produced link will
+   be used to generate a :class:`~scrapy.http.Request` object, which will contain the
+   link's text in its ``meta`` attribute.
+   The link text can be accessed from the callback method though ``response.meta['link_text']``
 
    ``callback`` is a callable or a string (in which case a method from the spider
    object with that name will be used) to be called for each link extracted with
@@ -438,6 +441,7 @@ Let's now take a look at an example CrawlSpider with rules::
             item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
             item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
             item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
+            item['link_text'] = response.meta['link_text']
             return item
 
 

From 8ebbc731b2bdf8e2a2b5a2f0673da838369f31b5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 1 Apr 2019 16:15:03 -0300
Subject: [PATCH 2014/2974] [Docs] Rephrase Rule docs

---
 docs/topics/spiders.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 5417ef129..7290bb844 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -380,13 +380,13 @@ Crawling rules
    defines how links will be extracted from each crawled page. Each produced link will
    be used to generate a :class:`~scrapy.http.Request` object, which will contain the
    link's text in its ``meta`` attribute.
-   The link text can be accessed from the callback method though ``response.meta['link_text']``
 
    ``callback`` is a callable or a string (in which case a method from the spider
    object with that name will be used) to be called for each link extracted with
-   the specified link_extractor. This callback receives a response as its first
-   argument and must return a list containing :class:`~scrapy.item.Item` and/or
-   :class:`~scrapy.http.Request` objects (or any subclass of them).
+   the specified link extractor. This callback receives a :class:`~scrapy.http.Response`
+   as its first argument and must return either a single instance or an iterable of
+   :class:`~scrapy.item.Item`, ``dict`` and/or :class:`~scrapy.http.Request` objects
+   (or any subclass of them).
 
    .. warning:: When writing crawl spider rules, avoid using ``parse`` as
        callback, since the :class:`CrawlSpider` uses the ``parse`` method

From 7a38623cecc6c60d7ffe14c75d1fe679bb04b774 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 1 Apr 2019 17:09:49 -0300
Subject: [PATCH 2015/2974] [Docs] Clarify comment about meta dictionary

---
 docs/topics/spiders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 7290bb844..3cd051cdf 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -379,7 +379,7 @@ Crawling rules
    ``link_extractor`` is a :ref:`Link Extractor <topics-link-extractors>` object which
    defines how links will be extracted from each crawled page. Each produced link will
    be used to generate a :class:`~scrapy.http.Request` object, which will contain the
-   link's text in its ``meta`` attribute.
+   link's text in its ``meta`` dictionary (under the ``link_text`` key).
 
    ``callback`` is a callable or a string (in which case a method from the spider
    object with that name will be used) to be called for each link extracted with

From 7acf4eec792f155a8b0e92c3bb1efa1ff5882ac8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 2 Apr 2019 18:36:03 +0200
Subject: [PATCH 2016/2974] Deprecate the scrapy.utils.gz.is_gzipped function

---
 scrapy/utils/gz.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index ec3949651..b3fb16b1e 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -9,6 +9,9 @@ from gzip import GzipFile
 import six
 import re
 
+from scrapy.utils.decorators import deprecated
+
+
 # - Python>=3.5 GzipFile's read() has issues returning leftover
 #   uncompressed data when input is corrupted
 #   (regression or bug-fix compared to Python 3.4)
@@ -53,6 +56,7 @@ def gunzip(data):
 _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
 _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search
 
+@deprecated
 def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
     ctype = response.headers.get('Content-Type', b'')

From 6336e1d1f31da8611a0b63dcb536529e9027d51b Mon Sep 17 00:00:00 2001
From: float13 <43447704+float13@users.noreply.github.com>
Date: Fri, 5 Apr 2019 00:54:46 -0400
Subject: [PATCH 2017/2974] grammar fix - delete unneeded apostrophe in "lets"

---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 41e61542a..b2f952fe2 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -511,7 +511,7 @@ We can try extracting it in the shell::
     '<a href="/page/2/">Next <span aria-hidden="true">→</span></a>'
 
 This gets the anchor element, but we want the attribute ``href``. For that,
-Scrapy supports a CSS extension that let's you select the attribute contents,
+Scrapy supports a CSS extension that lets you select the attribute contents,
 like this::
 
     >>> response.css('li.next a::attr(href)').get()

From d711ecfc18a01084f74bb2b9dc01c8bcb4772580 Mon Sep 17 00:00:00 2001
From: float13 <43447704+float13@users.noreply.github.com>
Date: Fri, 5 Apr 2019 00:56:51 -0400
Subject: [PATCH 2018/2974] grammar fix - delete extra word "shell"

---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index b2f952fe2..fc10adbe1 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -205,7 +205,7 @@ Extracting data
 ---------------
 
 The best way to learn how to extract data with Scrapy is trying selectors
-using the shell :ref:`Scrapy shell <topics-shell>`. Run::
+using the :ref:`Scrapy shell <topics-shell>`. Run::
 
     scrapy shell 'http://quotes.toscrape.com/page/1/'
 

From 77e3695686d9a46841778248f4c1a2da336b54f7 Mon Sep 17 00:00:00 2001
From: float13 <43447704+float13@users.noreply.github.com>
Date: Fri, 5 Apr 2019 01:04:59 -0400
Subject: [PATCH 2019/2974] grammar fix - add apostrophe-s to browser

---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index fc10adbe1..a97f96801 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -296,7 +296,7 @@ expressions`_::
 
 In order to find the proper CSS selectors to use, you might find useful opening
 the response page from the shell in your web browser using ``view(response)``.
-You can use your browser developer tools to inspect the HTML and come up
+You can use your browser's developer tools to inspect the HTML and come up
 with a selector (see section about :ref:`topics-developer-tools`).
 
 `Selector Gadget`_ is also a nice tool to quickly find CSS selector for

From a101d5fe5c215bfbb09732fb506d4cd016624d80 Mon Sep 17 00:00:00 2001
From: float13 <43447704+float13@users.noreply.github.com>
Date: Fri, 5 Apr 2019 01:12:20 -0400
Subject: [PATCH 2020/2974] text edit - delete 2 extra words

---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index a97f96801..8bd2d27dd 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -297,7 +297,7 @@ expressions`_::
 In order to find the proper CSS selectors to use, you might find useful opening
 the response page from the shell in your web browser using ``view(response)``.
 You can use your browser's developer tools to inspect the HTML and come up
-with a selector (see section about :ref:`topics-developer-tools`).
+with a selector (see :ref:`topics-developer-tools`).
 
 `Selector Gadget`_ is also a nice tool to quickly find CSS selector for
 visually selected elements, which works in many browsers.

From 3a493b60661760b26ebc9dd2f4c5c7e4b8df93c9 Mon Sep 17 00:00:00 2001
From: Kostis Anagnostopoulos <ankostis@gmail.com>
Date: Fri, 5 Apr 2019 11:52:00 +0200
Subject: [PATCH 2021/2974] fix: do not catch system exceptions like
 KeyboardInterrupt

---
 scrapy/contracts/__init__.py | 2 +-
 scrapy/core/spidermw.py      | 2 +-
 scrapy/utils/defer.py        | 4 ++--
 scrapy/utils/misc.py         | 2 +-
 tests/mockserver.py          | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 259220a72..536bbdafb 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -94,7 +94,7 @@ class ContractsManager(object):
             try:
                 output = cb(response)
                 output = list(iterate_spider_output(output))
-            except:
+            except Exception:
                 case = _create_testcase(method, 'callback')
                 results.addError(case, sys.exc_info())
 
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index e07f76bdf..b5f9837ff 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -49,7 +49,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                                              .format(fname(method), type(result)))
                 except _InvalidOutput:
                     raise
-                except:
+                except Exception:
                     return scrape_func(Failure(), request, spider)
             return scrape_func(response, request, spider)
 
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index bcf209511..69d621830 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -48,7 +48,7 @@ def mustbe_deferred(f, *args, **kw):
     # exception in Scrapy - see #125
     except IgnoreRequest as e:
         return defer_fail(failure.Failure(e))
-    except:
+    except Exception:
         return defer_fail(failure.Failure())
     else:
         return defer_result(result)
@@ -102,5 +102,5 @@ def iter_errback(iterable, errback, *a, **kw):
             yield next(it)
         except StopIteration:
             break
-        except:
+        except Exception:
             errback(failure.Failure(), *a, **kw)
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 6de36d45c..ddaa7f7bf 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -86,7 +86,7 @@ def extract_regex(regex, text, encoding='utf-8'):
 
     try:
         strings = [regex.search(text).group('extract')]   # named group
-    except:
+    except Exception:
         strings = regex.findall(text)    # full regex or numbered groups
     strings = flatten(strings)
 
diff --git a/tests/mockserver.py b/tests/mockserver.py
index bf62fe907..3fa4bc0f0 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -177,7 +177,7 @@ class Root(Resource):
         try:
             from tests import tests_datadir
             self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/')))
-        except:
+        except Exception:
             pass
         self.putChild(b"redirect-to", RedirectTo())
 

From 35ce92a4199b29a30e58026880b45002a1e4591e Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Fri, 5 Apr 2019 11:43:21 -0300
Subject: [PATCH 2022/2974] fix typo (Response -> Request)

check docs for more information https://github.com/scrapy/scrapy/blob/b5c552d17ff9e9629434712c3d0595c02853bcfc/docs/topics/spider-middleware.rst
---
 scrapy/templates/project/module/middlewares.py.tmpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
index 5debe1cd2..97b5db2e1 100644
--- a/scrapy/templates/project/module/middlewares.py.tmpl
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -39,7 +39,7 @@ class ${ProjectName}SpiderMiddleware(object):
         # Called when a spider or process_spider_input() method
         # (from other spider middleware) raises an exception.
 
-        # Should return either None or an iterable of Response, dict
+        # Should return either None or an iterable of Request, dict
         # or Item objects.
         pass
 

From a8f83ab9675ec4f0bcb90b2eb3f06c593f32732f Mon Sep 17 00:00:00 2001
From: Kostis Anagnostopoulos <ankostis@gmail.com>
Date: Sat, 6 Apr 2019 14:58:32 +0200
Subject: [PATCH 2023/2974] doc: document LOGSTATS_INTERVAL setting

---
 docs/topics/settings.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 062c4b2ca..145dcc136 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -897,6 +897,16 @@ Default: ``False``
 If ``True``, the logs will just contain the root path. If it is set to ``False``
 then it displays the component responsible for the log output
 
+.. setting:: LOGSTATS_INTERVAL
+
+LOGSTATS_INTERVAL
+-----------------
+
+Default: ``60.0``
+
+The interval (in seconds) between each logging printout of the stats 
+by :class:`~extensions.logstats.LogStats`.
+
 .. setting:: MEMDEBUG_ENABLED
 
 MEMDEBUG_ENABLED

From aa46e1995cd5cb1099aba17535372b538bd656b3 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Sun, 7 Apr 2019 00:33:40 +0530
Subject: [PATCH 2024/2974] [MRG+1] Show elapsed time in statscollector (#3638)

* Update corestats.py

* Update corestats.py

* corrected tests

* Update corestats.py

* Update scrapy/extensions/corestats.py
---
 scrapy/extensions/corestats.py | 6 +++++-
 tests/test_closespider.py      | 6 +-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 3d9a307b7..8cc5e18ac 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -24,7 +24,11 @@ class CoreStats(object):
         self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
 
     def spider_closed(self, spider, reason):
-        self.stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider)
+        finish_time = datetime.datetime.utcnow()
+        elapsed_time = finish_time - self.stats.get_value('start_time')
+        elapsed_time_seconds = elapsed_time.total_seconds()
+        self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)
+        self.stats.set_value('finish_time', finish_time, spider=spider)
         self.stats.set_value('finish_reason', reason, spider=spider)
 
     def item_scraped(self, item, spider):
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 0eb1b7944..4a56425b7 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -53,9 +53,5 @@ class TestCloseSpider(TestCase):
         yield crawler.crawl(total=1000000, mockserver=self.mockserver)
         reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_timeout')
-        stats = crawler.stats
-        start = stats.get_value('start_time')
-        stop = stats.get_value('finish_time')
-        diff = stop - start
-        total_seconds = diff.seconds + diff.microseconds
+        total_seconds = crawler.stats.get_value('elapsed_time_seconds')
         self.assertTrue(total_seconds >= close_on)

From e6048d55f905ca0be2b32f9b566d257f34752c71 Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Tue, 9 Apr 2019 17:34:20 +0530
Subject: [PATCH 2025/2974] changes parameter name

---
 scrapy/extensions/httpcache.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 1b5e05b1b..35c77add8 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -31,7 +31,7 @@ class DummyPolicy(object):
     def should_cache_response(self, response, request):
         return response.status not in self.ignore_http_codes
 
-    def is_cached_response_fresh(self, response, request):
+    def is_cached_response_fresh(self, cachedresponse, request):
         return True
 
     def is_cached_response_valid(self, cachedresponse, response, request):

From 4cfdc14974313f23d7bb8be9311195f9cfa74968 Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Tue, 9 Apr 2019 17:52:02 +0530
Subject: [PATCH 2026/2974] fixes a link in comment

---
 scrapy/extensions/httpcache.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 1b5e05b1b..03bba1530 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -70,7 +70,7 @@ class RFC2616Policy(object):
         return True
 
     def should_cache_response(self, response, request):
-        # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1
+        # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
         # Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4
         # Status code 206 is not included because cache can not deal with partial contents
         cc = self._parse_cachecontrol(response)

From d27c2c68ba4f201c18255a6ec8a735f140b13773 Mon Sep 17 00:00:00 2001
From: Matthijs Vos <vos1996@gmail.com>
Date: Wed, 10 Apr 2019 12:56:50 +0200
Subject: [PATCH 2027/2974] Wrap scrapy check in environment

---
 scrapy/commands/check.py | 21 +++++++++++----------
 scrapy/utils/misc.py     | 27 +++++++++++++++++++++++----
 2 files changed, 34 insertions(+), 14 deletions(-)

diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index b8a9ef989..b29587fa7 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -6,7 +6,7 @@ from unittest import TextTestRunner, TextTestResult as _TextTestResult
 
 from scrapy.commands import ScrapyCommand
 from scrapy.contracts import ContractsManager
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, set_environ
 from scrapy.utils.conf import build_component_list
 
 
@@ -68,16 +68,17 @@ class Command(ScrapyCommand):
 
         spider_loader = self.crawler_process.spider_loader
 
-        for spidername in args or spider_loader.list():
-            spidercls = spider_loader.load(spidername)
-            spidercls.start_requests = lambda s: conman.from_spider(s, result)
+        with set_environ(SCRAPY_CHECK=True):
+            for spidername in args or spider_loader.list():
+                spidercls = spider_loader.load(spidername)
+                spidercls.start_requests = lambda s: conman.from_spider(s, result)
 
-            tested_methods = conman.tested_methods_from_spidercls(spidercls)
-            if opts.list:
-                for method in tested_methods:
-                    contract_reqs[spidercls.name].append(method)
-            elif tested_methods:
-                self.crawler_process.crawl(spidercls)
+                tested_methods = conman.tested_methods_from_spidercls(spidercls)
+                if opts.list:
+                    for method in tested_methods:
+                        contract_reqs[spidercls.name].append(method)
+                elif tested_methods:
+                    self.crawler_process.crawl(spidercls)
 
         # start checks
         if opts.list:
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index ddaa7f7bf..7a2cd18ea 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -1,6 +1,8 @@
 """Helper functions which don't fit anywhere else"""
+import os
 import re
 import hashlib
+from contextlib import contextmanager
 from importlib import import_module
 from pkgutil import iter_modules
 
@@ -10,7 +12,6 @@ from w3lib.html import replace_entities
 from scrapy.utils.python import flatten, to_unicode
 from scrapy.item import BaseItem
 
-
 _ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes
 
 
@@ -40,7 +41,7 @@ def load_object(path):
     except ValueError:
         raise ValueError("Error loading object '%s': not a full path" % path)
 
-    module, name = path[:dot], path[dot+1:]
+    module, name = path[:dot], path[dot + 1:]
     mod = import_module(module)
 
     try:
@@ -85,9 +86,9 @@ def extract_regex(regex, text, encoding='utf-8'):
         regex = re.compile(regex, re.UNICODE)
 
     try:
-        strings = [regex.search(text).group('extract')]   # named group
+        strings = [regex.search(text).group('extract')]  # named group
     except Exception:
-        strings = regex.findall(text)    # full regex or numbered groups
+        strings = regex.findall(text)  # full regex or numbered groups
     strings = flatten(strings)
 
     if isinstance(text, six.text_type):
@@ -142,3 +143,21 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
         return objcls.from_settings(settings, *args, **kwargs)
     else:
         return objcls(*args, **kwargs)
+
+
+@contextmanager
+def set_environ(**kwargs):
+    """Temporarily set environment variables inside the context manager and
+    fully restore previous environment afterwards
+    """
+
+    original_env = {k: os.environ.get(k) for k in kwargs}
+    os.environ.update(kwargs)
+    try:
+        yield
+    finally:
+        for k, v in original_env:
+            if v is None:
+                del os.environ[k]
+            else:
+                os.environ[k] = v

From 50730ed2280dec6384986d34999cd277d7568ff9 Mon Sep 17 00:00:00 2001
From: Matthijs Vos <vos1996@gmail.com>
Date: Wed, 10 Apr 2019 13:01:01 +0200
Subject: [PATCH 2028/2974] Try it with a string

---
 scrapy/commands/check.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index b29587fa7..ab73e85e7 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -68,7 +68,7 @@ class Command(ScrapyCommand):
 
         spider_loader = self.crawler_process.spider_loader
 
-        with set_environ(SCRAPY_CHECK=True):
+        with set_environ(SCRAPY_CHECK='true'):
             for spidername in args or spider_loader.list():
                 spidercls = spider_loader.load(spidername)
                 spidercls.start_requests = lambda s: conman.from_spider(s, result)

From 07adca34e1378b11dff9e3f11d3760c54f5fa1ef Mon Sep 17 00:00:00 2001
From: Matthijs Vos <vos1996@gmail.com>
Date: Wed, 10 Apr 2019 13:01:46 +0200
Subject: [PATCH 2029/2974] Fix 'Too many values to unpack'

---
 scrapy/utils/misc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 7a2cd18ea..cdd5a11c9 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -156,7 +156,7 @@ def set_environ(**kwargs):
     try:
         yield
     finally:
-        for k, v in original_env:
+        for k, v in original_env.items():
             if v is None:
                 del os.environ[k]
             else:

From fbb42fe14ed23aaba37b42d12c7adbf513f9089e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 17 Apr 2019 08:25:22 +0200
Subject: [PATCH 2030/2974] Cover PEP 257 in the documentation policies

---
 docs/contributing.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index aac0f4496..c31a17609 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -167,8 +167,9 @@ Documentation policies
 
 For reference documentation of API members (classes, methods, etc.) use
 docstrings and make sure that the Sphinx documentation uses the autodoc_
-extension to pull the docstrings. API reference documentation should be
-IDE-friendly: short, to the point, and it may provide short examples.
+extension to pull the docstrings. API reference documentation should follow
+docstring conventions (`PEP 257`_) and be IDE-friendly: short, to the point,
+and it may provide short examples.
 
 Other types of documentation, such as tutorials or topics, should be covered in
 files within the ``docs/`` directory. This includes documentation that is
@@ -237,5 +238,6 @@ And their unit-tests are in::
 .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
 .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
 .. _open issues: https://github.com/scrapy/scrapy/issues
+.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
 .. _pull request: https://help.github.com/send-pull-requests/
 .. _tox: https://pypi.python.org/pypi/tox

From 5a6fb3daa6e6a15effe9377dbcc85e67bec9aec7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 29 Mar 2019 17:10:16 +0100
Subject: [PATCH 2031/2974] Use pytest-xdist

---
 docs/contributing.rst      | 26 ++++++++++++++++++++++++++
 tests/requirements-py2.txt |  5 +++--
 tests/requirements-py3.txt |  5 +++--
 3 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index aac0f4496..b462ae331 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -205,6 +205,29 @@ To run a specific test (say ``tests/test_loader.py``) use:
 
     ``tox -- tests/test_loader.py``
 
+To run the tests on a specific tox_ environment, use ``-e <name>`` with an
+environment name from ``tox.ini``. For example, to run the tests with Python
+3.6 use::
+
+    tox -e py36
+
+You can also specify a comma-separated list of environmets, and use `tox’s
+parallel mode`_ to run the tests on multiple environments in parallel::
+
+    tox -e py27,py36 -p auto
+
+To pass command-line options to pytest_, add them after ``--`` in your call to
+tox_. Using ``--`` overrides the default positional arguments defined in
+``tox.ini``, so you must include those default positional arguments
+(``scrapy tests``) after ``--`` as well::
+
+    tox -- scrapy tests -x  # stop after first failure
+
+You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
+the Python 3.6 tox_ environment using all your CPU cores::
+
+    tox -e py36 -- scrapy tests -n auto
+
 To see coverage report install `coverage`_ (``pip install coverage``) and run:
 
     ``coverage report``
@@ -238,4 +261,7 @@ And their unit-tests are in::
 .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
 .. _open issues: https://github.com/scrapy/scrapy/issues
 .. _pull request: https://help.github.com/send-pull-requests/
+.. _pytest: https://docs.pytest.org/en/latest/usage.html
+.. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html
 .. _tox: https://pypi.python.org/pypi/tox
+.. _tox’s parallel mode: https://tox.readthedocs.io/en/latest/example/basic.html#parallel-mode
diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txt
index 790f29d34..be809b151 100644
--- a/tests/requirements-py2.txt
+++ b/tests/requirements-py2.txt
@@ -2,9 +2,10 @@
 mock
 mitmproxy==0.10.1
 netlib==0.10.1
-pytest==2.9.2
+pytest
+pytest-cov
 pytest-twisted
-pytest-cov==2.2.1
+pytest-xdist
 jmespath
 brotlipy
 testfixtures
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 7c1aacd81..ed7bf0be0 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,6 +1,7 @@
-pytest==3.6.3
+pytest
+pytest-cov
 pytest-twisted
-pytest-cov==2.5.1
+pytest-xdist
 testfixtures
 jmespath
 leveldb; sys_platform != "win32"

From 29739989478bb1b10467b036df91326071b1ffbc Mon Sep 17 00:00:00 2001
From: Matthijs Vos <vos1996@gmail.com>
Date: Thu, 18 Apr 2019 14:50:02 +0200
Subject: [PATCH 2032/2974] Add set_environ test

---
 tests/test_utils_misc/__init__.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index fcb7772ab..738120a0b 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -3,7 +3,7 @@ import os
 import unittest
 
 from scrapy.item import Item, Field
-from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules
+from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules, set_environ
 
 from tests import mock
 
@@ -130,5 +130,12 @@ class UtilsMiscTestCase(unittest.TestCase):
         with self.assertRaises(ValueError):
             create_instance(m, None, None)
 
+    def test_set_environ(self):
+        assert os.environ.get('some_test_environ') is None
+        with set_environ(some_test_environ='test_value'):
+            assert os.environ.get('some_test_environ') == 'test_value'
+        assert os.environ.get('some_test_environ') is None
+
+
 if __name__ == "__main__":
     unittest.main()

From 6d52708579be05c29c58d6ccc63486f761466d18 Mon Sep 17 00:00:00 2001
From: Matthijs Vos <vos1996@gmail.com>
Date: Thu, 18 Apr 2019 15:19:23 +0200
Subject: [PATCH 2033/2974] Add reset case

---
 tests/test_utils_misc/__init__.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index 738120a0b..eeb995e48 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -136,6 +136,12 @@ class UtilsMiscTestCase(unittest.TestCase):
             assert os.environ.get('some_test_environ') == 'test_value'
         assert os.environ.get('some_test_environ') is None
 
+        os.environ['some_test_environ'] = 'test'
+        assert os.environ.get('some_test_environ') == 'test'
+        with set_environ(some_test_environ='test_value'):
+            assert os.environ.get('some_test_environ') == 'test_value'
+        assert os.environ.get('some_test_environ') == 'test'
+
 
 if __name__ == "__main__":
     unittest.main()

From 935387aaea2f0bdc28504b40dfc9ccbbb437bec4 Mon Sep 17 00:00:00 2001
From: Matthijs Vos <vos1996@gmail.com>
Date: Thu, 18 Apr 2019 22:10:23 +0200
Subject: [PATCH 2034/2974] Revert some non-changes

---
 scrapy/utils/misc.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index cdd5a11c9..f51012e3d 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -41,7 +41,7 @@ def load_object(path):
     except ValueError:
         raise ValueError("Error loading object '%s': not a full path" % path)
 
-    module, name = path[:dot], path[dot + 1:]
+    module, name = path[:dot], path[dot+1:]
     mod = import_module(module)
 
     try:
@@ -86,9 +86,9 @@ def extract_regex(regex, text, encoding='utf-8'):
         regex = re.compile(regex, re.UNICODE)
 
     try:
-        strings = [regex.search(text).group('extract')]  # named group
+        strings = [regex.search(text).group('extract')]   # named group
     except Exception:
-        strings = regex.findall(text)  # full regex or numbered groups
+        strings = regex.findall(text)   # full regex or numbered groups
     strings = flatten(strings)
 
     if isinstance(text, six.text_type):

From 7809c0b14e3ad62aea8e62c7309997ecb64fbbf1 Mon Sep 17 00:00:00 2001
From: Matthijs Vos <vos1996@gmail.com>
Date: Sat, 20 Apr 2019 09:25:01 +0200
Subject: [PATCH 2035/2974] Revert another non-change comment

---
 scrapy/utils/misc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index f51012e3d..b2164d4a8 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -88,7 +88,7 @@ def extract_regex(regex, text, encoding='utf-8'):
     try:
         strings = [regex.search(text).group('extract')]   # named group
     except Exception:
-        strings = regex.findall(text)   # full regex or numbered groups
+        strings = regex.findall(text)    # full regex or numbered groups
     strings = flatten(strings)
 
     if isinstance(text, six.text_type):

From 122ca6211935039825aff900e98bc1fbdb4dc0d6 Mon Sep 17 00:00:00 2001
From: Vandenn <evan.livelo@gmail.com>
Date: Thu, 2 May 2019 23:59:01 +0800
Subject: [PATCH 2036/2974] doc: update configure_logging docs to discourage
 use with CrawlerProcess

---
 docs/topics/logging.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 8e280d929..dea0528db 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -238,9 +238,10 @@ scrapy.utils.log module
 
 .. autofunction:: configure_logging
 
-    ``configure_logging`` is automatically called when using Scrapy commands,
-    but needs to be called explicitly when running custom scripts. In that
-    case, its usage is not required but it's recommended.
+    ``configure_logging`` is automatically called when using Scrapy commands
+    or :class:`~scrapy.crawler.CrawlerProcess`, but needs to be called explicitly
+    when running custom scripts using :class:`~scrapy.crawler.CrawlerRunner`.
+    In that case, its usage is not required but it's recommended.
 
     If you plan on configuring the handlers yourself is still recommended you
     call this function, passing ``install_root_handler=False``. Bear in mind

From 8bd207a2f639216eb51f61ae312dfd22f4b39781 Mon Sep 17 00:00:00 2001
From: Matthijs Vos <vos1996@gmail.com>
Date: Sun, 28 Apr 2019 21:47:47 +0200
Subject: [PATCH 2037/2974] Add documentation

---
 docs/topics/contracts.rst | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 70f20d4ed..3aa32cba8 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -120,3 +120,22 @@ get the failures pretty printed::
             for header in self.args:
                 if header not in response.headers:
                     raise ContractFail('X-CustomHeader not present')
+
+
+Detecting check run
+===================
+It is not encouraged to created different behaviour when running test.
+However, sometimes it is useful to know when a spider is started by scrapy check.
+It can for example be needed to enforce less settings to be set, or to disable some
+uploading of result data. When scrapy check is runned the ``SCRAPY_CHECK`` environment
+variable is set. This can be retrieved via ``os.environ``::
+
+    import os
+    import scrapy
+
+    class ExampleSpider(scrapy.Spider):
+        name = 'example'
+
+        def __init__(self):
+            if os.environ.get('SCRAPY_CHECK'):
+                # Do some scraper adjustments when check is running
\ No newline at end of file

From f6485e669772a940c3c319c71dbcca7bd747d57a Mon Sep 17 00:00:00 2001
From: Matthijs Vos <vos1996@gmail.com>
Date: Fri, 3 May 2019 13:53:45 +0200
Subject: [PATCH 2038/2974] Restore alphabetic order and two lines between
 import and code

---
 scrapy/utils/misc.py              | 1 +
 tests/test_utils_misc/__init__.py | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index b2164d4a8..f638adb25 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -12,6 +12,7 @@ from w3lib.html import replace_entities
 from scrapy.utils.python import flatten, to_unicode
 from scrapy.item import BaseItem
 
+
 _ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes
 
 
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index eeb995e48..e109d5343 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -3,12 +3,13 @@ import os
 import unittest
 
 from scrapy.item import Item, Field
-from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules, set_environ
+from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules
 
 from tests import mock
 
 __doctests__ = ['scrapy.utils.misc']
 
+
 class UtilsMiscTestCase(unittest.TestCase):
 
     def test_load_object(self):

From bc1a92921364de8f4616feb0ea7dcb7b6d42d2b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 3 May 2019 14:42:12 +0200
Subject: [PATCH 2039/2974] Improve the documentation about detecting check
 runs

---
 docs/topics/contracts.rst | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 3aa32cba8..9337375bb 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -122,13 +122,12 @@ get the failures pretty printed::
                     raise ContractFail('X-CustomHeader not present')
 
 
-Detecting check run
-===================
-It is not encouraged to created different behaviour when running test.
-However, sometimes it is useful to know when a spider is started by scrapy check.
-It can for example be needed to enforce less settings to be set, or to disable some
-uploading of result data. When scrapy check is runned the ``SCRAPY_CHECK`` environment
-variable is set. This can be retrieved via ``os.environ``::
+Detecting check runs
+====================
+
+When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
+set to the ``true`` string. You can use `os.environ`_ to perform any change to
+your spiders or your settings when ``scrapy check`` is used::
 
     import os
     import scrapy
@@ -138,4 +137,6 @@ variable is set. This can be retrieved via ``os.environ``::
 
         def __init__(self):
             if os.environ.get('SCRAPY_CHECK'):
-                # Do some scraper adjustments when check is running
\ No newline at end of file
+                pass  # Do some scraper adjustments when a check is running
+
+.. _os.environ: https://docs.python.org/3/library/os.html#os.environ

From 5814344adfc315a63f43237f02ed75db52765b7d Mon Sep 17 00:00:00 2001
From: Jeffallan <23423962+Jeffallan@users.noreply.github.com>
Date: Sat, 4 May 2019 14:15:47 -0500
Subject: [PATCH 2040/2974] Update telnetconsole.rst

Change spelling of bellow to below.
---
 docs/topics/telnetconsole.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index bf2ffa443..1eb705f05 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -45,7 +45,7 @@ the console you need to type::
     >>>
 
 By default Username is ``scrapy`` and Password is autogenerated. The 
-autogenerated Password can be seen on scrapy logs like the example bellow::
+autogenerated Password can be seen on scrapy logs like the example below::
 
     2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
 

From 3a7850fa158148e6c6096add09b555e46949bd51 Mon Sep 17 00:00:00 2001
From: Aditya <aditya.vrr@gmail.com>
Date: Sun, 5 May 2019 18:45:40 +0530
Subject: [PATCH 2041/2974] Update contributing.rst

---
 docs/contributing.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 2fbe30a00..51b5da59e 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -262,7 +262,7 @@ And their unit-tests are in::
 .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
 .. _open issues: https://github.com/scrapy/scrapy/issues
 .. _PEP 257: https://www.python.org/dev/peps/pep-0257/
-.. _pull request: https://help.github.com/send-pull-requests/
+.. _pull request: https://help.github.com/en/articles/creating-a-pull-request
 .. _pytest: https://docs.pytest.org/en/latest/usage.html
 .. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html
 .. _tox: https://pypi.python.org/pypi/tox

From 611249bb7f3a7bb5a92a67d90d6c97f17494768f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 8 May 2019 12:52:29 +0200
Subject: [PATCH 2042/2974] Implement the METAREFRESH_IGNORE_TAGS setting

---
 docs/topics/downloader-middleware.rst       | 10 ++++++++++
 scrapy/downloadermiddlewares/redirect.py    |  4 +++-
 scrapy/settings/default_settings.py         |  1 +
 scrapy/utils/response.py                    |  4 ++--
 tests/test_downloadermiddleware_redirect.py | 19 +++++++++++++++++++
 5 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index f2f3ef466..fa65f66ed 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -805,6 +805,7 @@ The :class:`MetaRefreshMiddleware` can be configured through the following
 settings (see the settings documentation for more info):
 
 * :setting:`METAREFRESH_ENABLED`
+* :setting:`METAREFRESH_IGNORE_TAGS`
 * :setting:`METAREFRESH_MAXDELAY`
 
 This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`,
@@ -826,6 +827,15 @@ Default: ``True``
 
 Whether the Meta Refresh middleware will be enabled.
 
+.. setting:: METAREFRESH_IGNORE_TAGS
+
+METAREFRESH_IGNORE_TAGS
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Default: ``['script', 'noscript']``
+
+Meta tags within these tags are ignored.
+
 .. setting:: METAREFRESH_MAXDELAY
 
 METAREFRESH_MAXDELAY
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index cb59d3fd2..49468a2e4 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -88,6 +88,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
 
     def __init__(self, settings):
         super(MetaRefreshMiddleware, self).__init__(settings)
+        self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
         self._maxdelay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY',
                                          settings.getint('METAREFRESH_MAXDELAY'))
 
@@ -96,7 +97,8 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
                 not isinstance(response, HtmlResponse):
             return response
 
-        interval, url = get_meta_refresh(response)
+        interval, url = get_meta_refresh(response,
+                                         ignore_tags=self._ignore_tags)
         if url and interval < self._maxdelay:
             redirected = self._redirect_request_using_get(request, url)
             return self._redirect(redirected, request, spider, 'meta refresh')
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 9986827d8..1ce1516e5 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -221,6 +221,7 @@ MEMUSAGE_NOTIFY_MAIL = []
 MEMUSAGE_WARNING_MB = 0
 
 METAREFRESH_ENABLED = True
+METAREFRESH_IGNORE_TAGS = ['script', 'noscript']
 METAREFRESH_MAXDELAY = 100
 
 NEWSPIDER_MODULE = ''
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index bf276b5ca..122af28b0 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -31,12 +31,12 @@ def get_base_url(response):
 
 
 _metaref_cache = weakref.WeakKeyDictionary()
-def get_meta_refresh(response):
+def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
     """Parse the http-equiv refrsh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.text[0:4096]
         _metaref_cache[response] = html.get_meta_refresh(text, response.url,
-            response.encoding, ignore_tags=('script', 'noscript'))
+            response.encoding, ignore_tags=ignore_tags)
     return _metaref_cache[response]
 
 
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 6c81c94ca..0e841489d 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -279,5 +279,24 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
         self.assertEqual(req2.meta['redirect_reasons'], ['meta refresh'])
         self.assertEqual(req3.meta['redirect_reasons'], ['meta refresh', 'meta refresh'])
 
+    def test_ignore_tags_default(self):
+        req = Request(url='http://example.org')
+        body = ('''<noscript><meta http-equiv="refresh" '''
+                '''content="0;URL='http://example.org/newpage'"></noscript>''')
+        rsp = HtmlResponse(req.url, body=body.encode())
+        response = self.mw.process_response(req, rsp, self.spider)
+        assert isinstance(response, Response)
+
+    def test_ignore_tags_empty_list(self):
+        crawler = get_crawler(Spider, {'METAREFRESH_IGNORE_TAGS': []})
+        mw = MetaRefreshMiddleware.from_crawler(crawler)
+        req = Request(url='http://example.org')
+        body = ('''<noscript><meta http-equiv="refresh" '''
+                '''content="0;URL='http://example.org/newpage'"></noscript>''')
+        rsp = HtmlResponse(req.url, body=body.encode())
+        req2 = mw.process_response(req, rsp, self.spider)
+        assert isinstance(req2, Request)
+        self.assertEqual(req2.url, 'http://example.org/newpage')
+
 if __name__ == "__main__":
     unittest.main()

From e667ca76820a53ac3abf34604fc284761f936bb9 Mon Sep 17 00:00:00 2001
From: Andrew Baxter <afb2@wustl.edu>
Date: Fri, 24 May 2019 21:45:53 +0900
Subject: [PATCH 2043/2974] Account for mangling when serializing requests with
 private callbacks

---
 scrapy/utils/reqser.py     | 6 +++++-
 tests/test_utils_reqser.py | 9 +++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 959dddbd5..8c99763cf 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -75,7 +75,11 @@ def _find_method(obj, func):
             pass
         else:
             if func_self is obj:
-                return six.get_method_function(func).__name__
+                name = six.get_method_function(func).__name__
+                if name.startswith('__'):
+                    classname = obj.__class__.__name__.lstrip('_')
+                    name = '_%s%s' % (classname, name)
+                return name
     raise ValueError("Function %s is not a method of: %s" % (func, obj))
 
 
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index dcc070b8f..f7191fcef 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -68,6 +68,12 @@ class RequestSerializationTest(unittest.TestCase):
                     errback=self.spider.handle_error)
         self._assert_serializes_ok(r, spider=self.spider)
 
+    def test_private_callback_serialization(self):
+        r = Request("http://www.example.com",
+                    callback=self.spider._TestSpider__parse_item_private,
+                    errback=self.spider.handle_error)
+        self._assert_serializes_ok(r, spider=self.spider)
+
     def test_unserializable_callback1(self):
         r = Request("http://www.example.com", callback=lambda x: x)
         self.assertRaises(ValueError, request_to_dict, r)
@@ -87,6 +93,9 @@ class TestSpider(Spider):
     def handle_error(self, failure):
         pass
 
+    def __parse_item_private(self, response):
+        pass
+
 
 class CustomRequest(Request):
     pass

From 7d36fa7435d2147c7dfd6a87733187823431b61c Mon Sep 17 00:00:00 2001
From: Capi Etheriel <barraponto@gmail.com>
Date: Fri, 24 May 2019 10:32:55 -0300
Subject: [PATCH 2044/2974] Fix documentation for spiderloader

---
 docs/topics/api.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index ba832ab5d..e1623287d 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -154,7 +154,7 @@ Settings API
 SpiderLoader API
 ================
 
-.. module:: scrapy.loader
+.. module:: scrapy.spiderloader
    :synopsis: The spider loader
 
 .. class:: SpiderLoader

From 0ee2284fcc23ccb2b8a4da8fb561a232cc328fe0 Mon Sep 17 00:00:00 2001
From: Capi Etheriel <barraponto@gmail.com>
Date: Fri, 24 May 2019 11:11:15 -0300
Subject: [PATCH 2045/2974] Add 429 to RETRY_HTTP_CODES

---
 scrapy/settings/default_settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 9986827d8..2afa7b321 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -238,7 +238,7 @@ REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
 
 RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
-RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408]
+RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
 RETRY_PRIORITY_ADJUST = -1
 
 ROBOTSTXT_OBEY = False

From 144afcee7973ab97d6c8d89fec007046cc878e3d Mon Sep 17 00:00:00 2001
From: Andrew Baxter <afb2@wustl.edu>
Date: Sat, 25 May 2019 00:52:00 +0900
Subject: [PATCH 2046/2974] Use regex to check for private methods

---
 scrapy/utils/reqser.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 8c99763cf..07c51aaff 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -2,12 +2,16 @@
 Helper functions for serializing (and deserializing) requests.
 """
 import six
+import re
 
 from scrapy.http import Request
 from scrapy.utils.python import to_unicode, to_native_str
 from scrapy.utils.misc import load_object
 
 
+private_name_regex = re.compile('^__[^_](.*[^_])?_?$')
+
+
 def request_to_dict(request, spider=None):
     """Convert Request object to a dict.
 
@@ -76,7 +80,7 @@ def _find_method(obj, func):
         else:
             if func_self is obj:
                 name = six.get_method_function(func).__name__
-                if name.startswith('__'):
+                if private_name_regex.search(name):
                     classname = obj.__class__.__name__.lstrip('_')
                     name = '_%s%s' % (classname, name)
                 return name

From 461682fc3dca72d9a34ddc22ad1896787c9dc518 Mon Sep 17 00:00:00 2001
From: Claudio Salazar <csalazar@spect.cl>
Date: Sat, 25 May 2019 11:01:19 +0200
Subject: [PATCH 2047/2974] Whitelist form methods in FormRequest.from_response
 method

---
 scrapy/http/request/form.py |  7 ++++++-
 tests/test_http_request.py  | 13 +++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index c2413b431..2182b9b53 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -18,6 +18,7 @@ from scrapy.utils.response import get_base_url
 
 
 class FormRequest(Request):
+    valid_form_methods = ['GET', 'POST', 'DIALOG']
 
     def __init__(self, *args, **kwargs):
         formdata = kwargs.pop('formdata', None)
@@ -48,7 +49,11 @@ class FormRequest(Request):
         form = _get_form(response, formname, formid, formnumber, formxpath)
         formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
         url = _get_form_url(form, kwargs.pop('url', None))
-        method = kwargs.pop('method', form.method)
+
+        method = kwargs.pop('method', form.method).upper()
+        if method not in cls.valid_form_methods:
+            raise ValueError('Invalid form method in chosen form')
+
         return cls(url=url, method=method, formdata=formdata, **kwargs)
 
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 49f148016..8fdafb286 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1100,6 +1100,19 @@ class FormRequestTest(RequestTest):
         self.assertRaises(ValueError, self.request_class.from_response,
                           response, formcss="input[name='abc']")
 
+    def test_from_response_valid_form_methods(self):
+        body = """<form action="post.php" method="%s">
+            <input type="hidden" name="one" value="1">
+            </form>"""
+
+        for method in self.request_class.valid_form_methods:
+            response = _buildresponse(body % method)
+            r1 = self.request_class.from_response(response)
+            self.assertEqual(r1.method, method)
+
+        response = _buildresponse(body % 'UNKNOWN')
+        self.assertRaises(ValueError, self.request_class.from_response, response)
+
 
 def _buildresponse(body, **kwargs):
     kwargs.setdefault('body', body)

From da82ede8a0751cf3e8496f35252eb0dcef4f197e Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Sat, 25 May 2019 17:19:10 +0530
Subject: [PATCH 2048/2974] describe method as a command

---
 docs/topics/downloader-middleware.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index dfbcdb8fa..236150059 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -526,7 +526,7 @@ defines the methods described below.
 
     .. method:: retrieve_response(spider, request)
 
-      Returns response if present in cache, or ``None`` otherwise.
+      Return response if present in cache, or ``None`` otherwise.
 
       :param spider: the spider which generated the request
       :type spider: :class:`~scrapy.spiders.Spider` object
@@ -536,7 +536,7 @@ defines the methods described below.
 
     .. method:: store_response(spider, request, response)
 
-      Stores the given response in the cache.
+      Store the given response in the cache.
 
       :param spider: the spider for which the response is intended
       :type spider: :class:`~scrapy.spiders.Spider` object

From 18f01ea6ecf1dba77b25d8d1f62c80ed0f9a13f5 Mon Sep 17 00:00:00 2001
From: mar-heaven <775650117@qq.com>
Date: Mon, 27 May 2019 17:15:30 +0800
Subject: [PATCH 2049/2974] remove a "is"

When I translated in Chinese, I found a needless "is"
---
 docs/topics/spiders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 8c4049f85..79eecfc3e 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -657,7 +657,7 @@ SitemapSpider
 
     .. attribute:: sitemap_follow
 
-        A list of regexes of sitemap that should be followed. This is is only
+        A list of regexes of sitemap that should be followed. This is only
         for sites that use `Sitemap index files`_ that point to other sitemap
         files.
 

From 72b7d3e90ac2d21ffdd0c44878ec1a5a5d0fa5ce Mon Sep 17 00:00:00 2001
From: Andrew Baxter <afb2@wustl.edu>
Date: Mon, 27 May 2019 23:30:23 +0900
Subject: [PATCH 2050/2974] Make the regex align to the spec better; add unit
 tests for name variations

---
 scrapy/utils/reqser.py     |  2 +-
 tests/test_utils_reqser.py | 24 +++++++++++++++++++++++-
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 07c51aaff..04665a2d4 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -9,7 +9,7 @@ from scrapy.utils.python import to_unicode, to_native_str
 from scrapy.utils.misc import load_object
 
 
-private_name_regex = re.compile('^__[^_](.*[^_])?_?$')
+private_name_regex = re.compile('^__.*[^_]_?$')
 
 
 def request_to_dict(request, spider=None):
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index f7191fcef..b49450ac5 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -3,7 +3,7 @@ import unittest
 
 from scrapy.http import Request, FormRequest
 from scrapy.spiders import Spider
-from scrapy.utils.reqser import request_to_dict, request_from_dict
+from scrapy.utils.reqser import request_to_dict, request_from_dict, private_name_regex
 
 
 class RequestSerializationTest(unittest.TestCase):
@@ -74,6 +74,28 @@ class RequestSerializationTest(unittest.TestCase):
                     errback=self.spider.handle_error)
         self._assert_serializes_ok(r, spider=self.spider)
 
+    def test_private_callback_name_matching(self):
+        self.assertTrue(private_name_regex.search('__a'))
+        self.assertTrue(private_name_regex.search('__a_'))
+        self.assertTrue(private_name_regex.search('__a_a'))
+        self.assertTrue(private_name_regex.search('__a_a_'))
+        self.assertTrue(private_name_regex.search('__a__a'))
+        self.assertTrue(private_name_regex.search('__a__a_'))
+        self.assertTrue(private_name_regex.search('__a___a'))
+        self.assertTrue(private_name_regex.search('__a___a_'))
+        self.assertTrue(private_name_regex.search('___a'))
+        self.assertTrue(private_name_regex.search('___a_'))
+        self.assertTrue(private_name_regex.search('___a_a'))
+        self.assertTrue(private_name_regex.search('___a_a_'))
+        self.assertTrue(private_name_regex.search('____a_a_'))
+
+        self.assertFalse(private_name_regex.search('_a'))
+        self.assertFalse(private_name_regex.search('_a_'))
+        self.assertFalse(private_name_regex.search('__a__'))
+        self.assertFalse(private_name_regex.search('__'))
+        self.assertFalse(private_name_regex.search('___'))
+        self.assertFalse(private_name_regex.search('____'))
+
     def test_unserializable_callback1(self):
         r = Request("http://www.example.com", callback=lambda x: x)
         self.assertRaises(ValueError, request_to_dict, r)

From 9af91a26b035a10e9303227ad9ddd5e043725514 Mon Sep 17 00:00:00 2001
From: Andrew Baxter <afb2@wustl.edu>
Date: Tue, 28 May 2019 01:40:26 +0900
Subject: [PATCH 2051/2974] Replace regex usage

---
 scrapy/utils/reqser.py     | 10 +++++-----
 tests/test_utils_reqser.py | 40 +++++++++++++++++++-------------------
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 04665a2d4..40223661f 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -2,16 +2,12 @@
 Helper functions for serializing (and deserializing) requests.
 """
 import six
-import re
 
 from scrapy.http import Request
 from scrapy.utils.python import to_unicode, to_native_str
 from scrapy.utils.misc import load_object
 
 
-private_name_regex = re.compile('^__.*[^_]_?$')
-
-
 def request_to_dict(request, spider=None):
     """Convert Request object to a dict.
 
@@ -71,6 +67,10 @@ def request_from_dict(d, spider=None):
         flags=d.get('flags'))
 
 
+def _is_private_method(name):
+    return name.startswith('__') and not name.endswith('__')
+
+
 def _find_method(obj, func):
     if obj:
         try:
@@ -80,7 +80,7 @@ def _find_method(obj, func):
         else:
             if func_self is obj:
                 name = six.get_method_function(func).__name__
-                if private_name_regex.search(name):
+                if _is_private_method(name):
                     classname = obj.__class__.__name__.lstrip('_')
                     name = '_%s%s' % (classname, name)
                 return name
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index b49450ac5..fad5b6003 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -3,7 +3,7 @@ import unittest
 
 from scrapy.http import Request, FormRequest
 from scrapy.spiders import Spider
-from scrapy.utils.reqser import request_to_dict, request_from_dict, private_name_regex
+from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method
 
 
 class RequestSerializationTest(unittest.TestCase):
@@ -75,26 +75,26 @@ class RequestSerializationTest(unittest.TestCase):
         self._assert_serializes_ok(r, spider=self.spider)
 
     def test_private_callback_name_matching(self):
-        self.assertTrue(private_name_regex.search('__a'))
-        self.assertTrue(private_name_regex.search('__a_'))
-        self.assertTrue(private_name_regex.search('__a_a'))
-        self.assertTrue(private_name_regex.search('__a_a_'))
-        self.assertTrue(private_name_regex.search('__a__a'))
-        self.assertTrue(private_name_regex.search('__a__a_'))
-        self.assertTrue(private_name_regex.search('__a___a'))
-        self.assertTrue(private_name_regex.search('__a___a_'))
-        self.assertTrue(private_name_regex.search('___a'))
-        self.assertTrue(private_name_regex.search('___a_'))
-        self.assertTrue(private_name_regex.search('___a_a'))
-        self.assertTrue(private_name_regex.search('___a_a_'))
-        self.assertTrue(private_name_regex.search('____a_a_'))
+        self.assertTrue(_is_private_method('__a'))
+        self.assertTrue(_is_private_method('__a_'))
+        self.assertTrue(_is_private_method('__a_a'))
+        self.assertTrue(_is_private_method('__a_a_'))
+        self.assertTrue(_is_private_method('__a__a'))
+        self.assertTrue(_is_private_method('__a__a_'))
+        self.assertTrue(_is_private_method('__a___a'))
+        self.assertTrue(_is_private_method('__a___a_'))
+        self.assertTrue(_is_private_method('___a'))
+        self.assertTrue(_is_private_method('___a_'))
+        self.assertTrue(_is_private_method('___a_a'))
+        self.assertTrue(_is_private_method('___a_a_'))
+        self.assertTrue(_is_private_method('____a_a_'))
 
-        self.assertFalse(private_name_regex.search('_a'))
-        self.assertFalse(private_name_regex.search('_a_'))
-        self.assertFalse(private_name_regex.search('__a__'))
-        self.assertFalse(private_name_regex.search('__'))
-        self.assertFalse(private_name_regex.search('___'))
-        self.assertFalse(private_name_regex.search('____'))
+        self.assertFalse(_is_private_method('_a'))
+        self.assertFalse(_is_private_method('_a_'))
+        self.assertFalse(_is_private_method('__a__'))
+        self.assertFalse(_is_private_method('__'))
+        self.assertFalse(_is_private_method('___'))
+        self.assertFalse(_is_private_method('____'))
 
     def test_unserializable_callback1(self):
         r = Request("http://www.example.com", callback=lambda x: x)

From bcad8947e8192448ab3bd59489444efb567f8793 Mon Sep 17 00:00:00 2001
From: Andrew Baxter <afb2@wustl.edu>
Date: Mon, 3 Jun 2019 20:41:02 +0900
Subject: [PATCH 2052/2974] Support inherited private method names

---
 scrapy/utils/reqser.py     |  9 +++++++--
 tests/test_utils_reqser.py | 16 +++++++++++++++-
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 40223661f..d1f472e6e 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -81,8 +81,13 @@ def _find_method(obj, func):
             if func_self is obj:
                 name = six.get_method_function(func).__name__
                 if _is_private_method(name):
-                    classname = obj.__class__.__name__.lstrip('_')
-                    name = '_%s%s' % (classname, name)
+                    qualname = getattr(func, '__qualname__', None)
+                    if qualname is None:
+                        classname = obj.__class__.__name__.lstrip('_')
+                        name = '_%s%s' % (classname, name)
+                    else:
+                        splits = qualname.split('.')
+                        name = '_%s%s' % (splits[-2], splits[-1])
                 return name
     raise ValueError("Function %s is not a method of: %s" % (func, obj))
 
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index fad5b6003..31577bc8c 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import unittest
+import sys
 
 from scrapy.http import Request, FormRequest
 from scrapy.spiders import Spider
@@ -74,6 +75,14 @@ class RequestSerializationTest(unittest.TestCase):
                     errback=self.spider.handle_error)
         self._assert_serializes_ok(r, spider=self.spider)
 
+    def test_mixin_private_callback_serialization(self):
+        if sys.version_info[0] < 3:
+            return
+        r = Request("http://www.example.com",
+                    callback=self.spider._TestSpiderMixin__mixin_callback,
+                    errback=self.spider.handle_error)
+        self._assert_serializes_ok(r, spider=self.spider)
+
     def test_private_callback_name_matching(self):
         self.assertTrue(_is_private_method('__a'))
         self.assertTrue(_is_private_method('__a_'))
@@ -106,7 +115,12 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertRaises(ValueError, request_to_dict, r)
 
 
-class TestSpider(Spider):
+class TestSpiderMixin(object):
+    def __mixin_callback(self, response):
+        pass
+
+
+class TestSpider(Spider, TestSpiderMixin):
     name = 'test'
 
     def parse_item(self, response):

From ea209a0ea7815f68a04a5ccab79a2b4f4a146647 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 3 Jun 2019 19:21:40 +0200
Subject: [PATCH 2053/2974] Fix module double indexing issues in the
 documentation

---
 docs/topics/stats.rst         | 3 +--
 docs/topics/telnetconsole.rst | 5 ++---
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst
index dd0c6216b..38648ec55 100644
--- a/docs/topics/stats.rst
+++ b/docs/topics/stats.rst
@@ -75,8 +75,7 @@ available in Scrapy which extend the basic Stats Collector. You can select
 which Stats Collector to use through the :setting:`STATS_CLASS` setting. The
 default Stats Collector used is the :class:`MemoryStatsCollector`. 
 
-.. module:: scrapy.statscollectors
-   :synopsis: Stats Collectors
+.. currentmodule:: scrapy.statscollectors
 
 MemoryStatsCollector
 --------------------
diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index 1eb705f05..7db7e4f6b 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -1,12 +1,11 @@
+.. currentmodule:: scrapy.extensions.telnet
+
 .. _topics-telnetconsole:
 
 ==============
 Telnet Console
 ==============
 
-.. module:: scrapy.extensions.telnet
-   :synopsis: The Telnet Console
-
 Scrapy comes with a built-in telnet console for inspecting and controlling a
 Scrapy running process. The telnet console is just a regular python shell
 running inside the Scrapy process, so you can do literally anything from it.

From c7b5ad0e20dc736a8a08b62134c9418439fb7077 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 4 Jan 2019 18:17:35 +0100
Subject: [PATCH 2054/2974] Add a Sphinx extension to generate documentation
 coverage information

---
 docs/Makefile         | 3 +++
 docs/conf.py          | 9 ++++++++-
 docs/contributing.rst | 9 +++++++++
 docs/requirements.txt | 2 +-
 tox.ini               | 6 ++++++
 5 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/docs/Makefile b/docs/Makefile
index 187f03c4c..ff68bf1ae 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -82,6 +82,9 @@ pydoc-topics: build
 	@echo "Building finished; now copy build/pydoc-topics/pydoc_topics.py " \
 	      "into the Lib/ directory"
 
+coverage: BUILDER = coverage
+coverage: build
+
 htmlview: html
 	 $(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \
 	 os.path.realpath('build/html/index.html'))"
diff --git a/docs/conf.py b/docs/conf.py
index a54a6bbe9..832626f6b 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -28,7 +28,8 @@ sys.path.insert(0, path.dirname(path.dirname(__file__)))
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
     'scrapydocs',
-    'sphinx.ext.autodoc'
+    'sphinx.ext.autodoc',
+    'sphinx.ext.coverage',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -218,3 +219,9 @@ linkcheck_ignore = [
     'http://localhost:\d+', 'http://hg.scrapy.org',
     'http://directory.google.com/'
 ]
+
+
+# Options for the Coverage extension
+# ----------------------------------
+coverage_ignore_pyobjects = [
+]
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 51b5da59e..b4f91ea8d 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -99,6 +99,15 @@ Well-written patches should:
   the documentation changes in the same patch.  See `Documentation policies`_
   below.
 
+* if you're adding a private API, please add a regular expression to the
+  ``coverage_ignore_pyobjects`` variable of ``docs/conf.py`` to exclude the new
+  private API from documentation coverage checks.
+
+  To see if your private API is skipped properly, generate a documentation
+  coverage report as follows::
+
+      tox -e docs-coverage
+
 .. _submitting-patches:
 
 Submitting patches
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 8e7611d21..379da9994 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,2 +1,2 @@
-Sphinx>=1.6
+Sphinx>=2.1
 sphinx_rtd_theme
\ No newline at end of file
diff --git a/tox.ini b/tox.ini
index 0c0f8f7b7..157a8b3ed 100644
--- a/tox.ini
+++ b/tox.ini
@@ -105,6 +105,12 @@ deps = {[docs]deps}
 commands =
     sphinx-build -W -b html . {envtmpdir}/html
 
+[testenv:docs-coverage]
+changedir = {[docs]changedir}
+deps = {[docs]deps}
+commands =
+    sphinx-build -b coverage . {envtmpdir}/coverage
+
 [testenv:docs-links]
 changedir = {[docs]changedir}
 deps = {[docs]deps}

From c81e15ed6ede552c499ae3ac4e03af27b1f9ed89 Mon Sep 17 00:00:00 2001
From: Artem Kuchumov <duketemon@users.noreply.github.com>
Date: Wed, 5 Jun 2019 13:15:23 +0500
Subject: [PATCH 2055/2974] Tutorial: scrapy shell example should say "text"
 not "title" (#3807)

Tutorial: scrapy shell example should say "text" not "title"
---
 docs/intro/tutorial.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 8bd2d27dd..a190ce407 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -379,11 +379,11 @@ variable, so that we can run our CSS selectors directly on a particular quote::
 
     >>> quote = response.css("div.quote")[0]
 
-Now, let's extract ``title``, ``author`` and the ``tags`` from that quote
+Now, let's extract ``text``, ``author`` and the ``tags`` from that quote
 using the ``quote`` object we just created::
 
-    >>> title = quote.css("span.text::text").get()
-    >>> title
+    >>> text = quote.css("span.text::text").get()
+    >>> text
     '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
     >>> author = quote.css("small.author::text").get()
     >>> author

From 9c81721c407ff41ef9dce2c33e26ac477355cf1f Mon Sep 17 00:00:00 2001
From: Andrew Baxter <afb2@wustl.edu>
Date: Wed, 5 Jun 2019 23:43:56 +0900
Subject: [PATCH 2056/2974] Add tests for private method name mangling

---
 scrapy/utils/reqser.py     | 18 +++++++++++-------
 tests/test_utils_reqser.py | 16 +++++++++++++++-
 2 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index d1f472e6e..3c463cfed 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -71,6 +71,16 @@ def _is_private_method(name):
     return name.startswith('__') and not name.endswith('__')
 
 
+def _mangle_private_name(obj, func, name):
+    qualname = getattr(func, '__qualname__', None)
+    if qualname is None:
+        classname = obj.__class__.__name__.lstrip('_')
+        return '_%s%s' % (classname, name)
+    else:
+        splits = qualname.split('.')
+        return '_%s%s' % (splits[-2], splits[-1])
+
+
 def _find_method(obj, func):
     if obj:
         try:
@@ -81,13 +91,7 @@ def _find_method(obj, func):
             if func_self is obj:
                 name = six.get_method_function(func).__name__
                 if _is_private_method(name):
-                    qualname = getattr(func, '__qualname__', None)
-                    if qualname is None:
-                        classname = obj.__class__.__name__.lstrip('_')
-                        name = '_%s%s' % (classname, name)
-                    else:
-                        splits = qualname.split('.')
-                        name = '_%s%s' % (splits[-2], splits[-1])
+                    return _mangle_private_name(obj, func, name)
                 return name
     raise ValueError("Function %s is not a method of: %s" % (func, obj))
 
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 31577bc8c..7f9e31daa 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -2,9 +2,11 @@
 import unittest
 import sys
 
+import six
+
 from scrapy.http import Request, FormRequest
 from scrapy.spiders import Spider
-from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method
+from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method, _mangle_private_name
 
 
 class RequestSerializationTest(unittest.TestCase):
@@ -105,6 +107,18 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertFalse(_is_private_method('___'))
         self.assertFalse(_is_private_method('____'))
 
+    def _assert_mangles_to(self, obj, name):
+        self.assertEqual(
+            _mangle_private_name(obj, getattr(obj, name), name),
+            name
+        )
+
+    def test_private_name_mangling(self):
+        self._assert_mangles_to(
+            self.spider, '_TestSpider__parse_item_private')
+        self._assert_mangles_to(
+            self.spider, '_TestSpiderMixin__mixin_callback')
+
     def test_unserializable_callback1(self):
         r = Request("http://www.example.com", callback=lambda x: x)
         self.assertRaises(ValueError, request_to_dict, r)

From 3dd3e8c29863683d60f9c4f74aacac3103703061 Mon Sep 17 00:00:00 2001
From: Andrew Baxter <afb2@wustl.edu>
Date: Wed, 5 Jun 2019 23:49:54 +0900
Subject: [PATCH 2057/2974] Restrict different class mangling tests to Py 3+

---
 tests/test_utils_reqser.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 7f9e31daa..57dc5db53 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -116,8 +116,9 @@ class RequestSerializationTest(unittest.TestCase):
     def test_private_name_mangling(self):
         self._assert_mangles_to(
             self.spider, '_TestSpider__parse_item_private')
-        self._assert_mangles_to(
-            self.spider, '_TestSpiderMixin__mixin_callback')
+        if sys.version_info[0] >= 3:
+            self._assert_mangles_to(
+                self.spider, '_TestSpiderMixin__mixin_callback')
 
     def test_unserializable_callback1(self):
         r = Request("http://www.example.com", callback=lambda x: x)

From 6af1dc89aa5988ebbfbef90afdafa84736f3993c Mon Sep 17 00:00:00 2001
From: Andrew Baxter <afb2@wustl.edu>
Date: Thu, 6 Jun 2019 04:25:19 +0900
Subject: [PATCH 2058/2974] Fix mangling test

---
 tests/test_utils_reqser.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 57dc5db53..e5a09dcf1 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -108,8 +108,9 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertFalse(_is_private_method('____'))
 
     def _assert_mangles_to(self, obj, name):
+        func = getattr(obj, name)
         self.assertEqual(
-            _mangle_private_name(obj, getattr(obj, name), name),
+            _mangle_private_name(obj, func, func.__name__),
             name
         )
 

From 0c50879568dee2363df5cbe25e9bdd7adaed5da4 Mon Sep 17 00:00:00 2001
From: Claudio Salazar <csalazar@spect.cl>
Date: Thu, 6 Jun 2019 22:10:59 +0200
Subject: [PATCH 2059/2974] Change behavior to use method GET when there are
 unknown methods in the form

---
 scrapy/http/request/form.py | 2 +-
 tests/test_http_request.py  | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 2182b9b53..8b29aae4b 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -52,7 +52,7 @@ class FormRequest(Request):
 
         method = kwargs.pop('method', form.method).upper()
         if method not in cls.valid_form_methods:
-            raise ValueError('Invalid form method in chosen form')
+            method = 'GET'
 
         return cls(url=url, method=method, formdata=formdata, **kwargs)
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 8fdafb286..258b48dce 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1107,11 +1107,12 @@ class FormRequestTest(RequestTest):
 
         for method in self.request_class.valid_form_methods:
             response = _buildresponse(body % method)
-            r1 = self.request_class.from_response(response)
-            self.assertEqual(r1.method, method)
+            r = self.request_class.from_response(response)
+            self.assertEqual(r.method, method)
 
         response = _buildresponse(body % 'UNKNOWN')
-        self.assertRaises(ValueError, self.request_class.from_response, response)
+        r = self.request_class.from_response(response)
+        self.assertEqual(r.method, 'GET')
 
 
 def _buildresponse(body, **kwargs):

From bd8a10384b462dd56b33668e8b92e4a148fd6fba Mon Sep 17 00:00:00 2001
From: Sortafreel <woutut@gmail.com>
Date: Fri, 7 Jun 2019 01:50:03 +0300
Subject: [PATCH 2060/2974] Add values (if there're any) when initiating items
 from dicts

https://github.com/scrapy/scrapy/issues/3804
---
 scrapy/loader/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index a7c75a46a..295a8e42d 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -35,6 +35,8 @@ class ItemLoader(object):
         self.parent = parent
         self._local_item = context['item'] = item
         self._local_values = defaultdict(list)
+        for field_name, value in item.items():
+            self.add_value(field_name, value)
 
     @property
     def _values(self):

From 754f52b02781097c8ca6835e057815c7653062d4 Mon Sep 17 00:00:00 2001
From: Sortafreel <woutut@gmail.com>
Date: Fri, 7 Jun 2019 03:20:45 +0300
Subject: [PATCH 2061/2974] Preprocess values if item built from dict.

https://github.com/scrapy/scrapy/issues/3804
---
 scrapy/loader/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 295a8e42d..7c7f66866 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -35,8 +35,9 @@ class ItemLoader(object):
         self.parent = parent
         self._local_item = context['item'] = item
         self._local_values = defaultdict(list)
+        # Preprocess values if item built from dict
         for field_name, value in item.items():
-            self.add_value(field_name, value)
+            self._values[field_name] = self._process_input_value(field_name, value)
 
     @property
     def _values(self):

From c7ba72b5dc9da3435eb1ec303b991d05ba40ba1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 4 Jun 2019 17:10:14 +0200
Subject: [PATCH 2062/2974] Skip scrapy.contracts private APIs in the
 documentation coverage report

---
 docs/conf.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/docs/conf.py b/docs/conf.py
index 832626f6b..bf222b361 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -224,4 +224,17 @@ linkcheck_ignore = [
 # Options for the Coverage extension
 # ----------------------------------
 coverage_ignore_pyobjects = [
+    # Contract’s add_pre_hook and add_post_hook are not documented because
+    # they should be transparent to contract developers, for whom pre_hook and
+    # post_hook should be the actual concern.
+    r'\bContract\.add_(pre|post)_hook$',
+
+    # ContractsManager is an internal class, developers are not expected to
+    # interact with it directly in any way.
+    r'\bContractsManager\b$',
+
+    # For default contracts we only want to document their general purpose in
+    # their constructor, the methods they reimplement to achieve that purpose
+    # should be irrelevant to developers using those contracts.
+    r'\w+Contract\.(adjust_request_args|(pre|post)_process)$',
 ]

From a1bca6a8e722af53241e51bbf758e7bd67671801 Mon Sep 17 00:00:00 2001
From: sortafreel <woutut@gmail.com>
Date: Tue, 11 Jun 2019 07:36:29 +0300
Subject: [PATCH 2063/2974] Add tests.

---
 scrapy/loader/__init__.py |  1 +
 tests/test_loader.py      | 65 ++++++++++++++++++++++++++-------------
 2 files changed, 45 insertions(+), 21 deletions(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 7c7f66866..20f0f90c3 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -36,6 +36,7 @@ class ItemLoader(object):
         self._local_item = context['item'] = item
         self._local_values = defaultdict(list)
         # Preprocess values if item built from dict
+        # Values need to be added to item._values if added them from dict (not with add_values)
         for field_name, value in item.items():
             self._values[field_name] = self._process_input_value(field_name, value)
 
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 8b58e4dbd..eb4a01572 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -419,6 +419,29 @@ class BasicItemLoaderTest(unittest.TestCase):
         self.assertEqual(item['url'], u'rabbit.hole')
         self.assertEqual(item['summary'], u'rabbithole')
 
+    def test_create_item_from_dict(self):
+        class TestItem(Item):
+            title = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+
+        input_item = {'title': 'Test item title 1'}
+        il = TestItemLoader(item=input_item)
+        # Getting output value mustn't remove value from item
+        self.assertEqual(il.load_item(), {
+            'title': 'Test item title 1',
+        })
+        self.assertEqual(il.get_output_value('title'), 'Test item title 1')
+        self.assertEqual(il.load_item(), {
+            'title': 'Test item title 1',
+        })
+
+        input_item = {'title': 'Test item title 2'}
+        il = TestItemLoader(item=input_item)
+        # Values from dict must be added to item _values
+        self.assertEqual(il._values.get('title'), 'Test item title 2')
+
 
 class ProcessorsTest(unittest.TestCase):
 
@@ -709,28 +732,28 @@ class SubselectorLoaderTest(unittest.TestCase):
 
 
 class SelectJmesTestCase(unittest.TestCase):
-        test_list_equals = {
-            'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
-            'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
-            'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
-            'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
-            'dict': (
-                'foo.bar[*].name',
-                {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
-                ['one', 'two']
-            ),
-            'list': ('[1]', [1, 2], 2)
-        }
+    test_list_equals = {
+        'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+        'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
+        'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
+        'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+        'dict': (
+            'foo.bar[*].name',
+            {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
+            ['one', 'two']
+        ),
+        'list': ('[1]', [1, 2], 2)
+    }
 
-        def test_output(self):
-            for l in self.test_list_equals:
-                expr, test_list, expected = self.test_list_equals[l]
-                test = SelectJmes(expr)(test_list)
-                self.assertEqual(
-                    test,
-                    expected,
-                    msg='test "{}" got {} expected {}'.format(l, test, expected)
-                )
+    def test_output(self):
+        for l in self.test_list_equals:
+            expr, test_list, expected = self.test_list_equals[l]
+            test = SelectJmes(expr)(test_list)
+            self.assertEqual(
+                test,
+                expected,
+                msg='test "{}" got {} expected {}'.format(l, test, expected)
+            )
 
 
 if __name__ == "__main__":

From 7dad2f7b130c426f2a8aee320ccbc378752a9568 Mon Sep 17 00:00:00 2001
From: sortafreel <woutut@gmail.com>
Date: Tue, 11 Jun 2019 07:43:03 +0300
Subject: [PATCH 2064/2974] Add more tests.

---
 tests/test_loader.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tests/test_loader.py b/tests/test_loader.py
index eb4a01572..241630ab3 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -442,6 +442,20 @@ class BasicItemLoaderTest(unittest.TestCase):
         # Values from dict must be added to item _values
         self.assertEqual(il._values.get('title'), 'Test item title 2')
 
+        input_item = {'title': [u'Test item title 3', u'Test item 4']}
+        il = TestItemLoader(item=input_item)
+        # Same rules must work for lists
+        self.assertEqual(il._values.get('title'),
+                         [u'Test item title 3', u'Test item 4'])
+        self.assertEqual(il.load_item(), {
+            'title': [u'Test item title 3', u'Test item 4'],
+        })
+        self.assertEqual(il.get_output_value('title'),
+                         [u'Test item title 3', u'Test item 4'])
+        self.assertEqual(il.load_item(), {
+            'title': [u'Test item title 3', u'Test item 4'],
+        })
+
 
 class ProcessorsTest(unittest.TestCase):
 

From 0da972339bb174156b08a3ae34ece7fddea1e48d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 11 Jun 2019 14:11:38 +0200
Subject: [PATCH 2065/2974] Require Twisted<=19.2.0 for Python 3.4

---
 setup.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index bd666e93c..4dc6d18c1 100644
--- a/setup.py
+++ b/setup.py
@@ -65,7 +65,8 @@ setup(
     ],
     python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     install_requires=[
-        'Twisted>=13.1.0',
+        'Twisted>=13.1.0;python_version!="3.4"',
+        'Twisted>=13.1.0,<=19.2.0;python_version=="3.4"',
         'w3lib>=1.17.0',
         'queuelib',
         'lxml',

From fe0f80f2f422d4047a8b6230d66eb853d443d90b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 11 Jun 2019 15:50:41 +0200
Subject: [PATCH 2066/2974] Set the cloned directory as PYTHONPATH in
 appveyor.yml

---
 appveyor.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index 93cfd469e..7fd636864 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -12,7 +12,8 @@ branches:
 
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
-  - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE"
+  - "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%"
+  - "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE"
   - "pip install -U tox"
 
 build: false

From cdeccac6d6ccd0034a5f007ed371c1d481b32c26 Mon Sep 17 00:00:00 2001
From: sortafreel <woutut@gmail.com>
Date: Tue, 11 Jun 2019 17:38:06 +0300
Subject: [PATCH 2067/2974] Linting (return previous indentation).

---
 tests/test_loader.py | 42 +++++++++++++++++++++---------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/tests/test_loader.py b/tests/test_loader.py
index 241630ab3..5a8ee1b2e 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -746,28 +746,28 @@ class SubselectorLoaderTest(unittest.TestCase):
 
 
 class SelectJmesTestCase(unittest.TestCase):
-    test_list_equals = {
-        'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
-        'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
-        'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
-        'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
-        'dict': (
-            'foo.bar[*].name',
-            {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
-            ['one', 'two']
-        ),
-        'list': ('[1]', [1, 2], 2)
-    }
+        test_list_equals = {
+            'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+            'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
+            'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
+            'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+            'dict': (
+                'foo.bar[*].name',
+                {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
+                ['one', 'two']
+            ),
+            'list': ('[1]', [1, 2], 2)
+        }
 
-    def test_output(self):
-        for l in self.test_list_equals:
-            expr, test_list, expected = self.test_list_equals[l]
-            test = SelectJmes(expr)(test_list)
-            self.assertEqual(
-                test,
-                expected,
-                msg='test "{}" got {} expected {}'.format(l, test, expected)
-            )
+        def test_output(self):
+            for l in self.test_list_equals:
+                expr, test_list, expected = self.test_list_equals[l]
+                test = SelectJmes(expr)(test_list)
+                self.assertEqual(
+                    test,
+                    expected,
+                    msg='test "{}" got {} expected {}'.format(l, test, expected)
+                )
 
 
 if __name__ == "__main__":

From b8900ec6a698cb4e27424de57ca5593f1c7300e7 Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Mon, 17 Jun 2019 00:06:44 +0530
Subject: [PATCH 2068/2974] removes unused var

---
 tests/test_downloadermiddleware.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 0f420b70d..03564e748 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -123,7 +123,6 @@ class ProcessRequestInvalidOutput(ManagerTestCase):
 
     def test_invalid_process_request(self):
         req = Request('http://example.com/index.html')
-        resp = Response('http://example.com/index.html')
 
         class InvalidProcessRequestMiddleware:
             def process_request(self, request, spider):
@@ -143,7 +142,6 @@ class ProcessResponseInvalidOutput(ManagerTestCase):
 
     def test_invalid_process_response(self):
         req = Request('http://example.com/index.html')
-        resp = Response('http://example.com/index.html')
 
         class InvalidProcessResponseMiddleware:
             def process_response(self, request, response, spider):
@@ -163,7 +161,6 @@ class ProcessExceptionInvalidOutput(ManagerTestCase):
 
     def test_invalid_process_exception(self):
         req = Request('http://example.com/index.html')
-        resp = Response('http://example.com/index.html')
 
         class InvalidProcessExceptionMiddleware:
             def process_request(self, request, spider):

From 8d1e0e09bb6fdeb4f1348b408a268c92dc9e7a8f Mon Sep 17 00:00:00 2001
From: Mabel Villalba <mabelvj@gmail.com>
Date: Thu, 20 Jun 2019 10:06:06 +0200
Subject: [PATCH 2069/2974] [itemloader-errors] added error message in
 get_value

---
 scrapy/loader/__init__.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 20f0f90c3..5055de015 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -106,11 +106,17 @@ class ItemLoader(object):
             value = arg_to_iter(value)
             value = flatten(extract_regex(regex, x) for x in value)
 
-        for proc in processors:
+        for _proc in processors:
             if value is None:
                 break
-            proc = wrap_loader_context(proc, self.context)
-            value = proc(value)
+            proc = wrap_loader_context(_proc, self.context)
+            try:
+                value = proc(value)
+            except Exception as e:
+                raise ValueError(
+                    "Error with processor %s value=%r error='%s: %s'" %
+                    (_proc.__class__.__name__, value, type(e).__name__,
+                     str(e)))
         return value
 
     def load_item(self):

From 663352b2a5250c377bbbe2d53c5d5b7da3a1836a Mon Sep 17 00:00:00 2001
From: Mabel Villalba <mabelvj@gmail.com>
Date: Thu, 20 Jun 2019 10:10:16 +0200
Subject: [PATCH 2070/2974] [itemloader-errors] added error message to
 _process_input_value

---
 scrapy/loader/__init__.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 5055de015..fc8b10bee 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -155,9 +155,15 @@ class ItemLoader(object):
         return proc
 
     def _process_input_value(self, field_name, value):
-        proc = self.get_input_processor(field_name)
-        proc = wrap_loader_context(proc, self.context)
-        return proc(value)
+        _proc = self.get_input_processor(field_name)
+        proc = wrap_loader_context(_proc, self.context)
+        try:
+            return proc(value)
+        except Exception as e:
+            raise ValueError(
+                "Error with inputput processor %s: field=%r value=%r "
+                "error='%s: %s'" % (_proc.__class__.__name__, field_name,
+                                    value, type(e).__name__, str(e)))
 
     def _get_item_field_attr(self, field_name, key, default=None):
         if isinstance(self.item, Item):

From f4f2b1695c4d7bc69e5cb19c33a3a47f69bd1e8d Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Mon, 24 Jun 2019 07:38:05 -0300
Subject: [PATCH 2071/2974] Fix a memory leak on the Media Pipeline (Files and
 Images) (#3813)

We're storing exceptions captured by Twisted on the media pipeline
cache, but we're also using the defer.returnValue method with our
own methods decorated with @defer.inlineCallbacks.

The defer.returnValue method passes returned values forward by
throwing a defer._DefGen_Return exception, which in its turn
extends the BaseException class and is captured by Twisted.

This way, the latest exception stored in the Failure's object may
also have an HtmlResponse object in its __context__ attribute. As
the Response object also keeps track of the Request object that
has originated it, you could figure it out how many RAM we're
wasting here.

This could easily lead to a Memory Leak problem when running
spiders with Media Pipeline enabled and a particular Request set
that tends to raise a significant number of exceptions.

Example triggers:
- media requests with 404 status responses
- user land exceptins coming from custom middlewares
- etc.
---
 scrapy/pipelines/media.py    | 26 +++++++++++-
 tests/test_pipeline_media.py | 77 +++++++++++++++++++++++++++++++++++-
 2 files changed, 101 insertions(+), 2 deletions(-)

diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 404bbf5bf..95dca9a3f 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -3,7 +3,7 @@ from __future__ import print_function
 import functools
 import logging
 from collections import defaultdict
-from twisted.internet.defer import Deferred, DeferredList
+from twisted.internet.defer import Deferred, DeferredList, _DefGen_Return
 from twisted.python.failure import Failure
 
 from scrapy.settings import Settings
@@ -139,6 +139,30 @@ class MediaPipeline(object):
             result.cleanFailure()
             result.frames = []
             result.stack = None
+
+            # This code fixes a memory leak by avoiding to keep references to
+            # the Request and Response objects on the Media Pipeline cache.
+            #
+            # Twisted inline callbacks pass return values using the function
+            # twisted.internet.defer.returnValue, which encapsulates the return
+            # value inside a _DefGen_Return base exception.
+            #
+            # What happens when the media_downloaded callback raises another
+            # exception, for example a FileException('download-error') when
+            # the Response status code is not 200 OK, is that it stores the
+            # _DefGen_Return exception on the FileException context.
+            #
+            # To avoid keeping references to the Response and therefore Request
+            # objects on the Media Pipeline cache, we should wipe the context of
+            # the exception encapsulated by the Twisted Failure when its a
+            # _DefGen_Return instance.
+            #
+            # This problem does not occur in Python 2.7 since we don't have
+            # Exception Chaining (https://www.python.org/dev/peps/pep-3134/).
+            context = getattr(result.value, '__context__', None)
+            if isinstance(context, _DefGen_Return):
+                setattr(result.value, '__context__', None)
+
         info.downloading.remove(fp)
         info.downloaded[fp] = result  # cache result
         for wad in info.waiting.pop(fp):
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 5f6a6d9e6..28e39cefa 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,15 +1,19 @@
 from __future__ import print_function
+
+import sys
+
 from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.python.failure import Failure
 from twisted.internet import reactor
-from twisted.internet.defer import Deferred, inlineCallbacks
+from twisted.internet.defer import Deferred, inlineCallbacks, returnValue
 
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.utils.request import request_fingerprint
 from scrapy.pipelines.media import MediaPipeline
+from scrapy.pipelines.files import FileException
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.signal import disconnect_all
 from scrapy import signals
@@ -90,6 +94,77 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         self.pipe._modify_media_request(request)
         assert request.meta == {'handle_httpstatus_all': True}
 
+    def test_should_remove_req_res_references_before_caching_the_results(self):
+        """Regression test case to prevent a memory leak in the Media Pipeline.
+
+        The memory leak is triggered when an exception is raised when a Response
+        scheduled by the Media Pipeline is being returned. For example, when a
+        FileException('download-error') is raised because the Response status
+        code is not 200 OK.
+
+        It happens because we are keeping a reference to the Response object
+        inside the FileException context. This is caused by the way Twisted
+        return values from inline callbacks. It raises a custom exception
+        encapsulating the original return value.
+
+        The solution is to remove the exception context when this context is a
+        _DefGen_Return instance, the BaseException used by Twisted to pass the
+        returned value from those inline callbacks.
+
+        Maybe there's a better and more reliable way to test the case described
+        here, but it would be more complicated and involve running - or at least
+        mocking - some async steps from the Media Pipeline. The current test
+        case is simple and detects the problem very fast. On the other hand, it
+        would not detect another kind of leak happening due to old object
+        references being kept inside the Media Pipeline cache.
+
+        This problem does not occur in Python 2.7 since we don't have Exception
+        Chaining (https://www.python.org/dev/peps/pep-3134/).
+        """
+        # Create sample pair of Request and Response objects
+        request = Request('http://url')
+        response = Response('http://url', body=b'', request=request)
+
+        # Simulate the Media Pipeline behavior to produce a Twisted Failure
+        try:
+            # Simulate a Twisted inline callback returning a Response
+            # The returnValue method raises an exception encapsulating the value
+            returnValue(response)
+        except BaseException as exc:
+            def_gen_return_exc = exc
+            try:
+                # Simulate the media_downloaded callback raising a FileException
+                # This usually happens when the status code is not 200 OK
+                raise FileException('download-error')
+            except Exception as exc:
+                file_exc = exc
+                # Simulate Twisted capturing the FileException
+                # It encapsulates the exception inside a Twisted Failure
+                failure = Failure(file_exc)
+
+        # The Failure should encapsulate a FileException ...
+        self.assertEqual(failure.value, file_exc)
+        # ... and if we're running on Python 3 ...
+        if sys.version_info.major >= 3:
+            # ... it should have the returnValue exception set as its context
+            self.assertEqual(failure.value.__context__, def_gen_return_exc)
+
+        # Let's calculate the request fingerprint and fake some runtime data...
+        fp = request_fingerprint(request)
+        info = self.pipe.spiderinfo
+        info.downloading.add(fp)
+        info.waiting[fp] = []
+
+        # When calling the method that caches the Request's result ...
+        self.pipe._cache_result_and_execute_waiters(failure, fp, info)
+        # ... it should store the Twisted Failure ...
+        self.assertEqual(info.downloaded[fp], failure)
+        # ... encapsulating the original FileException ...
+        self.assertEqual(info.downloaded[fp].value, file_exc)
+        # ... but it should not store the returnValue exception on its context
+        context = getattr(info.downloaded[fp].value, '__context__', None)
+        self.assertIsNone(context)
+
 
 class MockedMediaPipeline(MediaPipeline):
 

From 8a3b15eb91169ab262e4dca60105f56467ecd1ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 27 Mar 2019 08:50:33 +0100
Subject: [PATCH 2072/2974] Document how to select dynamically-loaded content

---
 docs/index.rst                  |   4 +
 docs/topics/dynamic-content.rst | 246 ++++++++++++++++++++++++++++++++
 2 files changed, 250 insertions(+)
 create mode 100644 docs/topics/dynamic-content.rst

diff --git a/docs/index.rst b/docs/index.rst
index cedde8f38..6d5f9e77d 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -158,6 +158,7 @@ Solving specific problems
    topics/practices
    topics/broad-crawls
    topics/developer-tools
+   topics/dynamic-content
    topics/leaks
    topics/media-pipeline
    topics/deploy
@@ -183,6 +184,9 @@ Solving specific problems
 :doc:`topics/developer-tools`
     Learn how to scrape with your browser's developer tools.
 
+:doc:`topics/dynamic-content`
+    Read webpage data that is loaded dynamically.
+
 :doc:`topics/leaks`
     Learn how to find and get rid of memory leaks in your crawler.
 
diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
new file mode 100644
index 000000000..8b5dacf56
--- /dev/null
+++ b/docs/topics/dynamic-content.rst
@@ -0,0 +1,246 @@
+.. _topics-dynamic-content:
+
+====================================
+Selecting dynamically-loaded content
+====================================
+
+Some webpages show the desired data when you load them in a web browser.
+However, when you download them using Scrapy, you cannot reach the desired data
+using :ref:`selectors <topics-selectors>`.
+
+When this happens, the recommended approach is to
+:ref:`find the data source <topics-finding-data-source>` and extract the data
+from it.
+
+If you fail to do that, and you can nonetheless access the desired data through
+the :ref:`DOM <topics-livedom>` from your web browser, see
+:ref:`topics-javascript-rendering`.
+
+.. _topics-finding-data-source:
+
+Finding the data source
+=======================
+
+To extract the desired data, you must first find its source location.
+
+If the data is in a non-text-based format, such as an image or a PDF document,
+use the :ref:`network tool <topics-network-tool>` of your web browser to find
+the corresponding request, and :ref:`reproduce it
+<topics-reproducing-requests>`.
+
+If your web browser lets you select the desired data as text, the data may be
+defined in embedded JavaScript code, or loaded from an external resource in a
+text-based format.
+
+In that case, you can use a tool like wgrep_ to find the URL of that resource.
+
+If the data turns out to come from the original URL itself, you must
+:ref:`inspect the source code of the webpage <topics-inspecting-source>` to
+determine where the data is located.
+
+If the data comes from a different URL, you will need to :ref:`reproduce the
+corresponding request <topics-reproducing-requests>`.
+
+.. _topics-inspecting-source:
+
+Inspecting the source code of a webpage
+=======================================
+
+Sometimes you need to inspect the source code of a webpage (not the
+:ref:`DOM <topics-livedom>`) to determine where some desired data is located.
+
+Use Scrapy’s :command:`fetch` command to download the webpage contents as seen
+by Scrapy::
+
+    scrapy fetch --nolog https://example.com > response.html
+
+If the desired data is in embedded JavaScript code within a ``<script/>``
+element, see :ref:`topics-parsing-javascript`.
+
+If you cannot find the desired data, first make sure it’s not just Scrapy:
+download the webpage with an HTTP client like curl_ or wget_ and see if the
+information can be found in the response they get.
+
+If they get a response with the desired data, modify your Scrapy
+:class:`~scrapy.http.Request` to match that of the other HTTP client. For
+example, try using the same user-agent string (:setting:`USER_AGENT`) or the
+same :attr:`~scrapy.http.Request.headers`.
+
+If they also get a response without the desired data, you’ll need to take
+steps to make your request more similar to that of the web browser. See
+:ref:`topics-reproducing-requests`.
+
+.. _topics-reproducing-requests:
+
+Reproducing requests
+====================
+
+Sometimes we need to reproduce a request the way our web browser performs it.
+
+Use the :ref:`network tool <topics-network-tool>` of your web browser to see
+how your web browser performs the desired request, and try to reproduce that
+request with Scrapy.
+
+It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP
+method and URL. However, you may also need to reproduce the body, headers and
+form parameters (see :class:`~scrapy.http.FormRequest`) of that request.
+
+Once you get the expected response, you can :ref:`extract the desired data from
+it <topics-handling-response-formats>`.
+
+You can reproduce any request with Scrapy. However, some times reproducing all
+necessary requests may not seem efficient in developer time. If that is your
+case, and crawling speed is not a major concern for you, you can alternatively
+consider :ref:`JavaScript pre-rendering <topics-javascript-rendering>`.
+
+If you get the expected response `sometimes`, but not always, the issue is
+probably not your request, but the target server. The target server might be
+buggy, overloaded, or :ref:`banning <bans>` some of your requests.
+
+.. _topics-handling-response-formats:
+
+Handling different response formats
+===================================
+
+Once you have a response with the desired data, how you extract the desired
+data from it depends on the type of response:
+
+-   If the response is HTML or XML, use :ref:`selectors
+    <topics-selectors>` as usual.
+
+-   If the response is JSON, use `json.loads`_ to load the desired data from
+    :attr:`response.text <scrapy.http.TextResponse.text>`::
+
+        data = json.loads(response.text)
+
+    If the desired data is inside HTML or XML code embedded within JSON data,
+    you can load that HTML or XML code into a
+    :class:`~scrapy.selector.Selector` and then
+    :ref:`use it <topics-selectors>` as usual::
+
+        selector = Selector(data['html'])
+
+-   If the response is JavaScript, or HTML with a ``<script/>`` element
+    containing the desired data, see :ref:`topics-parsing-javascript`.
+
+-   If the response is CSS, use a `regular expression`_ to extract the desired
+    data from :attr:`response.text <scrapy.http.TextResponse.text>`.
+
+.. _topics-parsing-images:
+
+-   If the response is an image or another format based on images (e.g. PDF),
+    read the response as bytes from
+    :attr:`response.body <scrapy.http.TextResponse.body>` and use an OCR
+    solution to extract the desired data as text.
+
+    For example, you can use pytesseract_. To read a table from a PDF,
+    `tabula-py`_ may be a better choice.
+
+-   If the response is SVG, or HTML with embedded SVG containing the desired
+    data, you may be able to extract the desired data using
+    :ref:`selectors <topics-selectors>`, since SVG is based on XML.
+
+    Otherwise, you might need to convert the SVG code into a raster image, and
+    :ref:`handle that raster image <topics-parsing-images>`.
+
+.. _topics-parsing-javascript:
+
+Parsing JavaScript code
+=======================
+
+If the desired data is hardcoded in JavaScript, you first need to get the
+JavaScript code:
+
+-   If the JavaScript code is in a JavaScript file, simply read
+    :attr:`response.text <scrapy.http.TextResponse.text>`.
+
+-   If the JavaScript code is within a ``<script/>`` element of an HTML page,
+    use :ref:`selectors <topics-selectors>` to extract the text within that
+    ``<script/>`` element.
+
+Once you have a string with the JavaScript code, you can extract the desired
+data from it:
+
+-   You might be able to use a `regular expression`_ to extract the desired
+    data in JSON format, which you can then parse with `json.loads`_.
+
+    For example, if the JavaScript code contains a separate line like
+    ``var data = {"field": "value"};`` you can extract that data as follows::
+
+        >>> pattern = r'\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n'
+        >>> json_data = response.css('script::text').re_first(pattern)
+        >>> json.loads(json_data)
+        {'field': 'value'}
+
+-   Otherwise, use js2xml_ to convert the JavaScript code into an XML document
+    that you can parse using :ref:`selectors <topics-selectors>`.
+
+    For example, if the JavaScript code contains
+    ``var data = {field: "value"};`` you can extract that data as follows::
+
+        >>> import js2xml
+        >>> import lxml.etree
+        >>> from parsel import Selector
+        >>> javascript = response.css('script::text').get()
+        >>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding='unicode')
+        >>> selector = Selector(text=xml)
+        >>> selector.css('var[name="data"]').get()
+        '<var name="data"><object><property name="field"><string>value</string></property></object></var>'
+
+.. _topics-javascript-rendering:
+
+Pre-rendering JavaScript
+========================
+
+On webpages that fetch data from additional requests, reproducing those
+requests that contain the desired data is the preferred approach. The effort is
+often worth the result: structured, complete data with minimum parsing time and
+network transfer.
+
+However, sometimes it can be really hard to reproduce certain requests. Or you
+may need something that no request can give you, such as a screenshot of a
+webpage as seen in a web browser.
+
+In these cases use the Splash_ JavaScript-rendering service, along with
+`scrapy-splash`_ for seamless integration.
+
+Splash returns as HTML the :ref:`DOM <topics-livedom>` of a webpage, so that
+you can parse it with :ref:`selectors <topics-selectors>`. It provides great
+flexibility through configuration_ or scripting_.
+
+If you need something beyond what Splash offers, such as interacting with the
+DOM on-the-fly from Python code instead of using a previously-written script,
+or handling multiple web browser windows, you might need to
+:ref:`use a headless browser <topics-headless-browsing>` instead.
+
+.. _configuration: https://splash.readthedocs.io/en/stable/api.html
+.. _scripting: https://splash.readthedocs.io/en/stable/scripting-tutorial.html
+
+.. _topics-headless-browsing:
+
+Using a headless browser
+========================
+
+A `headless browser`_ is a special web browser that provides an API for
+automation.
+
+The easiest way to use a headless browser with Scrapy is to use Selenium_,
+along with `scrapy-selenium`_ for seamless integration.
+
+
+.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
+.. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
+.. _curl: https://curl.haxx.se/
+.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
+.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
+.. _js2xml: https://github.com/scrapinghub/js2xml
+.. _json.loads: https://docs.python.org/library/json.html#json.loads
+.. _pytesseract: https://github.com/madmaze/pytesseract
+.. _regular expression: https://docs.python.org/library/re.html
+.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
+.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
+.. _Selenium: https://www.seleniumhq.org/
+.. _Splash: https://github.com/scrapinghub/splash
+.. _tabula-py: https://github.com/chezou/tabula-py
+.. _wget: https://www.gnu.org/software/wget/
+.. _wgrep: https://github.com/stav/wgrep
\ No newline at end of file

From 859008a10b8e126844662867195c6823c4a59e4f Mon Sep 17 00:00:00 2001
From: Mabel Villalba <mabelvj@gmail.com>
Date: Thu, 20 Jun 2019 10:13:59 +0200
Subject: [PATCH 2073/2974] [itemloader-errors] added error message to Compose
 and MapCompose

Fixes issue #3836
---
 scrapy/loader/__init__.py   | 16 ++++++++--------
 scrapy/loader/processors.py | 17 ++++++++++++++---
 2 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index fc8b10bee..ec4102b11 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -128,14 +128,6 @@ class ItemLoader(object):
 
         return item
 
-    def get_output_value(self, field_name):
-        proc = self.get_output_processor(field_name)
-        proc = wrap_loader_context(proc, self.context)
-        try:
-            return proc(self._values[field_name])
-        except Exception as e:
-            raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" % \
-                (field_name, self._values[field_name], type(e).__name__, str(e)))
 
     def get_collected_values(self, field_name):
         return self._values[field_name]
@@ -145,6 +137,14 @@ class ItemLoader(object):
         if not proc:
             proc = self._get_item_field_attr(field_name, 'input_processor', \
                 self.default_input_processor)
+    def get_output_value(self, field_name):
+        proc = self.get_output_processor(field_name)
+        proc = wrap_loader_context(proc, self.context)
+        try:
+            return proc(self._values[field_name])
+        except Exception as e:
+            raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" % \
+                (field_name, self._values[field_name], type(e).__name__, str(e)))
         return proc
 
     def get_output_processor(self, field_name):
diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index bf7c74bfe..85ac2c832 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -25,8 +25,13 @@ class MapCompose(object):
         for func in wrapped_funcs:
             next_values = []
             for v in values:
-                next_values += arg_to_iter(func(v))
-            values = next_values
+                try:
+                    next_values += arg_to_iter(func(v))
+                except Exception as e:
+                    raise ValueError("Error in MapCompose with "
+                                     "function %s value=%r error='%s: %s'" %
+                                     (func.__name__, value,
+                                      type(e).__name__, str(e)))
         return values
 
 
@@ -46,7 +51,13 @@ class Compose(object):
         for func in wrapped_funcs:
             if value is None and self.stop_on_none:
                 break
-            value = func(value)
+            try:
+                value = func(value)
+            except Exception as e:
+                raise ValueError("Error in Compose with "
+                                 "function %s value=%r error='%s: %s'" %
+                                 (func.__name__, value,
+                                  type(e).__name__, str(e)))
         return value
 
 

From e5d17b4efe23a617eacadf602559e481641d3590 Mon Sep 17 00:00:00 2001
From: Mabel Villalba <mabelvj@gmail.com>
Date: Tue, 25 Jun 2019 13:42:40 +0200
Subject: [PATCH 2074/2974] [itemloader-errors] reordered method

 #3836
---
 scrapy/loader/__init__.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index ec4102b11..38afa951a 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -128,6 +128,15 @@ class ItemLoader(object):
 
         return item
 
+    def get_output_value(self, field_name):
+        proc = self.get_output_processor(field_name)
+        proc = wrap_loader_context(proc, self.context)
+        try:
+            return proc(self._values[field_name])
+        except Exception as e:
+            raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" % \
+                (field_name, self._values[field_name], type(e).__name__, str(e)))
+        return proc
 
     def get_collected_values(self, field_name):
         return self._values[field_name]
@@ -137,14 +146,6 @@ class ItemLoader(object):
         if not proc:
             proc = self._get_item_field_attr(field_name, 'input_processor', \
                 self.default_input_processor)
-    def get_output_value(self, field_name):
-        proc = self.get_output_processor(field_name)
-        proc = wrap_loader_context(proc, self.context)
-        try:
-            return proc(self._values[field_name])
-        except Exception as e:
-            raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" % \
-                (field_name, self._values[field_name], type(e).__name__, str(e)))
         return proc
 
     def get_output_processor(self, field_name):

From f134b1daf3fb36372f8c1755aaa4c29ac65aced3 Mon Sep 17 00:00:00 2001
From: Mabel Villalba <mabelvj@gmail.com>
Date: Tue, 25 Jun 2019 13:45:06 +0200
Subject: [PATCH 2075/2974] [itemloader-errors] reordered method

 #3836
---
 scrapy/loader/__init__.py   | 1 -
 scrapy/loader/processors.py | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 38afa951a..fc8b10bee 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -136,7 +136,6 @@ class ItemLoader(object):
         except Exception as e:
             raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" % \
                 (field_name, self._values[field_name], type(e).__name__, str(e)))
-        return proc
 
     def get_collected_values(self, field_name):
         return self._values[field_name]
diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index 85ac2c832..56f2bc564 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -32,6 +32,7 @@ class MapCompose(object):
                                      "function %s value=%r error='%s: %s'" %
                                      (func.__name__, value,
                                       type(e).__name__, str(e)))
+            values = next_values
         return values
 
 

From ad8c980010b60d1d9ca3eb30c27fb60751b10508 Mon Sep 17 00:00:00 2001
From: Mabel Villalba <mabelvj@gmail.com>
Date: Tue, 25 Jun 2019 13:54:47 +0200
Subject: [PATCH 2076/2974] [itemloader-errors] undo _proc

 #3836
---
 scrapy/loader/__init__.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index fc8b10bee..30ff5a717 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -106,17 +106,17 @@ class ItemLoader(object):
             value = arg_to_iter(value)
             value = flatten(extract_regex(regex, x) for x in value)
 
-        for _proc in processors:
+        for proc in processors:
             if value is None:
                 break
-            proc = wrap_loader_context(_proc, self.context)
+            _proc = proc
+            proc = wrap_loader_context(proc, self.context)
             try:
                 value = proc(value)
             except Exception as e:
-                raise ValueError(
-                    "Error with processor %s value=%r error='%s: %s'" %
-                    (_proc.__class__.__name__, value, type(e).__name__,
-                     str(e)))
+                raise ValueError("Error with processor %s value=%r error='%s: %s'" %
+                                 (_proc.__class__.__name__, value,
+                                  type(e).__name__, str(e)))
         return value
 
     def load_item(self):
@@ -155,8 +155,9 @@ class ItemLoader(object):
         return proc
 
     def _process_input_value(self, field_name, value):
-        _proc = self.get_input_processor(field_name)
-        proc = wrap_loader_context(_proc, self.context)
+        proc = self.get_input_processor(field_name)
+        _proc = proc
+        proc = wrap_loader_context(proc, self.context)
         try:
             return proc(value)
         except Exception as e:

From ef56e34a4a9e02dd4bd02b3b82c5bd9b24453e49 Mon Sep 17 00:00:00 2001
From: Mabel Villalba <mabelvj@gmail.com>
Date: Tue, 25 Jun 2019 13:56:53 +0200
Subject: [PATCH 2077/2974] [itemloader-errors] fixed typo

 #3836
---
 scrapy/loader/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 30ff5a717..844e3828c 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -162,7 +162,7 @@ class ItemLoader(object):
             return proc(value)
         except Exception as e:
             raise ValueError(
-                "Error with inputput processor %s: field=%r value=%r "
+                "Error with input processor %s: field=%r value=%r "
                 "error='%s: %s'" % (_proc.__class__.__name__, field_name,
                                     value, type(e).__name__, str(e)))
 

From 99eb03a810273b7451d0ce3614d18b20cb604912 Mon Sep 17 00:00:00 2001
From: Mabel Villalba <mabelvj@gmail.com>
Date: Tue, 25 Jun 2019 14:17:54 +0200
Subject: [PATCH 2078/2974] [itemloader-errors] adapted compose test for the
 new error being returned by compose

 #3836
---
 tests/test_loader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_loader.py b/tests/test_loader.py
index 5a8ee1b2e..230242592 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -482,7 +482,7 @@ class ProcessorsTest(unittest.TestCase):
         proc = Compose(str.upper)
         self.assertEqual(proc(None), None)
         proc = Compose(str.upper, stop_on_none=False)
-        self.assertRaises(TypeError, proc, None)
+        self.assertRaises(ValueError, proc, None)
 
     def test_mapcompose(self):
         filter_world = lambda x: None if x == 'world' else x

From a753ea7e161e049a090d24b0807a4f20ad7cf36f Mon Sep 17 00:00:00 2001
From: Mabel Villalba <mabelvj@gmail.com>
Date: Tue, 25 Jun 2019 14:35:49 +0200
Subject: [PATCH 2079/2974] [itemloader-errors] added test for Compose and
 MapCompose

 #3836
---
 tests/test_loader.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/test_loader.py b/tests/test_loader.py
index 230242592..0f9070b8c 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -483,12 +483,21 @@ class ProcessorsTest(unittest.TestCase):
         self.assertEqual(proc(None), None)
         proc = Compose(str.upper, stop_on_none=False)
         self.assertRaises(ValueError, proc, None)
+        proc = Compose(str.upper, lambda x: x + 1)
+        self.assertRaises(ValueError, proc, 'hello')
 
     def test_mapcompose(self):
         filter_world = lambda x: None if x == 'world' else x
         proc = MapCompose(filter_world, six.text_type.upper)
         self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
                          [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
+        proc = MapCompose(filter_world, six.text_type.upper)
+        self.assertEqual(proc(None), [])
+        proc = MapCompose(filter_world, six.text_type.upper)
+        self.assertRaises(ValueError, proc, [1])
+        proc = MapCompose(filter_world, lambda x: x + 1)
+        self.assertRaises(ValueError, proc, 'hello')
+
 
 
 class SelectortemLoaderTest(unittest.TestCase):

From b1e348b2813880c616191d0135e759f4edcec64c Mon Sep 17 00:00:00 2001
From: Mabel Villalba <mabelvj@gmail.com>
Date: Tue, 25 Jun 2019 17:00:40 +0200
Subject: [PATCH 2080/2974] [itemloader-errors] updated Compose and MapCompose
 messages and added tests

 #3836
---
 scrapy/loader/processors.py | 12 ++++++++----
 tests/test_loader.py        | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index 56f2bc564..4b6d8624c 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -28,10 +28,12 @@ class MapCompose(object):
                 try:
                     next_values += arg_to_iter(func(v))
                 except Exception as e:
+                    type_name = type(func).__name__
+                    _name = (func.__name__ if type_name == 'function'
+                             else type_name)
                     raise ValueError("Error in MapCompose with "
                                      "function %s value=%r error='%s: %s'" %
-                                     (func.__name__, value,
-                                      type(e).__name__, str(e)))
+                                     (_name, value, type(e).__name__, str(e)))
             values = next_values
         return values
 
@@ -55,10 +57,12 @@ class Compose(object):
             try:
                 value = func(value)
             except Exception as e:
+                type_name = type(func).__name__
+                _name = (func.__name__ if type_name == 'function' 
+                         else type_name)
                 raise ValueError("Error in Compose with "
                                  "function %s value=%r error='%s: %s'" %
-                                 (func.__name__, value,
-                                  type(e).__name__, str(e)))
+                                 (_name, value, type(e).__name__, str(e)))
         return value
 
 
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 0f9070b8c..ce0fa0701 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -456,6 +456,42 @@ class BasicItemLoaderTest(unittest.TestCase):
             'title': [u'Test item title 3', u'Test item 4'],
         })
 
+    def test_error_input_processor(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+            name_in = MapCompose(float)
+
+        il = TestItemLoader()
+        self.assertRaises(ValueError, il.add_value, 'name',
+                          [u'marta', u'other'])
+
+    def test_error_output_processor(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+            name_out = Compose(Join(), float)
+
+        il = TestItemLoader()
+        il.add_value('name', u'marta')
+        with self.assertRaises(ValueError):
+            il.load_item()
+
+    def test_error_processor_as_argument(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+
+        il = TestItemLoader()
+        self.assertRaises(ValueError, il.add_value, 'name',
+                          [u'marta', u'other'], Compose(float))
+
 
 class ProcessorsTest(unittest.TestCase):
 

From d7795484dcadff27ae821567631d2141b8edcbc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 25 Jun 2019 17:29:49 +0200
Subject: [PATCH 2081/2974] Document scrapy.crawler.Crawler.stop()

---
 docs/topics/api.rst | 2 ++
 scrapy/crawler.py   | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index e1623287d..7c8c40b5f 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -99,6 +99,8 @@ how you :ref:`configure the downloader middlewares
 
         Returns a deferred that is fired when the crawl is finished.
 
+    .. automethod:: stop
+
 .. autoclass:: CrawlerRunner
    :members:
 
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 99bf48385..151ae4f62 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -111,6 +111,8 @@ class Crawler(object):
 
     @defer.inlineCallbacks
     def stop(self):
+        """Starts a graceful stop of the crawler and returns a deferred that is
+        fired when the crawler is stopped."""
         if self.crawling:
             self.crawling = False
             yield defer.maybeDeferred(self.engine.stop)

From 268a37cb7b40737a855e217e3ebe8a2b91238dba Mon Sep 17 00:00:00 2001
From: Mabel Villalba <mabelvj@gmail.com>
Date: Wed, 26 Jun 2019 13:23:10 +0200
Subject: [PATCH 2082/2974] [itemloader-errors] updated Compose and MapCompose
 function names

 #3836
---
 scrapy/loader/processors.py | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index 4b6d8624c..468aec2cc 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -28,12 +28,10 @@ class MapCompose(object):
                 try:
                     next_values += arg_to_iter(func(v))
                 except Exception as e:
-                    type_name = type(func).__name__
-                    _name = (func.__name__ if type_name == 'function'
-                             else type_name)
                     raise ValueError("Error in MapCompose with "
-                                     "function %s value=%r error='%s: %s'" %
-                                     (_name, value, type(e).__name__, str(e)))
+                                     "%s value=%r error='%s: %s'" %
+                                     (str(func), value, type(e).__name__,
+                                      str(e)))
             values = next_values
         return values
 
@@ -57,12 +55,9 @@ class Compose(object):
             try:
                 value = func(value)
             except Exception as e:
-                type_name = type(func).__name__
-                _name = (func.__name__ if type_name == 'function' 
-                         else type_name)
                 raise ValueError("Error in Compose with "
-                                 "function %s value=%r error='%s: %s'" %
-                                 (_name, value, type(e).__name__, str(e)))
+                                 "%s value=%r error='%s: %s'" %
+                                 (str(func), value, type(e).__name__, str(e)))
         return value
 
 

From 1f9f41b85055c25770951c899fe839d96aa2060c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 26 Jun 2019 12:31:12 -0300
Subject: [PATCH 2083/2974] Move request.cb_kwargs tests to their own test file

---
 tests/spiders.py                |  61 ------------------
 tests/test_crawl.py             |  28 +--------
 tests/test_request_cb_kwargs.py | 106 ++++++++++++++++++++++++++++++++
 3 files changed, 107 insertions(+), 88 deletions(-)
 create mode 100644 tests/test_request_cb_kwargs.py

diff --git a/tests/spiders.py b/tests/spiders.py
index 8c8d50ff5..7816bf7c7 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -28,67 +28,6 @@ class MetaSpider(MockServerSpider):
         self.meta['close_reason'] = reason
 
 
-class KeywordArgumentsSpider(MockServerSpider):
-
-    name = 'kwargs'
-    checks = list()
-
-    def start_requests(self):
-        data = {'key': 'value', 'number': 123}
-        yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data)
-        yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
-        yield Request(self.mockserver.url('/general_without'), self.parse_general)
-        yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
-        yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data)
-        yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)
-        yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)
-
-    def parse_first(self, response, key, number):
-        self.checks.append(key == 'value')
-        self.checks.append(number == 123)
-        self.crawler.stats.inc_value('boolean_checks', 2)
-        yield response.follow(
-            self.mockserver.url('/two'),
-            self.parse_second,
-            cb_kwargs={'new_key': 'new_value'})
-
-    def parse_second(self, response, new_key):
-        self.checks.append(new_key == 'new_value')
-        self.crawler.stats.inc_value('boolean_checks')
-
-    def parse_general(self, response, **kwargs):
-        if response.url.endswith('/general_with'):
-            self.checks.append(kwargs['key'] == 'value')
-            self.checks.append(kwargs['number'] == 123)
-            self.crawler.stats.inc_value('boolean_checks', 2)
-        elif response.url.endswith('/general_without'):
-            self.checks.append(kwargs == {})
-            self.crawler.stats.inc_value('boolean_checks')
-
-    def parse_no_kwargs(self, response):
-        self.checks.append(response.url.endswith('/no_kwargs'))
-        self.crawler.stats.inc_value('boolean_checks')
-
-    def parse_default(self, response, key, number=None, default=99):
-        self.checks.append(response.url.endswith('/default'))
-        self.checks.append(key == 'value')
-        self.checks.append(number == 123)
-        self.checks.append(default == 99)
-        self.crawler.stats.inc_value('boolean_checks', 4)
-
-    def parse_takes_less(self, response, key):
-        """
-        Should raise
-        TypeError: parse_takes_less() got an unexpected keyword argument 'number'
-        """
-
-    def parse_takes_more(self, response, key, number, other):
-        """
-        Should raise
-        TypeError: parse_takes_more() missing 1 required positional argument: 'other'
-        """
-
-
 class FollowAllSpider(MetaSpider):
 
     name = 'follow'
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 2b3e56ee9..3fc13eeb7 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -4,20 +4,17 @@ import logging
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
-import six
 
 from scrapy.http import Request
 from scrapy.crawler import CrawlerRunner
 from scrapy.utils.python import to_unicode
-from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, KeywordArgumentsSpider, \
+from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
     BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
 from tests.mockserver import MockServer
 
 
 class CrawlTestCase(TestCase):
 
-    maxDiff = None
-
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
@@ -26,29 +23,6 @@ class CrawlTestCase(TestCase):
     def tearDown(self):
         self.mockserver.__exit__(None, None, None)
 
-    @defer.inlineCallbacks
-    def test_callback_kwargs(self):
-        crawler = self.runner.create_crawler(KeywordArgumentsSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(mockserver=self.mockserver)
-        self.assertTrue(all(crawler.spider.checks))
-        self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks'))
-        # check exceptions for argument mismatch
-        exceptions = {}
-        for line in log.records:
-            for key in ('takes_less', 'takes_more'):
-                if key in line.getMessage():
-                    exceptions[key] = line
-        self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
-        self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
-        self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
-        # py2 and py3 messages are different
-        exc_message = str(exceptions['takes_more'].exc_info[1])
-        if six.PY2:
-            self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
-        elif six.PY3:
-            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
-
     @defer.inlineCallbacks
     def test_follow_all(self):
         crawler = self.runner.create_crawler(FollowAllSpider)
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
new file mode 100644
index 000000000..ee7117c8a
--- /dev/null
+++ b/tests/test_request_cb_kwargs.py
@@ -0,0 +1,106 @@
+from testfixtures import LogCapture
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+import six
+
+from scrapy.http import Request
+from scrapy.crawler import CrawlerRunner
+from tests.spiders import MockServerSpider
+from tests.mockserver import MockServer
+
+
+class KeywordArgumentsSpider(MockServerSpider):
+
+    name = 'kwargs'
+    checks = list()
+
+    def start_requests(self):
+        data = {'key': 'value', 'number': 123}
+        yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data)
+        yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
+        yield Request(self.mockserver.url('/general_without'), self.parse_general)
+        yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
+        yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data)
+        yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)
+        yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)
+
+    def parse_first(self, response, key, number):
+        self.checks.append(key == 'value')
+        self.checks.append(number == 123)
+        self.crawler.stats.inc_value('boolean_checks', 2)
+        yield response.follow(
+            self.mockserver.url('/two'),
+            self.parse_second,
+            cb_kwargs={'new_key': 'new_value'})
+
+    def parse_second(self, response, new_key):
+        self.checks.append(new_key == 'new_value')
+        self.crawler.stats.inc_value('boolean_checks')
+
+    def parse_general(self, response, **kwargs):
+        if response.url.endswith('/general_with'):
+            self.checks.append(kwargs['key'] == 'value')
+            self.checks.append(kwargs['number'] == 123)
+            self.crawler.stats.inc_value('boolean_checks', 2)
+        elif response.url.endswith('/general_without'):
+            self.checks.append(kwargs == {})
+            self.crawler.stats.inc_value('boolean_checks')
+
+    def parse_no_kwargs(self, response):
+        self.checks.append(response.url.endswith('/no_kwargs'))
+        self.crawler.stats.inc_value('boolean_checks')
+
+    def parse_default(self, response, key, number=None, default=99):
+        self.checks.append(response.url.endswith('/default'))
+        self.checks.append(key == 'value')
+        self.checks.append(number == 123)
+        self.checks.append(default == 99)
+        self.crawler.stats.inc_value('boolean_checks', 4)
+
+    def parse_takes_less(self, response, key):
+        """
+        Should raise
+        TypeError: parse_takes_less() got an unexpected keyword argument 'number'
+        """
+
+    def parse_takes_more(self, response, key, number, other):
+        """
+        Should raise
+        TypeError: parse_takes_more() missing 1 required positional argument: 'other'
+        """
+
+
+class CallbackKeywordArgumentsTestCase(TestCase):
+
+    maxDiff = None
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self.runner = CrawlerRunner()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_callback_kwargs(self):
+        crawler = self.runner.create_crawler(KeywordArgumentsSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(mockserver=self.mockserver)
+        self.assertTrue(all(crawler.spider.checks))
+        self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks'))
+        # check exceptions for argument mismatch
+        exceptions = {}
+        for line in log.records:
+            for key in ('takes_less', 'takes_more'):
+                if key in line.getMessage():
+                    exceptions[key] = line
+        self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
+        self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
+        self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
+        # py2 and py3 messages are different
+        exc_message = str(exceptions['takes_more'].exc_info[1])
+        if six.PY2:
+            self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
+        elif six.PY3:
+            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
\ No newline at end of file

From d4d68cf32187a129b556f371f8bcc24e0ee951ba Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 26 Jun 2019 12:31:41 -0300
Subject: [PATCH 2084/2974] Request.cb_kwargs: update in downloader middleware

---
 tests/test_request_cb_kwargs.py | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index ee7117c8a..25c0bcf54 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -9,9 +9,26 @@ from tests.spiders import MockServerSpider
 from tests.mockserver import MockServer
 
 
-class KeywordArgumentsSpider(MockServerSpider):
+class InjectArgumentsDownloaderMiddleware(object):
+    def process_request(self, request, spider):
+        if request.callback.__name__ == 'parse_downloader_mw':
+            request.cb_kwargs['from_process_request'] = True
+        return None
 
+    def process_response(self, request, response, spider):
+        if request.callback.__name__ == 'parse_downloader_mw':
+            request.cb_kwargs['from_process_response'] = True
+        return response
+
+
+class KeywordArgumentsSpider(MockServerSpider):
     name = 'kwargs'
+    custom_settings = {
+        'DOWNLOADER_MIDDLEWARES': {
+            __name__ + '.InjectArgumentsDownloaderMiddleware': 750,
+        }
+    }
+
     checks = list()
 
     def start_requests(self):
@@ -23,6 +40,7 @@ class KeywordArgumentsSpider(MockServerSpider):
         yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data)
         yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)
         yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)
+        yield Request(self.mockserver.url('/downloader_mw'), self.parse_downloader_mw)
 
     def parse_first(self, response, key, number):
         self.checks.append(key == 'value')
@@ -69,6 +87,14 @@ class KeywordArgumentsSpider(MockServerSpider):
         TypeError: parse_takes_more() missing 1 required positional argument: 'other'
         """
 
+    def parse_downloader_mw(self, response, from_process_request, from_process_response):
+        """
+        Check if downloader middlewares are able to update the keyword arguments
+        """
+        self.checks.append(bool(from_process_request))
+        self.checks.append(bool(from_process_response))
+        self.crawler.stats.inc_value('boolean_checks', 2)
+
 
 class CallbackKeywordArgumentsTestCase(TestCase):
 

From 312e57382c2c49d9676f861ac6cf4202f69a84e5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 26 Jun 2019 12:48:00 -0300
Subject: [PATCH 2085/2974] Request.cb_kwargs: update in spider middleware

---
 tests/test_request_cb_kwargs.py | 47 +++++++++++++++++++++++++++++----
 1 file changed, 42 insertions(+), 5 deletions(-)

diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 25c0bcf54..c9943faa8 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -10,6 +10,9 @@ from tests.mockserver import MockServer
 
 
 class InjectArgumentsDownloaderMiddleware(object):
+    """
+    Make sure downloader middlewares are able to update the keyword arguments
+    """
     def process_request(self, request, spider):
         if request.callback.__name__ == 'parse_downloader_mw':
             request.cb_kwargs['from_process_request'] = True
@@ -21,12 +24,38 @@ class InjectArgumentsDownloaderMiddleware(object):
         return response
 
 
+class InjectArgumentsSpiderMiddleware(object):
+    """
+    Make sure spider middlewares are able to update the keyword arguments
+    """
+    def process_start_requests(self, start_requests, spider):
+        for request in start_requests:
+            if request.callback.__name__ == 'parse_spider_mw':
+                request.cb_kwargs['from_process_start_requests'] = True
+            yield request
+
+    def process_spider_input(self, response, spider):
+        request = response.request
+        if request.callback.__name__ == 'parse_spider_mw':
+            request.cb_kwargs['from_process_spider_input'] = True
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        for element in result:
+            if isinstance(element, Request) and element.callback.__name__ == 'parse_spider_mw_2':
+                element.cb_kwargs['from_process_spider_output'] = True
+            yield element
+
+
 class KeywordArgumentsSpider(MockServerSpider):
     name = 'kwargs'
     custom_settings = {
         'DOWNLOADER_MIDDLEWARES': {
             __name__ + '.InjectArgumentsDownloaderMiddleware': 750,
-        }
+        },
+        'SPIDER_MIDDLEWARES': {
+            __name__ + '.InjectArgumentsSpiderMiddleware': 750,
+        },
     }
 
     checks = list()
@@ -41,6 +70,7 @@ class KeywordArgumentsSpider(MockServerSpider):
         yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)
         yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)
         yield Request(self.mockserver.url('/downloader_mw'), self.parse_downloader_mw)
+        yield Request(self.mockserver.url('/spider_mw'), self.parse_spider_mw)
 
     def parse_first(self, response, key, number):
         self.checks.append(key == 'value')
@@ -88,13 +118,20 @@ class KeywordArgumentsSpider(MockServerSpider):
         """
 
     def parse_downloader_mw(self, response, from_process_request, from_process_response):
-        """
-        Check if downloader middlewares are able to update the keyword arguments
-        """
         self.checks.append(bool(from_process_request))
         self.checks.append(bool(from_process_response))
         self.crawler.stats.inc_value('boolean_checks', 2)
 
+    def parse_spider_mw(self, response, from_process_spider_input, from_process_start_requests):
+        self.checks.append(bool(from_process_spider_input))
+        self.checks.append(bool(from_process_start_requests))
+        self.crawler.stats.inc_value('boolean_checks', 2)
+        return Request(self.mockserver.url('/spider_mw_2'), self.parse_spider_mw_2)
+
+    def parse_spider_mw_2(self, response, from_process_spider_output):
+        self.checks.append(bool(from_process_spider_output))
+        self.crawler.stats.inc_value('boolean_checks', 1)
+
 
 class CallbackKeywordArgumentsTestCase(TestCase):
 
@@ -129,4 +166,4 @@ class CallbackKeywordArgumentsTestCase(TestCase):
         if six.PY2:
             self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
         elif six.PY3:
-            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
\ No newline at end of file
+            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")

From 2e4dc20393c3a678a6576fb20817d4339f66235c Mon Sep 17 00:00:00 2001
From: Claudio Salazar <csalazar@spect.cl>
Date: Wed, 26 Jun 2019 21:36:28 +0200
Subject: [PATCH 2086/2974] Add backward compability when method=None in
 FormRequest

---
 scrapy/http/request/form.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 8b29aae4b..3ce8fc48e 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -18,7 +18,7 @@ from scrapy.utils.response import get_base_url
 
 
 class FormRequest(Request):
-    valid_form_methods = ['GET', 'POST', 'DIALOG']
+    valid_form_methods = ['GET', 'POST']
 
     def __init__(self, *args, **kwargs):
         formdata = kwargs.pop('formdata', None)
@@ -50,9 +50,11 @@ class FormRequest(Request):
         formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
         url = _get_form_url(form, kwargs.pop('url', None))
 
-        method = kwargs.pop('method', form.method).upper()
-        if method not in cls.valid_form_methods:
-            method = 'GET'
+        method = kwargs.pop('method', form.method)
+        if method is not None:
+            method = method.upper()
+            if method not in cls.valid_form_methods:
+                method = 'GET'
 
         return cls(url=url, method=method, formdata=formdata, **kwargs)
 

From 69b1d5d3d7050b5c60a95cc547febaded1b3685f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 5 Oct 2018 18:21:26 +0500
Subject: [PATCH 2087/2974] Log cipher, certificate and temp key info on
 establishing an SSL connection.

---
 scrapy/core/downloader/tls.py | 18 ++++++++++++-
 scrapy/utils/ssl.py           | 50 +++++++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+), 1 deletion(-)
 create mode 100644 scrapy/utils/ssl.py

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index df8051182..2ba72593f 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -2,6 +2,7 @@ import logging
 from OpenSSL import SSL
 
 from scrapy import twisted_version
+from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
 
 
 logger = logging.getLogger(__name__)
@@ -20,6 +21,7 @@ openssl_methods = {
     METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
 }
 
+
 if twisted_version >= (14, 0, 0):
     # ClientTLSOptions requires a recent-enough version of Twisted.
     # Not having ScrapyClientTLSOptions should not matter for older
@@ -65,13 +67,27 @@ if twisted_version >= (14, 0, 0):
         Same as Twisted's private _sslverify.ClientTLSOptions,
         except that VerificationError, CertificateError and ValueError
         exceptions are caught, so that the connection is not closed, only
-        logging warnings.
+        logging warnings. Also, HTTPS connection parameters logging is added.
         """
 
         def _identityVerifyingInfoCallback(self, connection, where, ret):
             if where & SSL_CB_HANDSHAKE_START:
                 set_tlsext_host_name(connection, self._hostnameBytes)
             elif where & SSL_CB_HANDSHAKE_DONE:
+                logger.debug('SSL connection to %s using protocol %s, cipher %s',
+                             self._hostnameASCII,
+                             connection.get_protocol_version_name(),
+                             connection.get_cipher_name(),
+                             )
+                server_cert = connection.get_peer_certificate()
+                logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
+                             x509name_to_string(server_cert.get_issuer()),
+                             x509name_to_string(server_cert.get_subject()),
+                             )
+                key_info = get_temp_key_info(connection._ssl)
+                if key_info:
+                    logger.debug('SSL temp key: %s', key_info)
+
                 try:
                     verifyHostname(connection, self._hostnameASCII)
                 except verification_errors as e:
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
new file mode 100644
index 000000000..5db1608bf
--- /dev/null
+++ b/scrapy/utils/ssl.py
@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+
+import OpenSSL._util as pyOpenSSLutil
+
+from scrapy.utils.python import to_native_str
+
+
+def ffi_buf_to_string(buf):
+    return to_native_str(pyOpenSSLutil.ffi.string(buf))
+
+
+def x509name_to_string(x509name):
+    # from OpenSSL.crypto.X509Name.__repr__
+    result_buffer = pyOpenSSLutil.ffi.new("char[]", 512)
+    pyOpenSSLutil.lib.X509_NAME_oneline(x509name._name, result_buffer, len(result_buffer))
+
+    return ffi_buf_to_string(result_buffer)
+
+
+def get_temp_key_info(ssl_object):
+    if not hasattr(pyOpenSSLutil.lib, 'SSL_get_server_tmp_key'):  # requires OpenSSL 1.0.2
+        return None
+
+    # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key()
+    temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **")
+    pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p)
+    if temp_key_p == pyOpenSSLutil.ffi.NULL:
+        return None
+
+    temp_key = temp_key_p[0]
+    pyOpenSSLutil.ffi.gc(temp_key, pyOpenSSLutil.lib.EVP_PKEY_free)
+    key_info = []
+    key_type = pyOpenSSLutil.lib.EVP_PKEY_id(temp_key)
+    if key_type == pyOpenSSLutil.lib.EVP_PKEY_RSA:
+        key_info.append('RSA')
+    elif key_type == pyOpenSSLutil.lib.EVP_PKEY_DH:
+        key_info.append('DH')
+    elif key_type == pyOpenSSLutil.lib.EVP_PKEY_EC:
+        key_info.append('ECDH')
+        ec_key = pyOpenSSLutil.lib.EVP_PKEY_get1_EC_KEY(temp_key)
+        pyOpenSSLutil.ffi.gc(ec_key, pyOpenSSLutil.lib.EC_KEY_free)
+        nid = pyOpenSSLutil.lib.EC_GROUP_get_curve_name(pyOpenSSLutil.lib.EC_KEY_get0_group(ec_key))
+        cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid)
+        if cname == pyOpenSSLutil.ffi.NULL:
+            cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid)
+        key_info.append(ffi_buf_to_string(cname))
+    else:
+        key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))
+    key_info.append('%s bits' % pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key))
+    return ', '.join(key_info)

From 8f52dada551a7467436e53756be4d8cdbe798380 Mon Sep 17 00:00:00 2001
From: mikolaje <xfl1991@163.com>
Date: Sat, 6 Jul 2019 19:55:09 +0800
Subject: [PATCH 2088/2974] PEP8 in cmdline.py

---
 scrapy/cmdline.py | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index fa2506eb0..fe30ca4f5 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -1,5 +1,6 @@
 from __future__ import print_function
-import sys, os
+import sys
+import os
 import optparse
 import cProfile
 import inspect
@@ -14,6 +15,7 @@ from scrapy.utils.project import inside_project, get_project_settings
 from scrapy.utils.python import garbage_collect
 from scrapy.settings.deprecated import check_deprecated_settings
 
+
 def _iter_command_classes(module_name):
     # TODO: add `name` attribute to commands and and merge this function with
     # scrapy.utils.spider.iter_spider_classes
@@ -25,6 +27,7 @@ def _iter_command_classes(module_name):
                     not obj == ScrapyCommand:
                 yield obj
 
+
 def _get_commands_from_module(module, inproject):
     d = {}
     for cmd in _iter_command_classes(module):
@@ -33,6 +36,7 @@ def _get_commands_from_module(module, inproject):
             d[cmdname] = cmd()
     return d
 
+
 def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
     cmds = {}
     for entry_point in pkg_resources.iter_entry_points(group):
@@ -43,6 +47,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
             raise Exception("Invalid entry point %s" % entry_point.name)
     return cmds
 
+
 def _get_commands_dict(settings, inproject):
     cmds = _get_commands_from_module('scrapy.commands', inproject)
     cmds.update(_get_commands_from_entry_points(inproject))
@@ -51,6 +56,7 @@ def _get_commands_dict(settings, inproject):
         cmds.update(_get_commands_from_module(cmds_module, inproject))
     return cmds
 
+
 def _pop_command_name(argv):
     i = 0
     for arg in argv[1:]:
@@ -59,13 +65,15 @@ def _pop_command_name(argv):
             return arg
         i += 1
 
+
 def _print_header(settings, inproject):
     if inproject:
         print("Scrapy %s - project: %s\n" % (scrapy.__version__, \
-            settings['BOT_NAME']))
+                                             settings['BOT_NAME']))
     else:
         print("Scrapy %s - no active project\n" % scrapy.__version__)
 
+
 def _print_commands(settings, inproject):
     _print_header(settings, inproject)
     print("Usage:")
@@ -80,11 +88,13 @@ def _print_commands(settings, inproject):
     print()
     print('Use "scrapy <command> -h" to see more info about a command')
 
+
 def _print_unknown_command(settings, cmdname, inproject):
     _print_header(settings, inproject)
     print("Unknown command: %s\n" % cmdname)
     print('Use "scrapy" to see available commands')
 
+
 def _run_print_help(parser, func, *a, **kw):
     try:
         func(*a, **kw)
@@ -95,6 +105,7 @@ def _run_print_help(parser, func, *a, **kw):
             parser.print_help()
         sys.exit(2)
 
+
 def execute(argv=None, settings=None):
     if argv is None:
         argv = sys.argv
@@ -111,7 +122,8 @@ def execute(argv=None, settings=None):
         # set EDITOR from environment if available
         try:
             editor = os.environ['EDITOR']
-        except KeyError: pass
+        except KeyError:
+            pass
         else:
             settings['EDITOR'] = editor
     check_deprecated_settings(settings)
@@ -129,7 +141,7 @@ def execute(argv=None, settings=None):
     cmds = _get_commands_dict(settings, inproject)
     cmdname = _pop_command_name(argv)
     parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
-        conflict_handler='resolve')
+                                   conflict_handler='resolve')
     if not cmdname:
         _print_commands(settings, inproject)
         sys.exit(0)
@@ -150,12 +162,14 @@ def execute(argv=None, settings=None):
     _run_print_help(parser, _run_command, cmd, args, opts)
     sys.exit(cmd.exitcode)
 
+
 def _run_command(cmd, args, opts):
     if opts.profile:
         _run_command_profiled(cmd, args, opts)
     else:
         cmd.run(args, opts)
 
+
 def _run_command_profiled(cmd, args, opts):
     if opts.profile:
         sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile)
@@ -165,6 +179,7 @@ def _run_command_profiled(cmd, args, opts):
     if opts.profile:
         p.dump_stats(opts.profile)
 
+
 if __name__ == '__main__':
     try:
         execute()

From 7a398b70868856f6bd17d36b7359e344f08f1440 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Wed, 9 Jan 2019 02:36:22 +0000
Subject: [PATCH 2089/2974] Deprecation removals for Scrapy 1.7

Removing deprecations of 2015 and prior (pre-1.1)
---
 .coveragerc                             |   2 -
 conftest.py                             |   4 -
 scrapy/cmdline.py                       |  16 ----
 scrapy/conf.py                          |  13 ---
 scrapy/core/downloader/__init__.py      |   4 -
 scrapy/core/downloader/handlers/http.py |  12 ---
 scrapy/crawler.py                       |   9 +-
 scrapy/loader/__init__.py               |   8 --
 scrapy/log.py                           |  60 ------------
 scrapy/pipelines/files.py               |  31 +-----
 scrapy/pipelines/images.py              |  64 +------------
 scrapy/selector/__init__.py             |   1 -
 scrapy/selector/csstranslator.py        |  15 ---
 scrapy/selector/lxmlsel.py              |  50 ----------
 scrapy/selector/unified.py              |  37 +------
 scrapy/spiders/__init__.py              |  26 -----
 scrapy/spiders/crawl.py                 |   4 -
 scrapy/telnet.py                        |   7 --
 scrapy/utils/python.py                  |  13 ---
 scrapy/utils/response.py                |   8 --
 tests/test_crawler.py                   |  21 ----
 tests/test_downloader_handlers.py       |  12 +--
 tests/test_pipeline_files.py            |  38 --------
 tests/test_pipeline_images.py           |  57 -----------
 tests/test_selector.py                  | 122 ------------------------
 tests/test_selector_csstranslator.py    |  22 -----
 tests/test_spider.py                    |  75 +--------------
 27 files changed, 8 insertions(+), 723 deletions(-)
 delete mode 100644 scrapy/conf.py
 delete mode 100644 scrapy/log.py
 delete mode 100644 scrapy/selector/csstranslator.py
 delete mode 100644 scrapy/selector/lxmlsel.py
 delete mode 100644 scrapy/telnet.py
 delete mode 100644 tests/test_selector_csstranslator.py

diff --git a/.coveragerc b/.coveragerc
index 1fde07e7e..914d697a0 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -4,5 +4,3 @@ include = scrapy/*
 omit =
   tests/*
   scrapy/xlib/*
-  scrapy/conf.py
-  scrapy/log.py
diff --git a/conftest.py b/conftest.py
index 2d015f5e9..d8531d6cc 100644
--- a/conftest.py
+++ b/conftest.py
@@ -9,10 +9,6 @@ def _py_files(folder):
 
 
 collect_ignore = [
-    # deprecated or moved modules
-    "scrapy/conf.py",
-    "scrapy/log.py",
-
     # not a test, but looks like a test
     "scrapy/utils/testsite.py",
 
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index fa2506eb0..8d5387718 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -99,13 +99,6 @@ def execute(argv=None, settings=None):
     if argv is None:
         argv = sys.argv
 
-    # --- backward compatibility for scrapy.conf.settings singleton ---
-    if settings is None and 'scrapy.conf' in sys.modules:
-        from scrapy import conf
-        if hasattr(conf, 'settings'):
-            settings = conf.settings
-    # ------------------------------------------------------------------
-
     if settings is None:
         settings = get_project_settings()
         # set EDITOR from environment if available
@@ -116,15 +109,6 @@ def execute(argv=None, settings=None):
             settings['EDITOR'] = editor
     check_deprecated_settings(settings)
 
-    # --- backward compatibility for scrapy.conf.settings singleton ---
-    import warnings
-    from scrapy.exceptions import ScrapyDeprecationWarning
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore", ScrapyDeprecationWarning)
-        from scrapy import conf
-        conf.settings = settings
-    # ------------------------------------------------------------------
-
     inproject = inside_project()
     cmds = _get_commands_dict(settings, inproject)
     cmdname = _pop_command_name(argv)
diff --git a/scrapy/conf.py b/scrapy/conf.py
deleted file mode 100644
index 6c40edcdd..000000000
--- a/scrapy/conf.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# This module is kept for backward compatibility, so users can import
-# scrapy.conf.settings and get the settings they expect
-
-import sys
-
-if 'scrapy.cmdline' not in sys.modules:
-    from scrapy.utils.project import get_project_settings
-    settings = get_project_settings()
-
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.conf` is deprecated, use `crawler.settings` attribute instead",
-    ScrapyDeprecationWarning, stacklevel=2)
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 4695d75f4..949dacbc8 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -60,10 +60,6 @@ class Slot(object):
 
 def _get_concurrency_delay(concurrency, spider, settings):
     delay = settings.getfloat('DOWNLOAD_DELAY')
-    if hasattr(spider, 'DOWNLOAD_DELAY'):
-        warnings.warn("%s.DOWNLOAD_DELAY attribute is deprecated, use %s.download_delay instead" %
-                      (type(spider).__name__, type(spider).__name__))
-        delay = spider.DOWNLOAD_DELAY
     if hasattr(spider, 'download_delay'):
         delay = spider.download_delay
 
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index e76823623..ac4b867c3 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -1,15 +1,3 @@
 from __future__ import absolute_import
 from .http10 import HTTP10DownloadHandler
 from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
-
-
-# backward compatibility
-class HttpDownloadHandler(HTTP10DownloadHandler):
-
-    def __init__(self, *args, **kwargs):
-        import warnings
-        from scrapy.exceptions import ScrapyDeprecationWarning
-        warnings.warn('HttpDownloadHandler is deprecated, import scrapy.core.downloader'
-                      '.handlers.http10.HTTP10DownloadHandler instead',
-                      category=ScrapyDeprecationWarning, stacklevel=1)
-        super(HttpDownloadHandler, self).__init__(*args, **kwargs)
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 151ae4f62..f573b2b7f 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -332,14 +332,7 @@ class CrawlerProcess(CrawlerRunner):
 
 def _get_spider_loader(settings):
     """ Get SpiderLoader instance from settings """
-    if settings.get('SPIDER_MANAGER_CLASS'):
-        warnings.warn(
-            'SPIDER_MANAGER_CLASS option is deprecated. '
-            'Please use SPIDER_LOADER_CLASS.',
-            category=ScrapyDeprecationWarning, stacklevel=2
-        )
-    cls_path = settings.get('SPIDER_MANAGER_CLASS',
-                            settings.get('SPIDER_LOADER_CLASS'))
+    cls_path = settings.get('SPIDER_LOADER_CLASS')
     loader_cls = load_object(cls_path)
     try:
         verifyClass(ISpiderLoader, loader_cls)
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 844e3828c..6665eba16 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -8,8 +8,6 @@ import six
 
 from scrapy.item import Item
 from scrapy.selector import Selector
-from scrapy.utils.decorators import deprecated
-from scrapy.utils.deprecate import create_deprecated_class
 from scrapy.utils.misc import arg_to_iter, extract_regex
 from scrapy.utils.python import flatten
 
@@ -191,10 +189,6 @@ class ItemLoader(object):
         values = self._get_xpathvalues(xpath, **kw)
         return self.get_value(values, *processors, **kw)
 
-    @deprecated(use_instead='._get_xpathvalues()')
-    def _get_values(self, xpaths, **kw):
-        return self._get_xpathvalues(xpaths, **kw)
-
     def _get_xpathvalues(self, xpaths, **kw):
         self._check_selector_method()
         xpaths = arg_to_iter(xpaths)
@@ -216,5 +210,3 @@ class ItemLoader(object):
         self._check_selector_method()
         csss = arg_to_iter(csss)
         return flatten(self.selector.css(css).getall() for css in csss)
-
-XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)
diff --git a/scrapy/log.py b/scrapy/log.py
deleted file mode 100644
index 777bd6dc4..000000000
--- a/scrapy/log.py
+++ /dev/null
@@ -1,60 +0,0 @@
-"""
-This module is kept to provide a helpful warning about its removal.
-"""
-import logging
-import warnings
-
-from twisted.python.failure import Failure
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.log import failure_to_exc_info
-
-logger = logging.getLogger(__name__)
-
-warnings.warn("Module `scrapy.log` has been deprecated, Scrapy now relies on "
-              "the builtin Python library for logging. Read the updated "
-              "logging entry in the documentation to learn more.",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-
-# Imports and level_names variable kept for backward-compatibility
-
-DEBUG = logging.DEBUG
-INFO = logging.INFO
-WARNING = logging.WARNING
-ERROR = logging.ERROR
-CRITICAL = logging.CRITICAL
-SILENT = CRITICAL + 1
-
-level_names = {
-    logging.DEBUG: "DEBUG",
-    logging.INFO: "INFO",
-    logging.WARNING: "WARNING",
-    logging.ERROR: "ERROR",
-    logging.CRITICAL: "CRITICAL",
-    SILENT: "SILENT",
-}
-
-
-def msg(message=None, _level=logging.INFO, **kw):
-    warnings.warn('log.msg has been deprecated, create a python logger and '
-                  'log through it instead',
-                  ScrapyDeprecationWarning, stacklevel=2)
-
-    level = kw.pop('level', _level)
-    message = kw.pop('format', message)
-    # NOTE: logger.log doesn't handle well passing empty dictionaries with format
-    # arguments because of some weird use-case:
-    # https://hg.python.org/cpython/file/648dcafa7e5f/Lib/logging/__init__.py#l269
-    logger.log(level, message, *[kw] if kw else [])
-
-
-def err(_stuff=None, _why=None, **kw):
-    warnings.warn('log.err has been deprecated, create a python logger and '
-                  'use its error method instead',
-                  ScrapyDeprecationWarning, stacklevel=2)
-
-    level = kw.pop('level', logging.ERROR)
-    failure = kw.pop('failure', _stuff) or Failure()
-    message = kw.pop('why', _why) or failure.value
-    logger.log(level, message, *[kw] if kw else [], exc_info=failure_to_exc_info(failure))
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 2d8091f5b..2145e6d2b 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -458,33 +458,6 @@ class FilesPipeline(MediaPipeline):
         return item
 
     def file_path(self, request, response=None, info=None):
-        ## start of deprecation warning block (can be removed in the future)
-        def _warn():
-            from scrapy.exceptions import ScrapyDeprecationWarning
-            import warnings
-            warnings.warn('FilesPipeline.file_key(url) method is deprecated, please use '
-                          'file_path(request, response=None, info=None) instead',
-                          category=ScrapyDeprecationWarning, stacklevel=1)
-
-        # check if called from file_key with url as first argument
-        if not isinstance(request, Request):
-            _warn()
-            url = request
-        else:
-            url = request.url
-
-        # detect if file_key() method has been overridden
-        if not hasattr(self.file_key, '_base'):
-            _warn()
-            return self.file_key(url)
-        ## end of deprecation warning block
-
-        media_guid = hashlib.sha1(to_bytes(url)).hexdigest()  # change to request.url after deprecation
-        media_ext = os.path.splitext(url)[1]  # change to request.url after deprecation
+        media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
+        media_ext = os.path.splitext(request.url)[1]
         return 'full/%s%s' % (media_guid, media_ext)
-
-    # deprecated
-    def file_key(self, url):
-        return self.file_path(url)
-
-    file_key._base = True
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index a1457c7e9..fa4d12ad1 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -165,69 +165,9 @@ class ImagesPipeline(FilesPipeline):
         return item
 
     def file_path(self, request, response=None, info=None):
-        ## start of deprecation warning block (can be removed in the future)
-        def _warn():
-            from scrapy.exceptions import ScrapyDeprecationWarning
-            import warnings
-            warnings.warn('ImagesPipeline.image_key(url) and file_key(url) methods are deprecated, '
-                          'please use file_path(request, response=None, info=None) instead',
-                          category=ScrapyDeprecationWarning, stacklevel=1)
-
-        # check if called from image_key or file_key with url as first argument
-        if not isinstance(request, Request):
-            _warn()
-            url = request
-        else:
-            url = request.url
-
-        # detect if file_key() or image_key() methods have been overridden
-        if not hasattr(self.file_key, '_base'):
-            _warn()
-            return self.file_key(url)
-        elif not hasattr(self.image_key, '_base'):
-            _warn()
-            return self.image_key(url)
-        ## end of deprecation warning block
-
-        image_guid = hashlib.sha1(to_bytes(url)).hexdigest()  # change to request.url after deprecation
+        image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         return 'full/%s.jpg' % (image_guid)
 
     def thumb_path(self, request, thumb_id, response=None, info=None):
-        ## start of deprecation warning block (can be removed in the future)
-        def _warn():
-            from scrapy.exceptions import ScrapyDeprecationWarning
-            import warnings
-            warnings.warn('ImagesPipeline.thumb_key(url) method is deprecated, please use '
-                          'thumb_path(request, thumb_id, response=None, info=None) instead',
-                          category=ScrapyDeprecationWarning, stacklevel=1)
-
-        # check if called from thumb_key with url as first argument
-        if not isinstance(request, Request):
-            _warn()
-            url = request
-        else:
-            url = request.url
-
-        # detect if thumb_key() method has been overridden
-        if not hasattr(self.thumb_key, '_base'):
-            _warn()
-            return self.thumb_key(url, thumb_id)
-        ## end of deprecation warning block
-
-        thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest()  # change to request.url after deprecation
+        thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         return 'thumbs/%s/%s.jpg' % (thumb_id, thumb_guid)
-
-    # deprecated
-    def file_key(self, url):
-        return self.image_key(url)
-    file_key._base = True
-
-    # deprecated
-    def image_key(self, url):
-        return self.file_path(url)
-    image_key._base = True
-
-    # deprecated
-    def thumb_key(self, url, thumb_id):
-        return self.thumb_path(url, thumb_id)
-    thumb_key._base = True
diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py
index bfbde4de9..90e96ee92 100644
--- a/scrapy/selector/__init__.py
+++ b/scrapy/selector/__init__.py
@@ -2,4 +2,3 @@
 Selectors
 """
 from scrapy.selector.unified import *
-from scrapy.selector.lxmlsel import *
diff --git a/scrapy/selector/csstranslator.py b/scrapy/selector/csstranslator.py
deleted file mode 100644
index 8d7f034af..000000000
--- a/scrapy/selector/csstranslator.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from parsel.csstranslator import XPathExpr, GenericTranslator, HTMLTranslator
-from scrapy.utils.deprecate import create_deprecated_class
-
-
-ScrapyXPathExpr = create_deprecated_class(
-    'ScrapyXPathExpr', XPathExpr,
-    new_class_path='parsel.csstranslator.XPathExpr')
-
-ScrapyGenericTranslator = create_deprecated_class(
-    'ScrapyGenericTranslator', GenericTranslator,
-    new_class_path='parsel.csstranslator.GenericTranslator')
-
-ScrapyHTMLTranslator = create_deprecated_class(
-    'ScrapyHTMLTranslator', HTMLTranslator,
-    new_class_path='parsel.csstranslator.HTMLTranslator')
diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py
deleted file mode 100644
index 070cb2306..000000000
--- a/scrapy/selector/lxmlsel.py
+++ /dev/null
@@ -1,50 +0,0 @@
-"""
-XPath selectors based on lxml
-"""
-from scrapy.utils.deprecate import create_deprecated_class
-from .unified import Selector, SelectorList
-
-
-__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector',
-           'XPathSelectorList']
-
-def _xpathselector_css(self, *a, **kw):
-    raise RuntimeError('.css() method not available for %s, '
-                        'instantiate scrapy.Selector '
-                        'instead' % type(self).__name__)
-
-XPathSelector = create_deprecated_class(
-    'XPathSelector',
-    Selector,
-    {
-        '__slots__': (),
-        '_default_type': 'html',
-        'css': _xpathselector_css,
-    },
-    new_class_path='scrapy.Selector',
-    old_class_path='scrapy.selector.XPathSelector',
-)
-
-XmlXPathSelector = create_deprecated_class(
-    'XmlXPathSelector',
-    XPathSelector,
-    clsdict={
-        '__slots__': (),
-        '_default_type': 'xml',
-    },
-    new_class_path='scrapy.Selector',
-    old_class_path='scrapy.selector.XmlXPathSelector',
-)
-
-HtmlXPathSelector = create_deprecated_class(
-    'HtmlXPathSelector',
-    XPathSelector,
-    clsdict={
-        '__slots__': (),
-        '_default_type': 'html',
-    },
-    new_class_path='scrapy.Selector',
-    old_class_path='scrapy.selector.HtmlXPathSelector',
-)
-
-XPathSelectorList = create_deprecated_class('XPathSelectorList', SelectorList)
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 8f6cb1d79..62fda40b0 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -8,7 +8,6 @@ from scrapy.utils.trackref import object_ref
 from scrapy.utils.python import to_bytes
 from scrapy.http import HtmlResponse, XmlResponse
 from scrapy.utils.decorators import deprecated
-from scrapy.exceptions import ScrapyDeprecationWarning
 
 
 __all__ = ['Selector', 'SelectorList']
@@ -31,17 +30,6 @@ class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
     The :class:`SelectorList` class is a subclass of the builtin ``list``
     class, which provides a few additional methods.
     """
-    @deprecated(use_instead='.extract()')
-    def extract_unquoted(self):
-        return [x.extract_unquoted() for x in self]
-
-    @deprecated(use_instead='.xpath()')
-    def x(self, xpath):
-        return self.select(xpath)
-
-    @deprecated(use_instead='.xpath()')
-    def select(self, xpath):
-        return self.xpath(xpath)
 
 
 class Selector(_ParselSelector, object_ref):
@@ -78,21 +66,13 @@ class Selector(_ParselSelector, object_ref):
     __slots__ = ['response']
     selectorlist_cls = SelectorList
 
-    def __init__(self, response=None, text=None, type=None, root=None, _root=None, **kwargs):
+    def __init__(self, response=None, text=None, type=None, root=None, **kwargs):
         if not(response is None or text is None):
            raise ValueError('%s.__init__() received both response and text'
                             % self.__class__.__name__)
 
         st = _st(response, type or self._default_type)
 
-        if _root is not None:
-            warnings.warn("Argument `_root` is deprecated, use `root` instead",
-                          ScrapyDeprecationWarning, stacklevel=2)
-            if root is None:
-                root = _root
-            else:
-                warnings.warn("Ignoring deprecated `_root` argument, using provided `root`")
-
         if text is not None:
             response = _response_from_text(text, st)
 
@@ -102,18 +82,3 @@ class Selector(_ParselSelector, object_ref):
 
         self.response = response
         super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
-
-    # Deprecated api
-    @property
-    def _root(self):
-        warnings.warn("Attribute `_root` is deprecated, use `root` instead",
-                      ScrapyDeprecationWarning, stacklevel=2)
-        return self.root
-
-    @deprecated(use_instead='.xpath()')
-    def select(self, xpath):
-        return self.xpath(xpath)
-
-    @deprecated(use_instead='.extract()')
-    def extract_unquoted(self):
-        return self.extract()
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index e9c131e3b..94095bc27 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -10,7 +10,6 @@ from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
-from scrapy.utils.deprecate import create_deprecated_class
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.deprecate import method_is_overridden
 
@@ -52,15 +51,6 @@ class Spider(object_ref):
         spider._set_crawler(crawler)
         return spider
 
-    def set_crawler(self, crawler):
-        warnings.warn("set_crawler is deprecated, instantiate and bound the "
-                      "spider to this crawler with from_crawler method "
-                      "instead.",
-                      category=ScrapyDeprecationWarning, stacklevel=2)
-        assert not hasattr(self, 'crawler'), "Spider already bounded to a " \
-                                             "crawler"
-        self._set_crawler(crawler)
-
     def _set_crawler(self, crawler):
         self.crawler = crawler
         self.settings = crawler.settings
@@ -109,22 +99,6 @@ class Spider(object_ref):
     __repr__ = __str__
 
 
-BaseSpider = create_deprecated_class('BaseSpider', Spider)
-
-
-class ObsoleteClass(object):
-    def __init__(self, message):
-        self.message = message
-
-    def __getattr__(self, name):
-        raise AttributeError(self.message)
-
-spiders = ObsoleteClass(
-    '"from scrapy.spider import spiders" no longer works - use '
-    '"from scrapy.spiderloader import SpiderLoader" and instantiate '
-    'it with your project settings"'
-)
-
 # Top-level imports
 from scrapy.spiders.crawl import CrawlSpider, Rule
 from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 6db3a1e06..90a6eb806 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -119,7 +119,3 @@ class CrawlSpider(Spider):
         spider._follow_links = crawler.settings.getbool(
             'CRAWLSPIDER_FOLLOW_LINKS', True)
         return spider
-
-    def set_crawler(self, crawler):
-        super(CrawlSpider, self).set_crawler(crawler)
-        self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
diff --git a/scrapy/telnet.py b/scrapy/telnet.py
deleted file mode 100644
index 70915f486..000000000
--- a/scrapy/telnet.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.telnet` is deprecated, "
-              "use `scrapy.extensions.telnet` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.telnet import *
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index d8eaaa80f..c6140f885 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -84,19 +84,6 @@ def unique(list_, key=lambda x: x):
     return result
 
 
-@deprecated("scrapy.utils.python.to_unicode")
-def str_to_unicode(text, encoding=None, errors='strict'):
-    """ This function is deprecated.
-    Please use scrapy.utils.python.to_unicode. """
-    return to_unicode(text, encoding, errors)
-
-
-@deprecated("scrapy.utils.python.to_bytes")
-def unicode_to_str(text, encoding=None, errors='strict'):
-    """ This function is deprecated. Please use scrapy.utils.python.to_bytes """
-    return to_bytes(text, encoding, errors)
-
-
 def to_unicode(text, encoding=None, errors='strict'):
     """Return the unicode representation of a bytes object ``text``. If
     ``text`` is already an unicode object, return it as-is."""
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 122af28b0..c3236afd4 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -11,14 +11,6 @@ from twisted.web import http
 from scrapy.utils.python import to_bytes, to_native_str
 from w3lib import html
 
-from scrapy.utils.decorators import deprecated
-
-
-@deprecated
-def body_or_str(*a, **kw):
-    from scrapy.utils.iterators import _body_or_str
-    return _body_or_str(*a, **kw)
-
 
 _baseurl_cache = weakref.WeakKeyDictionary()
 def get_base_url(response):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index e811c5757..8eb2389e2 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -176,27 +176,6 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
             sl_cls = load_object(runner.settings['SPIDER_LOADER_CLASS'])
             self.assertIsInstance(spiders, sl_cls)
 
-    def test_spidermanager_deprecation(self):
-        with warnings.catch_warnings(record=True) as w:
-            runner = CrawlerRunner({
-                'SPIDER_MANAGER_CLASS': 'tests.test_crawler.CustomSpiderLoader'
-            })
-            self.assertIsInstance(runner.spider_loader, CustomSpiderLoader)
-            is_one_warning = len(w) == 1
-            if not is_one_warning:
-                for warning in w:
-                    print(warning)
-            self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message))
-            self.assertTrue(is_one_warning)
-
-    def test_crawl_rejects_spider_objects(self):
-        with raises(ValueError):
-            CrawlerRunner().crawl(DefaultSpider())
-
-    def test_create_crawler_rejects_spider_objects(self):
-        with raises(ValueError):
-            CrawlerRunner().create_crawler(DefaultSpider())
-
 
 class CrawlerProcessTest(BaseCrawlerTest):
     def test_crawler_process_accepts_dict(self):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 81235a16f..d2151e10e 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -24,7 +24,7 @@ from w3lib.url import path_to_file_uri
 from scrapy.core.downloader.handlers import DownloadHandlers
 from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler
 from scrapy.core.downloader.handlers.file import FileDownloadHandler
-from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler
+from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
 from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
 from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
@@ -360,11 +360,6 @@ class HttpTestCase(unittest.TestCase):
         return d
 
 
-class DeprecatedHttpTestCase(HttpTestCase):
-    """HTTP 1.0 test case"""
-    download_handler_cls = HttpDownloadHandler
-
-
 class Http10TestCase(HttpTestCase):
     """HTTP 1.0 test case"""
     download_handler_cls = HTTP10DownloadHandler
@@ -656,11 +651,6 @@ class HttpProxyTestCase(unittest.TestCase):
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
 
-class DeprecatedHttpProxyTestCase(unittest.TestCase):
-    """Old deprecated reference to http10 downloader handler"""
-    download_handler_cls = HttpDownloadHandler
-
-
 class Http10ProxyTestCase(HttpProxyTestCase):
     download_handler_cls = HTTP10DownloadHandler
 
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 728a74803..0c5aaaa44 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -108,44 +108,6 @@ class FilesPipelineTestCase(unittest.TestCase):
             p.stop()
 
 
-class DeprecatedFilesPipeline(FilesPipeline):
-    def file_key(self, url):
-        media_guid = hashlib.sha1(to_bytes(url)).hexdigest()
-        media_ext = os.path.splitext(url)[1]
-        return 'empty/%s%s' % (media_guid, media_ext)
-
-
-class DeprecatedFilesPipelineTestCase(unittest.TestCase):
-    def setUp(self):
-        self.tempdir = mkdtemp()
-
-    def init_pipeline(self, pipeline_class):
-        self.pipeline = pipeline_class.from_settings(Settings({'FILES_STORE': self.tempdir}))
-        self.pipeline.download_func = _mocked_download_func
-        self.pipeline.open_spider(None)
-
-    def test_default_file_key_method(self):
-        self.init_pipeline(FilesPipeline)
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            self.assertEqual(self.pipeline.file_key("https://dev.mydeco.com/mydeco.pdf"),
-                             'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
-            self.assertEqual(len(w), 1)
-            self.assertTrue('file_key(url) method is deprecated' in str(w[-1].message))
-
-    def test_overridden_file_key_method(self):
-        self.init_pipeline(DeprecatedFilesPipeline)
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            self.assertEqual(self.pipeline.file_path(Request("https://dev.mydeco.com/mydeco.pdf")),
-                             'empty/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
-            self.assertEqual(len(w), 1)
-            self.assertTrue('file_key(url) method is deprecated' in str(w[-1].message))
-
-    def tearDown(self):
-        rmtree(self.tempdir)
-
-
 class FilesPipelineTestCaseFields(unittest.TestCase):
 
     def test_item_fields_default(self):
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index a7c652959..4f7265763 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -118,63 +118,6 @@ class DeprecatedImagesPipeline(ImagesPipeline):
         return 'thumbsup/%s/%s.jpg' % (thumb_id, thumb_guid)
 
 
-class DeprecatedImagesPipelineTestCase(unittest.TestCase):
-    def setUp(self):
-        self.tempdir = mkdtemp()
-
-    def init_pipeline(self, pipeline_class):
-        self.pipeline = pipeline_class(self.tempdir, download_func=_mocked_download_func)
-        self.pipeline.open_spider(None)
-
-    def test_default_file_key_method(self):
-        self.init_pipeline(ImagesPipeline)
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            self.assertEqual(self.pipeline.file_key("https://dev.mydeco.com/mydeco.gif"),
-                             'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
-            self.assertEqual(len(w), 1)
-            self.assertTrue('image_key(url) and file_key(url) methods are deprecated' in str(w[-1].message))
-
-    def test_default_image_key_method(self):
-        self.init_pipeline(ImagesPipeline)
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            self.assertEqual(self.pipeline.image_key("https://dev.mydeco.com/mydeco.gif"),
-                             'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
-            self.assertEqual(len(w), 1)
-            self.assertTrue('image_key(url) and file_key(url) methods are deprecated' in str(w[-1].message))
-
-    def test_overridden_file_key_method(self):
-        self.init_pipeline(DeprecatedImagesPipeline)
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            self.assertEqual(self.pipeline.file_path(Request("https://dev.mydeco.com/mydeco.gif")),
-                             'empty/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
-            self.assertEqual(len(w), 1)
-            self.assertTrue('image_key(url) and file_key(url) methods are deprecated' in str(w[-1].message))
-
-    def test_default_thumb_key_method(self):
-        self.init_pipeline(ImagesPipeline)
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            self.assertEqual(self.pipeline.thumb_key("file:///tmp/foo.jpg", 50),
-                             'thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg')
-            self.assertEqual(len(w), 1)
-            self.assertTrue('thumb_key(url) method is deprecated' in str(w[-1].message))
-
-    def test_overridden_thumb_key_method(self):
-        self.init_pipeline(DeprecatedImagesPipeline)
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            self.assertEqual(self.pipeline.thumb_path(Request("file:///tmp/foo.jpg"), 50),
-                             'thumbsup/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg')
-            self.assertEqual(len(w), 1)
-            self.assertTrue('thumb_key(url) method is deprecated' in str(w[-1].message))
-
-    def tearDown(self):
-        rmtree(self.tempdir)
-
-
 class ImagesPipelineTestCaseFields(unittest.TestCase):
 
     def test_item_fields_default(self):
diff --git a/tests/test_selector.py b/tests/test_selector.py
index bc4baf7ea..210e3256c 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -3,7 +3,6 @@ import weakref
 from twisted.trial import unittest
 from scrapy.http import TextResponse, HtmlResponse, XmlResponse
 from scrapy.selector import Selector
-from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector
 from lxml import etree
 
 
@@ -40,22 +39,6 @@ class SelectorTestCase(unittest.TestCase):
         sel = Selector(response)
         self.assertEqual(url, sel.root.base)
 
-    def test_deprecated_root_argument(self):
-        with warnings.catch_warnings(record=True) as w:
-            root = etree.fromstring(u'<html/>')
-            sel = Selector(_root=root)
-            self.assertIs(root, sel.root)
-            self.assertEqual(str(w[-1].message),
-                             'Argument `_root` is deprecated, use `root` instead')
-
-    def test_deprecated_root_argument_ambiguous(self):
-        with warnings.catch_warnings(record=True) as w:
-            _root = etree.fromstring(u'<xml/>')
-            root = etree.fromstring(u'<html/>')
-            sel = Selector(_root=_root, root=root)
-            self.assertIs(root, sel.root)
-            self.assertIn('Ignoring deprecated `_root` argument', str(w[-1].message))
-
     def test_flavor_detection(self):
         text = b'<div><img src="a.jpg"><p>Hello</div>'
         sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
@@ -101,111 +84,6 @@ class SelectorTestCase(unittest.TestCase):
         assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
             x.__class__.__name__
 
-    def test_deprecated_selector_methods(self):
-        sel = Selector(TextResponse(url="http://example.com", body=b'<p>some text</p>'))
-
-        with warnings.catch_warnings(record=True) as w:
-            sel.select('//p')
-            self.assertSubstring('Use .xpath() instead', str(w[-1].message))
-
-        with warnings.catch_warnings(record=True) as w:
-            sel.extract_unquoted()
-            self.assertSubstring('Use .extract() instead', str(w[-1].message))
-
-    def test_deprecated_selectorlist_methods(self):
-        sel = Selector(TextResponse(url="http://example.com", body=b'<p>some text</p>'))
-
-        with warnings.catch_warnings(record=True) as w:
-            sel.xpath('//p').select('.')
-            self.assertSubstring('Use .xpath() instead', str(w[-1].message))
-
-        with warnings.catch_warnings(record=True) as w:
-            sel.xpath('//p').extract_unquoted()
-            self.assertSubstring('Use .extract() instead', str(w[-1].message))
-
     def test_selector_bad_args(self):
         with self.assertRaisesRegexp(ValueError, 'received both response and text'):
             Selector(TextResponse(url='http://example.com', body=b''), text=u'')
-
-
-class DeprecatedXpathSelectorTest(unittest.TestCase):
-
-    text = '<div><img src="a.jpg"><p>Hello</div>'
-
-    def test_warnings_xpathselector(self):
-        cls = XPathSelector
-        with warnings.catch_warnings(record=True) as w:
-            class UserClass(cls):
-                pass
-
-            # subclassing must issue a warning
-            self.assertEqual(len(w), 1, str(cls))
-            self.assertIn('scrapy.Selector', str(w[0].message))
-
-            # subclass instance doesn't issue a warning
-            usel = UserClass(text=self.text)
-            self.assertEqual(len(w), 1)
-
-            # class instance must issue a warning
-            sel = cls(text=self.text)
-            self.assertEqual(len(w), 2, str((cls, [x.message for x in w])))
-            self.assertIn('scrapy.Selector', str(w[1].message))
-
-            # subclass and instance checks
-            self.assertTrue(issubclass(cls, Selector))
-            self.assertTrue(isinstance(sel, Selector))
-            self.assertTrue(isinstance(usel, Selector))
-
-    def test_warnings_xmlxpathselector(self):
-        cls = XmlXPathSelector
-        with warnings.catch_warnings(record=True) as w:
-            class UserClass(cls):
-                pass
-
-            # subclassing must issue a warning
-            self.assertEqual(len(w), 1, str(cls))
-            self.assertIn('scrapy.Selector', str(w[0].message))
-
-            # subclass instance doesn't issue a warning
-            usel = UserClass(text=self.text)
-            self.assertEqual(len(w), 1)
-
-            # class instance must issue a warning
-            sel = cls(text=self.text)
-            self.assertEqual(len(w), 2, str((cls, [x.message for x in w])))
-            self.assertIn('scrapy.Selector', str(w[1].message))
-
-            # subclass and instance checks
-            self.assertTrue(issubclass(cls, Selector))
-            self.assertTrue(issubclass(cls, XPathSelector))
-            self.assertTrue(isinstance(sel, Selector))
-            self.assertTrue(isinstance(usel, Selector))
-            self.assertTrue(isinstance(sel, XPathSelector))
-            self.assertTrue(isinstance(usel, XPathSelector))
-
-    def test_warnings_htmlxpathselector(self):
-        cls = HtmlXPathSelector
-        with warnings.catch_warnings(record=True) as w:
-            class UserClass(cls):
-                pass
-
-            # subclassing must issue a warning
-            self.assertEqual(len(w), 1, str(cls))
-            self.assertIn('scrapy.Selector', str(w[0].message))
-
-            # subclass instance doesn't issue a warning
-            usel = UserClass(text=self.text)
-            self.assertEqual(len(w), 1)
-
-            # class instance must issue a warning
-            sel = cls(text=self.text)
-            self.assertEqual(len(w), 2, str((cls, [x.message for x in w])))
-            self.assertIn('scrapy.Selector', str(w[1].message))
-
-            # subclass and instance checks
-            self.assertTrue(issubclass(cls, Selector))
-            self.assertTrue(issubclass(cls, XPathSelector))
-            self.assertTrue(isinstance(sel, Selector))
-            self.assertTrue(isinstance(usel, Selector))
-            self.assertTrue(isinstance(sel, XPathSelector))
-            self.assertTrue(isinstance(usel, XPathSelector))
diff --git a/tests/test_selector_csstranslator.py b/tests/test_selector_csstranslator.py
deleted file mode 100644
index 2d82fcba7..000000000
--- a/tests/test_selector_csstranslator.py
+++ /dev/null
@@ -1,22 +0,0 @@
-"""
-Selector tests for cssselect backend
-"""
-import warnings
-from twisted.trial import unittest
-from scrapy.selector.csstranslator import (
-    ScrapyHTMLTranslator,
-    ScrapyGenericTranslator,
-    ScrapyXPathExpr
-)
-
-
-class DeprecatedClassesTest(unittest.TestCase):
-
-    def test_deprecated_warnings(self):
-        for cls in [ScrapyHTMLTranslator, ScrapyGenericTranslator, ScrapyXPathExpr]:
-            with warnings.catch_warnings(record=True) as w:
-                obj = cls()
-                self.assertIn('%s is deprecated' % cls.__name__, str(w[-1].message),
-                              'Missing deprecate warning for %s' % cls.__name__)
-
-
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 83fb68c2f..1197ea653 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -10,7 +10,7 @@ from scrapy import signals
 from scrapy.settings import Settings
 from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse
 from scrapy.spiders.init import InitSpider
-from scrapy.spiders import Spider, BaseSpider, CrawlSpider, Rule, XMLFeedSpider, \
+from scrapy.spiders import Spider, CrawlSpider, Rule, XMLFeedSpider, \
     CSVFeedSpider, SitemapSpider
 from scrapy.linkextractors import LinkExtractor
 from scrapy.exceptions import ScrapyDeprecationWarning
@@ -51,17 +51,6 @@ class SpiderTest(unittest.TestCase):
         self.assertRaises(ValueError, self.spider_class)
         self.assertRaises(ValueError, self.spider_class, somearg='foo')
 
-    def test_deprecated_set_crawler_method(self):
-        spider = self.spider_class('example.com')
-        crawler = get_crawler()
-        with warnings.catch_warnings(record=True) as w:
-            spider.set_crawler(crawler)
-            self.assertIn("set_crawler", str(w[0].message))
-            self.assertTrue(hasattr(spider, 'crawler'))
-            self.assertIs(spider.crawler, crawler)
-            self.assertTrue(hasattr(spider, 'settings'))
-            self.assertIs(spider.settings, crawler.settings)
-
     def test_from_crawler_crawler_and_settings_population(self):
         crawler = get_crawler()
         spider = self.spider_class.from_crawler(crawler, 'example.com')
@@ -377,20 +366,6 @@ class CrawlSpiderTest(SpiderTest):
         self.assertTrue(hasattr(spider, '_follow_links'))
         self.assertFalse(spider._follow_links)
 
-    def test_follow_links_attribute_deprecated_population(self):
-        spider = self.spider_class('example.com')
-        self.assertFalse(hasattr(spider, '_follow_links'))
-
-        spider.set_crawler(get_crawler())
-        self.assertTrue(hasattr(spider, '_follow_links'))
-        self.assertTrue(spider._follow_links)
-
-        spider = self.spider_class('example.com')
-        settings_dict = {'CRAWLSPIDER_FOLLOW_LINKS': False}
-        spider.set_crawler(get_crawler(settings_dict=settings_dict))
-        self.assertTrue(hasattr(spider, '_follow_links'))
-        self.assertFalse(spider._follow_links)
-
 
 class SitemapSpiderTest(SpiderTest):
 
@@ -578,57 +553,9 @@ Sitemap: /sitemap-relative-url.xml
 
 class DeprecationTest(unittest.TestCase):
 
-    def test_basespider_is_deprecated(self):
-        with warnings.catch_warnings(record=True) as w:
-
-            class MySpider1(BaseSpider):
-                pass
-
-            self.assertEqual(len(w), 1)
-            self.assertEqual(w[0].category, ScrapyDeprecationWarning)
-            self.assertEqual(w[0].lineno, inspect.getsourcelines(MySpider1)[1])
-
-    def test_basespider_issubclass(self):
-        class MySpider2(Spider):
-            pass
-
-        class MySpider2a(MySpider2):
-            pass
-
-        class Foo(object):
-            pass
-
-        class Foo2(object_ref):
-            pass
-
-        assert issubclass(MySpider2, BaseSpider)
-        assert issubclass(MySpider2a, BaseSpider)
-        assert not issubclass(Foo, BaseSpider)
-        assert not issubclass(Foo2, BaseSpider)
-
-    def test_basespider_isinstance(self):
-        class MySpider3(Spider):
-            name = 'myspider3'
-
-        class MySpider3a(MySpider3):
-            pass
-
-        class Foo(object):
-            pass
-
-        class Foo2(object_ref):
-            pass
-
-        assert isinstance(MySpider3(), BaseSpider)
-        assert isinstance(MySpider3a(), BaseSpider)
-        assert not isinstance(Foo(), BaseSpider)
-        assert not isinstance(Foo2(), BaseSpider)
-
     def test_crawl_spider(self):
         assert issubclass(CrawlSpider, Spider)
-        assert issubclass(CrawlSpider, BaseSpider)
         assert isinstance(CrawlSpider(name='foo'), Spider)
-        assert isinstance(CrawlSpider(name='foo'), BaseSpider)
 
     def test_make_requests_from_url_deprecated(self):
         class MySpider4(Spider):

From 5442c2d3c3f201c7ad069585546b403f46720df5 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sat, 23 Feb 2019 17:49:58 +0000
Subject: [PATCH 2090/2974] Updating S3FeedStorage instancing without AWS key.

---
 scrapy/extensions/feedexport.py | 3 ++-
 tests/test_feedexport.py        | 7 +++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index b2f7267a2..39d05de49 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -98,7 +98,8 @@ class S3FeedStorage(BlockingFeedStorage):
         # without using from_crawler)
         no_defaults = access_key is None and secret_key is None
         if no_defaults:
-            from scrapy.conf import settings
+            from scrapy.utils.project import get_project_settings
+            settings = get_project_settings()
             if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:
                 import warnings
                 from scrapy.exceptions import ScrapyDeprecationWarning
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index eef0384cf..14bb72170 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -26,6 +26,7 @@ from scrapy.extensions.feedexport import (
     BlockingFeedStorage)
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
 from scrapy.utils.python import to_native_str
+from scrapy.utils.project import get_project_settings
 
 
 class FileFeedStorageTest(unittest.TestCase):
@@ -134,8 +135,10 @@ class BlockingFeedStorageTest(unittest.TestCase):
 
 class S3FeedStorageTest(unittest.TestCase):
 
-    @mock.patch('scrapy.conf.settings', new={'AWS_ACCESS_KEY_ID': 'conf_key',
-                'AWS_SECRET_ACCESS_KEY': 'conf_secret'}, create=True)
+    @mock.patch('scrapy.utils.project.get_project_settings',
+                new=mock.MagicMock(return_value={'AWS_ACCESS_KEY_ID': 'conf_key',
+                                                 'AWS_SECRET_ACCESS_KEY': 'conf_secret'}),
+                create=True)
     def test_parse_credentials(self):
         try:
             import boto

From 67a400092805ec0d643bd7de0481cc45d5ce8471 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 8 Jul 2019 10:31:52 +0500
Subject: [PATCH 2091/2974] Work around older pyOpenSSL not having
 get_cipher_name or get_protocol_version_name.

---
 scrapy/core/downloader/tls.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 2ba72593f..7e5882663 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -74,11 +74,18 @@ if twisted_version >= (14, 0, 0):
             if where & SSL_CB_HANDSHAKE_START:
                 set_tlsext_host_name(connection, self._hostnameBytes)
             elif where & SSL_CB_HANDSHAKE_DONE:
-                logger.debug('SSL connection to %s using protocol %s, cipher %s',
-                             self._hostnameASCII,
-                             connection.get_protocol_version_name(),
-                             connection.get_cipher_name(),
-                             )
+                if hasattr(connection, 'get_cipher_name'):  # requires pyOPenSSL 0.15
+                    if hasattr(connection, 'get_protocol_version_name'):  # requires pyOPenSSL 16.0.0
+                        logger.debug('SSL connection to %s using protocol %s, cipher %s',
+                                     self._hostnameASCII,
+                                     connection.get_protocol_version_name(),
+                                     connection.get_cipher_name(),
+                                     )
+                    else:
+                        logger.debug('SSL connection to %s using cipher %s',
+                                     self._hostnameASCII,
+                                     connection.get_cipher_name(),
+                                     )
                 server_cert = connection.get_peer_certificate()
                 logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
                              x509name_to_string(server_cert.get_issuer()),

From 1130711cc116b8902c8f743157299ee92b16ee96 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 8 Jul 2019 11:17:30 +0500
Subject: [PATCH 2092/2974] deprecate scrapy.utils.http

ChunkedTransferMiddleware is deprecated, so decode_chunked_transfer
can be deprecated as well.
---
 scrapy/utils/http.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py
index 7cc8d1884..b6e05c862 100644
--- a/scrapy/utils/http.py
+++ b/scrapy/utils/http.py
@@ -4,8 +4,19 @@ Transitional module for moving to the w3lib library.
 For new code, always import from w3lib.http instead of this module
 """
 
+import warnings
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.decorators import deprecated
 from w3lib.http import *
 
+
+warnings.warn("Module `scrapy.utils.http` is deprecated, "
+              "Please import from `w3lib.http` instead.",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+
+@deprecated
 def decode_chunked_transfer(chunked_body):
     """Parsed body received with chunked transfer encoding, and return the
     decoded body.

From 64ff3cd6aac9009ab0fe4777224b8457b6abe790 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 8 Jul 2019 11:18:15 +0500
Subject: [PATCH 2093/2974] deprecate scrapy.utils.multipart

It was a shim for w3lib.form, but w3lib.form is deprecated as well.
---
 scrapy/utils/multipart.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py
index ec26c0866..c2d8afd07 100644
--- a/scrapy/utils/multipart.py
+++ b/scrapy/utils/multipart.py
@@ -3,5 +3,13 @@ Transitional module for moving to the w3lib library.
 
 For new code, always import from w3lib.form instead of this module
 """
+import warnings
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from w3lib.form import *
+
+
+warnings.warn("Module `scrapy.utils.multipart` is deprecated. "
+              "If you're using `encode_multipart` function, please use "
+              "`urllib3.filepost.encode_multipart_formdata` instead",
+              ScrapyDeprecationWarning, stacklevel=2)
\ No newline at end of file

From cb4477db3e2ac4ee1972ac3832d036a08596fa7a Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 8 Jul 2019 11:18:40 +0500
Subject: [PATCH 2094/2974] deprecate scrapy.utils.markup

it was an import-only shim for w3lib.html
---
 scrapy/utils/markup.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py
index 977133f4e..a18f308a3 100644
--- a/scrapy/utils/markup.py
+++ b/scrapy/utils/markup.py
@@ -3,5 +3,12 @@ Transitional module for moving to the w3lib library.
 
 For new code, always import from w3lib.html instead of this module
 """
+import warnings
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from w3lib.html import *
+
+
+warnings.warn("Module `scrapy.utils.markup` is deprecated. "
+              "Please import from `w3lib.html` instead.",
+              ScrapyDeprecationWarning, stacklevel=2)
\ No newline at end of file

From 6ea62cacc71dbd62c5a722061364866d93b0bd4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 8 Mar 2019 16:06:45 +0100
Subject: [PATCH 2095/2974] Add a FAQ entry about splitting items in item
 pipelines

---
 docs/faq.rst                      | 23 +++++++++++++++++++++++
 docs/topics/spider-middleware.rst |  2 ++
 2 files changed, 25 insertions(+)

diff --git a/docs/faq.rst b/docs/faq.rst
index 7a0628f88..69edb0993 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -319,6 +319,29 @@ I'm scraping a XML document and my XPath selector doesn't return any items
 
 You may need to remove namespaces. See :ref:`removing-namespaces`.
 
+How to split an item into multiple items in an item pipeline?
+-------------------------------------------------------------
+
+:ref:`Item pipelines <topics-item-pipeline>` cannot yield multiple items per
+input item. :ref:`Create a spider middleware <custom-spider-middleware>`
+instead, and use its
+:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
+method for this puspose. For example::
+
+    from copy import deepcopy
+
+    from scrapy.item import BaseItem
+
+
+    class MultiplyItemsMiddleware:
+
+        def process_spider_output(self, response, result, spider):
+            for item in result:
+                if isinstance(item, (BaseItem, dict)):
+                    for _ in range(item['multiply_by']):
+                        yield deepcopy(item)
+
+
 .. _user agents: https://en.wikipedia.org/wiki/User_agent
 .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
 .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 30b8638bd..0e8210130 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -54,6 +54,8 @@ value.  For example, if you want to disable the off-site middleware::
 Finally, keep in mind that some middlewares may need to be enabled through a
 particular setting. See each middleware documentation for more info.
 
+.. _custom-spider-middleware:
+
 Writing your own spider middleware
 ==================================
 

From 783de8a8d8c6c3c0a91318c3354034a7eb46a627 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 12 Feb 2019 16:36:35 +0100
Subject: [PATCH 2096/2974] Document how request concurrency settings impact
 request order

---
 docs/faq.rst | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 7a0628f88..f6dc9cec0 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -129,13 +129,23 @@ Does Scrapy crawl in breadth-first or depth-first order?
 
 By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which
 basically means that it crawls in `DFO order`_. This order is more convenient
-in most cases. If you do want to crawl in true `BFO order`_, you can do it by
+in most cases.
+
+If you do want to crawl in true `BFO order`_, you can do it by
 setting the following settings::
 
     DEPTH_PRIORITY = 1
     SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue'
     SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue'
 
+While pending requests are below the configured values of
+:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
+:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sent
+concurrently. As a result, the first few requests of a crawl rarely follow the
+desired order. Lowering those settings to ``1`` enforces the desired order, but
+it significantly slows down the crawl as a whole.
+
+
 My Scrapy crawler has memory leaks. What can I do?
 --------------------------------------------------
 

From 0b9dce3a6c17d8dc827df57367383e0b82fa8b07 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 8 Jul 2019 17:40:56 +0500
Subject: [PATCH 2097/2974] Add DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING setting.

---
 docs/topics/settings.rst                  | 21 +++++++++--
 scrapy/core/downloader/contextfactory.py  | 14 ++++++-
 scrapy/core/downloader/handlers/http10.py |  7 ++--
 scrapy/core/downloader/handlers/http11.py | 11 +++---
 scrapy/core/downloader/tls.py             | 45 +++++++++++++----------
 scrapy/settings/default_settings.py       |  1 +
 6 files changed, 66 insertions(+), 33 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 371f21c72..5cc87bb64 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -438,9 +438,10 @@ or even enable client-side authentication (and various other things).
     which uses the platform's certificates to validate remote endpoints.
     **This is only available if you use Twisted>=14.0.**
 
-If you do use a custom ContextFactory, make sure it accepts a ``method``
-parameter at init (this is the ``OpenSSL.SSL`` method mapping
-:setting:`DOWNLOADER_CLIENT_TLS_METHOD`).
+If you do use a custom ContextFactory, make sure its ``__init__` method accepts
+a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
+:setting:`DOWNLOADER_CLIENT_TLS_METHOD`) and a ``settings`` parameter (this is
+the Scrapy settings object).
 
 .. setting:: DOWNLOADER_CLIENT_TLS_METHOD
 
@@ -468,6 +469,20 @@ This setting must be one of these string values:
     We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
     or above (Twisted>=14.0 if you can).
 
+.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
+
+DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
+-------------------------------------
+
+Default: ``False``
+
+Setting this to ``True`` will enable DEBUG level messages about TLS connection
+parameters after establishing HTTPS connections. The kind of information logged
+depends on the versions of OpenSSL and pyOpenSSL.
+
+This setting is only used for the default
+:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`.
+
 .. setting:: DOWNLOADER_MIDDLEWARES
 
 DOWNLOADER_MIDDLEWARES
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 783d4c383..80c784f5a 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -2,6 +2,7 @@ from OpenSSL import SSL
 from twisted.internet.ssl import ClientContextFactory
 
 from scrapy import twisted_version
+from scrapy.utils.misc import create_instance
 
 if twisted_version >= (14, 0, 0):
 
@@ -28,9 +29,17 @@ if twisted_version >= (14, 0, 0):
          understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
         """
 
-        def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs):
+        def __init__(self, method=SSL.SSLv23_METHOD, settings=None, *args, **kwargs):
             super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
             self._ssl_method = method
+            if settings:
+                self.tls_verbose_logging = settings['DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING']
+            else:
+                self.tls_verbose_logging = False
+
+        @classmethod
+        def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
+            return cls(method=method, settings=settings, *args, **kwargs)
 
         def getCertificateOptions(self):
             # setting verify=True will require you to provide CAs
@@ -56,7 +65,8 @@ if twisted_version >= (14, 0, 0):
             return self.getCertificateOptions().getContext()
 
         def creatorForNetloc(self, hostname, port):
-            return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext())
+            return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),
+                                          verbose_logging=self.tls_verbose_logging)
 
 
     @implementer(IPolicyForHTTPS)
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index d875fb1e4..be7298531 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -1,7 +1,7 @@
 """Download handlers for http and https schemes
 """
 from twisted.internet import reactor
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, create_instance
 from scrapy.utils.python import to_unicode
 
 
@@ -11,6 +11,7 @@ class HTTP10DownloadHandler(object):
     def __init__(self, settings):
         self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
         self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+        self._settings = settings
 
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
@@ -21,7 +22,7 @@ class HTTP10DownloadHandler(object):
     def _connect(self, factory):
         host, port = to_unicode(factory.host), factory.port
         if factory.scheme == b'https':
-            return reactor.connectSSL(host, port, factory,
-                                      self.ClientContextFactory())
+            client_context_factory = create_instance(self.ClientContextFactory, settings=self._settings, crawler=None)
+            return reactor.connectSSL(host, port, factory, client_context_factory)
         else:
             return reactor.connectTCP(host, port, factory)
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 0673188a1..9b0c7977d 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -25,7 +25,7 @@ from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.core.downloader.webclient import _parse
 from scrapy.core.downloader.tls import openssl_methods
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, create_instance
 from scrapy.utils.python import to_bytes, to_unicode
 from scrapy import twisted_version
 
@@ -44,14 +44,15 @@ class HTTP11DownloadHandler(object):
         self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
         # try method-aware context factory
         try:
-            self._contextFactory = self._contextFactoryClass(method=self._sslMethod)
+            self._contextFactory = create_instance(self._contextFactoryClass, settings=settings, crawler=None,
+                                                   method=self._sslMethod)
         except TypeError:
             # use context factory defaults
-            self._contextFactory = self._contextFactoryClass()
+            self._contextFactory = create_instance(self._contextFactoryClass, settings=settings, crawler=None)
             msg = """
  '%s' does not accept `method` argument (type OpenSSL.SSL method,\
- e.g. OpenSSL.SSL.SSLv23_METHOD).\
- Please upgrade your context factory class to handle it or ignore it.""" % (
+ e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `settings` argument.\
+ Please upgrade your context factory class to handle them or ignore them.""" % (
                 settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
             warnings.warn(msg)
         self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 7e5882663..74be85d52 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -70,30 +70,35 @@ if twisted_version >= (14, 0, 0):
         logging warnings. Also, HTTPS connection parameters logging is added.
         """
 
+        def __init__(self, hostname, ctx, verbose_logging=False):
+            super().__init__(hostname, ctx)
+            self.verbose_logging = verbose_logging
+
         def _identityVerifyingInfoCallback(self, connection, where, ret):
             if where & SSL_CB_HANDSHAKE_START:
                 set_tlsext_host_name(connection, self._hostnameBytes)
             elif where & SSL_CB_HANDSHAKE_DONE:
-                if hasattr(connection, 'get_cipher_name'):  # requires pyOPenSSL 0.15
-                    if hasattr(connection, 'get_protocol_version_name'):  # requires pyOPenSSL 16.0.0
-                        logger.debug('SSL connection to %s using protocol %s, cipher %s',
-                                     self._hostnameASCII,
-                                     connection.get_protocol_version_name(),
-                                     connection.get_cipher_name(),
-                                     )
-                    else:
-                        logger.debug('SSL connection to %s using cipher %s',
-                                     self._hostnameASCII,
-                                     connection.get_cipher_name(),
-                                     )
-                server_cert = connection.get_peer_certificate()
-                logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
-                             x509name_to_string(server_cert.get_issuer()),
-                             x509name_to_string(server_cert.get_subject()),
-                             )
-                key_info = get_temp_key_info(connection._ssl)
-                if key_info:
-                    logger.debug('SSL temp key: %s', key_info)
+                if self.verbose_logging:
+                    if hasattr(connection, 'get_cipher_name'):  # requires pyOPenSSL 0.15
+                        if hasattr(connection, 'get_protocol_version_name'):  # requires pyOPenSSL 16.0.0
+                            logger.debug('SSL connection to %s using protocol %s, cipher %s',
+                                         self._hostnameASCII,
+                                         connection.get_protocol_version_name(),
+                                         connection.get_cipher_name(),
+                                         )
+                        else:
+                            logger.debug('SSL connection to %s using cipher %s',
+                                         self._hostnameASCII,
+                                         connection.get_cipher_name(),
+                                         )
+                    server_cert = connection.get_peer_certificate()
+                    logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
+                                 x509name_to_string(server_cert.get_issuer()),
+                                 x509name_to_string(server_cert.get_subject()),
+                                 )
+                    key_info = get_temp_key_info(connection._ssl)
+                    if key_info:
+                        logger.debug('SSL temp key: %s', key_info)
 
                 try:
                     verifyHostname(connection, self._hostnameASCII)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 10b6cf9bc..af8305b25 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -87,6 +87,7 @@ DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClien
 DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
 DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
                                      # also allowing negotiation
+DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
 
 DOWNLOADER_MIDDLEWARES = {}
 

From a0b09e0193350426c71f9d321c9b56829ce10143 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 8 Jul 2019 14:44:46 +0200
Subject: [PATCH 2098/2974] Skip scrapy.downloadermiddlewares private APIs in
 the documentation coverage report

---
 docs/conf.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/conf.py b/docs/conf.py
index bf222b361..61d5b9600 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -237,4 +237,13 @@ coverage_ignore_pyobjects = [
     # their constructor, the methods they reimplement to achieve that purpose
     # should be irrelevant to developers using those contracts.
     r'\w+Contract\.(adjust_request_args|(pre|post)_process)$',
+
+    # Methods of downloader middlewares are not documented, only the classes
+    # themselves, since downloader middlewares are controlled through Scrapy
+    # settings.
+    r'^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.',
+
+    # Base classes of downloader middlewares are implementation details that
+    # are not meant for users.
+    r'^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware',
 ]

From 578bccf3bbe584ceaba01a37184bb8ad11763866 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 9 Jul 2019 12:33:07 +0200
Subject: [PATCH 2099/2974] Skip scrapy.exceptions.UsageError in the
 documentation coverage report

---
 docs/conf.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/conf.py b/docs/conf.py
index 61d5b9600..80b8e3e90 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -246,4 +246,7 @@ coverage_ignore_pyobjects = [
     # Base classes of downloader middlewares are implementation details that
     # are not meant for users.
     r'^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware',
+
+    # Private exception used by the command-line interface implementation.
+    r'^scrapy\.exceptions\.UsageError',
 ]

From a0bbbe8ffa7867f7bf7e1ba05a9362ce97e88085 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 9 Jul 2019 12:33:57 +0200
Subject: [PATCH 2100/2974] Add the API documentation of ContractFail to the
 contracts topic

---
 docs/topics/contracts.rst | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 9337375bb..05b1d161b 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -69,7 +69,7 @@ create and load your own contracts in the project by using the
         'myproject.contracts.ItemValidate': 10,
     }
 
-Each contract must inherit from :class:`scrapy.contracts.Contract` and can
+Each contract must inherit from :class:`~scrapy.contracts.Contract` and can
 override three methods:
 
 .. module:: scrapy.contracts
@@ -102,9 +102,14 @@ override three methods:
         This allows processing the output of the callback. Iterators are
         converted listified before being passed to this hook.
 
+Raise :class:`~scrapy.exceptions.ContractFail` from
+:class:`~scrapy.contracts.Contract.pre_process` or
+:class:`~scrapy.contracts.Contract.post_process` if expectations are not met:
+
+.. autoclass:: scrapy.exceptions.ContractFail
+
 Here is a demo contract which checks the presence of a custom header in the
-response received. Raise :class:`scrapy.exceptions.ContractFail` in order to
-get the failures pretty printed::
+response received::
 
     from scrapy.contracts import Contract
     from scrapy.exceptions import ContractFail

From 9fb0f8454eed974f5d2ed1808142169364145348 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 9 Jul 2019 15:30:22 -0300
Subject: [PATCH 2101/2974] Extend docs about Crawling Rules

---
 docs/topics/spiders.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 84a3a8fbe..869a61441 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -386,7 +386,9 @@ Crawling rules
    the specified link extractor. This callback receives a :class:`~scrapy.http.Response`
    as its first argument and must return either a single instance or an iterable of
    :class:`~scrapy.item.Item`, ``dict`` and/or :class:`~scrapy.http.Request` objects
-   (or any subclass of them).
+   (or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response`
+   object will contain the text of the link that produced the :class:`~scrapy.http.Request`
+   in its ``meta`` dictionary (under the ``link_text`` key)
 
    .. warning:: When writing crawl spider rules, avoid using ``parse`` as
        callback, since the :class:`CrawlSpider` uses the ``parse`` method

From df68c4b9b1789cc2c7f800f8e4b9f6e5aca7b7af Mon Sep 17 00:00:00 2001
From: Luiz Francisco Rodrigues da Silva <luizfrdasilva@gmail.com>
Date: Wed, 10 Jul 2019 09:25:52 -0300
Subject: [PATCH 2102/2974] Add a setting to use active mode in FTPFeedStorage
 (#3829)

Add a setting to use active mode in FTPFeedStorage

FTP servers can be configured in active and passive mode, by default
Python and Scrapy use passive mode and there was no way to use active.

This commit adds a setting FEED_STORAGE_FTP_ACTIVE to allow a feed
to be exported to a FTP server configured in active mode.
---
 docs/topics/feed-exports.rst        | 16 ++++++++++++++++
 scrapy/extensions/feedexport.py     | 12 +++++++++++-
 scrapy/settings/default_settings.py |  1 +
 tests/test_feedexport.py            | 20 +++++++++++++++++++-
 4 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index cf70b8aca..7e32da6f4 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -164,6 +164,11 @@ The feeds are stored in a FTP server.
  * Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
  * Required external libraries: none
 
+FTP supports two different connection modes: [active or passive](
+https://stackoverflow.com/a/1699163). Scrapy uses the passive connection mode
+by default. To use the active connection mode instead, set the
+:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
+
 .. _topics-feed-storage-s3:
 
 S3
@@ -209,6 +214,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_URI` (mandatory)
  * :setting:`FEED_FORMAT`
  * :setting:`FEED_STORAGES`
+ * :setting:`FEED_STORAGE_FTP_ACTIVE`
  * :setting:`FEED_STORAGE_S3_ACL`
  * :setting:`FEED_EXPORTERS`
  * :setting:`FEED_STORE_EMPTY`
@@ -307,6 +313,16 @@ Default: ``{}``
 A dict containing additional feed storage backends supported by your project.
 The keys are URI schemes and the values are paths to storage classes.
 
+.. setting:: FEED_STORAGE_FTP_ACTIVE
+
+FEED_STORAGE_FTP_ACTIVE
+-----------------------
+
+Default: ``False``
+
+Whether to use [active mode](https://stackoverflow.com/a/1699163) when exporting feeds
+to a FTP server.
+
 .. setting:: FEED_STORAGE_S3_ACL
 
 FEED_STORAGE_S3_ACL
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 39d05de49..d35551fdd 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -157,19 +157,29 @@ class S3FeedStorage(BlockingFeedStorage):
 
 class FTPFeedStorage(BlockingFeedStorage):
 
-    def __init__(self, uri):
+    def __init__(self, uri, use_active_mode=False):
         u = urlparse(uri)
         self.host = u.hostname
         self.port = int(u.port or '21')
         self.username = u.username
         self.password = u.password
         self.path = u.path
+        self.use_active_mode = use_active_mode
+
+    @classmethod
+    def from_crawler(cls, crawler, uri):
+        return cls(
+            uri=uri,
+            use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE')
+        )
 
     def _store_in_thread(self, file):
         file.seek(0)
         ftp = FTP()
         ftp.connect(self.host, self.port)
         ftp.login(self.username, self.password)
+        if self.use_active_mode:
+            ftp.set_pasv(False)
         dirname, filename = posixpath.split(self.path)
         ftp_makedirs_cwd(ftp, dirname)
         ftp.storbinary('STOR %s' % filename, file)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 10b6cf9bc..d17eb3125 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -158,6 +158,7 @@ FEED_EXPORTERS_BASE = {
 }
 FEED_EXPORT_INDENT = 0
 
+FEED_STORAGE_FTP_ACTIVE = False
 FEED_STORAGE_S3_ACL = ''
 
 FILES_STORE_S3_ACL = 'private'
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 14bb72170..c5063253a 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -71,6 +71,13 @@ class FileFeedStorageTest(unittest.TestCase):
 
 class FTPFeedStorageTest(unittest.TestCase):
 
+    def get_test_spider(self, settings=None):
+        class TestSpider(scrapy.Spider):
+            name = 'test_spider'
+        crawler = get_crawler(settings_dict=settings)
+        spider = TestSpider.from_crawler(crawler)
+        return spider
+
     def test_store(self):
         uri = os.environ.get('FEEDTEST_FTP_URI')
         path = os.environ.get('FEEDTEST_FTP_PATH')
@@ -80,9 +87,20 @@ class FTPFeedStorageTest(unittest.TestCase):
         verifyObject(IFeedStorage, st)
         return self._assert_stores(st, path)
 
+    def test_store_active_mode(self):
+        uri = os.environ.get('FEEDTEST_FTP_URI')
+        path = os.environ.get('FEEDTEST_FTP_PATH')
+        if not (uri and path):
+            raise unittest.SkipTest("No FTP server available for testing")
+        use_active_mode = {'FEED_STORAGE_FTP_ACTIVE': True}
+        crawler = get_crawler(settings_dict=use_active_mode)
+        st = FTPFeedStorage.from_crawler(crawler, uri)
+        verifyObject(IFeedStorage, st)
+        return self._assert_stores(st, path)
+
     @defer.inlineCallbacks
     def _assert_stores(self, storage, path):
-        spider = scrapy.Spider("default")
+        spider = self.get_test_spider()
         file = storage.open(spider)
         file.write(b"content")
         yield storage.store(file)

From 5cdf277017c38b4facd81e51f102061119fec7f9 Mon Sep 17 00:00:00 2001
From: Robin <mueller-bady@linux.com>
Date: Fri, 29 May 2015 16:02:02 +0200
Subject: [PATCH 2103/2974] Update broad-crawls.rst

Added section on how to treat memory consumption problems of broad crawls.
---
 docs/topics/broad-crawls.rst | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index b887b98af..5b9e7832f 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -174,3 +174,32 @@ It is turned OFF by default because it has some performance overhead,
 and enabling it for focused crawls doesn't make much sense.
 
 .. _ajax crawlable: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
+
+Reducing memory consumption for broad crawls
+============================================
+
+For broad crawls, the amount of memory used for storing `Requests`_, references, and further information may soon become pretty large.
+The following steps help to reduce the amount of memory used for broad crawls.
+
+1) **Change the queue type:** The default queue for crawls is "Last-In-First-Out ( `LIFO`_ )" using the concept of "Depth-First Search ( `DFS`_ )". In case the page scraping is faster than the processing of the spiders, early `Requests`_ might not be processed and therefore block memory until the final depth is reached. Setting the queue from LIFO to "First-In-First-Out ( `FIFO`_ )" and setting dispatching from `DFS`_ to "Breadth-First Search ( `BFS`_ )", as shown in the `FAQ`_ will solve this problem::
+
+    DEPTH_PRIORITY = 1
+    SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleFifoDiskQueue'
+    SCHEDULER_MEMORY_QUEUE = 'scrapy.squeue.FifoMemoryQueue'
+
+2) **Reduce the number of concurrent requests:** As stated before, the global concurrency level can be set using::
+ 
+    CONCURRENT_REQUESTS = 100
+
+However, if scraping is faster than processing, the queue will eventually exceed the memory size.
+Unfortunately, there is yet no autobalancing feature available, so you need to find concurrency values that fit your processing speed.
+
+3) **Use the profiling and trackref capabilities of scrapy:** scrapy provides an own and interactive profiling and reference tracking tool. See `debugging memory leaks`_ for more information.
+
+.. _debugging memory leaks: http://doc.scrapy.org/en/latest/topics/leaks.html
+.. _Requests: http://doc.scrapy.org/en/latest/topics/request-response.html#request-objects
+.. _LIFO: http://en.wikipedia.org/wiki/Stack_(abstract_data_type)
+.. _FIFO: http://en.wikipedia.org/wiki/FIFO_(computing_and_electronics)
+.. _BFS: http://en.wikipedia.org/wiki/Breadth-first_search
+.. _DFS: http://en.wikipedia.org/wiki/Depth-first_search
+.. _FAQ: http://doc.scrapy.org/en/latest/faq.html#does-scrapy-crawl-in-breadth-first-or-depth-first-order

From 0de6ffc8e1354ac7ffcdf7a75e3ec8d26ed23a01 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 11 Jul 2019 13:12:56 +0500
Subject: [PATCH 2104/2974] Fix super() call.

---
 scrapy/core/downloader/tls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 74be85d52..74afb3f10 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -71,7 +71,7 @@ if twisted_version >= (14, 0, 0):
         """
 
         def __init__(self, hostname, ctx, verbose_logging=False):
-            super().__init__(hostname, ctx)
+            super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
             self.verbose_logging = verbose_logging
 
         def _identityVerifyingInfoCallback(self, connection, where, ret):

From 98689b27a8839a21aed52ada44143456ead84e24 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 11 Jul 2019 14:02:35 +0500
Subject: [PATCH 2105/2974] Improve the DOWNLOADER_CLIENTCONTEXTFACTORY doc.

---
 docs/topics/settings.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 5cc87bb64..53c624679 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -438,10 +438,10 @@ or even enable client-side authentication (and various other things).
     which uses the platform's certificates to validate remote endpoints.
     **This is only available if you use Twisted>=14.0.**
 
-If you do use a custom ContextFactory, make sure its ``__init__` method accepts
-a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
+If you do use a custom ContextFactory, make sure its ``__init__`` method
+accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
 :setting:`DOWNLOADER_CLIENT_TLS_METHOD`) and a ``settings`` parameter (this is
-the Scrapy settings object).
+the Scrapy :class:`~scrapy.settings.Settings` object).
 
 .. setting:: DOWNLOADER_CLIENT_TLS_METHOD
 

From ed487b7db387a8687360b9d554ff8e7b3f38af93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 11 Jul 2019 09:18:00 +0200
Subject: [PATCH 2106/2974] broad-crawls.rst: Refactor the memory usage section

---
 docs/topics/broad-crawls.rst | 64 ++++++++++++++++++++----------------
 1 file changed, 36 insertions(+), 28 deletions(-)

diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index 5b9e7832f..1ab08d949 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -39,6 +39,8 @@ you need to keep in mind when using Scrapy for doing broad crawls, along with
 concrete suggestions of Scrapy settings to tune in order to achieve an
 efficient broad crawl.
 
+.. _broad-crawls-scheduler-priority-queue:
+
 Use the right :setting:`SCHEDULER_PRIORITY_QUEUE`
 =================================================
 
@@ -50,24 +52,37 @@ To apply the recommended priority queue use::
 
     SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
 
+.. _broad-crawls-concurrency:
+
 Increase concurrency
 ====================
 
 Concurrency is the number of requests that are processed in parallel. There is
-a global limit and a per-domain limit.
+a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that
+can be set either per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) or per
+IP (:setting:`CONCURRENT_REQUESTS_PER_IP`).
+
+.. note:: The scheduler priority queue :ref:`recommended for broad crawls
+          <broad-crawls-scheduler-priority-queue>` does not support
+          :setting:`CONCURRENT_REQUESTS_PER_IP`.
 
 The default global concurrency limit in Scrapy is not suitable for crawling
 many different domains in parallel, so you will want to increase it. How much
-to increase it will depend on how much CPU you crawler will have available. A
-good starting point is ``100``, but the best way to find out is by doing some
-trials and identifying at what concurrency your Scrapy process gets CPU
-bounded. For optimum performance, you should pick a concurrency where CPU usage
-is at 80-90%.
+to increase it will depend on how much CPU and memory you crawler will have
+available.
 
-To increase the global concurrency use::
+A good starting point is ``100``::
 
     CONCURRENT_REQUESTS = 100
 
+But the best way to find out is by doing some trials and identifying at what
+concurrency your Scrapy process gets CPU bounded. For optimum performance, you
+should pick a concurrency where CPU usage is at 80-90%.
+
+Increasing concurrency also increases memory usage. If memory usage is a
+concern, you might need to lower your global concurrency limit accordingly.
+
+
 Increase Twisted IO thread pool maximum size
 ============================================
 
@@ -175,31 +190,24 @@ and enabling it for focused crawls doesn't make much sense.
 
 .. _ajax crawlable: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
 
-Reducing memory consumption for broad crawls
-============================================
+.. _broad-crawls-bfo:
 
-For broad crawls, the amount of memory used for storing `Requests`_, references, and further information may soon become pretty large.
-The following steps help to reduce the amount of memory used for broad crawls.
+Crawl in BFO order
+==================
 
-1) **Change the queue type:** The default queue for crawls is "Last-In-First-Out ( `LIFO`_ )" using the concept of "Depth-First Search ( `DFS`_ )". In case the page scraping is faster than the processing of the spiders, early `Requests`_ might not be processed and therefore block memory until the final depth is reached. Setting the queue from LIFO to "First-In-First-Out ( `FIFO`_ )" and setting dispatching from `DFS`_ to "Breadth-First Search ( `BFS`_ )", as shown in the `FAQ`_ will solve this problem::
+:ref:`Scrapy crawls in DFO order by default <faq-bfo-dfo>`.
 
-    DEPTH_PRIORITY = 1
-    SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleFifoDiskQueue'
-    SCHEDULER_MEMORY_QUEUE = 'scrapy.squeue.FifoMemoryQueue'
+In broad crawls, however, page crawling tends to be faster than page
+processing. As a result, unprocessed early requests stay in memory until the
+final depth is reached, which can significantly increase memory usage.
 
-2) **Reduce the number of concurrent requests:** As stated before, the global concurrency level can be set using::
- 
-    CONCURRENT_REQUESTS = 100
+:ref:`Crawl in BFO order <faq-bfo-dfo>` instead to save memory.
 
-However, if scraping is faster than processing, the queue will eventually exceed the memory size.
-Unfortunately, there is yet no autobalancing feature available, so you need to find concurrency values that fit your processing speed.
 
-3) **Use the profiling and trackref capabilities of scrapy:** scrapy provides an own and interactive profiling and reference tracking tool. See `debugging memory leaks`_ for more information.
+Be mindful of memory leaks
+==========================
 
-.. _debugging memory leaks: http://doc.scrapy.org/en/latest/topics/leaks.html
-.. _Requests: http://doc.scrapy.org/en/latest/topics/request-response.html#request-objects
-.. _LIFO: http://en.wikipedia.org/wiki/Stack_(abstract_data_type)
-.. _FIFO: http://en.wikipedia.org/wiki/FIFO_(computing_and_electronics)
-.. _BFS: http://en.wikipedia.org/wiki/Breadth-first_search
-.. _DFS: http://en.wikipedia.org/wiki/Depth-first_search
-.. _FAQ: http://doc.scrapy.org/en/latest/faq.html#does-scrapy-crawl-in-breadth-first-or-depth-first-order
+If your broad crawl shows a high memory usage, in addition to :ref:`crawling in
+BFO order <broad-crawls-bfo>` and :ref:`lowering concurrency
+<broad-crawls-concurrency>` you should :ref:`debug your memory leaks
+<topics-leaks>`.

From a96a07bc762287a1f2056d1e142aff9f33e206fe Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 12 Jul 2019 18:44:45 +0500
Subject: [PATCH 2107/2974] Add a test for
 DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING.

---
 tests/test_downloader_handlers.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 81235a16f..8d0df6b5b 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -8,6 +8,7 @@ try:
 except ImportError:
     import mock
 
+from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.protocols.policies import WrappingFactory
 from twisted.python.filepath import FilePath
@@ -503,6 +504,24 @@ class Http11TestCase(HttpTestCase):
 class Https11TestCase(Http11TestCase):
     scheme = 'https'
 
+    tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", subject "/C=IE/O=Scrapy/CN=localhost"'
+
+    @defer.inlineCallbacks
+    def test_tls_logging(self):
+        download_handler = self.download_handler_cls(Settings({
+            'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True,
+        }))
+        try:
+            with LogCapture() as log_capture:
+                request = Request(self.getURL('file'))
+                d = download_handler.download_request(request, Spider('foo'))
+                d.addCallback(lambda r: r.body)
+                d.addCallback(self.assertEqual, b"0123456789")
+                yield d
+                log_capture.check_present(('scrapy.core.downloader.tls', 'DEBUG', self.tls_log_message))
+        finally:
+            yield download_handler.close()
+
 
 class Https11WrongHostnameTestCase(Http11TestCase):
     scheme = 'https'
@@ -523,6 +542,7 @@ class Https11InvalidDNSId(Https11TestCase):
         super(Https11InvalidDNSId, self).setUp()
         self.host = '127.0.0.1'
 
+
 class Https11InvalidDNSPattern(Https11TestCase):
     """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
 
@@ -534,6 +554,7 @@ class Https11InvalidDNSPattern(Https11TestCase):
             from service_identity.exceptions import CertificateError
         except ImportError:
             raise unittest.SkipTest("cryptography lib is too old")
+        self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
         super(Https11InvalidDNSPattern, self).setUp()
 
 

From 09e27d2d2e04fd44d9e49fbd328805ad9eca594f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 13 Jul 2019 18:08:07 -0300
Subject: [PATCH 2108/2974] [Tests] Monkey patch unittest.TestCase to prevent
 DeprecationWarning(s)

---
 tests/__init__.py                   | 13 +++++++++++++
 tests/test_exporters.py             |  2 +-
 tests/test_http_headers.py          | 16 ++++++++--------
 tests/test_http_request.py          |  6 +++---
 tests/test_http_response.py         | 18 +++++++++---------
 tests/test_selector.py              |  2 +-
 tests/test_spider.py                |  2 +-
 tests/test_spiderloader/__init__.py |  4 ++--
 8 files changed, 38 insertions(+), 25 deletions(-)

diff --git a/tests/__init__.py b/tests/__init__.py
index a54367f8c..e55c77734 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -35,3 +35,16 @@ def get_testdata(*paths):
     path = os.path.join(tests_datadir, *paths)
     with open(path, 'rb') as f:
         return f.read()
+
+
+# FIXME: delete after dropping py2 support
+# Monkey patch the unittest module to prevent the
+# DeprecationWarning about assertRaisesRegexp -> assertRaisesRegex
+import sys
+if sys.version_info[0] == 2:
+    import unittest
+    import twisted.trial.unittest
+    if not getattr(unittest.TestCase, 'assertRaisesRegex', None):
+        unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
+    if not getattr(twisted.trial.unittest.TestCase, 'assertRaisesRegex', None):
+        twisted.trial.unittest.TestCase.assertRaisesRegex = twisted.trial.unittest.TestCase.assertRaisesRegexp
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index cd72c661a..0046c5666 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -103,7 +103,7 @@ class PythonItemExporterTest(BaseItemExporterTest):
         return PythonItemExporter(binary=False, **kwargs)
 
     def test_invalid_option(self):
-        with self.assertRaisesRegexp(TypeError, "Unexpected options: invalid_option"):
+        with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"):
             PythonItemExporter(invalid_option='something')
 
     def test_nested_item(self):
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 504937295..69d906fbf 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -147,11 +147,11 @@ class HeadersTest(unittest.TestCase):
         self.assertEqual(h1.getlist('hey'), [b'5'])
 
     def test_invalid_value(self):
-        self.assertRaisesRegexp(TypeError, 'Unsupported value type',
-                                Headers, {'foo': object()})
-        self.assertRaisesRegexp(TypeError, 'Unsupported value type',
-                                Headers().__setitem__, 'foo', object())
-        self.assertRaisesRegexp(TypeError, 'Unsupported value type',
-                                Headers().setdefault, 'foo', object())
-        self.assertRaisesRegexp(TypeError, 'Unsupported value type',
-                                Headers().setlist, 'foo', [object()])
+        self.assertRaisesRegex(TypeError, 'Unsupported value type',
+                               Headers, {'foo': object()})
+        self.assertRaisesRegex(TypeError, 'Unsupported value type',
+                               Headers().__setitem__, 'foo', object())
+        self.assertRaisesRegex(TypeError, 'Unsupported value type',
+                               Headers().setdefault, 'foo', object())
+        self.assertRaisesRegex(TypeError, 'Unsupported value type',
+                               Headers().setlist, 'foo', [object()])
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 53d18d4aa..952e208de 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -989,9 +989,9 @@ class FormRequestTest(RequestTest):
 
         xpath = u"//form[@name='\u03b1']"
         encoded = xpath if six.PY3 else xpath.encode('unicode_escape')
-        self.assertRaisesRegexp(ValueError, re.escape(encoded),
-                                self.request_class.from_response,
-                                response, formxpath=xpath)
+        self.assertRaisesRegex(ValueError, re.escape(encoded),
+                               self.request_class.from_response,
+                               response, formxpath=xpath)
 
     def test_from_response_button_submit(self):
         response = _buildresponse(
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 3b90e3dac..cd5c3486e 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -135,9 +135,9 @@ class BaseResponseTest(unittest.TestCase):
         r = self.response_class("http://example.com", body=b'hello')
         if self.response_class == Response:
             msg = "Response content isn't text"
-            self.assertRaisesRegexp(AttributeError, msg, getattr, r, 'text')
-            self.assertRaisesRegexp(NotSupported, msg, r.css, 'body')
-            self.assertRaisesRegexp(NotSupported, msg, r.xpath, '//body')
+            self.assertRaisesRegex(AttributeError, msg, getattr, r, 'text')
+            self.assertRaisesRegex(NotSupported, msg, r.css, 'body')
+            self.assertRaisesRegex(NotSupported, msg, r.xpath, '//body')
         else:
             r.text
             r.css('body')
@@ -425,13 +425,13 @@ class TextResponseTest(BaseResponseTest):
 
     def test_follow_selector_list(self):
         resp = self._links_response()
-        self.assertRaisesRegexp(ValueError, 'SelectorList',
-                                resp.follow, resp.css('a'))
+        self.assertRaisesRegex(ValueError, 'SelectorList',
+                               resp.follow, resp.css('a'))
 
     def test_follow_selector_invalid(self):
         resp = self._links_response()
-        self.assertRaisesRegexp(ValueError, 'Unsupported',
-                                resp.follow, resp.xpath('count(//div)')[0])
+        self.assertRaisesRegex(ValueError, 'Unsupported',
+                               resp.follow, resp.xpath('count(//div)')[0])
 
     def test_follow_selector_attribute(self):
         resp = self._links_response()
@@ -443,8 +443,8 @@ class TextResponseTest(BaseResponseTest):
             url='http://example.com',
             body=b'<html><body><a name=123>click me</a></body></html>',
         )
-        self.assertRaisesRegexp(ValueError, 'no href',
-                                resp.follow, resp.css('a')[0])
+        self.assertRaisesRegex(ValueError, 'no href',
+                               resp.follow, resp.css('a')[0])
 
     def test_follow_whitespace_selector(self):
         resp = self.response_class(
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 210e3256c..b2565dd78 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -85,5 +85,5 @@ class SelectorTestCase(unittest.TestCase):
             x.__class__.__name__
 
     def test_selector_bad_args(self):
-        with self.assertRaisesRegexp(ValueError, 'received both response and text'):
+        with self.assertRaisesRegex(ValueError, 'received both response and text'):
             Selector(TextResponse(url='http://example.com', body=b''), text=u'')
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 1197ea653..e81e6d5f9 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -595,5 +595,5 @@ class NoParseMethodSpiderTest(unittest.TestCase):
         resp = TextResponse(url="http://www.example.com/random_url", body=text)
 
         exc_msg = 'Spider.parse callback is not defined'
-        with self.assertRaisesRegexp(NotImplementedError, exc_msg):
+        with self.assertRaisesRegex(NotImplementedError, exc_msg):
             spider.parse(resp)
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 1cd59b99a..106da798c 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -84,8 +84,8 @@ class SpiderLoaderTest(unittest.TestCase):
         module = 'tests.test_spiderloader.test_spiders.spider1'
         runner = CrawlerRunner({'SPIDER_MODULES': [module]})
 
-        self.assertRaisesRegexp(KeyError, 'Spider not found',
-                                runner.create_crawler, 'spider2')
+        self.assertRaisesRegex(KeyError, 'Spider not found',
+                               runner.create_crawler, 'spider2')
 
         crawler = runner.create_crawler('spider1')
         self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))

From 27e63e6890896c86fa4b11dc9951c4773bc2d629 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 13 Jul 2019 20:10:06 -0300
Subject: [PATCH 2109/2974] Monkey patch configparser module to ease the
 dropping of py2 support

---
 scrapy/_monkeypatches.py | 5 +++++
 scrapy/utils/conf.py     | 6 +++---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index f55ecc213..16bb4008b 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -14,6 +14,11 @@ if sys.version_info[0] == 2:
         from urlparse import uses_query
         uses_query.append('s3')
 
+    # Prevent the DeprecationWarning about SafeConfigParser -> ConfigParser
+    import configparser
+    if not getattr(configparser, 'ConfigParser', None):
+        configparser.ConfigParser = configparser.SafeConfigParser
+
 
 # Undo what Twisted's perspective broker adds to pickle register
 # to prevent bugs like Twisted#7989 while serializing requests
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index fbd297340..26d66eaf8 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -1,10 +1,10 @@
 import os
 import sys
 import numbers
+import configparser
 from operator import itemgetter
 
 import six
-from six.moves.configparser import SafeConfigParser
 
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
@@ -92,9 +92,9 @@ def init_env(project='default', set_syspath=True):
 
 
 def get_config(use_closest=True):
-    """Get Scrapy config file as a SafeConfigParser"""
+    """Get Scrapy config file as a ConfigParser"""
     sources = get_sources(use_closest)
-    cfg = SafeConfigParser()
+    cfg = configparser.ConfigParser()
     cfg.read(sources)
     return cfg
 

From b714a372e21bed55d10539297bd90ef091f9ef43 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 13 Jul 2019 20:23:10 -0300
Subject: [PATCH 2110/2974] [Tests] Prevent more DeprecationWarnings

* assertRegexpMatches -> assertRegex
* invalid escape sequence \[
---
 tests/test_command_parse.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index c18a6ce9f..98e415ad3 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -170,7 +170,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         status, out, stderr = yield self.execute(
             ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')]
         )
-        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
+        self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""Cannot find callback""", _textmode(stderr))
 
     @defer.inlineCallbacks
@@ -195,7 +195,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         status, out, stderr = yield self.execute(
             ['--spider', self.spider_name, '-r', self.url('/html')]
         )
-        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
+        self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""No CrawlSpider rules found""", _textmode(stderr))
 
     @defer.inlineCallbacks
@@ -203,7 +203,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         status, out, stderr = yield self.execute(
             ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')]
         )
-        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
+        self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
 
     @defer.inlineCallbacks
     def test_crawlspider_no_matching_rule(self):
@@ -211,5 +211,5 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         status, out, stderr = yield self.execute(
             ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')]
         )
-        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
+        self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))

From c24b80e1e6df7bd04cca77d068df2656d36a8753 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 13 Jul 2019 20:34:31 -0300
Subject: [PATCH 2111/2974] Prevent DeprecationWarning about logging.warn

---
 scrapy/core/downloader/handlers/http11.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 0673188a1..74ef82874 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -479,10 +479,10 @@ class _ResponseReader(protocol.Protocol):
                 return
 
             elif not self._fail_on_dataloss_warned:
-                logger.warn("Got data loss in %s. If you want to process broken "
-                            "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
-                            " -- This message won't be shown in further requests",
-                            self._txresponse.request.absoluteURI.decode())
+                logger.warning("Got data loss in %s. If you want to process broken "
+                               "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
+                               " -- This message won't be shown in further requests",
+                               self._txresponse.request.absoluteURI.decode())
                 self._fail_on_dataloss_warned = True
 
         self._finished.errback(reason)

From 92d624c16189ced0d03eea8cf9db5fd400ebfabd Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 13 Jul 2019 20:55:45 -0300
Subject: [PATCH 2112/2974] [Tests] Prevent more TestCase DeprecationWarnings

* assertRegexpMatches -> assertRegex
---
 tests/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/__init__.py b/tests/__init__.py
index e55c77734..72ae12989 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -44,6 +44,8 @@ import sys
 if sys.version_info[0] == 2:
     import unittest
     import twisted.trial.unittest
+    if not getattr(unittest.TestCase, 'assertRegex', None):
+        unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches
     if not getattr(unittest.TestCase, 'assertRaisesRegex', None):
         unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
     if not getattr(twisted.trial.unittest.TestCase, 'assertRaisesRegex', None):

From 40086dabb85a7c463f5a479e0c32d94c0b9463af Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 13 Jul 2019 20:57:24 -0300
Subject: [PATCH 2113/2974] Prevent more DeprecationWarnings

---
 scrapy/core/downloader/handlers/ftp.py     | 2 +-
 scrapy/core/downloader/handlers/http11.py  | 2 +-
 scrapy/utils/template.py                   | 2 +-
 tests/test_downloadermiddleware_cookies.py | 2 +-
 tests/test_engine.py                       | 6 +++---
 tests/test_linkextractors.py               | 2 +-
 tests/test_loader.py                       | 2 +-
 tests/test_utils_url.py                    | 4 ++--
 8 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index c342d4ab1..806a537d4 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -59,7 +59,7 @@ class ReceivedDataProtocol(Protocol):
     def close(self):
         self.body.close() if self.filename else self.body.seek(0)
 
-_CODE_RE = re.compile("\d+")
+_CODE_RE = re.compile(r"\d+")
 
 
 class FTPDownloadHandler(object):
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 74ef82874..cbaa36b2d 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -101,7 +101,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
     for it.
     """
 
-    _responseMatcher = re.compile(b'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
+    _responseMatcher = re.compile(br'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
 
     def __init__(self, reactor, host, port, proxyConf, contextFactory,
                  timeout=30, bindAddress=None):
diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py
index 1d7bd006c..615372fc8 100644
--- a/scrapy/utils/template.py
+++ b/scrapy/utils/template.py
@@ -18,7 +18,7 @@ def render_templatefile(path, **kwargs):
         os.remove(path)
 
 
-CAMELCASE_INVALID_CHARS = re.compile('[^a-zA-Z\d]')
+CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]')
 def string_camelcase(string):
     """ Convert a word  to its CamelCase version and remove invalid chars
 
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 17801e502..04884fb78 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -13,7 +13,7 @@ from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
 class CookiesMiddlewareTest(TestCase):
 
     def assertCookieValEqual(self, first, second, msg=None):
-        cookievaleq = lambda cv: re.split(';\s*', cv.decode('latin1'))
+        cookievaleq = lambda cv: re.split(r';\s*', cv.decode('latin1'))
         return self.assertEqual(
             sorted(cookievaleq(first)),
             sorted(cookievaleq(second)), msg)
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 856465161..30150391a 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -40,9 +40,9 @@ class TestSpider(Spider):
     name = "scrapytest.org"
     allowed_domains = ["scrapytest.org", "localhost"]
 
-    itemurl_re = re.compile("item\d+.html")
-    name_re = re.compile("<h1>(.*?)</h1>", re.M)
-    price_re = re.compile(">Price: \$(.*?)<", re.M)
+    itemurl_re = re.compile(r"item\d+.html")
+    name_re = re.compile(r"<h1>(.*?)</h1>", re.M)
+    price_re = re.compile(r">Price: \$(.*?)<", re.M)
 
     item_cls = TestItem
 
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index c9cd629f4..d96e259f6 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -288,7 +288,7 @@ class Base:
             response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
 
             def process_value(value):
-                m = re.search("javascript:goToPage\('(.*?)'", value)
+                m = re.search(r"javascript:goToPage\('(.*?)'", value)
                 if m:
                     return m.group(1)
 
diff --git a/tests/test_loader.py b/tests/test_loader.py
index ce0fa0701..2725b001a 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -691,7 +691,7 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertTrue(l.selector)
         l.add_css('url', 'a::attr(href)')
         self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
-        l.replace_css('url', 'a::attr(href)', re='http://www\.(.+)')
+        l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)')
         self.assertEqual(l.get_output_value('url'), [u'scrapy.org'])
 
 
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index c2b9fc176..e6588055c 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -233,8 +233,8 @@ for k, args in enumerate ([
     setattr (GuessSchemeTest, t_method.__name__, t_method)
 
 # TODO: the following tests do not pass with current implementation
-for k, args in enumerate ([
-            ('C:\absolute\path\to\a\file.html',     'file://',
+for k, args in enumerate([
+            (r'C:\absolute\path\to\a\file.html', 'file://',
              'Windows filepath are not supported for scrapy shell'),
         ], start=1):
     t_method = create_skipped_scheme_t(args)

From eced544d64ac7398133781ecefed178ed5178892 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 13 Jul 2019 22:14:47 -0300
Subject: [PATCH 2114/2974] Prevent even more DeprecationWarnings

---
 scrapy/item.py                               | 14 ++++++++----
 scrapy/utils/datatypes.py                    | 13 ++++++++---
 tests/test_downloadermiddleware_robotstxt.py | 23 +++++++++-----------
 3 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/scrapy/item.py b/scrapy/item.py
index 031b80a2d..393302dce 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -4,16 +4,22 @@ Scrapy Item
 See documentation in docs/topics/item.rst
 """
 
-from pprint import pformat
-from collections import MutableMapping
-from copy import deepcopy
-
 from abc import ABCMeta
+from pprint import pformat
+from copy import deepcopy
+import collections
+
 import six
 
 from scrapy.utils.trackref import object_ref
 
 
+if six.PY3:
+    MutableMapping = collections.abc.MutableMapping
+else:
+    MutableMapping = collections.MutableMapping
+
+
 class BaseItem(object_ref):
     """Base class for all scraped items."""
     pass
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index eb373c501..af39e887f 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -6,13 +6,20 @@ This module must not depend on any module outside the Standard Library.
 """
 
 import copy
-import six
+import collections
 import warnings
-from collections import OrderedDict, Mapping
+
+import six
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
+if six.PY3:
+    Mapping = collections.abc.Mapping
+else:
+    Mapping = collections.Mapping
+
+
 class MultiValueDictKeyError(KeyError):
     def __init__(self, *args, **kwargs):
         warnings.warn(
@@ -289,7 +296,7 @@ class MergeDict(object):
         return self.__copy__()
 
 
-class LocalCache(OrderedDict):
+class LocalCache(collections.OrderedDict):
     """Dictionary with a finite number of keys.
 
     Older items expires first.
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 60306eacb..2b3548bdd 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
-import re
 from twisted.internet import reactor, error
 from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
 from twisted.python import failure
@@ -31,18 +30,16 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
     def _get_successful_crawler(self):
         crawler = self.crawler
         crawler.settings.set('ROBOTSTXT_OBEY', True)
-        ROBOTS = re.sub(b'^\s+(?m)', b'', u'''
-        User-Agent: *
-        Disallow: /admin/
-        Disallow: /static/
-
-        # taken from https://en.wikipedia.org/robots.txt
-        Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
-        Disallow: /wiki/Käyttäjä:
-
-        User-Agent: UnicödeBöt
-        Disallow: /some/randome/page.html
-        '''.encode('utf-8'))
+        ROBOTS = u"""
+User-Agent: *
+Disallow: /admin/
+Disallow: /static/
+# taken from https://en.wikipedia.org/robots.txt
+Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
+Disallow: /wiki/Käyttäjä:
+User-Agent: UnicödeBöt
+Disallow: /some/randome/page.html
+""".encode('utf-8')
         response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
         def return_response(request, spider):
             deferred = Deferred()

From 62f3e22481145bf023cb43b95388dbcfd75a198d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 13 Jul 2019 22:44:57 -0300
Subject: [PATCH 2115/2974] Deprecate scrapy.utils.datatypes.MergeDict in favor
 of collections.ChainMap

---
 scrapy/loader/processors.py | 11 +++++++----
 scrapy/utils/datatypes.py   |  7 +++++++
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index 468aec2cc..2acdc8093 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -3,10 +3,13 @@ This module provides some commonly used processors for Item Loaders.
 
 See documentation in docs/topics/loaders.rst
 """
+try:
+    from collections import ChainMap
+except ImportError:
+    from scrapy.utils.datatypes import MergeDict as ChainMap
 
 from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.datatypes import MergeDict
-from .common import wrap_loader_context
+from scrapy.loader.common import wrap_loader_context
 
 
 class MapCompose(object):
@@ -18,7 +21,7 @@ class MapCompose(object):
     def __call__(self, value, loader_context=None):
         values = arg_to_iter(value)
         if loader_context:
-            context = MergeDict(loader_context, self.default_loader_context)
+            context = ChainMap(loader_context, self.default_loader_context)
         else:
             context = self.default_loader_context
         wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
@@ -45,7 +48,7 @@ class Compose(object):
 
     def __call__(self, value, loader_context=None):
         if loader_context:
-            context = MergeDict(loader_context, self.default_loader_context)
+            context = ChainMap(loader_context, self.default_loader_context)
         else:
             context = self.default_loader_context
         wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index eb373c501..fbc41f760 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -245,6 +245,13 @@ class MergeDict(object):
     first occurrence will be used.
     """
     def __init__(self, *dicts):
+        if six.PY3:
+            warnings.warn(
+                "scrapy.utils.datatypes.MergeDict is deprecated in favor "
+                "of collections.ChainMap (introduced in Python 3.3)",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
         self.dicts = dicts
 
     def __getitem__(self, key):

From ef9a61921482f1a0004616e0b2b6734b9314655d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 13 Jul 2019 23:36:39 -0300
Subject: [PATCH 2116/2974] More DeprecationWarnings

---
 scrapy/settings/__init__.py   | 10 ++++++++--
 tests/test_utils_datatypes.py |  9 ++++++++-
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 14c93bef2..6b53c7a79 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,11 +1,17 @@
 import six
 import json
 import copy
-from collections import MutableMapping
+import collections
 from importlib import import_module
 from pprint import pformat
 
-from . import default_settings
+from scrapy.settings import default_settings
+
+
+if six.PY3:
+    MutableMapping = collections.abc.MutableMapping
+else:
+    MutableMapping = collections.MutableMapping
 
 
 SETTINGS_PRIORITIES = {
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 5b83869b8..618050fef 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,11 +1,18 @@
 import copy
 import unittest
-from collections import Mapping, MutableMapping
+
+import six
+if six.PY3:
+    from collections.abc import Mapping, MutableMapping
+else:
+    from collections.abc import Mapping, MutableMapping
 
 from scrapy.utils.datatypes import CaselessDict, SequenceExclude
 
+
 __doctests__ = ['scrapy.utils.datatypes']
 
+
 class CaselessDictTest(unittest.TestCase):
 
     def test_init_dict(self):

From d5a2a7032979c6af8b074ec7fd60baa5256cc7cd Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 13 Jul 2019 23:47:41 -0300
Subject: [PATCH 2117/2974] Fix import

---
 tests/test_utils_datatypes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 618050fef..0d6e1eb71 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -5,7 +5,7 @@ import six
 if six.PY3:
     from collections.abc import Mapping, MutableMapping
 else:
-    from collections.abc import Mapping, MutableMapping
+    from collections import Mapping, MutableMapping
 
 from scrapy.utils.datatypes import CaselessDict, SequenceExclude
 

From 7e3a602d569bd81e9bd891a482f440a93dbe7bb4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 15 Jul 2019 12:37:09 -0300
Subject: [PATCH 2118/2974] Suggested changes

---
 scrapy/_monkeypatches.py      | 10 +++-------
 scrapy/item.py                |  6 +++---
 scrapy/settings/__init__.py   |  6 +++---
 scrapy/utils/datatypes.py     |  6 +++---
 tests/__init__.py             |  4 ++--
 tests/test_utils_datatypes.py |  6 +++---
 6 files changed, 17 insertions(+), 21 deletions(-)

diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index 16bb4008b..935c4bfa3 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -1,7 +1,8 @@
-import sys
+import six
 from six.moves import copyreg
 
-if sys.version_info[0] == 2:
+
+if six.PY2:
     from urlparse import urlparse
 
     # workaround for https://bugs.python.org/issue7904 - Python < 2.7
@@ -14,11 +15,6 @@ if sys.version_info[0] == 2:
         from urlparse import uses_query
         uses_query.append('s3')
 
-    # Prevent the DeprecationWarning about SafeConfigParser -> ConfigParser
-    import configparser
-    if not getattr(configparser, 'ConfigParser', None):
-        configparser.ConfigParser = configparser.SafeConfigParser
-
 
 # Undo what Twisted's perspective broker adds to pickle register
 # to prevent bugs like Twisted#7989 while serializing requests
diff --git a/scrapy/item.py b/scrapy/item.py
index 393302dce..9d4786788 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -14,10 +14,10 @@ import six
 from scrapy.utils.trackref import object_ref
 
 
-if six.PY3:
-    MutableMapping = collections.abc.MutableMapping
-else:
+if six.PY2:
     MutableMapping = collections.MutableMapping
+else:
+    MutableMapping = collections.abc.MutableMapping
 
 
 class BaseItem(object_ref):
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 6b53c7a79..f28c7940d 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -8,10 +8,10 @@ from pprint import pformat
 from scrapy.settings import default_settings
 
 
-if six.PY3:
-    MutableMapping = collections.abc.MutableMapping
-else:
+if six.PY2:
     MutableMapping = collections.MutableMapping
+else:
+    MutableMapping = collections.abc.MutableMapping
 
 
 SETTINGS_PRIORITIES = {
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index af39e887f..b98a1297e 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -14,10 +14,10 @@ import six
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
-if six.PY3:
-    Mapping = collections.abc.Mapping
-else:
+if six.PY2:
     Mapping = collections.Mapping
+else:
+    Mapping = collections.abc.Mapping
 
 
 class MultiValueDictKeyError(KeyError):
diff --git a/tests/__init__.py b/tests/__init__.py
index 72ae12989..9c9e35c35 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -40,8 +40,8 @@ def get_testdata(*paths):
 # FIXME: delete after dropping py2 support
 # Monkey patch the unittest module to prevent the
 # DeprecationWarning about assertRaisesRegexp -> assertRaisesRegex
-import sys
-if sys.version_info[0] == 2:
+import six
+if six.PY2:
     import unittest
     import twisted.trial.unittest
     if not getattr(unittest.TestCase, 'assertRegex', None):
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 0d6e1eb71..535095b8d 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -2,10 +2,10 @@ import copy
 import unittest
 
 import six
-if six.PY3:
-    from collections.abc import Mapping, MutableMapping
-else:
+if six.PY2:
     from collections import Mapping, MutableMapping
+else:
+    from collections.abc import Mapping, MutableMapping
 
 from scrapy.utils.datatypes import CaselessDict, SequenceExclude
 

From e892a484e823470cf37e66904ffbfdb240438842 Mon Sep 17 00:00:00 2001
From: "Deschner, Magdalena" <mdeschner@hotmail.de>
Date: Tue, 16 Jul 2019 13:53:56 +0200
Subject: [PATCH 2119/2974] add instructions about how to define output file
 when running scrapy from script instead of cmd

---
 docs/topics/practices.rst | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 298a078a7..4566eeb6e 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -35,12 +35,17 @@ Here's an example showing how to run a single spider with it.
         ...
 
     process = CrawlerProcess({
-        'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
+        'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
+        'FEED_FORMAT':'json',
+        'FEED_URI':'items.json'
     })
 
     process.crawl(MySpider)
     process.start() # the script will block here until the crawling is finished
 
+Define settings within dictionary in
+CrawlerProcess. FEED_FORMAT and FEED_URI are the equivalent to "-o items.json" when using the scrapy crawl shell command.
+
 Make sure to check :class:`~scrapy.crawler.CrawlerProcess` documentation to get
 acquainted with its usage details.
 

From d7074d86d26c936c6907dea7c550a4f251667d8b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 16 Jul 2019 14:13:45 -0300
Subject: [PATCH 2120/2974] Change condition to raise deprecation warning

---
 scrapy/utils/datatypes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index fbc41f760..231234e3d 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -245,7 +245,7 @@ class MergeDict(object):
     first occurrence will be used.
     """
     def __init__(self, *dicts):
-        if six.PY3:
+        if not six.PY2:
             warnings.warn(
                 "scrapy.utils.datatypes.MergeDict is deprecated in favor "
                 "of collections.ChainMap (introduced in Python 3.3)",

From 6660020ebb01c6eb240f56347d67106f308dd333 Mon Sep 17 00:00:00 2001
From: "Deschner, Magdalena" <mdeschner@hotmail.de>
Date: Wed, 17 Jul 2019 11:30:02 +0200
Subject: [PATCH 2121/2974] remove detailed description about individual
 settings

---
 docs/topics/practices.rst | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 4566eeb6e..610a39ea9 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -34,8 +34,7 @@ Here's an example showing how to run a single spider with it.
         # Your spider definition
         ...
 
-    process = CrawlerProcess({
-        'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
+    process = CrawlerProcess(settings={
         'FEED_FORMAT':'json',
         'FEED_URI':'items.json'
     })
@@ -43,11 +42,8 @@ Here's an example showing how to run a single spider with it.
     process.crawl(MySpider)
     process.start() # the script will block here until the crawling is finished
 
-Define settings within dictionary in
-CrawlerProcess. FEED_FORMAT and FEED_URI are the equivalent to "-o items.json" when using the scrapy crawl shell command.
-
-Make sure to check :class:`~scrapy.crawler.CrawlerProcess` documentation to get
-acquainted with its usage details.
+Define settings within dictionary in CrawlerProcess. Make sure to check :class:`~scrapy.crawler.CrawlerProcess`
+documentation to get acquainted with its usage details.
 
 If you are inside a Scrapy project there are some additional helpers you can
 use to import those components within the project. You can automatically import

From 0d51f9cc276fc6acaa815ee6d8af4e770a3ef9cf Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Wed, 17 Jul 2019 15:06:49 +0530
Subject: [PATCH 2122/2974] [MRG+1] Wrong value of log_count/INFO in stats
 (#3643)

* Update statscollectors.py

* Update statscollectors.py

* Update statscollectors.py

* Update crawler.py

* Update crawler.py

* corrected tests

* Update test_utils_log.py

* Update crawler.py

* Update crawler.py

* interchanged order

* correced
---
 scrapy/crawler.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index f573b2b7f..ded3c082b 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -39,14 +39,15 @@ class Crawler(object):
         self.settings = settings.copy()
         self.spidercls.update_settings(self.settings)
 
-        d = dict(overridden_settings(self.settings))
-        logger.info("Overridden settings: %(settings)r", {'settings': d})
-
         self.signals = SignalManager(self)
         self.stats = load_object(self.settings['STATS_CLASS'])(self)
 
         handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
         logging.root.addHandler(handler)
+
+        d = dict(overridden_settings(self.settings))
+        logger.info("Overridden settings: %(settings)r", {'settings': d})
+
         if get_scrapy_root_handler() is not None:
             # scrapy root handler already installed: update it with new settings
             install_scrapy_root_handler(self.settings)

From c44d49b238f1c6cfc07ffa2fbb65b267e19e381c Mon Sep 17 00:00:00 2001
From: "Deschner, Magdalena" <mdeschner@hotmail.de>
Date: Wed, 17 Jul 2019 13:13:52 +0200
Subject: [PATCH 2123/2974] minor PEP8 style changes

---
 docs/topics/practices.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 610a39ea9..a6d4f0d6d 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -35,8 +35,8 @@ Here's an example showing how to run a single spider with it.
         ...
 
     process = CrawlerProcess(settings={
-        'FEED_FORMAT':'json',
-        'FEED_URI':'items.json'
+        'FEED_FORMAT': 'json',
+        'FEED_URI': 'items.json'
     })
 
     process.crawl(MySpider)

From a94b5bef3a6ae658ec58a9a17bd149453aa855a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 27 Jun 2019 15:58:02 +0200
Subject: [PATCH 2124/2974] Write the 1.7 release notes and cover dropping
 Python 2 support in the upcoming 2.0

---
 docs/conf.py                          |  11 +
 docs/contributing.rst                 |   4 +
 docs/faq.rst                          |   2 +
 docs/news.rst                         | 344 +++++++++++++++++++++++++-
 docs/topics/contracts.rst             |   1 +
 docs/topics/downloader-middleware.rst |   2 +
 docs/topics/feed-exports.rst          |  13 +-
 docs/topics/items.rst                 |   2 +
 docs/topics/settings.rst              |   4 +
 9 files changed, 371 insertions(+), 12 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 80b8e3e90..f49f79cd5 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -30,6 +30,7 @@ extensions = [
     'scrapydocs',
     'sphinx.ext.autodoc',
     'sphinx.ext.coverage',
+    'sphinx.ext.intersphinx',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -74,6 +75,8 @@ language = 'en'
 # List of documents that shouldn't be included in the build.
 #unused_docs = []
 
+exclude_patterns = ['build']
+
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
 exclude_trees = ['.build']
@@ -250,3 +253,11 @@ coverage_ignore_pyobjects = [
     # Private exception used by the command-line interface implementation.
     r'^scrapy\.exceptions\.UsageError',
 ]
+
+
+# Options for the InterSphinx extension
+# -------------------------------------
+
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/3', None),
+}
diff --git a/docs/contributing.rst b/docs/contributing.rst
index b4f91ea8d..28dea74de 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -171,6 +171,8 @@ Scrapy:
   See https://help.github.com/articles/setting-your-username-in-git/ for
   setup instructions.
 
+.. _documentation-policies:
+
 Documentation policies
 ======================
 
@@ -196,6 +198,8 @@ Tests
 Tests are implemented using the `Twisted unit-testing framework`_, running
 tests requires `tox`_.
 
+.. _running-tests:
+
 Running tests
 -------------
 
diff --git a/docs/faq.rst b/docs/faq.rst
index 7105baeef..44f2b97bd 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -329,6 +329,8 @@ I'm scraping a XML document and my XPath selector doesn't return any items
 
 You may need to remove namespaces. See :ref:`removing-namespaces`.
 
+.. _faq-split-item:
+
 How to split an item into multiple items in an item pipeline?
 -------------------------------------------------------------
 
diff --git a/docs/news.rst b/docs/news.rst
index 75e3eef91..e0201ba11 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,326 @@
 Release notes
 =============
 
+.. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0,
+          planned for Q4 2019 or Q1 2020, will support **Python 3 only**.
+
+.. _release-1.7.0:
+
+Scrapy 1.7.0 (2019-07-18)
+-------------------------
+
+Highlights:
+
+* Improvements for crawls targeting multiple domains
+* A cleaner way to pass arguments to callbacks
+* A new class for JSON requests
+* Improvements for rule-based spiders
+* New features for feed exports
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+*   ``429`` is now part of the :setting:`RETRY_HTTP_CODES` setting by default
+
+    This change is **backward incompatible**. If you don’t want to retry
+    ``429``, you must override :setting:`RETRY_HTTP_CODES` accordingly.
+
+*   :class:`~scrapy.crawler.Crawler`,
+    :class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>` and
+    :class:`CrawlerRunner.create_crawler <scrapy.crawler.CrawlerRunner.create_crawler>`
+    no longer accept a :class:`~scrapy.spiders.Spider` subclass instance, they
+    only accept a :class:`~scrapy.spiders.Spider` subclass now.
+
+    :class:`~scrapy.spiders.Spider` subclass instances were never meant to
+    work, and they were not working as one would expect: instead of using the
+    passed :class:`~scrapy.spiders.Spider` subclass instance, their
+    :class:`~scrapy.spiders.Spider.from_crawler` method was called to generate
+    a new instance.
+
+*   Non-default values for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting
+    may stop working. Scheduler priority queue classes now need to handle
+    :class:`~scrapy.http.Request` objects instead of arbitrary Python data
+    structures.
+
+See also :ref:`1.7-deprecation-removals` below.
+
+
+New features
+~~~~~~~~~~~~
+
+*   A new scheduler priority queue,
+    :class:`scrapy.pqueues.DownloaderAwarePriorityQueue`, may be
+    :ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
+    scheduling improvement on crawls targetting multiple web domains, at the
+    cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
+
+*   A new :attr:`Request.cb_kwargs <scrapy.http.Request.cb_kwargs>` attribute
+    provides a cleaner way to pass keyword arguments to callback methods
+    (:issue:`1138`, :issue:`3563`)
+
+*   A new :class:`~scrapy.http.JSONRequest` class offers a more convenient way
+    to build JSON requests (:issue:`3504`, :issue:`3505`)
+
+*   A ``process_request`` callback passed to the :class:`~scrapy.spiders.Rule`
+    constructor now receives the :class:`~scrapy.http.Response` object that
+    originated the request as its second argument (:issue:`3682`)
+
+*   A new ``restrict_text`` parameter for the
+    :attr:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
+    constructor allows filtering links by linking text (:issue:`3622`,
+    :issue:`3635`)
+
+*   A new :setting:`FEED_STORAGE_S3_ACL` setting allows defining a custom ACL
+    for feeds exported to Amazon S3 (:issue:`3607`)
+
+*   A new :setting:`FEED_STORAGE_FTP_ACTIVE` setting allows using FTP’s active
+    connection mode for feeds exported to FTP servers (:issue:`3829`)
+
+*   A new :setting:`METAREFRESH_IGNORE_TAGS` setting allows overriding which
+    HTML tags are ignored when searching a response for HTML meta tags that
+    trigger a redirect (:issue:`1422`, :issue:`3768`)
+
+*   A new :reqmeta:`redirect_reasons` request meta key exposes the reason
+    (status code, meta refresh) behind every followed redirect (:issue:`3581`,
+    :issue:`3687`)
+
+*   The ``SCRAPY_CHECK`` variable is now set to the ``true`` string during runs
+    of the :command:`check` command, which allows :ref:`detecting contract
+    check runs from code <detecting-contract-check-runs>` (:issue:`3704`,
+    :issue:`3739`)
+
+*   A new :meth:`Item.deepcopy() <scrapy.item.Item.deepcopy>` method makes it
+    easier to :ref:`deep-copy items <copying-items>` (:issue:`1493`,
+    :issue:`3671`)
+
+*   :class:`~scrapy.extensions.corestats.CoreStats` also logs
+    ``elapsed_time_seconds`` now (:issue:`3638`)
+
+*   Exceptions from :class:`~scrapy.loader.ItemLoader` :ref:`input and output
+    processors <topics-loaders-processors>` are now more verbose
+    (:issue:`3836`, :issue:`3840`)
+
+*   :class:`~scrapy.crawler.Crawler`,
+    :class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>` and
+    :class:`CrawlerRunner.create_crawler <scrapy.crawler.CrawlerRunner.create_crawler>`
+    now fail gracefully if they receive a :class:`~scrapy.spiders.Spider`
+    subclass instance instead of the subclass itself (:issue:`2283`,
+    :issue:`3610`, :issue:`3872`)
+
+
+Bug fixes
+~~~~~~~~~
+
+*   :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception`
+    is now also invoked for generators (:issue:`220`, :issue:`2061`)
+
+*   System exceptions like KeyboardInterrupt_ are no longer caught
+    (:issue:`3726`)
+
+*   :meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` no
+    longer makes later calls to :meth:`ItemLoader.get_output_value()
+    <scrapy.loader.ItemLoader.get_output_value>` or
+    :meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` return
+    empty data (:issue:`3804`, :issue:`3819`)
+
+*   The images pipeline (:class:`~scrapy.pipelines.images.ImagesPipeline`) no
+    longer ignores these Amazon S3 settings: :setting:`AWS_ENDPOINT_URL`,
+    :setting:`AWS_REGION_NAME`, :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY`
+    (:issue:`3625`)
+
+*   Fixed a memory leak in :class:`~scrapy.pipelines.media.MediaPipeline`
+    affecting, for example, non-200 responses and exceptions from custom
+    middlewares (:issue:`3813`)
+
+*   Requests with private callbacks are now correctly unserialized from disk
+    (:issue:`3790`)
+
+*   :meth:`FormRequest.from_response() <scrapy.http.FormRequest.from_response>`
+    now handles invalid methods like major web browsers (:issue:`3777`,
+    :issue:`3794`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+*   A new topic, :ref:`topics-dynamic-content`, covers recommended approaches
+    to read dynamically-loaded data (:issue:`3703`)
+
+*   :ref:`topics-broad-crawls` now features information about memory usage
+    (:issue:`1264`, :issue:`3866`)
+
+*   The documentation of :class:`~scrapy.spiders.Rule` now covers how to access
+    the text of a link when using :class:`~scrapy.spiders.CrawlSpider`
+    (:issue:`3711`, :issue:`3712`)
+
+*   A new section, :ref:`httpcache-storage-custom`, covers writing a custom
+    cache storage backend for
+    :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
+    (:issue:`3683`, :issue:`3692`)
+
+*   A new :ref:`FAQ <faq>` entry, :ref:`faq-split-item`, explains what to do
+    when you want to split an item into multiple items from an item pipeline
+    (:issue:`2240`, :issue:`3672`)
+
+*   Updated the :ref:`FAQ entry about crawl order <faq-bfo-dfo>` to explain why
+    the first few requests rarely follow the desired order (:issue:`1739`,
+    :issue:`3621`)
+
+*   The :setting:`LOGSTATS_INTERVAL` setting (:issue:`3730`), the
+    :meth:`FilesPipeline.file_path <scrapy.pipelines.files.FilesPipeline.file_path>`
+    and
+    :meth:`ImagesPipeline.file_path <scrapy.pipelines.images.ImagesPipeline.file_path>`
+    methods (:issue:`2253`, :issue:`3609`) and the
+    :meth:`Crawler.stop() <scrapy.crawler.Crawler.stop>` method (:issue:`3842`)
+    are now documented
+
+*   Some parts of the documentation that were confusing or misleading are now
+    clearer (:issue:`1347`, :issue:`1789`, :issue:`2289`, :issue:`3069`,
+    :issue:`3615`, :issue:`3626`, :issue:`3668`, :issue:`3670`, :issue:`3673`,
+    :issue:`3728`, :issue:`3762`, :issue:`3861`, :issue:`3882`)
+
+*   Minor documentation fixes (:issue:`3648`, :issue:`3649`, :issue:`3662`,
+    :issue:`3674`, :issue:`3676`, :issue:`3694`, :issue:`3724`, :issue:`3764`,
+    :issue:`3767`, :issue:`3791`, :issue:`3797`, :issue:`3806`, :issue:`3812`)
+
+.. _1.7-deprecation-removals:
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+The following deprecated APIs have been removed (:issue:`3578`):
+
+*   ``scrapy.conf`` (use :attr:`Crawler.settings
+    <scrapy.crawler.Crawler.settings>`)
+
+*   From ``scrapy.core.downloader.handlers``:
+
+    *   ``http.HttpDownloadHandler`` (use ``http10.HTTP10DownloadHandler``)
+
+*   ``scrapy.loader.ItemLoader._get_values`` (use ``_get_xpathvalues``)
+
+*   ``scrapy.loader.XPathItemLoader`` (use :class:`~scrapy.loader.ItemLoader`)
+
+*   ``scrapy.log`` (see :ref:`topics-logging`)
+
+*   From ``scrapy.pipelines``:
+
+    *   ``files.FilesPipeline.file_key`` (use ``file_path``)
+
+    *   ``images.ImagesPipeline.file_key`` (use ``file_path``)
+
+    *   ``images.ImagesPipeline.image_key`` (use ``file_path``)
+
+    *   ``images.ImagesPipeline.thumb_key`` (use ``thumb_path``)
+
+*   From both ``scrapy.selector`` and ``scrapy.selector.lxmlsel``:
+
+    *   ``HtmlXPathSelector`` (use :class:`~scrapy.selector.Selector`)
+
+    *   ``XmlXPathSelector`` (use :class:`~scrapy.selector.Selector`)
+
+    *   ``XPathSelector`` (use :class:`~scrapy.selector.Selector`)
+
+    *   ``XPathSelectorList`` (use :class:`~scrapy.selector.Selector`)
+
+*   From ``scrapy.selector.csstranslator``:
+
+    *   ``ScrapyGenericTranslator`` (use parsel.csstranslator.GenericTranslator_)
+
+    *   ``ScrapyHTMLTranslator`` (use parsel.csstranslator.HTMLTranslator_)
+
+    *   ``ScrapyXPathExpr`` (use parsel.csstranslator.XPathExpr_)
+
+*   From :class:`~scrapy.selector.Selector`:
+
+    *   ``_root`` (both the constructor argument and the object property, use
+        ``root``)
+
+    *   ``extract_unquoted`` (use ``getall``)
+
+    *   ``select`` (use ``xpath``)
+
+*   From :class:`~scrapy.selector.SelectorList`:
+
+    *   ``extract_unquoted`` (use ``getall``)
+
+    *   ``select`` (use ``xpath``)
+
+    *   ``x`` (use ``xpath``)
+
+*   ``scrapy.spiders.BaseSpider`` (use :class:`~scrapy.spiders.Spider`)
+
+*   From :class:`~scrapy.spiders.Spider` (and subclasses):
+
+    *   ``DOWNLOAD_DELAY`` (use :ref:`download_delay
+        <spider-download_delay-attribute>`)
+
+    *   ``set_crawler`` (use :meth:`~scrapy.spiders.Spider.from_crawler`)
+
+*   ``scrapy.spiders.spiders`` (use :class:`~scrapy.spiderloader.SpiderLoader`)
+
+*   ``scrapy.telnet`` (use :mod:`scrapy.extensions.telnet`)
+
+*   From ``scrapy.utils.python``:
+
+    *   ``str_to_unicode`` (use ``to_unicode``)
+
+    *   ``unicode_to_str`` (use ``to_bytes``)
+
+*   ``scrapy.utils.response.body_or_str``
+
+The following deprecated settings have also been removed (:issue:`3578`):
+
+*   ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+*   The ``queuelib.PriorityQueue`` value for the
+    :setting:`SCHEDULER_PRIORITY_QUEUE` setting is deprecated. Use
+    :class:`scrapy.pqueues.ScrapyPriorityQueue` instead.
+
+*   ``process_request`` callbacks passed to :class:`~scrapy.spiders.Rule` that
+    do not accept two arguments are deprecated.
+
+*   The following modules are deprecated:
+
+    *   ``scrapy.utils.http`` (use `w3lib.http`_)
+
+    *   ``scrapy.utils.markup`` (use `w3lib.html`_)
+
+    *   ``scrapy.utils.multipart`` (use `urllib3`_)
+
+*   The ``scrapy.utils.datatypes.MergeDict`` class is deprecated for Python 3
+    code bases. Use :class:`~collections.ChainMap` instead. (:issue:`3878`)
+
+*   The ``scrapy.utils.gz.is_gzipped`` function is deprecated. Use
+    ``scrapy.utils.gz.gzip_magic_number`` instead.
+
+.. _urllib3: https://urllib3.readthedocs.io/en/latest/index.html
+.. _w3lib.html: https://w3lib.readthedocs.io/en/latest/w3lib.html#module-w3lib.html
+.. _w3lib.http: https://w3lib.readthedocs.io/en/latest/w3lib.html#module-w3lib.http
+
+
+Other changes
+~~~~~~~~~~~~~
+
+*   It is now possible to run all tests from the same tox_ environment in
+    parallel; the documentation now covers :ref:`this and other ways to run
+    tests <running-tests>` (:issue:`3707`)
+
+*   It is now possible to generate an API documentation coverage report
+    (:issue:`3806`, :issue:`3810`, :issue:`3860`)
+
+*   The :ref:`documentation policies <documentation-policies>` now require
+    docstrings_ (:issue:`3701`) that follow `PEP 257`_ (:issue:`3748`)
+
+*   Internal fixes and cleanup (:issue:`3629`, :issue:`3643`, :issue:`3684`,
+    :issue:`3698`, :issue:`3734`, :issue:`3735`, :issue:`3736`, :issue:`3737`,
+    :issue:`3809`, :issue:`3821`, :issue:`3825`, :issue:`3827`, :issue:`3833`,
+    :issue:`3857`, :issue:`3877`)
+
 .. _release-1.6.0:
 
 Scrapy 1.6.0 (2019-01-30)
@@ -2471,12 +2791,22 @@ First release of Scrapy.
 
 .. _AJAX crawleable urls: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started?csw=1
 .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
-.. _w3lib: https://github.com/scrapy/w3lib
-.. _scrapely: https://github.com/scrapy/scrapely
-.. _marshal: https://docs.python.org/2/library/marshal.html
-.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
-.. _lxml: http://lxml.de/
 .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
-.. _resource: https://docs.python.org/2/library/resource.html
+.. _Creating a pull request: https://help.github.com/en/articles/creating-a-pull-request
+.. _cssselect: https://github.com/scrapy/cssselect/
+.. _docstrings: https://docs.python.org/glossary.html#term-docstring
+.. _KeyboardInterrupt: https://docs.python.org/library/exceptions.html#KeyboardInterrupt
+.. _lxml: http://lxml.de/
+.. _marshal: https://docs.python.org/2/library/marshal.html
+.. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator
+.. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator
+.. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr
+.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
 .. _queuelib: https://github.com/scrapy/queuelib
-.. _cssselect: https://github.com/SimonSapin/cssselect
+.. _resource: https://docs.python.org/2/library/resource.html
+.. _scrapely: https://github.com/scrapy/scrapely
+.. _tox: https://pypi.python.org/pypi/tox
+.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
+.. _w3lib: https://github.com/scrapy/w3lib
+.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
+.. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 05b1d161b..957761b76 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -126,6 +126,7 @@ response received::
                 if header not in response.headers:
                     raise ContractFail('X-CustomHeader not present')
 
+.. _detecting-contract-check-runs:
 
 Detecting check runs
 ====================
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index d7add4ec4..38a4fdb25 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -52,6 +52,8 @@ as its value.  For example, if you want to disable the user-agent middleware::
 Finally, keep in mind that some middlewares may need to be enabled through a
 particular setting. See each middleware documentation for more info.
 
+.. _topics-downloader-middleware-custom:
+
 Writing your own downloader middleware
 ======================================
 
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 7e32da6f4..af541db78 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -164,9 +164,9 @@ The feeds are stored in a FTP server.
  * Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
  * Required external libraries: none
 
-FTP supports two different connection modes: [active or passive](
-https://stackoverflow.com/a/1699163). Scrapy uses the passive connection mode
-by default. To use the active connection mode instead, set the
+FTP supports two different connection modes: `active or passive
+<https://stackoverflow.com/a/1699163>`_. Scrapy uses the passive connection
+mode by default. To use the active connection mode instead, set the
 :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
 
 .. _topics-feed-storage-s3:
@@ -320,8 +320,11 @@ FEED_STORAGE_FTP_ACTIVE
 
 Default: ``False``
 
-Whether to use [active mode](https://stackoverflow.com/a/1699163) when exporting feeds
-to a FTP server.
+Whether to use the active connection mode when exporting feeds to an FTP server
+(``True``) or use the passive connection mode instead (``False``, default).
+
+For information about FTP connection modes, see `What is the difference between
+active and passive FTP? <https://stackoverflow.com/a/1699163>`_.
 
 .. setting:: FEED_STORAGE_S3_ACL
 
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index d744fd9ea..60fbc82f8 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -157,6 +157,8 @@ To access all populated values, just use the typical `dict API`_::
     [('price', 1000), ('name', 'Desktop PC')]
 
 
+.. _copying-items:
+
 Copying items
 -------------
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 371f21c72..fd46c614e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -30,6 +30,8 @@ Python `import search path`_.
 
 .. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path
 
+.. _populating-settings:
+
 Populating the settings
 =======================
 
@@ -538,6 +540,8 @@ amount of time between requests, but uses a random interval between 0.5 * :setti
 When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced
 per ip address instead of per domain.
 
+.. _spider-download_delay-attribute:
+
 You can also change this setting per spider by setting ``download_delay``
 spider attribute.
 

From 4e23d70dd34c180d19c4751265005cea6da43927 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 18 Jul 2019 15:38:25 +0200
Subject: [PATCH 2125/2974] =?UTF-8?q?Bump=20version:=201.6.0=20=E2=86=92?=
 =?UTF-8?q?=201.7.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 8cecb7ad4..70affe63f 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.6.0
+current_version = 1.7.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index dc1e644a1..bd8bf882d 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.6.0
+1.7.0

From ae4eab9843752e7cf75420a5d4f4fa58f8da8e50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 18 Jul 2019 17:02:48 +0200
Subject: [PATCH 2126/2974] Cover the 1.7.1 PyPI repackaging in the release
 notes

---
 docs/news.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index e0201ba11..921089ccd 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -11,6 +11,10 @@ Release notes
 Scrapy 1.7.0 (2019-07-18)
 -------------------------
 
+.. note:: Make sure you install Scrapy 1.7.1. The Scrapy 1.7.0 package in PyPI
+          is the result of an erroneous commit tagging and does not include all
+          the changes described below.
+
 Highlights:
 
 * Improvements for crawls targeting multiple domains

From 42743fd9dd9d7116848fd3ad6b657453dd0b117d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 18 Jul 2019 20:49:25 +0500
Subject: [PATCH 2127/2974] Move tls_verbose_logging extraction from __init__
 to from_settings.

---
 docs/topics/settings.rst                  |  4 ++--
 scrapy/core/downloader/contextfactory.py  | 13 +++++++------
 scrapy/core/downloader/handlers/http11.py |  2 +-
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 53c624679..8705a5249 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -440,8 +440,8 @@ or even enable client-side authentication (and various other things).
 
 If you do use a custom ContextFactory, make sure its ``__init__`` method
 accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
-:setting:`DOWNLOADER_CLIENT_TLS_METHOD`) and a ``settings`` parameter (this is
-the Scrapy :class:`~scrapy.settings.Settings` object).
+:setting:`DOWNLOADER_CLIENT_TLS_METHOD`) and a ``tls_verbose_logging``
+parameter (``bool``).
 
 .. setting:: DOWNLOADER_CLIENT_TLS_METHOD
 
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 80c784f5a..d5d238b9c 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -29,17 +29,18 @@ if twisted_version >= (14, 0, 0):
          understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
         """
 
-        def __init__(self, method=SSL.SSLv23_METHOD, settings=None, *args, **kwargs):
+        def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs):
             super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
             self._ssl_method = method
-            if settings:
-                self.tls_verbose_logging = settings['DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING']
-            else:
-                self.tls_verbose_logging = False
+            self.tls_verbose_logging = tls_verbose_logging
 
         @classmethod
         def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
-            return cls(method=method, settings=settings, *args, **kwargs)
+            if settings:
+                tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
+            else:
+                tls_verbose_logging = False
+            return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs)
 
         def getCertificateOptions(self):
             # setting verify=True will require you to provide CAs
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 9b0c7977d..deb0f9d21 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -51,7 +51,7 @@ class HTTP11DownloadHandler(object):
             self._contextFactory = create_instance(self._contextFactoryClass, settings=settings, crawler=None)
             msg = """
  '%s' does not accept `method` argument (type OpenSSL.SSL method,\
- e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `settings` argument.\
+ e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument.\
  Please upgrade your context factory class to handle them or ignore them.""" % (
                 settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
             warnings.warn(msg)

From 95dd2df7b5dc6836a784a2b373009b17ca2eb475 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 18 Jul 2019 20:51:26 +0500
Subject: [PATCH 2128/2974] Drop an unused import.

---
 scrapy/core/downloader/contextfactory.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index d5d238b9c..188d9f917 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -2,7 +2,6 @@ from OpenSSL import SSL
 from twisted.internet.ssl import ClientContextFactory
 
 from scrapy import twisted_version
-from scrapy.utils.misc import create_instance
 
 if twisted_version >= (14, 0, 0):
 

From c6453800cd612297aa635636477c68a109a4a542 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 18 Jul 2019 22:17:39 +0500
Subject: [PATCH 2129/2974] Remove an unneeded if.

---
 scrapy/core/downloader/contextfactory.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 188d9f917..5ac20c0bb 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -35,10 +35,7 @@ if twisted_version >= (14, 0, 0):
 
         @classmethod
         def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
-            if settings:
-                tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
-            else:
-                tls_verbose_logging = False
+            tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
             return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs)
 
         def getCertificateOptions(self):

From b8a43011e75da4353b0d5ef314c96cb1276f12f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 18 Jul 2019 18:47:29 +0200
Subject: [PATCH 2130/2974] Cover Scrapy 1.7.1 in the release notes

---
 docs/news.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 921089ccd..a0f0c5697 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -6,6 +6,11 @@ Release notes
 .. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0,
           planned for Q4 2019 or Q1 2020, will support **Python 3 only**.
 
+Scrapy 1.7.1 (2019-07-18)
+-------------------------
+
+Re-packaging of Scrapy 1.7.0, which was missing some changes in PyPI.
+
 .. _release-1.7.0:
 
 Scrapy 1.7.0 (2019-07-18)

From 43d5b5a524ff2cce6fd4620f8e2460489da39f42 Mon Sep 17 00:00:00 2001
From: Kristobal Junta <junta.kristobal@gmail.com>
Date: Mon, 22 Jul 2019 10:19:08 +0300
Subject: [PATCH 2131/2974] fix default RETRY_HTTP_CODES value in docs

---
 docs/topics/downloader-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 38a4fdb25..a3780a177 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -963,7 +963,7 @@ precedence over the :setting:`RETRY_TIMES` setting.
 RETRY_HTTP_CODES
 ^^^^^^^^^^^^^^^^
 
-Default: ``[500, 502, 503, 504, 522, 524, 408]``
+Default: ``[500, 502, 503, 504, 522, 524, 408, 429]``
 
 Which HTTP response codes to retry. Other errors (DNS lookup issues,
 connections lost, etc) are always retried.

From 7e622af4e5b0f49c88101ad370941b33e4833e1e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 22 Jul 2019 14:53:17 -0300
Subject: [PATCH 2132/2974] Fix ConfigParser import in py2

---
 scrapy/utils/conf.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 26d66eaf8..fb7ca3310 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -1,10 +1,13 @@
 import os
 import sys
 import numbers
-import configparser
 from operator import itemgetter
 
 import six
+if six.PY2:
+    from ConfigParser import SafeConfigParser as ConfigParser
+else:
+    from configparser import ConfigParser
 
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
@@ -94,7 +97,7 @@ def init_env(project='default', set_syspath=True):
 def get_config(use_closest=True):
     """Get Scrapy config file as a ConfigParser"""
     sources = get_sources(use_closest)
-    cfg = configparser.ConfigParser()
+    cfg = ConfigParser()
     cfg.read(sources)
     return cfg
 

From 7843101f9abad302e6c9c997f9d2a7adff98380b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 23 Jul 2019 12:04:26 +0200
Subject: [PATCH 2133/2974] Cover Scrapy 1.7.2 in the release notes

---
 docs/news.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index a0f0c5697..d79844ed2 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -6,6 +6,12 @@ Release notes
 .. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0,
           planned for Q4 2019 or Q1 2020, will support **Python 3 only**.
 
+Scrapy 1.7.2 (2019-07-23)
+-------------------------
+
+Fix Python 2 support (:issue:`3889`, :issue:`3893`, :issue:`3896`).
+
+
 Scrapy 1.7.1 (2019-07-18)
 -------------------------
 

From 7551689c75a1f2b4dbed72184f1dabab2f6c3c4a Mon Sep 17 00:00:00 2001
From: Lucy Wang <wxitb2017@gmail.com>
Date: Fri, 26 Jul 2019 09:07:29 +0800
Subject: [PATCH 2134/2974] s3 file store should accept all supported headers

---
 scrapy/pipelines/files.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 2145e6d2b..ea06d2ae8 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -189,6 +189,19 @@ class S3FilesStore(object):
             'X-Amz-Grant-Read': 'GrantRead',
             'X-Amz-Grant-Read-ACP': 'GrantReadACP',
             'X-Amz-Grant-Write-ACP': 'GrantWriteACP',
+            'X-Amz-Object-Lock-Legal-Hold': 'ObjectLockLegalHoldStatus',
+            'X-Amz-Object-Lock-Mode': 'ObjectLockMode',
+            'X-Amz-Object-Lock-Retain-Until-Date': 'ObjectLockRetainUntilDate',
+            'X-Amz-Request-Payer': 'RequestPayer',
+            'X-Amz-Server-Side-Encryption': 'ServerSideEncryption',
+            'X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id': 'SSEKMSKeyId',
+            'X-Amz-Server-Side-Encryption-Context': 'SSEKMSEncryptionContext',
+            'X-Amz-Server-Side-Encryption-Customer-Algorithm': 'SSECustomerAlgorithm',
+            'X-Amz-Server-Side-Encryption-Customer-Key': 'SSECustomerKey',
+            'X-Amz-Server-Side-Encryption-Customer-Key-Md5': 'SSECustomerKeyMD5',
+            'X-Amz-Storage-Class': 'StorageClass',
+            'X-Amz-Tagging': 'Tagging',
+            'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation',
         })
         extra = {}
         for key, value in six.iteritems(headers):

From f21dc24a266a9e45088fc71daee97cbe9b11d4a4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 30 Jul 2019 18:16:12 +0500
Subject: [PATCH 2135/2974] Fix memory handling and error handling in
 utils.ssl.get_temp_key_info.

---
 scrapy/utils/ssl.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index 5db1608bf..e54232abd 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -23,12 +23,12 @@ def get_temp_key_info(ssl_object):
 
     # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key()
     temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **")
-    pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p)
-    if temp_key_p == pyOpenSSLutil.ffi.NULL:
+    if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p):
         return None
-
     temp_key = temp_key_p[0]
-    pyOpenSSLutil.ffi.gc(temp_key, pyOpenSSLutil.lib.EVP_PKEY_free)
+    if temp_key == pyOpenSSLutil.ffi.NULL:
+        return None
+    temp_key = pyOpenSSLutil.ffi.gc(temp_key, pyOpenSSLutil.lib.EVP_PKEY_free)
     key_info = []
     key_type = pyOpenSSLutil.lib.EVP_PKEY_id(temp_key)
     if key_type == pyOpenSSLutil.lib.EVP_PKEY_RSA:
@@ -38,7 +38,7 @@ def get_temp_key_info(ssl_object):
     elif key_type == pyOpenSSLutil.lib.EVP_PKEY_EC:
         key_info.append('ECDH')
         ec_key = pyOpenSSLutil.lib.EVP_PKEY_get1_EC_KEY(temp_key)
-        pyOpenSSLutil.ffi.gc(ec_key, pyOpenSSLutil.lib.EC_KEY_free)
+        ec_key = pyOpenSSLutil.ffi.gc(ec_key, pyOpenSSLutil.lib.EC_KEY_free)
         nid = pyOpenSSLutil.lib.EC_GROUP_get_curve_name(pyOpenSSLutil.lib.EC_KEY_get0_group(ec_key))
         cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid)
         if cname == pyOpenSSLutil.ffi.NULL:

From 7333fc02aa842ec4506ca00c345dcba72739f35c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Tue, 30 Jul 2019 23:16:11 -0300
Subject: [PATCH 2136/2974] Pin Travis-ci build environment to previous
 default: Trusty

Travis-ci changed the default build environment to Xenial as explained in https://blog.travis-ci.com/2019-04-15-xenial-default-build-environment
This causes builds meant for Debian Jessie to break as noted by @wRAR in https://github.com/scrapy/scrapy/issues/3917#issuecomment-516426389

This change pins the environment to known working ubuntu trusty distribution prior to dropping Jessie support and upgrade to Xenial as base.

Closes #1369
---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 08b0bf119..3116d9b48 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,5 @@
 language: python
+dist: trusty
 branches:
   only:
     - master

From a25e09ecdd6bb53779b713dead19637128259ee2 Mon Sep 17 00:00:00 2001
From: Renne Rocha <renne@rennerocha.com>
Date: Mon, 29 Jul 2019 19:07:34 -0300
Subject: [PATCH 2137/2974] Added constrain on lxml version based on Python
 version

---
 requirements-py3.txt  | 3 ++-
 setup.py              | 3 ++-
 tests/constraints.txt | 3 +--
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/requirements-py3.txt b/requirements-py3.txt
index 5a5d4c95a..478ed0010 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -1,5 +1,6 @@
 Twisted>=17.9.0
-lxml>=3.2.4
+lxml;python_version!="3.4"
+lxml<=4.3.5;python_version=="3.4"
 pyOpenSSL>=0.13.1
 cssselect>=0.9
 queuelib>=1.1.1
diff --git a/setup.py b/setup.py
index 4dc6d18c1..ee0aaabf0 100644
--- a/setup.py
+++ b/setup.py
@@ -69,7 +69,8 @@ setup(
         'Twisted>=13.1.0,<=19.2.0;python_version=="3.4"',
         'w3lib>=1.17.0',
         'queuelib',
-        'lxml',
+        'lxml;python_version!="3.4"',
+        'lxml<=4.3.5;python_version=="3.4"',
         'pyOpenSSL',
         'cssselect>=0.9',
         'six>=1.5.2',
diff --git a/tests/constraints.txt b/tests/constraints.txt
index e59e68b3f..5655ac2d3 100644
--- a/tests/constraints.txt
+++ b/tests/constraints.txt
@@ -1,2 +1 @@
-Twisted!=18.4.0
-lxml!=4.2.2
\ No newline at end of file
+Twisted!=18.4.0
\ No newline at end of file

From 783d61d32aba53208b2b0bb9a1d82a56dbcbabd6 Mon Sep 17 00:00:00 2001
From: sbs2001 <shivam.sandbhor@gmail.com>
Date: Thu, 1 Aug 2019 14:11:27 +0530
Subject: [PATCH 2138/2974] [MRG+1] Update _monkeypatches.py (#3907)

* Update _monkeypatches.py

The workarounds are not required assuming the bugs regarding urlparse are absent in  Python versions >2.7. We already exit the program if Python  version<2.7 in the __init__.py(line 17).The monkeypatches are deployed after this check at line 27  in  the __init__.py .

* Update _monkeypatches.py

Added the second workaround.

* Update _monkeypatches.py

* Update _monkeypatches.py
---
 scrapy/_monkeypatches.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index 935c4bfa3..b68099cad 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -4,12 +4,7 @@ from six.moves import copyreg
 
 if six.PY2:
     from urlparse import urlparse
-
-    # workaround for https://bugs.python.org/issue7904 - Python < 2.7
-    if urlparse('s3://bucket/key').netloc != 'bucket':
-        from urlparse import uses_netloc
-        uses_netloc.append('s3')
-
+    
     # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
     if urlparse('s3://bucket/key?key=value').query != 'key=value':
         from urlparse import uses_query

From a12e8251e065182ab09b9b5a65a86c44a606c4a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 1 Aug 2019 17:06:53 +0200
Subject: [PATCH 2139/2974] Cover Scrapy 1.7.3 in the release notes

---
 docs/news.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index d79844ed2..ce5b8b406 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -6,6 +6,11 @@ Release notes
 .. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0,
           planned for Q4 2019 or Q1 2020, will support **Python 3 only**.
 
+Scrapy 1.7.3 (2019-08-01)
+-------------------------
+
+Enforce lxml 4.3.5 or lower for Python 3.4 (:issue:`3912`, :issue:`3918`).
+
 Scrapy 1.7.2 (2019-07-23)
 -------------------------
 

From 8e813953bd4b25b60d30981a948e48fee43f8dad Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Fri, 2 Aug 2019 13:13:29 +0530
Subject: [PATCH 2140/2974] [MRG+1] [GSoC 2019] Interface for robots.txt
 parsers (#3796)

Make the robots.txt parser configurable through the new ROBOTSTXT_PARSER setting, support the Reppy and Robotexclusionrulesparser parsers, and allow implementing custom robots.txt parsers.
---
 .travis.yml                                  |   6 +
 docs/topics/downloader-middleware.rst        |  79 +++++++++++
 docs/topics/settings.rst                     |  10 ++
 scrapy/downloadermiddlewares/robotstxt.py    |  41 ++----
 scrapy/robotstxt.py                          | 112 +++++++++++++++
 scrapy/settings/default_settings.py          |   1 +
 tests/test_downloadermiddleware_robotstxt.py |  25 ++++
 tests/test_robotstxt_interface.py            | 142 +++++++++++++++++++
 tox.ini                                      |  14 ++
 9 files changed, 404 insertions(+), 26 deletions(-)
 create mode 100644 scrapy/robotstxt.py
 create mode 100644 tests/test_robotstxt_interface.py

diff --git a/.travis.yml b/.travis.yml
index 3116d9b48..138e81c64 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,6 +27,12 @@ matrix:
       sudo: true
     - python: 3.6
       env: TOXENV=docs
+    - python: 3.7
+      env: TOXENV=py37-extra-deps
+      dist: xenial
+      sudo: true
+    - python: 2.7
+      env: TOXENV=py27-extra-deps
 install:
   - |
       if [ "$TOXENV" = "pypy" ]; then
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index a3780a177..616b56101 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -989,6 +989,17 @@ RobotsTxtMiddleware
     To make sure Scrapy respects robots.txt make sure the middleware is enabled
     and the :setting:`ROBOTSTXT_OBEY` setting is enabled.
 
+    This middleware has to be combined with a robots.txt_ parser.
+
+    Scrapy ships with support for the following robots.txt_ parsers:
+
+    * :ref:`RobotFileParser <python-robotfileparser>` (default)
+    * :ref:`Reppy <reppy-parser>`
+    * :ref:`Robotexclusionrulesparser <rerp-parser>`
+
+    You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
+    setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
+
 .. reqmeta:: dont_obey_robotstxt
 
 If :attr:`Request.meta <scrapy.http.Request.meta>` has
@@ -996,6 +1007,74 @@ If :attr:`Request.meta <scrapy.http.Request.meta>` has
 the request will be ignored by this middleware even if
 :setting:`ROBOTSTXT_OBEY` is enabled.
 
+.. _python-robotfileparser:
+
+RobotFileParser
+~~~~~~~~~~~~~~~
+
+`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ is 
+Python's inbuilt ``robots.txt`` parser. The parser is fully compliant with `Martijn Koster's 
+1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_. It lacks
+support for wildcard matching. Scrapy uses this parser by default.
+
+In order to use this parser, set:
+
+* :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser``
+
+.. _rerp-parser:
+
+Robotexclusionrulesparser
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+`Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ is fully compliant
+with `Martijn Koster's 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_,
+with support for wildcard matching.
+
+In order to use this parser:
+
+* Install `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ by running
+  ``pip install robotexclusionrulesparser``
+
+* Set :setting:`ROBOTSTXT_PARSER` setting to
+  ``scrapy.robotstxt.RerpRobotParser``
+
+.. _reppy-parser:
+
+Reppy parser
+~~~~~~~~~~~~
+
+`Reppy <https://github.com/seomoz/reppy/>`_ is a Python wrapper around `Robots Exclusion
+Protocol Parser for C++ <https://github.com/seomoz/rep-cpp>`_. The parser is fully compliant
+with `Martijn Koster's 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_,
+with support for wildcard matching. Unlike
+`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ and
+`Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_, it uses the length based
+rule, in particular for ``Allow`` and ``Disallow`` directives, where the most specific
+rule based on the length of the path trumps the less specific (shorter) rule.
+
+In order to use this parser:
+
+* Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
+
+* Set :setting:`ROBOTSTXT_PARSER` setting to
+  ``scrapy.robotstxt.ReppyRobotParser``
+
+.. _support-for-new-robots-parser:
+
+Implementing support for a new parser
+-------------------------------------
+
+You can implement support for a new robots.txt_ parser by subclassing
+the abstract base class :class:`~scrapy.robotstxt.RobotParser` and
+implementing the methods described below.
+
+.. module:: scrapy.robotstxt
+   :synopsis: robots.txt parser interface and implementations
+
+.. autoclass:: RobotParser
+   :members:
+
+.. _robots.txt: http://www.robotstxt.org/
 
 DownloaderStats
 ---------------
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 85ae2a305..12606fe47 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1141,6 +1141,16 @@ If enabled, Scrapy will respect robots.txt policies. For more information see
     this option is enabled by default in settings.py file generated
     by ``scrapy startproject`` command.
 
+.. setting:: ROBOTSTXT_PARSER
+
+ROBOTSTXT_PARSER
+----------------
+
+Default: ``'scrapy.robotstxt.PythonRobotParser'``
+
+The parser backend to use for parsing ``robots.txt`` files. For more information see
+:ref:`topics-dlmw-robots`.
+
 .. setting:: SCHEDULER
 
 SCHEDULER
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 200245210..c5a60d355 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -5,8 +5,8 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
 """
 
 import logging
-
-from six.moves.urllib import robotparser
+import sys
+import re
 
 from twisted.internet.defer import Deferred, maybeDeferred
 from scrapy.exceptions import NotConfigured, IgnoreRequest
@@ -14,6 +14,7 @@ from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.python import to_native_str
+from scrapy.utils.misc import load_object
 
 logger = logging.getLogger(__name__)
 
@@ -24,10 +25,13 @@ class RobotsTxtMiddleware(object):
     def __init__(self, crawler):
         if not crawler.settings.getbool('ROBOTSTXT_OBEY'):
             raise NotConfigured
-
+        self._default_useragent = crawler.settings.get('USER_AGENT', 'Scrapy')
         self.crawler = crawler
-        self._useragent = crawler.settings.get('USER_AGENT')
         self._parsers = {}
+        self._parserimpl = load_object(crawler.settings.get('ROBOTSTXT_PARSER'))
+
+        # check if parser dependencies are met, this should throw an error otherwise.
+        self._parserimpl.from_crawler(self.crawler, b'')
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -43,7 +47,8 @@ class RobotsTxtMiddleware(object):
     def process_request_2(self, rp, request, spider):
         if rp is None:
             return
-        if not rp.can_fetch(to_native_str(self._useragent), request.url):
+        useragent = request.headers.get(b'User-Agent', self._default_useragent)
+        if not rp.allowed(request.url, useragent):
             logger.debug("Forbidden by robots.txt: %(request)s",
                          {'request': request}, extra={'spider': spider})
             self.crawler.stats.inc_value('robotstxt/forbidden')
@@ -62,13 +67,14 @@ class RobotsTxtMiddleware(object):
                 meta={'dont_obey_robotstxt': True}
             )
             dfd = self.crawler.engine.download(robotsreq, spider)
-            dfd.addCallback(self._parse_robots, netloc)
+            dfd.addCallback(self._parse_robots, netloc, spider)
             dfd.addErrback(self._logerror, robotsreq, spider)
             dfd.addErrback(self._robots_error, netloc)
             self.crawler.stats.inc_value('robotstxt/request_count')
 
         if isinstance(self._parsers[netloc], Deferred):
             d = Deferred()
+
             def cb(result):
                 d.callback(result)
                 return result
@@ -85,27 +91,10 @@ class RobotsTxtMiddleware(object):
                          extra={'spider': spider})
         return failure
 
-    def _parse_robots(self, response, netloc):
+    def _parse_robots(self, response, netloc, spider):
         self.crawler.stats.inc_value('robotstxt/response_count')
-        self.crawler.stats.inc_value(
-            'robotstxt/response_status_count/{}'.format(response.status))
-        rp = robotparser.RobotFileParser(response.url)
-        body = ''
-        if hasattr(response, 'text'):
-            body = response.text
-        else:  # last effort try
-            try:
-                body = response.body.decode('utf-8')
-            except UnicodeDecodeError:
-                # If we found garbage, disregard it:,
-                # but keep the lookup cached (in self._parsers)
-                # Running rp.parse() will set rp state from
-                # 'disallow all' to 'allow any'.
-                self.crawler.stats.inc_value('robotstxt/unicode_error_count')
-        # stdlib's robotparser expects native 'str' ;
-        # with unicode input, non-ASCII encoded bytes decoding fails in Python2
-        rp.parse(to_native_str(body).splitlines())
-
+        self.crawler.stats.inc_value('robotstxt/response_status_count/{}'.format(response.status))
+        rp = self._parserimpl.from_crawler(self.crawler, response.body)
         rp_dfd = self._parsers[netloc]
         self._parsers[netloc] = rp
         rp_dfd.callback(rp)
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
new file mode 100644
index 000000000..4bfb275fd
--- /dev/null
+++ b/scrapy/robotstxt.py
@@ -0,0 +1,112 @@
+import sys
+import logging
+from abc import ABCMeta, abstractmethod
+from six import with_metaclass
+
+from scrapy.utils.python import to_native_str, to_unicode
+
+logger = logging.getLogger(__name__)
+
+
+class RobotParser(with_metaclass(ABCMeta)):
+    @classmethod
+    @abstractmethod
+    def from_crawler(cls, crawler, robotstxt_body):
+        """Parse the content of a robots.txt_ file as bytes. This must be a class method.
+        It must return a new instance of the parser backend.
+
+        :param crawler: crawler which made the request
+        :type crawler: :class:`~scrapy.crawler.Crawler` instance
+
+        :param robotstxt_body: content of a robots.txt_ file.
+        :type robotstxt_body: bytes
+        """
+        pass
+
+    @abstractmethod
+    def allowed(self, url, user_agent):
+        """Return ``True`` if  ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``.
+
+        :param url: Absolute URL
+        :type url: string
+
+        :param user_agent: User agent
+        :type user_agent: string
+        """
+        pass
+
+
+class PythonRobotParser(RobotParser):
+    def __init__(self, robotstxt_body, spider):
+        from six.moves.urllib_robotparser import RobotFileParser
+        self.spider = spider
+        try:
+            robotstxt_body = to_native_str(robotstxt_body)
+        except UnicodeDecodeError:
+            # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
+            # Switch to 'allow all' state.
+            logger.warning("Failure while parsing robots.txt using %(parser)s."
+                           " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
+                           {'parser': "RobotFileParser"},
+                           exc_info=sys.exc_info(),
+                           extra={'spider': self.spider})
+            robotstxt_body = ''
+        self.rp = RobotFileParser()
+        self.rp.parse(robotstxt_body.splitlines())
+
+    @classmethod
+    def from_crawler(cls, crawler, robotstxt_body):
+        spider = None if not crawler else crawler.spider
+        o = cls(robotstxt_body, spider)
+        return o
+
+    def allowed(self, url, user_agent):
+        user_agent = to_native_str(user_agent)
+        url = to_native_str(url)
+        return self.rp.can_fetch(user_agent, url)
+
+
+class ReppyRobotParser(RobotParser):
+    def __init__(self, robotstxt_body, spider):
+        from reppy.robots import Robots
+        self.spider = spider
+        self.rp = Robots.parse('', robotstxt_body)
+
+    @classmethod
+    def from_crawler(cls, crawler, robotstxt_body):
+        spider = None if not crawler else crawler.spider
+        o = cls(robotstxt_body, spider)
+        return o
+
+    def allowed(self, url, user_agent):
+        return self.rp.allowed(url, user_agent)
+
+
+class RerpRobotParser(RobotParser):
+    def __init__(self, robotstxt_body, spider):
+        from robotexclusionrulesparser import RobotExclusionRulesParser
+        self.spider = spider
+        self.rp = RobotExclusionRulesParser()
+        try:
+            robotstxt_body = robotstxt_body.decode('utf-8')
+        except UnicodeDecodeError:
+            # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
+            # Switch to 'allow all' state.
+            logger.warning("Failure while parsing robots.txt using %(parser)s."
+                           " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
+                           {'parser': "RobotExclusionRulesParser"},
+                           exc_info=sys.exc_info(),
+                           extra={'spider': self.spider})
+            robotstxt_body = ''
+        self.rp.parse(robotstxt_body)
+
+    @classmethod
+    def from_crawler(cls, crawler, robotstxt_body):
+        spider = None if not crawler else crawler.spider
+        o = cls(robotstxt_body, spider)
+        return o
+
+    def allowed(self, url, user_agent):
+        user_agent = to_unicode(user_agent)
+        url = to_unicode(url)
+        return self.rp.is_allowed(user_agent, url)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 086adf48e..81fee543f 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -245,6 +245,7 @@ RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
 RETRY_PRIORITY_ADJUST = -1
 
 ROBOTSTXT_OBEY = False
+ROBOTSTXT_PARSER = 'scrapy.robotstxt.PythonRobotParser'
 
 SCHEDULER = 'scrapy.core.scheduler.Scheduler'
 SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 2b3548bdd..79f172848 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -10,6 +10,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response, TextResponse
 from scrapy.settings import Settings
 from tests import mock
+from tests.test_robotstxt_interface import rerp_available, reppy_available
 
 
 class RobotsTxtMiddlewareTest(unittest.TestCase):
@@ -41,6 +42,7 @@ User-Agent: UnicödeBöt
 Disallow: /some/randome/page.html
 """.encode('utf-8')
         response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
+
         def return_response(request, spider):
             deferred = Deferred()
             reactor.callFromThread(deferred.callback, response)
@@ -77,6 +79,7 @@ Disallow: /some/randome/page.html
         crawler = self.crawler
         crawler.settings.set('ROBOTSTXT_OBEY', True)
         response = Response('http://site.local/robots.txt', body=b'GIF89a\xd3\x00\xfe\x00\xa2')
+
         def return_response(request, spider):
             deferred = Deferred()
             reactor.callFromThread(deferred.callback, response)
@@ -99,6 +102,7 @@ Disallow: /some/randome/page.html
         crawler = self.crawler
         crawler.settings.set('ROBOTSTXT_OBEY', True)
         response = Response('http://site.local/robots.txt')
+
         def return_response(request, spider):
             deferred = Deferred()
             reactor.callFromThread(deferred.callback, response)
@@ -118,6 +122,7 @@ Disallow: /some/randome/page.html
     def test_robotstxt_error(self):
         self.crawler.settings.set('ROBOTSTXT_OBEY', True)
         err = error.DNSLookupError('Robotstxt address not found')
+
         def return_failure(request, spider):
             deferred = Deferred()
             reactor.callFromThread(deferred.errback, failure.Failure(err))
@@ -133,6 +138,7 @@ Disallow: /some/randome/page.html
     def test_robotstxt_immediate_error(self):
         self.crawler.settings.set('ROBOTSTXT_OBEY', True)
         err = error.DNSLookupError('Robotstxt address not found')
+
         def immediate_failure(request, spider):
             deferred = Deferred()
             deferred.errback(failure.Failure(err))
@@ -144,6 +150,7 @@ Disallow: /some/randome/page.html
 
     def test_ignore_robotstxt_request(self):
         self.crawler.settings.set('ROBOTSTXT_OBEY', True)
+
         def ignore_request(request, spider):
             deferred = Deferred()
             reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest()))
@@ -167,3 +174,21 @@ Disallow: /some/randome/page.html
         spider = None  # not actually used
         return self.assertFailure(maybeDeferred(middleware.process_request, request, spider),
                                   IgnoreRequest)
+
+
+class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
+    if not rerp_available():
+        skip = "Rerp parser is not installed"
+
+    def setUp(self):
+        super(RobotsTxtMiddlewareWithRerpTest, self).setUp()
+        self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser')
+
+
+class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):
+    if not reppy_available():
+        skip = "Reppy parser is not installed"
+
+    def setUp(self):
+        super(RobotsTxtMiddlewareWithReppyTest, self).setUp()
+        self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser')
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
new file mode 100644
index 000000000..2819786b5
--- /dev/null
+++ b/tests/test_robotstxt_interface.py
@@ -0,0 +1,142 @@
+# coding=utf-8
+from twisted.trial import unittest
+from scrapy.utils.python import to_native_str
+
+
+def reppy_available():
+    # check if reppy parser is installed
+    try:
+        from reppy.robots import Robots
+    except ImportError:
+        return False
+    return True
+
+
+def rerp_available():
+    # check if robotexclusionrulesparser is installed
+    try:
+        from robotexclusionrulesparser import RobotExclusionRulesParser
+    except ImportError:
+        return False
+    return True
+
+
+class BaseRobotParserTest:
+    def _setUp(self, parser_cls):
+        self.parser_cls = parser_cls
+
+    def test_allowed(self):
+        robotstxt_robotstxt_body = ("User-agent: * \n"
+                                    "Disallow: /disallowed \n"
+                                    "Allow: /allowed \n"
+                                    "Crawl-delay: 10".encode('utf-8'))
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+        self.assertTrue(rp.allowed("https://www.site.local/allowed", "*"))
+        self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*"))
+
+    def test_allowed_wildcards(self):
+        robotstxt_robotstxt_body = """User-agent: first
+                                Disallow: /disallowed/*/end$    
+
+                                User-agent: second
+                                Allow: /*allowed
+                                Disallow: /
+                                """.encode('utf-8')
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+
+        self.assertTrue(rp.allowed("https://www.site.local/disallowed", "first"))
+        self.assertFalse(rp.allowed("https://www.site.local/disallowed/xyz/end", "first"))
+        self.assertFalse(rp.allowed("https://www.site.local/disallowed/abc/end", "first"))
+        self.assertTrue(rp.allowed("https://www.site.local/disallowed/xyz/endinglater", "first"))
+
+        self.assertTrue(rp.allowed("https://www.site.local/allowed", "second"))
+        self.assertTrue(rp.allowed("https://www.site.local/is_still_allowed", "second"))
+        self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second"))
+
+    def test_length_based_precedence(self):
+        robotstxt_robotstxt_body = ("User-agent: * \n"
+                                    "Disallow: / \n"
+                                    "Allow: /page".encode('utf-8'))
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+        self.assertTrue(rp.allowed("https://www.site.local/page", "*"))
+
+    def test_order_based_precedence(self):
+        robotstxt_robotstxt_body = ("User-agent: * \n"
+                                    "Disallow: / \n"
+                                    "Allow: /page".encode('utf-8'))
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+        self.assertFalse(rp.allowed("https://www.site.local/page", "*"))
+
+    def test_empty_response(self):
+        """empty response should equal 'allow all'"""
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b'')
+        self.assertTrue(rp.allowed("https://site.local/", "*"))
+        self.assertTrue(rp.allowed("https://site.local/", "chrome"))
+        self.assertTrue(rp.allowed("https://site.local/index.html", "*"))
+        self.assertTrue(rp.allowed("https://site.local/disallowed", "*"))
+
+    def test_garbage_response(self):
+        """garbage response should be discarded, equal 'allow all'"""
+        robotstxt_robotstxt_body = b'GIF89a\xd3\x00\xfe\x00\xa2'
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+        self.assertTrue(rp.allowed("https://site.local/", "*"))
+        self.assertTrue(rp.allowed("https://site.local/", "chrome"))
+        self.assertTrue(rp.allowed("https://site.local/index.html", "*"))
+        self.assertTrue(rp.allowed("https://site.local/disallowed", "*"))
+
+    def test_unicode_url_and_useragent(self):
+        robotstxt_robotstxt_body = u"""
+        User-Agent: *
+        Disallow: /admin/
+        Disallow: /static/
+        # taken from https://en.wikipedia.org/robots.txt
+        Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
+        Disallow: /wiki/Käyttäjä:
+
+        User-Agent: UnicödeBöt
+        Disallow: /some/randome/page.html""".encode('utf-8')
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+        self.assertTrue(rp.allowed("https://site.local/", "*"))
+        self.assertFalse(rp.allowed("https://site.local/admin/", "*"))
+        self.assertFalse(rp.allowed("https://site.local/static/", "*"))
+        self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt"))
+        self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*"))
+        self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*"))
+        self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*"))
+        self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt"))
+
+
+class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
+    def setUp(self):
+        from scrapy.robotstxt import PythonRobotParser
+        super(PythonRobotParserTest, self)._setUp(PythonRobotParser)
+
+    def test_length_based_precedence(self):
+        raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.")
+
+    def test_allowed_wildcards(self):
+        raise unittest.SkipTest("RobotFileParser does not support wildcards.")
+
+
+class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
+    if not reppy_available():
+        skip = "Reppy parser is not installed"
+
+    def setUp(self):
+        from scrapy.robotstxt import ReppyRobotParser
+        super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser)
+
+    def test_order_based_precedence(self):
+        raise unittest.SkipTest("Rerp does not support order based directives precedence.")
+
+
+class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
+    if not rerp_available():
+        skip = "Rerp parser is not installed"
+
+    def setUp(self):
+        from scrapy.robotstxt import RerpRobotParser
+        super(RerpRobotParserTest, self)._setUp(RerpRobotParser)
+
+    def test_length_based_precedence(self):
+        raise unittest.SkipTest("Rerp does not support length based directives precedence.")
diff --git a/tox.ini b/tox.ini
index 157a8b3ed..c918731f4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -116,3 +116,17 @@ changedir = {[docs]changedir}
 deps = {[docs]deps}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
+
+[testenv:py37-extra-deps]
+basepython = python3.7
+deps = 
+    {[testenv:py34]deps}
+    reppy
+    robotexclusionrulesparser
+
+[testenv:py27-extra-deps]
+basepython = python2.7
+deps = 
+    {[testenv]deps}
+    reppy
+    robotexclusionrulesparser

From 9a4cd94244a01a7029e9f03959805a2e45a07874 Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Sat, 3 Aug 2019 22:46:06 +0530
Subject: [PATCH 2141/2974] fixes typo

---
 scrapy/core/downloader/webclient.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 1c89a0f9e..3a5890ed0 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -95,7 +95,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
 
 class ScrapyHTTPClientFactory(HTTPClientFactory):
     """Scrapy implementation of the HTTPClientFactory overwriting the
-    serUrl method to make use of our Url object that cache the parse
+    setUrl method to make use of our Url object that cache the parse
     result.
     """
 

From 18d0affc015cd1be02cc9b227c4d89007dd8c816 Mon Sep 17 00:00:00 2001
From: Shivam Sandbhor <shivam.sandbhor@gmail.com>
Date: Mon, 5 Aug 2019 16:53:35 +0530
Subject: [PATCH 2142/2974] Update reactor.py, updated 'if' sequencing ,
 possibly eliminating a bug if portrange=None

This should be the proper ordering.This is the explanation.
  If 'not portrange' is True ,it is guaranteed that `not hasattr(portrange, '__iter__')`  is also True  the converse of this is not always true.(for example, consider portrange=None, for such case we were executing the logic for `not hasattr(portrange, '__iter__')` . ).Such case is eliminated by this PR.
---
 scrapy/utils/reactor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 83186a372..eda7867e3 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -3,10 +3,10 @@ from twisted.internet import reactor, error
 def listen_tcp(portrange, host, factory):
     """Like reactor.listenTCP but tries different ports in a range."""
     assert len(portrange) <= 2, "invalid portrange: %s" % portrange
-    if not hasattr(portrange, '__iter__'):
-        return reactor.listenTCP(portrange, factory, interface=host)
     if not portrange:
         return reactor.listenTCP(0, factory, interface=host)
+    if not hasattr(portrange, '__iter__'):
+        return reactor.listenTCP(portrange, factory, interface=host)
     if len(portrange) == 1:
         return reactor.listenTCP(portrange[0], factory, interface=host)
     for x in range(portrange[0], portrange[1]+1):

From a8621bbc2929b904dfe14985dd1f94aeedc86648 Mon Sep 17 00:00:00 2001
From: tpeng <pengtaoo@gmail.com>
Date: Thu, 26 Jun 2014 11:28:03 +0200
Subject: [PATCH 2143/2974] show all the missing field when scrapes contract
 fails

---
 scrapy/contracts/default.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 20582503d..0f6bdbad2 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -84,6 +84,6 @@ class ScrapesContract(Contract):
     def post_process(self, output):
         for x in output:
             if isinstance(x, (BaseItem, dict)):
-                for arg in self.args:
-                    if not arg in x:
-                        raise ContractFail("'%s' field is missing" % arg)
+                missing = [arg for arg in self.args if arg not in x]
+                if missing:
+                    raise ContractFail("'%s' field is missing" % " ".join(missing))

From bff335cf7f90a0dc3aeafc199aaa81697d5f122b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 5 Aug 2019 15:47:58 +0200
Subject: [PATCH 2144/2974] Improve the error message in contract failures due
 to multiple missing fields

---
 scrapy/contracts/default.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 0f6bdbad2..7745959a7 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -86,4 +86,5 @@ class ScrapesContract(Contract):
             if isinstance(x, (BaseItem, dict)):
                 missing = [arg for arg in self.args if arg not in x]
                 if missing:
-                    raise ContractFail("'%s' field is missing" % " ".join(missing))
+                    raise ContractFail(
+                        "Missing fields: %s" % ", ".join(missing))

From 7b755a41a1046e031015130971c9b9cc8885176e Mon Sep 17 00:00:00 2001
From: Pengyu Chen <pengyu@libstarrify.so>
Date: Tue, 6 Aug 2019 15:18:59 +0100
Subject: [PATCH 2145/2974] Added: Properly handling quoted passwords in
 FEED_URI for FTP

---
 scrapy/extensions/feedexport.py | 4 ++--
 tests/test_feedexport.py        | 9 ++++++++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index d35551fdd..ce2846eba 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -11,7 +11,7 @@ import posixpath
 from tempfile import NamedTemporaryFile
 from datetime import datetime
 import six
-from six.moves.urllib.parse import urlparse
+from six.moves.urllib.parse import urlparse, unquote
 from ftplib import FTP
 
 from zope.interface import Interface, implementer
@@ -162,7 +162,7 @@ class FTPFeedStorage(BlockingFeedStorage):
         self.host = u.hostname
         self.port = int(u.port or '21')
         self.username = u.username
-        self.password = u.password
+        self.password = unquote(u.password)
         self.path = u.path
         self.use_active_mode = use_active_mode
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c5063253a..f32ac2a4b 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -6,7 +6,8 @@ import warnings
 from io import BytesIO
 import tempfile
 import shutil
-from six.moves.urllib.parse import urljoin, urlparse
+import string
+from six.moves.urllib.parse import urljoin, urlparse, quote
 from six.moves.urllib.request import pathname2url
 
 from zope.interface.verify import verifyObject
@@ -98,6 +99,12 @@ class FTPFeedStorageTest(unittest.TestCase):
         verifyObject(IFeedStorage, st)
         return self._assert_stores(st, path)
 
+    def test_uri_auth_quote(self):
+        # RFC3986: 3.2.1. User Information
+        pw_quoted = quote(string.punctuation, safe='')
+        st = FTPFeedStorage('ftp://foo:%s@example.com/some_path' % pw_quoted)
+        self.assertEqual(st.password, string.punctuation)
+
     @defer.inlineCallbacks
     def _assert_stores(self, storage, path):
         spider = self.get_test_spider()

From 5dbeece8da6686709eb00af9ffb91dd30a832a2e Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Wed, 7 Aug 2019 04:36:52 -0300
Subject: [PATCH 2146/2974] [MRG+1] Drop py34 support - Update CI envs (#3892)

* Drop py34 support

* Travis experiments

* More Travis experiments

* Bump Twisted version for py35+ (stretch)

* Remove Debian build

* Remove pinned lxml for Py34

* Fix merge error

* Remove unused tox env

* Add environment with pinned versions for py36

* Bump minimum Twisted version in py27; Envs with pinned versions for py27 and py35

* Add botocore as extra dep for py27 tests

* Update requirements-py2.txt

* Add botocore and Pillow as extra dependencies
---
 .travis.yml                | 50 ++++++++++----------
 README.rst                 |  2 +-
 docs/faq.rst               |  2 +-
 docs/intro/install.rst     |  2 +-
 requirements-py2.txt       | 25 ++++++----
 requirements-py3.txt       | 26 +++++++----
 setup.py                   | 26 +++++------
 tests/requirements-py2.txt |  7 +--
 tests/requirements-py3.txt |  9 ++--
 tox.ini                    | 94 +++++++++++++++++++-------------------
 10 files changed, 128 insertions(+), 115 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 138e81c64..0190a7f4d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,5 @@
 language: python
-dist: trusty
+dist: xenial
 branches:
   only:
     - master
@@ -7,32 +7,28 @@ branches:
     - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
 matrix:
   include:
-    - python: 2.7
-      env: TOXENV=py27
-    - python: 2.7
-      env: TOXENV=jessie
-    - python: 2.7
-      env: TOXENV=pypy
-    - python: 2.7
-      env: TOXENV=pypy3
-    - python: 3.4
-      env: TOXENV=py34
-    - python: 3.5
-      env: TOXENV=py35
-    - python: 3.6
-      env: TOXENV=py36
-    - python: 3.7
-      env: TOXENV=py37
-      dist: xenial
-      sudo: true
-    - python: 3.6
-      env: TOXENV=docs
-    - python: 3.7
-      env: TOXENV=py37-extra-deps
-      dist: xenial
-      sudo: true
-    - python: 2.7
-      env: TOXENV=py27-extra-deps
+    - env: TOXENV=py27
+      python: 2.7
+    - env: TOXENV=py27-pinned
+      python: 2.7
+    - env: TOXENV=py27-extra-deps
+      python: 2.7
+    - env: TOXENV=pypy
+      python: 2.7
+    - env: TOXENV=pypy3
+      python: 3.5
+    - env: TOXENV=py35
+      python: 3.5
+    - env: TOXENV=py35-pinned
+      python: 3.5
+    - env: TOXENV=py36
+      python: 3.6
+    - env: TOXENV=py37
+      python: 3.7
+    - env: TOXENV=py37-extra-deps
+      python: 3.7
+    - env: TOXENV=docs
+      python: 3.6
 install:
   - |
       if [ "$TOXENV" = "pypy" ]; then
diff --git a/README.rst b/README.rst
index c28d217ff..bd82bff06 100644
--- a/README.rst
+++ b/README.rst
@@ -40,7 +40,7 @@ https://scrapy.org
 Requirements
 ============
 
-* Python 2.7 or Python 3.4+
+* Python 2.7 or Python 3.5+
 * Works on Linux, Windows, Mac OSX, BSD
 
 Install
diff --git a/docs/faq.rst b/docs/faq.rst
index 44f2b97bd..9733471bf 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
 What Python versions does Scrapy support?
 -----------------------------------------
 
-Scrapy is supported under Python 2.7 and Python 3.4+
+Scrapy is supported under Python 2.7 and Python 3.5+
 under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
 Python 2.6 support was dropped starting at Scrapy 0.20.
 Python 3 support was added in Scrapy 1.1.
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index daec7fcb7..2bf98dbdc 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -7,7 +7,7 @@ Installation guide
 Installing Scrapy
 =================
 
-Scrapy runs on Python 2.7 and Python 3.4 or above
+Scrapy runs on Python 2.7 and Python 3.5 or above
 under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
 
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
diff --git a/requirements-py2.txt b/requirements-py2.txt
index 0771aae3a..9e6944240 100644
--- a/requirements-py2.txt
+++ b/requirements-py2.txt
@@ -1,10 +1,17 @@
-Twisted>=13.1.0
-lxml
-pyOpenSSL
-cssselect>=0.9
-queuelib
-w3lib>=1.17.0
-six>=1.5.2
+parsel>=1.5.0
 PyDispatcher>=2.0.5
-parsel>=1.5
-service_identity
+w3lib>=1.17.0
+
+pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
+queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
+cryptography>=2.0  # Earlier versions would fail to install
+
+# Reference versions taken from
+# https://packages.ubuntu.com/xenial/python/
+# https://packages.ubuntu.com/xenial/zope/
+cssselect>=0.9.1
+lxml>=3.5.0
+service_identity>=16.0.0
+six>=1.10.0
+Twisted>=16.0.0
+zope.interface>=4.1.3
diff --git a/requirements-py3.txt b/requirements-py3.txt
index 478ed0010..cd183a525 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -1,11 +1,17 @@
-Twisted>=17.9.0
-lxml;python_version!="3.4"
-lxml<=4.3.5;python_version=="3.4"
-pyOpenSSL>=0.13.1
-cssselect>=0.9
-queuelib>=1.1.1
-w3lib>=1.17.0
-six>=1.5.2
+parsel>=1.5.0
 PyDispatcher>=2.0.5
-parsel>=1.5
-service_identity
+Twisted>=17.9.0
+w3lib>=1.17.0
+
+pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
+queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
+cryptography>=2.0  # Earlier versions would fail to install
+
+# Reference versions taken from
+# https://packages.ubuntu.com/xenial/python/
+# https://packages.ubuntu.com/xenial/zope/
+cssselect>=0.9.1
+lxml>=3.5.0
+service_identity>=16.0.0
+six>=1.10.0
+zope.interface>=4.1.3
diff --git a/setup.py b/setup.py
index ee0aaabf0..37892cfbf 100644
--- a/setup.py
+++ b/setup.py
@@ -53,7 +53,6 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
@@ -63,20 +62,21 @@ setup(
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
     install_requires=[
-        'Twisted>=13.1.0;python_version!="3.4"',
-        'Twisted>=13.1.0,<=19.2.0;python_version=="3.4"',
-        'w3lib>=1.17.0',
-        'queuelib',
-        'lxml;python_version!="3.4"',
-        'lxml<=4.3.5;python_version=="3.4"',
-        'pyOpenSSL',
-        'cssselect>=0.9',
-        'six>=1.5.2',
-        'parsel>=1.5',
+        'Twisted>=16.0.0;python_version=="2.7"',
+        'Twisted>=17.9.0;python_version>="3.5"',
+        'cryptography>=2.0',
+        'cssselect>=0.9.1',
+        'lxml>=3.5.0',
+        'parsel>=1.5.0',
         'PyDispatcher>=2.0.5',
-        'service_identity',
+        'pyOpenSSL>=16.2.0',
+        'queuelib>=1.4.2',
+        'service_identity>=16.0.0',
+        'six>=1.10.0',
+        'w3lib>=1.17.0',
+        'zope.interface>=4.1.3',
     ],
     extras_require=extras_require,
 )
diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txt
index be809b151..f621eb4eb 100644
--- a/tests/requirements-py2.txt
+++ b/tests/requirements-py2.txt
@@ -1,14 +1,15 @@
 # Tests requirements
-mock
+brotlipy
+jmespath
 mitmproxy==0.10.1
+mock
 netlib==0.10.1
 pytest
 pytest-cov
 pytest-twisted
 pytest-xdist
-jmespath
-brotlipy
 testfixtures
+
 # optional for shell wrapper tests
 bpython
 ipython<6.0
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index ed7bf0be0..cb67bc40e 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,13 +1,14 @@
+# Tests requirements
+jmespath
+leveldb; sys_platform != "win32"
 pytest
 pytest-cov
 pytest-twisted
 pytest-xdist
 testfixtures
-jmespath
-leveldb; sys_platform != "win32"
-botocore
+
 # optional for shell wrapper tests
 bpython
-ipython
 brotlipy
+ipython
 pywin32; sys_platform == "win32"
diff --git a/tox.ini b/tox.ini
index c918731f4..c3502c2ca 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,10 +11,10 @@ deps =
     -ctests/constraints.txt
     -rrequirements-py2.txt
     # Extras
-    botocore
+    botocore>=1.3.23
     google-cloud-storage
-    Pillow != 3.0.0
     leveldb
+    Pillow>=3.4.2
     -rtests/requirements-py2.txt
 passenv =
     S3_TEST_FILE_URI
@@ -25,72 +25,74 @@ passenv =
 commands =
     py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
 
-[testenv:trusty]
+[testenv:py27-pinned]
 basepython = python2.7
 deps =
-    pyOpenSSL==0.13
-    lxml==3.3.3
-    Twisted==13.2.0
-    boto==2.20.1
-    Pillow==2.3.0
+    -ctests/constraints.txt
+    cryptography==2.0
     cssselect==0.9.1
-    zope.interface==4.0.5
+    lxml==3.5.0
+    parsel==1.5.0
+    PyDispatcher==2.0.5
+    pyOpenSSL==16.2.0
+    queuelib==1.4.2
+    service_identity==16.0.0
+    six==1.10.0
+    Twisted==16.0.0
+    w3lib==1.17.0
+    zope.interface==4.1.3
     -rtests/requirements-py2.txt
-
-[testenv:jessie]
-# https://packages.debian.org/en/jessie/python/
-# https://packages.debian.org/en/jessie/zope/
-basepython = python2.7
-deps =
-    cryptography==0.6.1
-    pyOpenSSL==0.14
-    lxml==3.4.0
-    Twisted==14.0.2
-    boto==2.34.0
-    Pillow==2.6.1
-    cssselect==0.9.1
-    zope.interface==4.1.1
-    -rtests/requirements-py2.txt
-# Not used directly but allows boto GCE plugins to load.
-# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262
-    google-compute-engine==2.8.12
-
-[testenv:trunk]
-basepython = python2.7
-commands =
-    pip install -U https://github.com/scrapy/w3lib/archive/master.zip#egg=w3lib
-    pip install -U https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib
-    py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
+    # Extras
+    botocore==1.3.23
+    Pillow==3.4.2
 
 [testenv:pypy]
 basepython = pypy
 commands =
     py.test {posargs:scrapy tests}
 
-[testenv:py34]
-basepython = python3.4
+[testenv:py35]
+basepython = python3.5
 deps =
     -ctests/constraints.txt
     -rrequirements-py3.txt
-    # Extras
-    Pillow
     -rtests/requirements-py3.txt
+    # Extras
+    botocore>=1.3.23
+    Pillow>=3.4.2
 
-[testenv:py35]
+[testenv:py35-pinned]
 basepython = python3.5
-deps = {[testenv:py34]deps}
+deps =
+    -ctests/constraints.txt
+    cryptography==2.0
+    cssselect==0.9.1
+    lxml==3.5.0
+    parsel==1.5.0
+    PyDispatcher==2.0.5
+    pyOpenSSL==16.2.0
+    queuelib==1.4.2
+    service_identity==16.0.0
+    six==1.10.0
+    Twisted==17.9.0
+    w3lib==1.17.0
+    zope.interface==4.1.3
+    -rtests/requirements-py3.txt
+    # Extras
+    botocore==1.3.23
+    Pillow==3.4.2
 
 [testenv:py36]
 basepython = python3.6
-deps = {[testenv:py34]deps}
+deps = {[testenv:py35]deps}
 
 [testenv:py37]
 basepython = python3.7
-deps = {[testenv:py34]deps}
+deps = {[testenv:py35]deps}
 
 [testenv:pypy3]
 basepython = pypy3
-deps = {[testenv:py34]deps}
+deps = {[testenv:py35]deps}
 commands =
     py.test {posargs:scrapy tests}
 
@@ -119,14 +121,14 @@ commands =
 
 [testenv:py37-extra-deps]
 basepython = python3.7
-deps = 
-    {[testenv:py34]deps}
+deps =
+    {[testenv:py35]deps}
     reppy
     robotexclusionrulesparser
 
 [testenv:py27-extra-deps]
 basepython = python2.7
-deps = 
+deps =
     {[testenv]deps}
     reppy
     robotexclusionrulesparser

From 595c995ee640e360e5b574577594b273cc8af071 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 7 Aug 2019 15:38:04 -0300
Subject: [PATCH 2147/2974] Simplify version reporting

---
 scrapy/utils/versions.py | 26 +++++++-------------------
 1 file changed, 7 insertions(+), 19 deletions(-)

diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py
index 58c7aef85..3f8122154 100644
--- a/scrapy/utils/versions.py
+++ b/scrapy/utils/versions.py
@@ -1,8 +1,10 @@
 import platform
 import sys
 
+import cryptography
 import cssselect
 import lxml.etree
+import OpenSSL
 import parsel
 import twisted
 import w3lib
@@ -13,15 +15,6 @@ import scrapy
 def scrapy_components_versions():
     lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
     libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
-    try:
-        w3lib_version = w3lib.__version__
-    except AttributeError:
-        w3lib_version = "<1.14.3"
-    try:
-        import cryptography
-        cryptography_version = cryptography.__version__
-    except ImportError:
-        cryptography_version = "unknown"
 
     return [
         ("Scrapy", scrapy.__version__),
@@ -29,22 +22,17 @@ def scrapy_components_versions():
         ("libxml2", libxml2_version),
         ("cssselect", cssselect.__version__),
         ("parsel", parsel.__version__),
-        ("w3lib", w3lib_version),
+        ("w3lib", w3lib.__version__),
         ("Twisted", twisted.version.short()),
         ("Python", sys.version.replace("\n", "- ")),
         ("pyOpenSSL", _get_openssl_version()),
-        ("cryptography", cryptography_version),
+        ("cryptography", cryptography.__version__),
         ("Platform",  platform.platform()),
     ]
 
 
 def _get_openssl_version():
-    try:
-        import OpenSSL
-        openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\
-            .decode('ascii', errors='replace')
-    # pyOpenSSL 0.12 does not expose openssl version
-    except AttributeError:
-        openssl = 'Unknown OpenSSL version'
-
+    openssl = OpenSSL.SSL.SSLeay_version(
+        OpenSSL.SSL.SSLEAY_VERSION
+    ).decode('ascii', errors='replace')
     return '{} ({})'.format(OpenSSL.version.__version__, openssl)

From d76b6944c9081684e4961c15dc3426a4d4c67a3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Thu, 8 Aug 2019 09:43:42 +0200
Subject: [PATCH 2148/2974] Create Request from curl command  (#3862)

---
 docs/topics/developer-tools.rst  |  30 ++++-
 docs/topics/dynamic-content.rst  |   7 +
 docs/topics/request-response.rst |   2 +
 scrapy/http/request/__init__.py  |  32 +++++
 scrapy/utils/curl.py             |  95 ++++++++++++++
 tests/test_http_request.py       |  76 +++++++++++
 tests/test_utils_curl.py         | 211 +++++++++++++++++++++++++++++++
 7 files changed, 450 insertions(+), 3 deletions(-)
 create mode 100644 scrapy/utils/curl.py
 create mode 100644 tests/test_utils_curl.py

diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index 82857c9da..dcf8af365 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -252,9 +252,33 @@ If the handy ``has_next`` element is ``true`` (try loading
 `quotes.toscrape.com/api/quotes?page=10`_ in your browser or a
 page-number greater than 10), we increment the ``page`` attribute 
 and ``yield`` a new request, inserting the incremented page-number 
-into our ``url``. 
+into our ``url``.
 
-You can see that with a few inspections in the `Network`-tool we 
+.. _requests-from-curl:
+
+In more complex websites, it could be difficult to easily reproduce the
+requests, as we could need to add ``headers`` or ``cookies`` to make it work.
+In those cases you can export the requests in `cURL <https://curl.haxx.se/>`_
+format, by right-clicking on each of them in the network tool and using the
+:meth:`~scrapy.http.Request.from_curl()` method to generate an equivalent
+request::
+
+    from scrapy import Request
+
+    request = Request.from_curl(
+        "curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
+        "la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce"
+        "pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X"
+        "-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
+        "zEwZTAxLTk5MWUtNDFiNC1iZWRmLTJjNGI4M2ZiNDBmNDpAVEstMDMzMTBlMDEtOTkxZS00MW"
+        "I0LWJlZGYtMmM0YjgzZmI0MGY0' -H 'Connection: keep-alive' -H 'Referer: http"
+        "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
+
+Alternatively, if you want to know the arguments needed to recreate that
+request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
+function to get a dictionary with the equivalent arguments.
+
+As you can see, with a few inspections in the `Network`-tool we
 were able to easily replicate the dynamic requests of the scrolling 
 functionality of the page. Crawling dynamic pages can be quite
 daunting and pages can be very complex, but it (mostly) boils down
@@ -262,7 +286,7 @@ to identifying the correct request and replicating it in your spider.
 
 .. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools
 .. _quotes.toscrape.com: http://quotes.toscrape.com
-.. _quotes.toscrape.com/scroll: quotes.toscrape.com/scroll/
+.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll
 .. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10
 .. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions
 
diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index 8b5dacf56..8334ddcec 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -85,6 +85,13 @@ It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP
 method and URL. However, you may also need to reproduce the body, headers and
 form parameters (see :class:`~scrapy.http.FormRequest`) of that request.
 
+As all major browsers allow to export the requests in `cURL
+<https://curl.haxx.se/>`_ format, Scrapy incorporates the method
+:meth:`~scrapy.http.Request.from_curl()` to generate an equivalent
+:class:`~scrapy.http.Request` from a cURL command. To get more information
+visit :ref:`request from curl <requests-from-curl>` inside the network
+tool section.
+
 Once you get the expected response, you can :ref:`extract the desired data from
 it <topics-handling-response-formats>`.
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 4e81ce878..9a5c65b0d 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -194,6 +194,8 @@ Request objects
        copied by default (unless new values are given as arguments). See also
        :ref:`topics-request-response-ref-request-callback-arguments`.
 
+    .. automethod:: from_curl
+
 .. _topics-request-response-ref-request-callback-arguments:
 
 Passing additional data to callback functions
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index f5935c4ef..d09eaf849 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -12,6 +12,7 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import escape_ajax
 from scrapy.http.common import obsolete_setter
+from scrapy.utils.curl import curl_to_request_kwargs
 
 
 class Request(object_ref):
@@ -103,3 +104,34 @@ class Request(object_ref):
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)
+
+    @classmethod
+    def from_curl(cls, curl_command, ignore_unknown_options=True, **kwargs):
+        """Create a Request object from a string containing a `cURL
+        <https://curl.haxx.se/>`_ command. It populates the HTTP method, the
+        URL, the headers, the cookies and the body. It accepts the same
+        arguments as the :class:`Request` class, taking preference and
+        overriding the values of the same arguments contained in the cURL
+        command.
+
+        Unrecognized options are ignored by default. To raise an error when
+        finding unknown options call this method by passing
+        ``ignore_unknown_options=False``.
+
+        .. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request`
+                     subclasses, such as :class:`~scrapy.http.JSONRequest`, or
+                     :class:`~scrapy.http.XmlRpcRequest`, as well as having
+                     :ref:`downloader middlewares <topics-downloader-middleware>`
+                     and
+                     :ref:`spider middlewares <topics-spider-middleware>`
+                     enabled, such as
+                     :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`,
+                     :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`,
+                     or
+                     :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`,
+                     may modify the :class:`~scrapy.http.Request` object.
+
+       """
+        request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options)
+        request_kwargs.update(kwargs)
+        return cls(**request_kwargs)
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
new file mode 100644
index 000000000..b3fd0a497
--- /dev/null
+++ b/scrapy/utils/curl.py
@@ -0,0 +1,95 @@
+import argparse
+import warnings
+from shlex import split
+
+from six.moves.http_cookies import SimpleCookie
+from six.moves.urllib.parse import urlparse
+from six import string_types, iteritems
+from w3lib.http import basic_auth_header
+
+
+class CurlParser(argparse.ArgumentParser):
+    def error(self, message):
+        error_msg = \
+            'There was an error parsing the curl command: {}'.format(message)
+        raise ValueError(error_msg)
+
+
+curl_parser = CurlParser()
+curl_parser.add_argument('url')
+curl_parser.add_argument('-H', '--header', dest='headers', action='append')
+curl_parser.add_argument('-X', '--request', dest='method', default='get')
+curl_parser.add_argument('-d', '--data', dest='data')
+curl_parser.add_argument('-u', '--user', dest='auth')
+
+
+safe_to_ignore_arguments = [
+    ['--compressed'],
+    # `--compressed` argument is not safe to ignore, but it's included here
+    # because the `HttpCompressionMiddleware` is enabled by default
+    ['-s', '--silent'],
+    ['-v', '--verbose'],
+    ['-#', '--progress-bar']
+]
+
+for argument in safe_to_ignore_arguments:
+    curl_parser.add_argument(*argument, action='store_true')
+
+
+def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
+    """Convert a cURL command syntax to Request kwargs.
+
+    :param str curl_command: string containing the curl command
+    :param bool ignore_unknown_options: If true, only a warning is emitted when
+    cURL options are unknown. Otherwise raises an error. (default: True)
+    :return: dictionary of Request kwargs
+    """
+
+    curl_args = split(curl_command)
+
+    if curl_args[0] != 'curl':
+        raise ValueError('A curl command must start with "curl"')
+
+    parsed_args, argv = curl_parser.parse_known_args(curl_args[1:])
+
+    if argv:
+        msg = 'Unrecognized options: {}'.format(', '.join(argv))
+        if ignore_unknown_options:
+            warnings.warn(msg)
+        else:
+            raise ValueError(msg)
+
+    url = parsed_args.url
+
+    # curl automatically prepends 'http' if the scheme is missing, but Request
+    # needs the scheme to work
+    parsed_url = urlparse(url)
+    if not parsed_url.scheme:
+        url = 'http://' + url
+
+    result = {'method': parsed_args.method.upper(), 'url': url}
+
+    headers = []
+    cookies = {}
+    for header in parsed_args.headers or ():
+        name, val = header.split(':', 1)
+        name = name.strip()
+        val = val.strip()
+        if name.title() == 'Cookie':
+            for name, morsel in iteritems(SimpleCookie(val)):
+                cookies[name] = morsel.value
+        else:
+            headers.append((name, val))
+
+    if parsed_args.auth:
+        user, password = parsed_args.auth.split(':', 1)
+        headers.append(('Authorization', basic_auth_header(user, password)))
+
+    if headers:
+        result['headers'] = headers
+    if cookies:
+        result['cookies'] = cookies
+    if parsed_args.data:
+        result['body'] = parsed_args.data
+
+    return result
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 952e208de..60494d792 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -269,6 +269,82 @@ class RequestTest(unittest.TestCase):
         with self.assertRaises(TypeError):
             self.request_class('http://example.com', a_function, errback='a_function')
 
+    def test_from_curl(self):
+        # Note: more curated tests regarding curl conversion are in
+        # `test_utils_curl.py`
+        curl_command = (
+            "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique"
+            "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique"
+            "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H"
+            " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q"
+            "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use"
+            "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM"
+            "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S"
+            "afari/537.36' -H 'Content-Type: application /x-www-form-urlencode"
+            "d' -H 'Accept: text/html,application/xhtml+xml,application/xml;q="
+            "0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0"
+            "' -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: kee"
+            "p-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith"
+            "%40example.org&size=small&topping=cheese&topping=onion&delivery=1"
+            "2%3A15&comments=' --compressed"
+        )
+        r = self.request_class.from_curl(curl_command)
+        self.assertEqual(r.method, "POST")
+        self.assertEqual(r.url, "http://httpbin.org/post")
+        self.assertEqual(r.body,
+                         b"custname=John+Smith&custtel=500&custemail=jsmith%40"
+                         b"example.org&size=small&topping=cheese&topping=onion"
+                         b"&delivery=12%3A15&comments=")
+        self.assertEqual(r.cookies, {
+            '_gauges_unique_year': '1',
+            '_gauges_unique': '1',
+            '_gauges_unique_month': '1',
+            '_gauges_unique_hour': '1',
+            '_gauges_unique_day': '1'
+        })
+        self.assertEqual(r.headers, {
+            b'Origin': [b'http://httpbin.org'],
+            b'Accept-Encoding': [b'gzip, deflate'],
+            b'Accept-Language': [b'en-US,en;q=0.9,ru;q=0.8,es;q=0.7'],
+            b'Upgrade-Insecure-Requests': [b'1'],
+            b'User-Agent': [b'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.'
+                            b'36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202'
+                            b'.75 Chrome/62.0.3202.75 Safari/537.36'],
+            b'Content-Type': [b'application /x-www-form-urlencoded'],
+            b'Accept': [b'text/html,application/xhtml+xml,application/xml;q=0.'
+                        b'9,image/webp,image/apng,*/*;q=0.8'],
+            b'Cache-Control': [b'max-age=0'],
+            b'Referer': [b'http://httpbin.org/forms/post'],
+            b'Connection': [b'keep-alive']})
+
+    def test_from_curl_with_kwargs(self):
+        r = self.request_class.from_curl(
+            'curl -X PATCH "http://example.org"',
+            method="POST",
+            meta={'key': 'value'}
+        )
+        self.assertEqual(r.method, "POST")
+        self.assertEqual(r.meta, {"key": "value"})
+
+    def test_from_curl_ignore_unknown_options(self):
+        # By default: it works and ignores the unknown options: --foo and -z
+        with warnings.catch_warnings():  # avoid warning when executing tests
+            warnings.simplefilter('ignore')
+            r = self.request_class.from_curl(
+                'curl -X DELETE "http://example.org" --foo -z',
+            )
+            self.assertEqual(r.method, "DELETE")
+
+        # If `ignore_unknon_options` is set to `False` it raises an error with
+        # the unknown options: --foo and -z
+        self.assertRaises(
+            ValueError,
+            lambda: self.request_class.from_curl(
+                'curl -X PATCH "http://example.org" --foo -z',
+                ignore_unknown_options=False,
+            ),
+        )
+
 
 class FormRequestTest(RequestTest):
 
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
new file mode 100644
index 000000000..c5655df7e
--- /dev/null
+++ b/tests/test_utils_curl.py
@@ -0,0 +1,211 @@
+import unittest
+import warnings
+
+from six import assertRaisesRegex
+from w3lib.http import basic_auth_header
+
+from scrapy import Request
+from scrapy.utils.curl import curl_to_request_kwargs
+
+
+class CurlToRequestKwargsTest(unittest.TestCase):
+    maxDiff = 5000
+
+    def _test_command(self, curl_command, expected_result):
+        result = curl_to_request_kwargs(curl_command)
+        self.assertEqual(result, expected_result)
+        try:
+            Request(**result)
+        except TypeError as e:
+            self.fail("Request kwargs are not correct {}".format(e))
+
+    def test_get(self):
+        curl_command = "curl http://example.org/"
+        expected_result = {"method": "GET", "url": "http://example.org/"}
+        self._test_command(curl_command, expected_result)
+
+    def test_get_without_scheme(self):
+        curl_command = "curl www.example.org"
+        expected_result = {"method": "GET", "url": "http://www.example.org"}
+        self._test_command(curl_command, expected_result)
+
+    def test_get_basic_auth(self):
+        curl_command = 'curl "https://api.test.com/" -u ' \
+                       '"some_username:some_password"'
+        expected_result = {
+            "method": "GET",
+            "url": "https://api.test.com/",
+            "headers": [
+                (
+                    "Authorization",
+                    basic_auth_header("some_username", "some_password")
+                )
+            ],
+        }
+        self._test_command(curl_command, expected_result)
+
+    def test_get_complex(self):
+        curl_command = (
+            "curl 'http://httpbin.org/get' -H 'Accept-Encoding: gzip, deflate'"
+            " -H 'Accept-Language: en-US,en;q=0.9,ru;q=0.8,es;q=0.7' -H 'Upgra"
+            "de-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (X11; Linux "
+            "x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/62"
+            ".0.3202.75 Chrome/62.0.3202.75 Safari/537.36' -H 'Accept: text/ht"
+            "ml,application/xhtml+xml,application/xml;q=0.9,image/webp,image/a"
+            "png,*/*;q=0.8' -H 'Referer: http://httpbin.org/' -H 'Cookie: _gau"
+            "ges_unique_year=1; _gauges_unique=1; _gauges_unique_month=1; _gau"
+            "ges_unique_hour=1; _gauges_unique_day=1' -H 'Connection: keep-ali"
+            "ve' --compressed"
+        )
+        expected_result = {
+            "method": "GET",
+            "url": "http://httpbin.org/get",
+            "headers": [
+                ("Accept-Encoding", "gzip, deflate"),
+                ("Accept-Language", "en-US,en;q=0.9,ru;q=0.8,es;q=0.7"),
+                ("Upgrade-Insecure-Requests", "1"),
+                (
+                    "User-Agent",
+                    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"
+                    ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"
+                    "02.75 Safari/537.36",
+                ),
+                (
+                    "Accept",
+                    "text/html,application/xhtml+xml,application/xml;q=0.9,ima"
+                    "ge/webp,image/apng,*/*;q=0.8",
+                ),
+                ("Referer", "http://httpbin.org/"),
+                ("Connection", "keep-alive"),
+            ],
+            "cookies": {
+                '_gauges_unique_year': '1',
+                '_gauges_unique_hour': '1',
+                '_gauges_unique_day': '1',
+                '_gauges_unique': '1',
+                '_gauges_unique_month': '1'
+            },
+        }
+        self._test_command(curl_command, expected_result)
+
+    def test_post(self):
+        curl_command = (
+            "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique"
+            "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique"
+            "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H"
+            " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q"
+            "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use"
+            "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM"
+            "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S"
+            "afari/537.36' -H 'Content-Type: application/x-www-form-urlencoded"
+            "' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0"
+            ".9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0'"
+            " -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: keep"
+            "-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith%"
+            "40example.org&size=small&topping=cheese&topping=onion&delivery=12"
+            "%3A15&comments=' --compressed"
+        )
+        expected_result = {
+            "method": "POST",
+            "url": "http://httpbin.org/post",
+            "body": "custname=John+Smith&custtel=500&custemail=jsmith%40exampl"
+                    "e.org&size=small&topping=cheese&topping=onion&delivery=12"
+                    "%3A15&comments=",
+            "cookies": {
+                '_gauges_unique_year': '1',
+                '_gauges_unique_hour': '1',
+                '_gauges_unique_day': '1',
+                '_gauges_unique': '1',
+                '_gauges_unique_month': '1'
+            },
+            "headers": [
+                ("Origin", "http://httpbin.org"),
+                ("Accept-Encoding", "gzip, deflate"),
+                ("Accept-Language", "en-US,en;q=0.9,ru;q=0.8,es;q=0.7"),
+                ("Upgrade-Insecure-Requests", "1"),
+                (
+                    "User-Agent",
+                    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"
+                    ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"
+                    "02.75 Safari/537.36",
+                ),
+                ("Content-Type", "application/x-www-form-urlencoded"),
+                (
+                    "Accept",
+                    "text/html,application/xhtml+xml,application/xml;q=0.9,ima"
+                    "ge/webp,image/apng,*/*;q=0.8",
+                ),
+                ("Cache-Control", "max-age=0"),
+                ("Referer", "http://httpbin.org/forms/post"),
+                ("Connection", "keep-alive"),
+            ],
+        }
+        self._test_command(curl_command, expected_result)
+
+    def test_patch(self):
+        curl_command = (
+            'curl "https://example.com/api/fake" -u "username:password" -H "Ac'
+            'cept: application/vnd.go.cd.v4+json" -H "Content-Type: applicatio'
+            'n/json" -X PATCH -d \'{"hostname": "agent02.example.com",  "agent'
+            '_config_state": "Enabled", "resources": ["Java","Linux"], "enviro'
+            'nments": ["Dev"]}\''
+        )
+        expected_result = {
+            "method": "PATCH",
+            "url": "https://example.com/api/fake",
+            "headers": [
+                ("Accept", "application/vnd.go.cd.v4+json"),
+                ("Content-Type", "application/json"),
+                ("Authorization", basic_auth_header("username", "password")),
+            ],
+            "body": '{"hostname": "agent02.example.com",  "agent_config_state"'
+                    ': "Enabled", "resources": ["Java","Linux"], "environments'
+                    '": ["Dev"]}',
+        }
+        self._test_command(curl_command, expected_result)
+
+    def test_delete(self):
+        curl_command = 'curl -X "DELETE" https://www.url.com/page'
+        expected_result = {
+            "method": "DELETE", "url": "https://www.url.com/page"
+        }
+        self._test_command(curl_command, expected_result)
+
+    def test_get_silent(self):
+        curl_command = 'curl --silent "www.example.com"'
+        expected_result = {"method": "GET", "url": "http://www.example.com"}
+        self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
+
+    def test_too_few_arguments_error(self):
+        assertRaisesRegex(
+            self,
+            ValueError,
+            r"too few arguments|the following arguments are required:\s*url",
+            lambda: curl_to_request_kwargs("curl"),
+        )
+
+    def test_ignore_unknown_options(self):
+        # case 1: ignore_unknown_options=True:
+        with warnings.catch_warnings():  # avoid warning when executing tests
+            warnings.simplefilter('ignore')
+            curl_command = 'curl --bar --baz http://www.example.com'
+            expected_result = \
+                {"method": "GET", "url": "http://www.example.com"}
+            self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
+
+        # case 2: ignore_unknown_options=False (raise exception):
+        assertRaisesRegex(
+            self,
+            ValueError,
+            "Unrecognized options:.*--bar.*--baz",
+            lambda: curl_to_request_kwargs(
+                "curl --bar --baz http://www.example.com",
+                ignore_unknown_options=False
+            ),
+        )
+
+    def test_must_start_with_curl_error(self):
+        self.assertRaises(
+            ValueError,
+            lambda: curl_to_request_kwargs("carl -X POST http://example.org")
+        )

From 9119798a5ce10aaf015d1af647c5b85e312d2386 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 5 Aug 2019 15:49:07 +0200
Subject: [PATCH 2149/2974] Add test coverage for contract failures involving
 multiple missing fields

---
 tests/test_contracts.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index a06bb2cc3..a728099c0 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -123,6 +123,14 @@ class TestSpider(Spider):
         """
         return {'url': response.url}
 
+    def scrapes_multiple_missing_fields(self, response):
+        """ returns item with no name
+        @url http://scrapy.org
+        @returns items 1 1
+        @scrapes name url
+        """
+        return {}
+
     def parse_no_url(self, response):
         """ method with no url
         @returns items 1 1
@@ -256,6 +264,13 @@ class ContractsManagerTest(unittest.TestCase):
         request.callback(response)
         self.should_fail()
 
+        # scrapes_multiple_missing_fields
+        request = self.conman.from_method(spider.scrapes_multiple_missing_fields, self.results)
+        request.callback(response)
+        self.should_fail()
+        message = 'ContractFail: Missing fields: name, url'
+        assert message in self.results.failures[-1][-1]
+
     def test_custom_contracts(self):
         self.conman.from_spider(CustomContractSuccessSpider(), self.results)
         self.should_succeed()

From 3040f77468d7dd997afd2f8a35590cafaff454af Mon Sep 17 00:00:00 2001
From: Shivam Sandbhor <shivam.sandbhor@gmail.com>
Date: Thu, 8 Aug 2019 17:28:22 +0530
Subject: [PATCH 2150/2974] [MRG+1] Update project.py removed one 'hack', seems
 irrelevant. (#3910)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Update project.py removed one 'hack', seems irrelevant.

As mentioned by @Gallaecio in issue #3871, the 'hack'  is cleared. I also  double checked whether the environment variable "SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE" was ever set in our codebase and it turns out we didn't set it or used it anywhere else.So I guess the 'hack' was not used in the current version. Also the name of this environment variable rather doesn't suggest it was  a boolean(it is used in  an 'if' condition which has perplexed me )

* Update project.py

* Update project.py

How about this?

* Update project.py

* Update project.py

* Update scrapy/utils/project.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/project.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update project.py
---
 scrapy/utils/project.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index 95c6a8035..1cbda141a 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -8,6 +8,7 @@ from os.path import join, dirname, abspath, isabs, exists
 from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
 from scrapy.settings import Settings
 from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import ScrapyDeprecationWarning
 
 ENVVAR = 'SCRAPY_SETTINGS_MODULE'
 DATADIR_CFG_SECTION = 'datadir'
@@ -70,6 +71,9 @@ def get_project_settings():
     # XXX: remove this hack
     pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
     if pickled_settings:
+        warnings.warn("Use of environment variable "
+                      "'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE' "
+                      "is deprecated.", ScrapyDeprecationWarning)
         settings.setdict(pickle.loads(pickled_settings), priority='project')
 
     # XXX: deprecate and remove this functionality

From da385b56b10aaf3393cc10d13421d45c5a0edbdb Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 8 Aug 2019 12:44:23 -0300
Subject: [PATCH 2151/2974] Move get_openssl_version function to
 scrapy.utils.ssl

---
 scrapy/utils/ssl.py      |  8 ++++++++
 scrapy/utils/versions.py | 11 ++---------
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index e54232abd..632827471 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+import OpenSSL
 import OpenSSL._util as pyOpenSSLutil
 
 from scrapy.utils.python import to_native_str
@@ -48,3 +49,10 @@ def get_temp_key_info(ssl_object):
         key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))
     key_info.append('%s bits' % pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key))
     return ', '.join(key_info)
+
+
+def get_openssl_version():
+    system_openssl = OpenSSL.SSL.SSLeay_version(
+        OpenSSL.SSL.SSLEAY_VERSION
+    ).decode('ascii', errors='replace')
+    return '{} ({})'.format(OpenSSL.version.__version__, system_openssl)
diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py
index 3f8122154..48484b303 100644
--- a/scrapy/utils/versions.py
+++ b/scrapy/utils/versions.py
@@ -4,12 +4,12 @@ import sys
 import cryptography
 import cssselect
 import lxml.etree
-import OpenSSL
 import parsel
 import twisted
 import w3lib
 
 import scrapy
+from scrapy.utils.ssl import get_openssl_version
 
 
 def scrapy_components_versions():
@@ -25,14 +25,7 @@ def scrapy_components_versions():
         ("w3lib", w3lib.__version__),
         ("Twisted", twisted.version.short()),
         ("Python", sys.version.replace("\n", "- ")),
-        ("pyOpenSSL", _get_openssl_version()),
+        ("pyOpenSSL", get_openssl_version()),
         ("cryptography", cryptography.__version__),
         ("Platform",  platform.platform()),
     ]
-
-
-def _get_openssl_version():
-    openssl = OpenSSL.SSL.SSLeay_version(
-        OpenSSL.SSL.SSLEAY_VERSION
-    ).decode('ascii', errors='replace')
-    return '{} ({})'.format(OpenSSL.version.__version__, openssl)

From fa9a9033f0a73e5c3c1b3bd54408e3bab7ea1b91 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 8 Aug 2019 22:54:51 -0300
Subject: [PATCH 2152/2974] Remove check for Twisted>=14.0.0

16.0.0 is currently the minimum supported version
---
 scrapy/core/downloader/tls.py | 158 +++++++++++++++++-----------------
 1 file changed, 78 insertions(+), 80 deletions(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 74afb3f10..2e218dfb4 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -1,5 +1,8 @@
 import logging
+
 from OpenSSL import SSL
+from twisted.internet.ssl import AcceptableCiphers
+from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError
 
 from scrapy import twisted_version
 from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
@@ -22,95 +25,90 @@ openssl_methods = {
 }
 
 
-if twisted_version >= (14, 0, 0):
-    # ClientTLSOptions requires a recent-enough version of Twisted.
-    # Not having ScrapyClientTLSOptions should not matter for older
-    # Twisted versions because it is not used in the fallback
-    # ScrapyClientContextFactory.
+# ClientTLSOptions requires a recent-enough version of Twisted (14.0.0+)
+# Not having ScrapyClientTLSOptions should not matter for older
+# Twisted versions because it is not used in the fallback
+# ScrapyClientContextFactory.
 
-    # taken from twisted/twisted/internet/_sslverify.py
+# taken from twisted/twisted/internet/_sslverify.py
 
-    try:
-        # XXX: this try-except is not needed in Twisted 17.0.0+ because
-        # it requires pyOpenSSL 0.16+.
-        from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START
-    except ImportError:
-        SSL_CB_HANDSHAKE_START = 0x10
-        SSL_CB_HANDSHAKE_DONE = 0x20
+try:
+    # XXX: this try-except is not needed in Twisted 17.0.0+ because
+    # it requires pyOpenSSL 0.16+.
+    from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START
+except ImportError:
+    SSL_CB_HANDSHAKE_START = 0x10
+    SSL_CB_HANDSHAKE_DONE = 0x20
 
-    from twisted.internet.ssl import AcceptableCiphers
-    from twisted.internet._sslverify import (ClientTLSOptions,
-                                             verifyHostname,
-                                             VerificationError)
-    try:
-        # XXX: this import would fail on Debian jessie with system installed
-        # service_identity library, due to lack of cryptography.x509 dependency
-        # See https://github.com/pyca/service_identity/issues/21
-        from service_identity.exceptions import CertificateError
-        verification_errors = (CertificateError, VerificationError)
-    except ImportError:
-        verification_errors = VerificationError
+try:
+    # XXX: this import would fail on Debian jessie with system installed
+    # service_identity library, due to lack of cryptography.x509 dependency
+    # See https://github.com/pyca/service_identity/issues/21
+    from service_identity.exceptions import CertificateError
+    verification_errors = (CertificateError, VerificationError)
+except ImportError:
+    verification_errors = VerificationError
 
-    if twisted_version < (17, 0, 0):
-        from twisted.internet._sslverify import _maybeSetHostNameIndication
-        set_tlsext_host_name = _maybeSetHostNameIndication
-    else:
-        def set_tlsext_host_name(connection, hostNameBytes):
-            connection.set_tlsext_host_name(hostNameBytes)
+if twisted_version < (17, 0, 0):
+    from twisted.internet._sslverify import _maybeSetHostNameIndication
+    set_tlsext_host_name = _maybeSetHostNameIndication
+else:
+    def set_tlsext_host_name(connection, hostNameBytes):
+        connection.set_tlsext_host_name(hostNameBytes)
 
 
-    class ScrapyClientTLSOptions(ClientTLSOptions):
-        """
-        SSL Client connection creator ignoring certificate verification errors
-        (for genuinely invalid certificates or bugs in verification code).
+class ScrapyClientTLSOptions(ClientTLSOptions):
+    """
+    SSL Client connection creator ignoring certificate verification errors
+    (for genuinely invalid certificates or bugs in verification code).
 
-        Same as Twisted's private _sslverify.ClientTLSOptions,
-        except that VerificationError, CertificateError and ValueError
-        exceptions are caught, so that the connection is not closed, only
-        logging warnings. Also, HTTPS connection parameters logging is added.
-        """
+    Same as Twisted's private _sslverify.ClientTLSOptions,
+    except that VerificationError, CertificateError and ValueError
+    exceptions are caught, so that the connection is not closed, only
+    logging warnings. Also, HTTPS connection parameters logging is added.
+    """
 
-        def __init__(self, hostname, ctx, verbose_logging=False):
-            super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
-            self.verbose_logging = verbose_logging
+    def __init__(self, hostname, ctx, verbose_logging=False):
+        super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
+        self.verbose_logging = verbose_logging
 
-        def _identityVerifyingInfoCallback(self, connection, where, ret):
-            if where & SSL_CB_HANDSHAKE_START:
-                set_tlsext_host_name(connection, self._hostnameBytes)
-            elif where & SSL_CB_HANDSHAKE_DONE:
-                if self.verbose_logging:
-                    if hasattr(connection, 'get_cipher_name'):  # requires pyOPenSSL 0.15
-                        if hasattr(connection, 'get_protocol_version_name'):  # requires pyOPenSSL 16.0.0
-                            logger.debug('SSL connection to %s using protocol %s, cipher %s',
-                                         self._hostnameASCII,
-                                         connection.get_protocol_version_name(),
-                                         connection.get_cipher_name(),
-                                         )
-                        else:
-                            logger.debug('SSL connection to %s using cipher %s',
-                                         self._hostnameASCII,
-                                         connection.get_cipher_name(),
-                                         )
-                    server_cert = connection.get_peer_certificate()
-                    logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
-                                 x509name_to_string(server_cert.get_issuer()),
-                                 x509name_to_string(server_cert.get_subject()),
-                                 )
-                    key_info = get_temp_key_info(connection._ssl)
-                    if key_info:
-                        logger.debug('SSL temp key: %s', key_info)
+    def _identityVerifyingInfoCallback(self, connection, where, ret):
+        if where & SSL_CB_HANDSHAKE_START:
+            set_tlsext_host_name(connection, self._hostnameBytes)
+        elif where & SSL_CB_HANDSHAKE_DONE:
+            if self.verbose_logging:
+                if hasattr(connection, 'get_cipher_name'):  # requires pyOPenSSL 0.15
+                    if hasattr(connection, 'get_protocol_version_name'):  # requires pyOPenSSL 16.0.0
+                        logger.debug('SSL connection to %s using protocol %s, cipher %s',
+                                     self._hostnameASCII,
+                                     connection.get_protocol_version_name(),
+                                     connection.get_cipher_name(),
+                                     )
+                    else:
+                        logger.debug('SSL connection to %s using cipher %s',
+                                     self._hostnameASCII,
+                                     connection.get_cipher_name(),
+                                     )
+                server_cert = connection.get_peer_certificate()
+                logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
+                             x509name_to_string(server_cert.get_issuer()),
+                             x509name_to_string(server_cert.get_subject()),
+                             )
+                key_info = get_temp_key_info(connection._ssl)
+                if key_info:
+                    logger.debug('SSL temp key: %s', key_info)
 
-                try:
-                    verifyHostname(connection, self._hostnameASCII)
-                except verification_errors as e:
-                    logger.warning(
-                        'Remote certificate is not valid for hostname "{}"; {}'.format(
-                            self._hostnameASCII, e))
+            try:
+                verifyHostname(connection, self._hostnameASCII)
+            except verification_errors as e:
+                logger.warning(
+                    'Remote certificate is not valid for hostname "{}"; {}'.format(
+                        self._hostnameASCII, e))
 
-                except ValueError as e:
-                    logger.warning(
-                        'Ignoring error while verifying certificate '
-                        'from host "{}" (exception: {})'.format(
-                            self._hostnameASCII, repr(e)))
+            except ValueError as e:
+                logger.warning(
+                    'Ignoring error while verifying certificate '
+                    'from host "{}" (exception: {})'.format(
+                        self._hostnameASCII, repr(e)))
 
-    DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
+DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')

From a940a80f5886c631425c4790ac92928926b1cc92 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 8 Aug 2019 23:05:52 -0300
Subject: [PATCH 2153/2974] Remove check for pyOpenSSL>=0.16

16.2.0 is currently the minimum supported version
---
 scrapy/core/downloader/tls.py | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 2e218dfb4..d6b7967da 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -30,16 +30,6 @@ openssl_methods = {
 # Twisted versions because it is not used in the fallback
 # ScrapyClientContextFactory.
 
-# taken from twisted/twisted/internet/_sslverify.py
-
-try:
-    # XXX: this try-except is not needed in Twisted 17.0.0+ because
-    # it requires pyOpenSSL 0.16+.
-    from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START
-except ImportError:
-    SSL_CB_HANDSHAKE_START = 0x10
-    SSL_CB_HANDSHAKE_DONE = 0x20
-
 try:
     # XXX: this import would fail on Debian jessie with system installed
     # service_identity library, due to lack of cryptography.x509 dependency
@@ -73,9 +63,9 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
         self.verbose_logging = verbose_logging
 
     def _identityVerifyingInfoCallback(self, connection, where, ret):
-        if where & SSL_CB_HANDSHAKE_START:
+        if where & SSL.SSL_CB_HANDSHAKE_START:
             set_tlsext_host_name(connection, self._hostnameBytes)
-        elif where & SSL_CB_HANDSHAKE_DONE:
+        elif where & SSL.SSL_CB_HANDSHAKE_DONE:
             if self.verbose_logging:
                 if hasattr(connection, 'get_cipher_name'):  # requires pyOPenSSL 0.15
                     if hasattr(connection, 'get_protocol_version_name'):  # requires pyOPenSSL 16.0.0

From 3164543ed1659a11cea1f6a04d8545f9cfebe367 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 8 Aug 2019 23:15:03 -0300
Subject: [PATCH 2154/2974] Remove fallback ScrapyClientContextFactory class
 (used in Twisted < 14.0.0)

16.0.0 is currently the minimum supported version
---
 scrapy/core/downloader/contextfactory.py | 161 ++++++++++-------------
 scrapy/core/downloader/tls.py            |   7 +-
 2 files changed, 69 insertions(+), 99 deletions(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 5ac20c0bb..127a246f5 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,112 +1,85 @@
 from OpenSSL import SSL
-from twisted.internet.ssl import ClientContextFactory
+from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust
+from twisted.web.client import BrowserLikePolicyForHTTPS
+from twisted.web.iweb import IPolicyForHTTPS
+from zope.interface.declarations import implementer
 
-from scrapy import twisted_version
-
-if twisted_version >= (14, 0, 0):
-
-    from zope.interface.declarations import implementer
-
-    from twisted.internet.ssl import (optionsForClientTLS,
-                                      CertificateOptions,
-                                      platformTrust)
-    from twisted.web.client import BrowserLikePolicyForHTTPS
-    from twisted.web.iweb import IPolicyForHTTPS
-
-    from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
+from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
 
 
-    @implementer(IPolicyForHTTPS)
-    class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
-        """
-        Non-peer-certificate verifying HTTPS context factory
+@implementer(IPolicyForHTTPS)
+class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
+    """
+    Non-peer-certificate verifying HTTPS context factory
 
-        Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
-        which allows TLS protocol negotiation
+    Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
+    which allows TLS protocol negotiation
 
-        'A TLS/SSL connection established with [this method] may
-         understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
-        """
+    'A TLS/SSL connection established with [this method] may
+     understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
+    """
 
-        def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs):
-            super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
-            self._ssl_method = method
-            self.tls_verbose_logging = tls_verbose_logging
+    def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs):
+        super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
+        self._ssl_method = method
+        self.tls_verbose_logging = tls_verbose_logging
 
-        @classmethod
-        def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
-            tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
-            return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs)
+    @classmethod
+    def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
+        tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
+        return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs)
 
-        def getCertificateOptions(self):
-            # setting verify=True will require you to provide CAs
-            # to verify against; in other words: it's not that simple
+    def getCertificateOptions(self):
+        # setting verify=True will require you to provide CAs
+        # to verify against; in other words: it's not that simple
 
-            # backward-compatible SSL/TLS method:
-            #
-            # * this will respect `method` attribute in often recommended
-            #   `ScrapyClientContextFactory` subclass
-            #   (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
-            #
-            # * getattr() for `_ssl_method` attribute for context factories
-            #   not calling super(..., self).__init__
-            return CertificateOptions(verify=False,
-                        method=getattr(self, 'method',
-                                       getattr(self, '_ssl_method', None)),
-                        fixBrokenPeers=True,
-                        acceptableCiphers=DEFAULT_CIPHERS)
+        # backward-compatible SSL/TLS method:
+        #
+        # * this will respect `method` attribute in often recommended
+        #   `ScrapyClientContextFactory` subclass
+        #   (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
+        #
+        # * getattr() for `_ssl_method` attribute for context factories
+        #   not calling super(..., self).__init__
+        return CertificateOptions(verify=False,
+                    method=getattr(self, 'method',
+                                   getattr(self, '_ssl_method', None)),
+                    fixBrokenPeers=True,
+                    acceptableCiphers=DEFAULT_CIPHERS)
 
-        # kept for old-style HTTP/1.0 downloader context twisted calls,
-        # e.g. connectSSL()
-        def getContext(self, hostname=None, port=None):
-            return self.getCertificateOptions().getContext()
+    # kept for old-style HTTP/1.0 downloader context twisted calls,
+    # e.g. connectSSL()
+    def getContext(self, hostname=None, port=None):
+        return self.getCertificateOptions().getContext()
 
-        def creatorForNetloc(self, hostname, port):
-            return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),
-                                          verbose_logging=self.tls_verbose_logging)
+    def creatorForNetloc(self, hostname, port):
+        return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),
+                                      verbose_logging=self.tls_verbose_logging)
 
 
-    @implementer(IPolicyForHTTPS)
-    class BrowserLikeContextFactory(ScrapyClientContextFactory):
-        """
-        Twisted-recommended context factory for web clients.
+@implementer(IPolicyForHTTPS)
+class BrowserLikeContextFactory(ScrapyClientContextFactory):
+    """
+    Twisted-recommended context factory for web clients.
 
-        Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
-        "The default is to use a BrowserLikePolicyForHTTPS,
-        so unless you have special requirements you can leave this as-is."
+    Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
+    "The default is to use a BrowserLikePolicyForHTTPS,
+    so unless you have special requirements you can leave this as-is."
 
-        creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
-        except this context factory allows setting the TLS/SSL method to use.
+    creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
+    except this context factory allows setting the TLS/SSL method to use.
 
-        Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
-        which allows TLS protocol negotiation.
-        """
-        def creatorForNetloc(self, hostname, port):
+    Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
+    which allows TLS protocol negotiation.
+    """
+    def creatorForNetloc(self, hostname, port):
 
-            # trustRoot set to platformTrust() will use the platform's root CAs.
-            #
-            # This means that a website like https://www.cacert.org will be rejected
-            # by default, since CAcert.org CA certificate is seldom shipped.
-            return optionsForClientTLS(hostname.decode("ascii"),
-                                       trustRoot=platformTrust(),
-                                       extraCertificateOptions={
-                                            'method': self._ssl_method,
-                                       })
-
-else:
-
-    class ScrapyClientContextFactory(ClientContextFactory):
-        "A SSL context factory which is more permissive against SSL bugs."
-        # see https://github.com/scrapy/scrapy/issues/82
-        # and https://github.com/scrapy/scrapy/issues/26
-        # and https://github.com/scrapy/scrapy/issues/981
-
-        def __init__(self, method=SSL.SSLv23_METHOD):
-            self.method = method
-
-        def getContext(self, hostname=None, port=None):
-            ctx = ClientContextFactory.getContext(self)
-            # Enable all workarounds to SSL bugs as documented by
-            # https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_options.html
-            ctx.set_options(SSL.OP_ALL)
-            return ctx
+        # trustRoot set to platformTrust() will use the platform's root CAs.
+        #
+        # This means that a website like https://www.cacert.org will be rejected
+        # by default, since CAcert.org CA certificate is seldom shipped.
+        return optionsForClientTLS(hostname.decode("ascii"),
+                                   trustRoot=platformTrust(),
+                                   extraCertificateOptions={
+                                        'method': self._ssl_method,
+                                   })
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index d6b7967da..995f8cbba 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -10,12 +10,14 @@ from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
 
 logger = logging.getLogger(__name__)
 
+
 METHOD_SSLv3 = 'SSLv3'
 METHOD_TLS = 'TLS'
 METHOD_TLSv10 = 'TLSv1.0'
 METHOD_TLSv11 = 'TLSv1.1'
 METHOD_TLSv12 = 'TLSv1.2'
 
+
 openssl_methods = {
     METHOD_TLS:    SSL.SSLv23_METHOD,                   # protocol negotiation (recommended)
     METHOD_SSLv3:  SSL.SSLv3_METHOD,                    # SSL 3 (NOT recommended)
@@ -25,11 +27,6 @@ openssl_methods = {
 }
 
 
-# ClientTLSOptions requires a recent-enough version of Twisted (14.0.0+)
-# Not having ScrapyClientTLSOptions should not matter for older
-# Twisted versions because it is not used in the fallback
-# ScrapyClientContextFactory.
-
 try:
     # XXX: this import would fail on Debian jessie with system installed
     # service_identity library, due to lack of cryptography.x509 dependency

From b404941e0de86ddf2e108c42a4847f0341d53d14 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 8 Aug 2019 23:18:47 -0300
Subject: [PATCH 2155/2974] Remove import check for service_identity

service_identity.exceptions.CertificateError is available in the current minimum version (16.0.0)
---
 scrapy/core/downloader/tls.py | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 995f8cbba..1c3d94b29 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -1,8 +1,9 @@
 import logging
 
 from OpenSSL import SSL
-from twisted.internet.ssl import AcceptableCiphers
+from service_identity.exceptions import CertificateError
 from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError
+from twisted.internet.ssl import AcceptableCiphers
 
 from scrapy import twisted_version
 from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
@@ -27,15 +28,6 @@ openssl_methods = {
 }
 
 
-try:
-    # XXX: this import would fail on Debian jessie with system installed
-    # service_identity library, due to lack of cryptography.x509 dependency
-    # See https://github.com/pyca/service_identity/issues/21
-    from service_identity.exceptions import CertificateError
-    verification_errors = (CertificateError, VerificationError)
-except ImportError:
-    verification_errors = VerificationError
-
 if twisted_version < (17, 0, 0):
     from twisted.internet._sslverify import _maybeSetHostNameIndication
     set_tlsext_host_name = _maybeSetHostNameIndication
@@ -87,7 +79,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
 
             try:
                 verifyHostname(connection, self._hostnameASCII)
-            except verification_errors as e:
+            except (CertificateError, VerificationError) as e:
                 logger.warning(
                     'Remote certificate is not valid for hostname "{}"; {}'.format(
                         self._hostnameASCII, e))

From d92f1b18580940c27f303f1dfcf47bb66e508ce4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 8 Aug 2019 23:53:35 -0300
Subject: [PATCH 2156/2974] Simplify import + assignment

---
 scrapy/core/downloader/tls.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 1c3d94b29..4ed482058 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -29,8 +29,7 @@ openssl_methods = {
 
 
 if twisted_version < (17, 0, 0):
-    from twisted.internet._sslverify import _maybeSetHostNameIndication
-    set_tlsext_host_name = _maybeSetHostNameIndication
+    from twisted.internet._sslverify import _maybeSetHostNameIndication as set_tlsext_host_name
 else:
     def set_tlsext_host_name(connection, hostNameBytes):
         connection.set_tlsext_host_name(hostNameBytes)

From e17c9a48fdd41e838c235be75038fab1ab9bb8e2 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 8 Aug 2019 23:59:17 -0300
Subject: [PATCH 2157/2974] Remove check for Twisted>=15.0.0

16.0.0 is currently the minimum supported version
---
 scrapy/core/downloader/handlers/http11.py | 28 +++++++----------------
 1 file changed, 8 insertions(+), 20 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 2ccab2614..8b4ae6b24 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -27,7 +27,7 @@ from scrapy.core.downloader.webclient import _parse
 from scrapy.core.downloader.tls import openssl_methods
 from scrapy.utils.misc import load_object, create_instance
 from scrapy.utils.python import to_bytes, to_unicode
-from scrapy import twisted_version
+
 
 logger = logging.getLogger(__name__)
 
@@ -211,21 +211,14 @@ class TunnelingAgent(Agent):
         self._proxyConf = proxyConf
         self._contextFactory = contextFactory
 
-    if twisted_version >= (15, 0, 0):
-        def _getEndpoint(self, uri):
-            return TunnelingTCP4ClientEndpoint(
-                self._reactor, uri.host, uri.port, self._proxyConf,
-                self._contextFactory, self._endpointFactory._connectTimeout,
-                self._endpointFactory._bindAddress)
-    else:
-        def _getEndpoint(self, scheme, host, port):
-            return TunnelingTCP4ClientEndpoint(
-                self._reactor, host, port, self._proxyConf,
-                self._contextFactory, self._connectTimeout,
-                self._bindAddress)
+    def _getEndpoint(self, uri):
+        return TunnelingTCP4ClientEndpoint(
+            self._reactor, uri.host, uri.port, self._proxyConf,
+            self._contextFactory, self._endpointFactory._connectTimeout,
+            self._endpointFactory._bindAddress)
 
     def _requestWithEndpoint(self, key, endpoint, method, parsedURI,
-            headers, bodyProducer, requestPath):
+                             headers, bodyProducer, requestPath):
         # proxy host and port are required for HTTP pool `key`
         # otherwise, same remote host connection request could reuse
         # a cached tunneled connection to a different proxy
@@ -250,12 +243,7 @@ class ScrapyProxyAgent(Agent):
         """
         # Cache *all* connections under the same key, since we are only
         # connecting to a single destination, the proxy:
-        if twisted_version >= (15, 0, 0):
-            proxyEndpoint = self._getEndpoint(self._proxyURI)
-        else:
-            proxyEndpoint = self._getEndpoint(self._proxyURI.scheme,
-                                              self._proxyURI.host,
-                                              self._proxyURI.port)
+        proxyEndpoint = self._getEndpoint(self._proxyURI)
         key = ("http-proxy", self._proxyURI.host, self._proxyURI.port)
         return self._requestWithEndpoint(key, proxyEndpoint, method,
                                          URI.fromBytes(uri), headers,

From d3737d869b1b49497f53376283b02ac604543e2c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 9 Aug 2019 00:21:43 -0300
Subject: [PATCH 2158/2974] Remove check for Twisted>=14.0

---
 scrapy/core/downloader/handlers/http11.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 8b4ae6b24..f0ed1a4af 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -141,14 +141,8 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         self._protocol.dataReceived = self._protocolDataReceived
         respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
         if respm and int(respm.group('status')) == 200:
-            try:
-                # this sets proper Server Name Indication extension
-                # but is only available for Twisted>=14.0
-                sslOptions = self._contextFactory.creatorForNetloc(
-                    self._tunneledHost, self._tunneledPort)
-            except AttributeError:
-                # fall back to non-SNI SSL context factory
-                sslOptions = self._contextFactory
+            # set proper Server Name Indication extension
+            sslOptions = self._contextFactory.creatorForNetloc(self._tunneledHost, self._tunneledPort)
             self._protocol.transport.startTLS(sslOptions,
                                               self._protocolFactory)
             self._tunnelReadyDeferred.callback(self._protocol)

From d5dcc5eaef80ef383dff90f19349c0e06f1836a6 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 9 Aug 2019 00:30:58 -0300
Subject: [PATCH 2159/2974] Import twisted.web.client.URI directly

---
 scrapy/core/downloader/handlers/http11.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index f0ed1a4af..9da20e032 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -13,12 +13,8 @@ from twisted.web.http_headers import Headers as TxHeaders
 from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
 from twisted.internet.error import TimeoutError
 from twisted.web.http import _DataLoss, PotentialDataLoss
-from twisted.web.client import Agent, ProxyAgent, ResponseDone, \
-    HTTPConnectionPool, ResponseFailed
-try:
-    from twisted.web.client import URI
-except ImportError:
-    from twisted.web.client import _URI as URI
+from twisted.web.client import (Agent, ProxyAgent, ResponseDone,
+                                HTTPConnectionPool, ResponseFailed, URI)
 from twisted.internet.endpoints import TCP4ClientEndpoint
 
 from scrapy.http import Headers

From 26fb28b20f661e6d7cfd02c904f4b1dc2a79e1dc Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 9 Aug 2019 00:49:46 -0300
Subject: [PATCH 2160/2974] PEP8-ify HTTP/1.1 downloader handler

Signed-off-by: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
---
 scrapy/core/downloader/handlers/http11.py | 150 +++++++++++++---------
 1 file changed, 91 insertions(+), 59 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 9da20e032..e72052afc 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -13,8 +13,7 @@ from twisted.web.http_headers import Headers as TxHeaders
 from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
 from twisted.internet.error import TimeoutError
 from twisted.web.http import _DataLoss, PotentialDataLoss
-from twisted.web.client import (Agent, ProxyAgent, ResponseDone,
-                                HTTPConnectionPool, ResponseFailed, URI)
+from twisted.web.client import Agent, ResponseDone, HTTPConnectionPool, ResponseFailed, URI
 from twisted.internet.endpoints import TCP4ClientEndpoint
 
 from scrapy.http import Headers
@@ -40,11 +39,19 @@ class HTTP11DownloadHandler(object):
         self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
         # try method-aware context factory
         try:
-            self._contextFactory = create_instance(self._contextFactoryClass, settings=settings, crawler=None,
-                                                   method=self._sslMethod)
+            self._contextFactory = create_instance(
+                self._contextFactoryClass,
+                settings=settings,
+                crawler=None,
+                method=self._sslMethod,
+            )
         except TypeError:
             # use context factory defaults
-            self._contextFactory = create_instance(self._contextFactoryClass, settings=settings, crawler=None)
+            self._contextFactory = create_instance(
+                self._contextFactoryClass,
+                settings=settings,
+                crawler=None,
+            )
             msg = """
  '%s' does not accept `method` argument (type OpenSSL.SSL method,\
  e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument.\
@@ -58,10 +65,13 @@ class HTTP11DownloadHandler(object):
 
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
-        agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool,
+        agent = ScrapyAgent(
+            contextFactory=self._contextFactory,
+            pool=self._pool,
             maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
             warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
-            fail_on_dataloss=self._fail_on_dataloss)
+            fail_on_dataloss=self._fail_on_dataloss,
+        )
         return agent.download_request(request)
 
     def close(self):
@@ -100,11 +110,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
 
     _responseMatcher = re.compile(br'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
 
-    def __init__(self, reactor, host, port, proxyConf, contextFactory,
-                 timeout=30, bindAddress=None):
+    def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
         proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
-        super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost,
-            proxyPort, timeout, bindAddress)
+        super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
         self._tunnelReadyDeferred = defer.Deferred()
         self._tunneledHost = host
         self._tunneledPort = port
@@ -113,8 +121,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
 
     def requestTunnel(self, protocol):
         """Asks the proxy to open a tunnel."""
-        tunnelReq = tunnel_request_data(self._tunneledHost, self._tunneledPort,
-                                        self._proxyAuthHeader)
+        tunnelReq = tunnel_request_data(self._tunneledHost, self._tunneledPort, self._proxyAuthHeader)
         protocol.transport.write(tunnelReq)
         self._protocolDataReceived = protocol.dataReceived
         protocol.dataReceived = self.processProxyResponse
@@ -139,8 +146,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         if respm and int(respm.group('status')) == 200:
             # set proper Server Name Indication extension
             sslOptions = self._contextFactory.creatorForNetloc(self._tunneledHost, self._tunneledPort)
-            self._protocol.transport.startTLS(sslOptions,
-                                              self._protocolFactory)
+            self._protocol.transport.startTLS(sslOptions, self._protocolFactory)
             self._tunnelReadyDeferred.callback(self._protocol)
         else:
             if respm:
@@ -158,8 +164,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
 
     def connect(self, protocolFactory):
         self._protocolFactory = protocolFactory
-        connectDeferred = super(TunnelingTCP4ClientEndpoint,
-                                self).connect(protocolFactory)
+        connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory)
         connectDeferred.addCallback(self.requestTunnel)
         connectDeferred.addErrback(self.connectFailed)
         return self._tunnelReadyDeferred
@@ -196,35 +201,46 @@ class TunnelingAgent(Agent):
 
     def __init__(self, reactor, proxyConf, contextFactory=None,
                  connectTimeout=None, bindAddress=None, pool=None):
-        super(TunnelingAgent, self).__init__(reactor, contextFactory,
-            connectTimeout, bindAddress, pool)
+        super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
         self._proxyConf = proxyConf
         self._contextFactory = contextFactory
 
     def _getEndpoint(self, uri):
         return TunnelingTCP4ClientEndpoint(
-            self._reactor, uri.host, uri.port, self._proxyConf,
-            self._contextFactory, self._endpointFactory._connectTimeout,
-            self._endpointFactory._bindAddress)
+            reactor=self._reactor,
+            host=uri.host,
+            port=uri.port,
+            proxyConf=self._proxyConf,
+            contextFactory=self._contextFactory,
+            timeout=self._endpointFactory._connectTimeout,
+            bindAddress=self._endpointFactory._bindAddress,
+        )
 
-    def _requestWithEndpoint(self, key, endpoint, method, parsedURI,
-                             headers, bodyProducer, requestPath):
+    def _requestWithEndpoint(self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath):
         # proxy host and port are required for HTTP pool `key`
         # otherwise, same remote host connection request could reuse
         # a cached tunneled connection to a different proxy
         key = key + self._proxyConf
-        return super(TunnelingAgent, self)._requestWithEndpoint(key, endpoint, method, parsedURI,
-            headers, bodyProducer, requestPath)
+        return super(TunnelingAgent, self)._requestWithEndpoint(
+            key=key,
+            endpoint=endpoint,
+            method=method,
+            parsedURI=parsedURI,
+            headers=headers,
+            bodyProducer=bodyProducer,
+            requestPath=requestPath,
+        )
 
 
 class ScrapyProxyAgent(Agent):
 
-    def __init__(self, reactor, proxyURI,
-                 connectTimeout=None, bindAddress=None, pool=None):
-        super(ScrapyProxyAgent, self).__init__(reactor,
-                                               connectTimeout=connectTimeout,
-                                               bindAddress=bindAddress,
-                                               pool=pool)
+    def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
+        super(ScrapyProxyAgent, self).__init__(
+            reactor=reactor,
+            connectTimeout=connectTimeout,
+            bindAddress=bindAddress,
+            pool=pool,
+        )
         self._proxyURI = URI.fromBytes(proxyURI)
 
     def request(self, method, uri, headers=None, bodyProducer=None):
@@ -233,11 +249,15 @@ class ScrapyProxyAgent(Agent):
         """
         # Cache *all* connections under the same key, since we are only
         # connecting to a single destination, the proxy:
-        proxyEndpoint = self._getEndpoint(self._proxyURI)
-        key = ("http-proxy", self._proxyURI.host, self._proxyURI.port)
-        return self._requestWithEndpoint(key, proxyEndpoint, method,
-                                         URI.fromBytes(uri), headers,
-                                         bodyProducer, uri)
+        return self._requestWithEndpoint(
+            key=("http-proxy", self._proxyURI.host, self._proxyURI.port),
+            endpoint=self._getEndpoint(self._proxyURI),
+            method=method,
+            parsedURI=URI.fromBytes(uri),
+            headers=headers,
+            bodyProducer=bodyProducer,
+            requestPath=uri,
+        )
 
 
 class ScrapyAgent(object):
@@ -265,18 +285,33 @@ class ScrapyAgent(object):
             scheme = _parse(request.url)[0]
             proxyHost = to_unicode(proxyHost)
             omitConnectTunnel = b'noconnect' in proxyParams
-            if  scheme == b'https' and not omitConnectTunnel:
-                proxyConf = (proxyHost, proxyPort,
-                             request.headers.get(b'Proxy-Authorization', None))
-                return self._TunnelingAgent(reactor, proxyConf,
-                    contextFactory=self._contextFactory, connectTimeout=timeout,
-                    bindAddress=bindaddress, pool=self._pool)
+            if scheme == b'https' and not omitConnectTunnel:
+                proxyAuth = request.headers.get(b'Proxy-Authorization', None)
+                proxyConf = (proxyHost, proxyPort, proxyAuth)
+                return self._TunnelingAgent(
+                    reactor=reactor,
+                    proxyConf=proxyConf,
+                    contextFactory=self._contextFactory,
+                    connectTimeout=timeout,
+                    bindAddress=bindaddress,
+                    pool=self._pool,
+                )
             else:
-                return self._ProxyAgent(reactor, proxyURI=to_bytes(proxy, encoding='ascii'),
-                    connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)
+                return self._ProxyAgent(
+                    reactor=reactor,
+                    proxyURI=to_bytes(proxy, encoding='ascii'),
+                    connectTimeout=timeout,
+                    bindAddress=bindaddress,
+                    pool=self._pool,
+                )
 
-        return self._Agent(reactor, contextFactory=self._contextFactory,
-            connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)
+        return self._Agent(
+            reactor=reactor,
+            contextFactory=self._contextFactory,
+            connectTimeout=timeout,
+            bindAddress=bindaddress,
+            pool=self._pool,
+        )
 
     def download_request(self, request):
         timeout = request.meta.get('download_timeout') or self._connectTimeout
@@ -307,8 +342,7 @@ class ScrapyAgent(object):
         else:
             bodyproducer = None
         start_time = time()
-        d = agent.request(
-            method, to_bytes(url, encoding='ascii'), headers, bodyproducer)
+        d = agent.request(method, to_bytes(url, encoding='ascii'), headers, bodyproducer)
         # set download latency
         d.addCallback(self._cb_latency, request, start_time)
         # response body is ready to be consumed
@@ -363,8 +397,9 @@ class ScrapyAgent(object):
             txresponse._transport._producer.abortConnection()
 
         d = defer.Deferred(_cancel)
-        txresponse.deliverBody(_ResponseReader(
-            d, txresponse, request, maxsize, warnsize, fail_on_dataloss))
+        txresponse.deliverBody(
+            _ResponseReader(d, txresponse, request, maxsize, warnsize, fail_on_dataloss)
+        )
 
         # save response for timeouts
         self._txresponse = txresponse
@@ -399,22 +434,20 @@ class _RequestBodyProducer(object):
 
 class _ResponseReader(protocol.Protocol):
 
-    def __init__(self, finished, txresponse, request, maxsize, warnsize,
-                 fail_on_dataloss):
+    def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss):
         self._finished = finished
         self._txresponse = txresponse
         self._request = request
         self._bodybuf = BytesIO()
-        self._maxsize  = maxsize
-        self._warnsize  = warnsize
+        self._maxsize = maxsize
+        self._warnsize = warnsize
         self._fail_on_dataloss = fail_on_dataloss
         self._fail_on_dataloss_warned = False
         self._reached_warnsize = False
         self._bytes_received = 0
 
     def dataReceived(self, bodyBytes):
-        # This maybe called several times after cancel was called with buffered
-        # data.
+        # This maybe called several times after cancel was called with buffered data.
         if self._finished.called:
             return
 
@@ -427,8 +460,7 @@ class _ResponseReader(protocol.Protocol):
                          {'bytes': self._bytes_received,
                           'maxsize': self._maxsize,
                           'request': self._request})
-            # Clear buffer earlier to avoid keeping data in memory for a long
-            # time.
+            # Clear buffer earlier to avoid keeping data in memory for a long time.
             self._bodybuf.truncate(0)
             self._finished.cancel()
 

From 3384db92b4fb2bce66d01ddf5365478a695caad6 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 27 Sep 2018 19:51:27 +0500
Subject: [PATCH 2161/2974] Add support for setting SSL ciphers.

---
 scrapy/core/downloader/contextfactory.py | 13 +++++++++----
 scrapy/settings/default_settings.py      |  1 +
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 127a246f5..89d2776ae 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,5 +1,5 @@
 from OpenSSL import SSL
-from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust
+from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
 from twisted.web.client import BrowserLikePolicyForHTTPS
 from twisted.web.iweb import IPolicyForHTTPS
 from zope.interface.declarations import implementer
@@ -19,15 +19,20 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
      understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
     """
 
-    def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs):
+    def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
         super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
         self._ssl_method = method
         self.tls_verbose_logging = tls_verbose_logging
+        if tls_ciphers:
+            self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers)
+        else:
+            self.tls_ciphers = DEFAULT_CIPHERS
 
     @classmethod
     def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
         tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
-        return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs)
+        tls_ciphers = settings['DOWNLOADER_CLIENT_TLS_CIPHERS']
+        return cls(method=method, tls_verbose_logging=tls_verbose_logging, tls_ciphers=tls_ciphers, *args, **kwargs)
 
     def getCertificateOptions(self):
         # setting verify=True will require you to provide CAs
@@ -45,7 +50,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
                     method=getattr(self, 'method',
                                    getattr(self, '_ssl_method', None)),
                     fixBrokenPeers=True,
-                    acceptableCiphers=DEFAULT_CIPHERS)
+                    acceptableCiphers=self.tls_ciphers)
 
     # kept for old-style HTTP/1.0 downloader context twisted calls,
     # e.g. connectSSL()
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 81fee543f..742c8e8a1 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -85,6 +85,7 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader'
 
 DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
 DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
+DOWNLOADER_CLIENT_TLS_CIPHERS = 'DEFAULT'
 DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
                                      # also allowing negotiation
 DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False

From ce281d890dfbd905922e16552fdcee34eb0d6c42 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 28 Sep 2018 20:17:12 +0500
Subject: [PATCH 2162/2974] Documentation for DOWNLOADER_CLIENT_TLS_CIPHERS.

---
 docs/topics/settings.rst | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 12606fe47..c042d3f43 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -445,6 +445,24 @@ accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
 :setting:`DOWNLOADER_CLIENT_TLS_METHOD`) and a ``tls_verbose_logging``
 parameter (``bool``).
 
+.. setting:: DOWNLOADER_CLIENT_TLS_CIPHERS
+
+DOWNLOADER_CLIENT_TLS_CIPHERS
+-----------------------------
+
+Default: ``'DEFAULT'``
+
+Use  this setting to customize the TLS/SSL ciphers used by the default
+HTTP/1.1 downloader.
+
+The setting should contain a string in the `OpenSSL cipher list format`_,
+these ciphers will be used as client ciphers. Changing this setting may be
+necessary to access certain HTTPS websites: for example, you may need to use
+``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
+specific cipher that is not included in ``DEFAULT`` if a website requires it.
+
+.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT
+
 .. setting:: DOWNLOADER_CLIENT_TLS_METHOD
 
 DOWNLOADER_CLIENT_TLS_METHOD

From 9a8edf2bf1172e8eec282a70ad46a9dacff76d62 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 27 Sep 2018 19:52:29 +0500
Subject: [PATCH 2163/2974] Tests for setting SSL ciphers.

---
 tests/mockserver.py               | 10 +++++++
 tests/test_downloader_handlers.py | 43 +++++++++++++++++++++++++-
 tests/test_webclient.py           | 50 ++++++++++++++++++++++++++++++-
 3 files changed, 101 insertions(+), 2 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 3fa4bc0f0..8be8a36bb 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -3,6 +3,8 @@ import sys, time, random, os, json
 from six.moves.urllib.parse import urlencode
 from subprocess import Popen, PIPE
 
+from OpenSSL import SSL
+
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
 from twisted.web.static import File
@@ -222,6 +224,14 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c
          )
 
 
+def broken_ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string='DEFAULT'):
+    factory = ssl_context_factory(keyfile, certfile)
+    ctx = factory.getContext()
+    ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_2)
+    ctx.set_cipher_list(cipher_string)
+    return factory
+
+
 if __name__ == "__main__":
     root = Root()
     factory = Site(root)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index efef4192c..5df2ffe45 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -39,7 +39,7 @@ from scrapy.utils.test import get_crawler, skip_if_no_boto
 from scrapy.utils.python import to_bytes
 from scrapy.exceptions import NotConfigured
 
-from tests.mockserver import MockServer, ssl_context_factory, Echo
+from tests.mockserver import MockServer, ssl_context_factory, Echo, broken_ssl_context_factory
 from tests.spiders import SingleRequestSpider
 
 
@@ -553,6 +553,47 @@ class Https11InvalidDNSPattern(Https11TestCase):
         super(Https11InvalidDNSPattern, self).setUp()
 
 
+class Https11BadCiphers(unittest.TestCase):
+    scheme = 'https'
+    download_handler_cls = HTTP11DownloadHandler
+
+    keyfile = 'keys/localhost.key'
+    certfile = 'keys/localhost.crt'
+
+    def setUp(self):
+        self.tmpname = self.mktemp()
+        os.mkdir(self.tmpname)
+        FilePath(self.tmpname).child("file").setContent(b"0123456789")
+        r = static.File(self.tmpname)
+        self.site = server.Site(r, timeout=None)
+        self.wrapper = WrappingFactory(self.site)
+        self.host = 'localhost'
+        self.port = reactor.listenSSL(
+            0, self.wrapper, broken_ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),
+            interface=self.host)
+        self.portno = self.port.getHost().port
+        self.download_handler = self.download_handler_cls(
+            Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'}))
+        self.download_request = self.download_handler.download_request
+
+    @defer.inlineCallbacks
+    def tearDown(self):
+        yield self.port.stopListening()
+        if hasattr(self.download_handler, 'close'):
+            yield self.download_handler.close()
+        shutil.rmtree(self.tmpname)
+
+    def getURL(self, path):
+        return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path)
+
+    def test_download(self):
+        request = Request(self.getURL('file'))
+        d = self.download_request(request, Spider('foo'))
+        d.addCallback(lambda r: r.body)
+        d.addCallback(self.assertEqual, b"0123456789")
+        return d
+
+
 class Http11MockServerTestCase(unittest.TestCase):
     """HTTP 1.1 test case with MockServer"""
 
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 766329b57..2ebe075ab 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -8,15 +8,18 @@ import shutil
 
 from twisted.trial import unittest
 from twisted.web import server, static, util, resource
-from twisted.internet import reactor, defer
+from twisted.internet import reactor, defer, ssl
 from twisted.test.proto_helpers import StringTransport
 from twisted.python.filepath import FilePath
 from twisted.protocols.policies import WrappingFactory
 from twisted.internet.defer import inlineCallbacks
 
 from scrapy.core.downloader import webclient as client
+from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
 from scrapy.http import Request, Headers
+from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes, to_unicode
+from tests.mockserver import ssl_context_factory, broken_ssl_context_factory
 
 
 def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
@@ -363,3 +366,48 @@ class WebClientTestCase(unittest.TestCase):
         self.assertEqual(content_encoding, EncodingResource.out_encoding)
         self.assertEqual(
             response.body.decode(content_encoding), to_unicode(original_body))
+
+
+class WebClientSSLTestCase(unittest.TestCase):
+    context_factory = None
+
+    def _listen(self, site):
+        return reactor.listenSSL(
+            0, site,
+            contextFactory=self.context_factory or ssl_context_factory(),
+            interface="127.0.0.1")
+
+    def getURL(self, path):
+        return "https://127.0.0.1:%d/%s" % (self.portno, path)
+
+    def setUp(self):
+        self.tmpname = self.mktemp()
+        os.mkdir(self.tmpname)
+        FilePath(self.tmpname).child("file").setContent(b"0123456789")
+        r = static.File(self.tmpname)
+        r.putChild(b"payload", PayloadResource())
+        self.site = server.Site(r, timeout=None)
+        self.wrapper = WrappingFactory(self.site)
+        self.port = self._listen(self.wrapper)
+        self.portno = self.port.getHost().port
+
+    @inlineCallbacks
+    def tearDown(self):
+        yield self.port.stopListening()
+        shutil.rmtree(self.tmpname)
+
+    def testPayload(self):
+        s = "0123456789" * 10
+        return getPage(self.getURL("payload"), body=s).addCallback(
+            self.assertEqual, to_bytes(s))
+
+
+class WebClientBrokenSSLTestCase(WebClientSSLTestCase):
+    context_factory = broken_ssl_context_factory(cipher_string='CAMELLIA256-SHA')
+
+    def testPayload(self):
+        s = "0123456789" * 10
+        settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})
+        return getPage(self.getURL("payload"), body=s,
+                       contextFactory=ScrapyClientContextFactory(settings=settings)).addCallback(self.assertEqual,
+                                                                                                 to_bytes(s))

From aaa5229e5db4f1c014ae69e0fb9e1a933f4952b0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 15 Jul 2019 17:47:06 +0500
Subject: [PATCH 2164/2974] Fixes and improvements for
 DOWNLOADER_CLIENT_TLS_CIPHERS.

---
 docs/topics/settings.rst                  |  5 +++--
 scrapy/core/downloader/handlers/http11.py |  2 +-
 scrapy/utils/ssl.py                       |  5 +++++
 tests/mockserver.py                       | 19 ++++++++-----------
 tests/test_downloader_handlers.py         |  6 +++---
 tests/test_webclient.py                   | 23 ++++++++++++++++-------
 6 files changed, 36 insertions(+), 24 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index c042d3f43..0cb81c43e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -442,8 +442,9 @@ or even enable client-side authentication (and various other things).
 
 If you do use a custom ContextFactory, make sure its ``__init__`` method
 accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
-:setting:`DOWNLOADER_CLIENT_TLS_METHOD`) and a ``tls_verbose_logging``
-parameter (``bool``).
+:setting:`DOWNLOADER_CLIENT_TLS_METHOD`), a ``tls_verbose_logging``
+parameter (``bool``) and a ``tls_ciphers`` parameter (see
+:setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`).
 
 .. setting:: DOWNLOADER_CLIENT_TLS_CIPHERS
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index e72052afc..91b45a8fc 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -54,7 +54,7 @@ class HTTP11DownloadHandler(object):
             )
             msg = """
  '%s' does not accept `method` argument (type OpenSSL.SSL method,\
- e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument.\
+ e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
  Please upgrade your context factory class to handle them or ignore them.""" % (
                 settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
             warnings.warn(msg)
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index 632827471..02aed60ee 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -6,6 +6,11 @@ import OpenSSL._util as pyOpenSSLutil
 from scrapy.utils.python import to_native_str
 
 
+# The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL.
+# Using the binding directly, as this code does, requires cryptography 2.4.
+SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0)
+
+
 def ffi_buf_to_string(buf):
     return to_native_str(pyOpenSSLutil.ffi.string(buf))
 
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 8be8a36bb..77908284b 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -4,7 +4,6 @@ from six.moves.urllib.parse import urlencode
 from subprocess import Popen, PIPE
 
 from OpenSSL import SSL
-
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
 from twisted.web.static import File
@@ -15,8 +14,8 @@ from twisted.web.util import redirectTo
 from twisted.internet import reactor, ssl
 from twisted.internet.task import deferLater
 
-
 from scrapy.utils.python import to_bytes, to_unicode
+from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3
 
 
 def getarg(request, name, default=None, type=None):
@@ -217,18 +216,16 @@ class MockServer():
         return host + path
 
 
-def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt'):
-    return ssl.DefaultOpenSSLContextFactory(
+def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):
+    factory = ssl.DefaultOpenSSLContextFactory(
          os.path.join(os.path.dirname(__file__), keyfile),
          os.path.join(os.path.dirname(__file__), certfile),
          )
-
-
-def broken_ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string='DEFAULT'):
-    factory = ssl_context_factory(keyfile, certfile)
-    ctx = factory.getContext()
-    ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_2)
-    ctx.set_cipher_list(cipher_string)
+    if cipher_string:
+        ctx = factory.getContext()
+        # disabling TLS1.2+ because it unconditionally enables some strong ciphers
+        ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_2 | SSL_OP_NO_TLSv1_3)
+        ctx.set_cipher_list(to_bytes(cipher_string))
     return factory
 
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 5df2ffe45..109469503 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -39,7 +39,7 @@ from scrapy.utils.test import get_crawler, skip_if_no_boto
 from scrapy.utils.python import to_bytes
 from scrapy.exceptions import NotConfigured
 
-from tests.mockserver import MockServer, ssl_context_factory, Echo, broken_ssl_context_factory
+from tests.mockserver import MockServer, ssl_context_factory, Echo
 from tests.spiders import SingleRequestSpider
 
 
@@ -553,7 +553,7 @@ class Https11InvalidDNSPattern(Https11TestCase):
         super(Https11InvalidDNSPattern, self).setUp()
 
 
-class Https11BadCiphers(unittest.TestCase):
+class Https11CustomCiphers(unittest.TestCase):
     scheme = 'https'
     download_handler_cls = HTTP11DownloadHandler
 
@@ -569,7 +569,7 @@ class Https11BadCiphers(unittest.TestCase):
         self.wrapper = WrappingFactory(self.site)
         self.host = 'localhost'
         self.port = reactor.listenSSL(
-            0, self.wrapper, broken_ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),
+            0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),
             interface=self.host)
         self.portno = self.port.getHost().port
         self.download_handler = self.download_handler_cls(
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 2ebe075ab..a81946490 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -6,9 +6,10 @@ import os
 import six
 import shutil
 
+import OpenSSL.SSL
 from twisted.trial import unittest
 from twisted.web import server, static, util, resource
-from twisted.internet import reactor, defer, ssl
+from twisted.internet import reactor, defer
 from twisted.test.proto_helpers import StringTransport
 from twisted.python.filepath import FilePath
 from twisted.protocols.policies import WrappingFactory
@@ -18,8 +19,9 @@ from scrapy.core.downloader import webclient as client
 from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
 from scrapy.http import Request, Headers
 from scrapy.settings import Settings
+from scrapy.utils.misc import create_instance
 from scrapy.utils.python import to_bytes, to_unicode
-from tests.mockserver import ssl_context_factory, broken_ssl_context_factory
+from tests.mockserver import ssl_context_factory
 
 
 def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
@@ -402,12 +404,19 @@ class WebClientSSLTestCase(unittest.TestCase):
             self.assertEqual, to_bytes(s))
 
 
-class WebClientBrokenSSLTestCase(WebClientSSLTestCase):
-    context_factory = broken_ssl_context_factory(cipher_string='CAMELLIA256-SHA')
+class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
+    # we try to use a cipher that is not enabled by default in OpenSSL
+    custom_ciphers = 'CAMELLIA256-SHA'
+    context_factory = ssl_context_factory(cipher_string=custom_ciphers)
 
     def testPayload(self):
         s = "0123456789" * 10
-        settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})
+        settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': self.custom_ciphers})
+        client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)
         return getPage(self.getURL("payload"), body=s,
-                       contextFactory=ScrapyClientContextFactory(settings=settings)).addCallback(self.assertEqual,
-                                                                                                 to_bytes(s))
+                       contextFactory=client_context_factory).addCallback(self.assertEqual, to_bytes(s))
+
+    def testPayloadDefaultCiphers(self):
+        s = "0123456789" * 10
+        d = getPage(self.getURL("payload"), body=s, contextFactory=ScrapyClientContextFactory())
+        return self.assertFailure(d, OpenSSL.SSL.Error)

From 50c4cafe0ccc53064f38654fd7426ded876469f7 Mon Sep 17 00:00:00 2001
From: Tobias Hernstig <tobias@bubbies.se>
Date: Sun, 11 Mar 2018 11:46:07 +0100
Subject: [PATCH 2165/2974] Update documentation for logging manually

Usage of basicConfig() together with crawlerRunner is not recommended.
Update documentation to highlight this fact.

Closes #2149, #2352, #3146
---
 docs/topics/logging.rst | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 87ea43c7d..2fd85196d 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -254,18 +254,18 @@ scrapy.utils.log module
     when running custom scripts using :class:`~scrapy.crawler.CrawlerRunner`.
     In that case, its usage is not required but it's recommended.
 
-    If you plan on configuring the handlers yourself is still recommended you
-    call this function, passing ``install_root_handler=False``. Bear in mind
-    there won't be any log output set by default in that case.
+    Another option when running custom scripts is to manually configure the logging.
+    To do this you can use `logging.basicConfig()`_ to set a basic root handler.
 
-    To get you started on manually configuring logging's output, you can use
-    `logging.basicConfig()`_ to set a basic root handler. This is an example
-    on how to redirect ``INFO`` or higher messages to a file::
+    Note that ``scrapy.crawler.CrawlerProcess`` automatically calls ``configure_logging``,
+    so it is recommended to only use `logging.basicConfig()`_ together with
+    ``scrapy.crawler.CrawlerRunner``
+
+    This is an example on how to redirect ``INFO`` or higher messages to a file::
 
         import logging
         from scrapy.utils.log import configure_logging
 
-        configure_logging(install_root_handler=False)
         logging.basicConfig(
             filename='log.txt',
             format='%(levelname)s: %(message)s',

From 2b0de0606c3b1a379722e916e9ce350e8de08a20 Mon Sep 17 00:00:00 2001
From: Tobias Hernstig <sinizmaster@hotmail.com>
Date: Thu, 15 Aug 2019 18:54:28 +0200
Subject: [PATCH 2166/2974] Fix merge conflicts

---
 docs/topics/logging.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 2fd85196d..87ab6e19a 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -257,9 +257,9 @@ scrapy.utils.log module
     Another option when running custom scripts is to manually configure the logging.
     To do this you can use `logging.basicConfig()`_ to set a basic root handler.
 
-    Note that ``scrapy.crawler.CrawlerProcess`` automatically calls ``configure_logging``,
+    Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``,
     so it is recommended to only use `logging.basicConfig()`_ together with
-    ``scrapy.crawler.CrawlerRunner``
+    :class:`~scrapy.crawler.CrawlerRunner`.
 
     This is an example on how to redirect ``INFO`` or higher messages to a file::
 

From c025003da2c60a0a786c2ee158fe620402914273 Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Sun, 18 Aug 2019 04:44:09 +0200
Subject: [PATCH 2167/2974] Add FTPFileStore

---
 scrapy/pipelines/files.py | 50 ++++++++++++++++++++++++++++++++++++---
 1 file changed, 47 insertions(+), 3 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 2145e6d2b..6f66460b8 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -10,6 +10,7 @@ import os.path
 import time
 import logging
 from email.utils import parsedate_tz, mktime_tz
+from ftplib import FTP
 from six.moves.urllib.parse import urlparse
 from collections import defaultdict
 import six
@@ -31,6 +32,7 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.request import referer_str
 from scrapy.utils.boto import is_botocore
 from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.ftp import ftp_makedirs_cwd
 
 logger = logging.getLogger(__name__)
 
@@ -248,6 +250,42 @@ class GCSFilesStore(object):
         )
 
 
+class FTPFilesStore(object):
+    
+    def __init__(self, uri):
+        assert uri.startswith('ftp://')
+        u = urlparse(uri)
+        self.ftp = FTP()
+        self.ftp.connect(u.hostname, u.port or '21')
+        self.ftp.login(u.username, u.password)
+        self.basedir = u.path + '/'
+        ftp_makedirs_cwd(self.ftp, self.basedir+'full')
+        
+    def persist_file(self, path, buf, info, meta=None, headers=None):
+        buf.seek(0)
+        filename = path.split('/')[1]
+        return threads.deferToThread(
+            self.ftp.storbinary,
+            'STOR %s' % filename,
+            buf
+        )
+            
+    def stat_file(self, path, info):
+        def _stat_file(path):
+            try:
+                last_modified = float(self.ftp.voidcmd("MDTM " + self.basedir + '/' + path)[4:].strip())
+                m = hashlib.md5()
+                self.ftp.retrbinary('RETR %s' % self.basedir + path, m.update)
+                return {'last_modified': last_modified, 'checksum': m.hexdigest()}
+            # The file doesn't exist
+            except Exception as e :
+                return {} 
+        return threads.deferToThread(_stat_file, path)
+    
+    def close_connection(self):
+        self.ftp.quit()
+
+
 class FilesPipeline(MediaPipeline):
     """Abstract pipeline that implement the file downloading
 
@@ -274,6 +312,7 @@ class FilesPipeline(MediaPipeline):
         'file': FSFilesStore,
         's3': S3FilesStore,
         'gs': GCSFilesStore,
+        'ftp': FTPFilesStore
     }
     DEFAULT_FILES_URLS_FIELD = 'file_urls'
     DEFAULT_FILES_RESULT_FIELD = 'files'
@@ -284,7 +323,6 @@ class FilesPipeline(MediaPipeline):
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
-
         cls_name = "FilesPipeline"
         self.store = self._get_store(store_uri)
         resolve = functools.partial(self._key_for_pipe,
@@ -303,7 +341,6 @@ class FilesPipeline(MediaPipeline):
         self.files_result_field = settings.get(
             resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD
         )
-
         super(FilesPipeline, self).__init__(download_func=download_func, settings=settings)
 
     @classmethod
@@ -320,7 +357,7 @@ class FilesPipeline(MediaPipeline):
         gcs_store = cls.STORE_SCHEMES['gs']
         gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
         gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] or None
-
+        
         store_uri = settings['FILES_STORE']
         return cls(store_uri, settings=settings)
 
@@ -461,3 +498,10 @@ class FilesPipeline(MediaPipeline):
         media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         media_ext = os.path.splitext(request.url)[1]
         return 'full/%s%s' % (media_guid, media_ext)
+    
+    def close_spider(self, spider):
+        try:
+            self.store.close_connection()
+        # If the store doesn't implement this function, pass
+        except AttributeError:
+            pass

From 00fe05e53611d5c61fb23cdd6674fd89c99fd69e Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Mon, 19 Aug 2019 09:19:06 +0530
Subject: [PATCH 2168/2974] adds ROBOTSTXT_USER_AGENT setting

---
 docs/topics/downloader-middleware.rst        | 15 +++++++++++++++
 docs/topics/settings.rst                     | 16 +++++++++++++++-
 scrapy/downloadermiddlewares/robotstxt.py    |  6 +++++-
 scrapy/settings/default_settings.py          |  1 +
 tests/test_downloadermiddleware_robotstxt.py |  9 +++++++++
 5 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 616b56101..ae413dc84 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1074,6 +1074,21 @@ implementing the methods described below.
 .. autoclass:: RobotParser
    :members:
 
+RobotsTxtMiddleware Settings
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. setting:: ROBOTSTXT_USER_AGENT
+
+ROBOTSTXT_USER_AGENT
+^^^^^^^^^^^^^^^^^^^^
+
+Default: ``None``
+
+The user agent string to use for matching in the robots.txt_ file. If ``None``,
+the User-Agent header you are sending with the request or the
+:setting:`USER_AGENT` setting (in that order) will be used for determining
+the user agent to use in the robots.txt_ file.
+
 .. _robots.txt: http://www.robotstxt.org/
 
 DownloaderStats
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 12606fe47..c4b55cc7b 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1151,6 +1151,18 @@ Default: ``'scrapy.robotstxt.PythonRobotParser'``
 The parser backend to use for parsing ``robots.txt`` files. For more information see
 :ref:`topics-dlmw-robots`.
 
+.. setting:: ROBOTSTXT_USER_AGENT
+
+ROBOTSTXT_USER_AGENT
+^^^^^^^^^^^^^^^^^^^^
+
+Default: ``None``
+
+The user agent string to use for matching in the robots.txt file. If ``None``,
+the User-Agent header you are sending with the request or the
+:setting:`USER_AGENT` setting (in that order) will be used for determining
+the user agent to use in the robots.txt file.
+
 .. setting:: SCHEDULER
 
 SCHEDULER
@@ -1409,7 +1421,9 @@ USER_AGENT
 
 Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
 
-The default User-Agent to use when crawling, unless overridden.
+The default User-Agent to use when crawling, unless overridden. This user agent is
+also used in robots.txt if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
+there is no overridding User-Agent header specified for the request.
 
 
 Settings documented elsewhere:
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index c5a60d355..6a5dfb79c 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -26,6 +26,7 @@ class RobotsTxtMiddleware(object):
         if not crawler.settings.getbool('ROBOTSTXT_OBEY'):
             raise NotConfigured
         self._default_useragent = crawler.settings.get('USER_AGENT', 'Scrapy')
+        self._robotstxt_useragent = crawler.settings.get('ROBOTSTXT_USER_AGENT', None)
         self.crawler = crawler
         self._parsers = {}
         self._parserimpl = load_object(crawler.settings.get('ROBOTSTXT_PARSER'))
@@ -47,7 +48,10 @@ class RobotsTxtMiddleware(object):
     def process_request_2(self, rp, request, spider):
         if rp is None:
             return
-        useragent = request.headers.get(b'User-Agent', self._default_useragent)
+
+        useragent = self._robotstxt_useragent
+        if not useragent:
+            useragent = request.headers.get(b'User-Agent', self._default_useragent)
         if not rp.allowed(request.url, useragent):
             logger.debug("Forbidden by robots.txt: %(request)s",
                          {'request': request}, extra={'spider': spider})
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 81fee543f..efb257e2b 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -246,6 +246,7 @@ RETRY_PRIORITY_ADJUST = -1
 
 ROBOTSTXT_OBEY = False
 ROBOTSTXT_PARSER = 'scrapy.robotstxt.PythonRobotParser'
+ROBOTSTXT_USER_AGENT = None
 
 SCHEDULER = 'scrapy.core.scheduler.Scheduler'
 SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 79f172848..fbc46cba4 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -164,6 +164,15 @@ Disallow: /some/randome/page.html
         d.addCallback(lambda _: self.assertFalse(mw_module_logger.error.called))
         return d
 
+    def test_robotstxt_user_agent_setting(self):
+        crawler = self._get_successful_crawler()
+        crawler.settings.set('ROBOTSTXT_USER_AGENT', 'Examplebot')
+        crawler.settings.set('USER_AGENT', 'Mozilla/5.0 (X11; Linux x86_64)')
+        middleware = RobotsTxtMiddleware(crawler)
+        rp = mock.MagicMock(return_value=True)
+        middleware.process_request_2(rp, Request('http://site.local/allowed'), None)
+        rp.allowed.assert_called_once_with('http://site.local/allowed', 'Examplebot')
+
     def assertNotIgnored(self, request, middleware):
         spider = None  # not actually used
         dfd = maybeDeferred(middleware.process_request, request, spider)

From 9b1587ed1bc736f9fcc357ce425405adf5bd6d08 Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Mon, 19 Aug 2019 16:13:56 +0200
Subject: [PATCH 2169/2974] Credentials from settings-Support custom
 paths-Remove close conenction

---
 scrapy/pipelines/files.py | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 6f66460b8..c22404799 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -251,19 +251,30 @@ class GCSFilesStore(object):
 
 
 class FTPFilesStore(object):
+
+    FTP_USERNAME = None
+    FTP_PASSWORD = None
     
     def __init__(self, uri):
         assert uri.startswith('ftp://')
         u = urlparse(uri)
         self.ftp = FTP()
         self.ftp.connect(u.hostname, u.port or '21')
-        self.ftp.login(u.username, u.password)
-        self.basedir = u.path + '/'
-        ftp_makedirs_cwd(self.ftp, self.basedir+'full')
+        username = u.username or FTP_USERNAME
+        password = u.password or FTP_PASSWORD
+        self.ftp.login(username, password)
+        self.basedir = u.path
         
     def persist_file(self, path, buf, info, meta=None, headers=None):
         buf.seek(0)
-        filename = path.split('/')[1]
+        # If the path is like 'x/y/z.ext' the 'x/y' is rel_path and 
+        # 'z.ext' is file name
+        # If path is only the file name 'z.ext', then rel_path is
+        # the empty string and filename is 'z.ext'
+        x = path.rsplit('/',1)
+        rel_path, filename = ('/' + x[0], x[1]) if len(x) > 1 else ('', x[0])
+        abs_path = self.basedir + rel_path
+        ftp_makedirs_cwd(self.ftp, abs_path)
         return threads.deferToThread(
             self.ftp.storbinary,
             'STOR %s' % filename,
@@ -281,9 +292,6 @@ class FTPFilesStore(object):
             except Exception as e :
                 return {} 
         return threads.deferToThread(_stat_file, path)
-    
-    def close_connection(self):
-        self.ftp.quit()
 
 
 class FilesPipeline(MediaPipeline):
@@ -357,6 +365,10 @@ class FilesPipeline(MediaPipeline):
         gcs_store = cls.STORE_SCHEMES['gs']
         gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
         gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] or None
+
+        ftp_store = cls.STORE_SCHEMES['ftp']
+        ftp_store.FTP_USERNAME = settings['FTP_USER']           # Default is 'anonymous'
+        ftp_store.FTP_PASSWORD = settings['FTP_PASSWORD']       # Default is `guest`
         
         store_uri = settings['FILES_STORE']
         return cls(store_uri, settings=settings)
@@ -497,11 +509,4 @@ class FilesPipeline(MediaPipeline):
     def file_path(self, request, response=None, info=None):
         media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         media_ext = os.path.splitext(request.url)[1]
-        return 'full/%s%s' % (media_guid, media_ext)
-    
-    def close_spider(self, spider):
-        try:
-            self.store.close_connection()
-        # If the store doesn't implement this function, pass
-        except AttributeError:
-            pass
+        return 'full/%s%s' % (media_guid, media_ext)
\ No newline at end of file

From 0a5cb7745bc22b8e193c4b0e964b20e59ddc0bc2 Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Mon, 19 Aug 2019 17:12:11 +0200
Subject: [PATCH 2170/2974] Fix reference mistake

---
 scrapy/pipelines/files.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index c22404799..cbe588f9a 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -260,8 +260,8 @@ class FTPFilesStore(object):
         u = urlparse(uri)
         self.ftp = FTP()
         self.ftp.connect(u.hostname, u.port or '21')
-        username = u.username or FTP_USERNAME
-        password = u.password or FTP_PASSWORD
+        username = u.username or self.FTP_USERNAME
+        password = u.password or self.FTP_PASSWORD
         self.ftp.login(username, password)
         self.basedir = u.path
         

From 81ac1da3813c11a806aca845a5022e9964b03f80 Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Mon, 19 Aug 2019 17:17:21 +0200
Subject: [PATCH 2171/2974] Handle leading and trailing slashes

---
 scrapy/pipelines/files.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index cbe588f9a..74697fc1d 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -263,7 +263,7 @@ class FTPFilesStore(object):
         username = u.username or self.FTP_USERNAME
         password = u.password or self.FTP_PASSWORD
         self.ftp.login(username, password)
-        self.basedir = u.path
+        self.basedir = u.path.rstrip('/')
         
     def persist_file(self, path, buf, info, meta=None, headers=None):
         buf.seek(0)
@@ -272,7 +272,7 @@ class FTPFilesStore(object):
         # If path is only the file name 'z.ext', then rel_path is
         # the empty string and filename is 'z.ext'
         x = path.rsplit('/',1)
-        rel_path, filename = ('/' + x[0], x[1]) if len(x) > 1 else ('', x[0])
+        rel_path, filename = ('/' + x[0].lstrip('/'), x[1]) if len(x) > 1 else ('', x[0])
         abs_path = self.basedir + rel_path
         ftp_makedirs_cwd(self.ftp, abs_path)
         return threads.deferToThread(

From 790bf9031229261639a5c457f6dd3498bdff8830 Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Mon, 19 Aug 2019 19:16:47 +0200
Subject: [PATCH 2172/2974] Make FTP persiting files thread safe

---
 scrapy/pipelines/files.py | 43 +++++++++++++++++++++------------------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 74697fc1d..2959179b8 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -257,29 +257,32 @@ class FTPFilesStore(object):
     
     def __init__(self, uri):
         assert uri.startswith('ftp://')
-        u = urlparse(uri)
-        self.ftp = FTP()
-        self.ftp.connect(u.hostname, u.port or '21')
-        username = u.username or self.FTP_USERNAME
-        password = u.password or self.FTP_PASSWORD
-        self.ftp.login(username, password)
+        u = urlparse(uri)  
+        self.port = u.port
+        self.host = u.hostname
+        self.port = int(u.port or '21')
+        self.username = u.username or self.FTP_USERNAME
+        self.password = u.password or self.FTP_PASSWORD
         self.basedir = u.path.rstrip('/')
         
     def persist_file(self, path, buf, info, meta=None, headers=None):
-        buf.seek(0)
-        # If the path is like 'x/y/z.ext' the 'x/y' is rel_path and 
-        # 'z.ext' is file name
-        # If path is only the file name 'z.ext', then rel_path is
-        # the empty string and filename is 'z.ext'
-        x = path.rsplit('/',1)
-        rel_path, filename = ('/' + x[0].lstrip('/'), x[1]) if len(x) > 1 else ('', x[0])
-        abs_path = self.basedir + rel_path
-        ftp_makedirs_cwd(self.ftp, abs_path)
-        return threads.deferToThread(
-            self.ftp.storbinary,
-            'STOR %s' % filename,
-            buf
-        )
+        
+        def _persist_file(path, buf):
+            ftp = FTP()
+            ftp.connect(self.host, self.port)
+            ftp.login(self.username, self.password)
+            buf.seek(0)
+            # If the path is like 'x/y/z.ext' the 'x/y' is rel_path and 
+            # 'z.ext' is file name
+            # If path is only the file name 'z.ext', then rel_path is
+            # the empty string and filename is 'z.ext'
+            x = path.rsplit('/',1)
+            rel_path, filename = ('/' + x[0].lstrip('/'), x[1]) if len(x) > 1 else ('', x[0])
+            abs_path = self.basedir + rel_path
+            ftp_makedirs_cwd(ftp, abs_path)
+            ftp.storbinary('STOR %s' % filename, buf)
+
+        return threads.deferToThread(_persist_file, path, buf)
             
     def stat_file(self, path, info):
         def _stat_file(path):

From 8c970c636eb37deeb5caddbe5069bfcc0a79015e Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Wed, 21 Aug 2019 18:28:36 +0200
Subject: [PATCH 2173/2974] port from str to int
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 scrapy/pipelines/files.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 2959179b8..bbe4b9558 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -260,7 +260,7 @@ class FTPFilesStore(object):
         u = urlparse(uri)  
         self.port = u.port
         self.host = u.hostname
-        self.port = int(u.port or '21')
+        self.port = int(u.port or 21)
         self.username = u.username or self.FTP_USERNAME
         self.password = u.password or self.FTP_PASSWORD
         self.basedir = u.path.rstrip('/')
@@ -512,4 +512,4 @@ class FilesPipeline(MediaPipeline):
     def file_path(self, request, response=None, info=None):
         media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         media_ext = os.path.splitext(request.url)[1]
-        return 'full/%s%s' % (media_guid, media_ext)
\ No newline at end of file
+        return 'full/%s%s' % (media_guid, media_ext)

From bd22b25ef4e4223aafc6e326058c6d62b1fbf13c Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Thu, 22 Aug 2019 01:30:15 +0200
Subject: [PATCH 2174/2974] Make `stat_file` thread safe .. Refactor file
 storing..  Support act/psv

---
 scrapy/extensions/feedexport.py | 17 +++++---------
 scrapy/pipelines/files.py       | 39 +++++++++++++++------------------
 scrapy/utils/ftp.py             | 21 +++++++++++++++++-
 3 files changed, 44 insertions(+), 33 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index d35551fdd..1ddc55f93 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -19,7 +19,7 @@ from twisted.internet import defer, threads
 from w3lib.url import file_uri_to_path
 
 from scrapy import signals
-from scrapy.utils.ftp import ftp_makedirs_cwd
+from scrapy.utils.ftp import ftp_makedirs_cwd, ftp_store_file
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.log import failure_to_exc_info
@@ -174,16 +174,11 @@ class FTPFeedStorage(BlockingFeedStorage):
         )
 
     def _store_in_thread(self, file):
-        file.seek(0)
-        ftp = FTP()
-        ftp.connect(self.host, self.port)
-        ftp.login(self.username, self.password)
-        if self.use_active_mode:
-            ftp.set_pasv(False)
-        dirname, filename = posixpath.split(self.path)
-        ftp_makedirs_cwd(ftp, dirname)
-        ftp.storbinary('STOR %s' % filename, file)
-        ftp.quit()
+        ftp_store_file(
+            self.path, file, self.host,
+            self.port, self.username, 
+            self.password, self.use_active_mode
+        )
 
 
 class SpiderSlot(object):
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index bbe4b9558..04fbf3237 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -32,7 +32,7 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.request import referer_str
 from scrapy.utils.boto import is_botocore
 from scrapy.utils.datatypes import CaselessDict
-from scrapy.utils.ftp import ftp_makedirs_cwd
+from scrapy.utils.ftp import ftp_makedirs_cwd, ftp_store_file
 
 logger = logging.getLogger(__name__)
 
@@ -254,6 +254,7 @@ class FTPFilesStore(object):
 
     FTP_USERNAME = None
     FTP_PASSWORD = None
+    USE_ACTIVE_MODE = None
     
     def __init__(self, uri):
         assert uri.startswith('ftp://')
@@ -265,31 +266,26 @@ class FTPFilesStore(object):
         self.password = u.password or self.FTP_PASSWORD
         self.basedir = u.path.rstrip('/')
         
-    def persist_file(self, path, buf, info, meta=None, headers=None):
-        
-        def _persist_file(path, buf):
-            ftp = FTP()
-            ftp.connect(self.host, self.port)
-            ftp.login(self.username, self.password)
-            buf.seek(0)
-            # If the path is like 'x/y/z.ext' the 'x/y' is rel_path and 
-            # 'z.ext' is file name
-            # If path is only the file name 'z.ext', then rel_path is
-            # the empty string and filename is 'z.ext'
-            x = path.rsplit('/',1)
-            rel_path, filename = ('/' + x[0].lstrip('/'), x[1]) if len(x) > 1 else ('', x[0])
-            abs_path = self.basedir + rel_path
-            ftp_makedirs_cwd(ftp, abs_path)
-            ftp.storbinary('STOR %s' % filename, buf)
-
-        return threads.deferToThread(_persist_file, path, buf)
+    def persist_file(self, path, buf, info, meta=None, headers=None):     
+        path = '%s/%s' % (self.basedir, path)
+        return threads.deferToThread(
+            ftp_store_file, path,buf,
+            self.host, self.port,self.username,
+            self.password, self.USE_ACTIVE_MODE
+        )
             
     def stat_file(self, path, info):
         def _stat_file(path):
             try:
-                last_modified = float(self.ftp.voidcmd("MDTM " + self.basedir + '/' + path)[4:].strip())
+                ftp = FTP()
+                ftp.connect(self.host, self.port)
+                ftp.login(self.username, self.password)
+                if self.USE_ACTIVE_MODE:
+                    ftp.set_pasv(False)
+                file_path = "%s/%s" % (self.basedir, path)
+                last_modified = float(ftp.voidcmd("MDTM %s" % file_path)[4:].strip())
                 m = hashlib.md5()
-                self.ftp.retrbinary('RETR %s' % self.basedir + path, m.update)
+                ftp.retrbinary('RETR %s' % file_path, m.update)
                 return {'last_modified': last_modified, 'checksum': m.hexdigest()}
             # The file doesn't exist
             except Exception as e :
@@ -372,6 +368,7 @@ class FilesPipeline(MediaPipeline):
         ftp_store = cls.STORE_SCHEMES['ftp']
         ftp_store.FTP_USERNAME = settings['FTP_USER']           # Default is 'anonymous'
         ftp_store.FTP_PASSWORD = settings['FTP_PASSWORD']       # Default is `guest`
+        ftp_store.USE_ACTIVE_MODE = settings.getbool('FEED_STORAGE_FTP_ACTIVE')
         
         store_uri = settings['FILES_STORE']
         return cls(store_uri, settings=settings)
diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index 9eca6a4da..ba94ec142 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -1,4 +1,6 @@
-from ftplib import error_perm
+import posixpath
+
+from ftplib import error_perm, FTP
 from posixpath import dirname
 
 def ftp_makedirs_cwd(ftp, path, first_call=True):
@@ -13,3 +15,20 @@ def ftp_makedirs_cwd(ftp, path, first_call=True):
         ftp.mkd(path)
         if first_call:
             ftp.cwd(path)
+
+def ftp_store_file( 
+        path, file, host ,port,
+        username, password, use_active_mode=False):
+    """Opens a FTP connection with passed credentials,sets current directory
+    to the directory extracted from given path, then uploads the file to server
+    """
+    ftp = FTP()
+    ftp.connect(host, port)
+    ftp.login(username, password)
+    if use_active_mode:
+        ftp.set_pasv(False)
+    file.seek(0)
+    dirname, filename = posixpath.split(path)
+    ftp_makedirs_cwd(ftp, dirname)
+    ftp.storbinary('STOR %s' % filename, file)
+    ftp.quit()
\ No newline at end of file

From 2047124b3573d02ec60b13614f4e3dce85e71546 Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Thu, 22 Aug 2019 16:18:14 +0200
Subject: [PATCH 2175/2974] Follow PEP8 .. Remove unnecessary comments

---
 scrapy/pipelines/files.py | 6 ++++--
 scrapy/utils/ftp.py       | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 04fbf3237..5780f63bd 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -330,6 +330,7 @@ class FilesPipeline(MediaPipeline):
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
+
         cls_name = "FilesPipeline"
         self.store = self._get_store(store_uri)
         resolve = functools.partial(self._key_for_pipe,
@@ -348,6 +349,7 @@ class FilesPipeline(MediaPipeline):
         self.files_result_field = settings.get(
             resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD
         )
+
         super(FilesPipeline, self).__init__(download_func=download_func, settings=settings)
 
     @classmethod
@@ -366,8 +368,8 @@ class FilesPipeline(MediaPipeline):
         gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] or None
 
         ftp_store = cls.STORE_SCHEMES['ftp']
-        ftp_store.FTP_USERNAME = settings['FTP_USER']           # Default is 'anonymous'
-        ftp_store.FTP_PASSWORD = settings['FTP_PASSWORD']       # Default is `guest`
+        ftp_store.FTP_USERNAME = settings['FTP_USER']
+        ftp_store.FTP_PASSWORD = settings['FTP_PASSWORD']
         ftp_store.USE_ACTIVE_MODE = settings.getbool('FEED_STORAGE_FTP_ACTIVE')
         
         store_uri = settings['FILES_STORE']
diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index ba94ec142..bf67b9976 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -31,4 +31,4 @@ def ftp_store_file(
     dirname, filename = posixpath.split(path)
     ftp_makedirs_cwd(ftp, dirname)
     ftp.storbinary('STOR %s' % filename, file)
-    ftp.quit()
\ No newline at end of file
+    ftp.quit()

From 97d2f717ae30f53282a9cffacc40a879989f05af Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Thu, 22 Aug 2019 16:19:01 +0200
Subject: [PATCH 2176/2974] Support extracting ftp settings in `ImagesPipeline`

---
 scrapy/pipelines/images.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index fa4d12ad1..872342fc0 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -99,6 +99,11 @@ class ImagesPipeline(FilesPipeline):
         gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
         gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] or None
 
+        ftp_store = cls.STORE_SCHEMES['ftp']
+        ftp_store.FTP_USERNAME = settings['FTP_USER']
+        ftp_store.FTP_PASSWORD = settings['FTP_PASSWORD']
+        ftp_store.USE_ACTIVE_MODE = settings.getbool('FEED_STORAGE_FTP_ACTIVE')
+
         store_uri = settings['IMAGES_STORE']
         return cls(store_uri, settings=settings)
 

From 0fa384e80defaaac88b2f0d5449072f832611431 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 22 Aug 2019 20:10:42 +0200
Subject: [PATCH 2177/2974] Provide complete API documentation coverage of
 scrapy.exporters

---
 docs/conf.py              |  4 ++++
 docs/news.rst             |  8 ++++----
 docs/topics/exporters.rst | 15 +++++++++++++--
 scrapy/exporters.py       | 18 ++++++++++++++----
 4 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index f49f79cd5..eba416cd6 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -252,6 +252,10 @@ coverage_ignore_pyobjects = [
 
     # Private exception used by the command-line interface implementation.
     r'^scrapy\.exceptions\.UsageError',
+
+    # Methods of BaseItemExporter subclasses are only documented in
+    # BaseItemExporter.
+    r'^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.',
 ]
 
 
diff --git a/docs/news.rst b/docs/news.rst
index ce5b8b406..5915ba742 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1269,8 +1269,8 @@ This 1.1 release brings a lot of interesting features and bug fixes:
     this behavior, update :setting:`ROBOTSTXT_OBEY` in ``settings.py`` file
     after creating a new project.
   - Exporters now work on unicode, instead of bytes by default (:issue:`1080`).
-    If you use ``PythonItemExporter``, you may want to update your code to
-    disable binary mode which is now deprecated.
+    If you use :class:`~scrapy.exporters.PythonItemExporter`, you may want to
+    update your code to disable binary mode which is now deprecated.
   - Accept XML node names containing dots as valid (:issue:`1533`).
   - When uploading files or images to S3 (with ``FilesPipeline`` or
     ``ImagesPipeline``), the default ACL policy is now "private" instead
@@ -1408,8 +1408,8 @@ Bugfixes
 - Fixed bug on ``XMLItemExporter`` with non-string fields in
   items (:issue:`1738`).
 - Fixed startproject command in OS X (:issue:`1635`).
-- Fixed PythonItemExporter and CSVExporter for non-string item
-  types (:issue:`1737`).
+- Fixed :class:`~scrapy.exporters.PythonItemExporter` and CSVExporter for
+  non-string item types (:issue:`1737`).
 - Various logging related fixes (:issue:`1294`, :issue:`1419`, :issue:`1263`,
   :issue:`1624`, :issue:`1654`, :issue:`1722`, :issue:`1726` and :issue:`1303`).
 - Fixed bug in ``utils.template.render_templatefile()`` (:issue:`1212`).
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index f5048d2da..11b3045ec 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -165,9 +165,9 @@ BaseItemExporter
       value unchanged except for ``unicode`` values which are encoded to
       ``str`` using the encoding declared in the :attr:`encoding` attribute.
 
-      :param field: the field being serialized. If a raw dict is being 
+      :param field: the field being serialized. If a raw dict is being
           exported (not :class:`~.Item`) *field* value is an empty dict.
-      :type field: :class:`~scrapy.item.Field` object or an empty dict 
+      :type field: :class:`~scrapy.item.Field` object or an empty dict
 
       :param name: the name of the field being serialized
       :type name: str
@@ -223,6 +223,12 @@ BaseItemExporter
       * ``indent<=0`` each item on its own line, no indentation
       * ``indent>0`` each item on its own line, indented with the provided numeric value
 
+PythonItemExporter
+------------------
+
+.. autoclass:: PythonItemExporter
+
+
 .. highlight:: none
 
 XmlItemExporter
@@ -410,3 +416,8 @@ JsonLinesItemExporter
    this exporter is well suited for serializing large amounts of data.
 
 .. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
+
+MarshalItemExporter
+-------------------
+
+.. autoclass:: MarshalItemExporter
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 695c74fec..6fc87ed18 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -276,6 +276,13 @@ class PickleItemExporter(BaseItemExporter):
 
 
 class MarshalItemExporter(BaseItemExporter):
+    """Exports items in a Python-specific binary format (see
+    :mod:`marshal`).
+
+    :param file: The file-like object to use for exporting the data. Its
+                 ``write`` method should accept :class:`bytes` (a disk file
+                 opened in binary mode, a :class:`~io.BytesIO` object, etc)
+    """
 
     def __init__(self, file, **kwargs):
         self._configure(kwargs)
@@ -297,10 +304,13 @@ class PprintItemExporter(BaseItemExporter):
 
 
 class PythonItemExporter(BaseItemExporter):
-    """The idea behind this exporter is to have a mechanism to serialize items
-    to built-in python types so any serialization library (like
-    json, msgpack, binc, etc) can be used on top of it. Its main goal is to
-    seamless support what BaseItemExporter does plus nested items.
+    """This is a base class for item exporters that extends
+    :class:`BaseItemExporter` with support for nested items.
+
+    It serializes items to built-in Python types, so that any serialization
+    library (e.g. :mod:`json` or msgpack_) can be used on top of it.
+
+    .. _msgpack: https://pypi.org/project/msgpack/
     """
     def _configure(self, options, dont_fail=False):
         self.binary = options.pop('binary', True)

From 3abe7e6e6da38b1586b720749f53955b8d00390e Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Mon, 26 Aug 2019 04:35:44 -0300
Subject: [PATCH 2178/2974] Add Bug report and Feature request templates
 (#3471)

---
 .github/ISSUE_TEMPLATE/bug_report.md      | 41 +++++++++++++++++++++++
 .github/ISSUE_TEMPLATE/feature_request.md | 33 ++++++++++++++++++
 2 files changed, 74 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md
 create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 000000000..66821171f
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,41 @@
+---
+name: Bug report
+about: Report a problem to help us improve
+---
+
+<!--
+
+Thanks for taking an interest in Scrapy!
+
+If you have a question that starts with "How to...", please see the Scrapy Community page: https://scrapy.org/community/.
+The Github issue tracker's purpose is to deal with bug reports and feature requests for the project itself.
+
+Keep in mind that by filing an issue, you are expected to comply with Scrapy's Code of Conduct, including treating everyone with respect: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
+
+The following is a suggested template to structure your issue, you can find more guidelines at https://doc.scrapy.org/en/latest/contributing.html#reporting-bugs
+
+-->
+
+### Description
+
+[Description of the issue]
+
+### Steps to Reproduce
+
+1. [First Step]
+2. [Second Step]
+3. [and so on...]
+
+**Expected behavior:** [What you expect to happen]
+
+**Actual behavior:** [What actually happens]
+
+**Reproduces how often:** [What percentage of the time does it reproduce?]
+
+### Versions
+
+Please paste here the output of executing `scrapy version --verbose` in the command line.
+
+### Additional context
+
+Any additional information, configuration, data or output from commands that might be necessary to reproduce or understand the issue. Please try not to include screenshots of code or the command line, paste the contents as text instead. You can use [GitHub Flavored Markdown](https://help.github.com/en/articles/creating-and-highlighting-code-blocks) to make the text look better.
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 000000000..df5127b4c
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,33 @@
+---
+name: Feature request
+about: Suggest an idea for an enhancement or new feature
+---
+
+<!--
+
+Thanks for taking an interest in Scrapy!
+
+If you have a question that starts with "How to...", please see the Scrapy Community page: https://scrapy.org/community/.
+The Github issue tracker's purpose is to deal with bug reports and feature requests for the project itself.
+
+Keep in mind that by filing an issue, you are expected to comply with Scrapy's Code of Conduct, including treating everyone with respect: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
+
+The following is a suggested template to structure your pull request, you can find more guidelines at https://doc.scrapy.org/en/latest/contributing.html#writing-patches and https://doc.scrapy.org/en/latest/contributing.html#submitting-patches
+
+-->
+
+## Summary
+
+One paragraph explanation of the feature.
+
+## Motivation
+
+Why are we doing this? What use cases does it support? What is the expected outcome?
+
+## Describe alternatives you've considered
+
+A clear and concise description of the alternative solutions you've considered. Be sure to explain why Scrapy's existing customizability isn't suitable for this feature.
+
+## Additional context
+
+Any additional information about the feature request here.

From 3a7b949d6d8058cdc342dba683bc054d95ed6ccd Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Tue, 27 Aug 2019 13:11:31 +0530
Subject: [PATCH 2179/2974] Adds integration with Protego robots.txt parser
 (#3935)

---
 docs/topics/downloader-middleware.rst | 20 ++++++++-
 scrapy/robotstxt.py                   | 58 +++++++++++++++++----------
 tests/test_robotstxt_interface.py     | 21 +++++++++-
 tox.ini                               |  2 +
 4 files changed, 77 insertions(+), 24 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 616b56101..6aa714fb2 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -996,6 +996,7 @@ RobotsTxtMiddleware
     * :ref:`RobotFileParser <python-robotfileparser>` (default)
     * :ref:`Reppy <reppy-parser>`
     * :ref:`Robotexclusionrulesparser <rerp-parser>`
+    * :ref:`Protego <protego-parser>`
 
     You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
     setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
@@ -1013,7 +1014,7 @@ RobotFileParser
 ~~~~~~~~~~~~~~~
 
 `RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ is 
-Python's inbuilt ``robots.txt`` parser. The parser is fully compliant with `Martijn Koster's 
+Python's inbuilt robots.txt_ parser. The parser is fully compliant with `Martijn Koster's 
 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_. It lacks
 support for wildcard matching. Scrapy uses this parser by default.
 
@@ -1059,6 +1060,23 @@ In order to use this parser:
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.ReppyRobotParser``
 
+.. _protego-parser:
+
+Protego parser
+~~~~~~~~~~~~~~
+
+`Protego <https://github.com/scrapy/protego>`_ is a pure-Python robots.txt_ parser.
+The parser is fully compliant with `Google's Robots.txt Specification
+<https://developers.google.com/search/reference/robots_txt>`_ hence supports wildcard 
+matching, and uses the length based rule similar to `Reppy <https://github.com/seomoz/reppy/>`_.
+
+In order to use this parser:
+
+* Install `Protego <https://github.com/scrapy/protego>`_ by running ``pip install protego``
+
+* Set :setting:`ROBOTSTXT_PARSER` setting to
+  ``scrapy.robotstxt.ProtegoRobotParser``
+
 .. _support-for-new-robots-parser:
 
 Implementing support for a new parser
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 4bfb275fd..189f165d1 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -7,6 +7,21 @@ from scrapy.utils.python import to_native_str, to_unicode
 
 logger = logging.getLogger(__name__)
 
+def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
+    try:
+        if to_native_str_type:
+            robotstxt_body = to_native_str(robotstxt_body)
+        else:
+            robotstxt_body = robotstxt_body.decode('utf-8')
+    except UnicodeDecodeError:
+        # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
+        # Switch to 'allow all' state.
+        logger.warning("Failure while parsing robots.txt. "
+                       "File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
+                       exc_info=sys.exc_info(),
+                       extra={'spider': spider})
+        robotstxt_body = ''
+    return robotstxt_body
 
 class RobotParser(with_metaclass(ABCMeta)):
     @classmethod
@@ -40,17 +55,7 @@ class PythonRobotParser(RobotParser):
     def __init__(self, robotstxt_body, spider):
         from six.moves.urllib_robotparser import RobotFileParser
         self.spider = spider
-        try:
-            robotstxt_body = to_native_str(robotstxt_body)
-        except UnicodeDecodeError:
-            # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
-            # Switch to 'allow all' state.
-            logger.warning("Failure while parsing robots.txt using %(parser)s."
-                           " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
-                           {'parser': "RobotFileParser"},
-                           exc_info=sys.exc_info(),
-                           extra={'spider': self.spider})
-            robotstxt_body = ''
+        robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
         self.rp = RobotFileParser()
         self.rp.parse(robotstxt_body.splitlines())
 
@@ -87,17 +92,7 @@ class RerpRobotParser(RobotParser):
         from robotexclusionrulesparser import RobotExclusionRulesParser
         self.spider = spider
         self.rp = RobotExclusionRulesParser()
-        try:
-            robotstxt_body = robotstxt_body.decode('utf-8')
-        except UnicodeDecodeError:
-            # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
-            # Switch to 'allow all' state.
-            logger.warning("Failure while parsing robots.txt using %(parser)s."
-                           " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
-                           {'parser': "RobotExclusionRulesParser"},
-                           exc_info=sys.exc_info(),
-                           extra={'spider': self.spider})
-            robotstxt_body = ''
+        robotstxt_body = decode_robotstxt(robotstxt_body, spider)
         self.rp.parse(robotstxt_body)
 
     @classmethod
@@ -110,3 +105,22 @@ class RerpRobotParser(RobotParser):
         user_agent = to_unicode(user_agent)
         url = to_unicode(url)
         return self.rp.is_allowed(user_agent, url)
+
+
+class ProtegoRobotParser(RobotParser):
+    def __init__(self, robotstxt_body, spider):
+        from protego import Protego
+        self.spider = spider
+        robotstxt_body = decode_robotstxt(robotstxt_body, spider)
+        self.rp = Protego.parse(robotstxt_body)
+
+    @classmethod
+    def from_crawler(cls, crawler, robotstxt_body):
+        spider = None if not crawler else crawler.spider
+        o = cls(robotstxt_body, spider)
+        return o
+
+    def allowed(self, url, user_agent):
+        user_agent = to_unicode(user_agent)
+        url = to_unicode(url)
+        return self.rp.can_fetch(url, user_agent)
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 2819786b5..9aaab560a 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -20,6 +20,13 @@ def rerp_available():
         return False
     return True
 
+def protego_available():
+    # check if protego parser is installed
+    try:
+        from protego import Protego
+    except ImportError:
+        return False
+    return True
 
 class BaseRobotParserTest:
     def _setUp(self, parser_cls):
@@ -127,7 +134,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
         super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser)
 
     def test_order_based_precedence(self):
-        raise unittest.SkipTest("Rerp does not support order based directives precedence.")
+        raise unittest.SkipTest("Reppy does not support order based directives precedence.")
 
 
 class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
@@ -140,3 +147,15 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
 
     def test_length_based_precedence(self):
         raise unittest.SkipTest("Rerp does not support length based directives precedence.")
+
+
+class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase):
+    if not protego_available():
+        skip = "Protego parser is not installed"
+
+    def setUp(self):
+        from scrapy.robotstxt import ProtegoRobotParser
+        super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser)
+
+    def test_order_based_precedence(self):
+        raise unittest.SkipTest("Protego does not support order based directives precedence.")
diff --git a/tox.ini b/tox.ini
index c3502c2ca..cc845faf1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -125,6 +125,7 @@ deps =
     {[testenv:py35]deps}
     reppy
     robotexclusionrulesparser
+    protego
 
 [testenv:py27-extra-deps]
 basepython = python2.7
@@ -132,3 +133,4 @@ deps =
     {[testenv]deps}
     reppy
     robotexclusionrulesparser
+    protego
\ No newline at end of file

From ad824a264bfc69ec600c90ca380745b45da3dfe6 Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Tue, 27 Aug 2019 18:30:11 +0530
Subject: [PATCH 2180/2974] fixes a link in doc

---
 docs/topics/settings.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0cb81c43e..4cb76412e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -952,7 +952,7 @@ LOGSTATS_INTERVAL
 Default: ``60.0``
 
 The interval (in seconds) between each logging printout of the stats 
-by :class:`~extensions.logstats.LogStats`.
+by :class:`~scrapy.extensions.logstats.LogStats`.
 
 .. setting:: MEMDEBUG_ENABLED
 

From 77c8ab2e62f25496acfd812381d84aac369fd3b8 Mon Sep 17 00:00:00 2001
From: Anubhav Patel <anubhavp28@gmail.com>
Date: Tue, 27 Aug 2019 18:44:08 +0530
Subject: [PATCH 2181/2974] makes suggested changes

---
 docs/topics/downloader-middleware.rst | 21 ++++++---------------
 docs/topics/settings.rst              |  3 ++-
 2 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index ae413dc84..93c91f18a 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -989,6 +989,12 @@ RobotsTxtMiddleware
     To make sure Scrapy respects robots.txt make sure the middleware is enabled
     and the :setting:`ROBOTSTXT_OBEY` setting is enabled.
 
+    The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the
+    user agent string to use for matching in the robots.txt_ file. If it
+    is ``None``, the User-Agent header you are sending with the request or the
+    :setting:`USER_AGENT` setting (in that order) will be used for determining
+    the user agent to use in the robots.txt_ file.
+
     This middleware has to be combined with a robots.txt_ parser.
 
     Scrapy ships with support for the following robots.txt_ parsers:
@@ -1074,21 +1080,6 @@ implementing the methods described below.
 .. autoclass:: RobotParser
    :members:
 
-RobotsTxtMiddleware Settings
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. setting:: ROBOTSTXT_USER_AGENT
-
-ROBOTSTXT_USER_AGENT
-^^^^^^^^^^^^^^^^^^^^
-
-Default: ``None``
-
-The user agent string to use for matching in the robots.txt_ file. If ``None``,
-the User-Agent header you are sending with the request or the
-:setting:`USER_AGENT` setting (in that order) will be used for determining
-the user agent to use in the robots.txt_ file.
-
 .. _robots.txt: http://www.robotstxt.org/
 
 DownloaderStats
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index c4b55cc7b..d3ad777f2 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1422,7 +1422,8 @@ USER_AGENT
 Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
 
 The default User-Agent to use when crawling, unless overridden. This user agent is
-also used in robots.txt if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
+also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`
+if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
 there is no overridding User-Agent header specified for the request.
 
 

From b6b76df0574d65e053eb2e0aba5dd757a4907c24 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 28 Aug 2019 18:28:31 -0300
Subject: [PATCH 2182/2974] CallbackKeywordArgumentsContract

---
 scrapy/contracts/__init__.py        |  6 +++---
 scrapy/contracts/default.py         | 16 ++++++++++++++++
 scrapy/settings/default_settings.py |  5 +++--
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 536bbdafb..b3f02a291 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -92,7 +92,7 @@ class ContractsManager(object):
         @wraps(cb)
         def cb_wrapper(response):
             try:
-                output = cb(response)
+                output = cb(response, **request.cb_kwargs)
                 output = list(iterate_spider_output(output))
             except Exception:
                 case = _create_testcase(method, 'callback')
@@ -133,7 +133,7 @@ class Contract(object):
                 else:
                     results.addSuccess(self.testcase_pre)
                 finally:
-                    return list(iterate_spider_output(cb(response)))
+                    return list(iterate_spider_output(cb(response, **request.cb_kwargs)))
 
             request.callback = wrapper
 
@@ -145,7 +145,7 @@ class Contract(object):
 
             @wraps(cb)
             def wrapper(response):
-                output = list(iterate_spider_output(cb(response)))
+                output = list(iterate_spider_output(cb(response, **request.cb_kwargs)))
                 try:
                     results.startTest(self.testcase_post)
                     self.post_process(output)
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 7745959a7..24f6c2e77 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -1,3 +1,5 @@
+import json
+
 from scrapy.item import BaseItem
 from scrapy.http import Request
 from scrapy.exceptions import ContractFail
@@ -18,6 +20,20 @@ class UrlContract(Contract):
         return args
 
 
+class CallbackKeywordArgumentsContract(Contract):
+    """ Contract to set the keyword arguments for the request.
+        The value should be a JSON-encoded dictionary, e.g.:
+
+        @cb_kwargs {"arg1": "some value"}
+    """
+
+    name = 'cb_kwargs'
+
+    def adjust_request_args(self, args):
+        args['cb_kwargs'] = json.loads(' '.join(self.args))
+        return args
+
+
 class ReturnsContract(Contract):
     """ Contract to check the output of a callback
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 5884dfc60..52a701edf 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -291,6 +291,7 @@ TELNETCONSOLE_PASSWORD = None
 SPIDER_CONTRACTS = {}
 SPIDER_CONTRACTS_BASE = {
     'scrapy.contracts.default.UrlContract': 1,
-    'scrapy.contracts.default.ReturnsContract': 2,
-    'scrapy.contracts.default.ScrapesContract': 3,
+    'scrapy.contracts.default.CallbackKeywordArgumentsContract': 2,
+    'scrapy.contracts.default.ReturnsContract': 3,
+    'scrapy.contracts.default.ScrapesContract': 4,
 }

From 97a7d775f7021e485a88d4becb363c6e49ca3bb4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 29 Aug 2019 10:51:16 -0300
Subject: [PATCH 2183/2974] Aplly suggestions by @victor-torres

---
 scrapy/contracts/__init__.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index b3f02a291..7b6591d86 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -90,9 +90,9 @@ class ContractsManager(object):
         cb = request.callback
 
         @wraps(cb)
-        def cb_wrapper(response):
+        def cb_wrapper(response, **cb_kwargs):
             try:
-                output = cb(response, **request.cb_kwargs)
+                output = cb(response, **cb_kwargs)
                 output = list(iterate_spider_output(output))
             except Exception:
                 case = _create_testcase(method, 'callback')
@@ -121,7 +121,7 @@ class Contract(object):
             cb = request.callback
 
             @wraps(cb)
-            def wrapper(response):
+            def wrapper(response, **cb_kwargs):
                 try:
                     results.startTest(self.testcase_pre)
                     self.pre_process(response)
@@ -133,7 +133,7 @@ class Contract(object):
                 else:
                     results.addSuccess(self.testcase_pre)
                 finally:
-                    return list(iterate_spider_output(cb(response, **request.cb_kwargs)))
+                    return list(iterate_spider_output(cb(response, **cb_kwargs)))
 
             request.callback = wrapper
 
@@ -144,8 +144,8 @@ class Contract(object):
             cb = request.callback
 
             @wraps(cb)
-            def wrapper(response):
-                output = list(iterate_spider_output(cb(response, **request.cb_kwargs)))
+            def wrapper(response, **cb_kwargs):
+                output = list(iterate_spider_output(cb(response, **cb_kwargs)))
                 try:
                     results.startTest(self.testcase_post)
                     self.post_process(output)

From 110bc92e6b9c9c3ce1775a9ca1487df42a80d219 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Thu, 29 Aug 2019 11:10:00 -0300
Subject: [PATCH 2184/2974] Fix default value of FEED_STORAGE_GCS_ACL

---
 scrapy/extensions/feedexport.py     |  2 +-
 scrapy/settings/default_settings.py |  2 +-
 tests/test_feedexport.py            | 18 ++++++++++++++++++
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index fbbf9bb97..1e982c684 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -160,7 +160,7 @@ class GCSFeedStorage(BlockingFeedStorage):
         return cls(
             uri,
             crawler.settings['GCS_PROJECT_ID'],
-            crawler.settings['FEED_STORAGE_GCS_ACL']
+            crawler.settings['FEED_STORAGE_GCS_ACL'] or None
         )
 
     def _store_in_thread(self, file):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 50fcd1d0a..45257a61c 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -159,7 +159,7 @@ FEED_EXPORTERS_BASE = {
 }
 FEED_EXPORT_INDENT = 0
 
-FEED_STORAGE_GCS_ACL = None
+FEED_STORAGE_GCS_ACL = ''
 
 FILES_STORE_S3_ACL = 'private'
 FILES_STORE_GCS_ACL = ''
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 41df7d7af..69f144d07 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -206,6 +206,24 @@ class GCSFeedStorageTest(unittest.TestCase):
         assert storage.bucket_name == 'mybucket'
         assert storage.blob_name == 'export.csv'
 
+    @mock.patch('scrapy.conf.settings',
+                new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': '' }, create=True)
+    def test_parse_empty_acl(self):
+        try:
+            from google.cloud.storage import Client
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': '' }
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+        assert storage.acl is None
+
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None }
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+        assert storage.acl is None
+
     @defer.inlineCallbacks
     def test_store(self):
         try:

From eb0bd2daef8d91e6386390975b2e75d1a93b3161 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 29 Aug 2019 14:01:13 -0300
Subject: [PATCH 2185/2974] Revert backward-incompatible change (contract
 priorities)

---
 scrapy/settings/default_settings.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 52a701edf..05ab4b628 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -291,7 +291,7 @@ TELNETCONSOLE_PASSWORD = None
 SPIDER_CONTRACTS = {}
 SPIDER_CONTRACTS_BASE = {
     'scrapy.contracts.default.UrlContract': 1,
-    'scrapy.contracts.default.CallbackKeywordArgumentsContract': 2,
-    'scrapy.contracts.default.ReturnsContract': 3,
-    'scrapy.contracts.default.ScrapesContract': 4,
+    'scrapy.contracts.default.CallbackKeywordArgumentsContract': 1,
+    'scrapy.contracts.default.ReturnsContract': 2,
+    'scrapy.contracts.default.ScrapesContract': 3,
 }

From ace2df3d140ac783c622ea5403876fe76af2bb72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Fri, 30 Aug 2019 11:03:44 +0200
Subject: [PATCH 2186/2974] Fix JSONRequest naming (#3982)

---
 docs/news.rst                       |  4 ++--
 docs/topics/request-response.rst    | 14 +++++++-------
 scrapy/http/__init__.py             |  2 +-
 scrapy/http/request/json_request.py | 12 ++++++++----
 tests/test_http_request.py          | 10 +++++-----
 5 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 5915ba742..aac750601 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -80,8 +80,8 @@ New features
     provides a cleaner way to pass keyword arguments to callback methods
     (:issue:`1138`, :issue:`3563`)
 
-*   A new :class:`~scrapy.http.JSONRequest` class offers a more convenient way
-    to build JSON requests (:issue:`3504`, :issue:`3505`)
+*   A new :class:`JSONRequest <scrapy.http.JsonRequest>` class offers a more
+    convenient way to build JSON requests (:issue:`3504`, :issue:`3505`)
 
 *   A ``process_request`` callback passed to the :class:`~scrapy.spiders.Rule`
     constructor now receives the :class:`~scrapy.http.Response` object that
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 9a5c65b0d..ad1b9af10 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -535,19 +535,19 @@ method for this job. Here's an example spider which uses it::
 
             # continue scraping with authenticated session...
 
-JSONRequest
+JsonRequest
 -----------
 
-The JSONRequest class extends the base :class:`Request` class with functionality for
+The JsonRequest class extends the base :class:`Request` class with functionality for
 dealing with JSON requests.
 
-.. class:: JSONRequest(url, [... data, dumps_kwargs])
+.. class:: JsonRequest(url, [... data, dumps_kwargs])
 
-   The :class:`JSONRequest` class adds two new argument to the constructor. The
+   The :class:`JsonRequest` class adds two new argument to the constructor. The
    remaining arguments are the same as for the :class:`Request` class and are
    not documented here.
 
-   Using the :class:`JSONRequest` will set the ``Content-Type`` header to ``application/json``
+   Using the :class:`JsonRequest` will set the ``Content-Type`` header to ``application/json``
    and ``Accept`` header to ``application/json, text/javascript, */*; q=0.01``
 
    :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body.
@@ -562,7 +562,7 @@ dealing with JSON requests.
 
 .. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps
 
-JSONRequest usage example
+JsonRequest usage example
 -------------------------
 
 Sending a JSON POST request with a JSON payload::
@@ -571,7 +571,7 @@ Sending a JSON POST request with a JSON payload::
        'name1': 'value1',
        'name2': 'value2',
    }
-   yield JSONRequest(url='http://www.example.com/post/action', data=data)
+   yield JsonRequest(url='http://www.example.com/post/action', data=data)
 
 
 Response objects
diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py
index 4b2f7b33f..e6c58e1f1 100644
--- a/scrapy/http/__init__.py
+++ b/scrapy/http/__init__.py
@@ -10,7 +10,7 @@ from scrapy.http.headers import Headers
 from scrapy.http.request import Request
 from scrapy.http.request.form import FormRequest
 from scrapy.http.request.rpc import XmlRpcRequest
-from scrapy.http.request.json_request import JSONRequest
+from scrapy.http.request.json_request import JsonRequest
 
 from scrapy.http.response import Response
 from scrapy.http.response.html import HtmlResponse
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 8f7a61a6d..f08b25280 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -1,5 +1,5 @@
 """
-This module implements the JSONRequest class which is a more convenient class
+This module implements the JsonRequest class which is a more convenient class
 (than Request) to generate JSON Requests.
 
 See documentation in docs/topics/request-response.rst
@@ -10,9 +10,10 @@ import json
 import warnings
 
 from scrapy.http.request import Request
+from scrapy.utils.deprecate import create_deprecated_class
 
 
-class JSONRequest(Request):
+class JsonRequest(Request):
     def __init__(self, *args, **kwargs):
         dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {}))
         dumps_kwargs.setdefault('sort_keys', True)
@@ -31,7 +32,7 @@ class JSONRequest(Request):
             if 'method' not in kwargs:
                 kwargs['method'] = 'POST'
 
-        super(JSONRequest, self).__init__(*args, **kwargs)
+        super(JsonRequest, self).__init__(*args, **kwargs)
         self.headers.setdefault('Content-Type', 'application/json')
         self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
 
@@ -46,8 +47,11 @@ class JSONRequest(Request):
         elif not body_passed and data_passed:
             kwargs['body'] = self._dumps(data)
 
-        return super(JSONRequest, self).replace(*args, **kwargs)
+        return super(JsonRequest, self).replace(*args, **kwargs)
 
     def _dumps(self, data):
         """Convert to JSON """
         return json.dumps(data, **self._dumps_kwargs)
+
+
+JSONRequest = create_deprecated_class("JSONRequest", JsonRequest)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 60494d792..16d7a1cb8 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -11,7 +11,7 @@ from six.moves.urllib.parse import urlparse, parse_qs, unquote
 if six.PY3:
     from urllib.parse import unquote_to_bytes
 
-from scrapy.http import Request, FormRequest, XmlRpcRequest, JSONRequest, Headers, HtmlResponse
+from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_native_str
 
 from tests import mock
@@ -1246,14 +1246,14 @@ class XmlRpcRequestTest(RequestTest):
         self._test_request(params=(u'pas£',), encoding='latin1')
 
 
-class JSONRequestTest(RequestTest):
-    request_class = JSONRequest
+class JsonRequestTest(RequestTest):
+    request_class = JsonRequest
     default_method = 'GET'
     default_headers = {b'Content-Type': [b'application/json'], b'Accept': [b'application/json, text/javascript, */*; q=0.01']}
 
     def setUp(self):
         warnings.simplefilter("always")
-        super(JSONRequestTest, self).setUp()
+        super(JsonRequestTest, self).setUp()
 
     def test_data(self):
         r1 = self.request_class(url="http://www.example.com/")
@@ -1407,7 +1407,7 @@ class JSONRequestTest(RequestTest):
 
     def tearDown(self):
         warnings.resetwarnings()
-        super(JSONRequestTest, self).tearDown()
+        super(JsonRequestTest, self).tearDown()
 
 
 if __name__ == "__main__":

From 2828cb769f0618d3986cd12c2ef0f8143f31bfba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 30 Aug 2019 14:29:15 +0200
Subject: [PATCH 2187/2974] Provide complete API documentation coverage of
 scrapy.extensions

---
 docs/conf.py                          |   6 +
 docs/topics/downloader-middleware.rst | 153 +++++++++++++-------------
 docs/topics/extensions.rst            |  14 +--
 3 files changed, 90 insertions(+), 83 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index eba416cd6..fa257dead 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -256,6 +256,12 @@ coverage_ignore_pyobjects = [
     # Methods of BaseItemExporter subclasses are only documented in
     # BaseItemExporter.
     r'^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.',
+
+    # Extension behavior is only modified through settings. Methods of
+    # extension classes, as well as helper functions, are implementation
+    # details that are not documented.
+    r'^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.',  # methods
+    r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]',  # helper functions
 ]
 
 
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index c25e12bda..0845ef6e4 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -365,24 +365,25 @@ HttpCacheMiddleware
 
     You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals ``True``.
 
+.. module:: scrapy.extensions.httpcache
+   :noindex:
+
 .. _httpcache-policy-dummy:
 
 Dummy policy (default)
 ~~~~~~~~~~~~~~~~~~~~~~
 
-This policy has no awareness of any HTTP Cache-Control directives.
-Every request and its corresponding response are cached.  When the same
-request is seen again, the response is returned without transferring
-anything from the Internet.
+.. class:: DummyPolicy
 
-The Dummy policy is useful for testing spiders faster (without having
-to wait for downloads every time) and for trying your spider offline,
-when an Internet connection is not available. The goal is to be able to
-"replay" a spider run *exactly as it ran before*.
+    This policy has no awareness of any HTTP Cache-Control directives.
+    Every request and its corresponding response are cached.  When the same
+    request is seen again, the response is returned without transferring
+    anything from the Internet.
 
-In order to use this policy, set:
-
-* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.DummyPolicy``
+    The Dummy policy is useful for testing spiders faster (without having
+    to wait for downloads every time) and for trying your spider offline,
+    when an Internet connection is not available. The goal is to be able to
+    "replay" a spider run *exactly as it ran before*.
 
 
 .. _httpcache-policy-rfc2616:
@@ -390,45 +391,44 @@ In order to use this policy, set:
 RFC2616 policy
 ~~~~~~~~~~~~~~
 
-This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP
-Cache-Control awareness, aimed at production and used in continuous
-runs to avoid downloading unmodified data (to save bandwidth and speed up crawls).
+.. class:: RFC2616Policy
 
-what is implemented:
+    This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP
+    Cache-Control awareness, aimed at production and used in continuous
+    runs to avoid downloading unmodified data (to save bandwidth and speed up
+    crawls).
 
-* Do not attempt to store responses/requests with ``no-store`` cache-control directive set
-* Do not serve responses from cache if ``no-cache`` cache-control directive is set even for fresh responses
-* Compute freshness lifetime from ``max-age`` cache-control directive
-* Compute freshness lifetime from ``Expires`` response header
-* Compute freshness lifetime from ``Last-Modified`` response header (heuristic used by Firefox)
-* Compute current age from ``Age`` response header
-* Compute current age from ``Date`` header
-* Revalidate stale responses based on ``Last-Modified`` response header
-* Revalidate stale responses based on ``ETag`` response header
-* Set ``Date`` header for any received response missing it
-* Support ``max-stale`` cache-control directive in requests
+    What is implemented:
 
-  This allows spiders to be configured with the full RFC2616 cache policy,
-  but avoid revalidation on a request-by-request basis, while remaining
-  conformant with the HTTP spec.
+    * Do not attempt to store responses/requests with ``no-store`` cache-control directive set
+    * Do not serve responses from cache if ``no-cache`` cache-control directive is set even for fresh responses
+    * Compute freshness lifetime from ``max-age`` cache-control directive
+    * Compute freshness lifetime from ``Expires`` response header
+    * Compute freshness lifetime from ``Last-Modified`` response header (heuristic used by Firefox)
+    * Compute current age from ``Age`` response header
+    * Compute current age from ``Date`` header
+    * Revalidate stale responses based on ``Last-Modified`` response header
+    * Revalidate stale responses based on ``ETag`` response header
+    * Set ``Date`` header for any received response missing it
+    * Support ``max-stale`` cache-control directive in requests
 
-  Example:
+    This allows spiders to be configured with the full RFC2616 cache policy,
+    but avoid revalidation on a request-by-request basis, while remaining
+    conformant with the HTTP spec.
 
-  Add ``Cache-Control: max-stale=600`` to Request headers to accept responses that
-  have exceeded their expiration time by no more than 600 seconds.
+    Example:
 
-  See also: RFC2616, 14.9.3
+    Add ``Cache-Control: max-stale=600`` to Request headers to accept responses that
+    have exceeded their expiration time by no more than 600 seconds.
 
-what is missing:
+    See also: RFC2616, 14.9.3
 
-* ``Pragma: no-cache`` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
-* ``Vary`` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6
-* Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10
-* ... probably others ..
+    What is missing:
 
-In order to use this policy, set:
-
-* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.RFC2616Policy``
+    * ``Pragma: no-cache`` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
+    * ``Vary`` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6
+    * Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10
+    * ... probably others ..
 
 
 .. _httpcache-storage-fs:
@@ -436,67 +436,68 @@ In order to use this policy, set:
 Filesystem storage backend (default)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-File system storage backend is available for the HTTP cache middleware.
+.. class:: FilesystemCacheStorage
 
-In order to use this storage backend, set:
+    File system storage backend is available for the HTTP cache middleware.
 
-* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.FilesystemCacheStorage``
+    Each request/response pair is stored in a different directory containing
+    the following files:
 
-Each request/response pair is stored in a different directory containing
-the following files:
+    *   ``request_body`` - the plain request body
 
- * ``request_body`` - the plain request body
- * ``request_headers`` - the request headers (in raw HTTP format)
- * ``response_body`` - the plain response body
- * ``response_headers`` - the request headers (in raw HTTP format)
- * ``meta`` - some metadata of this cache resource in Python ``repr()`` format
-   (grep-friendly format)
- * ``pickled_meta`` - the same metadata in ``meta`` but pickled for more
-   efficient deserialization
+    *   ``request_headers`` - the request headers (in raw HTTP format)
 
-The directory name is made from the request fingerprint (see
-``scrapy.utils.request.fingerprint``), and one level of subdirectories is
-used to avoid creating too many files into the same directory (which is
-inefficient in many file systems). An example directory could be::
+    *   ``response_body`` - the plain response body
 
-   /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7
+    *   ``response_headers`` - the request headers (in raw HTTP format)
+
+    *   ``meta`` - some metadata of this cache resource in Python ``repr()``
+        format (grep-friendly format)
+
+    *   ``pickled_meta`` - the same metadata in ``meta`` but pickled for more
+        efficient deserialization
+
+    The directory name is made from the request fingerprint (see
+    ``scrapy.utils.request.fingerprint``), and one level of subdirectories is
+    used to avoid creating too many files into the same directory (which is
+    inefficient in many file systems). An example directory could be::
+
+        /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7
 
 .. _httpcache-storage-dbm:
 
 DBM storage backend
 ~~~~~~~~~~~~~~~~~~~
 
-.. versionadded:: 0.13
+.. class:: DbmCacheStorage
 
-A DBM_ storage backend is also available for the HTTP cache middleware.
+    .. versionadded:: 0.13
 
-By default, it uses the anydbm_ module, but you can change it with the
-:setting:`HTTPCACHE_DBM_MODULE` setting.
+    A DBM_ storage backend is also available for the HTTP cache middleware.
 
-In order to use this storage backend, set:
-
-* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.DbmCacheStorage``
+    By default, it uses the anydbm_ module, but you can change it with the
+    :setting:`HTTPCACHE_DBM_MODULE` setting.
 
 .. _httpcache-storage-leveldb:
 
 LevelDB storage backend
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-.. versionadded:: 0.23
+.. class:: LeveldbCacheStorage
 
-A LevelDB_ storage backend is also available for the HTTP cache middleware.
+    .. versionadded:: 0.23
 
-This backend is not recommended for development because only one process can
-access LevelDB databases at the same time, so you can't run a crawl and open
-the scrapy shell in parallel for the same spider.
+    A LevelDB_ storage backend is also available for the HTTP cache middleware.
 
-In order to use this storage backend:
+    This backend is not recommended for development because only one process
+    can access LevelDB databases at the same time, so you can't run a crawl and
+    open the scrapy shell in parallel for the same spider.
 
-* set :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.LeveldbCacheStorage``
-* install `LevelDB python bindings`_ like ``pip install leveldb``
+    In order to use this storage backend, install the `LevelDB python
+    bindings`_ (e.g. ``pip install leveldb``).
 
-.. _LevelDB: https://github.com/google/leveldb
-.. _leveldb python bindings: https://pypi.python.org/pypi/leveldb
+    .. _LevelDB: https://github.com/google/leveldb
+    .. _leveldb python bindings: https://pypi.python.org/pypi/leveldb
 
 .. _httpcache-storage-custom:
 
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index d6e7452a1..72c2290b5 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -183,7 +183,7 @@ Telnet console extension
 .. module:: scrapy.extensions.telnet
    :synopsis: Telnet console
 
-.. class:: scrapy.extensions.telnet.TelnetConsole
+.. class:: TelnetConsole
 
 Provides a telnet console for getting into a Python interpreter inside the
 currently running Scrapy process, which can be very useful for debugging.
@@ -200,7 +200,7 @@ Memory usage extension
 .. module:: scrapy.extensions.memusage
    :synopsis: Memory usage extension
 
-.. class:: scrapy.extensions.memusage.MemoryUsage
+.. class:: MemoryUsage
 
 .. note:: This extension does not work in Windows.
 
@@ -228,7 +228,7 @@ Memory debugger extension
 .. module:: scrapy.extensions.memdebug
    :synopsis: Memory debugger extension
 
-.. class:: scrapy.extensions.memdebug.MemoryDebugger
+.. class:: MemoryDebugger
 
 An extension for debugging memory usage. It collects information about:
 
@@ -244,7 +244,7 @@ Close spider extension
 .. module:: scrapy.extensions.closespider
    :synopsis: Close spider extension
 
-.. class:: scrapy.extensions.closespider.CloseSpider
+.. class:: CloseSpider
 
 Closes a spider automatically when some conditions are met, using a specific
 closing reason for each condition.
@@ -317,7 +317,7 @@ StatsMailer extension
 .. module:: scrapy.extensions.statsmailer
    :synopsis: StatsMailer extension
 
-.. class:: scrapy.extensions.statsmailer.StatsMailer
+.. class:: StatsMailer
 
 This simple extension can be used to send a notification e-mail every time a
 domain has finished scraping, including the Scrapy stats collected. The email
@@ -333,7 +333,7 @@ Debugging extensions
 Stack trace dump extension
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. class:: scrapy.extensions.debug.StackTraceDump
+.. class:: StackTraceDump
 
 Dumps information about the running process when a `SIGQUIT`_ or `SIGUSR2`_
 signal is received. The information dumped is the following:
@@ -362,7 +362,7 @@ There are at least two ways to send Scrapy the `SIGQUIT`_ signal:
 Debugger extension
 ~~~~~~~~~~~~~~~~~~
 
-.. class:: scrapy.extensions.debug.Debugger
+.. class:: Debugger
 
 Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_
 signal is received. After the debugger is exited, the Scrapy process continues

From 2061f2a382b136382e18d6302fcd53053654e1ce Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 31 Aug 2019 02:10:18 -0300
Subject: [PATCH 2188/2974] [doc] cb_kwargs contract

---
 docs/topics/contracts.rst | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 957761b76..62f9a743b 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -35,12 +35,20 @@ This callback is tested using three built-in contracts:
 
 .. class:: UrlContract
 
-    This contract (``@url``) sets the sample url used when checking other
+    This contract (``@url``) sets the sample URL used when checking other
     contract conditions for this spider. This contract is mandatory. All
     callbacks lacking this contract are ignored when running the checks::
 
     @url url
 
+.. class:: CallbackKeywordArgumentsContract
+
+    This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.http.Request.cb_kwargs>`
+    attribute for the sample request. It must be a valid JSON dictionary.
+    ::
+
+    @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
+
 .. class:: ReturnsContract
 
     This contract (``@returns``) sets lower and upper bounds for the items and

From b92b1146335f6b651ba8e6e56e579a6844ec3ebd Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 31 Aug 2019 02:44:09 -0300
Subject: [PATCH 2189/2974] [test] cb_kwargs contract

---
 tests/test_contracts.py | 78 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)

diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index a728099c0..b2e358700 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -14,6 +14,7 @@ from scrapy.item import Item, Field
 from scrapy.contracts import ContractsManager, Contract
 from scrapy.contracts.default import (
     UrlContract,
+    CallbackKeywordArgumentsContract,
     ReturnsContract,
     ScrapesContract,
 )
@@ -70,6 +71,37 @@ class TestSpider(Spider):
         """
         return TestItem(url=response.url)
 
+    def returns_request_cb_kwargs(self, response, url):
+        """ method which returns request
+        @url https://example.org
+        @cb_kwargs {"url": "http://scrapy.org"}
+        @returns requests 1
+        """
+        return Request(url, callback=self.returns_item_cb_kwargs)
+
+    def returns_item_cb_kwargs(self, response, name):
+        """ method which returns item
+        @url http://scrapy.org
+        @cb_kwargs {"name": "Scrapy"}
+        @returns items 1 1
+        """
+        return TestItem(name=name, url=response.url)
+
+    def returns_item_cb_kwargs_error_unexpected_keyword(self, response):
+        """ method which returns item
+        @url http://scrapy.org
+        @cb_kwargs {"arg": "value"}
+        @returns items 1 1
+        """
+        return TestItem(url=response.url)
+
+    def returns_item_cb_kwargs_error_missing_argument(self, response, arg):
+        """ method which returns item
+        @url http://scrapy.org
+        @returns items 1 1
+        """
+        return TestItem(url=response.url)
+
     def returns_dict_item(self, response):
         """ method which returns item
         @url http://scrapy.org
@@ -172,6 +204,7 @@ class InheritsTestSpider(TestSpider):
 class ContractsManagerTest(unittest.TestCase):
     contracts = [
         UrlContract,
+        CallbackKeywordArgumentsContract,
         ReturnsContract,
         ScrapesContract,
         CustomFormContract,
@@ -211,6 +244,51 @@ class ContractsManagerTest(unittest.TestCase):
         request = self.conman.from_method(spider.parse_no_url, self.results)
         self.assertEqual(request, None)
 
+    def test_cb_kwargs(self):
+        spider = TestSpider()
+        response = ResponseMock()
+
+        # extract contracts correctly
+        contracts = self.conman.extract_contracts(spider.returns_request_cb_kwargs)
+        self.assertEqual(len(contracts), 3)
+        self.assertEqual(frozenset(type(x) for x in contracts),
+                         frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))
+        
+        contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs)
+        self.assertEqual(len(contracts), 3)
+        self.assertEqual(frozenset(type(x) for x in contracts),
+                         frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))
+
+        contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs_error_unexpected_keyword)
+        self.assertEqual(len(contracts), 3)
+        self.assertEqual(frozenset(type(x) for x in contracts),
+                         frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))
+
+        contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs_error_missing_argument)
+        self.assertEqual(len(contracts), 2)
+        self.assertEqual(frozenset(type(x) for x in contracts),
+                         frozenset([UrlContract, ReturnsContract]))
+
+        # returns_request
+        request = self.conman.from_method(spider.returns_request_cb_kwargs, self.results)
+        request.callback(response, **request.cb_kwargs)
+        self.should_succeed()
+
+        # returns_item
+        request = self.conman.from_method(spider.returns_item_cb_kwargs, self.results)
+        request.callback(response, **request.cb_kwargs)
+        self.should_succeed()
+
+        # returns_item (error, callback doesn't take keyword arguments)
+        request = self.conman.from_method(spider.returns_item_cb_kwargs_error_unexpected_keyword, self.results)
+        request.callback(response, **request.cb_kwargs)
+        self.should_error()
+
+        # returns_item (error, contract doesn't provide keyword arguments)
+        request = self.conman.from_method(spider.returns_item_cb_kwargs_error_missing_argument, self.results)
+        request.callback(response, **request.cb_kwargs)
+        self.should_error()
+
     def test_returns(self):
         spider = TestSpider()
         response = ResponseMock()

From 0e8770a2f4e96ee18e2fd0fc7bfb5f9bcd2f623d Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Fri, 6 Sep 2019 15:47:57 +0200
Subject: [PATCH 2190/2974] test for files pipeline ftp store

---
 scrapy/utils/test.py         | 18 ++++++++++++++++++
 tests/test_pipeline_files.py | 25 ++++++++++++++++++++++++-
 2 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 4b935c51b..59467f105 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -3,6 +3,7 @@ This module contains some assorted functions used in tests
 """
 
 from __future__ import absolute_import
+from posixpath import split
 import os
 
 from importlib import import_module
@@ -61,6 +62,23 @@ def get_gcs_content_and_delete(bucket, path):
     bucket.delete_blob(path)
     return content, acl, blob
 
+def get_ftp_content_and_delete(path, host ,port,
+        username, password, use_active_mode=False):
+    from ftplib import FTP
+    ftp = FTP()
+    ftp.connect(host, port)
+    ftp.login(username, password)
+    if use_active_mode:
+        ftp.set_pasv(False)
+    ftp_data = []
+    def buffer_data(data):
+        ftp_data.append(data)
+    ftp.retrbinary('RETR %s' % path, buffer_data)
+    dirname, filename = split(path)
+    ftp.cwd(dirname)
+    ftp.delete(filename)
+    return "".join(ftp_data)
+    
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 0c5aaaa44..000c1e2e2 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -11,13 +11,14 @@ from six import BytesIO
 from twisted.trial import unittest
 from twisted.internet import defer
 
-from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore
+from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore, FTPFilesStore
 from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
 from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete
+from scrapy.utils.test import get_ftp_content_and_delete
 from scrapy.utils.boto import is_botocore
 
 from tests import mock
@@ -365,6 +366,28 @@ class TestGCSFilesStore(unittest.TestCase):
         self.assertEqual(blob.content_type, 'application/octet-stream')
         self.assertIn(expected_policy, acl)
 
+class TestFTPFileStore(unittest.TestCase):
+    @defer.inlineCallbacks
+    def test_persist(self):
+        uri = os.environ.get('FTP_TEST_FILE_URI')
+        if not uri:
+            raise unittest.SkipTest("No FTP URI available for testing")
+        data = b"TestFTPFilesStore: \xe2\x98\x83"
+        buf = BytesIO(data)
+        meta = {'foo': 'bar'}
+        path = 'full/filename'
+        store = FTPFilesStore(uri)
+        empty_dict = yield store.stat_file(path, info=None)
+        self.assertEqual(empty_dict, {})
+        yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
+        stat = yield store.stat_file(path, info=None)
+        self.assertIn('last_modified', stat)
+        self.assertIn('checksum', stat)
+        self.assertEqual(stat['checksum'], 'd113d66b2ec7258724a268bd88eef6b6')
+        path = '%s/%s' % (store.basedir, path)
+        content = get_ftp_content_and_delete(path, store.host, store.port,
+            store.username, store.password, store.USE_ACTIVE_MODE)
+        self.assertEqual(data.decode(), content)
 
 class ItemWithFiles(Item):
     file_urls = Field()

From 9578f490991fbefc4ad643588c87862a57a9f032 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Mon, 9 Sep 2019 07:36:55 +0000
Subject: [PATCH 2191/2974] use protego as a default robots.txt parser

---
 docs/topics/downloader-middleware.rst | 17 ++++++++---------
 requirements-py2.txt                  |  1 +
 requirements-py3.txt                  |  1 +
 scrapy/settings/default_settings.py   |  2 +-
 tox.ini                               |  2 --
 5 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 0845ef6e4..be96425cf 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -513,7 +513,7 @@ defines the methods described below.
 
     .. method:: open_spider(spider)
 
-      This method gets called after a spider has been opened for crawling. It handles 
+      This method gets called after a spider has been opened for crawling. It handles
       the :signal:`open_spider <spider_opened>` signal.
 
       :param spider: the spider which has been opened
@@ -521,8 +521,8 @@ defines the methods described below.
 
     .. method:: close_spider(spider)
 
-      This method gets called after a spider has been closed. It handles 
-      the :signal:`close_spider <spider_closed>` signal. 
+      This method gets called after a spider has been closed. It handles
+      the :signal:`close_spider <spider_closed>` signal.
 
       :param spider: the spider which has been closed
       :type spider: :class:`~scrapy.spiders.Spider` object
@@ -1020,10 +1020,10 @@ the request will be ignored by this middleware even if
 RobotFileParser
 ~~~~~~~~~~~~~~~
 
-`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ is 
-Python's inbuilt robots.txt_ parser. The parser is fully compliant with `Martijn Koster's 
+`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ is
+Python's inbuilt robots.txt_ parser. The parser is fully compliant with `Martijn Koster's
 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_. It lacks
-support for wildcard matching. Scrapy uses this parser by default.
+support for wildcard matching.
 
 In order to use this parser, set:
 
@@ -1074,13 +1074,12 @@ Protego parser
 
 `Protego <https://github.com/scrapy/protego>`_ is a pure-Python robots.txt_ parser.
 The parser is fully compliant with `Google's Robots.txt Specification
-<https://developers.google.com/search/reference/robots_txt>`_ hence supports wildcard 
+<https://developers.google.com/search/reference/robots_txt>`_ hence supports wildcard
 matching, and uses the length based rule similar to `Reppy <https://github.com/seomoz/reppy/>`_.
+Scrapy uses this parser by default.
 
 In order to use this parser:
 
-* Install `Protego <https://github.com/scrapy/protego>`_ by running ``pip install protego``
-
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.ProtegoRobotParser``
 
diff --git a/requirements-py2.txt b/requirements-py2.txt
index 9e6944240..61176bdba 100644
--- a/requirements-py2.txt
+++ b/requirements-py2.txt
@@ -15,3 +15,4 @@ service_identity>=16.0.0
 six>=1.10.0
 Twisted>=16.0.0
 zope.interface>=4.1.3
+protego
diff --git a/requirements-py3.txt b/requirements-py3.txt
index cd183a525..61e2e32d8 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -15,3 +15,4 @@ lxml>=3.5.0
 service_identity>=16.0.0
 six>=1.10.0
 zope.interface>=4.1.3
+protego
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 05ab4b628..9c22999cb 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -246,7 +246,7 @@ RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
 RETRY_PRIORITY_ADJUST = -1
 
 ROBOTSTXT_OBEY = False
-ROBOTSTXT_PARSER = 'scrapy.robotstxt.PythonRobotParser'
+ROBOTSTXT_PARSER = 'scrapy.robotstxt.ProtegoRobotParser'
 ROBOTSTXT_USER_AGENT = None
 
 SCHEDULER = 'scrapy.core.scheduler.Scheduler'
diff --git a/tox.ini b/tox.ini
index cc845faf1..c3502c2ca 100644
--- a/tox.ini
+++ b/tox.ini
@@ -125,7 +125,6 @@ deps =
     {[testenv:py35]deps}
     reppy
     robotexclusionrulesparser
-    protego
 
 [testenv:py27-extra-deps]
 basepython = python2.7
@@ -133,4 +132,3 @@ deps =
     {[testenv]deps}
     reppy
     robotexclusionrulesparser
-    protego
\ No newline at end of file

From 7af8c76649caf772627a189bb2d88c2d8fd620a2 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Mon, 9 Sep 2019 08:10:09 +0000
Subject: [PATCH 2192/2974] add pinned versions

---
 tox.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tox.ini b/tox.ini
index c3502c2ca..fdd227d02 100644
--- a/tox.ini
+++ b/tox.ini
@@ -33,6 +33,7 @@ deps =
     cssselect==0.9.1
     lxml==3.5.0
     parsel==1.5.0
+    Protego=0.1.15
     PyDispatcher==2.0.5
     pyOpenSSL==16.2.0
     queuelib==1.4.2
@@ -69,6 +70,7 @@ deps =
     cssselect==0.9.1
     lxml==3.5.0
     parsel==1.5.0
+    Protego=0.1.15
     PyDispatcher==2.0.5
     pyOpenSSL==16.2.0
     queuelib==1.4.2

From e418554c21cdd9da87b914472e32668cc41d7e87 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Mon, 9 Sep 2019 08:12:32 +0000
Subject: [PATCH 2193/2974] use proper equal

---
 tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index fdd227d02..ffe7360d3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -33,7 +33,7 @@ deps =
     cssselect==0.9.1
     lxml==3.5.0
     parsel==1.5.0
-    Protego=0.1.15
+    Protego==0.1.15
     PyDispatcher==2.0.5
     pyOpenSSL==16.2.0
     queuelib==1.4.2
@@ -70,7 +70,7 @@ deps =
     cssselect==0.9.1
     lxml==3.5.0
     parsel==1.5.0
-    Protego=0.1.15
+    Protego==0.1.15
     PyDispatcher==2.0.5
     pyOpenSSL==16.2.0
     queuelib==1.4.2

From 38828d3fd45e7846ef3556d384e4c1fdc7a0c8ba Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Mon, 9 Sep 2019 17:04:13 +0300
Subject: [PATCH 2194/2974] Update docs/topics/downloader-middleware.rst

Co-Authored-By: elacuesta <elacuesta@users.noreply.github.com>
---
 docs/topics/downloader-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index be96425cf..192bfd19a 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1021,7 +1021,7 @@ RobotFileParser
 ~~~~~~~~~~~~~~~
 
 `RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ is
-Python's inbuilt robots.txt_ parser. The parser is fully compliant with `Martijn Koster's
+Python's built-in robots.txt_ parser. The parser is fully compliant with `Martijn Koster's
 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_. It lacks
 support for wildcard matching.
 

From 7b33fa58fa46ae7fb96cb40d82ecf98eafcbd49d Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Mon, 9 Sep 2019 17:04:27 +0300
Subject: [PATCH 2195/2974] Update requirements-py2.txt

Co-Authored-By: elacuesta <elacuesta@users.noreply.github.com>
---
 requirements-py2.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-py2.txt b/requirements-py2.txt
index 61176bdba..c865cbaef 100644
--- a/requirements-py2.txt
+++ b/requirements-py2.txt
@@ -15,4 +15,4 @@ service_identity>=16.0.0
 six>=1.10.0
 Twisted>=16.0.0
 zope.interface>=4.1.3
-protego
+protego>=0.1.15

From db202487f06a1710b32d991220dbd6656da8b2a0 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Mon, 9 Sep 2019 14:05:45 +0000
Subject: [PATCH 2196/2974] newer version of protego and move up to top

---
 requirements-py2.txt | 2 +-
 requirements-py3.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements-py2.txt b/requirements-py2.txt
index c865cbaef..dde8d1c9c 100644
--- a/requirements-py2.txt
+++ b/requirements-py2.txt
@@ -1,6 +1,7 @@
 parsel>=1.5.0
 PyDispatcher>=2.0.5
 w3lib>=1.17.0
+protego>=0.1.15
 
 pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
 queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
@@ -15,4 +16,3 @@ service_identity>=16.0.0
 six>=1.10.0
 Twisted>=16.0.0
 zope.interface>=4.1.3
-protego>=0.1.15
diff --git a/requirements-py3.txt b/requirements-py3.txt
index 61e2e32d8..2c98e6f6d 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -2,6 +2,7 @@ parsel>=1.5.0
 PyDispatcher>=2.0.5
 Twisted>=17.9.0
 w3lib>=1.17.0
+protego>=0.1.15
 
 pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
 queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
@@ -15,4 +16,3 @@ lxml>=3.5.0
 service_identity>=16.0.0
 six>=1.10.0
 zope.interface>=4.1.3
-protego

From 6bd88711f2f5946bce6c72d46cd8a5ad3ce4ce86 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Tue, 10 Sep 2019 08:55:37 +0000
Subject: [PATCH 2197/2974] update documentation

---
 docs/topics/settings.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 943ba13ee..75e0af63b 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -951,7 +951,7 @@ LOGSTATS_INTERVAL
 
 Default: ``60.0``
 
-The interval (in seconds) between each logging printout of the stats 
+The interval (in seconds) between each logging printout of the stats
 by :class:`~scrapy.extensions.logstats.LogStats`.
 
 .. setting:: MEMDEBUG_ENABLED
@@ -1165,7 +1165,7 @@ If enabled, Scrapy will respect robots.txt policies. For more information see
 ROBOTSTXT_PARSER
 ----------------
 
-Default: ``'scrapy.robotstxt.PythonRobotParser'``
+Default: ``'scrapy.robotstxt.ProtegoRobotParser'``
 
 The parser backend to use for parsing ``robots.txt`` files. For more information see
 :ref:`topics-dlmw-robots`.

From c7f2bdfdbed0e82a385b05ea080768d38d6626cd Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Tue, 10 Sep 2019 08:58:52 +0000
Subject: [PATCH 2198/2974] add protego to install_requires

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index 37892cfbf..850456503 100644
--- a/setup.py
+++ b/setup.py
@@ -77,6 +77,7 @@ setup(
         'six>=1.10.0',
         'w3lib>=1.17.0',
         'zope.interface>=4.1.3',
+        'protego>=0.1.15',
     ],
     extras_require=extras_require,
 )

From 171fa1cd106f5a4ba34e6d57bdd65c8688564b82 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Tue, 10 Sep 2019 09:59:36 +0000
Subject: [PATCH 2199/2974] documentation rework

---
 docs/topics/downloader-middleware.rst | 116 +++++++++++++++++---------
 1 file changed, 76 insertions(+), 40 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 192bfd19a..52be8ded2 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1000,10 +1000,10 @@ RobotsTxtMiddleware
 
     Scrapy ships with support for the following robots.txt_ parsers:
 
-    * :ref:`RobotFileParser <python-robotfileparser>` (default)
+    * :ref:`Protego <protego-parser>` (default)
+    * :ref:`RobotFileParser <python-robotfileparser>`
     * :ref:`Reppy <reppy-parser>`
     * :ref:`Robotexclusionrulesparser <rerp-parser>`
-    * :ref:`Protego <protego-parser>`
 
     You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
     setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
@@ -1015,50 +1015,78 @@ If :attr:`Request.meta <scrapy.http.Request.meta>` has
 the request will be ignored by this middleware even if
 :setting:`ROBOTSTXT_OBEY` is enabled.
 
+Parsers varies in several aspects:
+
+* Language of implementation
+
+* Supported specification
+
+* Support for wildcard matching
+
+* usage of length based rule: in particular for ``Allow`` and
+  ``Disallow`` directives, where the most specific rule based on the length of
+  the path trumps the less specific (shorter) rule
+
+
+.. _protego-parser:
+
+Protego parser
+~~~~~~~~~~~~~~
+
+based on `Protego <https://github.com/scrapy/protego>`_:
+
+* implemented in Python
+
+* is compliant with `Google's Robots.txt Specification
+  <https://developers.google.com/search/reference/robots_txt>`_
+
+* supports wildcard matching
+
+* uses the length based rule,
+
+Scrapy uses this parser by default.
+
 .. _python-robotfileparser:
 
 RobotFileParser
 ~~~~~~~~~~~~~~~
 
-`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ is
-Python's built-in robots.txt_ parser. The parser is fully compliant with `Martijn Koster's
-1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_. It lacks
-support for wildcard matching.
+based on `RobotFileParser
+<https://docs.python.org/3.7/library/urllib.robotparser.html>`_:
+
+* is Python's built-in robots.txt_ parser.
+
+* is compliant with `Martijn Koster's 1996 draft specification
+  <http://www.robotstxt.org/norobots-rfc.txt>`_.
+
+* lacks support for wildcard matching.
+
+* doesn't use the length based rule,
+
+It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0 .
 
 In order to use this parser, set:
 
 * :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser``
 
-.. _rerp-parser:
-
-Robotexclusionrulesparser
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-`Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ is fully compliant
-with `Martijn Koster's 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_,
-with support for wildcard matching.
-
-In order to use this parser:
-
-* Install `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ by running
-  ``pip install robotexclusionrulesparser``
-
-* Set :setting:`ROBOTSTXT_PARSER` setting to
-  ``scrapy.robotstxt.RerpRobotParser``
-
 .. _reppy-parser:
 
 Reppy parser
 ~~~~~~~~~~~~
 
-`Reppy <https://github.com/seomoz/reppy/>`_ is a Python wrapper around `Robots Exclusion
-Protocol Parser for C++ <https://github.com/seomoz/rep-cpp>`_. The parser is fully compliant
-with `Martijn Koster's 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_,
-with support for wildcard matching. Unlike
-`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ and
-`Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_, it uses the length based
-rule, in particular for ``Allow`` and ``Disallow`` directives, where the most specific
-rule based on the length of the path trumps the less specific (shorter) rule.
+based on `Reppy <https://github.com/seomoz/reppy/>`_:
+
+* is a Python wrapper around `Robots Exclusion Protocol Parser for C++
+  <https://github.com/seomoz/rep-cpp>`_.
+
+* is compliant with `Martijn Koster's 1996 draft specification
+  <http://www.robotstxt.org/norobots-rfc.txt>`_.
+
+* supports wildcard matching
+
+* uses the length based rule,
+
+Native implementation provides better speed than Protego.
 
 In order to use this parser:
 
@@ -1067,21 +1095,29 @@ In order to use this parser:
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.ReppyRobotParser``
 
-.. _protego-parser:
+.. _rerp-parser:
 
-Protego parser
-~~~~~~~~~~~~~~
+Robotexclusionrulesparser
+~~~~~~~~~~~~~~~~~~~~~~~~~
 
-`Protego <https://github.com/scrapy/protego>`_ is a pure-Python robots.txt_ parser.
-The parser is fully compliant with `Google's Robots.txt Specification
-<https://developers.google.com/search/reference/robots_txt>`_ hence supports wildcard
-matching, and uses the length based rule similar to `Reppy <https://github.com/seomoz/reppy/>`_.
-Scrapy uses this parser by default.
+based on `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_:
+
+* implemented in Python
+
+* is compliant with `Martijn Koster's 1996 draft specification
+  <http://www.robotstxt.org/norobots-rfc.txt>`_.
+
+* supports wildcard matching
+
+* doesn't use the length based rule,
 
 In order to use this parser:
 
+* Install `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ by running
+  ``pip install robotexclusionrulesparser``
+
 * Set :setting:`ROBOTSTXT_PARSER` setting to
-  ``scrapy.robotstxt.ProtegoRobotParser``
+  ``scrapy.robotstxt.RerpRobotParser``
 
 .. _support-for-new-robots-parser:
 

From 66145b4eaf813929c45c17624d3cb4645b2e3ba0 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Thu, 12 Sep 2019 18:51:00 +0300
Subject: [PATCH 2200/2974] Update docs/topics/downloader-middleware.rst

Co-Authored-By: Mikhail Korobov <kmike84@gmail.com>
---
 docs/topics/downloader-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 52be8ded2..76ee77a35 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1015,7 +1015,7 @@ If :attr:`Request.meta <scrapy.http.Request.meta>` has
 the request will be ignored by this middleware even if
 :setting:`ROBOTSTXT_OBEY` is enabled.
 
-Parsers varies in several aspects:
+Parsers vary in several aspects:
 
 * Language of implementation
 

From c5612f387bcf34dc47044b81f802804a719e858f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 13 Sep 2019 14:21:09 -0300
Subject: [PATCH 2201/2974] Remove deprecated xlib module

---
 .coveragerc                         |  1 -
 conftest.py                         | 10 ----------
 debian/scrapy.lintian-overrides     |  1 -
 scrapy/xlib/__init__.py             |  2 --
 scrapy/xlib/pydispatch.py           | 19 -------------------
 scrapy/xlib/tx.py                   | 19 -------------------
 tests/test_pydispatch_deprecated.py | 12 ------------
 7 files changed, 64 deletions(-)
 delete mode 100644 scrapy/xlib/__init__.py
 delete mode 100644 scrapy/xlib/pydispatch.py
 delete mode 100644 scrapy/xlib/tx.py
 delete mode 100644 tests/test_pydispatch_deprecated.py

diff --git a/.coveragerc b/.coveragerc
index 914d697a0..02acbff8e 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -3,4 +3,3 @@ branch = true
 include = scrapy/*
 omit =
   tests/*
-  scrapy/xlib/*
diff --git a/conftest.py b/conftest.py
index d8531d6cc..06d65ba1d 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,22 +1,12 @@
-import glob
 import six
 import pytest
-from twisted import version as twisted_version
-
-
-def _py_files(folder):
-    return glob.glob(folder + "/*.py") + glob.glob(folder + "/*/*.py")
 
 
 collect_ignore = [
     # not a test, but looks like a test
     "scrapy/utils/testsite.py",
-
 ]
 
-if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, 5, 0):
-    collect_ignore += _py_files("scrapy/xlib/tx")
-
 
 if six.PY3:
     for line in open('tests/py3-ignores.txt'):
diff --git a/debian/scrapy.lintian-overrides b/debian/scrapy.lintian-overrides
index 955e7def0..b5de7f67d 100644
--- a/debian/scrapy.lintian-overrides
+++ b/debian/scrapy.lintian-overrides
@@ -1,2 +1 @@
 new-package-should-close-itp-bug
-extra-license-file usr/share/pyshared/scrapy/xlib/pydispatch/license.txt
diff --git a/scrapy/xlib/__init__.py b/scrapy/xlib/__init__.py
deleted file mode 100644
index 11f022087..000000000
--- a/scrapy/xlib/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-"""This package contains some third party modules that are distributed along
-with Scrapy"""
diff --git a/scrapy/xlib/pydispatch.py b/scrapy/xlib/pydispatch.py
deleted file mode 100644
index 5ffeaf579..000000000
--- a/scrapy/xlib/pydispatch.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from __future__ import absolute_import
-
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-from pydispatch import (
-    dispatcher,
-    errors,
-    robust,
-    robustapply,
-    saferef,
-)
-
-warnings.warn("Importing from scrapy.xlib.pydispatch is deprecated and will"
-              " no longer be supported in future Scrapy versions."
-              " If you just want to connect signals use the from_crawler class method,"
-              " otherwise import pydispatch directly if needed."
-              " See: https://github.com/scrapy/scrapy/issues/1762",
-              ScrapyDeprecationWarning, stacklevel=2)
diff --git a/scrapy/xlib/tx.py b/scrapy/xlib/tx.py
deleted file mode 100644
index 0d94307b7..000000000
--- a/scrapy/xlib/tx.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from __future__ import absolute_import
-
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-from twisted.web import client
-from twisted.internet import endpoints
-
-Agent = client.Agent  # since < 11.1
-ProxyAgent = client.ProxyAgent  # since 11.1
-ResponseDone = client.ResponseDone  # since 11.1
-ResponseFailed = client.ResponseFailed  # since 11.1
-HTTPConnectionPool = client.HTTPConnectionPool  # since 12.1
-TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint  # since 10.1
-
-warnings.warn("Importing from scrapy.xlib.tx is deprecated and will"
-              " no longer be supported in future Scrapy versions."
-              " Update your code to import from twisted proper.",
-              ScrapyDeprecationWarning, stacklevel=2)
diff --git a/tests/test_pydispatch_deprecated.py b/tests/test_pydispatch_deprecated.py
deleted file mode 100644
index 6d3237fe1..000000000
--- a/tests/test_pydispatch_deprecated.py
+++ /dev/null
@@ -1,12 +0,0 @@
-import unittest
-import warnings
-from six.moves import reload_module
-
-
-class DeprecatedPydispatchTest(unittest.TestCase):
-    def test_import_xlib_pydispatch_show_warning(self):
-        with warnings.catch_warnings(record=True) as w:
-            from scrapy.xlib import pydispatch
-            reload_module(pydispatch)
-        self.assertIn('Importing from scrapy.xlib.pydispatch is deprecated',
-                      str(w[0].message))

From 21ad8e20b9189eeca9bc3f2641eb6d27fe40ce4f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 13 Sep 2019 16:02:49 -0300
Subject: [PATCH 2202/2974] Crawling rules: make link extractors optional

---
 docs/topics/spiders.rst |  4 +++-
 scrapy/spiders/crawl.py | 12 ++++++++----
 tests/test_spider.py    | 20 ++++++++++++++++++++
 3 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 869a61441..45eea3e60 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -374,12 +374,14 @@ CrawlSpider
 Crawling rules
 ~~~~~~~~~~~~~~
 
-.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None)
+.. autoclass:: Rule
 
    ``link_extractor`` is a :ref:`Link Extractor <topics-link-extractors>` object which
    defines how links will be extracted from each crawled page. Each produced link will
    be used to generate a :class:`~scrapy.http.Request` object, which will contain the
    link's text in its ``meta`` dictionary (under the ``link_text`` key).
+   If omitted, a default link extractor created with no arguments will be used,
+   resulting in all links being extracted.
 
    ``callback`` is a callable or a string (in which case a method from the spider
    object with that name will be used) to be called for each link extracted with
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 90a6eb806..03000ce54 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -12,9 +12,10 @@ import six
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, HtmlResponse
-from scrapy.utils.spider import iterate_spider_output
-from scrapy.utils.python import get_func_args
+from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
+from scrapy.utils.python import get_func_args
+from scrapy.utils.spider import iterate_spider_output
 
 
 def _identity(request, response):
@@ -28,10 +29,13 @@ def _get_method(method, spider):
         return getattr(spider, method, None)
 
 
+_default_link_extractor = LinkExtractor()
+
+
 class Rule(object):
 
-    def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None):
-        self.link_extractor = link_extractor
+    def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None):
+        self.link_extractor = link_extractor or _default_link_extractor
         self.callback = callback
         self.cb_kwargs = cb_kwargs or {}
         self.process_links = process_links
diff --git a/tests/test_spider.py b/tests/test_spider.py
index e81e6d5f9..2220b8ffc 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -177,6 +177,26 @@ class CrawlSpiderTest(SpiderTest):
     </body></html>"""
     spider_class = CrawlSpider
 
+    def test_rule_without_link_extractor(self):
+
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+
+        class _CrawlSpider(self.spider_class):
+            name = "test"
+            allowed_domains = ['example.org']
+            rules = (
+                Rule(),
+            )
+
+        spider = _CrawlSpider()
+        output = list(spider._requests_to_follow(response))
+        self.assertEqual(len(output), 3)
+        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertEqual([r.url for r in output],
+                         ['http://example.org/somepage/item/12.html',
+                          'http://example.org/about.html',
+                          'http://example.org/nofollow.html'])
+
     def test_process_links(self):
 
         response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)

From 13735bcf34b0caf60a52e95a907ef390324fdddc Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Mon, 16 Sep 2019 14:04:06 +0200
Subject: [PATCH 2203/2974] Disallow media extensions unregistered with IANA 
 (#3954)

Co-Authored-By: s-sanjay <sanjay537@gmail.com>
---
 scrapy/pipelines/files.py    | 9 +++++++++
 tests/test_pipeline_files.py | 6 ++++++
 2 files changed, 15 insertions(+)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index ea06d2ae8..cc3d10b63 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -5,6 +5,7 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
+import mimetypes
 import os
 import os.path
 import time
@@ -14,6 +15,7 @@ from six.moves.urllib.parse import urlparse
 from collections import defaultdict
 import six
 
+
 try:
     from cStringIO import StringIO as BytesIO
 except ImportError:
@@ -473,4 +475,11 @@ class FilesPipeline(MediaPipeline):
     def file_path(self, request, response=None, info=None):
         media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         media_ext = os.path.splitext(request.url)[1]
+        # Handles empty and wild extensions by trying to guess the
+        # mime type then extension or default to empty string otherwise
+        if media_ext not in mimetypes.types_map:
+            media_ext = ''
+            media_type = mimetypes.guess_type(request.url)[0]
+            if media_type:
+                media_ext = mimetypes.guess_extension(media_type)
         return 'full/%s%s' % (media_guid, media_ext)
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 0c5aaaa44..cb8f8da18 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -57,6 +57,12 @@ class FilesPipelineTestCase(unittest.TestCase):
                                    response=Response("http://www.dorma.co.uk/images/product_details/2532"),
                                    info=object()),
                          'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
+        self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")),
+                         'full/76c00cef2ef669ae65052661f68d451162829507')
+        self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
+                                    //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")),
+                         'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')
+                         
 
     def test_fs_store(self):
         assert isinstance(self.pipeline.store, FSFilesStore)

From 0b52fa6ca9ec8916e4ebcaa7ee148e0b20c2068f Mon Sep 17 00:00:00 2001
From: watsta <tkrisztiana@gmail.com>
Date: Mon, 16 Sep 2019 14:12:04 +0200
Subject: [PATCH 2204/2974] LogFormatter: Add the ability to skip log messages
 (#3987)

---
 scrapy/core/engine.py      |  7 ++--
 scrapy/core/scraper.py     |  6 ++--
 scrapy/logformatter.py     |  4 +++
 tests/test_logformatter.py | 66 +++++++++++++++++++++++++++++++++++++-
 4 files changed, 77 insertions(+), 6 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 37fe0a873..fa913e528 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -233,10 +233,11 @@ class ExecutionEngine(object):
         def _on_success(response):
             assert isinstance(response, (Response, Request))
             if isinstance(response, Response):
-                response.request = request # tie request to response received
+                response.request = request  # tie request to response received
                 logkws = self.logformatter.crawled(request, response, spider)
-                logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
-                self.signals.send_catch_log(signal=signals.response_received, \
+                if logkws is not None:
+                    logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+                self.signals.send_catch_log(signal=signals.response_received,
                     response=response, request=request, spider=spider)
             return response
 
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 66f5d0e05..1f389cf2e 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -225,7 +225,8 @@ class Scraper(object):
             ex = output.value
             if isinstance(ex, DropItem):
                 logkws = self.logformatter.dropped(item, ex, response, spider)
-                logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+                if logkws is not None:
+                    logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
                 return self.signals.send_catch_log_deferred(
                     signal=signals.item_dropped, item=item, response=response,
                     spider=spider, exception=output.value)
@@ -238,7 +239,8 @@ class Scraper(object):
                     spider=spider, failure=output)
         else:
             logkws = self.logformatter.scraped(output, response, spider)
-            logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+            if logkws is not None:
+                logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
             return self.signals.send_catch_log_deferred(
                 signal=signals.item_scraped, item=output, response=response,
                 spider=spider)
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index b4d6787ff..f15940ed1 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -29,6 +29,10 @@ class LogFormatter(object):
     *   ``args`` should be a tuple or dict with the formatting placeholders for ``msg``.
         The final log message is computed as ``msg % args``.
 
+    Users can define their own ``LogFormatter`` class if they want to customise how
+    each action is logged or if they want to omit it entirely. In order to omit
+    logging an action the method must return ``None``.
+
     Here is an example on how to create a custom log formatter to lower the severity level of
     the log message when an item is dropped from the pipeline::
 
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 94e6c9fde..eb9c4a561 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -1,10 +1,18 @@
 import unittest
+
+from testfixtures import LogCapture
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase as TwistedTestCase
 import six
 
-from scrapy.spiders import Spider
+from scrapy.crawler import CrawlerRunner
+from scrapy.exceptions import DropItem
 from scrapy.http import Request, Response
 from scrapy.item import Item, Field
 from scrapy.logformatter import LogFormatter
+from scrapy.spiders import Spider
+from tests.mockserver import MockServer
+from tests.spiders import ItemSpider
 
 
 class CustomItem(Item):
@@ -89,5 +97,61 @@ class LogformatterSubclassTest(LoggingContribTest):
         pass
 
 
+class SkipMessagesLogFormatter(LogFormatter):
+    def crawled(self, *args, **kwargs):
+        return None
+
+    def scraped(self, *args, **kwargs):
+        return None
+
+    def dropped(self, *args, **kwargs):
+        return None
+
+
+class DropSomeItemsPipeline(object):
+    drop = True
+
+    def process_item(self, item, spider):
+        if self.drop:
+            self.drop = False
+            raise DropItem("Ignoring item")
+        else:
+            self.drop = True
+
+class ShowOrSkipMessagesTestCase(TwistedTestCase):
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self.base_settings = {
+            'LOG_LEVEL': 'DEBUG',
+            'ITEM_PIPELINES': {
+                __name__ + '.DropSomeItemsPipeline': 300,
+            },
+        }
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_show_messages(self):
+        crawler = CrawlerRunner(self.base_settings).create_crawler(ItemSpider)
+        with LogCapture() as lc:
+            yield crawler.crawl(mockserver=self.mockserver)
+        self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc))
+        self.assertIn("Crawled (200) <GET http://127.0.0.1:", str(lc))
+        self.assertIn("Dropped: Ignoring item", str(lc))
+
+    @defer.inlineCallbacks
+    def test_skip_messages(self):
+        settings = self.base_settings.copy()
+        settings['LOG_FORMATTER'] = __name__ + '.SkipMessagesLogFormatter'
+        crawler = CrawlerRunner(settings).create_crawler(ItemSpider)
+        with LogCapture() as lc:
+            yield crawler.crawl(mockserver=self.mockserver)
+        self.assertNotIn("Scraped from <200 http://127.0.0.1:", str(lc))
+        self.assertNotIn("Crawled (200) <GET http://127.0.0.1:", str(lc))
+        self.assertNotIn("Dropped: Ignoring item", str(lc))
+
+
 if __name__ == "__main__":
     unittest.main()

From 5197b39e01226e8153f7a1abb6760b202d2eaf3e Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Mon, 16 Sep 2019 14:24:25 +0000
Subject: [PATCH 2205/2974] fix capitalization, remove commas

---
 docs/topics/downloader-middleware.rst | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 52be8ded2..f2f754573 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1023,7 +1023,7 @@ Parsers varies in several aspects:
 
 * Support for wildcard matching
 
-* usage of length based rule: in particular for ``Allow`` and
+* Usage of length based rule: in particular for ``Allow`` and
   ``Disallow`` directives, where the most specific rule based on the length of
   the path trumps the less specific (shorter) rule
 
@@ -1033,7 +1033,7 @@ Parsers varies in several aspects:
 Protego parser
 ~~~~~~~~~~~~~~
 
-based on `Protego <https://github.com/scrapy/protego>`_:
+Based on `Protego <https://github.com/scrapy/protego>`_:
 
 * implemented in Python
 
@@ -1042,7 +1042,7 @@ based on `Protego <https://github.com/scrapy/protego>`_:
 
 * supports wildcard matching
 
-* uses the length based rule,
+* uses the length based rule
 
 Scrapy uses this parser by default.
 
@@ -1051,7 +1051,7 @@ Scrapy uses this parser by default.
 RobotFileParser
 ~~~~~~~~~~~~~~~
 
-based on `RobotFileParser
+Based on `RobotFileParser
 <https://docs.python.org/3.7/library/urllib.robotparser.html>`_:
 
 * is Python's built-in robots.txt_ parser.
@@ -1061,7 +1061,7 @@ based on `RobotFileParser
 
 * lacks support for wildcard matching.
 
-* doesn't use the length based rule,
+* doesn't use the length based rule
 
 It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0 .
 
@@ -1074,7 +1074,7 @@ In order to use this parser, set:
 Reppy parser
 ~~~~~~~~~~~~
 
-based on `Reppy <https://github.com/seomoz/reppy/>`_:
+Based on `Reppy <https://github.com/seomoz/reppy/>`_:
 
 * is a Python wrapper around `Robots Exclusion Protocol Parser for C++
   <https://github.com/seomoz/rep-cpp>`_.
@@ -1084,7 +1084,7 @@ based on `Reppy <https://github.com/seomoz/reppy/>`_:
 
 * supports wildcard matching
 
-* uses the length based rule,
+* uses the length based rule
 
 Native implementation provides better speed than Protego.
 
@@ -1100,7 +1100,7 @@ In order to use this parser:
 Robotexclusionrulesparser
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-based on `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_:
+Based on `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_:
 
 * implemented in Python
 
@@ -1109,7 +1109,7 @@ based on `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_:
 
 * supports wildcard matching
 
-* doesn't use the length based rule,
+* doesn't use the length based rule
 
 In order to use this parser:
 

From f6872189b96595625e428331ddd4d1c620047642 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 29 Aug 2019 10:38:49 -0300
Subject: [PATCH 2206/2974] Add LogFormatter.error method

---
 scrapy/core/scraper.py |  6 +++---
 scrapy/logformatter.py | 11 +++++++++++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 1f389cf2e..3273a1506 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -231,9 +231,9 @@ class Scraper(object):
                     signal=signals.item_dropped, item=item, response=response,
                     spider=spider, exception=output.value)
             else:
-                logger.error('Error processing %(item)s', {'item': item},
-                             exc_info=failure_to_exc_info(output),
-                             extra={'spider': spider})
+                logkws = self.logformatter.error(item, ex, response, spider)
+                logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
+                           exc_info=failure_to_exc_info(output))
                 return self.signals.send_catch_log_deferred(
                     signal=signals.item_error, item=item, response=response,
                     spider=spider, failure=output)
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index f15940ed1..4437d1106 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -8,6 +8,7 @@ from scrapy.utils.request import referer_str
 SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s"
 DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
 CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
+ERRORMSG = u"'Error processing %(item)s'"
 
 
 class LogFormatter(object):
@@ -92,6 +93,16 @@ class LogFormatter(object):
             }
         }
 
+    def error(self, item, exception, response, spider):
+        """Logs a message when an item causes an error while it is passing through the item pipeline."""
+        return {
+            'level': logging.ERROR,
+            'msg': ERRORMSG,
+            'args': {
+                'item': item,
+            }
+        }
+
     @classmethod
     def from_crawler(cls, crawler):
         return cls()

From 27436cbbc9e7331d18d4b63f3c894e6621226efb Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 29 Aug 2019 13:51:42 -0300
Subject: [PATCH 2207/2974] [test] LogFormatter.error

---
 tests/test_logformatter.py | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index eb9c4a561..502bc4ccc 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -23,7 +23,7 @@ class CustomItem(Item):
         return "name: %s" % self['name']
 
 
-class LoggingContribTest(unittest.TestCase):
+class LoggingFormatterTest(unittest.TestCase):
 
     def setUp(self):
         self.formatter = LogFormatter()
@@ -61,6 +61,16 @@ class LoggingContribTest(unittest.TestCase):
         lines = logline.splitlines()
         assert all(isinstance(x, six.text_type) for x in lines)
         self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
+    
+    def test_error(self):
+        # In practice, the complete traceback is shown by passing the
+        # 'exc_info' argument to the logging function
+        item = {'key': 'value'}
+        exception = Exception()
+        response = Response("http://www.example.com")
+        logkws = self.formatter.error(item, exception, response, self.spider)
+        logline = logkws['msg'] % logkws['args']
+        self.assertEqual(logline, u"'Error processing {'key': 'value'}'")
 
     def test_scraped(self):
         item = CustomItem()
@@ -75,26 +85,30 @@ class LoggingContribTest(unittest.TestCase):
 
 class LogFormatterSubclass(LogFormatter):
     def crawled(self, request, response, spider):
-        kwargs = super(LogFormatterSubclass, self).crawled(
-        request, response, spider)
+        kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider)
         CRAWLEDMSG = (
-            u"Crawled (%(status)s) %(request)s (referer: "
-            u"%(referer)s)%(flags)s"
+            u"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
         )
+        log_args = kwargs['args']
+        log_args['flags'] = str(request.flags)
         return {
             'level': kwargs['level'],
             'msg': CRAWLEDMSG,
-            'args': kwargs['args']
+            'args': log_args,
         }
 
 
-class LogformatterSubclassTest(LoggingContribTest):
+class LogformatterSubclassTest(unittest.TestCase):
     def setUp(self):
         self.formatter = LogFormatterSubclass()
         self.spider = Spider('default')
 
     def test_flags_in_request(self):
-        pass
+        req = Request("http://www.example.com", flags=['test','flag'])
+        res = Response("http://www.example.com")
+        logkws = self.formatter.crawled(req, res, self.spider)
+        logline = logkws['msg'] % logkws['args']
+        self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")
 
 
 class SkipMessagesLogFormatter(LogFormatter):

From b792dba5281c91dc6845e3e64771dde2157b20f7 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Tue, 17 Sep 2019 06:28:33 +0000
Subject: [PATCH 2208/2974] remove periods

---
 docs/topics/downloader-middleware.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index b67723b86..de5f72b80 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1054,12 +1054,12 @@ RobotFileParser
 Based on `RobotFileParser
 <https://docs.python.org/3.7/library/urllib.robotparser.html>`_:
 
-* is Python's built-in robots.txt_ parser.
+* is Python's built-in robots.txt_ parser
 
 * is compliant with `Martijn Koster's 1996 draft specification
-  <http://www.robotstxt.org/norobots-rfc.txt>`_.
+  <http://www.robotstxt.org/norobots-rfc.txt>`_
 
-* lacks support for wildcard matching.
+* lacks support for wildcard matching
 
 * doesn't use the length based rule
 
@@ -1077,10 +1077,10 @@ Reppy parser
 Based on `Reppy <https://github.com/seomoz/reppy/>`_:
 
 * is a Python wrapper around `Robots Exclusion Protocol Parser for C++
-  <https://github.com/seomoz/rep-cpp>`_.
+  <https://github.com/seomoz/rep-cpp>`_
 
 * is compliant with `Martijn Koster's 1996 draft specification
-  <http://www.robotstxt.org/norobots-rfc.txt>`_.
+  <http://www.robotstxt.org/norobots-rfc.txt>`_
 
 * supports wildcard matching
 
@@ -1105,7 +1105,7 @@ Based on `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_:
 * implemented in Python
 
 * is compliant with `Martijn Koster's 1996 draft specification
-  <http://www.robotstxt.org/norobots-rfc.txt>`_.
+  <http://www.robotstxt.org/norobots-rfc.txt>`_
 
 * supports wildcard matching
 

From d39ef77e6be5f1a24858d92018ea4ed7a38bd127 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Tue, 17 Sep 2019 06:34:33 +0000
Subject: [PATCH 2209/2974] add link to google description of lenght-based rule

---
 docs/topics/downloader-middleware.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index de5f72b80..9ce4293d2 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1023,10 +1023,10 @@ Parsers vary in several aspects:
 
 * Support for wildcard matching
 
-* Usage of length based rule: in particular for ``Allow`` and
-  ``Disallow`` directives, where the most specific rule based on the length of
-  the path trumps the less specific (shorter) rule
-
+* Usage of `length based rule <https://developers.google.com/search/reference/robots_txt#order-of-precedence-for-group-member-lines>`_:
+  in particular for ``Allow`` and ``Disallow`` directives, where the most
+  specific rule based on the length of the path trumps the less specific
+  (shorter) rule
 
 .. _protego-parser:
 

From 57e6f4c75087b7f2182d2ebc6b28e3afa18e12bd Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Tue, 17 Sep 2019 07:18:37 +0000
Subject: [PATCH 2210/2974] add link to performance comparison

---
 docs/topics/downloader-middleware.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 9ce4293d2..ec302f2eb 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1028,6 +1028,9 @@ Parsers vary in several aspects:
   specific rule based on the length of the path trumps the less specific
   (shorter) rule
 
+Performance comparison of different parsers is available at `the following link
+<https://anubhavp28.github.io/gsoc-weekly-checkin-12/>`_.
+
 .. _protego-parser:
 
 Protego parser

From d1d0bf8491da34d1a5a4bcf3d1241241346b62c8 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Tue, 17 Sep 2019 12:27:12 +0500
Subject: [PATCH 2211/2974] Update docs/topics/downloader-middleware.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/downloader-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index ec302f2eb..c08e13a9a 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1066,7 +1066,7 @@ Based on `RobotFileParser
 
 * doesn't use the length based rule
 
-It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0 .
+It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0.
 
 In order to use this parser, set:
 

From 2438ac529a647c6c665d402b59164d509404d584 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Tue, 17 Sep 2019 12:27:22 +0500
Subject: [PATCH 2212/2974] Update docs/topics/downloader-middleware.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/downloader-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index c08e13a9a..539832618 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1089,7 +1089,7 @@ Based on `Reppy <https://github.com/seomoz/reppy/>`_:
 
 * uses the length based rule
 
-Native implementation provides better speed than Protego.
+Native implementation, provides better speed than Protego.
 
 In order to use this parser:
 

From 9b65f9aa5b03353dcbb7daff0b85d3c3599b08fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 19 Sep 2019 09:17:23 +0200
Subject: [PATCH 2213/2974] Fix the item exporter example (#4022)

---
 docs/topics/exporters.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 11b3045ec..a698a6a4e 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -51,7 +51,6 @@ value of one of their fields::
         def close_spider(self, spider):
             for exporter in self.year_to_exporter.values():
                 exporter.finish_exporting()
-                exporter.file.close()
 
         def _exporter_for_item(self, item):
             year = item['year']

From c26a9015ad2f1f50e0dfc1510e9551414da0d04a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 19 Sep 2019 11:08:06 +0200
Subject: [PATCH 2214/2974] Clarify the effects of dont_merge_cookies

---
 docs/topics/request-response.rst | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index ad1b9af10..284d3479b 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -83,17 +83,21 @@ Request objects
         .. reqmeta:: dont_merge_cookies
 
         When some site returns cookies (in a response) those are stored in the
-        cookies for that domain and will be sent again in future requests. That's
-        the typical behaviour of any regular web browser. However, if, for some
-        reason, you want to avoid merging with existing cookies you can instruct
-        Scrapy to do so by setting the ``dont_merge_cookies`` key to True in the
-        :attr:`Request.meta`.
+        cookies for that domain and will be sent again in future requests.
+        That's the typical behaviour of any regular web browser.
 
-        Example of request without merging cookies::
+        To create a request that does not send stored cookies and does not
+        store received cookies, set the ``dont_merge_cookies`` key to ``True``
+        in :attr:`request.meta <scrapy.http.Request.meta>`.
 
-            request_with_cookies = Request(url="http://www.example.com",
-                                           cookies={'currency': 'USD', 'country': 'UY'},
-                                           meta={'dont_merge_cookies': True})
+        Example of a request that sends manually-defined cookies and ignores
+        cookie storage::
+
+            Request(
+                url="http://www.example.com",
+                cookies={'currency': 'USD', 'country': 'UY'},
+                meta={'dont_merge_cookies': True},
+            )
 
         For more info see :ref:`cookies-mw`.
     :type cookies: dict or list

From b14c3cb612becc28499409202e904c062745d52c Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Thu, 19 Sep 2019 23:33:57 +0200
Subject: [PATCH 2215/2974] Add media pipelines FTP documentation

---
 docs/topics/media-pipeline.rst | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 0ce431ff5..d3fed928c 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -148,6 +148,27 @@ Where:
 * ``full`` is a sub-directory to separate full images from thumbnails (if
   used). For more info see :ref:`topics-images-thumbnails`.
 
+FTP server storage
+------------------
+
+.. setting:: FTP_USER
+.. setting:: FTP_PASSWORD
+
+:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a FTP server.
+Scrapy will automatically upload the files to the server.
+
+:setting:`FILES_STORE` value: should be written in the form 
+`ftp://username:password@address:port/path` or `ftp://address:port/path`. In
+the second case, the `username` and `password` are taken from `FTP_USER` and
+`FTP_PASSWORD` settings respectively.
+
+.. note::
+  The `path` can be left empty
+
+FTP supports two different connection modes: active or passive. Scrapy uses
+the passive connection mode by default. To use the active connection mode instead,
+set the `FEED_STORAGE_FTP_ACTIVE` setting to True.
+
 Amazon S3 storage
 -----------------
 

From 447b3d9d8133da0448cba871314df01887b645d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 25 Sep 2019 11:13:37 +0200
Subject: [PATCH 2216/2974] =?UTF-8?q?Fix=20documentation=20typo:=20accesib?=
 =?UTF-8?q?le=20=E2=86=92=20accessible=20(#4033)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scrapy/utils/request.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 50bc3cb1e..9c143b83a 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -30,7 +30,7 @@ def request_fingerprint(request, include_headers=None):
     and are equivalent (ie. they should return the same response).
 
     Another example are cookies used to store session ids. Suppose the
-    following page is only accesible to authenticated users:
+    following page is only accessible to authenticated users:
 
     http://www.example.com/members/offers.html
 

From 1236e9e81ecd83de716cabc61374fbbacd82d0aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 3 Sep 2019 14:08:08 +0200
Subject: [PATCH 2217/2974] Provide complete API documentation coverage of
 scrapy.item

---
 docs/conf.py          |  3 +++
 docs/topics/items.rst |  6 ++++++
 scrapy/item.py        | 33 +++++++++++++++++++++++++++++----
 tests/test_item.py    | 30 +++++++++++++++++++++++++++++-
 4 files changed, 67 insertions(+), 5 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index fa257dead..34dd5bcb7 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -262,6 +262,9 @@ coverage_ignore_pyobjects = [
     # details that are not documented.
     r'^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.',  # methods
     r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]',  # helper functions
+
+    # Never documented before, and deprecated now.
+    r'^scrapy\.item\.DictItem$',
 ]
 
 
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 60fbc82f8..260f5882c 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -263,3 +263,9 @@ Field objects
 .. _dict: https://docs.python.org/2/library/stdtypes.html#dict
 
 
+Other classes related to Item
+=============================
+
+.. autoclass:: BaseItem
+
+.. autoclass:: ItemMeta
diff --git a/scrapy/item.py b/scrapy/item.py
index 9d4786788..73b8f54b0 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -4,13 +4,15 @@ Scrapy Item
 See documentation in docs/topics/item.rst
 """
 
-from abc import ABCMeta
-from pprint import pformat
-from copy import deepcopy
 import collections
+from abc import ABCMeta
+from copy import deepcopy
+from pprint import pformat
+from warnings import warn
 
 import six
 
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 
 
@@ -21,7 +23,19 @@ else:
 
 
 class BaseItem(object_ref):
-    """Base class for all scraped items."""
+    """Base class for all scraped items.
+
+    In Scrapy, an object is considered an *item* if it is an instance of either
+    :class:`BaseItem` or :class:`dict`. For example, when the output of a
+    spider callback is evaluated, only instances of :class:`BaseItem` or
+    :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
+
+    If you need instances of a custom class to be considered items by Scrapy,
+    you must inherit from either :class:`BaseItem` or :class:`dict`.
+
+    Unlike instances of :class:`dict`, instances of :class:`BaseItem` may be
+    :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
+    """
     pass
 
 
@@ -30,6 +44,10 @@ class Field(dict):
 
 
 class ItemMeta(ABCMeta):
+    """Metaclass_ of :class:`Item` that handles field definitions.
+
+    .. _metaclass: https://realpython.com/python-metaclasses
+    """
 
     def __new__(mcs, class_name, bases, attrs):
         classcell = attrs.pop('__classcell__', None)
@@ -56,6 +74,13 @@ class DictItem(MutableMapping, BaseItem):
 
     fields = {}
 
+    def __new__(cls, *args, **kwargs):
+        if issubclass(cls, DictItem) and not issubclass(cls, Item):
+            warn('scrapy.item.DictItem is deprecated, please use '
+                 'scrapy.item.Item instead',
+                 ScrapyDeprecationWarning, stacklevel=2)
+        return super(DictItem, cls).__new__(cls, *args, **kwargs)
+
     def __init__(self, *args, **kwargs):
         self._values = {}
         if args or kwargs:  # avoid creating dict for most common case
diff --git a/tests/test_item.py b/tests/test_item.py
index 010d3b141..947566686 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -1,9 +1,11 @@
 import sys
 import unittest
+from warnings import catch_warnings
 
 import six
 
-from scrapy.item import ABCMeta, Item, ItemMeta, Field
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
 from tests import mock
 
 
@@ -257,6 +259,17 @@ class ItemTest(unittest.TestCase):
         item['tags'].append('tag2')
         assert item['tags'] != copied_item['tags']
 
+    def test_dictitem_deprecation_warning(self):
+        """Make sure the DictItem deprecation warning is not issued for
+        Item"""
+        with catch_warnings(record=True) as warnings:
+            item = Item()
+            self.assertEqual(len(warnings), 0)
+            class SubclassedItem(Item):
+                pass
+            subclassed_item = SubclassedItem()
+            self.assertEqual(len(warnings), 0)
+
 
 class ItemMetaTest(unittest.TestCase):
 
@@ -302,5 +315,20 @@ class ItemMetaClassCellRegression(unittest.TestCase):
                 super(MyItem, self).__init__(*args, **kwargs)
 
 
+class DictItemTest(unittest.TestCase):
+
+    def test_deprecation_warning(self):
+        with catch_warnings(record=True) as warnings:
+            dict_item = DictItem()
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+        with catch_warnings(record=True) as warnings:
+            class SubclassedDictItem(DictItem):
+                pass
+            subclassed_dict_item = SubclassedDictItem()
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+
+
 if __name__ == "__main__":
     unittest.main()

From 2c14692e603873cf5b5783ad62ebe47474d2d7a8 Mon Sep 17 00:00:00 2001
From: s-sanjay <7111850+s-sanjay@users.noreply.github.com>
Date: Fri, 27 Sep 2019 00:56:43 -0700
Subject: [PATCH 2218/2974] remove .keys() to avoid creating a tmp list/keyview
 obj (#4031)

Also add --verbose and --nolinks for code coverage
---
 scrapy/commands/parse.py     | 15 ++++++++-------
 scrapy/commands/runspider.py |  5 ++---
 tests/test_command_parse.py  |  6 ++++++
 3 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index d4f2234b0..ef8acd29c 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -60,10 +60,12 @@ class Command(ScrapyCommand):
 
     @property
     def max_level(self):
-        levels = list(self.items.keys()) + list(self.requests.keys())
-        if not levels:
-            return 0
-        return max(levels)
+        max_items, max_requests = 0, 0
+        if self.items:
+            max_items = max(self.items)
+        if self.requests:
+            max_requests = max(self.requests)
+        return max(max_items, max_requests)
 
     def add_items(self, lvl, new_items):
         old_items = self.items.get(lvl, [])
@@ -84,9 +86,8 @@ class Command(ScrapyCommand):
 
     def print_requests(self, lvl=None, colour=True):
         if lvl is None:
-            levels = list(self.requests.keys())
-            if levels:
-                requests = self.requests[max(levels)]
+            if self.requests:
+                requests = self.requests[max(self.requests)]
             else:
                 requests = []
         else:
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 376d3c84e..57d8471ca 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -60,14 +60,13 @@ class Command(ScrapyCommand):
             else:
                 self.settings.set('FEED_URI', opts.output, priority='cmdline')
             feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))
-            valid_output_formats = feed_exporters.keys()
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
-            if opts.output_format not in valid_output_formats:
+            if opts.output_format not in feed_exporters:
                 raise UsageError("Unrecognized output format '%s', set one"
                                  " using the '-t' switch or as a file extension"
                                  " from the supported list %s" % (opts.output_format,
-                                                                  tuple(valid_output_formats)))
+                                                                  tuple(feed_exporters)))
             self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
 
     def run(self, args, opts):
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 98e415ad3..62d5d76b4 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -108,6 +108,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                            '-a', 'test_arg=1',
                                            '-c', 'parse',
+                                           '--verbose',
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
@@ -117,12 +118,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                            '--meta', raw_json_string,
                                            '-c', 'parse_request_with_meta',
+                                           '--verbose',
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                            '-m', raw_json_string,
                                            '-c', 'parse_request_with_meta',
+                                           '--verbose',
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
@@ -132,6 +135,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                            '--cbkwargs', raw_json_string,
                                            '-c', 'parse_request_with_cb_kwargs',
+                                           '--verbose',
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
@@ -139,6 +143,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
     def test_request_without_meta(self):
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                           '-c', 'parse_request_without_meta',
+                                          '--nolinks',
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
@@ -148,6 +153,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                            '--pipelines',
                                            '-c', 'parse',
+                                           '--verbose',
                                            self.url('/html')])
         self.assertIn("INFO: It Works!", _textmode(stderr))
 

From 28005b2872b897d84343d1e145fe50be880e91ff Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Sat, 28 Sep 2019 06:21:14 +0200
Subject: [PATCH 2219/2974] Update media-pipeline.rst

---
 docs/topics/media-pipeline.rst | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index d3fed928c..ceac317c0 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -151,23 +151,21 @@ Where:
 FTP server storage
 ------------------
 
-.. setting:: FTP_USER
-.. setting:: FTP_PASSWORD
-
-:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a FTP server.
+:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can point to an FTP server.
 Scrapy will automatically upload the files to the server.
 
-:setting:`FILES_STORE` value: should be written in the form 
-`ftp://username:password@address:port/path` or `ftp://address:port/path`. In
-the second case, the `username` and `password` are taken from `FTP_USER` and
-`FTP_PASSWORD` settings respectively.
+:setting:`FILES_STORE` and :setting:`IMAGES_STORE` should be written in one of the
+following forms::
 
-.. note::
-  The `path` can be left empty
+    ftp://username:password@address:port/path
+    ftp://address:port/path
+    
+If ``username`` and ``password`` are not provided, they are taken from :setting:`FTP_USER` and
+:setting:`FTP_PASSWORD` settings respectively.
 
 FTP supports two different connection modes: active or passive. Scrapy uses
 the passive connection mode by default. To use the active connection mode instead,
-set the `FEED_STORAGE_FTP_ACTIVE` setting to True.
+set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
 
 Amazon S3 storage
 -----------------

From 7f4f98fd38d3fdf6b45a9d0289df0cbb48bcd22b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 30 Sep 2019 18:22:28 +0200
Subject: [PATCH 2220/2974] Provide complete API documentation coverage of
 scrapy.linkextractors

---
 docs/conf.py                      |  4 +++
 docs/topics/link-extractors.rst   | 45 ++++++++++++-------------------
 scrapy/linkextractors/__init__.py | 11 ++++++++
 scrapy/linkextractors/lxmlhtml.py |  8 ++++++
 tests/test_linkextractors.py      | 30 +++++++++++++++++++++
 5 files changed, 70 insertions(+), 28 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 34dd5bcb7..5ba6b8d43 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -265,6 +265,10 @@ coverage_ignore_pyobjects = [
 
     # Never documented before, and deprecated now.
     r'^scrapy\.item\.DictItem$',
+    r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
+
+    # Implementation detail of LxmlLinkExtractor
+    r'^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor',
 ]
 
 
diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 713a94e10..f9936a498 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -4,46 +4,33 @@
 Link Extractors
 ===============
 
-Link extractors are objects whose only purpose is to extract links from web
-pages (:class:`scrapy.http.Response` objects) which will be eventually
-followed.
+A link extractor is an object that extracts links from responses.
 
-There is ``scrapy.linkextractors.LinkExtractor`` available
-in Scrapy, but you can create your own custom Link Extractors to suit your
-needs by implementing a simple interface.
-
-The only public method that every link extractor has is ``extract_links``,
-which receives a :class:`~scrapy.http.Response` object and returns a list
-of :class:`scrapy.link.Link` objects. Link extractors are meant to be
-instantiated once and their ``extract_links`` method called several times
-with different responses to extract links to follow.
-
-Link extractors are used in the :class:`~scrapy.spiders.CrawlSpider`
-class (available in Scrapy), through a set of rules, but you can also use it in
-your spiders, even if you don't subclass from
-:class:`~scrapy.spiders.CrawlSpider`, as its purpose is very simple: to
-extract links.
+The constructor of :class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`
+takes settings that determine which links may be extracted.
+:class:`LxmlLinkExtractor.extract_links
+<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` returns a
+list of matching :class:`scrapy.link.Link` objects from a
+:class:`~scrapy.http.Response` object.
 
+Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders
+through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link
+extractors in regular spiders.
 
 .. _topics-link-extractors-ref:
 
-Built-in link extractors reference
-==================================
+Link extractor reference
+========================
 
 .. module:: scrapy.linkextractors
    :synopsis: Link extractors classes
 
-Link extractors classes bundled with Scrapy are provided in the
-:mod:`scrapy.linkextractors` module.
-
-The default link extractor is ``LinkExtractor``, which is the same as
-:class:`~.LxmlLinkExtractor`::
+The link extractor class is
+:class:`scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`. For convenience it
+can also be imported as ``scrapy.linkextractors.LinkExtractor``::
 
     from scrapy.linkextractors import LinkExtractor
 
-There used to be other link extractor classes in previous Scrapy versions,
-but they are deprecated now.
-
 LxmlLinkExtractor
 -----------------
 
@@ -152,4 +139,6 @@ LxmlLinkExtractor
         from elements or attributes which allow leading/trailing whitespaces).
     :type strip: boolean
 
+    .. automethod:: extract_links
+
 .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index ebf3cd7d8..ca80dc339 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -6,11 +6,13 @@ This package contains a collection of Link Extractors.
 For more info see docs/topics/link-extractors.rst
 """
 import re
+from warnings import warn
 
 from six.moves.urllib.parse import urlparse
 from parsel.csstranslator import HTMLTranslator
 from w3lib.url import canonicalize_url
 
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.url import (
     url_is_from_any_domain, url_has_any_extension,
@@ -49,6 +51,15 @@ class FilteringLinkExtractor(object):
 
     _csstranslator = HTMLTranslator()
 
+    def __new__(cls, *args, **kwargs):
+        from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
+        if (issubclass(cls, FilteringLinkExtractor) and
+                not issubclass(cls, LxmlLinkExtractor)):
+            warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
+                 'please use scrapy.linkextractors.LinkExtractor instead',
+                 ScrapyDeprecationWarning, stacklevel=2)
+        return super(FilteringLinkExtractor, cls).__new__(cls)
+
     def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
                  restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):
 
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 8f6f93a44..41091ba23 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -117,6 +117,14 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
                                                 restrict_text=restrict_text)
 
     def extract_links(self, response):
+        """Returns a list of :class:`~scrapy.link.Link` objects from the
+        specified :class:`response <scrapy.http.Response>`.
+
+        Only links that match the settings passed to the link extractor
+        constructor are returned.
+
+        Duplicate links are omitted.
+        """
         base_url = get_base_url(response)
         if self.restrict_xpaths:
             docs = [subdoc
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index d96e259f6..ea6db28c0 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -1,10 +1,13 @@
 import re
 import unittest
+from warnings import catch_warnings
 
 import pytest
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import HtmlResponse, XmlResponse
 from scrapy.link import Link
+from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
 from tests import get_testdata
 
@@ -506,3 +509,30 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
     @pytest.mark.xfail
     def test_restrict_xpaths_with_html_entities(self):
         super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()
+
+    def test_filteringlinkextractor_deprecation_warning(self):
+        """Make sure the FilteringLinkExtractor deprecation warning is not
+        issued for LxmlLinkExtractor"""
+        with catch_warnings(record=True) as warnings:
+            extractor = LxmlLinkExtractor()
+            self.assertEqual(len(warnings), 0)
+            class SubclassedItem(LxmlLinkExtractor):
+                pass
+            subclassed_extractor = SubclassedItem()
+            self.assertEqual(len(warnings), 0)
+
+
+class FilteringLinkExtractorTest(unittest.TestCase):
+
+    def test_deprecation_warning(self):
+        args = [None]*10
+        with catch_warnings(record=True) as warnings:
+            extractor = FilteringLinkExtractor(*args)
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+        with catch_warnings(record=True) as warnings:
+            class SubclassedFilteringLinkExtractor(FilteringLinkExtractor):
+                pass
+            subclassed_extractor = SubclassedFilteringLinkExtractor(*args)
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)

From 175cd2ece5b9a8e2c735697e7a49d0baffc7cd52 Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Tue, 1 Oct 2019 07:27:31 +0200
Subject: [PATCH 2221/2974] Update docs/topics/media-pipeline.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/media-pipeline.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index ceac317c0..327517996 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -160,7 +160,7 @@ following forms::
     ftp://username:password@address:port/path
     ftp://address:port/path
     
-If ``username`` and ``password`` are not provided, they are taken from :setting:`FTP_USER` and
+If ``username`` and ``password`` are not provided, they are taken from the :setting:`FTP_USER` and
 :setting:`FTP_PASSWORD` settings respectively.
 
 FTP supports two different connection modes: active or passive. Scrapy uses

From 7632e375c268d31c570904452902799db180c415 Mon Sep 17 00:00:00 2001
From: Eugen <junta.kristobal@gmail.com>
Date: Tue, 1 Oct 2019 17:29:48 +0300
Subject: [PATCH 2222/2974] fix typo in ScrapyCommand.help docstring (#4046)

---
 scrapy/commands/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 43b420821..0b24193c2 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -47,7 +47,7 @@ class ScrapyCommand(object):
 
     def help(self):
         """An extensive help for the command. It will be shown when using the
-        "help" command. It can contain newlines, since not post-formatting will
+        "help" command. It can contain newlines, since no post-formatting will
         be applied to its contents.
         """
         return self.long_desc()

From c232bbdc426d91b98703d5f8e892b0c431c5a43a Mon Sep 17 00:00:00 2001
From: Kristobal Junta <junta.kristobal@gmail.com>
Date: Tue, 1 Oct 2019 17:41:38 +0300
Subject: [PATCH 2223/2974] fix typo in docs/topics/spiders

---
 docs/topics/spiders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 45eea3e60..d60c93be6 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -692,7 +692,7 @@ SitemapSpider
 
     .. method:: sitemap_filter(entries)
 
-        This is a filter funtion that could be overridden to select sitemap entries
+        This is a filter function that could be overridden to select sitemap entries
         based on their attributes.
 
         For example::

From 07a31b13db96b2aef3074157808f7ad300780c3d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 1 Oct 2019 17:55:57 -0300
Subject: [PATCH 2224/2974] Update LogFormatter tests

---
 tests/test_logformatter.py | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 502bc4ccc..f12ffc11b 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -23,13 +23,13 @@ class CustomItem(Item):
         return "name: %s" % self['name']
 
 
-class LoggingFormatterTest(unittest.TestCase):
+class LogFormatterTestCase(unittest.TestCase):
 
     def setUp(self):
         self.formatter = LogFormatter()
         self.spider = Spider('default')
 
-    def test_crawled(self):
+    def test_crawled_with_referer(self):
         req = Request("http://www.example.com")
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
@@ -37,6 +37,7 @@ class LoggingFormatterTest(unittest.TestCase):
         self.assertEqual(logline,
             "Crawled (200) <GET http://www.example.com> (referer: None)")
 
+    def test_crawled_without_referer(self):
         req = Request("http://www.example.com", headers={'referer': 'http://example.com'})
         res = Response("http://www.example.com", flags=['cached'])
         logkws = self.formatter.crawled(req, res, self.spider)
@@ -98,11 +99,27 @@ class LogFormatterSubclass(LogFormatter):
         }
 
 
-class LogformatterSubclassTest(unittest.TestCase):
+class LogformatterSubclassTest(LogFormatterTestCase):
     def setUp(self):
         self.formatter = LogFormatterSubclass()
         self.spider = Spider('default')
 
+    def test_crawled_with_referer(self):
+        req = Request("http://www.example.com")
+        res = Response("http://www.example.com")
+        logkws = self.formatter.crawled(req, res, self.spider)
+        logline = logkws['msg'] % logkws['args']
+        self.assertEqual(logline,
+            "Crawled (200) <GET http://www.example.com> (referer: None) []")
+
+    def test_crawled_without_referer(self):
+        req = Request("http://www.example.com", headers={'referer': 'http://example.com'}, flags=['cached'])
+        res = Response("http://www.example.com")
+        logkws = self.formatter.crawled(req, res, self.spider)
+        logline = logkws['msg'] % logkws['args']
+        self.assertEqual(logline,
+            "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
+
     def test_flags_in_request(self):
         req = Request("http://www.example.com", flags=['test','flag'])
         res = Response("http://www.example.com")

From e0fabab5cc740bfbef4c13d9d7611278bc7cc335 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 1 Oct 2019 11:54:15 -0300
Subject: [PATCH 2225/2974] Fix TypeError when using DummyStatsCollector

---
 scrapy/extensions/corestats.py | 11 +++++---
 tests/test_stats.py            | 49 ++++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+), 4 deletions(-)

diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 8cc5e18ac..20adfbe4b 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -1,14 +1,16 @@
 """
 Extension for collecting core stats like items scraped and start/finish times
 """
-import datetime
+from datetime import datetime
 
 from scrapy import signals
 
+
 class CoreStats(object):
 
     def __init__(self, stats):
         self.stats = stats
+        self.start_time = None
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -21,11 +23,12 @@ class CoreStats(object):
         return o
 
     def spider_opened(self, spider):
-        self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
+        self.start_time = datetime.utcnow()
+        self.stats.set_value('start_time', self.start_time, spider=spider)
 
     def spider_closed(self, spider, reason):
-        finish_time = datetime.datetime.utcnow()
-        elapsed_time = finish_time - self.stats.get_value('start_time')
+        finish_time = datetime.utcnow()
+        elapsed_time = finish_time - self.start_time
         elapsed_time_seconds = elapsed_time.total_seconds()
         self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)
         self.stats.set_value('finish_time', finish_time, spider=spider)
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 9f950ebc9..2033dbe07 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -1,10 +1,59 @@
+from datetime import datetime
 import unittest
 
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
+from scrapy.extensions.corestats import CoreStats
 from scrapy.spiders import Spider
 from scrapy.statscollectors import StatsCollector, DummyStatsCollector
 from scrapy.utils.test import get_crawler
 
 
+class CoreStatsExtensionTest(unittest.TestCase):
+
+    def setUp(self):
+        self.crawler = get_crawler(Spider)
+        self.spider = self.crawler._create_spider('foo')
+
+    @mock.patch('scrapy.extensions.corestats.datetime')
+    def test_core_stats_default_stats_collector(self, mock_datetime):
+        fixed_datetime = datetime(2019, 12, 1, 11, 38)
+        mock_datetime.utcnow = mock.Mock(return_value=fixed_datetime)
+        self.crawler.stats = StatsCollector(self.crawler)
+        ext = CoreStats.from_crawler(self.crawler)
+        ext.spider_opened(self.spider)
+        ext.item_scraped({}, self.spider)
+        ext.response_received(self.spider)
+        ext.item_dropped({}, self.spider, ZeroDivisionError())
+        ext.spider_closed(self.spider, 'finished')
+        self.assertEqual(
+            ext.stats._stats,
+            {
+                'start_time': fixed_datetime,
+                'finish_time': fixed_datetime,
+                'item_scraped_count': 1,
+                'response_received_count': 1,
+                'item_dropped_count': 1,
+                'item_dropped_reasons_count/ZeroDivisionError': 1,
+                'finish_reason': 'finished',
+                'elapsed_time_seconds': 0.0,
+            }
+        )
+
+    def test_core_stats_dummy_stats_collector(self):
+        self.crawler.stats = DummyStatsCollector(self.crawler)
+        ext = CoreStats.from_crawler(self.crawler)
+        ext.spider_opened(self.spider)
+        ext.item_scraped({}, self.spider)
+        ext.response_received(self.spider)
+        ext.item_dropped({}, self.spider, ZeroDivisionError())
+        ext.spider_closed(self.spider, 'finished')
+        self.assertEqual(ext.stats._stats, {})
+
+
 class StatsCollectorTest(unittest.TestCase):
 
     def setUp(self):

From 6ad5a89cb0aeac18a72ed41819c07e9c22990f10 Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Wed, 2 Oct 2019 07:18:36 -0300
Subject: [PATCH 2226/2974] [Doc] Use autoclass in topics/request-response.rst
 (#4055)

---
 docs/topics/request-response.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 284d3479b..727c67482 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
 Request objects
 ===============
 
-.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, cb_kwargs])
+.. autoclass:: Request
 
     A :class:`Request` object represents an HTTP request, which is usually
     generated in the Spider and executed by the Downloader, and thus generating
@@ -400,7 +400,7 @@ fields with form data from :class:`Response` objects.
 
 .. class:: FormRequest(url, [formdata, ...])
 
-    The :class:`FormRequest` class adds a new argument to the constructor. The
+    The :class:`FormRequest` class adds a new keyword parameter to the constructor. The
     remaining arguments are the same as for the :class:`Request` class and are
     not documented here.
 
@@ -547,7 +547,7 @@ dealing with JSON requests.
 
 .. class:: JsonRequest(url, [... data, dumps_kwargs])
 
-   The :class:`JsonRequest` class adds two new argument to the constructor. The
+   The :class:`JsonRequest` class adds two new keyword parameters to the constructor. The
    remaining arguments are the same as for the :class:`Request` class and are
    not documented here.
 
@@ -581,7 +581,7 @@ Sending a JSON POST request with a JSON payload::
 Response objects
 ================
 
-.. class:: Response(url, [status=200, headers=None, body=b'', flags=None, request=None])
+.. autoclass:: Response
 
     A :class:`Response` object represents an HTTP response, which is usually
     downloaded (by the Downloader) and fed to the Spiders for processing.

From a4aa5b8926153560c493fa84ed1d1d6c730d6f5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 2 Oct 2019 14:51:06 +0200
Subject: [PATCH 2227/2974] Fix internal links in the tutorial and release
 notes

---
 docs/intro/tutorial.rst  |  6 +++---
 docs/news.rst            | 44 ++++++++++++++++++++++++----------------
 docs/topics/items.rst    |  4 ++++
 docs/topics/settings.rst |  2 ++
 4 files changed, 35 insertions(+), 21 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index a190ce407..0629b9e19 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -78,9 +78,9 @@ Our first Spider
 
 Spiders are classes that you define and that Scrapy uses to scrape information
 from a website (or a group of websites). They must subclass
-:class:`scrapy.Spider` and define the initial requests to make, optionally how
-to follow links in the pages, and how to parse the downloaded page content to
-extract data.
+:class:`~scrapy.spiders.Spider` and define the initial requests to make,
+optionally how to follow links in the pages, and how to parse the downloaded
+page content to extract data.
 
 This is the code for our first Spider. Save it in a file named
 ``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
diff --git a/docs/news.rst b/docs/news.rst
index aac750601..c107e907c 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -71,7 +71,7 @@ New features
 ~~~~~~~~~~~~
 
 *   A new scheduler priority queue,
-    :class:`scrapy.pqueues.DownloaderAwarePriorityQueue`, may be
+    ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
     :ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
     scheduling improvement on crawls targetting multiple web domains, at the
     cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
@@ -150,9 +150,9 @@ Bug fixes
     :setting:`AWS_REGION_NAME`, :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY`
     (:issue:`3625`)
 
-*   Fixed a memory leak in :class:`~scrapy.pipelines.media.MediaPipeline`
-    affecting, for example, non-200 responses and exceptions from custom
-    middlewares (:issue:`3813`)
+*   Fixed a memory leak in ``scrapy.pipelines.media.MediaPipeline`` affecting,
+    for example, non-200 responses and exceptions from custom middlewares
+    (:issue:`3813`)
 
 *   Requests with private callbacks are now correctly unserialized from disk
     (:issue:`3790`)
@@ -301,7 +301,7 @@ Deprecations
 
 *   The ``queuelib.PriorityQueue`` value for the
     :setting:`SCHEDULER_PRIORITY_QUEUE` setting is deprecated. Use
-    :class:`scrapy.pqueues.ScrapyPriorityQueue` instead.
+    ``scrapy.pqueues.ScrapyPriorityQueue`` instead.
 
 *   ``process_request`` callbacks passed to :class:`~scrapy.spiders.Rule` that
     do not accept two arguments are deprecated.
@@ -551,7 +551,7 @@ Scrapy 1.5.2 (2019-01-22)
 
   *The fix is backward incompatible*, it enables telnet user-password
   authentication by default with a random generated password. If you can't
-  upgrade right away, please consider setting :setting:`TELNET_CONSOLE_PORT`
+  upgrade right away, please consider setting :setting:`TELNETCONSOLE_PORT`
   out of its default value.
 
   See :ref:`telnet console <topics-telnetconsole>` documentation for more info
@@ -758,7 +758,9 @@ Enjoy! (Or read on for the rest of changes in this release.)
 Deprecations and Backward Incompatible Changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor`
+- Default to ``canonicalize=False`` in
+  :class:`scrapy.linkextractors.LinkExtractor
+  <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
   (:issue:`2537`, fixes :issue:`1941` and :issue:`1982`):
   **warning, this is technically backward-incompatible**
 - Enable memusage extension by default (:issue:`2539`, fixes :issue:`2187`);
@@ -794,10 +796,13 @@ New Features
 - New ``data:`` URI download handler (:issue:`2334`, fixes :issue:`2156`)
 - Log cache directory when HTTP Cache is used (:issue:`2611`, fixes :issue:`2604`)
 - Warn users when project contains duplicate spider names (fixes :issue:`2181`)
-- :class:`CaselessDict` now accepts ``Mapping`` instances and not only dicts (:issue:`2646`)
-- :ref:`Media downloads <topics-media-pipeline>`, with :class:`FilesPipelines`
-  or :class:`ImagesPipelines`, can now optionally handle HTTP redirects
-  using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting (:issue:`2616`, fixes :issue:`2004`)
+- ``scrapy.utils.datatypes.CaselessDict`` now accepts ``Mapping`` instances and
+  not only dicts (:issue:`2646`)
+- :ref:`Media downloads <topics-media-pipeline>`, with
+  :class:`~scrapy.pipelines.files.FilesPipeline` or
+  :class:`~scrapy.pipelines.images.ImagesPipeline`, can now optionally handle
+  HTTP redirects using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting
+  (:issue:`2616`, fixes :issue:`2004`)
 - Accept non-complete responses from websites using a new
   :setting:`DOWNLOAD_FAIL_ON_DATALOSS` setting (:issue:`2590`, fixes :issue:`2586`)
 - Optional pretty-printing of JSON and XML items via
@@ -817,8 +822,8 @@ Bug fixes
 
 - LinkExtractor now strips leading and trailing whitespaces from attributes
   (:issue:`2547`, fixes :issue:`1614`)
-- Properly handle whitespaces in action attribute in :class:`FormRequest`
-  (:issue:`2548`)
+- Properly handle whitespaces in action attribute in
+  :class:`~scrapy.http.FormRequest` (:issue:`2548`)
 - Buffer CONNECT response bytes from proxy until all HTTP headers are received
   (:issue:`2495`, fixes :issue:`2491`)
 - FTP downloader now works on Python 3, provided you use Twisted>=17.1
@@ -851,7 +856,8 @@ Cleanups & Refactoring
   fixes :issue:`2560`)
 - Add omitted ``self`` arguments in default project middleware template (:issue:`2595`)
 - Remove redundant ``slot.add_request()`` call in ExecutionEngine (:issue:`2617`)
-- Catch more specific ``os.error`` exception in :class:`FSFilesStore` (:issue:`2644`)
+- Catch more specific ``os.error`` exception in
+  ``scrapy.pipelines.files.FSFilesStore`` (:issue:`2644`)
 - Change "localhost" test server certificate (:issue:`2720`)
 - Remove unused ``MEMUSAGE_REPORT`` setting (:issue:`2576`)
 
@@ -868,7 +874,8 @@ Documentation
   (:issue:`2477`, fixes :issue:`2475`)
 - FAQ: rewrite note on Python 3 support on Windows (:issue:`2690`)
 - Rearrange selector sections (:issue:`2705`)
-- Remove ``__nonzero__`` from :class:`SelectorList` docs (:issue:`2683`)
+- Remove ``__nonzero__`` from :class:`~scrapy.selector.SelectorList`
+  docs (:issue:`2683`)
 - Mention how to disable request filtering in documentation of
   :setting:`DUPEFILTER_CLASS` setting (:issue:`2714`)
 - Add sphinx_rtd_theme to docs setup readme (:issue:`2668`)
@@ -2327,7 +2334,7 @@ Scrapy 0.18.0 (released 2013-08-09)
 - Moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on
 - Add scrapy commands using external libraries (:issue:`260`)
 - Added ``--pdb`` option to ``scrapy`` command line tool
-- Added :meth:`XPathSelector.remove_namespaces` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
+- Added :meth:`XPathSelector.remove_namespaces <scrapy.selector.Selector.remove_namespaces>` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
 - Several improvements to spider contracts
 - New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections,
 - MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62
@@ -2448,7 +2455,7 @@ Scrapy changes:
 - added options ``-o`` and ``-t`` to the :command:`runspider` command
 - documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED`
 - major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals.
-- added :meth:`~scrapy.contrib.spidermiddleware.SpiderMiddleware.process_start_requests` method to spider middlewares
+- added :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method to spider middlewares
 - dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
 - dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
 - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info.
@@ -2609,7 +2616,8 @@ The numbers like #NNN reference tickets in the old issue tracker (Trac) which is
 New features and improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- Passed item is now sent in the ``item`` argument of the :signal:`item_passed` (#273)
+- Passed item is now sent in the ``item`` argument of the :signal:`item_passed
+  <item_scraped>` (#273)
 - Added verbose option to ``scrapy version`` command, useful for bug reports (#298)
 - HTTP cache now stored by default in the project data dir (#279)
 - Added project data storage directory (#276, #277)
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 260f5882c..fbc888f47 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -240,6 +240,10 @@ Item objects
     Items replicate the standard `dict API`_, including its constructor. The
     only additional attribute provided by Items is:
 
+    .. automethod:: copy
+
+    .. automethod:: deepcopy
+
     .. attribute:: fields
 
         A dictionary containing *all declared fields* for this Item, not only
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 75e0af63b..350c3c4b9 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -796,6 +796,7 @@ Default: ``True``
 
 Whether or not to use passive mode when initiating FTP transfers.
 
+.. reqmeta:: ftp_password
 .. setting:: FTP_PASSWORD
 
 FTP_PASSWORD
@@ -814,6 +815,7 @@ in ``Request`` meta.
 
 .. _RFC 1635: https://tools.ietf.org/html/rfc1635
 
+.. reqmeta:: ftp_user
 .. setting:: FTP_USER
 
 FTP_USER

From 39c9a3cc1c6a59e03abc6c2d4080ddc211289a8d Mon Sep 17 00:00:00 2001
From: John Bampton <jbampton@users.noreply.github.com>
Date: Sat, 5 Oct 2019 10:09:14 +1000
Subject: [PATCH 2228/2974] Fix case of GitHub.

---
 .github/ISSUE_TEMPLATE/bug_report.md      | 2 +-
 .github/ISSUE_TEMPLATE/feature_request.md | 2 +-
 docs/news.rst                             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index 66821171f..8ca10109b 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -8,7 +8,7 @@ about: Report a problem to help us improve
 Thanks for taking an interest in Scrapy!
 
 If you have a question that starts with "How to...", please see the Scrapy Community page: https://scrapy.org/community/.
-The Github issue tracker's purpose is to deal with bug reports and feature requests for the project itself.
+The GitHub issue tracker's purpose is to deal with bug reports and feature requests for the project itself.
 
 Keep in mind that by filing an issue, you are expected to comply with Scrapy's Code of Conduct, including treating everyone with respect: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
 
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
index df5127b4c..e05273fe2 100644
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -8,7 +8,7 @@ about: Suggest an idea for an enhancement or new feature
 Thanks for taking an interest in Scrapy!
 
 If you have a question that starts with "How to...", please see the Scrapy Community page: https://scrapy.org/community/.
-The Github issue tracker's purpose is to deal with bug reports and feature requests for the project itself.
+The GitHub issue tracker's purpose is to deal with bug reports and feature requests for the project itself.
 
 Keep in mind that by filing an issue, you are expected to comply with Scrapy's Code of Conduct, including treating everyone with respect: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
 
diff --git a/docs/news.rst b/docs/news.rst
index aac750601..2bcfe4d1c 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -2590,7 +2590,7 @@ Code rearranged and removed
    - `w3lib`_ (several functions from ``scrapy.utils.{http,markup,multipart,response,url}``, done in :rev:`2584`)
    - `scrapely`_ (was ``scrapy.contrib.ibl``, done in :rev:`2586`)
 - Removed unused function: ``scrapy.utils.request.request_info()`` (:rev:`2577`)
-- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on github: https://github.com/scrapy/dirbot
+- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on GitHub: https://github.com/scrapy/dirbot
 - Removed support for default field values in Scrapy items (:rev:`2616`)
 - Removed experimental crawlspider v2 (:rev:`2632`)
 - Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)

From f52148143be614779ccfdb34cac995b16f8264ff Mon Sep 17 00:00:00 2001
From: akhter wahab <akhter.wahab@gmail.com>
Date: Mon, 7 Oct 2019 23:28:33 +0500
Subject: [PATCH 2229/2974] Add dmg, iso & apk to ignored other extensions

---
 scrapy/linkextractors/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index ebf3cd7d8..6049c312c 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -35,7 +35,7 @@ IGNORED_EXTENSIONS = [
     'odp',
 
     # other
-    'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar',
+    'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar', 'dmg', 'iso', 'apk'
 ]
 
 

From 5f168cd459cfc026de1e4d0b43c45ea740fe1dd5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 2 Oct 2019 14:08:08 -0300
Subject: [PATCH 2230/2974] Response.follow_all

---
 docs/topics/request-response.rst |  4 ++
 scrapy/http/response/__init__.py | 63 +++++++++++++++++-----
 scrapy/http/response/text.py     | 61 ++++++++++++++++++---
 tests/test_http_response.py      | 91 ++++++++++++++++++++++++++++++++
 4 files changed, 199 insertions(+), 20 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 727c67482..9fe3c7518 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -701,6 +701,8 @@ Response objects
 
     .. automethod:: Response.follow
 
+    .. automethod:: Response.follow_all
+
 
 .. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
 
@@ -790,6 +792,8 @@ TextResponse objects
 
     .. automethod:: TextResponse.follow
 
+    .. automethod:: TextResponse.follow_all
+
     .. method:: TextResponse.body_as_unicode()
 
         The same as :attr:`text`, but available as a method. This method is
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index b0a526b72..96359705e 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -113,8 +113,8 @@ class Response(object_ref):
         It accepts the same arguments as ``Request.__init__`` method,
         but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
         not only an absolute URL.
-        
-        :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow` 
+
+        :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
         method which supports selectors in addition to absolute/relative URLs
         and Link objects.
         """
@@ -123,14 +123,51 @@ class Response(object_ref):
         elif url is None:
             raise ValueError("url can't be None")
         url = self.urljoin(url)
-        return Request(url, callback,
-                       method=method,
-                       headers=headers,
-                       body=body,
-                       cookies=cookies,
-                       meta=meta,
-                       encoding=encoding,
-                       priority=priority,
-                       dont_filter=dont_filter,
-                       errback=errback,
-                       cb_kwargs=cb_kwargs)
+        return Request(
+            url=url,
+            callback=callback,
+            method=method,
+            headers=headers,
+            body=body,
+            cookies=cookies,
+            meta=meta,
+            encoding=encoding,
+            priority=priority,
+            dont_filter=dont_filter,
+            errback=errback,
+            cb_kwargs=cb_kwargs,
+        )
+
+    def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,
+                   cookies=None, meta=None, encoding='utf-8', priority=0,
+                   dont_filter=False, errback=None, cb_kwargs=None):
+        # type: (...) -> Generator[Request, None, None]
+        """
+        Return an iterable of :class:`~.Request` instance to follow all links
+        in ``urls``. It accepts the same arguments as ``Request.__init__`` method,
+        but elements of ``urls`` can be relative URLs or ``scrapy.link.Link`` objects
+        not only absolute URLs.
+
+        :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all`
+        method which supports selectors in addition to absolute/relative URLs
+        and Link objects.
+        """
+        if not hasattr(urls, '__iter__'):
+            raise TypeError("'urls' argument must be an iterable")
+        return (
+            self.follow(
+                url=url,
+                callback=callback,
+                method=method,
+                headers=headers,
+                body=body,
+                cookies=cookies,
+                meta=meta,
+                encoding=encoding,
+                priority=priority,
+                dont_filter=dont_filter,
+                errback=errback,
+                cb_kwargs=cb_kwargs,
+            )
+            for url in urls
+        )
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 339913d4e..81e3f9b28 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -13,7 +13,6 @@ from w3lib.encoding import html_to_unicode, resolve_encoding, \
     html_body_declared_encoding, http_content_type_encoding
 from w3lib.html import strip_html5_whitespace
 
-from scrapy.http.request import Request
 from scrapy.http.response import Response
 from scrapy.utils.response import get_base_url
 from scrapy.utils.python import memoizemethod_noargs, to_native_str
@@ -44,7 +43,7 @@ class TextResponse(Response):
         if isinstance(body, six.text_type):
             if self._encoding is None:
                 raise TypeError('Cannot convert unicode body - %s has no encoding' %
-                    type(self).__name__)
+                                type(self).__name__)
             self._body = body.encode(self._encoding)
         else:
             super(TextResponse, self)._set_body(body)
@@ -90,8 +89,8 @@ class TextResponse(Response):
         if self._cached_benc is None:
             content_type = to_native_str(self.headers.get(b'Content-Type', b''))
             benc, ubody = html_to_unicode(content_type, self.body,
-                    auto_detect_fun=self._auto_detect_fun,
-                    default_encoding=self._DEFAULT_ENCODING)
+                                          auto_detect_fun=self._auto_detect_fun,
+                                          default_encoding=self._DEFAULT_ENCODING)
             self._cached_benc = benc
             self._cached_ubody = ubody
         return self._cached_benc
@@ -129,7 +128,7 @@ class TextResponse(Response):
         Return a :class:`~.Request` instance to follow a link ``url``.
         It accepts the same arguments as ``Request.__init__`` method,
         but ``url`` can be not only an absolute URL, but also
-        
+
         * a relative URL;
         * a scrapy.link.Link object (e.g. a link extractor result);
         * an attribute Selector (not SelectorList) - e.g.
@@ -137,7 +136,7 @@ class TextResponse(Response):
           ``response.xpath('//img/@src')[0]``.
         * a Selector for ``<a>`` or ``<link>`` element, e.g.
           ``response.css('a.my_link')[0]``.
-          
+
         See :ref:`response-follow-example` for usage examples.
         """
         if isinstance(url, parsel.Selector):
@@ -145,7 +144,9 @@ class TextResponse(Response):
         elif isinstance(url, parsel.SelectorList):
             raise ValueError("SelectorList is not supported")
         encoding = self.encoding if encoding is None else encoding
-        return super(TextResponse, self).follow(url, callback,
+        return super(TextResponse, self).follow(
+            url=url,
+            callback=callback,
             method=method,
             headers=headers,
             body=body,
@@ -158,6 +159,52 @@ class TextResponse(Response):
             cb_kwargs=cb_kwargs,
         )
 
+    def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,
+                   cookies=None, meta=None, encoding=None, priority=0,
+                   dont_filter=False, errback=None, cb_kwargs=None,
+                   css=None, xpath=None):
+        # type: (...) -> Generator[Request, None, None]
+        """
+        A generator that produces :class:`~.Request` instances to follow all
+        links in ``urls``. It accepts the same arguments as the :class:`~.Request`
+        initializer, except that each ``urls`` element does not need to be an absolute
+        URL, it can be any of the following:
+
+        * a relative URL;
+        * a scrapy.link.Link object (e.g. a link extractor result);
+        * an attribute Selector (not SelectorList) - e.g.
+          ``response.css('a::attr(href)')[0]`` or
+          ``response.xpath('//img/@src')[0]``.
+        * a Selector for ``<a>`` or ``<link>`` element, e.g.
+          ``response.css('a.my_link')[0]``.
+
+        In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction
+        within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` are accepted).
+        """
+        if len(list(filter(None, (urls, css, xpath)))) > 1:
+            raise ValueError('Please supply only one of the following arguments: {urls, css, xpath}')
+        if css:
+            urls = self.css(css)
+        elif xpath:
+            urls = self.xpath(xpath)
+        return (
+            self.follow(
+                url=url,
+                callback=callback,
+                method=method,
+                headers=headers,
+                body=body,
+                cookies=cookies,
+                meta=meta,
+                encoding=encoding,
+                priority=priority,
+                dont_filter=dont_filter,
+                errback=errback,
+                cb_kwargs=cb_kwargs,
+            )
+            for url in urls
+        )
+
 
 def _url_from_selector(sel):
     # type: (parsel.Selector) -> str
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index cd5c3486e..134856c78 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -143,6 +143,8 @@ class BaseResponseTest(unittest.TestCase):
             r.css('body')
             r.xpath('//body')
 
+    # Response.follow
+
     def test_follow_url_absolute(self):
         self._assert_followed_url('http://foo.example.com',
                                   'http://foo.example.com')
@@ -166,6 +168,64 @@ class BaseResponseTest(unittest.TestCase):
     def test_follow_whitespace_link(self):
         self._assert_followed_url(Link('http://example.com/foo '),
                                   'http://example.com/foo%20')
+
+    # Response.follow_all
+
+    def test_follow_all_absolute(self):
+        url_list = ['http://example.org', 'http://www.example.org',
+                    'http://example.com', 'http://www.example.com']
+        self._assert_followed_all_urls(url_list, url_list)
+
+    def test_follow_all_relative(self):
+        relative = ['foo', 'bar', 'foo/bar', 'bar/foo']
+        absolute = [
+            'http://example.com/foo',
+            'http://example.com/bar',
+            'http://example.com/foo/bar',
+            'http://example.com/bar/foo',
+        ]
+        self._assert_followed_all_urls(relative, absolute)
+
+    def test_follow_all_links(self):
+        absolute = [
+            'http://example.com/foo',
+            'http://example.com/bar',
+            'http://example.com/foo/bar',
+            'http://example.com/bar/foo',
+        ]
+        links = map(Link, absolute)
+        self._assert_followed_all_urls(links, absolute)
+
+    def test_follow_all_invalid(self):
+        r = self.response_class("http://example.com")
+        with self.assertRaises(TypeError):
+            list(r.follow_all(None))
+        with self.assertRaises(TypeError):
+            list(r.follow_all(12345))
+        with self.assertRaises(ValueError):
+            list(r.follow_all([None]))
+
+    def test_follow_all_whitespace(self):
+        relative = ['foo ', 'bar ', 'foo/bar ', 'bar/foo ']
+        absolute = [
+            'http://example.com/foo%20',
+            'http://example.com/bar%20',
+            'http://example.com/foo/bar%20',
+            'http://example.com/bar/foo%20',
+        ]
+        self._assert_followed_all_urls(relative, absolute)
+
+    def test_follow_all_whitespace_links(self):
+        absolute = [
+            'http://example.com/foo ',
+            'http://example.com/bar ',
+            'http://example.com/foo/bar ',
+            'http://example.com/bar/foo ',
+        ]
+        links = map(Link, absolute)
+        expected = [u.replace(' ', '%20') for u in absolute]
+        self._assert_followed_all_urls(links, expected)
+
     def _assert_followed_url(self, follow_obj, target_url, response=None):
         if response is None:
             response = self._links_response()
@@ -173,6 +233,14 @@ class BaseResponseTest(unittest.TestCase):
         self.assertEqual(req.url, target_url)
         return req
 
+    def _assert_followed_all_urls(self, follow_obj, target_urls, response=None):
+        if response is None:
+            response = self._links_response()
+        followed = response.follow_all(follow_obj)
+        for req, target in zip(followed, target_urls):
+            self.assertEqual(req.url, target)
+            yield req
+
     def _links_response(self):
         body = get_testdata('link_extractor', 'sgml_linkextractor.html')
         resp = self.response_class('http://example.com/index', body=body)
@@ -483,6 +551,29 @@ class TextResponseTest(BaseResponseTest):
         )
         self.assertEqual(req.encoding, 'cp1251')
 
+    def test_follow_all_css(self):
+        expected = [
+            'http://example.com/sample3.html',
+            'http://example.com/innertag.html',
+        ]
+        response = self._links_response()
+        extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')]
+        self.assertEqual(expected, extracted)
+
+    def test_follow_all_xpath(self):
+        expected = [
+            'http://example.com/sample3.html',
+            'http://example.com/innertag.html',
+        ]
+        response = self._links_response()
+        extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]')
+        self.assertEqual(expected, [r.url for r in extracted])
+
+    def test_follow_all_exception(self):
+        response = self._links_response()
+        with self.assertRaises(ValueError):
+            response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')
+
 
 class HtmlResponseTest(TextResponseTest):
 

From 877ef4269e5683bf87832e816cf20a3cac352440 Mon Sep 17 00:00:00 2001
From: akhter wahab <akhter.wahab@gmail.com>
Date: Wed, 9 Oct 2019 16:03:44 +0500
Subject: [PATCH 2231/2974] Add .webm to ignored video extensions

---
 scrapy/linkextractors/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 6049c312c..5f6df9c73 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -28,7 +28,7 @@ IGNORED_EXTENSIONS = [
 
     # video
     '3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv',
-    'm4a', 'm4v', 'flv',
+    'm4a', 'm4v', 'flv', 'webm',
 
     # office suites
     'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg',

From a25a2d5ee4755046d736efa0a03898d9d18671f9 Mon Sep 17 00:00:00 2001
From: akhter wahab <akhter.wahab@gmail.com>
Date: Wed, 9 Oct 2019 16:05:39 +0500
Subject: [PATCH 2232/2974] Add .tar.xz to ignored other extensions

---
 scrapy/linkextractors/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 5f6df9c73..0405462d6 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -35,7 +35,7 @@ IGNORED_EXTENSIONS = [
     'odp',
 
     # other
-    'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar', 'dmg', 'iso', 'apk'
+    'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar', 'dmg', 'iso', 'apk', 'tar.xz'
 ]
 
 

From 532770df5226757498a941746cf94ae47043e101 Mon Sep 17 00:00:00 2001
From: akhter wahab <akhter.wahab@gmail.com>
Date: Wed, 9 Oct 2019 22:53:14 +0500
Subject: [PATCH 2233/2974] instead of .tar.xz adding .xz in others extensions

---
 scrapy/linkextractors/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 0405462d6..3c75e683d 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -35,7 +35,7 @@ IGNORED_EXTENSIONS = [
     'odp',
 
     # other
-    'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar', 'dmg', 'iso', 'apk', 'tar.xz'
+    'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar', 'dmg', 'iso', 'apk', 'xz'
 ]
 
 

From e1fa1fd8ad62cb8958d21d3a33648ed4de6af757 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 10 Oct 2019 00:36:38 -0300
Subject: [PATCH 2234/2974] TextResponse.follow_all: skip invalid links

---
 scrapy/http/response/text.py                  | 26 +++++++---
 .../sgml_linkextractor_no_href.html           | 25 ++++++++++
 tests/test_http_response.py                   | 47 ++++++++++++++++---
 3 files changed, 85 insertions(+), 13 deletions(-)
 create mode 100644 tests/sample_data/link_extractor/sgml_linkextractor_no_href.html

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 81e3f9b28..ccacce550 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -180,13 +180,27 @@ class TextResponse(Response):
 
         In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction
         within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` are accepted).
+
+        Note that when using the ``css`` or ``xpath`` parameters, this method will not produce
+        requests for selectors from which links cannot be obtained (for instance, anchor tags
+        without ``href`` attribute)
         """
-        if len(list(filter(None, (urls, css, xpath)))) > 1:
-            raise ValueError('Please supply only one of the following arguments: {urls, css, xpath}')
-        if css:
-            urls = self.css(css)
-        elif xpath:
-            urls = self.xpath(xpath)
+        arg_count = len(list(filter(None, (urls, css, xpath))))
+        if arg_count != 1:
+            raise ValueError('Please supply exactly one of the following arguments: {urls, css, xpath}')
+        if not urls:
+            urls = []
+            if css:
+                selector_method = getattr(self, 'css')
+                expression = css
+            elif xpath:
+                selector_method = getattr(self, 'xpath')
+                expression = xpath
+            for selector in selector_method(expression):
+                try:
+                    urls.append(_url_from_selector(selector))
+                except ValueError:
+                    pass
         return (
             self.follow(
                 url=url,
diff --git a/tests/sample_data/link_extractor/sgml_linkextractor_no_href.html b/tests/sample_data/link_extractor/sgml_linkextractor_no_href.html
new file mode 100644
index 000000000..0b01cede8
--- /dev/null
+++ b/tests/sample_data/link_extractor/sgml_linkextractor_no_href.html
@@ -0,0 +1,25 @@
+<html>
+  <head>
+    <base href='http://example.com' />
+    <title>Sample page with anchor tags containing no href attribute, to test the TextResponse.follow_all method</title>
+  </head>
+
+<body>
+  <div class="quote">
+    <span class="text">“The world as we have created it is a process of our
+      thinking. It cannot be changed without changing our thinking.”</span>
+    <span>
+      by <small class="author">Albert Einstein</small>
+      <a href="/author/Albert-Einstein">(about)</a>
+    </span>
+    <div id="pagination" class="pagination">
+      Tags:
+      <a href="/page/1/">Page 1</a>
+      <a>Current</a>
+      <a href="/page/3/">Page 3</a>
+      <a href="/page/4/">Page 4</a>
+    </div>
+  </div>
+</body>
+
+</html>
\ No newline at end of file
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 134856c78..6d3c5cb9d 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -198,12 +198,20 @@ class BaseResponseTest(unittest.TestCase):
 
     def test_follow_all_invalid(self):
         r = self.response_class("http://example.com")
-        with self.assertRaises(TypeError):
-            list(r.follow_all(None))
-        with self.assertRaises(TypeError):
-            list(r.follow_all(12345))
-        with self.assertRaises(ValueError):
-            list(r.follow_all([None]))
+        if self.response_class == Response:
+            with self.assertRaises(TypeError):
+                list(r.follow_all(urls=None))
+            with self.assertRaises(TypeError):
+                list(r.follow_all(urls=12345))
+            with self.assertRaises(ValueError):
+                list(r.follow_all(urls=[None]))
+        else:
+            with self.assertRaises(ValueError):
+                list(r.follow_all(urls=None))
+            with self.assertRaises(TypeError):
+                list(r.follow_all(urls=12345))
+            with self.assertRaises(ValueError):
+                list(r.follow_all(urls=[None]))
 
     def test_follow_all_whitespace(self):
         relative = ['foo ', 'bar ', 'foo/bar ', 'bar/foo ']
@@ -246,6 +254,11 @@ class BaseResponseTest(unittest.TestCase):
         resp = self.response_class('http://example.com/index', body=body)
         return resp
 
+    def _links_response_no_href(self):
+        body = get_testdata('link_extractor', 'sgml_linkextractor_no_href.html')
+        resp = self.response_class('http://example.com/index', body=body)
+        return resp
+
 
 class TextResponseTest(BaseResponseTest):
 
@@ -560,6 +573,16 @@ class TextResponseTest(BaseResponseTest):
         extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')]
         self.assertEqual(expected, extracted)
 
+    def test_follow_all_css_skip_invalid(self):
+        expected = [
+            'http://example.com/page/1/',
+            'http://example.com/page/3/',
+            'http://example.com/page/4/',
+        ]
+        response = self._links_response_no_href()
+        extracted = [r.url for r in response.follow_all(css='.pagination a')]
+        self.assertEqual(expected, extracted)
+
     def test_follow_all_xpath(self):
         expected = [
             'http://example.com/sample3.html',
@@ -569,7 +592,17 @@ class TextResponseTest(BaseResponseTest):
         extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]')
         self.assertEqual(expected, [r.url for r in extracted])
 
-    def test_follow_all_exception(self):
+    def test_follow_all_xpath_skip_invalid(self):
+        expected = [
+            'http://example.com/page/1/',
+            'http://example.com/page/3/',
+            'http://example.com/page/4/',
+        ]
+        response = self._links_response_no_href()
+        extracted = [r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')]
+        self.assertEqual(expected, extracted)
+
+    def test_follow_all_too_many_arguments(self):
         response = self._links_response()
         with self.assertRaises(ValueError):
             response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')

From f0db1b4b6601e71caafe91f7970c9bfaa3d55395 Mon Sep 17 00:00:00 2001
From: "Matsievskiy S.V" <matsievskiysv@gmail.com>
Date: Fri, 11 Oct 2019 00:55:18 +0300
Subject: [PATCH 2235/2974] update zsh completion

---
 extras/scrapy_zsh_completion | 223 ++++++++++++++++++++++++++++++++---
 1 file changed, 204 insertions(+), 19 deletions(-)

diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion
index 564991aa8..86c52c36c 100644
--- a/extras/scrapy_zsh_completion
+++ b/extras/scrapy_zsh_completion
@@ -1,25 +1,210 @@
 #compdef scrapy
-
-# zsh completion for the Scrapy command-line tool
-
 _scrapy() {
-    local curcontext="$curcontext" cmd spiders
+    local context state state_descr line
     typeset -A opt_args
-    cmd=$words[2]
-    
-    case "$cmd" in
-        crawl|edit|check)
-            spiders=$(scrapy list 2>/dev/null) || spiders=""
-            if [[ -n "$spiders" ]]; then
-                compadd `echo $spiders`
-            fi
-        ;;
-        *)
-            if [[ CURRENT -eq 2 ]]; then
-                _arguments '*: :(check crawl edit fetch genspider list parse runspider settings shell startproject version view)'
-            fi
-        ;;
+    _arguments \
+	"(- 1 *)--help[Help]" \
+	"1: :->command" \
+	"*:: :->args"
+
+    case $state in
+	command)
+	    _scrapy_cmds
+	    ;;
+	args)
+	    case $words[1] in
+		bench)
+		    _scrapy_glb_opts
+		    ;;
+		fetch)
+		    local options=(
+			'--headers[print response HTTP headers instead of body]'
+			'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
+			'--spider[use this spider]:spider:_scrapy_spiders'
+			'1::URL:_httpie_urls'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		genspider)
+		    local options=(
+			{-l,--list}'[List available templates]'
+			{-e,--edit}'[Edit spider after creating it]'
+			'--force[If the spider already exists, overwrite it with the template]'
+			{-d,--dump=}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
+			{-t,--template=}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
+			'1:name:(NAME)'
+			'2:domain:_httpie_urls'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		runspider)
+		    local options=(
+			{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
+			{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
+			'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
+			'1:spider file:_files -g \*.py'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		settings)
+		    local options=(
+			'--get=[print raw setting value]:option:(SETTING)'
+			'--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)'
+			'--getint=[print setting value, interpreted as an integer]:option:(SETTING)'
+			'--getfloat=[print setting value, interpreted as a float]:option:(SETTING)'
+			'--getlist=[print setting value, interpreted as a list]:option:(SETTING)'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		shell)
+		    local options=(
+			'-c[evaluate the code in the shell, print the result and exit]:code:(CODE)'
+			'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
+			'--spider[use this spider]:spider:_scrapy_spiders'
+			'::file:_files -g \*.http'
+			'::URL:_httpie_urls'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		startproject)
+		    local options=(
+			'1:name:(NAME)'
+			'2:dir:_dir_list'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		version)
+		    local options=(
+			{-v,--verbose}'[also display twisted/python/platform info (useful for bug reports)]'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		view)
+		    local options=(
+			'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
+			'--spider[use this spider]:spider:_scrapy_spiders'
+			'1:URL:_httpie_urls'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		check)
+		    local options=(
+			'(- 1 *)'{-l,--list}'[only list contracts, without checking them]'
+			{-v,--verbose}'[print contract tests for all spiders]'
+			'1:spider:_scrapy_spiders'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		crawl)
+		    local options=(
+			{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
+			{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
+			'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
+			'1:spider:_scrapy_spiders'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		edit)
+		    local options=(
+			'1:spider:_scrapy_spiders'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		list)
+		    _scrapy_glb_opts
+		    ;;
+		parse)
+		    local options=(
+			'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
+			'--spider[use this spider without looking for one]:spider:_scrapy_spiders'
+			'--pipelines[process items through pipelines]'
+			"--nolinks[don't show links to follow (extracted requests)]"
+			"--noitems[don't show scraped items]"
+			'--nocolour[avoid using pygments to colorize the output]'
+			{-r,--rules}'[use CrawlSpider rules to discover the callback]'
+			{-c,--callback=}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
+			{-m,--meta=}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
+			'--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)'
+			{-d,--depth=}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
+			{-v,--verbose}'[print each depth level one by one]'
+			'1:URL:_httpie_urls'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+	    esac
+	    ;;
     esac
 }
 
-_scrapy
\ No newline at end of file
+_scrapy_cmds() {
+    local -a commands project_commands
+    commands=(
+        'bench:Run quick benchmark test'
+        'fetch:Fetch a URL using the Scrapy downloader'
+        'genspider:Generate new spider using pre-defined templates'
+        'runspider:Run a self-contained spider (without creating a project)'
+        'settings:Get settings values'
+        'shell:Interactive scraping console'
+        'startproject:Create new project'
+        'version:Print Scrapy version'
+        'view:Open URL in browser, as seen by Scrapy'
+    )
+    project_commands=(
+	'check:Check spider contracts'
+	'crawl:Run a spider'
+        'edit:Edit spider'
+        'list:List available spiders'
+        'parse:Parse URL (using its spider) and print the results'
+    )
+    if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
+	commands=(${commands[@]} ${project_commands[@]})
+    fi
+    _describe -t common-commands 'common commands' commands
+}
+
+_scrapy_glb_opts() {
+    local -a options
+    options=(
+	'(- *)'{-h,--help}'[show this help message and exit]'
+	'(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files'
+	'--pidfile=[write process ID to FILE]:file:_files'
+	'--profile=[write python cProfile stats to FILE]:file:_files'
+	'(--nolog)'{-L,--loglevel=}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
+	'(-L --loglevel --logfile)--nolog[disable logging completely]'
+	'--pdb[enable pdb on failure]'
+	'*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)'
+    )
+    options=(${options[@]} "$@")
+    _arguments $options
+}
+
+_httpie_urls() {
+
+  local ret=1
+
+  if ! [[ -prefix [-+.a-z0-9]#:// ]]; then
+    local expl
+    compset -S '[^:/]*' && compstate[to_end]=''
+    _wanted url-schemas expl 'URL schema' compadd -S '' http:// https:// && ret=0
+  else
+    _urls && ret=0
+  fi
+
+  return $ret
+
+}
+
+_scrapy_spiders() {
+
+  local ret=1
+
+  if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
+      compadd -S '' $(scrapy list) && ret=0
+  else
+      compadd -S '' SPIDER && ret=0
+  fi
+
+  return $ret
+}
+
+_scrapy $@

From 12f1e468e9f071e8f5c00921d0fbada460e41c57 Mon Sep 17 00:00:00 2001
From: Purva Udai <purvaudai@localhost.localdomain>
Date: Sun, 13 Oct 2019 15:55:27 +0530
Subject: [PATCH 2236/2974] Issue #3731

---
 scrapy/extensions/feedexport.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index ce2846eba..981efee55 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -199,9 +199,9 @@ class FeedExporter(object):
 
     def __init__(self, settings):
         self.settings = settings
-        self.urifmt = settings['FEED_URI']
-        if not self.urifmt:
+        if not settings['FEED_URI']:
             raise NotConfigured
+        self.urifmt=str(settings['FEED_URI'])
         self.format = settings['FEED_FORMAT'].lower()
         self.export_encoding = settings['FEED_EXPORT_ENCODING']
         self.storages = self._load_components('FEED_STORAGES')

From b970851299bf561af7ff0bb21caca9b6c3f296af Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Mon, 14 Oct 2019 13:35:06 -0300
Subject: [PATCH 2237/2974] Update scrapy/http/response/__init__.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 scrapy/http/response/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 96359705e..cdaababac 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -143,7 +143,7 @@ class Response(object_ref):
                    dont_filter=False, errback=None, cb_kwargs=None):
         # type: (...) -> Generator[Request, None, None]
         """
-        Return an iterable of :class:`~.Request` instance to follow all links
+        Return an iterable of :class:`~.Request` instances to follow all links
         in ``urls``. It accepts the same arguments as ``Request.__init__`` method,
         but elements of ``urls`` can be relative URLs or ``scrapy.link.Link`` objects
         not only absolute URLs.

From ba840c5a6bea95333b3b78fb767ac6f092d02177 Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Mon, 14 Oct 2019 13:35:24 -0300
Subject: [PATCH 2238/2974] Update scrapy/http/response/__init__.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 scrapy/http/response/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index cdaababac..92fa01621 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -145,7 +145,7 @@ class Response(object_ref):
         """
         Return an iterable of :class:`~.Request` instances to follow all links
         in ``urls``. It accepts the same arguments as ``Request.__init__`` method,
-        but elements of ``urls`` can be relative URLs or ``scrapy.link.Link`` objects
+        but elements of ``urls`` can be relative URLs or :class:`~scrapy.link.Link` objects,
         not only absolute URLs.
 
         :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all`

From 498d33aac37d5d7a0955e9b409fc3d5ff8627dc0 Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Mon, 14 Oct 2019 13:35:54 -0300
Subject: [PATCH 2239/2974] Update scrapy/http/response/text.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 scrapy/http/response/text.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index ccacce550..bb5a4eb9d 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -183,7 +183,7 @@ class TextResponse(Response):
 
         Note that when using the ``css`` or ``xpath`` parameters, this method will not produce
         requests for selectors from which links cannot be obtained (for instance, anchor tags
-        without ``href`` attribute)
+        without an ``href`` attribute)
         """
         arg_count = len(list(filter(None, (urls, css, xpath))))
         if arg_count != 1:

From c7c54f5453eaa16f2b6d6441be68b0a272606ec9 Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Mon, 14 Oct 2019 13:47:44 -0300
Subject: [PATCH 2240/2974] Update scrapy/http/response/text.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 scrapy/http/response/text.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index bb5a4eb9d..ecb582b4a 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -179,7 +179,7 @@ class TextResponse(Response):
           ``response.css('a.my_link')[0]``.
 
         In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction
-        within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` are accepted).
+        within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted).
 
         Note that when using the ``css`` or ``xpath`` parameters, this method will not produce
         requests for selectors from which links cannot be obtained (for instance, anchor tags

From 9d5398e7f2834940cbf9c2efa312bf5d96ffba98 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 14 Oct 2019 13:57:46 -0300
Subject: [PATCH 2241/2974] TextResponse.follow_all: improve docs

---
 scrapy/http/response/text.py | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index ecb582b4a..b2907baa4 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -129,13 +129,14 @@ class TextResponse(Response):
         It accepts the same arguments as ``Request.__init__`` method,
         but ``url`` can be not only an absolute URL, but also
 
-        * a relative URL;
-        * a scrapy.link.Link object (e.g. a link extractor result);
-        * an attribute Selector (not SelectorList) - e.g.
+        * a relative URL
+        * a :class:`~scrapy.link.Link` object, e.g. the result of
+          :ref:`topics-link-extractors`
+        * a :class:`~scrapy.selector.Selector` object for a ``<link>`` or ``<a>`` element, e.g.
+          ``response.css('a.my_link')[0]``
+        * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g.
           ``response.css('a::attr(href)')[0]`` or
-          ``response.xpath('//img/@src')[0]``.
-        * a Selector for ``<a>`` or ``<link>`` element, e.g.
-          ``response.css('a.my_link')[0]``.
+          ``response.xpath('//img/@src')[0]``
 
         See :ref:`response-follow-example` for usage examples.
         """
@@ -170,13 +171,14 @@ class TextResponse(Response):
         initializer, except that each ``urls`` element does not need to be an absolute
         URL, it can be any of the following:
 
-        * a relative URL;
-        * a scrapy.link.Link object (e.g. a link extractor result);
-        * an attribute Selector (not SelectorList) - e.g.
+        * a relative URL
+        * a :class:`~scrapy.link.Link` object, e.g. the result of
+          :ref:`topics-link-extractors`
+        * a :class:`~scrapy.selector.Selector` object for a ``<link>`` or ``<a>`` element, e.g.
+          ``response.css('a.my_link')[0]``
+        * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g.
           ``response.css('a::attr(href)')[0]`` or
-          ``response.xpath('//img/@src')[0]``.
-        * a Selector for ``<a>`` or ``<link>`` element, e.g.
-          ``response.css('a.my_link')[0]``.
+          ``response.xpath('//img/@src')[0]``
 
         In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction
         within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted).
@@ -187,7 +189,7 @@ class TextResponse(Response):
         """
         arg_count = len(list(filter(None, (urls, css, xpath))))
         if arg_count != 1:
-            raise ValueError('Please supply exactly one of the following arguments: {urls, css, xpath}')
+            raise ValueError('Please supply exactly one of the following arguments: urls, css, xpath')
         if not urls:
             urls = []
             if css:

From 7b1e69dec4cb1f1c66e9df6f754cdd280cff7e71 Mon Sep 17 00:00:00 2001
From: Baron Hou <houbaron@gmail.com>
Date: Tue, 15 Oct 2019 20:51:15 +0800
Subject: [PATCH 2242/2974] =?UTF-8?q?reponse=20=E2=86=92=20response=20(#40?=
 =?UTF-8?q?79)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/topics/developer-tools.rst       | 2 +-
 docs/topics/downloader-middleware.rst | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index dcf8af365..bf14643be 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -203,7 +203,7 @@ where our quotes are coming from:
 First click on the request with the name ``scroll``. On the right 
 you can now inspect the request. In ``Headers`` you'll find details
 about the request headers, such as the URL, the method, the IP-address,
-and so on. We'll ignore the other tabs and click directly on ``Reponse``.
+and so on. We'll ignore the other tabs and click directly on ``Response``.
 
 What you should see in the ``Preview`` pane is the rendered HTML-code, 
 that is exactly what we saw when we called ``view(response)`` in the 
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 539832618..2892b9b79 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -534,7 +534,7 @@ defines the methods described below.
       :param spider: the spider which generated the request
       :type spider: :class:`~scrapy.spiders.Spider` object
 
-      :param request: the request to find cached reponse for
+      :param request: the request to find cached response for
       :type request: :class:`~scrapy.http.Request` object
 
     .. method:: store_response(spider, request, response)

From d72ed46fe01f95e77554e58588675d5a1ff47e0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 15 Oct 2019 16:03:42 +0200
Subject: [PATCH 2243/2974] Improve how extra Item API members are introduced
 in the documentation

---
 docs/topics/items.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index fbc888f47..d70e7428b 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -237,8 +237,8 @@ Item objects
 
     Return a new Item optionally initialized from the given argument.
 
-    Items replicate the standard `dict API`_, including its constructor. The
-    only additional attribute provided by Items is:
+    Items replicate the standard `dict API`_, including its constructor, and
+    also provide the following additional API members:
 
     .. automethod:: copy
 

From 2a4d4a466aa6cede4829b62c7287332a627877ed Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 15 Oct 2019 11:52:12 -0300
Subject: [PATCH 2244/2974] TextResponse.follow_all: Simplify implementation

---
 scrapy/http/response/text.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index b2907baa4..25e115bf9 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -191,14 +191,12 @@ class TextResponse(Response):
         if arg_count != 1:
             raise ValueError('Please supply exactly one of the following arguments: urls, css, xpath')
         if not urls:
-            urls = []
             if css:
-                selector_method = getattr(self, 'css')
-                expression = css
-            elif xpath:
-                selector_method = getattr(self, 'xpath')
-                expression = xpath
-            for selector in selector_method(expression):
+                selector_list = self.css(css)
+            if xpath:
+                selector_list = self.xpath(xpath)
+            urls = []
+            for selector in selector_list:
                 try:
                     urls.append(_url_from_selector(selector))
                 except ValueError:

From 2c6f7fee6456168f4870293c442258a2470b1b48 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 15 Oct 2019 13:48:14 -0300
Subject: [PATCH 2245/2974] TextResponse.follow_all: invoke Response.follow_all

---
 scrapy/http/response/text.py | 29 +++++++++++++----------------
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 25e115bf9..f782f6217 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -201,22 +201,19 @@ class TextResponse(Response):
                     urls.append(_url_from_selector(selector))
                 except ValueError:
                     pass
-        return (
-            self.follow(
-                url=url,
-                callback=callback,
-                method=method,
-                headers=headers,
-                body=body,
-                cookies=cookies,
-                meta=meta,
-                encoding=encoding,
-                priority=priority,
-                dont_filter=dont_filter,
-                errback=errback,
-                cb_kwargs=cb_kwargs,
-            )
-            for url in urls
+        return super(TextResponse, self).follow_all(
+            urls=urls,
+            callback=callback,
+            method=method,
+            headers=headers,
+            body=body,
+            cookies=cookies,
+            meta=meta,
+            encoding=encoding,
+            priority=priority,
+            dont_filter=dont_filter,
+            errback=errback,
+            cb_kwargs=cb_kwargs,
         )
 
 

From c9614a5bdd1c8a3f50eeace148c59f0b52e293a2 Mon Sep 17 00:00:00 2001
From: Bulat <xb.bolo@gmail.com>
Date: Wed, 16 Oct 2019 12:07:19 +0300
Subject: [PATCH 2246/2974] Fixed BOT_NAME documentation

---
 docs/topics/settings.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 75e0af63b..dbe3e9e44 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -229,8 +229,7 @@ BOT_NAME
 Default: ``'scrapybot'``
 
 The name of the bot implemented by this Scrapy project (also known as the
-project name). This will be used to construct the User-Agent by default, and
-also for logging.
+project name) and also for logging.
 
 It's automatically populated with your project name when you create your
 project with the :command:`startproject` command.

From 84be6a941e3c4e80802dd52e43eb8e7064892cdb Mon Sep 17 00:00:00 2001
From: Bulat <xb.bolo@gmail.com>
Date: Wed, 16 Oct 2019 14:04:07 +0300
Subject: [PATCH 2247/2974] Refactor sentence.

---
 docs/topics/settings.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index dbe3e9e44..d41381dd1 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -229,7 +229,7 @@ BOT_NAME
 Default: ``'scrapybot'``
 
 The name of the bot implemented by this Scrapy project (also known as the
-project name) and also for logging.
+project name). This name will be used for the logging too.
 
 It's automatically populated with your project name when you create your
 project with the :command:`startproject` command.

From 865d58fd1b5676dcca44b64970b5cd48fc1ab715 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Alberto=20Orejuela=20Garc=C3=ADa?=
 <libre@josealberto4444.com>
Date: Thu, 3 Oct 2019 18:06:19 +0200
Subject: [PATCH 2248/2974] Make punctuation consistent

---
 CODE_OF_CONDUCT.md |  2 +-
 README.rst         | 18 +++++++++---------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
index d477168eb..d1cd3e517 100644
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -68,7 +68,7 @@ members of the project's leadership.
 ## Attribution
 
 This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at [http://contributor-covenant.org/version/1/4][version]
+available at [http://contributor-covenant.org/version/1/4][version].
 
 [homepage]: http://contributor-covenant.org
 [version]: http://contributor-covenant.org/version/1/4/
diff --git a/README.rst b/README.rst
index bd82bff06..87eaac2af 100644
--- a/README.rst
+++ b/README.rst
@@ -34,8 +34,8 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
 crawl websites and extract structured data from their pages. It can be used for
 a wide range of purposes, from data mining to monitoring and automated testing.
 
-For more information including a list of features check the Scrapy homepage at:
-https://scrapy.org
+Check the Scrapy homepage at https://scrapy.org for more information,
+including a list of features.
 
 Requirements
 ============
@@ -50,8 +50,8 @@ The quick way::
 
     pip install scrapy
 
-For more details see the install section in the documentation:
-https://docs.scrapy.org/en/latest/intro/install.html
+See the install section in the documentation at
+https://docs.scrapy.org/en/latest/intro/install.html for more details.
 
 Documentation
 =============
@@ -62,17 +62,17 @@ directory.
 Releases
 ========
 
-You can find release notes at https://docs.scrapy.org/en/latest/news.html
+You can check https://docs.scrapy.org/en/latest/news.html for release notes.
 
 Community (blog, twitter, mail list, IRC)
 =========================================
 
-See https://scrapy.org/community/
+See https://scrapy.org/community/ for details.
 
 Contributing
 ============
 
-See https://docs.scrapy.org/en/master/contributing.html
+See https://docs.scrapy.org/en/master/contributing.html for details.
 
 Code of Conduct
 ---------------
@@ -86,9 +86,9 @@ Please report unacceptable behavior to opensource@scrapinghub.com.
 Companies using Scrapy
 ======================
 
-See https://scrapy.org/companies/
+See https://scrapy.org/companies/ for a list.
 
 Commercial Support
 ==================
 
-See https://scrapy.org/support/
+See https://scrapy.org/support/ for details.

From 6df6b6dd6a5afd9172c03f6b9e8ce8e180867339 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 21 Oct 2019 03:56:45 -0300
Subject: [PATCH 2249/2974] Initializer -> __init__

---
 scrapy/http/response/text.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index f782f6217..74017b5aa 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -167,9 +167,9 @@ class TextResponse(Response):
         # type: (...) -> Generator[Request, None, None]
         """
         A generator that produces :class:`~.Request` instances to follow all
-        links in ``urls``. It accepts the same arguments as the :class:`~.Request`
-        initializer, except that each ``urls`` element does not need to be an absolute
-        URL, it can be any of the following:
+        links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s
+        ``__init__`` method, except that each ``urls`` element does not need to be
+        an absolute URL, it can be any of the following:
 
         * a relative URL
         * a :class:`~scrapy.link.Link` object, e.g. the result of

From 0fbd1ff4a91c68e552a3f919824c60437fc9a141 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 21 Oct 2019 14:06:45 +0200
Subject: [PATCH 2250/2974] =?UTF-8?q?constructor=20=E2=86=92=20=5F=5Finit?=
 =?UTF-8?q?=5F=5F=20method?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/topics/link-extractors.rst   | 6 +++---
 scrapy/linkextractors/lxmlhtml.py | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index f9936a498..2119cb8f8 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -6,9 +6,9 @@ Link Extractors
 
 A link extractor is an object that extracts links from responses.
 
-The constructor of :class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`
-takes settings that determine which links may be extracted.
-:class:`LxmlLinkExtractor.extract_links
+The ``__init__`` method of
+:class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that
+determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links
 <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` returns a
 list of matching :class:`scrapy.link.Link` objects from a
 :class:`~scrapy.http.Response` object.
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 41091ba23..37003720f 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -120,8 +120,8 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
         """Returns a list of :class:`~scrapy.link.Link` objects from the
         specified :class:`response <scrapy.http.Response>`.
 
-        Only links that match the settings passed to the link extractor
-        constructor are returned.
+        Only links that match the settings passed to the ``__init__`` method of
+        the link extractor are returned.
 
         Duplicate links are omitted.
         """

From 68a7d05ed86331ae70778c83a131e6dcc8b4ffc8 Mon Sep 17 00:00:00 2001
From: Ammar Najjar <najjarammar@protonmail.com>
Date: Mon, 21 Oct 2019 15:42:24 +0200
Subject: [PATCH 2251/2974] docs: use __init__ method instead of constructor

Issue #4086
---
 docs/conf.py                      |  2 +-
 docs/news.rst                     | 10 ++++----
 docs/topics/email.rst             |  4 ++--
 docs/topics/exporters.rst         | 40 +++++++++++++++----------------
 docs/topics/extensions.rst        |  2 +-
 docs/topics/items.rst             |  8 +++----
 docs/topics/loaders.rst           | 26 ++++++++++----------
 docs/topics/request-response.rst  | 14 +++++------
 scrapy/exporters.py               |  2 +-
 scrapy/extensions/feedexport.py   |  2 +-
 scrapy/utils/datatypes.py         |  2 +-
 scrapy/utils/misc.py              |  6 ++---
 scrapy/utils/python.py            |  2 +-
 sep/sep-009.rst                   | 12 +++++-----
 tests/test_http_request.py        |  6 ++---
 tests/test_http_response.py       |  2 +-
 tests/test_loader.py              | 12 +++++-----
 tests/test_spider.py              |  4 ++--
 tests/test_utils_misc/__init__.py | 10 ++++----
 19 files changed, 83 insertions(+), 83 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 34dd5bcb7..6ab5959d5 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -237,7 +237,7 @@ coverage_ignore_pyobjects = [
     r'\bContractsManager\b$',
 
     # For default contracts we only want to document their general purpose in
-    # their constructor, the methods they reimplement to achieve that purpose
+    # their __init__ method, the methods they reimplement to achieve that purpose
     # should be irrelevant to developers using those contracts.
     r'\w+Contract\.(adjust_request_args|(pre|post)_process)$',
 
diff --git a/docs/news.rst b/docs/news.rst
index 59317f5eb..c1f548060 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -84,12 +84,12 @@ New features
     convenient way to build JSON requests (:issue:`3504`, :issue:`3505`)
 
 *   A ``process_request`` callback passed to the :class:`~scrapy.spiders.Rule`
-    constructor now receives the :class:`~scrapy.http.Response` object that
+    ``__init__`` method now receives the :class:`~scrapy.http.Response` object that
     originated the request as its second argument (:issue:`3682`)
 
 *   A new ``restrict_text`` parameter for the
     :attr:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
-    constructor allows filtering links by linking text (:issue:`3622`,
+    ``__init__`` method allows filtering links by linking text (:issue:`3622`,
     :issue:`3635`)
 
 *   A new :setting:`FEED_STORAGE_S3_ACL` setting allows defining a custom ACL
@@ -255,7 +255,7 @@ The following deprecated APIs have been removed (:issue:`3578`):
 
 *   From :class:`~scrapy.selector.Selector`:
 
-    *   ``_root`` (both the constructor argument and the object property, use
+    *   ``_root`` (both the ``__init__`` method argument and the object property, use
         ``root``)
 
     *   ``extract_unquoted`` (use ``getall``)
@@ -2479,7 +2479,7 @@ Scrapy changes:
 - removed ``ENCODING_ALIASES`` setting, as encoding auto-detection has been moved to the `w3lib`_ library
 - promoted :ref:`topics-djangoitem` to main contrib
 - LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
-- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the constructor
+- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method
 - replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
 - removed signal: ``scrapy.mail.mail_sent``
 - removed ``TRACK_REFS`` setting, now :ref:`trackrefs <topics-leaks-trackrefs>` is always enabled
@@ -2693,7 +2693,7 @@ API changes
 - ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231)
 - Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default)
 - Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
-- Removed Spider Manager ``load()`` method. Now spiders are loaded in the constructor itself.
+- Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself.
 - Changes to Scrapy Manager (now called "Crawler"):
    - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler``
    - ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler``
diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index 949cdc638..12eedf2cd 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -21,7 +21,7 @@ Quick example
 =============
 
 There are two ways to instantiate the mail sender. You can instantiate it using
-the standard constructor::
+the standard ``__init__`` method::
 
     from scrapy.mail import MailSender
     mailer = MailSender()
@@ -111,7 +111,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
 Mail settings
 =============
 
-These settings define the default constructor values of the :class:`MailSender`
+These settings define the default ``__init__`` method values of the :class:`MailSender`
 class, and can be used to configure e-mail notifications in your project without
 writing any code (for those extensions and code that uses :class:`MailSender`).
 
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index a698a6a4e..d7ab7a5cc 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -87,8 +87,8 @@ described next.
 1. Declaring a serializer in the field
 --------------------------------------
 
-If you use :class:`~.Item` you can declare a serializer in the 
-:ref:`field metadata <topics-items-fields>`. The serializer must be 
+If you use :class:`~.Item` you can declare a serializer in the
+:ref:`field metadata <topics-items-fields>`. The serializer must be
 a callable which receives a value and returns its serialized form.
 
 Example::
@@ -144,7 +144,7 @@ BaseItemExporter
    defining what fields to export, whether to export empty fields, or which
    encoding to use.
 
-   These features can be configured through the constructor arguments which
+   These features can be configured through the `__init__` method arguments which
    populate their respective instance attributes: :attr:`fields_to_export`,
    :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
 
@@ -246,8 +246,8 @@ XmlItemExporter
    :param item_element: The name of each item element in the exported XML.
    :type item_element: str
 
-   The additional keyword arguments of this constructor are passed to the
-   :class:`BaseItemExporter` constructor.
+   The additional keyword arguments of this `__init__` method are passed to the
+   :class:`BaseItemExporter` `__init__` method
 
    A typical output of this exporter would be::
 
@@ -306,9 +306,9 @@ CsvItemExporter
       multi-valued fields, if found.
    :type include_headers_line: str
 
-   The additional keyword arguments of this constructor are passed to the
-   :class:`BaseItemExporter` constructor, and the leftover arguments to the
-   `csv.writer`_ constructor, so you can use any ``csv.writer`` constructor
+   The additional keyword arguments of this `__init__` method are passed to the
+   :class:`BaseItemExporter` `__init__` method, and the leftover arguments to the
+   `csv.writer`_ `__init__` method, so you can use any ``csv.writer`` `__init__` method
    argument to customize this exporter.
 
    A typical output of this exporter would be::
@@ -334,8 +334,8 @@ PickleItemExporter
 
    For more information, refer to the `pickle module documentation`_.
 
-   The additional keyword arguments of this constructor are passed to the
-   :class:`BaseItemExporter` constructor.
+   The additional keyword arguments of this `__init__` method are passed to the
+   :class:`BaseItemExporter` `__init__` method.
 
    Pickle isn't a human readable format, so no output examples are provided.
 
@@ -351,8 +351,8 @@ PprintItemExporter
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
 
-   The additional keyword arguments of this constructor are passed to the
-   :class:`BaseItemExporter` constructor.
+   The additional keyword arguments of this `__init__` method are passed to the
+   :class:`BaseItemExporter` `__init__` method
 
    A typical output of this exporter would be::
 
@@ -367,10 +367,10 @@ JsonItemExporter
 .. class:: JsonItemExporter(file, \**kwargs)
 
    Exports Items in JSON format to the specified file-like object, writing all
-   objects as a list of objects. The additional constructor arguments are
-   passed to the :class:`BaseItemExporter` constructor, and the leftover
-   arguments to the `JSONEncoder`_ constructor, so you can use any
-   `JSONEncoder`_ constructor argument to customize this exporter.
+   objects as a list of objects. The additional `__init__` method arguments are
+   passed to the :class:`BaseItemExporter` `__init__` method, and the leftover
+   arguments to the `JSONEncoder`_ `__init__` method, so you can use any
+   `JSONEncoder`_ `__init__` method argument to customize this exporter.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -398,10 +398,10 @@ JsonLinesItemExporter
 .. class:: JsonLinesItemExporter(file, \**kwargs)
 
    Exports Items in JSON format to the specified file-like object, writing one
-   JSON-encoded item per line. The additional constructor arguments are passed
-   to the :class:`BaseItemExporter` constructor, and the leftover arguments to
-   the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_
-   constructor argument to customize this exporter.
+   JSON-encoded item per line. The additional `__init__` method arguments are passed
+   to the :class:`BaseItemExporter` `__init__` method and the leftover arguments to
+   the `JSONEncoder`_ `__init__` method, so you can use any `JSONEncoder`_
+   `__init__` method argument to customize this exporter.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 72c2290b5..0a7455ec9 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -28,7 +28,7 @@ Loading & activating extensions
 
 Extensions are loaded and activated at startup by instantiating a single
 instance of the extension class. Therefore, all the extension initialization
-code must be performed in the class constructor (``__init__`` method).
+code must be performed in the class ``__init__`` method.
 
 To make an extension available, add it to the :setting:`EXTENSIONS` setting in
 your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index d70e7428b..8ea2635cc 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -16,12 +16,12 @@ especially in a larger project with many spiders.
 To define common output data format Scrapy provides the :class:`Item` class.
 :class:`Item` objects are simple containers used to collect the scraped data.
 They provide a `dictionary-like`_ API with a convenient syntax for declaring
-their available fields. 
+their available fields.
 
-Various Scrapy components use extra information provided by Items: 
+Various Scrapy components use extra information provided by Items:
 exporters look at declared fields to figure out columns to export,
 serialization can be customized using Item fields metadata, :mod:`trackref`
-tracks Item instances to help find memory leaks 
+tracks Item instances to help find memory leaks
 (see :ref:`topics-leaks-trackrefs`), etc.
 
 .. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
@@ -237,7 +237,7 @@ Item objects
 
     Return a new Item optionally initialized from the given argument.
 
-    Items replicate the standard `dict API`_, including its constructor, and
+    Items replicate the standard `dict API`_, including its `__init__` method and
     also provide the following additional API members:
 
     .. automethod:: copy
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 1c2f1da4d..db1175a13 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -26,7 +26,7 @@ Using Item Loaders to populate items
 
 To use an Item Loader, you must first instantiate it. You can either
 instantiate it with a dict-like object (e.g. Item or dict) or without one, in
-which case an Item is automatically instantiated in the Item Loader constructor
+which case an Item is automatically instantiated in the Item Loader ``__init__`` method
 using the Item class specified in the :attr:`ItemLoader.default_item_class`
 attribute.
 
@@ -265,7 +265,7 @@ There are several ways to modify Item Loader context values:
       loader.context['unit'] = 'cm'
 
 2. On Item Loader instantiation (the keyword arguments of Item Loader
-   constructor are stored in the Item Loader context)::
+   ``__init__`` methodare stored in the Item Loader context)::
 
       loader = ItemLoader(product, unit='cm')
 
@@ -494,7 +494,7 @@ ItemLoader objects
     .. attribute:: default_item_class
 
         An Item class (or factory), used to instantiate items when not given in
-        the constructor.
+        the `__init__` method
 
     .. attribute:: default_input_processor
 
@@ -509,15 +509,15 @@ ItemLoader objects
     .. attribute:: default_selector_class
 
         The class used to construct the :attr:`selector` of this
-        :class:`ItemLoader`, if only a response is given in the constructor.
-        If a selector is given in the constructor this attribute is ignored.
+        :class:`ItemLoader`, if only a response is given in the `__init__` method
+        If a selector is given in the `__init__` method this attribute is ignored.
         This attribute is sometimes overridden in subclasses.
 
     .. attribute:: selector
 
         The :class:`~scrapy.selector.Selector` object to extract data from.
-        It's either the selector given in the constructor or one created from
-        the response given in the constructor using the
+        It's either the selector given in the `__init__` methodor one created from
+        the response given in the `__init__` methodusing the
         :attr:`default_selector_class`. This attribute is meant to be
         read-only.
 
@@ -642,7 +642,7 @@ Here is a list of all built-in processors:
 .. class:: Identity
 
     The simplest processor, which doesn't do anything. It returns the original
-    values unchanged. It doesn't receive any constructor arguments, nor does it
+    values unchanged. It doesn't receive any `__init__` method arguments, nor does it
     accept Loader contexts.
 
     Example::
@@ -656,7 +656,7 @@ Here is a list of all built-in processors:
 
     Returns the first non-null/non-empty value from the values received,
     so it's typically used as an output processor to single-valued fields.
-    It doesn't receive any constructor arguments, nor does it accept Loader contexts.
+    It doesn't receive any `__init__` methodarguments, nor does it accept Loader contexts.
 
     Example::
 
@@ -667,7 +667,7 @@ Here is a list of all built-in processors:
 
 .. class:: Join(separator=u' ')
 
-    Returns the values joined with the separator given in the constructor, which
+    Returns the values joined with the separator given in the `__init__` method which
     defaults to ``u' '``. It doesn't accept Loader contexts.
 
     When using the default separator, this processor is equivalent to the
@@ -705,7 +705,7 @@ Here is a list of all built-in processors:
     those which do, this processor will pass the currently active :ref:`Loader
     context <topics-loaders-context>` through that parameter.
 
-    The keyword arguments passed in the constructor are used as the default
+    The keyword arguments passed in the `__init__` methodare used as the default
     Loader context values passed to each function call. However, the final
     Loader context values passed to functions are overridden with the currently
     active Loader context accessible through the :meth:`ItemLoader.context`
@@ -749,12 +749,12 @@ Here is a list of all built-in processors:
         ['HELLO, 'THIS', 'IS', 'SCRAPY']
 
     As with the Compose processor, functions can receive Loader contexts, and
-    constructor keyword arguments are used as default context values. See
+    `__init__` method keyword arguments are used as default context values. See
     :class:`Compose` processor for more info.
 
 .. class:: SelectJmes(json_path)
 
-    Queries the value using the json path provided to the constructor and returns the output.
+    Queries the value using the json path provided to the `__init__` methodand returns the output.
     Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
     This processor takes only one input at a time.
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 727c67482..d253064f2 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -137,7 +137,7 @@ Request objects
 
         A string containing the URL of this request. Keep in mind that this
         attribute contains the escaped URL, so it can differ from the URL passed in
-        the constructor.
+        the `__init__` method
 
         This attribute is read-only. To change the URL of a Request use
         :meth:`replace`.
@@ -400,7 +400,7 @@ fields with form data from :class:`Response` objects.
 
 .. class:: FormRequest(url, [formdata, ...])
 
-    The :class:`FormRequest` class adds a new keyword parameter to the constructor. The
+    The :class:`FormRequest` class adds a new keyword parameter to the `__init__` method The
     remaining arguments are the same as for the :class:`Request` class and are
     not documented here.
 
@@ -473,7 +473,7 @@ fields with form data from :class:`Response` objects.
        :type dont_click: boolean
 
        The other parameters of this class method are passed directly to the
-       :class:`FormRequest` constructor.
+       :class:`FormRequest` `__init__` method
 
        .. versionadded:: 0.10.3
           The ``formname`` parameter.
@@ -547,7 +547,7 @@ dealing with JSON requests.
 
 .. class:: JsonRequest(url, [... data, dumps_kwargs])
 
-   The :class:`JsonRequest` class adds two new keyword parameters to the constructor. The
+   The :class:`JsonRequest` class adds two new keyword parameters to the `__init__` method The
    remaining arguments are the same as for the :class:`Request` class and are
    not documented here.
 
@@ -556,7 +556,7 @@ dealing with JSON requests.
 
    :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body.
       if :attr:`Request.body` argument is provided this parameter will be ignored.
-      if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be 
+      if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
       set to ``'POST'`` automatically.
    :type data: JSON serializable object
 
@@ -721,7 +721,7 @@ TextResponse objects
     :class:`Response` class, which is meant to be used only for binary data,
     such as images, sounds or any media file.
 
-    :class:`TextResponse` objects support a new constructor argument, in
+    :class:`TextResponse` objects support a new `__init__` method argument, in
     addition to the base :class:`Response` objects. The remaining functionality
     is the same as for the :class:`Response` class and is not documented here.
 
@@ -755,7 +755,7 @@ TextResponse objects
        A string with the encoding of this response. The encoding is resolved by
        trying the following mechanisms, in order:
 
-       1. the encoding passed in the constructor ``encoding`` argument
+       1. the encoding passed in the `__init__` method`` ``encoding`` argument
 
        2. the encoding declared in the Content-Type HTTP header. If this
           encoding is not valid (ie. unknown), it is ignored and the next
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 6fc87ed18..2fdc86b63 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -31,7 +31,7 @@ class BaseItemExporter(object):
     def _configure(self, options, dont_fail=False):
         """Configure the exporter by poping options from the ``options`` dict.
         If dont_fail is set, it won't raise an exception on unexpected options
-        (useful for using with keyword arguments in subclasses constructors)
+        (useful for using with keyword arguments in subclasses __init__ methods)
         """
         self.encoding = options.pop('encoding', None)
         self.fields_to_export = options.pop('fields_to_export', None)
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index ce2846eba..655c10482 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -106,7 +106,7 @@ class S3FeedStorage(BlockingFeedStorage):
                 warnings.warn(
                     "Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
                     "without AWS keys is deprecated. Please supply credentials or "
-                    "use the `from_crawler()` constructor.",
+                    "use the `from_crawler()` __init__ method.",
                     category=ScrapyDeprecationWarning,
                     stacklevel=2
                 )
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index df2b99c28..0dfee1b27 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -246,7 +246,7 @@ class CaselessDict(dict):
 class MergeDict(object):
     """
     A simple class for creating new "virtual" dictionaries that actually look
-    up values in more than one dictionary, passed in the constructor.
+    up values in more than one dictionary, passed in the __init__ method.
 
     If a key appears in more than one of the given dictionaries, only the
     first occurrence will be used.
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index f638adb25..c892bd438 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -123,14 +123,14 @@ def rel_has_nofollow(rel):
 
 def create_instance(objcls, settings, crawler, *args, **kwargs):
     """Construct a class instance using its ``from_crawler`` or
-    ``from_settings`` constructors, if available.
+    ``from_settings`` __init__ method, if available.
 
     At least one of ``settings`` and ``crawler`` needs to be different from
     ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
-    If ``crawler`` is ``None``, only the ``from_settings`` constructor will be
+    If ``crawler`` is ``None``, only the ``from_settings`` __init__ method will be
     tried.
 
-    ``*args`` and ``**kwargs`` are forwarded to the constructors.
+    ``*args`` and ``**kwargs`` are forwarded to the __init__ methods.
 
     Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
     """
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index c6140f885..0d645543a 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -303,7 +303,7 @@ class WeakKeyCache(object):
 def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
     """Return a (new) dict with unicode keys (and values when "keys_only" is
     False) of the given dict converted to strings. ``dct_or_tuples`` can be a
-    dict or a list of tuples, like any dict constructor supports.
+    dict or a list of tuples, like any dict __init__ method supports.
     """
     d = {}
     for k, v in six.iteritems(dict(dct_or_tuples)):
diff --git a/sep/sep-009.rst b/sep/sep-009.rst
index 232a536a8..929cedc61 100644
--- a/sep/sep-009.rst
+++ b/sep/sep-009.rst
@@ -38,7 +38,7 @@ singletons members of that object, as explained below:
   ``scrapy.core.manager.ExecutionManager``) - instantiated with a ``Settings``
   object
 
-   - **crawler.settings**: ``scrapy.conf.Settings`` instance (passed in the constructor)
+   - **crawler.settings**: ``scrapy.conf.Settings`` instance (passed in the ``__init__`` method)
    - **crawler.extensions**: ``scrapy.extension.ExtensionManager`` instance
    - **crawler.engine**: ``scrapy.core.engine.ExecutionEngine`` instance
       - ``crawler.engine.scheduler``
@@ -55,7 +55,7 @@ singletons members of that object, as explained below:
      ``STATS_CLASS`` setting)
    - **crawler.log**: Logger class with methods replacing the current
      ``scrapy.log`` functions. Logging would be started (if enabled) on
-     ``Crawler`` constructor, so no log starting functions are required.
+     ``Crawler`` __init__ method, so no log starting functions are required.
 
       - ``crawler.log.msg``
    - **crawler.signals**: signal handling
@@ -69,12 +69,12 @@ Required code changes after singletons removal
 ==============================================
 
 All components (extensions, middlewares, etc) will receive this ``Crawler``
-object in their constructors, and this will be the only mechanism for accessing
+object in their ``__init__`` methods, and this will be the only mechanism for accessing
 any other components (as opposed to importing each singleton from their
 respective module). This will also serve to stabilize the core API, something
 which we haven't documented so far (partly because of this).
 
-So, for a typical middleware constructor code, instead of this:
+So, for a typical middleware ``__init__`` method code, instead of this:
 
 ::
 
@@ -125,13 +125,13 @@ Open issues to resolve
 
 - Should we pass ``Settings`` object to ``ScrapyCommand.add_options()``?
 - How should spiders access settings?
-   - Option 1. Pass ``Crawler`` object to spider constructors too
+   - Option 1. Pass ``Crawler`` object to spider ``__init__`` methods too
       - pro: one way to access all components (settings and signals being the
         most relevant to spiders)
       - con?: spider code can access (and control) any crawler component -
         since we don't want to support spiders messing with the crawler (write
         an extension or spider middleware if you need that)
-   - Option 2. Pass ``Settings`` object to spider constructors, which would
+   - Option 2. Pass ``Settings`` object to spider ``__init__`` methods, which would
      then be accessed through ``self.settings``, like logging which is accessed
      through ``self.log``
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 16d7a1cb8..96a4fb141 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -25,7 +25,7 @@ class RequestTest(unittest.TestCase):
     default_meta = {}
 
     def test_init(self):
-        # Request requires url in the constructor
+        # Request requires url in the __init__ method
         self.assertRaises(Exception, self.request_class)
 
         # url argument must be basestring
@@ -500,7 +500,7 @@ class FormRequestTest(RequestTest):
                 formdata=(('foo', 'bar'), ('foo', 'baz')))
         self.assertEqual(urlparse(req.url).hostname, 'www.example.com')
         self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz')
-    
+
     def test_from_response_override_duplicate_form_key(self):
         response = _buildresponse(
             """<form action="get.php" method="POST">
@@ -657,7 +657,7 @@ class FormRequestTest(RequestTest):
         req = self.request_class.from_response(response, dont_click=True)
         fs = _qs(req)
         self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']})
-    
+
     def test_from_response_clickdata_does_not_ignore_image(self):
         response = _buildresponse(
             """<form>
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index cd5c3486e..dfc8562f3 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -534,7 +534,7 @@ class XmlResponseTest(TextResponseTest):
         r2 = self.response_class("http://www.example.com", body=body)
         self._assert_response_values(r2, 'iso-8859-1', body)
 
-        # make sure replace() preserves the explicit encoding passed in the constructor
+        # make sure replace() preserves the explicit encoding passed in the __init__ method
         body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
         r3 = self.response_class("http://www.example.com", body=body, encoding='utf-8')
         body2 = b"New body"
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 2725b001a..bcc1e6421 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -548,11 +548,11 @@ class SelectortemLoaderTest(unittest.TestCase):
     </html>
     """)
 
-    def test_constructor(self):
+    def test_init_method(self):
         l = TestItemLoader()
         self.assertEqual(l.selector, None)
 
-    def test_constructor_errors(self):
+    def test_init_method_errors(self):
         l = TestItemLoader()
         self.assertRaises(RuntimeError, l.add_xpath, 'url', '//a/@href')
         self.assertRaises(RuntimeError, l.replace_xpath, 'url', '//a/@href')
@@ -561,7 +561,7 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertRaises(RuntimeError, l.replace_css, 'name', '#name::text')
         self.assertRaises(RuntimeError, l.get_css, '#name::text')
 
-    def test_constructor_with_selector(self):
+    def test_init_method_with_selector(self):
         sel = Selector(text=u"<html><body><div>marta</div></body></html>")
         l = TestItemLoader(selector=sel)
         self.assertIs(l.selector, sel)
@@ -569,7 +569,7 @@ class SelectortemLoaderTest(unittest.TestCase):
         l.add_xpath('name', '//div/text()')
         self.assertEqual(l.get_output_value('name'), [u'Marta'])
 
-    def test_constructor_with_selector_css(self):
+    def test_init_method_with_selector_css(self):
         sel = Selector(text=u"<html><body><div>marta</div></body></html>")
         l = TestItemLoader(selector=sel)
         self.assertIs(l.selector, sel)
@@ -577,14 +577,14 @@ class SelectortemLoaderTest(unittest.TestCase):
         l.add_css('name', 'div::text')
         self.assertEqual(l.get_output_value('name'), [u'Marta'])
 
-    def test_constructor_with_response(self):
+    def test_init_method_with_response(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
 
         l.add_xpath('name', '//div/text()')
         self.assertEqual(l.get_output_value('name'), [u'Marta'])
 
-    def test_constructor_with_response_css(self):
+    def test_init_method_with_response_css(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
 
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 2220b8ffc..64ff40b61 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -42,12 +42,12 @@ class SpiderTest(unittest.TestCase):
         self.assertEqual(list(start_requests), [])
 
     def test_spider_args(self):
-        """Constructor arguments are assigned to spider attributes"""
+        """__init__ method arguments are assigned to spider attributes"""
         spider = self.spider_class('example.com', foo='bar')
         self.assertEqual(spider.foo, 'bar')
 
     def test_spider_without_name(self):
-        """Constructor arguments are assigned to spider attributes"""
+        """__init__ method arguments are assigned to spider attributes"""
         self.assertRaises(ValueError, self.spider_class)
         self.assertRaises(ValueError, self.spider_class, somearg='foo')
 
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index e109d5343..457a2aa78 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -109,11 +109,11 @@ class UtilsMiscTestCase(unittest.TestCase):
             else:
                 mock.assert_called_once_with(*args, **kwargs)
 
-        # Check usage of correct constructor using four mocks:
-        #   1. with no alternative constructors
-        #   2. with from_settings() constructor
-        #   3. with from_crawler() constructor
-        #   4. with from_settings() and from_crawler() constructor
+        # Check usage of correct __init__ method using four mocks:
+        #   1. with no alternative __init__ methods
+        #   2. with from_settings() __init__ method
+        #   3. with from_crawler() __init__ method
+        #   4. with from_settings() and from_crawler() __init__ method
         spec_sets = ([], ['from_settings'], ['from_crawler'],
                      ['from_settings', 'from_crawler'])
         for specs in spec_sets:

From ad6075440c0285d903ec7238354d093ea300b8f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 21 Oct 2019 19:00:03 +0200
Subject: [PATCH 2252/2974] Fix references to Python types in parameter type
 fields

---
 docs/topics/contracts.rst        |  2 +-
 docs/topics/email.rst            | 10 ++++-----
 docs/topics/exporters.rst        |  2 +-
 docs/topics/leaks.rst            |  2 +-
 docs/topics/link-extractors.rst  | 14 ++++++-------
 docs/topics/loaders.rst          |  6 +++---
 docs/topics/request-response.rst | 36 ++++++++++++++++----------------
 scrapy/crawler.py                |  2 +-
 scrapy/robotstxt.py              |  4 ++--
 scrapy/settings/__init__.py      | 33 ++++++++++++-----------------
 scrapy/signalmanager.py          |  2 +-
 11 files changed, 53 insertions(+), 60 deletions(-)

diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 62f9a743b..15443f4cc 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -85,7 +85,7 @@ override three methods:
 .. class:: Contract(method, \*args)
 
     :param method: callback function to which the contract is associated
-    :type method: function
+    :type method: collections.abc.Callable
 
     :param args: list of arguments passed into the docstring (whitespace
         separated)
diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index 949cdc638..73b1bdc3b 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -63,10 +63,10 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
     :type smtpport: int
 
     :param smtptls: enforce using SMTP STARTTLS
-    :type smtptls: boolean
+    :type smtptls: bool
 
     :param smtpssl: enforce using a secure SSL connection
-    :type smtpssl: boolean
+    :type smtpssl: bool
 
     .. classmethod:: from_settings(settings)
 
@@ -81,13 +81,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
         Send email to the given recipients.
 
         :param to: the e-mail recipients
-        :type to: str or list of str
+        :type to: str or list
 
         :param subject: the subject of the e-mail
         :type subject: str
 
         :param cc: the e-mails to CC
-        :type cc: str or list of str
+        :type cc: str or list
 
         :param body: the e-mail body
         :type body: str
@@ -97,7 +97,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
           appear on the e-mail's attachment, ``mimetype`` is the mimetype of the
           attachment and ``file_object`` is a readable file object with the
           contents of the attachment
-        :type attachs: iterable
+        :type attachs: collections.abc.Iterable
 
         :param mimetype: the MIME type of the e-mail
         :type mimetype: str
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index a698a6a4e..da304922d 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -300,7 +300,7 @@ CsvItemExporter
    :param include_headers_line: If enabled, makes the exporter output a header
       line with the field names taken from
       :attr:`BaseItemExporter.fields_to_export` or the first exported item fields.
-   :type include_headers_line: boolean
+   :type include_headers_line: bool
 
    :param join_multivalued: The char (or chars) that will be used for joining
       multi-valued fields, if found.
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 8278e9849..657f1cc61 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -179,7 +179,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
 
     :param ignore: if given, all objects from the specified class (or tuple of
         classes) will be ignored.
-    :type ignore: class or classes tuple
+    :type ignore: type or tuple
 
 .. function:: get_oldest(class_name)
 
diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 713a94e10..13b9ad7a5 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -59,13 +59,13 @@ LxmlLinkExtractor
     :param allow: a single regular expression (or list of regular expressions)
         that the (absolute) urls must match in order to be extracted. If not
         given (or empty), it will match all links.
-    :type allow: a regular expression (or list of)
+    :type allow: str or list
 
     :param deny: a single regular expression (or list of regular expressions)
         that the (absolute) urls must match in order to be excluded (ie. not
         extracted). It has precedence over the ``allow`` parameter. If not
         given (or empty) it won't exclude any links.
-    :type deny: a regular expression (or list of)
+    :type deny: str or list
 
     :param allow_domains: a single value or a list of string containing
         domains which will be considered for extracting the links
@@ -97,7 +97,7 @@ LxmlLinkExtractor
         that the link's text must match in order to be extracted. If not
         given (or empty), it will match all links. If a list of regular expressions is
         given, the link will be extracted if it matches at least one.
-    :type restrict_text: a regular expression (or list of)
+    :type restrict_text: str or list
 
     :param tags: a tag or a list of tags to consider when extracting links.
         Defaults to ``('a', 'area')``.
@@ -115,11 +115,11 @@ LxmlLinkExtractor
         different for requests with canonicalized and raw URLs. If you're
         using LinkExtractor to follow links it is more robust to
         keep the default ``canonicalize=False``.
-    :type canonicalize: boolean
+    :type canonicalize: bool
 
     :param unique: whether duplicate filtering should be applied to extracted
         links.
-    :type unique: boolean
+    :type unique: bool
 
     :param process_value: a function which receives each value extracted from
         the tag and attributes scanned and can modify the value and return a
@@ -141,7 +141,7 @@ LxmlLinkExtractor
                 if m:
                     return m.group(1)
 
-    :type process_value: callable
+    :type process_value: collections.abc.Callable
 
     :param strip: whether to strip whitespaces from extracted attributes.
         According to HTML5 standard, leading and trailing whitespaces
@@ -150,6 +150,6 @@ LxmlLinkExtractor
         elements, etc., so LinkExtractor strips space chars by default.
         Set ``strip=False`` to turn it off (e.g. if you're extracting urls
         from elements or attributes which allow leading/trailing whitespaces).
-    :type strip: boolean
+    :type strip: bool
 
 .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 1c2f1da4d..4137fdd24 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -320,7 +320,7 @@ ItemLoader objects
         :param re: a regular expression to use for extracting data from the
             given value using :meth:`~scrapy.utils.misc.extract_regex` method,
             applied before processors
-        :type re: str or compiled regex
+        :type re: str or typing.Pattern
 
         Examples::
 
@@ -365,7 +365,7 @@ ItemLoader objects
 
         :param re: a regular expression to use for extracting data from the
             selected XPath region
-        :type re: str or compiled regex
+        :type re: str or typing.Pattern
 
         Examples::
 
@@ -408,7 +408,7 @@ ItemLoader objects
 
         :param re: a regular expression to use for extracting data from the
             selected CSS region
-        :type re: str or compiled regex
+        :type re: str or typing.Pattern
 
         Examples::
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 727c67482..2f99a72f5 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -31,7 +31,7 @@ Request objects
     a :class:`Response`.
 
     :param url: the URL of this request
-    :type url: string
+    :type url: str
 
     :param callback: the function that will be called with the response of this
        request (once its downloaded) as its first parameter. For more information
@@ -40,10 +40,10 @@ Request objects
        :meth:`~scrapy.spiders.Spider.parse` method will be used.
        Note that if exceptions are raised during processing, errback is called instead.
 
-    :type callback: callable
+    :type callback: collections.abc.Callable
 
     :param method: the HTTP method of this request. Defaults to ``'GET'``.
-    :type method: string
+    :type method: str
 
     :param meta: the initial values for the :attr:`Request.meta` attribute. If
        given, the dict passed in this parameter will be shallow copied.
@@ -54,7 +54,7 @@ Request objects
       ``body`` is not given, an empty string is stored. Regardless of the
       type of this argument, the final value stored will be a ``str`` (never
       ``unicode`` or ``None``).
-    :type body: str or unicode
+    :type body: str
 
     :param headers: the headers of this request. The dict values can be strings
        (for single valued headers) or lists (for multi-valued headers). If
@@ -105,7 +105,7 @@ Request objects
     :param encoding: the encoding of this request (defaults to ``'utf-8'``).
        This encoding will be used to percent-encode the URL and to convert the
        body to ``str`` (if given as ``unicode``).
-    :type encoding: string
+    :type encoding: str
 
     :param priority: the priority of this request (defaults to ``0``).
        The priority is used by the scheduler to define the order used to process
@@ -117,7 +117,7 @@ Request objects
        the scheduler. This is used when you want to perform an identical
        request multiple times, to ignore the duplicates filter. Use it with
        care, or you will get into crawling loops. Default to ``False``.
-    :type dont_filter: boolean
+    :type dont_filter: bool
 
     :param errback: a function that will be called if any exception was
        raised while processing the request. This includes pages that failed
@@ -125,7 +125,7 @@ Request objects
        as first parameter.
        For more information,
        see :ref:`topics-request-response-ref-errbacks` below.
-    :type errback: callable
+    :type errback: collections.abc.Callable
 
     :param flags:  Flags sent to the request, can be used for logging or similar purposes.
     :type flags: list
@@ -407,7 +407,7 @@ fields with form data from :class:`Response` objects.
     :param formdata: is a dictionary (or iterable of (key, value) tuples)
        containing HTML Form data which will be url-encoded and assigned to the
        body of the request.
-    :type formdata: dict or iterable of tuples
+    :type formdata: dict or collections.abc.Iterable
 
     The :class:`FormRequest` objects support the following class method in
     addition to the standard :class:`Request` methods:
@@ -439,20 +439,20 @@ fields with form data from :class:`Response` objects.
        :type response: :class:`Response` object
 
        :param formname: if given, the form with name attribute set to this value will be used.
-       :type formname: string
+       :type formname: str
 
        :param formid: if given, the form with id attribute set to this value will be used.
-       :type formid: string
+       :type formid: str
 
        :param formxpath: if given, the first form that matches the xpath will be used.
-       :type formxpath: string
+       :type formxpath: str
 
        :param formcss: if given, the first form that matches the css selector will be used.
-       :type formcss: string
+       :type formcss: str
 
        :param formnumber: the number of form to use, when the response contains
           multiple forms. The first one (and also the default) is ``0``.
-       :type formnumber: integer
+       :type formnumber: int
 
        :param formdata: fields to override in the form data. If a field was
           already present in the response ``<form>`` element, its value is
@@ -470,7 +470,7 @@ fields with form data from :class:`Response` objects.
 
        :param dont_click: If True, the form data will be submitted without
          clicking in any element.
-       :type dont_click: boolean
+       :type dont_click: bool
 
        The other parameters of this class method are passed directly to the
        :class:`FormRequest` constructor.
@@ -558,7 +558,7 @@ dealing with JSON requests.
       if :attr:`Request.body` argument is provided this parameter will be ignored.
       if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be 
       set to ``'POST'`` automatically.
-   :type data: JSON serializable object
+   :type data: object
 
    :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize
        data into JSON format.
@@ -587,10 +587,10 @@ Response objects
     downloaded (by the Downloader) and fed to the Spiders for processing.
 
     :param url: the URL of this response
-    :type url: string
+    :type url: str
 
     :param status: the HTTP status of the response. Defaults to ``200``.
-    :type status: integer
+    :type status: int
 
     :param headers: the headers of this response. The dict values can be strings
        (for single valued headers) or lists (for multi-valued headers).
@@ -730,7 +730,7 @@ TextResponse objects
        body, it will be encoded using this encoding (remember the body attribute
        is always a string). If ``encoding`` is ``None`` (default value), the
        encoding will be looked up in the response headers and body instead.
-    :type encoding: string
+    :type encoding: str
 
     :class:`TextResponse` objects support the following attributes in addition
     to the standard :class:`Response` ones:
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index ded3c082b..84acf543f 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -292,7 +292,7 @@ class CrawlerProcess(CrawlerRunner):
         If ``stop_after_crawl`` is True, the reactor will be stopped after all
         crawlers have finished, using :meth:`join`.
 
-        :param boolean stop_after_crawl: stop or not the reactor when all
+        :param bool stop_after_crawl: stop or not the reactor when all
             crawlers have finished
         """
         if stop_after_crawl:
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 189f165d1..7faad308a 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -43,10 +43,10 @@ class RobotParser(with_metaclass(ABCMeta)):
         """Return ``True`` if  ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``.
 
         :param url: Absolute URL
-        :type url: string
+        :type url: str
 
         :param user_agent: User agent
-        :type user_agent: string
+        :type user_agent: str
         """
         pass
 
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index f28c7940d..95c02021e 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -105,10 +105,9 @@ class BaseSettings(MutableMapping):
         Get a setting value without affecting its original type.
 
         :param name: the setting name
-        :type name: string
+        :type name: str
 
         :param default: the value to return if no setting is found
-        :type default: any
         """
         return self[name] if self[name] is not None else default
 
@@ -123,10 +122,9 @@ class BaseSettings(MutableMapping):
         ``'0'`` will return ``False`` when using this method.
 
         :param name: the setting name
-        :type name: string
+        :type name: str
 
         :param default: the value to return if no setting is found
-        :type default: any
         """
         got = self.get(name, default)
         try:
@@ -145,10 +143,9 @@ class BaseSettings(MutableMapping):
         Get a setting value as an int.
 
         :param name: the setting name
-        :type name: string
+        :type name: str
 
         :param default: the value to return if no setting is found
-        :type default: any
         """
         return int(self.get(name, default))
 
@@ -157,10 +154,9 @@ class BaseSettings(MutableMapping):
         Get a setting value as a float.
 
         :param name: the setting name
-        :type name: string
+        :type name: str
 
         :param default: the value to return if no setting is found
-        :type default: any
         """
         return float(self.get(name, default))
 
@@ -173,10 +169,9 @@ class BaseSettings(MutableMapping):
         ``'one,two'`` will return a list ['one', 'two'] when using this method.
 
         :param name: the setting name
-        :type name: string
+        :type name: str
 
         :param default: the value to return if no setting is found
-        :type default: any
         """
         value = self.get(name, default or [])
         if isinstance(value, six.string_types):
@@ -194,10 +189,9 @@ class BaseSettings(MutableMapping):
         and losing all information about priority and mutability.
 
         :param name: the setting name
-        :type name: string
+        :type name: str
 
         :param default: the value to return if no setting is found
-        :type default: any
         """
         value = self.get(name, default or {})
         if isinstance(value, six.string_types):
@@ -209,7 +203,7 @@ class BaseSettings(MutableMapping):
         counterpart.
 
         :param name: name of the dictionary-like setting
-        :type name: string
+        :type name: str
         """
         compbs = BaseSettings()
         compbs.update(self[name + '_BASE'])
@@ -222,7 +216,7 @@ class BaseSettings(MutableMapping):
         the given ``name`` does not exist.
 
         :param name: the setting name
-        :type name: string
+        :type name: str
         """
         if name not in self:
             return None
@@ -252,14 +246,13 @@ class BaseSettings(MutableMapping):
         otherwise they won't have any effect.
 
         :param name: the setting name
-        :type name: string
+        :type name: str
 
         :param value: the value to associate with the setting
-        :type value: any
 
         :param priority: the priority of the setting. Should be a key of
             :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
-        :type priority: string or int
+        :type priority: str or int
         """
         self._assert_mutability()
         priority = get_settings_priority(priority)
@@ -283,11 +276,11 @@ class BaseSettings(MutableMapping):
         uppercase variable of ``module`` with the provided ``priority``.
 
         :param module: the module or the path of the module
-        :type module: module object or string
+        :type module: types.ModuleType or str
 
         :param priority: the priority of the settings. Should be a key of
             :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
-        :type priority: string or int
+        :type priority: str or int
         """
         self._assert_mutability()
         if isinstance(module, six.string_types):
@@ -316,7 +309,7 @@ class BaseSettings(MutableMapping):
 
         :param priority: the priority of the settings. Should be a key of
             :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
-        :type priority: string or int
+        :type priority: str or int
         """
         self._assert_mutability()
         if isinstance(values, six.string_types):
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index 296d27ed8..c24b16fcb 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -17,7 +17,7 @@ class SignalManager(object):
         section.
 
         :param receiver: the function to be connected
-        :type receiver: callable
+        :type receiver: collections.abc.Callable
 
         :param signal: the signal to connect to
         :type signal: object

From 3d4317bfe4697955de1c2450ce4cb0d12471a9a3 Mon Sep 17 00:00:00 2001
From: Roy <roy.healy87+gh@gmail.com>
Date: Mon, 21 Oct 2019 18:32:30 +0100
Subject: [PATCH 2253/2974] [tox.ini] Added python 3.8 fields
 https://github.com/scrapy/scrapy/issues/4085

---
 tox.ini | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tox.ini b/tox.ini
index ffe7360d3..14afec23f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -92,6 +92,10 @@ deps = {[testenv:py35]deps}
 basepython = python3.7
 deps = {[testenv:py35]deps}
 
+[testenv:py38]
+basepython = python3.8
+deps = {[testenv:py35]deps}
+
 [testenv:pypy3]
 basepython = pypy3
 deps = {[testenv:py35]deps}
@@ -128,6 +132,13 @@ deps =
     reppy
     robotexclusionrulesparser
 
+[testenv:py38-extra-deps]
+basepython = python3.8
+deps =
+    {[testenv:py35]deps}
+    reppy
+    robotexclusionrulesparser
+
 [testenv:py27-extra-deps]
 basepython = python2.7
 deps =

From 4e939ca75d14daf5bc658fdbe5a97af6f4c3c498 Mon Sep 17 00:00:00 2001
From: Roy <roy.healy87+gh@gmail.com>
Date: Mon, 21 Oct 2019 18:33:18 +0100
Subject: [PATCH 2254/2974] [setup.py] Added python 3.8 fields
 https://github.com/scrapy/scrapy/issues/4085

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index 850456503..2f5fca4c9 100644
--- a/setup.py
+++ b/setup.py
@@ -56,6 +56,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
         'Topic :: Internet :: WWW/HTTP',

From c12a075164e95e98a136499faf7bf8eefdce0300 Mon Sep 17 00:00:00 2001
From: Roy <roy.healy87+gh@gmail.com>
Date: Mon, 21 Oct 2019 18:34:15 +0100
Subject: [PATCH 2255/2974] [.travis.yml] Added python 3.8 fields
 https://github.com/scrapy/scrapy/issues/4085

---
 .travis.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 0190a7f4d..2ba504972 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,6 +27,10 @@ matrix:
       python: 3.7
     - env: TOXENV=py37-extra-deps
       python: 3.7
+    - env: TOXENV=py38
+      python: 3.8
+    - env: TOXENV=py37-extra-deps
+      python: 3.8
     - env: TOXENV=docs
       python: 3.6
 install:

From da8cd9448de28bdeac7343bcd367e95237423789 Mon Sep 17 00:00:00 2001
From: Ammar Najjar <najjarammar@protonmail.com>
Date: Mon, 21 Oct 2019 19:48:13 +0200
Subject: [PATCH 2256/2974] docs: always surround __init__ with `` in docs

Issue #4086
---
 docs/topics/exporters.rst        | 36 ++++++++++++++++----------------
 docs/topics/items.rst            |  2 +-
 docs/topics/loaders.rst          | 22 +++++++++----------
 docs/topics/request-response.rst | 12 +++++------
 scrapy/exporters.py              |  2 +-
 scrapy/extensions/feedexport.py  |  2 +-
 scrapy/utils/datatypes.py        |  2 +-
 scrapy/utils/misc.py             |  6 +++---
 scrapy/utils/python.py           |  2 +-
 sep/sep-009.rst                  |  2 +-
 tests/test_spider.py             |  4 ++--
 11 files changed, 46 insertions(+), 46 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index d7ab7a5cc..1b8a69ca3 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -144,7 +144,7 @@ BaseItemExporter
    defining what fields to export, whether to export empty fields, or which
    encoding to use.
 
-   These features can be configured through the `__init__` method arguments which
+   These features can be configured through the ``__init__`` method arguments which
    populate their respective instance attributes: :attr:`fields_to_export`,
    :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
 
@@ -246,8 +246,8 @@ XmlItemExporter
    :param item_element: The name of each item element in the exported XML.
    :type item_element: str
 
-   The additional keyword arguments of this `__init__` method are passed to the
-   :class:`BaseItemExporter` `__init__` method
+   The additional keyword arguments of this ``__init__`` method are passed to the
+   :class:`BaseItemExporter` ``__init__`` method
 
    A typical output of this exporter would be::
 
@@ -306,9 +306,9 @@ CsvItemExporter
       multi-valued fields, if found.
    :type include_headers_line: str
 
-   The additional keyword arguments of this `__init__` method are passed to the
-   :class:`BaseItemExporter` `__init__` method, and the leftover arguments to the
-   `csv.writer`_ `__init__` method, so you can use any ``csv.writer`` `__init__` method
+   The additional keyword arguments of this ``__init__`` method are passed to the
+   :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
+   `csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
    argument to customize this exporter.
 
    A typical output of this exporter would be::
@@ -334,8 +334,8 @@ PickleItemExporter
 
    For more information, refer to the `pickle module documentation`_.
 
-   The additional keyword arguments of this `__init__` method are passed to the
-   :class:`BaseItemExporter` `__init__` method.
+   The additional keyword arguments of this ``__init__`` method are passed to the
+   :class:`BaseItemExporter` ``__init__`` method.
 
    Pickle isn't a human readable format, so no output examples are provided.
 
@@ -351,8 +351,8 @@ PprintItemExporter
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
 
-   The additional keyword arguments of this `__init__` method are passed to the
-   :class:`BaseItemExporter` `__init__` method
+   The additional keyword arguments of this ``__init__`` method are passed to the
+   :class:`BaseItemExporter` ``__init__`` method
 
    A typical output of this exporter would be::
 
@@ -367,10 +367,10 @@ JsonItemExporter
 .. class:: JsonItemExporter(file, \**kwargs)
 
    Exports Items in JSON format to the specified file-like object, writing all
-   objects as a list of objects. The additional `__init__` method arguments are
-   passed to the :class:`BaseItemExporter` `__init__` method, and the leftover
-   arguments to the `JSONEncoder`_ `__init__` method, so you can use any
-   `JSONEncoder`_ `__init__` method argument to customize this exporter.
+   objects as a list of objects. The additional ``__init__`` method arguments are
+   passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
+   arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
+   `JSONEncoder`_ ``__init__`` method argument to customize this exporter.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -398,10 +398,10 @@ JsonLinesItemExporter
 .. class:: JsonLinesItemExporter(file, \**kwargs)
 
    Exports Items in JSON format to the specified file-like object, writing one
-   JSON-encoded item per line. The additional `__init__` method arguments are passed
-   to the :class:`BaseItemExporter` `__init__` method and the leftover arguments to
-   the `JSONEncoder`_ `__init__` method, so you can use any `JSONEncoder`_
-   `__init__` method argument to customize this exporter.
+   JSON-encoded item per line. The additional ``__init__`` method arguments are passed
+   to the :class:`BaseItemExporter` ``__init__`` method and the leftover arguments to
+   the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
+   ``__init__`` method argument to customize this exporter.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 8ea2635cc..370409026 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -237,7 +237,7 @@ Item objects
 
     Return a new Item optionally initialized from the given argument.
 
-    Items replicate the standard `dict API`_, including its `__init__` method and
+    Items replicate the standard `dict API`_, including its ``__init__`` method and
     also provide the following additional API members:
 
     .. automethod:: copy
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index db1175a13..72610f645 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -494,7 +494,7 @@ ItemLoader objects
     .. attribute:: default_item_class
 
         An Item class (or factory), used to instantiate items when not given in
-        the `__init__` method
+        the ``__init__`` method
 
     .. attribute:: default_input_processor
 
@@ -509,15 +509,15 @@ ItemLoader objects
     .. attribute:: default_selector_class
 
         The class used to construct the :attr:`selector` of this
-        :class:`ItemLoader`, if only a response is given in the `__init__` method
-        If a selector is given in the `__init__` method this attribute is ignored.
+        :class:`ItemLoader`, if only a response is given in the ``__init__`` method
+        If a selector is given in the ``__init__`` method this attribute is ignored.
         This attribute is sometimes overridden in subclasses.
 
     .. attribute:: selector
 
         The :class:`~scrapy.selector.Selector` object to extract data from.
-        It's either the selector given in the `__init__` methodor one created from
-        the response given in the `__init__` methodusing the
+        It's either the selector given in the ``__init__`` methodor one created from
+        the response given in the ``__init__`` methodusing the
         :attr:`default_selector_class`. This attribute is meant to be
         read-only.
 
@@ -642,7 +642,7 @@ Here is a list of all built-in processors:
 .. class:: Identity
 
     The simplest processor, which doesn't do anything. It returns the original
-    values unchanged. It doesn't receive any `__init__` method arguments, nor does it
+    values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
     accept Loader contexts.
 
     Example::
@@ -656,7 +656,7 @@ Here is a list of all built-in processors:
 
     Returns the first non-null/non-empty value from the values received,
     so it's typically used as an output processor to single-valued fields.
-    It doesn't receive any `__init__` methodarguments, nor does it accept Loader contexts.
+    It doesn't receive any ``__init__`` methodarguments, nor does it accept Loader contexts.
 
     Example::
 
@@ -667,7 +667,7 @@ Here is a list of all built-in processors:
 
 .. class:: Join(separator=u' ')
 
-    Returns the values joined with the separator given in the `__init__` method which
+    Returns the values joined with the separator given in the ``__init__`` method which
     defaults to ``u' '``. It doesn't accept Loader contexts.
 
     When using the default separator, this processor is equivalent to the
@@ -705,7 +705,7 @@ Here is a list of all built-in processors:
     those which do, this processor will pass the currently active :ref:`Loader
     context <topics-loaders-context>` through that parameter.
 
-    The keyword arguments passed in the `__init__` methodare used as the default
+    The keyword arguments passed in the ``__init__`` methodare used as the default
     Loader context values passed to each function call. However, the final
     Loader context values passed to functions are overridden with the currently
     active Loader context accessible through the :meth:`ItemLoader.context`
@@ -749,12 +749,12 @@ Here is a list of all built-in processors:
         ['HELLO, 'THIS', 'IS', 'SCRAPY']
 
     As with the Compose processor, functions can receive Loader contexts, and
-    `__init__` method keyword arguments are used as default context values. See
+    ``__init__`` method keyword arguments are used as default context values. See
     :class:`Compose` processor for more info.
 
 .. class:: SelectJmes(json_path)
 
-    Queries the value using the json path provided to the `__init__` methodand returns the output.
+    Queries the value using the json path provided to the ``__init__`` methodand returns the output.
     Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
     This processor takes only one input at a time.
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index d253064f2..bf6a02a1d 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -137,7 +137,7 @@ Request objects
 
         A string containing the URL of this request. Keep in mind that this
         attribute contains the escaped URL, so it can differ from the URL passed in
-        the `__init__` method
+        the ``__init__`` method
 
         This attribute is read-only. To change the URL of a Request use
         :meth:`replace`.
@@ -400,7 +400,7 @@ fields with form data from :class:`Response` objects.
 
 .. class:: FormRequest(url, [formdata, ...])
 
-    The :class:`FormRequest` class adds a new keyword parameter to the `__init__` method The
+    The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method The
     remaining arguments are the same as for the :class:`Request` class and are
     not documented here.
 
@@ -473,7 +473,7 @@ fields with form data from :class:`Response` objects.
        :type dont_click: boolean
 
        The other parameters of this class method are passed directly to the
-       :class:`FormRequest` `__init__` method
+       :class:`FormRequest` ``__init__`` method
 
        .. versionadded:: 0.10.3
           The ``formname`` parameter.
@@ -547,7 +547,7 @@ dealing with JSON requests.
 
 .. class:: JsonRequest(url, [... data, dumps_kwargs])
 
-   The :class:`JsonRequest` class adds two new keyword parameters to the `__init__` method The
+   The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method The
    remaining arguments are the same as for the :class:`Request` class and are
    not documented here.
 
@@ -721,7 +721,7 @@ TextResponse objects
     :class:`Response` class, which is meant to be used only for binary data,
     such as images, sounds or any media file.
 
-    :class:`TextResponse` objects support a new `__init__` method argument, in
+    :class:`TextResponse` objects support a new ``__init__`` method argument, in
     addition to the base :class:`Response` objects. The remaining functionality
     is the same as for the :class:`Response` class and is not documented here.
 
@@ -755,7 +755,7 @@ TextResponse objects
        A string with the encoding of this response. The encoding is resolved by
        trying the following mechanisms, in order:
 
-       1. the encoding passed in the `__init__` method`` ``encoding`` argument
+       1. the encoding passed in the ``__init__`` method`` ``encoding`` argument
 
        2. the encoding declared in the Content-Type HTTP header. If this
           encoding is not valid (ie. unknown), it is ignored and the next
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 2fdc86b63..8ed8d55f1 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -31,7 +31,7 @@ class BaseItemExporter(object):
     def _configure(self, options, dont_fail=False):
         """Configure the exporter by poping options from the ``options`` dict.
         If dont_fail is set, it won't raise an exception on unexpected options
-        (useful for using with keyword arguments in subclasses __init__ methods)
+        (useful for using with keyword arguments in subclasses ``__init__`` methods)
         """
         self.encoding = options.pop('encoding', None)
         self.fields_to_export = options.pop('fields_to_export', None)
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 655c10482..bceb648a0 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -106,7 +106,7 @@ class S3FeedStorage(BlockingFeedStorage):
                 warnings.warn(
                     "Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
                     "without AWS keys is deprecated. Please supply credentials or "
-                    "use the `from_crawler()` __init__ method.",
+                    "use the `from_crawler()` ``__init__`` method.",
                     category=ScrapyDeprecationWarning,
                     stacklevel=2
                 )
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 0dfee1b27..70c2aebc8 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -246,7 +246,7 @@ class CaselessDict(dict):
 class MergeDict(object):
     """
     A simple class for creating new "virtual" dictionaries that actually look
-    up values in more than one dictionary, passed in the __init__ method.
+    up values in more than one dictionary, passed in the ``__init__`` method.
 
     If a key appears in more than one of the given dictionaries, only the
     first occurrence will be used.
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index c892bd438..b3ba2ccec 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -123,14 +123,14 @@ def rel_has_nofollow(rel):
 
 def create_instance(objcls, settings, crawler, *args, **kwargs):
     """Construct a class instance using its ``from_crawler`` or
-    ``from_settings`` __init__ method, if available.
+    ``from_settings`` ``__init__`` method, if available.
 
     At least one of ``settings`` and ``crawler`` needs to be different from
     ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
-    If ``crawler`` is ``None``, only the ``from_settings`` __init__ method will be
+    If ``crawler`` is ``None``, only the ``from_settings`` ``__init__`` method will be
     tried.
 
-    ``*args`` and ``**kwargs`` are forwarded to the __init__ methods.
+    ``*args`` and ``**kwargs`` are forwarded to the ``__init__`` methods.
 
     Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
     """
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 0d645543a..ea5193f12 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -303,7 +303,7 @@ class WeakKeyCache(object):
 def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
     """Return a (new) dict with unicode keys (and values when "keys_only" is
     False) of the given dict converted to strings. ``dct_or_tuples`` can be a
-    dict or a list of tuples, like any dict __init__ method supports.
+    dict or a list of tuples, like any dict ``__init__`` method supports.
     """
     d = {}
     for k, v in six.iteritems(dict(dct_or_tuples)):
diff --git a/sep/sep-009.rst b/sep/sep-009.rst
index 929cedc61..e46479a74 100644
--- a/sep/sep-009.rst
+++ b/sep/sep-009.rst
@@ -55,7 +55,7 @@ singletons members of that object, as explained below:
      ``STATS_CLASS`` setting)
    - **crawler.log**: Logger class with methods replacing the current
      ``scrapy.log`` functions. Logging would be started (if enabled) on
-     ``Crawler`` __init__ method, so no log starting functions are required.
+     ``Crawler`` ``__init__`` method, so no log starting functions are required.
 
       - ``crawler.log.msg``
    - **crawler.signals**: signal handling
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 64ff40b61..6f6cdb8ff 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -42,12 +42,12 @@ class SpiderTest(unittest.TestCase):
         self.assertEqual(list(start_requests), [])
 
     def test_spider_args(self):
-        """__init__ method arguments are assigned to spider attributes"""
+        """``__init__`` method arguments are assigned to spider attributes"""
         spider = self.spider_class('example.com', foo='bar')
         self.assertEqual(spider.foo, 'bar')
 
     def test_spider_without_name(self):
-        """__init__ method arguments are assigned to spider attributes"""
+        """``__init__`` method arguments are assigned to spider attributes"""
         self.assertRaises(ValueError, self.spider_class)
         self.assertRaises(ValueError, self.spider_class, somearg='foo')
 

From 85ac5c5c5778a116f762cb6200492aa6a2439e83 Mon Sep 17 00:00:00 2001
From: Roy Healy <roy.healy87+gh@gmail.com>
Date: Mon, 21 Oct 2019 19:06:43 +0100
Subject: [PATCH 2257/2974] Update .travis.yml

Co-Authored-By: Mikhail Korobov <kmike84@gmail.com>
---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 2ba504972..044fa9e95 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,7 +29,7 @@ matrix:
       python: 3.7
     - env: TOXENV=py38
       python: 3.8
-    - env: TOXENV=py37-extra-deps
+    - env: TOXENV=py38-extra-deps
       python: 3.8
     - env: TOXENV=docs
       python: 3.6

From 2ee38e8ddbac6841c8d2bb067ddc0ddba813151f Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Tue, 22 Oct 2019 14:43:47 +0530
Subject: [PATCH 2258/2974] Added Pathlib.Path test

---
 tests/test_feedexport.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index f32ac2a4b..842e8d8c0 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -29,6 +29,8 @@ from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get
 from scrapy.utils.python import to_native_str
 from scrapy.utils.project import get_project_settings
 
+from Pathlib import Path
+
 
 class FileFeedStorageTest(unittest.TestCase):
 
@@ -843,3 +845,17 @@ class FeedExportTest(unittest.TestCase):
         yield self.exported_data({}, settings)
         self.assertTrue(FromCrawlerCsvItemExporter.init_with_crawler)
         self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler)
+
+    @defer.inlineCallbacks
+    def test_pathlib_uri(self):
+        tmpdir = tempfile.mkdtemp()
+        feed_uri = Path(tmpdir) / 'res'
+        settings = {
+            'FEED_FORMAT': 'csv',
+            'FEED_STORE_EMPTY': True,
+            'FEED_URI': feed_uri,
+        }
+        
+    data = yield self.exported_no_data(settings)
+    self.assertEqual(data, b'')
+    shutil.rmtree(tmpdir, ignore_errors=True)
\ No newline at end of file

From ad96d6ef594c29e8648c3c3b310c42e8fcb210d3 Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Tue, 22 Oct 2019 14:53:59 +0530
Subject: [PATCH 2259/2974] Added Pathlib.Path test correctly

---
 tests/test_feedexport.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 842e8d8c0..664cfd6de 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -856,6 +856,6 @@ class FeedExportTest(unittest.TestCase):
             'FEED_URI': feed_uri,
         }
         
-    data = yield self.exported_no_data(settings)
-    self.assertEqual(data, b'')
-    shutil.rmtree(tmpdir, ignore_errors=True)
\ No newline at end of file
+        data = yield self.exported_no_data(settings)
+        self.assertEqual(data, b'')
+        shutil.rmtree(tmpdir, ignore_errors=True)
\ No newline at end of file

From 4226791481bb440b9542dda063d1b5ac920a8411 Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Tue, 22 Oct 2019 15:07:13 +0530
Subject: [PATCH 2260/2974] Added Pathlib.Path test

---
 requirements-py2.txt | 1 +
 requirements-py3.txt | 1 +
 2 files changed, 2 insertions(+)

diff --git a/requirements-py2.txt b/requirements-py2.txt
index dde8d1c9c..42e057417 100644
--- a/requirements-py2.txt
+++ b/requirements-py2.txt
@@ -16,3 +16,4 @@ service_identity>=16.0.0
 six>=1.10.0
 Twisted>=16.0.0
 zope.interface>=4.1.3
+pathlib2>=2.0
diff --git a/requirements-py3.txt b/requirements-py3.txt
index 2c98e6f6d..77296b91b 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -16,3 +16,4 @@ lxml>=3.5.0
 service_identity>=16.0.0
 six>=1.10.0
 zope.interface>=4.1.3
+pathlib2>=2.0

From 0b7d8a51b4dabeb4097f5fc2de396904f308c56d Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Tue, 22 Oct 2019 15:12:53 +0530
Subject: [PATCH 2261/2974] Added Pathlib.Path test

---
 requirements-py2.txt | 2 +-
 requirements-py3.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements-py2.txt b/requirements-py2.txt
index 42e057417..2a0bb49d3 100644
--- a/requirements-py2.txt
+++ b/requirements-py2.txt
@@ -16,4 +16,4 @@ service_identity>=16.0.0
 six>=1.10.0
 Twisted>=16.0.0
 zope.interface>=4.1.3
-pathlib2>=2.0
+pathlib==1.0.1
diff --git a/requirements-py3.txt b/requirements-py3.txt
index 77296b91b..c57cff5da 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -16,4 +16,4 @@ lxml>=3.5.0
 service_identity>=16.0.0
 six>=1.10.0
 zope.interface>=4.1.3
-pathlib2>=2.0
+pathlib==1.0.1

From a776554282aadc651f9e142aea9bc436f9f587a7 Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Tue, 22 Oct 2019 15:31:55 +0530
Subject: [PATCH 2262/2974] Added Pathlib.Path test

---
 tests/test_feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 664cfd6de..fcada3a45 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -29,7 +29,7 @@ from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get
 from scrapy.utils.python import to_native_str
 from scrapy.utils.project import get_project_settings
 
-from Pathlib import Path
+from pathlib import Path
 
 
 class FileFeedStorageTest(unittest.TestCase):

From 07822935eca497f3fe31de536afd76a2c31cc5a9 Mon Sep 17 00:00:00 2001
From: illgitthat <illgitthat@users.noreply.github.com>
Date: Tue, 22 Oct 2019 06:05:34 -0400
Subject: [PATCH 2263/2974] Updating link for miniconda (#4089)

---
 docs/intro/install.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 2bf98dbdc..51b41b4d7 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -290,5 +290,5 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
 .. _zsh: https://www.zsh.org/
 .. _Scrapinghub: https://scrapinghub.com
 .. _Anaconda: https://docs.anaconda.com/anaconda/
-.. _Miniconda: https://conda.io/docs/user-guide/install/index.html
+.. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
 .. _conda-forge: https://conda-forge.org/

From cd4c211f4b5c9d5716d0e49472a978c11d7351f8 Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Tue, 22 Oct 2019 15:38:06 +0530
Subject: [PATCH 2264/2974] Added Pathlib.Path test

---
 tests/test_feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index fcada3a45..f497bb32e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -29,7 +29,7 @@ from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get
 from scrapy.utils.python import to_native_str
 from scrapy.utils.project import get_project_settings
 
-from pathlib import Path
+from pathlib2 import Path
 
 
 class FileFeedStorageTest(unittest.TestCase):

From 5d75ed4cba3943ac88b4e9cd1d5e87259b85f11f Mon Sep 17 00:00:00 2001
From: WinterComes <andreysmirnou@gmail.com>
Date: Tue, 22 Oct 2019 13:19:07 +0300
Subject: [PATCH 2265/2974] Remove an old note about contracts (#4093)

---
 docs/topics/contracts.rst | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 62f9a743b..371ae62d5 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -6,10 +6,6 @@ Spiders Contracts
 
 .. versionadded:: 0.15
 
-.. note:: This is a new feature (introduced in Scrapy 0.15) and may be subject
-   to minor functionality/API updates. Check the :ref:`release notes <news>` to
-   be notified of updates.
-
 Testing spiders can get particularly annoying and while nothing prevents you
 from writing unit tests the task gets cumbersome quickly. Scrapy offers an
 integrated way of testing your spiders by the means of contracts.

From cd0964643879b5513807c8955c7e887c77624970 Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Tue, 22 Oct 2019 16:19:41 +0530
Subject: [PATCH 2266/2974] Added Pathlib.Path test

---
 requirements-py2.txt | 2 +-
 requirements-py3.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements-py2.txt b/requirements-py2.txt
index 2a0bb49d3..42e057417 100644
--- a/requirements-py2.txt
+++ b/requirements-py2.txt
@@ -16,4 +16,4 @@ service_identity>=16.0.0
 six>=1.10.0
 Twisted>=16.0.0
 zope.interface>=4.1.3
-pathlib==1.0.1
+pathlib2>=2.0
diff --git a/requirements-py3.txt b/requirements-py3.txt
index c57cff5da..77296b91b 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -16,4 +16,4 @@ lxml>=3.5.0
 service_identity>=16.0.0
 six>=1.10.0
 zope.interface>=4.1.3
-pathlib==1.0.1
+pathlib2>=2.0

From 7031e3a12422ae977448e7b338457f6c09af9b0e Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Tue, 22 Oct 2019 16:31:14 +0530
Subject: [PATCH 2267/2974] Added Pathlib.Path test

---
 tests/test_feedexport.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index f497bb32e..c4bfdb4fe 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -850,10 +850,11 @@ class FeedExportTest(unittest.TestCase):
     def test_pathlib_uri(self):
         tmpdir = tempfile.mkdtemp()
         feed_uri = Path(tmpdir) / 'res'
+        res_uri = urljoin('file:', pathname2url(feed_uri))
         settings = {
             'FEED_FORMAT': 'csv',
             'FEED_STORE_EMPTY': True,
-            'FEED_URI': feed_uri,
+            'FEED_URI': res_uri,
         }
         
         data = yield self.exported_no_data(settings)

From 85f56a92f0c753dfa55012e647f425a6a3d23076 Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Tue, 22 Oct 2019 16:43:17 +0530
Subject: [PATCH 2268/2974] Added Pathlib.Path test

---
 tests/test_feedexport.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c4bfdb4fe..17526716b 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -850,6 +850,7 @@ class FeedExportTest(unittest.TestCase):
     def test_pathlib_uri(self):
         tmpdir = tempfile.mkdtemp()
         feed_uri = Path(tmpdir) / 'res'
+        feed_uri=str(feeduri)
         res_uri = urljoin('file:', pathname2url(feed_uri))
         settings = {
             'FEED_FORMAT': 'csv',

From 4184bac0687d55a442f599ebcfc913231a7c98e2 Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Tue, 22 Oct 2019 16:57:14 +0530
Subject: [PATCH 2269/2974] Added Pathlib.Path test

---
 tests/test_feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 17526716b..8c0e5cd3d 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -850,7 +850,7 @@ class FeedExportTest(unittest.TestCase):
     def test_pathlib_uri(self):
         tmpdir = tempfile.mkdtemp()
         feed_uri = Path(tmpdir) / 'res'
-        feed_uri=str(feeduri)
+        feed_uri=str(feed_uri)
         res_uri = urljoin('file:', pathname2url(feed_uri))
         settings = {
             'FEED_FORMAT': 'csv',

From d21e1034f046c7e8d778e698614c47ebd6875c87 Mon Sep 17 00:00:00 2001
From: Ammar Najjar <najjarammar@protonmail.com>
Date: Tue, 22 Oct 2019 13:24:57 +0200
Subject: [PATCH 2270/2974] docs: correct point,comma and plural replacements

Issue #4086
---
 docs/topics/exporters.rst        |  6 +++---
 docs/topics/items.rst            |  2 +-
 docs/topics/loaders.rst          | 18 +++++++++---------
 docs/topics/request-response.rst | 10 +++++-----
 scrapy/utils/misc.py             |  2 +-
 5 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 1b8a69ca3..b8d898022 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -247,7 +247,7 @@ XmlItemExporter
    :type item_element: str
 
    The additional keyword arguments of this ``__init__`` method are passed to the
-   :class:`BaseItemExporter` ``__init__`` method
+   :class:`BaseItemExporter` ``__init__`` method.
 
    A typical output of this exporter would be::
 
@@ -352,7 +352,7 @@ PprintItemExporter
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
 
    The additional keyword arguments of this ``__init__`` method are passed to the
-   :class:`BaseItemExporter` ``__init__`` method
+   :class:`BaseItemExporter` ``__init__`` method.
 
    A typical output of this exporter would be::
 
@@ -399,7 +399,7 @@ JsonLinesItemExporter
 
    Exports Items in JSON format to the specified file-like object, writing one
    JSON-encoded item per line. The additional ``__init__`` method arguments are passed
-   to the :class:`BaseItemExporter` ``__init__`` method and the leftover arguments to
+   to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
    the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
    ``__init__`` method argument to customize this exporter.
 
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 370409026..cdf60208e 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -237,7 +237,7 @@ Item objects
 
     Return a new Item optionally initialized from the given argument.
 
-    Items replicate the standard `dict API`_, including its ``__init__`` method and
+    Items replicate the standard `dict API`_, including its ``__init__`` method, and
     also provide the following additional API members:
 
     .. automethod:: copy
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 72610f645..a4465f88d 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -265,7 +265,7 @@ There are several ways to modify Item Loader context values:
       loader.context['unit'] = 'cm'
 
 2. On Item Loader instantiation (the keyword arguments of Item Loader
-   ``__init__`` methodare stored in the Item Loader context)::
+   ``__init__`` method are stored in the Item Loader context)::
 
       loader = ItemLoader(product, unit='cm')
 
@@ -494,7 +494,7 @@ ItemLoader objects
     .. attribute:: default_item_class
 
         An Item class (or factory), used to instantiate items when not given in
-        the ``__init__`` method
+        the ``__init__`` method.
 
     .. attribute:: default_input_processor
 
@@ -509,15 +509,15 @@ ItemLoader objects
     .. attribute:: default_selector_class
 
         The class used to construct the :attr:`selector` of this
-        :class:`ItemLoader`, if only a response is given in the ``__init__`` method
+        :class:`ItemLoader`, if only a response is given in the ``__init__`` method.
         If a selector is given in the ``__init__`` method this attribute is ignored.
         This attribute is sometimes overridden in subclasses.
 
     .. attribute:: selector
 
         The :class:`~scrapy.selector.Selector` object to extract data from.
-        It's either the selector given in the ``__init__`` methodor one created from
-        the response given in the ``__init__`` methodusing the
+        It's either the selector given in the ``__init__`` method or one created from
+        the response given in the ``__init__`` method using the
         :attr:`default_selector_class`. This attribute is meant to be
         read-only.
 
@@ -656,7 +656,7 @@ Here is a list of all built-in processors:
 
     Returns the first non-null/non-empty value from the values received,
     so it's typically used as an output processor to single-valued fields.
-    It doesn't receive any ``__init__`` methodarguments, nor does it accept Loader contexts.
+    It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
 
     Example::
 
@@ -667,7 +667,7 @@ Here is a list of all built-in processors:
 
 .. class:: Join(separator=u' ')
 
-    Returns the values joined with the separator given in the ``__init__`` method which
+    Returns the values joined with the separator given in the ``__init__`` method, which
     defaults to ``u' '``. It doesn't accept Loader contexts.
 
     When using the default separator, this processor is equivalent to the
@@ -705,7 +705,7 @@ Here is a list of all built-in processors:
     those which do, this processor will pass the currently active :ref:`Loader
     context <topics-loaders-context>` through that parameter.
 
-    The keyword arguments passed in the ``__init__`` methodare used as the default
+    The keyword arguments passed in the ``__init__`` method are used as the default
     Loader context values passed to each function call. However, the final
     Loader context values passed to functions are overridden with the currently
     active Loader context accessible through the :meth:`ItemLoader.context`
@@ -754,7 +754,7 @@ Here is a list of all built-in processors:
 
 .. class:: SelectJmes(json_path)
 
-    Queries the value using the json path provided to the ``__init__`` methodand returns the output.
+    Queries the value using the json path provided to the ``__init__`` method and returns the output.
     Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
     This processor takes only one input at a time.
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index bf6a02a1d..123c2dde1 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -137,7 +137,7 @@ Request objects
 
         A string containing the URL of this request. Keep in mind that this
         attribute contains the escaped URL, so it can differ from the URL passed in
-        the ``__init__`` method
+        the ``__init__`` method.
 
         This attribute is read-only. To change the URL of a Request use
         :meth:`replace`.
@@ -400,7 +400,7 @@ fields with form data from :class:`Response` objects.
 
 .. class:: FormRequest(url, [formdata, ...])
 
-    The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method The
+    The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The
     remaining arguments are the same as for the :class:`Request` class and are
     not documented here.
 
@@ -473,7 +473,7 @@ fields with form data from :class:`Response` objects.
        :type dont_click: boolean
 
        The other parameters of this class method are passed directly to the
-       :class:`FormRequest` ``__init__`` method
+       :class:`FormRequest` ``__init__`` method.
 
        .. versionadded:: 0.10.3
           The ``formname`` parameter.
@@ -547,7 +547,7 @@ dealing with JSON requests.
 
 .. class:: JsonRequest(url, [... data, dumps_kwargs])
 
-   The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method The
+   The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method. The
    remaining arguments are the same as for the :class:`Request` class and are
    not documented here.
 
@@ -755,7 +755,7 @@ TextResponse objects
        A string with the encoding of this response. The encoding is resolved by
        trying the following mechanisms, in order:
 
-       1. the encoding passed in the ``__init__`` method`` ``encoding`` argument
+       1. the encoding passed in the ``__init__`` method ``encoding`` argument
 
        2. the encoding declared in the Content-Type HTTP header. If this
           encoding is not valid (ie. unknown), it is ignored and the next
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index b3ba2ccec..8060553ad 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -123,7 +123,7 @@ def rel_has_nofollow(rel):
 
 def create_instance(objcls, settings, crawler, *args, **kwargs):
     """Construct a class instance using its ``from_crawler`` or
-    ``from_settings`` ``__init__`` method, if available.
+    ``from_settings`` ``__init__`` methods, if available.
 
     At least one of ``settings`` and ``crawler`` needs to be different from
     ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.

From 1d5c270ce8caf954ce83c8db262e2a35707e0c5e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 22 Oct 2019 15:12:52 +0200
Subject: [PATCH 2271/2974] Fix dangling file descriptor in FeedExporter when
 FEED_STORE_EMPTY is False (#4023)

---
 scrapy/extensions/feedexport.py | 4 +++-
 tests/test_feedexport.py        | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index ce2846eba..6fb6397b1 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -242,7 +242,9 @@ class FeedExporter(object):
     def close_spider(self, spider):
         slot = self.slot
         if not slot.itemcount and not self.store_empty:
-            return
+            # We need to call slot.storage.store nonetheless to get the file
+            # properly closed.
+            return defer.maybeDeferred(slot.storage.store, slot.file)
         if self._exporting:
             slot.exporter.finish_exporting()
             self._exporting = False
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index f32ac2a4b..e1436fbe5 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -417,7 +417,7 @@ class FeedExportTest(unittest.TestCase):
                 content = f.read()
 
         finally:
-            shutil.rmtree(tmpdir, ignore_errors=True)
+            shutil.rmtree(tmpdir)
 
         defer.returnValue(content)
 

From 5479e7ecc7d30424dd2f3d9bbfb18abca765be92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 22 Oct 2019 15:24:44 +0200
Subject: [PATCH 2272/2974] Indicate that lists of emails may be provided as a
 single string or as a list of strings

---
 docs/topics/email.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index 73b1bdc3b..284849c56 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -80,13 +80,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
 
         Send email to the given recipients.
 
-        :param to: the e-mail recipients
+        :param to: the e-mail recipients as a string or as a list of strings
         :type to: str or list
 
         :param subject: the subject of the e-mail
         :type subject: str
 
-        :param cc: the e-mails to CC
+        :param cc: the e-mails to CC as a string or as a list of strings
         :type cc: str or list
 
         :param body: the e-mail body

From d96b9f860b01256a84bda641b190b423eb4910b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 22 Oct 2019 15:24:59 +0200
Subject: [PATCH 2273/2974] Use object as type for parameters that allow any
 value

---
 scrapy/settings/__init__.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 95c02021e..d1a5093a6 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -108,6 +108,7 @@ class BaseSettings(MutableMapping):
         :type name: str
 
         :param default: the value to return if no setting is found
+        :type default: object
         """
         return self[name] if self[name] is not None else default
 
@@ -125,6 +126,7 @@ class BaseSettings(MutableMapping):
         :type name: str
 
         :param default: the value to return if no setting is found
+        :type default: object
         """
         got = self.get(name, default)
         try:
@@ -146,6 +148,7 @@ class BaseSettings(MutableMapping):
         :type name: str
 
         :param default: the value to return if no setting is found
+        :type default: object
         """
         return int(self.get(name, default))
 
@@ -157,6 +160,7 @@ class BaseSettings(MutableMapping):
         :type name: str
 
         :param default: the value to return if no setting is found
+        :type default: object
         """
         return float(self.get(name, default))
 
@@ -172,6 +176,7 @@ class BaseSettings(MutableMapping):
         :type name: str
 
         :param default: the value to return if no setting is found
+        :type default: object
         """
         value = self.get(name, default or [])
         if isinstance(value, six.string_types):
@@ -192,6 +197,7 @@ class BaseSettings(MutableMapping):
         :type name: str
 
         :param default: the value to return if no setting is found
+        :type default: object
         """
         value = self.get(name, default or {})
         if isinstance(value, six.string_types):
@@ -249,6 +255,7 @@ class BaseSettings(MutableMapping):
         :type name: str
 
         :param value: the value to associate with the setting
+        :type default: object
 
         :param priority: the priority of the setting. Should be a key of
             :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer

From 7a84a4bdba45bab56cee0daed9d9c3a04d8d61c0 Mon Sep 17 00:00:00 2001
From: Ammar Najjar <najjarammar@protonmail.com>
Date: Tue, 22 Oct 2019 15:31:34 +0200
Subject: [PATCH 2274/2974] docs: use "constructor" for from_settings() &
 rom_crawler() factory methods

Issue #4086
---
 scrapy/utils/misc.py              |  6 +++---
 tests/test_utils_misc/__init__.py | 10 +++++-----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 8060553ad..f638adb25 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -123,14 +123,14 @@ def rel_has_nofollow(rel):
 
 def create_instance(objcls, settings, crawler, *args, **kwargs):
     """Construct a class instance using its ``from_crawler`` or
-    ``from_settings`` ``__init__`` methods, if available.
+    ``from_settings`` constructors, if available.
 
     At least one of ``settings`` and ``crawler`` needs to be different from
     ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
-    If ``crawler`` is ``None``, only the ``from_settings`` ``__init__`` method will be
+    If ``crawler`` is ``None``, only the ``from_settings`` constructor will be
     tried.
 
-    ``*args`` and ``**kwargs`` are forwarded to the ``__init__`` methods.
+    ``*args`` and ``**kwargs`` are forwarded to the constructors.
 
     Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
     """
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index 457a2aa78..e109d5343 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -109,11 +109,11 @@ class UtilsMiscTestCase(unittest.TestCase):
             else:
                 mock.assert_called_once_with(*args, **kwargs)
 
-        # Check usage of correct __init__ method using four mocks:
-        #   1. with no alternative __init__ methods
-        #   2. with from_settings() __init__ method
-        #   3. with from_crawler() __init__ method
-        #   4. with from_settings() and from_crawler() __init__ method
+        # Check usage of correct constructor using four mocks:
+        #   1. with no alternative constructors
+        #   2. with from_settings() constructor
+        #   3. with from_crawler() constructor
+        #   4. with from_settings() and from_crawler() constructor
         spec_sets = ([], ['from_settings'], ['from_crawler'],
                      ['from_settings', 'from_crawler'])
         for specs in spec_sets:

From f701f5b0db10faef08e4ed9a21b98fd72f9cfc9a Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Tue, 22 Oct 2019 10:48:02 -0300
Subject: [PATCH 2275/2974] fix #2552 by improving request schema check on its
 initialization

---
 scrapy/http/request/__init__.py | 2 +-
 tests/test_http_request.py      | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index d09eaf849..76a428199 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -66,7 +66,7 @@ class Request(object_ref):
         s = safe_url_string(url, self.encoding)
         self._url = escape_ajax(s)
 
-        if ':' not in self._url:
+        if ('://' not in self._url) and (not self._url.startswith('data:')):
             raise ValueError('Missing scheme in request url: %s' % self._url)
 
     url = property(_get_url, obsolete_setter(_set_url, 'url'))
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 16d7a1cb8..64f1184c3 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -52,6 +52,8 @@ class RequestTest(unittest.TestCase):
 
     def test_url_no_scheme(self):
         self.assertRaises(ValueError, self.request_class, 'foo')
+        self.assertRaises(ValueError, self.request_class, '/foo/')
+        self.assertRaises(ValueError, self.request_class, '/foo:bar')
 
     def test_headers(self):
         # Different ways of setting headers attribute

From 7fba8434f3997197f98d2b3e73099ad85429e377 Mon Sep 17 00:00:00 2001
From: Ammar Najjar <ammarnajjar@users.noreply.github.com>
Date: Tue, 22 Oct 2019 15:55:52 +0200
Subject: [PATCH 2276/2974] use instantiation for "Crawler"

Issue #4086

Co-Authored-By: Mikhail Korobov <kmike84@gmail.com>
---
 sep/sep-009.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sep/sep-009.rst b/sep/sep-009.rst
index e46479a74..da87fa9aa 100644
--- a/sep/sep-009.rst
+++ b/sep/sep-009.rst
@@ -55,7 +55,7 @@ singletons members of that object, as explained below:
      ``STATS_CLASS`` setting)
    - **crawler.log**: Logger class with methods replacing the current
      ``scrapy.log`` functions. Logging would be started (if enabled) on
-     ``Crawler`` ``__init__`` method, so no log starting functions are required.
+     ``Crawler`` instantiation, so no log starting functions are required.
 
       - ``crawler.log.msg``
    - **crawler.signals**: signal handling

From bf5c1a3dec02165b81dffb98fc560c43fb065f38 Mon Sep 17 00:00:00 2001
From: Ammar Najjar <najjarammar@protonmail.com>
Date: Tue, 22 Oct 2019 15:56:46 +0200
Subject: [PATCH 2277/2974] docs: use "constructor" for "from_crawler"

Issue #4086
---
 scrapy/extensions/feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index bceb648a0..ce2846eba 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -106,7 +106,7 @@ class S3FeedStorage(BlockingFeedStorage):
                 warnings.warn(
                     "Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
                     "without AWS keys is deprecated. Please supply credentials or "
-                    "use the `from_crawler()` ``__init__`` method.",
+                    "use the `from_crawler()` constructor.",
                     category=ScrapyDeprecationWarning,
                     stacklevel=2
                 )

From c623a16a223df31669d9f204b8af7d89062ee56c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 22 Oct 2019 17:52:34 +0200
Subject: [PATCH 2278/2974] Remove unused method from
 scrapy.pqueues._SlotPriorityQueues

---
 scrapy/pqueues.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 6ecd1b51a..717ed4d27 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -86,9 +86,6 @@ class _SlotPriorityQueues(object):
     def __len__(self):
         return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
 
-    def __contains__(self, slot):
-        return slot in self.pqueues
-
 
 class ScrapyPriorityQueue(PriorityQueue):
     """

From 3b6f7ac9f2f5b48b9f2f3ce106d1205599d2164f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 22 Oct 2019 19:43:02 +0200
Subject: [PATCH 2279/2974] Use pylint

---
 .travis.yml           |  2 +
 docs/utils/linkfix.py | 85 ++++++++++++++++++++++-------------------
 pylintrc              | 88 +++++++++++++++++++++++++++++++++++++++++++
 tox.ini               | 14 +++++++
 4 files changed, 150 insertions(+), 39 deletions(-)
 create mode 100644 pylintrc

diff --git a/.travis.yml b/.travis.yml
index 0190a7f4d..28a19f4f0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,6 +7,8 @@ branches:
     - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
 matrix:
   include:
+    - env: TOXENV=pylint
+      python: 3.7
     - env: TOXENV=py27
       python: 2.7
     - env: TOXENV=py27-pinned
diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
index 6290adbe2..9acfc3b23 100755
--- a/docs/utils/linkfix.py
+++ b/docs/utils/linkfix.py
@@ -14,50 +14,57 @@ Author: dufferzafar
 
 import re
 
-# Used for remembering the file (and its contents)
-# so we don't have to open the same file again.
-_filename = None
-_contents = None
 
-# A regex that matches standard linkcheck output lines
-line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
+def main():
 
-# Read lines from the linkcheck output file
-try:
-    with open("build/linkcheck/output.txt") as out:
-        output_lines = out.readlines()
-except IOError:
-    print("linkcheck output not found; please run linkcheck first.")
-    exit(1)
+    # Used for remembering the file (and its contents)
+    # so we don't have to open the same file again.
+    _filename = None
+    _contents = None
 
-# For every line, fix the respective file
-for line in output_lines:
-    match = re.match(line_re, line)
+    # A regex that matches standard linkcheck output lines
+    line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
 
-    if match:
-        newfilename = match.group(1)
-        errortype = match.group(2)
+    # Read lines from the linkcheck output file
+    try:
+        with open("build/linkcheck/output.txt") as out:
+            output_lines = out.readlines()
+    except IOError:
+        print("linkcheck output not found; please run linkcheck first.")
+        exit(1)
 
-        # Broken links can't be fixed and
-        # I am not sure what do with the local ones.
-        if errortype.lower() in ["broken", "local"]:
-            print("Not Fixed: " + line)
+    # For every line, fix the respective file
+    for line in output_lines:
+        match = re.match(line_re, line)
+
+        if match:
+            newfilename = match.group(1)
+            errortype = match.group(2)
+
+            # Broken links can't be fixed and
+            # I am not sure what do with the local ones.
+            if errortype.lower() in ["broken", "local"]:
+                print("Not Fixed: " + line)
+            else:
+                # If this is a new file
+                if newfilename != _filename:
+
+                    # Update the previous file
+                    if _filename:
+                        with open(_filename, "w") as _file:
+                            _file.write(_contents)
+
+                    _filename = newfilename
+
+                    # Read the new file to memory
+                    with open(_filename) as _file:
+                        _contents = _file.read()
+
+                _contents = _contents.replace(match.group(3), match.group(4))
         else:
-            # If this is a new file
-            if newfilename != _filename:
+            # We don't understand what the current line means!
+            print("Not Understood: " + line)
 
-                # Update the previous file
-                if _filename:
-                    with open(_filename, "w") as _file:
-                        _file.write(_contents)
 
-                _filename = newfilename
-
-                # Read the new file to memory
-                with open(_filename) as _file:
-                    _contents = _file.read()
-
-            _contents = _contents.replace(match.group(3), match.group(4))
-    else:
-        # We don't understand what the current line means!
-        print("Not Understood: " + line)
+if __name__ == '__main__':
+    main()
diff --git a/pylintrc b/pylintrc
new file mode 100644
index 000000000..b83bc9f82
--- /dev/null
+++ b/pylintrc
@@ -0,0 +1,88 @@
+[MASTER]
+persistent=no
+jobs=1  # >1 hides results
+
+[MESSAGES CONTROL]
+disable=abstract-method,
+        anomalous-backslash-in-string,
+        arguments-differ,
+        attribute-defined-outside-init,
+        bad-classmethod-argument,
+        bad-continuation,
+        bad-indentation,
+        bad-mcs-classmethod-argument,
+        bad-whitespace,
+        broad-except,
+        c-extension-no-member,
+        catching-non-exception,
+        cell-var-from-loop,
+        comparison-with-callable,
+        consider-using-in,
+        cyclic-import,
+        dangerous-default-value,
+        deprecated-method,
+        deprecated-module,
+        duplicate-code,  # https://github.com/PyCQA/pylint/issues/214
+        eval-used,
+        expression-not-assigned,
+        fixme,
+        function-redefined,
+        global-statement,
+        import-error,
+        import-outside-toplevel,
+        inconsistent-return-statements,
+        inherit-non-class,
+        invalid-name,
+        keyword-arg-before-vararg,
+        line-too-long,
+        logging-format-interpolation,
+        logging-not-lazy,
+        lost-exception,
+        method-hidden,
+        missing-docstring,
+        missing-final-newline,
+        multiple-imports,
+        multiple-statements,
+        no-else-continue,
+        no-else-raise,
+        no-else-return,
+        no-init,
+        no-member,
+        no-method-argument,
+        no-name-in-module,
+        no-self-argument,
+        no-self-use,
+        pointless-string-statement,
+        protected-access,
+        redefined-argument-from-local,
+        redefined-builtin,
+        redefined-outer-name,
+        reimported,
+        signature-differs,
+        super-init-not-called,
+        superfluous-parens,
+        too-few-public-methods,
+        too-many-ancestors,
+        too-many-arguments,
+        too-many-branches,
+        too-many-function-args,
+        too-many-instance-attributes,
+        too-many-locals,
+        too-many-return-statements,
+        trailing-newlines,
+        trailing-whitespace,
+        unexpected-special-method-signature,
+        ungrouped-imports,
+        unidiomatic-typecheck,
+        unnecessary-comprehension,
+        unnecessary-pass,
+        unsubscriptable-object,
+        unused-argument,
+        unused-import,
+        unused-variable,
+        unused-wildcard-import,
+        used-before-assignment,
+        useless-object-inheritance,  # Required for Python 2 support
+        wildcard-import,
+        wrong-import-order,
+        wrong-import-position
diff --git a/tox.ini b/tox.ini
index ffe7360d3..e7d366fe9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -98,6 +98,20 @@ deps = {[testenv:py35]deps}
 commands =
     py.test {posargs:scrapy tests}
 
+[testenv:pylint]
+basepython = python3.7
+deps =
+    {[testenv:py35]deps}
+    # Optional dependencies
+    boto
+    reppy
+    robotexclusionrulesparser
+    # Test dependencies
+    pylint
+
+commands =
+    pylint scrapy
+
 [docs]
 changedir = docs
 deps =

From 84fe4011b0063dae1a8efbcb563e772d3b9fce09 Mon Sep 17 00:00:00 2001
From: Kevin Lloyd Bernal <kevinoxy@gmail.com>
Date: Wed, 23 Oct 2019 20:39:53 +0800
Subject: [PATCH 2280/2974] update docs of scrapy.loader.ItemLoader.item

---
 docs/topics/loaders.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 1c2f1da4d..4bd564014 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -485,6 +485,8 @@ ItemLoader objects
     .. attribute:: item
 
         The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
+        This is mostly used as a property so when attempting to override this
+        value, you may want to check out :attr:`default_item_class` first.
 
     .. attribute:: context
 

From 02577f55a0586bc3e6c13a4a3ea572c7eefc82b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 24 Oct 2019 13:25:11 +0200
Subject: [PATCH 2281/2974] Have PyLint cover all Python files in the
 repository

---
 pylintrc | 19 +++++++++++++++++++
 tox.ini  |  2 +-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/pylintrc b/pylintrc
index b83bc9f82..ca3ea1c57 100644
--- a/pylintrc
+++ b/pylintrc
@@ -11,13 +11,18 @@ disable=abstract-method,
         bad-continuation,
         bad-indentation,
         bad-mcs-classmethod-argument,
+        bad-super-call,
         bad-whitespace,
+        blacklisted-name,
         broad-except,
         c-extension-no-member,
         catching-non-exception,
         cell-var-from-loop,
         comparison-with-callable,
+        consider-iterating-dictionary,
         consider-using-in,
+        consider-using-set-comprehension,
+        consider-using-sys-exit,
         cyclic-import,
         dangerous-default-value,
         deprecated-method,
@@ -30,6 +35,7 @@ disable=abstract-method,
         global-statement,
         import-error,
         import-outside-toplevel,
+        import-self,
         inconsistent-return-statements,
         inherit-non-class,
         invalid-name,
@@ -39,6 +45,7 @@ disable=abstract-method,
         logging-not-lazy,
         lost-exception,
         method-hidden,
+        misplaced-comparison-constant,
         missing-docstring,
         missing-final-newline,
         multiple-imports,
@@ -52,6 +59,9 @@ disable=abstract-method,
         no-name-in-module,
         no-self-argument,
         no-self-use,
+        no-value-for-parameter,
+        not-callable,
+        pointless-statement,
         pointless-string-statement,
         protected-access,
         redefined-argument-from-local,
@@ -59,6 +69,7 @@ disable=abstract-method,
         redefined-outer-name,
         reimported,
         signature-differs,
+        singleton-comparison,
         super-init-not-called,
         superfluous-parens,
         too-few-public-methods,
@@ -67,15 +78,21 @@ disable=abstract-method,
         too-many-branches,
         too-many-function-args,
         too-many-instance-attributes,
+        too-many-lines,
         too-many-locals,
+        too-many-public-methods,
         too-many-return-statements,
         trailing-newlines,
         trailing-whitespace,
+        unbalanced-tuple-unpacking,
+        undefined-variable,
         unexpected-special-method-signature,
         ungrouped-imports,
         unidiomatic-typecheck,
         unnecessary-comprehension,
+        unnecessary-lambda,
         unnecessary-pass,
+        unreachable,
         unsubscriptable-object,
         unused-argument,
         unused-import,
@@ -83,6 +100,8 @@ disable=abstract-method,
         unused-wildcard-import,
         used-before-assignment,
         useless-object-inheritance,  # Required for Python 2 support
+        useless-return,
+        useless-super-delegation,
         wildcard-import,
         wrong-import-order,
         wrong-import-position
diff --git a/tox.ini b/tox.ini
index e7d366fe9..428571ef2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -110,7 +110,7 @@ deps =
     pylint
 
 commands =
-    pylint scrapy
+    pylint conftest.py docs extras scrapy setup.py tests
 
 [docs]
 changedir = docs

From 179dc916ff3e3518d8874f3fcd93948e11d5a259 Mon Sep 17 00:00:00 2001
From: Roy Healy <roy.healy87+gh@gmail.com>
Date: Sun, 27 Oct 2019 16:48:20 +0000
Subject: [PATCH 2282/2974] Update setup.py

Updating setup.py to remove python 3.8 support for now
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 2f5fca4c9..4127d3191 100644
--- a/setup.py
+++ b/setup.py
@@ -56,7 +56,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
+        # 'Programming Language :: Python :: 3.8',  not supported yet
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
         'Topic :: Internet :: WWW/HTTP',

From 4068797558f6d8d58f81e12930f487fc0f59f0a5 Mon Sep 17 00:00:00 2001
From: Roy Healy <roy.healy87+gh@gmail.com>
Date: Sun, 27 Oct 2019 17:02:17 +0000
Subject: [PATCH 2283/2974] Update test_downloadermiddleware_httpcache.py

Adding xfail denoting that leveldb is not supported in 3.8
---
 tests/test_downloadermiddleware_httpcache.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 22946b98c..34bf5776a 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -154,6 +154,7 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
         new_settings.setdefault('HTTPCACHE_GZIP', True)
         return super(FilesystemStorageTest, self)._get_settings(**new_settings)
 
+@pytest.mark.xfail(reason='leveldb not supported in python 3.8')
 class LeveldbStorageTest(DefaultStorageTest):
 
     pytest.importorskip('leveldb')

From deacd34c8d2400b07e911dea9173b840cec7dece Mon Sep 17 00:00:00 2001
From: Roy <roy.healy87+gh@gmail.com>
Date: Sun, 27 Oct 2019 17:39:47 +0000
Subject: [PATCH 2284/2974] [test_downloadermiddleware_httpcache] Attempting to
 add xfail for leveldb related tests
 https://github.com/scrapy/scrapy/issues/4085

---
 tests/test_downloadermiddleware_httpcache.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 34bf5776a..e350da72c 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -104,6 +104,7 @@ class _BaseTest(unittest.TestCase):
 
 class DefaultStorageTest(_BaseTest):
 
+    @pytest.mark.xfail(reason='leveldb not supported in python 3.8')
     def test_storage(self):
         with self._storage() as storage:
             request2 = self.request.copy()
@@ -154,7 +155,6 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
         new_settings.setdefault('HTTPCACHE_GZIP', True)
         return super(FilesystemStorageTest, self)._get_settings(**new_settings)
 
-@pytest.mark.xfail(reason='leveldb not supported in python 3.8')
 class LeveldbStorageTest(DefaultStorageTest):
 
     pytest.importorskip('leveldb')

From 11942c436c78e998e3d58f0aad7cba490d74cc67 Mon Sep 17 00:00:00 2001
From: Roy <roy.healy87+gh@gmail.com>
Date: Sun, 27 Oct 2019 18:02:13 +0000
Subject: [PATCH 2285/2974] [test_downloadermiddleware_httpcache] Trying hack
 to handle systemerror whjen importing leveldb
 https://github.com/scrapy/scrapy/issues/4085

---
 tests/test_downloadermiddleware_httpcache.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index e350da72c..eec0feafc 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -6,6 +6,7 @@ import unittest
 import email.utils
 from contextlib import contextmanager
 import pytest
+import sys
 
 from scrapy.http import Response, HtmlResponse, Request
 from scrapy.spiders import Spider
@@ -157,7 +158,10 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
 
 class LeveldbStorageTest(DefaultStorageTest):
 
-    pytest.importorskip('leveldb')
+    try:
+        pytest.importorskip('leveldb')
+    except SystemError:
+        pass
     storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage'
 
 

From b3df0a84150f15ff00cdfaffc586f91b653baecd Mon Sep 17 00:00:00 2001
From: Roy <roy.healy87+gh@gmail.com>
Date: Sun, 27 Oct 2019 18:28:47 +0000
Subject: [PATCH 2286/2974] [test_downloadermiddleware_httpcache] Adding xfails
 to impacted tests following hack fix
 https://github.com/scrapy/scrapy/issues/4085

---
 tests/test_downloadermiddleware_httpcache.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index eec0feafc..605223088 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -6,7 +6,6 @@ import unittest
 import email.utils
 from contextlib import contextmanager
 import pytest
-import sys
 
 from scrapy.http import Response, HtmlResponse, Request
 from scrapy.spiders import Spider
@@ -90,6 +89,7 @@ class _BaseTest(unittest.TestCase):
         assert any(h in request2.headers for h in (b'If-None-Match', b'If-Modified-Since'))
         self.assertEqual(request1.body, request2.body)
 
+    @pytest.mark.xfail(reason='leveldb not supported in python 3.8')
     def test_dont_cache(self):
         with self._middleware() as mw:
             self.request.meta['dont_cache'] = True
@@ -119,6 +119,7 @@ class DefaultStorageTest(_BaseTest):
             time.sleep(2)  # wait for cache to expire
             assert storage.retrieve_response(self.spider, request2) is None
 
+    @pytest.mark.xfail(reason='leveldb not supported in python 3.8')
     def test_storage_never_expire(self):
         with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as storage:
             assert storage.retrieve_response(self.spider, self.request) is None
@@ -161,6 +162,8 @@ class LeveldbStorageTest(DefaultStorageTest):
     try:
         pytest.importorskip('leveldb')
     except SystemError:
+        # Happens in python 3.8
+        # This will cause xfail in DefaultStorageTest to trigger
         pass
     storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage'
 

From 70b2854590c1aa10324e6a4b50b6d74a5079c8e9 Mon Sep 17 00:00:00 2001
From: Roy <roy.healy87+gh@gmail.com>
Date: Sun, 27 Oct 2019 18:51:26 +0000
Subject: [PATCH 2287/2974] [test_downloadermiddleware_httpcache] Making xfails
 more informative https://github.com/scrapy/scrapy/issues/4085

---
 tests/test_downloadermiddleware_httpcache.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 605223088..faee37446 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -6,6 +6,7 @@ import unittest
 import email.utils
 from contextlib import contextmanager
 import pytest
+import sys
 
 from scrapy.http import Response, HtmlResponse, Request
 from scrapy.spiders import Spider
@@ -89,7 +90,7 @@ class _BaseTest(unittest.TestCase):
         assert any(h in request2.headers for h in (b'If-None-Match', b'If-Modified-Since'))
         self.assertEqual(request1.body, request2.body)
 
-    @pytest.mark.xfail(reason='leveldb not supported in python 3.8')
+    @pytest.mark.xfail(sys.version_info >= (3, 8), raises=RuntimeError, reason='leveldb not supported in python 3.8')
     def test_dont_cache(self):
         with self._middleware() as mw:
             self.request.meta['dont_cache'] = True
@@ -105,7 +106,7 @@ class _BaseTest(unittest.TestCase):
 
 class DefaultStorageTest(_BaseTest):
 
-    @pytest.mark.xfail(reason='leveldb not supported in python 3.8')
+    @pytest.mark.xfail(sys.version_info >= (3, 8), raises=RuntimeError, reason='leveldb not supported in python 3.8')
     def test_storage(self):
         with self._storage() as storage:
             request2 = self.request.copy()
@@ -119,7 +120,7 @@ class DefaultStorageTest(_BaseTest):
             time.sleep(2)  # wait for cache to expire
             assert storage.retrieve_response(self.spider, request2) is None
 
-    @pytest.mark.xfail(reason='leveldb not supported in python 3.8')
+    @pytest.mark.xfail(sys.version_info >= (3, 8), raises=RuntimeError, reason='leveldb not supported in python 3.8')
     def test_storage_never_expire(self):
         with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as storage:
             assert storage.retrieve_response(self.spider, self.request) is None

From 20ea912513053aa2b2122ba5bb0387189bbc5467 Mon Sep 17 00:00:00 2001
From: Roy <roy.healy87+gh@gmail.com>
Date: Sun, 27 Oct 2019 18:52:01 +0000
Subject: [PATCH 2288/2974] [test_downloadermiddleware_httpcache] Making xfails
 more informative https://github.com/scrapy/scrapy/issues/4085

---
 tests/test_downloadermiddleware_httpcache.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index faee37446..b832fc38c 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -90,7 +90,7 @@ class _BaseTest(unittest.TestCase):
         assert any(h in request2.headers for h in (b'If-None-Match', b'If-Modified-Since'))
         self.assertEqual(request1.body, request2.body)
 
-    @pytest.mark.xfail(sys.version_info >= (3, 8), raises=RuntimeError, reason='leveldb not supported in python 3.8')
+    @pytest.mark.xfail(sys.version_info >= (3, 8), raises=SystemError, reason='leveldb not supported in python 3.8')
     def test_dont_cache(self):
         with self._middleware() as mw:
             self.request.meta['dont_cache'] = True
@@ -106,7 +106,7 @@ class _BaseTest(unittest.TestCase):
 
 class DefaultStorageTest(_BaseTest):
 
-    @pytest.mark.xfail(sys.version_info >= (3, 8), raises=RuntimeError, reason='leveldb not supported in python 3.8')
+    @pytest.mark.xfail(sys.version_info >= (3, 8), raises=SystemError, reason='leveldb not supported in python 3.8')
     def test_storage(self):
         with self._storage() as storage:
             request2 = self.request.copy()
@@ -120,7 +120,7 @@ class DefaultStorageTest(_BaseTest):
             time.sleep(2)  # wait for cache to expire
             assert storage.retrieve_response(self.spider, request2) is None
 
-    @pytest.mark.xfail(sys.version_info >= (3, 8), raises=RuntimeError, reason='leveldb not supported in python 3.8')
+    @pytest.mark.xfail(sys.version_info >= (3, 8), raises=SystemError, reason='leveldb not supported in python 3.8')
     def test_storage_never_expire(self):
         with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as storage:
             assert storage.retrieve_response(self.spider, self.request) is None

From bb91f9c78c9c8c892495f6e6252cbeebbe12725b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 21 Oct 2019 12:08:35 +0200
Subject: [PATCH 2289/2974] Cover Scrapy 1.7.4 in the release notes

---
 docs/news.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 59317f5eb..8dfe8693c 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -6,6 +6,18 @@ Release notes
 .. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0,
           planned for Q4 2019 or Q1 2020, will support **Python 3 only**.
 
+Scrapy 1.7.4 (2019-10-21)
+-------------------------
+
+Revert the fix for :issue:`3804` (:issue:`3819`), which has a few undesired
+side effects (:issue:`3897`, :issue:`3976`).
+
+As a result, when an item loader is initialized with an item,
+:meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` once again
+makes later calls to :meth:`ItemLoader.get_output_value()
+<scrapy.loader.ItemLoader.get_output_value>` or :meth:`ItemLoader.load_item()
+<scrapy.loader.ItemLoader.load_item>` return empty data.
+
 Scrapy 1.7.3 (2019-08-01)
 -------------------------
 

From 7731814cc25c57fe31db9ba749450cd5a27eed39 Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Mon, 28 Oct 2019 06:53:53 -0300
Subject: [PATCH 2290/2974] ItemLoader: improve handling of initial item
 (#4036)

---
 docs/topics/loaders.rst   |  10 +-
 scrapy/loader/__init__.py |  31 ++--
 tests/test_loader.py      | 321 +++++++++++++++++++++++++++++---------
 3 files changed, 273 insertions(+), 89 deletions(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 1c2f1da4d..0318e37aa 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -35,6 +35,12 @@ Then, you start collecting values into the Item Loader, typically using
 the same item field; the Item Loader will know how to "join" those values later
 using a proper processing function.
 
+.. note:: Collected data is internally stored as lists,
+   allowing to add several values to the same field.
+   If an ``item`` argument is passed when creating a loader,
+   each of the item's values will be stored as-is if it's already
+   an iterable, or wrapped with a list if it's a single value.
+
 Here is a typical Item Loader usage in a :ref:`Spider <topics-spiders>`, using
 the :ref:`Product item <topics-items-declaring>` declared in the :ref:`Items
 chapter <topics-items>`::
@@ -128,9 +134,9 @@ So what happens is:
 It's worth noticing that processors are just callable objects, which are called
 with the data to be parsed, and return a parsed value. So you can use any
 function as input or output processor. The only requirement is that they must
-accept one (and only one) positional argument, which will be an iterator.
+accept one (and only one) positional argument, which will be an iterable.
 
-.. note:: Both input and output processors must receive an iterator as their
+.. note:: Both input and output processors must receive an iterable as their
    first argument. The output of those functions can be anything. The result of
    input processors will be appended to an internal list (in the Loader)
    containing the collected values (for that field). The result of the output
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 6665eba16..60fd6d222 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -1,19 +1,19 @@
-"""Item Loader
+"""
+Item Loader
 
 See documentation in docs/topics/loaders.rst
-
 """
 from collections import defaultdict
+
 import six
 
 from scrapy.item import Item
+from scrapy.loader.common import wrap_loader_context
+from scrapy.loader.processors import Identity
 from scrapy.selector import Selector
 from scrapy.utils.misc import arg_to_iter, extract_regex
 from scrapy.utils.python import flatten
 
-from .common import wrap_loader_context
-from .processors import Identity
-
 
 class ItemLoader(object):
 
@@ -33,10 +33,9 @@ class ItemLoader(object):
         self.parent = parent
         self._local_item = context['item'] = item
         self._local_values = defaultdict(list)
-        # Preprocess values if item built from dict
-        # Values need to be added to item._values if added them from dict (not with add_values)
+        # values from initial item
         for field_name, value in item.items():
-            self._values[field_name] = self._process_input_value(field_name, value)
+            self._values[field_name] += arg_to_iter(value)
 
     @property
     def _values(self):
@@ -132,8 +131,8 @@ class ItemLoader(object):
         try:
             return proc(self._values[field_name])
         except Exception as e:
-            raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" % \
-                (field_name, self._values[field_name], type(e).__name__, str(e)))
+            raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" %
+                             (field_name, self._values[field_name], type(e).__name__, str(e)))
 
     def get_collected_values(self, field_name):
         return self._values[field_name]
@@ -141,15 +140,15 @@ class ItemLoader(object):
     def get_input_processor(self, field_name):
         proc = getattr(self, '%s_in' % field_name, None)
         if not proc:
-            proc = self._get_item_field_attr(field_name, 'input_processor', \
-                self.default_input_processor)
+            proc = self._get_item_field_attr(field_name, 'input_processor',
+                                             self.default_input_processor)
         return proc
 
     def get_output_processor(self, field_name):
         proc = getattr(self, '%s_out' % field_name, None)
         if not proc:
-            proc = self._get_item_field_attr(field_name, 'output_processor', \
-                self.default_output_processor)
+            proc = self._get_item_field_attr(field_name, 'output_processor',
+                                             self.default_output_processor)
         return proc
 
     def _process_input_value(self, field_name, value):
@@ -174,8 +173,8 @@ class ItemLoader(object):
     def _check_selector_method(self):
         if self.selector is None:
             raise RuntimeError("To use XPath or CSS selectors, "
-                "%s must be instantiated with a selector "
-                "or a response" % self.__class__.__name__)
+                               "%s must be instantiated with a selector "
+                               "or a response" % self.__class__.__name__)
 
     def add_xpath(self, field_name, xpath, *processors, **kw):
         values = self._get_xpathvalues(xpath, **kw)
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 2725b001a..4a4264a2a 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -1,13 +1,15 @@
-import unittest
-import six
 from functools import partial
+import unittest
+
+import six
 
-from scrapy.loader import ItemLoader
-from scrapy.loader.processors import Join, Identity, TakeFirst, \
-    Compose, MapCompose, SelectJmes
-from scrapy.item import Item, Field
-from scrapy.selector import Selector
 from scrapy.http import HtmlResponse
+from scrapy.item import Item, Field
+from scrapy.loader import ItemLoader
+from scrapy.loader.processors import (Compose, Identity, Join,
+                                      MapCompose, SelectJmes, TakeFirst)
+from scrapy.selector import Selector
+
 
 # test items
 class NameItem(Item):
@@ -61,7 +63,7 @@ class BasicItemLoaderTest(unittest.TestCase):
         il.add_value('name', u'marta')
         item = il.load_item()
         assert item is i
-        self.assertEqual(item['summary'], u'lala')
+        self.assertEqual(item['summary'], [u'lala'])
         self.assertEqual(item['name'], [u'marta'])
 
     def test_load_item_using_custom_loader(self):
@@ -419,43 +421,6 @@ class BasicItemLoaderTest(unittest.TestCase):
         self.assertEqual(item['url'], u'rabbit.hole')
         self.assertEqual(item['summary'], u'rabbithole')
 
-    def test_create_item_from_dict(self):
-        class TestItem(Item):
-            title = Field()
-
-        class TestItemLoader(ItemLoader):
-            default_item_class = TestItem
-
-        input_item = {'title': 'Test item title 1'}
-        il = TestItemLoader(item=input_item)
-        # Getting output value mustn't remove value from item
-        self.assertEqual(il.load_item(), {
-            'title': 'Test item title 1',
-        })
-        self.assertEqual(il.get_output_value('title'), 'Test item title 1')
-        self.assertEqual(il.load_item(), {
-            'title': 'Test item title 1',
-        })
-
-        input_item = {'title': 'Test item title 2'}
-        il = TestItemLoader(item=input_item)
-        # Values from dict must be added to item _values
-        self.assertEqual(il._values.get('title'), 'Test item title 2')
-
-        input_item = {'title': [u'Test item title 3', u'Test item 4']}
-        il = TestItemLoader(item=input_item)
-        # Same rules must work for lists
-        self.assertEqual(il._values.get('title'),
-                         [u'Test item title 3', u'Test item 4'])
-        self.assertEqual(il.load_item(), {
-            'title': [u'Test item title 3', u'Test item 4'],
-        })
-        self.assertEqual(il.get_output_value('title'),
-                         [u'Test item title 3', u'Test item 4'])
-        self.assertEqual(il.load_item(), {
-            'title': [u'Test item title 3', u'Test item 4'],
-        })
-
     def test_error_input_processor(self):
         class TestItem(Item):
             name = Field()
@@ -493,6 +458,220 @@ class BasicItemLoaderTest(unittest.TestCase):
                           [u'marta', u'other'], Compose(float))
 
 
+class InitializationTestMixin(object):
+
+    item_class = None
+
+    def test_keep_single_value(self):
+        """Loaded item should contain values from the initial item"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo']})
+
+    def test_keep_list(self):
+        """Loaded item should contain values from the initial item"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+
+    def test_add_value_singlevalue_singlevalue(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        il.add_value('name', 'bar')
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+
+    def test_add_value_singlevalue_list(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        il.add_value('name', ['item', 'loader'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'item', 'loader']})
+
+    def test_add_value_list_singlevalue(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        il.add_value('name', 'qwerty')
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'qwerty']})
+
+    def test_add_value_list_list(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        il.add_value('name', ['item', 'loader'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'item', 'loader']})
+
+    def test_get_output_value_singlevalue(self):
+        """Getting output value must not remove value from item"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il.get_output_value('name'), ['foo'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(loaded_item, dict({'name': ['foo']}))
+
+    def test_get_output_value_list(self):
+        """Getting output value must not remove value from item"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il.get_output_value('name'), ['foo', 'bar'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(loaded_item, dict({'name': ['foo', 'bar']}))
+
+    def test_values_single(self):
+        """Values from initial item must be added to loader._values"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il._values.get('name'), ['foo'])
+
+    def test_values_list(self):
+        """Values from initial item must be added to loader._values"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il._values.get('name'), ['foo', 'bar'])
+
+
+class InitializationFromDictTest(InitializationTestMixin, unittest.TestCase):
+    item_class = dict
+
+
+class InitializationFromItemTest(InitializationTestMixin, unittest.TestCase):
+    item_class = NameItem
+
+
+class BaseNoInputReprocessingLoader(ItemLoader):
+    title_in = MapCompose(str.upper)
+    title_out = TakeFirst()
+
+
+class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader):
+    default_item_class = dict
+
+
+class NoInputReprocessingFromDictTest(unittest.TestCase):
+    """
+    Loaders initialized from loaded items must not reprocess fields (dict instances)
+    """
+    def test_avoid_reprocessing_with_initial_values_single(self):
+        il = NoInputReprocessingDictLoader(item=dict(title='foo'))
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='foo'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
+
+    def test_avoid_reprocessing_with_initial_values_list(self):
+        il = NoInputReprocessingDictLoader(item=dict(title=['foo', 'bar']))
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='foo'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
+
+    def test_avoid_reprocessing_without_initial_values_single(self):
+        il = NoInputReprocessingDictLoader()
+        il.add_value('title', 'foo')
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='FOO'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
+
+    def test_avoid_reprocessing_without_initial_values_list(self):
+        il = NoInputReprocessingDictLoader()
+        il.add_value('title', ['foo', 'bar'])
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='FOO'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
+
+
+class NoInputReprocessingItem(Item):
+    title = Field()
+
+
+class NoInputReprocessingItemLoader(BaseNoInputReprocessingLoader):
+    default_item_class = NoInputReprocessingItem
+
+
+class NoInputReprocessingFromItemTest(unittest.TestCase):
+    """
+    Loaders initialized from loaded items must not reprocess fields (BaseItem instances)
+    """
+    def test_avoid_reprocessing_with_initial_values_single(self):
+        il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title='foo'))
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, {'title': 'foo'})
+        self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'foo'})
+
+    def test_avoid_reprocessing_with_initial_values_list(self):
+        il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title=['foo', 'bar']))
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, {'title': 'foo'})
+        self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'foo'})
+
+    def test_avoid_reprocessing_without_initial_values_single(self):
+        il = NoInputReprocessingItemLoader()
+        il.add_value('title', 'FOO')
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, {'title': 'FOO'})
+        self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'FOO'})
+
+    def test_avoid_reprocessing_without_initial_values_list(self):
+        il = NoInputReprocessingItemLoader()
+        il.add_value('title', ['foo', 'bar'])
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, {'title': 'FOO'})
+        self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'FOO'})
+
+
+class TestOutputProcessorDict(unittest.TestCase):
+    def test_output_processor(self):
+
+        class TempDict(dict):
+            def __init__(self, *args, **kwargs):
+                super(TempDict, self).__init__(self, *args, **kwargs)
+                self.setdefault('temp', 0.3)
+
+        class TempLoader(ItemLoader):
+            default_item_class = TempDict
+            default_input_processor = Identity()
+            default_output_processor = Compose(TakeFirst())
+
+        loader = TempLoader()
+        item = loader.load_item()
+        self.assertIsInstance(item, TempDict)
+        self.assertEqual(dict(item), {'temp': 0.3})
+
+
+class TestOutputProcessorItem(unittest.TestCase):
+    def test_output_processor(self):
+
+        class TempItem(Item):
+            temp = Field()
+
+            def __init__(self, *args, **kwargs):
+                super(TempItem, self).__init__(self, *args, **kwargs)
+                self.setdefault('temp', 0.3)
+
+        class TempLoader(ItemLoader):
+            default_item_class = TempItem
+            default_input_processor = Identity()
+            default_output_processor = Compose(TakeFirst())
+
+        loader = TempLoader()
+        item = loader.load_item()
+        self.assertIsInstance(item, TempItem)
+        self.assertEqual(dict(item), {'temp': 0.3})
+
+
 class ProcessorsTest(unittest.TestCase):
 
     def test_take_first(self):
@@ -523,7 +702,8 @@ class ProcessorsTest(unittest.TestCase):
         self.assertRaises(ValueError, proc, 'hello')
 
     def test_mapcompose(self):
-        filter_world = lambda x: None if x == 'world' else x
+        def filter_world(x):
+            return None if x == 'world' else x
         proc = MapCompose(filter_world, six.text_type.upper)
         self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
                          [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
@@ -535,7 +715,6 @@ class ProcessorsTest(unittest.TestCase):
         self.assertRaises(ValueError, proc, 'hello')
 
 
-
 class SelectortemLoaderTest(unittest.TestCase):
     response = HtmlResponse(url="", encoding='utf-8', body=b"""
     <html>
@@ -672,7 +851,7 @@ class SelectortemLoaderTest(unittest.TestCase):
 
         self.assertEqual(l.get_css(['p::text', 'div::text']), [u'paragraph', 'marta'])
         self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']),
-            [u'http://www.scrapy.org', u'/images/logo.png'])
+                         [u'http://www.scrapy.org', u'/images/logo.png'])
 
     def test_replace_css_multi_fields(self):
         l = TestItemLoader(response=self.response)
@@ -720,7 +899,7 @@ class SubselectorLoaderTest(unittest.TestCase):
 
         self.assertEqual(l.get_output_value('name'), [u'marta'])
         self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
-        self.assertEqual(l.get_output_value('name_value'),  [u'marta'])
+        self.assertEqual(l.get_output_value('name_value'), [u'marta'])
 
         self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
         self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
@@ -735,7 +914,7 @@ class SubselectorLoaderTest(unittest.TestCase):
 
         self.assertEqual(l.get_output_value('name'), [u'marta'])
         self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
-        self.assertEqual(l.get_output_value('name_value'),  [u'marta'])
+        self.assertEqual(l.get_output_value('name_value'), [u'marta'])
 
         self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
         self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
@@ -791,28 +970,28 @@ class SubselectorLoaderTest(unittest.TestCase):
 
 
 class SelectJmesTestCase(unittest.TestCase):
-        test_list_equals = {
-            'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
-            'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
-            'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
-            'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
-            'dict': (
-                'foo.bar[*].name',
-                {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
-                ['one', 'two']
-            ),
-            'list': ('[1]', [1, 2], 2)
-        }
+    test_list_equals = {
+        'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+        'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
+        'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
+        'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+        'dict': (
+            'foo.bar[*].name',
+            {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
+            ['one', 'two']
+        ),
+        'list': ('[1]', [1, 2], 2)
+    }
 
-        def test_output(self):
-            for l in self.test_list_equals:
-                expr, test_list, expected = self.test_list_equals[l]
-                test = SelectJmes(expr)(test_list)
-                self.assertEqual(
-                    test,
-                    expected,
-                    msg='test "{}" got {} expected {}'.format(l, test, expected)
-                )
+    def test_output(self):
+        for l in self.test_list_equals:
+            expr, test_list, expected = self.test_list_equals[l]
+            test = SelectJmes(expr)(test_list)
+            self.assertEqual(
+                test,
+                expected,
+                msg='test "{}" got {} expected {}'.format(l, test, expected)
+            )
 
 
 if __name__ == "__main__":

From b5a00262ec48534b89750037060318326b4e349c Mon Sep 17 00:00:00 2001
From: Roy Healy <roy.healy87+gh@gmail.com>
Date: Mon, 28 Oct 2019 09:59:33 +0000
Subject: [PATCH 2291/2974] Update .travis.yml

Reverting change to 3.8 extra dependency environment.
---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 044fa9e95..2ba504972 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,7 +29,7 @@ matrix:
       python: 3.7
     - env: TOXENV=py38
       python: 3.8
-    - env: TOXENV=py38-extra-deps
+    - env: TOXENV=py37-extra-deps
       python: 3.8
     - env: TOXENV=docs
       python: 3.6

From 3d0df419c4eeb0ccf7934c8f06a38707eae8d722 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 28 Oct 2019 11:24:47 +0100
Subject: [PATCH 2292/2974] Mark the LevelDB storage backend as deprecated

---
 docs/topics/downloader-middleware.rst | 22 ----------------------
 scrapy/extensions/httpcache.py        | 17 ++++++++++++-----
 2 files changed, 12 insertions(+), 27 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 2892b9b79..8048e1c86 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -348,7 +348,6 @@ HttpCacheMiddleware
 
         * :ref:`httpcache-storage-fs`
         * :ref:`httpcache-storage-dbm`
-        * :ref:`httpcache-storage-leveldb`
 
     You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE`
     setting. Or you can also :ref:`implement your own storage backend. <httpcache-storage-custom>`
@@ -478,27 +477,6 @@ DBM storage backend
     By default, it uses the anydbm_ module, but you can change it with the
     :setting:`HTTPCACHE_DBM_MODULE` setting.
 
-.. _httpcache-storage-leveldb:
-
-LevelDB storage backend
-~~~~~~~~~~~~~~~~~~~~~~~
-
-.. class:: LeveldbCacheStorage
-
-    .. versionadded:: 0.23
-
-    A LevelDB_ storage backend is also available for the HTTP cache middleware.
-
-    This backend is not recommended for development because only one process
-    can access LevelDB databases at the same time, so you can't run a crawl and
-    open the scrapy shell in parallel for the same spider.
-
-    In order to use this storage backend, install the `LevelDB python
-    bindings`_ (e.g. ``pip install leveldb``).
-
-    .. _LevelDB: https://github.com/google/leveldb
-    .. _leveldb python bindings: https://pypi.python.org/pypi/leveldb
-
 .. _httpcache-storage-custom:
 
 Writing your own storage backend
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index c6094643d..7c650a91e 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,19 +1,24 @@
 from __future__ import print_function
-import os
+
 import gzip
 import logging
-from six.moves import cPickle as pickle
+import os
+from email.utils import mktime_tz, parsedate_tz
 from importlib import import_module
 from time import time
+from warnings import warn
 from weakref import WeakKeyDictionary
-from email.utils import mktime_tz, parsedate_tz
+
+from six.moves import cPickle as pickle
 from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
+
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Headers, Response
 from scrapy.responsetypes import responsetypes
-from scrapy.utils.request import request_fingerprint
-from scrapy.utils.project import data_path
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.project import data_path
 from scrapy.utils.python import to_bytes, to_unicode, garbage_collect
+from scrapy.utils.request import request_fingerprint
 
 
 logger = logging.getLogger(__name__)
@@ -345,6 +350,8 @@ class FilesystemCacheStorage(object):
 class LeveldbCacheStorage(object):
 
     def __init__(self, settings):
+        warn("The LevelDB storage backend is deprecated.",
+             ScrapyDeprecationWarning, stacklevel=2)
         import leveldb
         self._leveldb = leveldb
         self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)

From 16bb3ac20dae8b7c5fbccf4ab85b3a0393e7c55d Mon Sep 17 00:00:00 2001
From: Roy <roy.healy87+gh@gmail.com>
Date: Mon, 28 Oct 2019 11:24:09 +0000
Subject: [PATCH 2293/2974] [test_downloadermiddleware_httpcache] Using skipif
 approach https://github.com/scrapy/scrapy/issues/4085

---
 tests/test_downloadermiddleware_httpcache.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index b832fc38c..ba5027307 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -90,7 +90,6 @@ class _BaseTest(unittest.TestCase):
         assert any(h in request2.headers for h in (b'If-None-Match', b'If-Modified-Since'))
         self.assertEqual(request1.body, request2.body)
 
-    @pytest.mark.xfail(sys.version_info >= (3, 8), raises=SystemError, reason='leveldb not supported in python 3.8')
     def test_dont_cache(self):
         with self._middleware() as mw:
             self.request.meta['dont_cache'] = True
@@ -106,7 +105,6 @@ class _BaseTest(unittest.TestCase):
 
 class DefaultStorageTest(_BaseTest):
 
-    @pytest.mark.xfail(sys.version_info >= (3, 8), raises=SystemError, reason='leveldb not supported in python 3.8')
     def test_storage(self):
         with self._storage() as storage:
             request2 = self.request.copy()
@@ -120,7 +118,6 @@ class DefaultStorageTest(_BaseTest):
             time.sleep(2)  # wait for cache to expire
             assert storage.retrieve_response(self.spider, request2) is None
 
-    @pytest.mark.xfail(sys.version_info >= (3, 8), raises=SystemError, reason='leveldb not supported in python 3.8')
     def test_storage_never_expire(self):
         with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as storage:
             assert storage.retrieve_response(self.spider, self.request) is None
@@ -164,8 +161,10 @@ class LeveldbStorageTest(DefaultStorageTest):
         pytest.importorskip('leveldb')
     except SystemError:
         # Happens in python 3.8
-        # This will cause xfail in DefaultStorageTest to trigger
-        pass
+        pytest.mark.skipif(
+            sys.version_info >= (3, 8),
+            reason='leveldb not supported in python 3.8',
+        )
     storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage'
 
 

From 9b47dc6a703310d13c9470e50d4b14f81ee893c6 Mon Sep 17 00:00:00 2001
From: Roy <roy.healy87+gh@gmail.com>
Date: Mon, 28 Oct 2019 11:24:52 +0000
Subject: [PATCH 2294/2974] [travis, setup] Adding official python 3.8 support
 https://github.com/scrapy/scrapy/issues/4085

---
 .travis.yml | 4 +---
 setup.py    | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 2ba504972..4c2498053 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,11 +25,9 @@ matrix:
       python: 3.6
     - env: TOXENV=py37
       python: 3.7
-    - env: TOXENV=py37-extra-deps
-      python: 3.7
     - env: TOXENV=py38
       python: 3.8
-    - env: TOXENV=py37-extra-deps
+    - env: TOXENV=py38-extra-deps
       python: 3.8
     - env: TOXENV=docs
       python: 3.6
diff --git a/setup.py b/setup.py
index 4127d3191..2f5fca4c9 100644
--- a/setup.py
+++ b/setup.py
@@ -56,7 +56,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
-        # 'Programming Language :: Python :: 3.8',  not supported yet
+        'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
         'Topic :: Internet :: WWW/HTTP',

From 4432136ffff4d8af42f7a485c17ab7fbbb228078 Mon Sep 17 00:00:00 2001
From: Roy <roy.healy87+gh@gmail.com>
Date: Mon, 28 Oct 2019 12:22:21 +0000
Subject: [PATCH 2295/2974] [test_downloadermiddleware_httpcache] Fixing pytest
 skip behaviour https://github.com/scrapy/scrapy/issues/4085

---
 tests/test_downloadermiddleware_httpcache.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index ba5027307..32085b095 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -6,7 +6,6 @@ import unittest
 import email.utils
 from contextlib import contextmanager
 import pytest
-import sys
 
 from scrapy.http import Response, HtmlResponse, Request
 from scrapy.spiders import Spider
@@ -161,10 +160,7 @@ class LeveldbStorageTest(DefaultStorageTest):
         pytest.importorskip('leveldb')
     except SystemError:
         # Happens in python 3.8
-        pytest.mark.skipif(
-            sys.version_info >= (3, 8),
-            reason='leveldb not supported in python 3.8',
-        )
+        pytest.skip("'SystemError: bad call flags' occurs on Python 3.8")
     storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage'
 
 

From c51fb959e2985faf6f21fe7f03d2fb8160de064f Mon Sep 17 00:00:00 2001
From: Roy <roy.healy87+gh@gmail.com>
Date: Mon, 28 Oct 2019 12:36:54 +0000
Subject: [PATCH 2296/2974] [test_downloadermiddleware_httpcache] Fixing pytest
 skip behaviour https://github.com/scrapy/scrapy/issues/4085

---
 tests/test_downloadermiddleware_httpcache.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 32085b095..047526569 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -6,6 +6,7 @@ import unittest
 import email.utils
 from contextlib import contextmanager
 import pytest
+import sys
 
 from scrapy.http import Response, HtmlResponse, Request
 from scrapy.spiders import Spider
@@ -160,7 +161,7 @@ class LeveldbStorageTest(DefaultStorageTest):
         pytest.importorskip('leveldb')
     except SystemError:
         # Happens in python 3.8
-        pytest.skip("'SystemError: bad call flags' occurs on Python 3.8")
+        pytestmark = pytest.skip("'SystemError: bad call flags' occurs on Python 3.8")
     storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage'
 
 

From 74909030a55b59e3b858fc736b5b1f685d9596a6 Mon Sep 17 00:00:00 2001
From: Roy <roy.healy87+gh@gmail.com>
Date: Mon, 28 Oct 2019 12:52:36 +0000
Subject: [PATCH 2297/2974] [tox.ini] Removing obsolete py37 extra deps
 enviornment https://github.com/scrapy/scrapy/issues/4085

---
 tox.ini | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/tox.ini b/tox.ini
index 14afec23f..fe925951b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -125,13 +125,6 @@ deps = {[docs]deps}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
 
-[testenv:py37-extra-deps]
-basepython = python3.7
-deps =
-    {[testenv:py35]deps}
-    reppy
-    robotexclusionrulesparser
-
 [testenv:py38-extra-deps]
 basepython = python3.8
 deps =

From b73d217de5647a68c7b8dfda747cd3d0685c226d Mon Sep 17 00:00:00 2001
From: Roy <roy.healy87+gh@gmail.com>
Date: Mon, 28 Oct 2019 12:55:54 +0000
Subject: [PATCH 2298/2974] [test_downloadermiddleware_httpcache.py] Fixing
 pytest mark behaviour https://github.com/scrapy/scrapy/issues/4085

---
 tests/test_downloadermiddleware_httpcache.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 047526569..f5917d0f0 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -161,7 +161,7 @@ class LeveldbStorageTest(DefaultStorageTest):
         pytest.importorskip('leveldb')
     except SystemError:
         # Happens in python 3.8
-        pytestmark = pytest.skip("'SystemError: bad call flags' occurs on Python 3.8")
+        pytestmark = pytest.mark.skip("'SystemError: bad call flags' occurs on Python 3.8")
     storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage'
 
 

From 93e3dc1b826e44d1a5a24fbb39c090ce426aa862 Mon Sep 17 00:00:00 2001
From: Roy <roy.healy87+gh@gmail.com>
Date: Mon, 28 Oct 2019 16:12:03 +0000
Subject: [PATCH 2299/2974] [test_downloadermiddleware_httpcache.py] Cleaning
 text https://github.com/scrapy/scrapy/issues/4085

---
 tests/test_downloadermiddleware_httpcache.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index f5917d0f0..972d400a4 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -155,13 +155,13 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
         new_settings.setdefault('HTTPCACHE_GZIP', True)
         return super(FilesystemStorageTest, self)._get_settings(**new_settings)
 
+
 class LeveldbStorageTest(DefaultStorageTest):
 
     try:
         pytest.importorskip('leveldb')
     except SystemError:
-        # Happens in python 3.8
-        pytestmark = pytest.mark.skip("'SystemError: bad call flags' occurs on Python 3.8")
+        pytestmark = pytest.mark.skip("Test module skipped - 'SystemError: bad call flags' occurs when >= Python 3.8")
     storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage'
 
 

From 94f060fcc84853f28f3f91b6dde1d61c8e19251e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 29 Oct 2019 12:53:46 +0100
Subject: [PATCH 2300/2974] Cover Scrapy 1.8.0 in the release notes (#3952)

---
 docs/news.rst           | 226 +++++++++++++++++++++++++++++++++++++++-
 docs/topics/logging.rst |   5 +-
 scrapy/logformatter.py  |   2 +-
 3 files changed, 229 insertions(+), 4 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 8dfe8693c..669844045 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -6,6 +6,209 @@ Release notes
 .. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0,
           planned for Q4 2019 or Q1 2020, will support **Python 3 only**.
 
+.. _release-1.8.0:
+
+Scrapy 1.8.0 (2019-10-28)
+-------------------------
+
+Highlights:
+
+* Dropped Python 3.4 support and updated minimum requirements; made Python 3.8
+  support official
+* New :meth:`Request.from_curl <scrapy.http.Request.from_curl>` class method
+* New :setting:`ROBOTSTXT_PARSER` and :setting:`ROBOTSTXT_USER_AGENT` settings
+* New :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` and
+  :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` settings
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+*   Python 3.4 is no longer supported, and some of the minimum requirements of
+    Scrapy have also changed:
+
+    *   cssselect_ 0.9.1
+    *   cryptography_ 2.0
+    *   lxml_ 3.5.0
+    *   pyOpenSSL_ 16.2.0
+    *   queuelib_ 1.4.2
+    *   service_identity_ 16.0.0
+    *   six_ 1.10.0
+    *   Twisted_ 17.9.0 (16.0.0 with Python 2)
+    *   zope.interface_ 4.1.3
+
+    (:issue:`3892`)
+
+*   ``JSONRequest`` is now called :class:`~scrapy.http.JsonRequest` for
+    consistency with similar classes (:issue:`3929`, :issue:`3982`)
+
+*   If you are using a custom context factory
+    (:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`), its ``__init__`` method must
+    accept two new parameters: ``tls_verbose_logging`` and ``tls_ciphers``
+    (:issue:`2111`, :issue:`3392`, :issue:`3442`, :issue:`3450`)
+
+*   :class:`~scrapy.loader.ItemLoader` now turns the values of its input item
+    into lists::
+
+        >>> item = MyItem()
+        >>> item['field'] = 'value1'
+        >>> loader = ItemLoader(item=item)
+        >>> item['field']
+        ['value1']
+
+    This is needed to allow adding values to existing fields
+    (``loader.add_value('field', 'value2')``).
+
+    (:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`, :issue:`3998`,
+    :issue:`4036`)
+
+See also :ref:`1.8-deprecation-removals` below.
+
+
+New features
+~~~~~~~~~~~~
+
+*   A new :meth:`Request.from_curl <scrapy.http.Request.from_curl>` class
+    method allows :ref:`creating a request from a cURL command
+    <requests-from-curl>` (:issue:`2985`, :issue:`3862`)
+
+*   A new :setting:`ROBOTSTXT_PARSER` setting allows choosing which robots.txt_
+    parser to use. It includes built-in support for
+    :ref:`RobotFileParser <python-robotfileparser>`,
+    :ref:`Protego <protego-parser>` (default), :ref:`Reppy <reppy-parser>`, and
+    :ref:`Robotexclusionrulesparser <rerp-parser>`, and allows you to
+    :ref:`implement support for additional parsers
+    <support-for-new-robots-parser>` (:issue:`754`, :issue:`2669`,
+    :issue:`3796`, :issue:`3935`, :issue:`3969`, :issue:`4006`)
+
+*   A new :setting:`ROBOTSTXT_USER_AGENT` setting allows defining a separate
+    user agent string to use for robots.txt_ parsing (:issue:`3931`,
+    :issue:`3966`)
+
+*   :class:`~scrapy.spiders.Rule` no longer requires a :class:`LinkExtractor
+    <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` parameter
+    (:issue:`781`, :issue:`4016`)
+
+*   Use the new :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting to customize
+    the TLS/SSL ciphers used by the default HTTP/1.1 downloader (:issue:`3392`,
+    :issue:`3442`)
+
+*   Set the new :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` setting to
+    ``True`` to enable debug-level messages about TLS connection parameters
+    after establishing HTTPS connections (:issue:`2111`, :issue:`3450`)
+
+*   Callbacks that receive keyword arguments
+    (see :attr:`Request.cb_kwargs <scrapy.http.Request.cb_kwargs>`) can now be
+    tested using the new :class:`@cb_kwargs
+    <scrapy.contracts.default.CallbackKeywordArgumentsContract>`
+    :ref:`spider contract <topics-contracts>` (:issue:`3985`, :issue:`3988`)
+
+*   When a :class:`@scrapes <scrapy.contracts.default.ScrapesContract>` spider
+    contract fails, all missing fields are now reported (:issue:`766`,
+    :issue:`3939`)
+
+*   :ref:`Custom log formats <custom-log-formats>` can now drop messages by
+    having the corresponding methods of the configured :setting:`LOG_FORMATTER`
+    return ``None`` (:issue:`3984`, :issue:`3987`)
+
+*   A much improved completion definition is now available for Zsh_
+    (:issue:`4069`)
+
+
+Bug fixes
+~~~~~~~~~
+
+*   :meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` no
+    longer makes later calls to :meth:`ItemLoader.get_output_value()
+    <scrapy.loader.ItemLoader.get_output_value>` or
+    :meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` return
+    empty data (:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`,
+    :issue:`3998`, :issue:`4036`)
+
+*   Fixed :class:`~scrapy.statscollectors.DummyStatsCollector` raising a
+    :exc:`TypeError` exception (:issue:`4007`, :issue:`4052`)
+
+*   :meth:`FilesPipeline.file_path
+    <scrapy.pipelines.files.FilesPipeline.file_path>` and
+    :meth:`ImagesPipeline.file_path
+    <scrapy.pipelines.images.ImagesPipeline.file_path>` no longer choose
+    file extensions that are not `registered with IANA`_ (:issue:`1287`,
+    :issue:`3953`, :issue:`3954`)
+
+*   When using botocore_ to persist files in S3, all botocore-supported headers
+    are properly mapped now (:issue:`3904`, :issue:`3905`)
+
+*   FTP passwords in :setting:`FEED_URI` containing percent-escaped characters
+    are now properly decoded (:issue:`3941`)
+
+*   A memory-handling and error-handling issue in
+    :func:`scrapy.utils.ssl.get_temp_key_info` has been fixed (:issue:`3920`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+*   The documentation now covers how to define and configure a :ref:`custom log
+    format <custom-log-formats>` (:issue:`3616`, :issue:`3660`)
+
+*   API documentation added for :class:`~scrapy.exporters.MarshalItemExporter`
+    and :class:`~scrapy.exporters.PythonItemExporter` (:issue:`3973`)
+
+*   API documentation added for :class:`~scrapy.item.BaseItem` and
+    :class:`~scrapy.item.ItemMeta` (:issue:`3999`)
+
+*   Minor documentation fixes (:issue:`2998`, :issue:`3398`, :issue:`3597`,
+    :issue:`3894`, :issue:`3934`, :issue:`3978`, :issue:`3993`, :issue:`4022`,
+    :issue:`4028`, :issue:`4033`, :issue:`4046`, :issue:`4050`, :issue:`4055`,
+    :issue:`4056`, :issue:`4061`, :issue:`4072`, :issue:`4071`, :issue:`4079`,
+    :issue:`4081`, :issue:`4089`, :issue:`4093`)
+
+
+.. _1.8-deprecation-removals:
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+*   ``scrapy.xlib`` has been removed (:issue:`4015`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+*   The LevelDB_ storage backend
+    (``scrapy.extensions.httpcache.LeveldbCacheStorage``) of
+    :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` is
+    deprecated (:issue:`4085`, :issue:`4092`)
+
+*   Use of the undocumented ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment
+    variable is deprecated (:issue:`3910`)
+
+*   ``scrapy.item.DictItem`` is deprecated, use :class:`~scrapy.item.Item`
+    instead (:issue:`3999`)
+
+
+Other changes
+~~~~~~~~~~~~~
+
+*   Minimum versions of optional Scrapy requirements that are covered by
+    continuous integration tests have been updated:
+
+    *   botocore_ 1.3.23
+    *   Pillow_ 3.4.2
+
+    Lower versions of these optional requirements may work, but it is not
+    guaranteed (:issue:`3892`)
+
+*   GitHub templates for bug reports and feature requests (:issue:`3126`,
+    :issue:`3471`, :issue:`3749`, :issue:`3754`)
+
+*   Continuous integration fixes (:issue:`3923`)
+
+*   Code cleanup (:issue:`3391`, :issue:`3907`, :issue:`3946`, :issue:`3950`,
+    :issue:`4023`, :issue:`4031`)
+
+
+.. _release-1.7.4:
+
 Scrapy 1.7.4 (2019-10-21)
 -------------------------
 
@@ -18,22 +221,31 @@ makes later calls to :meth:`ItemLoader.get_output_value()
 <scrapy.loader.ItemLoader.get_output_value>` or :meth:`ItemLoader.load_item()
 <scrapy.loader.ItemLoader.load_item>` return empty data.
 
+
+.. _release-1.7.3:
+
 Scrapy 1.7.3 (2019-08-01)
 -------------------------
 
 Enforce lxml 4.3.5 or lower for Python 3.4 (:issue:`3912`, :issue:`3918`).
 
+
+.. _release-1.7.2:
+
 Scrapy 1.7.2 (2019-07-23)
 -------------------------
 
 Fix Python 2 support (:issue:`3889`, :issue:`3893`, :issue:`3896`).
 
 
+.. _release-1.7.1:
+
 Scrapy 1.7.1 (2019-07-18)
 -------------------------
 
 Re-packaging of Scrapy 1.7.0, which was missing some changes in PyPI.
 
+
 .. _release-1.7.0:
 
 Scrapy 1.7.0 (2019-07-18)
@@ -568,7 +780,7 @@ Scrapy 1.5.2 (2019-01-22)
 
   See :ref:`telnet console <topics-telnetconsole>` documentation for more info
 
-* Backport CI build failure under GCE environemnt due to boto import error.
+* Backport CI build failure under GCE environment due to boto import error.
 
 .. _release-1.5.1:
 
@@ -2830,23 +3042,35 @@ First release of Scrapy.
 
 
 .. _AJAX crawleable urls: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started?csw=1
+.. _botocore: https://github.com/boto/botocore
 .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
 .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
 .. _Creating a pull request: https://help.github.com/en/articles/creating-a-pull-request
+.. _cryptography: https://cryptography.io/en/latest/
 .. _cssselect: https://github.com/scrapy/cssselect/
 .. _docstrings: https://docs.python.org/glossary.html#term-docstring
 .. _KeyboardInterrupt: https://docs.python.org/library/exceptions.html#KeyboardInterrupt
+.. _LevelDB: https://github.com/google/leveldb
 .. _lxml: http://lxml.de/
 .. _marshal: https://docs.python.org/2/library/marshal.html
 .. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator
 .. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator
 .. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr
 .. _PEP 257: https://www.python.org/dev/peps/pep-0257/
+.. _Pillow: https://python-pillow.org/
+.. _pyOpenSSL: https://www.pyopenssl.org/en/stable/
 .. _queuelib: https://github.com/scrapy/queuelib
+.. _registered with IANA: https://www.iana.org/assignments/media-types/media-types.xhtml
 .. _resource: https://docs.python.org/2/library/resource.html
+.. _robots.txt: http://www.robotstxt.org/
 .. _scrapely: https://github.com/scrapy/scrapely
+.. _service_identity: https://service-identity.readthedocs.io/en/stable/
+.. _six: https://six.readthedocs.io/
 .. _tox: https://pypi.python.org/pypi/tox
+.. _Twisted: https://twistedmatrix.com/trac/
 .. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
 .. _w3lib: https://github.com/scrapy/w3lib
 .. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
 .. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
+.. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/
+.. _Zsh: https://www.zsh.org/
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 87ea43c7d..2db0ffddd 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -198,8 +198,9 @@ to override some of the Scrapy settings regarding logging.
 Custom Log Formats
 ------------------
 
-A custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class
-and making :setting:`LOG_FORMATTER` point to your new class.
+A custom log format can be set for different actions by extending
+:class:`~scrapy.logformatter.LogFormatter` class and making
+:setting:`LOG_FORMATTER` point to your new class.
  
 .. autoclass:: scrapy.logformatter.LogFormatter
    :members:
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index f15940ed1..3c61ed7e0 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -29,7 +29,7 @@ class LogFormatter(object):
     *   ``args`` should be a tuple or dict with the formatting placeholders for ``msg``.
         The final log message is computed as ``msg % args``.
 
-    Users can define their own ``LogFormatter`` class if they want to customise how
+    Users can define their own ``LogFormatter`` class if they want to customize how
     each action is logged or if they want to omit it entirely. In order to omit
     logging an action the method must return ``None``.
 

From be2e910dd06ba4904e7b10eb5a7e3251e8dab099 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 29 Oct 2019 12:57:02 +0100
Subject: [PATCH 2301/2974] =?UTF-8?q?Bump=20version:=201.7.0=20=E2=86=92?=
 =?UTF-8?q?=201.8.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 70affe63f..c9f1abea5 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.7.0
+current_version = 1.8.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index bd8bf882d..27f9cd322 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.7.0
+1.8.0

From 66cbceeb0a9104fc0fa238898e38d0d9ce9cbcf6 Mon Sep 17 00:00:00 2001
From: Amardeep Bhowmick <amardeep.bhowmick92@gmail.com>
Date: Wed, 30 Oct 2019 13:39:12 +0530
Subject: [PATCH 2302/2974] Fix redirection error when the Location header
 value starts with 3 slashes (#4042)

---
 scrapy/downloadermiddlewares/redirect.py    |  7 +++++--
 tests/test_downloadermiddleware_redirect.py | 16 ++++++++++++++++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 49468a2e4..b73f864dd 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -1,5 +1,5 @@
 import logging
-from six.moves.urllib.parse import urljoin
+from six.moves.urllib.parse import urljoin, urlparse
 
 from w3lib.url import safe_url_string
 
@@ -70,7 +70,10 @@ class RedirectMiddleware(BaseRedirectMiddleware):
         if 'Location' not in response.headers or response.status not in allowed_status:
             return response
 
-        location = safe_url_string(response.headers['location'])
+        location = safe_url_string(response.headers['Location'])
+        if response.headers['Location'].startswith(b'//'):
+            request_scheme = urlparse(request.url).scheme
+            location = request_scheme + '://' + location.lstrip('/')
 
         redirected_url = urljoin(request.url, location)
 
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 0e841489d..e7faf14a7 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -106,6 +106,22 @@ class RedirectMiddlewareTest(unittest.TestCase):
         del rsp.headers['Location']
         assert self.mw.process_response(req, rsp, self.spider) is rsp
 
+    def test_redirect_302_relative(self):
+        url = 'http://www.example.com/302'
+        url2 = '///i8n.example2.com/302'
+        url3 = 'http://i8n.example2.com/302'
+        req = Request(url, method='HEAD')
+        rsp = Response(url, headers={'Location': url2}, status=302)
+
+        req2 = self.mw.process_response(req, rsp, self.spider)
+        assert isinstance(req2, Request)
+        self.assertEqual(req2.url, url3)
+        self.assertEqual(req2.method, 'HEAD')
+
+        # response without Location header but with status code is 3XX should be ignored
+        del rsp.headers['Location']
+        assert self.mw.process_response(req, rsp, self.spider) is rsp
+
 
     def test_max_redirect_times(self):
         self.mw.max_redirect_times = 1

From 6d6da78eda3cc0bba1bfdf70194fdf655fac8aeb Mon Sep 17 00:00:00 2001
From: Benjamin Ooghe-Tabanou <benjamin.ooghe@sciences-po.fr>
Date: Wed, 30 Oct 2019 09:13:36 +0100
Subject: [PATCH 2303/2974] Add a keep_fragments parameter to the
 request_fingerprint function (#4104)

---
 scrapy/utils/request.py     | 16 +++++++++++-----
 tests/test_utils_request.py |  9 ++++++++-
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 9c143b83a..fb5af66a2 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -16,7 +16,7 @@ from scrapy.utils.httpobj import urlparse_cached
 
 
 _fingerprint_cache = weakref.WeakKeyDictionary()
-def request_fingerprint(request, include_headers=None):
+def request_fingerprint(request, include_headers=None, keep_fragments=False):
     """
     Return the request fingerprint.
 
@@ -42,15 +42,21 @@ def request_fingerprint(request, include_headers=None):
     the fingeprint. If you want to include specific headers use the
     include_headers argument, which is a list of Request headers to include.
 
+    Also, servers usually ignore fragments in urls when handling requests,
+    so they are also ignored by default when calculating the fingerprint.
+    If you want to include them, set the keep_fragments argument to True
+    (for instance when handling requests with a headless browser).
+
     """
     if include_headers:
         include_headers = tuple(to_bytes(h.lower())
                                  for h in sorted(include_headers))
     cache = _fingerprint_cache.setdefault(request, {})
-    if include_headers not in cache:
+    cache_key = (include_headers, keep_fragments)
+    if cache_key not in cache:
         fp = hashlib.sha1()
         fp.update(to_bytes(request.method))
-        fp.update(to_bytes(canonicalize_url(request.url)))
+        fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
         fp.update(request.body or b'')
         if include_headers:
             for hdr in include_headers:
@@ -58,8 +64,8 @@ def request_fingerprint(request, include_headers=None):
                     fp.update(hdr)
                     for v in request.headers.getlist(hdr):
                         fp.update(v)
-        cache[include_headers] = fp.hexdigest()
-    return cache[include_headers]
+        cache[cache_key] = fp.hexdigest()
+    return cache[cache_key]
 
 
 def request_authenticate(request, username, password):
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index e8a4eb3ea..625a32048 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -17,7 +17,7 @@ class UtilsRequestTest(unittest.TestCase):
         self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
 
         # make sure caching is working
-        self.assertEqual(request_fingerprint(r1), _fingerprint_cache[r1][None])
+        self.assertEqual(request_fingerprint(r1), _fingerprint_cache[r1][(None, False)])
 
         r1 = Request("http://www.example.com/members/offers.html")
         r2 = Request("http://www.example.com/members/offers.html")
@@ -42,6 +42,13 @@ class UtilsRequestTest(unittest.TestCase):
         self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']),
                          request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language']))
 
+        r1 = Request("http://www.example.com/test.html")
+        r2 = Request("http://www.example.com/test.html#fragment")
+        self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
+        self.assertEqual(request_fingerprint(r1), request_fingerprint(r1, keep_fragments=True))
+        self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r2, keep_fragments=True))
+        self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2, keep_fragments=True))
+
         r1 = Request("http://www.example.com")
         r2 = Request("http://www.example.com", method='POST')
         r3 = Request("http://www.example.com", method='POST', body=b'request body')

From 229e722a03aced0fb62ec8c631b036c3e13e2188 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 14:46:02 +0500
Subject: [PATCH 2304/2974] Initial Python 2 removal (#4091)

---
 .travis.yml                      | 10 +-----
 README.rst                       |  2 +-
 docs/faq.rst                     |  4 +--
 docs/intro/install.rst           | 16 +++-------
 docs/topics/feed-exports.rst     |  7 ++---
 docs/topics/leaks.rst            |  2 +-
 docs/topics/media-pipeline.rst   |  2 +-
 docs/topics/request-response.rst |  4 +--
 docs/topics/settings.rst         |  5 +--
 docs/topics/spiders.rst          |  2 --
 requirements-py2.txt             | 18 -----------
 scrapy/__init__.py               |  4 +--
 setup.py                         |  7 ++---
 tests/requirements-py2.txt       | 15 ---------
 tox.ini                          | 53 ++------------------------------
 15 files changed, 24 insertions(+), 127 deletions(-)
 delete mode 100644 requirements-py2.txt
 delete mode 100644 tests/requirements-py2.txt

diff --git a/.travis.yml b/.travis.yml
index 4c2498053..2352ef124 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,14 +7,6 @@ branches:
     - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
 matrix:
   include:
-    - env: TOXENV=py27
-      python: 2.7
-    - env: TOXENV=py27-pinned
-      python: 2.7
-    - env: TOXENV=py27-extra-deps
-      python: 2.7
-    - env: TOXENV=pypy
-      python: 2.7
     - env: TOXENV=pypy3
       python: 3.5
     - env: TOXENV=py35
@@ -70,4 +62,4 @@ deploy:
   on:
     tags: true
     repo: scrapy/scrapy
-    condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
+    condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
diff --git a/README.rst b/README.rst
index bd82bff06..fb4ca8e4f 100644
--- a/README.rst
+++ b/README.rst
@@ -40,7 +40,7 @@ https://scrapy.org
 Requirements
 ============
 
-* Python 2.7 or Python 3.5+
+* Python 3.5+
 * Works on Linux, Windows, Mac OSX, BSD
 
 Install
diff --git a/docs/faq.rst b/docs/faq.rst
index 9733471bf..080d81981 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -69,11 +69,11 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
 What Python versions does Scrapy support?
 -----------------------------------------
 
-Scrapy is supported under Python 2.7 and Python 3.5+
+Scrapy is supported under Python 3.5+
 under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
-Python 2.6 support was dropped starting at Scrapy 0.20.
 Python 3 support was added in Scrapy 1.1.
 PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
+Python 2 support was dropped in Scrapy 2.0.
 
 .. note::
     For Python 3 support on Windows, it is recommended to use
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 51b41b4d7..e924b5303 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -7,7 +7,7 @@ Installation guide
 Installing Scrapy
 =================
 
-Scrapy runs on Python 2.7 and Python 3.5 or above
+Scrapy runs on Python 3.5 or above
 under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
 
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
@@ -102,10 +102,8 @@ just like any other Python package.
 (See :ref:`platform-specific guides <intro-install-platform-notes>`
 below for non-Python dependencies that you may need to install beforehand).
 
-Python virtualenvs can be created to use Python 2 by default, or Python 3 by default.
-
-* If you want to install scrapy with Python 3, install scrapy within a Python 3 virtualenv.
-* And if you want to install scrapy with Python 2, install scrapy within a Python 2 virtualenv.
+Python virtualenvs can be created to use Python 2 by default, or Python 3 by default. As Scrapy
+only supports Python 3, make sure you created a Python 3 virtualenv.
 
 .. _virtualenv: https://virtualenv.pypa.io
 .. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/
@@ -149,16 +147,12 @@ typically too old and slow to catch up with latest Scrapy.
 To install scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
 these dependencies::
 
-    sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
+    sudo apt-get install python3 python3-dev python3-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
 
-- ``python-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
+- ``python3-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
   are required for ``lxml``
 - ``libssl-dev`` and ``libffi-dev`` are required for ``cryptography``
 
-If you want to install scrapy on Python 3, you’ll also need Python 3 development headers::
-
-    sudo apt-get install python3 python3-dev
-
 Inside a :ref:`virtualenv <intro-using-virtualenv>`,
 you can install Scrapy with ``pip`` after that::
 
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index af541db78..7481b1a99 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -99,12 +99,12 @@ The storages backends supported out of the box are:
 
  * :ref:`topics-feed-storage-fs`
  * :ref:`topics-feed-storage-ftp`
- * :ref:`topics-feed-storage-s3` (requires botocore_ or boto_)
+ * :ref:`topics-feed-storage-s3` (requires botocore_)
  * :ref:`topics-feed-storage-stdout`
 
 Some storage backends may be unavailable if the required external libraries are
 not available. For example, the S3 backend is only available if the botocore_
-or boto_ library is installed (Scrapy supports boto_ only on Python 2).
+library is installed.
 
 
 .. _topics-feed-uri-params:
@@ -182,7 +182,7 @@ The feeds are stored on `Amazon S3`_.
    * ``s3://mybucket/path/to/export.csv``
    * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
 
- * Required external libraries: `botocore`_ (Python 2 and Python 3) or `boto`_ (Python 2 only)
+ * Required external libraries: `botocore`_
 
 The AWS credentials can be passed as user/password in the URI, or they can be
 passed through the following settings:
@@ -399,6 +399,5 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 
 .. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
 .. _Amazon S3: https://aws.amazon.com/s3/
-.. _boto: https://github.com/boto/boto
 .. _botocore: https://github.com/boto/botocore
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 8278e9849..793636f59 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -260,7 +260,7 @@ knowledge about Python internals. For more info about Guppy, refer to the
 
 Debugging memory leaks with muppy
 =================================
-If you're using Python 3, you can use muppy from `Pympler`_.
+You can use muppy from `Pympler`_.
 
 .. _Pympler: https://pypi.org/project/Pympler/
 
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 0ce431ff5..431cc6027 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -171,7 +171,7 @@ policy::
 
 For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
 
-Because Scrapy uses ``boto`` / ``botocore`` internally you can also use other S3-like storages. Storages like
+Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like
 self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
 
     AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 727c67482..5a76e189e 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -596,8 +596,8 @@ Response objects
        (for single valued headers) or lists (for multi-valued headers).
     :type headers: dict
 
-    :param body: the response body. To access the decoded text as str (unicode
-       in Python 2) you can use ``response.text`` from an encoding-aware
+    :param body: the response body. To access the decoded text as str you can use
+       ``response.text`` from an encoding-aware
        :ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
        such as :class:`TextResponse`.
     :type body: bytes
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 56375664f..a1d15a760 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -188,7 +188,6 @@ AWS_ENDPOINT_URL
 Default: ``None``
 
 Endpoint URL used for S3-like storage, for example Minio or s3.scality.
-Only supported with ``botocore`` library.
 
 .. setting:: AWS_USE_SSL
 
@@ -199,7 +198,6 @@ Default: ``None``
 
 Use this option if you want to disable SSL connection for communication with
 S3 or S3-like storage. By default SSL will be used.
-Only supported with ``botocore`` library.
 
 .. setting:: AWS_VERIFY
 
@@ -209,7 +207,7 @@ AWS_VERIFY
 Default: ``None``
 
 Verify SSL connection between Scrapy and S3 or S3-like storage. By default
-SSL verification will occur. Only supported with ``botocore`` library.
+SSL verification will occur.
 
 .. setting:: AWS_REGION_NAME
 
@@ -219,7 +217,6 @@ AWS_REGION_NAME
 Default: ``None``
 
 The name of the region associated with the AWS client.
-Only supported with ``botocore`` library.
 
 .. setting:: BOT_NAME
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index d60c93be6..d65a43afd 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -72,8 +72,6 @@ scrapy.Spider
        spider that crawls ``mywebsite.com`` would often be called
        ``mywebsite``.
 
-       .. note:: In Python 2 this must be ASCII only.
-
    .. attribute:: allowed_domains
 
        An optional list of strings containing domains that this spider is
diff --git a/requirements-py2.txt b/requirements-py2.txt
deleted file mode 100644
index dde8d1c9c..000000000
--- a/requirements-py2.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-parsel>=1.5.0
-PyDispatcher>=2.0.5
-w3lib>=1.17.0
-protego>=0.1.15
-
-pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
-queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
-cryptography>=2.0  # Earlier versions would fail to install
-
-# Reference versions taken from
-# https://packages.ubuntu.com/xenial/python/
-# https://packages.ubuntu.com/xenial/zope/
-cssselect>=0.9.1
-lxml>=3.5.0
-service_identity>=16.0.0
-six>=1.10.0
-Twisted>=16.0.0
-zope.interface>=4.1.3
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 03ec6c667..230e5cee3 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -14,8 +14,8 @@ del pkgutil
 
 # Check minimum required Python version
 import sys
-if sys.version_info < (2, 7):
-    print("Scrapy %s requires Python 2.7" % __version__)
+if sys.version_info < (3, 5):
+    print("Scrapy %s requires Python 3.5" % __version__)
     sys.exit(1)
 
 # Ignore noisy twisted deprecation warnings
diff --git a/setup.py b/setup.py
index 2f5fca4c9..8f5f14f0d 100644
--- a/setup.py
+++ b/setup.py
@@ -50,8 +50,6 @@ setup(
         'License :: OSI Approved :: BSD License',
         'Operating System :: OS Independent',
         'Programming Language :: Python',
-        'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
@@ -63,10 +61,9 @@ setup(
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
+    python_requires='>=3.5',
     install_requires=[
-        'Twisted>=16.0.0;python_version=="2.7"',
-        'Twisted>=17.9.0;python_version>="3.5"',
+        'Twisted>=17.9.0',
         'cryptography>=2.0',
         'cssselect>=0.9.1',
         'lxml>=3.5.0',
diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txt
deleted file mode 100644
index f621eb4eb..000000000
--- a/tests/requirements-py2.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-# Tests requirements
-brotlipy
-jmespath
-mitmproxy==0.10.1
-mock
-netlib==0.10.1
-pytest
-pytest-cov
-pytest-twisted
-pytest-xdist
-testfixtures
-
-# optional for shell wrapper tests
-bpython
-ipython<6.0
diff --git a/tox.ini b/tox.ini
index fe925951b..821144381 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,18 +4,16 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py27
+envlist = py35
 
 [testenv]
 deps =
     -ctests/constraints.txt
-    -rrequirements-py2.txt
+    -rrequirements-py3.txt
+    -rtests/requirements-py3.txt
     # Extras
     botocore>=1.3.23
-    google-cloud-storage
-    leveldb
     Pillow>=3.4.2
-    -rtests/requirements-py2.txt
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
@@ -25,42 +23,8 @@ passenv =
 commands =
     py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
 
-[testenv:py27-pinned]
-basepython = python2.7
-deps =
-    -ctests/constraints.txt
-    cryptography==2.0
-    cssselect==0.9.1
-    lxml==3.5.0
-    parsel==1.5.0
-    Protego==0.1.15
-    PyDispatcher==2.0.5
-    pyOpenSSL==16.2.0
-    queuelib==1.4.2
-    service_identity==16.0.0
-    six==1.10.0
-    Twisted==16.0.0
-    w3lib==1.17.0
-    zope.interface==4.1.3
-    -rtests/requirements-py2.txt
-    # Extras
-    botocore==1.3.23
-    Pillow==3.4.2
-
-[testenv:pypy]
-basepython = pypy
-commands =
-    py.test {posargs:scrapy tests}
-
 [testenv:py35]
 basepython = python3.5
-deps =
-    -ctests/constraints.txt
-    -rrequirements-py3.txt
-    -rtests/requirements-py3.txt
-    # Extras
-    botocore>=1.3.23
-    Pillow>=3.4.2
 
 [testenv:py35-pinned]
 basepython = python3.5
@@ -86,19 +50,15 @@ deps =
 
 [testenv:py36]
 basepython = python3.6
-deps = {[testenv:py35]deps}
 
 [testenv:py37]
 basepython = python3.7
-deps = {[testenv:py35]deps}
 
 [testenv:py38]
 basepython = python3.8
-deps = {[testenv:py35]deps}
 
 [testenv:pypy3]
 basepython = pypy3
-deps = {[testenv:py35]deps}
 commands =
     py.test {posargs:scrapy tests}
 
@@ -127,13 +87,6 @@ commands =
 
 [testenv:py38-extra-deps]
 basepython = python3.8
-deps =
-    {[testenv:py35]deps}
-    reppy
-    robotexclusionrulesparser
-
-[testenv:py27-extra-deps]
-basepython = python2.7
 deps =
     {[testenv]deps}
     reppy

From 15c55d0c1d4a4384fd9725d47c305f52918e3831 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 31 Oct 2019 10:47:29 +0100
Subject: [PATCH 2305/2974] Remove LevelDB support (#4112)

---
 scrapy/extensions/httpcache.py               | 71 --------------------
 tests/requirements-py3.txt                   |  1 -
 tests/test_downloadermiddleware_httpcache.py |  9 ---
 3 files changed, 81 deletions(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 7c650a91e..f3fabf710 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -347,77 +347,6 @@ class FilesystemCacheStorage(object):
             return pickle.load(f)
 
 
-class LeveldbCacheStorage(object):
-
-    def __init__(self, settings):
-        warn("The LevelDB storage backend is deprecated.",
-             ScrapyDeprecationWarning, stacklevel=2)
-        import leveldb
-        self._leveldb = leveldb
-        self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
-        self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
-        self.db = None
-
-    def open_spider(self, spider):
-        dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name)
-        self.db = self._leveldb.LevelDB(dbpath)
-
-        logger.debug("Using LevelDB cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
-
-    def close_spider(self, spider):
-        # Do compactation each time to save space and also recreate files to
-        # avoid them being removed in storages with timestamp-based autoremoval.
-        self.db.CompactRange()
-        del self.db
-        garbage_collect()
-
-    def retrieve_response(self, spider, request):
-        data = self._read_data(spider, request)
-        if data is None:
-            return  # not cached
-        url = data['url']
-        status = data['status']
-        headers = Headers(data['headers'])
-        body = data['body']
-        respcls = responsetypes.from_args(headers=headers, url=url)
-        response = respcls(url=url, headers=headers, status=status, body=body)
-        return response
-
-    def store_response(self, spider, request, response):
-        key = self._request_key(request)
-        data = {
-            'status': response.status,
-            'url': response.url,
-            'headers': dict(response.headers),
-            'body': response.body,
-        }
-        batch = self._leveldb.WriteBatch()
-        batch.Put(key + b'_data', pickle.dumps(data, protocol=2))
-        batch.Put(key + b'_time', to_bytes(str(time())))
-        self.db.Write(batch)
-
-    def _read_data(self, spider, request):
-        key = self._request_key(request)
-        try:
-            ts = self.db.Get(key + b'_time')
-        except KeyError:
-            return  # not found or invalid entry
-
-        if 0 < self.expiration_secs < time() - float(ts):
-            return  # expired
-
-        try:
-            data = self.db.Get(key + b'_data')
-        except KeyError:
-            return  # invalid entry
-        else:
-            return pickle.loads(data)
-
-    def _request_key(self, request):
-        return to_bytes(request_fingerprint(request))
-
-
-
 def parse_cachecontrol(header):
     """Parse Cache-Control header
 
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index cb67bc40e..dd5b23cc3 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,6 +1,5 @@
 # Tests requirements
 jmespath
-leveldb; sys_platform != "win32"
 pytest
 pytest-cov
 pytest-twisted
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 972d400a4..950664ffe 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -156,15 +156,6 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
         return super(FilesystemStorageTest, self)._get_settings(**new_settings)
 
 
-class LeveldbStorageTest(DefaultStorageTest):
-
-    try:
-        pytest.importorskip('leveldb')
-    except SystemError:
-        pytestmark = pytest.mark.skip("Test module skipped - 'SystemError: bad call flags' occurs when >= Python 3.8")
-    storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage'
-
-
 class DummyPolicyTest(_BaseTest):
 
     policy_class = 'scrapy.extensions.httpcache.DummyPolicy'

From b44bd6f8250579dc9ddc25d5e0f10d8b3790f5ea Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 22 Jul 2019 20:51:03 +0500
Subject: [PATCH 2306/2974] Remove Python 2-only tests.

---
 conftest.py                             |  10 +-
 tests/py3-ignores.txt                   |   3 -
 tests/test_downloader_handlers.py       |  22 +--
 tests/test_linkextractors_deprecated.py | 233 ------------------------
 tests/test_proxy_connect.py             | 120 ------------
 tests/test_utils_python.py              |  27 ---
 tests/test_webclient.py                 |  20 --
 7 files changed, 14 insertions(+), 421 deletions(-)
 delete mode 100644 tests/test_linkextractors_deprecated.py
 delete mode 100644 tests/test_proxy_connect.py

diff --git a/conftest.py b/conftest.py
index 06d65ba1d..ede091e9f 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,4 +1,3 @@
-import six
 import pytest
 
 
@@ -8,11 +7,10 @@ collect_ignore = [
 ]
 
 
-if six.PY3:
-    for line in open('tests/py3-ignores.txt'):
-        file_path = line.strip()
-        if file_path and file_path[0] != '#':
-            collect_ignore.append(file_path)
+for line in open('tests/py3-ignores.txt'):
+    file_path = line.strip()
+    if file_path and file_path[0] != '#':
+        collect_ignore.append(file_path)
 
 
 @pytest.fixture()
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 313e74ec9..45cf6fb92 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,6 +1,3 @@
-tests/test_linkextractors_deprecated.py
-tests/test_proxy_connect.py
-
 scrapy/linkextractors/sgml.py
 scrapy/linkextractors/regex.py
 scrapy/linkextractors/htmlparser.py
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 109469503..4d3c4d4aa 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -633,18 +633,16 @@ class Http11MockServerTestCase(unittest.TestCase):
         # download_maxsize < 100, hence the CancelledError
         self.assertIsInstance(failure.value, defer.CancelledError)
 
-        if six.PY2:
-            request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
-            request = request.replace(url=self.mockserver.url('/xpayload'))
-            yield crawler.crawl(seed=request)
-            # download_maxsize = 50 is enough for the gzipped response
-            failure = crawler.spider.meta.get('failure')
-            self.assertTrue(failure == None)
-            reason = crawler.spider.meta['close_reason']
-            self.assertTrue(reason, 'finished')
-        else:
-            # See issue https://twistedmatrix.com/trac/ticket/8175
-            raise unittest.SkipTest("xpayload only enabled for PY2")
+        # See issue https://twistedmatrix.com/trac/ticket/8175
+        raise unittest.SkipTest("xpayload only enabled for PY2")
+        request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
+        request = request.replace(url=self.mockserver.url('/xpayload'))
+        yield crawler.crawl(seed=request)
+        # download_maxsize = 50 is enough for the gzipped response
+        failure = crawler.spider.meta.get('failure')
+        self.assertTrue(failure == None)
+        reason = crawler.spider.meta['close_reason']
+        self.assertTrue(reason, 'finished')
 
 
 class UriResource(resource.Resource):
diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py
deleted file mode 100644
index 1366971be..000000000
--- a/tests/test_linkextractors_deprecated.py
+++ /dev/null
@@ -1,233 +0,0 @@
-# -*- coding: utf-8 -*-
-import unittest
-from scrapy.linkextractors.regex import RegexLinkExtractor
-from scrapy.http import HtmlResponse
-from scrapy.link import Link
-from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor
-from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
-from tests import get_testdata
-
-from tests.test_linkextractors import Base
-
-
-class BaseSgmlLinkExtractorTestCase(unittest.TestCase):
-    # XXX: should we move some of these tests to base link extractor tests?
-
-    def test_basic(self):
-        html = """<html><head><title>Page title<title>
-        <body><p><a href="item/12.html">Item 12</a></p>
-        <p><a href="/about.html">About us</a></p>
-        <img src="/logo.png" alt="Company logo (not a link)" />
-        <p><a href="../othercat.html">Other category</a></p>
-        <p><a href="/">&gt;&gt;</a></p>
-        <p><a href="/" /></p>
-        </body></html>"""
-        response = HtmlResponse("http://example.org/somepage/index.html", body=html)
-
-        lx = BaseSgmlLinkExtractor()  # default: tag=a, attr=href
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://example.org/somepage/item/12.html', text='Item 12'),
-                          Link(url='http://example.org/about.html', text='About us'),
-                          Link(url='http://example.org/othercat.html', text='Other category'),
-                          Link(url='http://example.org/', text='>>'),
-                          Link(url='http://example.org/', text='')])
-
-    def test_base_url(self):
-        html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
-        <body><p><a href="item/12.html">Item 12</a></p>
-        </body></html>"""
-        response = HtmlResponse("http://example.org/somepage/index.html", body=html)
-
-        lx = BaseSgmlLinkExtractor()  # default: tag=a, attr=href
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
-
-        # base url is an absolute path and relative to host
-        html = """<html><head><title>Page title<title><base href="/" />
-        <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
-        response = HtmlResponse("https://example.org/somepage/index.html", body=html)
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='https://example.org/item/12.html', text='Item 12')])
-
-        # base url has no scheme
-        html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" />
-        <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
-        response = HtmlResponse("https://example.org/somepage/index.html", body=html)
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')])
-
-    def test_link_text_wrong_encoding(self):
-        html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>"""
-        response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8')
-        lx = BaseSgmlLinkExtractor()
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'),
-        ])
-
-    def test_extraction_encoding(self):
-        body = get_testdata('link_extractor', 'linkextractor_noenc.html')
-        response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})
-        response_noenc = HtmlResponse(url='http://example.com/noenc', body=body)
-        body = get_testdata('link_extractor', 'linkextractor_latin1.html')
-        response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body)
-
-        lx = BaseSgmlLinkExtractor()
-        self.assertEqual(lx.extract_links(response_utf8), [
-            Link(url='http://example.com/sample_%C3%B1.html', text=''),
-            Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
-        ])
-
-        self.assertEqual(lx.extract_links(response_noenc), [
-            Link(url='http://example.com/sample_%C3%B1.html', text=''),
-            Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
-        ])
-
-        # document encoding does not affect URL path component, only query part
-        # >>> u'sample_ñ.html'.encode('utf8')
-        # b'sample_\xc3\xb1.html'
-        # >>> u"sample_á.html".encode('utf8')
-        # b'sample_\xc3\xa1.html'
-        # >>> u"sample_ö.html".encode('utf8')
-        # b'sample_\xc3\xb6.html'
-        # >>> u"£32".encode('latin1')
-        # b'\xa332'
-        # >>> u"µ".encode('latin1')
-        # b'\xb5'
-        self.assertEqual(lx.extract_links(response_latin1), [
-            Link(url='http://example.com/sample_%C3%B1.html', text=''),
-            Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')),
-            Link(url='http://example.com/sample_%C3%B6.html?price=%A332&%B5=unit', text=''),
-        ])
-
-    def test_matches(self):
-        url1 = 'http://lotsofstuff.com/stuff1/index'
-        url2 = 'http://evenmorestuff.com/uglystuff/index'
-
-        lx = BaseSgmlLinkExtractor()
-        self.assertEqual(lx.matches(url1), True)
-        self.assertEqual(lx.matches(url2), True)
-
-
-class HtmlParserLinkExtractorTestCase(unittest.TestCase):
-
-    def setUp(self):
-        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
-        self.response = HtmlResponse(url='http://example.com/index', body=body)
-
-    def test_extraction(self):
-        # Default arguments
-        lx = HtmlParserLinkExtractor()
-        self.assertEqual(lx.extract_links(self.response), [
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
-            Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
-            Link(url='http://www.google.com/something', text=u''),
-            Link(url='http://example.com/innertag.html', text=u'inner tag'),
-            Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
-        ])
-
-    def test_link_wrong_href(self):
-        html = """
-        <a href="http://example.org/item1.html">Item 1</a>
-        <a href="http://[example.org/item2.html">Item 2</a>
-        <a href="http://example.org/item3.html">Item 3</a>
-        """
-        response = HtmlResponse("http://example.org/index.html", body=html)
-        lx = HtmlParserLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
-        ])
-
-
-class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
-    extractor_cls = SgmlLinkExtractor
-    escapes_whitespace = True
-
-    def test_deny_extensions(self):
-        html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
-        response = HtmlResponse("http://example.org/", body=html)
-        lx = SgmlLinkExtractor(deny_extensions="jpg")
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.org/page.html', text=u'asd'),
-        ])
-
-    def test_attrs_sgml(self):
-        html = """<html><area href="sample1.html"></area>
-        <a ref="sample2.html">sample text 2</a></html>"""
-        response = HtmlResponse("http://example.com/index.html", body=html)
-        lx = SgmlLinkExtractor(attrs="href")
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.com/sample1.html', text=u''),
-        ])
-
-    def test_link_nofollow(self):
-        html = """
-        <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
-        <a href="about.html">About us</a>
-        <a href="http://google.com/something" rel="external nofollow">Something</a>
-        """
-        response = HtmlResponse("http://example.org/page.html", body=html)
-        lx = SgmlLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
-            Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
-            Link(url='http://google.com/something', text=u'Something', nofollow=True),
-        ])
-
-
-class RegexLinkExtractorTestCase(unittest.TestCase):
-    # XXX: RegexLinkExtractor is not deprecated yet, but it must be rewritten
-    # not to depend on SgmlLinkExractor. Its speed is also much worse
-    # than it should be.
-
-    def setUp(self):
-        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
-        self.response = HtmlResponse(url='http://example.com/index', body=body)
-
-    def test_extraction(self):
-        # Default arguments
-        lx = RegexLinkExtractor()
-        self.assertEqual(lx.extract_links(self.response),
-                         [Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                          Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-                          Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
-                          Link(url='http://www.google.com/something', text=u''),
-                          Link(url='http://example.com/innertag.html', text=u'inner tag'),])
-
-    def test_link_wrong_href(self):
-        html = """
-        <a href="http://example.org/item1.html">Item 1</a>
-        <a href="http://[example.org/item2.html">Item 2</a>
-        <a href="http://example.org/item3.html">Item 3</a>
-        """
-        response = HtmlResponse("http://example.org/index.html", body=html)
-        lx = RegexLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
-        ])
-
-    def test_html_base_href(self):
-        html = """
-        <html>
-            <head>
-                <base href="http://b.com/">
-            </head>
-            <body>
-                <a href="test.html"></a>
-            </body>
-        </html>
-        """
-        response = HtmlResponse("http://a.com/", body=html)
-        lx = RegexLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://b.com/test.html', text=u'', nofollow=False),
-        ])
-
-    @unittest.expectedFailure
-    def test_extraction(self):
-        # RegexLinkExtractor doesn't parse URLs with leading/trailing
-        # whitespaces correctly.
-        super(RegexLinkExtractorTestCase, self).test_extraction()
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
deleted file mode 100644
index ae1236bcb..000000000
--- a/tests/test_proxy_connect.py
+++ /dev/null
@@ -1,120 +0,0 @@
-import json
-import os
-import time
-
-from six.moves.urllib.parse import urlsplit, urlunsplit
-from threading import Thread
-from libmproxy import controller, proxy
-from netlib import http_auth
-from testfixtures import LogCapture
-
-from twisted.internet import defer
-from twisted.trial.unittest import TestCase
-from scrapy.utils.test import get_crawler
-from scrapy.http import Request
-from tests.spiders import SimpleSpider, SingleRequestSpider
-from tests.mockserver import MockServer
-
-
-class HTTPSProxy(controller.Master, Thread):
-
-    def __init__(self):
-        password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy')
-        authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy")
-        cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
-            'keys', 'mitmproxy-ca.pem')
-        server = proxy.ProxyServer(proxy.ProxyConfig(
-            authenticator = authenticator,
-            cacert = cert_path),
-            0)
-        self.server = server
-        Thread.__init__(self)
-        controller.Master.__init__(self, server)
-
-    def http_address(self):
-        return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname()
-
-
-def _wrong_credentials(proxy_url):
-    bad_auth_proxy = list(urlsplit(proxy_url))
-    bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')
-    return urlunsplit(bad_auth_proxy)
-
-class ProxyConnectTestCase(TestCase):
-
-    def setUp(self):
-        self.mockserver = MockServer()
-        self.mockserver.__enter__()
-        self._oldenv = os.environ.copy()
-
-        self._proxy = HTTPSProxy()
-        self._proxy.start()
-
-        # Wait for the proxy to start.
-        time.sleep(1.0)
-        os.environ['https_proxy'] = self._proxy.http_address()
-        os.environ['http_proxy'] = self._proxy.http_address()
-
-    def tearDown(self):
-        self.mockserver.__exit__(None, None, None)
-        self._proxy.shutdown()
-        os.environ = self._oldenv
-
-    @defer.inlineCallbacks
-    def test_https_connect_tunnel(self):
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
-
-    @defer.inlineCallbacks
-    def test_https_noconnect(self):
-        proxy = os.environ['https_proxy']
-        os.environ['https_proxy'] = proxy + '?noconnect'
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
-
-    @defer.inlineCallbacks
-    def test_https_connect_tunnel_error(self):
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl("https://localhost:99999/status?n=200")
-        self._assert_got_tunnel_error(l)
-
-    @defer.inlineCallbacks
-    def test_https_tunnel_auth_error(self):
-        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        # The proxy returns a 407 error code but it does not reach the client;
-        # he just sees a TunnelError.
-        self._assert_got_tunnel_error(l)
-
-    @defer.inlineCallbacks
-    def test_https_tunnel_without_leak_proxy_authorization_header(self):
-        request = Request(self.mockserver.url("/echo", is_secure=True))
-        crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(seed=request)
-        self._assert_got_response_code(200, l)
-        echo = json.loads(crawler.spider.meta['responses'][0].body)
-        self.assertTrue('Proxy-Authorization' not in echo['headers'])
-
-    @defer.inlineCallbacks
-    def test_https_noconnect_auth_error(self):
-        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(407, l)
-
-    def _assert_got_response_code(self, code, log):
-        print(log)
-        self.assertEqual(str(log).count('Crawled (%d)' % code), 1)
-
-    def _assert_got_tunnel_error(self, log):
-        print(log)
-        self.assertIn('TunnelError', str(log))
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3e1148354..6cb32cbdd 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -163,33 +163,6 @@ class UtilsPythonTestCase(unittest.TestCase):
                 gc.collect()
         self.assertFalse(len(wk._weakdict))
 
-    @unittest.skipUnless(six.PY2, "deprecated function")
-    def test_stringify_dict(self):
-        d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'}
-        d2 = stringify_dict(d, keys_only=False)
-        self.assertEqual(d, d2)
-        self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
-
-    @unittest.skipUnless(six.PY2, "deprecated function")
-    def test_stringify_dict_tuples(self):
-        tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')]
-        d = dict(tuples)
-        d2 = stringify_dict(tuples, keys_only=False)
-        self.assertEqual(d, d2)
-        self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
-
-    @unittest.skipUnless(six.PY2, "deprecated function")
-    def test_stringify_dict_keys_only(self):
-        d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
-        d2 = stringify_dict(d)
-        self.assertEqual(d, d2)
-        self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
-
     def test_get_func_args(self):
         def f1(a, b, c):
             pass
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index a81946490..7b015ff8d 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -78,26 +78,6 @@ class ParseUrlTestCase(unittest.TestCase):
                 to_bytes(x) if not isinstance(x, int) else x for x in test)
             self.assertEqual(client._parse(url), test, url)
 
-    def test_externalUnicodeInterference(self):
-        """
-        L{client._parse} should return C{str} for the scheme, host, and path
-        elements of its return tuple, even when passed an URL which has
-        previously been passed to L{urlparse} as a C{unicode} string.
-        """
-        if not six.PY2:
-            raise unittest.SkipTest(
-                "Applies only to Py2, as urls can be ONLY unicode on Py3")
-        badInput = u'http://example.com/path'
-        goodInput = badInput.encode('ascii')
-        self._parse(badInput)  # cache badInput in urlparse_cached
-        scheme, netloc, host, port, path = self._parse(goodInput)
-        self.assertTrue(isinstance(scheme, str))
-        self.assertTrue(isinstance(netloc, str))
-        self.assertTrue(isinstance(host, str))
-        self.assertTrue(isinstance(path, str))
-        self.assertTrue(isinstance(port, int))
-
-
 
 class ScrapyHTTPPageGetterTests(unittest.TestCase):
 

From b0d6f4917d782f2396a702e90c36ffbc8bb42e7e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 3 Sep 2019 15:17:03 +0500
Subject: [PATCH 2307/2974] Restore tests/test_proxy_connect.py and update it
 to modern mitmproxy.

---
 tests/requirements-py3.txt  |   1 +
 tests/test_proxy_connect.py | 134 ++++++++++++++++++++++++++++++++++++
 2 files changed, 135 insertions(+)
 create mode 100644 tests/test_proxy_connect.py

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index dd5b23cc3..f27e45a54 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,5 +1,6 @@
 # Tests requirements
 jmespath
+mitmproxy
 pytest
 pytest-cov
 pytest-twisted
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
new file mode 100644
index 000000000..8142d9a41
--- /dev/null
+++ b/tests/test_proxy_connect.py
@@ -0,0 +1,134 @@
+import json
+import os
+import re
+from subprocess import Popen, PIPE
+import sys
+import time
+
+from six.moves.urllib.parse import urlsplit, urlunsplit
+from testfixtures import LogCapture
+
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+
+from scrapy.utils.test import get_crawler
+from scrapy.http import Request
+from tests.spiders import SimpleSpider, SingleRequestSpider
+from tests.mockserver import MockServer
+
+
+class MitmProxy:
+    auth_user = 'scrapy'
+    auth_pass = 'scrapy'
+
+    def start(self):
+        from scrapy.utils.test import get_testenv
+        script = """
+import sys
+from mitmproxy.tools.main import mitmdump
+sys.argv[0] = "mitmdump"
+sys.exit(mitmdump())
+        """
+        cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+            'keys', 'mitmproxy-ca.pem')
+        self.proc = Popen([sys.executable,
+                           '-c', script,
+                           '--listen-host', '127.0.0.1',
+                           '--listen-port', '0',
+                           '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
+                           '--certs', cert_path,
+                           '--ssl-insecure',
+                           ],
+                           stdout=PIPE, env=get_testenv())
+        line = self.proc.stdout.readline().decode('utf-8')
+        host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)
+        address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port)
+        return address
+
+    def stop(self):
+        self.proc.kill()
+        self.proc.wait()
+        time.sleep(0.2)
+
+
+def _wrong_credentials(proxy_url):
+    bad_auth_proxy = list(urlsplit(proxy_url))
+    bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')
+    return urlunsplit(bad_auth_proxy)
+
+
+class ProxyConnectTestCase(TestCase):
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self._oldenv = os.environ.copy()
+
+        self._proxy = MitmProxy()
+        proxy_url = self._proxy.start()
+        os.environ['https_proxy'] = proxy_url
+        os.environ['http_proxy'] = proxy_url
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+        self._proxy.stop()
+        os.environ = self._oldenv
+
+    @defer.inlineCallbacks
+    def test_https_connect_tunnel(self):
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        self._assert_got_response_code(200, l)
+
+    @defer.inlineCallbacks
+    def test_https_noconnect(self):
+        proxy = os.environ['https_proxy']
+        os.environ['https_proxy'] = proxy + '?noconnect'
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        self._assert_got_response_code(200, l)
+
+    @defer.inlineCallbacks
+    def test_https_connect_tunnel_error(self):
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl("https://localhost:99999/status?n=200")
+        self._assert_got_tunnel_error(l)
+
+    @defer.inlineCallbacks
+    def test_https_tunnel_auth_error(self):
+        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        # The proxy returns a 407 error code but it does not reach the client;
+        # he just sees a TunnelError.
+        self._assert_got_tunnel_error(l)
+
+    @defer.inlineCallbacks
+    def test_https_tunnel_without_leak_proxy_authorization_header(self):
+        request = Request(self.mockserver.url("/echo", is_secure=True))
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(seed=request)
+        self._assert_got_response_code(200, l)
+        echo = json.loads(crawler.spider.meta['responses'][0].body)
+        self.assertTrue('Proxy-Authorization' not in echo['headers'])
+
+    @defer.inlineCallbacks
+    def test_https_noconnect_auth_error(self):
+        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        self._assert_got_response_code(407, l)
+
+    def _assert_got_response_code(self, code, log):
+        print(log)
+        self.assertEqual(str(log).count('Crawled (%d)' % code), 1)
+
+    def _assert_got_tunnel_error(self, log):
+        print(log)
+        self.assertIn('TunnelError', str(log))

From 439e37fc7b2c192b4591d29dde74c9f589763cc8 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 3 Sep 2019 15:23:24 +0500
Subject: [PATCH 2308/2974] Mark failing proxy tests.

---
 tests/test_proxy_connect.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 8142d9a41..5e9470e39 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -5,6 +5,7 @@ from subprocess import Popen, PIPE
 import sys
 import time
 
+import pytest
 from six.moves.urllib.parse import urlsplit, urlunsplit
 from testfixtures import LogCapture
 
@@ -81,6 +82,7 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, l)
 
+    @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
     @defer.inlineCallbacks
     def test_https_noconnect(self):
         proxy = os.environ['https_proxy']
@@ -90,6 +92,7 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, l)
 
+    @pytest.mark.xfail(reason='Python 3 fails this earlier')
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
         crawler = get_crawler(SimpleSpider)
@@ -117,6 +120,7 @@ class ProxyConnectTestCase(TestCase):
         echo = json.loads(crawler.spider.meta['responses'][0].body)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
+    @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
     @defer.inlineCallbacks
     def test_https_noconnect_auth_error(self):
         os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'

From 186f9d88acd9043cf5d8f21358c758c42e433105 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 3 Sep 2019 15:24:06 +0500
Subject: [PATCH 2309/2974] Fix the skip message for
 test_download_gzip_response.

---
 tests/test_downloader_handlers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 4d3c4d4aa..9f74577a5 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -634,7 +634,7 @@ class Http11MockServerTestCase(unittest.TestCase):
         self.assertIsInstance(failure.value, defer.CancelledError)
 
         # See issue https://twistedmatrix.com/trac/ticket/8175
-        raise unittest.SkipTest("xpayload only enabled for PY2")
+        raise unittest.SkipTest("xpayload fails on PY3")
         request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
         request = request.replace(url=self.mockserver.url('/xpayload'))
         yield crawler.crawl(seed=request)

From bbd9f4be90e734df5c290aeb399e8e512f834224 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 22 Jul 2019 22:27:29 +0500
Subject: [PATCH 2310/2974] Remove six.PY2 and six.PY3 conditionals.

---
 docs/topics/downloader-middleware.rst |  6 +++---
 scrapy/_monkeypatches.py              | 10 ----------
 scrapy/commands/fetch.py              |  5 ++---
 scrapy/crawler.py                     | 11 -----------
 scrapy/exporters.py                   |  2 +-
 scrapy/extensions/feedexport.py       |  3 +--
 scrapy/http/request/form.py           |  3 +--
 scrapy/http/response/text.py          |  3 ---
 scrapy/item.py                        |  8 +-------
 scrapy/link.py                        | 15 ++-------------
 scrapy/mail.py                        |  9 ++-------
 scrapy/settings/__init__.py           |  8 +-------
 scrapy/settings/default_settings.py   |  4 +---
 scrapy/utils/boto.py                  | 10 +---------
 scrapy/utils/conf.py                  |  5 +----
 scrapy/utils/datatypes.py             | 20 +++++++-------------
 scrapy/utils/gz.py                    | 13 +++----------
 scrapy/utils/iterators.py             |  6 +-----
 scrapy/utils/python.py                | 10 ++--------
 tests/__init__.py                     | 15 ---------------
 tests/test_http_request.py            | 11 +++--------
 tests/test_http_response.py           |  3 +--
 tests/test_item.py                    |  8 ++------
 tests/test_link.py                    | 13 ++-----------
 tests/test_middleware.py              | 21 ++++++---------------
 tests/test_request_cb_kwargs.py       |  8 +-------
 tests/test_settings/__init__.py       |  8 +-------
 tests/test_utils_datatypes.py         |  7 +------
 tests/test_utils_python.py            |  7 +++----
 29 files changed, 50 insertions(+), 202 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 8048e1c86..366b95510 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -474,7 +474,7 @@ DBM storage backend
 
     A DBM_ storage backend is also available for the HTTP cache middleware.
 
-    By default, it uses the anydbm_ module, but you can change it with the
+    By default, it uses the dbm_ module, but you can change it with the
     :setting:`HTTPCACHE_DBM_MODULE` setting.
 
 .. _httpcache-storage-custom:
@@ -626,7 +626,7 @@ HTTPCACHE_DBM_MODULE
 
 .. versionadded:: 0.13
 
-Default: ``'anydbm'``
+Default: ``'dbm'``
 
 The database module to use in the :ref:`DBM storage backend
 <httpcache-storage-dbm>`. This setting is specific to the DBM backend.
@@ -1202,4 +1202,4 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
 
 
 .. _DBM: https://en.wikipedia.org/wiki/Dbm
-.. _anydbm: https://docs.python.org/2/library/anydbm.html
+.. _dbm: https://docs.python.org/3/library/dbm.html
diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index b68099cad..1f8067b35 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -1,16 +1,6 @@
-import six
 from six.moves import copyreg
 
 
-if six.PY2:
-    from urlparse import urlparse
-    
-    # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
-    if urlparse('s3://bucket/key?key=value').query != 'key=value':
-        from urlparse import uses_query
-        uses_query.append('s3')
-
-
 # Undo what Twisted's perspective broker adds to pickle register
 # to prevent bugs like Twisted#7989 while serializing requests
 import twisted.persisted.styles  # NOQA
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 7d4840529..d45133e0e 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,5 +1,5 @@
 from __future__ import print_function
-import sys, six
+import sys
 from w3lib.url import is_url
 
 from scrapy.commands import ScrapyCommand
@@ -45,8 +45,7 @@ class Command(ScrapyCommand):
             self._print_bytes(response.body)
 
     def _print_bytes(self, bytes_):
-        bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer
-        bytes_writer.write(bytes_ + b'\n')
+        sys.stdout.buffer.write(bytes_ + b'\n')
 
     def run(self, args, opts):
         if len(args) != 1 or not is_url(args[0]):
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index ded3c082b..19b998e0d 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -88,20 +88,9 @@ class Crawler(object):
             yield self.engine.open_spider(self.spider, start_requests)
             yield defer.maybeDeferred(self.engine.start)
         except Exception:
-            # In Python 2 reraising an exception after yield discards
-            # the original traceback (see https://bugs.python.org/issue7563),
-            # so sys.exc_info() workaround is used.
-            # This workaround also works in Python 3, but it is not needed,
-            # and it is slower, so in Python 3 we use native `raise`.
-            if six.PY2:
-                exc_info = sys.exc_info()
-
             self.crawling = False
             if self.engine is not None:
                 yield self.engine.close()
-
-            if six.PY2:
-                six.reraise(*exc_info)
             raise
 
     def _create_spider(self, *args, **kwargs):
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 6fc87ed18..5d1f1ad8f 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -216,7 +216,7 @@ class CsvItemExporter(BaseItemExporter):
             write_through=True,
             encoding=self.encoding,
             newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
-        ) if six.PY3 else file
+        )
         self.csv_writer = csv.writer(self.stream, **kwargs)
         self._headers_not_written = True
         self._join_multivalued = join_multivalued
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 6fb6397b1..07ffd3476 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -10,7 +10,6 @@ import logging
 import posixpath
 from tempfile import NamedTemporaryFile
 from datetime import datetime
-import six
 from six.moves.urllib.parse import urlparse, unquote
 from ftplib import FTP
 
@@ -65,7 +64,7 @@ class StdoutFeedStorage(object):
 
     def __init__(self, uri, _stdout=None):
         if not _stdout:
-            _stdout = sys.stdout if six.PY2 else sys.stdout.buffer
+            _stdout = sys.stdout.buffer
         self._stdout = _stdout
 
     def open(self, spider):
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 3ce8fc48e..b6feede07 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -104,8 +104,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
                 el = el.getparent()
                 if el is None:
                     break
-        encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape')
-        raise ValueError('No <form> element found with %s' % encoded)
+        raise ValueError('No <form> element found with %s' % formxpath)
 
     # If we get here, it means that either formname was None
     # or invalid
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 339913d4e..a8010877c 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -32,9 +32,6 @@ class TextResponse(Response):
 
     def _set_url(self, url):
         if isinstance(url, six.text_type):
-            if six.PY2 and self.encoding is None:
-                raise TypeError("Cannot convert unicode url - %s "
-                                "has no encoding" % type(self).__name__)
             self._url = to_native_str(url, self.encoding)
         else:
             super(TextResponse, self)._set_url(url)
diff --git a/scrapy/item.py b/scrapy/item.py
index 73b8f54b0..32f9b2ebb 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -4,8 +4,8 @@ Scrapy Item
 See documentation in docs/topics/item.rst
 """
 
-import collections
 from abc import ABCMeta
+from collections.abc import MutableMapping
 from copy import deepcopy
 from pprint import pformat
 from warnings import warn
@@ -16,12 +16,6 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 
 
-if six.PY2:
-    MutableMapping = collections.MutableMapping
-else:
-    MutableMapping = collections.abc.MutableMapping
-
-
 class BaseItem(object_ref):
     """Base class for all scraped items.
 
diff --git a/scrapy/link.py b/scrapy/link.py
index 2c8301680..a175b8afd 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -4,12 +4,6 @@ This module defines the Link object used in Link extractors.
 For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
 """
-import warnings
-import six
-
-from scrapy.utils.python import to_bytes
-
-
 class Link(object):
     """Link objects represent an extracted link by the LinkExtractor."""
 
@@ -17,13 +11,8 @@ class Link(object):
 
     def __init__(self, url, text='', fragment='', nofollow=False):
         if not isinstance(url, str):
-            if six.PY2:
-                warnings.warn("Link urls must be str objects. "
-                              "Assuming utf-8 encoding (which could be wrong)")
-                url = to_bytes(url, encoding='utf8')
-            else:
-                got = url.__class__.__name__
-                raise TypeError("Link urls must be str objects, got %s" % got)
+            got = url.__class__.__name__
+            raise TypeError("Link urls must be str objects, got %s" % got)
         self.url = url
         self.text = text
         self.fragment = fragment
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 5b944e1c4..746468e25 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -9,18 +9,13 @@ try:
     from cStringIO import StringIO as BytesIO
 except ImportError:
     from io import BytesIO
-import six
 
 from email.utils import COMMASPACE, formatdate
 from six.moves.email_mime_multipart import MIMEMultipart
 from six.moves.email_mime_text import MIMEText
 from six.moves.email_mime_base import MIMEBase
-if six.PY2:
-    from email.MIMENonMultipart import MIMENonMultipart
-    from email import Encoders
-else:
-    from email.mime.nonmultipart import MIMENonMultipart
-    from email import encoders as Encoders
+from email.mime.nonmultipart import MIMENonMultipart
+from email import encoders as Encoders
 
 from twisted.internet import defer, reactor, ssl
 
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index f28c7940d..c871e86e0 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,19 +1,13 @@
 import six
 import json
 import copy
-import collections
+from collections.abc import MutableMapping
 from importlib import import_module
 from pprint import pformat
 
 from scrapy.settings import default_settings
 
 
-if six.PY2:
-    MutableMapping = collections.MutableMapping
-else:
-    MutableMapping = collections.abc.MutableMapping
-
-
 SETTINGS_PRIORITIES = {
     'default': 0,
     'command': 10,
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 9c22999cb..5c9678c01 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -17,8 +17,6 @@ import sys
 from importlib import import_module
 from os.path import join, abspath, dirname
 
-import six
-
 AJAXCRAWL_ENABLED = False
 
 AUTOTHROTTLE_ENABLED = False
@@ -179,7 +177,7 @@ HTTPCACHE_ALWAYS_STORE = False
 HTTPCACHE_IGNORE_HTTP_CODES = []
 HTTPCACHE_IGNORE_SCHEMES = ['file']
 HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
-HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
+HTTPCACHE_DBM_MODULE = 'dbm'
 HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
 HTTPCACHE_GZIP = False
 
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index 421ab2f7e..c8fc911bb 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -1,7 +1,6 @@
 """Boto/botocore helpers"""
 
 from __future__ import absolute_import
-import six
 
 from scrapy.exceptions import NotConfigured
 
@@ -11,11 +10,4 @@ def is_botocore():
         import botocore
         return True
     except ImportError:
-        if six.PY2:
-            try:
-                import boto
-                return False
-            except ImportError:
-                raise NotConfigured('missing botocore or boto library')
-        else:
-            raise NotConfigured('missing botocore library')
+        raise NotConfigured('missing botocore library')
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index fb7ca3310..561bb72fc 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -1,13 +1,10 @@
+from configparser import ConfigParser
 import os
 import sys
 import numbers
 from operator import itemgetter
 
 import six
-if six.PY2:
-    from ConfigParser import SafeConfigParser as ConfigParser
-else:
-    from configparser import ConfigParser
 
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index df2b99c28..6e9de47f3 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -7,6 +7,7 @@ This module must not depend on any module outside the Standard Library.
 
 import copy
 import collections
+from collections.abc import Mapping
 import warnings
 
 import six
@@ -14,12 +15,6 @@ import six
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
-if six.PY2:
-    Mapping = collections.Mapping
-else:
-    Mapping = collections.abc.Mapping
-
-
 class MultiValueDictKeyError(KeyError):
     def __init__(self, *args, **kwargs):
         warnings.warn(
@@ -252,13 +247,12 @@ class MergeDict(object):
     first occurrence will be used.
     """
     def __init__(self, *dicts):
-        if not six.PY2:
-            warnings.warn(
-                "scrapy.utils.datatypes.MergeDict is deprecated in favor "
-                "of collections.ChainMap (introduced in Python 3.3)",
-                category=ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
+        warnings.warn(
+            "scrapy.utils.datatypes.MergeDict is deprecated in favor "
+            "of collections.ChainMap (introduced in Python 3.3)",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
         self.dicts = dicts
 
     def __getitem__(self, key):
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index b3fb16b1e..9984492f0 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -6,7 +6,6 @@ except ImportError:
     from io import BytesIO
 from gzip import GzipFile
 
-import six
 import re
 
 from scrapy.utils.decorators import deprecated
@@ -17,14 +16,8 @@ from scrapy.utils.decorators import deprecated
 #   (regression or bug-fix compared to Python 3.4)
 # - read1(), which fetches data before raising EOFError on next call
 #   works here but is only available from Python>=3.3
-# - scrapy does not support Python 3.2
-# - Python 2.7 GzipFile works fine with standard read() + extrabuf
-if six.PY2:
-    def read1(gzf, size=-1):
-        return gzf.read(size)
-else:
-    def read1(gzf, size=-1):
-        return gzf.read1(size)
+def read1(gzf, size=-1):
+    return gzf.read1(size)
 
 
 def gunzip(data):
@@ -37,7 +30,7 @@ def gunzip(data):
     chunk = b'.'
     while chunk:
         try:
-            chunk = read1(f, 8196)
+            chunk = f.read1(8196)
             output_list.append(chunk)
         except (IOError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index a12e14005..dbc1e0d20 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -102,11 +102,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     def row_to_unicode(row_):
         return [to_unicode(field, encoding) for field in row_]
 
-    # Python 3 csv reader input object needs to return strings
-    if six.PY3:
-        lines = StringIO(_body_or_str(obj, unicode=True))
-    else:
-        lines = BytesIO(_body_or_str(obj, unicode=False))
+    lines = StringIO(_body_or_str(obj, unicode=True))
 
     kwargs = {}
     if delimiter: kwargs["delimiter"] = delimiter
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index c6140f885..5009aab81 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -113,10 +113,7 @@ def to_bytes(text, encoding=None, errors='strict'):
 def to_native_str(text, encoding=None, errors='strict'):
     """ Return str representation of ``text``
     (bytes in Python 2.x and unicode in Python 3.x). """
-    if six.PY2:
-        return to_bytes(text, encoding, errors)
-    else:
-        return to_unicode(text, encoding, errors)
+    return to_unicode(text, encoding, errors)
 
 
 def re_rsearch(pattern, text, chunk_size=1024):
@@ -189,7 +186,7 @@ def _getargspec_py23(func):
     """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords,
                                                         defaults)
 
-    Identical to inspect.getargspec() in python2, but uses
+    Was identical to inspect.getargspec() in python2, but uses
     inspect.getfullargspec() for python3 behind the scenes to avoid
     DeprecationWarning.
 
@@ -199,9 +196,6 @@ def _getargspec_py23(func):
     >>> _getargspec_py23(f)
     ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,))
     """
-    if six.PY2:
-        return inspect.getargspec(func)
-
     return inspect.ArgSpec(*inspect.getfullargspec(func)[:4])
 
 
diff --git a/tests/__init__.py b/tests/__init__.py
index 9c9e35c35..a54367f8c 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -35,18 +35,3 @@ def get_testdata(*paths):
     path = os.path.join(tests_datadir, *paths)
     with open(path, 'rb') as f:
         return f.read()
-
-
-# FIXME: delete after dropping py2 support
-# Monkey patch the unittest module to prevent the
-# DeprecationWarning about assertRaisesRegexp -> assertRaisesRegex
-import six
-if six.PY2:
-    import unittest
-    import twisted.trial.unittest
-    if not getattr(unittest.TestCase, 'assertRegex', None):
-        unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches
-    if not getattr(unittest.TestCase, 'assertRaisesRegex', None):
-        unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
-    if not getattr(twisted.trial.unittest.TestCase, 'assertRaisesRegex', None):
-        twisted.trial.unittest.TestCase.assertRaisesRegex = twisted.trial.unittest.TestCase.assertRaisesRegexp
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 16d7a1cb8..828902b99 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -3,13 +3,12 @@ import cgi
 import unittest
 import re
 import json
+from urllib.parse import unquote_to_bytes
 import warnings
 
 import six
 from six.moves import xmlrpc_client as xmlrpclib
 from six.moves.urllib.parse import urlparse, parse_qs, unquote
-if six.PY3:
-    from urllib.parse import unquote_to_bytes
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_native_str
@@ -1064,8 +1063,7 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs, {})
 
         xpath = u"//form[@name='\u03b1']"
-        encoded = xpath if six.PY3 else xpath.encode('unicode_escape')
-        self.assertRaisesRegex(ValueError, re.escape(encoded),
+        self.assertRaisesRegex(ValueError, re.escape(xpath),
                                self.request_class.from_response,
                                response, formxpath=xpath)
 
@@ -1208,10 +1206,7 @@ def _qs(req, encoding='utf-8', to_unicode=False):
         qs = req.body
     else:
         qs = req.url.partition('?')[2]
-    if six.PY2:
-        uqs = unquote(to_native_str(qs, encoding))
-    elif six.PY3:
-        uqs = unquote_to_bytes(qs)
+    uqs = unquote_to_bytes(qs)
     if to_unicode:
         uqs = uqs.decode(encoding)
     return parse_qs(uqs, True)
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index cd5c3486e..d6e77d6b8 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -21,8 +21,7 @@ class BaseResponseTest(unittest.TestCase):
         # Response requires url in the consturctor
         self.assertRaises(Exception, self.response_class)
         self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class))
-        if not six.PY2:
-            self.assertRaises(TypeError, self.response_class, b"http://example.com")
+        self.assertRaises(TypeError, self.response_class, b"http://example.com")
         # body can be str or None
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class))
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class))
diff --git a/tests/test_item.py b/tests/test_item.py
index 947566686..0ad278701 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -62,12 +62,8 @@ class ItemTest(unittest.TestCase):
         i['number'] = 123
         itemrepr = repr(i)
 
-        if six.PY2:
-            self.assertEqual(itemrepr,
-                             "{'name': u'John Doe', 'number': 123}")
-        else:
-            self.assertEqual(itemrepr,
-                             "{'name': 'John Doe', 'number': 123}")
+        self.assertEqual(itemrepr,
+                         "{'name': 'John Doe', 'number': 123}")
 
         i2 = eval(itemrepr)
         self.assertEqual(i2['name'], 'John Doe')
diff --git a/tests/test_link.py b/tests/test_link.py
index 955430b37..5e2ce5eeb 100644
--- a/tests/test_link.py
+++ b/tests/test_link.py
@@ -1,6 +1,4 @@
 import unittest
-import warnings
-import six
 
 from scrapy.link import Link
 
@@ -46,12 +44,5 @@ class LinkTest(unittest.TestCase):
         self._assert_same_links(l1, l2)
 
     def test_non_str_url_py2(self):
-        if six.PY2:
-            with warnings.catch_warnings(record=True) as w:
-                link = Link(u"http://www.example.com/\xa3")
-                self.assertIsInstance(link.url, str)
-                self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3')
-            assert len(w) == 1, "warning not issued"
-        else:
-            with self.assertRaises(TypeError):
-                Link(b"http://www.example.com/\xc2\xa3")
+        with self.assertRaises(TypeError):
+            Link(b"http://www.example.com/\xc2\xa3")
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index aea0be825..af9b43d61 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -3,7 +3,6 @@ from twisted.trial import unittest
 from scrapy.settings import Settings
 from scrapy.exceptions import NotConfigured
 from scrapy.middleware import MiddlewareManager
-import six
 
 class M1(object):
 
@@ -66,20 +65,12 @@ class MiddlewareManagerTest(unittest.TestCase):
 
     def test_methods(self):
         mwman = TestMiddlewareManager(M1(), M2(), M3())
-        if six.PY2:
-            self.assertEqual([x.im_class for x in mwman.methods['open_spider']],
-                [M1, M2])
-            self.assertEqual([x.im_class for x in mwman.methods['close_spider']],
-                [M2, M1])
-            self.assertEqual([x.im_class for x in mwman.methods['process']],
-                [M1, M3])
-        else:
-            self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
-                [M1, M2])
-            self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
-                [M2, M1])
-            self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
-                [M1, M3])
+        self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
+            [M1, M2])
+        self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
+            [M2, M1])
+        self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
+            [M1, M3])
 
     def test_enabled(self):
         m1, m2, m3 = M1(), M2(), M3()
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index c9943faa8..a5cdc0de0 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -1,7 +1,6 @@
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
-import six
 
 from scrapy.http import Request
 from scrapy.crawler import CrawlerRunner
@@ -161,9 +160,4 @@ class CallbackKeywordArgumentsTestCase(TestCase):
         self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
         self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
         self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
-        # py2 and py3 messages are different
-        exc_message = str(exceptions['takes_more'].exc_info[1])
-        if six.PY2:
-            self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
-        elif six.PY3:
-            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
+        self.assertEqual(str(exceptions['takes_more'].exc_info[1]), "parse_takes_more() missing 1 required positional argument: 'other'")
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 1dbacbea3..08286ff02 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -60,9 +60,6 @@ class SettingsAttributeTest(unittest.TestCase):
 
 class BaseSettingsTest(unittest.TestCase):
 
-    if six.PY3:
-        assertItemsEqual = unittest.TestCase.assertCountEqual
-
     def setUp(self):
         self.settings = BaseSettings()
 
@@ -152,7 +149,7 @@ class BaseSettingsTest(unittest.TestCase):
         self.settings.setmodule(
             'tests.test_settings.default_settings', 10)
 
-        self.assertItemsEqual(six.iterkeys(self.settings.attributes),
+        self.assertCountEqual(six.iterkeys(self.settings.attributes),
                               six.iterkeys(ctrl_attributes))
 
         for key in six.iterkeys(ctrl_attributes):
@@ -343,9 +340,6 @@ class BaseSettingsTest(unittest.TestCase):
 
 class SettingsTest(unittest.TestCase):
 
-    if six.PY3:
-        assertItemsEqual = unittest.TestCase.assertCountEqual
-
     def setUp(self):
         self.settings = Settings()
 
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 535095b8d..47877f555 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,12 +1,7 @@
+from collections.abc import Mapping, MutableMapping
 import copy
 import unittest
 
-import six
-if six.PY2:
-    from collections import Mapping, MutableMapping
-else:
-    from collections.abc import Mapping, MutableMapping
-
 from scrapy.utils.datatypes import CaselessDict, SequenceExclude
 
 
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3e1148354..096aa50b7 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -231,12 +231,11 @@ class UtilsPythonTestCase(unittest.TestCase):
             self.assertEqual(get_func_args(" ".join), [])
             self.assertEqual(get_func_args(operator.itemgetter(2)), [])
         else:
-            stripself = not six.PY2  # PyPy3 exposes them as methods
             self.assertEqual(
-                get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit'])
-            self.assertEqual(get_func_args(" ".join, stripself), ['list'])
+                get_func_args(six.text_type.split, True), ['sep', 'maxsplit'])
+            self.assertEqual(get_func_args(" ".join, True), ['list'])
             self.assertEqual(
-                get_func_args(operator.itemgetter(2), stripself), ['obj'])
+                get_func_args(operator.itemgetter(2), True), ['obj'])
 
 
     def test_without_none_values(self):

From de7789e52df5d8dc8595570d36a60ae92b4fcb50 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 20:56:26 +0500
Subject: [PATCH 2311/2974] Remove unneeded and unused code from
 XmlItemExporter.

---
 scrapy/exporters.py | 22 ++++------------------
 1 file changed, 4 insertions(+), 18 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 5d1f1ad8f..40567f53b 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -143,11 +143,11 @@ class XmlItemExporter(BaseItemExporter):
 
     def _beautify_newline(self, new_item=False):
         if self.indent is not None and (self.indent > 0 or new_item):
-            self._xg_characters('\n')
+            self.xg.characters('\n')
 
     def _beautify_indent(self, depth=1):
         if self.indent:
-            self._xg_characters(' ' * self.indent * depth)
+            self.xg.characters(' ' * self.indent * depth)
 
     def start_exporting(self):
         self.xg.startDocument()
@@ -182,26 +182,12 @@ class XmlItemExporter(BaseItemExporter):
                 self._export_xml_field('value', value, depth=depth+1)
             self._beautify_indent(depth=depth)
         elif isinstance(serialized_value, six.text_type):
-            self._xg_characters(serialized_value)
+            self.xg.characters(serialized_value)
         else:
-            self._xg_characters(str(serialized_value))
+            self.xg.characters(str(serialized_value))
         self.xg.endElement(name)
         self._beautify_newline()
 
-    # Workaround for https://bugs.python.org/issue17606
-    # Before Python 2.7.4 xml.sax.saxutils required bytes;
-    # since 2.7.4 it requires unicode. The bug is likely to be
-    # fixed in 2.7.6, but 2.7.6 will still support unicode,
-    # and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
-    if sys.version_info[:3] >= (2, 7, 4):
-        def _xg_characters(self, serialized_value):
-            if not isinstance(serialized_value, six.text_type):
-                serialized_value = serialized_value.decode(self.encoding)
-            return self.xg.characters(serialized_value)
-    else:  # pragma: no cover
-        def _xg_characters(self, serialized_value):
-            return self.xg.characters(serialized_value)
-
 
 class CsvItemExporter(BaseItemExporter):
 

From c2898fdcf91cbbfa9d801196924960d3a3764b93 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 21:06:52 +0500
Subject: [PATCH 2312/2974] Deprecate scrapy.utils.gz.read1.

---
 scrapy/utils/gz.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 9984492f0..dc8316d8c 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -16,6 +16,7 @@ from scrapy.utils.decorators import deprecated
 #   (regression or bug-fix compared to Python 3.4)
 # - read1(), which fetches data before raising EOFError on next call
 #   works here but is only available from Python>=3.3
+@deprecated('GzipFile.read1')
 def read1(gzf, size=-1):
     return gzf.read1(size)
 

From cea2f5e244f30591bae667abf2b2689e566ab343 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 21:09:49 +0500
Subject: [PATCH 2313/2974] Remove cStringIO imports.

---
 scrapy/downloadermiddlewares/decompression.py | 6 +-----
 scrapy/mail.py                                | 6 +-----
 scrapy/pipelines/files.py                     | 7 +------
 scrapy/pipelines/images.py                    | 6 +-----
 scrapy/utils/gz.py                            | 9 ++-------
 scrapy/utils/iterators.py                     | 6 +-----
 tests/test_cmdline/__init__.py                | 5 +----
 tests/test_pipeline_media.py                  | 6 ++----
 8 files changed, 10 insertions(+), 41 deletions(-)

diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index 49313cc04..e2d73f347 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -4,6 +4,7 @@ and extract the potentially compressed responses that may arrive.
 
 import bz2
 import gzip
+from io import BytesIO
 import zipfile
 import tarfile
 import logging
@@ -11,11 +12,6 @@ from tempfile import mktemp
 
 import six
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 from scrapy.responsetypes import responsetypes
 
 logger = logging.getLogger(__name__)
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 746468e25..d24de2212 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -3,13 +3,9 @@ Mail sending helpers
 
 See documentation in docs/topics/email.rst
 """
+from io import BytesIO
 import logging
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 from email.utils import COMMASPACE, formatdate
 from six.moves.email_mime_multipart import MIMEMultipart
 from six.moves.email_mime_text import MIMEText
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index cc3d10b63..8d74c5011 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -5,6 +5,7 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
+from io import BytesIO
 import mimetypes
 import os
 import os.path
@@ -15,12 +16,6 @@ from six.moves.urllib.parse import urlparse
 from collections import defaultdict
 import six
 
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 from twisted.internet import defer, threads
 
 from scrapy.pipelines.media import MediaPipeline
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index fa4d12ad1..e77cef4ff 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -5,13 +5,9 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
+from io import BytesIO
 import six
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 from PIL import Image
 
 from scrapy.utils.misc import md5sum
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index dc8316d8c..f41e62fe3 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -1,12 +1,7 @@
-import struct
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
 from gzip import GzipFile
-
+from io import BytesIO
 import re
+import struct
 
 from scrapy.utils.decorators import deprecated
 
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index dbc1e0d20..9693ba768 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -1,11 +1,7 @@
 import re
 import csv
-import logging
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
 from io import StringIO
+import logging
 import six
 
 from scrapy.http import TextResponse, Response
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 68dfb1cca..56cfe642a 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -1,3 +1,4 @@
+from io import StringIO
 import json
 import os
 import pstats
@@ -7,10 +8,6 @@ from subprocess import Popen, PIPE
 import sys
 import tempfile
 import unittest
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from io import StringIO
 
 from scrapy.utils.test import get_testenv
 
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 28e39cefa..ad2618ec9 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -144,10 +144,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 
         # The Failure should encapsulate a FileException ...
         self.assertEqual(failure.value, file_exc)
-        # ... and if we're running on Python 3 ...
-        if sys.version_info.major >= 3:
-            # ... it should have the returnValue exception set as its context
-            self.assertEqual(failure.value.__context__, def_gen_return_exc)
+        # ... and it should have the returnValue exception set as its context
+        self.assertEqual(failure.value.__context__, def_gen_return_exc)
 
         # Let's calculate the request fingerprint and fake some runtime data...
         fp = request_fingerprint(request)

From 5b70b051a6cec699c3cb14c3ac9cf530671a8487 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 21:13:01 +0500
Subject: [PATCH 2314/2974] Some text function messages cleanup, deprecate
 to_native_str.

---
 scrapy/http/response/__init__.py | 2 +-
 scrapy/utils/python.py           | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index b0a526b72..a81404afb 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -88,7 +88,7 @@ class Response(object_ref):
     @property
     def text(self):
         """For subclasses of TextResponse, this will return the body
-        as text (unicode object in Python 2 and str in Python 3)
+        as str
         """
         raise AttributeError("Response content isn't text")
 
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 5009aab81..974abaeb1 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -90,7 +90,7 @@ def to_unicode(text, encoding=None, errors='strict'):
     if isinstance(text, six.text_type):
         return text
     if not isinstance(text, (bytes, six.text_type)):
-        raise TypeError('to_unicode must receive a bytes, str or unicode '
+        raise TypeError('to_unicode must receive a bytes or str '
                         'object, got %s' % type(text).__name__)
     if encoding is None:
         encoding = 'utf-8'
@@ -103,16 +103,16 @@ def to_bytes(text, encoding=None, errors='strict'):
     if isinstance(text, bytes):
         return text
     if not isinstance(text, six.string_types):
-        raise TypeError('to_bytes must receive a unicode, str or bytes '
+        raise TypeError('to_bytes must receive a str or bytes '
                         'object, got %s' % type(text).__name__)
     if encoding is None:
         encoding = 'utf-8'
     return text.encode(encoding, errors)
 
 
+@deprecated('to_unicode')
 def to_native_str(text, encoding=None, errors='strict'):
-    """ Return str representation of ``text``
-    (bytes in Python 2.x and unicode in Python 3.x). """
+    """ Return str representation of ``text``. """
     return to_unicode(text, encoding, errors)
 
 

From 3ac4b430ae0165f25d8cd2fb1ac25ac9f307df8b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 15:20:28 +0500
Subject: [PATCH 2315/2974] Remove an unused six import.

---
 tests/test_downloader_handlers.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 9f74577a5..e6856945c 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1,5 +1,4 @@
 import os
-import six
 import shutil
 import tempfile
 import contextlib

From 75b1d051d99cda17558a210326e77046eec851a7 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 21:27:52 +0500
Subject: [PATCH 2316/2974] Simplify some more imports.

---
 scrapy/downloadermiddlewares/httpproxy.py    | 5 +----
 scrapy/loader/processors.py                  | 5 +----
 tests/__init__.py                            | 5 -----
 tests/test_downloader_handlers.py            | 5 +----
 tests/test_downloadermiddleware.py           | 3 ++-
 tests/test_downloadermiddleware_robotstxt.py | 4 +++-
 tests/test_extension_telnet.py               | 5 -----
 tests/test_feedexport.py                     | 2 +-
 tests/test_http_request.py                   | 3 +--
 tests/test_item.py                           | 2 +-
 tests/test_pipeline_files.py                 | 6 +-----
 tests/test_settings/__init__.py              | 3 +--
 tests/test_spider.py                         | 3 +--
 tests/test_spidermiddleware.py               | 3 ++-
 tests/test_stats.py                          | 6 +-----
 tests/test_utils_deprecate.py                | 3 +--
 tests/test_utils_misc/__init__.py            | 2 +-
 tests/test_utils_trackref.py                 | 2 +-
 18 files changed, 20 insertions(+), 47 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 2c35d1b90..2212d9688 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -1,10 +1,7 @@
 import base64
 from six.moves.urllib.parse import unquote, urlunparse
 from six.moves.urllib.request import getproxies, proxy_bypass
-try:
-    from urllib2 import _parse_proxy
-except ImportError:
-    from urllib.request import _parse_proxy
+from urllib.request import _parse_proxy
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index 2acdc8093..02c625acc 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -3,10 +3,7 @@ This module provides some commonly used processors for Item Loaders.
 
 See documentation in docs/topics/loaders.rst
 """
-try:
-    from collections import ChainMap
-except ImportError:
-    from scrapy.utils.datatypes import MergeDict as ChainMap
+from collections import ChainMap
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.loader.common import wrap_loader_context
diff --git a/tests/__init__.py b/tests/__init__.py
index a54367f8c..12ce79fa9 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -21,11 +21,6 @@ if 'COV_CORE_CONFIG' in os.environ:
     os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot,
                                                  os.environ['COV_CORE_CONFIG'])
 
-try:
-    import unittest.mock as mock
-except ImportError:
-    import mock
-
 tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
                              'sample_data')
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 109469503..6090998d4 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -2,11 +2,8 @@ import os
 import six
 import shutil
 import tempfile
+from unittest import mock
 import contextlib
-try:
-    from unittest import mock
-except ImportError:
-    import mock
 
 from testfixtures import LogCapture
 from twisted.trial import unittest
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 03564e748..6b9a5bee8 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -1,3 +1,5 @@
+from unittest import mock
+
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
 
@@ -7,7 +9,6 @@ from scrapy.exceptions import _InvalidOutput
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.utils.test import get_crawler
 from scrapy.utils.python import to_bytes
-from tests import mock
 
 
 class ManagerTestCase(TestCase):
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index fbc46cba4..8266bf35f 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,5 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
+
+from unittest import mock
+
 from twisted.internet import reactor, error
 from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
 from twisted.python import failure
@@ -9,7 +12,6 @@ from scrapy.downloadermiddlewares.robotstxt import (RobotsTxtMiddleware,
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response, TextResponse
 from scrapy.settings import Settings
-from tests import mock
 from tests.test_robotstxt_interface import rerp_available, reppy_available
 
 
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index 4f389e5cb..875ceb83c 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -1,8 +1,3 @@
-try:
-    import unittest.mock as mock
-except ImportError:
-    import mock
-
 from twisted.trial import unittest
 from twisted.conch.telnet import ITelnetProtocol
 from twisted.cred import credentials
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e1436fbe5..7431f921f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -7,6 +7,7 @@ from io import BytesIO
 import tempfile
 import shutil
 import string
+from unittest import mock
 from six.moves.urllib.parse import urljoin, urlparse, quote
 from six.moves.urllib.request import pathname2url
 
@@ -15,7 +16,6 @@ from twisted.trial import unittest
 from twisted.internet import defer
 from scrapy.crawler import CrawlerRunner
 from scrapy.settings import Settings
-from tests import mock
 from tests.mockserver import MockServer
 from w3lib.url import path_to_file_uri
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 828902b99..effb9e53b 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -3,6 +3,7 @@ import cgi
 import unittest
 import re
 import json
+from unittest import mock
 from urllib.parse import unquote_to_bytes
 import warnings
 
@@ -13,8 +14,6 @@ from six.moves.urllib.parse import urlparse, parse_qs, unquote
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_native_str
 
-from tests import mock
-
 
 class RequestTest(unittest.TestCase):
 
diff --git a/tests/test_item.py b/tests/test_item.py
index 0ad278701..d98c63ddd 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -1,12 +1,12 @@
 import sys
 import unittest
+from unittest import mock
 from warnings import catch_warnings
 
 import six
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
-from tests import mock
 
 
 PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6)
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index cb8f8da18..bd40e4103 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,10 +1,9 @@
 import os
 import random
 import time
-import hashlib
-import warnings
 from tempfile import mkdtemp
 from shutil import rmtree
+from unittest import mock
 from six.moves.urllib.parse import urlparse
 from six import BytesIO
 
@@ -15,13 +14,10 @@ from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GC
 from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
-from scrapy.utils.python import to_bytes
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
 from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete
 from scrapy.utils.boto import is_botocore
 
-from tests import mock
-
 
 def _mocked_download_func(request, info):
     response = request.meta.get('response')
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 08286ff02..32e65bed5 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -1,10 +1,9 @@
 import six
 import unittest
-import warnings
+from unittest import mock
 
 from scrapy.settings import (BaseSettings, Settings, SettingsAttribute,
                              SETTINGS_PRIORITIES, get_settings_priority)
-from tests import mock
 from . import default_settings
 
 
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 2220b8ffc..b913a56b7 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -1,5 +1,6 @@
 import gzip
 import inspect
+from unittest import mock
 import warnings
 from io import BytesIO
 
@@ -17,8 +18,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.test import get_crawler
 
-from tests import mock
-
 
 class SpiderTest(unittest.TestCase):
 
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 832fd3330..55d665e79 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -1,3 +1,5 @@
+from unittest import mock
+
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
 
@@ -6,7 +8,6 @@ from scrapy.http import Request, Response
 from scrapy.exceptions import _InvalidOutput
 from scrapy.utils.test import get_crawler
 from scrapy.core.spidermw import SpiderMiddlewareManager
-from tests import mock
 
 
 class SpiderMiddlewareTestCase(TestCase):
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 2033dbe07..2bbbb9e2c 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -1,10 +1,6 @@
 from datetime import datetime
 import unittest
-
-try:
-    from unittest import mock
-except ImportError:
-    import mock
+from unittest import mock
 
 from scrapy.extensions.corestats import CoreStats
 from scrapy.spiders import Spider
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 3e7236fb1..ce04e7f29 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -2,12 +2,11 @@
 from __future__ import absolute_import
 import inspect
 import unittest
+from unittest import mock
 import warnings
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.deprecate import create_deprecated_class, update_classpath
 
-from tests import mock
-
 
 class MyWarning(UserWarning):
     pass
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index e109d5343..de9da9104 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -1,11 +1,11 @@
 import sys
 import os
 import unittest
+from unittest import mock
 
 from scrapy.item import Item, Field
 from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules
 
-from tests import mock
 
 __doctests__ = ['scrapy.utils.misc']
 
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index c6072fc0d..480a717e7 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -1,7 +1,7 @@
 import six
 import unittest
+from unittest import mock
 from scrapy.utils import trackref
-from tests import mock
 
 
 class Foo(trackref.object_ref):

From 397e8835564614608647c855c62c36232020f078 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 21:35:13 +0500
Subject: [PATCH 2317/2974] Replace to_native_str calls with to_unicode.

---
 scrapy/core/downloader/handlers/http11.py |  2 +-
 scrapy/downloadermiddlewares/cookies.py   |  6 +++---
 scrapy/downloadermiddlewares/robotstxt.py |  3 ---
 scrapy/exporters.py                       |  4 ++--
 scrapy/http/cookies.py                    | 10 +++++-----
 scrapy/http/response/text.py              |  8 ++++----
 scrapy/linkextractors/lxmlhtml.py         |  4 ++--
 scrapy/responsetypes.py                   |  6 +++---
 scrapy/robotstxt.py                       |  8 ++++----
 scrapy/spidermiddlewares/referer.py       |  5 ++---
 scrapy/utils/reqser.py                    |  4 ++--
 scrapy/utils/request.py                   |  4 ++--
 scrapy/utils/response.py                  |  4 ++--
 scrapy/utils/ssl.py                       |  4 ++--
 tests/test_command_parse.py               |  5 ++---
 tests/test_commands.py                    |  7 ++-----
 tests/test_feedexport.py                  |  7 +++----
 tests/test_http_request.py                |  7 +++----
 tests/test_http_response.py               |  6 +++---
 tests/test_robotstxt_interface.py         |  1 -
 20 files changed, 47 insertions(+), 58 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 91b45a8fc..7d917cb74 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -174,7 +174,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
     r"""
     Return binary content of a CONNECT request.
 
-    >>> from scrapy.utils.python import to_native_str as s
+    >>> from scrapy.utils.python import to_unicode as s
     >>> s(tunnel_request_data("example.com", 8080))
     'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
     >>> s(tunnel_request_data("example.com", 8080, b"123"))
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 321c0171b..0d2b9900c 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -6,7 +6,7 @@ from collections import defaultdict
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 
 logger = logging.getLogger(__name__)
 
@@ -53,7 +53,7 @@ class CookiesMiddleware(object):
 
     def _debug_cookie(self, request, spider):
         if self.debug:
-            cl = [to_native_str(c, errors='replace')
+            cl = [to_unicode(c, errors='replace')
                   for c in request.headers.getlist('Cookie')]
             if cl:
                 cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)
@@ -62,7 +62,7 @@ class CookiesMiddleware(object):
 
     def _debug_set_cookie(self, response, spider):
         if self.debug:
-            cl = [to_native_str(c, errors='replace')
+            cl = [to_unicode(c, errors='replace')
                   for c in response.headers.getlist('Set-Cookie')]
             if cl:
                 cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 6a5dfb79c..251706c50 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -5,15 +5,12 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
 """
 
 import logging
-import sys
-import re
 
 from twisted.internet.defer import Deferred, maybeDeferred
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.python import to_native_str
 from scrapy.utils.misc import load_object
 
 logger = logging.getLogger(__name__)
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 40567f53b..f276c28e8 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -12,7 +12,7 @@ from six.moves import cPickle as pickle
 from xml.sax.saxutils import XMLGenerator
 
 from scrapy.utils.serialize import ScrapyJSONEncoder
-from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike
+from scrapy.utils.python import to_bytes, to_unicode, is_listlike
 from scrapy.item import BaseItem
 from scrapy.exceptions import ScrapyDeprecationWarning
 import warnings
@@ -232,7 +232,7 @@ class CsvItemExporter(BaseItemExporter):
     def _build_row(self, values):
         for s in values:
             try:
-                yield to_native_str(s, self.encoding)
+                yield to_unicode(s, self.encoding)
             except TypeError:
                 yield s
 
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 4e8056750..4532c3ab7 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -3,7 +3,7 @@ from six.moves.http_cookiejar import (
     CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
 )
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 
 
 class CookieJar(object):
@@ -165,13 +165,13 @@ class WrappedRequest(object):
         return name in self.request.headers
 
     def get_header(self, name, default=None):
-        return to_native_str(self.request.headers.get(name, default),
+        return to_unicode(self.request.headers.get(name, default),
                              errors='replace')
 
     def header_items(self):
         return [
-            (to_native_str(k, errors='replace'),
-             [to_native_str(x, errors='replace') for x in v])
+            (to_unicode(k, errors='replace'),
+             [to_unicode(x, errors='replace') for x in v])
             for k, v in self.request.headers.items()
         ]
 
@@ -189,7 +189,7 @@ class WrappedResponse(object):
 
     # python3 cookiejars calls get_all
     def get_all(self, name, default=None):
-        return [to_native_str(v, errors='replace')
+        return [to_unicode(v, errors='replace')
                 for v in self.response.headers.getlist(name)]
     # python2 cookiejars calls getheaders
     getheaders = get_all
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index a8010877c..37f450e54 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -16,7 +16,7 @@ from w3lib.html import strip_html5_whitespace
 from scrapy.http.request import Request
 from scrapy.http.response import Response
 from scrapy.utils.response import get_base_url
-from scrapy.utils.python import memoizemethod_noargs, to_native_str
+from scrapy.utils.python import memoizemethod_noargs, to_unicode
 
 
 class TextResponse(Response):
@@ -32,7 +32,7 @@ class TextResponse(Response):
 
     def _set_url(self, url):
         if isinstance(url, six.text_type):
-            self._url = to_native_str(url, self.encoding)
+            self._url = to_unicode(url, self.encoding)
         else:
             super(TextResponse, self)._set_url(url)
 
@@ -81,11 +81,11 @@ class TextResponse(Response):
     @memoizemethod_noargs
     def _headers_encoding(self):
         content_type = self.headers.get(b'Content-Type', b'')
-        return http_content_type_encoding(to_native_str(content_type))
+        return http_content_type_encoding(to_unicode(content_type))
 
     def _body_inferred_encoding(self):
         if self._cached_benc is None:
-            content_type = to_native_str(self.headers.get(b'Content-Type', b''))
+            content_type = to_unicode(self.headers.get(b'Content-Type', b''))
             benc, ubody = html_to_unicode(content_type, self.body,
                     auto_detect_fun=self._auto_detect_fun,
                     default_encoding=self._DEFAULT_ENCODING)
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 8f6f93a44..890c019c8 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -10,7 +10,7 @@ from w3lib.url import canonicalize_url
 
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
-from scrapy.utils.python import unique as unique_list, to_native_str
+from scrapy.utils.python import unique as unique_list, to_unicode
 from scrapy.utils.response import get_base_url
 from scrapy.linkextractors import FilteringLinkExtractor
 
@@ -67,7 +67,7 @@ class LxmlParserLinkExtractor(object):
                 url = self.process_attr(attr_val)
                 if url is None:
                     continue
-            url = to_native_str(url, encoding=response_encoding)
+            url = to_unicode(url, encoding=response_encoding)
             # to fix relative links after process_value
             url = urljoin(response_url, url)
             link = Link(url, _collect_string_content(el) or u'',
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 4a2d5bf52..de62276c8 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -10,7 +10,7 @@ import six
 
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
-from scrapy.utils.python import binary_is_text, to_bytes, to_native_str
+from scrapy.utils.python import binary_is_text, to_bytes, to_unicode
 
 
 class ResponseTypes(object):
@@ -55,12 +55,12 @@ class ResponseTypes(object):
         header """
         if content_encoding:
             return Response
-        mimetype = to_native_str(content_type).split(';')[0].strip().lower()
+        mimetype = to_unicode(content_type).split(';')[0].strip().lower()
         return self.from_mimetype(mimetype)
 
     def from_content_disposition(self, content_disposition):
         try:
-            filename = to_native_str(content_disposition,
+            filename = to_unicode(content_disposition,
                 encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
             filename = filename.strip('"\'')
             return self.from_filename(filename)
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 189f165d1..95a8c09b8 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -3,14 +3,14 @@ import logging
 from abc import ABCMeta, abstractmethod
 from six import with_metaclass
 
-from scrapy.utils.python import to_native_str, to_unicode
+from scrapy.utils.python import to_unicode
 
 logger = logging.getLogger(__name__)
 
 def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
     try:
         if to_native_str_type:
-            robotstxt_body = to_native_str(robotstxt_body)
+            robotstxt_body = to_unicode(robotstxt_body)
         else:
             robotstxt_body = robotstxt_body.decode('utf-8')
     except UnicodeDecodeError:
@@ -66,8 +66,8 @@ class PythonRobotParser(RobotParser):
         return o
 
     def allowed(self, url, user_agent):
-        user_agent = to_native_str(user_agent)
-        url = to_native_str(url)
+        user_agent = to_unicode(user_agent)
+        url = to_unicode(url)
         return self.rp.can_fetch(user_agent, url)
 
 
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 1ddfb37f4..c76e4d5a2 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -10,8 +10,7 @@ from w3lib.url import safe_url_string
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
-from scrapy.utils.python import to_native_str
-from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 from scrapy.utils.url import strip_url
 
@@ -322,7 +321,7 @@ class RefererMiddleware(object):
             if isinstance(resp_or_url, Response):
                 policy_header = resp_or_url.headers.get('Referrer-Policy')
                 if policy_header is not None:
-                    policy_name = to_native_str(policy_header.decode('latin1'))
+                    policy_name = to_unicode(policy_header.decode('latin1'))
         if policy_name is None:
             return self.default_policy()
 
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index c7ea7b425..495564ac0 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -4,7 +4,7 @@ Helper functions for serializing (and deserializing) requests.
 import six
 
 from scrapy.http import Request
-from scrapy.utils.python import to_unicode, to_native_str
+from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 
 
@@ -54,7 +54,7 @@ def request_from_dict(d, spider=None):
         eb = _get_method(spider, eb)
     request_cls = load_object(d['_class']) if '_class' in d else Request
     return request_cls(
-        url=to_native_str(d['url']),
+        url=to_unicode(d['url']),
         callback=cb,
         errback=eb,
         method=d['method'],
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index fb5af66a2..63d0ae772 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -9,7 +9,7 @@ import weakref
 from six.moves.urllib.parse import urlunparse
 
 from w3lib.http import basic_auth_header
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
 
 from w3lib.url import canonicalize_url
 from scrapy.utils.httpobj import urlparse_cached
@@ -97,4 +97,4 @@ def referer_str(request):
     referrer = request.headers.get('Referer')
     if referrer is None:
         return referrer
-    return to_native_str(referrer, errors='replace')
+    return to_unicode(referrer, errors='replace')
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index c3236afd4..feab07431 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -8,7 +8,7 @@ import webbrowser
 import tempfile
 
 from twisted.web import http
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
 from w3lib import html
 
 
@@ -36,7 +36,7 @@ def response_status_message(status):
     """Return status code plus status text descriptive message
     """
     message = http.RESPONSES.get(int(status), "Unknown Status")
-    return '%s %s' % (status, to_native_str(message))
+    return '%s %s' % (status, to_unicode(message))
 
 
 def response_httprepr(response):
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index 02aed60ee..6e81b33ff 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -3,7 +3,7 @@
 import OpenSSL
 import OpenSSL._util as pyOpenSSLutil
 
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 
 
 # The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL.
@@ -12,7 +12,7 @@ SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0)
 
 
 def ffi_buf_to_string(buf):
-    return to_native_str(pyOpenSSLutil.ffi.string(buf))
+    return to_unicode(pyOpenSSLutil.ffi.string(buf))
 
 
 def x509name_to_string(x509name):
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 62d5d76b4..b134beb88 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,17 +1,16 @@
 import os
 from os.path import join, abspath
-from twisted.trial import unittest
 from twisted.internet import defer
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 from tests.test_commands import CommandTest
 
 
 def _textmode(bstr):
     """Normalize input the same as writing to a file
     and reading from it in text mode"""
-    return to_native_str(bstr).replace(os.linesep, '\n')
+    return to_unicode(bstr).replace(os.linesep, '\n')
 
 class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
     command = 'parse'
diff --git a/tests/test_commands.py b/tests/test_commands.py
index b8445ae6c..536379170 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -10,13 +10,10 @@ from contextlib import contextmanager
 from threading import Timer
 
 from twisted.trial import unittest
-from twisted.internet import defer
 
 import scrapy
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_testenv
-from scrapy.utils.testsite import SiteTest
-from scrapy.utils.testproc import ProcessTest
 from tests.test_crawler import ExceptionSpider, NoRequestsSpider
 
 
@@ -56,7 +53,7 @@ class ProjectTest(unittest.TestCase):
         finally:
             timer.cancel()
 
-        return p, to_native_str(stdout), to_native_str(stderr)
+        return p, to_unicode(stdout), to_unicode(stderr)
 
 
 class StartprojectTest(ProjectTest):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 7431f921f..abe2ab557 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -26,8 +26,7 @@ from scrapy.extensions.feedexport import (
     S3FeedStorage, StdoutFeedStorage,
     BlockingFeedStorage)
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
-from scrapy.utils.python import to_native_str
-from scrapy.utils.project import get_project_settings
+from scrapy.utils.python import to_unicode
 
 
 class FileFeedStorageTest(unittest.TestCase):
@@ -456,7 +455,7 @@ class FeedExportTest(unittest.TestCase):
         settings.update({'FEED_FORMAT': 'csv'})
         data = yield self.exported_data(items, settings)
 
-        reader = csv.DictReader(to_native_str(data).splitlines())
+        reader = csv.DictReader(to_unicode(data).splitlines())
         got_rows = list(reader)
         if ordered:
             self.assertEqual(reader.fieldnames, header)
@@ -470,7 +469,7 @@ class FeedExportTest(unittest.TestCase):
         settings = settings or {}
         settings.update({'FEED_FORMAT': 'jl'})
         data = yield self.exported_data(items, settings)
-        parsed = [json.loads(to_native_str(line)) for line in data.splitlines()]
+        parsed = [json.loads(to_unicode(line)) for line in data.splitlines()]
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         self.assertEqual(rows, parsed)
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index effb9e53b..807265981 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -7,12 +7,11 @@ from unittest import mock
 from urllib.parse import unquote_to_bytes
 import warnings
 
-import six
 from six.moves import xmlrpc_client as xmlrpclib
 from six.moves.urllib.parse import urlparse, parse_qs, unquote
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
 
 
 class RequestTest(unittest.TestCase):
@@ -349,8 +348,8 @@ class FormRequestTest(RequestTest):
     request_class = FormRequest
 
     def assertQueryEqual(self, first, second, msg=None):
-        first = to_native_str(first).split("&")
-        second = to_native_str(second).split("&")
+        first = to_unicode(first).split("&")
+        second = to_unicode(second).split("&")
         return self.assertEqual(sorted(first), sorted(second), msg)
 
     def test_empty_formdata(self):
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index d6e77d6b8..80bf51647 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -7,7 +7,7 @@ from w3lib.encoding import resolve_encoding
 from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
                          XmlResponse, Headers)
 from scrapy.selector import Selector
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 from scrapy.exceptions import NotSupported
 from scrapy.link import Link
 from tests import get_testdata
@@ -204,11 +204,11 @@ class TextResponseTest(BaseResponseTest):
         assert isinstance(resp.url, str)
 
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')
-        self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))
+        self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
         resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]})
-        self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))
+        self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
         resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
 
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 9aaab560a..cd7480e33 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -1,6 +1,5 @@
 # coding=utf-8
 from twisted.trial import unittest
-from scrapy.utils.python import to_native_str
 
 
 def reppy_available():

From 7299e91b1f1b231095e2a7ddbce5ad980ae7da8a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 28 Aug 2019 16:29:53 +0500
Subject: [PATCH 2318/2974] Remove Py2-only code that checks sys.version_info.

---
 tests/test_utils_reqser.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 11ac56897..92cd16de7 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -80,8 +80,6 @@ class RequestSerializationTest(unittest.TestCase):
         self._assert_serializes_ok(r, spider=self.spider)
 
     def test_mixin_private_callback_serialization(self):
-        if sys.version_info[0] < 3:
-            return
         r = Request("http://www.example.com",
                     callback=self.spider._TestSpiderMixin__mixin_callback,
                     errback=self.spider.handle_error)
@@ -119,9 +117,8 @@ class RequestSerializationTest(unittest.TestCase):
     def test_private_name_mangling(self):
         self._assert_mangles_to(
             self.spider, '_TestSpider__parse_item_private')
-        if sys.version_info[0] >= 3:
-            self._assert_mangles_to(
-                self.spider, '_TestSpiderMixin__mixin_callback')
+        self._assert_mangles_to(
+            self.spider, '_TestSpiderMixin__mixin_callback')
 
     def test_unserializable_callback1(self):
         r = Request("http://www.example.com", callback=lambda x: x)

From f02c3d1dcf3e4880388d19e961e7911be5dc54ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 31 Oct 2019 13:31:33 +0100
Subject: [PATCH 2319/2974] Use communicate() instead of wait() after killing
 the mock server (#4095)

---
 tests/mockserver.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 77908284b..b766bb653 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -206,8 +206,7 @@ class MockServer():
 
     def __exit__(self, exc_type, exc_value, traceback):
         self.proc.kill()
-        self.proc.wait()
-        time.sleep(0.2)
+        self.proc.communicate()
 
     def url(self, path, is_secure=False):
         host = self.http_address.replace('0.0.0.0', '127.0.0.1')

From 864123132a99dda61feb407a59367c6c06133517 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 22:55:58 +0500
Subject: [PATCH 2320/2974] Fix a duplicate ref name in docs.

---
 docs/topics/downloader-middleware.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 366b95510..e93645077 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -474,7 +474,7 @@ DBM storage backend
 
     A DBM_ storage backend is also available for the HTTP cache middleware.
 
-    By default, it uses the dbm_ module, but you can change it with the
+    By default, it uses the `dbm module`_, but you can change it with the
     :setting:`HTTPCACHE_DBM_MODULE` setting.
 
 .. _httpcache-storage-custom:
@@ -1202,4 +1202,4 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
 
 
 .. _DBM: https://en.wikipedia.org/wiki/Dbm
-.. _dbm: https://docs.python.org/3/library/dbm.html
+.. _dbm module: https://docs.python.org/3/library/dbm.html

From a5eb59b92d3311f702933753d85b15e6d698faa1 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 23:21:14 +0500
Subject: [PATCH 2321/2974] Fix test_proxy_connect.py for py3.5.

---
 tests/test_proxy_connect.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 5e9470e39..f6381b5b1 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -117,7 +117,7 @@ class ProxyConnectTestCase(TestCase):
         with LogCapture() as l:
             yield crawler.crawl(seed=request)
         self._assert_got_response_code(200, l)
-        echo = json.loads(crawler.spider.meta['responses'][0].body)
+        echo = json.loads(crawler.spider.meta['responses'][0].body.decode('utf-8'))
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
     @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')

From 5eb01b617d207aabc705038d8a0e3b46e358bdbb Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 23:21:30 +0500
Subject: [PATCH 2322/2974] Use an older mitmproxy for py3.5.

---
 tests/requirements-py3.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index f27e45a54..c2b16bec6 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,6 +1,7 @@
 # Tests requirements
 jmespath
-mitmproxy
+mitmproxy; python_version >= '3.6'
+mitmproxy==3.0.4; python_version < '3.6'
 pytest
 pytest-cov
 pytest-twisted

From e0c5c724969ebc35970daaf0203969dc2eb00d56 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:46:19 +0500
Subject: [PATCH 2323/2974] Improve the
 test_https_tunnel_without_leak_proxy_authorization_header change.

---
 tests/test_proxy_connect.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index f6381b5b1..651576c2c 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -117,7 +117,7 @@ class ProxyConnectTestCase(TestCase):
         with LogCapture() as l:
             yield crawler.crawl(seed=request)
         self._assert_got_response_code(200, l)
-        echo = json.loads(crawler.spider.meta['responses'][0].body.decode('utf-8'))
+        echo = json.loads(crawler.spider.meta['responses'][0].text)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
     @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')

From 3c9963ab049e49a1d490d9505ad22ae9c1415421 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:46:38 +0500
Subject: [PATCH 2324/2974] Only xfail test_https_connect_tunnel_error on 3.6+.

---
 tests/test_proxy_connect.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 651576c2c..ec3f0716c 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -92,7 +92,7 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, l)
 
-    @pytest.mark.xfail(reason='Python 3 fails this earlier')
+    @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
         crawler = get_crawler(SimpleSpider)

From 4b0cdf7f3ed94e81612cceacfab6d81d54117865 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:50:56 +0500
Subject: [PATCH 2325/2974] Use self.proc.communicate() after killing mitmdump.

---
 tests/test_proxy_connect.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index ec3f0716c..69925f80c 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -48,8 +48,7 @@ sys.exit(mitmdump())
 
     def stop(self):
         self.proc.kill()
-        self.proc.wait()
-        time.sleep(0.2)
+        self.proc.communicate()
 
 
 def _wrong_credentials(proxy_url):

From 350aa67c3dc8997ef9d3aac9ef3f596c83758e1c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:52:57 +0500
Subject: [PATCH 2326/2974] Rename tests/py3-ignores.txt to tests/ignores.txt.

---
 conftest.py                            | 2 +-
 tests/{py3-ignores.txt => ignores.txt} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename tests/{py3-ignores.txt => ignores.txt} (100%)

diff --git a/conftest.py b/conftest.py
index ede091e9f..7da4c4976 100644
--- a/conftest.py
+++ b/conftest.py
@@ -7,7 +7,7 @@ collect_ignore = [
 ]
 
 
-for line in open('tests/py3-ignores.txt'):
+for line in open('tests/ignores.txt'):
     file_path = line.strip()
     if file_path and file_path[0] != '#':
         collect_ignore.append(file_path)
diff --git a/tests/py3-ignores.txt b/tests/ignores.txt
similarity index 100%
rename from tests/py3-ignores.txt
rename to tests/ignores.txt

From 48b8ac60099c3751ad1f595a9a629be26ffb34cd Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 20:05:37 +0500
Subject: [PATCH 2327/2974] Improve the dbm module ref.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/downloader-middleware.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index e93645077..ae6d41809 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -474,7 +474,7 @@ DBM storage backend
 
     A DBM_ storage backend is also available for the HTTP cache middleware.
 
-    By default, it uses the `dbm module`_, but you can change it with the
+    By default, it uses the :mod:`dbm`, but you can change it with the
     :setting:`HTTPCACHE_DBM_MODULE` setting.
 
 .. _httpcache-storage-custom:
@@ -1202,4 +1202,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
 
 
 .. _DBM: https://en.wikipedia.org/wiki/Dbm
-.. _dbm module: https://docs.python.org/3/library/dbm.html

From 415526d922b6c70bfd5872c8d67d69fcce0ee1fb Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 2 Nov 2019 23:25:15 -0300
Subject: [PATCH 2328/2974] Remove __future__ imports

---
 extras/qps-bench-server.py                | 1 -
 scrapy/cmdline.py                         | 1 -
 scrapy/commands/check.py                  | 1 -
 scrapy/commands/fetch.py                  | 1 -
 scrapy/commands/genspider.py              | 1 -
 scrapy/commands/list.py                   | 1 -
 scrapy/commands/parse.py                  | 1 -
 scrapy/commands/settings.py               | 1 -
 scrapy/commands/startproject.py           | 1 -
 scrapy/commands/version.py                | 2 --
 scrapy/core/downloader/__init__.py        | 1 -
 scrapy/core/downloader/handlers/http.py   | 1 -
 scrapy/downloadermiddlewares/ajaxcrawl.py | 1 -
 scrapy/dupefilters.py                     | 1 -
 scrapy/extensions/httpcache.py            | 2 --
 scrapy/pipelines/media.py                 | 2 --
 scrapy/responsetypes.py                   | 1 -
 scrapy/shell.py                           | 2 --
 scrapy/signalmanager.py                   | 1 -
 scrapy/spiderloader.py                    | 1 -
 scrapy/utils/boto.py                      | 2 --
 scrapy/utils/display.py                   | 1 -
 scrapy/utils/engine.py                    | 1 -
 scrapy/utils/ossignal.py                  | 2 --
 scrapy/utils/request.py                   | 1 -
 scrapy/utils/test.py                      | 1 -
 scrapy/utils/testproc.py                  | 1 -
 scrapy/utils/testsite.py                  | 1 -
 scrapy/utils/trackref.py                  | 1 -
 29 files changed, 35 deletions(-)

diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py
index 3bef20bf3..da7a0022b 100755
--- a/extras/qps-bench-server.py
+++ b/extras/qps-bench-server.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import print_function
 from time import time
 from collections import deque
 from twisted.web.server import Site, NOT_DONE_YET
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 418dc1ac9..69e917004 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import sys
 import os
 import optparse
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index ab73e85e7..ac2a95c9b 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import time
 import sys
 from collections import defaultdict
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index d45133e0e..95f6f7b9a 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import sys
 from w3lib.url import is_url
 
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index d5498bb5c..adb01fa70 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import os
 import shutil
 import string
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index a255b3b94..60686f109 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 from scrapy.commands import ScrapyCommand
 
 class Command(ScrapyCommand):
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index ef8acd29c..ff6f1d8cd 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import json
 import logging
 
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index bee52f06a..a34338715 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import json
 
 from scrapy.commands import ScrapyCommand
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 67337c26e..34df25cef 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import re
 import os
 import string
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 577365c3b..494855500 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import scrapy
 from scrapy.commands import ScrapyCommand
 from scrapy.utils.versions import scrapy_components_versions
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 949dacbc8..f5f8be6e8 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 import random
 import warnings
 from time import time
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index ac4b867c3..6111e132a 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -1,3 +1,2 @@
-from __future__ import absolute_import
 from .http10 import HTTP10DownloadHandler
 from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 72715dba7..78b802673 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
 import re
 import logging
 
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 0bcdd3495..f8802eb7d 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import os
 import logging
 
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index f3fabf710..b1ed0b9f8 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import gzip
 import logging
 import os
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 95dca9a3f..c174addf9 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import functools
 import logging
 from collections import defaultdict
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index de62276c8..b64fbbd42 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -2,7 +2,6 @@
 This module implements a class which returns the appropriate Response class
 based on different criteria.
 """
-from __future__ import absolute_import
 from mimetypes import MimeTypes
 from pkgutil import get_data
 from io import StringIO
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 80b625633..a649d555f 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -3,8 +3,6 @@
 See documentation in docs/topics/shell.rst
 
 """
-from __future__ import print_function
-
 import os
 import signal
 import warnings
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index 296d27ed8..d474f1806 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 from pydispatch import dispatcher
 from scrapy.utils import signal as _signal
 
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 7478faa78..3beca4060 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
 from collections import defaultdict
 import traceback
 import warnings
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index c8fc911bb..46816b54d 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -1,7 +1,5 @@
 """Boto/botocore helpers"""
 
-from __future__ import absolute_import
-
 from scrapy.exceptions import NotConfigured
 
 
diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py
index f6a6c4645..536de6b88 100644
--- a/scrapy/utils/display.py
+++ b/scrapy/utils/display.py
@@ -2,7 +2,6 @@
 pprint and pformat wrappers with colorization support
 """
 
-from __future__ import print_function
 import sys
 from pprint import pformat as pformat_
 
diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py
index 11dd36d91..36ef8626a 100644
--- a/scrapy/utils/engine.py
+++ b/scrapy/utils/engine.py
@@ -1,6 +1,5 @@
 """Some debugging functions for working with the Scrapy engine"""
 
-from __future__ import print_function
 from time import time # used in global tests code
 
 def get_engine_status(engine):
diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py
index f87d5a803..7a7aec9be 100644
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@@ -1,5 +1,3 @@
-
-from __future__ import absolute_import
 import signal
 
 from twisted.internet import reactor
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 63d0ae772..45f1ef17e 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -3,7 +3,6 @@ This module provides some useful functions for working with
 scrapy.http.Request objects
 """
 
-from __future__ import print_function
 import hashlib
 import weakref
 from six.moves.urllib.parse import urlunparse
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 4b935c51b..febaa4dcc 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -2,7 +2,6 @@
 This module contains some assorted functions used in tests
 """
 
-from __future__ import absolute_import
 import os
 
 from importlib import import_module
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index f268e91ff..0f15cf60a 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 import sys
 import os
 
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index e50a989b3..05d06d53b 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 from six.moves.urllib.parse import urljoin
 
 from twisted.internet import reactor
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index eed14c5a1..78389e464 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -9,7 +9,6 @@ and no performance penalty at all when disabled (as object_ref becomes just an
 alias to object in that case).
 """
 
-from __future__ import print_function
 import weakref
 from time import time
 from operator import itemgetter

From c0bfaef37abe029c98072e7b1a1bf346ab79016c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 2 Nov 2019 23:27:04 -0300
Subject: [PATCH 2329/2974] Remove __future__ imports from tests

---
 tests/mockserver.py                          | 1 -
 tests/test_downloadermiddleware_httpcache.py | 1 -
 tests/test_downloadermiddleware_robotstxt.py | 2 --
 tests/test_engine.py                         | 1 -
 tests/test_exporters.py                      | 1 -
 tests/test_feedexport.py                     | 1 -
 tests/test_pipeline_media.py                 | 2 --
 tests/test_utils_deprecate.py                | 1 -
 tests/test_utils_log.py                      | 1 -
 tests/test_utils_request.py                  | 1 -
 10 files changed, 12 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 77908284b..8aff7b525 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import sys, time, random, os, json
 from six.moves.urllib.parse import urlencode
 from subprocess import Popen, PIPE
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 950664ffe..db0843b57 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import time
 import tempfile
 import shutil
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 8266bf35f..a1645ed96 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,6 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
-
 from unittest import mock
 
 from twisted.internet import reactor, error
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 30150391a..d5b911a40 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -10,7 +10,6 @@ module with the ``runserver`` argument::
     python test_engine.py runserver
 """
 
-from __future__ import print_function
 import sys, os, re
 from six.moves.urllib.parse import urlparse
 
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 0046c5666..7880c5bf8 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 import re
 import json
 import marshal
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index abe2ab557..b13b16b41 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 import os
 import csv
 import json
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index ad2618ec9..70f11466b 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import sys
 
 from testfixtures import LogCapture
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index ce04e7f29..159ef8f25 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
 import inspect
 import unittest
 from unittest import mock
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 742e04803..2c23f3616 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import print_function
 import sys
 import logging
 import unittest
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 625a32048..446497086 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import unittest
 from scrapy.http import Request
 from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \

From df00389c16fc8e74c5ab7737e5bffcbde6cc011b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 2 Nov 2019 23:48:11 -0300
Subject: [PATCH 2330/2974] Remove six.moves occurrences

---
 scrapy/_monkeypatches.py                  |  2 +-
 scrapy/commands/bench.py                  |  3 +--
 scrapy/core/downloader/handlers/ftp.py    |  2 +-
 scrapy/core/downloader/handlers/http11.py |  4 ++--
 scrapy/core/downloader/handlers/s3.py     |  2 +-
 scrapy/core/downloader/webclient.py       |  2 +-
 scrapy/downloadermiddlewares/httpproxy.py |  5 ++---
 scrapy/downloadermiddlewares/redirect.py  |  3 ++-
 scrapy/exporters.py                       |  7 ++++---
 scrapy/extensions/feedexport.py           |  2 +-
 scrapy/extensions/httpcache.py            |  2 +-
 scrapy/extensions/spiderstate.py          |  3 ++-
 scrapy/http/cookies.py                    |  5 ++---
 scrapy/http/request/form.py               |  4 ++--
 scrapy/http/request/rpc.py                |  2 +-
 scrapy/http/response/__init__.py          |  2 +-
 scrapy/http/response/text.py              |  4 ++--
 scrapy/linkextractors/__init__.py         |  2 +-
 scrapy/linkextractors/htmlparser.py       |  6 +++---
 scrapy/linkextractors/lxmlhtml.py         |  4 ++--
 scrapy/linkextractors/regex.py            |  3 ++-
 scrapy/linkextractors/sgml.py             |  4 ++--
 scrapy/mail.py                            | 14 +++++++-------
 scrapy/pipelines/files.py                 | 10 +++++-----
 scrapy/robotstxt.py                       |  2 +-
 scrapy/spidermiddlewares/referer.py       |  2 +-
 scrapy/squeues.py                         |  2 +-
 scrapy/utils/benchserver.py               |  3 ++-
 scrapy/utils/curl.py                      |  4 ++--
 scrapy/utils/httpobj.py                   |  2 +-
 scrapy/utils/project.py                   |  6 +++---
 scrapy/utils/python.py                    |  2 +-
 scrapy/utils/request.py                   |  6 +++---
 scrapy/utils/sitemap.py                   |  3 ++-
 scrapy/utils/testsite.py                  |  2 +-
 scrapy/utils/url.py                       |  2 +-
 36 files changed, 68 insertions(+), 65 deletions(-)

diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index 1f8067b35..f74f89bda 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -1,4 +1,4 @@
-from six.moves import copyreg
+import copyreg
 
 
 # Undo what Twisted's perspective broker adds to pickle register
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 90c8d56a2..7bbe362e7 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -1,8 +1,7 @@
 import sys
 import time
 import subprocess
-
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
 
 import scrapy
 from scrapy.commands import ScrapyCommand
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 806a537d4..2116a5a44 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
 
 import re
 from io import BytesIO
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
 
 from twisted.internet import reactor
 from twisted.protocols.ftp import FTPClient, CommandFailed
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 7d917cb74..63dedc19b 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -2,10 +2,10 @@
 
 import re
 import logging
+import warnings
 from io import BytesIO
 from time import time
-import warnings
-from six.moves.urllib.parse import urldefrag
+from urllib.parse import urldefrag
 
 from zope.interface import implementer
 from twisted.internet import defer, reactor, protocol
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index d8bbdd326..85b733228 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 3a5890ed0..9699da109 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -1,5 +1,5 @@
 from time import time
-from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
+from urllib.parse import urlparse, urlunparse, urldefrag
 
 from twisted.web.client import HTTPClientFactory
 from twisted.web.http import HTTPClient
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 2212d9688..814ce78fe 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -1,7 +1,6 @@
 import base64
-from six.moves.urllib.parse import unquote, urlunparse
-from six.moves.urllib.request import getproxies, proxy_bypass
-from urllib.request import _parse_proxy
+from urllib.parse import unquote, urlunparse
+from urllib.request import getproxies, proxy_bypass, _parse_proxy
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index b73f864dd..77cb5aa94 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -1,5 +1,5 @@
 import logging
-from six.moves.urllib.parse import urljoin, urlparse
+from urllib.parse import urljoin, urlparse
 
 from w3lib.url import safe_url_string
 
@@ -7,6 +7,7 @@ from scrapy.http import HtmlResponse
 from scrapy.utils.response import get_meta_refresh
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index f276c28e8..d9531e67c 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -7,15 +7,16 @@ import io
 import sys
 import pprint
 import marshal
-import six
-from six.moves import cPickle as pickle
+import warnings
+from import pickle
 from xml.sax.saxutils import XMLGenerator
 
+import six
+
 from scrapy.utils.serialize import ScrapyJSONEncoder
 from scrapy.utils.python import to_bytes, to_unicode, is_listlike
 from scrapy.item import BaseItem
 from scrapy.exceptions import ScrapyDeprecationWarning
-import warnings
 
 
 __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 07ffd3476..0b854e6f3 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -10,7 +10,7 @@ import logging
 import posixpath
 from tempfile import NamedTemporaryFile
 from datetime import datetime
-from six.moves.urllib.parse import urlparse, unquote
+from urllib.parse import urlparse, unquote
 from ftplib import FTP
 
 from zope.interface import Interface, implementer
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index b1ed0b9f8..b98ec218f 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,13 +1,13 @@
 import gzip
 import logging
 import os
+import pickle
 from email.utils import mktime_tz, parsedate_tz
 from importlib import import_module
 from time import time
 from warnings import warn
 from weakref import WeakKeyDictionary
 
-from six.moves import cPickle as pickle
 from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
 
 from scrapy.exceptions import ScrapyDeprecationWarning
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 2220cbd8f..2c8e46914 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -1,10 +1,11 @@
 import os
-from six.moves import cPickle as pickle
+import pickle
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.job import job_dir
 
+
 class SpiderState(object):
     """Store and load spider state during a scraping job"""
 
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 4532c3ab7..c39de0b52 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -1,7 +1,6 @@
 import time
-from six.moves.http_cookiejar import (
-    CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
-)
+from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
+
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
 
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index b6feede07..d2bcf77b7 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -5,10 +5,10 @@ This module implements the FormRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 
-import six
-from six.moves.urllib.parse import urljoin, urlencode
+from urllib.parse import urljoin, urlencode
 
 import lxml.html
+import six
 from parsel.selector import create_root_node
 from w3lib.html import strip_html5_whitespace
 
diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py
index bd09f7534..811d3ad6b 100644
--- a/scrapy/http/request/rpc.py
+++ b/scrapy/http/request/rpc.py
@@ -4,7 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
 
 See documentation in docs/topics/request-response.rst
 """
-from six.moves import xmlrpc_client as xmlrpclib
+import xmlrpc.client as xmlrpclib
 
 from scrapy.http.request import Request
 from scrapy.utils.python import get_func_args
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index a81404afb..64e9c6c20 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -4,7 +4,7 @@ responses in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 from scrapy.http.request import Request
 from scrapy.http.headers import Headers
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 37f450e54..69bcba577 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -5,10 +5,10 @@ discovering (through HTTP headers) to base Response class.
 See documentation in docs/topics/request-response.rst
 """
 
-import six
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 import parsel
+import six
 from w3lib.encoding import html_to_unicode, resolve_encoding, \
     html_body_declared_encoding, http_content_type_encoding
 from w3lib.html import strip_html5_whitespace
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index ebf3cd7d8..594c7264d 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -6,8 +6,8 @@ This package contains a collection of Link Extractors.
 For more info see docs/topics/link-extractors.rst
 """
 import re
+from urllib.parse import urlparse
 
-from six.moves.urllib.parse import urlparse
 from parsel.csstranslator import HTMLTranslator
 from w3lib.url import canonicalize_url
 
diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 27978a8a1..623732cee 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -2,10 +2,10 @@
 HTMLParser-based link extractor
 """
 import warnings
-import six
-from six.moves.html_parser import HTMLParser
-from six.moves.urllib.parse import urljoin
+from html.parser import HTMLParser
+from urllib.parse import urljoin
 
+import six
 from w3lib.url import safe_url_string
 from w3lib.html import strip_html5_whitespace
 
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 890c019c8..5b7e709de 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -1,9 +1,9 @@
 """
 Link extractor based on lxml.html
 """
-import six
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
+import six
 import lxml.etree as etree
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url
diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py
index e689b4727..f96db256b 100644
--- a/scrapy/linkextractors/regex.py
+++ b/scrapy/linkextractors/regex.py
@@ -1,11 +1,12 @@
 import re
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
 
 from scrapy.link import Link
 from .sgml import SgmlLinkExtractor
 
+
 linkre = re.compile(
         "<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
         re.DOTALL | re.IGNORECASE)
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 8940a4d77..98bed15e9 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -1,11 +1,11 @@
 """
 SGMLParser-based Link extractors
 """
-import six
-from six.moves.urllib.parse import urljoin
 import warnings
+from urllib.parse import urljoin
 from sgmllib import SGMLParser
 
+import six
 from w3lib.url import safe_url_string, canonicalize_url
 from w3lib.html import strip_html5_whitespace
 
diff --git a/scrapy/mail.py b/scrapy/mail.py
index d24de2212..891bb5e09 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -3,21 +3,21 @@ Mail sending helpers
 
 See documentation in docs/topics/email.rst
 """
-from io import BytesIO
 import logging
-
-from email.utils import COMMASPACE, formatdate
-from six.moves.email_mime_multipart import MIMEMultipart
-from six.moves.email_mime_text import MIMEText
-from six.moves.email_mime_base import MIMEBase
-from email.mime.nonmultipart import MIMENonMultipart
 from email import encoders as Encoders
+from email.mime.base import MIMEBase
+from email.mime.multipart import MIMEMultipart
+from email.mime.nonmultipart import MIMENonMultipart
+from email.mime.text import MIMEText
+from email.utils import COMMASPACE, formatdate
+from io import BytesIO
 
 from twisted.internet import defer, reactor, ssl
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
 
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 8d74c5011..432d4c182 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -5,17 +5,17 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
-from io import BytesIO
+import logging
 import mimetypes
 import os
 import os.path
 import time
-import logging
-from email.utils import parsedate_tz, mktime_tz
-from six.moves.urllib.parse import urlparse
 from collections import defaultdict
-import six
+from email.utils import parsedate_tz, mktime_tz
+from io import BytesIO
+from urllib.parse import urlparse
 
+import six
 from twisted.internet import defer, threads
 
 from scrapy.pipelines.media import MediaPipeline
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 95a8c09b8..397924110 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -53,7 +53,7 @@ class RobotParser(with_metaclass(ABCMeta)):
 
 class PythonRobotParser(RobotParser):
     def __init__(self, robotstxt_body, spider):
-        from six.moves.urllib_robotparser import RobotFileParser
+        from urllib.robotparser import RobotFileParser
         self.spider = spider
         robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
         self.rp = RobotFileParser()
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index c76e4d5a2..dce2b3598 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -2,8 +2,8 @@
 RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
-from six.moves.urllib.parse import urlparse
 import warnings
+from urllib.parse import urlparse
 
 from w3lib.url import safe_url_string
 
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 30cc926e5..d5d3be67e 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -3,7 +3,7 @@ Scheduler queues
 """
 
 import marshal
-from six.moves import cPickle as pickle
+import pickle
 
 from queuelib import queue
 
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index 5bbda6e27..cdbe21942 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -1,5 +1,6 @@
 import random
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
+
 from twisted.web.server import Site
 from twisted.web.resource import Resource
 from twisted.internet import reactor
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index b3fd0a497..a0a47e473 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -1,9 +1,9 @@
 import argparse
 import warnings
 from shlex import split
+from http.cookies import SimpleCookie
+from urllib.parse import urlparse
 
-from six.moves.http_cookies import SimpleCookie
-from six.moves.urllib.parse import urlparse
 from six import string_types, iteritems
 from w3lib.http import basic_auth_header
 
diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py
index b4c929b0e..54ffe086d 100644
--- a/scrapy/utils/httpobj.py
+++ b/scrapy/utils/httpobj.py
@@ -1,8 +1,8 @@
 """Helper functions for scrapy.http objects (Request, Response)"""
 
 import weakref
+from urllib.parse import urlparse
 
-from six.moves.urllib.parse import urlparse
 
 _urlparse_cache = weakref.WeakKeyDictionary()
 def urlparse_cached(request_or_response):
diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index 1cbda141a..f28c2eaa1 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -1,5 +1,5 @@
 import os
-from six.moves import cPickle as pickle
+import pickle
 import warnings
 
 from importlib import import_module
@@ -7,8 +7,8 @@ from os.path import join, dirname, abspath, isabs, exists
 
 from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
 from scrapy.settings import Settings
-from scrapy.exceptions import NotConfigured
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+
 
 ENVVAR = 'SCRAPY_SETTINGS_MODULE'
 DATADIR_CFG_SECTION = 'datadir'
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 974abaeb1..cb8ac3c8e 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -65,7 +65,7 @@ def is_listlike(x):
     True
     >>> is_listlike((x for x in range(3)))
     True
-    >>> is_listlike(six.moves.xrange(5))
+    >>> is_listlike(range(5))
     True
     """
     return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 45f1ef17e..c5e877acd 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -5,13 +5,13 @@ scrapy.http.Request objects
 
 import hashlib
 import weakref
-from six.moves.urllib.parse import urlunparse
+from urllib.parse import urlunparse
 
 from w3lib.http import basic_auth_header
-from scrapy.utils.python import to_bytes, to_unicode
-
 from w3lib.url import canonicalize_url
+
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_bytes, to_unicode
 
 
 _fingerprint_cache = weakref.WeakKeyDictionary()
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index 4742b3e13..2f10cf4de 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -5,8 +5,9 @@ Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
 
+from urllib.parse import urljoin
+
 import lxml.etree
-from six.moves.urllib.parse import urljoin
 
 
 class Sitemap(object):
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index 05d06d53b..6f5c21624 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 from twisted.internet import reactor
 from twisted.web import server, resource, static, util
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index b3a4be007..4b48868fe 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead.
 """
 import posixpath
 import re
-from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse)
+from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this
 # move doesn't break old code

From 5ab0f189ce33f235b847f96b66a243d808b5ce1f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 00:01:09 -0300
Subject: [PATCH 2331/2974] Remove six.moves occurrences from tests

---
 tests/mockserver.py                    | 8 ++++++--
 tests/spiders.py                       | 2 +-
 tests/test_crawl.py                    | 2 +-
 tests/test_engine.py                   | 6 ++++--
 tests/test_exporters.py                | 2 +-
 tests/test_feedexport.py               | 6 +++---
 tests/test_http_cookies.py             | 2 +-
 tests/test_http_request.py             | 8 +++-----
 tests/test_pipeline_files.py           | 4 ++--
 tests/test_proxy_connect.py            | 6 +++---
 tests/test_spidermiddleware_offsite.py | 9 ++++-----
 tests/test_spidermiddleware_referer.py | 2 +-
 tests/test_urlparse_monkeypatches.py   | 2 +-
 tests/test_utils_datatypes.py          | 8 --------
 tests/test_utils_defer.py              | 6 ++----
 tests/test_utils_httpobj.py            | 3 ++-
 tests/test_utils_response.py           | 3 ++-
 tests/test_utils_url.py                | 3 ++-
 18 files changed, 39 insertions(+), 43 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 8aff7b525..15b1b24f5 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,6 +1,10 @@
-import sys, time, random, os, json
-from six.moves.urllib.parse import urlencode
+import json
+import os
+import random
+import sys
+import time
 from subprocess import Popen, PIPE
+from urllib.parse import urlencode
 
 from OpenSSL import SSL
 from twisted.web.server import Site, NOT_DONE_YET
diff --git a/tests/spiders.py b/tests/spiders.py
index 7816bf7c7..72a428c50 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -3,7 +3,7 @@ Some spiders used for testing and benchmarking
 """
 
 import time
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
 
 from scrapy.spiders import Spider
 from scrapy.http import Request
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3fc13eeb7..3307899b7 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -140,7 +140,7 @@ class CrawlTestCase(TestCase):
     def test_unbounded_response(self):
         # Completeness of responses without Content-Length or Transfer-Encoding
         # can not be determined, we treat them as valid but flagged as "partial"
-        from six.moves.urllib.parse import urlencode
+        from urllib.parse import urlencode
         query = urlencode({'raw': '''\
 HTTP/1.1 200 OK
 Server: Apache-Coyote/1.1
diff --git a/tests/test_engine.py b/tests/test_engine.py
index d5b911a40..002c4e6bb 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -10,8 +10,10 @@ module with the ``runserver`` argument::
     python test_engine.py runserver
 """
 
-import sys, os, re
-from six.moves.urllib.parse import urlparse
+import os
+import re
+import sys
+from urllib.parse import urlparse
 
 from twisted.internet import reactor, defer
 from twisted.web import server, static, util
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 7880c5bf8..f151a1285 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -1,11 +1,11 @@
 import re
 import json
 import marshal
+import pickle
 import tempfile
 import unittest
 from io import BytesIO
 from datetime import datetime
-from six.moves import cPickle as pickle
 
 import lxml.etree
 import six
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index b13b16b41..7a71669c3 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -2,13 +2,13 @@ import os
 import csv
 import json
 import warnings
-from io import BytesIO
 import tempfile
 import shutil
 import string
+from io import BytesIO
 from unittest import mock
-from six.moves.urllib.parse import urljoin, urlparse, quote
-from six.moves.urllib.request import pathname2url
+from urllib.parse import urljoin, urlparse, quote
+from urllib.request import pathname2url
 
 from zope.interface.verify import verifyObject
 from twisted.trial import unittest
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index 0a9ed500a..45ddb42ba 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 from unittest import TestCase
 
 from scrapy.http import Request, Response
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 807265981..62d2847d7 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -3,12 +3,10 @@ import cgi
 import unittest
 import re
 import json
-from unittest import mock
-from urllib.parse import unquote_to_bytes
+import xmlrpc.client as xmlrpclib
 import warnings
-
-from six.moves import xmlrpc_client as xmlrpclib
-from six.moves.urllib.parse import urlparse, parse_qs, unquote
+from unittest import mock
+from urllib.parse import parse_qs, unquote, unquote_to_bytes, urlparse
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_unicode
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index bd40e4103..dede4bf12 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -4,9 +4,9 @@ import time
 from tempfile import mkdtemp
 from shutil import rmtree
 from unittest import mock
-from six.moves.urllib.parse import urlparse
-from six import BytesIO
+from urllib.parse import urlparse
 
+from six import BytesIO
 from twisted.trial import unittest
 from twisted.internet import defer
 
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index ae1236bcb..2b62a378d 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -1,15 +1,15 @@
 import json
 import os
 import time
-
-from six.moves.urllib.parse import urlsplit, urlunsplit
+from urllib.parse import urlsplit, urlunsplit
 from threading import Thread
+
 from libmproxy import controller, proxy
 from netlib import http_auth
 from testfixtures import LogCapture
-
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
+
 from scrapy.utils.test import get_crawler
 from scrapy.http import Request
 from tests.spiders import SimpleSpider, SingleRequestSpider
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 7e4af0d4c..5833591af 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -1,13 +1,12 @@
 from unittest import TestCase
-
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
+import warnings
 
 from scrapy.http import Response, Request
 from scrapy.spiders import Spider
-from scrapy.spidermiddlewares.offsite import OffsiteMiddleware
-from scrapy.spidermiddlewares.offsite import URLWarning
+from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning
 from scrapy.utils.test import get_crawler
-import warnings
+
 
 class TestOffsiteMiddleware(TestCase):
 
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 21439c20e..23c38d9b1 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 from unittest import TestCase
 import warnings
 
diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py
index 22e39821c..bea0cf3e5 100644
--- a/tests/test_urlparse_monkeypatches.py
+++ b/tests/test_urlparse_monkeypatches.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 import unittest
 
 
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 47877f555..8782e4c53 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -192,14 +192,6 @@ class SequenceExcludeTest(unittest.TestCase):
         self.assertIn(20, d)
         self.assertNotIn(15, d)
 
-    def test_six_range(self):
-        import six.moves
-        seq = six.moves.range(10**3, 10**6)
-        d = SequenceExclude(seq)
-        self.assertIn(10**2, d)
-        self.assertIn(10**7, d)
-        self.assertNotIn(10**4, d)
-
     def test_range_step(self):
         seq = range(10, 20, 3)
         d = SequenceExclude(seq)
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 003bb9b02..0d8c46657 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -5,8 +5,6 @@ from twisted.python.failure import Failure
 from scrapy.utils.defer import mustbe_deferred, process_chain, \
     process_chain_both, process_parallel, iter_errback
 
-from six.moves import xrange
-
 
 class MustbeDeferredTest(unittest.TestCase):
     def test_success_function(self):
@@ -83,7 +81,7 @@ class IterErrbackTest(unittest.TestCase):
 
     def test_iter_errback_good(self):
         def itergood():
-            for x in xrange(10):
+            for x in range(10):
                 yield x
 
         errors = []
@@ -93,7 +91,7 @@ class IterErrbackTest(unittest.TestCase):
 
     def test_iter_errback_bad(self):
         def iterbad():
-            for x in xrange(10):
+            for x in range(10):
                 if x == 5:
                     a = 1/0
                 yield x
diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py
index 4f9f7a370..cf8ad1f23 100644
--- a/tests/test_utils_httpobj.py
+++ b/tests/test_utils_httpobj.py
@@ -1,9 +1,10 @@
 import unittest
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 
+
 class HttpobjUtilsTest(unittest.TestCase):
 
     def test_urlparse_cached(self):
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index bea4dade3..6ebf290c0 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -1,12 +1,13 @@
 import os
 import unittest
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 
 from scrapy.http import Response, TextResponse, HtmlResponse
 from scrapy.utils.python import to_bytes
 from scrapy.utils.response import (response_httprepr, open_in_browser,
                                    get_meta_refresh, get_base_url, response_status_message)
 
+
 __doctests__ = ['scrapy.utils.response']
 
 
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index e6588055c..a8e37d7b8 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,14 +1,15 @@
 # -*- coding: utf-8 -*-
 import unittest
+from urllib.parse import urlparse
 
 import six
-from six.moves.urllib.parse import urlparse
 
 from scrapy.spiders import Spider
 from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
                               add_http_if_no_scheme, guess_scheme,
                               parse_url, strip_url)
 
+
 __doctests__ = ['scrapy.utils.url']
 
 

From 1aba5136939ff26503a59ca9fb9320a030aa77de Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 00:26:44 -0300
Subject: [PATCH 2332/2974] Remove six.iter* occurrences

---
 scrapy/core/downloader/__init__.py            |  3 +--
 scrapy/core/downloader/handlers/__init__.py   |  5 +++--
 scrapy/downloadermiddlewares/cookies.py       |  8 +++++---
 scrapy/downloadermiddlewares/decompression.py |  7 +++----
 scrapy/exporters.py                           |  6 +++---
 scrapy/extensions/memdebug.py                 |  3 +--
 scrapy/http/request/form.py                   |  3 +--
 scrapy/item.py                                |  2 +-
 scrapy/loader/__init__.py                     |  6 ++----
 scrapy/pipelines/files.py                     | 11 +++++------
 scrapy/pipelines/images.py                    |  3 +--
 scrapy/responsetypes.py                       |  3 +--
 scrapy/settings/__init__.py                   |  8 ++++----
 scrapy/utils/conf.py                          | 12 +++++-------
 scrapy/utils/datatypes.py                     |  7 ++-----
 scrapy/utils/python.py                        |  4 ++--
 scrapy/utils/spider.py                        |  5 ++---
 scrapy/utils/trackref.py                      | 13 ++++++-------
 tests/test_settings/__init__.py               | 10 +++++-----
 tests/test_webclient.py                       |  2 +-
 20 files changed, 54 insertions(+), 67 deletions(-)

diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index f5f8be6e8..73d84664f 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -4,7 +4,6 @@ from time import time
 from datetime import datetime
 from collections import deque
 
-import six
 from twisted.internet import reactor, defer, task
 
 from scrapy.utils.defer import mustbe_deferred
@@ -189,7 +188,7 @@ class Downloader(object):
 
     def close(self):
         self._slot_gc_loop.stop()
-        for slot in six.itervalues(self.slots):
+        for slot in self.slots.values():
             slot.close()
 
     def _slot_gc(self, age=60):
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 0b55d32fa..39a0b1f51 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -1,8 +1,9 @@
 """Download handlers for different schemes"""
 
 import logging
+
 from twisted.internet import defer
-import six
+
 from scrapy.exceptions import NotSupported, NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
@@ -22,7 +23,7 @@ class DownloadHandlers(object):
         self._notconfigured = {}  # remembers failed handlers
         handlers = without_none_values(
             crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
-        for scheme, clspath in six.iteritems(handlers):
+        for scheme, clspath in handlers.items():
             self._schemes[scheme] = clspath
             self._load_handler(scheme, skip_lazy=True)
 
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 0d2b9900c..9deba40d6 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -1,5 +1,4 @@
 import os
-import six
 import logging
 from collections import defaultdict
 
@@ -8,6 +7,7 @@ from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
 from scrapy.utils.python import to_unicode
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -82,8 +82,10 @@ class CookiesMiddleware(object):
 
     def _get_request_cookies(self, jar, request):
         if isinstance(request.cookies, dict):
-            cookie_list = [{'name': k, 'value': v} for k, v in \
-                    six.iteritems(request.cookies)]
+            cookie_list = [
+                {'name': k, 'value': v}
+                for k, v in request.cookies.items()
+            ]
         else:
             cookie_list = request.cookies
 
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index e2d73f347..dfd64c35f 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -4,16 +4,15 @@ and extract the potentially compressed responses that may arrive.
 
 import bz2
 import gzip
-from io import BytesIO
 import zipfile
 import tarfile
 import logging
+from io import BytesIO
 from tempfile import mktemp
 
-import six
-
 from scrapy.responsetypes import responsetypes
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -75,7 +74,7 @@ class DecompressionMiddleware(object):
         if not response.body:
             return response
 
-        for fmt, func in six.iteritems(self._formats):
+        for fmt, func in self._formats.items():
             new_response = func(response)
             if new_response:
                 logger.debug('Decompressed response with format: %(responsefmt)s',
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index d9531e67c..19af2d6e4 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -62,9 +62,9 @@ class BaseItemExporter(object):
             include_empty = self.export_empty_fields
         if self.fields_to_export is None:
             if include_empty and not isinstance(item, dict):
-                field_iter = six.iterkeys(item.fields)
+                field_iter = item.fields.keys()
             else:
-                field_iter = six.iterkeys(item)
+                field_iter = item.keys()
         else:
             if include_empty:
                 field_iter = self.fields_to_export
@@ -326,7 +326,7 @@ class PythonItemExporter(BaseItemExporter):
         return value
 
     def _serialize_dict(self, value):
-        for key, val in six.iteritems(value):
+        for key, val in value.items():
             key = to_bytes(key) if self.binary else key
             yield key, self._serialize_value(val)
 
diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py
index 263d8ce4c..892aa8a86 100644
--- a/scrapy/extensions/memdebug.py
+++ b/scrapy/extensions/memdebug.py
@@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst
 """
 
 import gc
-import six
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
@@ -28,7 +27,7 @@ class MemoryDebugger(object):
     def spider_closed(self, spider, reason):
         gc.collect()
         self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
-        for cls, wdict in six.iteritems(live_refs):
+        for cls, wdict in live_refs.items():
             if not wdict:
                 continue
             self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index d2bcf77b7..af02c8484 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -8,7 +8,6 @@ See documentation in docs/topics/request-response.rst
 from urllib.parse import urljoin, urlencode
 
 import lxml.html
-import six
 from parsel.selector import create_root_node
 from w3lib.html import strip_html5_whitespace
 
@@ -208,7 +207,7 @@ def _get_clickable(clickdata, form):
     # We didn't find it, so now we build an XPath expression out of the other
     # arguments, because they can be used as such
     xpath = u'.//*' + \
-            u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))
+            u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
     el = form.xpath(xpath)
     if len(el) == 1:
         return (el[0].get('name'), el[0].get('value') or '')
diff --git a/scrapy/item.py b/scrapy/item.py
index 32f9b2ebb..4e0f0ac44 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -78,7 +78,7 @@ class DictItem(MutableMapping, BaseItem):
     def __init__(self, *args, **kwargs):
         self._values = {}
         if args or kwargs:  # avoid creating dict for most common case
-            for k, v in six.iteritems(dict(*args, **kwargs)):
+            for k, v in dict(*args, **kwargs).items():
                 self[k] = v
 
     def __getitem__(self, key):
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 60fd6d222..fe01a856f 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -5,8 +5,6 @@ See documentation in docs/topics/loaders.rst
 """
 from collections import defaultdict
 
-import six
-
 from scrapy.item import Item
 from scrapy.loader.common import wrap_loader_context
 from scrapy.loader.processors import Identity
@@ -72,7 +70,7 @@ class ItemLoader(object):
         if value is None:
             return
         if not field_name:
-            for k, v in six.iteritems(value):
+            for k, v in value.items():
                 self._add_value(k, v)
         else:
             self._add_value(field_name, value)
@@ -82,7 +80,7 @@ class ItemLoader(object):
         if value is None:
             return
         if not field_name:
-            for k, v in six.iteritems(value):
+            for k, v in value.items():
                 self._replace_value(k, v)
         else:
             self._replace_value(field_name, value)
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 432d4c182..6d55c8980 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -8,14 +8,12 @@ import hashlib
 import logging
 import mimetypes
 import os
-import os.path
 import time
 from collections import defaultdict
 from email.utils import parsedate_tz, mktime_tz
 from io import BytesIO
 from urllib.parse import urlparse
 
-import six
 from twisted.internet import defer, threads
 
 from scrapy.pipelines.media import MediaPipeline
@@ -29,6 +27,7 @@ from scrapy.utils.request import referer_str
 from scrapy.utils.boto import is_botocore
 from scrapy.utils.datatypes import CaselessDict
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -153,14 +152,14 @@ class S3FilesStore(object):
                 Bucket=self.bucket,
                 Key=key_name,
                 Body=buf,
-                Metadata={k: str(v) for k, v in six.iteritems(meta or {})},
+                Metadata={k: str(v) for k, v in (meta or {}).items()},
                 ACL=self.POLICY,
                 **extra)
         else:
             b = self._get_boto_bucket()
             k = b.new_key(key_name)
             if meta:
-                for metakey, metavalue in six.iteritems(meta):
+                for metakey, metavalue in meta.items():
                     k.set_metadata(metakey, str(metavalue))
             h = self.HEADERS.copy()
             if headers:
@@ -201,7 +200,7 @@ class S3FilesStore(object):
             'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation',
         })
         extra = {}
-        for key, value in six.iteritems(headers):
+        for key, value in headers.items():
             try:
                 kwarg = mapping[key]
             except KeyError:
@@ -249,7 +248,7 @@ class GCSFilesStore(object):
     def persist_file(self, path, buf, info, meta=None, headers=None):
         blob = self.bucket.blob(self.prefix + path)
         blob.cache_control = self.CACHE_CONTROL
-        blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}
+        blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
         return threads.deferToThread(
             blob.upload_from_string,
             data=buf.getvalue(),
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index e77cef4ff..e9c6b759c 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -6,7 +6,6 @@ See documentation in topics/media-pipeline.rst
 import functools
 import hashlib
 from io import BytesIO
-import six
 
 from PIL import Image
 
@@ -126,7 +125,7 @@ class ImagesPipeline(FilesPipeline):
         image, buf = self.convert_image(orig_image)
         yield path, image, buf
 
-        for thumb_id, size in six.iteritems(self.thumbs):
+        for thumb_id, size in self.thumbs.items():
             thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
             thumb_image, thumb_buf = self.convert_image(image, size)
             yield thumb_path, thumb_image, thumb_buf
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index b64fbbd42..91d309147 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -5,7 +5,6 @@ based on different criteria.
 from mimetypes import MimeTypes
 from pkgutil import get_data
 from io import StringIO
-import six
 
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
@@ -36,7 +35,7 @@ class ResponseTypes(object):
         self.mimetypes = MimeTypes()
         mimedata = get_data('scrapy', 'mime.types').decode('utf8')
         self.mimetypes.readfp(StringIO(mimedata))
-        for mimetype, cls in six.iteritems(self.CLASSES):
+        for mimetype, cls in self.CLASSES.items():
             self.classes[mimetype] = load_object(cls)
 
     def from_mimetype(self, mimetype):
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index c871e86e0..d53b28895 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -317,10 +317,10 @@ class BaseSettings(MutableMapping):
             values = json.loads(values)
         if values is not None:
             if isinstance(values, BaseSettings):
-                for name, value in six.iteritems(values):
+                for name, value in values.items():
                     self.set(name, value, values.getpriority(name))
             else:
-                for name, value in six.iteritems(values):
+                for name, value in values.items():
                     self.set(name, value, priority)
 
     def delete(self, name, priority='project'):
@@ -377,7 +377,7 @@ class BaseSettings(MutableMapping):
 
     def _to_dict(self):
         return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
-                for k, v in six.iteritems(self)}
+                for k, v in self.items()}
 
     def copy_to_dict(self):
         """
@@ -445,7 +445,7 @@ class Settings(BaseSettings):
         self.setmodule(default_settings, 'default')
         # Promote default dictionaries to BaseSettings instances for per-key
         # priorities
-        for name, val in six.iteritems(self):
+        for name, val in self.items():
             if isinstance(val, dict):
                 self.set(name, BaseSettings(val, 'default'), 'default')
         self.update(values, priority)
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 561bb72fc..7a15e77ff 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -1,11 +1,9 @@
-from configparser import ConfigParser
 import os
 import sys
 import numbers
+from configparser import ConfigParser
 from operator import itemgetter
 
-import six
-
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
 from scrapy.utils.python import without_none_values
@@ -22,7 +20,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
     def _map_keys(compdict):
         if isinstance(compdict, BaseSettings):
             compbs = BaseSettings()
-            for k, v in six.iteritems(compdict):
+            for k, v in compdict.items():
                 prio = compdict.getpriority(k)
                 if compbs.getpriority(convert(k)) == prio:
                     raise ValueError('Some paths in {!r} convert to the same '
@@ -33,11 +31,11 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
             return compbs
         else:
             _check_components(compdict)
-            return {convert(k): v for k, v in six.iteritems(compdict)}
+            return {convert(k): v for k, v in compdict.items()}
 
     def _validate_values(compdict):
         """Fail if a value in the components dict is not a real number or None."""
-        for name, value in six.iteritems(compdict):
+        for name, value in compdict.items():
             if value is not None and not isinstance(value, numbers.Real):
                 raise ValueError('Invalid value {} for component {}, please provide ' \
                                  'a real number or None instead'.format(value, name))
@@ -53,7 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
 
     _validate_values(compdict)
     compdict = without_none_values(_map_keys(compdict))
-    return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
+    return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]
 
 
 def arglist_to_dict(arglist):
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 6e9de47f3..56d4d1b8e 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -7,11 +7,8 @@ This module must not depend on any module outside the Standard Library.
 
 import copy
 import collections
-from collections.abc import Mapping
 import warnings
 
-import six
-
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
@@ -151,7 +148,7 @@ class MultiValueDict(dict):
                         self.setlistdefault(key, []).append(value)
                 except TypeError:
                     raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary")
-        for key, value in six.iteritems(kwargs):
+        for key, value in kwargs.items():
             self.setlistdefault(key, []).append(value)
 
 
@@ -226,7 +223,7 @@ class CaselessDict(dict):
         return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
 
     def update(self, seq):
-        seq = seq.items() if isinstance(seq, Mapping) else seq
+        seq = seq.items() if isinstance(seq, collections.abc.Mapping) else seq
         iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
         super(CaselessDict, self).update(iseq)
 
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index cb8ac3c8e..845c19fb9 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -300,7 +300,7 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
     dict or a list of tuples, like any dict constructor supports.
     """
     d = {}
-    for k, v in six.iteritems(dict(dct_or_tuples)):
+    for k, v in dict(dct_or_tuples).items():
         k = k.encode(encoding) if isinstance(k, six.text_type) else k
         if not keys_only:
             v = v.encode(encoding) if isinstance(v, six.text_type) else v
@@ -345,7 +345,7 @@ def without_none_values(iterable):
     value ``None`` have been removed.
     """
     try:
-        return {k: v for k, v in six.iteritems(iterable) if v is not None}
+        return {k: v for k, v in iterable.items() if v is not None}
     except AttributeError:
         return type(iterable)((v for v in iterable if v is not None))
 
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 94b24f67e..48ad5041e 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -1,11 +1,10 @@
 import logging
 import inspect
 
-import six
-
 from scrapy.spiders import Spider
 from scrapy.utils.misc import  arg_to_iter
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -21,7 +20,7 @@ def iter_spider_classes(module):
     # singleton in scrapy.spider.spiders
     from scrapy.spiders import Spider
 
-    for obj in six.itervalues(vars(module)):
+    for obj in vars(module).values():
         if inspect.isclass(obj) and \
            issubclass(obj, Spider) and \
            obj.__module__ == module.__name__ and \
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index 78389e464..4842b95df 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -13,7 +13,6 @@ import weakref
 from time import time
 from operator import itemgetter
 from collections import defaultdict
-import six
 
 
 NoneType = type(None)
@@ -36,13 +35,13 @@ def format_live_refs(ignore=NoneType):
     """Return a tabular representation of tracked objects"""
     s = "Live References\n\n"
     now = time()
-    for cls, wdict in sorted(six.iteritems(live_refs),
+    for cls, wdict in sorted(live_refs.items(),
                              key=lambda x: x[0].__name__):
         if not wdict:
             continue
         if issubclass(cls, ignore):
             continue
-        oldest = min(six.itervalues(wdict))
+        oldest = min(wdict.values())
         s += "%-30s %6d   oldest: %ds ago\n" % (
             cls.__name__, len(wdict), now - oldest
         )
@@ -56,15 +55,15 @@ def print_live_refs(*a, **kw):
 
 def get_oldest(class_name):
     """Get the oldest object for a specific class name"""
-    for cls, wdict in six.iteritems(live_refs):
+    for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
             if not wdict:
                 break
-            return min(six.iteritems(wdict), key=itemgetter(1))[0]
+            return min(wdict.items(), key=itemgetter(1))[0]
 
 
 def iter_all(class_name):
     """Iterate over all objects of the same class by its class name"""
-    for cls, wdict in six.iteritems(live_refs):
+    for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
-            return six.iterkeys(wdict)
+            return wdict.keys()
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 32e65bed5..d5cbef6f5 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -10,7 +10,7 @@ from . import default_settings
 class SettingsGlobalFuncsTest(unittest.TestCase):
 
     def test_get_settings_priority(self):
-        for prio_str, prio_num in six.iteritems(SETTINGS_PRIORITIES):
+        for prio_str, prio_num in SETTINGS_PRIORITIES.items():
             self.assertEqual(get_settings_priority(prio_str), prio_num)
         self.assertEqual(get_settings_priority(99), 99)
 
@@ -148,10 +148,10 @@ class BaseSettingsTest(unittest.TestCase):
         self.settings.setmodule(
             'tests.test_settings.default_settings', 10)
 
-        self.assertCountEqual(six.iterkeys(self.settings.attributes),
-                              six.iterkeys(ctrl_attributes))
+        self.assertCountEqual(self.settings.attributes.keys(),
+                              ctrl_attributes.keys())
 
-        for key in six.iterkeys(ctrl_attributes):
+        for key in ctrl_attributes.keys():
             attr = self.settings.attributes[key]
             ctrl_attr = ctrl_attributes[key]
             self.assertEqual(attr.value, ctrl_attr.value)
@@ -227,7 +227,7 @@ class BaseSettingsTest(unittest.TestCase):
         }
         settings = self.settings
         settings.attributes = {key: SettingsAttribute(value, 0) for key, value
-                               in six.iteritems(test_configuration)}
+                               in test_configuration.items()}
 
         self.assertTrue(settings.getbool('TEST_ENABLED1'))
         self.assertTrue(settings.getbool('TEST_ENABLED2'))
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index a81946490..0c04e7114 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -318,7 +318,7 @@ class WebClientTestCase(unittest.TestCase):
         def cleanup(passthrough):
             # Clean up the server which is hanging around not doing
             # anything.
-            connected = list(six.iterkeys(self.wrapper.protocols))
+            connected = list(self.wrapper.protocols.keys())
             # There might be nothing here if the server managed to already see
             # that the connection was lost.
             if connected:

From 68bf192172e548da49613064086a6e02cf0bae54 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 00:32:07 -0300
Subject: [PATCH 2333/2974] Fix bad import

---
 scrapy/exporters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 19af2d6e4..2ff089ce9 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -8,7 +8,7 @@ import sys
 import pprint
 import marshal
 import warnings
-from import pickle
+import pickle
 from xml.sax.saxutils import XMLGenerator
 
 import six

From ce8e515fa8960d0229132ac33fc5f8a6e43bbd6f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 00:36:25 -0300
Subject: [PATCH 2334/2974] Remove six type wrappers

---
 scrapy/crawler.py                   |  2 +-
 scrapy/exporters.py                 |  4 ++--
 scrapy/http/headers.py              |  6 +++---
 scrapy/http/request/__init__.py     |  2 +-
 scrapy/http/response/text.py        |  6 +++---
 scrapy/linkextractors/htmlparser.py |  2 +-
 scrapy/linkextractors/lxmlhtml.py   |  2 +-
 scrapy/linkextractors/sgml.py       |  2 +-
 scrapy/settings/__init__.py         | 10 +++++-----
 scrapy/spiders/crawl.py             |  2 +-
 scrapy/spiders/sitemap.py           |  4 ++--
 scrapy/utils/iterators.py           |  6 +++---
 scrapy/utils/misc.py                |  6 +++---
 scrapy/utils/python.py              | 14 +++++++-------
 14 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 19b998e0d..b88df8f64 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -205,7 +205,7 @@ class CrawlerRunner(object):
         return self._create_crawler(crawler_or_spidercls)
 
     def _create_crawler(self, spidercls):
-        if isinstance(spidercls, six.string_types):
+        if isinstance(spidercls, str):
             spidercls = self.spider_loader.load(spidercls)
         return Crawler(spidercls, self.settings)
 
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 2ff089ce9..e45317491 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -182,7 +182,7 @@ class XmlItemExporter(BaseItemExporter):
             for value in serialized_value:
                 self._export_xml_field('value', value, depth=depth+1)
             self._beautify_indent(depth=depth)
-        elif isinstance(serialized_value, six.text_type):
+        elif isinstance(serialized_value, str):
             self.xg.characters(serialized_value)
         else:
             self.xg.characters(str(serialized_value))
@@ -321,7 +321,7 @@ class PythonItemExporter(BaseItemExporter):
         if is_listlike(value):
             return [self._serialize_value(v) for v in value]
         encode_func = to_bytes if self.binary else to_unicode
-        if isinstance(value, (six.text_type, bytes)):
+        if isinstance(value, (str, bytes)):
             return encode_func(value, encoding=self.encoding)
         return value
 
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 62507eb19..5bbe7d72a 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -19,7 +19,7 @@ class Headers(CaselessDict):
         """Normalize values to bytes"""
         if value is None:
             value = []
-        elif isinstance(value, (six.text_type, bytes)):
+        elif isinstance(value, (str, bytes)):
             value = [value]
         elif not hasattr(value, '__iter__'):
             value = [value]
@@ -29,10 +29,10 @@ class Headers(CaselessDict):
     def _tobytes(self, x):
         if isinstance(x, bytes):
             return x
-        elif isinstance(x, six.text_type):
+        elif isinstance(x, str):
             return x.encode(self.encoding)
         elif isinstance(x, int):
-            return six.text_type(x).encode(self.encoding)
+            return str(x).encode(self.encoding)
         else:
             raise TypeError('Unsupported value type: {}'.format(type(x)))
 
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index d09eaf849..ff7a44545 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -60,7 +60,7 @@ class Request(object_ref):
         return self._url
 
     def _set_url(self, url):
-        if not isinstance(url, six.string_types):
+        if not isinstance(url, str):
             raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
 
         s = safe_url_string(url, self.encoding)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 69bcba577..65400803f 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -31,14 +31,14 @@ class TextResponse(Response):
         super(TextResponse, self).__init__(*args, **kwargs)
 
     def _set_url(self, url):
-        if isinstance(url, six.text_type):
+        if isinstance(url, str):
             self._url = to_unicode(url, self.encoding)
         else:
             super(TextResponse, self)._set_url(url)
 
     def _set_body(self, body):
         self._body = b''  # used by encoding detection
-        if isinstance(body, six.text_type):
+        if isinstance(body, str):
             if self._encoding is None:
                 raise TypeError('Cannot convert unicode body - %s has no encoding' %
                     type(self).__name__)
@@ -158,7 +158,7 @@ class TextResponse(Response):
 
 def _url_from_selector(sel):
     # type: (parsel.Selector) -> str
-    if isinstance(sel.root, six.string_types):
+    if isinstance(sel.root, str):
         # e.g. ::attr(href) result
         return strip_html5_whitespace(sel.root)
     if not hasattr(sel.root, 'tag'):
diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 623732cee..2fec35799 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -42,7 +42,7 @@ class HtmlParserLinkExtractor(HTMLParser):
         ret = []
         base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
         for link in links:
-            if isinstance(link.url, six.text_type):
+            if isinstance(link.url, str):
                 link.url = link.url.encode(response_encoding)
             try:
                 link.url = urljoin(base_url, link.url)
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 5b7e709de..496ad3053 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -22,7 +22,7 @@ _collect_string_content = etree.XPath("string()")
 
 
 def _nons(tag):
-    if isinstance(tag, six.string_types):
+    if isinstance(tag, str):
         if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
             return tag.split('}')[-1]
     return tag
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 98bed15e9..a9dffcad3 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -49,7 +49,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
         if base_url is None:
             base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
         for link in self.links:
-            if isinstance(link.url, six.text_type):
+            if isinstance(link.url, str):
                 link.url = link.url.encode(response_encoding)
             try:
                 link.url = urljoin(base_url, link.url)
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index d53b28895..c88c9c0e2 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -23,7 +23,7 @@ def get_settings_priority(priority):
     :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its
     numerical value, or directly returns a given numerical priority.
     """
-    if isinstance(priority, six.string_types):
+    if isinstance(priority, str):
         return SETTINGS_PRIORITIES[priority]
     else:
         return priority
@@ -173,7 +173,7 @@ class BaseSettings(MutableMapping):
         :type default: any
         """
         value = self.get(name, default or [])
-        if isinstance(value, six.string_types):
+        if isinstance(value, str):
             value = value.split(',')
         return list(value)
 
@@ -194,7 +194,7 @@ class BaseSettings(MutableMapping):
         :type default: any
         """
         value = self.get(name, default or {})
-        if isinstance(value, six.string_types):
+        if isinstance(value, str):
             value = json.loads(value)
         return dict(value)
 
@@ -284,7 +284,7 @@ class BaseSettings(MutableMapping):
         :type priority: string or int
         """
         self._assert_mutability()
-        if isinstance(module, six.string_types):
+        if isinstance(module, str):
             module = import_module(module)
         for key in dir(module):
             if key.isupper():
@@ -313,7 +313,7 @@ class BaseSettings(MutableMapping):
         :type priority: string or int
         """
         self._assert_mutability()
-        if isinstance(values, six.string_types):
+        if isinstance(values, str):
             values = json.loads(values)
         if values is not None:
             if isinstance(values, BaseSettings):
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 03000ce54..59e2c5661 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -25,7 +25,7 @@ def _identity(request, response):
 def _get_method(method, spider):
     if callable(method):
         return method
-    elif isinstance(method, six.string_types):
+    elif isinstance(method, str):
         return getattr(spider, method, None)
 
 
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 534c45c70..2917daf57 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -22,7 +22,7 @@ class SitemapSpider(Spider):
         super(SitemapSpider, self).__init__(*a, **kw)
         self._cbs = []
         for r, c in self.sitemap_rules:
-            if isinstance(c, six.string_types):
+            if isinstance(c, str):
                 c = getattr(self, c)
             self._cbs.append((regex(r), c))
         self._follow = [regex(x) for x in self.sitemap_follow]
@@ -86,7 +86,7 @@ class SitemapSpider(Spider):
 
 
 def regex(x):
-    if isinstance(x, six.string_types):
+    if isinstance(x, str):
         return re.compile(x)
     return x
 
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 9693ba768..10481fe8d 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -60,7 +60,7 @@ class _StreamReader(object):
             self._text, self.encoding = obj.body, obj.encoding
         else:
             self._text, self.encoding = obj, 'utf-8'
-        self._is_unicode = isinstance(self._text, six.text_type)
+        self._is_unicode = isinstance(self._text, str)
 
     def read(self, n=65535):
         self.read = self._read_unicode if self._is_unicode else self._read_string
@@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
 
 def _body_or_str(obj, unicode=True):
-    expected_types = (Response, six.text_type, six.binary_type)
+    expected_types = (Response, str, bytes)
     assert isinstance(obj, expected_types), \
         "obj must be %s, not %s" % (
             " or ".join(t.__name__ for t in expected_types),
@@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True):
             return obj.text
         else:
             return obj.body.decode('utf-8')
-    elif isinstance(obj, six.text_type):
+    elif isinstance(obj, str):
         return obj if unicode else obj.encode('utf-8')
     else:
         return obj.decode('utf-8') if unicode else obj
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index f638adb25..9a44f3576 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -13,7 +13,7 @@ from scrapy.utils.python import flatten, to_unicode
 from scrapy.item import BaseItem
 
 
-_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes
+_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes
 
 
 def arg_to_iter(arg):
@@ -83,7 +83,7 @@ def extract_regex(regex, text, encoding='utf-8'):
     * if the regex doesn't contain any group the entire regex matching is returned
     """
 
-    if isinstance(regex, six.string_types):
+    if isinstance(regex, str):
         regex = re.compile(regex, re.UNICODE)
 
     try:
@@ -92,7 +92,7 @@ def extract_regex(regex, text, encoding='utf-8'):
         strings = regex.findall(text)    # full regex or numbered groups
     strings = flatten(strings)
 
-    if isinstance(text, six.text_type):
+    if isinstance(text, str):
         return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
     else:
         return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 845c19fb9..18fee1964 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -68,7 +68,7 @@ def is_listlike(x):
     >>> is_listlike(range(5))
     True
     """
-    return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))
+    return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))
 
 
 def unique(list_, key=lambda x: x):
@@ -87,9 +87,9 @@ def unique(list_, key=lambda x: x):
 def to_unicode(text, encoding=None, errors='strict'):
     """Return the unicode representation of a bytes object ``text``. If
     ``text`` is already an unicode object, return it as-is."""
-    if isinstance(text, six.text_type):
+    if isinstance(text, str):
         return text
-    if not isinstance(text, (bytes, six.text_type)):
+    if not isinstance(text, (bytes, str)):
         raise TypeError('to_unicode must receive a bytes or str '
                         'object, got %s' % type(text).__name__)
     if encoding is None:
@@ -102,7 +102,7 @@ def to_bytes(text, encoding=None, errors='strict'):
     is already a bytes object, return it as-is."""
     if isinstance(text, bytes):
         return text
-    if not isinstance(text, six.string_types):
+    if not isinstance(text, str):
         raise TypeError('to_bytes must receive a str or bytes '
                         'object, got %s' % type(text).__name__)
     if encoding is None:
@@ -138,7 +138,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
             yield (text[offset:], offset)
         yield (text, 0)
 
-    if isinstance(pattern, six.string_types):
+    if isinstance(pattern, str):
         pattern = re.compile(pattern)
 
     for chunk, offset in _chunk_iter():
@@ -301,9 +301,9 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
     """
     d = {}
     for k, v in dict(dct_or_tuples).items():
-        k = k.encode(encoding) if isinstance(k, six.text_type) else k
+        k = k.encode(encoding) if isinstance(k, str) else k
         if not keys_only:
-            v = v.encode(encoding) if isinstance(v, six.text_type) else v
+            v = v.encode(encoding) if isinstance(v, str) else v
         d[k] = v
     return d
 

From 54a786b102a3862c94fcdb123a26b54a39892697 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 00:58:47 -0300
Subject: [PATCH 2335/2974] Remove six imports

---
 scrapy/crawler.py                   | 6 +++---
 scrapy/exporters.py                 | 2 --
 scrapy/http/headers.py              | 4 ----
 scrapy/http/request/__init__.py     | 1 -
 scrapy/http/response/text.py        | 1 -
 scrapy/linkextractors/htmlparser.py | 1 -
 scrapy/linkextractors/lxmlhtml.py   | 1 -
 scrapy/linkextractors/sgml.py       | 1 -
 scrapy/settings/__init__.py         | 1 -
 scrapy/spiders/crawl.py             | 2 --
 scrapy/spiders/sitemap.py           | 1 -
 scrapy/utils/curl.py                | 3 +--
 scrapy/utils/iterators.py           | 6 +++---
 scrapy/utils/misc.py                | 1 -
 tests/test_http_headers.py          | 3 ---
 tests/test_http_request.py          | 2 +-
 16 files changed, 8 insertions(+), 28 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index b88df8f64..4d7d9bac4 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,9 +1,8 @@
-import six
-import signal
 import logging
+import signal
+import sys
 import warnings
 
-import sys
 from twisted.internet import reactor, defer
 from zope.interface.verify import verifyClass, DoesNotImplement
 
@@ -22,6 +21,7 @@ from scrapy.utils.log import (
     get_scrapy_root_handler, install_scrapy_root_handler)
 from scrapy import signals
 
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index e45317491..f2999daea 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -11,8 +11,6 @@ import warnings
 import pickle
 from xml.sax.saxutils import XMLGenerator
 
-import six
-
 from scrapy.utils.serialize import ScrapyJSONEncoder
 from scrapy.utils.python import to_bytes, to_unicode, is_listlike
 from scrapy.item import BaseItem
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 5bbe7d72a..860a5c9c6 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,4 +1,3 @@
-import six
 from w3lib.http import headers_dict_to_raw
 from scrapy.utils.datatypes import CaselessDict
 from scrapy.utils.python import to_unicode
@@ -68,9 +67,6 @@ class Headers(CaselessDict):
         self[key] = lst
 
     def items(self):
-        return list(self.iteritems())
-
-    def iteritems(self):
         return ((k, self.getlist(k)) for k in self.keys())
 
     def values(self):
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index ff7a44545..1e4a9c166 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -4,7 +4,6 @@ requests in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
-import six
 from w3lib.url import safe_url_string
 
 from scrapy.http.headers import Headers
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 65400803f..1079fd6e8 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -8,7 +8,6 @@ See documentation in docs/topics/request-response.rst
 from urllib.parse import urljoin
 
 import parsel
-import six
 from w3lib.encoding import html_to_unicode, resolve_encoding, \
     html_body_declared_encoding, http_content_type_encoding
 from w3lib.html import strip_html5_whitespace
diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 2fec35799..0425d4340 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -5,7 +5,6 @@ import warnings
 from html.parser import HTMLParser
 from urllib.parse import urljoin
 
-import six
 from w3lib.url import safe_url_string
 from w3lib.html import strip_html5_whitespace
 
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 496ad3053..cb55e805a 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -3,7 +3,6 @@ Link extractor based on lxml.html
 """
 from urllib.parse import urljoin
 
-import six
 import lxml.etree as etree
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index a9dffcad3..2ba6bca45 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -5,7 +5,6 @@ import warnings
 from urllib.parse import urljoin
 from sgmllib import SGMLParser
 
-import six
 from w3lib.url import safe_url_string, canonicalize_url
 from w3lib.html import strip_html5_whitespace
 
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index c88c9c0e2..b6133619c 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,4 +1,3 @@
-import six
 import json
 import copy
 from collections.abc import MutableMapping
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 59e2c5661..a5eb1a518 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -8,8 +8,6 @@ See documentation in docs/topics/spiders.rst
 import copy
 import warnings
 
-import six
-
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, HtmlResponse
 from scrapy.linkextractors import LinkExtractor
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 2917daf57..d368c7108 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -1,6 +1,5 @@
 import re
 import logging
-import six
 
 from scrapy.spiders import Spider
 from scrapy.http import Request, XmlResponse
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index a0a47e473..16639356e 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -4,7 +4,6 @@ from shlex import split
 from http.cookies import SimpleCookie
 from urllib.parse import urlparse
 
-from six import string_types, iteritems
 from w3lib.http import basic_auth_header
 
 
@@ -76,7 +75,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
         name = name.strip()
         val = val.strip()
         if name.title() == 'Cookie':
-            for name, morsel in iteritems(SimpleCookie(val)):
+            for name, morsel in SimpleCookie(val).items():
                 cookies[name] = morsel.value
         else:
             headers.append((name, val))
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 10481fe8d..3c0cb68c3 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -1,13 +1,13 @@
-import re
 import csv
-from io import StringIO
 import logging
-import six
+import re
+from io import StringIO
 
 from scrapy.http import TextResponse, Response
 from scrapy.selector import Selector
 from scrapy.utils.python import re_rsearch, to_unicode
 
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 9a44f3576..0de3f18b3 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -6,7 +6,6 @@ from contextlib import contextmanager
 from importlib import import_module
 from pkgutil import iter_modules
 
-import six
 from w3lib.html import replace_entities
 
 from scrapy.utils.python import flatten, to_unicode
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 69d906fbf..50763d8f7 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -85,9 +85,6 @@ class HeadersTest(unittest.TestCase):
         self.assertSortedEqual(h.items(),
                                [(b'X-Forwarded-For', [b'ip1', b'ip2']),
                                 (b'Content-Type', [b'text/html'])])
-        self.assertSortedEqual(h.iteritems(),
-                               [(b'X-Forwarded-For', [b'ip1', b'ip2']),
-                                (b'Content-Type', [b'text/html'])])
         self.assertSortedEqual(h.values(), [b'ip2', b'text/html'])
 
     def test_update(self):
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 62d2847d7..988c8a811 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -62,7 +62,7 @@ class RequestTest(unittest.TestCase):
         # headers must not be unicode
         h = Headers({'key1': u'val1', u'key2': 'val2'})
         h[u'newkey'] = u'newval'
-        for k, v in h.iteritems():
+        for k, v in h.items():
             self.assertIsInstance(k, bytes)
             for s in v:
                 self.assertIsInstance(s, bytes)

From ac62524824c590e9dd2d323bf418f71a683ecbf4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 01:00:54 -0300
Subject: [PATCH 2336/2974] Remove six.get_method_*

---
 scrapy/core/downloader/middleware.py | 6 +++---
 scrapy/core/spidermw.py              | 4 ++--
 scrapy/utils/reqser.py               | 4 ++--
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 7a6a4dfac..72432558a 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -38,7 +38,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 response = yield method(request=request, spider=spider)
                 if response is not None and not isinstance(response, (Response, Request)):
                     raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
-                                         (six.get_method_self(method).__class__.__name__, response.__class__.__name__))
+                                         (method.__self__.__class__.__name__, response.__class__.__name__))
                 if response:
                     defer.returnValue(response)
             defer.returnValue((yield download_func(request=request, spider=spider)))
@@ -53,7 +53,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 response = yield method(request=request, response=response, spider=spider)
                 if not isinstance(response, (Response, Request)):
                     raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
-                                         (six.get_method_self(method).__class__.__name__, type(response)))
+                                         (method.__self__.__class__.__name__, type(response)))
                 if isinstance(response, Request):
                     defer.returnValue(response)
             defer.returnValue(response)
@@ -65,7 +65,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 response = yield method(request=request, exception=exception, spider=spider)
                 if response is not None and not isinstance(response, (Response, Request)):
                     raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
-                                         (six.get_method_self(method).__class__.__name__, type(response)))
+                                         (method.__self__.__class__.__name__, type(response)))
                 if response:
                     defer.returnValue(response)
             defer.returnValue(_failure)
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index b5f9837ff..57436d2f6 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -37,8 +37,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
     def scrape_response(self, scrape_func, response, request, spider):
         fname = lambda f:'%s.%s' % (
-                six.get_method_self(f).__class__.__name__,
-                six.get_method_function(f).__name__)
+                f.__self__.__class__.__name__,
+                f.__func__.__name__)
 
         def process_spider_input(response):
             for method in self.methods['process_spider_input']:
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 495564ac0..e961ffca9 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -87,12 +87,12 @@ def _mangle_private_name(obj, func, name):
 def _find_method(obj, func):
     if obj:
         try:
-            func_self = six.get_method_self(func)
+            func_self = func.__self__
         except AttributeError:  # func has no __self__
             pass
         else:
             if func_self is obj:
-                name = six.get_method_function(func).__name__
+                name = func.__func__.__name__
                 if _is_private_method(name):
                     return _mangle_private_name(obj, func, name)
                 return name

From 5d8abdde59e7501e8bda25b27c8926fda66b9af1 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 01:01:10 -0300
Subject: [PATCH 2337/2974] Remove six.text_type from tests

---
 tests/test_exporters.py       |  2 +-
 tests/test_http_response.py   |  8 ++++----
 tests/test_loader.py          | 14 +++++++-------
 tests/test_logformatter.py    |  4 ++--
 tests/test_toplevel.py        |  2 +-
 tests/test_utils_iterators.py |  4 ++--
 tests/test_utils_python.py    | 14 +++++++-------
 7 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index f151a1285..8433fa4db 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -79,7 +79,7 @@ class BaseItemExporterTest(unittest.TestCase):
 
         ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
         _, name = list(ie._get_serialized_fields(self.i))[0]
-        assert isinstance(name, six.text_type)
+        assert isinstance(name, str)
         self.assertEqual(name, u'John\xa3')
 
     def test_field_custom_serializer(self):
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 80bf51647..1d121cc83 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -102,7 +102,7 @@ class BaseResponseTest(unittest.TestCase):
         self.assertEqual(r4.flags, [])
 
     def _assert_response_values(self, response, encoding, body):
-        if isinstance(body, six.text_type):
+        if isinstance(body, str):
             body_unicode = body
             body_bytes = body.encode(encoding)
         else:
@@ -110,7 +110,7 @@ class BaseResponseTest(unittest.TestCase):
             body_bytes = body
 
         assert isinstance(response.body, bytes)
-        assert isinstance(response.text, six.text_type)
+        assert isinstance(response.text, str)
         self._assert_response_encoding(response, encoding)
         self.assertEqual(response.body, body_bytes)
         self.assertEqual(response.body_as_unicode(), body_unicode)
@@ -220,11 +220,11 @@ class TextResponseTest(BaseResponseTest):
         r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
 
         # check body_as_unicode
-        self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type))
+        self.assertTrue(isinstance(r1.body_as_unicode(), str))
         self.assertEqual(r1.body_as_unicode(), unicode_string)
 
         # check response.text
-        self.assertTrue(isinstance(r1.text, six.text_type))
+        self.assertTrue(isinstance(r1.text, str))
         self.assertEqual(r1.text, unicode_string)
 
     def test_encoding(self):
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 4a4264a2a..f1cf0114f 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -157,7 +157,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_get_value(self):
         il = NameItemLoader()
-        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), six.text_type.upper))
+        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
         self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
         self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
 
@@ -258,7 +258,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_extend_custom_input_processors(self):
         class ChildItemLoader(TestItemLoader):
-            name_in = MapCompose(TestItemLoader.name_in, six.text_type.swapcase)
+            name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
 
         il = ChildItemLoader()
         il.add_value('name', u'marta')
@@ -266,7 +266,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_extend_default_input_processors(self):
         class ChildDefaultedItemLoader(DefaultedItemLoader):
-            name_in = MapCompose(DefaultedItemLoader.default_input_processor, six.text_type.swapcase)
+            name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
 
         il = ChildDefaultedItemLoader()
         il.add_value('name', u'marta')
@@ -689,7 +689,7 @@ class ProcessorsTest(unittest.TestCase):
         self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
         self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
         self.assertEqual(proc(['hello', 'world']), u'hello world')
-        self.assertIsInstance(proc(['hello', 'world']), six.text_type)
+        self.assertIsInstance(proc(['hello', 'world']), str)
 
     def test_compose(self):
         proc = Compose(lambda v: v[0], str.upper)
@@ -704,12 +704,12 @@ class ProcessorsTest(unittest.TestCase):
     def test_mapcompose(self):
         def filter_world(x):
             return None if x == 'world' else x
-        proc = MapCompose(filter_world, six.text_type.upper)
+        proc = MapCompose(filter_world, str.upper)
         self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
                          [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
-        proc = MapCompose(filter_world, six.text_type.upper)
+        proc = MapCompose(filter_world, str.upper)
         self.assertEqual(proc(None), [])
-        proc = MapCompose(filter_world, six.text_type.upper)
+        proc = MapCompose(filter_world, str.upper)
         self.assertRaises(ValueError, proc, [1])
         proc = MapCompose(filter_world, lambda x: x + 1)
         self.assertRaises(ValueError, proc, 'hello')
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index eb9c4a561..ca90de9d2 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -59,7 +59,7 @@ class LoggingContribTest(unittest.TestCase):
         logkws = self.formatter.dropped(item, exception, response, self.spider)
         logline = logkws['msg'] % logkws['args']
         lines = logline.splitlines()
-        assert all(isinstance(x, six.text_type) for x in lines)
+        assert all(isinstance(x, str) for x in lines)
         self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
 
     def test_scraped(self):
@@ -69,7 +69,7 @@ class LoggingContribTest(unittest.TestCase):
         logkws = self.formatter.scraped(item, response, self.spider)
         logline = logkws['msg'] % logkws['args']
         lines = logline.splitlines()
-        assert all(isinstance(x, six.text_type) for x in lines)
+        assert all(isinstance(x, str) for x in lines)
         self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])
 
 
diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py
index 91bbe43bc..6d305249a 100644
--- a/tests/test_toplevel.py
+++ b/tests/test_toplevel.py
@@ -6,7 +6,7 @@ import scrapy
 class ToplevelTestCase(TestCase):
 
     def test_version(self):
-        self.assertIs(type(scrapy.__version__), six.text_type)
+        self.assertIs(type(scrapy.__version__), str)
 
     def test_version_info(self):
         self.assertIs(type(scrapy.version_info), tuple)
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 2d845697e..0910bd560 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -255,8 +255,8 @@ class UtilsCsvTestCase(unittest.TestCase):
 
         # explicit type check cuz' we no like stinkin' autocasting! yarrr
         for result_row in result:
-            self.assertTrue(all((isinstance(k, six.text_type) for k in result_row.keys())))
-            self.assertTrue(all((isinstance(v, six.text_type) for v in result_row.values())))
+            self.assertTrue(all((isinstance(k, str) for k in result_row.keys())))
+            self.assertTrue(all((isinstance(v, str) for v in result_row.values())))
 
     def test_csviter_delimiter(self):
         body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 096aa50b7..bea431969 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -169,8 +169,8 @@ class UtilsPythonTestCase(unittest.TestCase):
         d2 = stringify_dict(d, keys_only=False)
         self.assertEqual(d, d2)
         self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
+        self.assertFalse(any(isinstance(x, str) for x in d2.keys()))
+        self.assertFalse(any(isinstance(x, str) for x in d2.values()))
 
     @unittest.skipUnless(six.PY2, "deprecated function")
     def test_stringify_dict_tuples(self):
@@ -179,8 +179,8 @@ class UtilsPythonTestCase(unittest.TestCase):
         d2 = stringify_dict(tuples, keys_only=False)
         self.assertEqual(d, d2)
         self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
+        self.assertFalse(any(isinstance(x, str) for x in d2.keys()), d2.keys())
+        self.assertFalse(any(isinstance(x, str) for x in d2.values()))
 
     @unittest.skipUnless(six.PY2, "deprecated function")
     def test_stringify_dict_keys_only(self):
@@ -188,7 +188,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         d2 = stringify_dict(d)
         self.assertEqual(d, d2)
         self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
+        self.assertFalse(any(isinstance(x, str) for x in d2.keys()))
 
     def test_get_func_args(self):
         def f1(a, b, c):
@@ -227,12 +227,12 @@ class UtilsPythonTestCase(unittest.TestCase):
 
         if platform.python_implementation() == 'CPython':
             # TODO: how do we fix this to return the actual argument names?
-            self.assertEqual(get_func_args(six.text_type.split), [])
+            self.assertEqual(get_func_args(str.split), [])
             self.assertEqual(get_func_args(" ".join), [])
             self.assertEqual(get_func_args(operator.itemgetter(2)), [])
         else:
             self.assertEqual(
-                get_func_args(six.text_type.split, True), ['sep', 'maxsplit'])
+                get_func_args(str.split, True), ['sep', 'maxsplit'])
             self.assertEqual(get_func_args(" ".join, True), ['list'])
             self.assertEqual(
                 get_func_args(operator.itemgetter(2), True), ['obj'])

From eaeaa40b991bcb2113ba63146fbe4b2dc9e93016 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 01:08:08 -0300
Subject: [PATCH 2338/2974] Remove six.PY* checks

---
 conftest.py             | 12 +++++-------
 tests/test_webclient.py | 20 --------------------
 2 files changed, 5 insertions(+), 27 deletions(-)

diff --git a/conftest.py b/conftest.py
index 06d65ba1d..5e6a42977 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,4 +1,3 @@
-import six
 import pytest
 
 
@@ -7,12 +6,11 @@ collect_ignore = [
     "scrapy/utils/testsite.py",
 ]
 
-
-if six.PY3:
-    for line in open('tests/py3-ignores.txt'):
-        file_path = line.strip()
-        if file_path and file_path[0] != '#':
-            collect_ignore.append(file_path)
+# FIXME: fix or delete these tests
+for line in open('tests/py3-ignores.txt'):
+    file_path = line.strip()
+    if file_path and file_path[0] != '#':
+        collect_ignore.append(file_path)
 
 
 @pytest.fixture()
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 0c04e7114..608cfe597 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -78,26 +78,6 @@ class ParseUrlTestCase(unittest.TestCase):
                 to_bytes(x) if not isinstance(x, int) else x for x in test)
             self.assertEqual(client._parse(url), test, url)
 
-    def test_externalUnicodeInterference(self):
-        """
-        L{client._parse} should return C{str} for the scheme, host, and path
-        elements of its return tuple, even when passed an URL which has
-        previously been passed to L{urlparse} as a C{unicode} string.
-        """
-        if not six.PY2:
-            raise unittest.SkipTest(
-                "Applies only to Py2, as urls can be ONLY unicode on Py3")
-        badInput = u'http://example.com/path'
-        goodInput = badInput.encode('ascii')
-        self._parse(badInput)  # cache badInput in urlparse_cached
-        scheme, netloc, host, port, path = self._parse(goodInput)
-        self.assertTrue(isinstance(scheme, str))
-        self.assertTrue(isinstance(netloc, str))
-        self.assertTrue(isinstance(host, str))
-        self.assertTrue(isinstance(path, str))
-        self.assertTrue(isinstance(port, int))
-
-
 
 class ScrapyHTTPPageGetterTests(unittest.TestCase):
 

From d72444b9c8db94dd5567e763ee53b2c70a423c9a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 01:11:23 -0300
Subject: [PATCH 2339/2974] Remove more six imports

---
 scrapy/core/downloader/middleware.py | 2 --
 scrapy/core/spidermw.py              | 1 -
 scrapy/utils/reqser.py               | 2 --
 tests/test_exporters.py              | 1 -
 tests/test_http_response.py          | 1 -
 tests/test_loader.py                 | 2 --
 tests/test_logformatter.py           | 1 -
 tests/test_toplevel.py               | 2 +-
 tests/test_utils_iterators.py        | 3 ++-
 tests/test_utils_reqser.py           | 2 --
 tests/test_utils_url.py              | 2 --
 tests/test_webclient.py              | 1 -
 12 files changed, 3 insertions(+), 17 deletions(-)

diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 72432558a..38608a429 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -3,8 +3,6 @@ Downloader Middleware manager
 
 See documentation in docs/topics/downloader-middleware.rst
 """
-import six
-
 from twisted.internet import defer
 
 from scrapy.exceptions import _InvalidOutput
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 57436d2f6..e4c6df8c7 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -5,7 +5,6 @@ See documentation in docs/topics/spider-middleware.rst
 """
 from itertools import chain, islice
 
-import six
 from twisted.python.failure import Failure
 from scrapy.exceptions import _InvalidOutput
 from scrapy.middleware import MiddlewareManager
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index e961ffca9..749bbc387 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -1,8 +1,6 @@
 """
 Helper functions for serializing (and deserializing) requests.
 """
-import six
-
 from scrapy.http import Request
 from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 8433fa4db..5d1f5c182 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -8,7 +8,6 @@ from io import BytesIO
 from datetime import datetime
 
 import lxml.etree
-import six
 
 from scrapy.item import Item, Field
 from scrapy.utils.python import to_unicode
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 1d121cc83..ee7177ceb 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 import unittest
 
-import six
 from w3lib.encoding import resolve_encoding
 
 from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
diff --git a/tests/test_loader.py b/tests/test_loader.py
index f1cf0114f..69ded4d50 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -1,8 +1,6 @@
 from functools import partial
 import unittest
 
-import six
-
 from scrapy.http import HtmlResponse
 from scrapy.item import Item, Field
 from scrapy.loader import ItemLoader
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index ca90de9d2..5dc077c5b 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -3,7 +3,6 @@ import unittest
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase as TwistedTestCase
-import six
 
 from scrapy.crawler import CrawlerRunner
 from scrapy.exceptions import DropItem
diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py
index 6d305249a..fdc5df166 100644
--- a/tests/test_toplevel.py
+++ b/tests/test_toplevel.py
@@ -1,5 +1,5 @@
 from unittest import TestCase
-import six
+
 import scrapy
 
 
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 0910bd560..4d69edb31 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 import os
-import six
+
 from twisted.trial import unittest
 
 from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
 from scrapy.http import XmlResponse, TextResponse, Response
 from tests import get_testdata
 
+
 FOOBAR_NL = u"foo\nbar"
 
 
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 92cd16de7..b5729b086 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -2,8 +2,6 @@
 import unittest
 import sys
 
-import six
-
 from scrapy.http import Request, FormRequest
 from scrapy.spiders import Spider
 from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method, _mangle_private_name
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index a8e37d7b8..c5fdc752b 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -2,8 +2,6 @@
 import unittest
 from urllib.parse import urlparse
 
-import six
-
 from scrapy.spiders import Spider
 from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
                               add_http_if_no_scheme, guess_scheme,
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 608cfe597..746367b41 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -3,7 +3,6 @@ from twisted.internet import defer
 Tests borrowed from the twisted.web.client tests.
 """
 import os
-import six
 import shutil
 
 import OpenSSL.SSL

From e461570f991cf70d17a784aa628496521c241873 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 2 Nov 2019 23:13:54 -0300
Subject: [PATCH 2340/2974] Remove protego from requirements file

---
 requirements-py3.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/requirements-py3.txt b/requirements-py3.txt
index 2c98e6f6d..cd183a525 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -2,7 +2,6 @@ parsel>=1.5.0
 PyDispatcher>=2.0.5
 Twisted>=17.9.0
 w3lib>=1.17.0
-protego>=0.1.15
 
 pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
 queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"

From 7f3cb05d8e20048eb3e37c5e83bcf495d28c6064 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 12:03:02 -0300
Subject: [PATCH 2341/2974] Remove metaclass-related six code

---
 scrapy/item.py     | 5 +----
 tests/test_item.py | 4 +---
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/scrapy/item.py b/scrapy/item.py
index 4e0f0ac44..1d39b48b2 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -10,8 +10,6 @@ from copy import deepcopy
 from pprint import pformat
 from warnings import warn
 
-import six
-
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 
@@ -130,6 +128,5 @@ class DictItem(MutableMapping, BaseItem):
         return deepcopy(self)
 
 
-@six.add_metaclass(ItemMeta)
-class Item(DictItem):
+class Item(DictItem, metaclass=ItemMeta):
     pass
diff --git a/tests/test_item.py b/tests/test_item.py
index d98c63ddd..0da8fa1ac 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -3,8 +3,6 @@ import unittest
 from unittest import mock
 from warnings import catch_warnings
 
-import six
-
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
 
@@ -302,7 +300,7 @@ class ItemMetaTest(unittest.TestCase):
 class ItemMetaClassCellRegression(unittest.TestCase):
 
     def test_item_meta_classcell_regression(self):
-        class MyItem(six.with_metaclass(ItemMeta, Item)):
+        class MyItem(Item, metaclass=ItemMeta):
             def __init__(self, *args, **kwargs):
                 # This call to super() trigger the __classcell__ propagation
                 # requirement. When not done properly raises an error:

From 586b25d27e1641433d505338901fa9a1409ccdd4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 12:10:37 -0300
Subject: [PATCH 2342/2974] Remove six types

---
 scrapy/downloadermiddlewares/ajaxcrawl.py | 3 +--
 scrapy/utils/python.py                    | 3 +--
 tests/test_utils_trackref.py              | 7 ++++---
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 78b802673..c618e9ffc 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -2,7 +2,6 @@
 import re
 import logging
 
-import six
 from w3lib import html
 
 from scrapy.exceptions import NotConfigured
@@ -66,7 +65,7 @@ class AjaxCrawlMiddleware(object):
 
 
 # XXX: move it to w3lib?
-_ajax_crawlable_re = re.compile(six.u(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'))
+_ajax_crawlable_re = re.compile(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>')
 def _has_ajaxcrawlable_meta(text):
     """
     >>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment"  content="!"/></head><body></body></html>')
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 18fee1964..d32ee5a3a 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -7,7 +7,6 @@ import re
 import inspect
 import weakref
 import errno
-import six
 from functools import partial, wraps
 from itertools import chain
 import sys
@@ -162,7 +161,7 @@ def memoizemethod_noargs(method):
     return new_method
 
 
-_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
+_BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
 _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
 
 @deprecated("scrapy.utils.python.binary_is_text")
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index 480a717e7..16e02f919 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -1,6 +1,7 @@
-import six
 import unittest
+from io import StringIO
 from unittest import mock
+
 from scrapy.utils import trackref
 
 
@@ -38,12 +39,12 @@ Live References
 Bar                                 1   oldest: 0s ago
 ''')
 
-    @mock.patch('sys.stdout', new_callable=six.StringIO)
+    @mock.patch('sys.stdout', new_callable=StringIO)
     def test_print_live_refs_empty(self, stdout):
         trackref.print_live_refs()
         self.assertEqual(stdout.getvalue(), 'Live References\n\n\n')
 
-    @mock.patch('sys.stdout', new_callable=six.StringIO)
+    @mock.patch('sys.stdout', new_callable=StringIO)
     def test_print_live_refs_with_objects(self, stdout):
         o1 = Foo()  # NOQA
         trackref.print_live_refs()

From 5797aefd4c561666c2c047494b7424d77eebb469 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 12:18:35 -0300
Subject: [PATCH 2343/2974] Remove six.assertCountEqual

---
 tests/test_cmdline/__init__.py  |  7 +++----
 tests/test_settings/__init__.py | 14 ++++++--------
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 56cfe642a..909ea90e0 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -1,13 +1,12 @@
-from io import StringIO
 import json
 import os
 import pstats
 import shutil
-import six
-from subprocess import Popen, PIPE
 import sys
 import tempfile
 import unittest
+from io import StringIO
+from subprocess import Popen, PIPE
 
 from scrapy.utils.test import get_testenv
 
@@ -65,5 +64,5 @@ class CmdlineTest(unittest.TestCase):
         for char in ("'", "<", ">", 'u"'):
             settingsstr = settingsstr.replace(char, '"')
         settingsdict = json.loads(settingsstr)
-        six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys())
+        self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())
         self.assertEqual(200, settingsdict[EXT_PATH])
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index d5cbef6f5..fda44653a 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -1,4 +1,3 @@
-import six
 import unittest
 from unittest import mock
 
@@ -43,14 +42,14 @@ class SettingsAttributeTest(unittest.TestCase):
         new_dict = {'three': 11, 'four': 21}
         attribute.set(new_dict, 10)
         self.assertIsInstance(attribute.value, BaseSettings)
-        six.assertCountEqual(self, attribute.value, new_dict)
-        six.assertCountEqual(self, original_settings, original_dict)
+        self.assertCountEqual(attribute.value, new_dict)
+        self.assertCountEqual(original_settings, original_dict)
 
         new_settings = BaseSettings({'five': 12}, 0)
         attribute.set(new_settings, 0)  # Insufficient priority
-        six.assertCountEqual(self, attribute.value, new_dict)
+        self.assertCountEqual(attribute.value, new_dict)
         attribute.set(new_settings, 10)
-        six.assertCountEqual(self, attribute.value, new_settings)
+        self.assertCountEqual(attribute.value, new_settings)
 
     def test_repr(self):
         self.assertEqual(repr(self.attribute),
@@ -276,9 +275,8 @@ class BaseSettingsTest(unittest.TestCase):
                           'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
                           'HASNOBASE': BaseSettings({3: 3000}, 'default')})
         s['TEST'].set(2, 200, 'cmdline')
-        six.assertCountEqual(self, s.getwithbase('TEST'),
-                             {1: 1, 2: 200, 3: 30})
-        six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE'])
+        self.assertCountEqual(s.getwithbase('TEST'), {1: 1, 2: 200, 3: 30})
+        self.assertCountEqual(s.getwithbase('HASNOBASE'), s['HASNOBASE'])
         self.assertEqual(s.getwithbase('NONEXISTENT'), {})
 
     def test_maxpriority(self):

From 00b793dc59a32d13c89ac5c0d54985677b228801 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 12:26:38 -0300
Subject: [PATCH 2344/2974] Remove elluding six occurrences

---
 scrapy/robotstxt.py          | 6 ++++--
 tests/test_contracts.py      | 5 ++---
 tests/test_pipeline_files.py | 2 +-
 tests/test_utils_curl.py     | 7 ++-----
 4 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 397924110..0a9af3a62 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -1,12 +1,13 @@
 import sys
 import logging
 from abc import ABCMeta, abstractmethod
-from six import with_metaclass
 
 from scrapy.utils.python import to_unicode
 
+
 logger = logging.getLogger(__name__)
 
+
 def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
     try:
         if to_native_str_type:
@@ -23,7 +24,8 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
         robotstxt_body = ''
     return robotstxt_body
 
-class RobotParser(with_metaclass(ABCMeta)):
+
+class RobotParser(metaclass=ABCMeta):
     @classmethod
     @abstractmethod
     def from_crawler(cls, crawler, robotstxt_body):
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index b2e358700..582e3d052 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -1,6 +1,5 @@
 from unittest import TextTestResult
 
-from six import get_unbound_function
 from twisted.internet import defer
 from twisted.python import failure
 from twisted.trial import unittest
@@ -395,8 +394,8 @@ class ContractsManagerTest(unittest.TestCase):
         with MockServer() as mockserver:
             contract_doc = '@url {}'.format(mockserver.url('/status?n=200'))
 
-            get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc
-            get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc
+            TestSameUrlSpider.parse_first.__doc__ = contract_doc
+            TestSameUrlSpider.parse_second.__doc__ = contract_doc
 
             crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
             yield crawler.crawl()
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index dede4bf12..52f2b554e 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,12 +1,12 @@
 import os
 import random
 import time
+from io import BytesIO
 from tempfile import mkdtemp
 from shutil import rmtree
 from unittest import mock
 from urllib.parse import urlparse
 
-from six import BytesIO
 from twisted.trial import unittest
 from twisted.internet import defer
 
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index c5655df7e..50e1bfd5f 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -1,7 +1,6 @@
 import unittest
 import warnings
 
-from six import assertRaisesRegex
 from w3lib.http import basic_auth_header
 
 from scrapy import Request
@@ -177,8 +176,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
         self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
 
     def test_too_few_arguments_error(self):
-        assertRaisesRegex(
-            self,
+        self.assertRaisesRegex(
             ValueError,
             r"too few arguments|the following arguments are required:\s*url",
             lambda: curl_to_request_kwargs("curl"),
@@ -194,8 +192,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
             self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
 
         # case 2: ignore_unknown_options=False (raise exception):
-        assertRaisesRegex(
-            self,
+        self.assertRaisesRegex(
             ValueError,
             "Unrecognized options:.*--bar.*--baz",
             lambda: curl_to_request_kwargs(

From 0c4e5b68ea0e077033fde5da28c6dc4fdd859d92 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 12:30:34 -0300
Subject: [PATCH 2345/2974] Remove six from requirements and setup files

---
 requirements-py3.txt | 1 -
 setup.py             | 1 -
 2 files changed, 2 deletions(-)

diff --git a/requirements-py3.txt b/requirements-py3.txt
index cd183a525..28c649e28 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -13,5 +13,4 @@ cryptography>=2.0  # Earlier versions would fail to install
 cssselect>=0.9.1
 lxml>=3.5.0
 service_identity>=16.0.0
-six>=1.10.0
 zope.interface>=4.1.3
diff --git a/setup.py b/setup.py
index 8f5f14f0d..85d797f88 100644
--- a/setup.py
+++ b/setup.py
@@ -72,7 +72,6 @@ setup(
         'pyOpenSSL>=16.2.0',
         'queuelib>=1.4.2',
         'service_identity>=16.0.0',
-        'six>=1.10.0',
         'w3lib>=1.17.0',
         'zope.interface>=4.1.3',
         'protego>=0.1.15',

From 439a3e59b8e858441f8d97dbc32f398db392330d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 4 Nov 2019 10:35:58 -0300
Subject: [PATCH 2346/2974] Fix scrapy.utils.datatypes.LocalCache limit issue

---
 scrapy/utils/datatypes.py     |  5 +++--
 tests/test_utils_datatypes.py | 29 +++++++++++++++++++++++++++--
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index df2b99c28..f7e3240c1 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -315,8 +315,9 @@ class LocalCache(collections.OrderedDict):
         self.limit = limit
 
     def __setitem__(self, key, value):
-        while len(self) >= self.limit:
-            self.popitem(last=False)
+        if self.limit:
+            while len(self) >= self.limit:
+                self.popitem(last=False)
         super(LocalCache, self).__setitem__(key, value)
 
 
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 535095b8d..6ffd7c73c 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -7,7 +7,7 @@ if six.PY2:
 else:
     from collections.abc import Mapping, MutableMapping
 
-from scrapy.utils.datatypes import CaselessDict, SequenceExclude
+from scrapy.utils.datatypes import CaselessDict, SequenceExclude, LocalCache
 
 
 __doctests__ = ['scrapy.utils.datatypes']
@@ -242,6 +242,31 @@ class SequenceExcludeTest(unittest.TestCase):
         for v in [-3, "test", 1.1]:
             self.assertNotIn(v, d)
 
+
+class LocalCacheTest(unittest.TestCase):
+
+    def test_cache_with_limit(self):
+        cache = LocalCache(limit=2)
+        cache['a'] = 1
+        cache['b'] = 2
+        cache['c'] = 3
+        self.assertEqual(len(cache), 2)
+        self.assertNotIn('a', cache)
+        self.assertIn('b', cache)
+        self.assertIn('c', cache)
+        self.assertEqual(cache['b'], 2)
+        self.assertEqual(cache['c'], 3)
+
+    def test_cache_without_limit(self):
+        max = 10**4
+        cache = LocalCache()
+        for x in range(max):
+            cache[str(x)] = x
+        self.assertEqual(len(cache), max)
+        for x in range(max):
+            self.assertIn(str(x), cache)
+            self.assertEqual(cache[str(x)], x)
+
+
 if __name__ == "__main__":
     unittest.main()
-

From fed9fbe62d54175d70660498f2fba6b5b7f68a92 Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Mon, 4 Nov 2019 15:34:27 -0300
Subject: [PATCH 2347/2974] Update tests/test_utils_datatypes.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 tests/test_utils_datatypes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 6ffd7c73c..fb2362829 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -7,7 +7,7 @@ if six.PY2:
 else:
     from collections.abc import Mapping, MutableMapping
 
-from scrapy.utils.datatypes import CaselessDict, SequenceExclude, LocalCache
+from scrapy.utils.datatypes import CaselessDict, LocalCache, SequenceExclude
 
 
 __doctests__ = ['scrapy.utils.datatypes']

From 613c66a034cebc885b90c8ba2a76aea2109ef72e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 5 Nov 2019 09:45:51 -0300
Subject: [PATCH 2348/2974] Do not override built-in max function

---
 tests/test_utils_datatypes.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index fb2362829..7e671f627 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -258,12 +258,12 @@ class LocalCacheTest(unittest.TestCase):
         self.assertEqual(cache['c'], 3)
 
     def test_cache_without_limit(self):
-        max = 10**4
+        maximum = 10**4
         cache = LocalCache()
-        for x in range(max):
+        for x in range(maximum):
             cache[str(x)] = x
-        self.assertEqual(len(cache), max)
-        for x in range(max):
+        self.assertEqual(len(cache), maximum)
+        for x in range(maximum):
             self.assertIn(str(x), cache)
             self.assertEqual(cache[str(x)], x)
 

From 698aa704b98e206cb755190f74d73a6ba47c5fac Mon Sep 17 00:00:00 2001
From: seregaxvm <seregaxvm.main@gmail.com>
Date: Tue, 5 Nov 2019 18:30:01 +0300
Subject: [PATCH 2349/2974] Fix zsh completion file extension (#4122)

---
 extras/scrapy_zsh_completion | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion
index 86c52c36c..e995947cb 100644
--- a/extras/scrapy_zsh_completion
+++ b/extras/scrapy_zsh_completion
@@ -61,7 +61,7 @@ _scrapy() {
 			'-c[evaluate the code in the shell, print the result and exit]:code:(CODE)'
 			'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
 			'--spider[use this spider]:spider:_scrapy_spiders'
-			'::file:_files -g \*.http'
+			'::file:_files -g \*.html'
 			'::URL:_httpie_urls'
 		    )
 		    _scrapy_glb_opts $options

From fe31695ba0266deaa94222fc01885b7270af4294 Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Tue, 5 Nov 2019 15:36:19 -0300
Subject: [PATCH 2350/2974] Remove unused import (urllib.parse.unquote)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 tests/test_http_request.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 988c8a811..05cac617c 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -6,7 +6,7 @@ import json
 import xmlrpc.client as xmlrpclib
 import warnings
 from unittest import mock
-from urllib.parse import parse_qs, unquote, unquote_to_bytes, urlparse
+from urllib.parse import parse_qs, unquote_to_bytes, urlparse
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_unicode

From 98caf055b5a343ff663f2c1150b301a3f4546a16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 6 Nov 2019 11:53:46 +0100
Subject: [PATCH 2351/2974] =?UTF-8?q?Fix=20a=20typo:=20specifiy=20?=
 =?UTF-8?q?=E2=86=92=20specify=20(#4128)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scrapy/utils/misc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index f638adb25..b74f34451 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -136,7 +136,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
     """
     if settings is None:
         if crawler is None:
-            raise ValueError("Specifiy at least one of settings and crawler.")
+            raise ValueError("Specify at least one of settings and crawler.")
         settings = crawler.settings
     if crawler and hasattr(objcls, 'from_crawler'):
         return objcls.from_crawler(crawler, *args, **kwargs)

From e8b1e46e85fbcdf22408d320f3cc61b6e802c5e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Thu, 7 Nov 2019 14:05:01 +0100
Subject: [PATCH 2352/2974] Add pytest-flake8 (#3945)

---
 .travis.yml |   2 +
 conftest.py |  10 +++
 pytest.ini  | 253 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 tox.ini     |   8 ++
 4 files changed, 273 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 2352ef124..f5b9d3f72 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,6 +7,8 @@ branches:
     - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
 matrix:
   include:
+    - env: TOXENV=flake8
+      python: 3.8
     - env: TOXENV=pypy3
       python: 3.5
     - env: TOXENV=py35
diff --git a/conftest.py b/conftest.py
index 06d65ba1d..d5d61ddd3 100644
--- a/conftest.py
+++ b/conftest.py
@@ -19,3 +19,13 @@ if six.PY3:
 def chdir(tmpdir):
     """Change to pytest-provided temporary directory"""
     tmpdir.chdir()
+
+
+def pytest_collection_modifyitems(session, config, items):
+    # Avoid executing tests when executing `--flake8` flag (pytest-flake8)
+    try:
+        from pytest_flake8 import Flake8Item
+        if config.getoption('--flake8'):
+            items[:] = [item for item in items if isinstance(item, Flake8Item)]
+    except ImportError:
+        pass
diff --git a/pytest.ini b/pytest.ini
index 73d169601..0e4ee9ed1 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -4,3 +4,256 @@ python_files=test_*.py __init__.py
 python_classes=
 addopts = --doctest-modules --assert=plain
 twisted = 1
+flake8-ignore =
+    # extras
+    extras/qps-bench-server.py E261 E501
+    extras/qpsclient.py E501 E261 E501
+    # scrapy/commands
+    scrapy/commands/__init__.py E128 E501
+    scrapy/commands/check.py F401 E501 W391
+    scrapy/commands/crawl.py E501
+    scrapy/commands/edit.py E501
+    scrapy/commands/fetch.py E401 E302 E501 E128 E502 E731
+    scrapy/commands/genspider.py E128 E501 E502
+    scrapy/commands/list.py E302
+    scrapy/commands/parse.py E128 E501 E731 E226
+    scrapy/commands/runspider.py E501
+    scrapy/commands/settings.py E302 E128
+    scrapy/commands/shell.py E128 E501 E502
+    scrapy/commands/startproject.py E502 E127 E501 E128 W391
+    scrapy/commands/version.py E501 E128 W391
+    scrapy/commands/view.py F401 E302
+    # scrapy/contracts
+    scrapy/contracts/__init__.py E501 W504
+    scrapy/contracts/default.py E502 E128
+    # scrapy/core
+    scrapy/core/engine.py E261 E501 E128 E127 E306 E502
+    scrapy/core/scheduler.py E501
+    scrapy/core/scraper.py E501 E306 E261 E128 W391 W504
+    scrapy/core/spidermw.py E501 E731 E502 E231 E126 E226
+    scrapy/core/downloader/__init__.py F401 E501
+    scrapy/core/downloader/contextfactory.py E501 E128 E126
+    scrapy/core/downloader/middleware.py E501 E502
+    scrapy/core/downloader/tls.py E501 E305 E241
+    scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 W391 E126 E226
+    scrapy/core/downloader/handlers/__init__.py E501
+    scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127 W391
+    scrapy/core/downloader/handlers/http.py F401
+    scrapy/core/downloader/handlers/http10.py E501
+    scrapy/core/downloader/handlers/http11.py E501
+    scrapy/core/downloader/handlers/s3.py E501 F401 E502 E128 E126
+    # scrapy/downloadermiddlewares
+    scrapy/downloadermiddlewares/ajaxcrawl.py E302 E501 E226
+    scrapy/downloadermiddlewares/decompression.py E501
+    scrapy/downloadermiddlewares/defaultheaders.py E501
+    scrapy/downloadermiddlewares/httpcache.py E501 E126
+    scrapy/downloadermiddlewares/httpcompression.py E502 E128
+    scrapy/downloadermiddlewares/httpproxy.py E501
+    scrapy/downloadermiddlewares/redirect.py E501 W504
+    scrapy/downloadermiddlewares/retry.py E501 E126
+    scrapy/downloadermiddlewares/robotstxt.py F401 E501
+    scrapy/downloadermiddlewares/stats.py E501
+    # scrapy/extensions
+    scrapy/extensions/closespider.py E501 E502 E128 E123
+    scrapy/extensions/corestats.py E302 E501
+    scrapy/extensions/feedexport.py E128 E501
+    scrapy/extensions/httpcache.py E128 E501 E303 F401
+    scrapy/extensions/memdebug.py E501
+    scrapy/extensions/spiderstate.py E302 E501
+    scrapy/extensions/telnet.py E501 W504
+    scrapy/extensions/throttle.py E501
+    # scrapy/http
+    scrapy/http/__init__.py F401
+    scrapy/http/common.py E501
+    scrapy/http/cookies.py E501
+    scrapy/http/headers.py W391
+    scrapy/http/request/__init__.py E501
+    scrapy/http/request/form.py E501 E123
+    scrapy/http/request/json_request.py E501
+    scrapy/http/response/__init__.py E501 E128 W293 W291
+    scrapy/http/response/html.py E302
+    scrapy/http/response/text.py E501 W293 E128 E124
+    scrapy/http/response/xml.py E302
+    # scrapy/linkextractors
+    scrapy/linkextractors/__init__.py E731 E502 E501 E402 F401
+    scrapy/linkextractors/lxmlhtml.py E501 E731 E226
+    # scrapy/loader
+    scrapy/loader/__init__.py E501 E502 E128
+    scrapy/loader/common.py E302
+    scrapy/loader/processors.py E501
+    # scrapy/pipelines
+    scrapy/pipelines/__init__.py E302
+    scrapy/pipelines/files.py E116 E501 E266
+    scrapy/pipelines/images.py E265 E501
+    scrapy/pipelines/media.py E125 E501 E266
+    # scrapy/selector
+    scrapy/selector/__init__.py F403 F401
+    scrapy/selector/unified.py F401 E501 E111
+    # scrapy/settings
+    scrapy/settings/__init__.py E501
+    scrapy/settings/default_settings.py E501 E261 E114 E116 E226
+    scrapy/settings/deprecated.py E501
+    # scrapy/spidermiddlewares
+    scrapy/spidermiddlewares/httperror.py E501
+    scrapy/spidermiddlewares/offsite.py E501
+    scrapy/spidermiddlewares/referer.py F401 E501 E129 W503 W504
+    scrapy/spidermiddlewares/urllength.py E501
+    # scrapy/spiders
+    scrapy/spiders/__init__.py F401 E501 E402
+    scrapy/spiders/crawl.py E501
+    scrapy/spiders/feed.py E501 E261 W391
+    scrapy/spiders/init.py W391
+    scrapy/spiders/sitemap.py E501
+    # scrapy/utils
+    scrapy/utils/benchserver.py E501
+    scrapy/utils/boto.py F401
+    scrapy/utils/conf.py E402 E502 E501
+    scrapy/utils/console.py E302 E261 F401 E306 E305
+    scrapy/utils/curl.py F401
+    scrapy/utils/datatypes.py E501 E226
+    scrapy/utils/decorators.py E501 E302
+    scrapy/utils/defer.py E501 E302 E128
+    scrapy/utils/deprecate.py E128 E501 E127 E502
+    scrapy/utils/display.py E302
+    scrapy/utils/engine.py F401 E261 E302
+    scrapy/utils/ftp.py E302
+    scrapy/utils/gz.py E305 E501 E302 W504
+    scrapy/utils/http.py F403 F401 W391 E226
+    scrapy/utils/httpobj.py E302 E501
+    scrapy/utils/iterators.py E501 E701
+    scrapy/utils/job.py E302
+    scrapy/utils/log.py E128 W503
+    scrapy/utils/markup.py F403 F401 W292
+    scrapy/utils/misc.py E501 E226
+    scrapy/utils/multipart.py F403 F401 W292
+    scrapy/utils/project.py E501
+    scrapy/utils/python.py E501 E302
+    scrapy/utils/reactor.py E302 E226
+    scrapy/utils/reqser.py E501
+    scrapy/utils/request.py E302 E127 E501
+    scrapy/utils/response.py E501 E302 E128
+    scrapy/utils/signal.py E501 E128
+    scrapy/utils/sitemap.py E501
+    scrapy/utils/spider.py E271 E302 E501
+    scrapy/utils/ssl.py E501
+    scrapy/utils/template.py E302
+    scrapy/utils/test.py E302 E501
+    scrapy/utils/url.py E501 F403 F401 E128 F405
+    # scrapy
+    scrapy/__init__.py E402 E501
+    scrapy/_monkeypatches.py W293
+    scrapy/cmdline.py E502 E501
+    scrapy/crawler.py E501
+    scrapy/dupefilters.py E302 E501 E202
+    scrapy/exceptions.py E302 E501
+    scrapy/exporters.py E501 E261 E226
+    scrapy/extension.py E302
+    scrapy/interfaces.py E302 E501 W391
+    scrapy/item.py E501 E128
+    scrapy/link.py E501 W391
+    scrapy/logformatter.py E501 W293
+    scrapy/mail.py E402 E128 E501 E502
+    scrapy/middleware.py E502 E128 E501
+    scrapy/pqueues.py E501
+    scrapy/resolver.py E302
+    scrapy/responsetypes.py E128 E501 E305
+    scrapy/robotstxt.py E302 E501
+    scrapy/shell.py E501
+    scrapy/signalmanager.py E501
+    scrapy/spiderloader.py E225 F841 E501 E126
+    scrapy/squeues.py E128
+    scrapy/statscollectors.py E501 W391
+    # tests
+    tests/__init__.py F401 E402 E501
+    tests/mockserver.py E401 E501 E126 E123 F401
+    tests/pipelines.py E302 F841 E226
+    tests/spiders.py E302 E501 E127
+    tests/test_closespider.py E501 E127
+    tests/test_command_fetch.py E501 E261
+    tests/test_command_parse.py F401 E302 E501 E128 E303 E226
+    tests/test_command_shell.py E501 E128
+    tests/test_commands.py F401 E128 E501
+    tests/test_contracts.py E501 E128 W293
+    tests/test_crawl.py E501 E741 E265
+    tests/test_crawler.py F841 E306 E501
+    tests/test_dependencies.py E302 F841 E501 E305
+    tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E711 E126 E226 E123
+    tests/test_downloadermiddleware.py E501
+    tests/test_downloadermiddleware_ajaxcrawlable.py E302 E501
+    tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
+    tests/test_downloadermiddleware_decompression.py E127
+    tests/test_downloadermiddleware_defaultheaders.py E501
+    tests/test_downloadermiddleware_downloadtimeout.py E501
+    tests/test_downloadermiddleware_httpcache.py E713 E501 E302 E305 F401
+    tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123
+    tests/test_downloadermiddleware_httpproxy.py F401 E501 E128
+    tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
+    tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
+    tests/test_downloadermiddleware_robotstxt.py E501
+    tests/test_downloadermiddleware_stats.py E501
+    tests/test_dupefilters.py E302 E221 E501 E741 W293 W291 E128 E124
+    tests/test_engine.py E401 E501 E502 E128 E261
+    tests/test_exporters.py E501 E731 E306 E128 E124
+    tests/test_extension_telnet.py F401 F841
+    tests/test_feedexport.py E501 F401 F841 E241
+    tests/test_http_cookies.py E501
+    tests/test_http_headers.py E302 E501
+    tests/test_http_request.py F401 E402 E501 E231 E261 E127 E128 W293 E502 E128 E502 E126 E123
+    tests/test_http_response.py E501 E301 E502 E128 E265
+    tests/test_item.py E701 E128 E231 F841 E306
+    tests/test_link.py E501
+    tests/test_linkextractors.py E501 E128 E231 E124
+    tests/test_loader.py E302 E501 E731 E303 E741 E128 E117 E241
+    tests/test_logformatter.py E128 E501 E231 E122 E302
+    tests/test_mail.py E302 E128 E501 E305
+    tests/test_middleware.py E302 E501 E128
+    tests/test_pipeline_crawl.py E131 E501 E128 E126
+    tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226
+    tests/test_pipeline_images.py F401 F841 E501 E303
+    tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
+    tests/test_request_cb_kwargs.py E501
+    tests/test_responsetypes.py E501 E302 E305
+    tests/test_robotstxt_interface.py F401 E302 E501 W291 E501
+    tests/test_scheduler.py E501 E126 E123
+    tests/test_selector.py F401 E501 E127
+    tests/test_spider.py E501 F401
+    tests/test_spidermiddleware.py E501 E226
+    tests/test_spidermiddleware_depth.py W391
+    tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
+    tests/test_spidermiddleware_offsite.py E302 E501 E128 E111 W293
+    tests/test_spidermiddleware_output_chain.py F401 E501 E302 W293 E226
+    tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 W391 E241 E121
+    tests/test_spidermiddleware_urllength.py W391
+    tests/test_squeues.py E501 E302 E701 E741
+    tests/test_utils_conf.py E501 E231 E303 E128
+    tests/test_utils_console.py E302 E231
+    tests/test_utils_curl.py E501
+    tests/test_utils_datatypes.py E402 E501 E305 W391
+    tests/test_utils_defer.py E306 E261 E501 E302 F841 E226
+    tests/test_utils_deprecate.py F841 E306 E501
+    tests/test_utils_http.py E302 E501 E502 E128 W391 W504
+    tests/test_utils_httpobj.py E302
+    tests/test_utils_iterators.py E501 E128 E129 E302 E303 E241
+    tests/test_utils_log.py E741 E226
+    tests/test_utils_python.py E501 E303 E731 E701 E305
+    tests/test_utils_reqser.py F401 E501 E128
+    tests/test_utils_request.py E302 E501 E128 E305
+    tests/test_utils_response.py E501
+    tests/test_utils_signal.py E741 F841 E302 E731 E226
+    tests/test_utils_sitemap.py E302 E128 E501 E124
+    tests/test_utils_spider.py E261 E302 E305 W391
+    tests/test_utils_template.py E305
+    tests/test_utils_url.py F401 E501 E127 E302 E305 E211 E125 E501 E226 E241 E126 E123
+    tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
+    tests/mocks/dummydbm.py E302
+    tests/test_cmdline/__init__.py E502 E501
+    tests/test_cmdline/extensions.py E302 W391
+    tests/test_settings/__init__.py F401 E501 E128
+    tests/test_settings/default_settings.py W391
+    tests/test_spiderloader/__init__.py E128 E501 E302
+    tests/test_spiderloader/test_spiders/spider0.py E302
+    tests/test_spiderloader/test_spiders/spider1.py E302
+    tests/test_spiderloader/test_spiders/spider2.py E302
+    tests/test_spiderloader/test_spiders/spider3.py E302
+    tests/test_spiderloader/test_spiders/nested/spider4.py E302
+    tests/test_utils_misc/__init__.py E501 E231
diff --git a/tox.ini b/tox.ini
index 821144381..cc3463a4d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -62,6 +62,14 @@ basepython = pypy3
 commands =
     py.test {posargs:scrapy tests}
 
+[testenv:flake8]
+basepython = python3.8
+deps =
+    {[testenv]deps}
+    pytest-flake8
+commands =
+    py.test --flake8 {posargs:scrapy tests}
+
 [docs]
 changedir = docs
 deps =

From c377c14e3263ce3c2bffa446cd0965006e845664 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Thu, 7 Nov 2019 17:47:35 +0100
Subject: [PATCH 2353/2974] Fix W391 Blank line at end of file (#4137)

---
 pytest.ini                               | 37 ++++++++++--------------
 scrapy/commands/check.py                 |  1 -
 scrapy/commands/startproject.py          |  1 -
 scrapy/commands/version.py               |  1 -
 scrapy/core/downloader/handlers/ftp.py   |  1 -
 scrapy/core/downloader/webclient.py      |  1 -
 scrapy/core/scraper.py                   |  1 -
 scrapy/http/headers.py                   |  2 --
 scrapy/interfaces.py                     |  1 -
 scrapy/link.py                           |  1 -
 scrapy/spiders/feed.py                   |  1 -
 scrapy/spiders/init.py                   |  1 -
 scrapy/statscollectors.py                |  2 --
 scrapy/utils/http.py                     |  1 -
 tests/test_cmdline/extensions.py         |  1 -
 tests/test_settings/default_settings.py  |  1 -
 tests/test_spidermiddleware_depth.py     |  1 -
 tests/test_spidermiddleware_urllength.py |  1 -
 tests/test_utils_datatypes.py            |  1 -
 tests/test_utils_http.py                 |  2 --
 tests/test_utils_spider.py               |  1 -
 21 files changed, 16 insertions(+), 44 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 0e4ee9ed1..db5bee228 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -10,7 +10,7 @@ flake8-ignore =
     extras/qpsclient.py E501 E261 E501
     # scrapy/commands
     scrapy/commands/__init__.py E128 E501
-    scrapy/commands/check.py F401 E501 W391
+    scrapy/commands/check.py F401 E501
     scrapy/commands/crawl.py E501
     scrapy/commands/edit.py E501
     scrapy/commands/fetch.py E401 E302 E501 E128 E502 E731
@@ -20,8 +20,8 @@ flake8-ignore =
     scrapy/commands/runspider.py E501
     scrapy/commands/settings.py E302 E128
     scrapy/commands/shell.py E128 E501 E502
-    scrapy/commands/startproject.py E502 E127 E501 E128 W391
-    scrapy/commands/version.py E501 E128 W391
+    scrapy/commands/startproject.py E502 E127 E501 E128
+    scrapy/commands/version.py E501 E128
     scrapy/commands/view.py F401 E302
     # scrapy/contracts
     scrapy/contracts/__init__.py E501 W504
@@ -29,15 +29,15 @@ flake8-ignore =
     # scrapy/core
     scrapy/core/engine.py E261 E501 E128 E127 E306 E502
     scrapy/core/scheduler.py E501
-    scrapy/core/scraper.py E501 E306 E261 E128 W391 W504
+    scrapy/core/scraper.py E501 E306 E261 E128 W504
     scrapy/core/spidermw.py E501 E731 E502 E231 E126 E226
     scrapy/core/downloader/__init__.py F401 E501
     scrapy/core/downloader/contextfactory.py E501 E128 E126
     scrapy/core/downloader/middleware.py E501 E502
     scrapy/core/downloader/tls.py E501 E305 E241
-    scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 W391 E126 E226
+    scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226
     scrapy/core/downloader/handlers/__init__.py E501
-    scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127 W391
+    scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
     scrapy/core/downloader/handlers/http.py F401
     scrapy/core/downloader/handlers/http10.py E501
     scrapy/core/downloader/handlers/http11.py E501
@@ -66,7 +66,6 @@ flake8-ignore =
     scrapy/http/__init__.py F401
     scrapy/http/common.py E501
     scrapy/http/cookies.py E501
-    scrapy/http/headers.py W391
     scrapy/http/request/__init__.py E501
     scrapy/http/request/form.py E501 E123
     scrapy/http/request/json_request.py E501
@@ -101,8 +100,7 @@ flake8-ignore =
     # scrapy/spiders
     scrapy/spiders/__init__.py F401 E501 E402
     scrapy/spiders/crawl.py E501
-    scrapy/spiders/feed.py E501 E261 W391
-    scrapy/spiders/init.py W391
+    scrapy/spiders/feed.py E501 E261
     scrapy/spiders/sitemap.py E501
     # scrapy/utils
     scrapy/utils/benchserver.py E501
@@ -118,7 +116,7 @@ flake8-ignore =
     scrapy/utils/engine.py F401 E261 E302
     scrapy/utils/ftp.py E302
     scrapy/utils/gz.py E305 E501 E302 W504
-    scrapy/utils/http.py F403 F401 W391 E226
+    scrapy/utils/http.py F403 F401 E226
     scrapy/utils/httpobj.py E302 E501
     scrapy/utils/iterators.py E501 E701
     scrapy/utils/job.py E302
@@ -148,9 +146,9 @@ flake8-ignore =
     scrapy/exceptions.py E302 E501
     scrapy/exporters.py E501 E261 E226
     scrapy/extension.py E302
-    scrapy/interfaces.py E302 E501 W391
+    scrapy/interfaces.py E302 E501
     scrapy/item.py E501 E128
-    scrapy/link.py E501 W391
+    scrapy/link.py E501
     scrapy/logformatter.py E501 W293
     scrapy/mail.py E402 E128 E501 E502
     scrapy/middleware.py E502 E128 E501
@@ -162,7 +160,7 @@ flake8-ignore =
     scrapy/signalmanager.py E501
     scrapy/spiderloader.py E225 F841 E501 E126
     scrapy/squeues.py E128
-    scrapy/statscollectors.py E501 W391
+    scrapy/statscollectors.py E501
     # tests
     tests/__init__.py F401 E402 E501
     tests/mockserver.py E401 E501 E126 E123 F401
@@ -218,20 +216,18 @@ flake8-ignore =
     tests/test_selector.py F401 E501 E127
     tests/test_spider.py E501 F401
     tests/test_spidermiddleware.py E501 E226
-    tests/test_spidermiddleware_depth.py W391
     tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
     tests/test_spidermiddleware_offsite.py E302 E501 E128 E111 W293
     tests/test_spidermiddleware_output_chain.py F401 E501 E302 W293 E226
-    tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 W391 E241 E121
-    tests/test_spidermiddleware_urllength.py W391
+    tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 E241 E121
     tests/test_squeues.py E501 E302 E701 E741
     tests/test_utils_conf.py E501 E231 E303 E128
     tests/test_utils_console.py E302 E231
     tests/test_utils_curl.py E501
-    tests/test_utils_datatypes.py E402 E501 E305 W391
+    tests/test_utils_datatypes.py E402 E501 E305
     tests/test_utils_defer.py E306 E261 E501 E302 F841 E226
     tests/test_utils_deprecate.py F841 E306 E501
-    tests/test_utils_http.py E302 E501 E502 E128 W391 W504
+    tests/test_utils_http.py E302 E501 E502 E128 W504
     tests/test_utils_httpobj.py E302
     tests/test_utils_iterators.py E501 E128 E129 E302 E303 E241
     tests/test_utils_log.py E741 E226
@@ -241,15 +237,14 @@ flake8-ignore =
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841 E302 E731 E226
     tests/test_utils_sitemap.py E302 E128 E501 E124
-    tests/test_utils_spider.py E261 E302 E305 W391
+    tests/test_utils_spider.py E261 E302 E305
     tests/test_utils_template.py E305
     tests/test_utils_url.py F401 E501 E127 E302 E305 E211 E125 E501 E226 E241 E126 E123
     tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
     tests/mocks/dummydbm.py E302
     tests/test_cmdline/__init__.py E502 E501
-    tests/test_cmdline/extensions.py E302 W391
+    tests/test_cmdline/extensions.py E302
     tests/test_settings/__init__.py F401 E501 E128
-    tests/test_settings/default_settings.py W391
     tests/test_spiderloader/__init__.py E128 E501 E302
     tests/test_spiderloader/test_spiders/spider0.py E302
     tests/test_spiderloader/test_spiders/spider1.py E302
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index ab73e85e7..3e6c11b7d 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -96,4 +96,3 @@ class Command(ScrapyCommand):
             result.printErrors()
             result.printSummary(start, stop)
             self.exitcode = int(not result.wasSuccessful())
-
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 67337c26e..3b9f6eabb 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -119,4 +119,3 @@ class Command(ScrapyCommand):
         _templates_base_dir = self.settings['TEMPLATES_DIR'] or \
             join(scrapy.__path__[0], 'templates')
         return join(_templates_base_dir, 'project')
-
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 577365c3b..8651948f7 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -30,4 +30,3 @@ class Command(ScrapyCommand):
                 print(patt % (name, version))
         else:
             print("Scrapy %s" % scrapy.__version__)
-
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 806a537d4..39ed67a1a 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -112,4 +112,3 @@ class FTPDownloadHandler(object):
                 httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])
                 return Response(url=request.url, status=httpcode, body=to_bytes(message))
         raise result.type(result.value)
-
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 3a5890ed0..3fe13414a 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -157,4 +157,3 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
     def gotHeaders(self, headers):
         self.headers_time = time()
         self.response_headers = headers
-
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 1f389cf2e..40de6b87a 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -244,4 +244,3 @@ class Scraper(object):
             return self.signals.send_catch_log_deferred(
                 signal=signals.item_scraped, item=output, response=response,
                 spider=spider)
-
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 62507eb19..f3b46b994 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -91,5 +91,3 @@ class Headers(CaselessDict):
     def __copy__(self):
         return self.__class__(self)
     copy = __copy__
-
-
diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py
index 89ad2b14f..d48babc3c 100644
--- a/scrapy/interfaces.py
+++ b/scrapy/interfaces.py
@@ -15,4 +15,3 @@ class ISpiderLoader(Interface):
 
     def find_by_request(request):
         """Return the list of spiders names that can handle the given request"""
-
diff --git a/scrapy/link.py b/scrapy/link.py
index 2c8301680..f0638ced2 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -39,4 +39,3 @@ class Link(object):
     def __repr__(self):
         return 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' % \
             (self.url, self.text, self.fragment, self.nofollow)
-
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 06e212e1c..197812a26 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -133,4 +133,3 @@ class CSVFeedSpider(Spider):
             raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
         response = self.adapt_response(response)
         return self.parse_rows(response)
-
diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py
index 2efb1a869..fd41133ea 100644
--- a/scrapy/spiders/init.py
+++ b/scrapy/spiders/init.py
@@ -29,4 +29,3 @@ class InitSpider(Spider):
         spider
         """
         return self.initialized()
-
diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py
index 6da9ddcd2..f0bfaed34 100644
--- a/scrapy/statscollectors.py
+++ b/scrapy/statscollectors.py
@@ -80,5 +80,3 @@ class DummyStatsCollector(StatsCollector):
 
     def min_value(self, key, value, spider=None):
         pass
-
-
diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py
index b6e05c862..ad49ef3e9 100644
--- a/scrapy/utils/http.py
+++ b/scrapy/utils/http.py
@@ -34,4 +34,3 @@ def decode_chunked_transfer(chunked_body):
         body += t[:size]
         t = t[size+2:]
     return body
-
diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py
index 72867eb56..28456b55d 100644
--- a/tests/test_cmdline/extensions.py
+++ b/tests/test_cmdline/extensions.py
@@ -12,4 +12,3 @@ class TestExtension(object):
 
 class DummyExtension(object):
     pass
-
diff --git a/tests/test_settings/default_settings.py b/tests/test_settings/default_settings.py
index c24b5a9b9..26a555275 100644
--- a/tests/test_settings/default_settings.py
+++ b/tests/test_settings/default_settings.py
@@ -2,4 +2,3 @@
 TEST_DEFAULT = 'defvalue'
 
 TEST_DICT = {'key': 'val'}
-
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index 3685d5a6f..71cca2472 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -40,4 +40,3 @@ class TestDepthMiddleware(TestCase):
 
     def tearDown(self):
         self.stats.close_spider(self.spider, '')
-
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index a0aae0fdd..5ef2b23fd 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -18,4 +18,3 @@ class TestUrlLengthMiddleware(TestCase):
         spider = Spider('foo')
         out = list(mw.process_spider_output(res, reqs, spider))
         self.assertEqual(out, [short_url_req])
-
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 535095b8d..9455172fa 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -244,4 +244,3 @@ class SequenceExcludeTest(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
-
diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py
index 583105673..2524153ea 100644
--- a/tests/test_utils_http.py
+++ b/tests/test_utils_http.py
@@ -16,5 +16,3 @@ class ChunkedTest(unittest.TestCase):
             "This is the data in the first chunk\r\n" +
             "and this is the second one\r\n" +
             "consequence")
-
-
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index 045e72117..d9de1ce77 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -34,4 +34,3 @@ class UtilsSpidersTestCase(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
-

From d874c4d90bcf96c7e5b507babaa2a45a233da506 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Thu, 7 Nov 2019 22:02:17 +0500
Subject: [PATCH 2354/2974] Remove the old Python 2 PyPy installation code from
 .travis.yml (#4138)

---
 .travis.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index f5b9d3f72..0e77af9fd 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,13 +27,6 @@ matrix:
       python: 3.6
 install:
   - |
-      if [ "$TOXENV" = "pypy" ]; then
-        export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable"
-        wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
-        tar -jxf ${PYPY_VERSION}.tar.bz2
-        virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
-        source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
-      fi
       if [ "$TOXENV" = "pypy3" ]; then
         export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
         wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"

From aef98188facfc79dc574d8a86200b4e95b96b880 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 7 Nov 2019 18:06:55 +0100
Subject: [PATCH 2355/2974] Improve the details about request serialization
 requirements for JOBDIR

---
 docs/topics/jobs.rst | 31 ++++---------------------------
 1 file changed, 4 insertions(+), 27 deletions(-)

diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 9fd311c69..f5542495b 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -71,34 +71,11 @@ on cookies.
 Request serialization
 ---------------------
 
-Requests must be serializable by the ``pickle`` module, in order for persistence
-to work, so you should make sure that your requests are serializable.
-
-The most common issue here is to use ``lambda`` functions on request callbacks that
-can't be persisted.
-
-So, for example, this won't work::
-
-    def some_callback(self, response):
-        somearg = 'test'
-        return scrapy.Request('http://www.example.com',
-                              callback=lambda r: self.other_callback(r, somearg))
-
-    def other_callback(self, response, somearg):
-        print("the argument passed is: %s" % somearg)
-
-But this will::
-
-    def some_callback(self, response):
-        somearg = 'test'
-        return scrapy.Request('http://www.example.com',
-                              callback=self.other_callback, cb_kwargs={'somearg': somearg})
-
-    def other_callback(self, response, somearg):
-        print("the argument passed is: %s" % somearg)
+For persistence to work, :class:`~scrapy.http.Request` objects must be
+serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
+values passed to their ``__init__`` method, which must be methods of the
+runnning :class:`~scrapy.spiders.Spider` class.
 
 If you wish to log the requests that couldn't be serialized, you can set the
 :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
 It is ``False`` by default.
-
-.. _pickle: https://docs.python.org/library/pickle.html

From 44f19df3119d553aa5c002321bd901a424c4bb2c Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Fri, 8 Nov 2019 11:32:50 -0300
Subject: [PATCH 2356/2974] [test] Update mitmproxy version
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 tests/requirements-py3.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index c2b16bec6..7abb66b9c 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,7 +1,7 @@
 # Tests requirements
 jmespath
 mitmproxy; python_version >= '3.6'
-mitmproxy==3.0.4; python_version < '3.6'
+mitmproxy<4.0.0; python_version < '3.6'
 pytest
 pytest-cov
 pytest-twisted

From 1df5755699eac5a98239ae73dfb82908706bf03b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 8 Nov 2019 16:00:10 +0100
Subject: [PATCH 2357/2974] Set the bases for testing code examples from the
 documentation

---
 docs/conftest.py           | 16 ++++++++++++++++
 pytest.ini                 | 20 +++++++++++++++++++-
 tests/requirements-py3.txt |  1 +
 tox.ini                    |  6 +++---
 4 files changed, 39 insertions(+), 4 deletions(-)
 create mode 100644 docs/conftest.py

diff --git a/docs/conftest.py b/docs/conftest.py
new file mode 100644
index 000000000..91c1d4428
--- /dev/null
+++ b/docs/conftest.py
@@ -0,0 +1,16 @@
+from doctest import ELLIPSIS
+
+from sybil import Sybil
+from sybil.parsers.codeblock import CodeBlockParser
+from sybil.parsers.doctest import DocTestParser
+from sybil.parsers.skip import skip
+
+
+pytest_collect_file = Sybil(
+    parsers=[
+        DocTestParser(optionflags=ELLIPSIS),
+        CodeBlockParser(future_imports=['print_function']),
+        skip,
+    ],
+    pattern='*.rst',
+).pytest()
diff --git a/pytest.ini b/pytest.ini
index db5bee228..8c5a2cd54 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -2,7 +2,25 @@
 usefixtures = chdir
 python_files=test_*.py __init__.py
 python_classes=
-addopts = --doctest-modules --assert=plain
+addopts =
+    --assert=plain
+    --doctest-modules
+    --ignore=docs/_ext
+    --ignore=docs/conf.py
+    --ignore=docs/intro/tutorial.rst
+    --ignore=docs/news.rst
+    --ignore=docs/topics/commands.rst
+    --ignore=docs/topics/debug.rst
+    --ignore=docs/topics/developer-tools.rst
+    --ignore=docs/topics/dynamic-content.rst
+    --ignore=docs/topics/items.rst
+    --ignore=docs/topics/leaks.rst
+    --ignore=docs/topics/loaders.rst
+    --ignore=docs/topics/selectors.rst
+    --ignore=docs/topics/shell.rst
+    --ignore=docs/topics/stats.rst
+    --ignore=docs/topics/telnetconsole.rst
+    --ignore=docs/utils
 twisted = 1
 flake8-ignore =
     # extras
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index dd5b23cc3..2e8d319d2 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -4,6 +4,7 @@ pytest
 pytest-cov
 pytest-twisted
 pytest-xdist
+sybil
 testfixtures
 
 # optional for shell wrapper tests
diff --git a/tox.ini b/tox.ini
index cc3463a4d..3668058c3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -21,7 +21,7 @@ passenv =
     GCS_TEST_FILE_URI
     GCS_PROJECT_ID
 commands =
-    py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
+    py.test --cov=scrapy --cov-report= {posargs:docs scrapy tests}
 
 [testenv:py35]
 basepython = python3.5
@@ -60,7 +60,7 @@ basepython = python3.8
 [testenv:pypy3]
 basepython = pypy3
 commands =
-    py.test {posargs:scrapy tests}
+    py.test {posargs:docs scrapy tests}
 
 [testenv:flake8]
 basepython = python3.8
@@ -68,7 +68,7 @@ deps =
     {[testenv]deps}
     pytest-flake8
 commands =
-    py.test --flake8 {posargs:scrapy tests}
+    py.test --flake8 {posargs:docs scrapy tests}
 
 [docs]
 changedir = docs

From 6cde428af43a5c8268208c6e4e239ab5ce507af4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 Nov 2019 12:26:40 -0300
Subject: [PATCH 2358/2974] Remove deprecated MergeDict class

---
 scrapy/utils/datatypes.py | 59 ---------------------------------------
 1 file changed, 59 deletions(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 56d4d1b8e..e194a7613 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -235,65 +235,6 @@ class CaselessDict(dict):
         return dict.pop(self, self.normkey(key), *args)
 
 
-class MergeDict(object):
-    """
-    A simple class for creating new "virtual" dictionaries that actually look
-    up values in more than one dictionary, passed in the constructor.
-
-    If a key appears in more than one of the given dictionaries, only the
-    first occurrence will be used.
-    """
-    def __init__(self, *dicts):
-        warnings.warn(
-            "scrapy.utils.datatypes.MergeDict is deprecated in favor "
-            "of collections.ChainMap (introduced in Python 3.3)",
-            category=ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-        self.dicts = dicts
-
-    def __getitem__(self, key):
-        for dict_ in self.dicts:
-            try:
-                return dict_[key]
-            except KeyError:
-                pass
-        raise KeyError
-
-    def __copy__(self):
-        return self.__class__(*self.dicts)
-
-    def get(self, key, default=None):
-        try:
-            return self[key]
-        except KeyError:
-            return default
-
-    def getlist(self, key):
-        for dict_ in self.dicts:
-            if key in dict_.keys():
-                return dict_.getlist(key)
-        return []
-
-    def items(self):
-        item_list = []
-        for dict_ in self.dicts:
-            item_list.extend(dict_.items())
-        return item_list
-
-    def has_key(self, key):
-        for dict_ in self.dicts:
-            if key in dict_:
-                return True
-        return False
-
-    __contains__ = has_key
-
-    def copy(self):
-        """Returns a copy of this object."""
-        return self.__copy__()
-
-
 class LocalCache(collections.OrderedDict):
     """Dictionary with a finite number of keys.
 

From b6bbb2819707a2202c87abd6b3dba6af13a7cc85 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 Nov 2019 22:13:03 -0300
Subject: [PATCH 2359/2974] PEP8 adjustments

---
 scrapy/crawler.py            |  1 -
 scrapy/exporters.py          |  1 -
 scrapy/http/cookies.py       |  2 +-
 scrapy/link.py               |  2 ++
 tests/test_pipeline_media.py |  2 --
 tests/test_proxy_connect.py  | 53 ++++++++++++++++++++----------------
 tests/test_utils_python.py   |  2 +-
 7 files changed, 34 insertions(+), 29 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 4d7d9bac4..ab62c678c 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,6 +1,5 @@
 import logging
 import signal
-import sys
 import warnings
 
 from twisted.internet import reactor, defer
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index f2999daea..e31ab1780 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -4,7 +4,6 @@ Item Exporters are used to export/serialize items into different formats.
 
 import csv
 import io
-import sys
 import pprint
 import marshal
 import warnings
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index c39de0b52..0903fd4f8 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -165,7 +165,7 @@ class WrappedRequest(object):
 
     def get_header(self, name, default=None):
         return to_unicode(self.request.headers.get(name, default),
-                             errors='replace')
+                          errors='replace')
 
     def header_items(self):
         return [
diff --git a/scrapy/link.py b/scrapy/link.py
index be1888ef0..a809c5ca4 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -4,6 +4,8 @@ This module defines the Link object used in Link extractors.
 For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
 """
+
+
 class Link(object):
     """Link objects represent an extracted link by the LinkExtractor."""
 
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 70f11466b..0d23f51cc 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,5 +1,3 @@
-import sys
-
 from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.python.failure import Failure
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index bf56136b1..4147dc944 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -28,17 +28,24 @@ from mitmproxy.tools.main import mitmdump
 sys.argv[0] = "mitmdump"
 sys.exit(mitmdump())
         """
-        cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
-            'keys', 'mitmproxy-ca.pem')
-        self.proc = Popen([sys.executable,
-                           '-c', script,
-                           '--listen-host', '127.0.0.1',
-                           '--listen-port', '0',
-                           '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
-                           '--certs', cert_path,
-                           '--ssl-insecure',
-                           ],
-                           stdout=PIPE, env=get_testenv())
+        cert_path = os.path.join(
+            os.path.abspath(os.path.dirname(__file__)),
+            'keys',
+            'mitmproxy-ca.pem'
+        )
+        self.proc = Popen(
+            [
+                sys.executable,
+                '-c', script,
+                '--listen-host', '127.0.0.1',
+                '--listen-port', '0',
+                '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
+                '--certs', cert_path,
+                '--ssl-insecure',
+            ],
+            stdout=PIPE,
+            env=get_testenv()
+        )
         line = self.proc.stdout.readline().decode('utf-8')
         host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)
         address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port)
@@ -75,9 +82,9 @@ class ProxyConnectTestCase(TestCase):
     @defer.inlineCallbacks
     def test_https_connect_tunnel(self):
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as logs:
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
+        self._assert_got_response_code(200, logs)
 
     @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
     @defer.inlineCallbacks
@@ -85,35 +92,35 @@ class ProxyConnectTestCase(TestCase):
         proxy = os.environ['https_proxy']
         os.environ['https_proxy'] = proxy + '?noconnect'
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as logs:
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
+        self._assert_got_response_code(200, logs)
 
     @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as logs:
             yield crawler.crawl("https://localhost:99999/status?n=200")
-        self._assert_got_tunnel_error(l)
+        self._assert_got_tunnel_error(logs)
 
     @defer.inlineCallbacks
     def test_https_tunnel_auth_error(self):
         os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as logs:
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         # The proxy returns a 407 error code but it does not reach the client;
         # he just sees a TunnelError.
-        self._assert_got_tunnel_error(l)
+        self._assert_got_tunnel_error(logs)
 
     @defer.inlineCallbacks
     def test_https_tunnel_without_leak_proxy_authorization_header(self):
         request = Request(self.mockserver.url("/echo", is_secure=True))
         crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as l:
+        with LogCapture() as logs:
             yield crawler.crawl(seed=request)
-        self._assert_got_response_code(200, l)
+        self._assert_got_response_code(200, logs)
         echo = json.loads(crawler.spider.meta['responses'][0].text)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
@@ -122,9 +129,9 @@ class ProxyConnectTestCase(TestCase):
     def test_https_noconnect_auth_error(self):
         os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as logs:
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(407, l)
+        self._assert_got_response_code(407, logs)
 
     def _assert_got_response_code(self, code, log):
         print(log)
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index faf0d4b73..b36c2a5e3 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -7,7 +7,7 @@ from itertools import count
 
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
-    WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode,
+    WeakKeyCache, get_func_args, to_bytes, to_unicode,
     without_none_values, MutableChain)
 
 

From 084a1cda6dfd94a3671d49c362db2ee6ea88a10d Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Mon, 11 Nov 2019 15:41:00 +0530
Subject: [PATCH 2360/2974] Adding test

---
 tests/test_feedexport.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 8c0e5cd3d..16916f728 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -415,7 +415,7 @@ class FeedExportTest(unittest.TestCase):
                 spider_cls.start_urls = [s.url('/')]
                 yield runner.crawl(spider_cls)
 
-            with open(res_path, 'rb') as f:
+            with open(res_uri, 'rb') as f:
                 content = f.read()
 
         finally:
@@ -850,12 +850,12 @@ class FeedExportTest(unittest.TestCase):
     def test_pathlib_uri(self):
         tmpdir = tempfile.mkdtemp()
         feed_uri = Path(tmpdir) / 'res'
-        feed_uri=str(feed_uri)
         res_uri = urljoin('file:', pathname2url(feed_uri))
         settings = {
             'FEED_FORMAT': 'csv',
             'FEED_STORE_EMPTY': True,
-            'FEED_URI': res_uri,
+            'FEED_URI': feed_uri,
+            'FEED_URI_ISPATH' : True
         }
         
         data = yield self.exported_no_data(settings)

From 0042c389eb2d44d017bc8af069c7dd7ebd9319bd Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Mon, 11 Nov 2019 15:57:58 +0530
Subject: [PATCH 2361/2974] Adding test

---
 tests/test_feedexport.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 16916f728..275579959 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -855,7 +855,6 @@ class FeedExportTest(unittest.TestCase):
             'FEED_FORMAT': 'csv',
             'FEED_STORE_EMPTY': True,
             'FEED_URI': feed_uri,
-            'FEED_URI_ISPATH' : True
         }
         
         data = yield self.exported_no_data(settings)

From 9e6e2dde2b7736278075d6a8268511a3bc44b8b5 Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Mon, 11 Nov 2019 16:10:37 +0530
Subject: [PATCH 2362/2974] Adding test

---
 tests/test_feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 275579959..b6e4a5449 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -415,7 +415,7 @@ class FeedExportTest(unittest.TestCase):
                 spider_cls.start_urls = [s.url('/')]
                 yield runner.crawl(spider_cls)
 
-            with open(res_uri, 'rb') as f:
+            with open(defaults['FEED_URI'], 'rb') as f:
                 content = f.read()
 
         finally:

From 970c3be1603483a61637b94afdb965eb24342744 Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Mon, 11 Nov 2019 18:34:15 +0530
Subject: [PATCH 2363/2974] Added Test

---
 scrapy/extensions/feedexport.py | 2 +-
 tests/test_feedexport.py        | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 981efee55..8dacce459 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -201,7 +201,7 @@ class FeedExporter(object):
         self.settings = settings
         if not settings['FEED_URI']:
             raise NotConfigured
-        self.urifmt=str(settings['FEED_URI'])
+        self.urifmt = str(settings['FEED_URI'])
         self.format = settings['FEED_FORMAT'].lower()
         self.export_encoding = settings['FEED_EXPORT_ENCODING']
         self.storages = self._load_components('FEED_STORAGES')
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index b6e4a5449..11d32bd14 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -407,6 +407,7 @@ class FeedExportTest(unittest.TestCase):
         defaults = {
             'FEED_URI': res_uri,
             'FEED_FORMAT': 'csv',
+            'FEED_PATH': res_path
         }
         defaults.update(settings or {})
         try:
@@ -415,7 +416,7 @@ class FeedExportTest(unittest.TestCase):
                 spider_cls.start_urls = [s.url('/')]
                 yield runner.crawl(spider_cls)
 
-            with open(defaults['FEED_URI'], 'rb') as f:
+            with open(defaults['FEED_PATH'], 'rb') as f:
                 content = f.read()
 
         finally:
@@ -855,8 +856,8 @@ class FeedExportTest(unittest.TestCase):
             'FEED_FORMAT': 'csv',
             'FEED_STORE_EMPTY': True,
             'FEED_URI': feed_uri,
+            'FEED_PATH': feed_uri
         }
-        
         data = yield self.exported_no_data(settings)
         self.assertEqual(data, b'')
-        shutil.rmtree(tmpdir, ignore_errors=True)
\ No newline at end of file
+        shutil.rmtree(tmpdir, ignore_errors=True)

From 0c2dcd5092eccf08399f0838d86d598329cc3a28 Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Mon, 11 Nov 2019 18:35:50 +0530
Subject: [PATCH 2364/2974] Added Test

---
 requirements-py2.txt | 19 -------------------
 1 file changed, 19 deletions(-)
 delete mode 100644 requirements-py2.txt

diff --git a/requirements-py2.txt b/requirements-py2.txt
deleted file mode 100644
index 42e057417..000000000
--- a/requirements-py2.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-parsel>=1.5.0
-PyDispatcher>=2.0.5
-w3lib>=1.17.0
-protego>=0.1.15
-
-pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
-queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
-cryptography>=2.0  # Earlier versions would fail to install
-
-# Reference versions taken from
-# https://packages.ubuntu.com/xenial/python/
-# https://packages.ubuntu.com/xenial/zope/
-cssselect>=0.9.1
-lxml>=3.5.0
-service_identity>=16.0.0
-six>=1.10.0
-Twisted>=16.0.0
-zope.interface>=4.1.3
-pathlib2>=2.0

From f39ff4945854fb5d98389fabfa2d2e5a059c0643 Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Mon, 11 Nov 2019 18:54:21 +0530
Subject: [PATCH 2365/2974] Added Test

---
 tests/test_feedexport.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 11d32bd14..9b07c2051 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -851,7 +851,6 @@ class FeedExportTest(unittest.TestCase):
     def test_pathlib_uri(self):
         tmpdir = tempfile.mkdtemp()
         feed_uri = Path(tmpdir) / 'res'
-        res_uri = urljoin('file:', pathname2url(feed_uri))
         settings = {
             'FEED_FORMAT': 'csv',
             'FEED_STORE_EMPTY': True,

From 50eaabe1fc540218f9f04197810367154eb3e102 Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Mon, 11 Nov 2019 20:00:26 +0530
Subject: [PATCH 2366/2974] Added Test

---
 tests/test_feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 9b07c2051..2819f8f0b 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -416,7 +416,7 @@ class FeedExportTest(unittest.TestCase):
                 spider_cls.start_urls = [s.url('/')]
                 yield runner.crawl(spider_cls)
 
-            with open(defaults['FEED_PATH'], 'rb') as f:
+            with open(str(defaults['FEED_PATH']), 'rb') as f:
                 content = f.read()
 
         finally:

From 79d2f99995a12ffc19ab7cd2fda10112cf5e9b65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 12 Nov 2019 08:08:50 +0100
Subject: [PATCH 2367/2974] Make tutorial doctests pass

---
 docs/_tests/quotes1.html | 281 +++++++++++++++++++++++++++++++++++++++
 docs/conftest.py         |  17 ++-
 docs/intro/tutorial.rst  |  23 ++--
 pytest.ini               |   1 -
 4 files changed, 310 insertions(+), 12 deletions(-)
 create mode 100644 docs/_tests/quotes1.html

diff --git a/docs/_tests/quotes1.html b/docs/_tests/quotes1.html
new file mode 100644
index 000000000..71aff8847
--- /dev/null
+++ b/docs/_tests/quotes1.html
@@ -0,0 +1,281 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+	<meta charset="UTF-8">
+	<title>Quotes to Scrape</title>
+    <link rel="stylesheet" href="/static/bootstrap.min.css">
+    <link rel="stylesheet" href="/static/main.css">
+</head>
+<body>
+    <div class="container">
+        <div class="row header-box">
+            <div class="col-md-8">
+                <h1>
+                    <a href="/" style="text-decoration: none">Quotes to Scrape</a>
+                </h1>
+            </div>
+            <div class="col-md-4">
+                <p>
+                
+                    <a href="/login">Login</a>
+                
+                </p>
+            </div>
+        </div>
+    
+
+<div class="row">
+    <div class="col-md-8">
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>
+        <span>by <small class="author" itemprop="author">Albert Einstein</small>
+        <a href="/author/Albert-Einstein">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="change,deep-thoughts,thinking,world" /    > 
+            
+            <a class="tag" href="/tag/change/page/1/">change</a>
+            
+            <a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
+            
+            <a class="tag" href="/tag/thinking/page/1/">thinking</a>
+            
+            <a class="tag" href="/tag/world/page/1/">world</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“It is our choices, Harry, that show what we truly are, far more than our abilities.”</span>
+        <span>by <small class="author" itemprop="author">J.K. Rowling</small>
+        <a href="/author/J-K-Rowling">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="abilities,choices" /    > 
+            
+            <a class="tag" href="/tag/abilities/page/1/">abilities</a>
+            
+            <a class="tag" href="/tag/choices/page/1/">choices</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</span>
+        <span>by <small class="author" itemprop="author">Albert Einstein</small>
+        <a href="/author/Albert-Einstein">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="inspirational,life,live,miracle,miracles" /    > 
+            
+            <a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
+            
+            <a class="tag" href="/tag/life/page/1/">life</a>
+            
+            <a class="tag" href="/tag/live/page/1/">live</a>
+            
+            <a class="tag" href="/tag/miracle/page/1/">miracle</a>
+            
+            <a class="tag" href="/tag/miracles/page/1/">miracles</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</span>
+        <span>by <small class="author" itemprop="author">Jane Austen</small>
+        <a href="/author/Jane-Austen">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="aliteracy,books,classic,humor" /    > 
+            
+            <a class="tag" href="/tag/aliteracy/page/1/">aliteracy</a>
+            
+            <a class="tag" href="/tag/books/page/1/">books</a>
+            
+            <a class="tag" href="/tag/classic/page/1/">classic</a>
+            
+            <a class="tag" href="/tag/humor/page/1/">humor</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“Imperfection is beauty, madness is genius and it&#39;s better to be absolutely ridiculous than absolutely boring.”</span>
+        <span>by <small class="author" itemprop="author">Marilyn Monroe</small>
+        <a href="/author/Marilyn-Monroe">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="be-yourself,inspirational" /    > 
+            
+            <a class="tag" href="/tag/be-yourself/page/1/">be-yourself</a>
+            
+            <a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“Try not to become a man of success. Rather become a man of value.”</span>
+        <span>by <small class="author" itemprop="author">Albert Einstein</small>
+        <a href="/author/Albert-Einstein">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="adulthood,success,value" /    > 
+            
+            <a class="tag" href="/tag/adulthood/page/1/">adulthood</a>
+            
+            <a class="tag" href="/tag/success/page/1/">success</a>
+            
+            <a class="tag" href="/tag/value/page/1/">value</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“It is better to be hated for what you are than to be loved for what you are not.”</span>
+        <span>by <small class="author" itemprop="author">André Gide</small>
+        <a href="/author/Andre-Gide">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="life,love" /    > 
+            
+            <a class="tag" href="/tag/life/page/1/">life</a>
+            
+            <a class="tag" href="/tag/love/page/1/">love</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“I have not failed. I&#39;ve just found 10,000 ways that won&#39;t work.”</span>
+        <span>by <small class="author" itemprop="author">Thomas A. Edison</small>
+        <a href="/author/Thomas-A-Edison">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="edison,failure,inspirational,paraphrased" /    > 
+            
+            <a class="tag" href="/tag/edison/page/1/">edison</a>
+            
+            <a class="tag" href="/tag/failure/page/1/">failure</a>
+            
+            <a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
+            
+            <a class="tag" href="/tag/paraphrased/page/1/">paraphrased</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“A woman is like a tea bag; you never know how strong it is until it&#39;s in hot water.”</span>
+        <span>by <small class="author" itemprop="author">Eleanor Roosevelt</small>
+        <a href="/author/Eleanor-Roosevelt">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="misattributed-eleanor-roosevelt" /    > 
+            
+            <a class="tag" href="/tag/misattributed-eleanor-roosevelt/page/1/">misattributed-eleanor-roosevelt</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“A day without sunshine is like, you know, night.”</span>
+        <span>by <small class="author" itemprop="author">Steve Martin</small>
+        <a href="/author/Steve-Martin">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="humor,obvious,simile" /    > 
+            
+            <a class="tag" href="/tag/humor/page/1/">humor</a>
+            
+            <a class="tag" href="/tag/obvious/page/1/">obvious</a>
+            
+            <a class="tag" href="/tag/simile/page/1/">simile</a>
+            
+        </div>
+    </div>
+
+    <nav>
+        <ul class="pager">
+            
+            
+            <li class="next">
+                <a href="/page/2/">Next <span aria-hidden="true">&rarr;</span></a>
+            </li>
+            
+        </ul>
+    </nav>
+    </div>
+    <div class="col-md-4 tags-box">
+        
+            <h2>Top Ten tags</h2>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 28px" href="/tag/love/">love</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 26px" href="/tag/inspirational/">inspirational</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 26px" href="/tag/life/">life</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 24px" href="/tag/humor/">humor</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 22px" href="/tag/books/">books</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 14px" href="/tag/reading/">reading</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 10px" href="/tag/friendship/">friendship</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 8px" href="/tag/friends/">friends</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 8px" href="/tag/truth/">truth</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 6px" href="/tag/simile/">simile</a>
+            </span>
+            
+        
+    </div>
+</div>
+
+    </div>
+    <footer class="footer">
+        <div class="container">
+            <p class="text-muted">
+                Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
+            </p>
+            <p class="copyright">
+                Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
+            </p>
+        </div>
+    </footer>
+</body>
+</html>
\ No newline at end of file
diff --git a/docs/conftest.py b/docs/conftest.py
index 91c1d4428..8c735e838 100644
--- a/docs/conftest.py
+++ b/docs/conftest.py
@@ -1,16 +1,29 @@
-from doctest import ELLIPSIS
+import os
+from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
 
+from scrapy.http.response.html import HtmlResponse
 from sybil import Sybil
 from sybil.parsers.codeblock import CodeBlockParser
 from sybil.parsers.doctest import DocTestParser
 from sybil.parsers.skip import skip
 
 
+def load_response(url, filename):
+    input_path = os.path.join(os.path.dirname(__file__), '_tests', filename)
+    with open(input_path, 'rb') as input_file:
+        return HtmlResponse(url, body=input_file.read())
+
+
+def setup(namespace):
+    namespace['load_response'] = load_response
+
+
 pytest_collect_file = Sybil(
     parsers=[
-        DocTestParser(optionflags=ELLIPSIS),
+        DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
         CodeBlockParser(future_imports=['print_function']),
         skip,
     ],
     pattern='*.rst',
+    setup=setup,
 ).pytest()
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 0629b9e19..996e3b475 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -235,13 +235,16 @@ You will see something like::
     [s]   shelp()           Shell help (print this help)
     [s]   fetch(req_or_url) Fetch request (or URL) and update local objects
     [s]   view(response)    View response in a browser
-    >>>
 
 Using the shell, you can try selecting elements using `CSS`_ with the response
-object::
+object:
 
-    >>> response.css('title')
-    [<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
+.. invisible-code-block: python
+
+    response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
+
+>>> response.css('title')
+[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
 
 The result of running ``response.css('title')`` is a list-like object called
 :class:`~scrapy.selector.SelectorList`, which represents a list of
@@ -372,6 +375,9 @@ we want::
 We get a list of selectors for the quote HTML elements with::
 
     >>> response.css("div.quote")
+    [<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
+     <Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
+     ...]
 
 Each of the selectors returned by the query above allows us to run further
 queries over their sub-elements. Let's assign the first selector to a
@@ -404,10 +410,9 @@ quotes elements and put them together into a Python dictionary::
     ...     author = quote.css("small.author::text").get()
     ...     tags = quote.css("div.tags a.tag::text").getall()
     ...     print(dict(text=text, author=author, tags=tags))
-    {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
-    {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
-        ... a few more of these, omitted for brevity
-    >>>
+    {'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
+    {'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
+    ...
 
 Extracting data in our spider
 -----------------------------
@@ -521,7 +526,7 @@ There is also an ``attrib`` property available
 (see :ref:`selecting-attributes` for more)::
 
     >>> response.css('li.next a').attrib['href']
-    '/page/2'
+    '/page/2/'
 
 Let's see now our spider modified to recursively follow the link to the next
 page, extracting data from it::
diff --git a/pytest.ini b/pytest.ini
index 8c5a2cd54..3f1cc5800 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -7,7 +7,6 @@ addopts =
     --doctest-modules
     --ignore=docs/_ext
     --ignore=docs/conf.py
-    --ignore=docs/intro/tutorial.rst
     --ignore=docs/news.rst
     --ignore=docs/topics/commands.rst
     --ignore=docs/topics/debug.rst

From 7b7bb028f45a06b3444950521ea582d0e83691ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 12 Nov 2019 08:49:06 +0100
Subject: [PATCH 2368/2974] Use intersphinx for links to the Sphinx
 documentation

---
 docs/conf.py          |  1 +
 docs/contributing.rst | 17 ++++++++---------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 34dd5bcb7..0cc1dc22b 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -273,4 +273,5 @@ coverage_ignore_pyobjects = [
 
 intersphinx_mapping = {
     'python': ('https://docs.python.org/3', None),
+    'sphinx': ('https://www.sphinx-doc.org/en/stable', None),
 }
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 28dea74de..f084bd23d 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -177,20 +177,19 @@ Documentation policies
 ======================
 
 For reference documentation of API members (classes, methods, etc.) use
-docstrings and make sure that the Sphinx documentation uses the autodoc_
-extension to pull the docstrings. API reference documentation should follow
-docstring conventions (`PEP 257`_) and be IDE-friendly: short, to the point,
-and it may provide short examples.
+docstrings and make sure that the Sphinx documentation uses the
+:mod:`~sphinx.ext.autodoc` extension to pull the docstrings. API reference
+documentation should follow docstring conventions (`PEP 257`_) and be
+IDE-friendly: short, to the point, and it may provide short examples.
 
 Other types of documentation, such as tutorials or topics, should be covered in
 files within the ``docs/`` directory. This includes documentation that is
 specific to an API member, but goes beyond API reference documentation.
 
-In any case, if something is covered in a docstring, use the autodoc_
-extension to pull the docstring into the documentation instead of duplicating
-the docstring in files within the ``docs/`` directory.
-
-.. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html
+In any case, if something is covered in a docstring, use the
+:mod:`~sphinx.ext.autodoc` extension to pull the docstring into the
+documentation instead of duplicating the docstring in files within the
+``docs/`` directory.
 
 Tests
 =====

From 8a6a063778d45e8ba68a59558d2930332c1e9a83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 12 Nov 2019 10:23:19 +0100
Subject: [PATCH 2369/2974] Allow opening the source code from the API
 documentation

---
 docs/conf.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/conf.py b/docs/conf.py
index 34dd5bcb7..b09000de0 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -31,6 +31,7 @@ extensions = [
     'sphinx.ext.autodoc',
     'sphinx.ext.coverage',
     'sphinx.ext.intersphinx',
+    'sphinx.ext.viewcode',
 ]
 
 # Add any paths that contain templates here, relative to this directory.

From 4b8b0345e58ee5990bd0c28205b5eb0b892680d1 Mon Sep 17 00:00:00 2001
From: purvaudai <purvaudai99@gmail.com>
Date: Tue, 12 Nov 2019 18:17:15 +0530
Subject: [PATCH 2370/2974] Mades Changes as per review

---
 requirements-py3.txt     | 1 -
 tests/test_feedexport.py | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/requirements-py3.txt b/requirements-py3.txt
index 77296b91b..2c98e6f6d 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -16,4 +16,3 @@ lxml>=3.5.0
 service_identity>=16.0.0
 six>=1.10.0
 zope.interface>=4.1.3
-pathlib2>=2.0
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 2819f8f0b..1f46ac04a 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -29,7 +29,7 @@ from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get
 from scrapy.utils.python import to_native_str
 from scrapy.utils.project import get_project_settings
 
-from pathlib2 import Path
+from pathlib import Path
 
 
 class FileFeedStorageTest(unittest.TestCase):

From 414e6e2fd568e0dfae699873d3c1ccd865261d09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 13 Nov 2019 07:56:45 +0100
Subject: [PATCH 2371/2974] Skip a doctest in Python 3.5- because of dictionary
 changes

---
 docs/intro/tutorial.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 996e3b475..30b1ddeab 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -402,6 +402,12 @@ to get all of them::
     >>> tags
     ['change', 'deep-thoughts', 'thinking', 'world']
 
+.. invisible-code-block: python
+
+  from sys import version_info
+
+.. skip: next if(version_info <= (3, 5), reason="Only Python 3.6+ dictionaries match the output")
+
 Having figured out how to extract each bit, we can now iterate over all the
 quotes elements and put them together into a Python dictionary::
 

From b642a1fca29852adf0ba3ddd62c2ecfdbaf9610e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 13 Nov 2019 09:14:20 +0100
Subject: [PATCH 2372/2974] Fix doctest skipping based on the running Python
 version

---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 30b1ddeab..6b15a5fbd 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -406,7 +406,7 @@ to get all of them::
 
   from sys import version_info
 
-.. skip: next if(version_info <= (3, 5), reason="Only Python 3.6+ dictionaries match the output")
+.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
 
 Having figured out how to extract each bit, we can now iterate over all the
 quotes elements and put them together into a Python dictionary::

From 76c31094dff2920778b42ed746c6a9cd5b08f4e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 13 Nov 2019 09:28:48 +0100
Subject: [PATCH 2373/2974] Install the sphinx-notfound-page Sphinx extension

---
 docs/conf.py          | 1 +
 docs/requirements.txt | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index 6ab5959d5..935c3c9a1 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -27,6 +27,7 @@ sys.path.insert(0, path.dirname(path.dirname(__file__)))
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
+    'notfound.extension',
     'scrapydocs',
     'sphinx.ext.autodoc',
     'sphinx.ext.coverage',
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 379da9994..f9db85146 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,2 +1,3 @@
 Sphinx>=2.1
-sphinx_rtd_theme
\ No newline at end of file
+sphinx-notfound-page
+sphinx_rtd_theme

From a3a3107bc45483e0d7c77e45412121fdb8d539b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 13 Nov 2019 09:46:54 +0100
Subject: [PATCH 2374/2974] MutableChain: return self from __iter__

---
 scrapy/utils/python.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index ea5193f12..64402a2bb 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -388,9 +388,7 @@ class MutableChain(object):
         self.data = chain(self.data, *iterables)
 
     def __iter__(self):
-        return self.data.__iter__()
+        return self
 
     def __next__(self):
         return next(self.data)
-
-    next = __next__

From 33ef24c757c797c804c8fd242b8ab5705219b452 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 13 Nov 2019 10:52:05 +0100
Subject: [PATCH 2375/2974] =?UTF-8?q?Add=20missing=20whitespace=20after=20?=
 =?UTF-8?q?=E2=80=98,=E2=80=99,=20=E2=80=98;=E2=80=99=20or=20=E2=80=98:?=
 =?UTF-8?q?=E2=80=99?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pytest.ini                        | 16 ++++++++--------
 scrapy/core/spidermw.py           |  2 +-
 tests/test_http_request.py        |  4 ++--
 tests/test_item.py                |  2 +-
 tests/test_linkextractors.py      |  4 ++--
 tests/test_logformatter.py        |  2 +-
 tests/test_utils_conf.py          |  2 +-
 tests/test_utils_console.py       |  2 +-
 tests/test_utils_misc/__init__.py |  2 +-
 9 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 8c5a2cd54..984930459 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -48,7 +48,7 @@ flake8-ignore =
     scrapy/core/engine.py E261 E501 E128 E127 E306 E502
     scrapy/core/scheduler.py E501
     scrapy/core/scraper.py E501 E306 E261 E128 W504
-    scrapy/core/spidermw.py E501 E731 E502 E231 E126 E226
+    scrapy/core/spidermw.py E501 E731 E502 E126 E226
     scrapy/core/downloader/__init__.py F401 E501
     scrapy/core/downloader/contextfactory.py E501 E128 E126
     scrapy/core/downloader/middleware.py E501 E502
@@ -214,13 +214,13 @@ flake8-ignore =
     tests/test_feedexport.py E501 F401 F841 E241
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E302 E501
-    tests/test_http_request.py F401 E402 E501 E231 E261 E127 E128 W293 E502 E128 E502 E126 E123
+    tests/test_http_request.py F401 E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
     tests/test_http_response.py E501 E301 E502 E128 E265
-    tests/test_item.py E701 E128 E231 F841 E306
+    tests/test_item.py E701 E128 F841 E306
     tests/test_link.py E501
-    tests/test_linkextractors.py E501 E128 E231 E124
+    tests/test_linkextractors.py E501 E128 E124
     tests/test_loader.py E302 E501 E731 E303 E741 E128 E117 E241
-    tests/test_logformatter.py E128 E501 E231 E122 E302
+    tests/test_logformatter.py E128 E501 E122 E302
     tests/test_mail.py E302 E128 E501 E305
     tests/test_middleware.py E302 E501 E128
     tests/test_pipeline_crawl.py E131 E501 E128 E126
@@ -239,8 +239,8 @@ flake8-ignore =
     tests/test_spidermiddleware_output_chain.py F401 E501 E302 W293 E226
     tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 E241 E121
     tests/test_squeues.py E501 E302 E701 E741
-    tests/test_utils_conf.py E501 E231 E303 E128
-    tests/test_utils_console.py E302 E231
+    tests/test_utils_conf.py E501 E303 E128
+    tests/test_utils_console.py E302
     tests/test_utils_curl.py E501
     tests/test_utils_datatypes.py E402 E501 E305
     tests/test_utils_defer.py E306 E261 E501 E302 F841 E226
@@ -269,4 +269,4 @@ flake8-ignore =
     tests/test_spiderloader/test_spiders/spider2.py E302
     tests/test_spiderloader/test_spiders/spider3.py E302
     tests/test_spiderloader/test_spiders/nested/spider4.py E302
-    tests/test_utils_misc/__init__.py E501 E231
+    tests/test_utils_misc/__init__.py E501
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index b5f9837ff..00cee3ada 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -36,7 +36,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         self.methods['process_spider_exception'].appendleft(getattr(mw, 'process_spider_exception', None))
 
     def scrape_response(self, scrape_func, response, request, spider):
-        fname = lambda f:'%s.%s' % (
+        fname = lambda f: '%s.%s' % (
                 six.get_method_self(f).__class__.__name__,
                 six.get_method_function(f).__name__)
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 96a4fb141..45a547f40 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -56,7 +56,7 @@ class RequestTest(unittest.TestCase):
     def test_headers(self):
         # Different ways of setting headers attribute
         url = 'http://www.scrapy.org'
-        headers = {b'Accept':'gzip', b'Custom-Header':'nothing to tell you'}
+        headers = {b'Accept': 'gzip', b'Custom-Header': 'nothing to tell you'}
         r = self.request_class(url=url, headers=headers)
         p = self.request_class(url=url, headers=r.headers)
 
@@ -816,7 +816,7 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="one" value="1">
             <input type="hidden" name="two" value="2">
             </form>""")
-        r1 = self.request_class.from_response(response, formdata={'two':'3'})
+        r1 = self.request_class.from_response(response, formdata={'two': '3'})
         self.assertEqual(r1.method, 'POST')
         self.assertEqual(r1.headers['Content-type'], b'application/x-www-form-urlencoded')
         fs = _qs(r1)
diff --git a/tests/test_item.py b/tests/test_item.py
index 947566686..7c9468f65 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -245,7 +245,7 @@ class ItemTest(unittest.TestCase):
     def test_copy(self):
         class TestItem(Item):
             name = Field()
-        item = TestItem({'name':'lower'})
+        item = TestItem({'name': 'lower'})
         copied_item = item.copy()
         self.assertNotEqual(id(item), id(copied_item))
         copied_item['name'] = copied_item['name'].upper()
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index d96e259f6..57ef1694a 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -322,7 +322,7 @@ class Base:
                 Link(url=page4_url, text=u'href with whitespaces'),
             ])
 
-            lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())
+            lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=())
             self.assertEqual(lx.extract_links(self.response), [
                 Link(url='http://example.com/sample1.html', text=u''),
                 Link(url='http://example.com/sample2.html', text=u'sample 2'),
@@ -360,7 +360,7 @@ class Base:
                 Link(url='http://example.com/sample2.html', text=u'sample 2'),
             ])
 
-            lx = self.extractor_cls(tags=("a","img"), attrs=("href", "src"), deny_extensions=())
+            lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=())
             self.assertEqual(lx.extract_links(response), [
                 Link(url='http://example.com/sample2.html', text=u'sample 2'),
                 Link(url='http://example.com/sample2.jpg', text=u''),
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index eb9c4a561..b4ea30bb7 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -45,7 +45,7 @@ class LoggingContribTest(unittest.TestCase):
             "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
 
     def test_flags_in_request(self):
-        req = Request("http://www.example.com", flags=['test','flag'])
+        req = Request("http://www.example.com", flags=['test', 'flag'])
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
         logline = logkws['msg'] % logkws['args']
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 29937c189..02d8ba51e 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -79,7 +79,7 @@ class BuildComponentListTest(unittest.TestCase):
         self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
         d = {'one': {'a': 'a', 'b': 2}}
         self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
-        d = {'one': 'lorem ipsum',}
+        d = {'one': 'lorem ipsum'}
         self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
 
 
diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py
index 65782747b..c2211848c 100644
--- a/tests/test_utils_console.py
+++ b/tests/test_utils_console.py
@@ -21,7 +21,7 @@ class UtilsConsoleTestCase(unittest.TestCase):
         shell = get_shell_embed_func(['invalid'])
         self.assertEqual(shell, None)
 
-        shell = get_shell_embed_func(['invalid','python'])
+        shell = get_shell_embed_func(['invalid', 'python'])
         self.assertTrue(callable(shell))
         self.assertEqual(shell.__name__, '_embed_standard_shell')
 
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index e109d5343..de6f173e0 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -74,7 +74,7 @@ class UtilsMiscTestCase(unittest.TestCase):
         self.assertEqual(list(arg_to_iter(100)), [100])
         self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c'])
         self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3])
-        self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}])
+        self.assertEqual(list(arg_to_iter({'a': 1})), [{'a': 1}])
         self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")])
 
     def test_create_instance(self):

From 1d7c8cb0b1d3aa225fd396f263938fd2f171fb73 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 22 Jul 2019 22:27:29 +0500
Subject: [PATCH 2376/2974] Remove six.PY2 and six.PY3 conditionals.

---
 docs/topics/downloader-middleware.rst |  6 +++---
 scrapy/_monkeypatches.py              | 10 ----------
 scrapy/commands/fetch.py              |  5 ++---
 scrapy/crawler.py                     | 11 -----------
 scrapy/exporters.py                   |  2 +-
 scrapy/extensions/feedexport.py       |  3 +--
 scrapy/http/request/form.py           |  3 +--
 scrapy/http/response/text.py          |  3 ---
 scrapy/item.py                        |  8 +-------
 scrapy/link.py                        | 15 ++-------------
 scrapy/mail.py                        |  9 ++-------
 scrapy/settings/__init__.py           |  8 +-------
 scrapy/settings/default_settings.py   |  4 +---
 scrapy/utils/boto.py                  | 10 +---------
 scrapy/utils/conf.py                  |  5 +----
 scrapy/utils/datatypes.py             | 20 +++++++-------------
 scrapy/utils/gz.py                    | 13 +++----------
 scrapy/utils/iterators.py             |  6 +-----
 scrapy/utils/python.py                | 10 ++--------
 tests/__init__.py                     | 15 ---------------
 tests/test_http_request.py            | 11 +++--------
 tests/test_http_response.py           |  3 +--
 tests/test_item.py                    |  8 ++------
 tests/test_link.py                    | 13 ++-----------
 tests/test_middleware.py              | 21 ++++++---------------
 tests/test_request_cb_kwargs.py       |  8 +-------
 tests/test_settings/__init__.py       |  8 +-------
 tests/test_utils_datatypes.py         |  7 +------
 tests/test_utils_python.py            |  7 +++----
 29 files changed, 50 insertions(+), 202 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 8048e1c86..366b95510 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -474,7 +474,7 @@ DBM storage backend
 
     A DBM_ storage backend is also available for the HTTP cache middleware.
 
-    By default, it uses the anydbm_ module, but you can change it with the
+    By default, it uses the dbm_ module, but you can change it with the
     :setting:`HTTPCACHE_DBM_MODULE` setting.
 
 .. _httpcache-storage-custom:
@@ -626,7 +626,7 @@ HTTPCACHE_DBM_MODULE
 
 .. versionadded:: 0.13
 
-Default: ``'anydbm'``
+Default: ``'dbm'``
 
 The database module to use in the :ref:`DBM storage backend
 <httpcache-storage-dbm>`. This setting is specific to the DBM backend.
@@ -1202,4 +1202,4 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
 
 
 .. _DBM: https://en.wikipedia.org/wiki/Dbm
-.. _anydbm: https://docs.python.org/2/library/anydbm.html
+.. _dbm: https://docs.python.org/3/library/dbm.html
diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index b68099cad..1f8067b35 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -1,16 +1,6 @@
-import six
 from six.moves import copyreg
 
 
-if six.PY2:
-    from urlparse import urlparse
-    
-    # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
-    if urlparse('s3://bucket/key?key=value').query != 'key=value':
-        from urlparse import uses_query
-        uses_query.append('s3')
-
-
 # Undo what Twisted's perspective broker adds to pickle register
 # to prevent bugs like Twisted#7989 while serializing requests
 import twisted.persisted.styles  # NOQA
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 7d4840529..d45133e0e 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,5 +1,5 @@
 from __future__ import print_function
-import sys, six
+import sys
 from w3lib.url import is_url
 
 from scrapy.commands import ScrapyCommand
@@ -45,8 +45,7 @@ class Command(ScrapyCommand):
             self._print_bytes(response.body)
 
     def _print_bytes(self, bytes_):
-        bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer
-        bytes_writer.write(bytes_ + b'\n')
+        sys.stdout.buffer.write(bytes_ + b'\n')
 
     def run(self, args, opts):
         if len(args) != 1 or not is_url(args[0]):
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index ded3c082b..19b998e0d 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -88,20 +88,9 @@ class Crawler(object):
             yield self.engine.open_spider(self.spider, start_requests)
             yield defer.maybeDeferred(self.engine.start)
         except Exception:
-            # In Python 2 reraising an exception after yield discards
-            # the original traceback (see https://bugs.python.org/issue7563),
-            # so sys.exc_info() workaround is used.
-            # This workaround also works in Python 3, but it is not needed,
-            # and it is slower, so in Python 3 we use native `raise`.
-            if six.PY2:
-                exc_info = sys.exc_info()
-
             self.crawling = False
             if self.engine is not None:
                 yield self.engine.close()
-
-            if six.PY2:
-                six.reraise(*exc_info)
             raise
 
     def _create_spider(self, *args, **kwargs):
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 8ed8d55f1..0d9c35654 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -216,7 +216,7 @@ class CsvItemExporter(BaseItemExporter):
             write_through=True,
             encoding=self.encoding,
             newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
-        ) if six.PY3 else file
+        )
         self.csv_writer = csv.writer(self.stream, **kwargs)
         self._headers_not_written = True
         self._join_multivalued = join_multivalued
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 41d68fb14..e2492d506 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -10,7 +10,6 @@ import logging
 import posixpath
 from tempfile import NamedTemporaryFile
 from datetime import datetime
-import six
 from six.moves.urllib.parse import urlparse, unquote
 from ftplib import FTP
 
@@ -65,7 +64,7 @@ class StdoutFeedStorage(object):
 
     def __init__(self, uri, _stdout=None):
         if not _stdout:
-            _stdout = sys.stdout if six.PY2 else sys.stdout.buffer
+            _stdout = sys.stdout.buffer
         self._stdout = _stdout
 
     def open(self, spider):
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 3ce8fc48e..b6feede07 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -104,8 +104,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
                 el = el.getparent()
                 if el is None:
                     break
-        encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape')
-        raise ValueError('No <form> element found with %s' % encoded)
+        raise ValueError('No <form> element found with %s' % formxpath)
 
     # If we get here, it means that either formname was None
     # or invalid
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 339913d4e..a8010877c 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -32,9 +32,6 @@ class TextResponse(Response):
 
     def _set_url(self, url):
         if isinstance(url, six.text_type):
-            if six.PY2 and self.encoding is None:
-                raise TypeError("Cannot convert unicode url - %s "
-                                "has no encoding" % type(self).__name__)
             self._url = to_native_str(url, self.encoding)
         else:
             super(TextResponse, self)._set_url(url)
diff --git a/scrapy/item.py b/scrapy/item.py
index 73b8f54b0..32f9b2ebb 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -4,8 +4,8 @@ Scrapy Item
 See documentation in docs/topics/item.rst
 """
 
-import collections
 from abc import ABCMeta
+from collections.abc import MutableMapping
 from copy import deepcopy
 from pprint import pformat
 from warnings import warn
@@ -16,12 +16,6 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 
 
-if six.PY2:
-    MutableMapping = collections.MutableMapping
-else:
-    MutableMapping = collections.abc.MutableMapping
-
-
 class BaseItem(object_ref):
     """Base class for all scraped items.
 
diff --git a/scrapy/link.py b/scrapy/link.py
index f0638ced2..be1888ef0 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -4,12 +4,6 @@ This module defines the Link object used in Link extractors.
 For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
 """
-import warnings
-import six
-
-from scrapy.utils.python import to_bytes
-
-
 class Link(object):
     """Link objects represent an extracted link by the LinkExtractor."""
 
@@ -17,13 +11,8 @@ class Link(object):
 
     def __init__(self, url, text='', fragment='', nofollow=False):
         if not isinstance(url, str):
-            if six.PY2:
-                warnings.warn("Link urls must be str objects. "
-                              "Assuming utf-8 encoding (which could be wrong)")
-                url = to_bytes(url, encoding='utf8')
-            else:
-                got = url.__class__.__name__
-                raise TypeError("Link urls must be str objects, got %s" % got)
+            got = url.__class__.__name__
+            raise TypeError("Link urls must be str objects, got %s" % got)
         self.url = url
         self.text = text
         self.fragment = fragment
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 5b944e1c4..746468e25 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -9,18 +9,13 @@ try:
     from cStringIO import StringIO as BytesIO
 except ImportError:
     from io import BytesIO
-import six
 
 from email.utils import COMMASPACE, formatdate
 from six.moves.email_mime_multipart import MIMEMultipart
 from six.moves.email_mime_text import MIMEText
 from six.moves.email_mime_base import MIMEBase
-if six.PY2:
-    from email.MIMENonMultipart import MIMENonMultipart
-    from email import Encoders
-else:
-    from email.mime.nonmultipart import MIMENonMultipart
-    from email import encoders as Encoders
+from email.mime.nonmultipart import MIMENonMultipart
+from email import encoders as Encoders
 
 from twisted.internet import defer, reactor, ssl
 
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index f28c7940d..c871e86e0 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,19 +1,13 @@
 import six
 import json
 import copy
-import collections
+from collections.abc import MutableMapping
 from importlib import import_module
 from pprint import pformat
 
 from scrapy.settings import default_settings
 
 
-if six.PY2:
-    MutableMapping = collections.MutableMapping
-else:
-    MutableMapping = collections.abc.MutableMapping
-
-
 SETTINGS_PRIORITIES = {
     'default': 0,
     'command': 10,
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 9c22999cb..5c9678c01 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -17,8 +17,6 @@ import sys
 from importlib import import_module
 from os.path import join, abspath, dirname
 
-import six
-
 AJAXCRAWL_ENABLED = False
 
 AUTOTHROTTLE_ENABLED = False
@@ -179,7 +177,7 @@ HTTPCACHE_ALWAYS_STORE = False
 HTTPCACHE_IGNORE_HTTP_CODES = []
 HTTPCACHE_IGNORE_SCHEMES = ['file']
 HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
-HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
+HTTPCACHE_DBM_MODULE = 'dbm'
 HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
 HTTPCACHE_GZIP = False
 
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index 421ab2f7e..c8fc911bb 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -1,7 +1,6 @@
 """Boto/botocore helpers"""
 
 from __future__ import absolute_import
-import six
 
 from scrapy.exceptions import NotConfigured
 
@@ -11,11 +10,4 @@ def is_botocore():
         import botocore
         return True
     except ImportError:
-        if six.PY2:
-            try:
-                import boto
-                return False
-            except ImportError:
-                raise NotConfigured('missing botocore or boto library')
-        else:
-            raise NotConfigured('missing botocore library')
+        raise NotConfigured('missing botocore library')
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index fb7ca3310..561bb72fc 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -1,13 +1,10 @@
+from configparser import ConfigParser
 import os
 import sys
 import numbers
 from operator import itemgetter
 
 import six
-if six.PY2:
-    from ConfigParser import SafeConfigParser as ConfigParser
-else:
-    from configparser import ConfigParser
 
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 87536e9d7..39d389fa6 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -7,6 +7,7 @@ This module must not depend on any module outside the Standard Library.
 
 import copy
 import collections
+from collections.abc import Mapping
 import warnings
 
 import six
@@ -14,12 +15,6 @@ import six
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
-if six.PY2:
-    Mapping = collections.Mapping
-else:
-    Mapping = collections.abc.Mapping
-
-
 class MultiValueDictKeyError(KeyError):
     def __init__(self, *args, **kwargs):
         warnings.warn(
@@ -252,13 +247,12 @@ class MergeDict(object):
     first occurrence will be used.
     """
     def __init__(self, *dicts):
-        if not six.PY2:
-            warnings.warn(
-                "scrapy.utils.datatypes.MergeDict is deprecated in favor "
-                "of collections.ChainMap (introduced in Python 3.3)",
-                category=ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
+        warnings.warn(
+            "scrapy.utils.datatypes.MergeDict is deprecated in favor "
+            "of collections.ChainMap (introduced in Python 3.3)",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
         self.dicts = dicts
 
     def __getitem__(self, key):
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index b3fb16b1e..9984492f0 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -6,7 +6,6 @@ except ImportError:
     from io import BytesIO
 from gzip import GzipFile
 
-import six
 import re
 
 from scrapy.utils.decorators import deprecated
@@ -17,14 +16,8 @@ from scrapy.utils.decorators import deprecated
 #   (regression or bug-fix compared to Python 3.4)
 # - read1(), which fetches data before raising EOFError on next call
 #   works here but is only available from Python>=3.3
-# - scrapy does not support Python 3.2
-# - Python 2.7 GzipFile works fine with standard read() + extrabuf
-if six.PY2:
-    def read1(gzf, size=-1):
-        return gzf.read(size)
-else:
-    def read1(gzf, size=-1):
-        return gzf.read1(size)
+def read1(gzf, size=-1):
+    return gzf.read1(size)
 
 
 def gunzip(data):
@@ -37,7 +30,7 @@ def gunzip(data):
     chunk = b'.'
     while chunk:
         try:
-            chunk = read1(f, 8196)
+            chunk = f.read1(8196)
             output_list.append(chunk)
         except (IOError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index a12e14005..dbc1e0d20 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -102,11 +102,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     def row_to_unicode(row_):
         return [to_unicode(field, encoding) for field in row_]
 
-    # Python 3 csv reader input object needs to return strings
-    if six.PY3:
-        lines = StringIO(_body_or_str(obj, unicode=True))
-    else:
-        lines = BytesIO(_body_or_str(obj, unicode=False))
+    lines = StringIO(_body_or_str(obj, unicode=True))
 
     kwargs = {}
     if delimiter: kwargs["delimiter"] = delimiter
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index ea5193f12..37e6be868 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -113,10 +113,7 @@ def to_bytes(text, encoding=None, errors='strict'):
 def to_native_str(text, encoding=None, errors='strict'):
     """ Return str representation of ``text``
     (bytes in Python 2.x and unicode in Python 3.x). """
-    if six.PY2:
-        return to_bytes(text, encoding, errors)
-    else:
-        return to_unicode(text, encoding, errors)
+    return to_unicode(text, encoding, errors)
 
 
 def re_rsearch(pattern, text, chunk_size=1024):
@@ -189,7 +186,7 @@ def _getargspec_py23(func):
     """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords,
                                                         defaults)
 
-    Identical to inspect.getargspec() in python2, but uses
+    Was identical to inspect.getargspec() in python2, but uses
     inspect.getfullargspec() for python3 behind the scenes to avoid
     DeprecationWarning.
 
@@ -199,9 +196,6 @@ def _getargspec_py23(func):
     >>> _getargspec_py23(f)
     ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,))
     """
-    if six.PY2:
-        return inspect.getargspec(func)
-
     return inspect.ArgSpec(*inspect.getfullargspec(func)[:4])
 
 
diff --git a/tests/__init__.py b/tests/__init__.py
index 9c9e35c35..a54367f8c 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -35,18 +35,3 @@ def get_testdata(*paths):
     path = os.path.join(tests_datadir, *paths)
     with open(path, 'rb') as f:
         return f.read()
-
-
-# FIXME: delete after dropping py2 support
-# Monkey patch the unittest module to prevent the
-# DeprecationWarning about assertRaisesRegexp -> assertRaisesRegex
-import six
-if six.PY2:
-    import unittest
-    import twisted.trial.unittest
-    if not getattr(unittest.TestCase, 'assertRegex', None):
-        unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches
-    if not getattr(unittest.TestCase, 'assertRaisesRegex', None):
-        unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
-    if not getattr(twisted.trial.unittest.TestCase, 'assertRaisesRegex', None):
-        twisted.trial.unittest.TestCase.assertRaisesRegex = twisted.trial.unittest.TestCase.assertRaisesRegexp
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 96a4fb141..bb451b5f4 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -3,13 +3,12 @@ import cgi
 import unittest
 import re
 import json
+from urllib.parse import unquote_to_bytes
 import warnings
 
 import six
 from six.moves import xmlrpc_client as xmlrpclib
 from six.moves.urllib.parse import urlparse, parse_qs, unquote
-if six.PY3:
-    from urllib.parse import unquote_to_bytes
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_native_str
@@ -1064,8 +1063,7 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs, {})
 
         xpath = u"//form[@name='\u03b1']"
-        encoded = xpath if six.PY3 else xpath.encode('unicode_escape')
-        self.assertRaisesRegex(ValueError, re.escape(encoded),
+        self.assertRaisesRegex(ValueError, re.escape(xpath),
                                self.request_class.from_response,
                                response, formxpath=xpath)
 
@@ -1208,10 +1206,7 @@ def _qs(req, encoding='utf-8', to_unicode=False):
         qs = req.body
     else:
         qs = req.url.partition('?')[2]
-    if six.PY2:
-        uqs = unquote(to_native_str(qs, encoding))
-    elif six.PY3:
-        uqs = unquote_to_bytes(qs)
+    uqs = unquote_to_bytes(qs)
     if to_unicode:
         uqs = uqs.decode(encoding)
     return parse_qs(uqs, True)
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index dfc8562f3..ec3b51086 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -21,8 +21,7 @@ class BaseResponseTest(unittest.TestCase):
         # Response requires url in the consturctor
         self.assertRaises(Exception, self.response_class)
         self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class))
-        if not six.PY2:
-            self.assertRaises(TypeError, self.response_class, b"http://example.com")
+        self.assertRaises(TypeError, self.response_class, b"http://example.com")
         # body can be str or None
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class))
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class))
diff --git a/tests/test_item.py b/tests/test_item.py
index 947566686..0ad278701 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -62,12 +62,8 @@ class ItemTest(unittest.TestCase):
         i['number'] = 123
         itemrepr = repr(i)
 
-        if six.PY2:
-            self.assertEqual(itemrepr,
-                             "{'name': u'John Doe', 'number': 123}")
-        else:
-            self.assertEqual(itemrepr,
-                             "{'name': 'John Doe', 'number': 123}")
+        self.assertEqual(itemrepr,
+                         "{'name': 'John Doe', 'number': 123}")
 
         i2 = eval(itemrepr)
         self.assertEqual(i2['name'], 'John Doe')
diff --git a/tests/test_link.py b/tests/test_link.py
index 955430b37..5e2ce5eeb 100644
--- a/tests/test_link.py
+++ b/tests/test_link.py
@@ -1,6 +1,4 @@
 import unittest
-import warnings
-import six
 
 from scrapy.link import Link
 
@@ -46,12 +44,5 @@ class LinkTest(unittest.TestCase):
         self._assert_same_links(l1, l2)
 
     def test_non_str_url_py2(self):
-        if six.PY2:
-            with warnings.catch_warnings(record=True) as w:
-                link = Link(u"http://www.example.com/\xa3")
-                self.assertIsInstance(link.url, str)
-                self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3')
-            assert len(w) == 1, "warning not issued"
-        else:
-            with self.assertRaises(TypeError):
-                Link(b"http://www.example.com/\xc2\xa3")
+        with self.assertRaises(TypeError):
+            Link(b"http://www.example.com/\xc2\xa3")
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index aea0be825..af9b43d61 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -3,7 +3,6 @@ from twisted.trial import unittest
 from scrapy.settings import Settings
 from scrapy.exceptions import NotConfigured
 from scrapy.middleware import MiddlewareManager
-import six
 
 class M1(object):
 
@@ -66,20 +65,12 @@ class MiddlewareManagerTest(unittest.TestCase):
 
     def test_methods(self):
         mwman = TestMiddlewareManager(M1(), M2(), M3())
-        if six.PY2:
-            self.assertEqual([x.im_class for x in mwman.methods['open_spider']],
-                [M1, M2])
-            self.assertEqual([x.im_class for x in mwman.methods['close_spider']],
-                [M2, M1])
-            self.assertEqual([x.im_class for x in mwman.methods['process']],
-                [M1, M3])
-        else:
-            self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
-                [M1, M2])
-            self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
-                [M2, M1])
-            self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
-                [M1, M3])
+        self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
+            [M1, M2])
+        self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
+            [M2, M1])
+        self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
+            [M1, M3])
 
     def test_enabled(self):
         m1, m2, m3 = M1(), M2(), M3()
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index c9943faa8..a5cdc0de0 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -1,7 +1,6 @@
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
-import six
 
 from scrapy.http import Request
 from scrapy.crawler import CrawlerRunner
@@ -161,9 +160,4 @@ class CallbackKeywordArgumentsTestCase(TestCase):
         self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
         self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
         self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
-        # py2 and py3 messages are different
-        exc_message = str(exceptions['takes_more'].exc_info[1])
-        if six.PY2:
-            self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
-        elif six.PY3:
-            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
+        self.assertEqual(str(exceptions['takes_more'].exc_info[1]), "parse_takes_more() missing 1 required positional argument: 'other'")
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 1dbacbea3..08286ff02 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -60,9 +60,6 @@ class SettingsAttributeTest(unittest.TestCase):
 
 class BaseSettingsTest(unittest.TestCase):
 
-    if six.PY3:
-        assertItemsEqual = unittest.TestCase.assertCountEqual
-
     def setUp(self):
         self.settings = BaseSettings()
 
@@ -152,7 +149,7 @@ class BaseSettingsTest(unittest.TestCase):
         self.settings.setmodule(
             'tests.test_settings.default_settings', 10)
 
-        self.assertItemsEqual(six.iterkeys(self.settings.attributes),
+        self.assertCountEqual(six.iterkeys(self.settings.attributes),
                               six.iterkeys(ctrl_attributes))
 
         for key in six.iterkeys(ctrl_attributes):
@@ -343,9 +340,6 @@ class BaseSettingsTest(unittest.TestCase):
 
 class SettingsTest(unittest.TestCase):
 
-    if six.PY3:
-        assertItemsEqual = unittest.TestCase.assertCountEqual
-
     def setUp(self):
         self.settings = Settings()
 
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 7e671f627..53228fc6e 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,11 +1,6 @@
 import copy
 import unittest
-
-import six
-if six.PY2:
-    from collections import Mapping, MutableMapping
-else:
-    from collections.abc import Mapping, MutableMapping
+from collections.abc import Mapping, MutableMapping
 
 from scrapy.utils.datatypes import CaselessDict, LocalCache, SequenceExclude
 
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3e1148354..096aa50b7 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -231,12 +231,11 @@ class UtilsPythonTestCase(unittest.TestCase):
             self.assertEqual(get_func_args(" ".join), [])
             self.assertEqual(get_func_args(operator.itemgetter(2)), [])
         else:
-            stripself = not six.PY2  # PyPy3 exposes them as methods
             self.assertEqual(
-                get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit'])
-            self.assertEqual(get_func_args(" ".join, stripself), ['list'])
+                get_func_args(six.text_type.split, True), ['sep', 'maxsplit'])
+            self.assertEqual(get_func_args(" ".join, True), ['list'])
             self.assertEqual(
-                get_func_args(operator.itemgetter(2), stripself), ['obj'])
+                get_func_args(operator.itemgetter(2), True), ['obj'])
 
 
     def test_without_none_values(self):

From 0e696ed06d2975ee86e7ce3a2d3892588c420a04 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 20:56:26 +0500
Subject: [PATCH 2377/2974] Remove unneeded and unused code from
 XmlItemExporter.

---
 scrapy/exporters.py | 22 ++++------------------
 1 file changed, 4 insertions(+), 18 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 0d9c35654..1aa195b0b 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -143,11 +143,11 @@ class XmlItemExporter(BaseItemExporter):
 
     def _beautify_newline(self, new_item=False):
         if self.indent is not None and (self.indent > 0 or new_item):
-            self._xg_characters('\n')
+            self.xg.characters('\n')
 
     def _beautify_indent(self, depth=1):
         if self.indent:
-            self._xg_characters(' ' * self.indent * depth)
+            self.xg.characters(' ' * self.indent * depth)
 
     def start_exporting(self):
         self.xg.startDocument()
@@ -182,26 +182,12 @@ class XmlItemExporter(BaseItemExporter):
                 self._export_xml_field('value', value, depth=depth+1)
             self._beautify_indent(depth=depth)
         elif isinstance(serialized_value, six.text_type):
-            self._xg_characters(serialized_value)
+            self.xg.characters(serialized_value)
         else:
-            self._xg_characters(str(serialized_value))
+            self.xg.characters(str(serialized_value))
         self.xg.endElement(name)
         self._beautify_newline()
 
-    # Workaround for https://bugs.python.org/issue17606
-    # Before Python 2.7.4 xml.sax.saxutils required bytes;
-    # since 2.7.4 it requires unicode. The bug is likely to be
-    # fixed in 2.7.6, but 2.7.6 will still support unicode,
-    # and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
-    if sys.version_info[:3] >= (2, 7, 4):
-        def _xg_characters(self, serialized_value):
-            if not isinstance(serialized_value, six.text_type):
-                serialized_value = serialized_value.decode(self.encoding)
-            return self.xg.characters(serialized_value)
-    else:  # pragma: no cover
-        def _xg_characters(self, serialized_value):
-            return self.xg.characters(serialized_value)
-
 
 class CsvItemExporter(BaseItemExporter):
 

From 065fe29d3cc6634893dfead320779498fb061cd4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 21:06:52 +0500
Subject: [PATCH 2378/2974] Deprecate scrapy.utils.gz.read1.

---
 scrapy/utils/gz.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 9984492f0..dc8316d8c 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -16,6 +16,7 @@ from scrapy.utils.decorators import deprecated
 #   (regression or bug-fix compared to Python 3.4)
 # - read1(), which fetches data before raising EOFError on next call
 #   works here but is only available from Python>=3.3
+@deprecated('GzipFile.read1')
 def read1(gzf, size=-1):
     return gzf.read1(size)
 

From 85e79ae792752353ea60bff3d93f9e77bea300fb Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 21:09:49 +0500
Subject: [PATCH 2379/2974] Remove cStringIO imports.

---
 scrapy/downloadermiddlewares/decompression.py | 6 +-----
 scrapy/mail.py                                | 6 +-----
 scrapy/pipelines/files.py                     | 7 +------
 scrapy/pipelines/images.py                    | 6 +-----
 scrapy/utils/gz.py                            | 9 ++-------
 scrapy/utils/iterators.py                     | 6 +-----
 tests/test_cmdline/__init__.py                | 5 +----
 tests/test_pipeline_media.py                  | 6 ++----
 8 files changed, 10 insertions(+), 41 deletions(-)

diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index 49313cc04..e2d73f347 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -4,6 +4,7 @@ and extract the potentially compressed responses that may arrive.
 
 import bz2
 import gzip
+from io import BytesIO
 import zipfile
 import tarfile
 import logging
@@ -11,11 +12,6 @@ from tempfile import mktemp
 
 import six
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 from scrapy.responsetypes import responsetypes
 
 logger = logging.getLogger(__name__)
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 746468e25..d24de2212 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -3,13 +3,9 @@ Mail sending helpers
 
 See documentation in docs/topics/email.rst
 """
+from io import BytesIO
 import logging
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 from email.utils import COMMASPACE, formatdate
 from six.moves.email_mime_multipart import MIMEMultipart
 from six.moves.email_mime_text import MIMEText
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index cc3d10b63..8d74c5011 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -5,6 +5,7 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
+from io import BytesIO
 import mimetypes
 import os
 import os.path
@@ -15,12 +16,6 @@ from six.moves.urllib.parse import urlparse
 from collections import defaultdict
 import six
 
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 from twisted.internet import defer, threads
 
 from scrapy.pipelines.media import MediaPipeline
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index fa4d12ad1..e77cef4ff 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -5,13 +5,9 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
+from io import BytesIO
 import six
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 from PIL import Image
 
 from scrapy.utils.misc import md5sum
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index dc8316d8c..f41e62fe3 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -1,12 +1,7 @@
-import struct
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
 from gzip import GzipFile
-
+from io import BytesIO
 import re
+import struct
 
 from scrapy.utils.decorators import deprecated
 
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index dbc1e0d20..9693ba768 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -1,11 +1,7 @@
 import re
 import csv
-import logging
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
 from io import StringIO
+import logging
 import six
 
 from scrapy.http import TextResponse, Response
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 68dfb1cca..56cfe642a 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -1,3 +1,4 @@
+from io import StringIO
 import json
 import os
 import pstats
@@ -7,10 +8,6 @@ from subprocess import Popen, PIPE
 import sys
 import tempfile
 import unittest
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from io import StringIO
 
 from scrapy.utils.test import get_testenv
 
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 28e39cefa..ad2618ec9 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -144,10 +144,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 
         # The Failure should encapsulate a FileException ...
         self.assertEqual(failure.value, file_exc)
-        # ... and if we're running on Python 3 ...
-        if sys.version_info.major >= 3:
-            # ... it should have the returnValue exception set as its context
-            self.assertEqual(failure.value.__context__, def_gen_return_exc)
+        # ... and it should have the returnValue exception set as its context
+        self.assertEqual(failure.value.__context__, def_gen_return_exc)
 
         # Let's calculate the request fingerprint and fake some runtime data...
         fp = request_fingerprint(request)

From cfa633f5e865fa491257215cc912d7321cc6da78 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 21:13:01 +0500
Subject: [PATCH 2380/2974] Some text function messages cleanup, deprecate
 to_native_str.

---
 scrapy/http/response/__init__.py | 2 +-
 scrapy/utils/python.py           | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index b0a526b72..a81404afb 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -88,7 +88,7 @@ class Response(object_ref):
     @property
     def text(self):
         """For subclasses of TextResponse, this will return the body
-        as text (unicode object in Python 2 and str in Python 3)
+        as str
         """
         raise AttributeError("Response content isn't text")
 
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 37e6be868..663a8ebaa 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -90,7 +90,7 @@ def to_unicode(text, encoding=None, errors='strict'):
     if isinstance(text, six.text_type):
         return text
     if not isinstance(text, (bytes, six.text_type)):
-        raise TypeError('to_unicode must receive a bytes, str or unicode '
+        raise TypeError('to_unicode must receive a bytes or str '
                         'object, got %s' % type(text).__name__)
     if encoding is None:
         encoding = 'utf-8'
@@ -103,16 +103,16 @@ def to_bytes(text, encoding=None, errors='strict'):
     if isinstance(text, bytes):
         return text
     if not isinstance(text, six.string_types):
-        raise TypeError('to_bytes must receive a unicode, str or bytes '
+        raise TypeError('to_bytes must receive a str or bytes '
                         'object, got %s' % type(text).__name__)
     if encoding is None:
         encoding = 'utf-8'
     return text.encode(encoding, errors)
 
 
+@deprecated('to_unicode')
 def to_native_str(text, encoding=None, errors='strict'):
-    """ Return str representation of ``text``
-    (bytes in Python 2.x and unicode in Python 3.x). """
+    """ Return str representation of ``text``. """
     return to_unicode(text, encoding, errors)
 
 

From 92ffd2f249ec276436277385e525397a2fb8b0a0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 21:27:52 +0500
Subject: [PATCH 2381/2974] Simplify some more imports.

---
 scrapy/downloadermiddlewares/httpproxy.py    | 5 +----
 scrapy/loader/processors.py                  | 5 +----
 tests/__init__.py                            | 5 -----
 tests/test_downloader_handlers.py            | 5 +----
 tests/test_downloadermiddleware.py           | 3 ++-
 tests/test_downloadermiddleware_robotstxt.py | 4 +++-
 tests/test_extension_telnet.py               | 5 -----
 tests/test_feedexport.py                     | 2 +-
 tests/test_http_request.py                   | 3 +--
 tests/test_item.py                           | 2 +-
 tests/test_pipeline_files.py                 | 6 +-----
 tests/test_settings/__init__.py              | 3 +--
 tests/test_spider.py                         | 3 +--
 tests/test_spidermiddleware.py               | 3 ++-
 tests/test_stats.py                          | 6 +-----
 tests/test_utils_deprecate.py                | 3 +--
 tests/test_utils_misc/__init__.py            | 2 +-
 tests/test_utils_trackref.py                 | 2 +-
 18 files changed, 20 insertions(+), 47 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 2c35d1b90..2212d9688 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -1,10 +1,7 @@
 import base64
 from six.moves.urllib.parse import unquote, urlunparse
 from six.moves.urllib.request import getproxies, proxy_bypass
-try:
-    from urllib2 import _parse_proxy
-except ImportError:
-    from urllib.request import _parse_proxy
+from urllib.request import _parse_proxy
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index 2acdc8093..02c625acc 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -3,10 +3,7 @@ This module provides some commonly used processors for Item Loaders.
 
 See documentation in docs/topics/loaders.rst
 """
-try:
-    from collections import ChainMap
-except ImportError:
-    from scrapy.utils.datatypes import MergeDict as ChainMap
+from collections import ChainMap
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.loader.common import wrap_loader_context
diff --git a/tests/__init__.py b/tests/__init__.py
index a54367f8c..12ce79fa9 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -21,11 +21,6 @@ if 'COV_CORE_CONFIG' in os.environ:
     os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot,
                                                  os.environ['COV_CORE_CONFIG'])
 
-try:
-    import unittest.mock as mock
-except ImportError:
-    import mock
-
 tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
                              'sample_data')
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 109469503..6090998d4 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -2,11 +2,8 @@ import os
 import six
 import shutil
 import tempfile
+from unittest import mock
 import contextlib
-try:
-    from unittest import mock
-except ImportError:
-    import mock
 
 from testfixtures import LogCapture
 from twisted.trial import unittest
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 03564e748..6b9a5bee8 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -1,3 +1,5 @@
+from unittest import mock
+
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
 
@@ -7,7 +9,6 @@ from scrapy.exceptions import _InvalidOutput
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.utils.test import get_crawler
 from scrapy.utils.python import to_bytes
-from tests import mock
 
 
 class ManagerTestCase(TestCase):
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index fbc46cba4..8266bf35f 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,5 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
+
+from unittest import mock
+
 from twisted.internet import reactor, error
 from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
 from twisted.python import failure
@@ -9,7 +12,6 @@ from scrapy.downloadermiddlewares.robotstxt import (RobotsTxtMiddleware,
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response, TextResponse
 from scrapy.settings import Settings
-from tests import mock
 from tests.test_robotstxt_interface import rerp_available, reppy_available
 
 
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index 4f389e5cb..875ceb83c 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -1,8 +1,3 @@
-try:
-    import unittest.mock as mock
-except ImportError:
-    import mock
-
 from twisted.trial import unittest
 from twisted.conch.telnet import ITelnetProtocol
 from twisted.cred import credentials
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 11a5a8279..0c70bf80e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -7,6 +7,7 @@ from io import BytesIO
 import tempfile
 import shutil
 import string
+from unittest import mock
 from six.moves.urllib.parse import urljoin, urlparse, quote
 from six.moves.urllib.request import pathname2url
 
@@ -15,7 +16,6 @@ from twisted.trial import unittest
 from twisted.internet import defer
 from scrapy.crawler import CrawlerRunner
 from scrapy.settings import Settings
-from tests import mock
 from tests.mockserver import MockServer
 from w3lib.url import path_to_file_uri
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index bb451b5f4..9fe201579 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -3,6 +3,7 @@ import cgi
 import unittest
 import re
 import json
+from unittest import mock
 from urllib.parse import unquote_to_bytes
 import warnings
 
@@ -13,8 +14,6 @@ from six.moves.urllib.parse import urlparse, parse_qs, unquote
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_native_str
 
-from tests import mock
-
 
 class RequestTest(unittest.TestCase):
 
diff --git a/tests/test_item.py b/tests/test_item.py
index 0ad278701..d98c63ddd 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -1,12 +1,12 @@
 import sys
 import unittest
+from unittest import mock
 from warnings import catch_warnings
 
 import six
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
-from tests import mock
 
 
 PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6)
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index cb8f8da18..bd40e4103 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,10 +1,9 @@
 import os
 import random
 import time
-import hashlib
-import warnings
 from tempfile import mkdtemp
 from shutil import rmtree
+from unittest import mock
 from six.moves.urllib.parse import urlparse
 from six import BytesIO
 
@@ -15,13 +14,10 @@ from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GC
 from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
-from scrapy.utils.python import to_bytes
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
 from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete
 from scrapy.utils.boto import is_botocore
 
-from tests import mock
-
 
 def _mocked_download_func(request, info):
     response = request.meta.get('response')
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 08286ff02..32e65bed5 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -1,10 +1,9 @@
 import six
 import unittest
-import warnings
+from unittest import mock
 
 from scrapy.settings import (BaseSettings, Settings, SettingsAttribute,
                              SETTINGS_PRIORITIES, get_settings_priority)
-from tests import mock
 from . import default_settings
 
 
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 6f6cdb8ff..c0fccfdd6 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -1,5 +1,6 @@
 import gzip
 import inspect
+from unittest import mock
 import warnings
 from io import BytesIO
 
@@ -17,8 +18,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.test import get_crawler
 
-from tests import mock
-
 
 class SpiderTest(unittest.TestCase):
 
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 832fd3330..55d665e79 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -1,3 +1,5 @@
+from unittest import mock
+
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
 
@@ -6,7 +8,6 @@ from scrapy.http import Request, Response
 from scrapy.exceptions import _InvalidOutput
 from scrapy.utils.test import get_crawler
 from scrapy.core.spidermw import SpiderMiddlewareManager
-from tests import mock
 
 
 class SpiderMiddlewareTestCase(TestCase):
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 2033dbe07..2bbbb9e2c 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -1,10 +1,6 @@
 from datetime import datetime
 import unittest
-
-try:
-    from unittest import mock
-except ImportError:
-    import mock
+from unittest import mock
 
 from scrapy.extensions.corestats import CoreStats
 from scrapy.spiders import Spider
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 3e7236fb1..ce04e7f29 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -2,12 +2,11 @@
 from __future__ import absolute_import
 import inspect
 import unittest
+from unittest import mock
 import warnings
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.deprecate import create_deprecated_class, update_classpath
 
-from tests import mock
-
 
 class MyWarning(UserWarning):
     pass
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index e109d5343..de9da9104 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -1,11 +1,11 @@
 import sys
 import os
 import unittest
+from unittest import mock
 
 from scrapy.item import Item, Field
 from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules
 
-from tests import mock
 
 __doctests__ = ['scrapy.utils.misc']
 
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index c6072fc0d..480a717e7 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -1,7 +1,7 @@
 import six
 import unittest
+from unittest import mock
 from scrapy.utils import trackref
-from tests import mock
 
 
 class Foo(trackref.object_ref):

From a138fb05d4f0d90e2002e85a348a5be34904d3d8 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 21:35:13 +0500
Subject: [PATCH 2382/2974] Replace to_native_str calls with to_unicode.

---
 scrapy/core/downloader/handlers/http11.py |  2 +-
 scrapy/downloadermiddlewares/cookies.py   |  6 +++---
 scrapy/downloadermiddlewares/robotstxt.py |  3 ---
 scrapy/exporters.py                       |  4 ++--
 scrapy/http/cookies.py                    | 10 +++++-----
 scrapy/http/response/text.py              |  8 ++++----
 scrapy/linkextractors/lxmlhtml.py         |  4 ++--
 scrapy/responsetypes.py                   |  6 +++---
 scrapy/robotstxt.py                       |  8 ++++----
 scrapy/spidermiddlewares/referer.py       |  5 ++---
 scrapy/utils/reqser.py                    |  4 ++--
 scrapy/utils/request.py                   |  4 ++--
 scrapy/utils/response.py                  |  4 ++--
 scrapy/utils/ssl.py                       |  4 ++--
 tests/test_command_parse.py               |  5 ++---
 tests/test_commands.py                    |  7 ++-----
 tests/test_feedexport.py                  |  7 +++----
 tests/test_http_request.py                |  7 +++----
 tests/test_http_response.py               |  6 +++---
 tests/test_robotstxt_interface.py         |  1 -
 20 files changed, 47 insertions(+), 58 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 91b45a8fc..7d917cb74 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -174,7 +174,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
     r"""
     Return binary content of a CONNECT request.
 
-    >>> from scrapy.utils.python import to_native_str as s
+    >>> from scrapy.utils.python import to_unicode as s
     >>> s(tunnel_request_data("example.com", 8080))
     'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
     >>> s(tunnel_request_data("example.com", 8080, b"123"))
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 321c0171b..0d2b9900c 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -6,7 +6,7 @@ from collections import defaultdict
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 
 logger = logging.getLogger(__name__)
 
@@ -53,7 +53,7 @@ class CookiesMiddleware(object):
 
     def _debug_cookie(self, request, spider):
         if self.debug:
-            cl = [to_native_str(c, errors='replace')
+            cl = [to_unicode(c, errors='replace')
                   for c in request.headers.getlist('Cookie')]
             if cl:
                 cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)
@@ -62,7 +62,7 @@ class CookiesMiddleware(object):
 
     def _debug_set_cookie(self, response, spider):
         if self.debug:
-            cl = [to_native_str(c, errors='replace')
+            cl = [to_unicode(c, errors='replace')
                   for c in response.headers.getlist('Set-Cookie')]
             if cl:
                 cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 6a5dfb79c..251706c50 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -5,15 +5,12 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
 """
 
 import logging
-import sys
-import re
 
 from twisted.internet.defer import Deferred, maybeDeferred
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.python import to_native_str
 from scrapy.utils.misc import load_object
 
 logger = logging.getLogger(__name__)
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 1aa195b0b..8eb52995e 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -12,7 +12,7 @@ from six.moves import cPickle as pickle
 from xml.sax.saxutils import XMLGenerator
 
 from scrapy.utils.serialize import ScrapyJSONEncoder
-from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike
+from scrapy.utils.python import to_bytes, to_unicode, is_listlike
 from scrapy.item import BaseItem
 from scrapy.exceptions import ScrapyDeprecationWarning
 import warnings
@@ -232,7 +232,7 @@ class CsvItemExporter(BaseItemExporter):
     def _build_row(self, values):
         for s in values:
             try:
-                yield to_native_str(s, self.encoding)
+                yield to_unicode(s, self.encoding)
             except TypeError:
                 yield s
 
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 4e8056750..4532c3ab7 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -3,7 +3,7 @@ from six.moves.http_cookiejar import (
     CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
 )
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 
 
 class CookieJar(object):
@@ -165,13 +165,13 @@ class WrappedRequest(object):
         return name in self.request.headers
 
     def get_header(self, name, default=None):
-        return to_native_str(self.request.headers.get(name, default),
+        return to_unicode(self.request.headers.get(name, default),
                              errors='replace')
 
     def header_items(self):
         return [
-            (to_native_str(k, errors='replace'),
-             [to_native_str(x, errors='replace') for x in v])
+            (to_unicode(k, errors='replace'),
+             [to_unicode(x, errors='replace') for x in v])
             for k, v in self.request.headers.items()
         ]
 
@@ -189,7 +189,7 @@ class WrappedResponse(object):
 
     # python3 cookiejars calls get_all
     def get_all(self, name, default=None):
-        return [to_native_str(v, errors='replace')
+        return [to_unicode(v, errors='replace')
                 for v in self.response.headers.getlist(name)]
     # python2 cookiejars calls getheaders
     getheaders = get_all
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index a8010877c..37f450e54 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -16,7 +16,7 @@ from w3lib.html import strip_html5_whitespace
 from scrapy.http.request import Request
 from scrapy.http.response import Response
 from scrapy.utils.response import get_base_url
-from scrapy.utils.python import memoizemethod_noargs, to_native_str
+from scrapy.utils.python import memoizemethod_noargs, to_unicode
 
 
 class TextResponse(Response):
@@ -32,7 +32,7 @@ class TextResponse(Response):
 
     def _set_url(self, url):
         if isinstance(url, six.text_type):
-            self._url = to_native_str(url, self.encoding)
+            self._url = to_unicode(url, self.encoding)
         else:
             super(TextResponse, self)._set_url(url)
 
@@ -81,11 +81,11 @@ class TextResponse(Response):
     @memoizemethod_noargs
     def _headers_encoding(self):
         content_type = self.headers.get(b'Content-Type', b'')
-        return http_content_type_encoding(to_native_str(content_type))
+        return http_content_type_encoding(to_unicode(content_type))
 
     def _body_inferred_encoding(self):
         if self._cached_benc is None:
-            content_type = to_native_str(self.headers.get(b'Content-Type', b''))
+            content_type = to_unicode(self.headers.get(b'Content-Type', b''))
             benc, ubody = html_to_unicode(content_type, self.body,
                     auto_detect_fun=self._auto_detect_fun,
                     default_encoding=self._DEFAULT_ENCODING)
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 8f6f93a44..890c019c8 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -10,7 +10,7 @@ from w3lib.url import canonicalize_url
 
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
-from scrapy.utils.python import unique as unique_list, to_native_str
+from scrapy.utils.python import unique as unique_list, to_unicode
 from scrapy.utils.response import get_base_url
 from scrapy.linkextractors import FilteringLinkExtractor
 
@@ -67,7 +67,7 @@ class LxmlParserLinkExtractor(object):
                 url = self.process_attr(attr_val)
                 if url is None:
                     continue
-            url = to_native_str(url, encoding=response_encoding)
+            url = to_unicode(url, encoding=response_encoding)
             # to fix relative links after process_value
             url = urljoin(response_url, url)
             link = Link(url, _collect_string_content(el) or u'',
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 4a2d5bf52..de62276c8 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -10,7 +10,7 @@ import six
 
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
-from scrapy.utils.python import binary_is_text, to_bytes, to_native_str
+from scrapy.utils.python import binary_is_text, to_bytes, to_unicode
 
 
 class ResponseTypes(object):
@@ -55,12 +55,12 @@ class ResponseTypes(object):
         header """
         if content_encoding:
             return Response
-        mimetype = to_native_str(content_type).split(';')[0].strip().lower()
+        mimetype = to_unicode(content_type).split(';')[0].strip().lower()
         return self.from_mimetype(mimetype)
 
     def from_content_disposition(self, content_disposition):
         try:
-            filename = to_native_str(content_disposition,
+            filename = to_unicode(content_disposition,
                 encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
             filename = filename.strip('"\'')
             return self.from_filename(filename)
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 189f165d1..95a8c09b8 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -3,14 +3,14 @@ import logging
 from abc import ABCMeta, abstractmethod
 from six import with_metaclass
 
-from scrapy.utils.python import to_native_str, to_unicode
+from scrapy.utils.python import to_unicode
 
 logger = logging.getLogger(__name__)
 
 def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
     try:
         if to_native_str_type:
-            robotstxt_body = to_native_str(robotstxt_body)
+            robotstxt_body = to_unicode(robotstxt_body)
         else:
             robotstxt_body = robotstxt_body.decode('utf-8')
     except UnicodeDecodeError:
@@ -66,8 +66,8 @@ class PythonRobotParser(RobotParser):
         return o
 
     def allowed(self, url, user_agent):
-        user_agent = to_native_str(user_agent)
-        url = to_native_str(url)
+        user_agent = to_unicode(user_agent)
+        url = to_unicode(url)
         return self.rp.can_fetch(user_agent, url)
 
 
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 1ddfb37f4..c76e4d5a2 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -10,8 +10,7 @@ from w3lib.url import safe_url_string
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
-from scrapy.utils.python import to_native_str
-from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 from scrapy.utils.url import strip_url
 
@@ -322,7 +321,7 @@ class RefererMiddleware(object):
             if isinstance(resp_or_url, Response):
                 policy_header = resp_or_url.headers.get('Referrer-Policy')
                 if policy_header is not None:
-                    policy_name = to_native_str(policy_header.decode('latin1'))
+                    policy_name = to_unicode(policy_header.decode('latin1'))
         if policy_name is None:
             return self.default_policy()
 
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index c7ea7b425..495564ac0 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -4,7 +4,7 @@ Helper functions for serializing (and deserializing) requests.
 import six
 
 from scrapy.http import Request
-from scrapy.utils.python import to_unicode, to_native_str
+from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 
 
@@ -54,7 +54,7 @@ def request_from_dict(d, spider=None):
         eb = _get_method(spider, eb)
     request_cls = load_object(d['_class']) if '_class' in d else Request
     return request_cls(
-        url=to_native_str(d['url']),
+        url=to_unicode(d['url']),
         callback=cb,
         errback=eb,
         method=d['method'],
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index fb5af66a2..63d0ae772 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -9,7 +9,7 @@ import weakref
 from six.moves.urllib.parse import urlunparse
 
 from w3lib.http import basic_auth_header
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
 
 from w3lib.url import canonicalize_url
 from scrapy.utils.httpobj import urlparse_cached
@@ -97,4 +97,4 @@ def referer_str(request):
     referrer = request.headers.get('Referer')
     if referrer is None:
         return referrer
-    return to_native_str(referrer, errors='replace')
+    return to_unicode(referrer, errors='replace')
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index c3236afd4..feab07431 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -8,7 +8,7 @@ import webbrowser
 import tempfile
 
 from twisted.web import http
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
 from w3lib import html
 
 
@@ -36,7 +36,7 @@ def response_status_message(status):
     """Return status code plus status text descriptive message
     """
     message = http.RESPONSES.get(int(status), "Unknown Status")
-    return '%s %s' % (status, to_native_str(message))
+    return '%s %s' % (status, to_unicode(message))
 
 
 def response_httprepr(response):
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index 02aed60ee..6e81b33ff 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -3,7 +3,7 @@
 import OpenSSL
 import OpenSSL._util as pyOpenSSLutil
 
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 
 
 # The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL.
@@ -12,7 +12,7 @@ SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0)
 
 
 def ffi_buf_to_string(buf):
-    return to_native_str(pyOpenSSLutil.ffi.string(buf))
+    return to_unicode(pyOpenSSLutil.ffi.string(buf))
 
 
 def x509name_to_string(x509name):
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 62d5d76b4..b134beb88 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,17 +1,16 @@
 import os
 from os.path import join, abspath
-from twisted.trial import unittest
 from twisted.internet import defer
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 from tests.test_commands import CommandTest
 
 
 def _textmode(bstr):
     """Normalize input the same as writing to a file
     and reading from it in text mode"""
-    return to_native_str(bstr).replace(os.linesep, '\n')
+    return to_unicode(bstr).replace(os.linesep, '\n')
 
 class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
     command = 'parse'
diff --git a/tests/test_commands.py b/tests/test_commands.py
index b8445ae6c..536379170 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -10,13 +10,10 @@ from contextlib import contextmanager
 from threading import Timer
 
 from twisted.trial import unittest
-from twisted.internet import defer
 
 import scrapy
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_testenv
-from scrapy.utils.testsite import SiteTest
-from scrapy.utils.testproc import ProcessTest
 from tests.test_crawler import ExceptionSpider, NoRequestsSpider
 
 
@@ -56,7 +53,7 @@ class ProjectTest(unittest.TestCase):
         finally:
             timer.cancel()
 
-        return p, to_native_str(stdout), to_native_str(stderr)
+        return p, to_unicode(stdout), to_unicode(stderr)
 
 
 class StartprojectTest(ProjectTest):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 0c70bf80e..87139e81f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -26,8 +26,7 @@ from scrapy.extensions.feedexport import (
     S3FeedStorage, StdoutFeedStorage,
     BlockingFeedStorage)
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
-from scrapy.utils.python import to_native_str
-from scrapy.utils.project import get_project_settings
+from scrapy.utils.python import to_unicode
 
 from pathlib import Path
 
@@ -459,7 +458,7 @@ class FeedExportTest(unittest.TestCase):
         settings.update({'FEED_FORMAT': 'csv'})
         data = yield self.exported_data(items, settings)
 
-        reader = csv.DictReader(to_native_str(data).splitlines())
+        reader = csv.DictReader(to_unicode(data).splitlines())
         got_rows = list(reader)
         if ordered:
             self.assertEqual(reader.fieldnames, header)
@@ -473,7 +472,7 @@ class FeedExportTest(unittest.TestCase):
         settings = settings or {}
         settings.update({'FEED_FORMAT': 'jl'})
         data = yield self.exported_data(items, settings)
-        parsed = [json.loads(to_native_str(line)) for line in data.splitlines()]
+        parsed = [json.loads(to_unicode(line)) for line in data.splitlines()]
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         self.assertEqual(rows, parsed)
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 9fe201579..3518da21c 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -7,12 +7,11 @@ from unittest import mock
 from urllib.parse import unquote_to_bytes
 import warnings
 
-import six
 from six.moves import xmlrpc_client as xmlrpclib
 from six.moves.urllib.parse import urlparse, parse_qs, unquote
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
 
 
 class RequestTest(unittest.TestCase):
@@ -349,8 +348,8 @@ class FormRequestTest(RequestTest):
     request_class = FormRequest
 
     def assertQueryEqual(self, first, second, msg=None):
-        first = to_native_str(first).split("&")
-        second = to_native_str(second).split("&")
+        first = to_unicode(first).split("&")
+        second = to_unicode(second).split("&")
         return self.assertEqual(sorted(first), sorted(second), msg)
 
     def test_empty_formdata(self):
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index ec3b51086..883c943da 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -7,7 +7,7 @@ from w3lib.encoding import resolve_encoding
 from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
                          XmlResponse, Headers)
 from scrapy.selector import Selector
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 from scrapy.exceptions import NotSupported
 from scrapy.link import Link
 from tests import get_testdata
@@ -204,11 +204,11 @@ class TextResponseTest(BaseResponseTest):
         assert isinstance(resp.url, str)
 
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')
-        self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))
+        self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
         resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]})
-        self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))
+        self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
         resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
 
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 9aaab560a..cd7480e33 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -1,6 +1,5 @@
 # coding=utf-8
 from twisted.trial import unittest
-from scrapy.utils.python import to_native_str
 
 
 def reppy_available():

From 87c23ba22d2ef714d778b62c794333acaf232f60 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 28 Aug 2019 16:29:53 +0500
Subject: [PATCH 2383/2974] Remove Py2-only code that checks sys.version_info.

---
 tests/test_utils_reqser.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 11ac56897..92cd16de7 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -80,8 +80,6 @@ class RequestSerializationTest(unittest.TestCase):
         self._assert_serializes_ok(r, spider=self.spider)
 
     def test_mixin_private_callback_serialization(self):
-        if sys.version_info[0] < 3:
-            return
         r = Request("http://www.example.com",
                     callback=self.spider._TestSpiderMixin__mixin_callback,
                     errback=self.spider.handle_error)
@@ -119,9 +117,8 @@ class RequestSerializationTest(unittest.TestCase):
     def test_private_name_mangling(self):
         self._assert_mangles_to(
             self.spider, '_TestSpider__parse_item_private')
-        if sys.version_info[0] >= 3:
-            self._assert_mangles_to(
-                self.spider, '_TestSpiderMixin__mixin_callback')
+        self._assert_mangles_to(
+            self.spider, '_TestSpiderMixin__mixin_callback')
 
     def test_unserializable_callback1(self):
         r = Request("http://www.example.com", callback=lambda x: x)

From a9c891399d1bf8a888392afe80c82a8ec8f2e8e7 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 22:55:58 +0500
Subject: [PATCH 2384/2974] Fix a duplicate ref name in docs.

---
 docs/topics/downloader-middleware.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 366b95510..e93645077 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -474,7 +474,7 @@ DBM storage backend
 
     A DBM_ storage backend is also available for the HTTP cache middleware.
 
-    By default, it uses the dbm_ module, but you can change it with the
+    By default, it uses the `dbm module`_, but you can change it with the
     :setting:`HTTPCACHE_DBM_MODULE` setting.
 
 .. _httpcache-storage-custom:
@@ -1202,4 +1202,4 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
 
 
 .. _DBM: https://en.wikipedia.org/wiki/Dbm
-.. _dbm: https://docs.python.org/3/library/dbm.html
+.. _dbm module: https://docs.python.org/3/library/dbm.html

From dd367438fa7a7fec923b28648c4e909cbed1b47d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 20:05:37 +0500
Subject: [PATCH 2385/2974] Improve the dbm module ref.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/downloader-middleware.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index e93645077..ae6d41809 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -474,7 +474,7 @@ DBM storage backend
 
     A DBM_ storage backend is also available for the HTTP cache middleware.
 
-    By default, it uses the `dbm module`_, but you can change it with the
+    By default, it uses the :mod:`dbm`, but you can change it with the
     :setting:`HTTPCACHE_DBM_MODULE` setting.
 
 .. _httpcache-storage-custom:
@@ -1202,4 +1202,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
 
 
 .. _DBM: https://en.wikipedia.org/wiki/Dbm
-.. _dbm module: https://docs.python.org/3/library/dbm.html

From 1a4a77d49fa580d35d1e023ab4b54a397b88088a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 14 Nov 2019 10:24:31 +0100
Subject: [PATCH 2386/2974] Remove Python 2 check from MutableChainTest

---
 tests/test_utils_python.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3e1148354..6cae9793d 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -21,9 +21,8 @@ class MutableChainTest(unittest.TestCase):
         m.extend([7, 8])
         m.extend([9, 10], (11, 12))
         self.assertEqual(next(m), 0)
-        self.assertEqual(m.next(), 1)
-        self.assertEqual(m.__next__(), 2)
-        self.assertEqual(list(m), list(range(3, 13)))
+        self.assertEqual(m.__next__(), 1)
+        self.assertEqual(list(m), list(range(2, 13)))
 
 
 class ToUnicodeTest(unittest.TestCase):

From be6da52019990c1db45b1101dd99787752a14313 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 14 Nov 2019 10:31:55 +0100
Subject: [PATCH 2387/2974] Include extensions from #2067

---
 scrapy/linkextractors/__init__.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 3c75e683d..a2ac963fe 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -19,9 +19,12 @@ from scrapy.utils.url import (
 
 # common file extensions that are not followed if they occur in links
 IGNORED_EXTENSIONS = [
+    # archives
+    '7z', '7zip', 'bz2', 'rar', 'tar', 'tar.gz', 'xz', 'zip',
+    
     # images
     'mng', 'pct', 'bmp', 'gif', 'jpg', 'jpeg', 'png', 'pst', 'psp', 'tif',
-    'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg',
+    'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg', 'cdr', 'ico',
 
     # audio
     'mp3', 'wma', 'ogg', 'wav', 'ra', 'aac', 'mid', 'au', 'aiff',
@@ -35,7 +38,7 @@ IGNORED_EXTENSIONS = [
     'odp',
 
     # other
-    'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar', 'dmg', 'iso', 'apk', 'xz'
+    'css', 'pdf', 'exe', 'bin', 'rss', 'dmg', 'iso', 'apk'
 ]
 
 

From 3631453bfb1fb2426916919dbfd489ba9ffd9505 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Thu, 14 Nov 2019 15:07:53 +0500
Subject: [PATCH 2388/2974] Remove spaces on a blank line.

---
 scrapy/linkextractors/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index a2ac963fe..e4c62f87b 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -21,7 +21,7 @@ from scrapy.utils.url import (
 IGNORED_EXTENSIONS = [
     # archives
     '7z', '7zip', 'bz2', 'rar', 'tar', 'tar.gz', 'xz', 'zip',
-    
+
     # images
     'mng', 'pct', 'bmp', 'gif', 'jpg', 'jpeg', 'png', 'pst', 'psp', 'tif',
     'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg', 'cdr', 'ico',

From e291460db67ef8c9e52de02a0a4a86f4bce39b9d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 14 Nov 2019 15:24:37 +0500
Subject: [PATCH 2389/2974] Fix flake8-detected errors.

---
 scrapy/crawler.py            | 1 -
 scrapy/exporters.py          | 1 -
 scrapy/http/cookies.py       | 2 +-
 scrapy/link.py               | 2 ++
 tests/test_pipeline_media.py | 2 --
 5 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 19b998e0d..8868a985b 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -3,7 +3,6 @@ import signal
 import logging
 import warnings
 
-import sys
 from twisted.internet import reactor, defer
 from zope.interface.verify import verifyClass, DoesNotImplement
 
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 8eb52995e..3defafd60 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -4,7 +4,6 @@ Item Exporters are used to export/serialize items into different formats.
 
 import csv
 import io
-import sys
 import pprint
 import marshal
 import six
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 4532c3ab7..60a14c6f8 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -166,7 +166,7 @@ class WrappedRequest(object):
 
     def get_header(self, name, default=None):
         return to_unicode(self.request.headers.get(name, default),
-                             errors='replace')
+                          errors='replace')
 
     def header_items(self):
         return [
diff --git a/scrapy/link.py b/scrapy/link.py
index be1888ef0..a809c5ca4 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -4,6 +4,8 @@ This module defines the Link object used in Link extractors.
 For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
 """
+
+
 class Link(object):
     """Link objects represent an extracted link by the LinkExtractor."""
 
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index ad2618ec9..ad958e25f 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,7 +1,5 @@
 from __future__ import print_function
 
-import sys
-
 from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.python.failure import Failure

From b8ef12cd4707f9e095abdd26cc137558009c335f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 14 Nov 2019 12:10:25 +0100
Subject: [PATCH 2390/2974] Add bandit to CI

---
 .bandit.yml | 16 ++++++++++++++++
 .travis.yml |  2 ++
 tox.ini     |  7 +++++++
 3 files changed, 25 insertions(+)
 create mode 100644 .bandit.yml

diff --git a/.bandit.yml b/.bandit.yml
new file mode 100644
index 000000000..00554587a
--- /dev/null
+++ b/.bandit.yml
@@ -0,0 +1,16 @@
+skips:
+- B101
+- B105
+- B303
+- B306
+- B307
+- B311
+- B320
+- B321
+- B402
+- B404
+- B406
+- B410
+- B503
+- B603
+- B605
diff --git a/.travis.yml b/.travis.yml
index 0e77af9fd..9f477e860 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,6 +7,8 @@ branches:
     - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
 matrix:
   include:
+    - env: TOXENV=security
+      python: 3.8
     - env: TOXENV=flake8
       python: 3.8
     - env: TOXENV=pypy3
diff --git a/tox.ini b/tox.ini
index 3668058c3..cd575f3c5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -62,6 +62,13 @@ basepython = pypy3
 commands =
     py.test {posargs:docs scrapy tests}
 
+[testenv:security]
+basepython = python3.8
+deps =
+    bandit
+commands =
+    bandit -r -c .bandit.yml {posargs:scrapy}
+
 [testenv:flake8]
 basepython = python3.8
 deps =

From 5ee5508cc33116b3700c9b745138b56cc67951f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 14 Nov 2019 15:42:34 +0100
Subject: [PATCH 2391/2974] Have CI record the 10 slowest tests

---
 tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index 3668058c3..ec04035f5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -21,7 +21,7 @@ passenv =
     GCS_TEST_FILE_URI
     GCS_PROJECT_ID
 commands =
-    py.test --cov=scrapy --cov-report= {posargs:docs scrapy tests}
+    py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
 
 [testenv:py35]
 basepython = python3.5
@@ -60,7 +60,7 @@ basepython = python3.8
 [testenv:pypy3]
 basepython = pypy3
 commands =
-    py.test {posargs:docs scrapy tests}
+    py.test {posargs:--durations=10 docs scrapy tests}
 
 [testenv:flake8]
 basepython = python3.8

From 058bdda0afe967f99a3ffd59b395566b063d9c3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 14 Nov 2019 16:51:47 +0100
Subject: [PATCH 2392/2974] Improve the performance of the DOWNLOAD_DELAY test

---
 tests/test_crawl.py | 51 +++++++++++++++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 16 deletions(-)

diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3fc13eeb7..a524287eb 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -30,25 +30,44 @@ class CrawlTestCase(TestCase):
         self.assertEqual(len(crawler.spider.urls_visited), 11)  # 10 + start_url
 
     @defer.inlineCallbacks
-    def test_delay(self):
-        # short to long delays
-        yield self._test_delay(0.2, False)
-        yield self._test_delay(1, False)
-        # randoms
-        yield self._test_delay(0.2, True)
-        yield self._test_delay(1, True)
+    def test_fixed_delay(self):
+        yield self._test_delay(total=3, delay=0.1)
 
     @defer.inlineCallbacks
-    def _test_delay(self, delay, randomize):
-        settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize}
+    def test_randomized_delay(self):
+        yield self._test_delay(total=3, delay=0.1, randomize=True)
+
+    @defer.inlineCallbacks
+    def _test_delay(self, total, delay, randomize=False):
+        crawl_kwargs = dict(
+            maxlatency=delay * 2,
+            mockserver=self.mockserver,
+            total=total,
+        )
+        tolerance = (1 - (0.6 if randomize else 0.2))
+
+        settings = {"DOWNLOAD_DELAY": delay,
+                    'RANDOMIZE_DOWNLOAD_DELAY': randomize}
         crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
-        yield crawler.crawl(maxlatency=delay * 2, mockserver=self.mockserver)
-        t = crawler.spider.times
-        totaltime = t[-1] - t[0]
-        avgd = totaltime / (len(t) - 1)
-        tolerance = 0.6 if randomize else 0.2
-        self.assertTrue(avgd > delay * (1 - tolerance),
-                        "download delay too small: %s" % avgd)
+        yield crawler.crawl(**crawl_kwargs)
+        times = crawler.spider.times
+        total_time = times[-1] - times[0]
+        average = total_time / (len(times) - 1)
+        self.assertTrue(average > delay * tolerance,
+                        "download delay too small: %s" % average)
+
+        # Ensure that the same test parameters would cause a failure if no
+        # download delay is set. Otherwise, it means we are using a combination
+        # of ``total`` and ``delay`` values that are too small for the test
+        # code above to have any meaning.
+        settings["DOWNLOAD_DELAY"] = 0
+        crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
+        yield crawler.crawl(**crawl_kwargs)
+        times = crawler.spider.times
+        total_time = times[-1] - times[0]
+        average = total_time / (len(times) - 1)
+        self.assertFalse(average > delay / tolerance,
+                         "test total or delay values are too small")
 
     @defer.inlineCallbacks
     def test_timeout_success(self):

From 0946eb335a285e1f210ba1185a564699f53b17d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 14 Nov 2019 17:56:21 +0100
Subject: [PATCH 2393/2974] =?UTF-8?q?Port=20code=20from=20Twisted=E2=80=99?=
 =?UTF-8?q?s=20deprecated=20HTTPClientFactory=20into=20ScrapyHTTPClientFac?=
 =?UTF-8?q?tory?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scrapy/core/downloader/webclient.py | 90 ++++++++++++++++++++++-------
 1 file changed, 70 insertions(+), 20 deletions(-)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 3fe13414a..16fd214a3 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -1,9 +1,9 @@
 from time import time
 from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
 
-from twisted.web.client import HTTPClientFactory
 from twisted.web.http import HTTPClient
-from twisted.internet import defer
+from twisted.internet import defer, reactor
+from twisted.internet.protocol import ClientFactory
 
 from scrapy.http import Headers
 from scrapy.utils.httpobj import urlparse_cached
@@ -93,18 +93,30 @@ class ScrapyHTTPPageGetter(HTTPClient):
                 (self.factory.url, self.factory.timeout)))
 
 
-class ScrapyHTTPClientFactory(HTTPClientFactory):
-    """Scrapy implementation of the HTTPClientFactory overwriting the
-    setUrl method to make use of our Url object that cache the parse
-    result.
-    """
+class ScrapyHTTPClientFactory(ClientFactory):
 
     protocol = ScrapyHTTPPageGetter
+
     waiting = 1
     noisy = False
     followRedirect = False
     afterFoundGet = False
 
+    def _build_response(self, body, request):
+        request.meta['download_latency'] = self.headers_time-self.start_time
+        status = int(self.status)
+        headers = Headers(self.response_headers)
+        respcls = responsetypes.from_args(headers=headers, url=self._url)
+        return respcls(url=self._url, status=status, headers=headers, body=body)
+
+    def _set_connection_attributes(self, request):
+        parsed = urlparse_cached(request)
+        self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
+        proxy = request.meta.get('proxy')
+        if proxy:
+            self.scheme, _, self.host, self.port, _ = _parse(proxy)
+            self.path = self.url
+
     def __init__(self, request, timeout=180):
         self._url = urldefrag(request.url)[0]
         # converting to bytes to comply to Twisted interface
@@ -139,21 +151,59 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
         elif self.method == b'POST':
             self.headers['Content-Length'] = 0
 
-    def _build_response(self, body, request):
-        request.meta['download_latency'] = self.headers_time-self.start_time
-        status = int(self.status)
-        headers = Headers(self.response_headers)
-        respcls = responsetypes.from_args(headers=headers, url=self._url)
-        return respcls(url=self._url, status=status, headers=headers, body=body)
+    def __repr__(self):
+        return "<%s: %s>" % (self.__class__.__name__, self.url)
 
-    def _set_connection_attributes(self, request):
-        parsed = urlparse_cached(request)
-        self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
-        proxy = request.meta.get('proxy')
-        if proxy:
-            self.scheme, _, self.host, self.port, _ = _parse(proxy)
-            self.path = self.url
+    def _cancelTimeout(self, result, timeoutCall):
+        if timeoutCall.active():
+            timeoutCall.cancel()
+        return result
+
+    def buildProtocol(self, addr):
+        p = ClientFactory.buildProtocol(self, addr)
+        p.followRedirect = self.followRedirect
+        p.afterFoundGet = self.afterFoundGet
+        if self.timeout:
+            timeoutCall = reactor.callLater(self.timeout, p.timeout)
+            self.deferred.addBoth(self._cancelTimeout, timeoutCall)
+        return p
 
     def gotHeaders(self, headers):
         self.headers_time = time()
         self.response_headers = headers
+
+    def gotStatus(self, version, status, message):
+        """
+        Set the status of the request on us.
+        @param version: The HTTP version.
+        @type version: L{bytes}
+        @param status: The HTTP status code, an integer represented as a
+            bytestring.
+        @type status: L{bytes}
+        @param message: The HTTP status message.
+        @type message: L{bytes}
+        """
+        self.version, self.status, self.message = version, status, message
+
+    def page(self, page):
+        if self.waiting:
+            self.waiting = 0
+            self.deferred.callback(page)
+
+    def noPage(self, reason):
+        if self.waiting:
+            self.waiting = 0
+            self.deferred.errback(reason)
+
+    def clientConnectionFailed(self, _, reason):
+        """
+        When a connection attempt fails, the request cannot be issued.  If no
+        result has yet been provided to the result Deferred, provide the
+        connection failure reason as an error result.
+        """
+        if self.waiting:
+            self.waiting = 0
+            # If the connection attempt failed, there is nothing more to
+            # disconnect, so just fire that Deferred now.
+            self._disconnectedDeferred.callback(None)
+            self.deferred.errback(reason)

From fe3a121f1358fc904915f5b32e276520523c553a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 14 Nov 2019 22:50:53 +0500
Subject: [PATCH 2394/2974] Use kwargs when calling get_func_args.

---
 tests/test_utils_python.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 096aa50b7..a94398796 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -232,10 +232,10 @@ class UtilsPythonTestCase(unittest.TestCase):
             self.assertEqual(get_func_args(operator.itemgetter(2)), [])
         else:
             self.assertEqual(
-                get_func_args(six.text_type.split, True), ['sep', 'maxsplit'])
-            self.assertEqual(get_func_args(" ".join, True), ['list'])
+                get_func_args(six.text_type.split, stripself=True), ['sep', 'maxsplit'])
+            self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
             self.assertEqual(
-                get_func_args(operator.itemgetter(2), True), ['obj'])
+                get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
 
 
     def test_without_none_values(self):

From 3b2289ad012043b94b495d411316b2778bf3db35 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 14 Nov 2019 22:53:28 +0500
Subject: [PATCH 2395/2974] Rename test_non_str_url_py2 to test_bytes_url.

---
 tests/test_link.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_link.py b/tests/test_link.py
index 5e2ce5eeb..e0f1efffa 100644
--- a/tests/test_link.py
+++ b/tests/test_link.py
@@ -43,6 +43,6 @@ class LinkTest(unittest.TestCase):
         l2 = eval(repr(l1))
         self._assert_same_links(l1, l2)
 
-    def test_non_str_url_py2(self):
+    def test_bytes_url(self):
         with self.assertRaises(TypeError):
             Link(b"http://www.example.com/\xc2\xa3")

From 77a84f620ffa5d001072c7122c0f38048fe15606 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Alberto=20/=20Speedy?=
 <libre@josealberto4444.com>
Date: Fri, 15 Nov 2019 11:09:24 +0100
Subject: [PATCH 2396/2974] Fix string
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 87eaac2af..4c038030f 100644
--- a/README.rst
+++ b/README.rst
@@ -62,7 +62,7 @@ directory.
 Releases
 ========
 
-You can check https://docs.scrapy.org/en/latest/news.html for release notes.
+You can check https://docs.scrapy.org/en/latest/news.html for the release notes.
 
 Community (blog, twitter, mail list, IRC)
 =========================================

From 0e252f5a13be3195fdc3ec1d66a111ae01a0ab80 Mon Sep 17 00:00:00 2001
From: Marc Hernandez Cabot <noviluni@gmail.com>
Date: Fri, 15 Nov 2019 19:12:43 +0100
Subject: [PATCH 2397/2974] fix E711 and E713

---
 pytest.ini                                   | 4 ++--
 tests/test_downloader_handlers.py            | 4 ++--
 tests/test_downloadermiddleware_httpcache.py | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index fa6e7287e..529ad5d27 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -192,14 +192,14 @@ flake8-ignore =
     tests/test_crawl.py E501 E741 E265
     tests/test_crawler.py F841 E306 E501
     tests/test_dependencies.py E302 F841 E501 E305
-    tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E711 E126 E226 E123
+    tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E126 E226 E123
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E302 E501
     tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
     tests/test_downloadermiddleware_decompression.py E127
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
-    tests/test_downloadermiddleware_httpcache.py E713 E501 E302 E305 F401
+    tests/test_downloadermiddleware_httpcache.py E501 E302 E305 F401
     tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123
     tests/test_downloadermiddleware_httpproxy.py F401 E501 E128
     tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 6090998d4..59d4a3eec 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -615,7 +615,7 @@ class Http11MockServerTestCase(unittest.TestCase):
         crawler = get_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=Request(url=self.mockserver.url('')))
         failure = crawler.spider.meta.get('failure')
-        self.assertTrue(failure == None)
+        self.assertTrue(failure is None)
         reason = crawler.spider.meta['close_reason']
         self.assertTrue(reason, 'finished')
 
@@ -636,7 +636,7 @@ class Http11MockServerTestCase(unittest.TestCase):
             yield crawler.crawl(seed=request)
             # download_maxsize = 50 is enough for the gzipped response
             failure = crawler.spider.meta.get('failure')
-            self.assertTrue(failure == None)
+            self.assertTrue(failure is None)
             reason = crawler.spider.meta['close_reason']
             self.assertTrue(reason, 'finished')
         else:
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 950664ffe..9d863b6e3 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -85,8 +85,8 @@ class _BaseTest(unittest.TestCase):
 
     def assertEqualRequestButWithCacheValidators(self, request1, request2):
         self.assertEqual(request1.url, request2.url)
-        assert not b'If-None-Match' in request1.headers
-        assert not b'If-Modified-Since' in request1.headers
+        assert b'If-None-Match' not in request1.headers
+        assert b'If-Modified-Since' not in request1.headers
         assert any(h in request2.headers for h in (b'If-None-Match', b'If-Modified-Since'))
         self.assertEqual(request1.body, request2.body)
 

From 393a2a197251cd4ac10671fbbff11113be42d930 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 18 Nov 2019 09:15:48 +0100
Subject: [PATCH 2398/2974] Include /requirements-py3.txt from
 /docs/requirements.txt

---
 docs/requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/requirements.txt b/docs/requirements.txt
index f9db85146..85812be9a 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,3 +1,4 @@
+-r ../requirements-py3.txt
 Sphinx>=2.1
 sphinx-notfound-page
 sphinx_rtd_theme

From 99d8b05a0b1997033b2240aa9f945bbe659ee6dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 18 Nov 2019 10:58:47 +0100
Subject: [PATCH 2399/2974] Deprecate scrapy.utils.python.MutableChain.next

---
 scrapy/utils/python.py     | 4 ++++
 tests/test_utils_python.py | 8 +++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 64402a2bb..6edf7d702 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -392,3 +392,7 @@ class MutableChain(object):
 
     def __next__(self):
         return next(self.data)
+
+    @deprecated("scrapy.utils.python.MutableChain.__next__")
+    def next(self):
+        return self.__next__()
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 6cae9793d..ca2c241e4 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -5,6 +5,7 @@ import unittest
 from itertools import count
 import platform
 import six
+from warnings import catch_warnings
 
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
@@ -22,7 +23,12 @@ class MutableChainTest(unittest.TestCase):
         m.extend([9, 10], (11, 12))
         self.assertEqual(next(m), 0)
         self.assertEqual(m.__next__(), 1)
-        self.assertEqual(list(m), list(range(2, 13)))
+        with catch_warnings(record=True) as warnings:
+            self.assertEqual(m.next(), 2)
+            self.assertEqual(len(warnings), 1)
+            self.assertIn('scrapy.utils.python.MutableChain.__next__',
+                          str(warnings[0].message))
+        self.assertEqual(list(m), list(range(3, 13)))
 
 
 class ToUnicodeTest(unittest.TestCase):

From 6d1667d5b8086dd12539a185680f9e119c5fdc55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 7 Nov 2019 18:32:33 +0100
Subject: [PATCH 2400/2974] Use the latest Python version to build the
 documentation

---
 .travis.yml |  2 +-
 tox.ini     | 10 ++++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 9f477e860..c9c64e990 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -26,7 +26,7 @@ matrix:
     - env: TOXENV=py38-extra-deps
       python: 3.8
     - env: TOXENV=docs
-      python: 3.6
+      python: 3.8
 install:
   - |
       if [ "$TOXENV" = "pypy3" ]; then
diff --git a/tox.ini b/tox.ini
index fd75d18e2..195cc106a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -6,6 +6,9 @@
 [tox]
 envlist = py35
 
+[latest]
+basepython = python3.8
+
 [testenv]
 deps =
     -ctests/constraints.txt
@@ -63,14 +66,14 @@ commands =
     py.test {posargs:--durations=10 docs scrapy tests}
 
 [testenv:security]
-basepython = python3.8
+basepython = {[latest]basepython}
 deps =
     bandit
 commands =
     bandit -r -c .bandit.yml {posargs:scrapy}
 
 [testenv:flake8]
-basepython = python3.8
+basepython = {[latest]basepython}
 deps =
     {[testenv]deps}
     pytest-flake8
@@ -83,18 +86,21 @@ deps =
     -rdocs/requirements.txt
 
 [testenv:docs]
+basepython = {[latest]basepython}
 changedir = {[docs]changedir}
 deps = {[docs]deps}
 commands =
     sphinx-build -W -b html . {envtmpdir}/html
 
 [testenv:docs-coverage]
+basepython = {[latest]basepython}
 changedir = {[docs]changedir}
 deps = {[docs]deps}
 commands =
     sphinx-build -b coverage . {envtmpdir}/coverage
 
 [testenv:docs-links]
+basepython = {[latest]basepython}
 changedir = {[docs]changedir}
 deps = {[docs]deps}
 commands =

From e1af85619f63fe9312920f0dea7454bb76fcc23f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 18 Nov 2019 11:06:25 +0100
Subject: [PATCH 2401/2974] Add a configuration file for Read the Docs

---
 .readthedocs.yml | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 .readthedocs.yml

diff --git a/.readthedocs.yml b/.readthedocs.yml
new file mode 100644
index 000000000..3c1c3e8be
--- /dev/null
+++ b/.readthedocs.yml
@@ -0,0 +1,7 @@
+version: 2
+sphinx:
+  configuration: docs/conf.py
+python:
+  version: 3.8
+  install:
+    - requirements: docs/requirements.txt

From 74589df02f2961110166959b49002fd8e5037291 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 18 Nov 2019 14:51:44 +0100
Subject: [PATCH 2402/2974] Make command doctests pass

---
 docs/topics/commands.rst | 28 ++++++++++++++++++++++++----
 pytest.ini               |  1 -
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index a93bee06b..5b3cd7e75 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -1,3 +1,5 @@
+.. highlight:: none
+
 .. _topics-commands:
 
 =================
@@ -66,7 +68,9 @@ structure by default, similar to this::
 
 The directory where the ``scrapy.cfg`` file resides is known as the *project
 root directory*. That file contains the name of the python module that defines
-the project settings. Here is an example::
+the project settings. Here is an example:
+
+.. code-block:: ini
 
     [settings]
     default = myproject.settings
@@ -80,7 +84,9 @@ A project root directory, the one that contains the ``scrapy.cfg``, may be
 shared by multiple Scrapy projects, each with its own settings module.
 
 In that case, you must define one or more aliases for those settings modules
-under ``[settings]`` in your ``scrapy.cfg`` file::
+under ``[settings]`` in your ``scrapy.cfg`` file:
+
+.. code-block:: ini
 
     [settings]
     default = myproject1.settings
@@ -277,6 +283,8 @@ check
 
 Run contract checks.
 
+.. skip: start
+
 Usage examples::
 
     $ scrapy check -l
@@ -294,6 +302,8 @@ Usage examples::
     [FAILED] first_spider:parse
     >>> Returned 92 requests, expected 0..4
 
+.. skip: end
+
 .. command:: list
 
 list
@@ -481,6 +491,8 @@ Supported options:
 
 * ``--verbose`` or ``-v``: display information for each depth level
 
+.. skip: start
+
 Usage example::
 
     $ scrapy parse http://www.example.com/ -c parse_item
@@ -495,6 +507,8 @@ Usage example::
     # Requests  -----------------------------------------------------------------
     []
 
+.. skip: end
+
 
 .. command:: settings
 
@@ -573,7 +587,9 @@ Default: ``''`` (empty string)
 A module to use for looking up custom Scrapy commands. This is used to add custom
 commands for your Scrapy project.
 
-Example::
+Example:
+
+.. code-block:: python
 
     COMMANDS_MODULE = 'mybot.commands'
 
@@ -588,7 +604,11 @@ You can also add Scrapy commands from an external library by adding a
 ``scrapy.commands`` section in the entry points of the library ``setup.py``
 file.
 
-The following example adds ``my_command`` command::
+The following example adds ``my_command`` command:
+
+.. skip: next
+
+.. code-block:: python
 
   from setuptools import setup, find_packages
 
diff --git a/pytest.ini b/pytest.ini
index 529ad5d27..dab91416f 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -8,7 +8,6 @@ addopts =
     --ignore=docs/_ext
     --ignore=docs/conf.py
     --ignore=docs/news.rst
-    --ignore=docs/topics/commands.rst
     --ignore=docs/topics/debug.rst
     --ignore=docs/topics/developer-tools.rst
     --ignore=docs/topics/dynamic-content.rst

From e84cb18ca0b5b09c68cc76d6c48929d9ff933e5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 18 Nov 2019 15:50:45 +0100
Subject: [PATCH 2403/2974] Use InterSphinx to link to the Twisted
 documentation

---
 docs/conf.py                             |  3 ++-
 docs/contributing.rst                    |  6 +++---
 docs/topics/api.rst                      |  2 --
 docs/topics/architecture.rst             |  3 +--
 docs/topics/email.rst                    | 13 +++++++------
 docs/topics/item-pipeline.rst            |  9 ++++-----
 docs/topics/media-pipeline.rst           |  6 +++---
 docs/topics/practices.rst                |  3 +--
 docs/topics/request-response.rst         |  9 ++++-----
 docs/topics/signals.rst                  | 16 +++++++---------
 scrapy/core/downloader/contextfactory.py | 17 ++++++++++-------
 scrapy/crawler.py                        | 15 ++++++++-------
 scrapy/signalmanager.py                  |  6 ++----
 13 files changed, 52 insertions(+), 56 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 6ec4582b1..6bfd2cb0e 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -275,5 +275,6 @@ coverage_ignore_pyobjects = [
 
 intersphinx_mapping = {
     'python': ('https://docs.python.org/3', None),
-    'sphinx': ('https://www.sphinx-doc.org/en/stable', None),
+    'sphinx': ('https://www.sphinx-doc.org/en/master', None),
+    'twisted': ('https://twistedmatrix.com/documents/current', None),
 }
diff --git a/docs/contributing.rst b/docs/contributing.rst
index f084bd23d..68ae2bf3c 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -194,8 +194,9 @@ documentation instead of duplicating the docstring in files within the
 Tests
 =====
 
-Tests are implemented using the `Twisted unit-testing framework`_, running
-tests requires `tox`_.
+Tests are implemented using the :doc:`Twisted unit-testing framework
+<twisted:core/development/policy/test-standard>`. Running tests requires
+`tox`_.
 
 .. _running-tests:
 
@@ -269,7 +270,6 @@ And their unit-tests are in::
 .. _issue tracker: https://github.com/scrapy/scrapy/issues
 .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
 .. _Scrapy subreddit: https://reddit.com/r/scrapy
-.. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
 .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
 .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
 .. _open issues: https://github.com/scrapy/scrapy/issues
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 7c8c40b5f..1c461a511 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -273,5 +273,3 @@ class (which they all inherit from).
 
         Close the given spider. After this is called, no more specific stats
         can be accessed or collected.
-
-.. _reactor: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index 2effe94dc..ae25dfa2f 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -166,11 +166,10 @@ for concurrency.
 For more information about asynchronous programming and Twisted see these
 links:
 
-* `Introduction to Deferreds in Twisted`_
+* :doc:`twisted:core/howto/defer-intro`
 * `Twisted - hello, asynchronous programming`_
 * `Twisted Introduction - Krondo`_
 
 .. _Twisted: https://twistedmatrix.com/trac/
-.. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
 .. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
 .. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/
diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index 12eedf2cd..72bf52227 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -9,13 +9,13 @@ Sending e-mail
 
 Although Python makes sending e-mails relatively easy via the `smtplib`_
 library, Scrapy provides its own facility for sending e-mails which is very
-easy to use and it's implemented using `Twisted non-blocking IO`_, to avoid
-interfering with the non-blocking IO of the crawler. It also provides a
-simple API for sending attachments and it's very easy to configure, with a few
-:ref:`settings <topics-email-settings>`.
+easy to use and it's implemented using :doc:`Twisted non-blocking IO
+<twisted:core/howto/defer-intro>`, to avoid interfering with the non-blocking
+IO of the crawler. It also provides a simple API for sending attachments and
+it's very easy to configure, with a few :ref:`settings
+<topics-email-settings>`.
 
 .. _smtplib: https://docs.python.org/2/library/smtplib.html
-.. _Twisted non-blocking IO: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
 
 Quick example
 =============
@@ -39,7 +39,8 @@ MailSender class reference
 ==========================
 
 MailSender is the preferred class to use for sending emails from Scrapy, as it
-uses `Twisted non-blocking IO`_, like the rest of the framework.
+uses :doc:`Twisted non-blocking IO <twisted:core/howto/defer-intro>`, like the
+rest of the framework.
 
 .. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None)
 
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index fae18200a..cdc4953c2 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -29,7 +29,8 @@ Each item pipeline component is a Python class that must implement the following
 
    This method is called for every item pipeline component. :meth:`process_item`
    must either: return a dict with data, return an :class:`~scrapy.item.Item`
-   (or any descendant class) object, return a `Twisted Deferred`_ or raise
+   (or any descendant class) object, return a
+   :class:`~twisted.internet.defer.Deferred` or raise
    :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
    processed by further pipeline components.
 
@@ -67,8 +68,6 @@ Additionally, they may also implement the following methods:
    :type crawler: :class:`~scrapy.crawler.Crawler` object
 
 
-.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
-
 Item pipeline example
 =====================
 
@@ -166,7 +165,8 @@ method and how to clean up the resources properly.::
 Take screenshot of item
 -----------------------
 
-This example demonstrates how to return Deferred_ from :meth:`process_item` method.
+This example demonstrates how to return a
+:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
 It uses Splash_ to render screenshot of item url. Pipeline
 makes request to locally running instance of Splash_. After request is downloaded
 and Deferred callback fires, it saves item to a file and adds filename to an item.
@@ -209,7 +209,6 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
             return item
 
 .. _Splash: https://splash.readthedocs.io/en/stable/
-.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
 
 Duplicates filter
 -----------------
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 431cc6027..206e7cfa5 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -441,8 +441,9 @@ See here the methods that you can override in your custom Files Pipeline:
       * ``success`` is a boolean which is ``True`` if the image was downloaded
         successfully or ``False`` if it failed for some reason
 
-      * ``file_info_or_error`` is a dict containing the following keys (if success
-        is ``True``) or a `Twisted Failure`_ if there was a problem.
+      * ``file_info_or_error`` is a dict containing the following keys (if
+        success is ``True``) or a :exc:`~twisted.python.failure.Failure` if
+        there was a problem.
 
         * ``url`` - the url where the file was downloaded from. This is the url of
           the request returned from the :meth:`~get_media_requests`
@@ -577,5 +578,4 @@ above::
             item['image_paths'] = image_paths
             return item
 
-.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
 .. _MD5 hash: https://en.wikipedia.org/wiki/MD5
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index a6d4f0d6d..e3e8fdc72 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -101,7 +101,7 @@ reactor after ``MySpider`` has finished running.
     d.addBoth(lambda _: reactor.stop())
     reactor.run() # the script will block here until the crawling is finished
 
-.. seealso:: `Twisted Reactor Overview`_.
+.. seealso:: :doc:`twisted:core/howto/reactor-basics`
 
 .. _run-multiple-spiders:
 
@@ -253,6 +253,5 @@ If you are still unable to prevent your bot getting banned, consider contacting
 .. _ProxyMesh: https://proxymesh.com/
 .. _Google cache: http://www.googleguide.com/cached_pages.html
 .. _testspiders: https://github.com/scrapinghub/testspiders
-.. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
 .. _Crawlera: https://scrapinghub.com/crawlera
 .. _scrapoxy: https://scrapoxy.io/
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index ee37f648e..4cf367d96 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -121,8 +121,8 @@ Request objects
 
     :param errback: a function that will be called if any exception was
        raised while processing the request. This includes pages that failed
-       with 404 HTTP errors and such. It receives a `Twisted Failure`_ instance
-       as first parameter.
+       with 404 HTTP errors and such. It receives a
+       :exc:`~twisted.python.failure.Failure` as first parameter.
        For more information,
        see :ref:`topics-request-response-ref-errbacks` below.
     :type errback: callable
@@ -254,8 +254,8 @@ Using errbacks to catch exceptions in request processing
 The errback of a request is a function that will be called when an exception
 is raise while processing it.
 
-It receives a `Twisted Failure`_ instance as first parameter and can be
-used to track connection establishment timeouts, DNS errors etc.
+It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
+be used to track connection establishment timeouts, DNS errors etc.
 
 Here's an example spider logging all errors and catching some specific
 errors if needed::
@@ -816,5 +816,4 @@ XmlResponse objects
     adds encoding auto-discovering support by looking into the XML declaration
     line.  See :attr:`TextResponse.encoding`.
 
-.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
 .. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index ff07b9d55..3f29aa323 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -50,10 +50,10 @@ Here is a simple example showing how you can catch signals and perform some acti
 Deferred signal handlers
 ========================
 
-Some signals support returning `Twisted deferreds`_ from their handlers, see
-the :ref:`topics-signals-ref` below to know which ones.
+Some signals support returning :class:`~twisted.internet.defer.Deferred`
+objects from their handlers, see the :ref:`topics-signals-ref` below to know
+which ones.
 
-.. _Twisted deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html
 
 .. _topics-signals-ref:
 
@@ -155,8 +155,8 @@ item_error
     :param spider: the spider which raised the exception
     :type spider: :class:`~scrapy.spiders.Spider` object
 
-    :param failure: the exception raised as a Twisted `Failure`_ object
-    :type failure: `Failure`_ object
+    :param failure: the exception raised
+    :type failure: twisted.python.failure.Failure
 
 spider_closed
 -------------
@@ -236,8 +236,8 @@ spider_error
 
     This signal does not support returning deferreds from their handlers.
 
-    :param failure: the exception raised as a Twisted `Failure`_ object
-    :type failure: `Failure`_ object
+    :param failure: the exception raised
+    :type failure: twisted.python.failure.Failure
 
     :param response: the response being processed when the exception was raised
     :type response: :class:`~scrapy.http.Response` object
@@ -333,5 +333,3 @@ response_downloaded
 
     :param spider: the spider for which the response is intended
     :type spider: :class:`~scrapy.spiders.Spider` object
-
-.. _Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 89d2776ae..6e023ebcc 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -67,15 +67,18 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
     """
     Twisted-recommended context factory for web clients.
 
-    Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
-    "The default is to use a BrowserLikePolicyForHTTPS,
-    so unless you have special requirements you can leave this as-is."
+    Quoting the documentation of the :class:`~twisted.web.client.Agent` class:
 
-    creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
-    except this context factory allows setting the TLS/SSL method to use.
+        The default is to use a
+        :class:`~twisted.web.client.BrowserLikePolicyForHTTPS`, so unless you
+        have special requirements you can leave this as-is.
 
-    Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
-    which allows TLS protocol negotiation.
+    :meth:`creatorForNetloc` is the same as
+    :class:`~twisted.web.client.BrowserLikePolicyForHTTPS` except this context
+    factory allows setting the TLS/SSL method to use.
+
+    The default OpenSSL method is ``TLS_METHOD`` (also called
+    ``SSLv23_METHOD``) which allows TLS protocol negotiation.
     """
     def creatorForNetloc(self, hostname, port):
 
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 8868a985b..f8c80880a 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -110,7 +110,7 @@ class Crawler(object):
 class CrawlerRunner(object):
     """
     This is a convenient helper class that keeps track of, manages and runs
-    crawlers inside an already setup Twisted `reactor`_.
+    crawlers inside an already setup :mod:`~twisted.internet.reactor`.
 
     The CrawlerRunner object must be instantiated with a
     :class:`~scrapy.settings.Settings` object.
@@ -233,12 +233,13 @@ class CrawlerProcess(CrawlerRunner):
     A class to run multiple scrapy crawlers in a process simultaneously.
 
     This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support
-    for starting a Twisted `reactor`_ and handling shutdown signals, like the
-    keyboard interrupt command Ctrl-C. It also configures top-level logging.
+    for starting a :mod:`~twisted.internet.reactor` and handling shutdown
+    signals, like the keyboard interrupt command Ctrl-C. It also configures
+    top-level logging.
 
     This utility should be a better fit than
     :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another
-    Twisted `reactor`_ within your application.
+    :mod:`~twisted.internet.reactor` within your application.
 
     The CrawlerProcess object must be instantiated with a
     :class:`~scrapy.settings.Settings` object.
@@ -273,9 +274,9 @@ class CrawlerProcess(CrawlerRunner):
 
     def start(self, stop_after_crawl=True):
         """
-        This method starts a Twisted `reactor`_, adjusts its pool size to
-        :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based
-        on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
+        This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
+        size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
+        based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
 
         If ``stop_after_crawl`` is True, the reactor will be stopped after all
         crawlers have finished, using :meth:`join`.
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index 296d27ed8..9a160f62e 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -46,16 +46,14 @@ class SignalManager(object):
 
     def send_catch_log_deferred(self, signal, **kwargs):
         """
-        Like :meth:`send_catch_log` but supports returning `deferreds`_ from
-        signal handlers.
+        Like :meth:`send_catch_log` but supports returning
+        :class:`~twisted.internet.defer.Deferred` objects from signal handlers.
 
         Returns a Deferred that gets fired once all signal handlers
         deferreds were fired. Send a signal, catch exceptions and log them.
 
         The keyword arguments are passed to the signal handlers (connected
         through the :meth:`connect` method).
-
-        .. _deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html
         """
         kwargs.setdefault('sender', self.sender)
         return _signal.send_catch_log_deferred(signal, **kwargs)

From fed93515de4e306eb3262125c09eb49decdb2944 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 18 Nov 2019 16:11:03 +0100
Subject: [PATCH 2404/2974] Add tooltips to documentation cross-references

---
 docs/conf.py          | 1 +
 docs/requirements.txt | 1 +
 2 files changed, 2 insertions(+)

diff --git a/docs/conf.py b/docs/conf.py
index 6ec4582b1..e2784cf17 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -27,6 +27,7 @@ sys.path.insert(0, path.dirname(path.dirname(__file__)))
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
+    'hoverxref.extension',
     'notfound.extension',
     'scrapydocs',
     'sphinx.ext.autodoc',
diff --git a/docs/requirements.txt b/docs/requirements.txt
index f9db85146..773b92cea 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,3 +1,4 @@
 Sphinx>=2.1
+sphinx-hoverxref
 sphinx-notfound-page
 sphinx_rtd_theme

From f261cf65e999573d95a575ba362c3e32b026f894 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 18 Nov 2019 17:16:09 +0100
Subject: [PATCH 2405/2974] Add missing blank lines between functions and
 classes

Also fixed 2 unrelated Flake8 issues
---
 pytest.ini                                    | 118 ++++++++----------
 scrapy/commands/fetch.py                      |   1 +
 scrapy/commands/list.py                       |   1 +
 scrapy/commands/settings.py                   |   1 +
 scrapy/commands/view.py                       |   1 +
 scrapy/core/downloader/handlers/datauri.py    |   4 +-
 scrapy/downloadermiddlewares/ajaxcrawl.py     |   2 +
 scrapy/dupefilters.py                         |   1 +
 scrapy/exceptions.py                          |  13 ++
 scrapy/extension.py                           |   1 +
 scrapy/extensions/spiderstate.py              |   1 +
 scrapy/http/response/html.py                  |   1 +
 scrapy/http/response/xml.py                   |   1 +
 scrapy/interfaces.py                          |   1 +
 scrapy/loader/common.py                       |   1 +
 scrapy/pipelines/__init__.py                  |   1 +
 scrapy/resolver.py                            |   1 +
 scrapy/robotstxt.py                           |   3 +
 scrapy/utils/console.py                       |   7 ++
 scrapy/utils/decorators.py                    |   1 +
 scrapy/utils/defer.py                         |   9 ++
 scrapy/utils/display.py                       |   3 +
 scrapy/utils/engine.py                        |   3 +
 scrapy/utils/ftp.py                           |   1 +
 scrapy/utils/gz.py                            |   1 +
 scrapy/utils/httpobj.py                       |   3 +
 scrapy/utils/job.py                           |   1 +
 scrapy/utils/python.py                        |   1 +
 scrapy/utils/reactor.py                       |   1 +
 scrapy/utils/request.py                       |   2 +
 scrapy/utils/response.py                      |   4 +
 scrapy/utils/spider.py                        |   1 +
 scrapy/utils/template.py                      |   2 +
 scrapy/utils/test.py                          |   6 +
 scrapy/utils/versions.py                      |   2 +-
 tests/mocks/dummydbm.py                       |   1 +
 tests/pipelines.py                            |   1 +
 tests/spiders.py                              |   1 +
 tests/test_cmdline/extensions.py              |   1 +
 tests/test_command_parse.py                   |   1 +
 tests/test_dependencies.py                    |   1 +
 ...test_downloadermiddleware_ajaxcrawlable.py |   2 +
 tests/test_downloadermiddleware_httpcache.py  |   1 +
 tests/test_dupefilters.py                     |   1 +
 tests/test_http_headers.py                    |   1 +
 tests/test_logformatter.py                    |   1 +
 tests/test_mail.py                            |   1 +
 tests/test_middleware.py                      |   4 +
 tests/test_responsetypes.py                   |   1 +
 tests/test_robotstxt_interface.py             |   2 +
 tests/test_spiderloader/__init__.py           |   1 +
 .../test_spiders/nested/spider4.py            |   1 +
 .../test_spiderloader/test_spiders/spider0.py |   1 +
 .../test_spiderloader/test_spiders/spider1.py |   1 +
 .../test_spiderloader/test_spiders/spider2.py |   1 +
 .../test_spiderloader/test_spiders/spider3.py |   1 +
 tests/test_spidermiddleware_offsite.py        |   2 +
 tests/test_spidermiddleware_output_chain.py   |   4 +
 tests/test_spidermiddleware_referer.py        |   4 +
 tests/test_squeues.py                         |  13 ++
 tests/test_utils_console.py                   |   1 +
 tests/test_utils_defer.py                     |   9 ++
 tests/test_utils_http.py                      |   1 +
 tests/test_utils_httpobj.py                   |   1 +
 tests/test_utils_iterators.py                 |   1 +
 tests/test_utils_request.py                   |   1 +
 tests/test_utils_signal.py                    |   1 +
 tests/test_utils_sitemap.py                   |   1 +
 tests/test_utils_spider.py                    |   3 +
 tests/test_utils_url.py                       |   2 +
 70 files changed, 199 insertions(+), 72 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 529ad5d27..a3693a778 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -30,16 +30,15 @@ flake8-ignore =
     scrapy/commands/check.py F401 E501
     scrapy/commands/crawl.py E501
     scrapy/commands/edit.py E501
-    scrapy/commands/fetch.py E401 E302 E501 E128 E502 E731
+    scrapy/commands/fetch.py E401 E501 E128 E502 E731
     scrapy/commands/genspider.py E128 E501 E502
-    scrapy/commands/list.py E302
     scrapy/commands/parse.py E128 E501 E731 E226
     scrapy/commands/runspider.py E501
-    scrapy/commands/settings.py E302 E128
+    scrapy/commands/settings.py E128
     scrapy/commands/shell.py E128 E501 E502
     scrapy/commands/startproject.py E502 E127 E501 E128
     scrapy/commands/version.py E501 E128
-    scrapy/commands/view.py F401 E302
+    scrapy/commands/view.py F401
     # scrapy/contracts
     scrapy/contracts/__init__.py E501 W504
     scrapy/contracts/default.py E502 E128
@@ -60,7 +59,7 @@ flake8-ignore =
     scrapy/core/downloader/handlers/http11.py E501
     scrapy/core/downloader/handlers/s3.py E501 F401 E502 E128 E126
     # scrapy/downloadermiddlewares
-    scrapy/downloadermiddlewares/ajaxcrawl.py E302 E501 E226
+    scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
     scrapy/downloadermiddlewares/decompression.py E501
     scrapy/downloadermiddlewares/defaultheaders.py E501
     scrapy/downloadermiddlewares/httpcache.py E501 E126
@@ -72,11 +71,11 @@ flake8-ignore =
     scrapy/downloadermiddlewares/stats.py E501
     # scrapy/extensions
     scrapy/extensions/closespider.py E501 E502 E128 E123
-    scrapy/extensions/corestats.py E302 E501
+    scrapy/extensions/corestats.py E501
     scrapy/extensions/feedexport.py E128 E501
     scrapy/extensions/httpcache.py E128 E501 E303 F401
     scrapy/extensions/memdebug.py E501
-    scrapy/extensions/spiderstate.py E302 E501
+    scrapy/extensions/spiderstate.py E501
     scrapy/extensions/telnet.py E501 W504
     scrapy/extensions/throttle.py E501
     # scrapy/http
@@ -87,18 +86,14 @@ flake8-ignore =
     scrapy/http/request/form.py E501 E123
     scrapy/http/request/json_request.py E501
     scrapy/http/response/__init__.py E501 E128 W293 W291
-    scrapy/http/response/html.py E302
     scrapy/http/response/text.py E501 W293 E128 E124
-    scrapy/http/response/xml.py E302
     # scrapy/linkextractors
     scrapy/linkextractors/__init__.py E731 E502 E501 E402 F401
     scrapy/linkextractors/lxmlhtml.py E501 E731 E226
     # scrapy/loader
     scrapy/loader/__init__.py E501 E502 E128
-    scrapy/loader/common.py E302
     scrapy/loader/processors.py E501
     # scrapy/pipelines
-    scrapy/pipelines/__init__.py E302
     scrapy/pipelines/files.py E116 E501 E266
     scrapy/pipelines/images.py E265 E501
     scrapy/pipelines/media.py E125 E501 E266
@@ -123,56 +118,50 @@ flake8-ignore =
     scrapy/utils/benchserver.py E501
     scrapy/utils/boto.py F401
     scrapy/utils/conf.py E402 E502 E501
-    scrapy/utils/console.py E302 E261 F401 E306 E305
+    scrapy/utils/console.py E261 F401 E306 E305
     scrapy/utils/curl.py F401
     scrapy/utils/datatypes.py E501 E226
-    scrapy/utils/decorators.py E501 E302
-    scrapy/utils/defer.py E501 E302 E128
+    scrapy/utils/decorators.py E501
+    scrapy/utils/defer.py E501 E128
     scrapy/utils/deprecate.py E128 E501 E127 E502
-    scrapy/utils/display.py E302
-    scrapy/utils/engine.py F401 E261 E302
-    scrapy/utils/ftp.py E302
-    scrapy/utils/gz.py E305 E501 E302 W504
+    scrapy/utils/engine.py F401 E261
+    scrapy/utils/gz.py E305 E501 W504
     scrapy/utils/http.py F403 F401 E226
-    scrapy/utils/httpobj.py E302 E501
+    scrapy/utils/httpobj.py E501
     scrapy/utils/iterators.py E501 E701
-    scrapy/utils/job.py E302
     scrapy/utils/log.py E128 W503
     scrapy/utils/markup.py F403 F401 W292
     scrapy/utils/misc.py E501 E226
     scrapy/utils/multipart.py F403 F401 W292
     scrapy/utils/project.py E501
-    scrapy/utils/python.py E501 E302
-    scrapy/utils/reactor.py E302 E226
+    scrapy/utils/python.py E501
+    scrapy/utils/reactor.py E226
     scrapy/utils/reqser.py E501
-    scrapy/utils/request.py E302 E127 E501
-    scrapy/utils/response.py E501 E302 E128
+    scrapy/utils/request.py E127 E501
+    scrapy/utils/response.py E501 E128
     scrapy/utils/signal.py E501 E128
     scrapy/utils/sitemap.py E501
-    scrapy/utils/spider.py E271 E302 E501
+    scrapy/utils/spider.py E271 E501
     scrapy/utils/ssl.py E501
-    scrapy/utils/template.py E302
-    scrapy/utils/test.py E302 E501
+    scrapy/utils/test.py E501
     scrapy/utils/url.py E501 F403 F401 E128 F405
     # scrapy
     scrapy/__init__.py E402 E501
     scrapy/_monkeypatches.py W293
     scrapy/cmdline.py E502 E501
     scrapy/crawler.py E501
-    scrapy/dupefilters.py E302 E501 E202
-    scrapy/exceptions.py E302 E501
+    scrapy/dupefilters.py E501 E202
+    scrapy/exceptions.py E501
     scrapy/exporters.py E501 E261 E226
-    scrapy/extension.py E302
-    scrapy/interfaces.py E302 E501
+    scrapy/interfaces.py E501
     scrapy/item.py E501 E128
     scrapy/link.py E501
     scrapy/logformatter.py E501 W293
     scrapy/mail.py E402 E128 E501 E502
     scrapy/middleware.py E502 E128 E501
     scrapy/pqueues.py E501
-    scrapy/resolver.py E302
     scrapy/responsetypes.py E128 E501 E305
-    scrapy/robotstxt.py E302 E501
+    scrapy/robotstxt.py E501
     scrapy/shell.py E501
     scrapy/signalmanager.py E501
     scrapy/spiderloader.py E225 F841 E501 E126
@@ -181,91 +170,82 @@ flake8-ignore =
     # tests
     tests/__init__.py F401 E402 E501
     tests/mockserver.py E401 E501 E126 E123 F401
-    tests/pipelines.py E302 F841 E226
-    tests/spiders.py E302 E501 E127
+    tests/pipelines.py F841 E226
+    tests/spiders.py E501 E127
     tests/test_closespider.py E501 E127
     tests/test_command_fetch.py E501 E261
-    tests/test_command_parse.py F401 E302 E501 E128 E303 E226
+    tests/test_command_parse.py F401 E501 E128 E303 E226
     tests/test_command_shell.py E501 E128
     tests/test_commands.py F401 E128 E501
     tests/test_contracts.py E501 E128 W293
     tests/test_crawl.py E501 E741 E265
     tests/test_crawler.py F841 E306 E501
-    tests/test_dependencies.py E302 F841 E501 E305
+    tests/test_dependencies.py F841 E501 E305
     tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E126 E226 E123
     tests/test_downloadermiddleware.py E501
-    tests/test_downloadermiddleware_ajaxcrawlable.py E302 E501
+    tests/test_downloadermiddleware_ajaxcrawlable.py E501
     tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
     tests/test_downloadermiddleware_decompression.py E127
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
-    tests/test_downloadermiddleware_httpcache.py E501 E302 E305 F401
+    tests/test_downloadermiddleware_httpcache.py E501 E305 F401
     tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123
     tests/test_downloadermiddleware_httpproxy.py F401 E501 E128
     tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
     tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
-    tests/test_dupefilters.py E302 E221 E501 E741 W293 W291 E128 E124
+    tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
     tests/test_engine.py E401 E501 E502 E128 E261
     tests/test_exporters.py E501 E731 E306 E128 E124
     tests/test_extension_telnet.py F401 F841
     tests/test_feedexport.py E501 F401 F841 E241
     tests/test_http_cookies.py E501
-    tests/test_http_headers.py E302 E501
+    tests/test_http_headers.py E501
     tests/test_http_request.py F401 E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
     tests/test_http_response.py E501 E301 E502 E128 E265
     tests/test_item.py E701 E128 F841 E306
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128 E124
-    tests/test_loader.py E302 E501 E731 E303 E741 E128 E117 E241
-    tests/test_logformatter.py E128 E501 E122 E302
-    tests/test_mail.py E302 E128 E501 E305
-    tests/test_middleware.py E302 E501 E128
+    tests/test_loader.py E501 E731 E303 E741 E128 E117 E241
+    tests/test_logformatter.py E128 E501 E122
+    tests/test_mail.py E128 E501 E305
+    tests/test_middleware.py E501 E128
     tests/test_pipeline_crawl.py E131 E501 E128 E126
     tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226
     tests/test_pipeline_images.py F401 F841 E501 E303
     tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
     tests/test_request_cb_kwargs.py E501
-    tests/test_responsetypes.py E501 E302 E305
-    tests/test_robotstxt_interface.py F401 E302 E501 W291 E501
+    tests/test_responsetypes.py E501 E305
+    tests/test_robotstxt_interface.py F401 E501 W291 E501
     tests/test_scheduler.py E501 E126 E123
     tests/test_selector.py F401 E501 E127
     tests/test_spider.py E501 F401
     tests/test_spidermiddleware.py E501 E226
     tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
-    tests/test_spidermiddleware_offsite.py E302 E501 E128 E111 W293
-    tests/test_spidermiddleware_output_chain.py F401 E501 E302 W293 E226
-    tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 E241 E121
-    tests/test_squeues.py E501 E302 E701 E741
+    tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
+    tests/test_spidermiddleware_output_chain.py F401 E501 W293 E226
+    tests/test_spidermiddleware_referer.py F401 E501 F841 E125 E201 E261 E124 E501 E241 E121
+    tests/test_squeues.py E501 E701 E741
     tests/test_utils_conf.py E501 E303 E128
-    tests/test_utils_console.py E302
     tests/test_utils_curl.py E501
     tests/test_utils_datatypes.py E402 E501 E305
-    tests/test_utils_defer.py E306 E261 E501 E302 F841 E226
+    tests/test_utils_defer.py E306 E261 E501 F841 E226
     tests/test_utils_deprecate.py F841 E306 E501
-    tests/test_utils_http.py E302 E501 E502 E128 W504
-    tests/test_utils_httpobj.py E302
-    tests/test_utils_iterators.py E501 E128 E129 E302 E303 E241
+    tests/test_utils_http.py E501 E502 E128 W504
+    tests/test_utils_iterators.py E501 E128 E129 E303 E241
     tests/test_utils_log.py E741 E226
     tests/test_utils_python.py E501 E303 E731 E701 E305
     tests/test_utils_reqser.py F401 E501 E128
-    tests/test_utils_request.py E302 E501 E128 E305
+    tests/test_utils_request.py E501 E128 E305
     tests/test_utils_response.py E501
-    tests/test_utils_signal.py E741 F841 E302 E731 E226
-    tests/test_utils_sitemap.py E302 E128 E501 E124
-    tests/test_utils_spider.py E261 E302 E305
+    tests/test_utils_signal.py E741 F841 E731 E226
+    tests/test_utils_sitemap.py E128 E501 E124
+    tests/test_utils_spider.py E261 E305
     tests/test_utils_template.py E305
-    tests/test_utils_url.py F401 E501 E127 E302 E305 E211 E125 E501 E226 E241 E126 E123
+    tests/test_utils_url.py F401 E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
     tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
-    tests/mocks/dummydbm.py E302
     tests/test_cmdline/__init__.py E502 E501
-    tests/test_cmdline/extensions.py E302
     tests/test_settings/__init__.py F401 E501 E128
-    tests/test_spiderloader/__init__.py E128 E501 E302
-    tests/test_spiderloader/test_spiders/spider0.py E302
-    tests/test_spiderloader/test_spiders/spider1.py E302
-    tests/test_spiderloader/test_spiders/spider2.py E302
-    tests/test_spiderloader/test_spiders/spider3.py E302
-    tests/test_spiderloader/test_spiders/nested/spider4.py E302
+    tests/test_spiderloader/__init__.py E128 E501
     tests/test_utils_misc/__init__.py E501
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index d45133e0e..724b4a1c4 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -8,6 +8,7 @@ from scrapy.exceptions import UsageError
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 
+
 class Command(ScrapyCommand):
 
     requires_project = False
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index a255b3b94..422183ac1 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -1,6 +1,7 @@
 from __future__ import print_function
 from scrapy.commands import ScrapyCommand
 
+
 class Command(ScrapyCommand):
 
     requires_project = True
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index bee52f06a..ffe3aa2eb 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -4,6 +4,7 @@ import json
 from scrapy.commands import ScrapyCommand
 from scrapy.settings import BaseSettings
 
+
 class Command(ScrapyCommand):
 
     requires_project = False
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index 59e665016..31c17c0ab 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -1,6 +1,7 @@
 from scrapy.commands import fetch, ScrapyCommand
 from scrapy.utils.response import open_in_browser
 
+
 class Command(fetch.Command):
 
     def short_desc(self):
diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py
index ad25beb3b..9e5020753 100644
--- a/scrapy/core/downloader/handlers/datauri.py
+++ b/scrapy/core/downloader/handlers/datauri.py
@@ -17,8 +17,8 @@ class DataURIDownloadHandler(object):
         respcls = responsetypes.from_mimetype(uri.media_type)
 
         resp_kwargs = {}
-        if (issubclass(respcls, TextResponse) and
-                uri.media_type.split('/')[0] == 'text'):
+        if (issubclass(respcls, TextResponse)
+                and uri.media_type.split('/')[0] == 'text'):
             charset = uri.media_type_parameters.get('charset')
             resp_kwargs['encoding'] = charset
 
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 72715dba7..ba50793bb 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -68,6 +68,8 @@ class AjaxCrawlMiddleware(object):
 
 # XXX: move it to w3lib?
 _ajax_crawlable_re = re.compile(six.u(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'))
+
+
 def _has_ajaxcrawlable_meta(text):
     """
     >>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment"  content="!"/></head><body></body></html>')
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 0bcdd3495..4d95eb847 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -5,6 +5,7 @@ import logging
 from scrapy.utils.job import job_dir
 from scrapy.utils.request import referer_str, request_fingerprint
 
+
 class BaseDupeFilter(object):
 
     @classmethod
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 96949bdd9..7c4bb3d00 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -7,10 +7,12 @@ new exceptions here without documenting them there.
 
 # Internal
 
+
 class NotConfigured(Exception):
     """Indicates a missing configuration situation"""
     pass
 
+
 class _InvalidOutput(TypeError):
     """
     Indicates an invalid value has been returned by a middleware's processing method.
@@ -18,15 +20,19 @@ class _InvalidOutput(TypeError):
     """
     pass
 
+
 # HTTP and crawling
 
+
 class IgnoreRequest(Exception):
     """Indicates a decision was made not to process a request"""
 
+
 class DontCloseSpider(Exception):
     """Request the spider not to be closed yet"""
     pass
 
+
 class CloseSpider(Exception):
     """Raise this from callbacks to request the spider to be closed"""
 
@@ -34,30 +40,37 @@ class CloseSpider(Exception):
         super(CloseSpider, self).__init__()
         self.reason = reason
 
+
 # Items
 
+
 class DropItem(Exception):
     """Drop item from the item pipeline"""
     pass
 
+
 class NotSupported(Exception):
     """Indicates a feature or method is not supported"""
     pass
 
+
 # Commands
 
+
 class UsageError(Exception):
     """To indicate a command-line usage error"""
     def __init__(self, *a, **kw):
         self.print_help = kw.pop('print_help', True)
         super(UsageError, self).__init__(*a, **kw)
 
+
 class ScrapyDeprecationWarning(Warning):
     """Warning category for deprecated features, since the default
     DeprecationWarning is silenced on Python 2.7+
     """
     pass
 
+
 class ContractFail(AssertionError):
     """Error raised in case of a failing contract"""
     pass
diff --git a/scrapy/extension.py b/scrapy/extension.py
index e39e456fa..050b87e5f 100644
--- a/scrapy/extension.py
+++ b/scrapy/extension.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.conf import build_component_list
 
+
 class ExtensionManager(MiddlewareManager):
 
     component_name = 'extension'
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 2220cbd8f..8ba770ec0 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -5,6 +5,7 @@ from scrapy import signals
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.job import job_dir
 
+
 class SpiderState(object):
     """Store and load spider state during a scraping job"""
 
diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py
index bd3559fbb..7eed052c2 100644
--- a/scrapy/http/response/html.py
+++ b/scrapy/http/response/html.py
@@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst
 
 from scrapy.http.response.text import TextResponse
 
+
 class HtmlResponse(TextResponse):
     pass
diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py
index 1df33fee5..abf474a2f 100644
--- a/scrapy/http/response/xml.py
+++ b/scrapy/http/response/xml.py
@@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst
 
 from scrapy.http.response.text import TextResponse
 
+
 class XmlResponse(TextResponse):
     pass
diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py
index d48babc3c..1896ec31e 100644
--- a/scrapy/interfaces.py
+++ b/scrapy/interfaces.py
@@ -1,5 +1,6 @@
 from zope.interface import Interface
 
+
 class ISpiderLoader(Interface):
 
     def from_settings(settings):
diff --git a/scrapy/loader/common.py b/scrapy/loader/common.py
index 916524947..42f8de636 100644
--- a/scrapy/loader/common.py
+++ b/scrapy/loader/common.py
@@ -3,6 +3,7 @@
 from functools import partial
 from scrapy.utils.python import get_func_args
 
+
 def wrap_loader_context(function, context):
     """Wrap functions that receive loader_context to contain the context
     "pre-loaded" and expose a interface that receives only one argument
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index 2ef8786d0..aa1bfb77f 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -7,6 +7,7 @@ See documentation in docs/item-pipeline.rst
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.conf import build_component_list
 
+
 class ItemPipelineManager(MiddlewareManager):
 
     component_name = 'item pipeline'
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 0aaced7e4..4df949015 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -7,6 +7,7 @@ from scrapy.utils.datatypes import LocalCache
 
 dnscache = LocalCache(10000)
 
+
 class CachingThreadedResolver(ThreadedResolver):
     def __init__(self, reactor, cache_size, timeout):
         super(CachingThreadedResolver, self).__init__(reactor)
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 95a8c09b8..f0f9c59dc 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -5,8 +5,10 @@ from six import with_metaclass
 
 from scrapy.utils.python import to_unicode
 
+
 logger = logging.getLogger(__name__)
 
+
 def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
     try:
         if to_native_str_type:
@@ -23,6 +25,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
         robotstxt_body = ''
     return robotstxt_body
 
+
 class RobotParser(with_metaclass(ABCMeta)):
     @classmethod
     @abstractmethod
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 2e9981556..a26e84d38 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -1,6 +1,7 @@
 from functools import wraps
 from collections import OrderedDict
 
+
 def _embed_ipython_shell(namespace={}, banner=''):
     """Start an IPython Shell"""
     try:
@@ -23,6 +24,7 @@ def _embed_ipython_shell(namespace={}, banner=''):
         shell()
     return wrapper
 
+
 def _embed_bpython_shell(namespace={}, banner=''):
     """Start a bpython shell"""
     import bpython
@@ -31,6 +33,7 @@ def _embed_bpython_shell(namespace={}, banner=''):
         bpython.embed(locals_=namespace, banner=banner)
     return wrapper
 
+
 def _embed_ptpython_shell(namespace={}, banner=''):
     """Start a ptpython shell"""
     import ptpython.repl
@@ -40,6 +43,7 @@ def _embed_ptpython_shell(namespace={}, banner=''):
         ptpython.repl.embed(locals=namespace)
     return wrapper
 
+
 def _embed_standard_shell(namespace={}, banner=''):
     """Start a standard python shell"""
     import code
@@ -55,6 +59,7 @@ def _embed_standard_shell(namespace={}, banner=''):
         code.interact(banner=banner, local=namespace)
     return wrapper
 
+
 DEFAULT_PYTHON_SHELLS = OrderedDict([
     ('ptpython', _embed_ptpython_shell),
     ('ipython', _embed_ipython_shell),
@@ -62,6 +67,7 @@ DEFAULT_PYTHON_SHELLS = OrderedDict([
     ('python', _embed_standard_shell),
 ])
 
+
 def get_shell_embed_func(shells=None, known_shells=None):
     """Return the first acceptable shell-embed function
     from a given list of shell names.
@@ -79,6 +85,7 @@ def get_shell_embed_func(shells=None, known_shells=None):
             except ImportError:
                 continue
 
+
 def start_python_console(namespace=None, banner='', shells=None):
     """Start Python console bound to the given namespace.
     Readline support and tab completion will be used on Unix, if available.
diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py
index 38bee1a6c..2e2c7adc1 100644
--- a/scrapy/utils/decorators.py
+++ b/scrapy/utils/decorators.py
@@ -34,6 +34,7 @@ def defers(func):
         return defer.maybeDeferred(func, *a, **kw)
     return wrapped
 
+
 def inthread(func):
     """Decorator to call a function in a thread and return a deferred with the
     result
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 69d621830..c5916c21c 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -7,6 +7,7 @@ from twisted.python import failure
 
 from scrapy.exceptions import IgnoreRequest
 
+
 def defer_fail(_failure):
     """Same as twisted.internet.defer.fail but delay calling errback until
     next reactor loop
@@ -18,6 +19,7 @@ def defer_fail(_failure):
     reactor.callLater(0.1, d.errback, _failure)
     return d
 
+
 def defer_succeed(result):
     """Same as twisted.internet.defer.succeed but delay calling callback until
     next reactor loop
@@ -29,6 +31,7 @@ def defer_succeed(result):
     reactor.callLater(0.1, d.callback, result)
     return d
 
+
 def defer_result(result):
     if isinstance(result, defer.Deferred):
         return result
@@ -37,6 +40,7 @@ def defer_result(result):
     else:
         return defer_succeed(result)
 
+
 def mustbe_deferred(f, *args, **kw):
     """Same as twisted.internet.defer.maybeDeferred, but delay calling
     callback/errback to next reactor loop
@@ -53,6 +57,7 @@ def mustbe_deferred(f, *args, **kw):
     else:
         return defer_result(result)
 
+
 def parallel(iterable, count, callable, *args, **named):
     """Execute a callable over the objects in the given iterable, in parallel,
     using no more than ``count`` concurrent calls.
@@ -63,6 +68,7 @@ def parallel(iterable, count, callable, *args, **named):
     work = (callable(elem, *args, **named) for elem in iterable)
     return defer.DeferredList([coop.coiterate(work) for _ in range(count)])
 
+
 def process_chain(callbacks, input, *a, **kw):
     """Return a Deferred built by chaining the given callbacks"""
     d = defer.Deferred()
@@ -71,6 +77,7 @@ def process_chain(callbacks, input, *a, **kw):
     d.callback(input)
     return d
 
+
 def process_chain_both(callbacks, errbacks, input, *a, **kw):
     """Return a Deferred built by chaining the given callbacks and errbacks"""
     d = defer.Deferred()
@@ -83,6 +90,7 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw):
         d.callback(input)
     return d
 
+
 def process_parallel(callbacks, input, *a, **kw):
     """Return a Deferred with the output of all successful calls to the given
     callbacks
@@ -92,6 +100,7 @@ def process_parallel(callbacks, input, *a, **kw):
     d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure)
     return d
 
+
 def iter_errback(iterable, errback, *a, **kw):
     """Wraps an iterable calling an errback if an error is caught while
     iterating it.
diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py
index f6a6c4645..91ebdae11 100644
--- a/scrapy/utils/display.py
+++ b/scrapy/utils/display.py
@@ -6,6 +6,7 @@ from __future__ import print_function
 import sys
 from pprint import pformat as pformat_
 
+
 def _colorize(text, colorize=True):
     if not colorize or not sys.stdout.isatty():
         return text
@@ -17,8 +18,10 @@ def _colorize(text, colorize=True):
     except ImportError:
         return text
 
+
 def pformat(obj, *args, **kwargs):
     return _colorize(pformat_(obj), kwargs.pop('colorize', True))
 
+
 def pprint(obj, *args, **kwargs):
     print(pformat(obj, *args, **kwargs))
diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py
index 11dd36d91..2c20b5c88 100644
--- a/scrapy/utils/engine.py
+++ b/scrapy/utils/engine.py
@@ -3,6 +3,7 @@
 from __future__ import print_function
 from time import time # used in global tests code
 
+
 def get_engine_status(engine):
     """Return a report of the current engine status"""
     tests = [
@@ -32,6 +33,7 @@ def get_engine_status(engine):
 
     return checks
 
+
 def format_engine_status(engine=None):
     checks = get_engine_status(engine)
     s = "Execution engine status\n\n"
@@ -41,5 +43,6 @@ def format_engine_status(engine=None):
 
     return s
 
+
 def print_engine_status(engine):
     print(format_engine_status(engine))
diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index 9eca6a4da..91d2439a9 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -1,6 +1,7 @@
 from ftplib import error_perm
 from posixpath import dirname
 
+
 def ftp_makedirs_cwd(ftp, path, first_call=True):
     """Set the current directory of the FTP connection given in the ``ftp``
     argument (as a ftplib.FTP object), creating all parent directories if they
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index f41e62fe3..9672e28da 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -45,6 +45,7 @@ def gunzip(data):
 _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
 _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search
 
+
 @deprecated
 def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py
index b4c929b0e..b2be0a901 100644
--- a/scrapy/utils/httpobj.py
+++ b/scrapy/utils/httpobj.py
@@ -4,7 +4,10 @@ import weakref
 
 from six.moves.urllib.parse import urlparse
 
+
 _urlparse_cache = weakref.WeakKeyDictionary()
+
+
 def urlparse_cached(request_or_response):
     """Return urlparse.urlparse caching the result, where the argument can be a
     Request or Response object
diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py
index 389fde73a..4f1e601fc 100644
--- a/scrapy/utils/job.py
+++ b/scrapy/utils/job.py
@@ -1,5 +1,6 @@
 import os
 
+
 def job_dir(settings):
     path = settings['JOBDIR']
     if path and not os.path.exists(path):
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 663a8ebaa..a4201bb04 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -165,6 +165,7 @@ def memoizemethod_noargs(method):
 _BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
 _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
 
+
 @deprecated("scrapy.utils.python.binary_is_text")
 def isbinarytext(text):
     """ This function is deprecated.
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 83186a372..b4b5f0645 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -1,5 +1,6 @@
 from twisted.internet import reactor, error
 
+
 def listen_tcp(portrange, host, factory):
     """Like reactor.listenTCP but tries different ports in a range."""
     assert len(portrange) <= 2, "invalid portrange: %s" % portrange
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 63d0ae772..0fce5a2e1 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -16,6 +16,8 @@ from scrapy.utils.httpobj import urlparse_cached
 
 
 _fingerprint_cache = weakref.WeakKeyDictionary()
+
+
 def request_fingerprint(request, include_headers=None, keep_fragments=False):
     """
     Return the request fingerprint.
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index feab07431..29fdaaf2c 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -13,6 +13,8 @@ from w3lib import html
 
 
 _baseurl_cache = weakref.WeakKeyDictionary()
+
+
 def get_base_url(response):
     """Return the base url of the given response, joined with the response url"""
     if response not in _baseurl_cache:
@@ -23,6 +25,8 @@ def get_base_url(response):
 
 
 _metaref_cache = weakref.WeakKeyDictionary()
+
+
 def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
     """Parse the http-equiv refrsh parameter from the given response"""
     if response not in _metaref_cache:
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 94b24f67e..bf4973fbf 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -28,6 +28,7 @@ def iter_spider_classes(module):
            getattr(obj, 'name', None):
             yield obj
 
+
 def spidercls_for_request(spider_loader, request, default_spidercls=None,
                           log_none=False, log_multiple=False):
     """Return a spider class that handles the given Request.
diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py
index 615372fc8..96ff4b09b 100644
--- a/scrapy/utils/template.py
+++ b/scrapy/utils/template.py
@@ -19,6 +19,8 @@ def render_templatefile(path, **kwargs):
 
 
 CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]')
+
+
 def string_camelcase(string):
     """ Convert a word  to its CamelCase version and remove invalid chars
 
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 4b935c51b..9754366df 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -32,6 +32,7 @@ def skip_if_no_boto():
     except NotConfigured as e:
         raise SkipTest(e)
 
+
 def get_s3_content_and_delete(bucket, path, with_key=False):
     """ Get content from s3 key, and delete key afterwards.
     """
@@ -51,6 +52,7 @@ def get_s3_content_and_delete(bucket, path, with_key=False):
         bucket.delete_key(path)
     return (content, key) if with_key else content
 
+
 def get_gcs_content_and_delete(bucket, path):
     from google.cloud import storage
     client = storage.Client(project=os.environ.get('GCS_PROJECT_ID'))
@@ -61,6 +63,7 @@ def get_gcs_content_and_delete(bucket, path):
     bucket.delete_blob(path)
     return content, acl, blob
 
+
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level
@@ -72,12 +75,14 @@ def get_crawler(spidercls=None, settings_dict=None):
     runner = CrawlerRunner(settings_dict)
     return runner.create_crawler(spidercls or Spider)
 
+
 def get_pythonpath():
     """Return a PYTHONPATH suitable to use in processes so that they find this
     installation of Scrapy"""
     scrapy_path = import_module('scrapy').__path__[0]
     return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '')
 
+
 def get_testenv():
     """Return a OS environment dict suitable to fork processes that need to import
     this installation of Scrapy, instead of a system installed one.
@@ -86,6 +91,7 @@ def get_testenv():
     env['PYTHONPATH'] = get_pythonpath()
     return env
 
+
 def assert_samelines(testcase, text1, text2, msg=None):
     """Asserts text1 and text2 have the same lines, ignoring differences in
     line endings between platforms
diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py
index 48484b303..b0737d3d5 100644
--- a/scrapy/utils/versions.py
+++ b/scrapy/utils/versions.py
@@ -27,5 +27,5 @@ def scrapy_components_versions():
         ("Python", sys.version.replace("\n", "- ")),
         ("pyOpenSSL", get_openssl_version()),
         ("cryptography", cryptography.__version__),
-        ("Platform",  platform.platform()),
+        ("Platform", platform.platform()),
     ]
diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py
index 431428331..75c74daf5 100644
--- a/tests/mocks/dummydbm.py
+++ b/tests/mocks/dummydbm.py
@@ -13,6 +13,7 @@ error = KeyError
 
 _DATABASES = collections.defaultdict(DummyDB)
 
+
 def open(file, flag='r', mode=0o666):
     """Open or create a dummy database compatible.
 
diff --git a/tests/pipelines.py b/tests/pipelines.py
index 7e2895a5c..d7d3b5259 100644
--- a/tests/pipelines.py
+++ b/tests/pipelines.py
@@ -2,6 +2,7 @@
 Some pipelines used for testing
 """
 
+
 class ZeroDivisionErrorPipeline(object):
 
     def open_spider(self, spider):
diff --git a/tests/spiders.py b/tests/spiders.py
index 7816bf7c7..2487ecc22 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -16,6 +16,7 @@ class MockServerSpider(Spider):
         super(MockServerSpider, self).__init__(*args, **kwargs)
         self.mockserver = mockserver
 
+
 class MetaSpider(MockServerSpider):
 
     name = 'meta'
diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py
index 28456b55d..c64e87d81 100644
--- a/tests/test_cmdline/extensions.py
+++ b/tests/test_cmdline/extensions.py
@@ -1,5 +1,6 @@
 """A test extension used to check the settings loading order"""
 
+
 class TestExtension(object):
 
     def __init__(self, settings):
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index b134beb88..b7035fdff 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -12,6 +12,7 @@ def _textmode(bstr):
     and reading from it in text mode"""
     return to_unicode(bstr).replace(os.linesep, '\n')
 
+
 class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
     command = 'parse'
 
diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
index 03bf2ffcf..e31ccd9b5 100644
--- a/tests/test_dependencies.py
+++ b/tests/test_dependencies.py
@@ -1,6 +1,7 @@
 from importlib import import_module
 from twisted.trial import unittest
 
+
 class ScrapyUtilsTest(unittest.TestCase):
     def test_required_openssl_version(self):
         try:
diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py
index 493691ea4..5a56c9db2 100644
--- a/tests/test_downloadermiddleware_ajaxcrawlable.py
+++ b/tests/test_downloadermiddleware_ajaxcrawlable.py
@@ -5,8 +5,10 @@ from scrapy.spiders import Spider
 from scrapy.http import Request, HtmlResponse, Response
 from scrapy.utils.test import get_crawler
 
+
 __doctests__ = ['scrapy.downloadermiddlewares.ajaxcrawl']
 
+
 class AjaxCrawlMiddlewareTest(unittest.TestCase):
     def setUp(self):
         crawler = get_crawler(Spider, {'AJAXCRAWL_ENABLED': True})
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 9d863b6e3..00e6c685e 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -149,6 +149,7 @@ class FilesystemStorageTest(DefaultStorageTest):
 
     storage_class = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
 
+
 class FilesystemStorageGzipTest(FilesystemStorageTest):
 
     def _get_settings(self, **new_settings):
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index d7eb98c97..e4b0bdf83 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -12,6 +12,7 @@ from scrapy.utils.job import job_dir
 from scrapy.utils.test import get_crawler
 from tests.spiders import SimpleSpider
 
+
 class FromCrawlerRFPDupeFilter(RFPDupeFilter):
 
     @classmethod
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 69d906fbf..c83cf3b66 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -3,6 +3,7 @@ import copy
 
 from scrapy.http import Headers
 
+
 class HeadersTest(unittest.TestCase):
 
     def assertSortedEqual(self, first, second, msg=None):
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index b4ea30bb7..0724d1807 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -118,6 +118,7 @@ class DropSomeItemsPipeline(object):
         else:
             self.drop = True
 
+
 class ShowOrSkipMessagesTestCase(TwistedTestCase):
     def setUp(self):
         self.mockserver = MockServer()
diff --git a/tests/test_mail.py b/tests/test_mail.py
index b139e98d8..ddb0f1e70 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -6,6 +6,7 @@ from email.charset import Charset
 
 from scrapy.mail import MailSender
 
+
 class MailSenderTest(unittest.TestCase):
 
     def test_send(self):
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index af9b43d61..ebf817c7e 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -4,6 +4,7 @@ from scrapy.settings import Settings
 from scrapy.exceptions import NotConfigured
 from scrapy.middleware import MiddlewareManager
 
+
 class M1(object):
 
     def open_spider(self, spider):
@@ -15,6 +16,7 @@ class M1(object):
     def process(self, response, request, spider):
         pass
 
+
 class M2(object):
 
     def open_spider(self, spider):
@@ -25,6 +27,7 @@ class M2(object):
 
     pass
 
+
 class M3(object):
 
     def process(self, response, request, spider):
@@ -54,6 +57,7 @@ class TestMiddlewareManager(MiddlewareManager):
         if hasattr(mw, 'process'):
             self.methods['process'].append(mw.process)
 
+
 class MiddlewareManagerTest(unittest.TestCase):
 
     def test_init(self):
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index f89042b3d..d5a3371ab 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -4,6 +4,7 @@ from scrapy.responsetypes import responsetypes
 
 from scrapy.http import Response, TextResponse, XmlResponse, HtmlResponse, Headers
 
+
 class ResponseTypesTest(unittest.TestCase):
 
     def test_from_filename(self):
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index cd7480e33..080507276 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -19,6 +19,7 @@ def rerp_available():
         return False
     return True
 
+
 def protego_available():
     # check if protego parser is installed
     try:
@@ -27,6 +28,7 @@ def protego_available():
         return False
     return True
 
+
 class BaseRobotParserTest:
     def _setUp(self, parser_cls):
         self.parser_cls = parser_cls
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 106da798c..d8be6e277 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -109,6 +109,7 @@ class SpiderLoaderTest(unittest.TestCase):
             spiders = spider_loader.list()
             self.assertEqual(spiders, [])
 
+
 class DuplicateSpiderNameLoaderTest(unittest.TestCase):
 
     def setUp(self):
diff --git a/tests/test_spiderloader/test_spiders/nested/spider4.py b/tests/test_spiderloader/test_spiders/nested/spider4.py
index 35b71870a..dbd1fb123 100644
--- a/tests/test_spiderloader/test_spiders/nested/spider4.py
+++ b/tests/test_spiderloader/test_spiders/nested/spider4.py
@@ -1,5 +1,6 @@
 from scrapy.spiders import Spider
 
+
 class Spider4(Spider):
     name = "spider4"
     allowed_domains = ['spider4.com']
diff --git a/tests/test_spiderloader/test_spiders/spider0.py b/tests/test_spiderloader/test_spiders/spider0.py
index 75a90794e..af679dbd6 100644
--- a/tests/test_spiderloader/test_spiders/spider0.py
+++ b/tests/test_spiderloader/test_spiders/spider0.py
@@ -1,4 +1,5 @@
 from scrapy.spiders import Spider
 
+
 class Spider0(Spider):
     allowed_domains = ["scrapy1.org", "scrapy3.org"]
diff --git a/tests/test_spiderloader/test_spiders/spider1.py b/tests/test_spiderloader/test_spiders/spider1.py
index 76efddc7f..6b4317a90 100644
--- a/tests/test_spiderloader/test_spiders/spider1.py
+++ b/tests/test_spiderloader/test_spiders/spider1.py
@@ -1,5 +1,6 @@
 from scrapy.spiders import Spider
 
+
 class Spider1(Spider):
     name = "spider1"
     allowed_domains = ["scrapy1.org", "scrapy3.org"]
diff --git a/tests/test_spiderloader/test_spiders/spider2.py b/tests/test_spiderloader/test_spiders/spider2.py
index 0badd8437..352601863 100644
--- a/tests/test_spiderloader/test_spiders/spider2.py
+++ b/tests/test_spiderloader/test_spiders/spider2.py
@@ -1,5 +1,6 @@
 from scrapy.spiders import Spider
 
+
 class Spider2(Spider):
     name = "spider2"
     allowed_domains = ["scrapy2.org", "scrapy3.org"]
diff --git a/tests/test_spiderloader/test_spiders/spider3.py b/tests/test_spiderloader/test_spiders/spider3.py
index d406f2d4f..84998ba35 100644
--- a/tests/test_spiderloader/test_spiders/spider3.py
+++ b/tests/test_spiderloader/test_spiders/spider3.py
@@ -1,5 +1,6 @@
 from scrapy.spiders import Spider
 
+
 class Spider3(Spider):
     name = "spider3"
     allowed_domains = ['spider3.com']
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 7e4af0d4c..b97d9b675 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -9,6 +9,7 @@ from scrapy.spidermiddlewares.offsite import URLWarning
 from scrapy.utils.test import get_crawler
 import warnings
 
+
 class TestOffsiteMiddleware(TestCase):
 
     def setUp(self):
@@ -53,6 +54,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware):
         out = list(self.mw.process_spider_output(res, reqs, self.spider))
         self.assertEqual(out, reqs)
 
+
 class TestOffsiteMiddleware3(TestOffsiteMiddleware2):
 
     def _get_spider(self):
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 6f8727a15..940e31070 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -34,6 +34,7 @@ class RecoverySpider(Spider):
         if not response.meta.get('dont_fail'):
             raise TabError()
 
+
 class RecoveryMiddleware:
     def process_spider_exception(self, response, exception, spider):
         spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
@@ -50,6 +51,7 @@ class FailProcessSpiderInputMiddleware:
         spider.logger.info('Middleware: will raise IndexError')
         raise IndexError()
 
+
 class ProcessSpiderInputSpiderWithoutErrback(Spider):
     name = 'ProcessSpiderInputSpiderWithoutErrback'
     custom_settings = {
@@ -177,6 +179,7 @@ class GeneratorRecoverMiddleware:
         spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
         yield {'processed': [method]}
 
+
 class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
     pass
 
@@ -247,6 +250,7 @@ class NotGeneratorRecoverMiddleware:
         spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
         return [{'processed': [method]}]
 
+
 class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware):
     pass
 
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 21439c20e..a9c31a983 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -349,6 +349,7 @@ class TestSettingsCustomPolicy(TestRefererMiddleware):
 
     ]
 
+
 # --- Tests using Request meta dict to set policy
 class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT}
@@ -518,14 +519,17 @@ class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
     resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}
 
+
 class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
     resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}
 
+
 class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
     resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
 
+
 class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
     """
     The empty string means "no-referrer-when-downgrade"
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 3ded5c027..d5fcf2f7f 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -7,16 +7,20 @@ from scrapy.http import Request
 from scrapy.loader import ItemLoader
 from scrapy.selector import Selector
 
+
 class TestItem(Item):
     name = Field()
 
+
 def _test_procesor(x):
     return x + x
 
+
 class TestLoader(ItemLoader):
     default_item_class = TestItem
     name_out = staticmethod(_test_procesor)
 
+
 def nonserializable_object_test(self):
     q = self.queue()
     try:
@@ -35,6 +39,7 @@ def nonserializable_object_test(self):
     sel = Selector(text='<html><body><p>some text</p></body></html>')
     self.assertRaises(ValueError, q.push, sel)
 
+
 class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
 
     chunksize = 100000
@@ -53,15 +58,19 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
 
     test_nonserializable_object = nonserializable_object_test
 
+
 class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
     chunksize = 1
 
+
 class ChunkSize2MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
     chunksize = 2
 
+
 class ChunkSize3MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
     chunksize = 3
 
+
 class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
     chunksize = 4
 
@@ -100,15 +109,19 @@ class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest):
         self.assertEqual(r.url, r2.url)
         assert r2.meta['request'] is r2
 
+
 class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
     chunksize = 1
 
+
 class ChunkSize2PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
     chunksize = 2
 
+
 class ChunkSize3PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
     chunksize = 3
 
+
 class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
     chunksize = 4
 
diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py
index c2211848c..380c41367 100644
--- a/tests/test_utils_console.py
+++ b/tests/test_utils_console.py
@@ -14,6 +14,7 @@ try:
 except ImportError:
     ipy = False
 
+
 class UtilsConsoleTestCase(unittest.TestCase):
 
     def test_get_shell_embed_func(self):
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 003bb9b02..d642ed3ed 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -33,14 +33,23 @@ class MustbeDeferredTest(unittest.TestCase):
         steps.append(2) # add another value, that should be catched by assertEqual
         return dfd
 
+
 def cb1(value, arg1, arg2):
     return "(cb1 %s %s %s)" % (value, arg1, arg2)
+
+
 def cb2(value, arg1, arg2):
     return defer.succeed("(cb2 %s %s %s)" % (value, arg1, arg2))
+
+
 def cb3(value, arg1, arg2):
     return "(cb3 %s %s %s)" % (value, arg1, arg2)
+
+
 def cb_fail(value, arg1, arg2):
     return Failure(TypeError())
+
+
 def eb1(failure, arg1, arg2):
     return "(eb1 %s %s %s)" % (failure.value.__class__.__name__, arg1, arg2)
 
diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py
index 2524153ea..f9af4bf87 100644
--- a/tests/test_utils_http.py
+++ b/tests/test_utils_http.py
@@ -2,6 +2,7 @@ import unittest
 
 from scrapy.utils.http import decode_chunked_transfer
 
+
 class ChunkedTest(unittest.TestCase):
 
     def test_decode_chunked_transfer(self):
diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py
index 4f9f7a370..2c3965bbc 100644
--- a/tests/test_utils_httpobj.py
+++ b/tests/test_utils_httpobj.py
@@ -4,6 +4,7 @@ from six.moves.urllib.parse import urlparse
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 
+
 class HttpobjUtilsTest(unittest.TestCase):
 
     def test_urlparse_cached(self):
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 2d845697e..f16ef8110 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -235,6 +235,7 @@ class LxmlXmliterTestCase(XmliterTestCase):
         i = self.xmliter(42, 'product')
         self.assertRaises(TypeError, next, i)
 
+
 class UtilsCsvTestCase(unittest.TestCase):
     sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')
     sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv')
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 625a32048..3da95b95a 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -4,6 +4,7 @@ from scrapy.http import Request
 from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \
     request_authenticate, request_httprepr
 
+
 class UtilsRequestTest(unittest.TestCase):
 
     def test_request_fingerprint(self):
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index 62edd420d..16b7c5c68 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -66,6 +66,7 @@ class SendCatchLogDeferredTest2(SendCatchLogTest):
     def _get_result(self, signal, *a, **kw):
         return send_catch_log_deferred(signal, *a, **kw)
 
+
 class SendCatchLogTest2(unittest.TestCase):
 
     def test_error_logged_if_deferred_not_supported(self):
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index 716bb44eb..db323ab31 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -2,6 +2,7 @@ import unittest
 
 from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
 
+
 class SitemapTest(unittest.TestCase):
 
     def test_sitemap(self):
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index d9de1ce77..edeeacc80 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -9,12 +9,15 @@ from scrapy.spiders import CrawlSpider
 class MyBaseSpider(CrawlSpider):
     pass # abstract spider
 
+
 class MySpider1(MyBaseSpider):
     name = 'myspider1'
 
+
 class MySpider2(MyBaseSpider):
     name = 'myspider2'
 
+
 class UtilsSpidersTestCase(unittest.TestCase):
 
     def test_iterate_spider_output(self):
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index e6588055c..93addc082 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -187,6 +187,7 @@ class AddHttpIfNoScheme(unittest.TestCase):
 class GuessSchemeTest(unittest.TestCase):
     pass
 
+
 def create_guess_scheme_t(args):
     def do_expected(self):
         url = guess_scheme(args[0])
@@ -195,6 +196,7 @@ def create_guess_scheme_t(args):
                 args[0], url, args[1])
     return do_expected
 
+
 def create_skipped_scheme_t(args):
     def do_expected(self):
         raise unittest.SkipTest(args[2])

From e18014d84d058d7a702c7b0f8c190ee5468abc7d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 22 Jul 2019 20:51:03 +0500
Subject: [PATCH 2406/2974] Remove Python 2-only tests.

---
 conftest.py                             |  10 +-
 tests/py3-ignores.txt                   |   3 -
 tests/test_downloader_handlers.py       |  22 +--
 tests/test_linkextractors_deprecated.py | 233 ------------------------
 tests/test_proxy_connect.py             | 120 ------------
 tests/test_utils_python.py              |  27 ---
 tests/test_webclient.py                 |  20 --
 7 files changed, 14 insertions(+), 421 deletions(-)
 delete mode 100644 tests/test_linkextractors_deprecated.py
 delete mode 100644 tests/test_proxy_connect.py

diff --git a/conftest.py b/conftest.py
index d5d61ddd3..74fb101e9 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,4 +1,3 @@
-import six
 import pytest
 
 
@@ -8,11 +7,10 @@ collect_ignore = [
 ]
 
 
-if six.PY3:
-    for line in open('tests/py3-ignores.txt'):
-        file_path = line.strip()
-        if file_path and file_path[0] != '#':
-            collect_ignore.append(file_path)
+for line in open('tests/py3-ignores.txt'):
+    file_path = line.strip()
+    if file_path and file_path[0] != '#':
+        collect_ignore.append(file_path)
 
 
 @pytest.fixture()
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 313e74ec9..45cf6fb92 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,6 +1,3 @@
-tests/test_linkextractors_deprecated.py
-tests/test_proxy_connect.py
-
 scrapy/linkextractors/sgml.py
 scrapy/linkextractors/regex.py
 scrapy/linkextractors/htmlparser.py
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 59d4a3eec..b06fcf6c3 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -630,18 +630,16 @@ class Http11MockServerTestCase(unittest.TestCase):
         # download_maxsize < 100, hence the CancelledError
         self.assertIsInstance(failure.value, defer.CancelledError)
 
-        if six.PY2:
-            request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
-            request = request.replace(url=self.mockserver.url('/xpayload'))
-            yield crawler.crawl(seed=request)
-            # download_maxsize = 50 is enough for the gzipped response
-            failure = crawler.spider.meta.get('failure')
-            self.assertTrue(failure is None)
-            reason = crawler.spider.meta['close_reason']
-            self.assertTrue(reason, 'finished')
-        else:
-            # See issue https://twistedmatrix.com/trac/ticket/8175
-            raise unittest.SkipTest("xpayload only enabled for PY2")
+        # See issue https://twistedmatrix.com/trac/ticket/8175
+        raise unittest.SkipTest("xpayload fails on PY3")
+        request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
+        request = request.replace(url=self.mockserver.url('/xpayload'))
+        yield crawler.crawl(seed=request)
+        # download_maxsize = 50 is enough for the gzipped response
+        failure = crawler.spider.meta.get('failure')
+        self.assertTrue(failure is None)
+        reason = crawler.spider.meta['close_reason']
+        self.assertTrue(reason, 'finished')
 
 
 class UriResource(resource.Resource):
diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py
deleted file mode 100644
index 1366971be..000000000
--- a/tests/test_linkextractors_deprecated.py
+++ /dev/null
@@ -1,233 +0,0 @@
-# -*- coding: utf-8 -*-
-import unittest
-from scrapy.linkextractors.regex import RegexLinkExtractor
-from scrapy.http import HtmlResponse
-from scrapy.link import Link
-from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor
-from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
-from tests import get_testdata
-
-from tests.test_linkextractors import Base
-
-
-class BaseSgmlLinkExtractorTestCase(unittest.TestCase):
-    # XXX: should we move some of these tests to base link extractor tests?
-
-    def test_basic(self):
-        html = """<html><head><title>Page title<title>
-        <body><p><a href="item/12.html">Item 12</a></p>
-        <p><a href="/about.html">About us</a></p>
-        <img src="/logo.png" alt="Company logo (not a link)" />
-        <p><a href="../othercat.html">Other category</a></p>
-        <p><a href="/">&gt;&gt;</a></p>
-        <p><a href="/" /></p>
-        </body></html>"""
-        response = HtmlResponse("http://example.org/somepage/index.html", body=html)
-
-        lx = BaseSgmlLinkExtractor()  # default: tag=a, attr=href
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://example.org/somepage/item/12.html', text='Item 12'),
-                          Link(url='http://example.org/about.html', text='About us'),
-                          Link(url='http://example.org/othercat.html', text='Other category'),
-                          Link(url='http://example.org/', text='>>'),
-                          Link(url='http://example.org/', text='')])
-
-    def test_base_url(self):
-        html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
-        <body><p><a href="item/12.html">Item 12</a></p>
-        </body></html>"""
-        response = HtmlResponse("http://example.org/somepage/index.html", body=html)
-
-        lx = BaseSgmlLinkExtractor()  # default: tag=a, attr=href
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
-
-        # base url is an absolute path and relative to host
-        html = """<html><head><title>Page title<title><base href="/" />
-        <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
-        response = HtmlResponse("https://example.org/somepage/index.html", body=html)
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='https://example.org/item/12.html', text='Item 12')])
-
-        # base url has no scheme
-        html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" />
-        <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
-        response = HtmlResponse("https://example.org/somepage/index.html", body=html)
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')])
-
-    def test_link_text_wrong_encoding(self):
-        html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>"""
-        response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8')
-        lx = BaseSgmlLinkExtractor()
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'),
-        ])
-
-    def test_extraction_encoding(self):
-        body = get_testdata('link_extractor', 'linkextractor_noenc.html')
-        response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})
-        response_noenc = HtmlResponse(url='http://example.com/noenc', body=body)
-        body = get_testdata('link_extractor', 'linkextractor_latin1.html')
-        response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body)
-
-        lx = BaseSgmlLinkExtractor()
-        self.assertEqual(lx.extract_links(response_utf8), [
-            Link(url='http://example.com/sample_%C3%B1.html', text=''),
-            Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
-        ])
-
-        self.assertEqual(lx.extract_links(response_noenc), [
-            Link(url='http://example.com/sample_%C3%B1.html', text=''),
-            Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
-        ])
-
-        # document encoding does not affect URL path component, only query part
-        # >>> u'sample_ñ.html'.encode('utf8')
-        # b'sample_\xc3\xb1.html'
-        # >>> u"sample_á.html".encode('utf8')
-        # b'sample_\xc3\xa1.html'
-        # >>> u"sample_ö.html".encode('utf8')
-        # b'sample_\xc3\xb6.html'
-        # >>> u"£32".encode('latin1')
-        # b'\xa332'
-        # >>> u"µ".encode('latin1')
-        # b'\xb5'
-        self.assertEqual(lx.extract_links(response_latin1), [
-            Link(url='http://example.com/sample_%C3%B1.html', text=''),
-            Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')),
-            Link(url='http://example.com/sample_%C3%B6.html?price=%A332&%B5=unit', text=''),
-        ])
-
-    def test_matches(self):
-        url1 = 'http://lotsofstuff.com/stuff1/index'
-        url2 = 'http://evenmorestuff.com/uglystuff/index'
-
-        lx = BaseSgmlLinkExtractor()
-        self.assertEqual(lx.matches(url1), True)
-        self.assertEqual(lx.matches(url2), True)
-
-
-class HtmlParserLinkExtractorTestCase(unittest.TestCase):
-
-    def setUp(self):
-        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
-        self.response = HtmlResponse(url='http://example.com/index', body=body)
-
-    def test_extraction(self):
-        # Default arguments
-        lx = HtmlParserLinkExtractor()
-        self.assertEqual(lx.extract_links(self.response), [
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
-            Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
-            Link(url='http://www.google.com/something', text=u''),
-            Link(url='http://example.com/innertag.html', text=u'inner tag'),
-            Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
-        ])
-
-    def test_link_wrong_href(self):
-        html = """
-        <a href="http://example.org/item1.html">Item 1</a>
-        <a href="http://[example.org/item2.html">Item 2</a>
-        <a href="http://example.org/item3.html">Item 3</a>
-        """
-        response = HtmlResponse("http://example.org/index.html", body=html)
-        lx = HtmlParserLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
-        ])
-
-
-class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
-    extractor_cls = SgmlLinkExtractor
-    escapes_whitespace = True
-
-    def test_deny_extensions(self):
-        html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
-        response = HtmlResponse("http://example.org/", body=html)
-        lx = SgmlLinkExtractor(deny_extensions="jpg")
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.org/page.html', text=u'asd'),
-        ])
-
-    def test_attrs_sgml(self):
-        html = """<html><area href="sample1.html"></area>
-        <a ref="sample2.html">sample text 2</a></html>"""
-        response = HtmlResponse("http://example.com/index.html", body=html)
-        lx = SgmlLinkExtractor(attrs="href")
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.com/sample1.html', text=u''),
-        ])
-
-    def test_link_nofollow(self):
-        html = """
-        <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
-        <a href="about.html">About us</a>
-        <a href="http://google.com/something" rel="external nofollow">Something</a>
-        """
-        response = HtmlResponse("http://example.org/page.html", body=html)
-        lx = SgmlLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
-            Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
-            Link(url='http://google.com/something', text=u'Something', nofollow=True),
-        ])
-
-
-class RegexLinkExtractorTestCase(unittest.TestCase):
-    # XXX: RegexLinkExtractor is not deprecated yet, but it must be rewritten
-    # not to depend on SgmlLinkExractor. Its speed is also much worse
-    # than it should be.
-
-    def setUp(self):
-        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
-        self.response = HtmlResponse(url='http://example.com/index', body=body)
-
-    def test_extraction(self):
-        # Default arguments
-        lx = RegexLinkExtractor()
-        self.assertEqual(lx.extract_links(self.response),
-                         [Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                          Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-                          Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
-                          Link(url='http://www.google.com/something', text=u''),
-                          Link(url='http://example.com/innertag.html', text=u'inner tag'),])
-
-    def test_link_wrong_href(self):
-        html = """
-        <a href="http://example.org/item1.html">Item 1</a>
-        <a href="http://[example.org/item2.html">Item 2</a>
-        <a href="http://example.org/item3.html">Item 3</a>
-        """
-        response = HtmlResponse("http://example.org/index.html", body=html)
-        lx = RegexLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
-        ])
-
-    def test_html_base_href(self):
-        html = """
-        <html>
-            <head>
-                <base href="http://b.com/">
-            </head>
-            <body>
-                <a href="test.html"></a>
-            </body>
-        </html>
-        """
-        response = HtmlResponse("http://a.com/", body=html)
-        lx = RegexLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://b.com/test.html', text=u'', nofollow=False),
-        ])
-
-    @unittest.expectedFailure
-    def test_extraction(self):
-        # RegexLinkExtractor doesn't parse URLs with leading/trailing
-        # whitespaces correctly.
-        super(RegexLinkExtractorTestCase, self).test_extraction()
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
deleted file mode 100644
index ae1236bcb..000000000
--- a/tests/test_proxy_connect.py
+++ /dev/null
@@ -1,120 +0,0 @@
-import json
-import os
-import time
-
-from six.moves.urllib.parse import urlsplit, urlunsplit
-from threading import Thread
-from libmproxy import controller, proxy
-from netlib import http_auth
-from testfixtures import LogCapture
-
-from twisted.internet import defer
-from twisted.trial.unittest import TestCase
-from scrapy.utils.test import get_crawler
-from scrapy.http import Request
-from tests.spiders import SimpleSpider, SingleRequestSpider
-from tests.mockserver import MockServer
-
-
-class HTTPSProxy(controller.Master, Thread):
-
-    def __init__(self):
-        password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy')
-        authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy")
-        cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
-            'keys', 'mitmproxy-ca.pem')
-        server = proxy.ProxyServer(proxy.ProxyConfig(
-            authenticator = authenticator,
-            cacert = cert_path),
-            0)
-        self.server = server
-        Thread.__init__(self)
-        controller.Master.__init__(self, server)
-
-    def http_address(self):
-        return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname()
-
-
-def _wrong_credentials(proxy_url):
-    bad_auth_proxy = list(urlsplit(proxy_url))
-    bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')
-    return urlunsplit(bad_auth_proxy)
-
-class ProxyConnectTestCase(TestCase):
-
-    def setUp(self):
-        self.mockserver = MockServer()
-        self.mockserver.__enter__()
-        self._oldenv = os.environ.copy()
-
-        self._proxy = HTTPSProxy()
-        self._proxy.start()
-
-        # Wait for the proxy to start.
-        time.sleep(1.0)
-        os.environ['https_proxy'] = self._proxy.http_address()
-        os.environ['http_proxy'] = self._proxy.http_address()
-
-    def tearDown(self):
-        self.mockserver.__exit__(None, None, None)
-        self._proxy.shutdown()
-        os.environ = self._oldenv
-
-    @defer.inlineCallbacks
-    def test_https_connect_tunnel(self):
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
-
-    @defer.inlineCallbacks
-    def test_https_noconnect(self):
-        proxy = os.environ['https_proxy']
-        os.environ['https_proxy'] = proxy + '?noconnect'
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
-
-    @defer.inlineCallbacks
-    def test_https_connect_tunnel_error(self):
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl("https://localhost:99999/status?n=200")
-        self._assert_got_tunnel_error(l)
-
-    @defer.inlineCallbacks
-    def test_https_tunnel_auth_error(self):
-        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        # The proxy returns a 407 error code but it does not reach the client;
-        # he just sees a TunnelError.
-        self._assert_got_tunnel_error(l)
-
-    @defer.inlineCallbacks
-    def test_https_tunnel_without_leak_proxy_authorization_header(self):
-        request = Request(self.mockserver.url("/echo", is_secure=True))
-        crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(seed=request)
-        self._assert_got_response_code(200, l)
-        echo = json.loads(crawler.spider.meta['responses'][0].body)
-        self.assertTrue('Proxy-Authorization' not in echo['headers'])
-
-    @defer.inlineCallbacks
-    def test_https_noconnect_auth_error(self):
-        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(407, l)
-
-    def _assert_got_response_code(self, code, log):
-        print(log)
-        self.assertEqual(str(log).count('Crawled (%d)' % code), 1)
-
-    def _assert_got_tunnel_error(self, log):
-        print(log)
-        self.assertIn('TunnelError', str(log))
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index a94398796..6857356f6 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -163,33 +163,6 @@ class UtilsPythonTestCase(unittest.TestCase):
                 gc.collect()
         self.assertFalse(len(wk._weakdict))
 
-    @unittest.skipUnless(six.PY2, "deprecated function")
-    def test_stringify_dict(self):
-        d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'}
-        d2 = stringify_dict(d, keys_only=False)
-        self.assertEqual(d, d2)
-        self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
-
-    @unittest.skipUnless(six.PY2, "deprecated function")
-    def test_stringify_dict_tuples(self):
-        tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')]
-        d = dict(tuples)
-        d2 = stringify_dict(tuples, keys_only=False)
-        self.assertEqual(d, d2)
-        self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
-
-    @unittest.skipUnless(six.PY2, "deprecated function")
-    def test_stringify_dict_keys_only(self):
-        d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
-        d2 = stringify_dict(d)
-        self.assertEqual(d, d2)
-        self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
-
     def test_get_func_args(self):
         def f1(a, b, c):
             pass
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index a81946490..7b015ff8d 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -78,26 +78,6 @@ class ParseUrlTestCase(unittest.TestCase):
                 to_bytes(x) if not isinstance(x, int) else x for x in test)
             self.assertEqual(client._parse(url), test, url)
 
-    def test_externalUnicodeInterference(self):
-        """
-        L{client._parse} should return C{str} for the scheme, host, and path
-        elements of its return tuple, even when passed an URL which has
-        previously been passed to L{urlparse} as a C{unicode} string.
-        """
-        if not six.PY2:
-            raise unittest.SkipTest(
-                "Applies only to Py2, as urls can be ONLY unicode on Py3")
-        badInput = u'http://example.com/path'
-        goodInput = badInput.encode('ascii')
-        self._parse(badInput)  # cache badInput in urlparse_cached
-        scheme, netloc, host, port, path = self._parse(goodInput)
-        self.assertTrue(isinstance(scheme, str))
-        self.assertTrue(isinstance(netloc, str))
-        self.assertTrue(isinstance(host, str))
-        self.assertTrue(isinstance(path, str))
-        self.assertTrue(isinstance(port, int))
-
-
 
 class ScrapyHTTPPageGetterTests(unittest.TestCase):
 

From f066257e95f1d025d1e581350902067c671fd800 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 3 Sep 2019 15:17:03 +0500
Subject: [PATCH 2407/2974] Restore tests/test_proxy_connect.py and update it
 to modern mitmproxy.

---
 tests/requirements-py3.txt  |   1 +
 tests/test_proxy_connect.py | 134 ++++++++++++++++++++++++++++++++++++
 2 files changed, 135 insertions(+)
 create mode 100644 tests/test_proxy_connect.py

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 2e8d319d2..8169febea 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,5 +1,6 @@
 # Tests requirements
 jmespath
+mitmproxy
 pytest
 pytest-cov
 pytest-twisted
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
new file mode 100644
index 000000000..8142d9a41
--- /dev/null
+++ b/tests/test_proxy_connect.py
@@ -0,0 +1,134 @@
+import json
+import os
+import re
+from subprocess import Popen, PIPE
+import sys
+import time
+
+from six.moves.urllib.parse import urlsplit, urlunsplit
+from testfixtures import LogCapture
+
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+
+from scrapy.utils.test import get_crawler
+from scrapy.http import Request
+from tests.spiders import SimpleSpider, SingleRequestSpider
+from tests.mockserver import MockServer
+
+
+class MitmProxy:
+    auth_user = 'scrapy'
+    auth_pass = 'scrapy'
+
+    def start(self):
+        from scrapy.utils.test import get_testenv
+        script = """
+import sys
+from mitmproxy.tools.main import mitmdump
+sys.argv[0] = "mitmdump"
+sys.exit(mitmdump())
+        """
+        cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+            'keys', 'mitmproxy-ca.pem')
+        self.proc = Popen([sys.executable,
+                           '-c', script,
+                           '--listen-host', '127.0.0.1',
+                           '--listen-port', '0',
+                           '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
+                           '--certs', cert_path,
+                           '--ssl-insecure',
+                           ],
+                           stdout=PIPE, env=get_testenv())
+        line = self.proc.stdout.readline().decode('utf-8')
+        host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)
+        address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port)
+        return address
+
+    def stop(self):
+        self.proc.kill()
+        self.proc.wait()
+        time.sleep(0.2)
+
+
+def _wrong_credentials(proxy_url):
+    bad_auth_proxy = list(urlsplit(proxy_url))
+    bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')
+    return urlunsplit(bad_auth_proxy)
+
+
+class ProxyConnectTestCase(TestCase):
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self._oldenv = os.environ.copy()
+
+        self._proxy = MitmProxy()
+        proxy_url = self._proxy.start()
+        os.environ['https_proxy'] = proxy_url
+        os.environ['http_proxy'] = proxy_url
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+        self._proxy.stop()
+        os.environ = self._oldenv
+
+    @defer.inlineCallbacks
+    def test_https_connect_tunnel(self):
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        self._assert_got_response_code(200, l)
+
+    @defer.inlineCallbacks
+    def test_https_noconnect(self):
+        proxy = os.environ['https_proxy']
+        os.environ['https_proxy'] = proxy + '?noconnect'
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        self._assert_got_response_code(200, l)
+
+    @defer.inlineCallbacks
+    def test_https_connect_tunnel_error(self):
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl("https://localhost:99999/status?n=200")
+        self._assert_got_tunnel_error(l)
+
+    @defer.inlineCallbacks
+    def test_https_tunnel_auth_error(self):
+        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        # The proxy returns a 407 error code but it does not reach the client;
+        # he just sees a TunnelError.
+        self._assert_got_tunnel_error(l)
+
+    @defer.inlineCallbacks
+    def test_https_tunnel_without_leak_proxy_authorization_header(self):
+        request = Request(self.mockserver.url("/echo", is_secure=True))
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(seed=request)
+        self._assert_got_response_code(200, l)
+        echo = json.loads(crawler.spider.meta['responses'][0].body)
+        self.assertTrue('Proxy-Authorization' not in echo['headers'])
+
+    @defer.inlineCallbacks
+    def test_https_noconnect_auth_error(self):
+        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        self._assert_got_response_code(407, l)
+
+    def _assert_got_response_code(self, code, log):
+        print(log)
+        self.assertEqual(str(log).count('Crawled (%d)' % code), 1)
+
+    def _assert_got_tunnel_error(self, log):
+        print(log)
+        self.assertIn('TunnelError', str(log))

From cbb6d0c6a71c709903048597cbbdb02616aed285 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 3 Sep 2019 15:23:24 +0500
Subject: [PATCH 2408/2974] Mark failing proxy tests.

---
 tests/test_proxy_connect.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 8142d9a41..5e9470e39 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -5,6 +5,7 @@ from subprocess import Popen, PIPE
 import sys
 import time
 
+import pytest
 from six.moves.urllib.parse import urlsplit, urlunsplit
 from testfixtures import LogCapture
 
@@ -81,6 +82,7 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, l)
 
+    @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
     @defer.inlineCallbacks
     def test_https_noconnect(self):
         proxy = os.environ['https_proxy']
@@ -90,6 +92,7 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, l)
 
+    @pytest.mark.xfail(reason='Python 3 fails this earlier')
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
         crawler = get_crawler(SimpleSpider)
@@ -117,6 +120,7 @@ class ProxyConnectTestCase(TestCase):
         echo = json.loads(crawler.spider.meta['responses'][0].body)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
+    @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
     @defer.inlineCallbacks
     def test_https_noconnect_auth_error(self):
         os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'

From c327ad9ba6baee934b35ab9cb832e2fbb3311e13 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 15:20:28 +0500
Subject: [PATCH 2409/2974] Remove an unused six import.

---
 tests/test_downloader_handlers.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index b06fcf6c3..a78762a64 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1,5 +1,4 @@
 import os
-import six
 import shutil
 import tempfile
 from unittest import mock

From 3ec6960732591b38a21bc0af9ae042007bb849f4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 23:21:14 +0500
Subject: [PATCH 2410/2974] Fix test_proxy_connect.py for py3.5.

---
 tests/test_proxy_connect.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 5e9470e39..f6381b5b1 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -117,7 +117,7 @@ class ProxyConnectTestCase(TestCase):
         with LogCapture() as l:
             yield crawler.crawl(seed=request)
         self._assert_got_response_code(200, l)
-        echo = json.loads(crawler.spider.meta['responses'][0].body)
+        echo = json.loads(crawler.spider.meta['responses'][0].body.decode('utf-8'))
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
     @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')

From c4ef950efda32d292840027e72aa5bb7844423bb Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 23:21:30 +0500
Subject: [PATCH 2411/2974] Use an older mitmproxy for py3.5.

---
 tests/requirements-py3.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 8169febea..c4bc1f278 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,6 +1,7 @@
 # Tests requirements
 jmespath
-mitmproxy
+mitmproxy; python_version >= '3.6'
+mitmproxy==3.0.4; python_version < '3.6'
 pytest
 pytest-cov
 pytest-twisted

From 5080180c759e74df9e20f7770534017272a9fea3 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:46:19 +0500
Subject: [PATCH 2412/2974] Improve the
 test_https_tunnel_without_leak_proxy_authorization_header change.

---
 tests/test_proxy_connect.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index f6381b5b1..651576c2c 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -117,7 +117,7 @@ class ProxyConnectTestCase(TestCase):
         with LogCapture() as l:
             yield crawler.crawl(seed=request)
         self._assert_got_response_code(200, l)
-        echo = json.loads(crawler.spider.meta['responses'][0].body.decode('utf-8'))
+        echo = json.loads(crawler.spider.meta['responses'][0].text)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
     @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')

From 5970d00eb9ade62523b0f61788cab50fe8f62eb1 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:46:38 +0500
Subject: [PATCH 2413/2974] Only xfail test_https_connect_tunnel_error on 3.6+.

---
 tests/test_proxy_connect.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 651576c2c..ec3f0716c 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -92,7 +92,7 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, l)
 
-    @pytest.mark.xfail(reason='Python 3 fails this earlier')
+    @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
         crawler = get_crawler(SimpleSpider)

From 8b730a36706f9b75e1d1b168901dad9805f4da78 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:50:56 +0500
Subject: [PATCH 2414/2974] Use self.proc.communicate() after killing mitmdump.

---
 tests/test_proxy_connect.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index ec3f0716c..69925f80c 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -48,8 +48,7 @@ sys.exit(mitmdump())
 
     def stop(self):
         self.proc.kill()
-        self.proc.wait()
-        time.sleep(0.2)
+        self.proc.communicate()
 
 
 def _wrong_credentials(proxy_url):

From a7b640991d527c265d581cba1b1d1119db99e4c8 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:52:57 +0500
Subject: [PATCH 2415/2974] Rename tests/py3-ignores.txt to tests/ignores.txt.

---
 conftest.py                            | 2 +-
 tests/{py3-ignores.txt => ignores.txt} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename tests/{py3-ignores.txt => ignores.txt} (100%)

diff --git a/conftest.py b/conftest.py
index 74fb101e9..d54ce155c 100644
--- a/conftest.py
+++ b/conftest.py
@@ -7,7 +7,7 @@ collect_ignore = [
 ]
 
 
-for line in open('tests/py3-ignores.txt'):
+for line in open('tests/ignores.txt'):
     file_path = line.strip()
     if file_path and file_path[0] != '#':
         collect_ignore.append(file_path)
diff --git a/tests/py3-ignores.txt b/tests/ignores.txt
similarity index 100%
rename from tests/py3-ignores.txt
rename to tests/ignores.txt

From 922a66cf07e68a9a42d59530ce9dc2dd29a093a8 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 14 Nov 2019 22:36:58 +0500
Subject: [PATCH 2416/2974] Fix or ignore flake8 problems.

---
 pytest.ini                  | 1 +
 tests/test_proxy_connect.py | 5 ++---
 tests/test_utils_python.py  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 529ad5d27..4f2548db2 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -226,6 +226,7 @@ flake8-ignore =
     tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226
     tests/test_pipeline_images.py F401 F841 E501 E303
     tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
+    tests/test_proxy_connect.py E501 E741
     tests/test_request_cb_kwargs.py E501
     tests/test_responsetypes.py E501 E302 E305
     tests/test_robotstxt_interface.py F401 E302 E501 W291 E501
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 69925f80c..2435999f9 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -3,7 +3,6 @@ import os
 import re
 from subprocess import Popen, PIPE
 import sys
-import time
 
 import pytest
 from six.moves.urllib.parse import urlsplit, urlunsplit
@@ -31,7 +30,7 @@ sys.argv[0] = "mitmdump"
 sys.exit(mitmdump())
         """
         cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
-            'keys', 'mitmproxy-ca.pem')
+                                 'keys', 'mitmproxy-ca.pem')
         self.proc = Popen([sys.executable,
                            '-c', script,
                            '--listen-host', '127.0.0.1',
@@ -40,7 +39,7 @@ sys.exit(mitmdump())
                            '--certs', cert_path,
                            '--ssl-insecure',
                            ],
-                           stdout=PIPE, env=get_testenv())
+                          stdout=PIPE, env=get_testenv())
         line = self.proc.stdout.readline().decode('utf-8')
         host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)
         address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port)
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 6857356f6..326a67c2e 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -8,7 +8,7 @@ import six
 
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
-    WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode,
+    WeakKeyCache, get_func_args, to_bytes, to_unicode,
     without_none_values, MutableChain)
 
 __doctests__ = ['scrapy.utils.python']

From beb7d80d6a8a82baf9cb8170758704b4ff5c63cf Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 14 Nov 2019 22:47:35 +0500
Subject: [PATCH 2417/2974] Add a comment about the noconnect tests.

---
 tests/test_proxy_connect.py | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 2435999f9..277455751 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -80,16 +80,6 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, l)
 
-    @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
-    @defer.inlineCallbacks
-    def test_https_noconnect(self):
-        proxy = os.environ['https_proxy']
-        os.environ['https_proxy'] = proxy + '?noconnect'
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
-
     @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
@@ -118,6 +108,23 @@ class ProxyConnectTestCase(TestCase):
         echo = json.loads(crawler.spider.meta['responses'][0].text)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
+    # The noconnect mode isn't supported by the current mitmproxy, it returns
+    # "Invalid request scheme: https" as it doesn't seem to support full URLs in GET at all,
+    # and it's not clear what behavior is intended by Scrapy and by mitmproxy here.
+    # https://github.com/mitmproxy/mitmproxy/issues/848 may be related.
+    # The Scrapy noconnect mode was required, at least in the past, to work with Crawlera,
+    # and https://github.com/scrapy-plugins/scrapy-crawlera/pull/44 seems to be related.
+
+    @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
+    @defer.inlineCallbacks
+    def test_https_noconnect(self):
+        proxy = os.environ['https_proxy']
+        os.environ['https_proxy'] = proxy + '?noconnect'
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        self._assert_got_response_code(200, l)
+
     @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
     @defer.inlineCallbacks
     def test_https_noconnect_auth_error(self):

From 78ad01632f16a97fb180d8c2972075b19f471380 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Tue, 19 Nov 2019 14:43:30 +0500
Subject: [PATCH 2418/2974] Fix flake8 problems in PR #3989 (#4176)

---
 tests/test_logformatter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 5b5d68f4f..afbd25d0c 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -62,7 +62,7 @@ class LogFormatterTestCase(unittest.TestCase):
         lines = logline.splitlines()
         assert all(isinstance(x, six.text_type) for x in lines)
         self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
-    
+
     def test_error(self):
         # In practice, the complete traceback is shown by passing the
         # 'exc_info' argument to the logging function
@@ -121,7 +121,7 @@ class LogformatterSubclassTest(LogFormatterTestCase):
             "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
 
     def test_flags_in_request(self):
-        req = Request("http://www.example.com", flags=['test','flag'])
+        req = Request("http://www.example.com", flags=['test', 'flag'])
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
         logline = logkws['msg'] % logkws['args']

From fc3af54dbd5b7fdcbc44c82ba4ced528a104cd88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 20 Nov 2019 07:57:09 +0100
Subject: [PATCH 2419/2974] Make tox configuration more user friendly

---
 .travis.yml           |  5 ++--
 docs/contributing.rst | 14 ++--------
 tox.ini               | 64 ++++++++++++++++++-------------------------
 3 files changed, 31 insertions(+), 52 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 9f477e860..4e28d6f11 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,10 +12,9 @@ matrix:
     - env: TOXENV=flake8
       python: 3.8
     - env: TOXENV=pypy3
-      python: 3.5
     - env: TOXENV=py35
       python: 3.5
-    - env: TOXENV=py35-pinned
+    - env: TOXENV=pinned
       python: 3.5
     - env: TOXENV=py36
       python: 3.6
@@ -23,7 +22,7 @@ matrix:
       python: 3.7
     - env: TOXENV=py38
       python: 3.8
-    - env: TOXENV=py38-extra-deps
+    - env: TOXENV=extra-deps
       python: 3.8
     - env: TOXENV=docs
       python: 3.6
diff --git a/docs/contributing.rst b/docs/contributing.rst
index f084bd23d..c4cb605ab 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -202,17 +202,9 @@ tests requires `tox`_.
 Running tests
 -------------
 
-Make sure you have a recent enough `tox`_ installation:
+To run all tests::
 
-    ``tox --version``
-
-If your version is older than 1.7.0, please update it first:
-
-    ``pip install -U tox``
-
-To run all tests go to the root directory of Scrapy source code and run:
-
-    ``tox``
+    tox
 
 To run a specific test (say ``tests/test_loader.py``) use:
 
@@ -227,7 +219,7 @@ environment name from ``tox.ini``. For example, to run the tests with Python
 You can also specify a comma-separated list of environmets, and use `tox’s
 parallel mode`_ to run the tests on multiple environments in parallel::
 
-    tox -e py27,py36 -p auto
+    tox -e py37,py38 -p auto
 
 To pass command-line options to pytest_, add them after ``--`` in your call to
 tox_. Using ``--`` overrides the default positional arguments defined in
diff --git a/tox.ini b/tox.ini
index fd75d18e2..0b41f6cc3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,7 +4,8 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py35
+envlist = security,flake8,py3
+minversion = 1.7.0
 
 [testenv]
 deps =
@@ -23,11 +24,28 @@ passenv =
 commands =
     py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
 
-[testenv:py35]
-basepython = python3.5
+[testenv:security]
+basepython = python3
+deps =
+    bandit
+commands =
+    bandit -r -c .bandit.yml {posargs:scrapy}
 
-[testenv:py35-pinned]
-basepython = python3.5
+[testenv:flake8]
+basepython = python3
+deps =
+    {[testenv]deps}
+    pytest-flake8
+commands =
+    py.test --flake8 {posargs:docs scrapy tests}
+
+[testenv:pypy3]
+basepython = pypy3
+commands =
+    py.test {posargs:--durations=10 docs scrapy tests}
+
+[testenv:pinned]
+basepython = python3
 deps =
     -ctests/constraints.txt
     cryptography==2.0
@@ -48,34 +66,11 @@ deps =
     botocore==1.3.23
     Pillow==3.4.2
 
-[testenv:py36]
-basepython = python3.6
-
-[testenv:py37]
-basepython = python3.7
-
-[testenv:py38]
-basepython = python3.8
-
-[testenv:pypy3]
-basepython = pypy3
-commands =
-    py.test {posargs:--durations=10 docs scrapy tests}
-
-[testenv:security]
-basepython = python3.8
-deps =
-    bandit
-commands =
-    bandit -r -c .bandit.yml {posargs:scrapy}
-
-[testenv:flake8]
-basepython = python3.8
+[testenv:extra-deps]
 deps =
     {[testenv]deps}
-    pytest-flake8
-commands =
-    py.test --flake8 {posargs:docs scrapy tests}
+    reppy
+    robotexclusionrulesparser
 
 [docs]
 changedir = docs
@@ -99,10 +94,3 @@ changedir = {[docs]changedir}
 deps = {[docs]deps}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
-
-[testenv:py38-extra-deps]
-basepython = python3.8
-deps =
-    {[testenv]deps}
-    reppy
-    robotexclusionrulesparser

From 42954d0df9a78e6641996fbf2eb76afdef9c1856 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 20 Nov 2019 08:16:33 +0100
Subject: [PATCH 2420/2974] Mention that ScrapyHTTPClientFactory has Twisted
 code

---
 scrapy/core/downloader/webclient.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 16fd214a3..26726afd4 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -93,6 +93,10 @@ class ScrapyHTTPPageGetter(HTTPClient):
                 (self.factory.url, self.factory.timeout)))
 
 
+# This class used to inherit from Twisted’s
+# twisted.web.client.HTTPClientFactory. When that class was deprecated in
+# Twisted (https://github.com/twisted/twisted/pull/643), we merged its
+# non-overriden code into this class.
 class ScrapyHTTPClientFactory(ClientFactory):
 
     protocol = ScrapyHTTPPageGetter

From e6c5292a7c4391642897ee31ea2ded889b3b88dc Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 20 Nov 2019 09:29:55 -0300
Subject: [PATCH 2421/2974] Response.follow_all: Specific exception for invalid
 selectors

---
 scrapy/http/response/text.py | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 74017b5aa..5110b4bd4 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -5,17 +5,18 @@ discovering (through HTTP headers) to base Response class.
 See documentation in docs/topics/request-response.rst
 """
 
-import six
-from six.moves.urllib.parse import urljoin
+from contextlib import suppress
 
 import parsel
-from w3lib.encoding import html_to_unicode, resolve_encoding, \
-    html_body_declared_encoding, http_content_type_encoding
+import six
+from six.moves.urllib.parse import urljoin
+from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
+                            http_content_type_encoding, resolve_encoding)
 from w3lib.html import strip_html5_whitespace
 
 from scrapy.http.response import Response
-from scrapy.utils.response import get_base_url
 from scrapy.utils.python import memoizemethod_noargs, to_native_str
+from scrapy.utils.response import get_base_url
 
 
 class TextResponse(Response):
@@ -197,10 +198,8 @@ class TextResponse(Response):
                 selector_list = self.xpath(xpath)
             urls = []
             for selector in selector_list:
-                try:
+                with suppress(_InvalidSelector):
                     urls.append(_url_from_selector(selector))
-                except ValueError:
-                    pass
         return super(TextResponse, self).follow_all(
             urls=urls,
             callback=callback,
@@ -217,18 +216,24 @@ class TextResponse(Response):
         )
 
 
+class _InvalidSelector(ValueError):
+    """
+    Raised when a URL cannot be obtained from a Selector
+    """
+
+
 def _url_from_selector(sel):
     # type: (parsel.Selector) -> str
     if isinstance(sel.root, six.string_types):
         # e.g. ::attr(href) result
         return strip_html5_whitespace(sel.root)
     if not hasattr(sel.root, 'tag'):
-        raise ValueError("Unsupported selector: %s" % sel)
+        raise _InvalidSelector("Unsupported selector: %s" % sel)
     if sel.root.tag not in ('a', 'link'):
-        raise ValueError("Only <a> and <link> elements are supported; got <%s>" %
-                         sel.root.tag)
+        raise _InvalidSelector("Only <a> and <link> elements are supported; got <%s>" %
+                               sel.root.tag)
     href = sel.root.get('href')
     if href is None:
-        raise ValueError("<%s> element has no href attribute: %s" %
-                         (sel.root.tag, sel))
+        raise _InvalidSelector("<%s> element has no href attribute: %s" %
+                               (sel.root.tag, sel))
     return strip_html5_whitespace(href)

From b602c61e1cc00e04ce435078644d5ea0c0249903 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 20 Nov 2019 09:38:54 -0300
Subject: [PATCH 2422/2974] [Test] Rename outdated sample files

---
 .../{sgml_linkextractor.html => linkextractor.html}           | 0
 ..._linkextractor_no_href.html => linkextractor_no_href.html} | 0
 tests/test_http_response.py                                   | 4 ++--
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename tests/sample_data/link_extractor/{sgml_linkextractor.html => linkextractor.html} (100%)
 rename tests/sample_data/link_extractor/{sgml_linkextractor_no_href.html => linkextractor_no_href.html} (100%)

diff --git a/tests/sample_data/link_extractor/sgml_linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html
similarity index 100%
rename from tests/sample_data/link_extractor/sgml_linkextractor.html
rename to tests/sample_data/link_extractor/linkextractor.html
diff --git a/tests/sample_data/link_extractor/sgml_linkextractor_no_href.html b/tests/sample_data/link_extractor/linkextractor_no_href.html
similarity index 100%
rename from tests/sample_data/link_extractor/sgml_linkextractor_no_href.html
rename to tests/sample_data/link_extractor/linkextractor_no_href.html
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 6d3c5cb9d..0ae1612b5 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -250,12 +250,12 @@ class BaseResponseTest(unittest.TestCase):
             yield req
 
     def _links_response(self):
-        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
+        body = get_testdata('link_extractor', 'linkextractor.html')
         resp = self.response_class('http://example.com/index', body=body)
         return resp
 
     def _links_response_no_href(self):
-        body = get_testdata('link_extractor', 'sgml_linkextractor_no_href.html')
+        body = get_testdata('link_extractor', 'linkextractor_no_href.html')
         resp = self.response_class('http://example.com/index', body=body)
         return resp
 

From 6f4e84ecf95feabe15fda0819bc428f8e0d4d340 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 20 Nov 2019 09:55:15 -0300
Subject: [PATCH 2423/2974] PEP8 adjustments for scrapy.http.response module

---
 scrapy/http/response/__init__.py | 6 ++++--
 scrapy/http/response/html.py     | 1 +
 scrapy/http/response/text.py     | 2 ++
 scrapy/http/response/xml.py      | 1 +
 4 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 79a8d0ca0..b9e638551 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -4,6 +4,8 @@ responses in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
+from typing import Generator
+
 from six.moves.urllib.parse import urljoin
 
 from scrapy.http.request import Request
@@ -41,8 +43,8 @@ class Response(object_ref):
         if isinstance(url, str):
             self._url = url
         else:
-            raise TypeError('%s url must be str, got %s:' % (type(self).__name__,
-                type(url).__name__))
+            raise TypeError('%s url must be str, got %s:' %
+                            (type(self).__name__, type(url).__name__))
 
     url = property(_get_url, obsolete_setter(_set_url, 'url'))
 
diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py
index bd3559fbb..7eed052c2 100644
--- a/scrapy/http/response/html.py
+++ b/scrapy/http/response/html.py
@@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst
 
 from scrapy.http.response.text import TextResponse
 
+
 class HtmlResponse(TextResponse):
     pass
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index b97420345..6acf1026f 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/request-response.rst
 """
 
 from contextlib import suppress
+from typing import Generator
 
 import parsel
 import six
@@ -14,6 +15,7 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
                             http_content_type_encoding, resolve_encoding)
 from w3lib.html import strip_html5_whitespace
 
+from scrapy.http import Request
 from scrapy.http.response import Response
 from scrapy.utils.python import memoizemethod_noargs, to_unicode
 from scrapy.utils.response import get_base_url
diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py
index 1df33fee5..abf474a2f 100644
--- a/scrapy/http/response/xml.py
+++ b/scrapy/http/response/xml.py
@@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst
 
 from scrapy.http.response.text import TextResponse
 
+
 class XmlResponse(TextResponse):
     pass

From 6781d2f5b261305a4f6aa41a2b485a2e5cc7bc76 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 20 Nov 2019 09:58:25 -0300
Subject: [PATCH 2424/2974] Update sample file references

---
 tests/test_linkextractors.py            | 2 +-
 tests/test_linkextractors_deprecated.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 57ef1694a..0b94f937f 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -16,7 +16,7 @@ class Base:
         escapes_whitespace = False
 
         def setUp(self):
-            body = get_testdata('link_extractor', 'sgml_linkextractor.html')
+            body = get_testdata('link_extractor', 'linkextractor.html')
             self.response = HtmlResponse(url='http://example.com/index', body=body)
 
         def test_urls_type(self):
diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py
index 1366971be..388ed6ad4 100644
--- a/tests/test_linkextractors_deprecated.py
+++ b/tests/test_linkextractors_deprecated.py
@@ -111,7 +111,7 @@ class BaseSgmlLinkExtractorTestCase(unittest.TestCase):
 class HtmlParserLinkExtractorTestCase(unittest.TestCase):
 
     def setUp(self):
-        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
+        body = get_testdata('link_extractor', 'linkextractor.html')
         self.response = HtmlResponse(url='http://example.com/index', body=body)
 
     def test_extraction(self):
@@ -183,7 +183,7 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
     # than it should be.
 
     def setUp(self):
-        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
+        body = get_testdata('link_extractor', 'linkextractor.html')
         self.response = HtmlResponse(url='http://example.com/index', body=body)
 
     def test_extraction(self):

From 4f80eff1e159ceb2e5f9375f6a95f69f75f6e1c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 21 Nov 2019 10:30:21 +0100
Subject: [PATCH 2425/2974] Enable sphinx-hoverxref for all references

---
 docs/conf.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/conf.py b/docs/conf.py
index 0e0df274c..04472cf86 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -279,3 +279,9 @@ intersphinx_mapping = {
     'sphinx': ('https://www.sphinx-doc.org/en/master', None),
     'twisted': ('https://twistedmatrix.com/documents/current', None),
 }
+
+
+# Options for sphinx-hoverxref options
+# ------------------------------------
+
+hoverxref_auto_ref = True

From f251dda2687ad0cd21c126580c4ddfff59444d93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 21 Nov 2019 11:59:10 +0100
Subject: [PATCH 2426/2974] Make debug doctests pass

---
 docs/topics/debug.rst | 8 ++++++++
 pytest.ini            | 1 -
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst
index 0aaad0c77..4b2588518 100644
--- a/docs/topics/debug.rst
+++ b/docs/topics/debug.rst
@@ -48,6 +48,10 @@ The most basic way of checking the output of your spider is to use the
 of the spider at the method level. It has the advantage of being flexible and
 simple to use, but does not allow debugging code inside a method.
 
+.. highlight:: none
+
+.. skip: start
+
 In order to see the item scraped from a specific url::
 
     $ scrapy parse --spider=myspider -c parse_item -d 2 <item_url>
@@ -85,6 +89,8 @@ using::
 
     $ scrapy parse --spider=myspider -d 3 'http://example.com/page1'
 
+.. skip: end
+
 
 Scrapy Shell
 ============
@@ -94,6 +100,8 @@ spider, it is of little help to check what happens inside a callback, besides
 showing the response received and the output. How to debug the situation when
 ``parse_details`` sometimes receives no item?
 
+.. highlight:: python
+
 Fortunately, the :command:`shell` is your bread and butter in this case (see
 :ref:`topics-shell-inspect-response`)::
 
diff --git a/pytest.ini b/pytest.ini
index 7be5d8572..9d99dcd22 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -8,7 +8,6 @@ addopts =
     --ignore=docs/_ext
     --ignore=docs/conf.py
     --ignore=docs/news.rst
-    --ignore=docs/topics/debug.rst
     --ignore=docs/topics/developer-tools.rst
     --ignore=docs/topics/dynamic-content.rst
     --ignore=docs/topics/items.rst

From fcfcabf1bdf6fc48105c6f58a4c4eb59e26a8f6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 21 Nov 2019 12:15:13 +0100
Subject: [PATCH 2427/2974] Use InterSphinx for links to the pytest and tox
 documentation

---
 docs/conf.py          |  2 ++
 docs/contributing.rst | 30 ++++++++++++++----------------
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 0e0df274c..e37df7e47 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -275,7 +275,9 @@ coverage_ignore_pyobjects = [
 # -------------------------------------
 
 intersphinx_mapping = {
+    'pytest': ('https://docs.pytest.org/en/latest', None),
     'python': ('https://docs.python.org/3', None),
     'sphinx': ('https://www.sphinx-doc.org/en/master', None),
+    'tox': ('https://tox.readthedocs.io/en/latest', None),
     'twisted': ('https://twistedmatrix.com/documents/current', None),
 }
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 68ae2bf3c..81bb50a77 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -196,14 +196,14 @@ Tests
 
 Tests are implemented using the :doc:`Twisted unit-testing framework
 <twisted:core/development/policy/test-standard>`. Running tests requires
-`tox`_.
+:doc:`tox <tox:index>`.
 
 .. _running-tests:
 
 Running tests
 -------------
 
-Make sure you have a recent enough `tox`_ installation:
+Make sure you have a recent enough :doc:`tox <tox:index>` installation:
 
     ``tox --version``
 
@@ -219,26 +219,27 @@ To run a specific test (say ``tests/test_loader.py``) use:
 
     ``tox -- tests/test_loader.py``
 
-To run the tests on a specific tox_ environment, use ``-e <name>`` with an
-environment name from ``tox.ini``. For example, to run the tests with Python
-3.6 use::
+To run the tests on a specific :doc:`tox <tox:index>` environment, use
+``-e <name>`` with an environment name from ``tox.ini``. For example, to run
+the tests with Python 3.6 use::
 
     tox -e py36
 
-You can also specify a comma-separated list of environmets, and use `tox’s
-parallel mode`_ to run the tests on multiple environments in parallel::
+You can also specify a comma-separated list of environmets, and use :ref:`tox’s
+parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
+parallel::
 
     tox -e py27,py36 -p auto
 
-To pass command-line options to pytest_, add them after ``--`` in your call to
-tox_. Using ``--`` overrides the default positional arguments defined in
-``tox.ini``, so you must include those default positional arguments
-(``scrapy tests``) after ``--`` as well::
+To pass command-line options to :doc:`pytest <pytest:index>`, add them after
+``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
+default positional arguments defined in ``tox.ini``, so you must include those
+default positional arguments (``scrapy tests``) after ``--`` as well::
 
     tox -- scrapy tests -x  # stop after first failure
 
 You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
-the Python 3.6 tox_ environment using all your CPU cores::
+the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
 
     tox -e py36 -- scrapy tests -n auto
 
@@ -275,7 +276,4 @@ And their unit-tests are in::
 .. _open issues: https://github.com/scrapy/scrapy/issues
 .. _PEP 257: https://www.python.org/dev/peps/pep-0257/
 .. _pull request: https://help.github.com/en/articles/creating-a-pull-request
-.. _pytest: https://docs.pytest.org/en/latest/usage.html
-.. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html
-.. _tox: https://pypi.python.org/pypi/tox
-.. _tox’s parallel mode: https://tox.readthedocs.io/en/latest/example/basic.html#parallel-mode
+.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist

From a2bf340bab796704ba6846f8ed755d3ffe37bb0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 21 Nov 2019 14:18:49 +0100
Subject: [PATCH 2428/2974] Remove unused imports

---
 pytest.ini                                    | 84 +++++++++----------
 scrapy/commands/check.py                      |  2 -
 scrapy/commands/view.py                       |  2 +-
 scrapy/core/downloader/__init__.py            |  2 -
 scrapy/core/downloader/handlers/http.py       |  1 -
 scrapy/core/downloader/handlers/s3.py         |  2 +-
 scrapy/extensions/httpcache.py                |  4 +-
 scrapy/linkextractors/__init__.py             |  2 +-
 scrapy/selector/__init__.py                   |  2 +-
 scrapy/selector/unified.py                    |  2 -
 scrapy/spiders/__init__.py                    |  7 +-
 scrapy/utils/boto.py                          |  2 +-
 scrapy/utils/console.py                       |  2 +-
 scrapy/utils/curl.py                          |  2 +-
 scrapy/utils/engine.py                        |  4 +-
 scrapy/utils/http.py                          |  2 +-
 scrapy/utils/markup.py                        |  2 +-
 scrapy/utils/multipart.py                     |  2 +-
 scrapy/utils/url.py                           |  1 -
 tests/mockserver.py                           |  8 +-
 tests/test_downloader_handlers.py             | 10 ++-
 tests/test_downloadermiddleware_httpcache.py  |  3 -
 ...st_downloadermiddleware_httpcompression.py |  2 +-
 tests/test_downloadermiddleware_httpproxy.py  |  5 +-
 tests/test_extension_telnet.py                |  2 +-
 tests/test_feedexport.py                      |  6 +-
 tests/test_http_request.py                    |  4 +-
 tests/test_pipeline_images.py                 |  1 -
 tests/test_robotstxt_interface.py             |  6 +-
 tests/test_selector.py                        |  4 +-
 tests/test_spider.py                          |  1 -
 tests/test_spidermiddleware_output_chain.py   |  1 -
 tests/test_spidermiddleware_referer.py        |  1 -
 tests/test_utils_reqser.py                    |  4 -
 tests/test_utils_url.py                       |  6 +-
 tox.ini                                       |  1 +
 36 files changed, 84 insertions(+), 108 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 7be5d8572..0b79860f7 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -21,12 +21,17 @@ addopts =
     --ignore=docs/utils
 twisted = 1
 flake8-ignore =
+    # Files that are only meant to provide top-level imports are expected not
+    # to use any of their imports:
+    scrapy/core/downloader/handlers/http.py F401
+    scrapy/http/__init__.py F401
+    # Issues pending a review:
     # extras
     extras/qps-bench-server.py E261 E501
     extras/qpsclient.py E501 E261 E501
     # scrapy/commands
     scrapy/commands/__init__.py E128 E501
-    scrapy/commands/check.py F401 E501
+    scrapy/commands/check.py E501
     scrapy/commands/crawl.py E501
     scrapy/commands/edit.py E501
     scrapy/commands/fetch.py E401 E501 E128 E502 E731
@@ -37,7 +42,6 @@ flake8-ignore =
     scrapy/commands/shell.py E128 E501 E502
     scrapy/commands/startproject.py E502 E127 E501 E128
     scrapy/commands/version.py E501 E128
-    scrapy/commands/view.py F401
     # scrapy/contracts
     scrapy/contracts/__init__.py E501 W504
     scrapy/contracts/default.py E502 E128
@@ -46,17 +50,16 @@ flake8-ignore =
     scrapy/core/scheduler.py E501
     scrapy/core/scraper.py E501 E306 E261 E128 W504
     scrapy/core/spidermw.py E501 E731 E502 E126 E226
-    scrapy/core/downloader/__init__.py F401 E501
+    scrapy/core/downloader/__init__.py E501
     scrapy/core/downloader/contextfactory.py E501 E128 E126
     scrapy/core/downloader/middleware.py E501 E502
     scrapy/core/downloader/tls.py E501 E305 E241
     scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226
     scrapy/core/downloader/handlers/__init__.py E501
     scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
-    scrapy/core/downloader/handlers/http.py F401
     scrapy/core/downloader/handlers/http10.py E501
     scrapy/core/downloader/handlers/http11.py E501
-    scrapy/core/downloader/handlers/s3.py E501 F401 E502 E128 E126
+    scrapy/core/downloader/handlers/s3.py E501 E502 E128 E126
     # scrapy/downloadermiddlewares
     scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
     scrapy/downloadermiddlewares/decompression.py E501
@@ -66,19 +69,18 @@ flake8-ignore =
     scrapy/downloadermiddlewares/httpproxy.py E501
     scrapy/downloadermiddlewares/redirect.py E501 W504
     scrapy/downloadermiddlewares/retry.py E501 E126
-    scrapy/downloadermiddlewares/robotstxt.py F401 E501
+    scrapy/downloadermiddlewares/robotstxt.py E501
     scrapy/downloadermiddlewares/stats.py E501
     # scrapy/extensions
     scrapy/extensions/closespider.py E501 E502 E128 E123
     scrapy/extensions/corestats.py E501
     scrapy/extensions/feedexport.py E128 E501
-    scrapy/extensions/httpcache.py E128 E501 E303 F401
+    scrapy/extensions/httpcache.py E128 E501 E303
     scrapy/extensions/memdebug.py E501
     scrapy/extensions/spiderstate.py E501
     scrapy/extensions/telnet.py E501 W504
     scrapy/extensions/throttle.py E501
     # scrapy/http
-    scrapy/http/__init__.py F401
     scrapy/http/common.py E501
     scrapy/http/cookies.py E501
     scrapy/http/request/__init__.py E501
@@ -87,7 +89,7 @@ flake8-ignore =
     scrapy/http/response/__init__.py E501 E128 W293 W291
     scrapy/http/response/text.py E501 W293 E128 E124
     # scrapy/linkextractors
-    scrapy/linkextractors/__init__.py E731 E502 E501 E402 F401
+    scrapy/linkextractors/__init__.py E731 E502 E501 E402
     scrapy/linkextractors/lxmlhtml.py E501 E731 E226
     # scrapy/loader
     scrapy/loader/__init__.py E501 E502 E128
@@ -97,8 +99,8 @@ flake8-ignore =
     scrapy/pipelines/images.py E265 E501
     scrapy/pipelines/media.py E125 E501 E266
     # scrapy/selector
-    scrapy/selector/__init__.py F403 F401
-    scrapy/selector/unified.py F401 E501 E111
+    scrapy/selector/__init__.py F403
+    scrapy/selector/unified.py E501 E111
     # scrapy/settings
     scrapy/settings/__init__.py E501
     scrapy/settings/default_settings.py E501 E261 E114 E116 E226
@@ -106,32 +108,30 @@ flake8-ignore =
     # scrapy/spidermiddlewares
     scrapy/spidermiddlewares/httperror.py E501
     scrapy/spidermiddlewares/offsite.py E501
-    scrapy/spidermiddlewares/referer.py F401 E501 E129 W503 W504
+    scrapy/spidermiddlewares/referer.py E501 E129 W503 W504
     scrapy/spidermiddlewares/urllength.py E501
     # scrapy/spiders
-    scrapy/spiders/__init__.py F401 E501 E402
+    scrapy/spiders/__init__.py E501 E402
     scrapy/spiders/crawl.py E501
     scrapy/spiders/feed.py E501 E261
     scrapy/spiders/sitemap.py E501
     # scrapy/utils
     scrapy/utils/benchserver.py E501
-    scrapy/utils/boto.py F401
     scrapy/utils/conf.py E402 E502 E501
-    scrapy/utils/console.py E261 F401 E306 E305
-    scrapy/utils/curl.py F401
+    scrapy/utils/console.py E261 E306 E305
     scrapy/utils/datatypes.py E501 E226
     scrapy/utils/decorators.py E501
     scrapy/utils/defer.py E501 E128
     scrapy/utils/deprecate.py E128 E501 E127 E502
-    scrapy/utils/engine.py F401 E261
+    scrapy/utils/engine.py E261
     scrapy/utils/gz.py E305 E501 W504
-    scrapy/utils/http.py F403 F401 E226
+    scrapy/utils/http.py F403 E226
     scrapy/utils/httpobj.py E501
     scrapy/utils/iterators.py E501 E701
     scrapy/utils/log.py E128 W503
-    scrapy/utils/markup.py F403 F401 W292
+    scrapy/utils/markup.py F403 W292
     scrapy/utils/misc.py E501 E226
-    scrapy/utils/multipart.py F403 F401 W292
+    scrapy/utils/multipart.py F403 W292
     scrapy/utils/project.py E501
     scrapy/utils/python.py E501
     scrapy/utils/reactor.py E226
@@ -143,7 +143,7 @@ flake8-ignore =
     scrapy/utils/spider.py E271 E501
     scrapy/utils/ssl.py E501
     scrapy/utils/test.py E501
-    scrapy/utils/url.py E501 F403 F401 E128 F405
+    scrapy/utils/url.py E501 F403 E128 F405
     # scrapy
     scrapy/__init__.py E402 E501
     scrapy/_monkeypatches.py W293
@@ -167,29 +167,29 @@ flake8-ignore =
     scrapy/squeues.py E128
     scrapy/statscollectors.py E501
     # tests
-    tests/__init__.py F401 E402 E501
-    tests/mockserver.py E401 E501 E126 E123 F401
+    tests/__init__.py E402 E501
+    tests/mockserver.py E401 E501 E126 E123
     tests/pipelines.py F841 E226
     tests/spiders.py E501 E127
     tests/test_closespider.py E501 E127
     tests/test_command_fetch.py E501 E261
-    tests/test_command_parse.py F401 E501 E128 E303 E226
+    tests/test_command_parse.py E501 E128 E303 E226
     tests/test_command_shell.py E501 E128
-    tests/test_commands.py F401 E128 E501
+    tests/test_commands.py E128 E501
     tests/test_contracts.py E501 E128 W293
     tests/test_crawl.py E501 E741 E265
     tests/test_crawler.py F841 E306 E501
     tests/test_dependencies.py F841 E501 E305
-    tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E126 E226 E123
+    tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
     tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
     tests/test_downloadermiddleware_decompression.py E127
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
-    tests/test_downloadermiddleware_httpcache.py E501 E305 F401
-    tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123
-    tests/test_downloadermiddleware_httpproxy.py F401 E501 E128
+    tests/test_downloadermiddleware_httpcache.py E501 E305
+    tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
+    tests/test_downloadermiddleware_httpproxy.py E501 E128
     tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
     tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
     tests/test_downloadermiddleware_robotstxt.py E501
@@ -197,11 +197,11 @@ flake8-ignore =
     tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
     tests/test_engine.py E401 E501 E502 E128 E261
     tests/test_exporters.py E501 E731 E306 E128 E124
-    tests/test_extension_telnet.py F401 F841
-    tests/test_feedexport.py E501 F401 F841 E241
+    tests/test_extension_telnet.py F841
+    tests/test_feedexport.py E501 F841 E241
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
-    tests/test_http_request.py F401 E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
+    tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
     tests/test_http_response.py E501 E301 E502 E128 E265
     tests/test_item.py E701 E128 F841 E306
     tests/test_link.py E501
@@ -211,20 +211,20 @@ flake8-ignore =
     tests/test_mail.py E128 E501 E305
     tests/test_middleware.py E501 E128
     tests/test_pipeline_crawl.py E131 E501 E128 E126
-    tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226
-    tests/test_pipeline_images.py F401 F841 E501 E303
+    tests/test_pipeline_files.py E501 W293 E303 E272 E226
+    tests/test_pipeline_images.py F841 E501 E303
     tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
     tests/test_request_cb_kwargs.py E501
     tests/test_responsetypes.py E501 E305
-    tests/test_robotstxt_interface.py F401 E501 W291 E501
+    tests/test_robotstxt_interface.py E501 W291 E501
     tests/test_scheduler.py E501 E126 E123
-    tests/test_selector.py F401 E501 E127
-    tests/test_spider.py E501 F401
+    tests/test_selector.py E501 E127
+    tests/test_spider.py E501
     tests/test_spidermiddleware.py E501 E226
     tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
     tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
-    tests/test_spidermiddleware_output_chain.py F401 E501 W293 E226
-    tests/test_spidermiddleware_referer.py F401 E501 F841 E125 E201 E261 E124 E501 E241 E121
+    tests/test_spidermiddleware_output_chain.py E501 W293 E226
+    tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121
     tests/test_squeues.py E501 E701 E741
     tests/test_utils_conf.py E501 E303 E128
     tests/test_utils_curl.py E501
@@ -235,16 +235,16 @@ flake8-ignore =
     tests/test_utils_iterators.py E501 E128 E129 E303 E241
     tests/test_utils_log.py E741 E226
     tests/test_utils_python.py E501 E303 E731 E701 E305
-    tests/test_utils_reqser.py F401 E501 E128
+    tests/test_utils_reqser.py E501 E128
     tests/test_utils_request.py E501 E128 E305
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841 E731 E226
     tests/test_utils_sitemap.py E128 E501 E124
     tests/test_utils_spider.py E261 E305
     tests/test_utils_template.py E305
-    tests/test_utils_url.py F401 E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
+    tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
     tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
     tests/test_cmdline/__init__.py E502 E501
-    tests/test_settings/__init__.py F401 E501 E128
+    tests/test_settings/__init__.py E501 E128
     tests/test_spiderloader/__init__.py E128 E501
     tests/test_utils_misc/__init__.py E501
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index 3e6c11b7d..9d4437a47 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -1,6 +1,4 @@
-from __future__ import print_function
 import time
-import sys
 from collections import defaultdict
 from unittest import TextTestRunner, TextTestResult as _TextTestResult
 
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index 31c17c0ab..41e77ba3b 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -1,4 +1,4 @@
-from scrapy.commands import fetch, ScrapyCommand
+from scrapy.commands import fetch
 from scrapy.utils.response import open_in_browser
 
 
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 949dacbc8..213268741 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -1,6 +1,4 @@
-from __future__ import absolute_import
 import random
-import warnings
 from time import time
 from datetime import datetime
 from collections import deque
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index ac4b867c3..6111e132a 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -1,3 +1,2 @@
-from __future__ import absolute_import
 from .http10 import HTTP10DownloadHandler
 from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index d8bbdd326..808d1bf21 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -21,7 +21,7 @@ def _get_boto_connection():
             return http_request.headers
 
     try:
-        import boto.auth
+        import boto.auth  # noqa: F401
     except ImportError:
         _S3Connection = _v19_S3Connection
     else:
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index f3fabf710..11403957c 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -6,18 +6,16 @@ import os
 from email.utils import mktime_tz, parsedate_tz
 from importlib import import_module
 from time import time
-from warnings import warn
 from weakref import WeakKeyDictionary
 
 from six.moves import cPickle as pickle
 from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Headers, Response
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.project import data_path
-from scrapy.utils.python import to_bytes, to_unicode, garbage_collect
+from scrapy.utils.python import to_bytes, to_unicode
 from scrapy.utils.request import request_fingerprint
 
 
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index e4c62f87b..8c3693f04 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -118,4 +118,4 @@ class FilteringLinkExtractor(object):
 
 
 # Top-level imports
-from .lxmlhtml import LxmlLinkExtractor as LinkExtractor
+from .lxmlhtml import LxmlLinkExtractor as LinkExtractor  # noqa: F401
diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py
index 90e96ee92..a9240c1f6 100644
--- a/scrapy/selector/__init__.py
+++ b/scrapy/selector/__init__.py
@@ -1,4 +1,4 @@
 """
 Selectors
 """
-from scrapy.selector.unified import *
+from scrapy.selector.unified import *  # noqa: F401
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 62fda40b0..a08955dc9 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -2,12 +2,10 @@
 XPath selectors based on lxml
 """
 
-import warnings
 from parsel import Selector as _ParselSelector
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.python import to_bytes
 from scrapy.http import HtmlResponse, XmlResponse
-from scrapy.utils.decorators import deprecated
 
 
 __all__ = ['Selector', 'SelectorList']
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 94095bc27..8d15dfceb 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -10,7 +10,6 @@ from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.deprecate import method_is_overridden
 
 
@@ -100,6 +99,6 @@ class Spider(object_ref):
 
 
 # Top-level imports
-from scrapy.spiders.crawl import CrawlSpider, Rule
-from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider
-from scrapy.spiders.sitemap import SitemapSpider
+from scrapy.spiders.crawl import CrawlSpider, Rule  # noqa: F401
+from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider  # noqa: F401
+from scrapy.spiders.sitemap import SitemapSpider  # noqa: F401
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index c8fc911bb..b76d5e56e 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -7,7 +7,7 @@ from scrapy.exceptions import NotConfigured
 
 def is_botocore():
     try:
-        import botocore
+        import botocore  # noqa: F401
         return True
     except ImportError:
         raise NotConfigured('missing botocore library')
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index a26e84d38..688e28c34 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -52,7 +52,7 @@ def _embed_standard_shell(namespace={}, banner=''):
     except ImportError:
         pass
     else:
-        import rlcompleter
+        import rlcompleter  # noqa: F401
         readline.parse_and_bind("tab:complete")
     @wraps(_embed_standard_shell)
     def wrapper(namespace=namespace, banner=''):
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index b3fd0a497..7fb25a71d 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -4,7 +4,7 @@ from shlex import split
 
 from six.moves.http_cookies import SimpleCookie
 from six.moves.urllib.parse import urlparse
-from six import string_types, iteritems
+from six import iteritems
 from w3lib.http import basic_auth_header
 
 
diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py
index 2c20b5c88..267c7ecd1 100644
--- a/scrapy/utils/engine.py
+++ b/scrapy/utils/engine.py
@@ -1,7 +1,7 @@
 """Some debugging functions for working with the Scrapy engine"""
 
-from __future__ import print_function
-from time import time # used in global tests code
+# used in global tests code
+from time import time  # noqa: F401
 
 
 def get_engine_status(engine):
diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py
index ad49ef3e9..bab262393 100644
--- a/scrapy/utils/http.py
+++ b/scrapy/utils/http.py
@@ -8,7 +8,7 @@ import warnings
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.decorators import deprecated
-from w3lib.http import *
+from w3lib.http import *  # noqa: F401
 
 
 warnings.warn("Module `scrapy.utils.http` is deprecated, "
diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py
index a18f308a3..2455fcc16 100644
--- a/scrapy/utils/markup.py
+++ b/scrapy/utils/markup.py
@@ -6,7 +6,7 @@ For new code, always import from w3lib.html instead of this module
 import warnings
 
 from scrapy.exceptions import ScrapyDeprecationWarning
-from w3lib.html import *
+from w3lib.html import *  # noqa: F401
 
 
 warnings.warn("Module `scrapy.utils.markup` is deprecated. "
diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py
index c2d8afd07..e81f63152 100644
--- a/scrapy/utils/multipart.py
+++ b/scrapy/utils/multipart.py
@@ -6,7 +6,7 @@ For new code, always import from w3lib.form instead of this module
 import warnings
 
 from scrapy.exceptions import ScrapyDeprecationWarning
-from w3lib.form import *
+from w3lib.form import *  # noqa: F401
 
 
 warnings.warn("Module `scrapy.utils.multipart` is deprecated. "
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index b3a4be007..a3e1bde63 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -12,7 +12,6 @@ from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse
 # scrapy.utils.url was moved to w3lib.url and import * ensures this
 # move doesn't break old code
 from w3lib.url import *
-from w3lib.url import _safe_chars, _unquotepath
 from scrapy.utils.python import to_unicode
 
 
diff --git a/tests/mockserver.py b/tests/mockserver.py
index b766bb653..7ebb8bb62 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,9 +1,11 @@
-from __future__ import print_function
-import sys, time, random, os, json
-from six.moves.urllib.parse import urlencode
+import json
+import os
+import random
+import sys
 from subprocess import Popen, PIPE
 
 from OpenSSL import SSL
+from six.moves.urllib.parse import urlencode
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
 from twisted.web.static import File
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 59d4a3eec..0421529a6 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -543,7 +543,7 @@ class Https11InvalidDNSPattern(Https11TestCase):
 
     def setUp(self):
         try:
-            from service_identity.exceptions import CertificateError
+            from service_identity.exceptions import CertificateError  # noqa: F401
         except ImportError:
             raise unittest.SkipTest("cryptography lib is too old")
         self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
@@ -778,7 +778,7 @@ class S3TestCase(unittest.TestCase):
     @contextlib.contextmanager
     def _mocked_date(self, date):
         try:
-            import botocore.auth
+            import botocore.auth  # noqa: F401
         except ImportError:
             yield
         else:
@@ -843,8 +843,10 @@ class S3TestCase(unittest.TestCase):
                 b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
 
     def test_request_signing5(self):
-        try: import botocore
-        except ImportError: pass
+        try:
+            import botocore  # noqa: F401
+        except ImportError:
+            pass
         else:
             raise unittest.SkipTest(
                 'botocore does not support overriding date with x-amz-date')
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 00e6c685e..9401dd66d 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -1,12 +1,9 @@
-from __future__ import print_function
 import time
 import tempfile
 import shutil
 import unittest
 import email.utils
 from contextlib import contextmanager
-import pytest
-import sys
 
 from scrapy.http import Response, HtmlResponse, Request
 from scrapy.spiders import Spider
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 0745c8dd3..c6a823b53 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -70,7 +70,7 @@ class HttpCompressionTest(TestCase):
 
     def test_process_response_br(self):
         try:
-            import brotli
+            import brotli  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         response = self._getresponse('br')
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 30920b2da..36743b1de 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -1,11 +1,10 @@
 import os
-import sys
 from functools import partial
-from twisted.trial.unittest import TestCase, SkipTest
+from twisted.trial.unittest import TestCase
 
 from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
 from scrapy.exceptions import NotConfigured
-from scrapy.http import Response, Request
+from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.crawler import Crawler
 from scrapy.settings import Settings
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index 875ceb83c..873a97248 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -3,7 +3,7 @@ from twisted.conch.telnet import ITelnetProtocol
 from twisted.cred import credentials
 from twisted.internet import defer
 
-from scrapy.extensions.telnet import TelnetConsole, logger
+from scrapy.extensions.telnet import TelnetConsole
 from scrapy.utils.test import get_crawler
 
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 87139e81f..ce3c4f059 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -167,7 +167,7 @@ class S3FeedStorageTest(unittest.TestCase):
                 create=True)
     def test_parse_credentials(self):
         try:
-            import boto
+            import boto  # noqa: F401
         except ImportError:
             raise unittest.SkipTest("S3FeedStorage requires boto")
         aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key',
@@ -268,7 +268,7 @@ class S3FeedStorageTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_store_botocore_without_acl(self):
         try:
-            import botocore
+            import botocore  # noqa: F401
         except ImportError:
             raise unittest.SkipTest('botocore is required')
 
@@ -288,7 +288,7 @@ class S3FeedStorageTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_store_botocore_with_acl(self):
         try:
-            import botocore
+            import botocore  # noqa: F401
         except ImportError:
             raise unittest.SkipTest('botocore is required')
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 5134a03b9..9df6ff67b 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-import cgi
 import unittest
 import re
 import json
@@ -8,7 +6,7 @@ from urllib.parse import unquote_to_bytes
 import warnings
 
 from six.moves import xmlrpc_client as xmlrpclib
-from six.moves.urllib.parse import urlparse, parse_qs, unquote
+from six.moves.urllib.parse import urlparse, parse_qs
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_unicode
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 4f7265763..7f1cb4a11 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -1,7 +1,6 @@
 import io
 import hashlib
 import random
-import warnings
 from tempfile import mkdtemp
 from shutil import rmtree
 
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 080507276..27d79437b 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -5,7 +5,7 @@ from twisted.trial import unittest
 def reppy_available():
     # check if reppy parser is installed
     try:
-        from reppy.robots import Robots
+        from reppy.robots import Robots  # noqa: F401
     except ImportError:
         return False
     return True
@@ -14,7 +14,7 @@ def reppy_available():
 def rerp_available():
     # check if robotexclusionrulesparser is installed
     try:
-        from robotexclusionrulesparser import RobotExclusionRulesParser
+        from robotexclusionrulesparser import RobotExclusionRulesParser  # noqa: F401
     except ImportError:
         return False
     return True
@@ -23,7 +23,7 @@ def rerp_available():
 def protego_available():
     # check if protego parser is installed
     try:
-        from protego import Protego
+        from protego import Protego  # noqa: F401
     except ImportError:
         return False
     return True
diff --git a/tests/test_selector.py b/tests/test_selector.py
index b2565dd78..09c2546fb 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -1,9 +1,9 @@
-import warnings
 import weakref
+
 from twisted.trial import unittest
+
 from scrapy.http import TextResponse, HtmlResponse, XmlResponse
 from scrapy.selector import Selector
-from lxml import etree
 
 
 class SelectorTestCase(unittest.TestCase):
diff --git a/tests/test_spider.py b/tests/test_spider.py
index c0fccfdd6..aa43e3b3a 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -15,7 +15,6 @@ from scrapy.spiders import Spider, CrawlSpider, Rule, XMLFeedSpider, \
     CSVFeedSpider, SitemapSpider
 from scrapy.linkextractors import LinkExtractor
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.trackref import object_ref
 from scrapy.utils.test import get_crawler
 
 
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 940e31070..5b7b5e7aa 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -6,7 +6,6 @@ from twisted.internet import defer
 from scrapy import Spider, Request
 from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
-from tests.spiders import MockServerSpider
 
 
 class LogExceptionMiddleware:
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index a9c31a983..2be6a1cd5 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -2,7 +2,6 @@ from six.moves.urllib.parse import urlparse
 from unittest import TestCase
 import warnings
 
-from scrapy.exceptions import NotConfigured
 from scrapy.http import Response, Request
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 92cd16de7..06d9c004c 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -1,8 +1,4 @@
-# -*- coding: utf-8 -*-
 import unittest
-import sys
-
-import six
 
 from scrapy.http import Request, FormRequest
 from scrapy.spiders import Spider
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 93addc082..c7bcaf88b 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,13 +1,9 @@
 # -*- coding: utf-8 -*-
 import unittest
 
-import six
-from six.moves.urllib.parse import urlparse
-
 from scrapy.spiders import Spider
 from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
-                              add_http_if_no_scheme, guess_scheme,
-                              parse_url, strip_url)
+                              add_http_if_no_scheme, guess_scheme, strip_url)
 
 __doctests__ = ['scrapy.utils.url']
 
diff --git a/tox.ini b/tox.ini
index fd75d18e2..e8672ea27 100644
--- a/tox.ini
+++ b/tox.ini
@@ -73,6 +73,7 @@ commands =
 basepython = python3.8
 deps =
     {[testenv]deps}
+    -r docs/requirements.txt
     pytest-flake8
 commands =
     py.test --flake8 {posargs:docs scrapy tests}

From b23288135633f9e800b575d84df0a109665a840e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 21 Nov 2019 14:30:10 +0100
Subject: [PATCH 2429/2974] Restore intentional import of unused objects

---
 scrapy/utils/url.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index a3e1bde63..2c7b324a1 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -12,6 +12,7 @@ from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse
 # scrapy.utils.url was moved to w3lib.url and import * ensures this
 # move doesn't break old code
 from w3lib.url import *
+from w3lib.url import _safe_chars, _unquotepath  # noqa: F401
 from scrapy.utils.python import to_unicode
 
 

From 1718e450ef9549a4fc71b01dba1e6faf7a63238a Mon Sep 17 00:00:00 2001
From: Mabel Villalba <mabelvj@gmail.com>
Date: Mon, 18 Nov 2019 12:33:55 +0100
Subject: [PATCH 2430/2974] [start_url] Fixes #4133: Raise AttributeError error
 when empty 'start_urls' and 'start_url' found. Added test.

---
 scrapy/spiders/__init__.py | 5 +++++
 tests/test_spider.py       | 7 +++++++
 2 files changed, 12 insertions(+)

diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index e9c131e3b..5a35fcdb6 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -68,6 +68,11 @@ class Spider(object_ref):
 
     def start_requests(self):
         cls = self.__class__
+        if not self.start_urls and hasattr(self, 'start_url'):
+            raise AttributeError(
+                "Crawling could not start: 'start_urls' not found "
+                "or empty (but found 'start_url' attribute instead, "
+                "did you miss an 's'?)")
         if method_is_overridden(cls, Spider, 'make_requests_from_url'):
             warnings.warn(
                 "Spider.make_requests_from_url method is deprecated; it "
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 83fb68c2f..0a6640cec 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -391,6 +391,13 @@ class CrawlSpiderTest(SpiderTest):
         self.assertTrue(hasattr(spider, '_follow_links'))
         self.assertFalse(spider._follow_links)
 
+    def test_start_url(self):
+        spider = self.spider_class("example.com")
+        spider.start_url = 'https://www.example.com'
+
+        with self.assertRaisesRegex(AttributeError,
+                                    r'^Crawling could not start.*$'):
+            list(spider.start_requests())
 
 class SitemapSpiderTest(SpiderTest):
 

From 9b5053c564fb465cfe85d0799388b3591638c5a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 21 Nov 2019 22:00:34 +0100
Subject: [PATCH 2431/2974] Undo unintended tox.ini changes

---
 tox.ini | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index e8672ea27..fd75d18e2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -73,7 +73,6 @@ commands =
 basepython = python3.8
 deps =
     {[testenv]deps}
-    -r docs/requirements.txt
     pytest-flake8
 commands =
     py.test --flake8 {posargs:docs scrapy tests}

From 55cc5c9068a7a39908f90e6403e6bd8c8ef11a4d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 22 Nov 2019 12:41:31 -0300
Subject: [PATCH 2432/2974] Skip pickle in bandit check

---
 .bandit.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.bandit.yml b/.bandit.yml
index 00554587a..cc7db3a66 100644
--- a/.bandit.yml
+++ b/.bandit.yml
@@ -1,6 +1,7 @@
 skips:
 - B101
 - B105
+- B301
 - B303
 - B306
 - B307
@@ -8,6 +9,7 @@ skips:
 - B320
 - B321
 - B402
+- B403
 - B404
 - B406
 - B410

From 40b5cfc0a4adbc51fa35018b09902255228e360c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 23 Jul 2019 18:33:19 -0300
Subject: [PATCH 2433/2974] Item loaders: allow single-argument processors
 (unbound methods)

---
 docs/topics/loaders.rst   | 16 +------------
 scrapy/loader/__init__.py | 22 +++++++++++++-----
 tests/test_loader.py      | 48 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 65 insertions(+), 21 deletions(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 12a5e5c60..81c8dab03 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -142,20 +142,6 @@ accept one (and only one) positional argument, which will be an iterable.
    containing the collected values (for that field). The result of the output
    processors is the value that will be finally assigned to the item.
 
-If you want to use a plain function as a processor, make sure it receives
-``self`` as the first argument::
-
-    def lowercase_processor(self, values):
-        for v in values:
-            yield v.lower()
-
-    class MyItemLoader(ItemLoader):
-        name_in = lowercase_processor
-
-This is because whenever a function is assigned as a class variable, it becomes
-a method and would be passed the instance as the the first argument when being
-called. See `this answer on stackoverflow`_ for more details.
-
 The other thing you need to keep in mind is that the values returned by input
 processors are collected internally (in lists) and then passed to output
 processors to populate the fields.
@@ -163,7 +149,7 @@ processors to populate the fields.
 Last, but not least, Scrapy comes with some :ref:`commonly used processors
 <topics-loaders-available-processors>` built-in for convenience.
 
-.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635
+
 
 Declaring Item Loaders
 ======================
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 60fd6d222..7cf67e29e 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -4,8 +4,7 @@ Item Loader
 See documentation in docs/topics/loaders.rst
 """
 from collections import defaultdict
-
-import six
+from contextlib import suppress
 
 from scrapy.item import Item
 from scrapy.loader.common import wrap_loader_context
@@ -15,6 +14,17 @@ from scrapy.utils.misc import arg_to_iter, extract_regex
 from scrapy.utils.python import flatten
 
 
+def unbound_method(method):
+    """
+    Allow to use single-argument functions as input or output processors
+    (no need to define an unused first 'self' argument)
+    """
+    with suppress(AttributeError):
+        if '.' not in method.__qualname__:
+            return method.__func__
+    return method
+
+
 class ItemLoader(object):
 
     default_item_class = Item
@@ -72,7 +82,7 @@ class ItemLoader(object):
         if value is None:
             return
         if not field_name:
-            for k, v in six.iteritems(value):
+            for k, v in value.items():
                 self._add_value(k, v)
         else:
             self._add_value(field_name, value)
@@ -82,7 +92,7 @@ class ItemLoader(object):
         if value is None:
             return
         if not field_name:
-            for k, v in six.iteritems(value):
+            for k, v in value.items():
                 self._replace_value(k, v)
         else:
             self._replace_value(field_name, value)
@@ -142,14 +152,14 @@ class ItemLoader(object):
         if not proc:
             proc = self._get_item_field_attr(field_name, 'input_processor',
                                              self.default_input_processor)
-        return proc
+        return unbound_method(proc)
 
     def get_output_processor(self, field_name):
         proc = getattr(self, '%s_out' % field_name, None)
         if not proc:
             proc = self._get_item_field_attr(field_name, 'output_processor',
                                              self.default_output_processor)
-        return proc
+        return unbound_method(proc)
 
     def _process_input_value(self, field_name, value):
         proc = self.get_input_processor(field_name)
diff --git a/tests/test_loader.py b/tests/test_loader.py
index b87602809..6bfc31dbf 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -994,5 +994,53 @@ class SelectJmesTestCase(unittest.TestCase):
             )
 
 
+# Functions as processors
+
+def function_processor_strip(iterable):
+    return [x.strip() for x in iterable]
+
+
+def function_processor_upper(iterable):
+    return [x.upper() for x in iterable]
+
+
+class FunctionProcessorItem(Item):
+    foo = Field(
+        input_processor=function_processor_strip,
+        output_processor=function_processor_upper,
+    )
+
+
+class FunctionProcessorItemLoader(ItemLoader):
+    default_item_class = FunctionProcessorItem
+
+
+class FunctionProcessorDictLoader(ItemLoader):
+    default_item_class = dict
+    foo_in = function_processor_strip
+    foo_out = function_processor_upper
+
+
+class FunctionProcessorTestCase(unittest.TestCase):
+
+    def test_processor_defined_in_item(self):
+        lo = FunctionProcessorItemLoader()
+        lo.add_value('foo', '  bar  ')
+        lo.add_value('foo', ['  asdf  ', '  qwerty  '])
+        self.assertEqual(
+            dict(lo.load_item()),
+            {'foo': ['BAR', 'ASDF', 'QWERTY']}
+        )
+
+    def test_processor_defined_in_item_loader(self):
+        lo = FunctionProcessorDictLoader()
+        lo.add_value('foo', '  bar  ')
+        lo.add_value('foo', ['  asdf  ', '  qwerty  '])
+        self.assertEqual(
+            dict(lo.load_item()),
+            {'foo': ['BAR', 'ASDF', 'QWERTY']}
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

From 7a7d13b1122dac397ee0bb8edd4e6fd61665e232 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 23 Nov 2019 19:04:02 -0300
Subject: [PATCH 2434/2974] Rename LogFormatter.error to item_error

---
 scrapy/core/scraper.py     | 2 +-
 scrapy/logformatter.py     | 6 +++---
 tests/test_logformatter.py | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index db463f989..c5bb48ea6 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -231,7 +231,7 @@ class Scraper(object):
                     signal=signals.item_dropped, item=item, response=response,
                     spider=spider, exception=output.value)
             else:
-                logkws = self.logformatter.error(item, ex, response, spider)
+                logkws = self.logformatter.item_error(item, ex, response, spider)
                 logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
                            exc_info=failure_to_exc_info(output))
                 return self.signals.send_catch_log_deferred(
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 5189d7cfa..79c752da4 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -8,7 +8,7 @@ from scrapy.utils.request import referer_str
 SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s"
 DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
 CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
-ERRORMSG = u"'Error processing %(item)s'"
+ITEMERRORMSG = u"'Error processing %(item)s'"
 
 
 class LogFormatter(object):
@@ -93,11 +93,11 @@ class LogFormatter(object):
             }
         }
 
-    def error(self, item, exception, response, spider):
+    def item_error(self, item, exception, response, spider):
         """Logs a message when an item causes an error while it is passing through the item pipeline."""
         return {
             'level': logging.ERROR,
-            'msg': ERRORMSG,
+            'msg': ITEMERRORMSG,
             'args': {
                 'item': item,
             }
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index d0b23a8c4..f2f8c0464 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -63,13 +63,13 @@ class LogFormatterTestCase(unittest.TestCase):
         assert all(isinstance(x, six.text_type) for x in lines)
         self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
 
-    def test_error(self):
+    def test_item_error(self):
         # In practice, the complete traceback is shown by passing the
         # 'exc_info' argument to the logging function
         item = {'key': 'value'}
         exception = Exception()
         response = Response("http://www.example.com")
-        logkws = self.formatter.error(item, exception, response, self.spider)
+        logkws = self.formatter.item_error(item, exception, response, self.spider)
         logline = logkws['msg'] % logkws['args']
         self.assertEqual(logline, u"'Error processing {'key': 'value'}'")
 

From facb9265421ead8afb839323af2e18f81dda560b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 23 Nov 2019 19:16:41 -0300
Subject: [PATCH 2435/2974] Remove quotes from item_error message

---
 scrapy/logformatter.py     | 8 ++++----
 tests/test_logformatter.py | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 79c752da4..9e038160f 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -5,10 +5,10 @@ from twisted.python.failure import Failure
 
 from scrapy.utils.request import referer_str
 
-SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s"
-DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
-CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
-ITEMERRORMSG = u"'Error processing %(item)s'"
+SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s"
+DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s"
+CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
+ITEMERRORMSG = "Error processing %(item)s"
 
 
 class LogFormatter(object):
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index f2f8c0464..990927f71 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -71,7 +71,7 @@ class LogFormatterTestCase(unittest.TestCase):
         response = Response("http://www.example.com")
         logkws = self.formatter.item_error(item, exception, response, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline, u"'Error processing {'key': 'value'}'")
+        self.assertEqual(logline, u"Error processing {'key': 'value'}")
 
     def test_scraped(self):
         item = CustomItem()

From 4756e7c587880997a54d9abf94b9a4c0b5bab71c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 23 Nov 2019 19:33:29 -0300
Subject: [PATCH 2436/2974] LogFormatter.spider_error

---
 scrapy/core/scraper.py     |  8 ++++----
 scrapy/logformatter.py     | 12 ++++++++++++
 tests/test_logformatter.py | 14 ++++++++++++++
 3 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index c5bb48ea6..21820e988 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -16,7 +16,7 @@ from scrapy import signals
 from scrapy.http import Request, Response
 from scrapy.item import BaseItem
 from scrapy.core.spidermw import SpiderMiddlewareManager
-from scrapy.utils.request import referer_str
+
 
 logger = logging.getLogger(__name__)
 
@@ -152,9 +152,9 @@ class Scraper(object):
         if isinstance(exc, CloseSpider):
             self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
             return
-        logger.error(
-            "Spider error processing %(request)s (referer: %(referer)s)",
-            {'request': request, 'referer': referer_str(request)},
+        logkws = self.logformatter.spider_error(_failure, request, response, spider)
+        logger.log(
+            *logformatter_adapter(logkws),
             exc_info=failure_to_exc_info(_failure),
             extra={'spider': spider}
         )
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 9e038160f..d87f685d5 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -9,6 +9,7 @@ SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s"
 DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s"
 CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
 ITEMERRORMSG = "Error processing %(item)s"
+SPIDERERRORMSG = "Spider error processing %(request)s (referer: %(referer)s)"
 
 
 class LogFormatter(object):
@@ -103,6 +104,17 @@ class LogFormatter(object):
             }
         }
 
+    def spider_error(self, failure, request, response, spider):
+        """Logs an error message from a spider."""
+        return {
+            'level': logging.ERROR,
+            'msg': SPIDERERRORMSG,
+            'args': {
+                'request': request,
+                'referer': referer_str(request),
+            }
+        }
+
     @classmethod
     def from_crawler(cls, crawler):
         return cls()
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 990927f71..47d2747c2 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -2,6 +2,7 @@ import unittest
 
 from testfixtures import LogCapture
 from twisted.internet import defer
+from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase as TwistedTestCase
 import six
 
@@ -73,6 +74,19 @@ class LogFormatterTestCase(unittest.TestCase):
         logline = logkws['msg'] % logkws['args']
         self.assertEqual(logline, u"Error processing {'key': 'value'}")
 
+    def test_spider_error(self):
+        # In practice, the complete traceback is shown by passing the
+        # 'exc_info' argument to the logging function
+        failure = Failure(Exception())
+        request = Request("http://www.example.com", headers={'Referer': 'http://example.org'})
+        response = Response("http://www.example.com", request=request)
+        logkws = self.formatter.spider_error(failure, request, response, self.spider)
+        logline = logkws['msg'] % logkws['args']
+        self.assertEqual(
+            logline,
+            "Spider error processing <GET http://www.example.com> (referer: http://example.org)"
+        )
+
     def test_scraped(self):
         item = CustomItem()
         item['name'] = u'\xa3'

From 03af8885ff475dc47a3de89517b1a5d627bd49c4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 23 Nov 2019 20:02:44 -0300
Subject: [PATCH 2437/2974] LogFormatter.download_error

---
 scrapy/core/scraper.py     | 22 +++++++++++++---------
 scrapy/logformatter.py     | 16 ++++++++++++++++
 tests/test_logformatter.py | 18 ++++++++++++++++++
 3 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 21820e988..427969f30 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -200,19 +200,23 @@ class Scraper(object):
         """Log and silence errors that come from the engine (typically download
         errors that got propagated thru here)
         """
-        if (isinstance(download_failure, Failure) and
-                not download_failure.check(IgnoreRequest)):
+        if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest):
             if download_failure.frames:
-                logger.error('Error downloading %(request)s',
-                             {'request': request},
-                             exc_info=failure_to_exc_info(download_failure),
-                             extra={'spider': spider})
+                logkws = self.logformatter.download_error(download_failure, request, spider)
+                logger.log(
+                    *logformatter_adapter(logkws),
+                    extra={'spider': spider},
+                    exc_info=failure_to_exc_info(download_failure),
+                )
             else:
                 errmsg = download_failure.getErrorMessage()
                 if errmsg:
-                    logger.error('Error downloading %(request)s: %(errmsg)s',
-                                 {'request': request, 'errmsg': errmsg},
-                                 extra={'spider': spider})
+                    logkws = self.logformatter.download_error(
+                        download_failure, request, spider, errmsg)
+                    logger.log(
+                        *logformatter_adapter(logkws),
+                        extra={'spider': spider},
+                    )
 
         if spider_failure is not download_failure:
             return spider_failure
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index d87f685d5..99bd5cfac 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -10,6 +10,8 @@ DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s"
 CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
 ITEMERRORMSG = "Error processing %(item)s"
 SPIDERERRORMSG = "Spider error processing %(request)s (referer: %(referer)s)"
+DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s"
+DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s"
 
 
 class LogFormatter(object):
@@ -115,6 +117,20 @@ class LogFormatter(object):
             }
         }
 
+    def download_error(self, failure, request, spider, errmsg=None):
+        """Logs a download error message from a spider (typically coming from the engine)."""
+        args = {'request': request}
+        if errmsg:
+            msg = DOWNLOADERRORMSG_LONG
+            args['errmsg'] = errmsg
+        else:
+            msg = DOWNLOADERRORMSG_SHORT
+        return {
+            'level': logging.ERROR,
+            'msg': msg,
+            'args': args,
+        }
+
     @classmethod
     def from_crawler(cls, crawler):
         return cls()
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 47d2747c2..697ac1d15 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -87,6 +87,24 @@ class LogFormatterTestCase(unittest.TestCase):
             "Spider error processing <GET http://www.example.com> (referer: http://example.org)"
         )
 
+    def test_download_error_short(self):
+        # In practice, the complete traceback is shown by passing the
+        # 'exc_info' argument to the logging function
+        failure = Failure(Exception())
+        request = Request("http://www.example.com")
+        logkws = self.formatter.download_error(failure, request, self.spider)
+        logline = logkws['msg'] % logkws['args']
+        self.assertEqual(logline, "Error downloading <GET http://www.example.com>")
+
+    def test_download_error_long(self):
+        # In practice, the complete traceback is shown by passing the
+        # 'exc_info' argument to the logging function
+        failure = Failure(Exception())
+        request = Request("http://www.example.com")
+        logkws = self.formatter.download_error(failure, request, self.spider, "Some message")
+        logline = logkws['msg'] % logkws['args']
+        self.assertEqual(logline, "Error downloading <GET http://www.example.com>: Some message")
+
     def test_scraped(self):
         item = CustomItem()
         item['name'] = u'\xa3'

From 54b056c4be045099f9f872c5455c003d68b578a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 25 Nov 2019 12:13:31 +0100
Subject: [PATCH 2438/2974] Make developer-tools doctests pass

---
 docs/_tests/quotes.html         | 281 ++++++++++++++++++++++++++++++++
 docs/topics/developer-tools.rst |  40 +++--
 pytest.ini                      |   1 -
 3 files changed, 308 insertions(+), 14 deletions(-)
 create mode 100644 docs/_tests/quotes.html

diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html
new file mode 100644
index 000000000..71aff8847
--- /dev/null
+++ b/docs/_tests/quotes.html
@@ -0,0 +1,281 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+	<meta charset="UTF-8">
+	<title>Quotes to Scrape</title>
+    <link rel="stylesheet" href="/static/bootstrap.min.css">
+    <link rel="stylesheet" href="/static/main.css">
+</head>
+<body>
+    <div class="container">
+        <div class="row header-box">
+            <div class="col-md-8">
+                <h1>
+                    <a href="/" style="text-decoration: none">Quotes to Scrape</a>
+                </h1>
+            </div>
+            <div class="col-md-4">
+                <p>
+                
+                    <a href="/login">Login</a>
+                
+                </p>
+            </div>
+        </div>
+    
+
+<div class="row">
+    <div class="col-md-8">
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>
+        <span>by <small class="author" itemprop="author">Albert Einstein</small>
+        <a href="/author/Albert-Einstein">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="change,deep-thoughts,thinking,world" /    > 
+            
+            <a class="tag" href="/tag/change/page/1/">change</a>
+            
+            <a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
+            
+            <a class="tag" href="/tag/thinking/page/1/">thinking</a>
+            
+            <a class="tag" href="/tag/world/page/1/">world</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“It is our choices, Harry, that show what we truly are, far more than our abilities.”</span>
+        <span>by <small class="author" itemprop="author">J.K. Rowling</small>
+        <a href="/author/J-K-Rowling">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="abilities,choices" /    > 
+            
+            <a class="tag" href="/tag/abilities/page/1/">abilities</a>
+            
+            <a class="tag" href="/tag/choices/page/1/">choices</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</span>
+        <span>by <small class="author" itemprop="author">Albert Einstein</small>
+        <a href="/author/Albert-Einstein">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="inspirational,life,live,miracle,miracles" /    > 
+            
+            <a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
+            
+            <a class="tag" href="/tag/life/page/1/">life</a>
+            
+            <a class="tag" href="/tag/live/page/1/">live</a>
+            
+            <a class="tag" href="/tag/miracle/page/1/">miracle</a>
+            
+            <a class="tag" href="/tag/miracles/page/1/">miracles</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</span>
+        <span>by <small class="author" itemprop="author">Jane Austen</small>
+        <a href="/author/Jane-Austen">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="aliteracy,books,classic,humor" /    > 
+            
+            <a class="tag" href="/tag/aliteracy/page/1/">aliteracy</a>
+            
+            <a class="tag" href="/tag/books/page/1/">books</a>
+            
+            <a class="tag" href="/tag/classic/page/1/">classic</a>
+            
+            <a class="tag" href="/tag/humor/page/1/">humor</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“Imperfection is beauty, madness is genius and it&#39;s better to be absolutely ridiculous than absolutely boring.”</span>
+        <span>by <small class="author" itemprop="author">Marilyn Monroe</small>
+        <a href="/author/Marilyn-Monroe">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="be-yourself,inspirational" /    > 
+            
+            <a class="tag" href="/tag/be-yourself/page/1/">be-yourself</a>
+            
+            <a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“Try not to become a man of success. Rather become a man of value.”</span>
+        <span>by <small class="author" itemprop="author">Albert Einstein</small>
+        <a href="/author/Albert-Einstein">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="adulthood,success,value" /    > 
+            
+            <a class="tag" href="/tag/adulthood/page/1/">adulthood</a>
+            
+            <a class="tag" href="/tag/success/page/1/">success</a>
+            
+            <a class="tag" href="/tag/value/page/1/">value</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“It is better to be hated for what you are than to be loved for what you are not.”</span>
+        <span>by <small class="author" itemprop="author">André Gide</small>
+        <a href="/author/Andre-Gide">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="life,love" /    > 
+            
+            <a class="tag" href="/tag/life/page/1/">life</a>
+            
+            <a class="tag" href="/tag/love/page/1/">love</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“I have not failed. I&#39;ve just found 10,000 ways that won&#39;t work.”</span>
+        <span>by <small class="author" itemprop="author">Thomas A. Edison</small>
+        <a href="/author/Thomas-A-Edison">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="edison,failure,inspirational,paraphrased" /    > 
+            
+            <a class="tag" href="/tag/edison/page/1/">edison</a>
+            
+            <a class="tag" href="/tag/failure/page/1/">failure</a>
+            
+            <a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
+            
+            <a class="tag" href="/tag/paraphrased/page/1/">paraphrased</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“A woman is like a tea bag; you never know how strong it is until it&#39;s in hot water.”</span>
+        <span>by <small class="author" itemprop="author">Eleanor Roosevelt</small>
+        <a href="/author/Eleanor-Roosevelt">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="misattributed-eleanor-roosevelt" /    > 
+            
+            <a class="tag" href="/tag/misattributed-eleanor-roosevelt/page/1/">misattributed-eleanor-roosevelt</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“A day without sunshine is like, you know, night.”</span>
+        <span>by <small class="author" itemprop="author">Steve Martin</small>
+        <a href="/author/Steve-Martin">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="humor,obvious,simile" /    > 
+            
+            <a class="tag" href="/tag/humor/page/1/">humor</a>
+            
+            <a class="tag" href="/tag/obvious/page/1/">obvious</a>
+            
+            <a class="tag" href="/tag/simile/page/1/">simile</a>
+            
+        </div>
+    </div>
+
+    <nav>
+        <ul class="pager">
+            
+            
+            <li class="next">
+                <a href="/page/2/">Next <span aria-hidden="true">&rarr;</span></a>
+            </li>
+            
+        </ul>
+    </nav>
+    </div>
+    <div class="col-md-4 tags-box">
+        
+            <h2>Top Ten tags</h2>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 28px" href="/tag/love/">love</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 26px" href="/tag/inspirational/">inspirational</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 26px" href="/tag/life/">life</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 24px" href="/tag/humor/">humor</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 22px" href="/tag/books/">books</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 14px" href="/tag/reading/">reading</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 10px" href="/tag/friendship/">friendship</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 8px" href="/tag/friends/">friends</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 8px" href="/tag/truth/">truth</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 6px" href="/tag/simile/">simile</a>
+            </span>
+            
+        
+    </div>
+</div>
+
+    </div>
+    <footer class="footer">
+        <div class="container">
+            <p class="text-muted">
+                Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
+            </p>
+            <p class="copyright">
+                Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
+            </p>
+        </div>
+    </footer>
+</body>
+</html>
\ No newline at end of file
diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index bf14643be..e67ce55f9 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -39,7 +39,7 @@ Therefore, you should keep in mind the following things:
 .. _topics-inspector:
 
 Inspecting a website
-===================================
+====================
 
 By far the most handy feature of the Developer Tools is the `Inspector` 
 feature, which allows you to inspect the underlying HTML code of 
@@ -79,13 +79,23 @@ sections and tags of a webpage, which greatly improves readability. You can
 expand and collapse a tag by clicking on the arrow in front of it or by double
 clicking directly on the tag. If we expand the ``span`` tag with the ``class=
 "text"`` we will see the quote-text we clicked on. The `Inspector` lets you
-copy XPaths to selected elements. Let's try it out: Right-click on the ``span`` 
-tag, select ``Copy > XPath`` and paste it in the scrapy shell like so::
+copy XPaths to selected elements. Let's try it out.
+
+First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal:
+
+.. code-block:: none
 
     $ scrapy shell "http://quotes.toscrape.com/"
-    (...)
-    >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
-    ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”]
+
+Then, back to your web browser, right-click on the ``span`` tag, select
+``Copy > XPath`` and paste it in the Scrapy shell like so:
+
+.. invisible-code-block: python
+
+    response = load_response('http://quotes.toscrape.com/', 'quotes.html')
+
+>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
+['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
 
 Adding ``text()`` at the end we are able to extract the first quote with this 
 basic selector. But this XPath is not really that clever. All it does is
@@ -112,13 +122,13 @@ see each quote:
 
 With this knowledge we can refine our XPath: Instead of a path to follow,
 we'll simply select all ``span`` tags with the ``class="text"`` by using 
-the `has-class-extension`_:: 
+the `has-class-extension`_:
 
-    >>> response.xpath('//span[has-class("text")]/text()').getall()
-   ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”,
-    '“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
-    '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
-    (...)]
+>>> response.xpath('//span[has-class("text")]/text()').getall()
+['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
+'“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
+'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
+...]
 
 And with one simple, cleverer XPath we are able to extract all quotes from 
 the page. We could have constructed a loop over our first XPath to increase 
@@ -159,7 +169,11 @@ The page is quite similar to the basic `quotes.toscrape.com`_-page,
 but instead of the above-mentioned ``Next`` button, the page 
 automatically loads new quotes when you scroll to the bottom. We 
 could go ahead and try out different XPaths directly, but instead 
-we'll check another quite useful command from the scrapy shell::
+we'll check another quite useful command from the scrapy shell:
+
+.. skip: next
+
+.. code-block:: none
 
   $ scrapy shell "quotes.toscrape.com/scroll"
   (...)
diff --git a/pytest.ini b/pytest.ini
index 33c34b8e8..0e830076d 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -8,7 +8,6 @@ addopts =
     --ignore=docs/_ext
     --ignore=docs/conf.py
     --ignore=docs/news.rst
-    --ignore=docs/topics/developer-tools.rst
     --ignore=docs/topics/dynamic-content.rst
     --ignore=docs/topics/items.rst
     --ignore=docs/topics/leaks.rst

From ed1e577610b77e92f9833ca96755b9f0727085f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 25 Nov 2019 13:35:29 +0100
Subject: [PATCH 2439/2974] Use super().__init__ in BaseItemExporter subclasses

---
 scrapy/exporters.py | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 3defafd60..0c28c2d7f 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -24,8 +24,9 @@ __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
 
 class BaseItemExporter(object):
 
-    def __init__(self, **kwargs):
-        self._configure(kwargs)
+    def __init__(self, dont_fail=False, **kwargs):
+        self._kwargs = kwargs
+        self._configure(kwargs, dont_fail=dont_fail)
 
     def _configure(self, options, dont_fail=False):
         """Configure the exporter by poping options from the ``options`` dict.
@@ -82,10 +83,10 @@ class BaseItemExporter(object):
 class JsonLinesItemExporter(BaseItemExporter):
 
     def __init__(self, file, **kwargs):
-        self._configure(kwargs, dont_fail=True)
+        super().__init__(dont_fail=True, **kwargs)
         self.file = file
-        kwargs.setdefault('ensure_ascii', not self.encoding)
-        self.encoder = ScrapyJSONEncoder(**kwargs)
+        self._kwargs.setdefault('ensure_ascii', not self.encoding)
+        self.encoder = ScrapyJSONEncoder(**self._kwargs)
 
     def export_item(self, item):
         itemdict = dict(self._get_serialized_fields(item))
@@ -96,15 +97,15 @@ class JsonLinesItemExporter(BaseItemExporter):
 class JsonItemExporter(BaseItemExporter):
 
     def __init__(self, file, **kwargs):
-        self._configure(kwargs, dont_fail=True)
+        super().__init__(dont_fail=True, **kwargs)
         self.file = file
         # there is a small difference between the behaviour or JsonItemExporter.indent
         # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent
         # the addition of newlines everywhere
         json_indent = self.indent if self.indent is not None and self.indent > 0 else None
-        kwargs.setdefault('indent', json_indent)
-        kwargs.setdefault('ensure_ascii', not self.encoding)
-        self.encoder = ScrapyJSONEncoder(**kwargs)
+        self._kwargs.setdefault('indent', json_indent)
+        self._kwargs.setdefault('ensure_ascii', not self.encoding)
+        self.encoder = ScrapyJSONEncoder(**self._kwargs)
         self.first_item = True
 
     def _beautify_newline(self):
@@ -135,7 +136,7 @@ class XmlItemExporter(BaseItemExporter):
     def __init__(self, file, **kwargs):
         self.item_element = kwargs.pop('item_element', 'item')
         self.root_element = kwargs.pop('root_element', 'items')
-        self._configure(kwargs)
+        super().__init__(**kwargs)
         if not self.encoding:
             self.encoding = 'utf-8'
         self.xg = XMLGenerator(file, encoding=self.encoding)
@@ -191,7 +192,7 @@ class XmlItemExporter(BaseItemExporter):
 class CsvItemExporter(BaseItemExporter):
 
     def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs):
-        self._configure(kwargs, dont_fail=True)
+        super().__init__(dont_fail=True, **kwargs)
         if not self.encoding:
             self.encoding = 'utf-8'
         self.include_headers_line = include_headers_line
@@ -202,7 +203,7 @@ class CsvItemExporter(BaseItemExporter):
             encoding=self.encoding,
             newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
         )
-        self.csv_writer = csv.writer(self.stream, **kwargs)
+        self.csv_writer = csv.writer(self.stream, **self._kwargs)
         self._headers_not_written = True
         self._join_multivalued = join_multivalued
 
@@ -251,7 +252,7 @@ class CsvItemExporter(BaseItemExporter):
 class PickleItemExporter(BaseItemExporter):
 
     def __init__(self, file, protocol=2, **kwargs):
-        self._configure(kwargs)
+        super().__init__(**kwargs)
         self.file = file
         self.protocol = protocol
 
@@ -270,7 +271,7 @@ class MarshalItemExporter(BaseItemExporter):
     """
 
     def __init__(self, file, **kwargs):
-        self._configure(kwargs)
+        super().__init__(**kwargs)
         self.file = file
 
     def export_item(self, item):
@@ -280,7 +281,7 @@ class MarshalItemExporter(BaseItemExporter):
 class PprintItemExporter(BaseItemExporter):
 
     def __init__(self, file, **kwargs):
-        self._configure(kwargs)
+        super().__init__(**kwargs)
         self.file = file
 
     def export_item(self, item):

From dd12f5fdcd5ce2c63e9a5d3626031f5d93c1628e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 25 Nov 2019 15:59:59 +0100
Subject: [PATCH 2440/2974] Use Response.follow_all in the documentation where
 appropiate

---
 docs/intro/tutorial.rst | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 6b15a5fbd..a2775e0bb 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -625,12 +625,12 @@ attribute automatically. So the code can be shortened further::
     for a in response.css('li.next a'):
         yield response.follow(a, callback=self.parse)
 
-.. note::
+To create multiple requests from an iterable, you can use
+:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` instead::
+
+    links = response.css('li.next a')
+    yield from response.follow_all(links, callback=self.parse)
 
-    ``response.follow(response.css('li.next a'))`` is not valid because
-    ``response.css`` returns a list-like object with selectors for all results,
-    not a single selector. A ``for`` loop like in the example above, or
-    ``response.follow(response.css('li.next a')[0])`` is fine.
 
 More examples and patterns
 --------------------------
@@ -647,13 +647,11 @@ this time for scraping author information::
         start_urls = ['http://quotes.toscrape.com/']
 
         def parse(self, response):
-            # follow links to author pages
-            for href in response.css('.author + a::attr(href)'):
-                yield response.follow(href, self.parse_author)
+            author_page_links = response.css('.author + a')
+            yield from response.follow_all(author_links, self.parse_author)
 
-            # follow pagination links
-            for href in response.css('li.next a::attr(href)'):
-                yield response.follow(href, self.parse)
+            pagination_links = response.css('li.next a')
+            yield from response.follow_all(pagination_links, self.parse)
 
         def parse_author(self, response):
             def extract_with_css(query):
@@ -669,8 +667,10 @@ This spider will start from the main page, it will follow all the links to the
 authors pages calling the ``parse_author`` callback for each of them, and also
 the pagination links with the ``parse`` callback as we saw before.
 
-Here we're passing callbacks to ``response.follow`` as positional arguments
-to make the code shorter; it also works for ``scrapy.Request``.
+Here we're passing callbacks to
+:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` as positional
+arguments to make the code shorter; it also works for
+:class:`~scrapy.http.Request`.
 
 The ``parse_author`` callback defines a helper function to extract and cleanup the
 data from a CSS query and yields the Python dict with the author data.

From b73fc99b60ed83be403e9570e84f5267d35dcc9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 26 Nov 2019 10:31:55 +0100
Subject: [PATCH 2441/2974] Use InterSphinx for coverage links

---
 docs/conf.py          | 1 +
 docs/contributing.rst | 5 ++---
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index eab366efd..914d1d05f 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -275,6 +275,7 @@ coverage_ignore_pyobjects = [
 # -------------------------------------
 
 intersphinx_mapping = {
+    'coverage': ('https://coverage.readthedocs.io/en/stable', None),
     'pytest': ('https://docs.pytest.org/en/latest', None),
     'python': ('https://docs.python.org/3', None),
     'sphinx': ('https://www.sphinx-doc.org/en/master', None),
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 81bb50a77..234c4bcee 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -243,14 +243,13 @@ the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
 
     tox -e py36 -- scrapy tests -n auto
 
-To see coverage report install `coverage`_ (``pip install coverage``) and run:
+To see coverage report install :doc:`coverage <coverage:index>`
+(``pip install coverage``) and run:
 
     ``coverage report``
 
 see output of ``coverage --help`` for more options like html or xml report.
 
-.. _coverage: https://pypi.python.org/pypi/coverage
-
 Writing tests
 -------------
 

From 63546cbf3e02380819732441ad55d95725dca7c5 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 27 Nov 2019 22:42:52 +0500
Subject: [PATCH 2442/2974] Deprecate the HTTPS proxy noconnect mode.

---
 scrapy/core/downloader/handlers/http11.py |  7 ++++++
 tests/test_downloader_handlers.py         | 10 ---------
 tests/test_proxy_connect.py               | 26 -----------------------
 3 files changed, 7 insertions(+), 36 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 7d917cb74..782eca89e 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -16,6 +16,7 @@ from twisted.web.http import _DataLoss, PotentialDataLoss
 from twisted.web.client import Agent, ResponseDone, HTTPConnectionPool, ResponseFailed, URI
 from twisted.internet.endpoints import TCP4ClientEndpoint
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.core.downloader.webclient import _parse
@@ -285,6 +286,12 @@ class ScrapyAgent(object):
             scheme = _parse(request.url)[0]
             proxyHost = to_unicode(proxyHost)
             omitConnectTunnel = b'noconnect' in proxyParams
+            if omitConnectTunnel:
+                warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
+                              "If you use Crawlera, it doesn't require this mode anymore, "
+                              "so you should update scrapy-crawlera to 1.3.0+ "
+                              "and remove '?noconnect' from the Crawlera URL.",
+                              ScrapyDeprecationWarning)
             if scheme == b'https' and not omitConnectTunnel:
                 proxyAuth = request.headers.get(b'Proxy-Authorization', None)
                 proxyConf = (proxyHost, proxyPort, proxyAuth)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 60124b93f..45d4aa952 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -687,16 +687,6 @@ class HttpProxyTestCase(unittest.TestCase):
         request = Request('http://example.com', meta={'proxy': http_proxy})
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
-    def test_download_with_proxy_https_noconnect(self):
-        def _test(response):
-            self.assertEqual(response.status, 200)
-            self.assertEqual(response.url, request.url)
-            self.assertEqual(response.body, b'https://example.com')
-
-        http_proxy = '%s?noconnect' % self.getURL('')
-        request = Request('https://example.com', meta={'proxy': http_proxy})
-        return self.download_request(request, Spider('foo')).addCallback(_test)
-
     def test_download_without_proxy(self):
         def _test(response):
             self.assertEqual(response.status, 200)
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 277455751..05d371b65 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -108,32 +108,6 @@ class ProxyConnectTestCase(TestCase):
         echo = json.loads(crawler.spider.meta['responses'][0].text)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
-    # The noconnect mode isn't supported by the current mitmproxy, it returns
-    # "Invalid request scheme: https" as it doesn't seem to support full URLs in GET at all,
-    # and it's not clear what behavior is intended by Scrapy and by mitmproxy here.
-    # https://github.com/mitmproxy/mitmproxy/issues/848 may be related.
-    # The Scrapy noconnect mode was required, at least in the past, to work with Crawlera,
-    # and https://github.com/scrapy-plugins/scrapy-crawlera/pull/44 seems to be related.
-
-    @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
-    @defer.inlineCallbacks
-    def test_https_noconnect(self):
-        proxy = os.environ['https_proxy']
-        os.environ['https_proxy'] = proxy + '?noconnect'
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
-
-    @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
-    @defer.inlineCallbacks
-    def test_https_noconnect_auth_error(self):
-        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(407, l)
-
     def _assert_got_response_code(self, code, log):
         print(log)
         self.assertEqual(str(log).count('Crawled (%d)' % code), 1)

From 17e648182332a1d231383c7416e08e89280cb1d0 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 27 Nov 2019 18:42:42 -0300
Subject: [PATCH 2443/2974] [Docs] Fix Twisted links

---
 docs/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index eab366efd..a79f3a8cb 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -279,7 +279,7 @@ intersphinx_mapping = {
     'python': ('https://docs.python.org/3', None),
     'sphinx': ('https://www.sphinx-doc.org/en/master', None),
     'tox': ('https://tox.readthedocs.io/en/latest', None),
-    'twisted': ('https://twistedmatrix.com/documents/current', None),
+    'twisted': ('https://twistedmatrix.com/documents/current/api', None),
 }
 
 

From 048cd74ae594f449ba97d07c927d7640f32a6770 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 27 Nov 2019 19:16:18 -0300
Subject: [PATCH 2444/2974] Add separate mapping for Twisted API docs

---
 docs/conf.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index a79f3a8cb..40e69c8ac 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -279,7 +279,8 @@ intersphinx_mapping = {
     'python': ('https://docs.python.org/3', None),
     'sphinx': ('https://www.sphinx-doc.org/en/master', None),
     'tox': ('https://tox.readthedocs.io/en/latest', None),
-    'twisted': ('https://twistedmatrix.com/documents/current/api', None),
+    'twisted': ('https://twistedmatrix.com/documents/current', None),
+    'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
 }
 
 

From d1cdfb47013330b0391a8db3b6b812697ee64b6a Mon Sep 17 00:00:00 2001
From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com>
Date: Fri, 29 Nov 2019 19:13:57 +1100
Subject: [PATCH 2445/2974] Use pprint.pformat on overridden settings (#4199)

Keeps consistency with scrapy.middleware
---
 scrapy/crawler.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index f8c80880a..19b61dc7e 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,3 +1,4 @@
+import pprint
 import six
 import signal
 import logging
@@ -45,7 +46,8 @@ class Crawler(object):
         logging.root.addHandler(handler)
 
         d = dict(overridden_settings(self.settings))
-        logger.info("Overridden settings: %(settings)r", {'settings': d})
+        logger.info("Overridden settings:\n%(settings)s",
+                    {'settings': pprint.pformat(d)})
 
         if get_scrapy_root_handler() is not None:
             # scrapy root handler already installed: update it with new settings

From 5980b0f2840f51f8f9c7c3a266be93527999dd11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 2 Dec 2019 16:47:44 +0100
Subject: [PATCH 2446/2974] =?UTF-8?q?Don=E2=80=99t=20use=20follow=5Fall=20?=
 =?UTF-8?q?where=20a=20single=20item=20is=20expected=20(#4)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/intro/tutorial.rst | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index a2775e0bb..2f97017fc 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -616,20 +616,19 @@ instance; you still have to yield this Request.
 You can also pass a selector to ``response.follow`` instead of a string;
 this selector should extract necessary attributes::
 
-    for href in response.css('li.next a::attr(href)'):
-        yield response.follow(href, callback=self.parse)
+    href = response.css('li.next a::attr(href)')[0]
+    yield response.follow(href, callback=self.parse)
 
 For ``<a>`` elements there is a shortcut: ``response.follow`` uses their href
 attribute automatically. So the code can be shortened further::
 
-    for a in response.css('li.next a'):
-        yield response.follow(a, callback=self.parse)
+    a = response.css('li.next a')[0]
+    yield response.follow(a, callback=self.parse)
 
 To create multiple requests from an iterable, you can use
 :meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` instead::
 
-    links = response.css('li.next a')
-    yield from response.follow_all(links, callback=self.parse)
+    yield from response.follow_all(response.css('a'), callback=self.parse)
 
 
 More examples and patterns

From 3d77f74e4089c1a9700fb9e2bc62fc196176250c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 5 Nov 2019 00:54:46 -0300
Subject: [PATCH 2447/2974] Download handlers: from_crawler factory method,
 take crawler instead of settings in __init__

---
 scrapy/core/downloader/handlers/__init__.py |  8 ++-
 scrapy/core/downloader/handlers/datauri.py  |  3 --
 scrapy/core/downloader/handlers/file.py     |  4 +-
 scrapy/core/downloader/handlers/ftp.py      | 13 +++--
 scrapy/core/downloader/handlers/http10.py   | 20 +++++--
 scrapy/core/downloader/handlers/http11.py   | 12 +++--
 scrapy/core/downloader/handlers/s3.py       | 15 +++---
 tests/test_downloader_handlers.py           | 58 +++++++++++++--------
 8 files changed, 86 insertions(+), 47 deletions(-)

diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 0b55d32fa..e8beb2f5a 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -5,7 +5,7 @@ from twisted.internet import defer
 import six
 from scrapy.exceptions import NotSupported, NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
 from scrapy import signals
 
@@ -48,7 +48,11 @@ class DownloadHandlers(object):
             dhcls = load_object(path)
             if skip_lazy and getattr(dhcls, 'lazy', True):
                 return None
-            dh = dhcls(self._crawler.settings)
+            dh = create_instance(
+                dhcls,
+                self._crawler.settings,
+                self._crawler,
+            )
         except NotConfigured as ex:
             self._notconfigured[scheme] = str(ex)
             return None
diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py
index 9e5020753..97134e618 100644
--- a/scrapy/core/downloader/handlers/datauri.py
+++ b/scrapy/core/downloader/handlers/datauri.py
@@ -8,9 +8,6 @@ from scrapy.utils.decorators import defers
 class DataURIDownloadHandler(object):
     lazy = False
 
-    def __init__(self, settings):
-        super(DataURIDownloadHandler, self).__init__()
-
     @defers
     def download_request(self, request, spider):
         uri = parse_data_uri(request.url)
diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py
index 23f25d28d..d445ba2e1 100644
--- a/scrapy/core/downloader/handlers/file.py
+++ b/scrapy/core/downloader/handlers/file.py
@@ -1,4 +1,5 @@
 from w3lib.url import file_uri_to_path
+
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.decorators import defers
 
@@ -6,9 +7,6 @@ from scrapy.utils.decorators import defers
 class FileDownloadHandler(object):
     lazy = False
 
-    def __init__(self, settings):
-        pass
-
     @defers
     def download_request(self, request, spider):
         filepath = file_uri_to_path(request.url)
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 39ed67a1a..7a98361ed 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -59,6 +59,7 @@ class ReceivedDataProtocol(Protocol):
     def close(self):
         self.body.close() if self.filename else self.body.seek(0)
 
+
 _CODE_RE = re.compile(r"\d+")
 
 
@@ -70,10 +71,14 @@ class FTPDownloadHandler(object):
         "default": 503,
     }
 
-    def __init__(self, settings):
-        self.default_user = settings['FTP_USER']
-        self.default_password = settings['FTP_PASSWORD']
-        self.passive_mode = settings['FTP_PASSIVE_MODE']
+    def __init__(self, crawler):
+        self.default_user = crawler.settings['FTP_USER']
+        self.default_password = crawler.settings['FTP_PASSWORD']
+        self.passive_mode = crawler.settings['FTP_PASSIVE_MODE']
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
 
     def download_request(self, request, spider):
         parsed_url = urlparse_cached(request)
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index be7298531..ce0801bcc 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -1,6 +1,7 @@
 """Download handlers for http and https schemes
 """
 from twisted.internet import reactor
+
 from scrapy.utils.misc import load_object, create_instance
 from scrapy.utils.python import to_unicode
 
@@ -8,10 +9,15 @@ from scrapy.utils.python import to_unicode
 class HTTP10DownloadHandler(object):
     lazy = False
 
-    def __init__(self, settings):
-        self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
-        self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
-        self._settings = settings
+    def __init__(self, crawler):
+        self.HTTPClientFactory = load_object(crawler.settings['DOWNLOADER_HTTPCLIENTFACTORY'])
+        self.ClientContextFactory = load_object(crawler.settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+        self._crawler = crawler
+        self._settings = crawler.settings
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
 
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
@@ -22,7 +28,11 @@ class HTTP10DownloadHandler(object):
     def _connect(self, factory):
         host, port = to_unicode(factory.host), factory.port
         if factory.scheme == b'https':
-            client_context_factory = create_instance(self.ClientContextFactory, settings=self._settings, crawler=None)
+            client_context_factory = create_instance(
+                self.ClientContextFactory,
+                settings=self._settings,
+                crawler=self._crawler,
+            )
             return reactor.connectSSL(host, port, factory, client_context_factory)
         else:
             return reactor.connectTCP(host, port, factory)
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 7d917cb74..b424a7999 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -30,7 +30,9 @@ logger = logging.getLogger(__name__)
 class HTTP11DownloadHandler(object):
     lazy = False
 
-    def __init__(self, settings):
+    def __init__(self, crawler):
+        settings = crawler.settings
+
         self._pool = HTTPConnectionPool(reactor, persistent=True)
         self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self._pool._factory.noisy = False
@@ -42,7 +44,7 @@ class HTTP11DownloadHandler(object):
             self._contextFactory = create_instance(
                 self._contextFactoryClass,
                 settings=settings,
-                crawler=None,
+                crawler=crawler,
                 method=self._sslMethod,
             )
         except TypeError:
@@ -50,7 +52,7 @@ class HTTP11DownloadHandler(object):
             self._contextFactory = create_instance(
                 self._contextFactoryClass,
                 settings=settings,
-                crawler=None,
+                crawler=crawler,
             )
             msg = """
  '%s' does not accept `method` argument (type OpenSSL.SSL method,\
@@ -63,6 +65,10 @@ class HTTP11DownloadHandler(object):
         self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
         self._disconnect_timeout = 1
 
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
         agent = ScrapyAgent(
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 808d1bf21..220296fb3 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -32,13 +32,12 @@ def _get_boto_connection():
 
 class S3DownloadHandler(object):
 
-    def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
-            httpdownloadhandler=HTTPDownloadHandler, **kw):
-
+    def __init__(self, crawler, aws_access_key_id=None, aws_secret_access_key=None,
+                 httpdownloadhandler=HTTPDownloadHandler, **kw):
         if not aws_access_key_id:
-            aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
+            aws_access_key_id = crawler.settings['AWS_ACCESS_KEY_ID']
         if not aws_secret_access_key:
-            aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
+            aws_secret_access_key = crawler.settings['AWS_SECRET_ACCESS_KEY']
 
         # If no credentials could be found anywhere,
         # consider this an anonymous connection request by default;
@@ -67,7 +66,11 @@ class S3DownloadHandler(object):
             except Exception as ex:
                 raise NotConfigured(str(ex))
 
-        self._download_http = httpdownloadhandler(settings).download_request
+        self._download_http = httpdownloadhandler(crawler).download_request
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
 
     def download_request(self, request, spider):
         p = urlparse_cached(request)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 60124b93f..815342219 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -30,7 +30,6 @@ from scrapy.spiders import Spider
 from scrapy.http import Headers, Request
 from scrapy.http.response.text import TextResponse
 from scrapy.responsetypes import responsetypes
-from scrapy.settings import Settings
 from scrapy.utils.test import get_crawler, skip_if_no_boto
 from scrapy.utils.python import to_bytes
 from scrapy.exceptions import NotConfigured
@@ -45,6 +44,10 @@ class DummyDH(object):
     def __init__(self, crawler):
         pass
 
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
 
 class DummyLazyDH(object):
     # Default is lazy for backward compatibility
@@ -52,6 +55,10 @@ class DummyLazyDH(object):
     def __init__(self, crawler):
         pass
 
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
 
 class OffDH(object):
     lazy = False
@@ -59,6 +66,10 @@ class OffDH(object):
     def __init__(self, crawler):
         raise NotConfigured
 
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
 
 class LoadTestCase(unittest.TestCase):
 
@@ -106,7 +117,7 @@ class FileTestCase(unittest.TestCase):
         self.tmpname = self.mktemp()
         with open(self.tmpname + '^', 'w') as f:
             f.write('0123456789')
-        self.download_request = FileDownloadHandler(Settings()).download_request
+        self.download_request = FileDownloadHandler().download_request
 
     def tearDown(self):
         os.unlink(self.tmpname + '^')
@@ -239,7 +250,7 @@ class HttpTestCase(unittest.TestCase):
         else:
             self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
         self.portno = self.port.getHost().port
-        self.download_handler = self.download_handler_cls(Settings())
+        self.download_handler = self.download_handler_cls(get_crawler())
         self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks
@@ -479,9 +490,9 @@ class Http11TestCase(HttpTestCase):
         return self.test_download_broken_content_allow_data_loss('broken-chunked')
 
     def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
-        download_handler = self.download_handler_cls(Settings({
-            'DOWNLOAD_FAIL_ON_DATALOSS': False,
-        }))
+        download_handler = self.download_handler_cls(
+            get_crawler(settings_dict={'DOWNLOAD_FAIL_ON_DATALOSS': False})
+        )
         request = Request(self.getURL(url))
         d = download_handler.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.flags)
@@ -499,9 +510,9 @@ class Https11TestCase(Http11TestCase):
 
     @defer.inlineCallbacks
     def test_tls_logging(self):
-        download_handler = self.download_handler_cls(Settings({
-            'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True,
-        }))
+        download_handler = self.download_handler_cls(
+            get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True})
+        )
         try:
             with LogCapture() as log_capture:
                 request = Request(self.getURL('file'))
@@ -569,7 +580,8 @@ class Https11CustomCiphers(unittest.TestCase):
             interface=self.host)
         self.portno = self.port.getHost().port
         self.download_handler = self.download_handler_cls(
-            Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'}))
+            get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})
+        )
         self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks
@@ -665,7 +677,7 @@ class HttpProxyTestCase(unittest.TestCase):
         wrapper = WrappingFactory(site)
         self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')
         self.portno = self.port.getHost().port
-        self.download_handler = self.download_handler_cls(Settings())
+        self.download_handler = self.download_handler_cls(get_crawler())
         self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks
@@ -738,9 +750,10 @@ class S3AnonTestCase(unittest.TestCase):
 
     def setUp(self):
         skip_if_no_boto()
-        self.s3reqh = S3DownloadHandler(Settings(),
-                httpdownloadhandler=HttpDownloadHandlerMock,
-                #anon=True, # is implicit
+        self.s3reqh = S3DownloadHandler(
+            crawler=get_crawler(),
+            httpdownloadhandler=HttpDownloadHandlerMock,
+            #anon=True, # is implicit
         )
         self.download_request = self.s3reqh.download_request
         self.spider = Spider('foo')
@@ -766,9 +779,12 @@ class S3TestCase(unittest.TestCase):
 
     def setUp(self):
         skip_if_no_boto()
-        s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID,
-                self.AWS_SECRET_ACCESS_KEY,
-                httpdownloadhandler=HttpDownloadHandlerMock)
+        s3reqh = S3DownloadHandler(
+            get_crawler(),
+            self.AWS_ACCESS_KEY_ID,
+            self.AWS_SECRET_ACCESS_KEY,
+            httpdownloadhandler=HttpDownloadHandlerMock,
+        )
         self.download_request = s3reqh.download_request
         self.spider = Spider('foo')
 
@@ -788,7 +804,7 @@ class S3TestCase(unittest.TestCase):
 
     def test_extra_kw(self):
         try:
-            S3DownloadHandler(Settings(), extra_kw=True)
+            S3DownloadHandler(get_crawler(), extra_kw=True)
         except Exception as e:
             self.assertIsInstance(e, (TypeError, NotConfigured))
         else:
@@ -928,7 +944,7 @@ class BaseFTPTestCase(unittest.TestCase):
         self.factory = FTPFactory(portal=p)
         self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
         self.portNum = self.port.getHost().port
-        self.download_handler = FTPDownloadHandler(Settings())
+        self.download_handler = FTPDownloadHandler(get_crawler())
         self.addCleanup(self.port.stopListening)
 
     def tearDown(self):
@@ -1042,7 +1058,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
                                   userAnonymous=self.username)
         self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
         self.portNum = self.port.getHost().port
-        self.download_handler = FTPDownloadHandler(Settings())
+        self.download_handler = FTPDownloadHandler(get_crawler())
         self.addCleanup(self.port.stopListening)
 
     def tearDown(self):
@@ -1052,7 +1068,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
 class DataURITestCase(unittest.TestCase):
 
     def setUp(self):
-        self.download_handler = DataURIDownloadHandler(Settings())
+        self.download_handler = DataURIDownloadHandler()
         self.download_request = self.download_handler.download_request
         self.spider = Spider('foo')
 

From e43f37fff3cbdb5c4de7af21594c6062859a50e0 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 5 Nov 2019 16:18:42 -0300
Subject: [PATCH 2448/2974] Pass args/kwargs in S3DownloadHandler.from_crawler,
 update tests

---
 scrapy/core/downloader/handlers/s3.py |  4 ++--
 tests/test_downloader_handlers.py     | 22 +++++++++++-----------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 220296fb3..99a3a7925 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -69,8 +69,8 @@ class S3DownloadHandler(object):
         self._download_http = httpdownloadhandler(crawler).download_request
 
     @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler)
+    def from_crawler(cls, crawler, *args, **kwargs):
+        return cls(crawler, *args, **kwargs)
 
     def download_request(self, request, spider):
         p = urlparse_cached(request)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 815342219..eff5653f0 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -250,7 +250,7 @@ class HttpTestCase(unittest.TestCase):
         else:
             self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
         self.portno = self.port.getHost().port
-        self.download_handler = self.download_handler_cls(get_crawler())
+        self.download_handler = self.download_handler_cls.from_crawler(get_crawler())
         self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks
@@ -490,7 +490,7 @@ class Http11TestCase(HttpTestCase):
         return self.test_download_broken_content_allow_data_loss('broken-chunked')
 
     def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
-        download_handler = self.download_handler_cls(
+        download_handler = self.download_handler_cls.from_crawler(
             get_crawler(settings_dict={'DOWNLOAD_FAIL_ON_DATALOSS': False})
         )
         request = Request(self.getURL(url))
@@ -510,7 +510,7 @@ class Https11TestCase(Http11TestCase):
 
     @defer.inlineCallbacks
     def test_tls_logging(self):
-        download_handler = self.download_handler_cls(
+        download_handler = self.download_handler_cls.from_crawler(
             get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True})
         )
         try:
@@ -579,7 +579,7 @@ class Https11CustomCiphers(unittest.TestCase):
             0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),
             interface=self.host)
         self.portno = self.port.getHost().port
-        self.download_handler = self.download_handler_cls(
+        self.download_handler = self.download_handler_cls.from_crawler(
             get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})
         )
         self.download_request = self.download_handler.download_request
@@ -677,7 +677,7 @@ class HttpProxyTestCase(unittest.TestCase):
         wrapper = WrappingFactory(site)
         self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')
         self.portno = self.port.getHost().port
-        self.download_handler = self.download_handler_cls(get_crawler())
+        self.download_handler = self.download_handler_cls.from_crawler(get_crawler())
         self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks
@@ -750,7 +750,7 @@ class S3AnonTestCase(unittest.TestCase):
 
     def setUp(self):
         skip_if_no_boto()
-        self.s3reqh = S3DownloadHandler(
+        self.s3reqh = S3DownloadHandler.from_crawler(
             crawler=get_crawler(),
             httpdownloadhandler=HttpDownloadHandlerMock,
             #anon=True, # is implicit
@@ -779,10 +779,10 @@ class S3TestCase(unittest.TestCase):
 
     def setUp(self):
         skip_if_no_boto()
-        s3reqh = S3DownloadHandler(
-            get_crawler(),
-            self.AWS_ACCESS_KEY_ID,
-            self.AWS_SECRET_ACCESS_KEY,
+        s3reqh = S3DownloadHandler.from_crawler(
+            crawler=get_crawler(),
+            aws_access_key_id=self.AWS_ACCESS_KEY_ID,
+            aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
             httpdownloadhandler=HttpDownloadHandlerMock,
         )
         self.download_request = s3reqh.download_request
@@ -804,7 +804,7 @@ class S3TestCase(unittest.TestCase):
 
     def test_extra_kw(self):
         try:
-            S3DownloadHandler(get_crawler(), extra_kw=True)
+            S3DownloadHandler.from_crawler(get_crawler(), extra_kw=True)
         except Exception as e:
             self.assertIsInstance(e, (TypeError, NotConfigured))
         else:

From 2a9f5a0aefae83fc0b1dc161d117a265148ad1ef Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 3 Dec 2019 15:56:50 -0300
Subject: [PATCH 2449/2974] Skip invalid links when passing SelectorLists to
 Response.follow_all

---
 scrapy/http/response/text.py | 19 +++++++++++--------
 tests/test_http_response.py  | 12 ++++++++----
 2 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 6acf1026f..e3646b2d5 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -7,10 +7,10 @@ See documentation in docs/topics/request-response.rst
 
 from contextlib import suppress
 from typing import Generator
+from urllib.parse import urljoin
 
 import parsel
 import six
-from six.moves.urllib.parse import urljoin
 from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
                             http_content_type_encoding, resolve_encoding)
 from w3lib.html import strip_html5_whitespace
@@ -183,22 +183,25 @@ class TextResponse(Response):
         In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction
         within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted).
 
-        Note that when using the ``css`` or ``xpath`` parameters, this method will not produce
-        requests for selectors from which links cannot be obtained (for instance, anchor tags
-        without an ``href`` attribute)
+        Note that when passing a ``SelectorList`` as argument for the ``urls`` parameter or
+        using the ``css`` or ``xpath`` parameters, this method will not produce requests for
+        selectors from which links cannot be obtained (for instance, anchor tags without an
+        ``href`` attribute)
         """
         arg_count = len(list(filter(None, (urls, css, xpath))))
         if arg_count != 1:
             raise ValueError('Please supply exactly one of the following arguments: urls, css, xpath')
         if not urls:
             if css:
-                selector_list = self.css(css)
+                urls = self.css(css)
             if xpath:
-                selector_list = self.xpath(xpath)
+                urls = self.xpath(xpath)
+        if isinstance(urls, parsel.SelectorList):
+            selectors = urls
             urls = []
-            for selector in selector_list:
+            for sel in selectors:
                 with suppress(_InvalidSelector):
-                    urls.append(_url_from_selector(selector))
+                    urls.append(_url_from_selector(sel))
         return super(TextResponse, self).follow_all(
             urls=urls,
             callback=callback,
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 36ccdfa1f..ce13650ce 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -579,8 +579,10 @@ class TextResponseTest(BaseResponseTest):
             'http://example.com/page/4/',
         ]
         response = self._links_response_no_href()
-        extracted = [r.url for r in response.follow_all(css='.pagination a')]
-        self.assertEqual(expected, extracted)
+        extracted1 = [r.url for r in response.follow_all(css='.pagination a')]
+        self.assertEqual(expected, extracted1)
+        extracted2 = [r.url for r in response.follow_all(response.css('.pagination a'))]
+        self.assertEqual(expected, extracted2)
 
     def test_follow_all_xpath(self):
         expected = [
@@ -598,8 +600,10 @@ class TextResponseTest(BaseResponseTest):
             'http://example.com/page/4/',
         ]
         response = self._links_response_no_href()
-        extracted = [r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')]
-        self.assertEqual(expected, extracted)
+        extracted1 = [r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')]
+        self.assertEqual(expected, extracted1)
+        extracted2 = [r.url for r in response.follow_all(response.xpath('//div[@id="pagination"]/a'))]
+        self.assertEqual(expected, extracted2)
 
     def test_follow_all_too_many_arguments(self):
         response = self._links_response()

From 62778cf23f6e0cad5839f31f9da8b3c5778dcbdb Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 11 Sep 2019 15:40:44 -0300
Subject: [PATCH 2450/2974] Request: remove restriction about errback without
 callback

---
 scrapy/http/request/__init__.py |  1 -
 tests/test_http_request.py      | 46 ++++++++++++++++++++++-----------
 2 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 76a428199..61c0a4c9e 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -32,7 +32,6 @@ class Request(object_ref):
             raise TypeError('callback must be a callable, got %s' % type(callback).__name__)
         if errback is not None and not callable(errback):
             raise TypeError('errback must be a callable, got %s' % type(errback).__name__)
-        assert callback or not errback, "Cannot use errback without a callback"
         self.callback = callback
         self.errback = errback
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 9df6ff67b..3449c7a40 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -246,25 +246,41 @@ class RequestTest(unittest.TestCase):
         self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com')
         self.assertRaises(AttributeError, setattr, r, 'body', 'xxx')
 
-    def test_callback_is_callable(self):
+    def test_callback_and_errback(self):
         def a_function():
             pass
-        r = self.request_class('http://example.com')
-        self.assertIsNone(r.callback)
-        r = self.request_class('http://example.com', a_function)
-        self.assertIs(r.callback, a_function)
-        with self.assertRaises(TypeError):
-            self.request_class('http://example.com', 'a_function')
 
-    def test_errback_is_callable(self):
-        def a_function():
-            pass
-        r = self.request_class('http://example.com')
-        self.assertIsNone(r.errback)
-        r = self.request_class('http://example.com', a_function, errback=a_function)
-        self.assertIs(r.errback, a_function)
+        r1 = self.request_class('http://example.com')
+        self.assertIsNone(r1.callback)
+        self.assertIsNone(r1.errback)
+
+        r2 = self.request_class('http://example.com', callback=a_function)
+        self.assertIs(r2.callback, a_function)
+        self.assertIsNone(r2.errback)
+
+        r3 = self.request_class('http://example.com', errback=a_function)
+        self.assertIsNone(r3.callback)
+        self.assertIs(r3.errback, a_function)
+
+        r4 = self.request_class(
+            url='http://example.com',
+            callback=a_function,
+            errback=a_function,
+        )
+        self.assertIs(r4.callback, a_function)
+        self.assertIs(r4.errback, a_function)
+
+    def test_callback_and_errback_type(self):
         with self.assertRaises(TypeError):
-            self.request_class('http://example.com', a_function, errback='a_function')
+            self.request_class('http://example.com', callback='a_function')
+        with self.assertRaises(TypeError):
+            self.request_class('http://example.com', errback='a_function')
+        with self.assertRaises(TypeError):
+            self.request_class(
+                url='http://example.com',
+                callback='a_function',
+                errback='a_function',
+            )
 
     def test_from_curl(self):
         # Note: more curated tests regarding curl conversion are in

From 5d8d4bb7d7d998ae2324c4995bafaafdef752572 Mon Sep 17 00:00:00 2001
From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com>
Date: Thu, 5 Dec 2019 00:22:10 +1100
Subject: [PATCH 2451/2974] Re-arrange the imports in the httpproxy module
 (#4210)

This commit re-arranges the imports in the httpproxy module to follow
pep8
---
 scrapy/downloadermiddlewares/httpproxy.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 2212d9688..5e4542b6c 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -1,7 +1,8 @@
 import base64
+from urllib.request import _parse_proxy
+
 from six.moves.urllib.parse import unquote, urlunparse
 from six.moves.urllib.request import getproxies, proxy_bypass
-from urllib.request import _parse_proxy
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached

From 702333478d072c3c043c64ec7ad3997befb87943 Mon Sep 17 00:00:00 2001
From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com>
Date: Thu, 5 Dec 2019 00:23:28 +1100
Subject: [PATCH 2452/2974] Re-arrange the imports in httpcache module (#4209)

This commit re-arrange the imports in httpcache module to follow pep8
---
 scrapy/downloadermiddlewares/httpcache.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 495b103d1..4e06f8236 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -1,11 +1,19 @@
 from email.utils import formatdate
+
 from twisted.internet import defer
-from twisted.internet.error import TimeoutError, DNSLookupError, \
-        ConnectionRefusedError, ConnectionDone, ConnectError, \
-        ConnectionLost, TCPTimedOutError
+from twisted.internet.error import (
+    ConnectError,
+    ConnectionDone,
+    ConnectionLost,
+    ConnectionRefusedError,
+    DNSLookupError,
+    TCPTimedOutError,
+    TimeoutError,
+)
 from twisted.web.client import ResponseFailed
+
 from scrapy import signals
-from scrapy.exceptions import NotConfigured, IgnoreRequest
+from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.utils.misc import load_object
 
 

From 74627033c4a1701f3e197216f9f2801d497f5535 Mon Sep 17 00:00:00 2001
From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com>
Date: Thu, 5 Dec 2019 00:24:14 +1100
Subject: [PATCH 2453/2974] Remove the used import and re-arrange the imports
 (#4208)

This commit removes unused import and re-arrange the imports in cookies
module
---
 scrapy/downloadermiddlewares/cookies.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 0d2b9900c..aeb7578b8 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -1,8 +1,8 @@
-import os
-import six
 import logging
 from collections import defaultdict
 
+import six
+
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar

From 1b35260625c3ffec9885265d9ac92771ade67ad9 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 25 Jul 2019 18:18:34 +0500
Subject: [PATCH 2454/2974] Add a test for downloader middlewares using
 Deferreds.

---
 tests/test_downloadermiddleware.py | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 6b9a5bee8..1b81ea949 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -1,5 +1,6 @@
 from unittest import mock
 
+from twisted.internet.defer import Deferred
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
 
@@ -177,3 +178,31 @@ class ProcessExceptionInvalidOutput(ManagerTestCase):
         dfd.addBoth(results.append)
         self.assertIsInstance(results[0], Failure)
         self.assertIsInstance(results[0].value, _InvalidOutput)
+
+
+class MiddlewareUsingDeferreds(ManagerTestCase):
+    """Middlewares using Deferreds should work"""
+
+    def test_deferred(self):
+        resp = Response('http://example.com/index.html')
+
+        class DeferredMiddleware:
+            def cb(self, result):
+                return result
+
+            def process_request(self, request, spider):
+                d = Deferred()
+                d.addCallback(self.cb)
+                d.callback(resp)
+                return d
+
+        self.mwman._add_middleware(DeferredMiddleware())
+        req = Request('http://example.com/index.html')
+        download_func = mock.MagicMock()
+        dfd = self.mwman.download(download_func, req, self.spider)
+        results = []
+        dfd.addBoth(results.append)
+        self._wait(dfd)
+
+        self.assertIs(results[0], resp)
+        self.assertFalse(download_func.called)

From 1b437bbe9fa0eb9736f35e510d486805706c783e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 30 Jul 2019 19:02:16 +0500
Subject: [PATCH 2455/2974] Install the asyncio reactor on "import scrapy".

---
 scrapy/__init__.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 230e5cee3..41eaee959 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -23,6 +23,28 @@ import warnings
 warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
 del warnings
 
+# Install twisted asyncio loop
+def _install_asyncio_reactor():
+    global asyncio_supported
+    try:
+        import asyncio
+        from twisted.internet import asyncioreactor
+    except ImportError:
+        pass
+    else:
+        from twisted.internet.error import ReactorAlreadyInstalledError
+        try:
+            asyncioreactor.install(asyncio.get_event_loop())
+            asyncio_supported = True
+        except ReactorAlreadyInstalledError:
+            import twisted.internet.reactor
+            if isinstance(twisted.internet.reactor,
+                              asyncioreactor.AsyncioSelectorReactor):
+                asyncio_supported = True
+asyncio_supported = False
+_install_asyncio_reactor()
+del _install_asyncio_reactor
+
 # Apply monkey patches to fix issues in external libraries
 from . import _monkeypatches
 del _monkeypatches

From 9777639533373951652f3865a6321a4dff73246a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 30 Jul 2019 19:02:59 +0500
Subject: [PATCH 2456/2974] Run tests using the asyncio reactor.

---
 pytest.ini                 | 1 +
 tests/mockserver.py        | 3 +++
 tests/requirements-py3.txt | 3 ++-
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/pytest.ini b/pytest.ini
index 33c34b8e8..6c4c21baf 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -5,6 +5,7 @@ python_classes=
 addopts =
     --assert=plain
     --doctest-modules
+    --reactor=asyncio
     --ignore=docs/_ext
     --ignore=docs/conf.py
     --ignore=docs/news.rst
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 7ebb8bb62..b6aee009a 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -6,6 +6,9 @@ from subprocess import Popen, PIPE
 
 from OpenSSL import SSL
 from six.moves.urllib.parse import urlencode
+
+import scrapy  # needed before importing twisted.internet.reactor
+
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
 from twisted.web.static import File
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index c4bc1f278..26ab08b04 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -4,7 +4,8 @@ mitmproxy; python_version >= '3.6'
 mitmproxy==3.0.4; python_version < '3.6'
 pytest
 pytest-cov
-pytest-twisted
+#pytest-twisted
+-e git+https://github.com/pytest-dev/pytest-twisted@81b91f17#egg=pytest-twisted
 pytest-xdist
 sybil
 testfixtures

From 63c3c62305a8c9c52d02ff5524301b7d45eb5724 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 30 Jul 2019 19:45:56 +0500
Subject: [PATCH 2457/2974] Add utils.deferred_from_coro.

---
 scrapy/utils/defer.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index c5916c21c..1f6a2584c 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -1,10 +1,14 @@
 """
 Helper functions for dealing with Twisted deferreds
 """
+import asyncio
+import asyncio.futures
+import inspect
 
 from twisted.internet import defer, reactor, task
 from twisted.python import failure
 
+from scrapy import asyncio_supported
 from scrapy.exceptions import IgnoreRequest
 
 
@@ -113,3 +117,21 @@ def iter_errback(iterable, errback, *a, **kw):
             break
         except Exception:
             errback(failure.Failure(), *a, **kw)
+
+
+def isfuture(o):
+    # workaround for Python before 3.5.3 not having asyncio.isfuture
+    if hasattr(asyncio, 'isfuture'):
+        return asyncio.isfuture(o)
+    return isinstance(o, asyncio.futures.Future)
+
+
+def deferred_from_coro(o):
+    """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
+    if isinstance(o, defer.Deferred):
+        return o
+    if asyncio.iscoroutine(o) or isfuture(o) or inspect.isawaitable(o):
+        if not asyncio_supported:
+            raise TypeError('Using coroutines requires installing AsyncioSelectorReactor')
+        return defer.Deferred.fromFuture(asyncio.ensure_future(o))
+    return o

From 8d8fbddbde133a94bb8741e48fabec05437a3df9 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 21 Aug 2019 00:07:08 +0500
Subject: [PATCH 2458/2974] Switch to the released version of pytest-twisted.

---
 tests/requirements-py3.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 26ab08b04..2ac434f41 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -4,8 +4,7 @@ mitmproxy; python_version >= '3.6'
 mitmproxy==3.0.4; python_version < '3.6'
 pytest
 pytest-cov
-#pytest-twisted
--e git+https://github.com/pytest-dev/pytest-twisted@81b91f17#egg=pytest-twisted
+pytest-twisted >= 1.11
 pytest-xdist
 sybil
 testfixtures

From b04b541372b219a02bb5fda2cc15cfd9fa1aac66 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 21 Aug 2019 17:14:46 +0500
Subject: [PATCH 2459/2974] Install the asyncio reactor only in scrapy.cmdline.

---
 scrapy/__init__.py      | 22 ----------------------
 scrapy/cmdline.py       |  8 +++++++-
 scrapy/utils/asyncio.py | 26 ++++++++++++++++++++++++++
 scrapy/utils/defer.py   |  5 +++--
 tests/mockserver.py     |  2 --
 5 files changed, 36 insertions(+), 27 deletions(-)
 create mode 100644 scrapy/utils/asyncio.py

diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 41eaee959..230e5cee3 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -23,28 +23,6 @@ import warnings
 warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
 del warnings
 
-# Install twisted asyncio loop
-def _install_asyncio_reactor():
-    global asyncio_supported
-    try:
-        import asyncio
-        from twisted.internet import asyncioreactor
-    except ImportError:
-        pass
-    else:
-        from twisted.internet.error import ReactorAlreadyInstalledError
-        try:
-            asyncioreactor.install(asyncio.get_event_loop())
-            asyncio_supported = True
-        except ReactorAlreadyInstalledError:
-            import twisted.internet.reactor
-            if isinstance(twisted.internet.reactor,
-                              asyncioreactor.AsyncioSelectorReactor):
-                asyncio_supported = True
-asyncio_supported = False
-_install_asyncio_reactor()
-del _install_asyncio_reactor
-
 # Apply monkey patches to fix issues in external libraries
 from . import _monkeypatches
 del _monkeypatches
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 418dc1ac9..d66f0cc2d 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -7,9 +7,9 @@ import inspect
 import pkg_resources
 
 import scrapy
-from scrapy.crawler import CrawlerProcess
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
+from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_supported
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import inside_project, get_project_settings
 from scrapy.utils.python import garbage_collect
@@ -121,6 +121,10 @@ def execute(argv=None, settings=None):
             settings['EDITOR'] = editor
     check_deprecated_settings(settings)
 
+    # needs to be before _get_commands_dict() as that imports the command modules
+    # which may import twisted.internet.reactor
+    install_asyncio_reactor()
+
     inproject = inside_project()
     cmds = _get_commands_dict(settings, inproject)
     cmdname = _pop_command_name(argv)
@@ -142,6 +146,8 @@ def execute(argv=None, settings=None):
     opts, args = parser.parse_args(args=argv[1:])
     _run_print_help(parser, cmd.process_options, args, opts)
 
+    # needs to be after install_asyncio_reactor() as it imports twisted.internet.reactor
+    from scrapy.crawler import CrawlerProcess
     cmd.crawler_process = CrawlerProcess(settings)
     _run_print_help(parser, _run_command, cmd, args, opts)
     sys.exit(cmd.exitcode)
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
new file mode 100644
index 000000000..e9e3bdd88
--- /dev/null
+++ b/scrapy/utils/asyncio.py
@@ -0,0 +1,26 @@
+#coding: utf-8
+
+
+def install_asyncio_reactor():
+    """ Tries to install AsyncioSelectorReactor
+    """
+    try:
+        import asyncio
+        from twisted.internet import asyncioreactor
+    except ImportError:
+        pass
+    else:
+        from twisted.internet.error import ReactorAlreadyInstalledError
+        try:
+            asyncioreactor.install(asyncio.get_event_loop())
+        except ReactorAlreadyInstalledError:
+            pass
+
+
+def is_asyncio_supported():
+    try:
+        import twisted.internet.reactor
+        from twisted.internet import asyncioreactor
+        return isinstance(twisted.internet.reactor, asyncioreactor.AsyncioSelectorReactor)
+    except ImportError:
+        return False
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 1f6a2584c..955fc820a 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -8,8 +8,9 @@ import inspect
 from twisted.internet import defer, reactor, task
 from twisted.python import failure
 
-from scrapy import asyncio_supported
 from scrapy.exceptions import IgnoreRequest
+from scrapy.utils.asyncio import is_asyncio_supported
+
 
 
 def defer_fail(_failure):
@@ -131,7 +132,7 @@ def deferred_from_coro(o):
     if isinstance(o, defer.Deferred):
         return o
     if asyncio.iscoroutine(o) or isfuture(o) or inspect.isawaitable(o):
-        if not asyncio_supported:
+        if not is_asyncio_supported():
             raise TypeError('Using coroutines requires installing AsyncioSelectorReactor')
         return defer.Deferred.fromFuture(asyncio.ensure_future(o))
     return o
diff --git a/tests/mockserver.py b/tests/mockserver.py
index b6aee009a..d09fbc171 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -7,8 +7,6 @@ from subprocess import Popen, PIPE
 from OpenSSL import SSL
 from six.moves.urllib.parse import urlencode
 
-import scrapy  # needed before importing twisted.internet.reactor
-
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
 from twisted.web.static import File

From 2fbe7d49dc084b7770cc4dc6bbe65eb380b5f498 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 21 Aug 2019 17:16:33 +0500
Subject: [PATCH 2460/2974] Log asyncio support on spider start.

---
 scrapy/utils/log.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index e07fb8698..b74b7a4af 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -11,6 +11,7 @@ from twisted.python import log as twisted_log
 import scrapy
 from scrapy.settings import Settings
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.asyncio import is_asyncio_supported
 from scrapy.utils.versions import scrapy_components_versions
 
 
@@ -148,6 +149,8 @@ def log_scrapy_info(settings):
                 {'versions': ", ".join("%s %s" % (name, version)
                     for name, version in scrapy_components_versions()
                     if name != "Scrapy")})
+    if is_asyncio_supported():
+        logger.debug("Asyncio support enabled")
 
 
 class StreamLogger(object):

From cc19ab5439f20ba6995528542cc064ddab86273c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 22 Aug 2019 18:15:02 +0500
Subject: [PATCH 2461/2974] Add tests that check asyncio support.

---
 conftest.py                 |  4 ++++
 tests/test_commands.py      |  7 +++++++
 tests/test_crawler.py       | 20 +++++++++++++++++++-
 tests/test_utils_asyncio.py | 17 +++++++++++++++++
 tox.ini                     |  6 ++++++
 5 files changed, 53 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_utils_asyncio.py

diff --git a/conftest.py b/conftest.py
index d54ce155c..24e31f130 100644
--- a/conftest.py
+++ b/conftest.py
@@ -27,3 +27,7 @@ def pytest_collection_modifyitems(session, config, items):
             items[:] = [item for item in items if isinstance(item, Flake8Item)]
     except ImportError:
         pass
+
+@pytest.fixture()
+def reactor_pytest(request):
+    request.cls.reactor_pytest = request.config.getoption("--reactor")
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 536379170..8aa7ee109 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -9,6 +9,7 @@ from tempfile import mkdtemp
 from contextlib import contextmanager
 from threading import Timer
 
+from pytest import mark
 from twisted.trial import unittest
 
 import scrapy
@@ -178,6 +179,7 @@ class MiscCommandsTest(CommandTest):
         self.assertEqual(0, self.call('list'))
 
 
+@mark.usefixtures('reactor_pytest')
 class RunSpiderCommandTest(CommandTest):
 
     debug_log_spider = """
@@ -295,6 +297,11 @@ class BadSpider(scrapy.Spider):
         self.assertIn("start_requests", log)
         self.assertIn("badspider.py", log)
 
+    def test_asyncio_supported(self):
+        if self.reactor_pytest == 'asyncio':
+            log = self.get_log(self.debug_log_spider)
+            self.assertIn("DEBUG: Asyncio support enabled", log)
+
 
 class BenchCommandTest(CommandTest):
 
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 8eb2389e2..151acb459 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,14 +1,16 @@
 import logging
 import warnings
 
+from pytest import raises, mark
+from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial import unittest
-from pytest import raises
 
 import scrapy
 from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
 from scrapy.settings import Settings, default_settings
 from scrapy.spiderloader import SpiderLoader
+from scrapy.utils.asyncio import is_asyncio_supported
 from scrapy.utils.log import configure_logging, get_scrapy_root_handler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
@@ -203,6 +205,15 @@ class NoRequestsSpider(scrapy.Spider):
         return []
 
 
+class AsyncioSpider(scrapy.Spider):
+    name = 'asyncio'
+
+    def start_requests(self):
+        self.logger.info('Asyncio support: %s', is_asyncio_supported())
+        return []
+
+
+@mark.usefixtures('reactor_pytest')
 class CrawlerRunnerHasSpider(unittest.TestCase):
 
     @defer.inlineCallbacks
@@ -245,3 +256,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
         yield runner.crawl(NoRequestsSpider)
 
         self.assertEqual(runner.bootstrap_failed, True)
+
+    @defer.inlineCallbacks
+    def test_asyncio_supported(self):
+        runner = CrawlerRunner()
+        with LogCapture() as log:
+            yield runner.crawl(AsyncioSpider)
+            log.check_present(('asyncio', 'INFO', 'Asyncio support: %s' % (self.reactor_pytest == 'asyncio')))
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
new file mode 100644
index 000000000..e34d3002a
--- /dev/null
+++ b/tests/test_utils_asyncio.py
@@ -0,0 +1,17 @@
+from unittest import TestCase
+
+from pytest import mark
+
+from scrapy.utils.asyncio import is_asyncio_supported, install_asyncio_reactor
+
+
+@mark.usefixtures('reactor_pytest')
+class AsyncioTest(TestCase):
+
+    def test_is_asyncio_supported(self):
+        # the result should depend only on the pytest --reactor argument
+        self.assertEquals(is_asyncio_supported(), self.reactor_pytest == 'asyncio')
+
+    def test_install_asyncio_reactor(self):
+        # this should do nothing
+        install_asyncio_reactor()
diff --git a/tox.ini b/tox.ini
index fd75d18e2..844956e5f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -106,3 +106,9 @@ deps =
     {[testenv]deps}
     reppy
     robotexclusionrulesparser
+
+[testenv:py38-no-asyncio]
+basepython = python3.8
+deps = {[testenv]deps}
+commands =
+    py.test --cov=scrapy --cov-report= --reactor=default {posargs:scrapy tests}

From f41c2f3874d2f9deac365d633a71f032e1339e3c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 22 Aug 2019 21:24:30 +0500
Subject: [PATCH 2462/2974] Add py38-no-asyncio to Travis.

---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 9f477e860..fdf40fdf1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,6 +25,8 @@ matrix:
       python: 3.8
     - env: TOXENV=py38-extra-deps
       python: 3.8
+    - env: TOXENV=py38-no-asyncio
+      python: 3.8
     - env: TOXENV=docs
       python: 3.6
 install:

From 3ba25ccbd3b456024b1d350645407557c81d73c7 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 8 Nov 2019 00:09:28 +0500
Subject: [PATCH 2463/2974] Don't use asyncio.iscoroutine, as it is True for
 generators.

---
 scrapy/utils/defer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 955fc820a..30163d2fb 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -131,7 +131,7 @@ def deferred_from_coro(o):
     """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
     if isinstance(o, defer.Deferred):
         return o
-    if asyncio.iscoroutine(o) or isfuture(o) or inspect.isawaitable(o):
+    if isfuture(o) or inspect.isawaitable(o):
         if not is_asyncio_supported():
             raise TypeError('Using coroutines requires installing AsyncioSelectorReactor')
         return defer.Deferred.fromFuture(asyncio.ensure_future(o))

From 794cf71806a94ba68238a93e75c396f355159ab5 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 14 Nov 2019 13:27:21 +0500
Subject: [PATCH 2464/2974] Fix or ignore flake8 problems.

---
 pytest.ini              | 2 ++
 scrapy/cmdline.py       | 2 +-
 scrapy/utils/asyncio.py | 3 ---
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 6c4c21baf..8b97237c3 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -116,6 +116,7 @@ flake8-ignore =
     scrapy/spiders/feed.py E501 E261
     scrapy/spiders/sitemap.py E501
     # scrapy/utils
+    scrapy/utils/asyncio.py E501
     scrapy/utils/benchserver.py E501
     scrapy/utils/conf.py E402 E502 E501
     scrapy/utils/console.py E261 E306 E305
@@ -227,6 +228,7 @@ flake8-ignore =
     tests/test_spidermiddleware_output_chain.py E501 W293 E226
     tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121
     tests/test_squeues.py E501 E701 E741
+    tests/test_utils_asyncio.py E501
     tests/test_utils_conf.py E501 E303 E128
     tests/test_utils_curl.py E501
     tests/test_utils_datatypes.py E402 E501 E305
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index d66f0cc2d..213e99bc0 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -9,7 +9,7 @@ import pkg_resources
 import scrapy
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
-from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_supported
+from scrapy.utils.asyncio import install_asyncio_reactor
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import inside_project, get_project_settings
 from scrapy.utils.python import garbage_collect
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
index e9e3bdd88..f732774f1 100644
--- a/scrapy/utils/asyncio.py
+++ b/scrapy/utils/asyncio.py
@@ -1,6 +1,3 @@
-#coding: utf-8
-
-
 def install_asyncio_reactor():
     """ Tries to install AsyncioSelectorReactor
     """

From c079d5002bae90dbd85bee1f61fdc359b9f39d29 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 21 Nov 2019 23:40:16 +0500
Subject: [PATCH 2465/2974] Run tests without asyncio support by default, add
 py35-asyncio and py38-asyncio envs.

---
 pytest.ini |  1 -
 tox.ini    | 10 ++++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 8b97237c3..336ef041d 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -5,7 +5,6 @@ python_classes=
 addopts =
     --assert=plain
     --doctest-modules
-    --reactor=asyncio
     --ignore=docs/_ext
     --ignore=docs/conf.py
     --ignore=docs/news.rst
diff --git a/tox.ini b/tox.ini
index 844956e5f..a4edae439 100644
--- a/tox.ini
+++ b/tox.ini
@@ -107,8 +107,14 @@ deps =
     reppy
     robotexclusionrulesparser
 
-[testenv:py38-no-asyncio]
+[testenv:py35-asyncio]
+basepython = python3.5
+deps = {[testenv]deps}
+commands =
+    py.test --cov=scrapy --cov-report= --reactor=asyncio {posargs:scrapy tests}
+
+[testenv:py38-asyncio]
 basepython = python3.8
 deps = {[testenv]deps}
 commands =
-    py.test --cov=scrapy --cov-report= --reactor=default {posargs:scrapy tests}
+    py.test --cov=scrapy --cov-report= --reactor=asyncio {posargs:scrapy tests}

From ed34ce14c0c06d4539d4fdeb0ad014f4e6fb5b94 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 4 Dec 2019 21:32:16 +0500
Subject: [PATCH 2466/2974] Add the ASYNCIO_SUPPORT setting, reshuffle other
 logic accordingly.

---
 docs/topics/settings.rst            | 25 +++++++++++++++++++++++++
 scrapy/cmdline.py                   |  8 ++------
 scrapy/commands/crawl.py            |  3 +++
 scrapy/commands/runspider.py        |  3 +++
 scrapy/settings/default_settings.py |  2 ++
 scrapy/utils/asyncio.py             |  2 +-
 scrapy/utils/defer.py               | 17 +++++++++++------
 scrapy/utils/log.py                 | 11 ++++++++---
 tests/test_commands.py              | 13 +++++++------
 tests/test_crawler.py               | 24 +++++++++++++++++++++---
 tests/test_utils_asyncio.py         |  6 +++---
 11 files changed, 86 insertions(+), 28 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index a1d15a760..43f59f7cc 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -160,6 +160,31 @@ to any particular component. In that case the module of that component will be
 shown, typically an extension, middleware or pipeline. It also means that the
 component must be enabled in order for the setting to have any effect.
 
+.. setting:: ASYNCIO_SUPPORT
+
+ASYNCIO_SUPPORT
+---------------
+
+Default: ``False``
+
+Whether to support ``async def`` methods and callbacks which use code that
+requires an asyncio loop.
+
+If an ``async def`` coroutine doesn't require the asyncio loop, it will work
+even if this is set to ``False``. Coroutines that require the asyncio loop may
+silently fail to run or raise errors unless this is set to ``True``.
+
+When this option is set to ``True``, Scrapy will require
+:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. It will
+install this reactor if no reactor is installed yet, such as when using the
+``scrapy`` script or :class:`~scrapy.crawler.CrawlerProcess`. If you are using
+:class:`~scrapy.crawler.CrawlerRunner`, you need to install the correct reactor
+manually.
+
+The default value for this option is currently ``False`` to maintain backward
+compatibility and avoid possible problems caused by using a different Twisted
+reactor.
+
 .. setting:: AWS_ACCESS_KEY_ID
 
 AWS_ACCESS_KEY_ID
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 213e99bc0..ce030cf75 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -9,7 +9,6 @@ import pkg_resources
 import scrapy
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
-from scrapy.utils.asyncio import install_asyncio_reactor
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import inside_project, get_project_settings
 from scrapy.utils.python import garbage_collect
@@ -121,10 +120,6 @@ def execute(argv=None, settings=None):
             settings['EDITOR'] = editor
     check_deprecated_settings(settings)
 
-    # needs to be before _get_commands_dict() as that imports the command modules
-    # which may import twisted.internet.reactor
-    install_asyncio_reactor()
-
     inproject = inside_project()
     cmds = _get_commands_dict(settings, inproject)
     cmdname = _pop_command_name(argv)
@@ -146,7 +141,8 @@ def execute(argv=None, settings=None):
     opts, args = parser.parse_args(args=argv[1:])
     _run_print_help(parser, cmd.process_options, args, opts)
 
-    # needs to be after install_asyncio_reactor() as it imports twisted.internet.reactor
+    # needs to be after cmd.process_options() as it imports twisted.internet.reactor
+    # while commands may want to install the asyncio reactor
     from scrapy.crawler import CrawlerProcess
     cmd.crawler_process = CrawlerProcess(settings)
     _run_print_help(parser, _run_command, cmd, args, opts)
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 8093fd402..e2e69be49 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,5 +1,6 @@
 import os
 from scrapy.commands import ScrapyCommand
+from scrapy.utils.asyncio import install_asyncio_reactor
 from scrapy.utils.conf import arglist_to_dict
 from scrapy.utils.python import without_none_values
 from scrapy.exceptions import UsageError
@@ -26,6 +27,8 @@ class Command(ScrapyCommand):
 
     def process_options(self, args, opts):
         ScrapyCommand.process_options(self, args, opts)
+        if self.settings.getbool('ASYNCIO_SUPPORT'):
+            install_asyncio_reactor()
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 57d8471ca..ebd4eb620 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -2,6 +2,7 @@ import sys
 import os
 from importlib import import_module
 
+from scrapy.utils.asyncio import install_asyncio_reactor
 from scrapy.utils.spider import iter_spider_classes
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
@@ -50,6 +51,8 @@ class Command(ScrapyCommand):
 
     def process_options(self, args, opts):
         ScrapyCommand.process_options(self, args, opts)
+        if self.settings.getbool('ASYNCIO_SUPPORT'):
+            install_asyncio_reactor()
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 5c9678c01..c9097bd1f 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -19,6 +19,8 @@ from os.path import join, abspath, dirname
 
 AJAXCRAWL_ENABLED = False
 
+ASYNCIO_SUPPORT = False
+
 AUTOTHROTTLE_ENABLED = False
 AUTOTHROTTLE_DEBUG = False
 AUTOTHROTTLE_MAX_DELAY = 60.0
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
index f732774f1..b5d5f92d9 100644
--- a/scrapy/utils/asyncio.py
+++ b/scrapy/utils/asyncio.py
@@ -14,7 +14,7 @@ def install_asyncio_reactor():
             pass
 
 
-def is_asyncio_supported():
+def is_asyncio_reactor_installed():
     try:
         import twisted.internet.reactor
         from twisted.internet import asyncioreactor
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 30163d2fb..3b7ef75ab 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -9,8 +9,7 @@ from twisted.internet import defer, reactor, task
 from twisted.python import failure
 
 from scrapy.exceptions import IgnoreRequest
-from scrapy.utils.asyncio import is_asyncio_supported
-
+from scrapy.utils.asyncio import is_asyncio_reactor_installed
 
 
 def defer_fail(_failure):
@@ -127,12 +126,18 @@ def isfuture(o):
     return isinstance(o, asyncio.futures.Future)
 
 
-def deferred_from_coro(o):
+def deferred_from_coro(o, asyncio_enabled=False):
     """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
     if isinstance(o, defer.Deferred):
         return o
     if isfuture(o) or inspect.isawaitable(o):
-        if not is_asyncio_supported():
-            raise TypeError('Using coroutines requires installing AsyncioSelectorReactor')
-        return defer.Deferred.fromFuture(asyncio.ensure_future(o))
+        if not asyncio_enabled:
+            # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
+            # that use asyncio, e.g. "await asyncio.sleep(1)"
+            return defer.ensureDeferred(o)
+        else:
+            # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
+            if not is_asyncio_reactor_installed():
+                raise TypeError('Using coroutines requires installing AsyncioSelectorReactor')
+            return defer.Deferred.fromFuture(asyncio.ensure_future(o))
     return o
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index b74b7a4af..8c56cfa42 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -11,7 +11,7 @@ from twisted.python import log as twisted_log
 import scrapy
 from scrapy.settings import Settings
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.asyncio import is_asyncio_supported
+from scrapy.utils.asyncio import is_asyncio_reactor_installed
 from scrapy.utils.versions import scrapy_components_versions
 
 
@@ -149,8 +149,13 @@ def log_scrapy_info(settings):
                 {'versions': ", ".join("%s %s" % (name, version)
                     for name, version in scrapy_components_versions()
                     if name != "Scrapy")})
-    if is_asyncio_supported():
-        logger.debug("Asyncio support enabled")
+    if settings.getbool('ASYNCIO_SUPPORT'):
+        if is_asyncio_reactor_installed():
+            logger.debug("Asyncio support enabled")
+        else:
+            logger.error("ASYNCIO_SUPPORT is on but the Twisted asyncio "
+                         "reactor is not installed, this is not supported "
+                         "and asyncio coroutines will not work.")
 
 
 class StreamLogger(object):
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 8aa7ee109..3b64bfa23 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -9,7 +9,6 @@ from tempfile import mkdtemp
 from contextlib import contextmanager
 from threading import Timer
 
-from pytest import mark
 from twisted.trial import unittest
 
 import scrapy
@@ -179,7 +178,6 @@ class MiscCommandsTest(CommandTest):
         self.assertEqual(0, self.call('list'))
 
 
-@mark.usefixtures('reactor_pytest')
 class RunSpiderCommandTest(CommandTest):
 
     debug_log_spider = """
@@ -297,10 +295,13 @@ class BadSpider(scrapy.Spider):
         self.assertIn("start_requests", log)
         self.assertIn("badspider.py", log)
 
-    def test_asyncio_supported(self):
-        if self.reactor_pytest == 'asyncio':
-            log = self.get_log(self.debug_log_spider)
-            self.assertIn("DEBUG: Asyncio support enabled", log)
+    def test_asyncio_support_true(self):
+        log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_SUPPORT=True'])
+        self.assertIn("DEBUG: Asyncio support enabled", log)
+
+    def test_asyncio_support_false(self):
+        log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_SUPPORT=False'])
+        self.assertNotIn("DEBUG: Asyncio support enabled", log)
 
 
 class BenchCommandTest(CommandTest):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 151acb459..3ac45ca1d 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -10,7 +10,7 @@ import scrapy
 from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
 from scrapy.settings import Settings, default_settings
 from scrapy.spiderloader import SpiderLoader
-from scrapy.utils.asyncio import is_asyncio_supported
+from scrapy.utils.asyncio import is_asyncio_reactor_installed
 from scrapy.utils.log import configure_logging, get_scrapy_root_handler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
@@ -209,7 +209,7 @@ class AsyncioSpider(scrapy.Spider):
     name = 'asyncio'
 
     def start_requests(self):
-        self.logger.info('Asyncio support: %s', is_asyncio_supported())
+        self.logger.info('Asyncio support: %s', is_asyncio_reactor_installed())
         return []
 
 
@@ -258,7 +258,25 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
         self.assertEqual(runner.bootstrap_failed, True)
 
     @defer.inlineCallbacks
-    def test_asyncio_supported(self):
+    def test_crawler_process_asyncio_supported_true(self):
+        with LogCapture(level=logging.DEBUG) as log:
+            runner = CrawlerProcess(settings={'ASYNCIO_SUPPORT': True})
+            yield runner.crawl(NoRequestsSpider)
+            if self.reactor_pytest == 'asyncio':
+                self.assertIn("Asyncio support enabled", str(log))
+            else:
+                self.assertNotIn("Asyncio support enabled", str(log))
+                self.assertIn("ASYNCIO_SUPPORT is on but the Twisted asyncio reactor is not installed", str(log))
+
+    @defer.inlineCallbacks
+    def test_crawler_process_asyncio_supported_false(self):
+        runner = CrawlerProcess(settings={'ASYNCIO_SUPPORT': False})
+        with LogCapture(level=logging.DEBUG) as log:
+            yield runner.crawl(NoRequestsSpider)
+            self.assertNotIn("Asyncio support enabled", str(log))
+
+    @defer.inlineCallbacks
+    def test_crawler_runner_asyncio_supported(self):
         runner = CrawlerRunner()
         with LogCapture() as log:
             yield runner.crawl(AsyncioSpider)
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index e34d3002a..a6ba24876 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -2,15 +2,15 @@ from unittest import TestCase
 
 from pytest import mark
 
-from scrapy.utils.asyncio import is_asyncio_supported, install_asyncio_reactor
+from scrapy.utils.asyncio import is_asyncio_reactor_installed, install_asyncio_reactor
 
 
 @mark.usefixtures('reactor_pytest')
 class AsyncioTest(TestCase):
 
-    def test_is_asyncio_supported(self):
+    def test_is_asyncio_reactor_installed(self):
         # the result should depend only on the pytest --reactor argument
-        self.assertEquals(is_asyncio_supported(), self.reactor_pytest == 'asyncio')
+        self.assertEquals(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
 
     def test_install_asyncio_reactor(self):
         # this should do nothing

From 97fb61cec846641eb1c8e224ae24e55558746f4f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 4 Dec 2019 21:53:07 +0500
Subject: [PATCH 2467/2974] Move an import to postpone another "import
 twisted.internet.reactor".

---
 scrapy/commands/shell.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index e05084272..7516e2aba 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -6,7 +6,6 @@ See documentation in docs/topics/shell.rst
 from threading import Thread
 
 from scrapy.commands import ScrapyCommand
-from scrapy.shell import Shell
 from scrapy.http import Request
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 from scrapy.utils.url import guess_scheme
@@ -70,6 +69,8 @@ class Command(ScrapyCommand):
 
         self._start_crawler_thread()
 
+        # moved from the top-level because it imports twisted.internet.reactor
+        from scrapy.shell import Shell
         shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
         shell.start(url=url, redirect=not opts.no_redirect)
 

From 0b9f29215ff7f81203efbbc25b8e9cf3e9719920 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 4 Dec 2019 22:06:35 +0500
Subject: [PATCH 2468/2974] Update .travis.yml.

---
 .travis.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index fdf40fdf1..98dab01f2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,6 +17,8 @@ matrix:
       python: 3.5
     - env: TOXENV=py35-pinned
       python: 3.5
+    - env: TOXENV=py35-asyncio
+      python: 3.5
     - env: TOXENV=py36
       python: 3.6
     - env: TOXENV=py37
@@ -25,7 +27,7 @@ matrix:
       python: 3.8
     - env: TOXENV=py38-extra-deps
       python: 3.8
-    - env: TOXENV=py38-no-asyncio
+    - env: TOXENV=py38-asyncio
       python: 3.8
     - env: TOXENV=docs
       python: 3.6

From 9b4b43f8ac3a39d5dd6137223699d3d9e7dacec2 Mon Sep 17 00:00:00 2001
From: Grammy Jiang <gjiang@buildingsalive.com>
Date: Thu, 5 Dec 2019 11:25:19 +1100
Subject: [PATCH 2469/2974] Convert the relative imports to absolute imports

This commits converts the relative imports to absolute imports in the
entire package
---
 scrapy/__init__.py                      | 2 +-
 scrapy/contracts/default.py             | 2 +-
 scrapy/core/downloader/__init__.py      | 4 ++--
 scrapy/core/downloader/handlers/http.py | 6 ++++--
 scrapy/core/downloader/handlers/s3.py   | 2 +-
 scrapy/linkextractors/__init__.py       | 2 +-
 scrapy/linkextractors/regex.py          | 2 +-
 7 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 230e5cee3..fb8357f3c 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
 del warnings
 
 # Apply monkey patches to fix issues in external libraries
-from . import _monkeypatches
+from scrapy import _monkeypatches
 del _monkeypatches
 
 from twisted import version as _txv
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 24f6c2e77..e0d425874 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -4,7 +4,7 @@ from scrapy.item import BaseItem
 from scrapy.http import Request
 from scrapy.exceptions import ContractFail
 
-from . import Contract
+from scrapy.contracts import Contract
 
 
 # contracts
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 213268741..c5474a57f 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -10,8 +10,8 @@ from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.resolver import dnscache
 from scrapy import signals
-from .middleware import DownloaderMiddlewareManager
-from .handlers import DownloadHandlers
+from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
+from scrapy.core.downloader.handlers import DownloadHandlers
 
 
 class Slot(object):
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index 6111e132a..52535bd8b 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -1,2 +1,4 @@
-from .http10 import HTTP10DownloadHandler
-from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
+from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
+from scrapy.core.downloader.handlers.http11 import (
+    HTTP11DownloadHandler as HTTPDownloadHandler,
+)
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 808d1bf21..f4a42ce12 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -3,7 +3,7 @@ from six.moves.urllib.parse import unquote
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.boto import is_botocore
-from .http import HTTPDownloadHandler
+from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
 
 
 def _get_boto_connection():
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 8c3693f04..8411c4d59 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -118,4 +118,4 @@ class FilteringLinkExtractor(object):
 
 
 # Top-level imports
-from .lxmlhtml import LxmlLinkExtractor as LinkExtractor  # noqa: F401
+from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor  # noqa: F401
diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py
index e689b4727..49aa2be46 100644
--- a/scrapy/linkextractors/regex.py
+++ b/scrapy/linkextractors/regex.py
@@ -4,7 +4,7 @@ from six.moves.urllib.parse import urljoin
 from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
 
 from scrapy.link import Link
-from .sgml import SgmlLinkExtractor
+from scrapy.linkextractors.sgml import SgmlLinkExtractor
 
 linkre = re.compile(
         "<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",

From af624ef414ab10d833925b4d6f9048468be01273 Mon Sep 17 00:00:00 2001
From: Wang Qin <37098874+dqwerter@users.noreply.github.com>
Date: Thu, 5 Dec 2019 09:29:12 +0800
Subject: [PATCH 2470/2974] Update overview.rst | Fix an inconsistency

There exists an inconsistency between the code (line 37 - 38) and the output 'quotes.json' (line 56 - 68).

Note that even though according to line 53 - 54  'quotes.json' is "reformatted here for better readability", it cannot explain why the "author" field precedes the "text" field.

Intended output for the code BEFORE change:
    [{
        "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d",
        "author": "Jane Austen"
    },
    {
        "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d",
        "author": "Groucho Marx"
    },
    {
        "text": "\u201cA day without sunshine is like, you know, night.\u201d",
        "author": "Steve Martin"
    },
    ...]

Intended output for the code After change (the inconsistency is fixed):
    [{
        "author": "Jane Austen",
        "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"
    },
    {
        "author": "Groucho Marx",
        "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d"
    },
    {
        "author": "Steve Martin",
        "text": "\u201cA day without sunshine is like, you know, night.\u201d"
    },
    ...]
---
 docs/intro/overview.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 8b2fef065..01986b594 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -34,8 +34,8 @@ http://quotes.toscrape.com, following the pagination::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').get(),
                     'author': quote.xpath('span/small/text()').get(),
+                    'text': quote.css('span.text::text').get(),
                 }
 
             next_page = response.css('li.next a::attr("href")').get()

From 83b8046fdcae660276634a03b99d51c629fa8b7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 8 Nov 2019 16:26:46 +0100
Subject: [PATCH 2471/2974] Do not indent doctests from the documentation
 unnecessarily

---
 docs/intro/tutorial.rst         | 120 +++----
 docs/news.rst                   |  12 +-
 docs/topics/developer-tools.rst |  12 +-
 docs/topics/dynamic-content.rst |  28 +-
 docs/topics/items.rst           | 106 +++---
 docs/topics/leaks.rst           | 140 ++++----
 docs/topics/loaders.rst         | 114 ++++---
 docs/topics/selectors.rst       | 565 ++++++++++++++++----------------
 docs/topics/shell.rst           |  77 +++--
 docs/topics/stats.rst           |  12 +-
 10 files changed, 589 insertions(+), 597 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 6b15a5fbd..33b1a969b 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -252,30 +252,30 @@ The result of running ``response.css('title')`` is a list-like object called
 and allow you to run further queries to fine-grain the selection or extract the
 data.
 
-To extract the text from the title above, you can do::
+To extract the text from the title above, you can do:
 
-    >>> response.css('title::text').getall()
-    ['Quotes to Scrape']
+>>> response.css('title::text').getall()
+['Quotes to Scrape']
 
 There are two things to note here: one is that we've added ``::text`` to the
 CSS query, to mean we want to select only the text elements directly inside
 ``<title>`` element.  If we don't specify ``::text``, we'd get the full title
-element, including its tags::
+element, including its tags:
 
-    >>> response.css('title').getall()
-    ['<title>Quotes to Scrape</title>']
+>>> response.css('title').getall()
+['<title>Quotes to Scrape</title>']
 
 The other thing is that the result of calling ``.getall()`` is a list: it is
 possible that a selector returns more than one result, so we extract them all.
-When you know you just want the first result, as in this case, you can do::
+When you know you just want the first result, as in this case, you can do:
 
-    >>> response.css('title::text').get()
-    'Quotes to Scrape'
+>>> response.css('title::text').get()
+'Quotes to Scrape'
 
-As an alternative, you could've written::
+As an alternative, you could've written:
 
-    >>> response.css('title::text')[0].get()
-    'Quotes to Scrape'
+>>> response.css('title::text')[0].get()
+'Quotes to Scrape'
 
 However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
 instance avoids an ``IndexError`` and returns ``None`` when it doesn't
@@ -288,14 +288,14 @@ to be scraped, you can at least get **some** data.
 Besides the :meth:`~scrapy.selector.SelectorList.getall` and
 :meth:`~scrapy.selector.SelectorList.get` methods, you can also use
 the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
-expressions`_::
+expressions`_:
 
-    >>> response.css('title::text').re(r'Quotes.*')
-    ['Quotes to Scrape']
-    >>> response.css('title::text').re(r'Q\w+')
-    ['Quotes']
-    >>> response.css('title::text').re(r'(\w+) to (\w+)')
-    ['Quotes', 'Scrape']
+>>> response.css('title::text').re(r'Quotes.*')
+['Quotes to Scrape']
+>>> response.css('title::text').re(r'Q\w+')
+['Quotes']
+>>> response.css('title::text').re(r'(\w+) to (\w+)')
+['Quotes', 'Scrape']
 
 In order to find the proper CSS selectors to use, you might find useful opening
 the response page from the shell in your web browser using ``view(response)``.
@@ -312,12 +312,12 @@ visually selected elements, which works in many browsers.
 XPath: a brief intro
 ^^^^^^^^^^^^^^^^^^^^
 
-Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions::
+Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:
 
-    >>> response.xpath('//title')
-    [<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
-    >>> response.xpath('//title/text()').get()
-    'Quotes to Scrape'
+>>> response.xpath('//title')
+[<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
+>>> response.xpath('//title/text()').get()
+'Quotes to Scrape'
 
 XPath expressions are very powerful, and are the foundation of Scrapy
 Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You
@@ -372,35 +372,35 @@ we want::
 
     $ scrapy shell 'http://quotes.toscrape.com'
 
-We get a list of selectors for the quote HTML elements with::
+We get a list of selectors for the quote HTML elements with:
 
-    >>> response.css("div.quote")
-    [<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
-     <Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
-     ...]
+>>> response.css("div.quote")
+[<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
+ <Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
+ ...]
 
 Each of the selectors returned by the query above allows us to run further
 queries over their sub-elements. Let's assign the first selector to a
-variable, so that we can run our CSS selectors directly on a particular quote::
+variable, so that we can run our CSS selectors directly on a particular quote:
 
-    >>> quote = response.css("div.quote")[0]
+>>> quote = response.css("div.quote")[0]
 
 Now, let's extract ``text``, ``author`` and the ``tags`` from that quote
-using the ``quote`` object we just created::
+using the ``quote`` object we just created:
 
-    >>> text = quote.css("span.text::text").get()
-    >>> text
-    '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
-    >>> author = quote.css("small.author::text").get()
-    >>> author
-    'Albert Einstein'
+>>> text = quote.css("span.text::text").get()
+>>> text
+'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
+>>> author = quote.css("small.author::text").get()
+>>> author
+'Albert Einstein'
 
 Given that the tags are a list of strings, we can use the ``.getall()`` method
-to get all of them::
+to get all of them:
 
-    >>> tags = quote.css("div.tags a.tag::text").getall()
-    >>> tags
-    ['change', 'deep-thoughts', 'thinking', 'world']
+>>> tags = quote.css("div.tags a.tag::text").getall()
+>>> tags
+['change', 'deep-thoughts', 'thinking', 'world']
 
 .. invisible-code-block: python
 
@@ -409,16 +409,16 @@ to get all of them::
 .. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
 
 Having figured out how to extract each bit, we can now iterate over all the
-quotes elements and put them together into a Python dictionary::
+quotes elements and put them together into a Python dictionary:
 
-    >>> for quote in response.css("div.quote"):
-    ...     text = quote.css("span.text::text").get()
-    ...     author = quote.css("small.author::text").get()
-    ...     tags = quote.css("div.tags a.tag::text").getall()
-    ...     print(dict(text=text, author=author, tags=tags))
-    {'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
-    {'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
-    ...
+>>> for quote in response.css("div.quote"):
+...     text = quote.css("span.text::text").get()
+...     author = quote.css("small.author::text").get()
+...     tags = quote.css("div.tags a.tag::text").getall()
+...     print(dict(text=text, author=author, tags=tags))
+{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
+{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
+...
 
 Extracting data in our spider
 -----------------------------
@@ -516,23 +516,23 @@ markup:
         </li>
     </ul>
 
-We can try extracting it in the shell::
+We can try extracting it in the shell:
 
-    >>> response.css('li.next a').get()
-    '<a href="/page/2/">Next <span aria-hidden="true">→</span></a>'
+>>> response.css('li.next a').get()
+'<a href="/page/2/">Next <span aria-hidden="true">→</span></a>'
 
 This gets the anchor element, but we want the attribute ``href``. For that,
 Scrapy supports a CSS extension that lets you select the attribute contents,
-like this::
+like this:
 
-    >>> response.css('li.next a::attr(href)').get()
-    '/page/2/'
+>>> response.css('li.next a::attr(href)').get()
+'/page/2/'
 
 There is also an ``attrib`` property available
-(see :ref:`selecting-attributes` for more)::
+(see :ref:`selecting-attributes` for more):
 
-    >>> response.css('li.next a').attrib['href']
-    '/page/2/'
+>>> response.css('li.next a').attrib['href']
+'/page/2/'
 
 Let's see now our spider modified to recursively follow the link to the next
 page, extracting data from it::
diff --git a/docs/news.rst b/docs/news.rst
index 9dfd28508..217382c57 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -47,13 +47,13 @@ Backward-incompatible changes
     (:issue:`2111`, :issue:`3392`, :issue:`3442`, :issue:`3450`)
 
 *   :class:`~scrapy.loader.ItemLoader` now turns the values of its input item
-    into lists::
+    into lists:
 
-        >>> item = MyItem()
-        >>> item['field'] = 'value1'
-        >>> loader = ItemLoader(item=item)
-        >>> item['field']
-        ['value1']
+    >>> item = MyItem()
+    >>> item['field'] = 'value1'
+    >>> loader = ItemLoader(item=item)
+    >>> item['field']
+    ['value1']
 
     This is needed to allow adding values to existing fields
     (``loader.add_value('field', 'value2')``).
diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index bf14643be..1c9315cd8 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -112,13 +112,13 @@ see each quote:
 
 With this knowledge we can refine our XPath: Instead of a path to follow,
 we'll simply select all ``span`` tags with the ``class="text"`` by using 
-the `has-class-extension`_:: 
+the `has-class-extension`_:
 
-    >>> response.xpath('//span[has-class("text")]/text()').getall()
-   ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”,
-    '“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
-    '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
-    (...)]
+>>> response.xpath('//span[has-class("text")]/text()').getall()
+['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
+ '“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
+ '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
+ ...]
 
 And with one simple, cleverer XPath we are able to extract all quotes from 
 the page. We could have constructed a loop over our first XPath to increase 
diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index 8334ddcec..1c3607860 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -172,27 +172,27 @@ data from it:
     data in JSON format, which you can then parse with `json.loads`_.
 
     For example, if the JavaScript code contains a separate line like
-    ``var data = {"field": "value"};`` you can extract that data as follows::
+    ``var data = {"field": "value"};`` you can extract that data as follows:
 
-        >>> pattern = r'\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n'
-        >>> json_data = response.css('script::text').re_first(pattern)
-        >>> json.loads(json_data)
-        {'field': 'value'}
+    >>> pattern = r'\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n'
+    >>> json_data = response.css('script::text').re_first(pattern)
+    >>> json.loads(json_data)
+    {'field': 'value'}
 
 -   Otherwise, use js2xml_ to convert the JavaScript code into an XML document
     that you can parse using :ref:`selectors <topics-selectors>`.
 
     For example, if the JavaScript code contains
-    ``var data = {field: "value"};`` you can extract that data as follows::
+    ``var data = {field: "value"};`` you can extract that data as follows:
 
-        >>> import js2xml
-        >>> import lxml.etree
-        >>> from parsel import Selector
-        >>> javascript = response.css('script::text').get()
-        >>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding='unicode')
-        >>> selector = Selector(text=xml)
-        >>> selector.css('var[name="data"]').get()
-        '<var name="data"><object><property name="field"><string>value</string></property></object></var>'
+    >>> import js2xml
+    >>> import lxml.etree
+    >>> from parsel import Selector
+    >>> javascript = response.css('script::text').get()
+    >>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding='unicode')
+    >>> selector = Selector(text=xml)
+    >>> selector.css('var[name="data"]').get()
+    '<var name="data"><object><property name="field"><string>value</string></property></object></var>'
 
 .. _topics-javascript-rendering:
 
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index cdf60208e..15313775b 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -84,77 +84,74 @@ notice the API is very similar to the `dict API`_.
 Creating items
 --------------
 
-::
+>>> product = Product(name='Desktop PC', price=1000)
+>>> print(product)
+Product(name='Desktop PC', price=1000)
 
-    >>> product = Product(name='Desktop PC', price=1000)
-    >>> print(product)
-    Product(name='Desktop PC', price=1000)
 
 Getting field values
 --------------------
 
-::
+>>> product['name']
+Desktop PC
+>>> product.get('name')
+Desktop PC
 
-    >>> product['name']
-    Desktop PC
-    >>> product.get('name')
-    Desktop PC
+>>> product['price']
+1000
 
-    >>> product['price']
-    1000
+>>> product['last_updated']
+Traceback (most recent call last):
+    ...
+KeyError: 'last_updated'
 
-    >>> product['last_updated']
-    Traceback (most recent call last):
-        ...
-    KeyError: 'last_updated'
+>>> product.get('last_updated', 'not set')
+not set
 
-    >>> product.get('last_updated', 'not set')
-    not set
+>>> product['lala'] # getting unknown field
+Traceback (most recent call last):
+    ...
+KeyError: 'lala'
 
-    >>> product['lala'] # getting unknown field
-    Traceback (most recent call last):
-        ...
-    KeyError: 'lala'
+>>> product.get('lala', 'unknown field')
+'unknown field'
 
-    >>> product.get('lala', 'unknown field')
-    'unknown field'
+>>> 'name' in product  # is name field populated?
+True
 
-    >>> 'name' in product  # is name field populated?
-    True
+>>> 'last_updated' in product  # is last_updated populated?
+False
 
-    >>> 'last_updated' in product  # is last_updated populated?
-    False
+>>> 'last_updated' in product.fields  # is last_updated a declared field?
+True
 
-    >>> 'last_updated' in product.fields  # is last_updated a declared field?
-    True
+>>> 'lala' in product.fields  # is lala a declared field?
+False
 
-    >>> 'lala' in product.fields  # is lala a declared field?
-    False
 
 Setting field values
 --------------------
 
-::
+>>> product['last_updated'] = 'today'
+>>> product['last_updated']
+today
 
-    >>> product['last_updated'] = 'today'
-    >>> product['last_updated']
-    today
+>>> product['lala'] = 'test' # setting unknown field
+Traceback (most recent call last):
+    ...
+KeyError: 'Product does not support field: lala'
 
-    >>> product['lala'] = 'test' # setting unknown field
-    Traceback (most recent call last):
-        ...
-    KeyError: 'Product does not support field: lala'
 
 Accessing all populated values
 ------------------------------
 
-To access all populated values, just use the typical `dict API`_::
+To access all populated values, just use the typical `dict API`_:
 
-    >>> product.keys()
-    ['price', 'name']
+>>> product.keys()
+['price', 'name']
 
-    >>> product.items()
-    [('price', 1000), ('name', 'Desktop PC')]
+>>> product.items()
+[('price', 1000), ('name', 'Desktop PC')]
 
 
 .. _copying-items:
@@ -194,20 +191,21 @@ To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
 Other common tasks
 ------------------
 
-Creating dicts from items::
+Creating dicts from items:
 
-    >>> dict(product) # create a dict from all populated values
-    {'price': 1000, 'name': 'Desktop PC'}
+>>> dict(product) # create a dict from all populated values
+{'price': 1000, 'name': 'Desktop PC'}
 
-Creating items from dicts::
+Creating items from dicts:
 
-    >>> Product({'name': 'Laptop PC', 'price': 1500})
-    Product(price=1500, name='Laptop PC')
+>>> Product({'name': 'Laptop PC', 'price': 1500})
+Product(price=1500, name='Laptop PC')
+
+>>> Product({'name': 'Laptop PC', 'lala': 1500}) # warning: unknown field in dict
+Traceback (most recent call last):
+    ...
+KeyError: 'Product does not support field: lala'
 
-    >>> Product({'name': 'Laptop PC', 'lala': 1500}) # warning: unknown field in dict
-    Traceback (most recent call last):
-        ...
-    KeyError: 'Product does not support field: lala'
 
 Extending Items
 ===============
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 793636f59..87d9d262f 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -132,21 +132,21 @@ and check the code of the spider to discover the nasty line that is
 generating the leaks (passing response references inside requests).
 
 Sometimes extra information about live objects can be helpful.
-Let's check the oldest response::
+Let's check the oldest response:
 
-    >>> from scrapy.utils.trackref import get_oldest
-    >>> r = get_oldest('HtmlResponse')
-    >>> r.url
-    'http://www.somenastyspider.com/product.php?pid=123'
+>>> from scrapy.utils.trackref import get_oldest
+>>> r = get_oldest('HtmlResponse')
+>>> r.url
+'http://www.somenastyspider.com/product.php?pid=123'
 
 If you want to iterate over all objects, instead of getting the oldest one, you
-can use the :func:`scrapy.utils.trackref.iter_all` function::
+can use the :func:`scrapy.utils.trackref.iter_all` function:
 
-    >>> from scrapy.utils.trackref import iter_all
-    >>> [r.url for r in iter_all('HtmlResponse')]
-    ['http://www.somenastyspider.com/product.php?pid=123',
-     'http://www.somenastyspider.com/product.php?pid=584',
-    ...
+>>> from scrapy.utils.trackref import iter_all
+>>> [r.url for r in iter_all('HtmlResponse')]
+['http://www.somenastyspider.com/product.php?pid=123',
+ 'http://www.somenastyspider.com/product.php?pid=584',
+...]
 
 Too many spiders?
 -----------------
@@ -155,10 +155,10 @@ If your project has too many spiders executed in parallel,
 the output of :func:`prefs()` can be difficult to read.
 For this reason, that function has a ``ignore`` argument which can be used to
 ignore a particular class (and all its subclases). For
-example, this won't show any live references to spiders::
+example, this won't show any live references to spiders:
 
-    >>> from scrapy.spiders import Spider
-    >>> prefs(ignore=Spider)
+>>> from scrapy.spiders import Spider
+>>> prefs(ignore=Spider)
 
 .. module:: scrapy.utils.trackref
    :synopsis: Track references of live objects
@@ -214,41 +214,41 @@ If you use ``pip``, you can install Guppy with the following command::
 
 The telnet console also comes with a built-in shortcut (``hpy``) for accessing
 Guppy heap objects. Here's an example to view all Python objects available in
-the heap using Guppy::
+the heap using Guppy:
 
-    >>> x = hpy.heap()
-    >>> x.bytype
-    Partition of a set of 297033 objects. Total size = 52587824 bytes.
-     Index  Count   %     Size   % Cumulative  % Type
-         0  22307   8 16423880  31  16423880  31 dict
-         1 122285  41 12441544  24  28865424  55 str
-         2  68346  23  5966696  11  34832120  66 tuple
-         3    227   0  5836528  11  40668648  77 unicode
-         4   2461   1  2222272   4  42890920  82 type
-         5  16870   6  2024400   4  44915320  85 function
-         6  13949   5  1673880   3  46589200  89 types.CodeType
-         7  13422   5  1653104   3  48242304  92 list
-         8   3735   1  1173680   2  49415984  94 _sre.SRE_Pattern
-         9   1209   0   456936   1  49872920  95 scrapy.http.headers.Headers
-    <1676 more rows. Type e.g. '_.more' to view.>
+>>> x = hpy.heap()
+>>> x.bytype
+Partition of a set of 297033 objects. Total size = 52587824 bytes.
+ Index  Count   %     Size   % Cumulative  % Type
+     0  22307   8 16423880  31  16423880  31 dict
+     1 122285  41 12441544  24  28865424  55 str
+     2  68346  23  5966696  11  34832120  66 tuple
+     3    227   0  5836528  11  40668648  77 unicode
+     4   2461   1  2222272   4  42890920  82 type
+     5  16870   6  2024400   4  44915320  85 function
+     6  13949   5  1673880   3  46589200  89 types.CodeType
+     7  13422   5  1653104   3  48242304  92 list
+     8   3735   1  1173680   2  49415984  94 _sre.SRE_Pattern
+     9   1209   0   456936   1  49872920  95 scrapy.http.headers.Headers
+<1676 more rows. Type e.g. '_.more' to view.>
 
 You can see that most space is used by dicts. Then, if you want to see from
-which attribute those dicts are referenced, you could do::
+which attribute those dicts are referenced, you could do:
 
-    >>> x.bytype[0].byvia
-    Partition of a set of 22307 objects. Total size = 16423880 bytes.
-     Index  Count   %     Size   % Cumulative  % Referred Via:
-         0  10982  49  9416336  57   9416336  57 '.__dict__'
-         1   1820   8  2681504  16  12097840  74 '.__dict__', '.func_globals'
-         2   3097  14  1122904   7  13220744  80
-         3    990   4   277200   2  13497944  82 "['cookies']"
-         4    987   4   276360   2  13774304  84 "['cache']"
-         5    985   4   275800   2  14050104  86 "['meta']"
-         6    897   4   251160   2  14301264  87 '[2]'
-         7      1   0   196888   1  14498152  88 "['moduleDict']", "['modules']"
-         8    672   3   188160   1  14686312  89 "['cb_kwargs']"
-         9     27   0   155016   1  14841328  90 '[1]'
-    <333 more rows. Type e.g. '_.more' to view.>
+>>> x.bytype[0].byvia
+Partition of a set of 22307 objects. Total size = 16423880 bytes.
+ Index  Count   %     Size   % Cumulative  % Referred Via:
+     0  10982  49  9416336  57   9416336  57 '.__dict__'
+     1   1820   8  2681504  16  12097840  74 '.__dict__', '.func_globals'
+     2   3097  14  1122904   7  13220744  80
+     3    990   4   277200   2  13497944  82 "['cookies']"
+     4    987   4   276360   2  13774304  84 "['cache']"
+     5    985   4   275800   2  14050104  86 "['meta']"
+     6    897   4   251160   2  14301264  87 '[2]'
+     7      1   0   196888   1  14498152  88 "['moduleDict']", "['modules']"
+     8    672   3   188160   1  14686312  89 "['cb_kwargs']"
+     9     27   0   155016   1  14841328  90 '[1]'
+<333 more rows. Type e.g. '_.more' to view.>
 
 As you can see, the Guppy module is very powerful but also requires some deep
 knowledge about Python internals. For more info about Guppy, refer to the
@@ -269,32 +269,32 @@ If you use ``pip``, you can install muppy with the following command::
     pip install Pympler
 
 Here's an example to view all Python objects available in
-the heap using muppy::
+the heap using muppy:
 
-    >>> from pympler import muppy
-    >>> all_objects = muppy.get_objects()
-    >>> len(all_objects)
-    28667
-    >>> from pympler import summary
-    >>> suml = summary.summarize(all_objects)
-    >>> summary.print_(suml)
-                                   types |   # objects |   total size
-    ==================================== | =========== | ============
-                             <class 'str |        9822 |      1.10 MB
-                            <class 'dict |        1658 |    856.62 KB
-                            <class 'type |         436 |    443.60 KB
-                            <class 'code |        2974 |    419.56 KB
-              <class '_io.BufferedWriter |           2 |    256.34 KB
-                             <class 'set |         420 |    159.88 KB
-              <class '_io.BufferedReader |           1 |    128.17 KB
-              <class 'wrapper_descriptor |        1130 |     88.28 KB
-                           <class 'tuple |        1304 |     86.57 KB
-                         <class 'weakref |        1013 |     79.14 KB
-      <class 'builtin_function_or_method |         958 |     67.36 KB
-               <class 'method_descriptor |         865 |     60.82 KB
-                     <class 'abc.ABCMeta |          62 |     59.96 KB
-                            <class 'list |         446 |     58.52 KB
-                             <class 'int |        1425 |     43.20 KB
+>>> from pympler import muppy
+>>> all_objects = muppy.get_objects()
+>>> len(all_objects)
+28667
+>>> from pympler import summary
+>>> suml = summary.summarize(all_objects)
+>>> summary.print_(suml)
+                               types |   # objects |   total size
+==================================== | =========== | ============
+                         <class 'str |        9822 |      1.10 MB
+                        <class 'dict |        1658 |    856.62 KB
+                        <class 'type |         436 |    443.60 KB
+                        <class 'code |        2974 |    419.56 KB
+          <class '_io.BufferedWriter |           2 |    256.34 KB
+                         <class 'set |         420 |    159.88 KB
+          <class '_io.BufferedReader |           1 |    128.17 KB
+          <class 'wrapper_descriptor |        1130 |     88.28 KB
+                       <class 'tuple |        1304 |     86.57 KB
+                     <class 'weakref |        1013 |     79.14 KB
+  <class 'builtin_function_or_method |         958 |     67.36 KB
+           <class 'method_descriptor |         865 |     60.82 KB
+                 <class 'abc.ABCMeta |          62 |     59.96 KB
+                        <class 'list |         446 |     58.52 KB
+                         <class 'int |        1425 |     43.20 KB
 
 For more info about muppy, refer to the `muppy documentation`_.
 
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index de3f38023..9d5fccbbc 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -206,14 +206,12 @@ metadata. Here is an example::
             output_processor=TakeFirst(),
         )
 
-::
-
-    >>> from scrapy.loader import ItemLoader
-    >>> il = ItemLoader(item=Product())
-    >>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
-    >>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
-    >>> il.load_item()
-    {'name': u'Welcome to my website', 'price': u'1000'}
+>>> from scrapy.loader import ItemLoader
+>>> il = ItemLoader(item=Product())
+>>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
+>>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
+>>> il.load_item()
+{'name': u'Welcome to my website', 'price': u'1000'}
 
 The precedence order, for both input and output processors, is as follows:
 
@@ -314,11 +312,11 @@ ItemLoader objects
             applied before processors
         :type re: str or compiled regex
 
-        Examples::
+        Examples:
 
-            >>> from scrapy.loader.processors import TakeFirst
-            >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
-            'FOO`
+        >>> from scrapy.loader.processors import TakeFirst
+        >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
+        'FOO`
 
     .. method:: add_value(field_name, value, \*processors, \**kwargs)
 
@@ -639,12 +637,12 @@ Here is a list of all built-in processors:
     values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
     accept Loader contexts.
 
-    Example::
+    Example:
 
-        >>> from scrapy.loader.processors import Identity
-        >>> proc = Identity()
-        >>> proc(['one', 'two', 'three'])
-        ['one', 'two', 'three']
+    >>> from scrapy.loader.processors import Identity
+    >>> proc = Identity()
+    >>> proc(['one', 'two', 'three'])
+    ['one', 'two', 'three']
 
 .. class:: TakeFirst
 
@@ -652,12 +650,12 @@ Here is a list of all built-in processors:
     so it's typically used as an output processor to single-valued fields.
     It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
 
-    Example::
+    Example:
 
-        >>> from scrapy.loader.processors import TakeFirst
-        >>> proc = TakeFirst()
-        >>> proc(['', 'one', 'two', 'three'])
-        'one'
+    >>> from scrapy.loader.processors import TakeFirst
+    >>> proc = TakeFirst()
+    >>> proc(['', 'one', 'two', 'three'])
+    'one'
 
 .. class:: Join(separator=u' ')
 
@@ -667,15 +665,15 @@ Here is a list of all built-in processors:
     When using the default separator, this processor is equivalent to the
     function: ``u' '.join``
 
-    Examples::
+    Examples:
 
-        >>> from scrapy.loader.processors import Join
-        >>> proc = Join()
-        >>> proc(['one', 'two', 'three'])
-        'one two three'
-        >>> proc = Join('<br>')
-        >>> proc(['one', 'two', 'three'])
-        'one<br>two<br>three'
+    >>> from scrapy.loader.processors import Join
+    >>> proc = Join()
+    >>> proc(['one', 'two', 'three'])
+    'one two three'
+    >>> proc = Join('<br>')
+    >>> proc(['one', 'two', 'three'])
+    'one<br>two<br>three'
 
 .. class:: Compose(\*functions, \**default_loader_context)
 
@@ -688,12 +686,12 @@ Here is a list of all built-in processors:
     By default, stop process on ``None`` value. This behaviour can be changed by
     passing keyword argument ``stop_on_none=False``.
 
-    Example::
+    Example:
 
-        >>> from scrapy.loader.processors import Compose
-        >>> proc = Compose(lambda v: v[0], str.upper)
-        >>> proc(['hello', 'world'])
-        'HELLO'
+    >>> from scrapy.loader.processors import Compose
+    >>> proc = Compose(lambda v: v[0], str.upper)
+    >>> proc(['hello', 'world'])
+    'HELLO'
 
     Each function can optionally receive a ``loader_context`` parameter. For
     those which do, this processor will pass the currently active :ref:`Loader
@@ -732,15 +730,15 @@ Here is a list of all built-in processors:
     :meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors
     <topics-selectors>`, which returns a list of unicode strings.
 
-    The example below should clarify how it works::
+    The example below should clarify how it works:
 
-        >>> def filter_world(x):
-        ...     return None if x == 'world' else x
-        ...
-        >>> from scrapy.loader.processors import MapCompose
-        >>> proc = MapCompose(filter_world, str.upper)
-        >>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
-        ['HELLO, 'THIS', 'IS', 'SCRAPY']
+    >>> def filter_world(x):
+    ...     return None if x == 'world' else x
+    ...
+    >>> from scrapy.loader.processors import MapCompose
+    >>> proc = MapCompose(filter_world, str.upper)
+    >>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
+    ['HELLO, 'THIS', 'IS', 'SCRAPY']
 
     As with the Compose processor, functions can receive Loader contexts, and
     ``__init__`` method keyword arguments are used as default context values. See
@@ -752,21 +750,21 @@ Here is a list of all built-in processors:
     Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
     This processor takes only one input at a time.
 
-    Example::
+    Example:
 
-        >>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
-        >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
-        >>> proc({'foo': 'bar'})
-        'bar'
-        >>> proc({'foo': {'bar': 'baz'}})
-        {'bar': 'baz'}
+    >>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
+    >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
+    >>> proc({'foo': 'bar'})
+    'bar'
+    >>> proc({'foo': {'bar': 'baz'}})
+    {'bar': 'baz'}
 
-    Working with Json::
+    Working with Json:
 
-        >>> import json
-        >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
-        >>> proc_single_json_str('{"foo": "bar"}')
-        'bar'
-        >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
-        >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
-        ['bar']
+    >>> import json
+    >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
+    >>> proc_single_json_str('{"foo": "bar"}')
+    'bar'
+    >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
+    >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
+    ['bar']
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 282a585d4..8ec758b0e 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -51,18 +51,18 @@ Constructing selectors
 .. highlight:: python
 
 Response objects expose a :class:`~scrapy.selector.Selector` instance
-on ``.selector`` attribute::
+on ``.selector`` attribute:
 
-    >>> response.selector.xpath('//span/text()').get()
-    'good'
+>>> response.selector.xpath('//span/text()').get()
+'good'
 
 Querying responses using XPath and CSS is so common that responses include two
-more shortcuts: ``response.xpath()`` and ``response.css()``::
+more shortcuts: ``response.xpath()`` and ``response.css()``:
 
-    >>> response.xpath('//span/text()').get()
-    'good'
-    >>> response.css('span::text').get()
-    'good'
+>>> response.xpath('//span/text()').get()
+'good'
+>>> response.css('span::text').get()
+'good'
 
 Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
 constructed by passing either :class:`~scrapy.http.TextResponse` object or
@@ -74,21 +74,21 @@ shortcuts. By using ``response.selector`` or one of these shortcuts
 you can also ensure the response body is parsed only once.
 
 But if required, it is possible to use ``Selector`` directly.
-Constructing from text::
+Constructing from text:
 
-    >>> from scrapy.selector import Selector
-    >>> body = '<html><body><span>good</span></body></html>'
-    >>> Selector(text=body).xpath('//span/text()').get()
-    'good'
+>>> from scrapy.selector import Selector
+>>> body = '<html><body><span>good</span></body></html>'
+>>> Selector(text=body).xpath('//span/text()').get()
+'good'
 
 Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of
-:class:`~scrapy.http.TextResponse` subclasses::
+:class:`~scrapy.http.TextResponse` subclasses:
 
-    >>> from scrapy.selector import Selector
-    >>> from scrapy.http import HtmlResponse
-    >>> response = HtmlResponse(url='http://example.com', body=body)
-    >>> Selector(response=response).xpath('//span/text()').get()
-    'good'
+>>> from scrapy.selector import Selector
+>>> from scrapy.http import HtmlResponse
+>>> response = HtmlResponse(url='http://example.com', body=body)
+>>> Selector(response=response).xpath('//span/text()').get()
+'good'
 
 ``Selector`` automatically chooses the best parsing rules
 (XML vs HTML) based on input type.
@@ -123,118 +123,118 @@ Since we're dealing with HTML, the selector will automatically use an HTML parse
 .. highlight:: python
 
 So, by looking at the :ref:`HTML code <topics-selectors-htmlcode>` of that
-page, let's construct an XPath for selecting the text inside the title tag::
+page, let's construct an XPath for selecting the text inside the title tag:
 
-    >>> response.xpath('//title/text()')
-    [<Selector xpath='//title/text()' data='Example website'>]
+>>> response.xpath('//title/text()')
+[<Selector xpath='//title/text()' data='Example website'>]
 
 To actually extract the textual data, you must call the selector ``.get()``
-or ``.getall()`` methods, as follows::
+or ``.getall()`` methods, as follows:
 
-    >>> response.xpath('//title/text()').getall()
-    ['Example website']
-    >>> response.xpath('//title/text()').get()
-    'Example website'
+>>> response.xpath('//title/text()').getall()
+['Example website']
+>>> response.xpath('//title/text()').get()
+'Example website'
 
 ``.get()`` always returns a single result; if there are several matches,
 content of a first match is returned; if there are no matches, None
 is returned. ``.getall()`` returns a list with all results.
 
 Notice that CSS selectors can select text or attribute nodes using CSS3
-pseudo-elements::
+pseudo-elements:
 
-    >>> response.css('title::text').get()
-    'Example website'
+>>> response.css('title::text').get()
+'Example website'
 
 As you can see, ``.xpath()`` and ``.css()`` methods return a
 :class:`~scrapy.selector.SelectorList` instance, which is a list of new
-selectors. This API can be used for quickly selecting nested data::
+selectors. This API can be used for quickly selecting nested data:
 
-    >>> response.css('img').xpath('@src').getall()
-    ['image1_thumb.jpg',
-     'image2_thumb.jpg',
-     'image3_thumb.jpg',
-     'image4_thumb.jpg',
-     'image5_thumb.jpg']
+>>> response.css('img').xpath('@src').getall()
+['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
 
 If you want to extract only the first matched element, you can call the
 selector ``.get()`` (or its alias ``.extract_first()`` commonly used in
-previous Scrapy versions)::
+previous Scrapy versions):
 
-    >>> response.xpath('//div[@id="images"]/a/text()').get()
-    'Name: My image 1 '
+>>> response.xpath('//div[@id="images"]/a/text()').get()
+'Name: My image 1 '
 
-It returns ``None`` if no element was found::
+It returns ``None`` if no element was found:
 
-    >>> response.xpath('//div[@id="not-exists"]/text()').get() is None
-    True
+>>> response.xpath('//div[@id="not-exists"]/text()').get() is None
+True
 
 A default return value can be provided as an argument, to be used instead
 of ``None``:
 
-    >>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found')
-    'not-found'
+>>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found')
+'not-found'
 
 Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes
-using ``.attrib`` property of a :class:`~scrapy.selector.Selector`::
+using ``.attrib`` property of a :class:`~scrapy.selector.Selector`:
 
-    >>> [img.attrib['src'] for img in response.css('img')]
-    ['image1_thumb.jpg',
-     'image2_thumb.jpg',
-     'image3_thumb.jpg',
-     'image4_thumb.jpg',
-     'image5_thumb.jpg']
+>>> [img.attrib['src'] for img in response.css('img')]
+['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
 
 As a shortcut, ``.attrib`` is also available on SelectorList directly;
-it returns attributes for the first matching element::
+it returns attributes for the first matching element:
 
-    >>> response.css('img').attrib['src']
-    'image1_thumb.jpg'
+>>> response.css('img').attrib['src']
+'image1_thumb.jpg'
 
 This is most useful when only a single result is expected, e.g. when selecting
-by id, or selecting unique elements on a web page::
+by id, or selecting unique elements on a web page:
 
-    >>> response.css('base').attrib['href']
-    'http://example.com/'
+>>> response.css('base').attrib['href']
+'http://example.com/'
 
-Now we're going to get the base URL and some image links::
+Now we're going to get the base URL and some image links:
 
-    >>> response.xpath('//base/@href').get()
-    'http://example.com/'
+>>> response.xpath('//base/@href').get()
+'http://example.com/'
 
-    >>> response.css('base::attr(href)').get()
-    'http://example.com/'
+>>> response.css('base::attr(href)').get()
+'http://example.com/'
 
-    >>> response.css('base').attrib['href']
-    'http://example.com/'
+>>> response.css('base').attrib['href']
+'http://example.com/'
 
-    >>> response.xpath('//a[contains(@href, "image")]/@href').getall()
-    ['image1.html',
-     'image2.html',
-     'image3.html',
-     'image4.html',
-     'image5.html']
+>>> response.xpath('//a[contains(@href, "image")]/@href').getall()
+['image1.html',
+ 'image2.html',
+ 'image3.html',
+ 'image4.html',
+ 'image5.html']
 
-    >>> response.css('a[href*=image]::attr(href)').getall()
-    ['image1.html',
-     'image2.html',
-     'image3.html',
-     'image4.html',
-     'image5.html']
+>>> response.css('a[href*=image]::attr(href)').getall()
+['image1.html',
+ 'image2.html',
+ 'image3.html',
+ 'image4.html',
+ 'image5.html']
 
-    >>> response.xpath('//a[contains(@href, "image")]/img/@src').getall()
-    ['image1_thumb.jpg',
-     'image2_thumb.jpg',
-     'image3_thumb.jpg',
-     'image4_thumb.jpg',
-     'image5_thumb.jpg']
+>>> response.xpath('//a[contains(@href, "image")]/img/@src').getall()
+['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
 
-    >>> response.css('a[href*=image] img::attr(src)').getall()
-    ['image1_thumb.jpg',
-     'image2_thumb.jpg',
-     'image3_thumb.jpg',
-     'image4_thumb.jpg',
-     'image5_thumb.jpg']
+>>> response.css('a[href*=image] img::attr(src)').getall()
+['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
 
 .. _topics-selectors-css-extensions:
 
@@ -259,47 +259,47 @@ that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**:
 
 Examples:
 
-* ``title::text`` selects children text nodes of a descendant ``<title>`` element::
+* ``title::text`` selects children text nodes of a descendant ``<title>`` element:
 
-    >>> response.css('title::text').get()
-    'Example website'
+>>> response.css('title::text').get()
+'Example website'
 
-* ``*::text`` selects all descendant text nodes of the current selector context::
+* ``*::text`` selects all descendant text nodes of the current selector context:
 
-    >>> response.css('#images *::text').getall()
-    ['\n   ',
-     'Name: My image 1 ',
-     '\n   ',
-     'Name: My image 2 ',
-     '\n   ',
-     'Name: My image 3 ',
-     '\n   ',
-     'Name: My image 4 ',
-     '\n   ',
-     'Name: My image 5 ',
-     '\n  ']
+>>> response.css('#images *::text').getall()
+['\n   ',
+ 'Name: My image 1 ',
+ '\n   ',
+ 'Name: My image 2 ',
+ '\n   ',
+ 'Name: My image 3 ',
+ '\n   ',
+ 'Name: My image 4 ',
+ '\n   ',
+ 'Name: My image 5 ',
+ '\n  ']
 
 * ``foo::text`` returns no results if ``foo`` element exists, but contains
-  no text (i.e. text is empty)::
+  no text (i.e. text is empty):
 
-    >>> response.css('img::text').getall()
-    []
+>>> response.css('img::text').getall()
+[]
 
   This means ``.css('foo::text').get()`` could return None even if an element
-  exists. Use ``default=''`` if you always want a string::
+  exists. Use ``default=''`` if you always want a string:
 
-    >>> response.css('img::text').get()
-    >>> response.css('img::text').get(default='')
-    ''
+>>> response.css('img::text').get()
+>>> response.css('img::text').get(default='')
+''
 
-* ``a::attr(href)`` selects the *href* attribute value of descendant links::
+* ``a::attr(href)`` selects the *href* attribute value of descendant links:
 
-    >>> response.css('a::attr(href)').getall()
-    ['image1.html',
-     'image2.html',
-     'image3.html',
-     'image4.html',
-     'image5.html']
+>>> response.css('a::attr(href)').getall()
+['image1.html',
+ 'image2.html',
+ 'image3.html',
+ 'image4.html',
+ 'image5.html']
 
 .. note::
     See also: :ref:`selecting-attributes`.
@@ -318,25 +318,24 @@ Nesting selectors
 
 The selection methods (``.xpath()`` or ``.css()``) return a list of selectors
 of the same type, so you can call the selection methods for those selectors
-too. Here's an example::
+too. Here's an example:
 
-    >>> links = response.xpath('//a[contains(@href, "image")]')
-    >>> links.getall()
-    ['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
-     '<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
-     '<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
-     '<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
-     '<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
+>>> links = response.xpath('//a[contains(@href, "image")]')
+>>> links.getall()
+['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
+ '<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
+ '<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
+ '<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
+ '<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
 
-    >>> for index, link in enumerate(links):
-    ...     args = (index, link.xpath('@href').get(), link.xpath('img/@src').get())
-    ...     print('Link number %d points to url %r and image %r' % args)
-
-    Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
-    Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
-    Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
-    Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg'
-    Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg'
+>>> for index, link in enumerate(links):
+...     args = (index, link.xpath('@href').get(), link.xpath('img/@src').get())
+...     print('Link number %d points to url %r and image %r' % args)
+Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
+Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
+Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
+Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg'
+Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg'
 
 .. _selecting-attributes:
 
@@ -344,42 +343,42 @@ Selecting element attributes
 ----------------------------
 
 There are several ways to get a value of an attribute. First, one can use
-XPath syntax::
+XPath syntax:
 
-    >>> response.xpath("//a/@href").getall()
-    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+>>> response.xpath("//a/@href").getall()
+['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
 
 XPath syntax has a few advantages: it is a standard XPath feature, and
 ``@attributes`` can be used in other parts of an XPath expression - e.g.
 it is possible to filter by attribute value.
 
 Scrapy also provides an extension to CSS selectors (``::attr(...)``)
-which allows to get attribute values::
+which allows to get attribute values:
 
-    >>> response.css('a::attr(href)').getall()
-    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+>>> response.css('a::attr(href)').getall()
+['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
 
 In addition to that, there is a ``.attrib`` property of Selector.
 You can use it if you prefer to lookup attributes in Python
-code, without using XPaths or CSS extensions::
+code, without using XPaths or CSS extensions:
 
-    >>> [a.attrib['href'] for a in response.css('a')]
-    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+>>> [a.attrib['href'] for a in response.css('a')]
+['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
 
 This property is also available on SelectorList; it returns a dictionary
 with attributes of a first matching element. It is convenient to use when
 a selector is expected to give a single result (e.g. when selecting by element
-ID, or when selecting an unique element on a page)::
+ID, or when selecting an unique element on a page):
 
-    >>> response.css('base').attrib
-    {'href': 'http://example.com/'}
-    >>> response.css('base').attrib['href']
-    'http://example.com/'
+>>> response.css('base').attrib
+{'href': 'http://example.com/'}
+>>> response.css('base').attrib['href']
+'http://example.com/'
 
-``.attrib`` property of an empty SelectorList is empty::
+``.attrib`` property of an empty SelectorList is empty:
 
-    >>> response.css('foo').attrib
-    {}
+>>> response.css('foo').attrib
+{}
 
 Using selectors with regular expressions
 ----------------------------------------
@@ -390,21 +389,21 @@ data using regular expressions. However, unlike using ``.xpath()`` or
 can't construct nested ``.re()`` calls.
 
 Here's an example used to extract image names from the :ref:`HTML code
-<topics-selectors-htmlcode>` above::
+<topics-selectors-htmlcode>` above:
 
-    >>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
-    ['My image 1',
-     'My image 2',
-     'My image 3',
-     'My image 4',
-     'My image 5']
+>>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
+['My image 1',
+ 'My image 2',
+ 'My image 3',
+ 'My image 4',
+ 'My image 5']
 
 There's an additional helper reciprocating ``.get()`` (and its
 alias ``.extract_first()``) for ``.re()``, named ``.re_first()``.
-Use it to extract just the first matching string::
+Use it to extract just the first matching string:
 
-    >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
-    'My image 1'
+>>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
+'My image 1'
 
 .. _old-extraction-api:
 
@@ -422,28 +421,28 @@ and readable code.
 
 The following examples show how these methods map to each other.
 
-1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``::
+1.  ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``:
 
-     >>> response.css('a::attr(href)').get()
-     'image1.html'
-     >>> response.css('a::attr(href)').extract_first()
-     'image1.html'
+    >>> response.css('a::attr(href)').get()
+    'image1.html'
+    >>> response.css('a::attr(href)').extract_first()
+    'image1.html'
 
-2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``::
+2.  ``SelectorList.getall()`` is the same as ``SelectorList.extract()``:
 
-     >>> response.css('a::attr(href)').getall()
-     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
-     >>> response.css('a::attr(href)').extract()
-     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+    >>> response.css('a::attr(href)').getall()
+    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+    >>> response.css('a::attr(href)').extract()
+    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
 
-3. ``Selector.get()`` is the same as ``Selector.extract()``::
+3.  ``Selector.get()`` is the same as ``Selector.extract()``:
 
-     >>> response.css('a::attr(href)')[0].get()
-     'image1.html'
-     >>> response.css('a::attr(href)')[0].extract()
-     'image1.html'
+    >>> response.css('a::attr(href)')[0].get()
+    'image1.html'
+    >>> response.css('a::attr(href)')[0].extract()
+    'image1.html'
 
-4. For consistency, there is also ``Selector.getall()``, which returns a list::
+4.  For consistency, there is also ``Selector.getall()``, which returns a list:
 
     >>> response.css('a::attr(href)')[0].getall()
     ['image1.html']
@@ -481,26 +480,26 @@ with ``/``, that XPath will be absolute to the document and not relative to the
 ``Selector`` you're calling it from.
 
 For example, suppose you want to extract all ``<p>`` elements inside ``<div>``
-elements. First, you would get all ``<div>`` elements::
+elements. First, you would get all ``<div>`` elements:
 
-    >>> divs = response.xpath('//div')
+>>> divs = response.xpath('//div')
 
 At first, you may be tempted to use the following approach, which is wrong, as
 it actually extracts all ``<p>`` elements from the document, not only those
-inside ``<div>`` elements::
+inside ``<div>`` elements:
 
-    >>> for p in divs.xpath('//p'):  # this is wrong - gets all <p> from the whole document
-    ...     print(p.get())
+>>> for p in divs.xpath('//p'):  # this is wrong - gets all <p> from the whole document
+...     print(p.get())
 
-This is the proper way to do it (note the dot prefixing the ``.//p`` XPath)::
+This is the proper way to do it (note the dot prefixing the ``.//p`` XPath):
 
-    >>> for p in divs.xpath('.//p'):  # extracts all <p> inside
-    ...     print(p.get())
+>>> for p in divs.xpath('.//p'):  # extracts all <p> inside
+...     print(p.get())
 
-Another common case would be to extract all direct ``<p>`` children::
+Another common case would be to extract all direct ``<p>`` children:
 
-    >>> for p in divs.xpath('p'):
-    ...     print(p.get())
+>>> for p in divs.xpath('p'):
+...     print(p.get())
 
 For more details about relative XPaths see the `Location Paths`_ section in the
 XPath specification.
@@ -521,12 +520,12 @@ for that you may end up with more elements that you want, if they have a differe
 class name that shares the string ``someclass``.
 
 As it turns out, Scrapy selectors allow you to chain selectors, so most of the time
-you can just select by class using CSS and then switch to XPath when needed::
+you can just select by class using CSS and then switch to XPath when needed:
 
-    >>> from scrapy import Selector
-    >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
-    >>> sel.css('.shout').xpath('./time/@datetime').getall()
-    ['2014-07-23 19:00']
+>>> from scrapy import Selector
+>>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
+>>> sel.css('.shout').xpath('./time/@datetime').getall()
+['2014-07-23 19:00']
 
 This is cleaner than using the verbose XPath trick shown above. Just remember
 to use the ``.`` in the XPath expressions that will follow.
@@ -538,41 +537,41 @@ Beware of the difference between //node[1] and (//node)[1]
 
 ``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them.
 
-Example::
+Example:
 
-    >>> from scrapy import Selector
-    >>> sel = Selector(text="""
-    ....:     <ul class="list">
-    ....:         <li>1</li>
-    ....:         <li>2</li>
-    ....:         <li>3</li>
-    ....:     </ul>
-    ....:     <ul class="list">
-    ....:         <li>4</li>
-    ....:         <li>5</li>
-    ....:         <li>6</li>
-    ....:     </ul>""")
-    >>> xp = lambda x: sel.xpath(x).getall()
+>>> from scrapy import Selector
+>>> sel = Selector(text="""
+....:     <ul class="list">
+....:         <li>1</li>
+....:         <li>2</li>
+....:         <li>3</li>
+....:     </ul>
+....:     <ul class="list">
+....:         <li>4</li>
+....:         <li>5</li>
+....:         <li>6</li>
+....:     </ul>""")
+>>> xp = lambda x: sel.xpath(x).getall()
 
-This gets all first ``<li>``  elements under whatever it is its parent::
+This gets all first ``<li>``  elements under whatever it is its parent:
 
-    >>> xp("//li[1]")
-    ['<li>1</li>', '<li>4</li>']
+>>> xp("//li[1]")
+['<li>1</li>', '<li>4</li>']
 
-And this gets the first ``<li>``  element in the whole document::
+And this gets the first ``<li>``  element in the whole document:
 
-    >>> xp("(//li)[1]")
-    ['<li>1</li>']
+>>> xp("(//li)[1]")
+['<li>1</li>']
 
-This gets all first ``<li>``  elements under an ``<ul>``  parent::
+This gets all first ``<li>``  elements under an ``<ul>``  parent:
 
-    >>> xp("//ul/li[1]")
-    ['<li>1</li>', '<li>4</li>']
+>>> xp("//ul/li[1]")
+['<li>1</li>', '<li>4</li>']
 
-And this gets the first ``<li>``  element under an ``<ul>``  parent in the whole document::
+And this gets the first ``<li>``  element under an ``<ul>``  parent in the whole document:
 
-    >>> xp("(//ul/li)[1]")
-    ['<li>1</li>']
+>>> xp("(//ul/li)[1]")
+['<li>1</li>']
 
 Using text nodes in a condition
 -------------------------------
@@ -584,34 +583,34 @@ This is because the expression ``.//text()`` yields a collection of text element
 And when a node-set is converted to a string, which happens when it is passed as argument to
 a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only.
 
-Example::
+Example:
 
-    >>> from scrapy import Selector
-    >>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
+>>> from scrapy import Selector
+>>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
 
-Converting a *node-set* to string::
+Converting a *node-set* to string:
 
-    >>> sel.xpath('//a//text()').getall() # take a peek at the node-set
-    ['Click here to go to the ', 'Next Page']
-    >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
-    ['Click here to go to the ']
+>>> sel.xpath('//a//text()').getall() # take a peek at the node-set
+['Click here to go to the ', 'Next Page']
+>>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
+['Click here to go to the ']
 
-A *node* converted to a string, however, puts together the text of itself plus of all its descendants::
+A *node* converted to a string, however, puts together the text of itself plus of all its descendants:
 
-    >>> sel.xpath("//a[1]").getall() # select the first node
-    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
-    >>> sel.xpath("string(//a[1])").getall() # convert it to string
-    ['Click here to go to the Next Page']
+>>> sel.xpath("//a[1]").getall() # select the first node
+['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+>>> sel.xpath("string(//a[1])").getall() # convert it to string
+['Click here to go to the Next Page']
 
-So, using the ``.//text()`` node-set won't select anything in this case::
+So, using the ``.//text()`` node-set won't select anything in this case:
 
-    >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
-    []
+>>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
+[]
 
-But using the ``.`` to mean the node, works::
+But using the ``.`` to mean the node, works:
 
-    >>> sel.xpath("//a[contains(., 'Next Page')]").getall()
-    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+>>> sel.xpath("//a[contains(., 'Next Page')]").getall()
+['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
 
 .. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
 
@@ -627,17 +626,17 @@ some arguments in your queries with placeholders like ``?``,
 which are then substituted with values passed with the query.
 
 Here's an example to match an element based on its "id" attribute value,
-without hard-coding it (that was shown previously)::
+without hard-coding it (that was shown previously):
 
-    >>> # `$val` used in the expression, a `val` argument needs to be passed
-    >>> response.xpath('//div[@id=$val]/a/text()', val='images').get()
-    'Name: My image 1 '
+>>> # `$val` used in the expression, a `val` argument needs to be passed
+>>> response.xpath('//div[@id=$val]/a/text()', val='images').get()
+'Name: My image 1 '
 
 Here's another example, to find the "id" attribute of a ``<div>`` tag containing
-five ``<a>`` children (here we pass the value ``5`` as an integer)::
+five ``<a>`` children (here we pass the value ``5`` as an integer):
 
-    >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get()
-    'images'
+>>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get()
+'images'
 
 All variable references must have a binding value when calling ``.xpath()``
 (otherwise you'll get a ``ValueError: XPath error:`` exception).
@@ -687,19 +686,19 @@ You can see several namespace declarations including a default
 .. highlight:: python
 
 Once in the shell we can try selecting all ``<link>`` objects and see that it
-doesn't work (because the Atom XML namespace is obfuscating those nodes)::
+doesn't work (because the Atom XML namespace is obfuscating those nodes):
 
-    >>> response.xpath("//link")
-    []
+>>> response.xpath("//link")
+[]
 
 But once we call the :meth:`Selector.remove_namespaces` method, all
-nodes can be accessed directly by their names::
+nodes can be accessed directly by their names:
 
-    >>> response.selector.remove_namespaces()
-    >>> response.xpath("//link")
-    [<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>,
-     <Selector xpath='//link' data='<link rel="next" type="application/atom+'>,
-     ...
+>>> response.selector.remove_namespaces()
+>>> response.xpath("//link")
+[<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>,
+    <Selector xpath='//link' data='<link rel="next" type="application/atom+'>,
+    ...
 
 If you wonder why the namespace removal procedure isn't always called by default
 instead of having to call it manually, this is because of two reasons, which, in order
@@ -734,26 +733,25 @@ Regular expressions
 The ``test()`` function, for example, can prove quite useful when XPath's
 ``starts-with()`` or ``contains()`` are not sufficient.
 
-Example selecting links in list item with a "class" attribute ending with a digit::
+Example selecting links in list item with a "class" attribute ending with a digit:
 
-    >>> from scrapy import Selector
-    >>> doc = u"""
-    ... <div>
-    ...     <ul>
-    ...         <li class="item-0"><a href="link1.html">first item</a></li>
-    ...         <li class="item-1"><a href="link2.html">second item</a></li>
-    ...         <li class="item-inactive"><a href="link3.html">third item</a></li>
-    ...         <li class="item-1"><a href="link4.html">fourth item</a></li>
-    ...         <li class="item-0"><a href="link5.html">fifth item</a></li>
-    ...     </ul>
-    ... </div>
-    ... """
-    >>> sel = Selector(text=doc, type="html")
-    >>> sel.xpath('//li//@href').getall()
-    ['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html']
-    >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall()
-    ['link1.html', 'link2.html', 'link4.html', 'link5.html']
-    >>>
+>>> from scrapy import Selector
+>>> doc = u"""
+... <div>
+...     <ul>
+...         <li class="item-0"><a href="link1.html">first item</a></li>
+...         <li class="item-1"><a href="link2.html">second item</a></li>
+...         <li class="item-inactive"><a href="link3.html">third item</a></li>
+...         <li class="item-1"><a href="link4.html">fourth item</a></li>
+...         <li class="item-0"><a href="link5.html">fifth item</a></li>
+...     </ul>
+... </div>
+... """
+>>> sel = Selector(text=doc, type="html")
+>>> sel.xpath('//li//@href').getall()
+['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html']
+>>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall()
+['link1.html', 'link2.html', 'link4.html', 'link5.html']
 
 .. warning:: C library ``libxslt`` doesn't natively support EXSLT regular
     expressions so `lxml`_'s implementation uses hooks to Python's ``re`` module.
@@ -849,7 +847,6 @@ with groups of itemscopes and corresponding itemprops::
     current scope: ['http://schema.org/Rating']
         properties: ['worstRating', 'ratingValue', 'bestRating']
 
-    >>>
 
 Here we first iterate over ``itemscope`` elements, and for each one,
 we look for all ``itemprops`` elements and exclude those that are themselves
@@ -877,15 +874,15 @@ For the following HTML::
 
 .. highlight:: python
 
-You can use it like this::
+You can use it like this:
 
-    >>> response.xpath('//p[has-class("foo")]')
-    [<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>,
-     <Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>]
-    >>> response.xpath('//p[has-class("foo", "bar-baz")]')
-    [<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>]
-    >>> response.xpath('//p[has-class("foo", "bar")]')
-    []
+>>> response.xpath('//p[has-class("foo")]')
+[<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>,
+ <Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>]
+>>> response.xpath('//p[has-class("foo", "bar-baz")]')
+[<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>]
+>>> response.xpath('//p[has-class("foo", "bar")]')
+[]
 
 So XPath ``//p[has-class("foo", "bar-baz")]`` is roughly equivalent to CSS
 ``p.foo.bar-baz``.  Please note, that it is slower in most of the cases,
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 68a0b19b5..c1fdfd221 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -177,47 +177,46 @@ all start with the ``[s]`` prefix)::
     >>>
 
 
-After that, we can start playing with the objects::
+After that, we can start playing with the objects:
 
-    >>> response.xpath('//title/text()').get()
-    'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
+>>> response.xpath('//title/text()').get()
+'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
 
-    >>> fetch("https://reddit.com")
+>>> fetch("https://reddit.com")
 
-    >>> response.xpath('//title/text()').get()
-    'reddit: the front page of the internet'
+>>> response.xpath('//title/text()').get()
+'reddit: the front page of the internet'
 
-    >>> request = request.replace(method="POST")
+>>> request = request.replace(method="POST")
 
-    >>> fetch(request)
+>>> fetch(request)
 
-    >>> response.status
-    404
+>>> response.status
+404
 
-    >>> from pprint import pprint
+>>> from pprint import pprint
 
-    >>> pprint(response.headers)
-    {'Accept-Ranges': ['bytes'],
-     'Cache-Control': ['max-age=0, must-revalidate'],
-     'Content-Type': ['text/html; charset=UTF-8'],
-     'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
-     'Server': ['snooserv'],
-     'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
-                    'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
-                    'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
-                    'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
-     'Vary': ['accept-encoding'],
-     'Via': ['1.1 varnish'],
-     'X-Cache': ['MISS'],
-     'X-Cache-Hits': ['0'],
-     'X-Content-Type-Options': ['nosniff'],
-     'X-Frame-Options': ['SAMEORIGIN'],
-     'X-Moose': ['majestic'],
-     'X-Served-By': ['cache-cdg8730-CDG'],
-     'X-Timer': ['S1481214079.394283,VS0,VE159'],
-     'X-Ua-Compatible': ['IE=edge'],
-     'X-Xss-Protection': ['1; mode=block']}
-    >>>
+>>> pprint(response.headers)
+{'Accept-Ranges': ['bytes'],
+ 'Cache-Control': ['max-age=0, must-revalidate'],
+ 'Content-Type': ['text/html; charset=UTF-8'],
+ 'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
+ 'Server': ['snooserv'],
+ 'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
+                'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
+                'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
+                'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
+ 'Vary': ['accept-encoding'],
+ 'Via': ['1.1 varnish'],
+ 'X-Cache': ['MISS'],
+ 'X-Cache-Hits': ['0'],
+ 'X-Content-Type-Options': ['nosniff'],
+ 'X-Frame-Options': ['SAMEORIGIN'],
+ 'X-Moose': ['majestic'],
+ 'X-Served-By': ['cache-cdg8730-CDG'],
+ 'X-Timer': ['S1481214079.394283,VS0,VE159'],
+ 'X-Ua-Compatible': ['IE=edge'],
+ 'X-Xss-Protection': ['1; mode=block']}
 
 
 .. _topics-shell-inspect-response:
@@ -263,16 +262,16 @@ When you run the spider, you will get something similar to this::
     >>> response.url
     'http://example.org'
 
-Then, you can check if the extraction code is working::
+Then, you can check if the extraction code is working:
 
-    >>> response.xpath('//h1[@class="fn"]')
-    []
+>>> response.xpath('//h1[@class="fn"]')
+[]
 
 Nope, it doesn't. So you can open the response in your web browser and see if
-it's the response you were expecting::
+it's the response you were expecting:
 
-    >>> view(response)
-    True
+>>> view(response)
+True
 
 Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the
 crawling::
diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst
index 38648ec55..3dd829ebe 100644
--- a/docs/topics/stats.rst
+++ b/docs/topics/stats.rst
@@ -57,15 +57,15 @@ Set stat value only if lower than previous::
 
     stats.min_value('min_free_memory_percent', value)
 
-Get stat value::
+Get stat value:
 
-    >>> stats.get_value('custom_count')
-    1
+>>> stats.get_value('custom_count')
+1
 
-Get all stats::
+Get all stats:
 
-    >>> stats.get_stats()
-    {'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
+>>> stats.get_stats()
+{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
 
 Available Stats Collectors
 ==========================

From 07b8cd28aa84fb322a072467b49e2557d4aa3881 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 5 Dec 2019 14:48:31 +0100
Subject: [PATCH 2472/2974] =?UTF-8?q?Mark=20bandit=E2=80=99s=20402=20check?=
 =?UTF-8?q?=20as=20addressed=20by=20#4180=20(#4181)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bandit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.bandit.yml b/.bandit.yml
index cc7db3a66..243379b0b 100644
--- a/.bandit.yml
+++ b/.bandit.yml
@@ -8,7 +8,7 @@ skips:
 - B311
 - B320
 - B321
-- B402
+- B402  # https://github.com/scrapy/scrapy/issues/4180
 - B403
 - B404
 - B406

From 02cdc53fb82e3cc5e51771180f1f79186b52670a Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Fri, 13 Dec 2019 18:04:05 +0500
Subject: [PATCH 2473/2974] Add a test for a CrawlerProcess script. (#4218)

* Add a test for a CrawlerProcess script.

* Add tests/CrawlerProcess to collect_ignore.

* Remove an extra line.

* Fix/improve conftest.py.
---
 conftest.py                    |  9 ++++++++-
 tests/CrawlerProcess/simple.py | 15 +++++++++++++++
 tests/test_crawler.py          | 20 ++++++++++++++++++++
 3 files changed, 43 insertions(+), 1 deletion(-)
 create mode 100644 tests/CrawlerProcess/simple.py

diff --git a/conftest.py b/conftest.py
index d54ce155c..d37c22436 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,12 +1,19 @@
+from pathlib import Path
+
 import pytest
 
 
+def _py_files(folder):
+    return (str(p) for p in Path(folder).rglob('*.py'))
+
+
 collect_ignore = [
     # not a test, but looks like a test
     "scrapy/utils/testsite.py",
+    # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
+    *_py_files("tests/CrawlerProcess")
 ]
 
-
 for line in open('tests/ignores.txt'):
     file_path = line.strip()
     if file_path and file_path[0] != '#':
diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py
new file mode 100644
index 000000000..5f6f1ae30
--- /dev/null
+++ b/tests/CrawlerProcess/simple.py
@@ -0,0 +1,15 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 8eb2389e2..e37a2ff0e 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,4 +1,7 @@
 import logging
+import os
+import subprocess
+import sys
 import warnings
 
 from twisted.internet import defer
@@ -14,6 +17,7 @@ from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
 from scrapy.extensions.throttle import AutoThrottle
 from scrapy.extensions import telnet
+from scrapy.utils.test import get_testenv
 
 
 class BaseCrawlerTest(unittest.TestCase):
@@ -245,3 +249,19 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
         yield runner.crawl(NoRequestsSpider)
 
         self.assertEqual(runner.bootstrap_failed, True)
+
+
+class CrawlerProcessSubprocess(unittest.TestCase):
+    script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
+
+    def run_script(self, script_name):
+        script_path = os.path.join(self.script_dir, script_name)
+        args = (sys.executable, script_path)
+        p = subprocess.Popen(args, env=get_testenv(),
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = p.communicate()
+        return stderr.decode('utf-8')
+
+    def test_simple(self):
+        log = self.run_script('simple.py')
+        self.assertIn('Spider closed (finished)', log)

From 3560123090c1660fcfad7c5e7da08c9af503940f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 5 Dec 2019 19:06:51 +0500
Subject: [PATCH 2474/2974] Rename ASYNCIO_SUPPORT to ASYNCIO_ENABLED.

---
 docs/topics/settings.rst            |  4 ++--
 scrapy/commands/crawl.py            |  2 +-
 scrapy/commands/runspider.py        |  2 +-
 scrapy/settings/default_settings.py |  2 +-
 scrapy/utils/log.py                 |  4 ++--
 tests/test_commands.py              |  8 ++++----
 tests/test_crawler.py               | 10 +++++-----
 7 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 43f59f7cc..5cbf7450e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -160,9 +160,9 @@ to any particular component. In that case the module of that component will be
 shown, typically an extension, middleware or pipeline. It also means that the
 component must be enabled in order for the setting to have any effect.
 
-.. setting:: ASYNCIO_SUPPORT
+.. setting:: ASYNCIO_ENABLED
 
-ASYNCIO_SUPPORT
+ASYNCIO_ENABLED
 ---------------
 
 Default: ``False``
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index e2e69be49..b50761e4a 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -27,7 +27,7 @@ class Command(ScrapyCommand):
 
     def process_options(self, args, opts):
         ScrapyCommand.process_options(self, args, opts)
-        if self.settings.getbool('ASYNCIO_SUPPORT'):
+        if self.settings.getbool('ASYNCIO_ENABLED'):
             install_asyncio_reactor()
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index ebd4eb620..bfe844eb5 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -51,7 +51,7 @@ class Command(ScrapyCommand):
 
     def process_options(self, args, opts):
         ScrapyCommand.process_options(self, args, opts)
-        if self.settings.getbool('ASYNCIO_SUPPORT'):
+        if self.settings.getbool('ASYNCIO_ENABLED'):
             install_asyncio_reactor()
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index c9097bd1f..153b8037a 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -19,7 +19,7 @@ from os.path import join, abspath, dirname
 
 AJAXCRAWL_ENABLED = False
 
-ASYNCIO_SUPPORT = False
+ASYNCIO_ENABLED = False
 
 AUTOTHROTTLE_ENABLED = False
 AUTOTHROTTLE_DEBUG = False
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 8c56cfa42..0fe3d1549 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -149,11 +149,11 @@ def log_scrapy_info(settings):
                 {'versions': ", ".join("%s %s" % (name, version)
                     for name, version in scrapy_components_versions()
                     if name != "Scrapy")})
-    if settings.getbool('ASYNCIO_SUPPORT'):
+    if settings.getbool('ASYNCIO_ENABLED'):
         if is_asyncio_reactor_installed():
             logger.debug("Asyncio support enabled")
         else:
-            logger.error("ASYNCIO_SUPPORT is on but the Twisted asyncio "
+            logger.error("ASYNCIO_ENABLED is on but the Twisted asyncio "
                          "reactor is not installed, this is not supported "
                          "and asyncio coroutines will not work.")
 
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 3b64bfa23..197d80217 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -295,12 +295,12 @@ class BadSpider(scrapy.Spider):
         self.assertIn("start_requests", log)
         self.assertIn("badspider.py", log)
 
-    def test_asyncio_support_true(self):
-        log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_SUPPORT=True'])
+    def test_asyncio_enabled_true(self):
+        log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_ENABLED=True'])
         self.assertIn("DEBUG: Asyncio support enabled", log)
 
-    def test_asyncio_support_false(self):
-        log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_SUPPORT=False'])
+    def test_asyncio_enabled_false(self):
+        log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_ENABLED=False'])
         self.assertNotIn("DEBUG: Asyncio support enabled", log)
 
 
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 9410b0e7a..05909d995 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -262,19 +262,19 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
         self.assertEqual(runner.bootstrap_failed, True)
 
     @defer.inlineCallbacks
-    def test_crawler_process_asyncio_supported_true(self):
+    def test_crawler_process_asyncio_enabled_true(self):
         with LogCapture(level=logging.DEBUG) as log:
-            runner = CrawlerProcess(settings={'ASYNCIO_SUPPORT': True})
+            runner = CrawlerProcess(settings={'ASYNCIO_ENABLED': True})
             yield runner.crawl(NoRequestsSpider)
             if self.reactor_pytest == 'asyncio':
                 self.assertIn("Asyncio support enabled", str(log))
             else:
                 self.assertNotIn("Asyncio support enabled", str(log))
-                self.assertIn("ASYNCIO_SUPPORT is on but the Twisted asyncio reactor is not installed", str(log))
+                self.assertIn("ASYNCIO_ENABLED is on but the Twisted asyncio reactor is not installed", str(log))
 
     @defer.inlineCallbacks
-    def test_crawler_process_asyncio_supported_false(self):
-        runner = CrawlerProcess(settings={'ASYNCIO_SUPPORT': False})
+    def test_crawler_process_asyncio_enabled_false(self):
+        runner = CrawlerProcess(settings={'ASYNCIO_ENABLED': False})
         with LogCapture(level=logging.DEBUG) as log:
             yield runner.crawl(NoRequestsSpider)
             self.assertNotIn("Asyncio support enabled", str(log))

From 69cd2e247efe1823ab188eacb241b9e5596a879c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 7 Dec 2019 00:09:53 +0500
Subject: [PATCH 2475/2974] Move a bunch of "from twisted.internet import
 reactor" inside functions.

---
 scrapy/cmdline.py        | 4 +---
 scrapy/commands/shell.py | 3 +--
 scrapy/crawler.py        | 7 ++++++-
 scrapy/shell.py          | 3 ++-
 scrapy/utils/defer.py    | 4 +++-
 scrapy/utils/ossignal.py | 3 +--
 scrapy/utils/reactor.py  | 4 +++-
 7 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 3c2efe58f..69e917004 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -6,6 +6,7 @@ import inspect
 import pkg_resources
 
 import scrapy
+from scrapy.crawler import CrawlerProcess
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.misc import walk_modules
@@ -140,9 +141,6 @@ def execute(argv=None, settings=None):
     opts, args = parser.parse_args(args=argv[1:])
     _run_print_help(parser, cmd.process_options, args, opts)
 
-    # needs to be after cmd.process_options() as it imports twisted.internet.reactor
-    # while commands may want to install the asyncio reactor
-    from scrapy.crawler import CrawlerProcess
     cmd.crawler_process = CrawlerProcess(settings)
     _run_print_help(parser, _run_command, cmd, args, opts)
     sys.exit(cmd.exitcode)
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 7516e2aba..d44a32d5f 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -7,6 +7,7 @@ from threading import Thread
 
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
+from scrapy.shell import Shell
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 from scrapy.utils.url import guess_scheme
 
@@ -69,8 +70,6 @@ class Command(ScrapyCommand):
 
         self._start_crawler_thread()
 
-        # moved from the top-level because it imports twisted.internet.reactor
-        from scrapy.shell import Shell
         shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
         shell.start(url=url, redirect=not opts.no_redirect)
 
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 6c7eb737b..450260004 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -3,7 +3,7 @@ import pprint
 import signal
 import warnings
 
-from twisted.internet import reactor, defer
+from twisted.internet import defer
 from zope.interface.verify import verifyClass, DoesNotImplement
 
 from scrapy import Spider
@@ -261,6 +261,7 @@ class CrawlerProcess(CrawlerRunner):
         log_scrapy_info(self.settings)
 
     def _signal_shutdown(self, signum, _):
+        from twisted.internet import reactor
         install_shutdown_handlers(self._signal_kill)
         signame = signal_names[signum]
         logger.info("Received %(signame)s, shutting down gracefully. Send again to force ",
@@ -268,6 +269,7 @@ class CrawlerProcess(CrawlerRunner):
         reactor.callFromThread(self._graceful_stop_reactor)
 
     def _signal_kill(self, signum, _):
+        from twisted.internet import reactor
         install_shutdown_handlers(signal.SIG_IGN)
         signame = signal_names[signum]
         logger.info('Received %(signame)s twice, forcing unclean shutdown',
@@ -286,6 +288,7 @@ class CrawlerProcess(CrawlerRunner):
         :param boolean stop_after_crawl: stop or not the reactor when all
             crawlers have finished
         """
+        from twisted.internet import reactor
         if stop_after_crawl:
             d = self.join()
             # Don't start the reactor if the deferreds are already fired
@@ -300,6 +303,7 @@ class CrawlerProcess(CrawlerRunner):
         reactor.run(installSignalHandlers=False)  # blocking call
 
     def _get_dns_resolver(self):
+        from twisted.internet import reactor
         if self.settings.getbool('DNSCACHE_ENABLED'):
             cache_size = self.settings.getint('DNSCACHE_SIZE')
         else:
@@ -316,6 +320,7 @@ class CrawlerProcess(CrawlerRunner):
         return d
 
     def _stop_reactor(self, _=None):
+        from twisted.internet import reactor
         try:
             reactor.stop()
         except RuntimeError:  # raised if already stopped or in shutdown stage
diff --git a/scrapy/shell.py b/scrapy/shell.py
index a649d555f..a23b04df9 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -7,7 +7,7 @@ import os
 import signal
 import warnings
 
-from twisted.internet import reactor, threads, defer
+from twisted.internet import threads, defer
 from twisted.python import threadable
 from w3lib.url import any_to_uri
 
@@ -98,6 +98,7 @@ class Shell(object):
         return spider
 
     def fetch(self, request_or_url, spider=None, redirect=True, **kwargs):
+        from twisted.internet import reactor
         if isinstance(request_or_url, Request):
             request = request_or_url
         else:
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 3b7ef75ab..6a91776c7 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -5,7 +5,7 @@ import asyncio
 import asyncio.futures
 import inspect
 
-from twisted.internet import defer, reactor, task
+from twisted.internet import defer, task
 from twisted.python import failure
 
 from scrapy.exceptions import IgnoreRequest
@@ -19,6 +19,7 @@ def defer_fail(_failure):
     It delays by 100ms so reactor has a chance to go through readers and writers
     before attending pending delayed calls, so do not set delay to zero.
     """
+    from twisted.internet import reactor
     d = defer.Deferred()
     reactor.callLater(0.1, d.errback, _failure)
     return d
@@ -31,6 +32,7 @@ def defer_succeed(result):
     It delays by 100ms so reactor has a chance to go trough readers and writers
     before attending pending delayed calls, so do not set delay to zero.
     """
+    from twisted.internet import reactor
     d = defer.Deferred()
     reactor.callLater(0.1, d.callback, result)
     return d
diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py
index 7a7aec9be..45c9cef0c 100644
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@@ -1,7 +1,5 @@
 import signal
 
-from twisted.internet import reactor
-
 
 signal_names = {}
 for signame in dir(signal):
@@ -17,6 +15,7 @@ def install_shutdown_handlers(function, override_sigint=True):
     SIGINT handler won't be install if there is already a handler in place
     (e.g.  Pdb)
     """
+    from twisted.internet import reactor
     reactor._handleSignals()
     signal.signal(signal.SIGTERM, function)
     if signal.getsignal(signal.SIGINT) == signal.default_int_handler or \
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 493d26d4c..b98fff6ec 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -1,8 +1,9 @@
-from twisted.internet import reactor, error
+from twisted.internet import error
 
 
 def listen_tcp(portrange, host, factory):
     """Like reactor.listenTCP but tries different ports in a range."""
+    from twisted.internet import reactor
     assert len(portrange) <= 2, "invalid portrange: %s" % portrange
     if not portrange:
         return reactor.listenTCP(0, factory, interface=host)
@@ -30,6 +31,7 @@ class CallLaterOnce(object):
         self._call = None
 
     def schedule(self, delay=0):
+        from twisted.internet import reactor
         if self._call is None:
             self._call = reactor.callLater(delay, self)
 

From 855bbebc8bb862aa02e48f65fd861b1ddf78b57a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 13 Dec 2019 18:11:49 +0500
Subject: [PATCH 2476/2974] Move install_asyncio_reactor() from commands to
 CrawlerProcess.

---
 scrapy/commands/crawl.py     | 3 ---
 scrapy/commands/runspider.py | 3 ---
 scrapy/crawler.py            | 3 +++
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index b50761e4a..8093fd402 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,6 +1,5 @@
 import os
 from scrapy.commands import ScrapyCommand
-from scrapy.utils.asyncio import install_asyncio_reactor
 from scrapy.utils.conf import arglist_to_dict
 from scrapy.utils.python import without_none_values
 from scrapy.exceptions import UsageError
@@ -27,8 +26,6 @@ class Command(ScrapyCommand):
 
     def process_options(self, args, opts):
         ScrapyCommand.process_options(self, args, opts)
-        if self.settings.getbool('ASYNCIO_ENABLED'):
-            install_asyncio_reactor()
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index bfe844eb5..57d8471ca 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -2,7 +2,6 @@ import sys
 import os
 from importlib import import_module
 
-from scrapy.utils.asyncio import install_asyncio_reactor
 from scrapy.utils.spider import iter_spider_classes
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
@@ -51,8 +50,6 @@ class Command(ScrapyCommand):
 
     def process_options(self, args, opts):
         ScrapyCommand.process_options(self, args, opts)
-        if self.settings.getbool('ASYNCIO_ENABLED'):
-            install_asyncio_reactor()
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 450260004..706c8a59d 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -14,6 +14,7 @@ from scrapy.extension import ExtensionManager
 from scrapy.settings import overridden_settings, Settings
 from scrapy.signalmanager import SignalManager
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.asyncio import install_asyncio_reactor
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 from scrapy.utils.misc import load_object
 from scrapy.utils.log import (
@@ -256,6 +257,8 @@ class CrawlerProcess(CrawlerRunner):
 
     def __init__(self, settings=None, install_root_handler=True):
         super(CrawlerProcess, self).__init__(settings)
+        if self.settings.getbool('ASYNCIO_ENABLED'):
+            install_asyncio_reactor()
         install_shutdown_handlers(self._signal_shutdown)
         configure_logging(self.settings, install_root_handler)
         log_scrapy_info(self.settings)

From bfb78b8dea44a5db3f4a3bca83ab58c7ca0e3ef3 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 13 Dec 2019 18:12:07 +0500
Subject: [PATCH 2477/2974] Add CrawlerProcess tests for ASYNCIO_ENABLED.

---
 .../asyncio_enabled_no_reactor.py             | 17 ++++++++++++++
 .../CrawlerProcess/asyncio_enabled_reactor.py | 22 +++++++++++++++++++
 tests/test_crawler.py                         | 11 ++++++++++
 3 files changed, 50 insertions(+)
 create mode 100644 tests/CrawlerProcess/asyncio_enabled_no_reactor.py
 create mode 100644 tests/CrawlerProcess/asyncio_enabled_reactor.py

diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
new file mode 100644
index 000000000..dfe028ef4
--- /dev/null
+++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
@@ -0,0 +1,17 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={
+    'ASYNCIO_ENABLED': True,
+})
+
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py
new file mode 100644
index 000000000..7a172ea28
--- /dev/null
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py
@@ -0,0 +1,22 @@
+import asyncio
+
+from twisted.internet import asyncioreactor
+asyncioreactor.install(asyncio.get_event_loop())
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={
+    'ASYNCIO_ENABLED': True,
+})
+
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 05909d995..0b2645280 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -301,3 +301,14 @@ class CrawlerProcessSubprocess(unittest.TestCase):
     def test_simple(self):
         log = self.run_script('simple.py')
         self.assertIn('Spider closed (finished)', log)
+        self.assertNotIn("DEBUG: Asyncio support enabled", log)
+
+    def test_asyncio_enabled_no_reactor(self):
+        log = self.run_script('asyncio_enabled_no_reactor.py')
+        self.assertIn('Spider closed (finished)', log)
+        self.assertIn("DEBUG: Asyncio support enabled", log)
+
+    def test_asyncio_enabled_reactor(self):
+        log = self.run_script('asyncio_enabled_reactor.py')
+        self.assertIn('Spider closed (finished)', log)
+        self.assertIn("DEBUG: Asyncio support enabled", log)

From b5c4c2cae89714479d6fbb5a497fe43d48d886bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 13 Dec 2019 14:20:48 +0100
Subject: [PATCH 2478/2974] Keep 2 spaces between code and inline comments
 (#4195)

---
 pytest.ini                             | 35 +++++++++++++-------------
 scrapy/core/downloader/webclient.py    |  2 +-
 scrapy/core/engine.py                  |  2 +-
 scrapy/core/scraper.py                 |  2 +-
 scrapy/exporters.py                    |  2 +-
 scrapy/settings/default_settings.py    |  4 +--
 scrapy/spiders/feed.py                 |  4 +--
 scrapy/utils/console.py                |  8 +++---
 tests/test_command_fetch.py            |  2 +-
 tests/test_downloader_handlers.py      |  2 +-
 tests/test_engine.py                   |  1 -
 tests/test_http_request.py             |  2 +-
 tests/test_pipeline_media.py           |  6 ++---
 tests/test_spidermiddleware_referer.py | 20 +++++++--------
 tests/test_utils_defer.py              |  8 +++---
 tests/test_utils_spider.py             | 13 ++++------
 16 files changed, 54 insertions(+), 59 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 33c34b8e8..1b23595c0 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -26,8 +26,8 @@ flake8-ignore =
     scrapy/http/__init__.py F401
     # Issues pending a review:
     # extras
-    extras/qps-bench-server.py E261 E501
-    extras/qpsclient.py E501 E261 E501
+    extras/qps-bench-server.py E501
+    extras/qpsclient.py E501 E501
     # scrapy/commands
     scrapy/commands/__init__.py E128 E501
     scrapy/commands/check.py E501
@@ -45,15 +45,15 @@ flake8-ignore =
     scrapy/contracts/__init__.py E501 W504
     scrapy/contracts/default.py E502 E128
     # scrapy/core
-    scrapy/core/engine.py E261 E501 E128 E127 E306 E502
+    scrapy/core/engine.py E501 E128 E127 E306 E502
     scrapy/core/scheduler.py E501
-    scrapy/core/scraper.py E501 E306 E261 E128 W504
+    scrapy/core/scraper.py E501 E306 E128 W504
     scrapy/core/spidermw.py E501 E731 E502 E126 E226
     scrapy/core/downloader/__init__.py E501
     scrapy/core/downloader/contextfactory.py E501 E128 E126
     scrapy/core/downloader/middleware.py E501 E502
     scrapy/core/downloader/tls.py E501 E305 E241
-    scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226
+    scrapy/core/downloader/webclient.py E731 E501 E502 E128 E126 E226
     scrapy/core/downloader/handlers/__init__.py E501
     scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
     scrapy/core/downloader/handlers/http10.py E501
@@ -102,7 +102,7 @@ flake8-ignore =
     scrapy/selector/unified.py E501 E111
     # scrapy/settings
     scrapy/settings/__init__.py E501
-    scrapy/settings/default_settings.py E501 E261 E114 E116 E226
+    scrapy/settings/default_settings.py E501 E114 E116 E226
     scrapy/settings/deprecated.py E501
     # scrapy/spidermiddlewares
     scrapy/spidermiddlewares/httperror.py E501
@@ -112,17 +112,16 @@ flake8-ignore =
     # scrapy/spiders
     scrapy/spiders/__init__.py E501 E402
     scrapy/spiders/crawl.py E501
-    scrapy/spiders/feed.py E501 E261
+    scrapy/spiders/feed.py E501
     scrapy/spiders/sitemap.py E501
     # scrapy/utils
     scrapy/utils/benchserver.py E501
     scrapy/utils/conf.py E402 E502 E501
-    scrapy/utils/console.py E261 E306 E305
+    scrapy/utils/console.py E306 E305
     scrapy/utils/datatypes.py E501 E226
     scrapy/utils/decorators.py E501
     scrapy/utils/defer.py E501 E128
     scrapy/utils/deprecate.py E128 E501 E127 E502
-    scrapy/utils/engine.py E261
     scrapy/utils/gz.py E305 E501 W504
     scrapy/utils/http.py F403 E226
     scrapy/utils/httpobj.py E501
@@ -150,7 +149,7 @@ flake8-ignore =
     scrapy/crawler.py E501
     scrapy/dupefilters.py E501 E202
     scrapy/exceptions.py E501
-    scrapy/exporters.py E501 E261 E226
+    scrapy/exporters.py E501 E226
     scrapy/interfaces.py E501
     scrapy/item.py E501 E128
     scrapy/link.py E501
@@ -171,7 +170,7 @@ flake8-ignore =
     tests/pipelines.py F841 E226
     tests/spiders.py E501 E127
     tests/test_closespider.py E501 E127
-    tests/test_command_fetch.py E501 E261
+    tests/test_command_fetch.py E501
     tests/test_command_parse.py E501 E128 E303 E226
     tests/test_command_shell.py E501 E128
     tests/test_commands.py E128 E501
@@ -179,7 +178,7 @@ flake8-ignore =
     tests/test_crawl.py E501 E741 E265
     tests/test_crawler.py F841 E306 E501
     tests/test_dependencies.py F841 E501 E305
-    tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123
+    tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E502 E701 E126 E226 E123
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
     tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
@@ -194,13 +193,13 @@ flake8-ignore =
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
     tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
-    tests/test_engine.py E401 E501 E502 E128 E261
+    tests/test_engine.py E401 E501 E502 E128
     tests/test_exporters.py E501 E731 E306 E128 E124
     tests/test_extension_telnet.py F841
     tests/test_feedexport.py E501 F841 E241
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
-    tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
+    tests/test_http_request.py E402 E501 E127 E128 W293 E502 E128 E502 E126 E123
     tests/test_http_response.py E501 E301 E502 E128 E265
     tests/test_item.py E701 E128 F841 E306
     tests/test_link.py E501
@@ -212,7 +211,7 @@ flake8-ignore =
     tests/test_pipeline_crawl.py E131 E501 E128 E126
     tests/test_pipeline_files.py E501 W293 E303 E272 E226
     tests/test_pipeline_images.py F841 E501 E303
-    tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
+    tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502
     tests/test_proxy_connect.py E501 E741
     tests/test_request_cb_kwargs.py E501
     tests/test_responsetypes.py E501 E305
@@ -224,12 +223,12 @@ flake8-ignore =
     tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
     tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
     tests/test_spidermiddleware_output_chain.py E501 W293 E226
-    tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121
+    tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
     tests/test_squeues.py E501 E701 E741
     tests/test_utils_conf.py E501 E303 E128
     tests/test_utils_curl.py E501
     tests/test_utils_datatypes.py E402 E501 E305
-    tests/test_utils_defer.py E306 E261 E501 F841 E226
+    tests/test_utils_defer.py E306 E501 F841 E226
     tests/test_utils_deprecate.py F841 E306 E501
     tests/test_utils_http.py E501 E502 E128 W504
     tests/test_utils_iterators.py E501 E128 E129 E303 E241
@@ -240,7 +239,7 @@ flake8-ignore =
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841 E731 E226
     tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_spider.py E261 E305
+    tests/test_utils_spider.py E305
     tests/test_utils_template.py E305
     tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
     tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 798346f19..f368c3bae 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -42,7 +42,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
     delimiter = b'\n'
 
     def connectionMade(self):
-        self.headers = Headers() # bucket for response headers
+        self.headers = Headers()  # bucket for response headers
 
         # Method command
         self.sendCommand(self.factory.method, self.factory.path)
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index fa913e528..829e69993 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -25,7 +25,7 @@ class Slot(object):
 
     def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
         self.closing = False
-        self.inprogress = set() # requests in progress
+        self.inprogress = set()  # requests in progress
         self.start_requests = iter(start_requests)
         self.close_if_idle = close_if_idle
         self.nextcall = nextcall
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index db463f989..b3d585cce 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -123,7 +123,7 @@ class Scraper(object):
         callback/errback"""
         assert isinstance(response, (Response, Failure))
 
-        dfd = self._scrape2(response, request, spider) # returns spiders processed output
+        dfd = self._scrape2(response, request, spider)  # returns spiders processed output
         dfd.addErrback(self.handle_spider_error, request, response, spider)
         dfd.addCallback(self.handle_spider_output, request, response, spider)
         return dfd
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index fcb55da67..5bf131312 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -199,7 +199,7 @@ class CsvItemExporter(BaseItemExporter):
             line_buffering=False,
             write_through=True,
             encoding=self.encoding,
-            newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
+            newline=''  # Windows needs this https://github.com/scrapy/scrapy/issues/3034
         )
         self.csv_writer = csv.writer(self.stream, **kwargs)
         self._headers_not_written = True
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 5c9678c01..1e163e1fb 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -84,8 +84,8 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader'
 DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
 DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
 DOWNLOADER_CLIENT_TLS_CIPHERS = 'DEFAULT'
-DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
-                                     # also allowing negotiation
+# Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation:
+DOWNLOADER_CLIENT_TLS_METHOD = 'TLS'
 DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
 
 DOWNLOADER_MIDDLEWARES = {}
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 197812a26..c566f0236 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -100,8 +100,8 @@ class CSVFeedSpider(Spider):
     and the file's headers.
     """
 
-    delimiter = None # When this is None, python's csv module's default delimiter is used
-    quotechar = None # When this is None, python's csv module's default quotechar is used
+    delimiter = None  # When this is None, python's csv module's default delimiter is used
+    quotechar = None  # When this is None, python's csv module's default quotechar is used
     headers = None
 
     def process_results(self, response, results):
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 688e28c34..7eb40f0ce 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -47,7 +47,7 @@ def _embed_ptpython_shell(namespace={}, banner=''):
 def _embed_standard_shell(namespace={}, banner=''):
     """Start a standard python shell"""
     import code
-    try: # readline module is only available on unix systems
+    try:  # readline module is only available on unix systems
         import readline
     except ImportError:
         pass
@@ -72,9 +72,9 @@ def get_shell_embed_func(shells=None, known_shells=None):
     """Return the first acceptable shell-embed function
     from a given list of shell names.
     """
-    if shells is None: # list, preference order of shells
+    if shells is None:  # list, preference order of shells
         shells = DEFAULT_PYTHON_SHELLS.keys()
-    if known_shells is None: # available embeddable shells
+    if known_shells is None:  # available embeddable shells
         known_shells = DEFAULT_PYTHON_SHELLS.copy()
     for shell in shells:
         if shell in known_shells:
@@ -97,5 +97,5 @@ def start_python_console(namespace=None, banner='', shells=None):
         shell = get_shell_embed_func(shells)
         if shell is not None:
             shell(namespace=namespace, banner=banner)
-    except SystemExit: # raised when using exit() in python code.interact
+    except SystemExit:  # raised when using exit() in python code.interact
         pass
diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py
index 3fa3ed930..9d3c8fe73 100644
--- a/tests/test_command_fetch.py
+++ b/tests/test_command_fetch.py
@@ -29,6 +29,6 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
     @defer.inlineCallbacks
     def test_headers(self):
         _, out, _ = yield self.execute([self.url('/text'), '--headers'])
-        out = out.replace(b'\r', b'') # required on win32
+        out = out.replace(b'\r', b'')  # required on win32
         assert b'Server: TwistedWeb' in out, out
         assert b'Content-Type: text/plain' in out
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 60124b93f..2db2417e8 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -349,7 +349,7 @@ class HttpTestCase(unittest.TestCase):
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
     def test_payload(self):
-        body = b'1'*100 # PayloadResource requires body length to be 100
+        body = b'1'*100  # PayloadResource requires body length to be 100
         request = Request(self.getURL('payload'), method='POST', body=body)
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.body)
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 002c4e6bb..537df8d91 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -271,7 +271,6 @@ class EngineTest(unittest.TestCase):
                          self.run.signals_catched[signals.spider_opened])
         self.assertEqual({'spider': self.run.spider},
                          self.run.signals_catched[signals.spider_idle])
-        self.run.signals_catched[signals.spider_closed].pop('spider_stats', None) # XXX: remove for scrapy 0.17
         self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},
                          self.run.signals_catched[signals.spider_closed])
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index a98aa1e6f..57e7b457d 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -149,7 +149,7 @@ class RequestTest(unittest.TestCase):
 
         r2 = self.request_class(url="http://www.example.com/", body=b"")
         assert isinstance(r2.body, bytes)
-        self.assertEqual(r2.encoding, 'utf-8') # default encoding
+        self.assertEqual(r2.encoding, 'utf-8')  # default encoding
 
         r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8')
         assert isinstance(r3.body, bytes)
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 0d23f51cc..1fcc5799e 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -238,10 +238,10 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         self.assertEqual(new_item['results'], [(True, rsp1), (False, fail)])
         m = self.pipe._mockcalled
         # only once
-        self.assertEqual(m[0], 'get_media_requests') # first hook called
+        self.assertEqual(m[0], 'get_media_requests')  # first hook called
         self.assertEqual(m.count('get_media_requests'), 1)
         self.assertEqual(m.count('item_completed'), 1)
-        self.assertEqual(m[-1], 'item_completed') # last hook called
+        self.assertEqual(m[-1], 'item_completed')  # last hook called
         # twice, one per request
         self.assertEqual(m.count('media_to_download'), 2)
         # one to handle success and other for failure
@@ -252,7 +252,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
     def test_get_media_requests(self):
         # returns single Request (without callback)
         req = Request('http://url')
-        item = dict(requests=req) # pass a single item
+        item = dict(requests=req)  # pass a single item
         new_item = yield self.pipe.process_item(item, self.spider)
         assert new_item is item
         assert request_fingerprint(req) in self.info.downloaded
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index ecec6135d..7cc17600c 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -548,8 +548,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
                 (301, 'http://scrapytest.org/3'),
                 (301, 'http://scrapytest.org/4'),
             ),
-            b'http://scrapytest.org/1', # expected initial referer
-            b'http://scrapytest.org/1', # expected referer for the redirection request
+            b'http://scrapytest.org/1',  # expected initial referer
+            b'http://scrapytest.org/1',  # expected referer for the redirection request
         ),
         (   'https://scrapytest.org/1',
             'https://scrapytest.org/2',
@@ -609,8 +609,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
                 (301, 'http://scrapytest.org/3'),
                 (301, 'http://scrapytest.org/4'),
             ),
-            None, # expected initial "Referer"
-            None, # expected "Referer" for the redirection request
+            None,  # expected initial "Referer"
+            None,  # expected "Referer" for the redirection request
         ),
         (   'https://scrapytest.org/1',
             'https://scrapytest.org/2',
@@ -648,8 +648,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
                 (301, 'http://scrapytest.org/103'),
                 (301, 'http://scrapytest.org/104'),
             ),
-            b'http://scrapytest.org/101', # expected initial "Referer"
-            b'http://scrapytest.org/101', # expected referer for the redirection request
+            b'http://scrapytest.org/101',  # expected initial "Referer"
+            b'http://scrapytest.org/101',  # expected referer for the redirection request
         ),
         (   'https://scrapytest.org/201',
             'https://scrapytest.org/202',
@@ -757,8 +757,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
                 (301, 'http://scrapytest.org/103'),
                 (301, 'http://scrapytest.org/104'),
             ),
-            b'http://scrapytest.org/101', # expected initial referer
-            b'http://scrapytest.org/101', # expected referer for the redirection request
+            b'http://scrapytest.org/101',  # expected initial referer
+            b'http://scrapytest.org/101',  # expected referer for the redirection request
         ),
         (   'https://scrapytest.org/201',
             'https://scrapytest.org/202',
@@ -827,8 +827,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
                 (301, 'http://scrapytest.org/103'),
                 (301, 'http://scrapytest.org/104'),
             ),
-            b'http://scrapytest.org/101', # expected initial referer
-            b'http://scrapytest.org/101', # expected referer for the redirection request
+            b'http://scrapytest.org/101',  # expected initial referer
+            b'http://scrapytest.org/101',  # expected referer for the redirection request
         ),
         (   'https://scrapytest.org/201',
             'https://scrapytest.org/202',
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 49c2befb5..dfbe71ae2 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -14,8 +14,8 @@ class MustbeDeferredTest(unittest.TestCase):
             return steps
 
         dfd = mustbe_deferred(_append, 1)
-        dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
-        steps.append(2) # add another value, that should be catched by assertEqual
+        dfd.addCallback(self.assertEqual, [1, 2])  # it is [1] with maybeDeferred
+        steps.append(2)  # add another value, that should be catched by assertEqual
         return dfd
 
     def test_unfired_deferred(self):
@@ -27,8 +27,8 @@ class MustbeDeferredTest(unittest.TestCase):
             return dfd
 
         dfd = mustbe_deferred(_append, 1)
-        dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
-        steps.append(2) # add another value, that should be catched by assertEqual
+        dfd.addCallback(self.assertEqual, [1, 2])  # it is [1] with maybeDeferred
+        steps.append(2)  # add another value, that should be catched by assertEqual
         return dfd
 
 
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index edeeacc80..ee7d17062 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -1,20 +1,16 @@
 import unittest
+
+from scrapy import Spider
 from scrapy.http import Request
 from scrapy.item import BaseItem
 from scrapy.utils.spider import iterate_spider_output, iter_spider_classes
 
-from scrapy.spiders import CrawlSpider
 
-
-class MyBaseSpider(CrawlSpider):
-    pass # abstract spider
-
-
-class MySpider1(MyBaseSpider):
+class MySpider1(Spider):
     name = 'myspider1'
 
 
-class MySpider2(MyBaseSpider):
+class MySpider2(Spider):
     name = 'myspider2'
 
 
@@ -35,5 +31,6 @@ class UtilsSpidersTestCase(unittest.TestCase):
         it = iter_spider_classes(tests.test_utils_spider)
         self.assertEqual(set(it), {MySpider1, MySpider2})
 
+
 if __name__ == "__main__":
     unittest.main()

From a4ef9750f9058fbe1041bad0fb28f39c693e5659 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 13 Dec 2019 14:32:06 +0100
Subject: [PATCH 2479/2974] Fix Flake8-reported issues

---
 pytest.ini                   |  2 +-
 tests/test_linkextractors.py | 12 +++++++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 33c34b8e8..02014d1a5 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -88,7 +88,7 @@ flake8-ignore =
     scrapy/http/response/__init__.py E501 E128 W293 W291
     scrapy/http/response/text.py E501 W293 E128 E124
     # scrapy/linkextractors
-    scrapy/linkextractors/__init__.py E731 E502 E501 E402
+    scrapy/linkextractors/__init__.py E731 E502 E501 E402 W504
     scrapy/linkextractors/lxmlhtml.py E501 E731 E226
     # scrapy/loader
     scrapy/loader/__init__.py E501 E502 E128
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index ebe497913..0ffeaecc3 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -514,25 +514,27 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
         """Make sure the FilteringLinkExtractor deprecation warning is not
         issued for LxmlLinkExtractor"""
         with catch_warnings(record=True) as warnings:
-            extractor = LxmlLinkExtractor()
+            LxmlLinkExtractor()
             self.assertEqual(len(warnings), 0)
+
             class SubclassedItem(LxmlLinkExtractor):
                 pass
-            subclassed_extractor = SubclassedItem()
+
+            SubclassedItem()
             self.assertEqual(len(warnings), 0)
 
 
 class FilteringLinkExtractorTest(unittest.TestCase):
 
     def test_deprecation_warning(self):
-        args = [None]*10
+        args = [None] * 10
         with catch_warnings(record=True) as warnings:
-            extractor = FilteringLinkExtractor(*args)
+            FilteringLinkExtractor(*args)
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
         with catch_warnings(record=True) as warnings:
             class SubclassedFilteringLinkExtractor(FilteringLinkExtractor):
                 pass
-            subclassed_extractor = SubclassedFilteringLinkExtractor(*args)
+            SubclassedFilteringLinkExtractor(*args)
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)

From afc886e57865e82e63f3f8f3326f481908919086 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 13 Dec 2019 19:34:47 +0500
Subject: [PATCH 2480/2974] Simplify tox.ini asyncio entries.

---
 tox.ini | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tox.ini b/tox.ini
index a4edae439..795c20233 100644
--- a/tox.ini
+++ b/tox.ini
@@ -107,14 +107,16 @@ deps =
     reppy
     robotexclusionrulesparser
 
+[asyncio]
+commands =
+    py.test --cov=scrapy --cov-report= --reactor=asyncio {posargs:scrapy tests}
+
 [testenv:py35-asyncio]
 basepython = python3.5
 deps = {[testenv]deps}
-commands =
-    py.test --cov=scrapy --cov-report= --reactor=asyncio {posargs:scrapy tests}
+commands = {[asyncio]commands}
 
 [testenv:py38-asyncio]
 basepython = python3.8
 deps = {[testenv]deps}
-commands =
-    py.test --cov=scrapy --cov-report= --reactor=asyncio {posargs:scrapy tests}
+commands = {[asyncio]commands}

From a1605cade6286dd5f7f1c9e4c9660d44ed15ed19 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 13 Dec 2019 19:35:09 +0500
Subject: [PATCH 2481/2974] Hide utils.defer.isfuture().

---
 scrapy/utils/defer.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 6a91776c7..530bf0e9d 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -2,7 +2,6 @@
 Helper functions for dealing with Twisted deferreds
 """
 import asyncio
-import asyncio.futures
 import inspect
 
 from twisted.internet import defer, task
@@ -121,18 +120,18 @@ def iter_errback(iterable, errback, *a, **kw):
             errback(failure.Failure(), *a, **kw)
 
 
-def isfuture(o):
+def _isfuture(o):
     # workaround for Python before 3.5.3 not having asyncio.isfuture
     if hasattr(asyncio, 'isfuture'):
         return asyncio.isfuture(o)
-    return isinstance(o, asyncio.futures.Future)
+    return isinstance(o, asyncio.Future)
 
 
 def deferred_from_coro(o, asyncio_enabled=False):
     """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
     if isinstance(o, defer.Deferred):
         return o
-    if isfuture(o) or inspect.isawaitable(o):
+    if _isfuture(o) or inspect.isawaitable(o):
         if not asyncio_enabled:
             # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
             # that use asyncio, e.g. "await asyncio.sleep(1)"

From e3a3ad4aaff6d07252bb4f65be4667e21070aea1 Mon Sep 17 00:00:00 2001
From: marc <Marc>
Date: Sat, 14 Dec 2019 10:34:31 +0100
Subject: [PATCH 2482/2974] remove reference to old (Python 2.7) environment

---
 docs/contributing.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 234c4bcee..eaaf86c29 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -229,7 +229,7 @@ You can also specify a comma-separated list of environmets, and use :ref:`tox’
 parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
 parallel::
 
-    tox -e py27,py36 -p auto
+    tox -e py36,py38 -p auto
 
 To pass command-line options to :doc:`pytest <pytest:index>`, add them after
 ``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the

From 1aab20e1cec9e2e9aac4ce9efd37b07811b8f7d6 Mon Sep 17 00:00:00 2001
From: marc <Marc>
Date: Sat, 14 Dec 2019 10:37:31 +0100
Subject: [PATCH 2483/2974] update copyright notice year

---
 docs/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index 914d1d05f..5fdaa4e4c 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -50,7 +50,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Scrapy'
-copyright = u'2008–2018, Scrapy developers'
+copyright = u'2008–2020, Scrapy developers'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

From a59bb279d18966e6a47f71d21c690598fc69678e Mon Sep 17 00:00:00 2001
From: marc <Marc>
Date: Sun, 15 Dec 2019 17:33:00 +0100
Subject: [PATCH 2484/2974] add year through code

---
 docs/conf.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 5fdaa4e4c..ed56c5cd1 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -12,6 +12,7 @@
 # serve to show the default.
 
 import sys
+from datetime import datetime
 from os import path
 
 # If your extensions are in another directory, add it here. If the directory
@@ -49,8 +50,8 @@ source_suffix = '.rst'
 master_doc = 'index'
 
 # General information about the project.
-project = u'Scrapy'
-copyright = u'2008–2020, Scrapy developers'
+project = 'Scrapy'
+copyright = '2008–{}, Scrapy developers'.format(datetime.now().year)
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -194,8 +195,8 @@ htmlhelp_basename = 'Scrapydoc'
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, document class [howto/manual]).
 latex_documents = [
-  ('index', 'Scrapy.tex', u'Scrapy Documentation',
-   u'Scrapy developers', 'manual'),
+  ('index', 'Scrapy.tex', 'Scrapy Documentation',
+   'Scrapy developers', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of

From 2db7d453788f5c638d0921b0f7f8bab58e2a58bc Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 16 Dec 2019 19:24:25 +0500
Subject: [PATCH 2485/2974] Enable skipping tests based on --reactor.

---
 conftest.py | 8 ++++++++
 pytest.ini  | 2 ++
 2 files changed, 10 insertions(+)

diff --git a/conftest.py b/conftest.py
index 64136b48d..56d552953 100644
--- a/conftest.py
+++ b/conftest.py
@@ -35,6 +35,14 @@ def pytest_collection_modifyitems(session, config, items):
     except ImportError:
         pass
 
+
 @pytest.fixture()
 def reactor_pytest(request):
     request.cls.reactor_pytest = request.config.getoption("--reactor")
+    return request.cls.reactor_pytest
+
+
+@pytest.fixture(autouse=True)
+def only_asyncio(request, reactor_pytest):
+    if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio':
+        pytest.skip('This test is only run with --reactor-asyncio')
diff --git a/pytest.ini b/pytest.ini
index 336ef041d..7b62a1bd8 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -19,6 +19,8 @@ addopts =
     --ignore=docs/topics/telnetconsole.rst
     --ignore=docs/utils
 twisted = 1
+markers =
+    only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
 flake8-ignore =
     # Files that are only meant to provide top-level imports are expected not
     # to use any of their imports:

From 451e7a616e6d23e88528836296a9abdc887834c6 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 12 Jul 2019 00:26:01 -0300
Subject: [PATCH 2486/2974] Scan callbacks/errbacks for return statements with
 values different than None

---
 scrapy/core/scraper.py                        | 19 ++++--
 scrapy/utils/datatypes.py                     | 31 ++++++++-
 scrapy/utils/misc.py                          | 46 +++++++++++++
 tests/test_utils_datatypes.py                 | 66 ++++++++++++++++++-
 ...t_return_with_argument_inside_generator.py | 37 +++++++++++
 5 files changed, 190 insertions(+), 9 deletions(-)
 create mode 100644 tests/test_utils_misc/test_return_with_argument_inside_generator.py

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index b3d585cce..99114d3bb 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -9,7 +9,7 @@ from twisted.internet import defer
 
 from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback
 from scrapy.utils.spider import iterate_spider_output
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
 from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
 from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy import signals
@@ -18,6 +18,7 @@ from scrapy.item import BaseItem
 from scrapy.core.spidermw import SpiderMiddlewareManager
 from scrapy.utils.request import referer_str
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -99,11 +100,13 @@ class Scraper(object):
     def enqueue_scrape(self, response, request, spider):
         slot = self.slot
         dfd = slot.add_response_request(response, request)
+
         def finish_scraping(_):
             slot.finish_response(response, request)
             self._check_if_closing(spider, slot)
             self._scrape_next(spider, slot)
             return _
+
         dfd.addBoth(finish_scraping)
         dfd.addErrback(
             lambda f: logger.error('Scraper bug processing %(request)s',
@@ -123,7 +126,7 @@ class Scraper(object):
         callback/errback"""
         assert isinstance(response, (Response, Failure))
 
-        dfd = self._scrape2(response, request, spider)  # returns spiders processed output
+        dfd = self._scrape2(response, request, spider)  # returns spider's processed output
         dfd.addErrback(self.handle_spider_error, request, response, spider)
         dfd.addCallback(self.handle_spider_output, request, response, spider)
         return dfd
@@ -142,7 +145,10 @@ class Scraper(object):
     def call_spider(self, result, request, spider):
         result.request = request
         dfd = defer_result(result)
-        dfd.addCallbacks(callback=request.callback or spider.parse,
+        callback = request.callback or spider.parse
+        warn_on_generator_with_return_value(spider, callback)
+        warn_on_generator_with_return_value(spider, request.errback)
+        dfd.addCallbacks(callback=callback,
                          errback=request.errback,
                          callbackKeywords=request.cb_kwargs)
         return dfd.addCallback(iterate_spider_output)
@@ -172,8 +178,8 @@ class Scraper(object):
         if not result:
             return defer_succeed(None)
         it = iter_errback(result, self.handle_spider_error, request, response, spider)
-        dfd = parallel(it, self.concurrent_items,
-            self._process_spidermw_output, request, response, spider)
+        dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
+                       request, response, spider)
         return dfd
 
     def _process_spidermw_output(self, output, request, response, spider):
@@ -200,8 +206,7 @@ class Scraper(object):
         """Log and silence errors that come from the engine (typically download
         errors that got propagated thru here)
         """
-        if (isinstance(download_failure, Failure) and
-                not download_failure.check(IgnoreRequest)):
+        if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest):
             if download_failure.frames:
                 logger.error('Error downloading %(request)s',
                              {'request': request},
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index ffd1537c3..a52bbc70e 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -8,6 +8,7 @@ This module must not depend on any module outside the Standard Library.
 import collections
 import copy
 import warnings
+import weakref
 from collections.abc import Mapping
 
 from scrapy.exceptions import ScrapyDeprecationWarning
@@ -240,7 +241,6 @@ class LocalCache(collections.OrderedDict):
     """Dictionary with a finite number of keys.
 
     Older items expires first.
-
     """
 
     def __init__(self, limit=None):
@@ -254,6 +254,35 @@ class LocalCache(collections.OrderedDict):
         super(LocalCache, self).__setitem__(key, value)
 
 
+class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
+    """
+    A weakref.WeakKeyDictionary implementation that uses LocalCache as its
+    underlying data structure, making it ordered and capable of being size-limited.
+
+    Useful for memoization, while avoiding keeping received
+    arguments in memory only because of the cached references.
+
+    Note: like LocalCache and unlike weakref.WeakKeyDictionary,
+    it cannot be instantiated with an initial dictionary.
+    """
+
+    def __init__(self, limit=None):
+        super(LocalWeakReferencedCache, self).__init__()
+        self.data = LocalCache(limit=limit)
+
+    def __setitem__(self, key, value):
+        try:
+            super(LocalWeakReferencedCache, self).__setitem__(key, value)
+        except TypeError:
+            pass  # key is not weak-referenceable, skip caching
+
+    def __getitem__(self, key):
+        try:
+            return super(LocalWeakReferencedCache, self).__getitem__(key)
+        except TypeError:
+            return None  # key is not weak-referenceable, it's not cached
+
+
 class SequenceExclude(object):
     """Object to test if an item is NOT within some sequence."""
 
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 9955fb1e7..cb0ee5af3 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -1,13 +1,18 @@
 """Helper functions which don't fit anywhere else"""
+import ast
+import inspect
 import os
 import re
 import hashlib
+import warnings
 from contextlib import contextmanager
 from importlib import import_module
 from pkgutil import iter_modules
+from textwrap import dedent
 
 from w3lib.html import replace_entities
 
+from scrapy.utils.datatypes import LocalWeakReferencedCache
 from scrapy.utils.python import flatten, to_unicode
 from scrapy.item import BaseItem
 
@@ -161,3 +166,44 @@ def set_environ(**kwargs):
                 del os.environ[k]
             else:
                 os.environ[k] = v
+
+
+_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
+
+
+def is_generator_with_return_value(callable):
+    """
+    Returns True if a callable is a generator function which includes a
+    'return' statement with a value different than None, False otherwise
+    """
+    if callable in _generator_callbacks_cache:
+        return _generator_callbacks_cache[callable]
+
+    def returns_none(return_node):
+        value = return_node.value
+        return value is None or isinstance(value, ast.NameConstant) and value.value is None
+
+    if inspect.isgeneratorfunction(callable):
+        tree = ast.parse(dedent(inspect.getsource(callable)))
+        for node in ast.walk(tree):
+            if isinstance(node, ast.Return) and not returns_none(node):
+                _generator_callbacks_cache[callable] = True
+                return _generator_callbacks_cache[callable]
+
+    _generator_callbacks_cache[callable] = False
+    return _generator_callbacks_cache[callable]
+
+
+def warn_on_generator_with_return_value(spider, callable):
+    """
+    Logs a warning if a callable is a generator function and includes
+    a 'return' statement with a value different than None
+    """
+    if is_generator_with_return_value(callable):
+        warnings.warn(
+            'The "{}.{}" method is a generator and includes a "return" statement with a '
+            'value different than None. This could lead to unexpected behaviour. Please see '
+            'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '
+            'for details about the semantics of the "return" statement within generators'
+            .format(spider.__class__.__name__, callable.__name__), stacklevel=2,
+        )
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 38a25778e..e5aa56eb9 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -2,7 +2,9 @@ import copy
 import unittest
 from collections.abc import Mapping, MutableMapping
 
-from scrapy.utils.datatypes import CaselessDict, LocalCache, SequenceExclude
+from scrapy.http import Request
+from scrapy.utils.datatypes import CaselessDict, LocalCache, LocalWeakReferencedCache, SequenceExclude
+from scrapy.utils.python import garbage_collect
 
 
 __doctests__ = ['scrapy.utils.datatypes']
@@ -255,5 +257,67 @@ class LocalCacheTest(unittest.TestCase):
             self.assertEqual(cache[str(x)], x)
 
 
+class LocalWeakReferencedCacheTest(unittest.TestCase):
+
+    def test_cache_with_limit(self):
+        cache = LocalWeakReferencedCache(limit=2)
+        r1 = Request('https://example.org')
+        r2 = Request('https://example.com')
+        r3 = Request('https://example.net')
+        cache[r1] = 1
+        cache[r2] = 2
+        cache[r3] = 3
+        self.assertEqual(len(cache), 2)
+        self.assertNotIn(r1, cache)
+        self.assertIn(r2, cache)
+        self.assertIn(r3, cache)
+        self.assertEqual(cache[r2], 2)
+        self.assertEqual(cache[r3], 3)
+        del r2
+
+        # PyPy takes longer to collect dead references
+        garbage_collect()
+
+        self.assertEqual(len(cache), 1)
+
+    def test_cache_non_weak_referenceable_objects(self):
+        cache = LocalWeakReferencedCache()
+        k1 = None
+        k2 = 1
+        k3 = [1, 2, 3]
+        cache[k1] = 1
+        cache[k2] = 2
+        cache[k3] = 3
+        self.assertNotIn(k1, cache)
+        self.assertNotIn(k2, cache)
+        self.assertNotIn(k3, cache)
+        self.assertEqual(len(cache), 0)
+
+    def test_cache_without_limit(self):
+        max = 10**4
+        cache = LocalWeakReferencedCache()
+        refs = []
+        for x in range(max):
+            refs.append(Request('https://example.org/{}'.format(x)))
+            cache[refs[-1]] = x
+        self.assertEqual(len(cache), max)
+        for i, r in enumerate(refs):
+            self.assertIn(r, cache)
+            self.assertEqual(cache[r], i)
+        del r  # delete reference to the last object in the list
+
+        # delete half of the objects, make sure that is reflected in the cache
+        for _ in range(max // 2):
+            refs.pop()
+
+        # PyPy takes longer to collect dead references
+        garbage_collect()
+
+        self.assertEqual(len(cache), max // 2)
+        for i, r in enumerate(refs):
+            self.assertIn(r, cache)
+            self.assertEqual(cache[r], i)
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
new file mode 100644
index 000000000..bdbec1beb
--- /dev/null
+++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
@@ -0,0 +1,37 @@
+import unittest
+
+from scrapy.utils.misc import is_generator_with_return_value
+
+
+class UtilsMiscPy3TestCase(unittest.TestCase):
+
+    def test_generators_with_return_statements(self):
+        def f():
+            yield 1
+            return 2
+
+        def g():
+            yield 1
+            return 'asdf'
+
+        def h():
+            yield 1
+            return None
+
+        def i():
+            yield 1
+            return
+
+        def j():
+            yield 1
+
+        def k():
+            yield 1
+            yield from g()
+
+        assert is_generator_with_return_value(f)
+        assert is_generator_with_return_value(g)
+        assert not is_generator_with_return_value(h)
+        assert not is_generator_with_return_value(i)
+        assert not is_generator_with_return_value(j)
+        assert not is_generator_with_return_value(k)  # not recursive

From 039e6fe6919341dbfd864c4a406d35389c8e2992 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 16 Dec 2019 20:17:41 +0500
Subject: [PATCH 2487/2974] Refactor install_asyncio_reactor slightly.

---
 scrapy/utils/asyncio.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
index b5d5f92d9..b53c8a8b0 100644
--- a/scrapy/utils/asyncio.py
+++ b/scrapy/utils/asyncio.py
@@ -1,3 +1,8 @@
+from contextlib import suppress
+
+from twisted.internet.error import ReactorAlreadyInstalledError
+
+
 def install_asyncio_reactor():
     """ Tries to install AsyncioSelectorReactor
     """
@@ -5,13 +10,10 @@ def install_asyncio_reactor():
         import asyncio
         from twisted.internet import asyncioreactor
     except ImportError:
-        pass
-    else:
-        from twisted.internet.error import ReactorAlreadyInstalledError
-        try:
-            asyncioreactor.install(asyncio.get_event_loop())
-        except ReactorAlreadyInstalledError:
-            pass
+        return
+
+    with suppress(ReactorAlreadyInstalledError):
+        asyncioreactor.install(asyncio.get_event_loop())
 
 
 def is_asyncio_reactor_installed():

From 900de7c14607fbe2936fa682d03747916337f075 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 16 Dec 2019 21:11:58 +0500
Subject: [PATCH 2488/2974] Fix the reactor_pytest fixture.

---
 conftest.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/conftest.py b/conftest.py
index 56d552953..6d9696a3f 100644
--- a/conftest.py
+++ b/conftest.py
@@ -36,8 +36,11 @@ def pytest_collection_modifyitems(session, config, items):
         pass
 
 
-@pytest.fixture()
+@pytest.fixture(scope='class')
 def reactor_pytest(request):
+    if not request.cls:
+        # doctests
+        return
     request.cls.reactor_pytest = request.config.getoption("--reactor")
     return request.cls.reactor_pytest
 

From 5980d3bbff67d060a1a1b15372293ced972dbe8b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 10 Sep 2019 14:23:11 +0500
Subject: [PATCH 2489/2974] Add simple tests for pipelines.

---
 tests/test_pipelines.py | 71 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 tests/test_pipelines.py

diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
new file mode 100644
index 000000000..bc53f5427
--- /dev/null
+++ b/tests/test_pipelines.py
@@ -0,0 +1,71 @@
+from twisted.internet import defer
+from twisted.internet.defer import Deferred
+from twisted.trial import unittest
+
+from scrapy import Spider, signals, Request
+from scrapy.utils.test import get_crawler
+
+from tests.mockserver import MockServer
+
+
+class SimplePipeline:
+    def process_item(self, item, spider):
+        item['pipeline_passed'] = True
+        return item
+
+
+class DeferredPipeline:
+    def cb(self, item):
+        item['pipeline_passed'] = True
+        return item
+
+    def process_item(self, item, spider):
+        d = Deferred()
+        d.addCallback(self.cb)
+        d.callback(item)
+        return d
+
+
+class ItemSpider(Spider):
+    name = 'itemspider'
+
+    def start_requests(self):
+        yield Request(self.mockserver.url('/status?n=200'))
+
+    def parse(self, response):
+        return {'field': 42}
+
+
+class PipelineTestCase(unittest.TestCase):
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    def _on_item_scraped(self, item):
+        self.assertIsInstance(item, dict)
+        self.assertTrue(item.get('pipeline_passed'))
+        self.items.append(item)
+
+    def _create_crawler(self, pipeline_class):
+        settings = {
+            'ITEM_PIPELINES': {__name__ + '.' + pipeline_class.__name__: 1},
+        }
+        crawler = get_crawler(ItemSpider, settings)
+        crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
+        self.items = []
+        return crawler
+
+    @defer.inlineCallbacks
+    def test_simple_pipeline(self):
+        crawler = self._create_crawler(SimplePipeline)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertEqual(len(self.items), 1)
+
+    @defer.inlineCallbacks
+    def test_deferred_pipeline(self):
+        crawler = self._create_crawler(DeferredPipeline)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertEqual(len(self.items), 1)

From 7d0096da6e37e508280e3af7388122bcdf0e3dcf Mon Sep 17 00:00:00 2001
From: apu <1173372284@qq.com>
Date: Tue, 17 Dec 2019 09:47:01 +0800
Subject: [PATCH 2490/2974] Fix mail attachs tcmime *** (#4229)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When the file name consists of alphanumeric characters, it is normal to receive the attachment name.
However，However, problems will occur if the file name is changed to Chinese.
This has nothing to do with the file type
---
 scrapy/mail.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/mail.py b/scrapy/mail.py
index 891bb5e09..9655b8114 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -73,8 +73,7 @@ class MailSender(object):
                 part = MIMEBase(*mimetype.split('/'))
                 part.set_payload(f.read())
                 Encoders.encode_base64(part)
-                part.add_header('Content-Disposition', 'attachment; filename="%s"' \
-                    % attach_name)
+                part.add_header('Content-Disposition', 'attachment', filename=attach_name)
                 msg.attach(part)
         else:
             msg.set_payload(body)

From 63cf5c75c850e48b6f267574e4a8f7ae3293deac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Tue, 17 Dec 2019 13:53:15 +0100
Subject: [PATCH 2491/2974] Fix E502: backslash is redundant between brackets
 (#4238)

---
 pytest.ini                                    | 40 +++++------
 scrapy/cmdline.py                             |  4 +-
 scrapy/commands/fetch.py                      | 11 ++-
 scrapy/commands/startproject.py               |  4 +-
 scrapy/contracts/default.py                   |  4 +-
 scrapy/core/downloader/handlers/s3.py         |  4 +-
 scrapy/core/downloader/webclient.py           |  6 +-
 scrapy/core/spidermw.py                       |  6 +-
 .../downloadermiddlewares/httpcompression.py  |  5 +-
 scrapy/extensions/closespider.py              |  6 +-
 scrapy/linkextractors/__init__.py             |  3 +-
 scrapy/middleware.py                          |  4 +-
 scrapy/utils/conf.py                          |  2 +-
 tests/test_cmdline/__init__.py                | 10 ++-
 tests/test_downloader_handlers.py             | 72 ++++++++++---------
 tests/test_downloadermiddleware_retry.py      |  6 +-
 tests/test_engine.py                          |  4 +-
 tests/test_http_request.py                    | 34 +++++----
 tests/test_http_response.py                   |  4 +-
 tests/test_utils_http.py                      |  8 +--
 20 files changed, 121 insertions(+), 116 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 1b23595c0..f088e10ef 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -33,45 +33,45 @@ flake8-ignore =
     scrapy/commands/check.py E501
     scrapy/commands/crawl.py E501
     scrapy/commands/edit.py E501
-    scrapy/commands/fetch.py E401 E501 E128 E502 E731
+    scrapy/commands/fetch.py E401 E501 E128 E731
     scrapy/commands/genspider.py E128 E501 E502
     scrapy/commands/parse.py E128 E501 E731 E226
     scrapy/commands/runspider.py E501
     scrapy/commands/settings.py E128
     scrapy/commands/shell.py E128 E501 E502
-    scrapy/commands/startproject.py E502 E127 E501 E128
+    scrapy/commands/startproject.py E127 E501 E128
     scrapy/commands/version.py E501 E128
     # scrapy/contracts
     scrapy/contracts/__init__.py E501 W504
-    scrapy/contracts/default.py E502 E128
+    scrapy/contracts/default.py E128
     # scrapy/core
     scrapy/core/engine.py E501 E128 E127 E306 E502
     scrapy/core/scheduler.py E501
     scrapy/core/scraper.py E501 E306 E128 W504
-    scrapy/core/spidermw.py E501 E731 E502 E126 E226
+    scrapy/core/spidermw.py E501 E731 E126 E226
     scrapy/core/downloader/__init__.py E501
     scrapy/core/downloader/contextfactory.py E501 E128 E126
     scrapy/core/downloader/middleware.py E501 E502
     scrapy/core/downloader/tls.py E501 E305 E241
-    scrapy/core/downloader/webclient.py E731 E501 E502 E128 E126 E226
+    scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226
     scrapy/core/downloader/handlers/__init__.py E501
     scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
     scrapy/core/downloader/handlers/http10.py E501
     scrapy/core/downloader/handlers/http11.py E501
-    scrapy/core/downloader/handlers/s3.py E501 E502 E128 E126
+    scrapy/core/downloader/handlers/s3.py E501 E128 E126
     # scrapy/downloadermiddlewares
     scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
     scrapy/downloadermiddlewares/decompression.py E501
     scrapy/downloadermiddlewares/defaultheaders.py E501
     scrapy/downloadermiddlewares/httpcache.py E501 E126
-    scrapy/downloadermiddlewares/httpcompression.py E502 E128
+    scrapy/downloadermiddlewares/httpcompression.py E501 E128
     scrapy/downloadermiddlewares/httpproxy.py E501
     scrapy/downloadermiddlewares/redirect.py E501 W504
     scrapy/downloadermiddlewares/retry.py E501 E126
     scrapy/downloadermiddlewares/robotstxt.py E501
     scrapy/downloadermiddlewares/stats.py E501
     # scrapy/extensions
-    scrapy/extensions/closespider.py E501 E502 E128 E123
+    scrapy/extensions/closespider.py E501 E128 E123
     scrapy/extensions/corestats.py E501
     scrapy/extensions/feedexport.py E128 E501
     scrapy/extensions/httpcache.py E128 E501 E303
@@ -88,10 +88,10 @@ flake8-ignore =
     scrapy/http/response/__init__.py E501 E128 W293 W291
     scrapy/http/response/text.py E501 W293 E128 E124
     # scrapy/linkextractors
-    scrapy/linkextractors/__init__.py E731 E502 E501 E402
+    scrapy/linkextractors/__init__.py E731 E501 E402
     scrapy/linkextractors/lxmlhtml.py E501 E731 E226
     # scrapy/loader
-    scrapy/loader/__init__.py E501 E502 E128
+    scrapy/loader/__init__.py E501 E128
     scrapy/loader/processors.py E501
     # scrapy/pipelines
     scrapy/pipelines/files.py E116 E501 E266
@@ -116,7 +116,7 @@ flake8-ignore =
     scrapy/spiders/sitemap.py E501
     # scrapy/utils
     scrapy/utils/benchserver.py E501
-    scrapy/utils/conf.py E402 E502 E501
+    scrapy/utils/conf.py E402 E501
     scrapy/utils/console.py E306 E305
     scrapy/utils/datatypes.py E501 E226
     scrapy/utils/decorators.py E501
@@ -145,7 +145,7 @@ flake8-ignore =
     # scrapy
     scrapy/__init__.py E402 E501
     scrapy/_monkeypatches.py W293
-    scrapy/cmdline.py E502 E501
+    scrapy/cmdline.py E501
     scrapy/crawler.py E501
     scrapy/dupefilters.py E501 E202
     scrapy/exceptions.py E501
@@ -155,7 +155,7 @@ flake8-ignore =
     scrapy/link.py E501
     scrapy/logformatter.py E501 W293
     scrapy/mail.py E402 E128 E501 E502
-    scrapy/middleware.py E502 E128 E501
+    scrapy/middleware.py E128 E501
     scrapy/pqueues.py E501
     scrapy/responsetypes.py E128 E501 E305
     scrapy/robotstxt.py E501
@@ -178,7 +178,7 @@ flake8-ignore =
     tests/test_crawl.py E501 E741 E265
     tests/test_crawler.py F841 E306 E501
     tests/test_dependencies.py F841 E501 E305
-    tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E502 E701 E126 E226 E123
+    tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
     tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
@@ -189,18 +189,18 @@ flake8-ignore =
     tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
     tests/test_downloadermiddleware_httpproxy.py E501 E128
     tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
-    tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
+    tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E303 E126
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
     tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
-    tests/test_engine.py E401 E501 E502 E128
+    tests/test_engine.py E401 E501 E128
     tests/test_exporters.py E501 E731 E306 E128 E124
     tests/test_extension_telnet.py F841
     tests/test_feedexport.py E501 F841 E241
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
-    tests/test_http_request.py E402 E501 E127 E128 W293 E502 E128 E502 E126 E123
-    tests/test_http_response.py E501 E301 E502 E128 E265
+    tests/test_http_request.py E402 E501 E127 E128 W293 E128 E126 E123
+    tests/test_http_response.py E501 E301 E128 E265
     tests/test_item.py E701 E128 F841 E306
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128 E124
@@ -230,7 +230,7 @@ flake8-ignore =
     tests/test_utils_datatypes.py E402 E501 E305
     tests/test_utils_defer.py E306 E501 F841 E226
     tests/test_utils_deprecate.py F841 E306 E501
-    tests/test_utils_http.py E501 E502 E128 W504
+    tests/test_utils_http.py E501 E128 W504
     tests/test_utils_iterators.py E501 E128 E129 E303 E241
     tests/test_utils_log.py E741 E226
     tests/test_utils_python.py E501 E303 E731 E701 E305
@@ -243,7 +243,7 @@ flake8-ignore =
     tests/test_utils_template.py E305
     tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
     tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
-    tests/test_cmdline/__init__.py E502 E501
+    tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
     tests/test_spiderloader/__init__.py E128 E501
     tests/test_utils_misc/__init__.py E501
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 69e917004..ec78f7c91 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -67,7 +67,7 @@ def _pop_command_name(argv):
 
 def _print_header(settings, inproject):
     if inproject:
-        print("Scrapy %s - project: %s\n" % (scrapy.__version__, \
+        print("Scrapy %s - project: %s\n" % (scrapy.__version__,
                                              settings['BOT_NAME']))
     else:
         print("Scrapy %s - no active project\n" % scrapy.__version__)
@@ -123,7 +123,7 @@ def execute(argv=None, settings=None):
     inproject = inside_project()
     cmds = _get_commands_dict(settings, inproject)
     cmdname = _pop_command_name(argv)
-    parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
+    parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
                                    conflict_handler='resolve')
     if not cmdname:
         _print_commands(settings, inproject)
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 8a22ebabe..0e149941d 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -24,12 +24,11 @@ class Command(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("--spider", dest="spider",
-            help="use this spider")
-        parser.add_option("--headers", dest="headers", action="store_true", \
-            help="print response HTTP headers instead of body")
-        parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
-            default=False, help="do not handle HTTP 3xx status codes and print response as-is")
+        parser.add_option("--spider", dest="spider", help="use this spider")
+        parser.add_option("--headers", dest="headers", action="store_true",
+                          help="print response HTTP headers instead of body")
+        parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
+                          default=False, help="do not handle HTTP 3xx status codes and print response as-is")
 
     def _print_headers(self, headers, prefix):
         for key, values in headers.items():
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index e65131ae8..b123e5c84 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -43,8 +43,8 @@ class Command(ScrapyCommand):
                 return False
 
         if not re.search(r'^[_a-zA-Z]\w*$', project_name):
-            print('Error: Project names must begin with a letter and contain'\
-                    ' only\nletters, numbers and underscores')
+            print('Error: Project names must begin with a letter and contain'
+                  ' only\nletters, numbers and underscores')
         elif _module_exists(project_name):
             print('Error: Module %r already exists' % project_name)
         else:
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index e0d425874..3002fc702 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -86,8 +86,8 @@ class ReturnsContract(Contract):
             else:
                 expected = '%s..%s' % (self.min_bound, self.max_bound)
 
-            raise ContractFail("Returned %s %s, expected %s" % \
-                (occurrences, self.obj_name, expected))
+            raise ContractFail("Returned %s %s, expected %s" %
+                               (occurrences, self.obj_name, expected))
 
 
 class ScrapesContract(Contract):
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index e2a07bdef..d6fbd54ee 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -32,8 +32,8 @@ def _get_boto_connection():
 
 class S3DownloadHandler(object):
 
-    def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
-            httpdownloadhandler=HTTPDownloadHandler, **kw):
+    def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None,
+                 httpdownloadhandler=HTTPDownloadHandler, **kw):
 
         if not aws_access_key_id:
             aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index f368c3bae..fc796e8bb 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -88,9 +88,9 @@ class ScrapyHTTPPageGetter(HTTPClient):
         if self.factory.url.startswith(b'https'):
             self.transport.stopProducing()
 
-        self.factory.noPage(\
-                defer.TimeoutError("Getting %s took longer than %s seconds." % \
-                (self.factory.url, self.factory.timeout)))
+        self.factory.noPage(
+                defer.TimeoutError("Getting %s took longer than %s seconds." %
+                                   (self.factory.url, self.factory.timeout)))
 
 
 class ScrapyHTTPClientFactory(HTTPClientFactory):
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index e2ade8256..097a374bf 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -44,7 +44,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 try:
                     result = method(response=response, spider=spider)
                     if result is not None:
-                        raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \
+                        raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}'
                                              .format(fname(method), type(result)))
                 except _InvalidOutput:
                     raise
@@ -69,7 +69,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 elif result is None:
                     continue
                 else:
-                    raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \
+                    raise _InvalidOutput('Middleware {} must return None or an iterable, got {}'
                                          .format(fname(method), type(result)))
             return _failure
 
@@ -103,7 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 if _isiterable(result):
                     result = evaluate_iterable(result, method_index)
                 else:
-                    raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
+                    raise _InvalidOutput('Middleware {} must return an iterable, got {}'
                                          .format(fname(method), type(result)))
 
             return chain(result, recovered)
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 203dee42d..65b652953 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -37,8 +37,9 @@ class HttpCompressionMiddleware(object):
             if content_encoding:
                 encoding = content_encoding.pop()
                 decoded_body = self._decode(response.body, encoding.lower())
-                respcls = responsetypes.from_args(headers=response.headers, \
-                    url=response.url, body=decoded_body)
+                respcls = responsetypes.from_args(
+                    headers=response.headers, url=response.url, body=decoded_body
+                )
                 kwargs = dict(cls=respcls, body=decoded_body)
                 if issubclass(respcls, TextResponse):
                     # force recalculating the encoding until we make sure the
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index 9ccf356ec..afb2ed049 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -54,9 +54,9 @@ class CloseSpider(object):
             self.crawler.engine.close_spider(spider, 'closespider_pagecount')
 
     def spider_opened(self, spider):
-        self.task = reactor.callLater(self.close_on['timeout'], \
-            self.crawler.engine.close_spider, spider, \
-            reason='closespider_timeout')
+        self.task = reactor.callLater(self.close_on['timeout'],
+                                      self.crawler.engine.close_spider, spider,
+                                      reason='closespider_timeout')
 
     def item_scraped(self, item, spider):
         self.counter['itemcount'] += 1
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 4a3e74fbe..bc65f41cc 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -44,8 +44,7 @@ IGNORED_EXTENSIONS = [
 
 _re_type = type(re.compile("", 0))
 _matches = lambda url, regexs: any(r.search(url) for r in regexs)
-_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', \
-                                                       'file', 'ftp'}
+_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
 
 
 class FilteringLinkExtractor(object):
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 1cfd8a782..53fa435bb 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -65,8 +65,8 @@ class MiddlewareManager(object):
         return process_chain(self.methods[methodname], obj, *args)
 
     def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args):
-        return process_chain_both(self.methods[cb_methodname], \
-            self.methods[eb_methodname], obj, *args)
+        return process_chain_both(self.methods[cb_methodname],
+                                  self.methods[eb_methodname], obj, *args)
 
     def open_spider(self, spider):
         return self._process_parallel('open_spider', spider)
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 7a15e77ff..23306ca28 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -37,7 +37,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
         """Fail if a value in the components dict is not a real number or None."""
         for name, value in compdict.items():
             if value is not None and not isinstance(value, numbers.Real):
-                raise ValueError('Invalid value {} for component {}, please provide ' \
+                raise ValueError('Invalid value {} for component {}, please provide '
                                  'a real number or None instead'.format(value, name))
 
     # BEGIN Backward compatibility for old (base, custom) call signature
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 909ea90e0..da99a6be8 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -25,17 +25,15 @@ class CmdlineTest(unittest.TestCase):
         return comm.decode(encoding)
 
     def test_default_settings(self):
-        self.assertEqual(self._execute('settings', '--get', 'TEST1'), \
-                         'default')
+        self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'default')
 
     def test_override_settings_using_set_arg(self):
-        self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s', 'TEST1=override'), \
-                         'override')
+        self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s',
+                                       'TEST1=override'), 'override')
 
     def test_override_settings_using_envvar(self):
         self.env['SCRAPY_TEST1'] = 'override'
-        self.assertEqual(self._execute('settings', '--get', 'TEST1'), \
-                         'override')
+        self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override')
 
     def test_profiling(self):
         path = tempfile.mkdtemp()
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 2db2417e8..82d7b18d6 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -800,8 +800,8 @@ class S3TestCase(unittest.TestCase):
         req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date})
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
-        self.assertEqual(httpreq.headers['Authorization'], \
-                b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
+        self.assertEqual(httpreq.headers['Authorization'],
+                         b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
 
     def test_request_signing2(self):
         # puts an object into the johnsmith bucket.
@@ -813,21 +813,22 @@ class S3TestCase(unittest.TestCase):
             })
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
-        self.assertEqual(httpreq.headers['Authorization'], \
-                b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
+        self.assertEqual(httpreq.headers['Authorization'],
+                         b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
 
     def test_request_signing3(self):
         # lists the content of the johnsmith bucket.
         date = 'Tue, 27 Mar 2007 19:42:41 +0000'
-        req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \
-                method='GET', headers={
-                    'User-Agent': 'Mozilla/5.0',
-                    'Date': date,
-                    })
+        req = Request(
+            's3://johnsmith/?prefix=photos&max-keys=50&marker=puppy',
+            method='GET', headers={
+                'User-Agent': 'Mozilla/5.0',
+                'Date': date,
+            })
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
-        self.assertEqual(httpreq.headers['Authorization'], \
-                b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
+        self.assertEqual(httpreq.headers['Authorization'],
+                         b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
 
     def test_request_signing4(self):
         # fetches the access control policy sub-resource for the 'johnsmith' bucket.
@@ -836,8 +837,8 @@ class S3TestCase(unittest.TestCase):
             method='GET', headers={'Date': date})
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
-        self.assertEqual(httpreq.headers['Authorization'], \
-                b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
+        self.assertEqual(httpreq.headers['Authorization'],
+                         b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
 
     def test_request_signing5(self):
         try:
@@ -850,11 +851,11 @@ class S3TestCase(unittest.TestCase):
         # deletes an object from the 'johnsmith' bucket using the
         # path-style and Date alternative.
         date = 'Tue, 27 Mar 2007 21:20:27 +0000'
-        req = Request('s3://johnsmith/photos/puppy.jpg', \
-                method='DELETE', headers={
-                    'Date': date,
-                    'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
-                    })
+        req = Request(
+            's3://johnsmith/photos/puppy.jpg', method='DELETE', headers={
+                'Date': date,
+                'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
+            })
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
         # botocore does not override Date with x-amz-date
@@ -864,25 +865,26 @@ class S3TestCase(unittest.TestCase):
     def test_request_signing6(self):
         # uploads an object to a CNAME style virtual hosted bucket with metadata.
         date = 'Tue, 27 Mar 2007 21:06:08 +0000'
-        req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \
-                method='PUT', headers={
-                    'User-Agent': 'curl/7.15.5',
-                    'Host': 'static.johnsmith.net:8080',
-                    'Date': date,
-                    'x-amz-acl': 'public-read',
-                    'content-type': 'application/x-download',
-                    'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',
-                    'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net',
-                    'X-Amz-Meta-FileChecksum': '0x02661779',
-                    'X-Amz-Meta-ChecksumAlgorithm': 'crc32',
-                    'Content-Disposition': 'attachment; filename=database.dat',
-                    'Content-Encoding': 'gzip',
-                    'Content-Length': '5913339',
-                    })
+        req = Request(
+            's3://static.johnsmith.net:8080/db-backup.dat.gz',
+            method='PUT', headers={
+                'User-Agent': 'curl/7.15.5',
+                'Host': 'static.johnsmith.net:8080',
+                'Date': date,
+                'x-amz-acl': 'public-read',
+                'content-type': 'application/x-download',
+                'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',
+                'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net',
+                'X-Amz-Meta-FileChecksum': '0x02661779',
+                'X-Amz-Meta-ChecksumAlgorithm': 'crc32',
+                'Content-Disposition': 'attachment; filename=database.dat',
+                'Content-Encoding': 'gzip',
+                'Content-Length': '5913339',
+            })
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
-        self.assertEqual(httpreq.headers['Authorization'], \
-                b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
+        self.assertEqual(httpreq.headers['Authorization'],
+                         b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
 
     def test_request_signing7(self):
         # ensure that spaces are quoted properly before signing
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 51b79b6c3..e09d66086 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -165,12 +165,12 @@ class MaxRetryTimesTest(unittest.TestCase):
         # SETTINGS: meta(max_retry_times) = 4
         meta_max_retry_times = 4
 
-        req = Request(self.invalid_url, meta= \
-            {'max_retry_times': meta_max_retry_times, 'dont_retry': True})
+        req = Request(self.invalid_url, meta={
+            'max_retry_times': meta_max_retry_times, 'dont_retry': True
+        })
 
         self._test_retry(req, DNSLookupError('foo'), 0)
 
-
     def _test_retry(self, req, exception, max_retry_times):
         
         for i in range(0, max_retry_times):
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 537df8d91..a48b63025 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -91,8 +91,8 @@ def start_test_site(debug=False):
 
     port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1")
     if debug:
-        print("Test server running at http://localhost:%d/ - hit Ctrl-C to finish." \
-            % port.getHost().port)
+        print("Test server running at http://localhost:%d/ - hit Ctrl-C to finish."
+              % port.getHost().port)
     return port
 
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 57e7b457d..e30417b30 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -622,8 +622,9 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="two" value="3">
             <input type="submit" name="clickable2" value="clicked2">
             </form>""")
-        req = self.request_class.from_response(response, formdata={'two': '2'}, \
-                                              clickdata={'name': 'clickable2'})
+        req = self.request_class.from_response(
+            response, formdata={'two': '2'}, clickdata={'name': 'clickable2'}
+        )
         fs = _qs(req)
         self.assertEqual(fs[b'clickable2'], [b'clicked2'])
         self.assertFalse(b'clickable1' in fs, fs)
@@ -671,8 +672,9 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="one" value="clicked1">
             <input type="hidden" name="two" value="clicked2">
             </form>""")
-        req = self.request_class.from_response(response, \
-                clickdata={u'name': u'clickable', u'value': u'clicked2'})
+        req = self.request_class.from_response(
+            response, clickdata={u'name': u'clickable', u'value': u'clicked2'}
+        )
         fs = _qs(req)
         self.assertEqual(fs[b'clickable'], [b'clicked2'])
         self.assertEqual(fs[b'one'], [b'clicked1'])
@@ -686,8 +688,9 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="poundsign" value="\u00a3">
             <input type="hidden" name="eurosign" value="\u20ac">
             </form>""")
-        req = self.request_class.from_response(response, \
-                clickdata={u'name': u'price in \u00a3'})
+        req = self.request_class.from_response(
+            response, clickdata={u'name': u'price in \u00a3'}
+        )
         fs = _qs(req, to_unicode=True)
         self.assertTrue(fs[u'price in \u00a3'])
 
@@ -700,8 +703,9 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="yensign" value="\u00a5">
             </form>""",
             encoding='latin1')
-        req = self.request_class.from_response(response, \
-                clickdata={u'name': u'price in \u00a5'})
+        req = self.request_class.from_response(
+            response, clickdata={u'name': u'price in \u00a5'}
+        )
         fs = _qs(req, to_unicode=True, encoding='latin1')
         self.assertTrue(fs[u'price in \u00a5'])
 
@@ -716,8 +720,9 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="field2" value="value2">
             </form>
             """)
-        req = self.request_class.from_response(response, formname='form2', \
-                clickdata={u'name': u'clickable'})
+        req = self.request_class.from_response(
+            response, formname='form2', clickdata={u'name': u'clickable'}
+        )
         fs = _qs(req)
         self.assertEqual(fs[b'clickable'], [b'clicked2'])
         self.assertEqual(fs[b'field2'], [b'value2'])
@@ -725,8 +730,9 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_override_clickable(self):
         response = _buildresponse('''<form><input type="submit" name="clickme" value="one"> </form>''')
-        req = self.request_class.from_response(response, \
-                formdata={'clickme': 'two'}, clickdata={'name': 'clickme'})
+        req = self.request_class.from_response(
+            response, formdata={'clickme': 'two'}, clickdata={'name': 'clickme'}
+        )
         fs = _qs(req)
         self.assertEqual(fs[b'clickme'], [b'two'])
 
@@ -853,7 +859,7 @@ class FormRequestTest(RequestTest):
             <form name="form2" action="post.php" method="POST">
             <input type="hidden" name="two" value="2">
             </form>""")
-        self.assertRaises(IndexError, self.request_class.from_response, \
+        self.assertRaises(IndexError, self.request_class.from_response,
                           response, formname="form3", formnumber=2)
 
     def test_from_response_formid_exists(self):
@@ -907,7 +913,7 @@ class FormRequestTest(RequestTest):
             <form id="form2" name="form2" action="post.php" method="POST">
             <input type="hidden" name="two" value="2">
             </form>""")
-        self.assertRaises(IndexError, self.request_class.from_response, \
+        self.assertRaises(IndexError, self.request_class.from_response,
                           response, formid="form3", formnumber=2)
 
     def test_from_response_select(self):
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 79bb745cc..960ecea3e 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -316,8 +316,8 @@ class TextResponseTest(BaseResponseTest):
         assert u'SUFFIX' in r.text, repr(r.text)
 
         # Do not destroy html tags due to encoding bugs
-        r = self.response_class("http://example.com", encoding='utf-8', \
-                body=b'\xf0<span>value</span>')
+        r = self.response_class("http://example.com", encoding='utf-8',
+                                body=b'\xf0<span>value</span>')
         assert u'<span>value</span>' in r.text, repr(r.text)
 
         # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py
index f9af4bf87..2fac3da1f 100644
--- a/tests/test_utils_http.py
+++ b/tests/test_utils_http.py
@@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase):
         chunked_body += "8\r\n" + "sequence\r\n"
         chunked_body += "0\r\n\r\n"
         body = decode_chunked_transfer(chunked_body)
-        self.assertEqual(body, \
-            "This is the data in the first chunk\r\n" +
-            "and this is the second one\r\n" +
-            "consequence")
+        self.assertEqual(body,
+                         "This is the data in the first chunk\r\n" +
+                         "and this is the second one\r\n" +
+                         "consequence")

From 2d92a39003a5a0b01b265b87cf722a61f121ca60 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 18 Dec 2019 12:07:08 +0500
Subject: [PATCH 2492/2974] Restore test_download_with_proxy_https_noconnect,
 check for a warning there.

---
 tests/test_downloader_handlers.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 45d4aa952..87ab4c9e5 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -33,7 +33,7 @@ from scrapy.responsetypes import responsetypes
 from scrapy.settings import Settings
 from scrapy.utils.test import get_crawler, skip_if_no_boto
 from scrapy.utils.python import to_bytes
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 
 from tests.mockserver import MockServer, ssl_context_factory, Echo
 from tests.spiders import SingleRequestSpider
@@ -687,6 +687,18 @@ class HttpProxyTestCase(unittest.TestCase):
         request = Request('http://example.com', meta={'proxy': http_proxy})
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
+    def test_download_with_proxy_https_noconnect(self):
+        def _test(response):
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.url, request.url)
+            self.assertEqual(response.body, b'https://example.com')
+
+        http_proxy = '%s?noconnect' % self.getURL('')
+        request = Request('https://example.com', meta={'proxy': http_proxy})
+        with self.assertWarnsRegex(ScrapyDeprecationWarning,
+                                   r'Using HTTPS proxies in the noconnect mode is deprecated'):
+            return self.download_request(request, Spider('foo')).addCallback(_test)
+
     def test_download_without_proxy(self):
         def _test(response):
             self.assertEqual(response.status, 200)

From bb2ff13e4c7c80cfc7925f60eadc97dec0d69026 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 18 Dec 2019 15:39:08 +0500
Subject: [PATCH 2493/2974] Skip
 Http10ProxyTestCase.test_download_with_proxy_https_noconnect

---
 tests/test_downloader_handlers.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 87ab4c9e5..412a9c084 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -712,6 +712,8 @@ class HttpProxyTestCase(unittest.TestCase):
 class Http10ProxyTestCase(HttpProxyTestCase):
     download_handler_cls = HTTP10DownloadHandler
 
+    def test_download_with_proxy_https_noconnect(self):
+        raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler')
 
 class Http11ProxyTestCase(HttpProxyTestCase):
     download_handler_cls = HTTP11DownloadHandler

From ac302c3f615d339ed36000231ca3e1e1c347478a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 18 Dec 2019 15:43:05 +0500
Subject: [PATCH 2494/2974] Fix a flake8 problem.

---
 tests/test_downloader_handlers.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 412a9c084..7412d7ebf 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -715,6 +715,7 @@ class Http10ProxyTestCase(HttpProxyTestCase):
     def test_download_with_proxy_https_noconnect(self):
         raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler')
 
+
 class Http11ProxyTestCase(HttpProxyTestCase):
     download_handler_cls = HTTP11DownloadHandler
 

From 12f9ffeb5d1c1f4d02f616852dcae82ef633e8d9 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Wed, 18 Dec 2019 10:53:27 +0000
Subject: [PATCH 2495/2974] remove requirements-py3.txt

---
 requirements-py3.txt | 16 ----------------
 tox.ini              |  1 -
 2 files changed, 17 deletions(-)
 delete mode 100644 requirements-py3.txt

diff --git a/requirements-py3.txt b/requirements-py3.txt
deleted file mode 100644
index 28c649e28..000000000
--- a/requirements-py3.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-parsel>=1.5.0
-PyDispatcher>=2.0.5
-Twisted>=17.9.0
-w3lib>=1.17.0
-
-pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
-queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
-cryptography>=2.0  # Earlier versions would fail to install
-
-# Reference versions taken from
-# https://packages.ubuntu.com/xenial/python/
-# https://packages.ubuntu.com/xenial/zope/
-cssselect>=0.9.1
-lxml>=3.5.0
-service_identity>=16.0.0
-zope.interface>=4.1.3
diff --git a/tox.ini b/tox.ini
index fd75d18e2..f37c381d0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,7 +9,6 @@ envlist = py35
 [testenv]
 deps =
     -ctests/constraints.txt
-    -rrequirements-py3.txt
     -rtests/requirements-py3.txt
     # Extras
     botocore>=1.3.23

From ee9881d2704798c9cd61b6da503bb0694227c58c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 18 Dec 2019 12:08:34 +0100
Subject: [PATCH 2496/2974] Improve FilteringLinkExtractor.__new__

---
 scrapy/linkextractors/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index a510fef70..7254bd79c 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -61,7 +61,7 @@ class FilteringLinkExtractor(object):
             warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
                  'please use scrapy.linkextractors.LinkExtractor instead',
                  ScrapyDeprecationWarning, stacklevel=2)
-        return super(FilteringLinkExtractor, cls).__new__(cls)
+        return super().__new__(cls, *args, **kwargs)
 
     def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
                  restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):

From 174769a3f08fcd84eaec8a88217a05f8ebc3f2cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 18 Dec 2019 12:09:03 +0100
Subject: [PATCH 2497/2974] Use a better name for the LxmlLinkExtractor
 subclassing test

---
 tests/test_linkextractors.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 0ffeaecc3..cfd4c6b85 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -517,10 +517,10 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
             LxmlLinkExtractor()
             self.assertEqual(len(warnings), 0)
 
-            class SubclassedItem(LxmlLinkExtractor):
+            class SubclassedLxmlLinkExtractor(LxmlLinkExtractor):
                 pass
 
-            SubclassedItem()
+            SubclassedLxmlLinkExtractor()
             self.assertEqual(len(warnings), 0)
 
 

From 012533924a799c7bf1d3f2d4a489d8ca1ac462f8 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Wed, 18 Dec 2019 11:13:36 +0000
Subject: [PATCH 2498/2974] remove requirements from here too

---
 docs/requirements.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/requirements.txt b/docs/requirements.txt
index 0ed11c4dc..773b92cea 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,4 +1,3 @@
--r ../requirements-py3.txt
 Sphinx>=2.1
 sphinx-hoverxref
 sphinx-notfound-page

From 7ccb169a27ccbedf3145b824e1bd655cb6902f32 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 18 Dec 2019 19:41:16 +0500
Subject: [PATCH 2499/2974] Split a long test in test_engine.py into three.

---
 tests/test_engine.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index a48b63025..25dee7c1f 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -179,7 +179,6 @@ class EngineTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_crawler(self):
-
         for spider in TestSpider, DictItemsSpider:
             self.run = CrawlerRun(spider)
             yield self.run.run()
@@ -189,11 +188,15 @@ class EngineTest(unittest.TestCase):
             self._assert_scraped_items()
             self._assert_signals_catched()
 
+    @defer.inlineCallbacks
+    def test_crawler_dupefilter(self):
         self.run = CrawlerRun(TestDupeFilterSpider)
         yield self.run.run()
         self._assert_scheduled_requests(urls_to_visit=7)
         self._assert_dropped_requests()
 
+    @defer.inlineCallbacks
+    def test_crawler_itemerror(self):
         self.run = CrawlerRun(ItemZeroDivisionErrorSpider)
         yield self.run.run()
         self._assert_items_error()

From 916382e109dc06a57f4631448555953d1fa540b5 Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Wed, 18 Dec 2019 12:05:33 -0300
Subject: [PATCH 2500/2974] Add errback parameter to scrapy.spiders.crawl.Rule
 (#4000)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add errback parameter to scrapy.spiders.crawl.Rule

* CrawlSpider: optimize by reducing iterations

* [test] Rule.errback

* [doc] Rule.errback

* [doc] Use autoclass in docs/topics/spiders.rst

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Rule.process_links takes a list

* Fix aesthetic issue reported by Flake8
---
 docs/topics/spiders.rst |  6 +++++
 scrapy/spiders/crawl.py | 60 ++++++++++++++++++++++++++---------------
 tests/mockserver.py     |  7 +++++
 tests/spiders.py        | 36 +++++++++++++++++++++++--
 tests/test_crawl.py     | 18 ++++++++++---
 5 files changed, 101 insertions(+), 26 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index d65a43afd..b0fb14e24 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -414,6 +414,12 @@ Crawling rules
    from which the request originated as second argument. It must return a
    ``Request`` object or ``None`` (to filter out the request).
 
+   ``errback`` is a callable or a string (in which case a method from the spider
+   object with that name will be used) to be called if any exception is
+   raised while processing a request generated by the rule.
+   It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
+   instance as first parameter.
+
 CrawlSpider example
 ~~~~~~~~~~~~~~~~~~~
 
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index a5eb1a518..a2c364c0e 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -16,7 +16,11 @@ from scrapy.utils.python import get_func_args
 from scrapy.utils.spider import iterate_spider_output
 
 
-def _identity(request, response):
+def _identity(x):
+    return x
+
+
+def _identity_process_request(request, response):
     return request
 
 
@@ -32,17 +36,20 @@ _default_link_extractor = LinkExtractor()
 
 class Rule(object):
 
-    def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None):
+    def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None,
+                 process_links=None, process_request=None, errback=None):
         self.link_extractor = link_extractor or _default_link_extractor
         self.callback = callback
+        self.errback = errback
         self.cb_kwargs = cb_kwargs or {}
-        self.process_links = process_links
-        self.process_request = process_request or _identity
+        self.process_links = process_links or _identity
+        self.process_request = process_request or _identity_process_request
         self.process_request_argcount = None
         self.follow = follow if follow is not None else not callback
 
     def _compile(self, spider):
         self.callback = _get_method(self.callback, spider)
+        self.errback = _get_method(self.errback, spider)
         self.process_links = _get_method(self.process_links, spider)
         self.process_request = _get_method(self.process_request, spider)
         self.process_request_argcount = len(get_func_args(self.process_request))
@@ -76,48 +83,59 @@ class CrawlSpider(Spider):
     def process_results(self, response, results):
         return results
 
-    def _build_request(self, rule, link):
-        r = Request(url=link.url, callback=self._response_downloaded)
-        r.meta.update(rule=rule, link_text=link.text)
-        return r
+    def _build_request(self, rule_index, link):
+        return Request(
+            url=link.url,
+            callback=self._callback,
+            errback=self._errback,
+            meta=dict(rule=rule_index, link_text=link.text),
+        )
 
     def _requests_to_follow(self, response):
         if not isinstance(response, HtmlResponse):
             return
         seen = set()
-        for n, rule in enumerate(self._rules):
+        for rule_index, rule in enumerate(self._rules):
             links = [lnk for lnk in rule.link_extractor.extract_links(response)
                      if lnk not in seen]
-            if links and rule.process_links:
-                links = rule.process_links(links)
-            for link in links:
+            for link in rule.process_links(links):
                 seen.add(link)
-                request = self._build_request(n, link)
+                request = self._build_request(rule_index, link)
                 yield rule._process_request(request, response)
 
-    def _response_downloaded(self, response):
+    def _callback(self, response):
         rule = self._rules[response.meta['rule']]
         return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow)
 
+    def _errback(self, failure):
+        rule = self._rules[failure.request.meta['rule']]
+        return self._handle_failure(failure, rule.errback)
+
     def _parse_response(self, response, callback, cb_kwargs, follow=True):
         if callback:
             cb_res = callback(response, **cb_kwargs) or ()
             cb_res = self.process_results(response, cb_res)
-            for requests_or_item in iterate_spider_output(cb_res):
-                yield requests_or_item
+            for request_or_item in iterate_spider_output(cb_res):
+                yield request_or_item
 
         if follow and self._follow_links:
             for request_or_item in self._requests_to_follow(response):
                 yield request_or_item
 
+    def _handle_failure(self, failure, errback):
+        if errback:
+            results = errback(failure) or ()
+            for request_or_item in iterate_spider_output(results):
+                yield request_or_item
+
     def _compile_rules(self):
-        self._rules = [copy.copy(r) for r in self.rules]
-        for rule in self._rules:
-            rule._compile(self)
+        self._rules = []
+        for rule in self.rules:
+            self._rules.append(copy.copy(rule))
+            self._rules[-1]._compile(self)
 
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):
         spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)
-        spider._follow_links = crawler.settings.getbool(
-            'CRAWLSPIDER_FOLLOW_LINKS', True)
+        spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
         return spider
diff --git a/tests/mockserver.py b/tests/mockserver.py
index fe28176d4..a45277db9 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -164,6 +164,12 @@ class Drop(Partial):
             request.finish()
 
 
+class ArbitraryLengthPayloadResource(LeafResource):
+
+    def render(self, request):
+        return request.content.read()
+
+
 class Root(Resource):
 
     def __init__(self):
@@ -177,6 +183,7 @@ class Root(Resource):
         self.putChild(b"echo", Echo())
         self.putChild(b"payload", PayloadResource())
         self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
+        self.putChild(b"alpayload", ArbitraryLengthPayloadResource())
         try:
             from tests import tests_datadir
             self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/')))
diff --git a/tests/spiders.py b/tests/spiders.py
index 981bd2eb8..39c8da0b6 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -1,14 +1,14 @@
 """
 Some spiders used for testing and benchmarking
 """
-
 import time
 from urllib.parse import urlencode
 
-from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.item import Item
 from scrapy.linkextractors import LinkExtractor
+from scrapy.spiders import Spider
+from scrapy.spiders.crawl import CrawlSpider, Rule
 
 
 class MockServerSpider(Spider):
@@ -184,3 +184,35 @@ class DuplicateStartRequestsSpider(MockServerSpider):
 
     def parse(self, response):
         self.visited += 1
+
+
+class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider):
+    name = 'crawl_spider_with_errback'
+    custom_settings = {
+        'RETRY_HTTP_CODES': [],  # no need to retry
+    }
+    rules = (
+        Rule(LinkExtractor(), callback='callback', errback='errback', follow=True),
+    )
+
+    def start_requests(self):
+        test_body = b"""
+        <html>
+            <head><title>Page title<title></head>
+            <body>
+                <p><a href="/status?n=200">Item 200</a></p>  <!-- callback -->
+                <p><a href="/status?n=201">Item 201</a></p>  <!-- callback -->
+                <p><a href="/status?n=404">Item 404</a></p>  <!-- errback -->
+                <p><a href="/status?n=500">Item 500</a></p>  <!-- errback -->
+                <p><a href="/status?n=501">Item 501</a></p>  <!-- errback -->
+            </body>
+        </html>
+        """
+        url = self.mockserver.url("/alpayload")
+        yield Request(url, method="POST", body=test_body)
+
+    def callback(self, response):
+        self.logger.info('[callback] status %i', response.status)
+
+    def errback(self, failure):
+        self.logger.info('[errback] status %i', failure.value.response.status)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3307899b7..76f87458b 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -5,12 +5,12 @@ from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 
-from scrapy.http import Request
 from scrapy.crawler import CrawlerRunner
+from scrapy.http import Request
 from scrapy.utils.python import to_unicode
-from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
-    BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
 from tests.mockserver import MockServer
+from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
+                           SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback)
 
 
 class CrawlTestCase(TestCase):
@@ -277,3 +277,15 @@ with multiples lines
 
         self._assert_retried(log)
         self.assertIn("Got response 200", str(log))
+
+    @defer.inlineCallbacks
+    def test_crawlspider_with_errback(self):
+        self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver)
+
+        with LogCapture() as log:
+            yield self.runner.join()
+
+        self.assertIn("[callback] status 200", str(log))
+        self.assertIn("[callback] status 201", str(log))
+        self.assertIn("[errback] status 404", str(log))
+        self.assertIn("[errback] status 500", str(log))

From a5de2c64e6f1233a98b1b180606fca0ae4dd0871 Mon Sep 17 00:00:00 2001
From: Marc Hernandez Cabot <noviluni@gmail.com>
Date: Wed, 18 Dec 2019 16:24:48 +0100
Subject: [PATCH 2501/2974] fix W291, W292, W293 (whitespaces)

---
 pytest.ini                                  | 27 ++++++++++-----------
 scrapy/http/response/__init__.py            |  4 +--
 scrapy/http/response/text.py                |  4 +--
 scrapy/logformatter.py                      |  4 +--
 scrapy/utils/markup.py                      |  2 +-
 scrapy/utils/multipart.py                   |  2 +-
 tests/test_contracts.py                     |  2 +-
 tests/test_downloadermiddleware_retry.py    |  8 +++---
 tests/test_dupefilters.py                   |  6 ++---
 tests/test_pipeline_files.py                |  1 -
 tests/test_robotstxt_interface.py           |  2 +-
 tests/test_spidermiddleware_offsite.py      |  2 +-
 tests/test_spidermiddleware_output_chain.py |  8 +++---
 13 files changed, 35 insertions(+), 37 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index f088e10ef..ac3c8cfb5 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -85,8 +85,8 @@ flake8-ignore =
     scrapy/http/request/__init__.py E501
     scrapy/http/request/form.py E501 E123
     scrapy/http/request/json_request.py E501
-    scrapy/http/response/__init__.py E501 E128 W293 W291
-    scrapy/http/response/text.py E501 W293 E128 E124
+    scrapy/http/response/__init__.py E501 E128
+    scrapy/http/response/text.py E501 E128 E124
     # scrapy/linkextractors
     scrapy/linkextractors/__init__.py E731 E501 E402
     scrapy/linkextractors/lxmlhtml.py E501 E731 E226
@@ -127,9 +127,9 @@ flake8-ignore =
     scrapy/utils/httpobj.py E501
     scrapy/utils/iterators.py E501 E701
     scrapy/utils/log.py E128 W503
-    scrapy/utils/markup.py F403 W292
+    scrapy/utils/markup.py F403
     scrapy/utils/misc.py E501 E226
-    scrapy/utils/multipart.py F403 W292
+    scrapy/utils/multipart.py F403
     scrapy/utils/project.py E501
     scrapy/utils/python.py E501
     scrapy/utils/reactor.py E226
@@ -144,7 +144,6 @@ flake8-ignore =
     scrapy/utils/url.py E501 F403 E128 F405
     # scrapy
     scrapy/__init__.py E402 E501
-    scrapy/_monkeypatches.py W293
     scrapy/cmdline.py E501
     scrapy/crawler.py E501
     scrapy/dupefilters.py E501 E202
@@ -153,7 +152,7 @@ flake8-ignore =
     scrapy/interfaces.py E501
     scrapy/item.py E501 E128
     scrapy/link.py E501
-    scrapy/logformatter.py E501 W293
+    scrapy/logformatter.py E501
     scrapy/mail.py E402 E128 E501 E502
     scrapy/middleware.py E128 E501
     scrapy/pqueues.py E501
@@ -174,7 +173,7 @@ flake8-ignore =
     tests/test_command_parse.py E501 E128 E303 E226
     tests/test_command_shell.py E501 E128
     tests/test_commands.py E128 E501
-    tests/test_contracts.py E501 E128 W293
+    tests/test_contracts.py E501 E128
     tests/test_crawl.py E501 E741 E265
     tests/test_crawler.py F841 E306 E501
     tests/test_dependencies.py F841 E501 E305
@@ -189,17 +188,17 @@ flake8-ignore =
     tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
     tests/test_downloadermiddleware_httpproxy.py E501 E128
     tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
-    tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E303 E126
+    tests/test_downloadermiddleware_retry.py E501 E128 E251 E303 E126
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
-    tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
+    tests/test_dupefilters.py E221 E501 E741 E128 E124
     tests/test_engine.py E401 E501 E128
     tests/test_exporters.py E501 E731 E306 E128 E124
     tests/test_extension_telnet.py F841
     tests/test_feedexport.py E501 F841 E241
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
-    tests/test_http_request.py E402 E501 E127 E128 W293 E128 E126 E123
+    tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
     tests/test_http_response.py E501 E301 E128 E265
     tests/test_item.py E701 E128 F841 E306
     tests/test_link.py E501
@@ -209,20 +208,20 @@ flake8-ignore =
     tests/test_mail.py E128 E501 E305
     tests/test_middleware.py E501 E128
     tests/test_pipeline_crawl.py E131 E501 E128 E126
-    tests/test_pipeline_files.py E501 W293 E303 E272 E226
+    tests/test_pipeline_files.py E501 E303 E272 E226
     tests/test_pipeline_images.py F841 E501 E303
     tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502
     tests/test_proxy_connect.py E501 E741
     tests/test_request_cb_kwargs.py E501
     tests/test_responsetypes.py E501 E305
-    tests/test_robotstxt_interface.py E501 W291 E501
+    tests/test_robotstxt_interface.py E501 E501
     tests/test_scheduler.py E501 E126 E123
     tests/test_selector.py E501 E127
     tests/test_spider.py E501
     tests/test_spidermiddleware.py E501 E226
     tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
-    tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
-    tests/test_spidermiddleware_output_chain.py E501 W293 E226
+    tests/test_spidermiddleware_offsite.py E501 E128 E111
+    tests/test_spidermiddleware_output_chain.py E501 E226
     tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
     tests/test_squeues.py E501 E701 E741
     tests/test_utils_conf.py E501 E303 E128
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 64e9c6c20..e79ce9acc 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -113,8 +113,8 @@ class Response(object_ref):
         It accepts the same arguments as ``Request.__init__`` method,
         but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
         not only an absolute URL.
-        
-        :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow` 
+
+        :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
         method which supports selectors in addition to absolute/relative URLs
         and Link objects.
         """
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 1079fd6e8..4f9afde87 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -125,7 +125,7 @@ class TextResponse(Response):
         Return a :class:`~.Request` instance to follow a link ``url``.
         It accepts the same arguments as ``Request.__init__`` method,
         but ``url`` can be not only an absolute URL, but also
-        
+
         * a relative URL;
         * a scrapy.link.Link object (e.g. a link extractor result);
         * an attribute Selector (not SelectorList) - e.g.
@@ -133,7 +133,7 @@ class TextResponse(Response):
           ``response.xpath('//img/@src')[0]``.
         * a Selector for ``<a>`` or ``<link>`` element, e.g.
           ``response.css('a.my_link')[0]``.
-          
+
         See :ref:`response-follow-example` for usage examples.
         """
         if isinstance(url, parsel.Selector):
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 5189d7cfa..4e5963e99 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -13,7 +13,7 @@ ERRORMSG = u"'Error processing %(item)s'"
 
 class LogFormatter(object):
     """Class for generating log messages for different actions.
-    
+
     All methods must return a dictionary listing the parameters ``level``, ``msg``
     and ``args`` which are going to be used for constructing the log message when
     calling ``logging.log``.
@@ -48,7 +48,7 @@ class LogFormatter(object):
                         }
                     }
     """
-    
+
     def crawled(self, request, response, spider):
         """Logs a message when the crawler finds a webpage."""
         request_flags = ' %s' % str(request.flags) if request.flags else ''
diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py
index 2455fcc16..9728c542a 100644
--- a/scrapy/utils/markup.py
+++ b/scrapy/utils/markup.py
@@ -11,4 +11,4 @@ from w3lib.html import *  # noqa: F401
 
 warnings.warn("Module `scrapy.utils.markup` is deprecated. "
               "Please import from `w3lib.html` instead.",
-              ScrapyDeprecationWarning, stacklevel=2)
\ No newline at end of file
+              ScrapyDeprecationWarning, stacklevel=2)
diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py
index e81f63152..5dcf791b8 100644
--- a/scrapy/utils/multipart.py
+++ b/scrapy/utils/multipart.py
@@ -12,4 +12,4 @@ from w3lib.form import *  # noqa: F401
 warnings.warn("Module `scrapy.utils.multipart` is deprecated. "
               "If you're using `encode_multipart` function, please use "
               "`urllib3.filepost.encode_multipart_formdata` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
\ No newline at end of file
+              ScrapyDeprecationWarning, stacklevel=2)
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 582e3d052..11d41c1fe 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -252,7 +252,7 @@ class ContractsManagerTest(unittest.TestCase):
         self.assertEqual(len(contracts), 3)
         self.assertEqual(frozenset(type(x) for x in contracts),
                          frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))
-        
+
         contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs)
         self.assertEqual(len(contracts), 3)
         self.assertEqual(frozenset(type(x) for x in contracts),
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index e09d66086..9c989977e 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -124,7 +124,7 @@ class MaxRetryTimesTest(unittest.TestCase):
 
         # SETTINGS: meta(max_retry_times) = 0
         meta_max_retry_times = 0
-        
+
         req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
         self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times)
 
@@ -137,7 +137,7 @@ class MaxRetryTimesTest(unittest.TestCase):
         self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times)
 
     def test_with_metakey_greater(self):
-        
+
         # SETINGS: RETRY_TIMES < meta(max_retry_times)
         self.mw.max_retry_times = 2
         meta_max_retry_times = 3
@@ -149,7 +149,7 @@ class MaxRetryTimesTest(unittest.TestCase):
         self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times)
 
     def test_with_metakey_lesser(self):
-        
+
         # SETINGS: RETRY_TIMES > meta(max_retry_times)
         self.mw.max_retry_times = 5
         meta_max_retry_times = 4
@@ -172,7 +172,7 @@ class MaxRetryTimesTest(unittest.TestCase):
         self._test_retry(req, DNSLookupError('foo'), 0)
 
     def _test_retry(self, req, exception, max_retry_times):
-        
+
         for i in range(0, max_retry_times):
             req = self.mw.process_exception(req, exception, self.spider)
             assert isinstance(req, Request)
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index e4b0bdf83..0546558bc 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -142,12 +142,12 @@ class RFPDupeFilterTest(unittest.TestCase):
 
             r1 = Request('http://scrapytest.org/index.html')
             r2 = Request('http://scrapytest.org/index.html')
-            
+
             dupefilter.log(r1, spider)
             dupefilter.log(r2, spider)
 
             assert crawler.stats.get_value('dupefilter/filtered') == 2
-            l.check_present(('scrapy.dupefilters', 'DEBUG', 
+            l.check_present(('scrapy.dupefilters', 'DEBUG',
                 ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
                 ' - no more duplicates will be shown'
                 ' (see DUPEFILTER_DEBUG to show all duplicates)')))
@@ -169,7 +169,7 @@ class RFPDupeFilterTest(unittest.TestCase):
             r2 = Request('http://scrapytest.org/index.html',
                 headers={'Referer': 'http://scrapytest.org/INDEX.html'}
             )
-            
+
             dupefilter.log(r1, spider)
             dupefilter.log(r2, spider)
 
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 52f2b554e..141141671 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -58,7 +58,6 @@ class FilesPipelineTestCase(unittest.TestCase):
         self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
                                     //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")),
                          'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')
-                         
 
     def test_fs_store(self):
         assert isinstance(self.pipeline.store, FSFilesStore)
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 27d79437b..24aaaf7ec 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -44,7 +44,7 @@ class BaseRobotParserTest:
 
     def test_allowed_wildcards(self):
         robotstxt_robotstxt_body = """User-agent: first
-                                Disallow: /disallowed/*/end$    
+                                Disallow: /disallowed/*/end$
 
                                 User-agent: second
                                 Allow: /*allowed
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 992e60be2..7511aa568 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -73,7 +73,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
 
 
 class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
-    
+
     def test_get_host_regex(self):
         self.spider.allowed_domains = ['http://scrapytest.org', 'scrapy.org', 'scrapy.test.org']
         with warnings.catch_warnings(record=True) as w:
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 5b7b5e7aa..739cf1c2d 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -156,7 +156,7 @@ class GeneratorFailMiddleware:
             r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__))
             yield r
             raise LookupError()
-    
+
     def process_spider_exception(self, response, exception, spider):
         method = '{}.process_spider_exception'.format(self.__class__.__name__)
         spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
@@ -264,7 +264,7 @@ class TestSpiderMiddleware(TestCase):
     @classmethod
     def tearDownClass(cls):
         cls.mockserver.__exit__(None, None, None)
-    
+
     @defer.inlineCallbacks
     def crawl_log(self, spider):
         crawler = get_crawler(spider)
@@ -308,7 +308,7 @@ class TestSpiderMiddleware(TestCase):
         self.assertIn("{'from': 'errback'}", str(log1))
         self.assertNotIn("{'from': 'callback'}", str(log1))
         self.assertIn("'item_scraped_count': 1", str(log1))
-    
+
     @defer.inlineCallbacks
     def test_generator_callback(self):
         """
@@ -319,7 +319,7 @@ class TestSpiderMiddleware(TestCase):
         log2 = yield self.crawl_log(GeneratorCallbackSpider)
         self.assertIn("Middleware: ImportError exception caught", str(log2))
         self.assertIn("'item_scraped_count': 2", str(log2))
-    
+
     @defer.inlineCallbacks
     def test_not_a_generator_callback(self):
         """

From c0d84f0962a4c269441db562e8cbc10298c53b72 Mon Sep 17 00:00:00 2001
From: Marc Hernandez Cabot <noviluni@gmail.com>
Date: Wed, 18 Dec 2019 19:39:21 +0100
Subject: [PATCH 2502/2974] fix typos

---
 docs/contributing.rst          |  2 +-
 docs/faq.rst                   |  2 +-
 docs/news.rst                  | 69 +++++++++++++++++-----------------
 docs/topics/jobs.rst           |  2 +-
 docs/topics/leaks.rst          |  2 +-
 docs/topics/media-pipeline.rst |  2 +-
 docs/topics/settings.rst       |  2 +-
 docs/topics/telnetconsole.rst  |  2 +-
 sep/sep-001.rst                |  4 +-
 sep/sep-019.rst                |  2 +-
 10 files changed, 44 insertions(+), 45 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 3aebb3d50..b56295027 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -217,7 +217,7 @@ the tests with Python 3.6 use::
 
     tox -e py36
 
-You can also specify a comma-separated list of environmets, and use :ref:`tox’s
+You can also specify a comma-separated list of environments, and use :ref:`tox’s
 parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
 parallel::
 
diff --git a/docs/faq.rst b/docs/faq.rst
index 080d81981..aae2411e0 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -338,7 +338,7 @@ How to split an item into multiple items in an item pipeline?
 input item. :ref:`Create a spider middleware <custom-spider-middleware>`
 instead, and use its
 :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
-method for this puspose. For example::
+method for this purpose. For example::
 
     from copy import deepcopy
 
diff --git a/docs/news.rst b/docs/news.rst
index 9dfd28508..28db40e3c 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -678,7 +678,7 @@ Usability improvements
 * a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`)
 * better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`)
 * non-zero exit code is returned from Scrapy commands when error happens
-  on spider inititalization (:issue:`3226`)
+  on spider initialization (:issue:`3226`)
 * Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`);
   "flv" is added to common video extensions (:issue:`3165`)
 * better error message when an exporter is disabled (:issue:`3358`);
@@ -1156,7 +1156,7 @@ Bug fixes
 - Fix :command:`view` command ; it was a regression in v1.3.0 (:issue:`2503`).
 - Fix tests regarding ``*_EXPIRES settings`` with Files/Images pipelines (:issue:`2460`).
 - Fix name of generated pipeline class when using basic project template (:issue:`2466`).
-- Fix compatiblity with Twisted 17+ (:issue:`2496`, :issue:`2528`).
+- Fix compatibility with Twisted 17+ (:issue:`2496`, :issue:`2528`).
 - Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`).
 - Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``,
   ``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`).
@@ -1164,7 +1164,7 @@ Bug fixes
 Documentation
 ~~~~~~~~~~~~~
 
-- Reword Code of Coduct section and upgrade to Contributor Covenant v1.4
+- Reword Code of Conduct section and upgrade to Contributor Covenant v1.4
   (:issue:`2469`).
 - Clarify that passing spider arguments converts them to spider attributes
   (:issue:`2483`).
@@ -1178,7 +1178,7 @@ Documentation
 Cleanups
 ~~~~~~~~
 
-- Remove reduntant check in ``MetaRefreshMiddleware`` (:issue:`2542`).
+- Remove redundant check in ``MetaRefreshMiddleware`` (:issue:`2542`).
 - Faster checks in ``LinkExtractor`` for allow/deny patterns (:issue:`2538`).
 - Remove dead code supporting old Twisted versions (:issue:`2544`).
 
@@ -1204,7 +1204,7 @@ New Features
 - ``MailSender`` now accepts single strings as values for ``to`` and ``cc``
   arguments (:issue:`2272`)
 - ``scrapy fetch url``, ``scrapy shell url`` and ``fetch(url)`` inside
-  scrapy shell now follow HTTP redirections by default (:issue:`2290`);
+  Scrapy shell now follow HTTP redirections by default (:issue:`2290`);
   See :command:`fetch` and :command:`shell` for details.
 - ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``;
   this is technically **backward incompatible** so please check your log parsers.
@@ -1705,7 +1705,7 @@ Scrapy 1.0.4 (2015-12-30)
 - fix ValueError: Invalid XPath: //div/[id="not-exists"]/text() on selectors.rst (:commit:`ca8d60f`)
 - Typos corrections (:commit:`7067117`)
 - fix typos in downloader-middleware.rst and exceptions.rst, middlware -> middleware (:commit:`32f115c`)
-- Add note to ubuntu install section about debian compatibility (:commit:`23fda69`)
+- Add note to Ubuntu install section about Debian compatibility (:commit:`23fda69`)
 - Replace alternative OSX install workaround with virtualenv (:commit:`98b63ee`)
 - Reference Homebrew's homepage for installation instructions (:commit:`1925db1`)
 - Add oldest supported tox version to contributing docs (:commit:`5d10d6d`)
@@ -1758,7 +1758,7 @@ Scrapy 1.0.1 (2015-07-01)
 - include tests/ to source distribution in MANIFEST.in (:commit:`eca227e`)
 - DOC Fix SelectJmes documentation (:commit:`b8567bc`)
 - DOC Bring Ubuntu and Archlinux outside of Windows subsection (:commit:`392233f`)
-- DOC remove version suffix from ubuntu package (:commit:`5303c66`)
+- DOC remove version suffix from Ubuntu package (:commit:`5303c66`)
 - DOC Update release date for 1.0 (:commit:`c89fa29`)
 
 .. _release-1.0.0:
@@ -2211,7 +2211,7 @@ Scrapy 0.24.2 (2014-07-08)
 
 - Use a mutable mapping to proxy deprecated settings.overrides and settings.defaults attribute (:commit:`e5e8133`)
 - there is not support for python3 yet (:commit:`3cd6146`)
-- Update python compatible version set to debian packages (:commit:`fa5d76b`)
+- Update python compatible version set to Debian packages (:commit:`fa5d76b`)
 - DOC fix formatting in release notes (:commit:`c6a9e20`)
 
 Scrapy 0.24.1 (2014-06-27)
@@ -2229,12 +2229,12 @@ Enhancements
 
 - Improve Scrapy top-level namespace (:issue:`494`, :issue:`684`)
 - Add selector shortcuts to responses (:issue:`554`, :issue:`690`)
-- Add new lxml based LinkExtractor to replace unmantained SgmlLinkExtractor
+- Add new lxml based LinkExtractor to replace unmaintained SgmlLinkExtractor
   (:issue:`559`, :issue:`761`, :issue:`763`)
 - Cleanup settings API - part of per-spider settings **GSoC project** (:issue:`737`)
 - Add UTF8 encoding header to templates (:issue:`688`, :issue:`762`)
 - Telnet console now binds to 127.0.0.1 by default (:issue:`699`)
-- Update debian/ubuntu install instructions (:issue:`509`, :issue:`549`)
+- Update Debian/Ubuntu install instructions (:issue:`509`, :issue:`549`)
 - Disable smart strings in lxml XPath evaluations (:issue:`535`)
 - Restore filesystem based cache as default for http
   cache middleware (:issue:`541`, :issue:`500`, :issue:`571`)
@@ -2267,7 +2267,7 @@ Enhancements
 - Tests and docs for ``request_fingerprint`` function (:issue:`597`)
 - Update SEP-19 for GSoC project ``per-spider settings`` (:issue:`705`)
 - Set exit code to non-zero when contracts fails (:issue:`727`)
-- Add a setting to control what class is instanciated as Downloader component
+- Add a setting to control what class is instantiated as Downloader component
   (:issue:`738`)
 - Pass response in ``item_dropped`` signal (:issue:`724`)
 - Improve ``scrapy check`` contracts command (:issue:`733`, :issue:`752`)
@@ -2276,7 +2276,7 @@ Enhancements
 - Add a note about reporting security issues (:issue:`697`)
 - Add LevelDB http cache storage backend (:issue:`626`, :issue:`500`)
 - Sort spider list output of ``scrapy list`` command (:issue:`742`)
-- Multiple documentation enhancemens and fixes
+- Multiple documentation enhancements and fixes
   (:issue:`575`, :issue:`587`, :issue:`590`, :issue:`596`, :issue:`610`,
   :issue:`617`, :issue:`618`, :issue:`627`, :issue:`613`, :issue:`643`,
   :issue:`654`, :issue:`675`, :issue:`663`, :issue:`711`, :issue:`714`)
@@ -2321,19 +2321,19 @@ Scrapy 0.22.1 (released 2014-02-08)
 - BaseSgmlLinkExtractor: Added unit test of a link with an inner tag (:commit:`c1cb418`)
 - BaseSgmlLinkExtractor: Fixed unknown_endtag() so that it only set current_link=None when the end tag match the opening tag (:commit:`7e4d627`)
 - Fix tests for Travis-CI build (:commit:`76c7e20`)
-- replace unencodeable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
+- replace unencodable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
 - RegexLinkExtractor: encode URL unicode value when creating Links (:commit:`d0ee545`)
 - Updated the tutorial crawl output with latest output. (:commit:`8da65de`)
 - Updated shell docs with the crawler reference and fixed the actual shell output. (:commit:`875b9ab`)
 - PEP8 minor edits. (:commit:`f89efaf`)
-- Expose current crawler in the scrapy shell. (:commit:`5349cec`)
+- Expose current crawler in the Scrapy shell. (:commit:`5349cec`)
 - Unused re import and PEP8 minor edits. (:commit:`387f414`)
 - Ignore None's values when using the ItemLoader. (:commit:`0632546`)
 - DOC Fixed HTTPCACHE_STORAGE typo in the default value which is now Filesystem instead Dbm. (:commit:`cde9a8c`)
-- show ubuntu setup instructions as literal code (:commit:`fb5c9c5`)
+- show Ubuntu setup instructions as literal code (:commit:`fb5c9c5`)
 - Update Ubuntu installation instructions (:commit:`70fb105`)
 - Merge pull request #550 from stray-leone/patch-1 (:commit:`6f70b6a`)
-- modify the version of scrapy ubuntu package (:commit:`725900d`)
+- modify the version of Scrapy Ubuntu package (:commit:`725900d`)
 - fix 0.22.0 release date (:commit:`af0219a`)
 - fix typos in news.rst and remove (not released yet) header (:commit:`b7f58f4`)
 
@@ -2354,7 +2354,7 @@ Enhancements
 - Improve test coverage and forthcoming Python 3 support (:issue:`525`)
 - Promote startup info on settings and middleware to INFO level (:issue:`520`)
 - Support partials in ``get_func_args`` util (:issue:`506`, issue:`504`)
-- Allow running indiviual tests via tox (:issue:`503`)
+- Allow running individual tests via tox (:issue:`503`)
 - Update extensions ignored by link extractors (:issue:`498`)
 - Add middleware methods to get files/images/thumbs paths (:issue:`490`)
 - Improve offsite middleware tests (:issue:`478`)
@@ -2411,7 +2411,7 @@ Enhancements
 - scrapy.mail.MailSender now can connect over TLS or upgrade using STARTTLS (:issue:`327`)
 - New FilesPipeline with functionality factored out from ImagesPipeline (:issue:`370`, :issue:`409`)
 - Recommend Pillow instead of PIL for image handling (:issue:`317`)
-- Added debian packages for Ubuntu quantal and raring (:commit:`86230c0`)
+- Added Debian packages for Ubuntu Quantal and raring (:commit:`86230c0`)
 - Mock server (used for tests) can listen for HTTPS requests (:issue:`410`)
 - Remove multi spider support from multiple core components
   (:issue:`422`, :issue:`421`, :issue:`420`, :issue:`419`, :issue:`423`, :issue:`418`)
@@ -2430,7 +2430,7 @@ Bugfixes
 - Fix tests under Django 1.6 (:commit:`b6bed44c`)
 - Lot of bugfixes to retry middleware under disconnections using HTTP 1.1 download handler
 - Fix inconsistencies among Twisted releases (:issue:`406`)
-- Fix scrapy shell bugs (:issue:`418`, :issue:`407`)
+- Fix Scrapy shell bugs (:issue:`418`, :issue:`407`)
 - Fix invalid variable name in setup.py (:issue:`429`)
 - Fix tutorial references (:issue:`387`)
 - Improve request-response docs (:issue:`391`)
@@ -2512,15 +2512,15 @@ Scrapy 0.18.1 (released 2013-08-27)
 - test PotentiaDataLoss errors on unbound responses (:commit:`b15470d`)
 - Treat responses without content-length or Transfer-Encoding as good responses (:commit:`c4bf324`)
 - do no include ResponseFailed if http11 handler is not enabled (:commit:`6cbe684`)
-- New HTTP client wraps connection losts in ResponseFailed exception. fix #373 (:commit:`1a20bba`)
+- New HTTP client wraps connection lost in ResponseFailed exception. fix #373 (:commit:`1a20bba`)
 - limit travis-ci build matrix (:commit:`3b01bb8`)
 - Merge pull request #375 from peterarenot/patch-1 (:commit:`fa766d7`)
 - Fixed so it refers to the correct folder (:commit:`3283809`)
-- added quantal & raring to support ubuntu releases (:commit:`1411923`)
+- added Quantal & raring to support Ubuntu releases (:commit:`1411923`)
 - fix retry middleware which didn't retry certain connection errors after the upgrade to http1 client, closes GH-373 (:commit:`bb35ed0`)
 - fix XmlItemExporter in Python 2.7.4 and 2.7.5 (:commit:`de3e451`)
 - minor updates to 0.18 release notes (:commit:`c45e5f1`)
-- fix contributters list format (:commit:`0b60031`)
+- fix contributors list format (:commit:`0b60031`)
 
 Scrapy 0.18.0 (released 2013-08-09)
 -----------------------------------
@@ -2617,7 +2617,7 @@ contributors sorted by number of commits::
 Scrapy 0.16.5 (released 2013-05-30)
 -----------------------------------
 
-- obey request method when scrapy deploy is redirected to a new endpoint (:commit:`8c4fcee`)
+- obey request method when Scrapy deploy is redirected to a new endpoint (:commit:`8c4fcee`)
 - fix inaccurate downloader middleware documentation. refs #280 (:commit:`40667cb`)
 - doc: remove links to diveintopython.org, which is no longer available. closes #246 (:commit:`bd58bfa`)
 - Find form nodes in invalid html5 documents (:commit:`e3d6945`)
@@ -2631,8 +2631,8 @@ Scrapy 0.16.4 (released 2013-01-23)
 - Fixed error message formatting. log.err() doesn't support cool formatting and when error occurred, the message was:    "ERROR: Error processing %(item)s" (:commit:`c16150c`)
 - lint and improve images pipeline error logging (:commit:`56b45fc`)
 - fixed doc typos (:commit:`243be84`)
-- add documentation topics: Broad Crawls & Common Practies (:commit:`1fbb715`)
-- fix bug in scrapy parse command when spider is not specified explicitly. closes #209 (:commit:`c72e682`)
+- add documentation topics: Broad Crawls & Common Practices (:commit:`1fbb715`)
+- fix bug in Scrapy parse command when spider is not specified explicitly. closes #209 (:commit:`c72e682`)
 - Update docs/topics/commands.rst (:commit:`28eac7a`)
 
 Scrapy 0.16.3 (released 2012-12-07)
@@ -2651,11 +2651,11 @@ Scrapy 0.16.3 (released 2012-12-07)
 Scrapy 0.16.2 (released 2012-11-09)
 -----------------------------------
 
-- scrapy contracts: python2.6 compat (:commit:`a4a9199`)
-- scrapy contracts verbose option (:commit:`ec41673`)
-- proper unittest-like output for scrapy contracts (:commit:`86635e4`)
+- Scrapy contracts: python2.6 compat (:commit:`a4a9199`)
+- Scrapy contracts verbose option (:commit:`ec41673`)
+- proper unittest-like output for Scrapy contracts (:commit:`86635e4`)
 - added open_in_browser to debugging doc (:commit:`c9b690d`)
-- removed reference to global scrapy stats from settings doc (:commit:`dd55067`)
+- removed reference to global Scrapy stats from settings doc (:commit:`dd55067`)
 - Fix SpiderState bug in Windows platforms (:commit:`58998f4`)
 
 
@@ -2665,7 +2665,7 @@ Scrapy 0.16.1 (released 2012-10-26)
 - fixed LogStats extension, which got broken after a wrong merge before the 0.16 release (:commit:`8c780fd`)
 - better backward compatibility for scrapy.conf.settings (:commit:`3403089`)
 - extended documentation on how to access crawler stats from extensions (:commit:`c4da0b5`)
-- removed .hgtags (no longer needed now that scrapy uses git) (:commit:`d52c188`)
+- removed .hgtags (no longer needed now that Scrapy uses git) (:commit:`d52c188`)
 - fix dashes under rst headers (:commit:`fa4f7f9`)
 - set release date for 0.16.0 in news (:commit:`e292246`)
 
@@ -2680,8 +2680,7 @@ Scrapy changes:
 - documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED`
 - major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals.
 - added :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method to spider middlewares
-- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
-- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
+- dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info.
 - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info.
 - documented :ref:`topics-api`
 - ``lxml`` is now the default selectors backend instead of ``libxml2``
@@ -2715,7 +2714,7 @@ Scrapy changes:
 Scrapy 0.14.4
 -------------
 
-- added precise to supported ubuntu distros (:commit:`b7e46df`)
+- added precise to supported Ubuntu distros (:commit:`b7e46df`)
 - fixed bug in json-rpc webservice reported in https://groups.google.com/forum/#!topic/scrapy-users/qgVBmFybNAQ/discussion. also removed no longer supported 'run' command from extras/scrapy-ws.py (:commit:`340fbdb`)
 - meta tag attributes for content-type http equiv can be in any order. #123 (:commit:`0cb68af`)
 - replace "import Image" by more standard "from PIL import Image". closes #88 (:commit:`4d17048`)
@@ -2728,11 +2727,11 @@ Scrapy 0.14.3
 - include egg files used by testsuite in source distribution. #118 (:commit:`c897793`)
 - update docstring in project template to avoid confusion with genspider command, which may be considered as an advanced feature. refs #107 (:commit:`2548dcc`)
 - added note to docs/topics/firebug.rst about google directory being shut down (:commit:`668e352`)
-- dont discard slot when empty, just save in another dict in order to recycle if needed again. (:commit:`8e9f607`)
+- don't discard slot when empty, just save in another dict in order to recycle if needed again. (:commit:`8e9f607`)
 - do not fail handling unicode xpaths in libxml2 backed selectors (:commit:`b830e95`)
 - fixed minor mistake in Request objects documentation (:commit:`bf3c9ee`)
 - fixed minor defect in link extractors documentation (:commit:`ba14f38`)
-- removed some obsolete remaining code related to sqlite support in scrapy (:commit:`0665175`)
+- removed some obsolete remaining code related to sqlite support in Scrapy (:commit:`0665175`)
 
 Scrapy 0.14.2
 -------------
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index f5542495b..8816a028c 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -74,7 +74,7 @@ Request serialization
 For persistence to work, :class:`~scrapy.http.Request` objects must be
 serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
 values passed to their ``__init__`` method, which must be methods of the
-runnning :class:`~scrapy.spiders.Spider` class.
+running :class:`~scrapy.spiders.Spider` class.
 
 If you wish to log the requests that couldn't be serialized, you can set the
 :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 793636f59..83b4d8154 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -110,7 +110,7 @@ ties the response lifetime to the requests' one, and that would definitely
 cause memory leaks.
 
 Let's see how we can discover the cause (without knowing it
-a-priori, of course) by using the ``trackref`` tool.
+a priori, of course) by using the ``trackref`` tool.
 
 After the crawler is running for a few minutes and we notice its memory usage
 has grown a lot, we can enter its telnet console and check the live
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 206e7cfa5..a40682e5b 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -558,7 +558,7 @@ See here the methods that you can override in your custom Images Pipeline:
 Custom Images pipeline example
 ==============================
 
-Here is a full example of the Images Pipeline whose methods are examplified
+Here is a full example of the Images Pipeline whose methods are exemplified
 above::
 
     import scrapy
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index a1d15a760..f138b7064 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -884,7 +884,7 @@ LOG_FORMAT
 
 Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
 
-String for formatting log messsages. Refer to the `Python logging documentation`_ for the whole list of available
+String for formatting log messages. Refer to the `Python logging documentation`_ for the whole list of available
 placeholders.
 
 .. _Python logging documentation: https://docs.python.org/2/library/logging.html#logrecord-attributes
diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index 7db7e4f6b..ecdc5423c 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -48,7 +48,7 @@ autogenerated Password can be seen on scrapy logs like the example below::
 
     2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
 
-Default Username and Password can be overriden by the settings 
+Default Username and Password can be overridden by the settings
 :setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`.
 
 .. warning::
diff --git a/sep/sep-001.rst b/sep/sep-001.rst
index 2a66f9802..00226283f 100644
--- a/sep/sep-001.rst
+++ b/sep/sep-001.rst
@@ -254,8 +254,8 @@ ItemForm
 
    #!python
    class MySiteForm(ItemForm):
-       witdth = adaptor(ItemForm.witdh, default_unit='cm')
-       volume = adaptor(ItemForm.witdh, default_unit='lt')
+       width = adaptor(ItemForm.width, default_unit='cm')
+       volume = adaptor(ItemForm.width, default_unit='lt')
 
    ia['width'] = x.x('//p[@class="width"]')
    ia['volume'] = x.x('//p[@class="volume"]')
diff --git a/sep/sep-019.rst b/sep/sep-019.rst
index 9fbf6a223..84f3a96c3 100644
--- a/sep/sep-019.rst
+++ b/sep/sep-019.rst
@@ -185,7 +185,7 @@ These ideas translate to the following changes on the ``SpiderManager`` class:
   will return a spider class, not an instance. It's basically a ``__get__``
   to ``self._spiders``.
 
-- All remaining functions should be deprecated or remove accordantly, since a
+- All remaining functions should be deprecated or remove accordingly, since a
   crawler reference is no longer needed.
 
 - New helper ``get_spider_manager_class_from_scrapycfg`` in

From 23a67cec271c849ad9b9c07a99531163dc3789fc Mon Sep 17 00:00:00 2001
From: Marc Hernandez Cabot <noviluni@gmail.com>
Date: Thu, 19 Dec 2019 09:57:17 +0100
Subject: [PATCH 2503/2974] fix first letter capitalization for Raring and
 Scrapy

---
 docs/contributing.rst           |  2 +-
 docs/index.rst                  |  2 +-
 docs/intro/install.rst          | 16 ++++++++--------
 docs/news.rst                   | 20 ++++++++++----------
 docs/topics/autothrottle.rst    |  2 +-
 docs/topics/commands.rst        |  2 +-
 docs/topics/contracts.rst       |  2 +-
 docs/topics/debug.rst           |  2 +-
 docs/topics/developer-tools.rst |  4 ++--
 docs/topics/logging.rst         |  4 ++--
 docs/topics/settings.rst        |  4 ++--
 docs/topics/shell.rst           |  2 +-
 docs/topics/telnetconsole.rst   |  2 +-
 sep/sep-004.rst                 |  2 +-
 14 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index b56295027..f40a6bba2 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -44,7 +44,7 @@ guidelines when you're going to report a new bug.
 * check the :ref:`FAQ <faq>` first to see if your issue is addressed in a
   well-known question
 
-* if you have a general question about scrapy usage, please ask it at
+* if you have a general question about Scrapy usage, please ask it at
   `Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__
   (use "scrapy" tag).
 
diff --git a/docs/index.rst b/docs/index.rst
index 6d5f9e77d..a4343b7e0 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -170,7 +170,7 @@ Solving specific problems
     Get answers to most frequently asked questions.
 
 :doc:`topics/debug`
-    Learn how to debug common problems of your scrapy spider.
+    Learn how to debug common problems of your Scrapy spider.
 
 :doc:`topics/contracts`
     Learn how to use contracts for testing your spiders.
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index e924b5303..0d6171884 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -78,9 +78,9 @@ TL;DR: We recommend installing Scrapy inside a virtual environment
 on all platforms.
 
 Python packages can be installed either globally (a.k.a system wide),
-or in user-space. We do not recommend installing scrapy system wide.
+or in user-space. We do not recommend installing Scrapy system wide.
 
-Instead, we recommend that you install scrapy within a so-called
+Instead, we recommend that you install Scrapy within a so-called
 "virtual environment" (`virtualenv`_).
 Virtualenvs allow you to not conflict with already-installed Python
 system packages (which could break some of your system tools and scripts),
@@ -97,7 +97,7 @@ Check this `user guide`_ on how to create your virtualenv.
 .. note::
     If you use Linux or OS X, `virtualenvwrapper`_ is a handy tool to create virtualenvs.
 
-Once you have created a virtualenv, you can install scrapy inside it with ``pip``,
+Once you have created a virtualenv, you can install Scrapy inside it with ``pip``,
 just like any other Python package.
 (See :ref:`platform-specific guides <intro-install-platform-notes>`
 below for non-Python dependencies that you may need to install beforehand).
@@ -144,7 +144,7 @@ albeit with potential issues with TLS connections.
 typically too old and slow to catch up with latest Scrapy.
 
 
-To install scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
+To install Scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
 these dependencies::
 
     sudo apt-get install python3 python3-dev python3-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
@@ -225,17 +225,17 @@ PyPy
 We recommend using the latest PyPy version. The version tested is 5.9.0.
 For PyPy3, only Linux installation was tested.
 
-Most scrapy dependencides now have binary wheels for CPython, but not for PyPy.
+Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy.
 This means that these dependecies will be built during installation.
 On OS X, you are likely to face an issue with building Cryptography dependency,
 solution to this problem is described
 `here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
 that is to ``brew install openssl`` and then export the flags that this command
-recommends (only needed when installing scrapy). Installing on Linux has no special
+recommends (only needed when installing Scrapy). Installing on Linux has no special
 issues besides installing build dependencies.
-Installing scrapy with PyPy on Windows is not tested.
+Installing Scrapy with PyPy on Windows is not tested.
 
-You can check that scrapy is installed correctly by running ``scrapy bench``.
+You can check that Scrapy is installed correctly by running ``scrapy bench``.
 If this command gives errors such as
 ``TypeError: ... got 2 unexpected keyword arguments``, this means
 that setuptools was unable to pick up one PyPy-specific dependency.
diff --git a/docs/news.rst b/docs/news.rst
index 28db40e3c..406e889f5 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1360,7 +1360,7 @@ Documentation
 
 - Grammar fixes: :issue:`2128`, :issue:`1566`.
 - Download stats badge removed from README (:issue:`2160`).
-- New scrapy :ref:`architecture diagram <topics-architecture>` (:issue:`2165`).
+- New Scrapy :ref:`architecture diagram <topics-architecture>` (:issue:`2165`).
 - Updated ``Response`` parameters documentation (:issue:`2197`).
 - Reworded misleading :setting:`RANDOMIZE_DOWNLOAD_DELAY` description (:issue:`2190`).
 - Add StackOverflow as a support channel (:issue:`2257`).
@@ -1450,7 +1450,7 @@ Documentation
 - Use "url" variable in downloader middleware example (:issue:`2015`)
 - Grammar fixes (:issue:`2054`, :issue:`2120`)
 - New FAQ entry on using BeautifulSoup in spider callbacks (:issue:`2048`)
-- Add notes about scrapy not working on Windows with Python 3 (:issue:`2060`)
+- Add notes about Scrapy not working on Windows with Python 3 (:issue:`2060`)
 - Encourage complete titles in pull requests (:issue:`2026`)
 
 Tests
@@ -1509,7 +1509,7 @@ This 1.1 release brings a lot of interesting features and bug fixes:
     You can use :setting:`FILES_STORE_S3_ACL` to change it.
   - We've reimplemented ``canonicalize_url()`` for more correct output,
     especially for URLs with non-ASCII characters (:issue:`1947`).
-    This could change link extractors output compared to previous scrapy versions.
+    This could change link extractors output compared to previous Scrapy versions.
     This may also invalidate some cache entries you could still have from pre-1.1 runs.
     **Warning: backward incompatible!**.
 
@@ -1722,7 +1722,7 @@ Scrapy 1.0.4 (2015-12-30)
 - Merge pull request #1513 from mgedmin/patch-2 (:commit:`5d4daf8`)
 - Typo (:commit:`f8d0682`)
 - Fix list formatting (:commit:`5f83a93`)
-- fix scrapy squeue tests after recent changes to queuelib (:commit:`3365c01`)
+- fix Scrapy squeue tests after recent changes to queuelib (:commit:`3365c01`)
 - Merge pull request #1475 from rweindl/patch-1 (:commit:`2d688cd`)
 - Update tutorial.rst (:commit:`fbc1f25`)
 - Merge pull request #1449 from rhoekman/patch-1 (:commit:`7d6538c`)
@@ -1734,7 +1734,7 @@ Scrapy 1.0.4 (2015-12-30)
 Scrapy 1.0.3 (2015-08-11)
 -------------------------
 
-- add service_identity to scrapy install_requires (:commit:`cbc2501`)
+- add service_identity to Scrapy install_requires (:commit:`cbc2501`)
 - Workaround for travis#296 (:commit:`66af9cd`)
 
 .. _release-1.0.2:
@@ -2411,7 +2411,7 @@ Enhancements
 - scrapy.mail.MailSender now can connect over TLS or upgrade using STARTTLS (:issue:`327`)
 - New FilesPipeline with functionality factored out from ImagesPipeline (:issue:`370`, :issue:`409`)
 - Recommend Pillow instead of PIL for image handling (:issue:`317`)
-- Added Debian packages for Ubuntu Quantal and raring (:commit:`86230c0`)
+- Added Debian packages for Ubuntu Quantal and Raring (:commit:`86230c0`)
 - Mock server (used for tests) can listen for HTTPS requests (:issue:`410`)
 - Remove multi spider support from multiple core components
   (:issue:`422`, :issue:`421`, :issue:`420`, :issue:`419`, :issue:`423`, :issue:`418`)
@@ -2516,7 +2516,7 @@ Scrapy 0.18.1 (released 2013-08-27)
 - limit travis-ci build matrix (:commit:`3b01bb8`)
 - Merge pull request #375 from peterarenot/patch-1 (:commit:`fa766d7`)
 - Fixed so it refers to the correct folder (:commit:`3283809`)
-- added Quantal & raring to support Ubuntu releases (:commit:`1411923`)
+- added Quantal & Raring to support Ubuntu releases (:commit:`1411923`)
 - fix retry middleware which didn't retry certain connection errors after the upgrade to http1 client, closes GH-373 (:commit:`bb35ed0`)
 - fix XmlItemExporter in Python 2.7.4 and 2.7.5 (:commit:`de3e451`)
 - minor updates to 0.18 release notes (:commit:`c45e5f1`)
@@ -2555,8 +2555,8 @@ Scrapy 0.18.0 (released 2013-08-09)
 - Collect idle downloader slots (:issue:`297`)
 - Add ``ftp://`` scheme downloader handler (:issue:`329`)
 - Added downloader benchmark webserver and spider tools :ref:`benchmarking`
-- Moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on
-- Add scrapy commands using external libraries (:issue:`260`)
+- Moved persistent (on disk) queues to a separate project (queuelib_) which Scrapy now depends on
+- Add Scrapy commands using external libraries (:issue:`260`)
 - Added ``--pdb`` option to ``scrapy`` command line tool
 - Added :meth:`XPathSelector.remove_namespaces <scrapy.selector.Selector.remove_namespaces>` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
 - Several improvements to spider contracts
@@ -2568,7 +2568,7 @@ Scrapy 0.18.0 (released 2013-08-09)
 - several more cleanups to singletons and multi-spider support (thanks Nicolas Ramirez)
 - support custom download slots
 - added --spider option to "shell" command.
-- log overridden settings when scrapy starts
+- log overridden settings when Scrapy starts
 
 Thanks to everyone who contribute to this release. Here is a list of
 contributors sorted by number of commits::
diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst
index c9bece753..4317019fc 100644
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@@ -11,7 +11,7 @@ Design goals
 ============
 
 1. be nicer to sites instead of using default download delay of zero
-2. automatically adjust scrapy to the optimum crawling speed, so the user
+2. automatically adjust Scrapy to the optimum crawling speed, so the user
    doesn't have to tune the download delays to find the optimum one.
    The user only needs to specify the maximum concurrent requests
    it allows, and the extension does the rest.
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 5b3cd7e75..a0dcba90d 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -29,7 +29,7 @@ in standard locations:
 1. ``/etc/scrapy.cfg`` or ``c:\scrapy\scrapy.cfg`` (system-wide),
 2. ``~/.config/scrapy.cfg`` (``$XDG_CONFIG_HOME``) and ``~/.scrapy.cfg`` (``$HOME``)
    for global (user-wide) settings, and
-3. ``scrapy.cfg`` inside a scrapy project's root (see next section).
+3. ``scrapy.cfg`` inside a Scrapy project's root (see next section).
 
 Settings from these files are merged in the listed order of preference:
 user-defined values have higher priority than system-wide defaults
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 371ae62d5..43db8f101 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -64,7 +64,7 @@ Use the :command:`check` command to run the contract checks.
 Custom Contracts
 ================
 
-If you find you need more power than the built-in scrapy contracts you can
+If you find you need more power than the built-in Scrapy contracts you can
 create and load your own contracts in the project by using the
 :setting:`SPIDER_CONTRACTS` setting::
 
diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst
index 4b2588518..d75f17301 100644
--- a/docs/topics/debug.rst
+++ b/docs/topics/debug.rst
@@ -5,7 +5,7 @@ Debugging Spiders
 =================
 
 This document explains the most common techniques for debugging spiders.
-Consider the following scrapy spider below::
+Consider the following Scrapy spider below::
 
     import scrapy
     from myproject.items import MyItem
diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index bf14643be..d1d4ebf5d 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -80,7 +80,7 @@ expand and collapse a tag by clicking on the arrow in front of it or by double
 clicking directly on the tag. If we expand the ``span`` tag with the ``class=
 "text"`` we will see the quote-text we clicked on. The `Inspector` lets you
 copy XPaths to selected elements. Let's try it out: Right-click on the ``span`` 
-tag, select ``Copy > XPath`` and paste it in the scrapy shell like so::
+tag, select ``Copy > XPath`` and paste it in the Scrapy shell like so::
 
     $ scrapy shell "http://quotes.toscrape.com/"
     (...)
@@ -159,7 +159,7 @@ The page is quite similar to the basic `quotes.toscrape.com`_-page,
 but instead of the above-mentioned ``Next`` button, the page 
 automatically loads new quotes when you scroll to the bottom. We 
 could go ahead and try out different XPaths directly, but instead 
-we'll check another quite useful command from the scrapy shell::
+we'll check another quite useful command from the Scrapy shell::
 
   $ scrapy shell "quotes.toscrape.com/scroll"
   (...)
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index dd09477b8..d4d22d889 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -171,9 +171,9 @@ listed in `logging's logrecord attributes docs
 <https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_
 respectively.
 
-If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the scrapy
+If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy
 component that prints the log. It is unset by default, hence logs contain the 
-scrapy component responsible for that log output.
+Scrapy component responsible for that log output.
 
 Command-line options
 --------------------
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index f138b7064..afe4fade1 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1264,7 +1264,7 @@ Default::
         'scrapy.contracts.default.ScrapesContract': 3,
     }
 
-A dict containing the scrapy contracts enabled by default in Scrapy. You should
+A dict containing the Scrapy contracts enabled by default in Scrapy. You should
 never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS`
 instead. For more info see :ref:`topics-contracts`.
 
@@ -1295,7 +1295,7 @@ SPIDER_LOADER_WARN_ONLY
 
 Default: ``False``
 
-By default, when scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
+By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
 it will fail loudly if there is any ``ImportError`` exception.
 But you can choose to silence this exception and turn it into a simple
 warning by setting ``SPIDER_LOADER_WARN_ONLY = True``.
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 68a0b19b5..4fe0dea06 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -31,7 +31,7 @@ for more info.
 Scrapy also has support for `bpython`_, and will try to use it where `IPython`_
 is unavailable.
 
-Through scrapy's settings you can configure it to use any one of
+Through Scrapy's settings you can configure it to use any one of
 ``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which
 are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment
 variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index ecdc5423c..47d8d393c 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -44,7 +44,7 @@ the console you need to type::
     >>>
 
 By default Username is ``scrapy`` and Password is autogenerated. The 
-autogenerated Password can be seen on scrapy logs like the example below::
+autogenerated Password can be seen on Scrapy logs like the example below::
 
     2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
 
diff --git a/sep/sep-004.rst b/sep/sep-004.rst
index 69edfa136..05b0eb99c 100644
--- a/sep/sep-004.rst
+++ b/sep/sep-004.rst
@@ -53,7 +53,7 @@ Here's a simple proof-of-concept code of such script:
    # ... do something more interesting with scraped_items ...
 
 The behaviour of the Scrapy crawler would be controller by the Scrapy settings,
-naturally, just like any typical scrapy project. But the default settings
+naturally, just like any typical Scrapy project. But the default settings
 should be sufficient so as to not require adding any specific setting. But, at
 the same time, you could do it if you need to, say, for specifying a custom
 middleware.

From f6bc1940a3394f74d8aa088faf2912a43690b260 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 19 Dec 2019 12:06:15 +0100
Subject: [PATCH 2504/2974] Use Python 3.7 to build the documentation

---
 .readthedocs.yml | 4 +++-
 .travis.yml      | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.readthedocs.yml b/.readthedocs.yml
index 3c1c3e8be..563add75f 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -2,6 +2,8 @@ version: 2
 sphinx:
   configuration: docs/conf.py
 python:
-  version: 3.8
+  # For available versions, see:
+  # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
+  version: 3.7  # Keep in sync with .travis.yml
   install:
     - requirements: docs/requirements.txt
diff --git a/.travis.yml b/.travis.yml
index c870934e1..f9d0dc8be 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,7 +25,7 @@ matrix:
     - env: TOXENV=extra-deps
       python: 3.8
     - env: TOXENV=docs
-      python: 3.8
+      python: 3.7  # Keep in sync with .readthedocs.yml
 install:
   - |
       if [ "$TOXENV" = "pypy3" ]; then

From e22c0c27d9d33383f4ac18a4142ccffe1d9a0d3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 19 Dec 2019 12:15:54 +0100
Subject: [PATCH 2505/2974] Revert "Improve FilteringLinkExtractor.__new__"

This reverts commit ee9881d2704798c9cd61b6da503bb0694227c58c.
---
 scrapy/linkextractors/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index d0d34035d..bdeab3a75 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -60,7 +60,7 @@ class FilteringLinkExtractor(object):
             warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
                  'please use scrapy.linkextractors.LinkExtractor instead',
                  ScrapyDeprecationWarning, stacklevel=2)
-        return super().__new__(cls, *args, **kwargs)
+        return super(FilteringLinkExtractor, cls).__new__(cls)
 
     def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
                  restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):

From c7f9b955bdf2405fce58907b0395abce2400a66d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 19 Dec 2019 12:44:52 +0100
Subject: [PATCH 2506/2974] Pylint: ignore not-an-iterable

---
 pylintrc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pylintrc b/pylintrc
index ca3ea1c57..c52a4c2d0 100644
--- a/pylintrc
+++ b/pylintrc
@@ -60,6 +60,7 @@ disable=abstract-method,
         no-self-argument,
         no-self-use,
         no-value-for-parameter,
+        not-an-iterable,
         not-callable,
         pointless-statement,
         pointless-string-statement,

From 40697dcbfa17dccf81adce7d033bc466ba6e98a2 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 20 Dec 2019 19:33:44 +0500
Subject: [PATCH 2507/2974] Remove deferred_from_coro from this PR.

---
 scrapy/utils/defer.py | 28 ----------------------------
 1 file changed, 28 deletions(-)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 530bf0e9d..20ce59297 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -1,14 +1,10 @@
 """
 Helper functions for dealing with Twisted deferreds
 """
-import asyncio
-import inspect
-
 from twisted.internet import defer, task
 from twisted.python import failure
 
 from scrapy.exceptions import IgnoreRequest
-from scrapy.utils.asyncio import is_asyncio_reactor_installed
 
 
 def defer_fail(_failure):
@@ -118,27 +114,3 @@ def iter_errback(iterable, errback, *a, **kw):
             break
         except Exception:
             errback(failure.Failure(), *a, **kw)
-
-
-def _isfuture(o):
-    # workaround for Python before 3.5.3 not having asyncio.isfuture
-    if hasattr(asyncio, 'isfuture'):
-        return asyncio.isfuture(o)
-    return isinstance(o, asyncio.Future)
-
-
-def deferred_from_coro(o, asyncio_enabled=False):
-    """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
-    if isinstance(o, defer.Deferred):
-        return o
-    if _isfuture(o) or inspect.isawaitable(o):
-        if not asyncio_enabled:
-            # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
-            # that use asyncio, e.g. "await asyncio.sleep(1)"
-            return defer.ensureDeferred(o)
-        else:
-            # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
-            if not is_asyncio_reactor_installed():
-                raise TypeError('Using coroutines requires installing AsyncioSelectorReactor')
-            return defer.Deferred.fromFuture(asyncio.ensure_future(o))
-    return o

From e342de5038e3757660f947fe1fadf35b54cd7113 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 20 Dec 2019 19:37:50 +0500
Subject: [PATCH 2508/2974] Remove a stray newline.

---
 tests/mockserver.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index d4e0362fb..a45277db9 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -6,7 +6,6 @@ from subprocess import Popen, PIPE
 from urllib.parse import urlencode
 
 from OpenSSL import SSL
-
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
 from twisted.web.static import File

From 8de80f59db19d739a056a7e58662f90544fece16 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 21 Dec 2019 13:08:29 +0500
Subject: [PATCH 2509/2974] Raise an exception if ASYNCIO_ENABLED but the
 reactor is wrong.

---
 scrapy/crawler.py     | 6 +++++-
 scrapy/utils/log.py   | 8 +-------
 tests/test_crawler.py | 9 +++++----
 3 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 706c8a59d..a9443f7ac 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -14,7 +14,7 @@ from scrapy.extension import ExtensionManager
 from scrapy.settings import overridden_settings, Settings
 from scrapy.signalmanager import SignalManager
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.asyncio import install_asyncio_reactor
+from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installed
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 from scrapy.utils.misc import load_object
 from scrapy.utils.log import (
@@ -259,6 +259,10 @@ class CrawlerProcess(CrawlerRunner):
         super(CrawlerProcess, self).__init__(settings)
         if self.settings.getbool('ASYNCIO_ENABLED'):
             install_asyncio_reactor()
+            if not is_asyncio_reactor_installed():
+                raise Exception("ASYNCIO_ENABLED is on but the Twisted asyncio "
+                                "reactor is not installed, this is not supported.")
+
         install_shutdown_handlers(self._signal_shutdown)
         configure_logging(self.settings, install_root_handler)
         log_scrapy_info(self.settings)
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 0fe3d1549..6179e1bd1 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -11,7 +11,6 @@ from twisted.python import log as twisted_log
 import scrapy
 from scrapy.settings import Settings
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.asyncio import is_asyncio_reactor_installed
 from scrapy.utils.versions import scrapy_components_versions
 
 
@@ -150,12 +149,7 @@ def log_scrapy_info(settings):
                     for name, version in scrapy_components_versions()
                     if name != "Scrapy")})
     if settings.getbool('ASYNCIO_ENABLED'):
-        if is_asyncio_reactor_installed():
-            logger.debug("Asyncio support enabled")
-        else:
-            logger.error("ASYNCIO_ENABLED is on but the Twisted asyncio "
-                         "reactor is not installed, this is not supported "
-                         "and asyncio coroutines will not work.")
+        logger.debug("Asyncio support enabled")
 
 
 class StreamLogger(object):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 0b2645280..a2865fcd1 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -264,13 +264,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
     @defer.inlineCallbacks
     def test_crawler_process_asyncio_enabled_true(self):
         with LogCapture(level=logging.DEBUG) as log:
-            runner = CrawlerProcess(settings={'ASYNCIO_ENABLED': True})
-            yield runner.crawl(NoRequestsSpider)
             if self.reactor_pytest == 'asyncio':
+                runner = CrawlerProcess(settings={'ASYNCIO_ENABLED': True})
+                yield runner.crawl(NoRequestsSpider)
                 self.assertIn("Asyncio support enabled", str(log))
             else:
-                self.assertNotIn("Asyncio support enabled", str(log))
-                self.assertIn("ASYNCIO_ENABLED is on but the Twisted asyncio reactor is not installed", str(log))
+                msg = "ASYNCIO_ENABLED is on but the Twisted asyncio reactor is not installed"
+                with self.assertRaisesRegex(Exception, msg):
+                    runner = CrawlerProcess(settings={'ASYNCIO_ENABLED': True})
 
     @defer.inlineCallbacks
     def test_crawler_process_asyncio_enabled_false(self):

From 931b7e68d33e06b624b49a7abeababb4e3540474 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Dec 2019 09:50:28 -0300
Subject: [PATCH 2510/2974] Update FileDownloadHandler test

---
 scrapy/core/downloader/handlers/file.py |  2 +-
 tests/test_downloader_handlers.py       | 26 +++++++++++++------------
 2 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py
index d445ba2e1..0d94e3df0 100644
--- a/scrapy/core/downloader/handlers/file.py
+++ b/scrapy/core/downloader/handlers/file.py
@@ -4,7 +4,7 @@ from scrapy.responsetypes import responsetypes
 from scrapy.utils.decorators import defers
 
 
-class FileDownloadHandler(object):
+class FileDownloadHandler:
     lazy = False
 
     @defers
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 4505f2bf7..ce4685eed 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1,21 +1,20 @@
+import contextlib
 import os
 import shutil
 import tempfile
 from unittest import mock
-import contextlib
 
 from testfixtures import LogCapture
-from twisted.trial import unittest
+from twisted.cred import checkers, credentials, portal
+from twisted.internet import defer, error, reactor
 from twisted.protocols.policies import WrappingFactory
 from twisted.python.filepath import FilePath
-from twisted.internet import reactor, defer, error
-from twisted.web import server, static, util, resource
+from twisted.trial import unittest
+from twisted.web import resource, server, static, util
 from twisted.web._newclient import ResponseFailed
 from twisted.web.http import _DataLoss
-from twisted.web.test.test_webclient import ForeverTakingResource, \
-        NoLengthResource, HostHeaderResource, \
-        PayloadResource
-from twisted.cred import portal, checkers, credentials
+from twisted.web.test.test_webclient import (ForeverTakingResource, HostHeaderResource,
+                                             NoLengthResource, PayloadResource)
 from w3lib.url import path_to_file_uri
 
 from scrapy.core.downloader.handlers import DownloadHandlers
@@ -26,13 +25,14 @@ from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
 from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 
-from scrapy.spiders import Spider
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Headers, Request
 from scrapy.http.response.text import TextResponse
 from scrapy.responsetypes import responsetypes
-from scrapy.utils.test import get_crawler, skip_if_no_boto
+from scrapy.spiders import Spider
+from scrapy.utils.misc import create_instance
 from scrapy.utils.python import to_bytes
-from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+from scrapy.utils.test import get_crawler, skip_if_no_boto
 
 from tests.mockserver import MockServer, ssl_context_factory, Echo
 from tests.spiders import SingleRequestSpider
@@ -117,7 +117,9 @@ class FileTestCase(unittest.TestCase):
         self.tmpname = self.mktemp()
         with open(self.tmpname + '^', 'w') as f:
             f.write('0123456789')
-        self.download_request = FileDownloadHandler().download_request
+        crawler = get_crawler()
+        handler = create_instance(FileDownloadHandler, crawler.settings, crawler)
+        self.download_request = handler.download_request
 
     def tearDown(self):
         os.unlink(self.tmpname + '^')

From 342bf3cd35df856f4f2eafbf2717e9244c9deaf8 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Dec 2019 09:52:55 -0300
Subject: [PATCH 2511/2974] Explicit keyword arguments

---
 scrapy/core/downloader/handlers/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index e8c4454d2..94e0e59ef 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -50,9 +50,9 @@ class DownloadHandlers(object):
             if skip_lazy and getattr(dhcls, 'lazy', True):
                 return None
             dh = create_instance(
-                dhcls,
-                self._crawler.settings,
-                self._crawler,
+                objcls=dhcls,
+                settings=self._crawler.settings,
+                crawler=self._crawler,
             )
         except NotConfigured as ex:
             self._notconfigured[scheme] = str(ex)

From 9e5d945ef27ff9efee54b2245e833fefc3df72d8 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Dec 2019 09:55:47 -0300
Subject: [PATCH 2512/2974] Use create_instance in downloader handler tests

---
 tests/test_downloader_handlers.py | 35 ++++++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index ce4685eed..b63e8405e 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -252,7 +252,12 @@ class HttpTestCase(unittest.TestCase):
         else:
             self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
         self.portno = self.port.getHost().port
-        self.download_handler = self.download_handler_cls.from_crawler(get_crawler())
+        crawler = get_crawler()
+        self.download_handler = create_instance(
+            objcls=self.download_handler_cls,
+            settings=crawler.settings,
+            crawler=crawler
+        )
         self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks
@@ -492,8 +497,11 @@ class Http11TestCase(HttpTestCase):
         return self.test_download_broken_content_allow_data_loss('broken-chunked')
 
     def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
-        download_handler = self.download_handler_cls.from_crawler(
-            get_crawler(settings_dict={'DOWNLOAD_FAIL_ON_DATALOSS': False})
+        crawler = get_crawler(settings_dict={'DOWNLOAD_FAIL_ON_DATALOSS': False})
+        download_handler = create_instance(
+            objcls=self.download_handler_cls,
+            settings=crawler.settings,
+            crawler=crawler
         )
         request = Request(self.getURL(url))
         d = download_handler.download_request(request, Spider('foo'))
@@ -512,8 +520,11 @@ class Https11TestCase(Http11TestCase):
 
     @defer.inlineCallbacks
     def test_tls_logging(self):
-        download_handler = self.download_handler_cls.from_crawler(
-            get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True})
+        crawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True})
+        download_handler = create_instance(
+            objcls=self.download_handler_cls,
+            settings=crawler.settings,
+            crawler=crawler
         )
         try:
             with LogCapture() as log_capture:
@@ -581,8 +592,11 @@ class Https11CustomCiphers(unittest.TestCase):
             0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),
             interface=self.host)
         self.portno = self.port.getHost().port
-        self.download_handler = self.download_handler_cls.from_crawler(
-            get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})
+        crawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})
+        self.download_handler = create_instance(
+            objcls=self.download_handler_cls,
+            settings=crawler.settings,
+            crawler=crawler
         )
         self.download_request = self.download_handler.download_request
 
@@ -679,7 +693,12 @@ class HttpProxyTestCase(unittest.TestCase):
         wrapper = WrappingFactory(site)
         self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')
         self.portno = self.port.getHost().port
-        self.download_handler = self.download_handler_cls.from_crawler(get_crawler())
+        crawler = get_crawler()
+        self.download_handler = create_instance(
+            objcls=self.download_handler_cls,
+            settings=crawler.settings,
+            crawler=crawler
+        )
         self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks

From fa21d8687a0f94e5d79cb28af11312ad58629954 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Dec 2019 10:00:25 -0300
Subject: [PATCH 2513/2974] Use create_instance in S3DownloadHandler tests

---
 tests/test_downloader_handlers.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index b63e8405e..ac0e94364 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -776,10 +776,13 @@ class S3AnonTestCase(unittest.TestCase):
 
     def setUp(self):
         skip_if_no_boto()
-        self.s3reqh = S3DownloadHandler.from_crawler(
-            crawler=get_crawler(),
+        crawler = get_crawler()
+        self.s3reqh = create_instance(
+            objcls=S3DownloadHandler,
+            settings=crawler.settings,
+            crawler=crawler,
             httpdownloadhandler=HttpDownloadHandlerMock,
-            #anon=True, # is implicit
+            # anon=True, # implicit
         )
         self.download_request = self.s3reqh.download_request
         self.spider = Spider('foo')
@@ -805,8 +808,11 @@ class S3TestCase(unittest.TestCase):
 
     def setUp(self):
         skip_if_no_boto()
-        s3reqh = S3DownloadHandler.from_crawler(
-            crawler=get_crawler(),
+        crawler = get_crawler()
+        s3reqh = create_instance(
+            objcls=S3DownloadHandler,
+            settings=crawler.settings,
+            crawler=crawler,
             aws_access_key_id=self.AWS_ACCESS_KEY_ID,
             aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
             httpdownloadhandler=HttpDownloadHandlerMock,
@@ -830,7 +836,13 @@ class S3TestCase(unittest.TestCase):
 
     def test_extra_kw(self):
         try:
-            S3DownloadHandler.from_crawler(get_crawler(), extra_kw=True)
+            crawler = get_crawler()
+            create_instance(
+                objcls=S3DownloadHandler,
+                settings=crawler.settings,
+                crawler=crawler,
+                extra_kw=True,
+            )
         except Exception as e:
             self.assertIsInstance(e, (TypeError, NotConfigured))
         else:

From 7e6387de407297a36dd0e9b8e2058ae809cf3123 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Dec 2019 10:02:58 -0300
Subject: [PATCH 2514/2974] Use create_instance in
 FTPDownloadHandler/DataURIDownloadHandler tests

---
 tests/test_downloader_handlers.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index ac0e94364..14d58b651 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -984,7 +984,8 @@ class BaseFTPTestCase(unittest.TestCase):
         self.factory = FTPFactory(portal=p)
         self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
         self.portNum = self.port.getHost().port
-        self.download_handler = FTPDownloadHandler(get_crawler())
+        crawler = get_crawler()
+        self.download_handler = create_instance(FTPDownloadHandler, crawler.settings, crawler)
         self.addCleanup(self.port.stopListening)
 
     def tearDown(self):
@@ -1098,7 +1099,8 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
                                   userAnonymous=self.username)
         self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
         self.portNum = self.port.getHost().port
-        self.download_handler = FTPDownloadHandler(get_crawler())
+        crawler = get_crawler()
+        self.download_handler = create_instance(FTPDownloadHandler, crawler.settings, crawler)
         self.addCleanup(self.port.stopListening)
 
     def tearDown(self):
@@ -1108,7 +1110,8 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
 class DataURITestCase(unittest.TestCase):
 
     def setUp(self):
-        self.download_handler = DataURIDownloadHandler()
+        crawler = get_crawler()
+        self.download_handler = create_instance(DataURIDownloadHandler, crawler.settings, crawler)
         self.download_request = self.download_handler.download_request
         self.spider = Spider('foo')
 

From 8a567e98bbb1c7f917462d0376061303baef5883 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Dec 2019 10:05:49 -0300
Subject: [PATCH 2515/2974] Remove unnecessary __init__ methods in downloader
 handler tests

---
 tests/test_downloader_handlers.py | 23 +++++------------------
 1 file changed, 5 insertions(+), 18 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 14d58b651..b66b8151e 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -38,29 +38,16 @@ from tests.mockserver import MockServer, ssl_context_factory, Echo
 from tests.spiders import SingleRequestSpider
 
 
-class DummyDH(object):
+class DummyDH:
     lazy = False
 
-    def __init__(self, crawler):
-        pass
 
-    @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler)
-
-
-class DummyLazyDH(object):
+class DummyLazyDH:
     # Default is lazy for backward compatibility
-
-    def __init__(self, crawler):
-        pass
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler)
+    pass
 
 
-class OffDH(object):
+class OffDH:
     lazy = False
 
     def __init__(self, crawler):
@@ -765,7 +752,7 @@ class Http11ProxyTestCase(HttpProxyTestCase):
 
 
 class HttpDownloadHandlerMock(object):
-    def __init__(self, settings):
+    def __init__(self, settings, crawler):
         pass
 
     def download_request(self, request, spider):

From a6ec89251eca6977f898e8341634bbe7288fd966 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Dec 2019 10:40:16 -0300
Subject: [PATCH 2516/2974] Downloader handlers: crawler=None in __init__

---
 scrapy/core/downloader/handlers/ftp.py    | 12 ++++++------
 scrapy/core/downloader/handlers/http10.py |  8 ++++----
 scrapy/core/downloader/handlers/http11.py |  8 +++-----
 scrapy/core/downloader/handlers/s3.py     | 15 +++++++++------
 tests/test_downloader_handlers.py         |  4 +---
 5 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index fafecc1a8..2b22465e0 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -63,7 +63,7 @@ class ReceivedDataProtocol(Protocol):
 _CODE_RE = re.compile(r"\d+")
 
 
-class FTPDownloadHandler(object):
+class FTPDownloadHandler:
     lazy = False
 
     CODE_MAPPING = {
@@ -71,14 +71,14 @@ class FTPDownloadHandler(object):
         "default": 503,
     }
 
-    def __init__(self, crawler):
-        self.default_user = crawler.settings['FTP_USER']
-        self.default_password = crawler.settings['FTP_PASSWORD']
-        self.passive_mode = crawler.settings['FTP_PASSIVE_MODE']
+    def __init__(self, settings, crawler=None):
+        self.default_user = settings['FTP_USER']
+        self.default_password = settings['FTP_PASSWORD']
+        self.passive_mode = settings['FTP_PASSIVE_MODE']
 
     @classmethod
     def from_crawler(cls, crawler):
-        return cls(crawler)
+        return cls(crawler.settings, crawler)
 
     def download_request(self, request, spider):
         parsed_url = urlparse_cached(request)
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index ce0801bcc..51c0acd1b 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -6,18 +6,18 @@ from scrapy.utils.misc import load_object, create_instance
 from scrapy.utils.python import to_unicode
 
 
-class HTTP10DownloadHandler(object):
+class HTTP10DownloadHandler:
     lazy = False
 
-    def __init__(self, crawler):
+    def __init__(self, settings, crawler=None):
         self.HTTPClientFactory = load_object(crawler.settings['DOWNLOADER_HTTPCLIENTFACTORY'])
         self.ClientContextFactory = load_object(crawler.settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+        self._settings = settings
         self._crawler = crawler
-        self._settings = crawler.settings
 
     @classmethod
     def from_crawler(cls, crawler):
-        return cls(crawler)
+        return cls(crawler.settings, crawler)
 
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 691937e97..25dc287df 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -28,12 +28,10 @@ from scrapy.utils.python import to_bytes, to_unicode
 logger = logging.getLogger(__name__)
 
 
-class HTTP11DownloadHandler(object):
+class HTTP11DownloadHandler:
     lazy = False
 
-    def __init__(self, crawler):
-        settings = crawler.settings
-
+    def __init__(self, settings, crawler=None):
         self._pool = HTTPConnectionPool(reactor, persistent=True)
         self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self._pool._factory.noisy = False
@@ -68,7 +66,7 @@ class HTTP11DownloadHandler(object):
 
     @classmethod
     def from_crawler(cls, crawler):
-        return cls(crawler)
+        return cls(crawler.settings, crawler)
 
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index b35b59f3a..93cad0662 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -4,6 +4,7 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.boto import is_botocore
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.misc import create_instance
 
 
 def _get_boto_connection():
@@ -30,14 +31,15 @@ def _get_boto_connection():
     return _S3Connection
 
 
-class S3DownloadHandler(object):
+class S3DownloadHandler:
 
-    def __init__(self, crawler, aws_access_key_id=None, aws_secret_access_key=None,
+    def __init__(self, settings, crawler=None,
+                 aws_access_key_id=None, aws_secret_access_key=None,
                  httpdownloadhandler=HTTPDownloadHandler, **kw):
         if not aws_access_key_id:
-            aws_access_key_id = crawler.settings['AWS_ACCESS_KEY_ID']
+            aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
         if not aws_secret_access_key:
-            aws_secret_access_key = crawler.settings['AWS_SECRET_ACCESS_KEY']
+            aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
 
         # If no credentials could be found anywhere,
         # consider this an anonymous connection request by default;
@@ -66,11 +68,12 @@ class S3DownloadHandler(object):
             except Exception as ex:
                 raise NotConfigured(str(ex))
 
-        self._download_http = httpdownloadhandler(crawler).download_request
+        _http_handler = create_instance(httpdownloadhandler, settings, crawler)
+        self._download_http = _http_handler.download_request
 
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):
-        return cls(crawler, *args, **kwargs)
+        return cls(crawler.settings, crawler, *args, **kwargs)
 
     def download_request(self, request, spider):
         p = urlparse_cached(request)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index b66b8151e..218360709 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -751,9 +751,7 @@ class Http11ProxyTestCase(HttpProxyTestCase):
         self.assertIn(domain, timeout.osError)
 
 
-class HttpDownloadHandlerMock(object):
-    def __init__(self, settings, crawler):
-        pass
+class HttpDownloadHandlerMock:
 
     def download_request(self, request, spider):
         return request

From e2e15d66510c3040234ef9222ff2a17961c50eff Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Dec 2019 10:48:19 -0300
Subject: [PATCH 2517/2974] Downloader handlers: sort imports

---
 scrapy/core/downloader/handlers/__init__.py |  6 +++---
 scrapy/core/downloader/handlers/datauri.py  |  2 +-
 scrapy/core/downloader/handlers/ftp.py      |  4 ++--
 scrapy/core/downloader/handlers/http10.py   |  2 +-
 scrapy/core/downloader/handlers/http11.py   | 20 ++++++++++----------
 5 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 94e0e59ef..e86680978 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -4,17 +4,17 @@ import logging
 
 from twisted.internet import defer
 
-from scrapy.exceptions import NotSupported, NotConfigured
+from scrapy import signals
+from scrapy.exceptions import NotConfigured, NotSupported
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
-from scrapy import signals
 
 
 logger = logging.getLogger(__name__)
 
 
-class DownloadHandlers(object):
+class DownloadHandlers:
 
     def __init__(self, crawler):
         self._crawler = crawler
diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py
index 97134e618..a45b4ff3c 100644
--- a/scrapy/core/downloader/handlers/datauri.py
+++ b/scrapy/core/downloader/handlers/datauri.py
@@ -5,7 +5,7 @@ from scrapy.responsetypes import responsetypes
 from scrapy.utils.decorators import defers
 
 
-class DataURIDownloadHandler(object):
+class DataURIDownloadHandler:
     lazy = False
 
     @defers
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 2b22465e0..89c88ad70 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -33,8 +33,8 @@ from io import BytesIO
 from urllib.parse import unquote
 
 from twisted.internet import reactor
-from twisted.protocols.ftp import FTPClient, CommandFailed
-from twisted.internet.protocol import Protocol, ClientCreator
+from twisted.internet.protocol import ClientCreator, Protocol
+from twisted.protocols.ftp import CommandFailed, FTPClient
 
 from scrapy.http import Response
 from scrapy.responsetypes import responsetypes
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index 51c0acd1b..1086a6cc0 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -2,7 +2,7 @@
 """
 from twisted.internet import reactor
 
-from scrapy.utils.misc import load_object, create_instance
+from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import to_unicode
 
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 25dc287df..d8a561792 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -1,27 +1,27 @@
 """Download handlers for http and https schemes"""
 
-import re
 import logging
+import re
 import warnings
 from io import BytesIO
 from time import time
 from urllib.parse import urldefrag
 
-from zope.interface import implementer
-from twisted.internet import defer, reactor, protocol
+from twisted.internet import defer, protocol, reactor
+from twisted.internet.endpoints import TCP4ClientEndpoint
+from twisted.internet.error import TimeoutError
+from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
+from twisted.web.http import _DataLoss, PotentialDataLoss
 from twisted.web.http_headers import Headers as TxHeaders
 from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
-from twisted.internet.error import TimeoutError
-from twisted.web.http import _DataLoss, PotentialDataLoss
-from twisted.web.client import Agent, ResponseDone, HTTPConnectionPool, ResponseFailed, URI
-from twisted.internet.endpoints import TCP4ClientEndpoint
+from zope.interface import implementer
 
+from scrapy.core.downloader.tls import openssl_methods
+from scrapy.core.downloader.webclient import _parse
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
-from scrapy.core.downloader.webclient import _parse
-from scrapy.core.downloader.tls import openssl_methods
-from scrapy.utils.misc import load_object, create_instance
+from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import to_bytes, to_unicode
 
 

From b9a58798eed39bf543b9682d9ce43b13378a5074 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sat, 24 May 2014 20:24:01 +0000
Subject: [PATCH 2518/2974] change Scraper API to call internal `_parse` method

A Spider class using internal pre-processing can have first dibs
at this and then call a public `parse` method for subclass hooking.
---
 scrapy/core/scraper.py     | 2 +-
 scrapy/spiders/__init__.py | 3 +++
 scrapy/spiders/crawl.py    | 2 +-
 scrapy/spiders/feed.py     | 4 ++--
 tests/test_spider.py       | 2 +-
 5 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 99114d3bb..ad6649134 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -145,7 +145,7 @@ class Scraper(object):
     def call_spider(self, result, request, spider):
         result.request = request
         dfd = defer_result(result)
-        callback = request.callback or spider.parse
+        callback = request.callback or spider._parse
         warn_on_generator_with_return_value(spider, callback)
         warn_on_generator_with_return_value(spider, request.errback)
         dfd.addCallbacks(callback=callback,
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 9429f6cb2..1011eb870 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -80,6 +80,9 @@ class Spider(object_ref):
         """ This method is deprecated. """
         return Request(url, dont_filter=True)
 
+    def _parse(self, response):
+        return self.parse(response)
+
     def parse(self, response):
         raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))
 
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index a2c364c0e..e28d17dcd 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -74,7 +74,7 @@ class CrawlSpider(Spider):
         super(CrawlSpider, self).__init__(*a, **kw)
         self._compile_rules()
 
-    def parse(self, response):
+    def _parse(self, response):
         return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True)
 
     def parse_start_url(self, response):
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index c566f0236..11bd17db4 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -61,7 +61,7 @@ class XMLFeedSpider(Spider):
             for result_item in self.process_results(response, ret):
                 yield result_item
 
-    def parse(self, response):
+    def _parse(self, response):
         if not hasattr(self, 'parse_node'):
             raise NotConfigured('You must define parse_node method in order to scrape this XML feed')
 
@@ -128,7 +128,7 @@ class CSVFeedSpider(Spider):
             for result_item in self.process_results(response, ret):
                 yield result_item
 
-    def parse(self, response):
+    def _parse(self, response):
         if not hasattr(self, 'parse_row'):
             raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
         response = self.adapt_response(response)
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 317a27076..6fbec7e58 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -142,7 +142,7 @@ class XMLFeedSpiderTest(SpiderTest):
 
         for iterator in ('iternodes', 'xml'):
             spider = _XMLSpider('example', iterator=iterator)
-            output = list(spider.parse(response))
+            output = list(spider._parse(response))
             self.assertEqual(len(output), 2, iterator)
             self.assertEqual(output, [
                 {'loc': [u'http://www.example.com/Special-Offers.html'],

From 5982e3477c732f4dff5accea6eab486e4ab52c3e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Dec 2019 14:12:21 -0300
Subject: [PATCH 2519/2974] Take keyword arguments in base parsing methods

---
 docs/topics/spiders.rst    | 12 +++++++-----
 scrapy/spiders/__init__.py |  6 +++---
 scrapy/spiders/crawl.py    | 11 ++++++++---
 scrapy/spiders/feed.py     |  4 ++--
 4 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index b0fb14e24..dd763b607 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -362,12 +362,14 @@ CrawlSpider
 
    This spider also exposes an overrideable method:
 
-   .. method:: parse_start_url(response)
+   .. method:: parse_start_url(response, **kwargs)
 
-      This method is called for the start_urls responses. It allows to parse
-      the initial responses and must return either an
-      :class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request`
-      object, or an iterable containing any of them.
+      This method is called for each response produced for the URLs in
+      the spider's ``start_urls`` attribute. It allows to parse
+      the initial responses and must return either an item
+      (:class:`scrapy.item.Item` or :class:`dict`),
+      a :class:`~scrapy.http.Request`,
+      or an iterable containing any of them.
 
 Crawling rules
 ~~~~~~~~~~~~~~
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 1011eb870..3e19f1e23 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -80,10 +80,10 @@ class Spider(object_ref):
         """ This method is deprecated. """
         return Request(url, dont_filter=True)
 
-    def _parse(self, response):
-        return self.parse(response)
+    def _parse(self, response, **kwargs):
+        return self.parse(response, **kwargs)
 
-    def parse(self, response):
+    def parse(self, response, **kwargs):
         raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))
 
     @classmethod
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index e28d17dcd..4ec0de78c 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -74,10 +74,15 @@ class CrawlSpider(Spider):
         super(CrawlSpider, self).__init__(*a, **kw)
         self._compile_rules()
 
-    def _parse(self, response):
-        return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True)
+    def _parse(self, response, **kwargs):
+        return self._parse_response(
+            response=response,
+            callback=self.parse_start_url,
+            cb_kwargs=kwargs,
+            follow=True,
+        )
 
-    def parse_start_url(self, response):
+    def parse_start_url(self, response, **kwargs):
         return []
 
     def process_results(self, response, results):
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 11bd17db4..4fa6009a5 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -61,7 +61,7 @@ class XMLFeedSpider(Spider):
             for result_item in self.process_results(response, ret):
                 yield result_item
 
-    def _parse(self, response):
+    def _parse(self, response, **kwargs):
         if not hasattr(self, 'parse_node'):
             raise NotConfigured('You must define parse_node method in order to scrape this XML feed')
 
@@ -128,7 +128,7 @@ class CSVFeedSpider(Spider):
             for result_item in self.process_results(response, ret):
                 yield result_item
 
-    def _parse(self, response):
+    def _parse(self, response, **kwargs):
         if not hasattr(self, 'parse_row'):
             raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
         response = self.adapt_response(response)

From 2fb160e3bac9e09373a49cb0b2d764ddf782987c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Dec 2019 20:24:16 -0300
Subject: [PATCH 2520/2974] Use settings instead of crawler

---
 scrapy/core/downloader/handlers/ftp.py    | 4 ++--
 scrapy/core/downloader/handlers/http10.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 89c88ad70..1681c6df8 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -71,14 +71,14 @@ class FTPDownloadHandler:
         "default": 503,
     }
 
-    def __init__(self, settings, crawler=None):
+    def __init__(self, settings):
         self.default_user = settings['FTP_USER']
         self.default_password = settings['FTP_PASSWORD']
         self.passive_mode = settings['FTP_PASSIVE_MODE']
 
     @classmethod
     def from_crawler(cls, crawler):
-        return cls(crawler.settings, crawler)
+        return cls(crawler.settings)
 
     def download_request(self, request, spider):
         parsed_url = urlparse_cached(request)
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index 1086a6cc0..87a42f1da 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -10,8 +10,8 @@ class HTTP10DownloadHandler:
     lazy = False
 
     def __init__(self, settings, crawler=None):
-        self.HTTPClientFactory = load_object(crawler.settings['DOWNLOADER_HTTPCLIENTFACTORY'])
-        self.ClientContextFactory = load_object(crawler.settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+        self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
+        self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
         self._settings = settings
         self._crawler = crawler
 

From 9a75b46fb8322d27ffd45ee6c187bc84a565e26d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Dec 2019 20:26:58 -0300
Subject: [PATCH 2521/2974] Explicit argument names

---
 scrapy/core/downloader/handlers/http10.py | 2 +-
 scrapy/core/downloader/handlers/http11.py | 4 ++--
 scrapy/core/downloader/handlers/s3.py     | 6 +++++-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index 87a42f1da..d4aa51bd1 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -29,7 +29,7 @@ class HTTP10DownloadHandler:
         host, port = to_unicode(factory.host), factory.port
         if factory.scheme == b'https':
             client_context_factory = create_instance(
-                self.ClientContextFactory,
+                objcls=self.ClientContextFactory,
                 settings=self._settings,
                 crawler=self._crawler,
             )
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index d8a561792..5a5f6cf0a 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -41,7 +41,7 @@ class HTTP11DownloadHandler:
         # try method-aware context factory
         try:
             self._contextFactory = create_instance(
-                self._contextFactoryClass,
+                objcls=self._contextFactoryClass,
                 settings=settings,
                 crawler=crawler,
                 method=self._sslMethod,
@@ -49,7 +49,7 @@ class HTTP11DownloadHandler:
         except TypeError:
             # use context factory defaults
             self._contextFactory = create_instance(
-                self._contextFactoryClass,
+                objcls=self._contextFactoryClass,
                 settings=settings,
                 crawler=crawler,
             )
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 93cad0662..b38d2bf86 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -68,7 +68,11 @@ class S3DownloadHandler:
             except Exception as ex:
                 raise NotConfigured(str(ex))
 
-        _http_handler = create_instance(httpdownloadhandler, settings, crawler)
+        _http_handler = create_instance(
+            objcls=httpdownloadhandler,
+            settings=settings,
+            crawler=crawler,
+        )
         self._download_http = _http_handler.download_request
 
     @classmethod

From 982a66f9fb627575d42f0ad4fb2eb38b0a55b784 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Dec 2019 20:28:17 -0300
Subject: [PATCH 2522/2974] [test] Download handler: avoid passing settings if
 not necessary

---
 tests/test_downloader_handlers.py | 41 +++++++------------------------
 1 file changed, 9 insertions(+), 32 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 218360709..8d95d7cac 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -104,8 +104,7 @@ class FileTestCase(unittest.TestCase):
         self.tmpname = self.mktemp()
         with open(self.tmpname + '^', 'w') as f:
             f.write('0123456789')
-        crawler = get_crawler()
-        handler = create_instance(FileDownloadHandler, crawler.settings, crawler)
+        handler = create_instance(FileDownloadHandler, None, get_crawler())
         self.download_request = handler.download_request
 
     def tearDown(self):
@@ -239,12 +238,7 @@ class HttpTestCase(unittest.TestCase):
         else:
             self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
         self.portno = self.port.getHost().port
-        crawler = get_crawler()
-        self.download_handler = create_instance(
-            objcls=self.download_handler_cls,
-            settings=crawler.settings,
-            crawler=crawler
-        )
+        self.download_handler = create_instance(self.download_handler_cls, None, get_crawler())
         self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks
@@ -485,11 +479,7 @@ class Http11TestCase(HttpTestCase):
 
     def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
         crawler = get_crawler(settings_dict={'DOWNLOAD_FAIL_ON_DATALOSS': False})
-        download_handler = create_instance(
-            objcls=self.download_handler_cls,
-            settings=crawler.settings,
-            crawler=crawler
-        )
+        download_handler = create_instance(self.download_handler_cls, None, crawler)
         request = Request(self.getURL(url))
         d = download_handler.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.flags)
@@ -508,11 +498,7 @@ class Https11TestCase(Http11TestCase):
     @defer.inlineCallbacks
     def test_tls_logging(self):
         crawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True})
-        download_handler = create_instance(
-            objcls=self.download_handler_cls,
-            settings=crawler.settings,
-            crawler=crawler
-        )
+        download_handler = create_instance(self.download_handler_cls, None, crawler)
         try:
             with LogCapture() as log_capture:
                 request = Request(self.getURL('file'))
@@ -580,11 +566,7 @@ class Https11CustomCiphers(unittest.TestCase):
             interface=self.host)
         self.portno = self.port.getHost().port
         crawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})
-        self.download_handler = create_instance(
-            objcls=self.download_handler_cls,
-            settings=crawler.settings,
-            crawler=crawler
-        )
+        self.download_handler = create_instance(self.download_handler_cls, None, crawler)
         self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks
@@ -680,12 +662,7 @@ class HttpProxyTestCase(unittest.TestCase):
         wrapper = WrappingFactory(site)
         self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')
         self.portno = self.port.getHost().port
-        crawler = get_crawler()
-        self.download_handler = create_instance(
-            objcls=self.download_handler_cls,
-            settings=crawler.settings,
-            crawler=crawler
-        )
+        self.download_handler = create_instance(self.download_handler_cls, None, get_crawler())
         self.download_request = self.download_handler.download_request
 
     @defer.inlineCallbacks
@@ -764,7 +741,7 @@ class S3AnonTestCase(unittest.TestCase):
         crawler = get_crawler()
         self.s3reqh = create_instance(
             objcls=S3DownloadHandler,
-            settings=crawler.settings,
+            settings=None,
             crawler=crawler,
             httpdownloadhandler=HttpDownloadHandlerMock,
             # anon=True, # implicit
@@ -796,7 +773,7 @@ class S3TestCase(unittest.TestCase):
         crawler = get_crawler()
         s3reqh = create_instance(
             objcls=S3DownloadHandler,
-            settings=crawler.settings,
+            settings=None,
             crawler=crawler,
             aws_access_key_id=self.AWS_ACCESS_KEY_ID,
             aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
@@ -824,7 +801,7 @@ class S3TestCase(unittest.TestCase):
             crawler = get_crawler()
             create_instance(
                 objcls=S3DownloadHandler,
-                settings=crawler.settings,
+                settings=None,
                 crawler=crawler,
                 extra_kw=True,
             )

From ab54e0d33e2a461059a9f1e9759c95b18e38da28 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Dec 2019 20:37:18 -0300
Subject: [PATCH 2523/2974] Keyword-only args for S3DownloadHandler

---
 scrapy/core/downloader/handlers/s3.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index b38d2bf86..40a1fa48e 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -33,7 +33,8 @@ def _get_boto_connection():
 
 class S3DownloadHandler:
 
-    def __init__(self, settings, crawler=None,
+    def __init__(self, settings, *,
+                 crawler=None,
                  aws_access_key_id=None, aws_secret_access_key=None,
                  httpdownloadhandler=HTTPDownloadHandler, **kw):
         if not aws_access_key_id:
@@ -76,8 +77,8 @@ class S3DownloadHandler:
         self._download_http = _http_handler.download_request
 
     @classmethod
-    def from_crawler(cls, crawler, *args, **kwargs):
-        return cls(crawler.settings, crawler, *args, **kwargs)
+    def from_crawler(cls, crawler, **kwargs):
+        return cls(crawler.settings, crawler=crawler, **kwargs)
 
     def download_request(self, request, spider):
         p = urlparse_cached(request)

From 87ece066ca320b07acda57c99aee8a62992ec144 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 26 Dec 2019 20:41:06 +0500
Subject: [PATCH 2524/2974] Remove conditional asyncio imports.

---
 scrapy/utils/asyncio.py | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
index b53c8a8b0..917973de2 100644
--- a/scrapy/utils/asyncio.py
+++ b/scrapy/utils/asyncio.py
@@ -1,25 +1,17 @@
+import asyncio
 from contextlib import suppress
 
+from twisted.internet import asyncioreactor
 from twisted.internet.error import ReactorAlreadyInstalledError
 
 
 def install_asyncio_reactor():
     """ Tries to install AsyncioSelectorReactor
     """
-    try:
-        import asyncio
-        from twisted.internet import asyncioreactor
-    except ImportError:
-        return
-
     with suppress(ReactorAlreadyInstalledError):
         asyncioreactor.install(asyncio.get_event_loop())
 
 
 def is_asyncio_reactor_installed():
-    try:
-        import twisted.internet.reactor
-        from twisted.internet import asyncioreactor
-        return isinstance(twisted.internet.reactor, asyncioreactor.AsyncioSelectorReactor)
-    except ImportError:
-        return False
+    from twisted.internet import reactor
+    return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)

From 37ac47ff8074959ea66566fcb8b0e9e62272f963 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 26 Dec 2019 20:46:54 +0500
Subject: [PATCH 2525/2974] Fix a deprecation warning.

---
 tests/test_utils_asyncio.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index a6ba24876..44acc24af 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -10,7 +10,7 @@ class AsyncioTest(TestCase):
 
     def test_is_asyncio_reactor_installed(self):
         # the result should depend only on the pytest --reactor argument
-        self.assertEquals(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
+        self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
 
     def test_install_asyncio_reactor(self):
         # this should do nothing

From 8d4948f6ca44a76ee7714c8b4b1c46ef73a8845e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 26 Dec 2019 14:38:11 -0300
Subject: [PATCH 2526/2974] [test] Override CrawlSpider.parse

---
 tests/spiders.py    | 35 +++++++++++++++++++++++++++++------
 tests/test_crawl.py | 31 +++++++++++++++++++++++++++----
 2 files changed, 56 insertions(+), 10 deletions(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index 39c8da0b6..dcc475ca7 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -186,13 +186,39 @@ class DuplicateStartRequestsSpider(MockServerSpider):
         self.visited += 1
 
 
-class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider):
-    name = 'crawl_spider_with_errback'
+class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
+    """
+    A CrawlSpider which overrides the 'parse' method
+    """
+    name = 'crawl_spider_with_parse_method'
     custom_settings = {
         'RETRY_HTTP_CODES': [],  # no need to retry
     }
     rules = (
-        Rule(LinkExtractor(), callback='callback', errback='errback', follow=True),
+        Rule(LinkExtractor(), callback='parse', follow=True),
+    )
+
+    def start_requests(self):
+        test_body = b"""
+        <html>
+            <head><title>Page title<title></head>
+            <body>
+                <p><a href="/status?n=200">Item 200</a></p>  <!-- callback -->
+                <p><a href="/status?n=201">Item 201</a></p>  <!-- callback -->
+            </body>
+        </html>
+        """
+        url = self.mockserver.url("/alpayload")
+        yield Request(url, method="POST", body=test_body)
+
+    def parse(self, response):
+        self.logger.info('[parse] status %i', response.status)
+
+
+class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
+    name = 'crawl_spider_with_errback'
+    rules = (
+        Rule(LinkExtractor(), callback='parse', errback='errback', follow=True),
     )
 
     def start_requests(self):
@@ -211,8 +237,5 @@ class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider):
         url = self.mockserver.url("/alpayload")
         yield Request(url, method="POST", body=test_body)
 
-    def callback(self, response):
-        self.logger.info('[callback] status %i', response.status)
-
     def errback(self, failure):
         self.logger.info('[errback] status %i', failure.value.response.status)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index f433fcea6..4299e4bbb 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -9,8 +9,9 @@ from scrapy.crawler import CrawlerRunner
 from scrapy.http import Request
 from scrapy.utils.python import to_unicode
 from tests.mockserver import MockServer
-from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
-                           SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback)
+from tests.spiders import (BrokenStartRequestsSpider, CrawlSpiderWithErrback,
+                           CrawlSpiderWithParseMethod, DelaySpider, SimpleSpider,
+                           DuplicateStartRequestsSpider, FollowAllSpider, SingleRequestSpider)
 
 
 class CrawlTestCase(TestCase):
@@ -297,6 +298,27 @@ with multiples lines
         self._assert_retried(log)
         self.assertIn("Got response 200", str(log))
 
+
+class CrawlSpiderTestCase(TestCase):
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self.runner = CrawlerRunner()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_crawlspider_with_parse(self):
+        self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver)
+
+        with LogCapture() as log:
+            yield self.runner.join()
+
+        self.assertIn("[parse] status 200", str(log))
+        self.assertIn("[parse] status 201", str(log))
+
     @defer.inlineCallbacks
     def test_crawlspider_with_errback(self):
         self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver)
@@ -304,7 +326,8 @@ with multiples lines
         with LogCapture() as log:
             yield self.runner.join()
 
-        self.assertIn("[callback] status 200", str(log))
-        self.assertIn("[callback] status 201", str(log))
+        self.assertIn("[parse] status 200", str(log))
+        self.assertIn("[parse] status 201", str(log))
         self.assertIn("[errback] status 404", str(log))
         self.assertIn("[errback] status 500", str(log))
+        self.assertIn("[errback] status 501", str(log))

From c54df8253a67bb6863a25bf7d667096adc73a040 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 26 Dec 2019 15:12:19 -0300
Subject: [PATCH 2527/2974] [test] Handle keyword args in CrawlSpider.parse

---
 tests/spiders.py    |  5 +++--
 tests/test_crawl.py | 10 ++++++----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index dcc475ca7..c042eb7fe 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -211,8 +211,9 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
         url = self.mockserver.url("/alpayload")
         yield Request(url, method="POST", body=test_body)
 
-    def parse(self, response):
-        self.logger.info('[parse] status %i', response.status)
+    def parse(self, response, foo=None):
+        self.logger.info('[parse] status %i (foo: %s)', response.status, foo)
+        yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"})
 
 
 class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 4299e4bbb..6247ced35 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -316,8 +316,9 @@ class CrawlSpiderTestCase(TestCase):
         with LogCapture() as log:
             yield self.runner.join()
 
-        self.assertIn("[parse] status 200", str(log))
-        self.assertIn("[parse] status 201", str(log))
+        self.assertIn("[parse] status 200 (foo: None)", str(log))
+        self.assertIn("[parse] status 201 (foo: None)", str(log))
+        self.assertIn("[parse] status 202 (foo: bar)", str(log))
 
     @defer.inlineCallbacks
     def test_crawlspider_with_errback(self):
@@ -326,8 +327,9 @@ class CrawlSpiderTestCase(TestCase):
         with LogCapture() as log:
             yield self.runner.join()
 
-        self.assertIn("[parse] status 200", str(log))
-        self.assertIn("[parse] status 201", str(log))
+        self.assertIn("[parse] status 200 (foo: None)", str(log))
+        self.assertIn("[parse] status 201 (foo: None)", str(log))
+        self.assertIn("[parse] status 202 (foo: bar)", str(log))
         self.assertIn("[errback] status 404", str(log))
         self.assertIn("[errback] status 500", str(log))
         self.assertIn("[errback] status 501", str(log))

From 8a1dc26d4662ec0c81d22c13cffa44255a7c29ca Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 26 Dec 2019 15:14:47 -0300
Subject: [PATCH 2528/2974] [doc] Note about the 'parse' method for
 CrawlSpider/XMLFeedSpider

---
 docs/topics/spiders.rst | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index dd763b607..406f50fb3 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -392,11 +392,6 @@ Crawling rules
    object will contain the text of the link that produced the :class:`~scrapy.http.Request`
    in its ``meta`` dictionary (under the ``link_text`` key)
 
-   .. warning:: When writing crawl spider rules, avoid using ``parse`` as
-       callback, since the :class:`CrawlSpider` uses the ``parse`` method
-       itself to implement its logic. So if you override the ``parse`` method,
-       the crawl spider will no longer work.
-
    ``cb_kwargs`` is a dict containing the keyword arguments to be passed to the
    callback function.
 
@@ -422,6 +417,12 @@ Crawling rules
    It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
    instance as first parameter.
 
+
+.. warning:: Because of its internal implementation, you must explicitly set
+   callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
+   unexpected behaviour can occur otherwise.
+
+
 CrawlSpider example
 ~~~~~~~~~~~~~~~~~~~
 
@@ -452,6 +453,11 @@ Let's now take a look at an example CrawlSpider with rules::
             item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
             item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
             item['link_text'] = response.meta['link_text']
+            url = response.xpath('//td[@id="additional_data"]/@href').get()
+            return response.follow(url, self.parse_additional_page, cb_kwargs=dict(item=item))
+
+        def parse_additional_page(self, response, item):
+            item['additional_data'] = response.xpath('//p[@id="additional_data"]/text()').get()
             return item
 
 
@@ -545,6 +551,11 @@ XMLFeedSpider
         those results. It must return a list of results (Items or Requests).
 
 
+.. warning:: Because of its internal implementation, you must explicitly set
+   callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;
+   unexpected behaviour can occur otherwise.
+
+
 XMLFeedSpider example
 ~~~~~~~~~~~~~~~~~~~~~
 

From 30ebd05a5f627262702ad1e1e488d6176a8c7882 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 27 Dec 2019 00:05:14 +0500
Subject: [PATCH 2529/2974] Simplify the tox asyncio entries.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index ed0d4c9ab..b62100026 100644
--- a/tox.ini
+++ b/tox.ini
@@ -99,7 +99,7 @@ commands =
 
 [asyncio]
 commands =
-    py.test --cov=scrapy --cov-report= --reactor=asyncio {posargs:scrapy tests}
+    {[testenv]commands} --reactor=asyncio
 
 [testenv:py35-asyncio]
 basepython = python3.5

From f75ccc997aa75fadf96a8d4b836248397ef89802 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 27 Dec 2019 19:48:54 +0500
Subject: [PATCH 2530/2974] FIx a typo in the only_asyncio fixture.

---
 conftest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/conftest.py b/conftest.py
index 6d9696a3f..c0de09909 100644
--- a/conftest.py
+++ b/conftest.py
@@ -48,4 +48,4 @@ def reactor_pytest(request):
 @pytest.fixture(autouse=True)
 def only_asyncio(request, reactor_pytest):
     if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio':
-        pytest.skip('This test is only run with --reactor-asyncio')
+        pytest.skip('This test is only run with --reactor=asyncio')

From dc1ee09481c7655a8ebf77a75ccc965a4ba5400d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 27 Dec 2019 21:55:58 +0500
Subject: [PATCH 2531/2974] Rename ASYNCIO_ENABLED to ASYNCIO_REACTOR, change
 the logic accordingly.

---
 docs/topics/settings.rst                      | 14 +++----
 scrapy/crawler.py                             | 17 +++++---
 scrapy/settings/default_settings.py           |  2 +-
 scrapy/utils/log.py                           |  5 ++-
 .../asyncio_enabled_no_reactor.py             |  2 +-
 .../CrawlerProcess/asyncio_enabled_reactor.py |  2 +-
 tests/test_commands.py                        |  8 ++--
 tests/test_crawler.py                         | 42 ++++++++-----------
 8 files changed, 43 insertions(+), 49 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 62b2870b2..c02f877fc 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -160,26 +160,22 @@ to any particular component. In that case the module of that component will be
 shown, typically an extension, middleware or pipeline. It also means that the
 component must be enabled in order for the setting to have any effect.
 
-.. setting:: ASYNCIO_ENABLED
+.. setting:: ASYNCIO_REACTOR
 
-ASYNCIO_ENABLED
+ASYNCIO_REACTOR
 ---------------
 
 Default: ``False``
 
-Whether to support ``async def`` methods and callbacks which use code that
-requires an asyncio loop.
-
-If an ``async def`` coroutine doesn't require the asyncio loop, it will work
-even if this is set to ``False``. Coroutines that require the asyncio loop may
-silently fail to run or raise errors unless this is set to ``True``.
+Whether to install and require the Twisted reactor that uses the asyncio loop.
 
 When this option is set to ``True``, Scrapy will require
 :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. It will
 install this reactor if no reactor is installed yet, such as when using the
 ``scrapy`` script or :class:`~scrapy.crawler.CrawlerProcess`. If you are using
 :class:`~scrapy.crawler.CrawlerRunner`, you need to install the correct reactor
-manually.
+manually. If a different reactor is installed outside Scrapy, it will raise an
+exception.
 
 The default value for this option is currently ``False`` to maintain backward
 compatibility and avoid possible problems caused by using a different Twisted
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index a9443f7ac..f87e67d93 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -137,6 +137,7 @@ class CrawlerRunner(object):
         self._crawlers = set()
         self._active = set()
         self.bootstrap_failed = False
+        self._handle_asyncio_reactor()
 
     @property
     def spiders(self):
@@ -230,6 +231,11 @@ class CrawlerRunner(object):
         while self._active:
             yield defer.DeferredList(self._active)
 
+    def _handle_asyncio_reactor(self):
+        if self.settings.getbool('ASYNCIO_REACTOR') and not is_asyncio_reactor_installed():
+            raise Exception("ASYNCIO_REACTOR is on but the Twisted asyncio "
+                            "reactor is not installed.")
+
 
 class CrawlerProcess(CrawlerRunner):
     """
@@ -257,12 +263,6 @@ class CrawlerProcess(CrawlerRunner):
 
     def __init__(self, settings=None, install_root_handler=True):
         super(CrawlerProcess, self).__init__(settings)
-        if self.settings.getbool('ASYNCIO_ENABLED'):
-            install_asyncio_reactor()
-            if not is_asyncio_reactor_installed():
-                raise Exception("ASYNCIO_ENABLED is on but the Twisted asyncio "
-                                "reactor is not installed, this is not supported.")
-
         install_shutdown_handlers(self._signal_shutdown)
         configure_logging(self.settings, install_root_handler)
         log_scrapy_info(self.settings)
@@ -333,6 +333,11 @@ class CrawlerProcess(CrawlerRunner):
         except RuntimeError:  # raised if already stopped or in shutdown stage
             pass
 
+    def _handle_asyncio_reactor(self):
+        if self.settings.getbool('ASYNCIO_REACTOR'):
+            install_asyncio_reactor()
+        super()._handle_asyncio_reactor()
+
 
 def _get_spider_loader(settings):
     """ Get SpiderLoader instance from settings """
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index a7792e248..d03fd37b0 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -19,7 +19,7 @@ from os.path import join, abspath, dirname
 
 AJAXCRAWL_ENABLED = False
 
-ASYNCIO_ENABLED = False
+ASYNCIO_REACTOR = False
 
 AUTOTHROTTLE_ENABLED = False
 AUTOTHROTTLE_DEBUG = False
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 6179e1bd1..e4cf0196b 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -11,6 +11,7 @@ from twisted.python import log as twisted_log
 import scrapy
 from scrapy.settings import Settings
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.asyncio import is_asyncio_reactor_installed
 from scrapy.utils.versions import scrapy_components_versions
 
 
@@ -148,8 +149,8 @@ def log_scrapy_info(settings):
                 {'versions': ", ".join("%s %s" % (name, version)
                     for name, version in scrapy_components_versions()
                     if name != "Scrapy")})
-    if settings.getbool('ASYNCIO_ENABLED'):
-        logger.debug("Asyncio support enabled")
+    if is_asyncio_reactor_installed():
+        logger.debug("Asyncio reactor is installed")
 
 
 class StreamLogger(object):
diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
index dfe028ef4..db1b75931 100644
--- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
@@ -10,7 +10,7 @@ class NoRequestsSpider(scrapy.Spider):
 
 
 process = CrawlerProcess(settings={
-    'ASYNCIO_ENABLED': True,
+    'ASYNCIO_REACTOR': True,
 })
 
 process.crawl(NoRequestsSpider)
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py
index 7a172ea28..cec3c9c25 100644
--- a/tests/CrawlerProcess/asyncio_enabled_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py
@@ -15,7 +15,7 @@ class NoRequestsSpider(scrapy.Spider):
 
 
 process = CrawlerProcess(settings={
-    'ASYNCIO_ENABLED': True,
+    'ASYNCIO_REACTOR': True,
 })
 
 process.crawl(NoRequestsSpider)
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 197d80217..6024af71c 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -296,12 +296,12 @@ class BadSpider(scrapy.Spider):
         self.assertIn("badspider.py", log)
 
     def test_asyncio_enabled_true(self):
-        log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_ENABLED=True'])
-        self.assertIn("DEBUG: Asyncio support enabled", log)
+        log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=True'])
+        self.assertIn("DEBUG: Asyncio reactor is installed", log)
 
     def test_asyncio_enabled_false(self):
-        log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_ENABLED=False'])
-        self.assertNotIn("DEBUG: Asyncio support enabled", log)
+        log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=False'])
+        self.assertNotIn("DEBUG: Asyncio reactor is installed", log)
 
 
 class BenchCommandTest(CommandTest):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index a2865fcd1..fce60ca37 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -13,7 +13,6 @@ import scrapy
 from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
 from scrapy.settings import Settings, default_settings
 from scrapy.spiderloader import SpiderLoader
-from scrapy.utils.asyncio import is_asyncio_reactor_installed
 from scrapy.utils.log import configure_logging, get_scrapy_root_handler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
@@ -209,14 +208,6 @@ class NoRequestsSpider(scrapy.Spider):
         return []
 
 
-class AsyncioSpider(scrapy.Spider):
-    name = 'asyncio'
-
-    def start_requests(self):
-        self.logger.info('Asyncio support: %s', is_asyncio_reactor_installed())
-        return []
-
-
 @mark.usefixtures('reactor_pytest')
 class CrawlerRunnerHasSpider(unittest.TestCase):
 
@@ -261,31 +252,32 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
         self.assertEqual(runner.bootstrap_failed, True)
 
+    def test_crawler_runner_asyncio_enabled_true(self):
+        if self.reactor_pytest == 'asyncio':
+            runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True})
+        else:
+            msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed"
+            with self.assertRaisesRegex(Exception, msg):
+                runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True})
+
     @defer.inlineCallbacks
     def test_crawler_process_asyncio_enabled_true(self):
         with LogCapture(level=logging.DEBUG) as log:
             if self.reactor_pytest == 'asyncio':
-                runner = CrawlerProcess(settings={'ASYNCIO_ENABLED': True})
+                runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True})
                 yield runner.crawl(NoRequestsSpider)
-                self.assertIn("Asyncio support enabled", str(log))
+                self.assertIn("Asyncio reactor is installed", str(log))
             else:
-                msg = "ASYNCIO_ENABLED is on but the Twisted asyncio reactor is not installed"
+                msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed"
                 with self.assertRaisesRegex(Exception, msg):
-                    runner = CrawlerProcess(settings={'ASYNCIO_ENABLED': True})
+                    runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True})
 
     @defer.inlineCallbacks
     def test_crawler_process_asyncio_enabled_false(self):
-        runner = CrawlerProcess(settings={'ASYNCIO_ENABLED': False})
+        runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': False})
         with LogCapture(level=logging.DEBUG) as log:
             yield runner.crawl(NoRequestsSpider)
-            self.assertNotIn("Asyncio support enabled", str(log))
-
-    @defer.inlineCallbacks
-    def test_crawler_runner_asyncio_supported(self):
-        runner = CrawlerRunner()
-        with LogCapture() as log:
-            yield runner.crawl(AsyncioSpider)
-            log.check_present(('asyncio', 'INFO', 'Asyncio support: %s' % (self.reactor_pytest == 'asyncio')))
+            self.assertNotIn("Asyncio reactor is installed", str(log))
 
 
 class CrawlerProcessSubprocess(unittest.TestCase):
@@ -302,14 +294,14 @@ class CrawlerProcessSubprocess(unittest.TestCase):
     def test_simple(self):
         log = self.run_script('simple.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertNotIn("DEBUG: Asyncio support enabled", log)
+        self.assertNotIn("DEBUG: Asyncio reactor is installed", log)
 
     def test_asyncio_enabled_no_reactor(self):
         log = self.run_script('asyncio_enabled_no_reactor.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertIn("DEBUG: Asyncio support enabled", log)
+        self.assertIn("DEBUG: Asyncio reactor is installed", log)
 
     def test_asyncio_enabled_reactor(self):
         log = self.run_script('asyncio_enabled_reactor.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertIn("DEBUG: Asyncio support enabled", log)
+        self.assertIn("DEBUG: Asyncio reactor is installed", log)

From 82861c73c8f74c3416f5fa5d77ded33802ca535f Mon Sep 17 00:00:00 2001
From: Atul Gopinathan <41539794+atul-g@users.noreply.github.com>
Date: Fri, 27 Dec 2019 22:57:58 +0530
Subject: [PATCH 2532/2974] Edited the link of the homepage of lxml website

The link "https://lxml.de" is redirecting to a completely different and unintended website. I changed the link to the index page of lxml's official website. I thought of changing it to the PyPi page of lxml, but even they are providing the same "https://lxml.de" link which doesn't seem to be working now.
---
 docs/intro/install.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 51b41b4d7..8ce13d715 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -278,7 +278,7 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
 
 .. _Python: https://www.python.org/
 .. _pip: https://pip.pypa.io/en/latest/installing/
-.. _lxml: http://lxml.de/
+.. _lxml: https://lxml.de/index.html
 .. _parsel: https://pypi.python.org/pypi/parsel
 .. _w3lib: https://pypi.python.org/pypi/w3lib
 .. _twisted: https://twistedmatrix.com/

From 14d4428e705e9ca739a4fd041ce7e3134749363c Mon Sep 17 00:00:00 2001
From: 1um0s <abitha95@gmail.com>
Date: Mon, 30 Dec 2019 01:26:22 +0530
Subject: [PATCH 2533/2974] Rephrasing documentation for image and file
 pipelines (#4252)

* scrapy#4034 Clarify documentation for image and file pipelines

* scrapy#4034 Clarify documentation for file pipeline

* scrapy#4034 Simplify documentation for pipeline

* scrapy#4034 Simplify documentation for pipeline

* scrapy#4034 Clarify documentation for image and file pipelines

* scrapy#4034 Clarify documentation for file pipeline

* scrapy#4034 Simplify documentation for pipeline

* scrapy#4034 Simplify documentation for pipeline

* scrapy#4034 Revert image, file pipeline docs. Enhance custom media pipeline docs.

* scrapy#4034 rebase master

* scrapy#4034 Clarify documentation for image and file pipelines

* scrapy#4034 Clarify documentation for file pipeline

* scrapy#4034 Simplify documentation for pipeline

* scrapy#4034 Simplify documentation for pipeline

* scrapy#4034 Clarify documentation for image and file pipelines

* scrapy#4034 Clarify documentation for file pipeline

* scrapy#4034 Simplify documentation for pipeline

* scrapy#4034 Simplify documentation for pipeline

* scrapy#4034 Revert image, file pipeline docs. Enhance custom media pipeline docs.

* scrapy#4034 rebase master

* Rebase master

* Add class to media pipeline docs

Co-Authored-By: elacuesta <elacuesta@users.noreply.github.com>

Co-authored-by: elacuesta <elacuesta@users.noreply.github.com>
---
 docs/topics/media-pipeline.rst | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index a40682e5b..332a14eb7 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -97,7 +97,6 @@ For Files Pipeline, use::
 
     ITEM_PIPELINES = {'scrapy.pipelines.files.FilesPipeline': 1}
 
-
 .. note::
     You can also use both the Files and Images Pipeline at the same time.
 
@@ -578,4 +577,12 @@ above::
             item['image_paths'] = image_paths
             return item
 
+
+To enable your custom media pipeline component you must add its class import path to the
+:setting:`ITEM_PIPELINES` setting, like in the following example::
+
+   ITEM_PIPELINES = {
+       'myproject.pipelines.MyImagesPipeline': 300
+   }
+
 .. _MD5 hash: https://en.wikipedia.org/wiki/MD5

From 21f50c795ac6978de9e73b64f31542a9df928ac5 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 30 Jul 2019 19:46:18 +0500
Subject: [PATCH 2534/2974] Add async def support to downloader middlewares.

---
 scrapy/core/downloader/middleware.py |  8 ++++----
 tests/test_downloadermiddleware.py   | 24 ++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 38608a429..9c0014206 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -8,7 +8,7 @@ from twisted.internet import defer
 from scrapy.exceptions import _InvalidOutput
 from scrapy.http import Request, Response
 from scrapy.middleware import MiddlewareManager
-from scrapy.utils.defer import mustbe_deferred
+from scrapy.utils.defer import mustbe_deferred, deferred_from_coro
 from scrapy.utils.conf import build_component_list
 
 
@@ -33,7 +33,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         @defer.inlineCallbacks
         def process_request(request):
             for method in self.methods['process_request']:
-                response = yield method(request=request, spider=spider)
+                response = yield deferred_from_coro(method(request=request, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
                     raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
                                          (method.__self__.__class__.__name__, response.__class__.__name__))
@@ -48,7 +48,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 defer.returnValue(response)
 
             for method in self.methods['process_response']:
-                response = yield method(request=request, response=response, spider=spider)
+                response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
                 if not isinstance(response, (Response, Request)):
                     raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
                                          (method.__self__.__class__.__name__, type(response)))
@@ -60,7 +60,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         def process_exception(_failure):
             exception = _failure.value
             for method in self.methods['process_exception']:
-                response = yield method(request=request, exception=exception, spider=spider)
+                response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
                     raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
                                          (method.__self__.__class__.__name__, type(response)))
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 1b81ea949..135321d0a 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -1,3 +1,4 @@
+import asyncio
 from unittest import mock
 
 from twisted.internet.defer import Deferred
@@ -206,3 +207,26 @@ class MiddlewareUsingDeferreds(ManagerTestCase):
 
         self.assertIs(results[0], resp)
         self.assertFalse(download_func.called)
+
+
+class MiddlewareUsingCoro(ManagerTestCase):
+    """Middlewares using asyncio coroutines should work"""
+
+    def test_asyncdef(self):
+        resp = Response('http://example.com/index.html')
+
+        class CoroMiddleware:
+            async def process_request(self, request, spider):
+                await asyncio.sleep(0.1)
+                return resp
+
+        self.mwman._add_middleware(CoroMiddleware())
+        req = Request('http://example.com/index.html')
+        download_func = mock.MagicMock()
+        dfd = self.mwman.download(download_func, req, self.spider)
+        results = []
+        dfd.addBoth(results.append)
+        self._wait(dfd)
+
+        self.assertIs(results[0], resp)
+        self.assertFalse(download_func.called)

From 3603644552f8d15d203abca221edb56047119528 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 12 Sep 2019 20:25:29 +0500
Subject: [PATCH 2535/2974] Add a non-asyncio async def middleware test.

---
 tests/test_downloadermiddleware.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 135321d0a..5b0cf1eb7 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -1,6 +1,7 @@
 import asyncio
 from unittest import mock
 
+from twisted.internet import defer
 from twisted.internet.defer import Deferred
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
@@ -215,6 +216,25 @@ class MiddlewareUsingCoro(ManagerTestCase):
     def test_asyncdef(self):
         resp = Response('http://example.com/index.html')
 
+        class CoroMiddleware:
+            async def process_request(self, request, spider):
+                await defer.succeed(42)
+                return resp
+
+        self.mwman._add_middleware(CoroMiddleware())
+        req = Request('http://example.com/index.html')
+        download_func = mock.MagicMock()
+        dfd = self.mwman.download(download_func, req, self.spider)
+        results = []
+        dfd.addBoth(results.append)
+        self._wait(dfd)
+
+        self.assertIs(results[0], resp)
+        self.assertFalse(download_func.called)
+
+    def test_asyncdef_asyncio(self):
+        resp = Response('http://example.com/index.html')
+
         class CoroMiddleware:
             async def process_request(self, request, spider):
                 await asyncio.sleep(0.1)

From 5cf1ac0005fa3a174b700d88c0e2536b689f13c7 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 16 Dec 2019 19:24:44 +0500
Subject: [PATCH 2536/2974] Move the asyncio downloader mw test to a separate
 class.

---
 tests/test_downloadermiddleware.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 5b0cf1eb7..c5c4d13bd 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -1,6 +1,7 @@
 import asyncio
 from unittest import mock
 
+from pytest import mark
 from twisted.internet import defer
 from twisted.internet.defer import Deferred
 from twisted.trial.unittest import TestCase
@@ -232,6 +233,12 @@ class MiddlewareUsingCoro(ManagerTestCase):
         self.assertIs(results[0], resp)
         self.assertFalse(download_func.called)
 
+
+@mark.only_asyncio()
+class MiddlewareUsingCoroAsyncio(ManagerTestCase):
+
+    settings_dict = {'ASYNCIO_ENABLED': True}
+
     def test_asyncdef_asyncio(self):
         resp = Response('http://example.com/index.html')
 

From 50aa6ef22cc9dec3655c9e3aaee225f159e94df1 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 21 Dec 2019 14:36:11 +0500
Subject: [PATCH 2537/2974] Add deferred_from_coro.

---
 scrapy/utils/defer.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 20ce59297..bbd5ebe52 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -1,10 +1,14 @@
 """
 Helper functions for dealing with Twisted deferreds
 """
+import asyncio
+import inspect
+
 from twisted.internet import defer, task
 from twisted.python import failure
 
 from scrapy.exceptions import IgnoreRequest
+from scrapy.utils.asyncio import is_asyncio_reactor_installed
 
 
 def defer_fail(_failure):
@@ -114,3 +118,25 @@ def iter_errback(iterable, errback, *a, **kw):
             break
         except Exception:
             errback(failure.Failure(), *a, **kw)
+
+
+def _isfuture(o):
+    # workaround for Python before 3.5.3 not having asyncio.isfuture
+    if hasattr(asyncio, 'isfuture'):
+        return asyncio.isfuture(o)
+    return isinstance(o, asyncio.Future)
+
+
+def deferred_from_coro(o):
+    """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
+    if isinstance(o, defer.Deferred):
+        return o
+    if _isfuture(o) or inspect.isawaitable(o):
+        if not is_asyncio_reactor_installed():
+            # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
+            # that use asyncio, e.g. "await asyncio.sleep(1)"
+            return defer.ensureDeferred(o)
+        else:
+            # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
+            return defer.Deferred.fromFuture(asyncio.ensure_future(o))
+    return o

From 16787f5bf4475fe1604c1e4cac7b491f1df6a1fb Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 30 Dec 2019 12:02:19 +0500
Subject: [PATCH 2538/2974] Merge middleware tests back as we don't need to set
 the setting anymore.

---
 tests/test_downloadermiddleware.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index c5c4d13bd..3943cecf7 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -233,12 +233,7 @@ class MiddlewareUsingCoro(ManagerTestCase):
         self.assertIs(results[0], resp)
         self.assertFalse(download_func.called)
 
-
-@mark.only_asyncio()
-class MiddlewareUsingCoroAsyncio(ManagerTestCase):
-
-    settings_dict = {'ASYNCIO_ENABLED': True}
-
+    @mark.only_asyncio()
     def test_asyncdef_asyncio(self):
         resp = Response('http://example.com/index.html')
 

From e3b8ba6188ce703e43563d8588b7d709b037a0e0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 31 Dec 2019 17:54:01 +0500
Subject: [PATCH 2539/2974] Run py35-asyncio also on 3.5.2 to test Xenial.

---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 82167e10a..c808b3436 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,6 +18,8 @@ matrix:
       python: 3.5
     - env: TOXENV=py35-asyncio
       python: 3.5
+    - env: TOXENV=py35-asyncio
+      python: 3.5.2
     - env: TOXENV=py36
       python: 3.6
     - env: TOXENV=py37

From 2b9254c2bde76995e81c54ad112ae884a1499386 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 31 Dec 2019 17:54:41 +0500
Subject: [PATCH 2540/2974] Add a test function that uses asyncio.Queue().

---
 scrapy/utils/test.py               | 9 ++++++++-
 tests/test_downloadermiddleware.py | 5 +++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 307c25352..0f4cf8091 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -1,7 +1,7 @@
 """
 This module contains some assorted functions used in tests
 """
-
+import asyncio
 import os
 
 from importlib import import_module
@@ -96,3 +96,10 @@ def assert_samelines(testcase, text1, text2, msg=None):
     line endings between platforms
     """
     testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
+
+
+def get_from_asyncio_queue(value):
+    q = asyncio.Queue()
+    getter = q.get()
+    q.put_nowait(value)
+    return getter
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 3943cecf7..3dd4f2351 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -11,7 +11,7 @@ from scrapy.http import Request, Response
 from scrapy.spiders import Spider
 from scrapy.exceptions import _InvalidOutput
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
-from scrapy.utils.test import get_crawler
+from scrapy.utils.test import get_crawler, get_from_asyncio_queue
 from scrapy.utils.python import to_bytes
 
 
@@ -240,7 +240,8 @@ class MiddlewareUsingCoro(ManagerTestCase):
         class CoroMiddleware:
             async def process_request(self, request, spider):
                 await asyncio.sleep(0.1)
-                return resp
+                result = await get_from_asyncio_queue(resp)
+                return result
 
         self.mwman._add_middleware(CoroMiddleware())
         req = Request('http://example.com/index.html')

From b2dd379bc2b93b4863dd36a296780481d758c4cb Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 3 Jan 2020 21:38:05 +0500
Subject: [PATCH 2541/2974] Remove the py35-asyncio env for 3.5 from Travis.

---
 .travis.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index c808b3436..66e1a9617 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,8 +16,6 @@ matrix:
       python: 3.5
     - env: TOXENV=pinned
       python: 3.5
-    - env: TOXENV=py35-asyncio
-      python: 3.5
     - env: TOXENV=py35-asyncio
       python: 3.5.2
     - env: TOXENV=py36

From 81175669746737f1ab0bd0236a70f787ed1855a4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 16 Dec 2019 22:12:27 +0500
Subject: [PATCH 2542/2974] Add utils.defer.deferred_f_from_coro_f.

---
 scrapy/utils/defer.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index bbd5ebe52..62b43a96c 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -2,6 +2,7 @@
 Helper functions for dealing with Twisted deferreds
 """
 import asyncio
+from functools import wraps
 import inspect
 
 from twisted.internet import defer, task
@@ -140,3 +141,15 @@ def deferred_from_coro(o):
             # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
             return defer.Deferred.fromFuture(asyncio.ensure_future(o))
     return o
+
+
+def deferred_f_from_coro_f(coro_f):
+    """ Converts a coroutine function into a function that returns a Deferred.
+
+    The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it.
+    This is useful for callback chains, as callback functions are called with the previous callback result.
+    """
+    @wraps(coro_f)
+    def f(*coro_args, **coro_kwargs):
+        return deferred_from_coro(coro_f(*coro_args, **coro_kwargs))
+    return f

From 1f9cef787d3ca0c12099f1b1b4c52efc510e381d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 10 Sep 2019 14:26:21 +0500
Subject: [PATCH 2543/2974] Add async def support to pipelines.

---
 scrapy/pipelines/__init__.py |  4 +++-
 tests/test_pipelines.py      | 13 +++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index aa1bfb77f..1a45e00a2 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -6,6 +6,8 @@ See documentation in docs/item-pipeline.rst
 
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.conf import build_component_list
+from scrapy.utils.defer import deferred_f_from_coro_f
+
 
 
 class ItemPipelineManager(MiddlewareManager):
@@ -19,7 +21,7 @@ class ItemPipelineManager(MiddlewareManager):
     def _add_middleware(self, pipe):
         super(ItemPipelineManager, self)._add_middleware(pipe)
         if hasattr(pipe, 'process_item'):
-            self.methods['process_item'].append(pipe.process_item)
+            self.methods['process_item'].append(deferred_f_from_coro_f(pipe.process_item))
 
     def process_item(self, item, spider):
         return self._process_chain('process_item', item, spider)
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index bc53f5427..cfe4471d7 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -26,6 +26,13 @@ class DeferredPipeline:
         return d
 
 
+class AsyncDefPipeline:
+    async def process_item(self, item, spider):
+        await defer.succeed(42)
+        item['pipeline_passed'] = True
+        return item
+
+
 class ItemSpider(Spider):
     name = 'itemspider'
 
@@ -69,3 +76,9 @@ class PipelineTestCase(unittest.TestCase):
         crawler = self._create_crawler(DeferredPipeline)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertEqual(len(self.items), 1)
+
+    @defer.inlineCallbacks
+    def test_asyncdef_pipeline(self):
+        crawler = self._create_crawler(AsyncDefPipeline)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertEqual(len(self.items), 1)

From bfdd552a32b3a67dfa2895b483c187d87b63b50e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 10 Sep 2019 14:57:07 +0500
Subject: [PATCH 2544/2974] Add a test for pipelines using asyncio.

---
 tests/test_pipelines.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index cfe4471d7..aba6d85b7 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -1,3 +1,5 @@
+import asyncio
+
 from twisted.internet import defer
 from twisted.internet.defer import Deferred
 from twisted.trial import unittest
@@ -33,6 +35,13 @@ class AsyncDefPipeline:
         return item
 
 
+class AsyncDefAsyncioPipeline:
+    async def process_item(self, item, spider):
+        await asyncio.sleep(0.2)
+        item['pipeline_passed'] = True
+        return item
+
+
 class ItemSpider(Spider):
     name = 'itemspider'
 
@@ -82,3 +91,9 @@ class PipelineTestCase(unittest.TestCase):
         crawler = self._create_crawler(AsyncDefPipeline)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertEqual(len(self.items), 1)
+
+    @defer.inlineCallbacks
+    def test_asyncdef_asyncio_pipeline(self):
+        crawler = self._create_crawler(AsyncDefAsyncioPipeline)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertEqual(len(self.items), 1)

From bdef948aaebced3d28ea7b81525ea9edf7cc650c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 16 Dec 2019 23:15:43 +0500
Subject: [PATCH 2545/2974] Mark the asyncio pipelines test as only_asyncio.

---
 tests/test_pipelines.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index aba6d85b7..6f33282f2 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -1,5 +1,6 @@
 import asyncio
 
+from pytest import mark
 from twisted.internet import defer
 from twisted.internet.defer import Deferred
 from twisted.trial import unittest
@@ -92,6 +93,7 @@ class PipelineTestCase(unittest.TestCase):
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertEqual(len(self.items), 1)
 
+    @mark.only_asyncio()
     @defer.inlineCallbacks
     def test_asyncdef_asyncio_pipeline(self):
         crawler = self._create_crawler(AsyncDefAsyncioPipeline)

From 9d8c54c0f2a98dc627efe8a2b58cfd311f2105cb Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 16 Dec 2019 22:43:55 +0500
Subject: [PATCH 2546/2974] Fix/ignore flake8 problems.

---
 pytest.ini                   | 1 +
 scrapy/pipelines/__init__.py | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/pytest.ini b/pytest.ini
index c3f3292bb..1030d5530 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -95,6 +95,7 @@ flake8-ignore =
     scrapy/loader/__init__.py E501 E128
     scrapy/loader/processors.py E501
     # scrapy/pipelines
+    scrapy/pipelines/__init__.py E501
     scrapy/pipelines/files.py E116 E501 E266
     scrapy/pipelines/images.py E265 E501
     scrapy/pipelines/media.py E125 E501 E266
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index 1a45e00a2..b5725a8ee 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -9,7 +9,6 @@ from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import deferred_f_from_coro_f
 
 
-
 class ItemPipelineManager(MiddlewareManager):
 
     component_name = 'item pipeline'

From 7d859848800ef05760e4f57dc206a4d756460a9d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 9 Jan 2020 14:48:07 +0500
Subject: [PATCH 2547/2974] Use get_from_asyncio_queue in the pipeline test.

---
 tests/test_pipelines.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index 6f33282f2..c72f1a338 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -6,7 +6,7 @@ from twisted.internet.defer import Deferred
 from twisted.trial import unittest
 
 from scrapy import Spider, signals, Request
-from scrapy.utils.test import get_crawler
+from scrapy.utils.test import get_crawler, get_from_asyncio_queue
 
 from tests.mockserver import MockServer
 
@@ -39,7 +39,7 @@ class AsyncDefPipeline:
 class AsyncDefAsyncioPipeline:
     async def process_item(self, item, spider):
         await asyncio.sleep(0.2)
-        item['pipeline_passed'] = True
+        item['pipeline_passed'] = await get_from_asyncio_queue(True)
         return item
 
 

From 3faef2d08277f43659400470ec408ee29400018a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 12 Sep 2019 20:10:58 +0500
Subject: [PATCH 2548/2974] Add async def support to signal handlers that
 already supported Deferreds.

---
 scrapy/utils/defer.py      | 17 +++++++++++++++++
 scrapy/utils/signal.py     |  6 ++++--
 tests/test_utils_signal.py | 26 +++++++++++++++++++++++---
 3 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index bbd5ebe52..a2c24e5fb 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -140,3 +140,20 @@ def deferred_from_coro(o):
             # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
             return defer.Deferred.fromFuture(asyncio.ensure_future(o))
     return o
+
+
+def maybeDeferred_coro(f, *args, **kw):
+    """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """
+    try:
+        result = f(*args, **kw)
+    except:  # noqa: E722
+        return defer.fail(failure.Failure(captureVars=defer.Deferred.debug))
+
+    if isinstance(result, defer.Deferred):
+        return result
+    elif _isfuture(result) or inspect.isawaitable(result):
+        return deferred_from_coro(result)
+    elif isinstance(result, failure.Failure):
+        return defer.fail(result)
+    else:
+        return defer.succeed(result)
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index de00bac49..60c561da6 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -2,12 +2,14 @@
 
 import logging
 
-from twisted.internet.defer import maybeDeferred, DeferredList, Deferred
+from twisted.internet.defer import DeferredList, Deferred
 from twisted.python.failure import Failure
 
 from pydispatch.dispatcher import Any, Anonymous, liveReceivers, \
     getAllReceivers, disconnect
 from pydispatch.robustapply import robustApply
+
+from scrapy.utils.defer import maybeDeferred_coro
 from scrapy.utils.log import failure_to_exc_info
 
 logger = logging.getLogger(__name__)
@@ -61,7 +63,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
     spider = named.get('spider', None)
     dfds = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
-        d = maybeDeferred(robustApply, receiver, signal=signal, sender=sender,
+        d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender,
                 *arguments, **named)
         d.addErrback(logerror, receiver)
         d.addBoth(lambda result: (receiver, result))
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index 16b7c5c68..e5f6f0ed4 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -1,3 +1,6 @@
+import asyncio
+
+from pytest import mark
 from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.python.failure import Failure
@@ -5,6 +8,7 @@ from twisted.internet import defer, reactor
 from pydispatch import dispatcher
 
 from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
+from scrapy.utils.test import get_from_asyncio_queue
 
 
 class SendCatchLogTest(unittest.TestCase):
@@ -54,7 +58,7 @@ class SendCatchLogDeferredTest(SendCatchLogTest):
         return send_catch_log_deferred(signal, *a, **kw)
 
 
-class SendCatchLogDeferredTest2(SendCatchLogTest):
+class SendCatchLogDeferredTest2(SendCatchLogDeferredTest):
 
     def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)
@@ -63,8 +67,24 @@ class SendCatchLogDeferredTest2(SendCatchLogTest):
         reactor.callLater(0, d.callback, "OK")
         return d
 
-    def _get_result(self, signal, *a, **kw):
-        return send_catch_log_deferred(signal, *a, **kw)
+
+class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
+
+    async def ok_handler(self, arg, handlers_called):
+        handlers_called.add(self.ok_handler)
+        assert arg == 'test'
+        await defer.succeed(42)
+        return "OK"
+
+
+@mark.only_asyncio()
+class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
+
+    async def ok_handler(self, arg, handlers_called):
+        handlers_called.add(self.ok_handler)
+        assert arg == 'test'
+        await asyncio.sleep(0.2)
+        return await get_from_asyncio_queue("OK")
 
 
 class SendCatchLogTest2(unittest.TestCase):

From a91a13b4434ae60be6fded94e4ed08ba322b4f69 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 12 Nov 2019 23:09:00 +0500
Subject: [PATCH 2549/2974] Support for async def callbacks.

---
 scrapy/utils/spider.py |  5 +++--
 tests/spiders.py       | 23 +++++++++++++++++++++++
 tests/test_crawl.py    | 20 +++++++++++++++++++-
 3 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 4061d1ea3..72775df5c 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -2,14 +2,15 @@ import logging
 import inspect
 
 from scrapy.spiders import Spider
-from scrapy.utils.misc import  arg_to_iter
+from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.misc import arg_to_iter
 
 
 logger = logging.getLogger(__name__)
 
 
 def iterate_spider_output(result):
-    return arg_to_iter(result)
+    return arg_to_iter(deferred_from_coro(result))
 
 
 def iter_spider_classes(module):
diff --git a/tests/spiders.py b/tests/spiders.py
index 39c8da0b6..e4f2d5474 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -1,14 +1,18 @@
 """
 Some spiders used for testing and benchmarking
 """
+import asyncio
 import time
 from urllib.parse import urlencode
 
+from twisted.internet import defer
+
 from scrapy.http import Request
 from scrapy.item import Item
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
 from scrapy.spiders.crawl import CrawlSpider, Rule
+from scrapy.utils.test import get_from_asyncio_queue
 
 
 class MockServerSpider(Spider):
@@ -83,6 +87,25 @@ class SimpleSpider(MetaSpider):
         self.logger.info("Got response %d" % response.status)
 
 
+class AsyncDefSpider(SimpleSpider):
+
+    name = 'asyncdef'
+
+    async def parse(self, response):
+        await defer.succeed(42)
+        self.logger.info("Got response %d" % response.status)
+
+
+class AsyncDefAsyncioSpider(SimpleSpider):
+
+    name = 'asyncdef_asyncio'
+
+    async def parse(self, response):
+        await asyncio.sleep(0.2)
+        status = await get_from_asyncio_queue(response.status)
+        self.logger.info("Got response %d" % status)
+
+
 class ItemSpider(FollowAllSpider):
 
     name = 'item'
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index f433fcea6..99b887ff6 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -1,6 +1,7 @@
 import json
 import logging
 
+from pytest import mark
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
@@ -10,7 +11,8 @@ from scrapy.http import Request
 from scrapy.utils.python import to_unicode
 from tests.mockserver import MockServer
 from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
-                           SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback)
+                           SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback,
+                           AsyncDefSpider, AsyncDefAsyncioSpider)
 
 
 class CrawlTestCase(TestCase):
@@ -308,3 +310,19 @@ with multiples lines
         self.assertIn("[callback] status 201", str(log))
         self.assertIn("[errback] status 404", str(log))
         self.assertIn("[errback] status 500", str(log))
+
+    @defer.inlineCallbacks
+    def test_async_def_parse(self):
+        self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        with LogCapture() as log:
+            yield self.runner.join()
+        self.assertIn("Got response 200", str(log))
+
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncio_parse(self):
+        runner = CrawlerRunner({"ASYNCIO_REACTOR": True})
+        runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        with LogCapture() as log:
+            yield runner.join()
+        self.assertIn("Got response 200", str(log))

From 6ce1ad31071326d22387f8444abefd6f3e18ed86 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 10 Jan 2020 04:20:37 -0300
Subject: [PATCH 2550/2974] [test] Spider middleware: catch exceptions right
 after the spider callback

---
 tests/test_spidermiddleware_output_chain.py | 50 +++++++++++++++++++--
 1 file changed, 46 insertions(+), 4 deletions(-)

diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 739cf1c2d..b19a74609 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -1,10 +1,10 @@
-
 from testfixtures import LogCapture
-from twisted.trial.unittest import TestCase
 from twisted.internet import defer
+from twisted.trial.unittest import TestCase
 
-from scrapy import Spider, Request
+from scrapy import Request, Spider
 from scrapy.utils.test import get_crawler
+
 from tests.mockserver import MockServer
 
 
@@ -74,7 +74,7 @@ class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback
     name = 'ProcessSpiderInputSpiderWithErrback'
 
     def start_requests(self):
-        yield Request(url=self.mockserver.url('/status?n=200'), callback=self.parse, errback=self.errback)
+        yield Request(self.mockserver.url('/status?n=200'), self.parse, errback=self.errback)
 
     def errback(self, failure):
         self.logger.info('Got a Failure on the Request errback')
@@ -100,6 +100,17 @@ class GeneratorCallbackSpider(Spider):
         raise ImportError()
 
 
+# ================================================================================
+# (2.1) exceptions from a spider callback (generator, middleware right after callback)
+class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider):
+    name = 'GeneratorCallbackSpiderMiddlewareRightAfterSpider'
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            __name__ + '.LogExceptionMiddleware': 100000,
+        },
+    }
+
+
 # ================================================================================
 # (3) exceptions from a spider callback (not a generator)
 class NotGeneratorCallbackSpider(Spider):
@@ -117,6 +128,17 @@ class NotGeneratorCallbackSpider(Spider):
         return [{'test': 1}, {'test': 1/0}]
 
 
+# ================================================================================
+# (3.1) exceptions from a spider callback (not a generator, middleware right after callback)
+class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackSpider):
+    name = 'NotGeneratorCallbackSpiderMiddlewareRightAfterSpider'
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            __name__ + '.LogExceptionMiddleware': 100000,
+        },
+    }
+
+
 # ================================================================================
 # (4) exceptions from a middleware process_spider_output method (generator)
 class GeneratorOutputChainSpider(Spider):
@@ -320,6 +342,16 @@ class TestSpiderMiddleware(TestCase):
         self.assertIn("Middleware: ImportError exception caught", str(log2))
         self.assertIn("'item_scraped_count': 2", str(log2))
 
+    @defer.inlineCallbacks
+    def test_generator_callback_right_after_callback(self):
+        """
+        (2.1) Special case of (2): Exceptions should be caught
+        even if the middleware is placed right after the spider
+        """
+        log21 = yield self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider)
+        self.assertIn("Middleware: ImportError exception caught", str(log21))
+        self.assertIn("'item_scraped_count': 2", str(log21))
+
     @defer.inlineCallbacks
     def test_not_a_generator_callback(self):
         """
@@ -330,6 +362,16 @@ class TestSpiderMiddleware(TestCase):
         self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3))
         self.assertNotIn("item_scraped_count", str(log3))
 
+    @defer.inlineCallbacks
+    def test_not_a_generator_callback_right_after_callback(self):
+        """
+        (3.1) Special case of (3): Exceptions should be caught
+        even if the middleware is placed right after the spider
+        """
+        log31 = yield self.crawl_log(NotGeneratorCallbackSpiderMiddlewareRightAfterSpider)
+        self.assertIn("Middleware: ZeroDivisionError exception caught", str(log31))
+        self.assertNotIn("item_scraped_count", str(log31))
+
     @defer.inlineCallbacks
     def test_generator_output_chain(self):
         """

From c088c04f449c3383b7867c25b387f13949b1d6c0 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 10 Jan 2020 04:20:55 -0300
Subject: [PATCH 2551/2974] Spider middleware: catch exceptions right after the
 spider callback

---
 scrapy/core/spidermw.py | 71 +++++++++++++++++++++++++----------------
 scrapy/utils/python.py  |  1 +
 2 files changed, 44 insertions(+), 28 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 097a374bf..180a0b1fe 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,13 +3,14 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
-from itertools import chain, islice
+from itertools import islice
 
 from twisted.python.failure import Failure
+
 from scrapy.exceptions import _InvalidOutput
 from scrapy.middleware import MiddlewareManager
-from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.conf import build_component_list
+from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.python import MutableChain
 
 
@@ -17,6 +18,13 @@ def _isiterable(possible_iterator):
     return hasattr(possible_iterator, '__iter__')
 
 
+def _fname(f):
+    return "%s.%s".format(
+        f.__self__.__class__.__name__,
+        f.__func__.__name__
+    )
+
+
 class SpiderMiddlewareManager(MiddlewareManager):
 
     component_name = 'spider middleware'
@@ -31,27 +39,36 @@ class SpiderMiddlewareManager(MiddlewareManager):
             self.methods['process_spider_input'].append(mw.process_spider_input)
         if hasattr(mw, 'process_start_requests'):
             self.methods['process_start_requests'].appendleft(mw.process_start_requests)
-        self.methods['process_spider_output'].appendleft(getattr(mw, 'process_spider_output', None))
-        self.methods['process_spider_exception'].appendleft(getattr(mw, 'process_spider_exception', None))
+        process_spider_output = getattr(mw, 'process_spider_output', None)
+        self.methods['process_spider_output'].appendleft(process_spider_output)
+        process_spider_exception = getattr(mw, 'process_spider_exception', None)
+        self.methods['process_spider_exception'].appendleft(process_spider_exception)
 
     def scrape_response(self, scrape_func, response, request, spider):
-        fname = lambda f: '%s.%s' % (
-                f.__self__.__class__.__name__,
-                f.__func__.__name__)
 
         def process_spider_input(response):
             for method in self.methods['process_spider_input']:
                 try:
                     result = method(response=response, spider=spider)
                     if result is not None:
-                        raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}'
-                                             .format(fname(method), type(result)))
+                        msg = "Middleware {} must return None or raise an exception, got {}"
+                        raise _InvalidOutput(msg.format(_fname(method), type(result)))
                 except _InvalidOutput:
                     raise
                 except Exception:
                     return scrape_func(Failure(), request, spider)
             return scrape_func(response, request, spider)
 
+        def _evaluate_iterable(iterable, method_index, recover_to):
+            try:
+                for r in iterable:
+                    yield r
+            except Exception as ex:
+                exception_result = process_spider_exception(Failure(ex), method_index)
+                if isinstance(exception_result, Failure):
+                    raise
+                recover_to.extend(exception_result)
+
         def process_spider_exception(_failure, start_index=0):
             exception = _failure.value
             # don't handle _InvalidOutput exception
@@ -69,8 +86,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 elif result is None:
                     continue
                 else:
-                    raise _InvalidOutput('Middleware {} must return None or an iterable, got {}'
-                                         .format(fname(method), type(result)))
+                    msg = "Middleware {} must return None or an iterable, got {}"
+                    raise _InvalidOutput(msg.format(_fname(method), type(result)))
             return _failure
 
         def process_spider_output(result, start_index=0):
@@ -78,38 +95,36 @@ class SpiderMiddlewareManager(MiddlewareManager):
             # chain, they went through it already from the process_spider_exception method
             recovered = MutableChain()
 
-            def evaluate_iterable(iterable, index):
-                try:
-                    for r in iterable:
-                        yield r
-                except Exception as ex:
-                    exception_result = process_spider_exception(Failure(ex), index+1)
-                    if isinstance(exception_result, Failure):
-                        raise
-                    recovered.extend(exception_result)
-
             method_list = islice(self.methods['process_spider_output'], start_index, None)
             for method_index, method in enumerate(method_list, start=start_index):
                 if method is None:
                     continue
-                # the following might fail directly if the output value is not a generator
                 try:
+                    # might fail directly if the output value is not a generator
                     result = method(response=response, result=result, spider=spider)
                 except Exception as ex:
                     exception_result = process_spider_exception(Failure(ex), method_index+1)
                     if isinstance(exception_result, Failure):
                         raise
                     return exception_result
-                if _isiterable(result):
-                    result = evaluate_iterable(result, method_index)
                 else:
-                    raise _InvalidOutput('Middleware {} must return an iterable, got {}'
-                                         .format(fname(method), type(result)))
+                    if _isiterable(result):
+                        result = _evaluate_iterable(result, method_index+1, recovered)
+                    else:
+                        msg = "Middleware {} must return an iterable, got {}"
+                        raise _InvalidOutput(msg.format(_fname(method), type(result)))
 
-            return chain(result, recovered)
+            return MutableChain(result, recovered)
+
+        def process_callback_output(result):
+            if isinstance(result, Failure):
+                return process_spider_exception(result)
+            recovered = MutableChain()
+            result = _evaluate_iterable(result, 0, recovered)
+            return MutableChain(process_spider_output(result), recovered)
 
         dfd = mustbe_deferred(process_spider_input, response)
-        dfd.addCallbacks(callback=process_spider_output, errback=process_spider_exception)
+        dfd.addCallbacks(callback=process_callback_output, errback=process_callback_output)
         return dfd
 
     def process_start_requests(self, start_requests, spider):
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 8d829c5a5..875650f3e 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -375,6 +375,7 @@ class MutableChain(object):
     """
     Thin wrapper around itertools.chain, allowing to add iterables "in-place"
     """
+
     def __init__(self, *args):
         self.data = chain(*args)
 

From d6e928f47209396d0f6c0b155eb07115c4855c93 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 10 Jan 2020 04:40:03 -0300
Subject: [PATCH 2552/2974] Remove object as base class for MutableChain

Plus some minor styling adjustments
---
 scrapy/utils/python.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 875650f3e..e5582cc18 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -1,15 +1,15 @@
 """
 This module contains essential stuff that should've come with Python itself ;)
 """
+import errno
 import gc
+import inspect
 import os
 import re
-import inspect
+import sys
 import weakref
-import errno
 from functools import partial, wraps
 from itertools import chain
-import sys
 
 from scrapy.utils.decorators import deprecated
 
@@ -371,7 +371,7 @@ else:
         gc.collect()
 
 
-class MutableChain(object):
+class MutableChain:
     """
     Thin wrapper around itertools.chain, allowing to add iterables "in-place"
     """

From 9770ca35fb494503298270902569ed897a365d32 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 10 Jan 2020 18:45:39 -0300
Subject: [PATCH 2553/2974] Spider middleware: simplify deferred errback
 handling

---
 scrapy/core/spidermw.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 180a0b1fe..ed02b306b 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -117,14 +117,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
             return MutableChain(result, recovered)
 
         def process_callback_output(result):
-            if isinstance(result, Failure):
-                return process_spider_exception(result)
             recovered = MutableChain()
             result = _evaluate_iterable(result, 0, recovered)
             return MutableChain(process_spider_output(result), recovered)
 
         dfd = mustbe_deferred(process_spider_input, response)
-        dfd.addCallbacks(callback=process_callback_output, errback=process_callback_output)
+        dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
         return dfd
 
     def process_start_requests(self, start_requests, spider):

From 03241aa4a66f9b0e8be4dc104807011f546840e8 Mon Sep 17 00:00:00 2001
From: abhishekh2001 <53903855+abhishekh2001@users.noreply.github.com>
Date: Wed, 15 Jan 2020 08:54:25 +0400
Subject: [PATCH 2554/2974] Fixed artwork/README formatting

---
 artwork/README.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/artwork/README.rst b/artwork/README.rst
index 92f6ecb7e..8a1028cde 100644
--- a/artwork/README.rst
+++ b/artwork/README.rst
@@ -1,5 +1,4 @@
-:orphan:
-
+==============
 Scrapy artwork
 ==============
 

From 735c0ceb7890dbea607dd6e4c0a14a4ce2b0afd2 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 9 Sep 2019 16:20:58 -0300
Subject: [PATCH 2555/2974] Custom name resolver implementing
 twisted.internet.interfaces.IHostnameResolver

---
 pytest.ini         |  1 +
 scrapy/crawler.py  | 38 ++++++++++++++++++----------------
 scrapy/resolver.py | 51 ++++++++++++++++++++++++++--------------------
 3 files changed, 50 insertions(+), 40 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index c3f3292bb..a0e89f0a9 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -158,6 +158,7 @@ flake8-ignore =
     scrapy/mail.py E402 E128 E501 E502
     scrapy/middleware.py E128 E501
     scrapy/pqueues.py E501
+    scrapy/resolver.py E501
     scrapy/responsetypes.py E128 E501 E305
     scrapy/robotstxt.py E501
     scrapy/shell.py E501
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index f87e67d93..1350ea84f 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -4,34 +4,36 @@ import signal
 import warnings
 
 from twisted.internet import defer
-from zope.interface.verify import verifyClass, DoesNotImplement
+from zope.interface.verify import DoesNotImplement, verifyClass
 
-from scrapy import Spider
+from scrapy import signals, Spider
 from scrapy.core.engine import ExecutionEngine
-from scrapy.resolver import CachingThreadedResolver
-from scrapy.interfaces import ISpiderLoader
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extension import ExtensionManager
+from scrapy.interfaces import ISpiderLoader
+from scrapy.resolver import CachingHostnameResolver
 from scrapy.settings import overridden_settings, Settings
 from scrapy.signalmanager import SignalManager
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installed
-from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
-from scrapy.utils.misc import load_object
 from scrapy.utils.log import (
-    LogCounterHandler, configure_logging, log_scrapy_info,
-    get_scrapy_root_handler, install_scrapy_root_handler)
-from scrapy import signals
+    configure_logging,
+    get_scrapy_root_handler,
+    install_scrapy_root_handler,
+    log_scrapy_info,
+    LogCounterHandler,
+)
+from scrapy.utils.misc import load_object
+from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 
 
 logger = logging.getLogger(__name__)
 
 
-class Crawler(object):
+class Crawler:
 
     def __init__(self, spidercls, settings=None):
         if isinstance(spidercls, Spider):
-            raise ValueError(
-                'The spidercls argument must be a class, not an object')
+            raise ValueError('The spidercls argument must be a class, not an object')
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
@@ -110,7 +112,7 @@ class Crawler(object):
             yield defer.maybeDeferred(self.engine.stop)
 
 
-class CrawlerRunner(object):
+class CrawlerRunner:
     """
     This is a convenient helper class that keeps track of, manages and runs
     crawlers inside an already setup :mod:`~twisted.internet.reactor`.
@@ -303,7 +305,7 @@ class CrawlerProcess(CrawlerRunner):
                 return
             d.addBoth(self._stop_reactor)
 
-        reactor.installResolver(self._get_dns_resolver())
+        reactor.installNameResolver(self._get_dns_resolver())
         tp = reactor.getThreadPool()
         tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
@@ -315,10 +317,10 @@ class CrawlerProcess(CrawlerRunner):
             cache_size = self.settings.getint('DNSCACHE_SIZE')
         else:
             cache_size = 0
-        return CachingThreadedResolver(
-            reactor=reactor,
+        return CachingHostnameResolver(
+            resolver=reactor.nameResolver,
             cache_size=cache_size,
-            timeout=self.settings.getfloat('DNS_TIMEOUT')
+            timeout=self.settings.getfloat('DNS_TIMEOUT'),
         )
 
     def _graceful_stop_reactor(self):
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 4df949015..03964f269 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -1,32 +1,39 @@
-from twisted.internet import defer
-from twisted.internet.base import ThreadedResolver
+from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver
+from zope.interface.declarations import implementer, provider
 
 from scrapy.utils.datatypes import LocalCache
 
-# TODO: cache misses
 
+# TODO: cache misses
 dnscache = LocalCache(10000)
 
 
-class CachingThreadedResolver(ThreadedResolver):
-    def __init__(self, reactor, cache_size, timeout):
-        super(CachingThreadedResolver, self).__init__(reactor)
-        dnscache.limit = cache_size
+@implementer(IHostnameResolver)
+class CachingHostnameResolver(object):
+
+    def __init__(self, resolver, cache_size, timeout):
+        self.resolver = resolver
         self.timeout = timeout
+        dnscache.limit = cache_size
 
-    def getHostByName(self, name, timeout=None):
-        if name in dnscache:
-            return defer.succeed(dnscache[name])
-        # in Twisted<=16.6, getHostByName() is always called with
-        # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple),
-        # so the input argument above is simply overridden
-        # to enforce Scrapy's DNS_TIMEOUT setting's value
-        timeout = (self.timeout,)
-        d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
-        if dnscache.limit:
-            d.addCallback(self._cache_result, name)
-        return d
+    def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
+                        addressTypes=None, transportSemantics='TCP'):
 
-    def _cache_result(self, result, name):
-        dnscache[name] = result
-        return result
+        @provider(IResolutionReceiver)
+        class CachingResolutionReceiver(resolutionReceiver):
+            def resolutionBegan(self, resolution):
+                super(CachingResolutionReceiver, self).resolutionBegan(resolution)
+                self.resolution = resolution
+
+            def resolutionComplete(self):
+                super(CachingResolutionReceiver, self).resolutionComplete()
+                dnscache[hostName] = self.resolution
+
+        try:
+            result = dnscache[hostName]
+        except KeyError:
+            result = self.resolver.resolveHostName(
+                CachingResolutionReceiver(), hostName, portNumber, addressTypes, transportSemantics
+            )
+        finally:
+            return result

From f1c184631e8cefc191dc0077083138c241bfd6a8 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 11 Dec 2019 17:44:05 -0300
Subject: [PATCH 2556/2974] Name resolver: timeout

---
 scrapy/resolver.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 03964f269..8792ed6ab 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -1,3 +1,4 @@
+from twisted.internet import reactor
 from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver
 from zope.interface.declarations import implementer, provider
 
@@ -21,9 +22,14 @@ class CachingHostnameResolver(object):
 
         @provider(IResolutionReceiver)
         class CachingResolutionReceiver(resolutionReceiver):
+
+            def __init__(self, timeout):
+                self.timeout = timeout
+
             def resolutionBegan(self, resolution):
                 super(CachingResolutionReceiver, self).resolutionBegan(resolution)
                 self.resolution = resolution
+                # reactor.callLater(self.timeout, resolution.cancel)
 
             def resolutionComplete(self):
                 super(CachingResolutionReceiver, self).resolutionComplete()
@@ -33,7 +39,11 @@ class CachingHostnameResolver(object):
             result = dnscache[hostName]
         except KeyError:
             result = self.resolver.resolveHostName(
-                CachingResolutionReceiver(), hostName, portNumber, addressTypes, transportSemantics
+                CachingResolutionReceiver(self.timeout),
+                hostName,
+                portNumber,
+                addressTypes,
+                transportSemantics
             )
         finally:
             return result

From 55babf9acd6cd357f4211a86af01a1f2abe2f0cb Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 15 Jan 2020 12:25:20 -0300
Subject: [PATCH 2557/2974] Cache resolution only if the DNS request was
 successful

---
 scrapy/resolver.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 8792ed6ab..0ba22ed0d 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -10,7 +10,7 @@ dnscache = LocalCache(10000)
 
 
 @implementer(IHostnameResolver)
-class CachingHostnameResolver(object):
+class CachingHostnameResolver:
 
     def __init__(self, resolver, cache_size, timeout):
         self.resolver = resolver
@@ -25,15 +25,21 @@ class CachingHostnameResolver(object):
 
             def __init__(self, timeout):
                 self.timeout = timeout
+                self.resolved = False
 
             def resolutionBegan(self, resolution):
                 super(CachingResolutionReceiver, self).resolutionBegan(resolution)
                 self.resolution = resolution
                 # reactor.callLater(self.timeout, resolution.cancel)
 
+            def addressResolved(self, address):
+                super(CachingResolutionReceiver, self).addressResolved(address)
+                self.resolved = True
+
             def resolutionComplete(self):
                 super(CachingResolutionReceiver, self).resolutionComplete()
-                dnscache[hostName] = self.resolution
+                if self.resolved:
+                    dnscache[hostName] = self.resolution
 
         try:
             result = dnscache[hostName]

From 8c3de288fa2564473f49198b40efdeaa2428f1ca Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 15 Jan 2020 12:31:36 -0300
Subject: [PATCH 2558/2974] Remove non-working DNS timeout code

---
 scrapy/crawler.py  |  1 -
 scrapy/resolver.py | 12 +++---------
 2 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 1350ea84f..61851acc3 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -320,7 +320,6 @@ class CrawlerProcess(CrawlerRunner):
         return CachingHostnameResolver(
             resolver=reactor.nameResolver,
             cache_size=cache_size,
-            timeout=self.settings.getfloat('DNS_TIMEOUT'),
         )
 
     def _graceful_stop_reactor(self):
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 0ba22ed0d..ddbae61a9 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -1,4 +1,3 @@
-from twisted.internet import reactor
 from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver
 from zope.interface.declarations import implementer, provider
 
@@ -12,9 +11,8 @@ dnscache = LocalCache(10000)
 @implementer(IHostnameResolver)
 class CachingHostnameResolver:
 
-    def __init__(self, resolver, cache_size, timeout):
+    def __init__(self, resolver, cache_size):
         self.resolver = resolver
-        self.timeout = timeout
         dnscache.limit = cache_size
 
     def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
@@ -23,14 +21,10 @@ class CachingHostnameResolver:
         @provider(IResolutionReceiver)
         class CachingResolutionReceiver(resolutionReceiver):
 
-            def __init__(self, timeout):
-                self.timeout = timeout
-                self.resolved = False
-
             def resolutionBegan(self, resolution):
                 super(CachingResolutionReceiver, self).resolutionBegan(resolution)
                 self.resolution = resolution
-                # reactor.callLater(self.timeout, resolution.cancel)
+                self.resolved = False
 
             def addressResolved(self, address):
                 super(CachingResolutionReceiver, self).addressResolved(address)
@@ -45,7 +39,7 @@ class CachingHostnameResolver:
             result = dnscache[hostName]
         except KeyError:
             result = self.resolver.resolveHostName(
-                CachingResolutionReceiver(self.timeout),
+                CachingResolutionReceiver(),
                 hostName,
                 portNumber,
                 addressTypes,

From e69cf415c8326349c871fb23faba2ed822aa08ee Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 16 Jan 2020 03:58:07 -0300
Subject: [PATCH 2559/2974] Ability to choose name resolver

---
 scrapy/crawler.py                   | 14 ++------
 scrapy/resolver.py                  | 54 ++++++++++++++++++++++++++++-
 scrapy/settings/default_settings.py |  1 +
 3 files changed, 56 insertions(+), 13 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 61851acc3..c5351c08f 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -305,23 +305,13 @@ class CrawlerProcess(CrawlerRunner):
                 return
             d.addBoth(self._stop_reactor)
 
-        reactor.installNameResolver(self._get_dns_resolver())
+        resolver_class = load_object(self.settings["DNS_RESOLVER"])
+        resolver_class.install(reactor, self.settings)
         tp = reactor.getThreadPool()
         tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
         reactor.run(installSignalHandlers=False)  # blocking call
 
-    def _get_dns_resolver(self):
-        from twisted.internet import reactor
-        if self.settings.getbool('DNSCACHE_ENABLED'):
-            cache_size = self.settings.getint('DNSCACHE_SIZE')
-        else:
-            cache_size = 0
-        return CachingHostnameResolver(
-            resolver=reactor.nameResolver,
-            cache_size=cache_size,
-        )
-
     def _graceful_stop_reactor(self):
         d = self.stop()
         d.addBoth(self._stop_reactor)
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index ddbae61a9..2bef9f1b8 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -1,4 +1,6 @@
-from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver
+from twisted.internet import defer
+from twisted.internet.base import ThreadedResolver
+from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple
 from zope.interface.declarations import implementer, provider
 
 from scrapy.utils.datatypes import LocalCache
@@ -8,8 +10,58 @@ from scrapy.utils.datatypes import LocalCache
 dnscache = LocalCache(10000)
 
 
+@implementer(IResolverSimple)
+class CachingThreadedResolver(ThreadedResolver):
+    """
+    Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests
+    """
+
+    @classmethod
+    def install(cls, reactor, settings):
+        if settings.getbool('DNSCACHE_ENABLED'):
+            cache_size = settings.getint('DNSCACHE_SIZE')
+        else:
+            cache_size = 0
+        resolver = cls(reactor, cache_size, settings.getfloat('DNS_TIMEOUT'))
+        reactor.installResolver(resolver)
+
+    def __init__(self, reactor, cache_size, timeout):
+        super(CachingThreadedResolver, self).__init__(reactor)
+        dnscache.limit = cache_size
+        self.timeout = timeout
+
+    def getHostByName(self, name, timeout=None):
+        if name in dnscache:
+            return defer.succeed(dnscache[name])
+        # in Twisted<=16.6, getHostByName() is always called with
+        # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple),
+        # so the input argument above is simply overridden
+        # to enforce Scrapy's DNS_TIMEOUT setting's value
+        timeout = (self.timeout,)
+        d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
+        if dnscache.limit:
+            d.addCallback(self._cache_result, name)
+        return d
+
+    def _cache_result(self, result, name):
+        dnscache[name] = result
+        return result
+
+
 @implementer(IHostnameResolver)
 class CachingHostnameResolver:
+    """
+    Experimental caching resolver, supporting IPv4 and IPv6
+    """
+
+    @classmethod
+    def install(cls, reactor, settings):
+        if settings.getbool('DNSCACHE_ENABLED'):
+            cache_size = settings.getint('DNSCACHE_SIZE')
+        else:
+            cache_size = 0
+        resolver = cls(reactor.nameResolver, cache_size)
+        reactor.installNameResolver(resolver)
 
     def __init__(self, resolver, cache_size):
         self.resolver = resolver
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index d03fd37b0..46ed3be96 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -60,6 +60,7 @@ DEPTH_PRIORITY = 0
 
 DNSCACHE_ENABLED = True
 DNSCACHE_SIZE = 10000
+DNS_RESOLVER = 'scrapy.resolver.CachingThreadedResolver'
 DNS_TIMEOUT = 60
 
 DOWNLOAD_DELAY = 0

From 0f155b059a43b2cc48149a26c6584910038a23f1 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 16 Jan 2020 04:27:13 -0300
Subject: [PATCH 2560/2974] Make Flake8 happy (remove unused import)

---
 scrapy/crawler.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index c5351c08f..5658e264b 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -11,7 +11,6 @@ from scrapy.core.engine import ExecutionEngine
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extension import ExtensionManager
 from scrapy.interfaces import ISpiderLoader
-from scrapy.resolver import CachingHostnameResolver
 from scrapy.settings import overridden_settings, Settings
 from scrapy.signalmanager import SignalManager
 from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installed

From f45b4c7f8d191c6855c12f3497ece6b6121289df Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 16 Jan 2020 10:09:34 -0300
Subject: [PATCH 2561/2974] from_crawler support for name resolvers

---
 scrapy/crawler.py  |  2 +-
 scrapy/resolver.py | 48 ++++++++++++++++++++++++++++------------------
 2 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 5658e264b..4531c3aac 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -305,7 +305,7 @@ class CrawlerProcess(CrawlerRunner):
             d.addBoth(self._stop_reactor)
 
         resolver_class = load_object(self.settings["DNS_RESOLVER"])
-        resolver_class.install(reactor, self.settings)
+        resolver_class.install_on_reactor(reactor, crawler=self)
         tp = reactor.getThreadPool()
         tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 2bef9f1b8..792563e79 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -4,6 +4,7 @@ from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver,
 from zope.interface.declarations import implementer, provider
 
 from scrapy.utils.datatypes import LocalCache
+from scrapy.utils.misc import create_instance
 
 
 # TODO: cache misses
@@ -13,23 +14,27 @@ dnscache = LocalCache(10000)
 @implementer(IResolverSimple)
 class CachingThreadedResolver(ThreadedResolver):
     """
-    Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests
+    Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests.
     """
 
-    @classmethod
-    def install(cls, reactor, settings):
-        if settings.getbool('DNSCACHE_ENABLED'):
-            cache_size = settings.getint('DNSCACHE_SIZE')
-        else:
-            cache_size = 0
-        resolver = cls(reactor, cache_size, settings.getfloat('DNS_TIMEOUT'))
-        reactor.installResolver(resolver)
-
     def __init__(self, reactor, cache_size, timeout):
         super(CachingThreadedResolver, self).__init__(reactor)
         dnscache.limit = cache_size
         self.timeout = timeout
 
+    @classmethod
+    def from_crawler(cls, crawler, reactor):
+        if crawler.settings.getbool('DNSCACHE_ENABLED'):
+            cache_size = crawler.settings.getint('DNSCACHE_SIZE')
+        else:
+            cache_size = 0
+        return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT'))
+
+    @classmethod
+    def install_on_reactor(cls, reactor, crawler):
+        resolver = create_instance(cls, None, crawler, reactor=reactor)
+        reactor.installResolver(resolver)
+
     def getHostByName(self, name, timeout=None):
         if name in dnscache:
             return defer.succeed(dnscache[name])
@@ -51,21 +56,26 @@ class CachingThreadedResolver(ThreadedResolver):
 @implementer(IHostnameResolver)
 class CachingHostnameResolver:
     """
-    Experimental caching resolver, supporting IPv4 and IPv6
+    Experimental caching resolver. Resolves IPv4 and IPv6 addresses,
+    does not support setting a timeout value for DNS requests.
     """
 
+    def __init__(self, reactor, cache_size):
+        self.resolver = reactor.nameResolver
+        dnscache.limit = cache_size
+
     @classmethod
-    def install(cls, reactor, settings):
-        if settings.getbool('DNSCACHE_ENABLED'):
-            cache_size = settings.getint('DNSCACHE_SIZE')
+    def from_crawler(cls, crawler, reactor):
+        if crawler.settings.getbool('DNSCACHE_ENABLED'):
+            cache_size = crawler.settings.getint('DNSCACHE_SIZE')
         else:
             cache_size = 0
-        resolver = cls(reactor.nameResolver, cache_size)
-        reactor.installNameResolver(resolver)
+        return cls(reactor, cache_size)
 
-    def __init__(self, resolver, cache_size):
-        self.resolver = resolver
-        dnscache.limit = cache_size
+    @classmethod
+    def install_on_reactor(cls, reactor, crawler):
+        resolver = create_instance(cls, None, crawler, reactor=reactor)
+        reactor.installNameResolver(resolver)
 
     def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
                         addressTypes=None, transportSemantics='TCP'):

From 3cfa73b8b12dfe2dd1364af856d2ec46a486b48d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 16 Jan 2020 18:01:18 -0300
Subject: [PATCH 2562/2974] Name resolvers: install_on_reactor as instance
 method

---
 scrapy/crawler.py  |  5 +++--
 scrapy/resolver.py | 13 ++++---------
 2 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 4531c3aac..0cef75a6a 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -21,7 +21,7 @@ from scrapy.utils.log import (
     log_scrapy_info,
     LogCounterHandler,
 )
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 
 
@@ -305,7 +305,8 @@ class CrawlerProcess(CrawlerRunner):
             d.addBoth(self._stop_reactor)
 
         resolver_class = load_object(self.settings["DNS_RESOLVER"])
-        resolver_class.install_on_reactor(reactor, crawler=self)
+        resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
+        resolver.install_on_reactor(reactor)
         tp = reactor.getThreadPool()
         tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 792563e79..2b97603da 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -4,7 +4,6 @@ from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver,
 from zope.interface.declarations import implementer, provider
 
 from scrapy.utils.datatypes import LocalCache
-from scrapy.utils.misc import create_instance
 
 
 # TODO: cache misses
@@ -30,10 +29,8 @@ class CachingThreadedResolver(ThreadedResolver):
             cache_size = 0
         return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT'))
 
-    @classmethod
-    def install_on_reactor(cls, reactor, crawler):
-        resolver = create_instance(cls, None, crawler, reactor=reactor)
-        reactor.installResolver(resolver)
+    def install_on_reactor(self, reactor):
+        reactor.installResolver(self)
 
     def getHostByName(self, name, timeout=None):
         if name in dnscache:
@@ -72,10 +69,8 @@ class CachingHostnameResolver:
             cache_size = 0
         return cls(reactor, cache_size)
 
-    @classmethod
-    def install_on_reactor(cls, reactor, crawler):
-        resolver = create_instance(cls, None, crawler, reactor=reactor)
-        reactor.installNameResolver(resolver)
+    def install_on_reactor(self, reactor):
+        reactor.installNameResolver(self)
 
     def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
                         addressTypes=None, transportSemantics='TCP'):

From 1040f581ec1ba3bcf8f08edb824d59c0e4a93700 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 16 Jan 2020 20:14:52 -0300
Subject: [PATCH 2563/2974] Name resolvers: do not pass the reactor to the
 install method

---
 scrapy/crawler.py  |  2 +-
 scrapy/resolver.py | 14 ++++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 0cef75a6a..35c6b7716 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -306,7 +306,7 @@ class CrawlerProcess(CrawlerRunner):
 
         resolver_class = load_object(self.settings["DNS_RESOLVER"])
         resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
-        resolver.install_on_reactor(reactor)
+        resolver.install_on_reactor()
         tp = reactor.getThreadPool()
         tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 2b97603da..7c776f75e 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -19,6 +19,7 @@ class CachingThreadedResolver(ThreadedResolver):
     def __init__(self, reactor, cache_size, timeout):
         super(CachingThreadedResolver, self).__init__(reactor)
         dnscache.limit = cache_size
+        self.reactor = reactor
         self.timeout = timeout
 
     @classmethod
@@ -29,8 +30,8 @@ class CachingThreadedResolver(ThreadedResolver):
             cache_size = 0
         return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT'))
 
-    def install_on_reactor(self, reactor):
-        reactor.installResolver(self)
+    def install_on_reactor(self,):
+        self.reactor.installResolver(self)
 
     def getHostByName(self, name, timeout=None):
         if name in dnscache:
@@ -58,7 +59,8 @@ class CachingHostnameResolver:
     """
 
     def __init__(self, reactor, cache_size):
-        self.resolver = reactor.nameResolver
+        self.reactor = reactor
+        self.original_resolver = reactor.nameResolver
         dnscache.limit = cache_size
 
     @classmethod
@@ -69,8 +71,8 @@ class CachingHostnameResolver:
             cache_size = 0
         return cls(reactor, cache_size)
 
-    def install_on_reactor(self, reactor):
-        reactor.installNameResolver(self)
+    def install_on_reactor(self):
+        self.reactor.installNameResolver(self)
 
     def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
                         addressTypes=None, transportSemantics='TCP'):
@@ -95,7 +97,7 @@ class CachingHostnameResolver:
         try:
             result = dnscache[hostName]
         except KeyError:
-            result = self.resolver.resolveHostName(
+            result = self.original_resolver.resolveHostName(
                 CachingResolutionReceiver(),
                 hostName,
                 portNumber,

From 90e3bd8715701aeea9187837bc32925e3225288f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 16 Jan 2020 20:32:40 -0300
Subject: [PATCH 2564/2974] [test] Name resolvers

---
 tests/CrawlerProcess/alternative_name_resolver.py | 15 +++++++++++++++
 tests/CrawlerProcess/default_name_resolver.py     | 12 ++++++++++++
 tests/test_crawler.py                             | 12 ++++++++++++
 3 files changed, 39 insertions(+)
 create mode 100644 tests/CrawlerProcess/alternative_name_resolver.py
 create mode 100644 tests/CrawlerProcess/default_name_resolver.py

diff --git a/tests/CrawlerProcess/alternative_name_resolver.py b/tests/CrawlerProcess/alternative_name_resolver.py
new file mode 100644
index 000000000..2c466da04
--- /dev/null
+++ b/tests/CrawlerProcess/alternative_name_resolver.py
@@ -0,0 +1,15 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class IPv6Spider(scrapy.Spider):
+    name = "ipv6_spider"
+    start_urls = ["http://[::1]"]
+
+
+process = CrawlerProcess(settings={
+    "RETRY_ENABLED": False,
+    "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+})
+process.crawl(IPv6Spider)
+process.start()
diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.py
new file mode 100644
index 000000000..60d91b68b
--- /dev/null
+++ b/tests/CrawlerProcess/default_name_resolver.py
@@ -0,0 +1,12 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class IPv6Spider(scrapy.Spider):
+    name = "ipv6_spider"
+    start_urls = ["http://[::1]"]
+
+
+process = CrawlerProcess(settings={"RETRY_ENABLED": False})
+process.crawl(IPv6Spider)
+process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index fce60ca37..d85f2bc41 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -305,3 +305,15 @@ class CrawlerProcessSubprocess(unittest.TestCase):
         log = self.run_script('asyncio_enabled_reactor.py')
         self.assertIn('Spider closed (finished)', log)
         self.assertIn("DEBUG: Asyncio reactor is installed", log)
+
+    def test_default_name_resolver(self):
+        log = self.run_script('default_name_resolver.py')
+        self.assertIn('Spider closed (finished)', log)
+        self.assertIn("twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", log)
+        self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)
+
+    def test_alternative_name_resolver(self):
+        log = self.run_script('alternative_name_resolver.py')
+        self.assertIn('Spider closed (finished)', log)
+        self.assertIn("twisted.internet.error.ConnectionRefusedError: Connection was refused by other side: 111: Connection refused.", log)
+        self.assertIn("'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1,", log)

From d487498cff893d2c507ef5ed0957f101dc3ab4a9 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 16 Jan 2020 22:02:01 -0300
Subject: [PATCH 2565/2974] Update name resolvers tests

---
 tests/test_crawler.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index d85f2bc41..14f4f8418 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -306,14 +306,20 @@ class CrawlerProcessSubprocess(unittest.TestCase):
         self.assertIn('Spider closed (finished)', log)
         self.assertIn("DEBUG: Asyncio reactor is installed", log)
 
-    def test_default_name_resolver(self):
+    def test_ipv6_default_name_resolver(self):
         log = self.run_script('default_name_resolver.py')
         self.assertIn('Spider closed (finished)', log)
         self.assertIn("twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", log)
         self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)
 
-    def test_alternative_name_resolver(self):
+    def test_ipv6_alternative_name_resolver(self):
         log = self.run_script('alternative_name_resolver.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertIn("twisted.internet.error.ConnectionRefusedError: Connection was refused by other side: 111: Connection refused.", log)
-        self.assertIn("'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1,", log)
+        self.assertTrue(any(
+            "twisted.internet.error.ConnectionRefusedError" in log,
+            "twisted.internet.error.ConnectError" in log,
+        ))
+        self.assertTrue(any(
+            "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
+            "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
+        ))

From dee420a69cb5c8319a7df99a5765c5ca90a7063e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 16 Jan 2020 23:48:16 -0300
Subject: [PATCH 2566/2974] Fix name resolvers tests

---
 tests/test_crawler.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 14f4f8418..0ce0674de 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -315,11 +315,11 @@ class CrawlerProcessSubprocess(unittest.TestCase):
     def test_ipv6_alternative_name_resolver(self):
         log = self.run_script('alternative_name_resolver.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertTrue(any(
+        self.assertTrue(any([
             "twisted.internet.error.ConnectionRefusedError" in log,
             "twisted.internet.error.ConnectError" in log,
-        ))
-        self.assertTrue(any(
+        ]))
+        self.assertTrue(any([
             "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
             "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
-        ))
+        ]))

From 41f7ebf3add25b2251082b206b062c00d1daba58 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 17 Jan 2020 12:40:49 -0300
Subject: [PATCH 2567/2974] CachingThreadedResolver: No need to store the
 reactor as an instance attribute

It's already done in the parent class
---
 scrapy/resolver.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 7c776f75e..7751f3796 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -19,7 +19,6 @@ class CachingThreadedResolver(ThreadedResolver):
     def __init__(self, reactor, cache_size, timeout):
         super(CachingThreadedResolver, self).__init__(reactor)
         dnscache.limit = cache_size
-        self.reactor = reactor
         self.timeout = timeout
 
     @classmethod

From 302d3f552b5a930a43fddac8eb8c5f769d8748bc Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 18 Jan 2020 01:41:57 -0300
Subject: [PATCH 2568/2974] [doc] DNS_RESOLVER setting

---
 docs/topics/settings.rst | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index c02f877fc..292eaea74 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -397,6 +397,19 @@ Default: ``10000``
 
 DNS in-memory cache size.
 
+.. setting:: DNS_RESOLVER
+
+DNS_RESOLVER
+------------
+
+Default: ``'scrapy.resolver.CachingThreadedResolver'``
+
+The class to be used to resolve DNS names. The default ``scrapy.resolver.CachingThreadedResolver``
+supports specifying a timeout for DNS requests via the :setting:`DNS_TIMEOUT` setting,
+but works only with IPv4 addresses. Scrapy provides an alternative resolver,
+``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not
+take the :setting:`DNS_TIMEOUT` setting into account.
+
 .. setting:: DNS_TIMEOUT
 
 DNS_TIMEOUT

From b471765d40a2a8a94f90af46ac6903e3786aec5f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 18 Jan 2020 01:52:29 -0300
Subject: [PATCH 2569/2974] [doc] FAQ entry about the IPv6 and the DNS_RESOLVER
 setting

---
 docs/faq.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/faq.rst b/docs/faq.rst
index aae2411e0..b789a8cdb 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -353,6 +353,13 @@ method for this purpose. For example::
                     for _ in range(item['multiply_by']):
                         yield deepcopy(item)
 
+Does Scrapy support IPv6 addresses?
+-----------------------------------
+
+Yes, by setting :setting:`DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
+Note that by doing so, you lose the ability to set a specific timeout for DNS requests
+(the value of the :setting:`DNS_TIMEOUT` setting is ignored).
+
 
 .. _user agents: https://en.wikipedia.org/wiki/User_agent
 .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)

From eaa8ed02d05d0cc5e7be1f773a22755676c7fe56 Mon Sep 17 00:00:00 2001
From: Juan Pablo Balarini <jpbalarini@gmail.com>
Date: Wed, 26 Dec 2018 13:11:23 -0300
Subject: [PATCH 2570/2974] Add ability to change max_active_size by settings

---
 docs/topics/settings.rst            | 11 +++++++++++
 scrapy/core/scraper.py              |  2 +-
 scrapy/settings/default_settings.py |  2 ++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index c02f877fc..d4480c879 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1262,6 +1262,17 @@ Type of priority queue used by the scheduler. Another available type is
 domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue``
 does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`.
 
+.. setting:: SCRAPER_SLOT_MAX_ACTIVE_SIZE
+
+SCRAPER_SLOT_MAX_ACTIVE_SIZE
+----------------------------
+Default: ``5000000``
+
+Soft limit (in bytes) for response data being processed.
+
+While the sum of the sizes of all responses being processed is above this value,
+Scrapy does not process new requests.
+
 .. setting:: SPIDER_CONTRACTS
 
 SPIDER_CONTRACTS
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 99114d3bb..facbd8b73 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -78,7 +78,7 @@ class Scraper(object):
     @defer.inlineCallbacks
     def open_spider(self, spider):
         """Open the given spider for scraping and allocate resources for it"""
-        self.slot = Slot()
+        self.slot = Slot(self.crawler.settings.getint('SCRAPER_SLOT_MAX_ACTIVE_SIZE'))
         yield self.itemproc.open_spider(spider)
 
     def close_spider(self, spider):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index d03fd37b0..ddd48d327 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -254,6 +254,8 @@ SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
 SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
 SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.ScrapyPriorityQueue'
 
+SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000
+
 SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
 SPIDER_LOADER_WARN_ONLY = False
 

From 0f2d871d88b6741aaf0348d1f11a1e020124099f Mon Sep 17 00:00:00 2001
From: JP Balarini <jpbalarini@gmail.com>
Date: Wed, 22 May 2019 16:26:27 -0300
Subject: [PATCH 2571/2974] Use PEP 515 style for SCRAPER_SLOT_MAX_ACTIVE_SIZE
 documentation

---
 docs/topics/settings.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index d4480c879..f4c3494f5 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1266,7 +1266,7 @@ does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`.
 
 SCRAPER_SLOT_MAX_ACTIVE_SIZE
 ----------------------------
-Default: ``5000000``
+Default: ``5_000_000``
 
 Soft limit (in bytes) for response data being processed.
 

From 8ea8f14827470f37c0e53d302aa65bcfa9604f3c Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Mon, 20 Jan 2020 18:19:36 +0200
Subject: [PATCH 2572/2974] Update scrapy/utils/ftp.py

Co-Authored-By: Mikhail Korobov <kmike84@gmail.com>
---
 scrapy/utils/ftp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index bf67b9976..b3e9ec2ed 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -17,7 +17,7 @@ def ftp_makedirs_cwd(ftp, path, first_call=True):
             ftp.cwd(path)
 
 def ftp_store_file( 
-        path, file, host ,port,
+        path, file, host, port,
         username, password, use_active_mode=False):
     """Opens a FTP connection with passed credentials,sets current directory
     to the directory extracted from given path, then uploads the file to server

From 06ab668ec7f880f9992dc669a374a6111cef5d04 Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Wed, 22 Jan 2020 03:48:07 +0200
Subject: [PATCH 2573/2974] Use kwargs-only parameters in `ftp_store_file`

---
 scrapy/extensions/feedexport.py | 6 +++---
 scrapy/pipelines/files.py       | 6 +++---
 scrapy/utils/ftp.py             | 2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 1ddc55f93..06b5a0dd9 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -175,9 +175,9 @@ class FTPFeedStorage(BlockingFeedStorage):
 
     def _store_in_thread(self, file):
         ftp_store_file(
-            self.path, file, self.host,
-            self.port, self.username, 
-            self.password, self.use_active_mode
+            path=self.path, file=file, host=self.host,
+            port=self.port, username=self.username, 
+            password=self.password, use_active_mode=self.use_active_mode
         )
 
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 5780f63bd..5383b05fe 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -269,9 +269,9 @@ class FTPFilesStore(object):
     def persist_file(self, path, buf, info, meta=None, headers=None):     
         path = '%s/%s' % (self.basedir, path)
         return threads.deferToThread(
-            ftp_store_file, path,buf,
-            self.host, self.port,self.username,
-            self.password, self.USE_ACTIVE_MODE
+            ftp_store_file, path=path, file=buf,
+            host=self.host, port=self.port, username=self.username,
+            password=self.password, use_active_mode=self.USE_ACTIVE_MODE
         )
             
     def stat_file(self, path, info):
diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index b3e9ec2ed..752e3c953 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -17,7 +17,7 @@ def ftp_makedirs_cwd(ftp, path, first_call=True):
             ftp.cwd(path)
 
 def ftp_store_file( 
-        path, file, host, port,
+        *, path, file, host, port,
         username, password, use_active_mode=False):
     """Opens a FTP connection with passed credentials,sets current directory
     to the directory extracted from given path, then uploads the file to server

From c75cf15b7a8293fd83f6e8b27abca2e50b0a04ed Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 22 Jan 2020 10:38:59 -0300
Subject: [PATCH 2574/2974] Update CSS selectors in tutorial

---
 docs/intro/tutorial.rst | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index c4d8d717b..c9d00eb74 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -616,19 +616,24 @@ instance; you still have to yield this Request.
 You can also pass a selector to ``response.follow`` instead of a string;
 this selector should extract necessary attributes::
 
-    href = response.css('li.next a::attr(href)')[0]
-    yield response.follow(href, callback=self.parse)
+    for href in response.css('ul.pager a::attr(href)'):
+        yield response.follow(href, callback=self.parse)
 
 For ``<a>`` elements there is a shortcut: ``response.follow`` uses their href
 attribute automatically. So the code can be shortened further::
 
-    a = response.css('li.next a')[0]
-    yield response.follow(a, callback=self.parse)
+    for a in response.css('ul.pager a'):
+        yield response.follow(a, callback=self.parse)
 
 To create multiple requests from an iterable, you can use
 :meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` instead::
 
-    yield from response.follow_all(response.css('a'), callback=self.parse)
+    anchors = response.css('ul.pager a')
+    yield from response.follow_all(anchors, callback=self.parse)
+
+or, shortening it further::
+
+    yield from response.follow_all(css='ul.pager a', callback=self.parse)
 
 
 More examples and patterns

From 7d5cebcf773d54d2f7b5ac0b90886847a4a70c26 Mon Sep 17 00:00:00 2001
From: Peter Vandenabeele <peter@vandenabeele.com>
Date: Thu, 23 Jan 2020 09:08:21 +0100
Subject: [PATCH 2575/2974] fix logical documentation error with PER_DOMAIN or
 PER_DOMAIN

---
 docs/faq.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index aae2411e0..169e1a479 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -140,7 +140,7 @@ setting the following settings::
 
 While pending requests are below the configured values of
 :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
-:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sent
+:setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent
 concurrently. As a result, the first few requests of a crawl rarely follow the
 desired order. Lowering those settings to ``1`` enforces the desired order, but
 it significantly slows down the crawl as a whole.

From 9899414300b4a6491ea17418c3403aed08e0faf6 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 23 Jan 2020 18:06:59 -0300
Subject: [PATCH 2576/2974] Name resolver: return result directly

---
 scrapy/resolver.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 7751f3796..554a3a14d 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -94,14 +94,12 @@ class CachingHostnameResolver:
                     dnscache[hostName] = self.resolution
 
         try:
-            result = dnscache[hostName]
+            return dnscache[hostName]
         except KeyError:
-            result = self.original_resolver.resolveHostName(
+            return self.original_resolver.resolveHostName(
                 CachingResolutionReceiver(),
                 hostName,
                 portNumber,
                 addressTypes,
                 transportSemantics
             )
-        finally:
-            return result

From c544c0d2b8356125d1a5465b44617aaaaeab0ea1 Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Fri, 24 Jan 2020 14:36:16 +0200
Subject: [PATCH 2577/2974] Use context management with `FTP`

---
 scrapy/utils/ftp.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index 752e3c953..9992a916e 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -22,13 +22,12 @@ def ftp_store_file(
     """Opens a FTP connection with passed credentials,sets current directory
     to the directory extracted from given path, then uploads the file to server
     """
-    ftp = FTP()
-    ftp.connect(host, port)
-    ftp.login(username, password)
-    if use_active_mode:
-        ftp.set_pasv(False)
-    file.seek(0)
-    dirname, filename = posixpath.split(path)
-    ftp_makedirs_cwd(ftp, dirname)
-    ftp.storbinary('STOR %s' % filename, file)
-    ftp.quit()
+    with FTP() as ftp:
+        ftp.connect(host, port)
+        ftp.login(username, password)
+        if use_active_mode:
+            ftp.set_pasv(False)
+        file.seek(0)
+        dirname, filename = posixpath.split(path)
+        ftp_makedirs_cwd(ftp, dirname)
+        ftp.storbinary('STOR %s' % filename, file)

From f5d9eb15f8b50c64c44a7f859a953b92d7a33e6b Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Fri, 24 Jan 2020 15:06:40 +0200
Subject: [PATCH 2578/2974] use `__future__` imports at the begining of the
 file

---
 scrapy/utils/test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index fd8f41137..65d24314e 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -2,10 +2,10 @@
 This module contains some assorted functions used in tests
 """
 
-import asyncio
-import os
 from __future__ import absolute_import
 from posixpath import split
+import asyncio
+import os
 
 from importlib import import_module
 from twisted.trial.unittest import SkipTest

From 40e0a11aa8dd499f725aaa206643aa36411fd514 Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Fri, 24 Jan 2020 15:51:48 +0200
Subject: [PATCH 2579/2974] Fix Flake8 errors

---
 scrapy/extensions/feedexport.py |  6 ++----
 scrapy/pipelines/files.py       | 18 +++++++++---------
 scrapy/utils/ftp.py             |  3 ++-
 scrapy/utils/test.py            |  8 +++++---
 tests/test_pipeline_files.py    |  5 ++++-
 5 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index c1ca0ca67..f1b101780 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -7,18 +7,16 @@ See documentation in docs/topics/feed-exports.rst
 import os
 import sys
 import logging
-import posixpath
 from tempfile import NamedTemporaryFile
 from datetime import datetime
 from urllib.parse import urlparse, unquote
-from ftplib import FTP
 
 from zope.interface import Interface, implementer
 from twisted.internet import defer, threads
 from w3lib.url import file_uri_to_path
 
 from scrapy import signals
-from scrapy.utils.ftp import ftp_makedirs_cwd, ftp_store_file
+from scrapy.utils.ftp import ftp_store_file
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.log import failure_to_exc_info
@@ -175,7 +173,7 @@ class FTPFeedStorage(BlockingFeedStorage):
     def _store_in_thread(self, file):
         ftp_store_file(
             path=self.path, file=file, host=self.host,
-            port=self.port, username=self.username, 
+            port=self.port, username=self.username,
             password=self.password, use_active_mode=self.use_active_mode
         )
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 2d286eed8..7e9b12c0e 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -28,7 +28,7 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.request import referer_str
 from scrapy.utils.boto import is_botocore
 from scrapy.utils.datatypes import CaselessDict
-from scrapy.utils.ftp import ftp_makedirs_cwd, ftp_store_file
+from scrapy.utils.ftp import ftp_store_file
 
 
 logger = logging.getLogger(__name__)
@@ -265,25 +265,25 @@ class FTPFilesStore(object):
     FTP_USERNAME = None
     FTP_PASSWORD = None
     USE_ACTIVE_MODE = None
-    
+
     def __init__(self, uri):
         assert uri.startswith('ftp://')
-        u = urlparse(uri)  
+        u = urlparse(uri)
         self.port = u.port
         self.host = u.hostname
         self.port = int(u.port or 21)
         self.username = u.username or self.FTP_USERNAME
         self.password = u.password or self.FTP_PASSWORD
         self.basedir = u.path.rstrip('/')
-        
-    def persist_file(self, path, buf, info, meta=None, headers=None):     
+
+    def persist_file(self, path, buf, info, meta=None, headers=None): 
         path = '%s/%s' % (self.basedir, path)
         return threads.deferToThread(
             ftp_store_file, path=path, file=buf,
             host=self.host, port=self.port, username=self.username,
             password=self.password, use_active_mode=self.USE_ACTIVE_MODE
         )
-            
+
     def stat_file(self, path, info):
         def _stat_file(path):
             try:
@@ -298,8 +298,8 @@ class FTPFilesStore(object):
                 ftp.retrbinary('RETR %s' % file_path, m.update)
                 return {'last_modified': last_modified, 'checksum': m.hexdigest()}
             # The file doesn't exist
-            except Exception as e :
-                return {} 
+            except Exception:
+                return {}
         return threads.deferToThread(_stat_file, path)
 
 
@@ -381,7 +381,7 @@ class FilesPipeline(MediaPipeline):
         ftp_store.FTP_USERNAME = settings['FTP_USER']
         ftp_store.FTP_PASSWORD = settings['FTP_PASSWORD']
         ftp_store.USE_ACTIVE_MODE = settings.getbool('FEED_STORAGE_FTP_ACTIVE')
-        
+
         store_uri = settings['FILES_STORE']
         return cls(store_uri, settings=settings)
 
diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index 1bb754a69..f07bdd748 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -17,7 +17,8 @@ def ftp_makedirs_cwd(ftp, path, first_call=True):
         if first_call:
             ftp.cwd(path)
 
-def ftp_store_file( 
+
+def ftp_store_file(
         *, path, file, host, port,
         username, password, use_active_mode=False):
     """Opens a FTP connection with passed credentials,sets current directory
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 65d24314e..61f2d059d 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -66,8 +66,9 @@ def get_gcs_content_and_delete(bucket, path):
     return content, acl, blob
 
 
-def get_ftp_content_and_delete(path, host ,port,
-        username, password, use_active_mode=False):
+def get_ftp_content_and_delete(
+        path, host, port,username,
+        password, use_active_mode=False):
     from ftplib import FTP
     ftp = FTP()
     ftp.connect(host, port)
@@ -75,6 +76,7 @@ def get_ftp_content_and_delete(path, host ,port,
     if use_active_mode:
         ftp.set_pasv(False)
     ftp_data = []
+
     def buffer_data(data):
         ftp_data.append(data)
     ftp.retrbinary('RETR %s' % path, buffer_data)
@@ -82,7 +84,7 @@ def get_ftp_content_and_delete(path, host ,port,
     ftp.cwd(dirname)
     ftp.delete(filename)
     return "".join(ftp_data)
-    
+
 
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index fc0453a97..e5bad2ed0 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -367,6 +367,7 @@ class TestGCSFilesStore(unittest.TestCase):
         self.assertEqual(blob.content_type, 'application/octet-stream')
         self.assertIn(expected_policy, acl)
 
+
 class TestFTPFileStore(unittest.TestCase):
     @defer.inlineCallbacks
     def test_persist(self):
@@ -386,10 +387,12 @@ class TestFTPFileStore(unittest.TestCase):
         self.assertIn('checksum', stat)
         self.assertEqual(stat['checksum'], 'd113d66b2ec7258724a268bd88eef6b6')
         path = '%s/%s' % (store.basedir, path)
-        content = get_ftp_content_and_delete(path, store.host, store.port,
+        content = get_ftp_content_and_delete(
+            path, store.host, store.port,
             store.username, store.password, store.USE_ACTIVE_MODE)
         self.assertEqual(data.decode(), content)
 
+
 class ItemWithFiles(Item):
     file_urls = Field()
     files = Field()

From 9e6d5573f1180bc70d7eca9f381204c238b3a550 Mon Sep 17 00:00:00 2001
From: OmarFarrag <omar.alaa.farrag@gmail.com>
Date: Fri, 24 Jan 2020 15:58:52 +0200
Subject: [PATCH 2580/2974] Fix Flake8 errors

---
 scrapy/pipelines/files.py | 3 +--
 scrapy/utils/test.py      | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 7e9b12c0e..9b7445755 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -13,7 +13,6 @@ from collections import defaultdict
 from email.utils import parsedate_tz, mktime_tz
 from ftplib import FTP
 from io import BytesIO
-from six.moves.urllib.parse import urlparse
 from urllib.parse import urlparse
 
 from twisted.internet import defer, threads
@@ -276,7 +275,7 @@ class FTPFilesStore(object):
         self.password = u.password or self.FTP_PASSWORD
         self.basedir = u.path.rstrip('/')
 
-    def persist_file(self, path, buf, info, meta=None, headers=None): 
+    def persist_file(self, path, buf, info, meta=None, headers=None):
         path = '%s/%s' % (self.basedir, path)
         return threads.deferToThread(
             ftp_store_file, path=path, file=buf,
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 61f2d059d..faac0b12f 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -67,7 +67,7 @@ def get_gcs_content_and_delete(bucket, path):
 
 
 def get_ftp_content_and_delete(
-        path, host, port,username,
+        path, host, port, username,
         password, use_active_mode=False):
     from ftplib import FTP
     ftp = FTP()

From f3374a50479fcfd9395919b204bcaff2405f7148 Mon Sep 17 00:00:00 2001
From: Peter Vandenabeele <peter@vandenabeele.com>
Date: Sat, 25 Jan 2020 16:52:30 +0100
Subject: [PATCH 2581/2974] Fix variable name `author_page_links`

I did not test this code, but the change from `href` to this
author_page_links seems to have a typo ?
---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index c9d00eb74..ee10048b5 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -652,7 +652,7 @@ this time for scraping author information::
 
         def parse(self, response):
             author_page_links = response.css('.author + a')
-            yield from response.follow_all(author_links, self.parse_author)
+            yield from response.follow_all(author_page_links, self.parse_author)
 
             pagination_links = response.css('li.next a')
             yield from response.follow_all(pagination_links, self.parse)

From f72d4e93e6b8a6774c515d55ca73fdbd6a01f13b Mon Sep 17 00:00:00 2001
From: Peter Vandenabeele <peter@vandenabeele.com>
Date: Sun, 26 Jan 2020 10:48:28 +0100
Subject: [PATCH 2582/2974] [Docs] 2 typos + 1 clarification in docs

Fixing 2 small typos and adding 1 word as clarification in the downloader-middlewares.

Also, I was confused with the entries like `ref:Reppy <reppy-parser>` and similar entries.
Are these supposed to be links to other parts of the doc, or is this the intended way of showing these references ?
---
 docs/topics/downloader-middleware.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index ae6d41809..8a760e53b 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -199,7 +199,7 @@ CookiesMiddleware
 
    This middleware enables working with sites that require cookies, such as
    those that use sessions. It keeps track of cookies sent by web servers, and
-   send them back on subsequent requests (from that spider), just like web
+   sends them back on subsequent requests (from that spider), just like web
    browsers do.
 
 The following settings can be used to configure the cookie middleware:
@@ -672,7 +672,7 @@ sometimes a more nuanced policy is desirable.
 
 This setting still respects ``Cache-Control: no-store`` directives in responses.
 If you don't want that, filter ``no-store`` out of the Cache-Control headers in
-responses you feedto the cache middleware.
+responses you feed to the cache middleware.
 
 .. setting:: HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS
 
@@ -686,7 +686,7 @@ Default: ``[]``
 List of Cache-Control directives in responses to be ignored.
 
 Sites often set "no-store", "no-cache", "must-revalidate", etc., but get
-upset at the traffic a spider can generate if it respects those
+upset at the traffic a spider can generate if it actually respects those
 directives. This allows to selectively ignore Cache-Control directives
 that are known to be unimportant for the sites being crawled.
 

From 80925ab845b7f55be97d9bb91015ceee90efc333 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 5 Aug 2019 11:39:07 -0300
Subject: [PATCH 2583/2974] Get server IP address for HTTP/1.1 responses

---
 docs/topics/request-response.rst          | 12 +++++++++-
 scrapy/core/downloader/__init__.py        |  2 +-
 scrapy/core/downloader/handlers/http11.py | 18 ++++++++++-----
 scrapy/http/response/__init__.py          |  5 +++--
 tests/test_crawl.py                       | 27 +++++++++++++++++++++++
 5 files changed, 54 insertions(+), 10 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 8997a7f19..a4cc1a7d7 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -34,7 +34,7 @@ Request objects
     :type url: string
 
     :param callback: the function that will be called with the response of this
-       request (once its downloaded) as its first parameter. For more information
+       request (once it's downloaded) as its first parameter. For more information
        see :ref:`topics-request-response-ref-request-callback-arguments` below.
        If a Request doesn't specify a callback, the spider's
        :meth:`~scrapy.spiders.Spider.parse` method will be used.
@@ -611,6 +611,12 @@ Response objects
         This represents the :class:`Request` that generated this response.
     :type request: :class:`Request` object
 
+    :param ip_address: The IP address of the server from which the Response originated.
+    :type ip_address: :class:`ipaddress.IPv4Address` object
+
+    .. FIXME: Add ipaddress.IPv6Address once it's supported
+
+
     .. attribute:: Response.url
 
         A string containing the URL of the response.
@@ -679,6 +685,10 @@ Response objects
         they're shown on the string representation of the Response (`__str__`
         method) which is used by the engine for logging.
 
+    .. attribute:: Response.ip_address
+
+        The IP address of the server from which the Response originated.
+
     .. method:: Response.copy()
 
        Returns a new Response which is a copy of this Response.
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 157dc3418..11c9dd908 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -172,7 +172,7 @@ class Downloader(object):
             return response
         dfd.addCallback(_downloaded)
 
-        # 3. After response arrives,  remove the request from transferring
+        # 3. After response arrives, remove the request from transferring
         # state to free up the transferring slot so it can be used by the
         # following requests (perhaps those which came from the downloader
         # middleware itself)
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 5a5f6cf0a..b690f439f 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -4,6 +4,7 @@ import logging
 import re
 import warnings
 from io import BytesIO
+from ipaddress import ip_address
 from time import time
 from urllib.parse import urldefrag
 
@@ -382,7 +383,7 @@ class ScrapyAgent(object):
     def _cb_bodyready(self, txresponse, request):
         # deliverBody hangs for responses without body
         if txresponse.length == 0:
-            return txresponse, b'', None
+            return txresponse, b'', None, None
 
         maxsize = request.meta.get('download_maxsize', self._maxsize)
         warnsize = request.meta.get('download_warnsize', self._warnsize)
@@ -418,11 +419,11 @@ class ScrapyAgent(object):
         return d
 
     def _cb_bodydone(self, result, request, url):
-        txresponse, body, flags = result
+        txresponse, body, flags, ip_address = result
         status = int(txresponse.code)
         headers = Headers(txresponse.headers.getAllRawHeaders())
         respcls = responsetypes.from_args(headers=headers, url=url, body=body)
-        return respcls(url=url, status=status, headers=headers, body=body, flags=flags)
+        return respcls(url=url, status=status, headers=headers, body=body, flags=flags, ip_address=ip_address)
 
 
 @implementer(IBodyProducer)
@@ -456,6 +457,11 @@ class _ResponseReader(protocol.Protocol):
         self._fail_on_dataloss_warned = False
         self._reached_warnsize = False
         self._bytes_received = 0
+        self._ip_address = None
+
+    def connectionMade(self):
+        if self._ip_address is None:
+            self._ip_address = ip_address(self.transport._producer.getPeer().host)
 
     def dataReceived(self, bodyBytes):
         # This maybe called several times after cancel was called with buffered data.
@@ -488,16 +494,16 @@ class _ResponseReader(protocol.Protocol):
 
         body = self._bodybuf.getvalue()
         if reason.check(ResponseDone):
-            self._finished.callback((self._txresponse, body, None))
+            self._finished.callback((self._txresponse, body, None, self._ip_address))
             return
 
         if reason.check(PotentialDataLoss):
-            self._finished.callback((self._txresponse, body, ['partial']))
+            self._finished.callback((self._txresponse, body, ['partial'], self._ip_address))
             return
 
         if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
             if not self._fail_on_dataloss:
-                self._finished.callback((self._txresponse, body, ['dataloss']))
+                self._finished.callback((self._txresponse, body, ['dataloss'], self._ip_address))
                 return
 
             elif not self._fail_on_dataloss_warned:
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index f92d0901c..ca5ecc02c 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -17,13 +17,14 @@ from scrapy.utils.trackref import object_ref
 
 class Response(object_ref):
 
-    def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None):
+    def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None, ip_address=None):
         self.headers = Headers(headers or {})
         self.status = int(status)
         self._set_body(body)
         self._set_url(url)
         self.request = request
         self.flags = [] if flags is None else list(flags)
+        self.ip_address = ip_address
 
     @property
     def meta(self):
@@ -76,7 +77,7 @@ class Response(object_ref):
         """Create a new Response with the same attributes except for those
         given new values.
         """
-        for x in ['url', 'status', 'headers', 'body', 'request', 'flags']:
+        for x in ['url', 'status', 'headers', 'body', 'request', 'flags', 'ip_address']:
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index f433fcea6..6281160ae 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -1,5 +1,7 @@
 import json
 import logging
+from ipaddress import IPv4Address
+from urllib.parse import urlparse
 
 from testfixtures import LogCapture
 from twisted.internet import defer
@@ -308,3 +310,28 @@ with multiples lines
         self.assertIn("[callback] status 201", str(log))
         self.assertIn("[errback] status 404", str(log))
         self.assertIn("[errback] status 500", str(log))
+
+    @defer.inlineCallbacks
+    def test_dns_server_ip_address(self):
+        from socket import gethostbyname
+
+        crawler = self.runner.create_crawler(SingleRequestSpider)
+        url = 'https://example.org'
+        yield crawler.crawl(seed=url)
+        ip_address = crawler.spider.meta['responses'][0].ip_address
+        self.assertIsInstance(ip_address, IPv4Address)
+        self.assertEqual(str(ip_address), gethostbyname(urlparse(url).netloc))
+
+        crawler = self.runner.create_crawler(SingleRequestSpider)
+        url = self.mockserver.url('/status?n=200')
+        yield crawler.crawl(seed=url, mockserver=self.mockserver)
+        ip_address = crawler.spider.meta['responses'][0].ip_address
+        self.assertIsNone(ip_address)
+
+        crawler = self.runner.create_crawler(SingleRequestSpider)
+        url = self.mockserver.url('/echo?body=test')
+        expected_netloc, _ = urlparse(url).netloc.split(':')
+        yield crawler.crawl(seed=url, mockserver=self.mockserver)
+        ip_address = crawler.spider.meta['responses'][0].ip_address
+        self.assertIsInstance(ip_address, IPv4Address)
+        self.assertEqual(str(ip_address), gethostbyname(expected_netloc))

From e8da7e296691d2b4eb63e2a442bb600e03e5766f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 26 Jan 2020 17:53:39 -0300
Subject: [PATCH 2584/2974] Test DNS resolution using CrawlerProcess

---
 tests/CrawlerProcess/ip_address.py | 51 ++++++++++++++++++++++++++++++
 tests/test_crawl.py                | 10 +-----
 tests/test_crawler.py              |  8 +++++
 3 files changed, 60 insertions(+), 9 deletions(-)
 create mode 100644 tests/CrawlerProcess/ip_address.py

diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.py
new file mode 100644
index 000000000..6b069cc90
--- /dev/null
+++ b/tests/CrawlerProcess/ip_address.py
@@ -0,0 +1,51 @@
+from urllib.parse import urlparse
+
+from twisted.internet import defer
+from twisted.internet.base import ThreadedResolver
+from twisted.internet.interfaces import IResolverSimple
+from zope.interface.declarations import implementer
+
+from scrapy import Spider, Request
+from scrapy.crawler import CrawlerProcess
+
+from tests.mockserver import MockServer
+
+
+@implementer(IResolverSimple)
+class MockThreadedResolver(ThreadedResolver):
+    """
+    Resolves all names to localhost
+    """
+
+    @classmethod
+    def from_crawler(cls, crawler, reactor):
+        return cls(reactor)
+
+    def install_on_reactor(self,):
+        self.reactor.installResolver(self)
+
+    def getHostByName(self, name, timeout=None):
+        return defer.succeed("127.0.0.1")
+
+
+class LocalhostSpider(Spider):
+    name = "localhost_spider"
+
+    def start_requests(self):
+        yield Request(self.url)
+
+    def parse(self, response):
+        netloc = urlparse(response.url).netloc
+        self.logger.info("Host: %s" % netloc.split(":")[0])
+        self.logger.info("Type: %s" % type(response.ip_address))
+        self.logger.info("IP address: %s" % response.ip_address)
+
+
+with MockServer() as mockserver:
+    settings = {"DNS_RESOLVER": __name__ + ".MockThreadedResolver"}
+    process = CrawlerProcess(settings)
+
+    port = urlparse(mockserver.http_address).port
+    url = "http://not.a.real.domain:{port}/echo?body=test".format(port=port)
+    process.crawl(LocalhostSpider, url=url)
+    process.start()
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 6281160ae..9896058dc 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -1,6 +1,7 @@
 import json
 import logging
 from ipaddress import IPv4Address
+from socket import gethostbyname
 from urllib.parse import urlparse
 
 from testfixtures import LogCapture
@@ -313,15 +314,6 @@ with multiples lines
 
     @defer.inlineCallbacks
     def test_dns_server_ip_address(self):
-        from socket import gethostbyname
-
-        crawler = self.runner.create_crawler(SingleRequestSpider)
-        url = 'https://example.org'
-        yield crawler.crawl(seed=url)
-        ip_address = crawler.spider.meta['responses'][0].ip_address
-        self.assertIsInstance(ip_address, IPv4Address)
-        self.assertEqual(str(ip_address), gethostbyname(urlparse(url).netloc))
-
         crawler = self.runner.create_crawler(SingleRequestSpider)
         url = self.mockserver.url('/status?n=200')
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 0ce0674de..dfc1cf448 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -107,6 +107,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
 
     def test_spider_custom_settings_log_level(self):
         log_file = self.mktemp()
+
         class MySpider(scrapy.Spider):
             name = 'spider'
             custom_settings = {
@@ -323,3 +324,10 @@ class CrawlerProcessSubprocess(unittest.TestCase):
             "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
             "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
         ]))
+
+    def test_response_ip_address(self):
+        log = self.run_script("ip_address.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Host: not.a.real.domain", log)
+        self.assertIn("Type: <class 'ipaddress.IPv4Address'>", log)
+        self.assertIn("IP address: 127.0.0.1", log)

From 8529dff41d3d2f6c81ee58c60b16dd9f2b8f72b4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 26 Jan 2020 18:00:56 -0300
Subject: [PATCH 2585/2974] Update docs regarding Response.ip_address and IPv6

---
 docs/topics/request-response.rst | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index a4cc1a7d7..17eb63064 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -612,10 +612,7 @@ Response objects
     :type request: :class:`Request` object
 
     :param ip_address: The IP address of the server from which the Response originated.
-    :type ip_address: :class:`ipaddress.IPv4Address` object
-
-    .. FIXME: Add ipaddress.IPv6Address once it's supported
-
+    :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
 
     .. attribute:: Response.url
 

From 72b8613ee9827af031862bd84f1bea9acefcbebe Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 27 Nov 2019 14:46:20 -0300
Subject: [PATCH 2586/2974] bytes_received signal (no tests)

---
 docs/topics/signals.rst                   | 45 ++++++++++++++++-------
 scrapy/core/downloader/handlers/http11.py | 25 +++++++++++--
 scrapy/signals.py                         |  1 +
 3 files changed, 54 insertions(+), 17 deletions(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 3f29aa323..6efb73abb 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -73,7 +73,7 @@ engine_started
 
     Sent when the Scrapy engine has started crawling.
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
 .. note:: This signal may be fired *after* the :signal:`spider_opened` signal,
     depending on how the spider was started. So **don't** rely on this signal
@@ -88,7 +88,7 @@ engine_stopped
     Sent when the Scrapy engine is stopped (for example, when a crawling
     process has finished).
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
 item_scraped
 ------------
@@ -99,7 +99,7 @@ item_scraped
     Sent when an item has been scraped, after it has passed all the
     :ref:`topics-item-pipeline` stages (without being dropped).
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
     :param item: the item scraped
     :type item: dict or :class:`~scrapy.item.Item` object
@@ -119,7 +119,7 @@ item_dropped
     Sent after an item has been dropped from the :ref:`topics-item-pipeline`
     when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception.
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
     :param item: the item dropped from the :ref:`topics-item-pipeline`
     :type item: dict or :class:`~scrapy.item.Item` object
@@ -144,7 +144,7 @@ item_error
     Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises
     an exception), except :exc:`~scrapy.exceptions.DropItem` exception.
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
     :param item: the item dropped from the :ref:`topics-item-pipeline`
     :type item: dict or :class:`~scrapy.item.Item` object
@@ -158,6 +158,23 @@ item_error
     :param failure: the exception raised
     :type failure: twisted.python.failure.Failure
 
+bytes_received
+--------------
+
+.. signal:: bytes_received
+.. function:: bytes_received(data, request)
+
+    Sent by the HTTP 1.1 download handler when a group of bytes is
+    received for a specific request.
+
+    This signal does not support returning deferreds from its handlers.
+
+    :param data: the data received by the download handler
+    :type spider: :class:`bytes` object
+
+    :param request: the request that generated the response
+    :type request: :class:`~scrapy.http.Request` object
+
 spider_closed
 -------------
 
@@ -167,7 +184,7 @@ spider_closed
     Sent after a spider has been closed. This can be used to release per-spider
     resources reserved on :signal:`spider_opened`.
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
     :param spider: the spider which has been closed
     :type spider: :class:`~scrapy.spiders.Spider` object
@@ -191,7 +208,7 @@ spider_opened
     reserve per-spider resources, but can be used for any task that needs to be
     performed when a spider is opened.
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
     :param spider: the spider which has been opened
     :type spider: :class:`~scrapy.spiders.Spider` object
@@ -215,7 +232,7 @@ spider_idle
     You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
     prevent the spider from being closed.
 
-    This signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param spider: the spider which has gone idle
     :type spider: :class:`~scrapy.spiders.Spider` object
@@ -234,7 +251,7 @@ spider_error
 
     Sent when a spider callback generates an error (ie. raises an exception).
 
-    This signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param failure: the exception raised
     :type failure: twisted.python.failure.Failure
@@ -254,7 +271,7 @@ request_scheduled
     Sent when the engine schedules a :class:`~scrapy.http.Request`, to be
     downloaded later.
 
-    The signal does not support returning deferreds from their handlers.
+    The signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached the scheduler
     :type request: :class:`~scrapy.http.Request` object
@@ -271,7 +288,7 @@ request_dropped
     Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be
     downloaded later, is rejected by the scheduler.
 
-    The signal does not support returning deferreds from their handlers.
+    The signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached the scheduler
     :type request: :class:`~scrapy.http.Request` object
@@ -287,7 +304,7 @@ request_reached_downloader
 
     Sent when a :class:`~scrapy.http.Request` reached downloader.
 
-    The signal does not support returning deferreds from their handlers.
+    The signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached downloader
     :type request: :class:`~scrapy.http.Request` object
@@ -304,7 +321,7 @@ response_received
     Sent when the engine receives a new :class:`~scrapy.http.Response` from the
     downloader.
 
-    This signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param response: the response received
     :type response: :class:`~scrapy.http.Response` object
@@ -323,7 +340,7 @@ response_downloaded
 
     Sent by the downloader right after a ``HTTPResponse`` is downloaded.
 
-    This signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param response: the response downloaded
     :type response: :class:`~scrapy.http.Response` object
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 5a5f6cf0a..92c3d5f5c 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -16,6 +16,7 @@ from twisted.web.http_headers import Headers as TxHeaders
 from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
 from zope.interface import implementer
 
+from scrapy import signals
 from scrapy.core.downloader.tls import openssl_methods
 from scrapy.core.downloader.webclient import _parse
 from scrapy.exceptions import ScrapyDeprecationWarning
@@ -32,6 +33,7 @@ class HTTP11DownloadHandler:
     lazy = False
 
     def __init__(self, settings, crawler=None):
+        self.crawler = crawler
         self._pool = HTTPConnectionPool(reactor, persistent=True)
         self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self._pool._factory.noisy = False
@@ -76,6 +78,7 @@ class HTTP11DownloadHandler:
             maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
             warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
             fail_on_dataloss=self._fail_on_dataloss,
+            crawler=self.crawler,
         )
         return agent.download_request(request)
 
@@ -272,7 +275,7 @@ class ScrapyAgent(object):
     _TunnelingAgent = TunnelingAgent
 
     def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
-                 maxsize=0, warnsize=0, fail_on_dataloss=True):
+                 maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None):
         self._contextFactory = contextFactory
         self._connectTimeout = connectTimeout
         self._bindAddress = bindAddress
@@ -281,6 +284,7 @@ class ScrapyAgent(object):
         self._warnsize = warnsize
         self._fail_on_dataloss = fail_on_dataloss
         self._txresponse = None
+        self._crawler = crawler
 
     def _get_agent(self, request, timeout):
         bindaddress = request.meta.get('bindaddress') or self._bindAddress
@@ -409,7 +413,15 @@ class ScrapyAgent(object):
 
         d = defer.Deferred(_cancel)
         txresponse.deliverBody(
-            _ResponseReader(d, txresponse, request, maxsize, warnsize, fail_on_dataloss)
+            _ResponseReader(
+                d,
+                txresponse,
+                request,
+                maxsize,
+                warnsize,
+                fail_on_dataloss,
+                self._crawler,
+            )
         )
 
         # save response for timeouts
@@ -445,7 +457,7 @@ class _RequestBodyProducer(object):
 
 class _ResponseReader(protocol.Protocol):
 
-    def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss):
+    def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler):
         self._finished = finished
         self._txresponse = txresponse
         self._request = request
@@ -456,6 +468,7 @@ class _ResponseReader(protocol.Protocol):
         self._fail_on_dataloss_warned = False
         self._reached_warnsize = False
         self._bytes_received = 0
+        self._crawler = crawler
 
     def dataReceived(self, bodyBytes):
         # This maybe called several times after cancel was called with buffered data.
@@ -465,6 +478,12 @@ class _ResponseReader(protocol.Protocol):
         self._bodybuf.write(bodyBytes)
         self._bytes_received += len(bodyBytes)
 
+        self._crawler.signals.send_catch_log(
+            signal=signals.bytes_received,
+            data=bodyBytes,
+            request=self._request,
+        )
+
         if self._maxsize and self._bytes_received > self._maxsize:
             logger.error("Received (%(bytes)s) bytes larger than download "
                          "max size (%(maxsize)s) in request %(request)s.",
diff --git a/scrapy/signals.py b/scrapy/signals.py
index 6b9125302..590421893 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -16,6 +16,7 @@ request_dropped = object()
 request_reached_downloader = object()
 response_received = object()
 response_downloaded = object()
+bytes_received = object()
 item_scraped = object()
 item_dropped = object()
 item_error = object()

From cab449b1952020b86fbe2915a537150fc885c567 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 29 Nov 2019 11:37:40 -0300
Subject: [PATCH 2587/2974] Typo fix

---
 tests/test_engine.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index 25dee7c1f..9d68836cc 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -107,7 +107,7 @@ class CrawlerRun(object):
         self.reqreached = []
         self.itemerror = []
         self.itemresp = []
-        self.signals_catched = {}
+        self.signals_caught = {}
         self.spider_class = spider_class
 
     def run(self):
@@ -172,7 +172,7 @@ class CrawlerRun(object):
         signalargs = kwargs.copy()
         sig = signalargs.pop('signal')
         signalargs.pop('sender', None)
-        self.signals_catched[sig] = signalargs
+        self.signals_caught[sig] = signalargs
 
 
 class EngineTest(unittest.TestCase):
@@ -186,7 +186,7 @@ class EngineTest(unittest.TestCase):
             self._assert_scheduled_requests(urls_to_visit=8)
             self._assert_downloaded_responses()
             self._assert_scraped_items()
-            self._assert_signals_catched()
+            self._assert_signals_caught()
 
     @defer.inlineCallbacks
     def test_crawler_dupefilter(self):
@@ -263,19 +263,19 @@ class EngineTest(unittest.TestCase):
                 self.assertEqual('Item 2 name', item['name'])
                 self.assertEqual('200', item['price'])
 
-    def _assert_signals_catched(self):
-        assert signals.engine_started in self.run.signals_catched
-        assert signals.engine_stopped in self.run.signals_catched
-        assert signals.spider_opened in self.run.signals_catched
-        assert signals.spider_idle in self.run.signals_catched
-        assert signals.spider_closed in self.run.signals_catched
+    def _assert_signals_caught(self):
+        assert signals.engine_started in self.run.signals_caught
+        assert signals.engine_stopped in self.run.signals_caught
+        assert signals.spider_opened in self.run.signals_caught
+        assert signals.spider_idle in self.run.signals_caught
+        assert signals.spider_closed in self.run.signals_caught
 
         self.assertEqual({'spider': self.run.spider},
-                         self.run.signals_catched[signals.spider_opened])
+                         self.run.signals_caught[signals.spider_opened])
         self.assertEqual({'spider': self.run.spider},
-                         self.run.signals_catched[signals.spider_idle])
+                         self.run.signals_caught[signals.spider_idle])
         self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},
-                         self.run.signals_catched[signals.spider_closed])
+                         self.run.signals_caught[signals.spider_closed])
 
     @defer.inlineCallbacks
     def test_close_downloader(self):

From bda37e38bd53d5aae691b56d4136fbff99f78158 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 29 Nov 2019 12:02:27 -0300
Subject: [PATCH 2588/2974] [Tests] bytes_received signal

---
 tests/test_engine.py | 53 +++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 47 insertions(+), 6 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index 9d68836cc..b63c7e232 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -13,22 +13,24 @@ module with the ``runserver`` argument::
 import os
 import re
 import sys
+from collections import defaultdict
 from urllib.parse import urlparse
 
 from twisted.internet import reactor, defer
-from twisted.web import server, static, util
 from twisted.trial import unittest
+from twisted.web import server, static, util
+from pydispatch import dispatcher
 
 from scrapy import signals
 from scrapy.core.engine import ExecutionEngine
-from scrapy.utils.test import get_crawler
-from pydispatch import dispatcher
-from tests import tests_datadir
-from scrapy.spiders import Spider
+from scrapy.http import Request
 from scrapy.item import Item, Field
 from scrapy.linkextractors import LinkExtractor
-from scrapy.http import Request
+from scrapy.spiders import Spider
 from scrapy.utils.signal import disconnect_all
+from scrapy.utils.test import get_crawler
+
+from tests import tests_datadir, get_testdata
 
 
 class TestItem(Item):
@@ -107,6 +109,7 @@ class CrawlerRun(object):
         self.reqreached = []
         self.itemerror = []
         self.itemresp = []
+        self.bytes = defaultdict(lambda: b"")
         self.signals_caught = {}
         self.spider_class = spider_class
 
@@ -124,6 +127,7 @@ class CrawlerRun(object):
         self.crawler = get_crawler(self.spider_class)
         self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
         self.crawler.signals.connect(self.item_error, signals.item_error)
+        self.crawler.signals.connect(self.bytes_received, signals.bytes_received)
         self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
         self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
         self.crawler.signals.connect(self.request_reached, signals.request_reached_downloader)
@@ -155,6 +159,9 @@ class CrawlerRun(object):
     def item_scraped(self, item, spider, response):
         self.itemresp.append((item, response))
 
+    def bytes_received(self, data, request):
+        self.bytes[request] += data
+
     def request_scheduled(self, request, spider):
         self.reqplug.append((request, spider))
 
@@ -187,6 +194,7 @@ class EngineTest(unittest.TestCase):
             self._assert_downloaded_responses()
             self._assert_scraped_items()
             self._assert_signals_caught()
+            self._assert_bytes_received()
 
     @defer.inlineCallbacks
     def test_crawler_dupefilter(self):
@@ -263,6 +271,39 @@ class EngineTest(unittest.TestCase):
                 self.assertEqual('Item 2 name', item['name'])
                 self.assertEqual('200', item['price'])
 
+    def _assert_bytes_received(self):
+        self.assertEqual(8, len(self.run.bytes))
+        for request, data in self.run.bytes.items():
+            if self.run.getpath(request.url) == "/":
+                self.assertEqual(data, get_testdata("test_site", "index.html"))
+            elif self.run.getpath(request.url) == "/item1.html":
+                self.assertEqual(data, get_testdata("test_site", "item1.html"))
+            elif self.run.getpath(request.url) == "/item2.html":
+                self.assertEqual(data, get_testdata("test_site", "item2.html"))
+            elif self.run.getpath(request.url) == "/redirected":
+                self.assertEqual(data, b"Redirected here")
+            elif self.run.getpath(request.url) == '/redirect':
+                self.assertEqual(data,
+                    b"\n<html>\n"
+                    b"    <head>\n"
+                    b"        <meta http-equiv=\"refresh\" content=\"0;URL=/redirected\">\n"
+                    b"    </head>\n"
+                    b"    <body bgcolor=\"#FFFFFF\" text=\"#000000\">\n"
+                    b"    <a href=\"/redirected\">click here</a>\n"
+                    b"    </body>\n"
+                    b"</html>\n"
+                )
+            elif self.run.getpath(request.url) == "/tem999.html":
+                self.assertEqual(data,
+                    b"\n<html>\n"
+                    b"  <head><title>404 - No Such Resource</title></head>\n"
+                    b"  <body>\n"
+                    b"    <h1>No Such Resource</h1>\n"
+                    b"    <p>File not found.</p>\n"
+                    b"  </body>\n"
+                    b"</html>\n"
+                )
+
     def _assert_signals_caught(self):
         assert signals.engine_started in self.run.signals_caught
         assert signals.engine_stopped in self.run.signals_caught

From 89483ce9f709e230ee5ff9050d206430d2d17c9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 3 Dec 2019 12:06:08 +0100
Subject: [PATCH 2589/2974] Fix Flake8 issues

---
 tests/test_engine.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index b63c7e232..c0769c992 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -283,7 +283,8 @@ class EngineTest(unittest.TestCase):
             elif self.run.getpath(request.url) == "/redirected":
                 self.assertEqual(data, b"Redirected here")
             elif self.run.getpath(request.url) == '/redirect':
-                self.assertEqual(data,
+                self.assertEqual(
+                    data,
                     b"\n<html>\n"
                     b"    <head>\n"
                     b"        <meta http-equiv=\"refresh\" content=\"0;URL=/redirected\">\n"
@@ -294,7 +295,8 @@ class EngineTest(unittest.TestCase):
                     b"</html>\n"
                 )
             elif self.run.getpath(request.url) == "/tem999.html":
-                self.assertEqual(data,
+                self.assertEqual(
+                    data,
                     b"\n<html>\n"
                     b"  <head><title>404 - No Such Resource</title></head>\n"
                     b"  <body>\n"

From c9d36522302ab73552d804137a3625552275a771 Mon Sep 17 00:00:00 2001
From: "Matsievskiy S.V" <matsievskiysv@gmail.com>
Date: Mon, 27 Jan 2020 18:24:57 +0300
Subject: [PATCH 2590/2974] add zsh -h autocomplete option

---
 extras/scrapy_zsh_completion | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion
index e995947cb..33f46eda8 100644
--- a/extras/scrapy_zsh_completion
+++ b/extras/scrapy_zsh_completion
@@ -1,11 +1,12 @@
 #compdef scrapy
 _scrapy() {
     local context state state_descr line
+    local ret=1
     typeset -A opt_args
     _arguments \
-	"(- 1 *)--help[Help]" \
+	"(- 1 *)"{-h,--help}"[Help]" \
 	"1: :->command" \
-	"*:: :->args"
+	"*:: :->args" && ret=0
 
     case $state in
 	command)
@@ -134,6 +135,8 @@ _scrapy() {
 	    esac
 	    ;;
     esac
+
+    return ret
 }
 
 _scrapy_cmds() {

From dbe20a863ff63dce937b2d3b159782d8268e6838 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 27 Jan 2020 12:21:18 -0300
Subject: [PATCH 2591/2974] bytes_received signal: send spider argument

---
 docs/topics/signals.rst                   | 5 ++++-
 scrapy/core/downloader/handlers/http11.py | 1 +
 tests/test_engine.py                      | 2 +-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 6efb73abb..3e70ca067 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -162,7 +162,7 @@ bytes_received
 --------------
 
 .. signal:: bytes_received
-.. function:: bytes_received(data, request)
+.. function:: bytes_received(data, request, spider)
 
     Sent by the HTTP 1.1 download handler when a group of bytes is
     received for a specific request.
@@ -175,6 +175,9 @@ bytes_received
     :param request: the request that generated the response
     :type request: :class:`~scrapy.http.Request` object
 
+    :param spider: the spider associated with the response
+    :type spider: :class:`~scrapy.spiders.Spider` object
+
 spider_closed
 -------------
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 92c3d5f5c..c53c9bb2d 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -482,6 +482,7 @@ class _ResponseReader(protocol.Protocol):
             signal=signals.bytes_received,
             data=bodyBytes,
             request=self._request,
+            spider=self._crawler.spider,
         )
 
         if self._maxsize and self._bytes_received > self._maxsize:
diff --git a/tests/test_engine.py b/tests/test_engine.py
index c0769c992..57cc89ba3 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -159,7 +159,7 @@ class CrawlerRun(object):
     def item_scraped(self, item, spider, response):
         self.itemresp.append((item, response))
 
-    def bytes_received(self, data, request):
+    def bytes_received(self, data, request, spider):
         self.bytes[request] += data
 
     def request_scheduled(self, request, spider):

From 613fd41f44d1455f9c9369087958674f3fdfcc8d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 27 Jan 2020 12:30:26 -0300
Subject: [PATCH 2592/2974] bytes_received signal: improve test performance

---
 tests/test_engine.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index 57cc89ba3..bb475958e 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -109,7 +109,7 @@ class CrawlerRun(object):
         self.reqreached = []
         self.itemerror = []
         self.itemresp = []
-        self.bytes = defaultdict(lambda: b"")
+        self.bytes = defaultdict(lambda: list())
         self.signals_caught = {}
         self.spider_class = spider_class
 
@@ -160,7 +160,7 @@ class CrawlerRun(object):
         self.itemresp.append((item, response))
 
     def bytes_received(self, data, request, spider):
-        self.bytes[request] += data
+        self.bytes[request].append(data)
 
     def request_scheduled(self, request, spider):
         self.reqplug.append((request, spider))
@@ -274,17 +274,18 @@ class EngineTest(unittest.TestCase):
     def _assert_bytes_received(self):
         self.assertEqual(8, len(self.run.bytes))
         for request, data in self.run.bytes.items():
+            joined_data = b"".join(data)
             if self.run.getpath(request.url) == "/":
-                self.assertEqual(data, get_testdata("test_site", "index.html"))
+                self.assertEqual(joined_data, get_testdata("test_site", "index.html"))
             elif self.run.getpath(request.url) == "/item1.html":
-                self.assertEqual(data, get_testdata("test_site", "item1.html"))
+                self.assertEqual(joined_data, get_testdata("test_site", "item1.html"))
             elif self.run.getpath(request.url) == "/item2.html":
-                self.assertEqual(data, get_testdata("test_site", "item2.html"))
+                self.assertEqual(joined_data, get_testdata("test_site", "item2.html"))
             elif self.run.getpath(request.url) == "/redirected":
-                self.assertEqual(data, b"Redirected here")
+                self.assertEqual(joined_data, b"Redirected here")
             elif self.run.getpath(request.url) == '/redirect':
                 self.assertEqual(
-                    data,
+                    joined_data,
                     b"\n<html>\n"
                     b"    <head>\n"
                     b"        <meta http-equiv=\"refresh\" content=\"0;URL=/redirected\">\n"
@@ -296,7 +297,7 @@ class EngineTest(unittest.TestCase):
                 )
             elif self.run.getpath(request.url) == "/tem999.html":
                 self.assertEqual(
-                    data,
+                    joined_data,
                     b"\n<html>\n"
                     b"  <head><title>404 - No Such Resource</title></head>\n"
                     b"  <body>\n"

From 4ffd18fb11ff89863569b8b4de44241e3ca2f86e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 27 Jan 2020 13:29:33 -0300
Subject: [PATCH 2593/2974] [docs] Mention that signals.bytes_received could be
 fired multiple times

---
 docs/topics/signals.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 3e70ca067..f490911f3 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -165,7 +165,8 @@ bytes_received
 .. function:: bytes_received(data, request, spider)
 
     Sent by the HTTP 1.1 download handler when a group of bytes is
-    received for a specific request.
+    received for a specific request. This signal might be fired
+    multiple times for the same request.
 
     This signal does not support returning deferreds from its handlers.
 

From ad4477d335bee8b10bc3bbca969defddd9b316f8 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 27 Jan 2020 14:16:43 -0300
Subject: [PATCH 2594/2974] Remove unnecessary else

---
 scrapy/core/spidermw.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index ed02b306b..8b36cbb04 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -107,12 +107,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
                     if isinstance(exception_result, Failure):
                         raise
                     return exception_result
+                if _isiterable(result):
+                    result = _evaluate_iterable(result, method_index+1, recovered)
                 else:
-                    if _isiterable(result):
-                        result = _evaluate_iterable(result, method_index+1, recovered)
-                    else:
-                        msg = "Middleware {} must return an iterable, got {}"
-                        raise _InvalidOutput(msg.format(_fname(method), type(result)))
+                    msg = "Middleware {} must return an iterable, got {}"
+                    raise _InvalidOutput(msg.format(_fname(method), type(result)))
 
             return MutableChain(result, recovered)
 

From a3b168948cb533427eceb68cf84bc8542732848b Mon Sep 17 00:00:00 2001
From: Kevin Lloyd Bernal <kevinoxy@gmail.com>
Date: Wed, 29 Jan 2020 04:53:25 +0800
Subject: [PATCH 2595/2974] Log an error when giving up requests after too many
 retries (#3566)

---
 scrapy/downloadermiddlewares/retry.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index dbc605a4c..7ab5b6e62 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -84,6 +84,6 @@ class RetryMiddleware(object):
             return retryreq
         else:
             stats.inc_value('retry/max_reached')
-            logger.debug("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
+            logger.error("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
                          {'request': request, 'retries': retries, 'reason': reason},
                          extra={'spider': spider})

From 752e8f7018cbfac9cbdf486046d6bd8171cca0e8 Mon Sep 17 00:00:00 2001
From: Daniel Kimsey <dekimsey@gmail.com>
Date: Sun, 26 Jan 2020 13:21:31 -0600
Subject: [PATCH 2596/2974] FilesPipeline.file_path has optional arguments

Documented signature doesn't match the actual interface in [files.py](https://github.com/scrapy/scrapy/blob/master/scrapy/pipelines/files.py#L520).

Specifically, it looks like it may be [called](https://github.com/scrapy/scrapy/blob/master/scrapy/pipelines/files.py#L422) without a response value.

I found this when I was implementing the pipeline with the signature `file_path(self, request, response, info)` and the following error was being return in my results :

    [(False, <twisted.python.failure.Failure builtins.TypeError: file_path() missing 1 required positional argument: 'response'>)]

Scrapy==1.8.0
---
 docs/topics/media-pipeline.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 1e0e0f18f..67a0bfdba 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -410,7 +410,7 @@ See here the methods that you can override in your custom Files Pipeline:
 
 .. class:: FilesPipeline
 
-   .. method:: file_path(request, response, info)
+   .. method:: file_path(self, request, response=None, info=None)
 
       This method is called once per downloaded item. It returns the
       download path of the file originating from the specified
@@ -434,7 +434,7 @@ See here the methods that you can override in your custom Files Pipeline:
 
         class MyFilesPipeline(FilesPipeline):
 
-            def file_path(self, request, response, info):
+            def file_path(self, request, response=None, info=None):
                 return 'files/' + os.path.basename(urlparse(request.url).path)
 
       By default the :meth:`file_path` method returns
@@ -524,7 +524,7 @@ See here the methods that you can override in your custom Images Pipeline:
     The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
     customizing the field names and adding custom behavior for images.
 
-   .. method:: file_path(request, response, info)
+   .. method:: file_path(self, request, response=None, info=None)
 
       This method is called once per downloaded item. It returns the
       download path of the file originating from the specified
@@ -548,7 +548,7 @@ See here the methods that you can override in your custom Images Pipeline:
 
         class MyImagesPipeline(ImagesPipeline):
 
-            def file_path(self, request, response, info):
+            def file_path(self, request, response=None, info=None):
                 return 'files/' + os.path.basename(urlparse(request.url).path)
 
       By default the :meth:`file_path` method returns

From 4e56571a196c09f8976b1b31a82a8ce2d0ee0be7 Mon Sep 17 00:00:00 2001
From: Evgeny Dorofeev <evgeny@scrapinghub.com>
Date: Wed, 29 Jan 2020 15:48:47 +0300
Subject: [PATCH 2597/2974] [HttpCompressionMiddleware] fix delimiter for
 Accept-Encoding header

---
 scrapy/downloadermiddlewares/httpcompression.py    | 2 +-
 tests/test_downloadermiddleware_httpcompression.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 65b652953..0010b2a8f 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -26,7 +26,7 @@ class HttpCompressionMiddleware(object):
 
     def process_request(self, request, spider):
         request.headers.setdefault('Accept-Encoding',
-                                   b",".join(ACCEPTED_ENCODINGS))
+                                   b", ".join(ACCEPTED_ENCODINGS))
 
     def process_response(self, request, response, spider):
 
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index c6a823b53..64488841a 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -48,7 +48,7 @@ class HttpCompressionTest(TestCase):
                 }
 
         response = Response('http://scrapytest.org/', body=body, headers=headers)
-        response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip,deflate'})
+        response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'})
         return response
 
     def test_process_request(self):
@@ -56,7 +56,7 @@ class HttpCompressionTest(TestCase):
         assert 'Accept-Encoding' not in request.headers
         self.mw.process_request(request, self.spider)
         self.assertEqual(request.headers.get('Accept-Encoding'),
-                         b','.join(ACCEPTED_ENCODINGS))
+                         b', '.join(ACCEPTED_ENCODINGS))
 
     def test_process_response_gzip(self):
         response = self._getresponse('gzip')

From 2c9643d38cc076c4d2032efd994fda4cfcc9f88a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 29 Jan 2020 14:11:56 -0300
Subject: [PATCH 2598/2974] Test: bytes_received signal fired multiple times

---
 tests/test_engine.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index bb475958e..3c5cc403b 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -12,6 +12,7 @@ module with the ``runserver`` argument::
 
 import os
 import re
+import string
 import sys
 from collections import defaultdict
 from urllib.parse import urlparse
@@ -90,6 +91,7 @@ def start_test_site(debug=False):
     r = static.File(root_dir)
     r.putChild(b"redirect", util.Redirect(b"/redirected"))
     r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
+    r.putChild(b"random", static.Data(string.ascii_letters.encode("utf8") * 2**14, "text/plain"))
 
     port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1")
     if debug:
@@ -117,8 +119,12 @@ class CrawlerRun(object):
         self.port = start_test_site()
         self.portno = self.port.getHost().port
 
-        start_urls = [self.geturl("/"), self.geturl("/redirect"),
-                      self.geturl("/redirect")]  # a duplicate
+        start_urls = [
+            self.geturl("/"),
+            self.geturl("/redirect"),
+            self.geturl("/redirect"),  # duplicate
+            self.geturl("/random"),
+        ]
 
         for name, signal in vars(signals).items():
             if not name.startswith('_'):
@@ -190,7 +196,7 @@ class EngineTest(unittest.TestCase):
             self.run = CrawlerRun(spider)
             yield self.run.run()
             self._assert_visited_urls()
-            self._assert_scheduled_requests(urls_to_visit=8)
+            self._assert_scheduled_requests(urls_to_visit=9)
             self._assert_downloaded_responses()
             self._assert_scraped_items()
             self._assert_signals_caught()
@@ -200,7 +206,7 @@ class EngineTest(unittest.TestCase):
     def test_crawler_dupefilter(self):
         self.run = CrawlerRun(TestDupeFilterSpider)
         yield self.run.run()
-        self._assert_scheduled_requests(urls_to_visit=7)
+        self._assert_scheduled_requests(urls_to_visit=8)
         self._assert_dropped_requests()
 
     @defer.inlineCallbacks
@@ -237,8 +243,8 @@ class EngineTest(unittest.TestCase):
 
     def _assert_downloaded_responses(self):
         # response tests
-        self.assertEqual(8, len(self.run.respplug))
-        self.assertEqual(8, len(self.run.reqreached))
+        self.assertEqual(9, len(self.run.respplug))
+        self.assertEqual(9, len(self.run.reqreached))
 
         for response, _ in self.run.respplug:
             if self.run.getpath(response.url) == '/item999.html':
@@ -272,7 +278,7 @@ class EngineTest(unittest.TestCase):
                 self.assertEqual('200', item['price'])
 
     def _assert_bytes_received(self):
-        self.assertEqual(8, len(self.run.bytes))
+        self.assertEqual(9, len(self.run.bytes))
         for request, data in self.run.bytes.items():
             joined_data = b"".join(data)
             if self.run.getpath(request.url) == "/":
@@ -306,6 +312,8 @@ class EngineTest(unittest.TestCase):
                     b"  </body>\n"
                     b"</html>\n"
                 )
+            elif self.run.getpath(request.url) == "/random":
+                self.assertTrue(len(data) > 1)  # signal was fired multiple times
 
     def _assert_signals_caught(self):
         assert signals.engine_started in self.run.signals_caught

From a499f38b14d16338d20084c0dcb24528a1f1f22f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 29 Jan 2020 14:35:17 -0300
Subject: [PATCH 2599/2974] Remove object parent class

---
 scrapy/core/downloader/handlers/http11.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index c53c9bb2d..6f1bd9ad6 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -268,7 +268,7 @@ class ScrapyProxyAgent(Agent):
         )
 
 
-class ScrapyAgent(object):
+class ScrapyAgent:
 
     _Agent = Agent
     _ProxyAgent = ScrapyProxyAgent
@@ -438,7 +438,7 @@ class ScrapyAgent(object):
 
 
 @implementer(IBodyProducer)
-class _RequestBodyProducer(object):
+class _RequestBodyProducer:
 
     def __init__(self, body):
         self.body = body

From 6f02a8dccb95373f22bac18c08d9fda8169dcb02 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 29 Jan 2020 14:53:23 -0300
Subject: [PATCH 2600/2974] Add source parameter to bytes_received signal

---
 docs/topics/signals.rst                   | 12 ++++++++----
 scrapy/core/downloader/handlers/http11.py | 18 +++++++++++++-----
 scrapy/core/downloader/handlers/s3.py     |  1 +
 tests/test_downloader_handlers.py         |  3 +++
 tests/test_engine.py                      |  5 ++++-
 5 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index f490911f3..3a15bf95c 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -162,11 +162,11 @@ bytes_received
 --------------
 
 .. signal:: bytes_received
-.. function:: bytes_received(data, request, spider)
+.. function:: bytes_received(data, request, spider, source)
 
-    Sent by the HTTP 1.1 download handler when a group of bytes is
-    received for a specific request. This signal might be fired
-    multiple times for the same request.
+    Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
+    received for a specific request. This signal might be fired multiple
+    times for the same request, with partial data each time.
 
     This signal does not support returning deferreds from its handlers.
 
@@ -179,6 +179,10 @@ bytes_received
     :param spider: the spider associated with the response
     :type spider: :class:`~scrapy.spiders.Spider` object
 
+    :param source: a string to identify which handler sent the signal
+        (current values could be "http11" or "s3")
+    :type source: :class:`str` object
+
 spider_closed
 -------------
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 6f1bd9ad6..49c9eacac 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -32,8 +32,9 @@ logger = logging.getLogger(__name__)
 class HTTP11DownloadHandler:
     lazy = False
 
-    def __init__(self, settings, crawler=None):
+    def __init__(self, settings, crawler=None, source="http11"):
         self.crawler = crawler
+        self.source = source
         self._pool = HTTPConnectionPool(reactor, persistent=True)
         self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self._pool._factory.noisy = False
@@ -67,8 +68,8 @@ class HTTP11DownloadHandler:
         self._disconnect_timeout = 1
 
     @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler.settings, crawler)
+    def from_crawler(cls, crawler, **kwargs):
+        return cls(crawler.settings, crawler, **kwargs)
 
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
@@ -79,6 +80,7 @@ class HTTP11DownloadHandler:
             warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
             fail_on_dataloss=self._fail_on_dataloss,
             crawler=self.crawler,
+            source=self.source,
         )
         return agent.download_request(request)
 
@@ -275,7 +277,7 @@ class ScrapyAgent:
     _TunnelingAgent = TunnelingAgent
 
     def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
-                 maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None):
+                 maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None, source=None):
         self._contextFactory = contextFactory
         self._connectTimeout = connectTimeout
         self._bindAddress = bindAddress
@@ -285,6 +287,7 @@ class ScrapyAgent:
         self._fail_on_dataloss = fail_on_dataloss
         self._txresponse = None
         self._crawler = crawler
+        self._source = source
 
     def _get_agent(self, request, timeout):
         bindaddress = request.meta.get('bindaddress') or self._bindAddress
@@ -421,6 +424,7 @@ class ScrapyAgent:
                 warnsize,
                 fail_on_dataloss,
                 self._crawler,
+                self._source,
             )
         )
 
@@ -457,7 +461,9 @@ class _RequestBodyProducer:
 
 class _ResponseReader(protocol.Protocol):
 
-    def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler):
+    def __init__(
+        self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler, source
+    ):
         self._finished = finished
         self._txresponse = txresponse
         self._request = request
@@ -469,6 +475,7 @@ class _ResponseReader(protocol.Protocol):
         self._reached_warnsize = False
         self._bytes_received = 0
         self._crawler = crawler
+        self._source = source
 
     def dataReceived(self, bodyBytes):
         # This maybe called several times after cancel was called with buffered data.
@@ -483,6 +490,7 @@ class _ResponseReader(protocol.Protocol):
             data=bodyBytes,
             request=self._request,
             spider=self._crawler.spider,
+            source=self._source,
         )
 
         if self._maxsize and self._bytes_received > self._maxsize:
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 40a1fa48e..2366b6394 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -73,6 +73,7 @@ class S3DownloadHandler:
             objcls=httpdownloadhandler,
             settings=settings,
             crawler=crawler,
+            source="s3",
         )
         self._download_http = _http_handler.download_request
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 8d95d7cac..22a813647 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -730,6 +730,9 @@ class Http11ProxyTestCase(HttpProxyTestCase):
 
 class HttpDownloadHandlerMock:
 
+    def __init__(self, *args, **kwargs):
+        pass
+
     def download_request(self, request, spider):
         return request
 
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 3c5cc403b..c83a23b55 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -112,6 +112,7 @@ class CrawlerRun(object):
         self.itemerror = []
         self.itemresp = []
         self.bytes = defaultdict(lambda: list())
+        self.bytes_source = set()
         self.signals_caught = {}
         self.spider_class = spider_class
 
@@ -165,8 +166,9 @@ class CrawlerRun(object):
     def item_scraped(self, item, spider, response):
         self.itemresp.append((item, response))
 
-    def bytes_received(self, data, request, spider):
+    def bytes_received(self, data, request, spider, source):
         self.bytes[request].append(data)
+        self.bytes_source.add(source)
 
     def request_scheduled(self, request, spider):
         self.reqplug.append((request, spider))
@@ -279,6 +281,7 @@ class EngineTest(unittest.TestCase):
 
     def _assert_bytes_received(self):
         self.assertEqual(9, len(self.run.bytes))
+        self.assertEqual(self.run.bytes_source, set(["http11"]))
         for request, data in self.run.bytes.items():
             joined_data = b"".join(data)
             if self.run.getpath(request.url) == "/":

From cc825c21deaa56875f2abf1b30b53abb60c566c7 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 30 Jan 2020 16:17:06 +0500
Subject: [PATCH 2601/2974] Test returning items from an async def callback.

---
 tests/spiders.py    | 11 +++++++++++
 tests/test_crawl.py | 19 ++++++++++++++++++-
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index e4f2d5474..3b1ee94b8 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -106,6 +106,17 @@ class AsyncDefAsyncioSpider(SimpleSpider):
         self.logger.info("Got response %d" % status)
 
 
+class AsyncDefAsyncioReturnSpider(SimpleSpider):
+
+    name = 'asyncdef_asyncio_return'
+
+    async def parse(self, response):
+        await asyncio.sleep(0.2)
+        status = await get_from_asyncio_queue(response.status)
+        self.logger.info("Got response %d" % status)
+        return [{'id': 1}, {'id': 2}]
+
+
 class ItemSpider(FollowAllSpider):
 
     name = 'item'
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 99b887ff6..85005eba4 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -6,13 +6,14 @@ from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 
+from scrapy import signals
 from scrapy.crawler import CrawlerRunner
 from scrapy.http import Request
 from scrapy.utils.python import to_unicode
 from tests.mockserver import MockServer
 from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
                            SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback,
-                           AsyncDefSpider, AsyncDefAsyncioSpider)
+                           AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider)
 
 
 class CrawlTestCase(TestCase):
@@ -326,3 +327,19 @@ with multiples lines
         with LogCapture() as log:
             yield runner.join()
         self.assertIn("Got response 200", str(log))
+
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncio_parse_list(self):
+        items = []
+
+        def _on_item_scraped(item):
+            items.append(item)
+
+        crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSpider)
+        crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+        with LogCapture() as log:
+            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        self.assertIn("Got response 200", str(log))
+        self.assertIn({'id': 1}, items)
+        self.assertIn({'id': 2}, items)

From 0641ba0faa97498ca5bee39c4e8faec58d5f0522 Mon Sep 17 00:00:00 2001
From: faizan2700 <syedfaizan824@gmail.com>
Date: Sun, 2 Feb 2020 16:54:22 +0530
Subject: [PATCH 2602/2974] SCRAPY_CHECK will be set while running contact

---
 scrapy/commands/check.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index 9d4437a47..09a76ca7a 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -78,19 +78,19 @@ class Command(ScrapyCommand):
                 elif tested_methods:
                     self.crawler_process.crawl(spidercls)
 
-        # start checks
-        if opts.list:
-            for spider, methods in sorted(contract_reqs.items()):
-                if not methods and not opts.verbose:
-                    continue
-                print(spider)
-                for method in sorted(methods):
-                    print('  * %s' % method)
-        else:
-            start = time.time()
-            self.crawler_process.start()
-            stop = time.time()
+            # start checks
+            if opts.list:
+                for spider, methods in sorted(contract_reqs.items()):
+                    if not methods and not opts.verbose:
+                        continue
+                    print(spider)
+                    for method in sorted(methods):
+                        print('  * %s' % method)
+            else:
+                start = time.time()
+                self.crawler_process.start()
+                stop = time.time()
 
-            result.printErrors()
-            result.printSummary(start, stop)
-            self.exitcode = int(not result.wasSuccessful())
+                result.printErrors()
+                result.printSummary(start, stop)
+                self.exitcode = int(not result.wasSuccessful())

From a2ae380efcaa5a3419a4f6a35541ae0fb71a2e7f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 3 Feb 2020 13:23:52 -0300
Subject: [PATCH 2603/2974] Remove unnecessary commas

---
 scrapy/resolver.py                 | 2 +-
 tests/CrawlerProcess/ip_address.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 554a3a14d..f69894b1e 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -29,7 +29,7 @@ class CachingThreadedResolver(ThreadedResolver):
             cache_size = 0
         return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT'))
 
-    def install_on_reactor(self,):
+    def install_on_reactor(self):
         self.reactor.installResolver(self)
 
     def getHostByName(self, name, timeout=None):
diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.py
index 6b069cc90..949e97172 100644
--- a/tests/CrawlerProcess/ip_address.py
+++ b/tests/CrawlerProcess/ip_address.py
@@ -21,7 +21,7 @@ class MockThreadedResolver(ThreadedResolver):
     def from_crawler(cls, crawler, reactor):
         return cls(reactor)
 
-    def install_on_reactor(self,):
+    def install_on_reactor(self):
         self.reactor.installResolver(self)
 
     def getHostByName(self, name, timeout=None):

From bb8f7dc609382153df79774ad9d8f6d33d064279 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 3 Feb 2020 14:50:14 -0300
Subject: [PATCH 2604/2974] Mock DNS server

---
 tests/mockserver.py | 90 ++++++++++++++++++++++++++++++++++-----------
 1 file changed, 68 insertions(+), 22 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index a45277db9..585741f1b 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,3 +1,4 @@
+import argparse
 import json
 import os
 import random
@@ -6,18 +7,19 @@ from subprocess import Popen, PIPE
 from urllib.parse import urlencode
 
 from OpenSSL import SSL
-from twisted.web.server import Site, NOT_DONE_YET
-from twisted.web.resource import Resource
+from twisted.internet import defer, reactor, ssl
+from twisted.internet.task import deferLater
+from twisted.names import dns, error
+from twisted.names.server import DNSServerFactory
+from twisted.web.resource import EncodingResourceWrapper, Resource
+from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site
 from twisted.web.static import File
 from twisted.web.test.test_webclient import PayloadResource
-from twisted.web.server import GzipEncoderFactory
-from twisted.web.resource import EncodingResourceWrapper
 from twisted.web.util import redirectTo
-from twisted.internet import reactor, ssl
-from twisted.internet.task import deferLater
 
 from scrapy.utils.python import to_bytes, to_unicode
 from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3
+from scrapy.utils.test import get_testenv
 
 
 def getarg(request, name, default=None, type=None):
@@ -198,12 +200,10 @@ class Root(Resource):
         return b'Scrapy mock HTTP server\n'
 
 
-class MockServer():
+class MockServer:
 
     def __enter__(self):
-        from scrapy.utils.test import get_testenv
-
-        self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'],
+        self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'http'],
                           stdout=PIPE, env=get_testenv())
         http_address = self.proc.stdout.readline().strip().decode('ascii')
         https_address = self.proc.stdout.readline().strip().decode('ascii')
@@ -224,6 +224,37 @@ class MockServer():
         return host + path
 
 
+class MockDNSResolver:
+    """
+    Implements twisted.internet.interfaces.IResolver partially
+    """
+
+    def _resolve(self, name):
+        record = dns.Record_A(address=b"127.0.0.1")
+        answer = dns.RRHeader(name=name, payload=record)
+        return [answer], [], []
+
+    def query(self, query, timeout=None):
+        if query.type == dns.A:
+            return defer.succeed(self._resolve(query.name.name))
+        return defer.fail(error.DomainError())
+
+    def lookupAllRecords(self, name, timeout=None):
+        return defer.succeed(self._resolve(name))
+
+
+class MockDNSServer():
+
+    def __enter__(self):
+        self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', 'dns'],
+                          stdout=PIPE, env=get_testenv())
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.proc.kill()
+        self.proc.communicate()
+
+
 def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):
     factory = ssl.DefaultOpenSSLContextFactory(
          os.path.join(os.path.dirname(__file__), keyfile),
@@ -238,19 +269,34 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c
 
 
 if __name__ == "__main__":
-    root = Root()
-    factory = Site(root)
-    httpPort = reactor.listenTCP(0, factory)
-    contextFactory = ssl_context_factory()
-    httpsPort = reactor.listenSSL(0, factory, contextFactory)
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-t", "--type", type=str, choices=("http", "dns"), default="http")
+    args = parser.parse_args()
 
-    def print_listening():
-        httpHost = httpPort.getHost()
-        httpsHost = httpsPort.getHost()
-        httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port)
-        httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port)
-        print(httpAddress)
-        print(httpsAddress)
+    if args.type == "http":
+        root = Root()
+        factory = Site(root)
+        httpPort = reactor.listenTCP(0, factory)
+        contextFactory = ssl_context_factory()
+        httpsPort = reactor.listenSSL(0, factory, contextFactory)
+
+        def print_listening():
+            httpHost = httpPort.getHost()
+            httpsHost = httpsPort.getHost()
+            httpAddress = "http://%s:%d" % (httpHost.host, httpHost.port)
+            httpsAddress = "https://%s:%d" % (httpsHost.host, httpsHost.port)
+            print(httpAddress)
+            print(httpsAddress)
+
+    elif args.type == "dns":
+        clients = [MockDNSResolver()]
+        factory = DNSServerFactory(clients=clients)
+        protocol = dns.DNSDatagramProtocol(controller=factory)
+        reactor.listenUDP(10053, protocol)
+        reactor.listenTCP(10053, factory)
+
+        def print_listening():
+            print("DNS server running on port 10053")
 
     reactor.callWhenRunning(print_listening)
     reactor.run()

From 4851efdfb0885a40a44a2834c6c69d0104326801 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 3 Feb 2020 14:50:54 -0300
Subject: [PATCH 2605/2974] Flake8 adjustments

---
 tests/mockserver.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 585741f1b..67139534e 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -257,9 +257,9 @@ class MockDNSServer():
 
 def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):
     factory = ssl.DefaultOpenSSLContextFactory(
-         os.path.join(os.path.dirname(__file__), keyfile),
-         os.path.join(os.path.dirname(__file__), certfile),
-         )
+        os.path.join(os.path.dirname(__file__), keyfile),
+        os.path.join(os.path.dirname(__file__), certfile),
+    )
     if cipher_string:
         ctx = factory.getContext()
         # disabling TLS1.2+ because it unconditionally enables some strong ciphers

From e0ef8ad2d6f958de6ce04cd7756e142efeb1a6a2 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 3 Feb 2020 15:52:15 -0300
Subject: [PATCH 2606/2974] CrawlerRunner test for Response.ip_address

---
 tests/CrawlerProcess/ip_address.py | 51 ------------------------------
 tests/CrawlerRunner/ip_address.py  | 37 ++++++++++++++++++++++
 tests/mockserver.py                | 11 ++++---
 tests/test_crawler.py              | 20 ++++++++----
 4 files changed, 57 insertions(+), 62 deletions(-)
 delete mode 100644 tests/CrawlerProcess/ip_address.py
 create mode 100644 tests/CrawlerRunner/ip_address.py

diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.py
deleted file mode 100644
index 949e97172..000000000
--- a/tests/CrawlerProcess/ip_address.py
+++ /dev/null
@@ -1,51 +0,0 @@
-from urllib.parse import urlparse
-
-from twisted.internet import defer
-from twisted.internet.base import ThreadedResolver
-from twisted.internet.interfaces import IResolverSimple
-from zope.interface.declarations import implementer
-
-from scrapy import Spider, Request
-from scrapy.crawler import CrawlerProcess
-
-from tests.mockserver import MockServer
-
-
-@implementer(IResolverSimple)
-class MockThreadedResolver(ThreadedResolver):
-    """
-    Resolves all names to localhost
-    """
-
-    @classmethod
-    def from_crawler(cls, crawler, reactor):
-        return cls(reactor)
-
-    def install_on_reactor(self):
-        self.reactor.installResolver(self)
-
-    def getHostByName(self, name, timeout=None):
-        return defer.succeed("127.0.0.1")
-
-
-class LocalhostSpider(Spider):
-    name = "localhost_spider"
-
-    def start_requests(self):
-        yield Request(self.url)
-
-    def parse(self, response):
-        netloc = urlparse(response.url).netloc
-        self.logger.info("Host: %s" % netloc.split(":")[0])
-        self.logger.info("Type: %s" % type(response.ip_address))
-        self.logger.info("IP address: %s" % response.ip_address)
-
-
-with MockServer() as mockserver:
-    settings = {"DNS_RESOLVER": __name__ + ".MockThreadedResolver"}
-    process = CrawlerProcess(settings)
-
-    port = urlparse(mockserver.http_address).port
-    url = "http://not.a.real.domain:{port}/echo?body=test".format(port=port)
-    process.crawl(LocalhostSpider, url=url)
-    process.start()
diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py
new file mode 100644
index 000000000..5a71536d8
--- /dev/null
+++ b/tests/CrawlerRunner/ip_address.py
@@ -0,0 +1,37 @@
+from urllib.parse import urlparse
+
+from twisted.internet import reactor
+from twisted.names.client import createResolver
+
+from scrapy import Spider, Request
+from scrapy.crawler import CrawlerRunner
+from scrapy.utils.log import configure_logging
+
+from tests.mockserver import MockServer, MockDNSServer
+
+
+class LocalhostSpider(Spider):
+    name = "localhost_spider"
+
+    def start_requests(self):
+        yield Request(self.url)
+
+    def parse(self, response):
+        netloc = urlparse(response.url).netloc
+        self.logger.info("Host: %s" % netloc.split(":")[0])
+        self.logger.info("Type: %s" % type(response.ip_address))
+        self.logger.info("IP address: %s" % response.ip_address)
+
+
+with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server:
+    port = urlparse(mock_http_server.http_address).port
+    url = "http://not.a.real.domain:{port}/echo".format(port=port)
+
+    servers = [(mock_dns_server.host, mock_dns_server.port)]
+    reactor.installResolver(createResolver(servers=servers))
+
+    configure_logging()
+    runner = CrawlerRunner()
+    d = runner.crawl(LocalhostSpider, url=url)
+    d.addBoth(lambda _: reactor.stop())
+    reactor.run()
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 67139534e..08a81418c 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -246,8 +246,11 @@ class MockDNSResolver:
 class MockDNSServer():
 
     def __enter__(self):
-        self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', 'dns'],
+        self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'],
                           stdout=PIPE, env=get_testenv())
+        host, port = self.proc.stdout.readline().strip().decode('ascii').split(":")
+        self.host = host
+        self.port = int(port)
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
@@ -292,11 +295,11 @@ if __name__ == "__main__":
         clients = [MockDNSResolver()]
         factory = DNSServerFactory(clients=clients)
         protocol = dns.DNSDatagramProtocol(controller=factory)
-        reactor.listenUDP(10053, protocol)
-        reactor.listenTCP(10053, factory)
+        listener = reactor.listenUDP(0, protocol)
 
         def print_listening():
-            print("DNS server running on port 10053")
+            host = listener.getHost()
+            print("%s:%s" % (host.host, host.port))
 
     reactor.callWhenRunning(print_listening)
     reactor.run()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index dfc1cf448..5d381c368 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -281,9 +281,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
             self.assertNotIn("Asyncio reactor is installed", str(log))
 
 
-class CrawlerProcessSubprocess(unittest.TestCase):
-    script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
-
+class ScriptRunnerMixin:
     def run_script(self, script_name):
         script_path = os.path.join(self.script_dir, script_name)
         args = (sys.executable, script_path)
@@ -292,6 +290,10 @@ class CrawlerProcessSubprocess(unittest.TestCase):
         stdout, stderr = p.communicate()
         return stderr.decode('utf-8')
 
+
+class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
+    script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
+
     def test_simple(self):
         log = self.run_script('simple.py')
         self.assertIn('Spider closed (finished)', log)
@@ -325,9 +327,13 @@ class CrawlerProcessSubprocess(unittest.TestCase):
             "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
         ]))
 
+
+class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
+    script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')
+
     def test_response_ip_address(self):
         log = self.run_script("ip_address.py")
-        self.assertIn("Spider closed (finished)", log)
-        self.assertIn("Host: not.a.real.domain", log)
-        self.assertIn("Type: <class 'ipaddress.IPv4Address'>", log)
-        self.assertIn("IP address: 127.0.0.1", log)
+        self.assertIn("INFO: Spider closed (finished)", log)
+        self.assertIn("INFO: Host: not.a.real.domain", log)
+        self.assertIn("INFO: Type: <class 'ipaddress.IPv4Address'>", log)
+        self.assertIn("INFO: IP address: 127.0.0.1", log)

From 13670f0397ba8dcec3dceb1852bad5751406d19d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 3 Feb 2020 16:16:43 -0300
Subject: [PATCH 2607/2974] Ignore tests/CrawlerRunner directory

---
 conftest.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/conftest.py b/conftest.py
index c0de09909..55294feca 100644
--- a/conftest.py
+++ b/conftest.py
@@ -11,7 +11,8 @@ collect_ignore = [
     # not a test, but looks like a test
     "scrapy/utils/testsite.py",
     # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
-    *_py_files("tests/CrawlerProcess")
+    *_py_files("tests/CrawlerProcess"),
+    *_py_files("tests/CrawlerRunner"),
 ]
 
 for line in open('tests/ignores.txt'):

From ad70497416527c3d882a64f7803e73155f3fa1da Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 4 Feb 2020 13:30:13 -0300
Subject: [PATCH 2608/2974] Remove unnecessary parentheses in class definition

---
 tests/mockserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 08a81418c..30d9bc0e8 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -243,7 +243,7 @@ class MockDNSResolver:
         return defer.succeed(self._resolve(name))
 
 
-class MockDNSServer():
+class MockDNSServer:
 
     def __enter__(self):
         self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'],

From fbea370c58c1d82b52fd9c1f7d3a6cee94477c7a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 5 Feb 2020 01:35:13 -0300
Subject: [PATCH 2609/2974] Rename function parameter

---
 scrapy/core/spidermw.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 8b36cbb04..dd9b3c376 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -59,12 +59,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
                     return scrape_func(Failure(), request, spider)
             return scrape_func(response, request, spider)
 
-        def _evaluate_iterable(iterable, method_index, recover_to):
+        def _evaluate_iterable(iterable, exception_processor_index, recover_to):
             try:
                 for r in iterable:
                     yield r
             except Exception as ex:
-                exception_result = process_spider_exception(Failure(ex), method_index)
+                exception_result = process_spider_exception(Failure(ex), exception_processor_index)
                 if isinstance(exception_result, Failure):
                     raise
                 recover_to.extend(exception_result)

From 898bc00811aac9d3e38d1863b95a10c2e8effb02 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Wed, 5 Feb 2020 11:31:27 +0000
Subject: [PATCH 2610/2974] new signal

---
 scrapy/signals.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/signals.py b/scrapy/signals.py
index 6b9125302..cd7ed7fb1 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -14,6 +14,7 @@ spider_error = object()
 request_scheduled = object()
 request_dropped = object()
 request_reached_downloader = object()
+request_left_downloader = object()
 response_received = object()
 response_downloaded = object()
 item_scraped = object()

From ae04174884eeb777d7b3caceed52bf522944ceb1 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Wed, 5 Feb 2020 11:32:31 +0000
Subject: [PATCH 2611/2974] emit new signal

---
 scrapy/core/downloader/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 157dc3418..5a2fdadf5 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -181,6 +181,9 @@ class Downloader(object):
         def finish_transferring(_):
             slot.transferring.remove(request)
             self._process_queue(spider, slot)
+            self.signals.send_catch_log(signal=signals.request_left_downloader,
+                                        request=request,
+                                        spider=spider)
             return _
 
         return dfd.addBoth(finish_transferring)

From 9916f6e556f9d4a41ea86d4a73687af1a40e43ba Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Wed, 5 Feb 2020 11:32:54 +0000
Subject: [PATCH 2612/2974] tests for new signal

---
 tests/test_request_left.py | 59 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 tests/test_request_left.py

diff --git a/tests/test_request_left.py b/tests/test_request_left.py
new file mode 100644
index 000000000..ddeca0499
--- /dev/null
+++ b/tests/test_request_left.py
@@ -0,0 +1,59 @@
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+from scrapy.signals import request_left_downloader
+from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
+from tests.mockserver import MockServer
+
+class SignalCatcherSpider(Spider):
+    name = 'signal_catcher'
+
+    def __init__(self, crawler, url, *args, **kwargs):
+        super(SignalCatcherSpider, self).__init__(*args, **kwargs)
+        crawler.signals.connect(self.on_response_download,
+                                signal=request_left_downloader)
+        self.catched_times = 0
+        self.start_urls = [url]
+
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        spider = cls(crawler, *args, **kwargs)
+        return spider
+
+    def on_response_download(self, request, spider):
+        self.catched_times = self.catched_times + 1
+
+
+class TestCatching(TestCase):
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_success(self):
+        crawler = get_crawler(SignalCatcherSpider)
+        yield crawler.crawl(self.mockserver.url("/status?n=200"))
+        self.assertEqual(crawler.spider.catched_times, 1)
+
+    @defer.inlineCallbacks
+    def test_timeout(self):
+        crawler = get_crawler(SignalCatcherSpider,
+                              {'DOWNLOAD_TIMEOUT': 0.1})
+        yield crawler.crawl(self.mockserver.url("/delay?n=0.2"))
+        self.assertEqual(crawler.spider.catched_times, 1)
+
+    @defer.inlineCallbacks
+    def test_disconnect(self):
+        crawler = get_crawler(SignalCatcherSpider)
+        yield crawler.crawl(self.mockserver.url("/drop"))
+        self.assertEqual(crawler.spider.catched_times, 1)
+
+    @defer.inlineCallbacks
+    def test_noconnect(self):
+        crawler = get_crawler(SignalCatcherSpider)
+        yield crawler.crawl('http://thereisdefinetelynosuchdomain.com')
+        self.assertEqual(crawler.spider.catched_times, 1)

From aab39f63412b4b7a0ae2713446859d6d8103e5f7 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Wed, 5 Feb 2020 11:35:03 +0000
Subject: [PATCH 2613/2974] docummentation for new signal

---
 docs/topics/signals.rst | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 3f29aa323..7fa5bc030 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -295,6 +295,23 @@ request_reached_downloader
     :param spider: the spider that yielded the request
     :type spider: :class:`~scrapy.spiders.Spider` object
 
+request_left_downloader
+---------------------------
+
+.. signal:: request_left_downloader
+.. function:: request_left_downloader(request, spider)
+
+    Sent when a :class:`~scrapy.http.Request` left downloader even in case of
+    failure.
+
+    The signal does not support returning deferreds from their handlers.
+
+    :param request: the request that reached downloader
+    :type request: :class:`~scrapy.http.Request` object
+
+    :param spider: the spider that yielded the request
+    :type spider: :class:`~scrapy.spiders.Spider` object
+
 response_received
 -----------------
 

From 3769f75386104c1a3072894b302d3c3239ff8c37 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Wed, 5 Feb 2020 12:08:08 +0000
Subject: [PATCH 2614/2974] pep8 E302

---
 tests/test_request_left.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_request_left.py b/tests/test_request_left.py
index ddeca0499..5d271190d 100644
--- a/tests/test_request_left.py
+++ b/tests/test_request_left.py
@@ -5,6 +5,7 @@ from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
 
+
 class SignalCatcherSpider(Spider):
     name = 'signal_catcher'
 

From 11941c324431e1f8822e64fb03d163b9e721eaa5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 5 Feb 2020 13:27:54 -0300
Subject: [PATCH 2615/2974] Remove elusive six occurrence from tox.ini

---
 tox.ini | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index b62100026..b1babc7fd 100644
--- a/tox.ini
+++ b/tox.ini
@@ -56,7 +56,6 @@ deps =
     pyOpenSSL==16.2.0
     queuelib==1.4.2
     service_identity==16.0.0
-    six==1.10.0
     Twisted==17.9.0
     w3lib==1.17.0
     zope.interface==4.1.3

From c2cca368213013f9dc9d4569b926da4d213f20a5 Mon Sep 17 00:00:00 2001
From: Respawnz <47511522+Respawnz@users.noreply.github.com>
Date: Thu, 6 Feb 2020 05:39:15 +0800
Subject: [PATCH 2616/2974] typo

---
 docs/topics/developer-tools.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index bf14643be..1fedf91df 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -132,7 +132,7 @@ a use case:
 
 Say you want to find the ``Next`` button on the page. Type ``Next`` into the 
 search bar on the top right of the `Inspector`. You should get two results. 
-The first is a ``li`` tag with the ``class="text"``, the second the text 
+The first is a ``li`` tag with the ``class="next"``, the second the text 
 of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``.
 If you hover over the tag, you'll see the button highlighted. From here
 we could easily create a :ref:`Link Extractor <topics-link-extractors>` to 

From 576663e5a778646aa4ef870641fa677ea94d21f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 6 Feb 2020 10:43:20 +0100
Subject: [PATCH 2617/2974] Make METAREFRESH_IGNORE_TAGS an empty list by
 default

---
 docs/topics/downloader-middleware.rst       |  2 +-
 scrapy/settings/default_settings.py         |  2 +-
 tests/test_downloadermiddleware_redirect.py | 16 +++++++++-------
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 8a760e53b..3ec6e0c17 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -868,7 +868,7 @@ Whether the Meta Refresh middleware will be enabled.
 METAREFRESH_IGNORE_TAGS
 ^^^^^^^^^^^^^^^^^^^^^^^
 
-Default: ``['script', 'noscript']``
+Default: ``[]``
 
 Meta tags within these tags are ignored.
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index c10dc1a1c..1a7d35b13 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -225,7 +225,7 @@ MEMUSAGE_NOTIFY_MAIL = []
 MEMUSAGE_WARNING_MB = 0
 
 METAREFRESH_ENABLED = True
-METAREFRESH_IGNORE_TAGS = ['script', 'noscript']
+METAREFRESH_IGNORE_TAGS = []
 METAREFRESH_MAXDELAY = 100
 
 NEWSPIDER_MODULE = ''
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index e7faf14a7..e0f145d0e 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -300,19 +300,21 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
         body = ('''<noscript><meta http-equiv="refresh" '''
                 '''content="0;URL='http://example.org/newpage'"></noscript>''')
         rsp = HtmlResponse(req.url, body=body.encode())
-        response = self.mw.process_response(req, rsp, self.spider)
-        assert isinstance(response, Response)
+        req2 = self.mw.process_response(req, rsp, self.spider)
+        assert isinstance(req2, Request)
+        self.assertEqual(req2.url, 'http://example.org/newpage')
 
-    def test_ignore_tags_empty_list(self):
-        crawler = get_crawler(Spider, {'METAREFRESH_IGNORE_TAGS': []})
+    def test_ignore_tags_1_x_list(self):
+        """Test that Scrapy 1.x behavior remains possible"""
+        settings = {'METAREFRESH_IGNORE_TAGS': ['script', 'noscript']}
+        crawler = get_crawler(Spider, settings)
         mw = MetaRefreshMiddleware.from_crawler(crawler)
         req = Request(url='http://example.org')
         body = ('''<noscript><meta http-equiv="refresh" '''
                 '''content="0;URL='http://example.org/newpage'"></noscript>''')
         rsp = HtmlResponse(req.url, body=body.encode())
-        req2 = mw.process_response(req, rsp, self.spider)
-        assert isinstance(req2, Request)
-        self.assertEqual(req2.url, 'http://example.org/newpage')
+        response = mw.process_response(req, rsp, self.spider)
+        assert isinstance(response, Response)
 
 if __name__ == "__main__":
     unittest.main()

From 6733f4d976150e0e5352d4ae9697880ae60ad638 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Thu, 6 Feb 2020 18:40:42 +0500
Subject: [PATCH 2618/2974] Update docs/topics/signals.rst

Co-Authored-By: elacuesta <elacuesta@users.noreply.github.com>
---
 docs/topics/signals.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 7fa5bc030..47be6b603 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -301,7 +301,7 @@ request_left_downloader
 .. signal:: request_left_downloader
 .. function:: request_left_downloader(request, spider)
 
-    Sent when a :class:`~scrapy.http.Request` left downloader even in case of
+    Sent when a :class:`~scrapy.http.Request` leaves the downloader even in case of
     failure.
 
     The signal does not support returning deferreds from their handlers.

From 4a91a5427df4846ed9fa11612cfeb9e31f34a1c8 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Thu, 6 Feb 2020 13:44:51 +0000
Subject: [PATCH 2619/2974] fix typo

---
 tests/test_request_left.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/test_request_left.py b/tests/test_request_left.py
index 5d271190d..8256d1c92 100644
--- a/tests/test_request_left.py
+++ b/tests/test_request_left.py
@@ -13,7 +13,7 @@ class SignalCatcherSpider(Spider):
         super(SignalCatcherSpider, self).__init__(*args, **kwargs)
         crawler.signals.connect(self.on_response_download,
                                 signal=request_left_downloader)
-        self.catched_times = 0
+        self.caught_times = 0
         self.start_urls = [url]
 
     @classmethod
@@ -22,7 +22,7 @@ class SignalCatcherSpider(Spider):
         return spider
 
     def on_response_download(self, request, spider):
-        self.catched_times = self.catched_times + 1
+        self.caught_times = self.caught_times + 1
 
 
 class TestCatching(TestCase):
@@ -38,23 +38,23 @@ class TestCatching(TestCase):
     def test_success(self):
         crawler = get_crawler(SignalCatcherSpider)
         yield crawler.crawl(self.mockserver.url("/status?n=200"))
-        self.assertEqual(crawler.spider.catched_times, 1)
+        self.assertEqual(crawler.spider.caught_times, 1)
 
     @defer.inlineCallbacks
     def test_timeout(self):
         crawler = get_crawler(SignalCatcherSpider,
                               {'DOWNLOAD_TIMEOUT': 0.1})
         yield crawler.crawl(self.mockserver.url("/delay?n=0.2"))
-        self.assertEqual(crawler.spider.catched_times, 1)
+        self.assertEqual(crawler.spider.caught_times, 1)
 
     @defer.inlineCallbacks
     def test_disconnect(self):
         crawler = get_crawler(SignalCatcherSpider)
         yield crawler.crawl(self.mockserver.url("/drop"))
-        self.assertEqual(crawler.spider.catched_times, 1)
+        self.assertEqual(crawler.spider.caught_times, 1)
 
     @defer.inlineCallbacks
     def test_noconnect(self):
         crawler = get_crawler(SignalCatcherSpider)
         yield crawler.crawl('http://thereisdefinetelynosuchdomain.com')
-        self.assertEqual(crawler.spider.catched_times, 1)
+        self.assertEqual(crawler.spider.caught_times, 1)

From 4be19e443e9c101a248c21509ae8000ce500d51a Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Thu, 6 Feb 2020 13:46:23 +0000
Subject: [PATCH 2620/2974] name signla catcher in accord with signal name

---
 tests/test_request_left.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_request_left.py b/tests/test_request_left.py
index 8256d1c92..5cfef8e7d 100644
--- a/tests/test_request_left.py
+++ b/tests/test_request_left.py
@@ -11,7 +11,7 @@ class SignalCatcherSpider(Spider):
 
     def __init__(self, crawler, url, *args, **kwargs):
         super(SignalCatcherSpider, self).__init__(*args, **kwargs)
-        crawler.signals.connect(self.on_response_download,
+        crawler.signals.connect(self.on_request_left,
                                 signal=request_left_downloader)
         self.caught_times = 0
         self.start_urls = [url]
@@ -21,7 +21,7 @@ class SignalCatcherSpider(Spider):
         spider = cls(crawler, *args, **kwargs)
         return spider
 
-    def on_response_download(self, request, spider):
+    def on_request_left(self, request, spider):
         self.caught_times = self.caught_times + 1
 
 

From 489ffcda5143a2ef28d4cbcf5418babd963f2b0f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 6 Feb 2020 22:39:00 +0500
Subject: [PATCH 2621/2974] Add a test for an async item_scraped handler.

---
 tests/test_signals.py | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 tests/test_signals.py

diff --git a/tests/test_signals.py b/tests/test_signals.py
new file mode 100644
index 000000000..001e798e5
--- /dev/null
+++ b/tests/test_signals.py
@@ -0,0 +1,39 @@
+from twisted.internet import defer
+from twisted.trial import unittest
+
+from scrapy import signals, Request, Spider
+from scrapy.utils.test import get_crawler
+
+from tests.mockserver import MockServer
+
+
+class ItemSpider(Spider):
+    name = 'itemspider'
+
+    def start_requests(self):
+        for _ in range(10):
+            yield Request(self.mockserver.url('/status?n=200'),
+                          dont_filter=True)
+
+    def parse(self, response):
+        return {'field': 42}
+
+
+class AsyncSignalTestCase(unittest.TestCase):
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self.items = []
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    async def _on_item_scraped(self, item):
+        self.items.append(item)
+
+    @defer.inlineCallbacks
+    def test_simple_pipeline(self):
+        crawler = get_crawler(ItemSpider)
+        crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertEqual(len(self.items), 10)

From 35dafef7f106dbf1c022d997b4e29e3eee84de7c Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Thu, 6 Feb 2020 14:42:34 -0300
Subject: [PATCH 2622/2974] Specify Twisted reactor (TWISTED_REACTOR setting)
 (#4294)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add the ability to install a specific reactor

* Add docs for the TWISTED_REACTOR setting

* Add tests for the TWISTED_REACTOR setting

* Update asyncio reactor test

* Ignore W503 globally

W503 is not PEP8-compliant:
https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b

* Line length adjustment

* Adjust asyncio reactor tests

* Merge ASYNCIO_ENABLED and TWISTED_REACTOR settings

* More docs about TWISTED_REACTOR

* Fix asyncio reactor test

* Docs: fix title

* Reword docs

* Check the TWISTED_REACTOR setting outside of the installing function

* Remove unrelated change

* Update scrapy/utils/log.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update docs/topics/settings.rst

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update docs/topics/settings.rst

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/faq.rst                                  | 11 +++++
 docs/topics/broad-crawls.rst                  |  7 +++
 docs/topics/settings.rst                      | 45 +++++++++---------
 pytest.ini                                    |  7 +--
 scrapy/crawler.py                             | 19 ++++----
 scrapy/settings/default_settings.py           |  4 +-
 scrapy/utils/asyncio.py                       | 17 -------
 scrapy/utils/defer.py                         |  4 +-
 scrapy/utils/log.py                           | 11 ++---
 scrapy/utils/reactor.py                       | 35 +++++++++++++-
 .../asyncio_enabled_no_reactor.py             |  3 +-
 .../CrawlerProcess/asyncio_enabled_reactor.py |  3 +-
 .../CrawlerProcess/twisted_reactor_asyncio.py | 13 +++++
 tests/CrawlerProcess/twisted_reactor_poll.py  | 13 +++++
 .../CrawlerProcess/twisted_reactor_select.py  | 13 +++++
 tests/test_commands.py                        | 10 ++--
 tests/test_crawler.py                         | 47 ++++++++++++++-----
 tests/test_utils_asyncio.py                   |  4 +-
 18 files changed, 182 insertions(+), 84 deletions(-)
 delete mode 100644 scrapy/utils/asyncio.py
 create mode 100644 tests/CrawlerProcess/twisted_reactor_asyncio.py
 create mode 100644 tests/CrawlerProcess/twisted_reactor_poll.py
 create mode 100644 tests/CrawlerProcess/twisted_reactor_select.py

diff --git a/docs/faq.rst b/docs/faq.rst
index b65908012..f72e4cf01 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -361,6 +361,17 @@ Note that by doing so, you lose the ability to set a specific timeout for DNS re
 (the value of the :setting:`DNS_TIMEOUT` setting is ignored).
 
 
+.. _faq-specific-reactor:
+
+How to deal with ``<class 'ValueError'>: filedescriptor out of range in select()`` exceptions?
+----------------------------------------------------------------------------------------------
+
+This issue `has been reported`_ to appear when running broad crawls in macOS, where the default
+Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a
+different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
+
+
+.. _has been reported: https://github.com/scrapy/scrapy/issues/2905
 .. _user agents: https://en.wikipedia.org/wiki/User_agent
 .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
 .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index 1ab08d949..4922694ee 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -211,3 +211,10 @@ If your broad crawl shows a high memory usage, in addition to :ref:`crawling in
 BFO order <broad-crawls-bfo>` and :ref:`lowering concurrency
 <broad-crawls-concurrency>` you should :ref:`debug your memory leaks
 <topics-leaks>`.
+
+
+Install a specific Twisted reactor
+==================================
+
+If the crawl is exceeding the system's capabilities, you might want to try
+installing a specific Twisted reactor, via the :setting:`TWISTED_REACTOR` setting.
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 4b770d249..fa63a5807 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -160,27 +160,6 @@ to any particular component. In that case the module of that component will be
 shown, typically an extension, middleware or pipeline. It also means that the
 component must be enabled in order for the setting to have any effect.
 
-.. setting:: ASYNCIO_REACTOR
-
-ASYNCIO_REACTOR
----------------
-
-Default: ``False``
-
-Whether to install and require the Twisted reactor that uses the asyncio loop.
-
-When this option is set to ``True``, Scrapy will require
-:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. It will
-install this reactor if no reactor is installed yet, such as when using the
-``scrapy`` script or :class:`~scrapy.crawler.CrawlerProcess`. If you are using
-:class:`~scrapy.crawler.CrawlerRunner`, you need to install the correct reactor
-manually. If a different reactor is installed outside Scrapy, it will raise an
-exception.
-
-The default value for this option is currently ``False`` to maintain backward
-compatibility and avoid possible problems caused by using a different Twisted
-reactor.
-
 .. setting:: AWS_ACCESS_KEY_ID
 
 AWS_ACCESS_KEY_ID
@@ -1463,6 +1442,30 @@ command.
 The project name must not conflict with the name of custom files or directories
 in the ``project`` subdirectory.
 
+.. setting:: TWISTED_REACTOR
+
+TWISTED_REACTOR
+---------------
+
+Default: ``None``
+
+Import path of a given Twisted reactor, for instance:
+:class:`twisted.internet.asyncioreactor.AsyncioSelectorReactor`.
+
+Scrapy will install this reactor if no other is installed yet, such as when
+the ``scrapy`` CLI program is invoked or when using the
+:class:`~scrapy.crawler.CrawlerProcess` class. If you are using the
+:class:`~scrapy.crawler.CrawlerRunner` class, you need to install the correct
+reactor manually. An exception will be raised if the installation fails.
+
+The default value for this option is currently ``None``, which means that Scrapy
+will not attempt to install any specific reactor, and the default one defined by
+Twisted for the current platform will be used. This is to maintain backward
+compatibility and avoid possible problems caused by using a non-default reactor.
+
+For additional information, please see
+:doc:`core/howto/choosing-reactor`.
+
 
 .. setting:: URLLENGTH_LIMIT
 
diff --git a/pytest.ini b/pytest.ini
index bae68cd3a..552829d4e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -21,6 +21,7 @@ twisted = 1
 markers =
     only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
 flake8-ignore =
+    W503
     # Files that are only meant to provide top-level imports are expected not
     # to use any of their imports:
     scrapy/core/downloader/handlers/http.py F401
@@ -109,7 +110,7 @@ flake8-ignore =
     # scrapy/spidermiddlewares
     scrapy/spidermiddlewares/httperror.py E501
     scrapy/spidermiddlewares/offsite.py E501
-    scrapy/spidermiddlewares/referer.py E501 E129 W503 W504
+    scrapy/spidermiddlewares/referer.py E501 E129 W504
     scrapy/spidermiddlewares/urllength.py E501
     # scrapy/spiders
     scrapy/spiders/__init__.py E501 E402
@@ -129,13 +130,13 @@ flake8-ignore =
     scrapy/utils/http.py F403 E226
     scrapy/utils/httpobj.py E501
     scrapy/utils/iterators.py E501 E701
-    scrapy/utils/log.py E128 W503
+    scrapy/utils/log.py E128 E501
     scrapy/utils/markup.py F403
     scrapy/utils/misc.py E501 E226
     scrapy/utils/multipart.py F403
     scrapy/utils/project.py E501
     scrapy/utils/python.py E501
-    scrapy/utils/reactor.py E226
+    scrapy/utils/reactor.py E226 E501
     scrapy/utils/reqser.py E501
     scrapy/utils/request.py E127 E501
     scrapy/utils/response.py E501 E128
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 35c6b7716..49b8e4511 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -13,7 +13,6 @@ from scrapy.extension import ExtensionManager
 from scrapy.interfaces import ISpiderLoader
 from scrapy.settings import overridden_settings, Settings
 from scrapy.signalmanager import SignalManager
-from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installed
 from scrapy.utils.log import (
     configure_logging,
     get_scrapy_root_handler,
@@ -23,6 +22,7 @@ from scrapy.utils.log import (
 )
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
+from scrapy.utils.reactor import install_reactor, verify_installed_reactor
 
 
 logger = logging.getLogger(__name__)
@@ -138,7 +138,7 @@ class CrawlerRunner:
         self._crawlers = set()
         self._active = set()
         self.bootstrap_failed = False
-        self._handle_asyncio_reactor()
+        self._handle_twisted_reactor()
 
     @property
     def spiders(self):
@@ -232,10 +232,9 @@ class CrawlerRunner:
         while self._active:
             yield defer.DeferredList(self._active)
 
-    def _handle_asyncio_reactor(self):
-        if self.settings.getbool('ASYNCIO_REACTOR') and not is_asyncio_reactor_installed():
-            raise Exception("ASYNCIO_REACTOR is on but the Twisted asyncio "
-                            "reactor is not installed.")
+    def _handle_twisted_reactor(self):
+        if self.settings.get("TWISTED_REACTOR"):
+            verify_installed_reactor(self.settings["TWISTED_REACTOR"])
 
 
 class CrawlerProcess(CrawlerRunner):
@@ -324,10 +323,10 @@ class CrawlerProcess(CrawlerRunner):
         except RuntimeError:  # raised if already stopped or in shutdown stage
             pass
 
-    def _handle_asyncio_reactor(self):
-        if self.settings.getbool('ASYNCIO_REACTOR'):
-            install_asyncio_reactor()
-        super()._handle_asyncio_reactor()
+    def _handle_twisted_reactor(self):
+        if self.settings.get("TWISTED_REACTOR"):
+            install_reactor(self.settings["TWISTED_REACTOR"])
+        super()._handle_twisted_reactor()
 
 
 def _get_spider_loader(settings):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index c10dc1a1c..c80833710 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -19,8 +19,6 @@ from os.path import join, abspath, dirname
 
 AJAXCRAWL_ENABLED = False
 
-ASYNCIO_REACTOR = False
-
 AUTOTHROTTLE_ENABLED = False
 AUTOTHROTTLE_DEBUG = False
 AUTOTHROTTLE_MAX_DELAY = 60.0
@@ -291,6 +289,8 @@ TELNETCONSOLE_HOST = '127.0.0.1'
 TELNETCONSOLE_USERNAME = 'scrapy'
 TELNETCONSOLE_PASSWORD = None
 
+TWISTED_REACTOR = None
+
 SPIDER_CONTRACTS = {}
 SPIDER_CONTRACTS_BASE = {
     'scrapy.contracts.default.UrlContract': 1,
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
deleted file mode 100644
index 917973de2..000000000
--- a/scrapy/utils/asyncio.py
+++ /dev/null
@@ -1,17 +0,0 @@
-import asyncio
-from contextlib import suppress
-
-from twisted.internet import asyncioreactor
-from twisted.internet.error import ReactorAlreadyInstalledError
-
-
-def install_asyncio_reactor():
-    """ Tries to install AsyncioSelectorReactor
-    """
-    with suppress(ReactorAlreadyInstalledError):
-        asyncioreactor.install(asyncio.get_event_loop())
-
-
-def is_asyncio_reactor_installed():
-    from twisted.internet import reactor
-    return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 62b43a96c..6a21a4903 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -2,14 +2,14 @@
 Helper functions for dealing with Twisted deferreds
 """
 import asyncio
-from functools import wraps
 import inspect
+from functools import wraps
 
 from twisted.internet import defer, task
 from twisted.python import failure
 
 from scrapy.exceptions import IgnoreRequest
-from scrapy.utils.asyncio import is_asyncio_reactor_installed
+from scrapy.utils.reactor import is_asyncio_reactor_installed
 
 
 def defer_fail(_failure):
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index e4cf0196b..afef2c93f 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -1,17 +1,16 @@
 # -*- coding: utf-8 -*-
 
-import sys
 import logging
+import sys
 import warnings
 from logging.config import dictConfig
 
-from twisted.python.failure import Failure
 from twisted.python import log as twisted_log
+from twisted.python.failure import Failure
 
 import scrapy
-from scrapy.settings import Settings
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.asyncio import is_asyncio_reactor_installed
+from scrapy.settings import Settings
 from scrapy.utils.versions import scrapy_components_versions
 
 
@@ -149,8 +148,8 @@ def log_scrapy_info(settings):
                 {'versions': ", ".join("%s %s" % (name, version)
                     for name, version in scrapy_components_versions()
                     if name != "Scrapy")})
-    if is_asyncio_reactor_installed():
-        logger.debug("Asyncio reactor is installed")
+    from twisted.internet import reactor
+    logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
 
 
 class StreamLogger(object):
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index b98fff6ec..80f52a4ef 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -1,4 +1,9 @@
-from twisted.internet import error
+import asyncio
+from contextlib import suppress
+
+from twisted.internet import asyncioreactor, error
+
+from scrapy.utils.misc import load_object
 
 
 def listen_tcp(portrange, host, factory):
@@ -42,3 +47,31 @@ class CallLaterOnce(object):
     def __call__(self):
         self._call = None
         return self._func(*self._a, **self._kw)
+
+
+def install_reactor(reactor_path):
+    reactor_class = load_object(reactor_path)
+    if reactor_class is asyncioreactor.AsyncioSelectorReactor:
+        with suppress(error.ReactorAlreadyInstalledError):
+            asyncioreactor.install(asyncio.get_event_loop())
+    else:
+        *module, _ = reactor_path.split(".")
+        installer_path = module + ["install"]
+        installer = load_object(".".join(installer_path))
+        with suppress(error.ReactorAlreadyInstalledError):
+            installer()
+
+
+def verify_installed_reactor(reactor_path):
+    from twisted.internet import reactor
+    reactor_class = load_object(reactor_path)
+    if not isinstance(reactor, reactor_class):
+        msg = "The installed reactor ({}.{}) does not match the requested one ({})".format(
+            reactor.__module__, reactor.__class__.__name__, reactor_path
+        )
+        raise Exception(msg)
+
+
+def is_asyncio_reactor_installed():
+    from twisted.internet import reactor
+    return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)
diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
index db1b75931..d1e4a7bb5 100644
--- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
@@ -10,8 +10,7 @@ class NoRequestsSpider(scrapy.Spider):
 
 
 process = CrawlerProcess(settings={
-    'ASYNCIO_REACTOR': True,
+    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
 })
-
 process.crawl(NoRequestsSpider)
 process.start()
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py
index cec3c9c25..8568bd8b8 100644
--- a/tests/CrawlerProcess/asyncio_enabled_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py
@@ -15,8 +15,7 @@ class NoRequestsSpider(scrapy.Spider):
 
 
 process = CrawlerProcess(settings={
-    'ASYNCIO_REACTOR': True,
+    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
 })
-
 process.crawl(NoRequestsSpider)
 process.start()
diff --git a/tests/CrawlerProcess/twisted_reactor_asyncio.py b/tests/CrawlerProcess/twisted_reactor_asyncio.py
new file mode 100644
index 000000000..c6cbf949b
--- /dev/null
+++ b/tests/CrawlerProcess/twisted_reactor_asyncio.py
@@ -0,0 +1,13 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class AsyncioReactorSpider(scrapy.Spider):
+    name = 'asyncio_reactor'
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+})
+process.crawl(AsyncioReactorSpider)
+process.start()
diff --git a/tests/CrawlerProcess/twisted_reactor_poll.py b/tests/CrawlerProcess/twisted_reactor_poll.py
new file mode 100644
index 000000000..27063260b
--- /dev/null
+++ b/tests/CrawlerProcess/twisted_reactor_poll.py
@@ -0,0 +1,13 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class PollReactorSpider(scrapy.Spider):
+    name = 'poll_reactor'
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor",
+})
+process.crawl(PollReactorSpider)
+process.start()
diff --git a/tests/CrawlerProcess/twisted_reactor_select.py b/tests/CrawlerProcess/twisted_reactor_select.py
new file mode 100644
index 000000000..9af8ceb4d
--- /dev/null
+++ b/tests/CrawlerProcess/twisted_reactor_select.py
@@ -0,0 +1,13 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class SelectReactorSpider(scrapy.Spider):
+    name = 'epoll_reactor'
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+})
+process.crawl(SelectReactorSpider)
+process.start()
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 6024af71c..3612b70c9 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -296,12 +296,14 @@ class BadSpider(scrapy.Spider):
         self.assertIn("badspider.py", log)
 
     def test_asyncio_enabled_true(self):
-        log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=True'])
-        self.assertIn("DEBUG: Asyncio reactor is installed", log)
+        log = self.get_log(self.debug_log_spider, args=[
+            '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'
+        ])
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
     def test_asyncio_enabled_false(self):
-        log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=False'])
-        self.assertNotIn("DEBUG: Asyncio reactor is installed", log)
+        log = self.get_log(self.debug_log_spider, args=[])
+        self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
 
 class BenchCommandTest(CommandTest):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 0ce0674de..f8fa26def 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -254,30 +254,38 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
     def test_crawler_runner_asyncio_enabled_true(self):
         if self.reactor_pytest == 'asyncio':
-            runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True})
+            runner = CrawlerRunner(settings={
+                "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+            })
         else:
-            msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed"
+            msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
             with self.assertRaisesRegex(Exception, msg):
-                runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True})
+                runner = CrawlerRunner(settings={
+                    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+                })
 
     @defer.inlineCallbacks
     def test_crawler_process_asyncio_enabled_true(self):
         with LogCapture(level=logging.DEBUG) as log:
             if self.reactor_pytest == 'asyncio':
-                runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True})
+                runner = CrawlerProcess(settings={
+                    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+                })
                 yield runner.crawl(NoRequestsSpider)
-                self.assertIn("Asyncio reactor is installed", str(log))
+                self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
             else:
-                msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed"
+                msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
                 with self.assertRaisesRegex(Exception, msg):
-                    runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True})
+                    runner = CrawlerProcess(settings={
+                        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+                    })
 
     @defer.inlineCallbacks
     def test_crawler_process_asyncio_enabled_false(self):
-        runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': False})
+        runner = CrawlerProcess(settings={"TWISTED_REACTOR": None})
         with LogCapture(level=logging.DEBUG) as log:
             yield runner.crawl(NoRequestsSpider)
-            self.assertNotIn("Asyncio reactor is installed", str(log))
+            self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
 
 
 class CrawlerProcessSubprocess(unittest.TestCase):
@@ -294,17 +302,17 @@ class CrawlerProcessSubprocess(unittest.TestCase):
     def test_simple(self):
         log = self.run_script('simple.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertNotIn("DEBUG: Asyncio reactor is installed", log)
+        self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
     def test_asyncio_enabled_no_reactor(self):
         log = self.run_script('asyncio_enabled_no_reactor.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertIn("DEBUG: Asyncio reactor is installed", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
     def test_asyncio_enabled_reactor(self):
         log = self.run_script('asyncio_enabled_reactor.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertIn("DEBUG: Asyncio reactor is installed", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
     def test_ipv6_default_name_resolver(self):
         log = self.run_script('default_name_resolver.py')
@@ -323,3 +331,18 @@ class CrawlerProcessSubprocess(unittest.TestCase):
             "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
             "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
         ]))
+
+    def test_reactor_select(self):
+        log = self.run_script("twisted_reactor_select.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
+
+    def test_reactor_poll(self):
+        log = self.run_script("twisted_reactor_poll.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
+
+    def test_reactor_asyncio(self):
+        log = self.run_script("twisted_reactor_asyncio.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 44acc24af..295323e4d 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -2,7 +2,7 @@ from unittest import TestCase
 
 from pytest import mark
 
-from scrapy.utils.asyncio import is_asyncio_reactor_installed, install_asyncio_reactor
+from scrapy.utils.reactor import is_asyncio_reactor_installed, install_reactor
 
 
 @mark.usefixtures('reactor_pytest')
@@ -14,4 +14,4 @@ class AsyncioTest(TestCase):
 
     def test_install_asyncio_reactor(self):
         # this should do nothing
-        install_asyncio_reactor()
+        install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")

From 3263441fbcec8f46d363926d9106572cb0ecac5e Mon Sep 17 00:00:00 2001
From: Lane Shaw <lanethegreat@gmail.com>
Date: Thu, 6 Feb 2020 16:14:40 -0500
Subject: [PATCH 2623/2974] Update RFPDupeFilter line separator for correct
 universal newlines mode usage (#4283)

---
 scrapy/dupefilters.py     |  2 +-
 tests/test_dupefilters.py | 48 +++++++++++++++++++++++++++++++--------
 2 files changed, 40 insertions(+), 10 deletions(-)

diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index ea6a4cfc3..a36c8304f 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -49,7 +49,7 @@ class RFPDupeFilter(BaseDupeFilter):
             return True
         self.fingerprints.add(fp)
         if self.file:
-            self.file.write(fp + os.linesep)
+            self.file.write(fp + '\n')
 
     def request_fingerprint(self, request):
         return request_fingerprint(request)
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 0546558bc..88ce9627f 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -2,6 +2,8 @@ import hashlib
 import tempfile
 import unittest
 import shutil
+import os
+import sys
 from testfixtures import LogCapture
 
 from scrapy.dupefilters import RFPDupeFilter
@@ -84,17 +86,21 @@ class RFPDupeFilterTest(unittest.TestCase):
         path = tempfile.mkdtemp()
         try:
             df = RFPDupeFilter(path)
-            df.open()
-            assert not df.request_seen(r1)
-            assert df.request_seen(r1)
-            df.close('finished')
+            try:
+                df.open()
+                assert not df.request_seen(r1)
+                assert df.request_seen(r1)
+            finally:
+                df.close('finished')
 
             df2 = RFPDupeFilter(path)
-            df2.open()
-            assert df2.request_seen(r1)
-            assert not df2.request_seen(r2)
-            assert df2.request_seen(r2)
-            df2.close('finished')
+            try:
+                df2.open()
+                assert df2.request_seen(r1)
+                assert not df2.request_seen(r2)
+                assert df2.request_seen(r2)
+            finally:
+                df2.close('finished')
         finally:
             shutil.rmtree(path)
 
@@ -129,6 +135,30 @@ class RFPDupeFilterTest(unittest.TestCase):
 
         case_insensitive_dupefilter.close('finished')
 
+    def test_seenreq_newlines(self):
+        """ Checks against adding duplicate \r to
+        line endings on Windows platforms. """
+
+        r1 = Request('http://scrapytest.org/1')
+
+        path = tempfile.mkdtemp()
+        try:
+            df = RFPDupeFilter(path)
+            df.open()
+            df.request_seen(r1)
+            df.close('finished')
+
+            with open(os.path.join(path, 'requests.seen'), 'rb') as seen_file:
+                line = next(seen_file).decode()
+                assert not line.endswith('\r\r\n')
+                if sys.platform == 'win32':
+                    assert line.endswith('\r\n')
+                else:
+                    assert line.endswith('\n')
+
+        finally:
+            shutil.rmtree(path)
+
     def test_log(self):
         with LogCapture() as l:
             settings = {'DUPEFILTER_DEBUG': False,

From 7f2d3051feb0bc8f868a3eea8310e2fc8c461287 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 6 Feb 2020 18:19:40 -0300
Subject: [PATCH 2624/2974] Fix Flake8 issue

---
 tests/test_crawl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index f1d502ff7..225fe7a0e 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -14,8 +14,8 @@ from tests.mockserver import MockServer
 from tests.spiders import (
     AsyncDefAsyncioReturnSpider,
     AsyncDefAsyncioSpider,
-    AsyncDefSpider, 
-    BrokenStartRequestsSpider, 
+    AsyncDefSpider,
+    BrokenStartRequestsSpider,
     CrawlSpiderWithErrback,
     CrawlSpiderWithParseMethod,
     DelaySpider,

From 4f31c3ce017db2b4f69949a81efd56fee60ee32d Mon Sep 17 00:00:00 2001
From: Joy Bhalla <joybhalla9@gmail.com>
Date: Fri, 7 Feb 2020 02:51:33 +0530
Subject: [PATCH 2625/2974] Document a backward incompatibility that may affect
 custom schedulers (#4274)

---
 docs/news.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 6d0d4b4ee..e4b985c77 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -288,6 +288,13 @@ Backward-incompatible changes
     :class:`~scrapy.http.Request` objects instead of arbitrary Python data
     structures.
 
+*   An additional ``crawler`` parameter has been added to the ``__init__`` method
+    of the :class:`scrapy.core.scheduler.Scheduler` class. 
+    Custom scheduler subclasses which don't accept arbitrary parameters in 
+    their ``__init__`` method might break because of this change.
+
+    For more information, refer to the documentation for the :setting:`SCHEDULER` setting.
+
 See also :ref:`1.7-deprecation-removals` below.
 
 

From 84b55b73646acca71461366ef98a1a501331f5d8 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 7 Feb 2020 11:07:35 +0500
Subject: [PATCH 2626/2974] Update docs/topics/signals.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/signals.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 47be6b603..60d9ce2bc 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -306,7 +306,7 @@ request_left_downloader
 
     The signal does not support returning deferreds from their handlers.
 
-    :param request: the request that reached downloader
+    :param request: the request that reached the downloader
     :type request: :class:`~scrapy.http.Request` object
 
     :param spider: the spider that yielded the request

From 2f83f3e2cb3497e89d42533b8f20f8398a696f46 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 7 Feb 2020 11:07:43 +0500
Subject: [PATCH 2627/2974] Update docs/topics/signals.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/signals.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 60d9ce2bc..49475c1af 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -301,7 +301,7 @@ request_left_downloader
 .. signal:: request_left_downloader
 .. function:: request_left_downloader(request, spider)
 
-    Sent when a :class:`~scrapy.http.Request` leaves the downloader even in case of
+    Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of
     failure.
 
     The signal does not support returning deferreds from their handlers.

From 8817b9e8e92f01147e7e44dd767165766093f408 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 7 Feb 2020 11:07:53 +0500
Subject: [PATCH 2628/2974] Update docs/topics/signals.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/signals.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 49475c1af..a7d60e9cb 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -296,7 +296,7 @@ request_reached_downloader
     :type spider: :class:`~scrapy.spiders.Spider` object
 
 request_left_downloader
----------------------------
+-----------------------
 
 .. signal:: request_left_downloader
 .. function:: request_left_downloader(request, spider)

From 153b78e53f5c0f4630d09d560e111ca68f357905 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 7 Feb 2020 11:08:55 +0500
Subject: [PATCH 2629/2974] Update docs/topics/signals.rst

Co-Authored-By: elacuesta <elacuesta@users.noreply.github.com>
---
 docs/topics/signals.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index a7d60e9cb..886d1b866 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -304,7 +304,7 @@ request_left_downloader
     Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of
     failure.
 
-    The signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached the downloader
     :type request: :class:`~scrapy.http.Request` object

From 31f6c7112fe8efce2105983b8350b1dabdce7a1c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 7 Feb 2020 17:14:52 +0500
Subject: [PATCH 2630/2974] Add a test for an async callbacks that returns
 requests.

---
 tests/spiders.py    | 18 ++++++++++++++++++
 tests/test_crawl.py | 14 ++++++++++++--
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index 3b1ee94b8..284c77829 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -117,6 +117,24 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):
         return [{'id': 1}, {'id': 2}]
 
 
+class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
+
+    name = 'asyncdef_asyncio_reqs_return'
+
+    async def parse(self, response):
+        await asyncio.sleep(0.2)
+        req_id = response.meta.get('req_id', 0)
+        status = await get_from_asyncio_queue(response.status)
+        self.logger.info("Got response %d, req_id %d" % (status, req_id))
+        if req_id > 0:
+            return
+        reqs = []
+        for i in range(1, 3):
+            req = Request(self.start_urls[0], dont_filter=True, meta={'req_id': i})
+            reqs.append(req)
+        return reqs
+
+
 class ItemSpider(FollowAllSpider):
 
     name = 'item'
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 85005eba4..b4b5bac1c 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -13,7 +13,8 @@ from scrapy.utils.python import to_unicode
 from tests.mockserver import MockServer
 from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
                            SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback,
-                           AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider)
+                           AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider,
+                           AsyncDefAsyncioReqsReturnSpider)
 
 
 class CrawlTestCase(TestCase):
@@ -330,7 +331,7 @@ with multiples lines
 
     @mark.only_asyncio()
     @defer.inlineCallbacks
-    def test_async_def_asyncio_parse_list(self):
+    def test_async_def_asyncio_parse_items_list(self):
         items = []
 
         def _on_item_scraped(item):
@@ -343,3 +344,12 @@ with multiples lines
         self.assertIn("Got response 200", str(log))
         self.assertIn({'id': 1}, items)
         self.assertIn({'id': 2}, items)
+
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncio_parse_reqs_list(self):
+        crawler = self.runner.create_crawler(AsyncDefAsyncioReqsReturnSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        for req_id in range(3):
+            self.assertIn("Got response 200, req_id %d" % req_id, str(log))

From 7323780c97e69b560bf9a4bd7e6ccd60fb2b8f13 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 31 Dec 2019 16:15:41 +0500
Subject: [PATCH 2631/2974] Support yield in async def callbacks.

---
 conftest.py               |  4 ++-
 scrapy/utils/py36.py      | 10 ++++++++
 scrapy/utils/spider.py    |  8 ++++++
 tests/py36/_test_crawl.py | 50 ++++++++++++++++++++++++++++++++++++++
 tests/test_crawl.py       | 51 +++++++++++++++++++++++++++++++++++++++
 5 files changed, 122 insertions(+), 1 deletion(-)
 create mode 100644 scrapy/utils/py36.py
 create mode 100644 tests/py36/_test_crawl.py

diff --git a/conftest.py b/conftest.py
index c0de09909..be5fbabf4 100644
--- a/conftest.py
+++ b/conftest.py
@@ -11,7 +11,9 @@ collect_ignore = [
     # not a test, but looks like a test
     "scrapy/utils/testsite.py",
     # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
-    *_py_files("tests/CrawlerProcess")
+    *_py_files("tests/CrawlerProcess"),
+    # Py36-only parts of respective tests
+    *_py_files("tests/py36"),
 ]
 
 for line in open('tests/ignores.txt'):
diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py
new file mode 100644
index 000000000..c8c24076e
--- /dev/null
+++ b/scrapy/utils/py36.py
@@ -0,0 +1,10 @@
+"""
+Helpers using Python 3.6+ syntax (ignore SyntaxError on import).
+"""
+
+
+async def collect_asyncgen(result):
+    results = []
+    async for x in result:
+        results.append(x)
+    return results
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 72775df5c..4e2a4d1bc 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -4,12 +4,20 @@ import inspect
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.misc import arg_to_iter
+try:
+    from scrapy.utils.py36 import collect_asyncgen
+except SyntaxError:
+    collect_asyncgen = None
 
 
 logger = logging.getLogger(__name__)
 
 
 def iterate_spider_output(result):
+    if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result):
+        d = deferred_from_coro(collect_asyncgen(result))
+        d.addCallback(iterate_spider_output)
+        return d
     return arg_to_iter(deferred_from_coro(result))
 
 
diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py
new file mode 100644
index 000000000..74c7daf53
--- /dev/null
+++ b/tests/py36/_test_crawl.py
@@ -0,0 +1,50 @@
+import asyncio
+
+from scrapy import Request
+from tests.spiders import SimpleSpider
+
+
+class AsyncDefAsyncioGenSpider(SimpleSpider):
+
+    name = 'asyncdef_asyncio_gen'
+
+    async def parse(self, response):
+        await asyncio.sleep(0.2)
+        yield {'foo': 42}
+        self.logger.info("Got response %d" % response.status)
+
+
+class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
+
+    name = 'asyncdef_asyncio_gen_loop'
+
+    async def parse(self, response):
+        for i in range(10):
+            await asyncio.sleep(0.1)
+            yield {'foo': i}
+        self.logger.info("Got response %d" % response.status)
+
+
+class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
+
+    name = 'asyncdef_asyncio_gen_complex'
+    initial_reqs = 4
+    following_reqs = 3
+    depth = 2
+
+    def _get_req(self, index):
+        return Request(self.mockserver.url("/status?n=200&request=%d" % index),
+                       meta={'index': index})
+
+    def start_requests(self):
+        for i in range(self.initial_reqs):
+            yield self._get_req(i)
+
+    async def parse(self, response):
+        index = response.meta['index']
+        yield {'index': index}
+        if index < 10 ** self.depth:
+            for new_index in range(10 * index, 10 * index + self.following_reqs):
+                yield self._get_req(new_index)
+        await asyncio.sleep(0.1)
+        yield {'index': index + 5}
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 85005eba4..856068465 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -1,5 +1,6 @@
 import json
 import logging
+import sys
 
 from pytest import mark
 from testfixtures import LogCapture
@@ -343,3 +344,53 @@ with multiples lines
         self.assertIn("Got response 200", str(log))
         self.assertIn({'id': 1}, items)
         self.assertIn({'id': 2}, items)
+
+    @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncgen_parse(self):
+        from tests.py36._test_crawl import AsyncDefAsyncioGenSpider
+        crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        self.assertIn("Got response 200", str(log))
+        itemcount = crawler.stats.get_value('item_scraped_count')
+        self.assertEqual(itemcount, 1)
+
+    @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncgen_parse_loop(self):
+        items = []
+
+        def _on_item_scraped(item):
+            items.append(item)
+
+        from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider
+        crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider)
+        crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+        with LogCapture() as log:
+            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        self.assertIn("Got response 200", str(log))
+        itemcount = crawler.stats.get_value('item_scraped_count')
+        self.assertEqual(itemcount, 10)
+        for i in range(10):
+            self.assertIn({'foo': i}, items)
+
+    @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncgen_parse_complex(self):
+        items = []
+
+        def _on_item_scraped(item):
+            items.append(item)
+
+        from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider
+        crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider)
+        crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+        yield crawler.crawl(mockserver=self.mockserver)
+        itemcount = crawler.stats.get_value('item_scraped_count')
+        self.assertEqual(itemcount, 80)
+        for i in [0, 3, 21, 22, 207, 311]:  # some random items
+            self.assertIn({'index': i}, items)

From 59653ebac609dc11c9d3b29624972d2aaddd5541 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 7 Feb 2020 21:07:57 +0100
Subject: [PATCH 2632/2974] Update installation instructions regarding Python 3
 and virtual environments

---
 docs/intro/install.rst | 32 ++++++--------------------------
 1 file changed, 6 insertions(+), 26 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 178be723c..49968437c 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -81,35 +81,18 @@ Python packages can be installed either globally (a.k.a system wide),
 or in user-space. We do not recommend installing Scrapy system wide.
 
 Instead, we recommend that you install Scrapy within a so-called
-"virtual environment" (`virtualenv`_).
-Virtualenvs allow you to not conflict with already-installed Python
+"virtual environment" (:mod:`venv`).
+Virtual environments allow you to not conflict with already-installed Python
 system packages (which could break some of your system tools and scripts),
 and still install packages normally with ``pip`` (without ``sudo`` and the likes).
 
-To get started with virtual environments, see `virtualenv installation instructions`_.
-To install it globally (having it globally installed actually helps here),
-it should be a matter of running::
+See :ref:`tut-venv` on how to create your virtual environment.
 
-    $ [sudo] pip install virtualenv
-
-Check this `user guide`_ on how to create your virtualenv.
-
-.. note::
-    If you use Linux or OS X, `virtualenvwrapper`_ is a handy tool to create virtualenvs.
-
-Once you have created a virtualenv, you can install Scrapy inside it with ``pip``,
+Once you have created a virtual environment, you can install Scrapy inside it with ``pip``,
 just like any other Python package.
 (See :ref:`platform-specific guides <intro-install-platform-notes>`
 below for non-Python dependencies that you may need to install beforehand).
 
-Python virtualenvs can be created to use Python 2 by default, or Python 3 by default. As Scrapy
-only supports Python 3, make sure you created a Python 3 virtualenv.
-
-.. _virtualenv: https://virtualenv.pypa.io
-.. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/
-.. _virtualenvwrapper: https://virtualenvwrapper.readthedocs.io/en/latest/install.html
-.. _user guide: https://virtualenv.pypa.io/en/stable/userguide/
-
 
 .. _intro-install-platform-notes:
 
@@ -205,15 +188,12 @@ solutions:
 
       brew update; brew upgrade python
 
-* *(Optional)* Install Scrapy inside an isolated python environment.
+*   *(Optional)* :ref:`Install Scrapy inside a Python virtual environment
+    <intro-using-virtualenv>`.
 
   This method is a workaround for the above OS X issue, but it's an overall
   good practice for managing dependencies and can complement the first method.
 
-  `virtualenv`_ is a tool you can use to create virtual environments in python.
-  We recommended reading a tutorial like
-  http://docs.python-guide.org/en/latest/dev/virtualenvs/ to get started.
-
 After any of these workarounds you should be able to install Scrapy::
 
   pip install Scrapy

From 35723d76c0c07575309810e776305e9ea22fc18d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 7 Feb 2020 22:59:53 +0100
Subject: [PATCH 2633/2974] Use canonicalize_url in link extraction

---
 scrapy/linkextractors/lxmlhtml.py | 4 ++--
 tests/test_linkextractors.py      | 5 +----
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index fdfa92370..da525d52e 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -9,7 +9,7 @@ from w3lib.url import canonicalize_url
 
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
-from scrapy.utils.python import unique as unique_list, to_unicode
+from scrapy.utils.python import unique as unique_list
 from scrapy.utils.response import get_base_url
 from scrapy.linkextractors import FilteringLinkExtractor
 
@@ -66,7 +66,7 @@ class LxmlParserLinkExtractor(object):
                 url = self.process_attr(attr_val)
                 if url is None:
                     continue
-            url = to_unicode(url, encoding=response_encoding)
+            url = canonicalize_url(url, encoding=response_encoding)
             # to fix relative links after process_value
             url = urljoin(response_url, url)
             link = Link(url, _collect_string_content(el) or u'',
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 38fb8fb4a..e9d6c0abe 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -2,8 +2,6 @@ import re
 import unittest
 from warnings import catch_warnings
 
-import pytest
-
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import HtmlResponse, XmlResponse
 from scrapy.link import Link
@@ -214,7 +212,7 @@ class Base:
             response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
             links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
             self.assertEqual(links,
-                             [Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')])
+                             [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')])
 
         def test_restrict_xpaths_concat_in_handle_data(self):
             """html entities cause SGMLParser to call handle_data hook twice"""
@@ -506,7 +504,6 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
             Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
         ])
 
-    @pytest.mark.xfail
     def test_restrict_xpaths_with_html_entities(self):
         super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()
 

From a64fa2f0866c10594f1e5cf00a0161f9fea1eb62 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 10 Feb 2020 10:16:05 -0300
Subject: [PATCH 2634/2974] Keyword arguments when creating a _ResponseReader

---
 scrapy/core/downloader/handlers/http11.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 49c9eacac..7a1a77b23 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -417,14 +417,14 @@ class ScrapyAgent:
         d = defer.Deferred(_cancel)
         txresponse.deliverBody(
             _ResponseReader(
-                d,
-                txresponse,
-                request,
-                maxsize,
-                warnsize,
-                fail_on_dataloss,
-                self._crawler,
-                self._source,
+                finished=d,
+                txresponse=txresponse,
+                request=request,
+                maxsize=maxsize,
+                warnsize=warnsize,
+                fail_on_dataloss=fail_on_dataloss,
+                crawler=self._crawler,
+                source=self._source,
             )
         )
 

From 122ce6d6fb3861d99ba2f2810b2370056bae1190 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 10 Feb 2020 10:20:26 -0300
Subject: [PATCH 2635/2974] Check bytes are received in order (bytes_received
 signal)

---
 tests/test_engine.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index c83a23b55..0d970928b 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -12,7 +12,6 @@ module with the ``runserver`` argument::
 
 import os
 import re
-import string
 import sys
 from collections import defaultdict
 from urllib.parse import urlparse
@@ -91,7 +90,8 @@ def start_test_site(debug=False):
     r = static.File(root_dir)
     r.putChild(b"redirect", util.Redirect(b"/redirected"))
     r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
-    r.putChild(b"random", static.Data(string.ascii_letters.encode("utf8") * 2**14, "text/plain"))
+    numbers = [str(x).encode("utf8") for x in range(2**14)]
+    r.putChild(b"numbers", static.Data(b"".join(numbers), "text/plain"))
 
     port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1")
     if debug:
@@ -124,7 +124,7 @@ class CrawlerRun(object):
             self.geturl("/"),
             self.geturl("/redirect"),
             self.geturl("/redirect"),  # duplicate
-            self.geturl("/random"),
+            self.geturl("/numbers"),
         ]
 
         for name, signal in vars(signals).items():
@@ -315,8 +315,12 @@ class EngineTest(unittest.TestCase):
                     b"  </body>\n"
                     b"</html>\n"
                 )
-            elif self.run.getpath(request.url) == "/random":
-                self.assertTrue(len(data) > 1)  # signal was fired multiple times
+            elif self.run.getpath(request.url) == "/numbers":
+                # signal was fired multiple times
+                self.assertTrue(len(data) > 1)
+                # bytes were received in order
+                numbers = [str(x).encode("utf8") for x in range(2**14)]
+                self.assertEqual(joined_data, b"".join(numbers))
 
     def _assert_signals_caught(self):
         assert signals.engine_started in self.run.signals_caught

From 42b4e9b3372ce3f9da57c7512b31a3c455b8a161 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 10 Feb 2020 11:23:38 -0300
Subject: [PATCH 2636/2974] Reword signal docs

---
 docs/topics/signals.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 3a15bf95c..dfb87cef3 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -279,7 +279,7 @@ request_scheduled
     Sent when the engine schedules a :class:`~scrapy.http.Request`, to be
     downloaded later.
 
-    The signal does not support returning deferreds from its handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached the scheduler
     :type request: :class:`~scrapy.http.Request` object
@@ -296,7 +296,7 @@ request_dropped
     Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be
     downloaded later, is rejected by the scheduler.
 
-    The signal does not support returning deferreds from its handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached the scheduler
     :type request: :class:`~scrapy.http.Request` object
@@ -312,7 +312,7 @@ request_reached_downloader
 
     Sent when a :class:`~scrapy.http.Request` reached downloader.
 
-    The signal does not support returning deferreds from its handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached downloader
     :type request: :class:`~scrapy.http.Request` object

From 13ba9bc629cb0a77ebaca36a10a0a4984d7cce68 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 10 Feb 2020 12:29:39 -0300
Subject: [PATCH 2637/2974] Note about Response.ip_address

---
 docs/topics/request-response.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 17eb63064..89e570028 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -685,6 +685,8 @@ Response objects
     .. attribute:: Response.ip_address
 
         The IP address of the server from which the Response originated.
+        This attribute is currently only populated by the HTTP 1.1 download
+        handler, i.e. for ``http(s)`` responses.
 
     .. method:: Response.copy()
 

From 7025c18b159f1ce14b5732dc7d262efbdc72cf5e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 10 Feb 2020 19:43:23 +0100
Subject: [PATCH 2638/2974] Clear line of spaces

---
 docs/topics/feed-exports.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 9f837221e..8dc08710f 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -184,7 +184,7 @@ passed through the following settings:
 
  * :setting:`AWS_ACCESS_KEY_ID`
  * :setting:`AWS_SECRET_ACCESS_KEY`
- 
+
 You can also define a custom ACL for exported feeds using this setting:
 
  * :setting:`FEED_STORAGE_S3_ACL`

From 4626e90df8ba4a945bb9cd6be47a915788e76f23 Mon Sep 17 00:00:00 2001
From: Abhishek Pratap Singh <35230163+Prime-5@users.noreply.github.com>
Date: Mon, 10 Feb 2020 18:48:31 +0000
Subject: [PATCH 2639/2974] Allow updating flags in follow and follow_all
 (#4279)

---
 scrapy/http/response/__init__.py |  7 +++++--
 scrapy/http/response/text.py     |  6 ++++--
 tests/test_http_response.py      | 31 +++++++++++++++++++++++++++++++
 3 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index f92d0901c..027fbac6f 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -107,7 +107,7 @@ class Response(object_ref):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding='utf-8', priority=0,
-               dont_filter=False, errback=None, cb_kwargs=None):
+               dont_filter=False, errback=None, cb_kwargs=None, flags=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -124,6 +124,7 @@ class Response(object_ref):
         elif url is None:
             raise ValueError("url can't be None")
         url = self.urljoin(url)
+
         return Request(
             url=url,
             callback=callback,
@@ -137,11 +138,12 @@ class Response(object_ref):
             dont_filter=dont_filter,
             errback=errback,
             cb_kwargs=cb_kwargs,
+            flags=flags,
         )
 
     def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,
                    cookies=None, meta=None, encoding='utf-8', priority=0,
-                   dont_filter=False, errback=None, cb_kwargs=None):
+                   dont_filter=False, errback=None, cb_kwargs=None, flags=None):
         # type: (...) -> Generator[Request, None, None]
         """
         Return an iterable of :class:`~.Request` instances to follow all links
@@ -169,6 +171,7 @@ class Response(object_ref):
                 dont_filter=dont_filter,
                 errback=errback,
                 cb_kwargs=cb_kwargs,
+                flags=flags,
             )
             for url in urls
         )
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 09049c157..33a485328 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -121,7 +121,7 @@ class TextResponse(Response):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding=None, priority=0,
-               dont_filter=False, errback=None, cb_kwargs=None):
+               dont_filter=False, errback=None, cb_kwargs=None, flags=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -157,11 +157,12 @@ class TextResponse(Response):
             dont_filter=dont_filter,
             errback=errback,
             cb_kwargs=cb_kwargs,
+            flags=flags,
         )
 
     def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,
                    cookies=None, meta=None, encoding=None, priority=0,
-                   dont_filter=False, errback=None, cb_kwargs=None,
+                   dont_filter=False, errback=None, cb_kwargs=None, flags=None,
                    css=None, xpath=None):
         # type: (...) -> Generator[Request, None, None]
         """
@@ -214,6 +215,7 @@ class TextResponse(Response):
             dont_filter=dont_filter,
             errback=errback,
             cb_kwargs=cb_kwargs,
+            flags=flags,
         )
 
 
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 4c1b2afc3..ff487cfa3 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -166,6 +166,10 @@ class BaseResponseTest(unittest.TestCase):
     def test_follow_whitespace_link(self):
         self._assert_followed_url(Link('http://example.com/foo '),
                                   'http://example.com/foo%20')
+    def test_follow_flags(self):
+        res = self.response_class('http://example.com/')
+        fol = res.follow('http://example.com/', flags=['cached', 'allowed'])
+        self.assertEqual(fol.flags, ['cached', 'allowed'])
 
     # Response.follow_all
 
@@ -232,6 +236,17 @@ class BaseResponseTest(unittest.TestCase):
         expected = [u.replace(' ', '%20') for u in absolute]
         self._assert_followed_all_urls(links, expected)
 
+    def test_follow_all_flags(self):
+        re = self.response_class('http://www.example.com/')
+        urls = [
+            'http://www.example.com/',
+            'http://www.example.com/2',
+            'http://www.example.com/foo',
+        ]
+        fol = re.follow_all(urls, flags=['cached', 'allowed'])
+        for req in fol:
+            self.assertEqual(req.flags, ['cached', 'allowed'])
+
     def _assert_followed_url(self, follow_obj, target_url, response=None):
         if response is None:
             response = self._links_response()
@@ -562,6 +577,22 @@ class TextResponseTest(BaseResponseTest):
         )
         self.assertEqual(req.encoding, 'cp1251')
 
+    def test_follow_flags(self):
+        res = self.response_class('http://example.com/')
+        fol = res.follow('http://example.com/', flags=['cached', 'allowed'])
+        self.assertEqual(fol.flags, ['cached', 'allowed'])
+
+    def test_follow_all_flags(self):
+        re = self.response_class('http://www.example.com/')
+        urls = [
+            'http://www.example.com/',
+            'http://www.example.com/2',
+            'http://www.example.com/foo',
+        ]
+        fol = re.follow_all(urls, flags=['cached', 'allowed'])
+        for req in fol:
+            self.assertEqual(req.flags, ['cached', 'allowed'])
+
     def test_follow_all_css(self):
         expected = [
             'http://example.com/sample3.html',

From 037ae5b22e6d6600dc537ee5073652ce74e5f47b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 10 Feb 2020 19:54:47 +0100
Subject: [PATCH 2640/2974] =?UTF-8?q?Explicitly=20indicate=20None=20as=20i?=
 =?UTF-8?q?p=5Faddress=E2=80=99s=20default=20value?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/topics/request-response.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 89e570028..8f2504a33 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -685,8 +685,10 @@ Response objects
     .. attribute:: Response.ip_address
 
         The IP address of the server from which the Response originated.
+        
         This attribute is currently only populated by the HTTP 1.1 download
-        handler, i.e. for ``http(s)`` responses.
+        handler, i.e. for ``http(s)`` responses. For other handlers, 
+        :attr:`ip_address` is always ``None``.
 
     .. method:: Response.copy()
 

From 36dcf901849014d7db00a0294ed86c6cc79b5cc6 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 11 Feb 2020 00:57:58 +0500
Subject: [PATCH 2641/2974] Also test non-default async callbacks.

---
 tests/py36/_test_crawl.py | 13 ++++++++++---
 tests/test_crawl.py       |  7 +++++--
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py
index 74c7daf53..162a53760 100644
--- a/tests/py36/_test_crawl.py
+++ b/tests/py36/_test_crawl.py
@@ -32,12 +32,14 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
     following_reqs = 3
     depth = 2
 
-    def _get_req(self, index):
+    def _get_req(self, index, cb=None):
         return Request(self.mockserver.url("/status?n=200&request=%d" % index),
-                       meta={'index': index})
+                       meta={'index': index},
+                       dont_filter=True,
+                       callback=cb)
 
     def start_requests(self):
-        for i in range(self.initial_reqs):
+        for i in range(1, self.initial_reqs + 1):
             yield self._get_req(i)
 
     async def parse(self, response):
@@ -46,5 +48,10 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
         if index < 10 ** self.depth:
             for new_index in range(10 * index, 10 * index + self.following_reqs):
                 yield self._get_req(new_index)
+        yield self._get_req(index, cb=self.parse2)
         await asyncio.sleep(0.1)
         yield {'index': index + 5}
+
+    async def parse2(self, response):
+        await asyncio.sleep(0.1)
+        yield {'index2': response.meta['index']}
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 626000147..64819acb6 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -392,9 +392,12 @@ with multiples lines
         crawler.signals.connect(_on_item_scraped, signals.item_scraped)
         yield crawler.crawl(mockserver=self.mockserver)
         itemcount = crawler.stats.get_value('item_scraped_count')
-        self.assertEqual(itemcount, 80)
-        for i in [0, 3, 21, 22, 207, 311]:  # some random items
+        self.assertEqual(itemcount, 156)
+        # some random items
+        for i in [1, 4, 21, 22, 207, 311]:
             self.assertIn({'index': i}, items)
+        for i in [10, 30, 122]:
+            self.assertIn({'index2': i}, items)
 
     @mark.only_asyncio()
     @defer.inlineCallbacks

From 1f0f52cbf7bdc9f11f7b83c482ad52ad7ad32ba0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 11 Feb 2020 01:05:45 +0500
Subject: [PATCH 2642/2974] Improve async signal tests.

---
 tests/test_signals.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/tests/test_signals.py b/tests/test_signals.py
index 001e798e5..d6ae526be 100644
--- a/tests/test_signals.py
+++ b/tests/test_signals.py
@@ -1,8 +1,9 @@
+from pytest import mark
 from twisted.internet import defer
 from twisted.trial import unittest
 
 from scrapy import signals, Request, Spider
-from scrapy.utils.test import get_crawler
+from scrapy.utils.test import get_crawler, get_from_asyncio_queue
 
 from tests.mockserver import MockServer
 
@@ -11,12 +12,12 @@ class ItemSpider(Spider):
     name = 'itemspider'
 
     def start_requests(self):
-        for _ in range(10):
-            yield Request(self.mockserver.url('/status?n=200'),
-                          dont_filter=True)
+        for index in range(10):
+            yield Request(self.mockserver.url('/status?n=200&id=%d' % index),
+                          meta={'index': index})
 
     def parse(self, response):
-        return {'field': 42}
+        return {'index': response.meta['index']}
 
 
 class AsyncSignalTestCase(unittest.TestCase):
@@ -29,11 +30,15 @@ class AsyncSignalTestCase(unittest.TestCase):
         self.mockserver.__exit__(None, None, None)
 
     async def _on_item_scraped(self, item):
+        item = await get_from_asyncio_queue(item)
         self.items.append(item)
 
+    @mark.only_asyncio()
     @defer.inlineCallbacks
     def test_simple_pipeline(self):
         crawler = get_crawler(ItemSpider)
         crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertEqual(len(self.items), 10)
+        for index in range(10):
+            self.assertIn({'index': index}, self.items)

From 61e74bac765de0f786d2125e876e4d7934f1722b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 10 Feb 2020 21:57:21 +0100
Subject: [PATCH 2643/2974] Extract links with safe_url_string

canonicalize_url changes links in undesirable ways.
---
 scrapy/linkextractors/lxmlhtml.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index da525d52e..f5ef56ea4 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -5,7 +5,7 @@ from urllib.parse import urljoin
 
 import lxml.etree as etree
 from w3lib.html import strip_html5_whitespace
-from w3lib.url import canonicalize_url
+from w3lib.url import canonicalize_url, safe_url_string
 
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
@@ -66,7 +66,7 @@ class LxmlParserLinkExtractor(object):
                 url = self.process_attr(attr_val)
                 if url is None:
                     continue
-            url = canonicalize_url(url, encoding=response_encoding)
+            url = safe_url_string(url, encoding=response_encoding)
             # to fix relative links after process_value
             url = urljoin(response_url, url)
             link = Link(url, _collect_string_content(el) or u'',

From 2d6d4fb2335ef24b1efca67dcedf5d264a642e0f Mon Sep 17 00:00:00 2001
From: Drew Seibert <drewjbert@gmail.com>
Date: Tue, 11 Feb 2020 03:35:23 -0600
Subject: [PATCH 2644/2974] Deprecate overriding settings with SCRAPY-prefixed
 environment variables (#4300)

---
 scrapy/utils/project.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index f28c2eaa1..d9a03ff63 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -68,7 +68,6 @@ def get_project_settings():
     if settings_module_path:
         settings.setmodule(settings_module_path, priority='project')
 
-    # XXX: remove this hack
     pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
     if pickled_settings:
         warnings.warn("Use of environment variable "
@@ -76,10 +75,9 @@ def get_project_settings():
                       "is deprecated.", ScrapyDeprecationWarning)
         settings.setdict(pickle.loads(pickled_settings), priority='project')
 
-    # XXX: deprecate and remove this functionality
     env_overrides = {k[7:]: v for k, v in os.environ.items() if
                      k.startswith('SCRAPY_')}
     if env_overrides:
+        warnings.warn("Use of 'SCRAPY_'-prefixed environment variables to override settings is deprecated.", ScrapyDeprecationWarning)
         settings.setdict(env_overrides, priority='project')
-
     return settings

From 73e88d036c72aa628af50371aace4cfce7e286e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 12 Feb 2020 17:17:38 +0100
Subject: [PATCH 2645/2974] Import mock from unittest

---
 scrapy/utils/test.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 4ff6d73a5..00002c303 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -2,14 +2,13 @@
 This module contains some assorted functions used in tests
 """
 
-from __future__ import absolute_import
-from posixpath import split
 import asyncio
 import os
+from posixpath import split
+from unittest import mock
 
 from importlib import import_module
 from twisted.trial.unittest import SkipTest
-from tests import mock
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.boto import is_botocore

From e1be078eaa17a8df72716932fde07e225f79745f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 12 Feb 2020 17:38:06 +0100
Subject: [PATCH 2646/2974] Fix Flake8-reported issues

---
 scrapy/utils/test.py     |  2 ++
 tests/test_feedexport.py | 27 ++++++++++++++-------------
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 00002c303..7442a2f33 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -120,12 +120,14 @@ def assert_samelines(testcase, text1, text2, msg=None):
     """
     testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
 
+
 def get_from_asyncio_queue(value):
     q = asyncio.Queue()
     getter = q.get()
     q.put_nowait(value)
     return getter
 
+
 def mock_google_cloud_storage():
     """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
     classes and set their proper return values.
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index d97b199fe..2b299503b 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -25,9 +25,13 @@ from scrapy.extensions.feedexport import (
     IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage,
     S3FeedStorage, StdoutFeedStorage,
     BlockingFeedStorage)
-from scrapy.utils.test import (assert_aws_environ, get_s3_content_and_delete,
-    get_crawler, mock_google_cloud_storage)
 from scrapy.utils.python import to_unicode
+from scrapy.utils.test import (
+    assert_aws_environ,
+    get_s3_content_and_delete,
+    get_crawler,
+    mock_google_cloud_storage,
+)
 
 
 class FileFeedStorageTest(unittest.TestCase):
@@ -362,15 +366,13 @@ class S3FeedStorageTest(unittest.TestCase):
 
 class GCSFeedStorageTest(unittest.TestCase):
 
-    @mock.patch('scrapy.conf.settings',
-                new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': None }, create=True)
     def test_parse_settings(self):
         try:
-            from google.cloud.storage import Client
+            from google.cloud.storage import Client  # noqa
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 
-        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' }
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead'}
         crawler = get_crawler(settings_dict=settings)
         storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
         assert storage.project_id == '123'
@@ -378,20 +380,18 @@ class GCSFeedStorageTest(unittest.TestCase):
         assert storage.bucket_name == 'mybucket'
         assert storage.blob_name == 'export.csv'
 
-    @mock.patch('scrapy.conf.settings',
-                new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': '' }, create=True)
     def test_parse_empty_acl(self):
         try:
-            from google.cloud.storage import Client
+            from google.cloud.storage import Client  # noqa
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 
-        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': '' }
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': ''}
         crawler = get_crawler(settings_dict=settings)
         storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
         assert storage.acl is None
 
-        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None }
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None}
         crawler = get_crawler(settings_dict=settings)
         storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
         assert storage.acl is None
@@ -399,7 +399,7 @@ class GCSFeedStorageTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_store(self):
         try:
-            from google.cloud.storage import Client
+            from google.cloud.storage import Client  # noqa
         except ImportError:
             raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
 
@@ -407,7 +407,7 @@ class GCSFeedStorageTest(unittest.TestCase):
         project_id = 'myproject-123'
         acl = 'publicRead'
         (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
-        with mock.patch('google.cloud.storage.Client') as m:    
+        with mock.patch('google.cloud.storage.Client') as m:
             m.return_value = client_mock
 
             f = mock.Mock()
@@ -420,6 +420,7 @@ class GCSFeedStorageTest(unittest.TestCase):
             bucket_mock.blob.assert_called_once_with('export.csv')
             blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
 
+
 class StdoutFeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks

From b4958358e89b6611f7dd852684ba6d599831fe27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 12 Feb 2020 19:00:04 +0100
Subject: [PATCH 2647/2974] Update tests to account for link extractors
 escaping spaces

---
 tests/test_linkextractors.py | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index e9d6c0abe..53968e60e 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -14,7 +14,6 @@ from tests import get_testdata
 class Base:
     class LinkExtractorTestCase(unittest.TestCase):
         extractor_cls = None
-        escapes_whitespace = False
 
         def setUp(self):
             body = get_testdata('link_extractor', 'linkextractor.html')
@@ -28,10 +27,7 @@ class Base:
 
         def test_extract_all_links(self):
             lx = self.extractor_cls()
-            if self.escapes_whitespace:
-                page4_url = 'http://example.com/page%204.html'
-            else:
-                page4_url = 'http://example.com/page 4.html'
+            page4_url = 'http://example.com/page%204.html'
 
             self.assertEqual([link for link in lx.extract_links(self.response)], [
                 Link(url='http://example.com/sample1.html', text=u''),
@@ -308,10 +304,7 @@ class Base:
 
         def test_attrs(self):
             lx = self.extractor_cls(attrs="href")
-            if self.escapes_whitespace:
-                page4_url = 'http://example.com/page%204.html'
-            else:
-                page4_url = 'http://example.com/page 4.html'
+            page4_url = 'http://example.com/page%204.html'
 
             self.assertEqual(lx.extract_links(self.response), [
                 Link(url='http://example.com/sample1.html', text=u''),

From df937d8280fe0781f6cf1715a3a0cd28c6e94eae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 13 Feb 2020 22:33:36 +0100
Subject: [PATCH 2648/2974] Implement Response.cb_kwargs

---
 docs/topics/request-response.rst | 12 ++++++++++++
 scrapy/http/response/__init__.py | 10 ++++++++++
 2 files changed, 22 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 4cf367d96..05b7bb5c7 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -672,6 +672,18 @@ Response objects
 
         .. seealso:: :attr:`Request.meta` attribute
 
+    .. attribute:: Response.cb_kwargs
+
+        A shortcut to the :attr:`Request.cb_kwargs` attribute of the
+        :attr:`Response.request` object (ie. ``self.request.cb_kwargs``).
+
+        Unlike the :attr:`Response.request` attribute, the
+        :attr:`Response.cb_kwargs` attribute is propagated along redirects and
+        retries, so you will get the original :attr:`Request.cb_kwargs` sent
+        from your spider.
+
+        .. seealso:: :attr:`Request.cb_kwargs` attribute
+
     .. attribute:: Response.flags
 
         A list that contains flags for this response. Flags are labels used for
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 64e9c6c20..ee9720d52 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -24,6 +24,16 @@ class Response(object_ref):
         self.request = request
         self.flags = [] if flags is None else list(flags)
 
+    @property
+    def cb_kwargs(self):
+        try:
+            return self.request.cb_kwargs
+        except AttributeError:
+            raise AttributeError(
+                "Response.cb_kwargs not available, this response "
+                "is not tied to any request"
+            )
+
     @property
     def meta(self):
         try:

From 5ff9eb90ea9d533d3f960db75071f0fe638503ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 13 Feb 2020 22:36:18 +0100
Subject: [PATCH 2649/2974] Add a test for the copy of cb_kwargs from Request
 to Response

---
 tests/test_http_response.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 960ecea3e..39f5fe750 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -72,6 +72,12 @@ class BaseResponseTest(unittest.TestCase):
         r1 = self.response_class("http://www.example.com", body=b"Some body", request=req)
         assert r1.meta is req.meta
 
+    def test_copy_cb_kwargs(self):
+        req = Request("http://www.example.com")
+        req.cb_kwargs['foo'] = 'bar'
+        r1 = self.response_class("http://www.example.com", body=b"Some body", request=req)
+        assert r1.cb_kwargs is req.cb_kwargs
+
     def test_copy_inherited_classes(self):
         """Test Response children copies preserve their class"""
 

From 43b43654a1dacae7b63fc067dc69929c262d9a15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 13 Feb 2020 22:39:58 +0100
Subject: [PATCH 2650/2974] Add tests for meta and cb_kwargs not being
 available

---
 tests/test_http_response.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 39f5fe750..5a19f9d54 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -78,6 +78,16 @@ class BaseResponseTest(unittest.TestCase):
         r1 = self.response_class("http://www.example.com", body=b"Some body", request=req)
         assert r1.cb_kwargs is req.cb_kwargs
 
+    def test_unavailable_meta(self):
+        r1 = self.response_class("http://www.example.com", body=b"Some body")
+        with self.assertRaisesRegex(AttributeError, r'Response\.meta not available'):
+            r1.meta
+
+    def test_unavailable_cb_kwargs(self):
+        r1 = self.response_class("http://www.example.com", body=b"Some body")
+        with self.assertRaisesRegex(AttributeError, r'Response\.cb_kwargs not available'):
+            r1.cb_kwargs
+
     def test_copy_inherited_classes(self):
         """Test Response children copies preserve their class"""
 

From 5ae3e1678fa99b3c44cb8981079df51ec34b860f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 14 Feb 2020 22:30:36 +0100
Subject: [PATCH 2651/2974] =?UTF-8?q?ie.=20=E2=86=92=20i.e.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: elacuesta <elacuesta@users.noreply.github.com>
---
 docs/topics/request-response.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 05b7bb5c7..260fe3caf 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -675,7 +675,7 @@ Response objects
     .. attribute:: Response.cb_kwargs
 
         A shortcut to the :attr:`Request.cb_kwargs` attribute of the
-        :attr:`Response.request` object (ie. ``self.request.cb_kwargs``).
+        :attr:`Response.request` object (i.e. ``self.request.cb_kwargs``).
 
         Unlike the :attr:`Response.request` attribute, the
         :attr:`Response.cb_kwargs` attribute is propagated along redirects and

From a04dd13cd08f1ff392a8bbe284fa0c8fe8924b57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 14 Feb 2020 22:31:30 +0100
Subject: [PATCH 2652/2974] =?UTF-8?q?ie.=20=E2=86=92=20i.e.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/topics/request-response.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 260fe3caf..d6c7cbec9 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -664,7 +664,7 @@ Response objects
     .. attribute:: Response.meta
 
         A shortcut to the :attr:`Request.meta` attribute of the
-        :attr:`Response.request` object (ie. ``self.request.meta``).
+        :attr:`Response.request` object (i.e. ``self.request.meta``).
 
         Unlike the :attr:`Response.request` attribute, the :attr:`Response.meta`
         attribute is propagated along redirects and retries, so you will get
@@ -770,7 +770,7 @@ TextResponse objects
        1. the encoding passed in the ``__init__`` method ``encoding`` argument
 
        2. the encoding declared in the Content-Type HTTP header. If this
-          encoding is not valid (ie. unknown), it is ignored and the next
+          encoding is not valid (i.e. unknown), it is ignored and the next
           resolution mechanism is tried.
 
        3. the encoding declared in the response body. The TextResponse class

From 182445f9d96130b1041ece8c4b2a9e9891107c73 Mon Sep 17 00:00:00 2001
From: Akshay Sharma <42249933+AKSHAYSHARMAJS@users.noreply.github.com>
Date: Tue, 18 Feb 2020 22:28:31 +0530
Subject: [PATCH 2653/2974] =?UTF-8?q?Fix=20a=20spelling=20error:=20ie.=20?=
 =?UTF-8?q?=E2=86=92=20i.e.=20(#4338)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/intro/tutorial.rst               | 2 +-
 docs/topics/downloader-middleware.rst | 4 ++--
 docs/topics/extensions.rst            | 6 +++---
 docs/topics/feed-exports.rst          | 2 +-
 docs/topics/jobs.rst                  | 2 +-
 docs/topics/link-extractors.rst       | 2 +-
 docs/topics/request-response.rst      | 4 ++--
 docs/topics/selectors.rst             | 4 ++--
 docs/topics/settings.rst              | 6 +++---
 docs/topics/signals.rst               | 4 ++--
 scrapy/shell.py                       | 2 +-
 scrapy/utils/misc.py                  | 4 ++--
 scrapy/utils/request.py               | 2 +-
 scrapy/utils/spider.py                | 2 +-
 sep/sep-003.rst                       | 4 ++--
 sep/sep-013.rst                       | 2 +-
 sep/sep-021.rst                       | 2 +-
 17 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index ee10048b5..798fe4a7a 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -212,7 +212,7 @@ using the :ref:`Scrapy shell <topics-shell>`. Run::
 .. note::
 
    Remember to always enclose urls in quotes when running Scrapy shell from
-   command-line, otherwise urls containing arguments (ie. ``&`` character)
+   command-line, otherwise urls containing arguments (i.e. ``&`` character)
    will not work.
 
    On Windows, use double quotes instead::
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 3ec6e0c17..a83cedcfd 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -259,8 +259,8 @@ COOKIES_DEBUG
 
 Default: ``False``
 
-If enabled, Scrapy will log all cookies sent in requests (ie. ``Cookie``
-header) and all cookies received in responses (ie. ``Set-Cookie`` header).
+If enabled, Scrapy will log all cookies sent in requests (i.e. ``Cookie``
+header) and all cookies received in responses (i.e. ``Set-Cookie`` header).
 
 Here's an example of a log with :setting:`COOKIES_DEBUG` enabled::
 
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 0a7455ec9..dc057f6b6 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -63,7 +63,7 @@ but disabled unless the :setting:`HTTPCACHE_ENABLED` setting is set.
 Disabling an extension
 ======================
 
-In order to disable an extension that comes enabled by default (ie. those
+In order to disable an extension that comes enabled by default (i.e. those
 included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to
 ``None``. For example::
 
@@ -345,7 +345,7 @@ signal is received. The information dumped is the following:
 After the stack trace and engine status is dumped, the Scrapy process continues
 running normally.
 
-This extension only works on POSIX-compliant platforms (ie. not Windows),
+This extension only works on POSIX-compliant platforms (i.e. not Windows),
 because the `SIGQUIT`_ and `SIGUSR2`_ signals are not available on Windows.
 
 There are at least two ways to send Scrapy the `SIGQUIT`_ signal:
@@ -370,7 +370,7 @@ running normally.
 
 For more info see `Debugging in Python`_.
 
-This extension only works on POSIX-compliant platforms (ie. not Windows).
+This extension only works on POSIX-compliant platforms (i.e. not Windows).
 
 .. _Python debugger: https://docs.python.org/2/library/pdb.html
 .. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 7481b1a99..1d94807a4 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -301,7 +301,7 @@ FEED_STORE_EMPTY
 
 Default: ``False``
 
-Whether to export empty feeds (ie. feeds with no items).
+Whether to export empty feeds (i.e. feeds with no items).
 
 .. setting:: FEED_STORAGES
 
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 8816a028c..c34ba336b 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -22,7 +22,7 @@ Job directory
 
 To enable persistence support you just need to define a *job directory* through
 the ``JOBDIR`` setting. This directory will be for storing all required data to
-keep the state of a single job (ie. a spider run).  It's important to note that
+keep the state of a single job (i.e. a spider run).  It's important to note that
 this directory must not be shared by different spiders, or even different
 jobs/runs of the same spider, as it's meant to be used for storing the state of
 a *single* job.
diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 2119cb8f8..8c8019438 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -49,7 +49,7 @@ LxmlLinkExtractor
     :type allow: a regular expression (or list of)
 
     :param deny: a single regular expression (or list of regular expressions)
-        that the (absolute) urls must match in order to be excluded (ie. not
+        that the (absolute) urls must match in order to be excluded (i.e. not
         extracted). It has precedence over the ``allow`` parameter. If not
         given (or empty) it won't exclude any links.
     :type deny: a regular expression (or list of)
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 8997a7f19..34cc41a02 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -664,7 +664,7 @@ Response objects
     .. attribute:: Response.meta
 
         A shortcut to the :attr:`Request.meta` attribute of the
-        :attr:`Response.request` object (ie. ``self.request.meta``).
+        :attr:`Response.request` object (i.e. ``self.request.meta``).
 
         Unlike the :attr:`Response.request` attribute, the :attr:`Response.meta`
         attribute is propagated along redirects and retries, so you will get
@@ -760,7 +760,7 @@ TextResponse objects
        1. the encoding passed in the ``__init__`` method ``encoding`` argument
 
        2. the encoding declared in the Content-Type HTTP header. If this
-          encoding is not valid (ie. unknown), it is ignored and the next
+          encoding is not valid (i.e. unknown), it is ignored and the next
           resolution mechanism is tried.
 
        3. the encoding declared in the response body. The TextResponse class
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 8ec758b0e..c3d431e2a 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -986,7 +986,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
       sel = Selector(html_response)
 
 1. Select all ``<h1>`` elements from an HTML response body, returning a list of
-   :class:`Selector` objects (ie. a :class:`SelectorList` object)::
+   :class:`Selector` objects (i.e. a :class:`SelectorList` object)::
 
       sel.xpath("//h1")
 
@@ -1013,7 +1013,7 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object::
       sel = Selector(xml_response)
 
 1. Select all ``<product>`` elements from an XML response body, returning a list
-   of :class:`Selector` objects (ie. a :class:`SelectorList` object)::
+   of :class:`Selector` objects (i.e. a :class:`SelectorList` object)::
 
       sel.xpath("//product")
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index fa63a5807..5394147da 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -248,7 +248,7 @@ CONCURRENT_REQUESTS
 
 Default: ``16``
 
-The maximum number of concurrent (ie. simultaneous) requests that will be
+The maximum number of concurrent (i.e. simultaneous) requests that will be
 performed by the Scrapy downloader.
 
 .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
@@ -258,7 +258,7 @@ CONCURRENT_REQUESTS_PER_DOMAIN
 
 Default: ``8``
 
-The maximum number of concurrent (ie. simultaneous) requests that will be
+The maximum number of concurrent (i.e. simultaneous) requests that will be
 performed to any single domain.
 
 See also: :ref:`topics-autothrottle` and its
@@ -272,7 +272,7 @@ CONCURRENT_REQUESTS_PER_IP
 
 Default: ``0``
 
-The maximum number of concurrent (ie. simultaneous) requests that will be
+The maximum number of concurrent (i.e. simultaneous) requests that will be
 performed to any single IP. If non-zero, the
 :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is ignored, and this one is
 used instead. In other words, concurrency limits will be applied per IP, not
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 886d1b866..d3cfb0307 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -141,7 +141,7 @@ item_error
 .. signal:: item_error
 .. function:: item_error(item, response, spider, failure)
 
-    Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises
+    Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises
     an exception), except :exc:`~scrapy.exceptions.DropItem` exception.
 
     This signal supports returning deferreds from their handlers.
@@ -232,7 +232,7 @@ spider_error
 .. signal:: spider_error
 .. function:: spider_error(failure, response, spider)
 
-    Sent when a spider callback generates an error (ie. raises an exception).
+    Sent when a spider callback generates an error (i.e. raises an exception).
 
     This signal does not support returning deferreds from their handlers.
 
diff --git a/scrapy/shell.py b/scrapy/shell.py
index a23b04df9..1d5341973 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -173,7 +173,7 @@ def _request_deferred(request):
 
     This returns a Deferred whose first pair of callbacks are the request
     callback and errback. The Deferred also triggers when the request
-    callback/errback is executed (ie. when the request is downloaded)
+    callback/errback is executed (i.e. when the request is downloaded)
 
     WARNING: Do not call request.replace() until after the deferred is called.
     """
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index cb0ee5af3..a3e55d6ea 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -37,8 +37,8 @@ def arg_to_iter(arg):
 def load_object(path):
     """Load an object given its absolute object path, and return it.
 
-    object can be a class, function, variable or an instance.
-    path ie: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'
+    object can be the import path of a class, function, variable or an
+    instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'
     """
 
     try:
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 356753ab5..b8c140a7e 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -28,7 +28,7 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
     http://www.example.com/query?cat=222&id=111
 
     Even though those are two different URLs both point to the same resource
-    and are equivalent (ie. they should return the same response).
+    and are equivalent (i.e. they should return the same response).
 
     Another example are cookies used to store session ids. Suppose the
     following page is only accessible to authenticated users:
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 72775df5c..e4a2d1ac2 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -15,7 +15,7 @@ def iterate_spider_output(result):
 
 def iter_spider_classes(module):
     """Return an iterator over all spider classes defined in the given module
-    that can be instantiated (ie. which have name)
+    that can be instantiated (i.e. which have name)
     """
     # this needs to be imported here until get rid of the spider manager
     # singleton in scrapy.spider.spiders
diff --git a/sep/sep-003.rst b/sep/sep-003.rst
index 184839525..e6357313d 100644
--- a/sep/sep-003.rst
+++ b/sep/sep-003.rst
@@ -18,7 +18,7 @@ Prerequisites
 
 This API proposal relies on the following API:
 
-1. instantiating a item with an item instance as its first argument (ie.
+1. instantiating a item with an item instance as its first argument (i.e.
    ``item2 = MyItem(item1)``) must return a **copy** of the first item
    instance)
 2. items can be instantiated using this syntax: ``item = Item(attr1=value1,
@@ -78,7 +78,7 @@ Defining an item containing ItemField's
        variants2 = ListField(ItemField(Variant), default=[])
 
 It's important to note here that the (perhaps most intuitive) way of defining a
-Product-Variant relationship (ie. defining a recursive !ItemField) doesn't
+Product-Variant relationship (i.e. defining a recursive !ItemField) doesn't
 work. For example, this fails to compile:
 
 ::
diff --git a/sep/sep-013.rst b/sep/sep-013.rst
index 5b18b7501..4bc9abd30 100644
--- a/sep/sep-013.rst
+++ b/sep/sep-013.rst
@@ -59,7 +59,7 @@ Global changes to all middlewares
 
 To be discussed:
 
-1. should we support returning deferreds (ie. ``maybeDeferred``) in middleware
+1. should we support returning deferreds (i.e. ``maybeDeferred``) in middleware
    methods?
 2. should we pass Twisted Failures instead of exceptions to error methods?
 
diff --git a/sep/sep-021.rst b/sep/sep-021.rst
index 628a95dd2..372429791 100644
--- a/sep/sep-021.rst
+++ b/sep/sep-021.rst
@@ -38,7 +38,7 @@ Goals:
 
 * simple to manage: adding or removing extensions should be just a matter of
   adding or removing lines in a ``scrapy.cfg`` file
-* backward compatibility with enabling extension the "old way" (ie. modifying
+* backward compatibility with enabling extension the "old way" (i.e. modifying
   settings directly)
 
 Non-goals:

From eb21dae5240d2b66feb72940cdd141dba31ecd7a Mon Sep 17 00:00:00 2001
From: Marc Hernandez Cabot <noviluni@gmail.com>
Date: Wed, 19 Feb 2020 17:49:42 +0100
Subject: [PATCH 2654/2974] deprecare sel shortcut in scrapy shell

---
 scrapy/shell.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/scrapy/shell.py b/scrapy/shell.py
index a23b04df9..e1b4a024e 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -126,7 +126,6 @@ class Shell(object):
         self.vars['spider'] = spider
         self.vars['request'] = request
         self.vars['response'] = response
-        self.vars['sel'] = _SelectorProxy(response)
         if self.inthread:
             self.vars['fetch'] = self.fetch
         self.vars['view'] = open_in_browser
@@ -192,15 +191,3 @@ def _request_deferred(request):
 
     request.callback, request.errback = d.callback, d.errback
     return d
-
-
-class _SelectorProxy(object):
-
-    def __init__(self, response):
-        self._proxiedresponse = response
-
-    def __getattr__(self, name):
-        warnings.warn('"sel" shortcut is deprecated. Use "response.xpath()", '
-                      '"response.css()" or "response.selector" instead',
-                      category=ScrapyDeprecationWarning, stacklevel=2)
-        return getattr(self._proxiedresponse.selector, name)

From 6972a197073af11bcb582cc03f6286fceda5ca6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 19 Feb 2020 18:59:09 +0100
Subject: [PATCH 2655/2974] Remove unused imports

---
 scrapy/shell.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/shell.py b/scrapy/shell.py
index e1b4a024e..e22c48dc5 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -5,14 +5,13 @@ See documentation in docs/topics/shell.rst
 """
 import os
 import signal
-import warnings
 
 from twisted.internet import threads, defer
 from twisted.python import threadable
 from w3lib.url import any_to_uri
 
 from scrapy.crawler import Crawler
-from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
+from scrapy.exceptions import IgnoreRequest
 from scrapy.http import Request, Response
 from scrapy.item import BaseItem
 from scrapy.settings import Settings

From 0f78a591f8796686dc65854c250d6ef5324024aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 19 Feb 2020 19:09:39 +0100
Subject: [PATCH 2656/2974] =?UTF-8?q?Fix=20Flake8-reported=20=E2=80=9CToo?=
 =?UTF-8?q?=20many=20blank=20lines=E2=80=9D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scrapy/core/scraper.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 7b62068f5..41f015017 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -18,7 +18,6 @@ from scrapy.item import BaseItem
 from scrapy.core.spidermw import SpiderMiddlewareManager
 
 
-
 logger = logging.getLogger(__name__)
 
 

From 91bbc70bc10cf326940eaf53294149444f43fb9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Fri, 21 Feb 2020 06:05:31 +0100
Subject: [PATCH 2657/2974] fix E30X flake8 (#4355)

---
 pytest.ini                                   | 67 ++++++++++----------
 scrapy/core/downloader/tls.py                |  1 +
 scrapy/core/engine.py                        |  1 +
 scrapy/responsetypes.py                      |  1 +
 scrapy/utils/console.py                      |  1 +
 scrapy/utils/gz.py                           |  1 +
 tests/test_command_parse.py                  |  1 -
 tests/test_crawler.py                        |  1 +
 tests/test_dependencies.py                   |  1 +
 tests/test_downloadermiddleware_cookies.py   |  1 -
 tests/test_downloadermiddleware_httpcache.py |  1 +
 tests/test_downloadermiddleware_redirect.py  |  4 +-
 tests/test_exporters.py                      |  1 +
 tests/test_http_response.py                  |  1 +
 tests/test_item.py                           |  1 +
 tests/test_mail.py                           |  1 +
 tests/test_pipeline_files.py                 |  1 -
 tests/test_pipeline_images.py                |  1 -
 tests/test_pipeline_media.py                 |  1 +
 tests/test_responsetypes.py                  |  1 +
 tests/test_utils_conf.py                     |  1 -
 tests/test_utils_defer.py                    |  2 +
 tests/test_utils_deprecate.py                |  3 +
 tests/test_utils_iterators.py                |  1 -
 tests/test_utils_python.py                   |  3 +-
 tests/test_utils_request.py                  |  1 +
 tests/test_utils_template.py                 |  1 +
 tests/test_utils_url.py                      |  1 +
 tests/test_webclient.py                      |  1 +
 29 files changed, 58 insertions(+), 45 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 552829d4e..0758d2f8b 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -47,17 +47,17 @@ flake8-ignore =
     scrapy/contracts/__init__.py E501 W504
     scrapy/contracts/default.py E128
     # scrapy/core
-    scrapy/core/engine.py E501 E128 E127 E306 E502
+    scrapy/core/engine.py E501 E128 E127 E502
     scrapy/core/scheduler.py E501
-    scrapy/core/scraper.py E501 E306 E128 W504
+    scrapy/core/scraper.py E501 E128 W504
     scrapy/core/spidermw.py E501 E731 E126 E226
     scrapy/core/downloader/__init__.py E501
     scrapy/core/downloader/contextfactory.py E501 E128 E126
     scrapy/core/downloader/middleware.py E501 E502
-    scrapy/core/downloader/tls.py E501 E305 E241
+    scrapy/core/downloader/tls.py E501 E241
     scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226
     scrapy/core/downloader/handlers/__init__.py E501
-    scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
+    scrapy/core/downloader/handlers/ftp.py E501 E128 E127
     scrapy/core/downloader/handlers/http10.py E501
     scrapy/core/downloader/handlers/http11.py E501
     scrapy/core/downloader/handlers/s3.py E501 E128 E126
@@ -76,7 +76,7 @@ flake8-ignore =
     scrapy/extensions/closespider.py E501 E128 E123
     scrapy/extensions/corestats.py E501
     scrapy/extensions/feedexport.py E128 E501
-    scrapy/extensions/httpcache.py E128 E501 E303
+    scrapy/extensions/httpcache.py E128 E501
     scrapy/extensions/memdebug.py E501
     scrapy/extensions/spiderstate.py E501
     scrapy/extensions/telnet.py E501 W504
@@ -121,12 +121,11 @@ flake8-ignore =
     scrapy/utils/asyncio.py E501
     scrapy/utils/benchserver.py E501
     scrapy/utils/conf.py E402 E501
-    scrapy/utils/console.py E306 E305
     scrapy/utils/datatypes.py E501 E226
     scrapy/utils/decorators.py E501
     scrapy/utils/defer.py E501 E128
     scrapy/utils/deprecate.py E128 E501 E127 E502
-    scrapy/utils/gz.py E305 E501 W504
+    scrapy/utils/gz.py E501 W504
     scrapy/utils/http.py F403 E226
     scrapy/utils/httpobj.py E501
     scrapy/utils/iterators.py E501 E701
@@ -161,7 +160,7 @@ flake8-ignore =
     scrapy/middleware.py E128 E501
     scrapy/pqueues.py E501
     scrapy/resolver.py E501
-    scrapy/responsetypes.py E128 E501 E305
+    scrapy/responsetypes.py E128 E501
     scrapy/robotstxt.py E501
     scrapy/shell.py E501
     scrapy/signalmanager.py E501
@@ -175,50 +174,50 @@ flake8-ignore =
     tests/spiders.py E501 E127
     tests/test_closespider.py E501 E127
     tests/test_command_fetch.py E501
-    tests/test_command_parse.py E501 E128 E303 E226
+    tests/test_command_parse.py E501 E128 E226
     tests/test_command_shell.py E501 E128
     tests/test_commands.py E128 E501
     tests/test_contracts.py E501 E128
     tests/test_crawl.py E501 E741 E265
-    tests/test_crawler.py F841 E306 E501
-    tests/test_dependencies.py F841 E501 E305
+    tests/test_crawler.py F841 E501
+    tests/test_dependencies.py F841 E501
     tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
-    tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
+    tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126
     tests/test_downloadermiddleware_decompression.py E127
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
-    tests/test_downloadermiddleware_httpcache.py E501 E305
+    tests/test_downloadermiddleware_httpcache.py E501
     tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
     tests/test_downloadermiddleware_httpproxy.py E501 E128
-    tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
-    tests/test_downloadermiddleware_retry.py E501 E128 E251 E303 E126
+    tests/test_downloadermiddleware_redirect.py E501 E128 E127
+    tests/test_downloadermiddleware_retry.py E501 E128 E251 E126
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
     tests/test_dupefilters.py E221 E501 E741 E128 E124
     tests/test_engine.py E401 E501 E128
-    tests/test_exporters.py E501 E731 E306 E128 E124
+    tests/test_exporters.py E501 E731 E128 E124
     tests/test_extension_telnet.py F841
     tests/test_feedexport.py E501 F841 E241
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
     tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
-    tests/test_http_response.py E501 E301 E128 E265
-    tests/test_item.py E701 E128 F841 E306
+    tests/test_http_response.py E501 E128 E265
+    tests/test_item.py E701 E128 F841
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128 E124
-    tests/test_loader.py E501 E731 E303 E741 E128 E117 E241
+    tests/test_loader.py E501 E731 E741 E128 E117 E241
     tests/test_logformatter.py E128 E501 E122
-    tests/test_mail.py E128 E501 E305
+    tests/test_mail.py E128 E501
     tests/test_middleware.py E501 E128
     tests/test_pipeline_crawl.py E131 E501 E128 E126
-    tests/test_pipeline_files.py E501 E303 E272 E226
-    tests/test_pipeline_images.py F841 E501 E303
-    tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502
+    tests/test_pipeline_files.py E501 E272 E226
+    tests/test_pipeline_images.py F841 E501
+    tests/test_pipeline_media.py E501 E741 E731 E128 E502
     tests/test_proxy_connect.py E501 E741
     tests/test_request_cb_kwargs.py E501
-    tests/test_responsetypes.py E501 E305
+    tests/test_responsetypes.py E501
     tests/test_robotstxt_interface.py E501 E501
     tests/test_scheduler.py E501 E126 E123
     tests/test_selector.py E501 E127
@@ -230,24 +229,22 @@ flake8-ignore =
     tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
     tests/test_squeues.py E501 E701 E741
     tests/test_utils_asyncio.py E501
-    tests/test_utils_conf.py E501 E303 E128
+    tests/test_utils_conf.py E501 E128
     tests/test_utils_curl.py E501
-    tests/test_utils_datatypes.py E402 E501 E305
-    tests/test_utils_defer.py E306 E501 F841 E226
-    tests/test_utils_deprecate.py F841 E306 E501
+    tests/test_utils_datatypes.py E402 E501
+    tests/test_utils_defer.py E501 F841 E226
+    tests/test_utils_deprecate.py F841 E501
     tests/test_utils_http.py E501 E128 W504
-    tests/test_utils_iterators.py E501 E128 E129 E303 E241
+    tests/test_utils_iterators.py E501 E128 E129 E241
     tests/test_utils_log.py E741 E226
-    tests/test_utils_python.py E501 E303 E731 E701 E305
+    tests/test_utils_python.py E501 E731 E701
     tests/test_utils_reqser.py E501 E128
-    tests/test_utils_request.py E501 E128 E305
+    tests/test_utils_request.py E501 E128
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841 E731 E226
     tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_spider.py E305
-    tests/test_utils_template.py E305
-    tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
-    tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
+    tests/test_utils_url.py E501 E127 E211 E125 E501 E226 E241 E126 E123
+    tests/test_webclient.py E501 E128 E122 E402 E226 E241 E123 E126
     tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
     tests/test_spiderloader/__init__.py E128 E501
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 4ed482058..a1c881d5e 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -89,4 +89,5 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
                     'from host "{}" (exception: {})'.format(
                         self._hostnameASCII, repr(e)))
 
+
 DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 829e69993..6ab8cde6b 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -230,6 +230,7 @@ class ExecutionEngine(object):
     def _download(self, request, spider):
         slot = self.slot
         slot.add_request(request)
+
         def _on_success(response):
             assert isinstance(response, (Response, Request))
             if isinstance(response, Response):
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 91d309147..64bf93e86 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -116,4 +116,5 @@ class ResponseTypes(object):
             cls = self.from_body(body)
         return cls
 
+
 responsetypes = ResponseTypes()
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 7eb40f0ce..c7a2ace88 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -54,6 +54,7 @@ def _embed_standard_shell(namespace={}, banner=''):
     else:
         import rlcompleter  # noqa: F401
         readline.parse_and_bind("tab:complete")
+
     @wraps(_embed_standard_shell)
     def wrapper(namespace=namespace, banner=''):
         code.interact(banner=banner, local=namespace)
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 9672e28da..c291ae237 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -42,6 +42,7 @@ def gunzip(data):
                 raise
     return b''.join(output_list)
 
+
 _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
 _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search
 
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index b7035fdff..8a54d2c74 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -147,7 +147,6 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
-
     @defer.inlineCallbacks
     def test_pipelines(self):
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index f8fa26def..7bd76601d 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -107,6 +107,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
 
     def test_spider_custom_settings_log_level(self):
         log_file = self.mktemp()
+
         class MySpider(scrapy.Spider):
             name = 'spider'
             custom_settings = {
diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
index e31ccd9b5..a169acbe6 100644
--- a/tests/test_dependencies.py
+++ b/tests/test_dependencies.py
@@ -13,5 +13,6 @@ class ScrapyUtilsTest(unittest.TestCase):
             installed_version = [int(x) for x in module.__version__.split('.')[:2]]
             assert installed_version >= [0, 6], "OpenSSL >= 0.6 required"
 
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 04884fb78..051f66680 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -145,7 +145,6 @@ class CookiesMiddlewareTest(TestCase):
                 {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},
                 {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}]
 
-
         req = Request('http://scrapytest.org/', cookies=cookies)
         self.mw.process_request(req, self.spider)
 
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 9401dd66d..9b77c97a8 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -501,5 +501,6 @@ class RFC2616PolicyTest(DefaultStorageTest):
                 self.assertEqualResponse(res1, res2)
                 assert 'cached' in res2.flags
 
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index e0f145d0e..053e26fc3 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -68,7 +68,6 @@ class RedirectMiddlewareTest(unittest.TestCase):
         assert isinstance(r, Response)
         assert r is rsp
 
-
     def test_redirect_302(self):
         url = 'http://www.example.com/302'
         url2 = 'http://www.example.com/redirected2'
@@ -122,7 +121,6 @@ class RedirectMiddlewareTest(unittest.TestCase):
         del rsp.headers['Location']
         assert self.mw.process_response(req, rsp, self.spider) is rsp
 
-
     def test_max_redirect_times(self):
         self.mw.max_redirect_times = 1
         req = Request('http://scrapytest.org/302')
@@ -178,6 +176,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
     def test_request_meta_handling(self):
         url = 'http://www.example.com/301'
         url2 = 'http://www.example.com/redirected'
+
         def _test_passthrough(req):
             rsp = Response(url, headers={'Location': url2}, status=301, request=req)
             r = self.mw.process_response(req, rsp, self.spider)
@@ -316,5 +315,6 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
         response = mw.process_response(req, rsp, self.spider)
         assert isinstance(response, Response)
 
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 5d1f5c182..6e2507508 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -312,6 +312,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
                         for child in children]
             else:
                 return [(elem.tag, [(elem.text, ())])]
+
         def xmlsplit(xmlcontent):
             doc = lxml.etree.fromstring(xmlcontent)
             return xmltuple(doc)
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 0dc603923..be17dfd6b 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -182,6 +182,7 @@ class BaseResponseTest(unittest.TestCase):
     def test_follow_whitespace_link(self):
         self._assert_followed_url(Link('http://example.com/foo '),
                                   'http://example.com/foo%20')
+
     def test_follow_flags(self):
         res = self.response_class('http://example.com/')
         fol = res.follow('http://example.com/', flags=['cached', 'allowed'])
diff --git a/tests/test_item.py b/tests/test_item.py
index 30463a0f5..823bf1ced 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -259,6 +259,7 @@ class ItemTest(unittest.TestCase):
         with catch_warnings(record=True) as warnings:
             item = Item()
             self.assertEqual(len(warnings), 0)
+
             class SubclassedItem(Item):
                 pass
             subclassed_item = SubclassedItem()
diff --git a/tests/test_mail.py b/tests/test_mail.py
index ddb0f1e70..f5cb81a8b 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -121,5 +121,6 @@ class MailSenderTest(unittest.TestCase):
         self.assertEqual(text.get_charset(), Charset('utf-8'))
         self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body)
 
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index e5bad2ed0..88ce1cf18 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -286,7 +286,6 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
         self.assertEqual(pipeline.files_result_field, "this")
         self.assertEqual(pipeline.files_urls_field, "that")
 
-
     def test_user_defined_subclass_default_key_names(self):
         """Test situation when user defines subclass of FilesPipeline,
         but uses attribute names for default pipeline (without prefixing
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 7f1cb4a11..5018d6802 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -177,7 +177,6 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
         IMAGES_RESULT_FIELD='images'
     )
 
-
     def setUp(self):
         self.tempdir = mkdtemp()
 
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 1fcc5799e..d369e147d 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -304,6 +304,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
             return response
 
         rsp1 = Response('http://url')
+
         def rsp1_func():
             dfd = Deferred().addCallback(_check_downloading)
             reactor.callLater(.1, dfd.callback, rsp1)
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index d5a3371ab..8cdf7a176 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -90,5 +90,6 @@ class ResponseTypesTest(unittest.TestCase):
         # check that mime.types files shipped with scrapy are loaded
         self.assertEqual(responsetypes.mimetypes.guess_type('x.scrapytest')[0], 'x-scrapy/test')
 
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 02d8ba51e..61e110845 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -83,7 +83,6 @@ class BuildComponentListTest(unittest.TestCase):
         self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
 
 
-
 class UtilsConfTestCase(unittest.TestCase):
 
     def test_arglist_to_dict(self):
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index dfbe71ae2..89b5fb4fb 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -9,6 +9,7 @@ from scrapy.utils.defer import mustbe_deferred, process_chain, \
 class MustbeDeferredTest(unittest.TestCase):
     def test_success_function(self):
         steps = []
+
         def _append(v):
             steps.append(v)
             return steps
@@ -20,6 +21,7 @@ class MustbeDeferredTest(unittest.TestCase):
 
     def test_unfired_deferred(self):
         steps = []
+
         def _append(v):
             steps.append(v)
             dfd = defer.Deferred()
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 159ef8f25..b3a90d314 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -110,6 +110,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
         # ignore subclassing warnings
         with warnings.catch_warnings():
             warnings.simplefilter('ignore', ScrapyDeprecationWarning)
+
             class UserClass(Deprecated):
                 pass
 
@@ -233,6 +234,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
 
         with warnings.catch_warnings(record=True) as w:
             AlsoDeprecated()
+
             class UserClass(AlsoDeprecated):
                 pass
 
@@ -247,6 +249,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
         with mock.patch('inspect.stack', side_effect=IndexError):
             with warnings.catch_warnings(record=True) as w:
                 DeprecatedName = create_deprecated_class('DeprecatedName', NewName)
+
                 class SubClass(DeprecatedName):
                     pass
 
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 9776dfb2a..33fc4d570 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -387,7 +387,6 @@ class TestHelper(unittest.TestCase):
             self.assertTrue(type(r1) is type(r2))
             self.assertTrue(type(r1) is not type(r3))
 
-
     def _assert_type_and_value(self, a, b, obj):
         self.assertTrue(type(a) is type(b),
                         'Got {}, expected {} for {!r}'.format(type(a), type(b), obj))
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index b79e0ac1c..4202e8c89 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -104,7 +104,6 @@ class BinaryIsTextTest(unittest.TestCase):
         assert not binary_is_text(b"\x02\xa3")
 
 
-
 class UtilsPythonTestCase(unittest.TestCase):
 
     def test_equal_attributes(self):
@@ -215,7 +214,6 @@ class UtilsPythonTestCase(unittest.TestCase):
             self.assertEqual(
                 get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
 
-
     def test_without_none_values(self):
         self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
         self.assertEqual(without_none_values((1, None, 3, 4)), (1, 3, 4))
@@ -223,5 +221,6 @@ class UtilsPythonTestCase(unittest.TestCase):
             without_none_values({'one': 1, 'none': None, 'three': 3, 'four': 4}),
             {'one': 1, 'three': 3, 'four': 4})
 
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 3e664fc74..45f0f59e4 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -83,5 +83,6 @@ class UtilsRequestTest(unittest.TestCase):
         request_httprepr(Request("file:///tmp/foo.txt"))
         request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
 
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index 40b733233..5a52dd695 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -38,5 +38,6 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
         os.remove(render_path)
         assert not os.path.exists(render_path)  # Failure of test iself
 
+
 if '__main__' == __name__:
     unittest.main()
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 21e9a056a..9f1acbc75 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -201,6 +201,7 @@ def create_skipped_scheme_t(args):
         assert url.startswith(args[1])
     return do_expected
 
+
 for k, args in enumerate ([
             ('/index',                              'file://'),
             ('/index.html',                         'file://'),
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 746367b41..b602a3ea0 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -294,6 +294,7 @@ class WebClientTestCase(unittest.TestCase):
         finished = self.assertFailure(
             getPage(self.getURL("wait"), timeout=0.000001),
             defer.TimeoutError)
+
         def cleanup(passthrough):
             # Clean up the server which is hanging around not doing
             # anything.

From 6fb85951ce3843156a801e71441a4a3e387588e2 Mon Sep 17 00:00:00 2001
From: Marc Hernandez Cabot <noviluni@gmail.com>
Date: Thu, 20 Feb 2020 16:32:58 +0100
Subject: [PATCH 2658/2974] fix E22X flake8

---
 pytest.ini                                  | 48 ++++++++++-----------
 scrapy/commands/parse.py                    |  6 +--
 scrapy/core/downloader/webclient.py         |  2 +-
 scrapy/core/spidermw.py                     |  6 +--
 scrapy/downloadermiddlewares/ajaxcrawl.py   |  2 +-
 scrapy/exporters.py                         |  4 +-
 scrapy/linkextractors/lxmlhtml.py           |  2 +-
 scrapy/settings/default_settings.py         |  4 +-
 scrapy/spiderloader.py                      |  2 +-
 scrapy/utils/datatypes.py                   |  6 +--
 scrapy/utils/http.py                        |  2 +-
 scrapy/utils/misc.py                        |  2 +-
 scrapy/utils/reactor.py                     |  2 +-
 tests/pipelines.py                          |  4 +-
 tests/test_command_parse.py                 |  8 ++--
 tests/test_downloader_handlers.py           |  4 +-
 tests/test_dupefilters.py                   | 10 ++---
 tests/test_pipeline_files.py                |  2 +-
 tests/test_spidermiddleware.py              |  2 +-
 tests/test_spidermiddleware_output_chain.py |  2 +-
 tests/test_utils_defer.py                   |  2 +-
 tests/test_utils_log.py                     |  2 +-
 tests/test_utils_signal.py                  |  2 +-
 tests/test_utils_url.py                     |  2 +-
 tests/test_webclient.py                     | 10 ++---
 25 files changed, 69 insertions(+), 69 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 0758d2f8b..7806620d5 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -37,7 +37,7 @@ flake8-ignore =
     scrapy/commands/edit.py E501
     scrapy/commands/fetch.py E401 E501 E128 E731
     scrapy/commands/genspider.py E128 E501 E502
-    scrapy/commands/parse.py E128 E501 E731 E226
+    scrapy/commands/parse.py E128 E501 E731
     scrapy/commands/runspider.py E501
     scrapy/commands/settings.py E128
     scrapy/commands/shell.py E128 E501 E502
@@ -50,19 +50,19 @@ flake8-ignore =
     scrapy/core/engine.py E501 E128 E127 E502
     scrapy/core/scheduler.py E501
     scrapy/core/scraper.py E501 E128 W504
-    scrapy/core/spidermw.py E501 E731 E126 E226
+    scrapy/core/spidermw.py E501 E731 E126
     scrapy/core/downloader/__init__.py E501
     scrapy/core/downloader/contextfactory.py E501 E128 E126
     scrapy/core/downloader/middleware.py E501 E502
     scrapy/core/downloader/tls.py E501 E241
-    scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226
+    scrapy/core/downloader/webclient.py E731 E501 E128 E126
     scrapy/core/downloader/handlers/__init__.py E501
     scrapy/core/downloader/handlers/ftp.py E501 E128 E127
     scrapy/core/downloader/handlers/http10.py E501
     scrapy/core/downloader/handlers/http11.py E501
     scrapy/core/downloader/handlers/s3.py E501 E128 E126
     # scrapy/downloadermiddlewares
-    scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
+    scrapy/downloadermiddlewares/ajaxcrawl.py E501
     scrapy/downloadermiddlewares/decompression.py E501
     scrapy/downloadermiddlewares/defaultheaders.py E501
     scrapy/downloadermiddlewares/httpcache.py E501 E126
@@ -91,7 +91,7 @@ flake8-ignore =
     scrapy/http/response/text.py E501 E128 E124
     # scrapy/linkextractors
     scrapy/linkextractors/__init__.py E731 E501 E402 W504
-    scrapy/linkextractors/lxmlhtml.py E501 E731 E226
+    scrapy/linkextractors/lxmlhtml.py E501 E731
     # scrapy/loader
     scrapy/loader/__init__.py E501 E128
     scrapy/loader/processors.py E501
@@ -105,7 +105,7 @@ flake8-ignore =
     scrapy/selector/unified.py E501 E111
     # scrapy/settings
     scrapy/settings/__init__.py E501
-    scrapy/settings/default_settings.py E501 E114 E116 E226
+    scrapy/settings/default_settings.py E501 E114 E116
     scrapy/settings/deprecated.py E501
     # scrapy/spidermiddlewares
     scrapy/spidermiddlewares/httperror.py E501
@@ -121,21 +121,21 @@ flake8-ignore =
     scrapy/utils/asyncio.py E501
     scrapy/utils/benchserver.py E501
     scrapy/utils/conf.py E402 E501
-    scrapy/utils/datatypes.py E501 E226
+    scrapy/utils/datatypes.py E501
     scrapy/utils/decorators.py E501
     scrapy/utils/defer.py E501 E128
     scrapy/utils/deprecate.py E128 E501 E127 E502
     scrapy/utils/gz.py E501 W504
-    scrapy/utils/http.py F403 E226
+    scrapy/utils/http.py F403
     scrapy/utils/httpobj.py E501
     scrapy/utils/iterators.py E501 E701
     scrapy/utils/log.py E128 E501
     scrapy/utils/markup.py F403
-    scrapy/utils/misc.py E501 E226
+    scrapy/utils/misc.py E501
     scrapy/utils/multipart.py F403
     scrapy/utils/project.py E501
     scrapy/utils/python.py E501
-    scrapy/utils/reactor.py E226 E501
+    scrapy/utils/reactor.py E501
     scrapy/utils/reqser.py E501
     scrapy/utils/request.py E127 E501
     scrapy/utils/response.py E501 E128
@@ -151,7 +151,7 @@ flake8-ignore =
     scrapy/crawler.py E501
     scrapy/dupefilters.py E501 E202
     scrapy/exceptions.py E501
-    scrapy/exporters.py E501 E226
+    scrapy/exporters.py E501
     scrapy/interfaces.py E501
     scrapy/item.py E501 E128
     scrapy/link.py E501
@@ -164,24 +164,24 @@ flake8-ignore =
     scrapy/robotstxt.py E501
     scrapy/shell.py E501
     scrapy/signalmanager.py E501
-    scrapy/spiderloader.py E225 F841 E501 E126
+    scrapy/spiderloader.py F841 E501 E126
     scrapy/squeues.py E128
     scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501
     tests/mockserver.py E401 E501 E126 E123
-    tests/pipelines.py F841 E226
+    tests/pipelines.py F841
     tests/spiders.py E501 E127
     tests/test_closespider.py E501 E127
     tests/test_command_fetch.py E501
-    tests/test_command_parse.py E501 E128 E226
+    tests/test_command_parse.py E501 E128
     tests/test_command_shell.py E501 E128
     tests/test_commands.py E128 E501
     tests/test_contracts.py E501 E128
     tests/test_crawl.py E501 E741 E265
     tests/test_crawler.py F841 E501
     tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123
+    tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E701 E126 E123
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
     tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126
@@ -195,7 +195,7 @@ flake8-ignore =
     tests/test_downloadermiddleware_retry.py E501 E128 E251 E126
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
-    tests/test_dupefilters.py E221 E501 E741 E128 E124
+    tests/test_dupefilters.py E501 E741 E128 E124
     tests/test_engine.py E401 E501 E128
     tests/test_exporters.py E501 E731 E128 E124
     tests/test_extension_telnet.py F841
@@ -212,7 +212,7 @@ flake8-ignore =
     tests/test_mail.py E128 E501
     tests/test_middleware.py E501 E128
     tests/test_pipeline_crawl.py E131 E501 E128 E126
-    tests/test_pipeline_files.py E501 E272 E226
+    tests/test_pipeline_files.py E501 E272
     tests/test_pipeline_images.py F841 E501
     tests/test_pipeline_media.py E501 E741 E731 E128 E502
     tests/test_proxy_connect.py E501 E741
@@ -222,29 +222,29 @@ flake8-ignore =
     tests/test_scheduler.py E501 E126 E123
     tests/test_selector.py E501 E127
     tests/test_spider.py E501
-    tests/test_spidermiddleware.py E501 E226
+    tests/test_spidermiddleware.py E501
     tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
     tests/test_spidermiddleware_offsite.py E501 E128 E111
-    tests/test_spidermiddleware_output_chain.py E501 E226
+    tests/test_spidermiddleware_output_chain.py E501
     tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
     tests/test_squeues.py E501 E701 E741
     tests/test_utils_asyncio.py E501
     tests/test_utils_conf.py E501 E128
     tests/test_utils_curl.py E501
     tests/test_utils_datatypes.py E402 E501
-    tests/test_utils_defer.py E501 F841 E226
+    tests/test_utils_defer.py E501 F841
     tests/test_utils_deprecate.py F841 E501
     tests/test_utils_http.py E501 E128 W504
     tests/test_utils_iterators.py E501 E128 E129 E241
-    tests/test_utils_log.py E741 E226
+    tests/test_utils_log.py E741
     tests/test_utils_python.py E501 E731 E701
     tests/test_utils_reqser.py E501 E128
     tests/test_utils_request.py E501 E128
     tests/test_utils_response.py E501
-    tests/test_utils_signal.py E741 F841 E731 E226
+    tests/test_utils_signal.py E741 F841 E731
     tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_url.py E501 E127 E211 E125 E501 E226 E241 E126 E123
-    tests/test_webclient.py E501 E128 E122 E402 E226 E241 E123 E126
+    tests/test_utils_url.py E501 E127 E211 E125 E501 E241 E126 E123
+    tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126
     tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
     tests/test_spiderloader/__init__.py E128 E501
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index ff6f1d8cd..3ef8ddcb3 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -80,7 +80,7 @@ class Command(ScrapyCommand):
         else:
             items = self.items.get(lvl, [])
 
-        print("# Scraped Items ", "-"*60)
+        print("# Scraped Items ", "-" * 60)
         display.pprint([dict(x) for x in items], colorize=colour)
 
     def print_requests(self, lvl=None, colour=True):
@@ -92,14 +92,14 @@ class Command(ScrapyCommand):
         else:
             requests = self.requests.get(lvl, [])
 
-        print("# Requests ", "-"*65)
+        print("# Requests ", "-" * 65)
         display.pprint(requests, colorize=colour)
 
     def print_results(self, opts):
         colour = not opts.nocolour
 
         if opts.verbose:
-            for level in range(1, self.max_level+1):
+            for level in range(1, self.max_level + 1):
                 print('\n>>> DEPTH LEVEL: %s <<<' % level)
                 if not opts.noitems:
                     self.print_items(level, colour)
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index fc796e8bb..a71dc5fb3 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -140,7 +140,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
             self.headers['Content-Length'] = 0
 
     def _build_response(self, body, request):
-        request.meta['download_latency'] = self.headers_time-self.start_time
+        request.meta['download_latency'] = self.headers_time - self.start_time
         status = int(self.status)
         headers = Headers(self.response_headers)
         respcls = responsetypes.from_args(headers=headers, url=self._url)
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index dd9b3c376..87d08cab7 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -82,7 +82,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 if _isiterable(result):
                     # stop exception handling by handing control over to the
                     # process_spider_output chain if an iterable has been returned
-                    return process_spider_output(result, method_index+1)
+                    return process_spider_output(result, method_index + 1)
                 elif result is None:
                     continue
                 else:
@@ -103,12 +103,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
                     # might fail directly if the output value is not a generator
                     result = method(response=response, result=result, spider=spider)
                 except Exception as ex:
-                    exception_result = process_spider_exception(Failure(ex), method_index+1)
+                    exception_result = process_spider_exception(Failure(ex), method_index + 1)
                     if isinstance(exception_result, Failure):
                         raise
                     return exception_result
                 if _isiterable(result):
-                    result = _evaluate_iterable(result, method_index+1, recovered)
+                    result = _evaluate_iterable(result, method_index + 1, recovered)
                 else:
                     msg = "Middleware {} must return an iterable, got {}"
                     raise _InvalidOutput(msg.format(_fname(method), type(result)))
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 7a140fcad..16b046e99 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -47,7 +47,7 @@ class AjaxCrawlMiddleware(object):
             return response
 
         # scrapy already handles #! links properly
-        ajax_crawl_request = request.replace(url=request.url+'#!')
+        ajax_crawl_request = request.replace(url=request.url + '#!')
         logger.debug("Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s",
                      {'ajax_crawl_request': ajax_crawl_request, 'request': request},
                      extra={'spider': spider})
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 1a3c9345f..96416f075 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -173,12 +173,12 @@ class XmlItemExporter(BaseItemExporter):
         if hasattr(serialized_value, 'items'):
             self._beautify_newline()
             for subname, value in serialized_value.items():
-                self._export_xml_field(subname, value, depth=depth+1)
+                self._export_xml_field(subname, value, depth=depth + 1)
             self._beautify_indent(depth=depth)
         elif is_listlike(serialized_value):
             self._beautify_newline()
             for value in serialized_value:
-                self._export_xml_field('value', value, depth=depth+1)
+                self._export_xml_field('value', value, depth=depth + 1)
             self._beautify_indent(depth=depth)
         elif isinstance(serialized_value, str):
             self.xg.characters(serialized_value)
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index f5ef56ea4..ab82e1915 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -22,7 +22,7 @@ _collect_string_content = etree.XPath("string()")
 
 def _nons(tag):
     if isinstance(tag, str):
-        if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
+        if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE:
             return tag.split('}')[-1]
     return tag
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index fc7b62e78..f8a0457ce 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -75,8 +75,8 @@ DOWNLOAD_HANDLERS_BASE = {
 
 DOWNLOAD_TIMEOUT = 180      # 3mins
 
-DOWNLOAD_MAXSIZE = 1024*1024*1024   # 1024m
-DOWNLOAD_WARNSIZE = 32*1024*1024    # 32m
+DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024   # 1024m
+DOWNLOAD_WARNSIZE = 32 * 1024 * 1024    # 32m
 
 DOWNLOAD_FAIL_ON_DATALOSS = True
 
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 3beca4060..048e84e4f 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -28,7 +28,7 @@ class SpiderLoader(object):
                                 module=mod, cls=cls, name=name)
                            for (mod, cls) in locations)
                  for name, locations in self._found.items()
-                 if len(locations)>1]
+                 if len(locations) > 1]
         if dupes:
             msg = ("There are several spiders with the same name:\n\n"
                    "{}\n\n  This can cause unexpected behavior.".format(
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index a52bbc70e..b07f995cf 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -175,12 +175,12 @@ class SiteNode(object):
         node.parent = self
 
     def to_string(self, level=0):
-        s = "%s%s\n" % ('  '*level, self.url)
+        s = "%s%s\n" % ('  ' * level, self.url)
         if self.itemnames:
             for n in self.itemnames:
-                s += "%sScraped: %s\n" % ('  '*(level+1), n)
+                s += "%sScraped: %s\n" % ('  ' * (level + 1), n)
         for node in self.children:
-            s += node.to_string(level+1)
+            s += node.to_string(level + 1)
         return s
 
 
diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py
index bab262393..ceb3f0509 100644
--- a/scrapy/utils/http.py
+++ b/scrapy/utils/http.py
@@ -32,5 +32,5 @@ def decode_chunked_transfer(chunked_body):
             break
         size = int(h, 16)
         body += t[:size]
-        t = t[size+2:]
+        t = t[size + 2:]
     return body
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index a3e55d6ea..52cfba208 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -46,7 +46,7 @@ def load_object(path):
     except ValueError:
         raise ValueError("Error loading object '%s': not a full path" % path)
 
-    module, name = path[:dot], path[dot+1:]
+    module, name = path[:dot], path[dot + 1:]
     mod = import_module(module)
 
     try:
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 80f52a4ef..6513e06c9 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -16,7 +16,7 @@ def listen_tcp(portrange, host, factory):
         return reactor.listenTCP(portrange, factory, interface=host)
     if len(portrange) == 1:
         return reactor.listenTCP(portrange[0], factory, interface=host)
-    for x in range(portrange[0], portrange[1]+1):
+    for x in range(portrange[0], portrange[1] + 1):
         try:
             return reactor.listenTCP(x, factory, interface=host)
         except error.CannotListenError:
diff --git a/tests/pipelines.py b/tests/pipelines.py
index d7d3b5259..de4894c32 100644
--- a/tests/pipelines.py
+++ b/tests/pipelines.py
@@ -6,7 +6,7 @@ Some pipelines used for testing
 class ZeroDivisionErrorPipeline(object):
 
     def open_spider(self, spider):
-        a = 1/0
+        a = 1 / 0
 
     def process_item(self, item, spider):
         return item
@@ -15,4 +15,4 @@ class ZeroDivisionErrorPipeline(object):
 class ProcessWithZeroDivisionErrorPipiline(object):
 
     def process_item(self, item, spider):
-        1/0
+        1 / 0
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 8a54d2c74..5bf92b71a 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -182,7 +182,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
     def test_crawlspider_matching_rule_callback_set(self):
         """If a rule matches the URL, use it's defined callback."""
         status, out, stderr = yield self.execute(
-            ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')]
+            ['--spider', 'goodcrawl' + self.spider_name, '-r', self.url('/html')]
         )
         self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
 
@@ -190,7 +190,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
     def test_crawlspider_matching_rule_default_callback(self):
         """If a rule match but it has no callback set, use the 'parse' callback."""
         status, out, stderr = yield self.execute(
-            ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')]
+            ['--spider', 'goodcrawl' + self.spider_name, '-r', self.url('/text')]
         )
         self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out))
 
@@ -206,7 +206,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
     @defer.inlineCallbacks
     def test_crawlspider_missing_callback(self):
         status, out, stderr = yield self.execute(
-            ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')]
+            ['--spider', 'badcrawl' + self.spider_name, '-r', self.url('/html')]
         )
         self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
 
@@ -214,7 +214,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
     def test_crawlspider_no_matching_rule(self):
         """The requested URL has no matching rule, so no items should be scraped"""
         status, out, stderr = yield self.execute(
-            ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')]
+            ['--spider', 'badcrawl' + self.spider_name, '-r', self.url('/enc-gb18030')]
         )
         self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 8d95d7cac..29d06bab4 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -348,7 +348,7 @@ class HttpTestCase(unittest.TestCase):
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
     def test_payload(self):
-        body = b'1'*100  # PayloadResource requires body length to be 100
+        body = b'1' * 100  # PayloadResource requires body length to be 100
         request = Request(self.getURL('payload'), method='POST', body=body)
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.body)
@@ -812,7 +812,7 @@ class S3TestCase(unittest.TestCase):
 
     def test_request_signing1(self):
         # gets an object from the johnsmith bucket.
-        date ='Tue, 27 Mar 2007 19:36:42 +0000'
+        date = 'Tue, 27 Mar 2007 19:36:42 +0000'
         req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date})
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 88ce9627f..9e24d86dd 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -43,7 +43,7 @@ class RFPDupeFilterTest(unittest.TestCase):
 
     def test_df_from_crawler_scheduler(self):
         settings = {'DUPEFILTER_DEBUG': True,
-                    'DUPEFILTER_CLASS': __name__  + '.FromCrawlerRFPDupeFilter'}
+                    'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
@@ -51,14 +51,14 @@ class RFPDupeFilterTest(unittest.TestCase):
 
     def test_df_from_settings_scheduler(self):
         settings = {'DUPEFILTER_DEBUG': True,
-                    'DUPEFILTER_CLASS': __name__  + '.FromSettingsRFPDupeFilter'}
+                    'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
         self.assertEqual(scheduler.df.method, 'from_settings')
 
     def test_df_direct_scheduler(self):
-        settings = {'DUPEFILTER_CLASS': __name__  + '.DirectDupeFilter'}
+        settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertEqual(scheduler.df.method, 'n/a')
@@ -162,7 +162,7 @@ class RFPDupeFilterTest(unittest.TestCase):
     def test_log(self):
         with LogCapture() as l:
             settings = {'DUPEFILTER_DEBUG': False,
-                        'DUPEFILTER_CLASS': __name__  + '.FromCrawlerRFPDupeFilter'}
+                        'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
             scheduler = Scheduler.from_crawler(crawler)
             spider = SimpleSpider.from_crawler(crawler)
@@ -187,7 +187,7 @@ class RFPDupeFilterTest(unittest.TestCase):
     def test_log_debug(self):
         with LogCapture() as l:
             settings = {'DUPEFILTER_DEBUG': True,
-                        'DUPEFILTER_CLASS': __name__  + '.FromCrawlerRFPDupeFilter'}
+                        'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
             scheduler = Scheduler.from_crawler(crawler)
             spider = SimpleSpider.from_crawler(crawler)
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 88ce1cf18..799782647 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -359,7 +359,7 @@ class TestGCSFilesStore(unittest.TestCase):
         self.assertIn('checksum', s)
         self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==')
         u = urlparse(uri)
-        content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path)
+        content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:] + path)
         self.assertEqual(content, data)
         self.assertEqual(blob.metadata, {'foo': 'bar'})
         self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL)
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 55d665e79..78e926adc 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -94,7 +94,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
 
         class RaiseExceptionProcessSpiderOutputMiddleware:
             def process_spider_output(self, response, result, spider):
-                1/0
+                1 / 0
 
         self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())
         self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index b19a74609..b26353d6c 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -125,7 +125,7 @@ class NotGeneratorCallbackSpider(Spider):
         yield Request(self.mockserver.url('/status?n=200'))
 
     def parse(self, response):
-        return [{'test': 1}, {'test': 1/0}]
+        return [{'test': 1}, {'test': 1 / 0}]
 
 
 # ================================================================================
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 89b5fb4fb..a3b6e64f1 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -104,7 +104,7 @@ class IterErrbackTest(unittest.TestCase):
         def iterbad():
             for x in range(10):
                 if x == 5:
-                    a = 1/0
+                    a = 1 / 0
                 yield x
 
         errors = []
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 2c23f3616..21100aeb8 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -16,7 +16,7 @@ class FailureToExcInfoTest(unittest.TestCase):
 
     def test_failure(self):
         try:
-            0/0
+            0 / 0
         except ZeroDivisionError:
             exc_info = sys.exc_info()
             failure = Failure()
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index e5f6f0ed4..9f6da09ed 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -44,7 +44,7 @@ class SendCatchLogTest(unittest.TestCase):
 
     def error_handler(self, arg, handlers_called):
         handlers_called.add(self.error_handler)
-        a = 1/0
+        a = 1 / 0
 
     def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 9f1acbc75..1e18494c3 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -30,7 +30,7 @@ class UrlUtilsTest(unittest.TestCase):
 
         url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29'
         self.assertFalse(url_is_from_any_domain(url, ['testdomain.com']))
-        self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com']))
+        self.assertFalse(url_is_from_any_domain(url + '.testdomain.com', ['testdomain.com']))
 
     def test_url_is_from_spider(self):
         spider = Spider(name='example.com')
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index b602a3ea0..99a998a46 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -51,22 +51,22 @@ class ParseUrlTestCase(unittest.TestCase):
     ("http://127.0.0.1?c=v&c2=v2#fragment",     ('http', lip, lip, 80, '/?c=v&c2=v2')),
     ("http://127.0.0.1/?c=v&c2=v2#fragment",    ('http', lip, lip, 80, '/?c=v&c2=v2')),
     ("http://127.0.0.1/foo?c=v&c2=v2#frag",     ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
-    ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip+':100', lip, 100, '/?c=v&c2=v2')),
-    ("http://127.0.0.1:100/?c=v&c2=v2#frag",    ('http', lip+':100', lip, 100, '/?c=v&c2=v2')),
-    ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip+':100', lip, 100, '/foo?c=v&c2=v2')),
+    ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
+    ("http://127.0.0.1:100/?c=v&c2=v2#frag",    ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
+    ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
 
     ("http://127.0.0.1",              ('http', lip, lip, 80, '/')),
     ("http://127.0.0.1/",             ('http', lip, lip, 80, '/')),
     ("http://127.0.0.1/foo",          ('http', lip, lip, 80, '/foo')),
     ("http://127.0.0.1?param=value",  ('http', lip, lip, 80, '/?param=value')),
     ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
-    ("http://127.0.0.1:12345/foo",    ('http', lip+':12345', lip, 12345, '/foo')),
+    ("http://127.0.0.1:12345/foo",    ('http', lip + ':12345', lip, 12345, '/foo')),
     ("http://spam:12345/foo",         ('http', 'spam:12345', 'spam', 12345, '/foo')),
     ("http://spam.test.org/foo",      ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
 
     ("https://127.0.0.1/foo",         ('https', lip, lip, 443, '/foo')),
     ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
-    ("https://127.0.0.1:12345/",      ('https', lip+':12345', lip, 12345, '/')),
+    ("https://127.0.0.1:12345/",      ('https', lip + ':12345', lip, 12345, '/')),
 
     ("http://scrapytest.org/foo ",    ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
     ("http://egg:7890 ",              ('http', 'egg:7890', 'egg', 7890, '/')),

From 03ed9e17867b8c7533d08ef28108a67305050e9a Mon Sep 17 00:00:00 2001
From: Marc Hernandez Cabot <noviluni@gmail.com>
Date: Fri, 21 Feb 2020 09:29:29 +0100
Subject: [PATCH 2659/2974] delete old deprecated functions from
 scrapy.utils.python

---
 scrapy/utils/python.py | 44 ------------------------------------------
 1 file changed, 44 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index e5582cc18..e95a4648e 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -4,7 +4,6 @@ This module contains essential stuff that should've come with Python itself ;)
 import errno
 import gc
 import inspect
-import os
 import re
 import sys
 import weakref
@@ -165,14 +164,6 @@ _BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\
 _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
 
 
-@deprecated("scrapy.utils.python.binary_is_text")
-def isbinarytext(text):
-    """ This function is deprecated.
-    Please use scrapy.utils.python.binary_is_text, which was created to be more
-    clear about the functions behavior: it is behaving inverted to this one. """
-    return not binary_is_text(text)
-
-
 def binary_is_text(data):
     """ Returns ``True`` if the given ``data`` argument (a ``bytes`` object)
     does not contain unprintable control characters.
@@ -293,41 +284,6 @@ class WeakKeyCache(object):
         return self._weakdict[key]
 
 
-@deprecated
-def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
-    """Return a (new) dict with unicode keys (and values when "keys_only" is
-    False) of the given dict converted to strings. ``dct_or_tuples`` can be a
-    dict or a list of tuples, like any dict ``__init__`` method supports.
-    """
-    d = {}
-    for k, v in dict(dct_or_tuples).items():
-        k = k.encode(encoding) if isinstance(k, str) else k
-        if not keys_only:
-            v = v.encode(encoding) if isinstance(v, str) else v
-        d[k] = v
-    return d
-
-
-@deprecated
-def is_writable(path):
-    """Return True if the given path can be written (if it exists) or created
-    (if it doesn't exist)
-    """
-    if os.path.exists(path):
-        return os.access(path, os.W_OK)
-    else:
-        return os.access(os.path.dirname(path), os.W_OK)
-
-
-@deprecated
-def setattr_default(obj, name, value):
-    """Set attribute value, but only if it's not already set. Similar to
-    setdefault() for dicts.
-    """
-    if not hasattr(obj, name):
-        setattr(obj, name, value)
-
-
 def retry_on_eintr(function, *args, **kw):
     """Run a function and retry it while getting EINTR errors"""
     while True:

From b49ece0b8781c1d53cdec77b96445826a089afc1 Mon Sep 17 00:00:00 2001
From: Marc Hernandez Cabot <noviluni@gmail.com>
Date: Fri, 21 Feb 2020 08:58:32 +0100
Subject: [PATCH 2660/2974] fix E701 and E271 flake8

---
 pytest.ini                   | 14 +++++++-------
 scrapy/utils/iterators.py    |  6 ++++--
 tests/test_item.py           | 21 ++++++++++++++-------
 tests/test_pipeline_files.py |  2 +-
 tests/test_squeues.py        |  4 +++-
 tests/test_utils_python.py   |  4 +++-
 6 files changed, 32 insertions(+), 19 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 7806620d5..acdb5a27a 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -128,7 +128,7 @@ flake8-ignore =
     scrapy/utils/gz.py E501 W504
     scrapy/utils/http.py F403
     scrapy/utils/httpobj.py E501
-    scrapy/utils/iterators.py E501 E701
+    scrapy/utils/iterators.py E501
     scrapy/utils/log.py E128 E501
     scrapy/utils/markup.py F403
     scrapy/utils/misc.py E501
@@ -141,7 +141,7 @@ flake8-ignore =
     scrapy/utils/response.py E501 E128
     scrapy/utils/signal.py E501 E128
     scrapy/utils/sitemap.py E501
-    scrapy/utils/spider.py E271 E501
+    scrapy/utils/spider.py E501
     scrapy/utils/ssl.py E501
     scrapy/utils/test.py E501
     scrapy/utils/url.py E501 F403 E128 F405
@@ -181,7 +181,7 @@ flake8-ignore =
     tests/test_crawl.py E501 E741 E265
     tests/test_crawler.py F841 E501
     tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E701 E126 E123
+    tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
     tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126
@@ -204,7 +204,7 @@ flake8-ignore =
     tests/test_http_headers.py E501
     tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
     tests/test_http_response.py E501 E128 E265
-    tests/test_item.py E701 E128 F841
+    tests/test_item.py E128 F841
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128 E124
     tests/test_loader.py E501 E731 E741 E128 E117 E241
@@ -212,7 +212,7 @@ flake8-ignore =
     tests/test_mail.py E128 E501
     tests/test_middleware.py E501 E128
     tests/test_pipeline_crawl.py E131 E501 E128 E126
-    tests/test_pipeline_files.py E501 E272
+    tests/test_pipeline_files.py E501
     tests/test_pipeline_images.py F841 E501
     tests/test_pipeline_media.py E501 E741 E731 E128 E502
     tests/test_proxy_connect.py E501 E741
@@ -227,7 +227,7 @@ flake8-ignore =
     tests/test_spidermiddleware_offsite.py E501 E128 E111
     tests/test_spidermiddleware_output_chain.py E501
     tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
-    tests/test_squeues.py E501 E701 E741
+    tests/test_squeues.py E501 E741
     tests/test_utils_asyncio.py E501
     tests/test_utils_conf.py E501 E128
     tests/test_utils_curl.py E501
@@ -237,7 +237,7 @@ flake8-ignore =
     tests/test_utils_http.py E501 E128 W504
     tests/test_utils_iterators.py E501 E128 E129 E241
     tests/test_utils_log.py E741
-    tests/test_utils_python.py E501 E731 E701
+    tests/test_utils_python.py E501 E731
     tests/test_utils_reqser.py E501 E128
     tests/test_utils_request.py E501 E128
     tests/test_utils_response.py E501
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 3c0cb68c3..7849174fb 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -101,8 +101,10 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     lines = StringIO(_body_or_str(obj, unicode=True))
 
     kwargs = {}
-    if delimiter: kwargs["delimiter"] = delimiter
-    if quotechar: kwargs["quotechar"] = quotechar
+    if delimiter:
+        kwargs["delimiter"] = delimiter
+    if quotechar:
+        kwargs["quotechar"] = quotechar
     csv_r = csv.reader(lines, **kwargs)
 
     if not headers:
diff --git a/tests/test_item.py b/tests/test_item.py
index 823bf1ced..f70632d57 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -149,13 +149,15 @@ class ItemTest(unittest.TestCase):
             fields = {'load': Field(default='A')}
             save = Field(default='A')
 
-        class B(A): pass
+        class B(A):
+            pass
 
         class C(Item):
             fields = {'load': Field(default='C')}
             save = Field(default='C')
 
-        class D(B, C): pass
+        class D(B, C):
+            pass
 
         item = D(save='X', load='Y')
         self.assertEqual(item['save'], 'X')
@@ -164,7 +166,8 @@ class ItemTest(unittest.TestCase):
             'save': {'default': 'A'}})
 
         # D class inverted
-        class E(C, B): pass
+        class E(C, B):
+            pass
 
         self.assertEqual(E(save='X')['save'], 'X')
         self.assertEqual(E(load='X')['load'], 'X')
@@ -177,7 +180,8 @@ class ItemTest(unittest.TestCase):
             save = Field(default='A')
             load = Field(default='A')
 
-        class B(A): pass
+        class B(A):
+            pass
 
         class C(A):
             fields = {'update': Field(default='C')}
@@ -206,14 +210,16 @@ class ItemTest(unittest.TestCase):
             fields = {'load': Field(default='A')}
             save = Field(default='A')
 
-        class B(A): pass
+        class B(A):
+            pass
 
         class C(object):
             fields = {'load': Field(default='C')}
             not_allowed = Field(default='not_allowed')
             save = Field(default='C')
 
-        class D(B, C): pass
+        class D(B, C):
+            pass
 
         self.assertRaises(KeyError, D, not_allowed='value')
         self.assertEqual(D(save='X')['save'], 'X')
@@ -221,7 +227,8 @@ class ItemTest(unittest.TestCase):
             'load': {'default': 'A'}})
 
         # D class inverted
-        class E(C, B): pass
+        class E(C, B):
+            pass
 
         self.assertRaises(KeyError, E, not_allowed='value')
         self.assertEqual(E(save='X')['save'], 'X')
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 799782647..f155db4ce 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -272,7 +272,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
         prefix = pipeline_cls.__name__.upper()
         settings = self._generate_fake_settings(prefix=prefix)
         user_pipeline = pipeline_cls.from_settings(Settings(settings))
-        for pipe_cls_attr, settings_attr, pipe_inst_attr  in self.file_cls_attr_settings_map:
+        for pipe_cls_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
             custom_value = settings.get(prefix + "_" + settings_attr)
             self.assertNotEqual(custom_value, self.default_cls_settings[pipe_cls_attr])
             self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value)
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index d5fcf2f7f..f6970162e 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -31,7 +31,9 @@ def nonserializable_object_test(self):
         self.assertRaises(ValueError, q.push, lambda x: x)
     else:
         # Use a different unpickleable object
-        class A(object): pass
+        class A(object):
+            pass
+
         a = A()
         a.__reduce__ = a.__reduce_ex__ = None
         self.assertRaises(ValueError, q.push, a)
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 4202e8c89..ec5b4c596 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -153,7 +153,9 @@ class UtilsPythonTestCase(unittest.TestCase):
         self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
 
     def test_weakkeycache(self):
-        class _Weakme(object): pass
+        class _Weakme(object):
+            pass
+
         _values = count()
         wk = WeakKeyCache(lambda k: next(_values))
         k = _Weakme()

From 9ad10bb6f727a3f1c5c59d490f444ebb32de97c6 Mon Sep 17 00:00:00 2001
From: Marc Hernandez Cabot <noviluni@gmail.com>
Date: Fri, 21 Feb 2020 09:05:42 +0100
Subject: [PATCH 2661/2974] fix E131

---
 pytest.ini                   |  2 +-
 tests/test_pipeline_crawl.py | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index acdb5a27a..2120264e0 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -211,7 +211,7 @@ flake8-ignore =
     tests/test_logformatter.py E128 E501 E122
     tests/test_mail.py E128 E501
     tests/test_middleware.py E501 E128
-    tests/test_pipeline_crawl.py E131 E501 E128 E126
+    tests/test_pipeline_crawl.py E501 E128 E126
     tests/test_pipeline_files.py E501
     tests/test_pipeline_images.py F841 E501
     tests/test_pipeline_media.py E501 E741 E731 E128 E502
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index fb72c9d6d..962c33144 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -26,10 +26,9 @@ class MediaDownloadSpider(SimpleSpider):
             self.media_key: [],
             self.media_urls_key: [
                 self._process_url(response.urljoin(href))
-                    for href in response.xpath('''
-                        //table[thead/tr/th="Filename"]
-                            /tbody//a/@href
-                        ''').getall()],
+                for href in response.xpath(
+                    '//table[thead/tr/th="Filename"]/tbody//a/@href'
+                ).getall()],
         }
         yield item
 
@@ -99,8 +98,9 @@ class FileDownloadCrawlTestCase(TestCase):
         if self.expected_checksums is not None:
             checksums = set(
                 i['checksum']
-                    for item in items
-                        for i in item[self.media_key])
+                for item in items
+                for i in item[self.media_key]
+            )
             self.assertEqual(checksums, self.expected_checksums)
 
         # check that the image files where actually written to the media store

From 69a8648bef6df38a5b7e79f9fbecb98869416654 Mon Sep 17 00:00:00 2001
From: Marc Hernandez Cabot <noviluni@gmail.com>
Date: Fri, 21 Feb 2020 09:13:28 +0100
Subject: [PATCH 2662/2974] fix E251

---
 pytest.ini                                         | 4 ++--
 tests/test_downloadermiddleware_httpcompression.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 2120264e0..58f1cfeb3 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -189,10 +189,10 @@ flake8-ignore =
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
     tests/test_downloadermiddleware_httpcache.py E501
-    tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
+    tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
     tests/test_downloadermiddleware_httpproxy.py E501 E128
     tests/test_downloadermiddleware_redirect.py E501 E128 E127
-    tests/test_downloadermiddleware_retry.py E501 E128 E251 E126
+    tests/test_downloadermiddleware_retry.py E501 E128 E126
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
     tests/test_dupefilters.py E501 E741 E128 E124
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 64488841a..106ca3360 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -245,7 +245,7 @@ class HttpCompressionTest(TestCase):
         response.headers['Content-Type'] = 'application/gzip'
         request = response.request
         request.method = 'HEAD'
-        response = response.replace(body = None)
+        response = response.replace(body=None)
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIs(newresponse, response)
         self.assertEqual(response.body, b'')

From 6e8e117aee4ddc5d6f6970019be212198d0b9e7a Mon Sep 17 00:00:00 2001
From: Marc Hernandez Cabot <noviluni@gmail.com>
Date: Fri, 21 Feb 2020 09:14:55 +0100
Subject: [PATCH 2663/2974] fix flake E211

---
 pytest.ini              | 2 +-
 tests/test_utils_url.py | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 58f1cfeb3..141a13a4f 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -243,7 +243,7 @@ flake8-ignore =
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841 E731
     tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_url.py E501 E127 E211 E125 E501 E241 E126 E123
+    tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123
     tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126
     tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 1e18494c3..7abff8281 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -202,7 +202,7 @@ def create_skipped_scheme_t(args):
     return do_expected
 
 
-for k, args in enumerate ([
+for k, args in enumerate([
             ('/index',                              'file://'),
             ('/index.html',                         'file://'),
             ('./index.html',                        'file://'),
@@ -230,7 +230,7 @@ for k, args in enumerate ([
         ], start=1):
     t_method = create_guess_scheme_t(args)
     t_method.__name__ = 'test_uri_%03d' % k
-    setattr (GuessSchemeTest, t_method.__name__, t_method)
+    setattr(GuessSchemeTest, t_method.__name__, t_method)
 
 # TODO: the following tests do not pass with current implementation
 for k, args in enumerate([
@@ -239,7 +239,7 @@ for k, args in enumerate([
         ], start=1):
     t_method = create_skipped_scheme_t(args)
     t_method.__name__ = 'test_uri_skipped_%03d' % k
-    setattr (GuessSchemeTest, t_method.__name__, t_method)
+    setattr(GuessSchemeTest, t_method.__name__, t_method)
 
 
 class StripUrl(unittest.TestCase):

From 67ee0b097fe15aefa787bce64f6fa085d38e69d8 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Sat, 22 Feb 2020 17:02:57 +0500
Subject: [PATCH 2664/2974] Remove specifics of downstream request queues from
 scheduler (#3884)

* move serialization/deserialization logic to downstream queues

* make memory queues conform to common interface

* make ScrapyPriorityQueue conform common interface

* ScrapyPriorityQueue works with disk

* make key as string

* return list instead of dict as earlier

* downloader aware pq works with new interface

* we don`t need these methods anymore

* create directories for files

* remove dummy priority

* remove priority as parameter, let every queue decide for itself

* rename obj to request

* DownloaderAwarePriorityQueue is too thin wrapper around _SlotPriorityQueues, just remove second one

* remove priority as parameter, let every queue decide for itself

* rename argument

* more granular class separation

* python2 compatible

* one more argument for common interface

* more simple downstream queue interface

* single place for easier customization

* rename function

* shorter

* shorter

* use named arguments

* fix typo

* add docstring

* Update scrapy/pqueues.py

Co-Authored-By: Mikhail Korobov <kmike84@gmail.com>

* Update scrapy/pqueues.py

Co-Authored-By: Mikhail Korobov <kmike84@gmail.com>

* 4 spaces indentation

* we ok with existing directories

* remove unused import

* rename method

* remove unused imports

* it has no sense now

* relining

* note about queues

* add value

* Revert "it has no sense now"

This reverts commit b61604275ba090ebd8e30a6d3a6fbe281c74c189.

* pep8 E261

* pep8 E303

* pep8 E501

* pep8 E123

* pep8 E123

* use create instance

* remove excessive import

Co-authored-by: Mikhail Korobov <kmike84@gmail.com>
---
 scrapy/core/scheduler.py |  30 +++---
 scrapy/pqueues.py        | 201 +++++++++++++++++++++------------------
 scrapy/squeues.py        | 100 +++++++++++++++++--
 tests/test_squeues.py    |   7 +-
 4 files changed, 214 insertions(+), 124 deletions(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 975aede0c..e184ed50e 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -119,7 +119,7 @@ class Scheduler(object):
         if self.dqs is None:
             return
         try:
-            self.dqs.push(request, -request.priority)
+            self.dqs.push(request)
         except ValueError as e:  # non serializable request
             if self.logunser:
                 msg = ("Unable to serialize request: %(request)s - reason:"
@@ -135,35 +135,29 @@ class Scheduler(object):
             return True
 
     def _mqpush(self, request):
-        self.mqs.push(request, -request.priority)
+        self.mqs.push(request)
 
     def _dqpop(self):
         if self.dqs:
             return self.dqs.pop()
 
-    def _newmq(self, priority):
-        """ Factory for creating memory queues. """
-        return self.mqclass()
-
-    def _newdq(self, priority):
-        """ Factory for creating disk queues. """
-        path = join(self.dqdir, 'p%s' % (priority, ))
-        return self.dqclass(path)
-
     def _mq(self):
         """ Create a new priority queue instance, with in-memory storage """
-        return create_instance(self.pqclass, None, self.crawler, self._newmq,
-                               serialize=False)
+        return create_instance(self.pqclass,
+                               settings=None,
+                               crawler=self.crawler,
+                               downstream_queue_cls=self.mqclass,
+                               key='')
 
     def _dq(self):
         """ Create a new priority queue instance, with disk storage """
         state = self._read_dqs_state(self.dqdir)
         q = create_instance(self.pqclass,
-                            None,
-                            self.crawler,
-                            self._newdq,
-                            state,
-                            serialize=True)
+                            settings=None,
+                            crawler=self.crawler,
+                            downstream_queue_cls=self.dqclass,
+                            key=self.dqdir,
+                            startprios=state)
         if q:
             logger.info("Resuming crawl (%(queuesize)d requests scheduled)",
                         {'queuesize': len(q)}, extra={'spider': self.spider})
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 717ed4d27..1afe58dab 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -1,11 +1,7 @@
 import hashlib
 import logging
-from collections import namedtuple
-
-from queuelib import PriorityQueue
-
-from scrapy.utils.reqser import request_to_dict, request_from_dict
 
+from scrapy.utils.misc import create_instance
 
 logger = logging.getLogger(__name__)
 
@@ -29,88 +25,89 @@ def _path_safe(text):
     return '-'.join([pathable_slot, unique_slot])
 
 
-class _Priority(namedtuple("_Priority", ["priority", "slot"])):
-    """ Slot-specific priority. It is a hack - ``(priority, slot)`` tuple
-    which can be used instead of int priorities in queues:
+class ScrapyPriorityQueue:
+    """A priority queue implemented using multiple internal queues (typically,
+    FIFO queues). It uses one internal queue for each priority value. The internal
+    queue must implement the following methods:
+
+        * push(obj)
+        * pop()
+        * close()
+        * __len__()
+
+    ``__init__`` method of ScrapyPriorityQueue receives a downstream_queue_cls
+    argument, which is a class used to instantiate a new (internal) queue when
+    a new priority is allocated.
+
+    Only integer priorities should be used. Lower numbers are higher
+    priorities.
+
+    startprios is a sequence of priorities to start with. If the queue was
+    previously closed leaving some priority buckets non-empty, those priorities
+    should be passed in startprios.
 
-    * they are ordered in the same way - order is still by priority value,
-      min(prios) works;
-    * str(p) representation is guaranteed to be different when slots
-      are different - this is important because str(p) is used to create
-      queue files on disk;
-    * they have readable str(p) representation which is safe
-      to use as a file name.
     """
-    __slots__ = ()
 
-    def __str__(self):
-        return '%s_%s' % (self.priority, _path_safe(str(self.slot)))
+    @classmethod
+    def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()):
+        return cls(crawler, downstream_queue_cls, key, startprios)
 
+    def __init__(self, crawler, downstream_queue_cls, key, startprios=()):
+        self.crawler = crawler
+        self.downstream_queue_cls = downstream_queue_cls
+        self.key = key
+        self.queues = {}
+        self.curprio = None
+        self.init_prios(startprios)
 
-class _SlotPriorityQueues(object):
-    """ Container for multiple priority queues. """
-    def __init__(self, pqfactory, slot_startprios=None):
-        """
-        ``pqfactory`` is a factory for creating new PriorityQueues.
-        It must be a function which accepts a single optional ``startprios``
-        argument, with a list of priorities to create queues for.
+    def init_prios(self, startprios):
+        if not startprios:
+            return
 
-        ``slot_startprios`` is a ``{slot: startprios}`` dict.
-        """
-        self.pqfactory = pqfactory
-        self.pqueues = {}  # slot -> priority queue
-        for slot, startprios in (slot_startprios or {}).items():
-            self.pqueues[slot] = self.pqfactory(startprios)
+        for priority in startprios:
+            self.queues[priority] = self.qfactory(priority)
 
-    def pop_slot(self, slot):
-        """ Pop an object from a priority queue for this slot """
-        queue = self.pqueues[slot]
-        request = queue.pop()
-        if len(queue) == 0:
-            del self.pqueues[slot]
-        return request
+        self.curprio = min(startprios)
 
-    def push_slot(self, slot, obj, priority):
-        """ Push an object to a priority queue for this slot """
-        if slot not in self.pqueues:
-            self.pqueues[slot] = self.pqfactory()
-        queue = self.pqueues[slot]
-        queue.push(obj, priority)
+    def qfactory(self, key):
+        return create_instance(self.downstream_queue_cls,
+                               None,
+                               self.crawler,
+                               self.key + '/' + str(key))
+
+    def priority(self, request):
+        return -request.priority
+
+    def push(self, request):
+        priority = self.priority(request)
+        if priority not in self.queues:
+            self.queues[priority] = self.qfactory(priority)
+        q = self.queues[priority]
+        q.push(request)  # this may fail (eg. serialization error)
+        if self.curprio is None or priority < self.curprio:
+            self.curprio = priority
+
+    def pop(self):
+        if self.curprio is None:
+            return
+        q = self.queues[self.curprio]
+        m = q.pop()
+        if not q:
+            del self.queues[self.curprio]
+            q.close()
+            prios = [p for p, q in self.queues.items() if q]
+            self.curprio = min(prios) if prios else None
+        return m
 
     def close(self):
-        active = {slot: queue.close()
-                  for slot, queue in self.pqueues.items()}
-        self.pqueues.clear()
+        active = []
+        for p, q in self.queues.items():
+            active.append(p)
+            q.close()
         return active
 
     def __len__(self):
-        return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
-
-
-class ScrapyPriorityQueue(PriorityQueue):
-    """
-    PriorityQueue which works with scrapy.Request instances and
-    can optionally convert them to/from dicts before/after putting to a queue.
-    """
-    def __init__(self, crawler, qfactory, startprios=(), serialize=False):
-        super(ScrapyPriorityQueue, self).__init__(qfactory, startprios)
-        self.serialize = serialize
-        self.spider = crawler.spider
-
-    @classmethod
-    def from_crawler(cls, crawler, qfactory, startprios=(), serialize=False):
-        return cls(crawler, qfactory, startprios, serialize)
-
-    def push(self, request, priority=0):
-        if self.serialize:
-            request = request_to_dict(request, self.spider)
-        super(ScrapyPriorityQueue, self).push(request, priority)
-
-    def pop(self):
-        request = super(ScrapyPriorityQueue, self).pop()
-        if request and self.serialize:
-            request = request_from_dict(request, self.spider)
-        return request
+        return sum(len(x) for x in self.queues.values()) if self.queues else 0
 
 
 class DownloaderInterface(object):
@@ -133,16 +130,16 @@ class DownloaderInterface(object):
 
 
 class DownloaderAwarePriorityQueue(object):
-    """ PriorityQueue which takes Downlaoder activity in account:
+    """ PriorityQueue which takes Downloader activity in account:
     domains (slots) with the least amount of active downloads are dequeued
     first.
     """
 
     @classmethod
-    def from_crawler(cls, crawler, qfactory, slot_startprios=None, serialize=False):
-        return cls(crawler, qfactory, slot_startprios, serialize)
+    def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()):
+        return cls(crawler, downstream_queue_cls, key, startprios)
 
-    def __init__(self, crawler, qfactory, slot_startprios=None, serialize=False):
+    def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()):
         if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0:
             raise ValueError('"%s" does not support CONCURRENT_REQUESTS_PER_IP'
                              % (self.__class__,))
@@ -156,35 +153,49 @@ class DownloaderAwarePriorityQueue(object):
                              "queue class can be resumed." %
                              slot_startprios.__class__)
 
-        slot_startprios = {
-            slot: [_Priority(p, slot) for p in startprios]
-            for slot, startprios in (slot_startprios or {}).items()}
-
-        def pqfactory(startprios=()):
-            return ScrapyPriorityQueue(crawler, qfactory, startprios, serialize)
-        self._slot_pqueues = _SlotPriorityQueues(pqfactory, slot_startprios)
-        self.serialize = serialize
         self._downloader_interface = DownloaderInterface(crawler)
+        self.downstream_queue_cls = downstream_queue_cls
+        self.key = key
+        self.crawler = crawler
+
+        self.pqueues = {}  # slot -> priority queue
+        for slot, startprios in (slot_startprios or {}).items():
+            self.pqueues[slot] = self.pqfactory(slot, startprios)
+
+    def pqfactory(self, slot, startprios=()):
+        return ScrapyPriorityQueue(self.crawler,
+                                   self.downstream_queue_cls,
+                                   self.key + '/' + _path_safe(slot),
+                                   startprios)
 
     def pop(self):
-        stats = self._downloader_interface.stats(self._slot_pqueues.pqueues)
+        stats = self._downloader_interface.stats(self.pqueues)
 
         if not stats:
             return
 
         slot = min(stats)[1]
-        request = self._slot_pqueues.pop_slot(slot)
+        queue = self.pqueues[slot]
+        request = queue.pop()
+        if len(queue) == 0:
+            del self.pqueues[slot]
         return request
 
-    def push(self, request, priority):
+    def push(self, request):
         slot = self._downloader_interface.get_slot_key(request)
-        priority_slot = _Priority(priority=priority, slot=slot)
-        self._slot_pqueues.push_slot(slot, request, priority_slot)
+        if slot not in self.pqueues:
+            self.pqueues[slot] = self.pqfactory(slot)
+        queue = self.pqueues[slot]
+        queue.push(request)
 
     def close(self):
-        active = self._slot_pqueues.close()
-        return {slot: [p.priority for p in startprios]
-                for slot, startprios in active.items()}
+        active = {slot: queue.close()
+                  for slot, queue in self.pqueues.items()}
+        self.pqueues.clear()
+        return active
 
     def __len__(self):
-        return len(self._slot_pqueues)
+        return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
+
+    def __contains__(self, slot):
+        return slot in self.pqueues
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index d5d3be67e..d0686dac3 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -3,10 +3,27 @@ Scheduler queues
 """
 
 import marshal
+import os
 import pickle
 
 from queuelib import queue
 
+from scrapy.utils.reqser import request_to_dict, request_from_dict
+
+
+def _with_mkdir(queue_class):
+
+    class DirectoriesCreated(queue_class):
+
+        def __init__(self, path, *args, **kwargs):
+            dirname = os.path.dirname(path)
+            if not os.path.exists(dirname):
+                os.makedirs(dirname, exist_ok=True)
+
+            super(DirectoriesCreated, self).__init__(path, *args, **kwargs)
+
+    return DirectoriesCreated
+
 
 def _serializable_queue(queue_class, serialize, deserialize):
 
@@ -24,6 +41,44 @@ def _serializable_queue(queue_class, serialize, deserialize):
     return SerializableQueue
 
 
+def _scrapy_serialization_queue(queue_class):
+
+    class ScrapyRequestQueue(queue_class):
+
+        def __init__(self, crawler, key):
+            self.spider = crawler.spider
+            super(ScrapyRequestQueue, self).__init__(key)
+
+        @classmethod
+        def from_crawler(cls, crawler, key, *args, **kwargs):
+            return cls(crawler, key)
+
+        def push(self, request):
+            request = request_to_dict(request, self.spider)
+            return super(ScrapyRequestQueue, self).push(request)
+
+        def pop(self):
+            request = super(ScrapyRequestQueue, self).pop()
+
+            if not request:
+                return None
+
+            request = request_from_dict(request, self.spider)
+            return request
+
+    return ScrapyRequestQueue
+
+
+def _scrapy_non_serialization_queue(queue_class):
+
+    class ScrapyRequestQueue(queue_class):
+        @classmethod
+        def from_crawler(cls, crawler, *args, **kwargs):
+            return cls()
+
+    return ScrapyRequestQueue
+
+
 def _pickle_serialize(obj):
     try:
         return pickle.dumps(obj, protocol=2)
@@ -34,13 +89,38 @@ def _pickle_serialize(obj):
         raise ValueError(str(e))
 
 
-PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue,
-    _pickle_serialize, pickle.loads)
-PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue,
-    _pickle_serialize, pickle.loads)
-MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue,
-    marshal.dumps, marshal.loads)
-MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue,
-    marshal.dumps, marshal.loads)
-FifoMemoryQueue = queue.FifoMemoryQueue
-LifoMemoryQueue = queue.LifoMemoryQueue
+PickleFifoDiskQueueNonRequest = _serializable_queue(
+    _with_mkdir(queue.FifoDiskQueue),
+    _pickle_serialize,
+    pickle.loads
+)
+PickleLifoDiskQueueNonRequest = _serializable_queue(
+    _with_mkdir(queue.LifoDiskQueue),
+    _pickle_serialize,
+    pickle.loads
+)
+MarshalFifoDiskQueueNonRequest = _serializable_queue(
+    _with_mkdir(queue.FifoDiskQueue),
+    marshal.dumps,
+    marshal.loads
+)
+MarshalLifoDiskQueueNonRequest = _serializable_queue(
+    _with_mkdir(queue.LifoDiskQueue),
+    marshal.dumps,
+    marshal.loads
+)
+
+PickleFifoDiskQueue = _scrapy_serialization_queue(
+    PickleFifoDiskQueueNonRequest
+)
+PickleLifoDiskQueue = _scrapy_serialization_queue(
+    PickleLifoDiskQueueNonRequest
+)
+MarshalFifoDiskQueue = _scrapy_serialization_queue(
+    MarshalFifoDiskQueueNonRequest
+)
+MarshalLifoDiskQueue = _scrapy_serialization_queue(
+    MarshalLifoDiskQueueNonRequest
+)
+FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue)
+LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue)
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index d5fcf2f7f..5c626fbcb 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -1,7 +1,12 @@
 import pickle
 
 from queuelib.tests import test_queue as t
-from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue
+from scrapy.squeues import (
+    MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue,
+    MarshalLifoDiskQueueNonRequest as MarshalLifoDiskQueue,
+    PickleFifoDiskQueueNonRequest as PickleFifoDiskQueue,
+    PickleLifoDiskQueueNonRequest as PickleLifoDiskQueue
+)
 from scrapy.item import Item, Field
 from scrapy.http import Request
 from scrapy.loader import ItemLoader

From 9d983c1b9962a018686111e20e42d25bbffb579e Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Sat, 22 Feb 2020 09:20:31 -0300
Subject: [PATCH 2665/2974] Expose certificate for HTTPS responses (#4054)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Expose certificate for HTTPS responses

* Fix test (missing inlineCallbacks decorator)

* Note about Response.certificate

* Explicitly cover None as the default value of Response.certificate

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/request-response.rst          | 12 +++++++++-
 scrapy/core/downloader/handlers/http11.py | 22 +++++++++++------
 scrapy/http/response/__init__.py          |  5 ++--
 tests/test_crawl.py                       | 29 +++++++++++++++++++++++
 4 files changed, 58 insertions(+), 10 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 672c0b3d6..f009facd6 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -609,7 +609,10 @@ Response objects
 
     :param request: the initial value of the :attr:`Response.request` attribute.
         This represents the :class:`Request` that generated this response.
-    :type request: :class:`Request` object
+    :type request: scrapy.http.Request
+
+    :param certificate: an object representing the server's SSL certificate.
+    :type certificate: twisted.internet.ssl.Certificate
 
     .. attribute:: Response.url
 
@@ -691,6 +694,13 @@ Response objects
         they're shown on the string representation of the Response (`__str__`
         method) which is used by the engine for logging.
 
+    .. attribute:: Response.certificate
+
+        A :class:`twisted.internet.ssl.Certificate` object representing
+        the server's SSL certificate.
+        
+        Only populated for ``https`` responses, ``None`` otherwise.
+
     .. method:: Response.copy()
 
        Returns a new Response which is a copy of this Response.
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 5a5f6cf0a..93951d3b5 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -3,11 +3,12 @@
 import logging
 import re
 import warnings
+from contextlib import suppress
 from io import BytesIO
 from time import time
 from urllib.parse import urldefrag
 
-from twisted.internet import defer, protocol, reactor
+from twisted.internet import defer, protocol, reactor, ssl
 from twisted.internet.endpoints import TCP4ClientEndpoint
 from twisted.internet.error import TimeoutError
 from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
@@ -382,7 +383,7 @@ class ScrapyAgent(object):
     def _cb_bodyready(self, txresponse, request):
         # deliverBody hangs for responses without body
         if txresponse.length == 0:
-            return txresponse, b'', None
+            return txresponse, b'', None, None
 
         maxsize = request.meta.get('download_maxsize', self._maxsize)
         warnsize = request.meta.get('download_warnsize', self._warnsize)
@@ -418,11 +419,12 @@ class ScrapyAgent(object):
         return d
 
     def _cb_bodydone(self, result, request, url):
-        txresponse, body, flags = result
+        txresponse, body, flags, certificate = result
         status = int(txresponse.code)
         headers = Headers(txresponse.headers.getAllRawHeaders())
         respcls = responsetypes.from_args(headers=headers, url=url, body=body)
-        return respcls(url=url, status=status, headers=headers, body=body, flags=flags)
+        return respcls(url=url, status=status, headers=headers, body=body,
+                       flags=flags, certificate=certificate)
 
 
 @implementer(IBodyProducer)
@@ -456,6 +458,12 @@ class _ResponseReader(protocol.Protocol):
         self._fail_on_dataloss_warned = False
         self._reached_warnsize = False
         self._bytes_received = 0
+        self._certificate = None
+
+    def connectionMade(self):
+        if self._certificate is None:
+            with suppress(AttributeError):
+                self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate())
 
     def dataReceived(self, bodyBytes):
         # This maybe called several times after cancel was called with buffered data.
@@ -488,16 +496,16 @@ class _ResponseReader(protocol.Protocol):
 
         body = self._bodybuf.getvalue()
         if reason.check(ResponseDone):
-            self._finished.callback((self._txresponse, body, None))
+            self._finished.callback((self._txresponse, body, None, self._certificate))
             return
 
         if reason.check(PotentialDataLoss):
-            self._finished.callback((self._txresponse, body, ['partial']))
+            self._finished.callback((self._txresponse, body, ['partial'], self._certificate))
             return
 
         if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
             if not self._fail_on_dataloss:
-                self._finished.callback((self._txresponse, body, ['dataloss']))
+                self._finished.callback((self._txresponse, body, ['dataloss'], self._certificate))
                 return
 
             elif not self._fail_on_dataloss_warned:
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index f60d09608..119dd2f63 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -17,13 +17,14 @@ from scrapy.utils.trackref import object_ref
 
 class Response(object_ref):
 
-    def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None):
+    def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None, certificate=None):
         self.headers = Headers(headers or {})
         self.status = int(status)
         self._set_body(body)
         self._set_url(url)
         self.request = request
         self.flags = [] if flags is None else list(flags)
+        self.certificate = certificate
 
     @property
     def cb_kwargs(self):
@@ -86,7 +87,7 @@ class Response(object_ref):
         """Create a new Response with the same attributes except for those
         given new values.
         """
-        for x in ['url', 'status', 'headers', 'body', 'request', 'flags']:
+        for x in ['url', 'status', 'headers', 'body', 'request', 'flags', 'certificate']:
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 64819acb6..bbe97d034 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -5,6 +5,7 @@ import sys
 from pytest import mark
 from testfixtures import LogCapture
 from twisted.internet import defer
+from twisted.internet.ssl import Certificate
 from twisted.trial.unittest import TestCase
 
 from scrapy import signals
@@ -407,3 +408,31 @@ with multiples lines
             yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
         for req_id in range(3):
             self.assertIn("Got response 200, req_id %d" % req_id, str(log))
+
+    @defer.inlineCallbacks
+    def test_response_ssl_certificate_none(self):
+        crawler = self.runner.create_crawler(SingleRequestSpider)
+        url = self.mockserver.url("/echo?body=test", is_secure=False)
+        yield crawler.crawl(seed=url, mockserver=self.mockserver)
+        self.assertIsNone(crawler.spider.meta['responses'][0].certificate)
+
+    @defer.inlineCallbacks
+    def test_response_ssl_certificate(self):
+        crawler = self.runner.create_crawler(SingleRequestSpider)
+        url = self.mockserver.url("/echo?body=test", is_secure=True)
+        yield crawler.crawl(seed=url, mockserver=self.mockserver)
+        cert = crawler.spider.meta['responses'][0].certificate
+        self.assertIsInstance(cert, Certificate)
+        self.assertEqual(cert.getSubject().commonName, b"localhost")
+        self.assertEqual(cert.getIssuer().commonName, b"localhost")
+
+    @mark.xfail(reason="Responses with no body return early and contain no certificate")
+    @defer.inlineCallbacks
+    def test_response_ssl_certificate_empty_response(self):
+        crawler = self.runner.create_crawler(SingleRequestSpider)
+        url = self.mockserver.url("/status?n=200", is_secure=True)
+        yield crawler.crawl(seed=url, mockserver=self.mockserver)
+        cert = crawler.spider.meta['responses'][0].certificate
+        self.assertIsInstance(cert, Certificate)
+        self.assertEqual(cert.getSubject().commonName, b"localhost")
+        self.assertEqual(cert.getIssuer().commonName, b"localhost")

From f85bf77da3c8943f0791dcae893e8294c4d118d7 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 23 Feb 2020 18:31:13 -0300
Subject: [PATCH 2666/2974] Restore unrelated change

---
 scrapy/resolver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index f69894b1e..554a3a14d 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -29,7 +29,7 @@ class CachingThreadedResolver(ThreadedResolver):
             cache_size = 0
         return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT'))
 
-    def install_on_reactor(self):
+    def install_on_reactor(self,):
         self.reactor.installResolver(self)
 
     def getHostByName(self, name, timeout=None):

From 889b4718520220d1a81e702ff754ec210a7d3c79 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 23 Feb 2020 18:40:43 -0300
Subject: [PATCH 2667/2974] Import changes

---
 scrapy/core/downloader/handlers/http11.py | 4 ++--
 tests/test_crawl.py                       | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index e72275021..190ae1d3b 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -1,11 +1,11 @@
 """Download handlers for http and https schemes"""
 
+import ipaddress
 import logging
 import re
 import warnings
 from contextlib import suppress
 from io import BytesIO
-from ipaddress import ip_address
 from time import time
 from urllib.parse import urldefrag
 
@@ -468,7 +468,7 @@ class _ResponseReader(protocol.Protocol):
                 self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate())
 
         if self._ip_address is None:
-            self._ip_address = ip_address(self.transport._producer.getPeer().host)
+            self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host)
 
     def dataReceived(self, bodyBytes):
         # This maybe called several times after cancel was called with buffered data.
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3a9b00ab3..3c110e7a6 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -441,13 +441,15 @@ with multiples lines
         self.assertEqual(cert.getIssuer().commonName, b"localhost")
 
     @defer.inlineCallbacks
-    def test_dns_server_ip_address(self):
+    def test_dns_server_ip_address_none(self):
         crawler = self.runner.create_crawler(SingleRequestSpider)
         url = self.mockserver.url('/status?n=200')
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         ip_address = crawler.spider.meta['responses'][0].ip_address
         self.assertIsNone(ip_address)
 
+    @defer.inlineCallbacks
+    def test_dns_server_ip_address(self):
         crawler = self.runner.create_crawler(SingleRequestSpider)
         url = self.mockserver.url('/echo?body=test')
         expected_netloc, _ = urlparse(url).netloc.split(':')

From 31f35c9c002178de20a3e124be4aab98c0a0f892 Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Mon, 24 Feb 2020 08:02:00 -0300
Subject: [PATCH 2668/2974] Remove unnecessary comma (#4369)

---
 scrapy/resolver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 554a3a14d..f69894b1e 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -29,7 +29,7 @@ class CachingThreadedResolver(ThreadedResolver):
             cache_size = 0
         return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT'))
 
-    def install_on_reactor(self,):
+    def install_on_reactor(self):
         self.reactor.installResolver(self)
 
     def getHostByName(self, name, timeout=None):

From 7417a9871c489686b7d3ac1b85b964eca253c979 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 24 Feb 2020 13:28:15 +0100
Subject: [PATCH 2669/2974] =?UTF-8?q?Make=20BaseItemExporter=E2=80=99s=20d?=
 =?UTF-8?q?ont=5Ffail=20parameter=20keyword-only?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scrapy/exporters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 96416f075..2e20a7180 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -23,7 +23,7 @@ __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
 
 class BaseItemExporter(object):
 
-    def __init__(self, dont_fail=False, **kwargs):
+    def __init__(self, *, dont_fail=False, **kwargs):
         self._kwargs = kwargs
         self._configure(kwargs, dont_fail=dont_fail)
 

From a34c366fa4f226cb107a19561ca64fac0d1dbdd5 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Fri, 21 Feb 2020 08:15:51 +0000
Subject: [PATCH 2670/2974] DOC linkcheck run; https and 301 link updates.

Closes #4359
---
 docs/conf.py                          |  1 +
 docs/contributing.rst                 |  6 +++---
 docs/faq.rst                          |  6 +++---
 docs/intro/install.rst                | 12 ++++++------
 docs/intro/tutorial.rst               |  4 ++--
 docs/news.rst                         | 19 +++++++++----------
 docs/topics/broad-crawls.rst          |  2 +-
 docs/topics/downloader-middleware.rst | 12 ++++++------
 docs/topics/dynamic-content.rst       |  6 +++---
 docs/topics/item-pipeline.rst         |  4 ++--
 docs/topics/items.rst                 |  4 ++--
 docs/topics/leaks.rst                 | 10 +++++-----
 docs/topics/request-response.rst      |  2 +-
 docs/topics/selectors.rst             | 17 ++++++++---------
 docs/topics/shell.rst                 |  6 +++---
 docs/topics/spiders.rst               |  6 +++---
 16 files changed, 58 insertions(+), 59 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index c3418cfb3..6e2399f66 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -281,6 +281,7 @@ coverage_ignore_pyobjects = [
 
 intersphinx_mapping = {
     'coverage': ('https://coverage.readthedocs.io/en/stable', None),
+    'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
     'pytest': ('https://docs.pytest.org/en/latest', None),
     'python': ('https://docs.python.org/3', None),
     'sphinx': ('https://www.sphinx-doc.org/en/master', None),
diff --git a/docs/contributing.rst b/docs/contributing.rst
index f40a6bba2..aed5ab92e 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -143,7 +143,7 @@ by running ``git fetch upstream pull/$PR_NUMBER/head:$BRANCH_NAME_TO_CREATE``
 (replace 'upstream' with a remote name for scrapy repository,
 ``$PR_NUMBER`` with an ID of the pull request, and ``$BRANCH_NAME_TO_CREATE``
 with a name of the branch you want to create locally).
-See also: https://help.github.com/articles/checking-out-pull-requests-locally/#modifying-an-inactive-pull-request-locally.
+See also: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally.
 
 When writing GitHub pull requests, try to keep titles short but descriptive.
 E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests"
@@ -168,7 +168,7 @@ Scrapy:
 
 * Don't put your name in the code you contribute; git provides enough
   metadata to identify author of the code.
-  See https://help.github.com/articles/setting-your-username-in-git/ for
+  See https://help.github.com/en/github/using-git/setting-your-username-in-git for
   setup instructions.
 
 .. _documentation-policies:
@@ -266,5 +266,5 @@ And their unit-tests are in::
 .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
 .. _open issues: https://github.com/scrapy/scrapy/issues
 .. _PEP 257: https://www.python.org/dev/peps/pep-0257/
-.. _pull request: https://help.github.com/en/articles/creating-a-pull-request
+.. _pull request: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request
 .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
diff --git a/docs/faq.rst b/docs/faq.rst
index f72e4cf01..75a0f4864 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -22,8 +22,8 @@ In other words, comparing `BeautifulSoup`_ (or `lxml`_) to Scrapy is like
 comparing `jinja2`_ to `Django`_.
 
 .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
-.. _lxml: http://lxml.de/
-.. _jinja2: http://jinja.pocoo.org/
+.. _lxml: https://lxml.de/
+.. _jinja2: https://palletsprojects.com/p/jinja/
 .. _Django: https://www.djangoproject.com/
 
 Can I use Scrapy with BeautifulSoup?
@@ -269,7 +269,7 @@ The ``__VIEWSTATE`` parameter is used in sites built with ASP.NET/VB.NET. For
 more info on how it works see `this page`_. Also, here's an `example spider`_
 which scrapes one of these sites.
 
-.. _this page: http://search.cpan.org/~ecarroll/HTML-TreeBuilderX-ASP_NET-0.09/lib/HTML/TreeBuilderX/ASP_NET.pm
+.. _this page: https://metacpan.org/pod/release/ECARROLL/HTML-TreeBuilderX-ASP_NET-0.09/lib/HTML/TreeBuilderX/ASP_NET.pm
 .. _example spider: https://github.com/AmbientLighter/rpn-fas/blob/master/fas/spiders/rnp.py
 
 What's the best way to parse big XML/CSV data feeds?
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 49968437c..871281460 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -65,7 +65,7 @@ please refer to their respective installation instructions:
 * `lxml installation`_
 * `cryptography installation`_
 
-.. _lxml installation: http://lxml.de/installation.html
+.. _lxml installation: https://lxml.de/installation.html
 .. _cryptography installation: https://cryptography.io/en/latest/installation/
 
 
@@ -253,11 +253,11 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
 .. _Python: https://www.python.org/
 .. _pip: https://pip.pypa.io/en/latest/installing/
 .. _lxml: https://lxml.de/index.html
-.. _parsel: https://pypi.python.org/pypi/parsel
-.. _w3lib: https://pypi.python.org/pypi/w3lib
-.. _twisted: https://twistedmatrix.com/
-.. _cryptography: https://cryptography.io/
-.. _pyOpenSSL: https://pypi.python.org/pypi/pyOpenSSL
+.. _parsel: https://pypi.org/project/parsel/
+.. _w3lib: https://pypi.org/project/w3lib/
+.. _twisted: https://twistedmatrix.com/trac/
+.. _cryptography: https://cryptography.io/en/latest/
+.. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/
 .. _setuptools: https://pypi.python.org/pypi/setuptools
 .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
 .. _homebrew: https://brew.sh/
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 798fe4a7a..1768badbb 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -306,7 +306,7 @@ with a selector (see :ref:`topics-developer-tools`).
 visually selected elements, which works in many browsers.
 
 .. _regular expressions: https://docs.python.org/3/library/re.html
-.. _Selector Gadget: http://selectorgadget.com/
+.. _Selector Gadget: https://selectorgadget.com/
 
 
 XPath: a brief intro
@@ -337,7 +337,7 @@ recommend `this tutorial to learn XPath through examples
 <http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this tutorial to learn "how
 to think in XPath" <http://plasmasturm.org/log/xpath101/>`_.
 
-.. _XPath: https://www.w3.org/TR/xpath
+.. _XPath: https://www.w3.org/TR/xpath/all/
 .. _CSS: https://www.w3.org/TR/selectors
 
 Extracting quotes and authors
diff --git a/docs/news.rst b/docs/news.rst
index e4b985c77..338b53dc4 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -26,7 +26,7 @@ Backward-incompatible changes
 *   Python 3.4 is no longer supported, and some of the minimum requirements of
     Scrapy have also changed:
 
-    *   cssselect_ 0.9.1
+    *   :doc:`cssselect <cssselect:index>` 0.9.1
     *   cryptography_ 2.0
     *   lxml_ 3.5.0
     *   pyOpenSSL_ 16.2.0
@@ -1616,7 +1616,7 @@ Deprecations and Removals
   + ``scrapy.utils.datatypes.SiteNode``
 
 - The previously bundled ``scrapy.xlib.pydispatch`` library was deprecated and
-  replaced by `pydispatcher <https://pypi.python.org/pypi/PyDispatcher>`_.
+  replaced by `pydispatcher <https://pypi.org/project/PyDispatcher/>`_.
 
 
 Relocations
@@ -2450,7 +2450,7 @@ Other
 ~~~~~
 
 - Dropped Python 2.6 support (:issue:`448`)
-- Add `cssselect`_ python package as install dependency
+- Add :doc:`cssselect <cssselect:index>` python package as install dependency
 - Drop libxml2 and multi selector's backend support, `lxml`_ is required from now on.
 - Minimum Twisted version increased to 10.0.0, dropped Twisted 8.0 support.
 - Running test suite now requires ``mock`` python library (:issue:`390`)
@@ -3047,17 +3047,16 @@ Scrapy 0.7
 First release of Scrapy.
 
 
-.. _AJAX crawleable urls: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started?csw=1
+.. _AJAX crawleable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1
 .. _botocore: https://github.com/boto/botocore
 .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
 .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
 .. _Creating a pull request: https://help.github.com/en/articles/creating-a-pull-request
 .. _cryptography: https://cryptography.io/en/latest/
-.. _cssselect: https://github.com/scrapy/cssselect/
-.. _docstrings: https://docs.python.org/glossary.html#term-docstring
-.. _KeyboardInterrupt: https://docs.python.org/library/exceptions.html#KeyboardInterrupt
+.. _docstrings: https://docs.python.org/3/glossary.html#term-docstring
+.. _KeyboardInterrupt: https://docs.python.org/3/library/exceptions.html#KeyboardInterrupt
 .. _LevelDB: https://github.com/google/leveldb
-.. _lxml: http://lxml.de/
+.. _lxml: https://lxml.de/
 .. _marshal: https://docs.python.org/2/library/marshal.html
 .. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator
 .. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator
@@ -3068,11 +3067,11 @@ First release of Scrapy.
 .. _queuelib: https://github.com/scrapy/queuelib
 .. _registered with IANA: https://www.iana.org/assignments/media-types/media-types.xhtml
 .. _resource: https://docs.python.org/2/library/resource.html
-.. _robots.txt: http://www.robotstxt.org/
+.. _robots.txt: https://www.robotstxt.org/
 .. _scrapely: https://github.com/scrapy/scrapely
 .. _service_identity: https://service-identity.readthedocs.io/en/stable/
 .. _six: https://six.readthedocs.io/
-.. _tox: https://pypi.python.org/pypi/tox
+.. _tox: https://pypi.org/project/tox/
 .. _Twisted: https://twistedmatrix.com/trac/
 .. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
 .. _w3lib: https://github.com/scrapy/w3lib
diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index 4922694ee..63b60312e 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -188,7 +188,7 @@ AjaxCrawlMiddleware helps to crawl them correctly.
 It is turned OFF by default because it has some performance overhead,
 and enabling it for focused crawls doesn't make much sense.
 
-.. _ajax crawlable: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
+.. _ajax crawlable: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started
 
 .. _broad-crawls-bfo:
 
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index a83cedcfd..0297ef3a0 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -709,7 +709,7 @@ HttpCompressionMiddleware
    provided `brotlipy`_ is installed.
 
 .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
-.. _brotlipy: https://pypi.python.org/pypi/brotlipy
+.. _brotlipy: https://pypi.org/project/brotlipy/
 
 HttpCompressionMiddleware Settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1038,7 +1038,7 @@ Based on `RobotFileParser
 * is Python's built-in robots.txt_ parser
 
 * is compliant with `Martijn Koster's 1996 draft specification
-  <http://www.robotstxt.org/norobots-rfc.txt>`_
+  <https://www.robotstxt.org/norobots-rfc.txt>`_
 
 * lacks support for wildcard matching
 
@@ -1061,7 +1061,7 @@ Based on `Reppy <https://github.com/seomoz/reppy/>`_:
   <https://github.com/seomoz/rep-cpp>`_
 
 * is compliant with `Martijn Koster's 1996 draft specification
-  <http://www.robotstxt.org/norobots-rfc.txt>`_
+  <https://www.robotstxt.org/norobots-rfc.txt>`_
 
 * supports wildcard matching
 
@@ -1086,7 +1086,7 @@ Based on `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_:
 * implemented in Python
 
 * is compliant with `Martijn Koster's 1996 draft specification
-  <http://www.robotstxt.org/norobots-rfc.txt>`_
+  <https://www.robotstxt.org/norobots-rfc.txt>`_
 
 * supports wildcard matching
 
@@ -1115,7 +1115,7 @@ implementing the methods described below.
 .. autoclass:: RobotParser
    :members:
 
-.. _robots.txt: http://www.robotstxt.org/
+.. _robots.txt: https://www.robotstxt.org/
 
 DownloaderStats
 ---------------
@@ -1155,7 +1155,7 @@ AjaxCrawlMiddleware
 
    Middleware that finds 'AJAX crawlable' page variants based
    on meta-fragment html tag. See
-   https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
+   https://developers.google.com/search/docs/ajax-crawling/docs/getting-started
    for more info.
 
    .. note::
diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index 1c3607860..b98133676 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -241,12 +241,12 @@ along with `scrapy-selenium`_ for seamless integration.
 .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
 .. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
 .. _js2xml: https://github.com/scrapinghub/js2xml
-.. _json.loads: https://docs.python.org/library/json.html#json.loads
+.. _json.loads: https://docs.python.org/3/library/json.html#json.loads
 .. _pytesseract: https://github.com/madmaze/pytesseract
-.. _regular expression: https://docs.python.org/library/re.html
+.. _regular expression: https://docs.python.org/3/library/re.html
 .. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
 .. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
-.. _Selenium: https://www.seleniumhq.org/
+.. _Selenium: https://www.selenium.dev/
 .. _Splash: https://github.com/scrapinghub/splash
 .. _tabula-py: https://github.com/chezou/tabula-py
 .. _wget: https://www.gnu.org/software/wget/
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index cdc4953c2..801d48fd5 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -158,8 +158,8 @@ method and how to clean up the resources properly.::
             self.db[self.collection_name].insert_one(dict(item))
             return item
 
-.. _MongoDB: https://www.mongodb.org/
-.. _pymongo: https://api.mongodb.org/python/current/
+.. _MongoDB: https://www.mongodb.com/
+.. _pymongo: https://api.mongodb.com/python/current/
 
 
 Take screenshot of item
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 15313775b..44643cb67 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -166,7 +166,7 @@ If your item contains mutable_ values like lists or dictionaries, a shallow
 copy will keep references to the same mutable values across all different
 copies.
 
-.. _mutable: https://docs.python.org/glossary.html#term-mutable
+.. _mutable: https://docs.python.org/3/glossary.html#term-mutable
 
 For example, if you have an item with a list of tags, and you create a shallow
 copy of that item, both the original item and the copy have the same list of
@@ -177,7 +177,7 @@ If that is not the desired behavior, use a deep copy instead.
 
 See the `documentation of the copy module`_ for more information.
 
-.. _documentation of the copy module: https://docs.python.org/library/copy.html
+.. _documentation of the copy module: https://docs.python.org/3/library/copy.html
 
 To create a shallow copy of an item, you can either call
 :meth:`~scrapy.item.Item.copy` on an existing item
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 9fee333ac..c0c83fc84 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -206,7 +206,7 @@ objects. If this is your case, and you can't find your leaks using ``trackref``,
 you still have another resource: the `Guppy library`_.
 If you're using Python3, see :ref:`topics-leaks-muppy`.
 
-.. _Guppy library: https://pypi.python.org/pypi/guppy
+.. _Guppy library: https://pypi.org/project/guppy/
 
 If you use ``pip``, you can install Guppy with the following command::
 
@@ -311,9 +311,9 @@ though neither Scrapy nor your project are leaking memory. This is due to a
 (not so well) known problem of Python, which may not return released memory to
 the operating system in some cases. For more information on this issue see:
 
-* `Python Memory Management <http://www.evanjones.ca/python-memory.html>`_
-* `Python Memory Management Part 2 <http://www.evanjones.ca/python-memory-part2.html>`_
-* `Python Memory Management Part 3 <http://www.evanjones.ca/python-memory-part3.html>`_
+* `Python Memory Management <https://www.evanjones.ca/python-memory.html>`_
+* `Python Memory Management Part 2 <https://www.evanjones.ca/python-memory-part2.html>`_
+* `Python Memory Management Part 3 <https://www.evanjones.ca/python-memory-part3.html>`_
 
 The improvements proposed by Evan Jones, which are detailed in `this paper`_,
 got merged in Python 2.5, but this only reduces the problem, it doesn't fix it
@@ -327,7 +327,7 @@ completely. To quote the paper:
     to move to a compacting garbage collector, which is able to move objects in
     memory. This would require significant changes to the Python interpreter.*
 
-.. _this paper: http://www.evanjones.ca/memoryallocator/
+.. _this paper: https://www.evanjones.ca/memoryallocator/
 
 To keep memory consumption reasonable you can split the job into several
 smaller jobs or enable :ref:`persistent job queue <topics-jobs>`
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index f009facd6..c4c2845c9 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -396,7 +396,7 @@ The FormRequest class extends the base :class:`Request` with functionality for
 dealing with HTML forms. It uses `lxml.html forms`_  to pre-populate form
 fields with form data from :class:`Response` objects.
 
-.. _lxml.html forms: http://lxml.de/lxmlhtml.html#forms
+.. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms
 
 .. class:: FormRequest(url, [formdata, ...])
 
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index c3d431e2a..1f7802c98 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -35,12 +35,11 @@ defines selectors to associate those styles with specific HTML elements.
     in speed and parsing accuracy to lxml.
 
 .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
-.. _lxml: http://lxml.de/
+.. _lxml: https://lxml.de/
 .. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html
-.. _cssselect: https://pypi.python.org/pypi/cssselect/
-.. _XPath: https://www.w3.org/TR/xpath
+.. _XPath: https://www.w3.org/TR/xpath/all/
 .. _CSS: https://www.w3.org/TR/selectors
-.. _parsel: https://parsel.readthedocs.io/
+.. _parsel: https://parsel.readthedocs.io/en/latest/
 
 Using selectors
 ===============
@@ -255,7 +254,7 @@ that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**:
     They will most probably not work with other libraries like
     `lxml`_ or `PyQuery`_.
 
-.. _PyQuery: https://pypi.python.org/pypi/pyquery
+.. _PyQuery: https://pypi.org/project/pyquery/
 
 Examples:
 
@@ -309,7 +308,7 @@ Examples:
     make much sense: text nodes do not have attributes, and attribute values
     are string values already and do not have children nodes.
 
-.. _CSS Selectors: https://www.w3.org/TR/css3-selectors/#selectors
+.. _CSS Selectors: https://www.w3.org/TR/selectors-3/#selectors
 
 .. _topics-selectors-nesting-selectors:
 
@@ -504,7 +503,7 @@ Another common case would be to extract all direct ``<p>`` children:
 For more details about relative XPaths see the `Location Paths`_ section in the
 XPath specification.
 
-.. _Location Paths: https://www.w3.org/TR/xpath#location-paths
+.. _Location Paths: https://www.w3.org/TR/xpath/all/#location-paths
 
 When querying by class, consider using CSS
 ------------------------------------------
@@ -612,7 +611,7 @@ But using the ``.`` to mean the node, works:
 >>> sel.xpath("//a[contains(., 'Next Page')]").getall()
 ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
 
-.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
+.. _`XPath string function`: https://www.w3.org/TR/xpath/all/#section-String-Functions
 
 .. _topics-selectors-xpath-variables:
 
@@ -764,7 +763,7 @@ Set operations
 These can be handy for excluding parts of a document tree before
 extracting text elements for example.
 
-Example extracting microdata (sample content taken from http://schema.org/Product)
+Example extracting microdata (sample content taken from https://schema.org/Product)
 with groups of itemscopes and corresponding itemprops::
 
     >>> doc = u"""
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 3cf8311a6..8f7518b19 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -41,7 +41,7 @@ variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
 
 .. _IPython: https://ipython.org/
 .. _IPython installation guide: https://ipython.org/install.html
-.. _bpython: https://www.bpython-interpreter.org/
+.. _bpython: https://bpython-interpreter.org/
 
 Launch the shell
 ================
@@ -142,7 +142,7 @@ Example of shell session
 ========================
 
 Here's an example of a typical shell session where we start by scraping the
-https://scrapy.org page, and then proceed to scrape the https://reddit.com
+https://scrapy.org page, and then proceed to scrape the https://old.reddit.com/
 page. Finally, we modify the (Reddit) request method to POST and re-fetch it
 getting an error. We end the session by typing Ctrl-D (in Unix systems) or
 Ctrl-Z in Windows.
@@ -182,7 +182,7 @@ After that, we can start playing with the objects:
 >>> response.xpath('//title/text()').get()
 'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
 
->>> fetch("https://reddit.com")
+>>> fetch("https://old.reddit.com/")
 
 >>> response.xpath('//title/text()').get()
 'reddit: the front page of the internet'
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index b0fb14e24..e0f33de66 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -299,8 +299,8 @@ The spider will not do any parsing on its own.
 If you were to set the ``start_urls`` attribute from the command line,
 you would have to parse it on your own into a list
 using something like
-`ast.literal_eval <https://docs.python.org/library/ast.html#ast.literal_eval>`_
-or `json.loads <https://docs.python.org/library/json.html#json.loads>`_
+`ast.literal_eval <https://docs.python.org/3/library/ast.html#ast.literal_eval>`_
+or `json.loads <https://docs.python.org/3/library/json.html#json.loads>`_
 and then set it as an attribute.
 Otherwise, you would cause iteration over a ``start_urls`` string
 (a very common python pitfall)
@@ -811,6 +811,6 @@ Combine SitemapSpider with other sources of urls::
 
 .. _Sitemaps: https://www.sitemaps.org/index.html
 .. _Sitemap index files: https://www.sitemaps.org/protocol.html#index
-.. _robots.txt: http://www.robotstxt.org/
+.. _robots.txt: https://www.robotstxt.org/
 .. _TLD: https://en.wikipedia.org/wiki/Top-level_domain
 .. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/

From 034e2c31c7d55333c3de208f80dcee1bf45ef9b9 Mon Sep 17 00:00:00 2001
From: gunblues <hsiao.powen@gmail.com>
Date: Wed, 26 Feb 2020 03:46:05 +0800
Subject: [PATCH 2671/2974] Use a non-zero exit code when a pipeline's
 open_spider method throws an exception (#4207)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix issue 4175 - Scrapy does not use a non-zero exit code when pipeline's open_spider throws the exception

* remove extra blank lines

* remove redundant code

* remove blank line at end of file

* more suitable naming for response and make if-condition shorter

* avoid error - AttributeError: 'Deferred' object has no attribute 'result'

* use getattr to make code concisely

* add test

* remove useless file

* modify test class name

* remove unneccessary files

* Fix Flake8-reported issue

* fix these items which are suggested by Gallaecio
・Sort those imports at tests/test_cmdline_crawl_with_pipeline/__init__.py
・Remove the unused setUp method.
・Remove comments generated by Scrapy’s project generation tool.
・Remove the [deploy] section from the scrapy.cfg file (I don’t think it’s needed here)
・Remove BOT_NAME and NEWSPIDER_MODULE from settings.py (I think there are not needed either, although I’m less sure about NEWSPIDER_MODULE)

* have to reserve BOT_NAME, SPIDER_MODULES in settings.py

* Remove unneeded empty lines

* Empty __init__.py file with unneeded comments

* Remove an unneeded empty line at the end

* Remove unneeed empty line from __init__.py file

* Update __init__.py

* Update __init__.py

* Update exception.py

* Update normal.py

* Update __init__.py

* Update __init__.py

* fix W391 blank line at end of file

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/commands/crawl.py                      | 11 +++++++---
 .../__init__.py                               | 20 +++++++++++++++++++
 .../scrapy.cfg                                |  2 ++
 .../test_spider/__init__.py                   |  0
 .../test_spider/pipelines.py                  | 16 +++++++++++++++
 .../test_spider/settings.py                   |  2 ++
 .../test_spider/spiders/__init__.py           |  0
 .../test_spider/spiders/exception.py          | 14 +++++++++++++
 .../test_spider/spiders/normal.py             | 14 +++++++++++++
 9 files changed, 76 insertions(+), 3 deletions(-)
 create mode 100644 tests/test_cmdline_crawl_with_pipeline/__init__.py
 create mode 100644 tests/test_cmdline_crawl_with_pipeline/scrapy.cfg
 create mode 100644 tests/test_cmdline_crawl_with_pipeline/test_spider/__init__.py
 create mode 100644 tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
 create mode 100644 tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py
 create mode 100644 tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/__init__.py
 create mode 100644 tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py
 create mode 100644 tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py

diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 8093fd402..7b417e2eb 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -54,8 +54,13 @@ class Command(ScrapyCommand):
             raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
         spname = args[0]
 
-        self.crawler_process.crawl(spname, **opts.spargs)
-        self.crawler_process.start()
+        crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
 
-        if self.crawler_process.bootstrap_failed:
+        if getattr(crawl_defer, 'result', None) is not None and issubclass(crawl_defer.result.type, Exception):
             self.exitcode = 1
+        else:
+            self.crawler_process.start()
+
+            if self.crawler_process.bootstrap_failed or \
+                    (hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception):
+                self.exitcode = 1
diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py
new file mode 100644
index 000000000..d341888d3
--- /dev/null
+++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py
@@ -0,0 +1,20 @@
+import os
+import sys
+import unittest
+from subprocess import Popen, PIPE
+
+
+class CmdlineCrawlPipelineTest(unittest.TestCase):
+
+    def _execute(self, spname):
+        args = (sys.executable, '-m', 'scrapy.cmdline', 'crawl', spname)
+        cwd = os.path.dirname(os.path.abspath(__file__))
+        proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
+        proc.communicate()
+        return proc.returncode
+
+    def test_open_spider_normally_in_pipeline(self):
+        self.assertEqual(self._execute('normal'), 0)
+
+    def test_exception_at_open_spider_in_pipeline(self):
+        self.assertEqual(self._execute('exception'), 1)
diff --git a/tests/test_cmdline_crawl_with_pipeline/scrapy.cfg b/tests/test_cmdline_crawl_with_pipeline/scrapy.cfg
new file mode 100644
index 000000000..2f238dba3
--- /dev/null
+++ b/tests/test_cmdline_crawl_with_pipeline/scrapy.cfg
@@ -0,0 +1,2 @@
+[settings]
+default = test_spider.settings
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/__init__.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
new file mode 100644
index 000000000..ce916f699
--- /dev/null
+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
@@ -0,0 +1,16 @@
+class TestSpiderPipeline(object):
+
+    def open_spider(self, spider):
+        pass
+
+    def process_item(self, item, spider):
+        return item
+
+
+class TestSpiderExceptionPipeline(object):
+
+    def open_spider(self, spider):
+        raise Exception('exception')
+
+    def process_item(self, item, spider):
+        return item
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py
new file mode 100644
index 000000000..ae782c0d8
--- /dev/null
+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py
@@ -0,0 +1,2 @@
+BOT_NAME = 'test_spider'
+SPIDER_MODULES = ['test_spider.spiders']
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/__init__.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py
new file mode 100644
index 000000000..300f45ebf
--- /dev/null
+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py
@@ -0,0 +1,14 @@
+import scrapy
+
+
+class ExceptionSpider(scrapy.Spider):
+    name = 'exception'
+
+    custom_settings = {
+        'ITEM_PIPELINES': {
+            'test_spider.pipelines.TestSpiderExceptionPipeline': 300
+        }
+    }
+
+    def parse(self, response):
+        pass
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py
new file mode 100644
index 000000000..87a40fdcb
--- /dev/null
+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py
@@ -0,0 +1,14 @@
+import scrapy
+
+
+class NormalSpider(scrapy.Spider):
+    name = 'normal'
+
+    custom_settings = {
+        'ITEM_PIPELINES': {
+            'test_spider.pipelines.TestSpiderPipeline': 300
+        }
+    }
+
+    def parse(self, response):
+        pass

From 7291173f6b6a8e1768ab9d5f52474cd8ada8381e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 25 Feb 2020 21:35:21 +0100
Subject: [PATCH 2672/2974] Have ReadTheDocs builds fail on warning

---
 .readthedocs.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.readthedocs.yml b/.readthedocs.yml
index 563add75f..0b9e15018 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -1,6 +1,7 @@
 version: 2
 sphinx:
   configuration: docs/conf.py
+  fail_on_warning: true
 python:
   # For available versions, see:
   # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image

From a9d7d8f064fe3086227d4737d07e3ca4a296b4b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 25 Feb 2020 21:41:07 +0100
Subject: [PATCH 2673/2974] Add Scrapy dependencies back to
 docs/requirements.txt

---
 docs/requirements.txt | 15 +++++++++++++++
 setup.py              |  1 +
 2 files changed, 16 insertions(+)

diff --git a/docs/requirements.txt b/docs/requirements.txt
index 773b92cea..215cdd64d 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -2,3 +2,18 @@ Sphinx>=2.1
 sphinx-hoverxref
 sphinx-notfound-page
 sphinx_rtd_theme
+
+# Required for ReadTheDocs
+# Keep in sync with setup.py
+Twisted>=17.9.0
+cryptography>=2.0
+cssselect>=0.9.1
+lxml>=3.5.0
+parsel>=1.5.0
+PyDispatcher>=2.0.5
+pyOpenSSL>=16.2.0
+queuelib>=1.4.2
+service_identity>=16.0.0
+w3lib>=1.17.0
+zope.interface>=4.1.3
+protego>=0.1.15
diff --git a/setup.py b/setup.py
index 85d797f88..6f15ca277 100644
--- a/setup.py
+++ b/setup.py
@@ -62,6 +62,7 @@ setup(
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
     python_requires='>=3.5',
+    # Keep in sync with docs/requirements.txt
     install_requires=[
         'Twisted>=17.9.0',
         'cryptography>=2.0',

From 778813717df0d5fcd4359266f6785068feea0785 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 25 Feb 2020 21:58:28 +0100
Subject: [PATCH 2674/2974] Use ReadTheDocs install.path

---
 .readthedocs.yml      |  1 +
 docs/requirements.txt | 15 ---------------
 setup.py              |  1 -
 3 files changed, 1 insertion(+), 16 deletions(-)

diff --git a/.readthedocs.yml b/.readthedocs.yml
index 0b9e15018..17eba34f3 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -8,3 +8,4 @@ python:
   version: 3.7  # Keep in sync with .travis.yml
   install:
     - requirements: docs/requirements.txt
+    - path: .
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 215cdd64d..773b92cea 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -2,18 +2,3 @@ Sphinx>=2.1
 sphinx-hoverxref
 sphinx-notfound-page
 sphinx_rtd_theme
-
-# Required for ReadTheDocs
-# Keep in sync with setup.py
-Twisted>=17.9.0
-cryptography>=2.0
-cssselect>=0.9.1
-lxml>=3.5.0
-parsel>=1.5.0
-PyDispatcher>=2.0.5
-pyOpenSSL>=16.2.0
-queuelib>=1.4.2
-service_identity>=16.0.0
-w3lib>=1.17.0
-zope.interface>=4.1.3
-protego>=0.1.15
diff --git a/setup.py b/setup.py
index 6f15ca277..85d797f88 100644
--- a/setup.py
+++ b/setup.py
@@ -62,7 +62,6 @@ setup(
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
     python_requires='>=3.5',
-    # Keep in sync with docs/requirements.txt
     install_requires=[
         'Twisted>=17.9.0',
         'cryptography>=2.0',

From 6109ad9aacd5897f67842685b2405627b4af4ad6 Mon Sep 17 00:00:00 2001
From: HEndo12345 <38522238+HEndo12345@users.noreply.github.com>
Date: Thu, 27 Feb 2020 23:15:30 +0900
Subject: [PATCH 2675/2974] Clean up the deprecated settings list (#4378)

---
 scrapy/settings/deprecated.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scrapy/settings/deprecated.py b/scrapy/settings/deprecated.py
index 91ed689e8..1211908df 100644
--- a/scrapy/settings/deprecated.py
+++ b/scrapy/settings/deprecated.py
@@ -9,10 +9,8 @@ DEPRECATED_SETTINGS = [
     ('ENCODING_ALIASES', 'no longer needed (encoding discovery uses w3lib now)'),
     ('STATS_ENABLED', 'no longer supported (change STATS_CLASS instead)'),
     ('SQLITE_DB', 'no longer supported'),
-    ('SELECTORS_BACKEND', 'use SCRAPY_SELECTORS_BACKEND environment variable instead'),
     ('AUTOTHROTTLE_MIN_DOWNLOAD_DELAY', 'use DOWNLOAD_DELAY instead'),
     ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'),
-    ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'),
     ('REDIRECT_MAX_METAREFRESH_DELAY', 'use METAREFRESH_MAXDELAY instead'),
     ('LOG_UNSERIALIZABLE_REQUESTS', 'use SCHEDULER_DEBUG instead'),
 ]

From 2acaa86231e8a743333928907c2933feadf40cd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 27 Feb 2020 15:39:49 +0100
Subject: [PATCH 2676/2974] Do not warn about valid environment variables

---
 scrapy/utils/project.py | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index d9a03ff63..d1dec2543 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -75,9 +75,24 @@ def get_project_settings():
                       "is deprecated.", ScrapyDeprecationWarning)
         settings.setdict(pickle.loads(pickled_settings), priority='project')
 
-    env_overrides = {k[7:]: v for k, v in os.environ.items() if
-                     k.startswith('SCRAPY_')}
-    if env_overrides:
-        warnings.warn("Use of 'SCRAPY_'-prefixed environment variables to override settings is deprecated.", ScrapyDeprecationWarning)
-        settings.setdict(env_overrides, priority='project')
+    scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if
+                      k.startswith('SCRAPY_')}
+    valid_envvars = {
+        'SCRAPY_CHECK',
+        'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE',
+        'SCRAPY_PROJECT',
+        'SCRAPY_PYTHON_SHELL',
+        'SCRAPY_SETTINGS_MODULE',
+    }
+    setting_envvars = {k for k in scrapy_envvars if k not in valid_envvars}
+    if setting_envvars:
+        setting_envvar_list = ', '.join(sorted(setting_envvars))
+        warnings.warn(
+            'Use of environment variables prefixed with SCRAPY_ to override '
+            'settings is deprecated. The following environment variables are '
+            'currently defined: {}'.format(setting_envvar_list),
+            ScrapyDeprecationWarning
+        )
+    settings.setdict(scrapy_envvars, priority='project')
+
     return settings

From 9aae4c0be7b42e27daa2750b6f01eb497edcd98a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 27 Feb 2020 16:31:43 +0100
Subject: [PATCH 2677/2974] Add tests for envvar setting warnings

---
 scrapy/utils/project.py     | 10 +++----
 tests/test_utils_project.py | 56 ++++++++++++++++++++++++++++++++++++-
 2 files changed, 60 insertions(+), 6 deletions(-)

diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index d1dec2543..b8d3ebf9d 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -78,11 +78,11 @@ def get_project_settings():
     scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if
                       k.startswith('SCRAPY_')}
     valid_envvars = {
-        'SCRAPY_CHECK',
-        'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE',
-        'SCRAPY_PROJECT',
-        'SCRAPY_PYTHON_SHELL',
-        'SCRAPY_SETTINGS_MODULE',
+        'CHECK',
+        'PICKLED_SETTINGS_TO_OVERRIDE',
+        'PROJECT',
+        'PYTHON_SHELL',
+        'SETTINGS_MODULE',
     }
     setting_envvars = {k for k in scrapy_envvars if k not in valid_envvars}
     if setting_envvars:
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index bd74b0c34..1ef4eeb14 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -3,7 +3,11 @@ import os
 import tempfile
 import shutil
 import contextlib
-from scrapy.utils.project import data_path
+
+from pytest import warns
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.project import data_path, get_project_settings
 
 
 @contextlib.contextmanager
@@ -41,3 +45,53 @@ class ProjectUtilsTest(unittest.TestCase):
             )
             abspath = os.path.join(os.path.sep, 'absolute', 'path')
             self.assertEqual(abspath, data_path(abspath))
+
+
+@contextlib.contextmanager
+def set_env(**update):
+    modified = set(update.keys()) & set(os.environ.keys())
+    update_after = {k: os.environ[k] for k in modified}
+    remove_after = frozenset(k for k in update if k not in os.environ)
+    try:
+        os.environ.update(update)
+        yield
+    finally:
+        os.environ.update(update_after)
+        for k in remove_after:
+            os.environ.pop(k)
+
+
+class GetProjectSettingsTestCase(unittest.TestCase):
+
+    def test_valid_envvar(self):
+        value = 'tests.test_cmdline.settings'
+        envvars = {
+            'SCRAPY_SETTINGS_MODULE': value,
+        }
+        with set_env(**envvars), warns(None) as warnings:
+            settings = get_project_settings()
+        assert not warnings
+        assert settings.get('SETTINGS_MODULE') == value
+
+    def test_invalid_envvar(self):
+        envvars = {
+            'SCRAPY_FOO': 'bar',
+        }
+        with set_env(**envvars), warns(None) as warnings:
+            get_project_settings()
+        assert len(warnings) == 1
+        assert warnings[0].category == ScrapyDeprecationWarning
+        assert str(warnings[0].message).endswith(': FOO')
+
+    def test_valid_and_invalid_envvars(self):
+        value = 'tests.test_cmdline.settings'
+        envvars = {
+            'SCRAPY_FOO': 'bar',
+            'SCRAPY_SETTINGS_MODULE': value,
+        }
+        with set_env(**envvars), warns(None) as warnings:
+            settings = get_project_settings()
+        assert len(warnings) == 1
+        assert warnings[0].category == ScrapyDeprecationWarning
+        assert str(warnings[0].message).endswith(': FOO')
+        assert settings.get('SETTINGS_MODULE') == value

From c411a51f42a5e6d241d69349b228f1584fdbd31b Mon Sep 17 00:00:00 2001
From: sakshamb2113 <44064539+sakshamb2113@users.noreply.github.com>
Date: Fri, 28 Feb 2020 17:47:02 +0530
Subject: [PATCH 2678/2974] Fix random failures from test_fixed_delay in some
 machines (#4372)

---
 tests/test_crawl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index bbe97d034..e93c668c5 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -37,7 +37,7 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_fixed_delay(self):
-        yield self._test_delay(total=3, delay=0.1)
+        yield self._test_delay(total=3, delay=0.2)
 
     @defer.inlineCallbacks
     def test_randomized_delay(self):

From ef00f8eb8eb4f5727409fd40c5826661db2bb665 Mon Sep 17 00:00:00 2001
From: MaliCN <40772522+MaliYudina@users.noreply.github.com>
Date: Fri, 28 Feb 2020 22:42:07 +0300
Subject: [PATCH 2679/2974] updated with new macOS name (#4308) (#4323)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* changed for new name as "macOS"  (issue #4308)

* updated macOS name

* update macOS name

* updated macOS name

* update for new macOS name

* docs/intro/install.rst: fix macOS header symbols

Co-Authored-By: elacuesta <elacuesta@users.noreply.github.com>

Co-authored-by: Adrián Chaves <adrian@chaves.io>
Co-authored-by: elacuesta <elacuesta@users.noreply.github.com>
---
 README.rst                    |  2 +-
 docs/intro/install.rst        | 12 ++++++------
 docs/news.rst                 | 12 ++++++------
 scrapy/extensions/memusage.py |  2 +-
 4 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/README.rst b/README.rst
index 7fefaeec9..ce5973bcd 100644
--- a/README.rst
+++ b/README.rst
@@ -41,7 +41,7 @@ Requirements
 ============
 
 * Python 3.5+
-* Works on Linux, Windows, Mac OSX, BSD
+* Works on Linux, Windows, macOS, BSD
 
 Install
 =======
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 871281460..89ba0c154 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -12,7 +12,7 @@ under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
 
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
 the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
-and OS X.
+and macOS.
 
 To install Scrapy using ``conda``, run::
 
@@ -148,11 +148,11 @@ you can install Scrapy with ``pip`` after that::
 
 .. _intro-install-macos:
 
-Mac OS X
---------
+macOS
+-----
 
 Building Scrapy's dependencies requires the presence of a C compiler and
-development headers. On OS X this is typically provided by Apple’s Xcode
+development headers. On macOS this is typically provided by Apple’s Xcode
 development tools. To install the Xcode command line tools open a terminal
 window and run::
 
@@ -191,7 +191,7 @@ solutions:
 *   *(Optional)* :ref:`Install Scrapy inside a Python virtual environment
     <intro-using-virtualenv>`.
 
-  This method is a workaround for the above OS X issue, but it's an overall
+  This method is a workaround for the above macOS issue, but it's an overall
   good practice for managing dependencies and can complement the first method.
 
 After any of these workarounds you should be able to install Scrapy::
@@ -207,7 +207,7 @@ For PyPy3, only Linux installation was tested.
 
 Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy.
 This means that these dependecies will be built during installation.
-On OS X, you are likely to face an issue with building Cryptography dependency,
+On macOS, you are likely to face an issue with building Cryptography dependency,
 solution to this problem is described
 `here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
 that is to ``brew install openssl`` and then export the flags that this command
diff --git a/docs/news.rst b/docs/news.rst
index 338b53dc4..c1daedaf2 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1076,7 +1076,7 @@ Cleanups & Refactoring
 ~~~~~~~~~~~~~~~~~~~~~~
 
 - Tests: remove temp files and folders (:issue:`2570`),
-  fixed ProjectUtilsTest on OS X (:issue:`2569`),
+  fixed ProjectUtilsTest on macOS (:issue:`2569`),
   use portable pypy for Linux on Travis CI (:issue:`2710`)
 - Separate building request from ``_requests_to_follow`` in CrawlSpider (:issue:`2562`)
 - Remove “Python 3 progress” badge (:issue:`2567`)
@@ -1645,7 +1645,7 @@ Bugfixes
 - Makes ``_monkeypatches`` more robust (:issue:`1634`).
 - Fixed bug on ``XMLItemExporter`` with non-string fields in
   items (:issue:`1738`).
-- Fixed startproject command in OS X (:issue:`1635`).
+- Fixed startproject command in macOS (:issue:`1635`).
 - Fixed :class:`~scrapy.exporters.PythonItemExporter` and CSVExporter for
   non-string item types (:issue:`1737`).
 - Various logging related fixes (:issue:`1294`, :issue:`1419`, :issue:`1263`,
@@ -1713,12 +1713,12 @@ Scrapy 1.0.4 (2015-12-30)
 - Typos corrections (:commit:`7067117`)
 - fix typos in downloader-middleware.rst and exceptions.rst, middlware -> middleware (:commit:`32f115c`)
 - Add note to Ubuntu install section about Debian compatibility (:commit:`23fda69`)
-- Replace alternative OSX install workaround with virtualenv (:commit:`98b63ee`)
+- Replace alternative macOS install workaround with virtualenv (:commit:`98b63ee`)
 - Reference Homebrew's homepage for installation instructions (:commit:`1925db1`)
 - Add oldest supported tox version to contributing docs (:commit:`5d10d6d`)
 - Note in install docs about pip being already included in python>=2.7.9 (:commit:`85c980e`)
 - Add non-python dependencies to Ubuntu install section in the docs (:commit:`fbd010d`)
-- Add OS X installation section to docs (:commit:`d8f4cba`)
+- Add macOS installation section to docs (:commit:`d8f4cba`)
 - DOC(ENH): specify path to rtd theme explicitly (:commit:`de73b1a`)
 - minor: scrapy.Spider docs grammar (:commit:`1ddcc7b`)
 - Make common practices sample code match the comments (:commit:`1b85bcf`)
@@ -2571,7 +2571,7 @@ Scrapy 0.18.0 (released 2013-08-09)
 - MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62
 - added from_crawler method to spiders
 - added system tests with mock server
-- more improvements to Mac OS compatibility (thanks Alex Cepoi)
+- more improvements to macOS compatibility (thanks Alex Cepoi)
 - several more cleanups to singletons and multi-spider support (thanks Nicolas Ramirez)
 - support custom download slots
 - added --spider option to "shell" command.
@@ -2647,7 +2647,7 @@ Scrapy 0.16.3 (released 2012-12-07)
 
 - Remove concurrency limitation when using download delays and still ensure inter-request delays are enforced (:commit:`487b9b5`)
 - add error details when image pipeline fails (:commit:`8232569`)
-- improve mac os compatibility (:commit:`8dcf8aa`)
+- improve macOS compatibility (:commit:`8dcf8aa`)
 - setup.py: use README.rst to populate long_description (:commit:`7b5310d`)
 - doc: removed obsolete references to ClientForm (:commit:`80f9bb6`)
 - correct docs for default storage backend (:commit:`2aa491b`)
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index c0570567e..14e0fb32d 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -47,7 +47,7 @@ class MemoryUsage(object):
     def get_virtual_size(self):
         size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss
         if sys.platform != 'darwin':
-            # on Mac OS X ru_maxrss is in bytes, on Linux it is in KB
+            # on macOS ru_maxrss is in bytes, on Linux it is in KB
             size *= 1024
         return size
 

From 6aa0ba45532a4fd8e868bb4ea15bf002e430e67f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 3 Mar 2020 09:11:11 +0100
Subject: [PATCH 2680/2974] Write release notes for Scrapy 2.0.0 (#4329)

---
 docs/index.rst                        |   8 +
 docs/intro/install.rst                |   4 +-
 docs/news.rst                         | 458 +++++++++++++++++++++++++-
 docs/topics/asyncio.rst               |  28 ++
 docs/topics/coroutines.rst            | 110 +++++++
 docs/topics/downloader-middleware.rst |   4 +
 docs/topics/exporters.rst             |   5 +-
 docs/topics/feed-exports.rst          |   3 +
 docs/topics/item-pipeline.rst         |  13 +-
 docs/topics/jobs.rst                  |   3 +
 docs/topics/link-extractors.rst       |  10 +-
 docs/topics/loaders.rst               |   3 +
 docs/topics/media-pipeline.rst        |  12 +-
 docs/topics/request-response.rst      |   8 +
 docs/topics/settings.rst              |  43 ++-
 docs/topics/signals.rst               |   3 +
 docs/topics/spiders.rst               |   3 +
 scrapy/http/response/__init__.py      |   5 +
 scrapy/logformatter.py                |  17 +-
 scrapy/utils/reactor.py               |   5 +
 tests/test_crawl.py                   |   2 +-
 21 files changed, 704 insertions(+), 43 deletions(-)
 create mode 100644 docs/topics/asyncio.rst
 create mode 100644 docs/topics/coroutines.rst

diff --git a/docs/index.rst b/docs/index.rst
index a4343b7e0..11aa5c9be 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -165,6 +165,8 @@ Solving specific problems
    topics/autothrottle
    topics/benchmarking
    topics/jobs
+   topics/coroutines
+   topics/asyncio
 
 :doc:`faq`
     Get answers to most frequently asked questions.
@@ -205,6 +207,12 @@ Solving specific problems
 :doc:`topics/jobs`
     Learn how to pause and resume crawls for large spiders.
 
+:doc:`topics/coroutines`
+    Use the :ref:`coroutine syntax <async>`.
+
+:doc:`topics/asyncio`
+    Use :mod:`asyncio` and :mod:`asyncio`-powered libraries.
+
 .. _extending-scrapy:
 
 Extending Scrapy
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 89ba0c154..6356e0eea 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -7,8 +7,8 @@ Installation guide
 Installing Scrapy
 =================
 
-Scrapy runs on Python 3.5 or above
-under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
+Scrapy runs on Python 3.5 or above under CPython (default Python
+implementation) and PyPy (starting with PyPy 5.9).
 
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
 the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
diff --git a/docs/news.rst b/docs/news.rst
index c1daedaf2..dd5e00223 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,8 +3,452 @@
 Release notes
 =============
 
-.. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0,
-          planned for Q4 2019 or Q1 2020, will support **Python 3 only**.
+.. _release-2.0.0:
+
+Scrapy 2.0.0 (2020-03-03)
+-------------------------
+
+Highlights:
+
+* Python 2 support has been removed
+* :doc:`Partial <topics/coroutines>` :ref:`coroutine syntax <async>` support
+  and :doc:`experimental <topics/asyncio>` :mod:`asyncio` support
+* New :meth:`Response.follow_all <scrapy.http.Response.follow_all>` method
+* :ref:`FTP support <media-pipeline-ftp>` for media pipelines
+* New :attr:`Response.certificate <scrapy.http.Response.certificate>`
+  attribute
+* IPv6 support through :setting:`DNS_RESOLVER`
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+*   Python 2 support has been removed, following `Python 2 end-of-life on
+    January 1, 2020`_ (:issue:`4091`, :issue:`4114`, :issue:`4115`,
+    :issue:`4121`, :issue:`4138`, :issue:`4231`, :issue:`4242`, :issue:`4304`,
+    :issue:`4309`, :issue:`4373`)
+
+*   Retry gaveups (see :setting:`RETRY_TIMES`) are now logged as errors instead
+    of as debug information (:issue:`3171`, :issue:`3566`)
+
+*   File extensions that
+    :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
+    ignores by default now also include ``7z``, ``7zip``, ``apk``, ``bz2``,
+    ``cdr``, ``dmg``, ``ico``, ``iso``, ``tar``, ``tar.gz``, ``webm``, and
+    ``xz`` (:issue:`1837`, :issue:`2067`, :issue:`4066`)
+
+*   The :setting:`METAREFRESH_IGNORE_TAGS` setting is now an empty list by
+    default, following web browser behavior (:issue:`3844`, :issue:`4311`)
+
+*   The
+    :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
+    now includes spaces after commas in the value of the ``Accept-Encoding``
+    header that it sets, following web browser behavior (:issue:`4293`)
+
+*   The ``__init__`` method of custom download handlers (see
+    :setting:`DOWNLOAD_HANDLERS`) or subclasses of the following downloader
+    handlers  no longer receives a ``settings`` parameter:
+
+    *   :class:`scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler`
+
+    *   :class:`scrapy.core.downloader.handlers.file.FileDownloadHandler`
+
+    Use the ``from_settings`` or ``from_crawler`` class methods to expose such
+    a parameter to your custom download handlers.
+
+    (:issue:`4126`)
+
+*   We have refactored the :class:`scrapy.core.scheduler.Scheduler` class and
+    related queue classes (see :setting:`SCHEDULER_PRIORITY_QUEUE`,
+    :setting:`SCHEDULER_DISK_QUEUE` and :setting:`SCHEDULER_MEMORY_QUEUE`) to
+    make it easier to implement custom scheduler queue classes. See
+    :ref:`2-0-0-scheduler-queue-changes` below for details.
+
+*   Overridden settings are now logged in a different format. This is more in
+    line with similar information logged at startup (:issue:`4199`)
+
+.. _Python 2 end-of-life on January 1, 2020: https://www.python.org/doc/sunset-python-2/
+
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+*   The :ref:`Scrapy shell <topics-shell>` no longer provides a `sel` proxy
+    object, use :meth:`response.selector <scrapy.http.Response.selector>`
+    instead (:issue:`4347`)
+
+*   LevelDB support has been removed (:issue:`4112`)
+
+*   The following functions have been removed from :mod:`scrapy.utils.python`:
+    ``isbinarytext``, ``is_writable``, ``setattr_default``, ``stringify_dict``
+    (:issue:`4362`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+*   Using environment variables prefixed with ``SCRAPY_`` to override settings
+    is deprecated (:issue:`4300`, :issue:`4374`, :issue:`4375`)
+
+*   :class:`scrapy.linkextractors.FilteringLinkExtractor` is deprecated, use
+    :class:`scrapy.linkextractors.LinkExtractor
+    <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` instead (:issue:`4045`)
+
+*   The ``noconnect`` query string argument of proxy URLs is deprecated and
+    should be removed from proxy URLs (:issue:`4198`)
+
+*   The :meth:`next <scrapy.utils.python.MutableChain.next>` method of
+    :class:`scrapy.utils.python.MutableChain` is deprecated, use the global
+    :func:`next` function or :meth:`MutableChain.__next__
+    <scrapy.utils.python.MutableChain.__next__>` instead (:issue:`4153`)
+
+
+New features
+~~~~~~~~~~~~
+
+*   Added :doc:`partial support <topics/coroutines>` for Python’s
+    :ref:`coroutine syntax <async>` and :doc:`experimental support
+    <topics/asyncio>` for :mod:`asyncio` and :mod:`asyncio`-powered libraries
+    (:issue:`4010`, :issue:`4259`, :issue:`4269`, :issue:`4270`, :issue:`4271`,
+    :issue:`4316`, :issue:`4318`)
+
+*   The new :meth:`Response.follow_all <scrapy.http.Response.follow_all>`
+    method offers the same functionality as
+    :meth:`Response.follow <scrapy.http.Response.follow>` but supports an
+    iterable of URLs as input and returns an iterable of requests
+    (:issue:`2582`, :issue:`4057`, :issue:`4286`)
+
+*   :ref:`Media pipelines <topics-media-pipeline>` now support :ref:`FTP
+    storage <media-pipeline-ftp>` (:issue:`3928`, :issue:`3961`)
+
+*   The new :attr:`Response.certificate <scrapy.http.Response.certificate>`
+    attribute exposes the SSL certificate of the server as a
+    :class:`twisted.internet.ssl.Certificate` object for HTTPS responses
+    (:issue:`2726`, :issue:`4054`)
+
+*   A new :setting:`DNS_RESOLVER` setting allows enabling IPv6 support
+    (:issue:`1031`, :issue:`4227`)
+
+*   A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuring
+    the existing soft limit that pauses request downloads when the total
+    response data being processed is too high (:issue:`1410`, :issue:`3551`)
+
+*   A new :setting:`TWISTED_REACTOR` setting allows customizing the
+    :mod:`~twisted.internet.reactor` that Scrapy uses, allowing to
+    :doc:`enable asyncio support <topics/asyncio>` or deal with a
+    :ref:`common macOS issue <faq-specific-reactor>` (:issue:`2905`,
+    :issue:`4294`)
+
+*   Scheduler disk and memory queues may now use the class methods
+    ``from_crawler`` or ``from_settings`` (:issue:`3884`)
+
+*   The new :attr:`Response.cb_kwargs <scrapy.http.Response.cb_kwargs>`
+    attribute serves as a shortcut for :attr:`Response.request.cb_kwargs
+    <scrapy.http.Request.cb_kwargs>` (:issue:`4331`)
+
+*   :meth:`Response.follow <scrapy.http.Response.follow>` now supports a
+    ``flags`` parameter, for consistency with :class:`~scrapy.http.Request`
+    (:issue:`4277`, :issue:`4279`)
+
+*   :ref:`Item loader processors <topics-loaders-processors>` can now be
+    regular functions, they no longer need to be methods (:issue:`3899`)
+
+*   :class:`~scrapy.spiders.Rule` now accepts an ``errback`` parameter
+    (:issue:`4000`)
+
+*   :class:`~scrapy.http.Request` no longer requires a ``callback`` parameter
+    when an ``errback`` parameter is specified (:issue:`3586`, :issue:`4008`)
+
+*   :class:`~scrapy.logformatter.LogFormatter` now supports some additional
+    methods:
+
+    *   :class:`~scrapy.logformatter.LogFormatter.download_error` for
+        download errors
+
+    *   :class:`~scrapy.logformatter.LogFormatter.item_error` for exceptions
+        raised during item processing by :ref:`item pipelines
+        <topics-item-pipeline>`
+
+    *   :class:`~scrapy.logformatter.LogFormatter.spider_error` for exceptions
+        raised from :ref:`spider callbacks <topics-spiders>`
+
+    (:issue:`374`, :issue:`3986`, :issue:`3989`, :issue:`4176`, :issue:`4188`)
+
+*   The :setting:`FEED_URI` setting now supports :class:`pathlib.Path` values
+    (:issue:`3731`, :issue:`4074`)
+
+*   A new :signal:`request_left_downloader` signal is sent when a request
+    leaves the downloader (:issue:`4303`)
+
+*   Scrapy logs a warning when it detects a request callback or errback that
+    uses ``yield`` but also returns a value, since the returned value would be
+    lost (:issue:`3484`, :issue:`3869`)
+
+*   :class:`~scrapy.spiders.Spider` objects now raise an :exc:`AttributeError`
+    exception if they do not have a :class:`~scrapy.spiders.Spider.start_urls`
+    attribute nor reimplement :class:`~scrapy.spiders.Spider.start_requests`,
+    but have a ``start_url`` attribute (:issue:`4133`, :issue:`4170`)
+
+*   :class:`~scrapy.exporters.BaseItemExporter` subclasses may now use
+    ``super().__init__(**kwargs)`` instead of ``self._configure(kwargs)`` in
+    their ``__init__`` method, passing ``dont_fail=True`` to the parent
+    ``__init__`` method if needed, and accessing ``kwargs`` at ``self._kwargs``
+    after calling their parent ``__init__`` method (:issue:`4193`,
+    :issue:`4370`)
+
+*   A new ``keep_fragments`` parameter of
+    :func:`scrapy.utils.request.request_fingerprint` allows to generate
+    different fingerprints for requests with different fragments in their URL
+    (:issue:`4104`)
+
+*   Download handlers (see :setting:`DOWNLOAD_HANDLERS`) may now use the
+    ``from_settings`` and ``from_crawler`` class methods that other Scrapy
+    components already supported (:issue:`4126`)
+
+*   :class:`scrapy.utils.python.MutableChain.__iter__` now returns ``self``,
+    `allowing it to be used as a sequence <https://lgtm.com/rules/4850080/>`_
+    (:issue:`4153`)
+
+
+Bug fixes
+~~~~~~~~~
+
+*   The :command:`crawl` command now also exits with exit code 1 when an
+    exception happens before the crawling starts (:issue:`4175`, :issue:`4207`)
+
+*   :class:`LinkExtractor.extract_links
+    <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` no longer
+    re-encodes the query string or URLs from non-UTF-8 responses in UTF-8
+    (:issue:`998`, :issue:`1403`, :issue:`1949`, :issue:`4321`)
+
+*   The first spider middleware (see :setting:`SPIDER_MIDDLEWARES`) now also
+    processes exceptions raised from callbacks that are generators
+    (:issue:`4260`, :issue:`4272`)
+
+*   Redirects to URLs starting with 3 slashes (``///``) are now supported
+    (:issue:`4032`, :issue:`4042`)
+
+*   :class:`~scrapy.http.Request` no longer accepts strings as ``url`` simply
+    because they have a colon (:issue:`2552`, :issue:`4094`)
+
+*   The correct encoding is now used for attach names in
+    :class:`~scrapy.mail.MailSender` (:issue:`4229`, :issue:`4239`)
+
+*   :class:`~scrapy.dupefilters.RFPDupeFilter`, the default
+    :setting:`DUPEFILTER_CLASS`, no longer writes an extra ``\r`` character on
+    each line in Windows, which made the size of the ``requests.seen`` file
+    unnecessarily large on that platform (:issue:`4283`)
+
+*   Z shell auto-completion now looks for ``.html`` files, not ``.http`` files,
+    and covers the ``-h`` command-line switch (:issue:`4122`, :issue:`4291`)
+
+*   Adding items to a :class:`scrapy.utils.datatypes.LocalCache` object
+    without a ``limit`` defined no longer raises a :exc:`TypeError` exception
+    (:issue:`4123`)
+
+*   Fixed a typo in the message of the :exc:`ValueError` exception raised when
+    :func:`scrapy.utils.misc.create_instance` gets both ``settings`` and
+    ``crawler`` set to ``None`` (:issue:`4128`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+*   API documentation now links to an online, syntax-highlighted view of the
+    corresponding source code (:issue:`4148`)
+
+*   Links to unexisting documentation pages now allow access to the sidebar
+    (:issue:`4152`, :issue:`4169`)
+
+*   Cross-references within our documentation now display a tooltip when
+    hovered (:issue:`4173`, :issue:`4183`)
+
+*   Improved the documentation about :meth:`LinkExtractor.extract_links
+    <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` and
+    simplified :ref:`topics-link-extractors` (:issue:`4045`)
+
+*   Clarified how :class:`ItemLoader.item <scrapy.loader.ItemLoader.item>`
+    works (:issue:`3574`, :issue:`4099`)
+
+*   Clarified that :func:`logging.basicConfig` should not be used when also
+    using :class:`~scrapy.crawler.CrawlerProcess` (:issue:`2149`,
+    :issue:`2352`, :issue:`3146`, :issue:`3960`)
+
+*   Clarified the requirements for :class:`~scrapy.http.Request` objects
+    :ref:`when using persistence <request-serialization>` (:issue:`4124`,
+    :issue:`4139`)
+
+*   Clarified how to install a :ref:`custom image pipeline
+    <media-pipeline-example>` (:issue:`4034`, :issue:`4252`)
+
+*   Fixed the signatures of the ``file_path`` method in :ref:`media pipeline
+    <topics-media-pipeline>` examples (:issue:`4290`)
+
+*   Covered a backward-incompatible change in Scrapy 1.7.0 affecting custom
+    :class:`scrapy.core.scheduler.Scheduler` subclasses (:issue:`4274`)
+
+*   Improved the ``README.rst`` and ``CODE_OF_CONDUCT.md`` files
+    (:issue:`4059`)
+
+*   Documentation examples are now checked as part of our test suite and we
+    have fixed some of the issues detected (:issue:`4142`, :issue:`4146`,
+    :issue:`4171`, :issue:`4184`, :issue:`4190`)
+
+*   Fixed logic issues, broken links and typos (:issue:`4247`, :issue:`4258`,
+    :issue:`4282`, :issue:`4288`, :issue:`4305`, :issue:`4308`, :issue:`4323`,
+    :issue:`4338`, :issue:`4359`, :issue:`4361`)
+
+*   Improved consistency when referring to the ``__init__`` method of an object
+    (:issue:`4086`, :issue:`4088`)
+
+*   Fixed an inconsistency between code and output in :ref:`intro-overview`
+    (:issue:`4213`)
+
+*   Extended :mod:`~sphinx.ext.intersphinx` usage (:issue:`4147`,
+    :issue:`4172`, :issue:`4185`, :issue:`4194`, :issue:`4197`)
+
+*   We now use a recent version of Python to build the documentation
+    (:issue:`4140`, :issue:`4249`)
+
+*   Cleaned up documentation (:issue:`4143`, :issue:`4275`)
+
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+*   Re-enabled proxy ``CONNECT`` tests (:issue:`2545`, :issue:`4114`)
+
+*   Added Bandit_ security checks to our test suite (:issue:`4162`,
+    :issue:`4181`)
+
+*   Added Flake8_ style checks to our test suite and applied many of the
+    corresponding changes (:issue:`3944`, :issue:`3945`, :issue:`4137`,
+    :issue:`4157`, :issue:`4167`, :issue:`4174`, :issue:`4186`, :issue:`4195`,
+    :issue:`4238`, :issue:`4246`, :issue:`4355`, :issue:`4360`, :issue:`4365`)
+
+*   Improved test coverage (:issue:`4097`, :issue:`4218`, :issue:`4236`)
+
+*   Started reporting slowest tests, and improved the performance of some of
+    them (:issue:`4163`, :issue:`4164`)
+
+*   Fixed broken tests and refactored some tests (:issue:`4014`, :issue:`4095`,
+    :issue:`4244`, :issue:`4268`, :issue:`4372`)
+
+*   Modified the :doc:`tox <tox:index>` configuration to allow running tests
+    with any Python version, run Bandit_ and Flake8_ tests by default, and
+    enforce a minimum tox version programmatically (:issue:`4179`)
+
+*   Cleaned up code (:issue:`3937`, :issue:`4208`, :issue:`4209`,
+    :issue:`4210`, :issue:`4212`, :issue:`4369`, :issue:`4376`, :issue:`4378`)
+
+.. _Bandit: https://bandit.readthedocs.io/
+.. _Flake8: https://flake8.pycqa.org/en/latest/
+
+
+.. _2-0-0-scheduler-queue-changes:
+
+Changes to scheduler queue classes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following changes may impact any custom queue classes of all types:
+
+*   The ``push`` method no longer receives a second positional parameter
+    containing ``request.priority * -1``. If you need that value, get it
+    from the first positional parameter, ``request``, instead, or use
+    the new :meth:`~scrapy.core.scheduler.ScrapyPriorityQueue.priority`
+    method in :class:`scrapy.core.scheduler.ScrapyPriorityQueue`
+    subclasses.
+
+The following changes may impact custom priority queue classes:
+
+*   In the ``__init__`` method or the ``from_crawler`` or ``from_settings``
+    class methods:
+
+    *   The parameter that used to contain a factory function,
+        ``qfactory``, is now passed as a keyword parameter named
+        ``downstream_queue_cls``.
+
+    *   A new keyword parameter has been added: ``key``. It is a string
+        that is always an empty string for memory queues and indicates the
+        :setting:`JOB_DIR` value for disk queues.
+
+    *   The parameter for disk queues that contains data from the previous
+        crawl, ``startprios`` or ``slot_startprios``, is now passed as a
+        keyword parameter named ``startprios``.
+
+    *   The ``serialize`` parameter is no longer passed. The disk queue
+        class must take care of request serialization on its own before
+        writing to disk, using the
+        :func:`~scrapy.utils.reqser.request_to_dict` and
+        :func:`~scrapy.utils.reqser.request_from_dict` functions from the
+        :mod:`scrapy.utils.reqser` module.
+
+The following changes may impact custom disk and memory queue classes:
+
+*   The signature of the ``__init__`` method is now
+    ``__init__(self, crawler, key)``.
+
+The following changes affect specifically the
+:class:`~scrapy.core.scheduler.ScrapyPriorityQueue` and
+:class:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue` classes from
+:mod:`scrapy.core.scheduler` and may affect subclasses:
+
+*   In the ``__init__`` method, most of the changes described above apply.
+
+    ``__init__`` may still receive all parameters as positional parameters,
+    however:
+
+    *   ``downstream_queue_cls``, which replaced ``qfactory``, must be
+        instantiated differently.
+
+        ``qfactory`` was instantiated with a priority value (integer).
+
+        Instances of ``downstream_queue_cls`` should be created using
+        the new
+        :meth:`ScrapyPriorityQueue.qfactory <scrapy.core.scheduler.ScrapyPriorityQueue.qfactory>`
+        or
+        :meth:`DownloaderAwarePriorityQueue.pqfactory <scrapy.core.scheduler.DownloaderAwarePriorityQueue.pqfactory>`
+        methods.
+
+    *   The new ``key`` parameter displaced the ``startprios``
+        parameter 1 position to the right.
+
+*   The following class attributes have been added:
+
+    *   :attr:`~scrapy.core.scheduler.ScrapyPriorityQueue.crawler`
+
+    *   :attr:`~scrapy.core.scheduler.ScrapyPriorityQueue.downstream_queue_cls`
+        (details above)
+
+    *   :attr:`~scrapy.core.scheduler.ScrapyPriorityQueue.key` (details above)
+
+*   The ``serialize`` attribute has been removed (details above)
+
+The following changes affect specifically the
+:class:`~scrapy.core.scheduler.ScrapyPriorityQueue` class and may affect
+subclasses:
+
+*   A new :meth:`~scrapy.core.scheduler.ScrapyPriorityQueue.priority`
+    method has been added which, given a request, returns
+    ``request.priority * -1``.
+
+    It is used in :meth:`~scrapy.core.scheduler.ScrapyPriorityQueue.push`
+    to make up for the removal of its ``priority`` parameter.
+
+*   The ``spider`` attribute has been removed. Use
+    :attr:`crawler.spider <scrapy.core.scheduler.ScrapyPriorityQueue.crawler>`
+    instead.
+
+The following changes affect specifically the
+:class:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue` class and may
+affect subclasses:
+
+*   A new :attr:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue.pqueues`
+    attribute offers a mapping of downloader slot names to the
+    corresponding instances of
+    :attr:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue.downstream_queue_cls`.
+
+(:issue:`3884`)
+
 
 .. _release-1.8.0:
 
@@ -288,12 +732,12 @@ Backward-incompatible changes
     :class:`~scrapy.http.Request` objects instead of arbitrary Python data
     structures.
 
-*   An additional ``crawler`` parameter has been added to the ``__init__`` method
-    of the :class:`scrapy.core.scheduler.Scheduler` class. 
-    Custom scheduler subclasses which don't accept arbitrary parameters in 
-    their ``__init__`` method might break because of this change.
+*   An additional ``crawler`` parameter has been added to the ``__init__``
+    method of the :class:`~scrapy.core.scheduler.Scheduler` class. Custom
+    scheduler subclasses which don't accept arbitrary parameters in their
+    ``__init__`` method might break because of this change.
 
-    For more information, refer to the documentation for the :setting:`SCHEDULER` setting.
+    For more information, see :setting:`SCHEDULER`.
 
 See also :ref:`1.7-deprecation-removals` below.
 
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
new file mode 100644
index 000000000..038a459fd
--- /dev/null
+++ b/docs/topics/asyncio.rst
@@ -0,0 +1,28 @@
+=======
+asyncio
+=======
+
+.. versionadded:: 2.0
+
+Scrapy has partial support :mod:`asyncio`. After you :ref:`install the asyncio
+reactor <install-asyncio>`, you may use :mod:`asyncio` and
+:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
+
+.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy
+             versions may introduce related changes without a deprecation
+             period or warning.
+
+.. _install-asyncio:
+
+Installing the asyncio reactor
+==============================
+
+To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to
+``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``.
+
+If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to
+install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`
+reactor manually. You can do that using
+:func:`~scrapy.utils.reactor.install_reactor`::
+
+    install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
new file mode 100644
index 000000000..487cf4c6c
--- /dev/null
+++ b/docs/topics/coroutines.rst
@@ -0,0 +1,110 @@
+==========
+Coroutines
+==========
+
+.. versionadded:: 2.0
+
+Scrapy has :ref:`partial support <coroutine-support>` for the
+:ref:`coroutine syntax <async>`.
+
+.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy
+             versions may introduce related API and behavior changes without a
+             deprecation period or warning.
+
+.. _coroutine-support:
+
+Supported callables
+===================
+
+The following callables may be defined as coroutines using ``async def``, and
+hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
+
+-   :class:`~scrapy.http.Request` callbacks.
+
+    The following are known caveats of the current implementation that we aim
+    to address in future versions of Scrapy:
+
+    -   The callback output is not processed until the whole callback finishes.
+
+        As a side effect, if the callback raises an exception, none of its
+        output is processed.
+
+    -   Because `asynchronous generators were introduced in Python 3.6`_, you
+        can only use ``yield`` if you are using Python 3.6 or later.
+
+        If you need to output multiple items or requests and you are using
+        Python 3.5, return an iterable (e.g. a list) instead.
+
+-   The :meth:`process_item` method of
+    :ref:`item pipelines <topics-item-pipeline>`.
+
+-   The
+    :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request`,
+    :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`,
+    and
+    :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`
+    methods of
+    :ref:`downloader middlewares <topics-downloader-middleware-custom>`.
+
+-   :ref:`Signal handlers that support deferreds <signal-deferred>`.
+
+.. _asynchronous generators were introduced in Python 3.6: https://www.python.org/dev/peps/pep-0525/
+
+Usage
+=====
+
+There are several use cases for coroutines in Scrapy. Code that would
+return Deferreds when written for previous Scrapy versions, such as downloader
+middlewares and signal handlers, can be rewritten to be shorter and cleaner::
+
+    class DbPipeline:
+        def _update_item(self, data, item):
+            item['field'] = data
+            return item
+
+        def process_item(self, item, spider):
+            dfd = db.get_some_data(item['id'])
+            dfd.addCallback(self._update_item, item)
+            return dfd
+
+becomes::
+
+    class DbPipeline:
+        async def process_item(self, item, spider):
+            item['field'] = await db.get_some_data(item['id'])
+            return item
+
+Coroutines may be used to call asynchronous code. This includes other
+coroutines, functions that return Deferreds and functions that return
+`awaitable objects`_ such as :class:`~asyncio.Future`. This means you can use
+many useful Python libraries providing such code::
+
+    class MySpider(Spider):
+        # ...
+        async def parse_with_deferred(self, response):
+            additional_response = await treq.get('https://additional.url')
+            additional_data = await treq.content(additional_response)
+            # ... use response and additional_data to yield items and requests
+
+        async def parse_with_asyncio(self, response):
+            async with aiohttp.ClientSession() as session:
+                async with session.get('https://additional.url') as additional_response:
+                    additional_data = await r.text()
+            # ... use response and additional_data to yield items and requests
+
+.. note:: Many libraries that use coroutines, such as `aio-libs`_, require the
+          :mod:`asyncio` loop and to use them you need to
+          :doc:`enable asyncio support in Scrapy<asyncio>`.
+
+Common use cases for asynchronous code include:
+
+* requesting data from websites, databases and other services (in callbacks,
+  pipelines and middlewares);
+* storing data in databases (in pipelines and middlewares);
+* delaying the spider initialization until some external event (in the
+  :signal:`spider_opened` handler);
+* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see
+  :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
+
+.. _aio-libs: https://github.com/aio-libs
+.. _awaitable objects: https://docs.python.org/3/glossary.html#term-awaitable
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 0297ef3a0..73648994d 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -872,6 +872,10 @@ Default: ``[]``
 
 Meta tags within these tags are ignored.
 
+.. versionchanged:: 2.0
+   The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
+   ``['script', 'noscript']`` to ``[]``.
+
 .. setting:: METAREFRESH_MAXDELAY
 
 METAREFRESH_MAXDELAY
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index b8d898022..d411e2eed 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -137,7 +137,7 @@ output examples, which assume you're exporting these two items::
 BaseItemExporter
 ----------------
 
-.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0)
+.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0, dont_fail=False)
 
    This is the (abstract) base class for all Item Exporters. It provides
    support for common features used by all (concrete) Item Exporters, such as
@@ -148,6 +148,9 @@ BaseItemExporter
    populate their respective instance attributes: :attr:`fields_to_export`,
    :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
 
+   .. versionadded:: 2.0
+      The *dont_fail* parameter.
+
    .. method:: export_item(item)
 
       Exports the given item. This method must be implemented in subclasses.
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 1d94807a4..42f1cad90 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -236,6 +236,9 @@ supported URI schemes.
 
 This setting is required for enabling the feed exports.
 
+.. versionchanged:: 2.0
+   Added :class:`pathlib.Path` support.
+
 .. setting:: FEED_FORMAT
 
 FEED_FORMAT
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 801d48fd5..98e2506e5 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -162,14 +162,16 @@ method and how to clean up the resources properly.::
 .. _pymongo: https://api.mongodb.com/python/current/
 
 
+.. _ScreenshotPipeline:
+
 Take screenshot of item
 -----------------------
 
 This example demonstrates how to return a
 :class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
 It uses Splash_ to render screenshot of item url. Pipeline
-makes request to locally running instance of Splash_. After request is downloaded
-and Deferred callback fires, it saves item to a file and adds filename to an item.
+makes request to locally running instance of Splash_. After request is downloaded,
+it saves the screenshot to a file and adds filename to the item.
 
 ::
 
@@ -184,15 +186,12 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
 
         SPLASH_URL = "http://localhost:8050/render.png?url={}"
 
-        def process_item(self, item, spider):
+        async def process_item(self, item, spider):
             encoded_item_url = quote(item["url"])
             screenshot_url = self.SPLASH_URL.format(encoded_item_url)
             request = scrapy.Request(screenshot_url)
-            dfd = spider.crawler.engine.download(request, spider)
-            dfd.addBoth(self.return_item, item)
-            return dfd
+            response = await spider.crawler.engine.download(request, spider)
 
-        def return_item(self, response, item):
             if response.status != 200:
                 # Error happened, return item.
                 return item
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index c34ba336b..58601824a 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -68,6 +68,9 @@ Cookies may expire. So, if you don't resume your spider quickly the requests
 scheduled may no longer work. This won't be an issue if you spider doesn't rely
 on cookies.
 
+
+.. _request-serialization:
+
 Request serialization
 ---------------------
 
diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 8c8019438..0162a331a 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -64,9 +64,13 @@ LxmlLinkExtractor
 
     :param deny_extensions: a single value or list of strings containing
         extensions that should be ignored when extracting links.
-        If not given, it will default to the
-        ``IGNORED_EXTENSIONS`` list defined in the
-        `scrapy.linkextractors`_ package.
+        If not given, it will default to
+        :data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.
+
+        .. versionchanged:: 2.0
+           :data:`~scrapy.linkextractors.IGNORED_EXTENSIONS` now includes
+           ``7z``, ``7zip``, ``apk``, ``bz2``, ``cdr``, ``dmg``, ``ico``,
+           ``iso``, ``tar``, ``tar.gz``, ``webm``, and ``xz``.
     :type deny_extensions: list
 
     :param restrict_xpaths: is an XPath (or list of XPath's) which defines
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 9d5fccbbc..5f75ccbff 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -136,6 +136,9 @@ with the data to be parsed, and return a parsed value. So you can use any
 function as input or output processor. The only requirement is that they must
 accept one (and only one) positional argument, which will be an iterable.
 
+.. versionchanged:: 2.0
+   Processors no longer need to be methods.
+
 .. note:: Both input and output processors must receive an iterable as their
    first argument. The output of those functions can be anything. The result of
    input processors will be appended to an internal list (in the Loader)
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 67a0bfdba..cd84905c5 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -116,12 +116,6 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
 Supported Storage
 =================
 
-File system is currently the only officially supported storage, but there are
-also support for storing files in `Amazon S3`_ and `Google Cloud Storage`_.
-
-.. _Amazon S3: https://aws.amazon.com/s3/
-.. _Google Cloud Storage: https://cloud.google.com/storage/
-
 File system storage
 -------------------
 
@@ -147,9 +141,13 @@ Where:
 * ``full`` is a sub-directory to separate full images from thumbnails (if
   used). For more info see :ref:`topics-images-thumbnails`.
 
+.. _media-pipeline-ftp:
+
 FTP server storage
 ------------------
 
+.. versionadded:: 2.0
+
 :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can point to an FTP server.
 Scrapy will automatically upload the files to the server.
 
@@ -573,6 +571,8 @@ See here the methods that you can override in your custom Images Pipeline:
       By default, the :meth:`item_completed` method returns the item.
 
 
+.. _media-pipeline-example:
+
 Custom Images pipeline example
 ==============================
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index c4c2845c9..b2a60ff39 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -31,6 +31,8 @@ Request objects
     a :class:`Response`.
 
     :param url: the URL of this request
+
+        If the URL is invalid, a :exc:`ValueError` exception is raised.
     :type url: string
 
     :param callback: the function that will be called with the response of this
@@ -125,6 +127,10 @@ Request objects
        :exc:`~twisted.python.failure.Failure` as first parameter.
        For more information,
        see :ref:`topics-request-response-ref-errbacks` below.
+
+       .. versionchanged:: 2.0
+          The *callback* parameter is no longer required when the *errback*
+          parameter is specified.
     :type errback: callable
 
     :param flags:  Flags sent to the request, can be used for logging or similar purposes.
@@ -677,6 +683,8 @@ Response objects
 
     .. attribute:: Response.cb_kwargs
 
+        .. versionadded:: 2.0
+
         A shortcut to the :attr:`Request.cb_kwargs` attribute of the
         :attr:`Response.request` object (i.e. ``self.request.cb_kwargs``).
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 5394147da..a70023efa 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -381,6 +381,8 @@ DNS in-memory cache size.
 DNS_RESOLVER
 ------------
 
+.. versionadded:: 2.0
+
 Default: ``'scrapy.resolver.CachingThreadedResolver'``
 
 The class to be used to resolve DNS names. The default ``scrapy.resolver.CachingThreadedResolver``
@@ -1258,6 +1260,9 @@ does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`.
 
 SCRAPER_SLOT_MAX_ACTIVE_SIZE
 ----------------------------
+
+.. versionadded:: 2.0
+
 Default: ``5_000_000``
 
 Soft limit (in bytes) for response data being processed.
@@ -1447,24 +1452,36 @@ in the ``project`` subdirectory.
 TWISTED_REACTOR
 ---------------
 
+.. versionadded:: 2.0
+
 Default: ``None``
 
-Import path of a given Twisted reactor, for instance:
-:class:`twisted.internet.asyncioreactor.AsyncioSelectorReactor`.
+Import path of a given :mod:`~twisted.internet.reactor`.
 
-Scrapy will install this reactor if no other is installed yet, such as when
-the ``scrapy`` CLI program is invoked or when using the
-:class:`~scrapy.crawler.CrawlerProcess` class. If you are using the
-:class:`~scrapy.crawler.CrawlerRunner` class, you need to install the correct
-reactor manually. An exception will be raised if the installation fails.
+Scrapy will install this reactor if no other reactor is installed yet, such as
+when the ``scrapy`` CLI program is invoked or when using the
+:class:`~scrapy.crawler.CrawlerProcess` class.
 
-The default value for this option is currently ``None``, which means that Scrapy
-will not attempt to install any specific reactor, and the default one defined by
-Twisted for the current platform will be used. This is to maintain backward
-compatibility and avoid possible problems caused by using a non-default reactor.
+If you are using the :class:`~scrapy.crawler.CrawlerRunner` class, you also
+need to install the correct reactor manually. You can do that using
+:func:`~scrapy.utils.reactor.install_reactor`:
 
-For additional information, please see
-:doc:`core/howto/choosing-reactor`.
+.. autofunction:: scrapy.utils.reactor.install_reactor
+
+If a reactor is already installed,
+:func:`~scrapy.utils.reactor.install_reactor` has no effect.
+
+:meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises
+:exc:`Exception` if the installed reactor does not match the
+:setting:`TWISTED_REACTOR` setting.
+
+The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
+means that Scrapy will not attempt to install any specific reactor, and the
+default reactor defined by Twisted for the current platform will be used. This
+is to maintain backward compatibility and avoid possible problems caused by
+using a non-default reactor.
+
+For additional information, see :doc:`core/howto/choosing-reactor`.
 
 
 .. setting:: URLLENGTH_LIMIT
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index d3cfb0307..2def53848 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -46,6 +46,7 @@ Here is a simple example showing how you can catch signals and perform some acti
         def parse(self, response):
             pass
 
+.. _signal-deferred:
 
 Deferred signal handlers
 ========================
@@ -301,6 +302,8 @@ request_left_downloader
 .. signal:: request_left_downloader
 .. function:: request_left_downloader(request, spider)
 
+    .. versionadded:: 2.0
+
     Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of
     failure.
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index e0f33de66..89609db7d 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -420,6 +420,9 @@ Crawling rules
    It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
    instance as first parameter.
 
+   .. versionadded:: 2.0
+      The *errback* parameter.
+
 CrawlSpider example
 ~~~~~~~~~~~~~~~~~~~
 
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 119dd2f63..682cec161 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -129,6 +129,9 @@ class Response(object_ref):
         :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
         method which supports selectors in addition to absolute/relative URLs
         and Link objects.
+
+        .. versionadded:: 2.0
+           The *flags* parameter.
         """
         if isinstance(url, Link):
             url = url.url
@@ -157,6 +160,8 @@ class Response(object_ref):
                    dont_filter=False, errback=None, cb_kwargs=None, flags=None):
         # type: (...) -> Generator[Request, None, None]
         """
+        .. versionadded:: 2.0
+
         Return an iterable of :class:`~.Request` instances to follow all links
         in ``urls``. It accepts the same arguments as ``Request.__init__`` method,
         but elements of ``urls`` can be relative URLs or :class:`~scrapy.link.Link` objects,
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 194013642..14cec44a6 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -97,7 +97,11 @@ class LogFormatter(object):
         }
 
     def item_error(self, item, exception, response, spider):
-        """Logs a message when an item causes an error while it is passing through the item pipeline."""
+        """Logs a message when an item causes an error while it is passing
+        through the item pipeline.
+
+        .. versionadded:: 2.0
+        """
         return {
             'level': logging.ERROR,
             'msg': ITEMERRORMSG,
@@ -107,7 +111,10 @@ class LogFormatter(object):
         }
 
     def spider_error(self, failure, request, response, spider):
-        """Logs an error message from a spider."""
+        """Logs an error message from a spider.
+
+        .. versionadded:: 2.0
+        """
         return {
             'level': logging.ERROR,
             'msg': SPIDERERRORMSG,
@@ -118,7 +125,11 @@ class LogFormatter(object):
         }
 
     def download_error(self, failure, request, spider, errmsg=None):
-        """Logs a download error message from a spider (typically coming from the engine)."""
+        """Logs a download error message from a spider (typically coming from
+        the engine).
+
+        .. versionadded:: 2.0
+        """
         args = {'request': request}
         if errmsg:
             msg = DOWNLOADERRORMSG_LONG
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 6513e06c9..17d6b2857 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -50,6 +50,8 @@ class CallLaterOnce(object):
 
 
 def install_reactor(reactor_path):
+    """Installs the :mod:`~twisted.internet.reactor` with the specified
+    import path."""
     reactor_class = load_object(reactor_path)
     if reactor_class is asyncioreactor.AsyncioSelectorReactor:
         with suppress(error.ReactorAlreadyInstalledError):
@@ -63,6 +65,9 @@ def install_reactor(reactor_path):
 
 
 def verify_installed_reactor(reactor_path):
+    """Raises :exc:`Exception` if the installed
+    :mod:`~twisted.internet.reactor` does not match the specified import
+    path."""
     from twisted.internet import reactor
     reactor_class = load_object(reactor_path)
     if not isinstance(reactor, reactor_class):
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index e93c668c5..3f8a7435c 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -325,7 +325,7 @@ with multiples lines
     @mark.only_asyncio()
     @defer.inlineCallbacks
     def test_async_def_asyncio_parse(self):
-        runner = CrawlerRunner({"ASYNCIO_REACTOR": True})
+        runner = CrawlerRunner({"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"})
         runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
         with LogCapture() as log:
             yield runner.join()

From a4dbb7754b999c8c6a5239bb3f58e951369e017e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 3 Mar 2020 09:13:00 +0100
Subject: [PATCH 2681/2974] =?UTF-8?q?Bump=20version:=201.8.0=20=E2=86=92?=
 =?UTF-8?q?=202.0.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 3 +--
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index c9f1abea5..f347a0cd0 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,8 +1,7 @@
 [bumpversion]
-current_version = 1.8.0
+current_version = 2.0.0
 commit = True
 tag = True
 tag_name = {new_version}
 
 [bumpversion:file:scrapy/VERSION]
-
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 27f9cd322..227cea215 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.8.0
+2.0.0

From 1b591ff061f2b38bf328e1d2a4acd9643d45ad80 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Fri, 28 Feb 2020 02:10:13 +0000
Subject: [PATCH 2682/2974] Obsolete deprecated settings

Obsolete REDIRECT_MAX_METAREFRESH_DELAY
  which has been deprecated since Scrapy 0.18

Obsolete LOG_UNSERIALIZABLE_REQUESTS
  which has been deprecated since Scrapy 1.2.0
  and is replaced by SCHEDULER_DEBUG
---
 scrapy/core/scheduler.py                 | 3 +--
 scrapy/downloadermiddlewares/redirect.py | 3 +--
 scrapy/settings/deprecated.py            | 2 --
 tests/test_scheduler.py                  | 2 +-
 4 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index e184ed50e..c96b9b719 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -66,8 +66,7 @@ class Scheduler(object):
 
         dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
         mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
-        logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS',
-                                    settings.getbool('SCHEDULER_DEBUG'))
+        logunser = settings.getbool('SCHEDULER_DEBUG')
         return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
                    stats=crawler.stats, pqclass=pqclass, dqclass=dqclass,
                    mqclass=mqclass, crawler=crawler)
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 77cb5aa94..08cff8a55 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -93,8 +93,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
     def __init__(self, settings):
         super(MetaRefreshMiddleware, self).__init__(settings)
         self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
-        self._maxdelay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY',
-                                         settings.getint('METAREFRESH_MAXDELAY'))
+        self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')
 
     def process_response(self, request, response, spider):
         if request.meta.get('dont_redirect', False) or request.method == 'HEAD' or \
diff --git a/scrapy/settings/deprecated.py b/scrapy/settings/deprecated.py
index 1211908df..f6f878725 100644
--- a/scrapy/settings/deprecated.py
+++ b/scrapy/settings/deprecated.py
@@ -11,8 +11,6 @@ DEPRECATED_SETTINGS = [
     ('SQLITE_DB', 'no longer supported'),
     ('AUTOTHROTTLE_MIN_DOWNLOAD_DELAY', 'use DOWNLOAD_DELAY instead'),
     ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'),
-    ('REDIRECT_MAX_METAREFRESH_DELAY', 'use METAREFRESH_MAXDELAY instead'),
-    ('LOG_UNSERIALIZABLE_REQUESTS', 'use SCHEDULER_DEBUG instead'),
 ]
 
 
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index e0e3600e5..13c297084 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -46,7 +46,7 @@ class MockCrawler(Crawler):
     def __init__(self, priority_queue_cls, jobdir):
 
         settings = dict(
-                LOG_UNSERIALIZABLE_REQUESTS=False,
+                SCHEDULER_DEBUG=False,
                 SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
                 SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
                 SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,

From 64002255554aa2aa79863b657e5d1674d72b228d Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Fri, 28 Feb 2020 00:06:11 +0000
Subject: [PATCH 2683/2974] Drop horribly outdated deb package build files

---
 Makefile.buildbot               | 24 --------------------
 debian/changelog                |  5 -----
 debian/compat                   |  1 -
 debian/control                  | 20 -----------------
 debian/copyright                | 40 ---------------------------------
 debian/pyversions               |  1 -
 debian/rules                    |  5 -----
 debian/scrapy.docs              |  2 --
 debian/scrapy.install           |  2 --
 debian/scrapy.lintian-overrides |  1 -
 debian/scrapy.manpages          |  1 -
 11 files changed, 102 deletions(-)
 delete mode 100644 Makefile.buildbot
 delete mode 100644 debian/changelog
 delete mode 100644 debian/compat
 delete mode 100644 debian/control
 delete mode 100644 debian/copyright
 delete mode 100644 debian/pyversions
 delete mode 100755 debian/rules
 delete mode 100644 debian/scrapy.docs
 delete mode 100644 debian/scrapy.install
 delete mode 100644 debian/scrapy.lintian-overrides
 delete mode 100644 debian/scrapy.manpages

diff --git a/Makefile.buildbot b/Makefile.buildbot
deleted file mode 100644
index 775538259..000000000
--- a/Makefile.buildbot
+++ /dev/null
@@ -1,24 +0,0 @@
-TRIAL := $(shell which trial)
-BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
-export PYTHONPATH=$(PWD)
-
-test:
-	coverage run --branch $(TRIAL) --reporter=text tests
-	rm -rf htmlcov && coverage html
-	-s3cmd sync -P htmlcov/ s3://static.scrapy.org/coverage-scrapy-$(BRANCH)/
-
-build:
-	git describe --tags --match '[0-9]*' |sed 's/-/.post/;s/-g/+g/' >scrapy/VERSION
-	debchange -m -D unstable --force-distribution -v \
-	  $$(python setup.py --version |sed -r 's/([0-9]+.[0-9]+.[0-9]+)(a|b|rc|dev)([0-9]*)/\1~\2\3/')-$$(date +%s) \
-	  "Automatic build"
-	debuild -us -uc -b
-
-clean:
-	git checkout debian scrapy/VERSION
-	git clean -dfq
-
-pypi:
-	umask 0022 &&  chmod -R a+rX . && python setup.py sdist upload
-
-.PHONY: clean test build
diff --git a/debian/changelog b/debian/changelog
deleted file mode 100644
index dde97f9e3..000000000
--- a/debian/changelog
+++ /dev/null
@@ -1,5 +0,0 @@
-scrapy (0.11) unstable; urgency=low
-
-  * Initial release.
-
- -- Scrapinghub Team <info@scrapinghub.com>  Thu, 10 Jun 2010 17:24:02 -0300
diff --git a/debian/compat b/debian/compat
deleted file mode 100644
index 7f8f011eb..000000000
--- a/debian/compat
+++ /dev/null
@@ -1 +0,0 @@
-7
diff --git a/debian/control b/debian/control
deleted file mode 100644
index 2cc8eedf4..000000000
--- a/debian/control
+++ /dev/null
@@ -1,20 +0,0 @@
-Source: scrapy
-Section: python
-Priority: optional
-Maintainer: Scrapinghub Team <info@scrapinghub.com>
-Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2)
-Standards-Version: 3.8.4
-Homepage: https://scrapy.org/
-
-Package: scrapy
-Architecture: all
-Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl,
- python-w3lib (>= 1.8.0), python-queuelib, python-cssselect (>= 0.9), python-six (>=1.5.2)
-Recommends: python-setuptools
-Conflicts: python-scrapy, scrapy-0.25
-Provides: python-scrapy, scrapy-0.25
-Description: Python web crawling and web scraping framework
- Scrapy is a fast high-level web crawling and web scraping framework,
- used to crawl websites and extract structured data from their pages.
- It can be used for a wide range of purposes, from data mining to
- monitoring and automated testing.
diff --git a/debian/copyright b/debian/copyright
deleted file mode 100644
index c1bf47565..000000000
--- a/debian/copyright
+++ /dev/null
@@ -1,40 +0,0 @@
-This package was debianized by the Scrapinghub team <info@scrapinghub.com>.
-
-It was downloaded from https://scrapy.org
-
-Upstream Author: Scrapy Developers
-
-Copyright: 2007-2013 Scrapy Developers
-
-License: bsd
-
-Copyright (c) Scrapy developers.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
-    1. Redistributions of source code must retain the above copyright notice,
-       this list of conditions and the following disclaimer.
-
-    2. Redistributions in binary form must reproduce the above copyright
-       notice, this list of conditions and the following disclaimer in the
-       documentation and/or other materials provided with the distribution.
-
-    3. Neither the name of Scrapy nor the names of its contributors may be used
-       to endorse or promote products derived from this software without
-       specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-The Debian packaging is (C) 2010-2013, Scrapinghub <info@scrapinghub.com> and
-is licensed under the BSD, see `/usr/share/common-licenses/BSD'.
diff --git a/debian/pyversions b/debian/pyversions
deleted file mode 100644
index 1effb0034..000000000
--- a/debian/pyversions
+++ /dev/null
@@ -1 +0,0 @@
-2.7
diff --git a/debian/rules b/debian/rules
deleted file mode 100755
index b8796e6e3..000000000
--- a/debian/rules
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/usr/bin/make -f
-# -*- makefile -*-
-
-%:
-	dh $@
diff --git a/debian/scrapy.docs b/debian/scrapy.docs
deleted file mode 100644
index c19ffba4d..000000000
--- a/debian/scrapy.docs
+++ /dev/null
@@ -1,2 +0,0 @@
-README.rst
-AUTHORS
diff --git a/debian/scrapy.install b/debian/scrapy.install
deleted file mode 100644
index c288ebed3..000000000
--- a/debian/scrapy.install
+++ /dev/null
@@ -1,2 +0,0 @@
-extras/scrapy_bash_completion etc/bash_completion.d/
-extras/scrapy_zsh_completion /usr/share/zsh/vendor-completions/_scrapy
diff --git a/debian/scrapy.lintian-overrides b/debian/scrapy.lintian-overrides
deleted file mode 100644
index b5de7f67d..000000000
--- a/debian/scrapy.lintian-overrides
+++ /dev/null
@@ -1 +0,0 @@
-new-package-should-close-itp-bug
diff --git a/debian/scrapy.manpages b/debian/scrapy.manpages
deleted file mode 100644
index 4818e9c92..000000000
--- a/debian/scrapy.manpages
+++ /dev/null
@@ -1 +0,0 @@
-extras/scrapy.1

From 6c35baae25517ed942c72d14d93363a389e5a9d3 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Wed, 4 Mar 2020 00:40:11 +0000
Subject: [PATCH 2684/2974] Remove deprecated SiteNode and MultiValueDict
 classes

---
 scrapy/utils/datatypes.py | 174 --------------------------------------
 1 file changed, 174 deletions(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index b07f995cf..175f92d77 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -6,183 +6,9 @@ This module must not depend on any module outside the Standard Library.
 """
 
 import collections
-import copy
-import warnings
 import weakref
 from collections.abc import Mapping
 
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-
-class MultiValueDictKeyError(KeyError):
-    def __init__(self, *args, **kwargs):
-        warnings.warn(
-            "scrapy.utils.datatypes.MultiValueDictKeyError is deprecated "
-            "and will be removed in future releases.",
-            category=ScrapyDeprecationWarning,
-            stacklevel=2
-        )
-        super(MultiValueDictKeyError, self).__init__(*args, **kwargs)
-
-
-class MultiValueDict(dict):
-    """
-    A subclass of dictionary customized to handle multiple values for the same key.
-
-    >>> d = MultiValueDict({'name': ['Adrian', 'Simon'], 'position': ['Developer']})
-    >>> d['name']
-    'Simon'
-    >>> d.getlist('name')
-    ['Adrian', 'Simon']
-    >>> d.get('lastname', 'nonexistent')
-    'nonexistent'
-    >>> d.setlist('lastname', ['Holovaty', 'Willison'])
-
-    This class exists to solve the irritating problem raised by cgi.parse_qs,
-    which returns a list for every key, even though most Web forms submit
-    single name-value pairs.
-    """
-    def __init__(self, key_to_list_mapping=()):
-        warnings.warn("scrapy.utils.datatypes.MultiValueDict is deprecated "
-                      "and will be removed in future releases.",
-                      category=ScrapyDeprecationWarning,
-                      stacklevel=2)
-        dict.__init__(self, key_to_list_mapping)
-
-    def __repr__(self):
-        return "<%s: %s>" % (self.__class__.__name__, dict.__repr__(self))
-
-    def __getitem__(self, key):
-        """
-        Returns the last data value for this key, or [] if it's an empty list;
-        raises KeyError if not found.
-        """
-        try:
-            list_ = dict.__getitem__(self, key)
-        except KeyError:
-            raise MultiValueDictKeyError("Key %r not found in %r" % (key, self))
-        try:
-            return list_[-1]
-        except IndexError:
-            return []
-
-    def __setitem__(self, key, value):
-        dict.__setitem__(self, key, [value])
-
-    def __copy__(self):
-        return self.__class__(dict.items(self))
-
-    def __deepcopy__(self, memo=None):
-        if memo is None:
-            memo = {}
-        result = self.__class__()
-        memo[id(self)] = result
-        for key, value in dict.items(self):
-            dict.__setitem__(result, copy.deepcopy(key, memo), copy.deepcopy(value, memo))
-        return result
-
-    def get(self, key, default=None):
-        "Returns the default value if the requested data doesn't exist"
-        try:
-            val = self[key]
-        except KeyError:
-            return default
-        if val == []:
-            return default
-        return val
-
-    def getlist(self, key):
-        "Returns an empty list if the requested data doesn't exist"
-        try:
-            return dict.__getitem__(self, key)
-        except KeyError:
-            return []
-
-    def setlist(self, key, list_):
-        dict.__setitem__(self, key, list_)
-
-    def setdefault(self, key, default=None):
-        if key not in self:
-            self[key] = default
-        return self[key]
-
-    def setlistdefault(self, key, default_list=()):
-        if key not in self:
-            self.setlist(key, default_list)
-        return self.getlist(key)
-
-    def appendlist(self, key, value):
-        "Appends an item to the internal list associated with key"
-        self.setlistdefault(key, [])
-        dict.__setitem__(self, key, self.getlist(key) + [value])
-
-    def items(self):
-        """
-        Returns a list of (key, value) pairs, where value is the last item in
-        the list associated with the key.
-        """
-        return [(key, self[key]) for key in self.keys()]
-
-    def lists(self):
-        "Returns a list of (key, list) pairs."
-        return dict.items(self)
-
-    def values(self):
-        "Returns a list of the last value on every key list."
-        return [self[key] for key in self.keys()]
-
-    def copy(self):
-        "Returns a copy of this object."
-        return self.__deepcopy__()
-
-    def update(self, *args, **kwargs):
-        "update() extends rather than replaces existing key lists. Also accepts keyword args."
-        if len(args) > 1:
-            raise TypeError("update expected at most 1 arguments, got %d" % len(args))
-        if args:
-            other_dict = args[0]
-            if isinstance(other_dict, MultiValueDict):
-                for key, value_list in other_dict.lists():
-                    self.setlistdefault(key, []).extend(value_list)
-            else:
-                try:
-                    for key, value in other_dict.items():
-                        self.setlistdefault(key, []).append(value)
-                except TypeError:
-                    raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary")
-        for key, value in kwargs.items():
-            self.setlistdefault(key, []).append(value)
-
-
-class SiteNode(object):
-    """Class to represent a site node (page, image or any other file)"""
-
-    def __init__(self, url):
-        warnings.warn(
-            "scrapy.utils.datatypes.SiteNode is deprecated "
-            "and will be removed in future releases.",
-            category=ScrapyDeprecationWarning,
-            stacklevel=2
-        )
-
-        self.url = url
-        self.itemnames = []
-        self.children = []
-        self.parent = None
-
-    def add_child(self, node):
-        self.children.append(node)
-        node.parent = self
-
-    def to_string(self, level=0):
-        s = "%s%s\n" % ('  ' * level, self.url)
-        if self.itemnames:
-            for n in self.itemnames:
-                s += "%sScraped: %s\n" % ('  ' * (level + 1), n)
-        for node in self.children:
-            s += node.to_string(level + 1)
-        return s
-
 
 class CaselessDict(dict):
 

From b1566a696217244d2d506ce8962418943d0b1edc Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Mon, 2 Mar 2020 23:01:26 +0000
Subject: [PATCH 2685/2974] Remove deprecated Crawler.spiders property

Deprecated since 419026615 (2014, Scrapy 0.25)
---
 scrapy/crawler.py     | 45 ++++++++++++++++---------------------------
 tests/test_crawler.py | 15 ---------------
 2 files changed, 17 insertions(+), 43 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 49b8e4511..77a13d0c1 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -68,17 +68,6 @@ class Crawler:
         self.spider = None
         self.engine = None
 
-    @property
-    def spiders(self):
-        if not hasattr(self, '_spiders'):
-            warnings.warn("Crawler.spiders is deprecated, use "
-                          "CrawlerRunner.spider_loader or instantiate "
-                          "scrapy.spiderloader.SpiderLoader with your "
-                          "settings.",
-                          category=ScrapyDeprecationWarning, stacklevel=2)
-            self._spiders = _get_spider_loader(self.settings.frozencopy())
-        return self._spiders
-
     @defer.inlineCallbacks
     def crawl(self, *args, **kwargs):
         assert not self.crawling, "Crawling already taking place"
@@ -130,11 +119,27 @@ class CrawlerRunner:
             ":meth:`crawl` and managed by this class."
     )
 
+    @staticmethod
+    def _get_spider_loader(settings):
+        """ Get SpiderLoader instance from settings """
+        cls_path = settings.get('SPIDER_LOADER_CLASS')
+        loader_cls = load_object(cls_path)
+        try:
+            verifyClass(ISpiderLoader, loader_cls)
+        except DoesNotImplement:
+            warnings.warn(
+                'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does '
+                'not fully implement scrapy.interfaces.ISpiderLoader interface. '
+                'Please add all missing methods to avoid unexpected runtime errors.',
+                category=ScrapyDeprecationWarning, stacklevel=2
+            )
+        return loader_cls.from_settings(settings.frozencopy())
+
     def __init__(self, settings=None):
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
         self.settings = settings
-        self.spider_loader = _get_spider_loader(settings)
+        self.spider_loader = self._get_spider_loader(settings)
         self._crawlers = set()
         self._active = set()
         self.bootstrap_failed = False
@@ -327,19 +332,3 @@ class CrawlerProcess(CrawlerRunner):
         if self.settings.get("TWISTED_REACTOR"):
             install_reactor(self.settings["TWISTED_REACTOR"])
         super()._handle_twisted_reactor()
-
-
-def _get_spider_loader(settings):
-    """ Get SpiderLoader instance from settings """
-    cls_path = settings.get('SPIDER_LOADER_CLASS')
-    loader_cls = load_object(cls_path)
-    try:
-        verifyClass(ISpiderLoader, loader_cls)
-    except DoesNotImplement:
-        warnings.warn(
-            'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does '
-            'not fully implement scrapy.interfaces.ISpiderLoader interface. '
-            'Please add all missing methods to avoid unexpected runtime errors.',
-            category=ScrapyDeprecationWarning, stacklevel=2
-        )
-    return loader_cls.from_settings(settings.frozencopy())
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 7bd76601d..37a069611 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -33,21 +33,6 @@ class CrawlerTestCase(BaseCrawlerTest):
     def setUp(self):
         self.crawler = Crawler(DefaultSpider, Settings())
 
-    def test_deprecated_attribute_spiders(self):
-        with warnings.catch_warnings(record=True) as w:
-            spiders = self.crawler.spiders
-            self.assertEqual(len(w), 1)
-            self.assertIn("Crawler.spiders", str(w[0].message))
-            sl_cls = load_object(self.crawler.settings['SPIDER_LOADER_CLASS'])
-            self.assertIsInstance(spiders, sl_cls)
-
-            self.crawler.spiders
-            is_one_warning = len(w) == 1
-            if not is_one_warning:
-                for warning in w:
-                    print(warning)
-            self.assertTrue(is_one_warning, "Warn deprecated access only once")
-
     def test_populate_spidercls_settings(self):
         spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
         project_settings = {'TEST1': 'project', 'TEST3': 'project'}

From ada37c5409047291ee5852fb2220e8e256424402 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 6 Jul 2019 22:37:31 -0300
Subject: [PATCH 2686/2974] Export to multiple formats in a single crawl

---
 docs/topics/feed-exports.rst        |  85 ++++++---
 scrapy/commands/crawl.py            |  23 +--
 scrapy/commands/runspider.py        |  20 +-
 scrapy/extensions/feedexport.py     | 191 +++++++++++--------
 scrapy/settings/default_settings.py |   3 +-
 scrapy/utils/conf.py                |  68 ++++++-
 tests/test_commands.py              |  32 +++-
 tests/test_feedexport.py            | 273 ++++++++++++++++++++--------
 tests/test_utils_conf.py            |  50 ++++-
 9 files changed, 511 insertions(+), 234 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 42f1cad90..6d6ba33c9 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -12,7 +12,7 @@ generating an "export file" with the scraped data (commonly called "export
 feed") to be consumed by other systems.
 
 Scrapy provides this functionality out of the box with the Feed Exports, which
-allows you to generate a feed with the scraped items, using multiple
+allows you to generate feeds with the scraped items, using multiple
 serialization formats and storage backends.
 
 .. _topics-feed-format:
@@ -36,7 +36,7 @@ But you can also extend the supported format through the
 JSON
 ----
 
- * :setting:`FEED_FORMAT`: ``json``
+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
  * Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
  * See :ref:`this warning <json-with-large-data>` if you're using JSON with
    large feeds.
@@ -46,7 +46,7 @@ JSON
 JSON lines
 ----------
 
- * :setting:`FEED_FORMAT`: ``jsonlines``
+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
  * Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
 
 .. _topics-feed-format-csv:
@@ -54,7 +54,7 @@ JSON lines
 CSV
 ---
 
- * :setting:`FEED_FORMAT`: ``csv``
+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
  * Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
  * To specify columns to export and their order use
    :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
@@ -66,7 +66,7 @@ CSV
 XML
 ---
 
- * :setting:`FEED_FORMAT`: ``xml``
+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
  * Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
 
 .. _topics-feed-format-pickle:
@@ -74,7 +74,7 @@ XML
 Pickle
 ------
 
- * :setting:`FEED_FORMAT`: ``pickle``
+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
  * Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
 
 .. _topics-feed-format-marshal:
@@ -82,7 +82,7 @@ Pickle
 Marshal
 -------
 
- * :setting:`FEED_FORMAT`: ``marshal``
+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
  * Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
 
 
@@ -91,8 +91,8 @@ Marshal
 Storages
 ========
 
-When using the feed exports you define where to store the feed using a URI_
-(through the :setting:`FEED_URI` setting). The feed exports supports multiple
+When using the feed exports you define where to store the feed using one or multiple URIs_
+(through the :setting:`FEEDS` setting). The feed exports supports multiple
 storage backend types which are defined by the URI scheme.
 
 The storages backends supported out of the box are:
@@ -211,41 +211,66 @@ Settings
 
 These are the settings used for configuring the feed exports:
 
- * :setting:`FEED_URI` (mandatory)
- * :setting:`FEED_FORMAT`
+ * :setting:`FEEDS` (mandatory)
+ * :setting:`FEED_EXPORT_ENCODING`
+ * :setting:`FEED_STORE_EMPTY`
+ * :setting:`FEED_EXPORT_FIELDS`
+ * :setting:`FEED_EXPORT_INDENT`
  * :setting:`FEED_STORAGES`
  * :setting:`FEED_STORAGE_FTP_ACTIVE`
  * :setting:`FEED_STORAGE_S3_ACL`
  * :setting:`FEED_EXPORTERS`
- * :setting:`FEED_STORE_EMPTY`
- * :setting:`FEED_EXPORT_ENCODING`
- * :setting:`FEED_EXPORT_FIELDS`
- * :setting:`FEED_EXPORT_INDENT`
 
 .. currentmodule:: scrapy.extensions.feedexport
 
-.. setting:: FEED_URI
+.. setting:: FEEDS
 
-FEED_URI
---------
+FEEDS
+-----
 
-Default: ``None``
+.. versionadded:: 2.1
 
-The URI of the export feed. See :ref:`topics-feed-storage-backends` for
-supported URI schemes.
+Default: ``{}``
 
-This setting is required for enabling the feed exports.
+A dictionary in which every key is a feed URI (or a :class:`pathlib.Path`
+object) and each value is a nested dictionary containing configuration
+parameters for the specific feed.
+This setting is required for enabling the feed export feature.
 
-.. versionchanged:: 2.0
-   Added :class:`pathlib.Path` support.
+See :ref:`topics-feed-storage-backends` for supported URI schemes.
 
-.. setting:: FEED_FORMAT
+For instance::
 
-FEED_FORMAT
------------
+    {
+        'items.json': {
+            'format': 'json',
+            'encoding': 'utf8',
+            'store_empty': False,
+            'fields': None,
+            'indent': 4,
+        }, 
+        'items.xml': {
+            'format': 'xml',
+            'fields': ['name', 'price'],
+            'encoding': 'latin1',
+            'indent': 8,
+        },
+        pathlib.Path('items.csv'): {
+            'format': 'csv',
+            'fields': ['price', 'name'],
+        },
+    }
 
-The serialization format to be used for the feed. See
-:ref:`topics-feed-format` for possible values.
+The following is a list of the accepted keys and the setting that is used
+as a fallback value if that key is not provided for a specific feed definition.
+
+* ``format``: the serialization format to be used for the feed.
+  See :ref:`topics-feed-format` for possible values. 
+  Mandatory, no fallback setting
+* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`
+* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
+* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
+* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
 
 .. setting:: FEED_EXPORT_ENCODING
 
@@ -400,7 +425,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
         'csv': None,
     }
 
-.. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
+.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
 .. _Amazon S3: https://aws.amazon.com/s3/
 .. _botocore: https://github.com/boto/botocore
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 7b417e2eb..4b2f9484b 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,7 +1,5 @@
-import os
 from scrapy.commands import ScrapyCommand
-from scrapy.utils.conf import arglist_to_dict
-from scrapy.utils.python import without_none_values
+from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 from scrapy.exceptions import UsageError
 
 
@@ -19,7 +17,7 @@ class Command(ScrapyCommand):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
                           help="set spider argument (may be repeated)")
-        parser.add_option("-o", "--output", metavar="FILE",
+        parser.add_option("-o", "--output", metavar="FILE", action="append",
                           help="dump scraped items into FILE (use - for stdout)")
         parser.add_option("-t", "--output-format", metavar="FORMAT",
                           help="format to use for dumping items with -o")
@@ -31,21 +29,8 @@ class Command(ScrapyCommand):
         except ValueError:
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
         if opts.output:
-            if opts.output == '-':
-                self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
-            else:
-                self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            feed_exporters = without_none_values(
-                self.settings.getwithbase('FEED_EXPORTERS'))
-            valid_output_formats = feed_exporters.keys()
-            if not opts.output_format:
-                opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
-            if opts.output_format not in valid_output_formats:
-                raise UsageError("Unrecognized output format '%s', set one"
-                                 " using the '-t' switch or as a file extension"
-                                 " from the supported list %s" % (opts.output_format,
-                                                                  tuple(valid_output_formats)))
-            self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
+            feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
+            self.settings.set('FEEDS', feeds, priority='cmdline')
 
     def run(self, args, opts):
         if len(args) < 1:
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 57d8471ca..62510609a 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -5,8 +5,7 @@ from importlib import import_module
 from scrapy.utils.spider import iter_spider_classes
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
-from scrapy.utils.conf import arglist_to_dict
-from scrapy.utils.python import without_none_values
+from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 
 
 def _import_file(filepath):
@@ -43,7 +42,7 @@ class Command(ScrapyCommand):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
                           help="set spider argument (may be repeated)")
-        parser.add_option("-o", "--output", metavar="FILE",
+        parser.add_option("-o", "--output", metavar="FILE", action="append",
                           help="dump scraped items into FILE (use - for stdout)")
         parser.add_option("-t", "--output-format", metavar="FORMAT",
                           help="format to use for dumping items with -o")
@@ -55,19 +54,8 @@ class Command(ScrapyCommand):
         except ValueError:
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
         if opts.output:
-            if opts.output == '-':
-                self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
-            else:
-                self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))
-            if not opts.output_format:
-                opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
-            if opts.output_format not in feed_exporters:
-                raise UsageError("Unrecognized output format '%s', set one"
-                                 " using the '-t' switch or as a file extension"
-                                 " from the supported list %s" % (opts.output_format,
-                                                                  tuple(feed_exporters)))
-            self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
+            feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
+            self.settings.set('FEEDS', feeds, priority='cmdline')
 
     def run(self, args, opts):
         if len(args) != 1:
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index f1b101780..108b6d35c 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -4,24 +4,27 @@ Feed Exports extension
 See documentation in docs/topics/feed-exports.rst
 """
 
+import logging
 import os
 import sys
-import logging
-from tempfile import NamedTemporaryFile
+import warnings
 from datetime import datetime
-from urllib.parse import urlparse, unquote
+from tempfile import NamedTemporaryFile
+from urllib.parse import unquote, urlparse
 
-from zope.interface import Interface, implementer
 from twisted.internet import defer, threads
 from w3lib.url import file_uri_to_path
+from zope.interface import implementer, Interface
 
 from scrapy import signals
-from scrapy.utils.ftp import ftp_store_file
-from scrapy.exceptions import NotConfigured
-from scrapy.utils.misc import create_instance, load_object
-from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.python import without_none_values
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.utils.boto import is_botocore
+from scrapy.utils.conf import feed_complete_default_values_from_settings
+from scrapy.utils.ftp import ftp_store_file
+from scrapy.utils.log import failure_to_exc_info
+from scrapy.utils.misc import create_instance, load_object
+from scrapy.utils.python import without_none_values
+
 
 logger = logging.getLogger(__name__)
 
@@ -98,8 +101,6 @@ class S3FeedStorage(BlockingFeedStorage):
             from scrapy.utils.project import get_project_settings
             settings = get_project_settings()
             if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:
-                import warnings
-                from scrapy.exceptions import ScrapyDeprecationWarning
                 warnings.warn(
                     "Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
                     "without AWS keys is deprecated. Please supply credentials or "
@@ -178,88 +179,117 @@ class FTPFeedStorage(BlockingFeedStorage):
         )
 
 
-class SpiderSlot(object):
-    def __init__(self, file, exporter, storage, uri):
+class _FeedSlot(object):
+    def __init__(self, file, exporter, storage, uri, format, store_empty):
         self.file = file
         self.exporter = exporter
         self.storage = storage
+        # feed params
         self.uri = uri
+        self.format = format
+        self.store_empty = store_empty
+        # flags
         self.itemcount = 0
+        self._exporting = False
+
+    def start_exporting(self):
+        if not self._exporting:
+            self.exporter.start_exporting()
+            self._exporting = True
+
+    def finish_exporting(self):
+        if self._exporting:
+            self.exporter.finish_exporting()
+            self._exporting = False
 
 
 class FeedExporter(object):
 
-    def __init__(self, settings):
-        self.settings = settings
-        if not settings['FEED_URI']:
-            raise NotConfigured
-        self.urifmt = str(settings['FEED_URI'])
-        self.format = settings['FEED_FORMAT'].lower()
-        self.export_encoding = settings['FEED_EXPORT_ENCODING']
-        self.storages = self._load_components('FEED_STORAGES')
-        self.exporters = self._load_components('FEED_EXPORTERS')
-        if not self._storage_supported(self.urifmt):
-            raise NotConfigured
-        if not self._exporter_supported(self.format):
-            raise NotConfigured
-        self.store_empty = settings.getbool('FEED_STORE_EMPTY')
-        self._exporting = False
-        self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
-        self.indent = None
-        if settings.get('FEED_EXPORT_INDENT') is not None:
-            self.indent = settings.getint('FEED_EXPORT_INDENT')
-        uripar = settings['FEED_URI_PARAMS']
-        self._uripar = load_object(uripar) if uripar else lambda x, y: None
-
     @classmethod
     def from_crawler(cls, crawler):
-        o = cls(crawler.settings)
-        o.crawler = crawler
-        crawler.signals.connect(o.open_spider, signals.spider_opened)
-        crawler.signals.connect(o.close_spider, signals.spider_closed)
-        crawler.signals.connect(o.item_scraped, signals.item_scraped)
-        return o
+        exporter = cls(crawler)
+        crawler.signals.connect(exporter.open_spider, signals.spider_opened)
+        crawler.signals.connect(exporter.close_spider, signals.spider_closed)
+        crawler.signals.connect(exporter.item_scraped, signals.item_scraped)
+        return exporter
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.feeds = {}
+        self.slots = []
+
+        if not self.settings['FEEDS'] and not self.settings['FEED_URI']:
+            raise NotConfigured
+
+        # Begin: Backward compatibility for FEED_URI and FEED_FORMAT settings
+        if self.settings['FEED_URI']:
+            warnings.warn(
+                'The `FEED_URI` and `FEED_FORMAT` settings have been deprecated in favor of '
+                'the `FEEDS` setting. Please see the `FEEDS` setting docs for more details',
+                category=ScrapyDeprecationWarning, stacklevel=2,
+            )
+            uri = str(self.settings['FEED_URI'])  # handle pathlib.Path objects
+            feed = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
+            self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
+        # End: Backward compatibility for FEED_URI and FEED_FORMAT settings
+
+        # 'FEEDS' setting takes precedence over 'FEED_URI'
+        for uri, feed in self.settings.getdict('FEEDS').items():
+            uri = str(uri)  # handle pathlib.Path objects
+            self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
+
+        self.storages = self._load_components('FEED_STORAGES')
+        self.exporters = self._load_components('FEED_EXPORTERS')
+        for uri, feed in self.feeds.items():
+            if not self._storage_supported(uri):
+                raise NotConfigured
+            if not self._exporter_supported(feed['format']):
+                raise NotConfigured
 
     def open_spider(self, spider):
-        uri = self.urifmt % self._get_uri_params(spider)
-        storage = self._get_storage(uri)
-        file = storage.open(spider)
-        exporter = self._get_exporter(file, fields_to_export=self.export_fields,
-            encoding=self.export_encoding, indent=self.indent)
-        if self.store_empty:
-            exporter.start_exporting()
-            self._exporting = True
-        self.slot = SpiderSlot(file, exporter, storage, uri)
+        for uri, feed in self.feeds.items():
+            uri = uri % self._get_uri_params(spider, feed['uri_params'])
+            storage = self._get_storage(uri)
+            file = storage.open(spider)
+            exporter = self._get_exporter(
+                file=file,
+                format=feed['format'],
+                fields_to_export=feed['fields'],
+                encoding=feed['encoding'],
+                indent=feed['indent'],
+            )
+            slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
+            self.slots.append(slot)
+            if slot.store_empty:
+                slot.start_exporting()
 
     def close_spider(self, spider):
-        slot = self.slot
-        if not slot.itemcount and not self.store_empty:
-            # We need to call slot.storage.store nonetheless to get the file
-            # properly closed.
-            return defer.maybeDeferred(slot.storage.store, slot.file)
-        if self._exporting:
-            slot.exporter.finish_exporting()
-            self._exporting = False
-        logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
-        log_args = {'format': self.format,
-                    'itemcount': slot.itemcount,
-                    'uri': slot.uri}
-        d = defer.maybeDeferred(slot.storage.store, slot.file)
-        d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
-                                            extra={'spider': spider}))
-        d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
-                                            exc_info=failure_to_exc_info(f),
-                                            extra={'spider': spider}))
-        return d
+        deferred_list = []
+        for slot in self.slots:
+            if not slot.itemcount and not slot.store_empty:
+                # We need to call slot.storage.store nonetheless to get the file
+                # properly closed.
+                return defer.maybeDeferred(slot.storage.store, slot.file)
+            slot.finish_exporting()
+            logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
+            log_args = {'format': slot.format,
+                        'itemcount': slot.itemcount,
+                        'uri': slot.uri}
+            d = defer.maybeDeferred(slot.storage.store, slot.file)
+            d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
+                                                extra={'spider': spider}))
+            d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
+                                                exc_info=failure_to_exc_info(f),
+                                                extra={'spider': spider}))
+            deferred_list.append(d)
+        return defer.DeferredList(deferred_list) if deferred_list else None
 
     def item_scraped(self, item, spider):
-        slot = self.slot
-        if not self._exporting:
-            slot.exporter.start_exporting()
-            self._exporting = True
-        slot.exporter.export_item(item)
-        slot.itemcount += 1
-        return item
+        for slot in self.slots:
+            slot.start_exporting()
+            slot.exporter.export_item(item)
+            slot.itemcount += 1
 
     def _load_components(self, setting_prefix):
         conf = without_none_values(self.settings.getwithbase(setting_prefix))
@@ -295,17 +325,18 @@ class FeedExporter(object):
             objcls, self.settings, getattr(self, 'crawler', None),
             *args, **kwargs)
 
-    def _get_exporter(self, *args, **kwargs):
-        return self._get_instance(self.exporters[self.format], *args, **kwargs)
+    def _get_exporter(self, file, format, *args, **kwargs):
+        return self._get_instance(self.exporters[format], file, *args, **kwargs)
 
     def _get_storage(self, uri):
         return self._get_instance(self.storages[urlparse(uri).scheme], uri)
 
-    def _get_uri_params(self, spider):
+    def _get_uri_params(self, spider, uri_params):
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
         ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
         params['time'] = ts
-        self._uripar(params, spider)
+        uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
+        uripar_function(params, spider)
         return params
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index f8a0457ce..077317c81 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -133,9 +133,8 @@ EXTENSIONS_BASE = {
 }
 
 FEED_TEMPDIR = None
-FEED_URI = None
+FEEDS = {}
 FEED_URI_PARAMS = None  # a function to extend uri arguments
-FEED_FORMAT = 'jsonlines'
 FEED_STORE_EMPTY = False
 FEED_EXPORT_ENCODING = None
 FEED_EXPORT_FIELDS = None
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 23306ca28..e01027491 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -1,9 +1,12 @@
+import numbers
 import os
 import sys
-import numbers
+import warnings
 from configparser import ConfigParser
 from operator import itemgetter
 
+from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
+
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
 from scrapy.utils.python import without_none_values
@@ -106,3 +109,66 @@ def get_sources(use_closest=True):
     if use_closest:
         sources.append(closest_scrapy_cfg())
     return sources
+
+
+def feed_complete_default_values_from_settings(feed, settings):
+    out = feed.copy()
+    if 'encoding' not in out:
+        out['encoding'] = settings['FEED_EXPORT_ENCODING']
+    if 'fields' not in out:
+        out['fields'] = settings.getlist('FEED_EXPORT_FIELDS') or None
+    if 'indent' not in out:
+        out['indent'] = None if settings['FEED_EXPORT_INDENT'] is None else settings.getint('FEED_EXPORT_INDENT')
+    if 'store_empty' not in out:
+        out['store_empty'] = settings.getbool('FEED_STORE_EMPTY')
+    if 'uri_params' not in out:
+        out['uri_params'] = settings['FEED_URI_PARAMS']
+    return out
+
+
+def feed_process_params_from_cli(settings, output, output_format=None):
+    """
+    Receives feed export params (from the 'crawl' or 'runspider' commands),
+    checks for inconsistencies in their quantities and returns a dictionary
+    suitable to be used as the FEEDS setting.
+    """
+    valid_output_formats = without_none_values(
+        settings.getwithbase('FEED_EXPORTERS')
+    ).keys()
+
+    def check_valid_format(output_format):
+        if output_format not in valid_output_formats:
+            raise UsageError("Unrecognized output format '%s', set one after a"
+                             " colon using the -o option (i.e. -o <URI>:<FORMAT>)"
+                             " or as a file extension, from the supported list %s" %
+                             (output_format, tuple(valid_output_formats)))
+
+    if output_format:
+        if len(output) == 1:
+            check_valid_format(output_format)
+            warnings.warn('The -t command line option is deprecated in favor'
+                          ' of specifying the output format within the -o'
+                          ' option, please check the -o option docs for more details',
+                          category=ScrapyDeprecationWarning, stacklevel=2)
+            return {output[0]: {'format': output_format}}
+        else:
+            raise UsageError('The -t command line option cannot be used if multiple'
+                             ' output files are specified with the -o option')
+
+    result = {}
+    for element in output:
+        try:
+            feed_uri, feed_format = element.rsplit(':', 1)
+        except ValueError:
+            feed_uri = element
+            feed_format = os.path.splitext(element)[1].replace('.', '')
+        else:
+            if feed_uri == '-':
+                feed_uri = 'stdout:'
+        check_valid_format(feed_format)
+        result[feed_uri] = {'format': feed_format}
+
+    # FEEDS setting should take precedence over the -o and -t CLI options
+    result.update(settings.getdict('FEEDS'))
+
+    return result
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 3612b70c9..24a341759 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -1,22 +1,46 @@
 import inspect
+import json
+import optparse
 import os
-import sys
 import subprocess
+import sys
 import tempfile
+from contextlib import contextmanager
 from os.path import exists, join, abspath
 from shutil import rmtree, copytree
 from tempfile import mkdtemp
-from contextlib import contextmanager
 from threading import Timer
 
 from twisted.trial import unittest
 
 import scrapy
+from scrapy.commands import ScrapyCommand
+from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_testenv
+
 from tests.test_crawler import ExceptionSpider, NoRequestsSpider
 
 
+class CommandSettings(unittest.TestCase):
+
+    def setUp(self):
+        self.command = ScrapyCommand()
+        self.command.settings = Settings()
+        self.parser = optparse.OptionParser(
+            formatter=optparse.TitledHelpFormatter(),
+            conflict_handler='resolve',
+        )
+        self.command.add_options(self.parser)
+
+    def test_settings_json_string(self):
+        feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}'
+        opts, args = self.parser.parse_args(args=['-s', 'FEEDS={}'.format(feeds_json), 'spider.py'])
+        self.command.process_options(args, opts)
+        self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings)
+        self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json))
+
+
 class ProjectTest(unittest.TestCase):
     project_name = 'testproject'
 
@@ -34,7 +58,7 @@ class ProjectTest(unittest.TestCase):
         with tempfile.TemporaryFile() as out:
             args = (sys.executable, '-m', 'scrapy.cmdline') + new_args
             return subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd,
-                env=self.env, **kwargs)
+                                   env=self.env, **kwargs)
 
     def proc(self, *new_args, **popen_kwargs):
         args = (sys.executable, '-m', 'scrapy.cmdline') + new_args
@@ -310,6 +334,6 @@ class BenchCommandTest(CommandTest):
 
     def test_run(self):
         _, _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
-                           '-s', 'CLOSESPIDER_TIMEOUT=0.01')
+                              '-s', 'CLOSESPIDER_TIMEOUT=0.01')
         self.assertIn('INFO: Crawled', log)
         self.assertNotIn('Unhandled Error', log)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 2ca57c19d..08e8dfc41 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1,32 +1,34 @@
-import os
 import csv
 import json
-import warnings
-import tempfile
+import os
+import random
 import shutil
 import string
+import tempfile
+import warnings
 from io import BytesIO
 from pathlib import Path
+from string import ascii_letters, digits
 from unittest import mock
 from urllib.parse import urljoin, urlparse, quote
 from urllib.request import pathname2url
 
-from zope.interface.verify import verifyObject
-from twisted.trial import unittest
+import lxml.etree
 from twisted.internet import defer
-from scrapy.crawler import CrawlerRunner
-from scrapy.settings import Settings
-from tests.mockserver import MockServer
+from twisted.trial import unittest
 from w3lib.url import path_to_file_uri
+from zope.interface.verify import verifyObject
 
 import scrapy
+from scrapy.crawler import CrawlerRunner
 from scrapy.exporters import CsvItemExporter
-from scrapy.extensions.feedexport import (
-    IFeedStorage, FileFeedStorage, FTPFeedStorage,
-    S3FeedStorage, StdoutFeedStorage,
-    BlockingFeedStorage)
-from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
+from scrapy.extensions.feedexport import (BlockingFeedStorage, FileFeedStorage, FTPFeedStorage,
+                                          IFeedStorage, S3FeedStorage, StdoutFeedStorage)
+from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
+from scrapy.utils.test import assert_aws_environ, get_crawler, get_s3_content_and_delete
+
+from tests.mockserver import MockServer
 
 
 class FileFeedStorageTest(unittest.TestCase):
@@ -395,29 +397,41 @@ class FeedExportTest(unittest.TestCase):
         egg = scrapy.Field()
         baz = scrapy.Field()
 
+    def setUp(self):
+        self.temp_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.temp_dir, ignore_errors=True)
+
+    def _random_temp_filename(self):
+        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
+        filename = ''.join(chars)
+        return os.path.join(self.temp_dir, filename)
+
     @defer.inlineCallbacks
-    def run_and_export(self, spider_cls, settings=None):
+    def run_and_export(self, spider_cls, settings):
         """ Run spider with specified settings; return exported data. """
-        tmpdir = tempfile.mkdtemp()
-        res_path = os.path.join(tmpdir, 'res')
-        res_uri = urljoin('file:', pathname2url(res_path))
-        defaults = {
-            'FEED_URI': res_uri,
-            'FEED_FORMAT': 'csv',
-            'FEED_PATH': res_path
+
+        FEEDS = settings.get('FEEDS') or {}
+        settings['FEEDS'] = {
+            urljoin('file:', pathname2url(str(file_path))): feed
+            for file_path, feed in FEEDS.items()
         }
-        defaults.update(settings or {})
+
+        content = {}
         try:
             with MockServer() as s:
-                runner = CrawlerRunner(Settings(defaults))
+                runner = CrawlerRunner(Settings(settings))
                 spider_cls.start_urls = [s.url('/')]
                 yield runner.crawl(spider_cls)
 
-            with open(str(defaults['FEED_PATH']), 'rb') as f:
-                content = f.read()
+            for file_path, feed in FEEDS.items():
+                with open(str(file_path), 'rb') as f:
+                    content[feed['format']] = f.read()
 
         finally:
-            shutil.rmtree(tmpdir)
+            for file_path in FEEDS.keys():
+                os.remove(str(file_path))
 
         defer.returnValue(content)
 
@@ -453,10 +467,14 @@ class FeedExportTest(unittest.TestCase):
     @defer.inlineCallbacks
     def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
         settings = settings or {}
-        settings.update({'FEED_FORMAT': 'csv'})
+        settings.update({
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'csv'},
+            },
+        })
         data = yield self.exported_data(items, settings)
 
-        reader = csv.DictReader(to_unicode(data).splitlines())
+        reader = csv.DictReader(to_unicode(data['csv']).splitlines())
         got_rows = list(reader)
         if ordered:
             self.assertEqual(reader.fieldnames, header)
@@ -468,51 +486,87 @@ class FeedExportTest(unittest.TestCase):
     @defer.inlineCallbacks
     def assertExportedJsonLines(self, items, rows, settings=None):
         settings = settings or {}
-        settings.update({'FEED_FORMAT': 'jl'})
+        settings.update({
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'jl'},
+            },
+        })
         data = yield self.exported_data(items, settings)
-        parsed = [json.loads(to_unicode(line)) for line in data.splitlines()]
+        parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()]
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         self.assertEqual(rows, parsed)
 
     @defer.inlineCallbacks
     def assertExportedXml(self, items, rows, settings=None):
         settings = settings or {}
-        settings.update({'FEED_FORMAT': 'xml'})
+        settings.update({
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'xml'},
+            },
+        })
         data = yield self.exported_data(items, settings)
         rows = [{k: v for k, v in row.items() if v} for row in rows]
-        import lxml.etree
-        root = lxml.etree.fromstring(data)
+        root = lxml.etree.fromstring(data['xml'])
         got_rows = [{e.tag: e.text for e in it} for it in root.findall('item')]
         self.assertEqual(rows, got_rows)
 
+    @defer.inlineCallbacks
+    def assertExportedMultiple(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'xml'},
+                self._random_temp_filename(): {'format': 'json'},
+            },
+        })
+        data = yield self.exported_data(items, settings)
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        # XML
+        root = lxml.etree.fromstring(data['xml'])
+        xml_rows = [{e.tag: e.text for e in it} for it in root.findall('item')]
+        self.assertEqual(rows, xml_rows)
+        # JSON
+        json_rows = json.loads(to_unicode(data['json']))
+        self.assertEqual(rows, json_rows)
+
     def _load_until_eof(self, data, load_func):
-        bytes_output = BytesIO(data)
         result = []
-        while True:
-            try:
-                result.append(load_func(bytes_output))
-            except EOFError:
-                break
+        with tempfile.TemporaryFile() as temp:
+            temp.write(data)
+            temp.seek(0)
+            while True:
+                try:
+                    result.append(load_func(temp))
+                except EOFError:
+                    break
         return result
 
     @defer.inlineCallbacks
     def assertExportedPickle(self, items, rows, settings=None):
         settings = settings or {}
-        settings.update({'FEED_FORMAT': 'pickle'})
+        settings.update({
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'pickle'},
+            },
+        })
         data = yield self.exported_data(items, settings)
         expected = [{k: v for k, v in row.items() if v} for row in rows]
         import pickle
-        result = self._load_until_eof(data, load_func=pickle.load)
+        result = self._load_until_eof(data['pickle'], load_func=pickle.load)
         self.assertEqual(expected, result)
 
     @defer.inlineCallbacks
     def assertExportedMarshal(self, items, rows, settings=None):
         settings = settings or {}
-        settings.update({'FEED_FORMAT': 'marshal'})
+        settings.update({
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'marshal'},
+            },
+        })
         data = yield self.exported_data(items, settings)
         expected = [{k: v for k, v in row.items() if v} for row in rows]
         import marshal
-        result = self._load_until_eof(data, load_func=marshal.load)
+        result = self._load_until_eof(data['marshal'], load_func=marshal.load)
         self.assertEqual(expected, result)
 
     @defer.inlineCallbacks
@@ -521,6 +575,8 @@ class FeedExportTest(unittest.TestCase):
         yield self.assertExportedJsonLines(items, rows, settings)
         yield self.assertExportedXml(items, rows, settings)
         yield self.assertExportedPickle(items, rows, settings)
+        yield self.assertExportedMarshal(items, rows, settings)
+        yield self.assertExportedMultiple(items, rows, settings)
 
     @defer.inlineCallbacks
     def test_export_items(self):
@@ -538,15 +594,14 @@ class FeedExportTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_export_no_items_not_store_empty(self):
-        formats = ('json',
-                   'jsonlines',
-                   'xml',
-                   'csv',)
-
-        for fmt in formats:
-            settings = {'FEED_FORMAT': fmt}
+        for fmt in ('json', 'jsonlines', 'xml', 'csv'):
+            settings = {
+                'FEEDS': {
+                    self._random_temp_filename(): {'format': fmt},
+                },
+            }
             data = yield self.exported_no_data(settings)
-            self.assertEqual(data, b'')
+            self.assertEqual(data[fmt], b'')
 
     @defer.inlineCallbacks
     def test_export_no_items_store_empty(self):
@@ -558,9 +613,15 @@ class FeedExportTest(unittest.TestCase):
         )
 
         for fmt, expctd in formats:
-            settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None}
+            settings = {
+                'FEEDS': {
+                    self._random_temp_filename(): {'format': fmt},
+                },
+                'FEED_STORE_EMPTY': True,
+                'FEED_EXPORT_INDENT': None,
+            }
             data = yield self.exported_no_data(settings)
-            self.assertEqual(data, expctd)
+            self.assertEqual(data[fmt], expctd)
 
     @defer.inlineCallbacks
     def test_export_multiple_item_classes(self):
@@ -581,9 +642,9 @@ class FeedExportTest(unittest.TestCase):
         header = self.MyItem.fields.keys()
         rows_csv = [
             {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
-            {'egg': '',      'foo': 'bar2', 'baz': ''},
+            {'egg': '', 'foo': 'bar2', 'baz': ''},
             {'egg': 'spam3', 'foo': 'bar3', 'baz': 'quux3'},
-            {'egg': 'spam4', 'foo': '',     'baz': ''},
+            {'egg': 'spam4', 'foo': '', 'baz': ''},
         ]
         rows_jl = [dict(row) for row in items]
         yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
@@ -598,10 +659,10 @@ class FeedExportTest(unittest.TestCase):
         header = ["foo", "baz", "hello"]
         settings = {'FEED_EXPORT_FIELDS': header}
         rows = [
-            {'foo': 'bar1', 'baz': '',      'hello': ''},
-            {'foo': 'bar2', 'baz': '',      'hello': 'world2'},
+            {'foo': 'bar1', 'baz': '', 'hello': ''},
+            {'foo': 'bar2', 'baz': '', 'hello': 'world2'},
             {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
-            {'foo': '',     'baz': '',      'hello': 'world4'},
+            {'foo': '', 'baz': '', 'hello': 'world4'},
         ]
         yield self.assertExported(items, header, rows,
                                   settings=settings, ordered=True)
@@ -663,10 +724,15 @@ class FeedExportTest(unittest.TestCase):
             'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
         }
 
-        for format, expected in formats.items():
-            settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT': None}
+        for fmt, expected in formats.items():
+            settings = {
+                'FEEDS': {
+                    self._random_temp_filename(): {'format': fmt},
+                },
+                'FEED_EXPORT_INDENT': None,
+            }
             data = yield self.exported_data(items, settings)
-            self.assertEqual(expected, data)
+            self.assertEqual(expected, data[fmt])
 
         formats = {
             'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'),
@@ -675,11 +741,53 @@ class FeedExportTest(unittest.TestCase):
             'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
         }
 
-        settings = {'FEED_EXPORT_INDENT': None, 'FEED_EXPORT_ENCODING': 'latin-1'}
-        for format, expected in formats.items():
-            settings['FEED_FORMAT'] = format
+        for fmt, expected in formats.items():
+            settings = {
+                'FEEDS': {
+                    self._random_temp_filename(): {'format': fmt},
+                },
+                'FEED_EXPORT_INDENT': None,
+                'FEED_EXPORT_ENCODING': 'latin-1',
+            }
             data = yield self.exported_data(items, settings)
-            self.assertEqual(expected, data)
+            self.assertEqual(expected, data[fmt])
+
+    @defer.inlineCallbacks
+    def test_export_multiple_configs(self):
+        items = [dict({'foo': u'FOO', 'bar': u'BAR'})]
+
+        formats = {
+            'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
+            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode('latin-1'),
+            'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {
+                    'format': 'json',
+                    'indent': 0,
+                    'fields': ['bar'],
+                    'encoding': 'utf-8',
+                },
+                self._random_temp_filename(): {
+                    'format': 'xml',
+                    'indent': 2,
+                    'fields': ['foo'],
+                    'encoding': 'latin-1',
+                },
+                self._random_temp_filename(): {
+                    'format': 'csv',
+                    'indent': None,
+                    'fields': ['bar', 'foo'],
+                    'encoding': 'utf-8',
+                },
+            },
+        }
+
+        data = yield self.exported_data(items, settings)
+        for fmt, expected in formats.items():
+            self.assertEqual(expected, data[fmt])
 
     @defer.inlineCallbacks
     def test_export_indentation(self):
@@ -827,33 +935,38 @@ class FeedExportTest(unittest.TestCase):
         ]
 
         for row in test_cases:
-            settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']}
+            settings = {
+                'FEEDS': {
+                    self._random_temp_filename(): {
+                        'format': row['format'],
+                        'indent': row['indent'],
+                    },
+                },
+            }
             data = yield self.exported_data(items, settings)
-            print(row['format'], row['indent'])
-            self.assertEqual(row['expected'], data)
+            self.assertEqual(row['expected'], data[row['format']])
 
     @defer.inlineCallbacks
     def test_init_exporters_storages_with_crawler(self):
         settings = {
-            'FEED_EXPORTERS': {'csv': 'tests.test_feedexport.'
-                                      'FromCrawlerCsvItemExporter'},
-            'FEED_STORAGES': {'file': 'tests.test_feedexport.'
-                                      'FromCrawlerFileFeedStorage'},
+            'FEED_EXPORTERS': {'csv': 'tests.test_feedexport.FromCrawlerCsvItemExporter'},
+            'FEED_STORAGES': {'file': 'tests.test_feedexport.FromCrawlerFileFeedStorage'},
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'csv'},
+            },
         }
-        yield self.exported_data({}, settings)
+        yield self.exported_data(items=[], settings=settings)
         self.assertTrue(FromCrawlerCsvItemExporter.init_with_crawler)
         self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler)
 
     @defer.inlineCallbacks
     def test_pathlib_uri(self):
-        tmpdir = tempfile.mkdtemp()
-        feed_uri = Path(tmpdir) / 'res'
+        feed_path = Path(self._random_temp_filename())
         settings = {
-            'FEED_FORMAT': 'csv',
             'FEED_STORE_EMPTY': True,
-            'FEED_URI': feed_uri,
-            'FEED_PATH': feed_uri
+            'FEEDS': {
+                feed_path: {'format': 'csv'}
+            },
         }
         data = yield self.exported_no_data(settings)
-        self.assertEqual(data, b'')
-        shutil.rmtree(tmpdir, ignore_errors=True)
+        self.assertEqual(data['csv'], b'')
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 61e110845..f064a646c 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -1,7 +1,9 @@
 import unittest
+import warnings
 
-from scrapy.settings import BaseSettings
-from scrapy.utils.conf import build_component_list, arglist_to_dict
+from scrapy.exceptions import UsageError, ScrapyDeprecationWarning
+from scrapy.settings import BaseSettings, Settings
+from scrapy.utils.conf import build_component_list, arglist_to_dict, feed_process_params_from_cli
 
 
 class BuildComponentListTest(unittest.TestCase):
@@ -90,5 +92,49 @@ class UtilsConfTestCase(unittest.TestCase):
             {'arg1': 'val1', 'arg2': 'val2'})
 
 
+class FeedExportConfigTestCase(unittest.TestCase):
+
+    def test_feed_export_config_invalid_format(self):
+        settings = Settings()
+        self.assertRaises(UsageError, feed_process_params_from_cli, settings, ['items.dat'], 'noformat')
+
+    def test_feed_export_config_mismatch(self):
+        settings = Settings()
+        self.assertRaises(
+            UsageError,
+            feed_process_params_from_cli, settings, ['items1.dat', 'items2.dat'], 'noformat'
+        )
+
+    def test_feed_export_config_backward_compatible(self):
+        with warnings.catch_warnings(record=True) as cw:
+            settings = Settings()
+            self.assertEqual(
+                {'items.dat': {'format': 'csv'}},
+                feed_process_params_from_cli(settings, ['items.dat'], 'csv')
+            )
+            self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
+
+    def test_feed_export_config_explicit_formats(self):
+        settings = Settings()
+        self.assertEqual(
+            {'items_1.dat': {'format': 'json'}, 'items_2.dat': {'format': 'xml'}, 'items_3.dat': {'format': 'csv'}},
+            feed_process_params_from_cli(settings, ['items_1.dat:json', 'items_2.dat:xml', 'items_3.dat:csv'])
+        )
+
+    def test_feed_export_config_implicit_formats(self):
+        settings = Settings()
+        self.assertEqual(
+            {'items_1.json': {'format': 'json'}, 'items_2.xml': {'format': 'xml'}, 'items_3.csv': {'format': 'csv'}},
+            feed_process_params_from_cli(settings, ['items_1.json', 'items_2.xml', 'items_3.csv'])
+        )
+
+    def test_feed_export_config_stdout(self):
+        settings = Settings()
+        self.assertEqual(
+            {'stdout:': {'format': 'pickle'}},
+            feed_process_params_from_cli(settings, ['-:pickle'])
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

From c2c6ea376ca2a1a0634946f95690c03fefb9990b Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Wed, 4 Mar 2020 21:30:32 +0000
Subject: [PATCH 2687/2974] Remove obsolete DEPRECATED_SETTINGS (deprecated.py)

---
 scrapy/cmdline.py             |  2 --
 scrapy/settings/deprecated.py | 23 -----------------------
 2 files changed, 25 deletions(-)
 delete mode 100644 scrapy/settings/deprecated.py

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index ec78f7c91..a4ec7c8ae 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -12,7 +12,6 @@ from scrapy.exceptions import UsageError
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import inside_project, get_project_settings
 from scrapy.utils.python import garbage_collect
-from scrapy.settings.deprecated import check_deprecated_settings
 
 
 def _iter_command_classes(module_name):
@@ -118,7 +117,6 @@ def execute(argv=None, settings=None):
             pass
         else:
             settings['EDITOR'] = editor
-    check_deprecated_settings(settings)
 
     inproject = inside_project()
     cmds = _get_commands_dict(settings, inproject)
diff --git a/scrapy/settings/deprecated.py b/scrapy/settings/deprecated.py
deleted file mode 100644
index f6f878725..000000000
--- a/scrapy/settings/deprecated.py
+++ /dev/null
@@ -1,23 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-DEPRECATED_SETTINGS = [
-    ('TRACK_REFS', 'no longer needed (trackref is always enabled)'),
-    ('RESPONSE_CLASSES', 'no longer supported'),
-    ('DEFAULT_RESPONSE_ENCODING', 'no longer supported'),
-    ('BOT_VERSION', 'no longer used (user agent defaults to Scrapy now)'),
-    ('ENCODING_ALIASES', 'no longer needed (encoding discovery uses w3lib now)'),
-    ('STATS_ENABLED', 'no longer supported (change STATS_CLASS instead)'),
-    ('SQLITE_DB', 'no longer supported'),
-    ('AUTOTHROTTLE_MIN_DOWNLOAD_DELAY', 'use DOWNLOAD_DELAY instead'),
-    ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'),
-]
-
-
-def check_deprecated_settings(settings):
-    deprecated = [x for x in DEPRECATED_SETTINGS if settings[x[0]] is not None]
-    if deprecated:
-        msg = "You are using the following settings which are deprecated or obsolete"
-        msg += " (ask scrapy-users@googlegroups.com for alternatives):"
-        msg = msg + "\n    " + "\n    ".join("%s: %s" % x for x in deprecated)
-        warnings.warn(msg, ScrapyDeprecationWarning)

From 915e363db5cb208de9043b61015b79c91ed0a6bb Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sat, 7 Mar 2020 18:03:25 +0000
Subject: [PATCH 2688/2974] Remove a 'twisted.test.proto_helpers' deprecation
 warning

---
 tests/test_webclient.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 99a998a46..6253d5c3f 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -9,7 +9,12 @@ import OpenSSL.SSL
 from twisted.trial import unittest
 from twisted.web import server, static, util, resource
 from twisted.internet import reactor, defer
-from twisted.test.proto_helpers import StringTransport
+try:
+    from twisted.internet.testing import StringTransport
+except ImportError:
+    # deprecated in Twisted 19.7.0
+    # (remove once we bump our requirement past that version)
+    from twisted.test.proto_helpers import StringTransport
 from twisted.python.filepath import FilePath
 from twisted.protocols.policies import WrappingFactory
 from twisted.internet.defer import inlineCallbacks

From 9d9dea0d69709ef0f7aef67ddba1bd7bda25d273 Mon Sep 17 00:00:00 2001
From: Lukas Anzinger <lukas@lukasanzinger.at>
Date: Sat, 7 Mar 2020 19:54:25 +0100
Subject: [PATCH 2689/2974] Fix handling of None in allowed_domains.

Nones in allowed_domains ought to be ignored and there are also tests
for that scenario. This commit fixes the handling of None and also the
accompanying tests which are now executed again.
---
 scrapy/spidermiddlewares/offsite.py    |  8 ++++++--
 tests/test_spidermiddleware_offsite.py | 18 +++++++++---------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 232e96cbb..36f809699 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -54,12 +54,16 @@ class OffsiteMiddleware(object):
         if not allowed_domains:
             return re.compile('')  # allow all by default
         url_pattern = re.compile("^https?://.*$")
+        domains = []
         for domain in allowed_domains:
-            if url_pattern.match(domain):
+            if domain is None:
+                continue
+            elif url_pattern.match(domain):
                 message = ("allowed_domains accepts only domains, not URLs. "
                            "Ignoring URL entry %s in allowed_domains." % domain)
                 warnings.warn(message, URLWarning)
-        domains = [re.escape(d) for d in allowed_domains if d is not None]
+            else:
+                domains.append(re.escape(domain))
         regex = r'^(.*\.)?(%s)$' % '|'.join(domains)
         return re.compile(regex)
 
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 7511aa568..51c328943 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -55,21 +55,21 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware):
 
 class TestOffsiteMiddleware3(TestOffsiteMiddleware2):
 
-    def _get_spider(self):
-        return Spider('foo')
+    def _get_spiderargs(self):
+        return dict(name='foo')
 
 
 class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
 
-    def _get_spider(self):
-      bad_hostname = urlparse('http:////scrapytest.org').hostname
-      return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname])
+    def _get_spiderargs(self):
+        bad_hostname = urlparse('http:////scrapytest.org').hostname
+        return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname])
 
     def test_process_spider_output(self):
-      res = Response('http://scrapytest.org')
-      reqs = [Request('http://scrapytest.org/1')]
-      out = list(self.mw.process_spider_output(res, reqs, self.spider))
-      self.assertEqual(out, reqs)
+        res = Response('http://scrapytest.org')
+        reqs = [Request('http://scrapytest.org/1')]
+        out = list(self.mw.process_spider_output(res, reqs, self.spider))
+        self.assertEqual(out, reqs)
 
 
 class TestOffsiteMiddleware5(TestOffsiteMiddleware4):

From 91a78eef3ee9de033e66db55c49321b2cc43740e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 8 Mar 2020 22:32:17 -0300
Subject: [PATCH 2690/2974] Pass callback results as dicts instead of tuples

---
 scrapy/core/downloader/handlers/http11.py | 56 ++++++++++++++++-------
 1 file changed, 40 insertions(+), 16 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 190ae1d3b..e904cbc05 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -384,7 +384,13 @@ class ScrapyAgent(object):
     def _cb_bodyready(self, txresponse, request):
         # deliverBody hangs for responses without body
         if txresponse.length == 0:
-            return txresponse, b'', None, None
+            return {
+                "txresponse": txresponse,
+                "body": b"",
+                "flags": None,
+                "certificate": None,
+                "ip_address": None,
+            }
 
         maxsize = request.meta.get('download_maxsize', self._maxsize)
         warnsize = request.meta.get('download_warnsize', self._warnsize)
@@ -420,12 +426,18 @@ class ScrapyAgent(object):
         return d
 
     def _cb_bodydone(self, result, request, url):
-        txresponse, body, flags, certificate, ip_address = result
-        status = int(txresponse.code)
-        headers = Headers(txresponse.headers.getAllRawHeaders())
-        respcls = responsetypes.from_args(headers=headers, url=url, body=body)
-        return respcls(url=url, status=status, headers=headers, body=body,
-                       flags=flags, certificate=certificate, ip_address=ip_address)
+        status = int(result["txresponse"].code)
+        headers = Headers(result["txresponse"].headers.getAllRawHeaders())
+        respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
+        return respcls(
+            url=url,
+            status=status,
+            headers=headers,
+            body=result["body"],
+            flags=result["flags"],
+            certificate=result["certificate"],
+            ip_address=result["ip_address"],
+        )
 
 
 @implementer(IBodyProducer)
@@ -501,22 +513,34 @@ class _ResponseReader(protocol.Protocol):
 
         body = self._bodybuf.getvalue()
         if reason.check(ResponseDone):
-            self._finished.callback(
-                (self._txresponse, body, None, self._certificate, self._ip_address)
-            )
+            self._finished.callback({
+                "txresponse": self._txresponse,
+                "body": body,
+                "flags": None,
+                "certificate": self._certificate,
+                "ip_address": self._ip_address,
+            })
             return
 
         if reason.check(PotentialDataLoss):
-            self._finished.callback(
-                (self._txresponse, body, ['partial'], self._certificate, self._ip_address)
-            )
+            self._finished.callback({
+                "txresponse": self._txresponse,
+                "body": body,
+                "flags": ["partial"],
+                "certificate": self._certificate,
+                "ip_address": self._ip_address,
+            })
             return
 
         if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
             if not self._fail_on_dataloss:
-                self._finished.callback(
-                    (self._txresponse, body, ['dataloss'], self._certificate, self._ip_address)
-                )
+                self._finished.callback({
+                    "txresponse": self._txresponse,
+                    "body": body,
+                    "flags": ["dataloss"],
+                    "certificate": self._certificate,
+                    "ip_address": self._ip_address,
+                })
                 return
 
             elif not self._fail_on_dataloss_warned:

From 1785095707dec53647c835c0b0861b220e8495af Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 11 Mar 2020 20:41:59 -0300
Subject: [PATCH 2691/2974] Remove single-use variable

---
 scrapy/core/downloader/handlers/http11.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index e904cbc05..a5b03a62b 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -426,12 +426,11 @@ class ScrapyAgent(object):
         return d
 
     def _cb_bodydone(self, result, request, url):
-        status = int(result["txresponse"].code)
         headers = Headers(result["txresponse"].headers.getAllRawHeaders())
         respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
         return respcls(
             url=url,
-            status=status,
+            status=int(result["txresponse"].code),
             headers=headers,
             body=result["body"],
             flags=result["flags"],

From 49156f2ecb0197c96e3889805a233b9a626c6d65 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 11 Mar 2020 20:45:54 -0300
Subject: [PATCH 2692/2974] [doc] Feed exports: full local path as example

---
 docs/topics/feed-exports.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 6d6ba33c9..9e5968a29 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -249,7 +249,7 @@ For instance::
             'fields': None,
             'indent': 4,
         }, 
-        'items.xml': {
+        '/home/user/documents/items.xml': {
             'format': 'xml',
             'fields': ['name', 'price'],
             'encoding': 'latin1',

From f3bab819ab92cc0750c9b73141abdcbc8da7c4ac Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 11 Mar 2020 20:56:25 -0300
Subject: [PATCH 2693/2974] Add tests for
 scrapy.utils.conf.feed_complete_default_values_from_settings

---
 tests/test_utils_conf.py | 45 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index f064a646c..332120021 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -3,7 +3,12 @@ import warnings
 
 from scrapy.exceptions import UsageError, ScrapyDeprecationWarning
 from scrapy.settings import BaseSettings, Settings
-from scrapy.utils.conf import build_component_list, arglist_to_dict, feed_process_params_from_cli
+from scrapy.utils.conf import (
+    arglist_to_dict,
+    build_component_list,
+    feed_complete_default_values_from_settings,
+    feed_process_params_from_cli
+)
 
 
 class BuildComponentListTest(unittest.TestCase):
@@ -135,6 +140,44 @@ class FeedExportConfigTestCase(unittest.TestCase):
             feed_process_params_from_cli(settings, ['-:pickle'])
         )
 
+    def test_feed_complete_default_values_from_settings_empty(self):
+        feed = {}
+        settings = Settings({
+            "FEED_EXPORT_ENCODING": "custom encoding",
+            "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
+            "FEED_EXPORT_INDENT": 42,
+            "FEED_STORE_EMPTY": True,
+            "FEED_URI_PARAMS": (1, 2, 3, 4),
+        })
+        new_feed = feed_complete_default_values_from_settings(feed, settings)
+        self.assertEqual(new_feed, {
+            "encoding": "custom encoding",
+            "fields": ["f1", "f2", "f3"],
+            "indent": 42,
+            "store_empty": True,
+            "uri_params": (1, 2, 3, 4),
+        })
+
+    def test_feed_complete_default_values_from_settings_non_empty(self):
+        feed = {
+            "encoding": "other encoding",
+            "fields": None,
+        }
+        settings = Settings({
+            "FEED_EXPORT_ENCODING": "custom encoding",
+            "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
+            "FEED_EXPORT_INDENT": 42,
+            "FEED_STORE_EMPTY": True,
+        })
+        new_feed = feed_complete_default_values_from_settings(feed, settings)
+        self.assertEqual(new_feed, {
+            "encoding": "other encoding",
+            "fields": None,
+            "indent": 42,
+            "store_empty": True,
+            "uri_params": None,
+        })
+
 
 if __name__ == "__main__":
     unittest.main()

From c886a70eae35e628040da0504af3fd9aaa6aea75 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 11 Mar 2020 21:06:51 -0300
Subject: [PATCH 2694/2974] Use dict.setdefault in
 scrapy.utils.conf.feed_complete_default_values_from_settings

---
 scrapy/utils/conf.py | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index e01027491..5921f82bf 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -113,16 +113,14 @@ def get_sources(use_closest=True):
 
 def feed_complete_default_values_from_settings(feed, settings):
     out = feed.copy()
-    if 'encoding' not in out:
-        out['encoding'] = settings['FEED_EXPORT_ENCODING']
-    if 'fields' not in out:
-        out['fields'] = settings.getlist('FEED_EXPORT_FIELDS') or None
-    if 'indent' not in out:
-        out['indent'] = None if settings['FEED_EXPORT_INDENT'] is None else settings.getint('FEED_EXPORT_INDENT')
-    if 'store_empty' not in out:
-        out['store_empty'] = settings.getbool('FEED_STORE_EMPTY')
-    if 'uri_params' not in out:
-        out['uri_params'] = settings['FEED_URI_PARAMS']
+    out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
+    out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
+    out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
+    out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
+    if settings["FEED_EXPORT_INDENT"] is None:
+        out.setdefault("indent", None)
+    else:
+        out.setdefault("indent", settings.getint("FEED_EXPORT_INDENT"))
     return out
 
 

From 8d30dc08882e3b97dbaa17b3254de708c358d0d2 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 12 Mar 2020 09:36:15 -0300
Subject: [PATCH 2695/2974] Response.follow_all: return empty generators for
 empty sequences

---
 scrapy/http/response/text.py | 8 +++++---
 tests/test_http_response.py  | 4 ++++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 33a485328..2f0f3820c 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -188,9 +188,11 @@ class TextResponse(Response):
         selectors from which links cannot be obtained (for instance, anchor tags without an
         ``href`` attribute)
         """
-        arg_count = len(list(filter(None, (urls, css, xpath))))
-        if arg_count != 1:
-            raise ValueError('Please supply exactly one of the following arguments: urls, css, xpath')
+        arguments = [x for x in (urls, css, xpath) if x is not None]
+        if len(arguments) != 1:
+            raise ValueError(
+                "Please supply exactly one of the following arguments: urls, css, xpath"
+            )
         if not urls:
             if css:
                 urls = self.css(css)
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index be17dfd6b..eafc3560e 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -215,6 +215,10 @@ class BaseResponseTest(unittest.TestCase):
         links = map(Link, absolute)
         self._assert_followed_all_urls(links, absolute)
 
+    def test_follow_all_empty(self):
+        r = self.response_class("http://example.com")
+        self.assertEqual([], list(r.follow_all([])))
+
     def test_follow_all_invalid(self):
         r = self.response_class("http://example.com")
         if self.response_class == Response:

From 3b0820d747e11a1a7722f0777baf923607fc0485 Mon Sep 17 00:00:00 2001
From: nyov <nyov@users.noreply.github.com>
Date: Thu, 12 Mar 2020 19:15:49 +0000
Subject: [PATCH 2696/2974] Deprecate Spider.make_requests_from_url, part 2
 (#4412)

---
 scrapy/spiders/__init__.py | 6 ++++++
 tests/test_spider.py       | 8 +++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 9429f6cb2..ba1c866f8 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -78,6 +78,12 @@ class Spider(object_ref):
 
     def make_requests_from_url(self, url):
         """ This method is deprecated. """
+        warnings.warn(
+            "Spider.make_requests_from_url method is deprecated: "
+            "it will be removed and not be called by the default "
+            "Spider.start_requests method in future Scrapy releases. "
+            "Please override Spider.start_requests method instead."
+        )
         return Request(url, dont_filter=True)
 
     def parse(self, response):
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 317a27076..bb00c8f42 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -602,13 +602,19 @@ class DeprecationTest(unittest.TestCase):
             self.assertEqual(len(list(spider1.start_requests())), 1)
             self.assertEqual(len(w), 0)
 
+            # spider without overridden make_requests_from_url method
+            # should issue a warning when called directly
+            request = spider1.make_requests_from_url("http://www.example.com")
+            self.assertTrue(isinstance(request, Request))
+            self.assertEqual(len(w), 1)
+
             # spider with overridden make_requests_from_url issues a warning,
             # but the method still works
             spider2 = MySpider5()
             requests = list(spider2.start_requests())
             self.assertEqual(len(requests), 1)
             self.assertEqual(requests[0].url, 'http://example.com/foo')
-            self.assertEqual(len(w), 1)
+            self.assertEqual(len(w), 2)
 
 
 class NoParseMethodSpiderTest(unittest.TestCase):

From ccc4d88779cf2827431ef9e73f976f755c04fe0e Mon Sep 17 00:00:00 2001
From: Lukas Anzinger <lukas@lukasanzinger.at>
Date: Thu, 12 Mar 2020 20:42:14 +0100
Subject: [PATCH 2697/2974] Ignore a domain in allowed_domains with port and
 issue a warning (#4413)

---
 scrapy/spidermiddlewares/offsite.py    | 11 ++++++++++-
 tests/test_spidermiddleware_offsite.py | 15 +++++++++++++--
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 36f809699..2fab572e6 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -53,7 +53,8 @@ class OffsiteMiddleware(object):
         allowed_domains = getattr(spider, 'allowed_domains', None)
         if not allowed_domains:
             return re.compile('')  # allow all by default
-        url_pattern = re.compile("^https?://.*$")
+        url_pattern = re.compile(r"^https?://.*$")
+        port_pattern = re.compile(r":\d+$")
         domains = []
         for domain in allowed_domains:
             if domain is None:
@@ -62,6 +63,10 @@ class OffsiteMiddleware(object):
                 message = ("allowed_domains accepts only domains, not URLs. "
                            "Ignoring URL entry %s in allowed_domains." % domain)
                 warnings.warn(message, URLWarning)
+            elif port_pattern.search(domain):
+                message = ("allowed_domains accepts only domains without ports. "
+                           "Ignoring entry %s in allowed_domains." % domain)
+                warnings.warn(message, PortWarning)
             else:
                 domains.append(re.escape(domain))
         regex = r'^(.*\.)?(%s)$' % '|'.join(domains)
@@ -74,3 +79,7 @@ class OffsiteMiddleware(object):
 
 class URLWarning(Warning):
     pass
+
+
+class PortWarning(Warning):
+    pass
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 51c328943..b96807bc2 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -4,7 +4,7 @@ import warnings
 
 from scrapy.http import Response, Request
 from scrapy.spiders import Spider
-from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning
+from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning, PortWarning
 from scrapy.utils.test import get_crawler
 
 
@@ -26,7 +26,8 @@ class TestOffsiteMiddleware(TestCase):
                        Request('http://scrapy.org/1'),
                        Request('http://sub.scrapy.org/1'),
                        Request('http://offsite.tld/letmepass', dont_filter=True),
-                       Request('http://scrapy.test.org/')]
+                       Request('http://scrapy.test.org/'),
+                       Request('http://scrapy.test.org:8000/')]
         offsite_reqs = [Request('http://scrapy2.org'),
                        Request('http://offsite.tld/'),
                        Request('http://offsite.tld/scrapytest.org'),
@@ -80,3 +81,13 @@ class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
             warnings.simplefilter("always")
             self.mw.get_host_regex(self.spider)
             assert issubclass(w[-1].category, URLWarning)
+
+
+class TestOffsiteMiddleware6(TestOffsiteMiddleware4):
+
+    def test_get_host_regex(self):
+        self.spider.allowed_domains = ['scrapytest.org:8000', 'scrapy.org', 'scrapy.test.org']
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+            self.mw.get_host_regex(self.spider)
+            assert issubclass(w[-1].category, PortWarning)

From 3f6cdcabceff5c5b2ac2935f8073b6efe817645b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 13 Mar 2020 13:25:53 +0100
Subject: [PATCH 2698/2974] Restrict pytest to versions prior to 5.4

---
 tests/requirements-py3.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index d97c4b8ee..d207c5fb0 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -2,7 +2,7 @@
 jmespath
 mitmproxy; python_version >= '3.6'
 mitmproxy<4.0.0; python_version < '3.6'
-pytest
+pytest < 5.4
 pytest-cov
 pytest-twisted >= 1.11
 pytest-xdist

From f9bf4b8d4dd64a1d65e949927b8ea7ad34e756d3 Mon Sep 17 00:00:00 2001
From: Aditya Kumar <k.aditya00@gmail.com>
Date: Sat, 14 Mar 2020 15:09:00 +0530
Subject: [PATCH 2699/2974] Remove all top-level imports for
 twisted.internet.reactor (#4406)

---
 docs/topics/settings.rst                  | 61 ++++++++++++++++++++++-
 scrapy/core/downloader/__init__.py        |  3 +-
 scrapy/core/downloader/handlers/ftp.py    |  2 +-
 scrapy/core/downloader/handlers/http10.py |  3 +-
 scrapy/core/downloader/handlers/http11.py |  6 ++-
 scrapy/extensions/closespider.py          |  3 +-
 scrapy/mail.py                            |  5 +-
 scrapy/utils/benchserver.py               |  2 +-
 scrapy/utils/testproc.py                  |  3 +-
 scrapy/utils/testsite.py                  |  3 +-
 10 files changed, 78 insertions(+), 13 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index a70023efa..c01202a10 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1473,7 +1473,66 @@ If a reactor is already installed,
 
 :meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises
 :exc:`Exception` if the installed reactor does not match the
-:setting:`TWISTED_REACTOR` setting.
+:setting:`TWISTED_REACTOR` setting; therfore, having top-level
+:mod:`~twisted.internet.reactor` imports in project files and imported
+third-party libraries will make Scrapy raise :exc:`Exception` when
+it checks which reactor is installed.
+
+In order to use the reactor installed by Scrapy::
+
+    import scrapy
+    from twisted.internet import reactor
+
+
+    class QuotesSpider(scrapy.Spider):
+        name = 'quotes'
+
+        def __init__(self, *args, **kwargs):
+            self.timeout = int(kwargs.pop('timeout', '60'))
+            super(QuotesSpider, self).__init__(*args, **kwargs)
+
+        def start_requests(self):
+            reactor.callLater(self.timeout, self.stop)
+
+            urls = ['http://quotes.toscrape.com/page/1']
+            for url in urls:
+                yield scrapy.Request(url=url, callback=self.parse)
+
+        def parse(self, response):
+            for quote in response.css('div.quote'):
+                yield {'text': quote.css('span.text::text').get()}
+
+        def stop(self):
+            self.crawler.engine.close_spider(self, 'timeout')
+
+
+which raises :exc:`Exception`, becomes::
+
+    import scrapy
+
+
+    class QuotesSpider(scrapy.Spider):
+        name = 'quotes'
+
+        def __init__(self, *args, **kwargs):
+            self.timeout = int(kwargs.pop('timeout', '60'))
+            super(QuotesSpider, self).__init__(*args, **kwargs)
+
+        def start_requests(self):
+            from twisted.internet import reactor
+            reactor.callLater(self.timeout, self.stop)
+
+            urls = ['http://quotes.toscrape.com/page/1']
+            for url in urls:
+                yield scrapy.Request(url=url, callback=self.parse)
+
+        def parse(self, response):
+            for quote in response.css('div.quote'):
+                yield {'text': quote.css('span.text::text').get()}
+
+        def stop(self):
+            self.crawler.engine.close_spider(self, 'timeout')
+
 
 The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
 means that Scrapy will not attempt to install any specific reactor, and the
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 5a2fdadf5..644be121f 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -3,7 +3,7 @@ from time import time
 from datetime import datetime
 from collections import deque
 
-from twisted.internet import reactor, defer, task
+from twisted.internet import defer, task
 
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.httpobj import urlparse_cached
@@ -133,6 +133,7 @@ class Downloader(object):
         return deferred
 
     def _process_queue(self, spider, slot):
+        from twisted.internet import reactor
         if slot.latercall and slot.latercall.active():
             return
 
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 1681c6df8..432cb1831 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -32,7 +32,6 @@ import re
 from io import BytesIO
 from urllib.parse import unquote
 
-from twisted.internet import reactor
 from twisted.internet.protocol import ClientCreator, Protocol
 from twisted.protocols.ftp import CommandFailed, FTPClient
 
@@ -81,6 +80,7 @@ class FTPDownloadHandler:
         return cls(crawler.settings)
 
     def download_request(self, request, spider):
+        from twisted.internet import reactor
         parsed_url = urlparse_cached(request)
         user = request.meta.get("ftp_user", self.default_user)
         password = request.meta.get("ftp_password", self.default_password)
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index d4aa51bd1..c0146a0a6 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -1,7 +1,5 @@
 """Download handlers for http and https schemes
 """
-from twisted.internet import reactor
-
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import to_unicode
 
@@ -26,6 +24,7 @@ class HTTP10DownloadHandler:
         return factory.deferred
 
     def _connect(self, factory):
+        from twisted.internet import reactor
         host, port = to_unicode(factory.host), factory.port
         if factory.scheme == b'https':
             client_context_factory = create_instance(
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 93951d3b5..04a8d617a 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -8,7 +8,7 @@ from io import BytesIO
 from time import time
 from urllib.parse import urldefrag
 
-from twisted.internet import defer, protocol, reactor, ssl
+from twisted.internet import defer, protocol, ssl
 from twisted.internet.endpoints import TCP4ClientEndpoint
 from twisted.internet.error import TimeoutError
 from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
@@ -33,6 +33,7 @@ class HTTP11DownloadHandler:
     lazy = False
 
     def __init__(self, settings, crawler=None):
+        from twisted.internet import reactor
         self._pool = HTTPConnectionPool(reactor, persistent=True)
         self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self._pool._factory.noisy = False
@@ -81,6 +82,7 @@ class HTTP11DownloadHandler:
         return agent.download_request(request)
 
     def close(self):
+        from twisted.internet import reactor
         d = self._pool.closeCachedConnections()
         # closeCachedConnections will hang on network or server issues, so
         # we'll manually timeout the deferred.
@@ -284,6 +286,7 @@ class ScrapyAgent(object):
         self._txresponse = None
 
     def _get_agent(self, request, timeout):
+        from twisted.internet import reactor
         bindaddress = request.meta.get('bindaddress') or self._bindAddress
         proxy = request.meta.get('proxy')
         if proxy:
@@ -326,6 +329,7 @@ class ScrapyAgent(object):
         )
 
     def download_request(self, request):
+        from twisted.internet import reactor
         timeout = request.meta.get('download_timeout') or self._connectTimeout
         agent = self._get_agent(request, timeout)
 
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index afb2ed049..260b2e86e 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -6,8 +6,6 @@ See documentation in docs/topics/extensions.rst
 
 from collections import defaultdict
 
-from twisted.internet import reactor
-
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
 
@@ -54,6 +52,7 @@ class CloseSpider(object):
             self.crawler.engine.close_spider(spider, 'closespider_pagecount')
 
     def spider_opened(self, spider):
+        from twisted.internet import reactor
         self.task = reactor.callLater(self.close_on['timeout'],
                                       self.crawler.engine.close_spider, spider,
                                       reason='closespider_timeout')
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 9655b8114..b2a24a3db 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -12,7 +12,7 @@ from email.mime.text import MIMEText
 from email.utils import COMMASPACE, formatdate
 from io import BytesIO
 
-from twisted.internet import defer, reactor, ssl
+from twisted.internet import defer, ssl
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
@@ -28,7 +28,6 @@ def _to_bytes_or_none(text):
 
 
 class MailSender(object):
-
     def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost',
             smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):
         self.smtphost = smtphost
@@ -47,6 +46,7 @@ class MailSender(object):
             settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL'))
 
     def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None):
+        from twisted.internet import reactor
         if attachs:
             msg = MIMEMultipart()
         else:
@@ -111,6 +111,7 @@ class MailSender(object):
 
     def _sendmail(self, to_addrs, msg):
         # Import twisted.mail here because it is not available in python3
+        from twisted.internet import reactor
         from twisted.mail.smtp import ESMTPSenderFactory
         msg = BytesIO(msg)
         d = defer.Deferred()
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index cdbe21942..9d8d64612 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -3,7 +3,6 @@ from urllib.parse import urlencode
 
 from twisted.web.server import Site
 from twisted.web.resource import Resource
-from twisted.internet import reactor
 
 
 class Root(Resource):
@@ -34,6 +33,7 @@ def _getarg(request, name, default=None, type=str):
 
 
 if __name__ == '__main__':
+    from twisted.internet import reactor
     root = Root()
     factory = Site(root)
     httpPort = reactor.listenTCP(8998, Site(root))
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index 0f15cf60a..37803b287 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -1,7 +1,7 @@
 import sys
 import os
 
-from twisted.internet import reactor, defer, protocol
+from twisted.internet import defer, protocol
 
 
 class ProcessTest(object):
@@ -11,6 +11,7 @@ class ProcessTest(object):
     cwd = os.getcwd()  # trial chdirs to temp dir
 
     def execute(self, args, check_code=True, settings=None):
+        from twisted.internet import reactor
         env = os.environ.copy()
         if settings is not None:
             env['SCRAPY_SETTINGS_MODULE'] = settings
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index 6f5c21624..9e1598805 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -1,12 +1,12 @@
 from urllib.parse import urljoin
 
-from twisted.internet import reactor
 from twisted.web import server, resource, static, util
 
 
 class SiteTest(object):
 
     def setUp(self):
+        from twisted.internet import reactor
         super(SiteTest, self).setUp()
         self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
         self.baseurl = "http://localhost:%d/" % self.site.getHost().port
@@ -38,6 +38,7 @@ def test_site():
 
 
 if __name__ == '__main__':
+    from twisted.internet import reactor
     port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
     print("http://localhost:%d/" % port.getHost().port)
     reactor.run()

From e5711127b162ce78e6c1b60ef209792cd6eef4e6 Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Mon, 16 Mar 2020 15:43:02 -0300
Subject: [PATCH 2700/2974] Remove deprecated ChunkedTransferMiddleware (#4431)

---
 scrapy/downloadermiddlewares/chunked.py | 21 ---------------------
 1 file changed, 21 deletions(-)
 delete mode 100644 scrapy/downloadermiddlewares/chunked.py

diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py
deleted file mode 100644
index 6748d0265..000000000
--- a/scrapy/downloadermiddlewares/chunked.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import warnings
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.http import decode_chunked_transfer
-
-
-warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, "
-              "chunked transfers are supported by default.",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-
-class ChunkedTransferMiddleware(object):
-    """This middleware adds support for chunked transfer encoding, as
-    documented in: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
-    """
-
-    def process_response(self, request, response, spider):
-        if response.headers.get('Transfer-Encoding') == 'chunked':
-            body = decode_chunked_transfer(response.body)
-            return response.replace(body=body)
-        return response

From dfbe1d95071acfbba159cea051530749b1684460 Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Mon, 16 Mar 2020 16:12:46 -0300
Subject: [PATCH 2701/2974] Remove object base class (#4430)

---
 docs/topics/exporters.rst                      |  2 +-
 docs/topics/extensions.rst                     |  2 +-
 docs/topics/item-pipeline.rst                  | 10 +++++-----
 docs/topics/leaks.rst                          |  2 +-
 docs/topics/settings.rst                       |  2 +-
 docs/topics/stats.rst                          |  2 +-
 scrapy/commands/__init__.py                    |  2 +-
 scrapy/commands/bench.py                       |  2 +-
 scrapy/contracts/__init__.py                   |  4 ++--
 scrapy/core/downloader/__init__.py             |  4 ++--
 scrapy/core/downloader/handlers/http11.py      |  4 ++--
 scrapy/core/engine.py                          |  4 ++--
 scrapy/core/scheduler.py                       |  2 +-
 scrapy/core/scraper.py                         |  4 ++--
 scrapy/downloadermiddlewares/ajaxcrawl.py      |  2 +-
 scrapy/downloadermiddlewares/cookies.py        |  2 +-
 scrapy/downloadermiddlewares/decompression.py  |  2 +-
 scrapy/downloadermiddlewares/defaultheaders.py |  2 +-
 .../downloadermiddlewares/downloadtimeout.py   |  2 +-
 scrapy/downloadermiddlewares/httpauth.py       |  2 +-
 scrapy/downloadermiddlewares/httpcache.py      |  2 +-
 .../downloadermiddlewares/httpcompression.py   |  2 +-
 scrapy/downloadermiddlewares/httpproxy.py      |  2 +-
 scrapy/downloadermiddlewares/redirect.py       |  2 +-
 scrapy/downloadermiddlewares/retry.py          |  2 +-
 scrapy/downloadermiddlewares/robotstxt.py      |  2 +-
 scrapy/downloadermiddlewares/stats.py          |  2 +-
 scrapy/downloadermiddlewares/useragent.py      |  2 +-
 scrapy/dupefilters.py                          |  2 +-
 scrapy/exporters.py                            |  2 +-
 scrapy/extensions/closespider.py               |  2 +-
 scrapy/extensions/corestats.py                 |  2 +-
 scrapy/extensions/debug.py                     |  4 ++--
 scrapy/extensions/feedexport.py                | 10 +++++-----
 scrapy/extensions/httpcache.py                 |  8 ++++----
 scrapy/extensions/logstats.py                  |  2 +-
 scrapy/extensions/memdebug.py                  |  2 +-
 scrapy/extensions/memusage.py                  |  2 +-
 scrapy/extensions/spiderstate.py               |  2 +-
 scrapy/extensions/statsmailer.py               |  2 +-
 scrapy/extensions/throttle.py                  |  2 +-
 scrapy/http/cookies.py                         |  8 ++++----
 scrapy/link.py                                 |  2 +-
 scrapy/linkextractors/__init__.py              |  2 +-
 scrapy/linkextractors/lxmlhtml.py              |  2 +-
 scrapy/loader/__init__.py                      |  2 +-
 scrapy/loader/processors.py                    | 12 ++++++------
 scrapy/logformatter.py                         |  2 +-
 scrapy/mail.py                                 |  2 +-
 scrapy/middleware.py                           |  2 +-
 scrapy/pipelines/files.py                      |  8 ++++----
 scrapy/pipelines/media.py                      |  4 ++--
 scrapy/pqueues.py                              |  6 +++---
 scrapy/responsetypes.py                        |  2 +-
 scrapy/settings/__init__.py                    |  2 +-
 scrapy/shell.py                                |  2 +-
 scrapy/signalmanager.py                        |  2 +-
 scrapy/spiderloader.py                         |  2 +-
 scrapy/spidermiddlewares/depth.py              |  2 +-
 scrapy/spidermiddlewares/httperror.py          |  2 +-
 scrapy/spidermiddlewares/offsite.py            |  2 +-
 scrapy/spidermiddlewares/referer.py            |  4 ++--
 scrapy/spidermiddlewares/urllength.py          |  2 +-
 scrapy/spiders/crawl.py                        |  2 +-
 scrapy/statscollectors.py                      |  2 +-
 .../project/module/middlewares.py.tmpl         |  4 ++--
 .../templates/project/module/pipelines.py.tmpl |  2 +-
 scrapy/utils/datatypes.py                      |  2 +-
 scrapy/utils/deprecate.py                      |  2 +-
 scrapy/utils/iterators.py                      |  2 +-
 scrapy/utils/log.py                            |  2 +-
 scrapy/utils/python.py                         |  4 ++--
 scrapy/utils/reactor.py                        |  2 +-
 scrapy/utils/sitemap.py                        |  2 +-
 scrapy/utils/testproc.py                       |  2 +-
 scrapy/utils/testsite.py                       |  2 +-
 scrapy/utils/trackref.py                       |  5 ++---
 tests/pipelines.py                             |  4 ++--
 tests/test_cmdline/extensions.py               |  4 ++--
 .../test_spider/pipelines.py                   |  4 ++--
 tests/test_command_parse.py                    |  2 +-
 tests/test_contracts.py                        |  2 +-
 tests/test_crawler.py                          |  2 +-
 tests/test_downloadermiddleware.py             |  2 +-
 tests/test_dupefilters.py                      |  2 +-
 tests/test_engine.py                           |  2 +-
 tests/test_feedexport.py                       |  2 +-
 tests/test_item.py                             |  2 +-
 tests/test_loader.py                           |  2 +-
 tests/test_logformatter.py                     |  2 +-
 tests/test_middleware.py                       |  8 ++++----
 tests/test_request_cb_kwargs.py                |  4 ++--
 tests/test_scheduler.py                        |  6 +++---
 tests/test_spidermiddleware_referer.py         | 18 +++++++++---------
 tests/test_squeues.py                          |  2 +-
 tests/test_utils_deprecate.py                  |  6 +++---
 tests/test_utils_python.py                     |  8 ++++----
 tests/test_utils_reqser.py                     |  2 +-
 98 files changed, 155 insertions(+), 156 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index d411e2eed..e52682690 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -42,7 +42,7 @@ value of one of their fields::
 
     from scrapy.exporters import XmlItemExporter
 
-    class PerYearXmlExportPipeline(object):
+    class PerYearXmlExportPipeline:
         """Distribute items across multiple XML files according to their 'year' field"""
 
         def open_spider(self, spider):
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index dc057f6b6..94fd2e36e 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -107,7 +107,7 @@ Here is the code of such extension::
 
     logger = logging.getLogger(__name__)
 
-    class SpiderOpenCloseLogging(object):
+    class SpiderOpenCloseLogging:
 
         def __init__(self, item_count):
             self.item_count = item_count
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 98e2506e5..533f84630 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -81,7 +81,7 @@ contain a price::
 
     from scrapy.exceptions import DropItem
 
-    class PricePipeline(object):
+    class PricePipeline:
 
         vat_factor = 1.15
 
@@ -103,7 +103,7 @@ format::
 
    import json
 
-   class JsonWriterPipeline(object):
+   class JsonWriterPipeline:
 
        def open_spider(self, spider):
            self.file = open('items.jl', 'w')
@@ -132,7 +132,7 @@ method and how to clean up the resources properly.::
 
     import pymongo
 
-    class MongoPipeline(object):
+    class MongoPipeline:
 
         collection_name = 'scrapy_items'
 
@@ -180,7 +180,7 @@ it saves the screenshot to a file and adds filename to the item.
     from urllib.parse import quote
 
 
-    class ScreenshotPipeline(object):
+    class ScreenshotPipeline:
         """Pipeline that uses Splash to render screenshot of
         every Scrapy item."""
 
@@ -219,7 +219,7 @@ returns multiples items with the same id::
 
     from scrapy.exceptions import DropItem
 
-    class DuplicatesPipeline(object):
+    class DuplicatesPipeline:
 
         def __init__(self):
             self.ids_seen = set()
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index c0c83fc84..4ee447065 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -170,7 +170,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
 
 .. class:: object_ref
 
-    Inherit from this class (instead of object) if you want to track live
+    Inherit from this class if you want to track live
     instances with the ``trackref`` module.
 
 .. function:: print_live_refs(class_name, ignore=NoneType)
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index c01202a10..dc6843d75 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -124,7 +124,7 @@ Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings`
 attribute of the Crawler that is passed to ``from_crawler`` method in
 extensions, middlewares and item pipelines::
 
-    class MyExtension(object):
+    class MyExtension:
         def __init__(self, log_is_enabled=False):
             if log_is_enabled:
                 print("log is enabled!")
diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst
index 3dd829ebe..af848b402 100644
--- a/docs/topics/stats.rst
+++ b/docs/topics/stats.rst
@@ -32,7 +32,7 @@ Common Stats Collector uses
 Access the stats collector through the :attr:`~scrapy.crawler.Crawler.stats`
 attribute. Here is an example of an extension that access stats::
 
-    class ExtensionThatAccessStats(object):
+    class ExtensionThatAccessStats:
 
         def __init__(self, stats):
             self.stats = stats
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 0b24193c2..a573a03d9 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -9,7 +9,7 @@ from scrapy.utils.conf import arglist_to_dict
 from scrapy.exceptions import UsageError
 
 
-class ScrapyCommand(object):
+class ScrapyCommand:
 
     requires_project = False
     crawler_process = None
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 7bbe362e7..c9f3b38e0 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -25,7 +25,7 @@ class Command(ScrapyCommand):
             self.crawler_process.start()
 
 
-class _BenchServer(object):
+class _BenchServer:
 
     def __enter__(self):
         from scrapy.utils.test import get_testenv
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 7b6591d86..41d4f25b2 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -9,7 +9,7 @@ from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.python import get_spec
 
 
-class ContractsManager(object):
+class ContractsManager:
     contracts = {}
 
     def __init__(self, contracts):
@@ -107,7 +107,7 @@ class ContractsManager(object):
         request.errback = eb_wrapper
 
 
-class Contract(object):
+class Contract:
     """ Abstract class for contracts """
     request_cls = None
 
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 644be121f..36aca4dae 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -13,7 +13,7 @@ from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.core.downloader.handlers import DownloadHandlers
 
 
-class Slot(object):
+class Slot:
     """Downloader slot"""
 
     def __init__(self, concurrency, delay, randomize_delay):
@@ -66,7 +66,7 @@ def _get_concurrency_delay(concurrency, spider, settings):
     return concurrency, delay
 
 
-class Downloader(object):
+class Downloader:
 
     DOWNLOAD_SLOT = 'download_slot'
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 04a8d617a..c970909d7 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -268,7 +268,7 @@ class ScrapyProxyAgent(Agent):
         )
 
 
-class ScrapyAgent(object):
+class ScrapyAgent:
 
     _Agent = Agent
     _ProxyAgent = ScrapyProxyAgent
@@ -432,7 +432,7 @@ class ScrapyAgent(object):
 
 
 @implementer(IBodyProducer)
-class _RequestBodyProducer(object):
+class _RequestBodyProducer:
 
     def __init__(self, body):
         self.body = body
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 6ab8cde6b..74f03344e 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -21,7 +21,7 @@ from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
 logger = logging.getLogger(__name__)
 
 
-class Slot(object):
+class Slot:
 
     def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
         self.closing = False
@@ -53,7 +53,7 @@ class Slot(object):
             self.closing.callback(None)
 
 
-class ExecutionEngine(object):
+class ExecutionEngine:
 
     def __init__(self, crawler, spider_closed_callback):
         self.crawler = crawler
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index c96b9b719..a18c26b17 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -14,7 +14,7 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning
 logger = logging.getLogger(__name__)
 
 
-class Scheduler(object):
+class Scheduler:
     """
     Scrapy Scheduler. It allows to enqueue requests and then get
     a next request to download. Scheduler is also handling duplication
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 41f015017..3e4826216 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -21,7 +21,7 @@ from scrapy.core.spidermw import SpiderMiddlewareManager
 logger = logging.getLogger(__name__)
 
 
-class Slot(object):
+class Slot:
     """Scraper slot (one per running spider)"""
 
     MIN_RESPONSE_SIZE = 1024
@@ -62,7 +62,7 @@ class Slot(object):
         return self.active_size > self.max_active_size
 
 
-class Scraper(object):
+class Scraper:
 
     def __init__(self, crawler):
         self.slot = None
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 16b046e99..ad7a81e6b 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -11,7 +11,7 @@ from scrapy.http import HtmlResponse
 logger = logging.getLogger(__name__)
 
 
-class AjaxCrawlMiddleware(object):
+class AjaxCrawlMiddleware:
     """
     Handle 'AJAX crawlable' pages marked as crawlable via meta tag.
     For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index d8dabdf13..d57f04bc3 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -10,7 +10,7 @@ from scrapy.utils.python import to_unicode
 logger = logging.getLogger(__name__)
 
 
-class CookiesMiddleware(object):
+class CookiesMiddleware:
     """This middleware enables working with sites that need cookies"""
 
     def __init__(self, debug=False):
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index fcea38ef5..0fcf8fb8c 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -16,7 +16,7 @@ from scrapy.responsetypes import responsetypes
 logger = logging.getLogger(__name__)
 
 
-class DecompressionMiddleware(object):
+class DecompressionMiddleware:
     """ This middleware tries to recognise and extract the possibly compressed
     responses that may arrive. """
 
diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py
index 93fe97673..f67961881 100644
--- a/scrapy/downloadermiddlewares/defaultheaders.py
+++ b/scrapy/downloadermiddlewares/defaultheaders.py
@@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst
 from scrapy.utils.python import without_none_values
 
 
-class DefaultHeadersMiddleware(object):
+class DefaultHeadersMiddleware:
 
     def __init__(self, headers):
         self._headers = headers
diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py
index 18123cfce..d373a22df 100644
--- a/scrapy/downloadermiddlewares/downloadtimeout.py
+++ b/scrapy/downloadermiddlewares/downloadtimeout.py
@@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst
 from scrapy import signals
 
 
-class DownloadTimeoutMiddleware(object):
+class DownloadTimeoutMiddleware:
 
     def __init__(self, timeout=180):
         self._timeout = timeout
diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py
index 7aa7a62bc..089bf0d85 100644
--- a/scrapy/downloadermiddlewares/httpauth.py
+++ b/scrapy/downloadermiddlewares/httpauth.py
@@ -9,7 +9,7 @@ from w3lib.http import basic_auth_header
 from scrapy import signals
 
 
-class HttpAuthMiddleware(object):
+class HttpAuthMiddleware:
     """Set Basic HTTP Authorization header
     (http_user and http_pass spider class attributes)"""
 
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 4e06f8236..6db57bd8b 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -17,7 +17,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.utils.misc import load_object
 
 
-class HttpCacheMiddleware(object):
+class HttpCacheMiddleware:
 
     DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
                            ConnectionRefusedError, ConnectionDone, ConnectError,
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 0010b2a8f..727c41466 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -15,7 +15,7 @@ except ImportError:
     pass
 
 
-class HttpCompressionMiddleware(object):
+class HttpCompressionMiddleware:
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
     @classmethod
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 814ce78fe..da89d3e9b 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -7,7 +7,7 @@ from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_bytes
 
 
-class HttpProxyMiddleware(object):
+class HttpProxyMiddleware:
 
     def __init__(self, auth_encoding='latin-1'):
         self.auth_encoding = auth_encoding
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 08cff8a55..09ee8377e 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -11,7 +11,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
 logger = logging.getLogger(__name__)
 
 
-class BaseRedirectMiddleware(object):
+class BaseRedirectMiddleware:
 
     enabled_setting = 'REDIRECT_ENABLED'
 
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 7ab5b6e62..bbf5fca05 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -25,7 +25,7 @@ from scrapy.utils.python import global_object_name
 logger = logging.getLogger(__name__)
 
 
-class RetryMiddleware(object):
+class RetryMiddleware:
 
     # IOError is raised by the HttpCompression middleware when trying to
     # decompress an empty response
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 251706c50..7f18b2bf2 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -16,7 +16,7 @@ from scrapy.utils.misc import load_object
 logger = logging.getLogger(__name__)
 
 
-class RobotsTxtMiddleware(object):
+class RobotsTxtMiddleware:
     DOWNLOAD_PRIORITY = 1000
 
     def __init__(self, crawler):
diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py
index ef0aafce0..46a2ad397 100644
--- a/scrapy/downloadermiddlewares/stats.py
+++ b/scrapy/downloadermiddlewares/stats.py
@@ -4,7 +4,7 @@ from scrapy.utils.response import response_httprepr
 from scrapy.utils.python import global_object_name
 
 
-class DownloaderStats(object):
+class DownloaderStats:
 
     def __init__(self, stats):
         self.stats = stats
diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py
index d24750c69..3ee7bd129 100644
--- a/scrapy/downloadermiddlewares/useragent.py
+++ b/scrapy/downloadermiddlewares/useragent.py
@@ -3,7 +3,7 @@
 from scrapy import signals
 
 
-class UserAgentMiddleware(object):
+class UserAgentMiddleware:
     """This middleware allows spiders to override the user_agent"""
 
     def __init__(self, user_agent='Scrapy'):
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index a36c8304f..d74c8ed36 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -5,7 +5,7 @@ from scrapy.utils.job import job_dir
 from scrapy.utils.request import referer_str, request_fingerprint
 
 
-class BaseDupeFilter(object):
+class BaseDupeFilter:
 
     @classmethod
     def from_settings(cls, settings):
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 2e20a7180..0cb6cef98 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -21,7 +21,7 @@ __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
            'JsonItemExporter', 'MarshalItemExporter']
 
 
-class BaseItemExporter(object):
+class BaseItemExporter:
 
     def __init__(self, *, dont_fail=False, **kwargs):
         self._kwargs = kwargs
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index 260b2e86e..e3f212bef 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -10,7 +10,7 @@ from scrapy import signals
 from scrapy.exceptions import NotConfigured
 
 
-class CloseSpider(object):
+class CloseSpider:
 
     def __init__(self, crawler):
         self.crawler = crawler
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 20adfbe4b..389cb65bc 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -6,7 +6,7 @@ from datetime import datetime
 from scrapy import signals
 
 
-class CoreStats(object):
+class CoreStats:
 
     def __init__(self, stats):
         self.stats = stats
diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py
index 625e13249..586399784 100644
--- a/scrapy/extensions/debug.py
+++ b/scrapy/extensions/debug.py
@@ -17,7 +17,7 @@ from scrapy.utils.trackref import format_live_refs
 logger = logging.getLogger(__name__)
 
 
-class StackTraceDump(object):
+class StackTraceDump:
 
     def __init__(self, crawler=None):
         self.crawler = crawler
@@ -52,7 +52,7 @@ class StackTraceDump(object):
         return dumps
 
 
-class Debugger(object):
+class Debugger:
     def __init__(self):
         try:
             signal.signal(signal.SIGUSR2, self._enter_debugger)
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 108b6d35c..998d2a5d1 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -44,7 +44,7 @@ class IFeedStorage(Interface):
 
 
 @implementer(IFeedStorage)
-class BlockingFeedStorage(object):
+class BlockingFeedStorage:
 
     def open(self, spider):
         path = spider.crawler.settings['FEED_TEMPDIR']
@@ -61,7 +61,7 @@ class BlockingFeedStorage(object):
 
 
 @implementer(IFeedStorage)
-class StdoutFeedStorage(object):
+class StdoutFeedStorage:
 
     def __init__(self, uri, _stdout=None):
         if not _stdout:
@@ -76,7 +76,7 @@ class StdoutFeedStorage(object):
 
 
 @implementer(IFeedStorage)
-class FileFeedStorage(object):
+class FileFeedStorage:
 
     def __init__(self, uri):
         self.path = file_uri_to_path(uri)
@@ -179,7 +179,7 @@ class FTPFeedStorage(BlockingFeedStorage):
         )
 
 
-class _FeedSlot(object):
+class _FeedSlot:
     def __init__(self, file, exporter, storage, uri, format, store_empty):
         self.file = file
         self.exporter = exporter
@@ -203,7 +203,7 @@ class _FeedSlot(object):
             self._exporting = False
 
 
-class FeedExporter(object):
+class FeedExporter:
 
     @classmethod
     def from_crawler(cls, crawler):
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 91850683f..8546628a8 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -20,7 +20,7 @@ from scrapy.utils.request import request_fingerprint
 logger = logging.getLogger(__name__)
 
 
-class DummyPolicy(object):
+class DummyPolicy:
 
     def __init__(self, settings):
         self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
@@ -39,7 +39,7 @@ class DummyPolicy(object):
         return True
 
 
-class RFC2616Policy(object):
+class RFC2616Policy:
 
     MAXAGE = 3600 * 24 * 365  # one year
 
@@ -213,7 +213,7 @@ class RFC2616Policy(object):
         return currentage
 
 
-class DbmCacheStorage(object):
+class DbmCacheStorage:
 
     def __init__(self, settings):
         self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
@@ -270,7 +270,7 @@ class DbmCacheStorage(object):
         return request_fingerprint(request)
 
 
-class FilesystemCacheStorage(object):
+class FilesystemCacheStorage:
 
     def __init__(self, settings):
         self.cachedir = data_path(settings['HTTPCACHE_DIR'])
diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py
index b685e7b19..0be2831a1 100644
--- a/scrapy/extensions/logstats.py
+++ b/scrapy/extensions/logstats.py
@@ -8,7 +8,7 @@ from scrapy import signals
 logger = logging.getLogger(__name__)
 
 
-class LogStats(object):
+class LogStats:
     """Log basic scraping stats periodically"""
 
     def __init__(self, stats, interval=60.0):
diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py
index 892aa8a86..dc8cdbb1d 100644
--- a/scrapy/extensions/memdebug.py
+++ b/scrapy/extensions/memdebug.py
@@ -11,7 +11,7 @@ from scrapy.exceptions import NotConfigured
 from scrapy.utils.trackref import live_refs
 
 
-class MemoryDebugger(object):
+class MemoryDebugger:
 
     def __init__(self, stats):
         self.stats = stats
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 14e0fb32d..a0540bf8f 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -19,7 +19,7 @@ from scrapy.utils.engine import get_engine_status
 logger = logging.getLogger(__name__)
 
 
-class MemoryUsage(object):
+class MemoryUsage:
 
     def __init__(self, crawler):
         if not crawler.settings.getbool('MEMUSAGE_ENABLED'):
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 2c8e46914..2e5ff569f 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -6,7 +6,7 @@ from scrapy.exceptions import NotConfigured
 from scrapy.utils.job import job_dir
 
 
-class SpiderState(object):
+class SpiderState:
     """Store and load spider state during a scraping job"""
 
     def __init__(self, jobdir=None):
diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py
index 6a982195d..320f13b29 100644
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@@ -8,7 +8,7 @@ from scrapy import signals
 from scrapy.mail import MailSender
 from scrapy.exceptions import NotConfigured
 
-class StatsMailer(object):
+class StatsMailer:
 
     def __init__(self, stats, recipients, mail):
         self.stats = stats
diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index 198d4bbb0..56e5ad2d2 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -6,7 +6,7 @@ from scrapy import signals
 logger = logging.getLogger(__name__)
 
 
-class AutoThrottle(object):
+class AutoThrottle:
 
     def __init__(self, crawler):
         self.crawler = crawler
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 0903fd4f8..3e810992c 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -5,7 +5,7 @@ from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
 
 
-class CookieJar(object):
+class CookieJar:
     def __init__(self, policy=None, check_expired_frequency=10000):
         self.policy = policy or DefaultCookiePolicy()
         self.jar = _CookieJar(self.policy)
@@ -100,7 +100,7 @@ def potential_domain_matches(domain):
     return matches + ['.' + d for d in matches]
 
 
-class _DummyLock(object):
+class _DummyLock:
     def acquire(self):
         pass
 
@@ -108,7 +108,7 @@ class _DummyLock(object):
         pass
 
 
-class WrappedRequest(object):
+class WrappedRequest:
     """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class
 
     see http://docs.python.org/library/urllib2.html#urllib2.Request
@@ -178,7 +178,7 @@ class WrappedRequest(object):
         self.request.headers.appendlist(name, value)
 
 
-class WrappedResponse(object):
+class WrappedResponse:
 
     def __init__(self, response):
         self.response = response
diff --git a/scrapy/link.py b/scrapy/link.py
index a809c5ca4..7cb0765cc 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -6,7 +6,7 @@ its documentation in: docs/topics/link-extractors.rst
 """
 
 
-class Link(object):
+class Link:
     """Link objects represent an extracted link by the LinkExtractor."""
 
     __slots__ = ['url', 'text', 'fragment', 'nofollow']
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index bdeab3a75..6afe867b5 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -49,7 +49,7 @@ _matches = lambda url, regexs: any(r.search(url) for r in regexs)
 _is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
 
 
-class FilteringLinkExtractor(object):
+class FilteringLinkExtractor:
 
     _csstranslator = HTMLTranslator()
 
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index ab82e1915..fbac1dc59 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -27,7 +27,7 @@ def _nons(tag):
     return tag
 
 
-class LxmlParserLinkExtractor(object):
+class LxmlParserLinkExtractor:
     def __init__(self, tag="a", attr="href", process=None, unique=False,
                  strip=True, canonicalized=False):
         self.scan_tag = tag if callable(tag) else lambda t: t == tag
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 7cf67e29e..21c4fb376 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -25,7 +25,7 @@ def unbound_method(method):
     return method
 
 
-class ItemLoader(object):
+class ItemLoader:
 
     default_item_class = Item
     default_input_processor = Identity()
diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index 02c625acc..a7be65609 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -9,7 +9,7 @@ from scrapy.utils.misc import arg_to_iter
 from scrapy.loader.common import wrap_loader_context
 
 
-class MapCompose(object):
+class MapCompose:
 
     def __init__(self, *functions, **default_loader_context):
         self.functions = functions
@@ -36,7 +36,7 @@ class MapCompose(object):
         return values
 
 
-class Compose(object):
+class Compose:
 
     def __init__(self, *functions, **default_loader_context):
         self.functions = functions
@@ -61,7 +61,7 @@ class Compose(object):
         return value
 
 
-class TakeFirst(object):
+class TakeFirst:
 
     def __call__(self, values):
         for value in values:
@@ -69,13 +69,13 @@ class TakeFirst(object):
                 return value
 
 
-class Identity(object):
+class Identity:
 
     def __call__(self, values):
         return values
 
 
-class SelectJmes(object):
+class SelectJmes:
     """
         Query the input string for the jmespath (given at instantiation),
         and return the answer
@@ -95,7 +95,7 @@ class SelectJmes(object):
         return self.compiled_path.search(value)
 
 
-class Join(object):
+class Join:
 
     def __init__(self, separator=u' '):
         self.separator = separator
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 14cec44a6..219145f13 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -14,7 +14,7 @@ DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s"
 DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s"
 
 
-class LogFormatter(object):
+class LogFormatter:
     """Class for generating log messages for different actions.
 
     All methods must return a dictionary listing the parameters ``level``, ``msg``
diff --git a/scrapy/mail.py b/scrapy/mail.py
index b2a24a3db..9d186f4f3 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -27,7 +27,7 @@ def _to_bytes_or_none(text):
     return to_bytes(text)
 
 
-class MailSender(object):
+class MailSender:
     def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost',
             smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):
         self.smtphost = smtphost
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 53fa435bb..5040378ea 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -9,7 +9,7 @@ from scrapy.utils.defer import process_parallel, process_chain, process_chain_bo
 logger = logging.getLogger(__name__)
 
 
-class MiddlewareManager(object):
+class MiddlewareManager:
     """Base class for implementing middleware managers"""
 
     component_name = 'foo middleware'
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 9b7445755..101bf5fbc 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -37,7 +37,7 @@ class FileException(Exception):
     """General media error exception"""
 
 
-class FSFilesStore(object):
+class FSFilesStore:
     def __init__(self, basedir):
         if '://' in basedir:
             basedir = basedir.split('://', 1)[1]
@@ -75,7 +75,7 @@ class FSFilesStore(object):
             seen.add(dirname)
 
 
-class S3FilesStore(object):
+class S3FilesStore:
     AWS_ACCESS_KEY_ID = None
     AWS_SECRET_ACCESS_KEY = None
     AWS_ENDPOINT_URL = None
@@ -213,7 +213,7 @@ class S3FilesStore(object):
         return extra
 
 
-class GCSFilesStore(object):
+class GCSFilesStore:
 
     GCS_PROJECT_ID = None
 
@@ -259,7 +259,7 @@ class GCSFilesStore(object):
         )
 
 
-class FTPFilesStore(object):
+class FTPFilesStore:
 
     FTP_USERNAME = None
     FTP_PASSWORD = None
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index c174addf9..562d9ee32 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -14,11 +14,11 @@ from scrapy.utils.log import failure_to_exc_info
 logger = logging.getLogger(__name__)
 
 
-class MediaPipeline(object):
+class MediaPipeline:
 
     LOG_FAILED_RESULTS = True
 
-    class SpiderInfo(object):
+    class SpiderInfo:
         def __init__(self, spider):
             self.spider = spider
             self.downloading = set()
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 1afe58dab..e13d389ee 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -110,7 +110,7 @@ class ScrapyPriorityQueue:
         return sum(len(x) for x in self.queues.values()) if self.queues else 0
 
 
-class DownloaderInterface(object):
+class DownloaderInterface:
 
     def __init__(self, crawler):
         self.downloader = crawler.engine.downloader
@@ -129,8 +129,8 @@ class DownloaderInterface(object):
         return len(self.downloader.slots[slot].active)
 
 
-class DownloaderAwarePriorityQueue(object):
-    """ PriorityQueue which takes Downloader activity in account:
+class DownloaderAwarePriorityQueue:
+    """ PriorityQueue which takes Downloader activity into account:
     domains (slots) with the least amount of active downloads are dequeued
     first.
     """
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 64bf93e86..ad89d9d22 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -11,7 +11,7 @@ from scrapy.utils.misc import load_object
 from scrapy.utils.python import binary_is_text, to_bytes, to_unicode
 
 
-class ResponseTypes(object):
+class ResponseTypes:
 
     CLASSES = {
         'text/html': 'scrapy.http.HtmlResponse',
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index b6133619c..b9a13c018 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -28,7 +28,7 @@ def get_settings_priority(priority):
         return priority
 
 
-class SettingsAttribute(object):
+class SettingsAttribute:
 
     """Class for storing data related to settings attributes.
 
diff --git a/scrapy/shell.py b/scrapy/shell.py
index a5e140484..08ce89481 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -24,7 +24,7 @@ from scrapy.utils.conf import get_config
 from scrapy.utils.console import DEFAULT_PYTHON_SHELLS
 
 
-class Shell(object):
+class Shell:
 
     relevant_classes = (Crawler, Spider, Request, Response, BaseItem,
                         Settings)
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index 481d97e9a..54eb7cfa3 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -2,7 +2,7 @@ from pydispatch import dispatcher
 from scrapy.utils import signal as _signal
 
 
-class SignalManager(object):
+class SignalManager:
 
     def __init__(self, sender=dispatcher.Anonymous):
         self.sender = sender
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 048e84e4f..3be5aaec5 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -11,7 +11,7 @@ from scrapy.utils.spider import iter_spider_classes
 
 
 @implementer(ISpiderLoader)
-class SpiderLoader(object):
+class SpiderLoader:
     """
     SpiderLoader is a class which locates and loads spiders
     in a Scrapy project.
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 34a87f2df..fa7f5bef9 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -11,7 +11,7 @@ from scrapy.http import Request
 logger = logging.getLogger(__name__)
 
 
-class DepthMiddleware(object):
+class DepthMiddleware:
 
     def __init__(self, maxdepth, stats, verbose_stats=False, prio=1):
         self.maxdepth = maxdepth
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index def697c2b..375042340 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -18,7 +18,7 @@ class HttpError(IgnoreRequest):
         super(HttpError, self).__init__(*args, **kwargs)
 
 
-class HttpErrorMiddleware(object):
+class HttpErrorMiddleware:
 
     @classmethod
     def from_crawler(cls, crawler):
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 2fab572e6..a006f3177 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -14,7 +14,7 @@ from scrapy.utils.httpobj import urlparse_cached
 logger = logging.getLogger(__name__)
 
 
-class OffsiteMiddleware(object):
+class OffsiteMiddleware:
 
     def __init__(self, stats):
         self.stats = stats
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index dce2b3598..3784de885 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -28,7 +28,7 @@ POLICY_UNSAFE_URL = "unsafe-url"
 POLICY_SCRAPY_DEFAULT = "scrapy-default"
 
 
-class ReferrerPolicy(object):
+class ReferrerPolicy:
 
     NOREFERRER_SCHEMES = LOCAL_SCHEMES
 
@@ -284,7 +284,7 @@ def _load_policy_class(policy, warning_only=False):
                 return None
 
 
-class RefererMiddleware(object):
+class RefererMiddleware:
 
     def __init__(self, settings=None):
         self.default_policy = DefaultReferrerPolicy
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index a904635d8..5be1f80cb 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -12,7 +12,7 @@ from scrapy.exceptions import NotConfigured
 logger = logging.getLogger(__name__)
 
 
-class UrlLengthMiddleware(object):
+class UrlLengthMiddleware:
 
     def __init__(self, maxlength):
         self.maxlength = maxlength
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index a2c364c0e..d76a96451 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -34,7 +34,7 @@ def _get_method(method, spider):
 _default_link_extractor = LinkExtractor()
 
 
-class Rule(object):
+class Rule:
 
     def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None,
                  process_links=None, process_request=None, errback=None):
diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py
index f0bfaed34..579c60180 100644
--- a/scrapy/statscollectors.py
+++ b/scrapy/statscollectors.py
@@ -7,7 +7,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 
-class StatsCollector(object):
+class StatsCollector:
 
     def __init__(self, crawler):
         self._dump = crawler.settings.getbool('STATS_DUMP')
diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
index 97b5db2e1..b3e58ff94 100644
--- a/scrapy/templates/project/module/middlewares.py.tmpl
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -8,7 +8,7 @@
 from scrapy import signals
 
 
-class ${ProjectName}SpiderMiddleware(object):
+class ${ProjectName}SpiderMiddleware:
     # Not all methods need to be defined. If a method is not defined,
     # scrapy acts as if the spider middleware does not modify the
     # passed objects.
@@ -56,7 +56,7 @@ class ${ProjectName}SpiderMiddleware(object):
         spider.logger.info('Spider opened: %s' % spider.name)
 
 
-class ${ProjectName}DownloaderMiddleware(object):
+class ${ProjectName}DownloaderMiddleware:
     # Not all methods need to be defined. If a method is not defined,
     # scrapy acts as if the downloader middleware does not modify the
     # passed objects.
diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl
index fb641d447..4876526a9 100644
--- a/scrapy/templates/project/module/pipelines.py.tmpl
+++ b/scrapy/templates/project/module/pipelines.py.tmpl
@@ -6,6 +6,6 @@
 # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
 
 
-class ${ProjectName}Pipeline(object):
+class ${ProjectName}Pipeline:
     def process_item(self, item, spider):
         return item
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 175f92d77..f59f4cc55 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -109,7 +109,7 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
             return None  # key is not weak-referenceable, it's not cached
 
 
-class SequenceExclude(object):
+class SequenceExclude:
     """Object to test if an item is NOT within some sequence."""
 
     def __init__(self, seq):
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index 2d3db431d..69334a918 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -144,7 +144,7 @@ def method_is_overridden(subclass, base_class, method_name):
     Return True if a method named ``method_name`` of a ``base_class``
     is overridden in a ``subclass``.
 
-    >>> class Base(object):
+    >>> class Base:
     ...     def foo(self):
     ...         pass
     >>> class Sub1(Base):
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 7849174fb..b71419111 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -52,7 +52,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
         yield xs.xpath(selxpath)[0]
 
 
-class _StreamReader(object):
+class _StreamReader:
 
     def __init__(self, obj):
         self._ptr = 0
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index afef2c93f..5998dc33b 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -152,7 +152,7 @@ def log_scrapy_info(settings):
     logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
 
 
-class StreamLogger(object):
+class StreamLogger:
     """Fake file-like stream object that redirects writes to a logger instance
 
     Taken from:
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index e95a4648e..3d02d9478 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -223,7 +223,7 @@ def get_spec(func):
     >>> get_spec(re.match)
     (['pattern', 'string'], {'flags': 0})
 
-    >>> class Test(object):
+    >>> class Test:
     ...     def __call__(self, val):
     ...         pass
     ...     def method(self, val, flags=0):
@@ -272,7 +272,7 @@ def equal_attributes(obj1, obj2, attributes):
     return True
 
 
-class WeakKeyCache(object):
+class WeakKeyCache:
 
     def __init__(self, default_factory):
         self.default_factory = default_factory
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 17d6b2857..5308812d6 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -24,7 +24,7 @@ def listen_tcp(portrange, host, factory):
                 raise
 
 
-class CallLaterOnce(object):
+class CallLaterOnce:
     """Schedule a function to be called in the next reactor loop, but only if
     it hasn't been already scheduled since the last time it ran.
     """
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index 2f10cf4de..a57a0c291 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -10,7 +10,7 @@ from urllib.parse import urljoin
 import lxml.etree
 
 
-class Sitemap(object):
+class Sitemap:
     """Class to parse Sitemap (type=urlset) and Sitemap Index
     (type=sitemapindex) files"""
 
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index 37803b287..a63c9a942 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -4,7 +4,7 @@ import os
 from twisted.internet import defer, protocol
 
 
-class ProcessTest(object):
+class ProcessTest:
 
     command = None
     prefix = [sys.executable, '-m', 'scrapy.cmdline']
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index 9e1598805..66930ad2c 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -3,7 +3,7 @@ from urllib.parse import urljoin
 from twisted.web import server, resource, static, util
 
 
-class SiteTest(object):
+class SiteTest:
 
     def setUp(self):
         from twisted.internet import reactor
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index 4842b95df..baed5c536 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -19,9 +19,8 @@ NoneType = type(None)
 live_refs = defaultdict(weakref.WeakKeyDictionary)
 
 
-class object_ref(object):
-    """Inherit from this class (instead of object) to a keep a record of live
-    instances"""
+class object_ref:
+    """Inherit from this class to a keep a record of live instances"""
 
     __slots__ = ()
 
diff --git a/tests/pipelines.py b/tests/pipelines.py
index de4894c32..cf677cc17 100644
--- a/tests/pipelines.py
+++ b/tests/pipelines.py
@@ -3,7 +3,7 @@ Some pipelines used for testing
 """
 
 
-class ZeroDivisionErrorPipeline(object):
+class ZeroDivisionErrorPipeline:
 
     def open_spider(self, spider):
         a = 1 / 0
@@ -12,7 +12,7 @@ class ZeroDivisionErrorPipeline(object):
         return item
 
 
-class ProcessWithZeroDivisionErrorPipiline(object):
+class ProcessWithZeroDivisionErrorPipiline:
 
     def process_item(self, item, spider):
         1 / 0
diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py
index c64e87d81..6504b4d2c 100644
--- a/tests/test_cmdline/extensions.py
+++ b/tests/test_cmdline/extensions.py
@@ -1,7 +1,7 @@
 """A test extension used to check the settings loading order"""
 
 
-class TestExtension(object):
+class TestExtension:
 
     def __init__(self, settings):
         settings.set('TEST1', "%s + %s" % (settings['TEST1'], 'started'))
@@ -11,5 +11,5 @@ class TestExtension(object):
         return cls(crawler.settings)
 
 
-class DummyExtension(object):
+class DummyExtension:
     pass
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
index ce916f699..bd1f9cd8c 100644
--- a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
@@ -1,4 +1,4 @@
-class TestSpiderPipeline(object):
+class TestSpiderPipeline:
 
     def open_spider(self, spider):
         pass
@@ -7,7 +7,7 @@ class TestSpiderPipeline(object):
         return item
 
 
-class TestSpiderExceptionPipeline(object):
+class TestSpiderExceptionPipeline:
 
     def open_spider(self, spider):
         raise Exception('exception')
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 5bf92b71a..85a24d0bc 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -89,7 +89,7 @@ class MyBadCrawlSpider(CrawlSpider):
             f.write("""
 import logging
 
-class MyPipeline(object):
+class MyPipeline:
     component_name = 'my_pipeline'
 
     def process_item(self, item, spider):
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 11d41c1fe..d1ce80f9d 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -25,7 +25,7 @@ class TestItem(Item):
     url = Field()
 
 
-class ResponseMock(object):
+class ResponseMock:
     url = 'http://scrapy.org'
 
 
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 37a069611..169e763f0 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -126,7 +126,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
         self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0)
 
 
-class SpiderLoaderWithWrongInterface(object):
+class SpiderLoaderWithWrongInterface:
 
     def unneeded_method(self):
         pass
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 3dd4f2351..a9190c62b 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -106,7 +106,7 @@ class ResponseFromProcessRequestTest(ManagerTestCase):
     def test_download_func_not_called(self):
         resp = Response('http://example.com/index.html')
 
-        class ResponseMiddleware(object):
+        class ResponseMiddleware:
             def process_request(self, request, spider):
                 return resp
 
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 9e24d86dd..ea0e664be 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -35,7 +35,7 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter):
         return df
 
 
-class DirectDupeFilter(object):
+class DirectDupeFilter:
     method = 'n/a'
 
 
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 25dee7c1f..5b7a4e676 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -96,7 +96,7 @@ def start_test_site(debug=False):
     return port
 
 
-class CrawlerRun(object):
+class CrawlerRun:
     """A class to run the crawler and keep track of events occurred"""
 
     def __init__(self, spider_class):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 08e8dfc41..c5589e52f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -373,7 +373,7 @@ class StdoutFeedStorageTest(unittest.TestCase):
         self.assertEqual(out.getvalue(), b"content")
 
 
-class FromCrawlerMixin(object):
+class FromCrawlerMixin:
     init_with_crawler = False
 
     @classmethod
diff --git a/tests/test_item.py b/tests/test_item.py
index f70632d57..4017f6e84 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -213,7 +213,7 @@ class ItemTest(unittest.TestCase):
         class B(A):
             pass
 
-        class C(object):
+        class C:
             fields = {'load': Field(default='C')}
             not_allowed = Field(default='not_allowed')
             save = Field(default='C')
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 579a85ff6..701d568dc 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -456,7 +456,7 @@ class BasicItemLoaderTest(unittest.TestCase):
                           [u'marta', u'other'], Compose(float))
 
 
-class InitializationTestMixin(object):
+class InitializationTestMixin:
 
     item_class = None
 
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index bf9fbe5e4..cd6cb8016 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -170,7 +170,7 @@ class SkipMessagesLogFormatter(LogFormatter):
         return None
 
 
-class DropSomeItemsPipeline(object):
+class DropSomeItemsPipeline:
     drop = True
 
     def process_item(self, item, spider):
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index ebf817c7e..3af514bb0 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -5,7 +5,7 @@ from scrapy.exceptions import NotConfigured
 from scrapy.middleware import MiddlewareManager
 
 
-class M1(object):
+class M1:
 
     def open_spider(self, spider):
         pass
@@ -17,7 +17,7 @@ class M1(object):
         pass
 
 
-class M2(object):
+class M2:
 
     def open_spider(self, spider):
         pass
@@ -28,13 +28,13 @@ class M2(object):
     pass
 
 
-class M3(object):
+class M3:
 
     def process(self, response, request, spider):
         pass
 
 
-class MOff(object):
+class MOff:
 
     def open_spider(self, spider):
         pass
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index a5cdc0de0..a3ddd50f4 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -8,7 +8,7 @@ from tests.spiders import MockServerSpider
 from tests.mockserver import MockServer
 
 
-class InjectArgumentsDownloaderMiddleware(object):
+class InjectArgumentsDownloaderMiddleware:
     """
     Make sure downloader middlewares are able to update the keyword arguments
     """
@@ -23,7 +23,7 @@ class InjectArgumentsDownloaderMiddleware(object):
         return response
 
 
-class InjectArgumentsSpiderMiddleware(object):
+class InjectArgumentsSpiderMiddleware:
     """
     Make sure spider middlewares are able to update the keyword arguments
     """
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 13c297084..00568aee9 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -20,7 +20,7 @@ MockEngine = collections.namedtuple('MockEngine', ['downloader'])
 MockSlot = collections.namedtuple('MockSlot', ['active'])
 
 
-class MockDownloader(object):
+class MockDownloader:
     def __init__(self):
         self.slots = dict()
 
@@ -57,7 +57,7 @@ class MockCrawler(Crawler):
         self.engine = MockEngine(downloader=MockDownloader())
 
 
-class SchedulerHandler(object):
+class SchedulerHandler:
     priority_queue_cls = None
     jobdir = None
 
@@ -245,7 +245,7 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots):
     return True
 
 
-class DownloaderAwareSchedulerTestMixin(object):
+class DownloaderAwareSchedulerTestMixin:
     priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
     reopen = False
 
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 7cc17600c..4c6ede70b 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -47,7 +47,7 @@ class TestRefererMiddleware(TestCase):
             self.assertEqual(out[0].headers.get('Referer'), referrer)
 
 
-class MixinDefault(object):
+class MixinDefault:
     """
     Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
 
@@ -72,7 +72,7 @@ class MixinDefault(object):
     ]
 
 
-class MixinNoReferrer(object):
+class MixinNoReferrer:
     scenarii = [
         ('https://example.com/page.html',       'https://example.com/', None),
         ('http://www.example.com/',             'https://scrapy.org/',  None),
@@ -82,7 +82,7 @@ class MixinNoReferrer(object):
     ]
 
 
-class MixinNoReferrerWhenDowngrade(object):
+class MixinNoReferrerWhenDowngrade:
     scenarii = [
         # TLS to TLS: send non-empty referrer
         ('https://example.com/page.html',       'https://not.example.com/', b'https://example.com/page.html'),
@@ -111,7 +111,7 @@ class MixinNoReferrerWhenDowngrade(object):
     ]
 
 
-class MixinSameOrigin(object):
+class MixinSameOrigin:
     scenarii = [
         # Same origin (protocol, host, port): send referrer
         ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
@@ -144,7 +144,7 @@ class MixinSameOrigin(object):
     ]
 
 
-class MixinOrigin(object):
+class MixinOrigin:
     scenarii = [
         # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)
         ('https://example.com/page.html',   'https://example.com/not-page.html',    b'https://example.com/'),
@@ -157,7 +157,7 @@ class MixinOrigin(object):
     ]
 
 
-class MixinStrictOrigin(object):
+class MixinStrictOrigin:
     scenarii = [
         # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades
         ('https://example.com/page.html',   'https://example.com/not-page.html',    b'https://example.com/'),
@@ -176,7 +176,7 @@ class MixinStrictOrigin(object):
     ]
 
 
-class MixinOriginWhenCrossOrigin(object):
+class MixinOriginWhenCrossOrigin:
     scenarii = [
         # Same origin (protocol, host, port): send referrer
         ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
@@ -211,7 +211,7 @@ class MixinOriginWhenCrossOrigin(object):
     ]
 
 
-class MixinStrictOriginWhenCrossOrigin(object):
+class MixinStrictOriginWhenCrossOrigin:
     scenarii = [
         # Same origin (protocol, host, port): send referrer
         ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
@@ -255,7 +255,7 @@ class MixinStrictOriginWhenCrossOrigin(object):
     ]
 
 
-class MixinUnsafeUrl(object):
+class MixinUnsafeUrl:
     scenarii = [
         # TLS to TLS: send referrer
         ('https://example.com/sekrit.html',     'http://not.example.com/',      b'https://example.com/sekrit.html'),
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index f0f3dd4c6..5ad8035f7 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -36,7 +36,7 @@ def nonserializable_object_test(self):
         self.assertRaises(ValueError, q.push, lambda x: x)
     else:
         # Use a different unpickleable object
-        class A(object):
+        class A:
             pass
 
         a = A()
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index b3a90d314..b17e17f2f 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -11,7 +11,7 @@ class MyWarning(UserWarning):
     pass
 
 
-class SomeBaseClass(object):
+class SomeBaseClass:
     pass
 
 
@@ -155,7 +155,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
             class OutdatedUserClass1a(DeprecatedName):
                 pass
 
-            class UnrelatedClass(object):
+            class UnrelatedClass:
                 pass
 
             class OldStyleClass:
@@ -191,7 +191,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
             class OutdatedUserClass2a(DeprecatedName):
                 pass
 
-            class UnrelatedClass(object):
+            class UnrelatedClass:
                 pass
 
             class OldStyleClass:
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index ec5b4c596..8cb8df15b 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -73,7 +73,7 @@ class ToBytesTest(unittest.TestCase):
 
 class MemoizedMethodTest(unittest.TestCase):
     def test_memoizemethod_noargs(self):
-        class A(object):
+        class A:
 
             @memoizemethod_noargs
             def cached(self):
@@ -153,7 +153,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
 
     def test_weakkeycache(self):
-        class _Weakme(object):
+        class _Weakme:
             pass
 
         _values = count()
@@ -176,14 +176,14 @@ class UtilsPythonTestCase(unittest.TestCase):
         def f2(a, b=None, c=None):
             pass
 
-        class A(object):
+        class A:
             def __init__(self, a, b, c):
                 pass
 
             def method(self, a, b, c):
                 pass
 
-        class Callable(object):
+        class Callable:
 
             def __call__(self, a, b, c):
                 pass
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 06d9c004c..c7572f02c 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -126,7 +126,7 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertRaises(ValueError, request_to_dict, r)
 
 
-class TestSpiderMixin(object):
+class TestSpiderMixin:
     def __mixin_callback(self, response):
         pass
 

From 533131a30fa944688dc54dd82a581739d1ed247c Mon Sep 17 00:00:00 2001
From: sakshamb2113 <44064539+sakshamb2113@users.noreply.github.com>
Date: Tue, 17 Mar 2020 14:42:49 +0530
Subject: [PATCH 2702/2974] Remove Guppy-specific code and documentation
 (#4343)

---
 docs/topics/leaks.rst       | 64 +++----------------------------------
 scrapy/extensions/telnet.py |  6 ----
 2 files changed, 5 insertions(+), 65 deletions(-)

diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 4ee447065..ceb708c7e 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -17,8 +17,8 @@ what is known as a "memory leak".
 
 To help debugging memory leaks, Scrapy provides a built-in mechanism for
 tracking objects references called :ref:`trackref <topics-leaks-trackrefs>`,
-and you can also use a third-party library called :ref:`Guppy
-<topics-leaks-guppy>` for more advanced memory debugging (see below for more
+and you can also use a third-party library called :ref:`muppy
+<topics-leaks-muppy>` for more advanced memory debugging (see below for more
 info). Both mechanisms must be used from the :ref:`Telnet Console
 <topics-telnetconsole>`.
 
@@ -193,9 +193,9 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
     ``None`` if none is found. Use :func:`print_live_refs` first to get a list
     of all tracked live objects per class name.
 
-.. _topics-leaks-guppy:
+.. _topics-leaks-muppy:
 
-Debugging memory leaks with Guppy
+Debugging memory leaks with muppy
 =================================
 
 ``trackref`` provides a very convenient mechanism for tracking down memory
@@ -203,63 +203,9 @@ leaks, but it only keeps track of the objects that are more likely to cause
 memory leaks (Requests, Responses, Items, and Selectors). However, there are
 other cases where the memory leaks could come from other (more or less obscure)
 objects. If this is your case, and you can't find your leaks using ``trackref``,
-you still have another resource: the `Guppy library`_.
-If you're using Python3, see :ref:`topics-leaks-muppy`.
+you still have another resource: the muppy library.
 
-.. _Guppy library: https://pypi.org/project/guppy/
 
-If you use ``pip``, you can install Guppy with the following command::
-
-    pip install guppy
-
-The telnet console also comes with a built-in shortcut (``hpy``) for accessing
-Guppy heap objects. Here's an example to view all Python objects available in
-the heap using Guppy:
-
->>> x = hpy.heap()
->>> x.bytype
-Partition of a set of 297033 objects. Total size = 52587824 bytes.
- Index  Count   %     Size   % Cumulative  % Type
-     0  22307   8 16423880  31  16423880  31 dict
-     1 122285  41 12441544  24  28865424  55 str
-     2  68346  23  5966696  11  34832120  66 tuple
-     3    227   0  5836528  11  40668648  77 unicode
-     4   2461   1  2222272   4  42890920  82 type
-     5  16870   6  2024400   4  44915320  85 function
-     6  13949   5  1673880   3  46589200  89 types.CodeType
-     7  13422   5  1653104   3  48242304  92 list
-     8   3735   1  1173680   2  49415984  94 _sre.SRE_Pattern
-     9   1209   0   456936   1  49872920  95 scrapy.http.headers.Headers
-<1676 more rows. Type e.g. '_.more' to view.>
-
-You can see that most space is used by dicts. Then, if you want to see from
-which attribute those dicts are referenced, you could do:
-
->>> x.bytype[0].byvia
-Partition of a set of 22307 objects. Total size = 16423880 bytes.
- Index  Count   %     Size   % Cumulative  % Referred Via:
-     0  10982  49  9416336  57   9416336  57 '.__dict__'
-     1   1820   8  2681504  16  12097840  74 '.__dict__', '.func_globals'
-     2   3097  14  1122904   7  13220744  80
-     3    990   4   277200   2  13497944  82 "['cookies']"
-     4    987   4   276360   2  13774304  84 "['cache']"
-     5    985   4   275800   2  14050104  86 "['meta']"
-     6    897   4   251160   2  14301264  87 '[2]'
-     7      1   0   196888   1  14498152  88 "['moduleDict']", "['modules']"
-     8    672   3   188160   1  14686312  89 "['cb_kwargs']"
-     9     27   0   155016   1  14841328  90 '[1]'
-<333 more rows. Type e.g. '_.more' to view.>
-
-As you can see, the Guppy module is very powerful but also requires some deep
-knowledge about Python internals. For more info about Guppy, refer to the
-`Guppy documentation`_.
-
-.. _Guppy documentation: http://guppy-pe.sourceforge.net/
-
-.. _topics-leaks-muppy:
-
-Debugging memory leaks with muppy
-=================================
 You can use muppy from `Pympler`_.
 
 .. _Pympler: https://pypi.org/project/Pympler/
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 26b214ee2..04ffd7235 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -26,11 +26,6 @@ from scrapy.utils.engine import print_engine_status
 from scrapy.utils.reactor import listen_tcp
 from scrapy.utils.decorators import defers
 
-try:
-    import guppy
-    hpy = guppy.hpy()
-except ImportError:
-    hpy = None
 
 logger = logging.getLogger(__name__)
 
@@ -110,7 +105,6 @@ class TelnetConsole(protocol.ServerFactory):
             'est': lambda: print_engine_status(self.crawler.engine),
             'p': pprint.pprint,
             'prefs': print_live_refs,
-            'hpy': hpy,
             'help': "This is Scrapy telnet console. For more info see: "
                     "https://docs.scrapy.org/en/latest/topics/telnetconsole.html",
         }

From 9ab45325ff6746f5c5325940fbbff1e020dd999a Mon Sep 17 00:00:00 2001
From: "Matsievskiy S.V" <matsievskiysv@gmail.com>
Date: Tue, 17 Mar 2020 18:45:00 +0300
Subject: [PATCH 2703/2974] edit zsh completion

- Fix bug introduced in https://github.com/scrapy/scrapy/pull/4291
- Enforce `[command] [options] [arguments]` syntax. Do not allow options after arguments
- Exclude already used option aliases from completion list
---
 extras/scrapy_zsh_completion | 76 ++++++++++++++++++------------------
 1 file changed, 38 insertions(+), 38 deletions(-)

diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion
index 33f46eda8..e2f2dc82b 100644
--- a/extras/scrapy_zsh_completion
+++ b/extras/scrapy_zsh_completion
@@ -14,40 +14,40 @@ _scrapy() {
 	    ;;
 	args)
 	    case $words[1] in
-		bench)
+		(bench)
 		    _scrapy_glb_opts
 		    ;;
-		fetch)
+		(fetch)
 		    local options=(
 			'--headers[print response HTTP headers instead of body]'
 			'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
-			'--spider[use this spider]:spider:_scrapy_spiders'
+			'--spider=[use this spider]:spider:_scrapy_spiders'
 			'1::URL:_httpie_urls'
 		    )
 		    _scrapy_glb_opts $options
 		    ;;
-		genspider)
+		(genspider)
 		    local options=(
-			{-l,--list}'[List available templates]'
-			{-e,--edit}'[Edit spider after creating it]'
+			{'(--list)-l','(-l)--list'}'[List available templates]'
+			{'(--edit)-e','(-e)--edit'}'[Edit spider after creating it]'
 			'--force[If the spider already exists, overwrite it with the template]'
-			{-d,--dump=}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
-			{-t,--template=}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
+			{'(--dump)-d','(-d)--dump='}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
+			{'(--template)-t','(-t)--template='}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
 			'1:name:(NAME)'
 			'2:domain:_httpie_urls'
 		    )
 		    _scrapy_glb_opts $options
 		    ;;
-		runspider)
+		(runspider)
 		    local options=(
-			{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
-			{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
+			{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
+			{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
 			'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
 			'1:spider file:_files -g \*.py'
 		    )
 		    _scrapy_glb_opts $options
 		    ;;
-		settings)
+		(settings)
 		    local options=(
 			'--get=[print raw setting value]:option:(SETTING)'
 			'--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)'
@@ -57,77 +57,77 @@ _scrapy() {
 		    )
 		    _scrapy_glb_opts $options
 		    ;;
-		shell)
+		(shell)
 		    local options=(
 			'-c[evaluate the code in the shell, print the result and exit]:code:(CODE)'
 			'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
-			'--spider[use this spider]:spider:_scrapy_spiders'
+			'--spider=[use this spider]:spider:_scrapy_spiders'
 			'::file:_files -g \*.html'
 			'::URL:_httpie_urls'
 		    )
 		    _scrapy_glb_opts $options
 		    ;;
-		startproject)
+		(startproject)
 		    local options=(
 			'1:name:(NAME)'
 			'2:dir:_dir_list'
 		    )
 		    _scrapy_glb_opts $options
 		    ;;
-		version)
+		(version)
 		    local options=(
-			{-v,--verbose}'[also display twisted/python/platform info (useful for bug reports)]'
+			{'(--verbose)-v','(-v)--verbose'}'[also display twisted/python/platform info (useful for bug reports)]'
 		    )
 		    _scrapy_glb_opts $options
 		    ;;
-		view)
+		(view)
 		    local options=(
 			'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
-			'--spider[use this spider]:spider:_scrapy_spiders'
+			'--spider=[use this spider]:spider:_scrapy_spiders'
 			'1:URL:_httpie_urls'
 		    )
 		    _scrapy_glb_opts $options
 		    ;;
-		check)
+		(check)
 		    local options=(
-			'(- 1 *)'{-l,--list}'[only list contracts, without checking them]'
-			{-v,--verbose}'[print contract tests for all spiders]'
+			{'(--list)-l','(-l)--list'}'[only list contracts, without checking them]'
+			{'(--verbose)-v','(-v)--verbose'}'[print contract tests for all spiders]'
 			'1:spider:_scrapy_spiders'
 		    )
 		    _scrapy_glb_opts $options
 		    ;;
-		crawl)
+		(crawl)
 		    local options=(
-			{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
-			{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
+			{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
+			{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
 			'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
 			'1:spider:_scrapy_spiders'
 		    )
 		    _scrapy_glb_opts $options
 		    ;;
-		edit)
+		(edit)
 		    local options=(
-			'1:spider:_scrapy_spiders'
+		      '1:spider:_scrapy_spiders'
 		    )
 		    _scrapy_glb_opts $options
 		    ;;
-		list)
+		(list)
 		    _scrapy_glb_opts
 		    ;;
-		parse)
+		(parse)
 		    local options=(
 			'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
-			'--spider[use this spider without looking for one]:spider:_scrapy_spiders'
+			'--spider=[use this spider without looking for one]:spider:_scrapy_spiders'
 			'--pipelines[process items through pipelines]'
 			"--nolinks[don't show links to follow (extracted requests)]"
 			"--noitems[don't show scraped items]"
 			'--nocolour[avoid using pygments to colorize the output]'
-			{-r,--rules}'[use CrawlSpider rules to discover the callback]'
-			{-c,--callback=}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
-			{-m,--meta=}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
+			{'(--rules)-r','(-r)--rules'}'[use CrawlSpider rules to discover the callback]'
+			{'(--callback)-c','(-c)--callback'}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
+			{'(--meta)-m','(-m)--meta='}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
 			'--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)'
-			{-d,--depth=}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
-			{-v,--verbose}'[print each depth level one by one]'
+			{'(--depth)-d','(-d)--depth='}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
+			{'(--verbose)-v','(-v)--verbose'}'[print each depth level one by one]'
 			'1:URL:_httpie_urls'
 		    )
 		    _scrapy_glb_opts $options
@@ -162,7 +162,7 @@ _scrapy_cmds() {
     if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
 	commands=(${commands[@]} ${project_commands[@]})
     fi
-    _describe -t common-commands 'common commands' commands
+    _describe -t common-commands 'common commands' commands && ret=0
 }
 
 _scrapy_glb_opts() {
@@ -172,13 +172,13 @@ _scrapy_glb_opts() {
 	'(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files'
 	'--pidfile=[write process ID to FILE]:file:_files'
 	'--profile=[write python cProfile stats to FILE]:file:_files'
-	'(--nolog)'{-L,--loglevel=}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
+	{'(--loglevel --nolog)-L','(-L --nolog)--loglevel='}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
 	'(-L --loglevel --logfile)--nolog[disable logging completely]'
 	'--pdb[enable pdb on failure]'
 	'*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)'
     )
     options=(${options[@]} "$@")
-    _arguments $options
+    _arguments -A "-*" $options && ret=0
 }
 
 _httpie_urls() {

From 6c747953f97d60e5668cff1af3f091979ceaaa83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 18 Mar 2020 18:33:41 +0100
Subject: [PATCH 2704/2974] Cover 2.0.1 in the release notes (#4437)

---
 docs/news.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index dd5e00223..e9b7140cd 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,22 @@
 Release notes
 =============
 
+.. _release-2.0.1:
+
+Scrapy 2.0.1 (2020-03-18)
+-------------------------
+
+*   :meth:`Response.follow_all <scrapy.http.Response.follow_all>` now supports
+    an empty URL iterable as input (:issue:`4408`, :issue:`4420`)
+
+*   Removed top-level :mod:`~twisted.internet.reactor` imports to prevent
+    errors about the wrong Twisted reactor being installed when setting a
+    different Twisted reactor using :setting:`TWISTED_REACTOR` (:issue:`4401`,
+    :issue:`4406`)
+
+*   Fixed tests (:issue:`4422`)
+
+
 .. _release-2.0.0:
 
 Scrapy 2.0.0 (2020-03-03)

From ca08e04198b94bd9583704f86316b57af3408adc Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Fri, 20 Mar 2020 02:31:35 +0530
Subject: [PATCH 2705/2974] [docs] update redirect links python2 -> python3

---
 docs/topics/downloader-middleware.rst |  5 ++---
 docs/topics/email.rst                 |  2 +-
 docs/topics/exporters.rst             |  8 ++++----
 docs/topics/extensions.rst            |  2 +-
 docs/topics/items.rst                 |  6 +++---
 docs/topics/logging.rst               | 16 ++++++++--------
 docs/topics/request-response.rst      | 10 +++++-----
 docs/topics/selectors.rst             |  2 +-
 docs/topics/settings.rst              |  6 +++---
 docs/topics/spider-middleware.rst     |  6 +++---
 10 files changed, 31 insertions(+), 32 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 73648994d..61a3806fb 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -739,7 +739,7 @@ HttpProxyMiddleware
    This middleware sets the HTTP proxy to use for requests, by setting the
    ``proxy`` meta value for :class:`~scrapy.http.Request` objects.
 
-   Like the Python standard library modules `urllib`_ and `urllib2`_, it obeys
+   Like the Python standard library module `urllib.request`_, it obeys
    the following environment variables:
 
    * ``http_proxy``
@@ -751,8 +751,7 @@ HttpProxyMiddleware
    Keep in mind this value will take precedence over ``http_proxy``/``https_proxy``
    environment variables, and it will also ignore ``no_proxy`` environment variable.
 
-.. _urllib: https://docs.python.org/2/library/urllib.html
-.. _urllib2: https://docs.python.org/2/library/urllib2.html
+.. _urllib.request: https://docs.python.org/3/library/urllib.request.html
 
 RedirectMiddleware
 ------------------
diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index 72bf52227..aed3deb2e 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -15,7 +15,7 @@ IO of the crawler. It also provides a simple API for sending attachments and
 it's very easy to configure, with a few :ref:`settings
 <topics-email-settings>`.
 
-.. _smtplib: https://docs.python.org/2/library/smtplib.html
+.. _smtplib: https://docs.python.org/3/library/smtplib.html
 
 Quick example
 =============
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index e52682690..4ba8714bd 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -320,7 +320,7 @@ CsvItemExporter
       Color TV,1200
       DVD player,200
 
-.. _csv.writer: https://docs.python.org/2/library/csv.html#csv.writer
+.. _csv.writer: https://docs.python.org/3/library/csv.html#csv.writer
 
 PickleItemExporter
 ------------------
@@ -342,7 +342,7 @@ PickleItemExporter
 
    Pickle isn't a human readable format, so no output examples are provided.
 
-.. _pickle module documentation: https://docs.python.org/2/library/pickle.html
+.. _pickle module documentation: https://docs.python.org/3/library/pickle.html
 
 PprintItemExporter
 ------------------
@@ -393,7 +393,7 @@ JsonItemExporter
       stream-friendly format, consider using :class:`JsonLinesItemExporter`
       instead, or splitting the output in multiple chunks.
 
-.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
+.. _JSONEncoder: https://docs.python.org/3/library/json.html#json.JSONEncoder
 
 JsonLinesItemExporter
 ---------------------
@@ -417,7 +417,7 @@ JsonLinesItemExporter
    Unlike the one produced by :class:`JsonItemExporter`, the format produced by
    this exporter is well suited for serializing large amounts of data.
 
-.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
+.. _JSONEncoder: https://docs.python.org/3/library/json.html#json.JSONEncoder
 
 MarshalItemExporter
 -------------------
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 94fd2e36e..f57e37e6f 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -372,5 +372,5 @@ For more info see `Debugging in Python`_.
 
 This extension only works on POSIX-compliant platforms (i.e. not Windows).
 
-.. _Python debugger: https://docs.python.org/2/library/pdb.html
+.. _Python debugger: https://docs.python.org/3/library/pdb.html
 .. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 44643cb67..36731571e 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -24,7 +24,7 @@ serialization can be customized using Item fields metadata, :mod:`trackref`
 tracks Item instances to help find memory leaks
 (see :ref:`topics-leaks-trackrefs`), etc.
 
-.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
+.. _dictionary-like: https://docs.python.org/3/library/stdtypes.html#dict
 
 .. _topics-items-declaring:
 
@@ -249,7 +249,7 @@ Item objects
         :class:`Field` objects used in the :ref:`Item declaration
         <topics-items-declaring>`.
 
-.. _dict API: https://docs.python.org/2/library/stdtypes.html#dict
+.. _dict API: https://docs.python.org/3/library/stdtypes.html#dict
 
 Field objects
 =============
@@ -262,7 +262,7 @@ Field objects
     to support the :ref:`item declaration syntax <topics-items-declaring>`
     based on class attributes.
 
-.. _dict: https://docs.python.org/2/library/stdtypes.html#dict
+.. _dict: https://docs.python.org/3/library/stdtypes.html#dict
 
 
 Other classes related to Item
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index d4d22d889..a85e1a769 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -83,10 +83,10 @@ path::
 
 .. seealso::
 
-    Module logging, `HowTo <https://docs.python.org/2/howto/logging.html>`_
+    Module logging, `HowTo <https://docs.python.org/3/howto/logging.html>`_
         Basic Logging Tutorial
 
-    Module logging, `Loggers <https://docs.python.org/2/library/logging.html#logger-objects>`_
+    Module logging, `Loggers <https://docs.python.org/3/library/logging.html#logger-objects>`_
         Further documentation on loggers
 
 .. _topics-logging-from-spiders:
@@ -166,13 +166,13 @@ possible levels listed in :ref:`topics-logging-levels`.
 :setting:`LOG_FORMAT` and :setting:`LOG_DATEFORMAT` specify formatting strings
 used as layouts for all messages. Those strings can contain any placeholders
 listed in `logging's logrecord attributes docs
-<https://docs.python.org/2/library/logging.html#logrecord-attributes>`_ and
+<https://docs.python.org/3/library/logging.html#logrecord-attributes>`_ and
 `datetime's strftime and strptime directives
-<https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_
+<https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior>`_
 respectively.
 
 If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy
-component that prints the log. It is unset by default, hence logs contain the 
+component that prints the log. It is unset by default, hence logs contain the
 Scrapy component responsible for that log output.
 
 Command-line options
@@ -190,7 +190,7 @@ to override some of the Scrapy settings regarding logging.
 
 .. seealso::
 
-    Module `logging.handlers <https://docs.python.org/2/library/logging.handlers.html>`_
+    Module `logging.handlers <https://docs.python.org/3/library/logging.handlers.html>`_
         Further documentation on available handlers
 
 .. _custom-log-formats:
@@ -201,7 +201,7 @@ Custom Log Formats
 A custom log format can be set for different actions by extending
 :class:`~scrapy.logformatter.LogFormatter` class and making
 :setting:`LOG_FORMATTER` point to your new class.
- 
+
 .. autoclass:: scrapy.logformatter.LogFormatter
    :members:
 
@@ -276,6 +276,6 @@ scrapy.utils.log module
     Refer to :ref:`run-from-script` for more details about using Scrapy this
     way.
 
-.. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig
+.. _logging.basicConfig(): https://docs.python.org/3/library/logging.html#logging.basicConfig
 
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index b2a60ff39..6c5a08409 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -189,7 +189,7 @@ Request objects
         ``copy()`` or ``replace()`` methods, and can also be accessed, in your
         spider, from the ``response.cb_kwargs`` attribute.
 
-    .. _shallow copied: https://docs.python.org/2/library/copy.html
+    .. _shallow copied: https://docs.python.org/3/library/copy.html
 
     .. method:: Request.copy()
 
@@ -706,7 +706,7 @@ Response objects
 
         A :class:`twisted.internet.ssl.Certificate` object representing
         the server's SSL certificate.
-        
+
         Only populated for ``https`` responses, ``None`` otherwise.
 
     .. method:: Response.copy()
@@ -724,17 +724,17 @@ Response objects
         Constructs an absolute url by combining the Response's :attr:`url` with
         a possible relative url.
 
-        This is a wrapper over `urlparse.urljoin`_, it's merely an alias for
+        This is a wrapper over `urllib.parse.urljoin`_, it's merely an alias for
         making this call::
 
-            urlparse.urljoin(response.url, url)
+            urllib.parse.urljoin(response.url, url)
 
     .. automethod:: Response.follow
 
     .. automethod:: Response.follow_all
 
 
-.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
+.. _urllib.parse.urljoin: https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin
 
 .. _topics-request-response-ref-response-subclasses:
 
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 1f7802c98..0f90b28c0 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -36,7 +36,7 @@ defines selectors to associate those styles with specific HTML elements.
 
 .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
 .. _lxml: https://lxml.de/
-.. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html
+.. _ElementTree: https://docs.python.org/3/library/xml.etree.elementtree.html
 .. _XPath: https://www.w3.org/TR/xpath/all/
 .. _CSS: https://www.w3.org/TR/selectors
 .. _parsel: https://parsel.readthedocs.io/en/latest/
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index dc6843d75..d78a6253e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -28,7 +28,7 @@ The value of ``SCRAPY_SETTINGS_MODULE`` should be in Python path syntax, e.g.
 ``myproject.settings``. Note that the settings module should be on the
 Python `import search path`_.
 
-.. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path
+.. _import search path: https://docs.python.org/3/tutorial/modules.html#the-module-search-path
 
 .. _populating-settings:
 
@@ -902,7 +902,7 @@ Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
 String for formatting log messages. Refer to the `Python logging documentation`_ for the whole list of available
 placeholders.
 
-.. _Python logging documentation: https://docs.python.org/2/library/logging.html#logrecord-attributes
+.. _Python logging documentation: https://docs.python.org/3/library/logging.html#logrecord-attributes
 
 .. setting:: LOG_DATEFORMAT
 
@@ -915,7 +915,7 @@ String for formatting date/time, expansion of the ``%(asctime)s`` placeholder
 in :setting:`LOG_FORMAT`. Refer to the `Python datetime documentation`_ for the whole list of available
 directives.
 
-.. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior
+.. _Python datetime documentation: https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior
 
 .. setting:: LOG_FORMATTER
 
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 0e8210130..3d7450c86 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -173,18 +173,18 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         :type spider: :class:`~scrapy.spiders.Spider` object
 
     .. method:: from_crawler(cls, crawler)
-    
+
        If present, this classmethod is called to create a middleware instance
        from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
        of the middleware. Crawler object provides access to all Scrapy core
        components like settings and signals; it is a way for middleware to
        access them and hook its functionality into Scrapy.
-    
+
        :param crawler: crawler that uses this middleware
        :type crawler: :class:`~scrapy.crawler.Crawler` object
 
 
-.. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception
+.. _Exception: https://docs.python.org/3/library/exceptions.html#Exception
 
 
 .. _topics-spider-middleware-ref:

From f37b1bdc5616f67460c645e26c49f9d5b34e3631 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Fri, 20 Mar 2020 05:22:51 +0530
Subject: [PATCH 2706/2974] [docs] update redirect links to python3

---
 docs/intro/tutorial.rst               | 10 +++++-----
 docs/topics/contracts.rst             |  4 +---
 docs/topics/downloader-middleware.rst | 11 +++--------
 docs/topics/dynamic-content.rst       | 10 ++++------
 docs/topics/email.rst                 |  4 +---
 docs/topics/exporters.rst             | 20 ++++++--------------
 docs/topics/extensions.rst            |  3 +--
 docs/topics/items.rst                 | 21 ++++++---------------
 docs/topics/logging.rst               | 15 +++++----------
 docs/topics/request-response.rst      |  8 ++------
 docs/topics/selectors.rst             |  3 +--
 docs/topics/spider-middleware.rst     |  6 +-----
 docs/topics/spiders.rst               |  4 +---
 docs/topics/telnetconsole.rst         | 11 ++++-------
 scrapy/item.py                        |  4 +---
 15 files changed, 42 insertions(+), 92 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 1768badbb..ab6fd4829 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -25,16 +25,16 @@ Scrapy.
 If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource.
 
 If you're new to programming and want to start with Python, the following books
-may be useful to you: 
+may be useful to you:
 
 * `Automate the Boring Stuff With Python`_
 
-* `How To Think Like a Computer Scientist`_ 
+* `How To Think Like a Computer Scientist`_
 
-* `Learn Python 3 The Hard Way`_ 
+* `Learn Python 3 The Hard Way`_
 
 You can also take a look at `this list of Python resources for non-programmers`_,
-as well as the `suggested resources in the learnpython-subreddit`_. 
+as well as the `suggested resources in the learnpython-subreddit`_.
 
 .. _Python: https://www.python.org/
 .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
@@ -62,7 +62,7 @@ This will create a ``tutorial`` directory with the following contents::
             __init__.py
 
             items.py          # project items definition file
-            
+
             middlewares.py    # project middlewares file
 
             pipelines.py      # project pipelines file
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 43db8f101..319f577bc 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -136,7 +136,7 @@ Detecting check runs
 ====================
 
 When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
-set to the ``true`` string. You can use `os.environ`_ to perform any change to
+set to the ``true`` string. You can use :data:`os.environ` to perform any change to
 your spiders or your settings when ``scrapy check`` is used::
 
     import os
@@ -148,5 +148,3 @@ your spiders or your settings when ``scrapy check`` is used::
         def __init__(self):
             if os.environ.get('SCRAPY_CHECK'):
                 pass  # Do some scraper adjustments when a check is running
-
-.. _os.environ: https://docs.python.org/3/library/os.html#os.environ
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 61a3806fb..d7ec53bfa 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -739,7 +739,7 @@ HttpProxyMiddleware
    This middleware sets the HTTP proxy to use for requests, by setting the
    ``proxy`` meta value for :class:`~scrapy.http.Request` objects.
 
-   Like the Python standard library module `urllib.request`_, it obeys
+   Like the Python standard library module :mod:`urllib.request`, it obeys
    the following environment variables:
 
    * ``http_proxy``
@@ -751,8 +751,6 @@ HttpProxyMiddleware
    Keep in mind this value will take precedence over ``http_proxy``/``https_proxy``
    environment variables, and it will also ignore ``no_proxy`` environment variable.
 
-.. _urllib.request: https://docs.python.org/3/library/urllib.request.html
-
 RedirectMiddleware
 ------------------
 
@@ -982,7 +980,7 @@ RobotsTxtMiddleware
     Scrapy ships with support for the following robots.txt_ parsers:
 
     * :ref:`Protego <protego-parser>` (default)
-    * :ref:`RobotFileParser <python-robotfileparser>`
+    * :class:`~urllib.robotparser.RobotFileParser`
     * :ref:`Reppy <reppy-parser>`
     * :ref:`Robotexclusionrulesparser <rerp-parser>`
 
@@ -1030,13 +1028,10 @@ Based on `Protego <https://github.com/scrapy/protego>`_:
 
 Scrapy uses this parser by default.
 
-.. _python-robotfileparser:
-
 RobotFileParser
 ~~~~~~~~~~~~~~~
 
-Based on `RobotFileParser
-<https://docs.python.org/3.7/library/urllib.robotparser.html>`_:
+Based on :class:`~urllib.robotparser.RobotFileParser`:
 
 * is Python's built-in robots.txt_ parser
 
diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index b98133676..22bcac268 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -115,7 +115,7 @@ data from it depends on the type of response:
 -   If the response is HTML or XML, use :ref:`selectors
     <topics-selectors>` as usual.
 
--   If the response is JSON, use `json.loads`_ to load the desired data from
+-   If the response is JSON, use :func:`json.loads` to load the desired data from
     :attr:`response.text <scrapy.http.TextResponse.text>`::
 
         data = json.loads(response.text)
@@ -130,7 +130,7 @@ data from it depends on the type of response:
 -   If the response is JavaScript, or HTML with a ``<script/>`` element
     containing the desired data, see :ref:`topics-parsing-javascript`.
 
--   If the response is CSS, use a `regular expression`_ to extract the desired
+-   If the response is CSS, use :mod:`re` to extract the desired
     data from :attr:`response.text <scrapy.http.TextResponse.text>`.
 
 .. _topics-parsing-images:
@@ -168,8 +168,8 @@ JavaScript code:
 Once you have a string with the JavaScript code, you can extract the desired
 data from it:
 
--   You might be able to use a `regular expression`_ to extract the desired
-    data in JSON format, which you can then parse with `json.loads`_.
+-   You might be able to use :mod:`re` to extract the desired
+    data in JSON format, which you can then parse with :func:`json.loads`.
 
     For example, if the JavaScript code contains a separate line like
     ``var data = {"field": "value"};`` you can extract that data as follows:
@@ -241,9 +241,7 @@ along with `scrapy-selenium`_ for seamless integration.
 .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
 .. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
 .. _js2xml: https://github.com/scrapinghub/js2xml
-.. _json.loads: https://docs.python.org/3/library/json.html#json.loads
 .. _pytesseract: https://github.com/madmaze/pytesseract
-.. _regular expression: https://docs.python.org/3/library/re.html
 .. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
 .. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
 .. _Selenium: https://www.selenium.dev/
diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index aed3deb2e..e347c3a35 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -7,7 +7,7 @@ Sending e-mail
 .. module:: scrapy.mail
    :synopsis: Email sending facility
 
-Although Python makes sending e-mails relatively easy via the `smtplib`_
+Although Python makes sending e-mails relatively easy via the :mod:`smtplib`
 library, Scrapy provides its own facility for sending e-mails which is very
 easy to use and it's implemented using :doc:`Twisted non-blocking IO
 <twisted:core/howto/defer-intro>`, to avoid interfering with the non-blocking
@@ -15,8 +15,6 @@ IO of the crawler. It also provides a simple API for sending attachments and
 it's very easy to configure, with a few :ref:`settings
 <topics-email-settings>`.
 
-.. _smtplib: https://docs.python.org/3/library/smtplib.html
-
 Quick example
 =============
 
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 4ba8714bd..f73c6728d 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -311,7 +311,7 @@ CsvItemExporter
 
    The additional keyword arguments of this ``__init__`` method are passed to the
    :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
-   `csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
+   :func:`csv.writer` ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
    argument to customize this exporter.
 
    A typical output of this exporter would be::
@@ -320,8 +320,6 @@ CsvItemExporter
       Color TV,1200
       DVD player,200
 
-.. _csv.writer: https://docs.python.org/3/library/csv.html#csv.writer
-
 PickleItemExporter
 ------------------
 
@@ -335,15 +333,13 @@ PickleItemExporter
    :param protocol: The pickle protocol to use.
    :type protocol: int
 
-   For more information, refer to the `pickle module documentation`_.
+   For more information, refer :mod:`pickle`.
 
    The additional keyword arguments of this ``__init__`` method are passed to the
    :class:`BaseItemExporter` ``__init__`` method.
 
    Pickle isn't a human readable format, so no output examples are provided.
 
-.. _pickle module documentation: https://docs.python.org/3/library/pickle.html
-
 PprintItemExporter
 ------------------
 
@@ -372,8 +368,8 @@ JsonItemExporter
    Exports Items in JSON format to the specified file-like object, writing all
    objects as a list of objects. The additional ``__init__`` method arguments are
    passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
-   arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
-   `JSONEncoder`_ ``__init__`` method argument to customize this exporter.
+   arguments to the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
+   :class:`~json.JSONEncoder` ``__init__`` method argument to customize this exporter.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -393,8 +389,6 @@ JsonItemExporter
       stream-friendly format, consider using :class:`JsonLinesItemExporter`
       instead, or splitting the output in multiple chunks.
 
-.. _JSONEncoder: https://docs.python.org/3/library/json.html#json.JSONEncoder
-
 JsonLinesItemExporter
 ---------------------
 
@@ -403,8 +397,8 @@ JsonLinesItemExporter
    Exports Items in JSON format to the specified file-like object, writing one
    JSON-encoded item per line. The additional ``__init__`` method arguments are passed
    to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
-   the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
-   ``__init__`` method argument to customize this exporter.
+   the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
+   :class:`~json.JSONEncoder` ``__init__`` method argument to customize this exporter.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -417,8 +411,6 @@ JsonLinesItemExporter
    Unlike the one produced by :class:`JsonItemExporter`, the format produced by
    this exporter is well suited for serializing large amounts of data.
 
-.. _JSONEncoder: https://docs.python.org/3/library/json.html#json.JSONEncoder
-
 MarshalItemExporter
 -------------------
 
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index f57e37e6f..1b8413abf 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -364,7 +364,7 @@ Debugger extension
 
 .. class:: Debugger
 
-Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_
+Invokes a :mod:`Python debugger <pdb>`: inside a running Scrapy process when a `SIGUSR2`_
 signal is received. After the debugger is exited, the Scrapy process continues
 running normally.
 
@@ -372,5 +372,4 @@ For more info see `Debugging in Python`_.
 
 This extension only works on POSIX-compliant platforms (i.e. not Windows).
 
-.. _Python debugger: https://docs.python.org/3/library/pdb.html
 .. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 36731571e..2e5c88054 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -15,7 +15,7 @@ especially in a larger project with many spiders.
 
 To define common output data format Scrapy provides the :class:`Item` class.
 :class:`Item` objects are simple containers used to collect the scraped data.
-They provide a `dictionary-like`_ API with a convenient syntax for declaring
+They provide a :class:`dict` like API with a convenient syntax for declaring
 their available fields.
 
 Various Scrapy components use extra information provided by Items:
@@ -24,8 +24,6 @@ serialization can be customized using Item fields metadata, :mod:`trackref`
 tracks Item instances to help find memory leaks
 (see :ref:`topics-leaks-trackrefs`), etc.
 
-.. _dictionary-like: https://docs.python.org/3/library/stdtypes.html#dict
-
 .. _topics-items-declaring:
 
 Declaring Items
@@ -79,7 +77,7 @@ Working with Items
 
 Here are some examples of common tasks performed with items, using the
 ``Product`` item :ref:`declared above  <topics-items-declaring>`. You will
-notice the API is very similar to the `dict API`_.
+notice the API is very similar to the :class:`dict` API.
 
 Creating items
 --------------
@@ -145,7 +143,7 @@ KeyError: 'Product does not support field: lala'
 Accessing all populated values
 ------------------------------
 
-To access all populated values, just use the typical `dict API`_:
+To access all populated values, just use the typical :class:`dict`:
 
 >>> product.keys()
 ['price', 'name']
@@ -175,9 +173,7 @@ other item as well.
 
 If that is not the desired behavior, use a deep copy instead.
 
-See the `documentation of the copy module`_ for more information.
-
-.. _documentation of the copy module: https://docs.python.org/3/library/copy.html
+See :mod:`copy` for more information.
 
 To create a shallow copy of an item, you can either call
 :meth:`~scrapy.item.Item.copy` on an existing item
@@ -235,7 +231,7 @@ Item objects
 
     Return a new Item optionally initialized from the given argument.
 
-    Items replicate the standard `dict API`_, including its ``__init__`` method, and
+    Items replicate the standard :class:`dict`, including its ``__init__`` method, and
     also provide the following additional API members:
 
     .. automethod:: copy
@@ -249,22 +245,17 @@ Item objects
         :class:`Field` objects used in the :ref:`Item declaration
         <topics-items-declaring>`.
 
-.. _dict API: https://docs.python.org/3/library/stdtypes.html#dict
-
 Field objects
 =============
 
 .. class:: Field([arg])
 
-    The :class:`Field` class is just an alias to the built-in `dict`_ class and
+    The :class:`Field` class is just an alias to the built-in :class:`dict` class and
     doesn't provide any extra functionality or attributes. In other words,
     :class:`Field` objects are plain-old Python dicts. A separate class is used
     to support the :ref:`item declaration syntax <topics-items-declaring>`
     based on class attributes.
 
-.. _dict: https://docs.python.org/3/library/stdtypes.html#dict
-
-
 Other classes related to Item
 =============================
 
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index a85e1a769..df631b3dc 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -9,8 +9,7 @@ Logging
     explicit calls to the Python standard logging. Keep reading to learn more
     about the new logging system.
 
-Scrapy uses `Python's builtin logging system
-<https://docs.python.org/3/library/logging.html>`_ for event logging. We'll
+Scrapy uses :mod:`logging` for event logging. We'll
 provide some simple examples to get you started, but for more advanced
 use-cases it's strongly suggested to read thoroughly its documentation.
 
@@ -86,7 +85,7 @@ path::
     Module logging, `HowTo <https://docs.python.org/3/howto/logging.html>`_
         Basic Logging Tutorial
 
-    Module logging, `Loggers <https://docs.python.org/3/library/logging.html#logger-objects>`_
+    Module logging, :class:`~logging.Logger`
         Further documentation on loggers
 
 .. _topics-logging-from-spiders:
@@ -190,7 +189,7 @@ to override some of the Scrapy settings regarding logging.
 
 .. seealso::
 
-    Module `logging.handlers <https://docs.python.org/3/library/logging.handlers.html>`_
+    Module :mod:`logging.handlers`
         Further documentation on available handlers
 
 .. _custom-log-formats:
@@ -256,10 +255,10 @@ scrapy.utils.log module
     In that case, its usage is not required but it's recommended.
 
     Another option when running custom scripts is to manually configure the logging.
-    To do this you can use `logging.basicConfig()`_ to set a basic root handler.
+    To do this you can use :func:`logging.basicConfig` to set a basic root handler.
 
     Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``,
-    so it is recommended to only use `logging.basicConfig()`_ together with
+    so it is recommended to only use :func:`logging.basicConfig` together with
     :class:`~scrapy.crawler.CrawlerRunner`.
 
     This is an example on how to redirect ``INFO`` or higher messages to a file::
@@ -275,7 +274,3 @@ scrapy.utils.log module
 
     Refer to :ref:`run-from-script` for more details about using Scrapy this
     way.
-
-.. _logging.basicConfig(): https://docs.python.org/3/library/logging.html#logging.basicConfig
-
-
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 6c5a08409..7260141e9 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -566,12 +566,10 @@ dealing with JSON requests.
       set to ``'POST'`` automatically.
    :type data: JSON serializable object
 
-   :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize
+   :param dumps_kwargs: Parameters that will be passed to underlying :func:`json.dumps` method which is used to serialize
        data into JSON format.
    :type dumps_kwargs: dict
 
-.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps
-
 JsonRequest usage example
 -------------------------
 
@@ -724,7 +722,7 @@ Response objects
         Constructs an absolute url by combining the Response's :attr:`url` with
         a possible relative url.
 
-        This is a wrapper over `urllib.parse.urljoin`_, it's merely an alias for
+        This is a wrapper over :func:`~urllib.parse.urljoin`, it's merely an alias for
         making this call::
 
             urllib.parse.urljoin(response.url, url)
@@ -734,8 +732,6 @@ Response objects
     .. automethod:: Response.follow_all
 
 
-.. _urllib.parse.urljoin: https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin
-
 .. _topics-request-response-ref-response-subclasses:
 
 Response subclasses
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 0f90b28c0..bb46ea80f 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -14,7 +14,7 @@ achieve this, such as:
    drawback: it's slow.
 
  * `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
-   API based on `ElementTree`_. (lxml is not part of the Python standard
+   API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python standard
    library.)
 
 Scrapy comes with its own mechanism for extracting data. They're called
@@ -36,7 +36,6 @@ defines selectors to associate those styles with specific HTML elements.
 
 .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
 .. _lxml: https://lxml.de/
-.. _ElementTree: https://docs.python.org/3/library/xml.etree.elementtree.html
 .. _XPath: https://www.w3.org/TR/xpath/all/
 .. _CSS: https://www.w3.org/TR/selectors
 .. _parsel: https://parsel.readthedocs.io/en/latest/
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 3d7450c86..d49a2209d 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -140,7 +140,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         :type response: :class:`~scrapy.http.Response` object
 
         :param exception: the exception raised
-        :type exception: `Exception`_ object
+        :type exception: :exc:`Exception` object
 
         :param spider: the spider which raised the exception
         :type spider: :class:`~scrapy.spiders.Spider` object
@@ -183,10 +183,6 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
        :param crawler: crawler that uses this middleware
        :type crawler: :class:`~scrapy.crawler.Crawler` object
 
-
-.. _Exception: https://docs.python.org/3/library/exceptions.html#Exception
-
-
 .. _topics-spider-middleware-ref:
 
 Built-in spider middleware reference
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 89609db7d..231db6cea 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -298,9 +298,7 @@ Keep in mind that spider arguments are only strings.
 The spider will not do any parsing on its own.
 If you were to set the ``start_urls`` attribute from the command line,
 you would have to parse it on your own into a list
-using something like
-`ast.literal_eval <https://docs.python.org/3/library/ast.html#ast.literal_eval>`_
-or `json.loads <https://docs.python.org/3/library/json.html#json.loads>`_
+using something like :func:`ast.literal_eval` or :func:`json.loads`
 and then set it as an attribute.
 Otherwise, you would cause iteration over a ``start_urls`` string
 (a very common python pitfall)
diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index 47d8d393c..9802a34a2 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -40,10 +40,10 @@ the console you need to type::
     Connected to localhost.
     Escape character is '^]'.
     Username:
-    Password:  
+    Password:
     >>>
 
-By default Username is ``scrapy`` and Password is autogenerated. The 
+By default Username is ``scrapy`` and Password is autogenerated. The
 autogenerated Password can be seen on Scrapy logs like the example below::
 
     2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
@@ -63,7 +63,7 @@ Available variables in the telnet console
 =========================================
 
 The telnet console is like a regular Python shell running inside the Scrapy
-process, so you can do anything from it including importing new modules, etc. 
+process, so you can do anything from it including importing new modules, etc.
 
 However, the telnet console comes with some default variables defined for
 convenience:
@@ -89,13 +89,11 @@ convenience:
 +----------------+-------------------------------------------------------------------+
 | ``prefs``      | for memory debugging (see :ref:`topics-leaks`)                    |
 +----------------+-------------------------------------------------------------------+
-| ``p``          | a shortcut to the `pprint.pprint`_ function                       |
+| ``p``          | a shortcut to the :func:`pprint.pprint` function                  |
 +----------------+-------------------------------------------------------------------+
 | ``hpy``        | for memory debugging (see :ref:`topics-leaks`)                    |
 +----------------+-------------------------------------------------------------------+
 
-.. _pprint.pprint: https://docs.python.org/library/pprint.html#pprint.pprint
-
 Telnet console usage examples
 =============================
 
@@ -208,4 +206,3 @@ Default: ``None``
 
 The password used for the telnet console, default behaviour is to have it
 autogenerated
-
diff --git a/scrapy/item.py b/scrapy/item.py
index 1d39b48b2..748368932 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -121,9 +121,7 @@ class DictItem(MutableMapping, BaseItem):
         return self.__class__(self)
 
     def deepcopy(self):
-        """Return a `deep copy`_ of this item.
-
-        .. _deep copy: https://docs.python.org/library/copy.html#copy.deepcopy
+        """Return a :func:`~copy.deepcopy` of this item.
         """
         return deepcopy(self)
 

From 532cd1d93ed58f038346ca6b753462563c85b489 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Fri, 20 Mar 2020 17:36:49 +0530
Subject: [PATCH 2707/2974] [fix] zope interface 5.0.0 unsupported

---
 scrapy/crawler.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 77a13d0c1..20990ea41 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -4,7 +4,15 @@ import signal
 import warnings
 
 from twisted.internet import defer
-from zope.interface.verify import DoesNotImplement, verifyClass
+from zope.interface.exceptions import DoesNotImplement
+
+try:
+    # zope >= 5.0 only supports MultipleInvalid
+    from zope.interface.exceptions import MultipleInvalid
+except ImportError:
+    MultipleInvalid = None
+
+from zope.interface.verify import verifyClass
 
 from scrapy import signals, Spider
 from scrapy.core.engine import ExecutionEngine
@@ -124,9 +132,10 @@ class CrawlerRunner:
         """ Get SpiderLoader instance from settings """
         cls_path = settings.get('SPIDER_LOADER_CLASS')
         loader_cls = load_object(cls_path)
+        excs = (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement
         try:
             verifyClass(ISpiderLoader, loader_cls)
-        except DoesNotImplement:
+        except excs:
             warnings.warn(
                 'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does '
                 'not fully implement scrapy.interfaces.ISpiderLoader interface. '

From 36a3913a41a033e57c6119edd8bbf967044c927f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 22 Mar 2020 21:34:20 -0300
Subject: [PATCH 2708/2974] E501 compliance

---
 pytest.ini                           | 176 +++++++++------------------
 scrapy/cmdline.py                    |   5 +-
 scrapy/core/downloader/middleware.py |  18 ++-
 scrapy/robotstxt.py                  |  10 +-
 scrapy/spiders/crawl.py              |   8 +-
 5 files changed, 85 insertions(+), 132 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 141a13a4f..781a45541 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -20,6 +20,7 @@ addopts =
 twisted = 1
 markers =
     only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
+flake8-max-line-length = 119
 flake8-ignore =
     W503
     # Files that are only meant to provide top-level imports are expected not
@@ -27,146 +28,85 @@ flake8-ignore =
     scrapy/core/downloader/handlers/http.py F401
     scrapy/http/__init__.py F401
     # Issues pending a review:
-    # extras
-    extras/qps-bench-server.py E501
-    extras/qpsclient.py E501 E501
     # scrapy/commands
-    scrapy/commands/__init__.py E128 E501
-    scrapy/commands/check.py E501
-    scrapy/commands/crawl.py E501
-    scrapy/commands/edit.py E501
-    scrapy/commands/fetch.py E401 E501 E128 E731
-    scrapy/commands/genspider.py E128 E501 E502
-    scrapy/commands/parse.py E128 E501 E731
-    scrapy/commands/runspider.py E501
+    scrapy/commands/__init__.py E128
+    scrapy/commands/fetch.py E401 E128 E731
+    scrapy/commands/genspider.py E128 E502
+    scrapy/commands/parse.py E128 E731
+    scrapy/commands/runspider.py
     scrapy/commands/settings.py E128
-    scrapy/commands/shell.py E128 E501 E502
-    scrapy/commands/startproject.py E127 E501 E128
-    scrapy/commands/version.py E501 E128
+    scrapy/commands/shell.py E128 E502
+    scrapy/commands/startproject.py E127 E128
+    scrapy/commands/version.py E128
     # scrapy/contracts
-    scrapy/contracts/__init__.py E501 W504
+    scrapy/contracts/__init__.py W504
     scrapy/contracts/default.py E128
     # scrapy/core
-    scrapy/core/engine.py E501 E128 E127 E502
-    scrapy/core/scheduler.py E501
-    scrapy/core/scraper.py E501 E128 W504
-    scrapy/core/spidermw.py E501 E731 E126
-    scrapy/core/downloader/__init__.py E501
-    scrapy/core/downloader/contextfactory.py E501 E128 E126
-    scrapy/core/downloader/middleware.py E501 E502
-    scrapy/core/downloader/tls.py E501 E241
-    scrapy/core/downloader/webclient.py E731 E501 E128 E126
-    scrapy/core/downloader/handlers/__init__.py E501
-    scrapy/core/downloader/handlers/ftp.py E501 E128 E127
-    scrapy/core/downloader/handlers/http10.py E501
-    scrapy/core/downloader/handlers/http11.py E501
-    scrapy/core/downloader/handlers/s3.py E501 E128 E126
+    scrapy/core/engine.py E128 E127 E502
+    scrapy/core/scheduler.py
+    scrapy/core/scraper.py E128 W504
+    scrapy/core/spidermw.py E731 E126
+    scrapy/core/downloader/__init__.py
+    scrapy/core/downloader/contextfactory.py E128 E126
+    scrapy/core/downloader/middleware.py E502
+    scrapy/core/downloader/tls.py E241
+    scrapy/core/downloader/webclient.py E731 E128 E126
+    scrapy/core/downloader/handlers/ftp.py E128 E127
+    scrapy/core/downloader/handlers/s3.py E128 E126
     # scrapy/downloadermiddlewares
-    scrapy/downloadermiddlewares/ajaxcrawl.py E501
-    scrapy/downloadermiddlewares/decompression.py E501
-    scrapy/downloadermiddlewares/defaultheaders.py E501
-    scrapy/downloadermiddlewares/httpcache.py E501 E126
-    scrapy/downloadermiddlewares/httpcompression.py E501 E128
-    scrapy/downloadermiddlewares/httpproxy.py E501
-    scrapy/downloadermiddlewares/redirect.py E501 W504
-    scrapy/downloadermiddlewares/retry.py E501 E126
-    scrapy/downloadermiddlewares/robotstxt.py E501
-    scrapy/downloadermiddlewares/stats.py E501
+    scrapy/downloadermiddlewares/httpcache.py E126
+    scrapy/downloadermiddlewares/httpcompression.py E128
+    scrapy/downloadermiddlewares/redirect.py W504
+    scrapy/downloadermiddlewares/retry.py E126
     # scrapy/extensions
-    scrapy/extensions/closespider.py E501 E128 E123
-    scrapy/extensions/corestats.py E501
-    scrapy/extensions/feedexport.py E128 E501
-    scrapy/extensions/httpcache.py E128 E501
-    scrapy/extensions/memdebug.py E501
-    scrapy/extensions/spiderstate.py E501
-    scrapy/extensions/telnet.py E501 W504
-    scrapy/extensions/throttle.py E501
+    scrapy/extensions/closespider.py E128 E123
+    scrapy/extensions/feedexport.py E128
+    scrapy/extensions/httpcache.py E128
+    scrapy/extensions/telnet.py W504
     # scrapy/http
-    scrapy/http/common.py E501
-    scrapy/http/cookies.py E501
-    scrapy/http/request/__init__.py E501
-    scrapy/http/request/form.py E501 E123
-    scrapy/http/request/json_request.py E501
-    scrapy/http/response/__init__.py E501 E128
-    scrapy/http/response/text.py E501 E128 E124
+    scrapy/http/request/form.py E123
+    scrapy/http/response/__init__.py E128
+    scrapy/http/response/text.py E128 E124
     # scrapy/linkextractors
-    scrapy/linkextractors/__init__.py E731 E501 E402 W504
-    scrapy/linkextractors/lxmlhtml.py E501 E731
+    scrapy/linkextractors/__init__.py E731 E402 W504
+    scrapy/linkextractors/lxmlhtml.py E731
     # scrapy/loader
-    scrapy/loader/__init__.py E501 E128
-    scrapy/loader/processors.py E501
+    scrapy/loader/__init__.py E128
+    scrapy/loader/processors.py
     # scrapy/pipelines
-    scrapy/pipelines/__init__.py E501
-    scrapy/pipelines/files.py E116 E501 E266
-    scrapy/pipelines/images.py E265 E501
-    scrapy/pipelines/media.py E125 E501 E266
+    scrapy/pipelines/files.py E116 E266
+    scrapy/pipelines/images.py E265
+    scrapy/pipelines/media.py E125 E266
     # scrapy/selector
     scrapy/selector/__init__.py F403
-    scrapy/selector/unified.py E501 E111
+    scrapy/selector/unified.py E111
     # scrapy/settings
-    scrapy/settings/__init__.py E501
-    scrapy/settings/default_settings.py E501 E114 E116
-    scrapy/settings/deprecated.py E501
+    scrapy/settings/default_settings.py E114 E116
     # scrapy/spidermiddlewares
-    scrapy/spidermiddlewares/httperror.py E501
-    scrapy/spidermiddlewares/offsite.py E501
-    scrapy/spidermiddlewares/referer.py E501 E129 W504
-    scrapy/spidermiddlewares/urllength.py E501
+    scrapy/spidermiddlewares/referer.py E129 W504
     # scrapy/spiders
-    scrapy/spiders/__init__.py E501 E402
-    scrapy/spiders/crawl.py E501
-    scrapy/spiders/feed.py E501
-    scrapy/spiders/sitemap.py E501
+    scrapy/spiders/__init__.py E402
     # scrapy/utils
-    scrapy/utils/asyncio.py E501
-    scrapy/utils/benchserver.py E501
-    scrapy/utils/conf.py E402 E501
-    scrapy/utils/datatypes.py E501
-    scrapy/utils/decorators.py E501
-    scrapy/utils/defer.py E501 E128
-    scrapy/utils/deprecate.py E128 E501 E127 E502
-    scrapy/utils/gz.py E501 W504
+    scrapy/utils/defer.py E128
+    scrapy/utils/deprecate.py E128 E127 E502
+    scrapy/utils/gz.py W504
     scrapy/utils/http.py F403
-    scrapy/utils/httpobj.py E501
-    scrapy/utils/iterators.py E501
-    scrapy/utils/log.py E128 E501
+    scrapy/utils/log.py E128
     scrapy/utils/markup.py F403
-    scrapy/utils/misc.py E501
     scrapy/utils/multipart.py F403
-    scrapy/utils/project.py E501
-    scrapy/utils/python.py E501
-    scrapy/utils/reactor.py E501
-    scrapy/utils/reqser.py E501
-    scrapy/utils/request.py E127 E501
-    scrapy/utils/response.py E501 E128
-    scrapy/utils/signal.py E501 E128
-    scrapy/utils/sitemap.py E501
-    scrapy/utils/spider.py E501
-    scrapy/utils/ssl.py E501
-    scrapy/utils/test.py E501
-    scrapy/utils/url.py E501 F403 E128 F405
+    scrapy/utils/request.py E127
+    scrapy/utils/response.py E128
+    scrapy/utils/signal.py E128
+    scrapy/utils/url.py F403 E128 F405
     # scrapy
-    scrapy/__init__.py E402 E501
-    scrapy/cmdline.py E501
-    scrapy/crawler.py E501
-    scrapy/dupefilters.py E501 E202
-    scrapy/exceptions.py E501
-    scrapy/exporters.py E501
-    scrapy/interfaces.py E501
-    scrapy/item.py E501 E128
-    scrapy/link.py E501
-    scrapy/logformatter.py E501
-    scrapy/mail.py E402 E128 E501 E502
-    scrapy/middleware.py E128 E501
-    scrapy/pqueues.py E501
-    scrapy/resolver.py E501
-    scrapy/responsetypes.py E128 E501
-    scrapy/robotstxt.py E501
-    scrapy/shell.py E501
-    scrapy/signalmanager.py E501
-    scrapy/spiderloader.py F841 E501 E126
+    scrapy/__init__.py E402
+    scrapy/dupefilters.py E202
+    scrapy/item.py E128
+    scrapy/mail.py E402 E128 E502
+    scrapy/middleware.py E128
+    scrapy/responsetypes.py E128
+    scrapy/spiderloader.py F841 E126
     scrapy/squeues.py E128
-    scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501
     tests/mockserver.py E401 E501 E126 E123
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index a4ec7c8ae..b189e016b 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -165,6 +165,7 @@ if __name__ == '__main__':
     try:
         execute()
     finally:
-        # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()
-        # on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies
+        # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() on exit:
+        # http://doc.pypy.org/en/latest/cpython_differences.html
+        # ?highlight=gc.collect#differences-related-to-garbage-collection-strategies
         garbage_collect()
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 9c0014206..83c7b1f19 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -35,8 +35,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             for method in self.methods['process_request']:
                 response = yield deferred_from_coro(method(request=request, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
-                    raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
-                                         (method.__self__.__class__.__name__, response.__class__.__name__))
+                    raise _InvalidOutput(
+                        "Middleware %s.process_request must return None, Response or Request, got %s"
+                        % (method.__self__.__class__.__name__, response.__class__.__name__)
+                    )
                 if response:
                     defer.returnValue(response)
             defer.returnValue((yield download_func(request=request, spider=spider)))
@@ -50,8 +52,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             for method in self.methods['process_response']:
                 response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
                 if not isinstance(response, (Response, Request)):
-                    raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
-                                         (method.__self__.__class__.__name__, type(response)))
+                    raise _InvalidOutput(
+                        "Middleware %s.process_response must return Response or Request, got %s"
+                        % (method.__self__.__class__.__name__, type(response))
+                    )
                 if isinstance(response, Request):
                     defer.returnValue(response)
             defer.returnValue(response)
@@ -62,8 +66,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             for method in self.methods['process_exception']:
                 response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
-                    raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
-                                         (method.__self__.__class__.__name__, type(response)))
+                    raise _InvalidOutput(
+                        "Middleware %s.process_exception must return None, Response or Request, got %s"
+                        % (method.__self__.__class__.__name__, type(response))
+                    )
                 if response:
                     defer.returnValue(response)
             defer.returnValue(_failure)
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 0a9af3a62..52cf09844 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -17,10 +17,12 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
     except UnicodeDecodeError:
         # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
         # Switch to 'allow all' state.
-        logger.warning("Failure while parsing robots.txt. "
-                       "File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
-                       exc_info=sys.exc_info(),
-                       extra={'spider': spider})
+        logger.warning(
+            "Failure while parsing robots.txt. File either contains garbage or "
+            "is in an encoding other than UTF-8, treating it as an empty file.",
+            exc_info=sys.exc_info(),
+            extra={'spider': spider},
+        )
         robotstxt_body = ''
     return robotstxt_body
 
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index d76a96451..3d62224bf 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -54,8 +54,12 @@ class Rule:
         self.process_request = _get_method(self.process_request, spider)
         self.process_request_argcount = len(get_func_args(self.process_request))
         if self.process_request_argcount == 1:
-            msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated'
-            warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
+            warnings.warn(
+                "Rule.process_request should accept two arguments"
+                " (request, response), accepting only one is deprecated",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
 
     def _process_request(self, request, response):
         """

From 182394bcecf556854291f1a0d2e0d2c406bc7b48 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 22 Mar 2020 23:29:30 -0300
Subject: [PATCH 2709/2974] E501 compliance (tests)

---
 pytest.ini                                    | 129 ++++++++----------
 tests/test_command_shell.py                   |   4 +-
 tests/test_crawler.py                         |   4 +-
 tests/test_downloader_handlers.py             |  10 +-
 ...st_downloadermiddleware_httpcompression.py |   9 +-
 tests/test_downloadermiddleware_redirect.py   |  10 +-
 tests/test_exporters.py                       |  25 +++-
 tests/test_feedexport.py                      |  31 +++--
 tests/test_http_request.py                    |   5 +-
 tests/test_http_response.py                   |  33 +++--
 tests/test_linkextractors.py                  |  49 ++++---
 tests/test_pipeline_files.py                  |  49 ++++---
 tests/test_pipeline_images.py                 |  39 +++---
 tests/test_request_cb_kwargs.py               |  10 +-
 tests/test_responsetypes.py                   |   9 +-
 tests/test_selector.py                        |  32 +++--
 tests/test_spider.py                          |   4 +-
 tests/test_spidermiddleware_output_chain.py   |  20 ++-
 tests/test_spidermiddleware_referer.py        |  86 ++++++++----
 tests/test_utils_iterators.py                 |  25 +++-
 tests/test_utils_request.py                   |   8 +-
 tests/test_utils_sitemap.py                   |  36 +++--
 tests/test_utils_url.py                       |  23 +++-
 23 files changed, 415 insertions(+), 235 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 781a45541..6f7e09478 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -108,84 +108,63 @@ flake8-ignore =
     scrapy/spiderloader.py F841 E126
     scrapy/squeues.py E128
     # tests
-    tests/__init__.py E402 E501
-    tests/mockserver.py E401 E501 E126 E123
+    tests/__init__.py E402
+    tests/mockserver.py E401 E126 E123
     tests/pipelines.py F841
-    tests/spiders.py E501 E127
-    tests/test_closespider.py E501 E127
-    tests/test_command_fetch.py E501
-    tests/test_command_parse.py E501 E128
-    tests/test_command_shell.py E501 E128
-    tests/test_commands.py E128 E501
-    tests/test_contracts.py E501 E128
-    tests/test_crawl.py E501 E741 E265
-    tests/test_crawler.py F841 E501
-    tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123
-    tests/test_downloadermiddleware.py E501
-    tests/test_downloadermiddleware_ajaxcrawlable.py E501
-    tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126
+    tests/spiders.py E127
+    tests/test_closespider.py E127
+    tests/test_command_parse.py E128
+    tests/test_command_shell.py E128
+    tests/test_commands.py E128
+    tests/test_contracts.py E128
+    tests/test_crawl.py E741 E265
+    tests/test_crawler.py F841
+    tests/test_dependencies.py F841
+    tests/test_downloader_handlers.py E124 E127 E128 E265 E126 E123
+    tests/test_downloadermiddleware_cookies.py E731 E741 E128 E265 E126
     tests/test_downloadermiddleware_decompression.py E127
-    tests/test_downloadermiddleware_defaultheaders.py E501
-    tests/test_downloadermiddleware_downloadtimeout.py E501
-    tests/test_downloadermiddleware_httpcache.py E501
-    tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
-    tests/test_downloadermiddleware_httpproxy.py E501 E128
-    tests/test_downloadermiddleware_redirect.py E501 E128 E127
-    tests/test_downloadermiddleware_retry.py E501 E128 E126
-    tests/test_downloadermiddleware_robotstxt.py E501
-    tests/test_downloadermiddleware_stats.py E501
-    tests/test_dupefilters.py E501 E741 E128 E124
-    tests/test_engine.py E401 E501 E128
-    tests/test_exporters.py E501 E731 E128 E124
+    tests/test_downloadermiddleware_httpcompression.py E126 E123
+    tests/test_downloadermiddleware_httpproxy.py E128
+    tests/test_downloadermiddleware_redirect.py E128 E127
+    tests/test_downloadermiddleware_retry.py E128 E126
+    tests/test_dupefilters.py E741 E128 E124
+    tests/test_engine.py E401 E128
+    tests/test_exporters.py E731 E128 E124
     tests/test_extension_telnet.py F841
-    tests/test_feedexport.py E501 F841 E241
-    tests/test_http_cookies.py E501
-    tests/test_http_headers.py E501
-    tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
-    tests/test_http_response.py E501 E128 E265
+    tests/test_feedexport.py F841 E241
+    tests/test_http_request.py E402 E127 E128 E128 E126 E123
+    tests/test_http_response.py E128 E265
     tests/test_item.py E128 F841
-    tests/test_link.py E501
-    tests/test_linkextractors.py E501 E128 E124
-    tests/test_loader.py E501 E731 E741 E128 E117 E241
-    tests/test_logformatter.py E128 E501 E122
-    tests/test_mail.py E128 E501
-    tests/test_middleware.py E501 E128
-    tests/test_pipeline_crawl.py E501 E128 E126
-    tests/test_pipeline_files.py E501
-    tests/test_pipeline_images.py F841 E501
-    tests/test_pipeline_media.py E501 E741 E731 E128 E502
-    tests/test_proxy_connect.py E501 E741
-    tests/test_request_cb_kwargs.py E501
-    tests/test_responsetypes.py E501
-    tests/test_robotstxt_interface.py E501 E501
-    tests/test_scheduler.py E501 E126 E123
-    tests/test_selector.py E501 E127
-    tests/test_spider.py E501
-    tests/test_spidermiddleware.py E501
-    tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
-    tests/test_spidermiddleware_offsite.py E501 E128 E111
-    tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
-    tests/test_squeues.py E501 E741
-    tests/test_utils_asyncio.py E501
-    tests/test_utils_conf.py E501 E128
-    tests/test_utils_curl.py E501
-    tests/test_utils_datatypes.py E402 E501
-    tests/test_utils_defer.py E501 F841
-    tests/test_utils_deprecate.py F841 E501
-    tests/test_utils_http.py E501 E128 W504
-    tests/test_utils_iterators.py E501 E128 E129 E241
+    tests/test_linkextractors.py E128 E124
+    tests/test_loader.py E731 E741 E128 E117 E241
+    tests/test_logformatter.py E128 E122
+    tests/test_mail.py E128
+    tests/test_middleware.py E128
+    tests/test_pipeline_crawl.py E128 E126
+    tests/test_pipeline_images.py F841
+    tests/test_pipeline_media.py E741 E731 E128 E502
+    tests/test_proxy_connect.py E741
+    tests/test_scheduler.py E126 E123
+    tests/test_selector.py E127
+    tests/test_spidermiddleware_httperror.py E128 E127 E121
+    tests/test_spidermiddleware_offsite.py E128 E111
+    tests/test_spidermiddleware_referer.py F841 E125 E201 E124 E241 E121
+    tests/test_squeues.py E741
+    tests/test_utils_conf.py E128
+    tests/test_utils_datatypes.py E402
+    tests/test_utils_defer.py F841
+    tests/test_utils_deprecate.py F841
+    tests/test_utils_http.py E128 W504
+    tests/test_utils_iterators.py E128 E129 E241
     tests/test_utils_log.py E741
-    tests/test_utils_python.py E501 E731
-    tests/test_utils_reqser.py E501 E128
-    tests/test_utils_request.py E501 E128
-    tests/test_utils_response.py E501
+    tests/test_utils_python.py E731
+    tests/test_utils_reqser.py E128
+    tests/test_utils_request.py E128
     tests/test_utils_signal.py E741 F841 E731
-    tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123
-    tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126
-    tests/test_cmdline/__init__.py E501
-    tests/test_settings/__init__.py E501 E128
-    tests/test_spiderloader/__init__.py E128 E501
-    tests/test_utils_misc/__init__.py E501
+    tests/test_utils_sitemap.py E128 E124
+    tests/test_utils_url.py E127 E125 E241 E126 E123
+    tests/test_webclient.py E128 E122 E402 E241 E123 E126
+    tests/test_cmdline/__init__.py
+    tests/test_settings/__init__.py E128
+    tests/test_spiderloader/__init__.py E128
+    tests/test_utils_misc/__init__.py
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index d664b6ade..5dc0ba3d2 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -56,7 +56,9 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_redirect_not_follow_302(self):
-        _, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status'])
+        _, out, _ = yield self.execute(
+            ['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status']
+        )
         assert out.strip().endswith(b'302')
 
     @defer.inlineCallbacks
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 169e763f0..bfc9a609a 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -303,8 +303,10 @@ class CrawlerProcessSubprocess(unittest.TestCase):
     def test_ipv6_default_name_resolver(self):
         log = self.run_script('default_name_resolver.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertIn("twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", log)
         self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)
+        self.assertIn(
+            "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.",
+            log)
 
     def test_ipv6_alternative_name_resolver(self):
         log = self.run_script('alternative_name_resolver.py')
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 29d06bab4..1a7be5c1b 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -493,7 +493,10 @@ class Http11TestCase(HttpTestCase):
 class Https11TestCase(Http11TestCase):
     scheme = 'https'
 
-    tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", subject "/C=IE/O=Scrapy/CN=localhost"'
+    tls_log_message = (
+        'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", '
+        'subject "/C=IE/O=Scrapy/CN=localhost"'
+    )
 
     @defer.inlineCallbacks
     def test_tls_logging(self):
@@ -542,7 +545,10 @@ class Https11InvalidDNSPattern(Https11TestCase):
             from service_identity.exceptions import CertificateError  # noqa: F401
         except ImportError:
             raise unittest.SkipTest("cryptography lib is too old")
-        self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
+        self.tls_log_message = (
+            'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
+            'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
+        )
         super(Https11InvalidDNSPattern, self).setUp()
 
 
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 106ca3360..4d43e164f 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -124,7 +124,8 @@ class HttpCompressionTest(TestCase):
             'Content-Encoding': 'gzip',
         }
         f = BytesIO()
-        plainbody = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
+        plainbody = (b'<html><head><title>Some page</title>'
+                     b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">')
         zf = GzipFile(fileobj=f, mode='wb')
         zf.write(plainbody)
         zf.close()
@@ -142,7 +143,8 @@ class HttpCompressionTest(TestCase):
             'Content-Encoding': 'gzip',
         }
         f = BytesIO()
-        plainbody = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
+        plainbody = (b'<html><head><title>Some page</title>'
+                     b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">')
         zf = GzipFile(fileobj=f, mode='wb')
         zf.write(plainbody)
         zf.close()
@@ -158,7 +160,8 @@ class HttpCompressionTest(TestCase):
         headers = {
             'Content-Encoding': 'identity',
         }
-        plainbody = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
+        plainbody = (b'<html><head><title>Some page</title>'
+                     b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">')
         respcls = responsetypes.from_args(url="http://www.example.com/index", headers=headers, body=plainbody)
         response = respcls("http://www.example.com/index", headers=headers, body=plainbody)
         request = Request("http://www.example.com/index")
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 053e26fc3..3044c5b21 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -151,7 +151,10 @@ class RedirectMiddlewareTest(unittest.TestCase):
         self.assertEqual(req2.url, 'http://scrapytest.org/redirected')
         self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first'])
         self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
-        self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])
+        self.assertEqual(
+            req3.meta['redirect_urls'],
+            ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']
+        )
 
     def test_redirect_reasons(self):
         req1 = Request('http://scrapytest.org/first')
@@ -282,7 +285,10 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
         self.assertEqual(req2.url, 'http://scrapytest.org/redirected')
         self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first'])
         self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
-        self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])
+        self.assertEqual(
+            req3.meta['redirect_urls'],
+            ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']
+        )
 
     def test_redirect_reasons(self):
         req1 = Request('http://scrapytest.org/first')
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 6e2507508..b8301a567 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -111,7 +111,10 @@ class PythonItemExporterTest(BaseItemExporterTest):
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(type(exported), dict)
-        self.assertEqual(exported, {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'})
+        self.assertEqual(
+            exported,
+            {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'}
+        )
         self.assertEqual(type(exported['age']), dict)
         self.assertEqual(type(exported['age']['age']), dict)
 
@@ -121,7 +124,10 @@ class PythonItemExporterTest(BaseItemExporterTest):
         i3 = TestItem(name=u'Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
-        self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'})
+        self.assertEqual(
+            exported,
+            {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
+        )
         self.assertEqual(type(exported['age'][0]), dict)
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
@@ -131,7 +137,10 @@ class PythonItemExporterTest(BaseItemExporterTest):
         i3 = TestItem(name=u'Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
-        self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'})
+        self.assertEqual(
+            exported,
+            {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
+        )
         self.assertEqual(type(exported['age'][0]), dict)
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
@@ -327,13 +336,19 @@ class XmlItemExporterTest(BaseItemExporterTest):
         self.assertXmlEquivalent(fp.getvalue(), expected_value)
 
     def _check_output(self):
-        expected_value = b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><age>22</age><name>John\xc2\xa3</name></item></items>'
+        expected_value = (
+            b'<?xml version="1.0" encoding="utf-8"?>\n'
+            b'<items><item><age>22</age><name>John\xc2\xa3</name></item></items>'
+        )
         self.assertXmlEquivalent(self.output.getvalue(), expected_value)
 
     def test_multivalued_fields(self):
         self.assertExportResult(
             TestItem(name=[u'John\xa3', u'Doe']),
-            b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
+            (
+                b'<?xml version="1.0" encoding="utf-8"?>\n'
+                b'<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
+            )
         )
 
     def test_nested_item(self):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c5589e52f..e248969db 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -718,10 +718,13 @@ class FeedExportTest(unittest.TestCase):
         header = ['foo']
 
         formats = {
-            'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
-            'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
-            'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
+            'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
+            'jsonlines': '{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
+            'xml': (
+                '<?xml version="1.0" encoding="utf-8"?>\n'
+                '<items><item><foo>Test\xd6</foo></item></items>'
+            ).encode('utf-8'),
+            'csv': 'foo\r\nTest\xd6\r\n'.encode('utf-8'),
         }
 
         for fmt, expected in formats.items():
@@ -735,10 +738,13 @@ class FeedExportTest(unittest.TestCase):
             self.assertEqual(expected, data[fmt])
 
         formats = {
-            'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'),
-            'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),
-            'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
+            'json': '[{"foo": "Test\xd6"}]'.encode('latin-1'),
+            'jsonlines': '{"foo": "Test\xd6"}\n'.encode('latin-1'),
+            'xml': (
+                '<?xml version="1.0" encoding="latin-1"?>\n'
+                '<items><item><foo>Test\xd6</foo></item></items>'
+            ).encode('latin-1'),
+            'csv': 'foo\r\nTest\xd6\r\n'.encode('latin-1'),
         }
 
         for fmt, expected in formats.items():
@@ -757,9 +763,12 @@ class FeedExportTest(unittest.TestCase):
         items = [dict({'foo': u'FOO', 'bar': u'BAR'})]
 
         formats = {
-            'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode('latin-1'),
-            'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
+            'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'),
+            'xml': (
+                '<?xml version="1.0" encoding="latin-1"?>\n'
+                '<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'
+            ).encode('latin-1'),
+            'csv': 'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
         }
 
         settings = {
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index cc2cddda4..87d98d6ee 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1262,7 +1262,10 @@ class XmlRpcRequestTest(RequestTest):
 class JsonRequestTest(RequestTest):
     request_class = JsonRequest
     default_method = 'GET'
-    default_headers = {b'Content-Type': [b'application/json'], b'Accept': [b'application/json, text/javascript, */*; q=0.01']}
+    default_headers = {
+        b'Content-Type': [b'application/json'],
+        b'Accept': [b'application/json, text/javascript, */*; q=0.01'],
+    }
 
     def setUp(self):
         warnings.simplefilter("always")
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index eafc3560e..a6b717bd1 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -25,7 +25,11 @@ class BaseResponseTest(unittest.TestCase):
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class))
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class))
         # test presence of all optional parameters
-        self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class))
+        self.assertTrue(
+            isinstance(
+                self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class
+            )
+        )
 
         r = self.response_class("http://www.example.com")
         assert isinstance(r.url, str)
@@ -323,13 +327,16 @@ class TextResponseTest(BaseResponseTest):
         self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
-        resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]})
+        resp = self.response_class(u"http://www.example.com/price/\xa3",
+                                   headers={"Content-type": ["text/html; charset=utf-8"]})
         self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
-        resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]})
+        resp = self.response_class(u"http://www.example.com/price/\xa3",
+                                   headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
 
     def test_unicode_body(self):
-        unicode_string = u'\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442'
+        unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 '
+                          '\u0442\u0435\u043a\u0441\u0442')
         self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body')
 
         original_string = unicode_string.encode('cp1251')
@@ -344,13 +351,18 @@ class TextResponseTest(BaseResponseTest):
         self.assertEqual(r1.text, unicode_string)
 
     def test_encoding(self):
-        r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xc2\xa3")
+        r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3",
+                                 headers={"Content-type": ["text/html; charset=utf-8"]})
         r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
-        r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=b"\xa3")
+        r3 = self.response_class("http://www.example.com", body=b"\xa3",
+                                 headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3")
-        r5 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=None"]}, body=b"\xc2\xa3")
-        r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gb2312"]}, body=b"\xa8D")
-        r7 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gbk"]}, body=b"\xa8D")
+        r5 = self.response_class("http://www.example.com", body=b"\xc2\xa3",
+                                 headers={"Content-type": ["text/html; charset=None"]})
+        r6 = self.response_class("http://www.example.com", body=b"\xa8D",
+                                 headers={"Content-type": ["text/html; charset=gb2312"]})
+        r7 = self.response_class("http://www.example.com", body=b"\xa8D",
+                                 headers={"Content-type": ["text/html; charset=gbk"]})
 
         self.assertEqual(r1._headers_encoding(), "utf-8")
         self.assertEqual(r2._headers_encoding(), None)
@@ -685,7 +697,8 @@ class HtmlResponseTest(TextResponseTest):
         body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=utf-8">
         </head><body>Price: \xa3100</body></html>'
         """
-        r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=body)
+        r3 = self.response_class("http://www.example.com", body=body,
+                                 headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         self._assert_response_values(r3, 'iso-8859-1', body)
 
         # make sure replace() preserves the encoding of the original response
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 53968e60e..ed5f2560a 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -279,8 +279,8 @@ class Base:
         def test_process_value(self):
             """Test restrict_xpaths with encodings"""
             html = b"""
-            <a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Link text</a>
-            <a href="/about.html">About us</a>
+<a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Text</a>
+<a href="/about.html">About us</a>
             """
             response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
 
@@ -291,7 +291,7 @@ class Base:
 
             lx = self.extractor_cls(process_value=process_value)
             self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.org/other/page.html', text='Link text')])
+                             [Link(url='http://example.org/other/page.html', text='Text')])
 
         def test_base_url_with_restrict_xpaths(self):
             html = b"""<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
@@ -332,7 +332,10 @@ class Base:
             self.assertEqual(lx.extract_links(self.response), [])
 
         def test_tags(self):
-            html = b"""<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>"""
+            html = (
+                b'<html><area href="sample1.html"></area>'
+                b'<a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>'
+            )
             response = HtmlResponse("http://example.com/index.html", body=html)
 
             lx = self.extractor_cls(tags=None)
@@ -413,24 +416,34 @@ class Base:
             response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)
 
             lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                              Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                              Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                              Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
-                              Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
-                            )
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='',
+                         nofollow=True),
+                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='',
+                         nofollow=False),
+                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                ]
+            )
 
             response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
 
             lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                              Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                              Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                              Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
-                              Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
-                            )
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='',
+                         nofollow=True),
+                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='',
+                         nofollow=False),
+                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                ]
+            )
 
         def test_link_wrong_href(self):
             html = b"""
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index f155db4ce..34fc8dcb6 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -38,27 +38,36 @@ class FilesPipelineTestCase(unittest.TestCase):
 
     def test_file_path(self):
         file_path = self.pipeline.file_path
-        self.assertEqual(file_path(Request("https://dev.mydeco.com/mydeco.pdf")),
-                         'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
-        self.assertEqual(file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt")),
-                         'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt')
-        self.assertEqual(file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")),
-                         'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc')
-        self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
-                         'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
-        self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
-                         'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2')
-        self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
-                         'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
-        self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
-                                   response=Response("http://www.dorma.co.uk/images/product_details/2532"),
-                                   info=object()),
-                         'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
-        self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")),
-                         'full/76c00cef2ef669ae65052661f68d451162829507')
-        self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
+        self.assertEqual(
+            file_path(Request("https://dev.mydeco.com/mydeco.pdf")),
+            'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
+        self.assertEqual(
+            file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt")),
+            'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt')
+        self.assertEqual(
+            file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")),
+            'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc')
+        self.assertEqual(
+            file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
+            'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
+        self.assertEqual(
+            file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
+            'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2')
+        self.assertEqual(
+            file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
+            'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
+        self.assertEqual(
+            file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
+                      response=Response("http://www.dorma.co.uk/images/product_details/2532"),
+                      info=object()),
+            'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
+        self.assertEqual(
+            file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")),
+            'full/76c00cef2ef669ae65052661f68d451162829507')
+        self.assertEqual(
+            file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
                                     //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")),
-                         'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')
+            'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')
 
     def test_fs_store(self):
         assert isinstance(self.pipeline.store, FSFilesStore)
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 5018d6802..e1c3a55dd 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -41,22 +41,29 @@ class ImagesPipelineTestCase(unittest.TestCase):
 
     def test_file_path(self):
         file_path = self.pipeline.file_path
-        self.assertEqual(file_path(Request("https://dev.mydeco.com/mydeco.gif")),
-                         'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
-        self.assertEqual(file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg")),
-                         'full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg')
-        self.assertEqual(file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")),
-                         'full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg')
-        self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
-                         'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
-        self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
-                         'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg')
-        self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
-                         'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
-        self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
-                                   response=Response("http://www.dorma.co.uk/images/product_details/2532"),
-                                   info=object()),
-                         'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
+        self.assertEqual(
+            file_path(Request("https://dev.mydeco.com/mydeco.gif")),
+            'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
+        self.assertEqual(
+            file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg")),
+            'full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg')
+        self.assertEqual(
+            file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")),
+            'full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg')
+        self.assertEqual(
+            file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
+            'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
+        self.assertEqual(
+            file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
+            'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg')
+        self.assertEqual(
+            file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
+            'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
+        self.assertEqual(
+            file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
+                      response=Response("http://www.dorma.co.uk/images/product_details/2532"),
+                      info=object()),
+            'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
 
     def test_thumbnail_name(self):
         thumb_path = self.pipeline.thumb_path
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index a3ddd50f4..bd49179aa 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -158,6 +158,12 @@ class CallbackKeywordArgumentsTestCase(TestCase):
                 if key in line.getMessage():
                     exceptions[key] = line
         self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
-        self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
+        self.assertEqual(
+            str(exceptions['takes_less'].exc_info[1]),
+            "parse_takes_less() got an unexpected keyword argument 'number'"
+        )
         self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
-        self.assertEqual(str(exceptions['takes_more'].exc_info[1]), "parse_takes_more() missing 1 required positional argument: 'other'")
+        self.assertEqual(
+            str(exceptions['takes_more'].exc_info[1]),
+            "parse_takes_more() missing 1 required positional argument: 'other'"
+        )
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 8cdf7a176..9f4d56412 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -64,8 +64,9 @@ class ResponseTypesTest(unittest.TestCase):
     def test_from_headers(self):
         mappings = [
             ({'Content-Type': ['text/html; charset=utf-8']}, HtmlResponse),
-            ({'Content-Type': ['application/octet-stream'], 'Content-Disposition': ['attachment; filename=data.txt']}, TextResponse),
             ({'Content-Type': ['text/html; charset=utf-8'], 'Content-Encoding': ['gzip']}, Response),
+            ({'Content-Type': ['application/octet-stream'],
+              'Content-Disposition': ['attachment; filename=data.txt']}, TextResponse),
         ]
         for source, cls in mappings:
             source = Headers(source)
@@ -77,8 +78,10 @@ class ResponseTypesTest(unittest.TestCase):
         mappings = [
             ({'url': 'http://www.example.com/data.csv'}, TextResponse),
             # headers takes precedence over url
-            ({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}), 'url': 'http://www.example.com/item/'}, HtmlResponse),
-            ({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}), 'url': 'http://www.example.com/page/'}, Response),
+            ({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}),
+              'url': 'http://www.example.com/item/'}, HtmlResponse),
+            ({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}),
+              'url': 'http://www.example.com/page/'}, Response),
 
 
         ]
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 09c2546fb..3629feb16 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -19,18 +19,26 @@ class SelectorTestCase(unittest.TestCase):
         for x in xl:
             assert isinstance(x, Selector)
 
-        self.assertEqual(sel.xpath('//input').getall(),
-                         [x.get() for x in sel.xpath('//input')])
-
-        self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")],
-                         [u'a'])
-        self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
-                         [u'12.0'])
-
-        self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(),
-                         [u'xpathrules'])
-        self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
-                         [u'12'])
+        self.assertEqual(
+            sel.xpath('//input').getall(),
+            [x.get() for x in sel.xpath('//input')]
+        )
+        self.assertEqual(
+            [x.get() for x in sel.xpath("//input[@name='a']/@name")],
+            [u'a']
+        )
+        self.assertEqual(
+            [x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
+            [u'12.0']
+        )
+        self.assertEqual(
+            sel.xpath("concat('xpath', 'rules')").getall(),
+            [u'xpathrules']
+        )
+        self.assertEqual(
+            [x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
+            [u'12']
+        )
 
     def test_root_base_url(self):
         body = b'<html><form action="/path"><input name="a" /></form></html>'
diff --git a/tests/test_spider.py b/tests/test_spider.py
index bb00c8f42..805d70459 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -120,7 +120,9 @@ class XMLFeedSpiderTest(SpiderTest):
         body = b"""<?xml version="1.0" encoding="UTF-8"?>
         <urlset xmlns:x="http://www.google.com/schemas/sitemap/0.84"
                 xmlns:y="http://www.example.com/schemas/extras/1.0">
-        <url><x:loc>http://www.example.com/Special-Offers.html</loc><y:updated>2009-08-16</updated><other value="bar" y:custom="fuu"/></url>
+        <url><x:loc>http://www.example.com/Special-Offers.html</loc><y:updated>2009-08-16</updated>
+            <other value="bar" y:custom="fuu"/>
+        </url>
         <url><loc>http://www.example.com/</loc><y:updated>2009-08-16</updated><other value="foo"/></url>
         </urlset>"""
         response = XmlResponse(url='http://example.com/sitemap.xml', body=body)
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index b26353d6c..78d2de347 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -385,9 +385,15 @@ class TestSpiderMiddleware(TestCase):
         log4 = yield self.crawl_log(GeneratorOutputChainSpider)
         self.assertIn("'item_scraped_count': 2", str(log4))
         self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", str(log4))
-        self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", str(log4))
-        self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: LookupError caught", str(log4))
-        self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", str(log4))
+        self.assertIn(
+            "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught",
+            str(log4))
+        self.assertNotIn(
+            "GeneratorFailMiddleware.process_spider_exception: LookupError caught",
+            str(log4))
+        self.assertNotIn(
+            "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught",
+            str(log4))
         item_from_callback = {'processed': [
             'parse-first-item',
             'GeneratorFailMiddleware.process_spider_output',
@@ -414,9 +420,13 @@ class TestSpiderMiddleware(TestCase):
         log5 = yield self.crawl_log(NotGeneratorOutputChainSpider)
         self.assertIn("'item_scraped_count': 1", str(log5))
         self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", str(log5))
-        self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", str(log5))
+        self.assertIn(
+            "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught",
+            str(log5))
         self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", str(log5))
-        self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", str(log5))
+        self.assertNotIn(
+            "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught",
+            str(log5))
         item_recovered = {'processed': [
             'NotGeneratorRecoverMiddleware.process_spider_exception',
             'NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']}
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 4c6ede70b..3974f7884 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -114,12 +114,16 @@ class MixinNoReferrerWhenDowngrade:
 class MixinSameOrigin:
     scenarii = [
         # Same origin (protocol, host, port): send referrer
-        ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
-        ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
+        (
+            'http://example.com:8888/page.html',
+            'http://example.com:8888/not-page.html',
+            b'http://example.com:8888/page.html',
+        ),
 
         # Different host: do NOT send referrer
         ('https://example.com/page.html',       'https://not.example.com/otherpage.html',   None),
@@ -139,8 +143,12 @@ class MixinSameOrigin:
         ('ftps://example.com/urls.zip',     'https://example.com/not-page.html',    None),
 
         # test for user/password stripping
-        ('https://user:password@example.com/page.html', 'https://example.com/not-page.html',    b'https://example.com/page.html'),
-        ('https://user:password@example.com/page.html', 'http://example.com/not-page.html',     None),
+        ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None),
+        (
+            'https://user:password@example.com/page.html',
+            'https://example.com/not-page.html',
+            b'https://example.com/page.html',
+        ),
     ]
 
 
@@ -179,12 +187,16 @@ class MixinStrictOrigin:
 class MixinOriginWhenCrossOrigin:
     scenarii = [
         # Same origin (protocol, host, port): send referrer
-        ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
-        ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
+        (
+            'http://example.com:8888/page.html',
+            'http://example.com:8888/not-page.html',
+            b'http://example.com:8888/page.html',
+        ),
 
         # Different host: send origin as referrer
         ('https://example2.com/page.html',  'https://scrapy.org/otherpage.html',        b'https://example2.com/'),
@@ -205,21 +217,33 @@ class MixinOriginWhenCrossOrigin:
         ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
 
         # test for user/password stripping
-        ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html',  b'https://example5.com/page.html'),
+        (
+            'https://user:password@example5.com/page.html',
+            'https://example5.com/not-page.html',
+            b'https://example5.com/page.html',
+        ),
         # TLS to non-TLS downgrade: send origin
-        ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html',   b'https://example5.com/'),
+        (
+            'https://user:password@example5.com/page.html',
+            'http://example5.com/not-page.html',
+            b'https://example5.com/',
+        ),
     ]
 
 
 class MixinStrictOriginWhenCrossOrigin:
     scenarii = [
         # Same origin (protocol, host, port): send referrer
-        ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
-        ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
+        (
+            'http://example.com:8888/page.html',
+            'http://example.com:8888/not-page.html',
+            b'http://example.com:8888/page.html',
+        ),
 
         # Different host: send origin as referrer
         ('https://example2.com/page.html',  'https://scrapy.org/otherpage.html',        b'https://example2.com/'),
@@ -248,7 +272,11 @@ class MixinStrictOriginWhenCrossOrigin:
         ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
 
         # test for user/password stripping
-        ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html',  b'https://example5.com/page.html'),
+        (
+            'https://user:password@example5.com/page.html',
+            'https://example5.com/not-page.html',
+            b'https://example5.com/page.html',
+        ),
 
         # TLS to non-TLS downgrade: send nothing
         ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html',   None),
@@ -281,8 +309,16 @@ class MixinUnsafeUrl:
         ('ftp://example3.com/urls.zip',         'https://scrapy.org/',          b'ftp://example3.com/urls.zip'),
 
         # test for user/password stripping
-        ('http://user:password@example4.com/page.html',     'https://not.example4.com/',    b'http://example4.com/page.html'),
-        ('https://user:password@example4.com/page.html',    'http://scrapy.org/',           b'https://example4.com/page.html'),
+        (
+            'http://user:password@example4.com/page.html',
+            'https://not.example4.com/',
+            b'http://example4.com/page.html',
+        ),
+        (
+            'https://user:password@example4.com/page.html',
+            'http://scrapy.org/',
+            b'https://example4.com/page.html',
+        ),
     ]
 
 
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 33fc4d570..7c4011142 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -17,7 +17,8 @@ class XmliterTestCase(unittest.TestCase):
 
     def test_xmliter(self):
         body = b"""<?xml version="1.0" encoding="UTF-8"?>\
-            <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="someschmea.xsd">\
+            <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                      xsi:noNamespaceSchemaLocation="someschmea.xsd">\
               <product id="001">\
                 <type>Type 1</type>\
                 <name>Name 1</name>\
@@ -107,7 +108,10 @@ class XmliterTestCase(unittest.TestCase):
                               (u'27', [u'A'], [u'27'])])
 
     def test_xmliter_text(self):
-        body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
+        body = (
+            '<?xml version="1.0" encoding="UTF-8"?>'
+            '<products><product>one</product><product>two</product></products>'
+        )
 
         self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
                          [[u'one'], [u'two']])
@@ -139,7 +143,10 @@ class XmliterTestCase(unittest.TestCase):
         self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])
         self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1'])
         self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1'])
-        self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
+        self.assertEqual(
+            node.xpath('g:image_link/text()').getall(),
+            ['http://www.mydummycompany.com/images/item1.jpg']
+        )
         self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1'])
         self.assertEqual(node.xpath('g:price/text()').getall(), ['400'])
         self.assertEqual(node.xpath('image_link/text()').getall(), [])
@@ -147,7 +154,10 @@ class XmliterTestCase(unittest.TestCase):
         self.assertEqual(node.xpath('price/text()').getall(), [])
 
     def test_xmliter_exception(self):
-        body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
+        body = (
+            '<?xml version="1.0" encoding="UTF-8"?>'
+            '<products><product>one</product><product>two</product></products>'
+        )
 
         iter = self.xmliter(body, 'product')
         next(iter)
@@ -160,7 +170,12 @@ class XmliterTestCase(unittest.TestCase):
         self.assertRaises(AssertionError, next, i)
 
     def test_xmliter_encoding(self):
-        body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
+        body = (
+            b'<?xml version="1.0" encoding="ISO-8859-9"?>\n'
+            b'<xml>\n'
+            b'    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n'
+            b'</xml>\n\n'
+        )
         response = XmlResponse('http://www.example.com', body=body)
         self.assertEqual(
             next(self.xmliter(response, 'item')).get(),
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 45f0f59e4..3b1994019 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -75,8 +75,12 @@ class UtilsRequestTest(unittest.TestCase):
         r1 = Request("http://www.example.com/some/page.html?arg=1")
         self.assertEqual(request_httprepr(r1), b'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
 
-        r1 = Request("http://www.example.com", method='POST', headers={"Content-type": b"text/html"}, body=b"Some body")
-        self.assertEqual(request_httprepr(r1), b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body')
+        r1 = Request("http://www.example.com", method='POST',
+                     headers={"Content-type": b"text/html"}, body=b"Some body")
+        self.assertEqual(
+            request_httprepr(r1),
+            b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body'
+        )
 
     def test_request_httprepr_for_non_http_request(self):
         # the representation is not important but it must not fail.
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index db323ab31..4c09d33b2 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -22,8 +22,14 @@ class SitemapTest(unittest.TestCase):
   </url>
 </urlset>""")
         assert s.type == 'urlset'
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html',
+                 'lastmod': '2009-08-16', 'changefreq': 'weekly'},
+            ]
+        )
 
     def test_sitemap_index(self):
         s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -38,7 +44,13 @@ class SitemapTest(unittest.TestCase):
    </sitemap>
 </sitemapindex>""")
         assert s.type == 'sitemapindex'
-        self.assertEqual(list(s), [{'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'}, {'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'}])
+        self.assertEqual(
+            list(s),
+            [
+                {'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'},
+                {'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'},
+            ]
+        )
 
     def test_sitemap_strip(self):
         """Assert we can deal with trailing spaces inside <loc> tags - we've
@@ -195,11 +207,19 @@ Disallow: /forum/active/
         </url>
     </urlset>""")
 
-        self.assertEqual(list(s), [
-            {'loc': 'http://www.example.com/english/',
-             'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/']
-            }
-        ])
+        self.assertEqual(
+            list(s),
+            [
+                {
+                    'loc': 'http://www.example.com/english/',
+                    'alternate': [
+                        'http://www.example.com/deutsch/',
+                        'http://www.example.com/schweiz-deutsch/',
+                        'http://www.example.com/english/',
+                    ],
+                }
+            ]
+        )
 
     def test_xml_entity_expansion(self):
         s = Sitemap(b"""<?xml version="1.0" encoding="utf-8"?>
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 7abff8281..a12b6dda2 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -28,7 +28,10 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertTrue(url_is_from_any_domain(url, ['192.169.0.15:8080']))
         self.assertFalse(url_is_from_any_domain(url, ['192.169.0.15']))
 
-        url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29'
+        url = (
+            'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20'
+            'javascript:%20document.orderform_2581_1190810811.submit%28%29'
+        )
         self.assertFalse(url_is_from_any_domain(url, ['testdomain.com']))
         self.assertFalse(url_is_from_any_domain(url + '.testdomain.com', ['testdomain.com']))
 
@@ -105,8 +108,10 @@ class AddHttpIfNoScheme(unittest.TestCase):
                                                'http://username:password@www.example.com')
 
     def test_complete_url(self):
-        self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'
+        )
 
     def test_preserve_http(self):
         self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
@@ -137,8 +142,10 @@ class AddHttpIfNoScheme(unittest.TestCase):
                                                'http://username:password@www.example.com')
 
     def test_preserve_http_complete_url(self):
-        self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'
+        )
 
     def test_protocol_relative(self):
         self.assertEqual(add_http_if_no_scheme('//www.example.com'),
@@ -169,8 +176,10 @@ class AddHttpIfNoScheme(unittest.TestCase):
                                                'http://username:password@www.example.com')
 
     def test_protocol_relative_complete_url(self):
-        self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'
+        )
 
     def test_preserve_https(self):
         self.assertEqual(add_http_if_no_scheme('https://www.example.com'),

From 4663f0b9df7f687bf05ca57e5b6e8442b0997df7 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 22 Mar 2020 23:38:51 -0300
Subject: [PATCH 2710/2974] Update pytest.ini after removing E501

---
 pytest.ini | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 6f7e09478..1b00bfb84 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -33,7 +33,6 @@ flake8-ignore =
     scrapy/commands/fetch.py E401 E128 E731
     scrapy/commands/genspider.py E128 E502
     scrapy/commands/parse.py E128 E731
-    scrapy/commands/runspider.py
     scrapy/commands/settings.py E128
     scrapy/commands/shell.py E128 E502
     scrapy/commands/startproject.py E127 E128
@@ -43,10 +42,8 @@ flake8-ignore =
     scrapy/contracts/default.py E128
     # scrapy/core
     scrapy/core/engine.py E128 E127 E502
-    scrapy/core/scheduler.py
     scrapy/core/scraper.py E128 W504
     scrapy/core/spidermw.py E731 E126
-    scrapy/core/downloader/__init__.py
     scrapy/core/downloader/contextfactory.py E128 E126
     scrapy/core/downloader/middleware.py E502
     scrapy/core/downloader/tls.py E241
@@ -72,7 +69,6 @@ flake8-ignore =
     scrapy/linkextractors/lxmlhtml.py E731
     # scrapy/loader
     scrapy/loader/__init__.py E128
-    scrapy/loader/processors.py
     # scrapy/pipelines
     scrapy/pipelines/files.py E116 E266
     scrapy/pipelines/images.py E265
@@ -87,6 +83,7 @@ flake8-ignore =
     # scrapy/spiders
     scrapy/spiders/__init__.py E402
     # scrapy/utils
+    scrapy/utils/conf.py E402
     scrapy/utils/defer.py E128
     scrapy/utils/deprecate.py E128 E127 E502
     scrapy/utils/gz.py W504
@@ -164,7 +161,5 @@ flake8-ignore =
     tests/test_utils_sitemap.py E128 E124
     tests/test_utils_url.py E127 E125 E241 E126 E123
     tests/test_webclient.py E128 E122 E402 E241 E123 E126
-    tests/test_cmdline/__init__.py
     tests/test_settings/__init__.py E128
     tests/test_spiderloader/__init__.py E128
-    tests/test_utils_misc/__init__.py

From 80c69d68addef99f8b6ea5d3ec894752a06e2a9c Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Tue, 24 Mar 2020 05:52:07 +0530
Subject: [PATCH 2711/2974] [docs] refactor python docs links using intersphinx

---
 docs/intro/tutorial.rst               |  5 ++---
 docs/topics/coroutines.rst            |  5 ++---
 docs/topics/downloader-middleware.rst |  4 +++-
 docs/topics/dynamic-content.rst       | 10 ++++++----
 docs/topics/exporters.rst             |  4 ++--
 docs/topics/extensions.rst            |  2 +-
 docs/topics/items.rst                 | 18 ++++++++----------
 docs/topics/logging.rst               | 10 ++++------
 docs/topics/request-response.rst      | 14 ++++++--------
 docs/topics/settings.rst              | 18 +++++++-----------
 10 files changed, 41 insertions(+), 49 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index ab6fd4829..5f35dc936 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -287,8 +287,8 @@ to be scraped, you can at least get **some** data.
 
 Besides the :meth:`~scrapy.selector.SelectorList.getall` and
 :meth:`~scrapy.selector.SelectorList.get` methods, you can also use
-the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
-expressions`_:
+the :meth:`~scrapy.selector.SelectorList.re` method to extract using
+:doc:`regular expressions <library/re>`:
 
 >>> response.css('title::text').re(r'Quotes.*')
 ['Quotes to Scrape']
@@ -305,7 +305,6 @@ with a selector (see :ref:`topics-developer-tools`).
 `Selector Gadget`_ is also a nice tool to quickly find CSS selector for
 visually selected elements, which works in many browsers.
 
-.. _regular expressions: https://docs.python.org/3/library/re.html
 .. _Selector Gadget: https://selectorgadget.com/
 
 
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 487cf4c6c..5f61d6796 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -76,8 +76,8 @@ becomes::
 
 Coroutines may be used to call asynchronous code. This includes other
 coroutines, functions that return Deferreds and functions that return
-`awaitable objects`_ such as :class:`~asyncio.Future`. This means you can use
-many useful Python libraries providing such code::
+:term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
+This means you can use many useful Python libraries providing such code::
 
     class MySpider(Spider):
         # ...
@@ -107,4 +107,3 @@ Common use cases for asynchronous code include:
   :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
 
 .. _aio-libs: https://github.com/aio-libs
-.. _awaitable objects: https://docs.python.org/3/glossary.html#term-awaitable
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index d7ec53bfa..d309bbc49 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -980,7 +980,7 @@ RobotsTxtMiddleware
     Scrapy ships with support for the following robots.txt_ parsers:
 
     * :ref:`Protego <protego-parser>` (default)
-    * :class:`~urllib.robotparser.RobotFileParser`
+    * :ref:`RobotFileParser <python-robotfileparser>`
     * :ref:`Reppy <reppy-parser>`
     * :ref:`Robotexclusionrulesparser <rerp-parser>`
 
@@ -1028,6 +1028,8 @@ Based on `Protego <https://github.com/scrapy/protego>`_:
 
 Scrapy uses this parser by default.
 
+.. _python-robotfileparser:
+
 RobotFileParser
 ~~~~~~~~~~~~~~~
 
diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index 22bcac268..a3f0d6ebb 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -130,8 +130,9 @@ data from it depends on the type of response:
 -   If the response is JavaScript, or HTML with a ``<script/>`` element
     containing the desired data, see :ref:`topics-parsing-javascript`.
 
--   If the response is CSS, use :mod:`re` to extract the desired
-    data from :attr:`response.text <scrapy.http.TextResponse.text>`.
+-   If the response is CSS, use a :doc:`regular expression <library/re>` to
+    extract the desired data from
+    :attr:`response.text <scrapy.http.TextResponse.text>`.
 
 .. _topics-parsing-images:
 
@@ -168,8 +169,9 @@ JavaScript code:
 Once you have a string with the JavaScript code, you can extract the desired
 data from it:
 
--   You might be able to use :mod:`re` to extract the desired
-    data in JSON format, which you can then parse with :func:`json.loads`.
+-   You might be able to use a :doc:`regular expression <library/re>` to
+    extract the desired data in JSON format, which you can then parse with
+    :func:`json.loads`.
 
     For example, if the JavaScript code contains a separate line like
     ``var data = {"field": "value"};`` you can extract that data as follows:
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index f73c6728d..de8b51195 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -311,7 +311,7 @@ CsvItemExporter
 
    The additional keyword arguments of this ``__init__`` method are passed to the
    :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
-   :func:`csv.writer` ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
+   :func:`csv.writer` function, so you can use any :func:`csv.writer` function
    argument to customize this exporter.
 
    A typical output of this exporter would be::
@@ -333,7 +333,7 @@ PickleItemExporter
    :param protocol: The pickle protocol to use.
    :type protocol: int
 
-   For more information, refer :mod:`pickle`.
+   For more information, see :mod:`pickle`.
 
    The additional keyword arguments of this ``__init__`` method are passed to the
    :class:`BaseItemExporter` ``__init__`` method.
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 1b8413abf..0fc83e645 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -364,7 +364,7 @@ Debugger extension
 
 .. class:: Debugger
 
-Invokes a :mod:`Python debugger <pdb>`: inside a running Scrapy process when a `SIGUSR2`_
+Invokes a :doc:`Python debugger <library/pdb>` inside a running Scrapy process when a `SIGUSR2`_
 signal is received. After the debugger is exited, the Scrapy process continues
 running normally.
 
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 2e5c88054..78612f524 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -15,8 +15,8 @@ especially in a larger project with many spiders.
 
 To define common output data format Scrapy provides the :class:`Item` class.
 :class:`Item` objects are simple containers used to collect the scraped data.
-They provide a :class:`dict` like API with a convenient syntax for declaring
-their available fields.
+They provide an API similar to :class:`dict` API with a convenient syntax
+for declaring their available fields.
 
 Various Scrapy components use extra information provided by Items:
 exporters look at declared fields to figure out columns to export,
@@ -143,7 +143,7 @@ KeyError: 'Product does not support field: lala'
 Accessing all populated values
 ------------------------------
 
-To access all populated values, just use the typical :class:`dict`:
+To access all populated values, just use the typical :class:`dict` API:
 
 >>> product.keys()
 ['price', 'name']
@@ -160,11 +160,9 @@ Copying items
 To copy an item, you must first decide whether you want a shallow copy or a
 deep copy.
 
-If your item contains mutable_ values like lists or dictionaries, a shallow
-copy will keep references to the same mutable values across all different
-copies.
-
-.. _mutable: https://docs.python.org/3/glossary.html#term-mutable
+If your item contains :term:`mutable` values like lists or dictionaries,
+a shallow copy will keep references to the same mutable values across all
+different copies.
 
 For example, if you have an item with a list of tags, and you create a shallow
 copy of that item, both the original item and the copy have the same list of
@@ -231,8 +229,8 @@ Item objects
 
     Return a new Item optionally initialized from the given argument.
 
-    Items replicate the standard :class:`dict`, including its ``__init__`` method, and
-    also provide the following additional API members:
+    Items replicate the standard :class:`dict` API, including its ``__init__``
+    method, and also provide the following additional API members:
 
     .. automethod:: copy
 
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index df631b3dc..675e65ef1 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -82,10 +82,10 @@ path::
 
 .. seealso::
 
-    Module logging, `HowTo <https://docs.python.org/3/howto/logging.html>`_
+    Module logging, :doc:`HowTo <howto/logging>`
         Basic Logging Tutorial
 
-    Module logging, :class:`~logging.Logger`
+    Module logging, :ref:`Loggers <logger>`
         Further documentation on loggers
 
 .. _topics-logging-from-spiders:
@@ -164,10 +164,8 @@ possible levels listed in :ref:`topics-logging-levels`.
 
 :setting:`LOG_FORMAT` and :setting:`LOG_DATEFORMAT` specify formatting strings
 used as layouts for all messages. Those strings can contain any placeholders
-listed in `logging's logrecord attributes docs
-<https://docs.python.org/3/library/logging.html#logrecord-attributes>`_ and
-`datetime's strftime and strptime directives
-<https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior>`_
+listed in :ref:`logging's logrecord attributes docs <logrecord-attributes>` and
+:ref:`datetime's strftime and strptime directives <strftime-strptime-behavior>`
 respectively.
 
 If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 7260141e9..69a51a17c 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -174,9 +174,9 @@ Request objects
         See :ref:`topics-request-meta` for a list of special meta keys
         recognized by Scrapy.
 
-        This dict is `shallow copied`_ when the request is cloned using the
-        ``copy()`` or ``replace()`` methods, and can also be accessed, in your
-        spider, from the ``response.meta`` attribute.
+        This dict is :mod:`shallow copied <copy>` when the request is
+        cloned using the ``copy()`` or ``replace()`` methods, and can also be
+        accessed, in your spider, from the ``response.meta`` attribute.
 
     .. attribute:: Request.cb_kwargs
 
@@ -185,11 +185,9 @@ Request objects
         for new Requests, which means by default callbacks only get a :class:`Response`
         object as argument.
 
-        This dict is `shallow copied`_ when the request is cloned using the
-        ``copy()`` or ``replace()`` methods, and can also be accessed, in your
-        spider, from the ``response.cb_kwargs`` attribute.
-
-    .. _shallow copied: https://docs.python.org/3/library/copy.html
+        This dict is :mod:`shallow copied <copy>` when the request is
+        cloned using the ``copy()`` or ``replace()`` methods, and can also be
+        accessed, in your spider, from the ``response.cb_kwargs`` attribute.
 
     .. method:: Request.copy()
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index d78a6253e..0049dbfca 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -26,9 +26,7 @@ do this by using an environment variable, ``SCRAPY_SETTINGS_MODULE``.
 
 The value of ``SCRAPY_SETTINGS_MODULE`` should be in Python path syntax, e.g.
 ``myproject.settings``. Note that the settings module should be on the
-Python `import search path`_.
-
-.. _import search path: https://docs.python.org/3/tutorial/modules.html#the-module-search-path
+Python :ref:`import search path <tut-searchpath>`.
 
 .. _populating-settings:
 
@@ -899,10 +897,9 @@ LOG_FORMAT
 
 Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
 
-String for formatting log messages. Refer to the `Python logging documentation`_ for the whole list of available
-placeholders.
-
-.. _Python logging documentation: https://docs.python.org/3/library/logging.html#logrecord-attributes
+String for formatting log messages. Refer to the
+:ref:`Python logging documentation <logrecord-attributes>` for the qwhole
+list of available placeholders.
 
 .. setting:: LOG_DATEFORMAT
 
@@ -912,10 +909,9 @@ LOG_DATEFORMAT
 Default: ``'%Y-%m-%d %H:%M:%S'``
 
 String for formatting date/time, expansion of the ``%(asctime)s`` placeholder
-in :setting:`LOG_FORMAT`. Refer to the `Python datetime documentation`_ for the whole list of available
-directives.
-
-.. _Python datetime documentation: https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior
+in :setting:`LOG_FORMAT`. Refer to the
+:ref:`Python datetime documentation <strftime-strptime-behavior>` for the
+whole list of available directives.
 
 .. setting:: LOG_FORMATTER
 

From ee510cf0d6dda87bd5d472d177c04ceef742d1e3 Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Tue, 24 Mar 2020 13:31:44 -0300
Subject: [PATCH 2712/2974] Update scrapy/spiders/crawl.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 scrapy/spiders/crawl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 3d62224bf..cb021a5a7 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -55,8 +55,8 @@ class Rule:
         self.process_request_argcount = len(get_func_args(self.process_request))
         if self.process_request_argcount == 1:
             warnings.warn(
-                "Rule.process_request should accept two arguments"
-                " (request, response), accepting only one is deprecated",
+                "Rule.process_request should accept two arguments "
+                "(request, response), accepting only one is deprecated",
                 category=ScrapyDeprecationWarning,
                 stacklevel=2,
             )

From 010edfe85caa72b4c366f2dada0f79f1f91e43ef Mon Sep 17 00:00:00 2001
From: Aditi Dutta <aditi011@e.ntu.edu.sg>
Date: Wed, 25 Mar 2020 14:38:22 -0400
Subject: [PATCH 2713/2974] [Docs] mention curl2scrapy in Request.from_curl

---
 docs/topics/developer-tools.rst | 3 +++
 docs/topics/dynamic-content.rst | 3 +++
 scrapy/http/request/__init__.py | 3 +++
 3 files changed, 9 insertions(+)

diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index f1b0964c6..4e87a00f2 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -292,6 +292,9 @@ Alternatively, if you want to know the arguments needed to recreate that
 request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
 function to get a dictionary with the equivalent arguments.
 
+Note that to translate a cURL command into a Scrapy request,
+you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
+
 As you can see, with a few inspections in the `Network`-tool we
 were able to easily replicate the dynamic requests of the scrolling 
 functionality of the page. Crawling dynamic pages can be quite
diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index b98133676..aa326868b 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -104,6 +104,9 @@ If you get the expected response `sometimes`, but not always, the issue is
 probably not your request, but the target server. The target server might be
 buggy, overloaded, or :ref:`banning <bans>` some of your requests.
 
+Note that to translate a cURL command into a Scrapy request,
+you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
+
 .. _topics-handling-response-formats:
 
 Handling different response formats
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 6c536cb71..0a6637af8 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -129,6 +129,9 @@ class Request(object_ref):
                      :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`,
                      may modify the :class:`~scrapy.http.Request` object.
 
+        To translate a cURL command into a Scrapy request,
+        you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
+
        """
         request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options)
         request_kwargs.update(kwargs)

From 16f2cb4a83033b63f3adf7d7cd8b7982b8b094ef Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Thu, 26 Mar 2020 12:57:39 +0000
Subject: [PATCH 2714/2974] project URLs in machine-readable format for showing
 in pypi

---
 setup.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/setup.py b/setup.py
index 85d797f88..1b3c6771a 100644
--- a/setup.py
+++ b/setup.py
@@ -30,6 +30,11 @@ setup(
     name='Scrapy',
     version=version,
     url='https://scrapy.org',
+    project_urls = {
+        'Documentation': 'https://docs.scrapy.org/',
+        'Source': 'https://github.com/scrapy/scrapy',
+        'Tracker': 'https://github.com/scrapy/scrapy/issues',
+    },
     description='A high-level Web Crawling and Web Scraping framework',
     long_description=open('README.rst').read(),
     author='Scrapy developers',

From b1904729d52d75bcf732b7ddccd7364e6efaa577 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Fri, 27 Mar 2020 04:37:26 +0530
Subject: [PATCH 2715/2974] [docs] change mod to doc redirect link

---
 docs/topics/request-response.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 69a51a17c..573efc05f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -174,7 +174,7 @@ Request objects
         See :ref:`topics-request-meta` for a list of special meta keys
         recognized by Scrapy.
 
-        This dict is :mod:`shallow copied <copy>` when the request is
+        This dict is :doc:`shallow copied <library/copy>` when the request is
         cloned using the ``copy()`` or ``replace()`` methods, and can also be
         accessed, in your spider, from the ``response.meta`` attribute.
 
@@ -185,7 +185,7 @@ Request objects
         for new Requests, which means by default callbacks only get a :class:`Response`
         object as argument.
 
-        This dict is :mod:`shallow copied <copy>` when the request is
+        This dict is :doc:`shallow copied <library/copy>` when the request is
         cloned using the ``copy()`` or ``replace()`` methods, and can also be
         accessed, in your spider, from the ``response.cb_kwargs`` attribute.
 

From a175b6efc319ecbfd03b19705c25bd8bd44ca339 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 27 Mar 2020 02:10:10 -0300
Subject: [PATCH 2716/2974] Set up CI with Azure Pipelines

[skip ci]
---
 azure-pipelines.yml | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 azure-pipelines.yml

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
new file mode 100644
index 000000000..9bc324c32
--- /dev/null
+++ b/azure-pipelines.yml
@@ -0,0 +1,32 @@
+# Python package
+# Create and test a Python package on multiple Python versions.
+# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/languages/python
+
+
+pool:
+  vmImage: 'windows-2019'
+strategy:
+  matrix:
+    Python35:
+      python.version: '3.5'
+    Python36:
+      python.version: '3.6'
+    Python37:
+      python.version: '3.7'
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '$(python.version)'
+  displayName: 'Use Python $(python.version)'
+
+- script: |
+    python -m pip install --upgrade pip
+    pip install -r requirements.txt
+  displayName: 'Install dependencies'
+
+- script: |
+    pip install pytest pytest-azurepipelines
+    pytest
+  displayName: 'pytest'

From 02206e5ffe74fe8107c272ad35920928544323b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 27 Mar 2020 02:20:39 -0300
Subject: [PATCH 2717/2974] Run tox

---
 azure-pipelines.yml        | 9 ++++++---
 tests/requirements-py3.txt | 1 +
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 9bc324c32..489cfe53b 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -10,10 +10,13 @@ strategy:
   matrix:
     Python35:
       python.version: '3.5'
+      TOXENV: py35
     Python36:
       python.version: '3.6'
+      TOXENV: py36
     Python37:
       python.version: '3.7'
+      TOXENV: py37
 
 steps:
 - task: UsePythonVersion@0
@@ -27,6 +30,6 @@ steps:
   displayName: 'Install dependencies'
 
 - script: |
-    pip install pytest pytest-azurepipelines
-    pytest
-  displayName: 'pytest'
+    pip install -U tox twine wheel codecov
+    tox
+  displayName: 'Run test suite'
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index d207c5fb0..8896f4614 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -6,6 +6,7 @@ pytest < 5.4
 pytest-cov
 pytest-twisted >= 1.11
 pytest-xdist
+pytest-azurepipelines
 sybil
 testfixtures
 

From 0699e6bb1600ff943e131a3b7a299aa038145198 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Fri, 27 Mar 2020 02:22:05 -0300
Subject: [PATCH 2718/2974] no need to install requirements.txt

---
 azure-pipelines.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 489cfe53b..ffc4d549b 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -24,11 +24,6 @@ steps:
     versionSpec: '$(python.version)'
   displayName: 'Use Python $(python.version)'
 
-- script: |
-    python -m pip install --upgrade pip
-    pip install -r requirements.txt
-  displayName: 'Install dependencies'
-
 - script: |
     pip install -U tox twine wheel codecov
     tox

From 3fb0027138ab44b16be6e11677626d69b8c90c95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Sat, 28 Mar 2020 17:36:50 +0100
Subject: [PATCH 2719/2974] =?UTF-8?q?Require=20sybil=20=E2=89=A5=201.3.0?=
 =?UTF-8?q?=20for=20tests?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tests/requirements-py3.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index d207c5fb0..e7c86e0e9 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -6,7 +6,7 @@ pytest < 5.4
 pytest-cov
 pytest-twisted >= 1.11
 pytest-xdist
-sybil
+sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
 testfixtures
 
 # optional for shell wrapper tests

From e2d5d357a7ae50eaee957d1a2f8fc8ad1d9f3f24 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 1 Apr 2020 13:45:00 -0300
Subject: [PATCH 2720/2974] Fix pycodestyle E502

---
 pytest.ini                           | 14 +++++++-------
 scrapy/commands/genspider.py         |  7 +++----
 scrapy/commands/shell.py             |  2 +-
 scrapy/core/downloader/middleware.py | 18 ++++++++++++------
 scrapy/core/engine.py                |  5 ++---
 scrapy/mail.py                       |  4 ++--
 scrapy/utils/deprecate.py            | 15 +++++++++------
 tests/test_pipeline_media.py         |  5 +++--
 8 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 141a13a4f..da0f68e20 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -36,24 +36,24 @@ flake8-ignore =
     scrapy/commands/crawl.py E501
     scrapy/commands/edit.py E501
     scrapy/commands/fetch.py E401 E501 E128 E731
-    scrapy/commands/genspider.py E128 E501 E502
+    scrapy/commands/genspider.py E128 E501
     scrapy/commands/parse.py E128 E501 E731
     scrapy/commands/runspider.py E501
     scrapy/commands/settings.py E128
-    scrapy/commands/shell.py E128 E501 E502
+    scrapy/commands/shell.py E128 E501
     scrapy/commands/startproject.py E127 E501 E128
     scrapy/commands/version.py E501 E128
     # scrapy/contracts
     scrapy/contracts/__init__.py E501 W504
     scrapy/contracts/default.py E128
     # scrapy/core
-    scrapy/core/engine.py E501 E128 E127 E502
+    scrapy/core/engine.py E501 E128 E127
     scrapy/core/scheduler.py E501
     scrapy/core/scraper.py E501 E128 W504
     scrapy/core/spidermw.py E501 E731 E126
     scrapy/core/downloader/__init__.py E501
     scrapy/core/downloader/contextfactory.py E501 E128 E126
-    scrapy/core/downloader/middleware.py E501 E502
+    scrapy/core/downloader/middleware.py E501
     scrapy/core/downloader/tls.py E501 E241
     scrapy/core/downloader/webclient.py E731 E501 E128 E126
     scrapy/core/downloader/handlers/__init__.py E501
@@ -124,7 +124,7 @@ flake8-ignore =
     scrapy/utils/datatypes.py E501
     scrapy/utils/decorators.py E501
     scrapy/utils/defer.py E501 E128
-    scrapy/utils/deprecate.py E128 E501 E127 E502
+    scrapy/utils/deprecate.py E128 E501 E127
     scrapy/utils/gz.py E501 W504
     scrapy/utils/http.py F403
     scrapy/utils/httpobj.py E501
@@ -156,7 +156,7 @@ flake8-ignore =
     scrapy/item.py E501 E128
     scrapy/link.py E501
     scrapy/logformatter.py E501
-    scrapy/mail.py E402 E128 E501 E502
+    scrapy/mail.py E402 E128 E501
     scrapy/middleware.py E128 E501
     scrapy/pqueues.py E501
     scrapy/resolver.py E501
@@ -214,7 +214,7 @@ flake8-ignore =
     tests/test_pipeline_crawl.py E501 E128 E126
     tests/test_pipeline_files.py E501
     tests/test_pipeline_images.py F841 E501
-    tests/test_pipeline_media.py E501 E741 E731 E128 E502
+    tests/test_pipeline_media.py E501 E741 E731 E128
     tests/test_proxy_connect.py E501 E741
     tests/test_request_cb_kwargs.py E501
     tests/test_responsetypes.py E501
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index adb01fa70..2e837abed 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -90,8 +90,7 @@ class Command(ScrapyCommand):
             'module': module,
             'name': name,
             'domain': domain,
-            'classname': '%sSpider' % ''.join(s.capitalize() \
-                for s in module.split('_'))
+            'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_'))
         }
         if self.settings.get('NEWSPIDER_MODULE'):
             spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
@@ -102,8 +101,8 @@ class Command(ScrapyCommand):
         spider_file = "%s.py" % join(spiders_dir, module)
         shutil.copyfile(template_file, spider_file)
         render_templatefile(spider_file, **tvars)
-        print("Created spider %r using template %r " % (name, \
-            template_name), end=('' if spiders_module else '\n'))
+        print("Created spider %r using template %r "
+              % (name, template_name), end=('' if spiders_module else '\n'))
         if spiders_module:
             print("in module:\n  %s.%s" % (spiders_module.__name__, module))
 
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index d44a32d5f..5946f21e8 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -37,7 +37,7 @@ class Command(ScrapyCommand):
             help="evaluate the code in the shell, print the result and exit")
         parser.add_option("--spider", dest="spider",
             help="use this spider")
-        parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
+        parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
             default=False, help="do not handle HTTP 3xx status codes and print response as-is")
 
     def update_vars(self, vars):
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 9c0014206..83c7b1f19 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -35,8 +35,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             for method in self.methods['process_request']:
                 response = yield deferred_from_coro(method(request=request, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
-                    raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
-                                         (method.__self__.__class__.__name__, response.__class__.__name__))
+                    raise _InvalidOutput(
+                        "Middleware %s.process_request must return None, Response or Request, got %s"
+                        % (method.__self__.__class__.__name__, response.__class__.__name__)
+                    )
                 if response:
                     defer.returnValue(response)
             defer.returnValue((yield download_func(request=request, spider=spider)))
@@ -50,8 +52,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             for method in self.methods['process_response']:
                 response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
                 if not isinstance(response, (Response, Request)):
-                    raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
-                                         (method.__self__.__class__.__name__, type(response)))
+                    raise _InvalidOutput(
+                        "Middleware %s.process_response must return Response or Request, got %s"
+                        % (method.__self__.__class__.__name__, type(response))
+                    )
                 if isinstance(response, Request):
                     defer.returnValue(response)
             defer.returnValue(response)
@@ -62,8 +66,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             for method in self.methods['process_exception']:
                 response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
-                    raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
-                                         (method.__self__.__class__.__name__, type(response)))
+                    raise _InvalidOutput(
+                        "Middleware %s.process_exception must return None, Response or Request, got %s"
+                        % (method.__self__.__class__.__name__, type(response))
+                    )
                 if response:
                     defer.returnValue(response)
             defer.returnValue(_failure)
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 74f03344e..66cf9ad9a 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -277,10 +277,9 @@ class ExecutionEngine:
         next loop and this function is guaranteed to be called (at least) once
         again for this spider.
         """
-        res = self.signals.send_catch_log(signal=signals.spider_idle, \
+        res = self.signals.send_catch_log(signal=signals.spider_idle,
             spider=spider, dont_log=DontCloseSpider)
-        if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \
-                for _, x in res):
+        if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
             return
 
         if self.spider_is_idle(spider):
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 9d186f4f3..9d7896ef6 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -115,8 +115,8 @@ class MailSender:
         from twisted.mail.smtp import ESMTPSenderFactory
         msg = BytesIO(msg)
         d = defer.Deferred()
-        factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \
-            to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \
+        factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom,
+            to_addrs, msg, d, heloFallback=True, requireAuthentication=False,
             requireTransportSecurity=self.smtptls)
         factory.noisy = False
 
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index 69334a918..36001d982 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -7,9 +7,12 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 
 def attribute(obj, oldattr, newattr, version='0.12'):
     cname = obj.__class__.__name__
-    warnings.warn("%s.%s attribute is deprecated and will be no longer supported "
-        "in Scrapy %s, use %s.%s attribute instead" % \
-        (cname, oldattr, version, cname, newattr), ScrapyDeprecationWarning, stacklevel=3)
+    warnings.warn(
+        "%s.%s attribute is deprecated and will be no longer supported "
+        "in Scrapy %s, use %s.%s attribute instead"
+        % (cname, oldattr, version, cname, newattr),
+        ScrapyDeprecationWarning,
+        stacklevel=3)
 
 
 def create_deprecated_class(name, new_class, clsdict=None,
@@ -17,10 +20,10 @@ def create_deprecated_class(name, new_class, clsdict=None,
                             warn_once=True,
                             old_class_path=None,
                             new_class_path=None,
-                            subclass_warn_message="{cls} inherits from "\
-                                    "deprecated class {old}, please inherit "\
+                            subclass_warn_message="{cls} inherits from "
+                                    "deprecated class {old}, please inherit "
                                     "from {new}.",
-                            instance_warn_message="{cls} is deprecated, "\
+                            instance_warn_message="{cls} is deprecated, "
                                     "instantiate {new} instead."):
     """
     Return a "deprecated" class that causes its subclasses to issue a warning.
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index d369e147d..ee1441227 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -325,8 +325,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         item = dict(requests=req)
         new_item = yield self.pipe.process_item(item, self.spider)
         self.assertEqual(new_item['results'], [(True, 'ITSME')])
-        self.assertEqual(self.pipe._mockcalled, \
-                ['get_media_requests', 'media_to_download', 'item_completed'])
+        self.assertEqual(
+            self.pipe._mockcalled,
+            ['get_media_requests', 'media_to_download', 'item_completed'])
 
 
 class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase):

From 4270e0a0da66a2cb3a8e904c5ea74f84b7f9d041 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 4 Apr 2020 21:51:02 -0300
Subject: [PATCH 2721/2974] Fix E731: do not assign a lambda expression

---
 pytest.ini                                 | 24 +++++++++++-----------
 scrapy/commands/fetch.py                   |  4 ++--
 scrapy/commands/parse.py                   |  4 ++--
 scrapy/core/downloader/webclient.py        |  9 ++++----
 scrapy/linkextractors/__init__.py          | 10 +++++++--
 scrapy/linkextractors/lxmlhtml.py          |  6 ++----
 tests/test_downloadermiddleware_cookies.py |  7 +++----
 tests/test_exporters.py                    | 11 +++++-----
 tests/test_pipeline_media.py               | 18 ++++++++++------
 tests/test_utils_python.py                 |  4 +++-
 tests/test_utils_signal.py                 |  4 +++-
 11 files changed, 57 insertions(+), 44 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 141a13a4f..4b655b8d5 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -35,9 +35,9 @@ flake8-ignore =
     scrapy/commands/check.py E501
     scrapy/commands/crawl.py E501
     scrapy/commands/edit.py E501
-    scrapy/commands/fetch.py E401 E501 E128 E731
+    scrapy/commands/fetch.py E401 E501 E128
     scrapy/commands/genspider.py E128 E501 E502
-    scrapy/commands/parse.py E128 E501 E731
+    scrapy/commands/parse.py E128 E501
     scrapy/commands/runspider.py E501
     scrapy/commands/settings.py E128
     scrapy/commands/shell.py E128 E501 E502
@@ -50,12 +50,12 @@ flake8-ignore =
     scrapy/core/engine.py E501 E128 E127 E502
     scrapy/core/scheduler.py E501
     scrapy/core/scraper.py E501 E128 W504
-    scrapy/core/spidermw.py E501 E731 E126
+    scrapy/core/spidermw.py E501 E126
     scrapy/core/downloader/__init__.py E501
     scrapy/core/downloader/contextfactory.py E501 E128 E126
     scrapy/core/downloader/middleware.py E501 E502
     scrapy/core/downloader/tls.py E501 E241
-    scrapy/core/downloader/webclient.py E731 E501 E128 E126
+    scrapy/core/downloader/webclient.py E501 E128 E126
     scrapy/core/downloader/handlers/__init__.py E501
     scrapy/core/downloader/handlers/ftp.py E501 E128 E127
     scrapy/core/downloader/handlers/http10.py E501
@@ -90,8 +90,8 @@ flake8-ignore =
     scrapy/http/response/__init__.py E501 E128
     scrapy/http/response/text.py E501 E128 E124
     # scrapy/linkextractors
-    scrapy/linkextractors/__init__.py E731 E501 E402 W504
-    scrapy/linkextractors/lxmlhtml.py E501 E731
+    scrapy/linkextractors/__init__.py E501 E402 W504
+    scrapy/linkextractors/lxmlhtml.py E501
     # scrapy/loader
     scrapy/loader/__init__.py E501 E128
     scrapy/loader/processors.py E501
@@ -184,7 +184,7 @@ flake8-ignore =
     tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
-    tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126
+    tests/test_downloadermiddleware_cookies.py E741 E501 E128 E265 E126
     tests/test_downloadermiddleware_decompression.py E127
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
@@ -197,7 +197,7 @@ flake8-ignore =
     tests/test_downloadermiddleware_stats.py E501
     tests/test_dupefilters.py E501 E741 E128 E124
     tests/test_engine.py E401 E501 E128
-    tests/test_exporters.py E501 E731 E128 E124
+    tests/test_exporters.py E501 E128 E124
     tests/test_extension_telnet.py F841
     tests/test_feedexport.py E501 F841 E241
     tests/test_http_cookies.py E501
@@ -207,14 +207,14 @@ flake8-ignore =
     tests/test_item.py E128 F841
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128 E124
-    tests/test_loader.py E501 E731 E741 E128 E117 E241
+    tests/test_loader.py E501 E741 E128 E117 E241
     tests/test_logformatter.py E128 E501 E122
     tests/test_mail.py E128 E501
     tests/test_middleware.py E501 E128
     tests/test_pipeline_crawl.py E501 E128 E126
     tests/test_pipeline_files.py E501
     tests/test_pipeline_images.py F841 E501
-    tests/test_pipeline_media.py E501 E741 E731 E128 E502
+    tests/test_pipeline_media.py E501 E741 E128 E502
     tests/test_proxy_connect.py E501 E741
     tests/test_request_cb_kwargs.py E501
     tests/test_responsetypes.py E501
@@ -237,11 +237,11 @@ flake8-ignore =
     tests/test_utils_http.py E501 E128 W504
     tests/test_utils_iterators.py E501 E128 E129 E241
     tests/test_utils_log.py E741
-    tests/test_utils_python.py E501 E731
+    tests/test_utils_python.py E501
     tests/test_utils_reqser.py E501 E128
     tests/test_utils_request.py E501 E128
     tests/test_utils_response.py E501
-    tests/test_utils_signal.py E741 F841 E731
+    tests/test_utils_signal.py E741 F841
     tests/test_utils_sitemap.py E128 E501 E124
     tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123
     tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 0e149941d..506d1f1b7 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -49,8 +49,8 @@ class Command(ScrapyCommand):
     def run(self, args, opts):
         if len(args) != 1 or not is_url(args[0]):
             raise UsageError()
-        cb = lambda x: self._print_response(x, opts)
-        request = Request(args[0], callback=cb, dont_filter=True)
+        request = Request(args[0], callback=self._print_response,
+                          cb_kwargs={"opts": opts}, dont_filter=True)
         # by default, let the framework handle redirects,
         # i.e. command handles all codes expect 3xx
         if not opts.no_redirect:
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 3ef8ddcb3..d5abe5930 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -147,8 +147,8 @@ class Command(ScrapyCommand):
                 logger.error('Unable to find spider for: %(url)s', {'url': url})
 
         # Request requires callback argument as callable or None, not string
-        request = Request(url, None)
-        _start_requests = lambda s: [self.prepare_request(s, request, opts)]
+        def _start_requests(spider):
+            yield self.prepare_request(spider, Request(url, None), opts)
         self.spidercls.start_requests = _start_requests
 
     def start_parsing(self, url, opts):
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index a71dc5fb3..a90a77b2b 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -14,13 +14,12 @@ from scrapy.responsetypes import responsetypes
 def _parsed_url_args(parsed):
     # Assume parsed is urlparse-d from Request.url,
     # which was passed via safe_url_string and is ascii-only.
-    b = lambda s: to_bytes(s, encoding='ascii')
     path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
-    path = b(path)
-    host = b(parsed.hostname)
+    path = to_bytes(path, encoding="ascii")
+    host = to_bytes(parsed.hostname, encoding="ascii")
     port = parsed.port
-    scheme = b(parsed.scheme)
-    netloc = b(parsed.netloc)
+    scheme = to_bytes(parsed.scheme, encoding="ascii")
+    netloc = to_bytes(parsed.netloc, encoding="ascii")
     if port is None:
         port = 443 if scheme == b'https' else 80
     return scheme, netloc, host, port, path
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 6afe867b5..d0b5066b6 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -45,8 +45,14 @@ IGNORED_EXTENSIONS = [
 
 
 _re_type = type(re.compile("", 0))
-_matches = lambda url, regexs: any(r.search(url) for r in regexs)
-_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
+
+
+def _matches(url, regexs):
+    return any(r.search(url) for r in regexs)
+
+
+def _is_valid_url(url):
+    return url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
 
 
 class FilteringLinkExtractor:
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index fbac1dc59..ceb37c5f1 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -98,11 +98,9 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
                  unique=True, process_value=None, deny_extensions=None, restrict_css=(),
                  strip=True, restrict_text=None):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
-        tag_func = lambda x: x in tags
-        attr_func = lambda x: x in attrs
         lx = LxmlParserLinkExtractor(
-            tag=tag_func,
-            attr=attr_func,
+            tag=lambda x: x in tags,
+            attr=lambda x: x in attrs,
             unique=unique,
             process=process_value,
             strip=strip,
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 051f66680..a8182e2ef 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -13,10 +13,9 @@ from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
 class CookiesMiddlewareTest(TestCase):
 
     def assertCookieValEqual(self, first, second, msg=None):
-        cookievaleq = lambda cv: re.split(r';\s*', cv.decode('latin1'))
-        return self.assertEqual(
-            sorted(cookievaleq(first)),
-            sorted(cookievaleq(second)), msg)
+        def split_cookies(cookies):
+            return sorted(re.split(r";\s*", cookies.decode("latin1")))
+        return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg)
 
     def setUp(self):
         self.spider = Spider('foo')
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 6e2507508..160912847 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -215,11 +215,12 @@ class CsvItemExporterTest(BaseItemExporterTest):
         return CsvItemExporter(self.output, **kwargs)
 
     def assertCsvEqual(self, first, second, msg=None):
-        first = to_unicode(first)
-        second = to_unicode(second)
-        csvsplit = lambda csv: [sorted(re.split(r'(,|\s+)', line))
-                                for line in csv.splitlines(True)]
-        return self.assertEqual(csvsplit(first), csvsplit(second), msg)
+        def split_csv(csv):
+            return [
+                sorted(re.split(r"(,|\s+)", line))
+                for line in to_unicode(csv).splitlines(True)
+            ]
+        return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
 
     def _check_output(self):
         self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index d369e147d..f84f47816 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -199,12 +199,19 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
 
     pipeline_class = MockedMediaPipeline
 
+    def _callback(self, result):
+        self.pipe._mockcalled.append('request_callback')
+        return result
+
+    def _errback(self, result):
+        self.pipe._mockcalled.append('request_errback')
+        return result
+
     @inlineCallbacks
     def test_result_succeed(self):
-        cb = lambda _: self.pipe._mockcalled.append('request_callback') or _
-        eb = lambda _: self.pipe._mockcalled.append('request_errback') or _
         rsp = Response('http://url1')
-        req = Request('http://url1', meta=dict(response=rsp), callback=cb, errback=eb)
+        req = Request('http://url1', meta=dict(response=rsp),
+                      callback=self._callback, errback=self._errback)
         item = dict(requests=req)
         new_item = yield self.pipe.process_item(item, self.spider)
         self.assertEqual(new_item['results'], [(True, rsp)])
@@ -215,10 +222,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
     @inlineCallbacks
     def test_result_failure(self):
         self.pipe.LOG_FAILED_RESULTS = False
-        cb = lambda _: self.pipe._mockcalled.append('request_callback') or _
-        eb = lambda _: self.pipe._mockcalled.append('request_errback') or _
         fail = Failure(Exception())
-        req = Request('http://url1', meta=dict(response=fail), callback=cb, errback=eb)
+        req = Request('http://url1', meta=dict(response=fail),
+                      callback=self._callback, errback=self._errback)
         item = dict(requests=req)
         new_item = yield self.pipe.process_item(item, self.spider)
         self.assertEqual(new_item['results'], [(False, fail)])
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 8cb8df15b..65e6ba876 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -145,7 +145,9 @@ class UtilsPythonTestCase(unittest.TestCase):
 
         get_z = operator.itemgetter('z')
         get_meta = operator.attrgetter('meta')
-        compare_z = lambda obj: get_z(get_meta(obj))
+
+        def compare_z(obj):
+            return get_z(get_meta(obj))
 
         self.assertTrue(equal_attributes(a, b, [compare_z, 'x']))
         # fail z equality
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index 9f6da09ed..bb211dc60 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -90,8 +90,10 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
 class SendCatchLogTest2(unittest.TestCase):
 
     def test_error_logged_if_deferred_not_supported(self):
+        def test_handler():
+            return defer.Deferred()
+
         test_signal = object()
-        test_handler = lambda: defer.Deferred()
         dispatcher.connect(test_handler, test_signal)
         with LogCapture() as l:
             send_catch_log(test_signal)

From c887fe37adfe529ed2afabd2d08c3aac00a819c0 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 4 Apr 2020 22:15:36 -0300
Subject: [PATCH 2722/2974] Simplify parse command

---
 scrapy/commands/parse.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index d5abe5930..1cefed106 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -146,9 +146,8 @@ class Command(ScrapyCommand):
             if not self.spidercls:
                 logger.error('Unable to find spider for: %(url)s', {'url': url})
 
-        # Request requires callback argument as callable or None, not string
         def _start_requests(spider):
-            yield self.prepare_request(spider, Request(url, None), opts)
+            yield self.prepare_request(spider, Request(url), opts)
         self.spidercls.start_requests = _start_requests
 
     def start_parsing(self, url, opts):

From 862f0301e2cb166ca87ce985d51683b46dcf56ad Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 5 Apr 2020 00:53:10 -0300
Subject: [PATCH 2723/2974] Remove empty _RequestBodyProducer for POST requests

---
 scrapy/core/downloader/handlers/http11.py | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index c970909d7..09f828419 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -341,20 +341,6 @@ class ScrapyAgent:
             headers.removeHeader(b'Proxy-Authorization')
         if request.body:
             bodyproducer = _RequestBodyProducer(request.body)
-        elif method == b'POST':
-            # Setting Content-Length: 0 even for POST requests is not a
-            # MUST per HTTP RFCs, but it's common behavior, and some
-            # servers require this, otherwise returning HTTP 411 Length required
-            #
-            # RFC 7230#section-3.3.2:
-            # "a Content-Length header field is normally sent in a POST
-            # request even when the value is 0 (indicating an empty payload body)."
-            #
-            # Twisted < 17 will not add "Content-Length: 0" by itself;
-            # Twisted >= 17 fixes this;
-            # Using a producer with an empty-string sends `0` as Content-Length
-            # for all versions of Twisted.
-            bodyproducer = _RequestBodyProducer(b'')
         else:
             bodyproducer = None
         start_time = time()

From c4a5e3f0da3e674ecb7393c0894098984d6aa571 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 6 Apr 2020 09:26:13 -0300
Subject: [PATCH 2724/2974] Simplify bytes_received signal

Remove "source" parameter
---
 docs/topics/signals.rst                   |  6 +-----
 scrapy/core/downloader/handlers/http11.py | 18 +++++-------------
 scrapy/core/downloader/handlers/s3.py     |  1 -
 tests/test_engine.py                      |  5 +----
 4 files changed, 7 insertions(+), 23 deletions(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 97be46f2a..02fa6e287 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -163,7 +163,7 @@ bytes_received
 --------------
 
 .. signal:: bytes_received
-.. function:: bytes_received(data, request, spider, source)
+.. function:: bytes_received(data, request, spider)
 
     Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
     received for a specific request. This signal might be fired multiple
@@ -180,10 +180,6 @@ bytes_received
     :param spider: the spider associated with the response
     :type spider: :class:`~scrapy.spiders.Spider` object
 
-    :param source: a string to identify which handler sent the signal
-        (current values could be "http11" or "s3")
-    :type source: :class:`str` object
-
 spider_closed
 -------------
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index c350cd3c2..bda21b6b9 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -33,9 +33,8 @@ logger = logging.getLogger(__name__)
 class HTTP11DownloadHandler:
     lazy = False
 
-    def __init__(self, settings, crawler=None, source="http11"):
+    def __init__(self, settings, crawler=None):
         self._crawler = crawler
-        self._source = source
 
         from twisted.internet import reactor
         self._pool = HTTPConnectionPool(reactor, persistent=True)
@@ -71,8 +70,8 @@ class HTTP11DownloadHandler:
         self._disconnect_timeout = 1
 
     @classmethod
-    def from_crawler(cls, crawler, **kwargs):
-        return cls(crawler.settings, crawler, **kwargs)
+    def from_crawler(cls, crawler):
+        return cls(crawler.settings, crawler)
 
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
@@ -83,7 +82,6 @@ class HTTP11DownloadHandler:
             warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
             fail_on_dataloss=self._fail_on_dataloss,
             crawler=self._crawler,
-            source=self._source,
         )
         return agent.download_request(request)
 
@@ -281,7 +279,7 @@ class ScrapyAgent:
     _TunnelingAgent = TunnelingAgent
 
     def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
-                 maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None, source=None):
+                 maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None):
         self._contextFactory = contextFactory
         self._connectTimeout = connectTimeout
         self._bindAddress = bindAddress
@@ -291,7 +289,6 @@ class ScrapyAgent:
         self._fail_on_dataloss = fail_on_dataloss
         self._txresponse = None
         self._crawler = crawler
-        self._source = source
 
     def _get_agent(self, request, timeout):
         from twisted.internet import reactor
@@ -430,7 +427,6 @@ class ScrapyAgent:
                 warnsize=warnsize,
                 fail_on_dataloss=fail_on_dataloss,
                 crawler=self._crawler,
-                source=self._source,
             )
         )
 
@@ -468,9 +464,7 @@ class _RequestBodyProducer:
 
 class _ResponseReader(protocol.Protocol):
 
-    def __init__(
-        self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler, source
-    ):
+    def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler):
         self._finished = finished
         self._txresponse = txresponse
         self._request = request
@@ -483,7 +477,6 @@ class _ResponseReader(protocol.Protocol):
         self._bytes_received = 0
         self._certificate = None
         self._crawler = crawler
-        self._source = source
 
     def connectionMade(self):
         if self._certificate is None:
@@ -503,7 +496,6 @@ class _ResponseReader(protocol.Protocol):
             data=bodyBytes,
             request=self._request,
             spider=self._crawler.spider,
-            source=self._source,
         )
 
         if self._maxsize and self._bytes_received > self._maxsize:
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 2366b6394..40a1fa48e 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -73,7 +73,6 @@ class S3DownloadHandler:
             objcls=httpdownloadhandler,
             settings=settings,
             crawler=crawler,
-            source="s3",
         )
         self._download_http = _http_handler.download_request
 
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 26f3163cf..acfe94f63 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -112,7 +112,6 @@ class CrawlerRun:
         self.itemerror = []
         self.itemresp = []
         self.bytes = defaultdict(lambda: list())
-        self.bytes_source = set()
         self.signals_caught = {}
         self.spider_class = spider_class
 
@@ -166,9 +165,8 @@ class CrawlerRun:
     def item_scraped(self, item, spider, response):
         self.itemresp.append((item, response))
 
-    def bytes_received(self, data, request, spider, source):
+    def bytes_received(self, data, request, spider):
         self.bytes[request].append(data)
-        self.bytes_source.add(source)
 
     def request_scheduled(self, request, spider):
         self.reqplug.append((request, spider))
@@ -281,7 +279,6 @@ class EngineTest(unittest.TestCase):
 
     def _assert_bytes_received(self):
         self.assertEqual(9, len(self.run.bytes))
-        self.assertEqual(self.run.bytes_source, set(["http11"]))
         for request, data in self.run.bytes.items():
             joined_data = b"".join(data)
             if self.run.getpath(request.url) == "/":

From f97fec5ebd34776fdd97220cb2b1eff1f639a409 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 10 Apr 2020 16:02:53 -0300
Subject: [PATCH 2725/2974] Pin Sphinx version, including extensions

---
 docs/requirements.txt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/requirements.txt b/docs/requirements.txt
index 773b92cea..a99d1b78f 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,4 +1,4 @@
-Sphinx>=2.1
-sphinx-hoverxref
-sphinx-notfound-page
-sphinx_rtd_theme
+Sphinx==3.0.1
+sphinx-hoverxref==0.2b1
+sphinx-notfound-page==0.4
+sphinx_rtd_theme==0.4.3

From 24a1d9acae776bc195e6078394ee159b42275833 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 10 Apr 2020 16:48:42 -0300
Subject: [PATCH 2726/2974] Get version in docs config

---
 docs/conf.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 6e2399f66..c59688fbe 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -14,6 +14,7 @@
 import sys
 from datetime import datetime
 from os import path
+from pathlib import Path
 
 # If your extensions are in another directory, add it here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
@@ -59,10 +60,10 @@ copyright = '2008–{}, Scrapy developers'.format(datetime.now().year)
 #
 # The short X.Y version.
 try:
-    import scrapy
-    version = '.'.join(map(str, scrapy.version_info[:2]))
-    release = scrapy.__version__
-except ImportError:
+    version_path = Path(__file__).parent.absolute().parent.joinpath("scrapy/VERSION")
+    version = version_path.read_text().strip()
+    release = version.rsplit(".", 1)[0]
+except Exception:
     version = ''
     release = ''
 
@@ -295,3 +296,5 @@ intersphinx_mapping = {
 # ------------------------------------
 
 hoverxref_auto_ref = True
+hoverxref_project = "scrapy"
+hoverxref_version = release

From 4383f452999464b623393288361ecf7f383666e2 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 10 Apr 2020 16:49:14 -0300
Subject: [PATCH 2727/2974] Replace os.path with pathlib in docs config

---
 docs/conf.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index c59688fbe..a0bbbc90a 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -13,14 +13,13 @@
 
 import sys
 from datetime import datetime
-from os import path
 from pathlib import Path
 
 # If your extensions are in another directory, add it here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
-sys.path.append(path.join(path.dirname(__file__), "_ext"))
-sys.path.insert(0, path.dirname(path.dirname(__file__)))
+sys.path.append(str(Path(__file__).absolute().parent / "_ext"))
+sys.path.insert(0, str(Path(__file__).absolute().parent.parent))
 
 
 # General configuration

From 34e81d0d74fb0c5b9e880afdf214c2fd2ec193c6 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 10 Apr 2020 17:29:02 -0300
Subject: [PATCH 2728/2974] Docs: remove duplicated setting definitions

---
 docs/topics/downloader-middleware.rst |  1 +
 docs/topics/settings.rst              | 22 ----------------------
 2 files changed, 1 insertion(+), 22 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 73648994d..cea5e4564 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -829,6 +829,7 @@ REDIRECT_MAX_TIMES
 Default: ``20``
 
 The maximum number of redirections that will be followed for a single request.
+After this maximum the request's response is returned as is.
 
 MetaRefreshMiddleware
 ---------------------
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index dc6843d75..90df9a02e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1116,17 +1116,6 @@ multi-purpose thread pool used by various Scrapy components. Threaded
 DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase
 this value if you're experiencing problems with insufficient blocking IO.
 
-.. setting:: REDIRECT_MAX_TIMES
-
-REDIRECT_MAX_TIMES
-------------------
-
-Default: ``20``
-
-Defines the maximum times a request can be redirected. After this maximum the
-request's response is returned as is. We used Firefox default value for the
-same task.
-
 .. setting:: REDIRECT_PRIORITY_ADJUST
 
 REDIRECT_PRIORITY_ADJUST
@@ -1422,17 +1411,6 @@ Default: ``True``
 A boolean which specifies if the :ref:`telnet console <topics-telnetconsole>`
 will be enabled (provided its extension is also enabled).
 
-.. setting:: TELNETCONSOLE_PORT
-
-TELNETCONSOLE_PORT
-------------------
-
-Default: ``[6023, 6073]``
-
-The port range to use for the telnet console. If set to ``None`` or ``0``, a
-dynamically assigned port is used. For more info see
-:ref:`topics-telnetconsole`.
-
 .. setting:: TEMPLATES_DIR
 
 TEMPLATES_DIR

From 2205f04631d97103f98a28f865e8ac6511c15c82 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 10 Apr 2020 18:08:04 -0300
Subject: [PATCH 2729/2974] Docs: Add hoverxref_role_types setting

---
 docs/conf.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/conf.py b/docs/conf.py
index a0bbbc90a..40de81342 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -297,3 +297,10 @@ intersphinx_mapping = {
 hoverxref_auto_ref = True
 hoverxref_project = "scrapy"
 hoverxref_version = release
+hoverxref_role_types = {
+    "class": "tooltip",
+    "confval": "tooltip",
+    "hoverxref": "tooltip",
+    "mod": "tooltip",
+    "ref": "tooltip",
+}

From 83a0cc6cdf4b8d55ebc594f2635beb75d93898cf Mon Sep 17 00:00:00 2001
From: ilias-ant <ilias.antonopoulos@yahoo.gr>
Date: Sun, 12 Apr 2020 23:22:17 +0300
Subject: [PATCH 2730/2974] Add status to files information

---
 docs/topics/media-pipeline.rst | 13 +++++++++++--
 scrapy/pipelines/files.py      |  4 ++--
 tests/test_pipeline_crawl.py   |  5 +++++
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index cd84905c5..86550d7a4 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -50,7 +50,7 @@ this:
 4. When the files are downloaded, another field (``files``) will be populated
    with the results. This field will contain a list of dicts with information
    about the downloaded files, such as the downloaded path, the original
-   scraped url (taken from the ``file_urls`` field) , and the file checksum.
+   scraped url (taken from the ``file_urls`` field), the file checksum and the file status.
    The files in the list of the ``files`` field will retain the same order of
    the original ``file_urls`` field. If some file failed downloading, an
    error will be logged and the file won't be present in the ``files`` field.
@@ -470,6 +470,14 @@ See here the methods that you can override in your custom Files Pipeline:
 
         * ``checksum`` - a `MD5 hash`_ of the image contents
 
+        * ``status`` - the file status indication. It can be one of the following:
+
+          * ``downloaded`` - file was downloaded.
+          * ``uptodate`` - file was not downloaded, as it was downloaded recently,
+            according to the file expiration policy.
+          * ``cached`` - file was already scheduled for download, by another item
+            sharing the same file.
+
       The list of tuples received by :meth:`~item_completed` is
       guaranteed to retain the same order of the requests returned from the
       :meth:`~get_media_requests` method.
@@ -479,7 +487,8 @@ See here the methods that you can override in your custom Files Pipeline:
           [(True,
             {'checksum': '2b00042f7481c7b056c4b410d28f33cf',
              'path': 'full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg',
-             'url': 'http://www.example.com/files/product1.pdf'}),
+             'url': 'http://www.example.com/files/product1.pdf',
+             'status': 'downloaded'}),
            (False,
             Failure(...))]
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 101bf5fbc..747dfa065 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -417,7 +417,7 @@ class FilesPipeline(MediaPipeline):
             self.inc_stats(info.spider, 'uptodate')
 
             checksum = result.get('checksum', None)
-            return {'url': request.url, 'path': path, 'checksum': checksum}
+            return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'}
 
         path = self.file_path(request, info=info)
         dfd = defer.maybeDeferred(self.store.stat_file, path, info)
@@ -494,7 +494,7 @@ class FilesPipeline(MediaPipeline):
             )
             raise FileException(str(exc))
 
-        return {'url': request.url, 'path': path, 'checksum': checksum}
+        return {'url': request.url, 'path': path, 'checksum': checksum, 'status': status}
 
     def inc_stats(self, spider, status):
         spider.crawler.stats.inc_value('file_count', spider=spider)
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 962c33144..73b03e9d2 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -94,6 +94,11 @@ class FileDownloadCrawlTestCase(TestCase):
         file_dl_success = 'File (downloaded): Downloaded file from'
         self.assertEqual(logs.count(file_dl_success), 3)
 
+        # check that the images/files status is `downloaded`
+        for item in items:
+            for i in item[self.media_key]:
+                self.assertEqual(i['status'], 'downloaded')
+
         # check that the images/files checksums are what we know they should be
         if self.expected_checksums is not None:
             checksums = set(

From 1bd8f392c92d5e856332ab99f547d2a4359bd5d1 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 13 Apr 2020 06:12:30 -0300
Subject: [PATCH 2731/2974] Initial removal of
 twisted.internet.defer.returnValue

---
 scrapy/core/downloader/middleware.py        | 18 +++++++++---------
 tests/test_feedexport.py                    |  6 +++---
 tests/test_spidermiddleware_output_chain.py |  2 +-
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 83c7b1f19..5a03dcdf7 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -40,14 +40,14 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                         % (method.__self__.__class__.__name__, response.__class__.__name__)
                     )
                 if response:
-                    defer.returnValue(response)
-            defer.returnValue((yield download_func(request=request, spider=spider)))
+                    return response
+            return (yield download_func(request=request, spider=spider))
 
         @defer.inlineCallbacks
         def process_response(response):
             assert response is not None, 'Received None in process_response'
             if isinstance(response, Request):
-                defer.returnValue(response)
+                return response
 
             for method in self.methods['process_response']:
                 response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
@@ -57,12 +57,12 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                         % (method.__self__.__class__.__name__, type(response))
                     )
                 if isinstance(response, Request):
-                    defer.returnValue(response)
-            defer.returnValue(response)
+                    return response
+            return response
 
         @defer.inlineCallbacks
-        def process_exception(_failure):
-            exception = _failure.value
+        def process_exception(failure):
+            exception = failure.value
             for method in self.methods['process_exception']:
                 response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
@@ -71,8 +71,8 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                         % (method.__self__.__class__.__name__, type(response))
                     )
                 if response:
-                    defer.returnValue(response)
-            defer.returnValue(_failure)
+                    return response
+            return failure
 
         deferred = mustbe_deferred(process_request, request)
         deferred.addErrback(process_exception)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c5589e52f..e02b0b840 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -433,7 +433,7 @@ class FeedExportTest(unittest.TestCase):
             for file_path in FEEDS.keys():
                 os.remove(str(file_path))
 
-        defer.returnValue(content)
+        return content
 
     @defer.inlineCallbacks
     def exported_data(self, items, settings):
@@ -448,7 +448,7 @@ class FeedExportTest(unittest.TestCase):
                     yield item
 
         data = yield self.run_and_export(TestSpider, settings)
-        defer.returnValue(data)
+        return data
 
     @defer.inlineCallbacks
     def exported_no_data(self, settings):
@@ -462,7 +462,7 @@ class FeedExportTest(unittest.TestCase):
                 pass
 
         data = yield self.run_and_export(TestSpider, settings)
-        defer.returnValue(data)
+        return data
 
     @defer.inlineCallbacks
     def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index b26353d6c..ad4d6fb98 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -292,7 +292,7 @@ class TestSpiderMiddleware(TestCase):
         crawler = get_crawler(spider)
         with LogCapture() as log:
             yield crawler.crawl(mockserver=self.mockserver)
-        raise defer.returnValue(log)
+        return log
 
     @defer.inlineCallbacks
     def test_recovery(self):

From 4023d5db33b588b4df861581948e39b41c0d1678 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 13 Apr 2020 06:35:17 -0300
Subject: [PATCH 2732/2974] Replace _DefGen_Return exception handling

Handle StopIteration instead
---
 scrapy/pipelines/media.py    | 24 +++++++++++++-----------
 tests/test_pipeline_media.py | 11 +++++------
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 562d9ee32..a31c37900 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -1,7 +1,7 @@
 import functools
 import logging
 from collections import defaultdict
-from twisted.internet.defer import Deferred, DeferredList, _DefGen_Return
+from twisted.internet.defer import Deferred, DeferredList
 from twisted.python.failure import Failure
 
 from scrapy.settings import Settings
@@ -141,24 +141,26 @@ class MediaPipeline:
             # This code fixes a memory leak by avoiding to keep references to
             # the Request and Response objects on the Media Pipeline cache.
             #
-            # Twisted inline callbacks pass return values using the function
-            # twisted.internet.defer.returnValue, which encapsulates the return
-            # value inside a _DefGen_Return base exception.
-            #
-            # What happens when the media_downloaded callback raises another
+            # What happens when the media_downloaded callback raises an
             # exception, for example a FileException('download-error') when
-            # the Response status code is not 200 OK, is that it stores the
-            # _DefGen_Return exception on the FileException context.
+            # the Response status code is not 200 OK, is that the original
+            # StopIteration exception (which in turn contains the failed
+            # Response and by extension, the original Request) gets encapsulated
+            # within the FileException context.
+            #
+            # Originally, Scrapy was using twisted.internet.defer.returnValue
+            # inside functions decorated with twisted.internet.defer.inlineCallbacks,
+            # encapsulating the returned Response in a _DefGen_Return exception
+            # instead of a StopIteration.
             #
             # To avoid keeping references to the Response and therefore Request
             # objects on the Media Pipeline cache, we should wipe the context of
-            # the exception encapsulated by the Twisted Failure when its a
-            # _DefGen_Return instance.
+            # the encapsulated exception when it is a StopIteration instance
             #
             # This problem does not occur in Python 2.7 since we don't have
             # Exception Chaining (https://www.python.org/dev/peps/pep-3134/).
             context = getattr(result.value, '__context__', None)
-            if isinstance(context, _DefGen_Return):
+            if isinstance(context, StopIteration):
                 setattr(result.value, '__context__', None)
 
         info.downloading.remove(fp)
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index ee1441227..e6e21601b 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -2,7 +2,7 @@ from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.python.failure import Failure
 from twisted.internet import reactor
-from twisted.internet.defer import Deferred, inlineCallbacks, returnValue
+from twisted.internet.defer import Deferred, inlineCallbacks
 
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
@@ -124,9 +124,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         # Simulate the Media Pipeline behavior to produce a Twisted Failure
         try:
             # Simulate a Twisted inline callback returning a Response
-            # The returnValue method raises an exception encapsulating the value
-            returnValue(response)
-        except BaseException as exc:
+            raise StopIteration(response)
+        except StopIteration as exc:
             def_gen_return_exc = exc
             try:
                 # Simulate the media_downloaded callback raising a FileException
@@ -140,7 +139,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 
         # The Failure should encapsulate a FileException ...
         self.assertEqual(failure.value, file_exc)
-        # ... and it should have the returnValue exception set as its context
+        # ... and it should have the StopIteration exception set as its context
         self.assertEqual(failure.value.__context__, def_gen_return_exc)
 
         # Let's calculate the request fingerprint and fake some runtime data...
@@ -155,7 +154,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         self.assertEqual(info.downloaded[fp], failure)
         # ... encapsulating the original FileException ...
         self.assertEqual(info.downloaded[fp].value, file_exc)
-        # ... but it should not store the returnValue exception on its context
+        # ... but it should not store the StopIteration exception on its context
         context = getattr(info.downloaded[fp].value, '__context__', None)
         self.assertIsNone(context)
 

From 0a4ef97fa3d9d25b3f6b4afcf2b4986c505605c9 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 14 Apr 2020 14:57:20 -0300
Subject: [PATCH 2733/2974] Loose restrictions for docs requirements

---
 docs/requirements.txt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/requirements.txt b/docs/requirements.txt
index a99d1b78f..3d34b47da 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,4 +1,4 @@
-Sphinx==3.0.1
-sphinx-hoverxref==0.2b1
-sphinx-notfound-page==0.4
-sphinx_rtd_theme==0.4.3
+Sphinx>=3.0
+sphinx-hoverxref>=0.2b1
+sphinx-notfound-page>=0.4
+sphinx_rtd_theme>=0.4

From ee4ee486b1b4f66deffccbbe15f056edaf135982 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 14 Apr 2020 15:06:54 -0300
Subject: [PATCH 2734/2974] Revert unnecessary changes to docs/conf.py

---
 docs/conf.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 40de81342..4414ef637 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -13,13 +13,13 @@
 
 import sys
 from datetime import datetime
-from pathlib import Path
+from os import path
 
 # If your extensions are in another directory, add it here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
-sys.path.append(str(Path(__file__).absolute().parent / "_ext"))
-sys.path.insert(0, str(Path(__file__).absolute().parent.parent))
+sys.path.append(path.join(path.dirname(__file__), "_ext"))
+sys.path.insert(0, path.dirname(path.dirname(__file__)))
 
 
 # General configuration
@@ -59,10 +59,10 @@ copyright = '2008–{}, Scrapy developers'.format(datetime.now().year)
 #
 # The short X.Y version.
 try:
-    version_path = Path(__file__).parent.absolute().parent.joinpath("scrapy/VERSION")
-    version = version_path.read_text().strip()
-    release = version.rsplit(".", 1)[0]
-except Exception:
+    import scrapy
+    version = '.'.join(map(str, scrapy.version_info[:2]))
+    release = scrapy.__version__
+except ImportError:
     version = ''
     release = ''
 

From 94d7ad76cb96f1623d5944c28db24744955103cd Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <elacuesta@users.noreply.github.com>
Date: Wed, 15 Apr 2020 09:11:37 -0300
Subject: [PATCH 2735/2974] Fix pycodestyle E2XX (whitespace) (#4468)

---
 pytest.ini                                 |  30 +-
 scrapy/core/downloader/tls.py              |   4 +-
 scrapy/dupefilters.py                      |   2 +-
 scrapy/pipelines/files.py                  |   2 +-
 scrapy/pipelines/images.py                 |   2 +-
 scrapy/pipelines/media.py                  |   2 +-
 tests/test_crawl.py                        |   6 +-
 tests/test_downloadermiddleware_cookies.py |   4 +-
 tests/test_http_response.py                |   4 +-
 tests/test_spidermiddleware_referer.py     | 333 +++++++++++----------
 tests/test_utils_iterators.py              |  50 ++--
 tests/test_utils_url.py                    |  40 +--
 tests/test_webclient.py                    |  30 +-
 13 files changed, 268 insertions(+), 241 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index da0f68e20..de0bccbf1 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -54,7 +54,7 @@ flake8-ignore =
     scrapy/core/downloader/__init__.py E501
     scrapy/core/downloader/contextfactory.py E501 E128 E126
     scrapy/core/downloader/middleware.py E501
-    scrapy/core/downloader/tls.py E501 E241
+    scrapy/core/downloader/tls.py E501
     scrapy/core/downloader/webclient.py E731 E501 E128 E126
     scrapy/core/downloader/handlers/__init__.py E501
     scrapy/core/downloader/handlers/ftp.py E501 E128 E127
@@ -97,9 +97,9 @@ flake8-ignore =
     scrapy/loader/processors.py E501
     # scrapy/pipelines
     scrapy/pipelines/__init__.py E501
-    scrapy/pipelines/files.py E116 E501 E266
-    scrapy/pipelines/images.py E265 E501
-    scrapy/pipelines/media.py E125 E501 E266
+    scrapy/pipelines/files.py E116 E501
+    scrapy/pipelines/images.py E501
+    scrapy/pipelines/media.py E125 E501
     # scrapy/selector
     scrapy/selector/__init__.py F403
     scrapy/selector/unified.py E501 E111
@@ -149,7 +149,7 @@ flake8-ignore =
     scrapy/__init__.py E402 E501
     scrapy/cmdline.py E501
     scrapy/crawler.py E501
-    scrapy/dupefilters.py E501 E202
+    scrapy/dupefilters.py E501
     scrapy/exceptions.py E501
     scrapy/exporters.py E501
     scrapy/interfaces.py E501
@@ -178,13 +178,13 @@ flake8-ignore =
     tests/test_command_shell.py E501 E128
     tests/test_commands.py E128 E501
     tests/test_contracts.py E501 E128
-    tests/test_crawl.py E501 E741 E265
+    tests/test_crawl.py E501 E741
     tests/test_crawler.py F841 E501
     tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123
+    tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
-    tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126
+    tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E126
     tests/test_downloadermiddleware_decompression.py E127
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
@@ -199,15 +199,15 @@ flake8-ignore =
     tests/test_engine.py E401 E501 E128
     tests/test_exporters.py E501 E731 E128 E124
     tests/test_extension_telnet.py F841
-    tests/test_feedexport.py E501 F841 E241
+    tests/test_feedexport.py E501 F841
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
     tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
-    tests/test_http_response.py E501 E128 E265
+    tests/test_http_response.py E501 E128
     tests/test_item.py E128 F841
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128 E124
-    tests/test_loader.py E501 E731 E741 E128 E117 E241
+    tests/test_loader.py E501 E731 E741 E128 E117
     tests/test_logformatter.py E128 E501 E122
     tests/test_mail.py E128 E501
     tests/test_middleware.py E501 E128
@@ -226,7 +226,7 @@ flake8-ignore =
     tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
     tests/test_spidermiddleware_offsite.py E501 E128 E111
     tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
+    tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121
     tests/test_squeues.py E501 E741
     tests/test_utils_asyncio.py E501
     tests/test_utils_conf.py E501 E128
@@ -235,7 +235,7 @@ flake8-ignore =
     tests/test_utils_defer.py E501 F841
     tests/test_utils_deprecate.py F841 E501
     tests/test_utils_http.py E501 E128 W504
-    tests/test_utils_iterators.py E501 E128 E129 E241
+    tests/test_utils_iterators.py E501 E128 E129
     tests/test_utils_log.py E741
     tests/test_utils_python.py E501 E731
     tests/test_utils_reqser.py E501 E128
@@ -243,8 +243,8 @@ flake8-ignore =
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841 E731
     tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123
-    tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126
+    tests/test_utils_url.py E501 E127 E125 E501 E126 E123
+    tests/test_webclient.py E501 E128 E122 E402 E123 E126
     tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
     tests/test_spiderloader/__init__.py E128 E501
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index a1c881d5e..e43a3c83e 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -20,8 +20,8 @@ METHOD_TLSv12 = 'TLSv1.2'
 
 
 openssl_methods = {
-    METHOD_TLS:    SSL.SSLv23_METHOD,                   # protocol negotiation (recommended)
-    METHOD_SSLv3:  SSL.SSLv3_METHOD,                    # SSL 3 (NOT recommended)
+    METHOD_TLS: SSL.SSLv23_METHOD,                      # protocol negotiation (recommended)
+    METHOD_SSLv3: SSL.SSLv3_METHOD,                     # SSL 3 (NOT recommended)
     METHOD_TLSv10: SSL.TLSv1_METHOD,                    # TLS 1.0 only
     METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only
     METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index d74c8ed36..ac5478e7c 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -61,7 +61,7 @@ class RFPDupeFilter(BaseDupeFilter):
     def log(self, request, spider):
         if self.debug:
             msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)"
-            args = {'request': request, 'referer': referer_str(request) }
+            args = {'request': request, 'referer': referer_str(request)}
             self.logger.debug(msg, args, extra={'spider': spider})
         elif self.logdupes:
             msg = ("Filtered duplicate request: %(request)s"
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 101bf5fbc..aab645d3d 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -500,7 +500,7 @@ class FilesPipeline(MediaPipeline):
         spider.crawler.stats.inc_value('file_count', spider=spider)
         spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider)
 
-    ### Overridable Interface
+    # Overridable Interface
     def get_media_requests(self, item, info):
         return [Request(x) for x in item.get(self.files_urls_field, [])]
 
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 2e646379c..aeb520442 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -14,7 +14,7 @@ from scrapy.utils.python import to_bytes
 from scrapy.http import Request
 from scrapy.settings import Settings
 from scrapy.exceptions import DropItem
-#TODO: from scrapy.pipelines.media import MediaPipeline
+# TODO: from scrapy.pipelines.media import MediaPipeline
 from scrapy.pipelines.files import FileException, FilesPipeline
 
 
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 562d9ee32..a6d99fa99 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -166,7 +166,7 @@ class MediaPipeline:
         for wad in info.waiting.pop(fp):
             defer_result(result).chainDeferred(wad)
 
-    ### Overridable Interface
+    # Overridable Interface
     def media_to_download(self, request, info):
         """Check request before starting download"""
         pass
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3f8a7435c..c02e6a70b 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -147,9 +147,9 @@ class CrawlTestCase(TestCase):
         settings = {"CONCURRENT_REQUESTS": 1}
         crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider)
         yield crawler.crawl(mockserver=self.mockserver)
-        #self.assertTrue(False, crawler.spider.seedsseen)
-        #self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
-        #                crawler.spider.seedsseen)
+        self.assertTrue(
+            crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
+            crawler.spider.seedsseen)
 
     @defer.inlineCallbacks
     def test_start_requests_dupes(self):
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 051f66680..f8e4851fc 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -202,7 +202,7 @@ class CookiesMiddlewareTest(TestCase):
         assert self.mw.process_request(req4, self.spider) is None
         self.assertCookieValEqual(req4.headers.get('Cookie'), b'C2=value2; galleta=dulce')
 
-        #cookies from hosts with port
+        # cookies from hosts with port
         req5_1 = Request('http://scrapytest.org:1104/')
         assert self.mw.process_request(req5_1, self.spider) is None
 
@@ -218,7 +218,7 @@ class CookiesMiddlewareTest(TestCase):
         assert self.mw.process_request(req5_3, self.spider) is None
         self.assertEqual(req5_3.headers.get('Cookie'), b'C1=value1')
 
-        #skip cookie retrieval for not http request
+        # skip cookie retrieval for not http request
         req6 = Request('file:///scrapy/sometempfile')
         assert self.mw.process_request(req6, self.spider) is None
         self.assertEqual(req6.headers.get('Cookie'), None)
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index eafc3560e..522ec4875 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -438,8 +438,8 @@ class TextResponseTest(BaseResponseTest):
         assert u'<span>value</span>' in r.text, repr(r.text)
 
         # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
-        #r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
-        #assert u'\ufffd' in r.text, repr(r.text)
+        # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
+        # assert u'\ufffd' in r.text, repr(r.text)
 
     def test_selector(self):
         body = b"<html><head><title>Some page</title><body></body></html>"
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 4c6ede70b..742adc64f 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -24,7 +24,7 @@ class TestRefererMiddleware(TestCase):
     resp_headers = {}
     settings = {}
     scenarii = [
-        ('http://scrapytest.org', 'http://scrapytest.org/',  b'http://scrapytest.org'),
+        ('http://scrapytest.org', 'http://scrapytest.org/', b'http://scrapytest.org'),
     ]
 
     def setUp(self):
@@ -54,57 +54,57 @@ class MixinDefault:
     with some additional filtering of s3://
     """
     scenarii = [
-        ('https://example.com/',    'https://scrapy.org/',  b'https://example.com/'),
-        ('http://example.com/',     'http://scrapy.org/',   b'http://example.com/'),
-        ('http://example.com/',     'https://scrapy.org/',  b'http://example.com/'),
-        ('https://example.com/',    'http://scrapy.org/',   None),
+        ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'),
+        ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'),
+        ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'),
+        ('https://example.com/', 'http://scrapy.org/', None),
 
         # no credentials leak
-        ('http://user:password@example.com/',  'https://scrapy.org/', b'http://example.com/'),
+        ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'),
 
         # no referrer leak for local schemes
-        ('file:///home/path/to/somefile.html',  'https://scrapy.org/', None),
-        ('file:///home/path/to/somefile.html',  'http://scrapy.org/',  None),
+        ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None),
+        ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),
 
         # no referrer leak for s3 origins
-        ('s3://mybucket/path/to/data.csv',  'https://scrapy.org/', None),
-        ('s3://mybucket/path/to/data.csv',  'http://scrapy.org/',  None),
+        ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None),
+        ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None),
     ]
 
 
 class MixinNoReferrer:
     scenarii = [
-        ('https://example.com/page.html',       'https://example.com/', None),
-        ('http://www.example.com/',             'https://scrapy.org/',  None),
-        ('http://www.example.com/',             'http://scrapy.org/',   None),
-        ('https://www.example.com/',            'http://scrapy.org/',   None),
-        ('file:///home/path/to/somefile.html',  'http://scrapy.org/',   None),
+        ('https://example.com/page.html', 'https://example.com/', None),
+        ('http://www.example.com/', 'https://scrapy.org/', None),
+        ('http://www.example.com/', 'http://scrapy.org/', None),
+        ('https://www.example.com/', 'http://scrapy.org/', None),
+        ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),
     ]
 
 
 class MixinNoReferrerWhenDowngrade:
     scenarii = [
         # TLS to TLS: send non-empty referrer
-        ('https://example.com/page.html',       'https://not.example.com/', b'https://example.com/page.html'),
-        ('https://example.com/page.html',       'https://scrapy.org/',      b'https://example.com/page.html'),
-        ('https://example.com:443/page.html',   'https://scrapy.org/',      b'https://example.com/page.html'),
-        ('https://example.com:444/page.html',   'https://scrapy.org/',      b'https://example.com:444/page.html'),
-        ('ftps://example.com/urls.zip',         'https://scrapy.org/',      b'ftps://example.com/urls.zip'),
+        ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'),
+        ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),
+        ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),
+        ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'),
+        ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'),
 
         # TLS to non-TLS: do not send referrer
-        ('https://example.com/page.html',       'http://not.example.com/',  None),
-        ('https://example.com/page.html',       'http://scrapy.org/',       None),
-        ('ftps://example.com/urls.zip',         'http://scrapy.org/',       None),
+        ('https://example.com/page.html', 'http://not.example.com/', None),
+        ('https://example.com/page.html', 'http://scrapy.org/', None),
+        ('ftps://example.com/urls.zip', 'http://scrapy.org/', None),
 
         # non-TLS to TLS or non-TLS: send referrer
-        ('http://example.com/page.html',        'https://not.example.com/', b'http://example.com/page.html'),
-        ('http://example.com/page.html',        'https://scrapy.org/',      b'http://example.com/page.html'),
-        ('http://example.com:8080/page.html',   'https://scrapy.org/',      b'http://example.com:8080/page.html'),
-        ('http://example.com:80/page.html',     'http://not.example.com/',  b'http://example.com/page.html'),
-        ('http://example.com/page.html',        'http://scrapy.org/',       b'http://example.com/page.html'),
-        ('http://example.com:443/page.html',    'http://scrapy.org/',       b'http://example.com:443/page.html'),
-        ('ftp://example.com/urls.zip',          'http://scrapy.org/',       b'ftp://example.com/urls.zip'),
-        ('ftp://example.com/urls.zip',          'https://scrapy.org/',      b'ftp://example.com/urls.zip'),
+        ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
+        ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'),
+        ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'),
+        ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'),
+        ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'),
+        ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'),
+        ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'),
+        ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'),
 
         # test for user/password stripping
         ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
@@ -114,43 +114,43 @@ class MixinNoReferrerWhenDowngrade:
 class MixinSameOrigin:
     scenarii = [
         # Same origin (protocol, host, port): send referrer
-        ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
-        ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
+        ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
 
         # Different host: do NOT send referrer
-        ('https://example.com/page.html',       'https://not.example.com/otherpage.html',   None),
-        ('http://example.com/page.html',        'http://not.example.com/otherpage.html',    None),
-        ('http://example.com/page.html',        'http://www.example.com/otherpage.html',    None),
+        ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None),
+        ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None),
+        ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None),
 
         # Different port: do NOT send referrer
-        ('https://example.com:444/page.html',   'https://example.com/not-page.html',    None),
-        ('http://example.com:81/page.html',     'http://example.com/not-page.html',     None),
-        ('http://example.com/page.html',        'http://example.com:81/not-page.html',  None),
+        ('https://example.com:444/page.html', 'https://example.com/not-page.html', None),
+        ('http://example.com:81/page.html', 'http://example.com/not-page.html', None),
+        ('http://example.com/page.html', 'http://example.com:81/not-page.html', None),
 
         # Different protocols: do NOT send refferer
-        ('https://example.com/page.html',   'http://example.com/not-page.html',     None),
-        ('https://example.com/page.html',   'http://not.example.com/',              None),
-        ('ftps://example.com/urls.zip',     'https://example.com/not-page.html',    None),
-        ('ftp://example.com/urls.zip',      'http://example.com/not-page.html',     None),
-        ('ftps://example.com/urls.zip',     'https://example.com/not-page.html',    None),
+        ('https://example.com/page.html', 'http://example.com/not-page.html', None),
+        ('https://example.com/page.html', 'http://not.example.com/', None),
+        ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
+        ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None),
+        ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
 
         # test for user/password stripping
-        ('https://user:password@example.com/page.html', 'https://example.com/not-page.html',    b'https://example.com/page.html'),
-        ('https://user:password@example.com/page.html', 'http://example.com/not-page.html',     None),
+        ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None),
     ]
 
 
 class MixinOrigin:
     scenarii = [
         # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)
-        ('https://example.com/page.html',   'https://example.com/not-page.html',    b'https://example.com/'),
-        ('https://example.com/page.html',   'https://scrapy.org',                   b'https://example.com/'),
-        ('https://example.com/page.html',   'http://scrapy.org',                    b'https://example.com/'),
-        ('http://example.com/page.html',    'http://scrapy.org',                    b'http://example.com/'),
+        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),
+        ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
+        ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
+        ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),
 
         # test for user/password stripping
         ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
@@ -160,129 +160,129 @@ class MixinOrigin:
 class MixinStrictOrigin:
     scenarii = [
         # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades
-        ('https://example.com/page.html',   'https://example.com/not-page.html',    b'https://example.com/'),
-        ('https://example.com/page.html',   'https://scrapy.org',                   b'https://example.com/'),
-        ('http://example.com/page.html',    'http://scrapy.org',                    b'http://example.com/'),
+        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),
+        ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
+        ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),
 
         # downgrade: send nothing
-        ('https://example.com/page.html',   'http://scrapy.org',                    None),
+        ('https://example.com/page.html', 'http://scrapy.org', None),
 
         # upgrade: send origin
-        ('http://example.com/page.html',    'https://scrapy.org',                   b'http://example.com/'),
+        ('http://example.com/page.html', 'https://scrapy.org', b'http://example.com/'),
 
         # test for user/password stripping
-        ('https://user:password@example.com/page.html', 'https://scrapy.org',       b'https://example.com/'),
-        ('https://user:password@example.com/page.html', 'http://scrapy.org',        None),
+        ('https://user:password@example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
+        ('https://user:password@example.com/page.html', 'http://scrapy.org', None),
     ]
 
 
 class MixinOriginWhenCrossOrigin:
     scenarii = [
         # Same origin (protocol, host, port): send referrer
-        ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
-        ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
+        ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
 
         # Different host: send origin as referrer
-        ('https://example2.com/page.html',  'https://scrapy.org/otherpage.html',        b'https://example2.com/'),
-        ('https://example2.com/page.html',  'https://not.example2.com/otherpage.html',  b'https://example2.com/'),
-        ('http://example2.com/page.html',   'http://not.example2.com/otherpage.html',   b'http://example2.com/'),
+        ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),
+        ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),
+        ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),
         # exact match required
-        ('http://example2.com/page.html',   'http://www.example2.com/otherpage.html',   b'http://example2.com/'),
+        ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),
 
         # Different port: send origin as referrer
-        ('https://example3.com:444/page.html',  'https://example3.com/not-page.html',   b'https://example3.com:444/'),
-        ('http://example3.com:81/page.html',    'http://example3.com/not-page.html',    b'http://example3.com:81/'),
+        ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),
+        ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),
 
         # Different protocols: send origin as referrer
-        ('https://example4.com/page.html',  'http://example4.com/not-page.html',    b'https://example4.com/'),
-        ('https://example4.com/page.html',  'http://not.example4.com/',             b'https://example4.com/'),
-        ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
-        ('ftp://example4.com/urls.zip',     'http://example4.com/not-page.html',    b'ftp://example4.com/'),
-        ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
+        ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'),
+        ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'),
+        ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
+        ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),
+        ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
 
         # test for user/password stripping
-        ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html',  b'https://example5.com/page.html'),
+        ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),
         # TLS to non-TLS downgrade: send origin
-        ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html',   b'https://example5.com/'),
+        ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'),
     ]
 
 
 class MixinStrictOriginWhenCrossOrigin:
     scenarii = [
         # Same origin (protocol, host, port): send referrer
-        ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
-        ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
-        ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
-        ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+        ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
+        ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
+        ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
+        ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
 
         # Different host: send origin as referrer
-        ('https://example2.com/page.html',  'https://scrapy.org/otherpage.html',        b'https://example2.com/'),
-        ('https://example2.com/page.html',  'https://not.example2.com/otherpage.html',  b'https://example2.com/'),
-        ('http://example2.com/page.html',   'http://not.example2.com/otherpage.html',   b'http://example2.com/'),
+        ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),
+        ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),
+        ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),
         # exact match required
-        ('http://example2.com/page.html',   'http://www.example2.com/otherpage.html',   b'http://example2.com/'),
+        ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),
 
         # Different port: send origin as referrer
-        ('https://example3.com:444/page.html',  'https://example3.com/not-page.html',   b'https://example3.com:444/'),
-        ('http://example3.com:81/page.html',    'http://example3.com/not-page.html',    b'http://example3.com:81/'),
+        ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),
+        ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),
 
         # downgrade
-        ('https://example4.com/page.html',  'http://example4.com/not-page.html',    None),
-        ('https://example4.com/page.html',  'http://not.example4.com/',             None),
+        ('https://example4.com/page.html', 'http://example4.com/not-page.html', None),
+        ('https://example4.com/page.html', 'http://not.example4.com/', None),
 
         # non-TLS to non-TLS
-        ('ftp://example4.com/urls.zip',     'http://example4.com/not-page.html',    b'ftp://example4.com/'),
+        ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),
 
         # upgrade
-        ('http://example4.com/page.html',  'https://example4.com/not-page.html',    b'http://example4.com/'),
-        ('http://example4.com/page.html',  'https://not.example4.com/',             b'http://example4.com/'),
+        ('http://example4.com/page.html', 'https://example4.com/not-page.html', b'http://example4.com/'),
+        ('http://example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/'),
 
         # Different protocols: send origin as referrer
-        ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
-        ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
+        ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
+        ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
 
         # test for user/password stripping
-        ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html',  b'https://example5.com/page.html'),
+        ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),
 
         # TLS to non-TLS downgrade: send nothing
-        ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html',   None),
+        ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None),
     ]
 
 
 class MixinUnsafeUrl:
     scenarii = [
         # TLS to TLS: send referrer
-        ('https://example.com/sekrit.html',     'http://not.example.com/',      b'https://example.com/sekrit.html'),
-        ('https://example1.com/page.html',      'https://not.example1.com/',    b'https://example1.com/page.html'),
-        ('https://example1.com/page.html',      'https://scrapy.org/',          b'https://example1.com/page.html'),
-        ('https://example1.com:443/page.html',  'https://scrapy.org/',          b'https://example1.com/page.html'),
-        ('https://example1.com:444/page.html',  'https://scrapy.org/',          b'https://example1.com:444/page.html'),
-        ('ftps://example1.com/urls.zip',        'https://scrapy.org/',          b'ftps://example1.com/urls.zip'),
+        ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'),
+        ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'),
+        ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),
+        ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),
+        ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'),
+        ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'),
 
         # TLS to non-TLS: send referrer (yes, it's unsafe)
-        ('https://example2.com/page.html',  'http://not.example2.com/', b'https://example2.com/page.html'),
-        ('https://example2.com/page.html',  'http://scrapy.org/',       b'https://example2.com/page.html'),
-        ('ftps://example2.com/urls.zip',    'http://scrapy.org/',       b'ftps://example2.com/urls.zip'),
+        ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'),
+        ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'),
+        ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'),
 
         # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)
-        ('http://example3.com/page.html',       'https://not.example3.com/',    b'http://example3.com/page.html'),
-        ('http://example3.com/page.html',       'https://scrapy.org/',          b'http://example3.com/page.html'),
-        ('http://example3.com:8080/page.html',  'https://scrapy.org/',          b'http://example3.com:8080/page.html'),
-        ('http://example3.com:80/page.html',    'http://not.example3.com/',     b'http://example3.com/page.html'),
-        ('http://example3.com/page.html',       'http://scrapy.org/',           b'http://example3.com/page.html'),
-        ('http://example3.com:443/page.html',   'http://scrapy.org/',           b'http://example3.com:443/page.html'),
-        ('ftp://example3.com/urls.zip',         'http://scrapy.org/',           b'ftp://example3.com/urls.zip'),
-        ('ftp://example3.com/urls.zip',         'https://scrapy.org/',          b'ftp://example3.com/urls.zip'),
+        ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'),
+        ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'),
+        ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'),
+        ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'),
+        ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'),
+        ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'),
+        ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'),
+        ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'),
 
         # test for user/password stripping
-        ('http://user:password@example4.com/page.html',     'https://not.example4.com/',    b'http://example4.com/page.html'),
-        ('https://user:password@example4.com/page.html',    'http://scrapy.org/',           b'https://example4.com/page.html'),
+        ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'),
+        ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'),
     ]
 
 
@@ -339,12 +339,12 @@ class CustomPythonOrgPolicy(ReferrerPolicy):
 class TestSettingsCustomPolicy(TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}
     scenarii = [
-        ('https://example.com/',    'https://scrapy.org/',  b'https://python.org/'),
-        ('http://example.com/',     'http://scrapy.org/',   b'http://python.org/'),
-        ('http://example.com/',     'https://scrapy.org/',  b'https://python.org/'),
-        ('https://example.com/',    'http://scrapy.org/',   b'http://python.org/'),
-        ('file:///home/path/to/somefile.html',  'https://scrapy.org/', b'https://python.org/'),
-        ('file:///home/path/to/somefile.html',  'http://scrapy.org/',  b'http://python.org/'),
+        ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'),
+        ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'),
+        ('http://example.com/', 'https://scrapy.org/', b'https://python.org/'),
+        ('https://example.com/', 'http://scrapy.org/', b'http://python.org/'),
+        ('file:///home/path/to/somefile.html', 'https://scrapy.org/', b'https://python.org/'),
+        ('file:///home/path/to/somefile.html', 'http://scrapy.org/', b'http://python.org/'),
 
     ]
 
@@ -541,7 +541,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
 
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
     scenarii = [
-        (   'http://scrapytest.org/1',      # parent
+        (
+            'http://scrapytest.org/1',      # parent
             'http://scrapytest.org/2',      # target
             (
                 # redirections: code, URL
@@ -551,7 +552,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
             b'http://scrapytest.org/1',  # expected initial referer
             b'http://scrapytest.org/1',  # expected referer for the redirection request
         ),
-        (   'https://scrapytest.org/1',
+        (
+            'https://scrapytest.org/1',
             'https://scrapytest.org/2',
             (
                 # redirecting to non-secure URL
@@ -560,7 +562,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
             b'https://scrapytest.org/1',
             b'https://scrapytest.org/1',
         ),
-        (   'https://scrapytest.org/1',
+        (
+            'https://scrapytest.org/1',
             'https://scrapytest.com/2',
             (
                 # redirecting to non-secure URL: different origin
@@ -602,7 +605,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
     """
     settings = {'REFERRER_POLICY': 'no-referrer'}
     scenarii = [
-        (   'http://scrapytest.org/1',      # parent
+        (
+            'http://scrapytest.org/1',      # parent
             'http://scrapytest.org/2',      # target
             (
                 # redirections: code, URL
@@ -612,7 +616,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
             None,  # expected initial "Referer"
             None,  # expected "Referer" for the redirection request
         ),
-        (   'https://scrapytest.org/1',
+        (
+            'https://scrapytest.org/1',
             'https://scrapytest.org/2',
             (
                 (301, 'http://scrapytest.org/3'),
@@ -620,7 +625,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
             None,
             None,
         ),
-        (   'https://scrapytest.org/1',
+        (
+            'https://scrapytest.org/1',
             'https://example.com/2',    # different origin
             (
                 (301, 'http://scrapytest.com/3'),
@@ -641,7 +647,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
     """
     settings = {'REFERRER_POLICY': 'same-origin'}
     scenarii = [
-        (   'http://scrapytest.org/101',      # origin
+        (
+            'http://scrapytest.org/101',      # origin
             'http://scrapytest.org/102',      # target
             (
                 # redirections: code, URL
@@ -651,7 +658,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
             b'http://scrapytest.org/101',  # expected initial "Referer"
             b'http://scrapytest.org/101',  # expected referer for the redirection request
         ),
-        (   'https://scrapytest.org/201',
+        (
+            'https://scrapytest.org/201',
             'https://scrapytest.org/202',
             (
                 # redirecting from secure to non-secure URL == different origin
@@ -660,7 +668,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
             b'https://scrapytest.org/201',
             None,
         ),
-        (   'https://scrapytest.org/301',
+        (
+            'https://scrapytest.org/301',
             'https://scrapytest.org/302',
             (
                 # different domain == different origin
@@ -683,7 +692,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
     """
     settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN}
     scenarii = [
-        (   'http://scrapytest.org/101',
+        (
+            'http://scrapytest.org/101',
             'http://scrapytest.org/102',
             (
                 (301, 'http://scrapytest.org/103'),
@@ -692,7 +702,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
             b'http://scrapytest.org/',  # send origin
             b'http://scrapytest.org/',  # redirects to same origin: send origin
         ),
-        (   'https://scrapytest.org/201',
+        (
+            'https://scrapytest.org/201',
             'https://scrapytest.org/202',
             (
                 # redirecting to non-secure URL: no referrer
@@ -701,7 +712,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
             b'https://scrapytest.org/',
             None,
         ),
-        (   'https://scrapytest.org/301',
+        (
+            'https://scrapytest.org/301',
             'https://scrapytest.org/302',
             (
                 # redirecting to non-secure URL (different domain): no referrer
@@ -710,7 +722,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
             b'https://scrapytest.org/',
             None,
         ),
-        (   'http://scrapy.org/401',
+        (
+            'http://scrapy.org/401',
             'http://example.com/402',
             (
                 (301, 'http://scrapytest.org/403'),
@@ -718,7 +731,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
             b'http://scrapy.org/',
             b'http://scrapy.org/',
         ),
-        (   'https://scrapy.org/501',
+        (
+            'https://scrapy.org/501',
             'https://example.com/502',
             (
                 # HTTPS all along, so origin referrer is kept as-is
@@ -728,7 +742,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
             b'https://scrapy.org/',
             b'https://scrapy.org/',
         ),
-        (   'https://scrapytest.org/601',
+        (
+            'https://scrapytest.org/601',
             'http://scrapytest.org/602',                # TLS to non-TLS: no referrer
             (
                 (301, 'https://scrapytest.org/603'),    # TLS URL again: (still) no referrer
@@ -750,7 +765,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
     """
     settings = {'REFERRER_POLICY': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
     scenarii = [
-        (   'http://scrapytest.org/101',      # origin
+        (
+            'http://scrapytest.org/101',      # origin
             'http://scrapytest.org/102',      # target + redirection
             (
                 # redirections: code, URL
@@ -760,7 +776,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
             b'http://scrapytest.org/101',  # expected initial referer
             b'http://scrapytest.org/101',  # expected referer for the redirection request
         ),
-        (   'https://scrapytest.org/201',
+        (
+            'https://scrapytest.org/201',
             'https://scrapytest.org/202',
             (
                 # redirecting to non-secure URL: send origin
@@ -769,7 +786,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
             b'https://scrapytest.org/201',
             b'https://scrapytest.org/',
         ),
-        (   'https://scrapytest.org/301',
+        (
+            'https://scrapytest.org/301',
             'https://scrapytest.org/302',
             (
                 # redirecting to non-secure URL (different domain): send origin
@@ -778,7 +796,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
             b'https://scrapytest.org/301',
             b'https://scrapytest.org/',
         ),
-        (   'http://scrapy.org/401',
+        (
+            'http://scrapy.org/401',
             'http://example.com/402',
             (
                 (301, 'http://scrapytest.org/403'),
@@ -786,7 +805,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
             b'http://scrapy.org/',
             b'http://scrapy.org/',
         ),
-        (   'https://scrapy.org/501',
+        (
+            'https://scrapy.org/501',
             'https://example.com/502',
             (
                 # all different domains: send origin
@@ -796,7 +816,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
             b'https://scrapy.org/',
             b'https://scrapy.org/',
         ),
-        (   'https://scrapytest.org/301',
+        (
+            'https://scrapytest.org/301',
             'http://scrapytest.org/302',                # TLS to non-TLS: send origin
             (
                 (301, 'https://scrapytest.org/303'),    # TLS URL again: send origin (also)
@@ -820,7 +841,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
     """
     settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
     scenarii = [
-        (   'http://scrapytest.org/101',      # origin
+        (
+            'http://scrapytest.org/101',      # origin
             'http://scrapytest.org/102',      # target + redirection
             (
                 # redirections: code, URL
@@ -830,7 +852,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
             b'http://scrapytest.org/101',  # expected initial referer
             b'http://scrapytest.org/101',  # expected referer for the redirection request
         ),
-        (   'https://scrapytest.org/201',
+        (
+            'https://scrapytest.org/201',
             'https://scrapytest.org/202',
             (
                 # redirecting to non-secure URL: do not send the "Referer" header
@@ -839,7 +862,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
             b'https://scrapytest.org/201',
             None,
         ),
-        (   'https://scrapytest.org/301',
+        (
+            'https://scrapytest.org/301',
             'https://scrapytest.org/302',
             (
                 # redirecting to non-secure URL (different domain): send origin
@@ -848,7 +872,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
             b'https://scrapytest.org/301',
             None,
         ),
-        (   'http://scrapy.org/401',
+        (
+            'http://scrapy.org/401',
             'http://example.com/402',
             (
                 (301, 'http://scrapytest.org/403'),
@@ -856,7 +881,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
             b'http://scrapy.org/',
             b'http://scrapy.org/',
         ),
-        (   'https://scrapy.org/501',
+        (
+            'https://scrapy.org/501',
             'https://example.com/502',
             (
                 # all different domains: send origin
@@ -866,7 +892,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
             b'https://scrapy.org/',
             b'https://scrapy.org/',
         ),
-        (   'https://scrapytest.org/601',
+        (
+            'https://scrapytest.org/601',
             'http://scrapytest.org/602',                # TLS to non-TLS: do not send "Referer"
             (
                 (301, 'https://scrapytest.org/603'),    # TLS URL again: (still) send nothing
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 33fc4d570..ec8311298 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -250,10 +250,10 @@ class UtilsCsvTestCase(unittest.TestCase):
 
         result = [row for row in csv]
         self.assertEqual(result,
-                         [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
+                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
                           {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi',   u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty',   u'value': u''}])
+                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
+                          {u'id': u'4', u'name': u'empty', u'value': u''}])
 
         # explicit type check cuz' we no like stinkin' autocasting! yarrr
         for result_row in result:
@@ -266,10 +266,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response, delimiter='\t')
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
+                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
                           {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi',   u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty',   u'value': u''}])
+                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
+                          {u'id': u'4', u'name': u'empty', u'value': u''}])
 
     def test_csviter_quotechar(self):
         body1 = get_testdata('feeds', 'feed-sample6.csv')
@@ -279,19 +279,19 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv1 = csviter(response1, quotechar="'")
 
         self.assertEqual([row for row in csv1],
-                         [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
+                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
                           {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi',   u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty',   u'value': u''}])
+                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
+                          {u'id': u'4', u'name': u'empty', u'value': u''}])
 
         response2 = TextResponse(url="http://example.com/", body=body2)
         csv2 = csviter(response2, delimiter="|", quotechar="'")
 
         self.assertEqual([row for row in csv2],
-                         [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
+                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
                           {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi',   u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty',   u'value': u''}])
+                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
+                          {u'id': u'4', u'name': u'empty', u'value': u''}])
 
     def test_csviter_wrong_quotechar(self):
         body = get_testdata('feeds', 'feed-sample6.csv')
@@ -299,10 +299,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response)
 
         self.assertEqual([row for row in csv],
-                         [{u"'id'": u"1",   u"'name'": u"'alpha'",   u"'value'": u"'foobar'"},
-                          {u"'id'": u"2",   u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"},
-                          {u"'id'": u"'3'", u"'name'": u"'multi'",   u"'value'": u"'foo"},
-                          {u"'id'": u"4",   u"'name'": u"'empty'",   u"'value'": u""}])
+                         [{u"'id'": u"1", u"'name'": u"'alpha'", u"'value'": u"'foobar'"},
+                          {u"'id'": u"2", u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"},
+                          {u"'id'": u"'3'", u"'name'": u"'multi'", u"'value'": u"'foo"},
+                          {u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}])
 
     def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
         body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
@@ -310,10 +310,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response, delimiter='\t')
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
+                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
                           {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi',   u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty',   u'value': u''}])
+                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
+                          {u'id': u'4', u'name': u'empty', u'value': u''}])
 
     def test_csviter_headers(self):
         sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()
@@ -323,10 +323,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response, headers=[h.decode('utf-8') for h in headers])
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
+                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
                           {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi',   u'value': u'foo\nbar'},
-                          {u'id': u'4', u'name': u'empty',   u'value': u''}])
+                          {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'},
+                          {u'id': u'4', u'name': u'empty', u'value': u''}])
 
     def test_csviter_falserow(self):
         body = get_testdata('feeds', 'feed-sample3.csv')
@@ -336,10 +336,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response)
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
+                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
                           {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi',   u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty',   u'value': u''}])
+                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
+                          {u'id': u'4', u'name': u'empty', u'value': u''}])
 
     def test_csviter_exception(self):
         body = get_testdata('feeds', 'feed-sample3.csv')
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 7abff8281..72a16e9b1 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -203,29 +203,29 @@ def create_skipped_scheme_t(args):
 
 
 for k, args in enumerate([
-            ('/index',                              'file://'),
-            ('/index.html',                         'file://'),
-            ('./index.html',                        'file://'),
-            ('../index.html',                       'file://'),
-            ('../../index.html',                    'file://'),
-            ('./data/index.html',                   'file://'),
-            ('.hidden/data/index.html',             'file://'),
-            ('/home/user/www/index.html',           'file://'),
-            ('//home/user/www/index.html',          'file://'),
-            ('file:///home/user/www/index.html',    'file://'),
+            ('/index', 'file://'),
+            ('/index.html', 'file://'),
+            ('./index.html', 'file://'),
+            ('../index.html', 'file://'),
+            ('../../index.html', 'file://'),
+            ('./data/index.html', 'file://'),
+            ('.hidden/data/index.html', 'file://'),
+            ('/home/user/www/index.html', 'file://'),
+            ('//home/user/www/index.html', 'file://'),
+            ('file:///home/user/www/index.html', 'file://'),
 
-            ('index.html',                          'http://'),
-            ('example.com',                         'http://'),
-            ('www.example.com',                     'http://'),
-            ('www.example.com/index.html',          'http://'),
-            ('http://example.com',                  'http://'),
-            ('http://example.com/index.html',       'http://'),
-            ('localhost',                           'http://'),
-            ('localhost/index.html',                'http://'),
+            ('index.html', 'http://'),
+            ('example.com', 'http://'),
+            ('www.example.com', 'http://'),
+            ('www.example.com/index.html', 'http://'),
+            ('http://example.com', 'http://'),
+            ('http://example.com/index.html', 'http://'),
+            ('localhost', 'http://'),
+            ('localhost/index.html', 'http://'),
 
             # some corner cases (default to http://)
-            ('/',                                   'http://'),
-            ('.../test',                            'http://'),
+            ('/', 'http://'),
+            ('.../test', 'http://'),
 
         ], start=1):
     t_method = create_guess_scheme_t(args)
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 6253d5c3f..d4abebbfb 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -53,28 +53,28 @@ class ParseUrlTestCase(unittest.TestCase):
     def testParse(self):
         lip = '127.0.0.1'
         tests = (
-    ("http://127.0.0.1?c=v&c2=v2#fragment",     ('http', lip, lip, 80, '/?c=v&c2=v2')),
-    ("http://127.0.0.1/?c=v&c2=v2#fragment",    ('http', lip, lip, 80, '/?c=v&c2=v2')),
-    ("http://127.0.0.1/foo?c=v&c2=v2#frag",     ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
+    ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
+    ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
+    ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
     ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
-    ("http://127.0.0.1:100/?c=v&c2=v2#frag",    ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
+    ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
     ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
 
-    ("http://127.0.0.1",              ('http', lip, lip, 80, '/')),
-    ("http://127.0.0.1/",             ('http', lip, lip, 80, '/')),
-    ("http://127.0.0.1/foo",          ('http', lip, lip, 80, '/foo')),
-    ("http://127.0.0.1?param=value",  ('http', lip, lip, 80, '/?param=value')),
+    ("http://127.0.0.1", ('http', lip, lip, 80, '/')),
+    ("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
+    ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
+    ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
     ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
-    ("http://127.0.0.1:12345/foo",    ('http', lip + ':12345', lip, 12345, '/foo')),
-    ("http://spam:12345/foo",         ('http', 'spam:12345', 'spam', 12345, '/foo')),
-    ("http://spam.test.org/foo",      ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
+    ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
+    ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
+    ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
 
-    ("https://127.0.0.1/foo",         ('https', lip, lip, 443, '/foo')),
+    ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
     ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
-    ("https://127.0.0.1:12345/",      ('https', lip + ':12345', lip, 12345, '/')),
+    ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
 
-    ("http://scrapytest.org/foo ",    ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
-    ("http://egg:7890 ",              ('http', 'egg:7890', 'egg', 7890, '/')),
+    ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
+    ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
     )
 
         for url, test in tests:

From c3257dc610ccdd963fae8dda330fa337deb53054 Mon Sep 17 00:00:00 2001
From: santoshkosgi <santoshkosgi@gmail.com>
Date: Wed, 15 Apr 2020 17:54:10 +0530
Subject: [PATCH 2736/2974] Change Content-type to Content-Type (#4481)

Co-authored-by: santosh <santosh@acalvio.com>
---
 scrapy/responsetypes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index ad89d9d22..7c5eeac21 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -71,7 +71,7 @@ class ResponseTypes:
         cls = Response
         if b'Content-Type' in headers:
             cls = self.from_content_type(
-                content_type=headers[b'Content-type'],
+                content_type=headers[b'Content-Type'],
                 content_encoding=headers.get(b'Content-Encoding')
             )
         if cls is Response and b'Content-Disposition' in headers:

From f242751b625c4a559d9e80f62a79223f911a8551 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 15 Apr 2020 09:39:28 -0300
Subject: [PATCH 2737/2974] Remove empty line

---
 pytest.ini | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pytest.ini b/pytest.ini
index 47f34e8f1..a9014ae55 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -68,7 +68,6 @@ flake8-ignore =
     # scrapy/loader
     scrapy/loader/__init__.py E128
     # scrapy/pipelines
-
     scrapy/pipelines/files.py E116
     scrapy/pipelines/media.py E125
     # scrapy/selector

From ac869181fb9118eb3e2dd0cb938cb1c8271bf6fc Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <elacuesta@users.noreply.github.com>
Date: Wed, 15 Apr 2020 13:42:35 -0300
Subject: [PATCH 2738/2974] Update docs/topics/downloader-middleware.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/downloader-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index cea5e4564..8d3ea51f3 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -829,7 +829,7 @@ REDIRECT_MAX_TIMES
 Default: ``20``
 
 The maximum number of redirections that will be followed for a single request.
-After this maximum the request's response is returned as is.
+After this maximum, the request's response is returned as is.
 
 MetaRefreshMiddleware
 ---------------------

From 2f510fd47d8d10217f7b18b531205dd8c252eaef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 15 Apr 2020 21:10:05 +0200
Subject: [PATCH 2739/2974] Fix ShellTest.test_local_file on Windows

---
 scrapy/utils/url.py         | 13 +++++++------
 tests/test_command_shell.py |  2 +-
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index c9abb12d5..c29ed4461 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -85,11 +85,9 @@ def add_http_if_no_scheme(url):
 
 
 def guess_scheme(url):
-    """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise."""
-    parts = urlparse(url)
-    if parts.scheme:
-        return url
-    # Note: this does not match Windows filepath
+    """Add an URL scheme if missing: file:// for filepath-like input or
+    http:// otherwise."""
+    # POSIX path
     if re.match(r'''^                   # start with...
                     (
                         \.              # ...a single dot,
@@ -99,7 +97,10 @@ def guess_scheme(url):
                     )?      # optional match of ".", ".." or ".blabla"
                     /       # at least one "/" for a file path,
                     .       # and something after the "/"
-                    ''', parts.path, flags=re.VERBOSE):
+                    ''', url, flags=re.VERBOSE):
+        return any_to_uri(url)
+    # Windows drive-letter path
+    elif re.match(r'''^[a-z]:\\''', url, flags=re.IGNORECASE):
         return any_to_uri(url)
     else:
         return add_http_if_no_scheme(url)
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index d664b6ade..acf8e9f71 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -94,7 +94,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_local_file(self):
-        filepath = join(tests_datadir, 'test_site/index.html')
+        filepath = join(tests_datadir, 'test_site', 'index.html')
         _, out, _ = yield self.execute([filepath, '-c', 'item'])
         assert b'{}' in out
 

From e5b23f4b00962df76d8302ebf869ef4a4319e142 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 18 Mar 2020 11:26:59 +0300
Subject: [PATCH 2740/2974] fix #4250: add batch deliveries

---
 scrapy/extensions/feedexport.py     | 54 +++++++++++++++++++++--------
 scrapy/settings/default_settings.py |  1 +
 2 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 998d2a5d1..906f99fee 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -241,6 +241,7 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
+        self.storage_batch = self.settings.getint('FEED_STORAGE_BATCH')
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
@@ -250,19 +251,7 @@ class FeedExporter:
     def open_spider(self, spider):
         for uri, feed in self.feeds.items():
             uri = uri % self._get_uri_params(spider, feed['uri_params'])
-            storage = self._get_storage(uri)
-            file = storage.open(spider)
-            exporter = self._get_exporter(
-                file=file,
-                format=feed['format'],
-                fields_to_export=feed['fields'],
-                encoding=feed['encoding'],
-                indent=feed['indent'],
-            )
-            slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
-            self.slots.append(slot)
-            if slot.store_empty:
-                slot.start_exporting()
+            self.slots.append(self._start_new_batch(None, uri, feed, spider))
 
     def close_spider(self, spider):
         deferred_list = []
@@ -285,11 +274,48 @@ class FeedExporter:
             deferred_list.append(d)
         return defer.DeferredList(deferred_list) if deferred_list else None
 
+    def _start_new_batch(self, previous_batch_slot, uri, feed, spider):
+        """
+        Redirect the output data stream to a new file.
+        Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero.
+        """
+        if previous_batch_slot is not None:
+            previous_batch_slot.exporter.finish_exporting()
+            previous_batch_slot.storage.store(previous_batch_slot.file)
+        storage = self._get_storage(uri)
+        file = storage.open(spider)
+        exporter = self._get_exporter(
+            file=file,
+            format=feed['format'],
+            fields_to_export=feed['fields'],
+            encoding=feed['encoding'],
+            indent=feed['indent']
+        )
+        slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
+        if slot.store_empty:
+            slot.start_exporting()
+        return slot
+
+    def _get_uri_of_partial(self, slot, feed, spider):
+        """Get uri for each partial using datetime.now().isoformat()"""
+        uri = (slot.uri % self._get_uri_params(spider, feed['uri_params'])).split('.')[0] + '.'
+        uri = uri + datetime.now().isoformat() + '.' + feed['format']
+        return uri
+
     def item_scraped(self, item, spider):
-        for slot in self.slots:
+        slots = []
+        for idx, slot in enumerate(self.slots):
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
+            if self.storage_batch and slot.itemcount % self.storage_batch == 0:
+                uri = self._get_uri_of_partial(slot, self.feeds[slot.uri], spider)
+                slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider))
+                self.feeds[uri] = self.feeds[slot.uri]
+                self.feeds.pop(slot.uri)
+                self.slots[idx] = None
+        self.slots = [slot for slot in self.slots if slot is not None]
+        self.slots.extend(slots)
 
     def _load_components(self, setting_prefix):
         conf = without_none_values(self.settings.getwithbase(setting_prefix))
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 077317c81..690e044c5 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,6 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
+FEED_STORAGE_BATCH = 0
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',

From 8b4566ff93843cdf17ada069dc09261a99971d26 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 18 Mar 2020 14:21:21 +0300
Subject: [PATCH 2741/2974] fix wrong name of first file in partial deliveries

---
 scrapy/extensions/feedexport.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 906f99fee..4f7c6bf07 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -249,6 +249,8 @@ class FeedExporter:
                 raise NotConfigured
 
     def open_spider(self, spider):
+        if self.storage_batch:
+            self.feeds = {self._get_uri_of_partial(uri, feed, spider): feed for uri, feed in self.feeds.items()}
         for uri, feed in self.feeds.items():
             uri = uri % self._get_uri_params(spider, feed['uri_params'])
             self.slots.append(self._start_new_batch(None, uri, feed, spider))
@@ -296,11 +298,11 @@ class FeedExporter:
             slot.start_exporting()
         return slot
 
-    def _get_uri_of_partial(self, slot, feed, spider):
+    def _get_uri_of_partial(self, template_uri, feed, spider):
         """Get uri for each partial using datetime.now().isoformat()"""
-        uri = (slot.uri % self._get_uri_params(spider, feed['uri_params'])).split('.')[0] + '.'
-        uri = uri + datetime.now().isoformat() + '.' + feed['format']
-        return uri
+        template_uri = (template_uri % self._get_uri_params(spider, feed['uri_params']))
+        uri_name = template_uri.split('.')[0]
+        return '{}.{}.{}'.format(uri_name, datetime.now().isoformat(), feed["format"])
 
     def item_scraped(self, item, spider):
         slots = []
@@ -309,11 +311,12 @@ class FeedExporter:
             slot.exporter.export_item(item)
             slot.itemcount += 1
             if self.storage_batch and slot.itemcount % self.storage_batch == 0:
-                uri = self._get_uri_of_partial(slot, self.feeds[slot.uri], spider)
+                uri = self._get_uri_of_partial(slot.uri, self.feeds[slot.uri], spider)
                 slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider))
                 self.feeds[uri] = self.feeds[slot.uri]
                 self.feeds.pop(slot.uri)
                 self.slots[idx] = None
+
         self.slots = [slot for slot in self.slots if slot is not None]
         self.slots.extend(slots)
 

From 0723e3f4f9777a87d0df3b2e2fddfeac9099dd3b Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 19 Mar 2020 21:17:02 +0300
Subject: [PATCH 2742/2974] add batch_id, add error if uri is specified
 incorrectly

---
 scrapy/extensions/feedexport.py     | 73 ++++++++++++++++++++---------
 scrapy/settings/default_settings.py |  2 +-
 2 files changed, 52 insertions(+), 23 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 4f7c6bf07..38b25bf4a 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -180,14 +180,16 @@ class FTPFeedStorage(BlockingFeedStorage):
 
 
 class _FeedSlot:
-    def __init__(self, file, exporter, storage, uri, format, store_empty):
+    def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri):
         self.file = file
         self.exporter = exporter
         self.storage = storage
         # feed params
-        self.uri = uri
+        self.batch_id = batch_id
         self.format = format
         self.store_empty = store_empty
+        self.template_uri = template_uri
+        self.uri = uri
         # flags
         self.itemcount = 0
         self._exporting = False
@@ -241,19 +243,28 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch = self.settings.getint('FEED_STORAGE_BATCH')
+        self.storage_batch_size = self.settings.getint('FEED_STORAGE_BATCH_SIZE')
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
+            if not self._batch_deliveries_supported(uri):
+                raise NotConfigured
             if not self._exporter_supported(feed['format']):
                 raise NotConfigured
 
     def open_spider(self, spider):
-        if self.storage_batch:
-            self.feeds = {self._get_uri_of_partial(uri, feed, spider): feed for uri, feed in self.feeds.items()}
         for uri, feed in self.feeds.items():
-            uri = uri % self._get_uri_params(spider, feed['uri_params'])
-            self.slots.append(self._start_new_batch(None, uri, feed, spider))
+            batch_id = 1
+            uri_params = self._get_uri_params(spider, feed['uri_params'])
+            uri_params['batch_id'] = batch_id
+            self.slots.append(self._start_new_batch(
+                previous_batch_slot=None,
+                uri=uri % uri_params,
+                feed=feed,
+                spider=spider,
+                batch_id=batch_id,
+                template_uri=uri
+            ))
 
     def close_spider(self, spider):
         deferred_list = []
@@ -276,10 +287,17 @@ class FeedExporter:
             deferred_list.append(d)
         return defer.DeferredList(deferred_list) if deferred_list else None
 
-    def _start_new_batch(self, previous_batch_slot, uri, feed, spider):
+    def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri):
         """
         Redirect the output data stream to a new file.
         Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero.
+        :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store
+        to get the file properly closed.
+        :param uri: uri of the new batch to start
+        :param feed: dict with parameters of feed
+        :param spider: user spider
+        :param batch_id: sequential batch id starting at 1
+        :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri
         """
         if previous_batch_slot is not None:
             previous_batch_slot.exporter.finish_exporting()
@@ -293,30 +311,30 @@ class FeedExporter:
             encoding=feed['encoding'],
             indent=feed['indent']
         )
-        slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
+        slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'], batch_id, template_uri)
         if slot.store_empty:
             slot.start_exporting()
         return slot
 
-    def _get_uri_of_partial(self, template_uri, feed, spider):
-        """Get uri for each partial using datetime.now().isoformat()"""
-        template_uri = (template_uri % self._get_uri_params(spider, feed['uri_params']))
-        uri_name = template_uri.split('.')[0]
-        return '{}.{}.{}'.format(uri_name, datetime.now().isoformat(), feed["format"])
-
     def item_scraped(self, item, spider):
         slots = []
         for idx, slot in enumerate(self.slots):
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
-            if self.storage_batch and slot.itemcount % self.storage_batch == 0:
-                uri = self._get_uri_of_partial(slot.uri, self.feeds[slot.uri], spider)
-                slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider))
-                self.feeds[uri] = self.feeds[slot.uri]
-                self.feeds.pop(slot.uri)
+            if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0:
+                batch_id = slot.batch_id + 1
+                uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'])
+                uri_params['batch_id'] = batch_id
+                self.slots.append(self._start_new_batch(
+                    previous_batch_slot=slot,
+                    uri=slot.template_uri % uri_params,
+                    feed=self.feeds[slot.template_uri],
+                    spider=spider,
+                    batch_id=batch_id,
+                    template_uri=slot.template_uri
+                ))
                 self.slots[idx] = None
-
         self.slots = [slot for slot in self.slots if slot is not None]
         self.slots.extend(slots)
 
@@ -335,6 +353,17 @@ class FeedExporter:
             return True
         logger.error("Unknown feed format: %(format)s", {'format': format})
 
+    def _batch_deliveries_supported(self, uri):
+        """
+        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s
+        to distinguish different files of partial output
+        """
+        if not self.storage_batch_size:
+            return True
+        if '%(time)s' in uri or '%(batch_id)s' in uri:
+            return True
+        logger.error('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+
     def _storage_supported(self, uri):
         scheme = urlparse(uri).scheme
         if scheme in self.storages:
@@ -364,7 +393,7 @@ class FeedExporter:
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
-        ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
+        ts = datetime.utcnow().isoformat().replace(':', '-')
         params['time'] = ts
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 690e044c5..7f90a2280 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_STORAGE_BATCH = 0
+FEED_STORAGE_BATCH_SIZE = 0
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',

From d11411b402ae68874c6ccc2883836be0b9cf8326 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sat, 21 Mar 2020 10:48:13 +0300
Subject: [PATCH 2743/2974] fix comments

---
 scrapy/extensions/feedexport.py     | 31 ++++++++++++++++++-----------
 scrapy/settings/default_settings.py |  2 +-
 2 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 38b25bf4a..ab0a0de37 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -25,7 +25,6 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -243,7 +242,7 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch_size = self.settings.getint('FEED_STORAGE_BATCH_SIZE')
+        self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_SIZE', None)
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
@@ -263,7 +262,7 @@ class FeedExporter:
                 feed=feed,
                 spider=spider,
                 batch_id=batch_id,
-                template_uri=uri
+                template_uri=uri,
             ))
 
     def close_spider(self, spider):
@@ -290,7 +289,7 @@ class FeedExporter:
     def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri):
         """
         Redirect the output data stream to a new file.
-        Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero.
+        Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified.
         :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store
         to get the file properly closed.
         :param uri: uri of the new batch to start
@@ -309,9 +308,18 @@ class FeedExporter:
             format=feed['format'],
             fields_to_export=feed['fields'],
             encoding=feed['encoding'],
-            indent=feed['indent']
+            indent=feed['indent'],
+        )
+        slot = _FeedSlot(
+            file=file,
+            exporter=exporter,
+            storage=storage,
+            uri=uri,
+            format=feed['format'],
+            store_empty=feed['store_empty'],
+            batch_id=batch_id,
+            template_uri=template_uri,
         )
-        slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'], batch_id, template_uri)
         if slot.store_empty:
             slot.start_exporting()
         return slot
@@ -326,13 +334,13 @@ class FeedExporter:
                 batch_id = slot.batch_id + 1
                 uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'])
                 uri_params['batch_id'] = batch_id
-                self.slots.append(self._start_new_batch(
+                slots.append(self._start_new_batch(
                     previous_batch_slot=slot,
                     uri=slot.template_uri % uri_params,
                     feed=self.feeds[slot.template_uri],
                     spider=spider,
                     batch_id=batch_id,
-                    template_uri=slot.template_uri
+                    template_uri=slot.template_uri,
                 ))
                 self.slots[idx] = None
         self.slots = [slot for slot in self.slots if slot is not None]
@@ -358,11 +366,10 @@ class FeedExporter:
         If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s
         to distinguish different files of partial output
         """
-        if not self.storage_batch_size:
+        if self.storage_batch_size is None or '%(time)s' in uri or '%(batch_id)s' in uri:
             return True
-        if '%(time)s' in uri or '%(batch_id)s' in uri:
-            return True
-        logger.error('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+        logger.warning('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+        return False
 
     def _storage_supported(self, uri):
         scheme = urlparse(uri).scheme
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 7f90a2280..c3463a505 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_STORAGE_BATCH_SIZE = 0
+FEED_STORAGE_BATCH_SIZE = None
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',

From 39d0d13d3f7bd671d5b29646b209c62e23373fab Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 26 Mar 2020 14:18:35 +0300
Subject: [PATCH 2744/2974] Add partial deliveries tests

---
 tests/test_feedexport.py | 195 +++++++++++++++++++++++++++++++--------
 1 file changed, 159 insertions(+), 36 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c5589e52f..1ebe44e12 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -6,6 +6,7 @@ import shutil
 import string
 import tempfile
 import warnings
+from abc import ABC, abstractmethod
 from io import BytesIO
 from pathlib import Path
 from string import ascii_letters, digits
@@ -21,8 +22,9 @@ from zope.interface.verify import verifyObject
 
 import scrapy
 from scrapy.crawler import CrawlerRunner
+from scrapy.exceptions import NotConfigured
 from scrapy.exporters import CsvItemExporter
-from scrapy.extensions.feedexport import (BlockingFeedStorage, FileFeedStorage, FTPFeedStorage,
+from scrapy.extensions.feedexport import (BlockingFeedStorage, FeedExporter, FileFeedStorage, FTPFeedStorage,
                                           IFeedStorage, S3FeedStorage, StdoutFeedStorage)
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
@@ -76,6 +78,7 @@ class FTPFeedStorageTest(unittest.TestCase):
     def get_test_spider(self, settings=None):
         class TestSpider(scrapy.Spider):
             name = 'test_spider'
+
         crawler = get_crawler(settings_dict=settings)
         spider = TestSpider.from_crawler(crawler)
         return spider
@@ -129,6 +132,7 @@ class BlockingFeedStorageTest(unittest.TestCase):
     def get_test_spider(self, settings=None):
         class TestSpider(scrapy.Spider):
             name = 'test_spider'
+
         crawler = get_crawler(settings_dict=settings)
         spider = TestSpider.from_crawler(crawler)
         return spider
@@ -390,23 +394,63 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
     pass
 
 
-class FeedExportTest(unittest.TestCase):
+class FeedExportTestBase(ABC, unittest.TestCase):
+    __test__ = False
 
     class MyItem(scrapy.Item):
         foo = scrapy.Field()
         egg = scrapy.Field()
         baz = scrapy.Field()
 
+    def _random_temp_filename(self, inter_dir=''):
+        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
+        filename = ''.join(chars)
+        return os.path.join(self.temp_dir, inter_dir, filename)
+
     def setUp(self):
         self.temp_dir = tempfile.mkdtemp()
 
     def tearDown(self):
         shutil.rmtree(self.temp_dir, ignore_errors=True)
 
-    def _random_temp_filename(self):
-        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
-        filename = ''.join(chars)
-        return os.path.join(self.temp_dir, filename)
+    @defer.inlineCallbacks
+    def exported_data(self, items, settings):
+        """
+        Return exported data which a spider yielding ``items`` would return.
+        """
+
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+
+            def parse(self, response):
+                for item in items:
+                    yield item
+
+        data = yield self.run_and_export(TestSpider, settings)
+        defer.returnValue(data)
+
+    @defer.inlineCallbacks
+    def exported_no_data(self, settings):
+        """
+        Return exported data which a spider yielding no ``items`` would return.
+        """
+
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+
+            def parse(self, response):
+                pass
+
+        data = yield self.run_and_export(TestSpider, settings)
+        defer.returnValue(data)
+
+    @abstractmethod
+    def run_and_export(self, spider_cls, settings):
+        pass
+
+
+class FeedExportTest(FeedExportTestBase):
+    __test__ = True
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
@@ -417,7 +461,6 @@ class FeedExportTest(unittest.TestCase):
             urljoin('file:', pathname2url(str(file_path))): feed
             for file_path, feed in FEEDS.items()
         }
-
         content = {}
         try:
             with MockServer() as s:
@@ -435,35 +478,6 @@ class FeedExportTest(unittest.TestCase):
 
         defer.returnValue(content)
 
-    @defer.inlineCallbacks
-    def exported_data(self, items, settings):
-        """
-        Return exported data which a spider yielding ``items`` would return.
-        """
-        class TestSpider(scrapy.Spider):
-            name = 'testspider'
-
-            def parse(self, response):
-                for item in items:
-                    yield item
-
-        data = yield self.run_and_export(TestSpider, settings)
-        defer.returnValue(data)
-
-    @defer.inlineCallbacks
-    def exported_no_data(self, settings):
-        """
-        Return exported data which a spider yielding no ``items`` would return.
-        """
-        class TestSpider(scrapy.Spider):
-            name = 'testspider'
-
-            def parse(self, response):
-                pass
-
-        data = yield self.run_and_export(TestSpider, settings)
-        defer.returnValue(data)
-
     @defer.inlineCallbacks
     def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
         settings = settings or {}
@@ -970,3 +984,112 @@ class FeedExportTest(unittest.TestCase):
         }
         data = yield self.exported_no_data(settings)
         self.assertEqual(data['csv'], b'')
+
+
+class PartialDeliveriesTest(FeedExportTestBase):
+    __test__ = True
+    _file_mark = '_%(time)s_#%(batch_id)s'
+
+    @defer.inlineCallbacks
+    def run_and_export(self, spider_cls, settings):
+        """ Run spider with specified settings; return exported data. """
+
+        FEEDS = settings.get('FEEDS') or {}
+        settings['FEEDS'] = {
+            urljoin('file:', file_path): feed
+            for file_path, feed in FEEDS.items()
+        }
+        from collections import defaultdict
+        content = defaultdict(list)
+        try:
+            with MockServer() as s:
+                runner = CrawlerRunner(Settings(settings))
+                spider_cls.start_urls = [s.url('/')]
+                yield runner.crawl(spider_cls)
+
+            for path, feed in FEEDS.items():
+                dir_name = os.path.dirname(path)
+                for file in sorted(os.listdir(dir_name)):
+                    with open(os.path.join(dir_name, file), 'rb') as f:
+                        data = f.read()
+                        content[feed['format']].append(data)
+        finally:
+            pass
+        defer.returnValue(content)
+
+    @defer.inlineCallbacks
+    def assertPartialExported(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
+            },
+        })
+        data = yield self.exported_data(items, settings)
+        data['jl'] = b''.join(data['jl'])
+        parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()]
+
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        self.assertEqual(rows, parsed)
+
+    @defer.inlineCallbacks
+    def test_partial_deliveries(self):
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+            self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+            self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
+        ]
+        rows = [
+            {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
+            {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'},
+            {'foo': 'bar3', 'baz': 'quux3'}
+        ]
+        settings = {
+            'FEED_STORAGE_BATCH_SIZE': 1
+        }
+        yield self.assertPartialExported(items, rows, settings=settings)
+
+    def test_wrong_path(self):
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'xml'},
+            },
+            'FEED_STORAGE_BATCH_SIZE': 1
+        }
+        crawler = get_crawler(settings_dict=settings)
+        self.assertRaises(NotConfigured, FeedExporter, crawler)
+
+    @defer.inlineCallbacks
+    def test_export_no_items_not_store_empty(self):
+        for fmt in ('json', 'jsonlines', 'xml', 'csv'):
+            settings = {
+                'FEEDS': {
+                    os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
+                },
+                'FEED_STORAGE_BATCH_SIZE': 1
+            }
+            data = yield self.exported_no_data(settings)
+            data[fmt] = b''.join(data[fmt])
+            self.assertEqual(data[fmt], b'')
+
+    @defer.inlineCallbacks
+    def test_export_no_items_store_empty(self):
+        formats = (
+            ('json', b'[]'),
+            ('jsonlines', b''),
+            ('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
+            ('csv', b''),
+        )
+
+        for fmt, expctd in formats:
+            settings = {
+                'FEEDS': {
+                    os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
+                },
+                'FEED_STORE_EMPTY': True,
+                'FEED_EXPORT_INDENT': None,
+                'FEED_STORAGE_BATCH_SIZE': 1
+            }
+            data = yield self.exported_no_data(settings)
+            data[fmt] = b''.join(data[fmt])
+            self.assertEqual(data[fmt], expctd)

From ffa8a533e74478a5c81fbf453f2c65601bb1d244 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sat, 28 Mar 2020 11:40:16 +0300
Subject: [PATCH 2745/2974] Set batch_id in _get_uri_params

---
 scrapy/extensions/feedexport.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index ab0a0de37..06ea6c5b2 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -253,15 +253,12 @@ class FeedExporter:
 
     def open_spider(self, spider):
         for uri, feed in self.feeds.items():
-            batch_id = 1
-            uri_params = self._get_uri_params(spider, feed['uri_params'])
-            uri_params['batch_id'] = batch_id
+            uri_params = self._get_uri_params(spider, feed['uri_params'], None)
             self.slots.append(self._start_new_batch(
                 previous_batch_slot=None,
                 uri=uri % uri_params,
                 feed=feed,
                 spider=spider,
-                batch_id=batch_id,
                 template_uri=uri,
             ))
 
@@ -286,7 +283,7 @@ class FeedExporter:
             deferred_list.append(d)
         return defer.DeferredList(deferred_list) if deferred_list else None
 
-    def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri):
+    def _start_new_batch(self, previous_batch_slot, uri, feed, spider, template_uri):
         """
         Redirect the output data stream to a new file.
         Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified.
@@ -295,12 +292,15 @@ class FeedExporter:
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
-        :param batch_id: sequential batch id starting at 1
         :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri
         """
         if previous_batch_slot is not None:
+            previous_batch_id = previous_batch_slot.batch_id
             previous_batch_slot.exporter.finish_exporting()
             previous_batch_slot.storage.store(previous_batch_slot.file)
+        else:
+            previous_batch_id = 0
+
         storage = self._get_storage(uri)
         file = storage.open(spider)
         exporter = self._get_exporter(
@@ -317,7 +317,7 @@ class FeedExporter:
             uri=uri,
             format=feed['format'],
             store_empty=feed['store_empty'],
-            batch_id=batch_id,
+            batch_id=previous_batch_id + 1,
             template_uri=template_uri,
         )
         if slot.store_empty:
@@ -331,15 +331,12 @@ class FeedExporter:
             slot.exporter.export_item(item)
             slot.itemcount += 1
             if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0:
-                batch_id = slot.batch_id + 1
-                uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'])
-                uri_params['batch_id'] = batch_id
+                uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
                 slots.append(self._start_new_batch(
                     previous_batch_slot=slot,
                     uri=slot.template_uri % uri_params,
                     feed=self.feeds[slot.template_uri],
                     spider=spider,
-                    batch_id=batch_id,
                     template_uri=slot.template_uri,
                 ))
                 self.slots[idx] = None
@@ -396,12 +393,12 @@ class FeedExporter:
     def _get_storage(self, uri):
         return self._get_instance(self.storages[urlparse(uri).scheme], uri)
 
-    def _get_uri_params(self, spider, uri_params):
+    def _get_uri_params(self, spider, uri_params, slot):
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
-        ts = datetime.utcnow().isoformat().replace(':', '-')
-        params['time'] = ts
+        params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
+        params['time'] = datetime.utcnow().isoformat().replace(':', '-')
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)
         return params

From 963580463b96315eb58319e6d35b4cd52672371a Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 15 Apr 2020 20:14:33 +0300
Subject: [PATCH 2746/2974] Update tests

---
 tests/test_feedexport.py | 203 ++++++++++++++++++++++++++++++---------
 1 file changed, 159 insertions(+), 44 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 1ebe44e12..c6cd867b1 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -7,6 +7,7 @@ import string
 import tempfile
 import warnings
 from abc import ABC, abstractmethod
+from collections import defaultdict
 from io import BytesIO
 from pathlib import Path
 from string import ascii_letters, digits
@@ -444,10 +445,31 @@ class FeedExportTestBase(ABC, unittest.TestCase):
         data = yield self.run_and_export(TestSpider, settings)
         defer.returnValue(data)
 
+    @defer.inlineCallbacks
+    def assertExported(self, items, header, rows, settings=None, ordered=True):
+        yield self.assertExportedCsv(items, header, rows, settings, ordered)
+        yield self.assertExportedJsonLines(items, rows, settings)
+        yield self.assertExportedXml(items, rows, settings)
+        yield self.assertExportedPickle(items, rows, settings)
+        yield self.assertExportedMarshal(items, rows, settings)
+        yield self.assertExportedMultiple(items, rows, settings)
+
     @abstractmethod
     def run_and_export(self, spider_cls, settings):
         pass
 
+    def _load_until_eof(self, data, load_func):
+        result = []
+        with tempfile.TemporaryFile() as temp:
+            temp.write(data)
+            temp.seek(0)
+            while True:
+                try:
+                    result.append(load_func(temp))
+                except EOFError:
+                    break
+        return result
+
 
 class FeedExportTest(FeedExportTestBase):
     __test__ = True
@@ -478,6 +500,22 @@ class FeedExportTest(FeedExportTestBase):
 
         defer.returnValue(content)
 
+    @defer.inlineCallbacks
+    def exported_data(self, items, settings):
+        """
+        Return exported data which a spider yielding ``items`` would return.
+        """
+
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+
+            def parse(self, response):
+                for item in items:
+                    yield item
+
+        data = yield self.run_and_export(TestSpider, settings)
+        defer.returnValue(data)
+
     @defer.inlineCallbacks
     def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
         settings = settings or {}
@@ -543,18 +581,6 @@ class FeedExportTest(FeedExportTestBase):
         json_rows = json.loads(to_unicode(data['json']))
         self.assertEqual(rows, json_rows)
 
-    def _load_until_eof(self, data, load_func):
-        result = []
-        with tempfile.TemporaryFile() as temp:
-            temp.write(data)
-            temp.seek(0)
-            while True:
-                try:
-                    result.append(load_func(temp))
-                except EOFError:
-                    break
-        return result
-
     @defer.inlineCallbacks
     def assertExportedPickle(self, items, rows, settings=None):
         settings = settings or {}
@@ -583,15 +609,6 @@ class FeedExportTest(FeedExportTestBase):
         result = self._load_until_eof(data['marshal'], load_func=marshal.load)
         self.assertEqual(expected, result)
 
-    @defer.inlineCallbacks
-    def assertExported(self, items, header, rows, settings=None, ordered=True):
-        yield self.assertExportedCsv(items, header, rows, settings, ordered)
-        yield self.assertExportedJsonLines(items, rows, settings)
-        yield self.assertExportedXml(items, rows, settings)
-        yield self.assertExportedPickle(items, rows, settings)
-        yield self.assertExportedMarshal(items, rows, settings)
-        yield self.assertExportedMultiple(items, rows, settings)
-
     @defer.inlineCallbacks
     def test_export_items(self):
         # feed exporters use field names from Item
@@ -615,7 +632,7 @@ class FeedExportTest(FeedExportTestBase):
                 },
             }
             data = yield self.exported_no_data(settings)
-            self.assertEqual(data[fmt], b'')
+            self.assertEqual(b'', data[fmt])
 
     @defer.inlineCallbacks
     def test_export_no_items_store_empty(self):
@@ -635,7 +652,7 @@ class FeedExportTest(FeedExportTestBase):
                 'FEED_EXPORT_INDENT': None,
             }
             data = yield self.exported_no_data(settings)
-            self.assertEqual(data[fmt], expctd)
+            self.assertEqual(expctd, data[fmt])
 
     @defer.inlineCallbacks
     def test_export_multiple_item_classes(self):
@@ -734,7 +751,8 @@ class FeedExportTest(FeedExportTestBase):
         formats = {
             'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
             'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
+            'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode(
+                'utf-8'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
         }
 
@@ -751,7 +769,8 @@ class FeedExportTest(FeedExportTestBase):
         formats = {
             'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'),
             'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),
+            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode(
+                'latin-1'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
         }
 
@@ -772,7 +791,8 @@ class FeedExportTest(FeedExportTestBase):
 
         formats = {
             'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode('latin-1'),
+            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode(
+                'latin-1'),
             'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
         }
 
@@ -988,7 +1008,7 @@ class FeedExportTest(FeedExportTestBase):
 
 class PartialDeliveriesTest(FeedExportTestBase):
     __test__ = True
-    _file_mark = '_%(time)s_#%(batch_id)s'
+    _file_mark = '_%(time)s_#%(batch_id)s_'
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
@@ -999,7 +1019,6 @@ class PartialDeliveriesTest(FeedExportTestBase):
             urljoin('file:', file_path): feed
             for file_path, feed in FEEDS.items()
         }
-        from collections import defaultdict
         content = defaultdict(list)
         try:
             with MockServer() as s:
@@ -1014,26 +1033,120 @@ class PartialDeliveriesTest(FeedExportTestBase):
                         data = f.read()
                         content[feed['format']].append(data)
         finally:
-            pass
+            self.tearDown()
         defer.returnValue(content)
 
     @defer.inlineCallbacks
-    def assertPartialExported(self, items, rows, settings=None):
+    def assertExportedJsonLines(self, items, rows, settings=None):
         settings = settings or {}
         settings.update({
             'FEEDS': {
                 os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
             },
         })
-        data = yield self.exported_data(items, settings)
-        data['jl'] = b''.join(data['jl'])
-        parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()]
-
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
-        self.assertEqual(rows, parsed)
+        data = yield self.exported_data(items, settings)
+        for batch in data['jl']:
+            got_batch = [json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines()]
+            expected_batch, rows = rows[:batch_size], rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
 
     @defer.inlineCallbacks
-    def test_partial_deliveries(self):
+    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
+            },
+        })
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        data = yield self.exported_data(items, settings)
+        for batch in data['csv']:
+            got_batch = csv.DictReader(to_unicode(batch).splitlines())
+            self.assertEqual(list(header), got_batch.fieldnames)
+            expected_batch, rows = rows[:batch_size], rows[batch_size:]
+            self.assertEqual(expected_batch, list(got_batch))
+
+    @defer.inlineCallbacks
+    def assertExportedXml(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
+            },
+        })
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        data = yield self.exported_data(items, settings)
+        for batch in data['xml']:
+            root = lxml.etree.fromstring(batch)
+            got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')]
+            expected_batch, rows = rows[:batch_size], rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
+
+    @defer.inlineCallbacks
+    def assertExportedMultiple(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
+                os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
+            },
+        })
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        data = yield self.exported_data(items, settings)
+        # XML
+        xml_rows = rows.copy()
+        for batch in data['xml']:
+            root = lxml.etree.fromstring(batch)
+            got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')]
+            expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
+        # JSON
+        json_rows = rows.copy()
+        for batch in data['json']:
+            got_batch = json.loads(batch)
+            expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
+
+    @defer.inlineCallbacks
+    def assertExportedPickle(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
+            },
+        })
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        data = yield self.exported_data(items, settings)
+        import pickle
+        for batch in data['pickle']:
+            got_batch = self._load_until_eof(batch, load_func=pickle.load)
+            expected_batch, rows = rows[:batch_size], rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
+
+    @defer.inlineCallbacks
+    def assertExportedMarshal(self, items, rows, settings=None):
+        settings = settings or {}
+        settings.update({
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
+            },
+        })
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        rows = [{k: v for k, v in row.items() if v} for row in rows]
+        data = yield self.exported_data(items, settings)
+        import marshal
+        for batch in data['marshal']:
+            got_batch = self._load_until_eof(batch, load_func=marshal.load)
+            expected_batch, rows = rows[:batch_size], rows[batch_size:]
+            self.assertEqual(expected_batch, got_batch)
+
+    @defer.inlineCallbacks
+    def test_export_items(self):
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
             self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
@@ -1042,14 +1155,16 @@ class PartialDeliveriesTest(FeedExportTestBase):
         rows = [
             {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
             {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'},
-            {'foo': 'bar3', 'baz': 'quux3'}
+            {'foo': 'bar3', 'baz': 'quux3', 'egg': ''}
         ]
         settings = {
-            'FEED_STORAGE_BATCH_SIZE': 1
+            'FEED_STORAGE_BATCH_SIZE': 2
         }
-        yield self.assertPartialExported(items, rows, settings=settings)
+        header = self.MyItem.fields.keys()
+        yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
+        """If path without %(time)s or %(batch_id)s an exception must be raised"""
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
@@ -1069,8 +1184,8 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 'FEED_STORAGE_BATCH_SIZE': 1
             }
             data = yield self.exported_no_data(settings)
-            data[fmt] = b''.join(data[fmt])
-            self.assertEqual(data[fmt], b'')
+            data = dict(data)
+            self.assertEqual(b'', data[fmt][0])
 
     @defer.inlineCallbacks
     def test_export_no_items_store_empty(self):
@@ -1088,8 +1203,8 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 },
                 'FEED_STORE_EMPTY': True,
                 'FEED_EXPORT_INDENT': None,
-                'FEED_STORAGE_BATCH_SIZE': 1
+                'FEED_STORAGE_BATCH_SIZE': 1,
             }
             data = yield self.exported_no_data(settings)
-            data[fmt] = b''.join(data[fmt])
-            self.assertEqual(data[fmt], expctd)
+            data = dict(data)
+            self.assertEqual(expctd, data[fmt][0])

From 47a992615a046d20c31bfca6f8b65c3194e9fd30 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Wed, 15 Apr 2020 19:57:34 -0300
Subject: [PATCH 2747/2974] serialize requests with callback references as
 spider attribute

You could define a spider attribute that references a callback method
but if this method has a different name than your spider attribute,
the request serializer is not able to find it on the spider class.

With this commit we're fixing this behavior as we're searching for
callback references in the spider object itself instead of looking
for attributes with the same function's name, that could be different.
---
 scrapy/utils/reqser.py     | 12 ++++++++----
 tests/test_utils_reqser.py | 40 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 749bbc387..78e13ec10 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -1,6 +1,8 @@
 """
 Helper functions for serializing (and deserializing) requests.
 """
+import inspect
+
 from scrapy.http import Request
 from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
@@ -90,10 +92,12 @@ def _find_method(obj, func):
             pass
         else:
             if func_self is obj:
-                name = func.__func__.__name__
-                if _is_private_method(name):
-                    return _mangle_private_name(obj, func, name)
-                return name
+                members = inspect.getmembers(obj, predicate=inspect.ismethod)
+                for name, obj_func in members:
+                    if obj_func.__func__ is func.__func__:
+                        if _is_private_method(name):
+                            return _mangle_private_name(obj, func, name)
+                        return name
     raise ValueError("Function %s is not a method of: %s" % (func, obj))
 
 
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index c7572f02c..cf84f8fbd 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -69,6 +69,26 @@ class RequestSerializationTest(unittest.TestCase):
                     errback=self.spider.handle_error)
         self._assert_serializes_ok(r, spider=self.spider)
 
+    def test_reference_callback_serialization(self):
+        r = Request("http://www.example.com",
+                    callback=self.spider.parse_item_reference,
+                    errback=self.spider.handle_error_reference)
+        self._assert_serializes_ok(r, spider=self.spider)
+        request_dict = request_to_dict(r, self.spider)
+        self.assertEqual(request_dict['callback'], 'parse_item_reference')
+        self.assertEqual(request_dict['errback'], 'handle_error_reference')
+
+    def test_private_reference_callback_serialization(self):
+        r = Request("http://www.example.com",
+                    callback=self.spider._TestSpider__parse_item_reference,
+                    errback=self.spider._TestSpider__handle_error_reference)
+        self._assert_serializes_ok(r, spider=self.spider)
+        request_dict = request_to_dict(r, self.spider)
+        self.assertEqual(request_dict['callback'],
+                         '_TestSpider__parse_item_reference')
+        self.assertEqual(request_dict['errback'],
+                         '_TestSpider__handle_error_reference')
+
     def test_private_callback_serialization(self):
         r = Request("http://www.example.com",
                     callback=self.spider._TestSpider__parse_item_private,
@@ -131,8 +151,28 @@ class TestSpiderMixin:
         pass
 
 
+def parse_item(response):
+    pass
+
+
+def handle_error(failure):
+    pass
+
+
+def private_parse_item(response):
+    pass
+
+
+def private_handle_error(failure):
+    pass
+
+
 class TestSpider(Spider, TestSpiderMixin):
     name = 'test'
+    parse_item_reference = parse_item
+    handle_error_reference = handle_error
+    __parse_item_reference = private_parse_item
+    __handle_error_reference = private_handle_error
 
     def parse_item(self, response):
         pass

From cac1f3a6adedc32977e0fb1830917a5e7d758bef Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 16 Apr 2020 10:06:56 +0300
Subject: [PATCH 2748/2974] Update documentation

---
 docs/topics/feed-exports.rst | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 9e5968a29..0bba03a7c 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -220,6 +220,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_STORAGE_FTP_ACTIVE`
  * :setting:`FEED_STORAGE_S3_ACL`
  * :setting:`FEED_EXPORTERS`
+ * :setting:`FEED_EXPORT_BATCH_SIZE`
 
 .. currentmodule:: scrapy.extensions.feedexport
 
@@ -429,3 +430,37 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. _Amazon S3: https://aws.amazon.com/s3/
 .. _botocore: https://github.com/boto/botocore
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+
+.. setting:: FEED_EXPORT_BATCH_SIZE
+
+FEED_EXPORT_BATCH_SIZE
+----------------------
+Default: ``None``
+
+An integer number which represent number of scraped items stored in each output
+file. Whenever the number of items exceeds this setting, a new file
+creates and output redirects to it.  The name of the new file will be selected
+based on timestamp when the feed is being created and/or batch sequence number.
+Therefore you must specify %(time)s or %(batch_id)s or both in the file path.
+
+* ``%(time)s`` - gets replaced by a timestamp when the feed is being created
+* ``%(batch_id)s`` - gets replaced by sequence number of batch
+
+For instance::
+
+    FEED_EXPORT_BATCH_SIZE=100
+
+Your request can be like::
+
+  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time)s.json
+
+The result directory tree of above can be like::
+
+->projectname
+-->dirname
+--->1-filename2020-03-28T14-45-08.237134.json
+--->2-filename2020-03-28T14-45-09.148903.json
+--->3-filename2020-03-28T14-45-10.046092.json
+
+Where first and second files contain exactly 100 items. The last one contains
+<= 100 items.
\ No newline at end of file

From 5980ae72c6cb177f47fbb41d17837e8d98d50025 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 16 Apr 2020 10:13:39 +0300
Subject: [PATCH 2749/2974] Some minor fixes and refactoring

---
 tests/test_feedexport.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 60e19d1df..e97e50e8e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -428,7 +428,7 @@ class FeedExportTestBase(ABC, unittest.TestCase):
                     yield item
 
         data = yield self.run_and_export(TestSpider, settings)
-        defer.returnValue(data)
+        return data
 
     @defer.inlineCallbacks
     def exported_no_data(self, settings):
@@ -443,7 +443,7 @@ class FeedExportTestBase(ABC, unittest.TestCase):
                 pass
 
         data = yield self.run_and_export(TestSpider, settings)
-        defer.returnValue(data)
+        return data
 
     @defer.inlineCallbacks
     def assertExported(self, items, header, rows, settings=None, ordered=True):
@@ -735,8 +735,7 @@ class FeedExportTest(FeedExportTestBase):
         formats = {
             'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
             'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode(
-                'utf-8'),
+            'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
         }
 
@@ -753,8 +752,7 @@ class FeedExportTest(FeedExportTestBase):
         formats = {
             'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'),
             'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode(
-                'latin-1'),
+            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
         }
 
@@ -775,8 +773,7 @@ class FeedExportTest(FeedExportTestBase):
 
         formats = {
             'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
-            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode(
-                'latin-1'),
+            'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode('latin-1'),
             'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
         }
 
@@ -1148,7 +1145,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
-        """If path without %(time)s or %(batch_id)s an exception must be raised"""
+        """If path is without %(time)s or %(batch_id)s an exception must be raised"""
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},

From 901892dab380d54186ae855bf65a20eb2467de04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 16 Apr 2020 14:48:38 +0200
Subject: [PATCH 2750/2974] Fix the hoverxref configuration

---
 docs/conf.py | 2 --
 tox.ini      | 6 ++++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 4414ef637..813417bae 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -295,8 +295,6 @@ intersphinx_mapping = {
 # ------------------------------------
 
 hoverxref_auto_ref = True
-hoverxref_project = "scrapy"
-hoverxref_version = release
 hoverxref_role_types = {
     "class": "tooltip",
     "confval": "tooltip",
diff --git a/tox.ini b/tox.ini
index b1babc7fd..cd118c921 100644
--- a/tox.ini
+++ b/tox.ini
@@ -74,11 +74,15 @@ deps =
 changedir = docs
 deps =
     -rdocs/requirements.txt
+setenv =
+    READTHEDOCS_PROJECT=scrapy
+    READTHEDOCS_VERSION=master
 
 [testenv:docs]
 basepython = python3
 changedir = {[docs]changedir}
 deps = {[docs]deps}
+setenv = {[docs]setenv}
 commands =
     sphinx-build -W -b html . {envtmpdir}/html
 
@@ -86,6 +90,7 @@ commands =
 basepython = python3
 changedir = {[docs]changedir}
 deps = {[docs]deps}
+setenv = {[docs]setenv}
 commands =
     sphinx-build -b coverage . {envtmpdir}/coverage
 
@@ -93,6 +98,7 @@ commands =
 basepython = python3
 changedir = {[docs]changedir}
 deps = {[docs]deps}
+setenv = {[docs]setenv}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
 

From 94ee68695a42ec8d3ccdff71bfc2fa33d5ea7049 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 16 Apr 2020 15:36:56 +0200
Subject: [PATCH 2751/2974] Mock server: use 127.0.0.1 also for HTTPS

Windows throws an error about 0.0.0.0 being external:
https://stackoverflow.com/a/23857995/939364
---
 tests/mockserver.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index a45277db9..e3dbdcc68 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -218,9 +218,8 @@ class MockServer():
         self.proc.communicate()
 
     def url(self, path, is_secure=False):
-        host = self.http_address.replace('0.0.0.0', '127.0.0.1')
-        if is_secure:
-            host = self.https_address
+        host = self.https_address if is_secure else self.http_address
+        host = host.replace('0.0.0.0', '127.0.0.1')
         return host + path
 
 

From e0921cab667a1fefe9730363ed45091edf1250e8 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Thu, 16 Apr 2020 11:18:56 -0300
Subject: [PATCH 2752/2974] remove not used code

This code is not needed anymore because we're getting the
already mangled name when matching func with spider attributes.
---
 scrapy/utils/reqser.py     | 16 ----------------
 tests/test_utils_reqser.py | 37 +------------------------------------
 2 files changed, 1 insertion(+), 52 deletions(-)

diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 78e13ec10..1392b2c61 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -70,20 +70,6 @@ def request_from_dict(d, spider=None):
     )
 
 
-def _is_private_method(name):
-    return name.startswith('__') and not name.endswith('__')
-
-
-def _mangle_private_name(obj, func, name):
-    qualname = getattr(func, '__qualname__', None)
-    if qualname is None:
-        classname = obj.__class__.__name__.lstrip('_')
-        return '_%s%s' % (classname, name)
-    else:
-        splits = qualname.split('.')
-        return '_%s%s' % (splits[-2], splits[-1])
-
-
 def _find_method(obj, func):
     if obj:
         try:
@@ -95,8 +81,6 @@ def _find_method(obj, func):
                 members = inspect.getmembers(obj, predicate=inspect.ismethod)
                 for name, obj_func in members:
                     if obj_func.__func__ is func.__func__:
-                        if _is_private_method(name):
-                            return _mangle_private_name(obj, func, name)
                         return name
     raise ValueError("Function %s is not a method of: %s" % (func, obj))
 
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index cf84f8fbd..47853d812 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -2,7 +2,7 @@ import unittest
 
 from scrapy.http import Request, FormRequest
 from scrapy.spiders import Spider
-from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method, _mangle_private_name
+from scrapy.utils.reqser import request_to_dict, request_from_dict
 
 
 class RequestSerializationTest(unittest.TestCase):
@@ -101,41 +101,6 @@ class RequestSerializationTest(unittest.TestCase):
                     errback=self.spider.handle_error)
         self._assert_serializes_ok(r, spider=self.spider)
 
-    def test_private_callback_name_matching(self):
-        self.assertTrue(_is_private_method('__a'))
-        self.assertTrue(_is_private_method('__a_'))
-        self.assertTrue(_is_private_method('__a_a'))
-        self.assertTrue(_is_private_method('__a_a_'))
-        self.assertTrue(_is_private_method('__a__a'))
-        self.assertTrue(_is_private_method('__a__a_'))
-        self.assertTrue(_is_private_method('__a___a'))
-        self.assertTrue(_is_private_method('__a___a_'))
-        self.assertTrue(_is_private_method('___a'))
-        self.assertTrue(_is_private_method('___a_'))
-        self.assertTrue(_is_private_method('___a_a'))
-        self.assertTrue(_is_private_method('___a_a_'))
-        self.assertTrue(_is_private_method('____a_a_'))
-
-        self.assertFalse(_is_private_method('_a'))
-        self.assertFalse(_is_private_method('_a_'))
-        self.assertFalse(_is_private_method('__a__'))
-        self.assertFalse(_is_private_method('__'))
-        self.assertFalse(_is_private_method('___'))
-        self.assertFalse(_is_private_method('____'))
-
-    def _assert_mangles_to(self, obj, name):
-        func = getattr(obj, name)
-        self.assertEqual(
-            _mangle_private_name(obj, func, func.__name__),
-            name
-        )
-
-    def test_private_name_mangling(self):
-        self._assert_mangles_to(
-            self.spider, '_TestSpider__parse_item_private')
-        self._assert_mangles_to(
-            self.spider, '_TestSpiderMixin__mixin_callback')
-
     def test_unserializable_callback1(self):
         r = Request("http://www.example.com", callback=lambda x: x)
         self.assertRaises(ValueError, request_to_dict, r)

From 94c95020b391c3298f4a7fd7608d48d74117bf43 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Thu, 16 Apr 2020 11:37:03 -0300
Subject: [PATCH 2753/2974] add comment to explain the use of __func__ instead
 of instance method objects

---
 scrapy/utils/reqser.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 1392b2c61..5ea2aafb8 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -80,6 +80,13 @@ def _find_method(obj, func):
             if func_self is obj:
                 members = inspect.getmembers(obj, predicate=inspect.ismethod)
                 for name, obj_func in members:
+                    # We need to use __func__ to access the original
+                    # function object because instance method objects
+                    # are generated each time attribute is retrieved from
+                    # instance.
+                    #
+                    # Reference: The standard type hierarchy
+                    # https://docs.python.org/3/reference/datamodel.html
                     if obj_func.__func__ is func.__func__:
                         return name
     raise ValueError("Function %s is not a method of: %s" % (func, obj))

From c9229922772a4d7f92a26786d6ea441609043a09 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 16 Apr 2020 11:37:37 -0300
Subject: [PATCH 2754/2974] Tests: Move code inside __main__ block

---
 tests/CrawlerRunner/ip_address.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py
index 5a71536d8..826374cd4 100644
--- a/tests/CrawlerRunner/ip_address.py
+++ b/tests/CrawlerRunner/ip_address.py
@@ -23,15 +23,16 @@ class LocalhostSpider(Spider):
         self.logger.info("IP address: %s" % response.ip_address)
 
 
-with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server:
-    port = urlparse(mock_http_server.http_address).port
-    url = "http://not.a.real.domain:{port}/echo".format(port=port)
+if __name__ == "__main__":
+    with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server:
+        port = urlparse(mock_http_server.http_address).port
+        url = "http://not.a.real.domain:{port}/echo".format(port=port)
 
-    servers = [(mock_dns_server.host, mock_dns_server.port)]
-    reactor.installResolver(createResolver(servers=servers))
+        servers = [(mock_dns_server.host, mock_dns_server.port)]
+        reactor.installResolver(createResolver(servers=servers))
 
-    configure_logging()
-    runner = CrawlerRunner()
-    d = runner.crawl(LocalhostSpider, url=url)
-    d.addBoth(lambda _: reactor.stop())
-    reactor.run()
+        configure_logging()
+        runner = CrawlerRunner()
+        d = runner.crawl(LocalhostSpider, url=url)
+        d.addBoth(lambda _: reactor.stop())
+        reactor.run()

From 7cc9601029274124804e63ebebfa8783ac175205 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 16 Apr 2020 16:57:48 +0200
Subject: [PATCH 2755/2974] Improve reporting on
 test_ipv6_alternative_name_resolver

---
 tests/test_crawler.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 169e763f0..a6b079395 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -309,14 +309,8 @@ class CrawlerProcessSubprocess(unittest.TestCase):
     def test_ipv6_alternative_name_resolver(self):
         log = self.run_script('alternative_name_resolver.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertTrue(any([
-            "twisted.internet.error.ConnectionRefusedError" in log,
-            "twisted.internet.error.ConnectError" in log,
-        ]))
-        self.assertTrue(any([
-            "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
-            "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
-        ]))
+        self.assertRegex(log, r"twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)")
+        self.assertRegex(log, r"'downloader/exception_type_count/twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)': 1,")
 
     def test_reactor_select(self):
         log = self.run_script("twisted_reactor_select.py")

From cf4180308982d9cd017167238a3df8de7900646d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 16 Apr 2020 17:07:29 +0200
Subject: [PATCH 2756/2974] Skip test_reactor_poll on Windows

---
 tests/test_crawler.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index a6b079395..3d166e14c 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,5 +1,6 @@
 import logging
 import os
+import platform
 import subprocess
 import sys
 import warnings
@@ -317,6 +318,7 @@ class CrawlerProcessSubprocess(unittest.TestCase):
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
 
+    @mark.skipif(platform.system() == 'Windows', reason="PollReactor is not supported on Windows")
     def test_reactor_poll(self):
         log = self.run_script("twisted_reactor_poll.py")
         self.assertIn("Spider closed (finished)", log)

From ea3e675801fe41c7517c58e21b46831940fbd064 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 16 Apr 2020 17:10:45 +0200
Subject: [PATCH 2757/2974] test_utils_iterators: use os.linesep

---
 tests/test_utils_iterators.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 33fc4d570..5b0073fd1 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -8,7 +8,7 @@ from scrapy.http import XmlResponse, TextResponse, Response
 from tests import get_testdata
 
 
-FOOBAR_NL = u"foo\nbar"
+FOOBAR_NL = "foo{}bar".format(os.linesep)
 
 
 class XmliterTestCase(unittest.TestCase):

From 1ade3fc723d1e5d7b6a3300b454d32656bdb8d28 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Fri, 17 Apr 2020 10:34:34 -0300
Subject: [PATCH 2758/2974] trying to improve test coverage

---
 tests/test_utils_reqser.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 47853d812..50b026d1c 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -110,6 +110,21 @@ class RequestSerializationTest(unittest.TestCase):
         r = Request("http://www.example.com", callback=self.spider.parse_item)
         self.assertRaises(ValueError, request_to_dict, r)
 
+    def test_unserializable_callback3(self):
+        """Parser method is removed or replaced dynamically."""
+
+        class MySpider(Spider):
+
+            name = 'my_spider'
+
+            def parse(self, response):
+                pass
+
+        spider = MySpider()
+        r = Request("http://www.example.com", callback=spider.parse)
+        setattr(spider, 'parse', None)
+        self.assertRaises(ValueError, request_to_dict, r, spider=spider)
+
 
 class TestSpiderMixin:
     def __mixin_callback(self, response):

From 04b6295a69174e81beceb0b1429fa3775949e99d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 17 Apr 2020 20:50:17 -0300
Subject: [PATCH 2759/2974] Docs: replace deprecated FEED_* settings

---
 docs/topics/practices.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index e3e8fdc72..cf1de1bd1 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -35,8 +35,9 @@ Here's an example showing how to run a single spider with it.
         ...
 
     process = CrawlerProcess(settings={
-        'FEED_FORMAT': 'json',
-        'FEED_URI': 'items.json'
+        "FEEDS": {
+            "items.json": {"format": "json"},
+        },
     })
 
     process.crawl(MySpider)

From ec76445dd6753074c1531571f66467eecf22b498 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sat, 18 Apr 2020 09:29:23 +0300
Subject: [PATCH 2760/2974] Update tests

---
 tests/test_feedexport.py | 66 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 65 insertions(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e97e50e8e..8e03a91c8 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1128,6 +1128,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_items(self):
+        """ Test partial deliveries in all supported formats """
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
             self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
@@ -1145,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
-        """If path is without %(time)s or %(batch_id)s an exception must be raised"""
+        """ If path is without %(time)s or %(batch_id)s an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
@@ -1189,3 +1190,66 @@ class PartialDeliveriesTest(FeedExportTestBase):
             data = yield self.exported_no_data(settings)
             data = dict(data)
             self.assertEqual(expctd, data[fmt][0])
+
+    @defer.inlineCallbacks
+    def test_export_multiple_configs(self):
+        items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
+
+        formats = {
+            'json': [u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
+                     u'[\n{"bar": "BAR1"}\n]'.encode('utf-8')],
+            'xml': [u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode('latin-1'),
+                    u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>'.encode('latin-1')],
+            'csv': [u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
+                    u'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],
+        }
+
+        settings = {
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'json', self._file_mark): {
+                    'format': 'json',
+                    'indent': 0,
+                    'fields': ['bar'],
+                    'encoding': 'utf-8',
+                },
+                os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {
+                    'format': 'xml',
+                    'indent': 2,
+                    'fields': ['foo'],
+                    'encoding': 'latin-1',
+                },
+                os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
+                    'format': 'csv',
+                    'indent': None,
+                    'fields': ['bar', 'foo'],
+                    'encoding': 'utf-8',
+                },
+            },
+            'FEED_STORAGE_BATCH_SIZE': 1,
+        }
+        data = yield self.exported_data(items, settings)
+        for fmt, expected in formats.items():
+            for expected_batch, got_batch in zip(expected, data[fmt]):
+                self.assertEqual(expected_batch, got_batch)
+
+    @defer.inlineCallbacks
+    def test_batch_path_differ(self):
+        """
+        Test that the name of all batch files differ from each other.
+        So %(time)s replaced with the current date.
+        """
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+            self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+            self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
+        ]
+        settings = {
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), '%(time)s'): {
+                    'format': 'json',
+                },
+            },
+            'FEED_STORAGE_BATCH_SIZE': 1,
+        }
+        data = yield self.exported_data(items, settings)
+        self.assertEqual(len(items) + 1, len(data['json']))

From bfeb2c8c13de0c45af21228f69395a1131913da5 Mon Sep 17 00:00:00 2001
From: sakshamb2113 <44064539+sakshamb2113@users.noreply.github.com>
Date: Sat, 18 Apr 2020 20:51:26 +0530
Subject: [PATCH 2761/2974] Added warning to use double quotes in Windows for
 scrapy shell in shell.rst (#4450)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* modified debugging memory leaks with guppy in leaks.rst

* modified leaks.rst(issue #4285)

* removed guppy from telnet.py

* Fix undefined name error

* removed hpy key from telnet_vars in telnet.py

* updated shell.rst

* Update docs/topics/shell.rst

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/shell.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 8f7518b19..0f46f1c87 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -156,6 +156,17 @@ First, we launch the shell::
 
     scrapy shell 'https://scrapy.org' --nolog
 
+.. note::
+
+   Remember to always enclose URLs in quotes when running the Scrapy shell from
+   the command line, otherwise URLs containing arguments (i.e. the ``&`` character)
+   will not work.
+
+   On Windows, use double quotes instead::
+
+       scrapy shell "https://scrapy.org" --nolog
+
+
 Then, the shell fetches the URL (using the Scrapy downloader) and prints the
 list of available objects and useful shortcuts (you'll notice that these lines
 all start with the ``[s]`` prefix)::

From 773ddf77ba36613a90d35e583b29e9020cc2aace Mon Sep 17 00:00:00 2001
From: ilias-ant <ilias.antonopoulos@yahoo.gr>
Date: Sun, 19 Apr 2020 14:14:17 +0300
Subject: [PATCH 2762/2974] added more tests to cover the (downloaded,
 uptodate, cached) status functionality

---
 tests/test_pipeline_files.py | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index f155db4ce..9eb6462f2 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -84,6 +84,7 @@ class FilesPipelineTestCase(unittest.TestCase):
 
         result = yield self.pipeline.process_item(item, None)
         self.assertEqual(result['files'][0]['checksum'], 'abc')
+        self.assertEqual(result['files'][0]['status'], 'uptodate')
 
         for p in patchers:
             p.stop()
@@ -105,6 +106,29 @@ class FilesPipelineTestCase(unittest.TestCase):
 
         result = yield self.pipeline.process_item(item, None)
         self.assertNotEqual(result['files'][0]['checksum'], 'abc')
+        self.assertEqual(result['files'][0]['status'], 'downloaded')
+
+        for p in patchers:
+            p.stop()
+
+    @defer.inlineCallbacks
+    def test_file_cached(self):
+        item_url = "http://example.com/file3.pdf"
+        item = _create_item_with_files(item_url)
+        patchers = [
+            mock.patch.object(FilesPipeline, 'inc_stats', return_value=True),
+            mock.patch.object(FSFilesStore, 'stat_file', return_value={
+                'checksum': 'abc',
+                'last_modified': time.time() - (self.pipeline.expires * 60 * 60 * 24 * 2)}),
+            mock.patch.object(FilesPipeline, 'get_media_requests',
+                              return_value=[_prepare_request_object(item_url, flags=['cached'])])
+        ]
+        for p in patchers:
+            p.start()
+
+        result = yield self.pipeline.process_item(item, None)
+        self.assertNotEqual(result['files'][0]['checksum'], 'abc')
+        self.assertEqual(result['files'][0]['status'], 'cached')
 
         for p in patchers:
             p.stop()
@@ -403,10 +427,10 @@ def _create_item_with_files(*files):
     return item
 
 
-def _prepare_request_object(item_url):
+def _prepare_request_object(item_url, flags=None):
     return Request(
         item_url,
-        meta={'response': Response(item_url, status=200, body=b'data')})
+        meta={'response': Response(item_url, status=200, body=b'data', flags=flags)})
 
 
 if __name__ == "__main__":

From 1fecacbb1a3c2c4c61c202561f9c51f7f6b191ba Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 20 Apr 2020 12:05:15 -0300
Subject: [PATCH 2763/2974] IPv6 test: check for the absence of DNSLookupError

---
 tests/test_crawler.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 3d166e14c..d6756c266 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -310,8 +310,7 @@ class CrawlerProcessSubprocess(unittest.TestCase):
     def test_ipv6_alternative_name_resolver(self):
         log = self.run_script('alternative_name_resolver.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertRegex(log, r"twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)")
-        self.assertRegex(log, r"'downloader/exception_type_count/twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)': 1,")
+        self.assertNotIn("twisted.internet.error.DNSLookupError", log)
 
     def test_reactor_select(self):
         log = self.run_script("twisted_reactor_select.py")

From e4750f2fbdacbeb7a20ae7c6b13bba3fb0f7ad54 Mon Sep 17 00:00:00 2001
From: Aditya Kumar <k.aditya00@gmail.com>
Date: Mon, 20 Apr 2020 21:17:57 +0530
Subject: [PATCH 2764/2974] async/deferred signal handlers (#4390)

* [docs] async/deferred signal handlers

* [docs] update deferred signals example

* [docs] add subsections for built-in signals

* docs(signals): update signal handler example

* docs(signals): update signal handler example
---
 docs/topics/signals.rst | 96 +++++++++++++++++++++++++++++++++--------
 1 file changed, 77 insertions(+), 19 deletions(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 2def53848..8661f86a0 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -16,8 +16,7 @@ deliver the arguments that the handler receives.
 You can connect to signals (or send your own) through the
 :ref:`topics-api-signals`.
 
-Here is a simple example showing how you can catch signals and perform some action:
-::
+Here is a simple example showing how you can catch signals and perform some action::
 
     from scrapy import signals
     from scrapy import Spider
@@ -52,9 +51,45 @@ Deferred signal handlers
 ========================
 
 Some signals support returning :class:`~twisted.internet.defer.Deferred`
-objects from their handlers, see the :ref:`topics-signals-ref` below to know
-which ones.
+objects from their handlers, allowing you to run asynchronous code that
+does not block Scrapy. If a signal handler returns a
+:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
+:class:`~twisted.internet.defer.Deferred` to fire.
 
+Let's take an example::
+
+    class SignalSpider(scrapy.Spider):
+        name = 'signals'
+        start_urls = ['http://quotes.toscrape.com/page/1/']
+
+        @classmethod
+        def from_crawler(cls, crawler, *args, **kwargs):
+            spider = super(SignalSpider, cls).from_crawler(crawler, *args, **kwargs)
+            crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
+            return spider
+
+        def item_scraped(self, item):
+            # Send the scraped item to the server
+            d = treq.post(
+                'http://example.com/post',
+                json.dumps(item).encode('ascii'),
+                headers={b'Content-Type': [b'application/json']}
+            )
+
+            # The next item will be scraped only after
+            # deferred (d) is fired
+            return d
+
+        def parse(self, response):
+            for quote in response.css('div.quote'):
+                yield {
+                    'text': quote.css('span.text::text').get(),
+                    'author': quote.css('small.author::text').get(),
+                    'tags': quote.css('div.tags a.tag::text').getall(),
+                }
+
+See the :ref:`topics-signals-ref` below to know which signals support
+:class:`~twisted.internet.defer.Deferred`.
 
 .. _topics-signals-ref:
 
@@ -66,9 +101,12 @@ Built-in signals reference
 
 Here's the list of Scrapy built-in signals and their meaning.
 
-engine_started
+Engine signals
 --------------
 
+engine_started
+~~~~~~~~~~~~~~
+
 .. signal:: engine_started
 .. function:: engine_started()
 
@@ -81,7 +119,7 @@ engine_started
     getting fired before :signal:`spider_opened`.
 
 engine_stopped
---------------
+~~~~~~~~~~~~~~
 
 .. signal:: engine_stopped
 .. function:: engine_stopped()
@@ -91,9 +129,20 @@ engine_stopped
 
     This signal supports returning deferreds from their handlers.
 
-item_scraped
+Item signals
 ------------
 
+.. note::
+    As at max :setting:`CONCURRENT_ITEMS` items are processed in
+    parallel, many deferreds are fired together using
+    :class:`~twisted.internet.defer.DeferredList`. Hence the next
+    batch waits for the :class:`~twisted.internet.defer.DeferredList`
+    to fire and then runs the respective item signal handler for
+    the next batch of scraped items.
+
+item_scraped
+~~~~~~~~~~~~
+
 .. signal:: item_scraped
 .. function:: item_scraped(item, response, spider)
 
@@ -112,7 +161,7 @@ item_scraped
     :type response: :class:`~scrapy.http.Response` object
 
 item_dropped
-------------
+~~~~~~~~~~~~
 
 .. signal:: item_dropped
 .. function:: item_dropped(item, response, exception, spider)
@@ -137,7 +186,7 @@ item_dropped
     :type exception: :exc:`~scrapy.exceptions.DropItem` exception
 
 item_error
-------------
+~~~~~~~~~~
 
 .. signal:: item_error
 .. function:: item_error(item, response, spider, failure)
@@ -159,8 +208,11 @@ item_error
     :param failure: the exception raised
     :type failure: twisted.python.failure.Failure
 
+Spider signals
+--------------
+
 spider_closed
--------------
+~~~~~~~~~~~~~
 
 .. signal:: spider_closed
 .. function:: spider_closed(spider, reason)
@@ -183,7 +235,7 @@ spider_closed
     :type reason: str
 
 spider_opened
--------------
+~~~~~~~~~~~~~
 
 .. signal:: spider_opened
 .. function:: spider_opened(spider)
@@ -198,7 +250,7 @@ spider_opened
     :type spider: :class:`~scrapy.spiders.Spider` object
 
 spider_idle
------------
+~~~~~~~~~~~
 
 .. signal:: spider_idle
 .. function:: spider_idle(spider)
@@ -228,7 +280,7 @@ spider_idle
     due to duplication).
 
 spider_error
-------------
+~~~~~~~~~~~~
 
 .. signal:: spider_error
 .. function:: spider_error(failure, response, spider)
@@ -246,8 +298,11 @@ spider_error
     :param spider: the spider which raised the exception
     :type spider: :class:`~scrapy.spiders.Spider` object
 
+Request signals
+---------------
+
 request_scheduled
------------------
+~~~~~~~~~~~~~~~~~
 
 .. signal:: request_scheduled
 .. function:: request_scheduled(request, spider)
@@ -264,7 +319,7 @@ request_scheduled
     :type spider: :class:`~scrapy.spiders.Spider` object
 
 request_dropped
----------------
+~~~~~~~~~~~~~~~
 
 .. signal:: request_dropped
 .. function:: request_dropped(request, spider)
@@ -281,7 +336,7 @@ request_dropped
     :type spider: :class:`~scrapy.spiders.Spider` object
 
 request_reached_downloader
----------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. signal:: request_reached_downloader
 .. function:: request_reached_downloader(request, spider)
@@ -297,7 +352,7 @@ request_reached_downloader
     :type spider: :class:`~scrapy.spiders.Spider` object
 
 request_left_downloader
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
 
 .. signal:: request_left_downloader
 .. function:: request_left_downloader(request, spider)
@@ -315,8 +370,11 @@ request_left_downloader
     :param spider: the spider that yielded the request
     :type spider: :class:`~scrapy.spiders.Spider` object
 
+Response signals
+----------------
+
 response_received
------------------
+~~~~~~~~~~~~~~~~~
 
 .. signal:: response_received
 .. function:: response_received(response, request, spider)
@@ -336,7 +394,7 @@ response_received
     :type spider: :class:`~scrapy.spiders.Spider` object
 
 response_downloaded
--------------------
+~~~~~~~~~~~~~~~~~~~
 
 .. signal:: response_downloaded
 .. function:: response_downloaded(response, request, spider)

From efb6f13debf9406a214a9cee3d94d47875d542f5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Thu, 23 Apr 2020 07:40:10 -0300
Subject: [PATCH 2765/2974] Remove assertions from production code (#4440)

---
 scrapy/commands/__init__.py          |  3 ++-
 scrapy/contracts/default.py          |  6 +++++-
 scrapy/core/downloader/middleware.py |  5 +++--
 scrapy/core/engine.py                | 26 ++++++++++++++++++--------
 scrapy/core/scraper.py               |  6 +++++-
 scrapy/crawler.py                    |  3 ++-
 scrapy/http/request/__init__.py      |  3 ++-
 scrapy/pipelines/files.py            |  6 ++++--
 scrapy/utils/iterators.py            | 10 ++++++----
 scrapy/utils/reactor.py              |  3 ++-
 tests/test_utils_iterators.py        |  2 +-
 11 files changed, 50 insertions(+), 23 deletions(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index a573a03d9..9f8e6986a 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -23,7 +23,8 @@ class ScrapyCommand:
         self.settings = None  # set in scrapy.cmdline
 
     def set_crawler(self, crawler):
-        assert not hasattr(self, '_crawler'), "crawler already set"
+        if hasattr(self, '_crawler'):
+            raise RuntimeError("crawler already set")
         self._crawler = crawler
 
     def syntax(self):
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 3002fc702..a1b0f8f22 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -58,7 +58,11 @@ class ReturnsContract(Contract):
     def __init__(self, *args, **kwargs):
         super(ReturnsContract, self).__init__(*args, **kwargs)
 
-        assert len(self.args) in [1, 2, 3]
+        if len(self.args) not in [1, 2, 3]:
+            raise ValueError(
+                "Incorrect argument quantity: expected 1, 2 or 3, got %i"
+                % len(self.args)
+            )
         self.obj_name = self.args[0] or None
         self.obj_type = self.objects[self.obj_name]
 
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 5a03dcdf7..4c2eea522 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -45,8 +45,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
 
         @defer.inlineCallbacks
         def process_response(response):
-            assert response is not None, 'Received None in process_response'
-            if isinstance(response, Request):
+            if response is None:
+                raise TypeError("Received None in process_response")
+            elif isinstance(response, Request):
                 return response
 
             for method in self.methods['process_response']:
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 66cf9ad9a..77d71846e 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -73,7 +73,8 @@ class ExecutionEngine:
     @defer.inlineCallbacks
     def start(self):
         """Start the execution engine"""
-        assert not self.running, "Engine already running"
+        if self.running:
+            raise RuntimeError("Engine already running")
         self.start_time = time()
         yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
         self.running = True
@@ -82,7 +83,8 @@ class ExecutionEngine:
 
     def stop(self):
         """Stop the execution engine gracefully"""
-        assert self.running, "Engine not running"
+        if not self.running:
+            raise RuntimeError("Engine not running")
         self.running = False
         dfd = self._close_all_spiders()
         return dfd.addBoth(lambda _: self._finish_stopping_engine())
@@ -165,7 +167,11 @@ class ExecutionEngine:
         return d
 
     def _handle_downloader_output(self, response, request, spider):
-        assert isinstance(response, (Request, Response, Failure)), response
+        if not isinstance(response, (Request, Response, Failure)):
+            raise TypeError(
+                "Incorrect type: expected Request, Response or Failure, got %s: %r"
+                % (type(response), response)
+            )
         # downloader middleware can return requests (for example, redirects)
         if isinstance(response, Request):
             self.crawl(response, spider)
@@ -205,8 +211,8 @@ class ExecutionEngine:
         return not bool(self.slot)
 
     def crawl(self, request, spider):
-        assert spider in self.open_spiders, \
-            "Spider %r not opened when crawling: %s" % (spider.name, request)
+        if spider not in self.open_spiders:
+            raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request))
         self.schedule(request, spider)
         self.slot.nextcall.schedule()
 
@@ -232,7 +238,11 @@ class ExecutionEngine:
         slot.add_request(request)
 
         def _on_success(response):
-            assert isinstance(response, (Response, Request))
+            if not isinstance(response, (Response, Request)):
+                raise TypeError(
+                    "Incorrect type: expected Response or Request, got %s: %r"
+                    % (type(response), response)
+                )
             if isinstance(response, Response):
                 response.request = request  # tie request to response received
                 logkws = self.logformatter.crawled(request, response, spider)
@@ -253,8 +263,8 @@ class ExecutionEngine:
 
     @defer.inlineCallbacks
     def open_spider(self, spider, start_requests=(), close_if_idle=True):
-        assert self.has_capacity(), "No free spider slot when opening %r" % \
-            spider.name
+        if not self.has_capacity():
+            raise RuntimeError("No free spider slot when opening %r" % spider.name)
         logger.info("Spider opened", extra={'spider': spider})
         nextcall = CallLaterOnce(self._next_request, spider)
         scheduler = self.scheduler_cls.from_crawler(self.crawler)
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 3e4826216..edbb4dd66 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -123,7 +123,11 @@ class Scraper:
     def _scrape(self, response, request, spider):
         """Handle the downloaded response or failure through the spider
         callback/errback"""
-        assert isinstance(response, (Response, Failure))
+        if not isinstance(response, (Response, Failure)):
+            raise TypeError(
+                "Incorrect type: expected Response or Failure, got %s: %r"
+                % (type(response), response)
+            )
 
         dfd = self._scrape2(response, request, spider)  # returns spider's processed output
         dfd.addErrback(self.handle_spider_error, request, response, spider)
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 20990ea41..6f43771e2 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -78,7 +78,8 @@ class Crawler:
 
     @defer.inlineCallbacks
     def crawl(self, *args, **kwargs):
-        assert not self.crawling, "Crawling already taking place"
+        if self.crawling:
+            raise RuntimeError("Crawling already taking place")
         self.crawling = True
 
         try:
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 0a6637af8..a98ba9960 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -24,7 +24,8 @@ class Request(object_ref):
         self.method = str(method).upper()
         self._set_url(url)
         self._set_body(body)
-        assert isinstance(priority, int), "Request priority not an integer: %r" % priority
+        if not isinstance(priority, int):
+            raise TypeError("Request priority not an integer: %r" % priority)
         self.priority = priority
 
         if callback is not None and not callable(callback):
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index aab645d3d..ae365db5b 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -106,7 +106,8 @@ class S3FilesStore:
         else:
             from boto.s3.connection import S3Connection
             self.S3Connection = S3Connection
-        assert uri.startswith('s3://')
+        if not uri.startswith("s3://"):
+            raise ValueError("Incorrect URI scheme in %s, expected 's3'" % uri)
         self.bucket, self.prefix = uri[5:].split('/', 1)
 
     def stat_file(self, path, info):
@@ -266,7 +267,8 @@ class FTPFilesStore:
     USE_ACTIVE_MODE = None
 
     def __init__(self, uri):
-        assert uri.startswith('ftp://')
+        if not uri.startswith("ftp://"):
+            raise ValueError("Incorrect URI scheme in %s, expected 'ftp'" % uri)
         u = urlparse(uri)
         self.port = u.port
         self.host = u.hostname
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index b71419111..5e15bf0c8 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -128,10 +128,12 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
 def _body_or_str(obj, unicode=True):
     expected_types = (Response, str, bytes)
-    assert isinstance(obj, expected_types), \
-        "obj must be %s, not %s" % (
-            " or ".join(t.__name__ for t in expected_types),
-            type(obj).__name__)
+    if not isinstance(obj, expected_types):
+        expected_types_str = " or ".join(t.__name__ for t in expected_types)
+        raise TypeError(
+            "Object %r must be %s, not %s"
+            % (obj, expected_types_str, type(obj).__name__)
+        )
     if isinstance(obj, Response):
         if not unicode:
             return obj.body
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 5308812d6..3c705f69b 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -9,7 +9,8 @@ from scrapy.utils.misc import load_object
 def listen_tcp(portrange, host, factory):
     """Like reactor.listenTCP but tries different ports in a range."""
     from twisted.internet import reactor
-    assert len(portrange) <= 2, "invalid portrange: %s" % portrange
+    if len(portrange) > 2:
+        raise ValueError("invalid portrange: %s" % portrange)
     if not portrange:
         return reactor.listenTCP(0, factory, interface=host)
     if not hasattr(portrange, '__iter__'):
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index ec8311298..a85087619 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -157,7 +157,7 @@ class XmliterTestCase(unittest.TestCase):
 
     def test_xmliter_objtype_exception(self):
         i = self.xmliter(42, 'product')
-        self.assertRaises(AssertionError, next, i)
+        self.assertRaises(TypeError, next, i)
 
     def test_xmliter_encoding(self):
         body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'

From ffe576c4ed192882d1e40fef815f0c1d5354249a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 24 Apr 2020 11:44:36 +0200
Subject: [PATCH 2766/2974] Cover Scrapy 2.1 in the release notes (#4499)

Co-authored-by: Mikhail Korobov <kmike84@gmail.com>
---
 docs/news.rst                    | 147 +++++++++++++++++++++++++++++++
 docs/topics/request-response.rst |   5 ++
 2 files changed, 152 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index e9b7140cd..a158246eb 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,153 @@
 Release notes
 =============
 
+.. _release-2.1.0:
+
+Scrapy 2.1.0 (2020-04-24)
+-------------------------
+
+Highlights:
+
+* New :setting:`FEEDS` setting to export to multiple feeds
+* New :attr:`Response.ip_address <scrapy.http.Response.ip_address>` attribute
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+*   :exc:`AssertionError` exceptions triggered by :ref:`assert <assert>`
+    statements have been replaced by new exception types, to support running
+    Python in optimized mode (see :option:`-O`) without changing Scrapy’s
+    behavior in any unexpected ways.
+
+    If you catch an :exc:`AssertionError` exception from Scrapy, update your
+    code to catch the corresponding new exception.
+
+    (:issue:`4440`)
+
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+*   The ``LOG_UNSERIALIZABLE_REQUESTS`` setting is no longer supported, use
+    :setting:`SCHEDULER_DEBUG` instead (:issue:`4385`)
+
+*   The ``REDIRECT_MAX_METAREFRESH_DELAY`` setting is no longer supported, use
+    :setting:`METAREFRESH_MAXDELAY` instead (:issue:`4385`)
+
+*   The :class:`~scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware`
+    middleware has been removed, including the entire
+    :class:`scrapy.downloadermiddlewares.chunked` module; chunked transfers
+    work out of the box (:issue:`4431`)
+
+*   The ``spiders`` property has been removed from
+    :class:`~scrapy.crawler.Crawler`, use :class:`CrawlerRunner.spider_loader
+    <scrapy.crawler.CrawlerRunner.spider_loader>` or instantiate
+    :setting:`SPIDER_LOADER_CLASS` with your settings instead (:issue:`4398`)
+
+*   The ``MultiValueDict``, ``MultiValueDictKeyError``, and ``SiteNode``
+    classes have been removed from :mod:`scrapy.utils.datatypes`
+    (:issue:`4400`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+*   The ``FEED_FORMAT`` and ``FEED_URI`` settings have been deprecated in
+    favor of the new :setting:`FEEDS` setting (:issue:`1336`, :issue:`3858`,
+    :issue:`4507`)
+
+
+New features
+~~~~~~~~~~~~
+
+*   A new setting, :setting:`FEEDS`, allows configuring multiple output feeds
+    with different settings each (:issue:`1336`, :issue:`3858`, :issue:`4507`)
+
+*   The :command:`crawl` and :command:`runspider` commands now support multiple
+    ``-o`` parameters (:issue:`1336`, :issue:`3858`, :issue:`4507`)
+
+*   The :command:`crawl` and :command:`runspider` commands now support
+    specifying an output format by appending ``:<format>`` to the output file
+    (:issue:`1336`, :issue:`3858`, :issue:`4507`)
+
+*   The new :attr:`Response.ip_address <scrapy.http.Response.ip_address>`
+    attribute gives access to the IP address that originated a response
+    (:issue:`3903`, :issue:`3940`)
+
+*   A warning is now issued when a value in
+    :attr:`~scrapy.spiders.Spider.allowed_domains` includes a port
+    (:issue:`50`, :issue:`3198`, :issue:`4413`)
+
+*   Zsh completion now excludes used option aliases from the completion list
+    (:issue:`4438`)
+
+
+Bug fixes
+~~~~~~~~~
+
+*   :ref:`Request serialization <request-serialization>` no longer breaks for
+    callbacks that are spider attributes which are assigned a function with a
+    different name (:issue:`4500`)
+
+*   ``None`` values in :attr:`~scrapy.spiders.Spider.allowed_domains` no longer
+    cause a :exc:`TypeError` exception (:issue:`4410`)
+
+*   Zsh completion no longer allows options after arguments (:issue:`4438`)
+
+*   zope.interface 5.0.0 and later versions are now supported
+    (:issue:`4447`, :issue:`4448`)
+
+*   :meth:`Spider.make_requests_from_url
+    <scrapy.spiders.Spider.make_requests_from_url>`, deprecated in Scrapy
+    1.4.0, now issues a warning when used (:issue:`4412`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+*   Improved the documentation about signals that allow their handlers to
+    return a :class:`~twisted.internet.defer.Deferred` (:issue:`4295`,
+    :issue:`4390`)
+
+*   Our PyPI entry now includes links for our documentation, our source code
+    repository and our issue tracker (:issue:`4456`)
+
+*   Covered the `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_
+    service in the documentation (:issue:`4206`, :issue:`4455`)
+
+*   Removed references to the Guppy library, which only works in Python 2
+    (:issue:`4285`, :issue:`4343`)
+
+*   Extended use of InterSphinx to link to Python 3 documentation
+    (:issue:`4444`, :issue:`4445`)
+
+*   Added support for Sphinx 3.0 and later (:issue:`4475`, :issue:`4480`,
+    :issue:`4496`, :issue:`4503`)
+
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+*   Removed warnings about using old, removed settings (:issue:`4404`)
+
+*   Removed a warning about importing
+    :class:`~twisted.internet.testing.StringTransport` from
+    ``twisted.test.proto_helpers`` in Twisted 19.7.0 or newer (:issue:`4409`)
+
+*   Removed outdated Debian package build files (:issue:`4384`)
+
+*   Removed :class:`object` usage as a base class (:issue:`4430`)
+
+*   Removed code that added support for old versions of Twisted that we no
+    longer support (:issue:`4472`)
+
+*   Fixed code style issues (:issue:`4468`, :issue:`4469`, :issue:`4471`,
+    :issue:`4481`)
+
+*   Removed :func:`twisted.internet.defer.returnValue` calls (:issue:`4443`,
+    :issue:`4446`, :issue:`4489`)
+
+
 .. _release-2.0.1:
 
 Scrapy 2.0.1 (2020-03-18)
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 5eb4915cd..024f46466 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -619,6 +619,9 @@ Response objects
     :param ip_address: The IP address of the server from which the Response originated.
     :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
 
+    .. versionadded:: 2.1.0
+       The ``ip_address`` parameter.
+
     .. attribute:: Response.url
 
         A string containing the URL of the response.
@@ -710,6 +713,8 @@ Response objects
 
     .. attribute:: Response.ip_address
 
+        .. versionadded:: 2.1.0
+
         The IP address of the server from which the Response originated.
         
         This attribute is currently only populated by the HTTP 1.1 download

From 3878b67a3771102d4b6668ac749afbec7dc85a8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 24 Apr 2020 11:46:54 +0200
Subject: [PATCH 2767/2974] =?UTF-8?q?Bump=20version:=202.0.0=20=E2=86=92?=
 =?UTF-8?q?=202.1.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index f347a0cd0..de22a2783 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.0.0
+current_version = 2.1.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 227cea215..7ec1d6db4 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.0.0
+2.1.0

From f0f1be76d1e6cef65ac9a01d13c5d5060a03f648 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Mon, 27 Apr 2020 09:56:57 +0300
Subject: [PATCH 2768/2974] Using time_id instead of time as a timestamp

---
 docs/topics/feed-exports.rst    |  6 +++---
 scrapy/extensions/feedexport.py | 11 ++++++-----
 tests/test_feedexport.py        |  8 ++++----
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 0bba03a7c..2017be78f 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -441,9 +441,9 @@ An integer number which represent number of scraped items stored in each output
 file. Whenever the number of items exceeds this setting, a new file
 creates and output redirects to it.  The name of the new file will be selected
 based on timestamp when the feed is being created and/or batch sequence number.
-Therefore you must specify %(time)s or %(batch_id)s or both in the file path.
+Therefore you must specify %(time_id)s or %(batch_id)s or both in the file path.
 
-* ``%(time)s`` - gets replaced by a timestamp when the feed is being created
+* ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created
 * ``%(batch_id)s`` - gets replaced by sequence number of batch
 
 For instance::
@@ -452,7 +452,7 @@ For instance::
 
 Your request can be like::
 
-  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time)s.json
+  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time_id)s.json
 
 The result directory tree of above can be like::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 06ea6c5b2..72baa6269 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -292,7 +292,7 @@ class FeedExporter:
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
-        :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri
+        :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri
         """
         if previous_batch_slot is not None:
             previous_batch_id = previous_batch_slot.batch_id
@@ -360,12 +360,12 @@ class FeedExporter:
 
     def _batch_deliveries_supported(self, uri):
         """
-        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s
+        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time_id)s or %(batch_id)s
         to distinguish different files of partial output
         """
-        if self.storage_batch_size is None or '%(time)s' in uri or '%(batch_id)s' in uri:
+        if self.storage_batch_size is None or '%(time_id)s' in uri or '%(batch_id)s' in uri:
             return True
-        logger.warning('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+        logger.warning('%(time_id)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
         return False
 
     def _storage_supported(self, uri):
@@ -397,8 +397,9 @@ class FeedExporter:
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
+        params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
+        params['time_id'] = datetime.utcnow().isoformat().replace(':', '-')
         params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
-        params['time'] = datetime.utcnow().isoformat().replace(':', '-')
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)
         return params
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 8e03a91c8..da759917a 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -989,7 +989,7 @@ class FeedExportTest(FeedExportTestBase):
 
 class PartialDeliveriesTest(FeedExportTestBase):
     __test__ = True
-    _file_mark = '_%(time)s_#%(batch_id)s_'
+    _file_mark = '_%(time_id)s_#%(batch_id)s_'
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
@@ -1146,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
-        """ If path is without %(time)s or %(batch_id)s an exception must be raised """
+        """ If path is without %(time_id)s or %(batch_id)s an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
@@ -1236,7 +1236,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
     def test_batch_path_differ(self):
         """
         Test that the name of all batch files differ from each other.
-        So %(time)s replaced with the current date.
+        So %(time_id)s replaced with the current date.
         """
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
@@ -1245,7 +1245,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         ]
         settings = {
             'FEEDS': {
-                os.path.join(self._random_temp_filename(), '%(time)s'): {
+                os.path.join(self._random_temp_filename(), '%(time_id)s'): {
                     'format': 'json',
                 },
             },

From 2eee6c81017e08bb492da560bc73c03f4f375fcc Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Mon, 27 Apr 2020 09:58:14 +0300
Subject: [PATCH 2769/2974] Documentation spelling fix

---
 docs/topics/feed-exports.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 2017be78f..6c463fc27 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -438,10 +438,10 @@ FEED_EXPORT_BATCH_SIZE
 Default: ``None``
 
 An integer number which represent number of scraped items stored in each output
-file. Whenever the number of items exceeds this setting, a new file
-creates and output redirects to it.  The name of the new file will be selected
+file. Whenever the number of items exceeds this setting, a new file is
+created and output redirects to it.  The name of the new file will be selected
 based on timestamp when the feed is being created and/or batch sequence number.
-Therefore you must specify %(time_id)s or %(batch_id)s or both in the file path.
+Therefore you must specify %(time_id)s or %(batch_id)s or both in FEED_URI.
 
 * ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created
 * ``%(batch_id)s`` - gets replaced by sequence number of batch

From 204737042ac6672eee73c975d0bd6735893d684c Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Mon, 27 Apr 2020 12:52:18 +0300
Subject: [PATCH 2770/2974] Extract the slot closing functionality to the
 function; minor changes

---
 scrapy/extensions/feedexport.py | 56 ++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 29 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 72baa6269..fe6061c33 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -255,7 +255,7 @@ class FeedExporter:
         for uri, feed in self.feeds.items():
             uri_params = self._get_uri_params(spider, feed['uri_params'], None)
             self.slots.append(self._start_new_batch(
-                previous_batch_slot=None,
+                batch_id=1,
                 uri=uri % uri_params,
                 feed=feed,
                 spider=spider,
@@ -265,42 +265,38 @@ class FeedExporter:
     def close_spider(self, spider):
         deferred_list = []
         for slot in self.slots:
-            if not slot.itemcount and not slot.store_empty:
-                # We need to call slot.storage.store nonetheless to get the file
-                # properly closed.
-                return defer.maybeDeferred(slot.storage.store, slot.file)
-            slot.finish_exporting()
-            logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
-            log_args = {'format': slot.format,
-                        'itemcount': slot.itemcount,
-                        'uri': slot.uri}
-            d = defer.maybeDeferred(slot.storage.store, slot.file)
-            d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
-                                                extra={'spider': spider}))
-            d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
-                                                exc_info=failure_to_exc_info(f),
-                                                extra={'spider': spider}))
+            d = self._close_slot(slot, spider)
             deferred_list.append(d)
         return defer.DeferredList(deferred_list) if deferred_list else None
 
-    def _start_new_batch(self, previous_batch_slot, uri, feed, spider, template_uri):
+    def _close_slot(self, slot, spider):
+        if not slot.itemcount and not slot.store_empty:
+            # We need to call slot.storage.store nonetheless to get the file
+            # properly closed.
+            return defer.maybeDeferred(slot.storage.store, slot.file)
+        slot.finish_exporting()
+        logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
+        log_args = {'format': slot.format,
+                    'itemcount': slot.itemcount,
+                    'uri': slot.uri}
+        d = defer.maybeDeferred(slot.storage.store, slot.file)
+        d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
+                                            extra={'spider': spider}))
+        d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
+                                            exc_info=failure_to_exc_info(f),
+                                            extra={'spider': spider}))
+        return d
+
+    def _start_new_batch(self, batch_id, uri, feed, spider, template_uri):
         """
         Redirect the output data stream to a new file.
         Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified.
-        :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store
-        to get the file properly closed.
+        :param batch_id: sequence number of current batch
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
         :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri
         """
-        if previous_batch_slot is not None:
-            previous_batch_id = previous_batch_slot.batch_id
-            previous_batch_slot.exporter.finish_exporting()
-            previous_batch_slot.storage.store(previous_batch_slot.file)
-        else:
-            previous_batch_id = 0
-
         storage = self._get_storage(uri)
         file = storage.open(spider)
         exporter = self._get_exporter(
@@ -317,7 +313,7 @@ class FeedExporter:
             uri=uri,
             format=feed['format'],
             store_empty=feed['store_empty'],
-            batch_id=previous_batch_id + 1,
+            batch_id=batch_id,
             template_uri=template_uri,
         )
         if slot.store_empty:
@@ -330,10 +326,12 @@ class FeedExporter:
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
-            if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0:
+            # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_SIZE and close the old one
+            if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
                 uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
+                self._close_slot(slot, spider)
                 slots.append(self._start_new_batch(
-                    previous_batch_slot=slot,
+                    batch_id=slot.batch_id + 1,
                     uri=slot.template_uri % uri_params,
                     feed=self.feeds[slot.template_uri],
                     spider=spider,

From c207dbf939811176a7b094e0f2547aa7846b1cf8 Mon Sep 17 00:00:00 2001
From: Ashe <sgy@tpay.co.kr>
Date: Tue, 28 Apr 2020 02:45:19 +0900
Subject: [PATCH 2771/2974] Remove the asyncio warning from coroutines page
 (#4513)

---
 docs/topics/coroutines.rst | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 5f61d6796..7a9ecd4d5 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -7,10 +7,6 @@ Coroutines
 Scrapy has :ref:`partial support <coroutine-support>` for the
 :ref:`coroutine syntax <async>`.
 
-.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy
-             versions may introduce related API and behavior changes without a
-             deprecation period or warning.
-
 .. _coroutine-support:
 
 Supported callables

From e3c3ec2ba988f654be1676586714fd96dba32c23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 28 Apr 2020 13:48:50 +0200
Subject: [PATCH 2772/2974] Run quick tests first in Travis CI

---
 .travis.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 66e1a9617..dc91dfe4c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,6 +11,9 @@ matrix:
       python: 3.8
     - env: TOXENV=flake8
       python: 3.8
+    - env: TOXENV=docs
+      python: 3.7  # Keep in sync with .readthedocs.yml
+
     - env: TOXENV=pypy3
     - env: TOXENV=py35
       python: 3.5
@@ -28,8 +31,6 @@ matrix:
       python: 3.8
     - env: TOXENV=py38-asyncio
       python: 3.8
-    - env: TOXENV=docs
-      python: 3.7  # Keep in sync with .readthedocs.yml
 install:
   - |
       if [ "$TOXENV" = "pypy3" ]; then

From 15d96ab8b5fa6d349a45920f93d349b8ea1d0372 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 28 Apr 2020 17:09:05 +0200
Subject: [PATCH 2773/2974] Test the latest Ubuntu along the latest Python

---
 .travis.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 66e1a9617..02c8885e5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,10 +24,13 @@ matrix:
       python: 3.7
     - env: TOXENV=py38
       python: 3.8
+      dist: bionic
     - env: TOXENV=extra-deps
       python: 3.8
+      dist: bionic
     - env: TOXENV=py38-asyncio
       python: 3.8
+      dist: bionic
     - env: TOXENV=docs
       python: 3.7  # Keep in sync with .readthedocs.yml
 install:

From 5c0f11b4ef1d58de4245d9f4ac9a26f21faf082c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 28 Apr 2020 17:32:53 +0200
Subject: [PATCH 2774/2974] Simplify the asyncio Tox environment

---
 .travis.yml |  4 ++--
 tox.ini     | 12 +-----------
 2 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 66e1a9617..a924eb68c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,7 +16,7 @@ matrix:
       python: 3.5
     - env: TOXENV=pinned
       python: 3.5
-    - env: TOXENV=py35-asyncio
+    - env: TOXENV=asyncio
       python: 3.5.2
     - env: TOXENV=py36
       python: 3.6
@@ -26,7 +26,7 @@ matrix:
       python: 3.8
     - env: TOXENV=extra-deps
       python: 3.8
-    - env: TOXENV=py38-asyncio
+    - env: TOXENV=asyncio
       python: 3.8
     - env: TOXENV=docs
       python: 3.7  # Keep in sync with .readthedocs.yml
diff --git a/tox.ini b/tox.ini
index cd118c921..697328ebd 100644
--- a/tox.ini
+++ b/tox.ini
@@ -102,16 +102,6 @@ setenv = {[docs]setenv}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
 
-[asyncio]
+[testenv:asyncio]
 commands =
     {[testenv]commands} --reactor=asyncio
-
-[testenv:py35-asyncio]
-basepython = python3.5
-deps = {[testenv]deps}
-commands = {[asyncio]commands}
-
-[testenv:py38-asyncio]
-basepython = python3.8
-deps = {[testenv]deps}
-commands = {[asyncio]commands}

From 3a64f3eb2902ed8168b78c43f3516cf657873cef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 28 Apr 2020 17:44:19 +0200
Subject: [PATCH 2775/2974] Remove TOXENV from .travis.yml unless needed

---
 .travis.yml | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 66e1a9617..b029d8bda 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,17 +12,14 @@ matrix:
     - env: TOXENV=flake8
       python: 3.8
     - env: TOXENV=pypy3
-    - env: TOXENV=py35
-      python: 3.5
+    - python: 3.5
     - env: TOXENV=pinned
       python: 3.5
     - env: TOXENV=py35-asyncio
       python: 3.5.2
-    - env: TOXENV=py36
-      python: 3.6
-    - env: TOXENV=py37
-      python: 3.7
-    - env: TOXENV=py38
+    - python: 3.6
+    - python: 3.7
+    - env: PYPI_RELEASE_JOB=true
       python: 3.8
     - env: TOXENV=extra-deps
       python: 3.8
@@ -62,4 +59,4 @@ deploy:
   on:
     tags: true
     repo: scrapy/scrapy
-    condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
+    condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"

From f787b8483ceb37ad8c9764d5a28be07028d85f70 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 20 Apr 2020 12:05:15 -0300
Subject: [PATCH 2776/2974] IPv6 test: check for the absence of DNSLookupError

---
 tests/test_crawler.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index b4144ea1d..9151278a5 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -311,14 +311,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
     def test_ipv6_alternative_name_resolver(self):
         log = self.run_script('alternative_name_resolver.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertTrue(any([
-            "twisted.internet.error.ConnectionRefusedError" in log,
-            "twisted.internet.error.ConnectError" in log,
-        ]))
-        self.assertTrue(any([
-            "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
-            "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
-        ]))
+        self.assertNotIn("twisted.internet.error.DNSLookupError", log)
 
     def test_reactor_select(self):
         log = self.run_script("twisted_reactor_select.py")

From 3f9874fac9f93c0956afa5975d7b2bbb21816894 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 1 May 2020 11:52:16 +0300
Subject: [PATCH 2777/2974] Add test s3 export

---
 tests/test_feedexport.py | 70 ++++++++++++++++++++++++++++++++++++++++
 tox.ini                  |  1 +
 2 files changed, 71 insertions(+)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index da759917a..9fc39c3a6 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1253,3 +1253,73 @@ class PartialDeliveriesTest(FeedExportTestBase):
         }
         data = yield self.exported_data(items, settings)
         self.assertEqual(len(items) + 1, len(data['json']))
+
+    @defer.inlineCallbacks
+    def test_s3_export(self):
+        """
+        Test export of items into s3 bucket.
+        S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini
+        to perform this test:
+        [testenv]
+        setenv =
+            AWS_SECRET_ACCESS_KEY = ABCD
+            AWS_ACCESS_KEY_ID = ABCD
+            S3_TEST_BUCKET_NAME = ABCD
+        """
+        try:
+            import boto3
+        except ImportError:
+            raise unittest.SkipTest("S3FeedStorage requires boto3")
+
+        assert_aws_environ()
+        s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME')
+        access_key = os.environ.get('AWS_ACCESS_KEY_ID')
+        secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
+        if not s3_test_bucket_name:
+            raise unittest.SkipTest("No S3 BUCKET available for testing")
+
+        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
+        filename = ''.join(chars)
+        prefix = 'tmp/{filename}'.format(filename=filename)
+        s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(time_id)s.json'.format(
+            bucket_name=s3_test_bucket_name, prefix=prefix
+        )
+        storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)
+        settings = {
+            'FEEDS': {
+                s3_test_file_uri: {
+                    'format': 'json',
+                },
+            },
+            'FEED_STORAGE_BATCH_SIZE': 1,
+        }
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+            self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+            self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
+        ]
+        verifyObject(IFeedStorage, storage)
+
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+
+            def parse(self, response):
+                for item in items:
+                    yield item
+
+        s3 = boto3.resource('s3')
+        my_bucket = s3.Bucket(s3_test_bucket_name)
+        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+
+        with MockServer() as s:
+            runner = CrawlerRunner(Settings(settings))
+            TestSpider.start_urls = [s.url('/')]
+            yield runner.crawl(TestSpider)
+
+        for file_uri in my_bucket.objects.filter(Prefix=prefix):
+            content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key)
+            if not content and not items:
+                break
+            content = json.loads(content.decode('utf-8'))
+            expected_batch, items = items[:batch_size], items[batch_size:]
+            self.assertEqual(expected_batch, content)
diff --git a/tox.ini b/tox.ini
index cd118c921..c77fae1f0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -14,6 +14,7 @@ deps =
     # Extras
     botocore>=1.3.23
     Pillow>=3.4.2
+    boto3>=1.13.0
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID

From dad2ea75222d6240c569440d3221f5fc00925682 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sat, 2 May 2020 01:21:03 +0300
Subject: [PATCH 2778/2974] Change time_id to batch_time

---
 docs/topics/feed-exports.rst    |  6 +++---
 scrapy/extensions/feedexport.py | 10 +++++-----
 tests/test_feedexport.py        | 10 +++++-----
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 6c463fc27..2106b41f5 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -441,9 +441,9 @@ An integer number which represent number of scraped items stored in each output
 file. Whenever the number of items exceeds this setting, a new file is
 created and output redirects to it.  The name of the new file will be selected
 based on timestamp when the feed is being created and/or batch sequence number.
-Therefore you must specify %(time_id)s or %(batch_id)s or both in FEED_URI.
+Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI.
 
-* ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created
+* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
 * ``%(batch_id)s`` - gets replaced by sequence number of batch
 
 For instance::
@@ -452,7 +452,7 @@ For instance::
 
 Your request can be like::
 
-  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time_id)s.json
+  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
 
 The result directory tree of above can be like::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index fe6061c33..a262f5d18 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -295,7 +295,7 @@ class FeedExporter:
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
-        :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri
+        :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri
         """
         storage = self._get_storage(uri)
         file = storage.open(spider)
@@ -358,12 +358,12 @@ class FeedExporter:
 
     def _batch_deliveries_supported(self, uri):
         """
-        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time_id)s or %(batch_id)s
+        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s
         to distinguish different files of partial output
         """
-        if self.storage_batch_size is None or '%(time_id)s' in uri or '%(batch_id)s' in uri:
+        if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
             return True
-        logger.warning('%(time_id)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+        logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
         return False
 
     def _storage_supported(self, uri):
@@ -396,7 +396,7 @@ class FeedExporter:
         for k in dir(spider):
             params[k] = getattr(spider, k)
         params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
-        params['time_id'] = datetime.utcnow().isoformat().replace(':', '-')
+        params['batch_time'] = datetime.utcnow().isoformat().replace(':', '-')
         params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 9fc39c3a6..2217bb4ed 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -989,7 +989,7 @@ class FeedExportTest(FeedExportTestBase):
 
 class PartialDeliveriesTest(FeedExportTestBase):
     __test__ = True
-    _file_mark = '_%(time_id)s_#%(batch_id)s_'
+    _file_mark = '_%(batch_time)s_#%(batch_id)s_'
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
@@ -1146,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
-        """ If path is without %(time_id)s or %(batch_id)s an exception must be raised """
+        """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
@@ -1236,7 +1236,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
     def test_batch_path_differ(self):
         """
         Test that the name of all batch files differ from each other.
-        So %(time_id)s replaced with the current date.
+        So %(batch_time)s replaced with the current date.
         """
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
@@ -1245,7 +1245,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         ]
         settings = {
             'FEEDS': {
-                os.path.join(self._random_temp_filename(), '%(time_id)s'): {
+                os.path.join(self._random_temp_filename(), '%(batch_time)s'): {
                     'format': 'json',
                 },
             },
@@ -1281,7 +1281,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
         chars = [random.choice(ascii_letters + digits) for _ in range(15)]
         filename = ''.join(chars)
         prefix = 'tmp/{filename}'.format(filename=filename)
-        s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(time_id)s.json'.format(
+        s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(batch_time)s.json'.format(
             bucket_name=s3_test_bucket_name, prefix=prefix
         )
         storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)

From 83d7360bb709cf2c73680260c58b767006f42b12 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 4 May 2020 02:00:11 +0500
Subject: [PATCH 2779/2974] Don't mention unsupported package versions in docs

---
 docs/topics/settings.rst | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 18f81838f..e3da1bd12 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -420,10 +420,9 @@ connections (for ``HTTP10DownloadHandler``).
 .. note::
 
     HTTP/1.0 is rarely used nowadays so you can safely ignore this setting,
-    unless you use Twisted<11.1, or if you really want to use HTTP/1.0
-    and override :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme
-    accordingly, i.e. to
-    ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
+    unless you really want to use HTTP/1.0 and override
+    :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme accordingly,
+    i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
 
 .. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY
 
@@ -447,7 +446,6 @@ or even enable client-side authentication (and various other things).
     Scrapy also has another context factory class that you can set,
     ``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``,
     which uses the platform's certificates to validate remote endpoints.
-    **This is only available if you use Twisted>=14.0.**
 
 If you do use a custom ContextFactory, make sure its ``__init__`` method
 accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
@@ -494,10 +492,6 @@ This setting must be one of these string values:
 - ``'TLSv1.2'``: forces TLS version 1.2
 - ``'SSLv3'``: forces SSL version 3 (**not recommended**)
 
-.. note::
-
-    We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
-    or above (Twisted>=14.0 if you can).
 
 .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
 
@@ -660,8 +654,6 @@ If you want to disable it set to 0.
     spider attribute and per-request using :reqmeta:`download_maxsize`
     Request.meta key.
 
-    This feature needs Twisted >= 11.1.
-
 .. setting:: DOWNLOAD_WARNSIZE
 
 DOWNLOAD_WARNSIZE
@@ -679,8 +671,6 @@ If you want to disable it set to 0.
     spider attribute and per-request using :reqmeta:`download_warnsize`
     Request.meta key.
 
-    This feature needs Twisted >= 11.1.
-
 .. setting:: DOWNLOAD_FAIL_ON_DATALOSS
 
 DOWNLOAD_FAIL_ON_DATALOSS

From f75941f79dfeb931649f9e8880cdbd1b0a8e4681 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 30 Apr 2020 11:56:52 -0300
Subject: [PATCH 2780/2974] Deprecate scrapy.item.BaseItem

---
 scrapy/item.py     |  7 ++++++-
 tests/test_item.py | 25 ++++++++++++++++++++-----
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/scrapy/item.py b/scrapy/item.py
index 748368932..3558b2231 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -28,7 +28,12 @@ class BaseItem(object_ref):
     Unlike instances of :class:`dict`, instances of :class:`BaseItem` may be
     :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
     """
-    pass
+
+    def __new__(cls, *args, **kwargs):
+        if issubclass(cls, BaseItem) and not (issubclass(cls, Item) or issubclass(cls, DictItem)):
+            warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
+                 ScrapyDeprecationWarning, stacklevel=2)
+        return super(BaseItem, cls).__new__(cls, *args, **kwargs)
 
 
 class Field(dict):
diff --git a/tests/test_item.py b/tests/test_item.py
index 4017f6e84..1220bc98a 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -4,7 +4,7 @@ from unittest import mock
 from warnings import catch_warnings
 
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
+from scrapy.item import ABCMeta, BaseItem, DictItem, Field, Item, ItemMeta
 
 
 PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6)
@@ -131,12 +131,12 @@ class ItemTest(unittest.TestCase):
         self.assertSortedEqual(list(item.values()), [u'New'])
 
     def test_metaclass_inheritance(self):
-        class BaseItem(Item):
+        class ParentItem(Item):
             name = Field()
             keys = Field()
             values = Field()
 
-        class TestItem(BaseItem):
+        class TestItem(ParentItem):
             keys = Field()
 
         i = TestItem()
@@ -321,13 +321,28 @@ class DictItemTest(unittest.TestCase):
 
     def test_deprecation_warning(self):
         with catch_warnings(record=True) as warnings:
-            dict_item = DictItem()
+            DictItem()
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
         with catch_warnings(record=True) as warnings:
             class SubclassedDictItem(DictItem):
                 pass
-            subclassed_dict_item = SubclassedDictItem()
+            SubclassedDictItem()
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+
+
+class BaseItemTest(unittest.TestCase):
+
+    def test_deprecation_warning(self):
+        with catch_warnings(record=True) as warnings:
+            BaseItem()
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+        with catch_warnings(record=True) as warnings:
+            class SubclassedBaseItem(BaseItem):
+                pass
+            SubclassedBaseItem()
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 

From e1948b492317eb5b11550d119d91c61b74b3a37f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 4 May 2020 09:07:27 -0300
Subject: [PATCH 2781/2974] Add example about bytes_received signal

---
 docs/topics/signals.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index bc04faad5..7fe63a7b0 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -378,7 +378,9 @@ bytes_received
 
     Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
     received for a specific request. This signal might be fired multiple
-    times for the same request, with partial data each time.
+    times for the same request, with partial data each time. For instance,
+    a possible scenario for a 25 kb response would be two signals fired
+    with 10 kb of data, and a final one with 5 kb of data.
 
     This signal does not support returning deferreds from its handlers.
 

From fe6154e4faee375e7f47d61ceafabde7a3289bf3 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 4 May 2020 18:18:38 +0500
Subject: [PATCH 2782/2974] clarify DOWNLOADER_HTTPCLIENTFACTORY docs

---
 docs/topics/settings.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index e3da1bd12..f06d9db3c 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -421,7 +421,7 @@ connections (for ``HTTP10DownloadHandler``).
 
     HTTP/1.0 is rarely used nowadays so you can safely ignore this setting,
     unless you really want to use HTTP/1.0 and override
-    :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme accordingly,
+    :setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly,
     i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
 
 .. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY

From 622ce860669b8bd7fc581b74d414aef24f4fb041 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 4 May 2020 16:22:24 -0300
Subject: [PATCH 2783/2974] Test: make sure scrapy.item.Item does not issue a
 deprecation warning

---
 tests/test_item.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/test_item.py b/tests/test_item.py
index 1220bc98a..f35a2b9f9 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -347,5 +347,18 @@ class BaseItemTest(unittest.TestCase):
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 
 
+class ItemNoDeprecationWarningTest(unittest.TestCase):
+
+    def test_no_deprecation_warning(self):
+        with catch_warnings(record=True) as warnings:
+            Item()
+            self.assertEqual(len(warnings), 0)
+        with catch_warnings(record=True) as warnings:
+            class SubclassedItem(Item):
+                pass
+            SubclassedItem()
+            self.assertEqual(len(warnings), 0)
+
+
 if __name__ == "__main__":
     unittest.main()

From 7988c676a956457843416f8586544a58bf121f63 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 5 May 2020 13:11:01 -0300
Subject: [PATCH 2784/2974] Update Item docstring, update BaseItem occurrences

---
 docs/faq.rst               |  4 ++--
 pytest.ini                 |  2 +-
 scrapy/item.py             | 35 ++++++++++++++++++-----------------
 scrapy/spiders/feed.py     |  2 +-
 tests/test_loader.py       |  2 +-
 tests/test_utils_spider.py |  4 ++--
 6 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 75a0f4864..79ef6ca85 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -342,14 +342,14 @@ method for this purpose. For example::
 
     from copy import deepcopy
 
-    from scrapy.item import BaseItem
+    from scrapy.item import Item
 
 
     class MultiplyItemsMiddleware:
 
         def process_spider_output(self, response, result, spider):
             for item in result:
-                if isinstance(item, (BaseItem, dict)):
+                if isinstance(item, (Item, dict)):
                     for _ in range(item['multiply_by']):
                         yield deepcopy(item)
 
diff --git a/pytest.ini b/pytest.ini
index e8911ee3f..5a86ce2a7 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -153,7 +153,7 @@ flake8-ignore =
     scrapy/exceptions.py E501
     scrapy/exporters.py E501
     scrapy/interfaces.py E501
-    scrapy/item.py E501 E128
+    scrapy/item.py E501
     scrapy/link.py E501
     scrapy/logformatter.py E501
     scrapy/mail.py E402 E128 E501
diff --git a/scrapy/item.py b/scrapy/item.py
index 3558b2231..46d20d017 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -15,18 +15,8 @@ from scrapy.utils.trackref import object_ref
 
 
 class BaseItem(object_ref):
-    """Base class for all scraped items.
-
-    In Scrapy, an object is considered an *item* if it is an instance of either
-    :class:`BaseItem` or :class:`dict`. For example, when the output of a
-    spider callback is evaluated, only instances of :class:`BaseItem` or
-    :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
-
-    If you need instances of a custom class to be considered items by Scrapy,
-    you must inherit from either :class:`BaseItem` or :class:`dict`.
-
-    Unlike instances of :class:`dict`, instances of :class:`BaseItem` may be
-    :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
+    """
+    Deprecated, please use :class:`scrapy.item.Item` instead
     """
 
     def __new__(cls, *args, **kwargs):
@@ -91,8 +81,7 @@ class DictItem(MutableMapping, BaseItem):
         if key in self.fields:
             self._values[key] = value
         else:
-            raise KeyError("%s does not support field: %s" %
-                (self.__class__.__name__, key))
+            raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key))
 
     def __delitem__(self, key):
         del self._values[key]
@@ -104,8 +93,7 @@ class DictItem(MutableMapping, BaseItem):
 
     def __setattr__(self, name, value):
         if not name.startswith('_'):
-            raise AttributeError("Use item[%r] = %r to set field value" %
-                (name, value))
+            raise AttributeError("Use item[%r] = %r to set field value" % (name, value))
         super(DictItem, self).__setattr__(name, value)
 
     def __len__(self):
@@ -132,4 +120,17 @@ class DictItem(MutableMapping, BaseItem):
 
 
 class Item(DictItem, metaclass=ItemMeta):
-    pass
+    """
+    Base class for scraped items.
+
+    In Scrapy, an object is considered an *item* if it is an instance of either
+    :class:`Item` or :class:`dict`. For example, when the output of a
+    spider callback is evaluated, only instances of :class:`Item` or
+    :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
+
+    If you need instances of a custom class to be considered items by Scrapy,
+    you must inherit from either :class:`Item` or :class:`dict`.
+
+    Unlike instances of :class:`dict`, instances of :class:`Item` may be
+    :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
+    """
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index c566f0236..a4ff8010d 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -52,7 +52,7 @@ class XMLFeedSpider(Spider):
         """This method is called for the nodes matching the provided tag name
         (itertag). Receives the response and an Selector for each node.
         Overriding this method is mandatory. Otherwise, you spider won't work.
-        This method must return either a BaseItem, a Request, or a list
+        This method must return either an item, a request, or a list
         containing any of them.
         """
 
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 701d568dc..f14714c75 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -601,7 +601,7 @@ class NoInputReprocessingItemLoader(BaseNoInputReprocessingLoader):
 
 class NoInputReprocessingFromItemTest(unittest.TestCase):
     """
-    Loaders initialized from loaded items must not reprocess fields (BaseItem instances)
+    Loaders initialized from loaded items must not reprocess fields (Item instances)
     """
     def test_avoid_reprocessing_with_initial_values_single(self):
         il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title='foo'))
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index ee7d17062..3c87268ab 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -2,7 +2,7 @@ import unittest
 
 from scrapy import Spider
 from scrapy.http import Request
-from scrapy.item import BaseItem
+from scrapy.item import Item
 from scrapy.utils.spider import iterate_spider_output, iter_spider_classes
 
 
@@ -17,7 +17,7 @@ class MySpider2(Spider):
 class UtilsSpidersTestCase(unittest.TestCase):
 
     def test_iterate_spider_output(self):
-        i = BaseItem()
+        i = Item()
         r = Request('http://scrapytest.org')
         o = object()
 

From 17c0cf64aee1641e1ad33c5b46a61435c5969f2f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Tue, 5 May 2020 19:14:48 -0300
Subject: [PATCH 2785/2974] Flake8: remove W504 code (#4525)

Co-authored-by: Mikhail Korobov <kmike84@gmail.com>
---
 pytest.ini                               | 16 ++++++++--------
 scrapy/contracts/__init__.py             |  4 ++--
 scrapy/downloadermiddlewares/redirect.py | 11 +++++++----
 scrapy/extensions/telnet.py              |  6 ++++--
 scrapy/linkextractors/__init__.py        |  3 +--
 scrapy/spidermiddlewares/referer.py      | 14 ++++++++------
 scrapy/utils/gz.py                       |  3 +--
 tests/test_utils_http.py                 |  8 ++++----
 8 files changed, 35 insertions(+), 30 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index e8911ee3f..4f3494e0e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -44,12 +44,12 @@ flake8-ignore =
     scrapy/commands/startproject.py E127 E501 E128
     scrapy/commands/version.py E501 E128
     # scrapy/contracts
-    scrapy/contracts/__init__.py E501 W504
+    scrapy/contracts/__init__.py E501
     scrapy/contracts/default.py E128
     # scrapy/core
     scrapy/core/engine.py E501 E128 E127
     scrapy/core/scheduler.py E501
-    scrapy/core/scraper.py E501 E128 W504
+    scrapy/core/scraper.py E501 E128
     scrapy/core/spidermw.py E501 E126
     scrapy/core/downloader/__init__.py E501
     scrapy/core/downloader/contextfactory.py E501 E128 E126
@@ -68,7 +68,7 @@ flake8-ignore =
     scrapy/downloadermiddlewares/httpcache.py E501 E126
     scrapy/downloadermiddlewares/httpcompression.py E501 E128
     scrapy/downloadermiddlewares/httpproxy.py E501
-    scrapy/downloadermiddlewares/redirect.py E501 W504
+    scrapy/downloadermiddlewares/redirect.py E501
     scrapy/downloadermiddlewares/retry.py E501 E126
     scrapy/downloadermiddlewares/robotstxt.py E501
     scrapy/downloadermiddlewares/stats.py E501
@@ -79,7 +79,7 @@ flake8-ignore =
     scrapy/extensions/httpcache.py E128 E501
     scrapy/extensions/memdebug.py E501
     scrapy/extensions/spiderstate.py E501
-    scrapy/extensions/telnet.py E501 W504
+    scrapy/extensions/telnet.py E501
     scrapy/extensions/throttle.py E501
     # scrapy/http
     scrapy/http/common.py E501
@@ -90,7 +90,7 @@ flake8-ignore =
     scrapy/http/response/__init__.py E501 E128
     scrapy/http/response/text.py E501 E128 E124
     # scrapy/linkextractors
-    scrapy/linkextractors/__init__.py E501 E402 W504
+    scrapy/linkextractors/__init__.py E501 E402
     scrapy/linkextractors/lxmlhtml.py E501
     # scrapy/loader
     scrapy/loader/__init__.py E501 E128
@@ -110,7 +110,7 @@ flake8-ignore =
     # scrapy/spidermiddlewares
     scrapy/spidermiddlewares/httperror.py E501
     scrapy/spidermiddlewares/offsite.py E501
-    scrapy/spidermiddlewares/referer.py E501 E129 W504
+    scrapy/spidermiddlewares/referer.py E501 E129
     scrapy/spidermiddlewares/urllength.py E501
     # scrapy/spiders
     scrapy/spiders/__init__.py E501 E402
@@ -125,7 +125,7 @@ flake8-ignore =
     scrapy/utils/decorators.py E501
     scrapy/utils/defer.py E501 E128
     scrapy/utils/deprecate.py E128 E501 E127
-    scrapy/utils/gz.py E501 W504
+    scrapy/utils/gz.py E501
     scrapy/utils/http.py F403
     scrapy/utils/httpobj.py E501
     scrapy/utils/iterators.py E501
@@ -234,7 +234,7 @@ flake8-ignore =
     tests/test_utils_datatypes.py E402 E501
     tests/test_utils_defer.py E501 F841
     tests/test_utils_deprecate.py F841 E501
-    tests/test_utils_http.py E501 E128 W504
+    tests/test_utils_http.py E501 E128
     tests/test_utils_iterators.py E501 E128 E129
     tests/test_utils_log.py E741
     tests/test_utils_python.py E501
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 41d4f25b2..5af3831a2 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -17,10 +17,10 @@ class ContractsManager:
             self.contracts[contract.name] = contract
 
     def tested_methods_from_spidercls(self, spidercls):
+        is_method = re.compile(r"^\s*@", re.MULTILINE).search
         methods = []
         for key, value in getmembers(spidercls):
-            if (callable(value) and value.__doc__ and
-                    re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
+            if callable(value) and value.__doc__ and is_method(value.__doc__):
                 methods.append(key)
 
         return methods
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 09ee8377e..b32afb8e4 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -60,11 +60,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
     Handle redirection of requests based on response status
     and meta-refresh html tag.
     """
+
     def process_response(self, request, response, spider):
-        if (request.meta.get('dont_redirect', False) or
-                response.status in getattr(spider, 'handle_httpstatus_list', []) or
-                response.status in request.meta.get('handle_httpstatus_list', []) or
-                request.meta.get('handle_httpstatus_all', False)):
+        if (
+            request.meta.get('dont_redirect', False)
+            or response.status in getattr(spider, 'handle_httpstatus_list', [])
+            or response.status in request.meta.get('handle_httpstatus_list', [])
+            or request.meta.get('handle_httpstatus_all', False)
+        ):
             return response
 
         allowed_status = (301, 302, 303, 307, 308)
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 04ffd7235..1663604e7 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -76,8 +76,10 @@ class TelnetConsole(protocol.ServerFactory):
             """An implementation of IPortal"""
             @defers
             def login(self_, credentials, mind, *interfaces):
-                if not (credentials.username == self.username.encode('utf8') and
-                        credentials.checkPassword(self.password.encode('utf8'))):
+                if not (
+                    credentials.username == self.username.encode('utf8')
+                    and credentials.checkPassword(self.password.encode('utf8'))
+                ):
                     raise ValueError("Invalid credentials")
 
                 protocol = telnet.TelnetBootstrapProtocol(
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index d0b5066b6..ae019c70f 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -61,8 +61,7 @@ class FilteringLinkExtractor:
 
     def __new__(cls, *args, **kwargs):
         from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
-        if (issubclass(cls, FilteringLinkExtractor) and
-                not issubclass(cls, LxmlLinkExtractor)):
+        if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor):
             warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
                  'please use scrapy.linkextractors.LinkExtractor instead',
                  ScrapyDeprecationWarning, stacklevel=2)
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 3784de885..434067b00 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -163,9 +163,10 @@ class StrictOriginPolicy(ReferrerPolicy):
     name = POLICY_STRICT_ORIGIN
 
     def referrer(self, response_url, request_url):
-        if ((self.tls_protected(response_url) and
-             self.potentially_trustworthy(request_url))
-            or not self.tls_protected(response_url)):
+        if (
+            self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
+            or not self.tls_protected(response_url)
+        ):
             return self.origin_referrer(response_url)
 
 
@@ -213,9 +214,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
         origin = self.origin(response_url)
         if origin == self.origin(request_url):
             return self.stripped_referrer(response_url)
-        elif ((self.tls_protected(response_url) and
-               self.potentially_trustworthy(request_url))
-              or not self.tls_protected(response_url)):
+        elif (
+            self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
+            or not self.tls_protected(response_url)
+        ):
             return self.origin_referrer(response_url)
 
 
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index c291ae237..fbd7bd18f 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -52,8 +52,7 @@ def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
     ctype = response.headers.get('Content-Type', b'')
     cenc = response.headers.get('Content-Encoding', b'').lower()
-    return (_is_gzipped(ctype) or
-            (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')))
+    return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')
 
 
 def gzip_magic_number(response):
diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py
index 2fac3da1f..363b015a8 100644
--- a/tests/test_utils_http.py
+++ b/tests/test_utils_http.py
@@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase):
         chunked_body += "8\r\n" + "sequence\r\n"
         chunked_body += "0\r\n\r\n"
         body = decode_chunked_transfer(chunked_body)
-        self.assertEqual(body,
-                         "This is the data in the first chunk\r\n" +
-                         "and this is the second one\r\n" +
-                         "consequence")
+        self.assertEqual(
+            body,
+            "This is the data in the first chunk\r\nand this is the second one\r\nconsequence"
+        )

From 418b9b5f5222e05bec497a83af8c7dfcca30c6cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 6 May 2020 11:15:02 +0200
Subject: [PATCH 2786/2974] Travis CI: do not run security and Flake8 on
 multiple jobs

---
 .travis.yml | 11 +++++++----
 tox.ini     |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 6bde973f4..75d3c5a98 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,14 +15,17 @@ matrix:
       python: 3.7  # Keep in sync with .readthedocs.yml
 
     - env: TOXENV=pypy3
-    - python: 3.5
+    - env: TOXENV=py
+      python: 3.5
     - env: TOXENV=pinned
       python: 3.5
     - env: TOXENV=asyncio
       python: 3.5.2
-    - python: 3.6
-    - python: 3.7
-    - env: PYPI_RELEASE_JOB=true
+    - env: TOXENV=py
+      python: 3.6
+    - env: TOXENV=py
+      python: 3.7
+    - env: TOXENV=py PYPI_RELEASE_JOB=true
       python: 3.8
     - env: TOXENV=extra-deps
       python: 3.8
diff --git a/tox.ini b/tox.ini
index 697328ebd..2102fc602 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = security,flake8,py3
+envlist = security,flake8,py
 minversion = 1.7.0
 
 [testenv]

From 8d1e3ee0dd8d48428f719a0fffda964d30ab956d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 09:24:32 -0300
Subject: [PATCH 2787/2974] Remove deprecated BaseItem from the docs

---
 docs/topics/items.rst | 2 --
 1 file changed, 2 deletions(-)

diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 78612f524..0941a8a1b 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -257,6 +257,4 @@ Field objects
 Other classes related to Item
 =============================
 
-.. autoclass:: BaseItem
-
 .. autoclass:: ItemMeta

From 49e8a337f78ec5e30eacfcd201b66d68deeecb56 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 09:37:01 -0300
Subject: [PATCH 2788/2974] Flake8: remove E127 (continuation line
 over-indented for visual indent)

---
 pytest.ini                             | 10 +++++-----
 scrapy/core/downloader/handlers/ftp.py | 11 ++++++-----
 scrapy/core/engine.py                  |  3 +--
 scrapy/utils/deprecate.py              | 21 +++++++++++----------
 scrapy/utils/request.py                |  3 +--
 5 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 4f3494e0e..fa65a0da2 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -41,13 +41,13 @@ flake8-ignore =
     scrapy/commands/runspider.py E501
     scrapy/commands/settings.py E128
     scrapy/commands/shell.py E128 E501
-    scrapy/commands/startproject.py E127 E501 E128
+    scrapy/commands/startproject.py E501 E128
     scrapy/commands/version.py E501 E128
     # scrapy/contracts
     scrapy/contracts/__init__.py E501
     scrapy/contracts/default.py E128
     # scrapy/core
-    scrapy/core/engine.py E501 E128 E127
+    scrapy/core/engine.py E501 E128
     scrapy/core/scheduler.py E501
     scrapy/core/scraper.py E501 E128
     scrapy/core/spidermw.py E501 E126
@@ -57,7 +57,7 @@ flake8-ignore =
     scrapy/core/downloader/tls.py E501
     scrapy/core/downloader/webclient.py E501 E128 E126
     scrapy/core/downloader/handlers/__init__.py E501
-    scrapy/core/downloader/handlers/ftp.py E501 E128 E127
+    scrapy/core/downloader/handlers/ftp.py E501 E128
     scrapy/core/downloader/handlers/http10.py E501
     scrapy/core/downloader/handlers/http11.py E501
     scrapy/core/downloader/handlers/s3.py E501 E128 E126
@@ -124,7 +124,7 @@ flake8-ignore =
     scrapy/utils/datatypes.py E501
     scrapy/utils/decorators.py E501
     scrapy/utils/defer.py E501 E128
-    scrapy/utils/deprecate.py E128 E501 E127
+    scrapy/utils/deprecate.py E501
     scrapy/utils/gz.py E501
     scrapy/utils/http.py F403
     scrapy/utils/httpobj.py E501
@@ -137,7 +137,7 @@ flake8-ignore =
     scrapy/utils/python.py E501
     scrapy/utils/reactor.py E501
     scrapy/utils/reqser.py E501
-    scrapy/utils/request.py E127 E501
+    scrapy/utils/request.py E501
     scrapy/utils/response.py E501 E128
     scrapy/utils/signal.py E501 E128
     scrapy/utils/sitemap.py E501
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 432cb1831..94b55c347 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -94,11 +94,12 @@ class FTPDownloadHandler:
     def gotClient(self, client, request, filepath):
         self.client = client
         protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))
-        return client.retrieveFile(filepath, protocol)\
-                .addCallbacks(callback=self._build_response,
-                        callbackArgs=(request, protocol),
-                        errback=self._failed,
-                        errbackArgs=(request,))
+        return client.retrieveFile(filepath, protocol).addCallbacks(
+            callback=self._build_response,
+            callbackArgs=(request, protocol),
+            errback=self._failed,
+            errbackArgs=(request,),
+        )
 
     def _build_response(self, result, request, protocol):
         self.result = result
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 77d71846e..324d21716 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -230,8 +230,7 @@ class ExecutionEngine:
 
     def _downloaded(self, response, slot, request, spider):
         slot.remove_request(request)
-        return self.download(response, spider) \
-                if isinstance(response, Request) else response
+        return self.download(response, spider) if isinstance(response, Request) else response
 
     def _download(self, request, spider):
         slot = self.slot
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index 36001d982..3dbea5fee 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -15,16 +15,17 @@ def attribute(obj, oldattr, newattr, version='0.12'):
         stacklevel=3)
 
 
-def create_deprecated_class(name, new_class, clsdict=None,
-                            warn_category=ScrapyDeprecationWarning,
-                            warn_once=True,
-                            old_class_path=None,
-                            new_class_path=None,
-                            subclass_warn_message="{cls} inherits from "
-                                    "deprecated class {old}, please inherit "
-                                    "from {new}.",
-                            instance_warn_message="{cls} is deprecated, "
-                                    "instantiate {new} instead."):
+def create_deprecated_class(
+    name,
+    new_class,
+    clsdict=None,
+    warn_category=ScrapyDeprecationWarning,
+    warn_once=True,
+    old_class_path=None,
+    new_class_path=None,
+    subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.",
+    instance_warn_message="{cls} is deprecated, instantiate {new} instead."
+):
     """
     Return a "deprecated" class that causes its subclasses to issue a warning.
     Subclasses of ``new_class`` are considered subclasses of this class.
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index b8c140a7e..12c03d78e 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -50,8 +50,7 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
 
     """
     if include_headers:
-        include_headers = tuple(to_bytes(h.lower())
-                                 for h in sorted(include_headers))
+        include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
     cache = _fingerprint_cache.setdefault(request, {})
     cache_key = (include_headers, keep_fragments)
     if cache_key not in cache:

From fe0c582ee083ad8085a33443af0ffbc67b44fc16 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 09:49:10 -0300
Subject: [PATCH 2789/2974] Flake8: remove E127 in tests (continuation line
 over-indented for visual indent)

---
 pytest.ini                                    |  18 +--
 tests/spiders.py                              |   3 +-
 tests/test_closespider.py                     |   3 +-
 tests/test_downloader_handlers.py             |   9 +-
 ...test_downloadermiddleware_decompression.py |   4 +-
 tests/test_downloadermiddleware_redirect.py   |   3 +-
 tests/test_http_request.py                    |   6 +-
 tests/test_selector.py                        |   3 +-
 tests/test_spidermiddleware_httperror.py      |   6 +-
 tests/test_utils_url.py                       | 120 ++++++++++--------
 10 files changed, 90 insertions(+), 85 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index fa65a0da2..3eefe70f1 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -171,8 +171,8 @@ flake8-ignore =
     tests/__init__.py E402 E501
     tests/mockserver.py E401 E501 E126 E123
     tests/pipelines.py F841
-    tests/spiders.py E501 E127
-    tests/test_closespider.py E501 E127
+    tests/spiders.py E501
+    tests/test_closespider.py E501
     tests/test_command_fetch.py E501
     tests/test_command_parse.py E501 E128
     tests/test_command_shell.py E501 E128
@@ -181,17 +181,17 @@ flake8-ignore =
     tests/test_crawl.py E501 E741
     tests/test_crawler.py F841 E501
     tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123
+    tests/test_downloader_handlers.py E124 E128 E501 E126 E123
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
     tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
-    tests/test_downloadermiddleware_decompression.py E127
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
     tests/test_downloadermiddleware_httpcache.py E501
     tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
+    tests/test_downloadermiddleware_decompression.py E501
     tests/test_downloadermiddleware_httpproxy.py E501 E128
-    tests/test_downloadermiddleware_redirect.py E501 E128 E127
+    tests/test_downloadermiddleware_redirect.py E501 E128
     tests/test_downloadermiddleware_retry.py E501 E128 E126
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
@@ -202,7 +202,7 @@ flake8-ignore =
     tests/test_feedexport.py E501 F841
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
-    tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
+    tests/test_http_request.py E402 E501 E128 E128 E126 E123
     tests/test_http_response.py E501 E128
     tests/test_item.py E128 F841
     tests/test_link.py E501
@@ -220,10 +220,10 @@ flake8-ignore =
     tests/test_responsetypes.py E501
     tests/test_robotstxt_interface.py E501 E501
     tests/test_scheduler.py E501 E126 E123
-    tests/test_selector.py E501 E127
+    tests/test_selector.py E501
     tests/test_spider.py E501
     tests/test_spidermiddleware.py E501
-    tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
+    tests/test_spidermiddleware_httperror.py E128 E501 E121
     tests/test_spidermiddleware_offsite.py E501 E128 E111
     tests/test_spidermiddleware_output_chain.py E501
     tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121
@@ -243,7 +243,7 @@ flake8-ignore =
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841
     tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_url.py E501 E127 E125 E501 E126 E123
+    tests/test_utils_url.py E501 E125 E501 E126 E123
     tests/test_webclient.py E501 E128 E122 E402 E123 E126
     tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
diff --git a/tests/spiders.py b/tests/spiders.py
index 284c77829..33d5d02e1 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -184,8 +184,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
             if self.fail_yielding:
                 2 / 0
 
-        assert self.seedsseen, \
-                'All start requests consumed before any download happened'
+        assert self.seedsseen, 'All start requests consumed before any download happened'
 
     def parse(self, response):
         self.seedsseen.append(response.meta.get('seed'))
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 4a56425b7..5ec5e2989 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -41,8 +41,7 @@ class TestCloseSpider(TestCase):
         yield crawler.crawl(total=1000000, mockserver=self.mockserver)
         reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_errorcount')
-        key = 'spider_exceptions/{name}'\
-                .format(name=crawler.spider.exception_cls.__name__)
+        key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__)
         errorcount = crawler.stats.get_value(key)
         self.assertTrue(errorcount >= close_on)
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 29d06bab4..24ef560c1 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1090,8 +1090,7 @@ class DataURITestCase(unittest.TestCase):
     def test_default_mediatype_encoding(self):
         def _test(response):
             self.assertEqual(response.text, 'A brief note')
-            self.assertEqual(type(response),
-                              responsetypes.from_mimetype("text/plain"))
+            self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "US-ASCII")
 
         request = Request("data:,A%20brief%20note")
@@ -1100,8 +1099,7 @@ class DataURITestCase(unittest.TestCase):
     def test_default_mediatype(self):
         def _test(response):
             self.assertEqual(response.text, u'\u038e\u03a3\u038e')
-            self.assertEqual(type(response),
-                              responsetypes.from_mimetype("text/plain"))
+            self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "iso-8859-7")
 
         request = Request("data:;charset=iso-8859-7,%be%d3%be")
@@ -1119,8 +1117,7 @@ class DataURITestCase(unittest.TestCase):
     def test_mediatype_parameters(self):
         def _test(response):
             self.assertEqual(response.text, u'\u038e\u03a3\u038e')
-            self.assertEqual(type(response),
-                              responsetypes.from_mimetype("text/plain"))
+            self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "utf-8")
 
         request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;'
diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py
index 77b35a8c3..dbae4d3ae 100644
--- a/tests/test_downloadermiddleware_decompression.py
+++ b/tests/test_downloadermiddleware_decompression.py
@@ -28,8 +28,8 @@ class DecompressionMiddlewareTest(TestCase):
         for fmt in self.test_formats:
             rsp = self.test_responses[fmt]
             new = self.mw.process_response(None, rsp, self.spider)
-            assert isinstance(new, XmlResponse), \
-                    'Failed %s, response type %s' % (fmt, type(new).__name__)
+            error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__)
+            assert isinstance(new, XmlResponse), error_msg
             assert_samelines(self, new.body, self.uncompressed_body, fmt)
 
     def test_plain_response(self):
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 053e26fc3..551e124ab 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -181,8 +181,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
             rsp = Response(url, headers={'Location': url2}, status=301, request=req)
             r = self.mw.process_response(req, rsp, self.spider)
             self.assertIs(r, rsp)
-        _test_passthrough(Request(url, meta={'handle_httpstatus_list':
-                                                           [404, 301, 302]}))
+        _test_passthrough(Request(url, meta={'handle_httpstatus_list': [404, 301, 302]}))
         _test_passthrough(Request(url, meta={'handle_httpstatus_all': True}))
 
     def test_latin1_location(self):
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index cc2cddda4..b12841ba2 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -399,8 +399,7 @@ class FormRequestTest(RequestTest):
 
     def test_custom_encoding_bytes(self):
         data = {b'\xb5 one': b'two', b'price': b'\xa3 100'}
-        r2 = self.request_class("http://www.example.com", formdata=data,
-                                    encoding='latin1')
+        r2 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
         self.assertEqual(r2.method, 'POST')
         self.assertEqual(r2.encoding, 'latin1')
         self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two')
@@ -408,8 +407,7 @@ class FormRequestTest(RequestTest):
 
     def test_custom_encoding_textual_data(self):
         data = {'price': u'£ 100'}
-        r3 = self.request_class("http://www.example.com", formdata=data,
-                                    encoding='latin1')
+        r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
         self.assertEqual(r3.encoding, 'latin1')
         self.assertEqual(r3.body, b'price=%A3+100')
 
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 09c2546fb..65b0f5860 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -67,8 +67,7 @@ class SelectorTestCase(unittest.TestCase):
         headers = {'Content-Type': ['text/html; charset=utf-8']}
         response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
         x = Selector(response)
-        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),
-                          [u'\xa3'])
+        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])
 
     def test_badly_encoded_body(self):
         # \xe9 alone isn't valid utf8 sequence
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index dacd0147f..6b61df56f 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -111,8 +111,7 @@ class TestHttpErrorMiddlewareSettings(TestCase):
                 self.mw.process_spider_input(self.res402, self.spider))
 
     def test_meta_overrides_settings(self):
-        request = Request('http://scrapytest.org',
-                              meta={'handle_httpstatus_list': [404]})
+        request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
         res404 = self.res404.copy()
         res404.request = request
         res402 = self.res402.copy()
@@ -146,8 +145,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
                 self.mw.process_spider_input(self.res404, self.spider))
 
     def test_meta_overrides_settings(self):
-        request = Request('http://scrapytest.org',
-                              meta={'handle_httpstatus_list': [404]})
+        request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
         res404 = self.res404.copy()
         res404.request = request
         res402 = self.res402.copy()
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 72a16e9b1..3bb6d40db 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -77,108 +77,124 @@ class UrlUtilsTest(unittest.TestCase):
 class AddHttpIfNoScheme(unittest.TestCase):
 
     def test_add_scheme(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com'),
-                                               'http://www.example.com')
+        self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com')
 
     def test_without_subdomain(self):
-        self.assertEqual(add_http_if_no_scheme('example.com'),
-                                               'http://example.com')
+        self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com')
 
     def test_path(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'),
-                                               'http://www.example.com/some/page.html')
+        self.assertEqual(
+            add_http_if_no_scheme('www.example.com/some/page.html'),
+            'http://www.example.com/some/page.html')
 
     def test_port(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com:80'),
-                                               'http://www.example.com:80')
+        self.assertEqual(
+            add_http_if_no_scheme('www.example.com:80'),
+            'http://www.example.com:80')
 
     def test_fragment(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'),
-                                               'http://www.example.com/some/page#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('www.example.com/some/page#frag'),
+            'http://www.example.com/some/page#frag')
 
     def test_query(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
-                                               'http://www.example.com/do?a=1&b=2&c=3')
+        self.assertEqual(
+            add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
+            'http://www.example.com/do?a=1&b=2&c=3')
 
     def test_username_password(self):
-        self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'),
-                                               'http://username:password@www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('username:password@www.example.com'),
+            'http://username:password@www.example.com')
 
     def test_complete_url(self):
-        self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
 
     def test_preserve_http(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
-                                               'http://www.example.com')
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com')
 
     def test_preserve_http_without_subdomain(self):
-        self.assertEqual(add_http_if_no_scheme('http://example.com'),
-                                               'http://example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('http://example.com'),
+            'http://example.com')
 
     def test_preserve_http_path(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'),
-                                               'http://www.example.com/some/page.html')
+        self.assertEqual(
+            add_http_if_no_scheme('http://www.example.com/some/page.html'),
+            'http://www.example.com/some/page.html')
 
     def test_preserve_http_port(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'),
-                                               'http://www.example.com:80')
+        self.assertEqual(
+            add_http_if_no_scheme('http://www.example.com:80'),
+            'http://www.example.com:80')
 
     def test_preserve_http_fragment(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'),
-                                               'http://www.example.com/some/page#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('http://www.example.com/some/page#frag'),
+            'http://www.example.com/some/page#frag')
 
     def test_preserve_http_query(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
-                                               'http://www.example.com/do?a=1&b=2&c=3')
+        self.assertEqual(
+            add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
+            'http://www.example.com/do?a=1&b=2&c=3')
 
     def test_preserve_http_username_password(self):
-        self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'),
-                                               'http://username:password@www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('http://username:password@www.example.com'),
+            'http://username:password@www.example.com')
 
     def test_preserve_http_complete_url(self):
-        self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
 
     def test_protocol_relative(self):
-        self.assertEqual(add_http_if_no_scheme('//www.example.com'),
-                                               'http://www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('//www.example.com'), 'http://www.example.com')
 
     def test_protocol_relative_without_subdomain(self):
-        self.assertEqual(add_http_if_no_scheme('//example.com'),
-                                               'http://example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('//example.com'), 'http://example.com')
 
     def test_protocol_relative_path(self):
-        self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'),
-                                               'http://www.example.com/some/page.html')
+        self.assertEqual(
+            add_http_if_no_scheme('//www.example.com/some/page.html'),
+            'http://www.example.com/some/page.html')
 
     def test_protocol_relative_port(self):
-        self.assertEqual(add_http_if_no_scheme('//www.example.com:80'),
-                                               'http://www.example.com:80')
+        self.assertEqual(
+            add_http_if_no_scheme('//www.example.com:80'),
+            'http://www.example.com:80')
 
     def test_protocol_relative_fragment(self):
-        self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'),
-                                               'http://www.example.com/some/page#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('//www.example.com/some/page#frag'),
+            'http://www.example.com/some/page#frag')
 
     def test_protocol_relative_query(self):
-        self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
-                                               'http://www.example.com/do?a=1&b=2&c=3')
+        self.assertEqual(
+            add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
+            'http://www.example.com/do?a=1&b=2&c=3')
 
     def test_protocol_relative_username_password(self):
-        self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'),
-                                               'http://username:password@www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('//username:password@www.example.com'),
+            'http://username:password@www.example.com')
 
     def test_protocol_relative_complete_url(self):
-        self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
 
     def test_preserve_https(self):
-        self.assertEqual(add_http_if_no_scheme('https://www.example.com'),
-                                               'https://www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('https://www.example.com'),
+            'https://www.example.com')
 
     def test_preserve_ftp(self):
-        self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'),
-                                               'ftp://www.example.com')
+        self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), 'ftp://www.example.com')
 
 
 class GuessSchemeTest(unittest.TestCase):

From 63600243e08cb7e783798bd6c59fb97595488e9e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 10:21:01 -0300
Subject: [PATCH 2790/2974] Flake8: remove E125 (Continuation line with same
 indent as next logical line)

Also remove E401 from pytest.ini - no occurrences in the codebase
---
 pytest.ini                             | 12 ++++----
 scrapy/pipelines/media.py              | 10 ++++---
 tests/test_spidermiddleware_referer.py | 40 +++++++++++++-------------
 tests/test_utils_url.py                | 14 ++++-----
 4 files changed, 39 insertions(+), 37 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 3eefe70f1..8ed1ad0cf 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -35,7 +35,7 @@ flake8-ignore =
     scrapy/commands/check.py E501
     scrapy/commands/crawl.py E501
     scrapy/commands/edit.py E501
-    scrapy/commands/fetch.py E401 E501 E128
+    scrapy/commands/fetch.py E501 E128
     scrapy/commands/genspider.py E128 E501
     scrapy/commands/parse.py E128 E501
     scrapy/commands/runspider.py E501
@@ -99,7 +99,7 @@ flake8-ignore =
     scrapy/pipelines/__init__.py E501
     scrapy/pipelines/files.py E116 E501
     scrapy/pipelines/images.py E501
-    scrapy/pipelines/media.py E125 E501
+    scrapy/pipelines/media.py E501
     # scrapy/selector
     scrapy/selector/__init__.py F403
     scrapy/selector/unified.py E501 E111
@@ -169,7 +169,7 @@ flake8-ignore =
     scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501
-    tests/mockserver.py E401 E501 E126 E123
+    tests/mockserver.py E501 E126 E123
     tests/pipelines.py F841
     tests/spiders.py E501
     tests/test_closespider.py E501
@@ -196,7 +196,7 @@ flake8-ignore =
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
     tests/test_dupefilters.py E501 E741 E128 E124
-    tests/test_engine.py E401 E501 E128
+    tests/test_engine.py E501 E128
     tests/test_exporters.py E501 E128 E124
     tests/test_extension_telnet.py F841
     tests/test_feedexport.py E501 F841
@@ -226,7 +226,7 @@ flake8-ignore =
     tests/test_spidermiddleware_httperror.py E128 E501 E121
     tests/test_spidermiddleware_offsite.py E501 E128 E111
     tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121
+    tests/test_spidermiddleware_referer.py E501 F841 E124 E501 E121
     tests/test_squeues.py E501 E741
     tests/test_utils_asyncio.py E501
     tests/test_utils_conf.py E501 E128
@@ -243,7 +243,7 @@ flake8-ignore =
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841
     tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_url.py E501 E125 E501 E126 E123
+    tests/test_utils_url.py E501 E501 E126 E123
     tests/test_webclient.py E501 E128 E122 E402 E123 E126
     tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 8a0636264..aa65f4f0e 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -43,8 +43,7 @@ class MediaPipeline:
         if allow_redirects:
             self.handle_httpstatus_list = SequenceExclude(range(300, 400))
 
-    def _key_for_pipe(self, key, base_class_name=None,
-                      settings=None):
+    def _key_for_pipe(self, key, base_class_name=None, settings=None):
         """
         >>> MediaPipeline()._key_for_pipe("IMAGES")
         'IMAGES'
@@ -55,8 +54,11 @@ class MediaPipeline:
         """
         class_name = self.__class__.__name__
         formatted_key = "{}_{}".format(class_name.upper(), key)
-        if class_name == base_class_name or not base_class_name \
-            or (settings and not settings.get(formatted_key)):
+        if (
+            not base_class_name
+            or class_name == base_class_name
+            or settings and not settings.get(formatted_key)
+        ):
             return key
         return formatted_key
 
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 742adc64f..41589177a 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -478,32 +478,32 @@ class TestSettingsPolicyByName(TestCase):
 
     def test_valid_name(self):
         for s, p in [
-                (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
-                (POLICY_NO_REFERRER, NoReferrerPolicy),
-                (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
-                (POLICY_SAME_ORIGIN, SameOriginPolicy),
-                (POLICY_ORIGIN, OriginPolicy),
-                (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
-                (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
-                (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
-                (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
-            ]:
+            (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
+            (POLICY_NO_REFERRER, NoReferrerPolicy),
+            (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
+            (POLICY_SAME_ORIGIN, SameOriginPolicy),
+            (POLICY_ORIGIN, OriginPolicy),
+            (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
+            (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
+            (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
+            (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
+        ]:
             settings = Settings({'REFERRER_POLICY': s})
             mw = RefererMiddleware(settings)
             self.assertEqual(mw.default_policy, p)
 
     def test_valid_name_casevariants(self):
         for s, p in [
-                (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
-                (POLICY_NO_REFERRER, NoReferrerPolicy),
-                (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
-                (POLICY_SAME_ORIGIN, SameOriginPolicy),
-                (POLICY_ORIGIN, OriginPolicy),
-                (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
-                (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
-                (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
-                (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
-            ]:
+            (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
+            (POLICY_NO_REFERRER, NoReferrerPolicy),
+            (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
+            (POLICY_SAME_ORIGIN, SameOriginPolicy),
+            (POLICY_ORIGIN, OriginPolicy),
+            (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
+            (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
+            (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
+            (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
+        ]:
             settings = Settings({'REFERRER_POLICY': s.upper()})
             mw = RefererMiddleware(settings)
             self.assertEqual(mw.default_policy, p)
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 3bb6d40db..bed1a5634 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -288,7 +288,7 @@ class StripUrl(unittest.TestCase):
             ('http://www.example.com',
              True,
              'http://www.example.com/'),
-            ]:
+        ]:
             self.assertEqual(strip_url(input_url, origin_only=origin), output_url)
 
     def test_credentials(self):
@@ -301,7 +301,7 @@ class StripUrl(unittest.TestCase):
 
             ('ftp://username:password@www.example.com/index.html?somekey=somevalue#section',
              'ftp://www.example.com/index.html?somekey=somevalue'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i, strip_credentials=True), o)
 
     def test_credentials_encoded_delims(self):
@@ -320,7 +320,7 @@ class StripUrl(unittest.TestCase):
             # password: "user@domain.com"
             ('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section',
              'ftp://www.example.com/index.html?somekey=somevalue'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i, strip_credentials=True), o)
 
     def test_default_ports_creds_off(self):
@@ -348,7 +348,7 @@ class StripUrl(unittest.TestCase):
 
             ('ftp://username:password@www.example.com:221/file.txt',
              'ftp://www.example.com:221/file.txt'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i), o)
 
     def test_default_ports(self):
@@ -376,7 +376,7 @@ class StripUrl(unittest.TestCase):
 
             ('ftp://username:password@www.example.com:221/file.txt',
              'ftp://username:password@www.example.com:221/file.txt'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o)
 
     def test_default_ports_keep(self):
@@ -404,7 +404,7 @@ class StripUrl(unittest.TestCase):
 
             ('ftp://username:password@www.example.com:221/file.txt',
              'ftp://username:password@www.example.com:221/file.txt'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o)
 
     def test_origin_only(self):
@@ -420,7 +420,7 @@ class StripUrl(unittest.TestCase):
 
             ('https://username:password@www.example.com:443/index.html',
              'https://www.example.com/'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i, origin_only=True), o)
 
 

From 286fca733f23fa41165edcdcc7ab7593cc6b074f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 6 May 2020 16:20:33 +0200
Subject: [PATCH 2791/2974] Fix parameter name, broken by copy-pasting

---
 scrapy/settings/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 8f6fd3e6a..99ffa0dc9 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -248,7 +248,7 @@ class BaseSettings(MutableMapping):
         :type name: str
 
         :param value: the value to associate with the setting
-        :type default: object
+        :type value: object
 
         :param priority: the priority of the setting. Should be a key of
             :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer

From 628c4a531914b6803ae0ec4991363aad52069ca1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Panek?= <michal@panek.cloud>
Date: Wed, 6 May 2020 17:09:20 +0200
Subject: [PATCH 2792/2974] Add a warning/error in case of incorrect gcs
 permissions (#4508)

---
 scrapy/pipelines/files.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index ae365db5b..a9066986b 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -230,6 +230,20 @@ class GCSFilesStore:
         bucket, prefix = uri[5:].split('/', 1)
         self.bucket = client.bucket(bucket)
         self.prefix = prefix
+        permissions = self.bucket.test_iam_permissions(
+            ['storage.objects.get', 'storage.objects.create']
+        )
+        if 'storage.objects.get' not in permissions:
+            logger.warning(
+                "No 'storage.objects.get' permission for GSC bucket %(bucket)s. "
+                "Checking if files are up to date will be impossible. Files will be downloaded every time.",
+                {'bucket': bucket}
+            )
+        if 'storage.objects.create' not in permissions:
+            logger.error(
+                "No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!",
+                {'bucket': bucket}
+            )
 
     def stat_file(self, path, info):
         def _onsuccess(blob):

From 8643e8d3557449393989b15b9b8f2ec813f3e6ad Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 12:26:04 -0300
Subject: [PATCH 2793/2974] Flake8: remove E123 (Closing bracket does not match
 indentation of opening bracket's line)

---
 pytest.ini                                    | 18 +++---
 scrapy/extensions/closespider.py              |  2 +-
 scrapy/http/request/form.py                   | 11 ++--
 tests/test_downloader_handlers.py             | 19 +++---
 ...st_downloadermiddleware_httpcompression.py | 24 +++----
 tests/test_http_request.py                    |  2 +-
 tests/test_scheduler.py                       | 30 +++++----
 tests/test_utils_url.py                       | 64 +++++++++++--------
 tests/test_webclient.py                       |  6 +-
 9 files changed, 95 insertions(+), 81 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 8ed1ad0cf..1a73b41be 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -73,7 +73,7 @@ flake8-ignore =
     scrapy/downloadermiddlewares/robotstxt.py E501
     scrapy/downloadermiddlewares/stats.py E501
     # scrapy/extensions
-    scrapy/extensions/closespider.py E501 E128 E123
+    scrapy/extensions/closespider.py E501 E128
     scrapy/extensions/corestats.py E501
     scrapy/extensions/feedexport.py E128 E501
     scrapy/extensions/httpcache.py E128 E501
@@ -85,7 +85,7 @@ flake8-ignore =
     scrapy/http/common.py E501
     scrapy/http/cookies.py E501
     scrapy/http/request/__init__.py E501
-    scrapy/http/request/form.py E501 E123
+    scrapy/http/request/form.py E501
     scrapy/http/request/json_request.py E501
     scrapy/http/response/__init__.py E501 E128
     scrapy/http/response/text.py E501 E128 E124
@@ -169,7 +169,7 @@ flake8-ignore =
     scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501
-    tests/mockserver.py E501 E126 E123
+    tests/mockserver.py E501 E126
     tests/pipelines.py F841
     tests/spiders.py E501
     tests/test_closespider.py E501
@@ -181,14 +181,14 @@ flake8-ignore =
     tests/test_crawl.py E501 E741
     tests/test_crawler.py F841 E501
     tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E124 E128 E501 E126 E123
+    tests/test_downloader_handlers.py E124 E128 E501 E126
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
     tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
     tests/test_downloadermiddleware_httpcache.py E501
-    tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
+    tests/test_downloadermiddleware_httpcompression.py E501 E126
     tests/test_downloadermiddleware_decompression.py E501
     tests/test_downloadermiddleware_httpproxy.py E501 E128
     tests/test_downloadermiddleware_redirect.py E501 E128
@@ -202,7 +202,7 @@ flake8-ignore =
     tests/test_feedexport.py E501 F841
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
-    tests/test_http_request.py E402 E501 E128 E128 E126 E123
+    tests/test_http_request.py E402 E501 E128 E128 E126
     tests/test_http_response.py E501 E128
     tests/test_item.py E128 F841
     tests/test_link.py E501
@@ -219,7 +219,7 @@ flake8-ignore =
     tests/test_request_cb_kwargs.py E501
     tests/test_responsetypes.py E501
     tests/test_robotstxt_interface.py E501 E501
-    tests/test_scheduler.py E501 E126 E123
+    tests/test_scheduler.py E501 E126
     tests/test_selector.py E501
     tests/test_spider.py E501
     tests/test_spidermiddleware.py E501
@@ -243,8 +243,8 @@ flake8-ignore =
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841
     tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_url.py E501 E501 E126 E123
-    tests/test_webclient.py E501 E128 E122 E402 E123 E126
+    tests/test_utils_url.py E501 E501 E126
+    tests/test_webclient.py E501 E128 E122 E402 E126
     tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
     tests/test_spiderloader/__init__.py E128 E501
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index e3f212bef..812844c0a 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -20,7 +20,7 @@ class CloseSpider:
             'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'),
             'pagecount': crawler.settings.getint('CLOSESPIDER_PAGECOUNT'),
             'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'),
-            }
+        }
 
         if not any(self.close_on.values()):
             raise NotConfigured
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index af02c8484..cd4e3373f 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -178,12 +178,11 @@ def _get_clickable(clickdata, form):
     if the latter is given. If not, it returns the first
     clickable element found
     """
-    clickables = [
-        el for el in form.xpath(
-            'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
-            '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
-            namespaces={"re": "http://exslt.org/regular-expressions"})
-        ]
+    clickables = list(form.xpath(
+        'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
+        '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
+        namespaces={"re": "http://exslt.org/regular-expressions"}
+    ))
     if not clickables:
         return
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 24ef560c1..f93bce8ef 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -822,11 +822,15 @@ class S3TestCase(unittest.TestCase):
     def test_request_signing2(self):
         # puts an object into the johnsmith bucket.
         date = 'Tue, 27 Mar 2007 21:15:45 +0000'
-        req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={
-            'Content-Type': 'image/jpeg',
-            'Date': date,
-            'Content-Length': '94328',
-            })
+        req = Request(
+            's3://johnsmith/photos/puppy.jpg',
+            method='PUT',
+            headers={
+                'Content-Type': 'image/jpeg',
+                'Date': date,
+                'Content-Length': '94328',
+            },
+        )
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'],
@@ -906,11 +910,10 @@ class S3TestCase(unittest.TestCase):
         # ensure that spaces are quoted properly before signing
         date = 'Tue, 27 Mar 2007 19:42:41 +0000'
         req = Request(
-            ("s3://johnsmith/photos/my puppy.jpg"
-             "?response-content-disposition=my puppy.jpg"),
+            "s3://johnsmith/photos/my puppy.jpg?response-content-disposition=my puppy.jpg",
             method='GET',
             headers={'Date': date},
-            )
+        )
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
         self.assertEqual(
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 106ca3360..e86568bfb 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -16,12 +16,12 @@ from w3lib.encoding import resolve_encoding
 SAMPLEDIR = join(tests_datadir, 'compressed')
 
 FORMAT = {
-        'gzip': ('html-gzip.bin', 'gzip'),
-        'x-gzip': ('html-gzip.bin', 'gzip'),
-        'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
-        'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
-        'br': ('html-br.bin', 'br')
-        }
+    'gzip': ('html-gzip.bin', 'gzip'),
+    'x-gzip': ('html-gzip.bin', 'gzip'),
+    'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
+    'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
+    'br': ('html-br.bin', 'br'),
+}
 
 
 class HttpCompressionTest(TestCase):
@@ -40,12 +40,12 @@ class HttpCompressionTest(TestCase):
             body = sample.read()
 
         headers = {
-                'Server': 'Yaws/1.49 Yet Another Web Server',
-                'Date': 'Sun, 08 Mar 2009 00:41:03 GMT',
-                'Content-Length': len(body),
-                'Content-Type': 'text/html',
-                'Content-Encoding': contentencoding,
-                }
+            'Server': 'Yaws/1.49 Yet Another Web Server',
+            'Date': 'Sun, 08 Mar 2009 00:41:03 GMT',
+            'Content-Length': len(body),
+            'Content-Type': 'text/html',
+            'Content-Encoding': contentencoding,
+        }
 
         response = Response('http://scrapytest.org/', body=body, headers=headers)
         response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'})
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index b12841ba2..3b6d119a9 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -467,7 +467,7 @@ class FormRequestTest(RequestTest):
             </form>""",
             url="http://www.example.com/this/list.html",
             encoding='latin1',
-            )
+        )
         req = self.request_class.from_response(response,
                 formdata={'one': ['two', 'three'], 'six': 'seven'})
 
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 00568aee9..930a5dd99 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -46,13 +46,13 @@ class MockCrawler(Crawler):
     def __init__(self, priority_queue_cls, jobdir):
 
         settings = dict(
-                SCHEDULER_DEBUG=False,
-                SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
-                SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
-                SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
-                JOBDIR=jobdir,
-                DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter'
-                )
+            SCHEDULER_DEBUG=False,
+            SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
+            SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
+            SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
+            JOBDIR=jobdir,
+            DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
+        )
         super(MockCrawler, self).__init__(Spider, settings)
         self.engine = MockEngine(downloader=MockDownloader())
 
@@ -305,10 +305,12 @@ class StartUrlsSpider(Spider):
 class TestIntegrationWithDownloaderAwareInMemory(TestCase):
     def setUp(self):
         self.crawler = get_crawler(
-                    StartUrlsSpider,
-                    {'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
-                     'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'}
-                    )
+            spidercls=StartUrlsSpider,
+            settings_dict={
+                'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
+                'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter',
+            },
+        )
 
     @defer.inlineCallbacks
     def tearDown(self):
@@ -329,9 +331,9 @@ class TestIncompatibility(unittest.TestCase):
 
     def _incompatible(self):
         settings = dict(
-                SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
-                CONCURRENT_REQUESTS_PER_IP=1
-                )
+            SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
+            CONCURRENT_REQUESTS_PER_IP=1,
+        )
         crawler = Crawler(Spider, settings)
         scheduler = Scheduler.from_crawler(crawler)
         spider = Spider(name='spider')
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index bed1a5634..1f8388957 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -218,41 +218,49 @@ def create_skipped_scheme_t(args):
     return do_expected
 
 
-for k, args in enumerate([
-            ('/index', 'file://'),
-            ('/index.html', 'file://'),
-            ('./index.html', 'file://'),
-            ('../index.html', 'file://'),
-            ('../../index.html', 'file://'),
-            ('./data/index.html', 'file://'),
-            ('.hidden/data/index.html', 'file://'),
-            ('/home/user/www/index.html', 'file://'),
-            ('//home/user/www/index.html', 'file://'),
-            ('file:///home/user/www/index.html', 'file://'),
+for k, args in enumerate(
+    [
+        ('/index', 'file://'),
+        ('/index.html', 'file://'),
+        ('./index.html', 'file://'),
+        ('../index.html', 'file://'),
+        ('../../index.html', 'file://'),
+        ('./data/index.html', 'file://'),
+        ('.hidden/data/index.html', 'file://'),
+        ('/home/user/www/index.html', 'file://'),
+        ('//home/user/www/index.html', 'file://'),
+        ('file:///home/user/www/index.html', 'file://'),
 
-            ('index.html', 'http://'),
-            ('example.com', 'http://'),
-            ('www.example.com', 'http://'),
-            ('www.example.com/index.html', 'http://'),
-            ('http://example.com', 'http://'),
-            ('http://example.com/index.html', 'http://'),
-            ('localhost', 'http://'),
-            ('localhost/index.html', 'http://'),
+        ('index.html', 'http://'),
+        ('example.com', 'http://'),
+        ('www.example.com', 'http://'),
+        ('www.example.com/index.html', 'http://'),
+        ('http://example.com', 'http://'),
+        ('http://example.com/index.html', 'http://'),
+        ('localhost', 'http://'),
+        ('localhost/index.html', 'http://'),
 
-            # some corner cases (default to http://)
-            ('/', 'http://'),
-            ('.../test', 'http://'),
-
-        ], start=1):
+        # some corner cases (default to http://)
+        ('/', 'http://'),
+        ('.../test', 'http://'),
+    ],
+    start=1,
+):
     t_method = create_guess_scheme_t(args)
     t_method.__name__ = 'test_uri_%03d' % k
     setattr(GuessSchemeTest, t_method.__name__, t_method)
 
 # TODO: the following tests do not pass with current implementation
-for k, args in enumerate([
-            (r'C:\absolute\path\to\a\file.html', 'file://',
-             'Windows filepath are not supported for scrapy shell'),
-        ], start=1):
+for k, args in enumerate(
+    [
+        (
+            r'C:\absolute\path\to\a\file.html',
+            'file://',
+            'Windows filepath are not supported for scrapy shell',
+        ),
+    ],
+    start=1,
+):
     t_method = create_skipped_scheme_t(args)
     t_method.__name__ = 'test_uri_skipped_%03d' % k
     setattr(GuessSchemeTest, t_method.__name__, t_method)
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index d4abebbfb..de61e2125 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -149,7 +149,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             headers={
                 'X-Meta-Single': 'single',
                 'X-Meta-Multivalued': ['value1', 'value2'],
-                }))
+            },
+        ))
 
         self._test(factory,
             b"GET /bar HTTP/1.0\r\n"
@@ -165,7 +166,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             headers=Headers({
                 'X-Meta-Single': 'single',
                 'X-Meta-Multivalued': ['value1', 'value2'],
-                })))
+            }),
+        ))
 
         self._test(factory,
             b"GET /bar HTTP/1.0\r\n"

From d0bb04f08936435202488404d08c0b82f25aa1e5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 13:37:23 -0300
Subject: [PATCH 2794/2974] Switch to pickle protocol 4

---
 scrapy/exporters.py              | 2 +-
 scrapy/extensions/httpcache.py   | 4 ++--
 scrapy/extensions/spiderstate.py | 2 +-
 scrapy/squeues.py                | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 0cb6cef98..349a9586b 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -250,7 +250,7 @@ class CsvItemExporter(BaseItemExporter):
 
 class PickleItemExporter(BaseItemExporter):
 
-    def __init__(self, file, protocol=2, **kwargs):
+    def __init__(self, file, protocol=4, **kwargs):
         super().__init__(**kwargs)
         self.file = file
         self.protocol = protocol
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 8546628a8..7972b58b1 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -250,7 +250,7 @@ class DbmCacheStorage:
             'headers': dict(response.headers),
             'body': response.body,
         }
-        self.db['%s_data' % key] = pickle.dumps(data, protocol=2)
+        self.db['%s_data' % key] = pickle.dumps(data, protocol=4)
         self.db['%s_time' % key] = str(time())
 
     def _read_data(self, spider, request):
@@ -317,7 +317,7 @@ class FilesystemCacheStorage:
         with self._open(os.path.join(rpath, 'meta'), 'wb') as f:
             f.write(to_bytes(repr(metadata)))
         with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f:
-            pickle.dump(metadata, f, protocol=2)
+            pickle.dump(metadata, f, protocol=4)
         with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f:
             f.write(headers_dict_to_raw(response.headers))
         with self._open(os.path.join(rpath, 'response_body'), 'wb') as f:
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 2e5ff569f..bea00596e 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -26,7 +26,7 @@ class SpiderState:
     def spider_closed(self, spider):
         if self.jobdir:
             with open(self.statefn, 'wb') as f:
-                pickle.dump(spider.state, f, protocol=2)
+                pickle.dump(spider.state, f, protocol=4)
 
     def spider_opened(self, spider):
         if self.jobdir and os.path.exists(self.statefn):
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index d0686dac3..8d05bd0d0 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -81,7 +81,7 @@ def _scrapy_non_serialization_queue(queue_class):
 
 def _pickle_serialize(obj):
     try:
-        return pickle.dumps(obj, protocol=2)
+        return pickle.dumps(obj, protocol=4)
     # Python <= 3.4 raises pickle.PicklingError here while
     # 3.5 <= Python < 3.6 raises AttributeError and
     # Python >= 3.6 raises TypeError

From b1ddd7bd7b84d8d8417228aa7392d418463c9728 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 13:44:02 -0300
Subject: [PATCH 2795/2974] Refactor test_squeues.py

---
 tests/test_squeues.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 5ad8035f7..7e997a25e 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -47,12 +47,7 @@ def nonserializable_object_test(self):
     self.assertRaises(ValueError, q.push, sel)
 
 
-class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
-
-    chunksize = 100000
-
-    def queue(self):
-        return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize)
+class FifoDiskQueueTestMixin:
 
     def test_serialize(self):
         q = self.queue()
@@ -66,6 +61,13 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
     test_nonserializable_object = nonserializable_object_test
 
 
+class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
+    chunksize = 100000
+
+    def queue(self):
+        return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize)
+
+
 class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
     chunksize = 1
 
@@ -82,7 +84,7 @@ class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
     chunksize = 4
 
 
-class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest):
+class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
 
     chunksize = 100000
 
@@ -133,10 +135,7 @@ class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
     chunksize = 4
 
 
-class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
-
-    def queue(self):
-        return MarshalLifoDiskQueue(self.qpath)
+class LifoDiskQueueTestMixin:
 
     def test_serialize(self):
         q = self.queue()
@@ -150,7 +149,13 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
     test_nonserializable_object = nonserializable_object_test
 
 
-class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest):
+class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
+
+    def queue(self):
+        return MarshalLifoDiskQueue(self.qpath)
+
+
+class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
 
     def queue(self):
         return PickleLifoDiskQueue(self.qpath)

From 93436f9d3a67cd8abe3b321321c2d36d94f75b8b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 14:05:27 -0300
Subject: [PATCH 2796/2974] Chain pickling exception, test_squeues.py updates

---
 scrapy/squeues.py     |  7 +++----
 tests/test_squeues.py | 28 ++++++++++++++--------------
 2 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 8d05bd0d0..c7ad4d53d 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -82,11 +82,10 @@ def _scrapy_non_serialization_queue(queue_class):
 def _pickle_serialize(obj):
     try:
         return pickle.dumps(obj, protocol=4)
-    # Python <= 3.4 raises pickle.PicklingError here while
-    # 3.5 <= Python < 3.6 raises AttributeError and
-    # Python >= 3.6 raises TypeError
+    # Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s)
+    # TypeError is raised from parsel.Selector
     except (pickle.PicklingError, AttributeError, TypeError) as e:
-        raise ValueError(str(e))
+        raise ValueError(str(e)) from e
 
 
 PickleFifoDiskQueueNonRequest = _serializable_queue(
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 7e997a25e..a20d242f4 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -28,20 +28,7 @@ class TestLoader(ItemLoader):
 
 def nonserializable_object_test(self):
     q = self.queue()
-    try:
-        pickle.dumps(lambda x: x)
-    except Exception:
-        # Trigger Twisted bug #7989
-        import twisted.persisted.styles  # NOQA
-        self.assertRaises(ValueError, q.push, lambda x: x)
-    else:
-        # Use a different unpickleable object
-        class A:
-            pass
-
-        a = A()
-        a.__reduce__ = a.__reduce_ex__ = None
-        self.assertRaises(ValueError, q.push, a)
+    self.assertRaises(ValueError, q.push, lambda x: x)
     # Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
     sel = Selector(text='<html><body><p>some text</p></body></html>')
     self.assertRaises(ValueError, q.push, sel)
@@ -118,6 +105,19 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
         self.assertEqual(r.url, r2.url)
         assert r2.meta['request'] is r2
 
+    def test_non_pickable_object(self):
+        q = self.queue()
+        try:
+            q.push(lambda x: x)
+        except ValueError as exc:
+            self.assertIsInstance(exc.__context__, AttributeError)
+
+        sel = Selector(text='<html><body><p>some text</p></body></html>')
+        try:
+            q.push(sel)
+        except ValueError as exc:
+            self.assertIsInstance(exc.__context__, TypeError)
+
 
 class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
     chunksize = 1

From 0e382c816024baffca05b0da29def95f723d27fd Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 14:09:10 -0300
Subject: [PATCH 2797/2974] Remove unused import

---
 tests/test_squeues.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index a20d242f4..51c0c028a 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -1,5 +1,3 @@
-import pickle
-
 from queuelib.tests import test_queue as t
 from scrapy.squeues import (
     MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue,

From d71804ef29a00fb526ac496930356a47006c639d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 15:23:36 -0300
Subject: [PATCH 2798/2974] Flake8: Remove E122

---
 pytest.ini              |  4 ++--
 tests/test_webclient.py | 40 ++++++++++++++++++++--------------------
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 1a73b41be..f8c4ce19a 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -208,7 +208,7 @@ flake8-ignore =
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128 E124
     tests/test_loader.py E501 E741 E128 E117
-    tests/test_logformatter.py E128 E501 E122
+    tests/test_logformatter.py E128 E501
     tests/test_mail.py E128 E501
     tests/test_middleware.py E501 E128
     tests/test_pipeline_crawl.py E501 E128 E126
@@ -244,7 +244,7 @@ flake8-ignore =
     tests/test_utils_signal.py E741 F841
     tests/test_utils_sitemap.py E128 E501 E124
     tests/test_utils_url.py E501 E501 E126
-    tests/test_webclient.py E501 E128 E122 E402 E126
+    tests/test_webclient.py E501 E128 E402 E126
     tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
     tests/test_spiderloader/__init__.py E128 E501
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index de61e2125..b657c7ab6 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -53,29 +53,29 @@ class ParseUrlTestCase(unittest.TestCase):
     def testParse(self):
         lip = '127.0.0.1'
         tests = (
-    ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
-    ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
-    ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
-    ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
-    ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
-    ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
+            ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
+            ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
+            ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
+            ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
+            ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
+            ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
 
-    ("http://127.0.0.1", ('http', lip, lip, 80, '/')),
-    ("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
-    ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
-    ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
-    ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
-    ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
-    ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
-    ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
+            ("http://127.0.0.1", ('http', lip, lip, 80, '/')),
+            ("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
+            ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
+            ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
+            ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
+            ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
+            ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
+            ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
 
-    ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
-    ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
-    ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
+            ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
+            ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
+            ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
 
-    ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
-    ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
-    )
+            ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
+            ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
+        )
 
         for url, test in tests:
             test = tuple(

From cc23d1cb580795f6fde6e27f92f39ce4e3b8b558 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 15:40:37 -0300
Subject: [PATCH 2799/2974] Flake8: Remove E124

---
 pytest.ini                   | 14 +++----
 tests/test_dupefilters.py    |  3 +-
 tests/test_exporters.py      | 76 +++++++++++++++++++-----------------
 tests/test_linkextractors.py | 34 +++++++++-------
 tests/test_utils_sitemap.py  | 51 ++++++++++++++++--------
 5 files changed, 103 insertions(+), 75 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index f8c4ce19a..998633d54 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -88,7 +88,7 @@ flake8-ignore =
     scrapy/http/request/form.py E501
     scrapy/http/request/json_request.py E501
     scrapy/http/response/__init__.py E501 E128
-    scrapy/http/response/text.py E501 E128 E124
+    scrapy/http/response/text.py E501 E128
     # scrapy/linkextractors
     scrapy/linkextractors/__init__.py E501 E402
     scrapy/linkextractors/lxmlhtml.py E501
@@ -181,7 +181,7 @@ flake8-ignore =
     tests/test_crawl.py E501 E741
     tests/test_crawler.py F841 E501
     tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E124 E128 E501 E126
+    tests/test_downloader_handlers.py E128 E501 E126
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
     tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
@@ -195,9 +195,9 @@ flake8-ignore =
     tests/test_downloadermiddleware_retry.py E501 E128 E126
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
-    tests/test_dupefilters.py E501 E741 E128 E124
+    tests/test_dupefilters.py E501 E741 E128
     tests/test_engine.py E501 E128
-    tests/test_exporters.py E501 E128 E124
+    tests/test_exporters.py E501 E128
     tests/test_extension_telnet.py F841
     tests/test_feedexport.py E501 F841
     tests/test_http_cookies.py E501
@@ -206,7 +206,7 @@ flake8-ignore =
     tests/test_http_response.py E501 E128
     tests/test_item.py E128 F841
     tests/test_link.py E501
-    tests/test_linkextractors.py E501 E128 E124
+    tests/test_linkextractors.py E501 E128
     tests/test_loader.py E501 E741 E128 E117
     tests/test_logformatter.py E128 E501
     tests/test_mail.py E128 E501
@@ -226,7 +226,7 @@ flake8-ignore =
     tests/test_spidermiddleware_httperror.py E128 E501 E121
     tests/test_spidermiddleware_offsite.py E501 E128 E111
     tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 F841 E124 E501 E121
+    tests/test_spidermiddleware_referer.py E501 F841 E501 E121
     tests/test_squeues.py E501 E741
     tests/test_utils_asyncio.py E501
     tests/test_utils_conf.py E501 E128
@@ -242,7 +242,7 @@ flake8-ignore =
     tests/test_utils_request.py E501 E128
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841
-    tests/test_utils_sitemap.py E128 E501 E124
+    tests/test_utils_sitemap.py E128 E501
     tests/test_utils_url.py E501 E501 E126
     tests/test_webclient.py E501 E128 E402 E126
     tests/test_cmdline/__init__.py E501
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index ea0e664be..7426107c1 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -197,8 +197,7 @@ class RFPDupeFilterTest(unittest.TestCase):
 
             r1 = Request('http://scrapytest.org/index.html')
             r2 = Request('http://scrapytest.org/index.html',
-                headers={'Referer': 'http://scrapytest.org/INDEX.html'}
-            )
+                         headers={'Referer': 'http://scrapytest.org/INDEX.html'})
 
             dupefilter.log(r1, spider)
             dupefilter.log(r2, spider)
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 160912847..0f9dafcaa 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -342,20 +342,22 @@ class XmlItemExporterTest(BaseItemExporterTest):
         i2 = dict(name=u'bar', age=i1)
         i3 = TestItem(name=u'buz', age=i2)
 
-        self.assertExportResult(i3,
-            b'<?xml version="1.0" encoding="utf-8"?>\n'
-            b'<items>'
-                b'<item>'
-                    b'<age>'
-                        b'<age>'
-                            b'<age>22</age>'
-                            b'<name>foo\xc2\xa3hoo</name>'
-                        b'</age>'
-                        b'<name>bar</name>'
-                    b'</age>'
-                    b'<name>buz</name>'
-                b'</item>'
-            b'</items>'
+        self.assertExportResult(
+            i3,
+            b"""<?xml version="1.0" encoding="utf-8"?>\n
+                <items>
+                    <item>
+                        <age>
+                            <age>
+                                <age>22</age>
+                                <name>foo\xc2\xa3hoo</name>
+                            </age>
+                            <name>bar</name>
+                        </age>
+                        <name>buz</name>
+                    </item>
+                </items>
+            """
         )
 
     def test_nested_list_item(self):
@@ -363,31 +365,35 @@ class XmlItemExporterTest(BaseItemExporterTest):
         i2 = dict(name=u'bar', v2={"egg": ["spam"]})
         i3 = TestItem(name=u'buz', age=[i1, i2])
 
-        self.assertExportResult(i3,
-            b'<?xml version="1.0" encoding="utf-8"?>\n'
-            b'<items>'
-                b'<item>'
-                    b'<age>'
-                        b'<value><name>foo</name></value>'
-                        b'<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
-                    b'</age>'
-                    b'<name>buz</name>'
-                b'</item>'
-            b'</items>'
+        self.assertExportResult(
+            i3,
+            b"""<?xml version="1.0" encoding="utf-8"?>\n
+                <items>
+                    <item>
+                        <age>
+                            <value><name>foo</name></value>
+                            <value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>
+                        </age>
+                        <name>buz</name>
+                    </item>
+                </items>
+            """
         )
 
     def test_nonstring_types_item(self):
         item = self._get_nonstring_types_item()
-        self.assertExportResult(item,
-            b'<?xml version="1.0" encoding="utf-8"?>\n'
-            b'<items>'
-               b'<item>'
-                   b'<float>3.14</float>'
-                   b'<boolean>False</boolean>'
-                   b'<number>22</number>'
-                   b'<time>2015-01-01 01:01:01</time>'
-               b'</item>'
-            b'</items>'
+        self.assertExportResult(
+            item,
+            b"""<?xml version="1.0" encoding="utf-8"?>\n
+                <items>
+                   <item>
+                       <float>3.14</float>
+                       <boolean>False</boolean>
+                       <number>22</number>
+                       <time>2015-01-01 01:01:01</time>
+                   </item>
+                </items>
+            """
         )
 
 
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 53968e60e..68e8514ba 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -413,24 +413,30 @@ class Base:
             response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)
 
             lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                              Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                              Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                              Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
-                              Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
-                            )
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
+                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
+                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                ]
+            )
 
             response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
 
             lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                              Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                              Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                              Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
-                              Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
-                            )
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
+                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
+                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                ]
+            )
 
         def test_link_wrong_href(self):
             html = b"""
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index db323ab31..08b215434 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -58,10 +58,13 @@ class SitemapTest(unittest.TestCase):
   </url>
 </urlset>
 """)
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-             {'loc': 'http://www.example.com/2', 'lastmod': ''},
-            ])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'loc': 'http://www.example.com/2', 'lastmod': ''},
+            ]
+        )
 
     def test_sitemap_wrong_ns(self):
         """We have seen sitemaps with wrongs ns. Presumably, Google still works
@@ -80,10 +83,13 @@ class SitemapTest(unittest.TestCase):
   </url>
 </urlset>
 """)
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-             {'loc': 'http://www.example.com/2', 'lastmod': ''},
-            ])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'loc': 'http://www.example.com/2', 'lastmod': ''},
+            ]
+        )
 
     def test_sitemap_wrong_ns2(self):
         """We have seen sitemaps with wrongs ns. Presumably, Google still works
@@ -103,10 +109,13 @@ class SitemapTest(unittest.TestCase):
 </urlset>
 """)
         assert s.type == 'urlset'
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-             {'loc': 'http://www.example.com/2', 'lastmod': ''},
-            ])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'loc': 'http://www.example.com/2', 'lastmod': ''},
+            ]
+        )
 
     def test_sitemap_urls_from_robots(self):
         robots = """User-agent: *
@@ -195,11 +204,19 @@ Disallow: /forum/active/
         </url>
     </urlset>""")
 
-        self.assertEqual(list(s), [
-            {'loc': 'http://www.example.com/english/',
-             'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/']
-            }
-        ])
+        self.assertEqual(
+            list(s),
+            [
+                {
+                    'loc': 'http://www.example.com/english/',
+                    'alternate': [
+                        'http://www.example.com/deutsch/',
+                        'http://www.example.com/schweiz-deutsch/',
+                        'http://www.example.com/english/',
+                    ],
+                }
+            ]
+        )
 
     def test_xml_entity_expansion(self):
         s = Sitemap(b"""<?xml version="1.0" encoding="utf-8"?>

From 4c12a234ae65d49678a9840708ff5e7b9d6dcecc Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 16:10:21 -0300
Subject: [PATCH 2800/2974] Flake8: Remove E126

---
 pytest.ini                                 | 34 +++++++++++-----------
 scrapy/core/downloader/contextfactory.py   | 10 +++----
 scrapy/core/downloader/handlers/s3.py      | 13 +++++----
 scrapy/core/downloader/webclient.py        |  4 +--
 scrapy/downloadermiddlewares/retry.py      | 12 ++++++--
 scrapy/spiderloader.py                     | 25 +++++++++-------
 tests/test_downloadermiddleware_cookies.py | 10 ++++---
 tests/test_downloadermiddleware_retry.py   | 12 ++++++--
 tests/test_http_request.py                 | 32 +++++++++++---------
 tests/test_pipeline_crawl.py               |  4 +--
 tests/test_webclient.py                    | 13 +++++----
 11 files changed, 96 insertions(+), 73 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 998633d54..1570a3a75 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -50,26 +50,26 @@ flake8-ignore =
     scrapy/core/engine.py E501 E128
     scrapy/core/scheduler.py E501
     scrapy/core/scraper.py E501 E128
-    scrapy/core/spidermw.py E501 E126
+    scrapy/core/spidermw.py E501
     scrapy/core/downloader/__init__.py E501
-    scrapy/core/downloader/contextfactory.py E501 E128 E126
+    scrapy/core/downloader/contextfactory.py E501 E128
     scrapy/core/downloader/middleware.py E501
     scrapy/core/downloader/tls.py E501
-    scrapy/core/downloader/webclient.py E501 E128 E126
+    scrapy/core/downloader/webclient.py E501 E128
     scrapy/core/downloader/handlers/__init__.py E501
     scrapy/core/downloader/handlers/ftp.py E501 E128
     scrapy/core/downloader/handlers/http10.py E501
     scrapy/core/downloader/handlers/http11.py E501
-    scrapy/core/downloader/handlers/s3.py E501 E128 E126
+    scrapy/core/downloader/handlers/s3.py E501 E128
     # scrapy/downloadermiddlewares
     scrapy/downloadermiddlewares/ajaxcrawl.py E501
     scrapy/downloadermiddlewares/decompression.py E501
     scrapy/downloadermiddlewares/defaultheaders.py E501
-    scrapy/downloadermiddlewares/httpcache.py E501 E126
+    scrapy/downloadermiddlewares/httpcache.py E501
     scrapy/downloadermiddlewares/httpcompression.py E501 E128
     scrapy/downloadermiddlewares/httpproxy.py E501
     scrapy/downloadermiddlewares/redirect.py E501
-    scrapy/downloadermiddlewares/retry.py E501 E126
+    scrapy/downloadermiddlewares/retry.py E501
     scrapy/downloadermiddlewares/robotstxt.py E501
     scrapy/downloadermiddlewares/stats.py E501
     # scrapy/extensions
@@ -164,12 +164,12 @@ flake8-ignore =
     scrapy/robotstxt.py E501
     scrapy/shell.py E501
     scrapy/signalmanager.py E501
-    scrapy/spiderloader.py F841 E501 E126
+    scrapy/spiderloader.py F841 E501
     scrapy/squeues.py E128
     scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501
-    tests/mockserver.py E501 E126
+    tests/mockserver.py E501
     tests/pipelines.py F841
     tests/spiders.py E501
     tests/test_closespider.py E501
@@ -181,18 +181,18 @@ flake8-ignore =
     tests/test_crawl.py E501 E741
     tests/test_crawler.py F841 E501
     tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E128 E501 E126
+    tests/test_downloader_handlers.py E128 E501
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
-    tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
+    tests/test_downloadermiddleware_cookies.py E741 E501 E128
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
     tests/test_downloadermiddleware_httpcache.py E501
-    tests/test_downloadermiddleware_httpcompression.py E501 E126
+    tests/test_downloadermiddleware_httpcompression.py E501
     tests/test_downloadermiddleware_decompression.py E501
     tests/test_downloadermiddleware_httpproxy.py E501 E128
     tests/test_downloadermiddleware_redirect.py E501 E128
-    tests/test_downloadermiddleware_retry.py E501 E128 E126
+    tests/test_downloadermiddleware_retry.py E501 E128
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
     tests/test_dupefilters.py E501 E741 E128
@@ -202,7 +202,7 @@ flake8-ignore =
     tests/test_feedexport.py E501 F841
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
-    tests/test_http_request.py E402 E501 E128 E128 E126
+    tests/test_http_request.py E402 E501 E128 E128
     tests/test_http_response.py E501 E128
     tests/test_item.py E128 F841
     tests/test_link.py E501
@@ -211,7 +211,7 @@ flake8-ignore =
     tests/test_logformatter.py E128 E501
     tests/test_mail.py E128 E501
     tests/test_middleware.py E501 E128
-    tests/test_pipeline_crawl.py E501 E128 E126
+    tests/test_pipeline_crawl.py E501 E128
     tests/test_pipeline_files.py E501
     tests/test_pipeline_images.py F841 E501
     tests/test_pipeline_media.py E501 E741 E128
@@ -219,7 +219,7 @@ flake8-ignore =
     tests/test_request_cb_kwargs.py E501
     tests/test_responsetypes.py E501
     tests/test_robotstxt_interface.py E501 E501
-    tests/test_scheduler.py E501 E126
+    tests/test_scheduler.py E501
     tests/test_selector.py E501
     tests/test_spider.py E501
     tests/test_spidermiddleware.py E501
@@ -243,8 +243,8 @@ flake8-ignore =
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841
     tests/test_utils_sitemap.py E128 E501
-    tests/test_utils_url.py E501 E501 E126
-    tests/test_webclient.py E501 E128 E402 E126
+    tests/test_utils_url.py E501 E501
+    tests/test_webclient.py E501 E128 E402
     tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
     tests/test_spiderloader/__init__.py E128 E501
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 6e023ebcc..ab73e12c8 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -86,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
         #
         # This means that a website like https://www.cacert.org will be rejected
         # by default, since CAcert.org CA certificate is seldom shipped.
-        return optionsForClientTLS(hostname.decode("ascii"),
-                                   trustRoot=platformTrust(),
-                                   extraCertificateOptions={
-                                        'method': self._ssl_method,
-                                   })
+        return optionsForClientTLS(
+            hostname=hostname.decode("ascii"),
+            trustRoot=platformTrust(),
+            extraCertificateOptions={'method': self._ssl_method},
+        )
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 40a1fa48e..8f63ad974 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -100,11 +100,12 @@ class S3DownloadHandler:
                 url=url, headers=awsrequest.headers.items())
         else:
             signed_headers = self.conn.make_request(
-                    method=request.method,
-                    bucket=bucket,
-                    key=unquote(p.path),
-                    query_args=unquote(p.query),
-                    headers=request.headers,
-                    data=request.body)
+                method=request.method,
+                bucket=bucket,
+                key=unquote(p.path),
+                query_args=unquote(p.query),
+                headers=request.headers,
+                data=request.body,
+            )
             request = request.replace(url=url, headers=signed_headers)
         return self._download_http(request, spider)
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index a90a77b2b..355045d74 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -88,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient):
             self.transport.stopProducing()
 
         self.factory.noPage(
-                defer.TimeoutError("Getting %s took longer than %s seconds." %
-                                   (self.factory.url, self.factory.timeout)))
+            defer.TimeoutError("Getting %s took longer than %s seconds."
+                               % (self.factory.url, self.factory.timeout)))
 
 
 class ScrapyHTTPClientFactory(HTTPClientFactory):
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index bbf5fca05..6d11af5b2 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -12,9 +12,15 @@ once the spider has finished crawling all regular (non failed) pages.
 import logging
 
 from twisted.internet import defer
-from twisted.internet.error import TimeoutError, DNSLookupError, \
-        ConnectionRefusedError, ConnectionDone, ConnectError, \
-        ConnectionLost, TCPTimedOutError
+from twisted.internet.error import (
+    ConnectError,
+    ConnectionDone,
+    ConnectionLost,
+    ConnectionRefusedError,
+    DNSLookupError,
+    TCPTimedOutError,
+    TimeoutError,
+)
 from twisted.web.client import ResponseFailed
 
 from scrapy.exceptions import NotConfigured
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 3be5aaec5..8dc89c2e9 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -24,15 +24,17 @@ class SpiderLoader:
         self._load_all_spiders()
 
     def _check_name_duplicates(self):
-        dupes = ["\n".join("  {cls} named {name!r} (in {module})".format(
-                                module=mod, cls=cls, name=name)
-                           for (mod, cls) in locations)
-                 for name, locations in self._found.items()
-                 if len(locations) > 1]
+        dupes = []
+        for name, locations in self._found.items():
+            dupes.extend([
+                "  {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name)
+                for mod, cls in locations
+            ])
+
         if dupes:
+            dupes_string = "\n\n".join(dupes)
             msg = ("There are several spiders with the same name:\n\n"
-                   "{}\n\n  This can cause unexpected behavior.".format(
-                        "\n\n".join(dupes)))
+                   "{}\n\n  This can cause unexpected behavior.".format(dupes_string))
             warnings.warn(msg, UserWarning)
 
     def _load_spiders(self, module):
@@ -45,11 +47,12 @@ class SpiderLoader:
             try:
                 for module in walk_modules(name):
                     self._load_spiders(module)
-            except ImportError as e:
+            except ImportError:
                 if self.warn_only:
-                    msg = ("\n{tb}Could not load spiders from module '{modname}'. "
-                           "See above traceback for details.".format(
-                                modname=name, tb=traceback.format_exc()))
+                    msg = (
+                        "\n{tb}Could not load spiders from module '{modname}'. "
+                        "See above traceback for details.".format(modname=name, tb=traceback.format_exc())
+                    )
                     warnings.warn(msg, RuntimeWarning)
                 else:
                     raise
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index b686a14d6..f86c50f50 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -139,10 +139,12 @@ class CookiesMiddlewareTest(TestCase):
 
     def test_complex_cookies(self):
         # merge some cookies into jar
-        cookies = [{'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'},
-                {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'},
-                {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},
-                {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}]
+        cookies = [
+            {'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'},
+            {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'},
+            {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},
+            {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'},
+        ]
 
         req = Request('http://scrapytest.org/', cookies=cookies)
         self.mw.process_request(req, self.spider)
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 9c989977e..e118750e3 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -1,8 +1,14 @@
 import unittest
 from twisted.internet import defer
-from twisted.internet.error import TimeoutError, DNSLookupError, \
-        ConnectionRefusedError, ConnectionDone, ConnectError, \
-        ConnectionLost, TCPTimedOutError
+from twisted.internet.error import (
+    ConnectError,
+    ConnectionDone,
+    ConnectionLost,
+    ConnectionRefusedError,
+    DNSLookupError,
+    TCPTimedOutError,
+    TimeoutError,
+)
 from twisted.web.client import ResponseFailed
 
 from scrapy.downloadermiddlewares.retry import RetryMiddleware
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 3b6d119a9..77da15ce6 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -502,11 +502,13 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_duplicate_form_key(self):
         response = _buildresponse(
-                '<form></form>',
-                url='http://www.example.com')
-        req = self.request_class.from_response(response,
-                method='GET',
-                formdata=(('foo', 'bar'), ('foo', 'baz')))
+            '<form></form>',
+            url='http://www.example.com')
+        req = self.request_class.from_response(
+            response=response,
+            method='GET',
+            formdata=(('foo', 'bar'), ('foo', 'baz')),
+        )
         self.assertEqual(urlparse(req.url).hostname, 'www.example.com')
         self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz')
 
@@ -530,9 +532,11 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test" value="val2">
             <input type="hidden" name="test2" value="xxx">
             </form>""")
-        req = self.request_class.from_response(response,
-                formdata={'one': ['two', 'three'], 'six': 'seven'},
-                headers={"Accept-Encoding": "gzip,deflate"})
+        req = self.request_class.from_response(
+            response=response,
+            formdata={'one': ['two', 'three'], 'six': 'seven'},
+            headers={"Accept-Encoding": "gzip,deflate"},
+        )
         self.assertEqual(req.method, 'POST')
         self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
         self.assertEqual(req.headers['Accept-Encoding'], b'gzip,deflate')
@@ -580,9 +584,9 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_override_method(self):
         response = _buildresponse(
-                '''<html><body>
-                <form action="/app"></form>
-                </body></html>''')
+            '''<html><body>
+            <form action="/app"></form>
+            </body></html>''')
         request = FormRequest.from_response(response)
         self.assertEqual(request.method, 'GET')
         request = FormRequest.from_response(response, method='POST')
@@ -590,9 +594,9 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_override_url(self):
         response = _buildresponse(
-                '''<html><body>
-                <form action="/app"></form>
-                </body></html>''')
+            '''<html><body>
+            <form action="/app"></form>
+            </body></html>''')
         request = FormRequest.from_response(response)
         self.assertEqual(request.url, 'http://example.com/app')
         request = FormRequest.from_response(response, url='http://foo.bar/absolute')
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 962c33144..24c516473 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -44,9 +44,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):
     name = 'redirectedmedia'
 
     def _process_url(self, url):
-        return add_or_replace_parameter(
-                    self.mockserver.url('/redirect-to'),
-                    'goto', url)
+        return add_or_replace_parameter(self.mockserver.url('/redirect-to'), 'goto', url)
 
 
 class FileDownloadCrawlTestCase(TestCase):
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index b657c7ab6..307fadb5c 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -18,6 +18,14 @@ except ImportError:
 from twisted.python.filepath import FilePath
 from twisted.protocols.policies import WrappingFactory
 from twisted.internet.defer import inlineCallbacks
+from twisted.web.test.test_webclient import (
+    ForeverTakingResource,
+    ErrorResource,
+    NoLengthResource,
+    HostHeaderResource,
+    PayloadResource,
+    BrokenDownloadResource,
+)
 
 from scrapy.core.downloader import webclient as client
 from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
@@ -202,11 +210,6 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             Headers({'Hello': ['World'], 'Foo': ['Bar']}))
 
 
-from twisted.web.test.test_webclient import ForeverTakingResource, \
-        ErrorResource, NoLengthResource, HostHeaderResource, \
-        PayloadResource, BrokenDownloadResource
-
-
 class EncodingResource(resource.Resource):
     out_encoding = 'cp1251'
 

From 88efc988473fb0db8ca8fb512b2aab834c5aa7af Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 16:42:47 -0300
Subject: [PATCH 2801/2974] Flake8: remove E129

---
 pytest.ini                    | 4 ++--
 tests/test_utils_iterators.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 1a73b41be..ff0bb010f 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -110,7 +110,7 @@ flake8-ignore =
     # scrapy/spidermiddlewares
     scrapy/spidermiddlewares/httperror.py E501
     scrapy/spidermiddlewares/offsite.py E501
-    scrapy/spidermiddlewares/referer.py E501 E129
+    scrapy/spidermiddlewares/referer.py E501
     scrapy/spidermiddlewares/urllength.py E501
     # scrapy/spiders
     scrapy/spiders/__init__.py E501 E402
@@ -235,7 +235,7 @@ flake8-ignore =
     tests/test_utils_defer.py E501 F841
     tests/test_utils_deprecate.py F841 E501
     tests/test_utils_http.py E501 E128
-    tests/test_utils_iterators.py E501 E128 E129
+    tests/test_utils_iterators.py E501 E128
     tests/test_utils_log.py E741
     tests/test_utils_python.py E501
     tests/test_utils_reqser.py E501 E128
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index a85087619..46aaaecbc 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -93,8 +93,8 @@ class XmliterTestCase(unittest.TestCase):
             # with bytes
             XmlResponse(url="http://example.com", body=body.encode('utf-8')),
             # Unicode body needs encoding information
-            XmlResponse(url="http://example.com", body=body, encoding='utf-8')):
-
+            XmlResponse(url="http://example.com", body=body, encoding='utf-8'),
+        ):
             attrs = []
             for x in self.xmliter(r, u'þingflokkur'):
                 attrs.append((x.attrib['id'],

From 2af34873cc385e47cc962903ff3d6a13c9847615 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 17:10:10 -0300
Subject: [PATCH 2802/2974] Flake8: Remove E128 (commands module)

---
 pytest.ini                      | 16 ++++++++--------
 scrapy/commands/__init__.py     | 12 ++++++------
 scrapy/commands/fetch.py        |  4 ++--
 scrapy/commands/genspider.py    | 10 +++++-----
 scrapy/commands/parse.py        | 24 ++++++++++++------------
 scrapy/commands/settings.py     | 10 +++++-----
 scrapy/commands/shell.py        |  8 ++++----
 scrapy/commands/startproject.py |  6 ++----
 scrapy/commands/version.py      |  2 +-
 9 files changed, 45 insertions(+), 47 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 1570a3a75..d96ac9684 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -31,18 +31,18 @@ flake8-ignore =
     extras/qps-bench-server.py E501
     extras/qpsclient.py E501 E501
     # scrapy/commands
-    scrapy/commands/__init__.py E128 E501
+    scrapy/commands/__init__.py E501
     scrapy/commands/check.py E501
     scrapy/commands/crawl.py E501
     scrapy/commands/edit.py E501
-    scrapy/commands/fetch.py E501 E128
-    scrapy/commands/genspider.py E128 E501
-    scrapy/commands/parse.py E128 E501
+    scrapy/commands/fetch.py E501
+    scrapy/commands/genspider.py E501
+    scrapy/commands/parse.py E501
     scrapy/commands/runspider.py E501
-    scrapy/commands/settings.py E128
-    scrapy/commands/shell.py E128 E501
-    scrapy/commands/startproject.py E501 E128
-    scrapy/commands/version.py E501 E128
+    scrapy/commands/settings.py E501
+    scrapy/commands/shell.py E501
+    scrapy/commands/startproject.py E501
+    scrapy/commands/version.py E501
     # scrapy/contracts
     scrapy/contracts/__init__.py E501
     scrapy/contracts/default.py E128
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 9f8e6986a..97f818cb2 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -59,17 +59,17 @@ class ScrapyCommand:
         """
         group = OptionGroup(parser, "Global Options")
         group.add_option("--logfile", metavar="FILE",
-            help="log file. if omitted stderr will be used")
+                         help="log file. if omitted stderr will be used")
         group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
-            help="log level (default: %s)" % self.settings['LOG_LEVEL'])
+                         help="log level (default: %s)" % self.settings['LOG_LEVEL'])
         group.add_option("--nolog", action="store_true",
-            help="disable logging completely")
+                         help="disable logging completely")
         group.add_option("--profile", metavar="FILE", default=None,
-            help="write python cProfile stats to FILE")
+                         help="write python cProfile stats to FILE")
         group.add_option("--pidfile", metavar="FILE",
-            help="write process ID to FILE")
+                         help="write process ID to FILE")
         group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
-            help="set/override setting (may be repeated)")
+                         help="set/override setting (may be repeated)")
         group.add_option("--pdb", action="store_true", help="enable pdb on failure")
 
         parser.add_option_group(group)
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 506d1f1b7..063195f50 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -27,8 +27,8 @@ class Command(ScrapyCommand):
         parser.add_option("--spider", dest="spider", help="use this spider")
         parser.add_option("--headers", dest="headers", action="store_true",
                           help="print response HTTP headers instead of body")
-        parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
-                          default=False, help="do not handle HTTP 3xx status codes and print response as-is")
+        parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
+                          help="do not handle HTTP 3xx status codes and print response as-is")
 
     def _print_headers(self, headers, prefix):
         for key, values in headers.items():
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 2e837abed..abf3b7a5c 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -36,15 +36,15 @@ class Command(ScrapyCommand):
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("-l", "--list", dest="list", action="store_true",
-            help="List available templates")
+                          help="List available templates")
         parser.add_option("-e", "--edit", dest="edit", action="store_true",
-            help="Edit spider after creating it")
+                          help="Edit spider after creating it")
         parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
-            help="Dump template to standard output")
+                          help="Dump template to standard output")
         parser.add_option("-t", "--template", dest="template", default="basic",
-            help="Uses a custom template.")
+                          help="Uses a custom template.")
         parser.add_option("--force", dest="force", action="store_true",
-            help="If the spider already exists, overwrite it with the template")
+                          help="If the spider already exists, overwrite it with the template")
 
     def run(self, args, opts):
         if opts.list:
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 1cefed106..d9ab2126a 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -33,29 +33,29 @@ class Command(ScrapyCommand):
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("--spider", dest="spider", default=None,
-            help="use this spider without looking for one")
+                          help="use this spider without looking for one")
         parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
-            help="set spider argument (may be repeated)")
+                          help="set spider argument (may be repeated)")
         parser.add_option("--pipelines", action="store_true",
-            help="process items through pipelines")
+                          help="process items through pipelines")
         parser.add_option("--nolinks", dest="nolinks", action="store_true",
-            help="don't show links to follow (extracted requests)")
+                          help="don't show links to follow (extracted requests)")
         parser.add_option("--noitems", dest="noitems", action="store_true",
-            help="don't show scraped items")
+                          help="don't show scraped items")
         parser.add_option("--nocolour", dest="nocolour", action="store_true",
-            help="avoid using pygments to colorize the output")
+                          help="avoid using pygments to colorize the output")
         parser.add_option("-r", "--rules", dest="rules", action="store_true",
-            help="use CrawlSpider rules to discover the callback")
+                          help="use CrawlSpider rules to discover the callback")
         parser.add_option("-c", "--callback", dest="callback",
-            help="use this callback for parsing, instead looking for a callback")
+                          help="use this callback for parsing, instead looking for a callback")
         parser.add_option("-m", "--meta", dest="meta",
-            help="inject extra meta into the Request, it must be a valid raw json string")
+                          help="inject extra meta into the Request, it must be a valid raw json string")
         parser.add_option("--cbkwargs", dest="cbkwargs",
-            help="inject extra callback kwargs into the Request, it must be a valid raw json string")
+                          help="inject extra callback kwargs into the Request, it must be a valid raw json string")
         parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
-            help="maximum depth for parsing requests [default: %default]")
+                          help="maximum depth for parsing requests [default: %default]")
         parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
-            help="print each depth level one by one")
+                          help="print each depth level one by one")
 
     @property
     def max_level(self):
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index 603bafb9f..8d49e440f 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -19,15 +19,15 @@ class Command(ScrapyCommand):
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("--get", dest="get", metavar="SETTING",
-            help="print raw setting value")
+                          help="print raw setting value")
         parser.add_option("--getbool", dest="getbool", metavar="SETTING",
-            help="print setting value, interpreted as a boolean")
+                          help="print setting value, interpreted as a boolean")
         parser.add_option("--getint", dest="getint", metavar="SETTING",
-            help="print setting value, interpreted as an integer")
+                          help="print setting value, interpreted as an integer")
         parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
-            help="print setting value, interpreted as a float")
+                          help="print setting value, interpreted as a float")
         parser.add_option("--getlist", dest="getlist", metavar="SETTING",
-            help="print setting value, interpreted as a list")
+                          help="print setting value, interpreted as a list")
 
     def run(self, args, opts):
         settings = self.crawler_process.settings
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 5946f21e8..d1944df3d 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -34,11 +34,11 @@ class Command(ScrapyCommand):
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("-c", dest="code",
-            help="evaluate the code in the shell, print the result and exit")
+                          help="evaluate the code in the shell, print the result and exit")
         parser.add_option("--spider", dest="spider",
-            help="use this spider")
-        parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
-            default=False, help="do not handle HTTP 3xx status codes and print response as-is")
+                          help="use this spider")
+        parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
+                          help="do not handle HTTP 3xx status codes and print response as-is")
 
     def update_vars(self, vars):
         """You can use this function to update the Scrapy objects that will be
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index b123e5c84..ebe3a9c2c 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -102,10 +102,8 @@ class Command(ScrapyCommand):
         move(join(project_dir, 'module'), join(project_dir, project_name))
         for paths in TEMPLATES_TO_RENDER:
             path = join(*paths)
-            tplfile = join(project_dir,
-                string.Template(path).substitute(project_name=project_name))
-            render_templatefile(tplfile, project_name=project_name,
-                ProjectName=string_camelcase(project_name))
+            tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
+            render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
         print("New Scrapy project '%s', using template directory '%s', "
               "created in:" % (project_name, self.templates_dir))
         print("    %s\n" % abspath(project_dir))
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 1516c5997..d0ea72a67 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -17,7 +17,7 @@ class Command(ScrapyCommand):
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
-            help="also display twisted/python/platform info (useful for bug reports)")
+                          help="also display twisted/python/platform info (useful for bug reports)")
 
     def run(self, args, opts):
         if opts.verbose:

From 7383b2b428b08e84d6e3e40e6229c892e5802f7a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 17:27:44 -0300
Subject: [PATCH 2803/2974] Flake8: Remove E128 (core module)

---
 pytest.ini                               | 12 ++++++------
 scrapy/core/downloader/contextfactory.py | 11 ++++++-----
 scrapy/core/downloader/handlers/ftp.py   |  7 +++----
 scrapy/core/engine.py                    | 13 +++++--------
 4 files changed, 20 insertions(+), 23 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index d96ac9684..7c6ee07d6 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -47,20 +47,20 @@ flake8-ignore =
     scrapy/contracts/__init__.py E501
     scrapy/contracts/default.py E128
     # scrapy/core
-    scrapy/core/engine.py E501 E128
+    scrapy/core/engine.py E501
     scrapy/core/scheduler.py E501
-    scrapy/core/scraper.py E501 E128
+    scrapy/core/scraper.py E501
     scrapy/core/spidermw.py E501
     scrapy/core/downloader/__init__.py E501
-    scrapy/core/downloader/contextfactory.py E501 E128
+    scrapy/core/downloader/contextfactory.py E501
     scrapy/core/downloader/middleware.py E501
     scrapy/core/downloader/tls.py E501
-    scrapy/core/downloader/webclient.py E501 E128
+    scrapy/core/downloader/webclient.py E501
     scrapy/core/downloader/handlers/__init__.py E501
-    scrapy/core/downloader/handlers/ftp.py E501 E128
+    scrapy/core/downloader/handlers/ftp.py E501
     scrapy/core/downloader/handlers/http10.py E501
     scrapy/core/downloader/handlers/http11.py E501
-    scrapy/core/downloader/handlers/s3.py E501 E128
+    scrapy/core/downloader/handlers/s3.py E501
     # scrapy/downloadermiddlewares
     scrapy/downloadermiddlewares/ajaxcrawl.py E501
     scrapy/downloadermiddlewares/decompression.py E501
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index ab73e12c8..452242d47 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -46,11 +46,12 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         #
         # * getattr() for `_ssl_method` attribute for context factories
         #   not calling super(..., self).__init__
-        return CertificateOptions(verify=False,
-                    method=getattr(self, 'method',
-                                   getattr(self, '_ssl_method', None)),
-                    fixBrokenPeers=True,
-                    acceptableCiphers=self.tls_ciphers)
+        return CertificateOptions(
+            verify=False,
+            method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
+            fixBrokenPeers=True,
+            acceptableCiphers=self.tls_ciphers,
+        )
 
     # kept for old-style HTTP/1.0 downloader context twisted calls,
     # e.g. connectSSL()
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 94b55c347..3ef129587 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -86,10 +86,9 @@ class FTPDownloadHandler:
         password = request.meta.get("ftp_password", self.default_password)
         passive_mode = 1 if bool(request.meta.get("ftp_passive",
                                                   self.passive_mode)) else 0
-        creator = ClientCreator(reactor, FTPClient, user, password,
-            passive=passive_mode)
-        return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient,
-                                request, unquote(parsed_url.path))
+        creator = ClientCreator(reactor, FTPClient, user, password, passive=passive_mode)
+        dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21)
+        return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path))
 
     def gotClient(self, client, request, filepath):
         self.client = client
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 324d21716..de0da4b70 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -217,11 +217,9 @@ class ExecutionEngine:
         self.slot.nextcall.schedule()
 
     def schedule(self, request, spider):
-        self.signals.send_catch_log(signal=signals.request_scheduled,
-                request=request, spider=spider)
+        self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider)
         if not self.slot.scheduler.enqueue_request(request):
-            self.signals.send_catch_log(signal=signals.request_dropped,
-                                        request=request, spider=spider)
+            self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider)
 
     def download(self, request, spider):
         d = self._download(request, spider)
@@ -247,8 +245,8 @@ class ExecutionEngine:
                 logkws = self.logformatter.crawled(request, response, spider)
                 if logkws is not None:
                     logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
-                self.signals.send_catch_log(signal=signals.response_received,
-                    response=response, request=request, spider=spider)
+                self.signals.send_catch_log(signals.response_received,
+                                            response=response, request=request, spider=spider)
             return response
 
         def _on_complete(_):
@@ -286,8 +284,7 @@ class ExecutionEngine:
         next loop and this function is guaranteed to be called (at least) once
         again for this spider.
         """
-        res = self.signals.send_catch_log(signal=signals.spider_idle,
-            spider=spider, dont_log=DontCloseSpider)
+        res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider)
         if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
             return
 

From a72f5aadcf0b7ee8ef77f43602aecf34ffea7fd6 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 17:38:39 -0300
Subject: [PATCH 2804/2974] Flake8: Remove E128 (utils module)

---
 pytest.ini               | 10 +++++-----
 scrapy/utils/defer.py    |  7 +++++--
 scrapy/utils/log.py      | 10 ++++++----
 scrapy/utils/response.py |  7 +++----
 scrapy/utils/signal.py   |  6 ++----
 scrapy/utils/url.py      |  3 +--
 6 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 7c6ee07d6..63a3051cf 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -123,13 +123,13 @@ flake8-ignore =
     scrapy/utils/conf.py E402 E501
     scrapy/utils/datatypes.py E501
     scrapy/utils/decorators.py E501
-    scrapy/utils/defer.py E501 E128
+    scrapy/utils/defer.py E501
     scrapy/utils/deprecate.py E501
     scrapy/utils/gz.py E501
     scrapy/utils/http.py F403
     scrapy/utils/httpobj.py E501
     scrapy/utils/iterators.py E501
-    scrapy/utils/log.py E128 E501
+    scrapy/utils/log.py E501
     scrapy/utils/markup.py F403
     scrapy/utils/misc.py E501
     scrapy/utils/multipart.py F403
@@ -138,13 +138,13 @@ flake8-ignore =
     scrapy/utils/reactor.py E501
     scrapy/utils/reqser.py E501
     scrapy/utils/request.py E501
-    scrapy/utils/response.py E501 E128
-    scrapy/utils/signal.py E501 E128
+    scrapy/utils/response.py E501
+    scrapy/utils/signal.py E501
     scrapy/utils/sitemap.py E501
     scrapy/utils/spider.py E501
     scrapy/utils/ssl.py E501
     scrapy/utils/test.py E501
-    scrapy/utils/url.py E501 F403 E128 F405
+    scrapy/utils/url.py E501 F403 F405
     # scrapy
     scrapy/__init__.py E402 E501
     scrapy/cmdline.py E501
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 34b8d9774..a3950db75 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -88,8 +88,11 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw):
     """Return a Deferred built by chaining the given callbacks and errbacks"""
     d = defer.Deferred()
     for cb, eb in zip(callbacks, errbacks):
-        d.addCallbacks(cb, eb, callbackArgs=a, callbackKeywords=kw,
-            errbackArgs=a, errbackKeywords=kw)
+        d.addCallbacks(
+            callback=cb, errback=eb,
+            callbackArgs=a, callbackKeywords=kw,
+            errbackArgs=a, errbackKeywords=kw,
+        )
     if isinstance(input, failure.Failure):
         d.errback(input)
     else:
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 5998dc33b..83c359bd4 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -144,10 +144,12 @@ def _get_handler(settings):
 def log_scrapy_info(settings):
     logger.info("Scrapy %(version)s started (bot: %(bot)s)",
                 {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
-    logger.info("Versions: %(versions)s",
-                {'versions': ", ".join("%s %s" % (name, version)
-                    for name, version in scrapy_components_versions()
-                    if name != "Scrapy")})
+    versions = [
+        "%s %s" % (name, version)
+        for name, version in scrapy_components_versions()
+        if name != "Scrapy"
+    ]
+    logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
     from twisted.internet import reactor
     logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
 
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 29fdaaf2c..edbc0db25 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -19,8 +19,7 @@ def get_base_url(response):
     """Return the base url of the given response, joined with the response url"""
     if response not in _baseurl_cache:
         text = response.text[0:4096]
-        _baseurl_cache[response] = html.get_base_url(text, response.url,
-            response.encoding)
+        _baseurl_cache[response] = html.get_base_url(text, response.url, response.encoding)
     return _baseurl_cache[response]
 
 
@@ -31,8 +30,8 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
     """Parse the http-equiv refrsh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.text[0:4096]
-        _metaref_cache[response] = html.get_meta_refresh(text, response.url,
-            response.encoding, ignore_tags=ignore_tags)
+        _metaref_cache[response] = html.get_meta_refresh(
+            text, response.url, response.encoding, ignore_tags=ignore_tags)
     return _metaref_cache[response]
 
 
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index 60c561da6..a311e9257 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -28,8 +28,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
     responses = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
         try:
-            response = robustApply(receiver, signal=signal, sender=sender,
-                *arguments, **named)
+            response = robustApply(receiver, signal=signal, sender=sender, *arguments, **named)
             if isinstance(response, Deferred):
                 logger.error("Cannot return deferreds from signal handler: %(receiver)s",
                              {'receiver': receiver}, extra={'spider': spider})
@@ -63,8 +62,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
     spider = named.get('spider', None)
     dfds = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
-        d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender,
-                *arguments, **named)
+        d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, *arguments, **named)
         d.addErrback(logerror, receiver)
         d.addBoth(lambda result: (receiver, result))
         dfds.append(d)
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index c9abb12d5..955b63d4b 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -27,8 +27,7 @@ def url_is_from_any_domain(url, domains):
 
 def url_is_from_spider(url, spider):
     """Return True if the url belongs to the given spider"""
-    return url_is_from_any_domain(url,
-        [spider.name] + list(getattr(spider, 'allowed_domains', [])))
+    return url_is_from_any_domain(url, [spider.name] + list(getattr(spider, 'allowed_domains', [])))
 
 
 def url_has_any_extension(url, extensions):

From 4171b3f663aea5706ba6acf15f64e92be0e34f2a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 18:28:12 -0300
Subject: [PATCH 2805/2974] Flake8: Remove E128 (extensions module)

---
 pytest.ini                     | 6 +++---
 scrapy/extensions/httpcache.py | 5 +++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 63a3051cf..1af3ea0d1 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -73,10 +73,10 @@ flake8-ignore =
     scrapy/downloadermiddlewares/robotstxt.py E501
     scrapy/downloadermiddlewares/stats.py E501
     # scrapy/extensions
-    scrapy/extensions/closespider.py E501 E128
+    scrapy/extensions/closespider.py E501
     scrapy/extensions/corestats.py E501
-    scrapy/extensions/feedexport.py E128 E501
-    scrapy/extensions/httpcache.py E128 E501
+    scrapy/extensions/feedexport.py E501
+    scrapy/extensions/httpcache.py E501
     scrapy/extensions/memdebug.py E501
     scrapy/extensions/spiderstate.py E501
     scrapy/extensions/telnet.py E501
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 8546628a8..6289efec0 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -46,9 +46,10 @@ class RFC2616Policy:
     def __init__(self, settings):
         self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE')
         self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
-        self.ignore_response_cache_controls = [to_bytes(cc) for cc in
-            settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')]
         self._cc_parsed = WeakKeyDictionary()
+        self.ignore_response_cache_controls = [
+            to_bytes(cc) for cc in settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')
+        ]
 
     def _parse_cachecontrol(self, r):
         if r not in self._cc_parsed:

From 2fbbca56fb1e3bba98bf6c4c84591121c4c1da57 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 18:34:40 -0300
Subject: [PATCH 2806/2974] pytest.ini: remove unnecessary E128 lines

---
 pytest.ini | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 1af3ea0d1..c0d74f7d9 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -45,7 +45,6 @@ flake8-ignore =
     scrapy/commands/version.py E501
     # scrapy/contracts
     scrapy/contracts/__init__.py E501
-    scrapy/contracts/default.py E128
     # scrapy/core
     scrapy/core/engine.py E501
     scrapy/core/scheduler.py E501
@@ -66,7 +65,7 @@ flake8-ignore =
     scrapy/downloadermiddlewares/decompression.py E501
     scrapy/downloadermiddlewares/defaultheaders.py E501
     scrapy/downloadermiddlewares/httpcache.py E501
-    scrapy/downloadermiddlewares/httpcompression.py E501 E128
+    scrapy/downloadermiddlewares/httpcompression.py E501
     scrapy/downloadermiddlewares/httpproxy.py E501
     scrapy/downloadermiddlewares/redirect.py E501
     scrapy/downloadermiddlewares/retry.py E501
@@ -87,13 +86,13 @@ flake8-ignore =
     scrapy/http/request/__init__.py E501
     scrapy/http/request/form.py E501
     scrapy/http/request/json_request.py E501
-    scrapy/http/response/__init__.py E501 E128
-    scrapy/http/response/text.py E501 E128
+    scrapy/http/response/__init__.py E501
+    scrapy/http/response/text.py E501
     # scrapy/linkextractors
     scrapy/linkextractors/__init__.py E501 E402
     scrapy/linkextractors/lxmlhtml.py E501
     # scrapy/loader
-    scrapy/loader/__init__.py E501 E128
+    scrapy/loader/__init__.py E501
     scrapy/loader/processors.py E501
     # scrapy/pipelines
     scrapy/pipelines/__init__.py E501
@@ -157,7 +156,7 @@ flake8-ignore =
     scrapy/link.py E501
     scrapy/logformatter.py E501
     scrapy/mail.py E402 E128 E501
-    scrapy/middleware.py E128 E501
+    scrapy/middleware.py E501
     scrapy/pqueues.py E501
     scrapy/resolver.py E501
     scrapy/responsetypes.py E128 E501
@@ -165,7 +164,6 @@ flake8-ignore =
     scrapy/shell.py E501
     scrapy/signalmanager.py E501
     scrapy/spiderloader.py F841 E501
-    scrapy/squeues.py E128
     scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501

From 2851f641f2c105350948a34be55aa00dfed9933a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 18:35:50 -0300
Subject: [PATCH 2807/2974] Flake8: Remove E128 (item, mail, responsetypes)

---
 pytest.ini              |  6 +++---
 scrapy/item.py          |  6 ++----
 scrapy/mail.py          | 32 ++++++++++++++++++++++----------
 scrapy/responsetypes.py |  6 +++---
 4 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index c0d74f7d9..0a5a172b3 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -152,14 +152,14 @@ flake8-ignore =
     scrapy/exceptions.py E501
     scrapy/exporters.py E501
     scrapy/interfaces.py E501
-    scrapy/item.py E501 E128
+    scrapy/item.py E501
     scrapy/link.py E501
     scrapy/logformatter.py E501
-    scrapy/mail.py E402 E128 E501
+    scrapy/mail.py E402 E501
     scrapy/middleware.py E501
     scrapy/pqueues.py E501
     scrapy/resolver.py E501
-    scrapy/responsetypes.py E128 E501
+    scrapy/responsetypes.py E501
     scrapy/robotstxt.py E501
     scrapy/shell.py E501
     scrapy/signalmanager.py E501
diff --git a/scrapy/item.py b/scrapy/item.py
index 748368932..b75d04404 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -86,8 +86,7 @@ class DictItem(MutableMapping, BaseItem):
         if key in self.fields:
             self._values[key] = value
         else:
-            raise KeyError("%s does not support field: %s" %
-                (self.__class__.__name__, key))
+            raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key))
 
     def __delitem__(self, key):
         del self._values[key]
@@ -99,8 +98,7 @@ class DictItem(MutableMapping, BaseItem):
 
     def __setattr__(self, name, value):
         if not name.startswith('_'):
-            raise AttributeError("Use item[%r] = %r to set field value" %
-                (name, value))
+            raise AttributeError("Use item[%r] = %r to set field value" % (name, value))
         super(DictItem, self).__setattr__(name, value)
 
     def __len__(self):
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 9d7896ef6..7d7a2c435 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -28,8 +28,10 @@ def _to_bytes_or_none(text):
 
 
 class MailSender:
-    def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost',
-            smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):
+    def __init__(
+        self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None,
+        smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False
+    ):
         self.smtphost = smtphost
         self.smtpport = smtpport
         self.smtpuser = _to_bytes_or_none(smtpuser)
@@ -41,9 +43,15 @@ class MailSender:
 
     @classmethod
     def from_settings(cls, settings):
-        return cls(settings['MAIL_HOST'], settings['MAIL_FROM'], settings['MAIL_USER'],
-            settings['MAIL_PASS'], settings.getint('MAIL_PORT'),
-            settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL'))
+        return cls(
+            smtphost=settings['MAIL_HOST'],
+            mailfrom=settings['MAIL_FROM'],
+            smtpuser=settings['MAIL_USER'],
+            smtppass=settings['MAIL_PASS'],
+            smtpport=settings.getint('MAIL_PORT'),
+            smtptls=settings.getbool('MAIL_TLS'),
+            smtpssl=settings.getbool('MAIL_SSL'),
+        )
 
     def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None):
         from twisted.internet import reactor
@@ -89,9 +97,12 @@ class MailSender:
             return
 
         dfd = self._sendmail(rcpts, msg.as_string().encode(charset or 'utf-8'))
-        dfd.addCallbacks(self._sent_ok, self._sent_failed,
+        dfd.addCallbacks(
+            callback=self._sent_ok,
+            errback=self._sent_failed,
             callbackArgs=[to, cc, subject, len(attachs)],
-            errbackArgs=[to, cc, subject, len(attachs)])
+            errbackArgs=[to, cc, subject, len(attachs)],
+        )
         reactor.addSystemEventTrigger('before', 'shutdown', lambda: dfd)
         return dfd
 
@@ -115,9 +126,10 @@ class MailSender:
         from twisted.mail.smtp import ESMTPSenderFactory
         msg = BytesIO(msg)
         d = defer.Deferred()
-        factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom,
-            to_addrs, msg, d, heloFallback=True, requireAuthentication=False,
-            requireTransportSecurity=self.smtptls)
+        factory = ESMTPSenderFactory(
+            self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d,
+            heloFallback=True, requireAuthentication=False, requireTransportSecurity=self.smtptls,
+        )
         factory.noisy = False
 
         if self.smtpssl:
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 7c5eeac21..d207088e6 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -58,9 +58,9 @@ class ResponseTypes:
 
     def from_content_disposition(self, content_disposition):
         try:
-            filename = to_unicode(content_disposition,
-                encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
-            filename = filename.strip('"\'')
+            filename = to_unicode(
+                content_disposition, encoding='latin-1', errors='replace'
+            ).split(';')[1].split('=')[1].strip('"\'')
             return self.from_filename(filename)
         except IndexError:
             return Response

From 9c6d13075a2ab06c0d670e19457f9d9f219ceeee Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 18:56:14 -0300
Subject: [PATCH 2808/2974] Flake8: Remove E128 (tests, part 1)

---
 pytest.ini                                   | 28 +++++++++---------
 tests/test_command_parse.py                  |  4 +--
 tests/test_command_shell.py                  |  6 ++--
 tests/test_contracts.py                      |  3 +-
 tests/test_downloader_handlers.py            |  8 ++---
 tests/test_downloadermiddleware_httpproxy.py |  7 +++--
 tests/test_downloadermiddleware_redirect.py  |  2 +-
 tests/test_downloadermiddleware_retry.py     | 14 +++++++--
 tests/test_dupefilters.py                    | 31 +++++++++++++-------
 tests/test_http_request.py                   | 18 ++++--------
 tests/test_http_response.py                  | 20 ++++++++-----
 11 files changed, 80 insertions(+), 61 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 0a5a172b3..d8eefea7e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -172,37 +172,37 @@ flake8-ignore =
     tests/spiders.py E501
     tests/test_closespider.py E501
     tests/test_command_fetch.py E501
-    tests/test_command_parse.py E501 E128
-    tests/test_command_shell.py E501 E128
-    tests/test_commands.py E128 E501
-    tests/test_contracts.py E501 E128
+    tests/test_command_parse.py E501
+    tests/test_command_shell.py E501
+    tests/test_commands.py E501
+    tests/test_contracts.py E501
     tests/test_crawl.py E501 E741
     tests/test_crawler.py F841 E501
     tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E128 E501
+    tests/test_downloader_handlers.py E501
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
-    tests/test_downloadermiddleware_cookies.py E741 E501 E128
+    tests/test_downloadermiddleware_cookies.py E741 E501
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
     tests/test_downloadermiddleware_httpcache.py E501
     tests/test_downloadermiddleware_httpcompression.py E501
     tests/test_downloadermiddleware_decompression.py E501
-    tests/test_downloadermiddleware_httpproxy.py E501 E128
-    tests/test_downloadermiddleware_redirect.py E501 E128
-    tests/test_downloadermiddleware_retry.py E501 E128
+    tests/test_downloadermiddleware_httpproxy.py E501
+    tests/test_downloadermiddleware_redirect.py E501
+    tests/test_downloadermiddleware_retry.py E501
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
-    tests/test_dupefilters.py E501 E741 E128
-    tests/test_engine.py E501 E128
-    tests/test_exporters.py E501 E128
+    tests/test_dupefilters.py E501 E741
+    tests/test_engine.py E501
+    tests/test_exporters.py E501
     tests/test_extension_telnet.py F841
     tests/test_feedexport.py E501 F841
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
-    tests/test_http_request.py E402 E501 E128 E128
-    tests/test_http_response.py E501 E128
     tests/test_item.py E128 F841
+    tests/test_http_request.py E402 E501
+    tests/test_http_response.py E501
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128
     tests/test_loader.py E501 E741 E128 E117
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 85a24d0bc..a09dcf072 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -142,8 +142,8 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
     @defer.inlineCallbacks
     def test_request_without_meta(self):
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
-                                          '-c', 'parse_request_without_meta',
-                                          '--nolinks',
+                                           '-c', 'parse_request_without_meta',
+                                           '--nolinks',
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index d664b6ade..93fda2648 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -101,15 +101,13 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
     @defer.inlineCallbacks
     def test_local_nofile(self):
         filepath = 'file:///tests/sample_data/test_site/nothinghere.html'
-        errcode, out, err = yield self.execute([filepath, '-c', 'item'],
-                                       check_code=False)
+        errcode, out, err = yield self.execute([filepath, '-c', 'item'], check_code=False)
         self.assertEqual(errcode, 1, out or err)
         self.assertIn(b'No such file or directory', err)
 
     @defer.inlineCallbacks
     def test_dns_failures(self):
         url = 'www.somedomainthatdoesntexi.st'
-        errcode, out, err = yield self.execute([url, '-c', 'item'],
-                                       check_code=False)
+        errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False)
         self.assertEqual(errcode, 1, out or err)
         self.assertIn(b'DNS lookup failed', err)
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index d1ce80f9d..99120b128 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -232,7 +232,8 @@ class ContractsManagerTest(unittest.TestCase):
         # extract contracts correctly
         contracts = self.conman.extract_contracts(spider.returns_request)
         self.assertEqual(len(contracts), 2)
-        self.assertEqual(frozenset(type(x) for x in contracts),
+        self.assertEqual(
+            frozenset(type(x) for x in contracts),
             frozenset([UrlContract, ReturnsContract]))
 
         # returns request for valid method
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index f93bce8ef..1a05b679a 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -853,8 +853,7 @@ class S3TestCase(unittest.TestCase):
     def test_request_signing4(self):
         # fetches the access control policy sub-resource for the 'johnsmith' bucket.
         date = 'Tue, 27 Mar 2007 19:44:46 +0000'
-        req = Request('s3://johnsmith/?acl',
-            method='GET', headers={'Date': date})
+        req = Request('s3://johnsmith/?acl', method='GET', headers={'Date': date})
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'],
@@ -879,8 +878,9 @@ class S3TestCase(unittest.TestCase):
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
         # botocore does not override Date with x-amz-date
-        self.assertEqual(httpreq.headers['Authorization'],
-                b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
+        self.assertEqual(
+            httpreq.headers['Authorization'],
+            b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
 
     def test_request_signing6(self):
         # uploads an object to a CNAME style virtual hosted bucket with metadata.
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 36743b1de..9841d7a76 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -43,8 +43,11 @@ class TestHttpProxyMiddleware(TestCase):
         os.environ.pop('file_proxy', None)
         mw = HttpProxyMiddleware()
 
-        for url, proxy in [('http://e.com', http_proxy),
-                ('https://e.com', https_proxy), ('file://tmp/a', None)]:
+        for url, proxy in [
+            ('http://e.com', http_proxy),
+            ('https://e.com', https_proxy),
+            ('file://tmp/a', None),
+        ]:
             req = Request(url)
             assert mw.process_request(req, spider) is None
             self.assertEqual(req.url, url)
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 551e124ab..52c4d71a6 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -72,7 +72,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
         url = 'http://www.example.com/302'
         url2 = 'http://www.example.com/redirected2'
         req = Request(url, method='POST', body='test',
-            headers={'Content-Type': 'text/plain', 'Content-length': '4'})
+                      headers={'Content-Type': 'text/plain', 'Content-length': '4'})
         rsp = Response(url, headers={'Location': url2}, status=302)
 
         req2 = self.mw.process_response(req, rsp, self.spider)
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index e118750e3..29357ba94 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -81,9 +81,17 @@ class RetryTest(unittest.TestCase):
         assert self.crawler.stats.get_value('retry/count') == 2
 
     def test_twistederrors(self):
-        exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError,
-                DNSLookupError, ConnectionRefusedError, ConnectionDone,
-                ConnectError, ConnectionLost, ResponseFailed]
+        exceptions = [
+            ConnectError,
+            ConnectionDone,
+            ConnectionLost,
+            ConnectionRefusedError,
+            defer.TimeoutError,
+            DNSLookupError,
+            ResponseFailed,
+            TCPTimedOutError,
+            TimeoutError,
+        ]
 
         for exc in exceptions:
             req = Request('http://www.scrapytest.org/%s' % exc.__name__)
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 7426107c1..38b95b710 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -177,10 +177,13 @@ class RFPDupeFilterTest(unittest.TestCase):
             dupefilter.log(r2, spider)
 
             assert crawler.stats.get_value('dupefilter/filtered') == 2
-            l.check_present(('scrapy.dupefilters', 'DEBUG',
-                ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
-                ' - no more duplicates will be shown'
-                ' (see DUPEFILTER_DEBUG to show all duplicates)')))
+            l.check_present(
+                (
+                    'scrapy.dupefilters',
+                    'DEBUG',
+                    'Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)'
+                )
+            )
 
             dupefilter.close('finished')
 
@@ -203,11 +206,19 @@ class RFPDupeFilterTest(unittest.TestCase):
             dupefilter.log(r2, spider)
 
             assert crawler.stats.get_value('dupefilter/filtered') == 2
-            l.check_present(('scrapy.dupefilters', 'DEBUG',
-                ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
-                ' (referer: None)')))
-            l.check_present(('scrapy.dupefilters', 'DEBUG',
-                ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
-                ' (referer: http://scrapytest.org/INDEX.html)')))
+            l.check_present(
+                (
+                    'scrapy.dupefilters',
+                    'DEBUG',
+                    'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)'
+                )
+            )
+            l.check_present(
+                (
+                    'scrapy.dupefilters',
+                    'DEBUG',
+                    'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: http://scrapytest.org/INDEX.html)'
+                )
+            )
 
             dupefilter.close('finished')
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 77da15ce6..a672963f3 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -415,8 +415,7 @@ class FormRequestTest(RequestTest):
         # using multiples values for a single key
         data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']}
         r3 = self.request_class("http://www.example.com", formdata=data)
-        self.assertQueryEqual(r3.body,
-            b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
+        self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
 
     def test_from_response_post(self):
         response = _buildresponse(
@@ -426,8 +425,7 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test2" value="xxx">
             </form>""",
             url="http://www.example.com/this/list.html")
-        req = self.request_class.from_response(response,
-                formdata={'one': ['two', 'three'], 'six': 'seven'})
+        req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
 
         self.assertEqual(req.method, 'POST')
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@@ -446,8 +444,7 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test2" value="xxx \xc2\xb5">
             </form>""",
             url="http://www.example.com/this/list.html")
-        req = self.request_class.from_response(response,
-                formdata={'one': ['two', 'three'], 'six': 'seven'})
+        req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
 
         self.assertEqual(req.method, 'POST')
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@@ -468,8 +465,7 @@ class FormRequestTest(RequestTest):
             url="http://www.example.com/this/list.html",
             encoding='latin1',
         )
-        req = self.request_class.from_response(response,
-                formdata={'one': ['two', 'three'], 'six': 'seven'})
+        req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
 
         self.assertEqual(req.method, 'POST')
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@@ -488,8 +484,7 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test2" value="xxx µ">
             </form>""",
             url="http://www.example.com/this/list.html")
-        req = self.request_class.from_response(response,
-                formdata={'one': ['two', 'three'], 'six': 'seven'})
+        req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
 
         self.assertEqual(req.method, 'POST')
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@@ -549,8 +544,7 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test2" value="xxx">
             </form>""",
             url="http://www.example.com/this/list.html")
-        r1 = self.request_class.from_response(response,
-                formdata={'one': ['two', 'three'], 'six': 'seven'})
+        r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
         self.assertEqual(r1.method, 'GET')
         self.assertEqual(urlparse(r1.url).hostname, "www.example.com")
         self.assertEqual(urlparse(r1.url).path, "/this/get.php")
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 522ec4875..04f58d305 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -485,8 +485,10 @@ class TextResponseTest(BaseResponseTest):
             response.xpath("normalize-space(//p[@class=\"content\"])").getall(),
         )
         self.assertEqual(
-            response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()",
-                pclass="content", pcount=1).getall(),
+            response.xpath(
+                "//title[count(following::p[@class=$pclass])=$pcount]/text()",
+                pclass="content", pcount=1,
+            ).getall(),
             response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(),
         )
 
@@ -566,12 +568,14 @@ class TextResponseTest(BaseResponseTest):
             'http://example.com',
             body=b'''<html><body><a href=" foo\n">click me</a></body></html>'''
         )
-        self._assert_followed_url(resp.css('a')[0],
-                                 'http://example.com/foo',
-                                  response=resp)
-        self._assert_followed_url(resp.css('a::attr(href)')[0],
-                                 'http://example.com/foo',
-                                  response=resp)
+        self._assert_followed_url(
+            resp.css('a')[0],
+            'http://example.com/foo',
+            response=resp)
+        self._assert_followed_url(
+            resp.css('a::attr(href)')[0],
+            'http://example.com/foo',
+            response=resp)
 
     def test_follow_encoding(self):
         resp1 = self.response_class(

From bf56517abfc3d2287c08e1c88f07de7d3b1c499c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 19:18:29 -0300
Subject: [PATCH 2809/2974] Flake8: Remove E128 (tests, part 2)

---
 pytest.ini                               | 20 ++++-----
 tests/test_item.py                       | 22 +++++-----
 tests/test_linkextractors.py             |  6 +--
 tests/test_logformatter.py               | 21 +++++----
 tests/test_mail.py                       |  2 +-
 tests/test_middleware.py                 |  9 ++--
 tests/test_pipeline_crawl.py             | 12 ++++--
 tests/test_pipeline_media.py             | 12 +++---
 tests/test_spidermiddleware_httperror.py | 55 ++++++++----------------
 tests/test_spidermiddleware_offsite.py   | 32 ++++++++------
 10 files changed, 94 insertions(+), 97 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index d8eefea7e..0b2766031 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -200,19 +200,19 @@ flake8-ignore =
     tests/test_feedexport.py E501 F841
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
-    tests/test_item.py E128 F841
     tests/test_http_request.py E402 E501
     tests/test_http_response.py E501
+    tests/test_item.py E501 F841
     tests/test_link.py E501
-    tests/test_linkextractors.py E501 E128
-    tests/test_loader.py E501 E741 E128 E117
-    tests/test_logformatter.py E128 E501
-    tests/test_mail.py E128 E501
-    tests/test_middleware.py E501 E128
-    tests/test_pipeline_crawl.py E501 E128
+    tests/test_linkextractors.py E501
+    tests/test_loader.py E501 E741 E117
+    tests/test_logformatter.py E501
+    tests/test_mail.py E501
+    tests/test_middleware.py E501
+    tests/test_pipeline_crawl.py E501
     tests/test_pipeline_files.py E501
     tests/test_pipeline_images.py F841 E501
-    tests/test_pipeline_media.py E501 E741 E128
+    tests/test_pipeline_media.py E501 E741
     tests/test_proxy_connect.py E501 E741
     tests/test_request_cb_kwargs.py E501
     tests/test_responsetypes.py E501
@@ -221,8 +221,8 @@ flake8-ignore =
     tests/test_selector.py E501
     tests/test_spider.py E501
     tests/test_spidermiddleware.py E501
-    tests/test_spidermiddleware_httperror.py E128 E501 E121
-    tests/test_spidermiddleware_offsite.py E501 E128 E111
+    tests/test_spidermiddleware_httperror.py E501 E121
+    tests/test_spidermiddleware_offsite.py E501 E111
     tests/test_spidermiddleware_output_chain.py E501
     tests/test_spidermiddleware_referer.py E501 F841 E501 E121
     tests/test_squeues.py E501 E741
diff --git a/tests/test_item.py b/tests/test_item.py
index 4017f6e84..4fe247cda 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -162,8 +162,7 @@ class ItemTest(unittest.TestCase):
         item = D(save='X', load='Y')
         self.assertEqual(item['save'], 'X')
         self.assertEqual(item['load'], 'Y')
-        self.assertEqual(D.fields, {'load': {'default': 'A'},
-            'save': {'default': 'A'}})
+        self.assertEqual(D.fields, {'load': {'default': 'A'}, 'save': {'default': 'A'}})
 
         # D class inverted
         class E(C, B):
@@ -171,8 +170,7 @@ class ItemTest(unittest.TestCase):
 
         self.assertEqual(E(save='X')['save'], 'X')
         self.assertEqual(E(load='X')['load'], 'X')
-        self.assertEqual(E.fields, {'load': {'default': 'C'},
-            'save': {'default': 'C'}})
+        self.assertEqual(E.fields, {'load': {'default': 'C'}, 'save': {'default': 'C'}})
 
     def test_metaclass_multiple_inheritance_diamond(self):
         class A(Item):
@@ -193,8 +191,9 @@ class ItemTest(unittest.TestCase):
 
         self.assertEqual(D(save='X')['save'], 'X')
         self.assertEqual(D(load='X')['load'], 'X')
-        self.assertEqual(D.fields, {'save': {'default': 'C'},
-            'load': {'default': 'D'}, 'update': {'default': 'D'}})
+        self.assertEqual(
+            D.fields,
+            {'save': {'default': 'C'}, 'load': {'default': 'D'}, 'update': {'default': 'D'}})
 
         # D class inverted
         class E(C, B):
@@ -202,8 +201,9 @@ class ItemTest(unittest.TestCase):
 
         self.assertEqual(E(save='X')['save'], 'X')
         self.assertEqual(E(load='X')['load'], 'X')
-        self.assertEqual(E.fields, {'save': {'default': 'C'},
-            'load': {'default': 'E'}, 'update': {'default': 'C'}})
+        self.assertEqual(
+            E.fields,
+            {'save': {'default': 'C'}, 'load': {'default': 'E'}, 'update': {'default': 'C'}})
 
     def test_metaclass_multiple_inheritance_without_metaclass(self):
         class A(Item):
@@ -223,8 +223,7 @@ class ItemTest(unittest.TestCase):
 
         self.assertRaises(KeyError, D, not_allowed='value')
         self.assertEqual(D(save='X')['save'], 'X')
-        self.assertEqual(D.fields, {'save': {'default': 'A'},
-            'load': {'default': 'A'}})
+        self.assertEqual(D.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}})
 
         # D class inverted
         class E(C, B):
@@ -232,8 +231,7 @@ class ItemTest(unittest.TestCase):
 
         self.assertRaises(KeyError, E, not_allowed='value')
         self.assertEqual(E(save='X')['save'], 'X')
-        self.assertEqual(E.fields, {'save': {'default': 'A'},
-            'load': {'default': 'A'}})
+        self.assertEqual(E.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}})
 
     def test_to_dict(self):
         class TestItem(Item):
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 68e8514ba..7e1b62b7f 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -171,9 +171,9 @@ class Base:
             self.assertEqual(lx.matches(url1), False)
             self.assertEqual(lx.matches(url2), True)
 
-            lx = self.extractor_cls(allow=('blah1',), deny=('blah2',),
-                                   allow_domains=('blah1.com',),
-                                   deny_domains=('blah2.com',))
+            lx = self.extractor_cls(allow=['blah1'], deny=['blah2'],
+                                    allow_domains=['blah1.com'],
+                                    deny_domains=['blah2.com'])
             self.assertEqual(lx.matches('http://blah1.com/blah1'), True)
             self.assertEqual(lx.matches('http://blah1.com/blah2'), False)
             self.assertEqual(lx.matches('http://blah2.com/blah1'), False)
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index cd6cb8016..7064337ad 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -34,15 +34,15 @@ class LogFormatterTestCase(unittest.TestCase):
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline,
-            "Crawled (200) <GET http://www.example.com> (referer: None)")
+        self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None)")
 
     def test_crawled_without_referer(self):
         req = Request("http://www.example.com", headers={'referer': 'http://example.com'})
         res = Response("http://www.example.com", flags=['cached'])
         logkws = self.formatter.crawled(req, res, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline,
+        self.assertEqual(
+            logline,
             "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
 
     def test_flags_in_request(self):
@@ -50,8 +50,9 @@ class LogFormatterTestCase(unittest.TestCase):
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline,
-        "Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")
+        self.assertEqual(
+            logline,
+            "Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")
 
     def test_dropped(self):
         item = {}
@@ -140,7 +141,8 @@ class LogformatterSubclassTest(LogFormatterTestCase):
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline,
+        self.assertEqual(
+            logline,
             "Crawled (200) <GET http://www.example.com> (referer: None) []")
 
     def test_crawled_without_referer(self):
@@ -148,7 +150,8 @@ class LogformatterSubclassTest(LogFormatterTestCase):
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline,
+        self.assertEqual(
+            logline,
             "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
 
     def test_flags_in_request(self):
@@ -156,7 +159,9 @@ class LogformatterSubclassTest(LogFormatterTestCase):
         res = Response("http://www.example.com")
         logkws = self.formatter.crawled(req, res, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")
+        self.assertEqual(
+            logline,
+            "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")
 
 
 class SkipMessagesLogFormatter(LogFormatter):
diff --git a/tests/test_mail.py b/tests/test_mail.py
index f5cb81a8b..53dbc0686 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -49,7 +49,7 @@ class MailSenderTest(unittest.TestCase):
 
         mailsender = MailSender(debug=True)
         mailsender.send(to=['test@scrapy.org'], subject='subject', body='body',
-                       attachs=attachs, _callback=self._catch_mail_sent)
+                        attachs=attachs, _callback=self._catch_mail_sent)
 
         assert self.catched_msg
         self.assertEqual(self.catched_msg['to'], ['test@scrapy.org'])
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index 3af514bb0..3364d2258 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -69,11 +69,14 @@ class MiddlewareManagerTest(unittest.TestCase):
 
     def test_methods(self):
         mwman = TestMiddlewareManager(M1(), M2(), M3())
-        self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
+        self.assertEqual(
+            [x.__self__.__class__ for x in mwman.methods['open_spider']],
             [M1, M2])
-        self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
+        self.assertEqual(
+            [x.__self__.__class__ for x in mwman.methods['close_spider']],
             [M2, M1])
-        self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
+        self.assertEqual(
+            [x.__self__.__class__ for x in mwman.methods['process']],
             [M1, M3])
 
     def test_enabled(self):
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 24c516473..188c5c3cf 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -132,7 +132,8 @@ class FileDownloadCrawlTestCase(TestCase):
     def test_download_media(self):
         crawler = self._create_crawler(MediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/files/images/"),
+            yield crawler.crawl(
+                self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key)
         self._assert_files_downloaded(self.items, str(log))
@@ -141,7 +142,8 @@ class FileDownloadCrawlTestCase(TestCase):
     def test_download_media_wrong_urls(self):
         crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/files/images/"),
+            yield crawler.crawl(
+                self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key)
         self._assert_files_download_failure(crawler, self.items, 404, str(log))
@@ -150,7 +152,8 @@ class FileDownloadCrawlTestCase(TestCase):
     def test_download_media_redirected_default_failure(self):
         crawler = self._create_crawler(RedirectedMediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/files/images/"),
+            yield crawler.crawl(
+                self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
                 mockserver=self.mockserver)
@@ -164,7 +167,8 @@ class FileDownloadCrawlTestCase(TestCase):
 
         crawler = self._create_crawler(RedirectedMediaDownloadSpider)
         with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/files/images/"),
+            yield crawler.crawl(
+                self.mockserver.url("/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
                 mockserver=self.mockserver)
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 949f0dea1..b51f82422 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -214,9 +214,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         item = dict(requests=req)
         new_item = yield self.pipe.process_item(item, self.spider)
         self.assertEqual(new_item['results'], [(True, rsp)])
-        self.assertEqual(self.pipe._mockcalled,
-                ['get_media_requests', 'media_to_download',
-                    'media_downloaded', 'request_callback', 'item_completed'])
+        self.assertEqual(
+            self.pipe._mockcalled,
+            ['get_media_requests', 'media_to_download', 'media_downloaded', 'request_callback', 'item_completed'])
 
     @inlineCallbacks
     def test_result_failure(self):
@@ -227,9 +227,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         item = dict(requests=req)
         new_item = yield self.pipe.process_item(item, self.spider)
         self.assertEqual(new_item['results'], [(False, fail)])
-        self.assertEqual(self.pipe._mockcalled,
-                ['get_media_requests', 'media_to_download',
-                    'media_failed', 'request_errback', 'item_completed'])
+        self.assertEqual(
+            self.pipe._mockcalled,
+            ['get_media_requests', 'media_to_download', 'media_failed', 'request_errback', 'item_completed'])
 
     @inlineCallbacks
     def test_mix_of_success_and_failure(self):
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 6b61df56f..714279ae0 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -68,29 +68,23 @@ class TestHttpErrorMiddleware(TestCase):
         self.res200, self.res404 = _responses(self.req, [200, 404])
 
     def test_process_spider_input(self):
-        self.assertEqual(None,
-                self.mw.process_spider_input(self.res200, self.spider))
-        self.assertRaises(HttpError,
-                self.mw.process_spider_input, self.res404, self.spider)
+        self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
+        self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
 
     def test_process_spider_exception(self):
-        self.assertEqual([],
-                self.mw.process_spider_exception(self.res404,
-                        HttpError(self.res404), self.spider))
-        self.assertEqual(None,
-                self.mw.process_spider_exception(self.res404,
-                        Exception(), self.spider))
+        self.assertEqual(
+            [],
+            self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider))
+        self.assertIsNone(self.mw.process_spider_exception(self.res404, Exception(), self.spider))
 
     def test_handle_httpstatus_list(self):
         res = self.res404.copy()
         res.request = Request('http://scrapytest.org',
                               meta={'handle_httpstatus_list': [404]})
-        self.assertEqual(None,
-            self.mw.process_spider_input(res, self.spider))
+        self.assertIsNone(self.mw.process_spider_input(res, self.spider))
 
         self.spider.handle_httpstatus_list = [404]
-        self.assertEqual(None,
-            self.mw.process_spider_input(self.res404, self.spider))
+        self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
 
 
 class TestHttpErrorMiddlewareSettings(TestCase):
@@ -103,12 +97,9 @@ class TestHttpErrorMiddlewareSettings(TestCase):
         self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
 
     def test_process_spider_input(self):
-        self.assertEqual(None,
-                self.mw.process_spider_input(self.res200, self.spider))
-        self.assertRaises(HttpError,
-                self.mw.process_spider_input, self.res404, self.spider)
-        self.assertEqual(None,
-                self.mw.process_spider_input(self.res402, self.spider))
+        self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
+        self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
+        self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider))
 
     def test_meta_overrides_settings(self):
         request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
@@ -117,17 +108,13 @@ class TestHttpErrorMiddlewareSettings(TestCase):
         res402 = self.res402.copy()
         res402.request = request
 
-        self.assertEqual(None,
-            self.mw.process_spider_input(res404, self.spider))
-        self.assertRaises(HttpError,
-                self.mw.process_spider_input, res402, self.spider)
+        self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
+        self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
 
     def test_spider_override_settings(self):
         self.spider.handle_httpstatus_list = [404]
-        self.assertEqual(None,
-            self.mw.process_spider_input(self.res404, self.spider))
-        self.assertRaises(HttpError,
-                self.mw.process_spider_input, self.res402, self.spider)
+        self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
+        self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider)
 
 
 class TestHttpErrorMiddlewareHandleAll(TestCase):
@@ -139,10 +126,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
         self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
 
     def test_process_spider_input(self):
-        self.assertEqual(None,
-                self.mw.process_spider_input(self.res200, self.spider))
-        self.assertEqual(None,
-                self.mw.process_spider_input(self.res404, self.spider))
+        self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
+        self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
 
     def test_meta_overrides_settings(self):
         request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
@@ -151,10 +136,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
         res402 = self.res402.copy()
         res402.request = request
 
-        self.assertEqual(None,
-            self.mw.process_spider_input(res404, self.spider))
-        self.assertRaises(HttpError,
-                self.mw.process_spider_input, res402, self.spider)
+        self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
+        self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
 
 
 class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index b96807bc2..0f4b98a07 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -22,20 +22,24 @@ class TestOffsiteMiddleware(TestCase):
     def test_process_spider_output(self):
         res = Response('http://scrapytest.org')
 
-        onsite_reqs = [Request('http://scrapytest.org/1'),
-                       Request('http://scrapy.org/1'),
-                       Request('http://sub.scrapy.org/1'),
-                       Request('http://offsite.tld/letmepass', dont_filter=True),
-                       Request('http://scrapy.test.org/'),
-                       Request('http://scrapy.test.org:8000/')]
-        offsite_reqs = [Request('http://scrapy2.org'),
-                       Request('http://offsite.tld/'),
-                       Request('http://offsite.tld/scrapytest.org'),
-                       Request('http://offsite.tld/rogue.scrapytest.org'),
-                       Request('http://rogue.scrapytest.org.haha.com'),
-                       Request('http://roguescrapytest.org'),
-                       Request('http://test.org/'),
-                       Request('http://notscrapy.test.org/')]
+        onsite_reqs = [
+            Request('http://scrapytest.org/1'),
+            Request('http://scrapy.org/1'),
+            Request('http://sub.scrapy.org/1'),
+            Request('http://offsite.tld/letmepass', dont_filter=True),
+            Request('http://scrapy.test.org/'),
+            Request('http://scrapy.test.org:8000/'),
+        ]
+        offsite_reqs = [
+            Request('http://scrapy2.org'),
+            Request('http://offsite.tld/'),
+            Request('http://offsite.tld/scrapytest.org'),
+            Request('http://offsite.tld/rogue.scrapytest.org'),
+            Request('http://rogue.scrapytest.org.haha.com'),
+            Request('http://roguescrapytest.org'),
+            Request('http://test.org/'),
+            Request('http://notscrapy.test.org/'),
+        ]
         reqs = onsite_reqs + offsite_reqs
 
         out = list(self.mw.process_spider_output(res, reqs, self.spider))

From 881b4f417f645fa9719e6c54eb3788cc67d56053 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 19:33:22 -0300
Subject: [PATCH 2810/2974] Flake8: Remove E128 (tests, part 3)

---
 pytest.ini                          | 18 ++++++-------
 tests/test_settings/__init__.py     | 18 ++++++++-----
 tests/test_spiderloader/__init__.py | 21 ++++++++++-----
 tests/test_utils_conf.py            |  3 ++-
 tests/test_utils_iterators.py       | 23 +++++++++++------
 tests/test_utils_reqser.py          |  3 ++-
 tests/test_utils_request.py         |  5 ++--
 tests/test_utils_sitemap.py         |  9 +++++--
 tests/test_webclient.py             | 40 +++++++++++++++++------------
 9 files changed, 87 insertions(+), 53 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 0b2766031..7c3c9cc5b 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -227,23 +227,23 @@ flake8-ignore =
     tests/test_spidermiddleware_referer.py E501 F841 E501 E121
     tests/test_squeues.py E501 E741
     tests/test_utils_asyncio.py E501
-    tests/test_utils_conf.py E501 E128
+    tests/test_utils_conf.py E501
     tests/test_utils_curl.py E501
     tests/test_utils_datatypes.py E402 E501
     tests/test_utils_defer.py E501 F841
     tests/test_utils_deprecate.py F841 E501
-    tests/test_utils_http.py E501 E128
-    tests/test_utils_iterators.py E501 E128 E129
+    tests/test_utils_http.py E501
+    tests/test_utils_iterators.py E501 E129
     tests/test_utils_log.py E741
     tests/test_utils_python.py E501
-    tests/test_utils_reqser.py E501 E128
-    tests/test_utils_request.py E501 E128
+    tests/test_utils_reqser.py E501
+    tests/test_utils_request.py E501
     tests/test_utils_response.py E501
     tests/test_utils_signal.py E741 F841
-    tests/test_utils_sitemap.py E128 E501
+    tests/test_utils_sitemap.py E501
     tests/test_utils_url.py E501 E501
-    tests/test_webclient.py E501 E128 E402
+    tests/test_webclient.py E501 E402
     tests/test_cmdline/__init__.py E501
-    tests/test_settings/__init__.py E501 E128
-    tests/test_spiderloader/__init__.py E128 E501
+    tests/test_settings/__init__.py E501
+    tests/test_spiderloader/__init__.py E501
     tests/test_utils_misc/__init__.py E501
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index fda44653a..2da6aa4b5 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -314,13 +314,17 @@ class BaseSettingsTest(unittest.TestCase):
                           'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
                           'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
                           'HASNOBASE': BaseSettings({3: 3000}, 'default')})
-        self.assertDictEqual(s.copy_to_dict(),
-                            {'HASNOBASE': {3: 3000},
-                             'TEST': {1: 10, 3: 30},
-                             'TEST_BASE': {1: 1, 2: 2},
-                             'TEST_BOOLEAN': False,
-                             'TEST_LIST': [1, 2],
-                             'TEST_STRING': 'a string'})
+        self.assertDictEqual(
+            s.copy_to_dict(),
+            {
+                'HASNOBASE': {3: 3000},
+                'TEST': {1: 10, 3: 30},
+                'TEST_BASE': {1: 1, 2: 2},
+                'TEST_LIST': [1, 2],
+                'TEST_BOOLEAN': False,
+                'TEST_STRING': 'a string',
+            }
+        )
 
     def test_freeze(self):
         self.settings.freeze()
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index d8be6e277..b20e1b3f2 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -40,7 +40,8 @@ class SpiderLoaderTest(unittest.TestCase):
         verifyObject(ISpiderLoader, self.spider_loader)
 
     def test_list(self):
-        self.assertEqual(set(self.spider_loader.list()),
+        self.assertEqual(
+            set(self.spider_loader.list()),
             set(['spider1', 'spider2', 'spider3', 'spider4']))
 
     def test_load(self):
@@ -48,17 +49,23 @@ class SpiderLoaderTest(unittest.TestCase):
         self.assertEqual(spider1.__name__, 'Spider1')
 
     def test_find_by_request(self):
-        self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy1.org/test')),
+        self.assertEqual(
+            self.spider_loader.find_by_request(Request('http://scrapy1.org/test')),
             ['spider1'])
-        self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy2.org/test')),
+        self.assertEqual(
+            self.spider_loader.find_by_request(Request('http://scrapy2.org/test')),
             ['spider2'])
-        self.assertEqual(set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
+        self.assertEqual(
+            set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
             set(['spider1', 'spider2']))
-        self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
+        self.assertEqual(
+            self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
             [])
-        self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com')),
+        self.assertEqual(
+            self.spider_loader.find_by_request(Request('http://spider3.com')),
             [])
-        self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')),
+        self.assertEqual(
+            self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')),
             ['spider3'])
 
     def test_load_spider_module(self):
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 332120021..e5d3ef582 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -93,7 +93,8 @@ class BuildComponentListTest(unittest.TestCase):
 class UtilsConfTestCase(unittest.TestCase):
 
     def test_arglist_to_dict(self):
-        self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']),
+        self.assertEqual(
+            arglist_to_dict(['arg1=val1', 'arg2=val2']),
             {'arg1': 'val1', 'arg2': 'val2'})
 
 
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index a85087619..c677dabe5 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -47,8 +47,7 @@ class XmliterTestCase(unittest.TestCase):
             </root>
         """
         response = XmlResponse(url="http://example.com", body=body)
-        nodenames = [e.xpath('name()').getall()
-                 for e in self.xmliter(response, 'matchme...')]
+        nodenames = [e.xpath('name()').getall() for e in self.xmliter(response, 'matchme...')]
         self.assertEqual(nodenames, [['matchme...']])
 
     def test_xmliter_unicode(self):
@@ -359,15 +358,23 @@ class UtilsCsvTestCase(unittest.TestCase):
 
         response = TextResponse(url="http://example.com/", body=body1, encoding='latin1')
         csv = csviter(response)
-        self.assertEqual([row for row in csv],
-            [{u'id': u'1', u'name': u'latin1', u'value': u'test'},
-             {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}])
+        self.assertEqual(
+            list(csv),
+            [
+                {u'id': u'1', u'name': u'latin1', u'value': u'test'},
+                {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'},
+            ]
+        )
 
         response = TextResponse(url="http://example.com/", body=body2, encoding='cp852')
         csv = csviter(response)
-        self.assertEqual([row for row in csv],
-            [{u'id': u'1', u'name': u'cp852', u'value': u'test'},
-             {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}])
+        self.assertEqual(
+            list(csv),
+            [
+                {u'id': u'1', u'name': u'cp852', u'value': u'test'},
+                {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
+            ]
+        )
 
 
 class TestHelper(unittest.TestCase):
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 50b026d1c..450e4bdca 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -15,7 +15,8 @@ class RequestSerializationTest(unittest.TestCase):
         self._assert_serializes_ok(r)
 
     def test_all_attributes(self):
-        r = Request("http://www.example.com",
+        r = Request(
+            url="http://www.example.com",
             callback=self.spider.parse_item,
             errback=self.spider.handle_error,
             method="POST",
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 45f0f59e4..50efb63ca 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -36,8 +36,9 @@ class UtilsRequestTest(unittest.TestCase):
         self.assertEqual(request_fingerprint(r1),
                          request_fingerprint(r1, include_headers=['Accept-Language']))
 
-        self.assertNotEqual(request_fingerprint(r1),
-                         request_fingerprint(r2, include_headers=['Accept-Language']))
+        self.assertNotEqual(
+            request_fingerprint(r1),
+            request_fingerprint(r2, include_headers=['Accept-Language']))
 
         self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']),
                          request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language']))
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index 08b215434..d77978ff1 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -22,8 +22,13 @@ class SitemapTest(unittest.TestCase):
   </url>
 </urlset>""")
         assert s.type == 'urlset'
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'},
+            ]
+        )
 
     def test_sitemap_index(self):
         s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?>
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 307fadb5c..188e54602 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -47,8 +47,9 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
         return f
 
     from twisted.web.client import _makeGetterFactory
-    return _makeGetterFactory(to_bytes(url), _clientfactory,
-        contextFactory=contextFactory, *args, **kwargs).deferred
+    return _makeGetterFactory(
+        to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs
+    ).deferred
 
 
 class ParseUrlTestCase(unittest.TestCase):
@@ -105,7 +106,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
                 'Content-Length': '12981',
                 'Useful': 'value'}))
 
-        self._test(factory,
+        self._test(
+            factory,
             b"GET /bar HTTP/1.0\r\n"
             b"Content-Length: 9\r\n"
             b"Useful: value\r\n"
@@ -118,7 +120,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
 
         # test minimal sent headers
         factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar'))
-        self._test(factory,
+        self._test(
+            factory,
             b"GET /bar HTTP/1.0\r\n"
             b"Host: foo\r\n"
             b"\r\n")
@@ -130,7 +133,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             body='name=value',
             headers={'Content-Type': 'application/x-www-form-urlencoded'}))
 
-        self._test(factory,
+        self._test(
+            factory,
             b"POST /bar HTTP/1.0\r\n"
             b"Host: foo\r\n"
             b"Connection: close\r\n"
@@ -145,7 +149,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             url='http://foo/bar'
         ))
 
-        self._test(factory,
+        self._test(
+            factory,
             b"POST /bar HTTP/1.0\r\n"
             b"Host: foo\r\n"
             b"Content-Length: 0\r\n"
@@ -160,7 +165,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             },
         ))
 
-        self._test(factory,
+        self._test(
+            factory,
             b"GET /bar HTTP/1.0\r\n"
             b"Host: foo\r\n"
             b"X-Meta-Multivalued: value1\r\n"
@@ -177,7 +183,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             }),
         ))
 
-        self._test(factory,
+        self._test(
+            factory,
             b"GET /bar HTTP/1.0\r\n"
             b"Host: foo\r\n"
             b"X-Meta-Multivalued: value1\r\n"
@@ -206,8 +213,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
         protocol.dataReceived(b"Hello: World\n")
         protocol.dataReceived(b"Foo: Bar\n")
         protocol.dataReceived(b"\n")
-        self.assertEqual(protocol.headers,
-            Headers({'Hello': ['World'], 'Foo': ['Bar']}))
+        self.assertEqual(protocol.headers, Headers({'Hello': ['World'], 'Foo': ['Bar']}))
 
 
 class EncodingResource(resource.Resource):
@@ -340,10 +346,11 @@ class WebClientTestCase(unittest.TestCase):
         return getPage(self.getURL("redirect")).addCallback(self._cbRedirect)
 
     def _cbRedirect(self, pageData):
-        self.assertEqual(pageData,
-                b'\n<html>\n    <head>\n        <meta http-equiv="refresh" content="0;URL=/file">\n'
-                b'    </head>\n    <body bgcolor="#FFFFFF" text="#000000">\n    '
-                b'<a href="/file">click here</a>\n    </body>\n</html>\n')
+        self.assertEqual(
+            pageData,
+            b'\n<html>\n    <head>\n        <meta http-equiv="refresh" content="0;URL=/file">\n'
+            b'    </head>\n    <body bgcolor="#FFFFFF" text="#000000">\n    '
+            b'<a href="/file">click here</a>\n    </body>\n</html>\n')
 
     def test_encoding(self):
         """ Test that non-standart body encoding matches
@@ -403,8 +410,9 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
         s = "0123456789" * 10
         settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': self.custom_ciphers})
         client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)
-        return getPage(self.getURL("payload"), body=s,
-                       contextFactory=client_context_factory).addCallback(self.assertEqual, to_bytes(s))
+        return getPage(
+            self.getURL("payload"), body=s, contextFactory=client_context_factory
+        ).addCallback(self.assertEqual, to_bytes(s))
 
     def testPayloadDefaultCiphers(self):
         s = "0123456789" * 10

From d472402a0232781753515d9552b7a1997b43543a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 6 May 2020 14:39:17 -0300
Subject: [PATCH 2811/2974] Fix pickle test for pypy

---
 pytest.ini            | 1 +
 tests/test_squeues.py | 9 +++++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 4f3494e0e..d107c1fbe 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -166,6 +166,7 @@ flake8-ignore =
     scrapy/signalmanager.py E501
     scrapy/spiderloader.py F841 E501 E126
     scrapy/squeues.py E128
+    scrapy/squeues.py E501
     scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 51c0c028a..d2cf9135f 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -1,3 +1,6 @@
+import pickle
+import sys
+
 from queuelib.tests import test_queue as t
 from scrapy.squeues import (
     MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue,
@@ -108,8 +111,10 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
         try:
             q.push(lambda x: x)
         except ValueError as exc:
-            self.assertIsInstance(exc.__context__, AttributeError)
-
+            if hasattr(sys, "pypy_version_info"):
+                self.assertIsInstance(exc.__context__, pickle.PicklingError)
+            else:
+                self.assertIsInstance(exc.__context__, AttributeError)
         sel = Selector(text='<html><body><p>some text</p></body></html>')
         try:
             q.push(sel)

From bbd9d05dbff7abb6ab3a5ea575d75cfe88cb2ef3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 7 May 2020 11:44:43 +0200
Subject: [PATCH 2812/2974] request-response.rst: review type references around
 body mentions

---
 docs/topics/request-response.rst | 38 +++++++++++++++-----------------
 1 file changed, 18 insertions(+), 20 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 1638c202d..4fec70e13 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -51,12 +51,10 @@ Request objects
        given, the dict passed in this parameter will be shallow copied.
     :type meta: dict
 
-    :param body: the request body. If a ``unicode`` is passed, then it's encoded to
-      ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If
-      ``body`` is not given, an empty string is stored. Regardless of the
-      type of this argument, the final value stored will be a ``str`` (never
-      ``unicode`` or ``None``).
-    :type body: str
+    :param body: the request body. If a string is passed, it is converted to
+        bytes using *encoding*, which defaults to ``utf-8``. If not passed or
+        ``None`` is passed, an empty bytes array is stored.
+    :type body: bytes
 
     :param headers: the headers of this request. The dict values can be strings
        (for single valued headers) or lists (for multi-valued headers). If
@@ -106,7 +104,7 @@ Request objects
 
     :param encoding: the encoding of this request (defaults to ``'utf-8'``).
        This encoding will be used to percent-encode the URL and to convert the
-       body to ``str`` (if given as ``unicode``).
+       body to bytes if given as a string.
     :type encoding: str
 
     :param priority: the priority of this request (defaults to ``0``).
@@ -159,7 +157,7 @@ Request objects
 
     .. attribute:: Request.body
 
-        A str that contains the request body.
+        The request body as bytes.
 
         This attribute is read-only. To change the body of a Request use
         :meth:`replace`.
@@ -598,7 +596,7 @@ Response objects
        (for single valued headers) or lists (for multi-valued headers).
     :type headers: dict
 
-    :param body: the response body. To access the decoded text as str you can use
+    :param body: the response body. To access the decoded text as a string, use
        ``response.text`` from an encoding-aware
        :ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
        such as :class:`TextResponse`.
@@ -646,10 +644,10 @@ Response objects
 
     .. attribute:: Response.body
 
-        The body of this Response. Keep in mind that Response.body
-        is always a bytes object. If you want the unicode version use
-        :attr:`TextResponse.text` (only available in :class:`TextResponse`
-        and subclasses).
+        The response body as bytes.
+
+        If you want the body as a string, use :attr:`TextResponse.text` (only
+        available in :class:`TextResponse` and subclasses).
 
         This attribute is read-only. To change the body of a Response use
         :meth:`replace`.
@@ -768,10 +766,10 @@ TextResponse objects
     is the same as for the :class:`Response` class and is not documented here.
 
     :param encoding: is a string which contains the encoding to use for this
-       response. If you create a :class:`TextResponse` object with a unicode
-       body, it will be encoded using this encoding (remember the body attribute
-       is always a string). If ``encoding`` is ``None`` (default value), the
-       encoding will be looked up in the response headers and body instead.
+       response. If you create a :class:`TextResponse` object with a string as
+       body, it will be converted to bytes encoded using this encoding. If
+       *encoding* is ``None`` (default), the encoding will be looked up in the
+       response headers and body instead.
     :type encoding: str
 
     :class:`TextResponse` objects support the following attributes in addition
@@ -779,7 +777,7 @@ TextResponse objects
 
     .. attribute:: TextResponse.text
 
-       Response body, as unicode.
+       Response body as a string.
 
        The same as ``response.body.decode(response.encoding)``, but the
        result is cached after the first call, so you can access
@@ -787,8 +785,8 @@ TextResponse objects
 
        .. note::
 
-            ``unicode(response.body)`` is not a correct way to convert response
-            body to unicode: you would be using the system default encoding
+            ``str(response.body)`` is not a correct way to convert the response
+            body into a string: you would be using the system default encoding
             (typically ``ascii``) instead of the response encoding.
 
 

From b59dfb75fa72346f8268b83dedd2c1f9af460c71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 7 May 2020 14:14:59 +0200
Subject: [PATCH 2813/2974] Update disabled Pylint checks

---
 pylintrc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pylintrc b/pylintrc
index c52a4c2d0..129c7bf7d 100644
--- a/pylintrc
+++ b/pylintrc
@@ -13,6 +13,7 @@ disable=abstract-method,
         bad-mcs-classmethod-argument,
         bad-super-call,
         bad-whitespace,
+        bare-except,
         blacklisted-name,
         broad-except,
         c-extension-no-member,
@@ -39,6 +40,8 @@ disable=abstract-method,
         inconsistent-return-statements,
         inherit-non-class,
         invalid-name,
+        invalid-overridden-method,
+        isinstance-second-argument-not-valid-type,
         keyword-arg-before-vararg,
         line-too-long,
         logging-format-interpolation,
@@ -77,6 +80,7 @@ disable=abstract-method,
         too-many-ancestors,
         too-many-arguments,
         too-many-branches,
+        too-many-format-args,
         too-many-function-args,
         too-many-instance-attributes,
         too-many-lines,
@@ -87,6 +91,7 @@ disable=abstract-method,
         trailing-whitespace,
         unbalanced-tuple-unpacking,
         undefined-variable,
+        undefined-loop-variable,
         unexpected-special-method-signature,
         ungrouped-imports,
         unidiomatic-typecheck,

From 422e6429b56e42b8344a0e46c45f4106d374d024 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 7 May 2020 09:22:14 -0300
Subject: [PATCH 2814/2974] Add mising len check in spiderloader

---
 scrapy/spiderloader.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 8dc89c2e9..92aed9b8e 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
-from collections import defaultdict
 import traceback
 import warnings
+from collections import defaultdict
 
 from zope.interface import implementer
 
@@ -16,6 +16,7 @@ class SpiderLoader:
     SpiderLoader is a class which locates and loads spiders
     in a Scrapy project.
     """
+
     def __init__(self, settings):
         self.spider_modules = settings.getlist('SPIDER_MODULES')
         self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')
@@ -29,6 +30,7 @@ class SpiderLoader:
             dupes.extend([
                 "  {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name)
                 for mod, cls in locations
+                if len(locations) > 1
             ])
 
         if dupes:
@@ -49,10 +51,9 @@ class SpiderLoader:
                     self._load_spiders(module)
             except ImportError:
                 if self.warn_only:
-                    msg = (
-                        "\n{tb}Could not load spiders from module '{modname}'. "
-                        "See above traceback for details.".format(modname=name, tb=traceback.format_exc())
-                    )
+                    msg = ("\n{tb}Could not load spiders from module '{modname}'. "
+                           "See above traceback for details.".format(
+                                modname=name, tb=traceback.format_exc()))
                     warnings.warn(msg, RuntimeWarning)
                 else:
                     raise
@@ -76,8 +77,10 @@ class SpiderLoader:
         """
         Return the list of spider names that can handle the given request.
         """
-        return [name for name, cls in self._spiders.items()
-                if cls.handles_request(request)]
+        return [
+            name for name, cls in self._spiders.items()
+            if cls.handles_request(request)
+        ]
 
     def list(self):
         """

From e0127a31230d4be13b1bd29e62d75c2954b47d9e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 7 May 2020 12:48:43 -0300
Subject: [PATCH 2815/2974] Refactor warnings in spiderloader

---
 scrapy/spiderloader.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 92aed9b8e..63da55718 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -35,9 +35,11 @@ class SpiderLoader:
 
         if dupes:
             dupes_string = "\n\n".join(dupes)
-            msg = ("There are several spiders with the same name:\n\n"
-                   "{}\n\n  This can cause unexpected behavior.".format(dupes_string))
-            warnings.warn(msg, UserWarning)
+            warnings.warn(
+                "There are several spiders with the same name:\n\n"
+                "{}\n\n  This can cause unexpected behavior.".format(dupes_string),
+                category=UserWarning,
+            )
 
     def _load_spiders(self, module):
         for spcls in iter_spider_classes(module):
@@ -51,10 +53,13 @@ class SpiderLoader:
                     self._load_spiders(module)
             except ImportError:
                 if self.warn_only:
-                    msg = ("\n{tb}Could not load spiders from module '{modname}'. "
-                           "See above traceback for details.".format(
-                                modname=name, tb=traceback.format_exc()))
-                    warnings.warn(msg, RuntimeWarning)
+                    warnings.warn(
+                        "\n{tb}Could not load spiders from module '{modname}'. "
+                        "See above traceback for details.".format(
+                            modname=name, tb=traceback.format_exc()
+                        ),
+                        category=RuntimeWarning,
+                    )
                 else:
                     raise
         self._check_name_duplicates()

From 5256eae60d3685de51c1f3891abe157e15d14def Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 7 May 2020 14:37:41 -0300
Subject: [PATCH 2816/2974] Meta class to handle isinstance checks for BaseItem

---
 pytest.ini                  |  2 +-
 scrapy/commands/parse.py    |  4 +--
 scrapy/contracts/default.py |  8 +++---
 scrapy/core/scraper.py      |  4 +--
 scrapy/exporters.py         |  4 +--
 scrapy/item.py              | 20 +++++++++++--
 scrapy/shell.py             |  5 ++--
 scrapy/utils/misc.py        |  4 +--
 scrapy/utils/serialize.py   |  4 +--
 tests/test_item.py          | 56 +++++++++++++++++++++++++++++++++----
 10 files changed, 85 insertions(+), 26 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 5a86ce2a7..292dbce41 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -204,7 +204,7 @@ flake8-ignore =
     tests/test_http_headers.py E501
     tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
     tests/test_http_response.py E501 E128
-    tests/test_item.py E128 F841
+    tests/test_item.py E128 F841 E501
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128 E124
     tests/test_loader.py E501 E741 E128 E117
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 1cefed106..098827ab9 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -5,7 +5,7 @@ from w3lib.url import is_url
 
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
-from scrapy.item import BaseItem
+from scrapy.item import _BaseItem
 from scrapy.utils import display
 from scrapy.utils.conf import arglist_to_dict
 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
@@ -117,7 +117,7 @@ class Command(ScrapyCommand):
         items, requests = [], []
 
         for x in iterate_spider_output(callback(response, **cb_kwargs)):
-            if isinstance(x, (BaseItem, dict)):
+            if isinstance(x, (_BaseItem, dict)):
                 items.append(x)
             elif isinstance(x, Request):
                 requests.append(x)
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index a1b0f8f22..cdc2bac15 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -1,6 +1,6 @@
 import json
 
-from scrapy.item import BaseItem
+from scrapy.item import _BaseItem
 from scrapy.http import Request
 from scrapy.exceptions import ContractFail
 
@@ -51,8 +51,8 @@ class ReturnsContract(Contract):
     objects = {
         'request': Request,
         'requests': Request,
-        'item': (BaseItem, dict),
-        'items': (BaseItem, dict),
+        'item': (_BaseItem, dict),
+        'items': (_BaseItem, dict),
     }
 
     def __init__(self, *args, **kwargs):
@@ -103,7 +103,7 @@ class ScrapesContract(Contract):
 
     def post_process(self, output):
         for x in output:
-            if isinstance(x, (BaseItem, dict)):
+            if isinstance(x, (_BaseItem, dict)):
                 missing = [arg for arg in self.args if arg not in x]
                 if missing:
                     raise ContractFail(
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index edbb4dd66..6785e103d 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -14,7 +14,7 @@ from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
 from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy import signals
 from scrapy.http import Request, Response
-from scrapy.item import BaseItem
+from scrapy.item import _BaseItem
 from scrapy.core.spidermw import SpiderMiddlewareManager
 
 
@@ -191,7 +191,7 @@ class Scraper:
         """
         if isinstance(output, Request):
             self.crawler.engine.crawl(request=output, spider=spider)
-        elif isinstance(output, (BaseItem, dict)):
+        elif isinstance(output, (_BaseItem, dict)):
             self.slot.itemproc_size += 1
             dfd = self.itemproc.process_item(output, spider)
             dfd.addBoth(self._itemproc_finished, output, response, spider)
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 0cb6cef98..4731b925a 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -12,7 +12,7 @@ from xml.sax.saxutils import XMLGenerator
 
 from scrapy.utils.serialize import ScrapyJSONEncoder
 from scrapy.utils.python import to_bytes, to_unicode, is_listlike
-from scrapy.item import BaseItem
+from scrapy.item import _BaseItem
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
@@ -312,7 +312,7 @@ class PythonItemExporter(BaseItemExporter):
         return serializer(value)
 
     def _serialize_value(self, value):
-        if isinstance(value, BaseItem):
+        if isinstance(value, _BaseItem):
             return self.export_item(value)
         if isinstance(value, dict):
             return dict(self._serialize_dict(value))
diff --git a/scrapy/item.py b/scrapy/item.py
index 46d20d017..f468ff86f 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -14,7 +14,23 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 
 
-class BaseItem(object_ref):
+class _BaseItem(object_ref):
+    """
+    Temporary class used internally to avoid the deprecation
+    warning raised by isinstance checks using BaseItem.
+    """
+    pass
+
+
+class _BaseItemMeta(ABCMeta):
+    def __instancecheck__(cls, instance):
+        if cls is BaseItem:
+            warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
+                 ScrapyDeprecationWarning, stacklevel=2)
+        return super().__instancecheck__(instance)
+
+
+class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
     """
     Deprecated, please use :class:`scrapy.item.Item` instead
     """
@@ -30,7 +46,7 @@ class Field(dict):
     """Container of field metadata"""
 
 
-class ItemMeta(ABCMeta):
+class ItemMeta(_BaseItemMeta):
     """Metaclass_ of :class:`Item` that handles field definitions.
 
     .. _metaclass: https://realpython.com/python-metaclasses
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 08ce89481..83afb74c9 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -13,7 +13,7 @@ from w3lib.url import any_to_uri
 from scrapy.crawler import Crawler
 from scrapy.exceptions import IgnoreRequest
 from scrapy.http import Request, Response
-from scrapy.item import BaseItem
+from scrapy.item import _BaseItem
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.utils.console import start_python_console
@@ -26,8 +26,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS
 
 class Shell:
 
-    relevant_classes = (Crawler, Spider, Request, Response, BaseItem,
-                        Settings)
+    relevant_classes = (Crawler, Spider, Request, Response, _BaseItem, Settings)
 
     def __init__(self, crawler, update_vars=None, code=None):
         self.crawler = crawler
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 52cfba208..bfe3ccd40 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -14,10 +14,10 @@ from w3lib.html import replace_entities
 
 from scrapy.utils.datatypes import LocalWeakReferencedCache
 from scrapy.utils.python import flatten, to_unicode
-from scrapy.item import BaseItem
+from scrapy.item import _BaseItem
 
 
-_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes
+_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
 
 
 def arg_to_iter(arg):
diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py
index 9dd72ea71..bf73dfa18 100644
--- a/scrapy/utils/serialize.py
+++ b/scrapy/utils/serialize.py
@@ -5,7 +5,7 @@ import decimal
 from twisted.internet import defer
 
 from scrapy.http import Request, Response
-from scrapy.item import BaseItem
+from scrapy.item import _BaseItem
 
 
 class ScrapyJSONEncoder(json.JSONEncoder):
@@ -26,7 +26,7 @@ class ScrapyJSONEncoder(json.JSONEncoder):
             return str(o)
         elif isinstance(o, defer.Deferred):
             return str(o)
-        elif isinstance(o, BaseItem):
+        elif isinstance(o, _BaseItem):
             return dict(o)
         elif isinstance(o, Request):
             return "<%s %s %s>" % (type(o).__name__, o.method, o.url)
diff --git a/tests/test_item.py b/tests/test_item.py
index f35a2b9f9..6fdd7e302 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -4,7 +4,7 @@ from unittest import mock
 from warnings import catch_warnings
 
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.item import ABCMeta, BaseItem, DictItem, Field, Item, ItemMeta
+from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta
 
 
 PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6)
@@ -334,29 +334,73 @@ class DictItemTest(unittest.TestCase):
 
 class BaseItemTest(unittest.TestCase):
 
+    def test_isinstance_check(self):
+
+        class SubclassedBaseItem(BaseItem):
+            pass
+
+        class SubclassedItem(Item):
+            pass
+
+        self.assertTrue(isinstance(BaseItem(), BaseItem))
+        self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem))
+        self.assertTrue(isinstance(Item(), BaseItem))
+        self.assertTrue(isinstance(SubclassedItem(), BaseItem))
+
+        # make sure internal checks using private _BaseItem class succeed
+        self.assertTrue(isinstance(BaseItem(), _BaseItem))
+        self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem))
+        self.assertTrue(isinstance(Item(), _BaseItem))
+        self.assertTrue(isinstance(SubclassedItem(), _BaseItem))
+
     def test_deprecation_warning(self):
+        """
+        Make sure deprecation warnings are logged whenever BaseItem is used,
+        either instantiated or in an isinstance check
+        """
         with catch_warnings(record=True) as warnings:
             BaseItem()
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+
         with catch_warnings(record=True) as warnings:
+
             class SubclassedBaseItem(BaseItem):
                 pass
+
             SubclassedBaseItem()
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 
+        with catch_warnings(record=True) as warnings:
+            self.assertFalse(isinstance("foo", BaseItem))
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+
+        with catch_warnings(record=True) as warnings:
+            self.assertTrue(isinstance(BaseItem(), BaseItem))
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+
 
 class ItemNoDeprecationWarningTest(unittest.TestCase):
-
     def test_no_deprecation_warning(self):
+        """
+        Make sure deprecation warnings are NOT logged whenever BaseItem subclasses are used.
+        """
+        class SubclassedItem(Item):
+            pass
+
         with catch_warnings(record=True) as warnings:
             Item()
-            self.assertEqual(len(warnings), 0)
-        with catch_warnings(record=True) as warnings:
-            class SubclassedItem(Item):
-                pass
             SubclassedItem()
+            _BaseItem()
+            self.assertFalse(isinstance("foo", _BaseItem))
+            self.assertFalse(isinstance("foo", Item))
+            self.assertFalse(isinstance("foo", SubclassedItem))
+            self.assertTrue(isinstance(_BaseItem(), _BaseItem))
+            self.assertTrue(isinstance(Item(), Item))
+            self.assertTrue(isinstance(SubclassedItem(), SubclassedItem))
             self.assertEqual(len(warnings), 0)
 
 

From dcf7235f0e44c0199360cea9472b7a0da6bab1a7 Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Fri, 8 May 2020 01:48:23 +0530
Subject: [PATCH 2817/2974] Code sharing between crawl and runspider command

---
 scrapy/commands/common_commands.py | 29 +++++++++++++++++++++++++++++
 scrapy/commands/crawl.py           | 27 ++++-----------------------
 scrapy/commands/runspider.py       | 24 ++----------------------
 3 files changed, 35 insertions(+), 45 deletions(-)
 create mode 100644 scrapy/commands/common_commands.py

diff --git a/scrapy/commands/common_commands.py b/scrapy/commands/common_commands.py
new file mode 100644
index 000000000..7da7494ac
--- /dev/null
+++ b/scrapy/commands/common_commands.py
@@ -0,0 +1,29 @@
+from scrapy.commands import ScrapyCommand
+from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
+from scrapy.exceptions import UsageError
+
+
+class CommonCommands(ScrapyCommand):
+
+    def add_options(self, parser):
+        ScrapyCommand.add_options(self, parser)
+        parser.add_option("-a", dest="spargs", action="append", default=[],
+                          metavar="NAME=VALUE",
+                          help="set spider argument (may be repeated)")
+        parser.add_option("-o", "--output", metavar="FILE", action="append",
+                          help="dump scraped items into FILE"
+                          + "(use - for stdout)")
+        parser.add_option("-t", "--output-format", metavar="FORMAT",
+                          help="format to use for dumping items with -o")
+
+    def process_options(self, args, opts):
+        ScrapyCommand.process_options(self, args, opts)
+        try:
+            opts.spargs = arglist_to_dict(opts.spargs)
+        except ValueError:
+            raise UsageError(
+                "Invalid -a value, use -a NAME=VALUE", print_help=False)
+        if opts.output:
+            feeds = feed_process_params_from_cli(
+                self.settings, opts.output, opts.output_format)
+            self.settings.set('FEEDS', feeds, priority='cmdline')
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 4b2f9484b..b477d7c71 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,9 +1,8 @@
-from scrapy.commands import ScrapyCommand
-from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 from scrapy.exceptions import UsageError
+from scrapy.commands.common_commands import CommonCommands
 
 
-class Command(ScrapyCommand):
+class Command(CommonCommands):
 
     requires_project = True
 
@@ -13,30 +12,12 @@ class Command(ScrapyCommand):
     def short_desc(self):
         return "Run a spider"
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
-        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
-                          help="set spider argument (may be repeated)")
-        parser.add_option("-o", "--output", metavar="FILE", action="append",
-                          help="dump scraped items into FILE (use - for stdout)")
-        parser.add_option("-t", "--output-format", metavar="FORMAT",
-                          help="format to use for dumping items with -o")
-
-    def process_options(self, args, opts):
-        ScrapyCommand.process_options(self, args, opts)
-        try:
-            opts.spargs = arglist_to_dict(opts.spargs)
-        except ValueError:
-            raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
-        if opts.output:
-            feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
-            self.settings.set('FEEDS', feeds, priority='cmdline')
-
     def run(self, args, opts):
         if len(args) < 1:
             raise UsageError()
         elif len(args) > 1:
-            raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
+            raise UsageError(
+                "running 'scrapy crawl' with more than one spider is no longer supported")
         spname = args[0]
 
         crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 62510609a..9959f6b0d 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -3,9 +3,8 @@ import os
 from importlib import import_module
 
 from scrapy.utils.spider import iter_spider_classes
-from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
-from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
+from scrapy.commands.common_commands import CommonCommands
 
 
 def _import_file(filepath):
@@ -24,7 +23,7 @@ def _import_file(filepath):
     return module
 
 
-class Command(ScrapyCommand):
+class Command(CommonCommands):
 
     requires_project = False
     default_settings = {'SPIDER_LOADER_WARN_ONLY': True}
@@ -38,25 +37,6 @@ class Command(ScrapyCommand):
     def long_desc(self):
         return "Run the spider defined in the given file"
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
-        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
-                          help="set spider argument (may be repeated)")
-        parser.add_option("-o", "--output", metavar="FILE", action="append",
-                          help="dump scraped items into FILE (use - for stdout)")
-        parser.add_option("-t", "--output-format", metavar="FORMAT",
-                          help="format to use for dumping items with -o")
-
-    def process_options(self, args, opts):
-        ScrapyCommand.process_options(self, args, opts)
-        try:
-            opts.spargs = arglist_to_dict(opts.spargs)
-        except ValueError:
-            raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
-        if opts.output:
-            feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
-            self.settings.set('FEEDS', feeds, priority='cmdline')
-
     def run(self, args, opts):
         if len(args) != 1:
             raise UsageError()

From cf09af787eafa6770bc5ab00bb0ee9759c75df23 Mon Sep 17 00:00:00 2001
From: Antonio Gordillo Toledo <antonio.gor@hotmail.com>
Date: Fri, 8 May 2020 06:45:19 -0700
Subject: [PATCH 2818/2974] Remove Python 2 encoding header from files (#4553)

---
 docs/conf.py                                        | 2 --
 scrapy/downloadermiddlewares/ajaxcrawl.py           | 1 -
 scrapy/spiderloader.py                              | 1 -
 scrapy/templates/project/module/items.py.tmpl       | 2 --
 scrapy/templates/project/module/middlewares.py.tmpl | 2 --
 scrapy/templates/project/module/pipelines.py.tmpl   | 2 --
 scrapy/templates/project/module/settings.py.tmpl    | 2 --
 scrapy/templates/spiders/basic.tmpl                 | 1 -
 scrapy/templates/spiders/crawl.tmpl                 | 1 -
 scrapy/templates/spiders/csvfeed.tmpl               | 1 -
 scrapy/templates/spiders/xmlfeed.tmpl               | 1 -
 scrapy/utils/log.py                                 | 2 --
 scrapy/utils/ssl.py                                 | 2 --
 tests/test_downloadermiddleware_redirect.py         | 2 --
 tests/test_downloadermiddleware_robotstxt.py        | 1 -
 tests/test_http_response.py                         | 1 -
 tests/test_pipeline_crawl.py                        | 1 -
 tests/test_responsetypes.py                         | 1 -
 tests/test_utils_deprecate.py                       | 1 -
 tests/test_utils_iterators.py                       | 1 -
 tests/test_utils_log.py                             | 1 -
 tests/test_utils_url.py                             | 1 -
 22 files changed, 30 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 813417bae..8ab38a090 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-#
 # Scrapy documentation build configuration file, created by
 # sphinx-quickstart on Mon Nov 24 12:02:52 2008.
 #
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index ad7a81e6b..4e12a5044 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import re
 import logging
 
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 63da55718..db4193430 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import traceback
 import warnings
 from collections import defaultdict
diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl
index a12d08414..88a18331c 100644
--- a/scrapy/templates/project/module/items.py.tmpl
+++ b/scrapy/templates/project/module/items.py.tmpl
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Define here the models for your scraped items
 #
 # See documentation in:
diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
index b3e58ff94..6490f52a7 100644
--- a/scrapy/templates/project/module/middlewares.py.tmpl
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Define here the models for your spider middleware
 #
 # See documentation in:
diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl
index 4876526a9..ce0edd335 100644
--- a/scrapy/templates/project/module/pipelines.py.tmpl
+++ b/scrapy/templates/project/module/pipelines.py.tmpl
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Define your item pipelines here
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index cb220eafc..a414b5fde 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Scrapy settings for $project_name project
 #
 # For simplicity, this file contains only settings considered important or
diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl
index 1cfe9cc9d..e9112bc95 100644
--- a/scrapy/templates/spiders/basic.tmpl
+++ b/scrapy/templates/spiders/basic.tmpl
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import scrapy
 
 
diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 878425125..356496487 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import scrapy
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule
diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl
index c2e4bacfe..cbcbe9e2c 100644
--- a/scrapy/templates/spiders/csvfeed.tmpl
+++ b/scrapy/templates/spiders/csvfeed.tmpl
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from scrapy.spiders import CSVFeedSpider
 
 
diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl
index 863c9772f..5aa2aa8b0 100644
--- a/scrapy/templates/spiders/xmlfeed.tmpl
+++ b/scrapy/templates/spiders/xmlfeed.tmpl
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from scrapy.spiders import XMLFeedSpider
 
 
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 5998dc33b..203842fc8 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import logging
 import sys
 import warnings
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index 6e81b33ff..c3c5e329b 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import OpenSSL
 import OpenSSL._util as pyOpenSSLutil
 
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 551e124ab..61c9eddbc 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import unittest
 
 from scrapy.downloadermiddlewares.redirect import RedirectMiddleware, MetaRefreshMiddleware
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index a1645ed96..b9452a0e7 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from unittest import mock
 
 from twisted.internet import reactor, error
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 522ec4875..43d6d936a 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from w3lib.encoding import resolve_encoding
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 24c516473..e2578a9c9 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import os
 import shutil
 
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 8cdf7a176..9e63ac924 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 from scrapy.responsetypes import responsetypes
 
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index b17e17f2f..adef66c1d 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import inspect
 import unittest
 from unittest import mock
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 46aaaecbc..69339256e 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import os
 
 from twisted.trial import unittest
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 21100aeb8..25cd904bc 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import sys
 import logging
 import unittest
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 1f8388957..16e7449c9 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from scrapy.spiders import Spider

From b852fff6f82e24c535c0dd9b4ef7fa78e7946497 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 15:19:22 -0300
Subject: [PATCH 2819/2974] Style changes in link extractor

---
 scrapy/linkextractors/lxmlhtml.py | 52 ++++++++++++++++++++++---------
 1 file changed, 37 insertions(+), 15 deletions(-)

diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index ceb37c5f1..9ebf8e7c7 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -28,8 +28,9 @@ def _nons(tag):
 
 
 class LxmlParserLinkExtractor:
-    def __init__(self, tag="a", attr="href", process=None, unique=False,
-                 strip=True, canonicalized=False):
+    def __init__(
+        self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False
+    ):
         self.scan_tag = tag if callable(tag) else lambda t: t == tag
         self.scan_attr = attr if callable(attr) else lambda a: a == attr
         self.process_attr = process if callable(process) else lambda v: v
@@ -93,10 +94,23 @@ class LxmlParserLinkExtractor:
 
 class LxmlLinkExtractor(FilteringLinkExtractor):
 
-    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
-                 tags=('a', 'area'), attrs=('href',), canonicalize=False,
-                 unique=True, process_value=None, deny_extensions=None, restrict_css=(),
-                 strip=True, restrict_text=None):
+    def __init__(
+        self,
+        allow=(),
+        deny=(),
+        allow_domains=(),
+        deny_domains=(),
+        restrict_xpaths=(),
+        tags=('a', 'area'),
+        attrs=('href',),
+        canonicalize=False,
+        unique=True,
+        process_value=None,
+        deny_extensions=None,
+        restrict_css=(),
+        strip=True,
+        restrict_text=None,
+    ):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
         lx = LxmlParserLinkExtractor(
             tag=lambda x: x in tags,
@@ -106,12 +120,18 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
             strip=strip,
             canonicalized=canonicalize
         )
-
-        super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
-                                                allow_domains=allow_domains, deny_domains=deny_domains,
-                                                restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
-                                                canonicalize=canonicalize, deny_extensions=deny_extensions,
-                                                restrict_text=restrict_text)
+        super(LxmlLinkExtractor, self).__init__(
+            link_extractor=lx,
+            allow=allow,
+            deny=deny,
+            allow_domains=allow_domains,
+            deny_domains=deny_domains,
+            restrict_xpaths=restrict_xpaths,
+            restrict_css=restrict_css,
+            canonicalize=canonicalize,
+            deny_extensions=deny_extensions,
+            restrict_text=restrict_text,
+        )
 
     def extract_links(self, response):
         """Returns a list of :class:`~scrapy.link.Link` objects from the
@@ -124,9 +144,11 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
         """
         base_url = get_base_url(response)
         if self.restrict_xpaths:
-            docs = [subdoc
-                    for x in self.restrict_xpaths
-                    for subdoc in response.xpath(x)]
+            docs = [
+                subdoc
+                for x in self.restrict_xpaths
+                for subdoc in response.xpath(x)
+            ]
         else:
             docs = [response.selector]
         all_links = []

From 3ebf2a0d82b0ae5b9e701972ba2a9e1420c6d2c7 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 15:17:33 -0300
Subject: [PATCH 2820/2974] Remove lambdas in link extractor

---
 scrapy/linkextractors/lxmlhtml.py | 28 +++++++++++++++++-----------
 tests/test_linkextractors.py      |  5 +++++
 2 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 9ebf8e7c7..1615d44d7 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -1,6 +1,8 @@
 """
 Link extractor based on lxml.html
 """
+import operator
+from functools import partial
 from urllib.parse import urljoin
 
 import lxml.etree as etree
@@ -8,10 +10,10 @@ from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
 
 from scrapy.link import Link
+from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list
 from scrapy.utils.response import get_base_url
-from scrapy.linkextractors import FilteringLinkExtractor
 
 
 # from lxml/src/lxml/html/__init__.py
@@ -27,20 +29,24 @@ def _nons(tag):
     return tag
 
 
+def _identity(x):
+    return x
+
+
+def _canonicalize_link_url(link):
+    return canonicalize_url(link.url, keep_fragments=True)
+
+
 class LxmlParserLinkExtractor:
     def __init__(
         self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False
     ):
-        self.scan_tag = tag if callable(tag) else lambda t: t == tag
-        self.scan_attr = attr if callable(attr) else lambda a: a == attr
-        self.process_attr = process if callable(process) else lambda v: v
+        self.scan_tag = tag if callable(tag) else partial(operator.eq, tag)
+        self.scan_attr = attr if callable(attr) else partial(operator.eq, attr)
+        self.process_attr = process if callable(process) else _identity
         self.unique = unique
         self.strip = strip
-        if canonicalized:
-            self.link_key = lambda link: link.url
-        else:
-            self.link_key = lambda link: canonicalize_url(link.url,
-                                                          keep_fragments=True)
+        self.link_key = operator.attrgetter("url") if canonicalized else _canonicalize_link_url
 
     def _iter_links(self, document):
         for el in document.iter(etree.Element):
@@ -113,8 +119,8 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
     ):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
         lx = LxmlParserLinkExtractor(
-            tag=lambda x: x in tags,
-            attr=lambda x: x in attrs,
+            tag=partial(operator.contains, tags),
+            attr=partial(operator.contains, attrs),
             unique=unique,
             process=process_value,
             strip=strip,
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 68e8514ba..46d8c13af 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -1,3 +1,4 @@
+import pickle
 import re
 import unittest
 from warnings import catch_warnings
@@ -462,6 +463,10 @@ class Base:
                 Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False),
             ])
 
+        def test_pickle_extractor(self):
+            lx = self.extractor_cls()
+            self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls)
+
 
 class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
     extractor_cls = LxmlLinkExtractor

From 81d0b2f61ac119efab4d5970bd235dbc288496ef Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 16:23:53 -0300
Subject: [PATCH 2821/2974] Flake8: remove E111

---
 pytest.ini                 | 4 ++--
 scrapy/selector/unified.py | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 604bbfe1d..20639baa3 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -102,7 +102,7 @@ flake8-ignore =
     scrapy/pipelines/media.py E501
     # scrapy/selector
     scrapy/selector/__init__.py F403
-    scrapy/selector/unified.py E501 E111
+    scrapy/selector/unified.py E501
     # scrapy/settings
     scrapy/settings/__init__.py E501
     scrapy/settings/default_settings.py E501 E114 E116
@@ -224,7 +224,7 @@ flake8-ignore =
     tests/test_spider.py E501
     tests/test_spidermiddleware.py E501
     tests/test_spidermiddleware_httperror.py E128 E501 E121
-    tests/test_spidermiddleware_offsite.py E501 E128 E111
+    tests/test_spidermiddleware_offsite.py E501 E128
     tests/test_spidermiddleware_output_chain.py E501
     tests/test_spidermiddleware_referer.py E501 F841 E501 E121
     tests/test_squeues.py E501 E741
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index a08955dc9..85a9bb526 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -65,9 +65,9 @@ class Selector(_ParselSelector, object_ref):
     selectorlist_cls = SelectorList
 
     def __init__(self, response=None, text=None, type=None, root=None, **kwargs):
-        if not(response is None or text is None):
-           raise ValueError('%s.__init__() received both response and text'
-                            % self.__class__.__name__)
+        if response is not None and text is not None:
+            raise ValueError('%s.__init__() received both response and text'
+                             % self.__class__.__name__)
 
         st = _st(response, type or self._default_type)
 

From 1a157f2e26274455405b7e272bab5aede0fa59fa Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 16:27:21 -0300
Subject: [PATCH 2822/2974] Flake8: remove E116

---
 pytest.ini                | 4 ++--
 scrapy/pipelines/files.py | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 20639baa3..295609435 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -97,7 +97,7 @@ flake8-ignore =
     scrapy/loader/processors.py E501
     # scrapy/pipelines
     scrapy/pipelines/__init__.py E501
-    scrapy/pipelines/files.py E116 E501
+    scrapy/pipelines/files.py E501
     scrapy/pipelines/images.py E501
     scrapy/pipelines/media.py E501
     # scrapy/selector
@@ -105,7 +105,7 @@ flake8-ignore =
     scrapy/selector/unified.py E501
     # scrapy/settings
     scrapy/settings/__init__.py E501
-    scrapy/settings/default_settings.py E501 E114 E116
+    scrapy/settings/default_settings.py E501 E114
     scrapy/settings/deprecated.py E501
     # scrapy/spidermiddlewares
     scrapy/spidermiddlewares/httperror.py E501
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index a9066986b..cd3e29057 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -83,8 +83,7 @@ class S3FilesStore:
     AWS_USE_SSL = None
     AWS_VERIFY = None
 
-    POLICY = 'private'  # Overriden from settings.FILES_STORE_S3_ACL in
-                        # FilesPipeline.from_settings.
+    POLICY = 'private'  # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
     HEADERS = {
         'Cache-Control': 'max-age=172800',
     }

From 83ce82f400d9686f7fbaa526f21f635fca8491de Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 16:28:26 -0300
Subject: [PATCH 2823/2974] Flake8: remove E114 and E117 (unused)

---
 pytest.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 295609435..59cce9ac4 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -105,7 +105,7 @@ flake8-ignore =
     scrapy/selector/unified.py E501
     # scrapy/settings
     scrapy/settings/__init__.py E501
-    scrapy/settings/default_settings.py E501 E114
+    scrapy/settings/default_settings.py E501
     scrapy/settings/deprecated.py E501
     # scrapy/spidermiddlewares
     scrapy/spidermiddlewares/httperror.py E501
@@ -207,7 +207,7 @@ flake8-ignore =
     tests/test_item.py E128 F841
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128
-    tests/test_loader.py E501 E741 E128 E117
+    tests/test_loader.py E501 E741 E128
     tests/test_logformatter.py E128 E501
     tests/test_mail.py E128 E501
     tests/test_middleware.py E501 E128

From c2c3054ac13838b89ad5062d6583cc52dd0ba317 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 16:32:02 -0300
Subject: [PATCH 2824/2974] Flake8: remove E121

---
 pytest.ini                               | 4 ++--
 tests/test_spidermiddleware_httperror.py | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 59cce9ac4..61139f7fe 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -223,10 +223,10 @@ flake8-ignore =
     tests/test_selector.py E501
     tests/test_spider.py E501
     tests/test_spidermiddleware.py E501
-    tests/test_spidermiddleware_httperror.py E128 E501 E121
+    tests/test_spidermiddleware_httperror.py E128 E501
     tests/test_spidermiddleware_offsite.py E501 E128
     tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 F841 E501 E121
+    tests/test_spidermiddleware_referer.py E501 F841 E501
     tests/test_squeues.py E501 E741
     tests/test_utils_asyncio.py E501
     tests/test_utils_conf.py E501 E128
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 6b61df56f..29584f21b 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -21,10 +21,10 @@ class _HttpErrorSpider(MockServerSpider):
     def __init__(self, *args, **kwargs):
         super(_HttpErrorSpider, self).__init__(*args, **kwargs)
         self.start_urls = [
-           self.mockserver.url("/status?n=200"),
-           self.mockserver.url("/status?n=404"),
-           self.mockserver.url("/status?n=402"),
-           self.mockserver.url("/status?n=500"),
+            self.mockserver.url("/status?n=200"),
+            self.mockserver.url("/status?n=404"),
+            self.mockserver.url("/status?n=402"),
+            self.mockserver.url("/status?n=500"),
         ]
         self.failed = set()
         self.skipped = set()

From f689e917bf2caa2c48def7399317dd019264663d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 19:44:05 -0300
Subject: [PATCH 2825/2974] Update Item docstring

---
 scrapy/item.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scrapy/item.py b/scrapy/item.py
index f468ff86f..b9a0acd3d 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -79,8 +79,7 @@ class DictItem(MutableMapping, BaseItem):
 
     def __new__(cls, *args, **kwargs):
         if issubclass(cls, DictItem) and not issubclass(cls, Item):
-            warn('scrapy.item.DictItem is deprecated, please use '
-                 'scrapy.item.Item instead',
+            warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
                  ScrapyDeprecationWarning, stacklevel=2)
         return super(DictItem, cls).__new__(cls, *args, **kwargs)
 
@@ -147,6 +146,11 @@ class Item(DictItem, metaclass=ItemMeta):
     If you need instances of a custom class to be considered items by Scrapy,
     you must inherit from either :class:`Item` or :class:`dict`.
 
+    Items offer the ability to declare :class:`Field` attributes, which can be
+    used to define metadata and control the way data is processed internally.
+    Please refer to the :ref:`documentation about fields <topics-items-fields>`
+    for additional information.
+
     Unlike instances of :class:`dict`, instances of :class:`Item` may be
     :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
     """

From 92792cc3f7c66a81a9e7a0576be0848266950c7e Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Sat, 9 May 2020 17:28:10 +0530
Subject: [PATCH 2826/2974] Moved common_commands.py to __init__.py

---
 scrapy/commands/__init__.py        | 52 ++++++++++++++++++++++++------
 scrapy/commands/common_commands.py | 29 -----------------
 scrapy/commands/crawl.py           |  4 +--
 scrapy/commands/runspider.py       |  4 +--
 4 files changed, 47 insertions(+), 42 deletions(-)
 delete mode 100644 scrapy/commands/common_commands.py

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 9f8e6986a..b95d395a1 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -5,7 +5,7 @@ import os
 from optparse import OptionGroup
 from twisted.python import failure
 
-from scrapy.utils.conf import arglist_to_dict
+from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 from scrapy.exceptions import UsageError
 
 
@@ -59,18 +59,19 @@ class ScrapyCommand:
         """
         group = OptionGroup(parser, "Global Options")
         group.add_option("--logfile", metavar="FILE",
-            help="log file. if omitted stderr will be used")
+                         help="log file. if omitted stderr will be used")
         group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
-            help="log level (default: %s)" % self.settings['LOG_LEVEL'])
+                         help="log level (default: %s)" % self.settings['LOG_LEVEL'])
         group.add_option("--nolog", action="store_true",
-            help="disable logging completely")
+                         help="disable logging completely")
         group.add_option("--profile", metavar="FILE", default=None,
-            help="write python cProfile stats to FILE")
+                         help="write python cProfile stats to FILE")
         group.add_option("--pidfile", metavar="FILE",
-            help="write process ID to FILE")
+                         help="write process ID to FILE")
         group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
-            help="set/override setting (may be repeated)")
-        group.add_option("--pdb", action="store_true", help="enable pdb on failure")
+                         help="set/override setting (may be repeated)")
+        group.add_option("--pdb", action="store_true",
+                         help="enable pdb on failure")
 
         parser.add_option_group(group)
 
@@ -79,7 +80,8 @@ class ScrapyCommand:
             self.settings.setdict(arglist_to_dict(opts.set),
                                   priority='cmdline')
         except ValueError:
-            raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False)
+            raise UsageError(
+                "Invalid -s value, use -s NAME=VALUE", print_help=False)
 
         if opts.logfile:
             self.settings.set('LOG_ENABLED', True, priority='cmdline')
@@ -104,3 +106,35 @@ class ScrapyCommand:
         Entry point for running commands
         """
         raise NotImplementedError
+
+
+''''
+The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for
+performing common functionality between crawl.py and runspider.py
+'''
+
+
+class BaseRunSpiderCommands(ScrapyCommand):
+
+    def add_options(self, parser):
+        ScrapyCommand.add_options(self, parser)
+        parser.add_option("-a", dest="spargs", action="append", default=[],
+                          metavar="NAME=VALUE",
+                          help="set spider argument (may be repeated)")
+        parser.add_option("-o", "--output", metavar="FILE", action="append",
+                          help="dump scraped items into FILE"
+                          + "(use - for stdout)")
+        parser.add_option("-t", "--output-format", metavar="FORMAT",
+                          help="format to use for dumping items with -o")
+
+    def process_options(self, args, opts):
+        ScrapyCommand.process_options(self, args, opts)
+        try:
+            opts.spargs = arglist_to_dict(opts.spargs)
+        except ValueError:
+            raise UsageError(
+                "Invalid -a value, use -a NAME=VALUE", print_help=False)
+        if opts.output:
+            feeds = feed_process_params_from_cli(
+                self.settings, opts.output, opts.output_format)
+            self.settings.set('FEEDS', feeds, priority='cmdline')
diff --git a/scrapy/commands/common_commands.py b/scrapy/commands/common_commands.py
deleted file mode 100644
index 7da7494ac..000000000
--- a/scrapy/commands/common_commands.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from scrapy.commands import ScrapyCommand
-from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
-from scrapy.exceptions import UsageError
-
-
-class CommonCommands(ScrapyCommand):
-
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
-        parser.add_option("-a", dest="spargs", action="append", default=[],
-                          metavar="NAME=VALUE",
-                          help="set spider argument (may be repeated)")
-        parser.add_option("-o", "--output", metavar="FILE", action="append",
-                          help="dump scraped items into FILE"
-                          + "(use - for stdout)")
-        parser.add_option("-t", "--output-format", metavar="FORMAT",
-                          help="format to use for dumping items with -o")
-
-    def process_options(self, args, opts):
-        ScrapyCommand.process_options(self, args, opts)
-        try:
-            opts.spargs = arglist_to_dict(opts.spargs)
-        except ValueError:
-            raise UsageError(
-                "Invalid -a value, use -a NAME=VALUE", print_help=False)
-        if opts.output:
-            feeds = feed_process_params_from_cli(
-                self.settings, opts.output, opts.output_format)
-            self.settings.set('FEEDS', feeds, priority='cmdline')
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index b477d7c71..ca19b1367 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,8 +1,8 @@
 from scrapy.exceptions import UsageError
-from scrapy.commands.common_commands import CommonCommands
+from scrapy.commands import BaseRunSpiderCommands
 
 
-class Command(CommonCommands):
+class Command(BaseRunSpiderCommands):
 
     requires_project = True
 
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 9959f6b0d..bd24a369e 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -4,7 +4,7 @@ from importlib import import_module
 
 from scrapy.utils.spider import iter_spider_classes
 from scrapy.exceptions import UsageError
-from scrapy.commands.common_commands import CommonCommands
+from scrapy.commands import BaseRunSpiderCommands
 
 
 def _import_file(filepath):
@@ -23,7 +23,7 @@ def _import_file(filepath):
     return module
 
 
-class Command(CommonCommands):
+class Command(BaseRunSpiderCommands):
 
     requires_project = False
     default_settings = {'SPIDER_LOADER_WARN_ONLY': True}

From c6746f0e381a44f3d66efad86a4aca87805138bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Sat, 9 May 2020 15:51:11 +0200
Subject: [PATCH 2827/2974] =?UTF-8?q?bytes=20array=20=E2=86=92=20bytes=20o?=
 =?UTF-8?q?bject?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/topics/request-response.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 4fec70e13..ad6c10b6e 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -53,7 +53,7 @@ Request objects
 
     :param body: the request body. If a string is passed, it is converted to
         bytes using *encoding*, which defaults to ``utf-8``. If not passed or
-        ``None`` is passed, an empty bytes array is stored.
+        ``None`` is passed, an empty :class:`bytes` object is stored.
     :type body: bytes
 
     :param headers: the headers of this request. The dict values can be strings

From e07708e3744fb26fa72042720b33deab59331cca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Sat, 9 May 2020 15:54:31 +0200
Subject: [PATCH 2828/2974] request-response: update the consequences of
 str(b'')

---
 docs/topics/request-response.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index ad6c10b6e..397632932 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -786,8 +786,7 @@ TextResponse objects
        .. note::
 
             ``str(response.body)`` is not a correct way to convert the response
-            body into a string: you would be using the system default encoding
-            (typically ``ascii``) instead of the response encoding.
+            body into a string: ``str(b'')`` returns ``"b''"``.
 
 
     .. attribute:: TextResponse.encoding

From ed4f4f84082f7023734d1200d88e8f1aec57904c Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Sun, 10 May 2020 00:08:34 +0530
Subject: [PATCH 2829/2974] Applied suggested format changes

---
 scrapy/commands/__init__.py | 22 ++++++++--------------
 scrapy/commands/crawl.py    |  5 ++---
 2 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index b95d395a1..1dada1ceb 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -80,8 +80,7 @@ class ScrapyCommand:
             self.settings.setdict(arglist_to_dict(opts.set),
                                   priority='cmdline')
         except ValueError:
-            raise UsageError(
-                "Invalid -s value, use -s NAME=VALUE", print_help=False)
+            raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False)
 
         if opts.logfile:
             self.settings.set('LOG_ENABLED', True, priority='cmdline')
@@ -108,18 +107,15 @@ class ScrapyCommand:
         raise NotImplementedError
 
 
-''''
-The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for
-performing common functionality between crawl.py and runspider.py
-'''
-
-
 class BaseRunSpiderCommands(ScrapyCommand):
+    ''''
+    The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for
+    performing common functionality between crawl.py and runspider.py
+    '''
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("-a", dest="spargs", action="append", default=[],
-                          metavar="NAME=VALUE",
+        parser.add_option("-a", dest="spargs", action="append", default=[],metavar="NAME=VALUE",
                           help="set spider argument (may be repeated)")
         parser.add_option("-o", "--output", metavar="FILE", action="append",
                           help="dump scraped items into FILE"
@@ -132,9 +128,7 @@ class BaseRunSpiderCommands(ScrapyCommand):
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
-            raise UsageError(
-                "Invalid -a value, use -a NAME=VALUE", print_help=False)
+            raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
         if opts.output:
-            feeds = feed_process_params_from_cli(
-                self.settings, opts.output, opts.output_format)
+            feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
             self.settings.set('FEEDS', feeds, priority='cmdline')
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index ca19b1367..c39c5a9ef 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,5 +1,5 @@
-from scrapy.exceptions import UsageError
 from scrapy.commands import BaseRunSpiderCommands
+from scrapy.exceptions import UsageError
 
 
 class Command(BaseRunSpiderCommands):
@@ -16,8 +16,7 @@ class Command(BaseRunSpiderCommands):
         if len(args) < 1:
             raise UsageError()
         elif len(args) > 1:
-            raise UsageError(
-                "running 'scrapy crawl' with more than one spider is no longer supported")
+            raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
         spname = args[0]
 
         crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)

From 02ac6664a5c3510104b67f7cfc9a71c831d5bd5f Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Sun, 10 May 2020 00:26:48 +0530
Subject: [PATCH 2830/2974] Travis CI fixes

---
 scrapy/commands/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 1dada1ceb..38b1b77bd 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -115,7 +115,7 @@ class BaseRunSpiderCommands(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("-a", dest="spargs", action="append", default=[],metavar="NAME=VALUE",
+        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
                           help="set spider argument (may be repeated)")
         parser.add_option("-o", "--output", metavar="FILE", action="append",
                           help="dump scraped items into FILE"

From 1bfbcc618090d4e71e06471b78aaac3071a51956 Mon Sep 17 00:00:00 2001
From: Will Beaufoy <will@willbeaufoy.net>
Date: Sun, 10 May 2020 13:48:09 +0100
Subject: [PATCH 2831/2974] Extend hoverxref_roles to custom crossrefs (#4495)

---
 docs/conf.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/conf.py b/docs/conf.py
index 8ab38a090..3ae709a04 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -300,3 +300,4 @@ hoverxref_role_types = {
     "mod": "tooltip",
     "ref": "tooltip",
 }
+hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']

From b5684909d1cb01ad138a389caa750485b51f79cf Mon Sep 17 00:00:00 2001
From: Jacty <jacty@users.noreply.github.com>
Date: Mon, 11 May 2020 11:18:25 +0800
Subject: [PATCH 2832/2974] Unnecessary update when value is None

When value is None, it is not necessary to invoke update and run other methods and conditions to make the code complicated there.
---
 scrapy/settings/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index b9a13c018..f28fbfaf9 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -83,7 +83,8 @@ class BaseSettings(MutableMapping):
     def __init__(self, values=None, priority='project'):
         self.frozen = False
         self.attributes = {}
-        self.update(values, priority)
+        if values is not None:
+            self.update(values, priority)
 
     def __getitem__(self, opt_name):
         if opt_name not in self:

From abfdc1b5425997f1aa69b29465c6ab2324f37fd0 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 11 May 2020 13:20:06 -0300
Subject: [PATCH 2833/2974] Update docstring for Item class

---
 scrapy/item.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/scrapy/item.py b/scrapy/item.py
index b9a0acd3d..97dfed976 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -138,18 +138,20 @@ class Item(DictItem, metaclass=ItemMeta):
     """
     Base class for scraped items.
 
-    In Scrapy, an object is considered an *item* if it is an instance of either
-    :class:`Item` or :class:`dict`. For example, when the output of a
+    In Scrapy, an object is considered an ``item`` if it is an instance of either
+    :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
     spider callback is evaluated, only instances of :class:`Item` or
     :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
 
     If you need instances of a custom class to be considered items by Scrapy,
     you must inherit from either :class:`Item` or :class:`dict`.
 
-    Items offer the ability to declare :class:`Field` attributes, which can be
-    used to define metadata and control the way data is processed internally.
-    Please refer to the :ref:`documentation about fields <topics-items-fields>`
-    for additional information.
+    Items must declare :class:`Field` attributes, which are processed and stored
+    in the ``fields`` attribute. This restricts the set of allowed field names
+    and prevents typos, raising ``KeyError`` when referring to undefined fields.
+    Additionally, fields can be used to define metadata and control the way
+    data is processed internally. Please refer to the :ref:`documentation
+    about fields <topics-items-fields>` for additional information.
 
     Unlike instances of :class:`dict`, instances of :class:`Item` may be
     :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.

From 6f8758624c8d3df7472948ce9805601f6037548a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 11 May 2020 13:50:34 -0300
Subject: [PATCH 2834/2974] Flake8: remove F841

---
 pytest.ini                             | 22 ++++++++++------------
 tests/pipelines.py                     |  2 +-
 tests/test_crawler.py                  |  6 +++---
 tests/test_dependencies.py             |  2 +-
 tests/test_extension_telnet.py         |  2 --
 tests/test_feedexport.py               |  1 -
 tests/test_item.py                     |  8 ++++----
 tests/test_pipeline_images.py          |  2 +-
 tests/test_spidermiddleware_referer.py |  3 +--
 tests/test_utils_defer.py              |  4 ++--
 tests/test_utils_deprecate.py          |  4 ++--
 tests/test_utils_signal.py             |  2 +-
 12 files changed, 26 insertions(+), 32 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 32fd76445..fee54dcbd 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -164,14 +164,13 @@ flake8-ignore =
     scrapy/robotstxt.py E501
     scrapy/shell.py E501
     scrapy/signalmanager.py E501
-    scrapy/spiderloader.py F841 E501
+    scrapy/spiderloader.py E501
     scrapy/squeues.py E128
     scrapy/squeues.py E501
     scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501
     tests/mockserver.py E501
-    tests/pipelines.py F841
     tests/spiders.py E501
     tests/test_closespider.py E501
     tests/test_command_fetch.py E501
@@ -180,8 +179,8 @@ flake8-ignore =
     tests/test_commands.py E128 E501
     tests/test_contracts.py E501 E128
     tests/test_crawl.py E501 E741
-    tests/test_crawler.py F841 E501
-    tests/test_dependencies.py F841 E501
+    tests/test_crawler.py E501
+    tests/test_dependencies.py E501
     tests/test_downloader_handlers.py E128 E501
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
@@ -199,13 +198,12 @@ flake8-ignore =
     tests/test_dupefilters.py E501 E741 E128
     tests/test_engine.py E501 E128
     tests/test_exporters.py E501 E128
-    tests/test_extension_telnet.py F841
-    tests/test_feedexport.py E501 F841
+    tests/test_feedexport.py E501
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
     tests/test_http_request.py E402 E501 E128 E128
     tests/test_http_response.py E501 E128
-    tests/test_item.py E128 F841
+    tests/test_item.py E128
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128
     tests/test_loader.py E501 E741 E128
@@ -214,7 +212,7 @@ flake8-ignore =
     tests/test_middleware.py E501 E128
     tests/test_pipeline_crawl.py E501 E128
     tests/test_pipeline_files.py E501
-    tests/test_pipeline_images.py F841 E501
+    tests/test_pipeline_images.py E501
     tests/test_pipeline_media.py E501 E741 E128
     tests/test_proxy_connect.py E501 E741
     tests/test_request_cb_kwargs.py E501
@@ -227,14 +225,14 @@ flake8-ignore =
     tests/test_spidermiddleware_httperror.py E128 E501
     tests/test_spidermiddleware_offsite.py E501 E128
     tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 F841 E501
+    tests/test_spidermiddleware_referer.py E501 E501
     tests/test_squeues.py E501 E741
     tests/test_utils_asyncio.py E501
     tests/test_utils_conf.py E501 E128
     tests/test_utils_curl.py E501
     tests/test_utils_datatypes.py E402 E501
-    tests/test_utils_defer.py E501 F841
-    tests/test_utils_deprecate.py F841 E501
+    tests/test_utils_defer.py E501
+    tests/test_utils_deprecate.py E501
     tests/test_utils_http.py E501 E128
     tests/test_utils_iterators.py E501 E128
     tests/test_utils_log.py E741
@@ -242,7 +240,7 @@ flake8-ignore =
     tests/test_utils_reqser.py E501 E128
     tests/test_utils_request.py E501 E128
     tests/test_utils_response.py E501
-    tests/test_utils_signal.py E741 F841
+    tests/test_utils_signal.py E741
     tests/test_utils_sitemap.py E128 E501
     tests/test_utils_url.py E501 E501
     tests/test_webclient.py E501 E128 E402
diff --git a/tests/pipelines.py b/tests/pipelines.py
index cf677cc17..fed2af7d3 100644
--- a/tests/pipelines.py
+++ b/tests/pipelines.py
@@ -6,7 +6,7 @@ Some pipelines used for testing
 class ZeroDivisionErrorPipeline:
 
     def open_spider(self, spider):
-        a = 1 / 0
+        1 / 0
 
     def process_item(self, item, spider):
         return item
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 9151278a5..ecc0cd7af 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -87,7 +87,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
         class MySpider(scrapy.Spider):
             name = 'spider'
 
-        crawler = Crawler(MySpider, {})
+        Crawler(MySpider, {})
         assert get_scrapy_root_handler() is None
 
     def test_spider_custom_settings_log_level(self):
@@ -240,13 +240,13 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
     def test_crawler_runner_asyncio_enabled_true(self):
         if self.reactor_pytest == 'asyncio':
-            runner = CrawlerRunner(settings={
+            CrawlerRunner(settings={
                 "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
             })
         else:
             msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
             with self.assertRaisesRegex(Exception, msg):
-                runner = CrawlerRunner(settings={
+                CrawlerRunner(settings={
                     "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
                 })
 
diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
index a169acbe6..5d0a1d0c9 100644
--- a/tests/test_dependencies.py
+++ b/tests/test_dependencies.py
@@ -6,7 +6,7 @@ class ScrapyUtilsTest(unittest.TestCase):
     def test_required_openssl_version(self):
         try:
             module = import_module('OpenSSL')
-        except ImportError as ex:
+        except ImportError:
             raise unittest.SkipTest("OpenSSL is not available")
 
         if hasattr(module, '__version__'):
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index 873a97248..1e716b94a 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -11,8 +11,6 @@ class TelnetExtensionTest(unittest.TestCase):
     def _get_console_and_portal(self, settings=None):
         crawler = get_crawler(settings_dict=settings)
         console = TelnetConsole(crawler)
-        username = console.username
-        password = console.password
 
         # This function has some side effects we don't need for this test
         console._get_telnet_vars = lambda: {}
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e02b0b840..cbc81bc35 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -715,7 +715,6 @@ class FeedExportTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_export_encoding(self):
         items = [dict({'foo': u'Test\xd6'})]
-        header = ['foo']
 
         formats = {
             'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
diff --git a/tests/test_item.py b/tests/test_item.py
index 4017f6e84..58735cc5c 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -264,12 +264,12 @@ class ItemTest(unittest.TestCase):
         """Make sure the DictItem deprecation warning is not issued for
         Item"""
         with catch_warnings(record=True) as warnings:
-            item = Item()
+            Item()
             self.assertEqual(len(warnings), 0)
 
             class SubclassedItem(Item):
                 pass
-            subclassed_item = SubclassedItem()
+            SubclassedItem()
             self.assertEqual(len(warnings), 0)
 
 
@@ -321,13 +321,13 @@ class DictItemTest(unittest.TestCase):
 
     def test_deprecation_warning(self):
         with catch_warnings(record=True) as warnings:
-            dict_item = DictItem()
+            DictItem()
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
         with catch_warnings(record=True) as warnings:
             class SubclassedDictItem(DictItem):
                 pass
-            subclassed_dict_item = SubclassedDictItem()
+            SubclassedDictItem()
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 5018d6802..5ba03ff4c 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -15,7 +15,7 @@ from scrapy.utils.python import to_bytes
 skip = False
 try:
     from PIL import Image
-except ImportError as e:
+except ImportError:
     skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
 else:
     encoders = set(('jpeg_encoder', 'jpeg_decoder'))
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 41589177a..ca765518b 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -459,7 +459,6 @@ class TestRequestMetaSettingFallback(TestCase):
         target = 'http://www.example.com'
 
         for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]:
-            spider = Spider('foo')
             mw = RefererMiddleware(Settings(settings))
 
             response = Response(origin, headers=response_headers)
@@ -511,7 +510,7 @@ class TestSettingsPolicyByName(TestCase):
     def test_invalid_name(self):
         settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'})
         with self.assertRaises(RuntimeError):
-            mw = RefererMiddleware(settings)
+            RefererMiddleware(settings)
 
 
 class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index a3b6e64f1..2d4b88121 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -64,7 +64,7 @@ class DeferUtilsTest(unittest.TestCase):
         gotexc = False
         try:
             yield process_chain([cb1, cb_fail, cb3], 'res', 'v1', 'v2')
-        except TypeError as e:
+        except TypeError:
             gotexc = True
         self.assertTrue(gotexc)
 
@@ -104,7 +104,7 @@ class IterErrbackTest(unittest.TestCase):
         def iterbad():
             for x in range(10):
                 if x == 5:
-                    a = 1 / 0
+                    1 / 0
                 yield x
 
         errors = []
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index adef66c1d..35d35b45d 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -25,7 +25,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
 
     def test_no_warning_on_definition(self):
         with warnings.catch_warnings(record=True) as w:
-            Deprecated = create_deprecated_class('Deprecated', NewName)
+            create_deprecated_class('Deprecated', NewName)
 
         w = self._mywarnings(w)
         self.assertEqual(w, [])
@@ -217,7 +217,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
     def test_deprecate_a_class_with_custom_metaclass(self):
         Meta1 = type('Meta1', (type,), {})
         New = Meta1('New', (), {})
-        Deprecated = create_deprecated_class('Deprecated', New)
+        create_deprecated_class('Deprecated', New)
 
     def test_deprecate_subclass_of_deprecated_class(self):
         with warnings.catch_warnings(record=True) as w:
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index bb211dc60..c83c9398c 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -44,7 +44,7 @@ class SendCatchLogTest(unittest.TestCase):
 
     def error_handler(self, arg, handlers_called):
         handlers_called.add(self.error_handler)
-        a = 1 / 0
+        1 / 0
 
     def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)

From cb8140a42a27ede87b0880372024f2f1804618b8 Mon Sep 17 00:00:00 2001
From: nsirletti <40069643+nsirletti@users.noreply.github.com>
Date: Mon, 11 May 2020 20:20:31 +0200
Subject: [PATCH 2835/2974] Deprecate Response.body_as_unicode() (#4555)

Co-authored-by: Nicolas Sirletti <n.sirletti@gmail.com>
---
 docs/topics/request-response.rst | 5 -----
 scrapy/http/response/text.py     | 5 +++++
 tests/test_http_response.py      | 9 +++++++++
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 024f46466..15a83f453 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -834,11 +834,6 @@ TextResponse objects
 
     .. automethod:: TextResponse.follow_all
 
-    .. method:: TextResponse.body_as_unicode()
-
-        The same as :attr:`text`, but available as a method. This method is
-        kept for backward compatibility; please prefer ``response.text``.
-
 
 HtmlResponse objects
 --------------------
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 2f0f3820c..5614e6e55 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -5,6 +5,7 @@ discovering (through HTTP headers) to base Response class.
 See documentation in docs/topics/request-response.rst
 """
 
+import warnings
 from contextlib import suppress
 from typing import Generator
 from urllib.parse import urljoin
@@ -14,6 +15,7 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
                             http_content_type_encoding, resolve_encoding)
 from w3lib.html import strip_html5_whitespace
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.http.response import Response
 from scrapy.utils.python import memoizemethod_noargs, to_unicode
@@ -61,6 +63,9 @@ class TextResponse(Response):
 
     def body_as_unicode(self):
         """Return body as unicode"""
+        warnings.warn('Response.body_as_unicode() is deprecated, '
+                      'please use Response.text instead.',
+                      ScrapyDeprecationWarning)
         return self.text
 
     @property
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 43d6d936a..2f73afe56 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,7 +1,9 @@
 import unittest
+from warnings import catch_warnings
 
 from w3lib.encoding import resolve_encoding
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
                          XmlResponse, Headers)
 from scrapy.selector import Selector
@@ -660,6 +662,13 @@ class TextResponseTest(BaseResponseTest):
         with self.assertRaises(ValueError):
             response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')
 
+    def test_body_as_unicode_deprecation_warning(self):
+        with catch_warnings(record=True) as warnings:
+            r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8')
+            self.assertEqual(r1.body_as_unicode(), u'Hello')
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+
 
 class HtmlResponseTest(TextResponseTest):
 

From cf9be5344a89dd8e14f8241ec69de9c984ec1e05 Mon Sep 17 00:00:00 2001
From: willbeaufoy <will@willbeaufoy.net>
Date: Mon, 11 May 2020 19:35:25 +0100
Subject: [PATCH 2836/2974] Prevent create_instance() returning None (#4532)

Currently create_instance() can return None if an extension is
incorrectly implemented, but the extension will still show up as
enabled in the logs. This can cause confusion, as in the linked bug.

This change prevents this occurring by throwing an error if
create_instance() will return None.
---
 scrapy/utils/misc.py              | 15 ++++++++++++---
 tests/test_utils_misc/__init__.py | 14 +++++++++++---
 2 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 52cfba208..ab7cf9deb 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -137,17 +137,26 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
     ``*args`` and ``**kwargs`` are forwarded to the constructors.
 
     Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
+
+    Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
+    extension has not been implemented correctly).
     """
     if settings is None:
         if crawler is None:
             raise ValueError("Specify at least one of settings and crawler.")
         settings = crawler.settings
     if crawler and hasattr(objcls, 'from_crawler'):
-        return objcls.from_crawler(crawler, *args, **kwargs)
+        instance = objcls.from_crawler(crawler, *args, **kwargs)
+        method_name = 'from_crawler'
     elif hasattr(objcls, 'from_settings'):
-        return objcls.from_settings(settings, *args, **kwargs)
+        instance = objcls.from_settings(settings, *args, **kwargs)
+        method_name = 'from_settings'
     else:
-        return objcls(*args, **kwargs)
+        instance = objcls(*args, **kwargs)
+        method_name = '__new__'
+    if instance is None:
+        raise TypeError("%s.%s returned None" % (objcls.__qualname__, method_name))
+    return instance
 
 
 @contextmanager
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index 6f945cd01..015a0e5a2 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -114,8 +114,12 @@ class UtilsMiscTestCase(unittest.TestCase):
         #   2. with from_settings() constructor
         #   3. with from_crawler() constructor
         #   4. with from_settings() and from_crawler() constructor
-        spec_sets = ([], ['from_settings'], ['from_crawler'],
-                     ['from_settings', 'from_crawler'])
+        spec_sets = (
+            ['__qualname__'],
+            ['__qualname__', 'from_settings'],
+            ['__qualname__', 'from_crawler'],
+            ['__qualname__', 'from_settings', 'from_crawler'],
+        )
         for specs in spec_sets:
             m = mock.MagicMock(spec_set=specs)
             _test_with_settings(m, settings)
@@ -123,7 +127,7 @@ class UtilsMiscTestCase(unittest.TestCase):
             _test_with_crawler(m, settings, crawler)
 
         # Check adoption of crawler settings
-        m = mock.MagicMock(spec_set=['from_settings'])
+        m = mock.MagicMock(spec_set=['__qualname__', 'from_settings'])
         create_instance(m, None, crawler, *args, **kwargs)
         m.from_settings.assert_called_once_with(crawler.settings, *args,
                                                 **kwargs)
@@ -131,6 +135,10 @@ class UtilsMiscTestCase(unittest.TestCase):
         with self.assertRaises(ValueError):
             create_instance(m, None, None)
 
+        m.from_settings.return_value = None
+        with self.assertRaises(TypeError):
+            create_instance(m, settings, None)
+
     def test_set_environ(self):
         assert os.environ.get('some_test_environ') is None
         with set_environ(some_test_environ='test_value'):

From e01c30f0d54ba838aa3212634cddf3630f77b2d4 Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Tue, 12 May 2020 01:05:20 +0530
Subject: [PATCH 2837/2974] Update scrapy/commands/__init__.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/commands/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 38b1b77bd..2ba796562 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -108,10 +108,10 @@ class ScrapyCommand:
 
 
 class BaseRunSpiderCommands(ScrapyCommand):
-    ''''
+    """
     The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for
     performing common functionality between crawl.py and runspider.py
-    '''
+    """
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)

From 28d223dd87fb655743f67087be0009b094ddad10 Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Tue, 12 May 2020 15:28:22 +0530
Subject: [PATCH 2838/2974] Update __init__.py

---
 scrapy/commands/__init__.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 2ba796562..99276515b 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -118,8 +118,7 @@ class BaseRunSpiderCommands(ScrapyCommand):
         parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
                           help="set spider argument (may be repeated)")
         parser.add_option("-o", "--output", metavar="FILE", action="append",
-                          help="dump scraped items into FILE"
-                          + "(use - for stdout)")
+                          help="dump scraped items into FILE (use - for stdout)")
         parser.add_option("-t", "--output-format", metavar="FORMAT",
                           help="format to use for dumping items with -o")
 

From 07e125f4c916fcd2d4d978a89c7d7e508cc63620 Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Tue, 12 May 2020 16:31:56 +0530
Subject: [PATCH 2839/2974] Travis CI fixes in __init__.py

---
 scrapy/commands/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 99276515b..0ddbf2ca0 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -112,7 +112,6 @@ class BaseRunSpiderCommands(ScrapyCommand):
     The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for
     performing common functionality between crawl.py and runspider.py
     """
-
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",

From 97532a9144cc011e5a6ede84fa7d317c3af60131 Mon Sep 17 00:00:00 2001
From: Aditya Kumar <k.aditya00@gmail.com>
Date: Tue, 12 May 2020 20:40:09 +0530
Subject: [PATCH 2840/2974] test(spiderloader): no duplicate spider names
 (#4560)

---
 tests/test_spiderloader/__init__.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index d8be6e277..b6fb27ffe 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -137,6 +137,11 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             msg = str(w[0].message)
             self.assertIn("several spiders with the same name", msg)
             self.assertIn("'spider3'", msg)
+            self.assertTrue(msg.count("'spider3'") == 2)
+
+            self.assertNotIn("'spider1'", msg)
+            self.assertNotIn("'spider2'", msg)
+            self.assertNotIn("'spider4'", msg)
 
             spiders = set(spider_loader.list())
             self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))
@@ -156,7 +161,13 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             msg = str(w[0].message)
             self.assertIn("several spiders with the same name", msg)
             self.assertIn("'spider1'", msg)
+            self.assertTrue(msg.count("'spider1'") == 2)
+
             self.assertIn("'spider2'", msg)
+            self.assertTrue(msg.count("'spider2'") == 2)
+
+            self.assertNotIn("'spider3'", msg)
+            self.assertNotIn("'spider4'", msg)
 
             spiders = set(spider_loader.list())
             self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))

From 8d1269bcbc81fa0bb5a69068e07bdbcb0dba8889 Mon Sep 17 00:00:00 2001
From: Aditya Kumar <k.aditya00@gmail.com>
Date: Wed, 13 May 2020 00:12:28 +0530
Subject: [PATCH 2841/2974] Cover chompjs in documentation (#4562)

---
 docs/topics/dynamic-content.rst | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index 3b85bfe8a..495111b56 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -184,6 +184,18 @@ data from it:
     >>> json.loads(json_data)
     {'field': 'value'}
 
+-   chompjs_ provides an API to parse JavaScript objects into a :class:`dict`.
+
+    For example, if the JavaScript code contains
+    ``var data = {field: "value", secondField: "second value"};``
+    you can extract that data as follows:
+
+    >>> import chompjs
+    >>> javascript = response.css('script::text').get()
+    >>> data = chompjs.parse_js_object(javascript)
+    >>> data
+    {'field': 'value', 'secondField': 'second value'}
+
 -   Otherwise, use js2xml_ to convert the JavaScript code into an XML document
     that you can parse using :ref:`selectors <topics-selectors>`.
 
@@ -241,6 +253,7 @@ along with `scrapy-selenium`_ for seamless integration.
 
 
 .. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
+.. _chompjs: https://github.com/Nykakin/chompjs
 .. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
 .. _curl: https://curl.haxx.se/
 .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser

From 33ab0a36635fbd45debbc44584002bd7a4ef7fed Mon Sep 17 00:00:00 2001
From: Jacty <jacty@users.noreply.github.com>
Date: Wed, 13 May 2020 06:11:07 +0800
Subject: [PATCH 2842/2974] Update __init__.py

---
 scrapy/settings/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index f28fbfaf9..0425b48b3 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -83,7 +83,7 @@ class BaseSettings(MutableMapping):
     def __init__(self, values=None, priority='project'):
         self.frozen = False
         self.attributes = {}
-        if values is not None:
+        if values:
             self.update(values, priority)
 
     def __getitem__(self, opt_name):

From 8971878c1d4d95aff10a1e910688502de892662b Mon Sep 17 00:00:00 2001
From: marc <Marc>
Date: Wed, 13 May 2020 14:11:10 +0200
Subject: [PATCH 2843/2974] fix new detected flake8 cases

---
 scrapy/core/spidermw.py | 2 +-
 scrapy/utils/console.py | 2 ++
 scrapy/utils/python.py  | 1 +
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 87d08cab7..35264a92b 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -19,7 +19,7 @@ def _isiterable(possible_iterator):
 
 
 def _fname(f):
-    return "%s.%s".format(
+    return "{}.{}".format(
         f.__self__.__class__.__name__,
         f.__func__.__name__
     )
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index c7a2ace88..133261fd7 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -28,6 +28,7 @@ def _embed_ipython_shell(namespace={}, banner=''):
 def _embed_bpython_shell(namespace={}, banner=''):
     """Start a bpython shell"""
     import bpython
+
     @wraps(_embed_bpython_shell)
     def wrapper(namespace=namespace, banner=''):
         bpython.embed(locals_=namespace, banner=banner)
@@ -37,6 +38,7 @@ def _embed_bpython_shell(namespace={}, banner=''):
 def _embed_ptpython_shell(namespace={}, banner=''):
     """Start a ptpython shell"""
     import ptpython.repl
+
     @wraps(_embed_ptpython_shell)
     def wrapper(namespace=namespace, banner=''):
         print(banner)
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 3d02d9478..9c1f3c2fe 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -152,6 +152,7 @@ def memoizemethod_noargs(method):
     weak reference to its object
     """
     cache = weakref.WeakKeyDictionary()
+
     @wraps(method)
     def new_method(self, *args, **kwargs):
         if self not in cache:

From 2327ecead085a41d1a71a70a12eb988bbf982268 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 13 May 2020 22:50:04 +0300
Subject: [PATCH 2844/2974] Rename FEED_STORAGE_BATCH_SIZE to
 FEED_STORAGE_BATCH_ITEM_COUNT

---
 docs/topics/feed-exports.rst        |  8 ++++----
 scrapy/extensions/feedexport.py     | 10 +++++-----
 scrapy/settings/default_settings.py |  2 +-
 tests/test_feedexport.py            | 28 ++++++++++++++--------------
 4 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 2106b41f5..917240d4d 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_STORAGE_FTP_ACTIVE`
  * :setting:`FEED_STORAGE_S3_ACL`
  * :setting:`FEED_EXPORTERS`
- * :setting:`FEED_EXPORT_BATCH_SIZE`
+ * :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`
 
 .. currentmodule:: scrapy.extensions.feedexport
 
@@ -431,9 +431,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. _botocore: https://github.com/boto/botocore
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
-.. setting:: FEED_EXPORT_BATCH_SIZE
+.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT
 
-FEED_EXPORT_BATCH_SIZE
+FEED_STORAGE_BATCH_ITEM_COUNT
 ----------------------
 Default: ``None``
 
@@ -448,7 +448,7 @@ Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI.
 
 For instance::
 
-    FEED_EXPORT_BATCH_SIZE=100
+    FEED_STORAGE_BATCH_ITEM_COUNT=100
 
 Your request can be like::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index a262f5d18..5bc946634 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -242,7 +242,7 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_SIZE', None)
+        self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
@@ -290,7 +290,7 @@ class FeedExporter:
     def _start_new_batch(self, batch_id, uri, feed, spider, template_uri):
         """
         Redirect the output data stream to a new file.
-        Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified.
+        Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.
         :param batch_id: sequence number of current batch
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
@@ -326,7 +326,7 @@ class FeedExporter:
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
-            # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_SIZE and close the old one
+            # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
             if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
                 uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
                 self._close_slot(slot, spider)
@@ -358,12 +358,12 @@ class FeedExporter:
 
     def _batch_deliveries_supported(self, uri):
         """
-        If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s
+        If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s
         to distinguish different files of partial output
         """
         if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
             return True
-        logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')
+        logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
         return False
 
     def _storage_supported(self, uri):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index c3463a505..5a7dc533e 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_STORAGE_BATCH_SIZE = None
+FEED_STORAGE_BATCH_ITEM_COUNT = None
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 2217bb4ed..1a21eeba9 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1025,7 +1025,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['jl']:
@@ -1041,7 +1041,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         data = yield self.exported_data(items, settings)
         for batch in data['csv']:
             got_batch = csv.DictReader(to_unicode(batch).splitlines())
@@ -1057,7 +1057,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['xml']:
@@ -1075,7 +1075,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         # XML
@@ -1100,7 +1100,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import pickle
@@ -1117,7 +1117,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import marshal
@@ -1140,7 +1140,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
             {'foo': 'bar3', 'baz': 'quux3', 'egg': ''}
         ]
         settings = {
-            'FEED_STORAGE_BATCH_SIZE': 2
+            'FEED_STORAGE_BATCH_ITEM_COUNT': 2
         }
         header = self.MyItem.fields.keys()
         yield self.assertExported(items, header, rows, settings=settings)
@@ -1151,7 +1151,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
             },
-            'FEED_STORAGE_BATCH_SIZE': 1
+            'FEED_STORAGE_BATCH_ITEM_COUNT': 1
         }
         crawler = get_crawler(settings_dict=settings)
         self.assertRaises(NotConfigured, FeedExporter, crawler)
@@ -1163,7 +1163,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 'FEEDS': {
                     os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
                 },
-                'FEED_STORAGE_BATCH_SIZE': 1
+                'FEED_STORAGE_BATCH_ITEM_COUNT': 1
             }
             data = yield self.exported_no_data(settings)
             data = dict(data)
@@ -1185,7 +1185,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                 },
                 'FEED_STORE_EMPTY': True,
                 'FEED_EXPORT_INDENT': None,
-                'FEED_STORAGE_BATCH_SIZE': 1,
+                'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
             }
             data = yield self.exported_no_data(settings)
             data = dict(data)
@@ -1225,7 +1225,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                     'encoding': 'utf-8',
                 },
             },
-            'FEED_STORAGE_BATCH_SIZE': 1,
+            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
         }
         data = yield self.exported_data(items, settings)
         for fmt, expected in formats.items():
@@ -1249,7 +1249,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                     'format': 'json',
                 },
             },
-            'FEED_STORAGE_BATCH_SIZE': 1,
+            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
         }
         data = yield self.exported_data(items, settings)
         self.assertEqual(len(items) + 1, len(data['json']))
@@ -1291,7 +1291,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
                     'format': 'json',
                 },
             },
-            'FEED_STORAGE_BATCH_SIZE': 1,
+            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
         }
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
@@ -1309,7 +1309,7 @@ class PartialDeliveriesTest(FeedExportTestBase):
 
         s3 = boto3.resource('s3')
         my_bucket = s3.Bucket(s3_test_bucket_name)
-        batch_size = settings['FEED_STORAGE_BATCH_SIZE']
+        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
 
         with MockServer() as s:
             runner = CrawlerRunner(Settings(settings))

From 8662d3587df74841d4ea640c0432446569e59262 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 13 May 2020 23:41:01 +0300
Subject: [PATCH 2845/2974] Documentation and code refactoring

---
 docs/topics/feed-exports.rst    | 21 ++++++++++++---------
 scrapy/extensions/feedexport.py |  7 ++++---
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 917240d4d..0f15044b3 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -437,24 +437,27 @@ FEED_STORAGE_BATCH_ITEM_COUNT
 ----------------------
 Default: ``None``
 
-An integer number which represent number of scraped items stored in each output
+An integer number that represents the number of scraped items stored in each output
 file. Whenever the number of items exceeds this setting, a new file is
-created and output redirects to it.  The name of the new file will be selected
-based on timestamp when the feed is being created and/or batch sequence number.
-Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI.
+created and the output is redirected to it.  The name of the new file will be selected
+based on the timestamp when the feed is being created and/or on the batch sequence number.
+Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`.
 
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
-* ``%(batch_id)s`` - gets replaced by sequence number of batch
+(e.g. `2020-03-28T14-45-08.237134`)
 
-For instance::
+* ``%(batch_id)s`` - gets replaced by the batch  sequence number of batch
+(e.g. `2` for the second file)
+
+For instance, if your settings include::
 
     FEED_STORAGE_BATCH_ITEM_COUNT=100
 
-Your request can be like::
+And your :command:`crawl` command line is::
 
   scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
 
-The result directory tree of above can be like::
+The resulting directory tree of above can be like::
 
 ->projectname
 -->dirname
@@ -462,5 +465,5 @@ The result directory tree of above can be like::
 --->2-filename2020-03-28T14-45-09.148903.json
 --->3-filename2020-03-28T14-45-10.046092.json
 
-Where first and second files contain exactly 100 items. The last one contains
+Where the first and second files contain exactly 100 items. The last one contains
 <= 100 items.
\ No newline at end of file
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 5bc946634..4c9362f3a 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -25,6 +25,7 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -337,9 +338,9 @@ class FeedExporter:
                     spider=spider,
                     template_uri=slot.template_uri,
                 ))
-                self.slots[idx] = None
-        self.slots = [slot for slot in self.slots if slot is not None]
-        self.slots.extend(slots)
+            else:
+                slots.append(slot)
+        self.slots = slots
 
     def _load_components(self, setting_prefix):
         conf = without_none_values(self.settings.getwithbase(setting_prefix))

From 69c005f013eb0dc000611853e66371fad17dea9d Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 14 May 2020 10:35:56 +0300
Subject: [PATCH 2846/2974] Documentation indent fix

---
 docs/topics/feed-exports.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 42c4e2267..dfeea5b7f 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -444,10 +444,10 @@ based on the timestamp when the feed is being created and/or on the batch sequen
 Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`.
 
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
-(e.g. `2020-03-28T14-45-08.237134`)
+  (e.g. `2020-03-28T14-45-08.237134`)
 
 * ``%(batch_id)s`` - gets replaced by the batch  sequence number of batch
-(e.g. `2` for the second file)
+  (e.g. `2` for the second file)
 
 For instance, if your settings include::
 

From df8a1d1c0108b15bebdad064d8b7bc61a894f062 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 May 2020 20:09:35 -0300
Subject: [PATCH 2847/2974] Flake8: Remove E741

---
 pytest.ini                                 | 18 +++++------
 scrapy/shell.py                            |  5 ++-
 scrapy/spiders/sitemap.py                  |  3 +-
 scrapy/utils/log.py                        |  2 +-
 tests/test_command_version.py              |  6 ++--
 tests/test_crawl.py                        | 36 +++++++++++-----------
 tests/test_downloadermiddleware_cookies.py |  8 ++---
 tests/test_dupefilters.py                  | 10 +++---
 tests/test_pipeline_media.py               | 10 +++---
 tests/test_proxy_connect.py                | 16 +++++-----
 tests/test_squeues.py                      | 24 +++++++--------
 tests/test_utils_log.py                    | 24 ++++++---------
 tests/test_utils_misc/__init__.py          |  4 +--
 tests/test_utils_signal.py                 | 12 ++++----
 14 files changed, 86 insertions(+), 92 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index fee54dcbd..c141d49a4 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -178,13 +178,13 @@ flake8-ignore =
     tests/test_command_shell.py E501 E128
     tests/test_commands.py E128 E501
     tests/test_contracts.py E501 E128
-    tests/test_crawl.py E501 E741
+    tests/test_crawl.py E501
     tests/test_crawler.py E501
     tests/test_dependencies.py E501
     tests/test_downloader_handlers.py E128 E501
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
-    tests/test_downloadermiddleware_cookies.py E741 E501 E128
+    tests/test_downloadermiddleware_cookies.py E501 E128
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
     tests/test_downloadermiddleware_httpcache.py E501
@@ -195,7 +195,7 @@ flake8-ignore =
     tests/test_downloadermiddleware_retry.py E501 E128
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
-    tests/test_dupefilters.py E501 E741 E128
+    tests/test_dupefilters.py E501 E128
     tests/test_engine.py E501 E128
     tests/test_exporters.py E501 E128
     tests/test_feedexport.py E501
@@ -206,15 +206,15 @@ flake8-ignore =
     tests/test_item.py E128
     tests/test_link.py E501
     tests/test_linkextractors.py E501 E128
-    tests/test_loader.py E501 E741 E128
+    tests/test_loader.py E128 E501 E741
     tests/test_logformatter.py E128 E501
     tests/test_mail.py E128 E501
     tests/test_middleware.py E501 E128
     tests/test_pipeline_crawl.py E501 E128
     tests/test_pipeline_files.py E501
     tests/test_pipeline_images.py E501
-    tests/test_pipeline_media.py E501 E741 E128
-    tests/test_proxy_connect.py E501 E741
+    tests/test_pipeline_media.py E501 E128
+    tests/test_proxy_connect.py E501
     tests/test_request_cb_kwargs.py E501
     tests/test_responsetypes.py E501
     tests/test_robotstxt_interface.py E501 E501
@@ -225,8 +225,8 @@ flake8-ignore =
     tests/test_spidermiddleware_httperror.py E128 E501
     tests/test_spidermiddleware_offsite.py E501 E128
     tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 E501
-    tests/test_squeues.py E501 E741
+    tests/test_spidermiddleware_referer.py E501
+    tests/test_squeues.py E501
     tests/test_utils_asyncio.py E501
     tests/test_utils_conf.py E501 E128
     tests/test_utils_curl.py E501
@@ -235,12 +235,10 @@ flake8-ignore =
     tests/test_utils_deprecate.py E501
     tests/test_utils_http.py E501 E128
     tests/test_utils_iterators.py E501 E128
-    tests/test_utils_log.py E741
     tests/test_utils_python.py E501
     tests/test_utils_reqser.py E501 E128
     tests/test_utils_request.py E501 E128
     tests/test_utils_response.py E501
-    tests/test_utils_signal.py E741
     tests/test_utils_sitemap.py E128 E501
     tests/test_utils_url.py E501 E501
     tests/test_webclient.py E501 E128 E402
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 08ce89481..2a3e13ddd 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -146,14 +146,13 @@ class Shell:
         b.append("Useful shortcuts:")
         if self.inthread:
             b.append("  fetch(url[, redirect=True]) "
-                     "Fetch URL and update local objects "
-                     "(by default, redirects are followed)")
+                     "Fetch URL and update local objects (by default, redirects are followed)")
             b.append("  fetch(req)                  "
                      "Fetch a scrapy.Request and update local objects ")
         b.append("  shelp()           Shell help (print this help)")
         b.append("  view(response)    View response in a browser")
 
-        return "\n".join("[s] %s" % l for l in b)
+        return "\n".join("[s] %s" % line for line in b)
 
     def _is_relevant(self, value):
         return isinstance(value, self.relevant_classes)
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index d368c7108..c5360bfa7 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -96,5 +96,4 @@ def iterloc(it, alt=False):
 
         # Also consider alternate URLs (xhtml:link rel="alternate")
         if alt and 'alternate' in d:
-            for l in d['alternate']:
-                yield l
+            yield from d['alternate']
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 203842fc8..6392862ee 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -37,7 +37,7 @@ class TopLevelFormatter(logging.Filter):
         self.loggers = loggers or []
 
     def filter(self, record):
-        if any(record.name.startswith(l + '.') for l in self.loggers):
+        if any(record.name.startswith(logger + '.') for logger in self.loggers):
             record.name = record.name.split('.', 1)[0]
         return True
 
diff --git a/tests/test_command_version.py b/tests/test_command_version.py
index 4ac7fb786..99c01c2b7 100644
--- a/tests/test_command_version.py
+++ b/tests/test_command_version.py
@@ -23,8 +23,10 @@ class VersionTest(ProcessTest, unittest.TestCase):
     def test_verbose_output(self):
         encoding = getattr(sys.stdout, 'encoding') or 'utf-8'
         _, out, _ = yield self.execute(['-v'])
-        headers = [l.partition(":")[0].strip()
-                   for l in out.strip().decode(encoding).splitlines()]
+        headers = [
+            line.partition(":")[0].strip()
+            for line in out.strip().decode(encoding).splitlines()
+        ]
         self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2',
                                    'cssselect', 'parsel', 'w3lib',
                                    'Twisted', 'Python', 'pyOpenSSL',
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 4215ca56c..84f80d103 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -104,44 +104,44 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_retry_503(self):
         crawler = self.runner.create_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
-        self._assert_retried(l)
+        self._assert_retried(log)
 
     @defer.inlineCallbacks
     def test_retry_conn_failed(self):
         crawler = self.runner.create_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver)
-        self._assert_retried(l)
+        self._assert_retried(log)
 
     @defer.inlineCallbacks
     def test_retry_dns_error(self):
         crawler = self.runner.create_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             # try to fetch the homepage of a non-existent domain
             yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)
-        self._assert_retried(l)
+        self._assert_retried(log)
 
     @defer.inlineCallbacks
     def test_start_requests_bug_before_yield(self):
-        with LogCapture('scrapy', level=logging.ERROR) as l:
+        with LogCapture('scrapy', level=logging.ERROR) as log:
             crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
 
-        self.assertEqual(len(l.records), 1)
-        record = l.records[0]
+        self.assertEqual(len(log.records), 1)
+        record = log.records[0]
         self.assertIsNotNone(record.exc_info)
         self.assertIs(record.exc_info[0], ZeroDivisionError)
 
     @defer.inlineCallbacks
     def test_start_requests_bug_yielding(self):
-        with LogCapture('scrapy', level=logging.ERROR) as l:
+        with LogCapture('scrapy', level=logging.ERROR) as log:
             crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
 
-        self.assertEqual(len(l.records), 1)
-        record = l.records[0]
+        self.assertEqual(len(log.records), 1)
+        record = log.records[0]
         self.assertIsNotNone(record.exc_info)
         self.assertIs(record.exc_info[0], ZeroDivisionError)
 
@@ -187,25 +187,25 @@ foo body
 with multiples lines
 '''})
         crawler = self.runner.create_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver)
-        self.assertEqual(str(l).count("Got response 200"), 1)
+        self.assertEqual(str(log).count("Got response 200"), 1)
 
     @defer.inlineCallbacks
     def test_retry_conn_lost(self):
         # connection lost after receiving data
         crawler = self.runner.create_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver)
-        self._assert_retried(l)
+        self._assert_retried(log)
 
     @defer.inlineCallbacks
     def test_retry_conn_aborted(self):
         # connection lost before receiving data
         crawler = self.runner.create_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver)
-        self._assert_retried(l)
+        self._assert_retried(log)
 
     def _assert_retried(self, log):
         self.assertEqual(str(log).count("Retrying"), 2)
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index f86c50f50..d54434c8f 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -63,7 +63,7 @@ class CookiesMiddlewareTest(TestCase):
         mw = CookiesMiddleware.from_crawler(crawler)
         with LogCapture('scrapy.downloadermiddlewares.cookies',
                         propagate=False,
-                        level=logging.DEBUG) as l:
+                        level=logging.DEBUG) as log:
             req = Request('http://scrapytest.org/')
             res = Response('http://scrapytest.org/',
                            headers={'Set-Cookie': 'C1=value1; path=/'})
@@ -71,7 +71,7 @@ class CookiesMiddlewareTest(TestCase):
             req2 = Request('http://scrapytest.org/sub1/')
             mw.process_request(req2, crawler.spider)
 
-            l.check(
+            log.check(
                 ('scrapy.downloadermiddlewares.cookies',
                  'DEBUG',
                  'Received cookies from: <200 http://scrapytest.org/>\n'
@@ -87,7 +87,7 @@ class CookiesMiddlewareTest(TestCase):
         mw = CookiesMiddleware.from_crawler(crawler)
         with LogCapture('scrapy.downloadermiddlewares.cookies',
                         propagate=False,
-                        level=logging.DEBUG) as l:
+                        level=logging.DEBUG) as log:
             req = Request('http://scrapytest.org/')
             res = Response('http://scrapytest.org/',
                            headers={'Set-Cookie': 'C1=value1; path=/'})
@@ -95,7 +95,7 @@ class CookiesMiddlewareTest(TestCase):
             req2 = Request('http://scrapytest.org/sub1/')
             mw.process_request(req2, crawler.spider)
 
-            l.check()
+            log.check()
 
     def test_do_not_break_on_non_utf8_header(self):
         req = Request('http://scrapytest.org/')
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 7426107c1..ee1794f4f 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -160,7 +160,7 @@ class RFPDupeFilterTest(unittest.TestCase):
             shutil.rmtree(path)
 
     def test_log(self):
-        with LogCapture() as l:
+        with LogCapture() as log:
             settings = {'DUPEFILTER_DEBUG': False,
                         'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
@@ -177,7 +177,7 @@ class RFPDupeFilterTest(unittest.TestCase):
             dupefilter.log(r2, spider)
 
             assert crawler.stats.get_value('dupefilter/filtered') == 2
-            l.check_present(('scrapy.dupefilters', 'DEBUG',
+            log.check_present(('scrapy.dupefilters', 'DEBUG',
                 ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
                 ' - no more duplicates will be shown'
                 ' (see DUPEFILTER_DEBUG to show all duplicates)')))
@@ -185,7 +185,7 @@ class RFPDupeFilterTest(unittest.TestCase):
             dupefilter.close('finished')
 
     def test_log_debug(self):
-        with LogCapture() as l:
+        with LogCapture() as log:
             settings = {'DUPEFILTER_DEBUG': True,
                         'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
@@ -203,10 +203,10 @@ class RFPDupeFilterTest(unittest.TestCase):
             dupefilter.log(r2, spider)
 
             assert crawler.stats.get_value('dupefilter/filtered') == 2
-            l.check_present(('scrapy.dupefilters', 'DEBUG',
+            log.check_present(('scrapy.dupefilters', 'DEBUG',
                 ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
                 ' (referer: None)')))
-            l.check_present(('scrapy.dupefilters', 'DEBUG',
+            log.check_present(('scrapy.dupefilters', 'DEBUG',
                 ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
                 ' (referer: http://scrapytest.org/INDEX.html)')))
 
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 949f0dea1..d7b0d32b2 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -63,21 +63,21 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         fail = Failure(Exception())
         results = [(True, 1), (False, fail)]
 
-        with LogCapture() as l:
+        with LogCapture() as log:
             new_item = self.pipe.item_completed(results, item, self.info)
 
         assert new_item is item
-        assert len(l.records) == 1
-        record = l.records[0]
+        assert len(log.records) == 1
+        record = log.records[0]
         assert record.levelname == 'ERROR'
         self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail))
 
         # disable failure logging and check again
         self.pipe.LOG_FAILED_RESULTS = False
-        with LogCapture() as l:
+        with LogCapture() as log:
             new_item = self.pipe.item_completed(results, item, self.info)
         assert new_item is item
-        assert len(l.records) == 0
+        assert len(log.records) == 0
 
     @inlineCallbacks
     def test_default_process_item(self):
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 188ec68dd..4763a5417 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -76,35 +76,35 @@ class ProxyConnectTestCase(TestCase):
     @defer.inlineCallbacks
     def test_https_connect_tunnel(self):
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
+        self._assert_got_response_code(200, log)
 
     @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl("https://localhost:99999/status?n=200")
-        self._assert_got_tunnel_error(l)
+        self._assert_got_tunnel_error(log)
 
     @defer.inlineCallbacks
     def test_https_tunnel_auth_error(self):
         os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         # The proxy returns a 407 error code but it does not reach the client;
         # he just sees a TunnelError.
-        self._assert_got_tunnel_error(l)
+        self._assert_got_tunnel_error(log)
 
     @defer.inlineCallbacks
     def test_https_tunnel_without_leak_proxy_authorization_header(self):
         request = Request(self.mockserver.url("/echo", is_secure=True))
         crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as l:
+        with LogCapture() as log:
             yield crawler.crawl(seed=request)
-        self._assert_got_response_code(200, l)
+        self._assert_got_response_code(200, log)
         echo = json.loads(crawler.spider.meta['responses'][0].text)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index d2cf9135f..becacce62 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -89,12 +89,12 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
 
     def test_serialize_loader(self):
         q = self.queue()
-        l = TestLoader()
-        q.push(l)
-        l2 = q.pop()
-        assert isinstance(l2, TestLoader)
-        assert l2.default_item_class is TestItem
-        self.assertEqual(l2.name_out('x'), 'xx')
+        loader = TestLoader()
+        q.push(loader)
+        loader2 = q.pop()
+        assert isinstance(loader2, TestLoader)
+        assert loader2.default_item_class is TestItem
+        self.assertEqual(loader2.name_out('x'), 'xx')
 
     def test_serialize_request_recursive(self):
         q = self.queue()
@@ -173,12 +173,12 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
 
     def test_serialize_loader(self):
         q = self.queue()
-        l = TestLoader()
-        q.push(l)
-        l2 = q.pop()
-        assert isinstance(l2, TestLoader)
-        assert l2.default_item_class is TestItem
-        self.assertEqual(l2.name_out('x'), 'xx')
+        loader = TestLoader()
+        q.push(loader)
+        loader2 = q.pop()
+        assert isinstance(loader2, TestLoader)
+        assert loader2.default_item_class is TestItem
+        self.assertEqual(loader2.name_out('x'), 'xx')
 
     def test_serialize_request_recursive(self):
         q = self.queue()
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 25cd904bc..535f56691 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -34,31 +34,27 @@ class TopLevelFormatterTest(unittest.TestCase):
 
     def test_top_level_logger(self):
         logger = logging.getLogger('test')
-        with self.handler as l:
+        with self.handler as log:
             logger.warning('test log msg')
-
-        l.check(('test', 'WARNING', 'test log msg'))
+        log.check(('test', 'WARNING', 'test log msg'))
 
     def test_children_logger(self):
         logger = logging.getLogger('test.test1')
-        with self.handler as l:
+        with self.handler as log:
             logger.warning('test log msg')
-
-        l.check(('test', 'WARNING', 'test log msg'))
+        log.check(('test', 'WARNING', 'test log msg'))
 
     def test_overlapping_name_logger(self):
         logger = logging.getLogger('test2')
-        with self.handler as l:
+        with self.handler as log:
             logger.warning('test log msg')
-
-        l.check(('test2', 'WARNING', 'test log msg'))
+        log.check(('test2', 'WARNING', 'test log msg'))
 
     def test_different_name_logger(self):
         logger = logging.getLogger('different')
-        with self.handler as l:
+        with self.handler as log:
             logger.warning('test log msg')
-
-        l.check(('different', 'WARNING', 'test log msg'))
+        log.check(('different', 'WARNING', 'test log msg'))
 
 
 class LogCounterHandlerTest(unittest.TestCase):
@@ -107,6 +103,6 @@ class StreamLoggerTest(unittest.TestCase):
         sys.stdout = self.stdout
 
     def test_redirect(self):
-        with LogCapture() as l:
+        with LogCapture() as log:
             print('test log msg')
-        l.check(('test', 'ERROR', 'test log msg'))
+        log.check(('test', 'ERROR', 'test log msg'))
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index 015a0e5a2..28205e0d9 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -67,12 +67,12 @@ class UtilsMiscTestCase(unittest.TestCase):
         assert hasattr(arg_to_iter(100), '__iter__')
         assert hasattr(arg_to_iter('lala'), '__iter__')
         assert hasattr(arg_to_iter([1, 2, 3]), '__iter__')
-        assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__')
+        assert hasattr(arg_to_iter(c for c in 'abcd'), '__iter__')
 
         self.assertEqual(list(arg_to_iter(None)), [])
         self.assertEqual(list(arg_to_iter('lala')), ['lala'])
         self.assertEqual(list(arg_to_iter(100)), [100])
-        self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c'])
+        self.assertEqual(list(arg_to_iter(c for c in 'abc')), ['a', 'b', 'c'])
         self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3])
         self.assertEqual(list(arg_to_iter({'a': 1})), [{'a': 1}])
         self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")])
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index c83c9398c..b66588efb 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -20,7 +20,7 @@ class SendCatchLogTest(unittest.TestCase):
 
         dispatcher.connect(self.error_handler, signal=test_signal)
         dispatcher.connect(self.ok_handler, signal=test_signal)
-        with LogCapture() as l:
+        with LogCapture() as log:
             result = yield defer.maybeDeferred(
                 self._get_result, test_signal, arg='test',
                 handlers_called=handlers_called
@@ -28,8 +28,8 @@ class SendCatchLogTest(unittest.TestCase):
 
         assert self.error_handler in handlers_called
         assert self.ok_handler in handlers_called
-        self.assertEqual(len(l.records), 1)
-        record = l.records[0]
+        self.assertEqual(len(log.records), 1)
+        record = log.records[0]
         self.assertIn('error_handler', record.getMessage())
         self.assertEqual(record.levelname, 'ERROR')
         self.assertEqual(result[0][0], self.error_handler)
@@ -95,8 +95,8 @@ class SendCatchLogTest2(unittest.TestCase):
 
         test_signal = object()
         dispatcher.connect(test_handler, test_signal)
-        with LogCapture() as l:
+        with LogCapture() as log:
             send_catch_log(test_signal)
-        self.assertEqual(len(l.records), 1)
-        self.assertIn("Cannot return deferreds from signal handler", str(l))
+        self.assertEqual(len(log.records), 1)
+        self.assertIn("Cannot return deferreds from signal handler", str(log))
         dispatcher.disconnect(test_handler, test_signal)

From fffb0a5b6a7eaf365ecd7ef43e7e45cf2ea7ff2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Fri, 15 May 2020 15:23:09 +0200
Subject: [PATCH 2848/2974] replace set constructor with set literals (#4573)

---
 tests/test_engine.py                | 8 ++++----
 tests/test_http_request.py          | 6 +++---
 tests/test_pipeline_crawl.py        | 4 ++--
 tests/test_pipeline_images.py       | 2 +-
 tests/test_spiderloader/__init__.py | 8 ++++----
 tests/test_utils_datatypes.py       | 2 +-
 tests/test_utils_misc/__init__.py   | 8 ++++----
 tests/test_utils_url.py             | 2 +-
 8 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index acfe94f63..d781665dc 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -218,8 +218,8 @@ class EngineTest(unittest.TestCase):
     def _assert_visited_urls(self):
         must_be_visited = ["/", "/redirect", "/redirected",
                            "/item1.html", "/item2.html", "/item999.html"]
-        urls_visited = set([rp[0].url for rp in self.run.respplug])
-        urls_expected = set([self.run.geturl(p) for p in must_be_visited])
+        urls_visited = {rp[0].url for rp in self.run.respplug}
+        urls_expected = {self.run.geturl(p) for p in must_be_visited}
         assert urls_expected <= urls_visited, "URLs not visited: %s" % list(urls_expected - urls_visited)
 
     def _assert_scheduled_requests(self, urls_to_visit=None):
@@ -227,8 +227,8 @@ class EngineTest(unittest.TestCase):
 
         paths_expected = ['/item999.html', '/item2.html', '/item1.html']
 
-        urls_requested = set([rq[0].url for rq in self.run.reqplug])
-        urls_expected = set([self.run.geturl(p) for p in paths_expected])
+        urls_requested = {rq[0].url for rq in self.run.reqplug}
+        urls_expected = {self.run.geturl(p) for p in paths_expected}
         assert urls_expected <= urls_requested
         scheduled_requests_count = len(self.run.reqplug)
         dropped_requests_count = len(self.run.reqdropped)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index a672963f3..349c23f36 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -549,8 +549,8 @@ class FormRequestTest(RequestTest):
         self.assertEqual(urlparse(r1.url).hostname, "www.example.com")
         self.assertEqual(urlparse(r1.url).path, "/this/get.php")
         fs = _qs(r1)
-        self.assertEqual(set(fs[b'test']), set([b'val1', b'val2']))
-        self.assertEqual(set(fs[b'one']), set([b'two', b'three']))
+        self.assertEqual(set(fs[b'test']), {b'val1', b'val2'})
+        self.assertEqual(set(fs[b'one']), {b'two', b'three'})
         self.assertEqual(fs[b'test2'], [b'xxx'])
         self.assertEqual(fs[b'six'], [b'seven'])
 
@@ -1047,7 +1047,7 @@ class FormRequestTest(RequestTest):
             </form>''')
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(set(fs), set([b'h2', b'i2', b'i1', b'i3', b'h1', b'i5', b'i4']))
+        self.assertEqual(set(fs), {b'h2', b'i2', b'i1', b'i3', b'h1', b'i5', b'i4'})
 
     def test_from_response_xpath(self):
         response = _buildresponse(
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 6d15aaf31..74263e50c 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -51,10 +51,10 @@ class FileDownloadCrawlTestCase(TestCase):
     store_setting_key = 'FILES_STORE'
     media_key = 'files'
     media_urls_key = 'file_urls'
-    expected_checksums = set([
+    expected_checksums = {
         '5547178b89448faf0015a13f904c936e',
         'c2281c83670e31d8aaab7cb642b824db',
-        'ed3f6538dc15d4d9179dae57319edc5f'])
+        'ed3f6538dc15d4d9179dae57319edc5f'}
 
     def setUp(self):
         self.mockserver = MockServer()
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 5ba03ff4c..76aea470f 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -18,7 +18,7 @@ try:
 except ImportError:
     skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
 else:
-    encoders = set(('jpeg_encoder', 'jpeg_decoder'))
+    encoders = {'jpeg_encoder', 'jpeg_decoder'}
     if not encoders.issubset(set(Image.core.__dict__)):
         skip = 'Missing JPEG encoders'
 
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 265970b43..d922c6059 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -42,7 +42,7 @@ class SpiderLoaderTest(unittest.TestCase):
     def test_list(self):
         self.assertEqual(
             set(self.spider_loader.list()),
-            set(['spider1', 'spider2', 'spider3', 'spider4']))
+            {'spider1', 'spider2', 'spider3', 'spider4'})
 
     def test_load(self):
         spider1 = self.spider_loader.load("spider1")
@@ -57,7 +57,7 @@ class SpiderLoaderTest(unittest.TestCase):
             ['spider2'])
         self.assertEqual(
             set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
-            set(['spider1', 'spider2']))
+            {'spider1', 'spider2'})
         self.assertEqual(
             self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
             [])
@@ -151,7 +151,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             self.assertNotIn("'spider4'", msg)
 
             spiders = set(spider_loader.list())
-            self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))
+            self.assertEqual(spiders, {'spider1', 'spider2', 'spider3', 'spider4'})
 
     def test_multiple_dupename_warning(self):
         # copy 2 spider modules so as to have duplicate spider name
@@ -177,4 +177,4 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             self.assertNotIn("'spider4'", msg)
 
             spiders = set(spider_loader.list())
-            self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))
+            self.assertEqual(spiders, {'spider1', 'spider2', 'spider3', 'spider4'})
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index e5aa56eb9..0a4c6034a 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -217,7 +217,7 @@ class SequenceExcludeTest(unittest.TestCase):
 
     def test_set(self):
         """Anything that is not in the supplied sequence will evaluate as 'in' the container."""
-        seq = set([-3, "test", 1.1])
+        seq = {-3, "test", 1.1}
         d = SequenceExclude(seq)
         self.assertIn(0, d)
         self.assertIn("foo", d)
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index 28205e0d9..9bb996d27 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -26,20 +26,20 @@ class UtilsMiscTestCase(unittest.TestCase):
             'tests.test_utils_misc.test_walk_modules.mod.mod0',
             'tests.test_utils_misc.test_walk_modules.mod1',
         ]
-        self.assertEqual(set([m.__name__ for m in mods]), set(expected))
+        self.assertEqual({m.__name__ for m in mods}, set(expected))
 
         mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod')
         expected = [
             'tests.test_utils_misc.test_walk_modules.mod',
             'tests.test_utils_misc.test_walk_modules.mod.mod0',
         ]
-        self.assertEqual(set([m.__name__ for m in mods]), set(expected))
+        self.assertEqual({m.__name__ for m in mods}, set(expected))
 
         mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod1')
         expected = [
             'tests.test_utils_misc.test_walk_modules.mod1',
         ]
-        self.assertEqual(set([m.__name__ for m in mods]), set(expected))
+        self.assertEqual({m.__name__ for m in mods}, set(expected))
 
         self.assertRaises(ImportError, walk_modules, 'nomodule999')
 
@@ -54,7 +54,7 @@ class UtilsMiscTestCase(unittest.TestCase):
                 'testegg.spiders.b',
                 'testegg'
             ]
-            self.assertEqual(set([m.__name__ for m in mods]), set(expected))
+            self.assertEqual({m.__name__ for m in mods}, set(expected))
         finally:
             sys.path.remove(egg)
 
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 16e7449c9..4ab944949 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -55,7 +55,7 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider))
         self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider))
 
-        spider = Spider(name='example.com', allowed_domains=set(('example.com', 'example.net')))
+        spider = Spider(name='example.com', allowed_domains={'example.com', 'example.net'})
         self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider))
 
         spider = Spider(name='example.com', allowed_domains=('example.com', 'example.net'))

From 1cdcf8b08b8f1e68c5b107b6ae39b2da1aedd245 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 15 May 2020 19:46:36 +0300
Subject: [PATCH 2849/2974] Minor fixes

---
 docs/topics/feed-exports.rst    | 19 ++++++++++---------
 scrapy/extensions/feedexport.py | 20 ++++++++++----------
 2 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index dfeea5b7f..638733b6a 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -437,17 +437,18 @@ FEED_STORAGE_BATCH_ITEM_COUNT
 -----------------------------
 Default: ``None``
 
-An integer number that represents the number of scraped items stored in each output
-file. Whenever the number of items exceeds this setting, a new file is
-created and the output is redirected to it.  The name of the new file will be selected
-based on the timestamp when the feed is being created and/or on the batch sequence number.
-Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`.
+If assigned an integer number higher than ``0``, Scrapy generates multiple output files
+storing up to the specified number of items in each output file.
+
+When generating multiple output files, you must use at least one of the following
+placeholders in :setting:`FEED_URI` to indicate how the different output file names are
+generated:
 
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
-  (e.g. `2020-03-28T14-45-08.237134`)
+  (e.g. ``2020-03-28T14-45-08.237134``)
 
 * ``%(batch_id)s`` - gets replaced by the batch  sequence number of batch
-  (e.g. `2` for the second file)
+  (e.g. ``2`` for the second file)
 
 For instance, if your settings include::
 
@@ -457,7 +458,7 @@ And your :command:`crawl` command line is::
 
   scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
 
-The resulting directory tree of above can be like::
+The command line above can generate a directory tree like::
 
 ->projectname
 -->dirname
@@ -466,4 +467,4 @@ The resulting directory tree of above can be like::
 --->3-filename2020-03-28T14-45-10.046092.json
 
 Where the first and second files contain exactly 100 items. The last one contains
-<= 100 items.
\ No newline at end of file
+100 items or fever.
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 4c9362f3a..3d691c580 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -180,7 +180,7 @@ class FTPFeedStorage(BlockingFeedStorage):
 
 
 class _FeedSlot:
-    def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri):
+    def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):
         self.file = file
         self.exporter = exporter
         self.storage = storage
@@ -188,7 +188,7 @@ class _FeedSlot:
         self.batch_id = batch_id
         self.format = format
         self.store_empty = store_empty
-        self.template_uri = template_uri
+        self.uri_template = uri_template
         self.uri = uri
         # flags
         self.itemcount = 0
@@ -260,7 +260,7 @@ class FeedExporter:
                 uri=uri % uri_params,
                 feed=feed,
                 spider=spider,
-                template_uri=uri,
+                uri_template=uri,
             ))
 
     def close_spider(self, spider):
@@ -288,7 +288,7 @@ class FeedExporter:
                                             extra={'spider': spider}))
         return d
 
-    def _start_new_batch(self, batch_id, uri, feed, spider, template_uri):
+    def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
         """
         Redirect the output data stream to a new file.
         Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.
@@ -296,7 +296,7 @@ class FeedExporter:
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
-        :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri
+        :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri
         """
         storage = self._get_storage(uri)
         file = storage.open(spider)
@@ -315,7 +315,7 @@ class FeedExporter:
             format=feed['format'],
             store_empty=feed['store_empty'],
             batch_id=batch_id,
-            template_uri=template_uri,
+            uri_template=uri_template,
         )
         if slot.store_empty:
             slot.start_exporting()
@@ -329,14 +329,14 @@ class FeedExporter:
             slot.itemcount += 1
             # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
             if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
-                uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
+                uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
                 self._close_slot(slot, spider)
                 slots.append(self._start_new_batch(
                     batch_id=slot.batch_id + 1,
-                    uri=slot.template_uri % uri_params,
-                    feed=self.feeds[slot.template_uri],
+                    uri=slot.uri_template % uri_params,
+                    feed=self.feeds[slot.uri_template],
                     spider=spider,
-                    template_uri=slot.template_uri,
+                    uri_template=slot.uri_template,
                 ))
             else:
                 slots.append(slot)

From 36c3c9713e59f5d22bf51354920b5093e2d30b73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 15 May 2020 19:37:56 +0200
Subject: [PATCH 2850/2974] Run tests with Python 3.5.0 (#4518)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Run tests with Python 3.5.0

* Use Ubuntu Trusty 14.04 to test Python 3.5.0

* Use mitmproxy<3.0.0 on Python 3.5.0

* Skip tests requiring mitmproxy in Python 3.5.0

* Change the minimum Python version from 3.5 to 3.5.1

* Do not prevent Scrapy from working with Python 3.5.0

* Force system Python 3.5.1

* Do not install a non-system Python in Python 3.5.1 jobs

* Switch to Trusty to be able to test Xenial’s Python version ¯\_(ツ)_/¯

* Add missing trusty

* Stop breaking old PyPy

* Allow installing Scrapy on Python 3.5.0
---
 .travis.yml                 | 10 +++++++---
 README.rst                  |  2 +-
 docs/faq.rst                |  2 +-
 docs/intro/install.rst      |  2 +-
 tests/test_proxy_connect.py |  5 ++++-
 5 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index bcbf75a43..d6ec88e06 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,12 +17,16 @@ matrix:
       python: 3.7  # Keep in sync with .readthedocs.yml
 
     - env: TOXENV=pypy3
+    - env: TOXENV=pinned
+      python: 3.5.1
+      dist: trusty
+    - env: TOXENV=asyncio
+      python: 3.5.1  # We use additional code to support 3.5.3 and earlier
+      dist: trusty
     - env: TOXENV=py
       python: 3.5
-    - env: TOXENV=pinned
-      python: 3.5
     - env: TOXENV=asyncio
-      python: 3.5.2
+      python: 3.5  # We use specific code to support >= 3.5.4, < 3.6
     - env: TOXENV=py
       python: 3.6
     - env: TOXENV=py
diff --git a/README.rst b/README.rst
index ce5973bcd..fd84e127e 100644
--- a/README.rst
+++ b/README.rst
@@ -40,7 +40,7 @@ including a list of features.
 Requirements
 ============
 
-* Python 3.5+
+* Python 3.5.1+
 * Works on Linux, Windows, macOS, BSD
 
 Install
diff --git a/docs/faq.rst b/docs/faq.rst
index 75a0f4864..936f315b3 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
 What Python versions does Scrapy support?
 -----------------------------------------
 
-Scrapy is supported under Python 3.5+
+Scrapy is supported under Python 3.5.1+
 under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
 Python 3 support was added in Scrapy 1.1.
 PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 6356e0eea..4af80d801 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -7,7 +7,7 @@ Installation guide
 Installing Scrapy
 =================
 
-Scrapy runs on Python 3.5 or above under CPython (default Python
+Scrapy runs on Python 3.5.1 or above under CPython (default Python
 implementation) and PyPy (starting with PyPy 5.9).
 
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 4763a5417..eb4ecc91d 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -4,6 +4,7 @@ import re
 import sys
 from subprocess import Popen, PIPE
 from urllib.parse import urlsplit, urlunsplit
+from unittest import skipIf
 
 import pytest
 from testfixtures import LogCapture
@@ -56,6 +57,8 @@ def _wrong_credentials(proxy_url):
     return urlunsplit(bad_auth_proxy)
 
 
+@skipIf(sys.version_info < (3, 5, 4),
+        "requires mitmproxy < 3.0.0, which these tests do not support")
 class ProxyConnectTestCase(TestCase):
 
     def setUp(self):
@@ -80,7 +83,7 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, log)
 
-    @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
+    @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info >= (3, 6))
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
         crawler = get_crawler(SimpleSpider)

From 4cdd00e21f4bfe22ba9b8fabe034a5e4d34dab75 Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Sat, 16 May 2020 00:25:57 +0530
Subject: [PATCH 2851/2974] Changed BaseRunSpiderCommands to
 BaseRunSpiderCommand

---
 scrapy/commands/__init__.py  | 2 +-
 scrapy/commands/crawl.py     | 4 ++--
 scrapy/commands/runspider.py | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 0ddbf2ca0..81fae089b 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -107,7 +107,7 @@ class ScrapyCommand:
         raise NotImplementedError
 
 
-class BaseRunSpiderCommands(ScrapyCommand):
+class BaseRunSpiderCommand(ScrapyCommand):
     """
     The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for
     performing common functionality between crawl.py and runspider.py
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index c39c5a9ef..e1724c1e6 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,8 +1,8 @@
-from scrapy.commands import BaseRunSpiderCommands
+from scrapy.commands import BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
 
 
-class Command(BaseRunSpiderCommands):
+class Command(BaseRunSpiderCommand):
 
     requires_project = True
 
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index bd24a369e..befee021b 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -4,7 +4,7 @@ from importlib import import_module
 
 from scrapy.utils.spider import iter_spider_classes
 from scrapy.exceptions import UsageError
-from scrapy.commands import BaseRunSpiderCommands
+from scrapy.commands import BaseRunSpiderCommand
 
 
 def _import_file(filepath):
@@ -23,7 +23,7 @@ def _import_file(filepath):
     return module
 
 
-class Command(BaseRunSpiderCommands):
+class Command(BaseRunSpiderCommand):
 
     requires_project = False
     default_settings = {'SPIDER_LOADER_WARN_ONLY': True}

From 10ae1a284f759b541d086e3d1a13cda96b6e2040 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 15 May 2020 22:50:54 +0300
Subject: [PATCH 2852/2974] Minor fixes

---
 docs/topics/feed-exports.rst    |  2 +-
 scrapy/extensions/feedexport.py | 10 +++++-----
 tests/test_feedexport.py        |  2 +-
 tox.ini                         |  2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 638733b6a..6f7db20c4 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -467,4 +467,4 @@ The command line above can generate a directory tree like::
 --->3-filename2020-03-28T14-45-10.046092.json
 
 Where the first and second files contain exactly 100 items. The last one contains
-100 items or fever.
+100 items or fewer.
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 3d691c580..cc26ae173 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -243,11 +243,11 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
+        self.storage_batch_item_count = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
-            if not self._batch_deliveries_supported(uri):
+            if not self._settings_are_valid(uri):
                 raise NotConfigured
             if not self._exporter_supported(feed['format']):
                 raise NotConfigured
@@ -328,7 +328,7 @@ class FeedExporter:
             slot.exporter.export_item(item)
             slot.itemcount += 1
             # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
-            if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
+            if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count:
                 uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
                 self._close_slot(slot, spider)
                 slots.append(self._start_new_batch(
@@ -357,12 +357,12 @@ class FeedExporter:
             return True
         logger.error("Unknown feed format: %(format)s", {'format': format})
 
-    def _batch_deliveries_supported(self, uri):
+    def _settings_are_valid(self, uri):
         """
         If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s
         to distinguish different files of partial output
         """
-        if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
+        if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
             return True
         logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
         return False
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index d1374f291..88f9a5933 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -986,7 +986,7 @@ class FeedExportTest(FeedExportTestBase):
         self.assertEqual(data['csv'], b'')
 
 
-class PartialDeliveriesTest(FeedExportTestBase):
+class BatchDeliveriesTest(FeedExportTestBase):
     __test__ = True
     _file_mark = '_%(batch_time)s_#%(batch_id)s_'
 
diff --git a/tox.ini b/tox.ini
index 6dd944dff..7507a14a6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,9 +12,9 @@ deps =
     -ctests/constraints.txt
     -rtests/requirements-py3.txt
     # Extras
+    boto3>=1.13.0
     botocore>=1.3.23
     Pillow>=3.4.2
-    boto3>=1.13.0
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID

From 604fe33bad36f1269677e98d0bfec1f60c95aa53 Mon Sep 17 00:00:00 2001
From: Jay Rajput <jay24rajput@gmail.com>
Date: Sat, 16 May 2020 01:53:49 +0530
Subject: [PATCH 2853/2974] Update scrapy/commands/__init__.py

Changed typo in a comment for BaseRunSpiderCommand

Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
---
 scrapy/commands/__init__.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 49152ca28..ab850dcb3 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -108,8 +108,7 @@ class ScrapyCommand:
 
 class BaseRunSpiderCommand(ScrapyCommand):
     """
-    The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for
-    performing common functionality between crawl.py and runspider.py
+    Common class used to share functionality between the crawl and runspider commands
     """
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)

From a915af2e4592e8a0367c44a73c95cee3f835887d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 11 May 2020 16:13:54 -0300
Subject: [PATCH 2854/2974] Remove monkeypatches module, E402 flake8 code

---
 pytest.ini                        |  4 +--
 scrapy/__init__.py                | 47 ++++++++++++++++---------------
 scrapy/_monkeypatches.py          | 11 --------
 scrapy/linkextractors/__init__.py |  2 +-
 scrapy/spiders/__init__.py        |  6 ++--
 5 files changed, 30 insertions(+), 40 deletions(-)
 delete mode 100644 scrapy/_monkeypatches.py

diff --git a/pytest.ini b/pytest.ini
index 0e289866c..0797e8ac8 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -27,18 +27,16 @@ flake8-ignore =
     # Exclude files that are meant to provide top-level imports
     # E402: Module level import not at top of file
     # F401: Module imported but unused
+    scrapy/__init__.py E402
     scrapy/core/downloader/handlers/http.py F401
     scrapy/http/__init__.py F401
     scrapy/linkextractors/__init__.py E402 F401
     scrapy/spiders/__init__.py E402 F401
 
     # Issues pending a review:
-    scrapy/__init__.py E402
     scrapy/selector/__init__.py F403
-    scrapy/spiders/__init__.py E402
     scrapy/utils/http.py F403
     scrapy/utils/markup.py F403
     scrapy/utils/multipart.py F403
     scrapy/utils/url.py F403 F405
     tests/test_loader.py E741
-    tests/test_webclient.py E402
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index fb8357f3c..e791deaa6 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -2,33 +2,11 @@
 Scrapy - a web crawling and web scraping framework written for Python
 """
 
-__all__ = ['__version__', 'version_info', 'twisted_version',
-           'Spider', 'Request', 'FormRequest', 'Selector', 'Item', 'Field']
-
-# Scrapy version
 import pkgutil
-__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
-version_info = tuple(int(v) if v.isdigit() else v
-                     for v in __version__.split('.'))
-del pkgutil
-
-# Check minimum required Python version
 import sys
-if sys.version_info < (3, 5):
-    print("Scrapy %s requires Python 3.5" % __version__)
-    sys.exit(1)
-
-# Ignore noisy twisted deprecation warnings
 import warnings
-warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
-del warnings
-
-# Apply monkey patches to fix issues in external libraries
-from scrapy import _monkeypatches
-del _monkeypatches
 
 from twisted import version as _txv
-twisted_version = (_txv.major, _txv.minor, _txv.micro)
 
 # Declare top-level shortcuts
 from scrapy.spiders import Spider
@@ -36,4 +14,29 @@ from scrapy.http import Request, FormRequest
 from scrapy.selector import Selector
 from scrapy.item import Item, Field
 
+
+__all__ = [
+    '__version__', 'version_info', 'twisted_version', 'Spider',
+    'Request', 'FormRequest', 'Selector', 'Item', 'Field',
+]
+
+
+# Scrapy and Twisted versions
+__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
+version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
+twisted_version = (_txv.major, _txv.minor, _txv.micro)
+
+
+# Check minimum required Python version
+if sys.version_info < (3, 5):
+    print("Scrapy %s requires Python 3.5" % __version__)
+    sys.exit(1)
+
+
+# Ignore noisy twisted deprecation warnings
+warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
+
+
+del pkgutil
 del sys
+del warnings
diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
deleted file mode 100644
index f74f89bda..000000000
--- a/scrapy/_monkeypatches.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import copyreg
-
-
-# Undo what Twisted's perspective broker adds to pickle register
-# to prevent bugs like Twisted#7989 while serializing requests
-import twisted.persisted.styles  # NOQA
-# Remove only entries with twisted serializers for non-twisted types.
-for k, v in frozenset(copyreg.dispatch_table.items()):
-    if not str(getattr(k, '__module__', '')).startswith('twisted') \
-            and str(getattr(v, '__module__', '')).startswith('twisted'):
-        copyreg.dispatch_table.pop(k)
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index ae019c70f..984a5c4e1 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -133,4 +133,4 @@ class FilteringLinkExtractor:
 
 
 # Top-level imports
-from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor  # noqa: F401
+from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index ba1c866f8..02f87f8f5 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -110,6 +110,6 @@ class Spider(object_ref):
 
 
 # Top-level imports
-from scrapy.spiders.crawl import CrawlSpider, Rule  # noqa: F401
-from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider  # noqa: F401
-from scrapy.spiders.sitemap import SitemapSpider  # noqa: F401
+from scrapy.spiders.crawl import CrawlSpider, Rule
+from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider
+from scrapy.spiders.sitemap import SitemapSpider

From a22f97052e9b14631a977600c1f59bd468c85601 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 15 May 2020 21:18:26 -0300
Subject: [PATCH 2855/2974] Cleanup import in selector module

---
 pytest.ini                  | 2 +-
 scrapy/selector/__init__.py | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 0797e8ac8..663c5cc78 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -31,10 +31,10 @@ flake8-ignore =
     scrapy/core/downloader/handlers/http.py F401
     scrapy/http/__init__.py F401
     scrapy/linkextractors/__init__.py E402 F401
+    scrapy/selector/__init__.py F401
     scrapy/spiders/__init__.py E402 F401
 
     # Issues pending a review:
-    scrapy/selector/__init__.py F403
     scrapy/utils/http.py F403
     scrapy/utils/markup.py F403
     scrapy/utils/multipart.py F403
diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py
index a9240c1f6..85c500d66 100644
--- a/scrapy/selector/__init__.py
+++ b/scrapy/selector/__init__.py
@@ -1,4 +1,6 @@
 """
 Selectors
 """
-from scrapy.selector.unified import *  # noqa: F401
+
+# top-level imports
+from scrapy.selector.unified import Selector, SelectorList

From bcc40c40771ad223471ab77ad47233c498312095 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 18 May 2020 22:12:31 +0500
Subject: [PATCH 2856/2974] better deprecation warning for
 Response.body_as_unicode() (#4579)

---
 scrapy/http/response/text.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 5614e6e55..0603b6653 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -65,7 +65,7 @@ class TextResponse(Response):
         """Return body as unicode"""
         warnings.warn('Response.body_as_unicode() is deprecated, '
                       'please use Response.text instead.',
-                      ScrapyDeprecationWarning)
+                      ScrapyDeprecationWarning, stacklevel=2)
         return self.text
 
     @property

From a7d070f3bb350cbe1f7b580350d5f491f59d47d8 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Mon, 18 May 2020 22:25:29 +0300
Subject: [PATCH 2857/2974] Change log level to error

---
 scrapy/extensions/feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index cc26ae173..ce7fc372d 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -364,7 +364,7 @@ class FeedExporter:
         """
         if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
             return True
-        logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
+        logger.error('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
         return False
 
     def _storage_supported(self, uri):

From afb2e5011fb9d9aa0ca568d376b0c4cef4193a9e Mon Sep 17 00:00:00 2001
From: nsirletti <n.sirletti@gmail.com>
Date: Sat, 16 May 2020 12:48:21 +0200
Subject: [PATCH 2858/2974] Allow doc to be downloadable on readthedocs.org

---
 .readthedocs.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.readthedocs.yml b/.readthedocs.yml
index 17eba34f3..e4d3f02cc 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -1,4 +1,5 @@
 version: 2
+formats: all
 sphinx:
   configuration: docs/conf.py
   fail_on_warning: true

From c2a0cca0fe6bc8342efa6034a78b9b8161aa2177 Mon Sep 17 00:00:00 2001
From: nsirletti <n.sirletti@gmail.com>
Date: Sat, 16 May 2020 15:07:16 +0200
Subject: [PATCH 2859/2974] Suppress Sphinx warnings about MIME types other
 than .rst

---
 docs/conf.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/conf.py b/docs/conf.py
index 8ab38a090..468c1d190 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -100,6 +100,9 @@ exclude_trees = ['.build']
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
 
+# List of Sphinx warnings that will not be raised
+suppress_warnings = ['epub.unknown_project_files']
+
 
 # Options for HTML output
 # -----------------------

From f418c6d5f7307c3beae57dfc519be04657a8e1b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 19 May 2020 20:04:47 +0200
Subject: [PATCH 2860/2974] logging.rst: remove unused, misleading import

---
 docs/topics/logging.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 675e65ef1..e81091651 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -262,7 +262,6 @@ scrapy.utils.log module
     This is an example on how to redirect ``INFO`` or higher messages to a file::
 
         import logging
-        from scrapy.utils.log import configure_logging
 
         logging.basicConfig(
             filename='log.txt',

From eb8493f09c56dea4dad129882fbf7b4a4da08e1a Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Wed, 20 May 2020 09:21:34 -0400
Subject: [PATCH 2861/2974] Incompatible with pytest 5.4 and 5.4.1

---
 tests/requirements-py3.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index d207c5fb0..05b794d81 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -2,7 +2,7 @@
 jmespath
 mitmproxy; python_version >= '3.6'
 mitmproxy<4.0.0; python_version < '3.6'
-pytest < 5.4
+pytest != 5.4, != 5.4.1
 pytest-cov
 pytest-twisted >= 1.11
 pytest-xdist

From 276721a5dc96a4239645db1a78fe056df671e10e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 9 May 2020 22:02:40 -0300
Subject: [PATCH 2862/2974] Stop response download

---
 docs/faq.rst                              | 13 +++++
 docs/topics/exceptions.rst                | 46 +++++++++++++++---
 docs/topics/request-response.rst          | 49 ++++++++++++++++++-
 docs/topics/signals.rst                   | 11 ++++-
 scrapy/core/downloader/handlers/http11.py | 59 +++++++++++++----------
 scrapy/core/scraper.py                    | 10 +++-
 scrapy/exceptions.py                      | 13 +++++
 scrapy/utils/signal.py                    |  7 +--
 tests/spiders.py                          | 33 +++++++++++++
 tests/test_crawl.py                       | 40 +++++++++++++--
 tests/test_engine.py                      | 57 ++++++++++++++++++++--
 11 files changed, 291 insertions(+), 47 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 0b6bd6a86..c06cb945b 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -371,6 +371,19 @@ Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switch
 different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
 
 
+.. _faq-stop-response-download:
+
+How can I cancel the download of a given response?
+--------------------------------------------------
+
+In some situations, it might be useful to stop the download of a certain response.
+For instance, if you only need the first part of a large response and you would like
+to save resources by avoiding the download of the whole body.
+In that case, you could attach a handler to the :class:`~scrapy.signals.bytes_received`
+signal and raise a :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to
+the :ref:`topics-stop-response-download` topic for additional information and examples.
+
+
 .. _has been reported: https://github.com/scrapy/scrapy/issues/2905
 .. _user agents: https://en.wikipedia.org/wiki/User_agent
 .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index 09cb8ed66..10b23edc4 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -14,13 +14,6 @@ Built-in Exceptions reference
 
 Here's a list of all exceptions included in Scrapy and their usage.
 
-DropItem
---------
-
-.. exception:: DropItem
-
-The exception that must be raised by item pipeline stages to stop processing an
-Item. For more information see :ref:`topics-item-pipeline`.
 
 CloseSpider
 -----------
@@ -47,6 +40,14 @@ DontCloseSpider
 This exception can be raised in a :signal:`spider_idle` signal handler to
 prevent the spider from being closed.
 
+DropItem
+--------
+
+.. exception:: DropItem
+
+The exception that must be raised by item pipeline stages to stop processing an
+Item. For more information see :ref:`topics-item-pipeline`.
+
 IgnoreRequest
 -------------
 
@@ -77,3 +78,34 @@ NotSupported
 
 This exception is raised to indicate an unsupported feature.
 
+StopDownload
+-------------
+
+.. versionadded:: 2.2
+
+.. exception:: StopDownload(fail=True)
+
+Raised from a :class:`~scrapy.signals.bytes_received` signal handler to
+indicate that no further bytes should be downloaded for a response.
+
+The ``fail`` boolean parameter controls which method will handle the resulting
+response:
+
+* If ``fail=True``, the request errback is called. The response object is
+  available as the ``response`` attribute of the received
+  :class:`~twisted.python.failure.Failure` object. This is the default behaviour.
+
+* If ``fail=False``, the request callback is called instead.
+
+In both cases, the response could have its body truncated: the body contains
+all bytes received up until the exception is raised, including the bytes
+received in the signal handler that raises the exception. Also, the response
+object is marked with ``"download_stopped"`` in its :attr:`Response.flags`
+attribute.
+
+.. note:: ``fail`` is a keyword-only parameter, i.e. raising
+    ``StopDownload(False)`` or ``StopDownload(True)`` will raise
+    a :class:`TypeError`.
+
+See the documentation for the :class:`~scrapy.signals.bytes_received` signal
+and the :ref:`topics-stop-response-download` topic for additional information and examples.
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 15a83f453..ae25ff7e4 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -385,6 +385,51 @@ The meta key is used set retry times per request. When initialized, the
 :reqmeta:`max_retry_times` meta key takes higher precedence over the
 :setting:`RETRY_TIMES` setting.
 
+
+.. _topics-stop-response-download:
+
+Stopping the download of a Response
+===================================
+
+Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a
+:class:`~scrapy.signals.bytes_received` signal handler will stop the
+download of a given response. See the following example::
+
+    import scrapy
+
+
+    class StopSpider(scrapy.Spider):
+        name = "stop"
+        start_urls = ["https://docs.scrapy.org/en/latest/"]
+
+        @classmethod
+        def from_crawler(cls, crawler):
+            spider = super().from_crawler(crawler)
+            crawler.signals.connect(spider.on_bytes_received, signal=scrapy.signals.bytes_received)
+            return spider
+
+        def parse(self, response):
+            # 'last_chars' show that the full response was not downloaded
+            yield {"len": len(response.text), "last_chars": response.text[-40:]}
+
+        def on_bytes_received(self, data, request, spider):
+            raise scrapy.exceptions.StopDownload(fail=False)
+
+which produces the following output::
+
+    2020-05-19 17:26:12 [scrapy.core.engine] INFO: Spider opened
+    2020-05-19 17:26:12 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
+    2020-05-19 17:26:13 [scrapy.core.downloader.handlers.http11] DEBUG: Download stopped for <GET https://docs.scrapy.org/en/latest/> from signal handler StopSpider.on_bytes_received
+    2020-05-19 17:26:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://docs.scrapy.org/en/latest/> (referer: None) ['download_stopped']
+    2020-05-19 17:26:13 [scrapy.core.scraper] DEBUG: Scraped from <200 https://docs.scrapy.org/en/latest/>
+    {'len': 279, 'last_chars': 'dth, initial-scale=1.0">\n  \n  <title>Scr'}
+    2020-05-19 17:26:13 [scrapy.core.engine] INFO: Closing spider (finished)
+
+By default, resulting responses are handled by their corresponding errbacks. To
+call their callback instead, like in this example, pass ``fail=False`` to the
+:exc:`~scrapy.exceptions.StopDownload` exception.
+
+
 .. _topics-request-response-ref-request-subclasses:
 
 Request subclasses
@@ -716,9 +761,9 @@ Response objects
         .. versionadded:: 2.1.0
 
         The IP address of the server from which the Response originated.
-        
+
         This attribute is currently only populated by the HTTP 1.1 download
-        handler, i.e. for ``http(s)`` responses. For other handlers, 
+        handler, i.e. for ``http(s)`` responses. For other handlers,
         :attr:`ip_address` is always ``None``.
 
     .. method:: Response.copy()
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 7fe63a7b0..fe4fb0834 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -373,6 +373,8 @@ request_left_downloader
 bytes_received
 ~~~~~~~~~~~~~~
 
+.. versionadded:: 2.2
+
 .. signal:: bytes_received
 .. function:: bytes_received(data, request, spider)
 
@@ -385,14 +387,19 @@ bytes_received
     This signal does not support returning deferreds from its handlers.
 
     :param data: the data received by the download handler
-    :type spider: :class:`bytes` object
+    :type data: :class:`bytes` object
 
-    :param request: the request that generated the response
+    :param request: the request that generated the download
     :type request: :class:`~scrapy.http.Request` object
 
     :param spider: the spider associated with the response
     :type spider: :class:`~scrapy.spiders.Spider` object
 
+.. note:: Handlers of this signal can stop the download of a response while it
+    is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
+    exception. Please refer to the :ref:`topics-stop-response-download` topic
+    for additional information and examples.
+
 Response signals
 ----------------
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index c21491f52..555a3a032 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -12,6 +12,7 @@ from urllib.parse import urldefrag
 from twisted.internet import defer, protocol, ssl
 from twisted.internet.endpoints import TCP4ClientEndpoint
 from twisted.internet.error import TimeoutError
+from twisted.python.failure import Failure
 from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
 from twisted.web.http import _DataLoss, PotentialDataLoss
 from twisted.web.http_headers import Headers as TxHeaders
@@ -21,7 +22,7 @@ from zope.interface import implementer
 from scrapy import signals
 from scrapy.core.downloader.tls import openssl_methods
 from scrapy.core.downloader.webclient import _parse
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
 from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.misc import create_instance, load_object
@@ -431,7 +432,7 @@ class ScrapyAgent:
     def _cb_bodydone(self, result, request, url):
         headers = Headers(result["txresponse"].headers.getAllRawHeaders())
         respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
-        return respcls(
+        response = respcls(
             url=url,
             status=int(result["txresponse"].code),
             headers=headers,
@@ -440,6 +441,14 @@ class ScrapyAgent:
             certificate=result["certificate"],
             ip_address=result["ip_address"],
         )
+        if result.get("failure"):
+            # This failure is not the same object that will reach the errback,
+            # so we need to temporarily store the response in the exception.
+            # It will be moved to the failure in core/scraper.py
+            failure = result["failure"]
+            failure.value.response = response
+            return failure
+        return response
 
 
 @implementer(IBodyProducer)
@@ -477,6 +486,16 @@ class _ResponseReader(protocol.Protocol):
         self._ip_address = None
         self._crawler = crawler
 
+    def _finish_response(self, flags=None, failure=None):
+        self._finished.callback({
+            "txresponse": self._txresponse,
+            "body": self._bodybuf.getvalue(),
+            "flags": flags,
+            "certificate": self._certificate,
+            "ip_address": self._ip_address,
+            "failure": failure,
+        })
+
     def connectionMade(self):
         if self._certificate is None:
             with suppress(AttributeError):
@@ -493,12 +512,21 @@ class _ResponseReader(protocol.Protocol):
         self._bodybuf.write(bodyBytes)
         self._bytes_received += len(bodyBytes)
 
-        self._crawler.signals.send_catch_log(
+        bytes_received_result = self._crawler.signals.send_catch_log(
             signal=signals.bytes_received,
             data=bodyBytes,
             request=self._request,
             spider=self._crawler.spider,
         )
+        for handler, result in bytes_received_result:
+            if isinstance(result, Failure) and isinstance(result.value, StopDownload):
+                logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
+                             {"request": self._request, "handler": handler.__qualname__})
+                self.transport._producer.loseConnection()
+                self._finish_response(
+                    flags=["download_stopped"],
+                    failure=result if result.value.fail else None,
+                )
 
         if self._maxsize and self._bytes_received > self._maxsize:
             logger.error("Received (%(bytes)s) bytes larger than download "
@@ -521,36 +549,17 @@ class _ResponseReader(protocol.Protocol):
         if self._finished.called:
             return
 
-        body = self._bodybuf.getvalue()
         if reason.check(ResponseDone):
-            self._finished.callback({
-                "txresponse": self._txresponse,
-                "body": body,
-                "flags": None,
-                "certificate": self._certificate,
-                "ip_address": self._ip_address,
-            })
+            self._finish_response()
             return
 
         if reason.check(PotentialDataLoss):
-            self._finished.callback({
-                "txresponse": self._txresponse,
-                "body": body,
-                "flags": ["partial"],
-                "certificate": self._certificate,
-                "ip_address": self._ip_address,
-            })
+            self._finish_response(flags=["partial"])
             return
 
         if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
             if not self._fail_on_dataloss:
-                self._finished.callback({
-                    "txresponse": self._txresponse,
-                    "body": body,
-                    "flags": ["dataloss"],
-                    "certificate": self._certificate,
-                    "ip_address": self._ip_address,
-                })
+                self._finish_response(flags=["dataloss"])
                 return
 
             elif not self._fail_on_dataloss_warned:
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 6785e103d..c52ada83b 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -11,7 +11,7 @@ from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errba
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
 from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
-from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
+from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest, StopDownload
 from scrapy import signals
 from scrapy.http import Request, Response
 from scrapy.item import _BaseItem
@@ -147,6 +147,14 @@ class Scraper:
 
     def call_spider(self, result, request, spider):
         result.request = request
+        # StopDownload exceptions: make the partial response an attribute of the failure
+        if (
+            isinstance(result, Failure)
+            and isinstance(result.value, StopDownload)
+            and hasattr(result.value, "response")
+        ):
+            result.response = result.value.response
+            delattr(result.value, "response")
         dfd = defer_result(result)
         callback = request.callback or spider.parse
         warn_on_generator_with_return_value(spider, callback)
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 7c4bb3d00..45f152321 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -41,6 +41,18 @@ class CloseSpider(Exception):
         self.reason = reason
 
 
+class StopDownload(Exception):
+    """
+    Stop the download of the body for a given response.
+    The 'fail' boolean parameter indicates whether or not the resulting partial response
+    should be handled by the request errback. Note that 'fail' is a keyword-only argument.
+    """
+
+    def __init__(self, *, fail=True):
+        super().__init__()
+        self.fail = fail
+
+
 # Items
 
 
@@ -59,6 +71,7 @@ class NotSupported(Exception):
 
 class UsageError(Exception):
     """To indicate a command-line usage error"""
+
     def __init__(self, *a, **kw):
         self.print_help = kw.pop('print_help', True)
         super(UsageError, self).__init__(*a, **kw)
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index a311e9257..115707182 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -5,13 +5,14 @@ import logging
 from twisted.internet.defer import DeferredList, Deferred
 from twisted.python.failure import Failure
 
-from pydispatch.dispatcher import Any, Anonymous, liveReceivers, \
-    getAllReceivers, disconnect
+from pydispatch.dispatcher import Anonymous, Any, disconnect, getAllReceivers, liveReceivers
 from pydispatch.robustapply import robustApply
 
+from scrapy.exceptions import StopDownload
 from scrapy.utils.defer import maybeDeferred_coro
 from scrapy.utils.log import failure_to_exc_info
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -23,7 +24,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
     """Like pydispatcher.robust.sendRobust but it also logs errors and returns
     Failures instead of exceptions.
     """
-    dont_log = named.pop('dont_log', _IgnoredException)
+    dont_log = (named.pop('dont_log', _IgnoredException), StopDownload)
     spider = named.get('spider', None)
     responses = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
diff --git a/tests/spiders.py b/tests/spiders.py
index 33d5d02e1..ebe81a1a9 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -7,6 +7,8 @@ from urllib.parse import urlencode
 
 from twisted.internet import defer
 
+from scrapy import signals
+from scrapy.exceptions import StopDownload
 from scrapy.http import Request
 from scrapy.item import Item
 from scrapy.linkextractors import LinkExtractor
@@ -267,3 +269,34 @@ class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider):
 
     def errback(self, failure):
         self.logger.info('[errback] status %i', failure.value.response.status)
+
+
+class BytesReceivedCallbackSpider(MetaSpider):
+
+    full_response_length = 2**18
+
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        spider = super().from_crawler(crawler, *args, **kwargs)
+        crawler.signals.connect(spider.bytes_received, signals.bytes_received)
+        return spider
+
+    def start_requests(self):
+        body = b"a" * self.full_response_length
+        url = self.mockserver.url("/alpayload")
+        yield Request(url, method="POST", body=body, errback=self.errback)
+
+    def parse(self, response):
+        self.meta["response"] = response
+
+    def errback(self, failure):
+        self.meta["failure"] = failure
+
+    def bytes_received(self, data, request, spider):
+        raise StopDownload(fail=False)
+
+
+class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):
+
+    def bytes_received(self, data, request, spider):
+        raise StopDownload(fail=True)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 84f80d103..5e70064c2 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -9,17 +9,31 @@ from pytest import mark
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.internet.ssl import Certificate
+from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase
 
 from scrapy import signals
 from scrapy.crawler import CrawlerRunner
+from scrapy.exceptions import StopDownload
 from scrapy.http import Request
+from scrapy.http.response import Response
 from scrapy.utils.python import to_unicode
 from tests.mockserver import MockServer
-from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
-                           SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback,
-                           AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider,
-                           AsyncDefAsyncioReqsReturnSpider)
+from tests.spiders import (
+    AsyncDefAsyncioReqsReturnSpider,
+    AsyncDefAsyncioReturnSpider,
+    AsyncDefAsyncioSpider,
+    AsyncDefSpider,
+    BrokenStartRequestsSpider,
+    BytesReceivedCallbackSpider,
+    BytesReceivedErrbackSpider,
+    CrawlSpiderWithErrback,
+    DelaySpider,
+    DuplicateStartRequestsSpider,
+    FollowAllSpider,
+    SimpleSpider,
+    SingleRequestSpider,
+)
 
 
 class CrawlTestCase(TestCase):
@@ -457,3 +471,21 @@ with multiples lines
         ip_address = crawler.spider.meta['responses'][0].ip_address
         self.assertIsInstance(ip_address, IPv4Address)
         self.assertEqual(str(ip_address), gethostbyname(expected_netloc))
+
+    @defer.inlineCallbacks
+    def test_stop_download_callback(self):
+        crawler = self.runner.create_crawler(BytesReceivedCallbackSpider)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertIsNone(crawler.spider.meta.get("failure"))
+        self.assertIsInstance(crawler.spider.meta["response"], Response)
+        self.assertLess(len(crawler.spider.meta["response"].text), crawler.spider.full_response_length)
+
+    @defer.inlineCallbacks
+    def test_stop_download_errback(self):
+        crawler = self.runner.create_crawler(BytesReceivedErrbackSpider)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertIsNone(crawler.spider.meta.get("response"))
+        self.assertIsInstance(crawler.spider.meta["failure"], Failure)
+        self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload)
+        self.assertIsInstance(crawler.spider.meta["failure"].response, Response)
+        self.assertLess(len(crawler.spider.meta["failure"].response.text), crawler.spider.full_response_length)
diff --git a/tests/test_engine.py b/tests/test_engine.py
index d781665dc..6696ee52e 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -16,13 +16,15 @@ import sys
 from collections import defaultdict
 from urllib.parse import urlparse
 
+from pydispatch import dispatcher
+from testfixtures import LogCapture
 from twisted.internet import reactor, defer
 from twisted.trial import unittest
 from twisted.web import server, static, util
-from pydispatch import dispatcher
 
 from scrapy import signals
 from scrapy.core.engine import ExecutionEngine
+from scrapy.exceptions import StopDownload
 from scrapy.http import Request
 from scrapy.item import Item, Field
 from scrapy.linkextractors import LinkExtractor
@@ -90,7 +92,7 @@ def start_test_site(debug=False):
     r = static.File(root_dir)
     r.putChild(b"redirect", util.Redirect(b"/redirected"))
     r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
-    numbers = [str(x).encode("utf8") for x in range(2**14)]
+    numbers = [str(x).encode("utf8") for x in range(2**18)]
     r.putChild(b"numbers", static.Data(b"".join(numbers), "text/plain"))
 
     port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1")
@@ -188,6 +190,16 @@ class CrawlerRun:
         self.signals_caught[sig] = signalargs
 
 
+class StopDownloadCrawlerRun(CrawlerRun):
+    """
+    Make sure raising the StopDownload exception stops the download of the response body
+    """
+
+    def bytes_received(self, data, request, spider):
+        super().bytes_received(data, request, spider)
+        raise StopDownload(fail=False)
+
+
 class EngineTest(unittest.TestCase):
 
     @defer.inlineCallbacks
@@ -316,7 +328,7 @@ class EngineTest(unittest.TestCase):
                 # signal was fired multiple times
                 self.assertTrue(len(data) > 1)
                 # bytes were received in order
-                numbers = [str(x).encode("utf8") for x in range(2**14)]
+                numbers = [str(x).encode("utf8") for x in range(2**18)]
                 self.assertEqual(joined_data, b"".join(numbers))
 
     def _assert_signals_caught(self):
@@ -357,6 +369,45 @@ class EngineTest(unittest.TestCase):
         self.assertEqual(len(e.open_spiders), 0)
 
 
+class StopDownloadEngineTest(EngineTest):
+
+    @defer.inlineCallbacks
+    def test_crawler(self):
+        for spider in TestSpider, DictItemsSpider:
+            self.run = StopDownloadCrawlerRun(spider)
+            with LogCapture() as log:
+                yield self.run.run()
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   "Download stopped for <GET http://localhost:{}/redirected> from signal handler"
+                                   " StopDownloadCrawlerRun.bytes_received".format(self.run.portno)))
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   "Download stopped for <GET http://localhost:{}/> from signal handler"
+                                   " StopDownloadCrawlerRun.bytes_received".format(self.run.portno)))
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   "Download stopped for <GET http://localhost:{}/numbers> from signal handler"
+                                   " StopDownloadCrawlerRun.bytes_received".format(self.run.portno)))
+            self._assert_visited_urls()
+            self._assert_scheduled_requests(urls_to_visit=9)
+            self._assert_downloaded_responses()
+            self._assert_signals_caught()
+            self._assert_bytes_received()
+
+    def _assert_bytes_received(self):
+        self.assertEqual(9, len(self.run.bytes))
+        for request, data in self.run.bytes.items():
+            joined_data = b"".join(data)
+            self.assertTrue(len(data) == 1)  # signal was fired only once
+            if self.run.getpath(request.url) == "/numbers":
+                # Received bytes are not the complete response. The exact amount depends
+                # on the buffer size, which can vary, so we only check that the amount
+                # of received bytes is strictly less than the full response.
+                numbers = [str(x).encode("utf8") for x in range(2**18)]
+                self.assertTrue(len(joined_data) < len(b"".join(numbers)))
+
+
 if __name__ == "__main__":
     if len(sys.argv) > 1 and sys.argv[1] == 'runserver':
         start_test_site(debug=True)

From 9514393bd1c9137c6a3b9964679de04bccbd0570 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 20 May 2020 15:28:54 -0300
Subject: [PATCH 2863/2974] Reduce amount of lines

---
 scrapy/core/downloader/handlers/http11.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 555a3a032..a710761c5 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -523,10 +523,8 @@ class _ResponseReader(protocol.Protocol):
                 logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
                              {"request": self._request, "handler": handler.__qualname__})
                 self.transport._producer.loseConnection()
-                self._finish_response(
-                    flags=["download_stopped"],
-                    failure=result if result.value.fail else None,
-                )
+                failure = result if result.value.fail else None
+                self._finish_response(flags=["download_stopped"], failure=failure)
 
         if self._maxsize and self._bytes_received > self._maxsize:
             logger.error("Received (%(bytes)s) bytes larger than download "

From fbe915d9cbd8ab801a679b05265cff416764d558 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 20 May 2020 20:20:30 -0300
Subject: [PATCH 2864/2974] StopDownload: check partial response contents

---
 tests/spiders.py    | 2 ++
 tests/test_crawl.py | 6 ++++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index ebe81a1a9..05078cc04 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -293,10 +293,12 @@ class BytesReceivedCallbackSpider(MetaSpider):
         self.meta["failure"] = failure
 
     def bytes_received(self, data, request, spider):
+        self.meta["bytes_received"] = data
         raise StopDownload(fail=False)
 
 
 class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):
 
     def bytes_received(self, data, request, spider):
+        self.meta["bytes_received"] = data
         raise StopDownload(fail=True)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 5e70064c2..e2426e0da 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -478,7 +478,8 @@ with multiples lines
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("failure"))
         self.assertIsInstance(crawler.spider.meta["response"], Response)
-        self.assertLess(len(crawler.spider.meta["response"].text), crawler.spider.full_response_length)
+        self.assertEqual(crawler.spider.meta["response"].body, crawler.spider.meta.get("bytes_received"))
+        self.assertLess(len(crawler.spider.meta["response"].body), crawler.spider.full_response_length)
 
     @defer.inlineCallbacks
     def test_stop_download_errback(self):
@@ -488,4 +489,5 @@ with multiples lines
         self.assertIsInstance(crawler.spider.meta["failure"], Failure)
         self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload)
         self.assertIsInstance(crawler.spider.meta["failure"].response, Response)
-        self.assertLess(len(crawler.spider.meta["failure"].response.text), crawler.spider.full_response_length)
+        self.assertEqual(crawler.spider.meta["failure"].response.body, crawler.spider.meta.get("bytes_received"))
+        self.assertLess(len(crawler.spider.meta["failure"].response.body), crawler.spider.full_response_length)

From 677e619d3761e6669c247786bb95822ce38c8080 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 21 May 2020 14:57:03 +0300
Subject: [PATCH 2865/2974] Fix too long lines

---
 scrapy/extensions/feedexport.py |  4 +++-
 tests/test_feedexport.py        | 20 ++++++++++++++------
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index ce7fc372d..1f745be98 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -364,7 +364,9 @@ class FeedExporter:
         """
         if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
             return True
-        logger.error('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')
+        logger.error(
+            '%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified'
+        )
         return False
 
     def _storage_supported(self, uri):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 08ee24768..fecb17e29 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1204,12 +1204,20 @@ class BatchDeliveriesTest(FeedExportTestBase):
         items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
 
         formats = {
-            'json': [u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
-                     u'[\n{"bar": "BAR1"}\n]'.encode('utf-8')],
-            'xml': [u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'.encode('latin-1'),
-                    u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>'.encode('latin-1')],
-            'csv': [u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
-                    u'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],
+            'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'),
+                     '[\n{"bar": "BAR1"}\n]'.encode('utf-8')],
+            'xml': [
+                (
+                    '<?xml version="1.0" encoding="latin-1"?>\n'
+                    '<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>'
+                ).encode('latin-1'),
+                (
+                    '<?xml version="1.0" encoding="latin-1"?>\n'
+                    '<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>'
+                ).encode('latin-1')
+            ],
+            'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
+                    'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],
         }
 
         settings = {

From f6ed5edc31e7cc66225c0860e1534a6230511954 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 18 Nov 2016 09:14:54 -0300
Subject: [PATCH 2866/2974] CookiesMiddleware: keep cookies from 'Cookie'
 request header

---
 docs/topics/downloader-middleware.rst      |   5 +
 docs/topics/logging.rst                    |   3 +
 scrapy/downloadermiddlewares/cookies.py    |  78 ++++++++---
 tests/test_downloadermiddleware_cookies.py | 145 ++++++++++++++++++---
 4 files changed, 190 insertions(+), 41 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 1a87d07b6..323e553e5 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -202,6 +202,11 @@ CookiesMiddleware
    sends them back on subsequent requests (from that spider), just like web
    browsers do.
 
+   .. caution:: When non-UTF8 encoded byte sequences are passed to a
+      :class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log
+      a warning. Refer to :ref:`topics-logging-advanced-customization`
+      to customize the logging behaviour.
+
 The following settings can be used to configure the cookie middleware:
 
 * :setting:`COOKIES_ENABLED`
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index e81091651..55065a1a3 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -202,6 +202,9 @@ A custom log format can be set for different actions by extending
 .. autoclass:: scrapy.logformatter.LogFormatter
    :members:
 
+
+.. _topics-logging-advanced-customization:
+
 Advanced customization
 ----------------------
 
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index d57f04bc3..77048f389 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -29,8 +29,7 @@ class CookiesMiddleware:
 
         cookiejarkey = request.meta.get("cookiejar")
         jar = self.jars[cookiejarkey]
-        cookies = self._get_request_cookies(jar, request)
-        for cookie in cookies:
+        for cookie in self._get_request_cookies(jar, request):
             jar.set_cookie_if_ok(cookie, request)
 
         # set Cookie header
@@ -68,28 +67,65 @@ class CookiesMiddleware:
                 msg = "Received cookies from: {}\n{}".format(response, cookies)
                 logger.debug(msg, extra={'spider': spider})
 
-    def _format_cookie(self, cookie):
-        # build cookie string
-        cookie_str = '%s=%s' % (cookie['name'], cookie['value'])
-
-        if cookie.get('path', None):
-            cookie_str += '; Path=%s' % cookie['path']
-        if cookie.get('domain', None):
-            cookie_str += '; Domain=%s' % cookie['domain']
+    def _format_cookie(self, cookie, request):
+        """
+        Given a dict consisting of cookie components, return its string representation.
+        Decode from bytes if necessary.
+        """
+        decoded = {}
+        for key in ("name", "value", "path", "domain"):
+            if not cookie.get(key):
+                if key in ("name", "value"):
+                    msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
+                    logger.warning(msg.format(request, cookie, key))
+                    return
+                continue
+            if isinstance(cookie[key], str):
+                decoded[key] = cookie[key]
+            else:
+                try:
+                    decoded[key] = cookie[key].decode("utf8")
+                except UnicodeDecodeError:
+                    logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
+                                   request, cookie)
+                    decoded[key] = cookie[key].decode("latin1", errors="replace")
 
+        cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value"))
+        for key, value in decoded.items():  # path, domain
+            cookie_str += "; {}={}".format(key.capitalize(), value)
         return cookie_str
 
     def _get_request_cookies(self, jar, request):
-        if isinstance(request.cookies, dict):
-            cookie_list = [
-                {'name': k, 'value': v}
-                for k, v in request.cookies.items()
-            ]
-        else:
-            cookie_list = request.cookies
+        """
+        Extract cookies from a Request. Values from the `Request.cookies` attribute
+        take precedence over values from the `Cookie` request header.
+        """
+        def get_cookies_from_header(jar, request):
+            cookie_header = request.headers.get("Cookie")
+            if not cookie_header:
+                return []
+            cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";"))
+            cookie_list_unicode = []
+            for cookie_bytes in cookie_gen_bytes:
+                try:
+                    cookie_unicode = cookie_bytes.decode("utf8")
+                except UnicodeDecodeError:
+                    logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
+                                   request, cookie_bytes)
+                    cookie_unicode = cookie_bytes.decode("latin1", errors="replace")
+                cookie_list_unicode.append(cookie_unicode)
+            response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode})
+            return jar.make_cookies(response, request)
 
-        cookies = [self._format_cookie(x) for x in cookie_list]
-        headers = {'Set-Cookie': cookies}
-        response = Response(request.url, headers=headers)
+        def get_cookies_from_attribute(jar, request):
+            if not request.cookies:
+                return []
+            elif isinstance(request.cookies, dict):
+                cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
+            else:
+                cookies = request.cookies
+            formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
+            response = Response(request.url, headers={"Set-Cookie": formatted})
+            return jar.make_cookies(response, request)
 
-        return jar.make_cookies(response, request)
+        return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request)
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index d54434c8f..9ccc2110b 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -1,20 +1,21 @@
-import re
 import logging
-from unittest import TestCase
 from testfixtures import LogCapture
+from unittest import TestCase
 
+from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
+from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
+from scrapy.exceptions import NotConfigured
 from scrapy.http import Response, Request
 from scrapy.spiders import Spider
+from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler
-from scrapy.exceptions import NotConfigured
-from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
 
 
 class CookiesMiddlewareTest(TestCase):
 
     def assertCookieValEqual(self, first, second, msg=None):
         def split_cookies(cookies):
-            return sorted(re.split(r";\s*", cookies.decode("latin1")))
+            return sorted([s.strip() for s in to_bytes(cookies).split(b";")])
         return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg)
 
     def setUp(self):
@@ -61,12 +62,13 @@ class CookiesMiddlewareTest(TestCase):
     def test_setting_enabled_cookies_debug(self):
         crawler = get_crawler(settings_dict={'COOKIES_DEBUG': True})
         mw = CookiesMiddleware.from_crawler(crawler)
-        with LogCapture('scrapy.downloadermiddlewares.cookies',
-                        propagate=False,
-                        level=logging.DEBUG) as log:
+        with LogCapture(
+            'scrapy.downloadermiddlewares.cookies',
+            propagate=False,
+            level=logging.DEBUG,
+        ) as log:
             req = Request('http://scrapytest.org/')
-            res = Response('http://scrapytest.org/',
-                           headers={'Set-Cookie': 'C1=value1; path=/'})
+            res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'})
             mw.process_response(req, res, crawler.spider)
             req2 = Request('http://scrapytest.org/sub1/')
             mw.process_request(req2, crawler.spider)
@@ -85,12 +87,13 @@ class CookiesMiddlewareTest(TestCase):
     def test_setting_disabled_cookies_debug(self):
         crawler = get_crawler(settings_dict={'COOKIES_DEBUG': False})
         mw = CookiesMiddleware.from_crawler(crawler)
-        with LogCapture('scrapy.downloadermiddlewares.cookies',
-                        propagate=False,
-                        level=logging.DEBUG) as log:
+        with LogCapture(
+            'scrapy.downloadermiddlewares.cookies',
+            propagate=False,
+            level=logging.DEBUG,
+        ) as log:
             req = Request('http://scrapytest.org/')
-            res = Response('http://scrapytest.org/',
-                           headers={'Set-Cookie': 'C1=value1; path=/'})
+            res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'})
             mw.process_response(req, res, crawler.spider)
             req2 = Request('http://scrapytest.org/sub1/')
             mw.process_request(req2, crawler.spider)
@@ -102,8 +105,7 @@ class CookiesMiddlewareTest(TestCase):
         assert self.mw.process_request(req, self.spider) is None
         assert 'Cookie' not in req.headers
 
-        headers = {'Set-Cookie': b'C1=in\xa3valid; path=/',
-                   'Other': b'ignore\xa3me'}
+        headers = {'Set-Cookie': b'C1=in\xa3valid; path=/', 'Other': b'ignore\xa3me'}
         res = Response('http://scrapytest.org/', headers=headers)
         assert self.mw.process_response(req, res, self.spider) is res
 
@@ -124,7 +126,10 @@ class CookiesMiddlewareTest(TestCase):
         assert 'Cookie' not in req.headers
 
         # check that returned cookies are not merged back to jar
-        res = Response('http://scrapytest.org/dontmerge', headers={'Set-Cookie': 'dont=mergeme; path=/'})
+        res = Response(
+            'http://scrapytest.org/dontmerge',
+            headers={'Set-Cookie': 'dont=mergeme; path=/'},
+        )
         assert self.mw.process_response(req, res, self.spider) is res
 
         # check that cookies are merged back
@@ -179,7 +184,11 @@ class CookiesMiddlewareTest(TestCase):
         self.assertCookieValEqual(req2.headers.get('Cookie'), b"C1=value1; galleta=salada")
 
     def test_cookiejar_key(self):
-        req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}, meta={'cookiejar': "store1"})
+        req = Request(
+            'http://scrapytest.org/',
+            cookies={'galleta': 'salada'},
+            meta={'cookiejar': "store1"},
+        )
         assert self.mw.process_request(req, self.spider) is None
         self.assertEqual(req.headers.get('Cookie'), b'galleta=salada')
 
@@ -191,7 +200,11 @@ class CookiesMiddlewareTest(TestCase):
         assert self.mw.process_request(req2, self.spider) is None
         self.assertCookieValEqual(req2.headers.get('Cookie'), b'C1=value1; galleta=salada')
 
-        req3 = Request('http://scrapytest.org/', cookies={'galleta': 'dulce'}, meta={'cookiejar': "store2"})
+        req3 = Request(
+            'http://scrapytest.org/',
+            cookies={'galleta': 'dulce'},
+            meta={'cookiejar': "store2"},
+        )
         assert self.mw.process_request(req3, self.spider) is None
         self.assertEqual(req3.headers.get('Cookie'), b'galleta=dulce')
 
@@ -229,3 +242,95 @@ class CookiesMiddlewareTest(TestCase):
         assert self.mw.process_request(request, self.spider) is None
         self.assertIn('Cookie', request.headers)
         self.assertEqual(b'currencyCookie=USD', request.headers['Cookie'])
+
+    def test_keep_cookie_from_default_request_headers_middleware(self):
+        DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty')
+        mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items())
+        # overwrite with values from 'cookies' request argument
+        req1 = Request('http://example.org', cookies={'default': 'something'})
+        assert mw_default_headers.process_request(req1, self.spider) is None
+        assert self.mw.process_request(req1, self.spider) is None
+        self.assertCookieValEqual(req1.headers['Cookie'], b'default=something; asdf=qwerty')
+        # keep both
+        req2 = Request('http://example.com', cookies={'a': 'b'})
+        assert mw_default_headers.process_request(req2, self.spider) is None
+        assert self.mw.process_request(req2, self.spider) is None
+        self.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty')
+
+    def test_keep_cookie_header(self):
+        # keep only cookies from 'Cookie' request header
+        req1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'})
+        assert self.mw.process_request(req1, self.spider) is None
+        self.assertCookieValEqual(req1.headers['Cookie'], 'a=b; c=d')
+        # keep cookies from both 'Cookie' request header and 'cookies' keyword
+        req2 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'}, cookies={'e': 'f'})
+        assert self.mw.process_request(req2, self.spider) is None
+        self.assertCookieValEqual(req2.headers['Cookie'], 'a=b; c=d; e=f')
+        # overwrite values from 'Cookie' request header with 'cookies' keyword
+        req3 = Request(
+            'http://scrapytest.org',
+            headers={'Cookie': 'a=b; c=d'},
+            cookies={'a': 'new', 'e': 'f'},
+        )
+        assert self.mw.process_request(req3, self.spider) is None
+        self.assertCookieValEqual(req3.headers['Cookie'], 'a=new; c=d; e=f')
+
+    def test_request_cookies_encoding(self):
+        # 1) UTF8-encoded bytes
+        req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')})
+        assert self.mw.process_request(req1, self.spider) is None
+        self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
+
+        # 2) Non UTF8-encoded bytes
+        req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')})
+        assert self.mw.process_request(req2, self.spider) is None
+        self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
+
+        # 3) Unicode string
+        req3 = Request('http://example.org', cookies={'a': u'á'})
+        assert self.mw.process_request(req3, self.spider) is None
+        self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
+
+    def test_request_headers_cookie_encoding(self):
+        # 1) UTF8-encoded bytes
+        req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')})
+        assert self.mw.process_request(req1, self.spider) is None
+        self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
+
+        # 2) Non UTF8-encoded bytes
+        req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')})
+        assert self.mw.process_request(req2, self.spider) is None
+        self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
+
+        # 3) Unicode string
+        req3 = Request('http://example.org', headers={'Cookie': u'a=á'})
+        assert self.mw.process_request(req3, self.spider) is None
+        self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
+
+    def test_invalid_cookies(self):
+        """
+        Invalid cookies are logged as warnings and discarded
+        """
+        with LogCapture(
+            'scrapy.downloadermiddlewares.cookies',
+            propagate=False,
+            level=logging.INFO,
+        ) as lc:
+            cookies1 = [{'value': 'bar'}, {'name': 'key', 'value': 'value1'}]
+            req1 = Request('http://example.org/1', cookies=cookies1)
+            assert self.mw.process_request(req1, self.spider) is None
+            cookies2 = [{'name': 'foo'}, {'name': 'key', 'value': 'value2'}]
+            req2 = Request('http://example.org/2', cookies=cookies2)
+            assert self.mw.process_request(req2, self.spider) is None
+            lc.check(
+                ("scrapy.downloadermiddlewares.cookies",
+                 "WARNING",
+                 "Invalid cookie found in request <GET http://example.org/1>:"
+                 " {'value': 'bar'} ('name' is missing)"),
+                ("scrapy.downloadermiddlewares.cookies",
+                 "WARNING",
+                 "Invalid cookie found in request <GET http://example.org/2>:"
+                 " {'name': 'foo'} ('value' is missing)"),
+            )
+        self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1')
+        self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2')

From 08756cd87a7699dd0f73dcdc3178671fb6fec2d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 22 May 2020 18:06:54 +0200
Subject: [PATCH 2867/2974] Fix the description of the ScreenshotPipeline
 example item pipeline

---
 docs/topics/item-pipeline.rst | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 533f84630..c9194caa1 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -167,11 +167,13 @@ method and how to clean up the resources properly.::
 Take screenshot of item
 -----------------------
 
-This example demonstrates how to return a
-:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
-It uses Splash_ to render screenshot of item url. Pipeline
-makes request to locally running instance of Splash_. After request is downloaded,
-it saves the screenshot to a file and adds filename to the item.
+This example demonstrates how to use :doc:`coroutine syntax <coroutines>` in
+the :meth:`process_item` method.
+
+This item pipeline makes a request to a locally-running instance of Splash_ to
+render a screenshot of the item URL. After the request response is downloaded,
+the item pipeline saves the screenshot to a file and adds the filename to the
+item.
 
 ::
 

From 5b2af852892faa3bf293225e6198f0cb1bc3bd85 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 22 May 2020 15:09:58 -0400
Subject: [PATCH 2868/2974] Link to issue describing troublesome pytest
 versions

---
 tests/requirements-py3.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 05b794d81..91fa1c5b5 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -2,6 +2,7 @@
 jmespath
 mitmproxy; python_version >= '3.6'
 mitmproxy<4.0.0; python_version < '3.6'
+# https://github.com/pytest-dev/pytest-twisted/issues/93
 pytest != 5.4, != 5.4.1
 pytest-cov
 pytest-twisted >= 1.11

From dd96f94e8cc1517b7021e35e46cbdc92580c6333 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 22 May 2020 23:30:33 +0300
Subject: [PATCH 2869/2974] Push datetime.utcnow() to its own variable

---
 scrapy/extensions/feedexport.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 1f745be98..45c2971a6 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -398,8 +398,9 @@ class FeedExporter:
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
-        params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
-        params['batch_time'] = datetime.utcnow().isoformat().replace(':', '-')
+        utc_now = datetime.utcnow()
+        params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
+        params['batch_time'] = utc_now.isoformat().replace(':', '-')
         params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)

From 0cc0e51ca330ae5d88d7daaf027d13dd8783603f Mon Sep 17 00:00:00 2001
From: Rohit Sharma <rohit_sh1@yahoo.ca>
Date: Sun, 24 May 2020 13:38:44 -0700
Subject: [PATCH 2870/2974] remove uneeded escape sequence

removed uneeded wscape sequences from method arguments in the docs
folder
---
 docs/topics/api.rst       |  2 +-
 docs/topics/contracts.rst |  2 +-
 docs/topics/exporters.rst | 12 ++++++------
 docs/topics/loaders.rst   | 24 ++++++++++++------------
 docs/topics/spiders.rst   |  2 +-
 5 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 1c461a511..52509ffdf 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -91,7 +91,7 @@ how you :ref:`configure the downloader middlewares
         provided while constructing the crawler, and it is created after the
         arguments given in the :meth:`crawl` method.
 
-    .. method:: crawl(\*args, \**kwargs)
+    .. method:: crawl(*args, **kwargs)
 
         Starts the crawler by instantiating its spider class with the given
         ``args`` and ``kwargs`` arguments, while setting the execution engine in
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 319f577bc..b8b3078c4 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -78,7 +78,7 @@ override three methods:
 
 .. module:: scrapy.contracts
 
-.. class:: Contract(method, \*args)
+.. class:: Contract(method, *args)
 
     :param method: callback function to which the contract is associated
     :type method: function
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index de8b51195..7daf25ab3 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -236,7 +236,7 @@ PythonItemExporter
 XmlItemExporter
 ---------------
 
-.. class:: XmlItemExporter(file, item_element='item', root_element='items', \**kwargs)
+.. class:: XmlItemExporter(file, item_element='item', root_element='items', **kwargs)
 
    Exports Items in XML format to the specified file object.
 
@@ -290,7 +290,7 @@ XmlItemExporter
 CsvItemExporter
 ---------------
 
-.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', \**kwargs)
+.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs)
 
    Exports Items in CSV format to the given file-like object. If the
    :attr:`fields_to_export` attribute is set, it will be used to define the
@@ -323,7 +323,7 @@ CsvItemExporter
 PickleItemExporter
 ------------------
 
-.. class:: PickleItemExporter(file, protocol=0, \**kwargs)
+.. class:: PickleItemExporter(file, protocol=0, **kwargs)
 
    Exports Items in pickle format to the given file-like object.
 
@@ -343,7 +343,7 @@ PickleItemExporter
 PprintItemExporter
 ------------------
 
-.. class:: PprintItemExporter(file, \**kwargs)
+.. class:: PprintItemExporter(file, **kwargs)
 
    Exports Items in pretty print format to the specified file object.
 
@@ -363,7 +363,7 @@ PprintItemExporter
 JsonItemExporter
 ----------------
 
-.. class:: JsonItemExporter(file, \**kwargs)
+.. class:: JsonItemExporter(file, **kwargs)
 
    Exports Items in JSON format to the specified file-like object, writing all
    objects as a list of objects. The additional ``__init__`` method arguments are
@@ -392,7 +392,7 @@ JsonItemExporter
 JsonLinesItemExporter
 ---------------------
 
-.. class:: JsonLinesItemExporter(file, \**kwargs)
+.. class:: JsonLinesItemExporter(file, **kwargs)
 
    Exports Items in JSON format to the specified file-like object, writing one
    JSON-encoded item per line. The additional ``__init__`` method arguments are passed
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 5f75ccbff..eb804f1db 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -273,7 +273,7 @@ There are several ways to modify Item Loader context values:
 ItemLoader objects
 ==================
 
-.. class:: ItemLoader([item, selector, response], \**kwargs)
+.. class:: ItemLoader([item, selector, response], **kwargs)
 
     Return a new Item Loader for populating the given Item. If no item is
     given, one is instantiated automatically using the class in
@@ -303,7 +303,7 @@ ItemLoader objects
 
     :class:`ItemLoader` instances have the following methods:
 
-    .. method:: get_value(value, \*processors, \**kwargs)
+    .. method:: get_value(value, *processors, **kwargs)
 
         Process the given ``value`` by the given ``processors`` and keyword
         arguments.
@@ -321,7 +321,7 @@ ItemLoader objects
         >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
         'FOO`
 
-    .. method:: add_value(field_name, value, \*processors, \**kwargs)
+    .. method:: add_value(field_name, value, *processors, **kwargs)
 
         Process and then add the given ``value`` for the given field.
 
@@ -343,11 +343,11 @@ ItemLoader objects
             loader.add_value('name', u'name: foo', TakeFirst(), re='name: (.+)')
             loader.add_value(None, {'name': u'foo', 'sex': u'male'})
 
-    .. method:: replace_value(field_name, value, \*processors, \**kwargs)
+    .. method:: replace_value(field_name, value, *processors, **kwargs)
 
         Similar to :meth:`add_value` but replaces the collected data with the
         new value instead of adding it.
-    .. method:: get_xpath(xpath, \*processors, \**kwargs)
+    .. method:: get_xpath(xpath, *processors, **kwargs)
 
         Similar to :meth:`ItemLoader.get_value` but receives an XPath instead of a
         value, which is used to extract a list of unicode strings from the
@@ -367,7 +367,7 @@ ItemLoader objects
             # HTML snippet: <p id="price">the price is $1200</p>
             loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)')
 
-    .. method:: add_xpath(field_name, xpath, \*processors, \**kwargs)
+    .. method:: add_xpath(field_name, xpath, *processors, **kwargs)
 
         Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a
         value, which is used to extract a list of unicode strings from the
@@ -385,12 +385,12 @@ ItemLoader objects
             # HTML snippet: <p id="price">the price is $1200</p>
             loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)')
 
-    .. method:: replace_xpath(field_name, xpath, \*processors, \**kwargs)
+    .. method:: replace_xpath(field_name, xpath, *processors, **kwargs)
 
         Similar to :meth:`add_xpath` but replaces collected data instead of
         adding it.
 
-    .. method:: get_css(css, \*processors, \**kwargs)
+    .. method:: get_css(css, *processors, **kwargs)
 
         Similar to :meth:`ItemLoader.get_value` but receives a CSS selector
         instead of a value, which is used to extract a list of unicode strings
@@ -410,7 +410,7 @@ ItemLoader objects
             # HTML snippet: <p id="price">the price is $1200</p>
             loader.get_css('p#price', TakeFirst(), re='the price is (.*)')
 
-    .. method:: add_css(field_name, css, \*processors, \**kwargs)
+    .. method:: add_css(field_name, css, *processors, **kwargs)
 
         Similar to :meth:`ItemLoader.add_value` but receives a CSS selector
         instead of a value, which is used to extract a list of unicode strings
@@ -428,7 +428,7 @@ ItemLoader objects
             # HTML snippet: <p id="price">the price is $1200</p>
             loader.add_css('price', 'p#price', re='the price is (.*)')
 
-    .. method:: replace_css(field_name, css, \*processors, \**kwargs)
+    .. method:: replace_css(field_name, css, *processors, **kwargs)
 
         Similar to :meth:`add_css` but replaces collected data instead of
         adding it.
@@ -678,7 +678,7 @@ Here is a list of all built-in processors:
     >>> proc(['one', 'two', 'three'])
     'one<br>two<br>three'
 
-.. class:: Compose(\*functions, \**default_loader_context)
+.. class:: Compose(*functions, **default_loader_context)
 
     A processor which is constructed from the composition of the given
     functions. This means that each input value of this processor is passed to
@@ -706,7 +706,7 @@ Here is a list of all built-in processors:
     active Loader context accessible through the :meth:`ItemLoader.context`
     attribute.
 
-.. class:: MapCompose(\*functions, \**default_loader_context)
+.. class:: MapCompose(*functions, **default_loader_context)
 
     A processor which is constructed from the composition of the given
     functions, similar to the :class:`Compose` processor. The difference with
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 231db6cea..8ff5e7292 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -121,7 +121,7 @@ scrapy.Spider
       send log messages through it as described on
       :ref:`topics-logging-from-spiders`.
 
-   .. method:: from_crawler(crawler, \*args, \**kwargs)
+   .. method:: from_crawler(crawler, *args, **kwargs)
 
        This is the class method used by Scrapy to create your spiders.
 

From 898bdd3feb23de793ae0b4c725545c32050fc0ea Mon Sep 17 00:00:00 2001
From: Will Beaufoy <will@willbeaufoy.net>
Date: Sun, 10 May 2020 13:49:43 +0100
Subject: [PATCH 2871/2974] Update docs README with build instructions using
 tox

None of the existing commands built the docs properly for me (I had to
revert the changes in 901892d to docs/conf.py to get them to build
properly, and even then no tooltips displayed).

Building them with tox worked for me, but other developers say they
can still use the original method, so the docs now contain both.
---
 docs/README.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/README.rst b/docs/README.rst
index 0a343cd19..0b7afa548 100644
--- a/docs/README.rst
+++ b/docs/README.rst
@@ -57,3 +57,12 @@ There is a way to recreate the doc automatically when you make changes, you
 need to install watchdog (``pip install watchdog``) and then use::
 
     make watch
+
+Alternative method using tox
+----------------------------
+
+To compile the documentation to HTML run the following command::
+
+    tox -e docs
+
+Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir.

From 492197e4773798fcb19ba934c03d5b48e0b7c1f2 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 27 May 2020 10:45:07 -0300
Subject: [PATCH 2872/2974] StopDownload: store response in the exception
 instead of the failure

---
 docs/topics/exceptions.rst                |  9 ++++++---
 scrapy/core/downloader/handlers/http11.py |  8 ++------
 scrapy/core/scraper.py                    | 10 +---------
 tests/test_crawl.py                       | 10 +++++++---
 4 files changed, 16 insertions(+), 21 deletions(-)

diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index 10b23edc4..583a50ab8 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -91,9 +91,12 @@ indicate that no further bytes should be downloaded for a response.
 The ``fail`` boolean parameter controls which method will handle the resulting
 response:
 
-* If ``fail=True``, the request errback is called. The response object is
-  available as the ``response`` attribute of the received
-  :class:`~twisted.python.failure.Failure` object. This is the default behaviour.
+* If ``fail=True`` (default), the request errback is called. The response object is
+  available as the ``response`` attribute of the ``StopDownload`` exception,
+  which is in turn stored as the ``value`` attribute of the received
+  :class:`~twisted.python.failure.Failure` object. This means that in an errback
+  defined as ``def errback(self, failure)``, the response can be accessed though
+  ``failure.value.response``.
 
 * If ``fail=False``, the request callback is called instead.
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index a710761c5..22c9ac520 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -442,12 +442,8 @@ class ScrapyAgent:
             ip_address=result["ip_address"],
         )
         if result.get("failure"):
-            # This failure is not the same object that will reach the errback,
-            # so we need to temporarily store the response in the exception.
-            # It will be moved to the failure in core/scraper.py
-            failure = result["failure"]
-            failure.value.response = response
-            return failure
+            result["failure"].value.response = response
+            return result["failure"]
         return response
 
 
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index c52ada83b..6785e103d 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -11,7 +11,7 @@ from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errba
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
 from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
-from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest, StopDownload
+from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy import signals
 from scrapy.http import Request, Response
 from scrapy.item import _BaseItem
@@ -147,14 +147,6 @@ class Scraper:
 
     def call_spider(self, result, request, spider):
         result.request = request
-        # StopDownload exceptions: make the partial response an attribute of the failure
-        if (
-            isinstance(result, Failure)
-            and isinstance(result.value, StopDownload)
-            and hasattr(result.value, "response")
-        ):
-            result.response = result.value.response
-            delattr(result.value, "response")
         dfd = defer_result(result)
         callback = request.callback or spider.parse
         warn_on_generator_with_return_value(spider, callback)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index e2426e0da..0115b8fb9 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -488,6 +488,10 @@ with multiples lines
         self.assertIsNone(crawler.spider.meta.get("response"))
         self.assertIsInstance(crawler.spider.meta["failure"], Failure)
         self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload)
-        self.assertIsInstance(crawler.spider.meta["failure"].response, Response)
-        self.assertEqual(crawler.spider.meta["failure"].response.body, crawler.spider.meta.get("bytes_received"))
-        self.assertLess(len(crawler.spider.meta["failure"].response.body), crawler.spider.full_response_length)
+        self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response)
+        self.assertEqual(
+            crawler.spider.meta["failure"].value.response.body,
+            crawler.spider.meta.get("bytes_received"))
+        self.assertLess(
+            len(crawler.spider.meta["failure"].value.response.body),
+            crawler.spider.full_response_length)

From 44d5801b05280ad633e63a6b25db264d6586d882 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 27 May 2020 11:13:21 -0300
Subject: [PATCH 2873/2974] Fix KeyError in LocalWeakReferencedCache

---
 scrapy/utils/datatypes.py     | 4 ++--
 tests/test_utils_datatypes.py | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index f59f4cc55..2a92d0588 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -105,8 +105,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
     def __getitem__(self, key):
         try:
             return super(LocalWeakReferencedCache, self).__getitem__(key)
-        except TypeError:
-            return None  # key is not weak-referenceable, it's not cached
+        except (TypeError, KeyError):
+            return None  # key is either not weak-referenceable or not cached
 
 
 class SequenceExclude:
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 0a4c6034a..aa18ef1f3 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -271,6 +271,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase):
         self.assertNotIn(r1, cache)
         self.assertIn(r2, cache)
         self.assertIn(r3, cache)
+        self.assertEqual(cache[r1], None)
         self.assertEqual(cache[r2], 2)
         self.assertEqual(cache[r3], 3)
         del r2

From 5cef927944152f11d296566767c1d794b34b38e4 Mon Sep 17 00:00:00 2001
From: Bulat Khabibullin <xb.bolo@gmail.com>
Date: Mon, 1 Jun 2020 07:57:23 +0300
Subject: [PATCH 2874/2974] Implement TextResponse.json() (#4574)

---
 docs/topics/request-response.rst |  5 +++++
 scrapy/http/response/text.py     | 12 ++++++++++++
 tests/test_http_response.py      | 21 +++++++++++++++++++++
 3 files changed, 38 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index ae25ff7e4..bbd715766 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -879,6 +879,11 @@ TextResponse objects
 
     .. automethod:: TextResponse.follow_all
 
+    .. automethod:: TextResponse.json()
+
+        Returns a Python object from deserialized JSON document.
+        The result is cached after the first call.
+
 
 HtmlResponse objects
 --------------------
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 0603b6653..40cf3f483 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -5,6 +5,7 @@ discovering (through HTTP headers) to base Response class.
 See documentation in docs/topics/request-response.rst
 """
 
+import json
 import warnings
 from contextlib import suppress
 from typing import Generator
@@ -21,10 +22,13 @@ from scrapy.http.response import Response
 from scrapy.utils.python import memoizemethod_noargs, to_unicode
 from scrapy.utils.response import get_base_url
 
+_NONE = object()
+
 
 class TextResponse(Response):
 
     _DEFAULT_ENCODING = 'ascii'
+    _cached_decoded_json = _NONE
 
     def __init__(self, *args, **kwargs):
         self._encoding = kwargs.pop('encoding', None)
@@ -68,6 +72,14 @@ class TextResponse(Response):
                       ScrapyDeprecationWarning, stacklevel=2)
         return self.text
 
+    def json(self):
+        """
+        Deserialize a JSON document to a Python object.
+        """
+        if self._cached_decoded_json is _NONE:
+            self._cached_decoded_json = json.loads(self.text)
+        return self._cached_decoded_json
+
     @property
     def text(self):
         """ Body as unicode """
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 039e863f4..e0ca3c0e6 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,4 +1,5 @@
 import unittest
+from unittest import mock
 from warnings import catch_warnings
 
 from w3lib.encoding import resolve_encoding
@@ -685,6 +686,26 @@ class TextResponseTest(BaseResponseTest):
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 
+    def test_json_response(self):
+        json_body = b"""{"ip": "109.187.217.200"}"""
+        json_response = self.response_class("http://www.example.com", body=json_body)
+        self.assertEqual(json_response.json(), {'ip': '109.187.217.200'})
+
+        text_body = b"""<html><body>text</body></html>"""
+        text_response = self.response_class("http://www.example.com", body=text_body)
+        with self.assertRaises(ValueError):
+            text_response.json()
+
+    def test_cache_json_response(self):
+        json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""]
+        for json_body in json_valid_bodies:
+            json_response = self.response_class("http://www.example.com", body=json_body)
+
+            with mock.patch('json.loads') as mock_json:
+                for _ in range(2):
+                    json_response.json()
+                mock_json.assert_called_once_with(json_body.decode())
+
 
 class HtmlResponseTest(TextResponseTest):
 

From 91e505edfe811c8faa6e457ecdae42c3644e6537 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Tue, 2 Jun 2020 05:32:14 -0300
Subject: [PATCH 2875/2974] Return single element from coroutine callback
 (#4609)

---
 scrapy/utils/spider.py |  8 ++++++--
 tests/spiders.py       | 11 +++++++++++
 tests/test_crawl.py    | 16 ++++++++++++++++
 3 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 1b8a82829..7e7a50c88 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -1,5 +1,5 @@
-import logging
 import inspect
+import logging
 
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
@@ -18,7 +18,11 @@ def iterate_spider_output(result):
         d = deferred_from_coro(collect_asyncgen(result))
         d.addCallback(iterate_spider_output)
         return d
-    return arg_to_iter(deferred_from_coro(result))
+    elif inspect.iscoroutine(result):
+        d = deferred_from_coro(result)
+        d.addCallback(iterate_spider_output)
+        return d
+    return arg_to_iter(result)
 
 
 def iter_spider_classes(module):
diff --git a/tests/spiders.py b/tests/spiders.py
index 05078cc04..a360d8206 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -119,6 +119,17 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):
         return [{'id': 1}, {'id': 2}]
 
 
+class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
+
+    name = "asyncdef_asyncio_return_single_element"
+
+    async def parse(self, response):
+        await asyncio.sleep(0.1)
+        status = await get_from_asyncio_queue(response.status)
+        self.logger.info("Got response %d" % status)
+        return {"foo": 42}
+
+
 class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
 
     name = 'asyncdef_asyncio_reqs_return'
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 0115b8fb9..df920f2a2 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -21,6 +21,7 @@ from scrapy.utils.python import to_unicode
 from tests.mockserver import MockServer
 from tests.spiders import (
     AsyncDefAsyncioReqsReturnSpider,
+    AsyncDefAsyncioReturnSingleElementSpider,
     AsyncDefAsyncioReturnSpider,
     AsyncDefAsyncioSpider,
     AsyncDefSpider,
@@ -364,6 +365,21 @@ with multiples lines
         self.assertIn({'id': 1}, items)
         self.assertIn({'id': 2}, items)
 
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncio_parse_items_single_element(self):
+        items = []
+
+        def _on_item_scraped(item):
+            items.append(item)
+
+        crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSingleElementSpider)
+        crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+        with LogCapture() as log:
+            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        self.assertIn("Got response 200", str(log))
+        self.assertIn({"foo": 42}, items)
+
     @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
     @mark.only_asyncio()
     @defer.inlineCallbacks

From 0cabf406d5c50060ac2da4e5c76d704606bee990 Mon Sep 17 00:00:00 2001
From: Matthias Meschede <MMesch@users.noreply.github.com>
Date: Mon, 1 Jun 2020 17:41:52 +0200
Subject: [PATCH 2876/2974] set write permission to startproject folder

---
 scrapy/commands/startproject.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index ebe3a9c2c..cfaa25154 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -4,6 +4,7 @@ import string
 from importlib import import_module
 from os.path import join, exists, abspath
 from shutil import ignore_patterns, move, copy2, copystat
+import stat
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -79,6 +80,28 @@ class Command(ScrapyCommand):
                 copy2(srcname, dstname)
         copystat(src, dst)
 
+    def _set_rw_permissions(self, path):
+        """
+        Sets permissions of a directory tree to +rw and +rwx for folders.
+        This is necessary if the start template files come without write
+        permissions.
+        """
+        mode_rw = (stat.S_IRUSR
+                   | stat.S_IWUSR
+                   | stat.S_IRGRP
+                   | stat.S_IROTH)
+
+        mode_x = (stat.S_IXUSR
+                  | stat.S_IXGRP
+                  | stat.S_IXOTH)
+
+        os.chmod(path, mode_rw | mode_x)
+        for root, dirs, files in os.walk(path):
+            for dir in dirs:
+                os.chmod(join(root, dir), mode_rw | mode_x)
+            for file in files:
+                os.chmod(join(root, file), mode_rw)
+
     def run(self, args, opts):
         if len(args) not in (1, 2):
             raise UsageError()
@@ -99,6 +122,9 @@ class Command(ScrapyCommand):
             return
 
         self._copytree(self.templates_dir, abspath(project_dir))
+
+        self._set_rw_permissions(abspath(project_dir))
+
         move(join(project_dir, 'module'), join(project_dir, project_name))
         for paths in TEMPLATES_TO_RENDER:
             path = join(*paths)

From 2df3b54c7d2666a95793ea6788e88d1ae945e0f5 Mon Sep 17 00:00:00 2001
From: Matthias Meschede <MMesch@users.noreply.github.com>
Date: Fri, 5 Jun 2020 09:29:05 +0200
Subject: [PATCH 2877/2974] refactor

---
 scrapy/commands/startproject.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index cfaa25154..3d1f5914f 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -79,6 +79,7 @@ class Command(ScrapyCommand):
             else:
                 copy2(srcname, dstname)
         copystat(src, dst)
+        self._set_rw_permissions(dst)
 
     def _set_rw_permissions(self, path):
         """
@@ -123,8 +124,6 @@ class Command(ScrapyCommand):
 
         self._copytree(self.templates_dir, abspath(project_dir))
 
-        self._set_rw_permissions(abspath(project_dir))
-
         move(join(project_dir, 'module'), join(project_dir, project_name))
         for paths in TEMPLATES_TO_RENDER:
             path = join(*paths)

From 76a2cbf0ff7a060833812704a0416ba617ddc8b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 9 Jun 2020 21:30:19 +0200
Subject: [PATCH 2878/2974] Apply minor style changes

---
 scrapy/commands/startproject.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 3d1f5914f..852281959 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,10 +1,10 @@
 import re
 import os
+import stat
 import string
 from importlib import import_module
 from os.path import join, exists, abspath
 from shutil import ignore_patterns, move, copy2, copystat
-import stat
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -123,7 +123,6 @@ class Command(ScrapyCommand):
             return
 
         self._copytree(self.templates_dir, abspath(project_dir))
-
         move(join(project_dir, 'module'), join(project_dir, project_name))
         for paths in TEMPLATES_TO_RENDER:
             path = join(*paths)

From 8b549392f924ddad9536e55c6120638daf688dfd Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Thu, 11 Jun 2020 09:53:59 -0300
Subject: [PATCH 2879/2974] Bump minimum Python version to 3.5.2 (#4615)

---
 .travis.yml            | 6 ++----
 README.rst             | 2 +-
 docs/faq.rst           | 2 +-
 docs/intro/install.rst | 2 +-
 scrapy/__init__.py     | 4 ++--
 setup.py               | 2 +-
 6 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index d6ec88e06..e44f85237 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,11 +18,9 @@ matrix:
 
     - env: TOXENV=pypy3
     - env: TOXENV=pinned
-      python: 3.5.1
-      dist: trusty
+      python: 3.5.2
     - env: TOXENV=asyncio
-      python: 3.5.1  # We use additional code to support 3.5.3 and earlier
-      dist: trusty
+      python: 3.5.2  # We use additional code to support 3.5.3 and earlier
     - env: TOXENV=py
       python: 3.5
     - env: TOXENV=asyncio
diff --git a/README.rst b/README.rst
index fd84e127e..0e3939e9b 100644
--- a/README.rst
+++ b/README.rst
@@ -40,7 +40,7 @@ including a list of features.
 Requirements
 ============
 
-* Python 3.5.1+
+* Python 3.5.2+
 * Works on Linux, Windows, macOS, BSD
 
 Install
diff --git a/docs/faq.rst b/docs/faq.rst
index c06cb945b..9cdb7d09d 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
 What Python versions does Scrapy support?
 -----------------------------------------
 
-Scrapy is supported under Python 3.5.1+
+Scrapy is supported under Python 3.5.2+
 under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
 Python 3 support was added in Scrapy 1.1.
 PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 4af80d801..fb64d443c 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -7,7 +7,7 @@ Installation guide
 Installing Scrapy
 =================
 
-Scrapy runs on Python 3.5.1 or above under CPython (default Python
+Scrapy runs on Python 3.5.2 or above under CPython (default Python
 implementation) and PyPy (starting with PyPy 5.9).
 
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index e791deaa6..f0259a9b7 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
 
 
 # Check minimum required Python version
-if sys.version_info < (3, 5):
-    print("Scrapy %s requires Python 3.5" % __version__)
+if sys.version_info < (3, 5, 2):
+    print("Scrapy %s requires Python 3.5.2" % __version__)
     sys.exit(1)
 
 
diff --git a/setup.py b/setup.py
index 1b3c6771a..71dc3232d 100644
--- a/setup.py
+++ b/setup.py
@@ -66,7 +66,7 @@ setup(
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-    python_requires='>=3.5',
+    python_requires='>=3.5.2',
     install_requires=[
         'Twisted>=17.9.0',
         'cryptography>=2.0',

From ec98dabfab60283303a9208ccd8177d9f995ba72 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Sun, 14 Jun 2020 06:45:27 -0300
Subject: [PATCH 2880/2974] Support for dataclass and attrs items (#3881)

---
 docs/conf.py                                  |   1 +
 docs/faq.rst                                  |   8 +-
 docs/topics/architecture.rst                  |   2 +-
 docs/topics/coroutines.rst                    |  13 +-
 docs/topics/exporters.rst                     |  34 ++-
 docs/topics/feed-exports.rst                  |   4 +-
 docs/topics/item-pipeline.rst                 |  54 ++--
 docs/topics/items.rst                         | 241 ++++++++++++++----
 docs/topics/leaks.rst                         |  15 +-
 docs/topics/loaders.rst                       |  53 ++--
 docs/topics/media-pipeline.rst                |  33 ++-
 docs/topics/settings.rst                      |   4 +-
 docs/topics/signals.rst                       |  10 +-
 docs/topics/spider-middleware.rst             |  13 +-
 docs/topics/spiders.rst                       |  18 +-
 scrapy/commands/parse.py                      |   6 +-
 scrapy/contracts/default.py                   |  28 +-
 scrapy/core/scraper.py                        |  27 +-
 scrapy/exporters.py                           |  38 +--
 scrapy/item.py                                |   2 +-
 scrapy/loader/__init__.py                     |  18 +-
 scrapy/pipelines/files.py                     |  23 +-
 scrapy/pipelines/images.py                    |  19 +-
 scrapy/shell.py                               |  11 +-
 scrapy/spiders/feed.py                        |   2 +-
 .../project/module/middlewares.py.tmpl        |   8 +-
 .../project/module/pipelines.py.tmpl          |   4 +
 scrapy/utils/serialize.py                     |   6 +-
 setup.py                                      |   1 +
 tests/requirements-py3.txt                    |   2 +
 tests/test_engine.py                          |  40 ++-
 tests/test_loader.py                          |  47 +++-
 tests/test_pipeline_files.py                  | 144 ++++++++---
 tests/test_pipeline_images.py                 | 129 +++++++---
 tests/test_utils_serialize.py                 |  43 +++-
 tox.ini                                       |   3 +-
 36 files changed, 753 insertions(+), 351 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 29b2fc406..86734fae7 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -281,6 +281,7 @@ coverage_ignore_pyobjects = [
 # -------------------------------------
 
 intersphinx_mapping = {
+    'attrs': ('https://www.attrs.org/en/stable/', None),
     'coverage': ('https://coverage.readthedocs.io/en/stable', None),
     'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
     'pytest': ('https://docs.pytest.org/en/latest', None),
diff --git a/docs/faq.rst b/docs/faq.rst
index 9cdb7d09d..d5ea3cb87 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -342,15 +342,15 @@ method for this purpose. For example::
 
     from copy import deepcopy
 
-    from scrapy.item import Item
-
+    from itemadapter import is_item, ItemAdapter
 
     class MultiplyItemsMiddleware:
 
         def process_spider_output(self, response, result, spider):
             for item in result:
-                if isinstance(item, (Item, dict)):
-                    for _ in range(item['multiply_by']):
+                if is_item(item):
+                    adapter = ItemAdapter(item)
+                    for _ in range(adapter['multiply_by']):
                         yield deepcopy(item)
 
 Does Scrapy support IPv6 addresses?
diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index ae25dfa2f..074c59241 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -104,7 +104,7 @@ Spiders
 -------
 
 Spiders are custom classes written by Scrapy users to parse responses and
-extract items (aka scraped items) from them or additional requests to
+extract :ref:`items <topics-items>` from them or additional requests to
 follow. For more information see :ref:`topics-spiders`.
 
 .. _component-pipelines:
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 7a9ecd4d5..a0952d323 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -53,21 +53,28 @@ There are several use cases for coroutines in Scrapy. Code that would
 return Deferreds when written for previous Scrapy versions, such as downloader
 middlewares and signal handlers, can be rewritten to be shorter and cleaner::
 
+    from itemadapter import ItemAdapter
+
     class DbPipeline:
         def _update_item(self, data, item):
-            item['field'] = data
+            adapter = ItemAdapter(item)
+            adapter['field'] = data
             return item
 
         def process_item(self, item, spider):
-            dfd = db.get_some_data(item['id'])
+            adapter = ItemAdapter(item)
+            dfd = db.get_some_data(adapter['id'])
             dfd.addCallback(self._update_item, item)
             return dfd
 
 becomes::
 
+    from itemadapter import ItemAdapter
+
     class DbPipeline:
         async def process_item(self, item, spider):
-            item['field'] = await db.get_some_data(item['id'])
+            adapter = ItemAdapter(item)
+            adapter['field'] = await db.get_some_data(adapter['id'])
             return item
 
 Coroutines may be used to call asynchronous code. This includes other
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 7daf25ab3..e5c99e5b1 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -40,6 +40,7 @@ Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
 Item Exporters to group scraped items to different files according to the
 value of one of their fields::
 
+    from itemadapter import ItemAdapter
     from scrapy.exporters import XmlItemExporter
 
     class PerYearXmlExportPipeline:
@@ -53,7 +54,8 @@ value of one of their fields::
                 exporter.finish_exporting()
 
         def _exporter_for_item(self, item):
-            year = item['year']
+            adapter = ItemAdapter(item)
+            year = adapter['year']
             if year not in self.year_to_exporter:
                 f = open('{}.xml'.format(year), 'wb')
                 exporter = XmlItemExporter(f)
@@ -167,9 +169,10 @@ BaseItemExporter
       value unchanged except for ``unicode`` values which are encoded to
       ``str`` using the encoding declared in the :attr:`encoding` attribute.
 
-      :param field: the field being serialized. If a raw dict is being
-          exported (not :class:`~.Item`) *field* value is an empty dict.
-      :type field: :class:`~scrapy.item.Field` object or an empty dict
+      :param field: the field being serialized. If the source :ref:`item object
+          <item-types>` does not define field metadata, *field* is an empty
+          :class:`dict`.
+      :type field: :class:`~scrapy.item.Field` object or a :class:`dict` instance
 
       :param name: the name of the field being serialized
       :type name: str
@@ -192,14 +195,17 @@ BaseItemExporter
 
    .. attribute:: fields_to_export
 
-      A list with the name of the fields that will be exported, or None if you
-      want to export all fields. Defaults to None.
+      A list with the name of the fields that will be exported, or ``None`` if
+      you want to export all fields. Defaults to ``None``.
 
       Some exporters (like :class:`CsvItemExporter`) respect the order of the
       fields defined in this attribute.
 
-      Some exporters may require fields_to_export list in order to export the
-      data properly when spiders return dicts (not :class:`~Item` instances).
+      When using :ref:`item objects <item-types>` that do not expose all their
+      possible fields, exporters that do not support exporting a different
+      subset of fields per item will only export the fields found in the first
+      item exported. Use ``fields_to_export`` to define all the fields to be
+      exported.
 
    .. attribute:: export_empty_fields
 
@@ -238,7 +244,7 @@ XmlItemExporter
 
 .. class:: XmlItemExporter(file, item_element='item', root_element='items', **kwargs)
 
-   Exports Items in XML format to the specified file object.
+   Exports items in XML format to the specified file object.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -292,7 +298,7 @@ CsvItemExporter
 
 .. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs)
 
-   Exports Items in CSV format to the given file-like object. If the
+   Exports items in CSV format to the given file-like object. If the
    :attr:`fields_to_export` attribute is set, it will be used to define the
    CSV columns and their order. The :attr:`export_empty_fields` attribute has
    no effect on this exporter.
@@ -325,7 +331,7 @@ PickleItemExporter
 
 .. class:: PickleItemExporter(file, protocol=0, **kwargs)
 
-   Exports Items in pickle format to the given file-like object.
+   Exports items in pickle format to the given file-like object.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -345,7 +351,7 @@ PprintItemExporter
 
 .. class:: PprintItemExporter(file, **kwargs)
 
-   Exports Items in pretty print format to the specified file object.
+   Exports items in pretty print format to the specified file object.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -365,7 +371,7 @@ JsonItemExporter
 
 .. class:: JsonItemExporter(file, **kwargs)
 
-   Exports Items in JSON format to the specified file-like object, writing all
+   Exports items in JSON format to the specified file-like object, writing all
    objects as a list of objects. The additional ``__init__`` method arguments are
    passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
    arguments to the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
@@ -394,7 +400,7 @@ JsonLinesItemExporter
 
 .. class:: JsonLinesItemExporter(file, **kwargs)
 
-   Exports Items in JSON format to the specified file-like object, writing one
+   Exports items in JSON format to the specified file-like object, writing one
    JSON-encoded item per line. The additional ``__init__`` method arguments are passed
    to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
    the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 9e5968a29..24d69040c 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -298,8 +298,8 @@ Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
 
 Use FEED_EXPORT_FIELDS option to define fields to export and their order.
 
-When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses fields
-defined in dicts or :class:`~.Item` subclasses a spider is yielding.
+When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses the fields
+defined in :ref:`item objects <topics-items>` yielded by your spider.
 
 If an exporter requires a fixed set of fields (this is the case for
 :ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index c9194caa1..cd6a6d47e 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -27,15 +27,19 @@ Each item pipeline component is a Python class that must implement the following
 
 .. method:: process_item(self, item, spider)
 
-   This method is called for every item pipeline component. :meth:`process_item`
-   must either: return a dict with data, return an :class:`~scrapy.item.Item`
-   (or any descendant class) object, return a
-   :class:`~twisted.internet.defer.Deferred` or raise
-   :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
-   processed by further pipeline components.
+   This method is called for every item pipeline component.
 
-   :param item: the item scraped
-   :type item: :class:`~scrapy.item.Item` object or a dict
+   `item` is an :ref:`item object <item-types>`, see
+   :ref:`supporting-item-types`.
+
+   :meth:`process_item` must either: return an :ref:`item object <item-types>`,
+   return a :class:`~twisted.internet.defer.Deferred` or raise a
+   :exc:`~scrapy.exceptions.DropItem` exception.
+
+   Dropped items are no longer processed by further pipeline components.
+
+   :param item: the scraped item
+   :type item: :ref:`item object <item-types>`
 
    :param spider: the spider which scraped the item
    :type spider: :class:`~scrapy.spiders.Spider` object
@@ -79,16 +83,17 @@ Let's take a look at the following hypothetical pipeline that adjusts the
 (``price_excludes_vat`` attribute), and drops those items which don't
 contain a price::
 
+    from itemadapter import ItemAdapter
     from scrapy.exceptions import DropItem
-
     class PricePipeline:
 
         vat_factor = 1.15
 
         def process_item(self, item, spider):
-            if item.get('price'):
-                if item.get('price_excludes_vat'):
-                    item['price'] = item['price'] * self.vat_factor
+            adapter = ItemAdapter(item)
+            if adapter.get('price'):
+                if adapter.get('price_excludes_vat'):
+                    adapter['price'] = adapter['price'] * self.vat_factor
                 return item
             else:
                 raise DropItem("Missing price in %s" % item)
@@ -103,6 +108,8 @@ format::
 
    import json
 
+   from itemadapter import ItemAdapter
+
    class JsonWriterPipeline:
 
        def open_spider(self, spider):
@@ -112,7 +119,7 @@ format::
            self.file.close()
 
        def process_item(self, item, spider):
-           line = json.dumps(dict(item)) + "\n"
+           line = json.dumps(ItemAdapter(item).asdict()) + "\n"
            self.file.write(line)
            return item
 
@@ -131,6 +138,7 @@ The main point of this example is to show how to use :meth:`from_crawler`
 method and how to clean up the resources properly.::
 
     import pymongo
+    from itemadapter import ItemAdapter
 
     class MongoPipeline:
 
@@ -155,7 +163,7 @@ method and how to clean up the resources properly.::
             self.client.close()
 
         def process_item(self, item, spider):
-            self.db[self.collection_name].insert_one(dict(item))
+            self.db[self.collection_name].insert_one(ItemAdapter(item).asdict())
             return item
 
 .. _MongoDB: https://www.mongodb.com/
@@ -177,10 +185,11 @@ item.
 
 ::
 
-    import scrapy
     import hashlib
     from urllib.parse import quote
 
+    import scrapy
+    from itemadapter import ItemAdapter
 
     class ScreenshotPipeline:
         """Pipeline that uses Splash to render screenshot of
@@ -189,7 +198,8 @@ item.
         SPLASH_URL = "http://localhost:8050/render.png?url={}"
 
         async def process_item(self, item, spider):
-            encoded_item_url = quote(item["url"])
+            adapter = ItemAdapter(item)
+            encoded_item_url = quote(adapter["url"])
             screenshot_url = self.SPLASH_URL.format(encoded_item_url)
             request = scrapy.Request(screenshot_url)
             response = await spider.crawler.engine.download(request, spider)
@@ -199,14 +209,14 @@ item.
                 return item
 
             # Save screenshot to file, filename will be hash of url.
-            url = item["url"]
+            url = adapter["url"]
             url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
             filename = "{}.png".format(url_hash)
             with open(filename, "wb") as f:
                 f.write(response.body)
 
             # Store filename in item.
-            item["screenshot_filename"] = filename
+            adapter["screenshot_filename"] = filename
             return item
 
 .. _Splash: https://splash.readthedocs.io/en/stable/
@@ -219,6 +229,7 @@ already processed. Let's say that our items have a unique id, but our spider
 returns multiples items with the same id::
 
 
+    from itemadapter import ItemAdapter
     from scrapy.exceptions import DropItem
 
     class DuplicatesPipeline:
@@ -227,10 +238,11 @@ returns multiples items with the same id::
             self.ids_seen = set()
 
         def process_item(self, item, spider):
-            if item['id'] in self.ids_seen:
-                raise DropItem("Duplicate item found: %s" % item)
+            adapter = ItemAdapter(item)
+            if adapter['id'] in self.ids_seen:
+                raise DropItem("Duplicate item found: %r" % item)
             else:
-                self.ids_seen.add(item['id'])
+                self.ids_seen.add(adapter['id'])
                 return item
 
 
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 0941a8a1b..65bf156ac 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -8,29 +8,155 @@ Items
    :synopsis: Item and Field classes
 
 The main goal in scraping is to extract structured data from unstructured
-sources, typically, web pages. Scrapy spiders can return the extracted data
-as Python dicts. While convenient and familiar, Python dicts lack structure:
-it is easy to make a typo in a field name or return inconsistent data,
-especially in a larger project with many spiders.
+sources, typically, web pages. :ref:`Spiders <topics-spiders>` may return the
+extracted data as `items`, Python objects that define key-value pairs.
 
-To define common output data format Scrapy provides the :class:`Item` class.
-:class:`Item` objects are simple containers used to collect the scraped data.
-They provide an API similar to :class:`dict` API with a convenient syntax
-for declaring their available fields.
+Scrapy supports :ref:`multiple types of items <item-types>`. When you create an
+item, you may use whichever type of item you want. When you write code that
+receives an item, your code should :ref:`work for any item type
+<supporting-item-types>`.
 
-Various Scrapy components use extra information provided by Items:
-exporters look at declared fields to figure out columns to export,
-serialization can be customized using Item fields metadata, :mod:`trackref`
-tracks Item instances to help find memory leaks
-(see :ref:`topics-leaks-trackrefs`), etc.
+.. _item-types:
+
+Item Types
+==========
+
+Scrapy supports the following types of items, via the `itemadapter`_ library:
+:ref:`dictionaries <dict-items>`, :ref:`Item objects <item-objects>`,
+:ref:`dataclass objects <dataclass-items>`, and :ref:`attrs objects <attrs-items>`.
+
+.. _itemadapter: https://github.com/scrapy/itemadapter
+
+.. _dict-items:
+
+Dictionaries
+------------
+
+As an item type, :class:`dict` is convenient and familiar.
+
+.. _item-objects:
+
+Item objects
+------------
+
+:class:`Item` provides a :class:`dict`-like API plus additional features that
+make it the most feature-complete item type:
+
+.. class:: Item([arg])
+
+    :class:`Item` objects replicate the standard :class:`dict` API, including
+    its ``__init__`` method.
+
+    :class:`Item` allows defining field names, so that:
+
+    -   :class:`KeyError` is raised when using undefined field names (i.e.
+        prevents typos going unnoticed)
+
+    -   :ref:`Item exporters <topics-exporters>` can export all fields by
+        default even if the first scraped object does not have values for all
+        of them
+
+    :class:`Item` also allows defining field metadata, which can be used to
+    :ref:`customize serialization <topics-exporters-field-serialization>`.
+
+    :mod:`trackref` tracks :class:`Item` objects to help find memory leaks
+    (see :ref:`topics-leaks-trackrefs`).
+
+    :class:`Item` objects also provide the following additional API members:
+
+    .. automethod:: copy
+
+    .. automethod:: deepcopy
+
+    .. attribute:: fields
+
+        A dictionary containing *all declared fields* for this Item, not only
+        those populated. The keys are the field names and the values are the
+        :class:`Field` objects used in the :ref:`Item declaration
+        <topics-items-declaring>`.
+
+Example::
+
+    from scrapy.item import Item, Field
+
+    class CustomItem(Item):
+        one_field = Field()
+        another_field = Field()
+
+.. _dataclass-items:
+
+Dataclass objects
+-----------------
+
+.. versionadded:: 2.2
+
+:func:`~dataclasses.dataclass` allows defining item classes with field names,
+so that :ref:`item exporters <topics-exporters>` can export all fields by
+default even if the first scraped object does not have values for all of them.
+
+Additionally, ``dataclass`` items also allow to:
+
+* define the type and default value of each defined field.
+
+* define custom field metadata through :func:`dataclasses.field`, which can be used to
+  :ref:`customize serialization <topics-exporters-field-serialization>`.
+
+They work natively in Python 3.7 or later, or using the `dataclasses
+backport`_ in Python 3.6.
+
+.. _dataclasses backport: https://pypi.org/project/dataclasses/
+
+Example::
+
+    from dataclasses import dataclass
+
+    @dataclass
+    class CustomItem:
+        one_field: str
+        another_field: int
+
+.. note:: Field types are not enforced at run time.
+
+.. _attrs-items:
+
+attr.s objects
+--------------
+
+.. versionadded:: 2.2
+
+:func:`attr.s` allows defining item classes with field names,
+so that :ref:`item exporters <topics-exporters>` can export all fields by
+default even if the first scraped object does not have values for all of them.
+
+Additionally, ``attr.s`` items also allow to:
+
+* define the type and default value of each defined field.
+
+* define custom field :ref:`metadata <attrs:metadata>`, which can be used to
+  :ref:`customize serialization <topics-exporters-field-serialization>`.
+
+In order to use this type, the :doc:`attrs package <attrs:index>` needs to be installed.
+
+Example::
+
+    import attr
+
+    @attr.s
+    class CustomItem:
+        one_field = attr.ib()
+        another_field = attr.ib()
+
+
+Working with Item objects
+=========================
 
 .. _topics-items-declaring:
 
-Declaring Items
-===============
+Declaring Item subclasses
+-------------------------
 
-Items are declared using a simple class definition syntax and :class:`Field`
-objects. Here is an example::
+Item subclasses are declared using a simple class definition syntax and
+:class:`Field` objects. Here is an example::
 
     import scrapy
 
@@ -48,10 +174,11 @@ objects. Here is an example::
 .. _Django: https://www.djangoproject.com/
 .. _Django Models: https://docs.djangoproject.com/en/dev/topics/db/models/
 
+
 .. _topics-items-fields:
 
-Item Fields
-===========
+Declaring fields
+----------------
 
 :class:`Field` objects are used to specify metadata for each field. For
 example, the serializer function for the ``last_updated`` field illustrated in
@@ -72,15 +199,31 @@ It's important to note that the :class:`Field` objects used to declare the item
 do not stay assigned as class attributes. Instead, they can be accessed through
 the :attr:`Item.fields` attribute.
 
-Working with Items
-==================
+.. class:: Field([arg])
+
+    The :class:`Field` class is just an alias to the built-in :class:`dict` class and
+    doesn't provide any extra functionality or attributes. In other words,
+    :class:`Field` objects are plain-old Python dicts. A separate class is used
+    to support the :ref:`item declaration syntax <topics-items-declaring>`
+    based on class attributes.
+
+.. note:: Field metadata can also be declared for ``dataclass`` and ``attrs``
+    items. Please refer to the documentation for `dataclasses.field`_ and
+    `attr.ib`_ for additional information.
+
+    .. _dataclasses.field: https://docs.python.org/3/library/dataclasses.html#dataclasses.field
+    .. _attr.ib: https://www.attrs.org/en/stable/api.html#attr.ib
+
+
+Working with Item objects
+-------------------------
 
 Here are some examples of common tasks performed with items, using the
 ``Product`` item :ref:`declared above  <topics-items-declaring>`. You will
 notice the API is very similar to the :class:`dict` API.
 
 Creating items
---------------
+''''''''''''''
 
 >>> product = Product(name='Desktop PC', price=1000)
 >>> print(product)
@@ -88,7 +231,7 @@ Product(name='Desktop PC', price=1000)
 
 
 Getting field values
---------------------
+''''''''''''''''''''
 
 >>> product['name']
 Desktop PC
@@ -128,7 +271,7 @@ False
 
 
 Setting field values
---------------------
+''''''''''''''''''''
 
 >>> product['last_updated'] = 'today'
 >>> product['last_updated']
@@ -141,7 +284,7 @@ KeyError: 'Product does not support field: lala'
 
 
 Accessing all populated values
-------------------------------
+''''''''''''''''''''''''''''''
 
 To access all populated values, just use the typical :class:`dict` API:
 
@@ -155,7 +298,7 @@ To access all populated values, just use the typical :class:`dict` API:
 .. _copying-items:
 
 Copying items
--------------
+'''''''''''''
 
 To copy an item, you must first decide whether you want a shallow copy or a
 deep copy.
@@ -183,7 +326,7 @@ To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
 
 
 Other common tasks
-------------------
+''''''''''''''''''
 
 Creating dicts from items:
 
@@ -201,8 +344,8 @@ Traceback (most recent call last):
 KeyError: 'Product does not support field: lala'
 
 
-Extending Items
-===============
+Extending Item subclasses
+-------------------------
 
 You can extend Items (to add more fields or to change some metadata for some
 fields) by declaring a subclass of your original Item.
@@ -222,39 +365,25 @@ appending more values, or changing existing values, like this::
 That adds (or replaces) the ``serializer`` metadata key for the ``name`` field,
 keeping all the previously existing metadata values.
 
-Item objects
-============
 
-.. class:: Item([arg])
+.. _supporting-item-types:
 
-    Return a new Item optionally initialized from the given argument.
+Supporting All Item Types
+=========================
 
-    Items replicate the standard :class:`dict` API, including its ``__init__``
-    method, and also provide the following additional API members:
+In code that receives an item, such as methods of :ref:`item pipelines
+<topics-item-pipeline>` or :ref:`spider middlewares
+<topics-spider-middleware>`, it is a good practice to use the
+:class:`~itemadapter.ItemAdapter` class and the
+:func:`~itemadapter.is_item` function to write code that works for
+any :ref:`supported item type <item-types>`:
 
-    .. automethod:: copy
+.. autoclass:: itemadapter.ItemAdapter
 
-    .. automethod:: deepcopy
+.. autofunction:: itemadapter.is_item
 
-    .. attribute:: fields
 
-        A dictionary containing *all declared fields* for this Item, not only
-        those populated. The keys are the field names and the values are the
-        :class:`Field` objects used in the :ref:`Item declaration
-        <topics-items-declaring>`.
-
-Field objects
-=============
-
-.. class:: Field([arg])
-
-    The :class:`Field` class is just an alias to the built-in :class:`dict` class and
-    doesn't provide any extra functionality or attributes. In other words,
-    :class:`Field` objects are plain-old Python dicts. A separate class is used
-    to support the :ref:`item declaration syntax <topics-items-declaring>`
-    based on class attributes.
-
-Other classes related to Item
-=============================
+Other classes related to items
+==============================
 
 .. autoclass:: ItemMeta
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index ceb708c7e..3224241fc 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -4,7 +4,7 @@
 Debugging memory leaks
 ======================
 
-In Scrapy, objects such as Requests, Responses and Items have a finite
+In Scrapy, objects such as requests, responses and items have a finite
 lifetime: they are created, used for a while, and finally destroyed.
 
 From all those objects, the Request is probably the one with the longest
@@ -61,8 +61,8 @@ Debugging memory leaks with ``trackref``
 ========================================
 
 :mod:`trackref` is a module provided by Scrapy to debug the most common cases of
-memory leaks. It basically tracks the references to all live Requests,
-Responses, Item and Selector objects.
+memory leaks. It basically tracks the references to all live Request,
+Response, Item, Spider and Selector objects.
 
 You can enter the telnet console and inspect how many objects (of the classes
 mentioned above) are currently alive using the ``prefs()`` function which is an
@@ -200,11 +200,10 @@ Debugging memory leaks with muppy
 
 ``trackref`` provides a very convenient mechanism for tracking down memory
 leaks, but it only keeps track of the objects that are more likely to cause
-memory leaks (Requests, Responses, Items, and Selectors). However, there are
-other cases where the memory leaks could come from other (more or less obscure)
-objects. If this is your case, and you can't find your leaks using ``trackref``,
-you still have another resource: the muppy library.
-
+memory leaks. However, there are other cases where the memory leaks could come
+from other (more or less obscure) objects. If this is your case, and you can't
+find your leaks using ``trackref``, you still have another resource: the muppy
+library.
 
 You can use muppy from `Pympler`_.
 
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index eb804f1db..6645bf123 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -7,13 +7,12 @@ Item Loaders
 .. module:: scrapy.loader
    :synopsis: Item Loader class
 
-Item Loaders provide a convenient mechanism for populating scraped :ref:`Items
-<topics-items>`. Even though Items can be populated using their own
-dictionary-like API, Item Loaders provide a much more convenient API for
-populating them from a scraping process, by automating some common tasks like
-parsing the raw extracted data before assigning it.
+Item Loaders provide a convenient mechanism for populating scraped :ref:`items
+<topics-items>`. Even though items can be populated directly, Item Loaders provide a
+much more convenient API for populating them from a scraping process, by automating
+some common tasks like parsing the raw extracted data before assigning it.
 
-In other words, :ref:`Items <topics-items>` provide the *container* of
+In other words, :ref:`items <topics-items>` provide the *container* of
 scraped data, while Item Loaders provide the mechanism for *populating* that
 container.
 
@@ -25,10 +24,10 @@ Using Item Loaders to populate items
 ====================================
 
 To use an Item Loader, you must first instantiate it. You can either
-instantiate it with a dict-like object (e.g. Item or dict) or without one, in
-which case an Item is automatically instantiated in the Item Loader ``__init__`` method
-using the Item class specified in the :attr:`ItemLoader.default_item_class`
-attribute.
+instantiate it with an :ref:`item object <topics-items>` or without one, in which
+case an instance of :class:`~scrapy.item.Item` is automatically created in the
+Item Loader ``__init__`` method using the :class:`~scrapy.item.Item` subclass
+specified in the :attr:`ItemLoader.default_item_class` attribute.
 
 Then, you start collecting values into the Item Loader, typically using
 :ref:`Selectors <topics-selectors>`. You can add more than one value to
@@ -88,7 +87,7 @@ received (through the :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`
 :meth:`~ItemLoader.add_value` methods) and the result of the input processor is
 collected and kept inside the ItemLoader. After collecting all data, the
 :meth:`ItemLoader.load_item` method is called to populate and get the populated
-:class:`~scrapy.item.Item` object.  That's when the output processor is
+:ref:`item object <topics-items>`.  That's when the output processor is
 called with the data previously collected (and processed using the input
 processor). The result of the output processor is the final value that gets
 assigned to the item.
@@ -153,12 +152,10 @@ Last, but not least, Scrapy comes with some :ref:`commonly used processors
 <topics-loaders-available-processors>` built-in for convenience.
 
 
-
 Declaring Item Loaders
 ======================
 
-Item Loaders are declared like Items, by using a class definition syntax. Here
-is an example::
+Item Loaders are declared using a class definition syntax. Here is an example::
 
     from scrapy.loader import ItemLoader
     from scrapy.loader.processors import TakeFirst, MapCompose, Join
@@ -275,9 +272,9 @@ ItemLoader objects
 
 .. class:: ItemLoader([item, selector, response], **kwargs)
 
-    Return a new Item Loader for populating the given Item. If no item is
-    given, one is instantiated automatically using the class in
-    :attr:`default_item_class`.
+    Return a new Item Loader for populating the given :ref:`item object
+    <topics-items>`. If no item object is given, one is instantiated
+    automatically using the class in :attr:`default_item_class`.
 
     When instantiated with a ``selector`` or a ``response`` parameters
     the :class:`ItemLoader` class provides convenient mechanisms for extracting
@@ -286,7 +283,7 @@ ItemLoader objects
     :param item: The item instance to populate using subsequent calls to
         :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
         or :meth:`~ItemLoader.add_value`.
-    :type item: :class:`~scrapy.item.Item` object
+    :type item: :ref:`item object <topics-items>`
 
     :param selector: The selector to extract data from, when using the
         :meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath`
@@ -444,17 +441,19 @@ ItemLoader objects
 
         Create a nested loader with an xpath selector.
         The supplied selector is applied relative to selector associated
-        with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
-        with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
-        :meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
+        with this :class:`ItemLoader`. The nested loader shares the :ref:`item
+        object <topics-items>` with the parent :class:`ItemLoader` so calls to
+        :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
+        behave as expected.
 
     .. method:: nested_css(css)
 
         Create a nested loader with a css selector.
         The supplied selector is applied relative to selector associated
-        with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
-        with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
-        :meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
+        with this :class:`ItemLoader`. The nested loader shares the :ref:`item
+        object <topics-items>` with the parent :class:`ItemLoader` so calls to
+        :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
+        behave as expected.
 
     .. method:: get_collected_values(field_name)
 
@@ -477,7 +476,7 @@ ItemLoader objects
 
     .. attribute:: item
 
-        The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
+        The :ref:`item object <topics-items>` being parsed by this Item Loader.
         This is mostly used as a property so when attempting to override this
         value, you may want to check out :attr:`default_item_class` first.
 
@@ -488,8 +487,8 @@ ItemLoader objects
 
     .. attribute:: default_item_class
 
-        An Item class (or factory), used to instantiate items when not given in
-        the ``__init__`` method.
+        An :ref:`item object <topics-items>` class or factory, used to
+        instantiate items when not given in the ``__init__`` method.
 
     .. attribute:: default_input_processor
 
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 86550d7a4..01de3dedb 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -156,7 +156,7 @@ following forms::
 
     ftp://username:password@address:port/path
     ftp://address:port/path
-    
+
 If ``username`` and ``password`` are not provided, they are taken from the :setting:`FTP_USER` and
 :setting:`FTP_PASSWORD` settings respectively.
 
@@ -243,20 +243,22 @@ Usage example
 .. setting:: IMAGES_URLS_FIELD
 .. setting:: IMAGES_RESULT_FIELD
 
-In order to use a media pipeline first, :ref:`enable it
+In order to use a media pipeline, first :ref:`enable it
 <topics-media-pipeline-enabling>`.
 
-Then, if a spider returns a dict with the URLs key (``file_urls`` or
-``image_urls``, for the Files or Images Pipeline respectively), the pipeline will
-put the results under respective key (``files`` or ``images``).
+Then, if a spider returns an :ref:`item object <topics-items>` with the URLs
+field (``file_urls`` or ``image_urls``, for the Files or Images Pipeline
+respectively), the pipeline will put the results under the respective field
+(``files`` or ``images``).
 
-If you prefer to use :class:`~.Item`, then define a custom item with the
-necessary fields, like in this example for Images Pipeline::
+When using :ref:`item types <item-types>` for which fields are defined beforehand,
+you must define both the URLs field and the results field. For example, when
+using the images pipeline, items must define both the ``image_urls`` and the
+``images`` field. For instance, using the :class:`~scrapy.item.Item` class::
 
     import scrapy
 
     class MyItem(scrapy.Item):
-
         # ... other item fields ...
         image_urls = scrapy.Field()
         images = scrapy.Field()
@@ -445,8 +447,11 @@ See here the methods that you can override in your custom Files Pipeline:
       :meth:`~get_media_requests` method and return a Request for each
       file URL::
 
+         from itemadapter import ItemAdapter
+
          def get_media_requests(self, item, info):
-             for file_url in item['file_urls']:
+             adapter = ItemAdapter(item)
+             for file_url in adapter['file_urls']:
                  yield scrapy.Request(file_url)
 
       Those requests will be processed by the pipeline and, when they have finished
@@ -509,13 +514,15 @@ See here the methods that you can override in your custom Files Pipeline:
       store the downloaded file paths (passed in results) in the ``file_paths``
       item field, and we drop the item if it doesn't contain any files::
 
+          from itemadapter import ItemAdapter
           from scrapy.exceptions import DropItem
 
           def item_completed(self, results, item, info):
               file_paths = [x['path'] for ok, x in results if ok]
               if not file_paths:
                   raise DropItem("Item contains no files")
-              item['file_paths'] = file_paths
+              adapter = ItemAdapter(item)
+              adapter['file_paths'] = file_paths
               return item
 
       By default, the :meth:`item_completed` method returns the item.
@@ -589,8 +596,9 @@ Here is a full example of the Images Pipeline whose methods are exemplified
 above::
 
     import scrapy
-    from scrapy.pipelines.images import ImagesPipeline
+    from itemadapter import ItemAdapter
     from scrapy.exceptions import DropItem
+    from scrapy.pipelines.images import ImagesPipeline
 
     class MyImagesPipeline(ImagesPipeline):
 
@@ -602,7 +610,8 @@ above::
             image_paths = [x['path'] for ok, x in results if ok]
             if not image_paths:
                 raise DropItem("Item contains no images")
-            item['image_paths'] = image_paths
+            adapter = ItemAdapter(item)
+            adapter['image_paths'] = image_paths
             return item
 
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index f06d9db3c..5178f272f 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -236,8 +236,8 @@ CONCURRENT_ITEMS
 
 Default: ``100``
 
-Maximum number of concurrent items (per response) to process in parallel in the
-Item Processor (also known as the :ref:`Item Pipeline <topics-item-pipeline>`).
+Maximum number of concurrent items (per response) to process in parallel in
+:ref:`item pipelines <topics-item-pipeline>`.
 
 .. setting:: CONCURRENT_REQUESTS
 
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index fe4fb0834..255ba9d3f 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -151,8 +151,8 @@ item_scraped
 
     This signal supports returning deferreds from its handlers.
 
-    :param item: the item scraped
-    :type item: dict or :class:`~scrapy.item.Item` object
+    :param item: the scraped item
+    :type item: :ref:`item object <item-types>`
 
     :param spider: the spider which scraped the item
     :type spider: :class:`~scrapy.spiders.Spider` object
@@ -172,7 +172,7 @@ item_dropped
     This signal supports returning deferreds from its handlers.
 
     :param item: the item dropped from the :ref:`topics-item-pipeline`
-    :type item: dict or :class:`~scrapy.item.Item` object
+    :type item: :ref:`item object <item-types>`
 
     :param spider: the spider which scraped the item
     :type spider: :class:`~scrapy.spiders.Spider` object
@@ -196,8 +196,8 @@ item_error
 
     This signal supports returning deferreds from its handlers.
 
-    :param item: the item dropped from the :ref:`topics-item-pipeline`
-    :type item: dict or :class:`~scrapy.item.Item` object
+    :param item: the item that caused the error in the :ref:`topics-item-pipeline`
+    :type item: :ref:`item object <item-types>`
 
     :param response: the response being processed when the exception was raised
     :type response: :class:`~scrapy.http.Response` object
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index d49a2209d..c6cbdba76 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -102,29 +102,28 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         it has processed the response.
 
         :meth:`process_spider_output` must return an iterable of
-        :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item`
-        objects.
+        :class:`~scrapy.http.Request` objects and :ref:`item object
+        <topics-items>`.
 
         :param response: the response which generated this output from the
           spider
         :type response: :class:`~scrapy.http.Response` object
 
         :param result: the result returned by the spider
-        :type result: an iterable of :class:`~scrapy.http.Request`, dict
-          or :class:`~scrapy.item.Item` objects
+        :type result: an iterable of :class:`~scrapy.http.Request` objects and
+          :ref:`item object <topics-items>`
 
         :param spider: the spider whose result is being processed
         :type spider: :class:`~scrapy.spiders.Spider` object
 
-
     .. method:: process_spider_exception(response, exception, spider)
 
         This method is called when a spider or :meth:`process_spider_output`
         method (from a previous spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
-        iterable of :class:`~scrapy.http.Request`, dict or
-        :class:`~scrapy.item.Item` objects.
+        iterable of :class:`~scrapy.http.Request` objects and :ref:`item object
+        <topics-items>`.
 
         If it returns ``None``, Scrapy will continue processing this exception,
         executing any other :meth:`process_spider_exception` in the following
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 8ff5e7292..d4d6e2ea0 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -23,8 +23,8 @@ For spiders, the scraping cycle goes through something like this:
    :attr:`~scrapy.spiders.Spider.parse` method as callback function for the
    Requests.
 
-2. In the callback function, you parse the response (web page) and return either
-   dicts with extracted data, :class:`~scrapy.item.Item` objects,
+2. In the callback function, you parse the response (web page) and return
+   :ref:`item objects <topics-items>`,
    :class:`~scrapy.http.Request` objects, or an iterable of these objects.
    Those Requests will also contain a callback (maybe
    the same) and will then be downloaded by Scrapy and then their
@@ -179,8 +179,8 @@ scrapy.Spider
        the same requirements as the :class:`Spider` class.
 
        This method, as well as any other Request callback, must return an
-       iterable of :class:`~scrapy.http.Request` and/or
-       dicts or :class:`~scrapy.item.Item` objects.
+       iterable of :class:`~scrapy.http.Request` and/or :ref:`item objects
+       <topics-items>`.
 
        :param response: the response to parse
        :type response: :class:`~scrapy.http.Response`
@@ -234,7 +234,7 @@ Return multiple Requests and items from a single callback::
                 yield scrapy.Request(response.urljoin(href), self.parse)
 
 Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
-to give data more structure you can use :ref:`topics-items`::
+to give data more structure you can use :class:`~scrapy.item.Item` objects::
 
     import scrapy
     from myproject.items import MyItem
@@ -364,7 +364,7 @@ CrawlSpider
 
       This method is called for the start_urls responses. It allows to parse
       the initial responses and must return either an
-      :class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request`
+      :ref:`item object <topics-items>`, a :class:`~scrapy.http.Request`
       object, or an iterable containing any of them.
 
 Crawling rules
@@ -383,7 +383,7 @@ Crawling rules
    object with that name will be used) to be called for each link extracted with
    the specified link extractor. This callback receives a :class:`~scrapy.http.Response`
    as its first argument and must return either a single instance or an iterable of
-   :class:`~scrapy.item.Item`, ``dict`` and/or :class:`~scrapy.http.Request` objects
+   :ref:`item objects <topics-items>` and/or :class:`~scrapy.http.Request` objects
    (or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response`
    object will contain the text of the link that produced the :class:`~scrapy.http.Request`
    in its ``meta`` dictionary (under the ``link_text`` key)
@@ -531,7 +531,7 @@ XMLFeedSpider
         (``itertag``).  Receives the response and an
         :class:`~scrapy.selector.Selector` for each node.  Overriding this
         method is mandatory. Otherwise, you spider won't work.  This method
-        must return either a :class:`~scrapy.item.Item` object, a
+        must return an :ref:`item object <topics-items>`, a
         :class:`~scrapy.http.Request` object, or an iterable containing any of
         them.
 
@@ -541,7 +541,7 @@ XMLFeedSpider
         spider, and it's intended to perform any last time processing required
         before returning the results to the framework core, for example setting the
         item IDs. It receives a list of results and the response which originated
-        those results. It must return a list of results (Items or Requests).
+        those results. It must return a list of results (items or requests).
 
 
 XMLFeedSpider example
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 580fd2828..8b7fa8b58 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,11 +1,11 @@
 import json
 import logging
 
+from itemadapter import is_item, ItemAdapter
 from w3lib.url import is_url
 
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
-from scrapy.item import _BaseItem
 from scrapy.utils import display
 from scrapy.utils.conf import arglist_to_dict
 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
@@ -81,7 +81,7 @@ class Command(ScrapyCommand):
             items = self.items.get(lvl, [])
 
         print("# Scraped Items ", "-" * 60)
-        display.pprint([dict(x) for x in items], colorize=colour)
+        display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)
 
     def print_requests(self, lvl=None, colour=True):
         if lvl is None:
@@ -117,7 +117,7 @@ class Command(ScrapyCommand):
         items, requests = [], []
 
         for x in iterate_spider_output(callback(response, **cb_kwargs)):
-            if isinstance(x, (_BaseItem, dict)):
+            if is_item(x):
                 items.append(x)
             elif isinstance(x, Request):
                 requests.append(x)
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index cdc2bac15..34f0d36d4 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -1,10 +1,10 @@
 import json
 
-from scrapy.item import _BaseItem
-from scrapy.http import Request
-from scrapy.exceptions import ContractFail
+from itemadapter import is_item, ItemAdapter
 
 from scrapy.contracts import Contract
+from scrapy.exceptions import ContractFail
+from scrapy.http import Request
 
 
 # contracts
@@ -48,11 +48,11 @@ class ReturnsContract(Contract):
     """
 
     name = 'returns'
-    objects = {
-        'request': Request,
-        'requests': Request,
-        'item': (_BaseItem, dict),
-        'items': (_BaseItem, dict),
+    object_type_verifiers = {
+        'request': lambda x: isinstance(x, Request),
+        'requests': lambda x: isinstance(x, Request),
+        'item': is_item,
+        'items': is_item,
     }
 
     def __init__(self, *args, **kwargs):
@@ -64,7 +64,7 @@ class ReturnsContract(Contract):
                 % len(self.args)
             )
         self.obj_name = self.args[0] or None
-        self.obj_type = self.objects[self.obj_name]
+        self.obj_type_verifier = self.object_type_verifiers[self.obj_name]
 
         try:
             self.min_bound = int(self.args[1])
@@ -79,7 +79,7 @@ class ReturnsContract(Contract):
     def post_process(self, output):
         occurrences = 0
         for x in output:
-            if isinstance(x, self.obj_type):
+            if self.obj_type_verifier(x):
                 occurrences += 1
 
         assertion = (self.min_bound <= occurrences <= self.max_bound)
@@ -103,8 +103,8 @@ class ScrapesContract(Contract):
 
     def post_process(self, output):
         for x in output:
-            if isinstance(x, (_BaseItem, dict)):
-                missing = [arg for arg in self.args if arg not in x]
+            if is_item(x):
+                missing = [arg for arg in self.args if arg not in ItemAdapter(x)]
                 if missing:
-                    raise ContractFail(
-                        "Missing fields: %s" % ", ".join(missing))
+                    missing_str = ", ".join(missing)
+                    raise ContractFail("Missing fields: %s" % missing_str)
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 6785e103d..d07c7aa62 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -4,18 +4,18 @@ extracts information from them"""
 import logging
 from collections import deque
 
-from twisted.python.failure import Failure
+from itemadapter import is_item
 from twisted.internet import defer
+from twisted.python.failure import Failure
 
-from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback
-from scrapy.utils.spider import iterate_spider_output
-from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
-from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
-from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy import signals
-from scrapy.http import Request, Response
-from scrapy.item import _BaseItem
 from scrapy.core.spidermw import SpiderMiddlewareManager
+from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
+from scrapy.http import Request, Response
+from scrapy.utils.defer import defer_result, defer_succeed, iter_errback, parallel
+from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
+from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
+from scrapy.utils.spider import iterate_spider_output
 
 
 logger = logging.getLogger(__name__)
@@ -191,7 +191,7 @@ class Scraper:
         """
         if isinstance(output, Request):
             self.crawler.engine.crawl(request=output, spider=spider)
-        elif isinstance(output, (_BaseItem, dict)):
+        elif is_item(output):
             self.slot.itemproc_size += 1
             dfd = self.itemproc.process_item(output, spider)
             dfd.addBoth(self._itemproc_finished, output, response, spider)
@@ -200,10 +200,11 @@ class Scraper:
             pass
         else:
             typename = type(output).__name__
-            logger.error('Spider must return Request, BaseItem, dict or None, '
-                         'got %(typename)r in %(request)s',
-                         {'request': request, 'typename': typename},
-                         extra={'spider': spider})
+            logger.error(
+                'Spider must return request, item, or None, got %(typename)r in %(request)s',
+                {'request': request, 'typename': typename},
+                extra={'spider': spider},
+            )
 
     def _log_download_errors(self, spider_failure, download_failure, request, spider):
         """Log and silence errors that come from the engine (typically download
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index de009082a..712572673 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -4,16 +4,18 @@ Item Exporters are used to export/serialize items into different formats.
 
 import csv
 import io
-import pprint
 import marshal
-import warnings
 import pickle
+import pprint
+import warnings
 from xml.sax.saxutils import XMLGenerator
 
-from scrapy.utils.serialize import ScrapyJSONEncoder
-from scrapy.utils.python import to_bytes, to_unicode, is_listlike
-from scrapy.item import _BaseItem
+from itemadapter import is_item, ItemAdapter
+
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.item import _BaseItem
+from scrapy.utils.python import is_listlike, to_bytes, to_unicode
+from scrapy.utils.serialize import ScrapyJSONEncoder
 
 
 __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
@@ -56,11 +58,14 @@ class BaseItemExporter:
         """Return the fields to export as an iterable of tuples
         (name, serialized_value)
         """
+        item = ItemAdapter(item)
+
         if include_empty is None:
             include_empty = self.export_empty_fields
+
         if self.fields_to_export is None:
-            if include_empty and not isinstance(item, dict):
-                field_iter = item.fields.keys()
+            if include_empty:
+                field_iter = item.field_names()
             else:
                 field_iter = item.keys()
         else:
@@ -71,8 +76,8 @@ class BaseItemExporter:
 
         for field_name in field_iter:
             if field_name in item:
-                field = {} if isinstance(item, dict) else item.fields[field_name]
-                value = self.serialize_field(field, field_name, item[field_name])
+                field_meta = item.get_field_meta(field_name)
+                value = self.serialize_field(field_meta, field_name, item[field_name])
             else:
                 value = default_value
 
@@ -297,6 +302,7 @@ class PythonItemExporter(BaseItemExporter):
 
     .. _msgpack: https://pypi.org/project/msgpack/
     """
+
     def _configure(self, options, dont_fail=False):
         self.binary = options.pop('binary', True)
         super(PythonItemExporter, self)._configure(options, dont_fail)
@@ -314,22 +320,22 @@ class PythonItemExporter(BaseItemExporter):
     def _serialize_value(self, value):
         if isinstance(value, _BaseItem):
             return self.export_item(value)
-        if isinstance(value, dict):
-            return dict(self._serialize_dict(value))
-        if is_listlike(value):
+        elif is_item(value):
+            return dict(self._serialize_item(value))
+        elif is_listlike(value):
             return [self._serialize_value(v) for v in value]
         encode_func = to_bytes if self.binary else to_unicode
         if isinstance(value, (str, bytes)):
             return encode_func(value, encoding=self.encoding)
         return value
 
-    def _serialize_dict(self, value):
-        for key, val in value.items():
+    def _serialize_item(self, item):
+        for key, value in ItemAdapter(item).items():
             key = to_bytes(key) if self.binary else key
-            yield key, self._serialize_value(val)
+            yield key, self._serialize_value(value)
 
     def export_item(self, item):
         result = dict(self._get_serialized_fields(item))
         if self.binary:
-            result = dict(self._serialize_dict(result))
+            result = dict(self._serialize_item(result))
         return result
diff --git a/scrapy/item.py b/scrapy/item.py
index 97dfed976..4ab83d1a0 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -36,7 +36,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
     """
 
     def __new__(cls, *args, **kwargs):
-        if issubclass(cls, BaseItem) and not (issubclass(cls, Item) or issubclass(cls, DictItem)):
+        if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
             warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
                  ScrapyDeprecationWarning, stacklevel=2)
         return super(BaseItem, cls).__new__(cls, *args, **kwargs)
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 21c4fb376..18f57945f 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -6,6 +6,8 @@ See documentation in docs/topics/loaders.rst
 from collections import defaultdict
 from contextlib import suppress
 
+from itemadapter import ItemAdapter
+
 from scrapy.item import Item
 from scrapy.loader.common import wrap_loader_context
 from scrapy.loader.processors import Identity
@@ -44,7 +46,7 @@ class ItemLoader:
         self._local_item = context['item'] = item
         self._local_values = defaultdict(list)
         # values from initial item
-        for field_name, value in item.items():
+        for field_name, value in ItemAdapter(item).items():
             self._values[field_name] += arg_to_iter(value)
 
     @property
@@ -127,13 +129,12 @@ class ItemLoader:
         return value
 
     def load_item(self):
-        item = self.item
+        adapter = ItemAdapter(self.item)
         for field_name in tuple(self._values):
             value = self.get_output_value(field_name)
             if value is not None:
-                item[field_name] = value
-
-        return item
+                adapter[field_name] = value
+        return adapter.item
 
     def get_output_value(self, field_name):
         proc = self.get_output_processor(field_name)
@@ -174,11 +175,8 @@ class ItemLoader:
                                     value, type(e).__name__, str(e)))
 
     def _get_item_field_attr(self, field_name, key, default=None):
-        if isinstance(self.item, Item):
-            value = self.item.fields[field_name].get(key, default)
-        else:
-            value = default
-        return value
+        field_meta = ItemAdapter(self.item).get_field_meta(field_name)
+        return field_meta.get(key, default)
 
     def _check_selector_method(self):
         if self.selector is None:
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 7d86d0d56..487382a38 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -10,24 +10,26 @@ import mimetypes
 import os
 import time
 from collections import defaultdict
-from email.utils import parsedate_tz, mktime_tz
+from contextlib import suppress
+from email.utils import mktime_tz, parsedate_tz
 from ftplib import FTP
 from io import BytesIO
 from urllib.parse import urlparse
 
+from itemadapter import ItemAdapter
 from twisted.internet import defer, threads
 
+from scrapy.exceptions import IgnoreRequest, NotConfigured
+from scrapy.http import Request
 from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
-from scrapy.exceptions import NotConfigured, IgnoreRequest
-from scrapy.http import Request
-from scrapy.utils.misc import md5sum
-from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.python import to_bytes
-from scrapy.utils.request import referer_str
 from scrapy.utils.boto import is_botocore
 from scrapy.utils.datatypes import CaselessDict
 from scrapy.utils.ftp import ftp_store_file
+from scrapy.utils.log import failure_to_exc_info
+from scrapy.utils.misc import md5sum
+from scrapy.utils.python import to_bytes
+from scrapy.utils.request import referer_str
 
 
 logger = logging.getLogger(__name__)
@@ -517,7 +519,8 @@ class FilesPipeline(MediaPipeline):
 
     # Overridable Interface
     def get_media_requests(self, item, info):
-        return [Request(x) for x in item.get(self.files_urls_field, [])]
+        urls = ItemAdapter(item).get(self.files_urls_field, [])
+        return [Request(u) for u in urls]
 
     def file_downloaded(self, response, request, info):
         path = self.file_path(request, response=response, info=info)
@@ -528,8 +531,8 @@ class FilesPipeline(MediaPipeline):
         return checksum
 
     def item_completed(self, results, item, info):
-        if isinstance(item, dict) or self.files_result_field in item.fields:
-            item[self.files_result_field] = [x for ok, x in results if ok]
+        with suppress(KeyError):
+            ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok]
         return item
 
     def file_path(self, request, response=None, info=None):
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index aeb520442..46f2bfb58 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -5,17 +5,19 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
+from contextlib import suppress
 from io import BytesIO
 
+from itemadapter import ItemAdapter
 from PIL import Image
 
+from scrapy.exceptions import DropItem
+from scrapy.http import Request
+from scrapy.pipelines.files import FileException, FilesPipeline
+# TODO: from scrapy.pipelines.media import MediaPipeline
+from scrapy.settings import Settings
 from scrapy.utils.misc import md5sum
 from scrapy.utils.python import to_bytes
-from scrapy.http import Request
-from scrapy.settings import Settings
-from scrapy.exceptions import DropItem
-# TODO: from scrapy.pipelines.media import MediaPipeline
-from scrapy.pipelines.files import FileException, FilesPipeline
 
 
 class NoimagesDrop(DropItem):
@@ -157,11 +159,12 @@ class ImagesPipeline(FilesPipeline):
         return image, buf
 
     def get_media_requests(self, item, info):
-        return [Request(x) for x in item.get(self.images_urls_field, [])]
+        urls = ItemAdapter(item).get(self.images_urls_field, [])
+        return [Request(u) for u in urls]
 
     def item_completed(self, results, item, info):
-        if isinstance(item, dict) or self.images_result_field in item.fields:
-            item[self.images_result_field] = [x for ok, x in results if ok]
+        with suppress(KeyError):
+            ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok]
         return item
 
     def file_path(self, request, response=None, info=None):
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 3ff5a8ad8..10de119ce 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/shell.rst
 import os
 import signal
 
+from itemadapter import is_item
 from twisted.internet import threads, defer
 from twisted.python import threadable
 from w3lib.url import any_to_uri
@@ -13,20 +14,18 @@ from w3lib.url import any_to_uri
 from scrapy.crawler import Crawler
 from scrapy.exceptions import IgnoreRequest
 from scrapy.http import Request, Response
-from scrapy.item import _BaseItem
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
-from scrapy.utils.console import start_python_console
+from scrapy.utils.conf import get_config
+from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
-from scrapy.utils.conf import get_config
-from scrapy.utils.console import DEFAULT_PYTHON_SHELLS
 
 
 class Shell:
 
-    relevant_classes = (Crawler, Spider, Request, Response, _BaseItem, Settings)
+    relevant_classes = (Crawler, Spider, Request, Response, Settings)
 
     def __init__(self, crawler, update_vars=None, code=None):
         self.crawler = crawler
@@ -154,7 +153,7 @@ class Shell:
         return "\n".join("[s] %s" % line for line in b)
 
     def _is_relevant(self, value):
-        return isinstance(value, self.relevant_classes)
+        return isinstance(value, self.relevant_classes) or is_item(value)
 
 
 def inspect_response(response, spider):
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index a4ff8010d..5aad7398a 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -31,7 +31,7 @@ class XMLFeedSpider(Spider):
         processing required before returning the results to the framework core,
         for example setting the item GUIDs. It receives a list of results and
         the response which originated that results. It must return a list of
-        results (Items or Requests).
+        results (items or requests).
         """
         return results
 
diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
index 6490f52a7..bd09890fe 100644
--- a/scrapy/templates/project/module/middlewares.py.tmpl
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -5,6 +5,9 @@
 
 from scrapy import signals
 
+# useful for handling different item types with a single interface
+from itemadapter import is_item, ItemAdapter
+
 
 class ${ProjectName}SpiderMiddleware:
     # Not all methods need to be defined. If a method is not defined,
@@ -29,7 +32,7 @@ class ${ProjectName}SpiderMiddleware:
         # Called with the results returned from the Spider, after
         # it has processed the response.
 
-        # Must return an iterable of Request, dict or Item objects.
+        # Must return an iterable of Request, or item objects.
         for i in result:
             yield i
 
@@ -37,8 +40,7 @@ class ${ProjectName}SpiderMiddleware:
         # Called when a spider or process_spider_input() method
         # (from other spider middleware) raises an exception.
 
-        # Should return either None or an iterable of Request, dict
-        # or Item objects.
+        # Should return either None or an iterable of Request or item objects.
         pass
 
     def process_start_requests(self, start_requests, spider):
diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl
index ce0edd335..e845f43e9 100644
--- a/scrapy/templates/project/module/pipelines.py.tmpl
+++ b/scrapy/templates/project/module/pipelines.py.tmpl
@@ -4,6 +4,10 @@
 # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
 
 
+# useful for handling different item types with a single interface
+from itemadapter import ItemAdapter
+
+
 class ${ProjectName}Pipeline:
     def process_item(self, item, spider):
         return item
diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py
index bf73dfa18..dc9604578 100644
--- a/scrapy/utils/serialize.py
+++ b/scrapy/utils/serialize.py
@@ -2,10 +2,10 @@ import json
 import datetime
 import decimal
 
+from itemadapter import is_item, ItemAdapter
 from twisted.internet import defer
 
 from scrapy.http import Request, Response
-from scrapy.item import _BaseItem
 
 
 class ScrapyJSONEncoder(json.JSONEncoder):
@@ -26,8 +26,8 @@ class ScrapyJSONEncoder(json.JSONEncoder):
             return str(o)
         elif isinstance(o, defer.Deferred):
             return str(o)
-        elif isinstance(o, _BaseItem):
-            return dict(o)
+        elif is_item(o):
+            return ItemAdapter(o).asdict()
         elif isinstance(o, Request):
             return "<%s %s %s>" % (type(o).__name__, o.method, o.url)
         elif isinstance(o, Response):
diff --git a/setup.py b/setup.py
index 71dc3232d..5a99fd1bf 100644
--- a/setup.py
+++ b/setup.py
@@ -80,6 +80,7 @@ setup(
         'w3lib>=1.17.0',
         'zope.interface>=4.1.3',
         'protego>=0.1.15',
+        'itemadapter>=0.1.0',
     ],
     extras_require=extras_require,
 )
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 91fa1c5b5..dacb86e56 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,4 +1,6 @@
 # Tests requirements
+attrs
+dataclasses; python_version == '3.6'
 jmespath
 mitmproxy; python_version >= '3.6'
 mitmproxy<4.0.0; python_version < '3.6'
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 6696ee52e..1b848ac72 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -16,9 +16,11 @@ import sys
 from collections import defaultdict
 from urllib.parse import urlparse
 
+import attr
+from itemadapter import ItemAdapter
 from pydispatch import dispatcher
 from testfixtures import LogCapture
-from twisted.internet import reactor, defer
+from twisted.internet import defer, reactor
 from twisted.trial import unittest
 from twisted.web import server, static, util
 
@@ -32,7 +34,7 @@ from scrapy.spiders import Spider
 from scrapy.utils.signal import disconnect_all
 from scrapy.utils.test import get_crawler
 
-from tests import tests_datadir, get_testdata
+from tests import get_testdata, tests_datadir
 
 
 class TestItem(Item):
@@ -41,6 +43,13 @@ class TestItem(Item):
     price = Field()
 
 
+@attr.s
+class AttrsItem:
+    name = attr.ib(default="")
+    url = attr.ib(default="")
+    price = attr.ib(default=0)
+
+
 class TestSpider(Spider):
     name = "scrapytest.org"
     allowed_domains = ["scrapytest.org", "localhost"]
@@ -79,6 +88,27 @@ class DictItemsSpider(TestSpider):
     item_cls = dict
 
 
+class AttrsItemsSpider(TestSpider):
+    item_class = AttrsItem
+
+
+try:
+    from dataclasses import make_dataclass
+except ImportError:
+    DataClassItemsSpider = None
+else:
+    TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)])
+
+    class DataClassItemsSpider(DictItemsSpider):
+        def parse_item(self, response):
+            item = super().parse_item(response)
+            return TestDataClass(
+                name=item.get('name'),
+                url=item.get('url'),
+                price=item.get('price'),
+            )
+
+
 class ItemZeroDivisionErrorSpider(TestSpider):
     custom_settings = {
         "ITEM_PIPELINES": {
@@ -204,7 +234,10 @@ class EngineTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_crawler(self):
-        for spider in TestSpider, DictItemsSpider:
+
+        for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
+            if spider is None:
+                continue
             self.run = CrawlerRun(spider)
             yield self.run.run()
             self._assert_visited_urls()
@@ -281,6 +314,7 @@ class EngineTest(unittest.TestCase):
     def _assert_scraped_items(self):
         self.assertEqual(2, len(self.run.itemresp))
         for item, response in self.run.itemresp:
+            item = ItemAdapter(item)
             self.assertEqual(item['url'], response.url)
             if 'item1.html' in item['url']:
                 self.assertEqual('Item 1 name', item['name'])
diff --git a/tests/test_loader.py b/tests/test_loader.py
index f14714c75..8a9c6fca9 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -1,6 +1,9 @@
 from functools import partial
 import unittest
 
+import attr
+from itemadapter import ItemAdapter
+
 from scrapy.http import HtmlResponse
 from scrapy.item import Item, Field
 from scrapy.loader import ItemLoader
@@ -9,6 +12,13 @@ from scrapy.loader.processors import (Compose, Identity, Join,
 from scrapy.selector import Selector
 
 
+try:
+    from dataclasses import make_dataclass, field as dataclass_field
+except ImportError:
+    make_dataclass = None
+    dataclass_field = None
+
+
 # test items
 class NameItem(Item):
     name = Field()
@@ -28,6 +38,11 @@ class TestNestedItem(Item):
     image = Field()
 
 
+@attr.s
+class AttrsNameItem:
+    name = attr.ib(default="")
+
+
 # test item loaders
 class NameItemLoader(ItemLoader):
     default_item_class = TestItem
@@ -466,7 +481,7 @@ class InitializationTestMixin:
         il = ItemLoader(item=input_item)
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo']})
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo']})
 
     def test_keep_list(self):
         """Loaded item should contain values from the initial item"""
@@ -474,7 +489,7 @@ class InitializationTestMixin:
         il = ItemLoader(item=input_item)
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar']})
 
     def test_add_value_singlevalue_singlevalue(self):
         """Values added after initialization should be appended"""
@@ -483,7 +498,7 @@ class InitializationTestMixin:
         il.add_value('name', 'bar')
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar']})
 
     def test_add_value_singlevalue_list(self):
         """Values added after initialization should be appended"""
@@ -492,7 +507,7 @@ class InitializationTestMixin:
         il.add_value('name', ['item', 'loader'])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo', 'item', 'loader']})
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'item', 'loader']})
 
     def test_add_value_list_singlevalue(self):
         """Values added after initialization should be appended"""
@@ -501,7 +516,7 @@ class InitializationTestMixin:
         il.add_value('name', 'qwerty')
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'qwerty']})
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar', 'qwerty']})
 
     def test_add_value_list_list(self):
         """Values added after initialization should be appended"""
@@ -510,7 +525,7 @@ class InitializationTestMixin:
         il.add_value('name', ['item', 'loader'])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'item', 'loader']})
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar', 'item', 'loader']})
 
     def test_get_output_value_singlevalue(self):
         """Getting output value must not remove value from item"""
@@ -519,7 +534,7 @@ class InitializationTestMixin:
         self.assertEqual(il.get_output_value('name'), ['foo'])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(loaded_item, dict({'name': ['foo']}))
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({'name': ['foo']}))
 
     def test_get_output_value_list(self):
         """Getting output value must not remove value from item"""
@@ -528,7 +543,7 @@ class InitializationTestMixin:
         self.assertEqual(il.get_output_value('name'), ['foo', 'bar'])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(loaded_item, dict({'name': ['foo', 'bar']}))
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({'name': ['foo', 'bar']}))
 
     def test_values_single(self):
         """Values from initial item must be added to loader._values"""
@@ -551,6 +566,22 @@ class InitializationFromItemTest(InitializationTestMixin, unittest.TestCase):
     item_class = NameItem
 
 
+class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase):
+    item_class = AttrsNameItem
+
+
+@unittest.skipIf(not make_dataclass, "dataclasses module is not available")
+class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if make_dataclass:
+            self.item_class = make_dataclass(
+                "TestDataClass",
+                [("name", list, dataclass_field(default_factory=list))],
+            )
+
+
 class BaseNoInputReprocessingLoader(ItemLoader):
     title_in = MapCompose(str.upper)
     title_out = TakeFirst()
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 6bbcbc2e9..a023dfcc8 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -2,22 +2,41 @@ import os
 import random
 import time
 from io import BytesIO
-from tempfile import mkdtemp
 from shutil import rmtree
-from unittest import mock
+from tempfile import mkdtemp
+from unittest import mock, skipIf
 from urllib.parse import urlparse
 
-from twisted.trial import unittest
+import attr
+from itemadapter import ItemAdapter
 from twisted.internet import defer
+from twisted.trial import unittest
 
-from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore, FTPFilesStore
-from scrapy.item import Item, Field
 from scrapy.http import Request, Response
+from scrapy.item import Field, Item
+from scrapy.pipelines.files import (
+    FilesPipeline,
+    FSFilesStore,
+    FTPFilesStore,
+    GCSFilesStore,
+    S3FilesStore,
+)
 from scrapy.settings import Settings
-from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
-from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete
-from scrapy.utils.test import get_ftp_content_and_delete
 from scrapy.utils.boto import is_botocore
+from scrapy.utils.test import (
+    assert_aws_environ,
+    assert_gcs_environ,
+    get_ftp_content_and_delete,
+    get_gcs_content_and_delete,
+    get_s3_content_and_delete,
+)
+
+
+try:
+    from dataclasses import make_dataclass, field as dataclass_field
+except ImportError:
+    make_dataclass = None
+    dataclass_field = None
 
 
 def _mocked_download_func(request, info):
@@ -143,43 +162,88 @@ class FilesPipelineTestCase(unittest.TestCase):
             p.stop()
 
 
-class FilesPipelineTestCaseFields(unittest.TestCase):
+class FilesPipelineTestCaseFieldsMixin:
 
     def test_item_fields_default(self):
-        class TestItem(Item):
-            name = Field()
-            file_urls = Field()
-            files = Field()
-
-        for cls in TestItem, dict:
-            url = 'http://www.example.com/files/1.txt'
-            item = cls({'name': 'item1', 'file_urls': [url]})
-            pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'}))
-            requests = list(pipeline.get_media_requests(item, None))
-            self.assertEqual(requests[0].url, url)
-            results = [(True, {'url': url})]
-            pipeline.item_completed(results, item, None)
-            self.assertEqual(item['files'], [results[0][1]])
+        url = 'http://www.example.com/files/1.txt'
+        item = self.item_class(name='item1', file_urls=[url])
+        pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'}))
+        requests = list(pipeline.get_media_requests(item, None))
+        self.assertEqual(requests[0].url, url)
+        results = [(True, {'url': url})]
+        item = pipeline.item_completed(results, item, None)
+        files = ItemAdapter(item).get("files")
+        self.assertEqual(files, [results[0][1]])
+        self.assertIsInstance(item, self.item_class)
 
     def test_item_fields_override_settings(self):
-        class TestItem(Item):
-            name = Field()
-            files = Field()
-            stored_file = Field()
+        url = 'http://www.example.com/files/1.txt'
+        item = self.item_class(name='item1', custom_file_urls=[url])
+        pipeline = FilesPipeline.from_settings(Settings({
+            'FILES_STORE': 's3://example/files/',
+            'FILES_URLS_FIELD': 'custom_file_urls',
+            'FILES_RESULT_FIELD': 'custom_files'
+        }))
+        requests = list(pipeline.get_media_requests(item, None))
+        self.assertEqual(requests[0].url, url)
+        results = [(True, {'url': url})]
+        item = pipeline.item_completed(results, item, None)
+        custom_files = ItemAdapter(item).get("custom_files")
+        self.assertEqual(custom_files, [results[0][1]])
+        self.assertIsInstance(item, self.item_class)
 
-        for cls in TestItem, dict:
-            url = 'http://www.example.com/files/1.txt'
-            item = cls({'name': 'item1', 'files': [url]})
-            pipeline = FilesPipeline.from_settings(Settings({
-                'FILES_STORE': 's3://example/files/',
-                'FILES_URLS_FIELD': 'files',
-                'FILES_RESULT_FIELD': 'stored_file'
-            }))
-            requests = list(pipeline.get_media_requests(item, None))
-            self.assertEqual(requests[0].url, url)
-            results = [(True, {'url': url})]
-            pipeline.item_completed(results, item, None)
-            self.assertEqual(item['stored_file'], [results[0][1]])
+
+class FilesPipelineTestCaseFieldsDict(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = dict
+
+
+class FilesPipelineTestItem(Item):
+    name = Field()
+    # default fields
+    file_urls = Field()
+    files = Field()
+    # overridden fields
+    custom_file_urls = Field()
+    custom_files = Field()
+
+
+class FilesPipelineTestCaseFieldsItem(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = FilesPipelineTestItem
+
+
+@skipIf(not make_dataclass, "dataclasses module is not available")
+class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if make_dataclass:
+            self.item_class = make_dataclass(
+                "FilesPipelineTestDataClass",
+                [
+                    ("name", str),
+                    # default fields
+                    ("file_urls", list, dataclass_field(default_factory=list)),
+                    ("files", list, dataclass_field(default_factory=list)),
+                    # overridden fields
+                    ("custom_file_urls", list, dataclass_field(default_factory=list)),
+                    ("custom_files", list, dataclass_field(default_factory=list)),
+                ],
+            )
+
+
+@attr.s
+class FilesPipelineTestAttrsItem:
+    name = attr.ib(default="")
+    # default fields
+    file_urls = attr.ib(default=lambda: [])
+    files = attr.ib(default=lambda: [])
+    # overridden fields
+    custom_file_urls = attr.ib(default=lambda: [])
+    custom_files = attr.ib(default=lambda: [])
+
+
+class FilesPipelineTestCaseFieldsAttrsItem(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = FilesPipelineTestAttrsItem
 
 
 class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 8ef27fce7..082e9ee21 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -1,17 +1,28 @@
-import io
 import hashlib
+import io
 import random
-from tempfile import mkdtemp
 from shutil import rmtree
+from tempfile import mkdtemp
+from unittest import skipIf
 
+import attr
+from itemadapter import ItemAdapter
 from twisted.trial import unittest
 
-from scrapy.item import Item, Field
 from scrapy.http import Request, Response
-from scrapy.settings import Settings
+from scrapy.item import Field, Item
 from scrapy.pipelines.images import ImagesPipeline
+from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes
 
+
+try:
+    from dataclasses import make_dataclass, field as dataclass_field
+except ImportError:
+    make_dataclass = None
+    dataclass_field = None
+
+
 skip = False
 try:
     from PIL import Image
@@ -124,43 +135,89 @@ class DeprecatedImagesPipeline(ImagesPipeline):
         return 'thumbsup/%s/%s.jpg' % (thumb_id, thumb_guid)
 
 
-class ImagesPipelineTestCaseFields(unittest.TestCase):
+class ImagesPipelineTestCaseFieldsMixin:
 
     def test_item_fields_default(self):
-        class TestItem(Item):
-            name = Field()
-            image_urls = Field()
-            images = Field()
-
-        for cls in TestItem, dict:
-            url = 'http://www.example.com/images/1.jpg'
-            item = cls({'name': 'item1', 'image_urls': [url]})
-            pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': 's3://example/images/'}))
-            requests = list(pipeline.get_media_requests(item, None))
-            self.assertEqual(requests[0].url, url)
-            results = [(True, {'url': url})]
-            pipeline.item_completed(results, item, None)
-            self.assertEqual(item['images'], [results[0][1]])
+        url = 'http://www.example.com/images/1.jpg'
+        item = self.item_class(name='item1', image_urls=[url])
+        pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': 's3://example/images/'}))
+        requests = list(pipeline.get_media_requests(item, None))
+        self.assertEqual(requests[0].url, url)
+        results = [(True, {'url': url})]
+        item = pipeline.item_completed(results, item, None)
+        images = ItemAdapter(item).get("images")
+        self.assertEqual(images, [results[0][1]])
+        self.assertIsInstance(item, self.item_class)
 
     def test_item_fields_override_settings(self):
-        class TestItem(Item):
-            name = Field()
-            image = Field()
-            stored_image = Field()
+        url = 'http://www.example.com/images/1.jpg'
+        item = self.item_class(name='item1', custom_image_urls=[url])
+        pipeline = ImagesPipeline.from_settings(Settings({
+            'IMAGES_STORE': 's3://example/images/',
+            'IMAGES_URLS_FIELD': 'custom_image_urls',
+            'IMAGES_RESULT_FIELD': 'custom_images'
+        }))
+        requests = list(pipeline.get_media_requests(item, None))
+        self.assertEqual(requests[0].url, url)
+        results = [(True, {'url': url})]
+        item = pipeline.item_completed(results, item, None)
+        custom_images = ItemAdapter(item).get("custom_images")
+        self.assertEqual(custom_images, [results[0][1]])
+        self.assertIsInstance(item, self.item_class)
 
-        for cls in TestItem, dict:
-            url = 'http://www.example.com/images/1.jpg'
-            item = cls({'name': 'item1', 'image': [url]})
-            pipeline = ImagesPipeline.from_settings(Settings({
-                'IMAGES_STORE': 's3://example/images/',
-                'IMAGES_URLS_FIELD': 'image',
-                'IMAGES_RESULT_FIELD': 'stored_image'
-            }))
-            requests = list(pipeline.get_media_requests(item, None))
-            self.assertEqual(requests[0].url, url)
-            results = [(True, {'url': url})]
-            pipeline.item_completed(results, item, None)
-            self.assertEqual(item['stored_image'], [results[0][1]])
+
+class ImagesPipelineTestCaseFieldsDict(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = dict
+
+
+class ImagesPipelineTestItem(Item):
+    name = Field()
+    # default fields
+    image_urls = Field()
+    images = Field()
+    # overridden fields
+    custom_image_urls = Field()
+    custom_images = Field()
+
+
+class ImagesPipelineTestCaseFieldsItem(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = ImagesPipelineTestItem
+
+
+@skipIf(not make_dataclass, "dataclasses module is not available")
+class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if make_dataclass:
+            self.item_class = make_dataclass(
+                "FilesPipelineTestDataClass",
+                [
+                    ("name", str),
+                    # default fields
+                    ("image_urls", list, dataclass_field(default_factory=list)),
+                    ("images", list, dataclass_field(default_factory=list)),
+                    # overridden fields
+                    ("custom_image_urls", list, dataclass_field(default_factory=list)),
+                    ("custom_images", list, dataclass_field(default_factory=list)),
+                ],
+            )
+
+
+@attr.s
+class ImagesPipelineTestAttrsItem:
+    name = attr.ib(default="")
+    # default fields
+    image_urls = attr.ib(default=lambda: [])
+    images = attr.ib(default=lambda: [])
+    # overridden fields
+    custom_image_urls = attr.ib(default=lambda: [])
+    custom_images = attr.ib(default=lambda: [])
+
+
+class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = ImagesPipelineTestAttrsItem
 
 
 class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py
index 6dc117779..daf022aee 100644
--- a/tests/test_utils_serialize.py
+++ b/tests/test_utils_serialize.py
@@ -1,18 +1,25 @@
+import datetime
 import json
 import unittest
-import datetime
 from decimal import Decimal
 
+import attr
 from twisted.internet import defer
 
-from scrapy.utils.serialize import ScrapyJSONEncoder
 from scrapy.http import Request, Response
+from scrapy.utils.serialize import ScrapyJSONEncoder
+
+
+try:
+    from dataclasses import make_dataclass
+except ImportError:
+    make_dataclass = None
 
 
 class JsonEncoderTestCase(unittest.TestCase):
 
     def setUp(self):
-        self.encoder = ScrapyJSONEncoder()
+        self.encoder = ScrapyJSONEncoder(sort_keys=True)
 
     def test_encode_decode(self):
         dt = datetime.datetime(2010, 1, 2, 10, 11, 12)
@@ -31,7 +38,8 @@ class JsonEncoderTestCase(unittest.TestCase):
         for input, output in [('foo', 'foo'), (d, ds), (t, ts), (dt, dts),
                               (dec, decs), (['foo', d], ['foo', ds]), (s, ss),
                               (dt_set, dt_sets)]:
-            self.assertEqual(self.encoder.encode(input), json.dumps(output))
+            self.assertEqual(self.encoder.encode(input),
+                             json.dumps(output, sort_keys=True))
 
     def test_encode_deferred(self):
         self.assertIn('Deferred', self.encoder.encode(defer.Deferred()))
@@ -47,3 +55,30 @@ class JsonEncoderTestCase(unittest.TestCase):
         rs = self.encoder.encode(r)
         self.assertIn(r.url, rs)
         self.assertIn(str(r.status), rs)
+
+    @unittest.skipIf(not make_dataclass, "No dataclass support")
+    def test_encode_dataclass_item(self):
+        TestDataClass = make_dataclass(
+            "TestDataClass",
+            [("name", str), ("url", str), ("price", int)],
+        )
+        item = TestDataClass(name="Product", url="http://product.org", price=1)
+        encoded = self.encoder.encode(item)
+        self.assertEqual(
+            encoded,
+            '{"name": "Product", "price": 1, "url": "http://product.org"}'
+        )
+
+    def test_encode_attrs_item(self):
+        @attr.s
+        class AttrsItem:
+            name = attr.ib(type=str)
+            url = attr.ib(type=str)
+            price = attr.ib(type=int)
+
+        item = AttrsItem(name="Product", url="http://product.org", price=1)
+        encoded = self.encoder.encode(item)
+        self.assertEqual(
+            encoded,
+            '{"name": "Product", "price": 1, "url": "http://product.org"}'
+        )
diff --git a/tox.ini b/tox.ini
index 69b1bdfdd..4c790158d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -37,7 +37,7 @@ deps =
     pytest-flake8
 commands =
     py.test --flake8 {posargs:docs scrapy tests}
-    
+
 [testenv:pylint]
 basepython = python3
 deps =
@@ -62,6 +62,7 @@ deps =
     -ctests/constraints.txt
     cryptography==2.0
     cssselect==0.9.1
+    itemadapter==0.1.0
     lxml==3.5.0
     parsel==1.5.0
     Protego==0.1.15

From 214da8e5235c9676bc4b0427c8bf58f328fe6570 Mon Sep 17 00:00:00 2001
From: Ram Rachum <ram@rachum.com>
Date: Wed, 17 Jun 2020 13:50:54 +0300
Subject: [PATCH 2881/2974] Use chain.from_iterable in python.py

---
 scrapy/utils/python.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 9c1f3c2fe..afa8a8135 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -334,10 +334,10 @@ class MutableChain:
     """
 
     def __init__(self, *args):
-        self.data = chain(*args)
+        self.data = chain.from_iterable(args)
 
     def extend(self, *iterables):
-        self.data = chain(self.data, *iterables)
+        self.data = chain(self.data, chain.from_iterable(iterables))
 
     def __iter__(self):
         return self

From 3d027fb578532d504b3dbfaa77a06c3560f85d3c Mon Sep 17 00:00:00 2001
From: Stas Glubokiy <glubokiy.stas@gmail.com>
Date: Wed, 17 Jun 2020 18:08:14 +0300
Subject: [PATCH 2882/2974] Fix missing storage.store calls in
 FeedExporter.close_spider (#4626)

---
 scrapy/extensions/feedexport.py |  4 ++-
 tests/test_feedexport.py        | 49 ++++++++++++++++++++++++++++++++-
 2 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 998d2a5d1..30e6349d6 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -270,7 +270,9 @@ class FeedExporter:
             if not slot.itemcount and not slot.store_empty:
                 # We need to call slot.storage.store nonetheless to get the file
                 # properly closed.
-                return defer.maybeDeferred(slot.storage.store, slot.file)
+                d = defer.maybeDeferred(slot.storage.store, slot.file)
+                deferred_list.append(d)
+                continue
             slot.finish_exporting()
             logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
             log_args = {'format': slot.format,
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 8eeb29b6d..f7013bc44 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -7,6 +7,7 @@ import string
 import tempfile
 import warnings
 from io import BytesIO
+from logging import getLogger
 from pathlib import Path
 from string import ascii_letters, digits
 from unittest import mock
@@ -14,9 +15,11 @@ from urllib.parse import urljoin, urlparse, quote
 from urllib.request import pathname2url
 
 import lxml.etree
+from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial import unittest
-from w3lib.url import path_to_file_uri
+from w3lib.url import file_uri_to_path, path_to_file_uri
+from zope.interface import implementer
 from zope.interface.verify import verifyObject
 
 import scrapy
@@ -390,6 +393,25 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
     pass
 
 
+@implementer(IFeedStorage)
+class LogOnStoreFileStorage:
+    """
+    This storage logs inside `store` method.
+    It can be used to make sure `store` method is invoked.
+    """
+
+    def __init__(self, uri):
+        self.path = file_uri_to_path(uri)
+        self.logger = getLogger()
+
+    def open(self, spider):
+        return tempfile.NamedTemporaryFile(prefix='feed-')
+
+    def store(self, file):
+        self.logger.info('Storage.store is called')
+        file.close()
+
+
 class FeedExportTest(unittest.TestCase):
 
     class MyItem(scrapy.Item):
@@ -426,11 +448,17 @@ class FeedExportTest(unittest.TestCase):
                 yield runner.crawl(spider_cls)
 
             for file_path, feed in FEEDS.items():
+                if not os.path.exists(str(file_path)):
+                    continue
+
                 with open(str(file_path), 'rb') as f:
                     content[feed['format']] = f.read()
 
         finally:
             for file_path in FEEDS.keys():
+                if not os.path.exists(str(file_path)):
+                    continue
+
                 os.remove(str(file_path))
 
         return content
@@ -623,6 +651,25 @@ class FeedExportTest(unittest.TestCase):
             data = yield self.exported_no_data(settings)
             self.assertEqual(data[fmt], expctd)
 
+    @defer.inlineCallbacks
+    def test_export_no_items_multiple_feeds(self):
+        """ Make sure that `storage.store` is called for every feed. """
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'json'},
+                self._random_temp_filename(): {'format': 'xml'},
+                self._random_temp_filename(): {'format': 'csv'},
+            },
+            'FEED_STORAGES': {'file': 'tests.test_feedexport.LogOnStoreFileStorage'},
+            'FEED_STORE_EMPTY': False
+        }
+
+        with LogCapture() as log:
+            yield self.exported_no_data(settings)
+
+        print(log)
+        self.assertEqual(str(log).count('Storage.store is called'), 3)
+
     @defer.inlineCallbacks
     def test_export_multiple_item_classes(self):
 

From 5d541731870eaaa3dd658673b82a7d0ca056f689 Mon Sep 17 00:00:00 2001
From: Devi Sandeep <sandeep0138@gmail.com>
Date: Thu, 18 Jun 2020 05:01:38 -0500
Subject: [PATCH 2883/2974] Update docs on accessing callback arguments in
 errback (#4634)

---
 docs/topics/request-response.rst | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index bbd715766..d88d40b00 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -189,6 +189,10 @@ Request objects
         cloned using the ``copy()`` or ``replace()`` methods, and can also be
         accessed, in your spider, from the ``response.cb_kwargs`` attribute.
 
+        In case of a failure to process the request, this dict can be accessed as
+        ``failure.request.cb_kwargs`` in the request's errback. For more information,
+        see :ref:`topics-request-response-ref-accessing-callback-arguments-in-errback`.
+
     .. method:: Request.copy()
 
        Return a new Request which is a copy of this Request. See also:
@@ -312,6 +316,31 @@ errors if needed::
                 request = failure.request
                 self.logger.error('TimeoutError on %s', request.url)
 
+.. _topics-request-response-ref-accessing-callback-arguments-in-errback:
+
+Accessing additional data in errback functions
+----------------------------------------------
+
+In case of a failure to process the request, you may be interested in
+accessing arguments to the callback functions so you can process further
+based on the arguments in the errback. The following example shows how to
+achieve this by using ``Failure.request.cb_kwargs``::
+
+    def parse(self, response):
+        request = scrapy.Request('http://www.example.com/index.html',
+                                 callback=self.parse_page2,
+                                 errback=self.errback_page2,
+                                 cb_kwargs=dict(main_url=response.url))
+        yield request
+
+    def parse_page2(self, response, main_url):
+        pass
+
+    def errback_page2(self, failure):
+        yield dict(
+            main_url=failure.request.cb_kwargs['main_url'],
+        )
+
 .. _topics-request-meta:
 
 Request.meta special keys

From 7babf359e0221613b872b5b204e7c523b7b84486 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 18 Jun 2020 13:52:04 -0300
Subject: [PATCH 2884/2974] Typing: Tox env, CI job

---
 .gitignore  |   1 +
 .travis.yml |   2 +
 setup.cfg   | 171 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 tox.ini     |   7 +++
 4 files changed, 181 insertions(+)

diff --git a/.gitignore b/.gitignore
index ff6e2ea65..83a2569dd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,7 @@ htmlcov/
 .pytest_cache/
 .coverage.*
 .cache/
+.mypy_cache/
 
 # Windows
 Thumbs.db
diff --git a/.travis.yml b/.travis.yml
index e44f85237..b403ac54c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,6 +15,8 @@ matrix:
       python: 3.8
     - env: TOXENV=docs
       python: 3.7  # Keep in sync with .readthedocs.yml
+    - env: TOXENV=typing
+      python: 3.8
 
     - env: TOXENV=pypy3
     - env: TOXENV=pinned
diff --git a/setup.cfg b/setup.cfg
index 2296a1052..a9138c1c0 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -3,3 +3,174 @@ doc_files = docs AUTHORS INSTALL LICENSE README.rst
 
 [bdist_wheel]
 universal=1
+
+[mypy]
+ignore_missing_imports = true
+follow_imports = skip
+
+# FIXME: remove the following sections once the issues are solved
+
+[mypy-scrapy]
+ignore_errors = True
+
+[mypy-scrapy._monkeypatches]
+ignore_errors = True
+
+[mypy-scrapy.commands]
+ignore_errors = True
+
+[mypy-scrapy.commands.bench]
+ignore_errors = True
+
+[mypy-scrapy.commands.parse]
+ignore_errors = True
+
+[mypy-scrapy.downloadermiddlewares.httpproxy]
+ignore_errors = True
+
+[mypy-scrapy.contracts]
+ignore_errors = True
+
+[mypy-scrapy.core.spidermw]
+ignore_errors = True
+
+[mypy-scrapy.interfaces]
+ignore_errors = True
+
+[mypy-scrapy.item]
+ignore_errors = True
+
+[mypy-scrapy.http.cookies]
+ignore_errors = True
+
+[mypy-scrapy.mail]
+ignore_errors = True
+
+[mypy-scrapy.pipelines.images]
+ignore_errors = True
+
+[mypy-scrapy.settings.default_settings]
+ignore_errors = True
+
+[mypy-scrapy.spidermiddlewares.referer]
+ignore_errors = True
+
+[mypy-scrapy.utils.httpobj]
+ignore_errors = True
+
+[mypy-scrapy.utils.request]
+ignore_errors = True
+
+[mypy-scrapy.utils.response]
+ignore_errors = True
+
+[mypy-scrapy.utils.spider]
+ignore_errors = True
+
+[mypy-scrapy.utils.trackref]
+ignore_errors = True
+
+[mypy-tests.mocks.dummydbm]
+ignore_errors = True
+
+[mypy-tests.spiders]
+ignore_errors = True
+
+[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.exception]
+ignore_errors = True
+
+[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.normal]
+ignore_errors = True
+
+[mypy-tests.test_command_fetch]
+ignore_errors = True
+
+[mypy-tests.test_command_parse]
+ignore_errors = True
+
+[mypy-tests.test_command_shell]
+ignore_errors = True
+
+[mypy-tests.test_command_version]
+ignore_errors = True
+
+[mypy-tests.test_contracts]
+ignore_errors = True
+
+[mypy-tests.test_crawler]
+ignore_errors = True
+
+[mypy-tests.test_downloader_handlers]
+ignore_errors = True
+
+[mypy-tests.test_engine]
+ignore_errors = True
+
+[mypy-tests.test_exporters]
+ignore_errors = True
+
+[mypy-tests.test_http_request]
+ignore_errors = True
+
+[mypy-tests.test_linkextractors]
+ignore_errors = True
+
+[mypy-tests.test_loader]
+ignore_errors = True
+
+[mypy-tests.test_pipeline_crawl]
+ignore_errors = True
+
+[mypy-tests.test_pipeline_files]
+ignore_errors = True
+
+[mypy-tests.test_pipeline_images]
+ignore_errors = True
+
+[mypy-tests.test_pipelines]
+ignore_errors = True
+
+[mypy-tests.test_request_cb_kwargs]
+ignore_errors = True
+
+[mypy-tests.test_request_left]
+ignore_errors = True
+
+[mypy-tests.test_scheduler]
+ignore_errors = True
+
+[mypy-tests.test_signals]
+ignore_errors = True
+
+[mypy-tests.test_spiderloader.test_spiders.nested.spider4]
+ignore_errors = True
+
+[mypy-tests.test_spiderloader.test_spiders.spider1]
+ignore_errors = True
+
+[mypy-tests.test_spiderloader.test_spiders.spider2]
+ignore_errors = True
+
+[mypy-tests.test_spiderloader.test_spiders.spider3]
+ignore_errors = True
+
+[mypy-tests.test_spidermiddleware_httperror]
+ignore_errors = True
+
+[mypy-tests.test_spidermiddleware_output_chain]
+ignore_errors = True
+
+[mypy-tests.test_spidermiddleware_referer]
+ignore_errors = True
+
+[mypy-tests.test_utils_reqser]
+ignore_errors = True
+
+[mypy-tests.test_utils_serialize]
+ignore_errors = True
+
+[mypy-tests.test_utils_spider]
+ignore_errors = True
+
+[mypy-tests.test_utils_url]
+ignore_errors = True
diff --git a/tox.ini b/tox.ini
index 4c790158d..27d21ade2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,6 +23,13 @@ passenv =
 commands =
     py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
 
+[testenv:typing]
+basepython = python3
+deps =
+    mypy==0.780
+commands =
+    mypy {posargs: scrapy tests}
+
 [testenv:security]
 basepython = python3
 deps =

From a4bfd5ab6fd75c4badac1c5d9b40706181c41bd9 Mon Sep 17 00:00:00 2001
From: Stanislau Hluboki <glubokiy.stas@gmail.com>
Date: Sat, 13 Jun 2020 18:04:38 +0300
Subject: [PATCH 2885/2974] Fix duplicated feed logs

---
 scrapy/extensions/feedexport.py | 19 +++++++---
 tests/test_feedexport.py        | 63 +++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 5 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 30e6349d6..61dad8726 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -279,11 +279,20 @@ class FeedExporter:
                         'itemcount': slot.itemcount,
                         'uri': slot.uri}
             d = defer.maybeDeferred(slot.storage.store, slot.file)
-            d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
-                                                extra={'spider': spider}))
-            d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
-                                                exc_info=failure_to_exc_info(f),
-                                                extra={'spider': spider}))
+
+            # Use `largs=log_args` to copy log_args into function's scope
+            # instead of using `log_args` from the outer scope
+            d.addCallback(
+                lambda _, largs=log_args: logger.info(
+                    logfmt % "Stored", largs, extra={'spider': spider}
+                )
+            )
+            d.addErrback(
+                lambda f, largs=log_args: logger.error(
+                    logfmt % "Error storing", largs,
+                    exc_info=failure_to_exc_info(f), extra={'spider': spider}
+                )
+            )
             deferred_list.append(d)
         return defer.DeferredList(deferred_list) if deferred_list else None
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index f7013bc44..e38644214 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -393,6 +393,27 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
     pass
 
 
+class DummyBlockingFeedStorage(BlockingFeedStorage):
+
+    def __init__(self, uri):
+        self.path = file_uri_to_path(uri)
+
+    def _store_in_thread(self, file):
+        dirname = os.path.dirname(self.path)
+        if dirname and not os.path.exists(dirname):
+            os.makedirs(dirname)
+        with open(self.path, 'ab') as output_file:
+            output_file.write(file.read())
+
+        file.close()
+
+
+class FailingBlockingFeedStorage(DummyBlockingFeedStorage):
+
+    def _store_in_thread(self, file):
+        raise OSError('Cannot store')
+
+
 @implementer(IFeedStorage)
 class LogOnStoreFileStorage:
     """
@@ -1025,3 +1046,45 @@ class FeedExportTest(unittest.TestCase):
         }
         data = yield self.exported_no_data(settings)
         self.assertEqual(data['csv'], b'')
+
+    @defer.inlineCallbacks
+    def test_multiple_feeds_success_logs_blocking_feed_storage(self):
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'json'},
+                self._random_temp_filename(): {'format': 'xml'},
+                self._random_temp_filename(): {'format': 'csv'},
+            },
+            'FEED_STORAGES': {'file': 'tests.test_feedexport.DummyBlockingFeedStorage'},
+        }
+        items = [
+            {'foo': 'bar1', 'baz': ''},
+            {'foo': 'bar2', 'baz': 'quux'},
+        ]
+        with LogCapture() as log:
+            yield self.exported_data(items, settings)
+
+        print(log)
+        for fmt in ['json', 'xml', 'csv']:
+            self.assertIn('Stored %s feed (2 items)' % fmt, str(log))
+
+    @defer.inlineCallbacks
+    def test_multiple_feeds_failing_logs_blocking_feed_storage(self):
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'json'},
+                self._random_temp_filename(): {'format': 'xml'},
+                self._random_temp_filename(): {'format': 'csv'},
+            },
+            'FEED_STORAGES': {'file': 'tests.test_feedexport.FailingBlockingFeedStorage'},
+        }
+        items = [
+            {'foo': 'bar1', 'baz': ''},
+            {'foo': 'bar2', 'baz': 'quux'},
+        ]
+        with LogCapture() as log:
+            yield self.exported_data(items, settings)
+
+        print(log)
+        for fmt in ['json', 'xml', 'csv']:
+            self.assertIn('Error storing %s feed (2 items)' % fmt, str(log))

From b99fe4aa4c35ff8f5c2355a09988ddb1e90d6b70 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 19 Jun 2020 21:41:15 -0300
Subject: [PATCH 2886/2974] Add google-cloud-storage to the 'pinned' tox
 environment

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 4c790158d..d28eab131 100644
--- a/tox.ini
+++ b/tox.ini
@@ -76,6 +76,7 @@ deps =
     -rtests/requirements-py3.txt
     # Extras
     botocore==1.3.23
+    google-cloud-storage==1.29.0
     Pillow==3.4.2
 
 [testenv:extra-deps]

From 3efea98e0518cd0d3b92b06f6b44b701dff1e53d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 22 Jun 2020 12:41:14 -0300
Subject: [PATCH 2887/2974] Docs: add note about dataclass items and loaders

---
 docs/topics/loaders.rst | 37 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 6645bf123..d70e03ad4 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -26,7 +26,7 @@ Using Item Loaders to populate items
 To use an Item Loader, you must first instantiate it. You can either
 instantiate it with an :ref:`item object <topics-items>` or without one, in which
 case an instance of :class:`~scrapy.item.Item` is automatically created in the
-Item Loader ``__init__`` method using the :class:`~scrapy.item.Item` subclass
+Item Loader ``__init__`` method using the :ref:`item <topics-items>` class
 specified in the :attr:`ItemLoader.default_item_class` attribute.
 
 Then, you start collecting values into the Item Loader, typically using
@@ -76,6 +76,41 @@ called which actually returns the item populated with the data
 previously extracted and collected with the :meth:`~ItemLoader.add_xpath`,
 :meth:`~ItemLoader.add_css`, and :meth:`~ItemLoader.add_value` calls.
 
+
+.. _topics-loaders-dataclass:
+
+Working with dataclass items
+============================
+
+By default, :ref:`dataclass items <dataclass-items>` require all fields to be
+passed when created. This could be an issue when using dataclass items with
+item loaders, since fields could be populated incrementally.
+
+Given the way that item loaders store data internally, the recommended approach
+to overcome this is to define items using the :func:`~dataclasses.field`
+function, with ``list`` as the ``default_factory`` argument::
+
+    from dataclasses import dataclass, field
+
+    @dataclass
+    class InventoryItem:
+        name: str = field(default_factory=list)
+        price: float = field(default_factory=list)
+        stock: int = field(default_factory=list)
+
+Note that in order to keep the example simple, the types do not match
+completely. A more accurate but verbose definition would be::
+
+    from dataclasses import dataclass, field
+    from typing import List, Union
+
+    @dataclass
+    class InventoryItem:
+        name: Union[str, List[str]] = field(default_factory=list)
+        price: Union[float, List[float]] = field(default_factory=list)
+        stock: Union[int, List[int]] = field(default_factory=list)
+
+
 .. _topics-loaders-processors:
 
 Input and Output processors

From 1335d9053e08c0321927728060d4eba0b6da687b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Mon, 22 Jun 2020 14:05:44 -0300
Subject: [PATCH 2888/2974] Update docs/topics/loaders.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/loaders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index d70e03ad4..c5f121dbf 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -25,7 +25,7 @@ Using Item Loaders to populate items
 
 To use an Item Loader, you must first instantiate it. You can either
 instantiate it with an :ref:`item object <topics-items>` or without one, in which
-case an instance of :class:`~scrapy.item.Item` is automatically created in the
+case an :ref:`item object <topics-items>` is automatically created in the
 Item Loader ``__init__`` method using the :ref:`item <topics-items>` class
 specified in the :attr:`ItemLoader.default_item_class` attribute.
 

From 73b6ce8cb560da71b082efdba44b90e1ea932b17 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 22 Jun 2020 14:13:37 -0300
Subject: [PATCH 2889/2974] Update docs about dataclass items and loaders

---
 docs/topics/loaders.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index c5f121dbf..e921395d2 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -84,9 +84,11 @@ Working with dataclass items
 
 By default, :ref:`dataclass items <dataclass-items>` require all fields to be
 passed when created. This could be an issue when using dataclass items with
-item loaders, since fields could be populated incrementally.
+item loaders: unless a pre-populated item is passed to the loader, fields
+will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`,
+:meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods.
 
-Given the way that item loaders store data internally, the recommended approach
+Given the way that item loaders store data internally, one approach
 to overcome this is to define items using the :func:`~dataclasses.field`
 function, with ``list`` as the ``default_factory`` argument::
 

From cfd039aeb6d46f7c120edd82ce06d6b9f4f8e7db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 22 Jun 2020 19:28:33 +0200
Subject: [PATCH 2890/2974] Remove a duplicate GCS_PROJECT_ID reference target

---
 docs/topics/media-pipeline.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 01de3dedb..096618648 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -204,7 +204,6 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
 Google Cloud Storage
 ---------------------
 
-.. setting:: GCS_PROJECT_ID
 .. setting:: FILES_STORE_GCS_ACL
 .. setting:: IMAGES_STORE_GCS_ACL
 

From 3672f5f988cbb29c6bebb7ed535c0d5ad941d868 Mon Sep 17 00:00:00 2001
From: Lukas Anzinger <lukas@lukasanzinger.at>
Date: Tue, 23 Jun 2020 14:51:21 +0200
Subject: [PATCH 2891/2974] Spider constructor expects name as argument, not
 start_urls.

Fixes #4644
---
 tests/test_scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 930a5dd99..2b6cb0902 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -296,7 +296,7 @@ class StartUrlsSpider(Spider):
 
     def __init__(self, start_urls):
         self.start_urls = start_urls
-        super(StartUrlsSpider, self).__init__(start_urls)
+        super(StartUrlsSpider, self).__init__(name='StartUrlsSpider')
 
     def parse(self, response):
         pass

From 6f4c964aa4949584cfd251cf2aa2c0c7de6cf251 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 24 Jun 2020 12:26:38 +0200
Subject: [PATCH 2892/2974] Cover Scrapy 2.2.0 in the release notes (#4630)

---
 docs/contributing.rst            |   5 +-
 docs/news.rst                    | 195 +++++++++++++++++++++++++++++++
 docs/topics/media-pipeline.rst   |   9 +-
 docs/topics/request-response.rst |   4 +-
 scrapy/http/response/text.py     |   2 +
 scrapy/utils/misc.py             |   5 +-
 6 files changed, 214 insertions(+), 6 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index aed5ab92e..7b901dd00 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -155,6 +155,9 @@ Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports
 removal, etc) in separate commits from functional changes. This will make pull
 requests easier to review and more likely to get merged.
 
+
+.. _coding-style:
+
 Coding style
 ============
 
@@ -163,7 +166,7 @@ Scrapy:
 
 * Unless otherwise specified, follow :pep:`8`.
 
-* It's OK to use lines longer than 80 chars if it improves the code
+* It's OK to use lines longer than 79 chars if it improves the code
   readability.
 
 * Don't put your name in the code you contribute; git provides enough
diff --git a/docs/news.rst b/docs/news.rst
index a158246eb..80d130e4a 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,201 @@
 Release notes
 =============
 
+.. _release-2.2.0:
+
+Scrapy 2.2.0 (2020-06-24)
+-------------------------
+
+Highlights:
+
+* Python 3.5.2+ is required now
+* :ref:`dataclass objects <dataclass-items>` and
+  :ref:`attrs objects <attrs-items>` are now valid :ref:`item types
+  <item-types>`
+* New :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method
+* New :signal:`bytes_received` signal that allows canceling response download
+* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` fixes
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+*   Support for Python 3.5.0 and 3.5.1 has been dropped; Scrapy now refuses to
+    run with a Python version lower than 3.5.2, which introduced
+    :class:`typing.Type` (:issue:`4615`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+*   :meth:`TextResponse.body_as_unicode
+    <scrapy.http.TextResponse.body_as_unicode>` is now deprecated, use
+    :attr:`TextResponse.text <scrapy.http.TextResponse.text>` instead
+    (:issue:`4546`, :issue:`4555`, :issue:`4579`)
+
+*   :class:`scrapy.item.BaseItem` is now deprecated, use
+    :class:`scrapy.item.Item` instead (:issue:`4534`)
+
+
+New features
+~~~~~~~~~~~~
+
+*   :ref:`dataclass objects <dataclass-items>` and
+    :ref:`attrs objects <attrs-items>` are now valid :ref:`item types
+    <item-types>`, and a new itemadapter_ library makes it easy to
+    write code that :ref:`supports any item type <supporting-item-types>`
+    (:issue:`2749`, :issue:`2807`, :issue:`3761`, :issue:`3881`, :issue:`4642`)
+
+*   A new :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method
+    allows to deserialize JSON responses (:issue:`2444`, :issue:`4460`,
+    :issue:`4574`)
+
+*   A new :signal:`bytes_received` signal allows monitoring response download
+    progress and :ref:`stopping downloads <topics-stop-response-download>`
+    (:issue:`4205`, :issue:`4559`)
+
+*   The dictionaries in the result list of a :ref:`media pipeline
+    <topics-media-pipeline>` now include a new key, ``status``, which indicates
+    if the file was downloaded or, if the file was not downloaded, why it was
+    not downloaded; see :meth:`FilesPipeline.get_media_requests
+    <scrapy.pipelines.files.FilesPipeline.get_media_requests>` for more
+    information (:issue:`2893`, :issue:`4486`)
+
+*   When using :ref:`Google Cloud Storage <media-pipeline-gcs>` for
+    a :ref:`media pipeline <topics-media-pipeline>`, a warning is now logged if
+    the configured credentials do not grant the required permissions
+    (:issue:`4346`, :issue:`4508`)
+
+*   :ref:`Link extractors <topics-link-extractors>` are now serializable,
+    as long as you do not use :ref:`lambdas <lambda>` for parameters; for
+    example, you can now pass link extractors in :attr:`Request.cb_kwargs
+    <scrapy.http.Request.cb_kwargs>` or
+    :attr:`Request.meta <scrapy.http.Request.meta>` when :ref:`persisting
+    scheduled requests <topics-jobs>` (:issue:`4554`)
+
+*   Upgraded the :ref:`pickle protocol <pickle-protocols>` that Scrapy uses
+    from protocol 2 to protocol 4, improving serialization capabilities and
+    performance (:issue:`4135`, :issue:`4541`)
+
+*   :func:`scrapy.utils.misc.create_instance` now raises a :exc:`TypeError`
+    exception if the resulting instance is ``None`` (:issue:`4528`,
+    :issue:`4532`)
+
+.. _itemadapter: https://github.com/scrapy/itemadapter
+
+
+Bug fixes
+~~~~~~~~~
+
+*   :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer
+    discards cookies defined in :attr:`Request.headers
+    <scrapy.http.Request.headers>` (:issue:`1992`, :issue:`2400`)
+
+*   :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer
+    re-encodes cookies defined as :class:`bytes` in the ``cookies`` parameter
+    of the ``__init__`` method of :class:`~scrapy.http.Request`
+    (:issue:`2400`, :issue:`3575`)
+
+*   When :setting:`FEEDS` defines multiple URIs, :setting:`FEED_STORE_EMPTY` is
+    ``False`` and the crawl yields no items, Scrapy no longer stops feed
+    exports after the first URI (:issue:`4621`, :issue:`4626`)
+
+*   :class:`~scrapy.spiders.Spider` callbacks defined using :doc:`coroutine
+    syntax <topics/coroutines>` no longer need to return an iterable, and may
+    instead return a :class:`~scrapy.http.Request` object, an
+    :ref:`item <topics-items>`, or ``None`` (:issue:`4609`)
+
+*   The :command:`startproject` command now ensures that the generated project
+    folders and files have the right permissions (:issue:`4604`)
+
+*   Fix a :exc:`KeyError` exception being sometimes raised from
+    :class:`scrapy.utils.datatypes.LocalWeakReferencedCache` (:issue:`4597`,
+    :issue:`4599`)
+
+*   When :setting:`FEEDS` defines multiple URIs, log messages about items being
+    stored now contain information from the corresponding feed, instead of
+    always containing information about only one of the feeds (:issue:`4619`,
+    :issue:`4629`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+*   Added a new section about :ref:`accessing cb_kwargs from errbacks
+    <errback-cb_kwargs>` (:issue:`4598`, :issue:`4634`)
+
+*   Covered chompjs_ in :ref:`topics-parsing-javascript` (:issue:`4556`,
+    :issue:`4562`)
+
+*   Removed from :doc:`topics/coroutines` the warning about the API being
+    experimental (:issue:`4511`, :issue:`4513`)
+
+*   Removed references to unsupported versions of :doc:`Twisted
+    <twisted:index>` (:issue:`4533`)
+
+*   Updated the description of the :ref:`screenshot pipeline example
+    <ScreenshotPipeline>`, which now uses :doc:`coroutine syntax
+    <topics/coroutines>` instead of returning a
+    :class:`~twisted.internet.defer.Deferred` (:issue:`4514`, :issue:`4593`)
+
+*   Removed a misleading import line from the
+    :func:`scrapy.utils.log.configure_logging` code example (:issue:`4510`,
+    :issue:`4587`)
+
+*   The display-on-hover behavior of internal documentation references now also
+    covers links to :ref:`commands <topics-commands>`, :attr:`Request.meta
+    <scrapy.http.Request.meta>` keys, :ref:`settings <topics-settings>` and
+    :ref:`signals <topics-signals>` (:issue:`4495`, :issue:`4563`)
+
+*   It is again possible to download the documentation for offline reading
+    (:issue:`4578`, :issue:`4585`)
+
+*   Removed backslashes preceding ``*args`` and ``**kwargs`` in some function
+    and method signatures (:issue:`4592`, :issue:`4596`)
+
+.. _chompjs: https://github.com/Nykakin/chompjs
+
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+*   Adjusted the code base further to our :ref:`style guidelines
+    <coding-style>` (:issue:`4237`, :issue:`4525`, :issue:`4538`,
+    :issue:`4539`, :issue:`4540`, :issue:`4542`, :issue:`4543`, :issue:`4544`,
+    :issue:`4545`, :issue:`4557`, :issue:`4558`, :issue:`4566`, :issue:`4568`,
+    :issue:`4572`)
+
+*   Removed remnants of Python 2 support (:issue:`4550`, :issue:`4553`,
+    :issue:`4568`)
+
+*   Improved code sharing between the :command:`crawl` and :command:`runspider`
+    commands (:issue:`4548`, :issue:`4552`)
+
+*   Replaced ``chain(*iterable)`` with ``chain.from_iterable(iterable)``
+    (:issue:`4635`)
+
+*   You may now run the :mod:`asyncio` tests with Tox on any Python version
+    (:issue:`4521`)
+
+*   Updated test requirements to reflect an incompatibility with pytest 5.4 and
+    5.4.1 (:issue:`4588`)
+
+*   Improved :class:`~scrapy.spiderloader.SpiderLoader` test coverage for
+    scenarios involving duplicate spider names (:issue:`4549`, :issue:`4560`)
+
+*   Configured Travis CI to also run the tests with Python 3.5.2
+    (:issue:`4518`, :issue:`4615`)
+
+*   Added a `Pylint <https://www.pylint.org/>`_ job to Travis CI
+    (:issue:`3727`)
+
+*   Added a `Mypy <http://mypy-lang.org/>`_ job to Travis CI (:issue:`4637`)
+
+*   Made use of set literals in tests (:issue:`4573`)
+
+*   Cleaned up the Travis CI configuration (:issue:`4517`, :issue:`4519`,
+    :issue:`4522`, :issue:`4537`)
+
+
 .. _release-2.1.0:
 
 Scrapy 2.1.0 (2020-04-24)
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 01de3dedb..9f2a06dd7 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -201,6 +201,9 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
 .. _s3.scality: https://s3.scality.com/
 .. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
+
+.. _media-pipeline-gcs:
+
 Google Cloud Storage
 ---------------------
 
@@ -475,7 +478,11 @@ See here the methods that you can override in your custom Files Pipeline:
 
         * ``checksum`` - a `MD5 hash`_ of the image contents
 
-        * ``status`` - the file status indication. It can be one of the following:
+        * ``status`` - the file status indication.
+
+          .. versionadded:: 2.2
+
+          It can be one of the following:
 
           * ``downloaded`` - file was downloaded.
           * ``uptodate`` - file was not downloaded, as it was downloaded recently,
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index d88d40b00..fbd8e4b73 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -191,7 +191,7 @@ Request objects
 
         In case of a failure to process the request, this dict can be accessed as
         ``failure.request.cb_kwargs`` in the request's errback. For more information,
-        see :ref:`topics-request-response-ref-accessing-callback-arguments-in-errback`.
+        see :ref:`errback-cb_kwargs`.
 
     .. method:: Request.copy()
 
@@ -316,7 +316,7 @@ errors if needed::
                 request = failure.request
                 self.logger.error('TimeoutError on %s', request.url)
 
-.. _topics-request-response-ref-accessing-callback-arguments-in-errback:
+.. _errback-cb_kwargs:
 
 Accessing additional data in errback functions
 ----------------------------------------------
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 40cf3f483..b43fe5c19 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -74,6 +74,8 @@ class TextResponse(Response):
 
     def json(self):
         """
+        .. versionadded:: 2.2
+
         Deserialize a JSON document to a Python object.
         """
         if self._cached_decoded_json is _NONE:
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index a7808cb2c..8e5fde246 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -138,8 +138,9 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
 
     Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
 
-    Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
-    extension has not been implemented correctly).
+    .. versionchanged:: 2.2
+       Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
+       extension has not been implemented correctly).
     """
     if settings is None:
         if crawler is None:

From 9f60481360628bafe467f59b2144d69204b8b4e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 24 Jun 2020 12:27:39 +0200
Subject: [PATCH 2893/2974] =?UTF-8?q?Bump=20version:=202.1.0=20=E2=86=92?=
 =?UTF-8?q?=202.2.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index de22a2783..8d4d74bc5 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.1.0
+current_version = 2.2.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 7ec1d6db4..ccbccc3dc 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.1.0
+2.2.0

From c3cee74fd401e6a6307b5eb1786e532bb2cd5aa8 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 26 Jun 2020 18:45:21 +0300
Subject: [PATCH 2894/2974] Change default value of
 FEED_STORAGE_BATCH_ITEM_COUNT to 0

---
 docs/topics/feed-exports.rst        |  2 +-
 scrapy/extensions/feedexport.py     |  2 +-
 scrapy/settings/default_settings.py |  2 +-
 tests/test_feedexport.py            | 20 ++++++++++----------
 4 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 866ce78eb..0b37e9a7d 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -435,7 +435,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 
 FEED_STORAGE_BATCH_ITEM_COUNT
 -----------------------------
-Default: ``None``
+Default: ``0``
 
 If assigned an integer number higher than ``0``, Scrapy generates multiple output files
 storing up to the specified number of items in each output file.
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 1331782e3..e06116acd 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -243,7 +243,7 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch_item_count = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
+        self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 5a7dc533e..810acd5a3 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_STORAGE_BATCH_ITEM_COUNT = None
+FEED_STORAGE_BATCH_ITEM_COUNT = 0
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 1a6a5624b..578cd396b 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1144,7 +1144,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['jl']:
@@ -1160,7 +1160,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         data = yield self.exported_data(items, settings)
         for batch in data['csv']:
             got_batch = csv.DictReader(to_unicode(batch).splitlines())
@@ -1176,7 +1176,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['xml']:
@@ -1194,7 +1194,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         # XML
@@ -1219,7 +1219,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import pickle
@@ -1236,7 +1236,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
             },
         })
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import marshal
@@ -1262,7 +1262,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             'FEED_STORAGE_BATCH_ITEM_COUNT': 2
         }
         header = self.MyItem.fields.keys()
-        yield self.assertExported(items, header, rows, settings=settings)
+        yield self.assertExported(items, header, rows, settings=Settings(settings))
 
     def test_wrong_path(self):
         """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """
@@ -1412,14 +1412,14 @@ class BatchDeliveriesTest(FeedExportTestBase):
             bucket_name=s3_test_bucket_name, prefix=prefix
         )
         storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)
-        settings = {
+        settings = Settings({
             'FEEDS': {
                 s3_test_file_uri: {
                     'format': 'json',
                 },
             },
             'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
-        }
+        })
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
             self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
@@ -1436,7 +1436,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
         s3 = boto3.resource('s3')
         my_bucket = s3.Bucket(s3_test_bucket_name)
-        batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
 
         with MockServer() as s:
             runner = CrawlerRunner(Settings(settings))

From 88a52198b90faa0129c8e05072197cdffbb9653b Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sat, 27 Jun 2020 11:50:26 +0300
Subject: [PATCH 2895/2974] Add batch_item_count support in FEEDS setting

---
 scrapy/extensions/feedexport.py |  5 +++--
 tests/test_feedexport.py        | 39 ++++++++++++++++++++++++++++++---
 2 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index e06116acd..2312c994e 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -25,7 +25,6 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -337,7 +336,9 @@ class FeedExporter:
             slot.exporter.export_item(item)
             slot.itemcount += 1
             # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
-            if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count:
+            if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \
+                    and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count',
+                                                                            self.storage_batch_item_count):
                 uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
                 self._close_slot(slot, spider)
                 slots.append(self._start_new_batch(
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 578cd396b..3bc0c083c 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1327,8 +1327,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     '<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>'
                 ).encode('latin-1')
             ],
-            'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
-                    'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],
+            'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'),
+                    'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')],
+            'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
         }
 
         settings = {
@@ -1348,9 +1349,16 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
                     'format': 'csv',
                     'indent': None,
-                    'fields': ['bar', 'foo'],
+                    'fields': ['foo', 'bar'],
                     'encoding': 'utf-8',
                 },
+                os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
+                    'format': 'jsonlines',
+                    'indent': None,
+                    'fields': ['foo', 'bar'],
+                    'encoding': 'utf-8',
+                    'batch_item_count': 0,
+                },
             },
             'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
         }
@@ -1359,6 +1367,31 @@ class BatchDeliveriesTest(FeedExportTestBase):
             for expected_batch, got_batch in zip(expected, data[fmt]):
                 self.assertEqual(expected_batch, got_batch)
 
+    @defer.inlineCallbacks
+    def test_batch_item_count_feeds_setting(self):
+        items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
+
+        formats = {
+            'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'),
+                          '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
+        }
+
+        settings = {
+            'FEEDS': {
+                os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {
+                    'format': 'jsonlines',
+                    'indent': None,
+                    'fields': ['foo', 'bar'],
+                    'encoding': 'utf-8',
+                    'batch_item_count': 1,
+                },
+            },
+        }
+        data = yield self.exported_data(items, settings)
+        for fmt, expected in formats.items():
+            for expected_batch, got_batch in zip(expected, data[fmt]):
+                self.assertEqual(expected_batch, got_batch)
+
     @defer.inlineCallbacks
     def test_batch_path_differ(self):
         """

From 05c2587c6a32b84a94463f2b1187e49f94957aa2 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sun, 28 Jun 2020 09:45:45 +0300
Subject: [PATCH 2896/2974] Docs update and tiny fixes

---
 docs/topics/feed-exports.rst | 1 +
 tests/test_feedexport.py     | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 0b37e9a7d..3da56821e 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -272,6 +272,7 @@ as a fallback value if that key is not provided for a specific feed definition.
 * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
 * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
 * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
+* ``batch_item_count``: falls back to :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`
 
 .. setting:: FEED_EXPORT_ENCODING
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 3bc0c083c..542cce70f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1352,7 +1352,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     'fields': ['foo', 'bar'],
                     'encoding': 'utf-8',
                 },
-                os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
+                os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {
                     'format': 'jsonlines',
                     'indent': None,
                     'fields': ['foo', 'bar'],
@@ -1423,8 +1423,8 @@ class BatchDeliveriesTest(FeedExportTestBase):
         [testenv]
         setenv =
             AWS_SECRET_ACCESS_KEY = ABCD
-            AWS_ACCESS_KEY_ID = ABCD
-            S3_TEST_BUCKET_NAME = ABCD
+            AWS_ACCESS_KEY_ID = EFGH
+            S3_TEST_BUCKET_NAME = IJKL
         """
         try:
             import boto3

From 464f24f8c160466f93eb8ebb4ec8b84d1824eaa0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Mon, 29 Jun 2020 14:20:29 +0200
Subject: [PATCH 2897/2974] Add --data-raw to utils.curl and fix missing method
 with data (#4612)

---
 scrapy/utils/curl.py     | 12 +++++++++---
 tests/test_utils_curl.py | 25 +++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index 16639356e..67b22dbc5 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -17,8 +17,8 @@ class CurlParser(argparse.ArgumentParser):
 curl_parser = CurlParser()
 curl_parser.add_argument('url')
 curl_parser.add_argument('-H', '--header', dest='headers', action='append')
-curl_parser.add_argument('-X', '--request', dest='method', default='get')
-curl_parser.add_argument('-d', '--data', dest='data')
+curl_parser.add_argument('-X', '--request', dest='method')
+curl_parser.add_argument('-d', '--data', '--data-raw', dest='data')
 curl_parser.add_argument('-u', '--user', dest='auth')
 
 
@@ -66,7 +66,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
     if not parsed_url.scheme:
         url = 'http://' + url
 
-    result = {'method': parsed_args.method.upper(), 'url': url}
+    method = parsed_args.method or 'GET'
+
+    result = {'method': method.upper(), 'url': url}
 
     headers = []
     cookies = {}
@@ -90,5 +92,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
         result['cookies'] = cookies
     if parsed_args.data:
         result['body'] = parsed_args.data
+        if not parsed_args.method:
+            # if the "data" is specified but the "method" is not specified,
+            # the default method is 'POST'
+            result['method'] = 'POST'
 
     return result
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index 50e1bfd5f..299a51efe 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -141,6 +141,31 @@ class CurlToRequestKwargsTest(unittest.TestCase):
         }
         self._test_command(curl_command, expected_result)
 
+    def test_post_data_raw(self):
+        curl_command = (
+            "curl 'https://www.example.org/' --data-raw 'excerptLength=200&ena"
+            "bleDidYouMean=true&sortCriteria=ffirstz32xnamez32x201740686%20asc"
+            "ending&queryFunctions=%5B%5D&rankingFunctions=%5B%5D'"
+        )
+        expected_result = {
+            "method": "POST",
+            "url": "https://www.example.org/",
+            "body": (
+                "excerptLength=200&enableDidYouMean=true&sortCriteria=ffirstz3"
+                "2xnamez32x201740686%20ascending&queryFunctions=%5B%5D&ranking"
+                "Functions=%5B%5D")
+        }
+        self._test_command(curl_command, expected_result)
+
+    def test_explicit_get_with_data(self):
+        curl_command = 'curl httpbin.org/anything -X GET --data asdf'
+        expected_result = {
+            "method": "GET",
+            "url": "http://httpbin.org/anything",
+            "body": "asdf"
+        }
+        self._test_command(curl_command, expected_result)
+
     def test_patch(self):
         curl_command = (
             'curl "https://example.com/api/fake" -u "username:password" -H "Ac'

From e46b47c365b27e3f3f383a102122361db0a86a42 Mon Sep 17 00:00:00 2001
From: Aditya Kumar <k.aditya00@gmail.com>
Date: Mon, 29 Jun 2020 18:35:13 +0530
Subject: [PATCH 2898/2974] Renew the localhost certificate for tests (#4650)

Validity
  Not Before: Jun 28 12:54:15 2020 GMT
  Not After : Jun 28 12:54:15 2021 GMT
Subject: C = IE, O = Scrapy, CN = localhost
---
 tests/keys/localhost.crt | 36 ++++++++++++++--------------
 tests/keys/localhost.key | 52 ++++++++++++++++++++--------------------
 2 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt
index 13c5b5bd6..0cf5256d8 100644
--- a/tests/keys/localhost.crt
+++ b/tests/keys/localhost.crt
@@ -1,20 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV
-BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x
-NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD
-VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB
-BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K
-o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y
-njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY
-ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/
-8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52
-gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O
-BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j
-xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES
-9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i
-jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+
-TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own
-tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM
-am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U
-Mjybr/hQK3mnVdE=
+MIIDRTCCAi2gAwIBAgIUGoISfeW3LwSWHC52ORXdZY9pNLswDQYJKoZIhvcNAQEL
+BQAwMjELMAkGA1UEBhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9j
+YWxob3N0MB4XDTIwMDYyODEyNTQxNVoXDTIxMDYyODEyNTQxNVowMjELMAkGA1UE
+BhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjAN
+BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvCLxfTEQuIdf8JhiHrbVkGHYrNSK
+2XD2TCPaSIpJ2KKlFUrIz3A9tWlOfLnWabS5od89yOebhYj4DN/Qm2TViGg1mtWe
+pD1K2YWd1Af+hhAw5D+TpW2RH9TVhX7Ey5osWcl+0uy+RlKZE8qum72xi1vxWOmH
+wYw06iN8klQ3JfP2/eLRXBQjsh7WW0dbJ7yLvG6UFz1RbhFTtlxeIMenzNsHaMg7
+56Ru57/MMbaBwdBttXVzJDQ7imo8njuxDMszliC/QgIdBUBFzA2LB5qpr+v+laDN
+cN9t9Q9stsu446dFnRoofxJjMFW7lLu6h/lwP5r0kfeUkMDhXJ4mb6KwfwIDAQAB
+o1MwUTAdBgNVHQ4EFgQUVEdXn8ha2FA73zcy1Ia0FQMzMEYwHwYDVR0jBBgwFoAU
+VEdXn8ha2FA73zcy1Ia0FQMzMEYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
+AQsFAAOCAQEAZpGBPsexMD+IwcMNIgc7FiaJsb8E30C9vWxgdnkpapi9zLJ4yiHQ
+VxkV9RTezUEADkaDj+2qFveamWTzJLnphgaaUpVeMcYACPhRVOYXidNrZyTmHIsX
+FwaTzAggW6CP7JxAcpxH0f9+NWFCZI36FihRdwuWyvrUl7rsXaexu0SOI/Ck0oWf
+2IW+jo67TSmcbte+J8wq77DX32mVLb/2nqpItH4T2Di+XjVBARACVOSdgdlo7lZE
+W8mSEXqP2BVx8JGG8X1znNLHcmjVj4EtkpH0wkYzpC4cvGkTsUcU7CU7ZyVUp+Bb
+dPMVxyRKWfAjRJc8o5Ot1mgHrx5coOtzAA==
 -----END CERTIFICATE-----
diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key
index da975e6d3..8fc373bdd 100644
--- a/tests/keys/localhost.key
+++ b/tests/keys/localhost.key
@@ -1,28 +1,28 @@
 -----BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5
-ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4
-JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz
-vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2
-RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY
-b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/
-XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0
-gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk
-i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu
-Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss
-ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4
-hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD
-7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa
-EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf
-uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG
-HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR
-UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO
-j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri
-fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ
-6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI
-xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g
-DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f
-npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM
-8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6
-J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj
-Z+pMCFLbwAbFZ9Qb/NFlUv0=
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC8IvF9MRC4h1/w
+mGIettWQYdis1IrZcPZMI9pIiknYoqUVSsjPcD21aU58udZptLmh3z3I55uFiPgM
+39CbZNWIaDWa1Z6kPUrZhZ3UB/6GEDDkP5OlbZEf1NWFfsTLmixZyX7S7L5GUpkT
+yq6bvbGLW/FY6YfBjDTqI3ySVDcl8/b94tFcFCOyHtZbR1snvIu8bpQXPVFuEVO2
+XF4gx6fM2wdoyDvnpG7nv8wxtoHB0G21dXMkNDuKajyeO7EMyzOWIL9CAh0FQEXM
+DYsHmqmv6/6VoM1w3231D2y2y7jjp0WdGih/EmMwVbuUu7qH+XA/mvSR95SQwOFc
+niZvorB/AgMBAAECggEAHVpSVRb/pdqxNEeCH4qlHWa2uJhcpXpDYzPAzcqNpPgT
+S5QkaoD3j8NDVKBl/I4O3FuJNzwzfo0VLmUJFgWQbzzbCDJGExfhArkfG8K3ilEi
+X6ovrgK/PrklKzPRHncKbmPKnrwDH9OpQHZB8diRx81rhVTCModehh1NRUNQa2I1
+QzFC7uyXx3duoIsI5QXVeEGuwHZfqIY/z+9SscdVFL6elXTPFUzBzcmAqQgdgWKN
+HXgX22LE0rAu8NnRvOZZWt4/nOjvlCFCPTB11NgthmKlVnsx4H7gpQ2OPh4bZ+0W
+birVEtZ3E1jxoGvw1FzxyqqpGkcanRMa8QWzK4JwuQKBgQDrgclpkqZrgHB/TC1p
+hLvsdflGI2SGs+c/mYR3GEjf0kJtI88WL5fj1QezdkDyOpwxFvnLslswfzdtzvis
+vksGysV35vhMPQUcmWhvzA7Pdxdv4BZr+ckER0SAYBBxg9KYZyxewGb5XzB8Cz2o
+8V+YpwrMAOYGuXHTfafv4CKlTQKBgQDMgetvV9/E3HNtKsATiPIwT3e1MzyPXigq
+12NkHSZa6s4yqm/h/fSUn54sJbhx+OtRRhktOo0aB34tcogtrJyClvCPdRAP/4Qi
+M43FjKo2cWiubWvtWlOZU04bpClG324q420rK7dCA2stID/Fa0sMQgAAyPH8TGMo
+gbvyrk4W+wKBgQDMIOnYZTF0epaH8BponJFaqwMOhTzr+OGW4dTMebMotZG4EdK8
+kzIfW5XaOsSecKjTb+vCYGzkA1CjEEPBTwuu7nDstblAM5/Lozi/tmqb7sjUwrIM
+kyxmVfONJjb6fV07lioCUtiui5B15DRkzBqlMRyNqLW43GJKA19d7rN4/QKBgCzy
+kRBTu/bEjQn9T2H7w18i2CiXLkREaYeg91NVpMxutwsjspt0+YCA5H7He5ZxIycl
+xPrP15tU8kKC3bNMMMny6sRc8j7R5fSuaAZ3OCHnIx7TJdlw9NbKHGyu0/Ojv87l
+VWUbopd7sN6mK930CvaSuvVxNN5C27hXazuXW8ppAoGBANcWsenNKpCJgF0cNPHX
+abPaWfcs5FKMNz8gEdGk3B1z/KBpYz59smPwurYVCXaWE6iv99sDOP7CVneF02sV
+SqyNzVhcVSG788uB3CwnpEvm7ydoH89L5dvYekAHP8RJulhWCK45lXkHLiYGKvhv
+PWuPk5VX+qF78JhUhPO3nfnu
 -----END PRIVATE KEY-----

From 5b88c522ac1b1a9ba1588573d90cf3bc01339282 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 30 Jun 2020 12:18:21 -0300
Subject: [PATCH 2899/2974] Simplify dataclass example in item loader docs

---
 docs/topics/loaders.rst | 24 ++++++------------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index e921395d2..9c82bb4d9 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -88,29 +88,17 @@ item loaders: unless a pre-populated item is passed to the loader, fields
 will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`,
 :meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods.
 
-Given the way that item loaders store data internally, one approach
-to overcome this is to define items using the :func:`~dataclasses.field`
-function, with ``list`` as the ``default_factory`` argument::
+One approach to overcome this is to define items using the
+:func:`~dataclasses.field` function, with a ``default`` argument::
 
     from dataclasses import dataclass, field
+    from typing import Optional
 
     @dataclass
     class InventoryItem:
-        name: str = field(default_factory=list)
-        price: float = field(default_factory=list)
-        stock: int = field(default_factory=list)
-
-Note that in order to keep the example simple, the types do not match
-completely. A more accurate but verbose definition would be::
-
-    from dataclasses import dataclass, field
-    from typing import List, Union
-
-    @dataclass
-    class InventoryItem:
-        name: Union[str, List[str]] = field(default_factory=list)
-        price: Union[float, List[float]] = field(default_factory=list)
-        stock: Union[int, List[int]] = field(default_factory=list)
+        name: Optional[str] = field(default=None)
+        price: Optional[float] = field(default=None)
+        stock: Optional[int] = field(default=None)
 
 
 .. _topics-loaders-processors:

From 7b1d3c35ea3bfde2ac7fc69a2a26bbcb94aec1bf Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Wed, 1 Jul 2020 11:54:39 +0300
Subject: [PATCH 2900/2974] Minor updates

---
 docs/topics/feed-exports.rst    |  4 ++--
 scrapy/extensions/feedexport.py | 34 +++++++++++++++++----------------
 scrapy/utils/conf.py            |  4 ++++
 tests/test_feedexport.py        | 23 ++++++----------------
 tests/test_utils_conf.py        |  4 ++++
 5 files changed, 34 insertions(+), 35 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 3da56821e..0b659f30e 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -442,7 +442,7 @@ If assigned an integer number higher than ``0``, Scrapy generates multiple outpu
 storing up to the specified number of items in each output file.
 
 When generating multiple output files, you must use at least one of the following
-placeholders in :setting:`FEED_URI` to indicate how the different output file names are
+placeholders in the feed URI to indicate how the different output file names are
 generated:
 
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
@@ -457,7 +457,7 @@ For instance, if your settings include::
 
 And your :command:`crawl` command line is::
 
-  scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
+    scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
 
 The command line above can generate a directory tree like::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 2312c994e..5908987a3 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -242,7 +242,6 @@ class FeedExporter:
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
@@ -253,7 +252,7 @@ class FeedExporter:
 
     def open_spider(self, spider):
         for uri, feed in self.feeds.items():
-            uri_params = self._get_uri_params(spider, feed['uri_params'], None)
+            uri_params = self._get_uri_params(spider, feed['uri_params'])
             self.slots.append(self._start_new_batch(
                 batch_id=1,
                 uri=uri % uri_params,
@@ -299,7 +298,7 @@ class FeedExporter:
     def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
         """
         Redirect the output data stream to a new file.
-        Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.
+        Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
         :param batch_id: sequence number of current batch
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
@@ -331,14 +330,15 @@ class FeedExporter:
 
     def item_scraped(self, item, spider):
         slots = []
-        for idx, slot in enumerate(self.slots):
+        for slot in self.slots:
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
             # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
-            if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \
-                    and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count',
-                                                                            self.storage_batch_item_count):
+            if (
+                    self.feeds[slot.uri_template]['batch_item_count']
+                    and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
+            ):
                 uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
                 self._close_slot(slot, spider)
                 slots.append(self._start_new_batch(
@@ -369,15 +369,17 @@ class FeedExporter:
 
     def _settings_are_valid(self, uri):
         """
-        If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s
-        to distinguish different files of partial output
+        If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
+        %(batch_time)s or %(batch_id)s to distinguish different files of partial output
         """
-        if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
-            return True
-        logger.error(
-            '%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified'
-        )
-        return False
+        for uri_template, values in self.feeds.items():
+            if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']):
+                logger.error(
+                    '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting '
+                    'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template)
+                )
+                return False
+        return True
 
     def _storage_supported(self, uri):
         scheme = urlparse(uri).scheme
@@ -404,7 +406,7 @@ class FeedExporter:
     def _get_storage(self, uri):
         return self._get_instance(self.storages[urlparse(uri).scheme], uri)
 
-    def _get_uri_params(self, spider, uri_params, slot):
+    def _get_uri_params(self, spider, uri_params, slot=None):
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 5921f82bf..0e02f0f28 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -115,6 +115,10 @@ def feed_complete_default_values_from_settings(feed, settings):
     out = feed.copy()
     out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
     out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
+    out.setdefault(
+        "batch_item_count",
+        out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT'))
+    )
     out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
     out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
     if settings["FEED_EXPORT_INDENT"] is None:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 542cce70f..db14b20b9 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1265,7 +1265,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=Settings(settings))
 
     def test_wrong_path(self):
-        """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """
+        """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
@@ -1329,7 +1329,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
             ],
             'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'),
                     'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')],
-            'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
         }
 
         settings = {
@@ -1352,13 +1351,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     'fields': ['foo', 'bar'],
                     'encoding': 'utf-8',
                 },
-                os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {
-                    'format': 'jsonlines',
-                    'indent': None,
-                    'fields': ['foo', 'bar'],
-                    'encoding': 'utf-8',
-                    'batch_item_count': 0,
-                },
             },
             'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
         }
@@ -1369,19 +1361,16 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_batch_item_count_feeds_setting(self):
-        items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
-
+        items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})]
         formats = {
-            'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'),
-                          '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
+            'json': ['[{"foo": "FOO"}]'.encode('utf-8'),
+                     '[{"foo": "FOO1"}]'.encode('utf-8')],
         }
-
         settings = {
             'FEEDS': {
-                os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {
-                    'format': 'jsonlines',
+                os.path.join(self._random_temp_filename(), 'json', self._file_mark): {
+                    'format': 'json',
                     'indent': None,
-                    'fields': ['foo', 'bar'],
                     'encoding': 'utf-8',
                     'batch_item_count': 1,
                 },
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index e5d3ef582..95ec2b64a 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -149,6 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "FEED_EXPORT_INDENT": 42,
             "FEED_STORE_EMPTY": True,
             "FEED_URI_PARAMS": (1, 2, 3, 4),
+            "FEED_STORAGE_BATCH_ITEM_COUNT": 2,
         })
         new_feed = feed_complete_default_values_from_settings(feed, settings)
         self.assertEqual(new_feed, {
@@ -157,6 +158,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "indent": 42,
             "store_empty": True,
             "uri_params": (1, 2, 3, 4),
+            "batch_item_count": 2,
         })
 
     def test_feed_complete_default_values_from_settings_non_empty(self):
@@ -169,6 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
             "FEED_EXPORT_INDENT": 42,
             "FEED_STORE_EMPTY": True,
+            "FEED_STORAGE_BATCH_ITEM_COUNT": 2,
         })
         new_feed = feed_complete_default_values_from_settings(feed, settings)
         self.assertEqual(new_feed, {
@@ -177,6 +180,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "indent": 42,
             "store_empty": True,
             "uri_params": None,
+            "batch_item_count": 2,
         })
 
 

From af55d23167f9cec22f815bc9f9884b10a9a35f5b Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Wed, 1 Jul 2020 19:46:54 +0500
Subject: [PATCH 2901/2974] Update the OpenSSL cipher list format link

OpenSSL `ciphers(1)` is now almost empty: https://www.openssl.org/docs/manmaster/man1/ciphers.html

Alternative would be linking to 1.1.1 docs specifically.
---
 docs/topics/settings.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 5178f272f..8cc8806a5 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -469,7 +469,7 @@ necessary to access certain HTTPS websites: for example, you may need to use
 ``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
 specific cipher that is not included in ``DEFAULT`` if a website requires it.
 
-.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT
+.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
 
 .. setting:: DOWNLOADER_CLIENT_TLS_METHOD
 

From 1e245046ed8ac3d9f89860501c2da95b69aaabf6 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 2 Jul 2020 12:38:08 +0300
Subject: [PATCH 2902/2974] Change setting name. Add leading zeroes to
 batch_id. Minor fixes.

---
 docs/topics/feed-exports.rst        | 19 +++++++++--------
 scrapy/extensions/feedexport.py     | 24 +++++++++++++---------
 scrapy/settings/default_settings.py |  2 +-
 scrapy/utils/conf.py                |  5 +----
 tests/test_feedexport.py            | 32 ++++++++++++++---------------
 tests/test_utils_conf.py            |  4 ++--
 6 files changed, 45 insertions(+), 41 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 0b659f30e..56efa80a7 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_STORAGE_FTP_ACTIVE`
  * :setting:`FEED_STORAGE_S3_ACL`
  * :setting:`FEED_EXPORTERS`
- * :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`
+ * :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
 
 .. currentmodule:: scrapy.extensions.feedexport
 
@@ -272,7 +272,7 @@ as a fallback value if that key is not provided for a specific feed definition.
 * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
 * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
 * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
-* ``batch_item_count``: falls back to :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`
+* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
 
 .. setting:: FEED_EXPORT_ENCODING
 
@@ -432,9 +432,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. _botocore: https://github.com/boto/botocore
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
-.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT
+.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT
 
-FEED_STORAGE_BATCH_ITEM_COUNT
+FEED_EXPORT_BATCH_ITEM_COUNT
 -----------------------------
 Default: ``0``
 
@@ -448,16 +448,19 @@ generated:
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
   (e.g. ``2020-03-28T14-45-08.237134``)
 
-* ``%(batch_id)s`` - gets replaced by the batch  sequence number of batch
-  (e.g. ``2`` for the second file)
+* ``%(batch_id)0xd`` - gets replaced by the sequence number of the batch.
+By replacing ``x`` with an integer you set the number of leading zeroes to prevent
+inappropriate sorting like this: [``'1'``, ``'10'``, ``'2'``]. Here are some examples:
+    ``%(batch_id)01d`` for the second batch gets replaced by ``2``
+    ``%(batch_id)05d`` for the third batch gets replaced by ``00003``
 
 For instance, if your settings include::
 
-    FEED_STORAGE_BATCH_ITEM_COUNT=100
+    FEED_EXPORT_BATCH_ITEM_COUNT=100
 
 And your :command:`crawl` command line is::
 
-    scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
+    scrapy crawl spidername -o dirname/%(batch_id)d-filename%(batch_time)s.json
 
 The command line above can generate a directory tree like::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 5908987a3..adb6ea2e4 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst
 
 import logging
 import os
+import re
 import sys
 import warnings
 from datetime import datetime
@@ -25,6 +26,7 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -245,7 +247,7 @@ class FeedExporter:
         for uri, feed in self.feeds.items():
             if not self._storage_supported(uri):
                 raise NotConfigured
-            if not self._settings_are_valid(uri):
+            if not self._settings_are_valid():
                 raise NotConfigured
             if not self._exporter_supported(feed['format']):
                 raise NotConfigured
@@ -298,7 +300,7 @@ class FeedExporter:
     def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
         """
         Redirect the output data stream to a new file.
-        Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
+        Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
         :param batch_id: sequence number of current batch
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
@@ -334,10 +336,10 @@ class FeedExporter:
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
-            # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
+            # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one
             if (
-                    self.feeds[slot.uri_template]['batch_item_count']
-                    and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
+                self.feeds[slot.uri_template]['batch_item_count']
+                and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
             ):
                 uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
                 self._close_slot(slot, spider)
@@ -367,16 +369,18 @@ class FeedExporter:
             return True
         logger.error("Unknown feed format: %(format)s", {'format': format})
 
-    def _settings_are_valid(self, uri):
+    def _settings_are_valid(self):
         """
-        If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
+        If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
         %(batch_time)s or %(batch_id)s to distinguish different files of partial output
         """
         for uri_template, values in self.feeds.items():
-            if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']):
+            if values['batch_item_count'] and not re.findall(r'(%\(batch_time\)s|(%\(batch_id\)0\d*d))', uri_template):
                 logger.error(
-                    '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting '
-                    'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template)
+                    '%(batch_time)s or %(batch_id)0xd must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting '
+                    'or FEEDS.batch_item_count is specified and greater than 0. For more info see:'
+                    'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
+                    ''.format(uri_template)
                 )
                 return False
         return True
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 810acd5a3..0016bbe1b 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
 }
-FEED_STORAGE_BATCH_ITEM_COUNT = 0
+FEED_EXPORT_BATCH_ITEM_COUNT = 0
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 0e02f0f28..64f9c824b 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -113,12 +113,9 @@ def get_sources(use_closest=True):
 
 def feed_complete_default_values_from_settings(feed, settings):
     out = feed.copy()
+    out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
     out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
     out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
-    out.setdefault(
-        "batch_item_count",
-        out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT'))
-    )
     out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
     out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
     if settings["FEED_EXPORT_INDENT"] is None:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index db14b20b9..d20b40e2f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1108,7 +1108,7 @@ class FeedExportTest(FeedExportTestBase):
 
 class BatchDeliveriesTest(FeedExportTestBase):
     __test__ = True
-    _file_mark = '_%(batch_time)s_#%(batch_id)s_'
+    _file_mark = '_%(batch_time)s_#%(batch_id)02d_'
 
     @defer.inlineCallbacks
     def run_and_export(self, spider_cls, settings):
@@ -1144,7 +1144,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['jl']:
@@ -1160,7 +1160,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         data = yield self.exported_data(items, settings)
         for batch in data['csv']:
             got_batch = csv.DictReader(to_unicode(batch).splitlines())
@@ -1176,7 +1176,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['xml']:
@@ -1194,7 +1194,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         # XML
@@ -1219,7 +1219,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import pickle
@@ -1236,7 +1236,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
             },
         })
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import marshal
@@ -1259,18 +1259,18 @@ class BatchDeliveriesTest(FeedExportTestBase):
             {'foo': 'bar3', 'baz': 'quux3', 'egg': ''}
         ]
         settings = {
-            'FEED_STORAGE_BATCH_ITEM_COUNT': 2
+            'FEED_EXPORT_BATCH_ITEM_COUNT': 2
         }
         header = self.MyItem.fields.keys()
         yield self.assertExported(items, header, rows, settings=Settings(settings))
 
     def test_wrong_path(self):
-        """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """
+        """ If path is without %(batch_time)s and %(batch_id)0xd an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},
             },
-            'FEED_STORAGE_BATCH_ITEM_COUNT': 1
+            'FEED_EXPORT_BATCH_ITEM_COUNT': 1
         }
         crawler = get_crawler(settings_dict=settings)
         self.assertRaises(NotConfigured, FeedExporter, crawler)
@@ -1282,7 +1282,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 'FEEDS': {
                     os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
                 },
-                'FEED_STORAGE_BATCH_ITEM_COUNT': 1
+                'FEED_EXPORT_BATCH_ITEM_COUNT': 1
             }
             data = yield self.exported_no_data(settings)
             data = dict(data)
@@ -1304,7 +1304,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 },
                 'FEED_STORE_EMPTY': True,
                 'FEED_EXPORT_INDENT': None,
-                'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+                'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
             }
             data = yield self.exported_no_data(settings)
             data = dict(data)
@@ -1352,7 +1352,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     'encoding': 'utf-8',
                 },
             },
-            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+            'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
         }
         data = yield self.exported_data(items, settings)
         for fmt, expected in formats.items():
@@ -1398,7 +1398,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     'format': 'json',
                 },
             },
-            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+            'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
         }
         data = yield self.exported_data(items, settings)
         self.assertEqual(len(items) + 1, len(data['json']))
@@ -1440,7 +1440,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     'format': 'json',
                 },
             },
-            'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+            'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
         })
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
@@ -1458,7 +1458,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
         s3 = boto3.resource('s3')
         my_bucket = s3.Bucket(s3_test_bucket_name)
-        batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
+        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
 
         with MockServer() as s:
             runner = CrawlerRunner(Settings(settings))
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 95ec2b64a..f3ef36127 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -149,7 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "FEED_EXPORT_INDENT": 42,
             "FEED_STORE_EMPTY": True,
             "FEED_URI_PARAMS": (1, 2, 3, 4),
-            "FEED_STORAGE_BATCH_ITEM_COUNT": 2,
+            "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
         })
         new_feed = feed_complete_default_values_from_settings(feed, settings)
         self.assertEqual(new_feed, {
@@ -171,7 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
             "FEED_EXPORT_INDENT": 42,
             "FEED_STORE_EMPTY": True,
-            "FEED_STORAGE_BATCH_ITEM_COUNT": 2,
+            "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
         })
         new_feed = feed_complete_default_values_from_settings(feed, settings)
         self.assertEqual(new_feed, {

From 3199048520ebe3798c14cfa7362612b51d156b55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 2 Jul 2020 20:10:08 +0200
Subject: [PATCH 2903/2974] Complete Azure Pipelines CI setup

---
 azure-pipelines.yml                 | 18 ++++++------------
 tests/CrawlerRunner/ip_address.py   | 15 ++++++++++++++-
 tests/mockserver.py                 |  5 ++---
 tests/test_commands.py              |  5 +++++
 tests/test_crawler.py               | 13 +++++++++++++
 tests/test_feedexport.py            |  8 +++++++-
 tests/test_proxy_connect.py         |  3 +++
 tests/test_spiderloader/__init__.py | 23 +++++++++++++++--------
 tests/test_utils_asyncio.py         |  7 ++++++-
 tox.ini                             | 18 +++++++++++++++---
 10 files changed, 86 insertions(+), 29 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index ffc4d549b..710e42090 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -1,29 +1,23 @@
-# Python package
-# Create and test a Python package on multiple Python versions.
-# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/python
-
-
+variables:
+  TOXENV: py
 pool:
-  vmImage: 'windows-2019'
+  vmImage: 'windows-latest'
 strategy:
   matrix:
     Python35:
       python.version: '3.5'
-      TOXENV: py35
+      TOXENV: windows-pinned
     Python36:
       python.version: '3.6'
-      TOXENV: py36
     Python37:
       python.version: '3.7'
-      TOXENV: py37
-
+    Python38:
+      python.version: '3.8'
 steps:
 - task: UsePythonVersion@0
   inputs:
     versionSpec: '$(python.version)'
   displayName: 'Use Python $(python.version)'
-
 - script: |
     pip install -U tox twine wheel codecov
     tox
diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py
index 826374cd4..ea75bc3c9 100644
--- a/tests/CrawlerRunner/ip_address.py
+++ b/tests/CrawlerRunner/ip_address.py
@@ -1,7 +1,9 @@
 from urllib.parse import urlparse
 
 from twisted.internet import reactor
-from twisted.names.client import createResolver
+from twisted.names import cache, hosts as hostsModule, resolve
+from twisted.names.client import Resolver
+from twisted.python.runtime import platform
 
 from scrapy import Spider, Request
 from scrapy.crawler import CrawlerRunner
@@ -10,6 +12,17 @@ from scrapy.utils.log import configure_logging
 from tests.mockserver import MockServer, MockDNSServer
 
 
+# https://stackoverflow.com/a/32784190
+def createResolver(servers=None, resolvconf=None, hosts=None):
+    if hosts is None:
+        hosts = (b'/etc/hosts' if platform.getType() == 'posix'
+                 else r'c:\windows\hosts')
+    theResolver = Resolver(resolvconf, servers)
+    hostResolver = hostsModule.Resolver(hosts)
+    L = [hostResolver, cache.CacheResolver(), theResolver]
+    return resolve.ResolverChain(L)
+
+
 class LocalhostSpider(Spider):
     name = "localhost_spider"
 
diff --git a/tests/mockserver.py b/tests/mockserver.py
index df30feab6..1f40473ba 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -247,9 +247,8 @@ class MockDNSServer:
     def __enter__(self):
         self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'],
                           stdout=PIPE, env=get_testenv())
-        host, port = self.proc.stdout.readline().strip().decode('ascii').split(":")
-        self.host = host
-        self.port = int(port)
+        self.host = '127.0.0.1'
+        self.port = int(self.proc.stdout.readline().strip().decode('ascii').split(":")[1])
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 24a341759..ee0e4511a 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -2,6 +2,7 @@ import inspect
 import json
 import optparse
 import os
+import platform
 import subprocess
 import sys
 import tempfile
@@ -10,6 +11,7 @@ from os.path import exists, join, abspath
 from shutil import rmtree, copytree
 from tempfile import mkdtemp
 from threading import Timer
+from unittest import skipIf
 
 from twisted.trial import unittest
 
@@ -319,6 +321,9 @@ class BadSpider(scrapy.Spider):
         self.assertIn("start_requests", log)
         self.assertIn("badspider.py", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_true(self):
         log = self.get_log(self.debug_log_spider, args=[
             '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 78704fb2c..1a4cfe813 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -4,6 +4,7 @@ import platform
 import subprocess
 import sys
 import warnings
+from unittest import skipIf
 
 from pytest import raises, mark
 from testfixtures import LogCapture
@@ -252,6 +253,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
                 })
 
     @defer.inlineCallbacks
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_crawler_process_asyncio_enabled_true(self):
         with LogCapture(level=logging.DEBUG) as log:
             if self.reactor_pytest == 'asyncio':
@@ -293,11 +297,17 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn('Spider closed (finished)', log)
         self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_no_reactor(self):
         log = self.run_script('asyncio_enabled_no_reactor.py')
         self.assertIn('Spider closed (finished)', log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_reactor(self):
         log = self.run_script('asyncio_enabled_reactor.py')
         self.assertIn('Spider closed (finished)', log)
@@ -327,6 +337,9 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_reactor_asyncio(self):
         log = self.run_script("twisted_reactor_asyncio.py")
         self.assertIn("Spider closed (finished)", log)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e38644214..f7b997560 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -455,9 +455,15 @@ class FeedExportTest(unittest.TestCase):
     def run_and_export(self, spider_cls, settings):
         """ Run spider with specified settings; return exported data. """
 
+        def path_to_url(path):
+            return urljoin('file:', pathname2url(str(path)))
+
+        def printf_escape(string):
+            return string.replace('%', '%%')
+
         FEEDS = settings.get('FEEDS') or {}
         settings['FEEDS'] = {
-            urljoin('file:', pathname2url(str(file_path))): feed
+            printf_escape(path_to_url(file_path)): feed
             for file_path, feed in FEEDS.items()
         }
 
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index eb4ecc91d..fc5658ae7 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -1,5 +1,6 @@
 import json
 import os
+import platform
 import re
 import sys
 from subprocess import Popen, PIPE
@@ -59,6 +60,8 @@ def _wrong_credentials(proxy_url):
 
 @skipIf(sys.version_info < (3, 5, 4),
         "requires mitmproxy < 3.0.0, which these tests do not support")
+@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7),
+        "mitmproxy does not support Windows when running Python < 3.7")
 class ProxyConnectTestCase(TestCase):
 
     def setUp(self):
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index d922c6059..4929f1e3e 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -20,13 +20,20 @@ from scrapy.crawler import CrawlerRunner
 module_dir = os.path.dirname(os.path.abspath(__file__))
 
 
+def _copytree(source, target):
+    try:
+        shutil.copytree(source, target)
+    except shutil.Error:
+        pass
+
+
 class SpiderLoaderTest(unittest.TestCase):
 
     def setUp(self):
         orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
         self.tmpdir = tempfile.mkdtemp()
         self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
-        shutil.copytree(orig_spiders_dir, self.spiders_dir)
+        _copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(self.tmpdir)
         settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
         self.spider_loader = SpiderLoader.from_settings(settings)
@@ -124,7 +131,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
         self.tmpdir = self.mktemp()
         os.mkdir(self.tmpdir)
         self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
-        shutil.copytree(orig_spiders_dir, self.spiders_dir)
+        _copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(self.tmpdir)
         self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
 
@@ -134,8 +141,8 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
 
     def test_dupename_warning(self):
         # copy 1 spider module so as to have duplicate spider name
-        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider3.py'),
-                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider3dupe.py'))
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3dupe.py'))
 
         with warnings.catch_warnings(record=True) as w:
             spider_loader = SpiderLoader.from_settings(self.settings)
@@ -156,10 +163,10 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
     def test_multiple_dupename_warning(self):
         # copy 2 spider modules so as to have duplicate spider name
         # This should issue 2 warning, 1 for each duplicate spider name
-        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider1.py'),
-                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider1dupe.py'))
-        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider2.py'),
-                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider2dupe.py'))
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1dupe.py'))
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2dupe.py'))
 
         with warnings.catch_warnings(record=True) as w:
             spider_loader = SpiderLoader.from_settings(self.settings)
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 295323e4d..a2114bd18 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -1,4 +1,6 @@
-from unittest import TestCase
+import platform
+import sys
+from unittest import skipIf, TestCase
 
 from pytest import mark
 
@@ -12,6 +14,9 @@ class AsyncioTest(TestCase):
         # the result should depend only on the pytest --reactor argument
         self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_install_asyncio_reactor(self):
         # this should do nothing
         install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
diff --git a/tox.ini b/tox.ini
index 27d21ade2..f729327ca 100644
--- a/tox.ini
+++ b/tox.ini
@@ -63,14 +63,12 @@ basepython = pypy3
 commands =
     py.test {posargs:--durations=10 docs scrapy tests}
 
-[testenv:pinned]
-basepython = python3
+[pinned]
 deps =
     -ctests/constraints.txt
     cryptography==2.0
     cssselect==0.9.1
     itemadapter==0.1.0
-    lxml==3.5.0
     parsel==1.5.0
     Protego==0.1.15
     PyDispatcher==2.0.5
@@ -85,6 +83,20 @@ deps =
     botocore==1.3.23
     Pillow==3.4.2
 
+[testenv:pinned]
+basepython = python3
+deps =
+    {[pinned]deps}
+    lxml==3.5.0
+
+[testenv:windows-pinned]
+basepython = python3
+deps =
+    {[pinned]deps}
+    # First lxml version that includes a Windows wheel for Python 3.5, so we do
+    # not need to build lxml from sources in a CI Windows job:
+    lxml==3.8.0
+
 [testenv:extra-deps]
 deps =
     {[testenv]deps}

From eb937742566105f3525a9f76e4ae68cc18e9fd8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 3 Jul 2020 01:41:47 +0200
Subject: [PATCH 2904/2974] TrackrefTestCase.test_get_oldest: protect from lack
 of precision

---
 tests/test_utils_trackref.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index 16e02f919..b8e8c3130 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -1,7 +1,10 @@
 import unittest
 from io import StringIO
+from time import sleep, time
 from unittest import mock
 
+from twisted.trial.unittest import SkipTest
+
 from scrapy.utils import trackref
 
 
@@ -55,7 +58,18 @@ Foo                                 1   oldest: 0s ago\n\n''')
 
     def test_get_oldest(self):
         o1 = Foo()  # NOQA
+
+        o1_time = time()
+
         o2 = Bar()  # NOQA
+
+        o3_time = time()
+        if o3_time <= o1_time:
+            sleep(0.01)
+            o3_time = time()
+        if o3_time <= o1_time:
+            raise SkipTest('time.time is not precise enough')
+
         o3 = Foo()  # NOQA
         self.assertIs(trackref.get_oldest('Foo'), o1)
         self.assertIs(trackref.get_oldest('Bar'), o2)

From 6454d456d2bcab0828aba6d81d98f7393ab7e04d Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 3 Jul 2020 08:29:54 +0300
Subject: [PATCH 2905/2974] Make check of placeholder less strict

---
 docs/topics/feed-exports.rst    | 11 ++++++-----
 scrapy/extensions/feedexport.py |  4 ++--
 tests/test_feedexport.py        |  2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 56efa80a7..0bb5f1733 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -448,11 +448,12 @@ generated:
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
   (e.g. ``2020-03-28T14-45-08.237134``)
 
-* ``%(batch_id)0xd`` - gets replaced by the sequence number of the batch.
-By replacing ``x`` with an integer you set the number of leading zeroes to prevent
-inappropriate sorting like this: [``'1'``, ``'10'``, ``'2'``]. Here are some examples:
-    ``%(batch_id)01d`` for the second batch gets replaced by ``2``
-    ``%(batch_id)05d`` for the third batch gets replaced by ``00003``
+* ``%(batch_id)d`` - gets replaced by the sequence number of the batch.
+
+  Use :ref:`printf-style string formatting <python:old-string-formatting>` to
+  alter the number format. For example, to make the batch ID a 5-digit
+  number by introducing leading zeroes as needed, use ``%(batch_id)05d``
+  (e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``).
 
 For instance, if your settings include::
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index adb6ea2e4..e15c1a09c 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -375,9 +375,9 @@ class FeedExporter:
         %(batch_time)s or %(batch_id)s to distinguish different files of partial output
         """
         for uri_template, values in self.feeds.items():
-            if values['batch_item_count'] and not re.findall(r'(%\(batch_time\)s|(%\(batch_id\)0\d*d))', uri_template):
+            if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
                 logger.error(
-                    '%(batch_time)s or %(batch_id)0xd must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting '
+                    '%(batch_time)s or %(batch_id) must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting '
                     'or FEEDS.batch_item_count is specified and greater than 0. For more info see:'
                     'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
                     ''.format(uri_template)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index d20b40e2f..4e0b867a4 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1265,7 +1265,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows, settings=Settings(settings))
 
     def test_wrong_path(self):
-        """ If path is without %(batch_time)s and %(batch_id)0xd an exception must be raised """
+        """ If path is without %(batch_time)s and %(batch_id) an exception must be raised """
         settings = {
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'xml'},

From ec06cf79a6a7264ec3e32d7de8c4e305c2afa05e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 6 Jul 2020 10:47:11 +0200
Subject: [PATCH 2906/2974] Update tests/CrawlerRunner/ip_address.py

Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
---
 tests/CrawlerRunner/ip_address.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py
index ea75bc3c9..b8254afdf 100644
--- a/tests/CrawlerRunner/ip_address.py
+++ b/tests/CrawlerRunner/ip_address.py
@@ -15,8 +15,7 @@ from tests.mockserver import MockServer, MockDNSServer
 # https://stackoverflow.com/a/32784190
 def createResolver(servers=None, resolvconf=None, hosts=None):
     if hosts is None:
-        hosts = (b'/etc/hosts' if platform.getType() == 'posix'
-                 else r'c:\windows\hosts')
+        hosts = b'/etc/hosts' if platform.getType() == 'posix' else r'c:\windows\hosts'
     theResolver = Resolver(resolvconf, servers)
     hostResolver = hostsModule.Resolver(hosts)
     L = [hostResolver, cache.CacheResolver(), theResolver]

From 17aec5944cab33b3cdcd497d2362cacbf7773e47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 6 Jul 2020 10:47:25 +0200
Subject: [PATCH 2907/2974] Update tests/CrawlerRunner/ip_address.py

Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
---
 tests/CrawlerRunner/ip_address.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py
index b8254afdf..3f9738798 100644
--- a/tests/CrawlerRunner/ip_address.py
+++ b/tests/CrawlerRunner/ip_address.py
@@ -18,8 +18,8 @@ def createResolver(servers=None, resolvconf=None, hosts=None):
         hosts = b'/etc/hosts' if platform.getType() == 'posix' else r'c:\windows\hosts'
     theResolver = Resolver(resolvconf, servers)
     hostResolver = hostsModule.Resolver(hosts)
-    L = [hostResolver, cache.CacheResolver(), theResolver]
-    return resolve.ResolverChain(L)
+    chain = [hostResolver, cache.CacheResolver(), theResolver]
+    return resolve.ResolverChain(chain)
 
 
 class LocalhostSpider(Spider):

From f1020e0e6af064ab31b812c25bda6b0f08827222 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Mon, 6 Jul 2020 15:40:53 +0300
Subject: [PATCH 2908/2974] Tiny changes

---
 scrapy/extensions/feedexport.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index e15c1a09c..21177b1b0 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -305,7 +305,7 @@ class FeedExporter:
         :param uri: uri of the new batch to start
         :param feed: dict with parameters of feed
         :param spider: user spider
-        :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri
+        :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri
         """
         storage = self._get_storage(uri)
         file = storage.open(spider)
@@ -372,13 +372,13 @@ class FeedExporter:
     def _settings_are_valid(self):
         """
         If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
-        %(batch_time)s or %(batch_id)s to distinguish different files of partial output
+        %(batch_time)s or %(batch_id)d to distinguish different files of partial output
         """
         for uri_template, values in self.feeds.items():
             if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
                 logger.error(
-                    '%(batch_time)s or %(batch_id) must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting '
-                    'or FEEDS.batch_item_count is specified and greater than 0. For more info see:'
+                    '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT '
+                    'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
                     'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
                     ''.format(uri_template)
                 )

From 770a8127e8e76d95243c1d586b4bb6113a38870a Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Tue, 7 Jul 2020 15:23:29 +0530
Subject: [PATCH 2909/2974] Added basic `scrapy check` tests

---
 tests/test_command_check.py | 96 +++++++++++++++++++++++++++++++++++++
 1 file changed, 96 insertions(+)
 create mode 100644 tests/test_command_check.py

diff --git a/tests/test_command_check.py b/tests/test_command_check.py
new file mode 100644
index 000000000..52005a4c5
--- /dev/null
+++ b/tests/test_command_check.py
@@ -0,0 +1,96 @@
+from os.path import join, abspath
+
+from tests.test_commands import CommandTest
+
+
+class CheckCommandTest(CommandTest):
+
+    command = 'check'
+
+    def setUp(self):
+        super(CheckCommandTest, self).setUp()
+        self.spider_name = 'check_spider'
+        self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py'))
+
+    def _write_contract(self, contracts, parse_def):
+        with open(self.spider, 'w') as file:
+            file.write(f"""
+import scrapy
+
+class CheckSpider(scrapy.Spider):
+    name = '{self.spider_name}'
+    start_urls = ['http://example.com']
+
+    def parse(self, response, **cb_kwargs):
+        \"\"\"
+        @url http://www.amazon.com/s?field-keywords=selfish+gene
+        {contracts}
+        \"\"\"
+        {parse_def}
+            """)
+
+    def _test_contract(self, contracts='', parse_def='pass'):
+        self._write_contract(contracts, parse_def)
+        p, out, err = self.proc('check')
+        self.assertIn('OK', err)
+        self.assertEqual(p.returncode, 0)
+
+    def test_check_returns_requests_contract(self):
+        contracts = """
+        @returns requests 1
+        """
+        parse_def = """
+        yield scrapy.Request(url='http://next-url.com')
+        """
+        self._test_contract(contracts, parse_def)
+
+    def test_check_returns_items_contract(self):
+        contracts = """
+        @returns items 1
+        """
+        parse_def = """
+        yield {'key1': 'val1', 'key2': 'val2'}
+        """
+        self._test_contract(contracts, parse_def)
+
+    def test_check_cb_kwargs_contract(self):
+        contracts = """
+        @cb_kwargs {"arg1": "val1", "arg2": "val2"}
+        """
+        parse_def = """
+        if len(cb_kwargs.items()) == 0:
+            raise Exception("Callback args not set")
+        """
+        self._test_contract(contracts, parse_def)
+
+    def test_check_scrapes_contract(self):
+        contracts = """
+        @scrapes key1 key2
+        """
+        parse_def = """
+        yield {'key1': 'val1', 'key2': 'val2'}
+        """
+        self._test_contract(contracts, parse_def)
+
+    def test_check_all_default_contracts(self):
+        contracts = """
+        @returns items 1
+        @returns requests 1
+        @scrapes key1 key2
+        @cb_kwargs {"arg1": "val1", "arg2": "val2"}
+        """
+        parse_def = """
+        yield {'key1': 'val1', 'key2': 'val2'}
+        yield scrapy.Request(url='http://next-url.com')
+        if len(cb_kwargs.items()) == 0:
+            raise Exception("Callback args not set")
+        """
+        self._test_contract(contracts, parse_def)
+
+    def test_SCRAPY_CHECK_set(self):
+        parse_def = """
+        import os
+        if not os.environ.get('SCRAPY_CHECK'):
+            raise Exception('SCRAPY_CHECK not set')
+        """
+        self._test_contract(parse_def=parse_def)

From d014840672820b3970282f31a51b9ff24cd46bd3 Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Tue, 7 Jul 2020 15:24:33 +0530
Subject: [PATCH 2910/2974] Ignore flake8 E501 for `scrapy check` tests`

---
 pytest.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pytest.ini b/pytest.ini
index bae68cd3a..97320a008 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -173,6 +173,7 @@ flake8-ignore =
     tests/pipelines.py F841 E226
     tests/spiders.py E501 E127
     tests/test_closespider.py E501 E127
+    tests/test_command_check.py E501
     tests/test_command_fetch.py E501
     tests/test_command_parse.py E501 E128 E303 E226
     tests/test_command_shell.py E501 E128

From 79b4dfc53e431bdad31925aaf16831a0dde536ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 7 Jul 2020 14:07:04 +0200
Subject: [PATCH 2911/2974] Fix permission handling on project generation from
 template files

---
 scrapy/commands/startproject.py               |  30 +---
 .../project/module/__init__.py                |   0
 .../project/module/items.py.tmpl              |  12 ++
 .../project/module/middlewares.py.tmpl        | 103 +++++++++++
 .../project/module/pipelines.py.tmpl          |  13 ++
 .../project/module/settings.py.tmpl           |  88 ++++++++++
 .../project/module/spiders/__init__.py        |   4 +
 .../read_only_templates/project/scrapy.cfg    |  11 ++
 .../read_only_templates/spiders/basic.tmpl    |  10 ++
 .../read_only_templates/spiders/crawl.tmpl    |  20 +++
 .../read_only_templates/spiders/csvfeed.tmpl  |  20 +++
 .../read_only_templates/spiders/xmlfeed.tmpl  |  16 ++
 tests/test_commands.py                        | 164 ++++++++++++++++++
 13 files changed, 467 insertions(+), 24 deletions(-)
 create mode 100644 tests/sample_data/read_only_templates/project/module/__init__.py
 create mode 100644 tests/sample_data/read_only_templates/project/module/items.py.tmpl
 create mode 100644 tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl
 create mode 100644 tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl
 create mode 100644 tests/sample_data/read_only_templates/project/module/settings.py.tmpl
 create mode 100644 tests/sample_data/read_only_templates/project/module/spiders/__init__.py
 create mode 100644 tests/sample_data/read_only_templates/project/scrapy.cfg
 create mode 100644 tests/sample_data/read_only_templates/spiders/basic.tmpl
 create mode 100644 tests/sample_data/read_only_templates/spiders/crawl.tmpl
 create mode 100644 tests/sample_data/read_only_templates/spiders/csvfeed.tmpl
 create mode 100644 tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 852281959..e702d7cdc 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,10 +1,10 @@
 import re
 import os
-import stat
 import string
 from importlib import import_module
 from os.path import join, exists, abspath
 from shutil import ignore_patterns, move, copy2, copystat
+from stat import S_IWUSR as OWNER_WRITE_PERMISSION
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -78,30 +78,12 @@ class Command(ScrapyCommand):
                 self._copytree(srcname, dstname)
             else:
                 copy2(srcname, dstname)
+                current_permissions = os.stat(dstname).st_mode
+                os.chmod(dstname, current_permissions | OWNER_WRITE_PERMISSION)
+
         copystat(src, dst)
-        self._set_rw_permissions(dst)
-
-    def _set_rw_permissions(self, path):
-        """
-        Sets permissions of a directory tree to +rw and +rwx for folders.
-        This is necessary if the start template files come without write
-        permissions.
-        """
-        mode_rw = (stat.S_IRUSR
-                   | stat.S_IWUSR
-                   | stat.S_IRGRP
-                   | stat.S_IROTH)
-
-        mode_x = (stat.S_IXUSR
-                  | stat.S_IXGRP
-                  | stat.S_IXOTH)
-
-        os.chmod(path, mode_rw | mode_x)
-        for root, dirs, files in os.walk(path):
-            for dir in dirs:
-                os.chmod(join(root, dir), mode_rw | mode_x)
-            for file in files:
-                os.chmod(join(root, file), mode_rw)
+        current_permissions = os.stat(dst).st_mode
+        os.chmod(dst, current_permissions | OWNER_WRITE_PERMISSION)
 
     def run(self, args, opts):
         if len(args) not in (1, 2):
diff --git a/tests/sample_data/read_only_templates/project/module/__init__.py b/tests/sample_data/read_only_templates/project/module/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/sample_data/read_only_templates/project/module/items.py.tmpl b/tests/sample_data/read_only_templates/project/module/items.py.tmpl
new file mode 100644
index 000000000..88a18331c
--- /dev/null
+++ b/tests/sample_data/read_only_templates/project/module/items.py.tmpl
@@ -0,0 +1,12 @@
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class ${ProjectName}Item(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    pass
diff --git a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl b/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl
new file mode 100644
index 000000000..bd09890fe
--- /dev/null
+++ b/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl
@@ -0,0 +1,103 @@
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+# useful for handling different item types with a single interface
+from itemadapter import is_item, ItemAdapter
+
+
+class ${ProjectName}SpiderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, or item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request or item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class ${ProjectName}DownloaderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
diff --git a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl b/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl
new file mode 100644
index 000000000..e845f43e9
--- /dev/null
+++ b/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl
@@ -0,0 +1,13 @@
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+
+
+# useful for handling different item types with a single interface
+from itemadapter import ItemAdapter
+
+
+class ${ProjectName}Pipeline:
+    def process_item(self, item, spider):
+        return item
diff --git a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl b/tests/sample_data/read_only_templates/project/module/settings.py.tmpl
new file mode 100644
index 000000000..a414b5fde
--- /dev/null
+++ b/tests/sample_data/read_only_templates/project/module/settings.py.tmpl
@@ -0,0 +1,88 @@
+# Scrapy settings for $project_name project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://docs.scrapy.org/en/latest/topics/settings.html
+#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = '$project_name'
+
+SPIDER_MODULES = ['$project_name.spiders']
+NEWSPIDER_MODULE = '$project_name.spiders'
+
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+#USER_AGENT = '$project_name (+http://www.yourdomain.com)'
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = True
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+#CONCURRENT_REQUESTS = 32
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+#DOWNLOAD_DELAY = 3
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN = 16
+#CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+#COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+#DEFAULT_REQUEST_HEADERS = {
+#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#   'Accept-Language': 'en',
+#}
+
+# Enable or disable spider middlewares
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543,
+#}
+
+# Enable or disable downloader middlewares
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#DOWNLOADER_MIDDLEWARES = {
+#    '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543,
+#}
+
+# Enable or disable extensions
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+#}
+
+# Configure item pipelines
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+#ITEM_PIPELINES = {
+#    '$project_name.pipelines.${ProjectName}Pipeline': 300,
+#}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
diff --git a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py b/tests/sample_data/read_only_templates/project/module/spiders/__init__.py
new file mode 100644
index 000000000..ebd689ac5
--- /dev/null
+++ b/tests/sample_data/read_only_templates/project/module/spiders/__init__.py
@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.
diff --git a/tests/sample_data/read_only_templates/project/scrapy.cfg b/tests/sample_data/read_only_templates/project/scrapy.cfg
new file mode 100644
index 000000000..1daeaa541
--- /dev/null
+++ b/tests/sample_data/read_only_templates/project/scrapy.cfg
@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = ${project_name}.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = ${project_name}
diff --git a/tests/sample_data/read_only_templates/spiders/basic.tmpl b/tests/sample_data/read_only_templates/spiders/basic.tmpl
new file mode 100644
index 000000000..e9112bc95
--- /dev/null
+++ b/tests/sample_data/read_only_templates/spiders/basic.tmpl
@@ -0,0 +1,10 @@
+import scrapy
+
+
+class $classname(scrapy.Spider):
+    name = '$name'
+    allowed_domains = ['$domain']
+    start_urls = ['http://$domain/']
+
+    def parse(self, response):
+        pass
diff --git a/tests/sample_data/read_only_templates/spiders/crawl.tmpl b/tests/sample_data/read_only_templates/spiders/crawl.tmpl
new file mode 100644
index 000000000..356496487
--- /dev/null
+++ b/tests/sample_data/read_only_templates/spiders/crawl.tmpl
@@ -0,0 +1,20 @@
+import scrapy
+from scrapy.linkextractors import LinkExtractor
+from scrapy.spiders import CrawlSpider, Rule
+
+
+class $classname(CrawlSpider):
+    name = '$name'
+    allowed_domains = ['$domain']
+    start_urls = ['http://$domain/']
+
+    rules = (
+        Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
+    )
+
+    def parse_item(self, response):
+        item = {}
+        #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
+        #item['name'] = response.xpath('//div[@id="name"]').get()
+        #item['description'] = response.xpath('//div[@id="description"]').get()
+        return item
diff --git a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl b/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl
new file mode 100644
index 000000000..cbcbe9e2c
--- /dev/null
+++ b/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl
@@ -0,0 +1,20 @@
+from scrapy.spiders import CSVFeedSpider
+
+
+class $classname(CSVFeedSpider):
+    name = '$name'
+    allowed_domains = ['$domain']
+    start_urls = ['http://$domain/feed.csv']
+    # headers = ['id', 'name', 'description', 'image_link']
+    # delimiter = '\t'
+
+    # Do any adaptations you need here
+    #def adapt_response(self, response):
+    #    return response
+
+    def parse_row(self, response, row):
+        i = {}
+        #i['url'] = row['url']
+        #i['name'] = row['name']
+        #i['description'] = row['description']
+        return i
diff --git a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl b/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl
new file mode 100644
index 000000000..5aa2aa8b0
--- /dev/null
+++ b/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl
@@ -0,0 +1,16 @@
+from scrapy.spiders import XMLFeedSpider
+
+
+class $classname(XMLFeedSpider):
+    name = '$name'
+    allowed_domains = ['$domain']
+    start_urls = ['http://$domain/feed.xml']
+    iterator = 'iternodes' # you can change this; see the docs
+    itertag = 'item' # change it accordingly
+
+    def parse_node(self, response, selector):
+        item = {}
+        #item['url'] = selector.select('url').get()
+        #item['name'] = selector.select('name').get()
+        #item['description'] = selector.select('description').get()
+        return item
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 24a341759..8336c8759 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -6,7 +6,9 @@ import subprocess
 import sys
 import tempfile
 from contextlib import contextmanager
+from itertools import chain
 from os.path import exists, join, abspath
+from pathlib import Path
 from shutil import rmtree, copytree
 from tempfile import mkdtemp
 from threading import Timer
@@ -15,6 +17,7 @@ from twisted.trial import unittest
 
 import scrapy
 from scrapy.commands import ScrapyCommand
+from scrapy.commands.startproject import IGNORE
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_testenv
@@ -119,8 +122,34 @@ class StartprojectTest(ProjectTest):
         self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params'))
 
 
+def get_permissions_dict(path, renamings=None, ignore=None):
+    renamings = renamings or tuple()
+    permissions_dict = {
+        '.': os.stat(path).st_mode,
+    }
+    for root, dirs, files in os.walk(path):
+        nodes = list(chain(dirs, files))
+        if ignore:
+            ignored_names = ignore(root, nodes)
+            nodes = [node for node in nodes
+                        if node not in ignored_names]
+        for node in nodes:
+            absolute_path = os.path.join(root, node)
+            relative_path = os.path.relpath(absolute_path, path)
+            for search_string, replacement in renamings:
+                relative_path = relative_path.replace(
+                    search_string,
+                    replacement
+                )
+            permissions = os.stat(absolute_path).st_mode
+            permissions_dict[relative_path] = permissions
+    return permissions_dict
+
+
 class StartprojectTemplatesTest(ProjectTest):
 
+    maxDiff = None
+
     def setUp(self):
         super(StartprojectTemplatesTest, self).setUp()
         self.tmpl = join(self.temp_path, 'templates')
@@ -139,6 +168,141 @@ class StartprojectTemplatesTest(ProjectTest):
         self.assertIn(self.tmpl_proj, out)
         assert exists(join(self.proj_path, 'root_template'))
 
+    def test_startproject_permissions_from_writable(self):
+        """Check that generated files have the right permissions when the
+        template folder has the same permissions as in the project, i.e.
+        everything is writable."""
+        scrapy_path = scrapy.__path__[0]
+        templates_dir = os.path.join(scrapy_path, 'templates', 'project')
+        project_name = 'startproject1'
+        renamings = (
+            ('module', project_name),
+            ('.tmpl', ''),
+        )
+        expected_permissions = get_permissions_dict(
+            templates_dir,
+            renamings,
+            IGNORE,
+        )
+
+        destination = mkdtemp()
+        process = subprocess.Popen(
+            (
+                sys.executable,
+                '-m',
+                'scrapy.cmdline',
+                'startproject',
+                project_name,
+            ),
+            cwd=destination,
+            env=self.env,
+        )
+        process.wait()
+
+        project_dir = os.path.join(destination, project_name)
+        actual_permissions = get_permissions_dict(project_dir)
+
+        self.assertEqual(actual_permissions, expected_permissions)
+
+    def test_startproject_permissions_from_read_only(self):
+        """Check that generated files have the right permissions when the
+        template folder has been made read-only, which is something that some
+        systems do.
+
+        See https://github.com/scrapy/scrapy/pull/4604
+        """
+        scrapy_path = scrapy.__path__[0]
+        templates_dir = os.path.join(scrapy_path, 'templates', 'project')
+        project_name = 'startproject2'
+        renamings = (
+            ('module', project_name),
+            ('.tmpl', ''),
+        )
+        expected_permissions = get_permissions_dict(
+            templates_dir,
+            renamings,
+            IGNORE,
+        )
+
+        tests_path = os.path.dirname(__file__)
+        read_only_templates_dir = os.path.join(
+            tests_path, 'sample_data', 'read_only_templates'
+        )
+        destination = mkdtemp()
+        process = subprocess.Popen(
+            (
+                sys.executable,
+                '-m',
+                'scrapy.cmdline',
+                'startproject',
+                project_name,
+                '--set',
+                'TEMPLATES_DIR={}'.format(read_only_templates_dir),
+            ),
+            cwd=destination,
+            env=self.env,
+        )
+        process.wait()
+
+        project_dir = os.path.join(destination, project_name)
+        actual_permissions = get_permissions_dict(project_dir)
+
+        self.assertEqual(actual_permissions, expected_permissions)
+
+    def test_startproject_permissions_unchanged_in_destination(self):
+        """Check that pre-existing folders and files in the destination folder
+        do not see their permissions modified."""
+        scrapy_path = scrapy.__path__[0]
+        templates_dir = os.path.join(scrapy_path, 'templates', 'project')
+        project_name = 'startproject3'
+        renamings = (
+            ('module', project_name),
+            ('.tmpl', ''),
+        )
+        expected_permissions = get_permissions_dict(
+            templates_dir,
+            renamings,
+            IGNORE,
+        )
+
+        destination = mkdtemp()
+        project_dir = os.path.join(destination, project_name)
+
+        existing_nodes = {
+            oct(permissions)[2:] + extension: permissions
+            for extension in ('', '.d')
+            for permissions in (
+                0o444, 0o555, 0o644, 0o666, 0o755, 0o777,
+            )
+        }
+        os.mkdir(project_dir)
+        project_dir_path = Path(project_dir)
+        for node, permissions in existing_nodes.items():
+            path = project_dir_path / node
+            if node.endswith('.d'):
+                path.mkdir(mode=permissions)
+            else:
+                path.touch(mode=permissions)
+            expected_permissions[node] = path.stat().st_mode
+
+        process = subprocess.Popen(
+            (
+                sys.executable,
+                '-m',
+                'scrapy.cmdline',
+                'startproject',
+                project_name,
+                '.',
+            ),
+            cwd=project_dir,
+            env=self.env,
+        )
+        process.wait()
+
+        actual_permissions = get_permissions_dict(project_dir)
+
+        self.assertEqual(actual_permissions, expected_permissions)
+
 
 class CommandTest(ProjectTest):
 

From a3afff4a0e1b25903d1de5c5501846d1f1288d82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 7 Jul 2020 14:11:02 +0200
Subject: [PATCH 2912/2974] Fix style issue

---
 tests/test_commands.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 8336c8759..bd799817d 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -131,8 +131,7 @@ def get_permissions_dict(path, renamings=None, ignore=None):
         nodes = list(chain(dirs, files))
         if ignore:
             ignored_names = ignore(root, nodes)
-            nodes = [node for node in nodes
-                        if node not in ignored_names]
+            nodes = [node for node in nodes if node not in ignored_names]
         for node in nodes:
             absolute_path = os.path.join(root, node)
             relative_path = os.path.relpath(absolute_path, path)

From e1450799ce2dfa32e248cb0b4069668ee5e6f4ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 7 Jul 2020 14:11:37 +0200
Subject: [PATCH 2913/2974] Remove debug test case variable

---
 tests/test_commands.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index bd799817d..c25495d16 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -147,8 +147,6 @@ def get_permissions_dict(path, renamings=None, ignore=None):
 
 class StartprojectTemplatesTest(ProjectTest):
 
-    maxDiff = None
-
     def setUp(self):
         super(StartprojectTemplatesTest, self).setUp()
         self.tmpl = join(self.temp_path, 'templates')

From ca77ca1f751a614b0c9394d1e8c6d0b9cfcdd957 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 7 Jul 2020 14:44:03 +0200
Subject: [PATCH 2914/2974] Generate read-only files on the fly

---
 scrapy/commands/startproject.py               |  13 ++-
 .../project/module/__init__.py                |   0
 .../project/module/items.py.tmpl              |  12 --
 .../project/module/middlewares.py.tmpl        | 103 ------------------
 .../project/module/pipelines.py.tmpl          |  13 ---
 .../project/module/settings.py.tmpl           |  88 ---------------
 .../project/module/spiders/__init__.py        |   4 -
 .../read_only_templates/project/scrapy.cfg    |  11 --
 .../read_only_templates/spiders/basic.tmpl    |  10 --
 .../read_only_templates/spiders/crawl.tmpl    |  20 ----
 .../read_only_templates/spiders/csvfeed.tmpl  |  20 ----
 .../read_only_templates/spiders/xmlfeed.tmpl  |  16 ---
 tests/test_commands.py                        |  31 ++++--
 13 files changed, 28 insertions(+), 313 deletions(-)
 delete mode 100644 tests/sample_data/read_only_templates/project/module/__init__.py
 delete mode 100644 tests/sample_data/read_only_templates/project/module/items.py.tmpl
 delete mode 100644 tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl
 delete mode 100644 tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl
 delete mode 100644 tests/sample_data/read_only_templates/project/module/settings.py.tmpl
 delete mode 100644 tests/sample_data/read_only_templates/project/module/spiders/__init__.py
 delete mode 100644 tests/sample_data/read_only_templates/project/scrapy.cfg
 delete mode 100644 tests/sample_data/read_only_templates/spiders/basic.tmpl
 delete mode 100644 tests/sample_data/read_only_templates/spiders/crawl.tmpl
 delete mode 100644 tests/sample_data/read_only_templates/spiders/csvfeed.tmpl
 delete mode 100644 tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index e702d7cdc..eccc2a3e1 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -20,7 +20,12 @@ TEMPLATES_TO_RENDER = (
     ('${project_name}', 'middlewares.py.tmpl'),
 )
 
-IGNORE = ignore_patterns('*.pyc', '.svn')
+IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn')
+
+
+def _make_writable(path):
+    current_permissions = os.stat(path).st_mode
+    os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)
 
 
 class Command(ScrapyCommand):
@@ -78,12 +83,10 @@ class Command(ScrapyCommand):
                 self._copytree(srcname, dstname)
             else:
                 copy2(srcname, dstname)
-                current_permissions = os.stat(dstname).st_mode
-                os.chmod(dstname, current_permissions | OWNER_WRITE_PERMISSION)
+                _make_writable(dstname)
 
         copystat(src, dst)
-        current_permissions = os.stat(dst).st_mode
-        os.chmod(dst, current_permissions | OWNER_WRITE_PERMISSION)
+        _make_writable(dst)
 
     def run(self, args, opts):
         if len(args) not in (1, 2):
diff --git a/tests/sample_data/read_only_templates/project/module/__init__.py b/tests/sample_data/read_only_templates/project/module/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/tests/sample_data/read_only_templates/project/module/items.py.tmpl b/tests/sample_data/read_only_templates/project/module/items.py.tmpl
deleted file mode 100644
index 88a18331c..000000000
--- a/tests/sample_data/read_only_templates/project/module/items.py.tmpl
+++ /dev/null
@@ -1,12 +0,0 @@
-# Define here the models for your scraped items
-#
-# See documentation in:
-# https://docs.scrapy.org/en/latest/topics/items.html
-
-import scrapy
-
-
-class ${ProjectName}Item(scrapy.Item):
-    # define the fields for your item here like:
-    # name = scrapy.Field()
-    pass
diff --git a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl b/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl
deleted file mode 100644
index bd09890fe..000000000
--- a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl
+++ /dev/null
@@ -1,103 +0,0 @@
-# Define here the models for your spider middleware
-#
-# See documentation in:
-# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
-
-from scrapy import signals
-
-# useful for handling different item types with a single interface
-from itemadapter import is_item, ItemAdapter
-
-
-class ${ProjectName}SpiderMiddleware:
-    # Not all methods need to be defined. If a method is not defined,
-    # scrapy acts as if the spider middleware does not modify the
-    # passed objects.
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        # This method is used by Scrapy to create your spiders.
-        s = cls()
-        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
-        return s
-
-    def process_spider_input(self, response, spider):
-        # Called for each response that goes through the spider
-        # middleware and into the spider.
-
-        # Should return None or raise an exception.
-        return None
-
-    def process_spider_output(self, response, result, spider):
-        # Called with the results returned from the Spider, after
-        # it has processed the response.
-
-        # Must return an iterable of Request, or item objects.
-        for i in result:
-            yield i
-
-    def process_spider_exception(self, response, exception, spider):
-        # Called when a spider or process_spider_input() method
-        # (from other spider middleware) raises an exception.
-
-        # Should return either None or an iterable of Request or item objects.
-        pass
-
-    def process_start_requests(self, start_requests, spider):
-        # Called with the start requests of the spider, and works
-        # similarly to the process_spider_output() method, except
-        # that it doesn’t have a response associated.
-
-        # Must return only requests (not items).
-        for r in start_requests:
-            yield r
-
-    def spider_opened(self, spider):
-        spider.logger.info('Spider opened: %s' % spider.name)
-
-
-class ${ProjectName}DownloaderMiddleware:
-    # Not all methods need to be defined. If a method is not defined,
-    # scrapy acts as if the downloader middleware does not modify the
-    # passed objects.
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        # This method is used by Scrapy to create your spiders.
-        s = cls()
-        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
-        return s
-
-    def process_request(self, request, spider):
-        # Called for each request that goes through the downloader
-        # middleware.
-
-        # Must either:
-        # - return None: continue processing this request
-        # - or return a Response object
-        # - or return a Request object
-        # - or raise IgnoreRequest: process_exception() methods of
-        #   installed downloader middleware will be called
-        return None
-
-    def process_response(self, request, response, spider):
-        # Called with the response returned from the downloader.
-
-        # Must either;
-        # - return a Response object
-        # - return a Request object
-        # - or raise IgnoreRequest
-        return response
-
-    def process_exception(self, request, exception, spider):
-        # Called when a download handler or a process_request()
-        # (from other downloader middleware) raises an exception.
-
-        # Must either:
-        # - return None: continue processing this exception
-        # - return a Response object: stops process_exception() chain
-        # - return a Request object: stops process_exception() chain
-        pass
-
-    def spider_opened(self, spider):
-        spider.logger.info('Spider opened: %s' % spider.name)
diff --git a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl b/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl
deleted file mode 100644
index e845f43e9..000000000
--- a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl
+++ /dev/null
@@ -1,13 +0,0 @@
-# Define your item pipelines here
-#
-# Don't forget to add your pipeline to the ITEM_PIPELINES setting
-# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
-
-
-# useful for handling different item types with a single interface
-from itemadapter import ItemAdapter
-
-
-class ${ProjectName}Pipeline:
-    def process_item(self, item, spider):
-        return item
diff --git a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl b/tests/sample_data/read_only_templates/project/module/settings.py.tmpl
deleted file mode 100644
index a414b5fde..000000000
--- a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl
+++ /dev/null
@@ -1,88 +0,0 @@
-# Scrapy settings for $project_name project
-#
-# For simplicity, this file contains only settings considered important or
-# commonly used. You can find more settings consulting the documentation:
-#
-#     https://docs.scrapy.org/en/latest/topics/settings.html
-#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
-#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
-
-BOT_NAME = '$project_name'
-
-SPIDER_MODULES = ['$project_name.spiders']
-NEWSPIDER_MODULE = '$project_name.spiders'
-
-
-# Crawl responsibly by identifying yourself (and your website) on the user-agent
-#USER_AGENT = '$project_name (+http://www.yourdomain.com)'
-
-# Obey robots.txt rules
-ROBOTSTXT_OBEY = True
-
-# Configure maximum concurrent requests performed by Scrapy (default: 16)
-#CONCURRENT_REQUESTS = 32
-
-# Configure a delay for requests for the same website (default: 0)
-# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
-# See also autothrottle settings and docs
-#DOWNLOAD_DELAY = 3
-# The download delay setting will honor only one of:
-#CONCURRENT_REQUESTS_PER_DOMAIN = 16
-#CONCURRENT_REQUESTS_PER_IP = 16
-
-# Disable cookies (enabled by default)
-#COOKIES_ENABLED = False
-
-# Disable Telnet Console (enabled by default)
-#TELNETCONSOLE_ENABLED = False
-
-# Override the default request headers:
-#DEFAULT_REQUEST_HEADERS = {
-#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-#   'Accept-Language': 'en',
-#}
-
-# Enable or disable spider middlewares
-# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
-#SPIDER_MIDDLEWARES = {
-#    '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543,
-#}
-
-# Enable or disable downloader middlewares
-# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
-#DOWNLOADER_MIDDLEWARES = {
-#    '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543,
-#}
-
-# Enable or disable extensions
-# See https://docs.scrapy.org/en/latest/topics/extensions.html
-#EXTENSIONS = {
-#    'scrapy.extensions.telnet.TelnetConsole': None,
-#}
-
-# Configure item pipelines
-# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
-#ITEM_PIPELINES = {
-#    '$project_name.pipelines.${ProjectName}Pipeline': 300,
-#}
-
-# Enable and configure the AutoThrottle extension (disabled by default)
-# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
-#AUTOTHROTTLE_ENABLED = True
-# The initial download delay
-#AUTOTHROTTLE_START_DELAY = 5
-# The maximum download delay to be set in case of high latencies
-#AUTOTHROTTLE_MAX_DELAY = 60
-# The average number of requests Scrapy should be sending in parallel to
-# each remote server
-#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
-# Enable showing throttling stats for every response received:
-#AUTOTHROTTLE_DEBUG = False
-
-# Enable and configure HTTP caching (disabled by default)
-# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
-#HTTPCACHE_ENABLED = True
-#HTTPCACHE_EXPIRATION_SECS = 0
-#HTTPCACHE_DIR = 'httpcache'
-#HTTPCACHE_IGNORE_HTTP_CODES = []
-#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
diff --git a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py b/tests/sample_data/read_only_templates/project/module/spiders/__init__.py
deleted file mode 100644
index ebd689ac5..000000000
--- a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-# This package will contain the spiders of your Scrapy project
-#
-# Please refer to the documentation for information on how to create and manage
-# your spiders.
diff --git a/tests/sample_data/read_only_templates/project/scrapy.cfg b/tests/sample_data/read_only_templates/project/scrapy.cfg
deleted file mode 100644
index 1daeaa541..000000000
--- a/tests/sample_data/read_only_templates/project/scrapy.cfg
+++ /dev/null
@@ -1,11 +0,0 @@
-# Automatically created by: scrapy startproject
-#
-# For more information about the [deploy] section see:
-# https://scrapyd.readthedocs.io/en/latest/deploy.html
-
-[settings]
-default = ${project_name}.settings
-
-[deploy]
-#url = http://localhost:6800/
-project = ${project_name}
diff --git a/tests/sample_data/read_only_templates/spiders/basic.tmpl b/tests/sample_data/read_only_templates/spiders/basic.tmpl
deleted file mode 100644
index e9112bc95..000000000
--- a/tests/sample_data/read_only_templates/spiders/basic.tmpl
+++ /dev/null
@@ -1,10 +0,0 @@
-import scrapy
-
-
-class $classname(scrapy.Spider):
-    name = '$name'
-    allowed_domains = ['$domain']
-    start_urls = ['http://$domain/']
-
-    def parse(self, response):
-        pass
diff --git a/tests/sample_data/read_only_templates/spiders/crawl.tmpl b/tests/sample_data/read_only_templates/spiders/crawl.tmpl
deleted file mode 100644
index 356496487..000000000
--- a/tests/sample_data/read_only_templates/spiders/crawl.tmpl
+++ /dev/null
@@ -1,20 +0,0 @@
-import scrapy
-from scrapy.linkextractors import LinkExtractor
-from scrapy.spiders import CrawlSpider, Rule
-
-
-class $classname(CrawlSpider):
-    name = '$name'
-    allowed_domains = ['$domain']
-    start_urls = ['http://$domain/']
-
-    rules = (
-        Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
-    )
-
-    def parse_item(self, response):
-        item = {}
-        #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
-        #item['name'] = response.xpath('//div[@id="name"]').get()
-        #item['description'] = response.xpath('//div[@id="description"]').get()
-        return item
diff --git a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl b/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl
deleted file mode 100644
index cbcbe9e2c..000000000
--- a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl
+++ /dev/null
@@ -1,20 +0,0 @@
-from scrapy.spiders import CSVFeedSpider
-
-
-class $classname(CSVFeedSpider):
-    name = '$name'
-    allowed_domains = ['$domain']
-    start_urls = ['http://$domain/feed.csv']
-    # headers = ['id', 'name', 'description', 'image_link']
-    # delimiter = '\t'
-
-    # Do any adaptations you need here
-    #def adapt_response(self, response):
-    #    return response
-
-    def parse_row(self, response, row):
-        i = {}
-        #i['url'] = row['url']
-        #i['name'] = row['name']
-        #i['description'] = row['description']
-        return i
diff --git a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl b/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl
deleted file mode 100644
index 5aa2aa8b0..000000000
--- a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl
+++ /dev/null
@@ -1,16 +0,0 @@
-from scrapy.spiders import XMLFeedSpider
-
-
-class $classname(XMLFeedSpider):
-    name = '$name'
-    allowed_domains = ['$domain']
-    start_urls = ['http://$domain/feed.xml']
-    iterator = 'iternodes' # you can change this; see the docs
-    itertag = 'item' # change it accordingly
-
-    def parse_node(self, response, selector):
-        item = {}
-        #item['url'] = selector.select('url').get()
-        #item['name'] = selector.select('name').get()
-        #item['description'] = selector.select('description').get()
-        return item
diff --git a/tests/test_commands.py b/tests/test_commands.py
index c25495d16..f3fe45139 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -2,6 +2,7 @@ import inspect
 import json
 import optparse
 import os
+from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
 import subprocess
 import sys
 import tempfile
@@ -17,7 +18,7 @@ from twisted.trial import unittest
 
 import scrapy
 from scrapy.commands import ScrapyCommand
-from scrapy.commands.startproject import IGNORE
+from scrapy.commands.startproject import IGNORE, _make_writable
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_testenv
@@ -170,14 +171,14 @@ class StartprojectTemplatesTest(ProjectTest):
         template folder has the same permissions as in the project, i.e.
         everything is writable."""
         scrapy_path = scrapy.__path__[0]
-        templates_dir = os.path.join(scrapy_path, 'templates', 'project')
+        project_template = os.path.join(scrapy_path, 'templates', 'project')
         project_name = 'startproject1'
         renamings = (
             ('module', project_name),
             ('.tmpl', ''),
         )
         expected_permissions = get_permissions_dict(
-            templates_dir,
+            project_template,
             renamings,
             IGNORE,
         )
@@ -209,22 +210,30 @@ class StartprojectTemplatesTest(ProjectTest):
         See https://github.com/scrapy/scrapy/pull/4604
         """
         scrapy_path = scrapy.__path__[0]
-        templates_dir = os.path.join(scrapy_path, 'templates', 'project')
+        templates_dir = os.path.join(scrapy_path, 'templates')
+        project_template = os.path.join(templates_dir, 'project')
         project_name = 'startproject2'
         renamings = (
             ('module', project_name),
             ('.tmpl', ''),
         )
         expected_permissions = get_permissions_dict(
-            templates_dir,
+            project_template,
             renamings,
             IGNORE,
         )
 
-        tests_path = os.path.dirname(__file__)
-        read_only_templates_dir = os.path.join(
-            tests_path, 'sample_data', 'read_only_templates'
-        )
+        def _make_read_only(path):
+            current_permissions = os.stat(path).st_mode
+            os.chmod(path, current_permissions & ~ANYONE_WRITE_PERMISSION)
+
+        read_only_templates_dir = str(Path(mkdtemp()) / 'templates')
+        copytree(templates_dir, read_only_templates_dir)
+
+        for root, dirs, files in os.walk(read_only_templates_dir):
+            for node in chain(dirs, files):
+                _make_read_only(os.path.join(root, node))
+
         destination = mkdtemp()
         process = subprocess.Popen(
             (
@@ -250,14 +259,14 @@ class StartprojectTemplatesTest(ProjectTest):
         """Check that pre-existing folders and files in the destination folder
         do not see their permissions modified."""
         scrapy_path = scrapy.__path__[0]
-        templates_dir = os.path.join(scrapy_path, 'templates', 'project')
+        project_template = os.path.join(scrapy_path, 'templates', 'project')
         project_name = 'startproject3'
         renamings = (
             ('module', project_name),
             ('.tmpl', ''),
         )
         expected_permissions = get_permissions_dict(
-            templates_dir,
+            project_template,
             renamings,
             IGNORE,
         )

From 7e386157033e69bccc56e3a34d5545daa0174d44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 7 Jul 2020 15:30:19 +0200
Subject: [PATCH 2915/2974] Remove unused import

---
 tests/test_commands.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index f3fe45139..002237824 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -18,7 +18,7 @@ from twisted.trial import unittest
 
 import scrapy
 from scrapy.commands import ScrapyCommand
-from scrapy.commands.startproject import IGNORE, _make_writable
+from scrapy.commands.startproject import IGNORE
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_testenv

From 3e98ed24b6e9189d7fc7b4209d24971068274ddb Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Wed, 8 Jul 2020 17:13:57 +0530
Subject: [PATCH 2916/2974] Convert f-string to .format()

---
 tests/test_command_check.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 52005a4c5..72acd817c 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -14,20 +14,20 @@ class CheckCommandTest(CommandTest):
 
     def _write_contract(self, contracts, parse_def):
         with open(self.spider, 'w') as file:
-            file.write(f"""
+            file.write("""
 import scrapy
 
 class CheckSpider(scrapy.Spider):
-    name = '{self.spider_name}'
+    name = '{0}'
     start_urls = ['http://example.com']
 
     def parse(self, response, **cb_kwargs):
         \"\"\"
         @url http://www.amazon.com/s?field-keywords=selfish+gene
-        {contracts}
+        {1}
         \"\"\"
-        {parse_def}
-            """)
+        {2}
+            """.format(self.spider_name, contracts, parse_def))
 
     def _test_contract(self, contracts='', parse_def='pass'):
         self._write_contract(contracts, parse_def)

From 75bff7b6d33bdc74c1a8eb0e43e4b484473c3062 Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Wed, 8 Jul 2020 19:48:42 +0530
Subject: [PATCH 2917/2974] Update url contract value

---
 tests/test_command_check.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 72acd817c..f27f526a3 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -23,7 +23,7 @@ class CheckSpider(scrapy.Spider):
 
     def parse(self, response, **cb_kwargs):
         \"\"\"
-        @url http://www.amazon.com/s?field-keywords=selfish+gene
+        @url http://example.com
         {1}
         \"\"\"
         {2}
@@ -32,6 +32,7 @@ class CheckSpider(scrapy.Spider):
     def _test_contract(self, contracts='', parse_def='pass'):
         self._write_contract(contracts, parse_def)
         p, out, err = self.proc('check')
+        self.assertNotIn('F', out)
         self.assertIn('OK', err)
         self.assertEqual(p.returncode, 0)
 

From 9e99be982a2916559219693e6665ec7e9319f0e9 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 17 Jun 2020 15:52:57 -0300
Subject: [PATCH 2918/2974] Remove backslash

---
 scrapy/cmdline.py                        | 10 ++++++----
 scrapy/commands/crawl.py                 |  6 ++++--
 scrapy/commands/fetch.py                 |  6 ++++--
 scrapy/commands/genspider.py             |  7 ++++---
 scrapy/commands/startproject.py          |  7 ++++---
 scrapy/commands/view.py                  |  3 +--
 scrapy/core/engine.py                    |  8 +++++---
 scrapy/downloadermiddlewares/redirect.py | 13 +++++++------
 scrapy/downloadermiddlewares/retry.py    |  6 ++++--
 scrapy/extensions/memusage.py            | 12 ++++++++----
 scrapy/http/request/form.py              |  3 +--
 scrapy/http/response/text.py             |  5 ++++-
 scrapy/link.py                           | 14 ++++++++++----
 scrapy/settings/__init__.py              |  3 +--
 scrapy/utils/benchserver.py              |  3 +--
 scrapy/utils/conf.py                     | 12 +++++++-----
 scrapy/utils/curl.py                     |  3 +--
 scrapy/utils/ossignal.py                 |  3 +--
 scrapy/utils/response.py                 | 16 ++++++++++------
 scrapy/utils/spider.py                   | 10 ++++++----
 20 files changed, 89 insertions(+), 61 deletions(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index b189e016b..3e88536e4 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -19,10 +19,12 @@ def _iter_command_classes(module_name):
     # scrapy.utils.spider.iter_spider_classes
     for module in walk_modules(module_name):
         for obj in vars(module).values():
-            if inspect.isclass(obj) and \
-                    issubclass(obj, ScrapyCommand) and \
-                    obj.__module__ == module.__name__ and \
-                    not obj == ScrapyCommand:
+            if (
+                inspect.isclass(obj)
+                and issubclass(obj, ScrapyCommand)
+                and obj.__module__ == module.__name__
+                and not obj == ScrapyCommand
+            ):
                 yield obj
 
 
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index e1724c1e6..f205c40b0 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -26,6 +26,8 @@ class Command(BaseRunSpiderCommand):
         else:
             self.crawler_process.start()
 
-            if self.crawler_process.bootstrap_failed or \
-                    (hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception):
+            if (
+                self.crawler_process.bootstrap_failed
+                or hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception
+            ):
                 self.exitcode = 1
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 063195f50..95f87e8c3 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -19,8 +19,10 @@ class Command(ScrapyCommand):
         return "Fetch a URL using the Scrapy downloader"
 
     def long_desc(self):
-        return "Fetch a URL using the Scrapy downloader and print its content " \
-            "to stdout. You may want to use --nolog to disable logging"
+        return (
+            "Fetch a URL using the Scrapy downloader and print its content"
+            " to stdout. You may want to use --nolog to disable logging"
+        )
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index abf3b7a5c..4c7548e9c 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -121,6 +121,7 @@ class Command(ScrapyCommand):
 
     @property
     def templates_dir(self):
-        _templates_base_dir = self.settings['TEMPLATES_DIR'] or \
-            join(scrapy.__path__[0], 'templates')
-        return join(_templates_base_dir, 'spiders')
+        return join(
+            self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
+            'spiders'
+        )
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 852281959..ae4a15b0f 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -137,6 +137,7 @@ class Command(ScrapyCommand):
 
     @property
     def templates_dir(self):
-        _templates_base_dir = self.settings['TEMPLATES_DIR'] or \
-            join(scrapy.__path__[0], 'templates')
-        return join(_templates_base_dir, 'project')
+        return join(
+            self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
+            'project'
+        )
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index 41e77ba3b..908bee966 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -8,8 +8,7 @@ class Command(fetch.Command):
         return "Open URL in browser, as seen by Scrapy"
 
     def long_desc(self):
-        return "Fetch a URL using the Scrapy downloader and show its " \
-            "contents in a browser"
+        return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
 
     def add_options(self, parser):
         super(Command, self).add_options(parser)
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index de0da4b70..86a6abb23 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -141,10 +141,12 @@ class ExecutionEngine:
 
     def _needs_backout(self, spider):
         slot = self.slot
-        return not self.running \
-            or slot.closing \
-            or self.downloader.needs_backout() \
+        return (
+            not self.running
+            or slot.closing
+            or self.downloader.needs_backout()
             or self.scraper.slot.needs_backout()
+        )
 
     def _next_request_from_scheduler(self, spider):
         slot = self.slot
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index b32afb8e4..366d60dcb 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -33,10 +33,8 @@ class BaseRedirectMiddleware:
         if ttl and redirects <= self.max_redirect_times:
             redirected.meta['redirect_times'] = redirects
             redirected.meta['redirect_ttl'] = ttl - 1
-            redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \
-                [request.url]
-            redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \
-                [reason]
+            redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + [request.url]
+            redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + [reason]
             redirected.dont_filter = request.dont_filter
             redirected.priority = request.priority + self.priority_adjust
             logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",
@@ -99,8 +97,11 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
         self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')
 
     def process_response(self, request, response, spider):
-        if request.meta.get('dont_redirect', False) or request.method == 'HEAD' or \
-                not isinstance(response, HtmlResponse):
+        if (
+            request.meta.get('dont_redirect', False)
+            or request.method == 'HEAD'
+            or not isinstance(response, HtmlResponse)
+        ):
             return response
 
         interval, url = get_meta_refresh(response,
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 6d11af5b2..67be8c282 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -60,8 +60,10 @@ class RetryMiddleware:
         return response
 
     def process_exception(self, request, exception, spider):
-        if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \
-                and not request.meta.get('dont_retry', False):
+        if (
+            isinstance(exception, self.EXCEPTIONS_TO_RETRY)
+            and not request.meta.get('dont_retry', False)
+        ):
             return self._retry(request, exception, spider)
 
     def _retry(self, request, reason, spider):
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index a0540bf8f..ab2e43e8c 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -81,8 +81,10 @@ class MemoryUsage:
             logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
                          {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
-                subj = "%s terminated: memory usage exceeded %dM at %s" % \
-                        (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
+                subj = (
+                    "%s terminated: memory usage exceeded %dM at %s"
+                    % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
+                )
                 self._send_report(self.notify_mails, subj)
                 self.crawler.stats.set_value('memusage/limit_notified', 1)
 
@@ -102,8 +104,10 @@ class MemoryUsage:
             logger.warning("Memory usage reached %(memusage)dM",
                            {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
-                subj = "%s warning: memory usage reached %dM at %s" % \
-                        (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
+                subj = (
+                    "%s warning: memory usage reached %dM at %s"
+                    % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
+                )
                 self._send_report(self.notify_mails, subj)
                 self.crawler.stats.set_value('memusage/warning_notified', 1)
             self.warned = True
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index cd4e3373f..0e6ceef0b 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -205,8 +205,7 @@ def _get_clickable(clickdata, form):
 
     # We didn't find it, so now we build an XPath expression out of the other
     # arguments, because they can be used as such
-    xpath = u'.//*' + \
-            u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
+    xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
     el = form.xpath(xpath)
     if len(el) == 1:
         return (el[0].get('name'), el[0].get('value') or '')
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index b43fe5c19..0f300c8da 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -62,8 +62,11 @@ class TextResponse(Response):
         return self._declared_encoding() or self._body_inferred_encoding()
 
     def _declared_encoding(self):
-        return self._encoding or self._headers_encoding() \
+        return (
+            self._encoding
+            or self._headers_encoding()
             or self._body_declared_encoding()
+        )
 
     def body_as_unicode(self):
         """Return body as unicode"""
diff --git a/scrapy/link.py b/scrapy/link.py
index 7cb0765cc..1ef50b113 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -21,12 +21,18 @@ class Link:
         self.nofollow = nofollow
 
     def __eq__(self, other):
-        return self.url == other.url and self.text == other.text and \
-            self.fragment == other.fragment and self.nofollow == other.nofollow
+        return (
+            self.url == other.url
+            and self.text == other.text
+            and self.fragment == other.fragment
+            and self.nofollow == other.nofollow
+        )
 
     def __hash__(self):
         return hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow)
 
     def __repr__(self):
-        return 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' % \
-            (self.url, self.text, self.fragment, self.nofollow)
+        return (
+            'Link(url=%r, text=%r, fragment=%r, nofollow=%r)'
+            % (self.url, self.text, self.fragment, self.nofollow)
+        )
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index b9a13c018..ff8317cd1 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -52,8 +52,7 @@ class SettingsAttribute:
             self.priority = priority
 
     def __str__(self):
-        return "<SettingsAttribute value={self.value!r} " \
-               "priority={self.priority}>".format(self=self)
+        return "<SettingsAttribute value={self.value!r} priority={self.priority}>".format(self=self)
 
     __repr__ = __str__
 
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index 9d8d64612..f595a1acb 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -28,8 +28,7 @@ class Root(Resource):
 
 
 def _getarg(request, name, default=None, type=str):
-    return type(request.args[name][0]) \
-        if name in request.args else default
+    return type(request.args[name][0]) if name in request.args else default
 
 
 if __name__ == '__main__':
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 5921f82bf..728bb5f1b 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -101,11 +101,13 @@ def get_config(use_closest=True):
 
 
 def get_sources(use_closest=True):
-    xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \
-        os.path.expanduser('~/.config')
-    sources = ['/etc/scrapy.cfg', r'c:\scrapy\scrapy.cfg',
-               xdg_config_home + '/scrapy.cfg',
-               os.path.expanduser('~/.scrapy.cfg')]
+    xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.expanduser('~/.config')
+    sources = [
+        '/etc/scrapy.cfg',
+        r'c:\scrapy\scrapy.cfg',
+        xdg_config_home + '/scrapy.cfg',
+        os.path.expanduser('~/.scrapy.cfg'),
+    ]
     if use_closest:
         sources.append(closest_scrapy_cfg())
     return sources
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index 67b22dbc5..aa681522f 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -9,8 +9,7 @@ from w3lib.http import basic_auth_header
 
 class CurlParser(argparse.ArgumentParser):
     def error(self, message):
-        error_msg = \
-            'There was an error parsing the curl command: {}'.format(message)
+        error_msg = 'There was an error parsing the curl command: {}'.format(message)
         raise ValueError(error_msg)
 
 
diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py
index 45c9cef0c..cf867f3f8 100644
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@@ -18,8 +18,7 @@ def install_shutdown_handlers(function, override_sigint=True):
     from twisted.internet import reactor
     reactor._handleSignals()
     signal.signal(signal.SIGTERM, function)
-    if signal.getsignal(signal.SIGINT) == signal.default_int_handler or \
-            override_sigint:
+    if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint:
         signal.signal(signal.SIGINT, function)
     # Catch Ctrl-Break in windows
     if hasattr(signal, 'SIGBREAK'):
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index edbc0db25..c29b619ce 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -47,13 +47,17 @@ def response_httprepr(response):
     is provided only for reference, since it's not the exact stream of bytes
     that was received (that's not exposed by Twisted).
     """
-    s = b"HTTP/1.1 " + to_bytes(str(response.status)) + b" " + \
-        to_bytes(http.RESPONSES.get(response.status, b'')) + b"\r\n"
+    values = [
+        b"HTTP/1.1 ",
+        to_bytes(str(response.status)),
+        b" ",
+        to_bytes(http.RESPONSES.get(response.status, b'')),
+        b"\r\n",
+    ]
     if response.headers:
-        s += response.headers.to_string() + b"\r\n"
-    s += b"\r\n"
-    s += response.body
-    return s
+        values.extend([response.headers.to_string(), b"\r\n"])
+    values.extend([b"\r\n", response.body])
+    return b"".join(values)
 
 
 def open_in_browser(response, _openfunc=webbrowser.open):
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 7e7a50c88..f3a9a67a3 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -34,10 +34,12 @@ def iter_spider_classes(module):
     from scrapy.spiders import Spider
 
     for obj in vars(module).values():
-        if inspect.isclass(obj) and \
-           issubclass(obj, Spider) and \
-           obj.__module__ == module.__name__ and \
-           getattr(obj, 'name', None):
+        if (
+            inspect.isclass(obj)
+            and issubclass(obj, Spider)
+            and obj.__module__ == module.__name__
+            and getattr(obj, 'name', None)
+        ):
             yield obj
 
 

From 9aea1f096171d38348b1403302c6c40eeef7f0a6 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 9 Jul 2020 11:04:46 -0300
Subject: [PATCH 2919/2974] Remove backslash (tests)

---
 ...st_downloadermiddleware_httpcompression.py |  3 +-
 tests/test_downloadermiddleware_redirect.py   | 18 ++++-------
 tests/test_selector.py                        |  3 +-
 tests/test_settings/__init__.py               |  3 +-
 tests/test_spider.py                          | 10 ++++--
 tests/test_spidermiddleware_referer.py        | 32 +++++++++++++------
 tests/test_utils_curl.py                      |  6 ++--
 tests/test_utils_defer.py                     |  9 ++++--
 tests/test_utils_iterators.py                 | 30 +++++++++--------
 tests/test_utils_request.py                   |  8 +++--
 tests/test_utils_response.py                  |  3 +-
 tests/test_utils_sitemap.py                   |  3 +-
 tests/test_utils_url.py                       |  3 +-
 tests/test_webclient.py                       |  5 ++-
 14 files changed, 75 insertions(+), 61 deletions(-)

diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 87304d76c..a806f55ce 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -5,8 +5,7 @@ from gzip import GzipFile
 
 from scrapy.spiders import Spider
 from scrapy.http import Response, Request, HtmlResponse
-from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \
-    ACCEPTED_ENCODINGS
+from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.gz import gunzip
 from tests import tests_datadir
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index c46b1bb87..919dbed23 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -77,12 +77,9 @@ class RedirectMiddlewareTest(unittest.TestCase):
         assert isinstance(req2, Request)
         self.assertEqual(req2.url, url2)
         self.assertEqual(req2.method, 'GET')
-        assert 'Content-Type' not in req2.headers, \
-            "Content-Type header must not be present in redirected request"
-        assert 'Content-Length' not in req2.headers, \
-            "Content-Length header must not be present in redirected request"
-        assert not req2.body, \
-            "Redirected body must be empty, not '%s'" % req2.body
+        assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request"
+        assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request"
+        assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body
 
         # response without Location header but with status code is 3XX should be ignored
         del rsp.headers['Location']
@@ -244,12 +241,9 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
         assert isinstance(req2, Request)
         self.assertEqual(req2.url, 'http://example.org/newpage')
         self.assertEqual(req2.method, 'GET')
-        assert 'Content-Type' not in req2.headers, \
-            "Content-Type header must not be present in redirected request"
-        assert 'Content-Length' not in req2.headers, \
-            "Content-Length header must not be present in redirected request"
-        assert not req2.body, \
-            "Redirected body must be empty, not '%s'" % req2.body
+        assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request"
+        assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request"
+        assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body
 
     def test_max_redirect_times(self):
         self.mw.max_redirect_times = 1
diff --git a/tests/test_selector.py b/tests/test_selector.py
index bcf653444..00e663c11 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -88,8 +88,7 @@ class SelectorTestCase(unittest.TestCase):
         """Check that classes are using slots and are weak-referenceable"""
         x = Selector(text='')
         weakref.ref(x)
-        assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
-            x.__class__.__name__
+        assert not hasattr(x, '__dict__'), "%s does not use __slots__" % x.__class__.__name__
 
     def test_selector_bad_args(self):
         with self.assertRaisesRegex(ValueError, 'received both response and text'):
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 2da6aa4b5..6e56a28f5 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -86,8 +86,7 @@ class BaseSettingsTest(unittest.TestCase):
 
     def test_set_calls_settings_attributes_methods_on_update(self):
         attr = SettingsAttribute('value', 10)
-        with mock.patch.object(attr, '__setattr__') as mock_setattr, \
-                mock.patch.object(attr, 'set') as mock_set:
+        with mock.patch.object(attr, '__setattr__') as mock_setattr, mock.patch.object(attr, 'set') as mock_set:
 
             self.settings.attributes = {'TEST_OPTION': attr}
 
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 805d70459..bd9238810 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -11,8 +11,14 @@ from scrapy import signals
 from scrapy.settings import Settings
 from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse
 from scrapy.spiders.init import InitSpider
-from scrapy.spiders import Spider, CrawlSpider, Rule, XMLFeedSpider, \
-    CSVFeedSpider, SitemapSpider
+from scrapy.spiders import (
+    CSVFeedSpider,
+    CrawlSpider,
+    Rule,
+    SitemapSpider,
+    Spider,
+    XMLFeedSpider,
+)
 from scrapy.linkextractors import LinkExtractor
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.test import get_crawler
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 067118cf0..5141f47af 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -6,16 +6,28 @@ from scrapy.http import Response, Request
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
-from scrapy.spidermiddlewares.referer import RefererMiddleware, \
-    POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \
-    POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \
-    POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \
-    POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, \
-    DefaultReferrerPolicy, \
-    NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \
-    OriginWhenCrossOriginPolicy, OriginPolicy, \
-    StrictOriginWhenCrossOriginPolicy, StrictOriginPolicy, \
-    SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy
+from scrapy.spidermiddlewares.referer import (
+    DefaultReferrerPolicy,
+    NoReferrerPolicy,
+    NoReferrerWhenDowngradePolicy,
+    OriginPolicy,
+    OriginWhenCrossOriginPolicy,
+    POLICY_NO_REFERRER,
+    POLICY_NO_REFERRER_WHEN_DOWNGRADE,
+    POLICY_ORIGIN,
+    POLICY_ORIGIN_WHEN_CROSS_ORIGIN,
+    POLICY_SAME_ORIGIN,
+    POLICY_SCRAPY_DEFAULT,
+    POLICY_STRICT_ORIGIN,
+    POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN,
+    POLICY_UNSAFE_URL,
+    RefererMiddleware,
+    ReferrerPolicy,
+    SameOriginPolicy,
+    StrictOriginPolicy,
+    StrictOriginWhenCrossOriginPolicy,
+    UnsafeUrlPolicy,
+)
 
 
 class TestRefererMiddleware(TestCase):
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index 299a51efe..6b05c8771 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -29,8 +29,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
         self._test_command(curl_command, expected_result)
 
     def test_get_basic_auth(self):
-        curl_command = 'curl "https://api.test.com/" -u ' \
-                       '"some_username:some_password"'
+        curl_command = 'curl "https://api.test.com/" -u "some_username:some_password"'
         expected_result = {
             "method": "GET",
             "url": "https://api.test.com/",
@@ -212,8 +211,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
         with warnings.catch_warnings():  # avoid warning when executing tests
             warnings.simplefilter('ignore')
             curl_command = 'curl --bar --baz http://www.example.com'
-            expected_result = \
-                {"method": "GET", "url": "http://www.example.com"}
+            expected_result = {"method": "GET", "url": "http://www.example.com"}
             self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
 
         # case 2: ignore_unknown_options=False (raise exception):
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 2d4b88121..8c84331b9 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -2,8 +2,13 @@ from twisted.trial import unittest
 from twisted.internet import reactor, defer
 from twisted.python.failure import Failure
 
-from scrapy.utils.defer import mustbe_deferred, process_chain, \
-    process_chain_both, process_parallel, iter_errback
+from scrapy.utils.defer import (
+    iter_errback,
+    mustbe_deferred,
+    process_chain,
+    process_chain_both,
+    process_parallel,
+)
 
 
 class MustbeDeferredTest(unittest.TestCase):
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 8344c6701..3ebe3ac24 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -15,18 +15,20 @@ class XmliterTestCase(unittest.TestCase):
     xmliter = staticmethod(xmliter)
 
     def test_xmliter(self):
-        body = b"""<?xml version="1.0" encoding="UTF-8"?>\
+        body = b"""
+            <?xml version="1.0" encoding="UTF-8"?>
             <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-                      xsi:noNamespaceSchemaLocation="someschmea.xsd">\
-              <product id="001">\
-                <type>Type 1</type>\
-                <name>Name 1</name>\
-              </product>\
-              <product id="002">\
-                <type>Type 2</type>\
-                <name>Name 2</name>\
-              </product>\
-            </products>"""
+                      xsi:noNamespaceSchemaLocation="someschmea.xsd">
+              <product id="001">
+                <type>Type 1</type>
+                <name>Name 1</name>
+              </product>
+              <product id="002">
+                <type>Type 2</type>
+                <name>Name 2</name>
+              </product>
+            </products>
+        """
 
         response = XmlResponse(url="http://example.com", body=body)
         attrs = []
@@ -115,7 +117,7 @@ class XmliterTestCase(unittest.TestCase):
                          [[u'one'], [u'two']])
 
     def test_xmliter_namespaces(self):
-        body = b"""\
+        body = b"""
             <?xml version="1.0" encoding="UTF-8"?>
             <rss version="2.0" xmlns:g="http://base.google.com/ns/1.0">
                 <channel>
@@ -185,7 +187,7 @@ class LxmlXmliterTestCase(XmliterTestCase):
     xmliter = staticmethod(xmliter_lxml)
 
     def test_xmliter_iterate_namespace(self):
-        body = b"""\
+        body = b"""
             <?xml version="1.0" encoding="UTF-8"?>
             <rss version="2.0" xmlns="http://base.google.com/ns/1.0">
                 <channel>
@@ -214,7 +216,7 @@ class LxmlXmliterTestCase(XmliterTestCase):
         self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg'])
 
     def test_xmliter_namespaces_prefix(self):
-        body = b"""\
+        body = b"""
         <?xml version="1.0" encoding="UTF-8"?>
         <root>
             <h:table xmlns:h="http://www.w3.org/TR/html4/">
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 4cd4b7010..7e0049b1d 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,7 +1,11 @@
 import unittest
 from scrapy.http import Request
-from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \
-    request_authenticate, request_httprepr
+from scrapy.utils.request import (
+    _fingerprint_cache,
+    request_authenticate,
+    request_fingerprint,
+    request_httprepr,
+)
 
 
 class UtilsRequestTest(unittest.TestCase):
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 6ebf290c0..d6f4c0bb5 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -37,8 +37,7 @@ class ResponseUtilsTest(unittest.TestCase):
             self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody)
             return True
         response = HtmlResponse(url, body=body)
-        assert open_in_browser(response, _openfunc=browser_open), \
-            "Browser not called"
+        assert open_in_browser(response, _openfunc=browser_open), "Browser not called"
 
         resp = Response(url, body=body)
         self.assertRaises(TypeError, open_in_browser, resp, debug=True)
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index bfbf9abb3..23eb261b7 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -156,8 +156,7 @@ Disallow: /forum/active/
 
     def test_sitemap_blanklines(self):
         """Assert we can deal with starting blank lines before <xml> tag"""
-        s = Sitemap(b"""\
-
+        s = Sitemap(b"""
 <?xml version="1.0" encoding="UTF-8"?>
 <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 09a6d6c70..a194a0998 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -207,8 +207,7 @@ def create_guess_scheme_t(args):
     def do_expected(self):
         url = guess_scheme(args[0])
         assert url.startswith(args[1]), \
-            'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (
-                args[0], url, args[1])
+            'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (args[0], url, args[1])
     return do_expected
 
 
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 188e54602..c1c5945c2 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -356,9 +356,8 @@ class WebClientTestCase(unittest.TestCase):
         """ Test that non-standart body encoding matches
         Content-Encoding header """
         body = b'\xd0\x81\xd1\x8e\xd0\xaf'
-        return getPage(
-            self.getURL('encoding'), body=body, response_transform=lambda r: r)\
-            .addCallback(self._check_Encoding, body)
+        dfd = getPage(self.getURL('encoding'), body=body, response_transform=lambda r: r)
+        return dfd.addCallback(self._check_Encoding, body)
 
     def _check_Encoding(self, response, original_body):
         content_encoding = to_unicode(response.headers[b'Content-Encoding'])

From cbe4dc57f3f65ecb851941dcfae0bc18c6c8582a Mon Sep 17 00:00:00 2001
From: Ajay Mittur <ajay.cs18@bmsce.ac.in>
Date: Fri, 10 Jul 2020 18:22:43 +0530
Subject: [PATCH 2920/2974] Update pytest.ini

---
 pytest.ini | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pytest.ini b/pytest.ini
index 92c5bcb75..ca8191f42 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -39,4 +39,5 @@ flake8-ignore =
     scrapy/utils/markup.py F403
     scrapy/utils/multipart.py F403
     scrapy/utils/url.py F403 F405
-    tests/test_loader.py E741
\ No newline at end of file
+    tests/test_loader.py E741
+    

From a6a5fa91da8944943e2c9d8f34f09662be17b781 Mon Sep 17 00:00:00 2001
From: Artur Shellunts <shellunts.artur@gmail.com>
Date: Fri, 10 Jul 2020 23:10:49 +0200
Subject: [PATCH 2921/2974] Remove deprecated class HtmlParserLinkExtractor

Issue #4356
---
 scrapy/linkextractors/htmlparser.py | 91 -----------------------------
 1 file changed, 91 deletions(-)
 delete mode 100644 scrapy/linkextractors/htmlparser.py

diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
deleted file mode 100644
index 0425d4340..000000000
--- a/scrapy/linkextractors/htmlparser.py
+++ /dev/null
@@ -1,91 +0,0 @@
-"""
-HTMLParser-based link extractor
-"""
-import warnings
-from html.parser import HTMLParser
-from urllib.parse import urljoin
-
-from w3lib.url import safe_url_string
-from w3lib.html import strip_html5_whitespace
-
-from scrapy.link import Link
-from scrapy.utils.python import unique as unique_list
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-
-class HtmlParserLinkExtractor(HTMLParser):
-
-    def __init__(self, tag="a", attr="href", process=None, unique=False,
-                 strip=True):
-        HTMLParser.__init__(self)
-
-        warnings.warn(
-            "HtmlParserLinkExtractor is deprecated and will be removed in "
-            "future releases. Please use scrapy.linkextractors.LinkExtractor",
-            ScrapyDeprecationWarning, stacklevel=2,
-        )
-
-        self.scan_tag = tag if callable(tag) else lambda t: t == tag
-        self.scan_attr = attr if callable(attr) else lambda a: a == attr
-        self.process_attr = process if callable(process) else lambda v: v
-        self.unique = unique
-        self.strip = strip
-
-    def _extract_links(self, response_text, response_url, response_encoding):
-        self.reset()
-        self.feed(response_text)
-        self.close()
-
-        links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links
-
-        ret = []
-        base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
-        for link in links:
-            if isinstance(link.url, str):
-                link.url = link.url.encode(response_encoding)
-            try:
-                link.url = urljoin(base_url, link.url)
-            except ValueError:
-                continue
-            link.url = safe_url_string(link.url, response_encoding)
-            link.text = link.text.decode(response_encoding)
-            ret.append(link)
-
-        return ret
-
-    def extract_links(self, response):
-        # wrapper needed to allow to work directly with text
-        return self._extract_links(response.body, response.url, response.encoding)
-
-    def reset(self):
-        HTMLParser.reset(self)
-
-        self.base_url = None
-        self.current_link = None
-        self.links = []
-
-    def handle_starttag(self, tag, attrs):
-        if tag == 'base':
-            self.base_url = dict(attrs).get('href')
-        if self.scan_tag(tag):
-            for attr, value in attrs:
-                if self.scan_attr(attr):
-                    if self.strip:
-                        value = strip_html5_whitespace(value)
-                    url = self.process_attr(value)
-                    link = Link(url=url)
-                    self.links.append(link)
-                    self.current_link = link
-
-    def handle_endtag(self, tag):
-        if self.scan_tag(tag):
-            self.current_link = None
-
-    def handle_data(self, data):
-        if self.current_link:
-            self.current_link.text = self.current_link.text + data
-
-    def matches(self, url):
-        """This extractor matches with any url, since
-        it doesn't contain any patterns"""
-        return True

From 3f7e8635f479f4de2ab1c3d518010730a5f6f0a6 Mon Sep 17 00:00:00 2001
From: Aditya Kumar <k.aditya00@gmail.com>
Date: Sat, 11 Jul 2020 12:18:24 +0530
Subject: [PATCH 2922/2974] Allow the parse command to write data to a file 
 (#4377)

---
 docs/topics/commands.rst    |  4 +++-
 scrapy/commands/__init__.py |  2 +-
 scrapy/commands/parse.py    | 24 ++++++++----------------
 tests/test_command_parse.py | 23 ++++++++++++++++++++++-
 4 files changed, 34 insertions(+), 19 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index a0dcba90d..4fce51abc 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -468,7 +468,7 @@ Supported options:
 * ``--callback`` or ``-c``: spider method to use as callback for parsing the
   response
 
-* ``--meta`` or ``-m``: additional request meta that will be passed to the callback 
+* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
   request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
 
 * ``--cbkwargs``: additional keyword arguments that will be passed to the callback.
@@ -491,6 +491,8 @@ Supported options:
 
 * ``--verbose`` or ``-v``: display information for each depth level
 
+* ``--output`` or ``-o``: dump scraped items to a file
+
 .. skip: start
 
 Usage example::
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index ab850dcb3..57ce4e522 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -108,7 +108,7 @@ class ScrapyCommand:
 
 class BaseRunSpiderCommand(ScrapyCommand):
     """
-    Common class used to share functionality between the crawl and runspider commands
+    Common class used to share functionality between the crawl, parse and runspider commands
     """
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 8b7fa8b58..abc8ba9ff 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -4,18 +4,16 @@ import logging
 from itemadapter import is_item, ItemAdapter
 from w3lib.url import is_url
 
-from scrapy.commands import ScrapyCommand
+from scrapy.commands import BaseRunSpiderCommand
 from scrapy.http import Request
 from scrapy.utils import display
-from scrapy.utils.conf import arglist_to_dict
 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
 from scrapy.exceptions import UsageError
 
 logger = logging.getLogger(__name__)
 
 
-class Command(ScrapyCommand):
-
+class Command(BaseRunSpiderCommand):
     requires_project = True
 
     spider = None
@@ -31,11 +29,9 @@ class Command(ScrapyCommand):
         return "Parse URL (using its spider) and print the results"
 
     def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+        BaseRunSpiderCommand.add_options(self, parser)
         parser.add_option("--spider", dest="spider", default=None,
                           help="use this spider without looking for one")
-        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
-                          help="set spider argument (may be repeated)")
         parser.add_option("--pipelines", action="store_true",
                           help="process items through pipelines")
         parser.add_option("--nolinks", dest="nolinks", action="store_true",
@@ -200,12 +196,15 @@ class Command(ScrapyCommand):
             self.add_items(depth, items)
             self.add_requests(depth, requests)
 
+            scraped_data = items if opts.output else []
             if depth < opts.depth:
                 for req in requests:
                     req.meta['_depth'] = depth + 1
                     req.meta['_callback'] = req.callback
                     req.callback = callback
-                return requests
+                scraped_data += requests
+
+            return scraped_data
 
         # update request meta if any extra meta was passed through the --meta/-m opts.
         if opts.meta:
@@ -221,18 +220,11 @@ class Command(ScrapyCommand):
         return request
 
     def process_options(self, args, opts):
-        ScrapyCommand.process_options(self, args, opts)
+        BaseRunSpiderCommand.process_options(self, args, opts)
 
-        self.process_spider_arguments(opts)
         self.process_request_meta(opts)
         self.process_request_cb_kwargs(opts)
 
-    def process_spider_arguments(self, opts):
-        try:
-            opts.spargs = arglist_to_dict(opts.spargs)
-        except ValueError:
-            raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
-
     def process_request_meta(self, opts):
         if opts.meta:
             try:
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index a09dcf072..5754a5478 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,5 +1,5 @@
 import os
-from os.path import join, abspath
+from os.path import join, abspath, isfile, exists
 from twisted.internet import defer
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
@@ -218,3 +218,24 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         )
         self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))
+
+    @defer.inlineCallbacks
+    def test_output_flag(self):
+        """Checks if a file was created successfully having
+        correct format containing correct data in it.
+        """
+        file_name = 'data.json'
+        file_path = join(self.proj_path, file_name)
+        yield self.execute([
+            '--spider', self.spider_name,
+            '-c', 'parse',
+            '-o', file_name,
+            self.url('/html')
+        ])
+
+        self.assertTrue(exists(file_path))
+        self.assertTrue(isfile(file_path))
+
+        content = '[\n{},\n{"foo": "bar"}\n]'
+        with open(file_path, 'r') as f:
+            self.assertEqual(f.read(), content)

From d54c4496ee57785f3d6f882e2d128bb64b6b262c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 13 Jul 2020 14:36:33 +0200
Subject: [PATCH 2923/2974] Refactor guess_scheme

---
 scrapy/utils/url.py     | 60 ++++++++++++++++++++++++++++++-----------
 tests/test_utils_url.py | 33 +++++++++++++++++++++--
 2 files changed, 75 insertions(+), 18 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 60e2b50eb..1e431d3bf 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -83,26 +83,54 @@ def add_http_if_no_scheme(url):
     return url
 
 
+def _is_posix_path(string):
+    return bool(
+        re.match(
+            r'''
+            ^                   # start with...
+            (
+                \.              # ...a single dot,
+                (
+                    \. | [^/\.]+  # optionally followed by
+                )?                # either a second dot or some characters
+                |
+                ~   # $HOME
+            )?      # optional match of ".", ".." or ".blabla"
+            /       # at least one "/" for a file path,
+            .       # and something after the "/"
+            ''',
+            string,
+            flags=re.VERBOSE,
+        )
+    )
+
+
+def _is_windows_path(string):
+    return bool(
+        re.match(
+            r'''
+            ^
+            (
+                [a-z]:\\
+                | \\\\
+            )
+            ''',
+            string,
+            flags=re.IGNORECASE | re.VERBOSE,
+        )
+    )
+
+
+def _is_path(string):
+    return _is_posix_path(string) or _is_windows_path(string)
+
+
 def guess_scheme(url):
     """Add an URL scheme if missing: file:// for filepath-like input or
     http:// otherwise."""
-    # POSIX path
-    if re.match(r'''^                   # start with...
-                    (
-                        \.              # ...a single dot,
-                        (
-                            \. | [^/\.]+  # optionally followed by
-                        )?                # either a second dot or some characters
-                    )?      # optional match of ".", ".." or ".blabla"
-                    /       # at least one "/" for a file path,
-                    .       # and something after the "/"
-                    ''', url, flags=re.VERBOSE):
+    if _is_path(url):
         return any_to_uri(url)
-    # Windows drive-letter path
-    elif re.match(r'''^[a-z]:\\''', url, flags=re.IGNORECASE):
-        return any_to_uri(url)
-    else:
-        return add_http_if_no_scheme(url)
+    return add_http_if_no_scheme(url)
 
 
 def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True):
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 09a6d6c70..6a5254d54 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,8 +1,14 @@
 import unittest
 
 from scrapy.spiders import Spider
-from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
-                              add_http_if_no_scheme, guess_scheme, strip_url)
+from scrapy.utils.url import (
+    add_http_if_no_scheme,
+    guess_scheme,
+    _is_path,
+    strip_url,
+    url_is_from_any_domain,
+    url_is_from_spider,
+)
 
 
 __doctests__ = ['scrapy.utils.url']
@@ -434,5 +440,28 @@ class StripUrl(unittest.TestCase):
             self.assertEqual(strip_url(i, origin_only=True), o)
 
 
+class IsPathTestCase(unittest.TestCase):
+
+    def test_path(self):
+        for input_value, output_value in (
+            # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell
+            # Unix-like OS, Microsoft Windows / cmd.exe
+            ("/home/user/docs/Letter.txt", True),
+            ("./inthisdir", True),
+            ("../../greatgrandparent", True),
+            ("~/.rcinfo", True),
+            (r"C:\user\docs\Letter.txt", True),
+            ("/user/docs/Letter.txt", True),
+            (r"C:\Letter.txt", True),
+            (r"\\Server01\user\docs\Letter.txt", True),
+            (r"\\?\UNC\Server01\user\docs\Letter.txt", True),
+            (r"\\?\C:\user\docs\Letter.txt", True),
+            (r"C:\user\docs\somefile.ext:alternate_stream_name", True),
+
+            (r"https://example.com", False),
+        ):
+            self.assertEqual(_is_path(input_value), output_value, input_value)
+
+
 if __name__ == "__main__":
     unittest.main()

From 53c323b19d81784e6c376ce8b9602de24d8e3037 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 13 Jul 2020 15:29:30 +0200
Subject: [PATCH 2924/2974] =?UTF-8?q?=5Fis=5Fpath=20=E2=86=92=20=5Fis=5Ffi?=
 =?UTF-8?q?lesystem=5Fpath?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scrapy/utils/url.py     | 4 ++--
 tests/test_utils_url.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 1e431d3bf..b23ddb459 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -121,14 +121,14 @@ def _is_windows_path(string):
     )
 
 
-def _is_path(string):
+def _is_filesystem_path(string):
     return _is_posix_path(string) or _is_windows_path(string)
 
 
 def guess_scheme(url):
     """Add an URL scheme if missing: file:// for filepath-like input or
     http:// otherwise."""
-    if _is_path(url):
+    if _is_filesystem_path(url):
         return any_to_uri(url)
     return add_http_if_no_scheme(url)
 
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 6a5254d54..3a143ba2f 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -4,7 +4,7 @@ from scrapy.spiders import Spider
 from scrapy.utils.url import (
     add_http_if_no_scheme,
     guess_scheme,
-    _is_path,
+    _is_filesystem_path,
     strip_url,
     url_is_from_any_domain,
     url_is_from_spider,
@@ -460,7 +460,7 @@ class IsPathTestCase(unittest.TestCase):
 
             (r"https://example.com", False),
         ):
-            self.assertEqual(_is_path(input_value), output_value, input_value)
+            self.assertEqual(_is_filesystem_path(input_value), output_value, input_value)
 
 
 if __name__ == "__main__":

From ed5247ca4cbef98d1acf499c888ded74e444ef48 Mon Sep 17 00:00:00 2001
From: Artur Shellunts <shellunts.artur@gmail.com>
Date: Tue, 14 Jul 2020 18:06:11 +0200
Subject: [PATCH 2925/2974] Remove htmlparser.py from tests/ignore.txt

---
 tests/ignores.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/ignores.txt b/tests/ignores.txt
index 45cf6fb92..f6e0d6fbe 100644
--- a/tests/ignores.txt
+++ b/tests/ignores.txt
@@ -1,6 +1,5 @@
 scrapy/linkextractors/sgml.py
 scrapy/linkextractors/regex.py
-scrapy/linkextractors/htmlparser.py
 scrapy/downloadermiddlewares/cookies.py
 scrapy/extensions/statsmailer.py
 scrapy/extensions/memusage.py

From 38496a00b7d2bcfa9d435551409e2c44007d168d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com>
Date: Wed, 15 Jul 2020 07:08:36 -0300
Subject: [PATCH 2926/2974] Use the itemlaoders library (#4516)

---
 docs/conf.py                    |  13 +
 docs/topics/loaders.rst         | 418 +-----------------
 scrapy/loader/__init__.py       | 269 +++---------
 scrapy/loader/common.py         |  19 +-
 scrapy/loader/processors.py     |  99 +----
 scrapy/utils/misc.py            |   6 +
 setup.cfg                       |   3 +
 setup.py                        |   1 +
 tests/requirements-py3.txt      |   1 -
 tests/test_loader.py            | 531 +----------------------
 tests/test_loader_deprecated.py | 720 ++++++++++++++++++++++++++++++++
 11 files changed, 854 insertions(+), 1226 deletions(-)
 create mode 100644 tests/test_loader_deprecated.py

diff --git a/docs/conf.py b/docs/conf.py
index 86734fae7..427c79481 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -284,6 +284,7 @@ intersphinx_mapping = {
     'attrs': ('https://www.attrs.org/en/stable/', None),
     'coverage': ('https://coverage.readthedocs.io/en/stable', None),
     'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
+    'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
     'pytest': ('https://docs.pytest.org/en/latest', None),
     'python': ('https://docs.python.org/3', None),
     'sphinx': ('https://www.sphinx-doc.org/en/master', None),
@@ -305,3 +306,15 @@ hoverxref_role_types = {
     "ref": "tooltip",
 }
 hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
+
+
+def setup(app):
+    app.connect('autodoc-skip-member', maybe_skip_member)
+
+
+def maybe_skip_member(app, what, name, obj, skip, options):
+    if not skip:
+        # autodocs was generating a text "alias of" for the following members
+        # https://github.com/sphinx-doc/sphinx/issues/4422
+        return name in {'default_item_class', 'default_selector_class'}
+    return skip
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 9c82bb4d9..d0eeb4097 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -20,6 +20,10 @@ Item Loaders are designed to provide a flexible, efficient and easy mechanism
 for extending and overriding different field parsing rules, either by spider,
 or by source format (HTML, XML, etc) without becoming a nightmare to maintain.
 
+.. note:: Item Loaders are an extension of the itemloaders_ library that make it
+    easier to work with Scrapy by adding support for
+    :ref:`responses <topics-request-response>`.
+
 Using Item Loaders to populate items
 ====================================
 
@@ -173,8 +177,8 @@ The other thing you need to keep in mind is that the values returned by input
 processors are collected internally (in lists) and then passed to output
 processors to populate the fields.
 
-Last, but not least, Scrapy comes with some :ref:`commonly used processors
-<topics-loaders-available-processors>` built-in for convenience.
+Last, but not least, itemloaders_ comes with some :ref:`commonly used
+processors <itemloaders:built-in-processors>` built-in for convenience.
 
 
 Declaring Item Loaders
@@ -182,8 +186,8 @@ Declaring Item Loaders
 
 Item Loaders are declared using a class definition syntax. Here is an example::
 
+    from itemloaders.processors import TakeFirst, MapCompose, Join
     from scrapy.loader import ItemLoader
-    from scrapy.loader.processors import TakeFirst, MapCompose, Join
 
     class ProductLoader(ItemLoader):
 
@@ -214,7 +218,7 @@ output processors to use: in the :ref:`Item Field <topics-items-fields>`
 metadata. Here is an example::
 
     import scrapy
-    from scrapy.loader.processors import Join, MapCompose, TakeFirst
+    from itemloaders.processors import Join, MapCompose, TakeFirst
     from w3lib.html import remove_tags
 
     def filter_price(value):
@@ -295,250 +299,9 @@ There are several ways to modify Item Loader context values:
 ItemLoader objects
 ==================
 
-.. class:: ItemLoader([item, selector, response], **kwargs)
-
-    Return a new Item Loader for populating the given :ref:`item object
-    <topics-items>`. If no item object is given, one is instantiated
-    automatically using the class in :attr:`default_item_class`.
-
-    When instantiated with a ``selector`` or a ``response`` parameters
-    the :class:`ItemLoader` class provides convenient mechanisms for extracting
-    data from web pages using :ref:`selectors <topics-selectors>`.
-
-    :param item: The item instance to populate using subsequent calls to
-        :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
-        or :meth:`~ItemLoader.add_value`.
-    :type item: :ref:`item object <topics-items>`
-
-    :param selector: The selector to extract data from, when using the
-        :meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath`
-        (resp. :meth:`replace_css`) method.
-    :type selector: :class:`~scrapy.selector.Selector` object
-
-    :param response: The response used to construct the selector using the
-        :attr:`default_selector_class`, unless the selector argument is given,
-        in which case this argument is ignored.
-    :type response: :class:`~scrapy.http.Response` object
-
-    The item, selector, response and the remaining keyword arguments are
-    assigned to the Loader context (accessible through the :attr:`context` attribute).
-
-    :class:`ItemLoader` instances have the following methods:
-
-    .. method:: get_value(value, *processors, **kwargs)
-
-        Process the given ``value`` by the given ``processors`` and keyword
-        arguments.
-
-        Available keyword arguments:
-
-        :param re: a regular expression to use for extracting data from the
-            given value using :meth:`~scrapy.utils.misc.extract_regex` method,
-            applied before processors
-        :type re: str or compiled regex
-
-        Examples:
-
-        >>> from scrapy.loader.processors import TakeFirst
-        >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
-        'FOO`
-
-    .. method:: add_value(field_name, value, *processors, **kwargs)
-
-        Process and then add the given ``value`` for the given field.
-
-        The value is first passed through :meth:`get_value` by giving the
-        ``processors`` and ``kwargs``, and then passed through the
-        :ref:`field input processor <topics-loaders-processors>` and its result
-        appended to the data collected for that field. If the field already
-        contains collected data, the new data is added.
-
-        The given ``field_name`` can be ``None``, in which case values for
-        multiple fields may be added. And the processed value should be a dict
-        with field_name mapped to values.
-
-        Examples::
-
-            loader.add_value('name', u'Color TV')
-            loader.add_value('colours', [u'white', u'blue'])
-            loader.add_value('length', u'100')
-            loader.add_value('name', u'name: foo', TakeFirst(), re='name: (.+)')
-            loader.add_value(None, {'name': u'foo', 'sex': u'male'})
-
-    .. method:: replace_value(field_name, value, *processors, **kwargs)
-
-        Similar to :meth:`add_value` but replaces the collected data with the
-        new value instead of adding it.
-    .. method:: get_xpath(xpath, *processors, **kwargs)
-
-        Similar to :meth:`ItemLoader.get_value` but receives an XPath instead of a
-        value, which is used to extract a list of unicode strings from the
-        selector associated with this :class:`ItemLoader`.
-
-        :param xpath: the XPath to extract data from
-        :type xpath: str
-
-        :param re: a regular expression to use for extracting data from the
-            selected XPath region
-        :type re: str or compiled regex
-
-        Examples::
-
-            # HTML snippet: <p class="product-name">Color TV</p>
-            loader.get_xpath('//p[@class="product-name"]')
-            # HTML snippet: <p id="price">the price is $1200</p>
-            loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)')
-
-    .. method:: add_xpath(field_name, xpath, *processors, **kwargs)
-
-        Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a
-        value, which is used to extract a list of unicode strings from the
-        selector associated with this :class:`ItemLoader`.
-
-        See :meth:`get_xpath` for ``kwargs``.
-
-        :param xpath: the XPath to extract data from
-        :type xpath: str
-
-        Examples::
-
-            # HTML snippet: <p class="product-name">Color TV</p>
-            loader.add_xpath('name', '//p[@class="product-name"]')
-            # HTML snippet: <p id="price">the price is $1200</p>
-            loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)')
-
-    .. method:: replace_xpath(field_name, xpath, *processors, **kwargs)
-
-        Similar to :meth:`add_xpath` but replaces collected data instead of
-        adding it.
-
-    .. method:: get_css(css, *processors, **kwargs)
-
-        Similar to :meth:`ItemLoader.get_value` but receives a CSS selector
-        instead of a value, which is used to extract a list of unicode strings
-        from the selector associated with this :class:`ItemLoader`.
-
-        :param css: the CSS selector to extract data from
-        :type css: str
-
-        :param re: a regular expression to use for extracting data from the
-            selected CSS region
-        :type re: str or compiled regex
-
-        Examples::
-
-            # HTML snippet: <p class="product-name">Color TV</p>
-            loader.get_css('p.product-name')
-            # HTML snippet: <p id="price">the price is $1200</p>
-            loader.get_css('p#price', TakeFirst(), re='the price is (.*)')
-
-    .. method:: add_css(field_name, css, *processors, **kwargs)
-
-        Similar to :meth:`ItemLoader.add_value` but receives a CSS selector
-        instead of a value, which is used to extract a list of unicode strings
-        from the selector associated with this :class:`ItemLoader`.
-
-        See :meth:`get_css` for ``kwargs``.
-
-        :param css: the CSS selector to extract data from
-        :type css: str
-
-        Examples::
-
-            # HTML snippet: <p class="product-name">Color TV</p>
-            loader.add_css('name', 'p.product-name')
-            # HTML snippet: <p id="price">the price is $1200</p>
-            loader.add_css('price', 'p#price', re='the price is (.*)')
-
-    .. method:: replace_css(field_name, css, *processors, **kwargs)
-
-        Similar to :meth:`add_css` but replaces collected data instead of
-        adding it.
-
-    .. method:: load_item()
-
-        Populate the item with the data collected so far, and return it. The
-        data collected is first passed through the :ref:`output processors
-        <topics-loaders-processors>` to get the final value to assign to each
-        item field.
-
-    .. method:: nested_xpath(xpath)
-
-        Create a nested loader with an xpath selector.
-        The supplied selector is applied relative to selector associated
-        with this :class:`ItemLoader`. The nested loader shares the :ref:`item
-        object <topics-items>` with the parent :class:`ItemLoader` so calls to
-        :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
-        behave as expected.
-
-    .. method:: nested_css(css)
-
-        Create a nested loader with a css selector.
-        The supplied selector is applied relative to selector associated
-        with this :class:`ItemLoader`. The nested loader shares the :ref:`item
-        object <topics-items>` with the parent :class:`ItemLoader` so calls to
-        :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
-        behave as expected.
-
-    .. method:: get_collected_values(field_name)
-
-        Return the collected values for the given field.
-
-    .. method:: get_output_value(field_name)
-
-        Return the collected values parsed using the output processor, for the
-        given field. This method doesn't populate or modify the item at all.
-
-    .. method:: get_input_processor(field_name)
-
-        Return the input processor for the given field.
-
-    .. method:: get_output_processor(field_name)
-
-        Return the output processor for the given field.
-
-    :class:`ItemLoader` instances have the following attributes:
-
-    .. attribute:: item
-
-        The :ref:`item object <topics-items>` being parsed by this Item Loader.
-        This is mostly used as a property so when attempting to override this
-        value, you may want to check out :attr:`default_item_class` first.
-
-    .. attribute:: context
-
-        The currently active :ref:`Context <topics-loaders-context>` of this
-        Item Loader.
-
-    .. attribute:: default_item_class
-
-        An :ref:`item object <topics-items>` class or factory, used to
-        instantiate items when not given in the ``__init__`` method.
-
-    .. attribute:: default_input_processor
-
-        The default input processor to use for those fields which don't specify
-        one.
-
-    .. attribute:: default_output_processor
-
-        The default output processor to use for those fields which don't specify
-        one.
-
-    .. attribute:: default_selector_class
-
-        The class used to construct the :attr:`selector` of this
-        :class:`ItemLoader`, if only a response is given in the ``__init__`` method.
-        If a selector is given in the ``__init__`` method this attribute is ignored.
-        This attribute is sometimes overridden in subclasses.
-
-    .. attribute:: selector
-
-        The :class:`~scrapy.selector.Selector` object to extract data from.
-        It's either the selector given in the ``__init__`` method or one created from
-        the response given in the ``__init__`` method using the
-        :attr:`default_selector_class`. This attribute is meant to be
-        read-only.
+.. autoclass:: scrapy.loader.ItemLoader
+    :members:
+    :inherited-members:
 
 .. _topics-loaders-nested:
 
@@ -609,7 +372,7 @@ those dashes in the final product names.
 Here's how you can remove those dashes by reusing and extending the default
 Product Item Loader (``ProductLoader``)::
 
-    from scrapy.loader.processors import MapCompose
+    from itemloaders.processors import MapCompose
     from myproject.ItemLoaders import ProductLoader
 
     def strip_dashes(x):
@@ -622,7 +385,7 @@ Another case where extending Item Loaders can be very helpful is when you have
 multiple source formats, for example XML and HTML. In the XML version you may
 want to remove ``CDATA`` occurrences. Here's an example of how to do it::
 
-    from scrapy.loader.processors import MapCompose
+    from itemloaders.processors import MapCompose
     from myproject.ItemLoaders import ProductLoader
     from myproject.utils.xml import remove_cdata
 
@@ -642,156 +405,5 @@ projects. Scrapy only provides the mechanism; it doesn't impose any specific
 organization of your Loaders collection - that's up to you and your project's
 needs.
 
-.. _topics-loaders-available-processors:
-
-Available built-in processors
-=============================
-
-.. module:: scrapy.loader.processors
-   :synopsis: A collection of processors to use with Item Loaders
-
-Even though you can use any callable function as input and output processors,
-Scrapy provides some commonly used processors, which are described below. Some
-of them, like the :class:`MapCompose` (which is typically used as input
-processor) compose the output of several functions executed in order, to
-produce the final parsed value.
-
-Here is a list of all built-in processors:
-
-.. class:: Identity
-
-    The simplest processor, which doesn't do anything. It returns the original
-    values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
-    accept Loader contexts.
-
-    Example:
-
-    >>> from scrapy.loader.processors import Identity
-    >>> proc = Identity()
-    >>> proc(['one', 'two', 'three'])
-    ['one', 'two', 'three']
-
-.. class:: TakeFirst
-
-    Returns the first non-null/non-empty value from the values received,
-    so it's typically used as an output processor to single-valued fields.
-    It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
-
-    Example:
-
-    >>> from scrapy.loader.processors import TakeFirst
-    >>> proc = TakeFirst()
-    >>> proc(['', 'one', 'two', 'three'])
-    'one'
-
-.. class:: Join(separator=u' ')
-
-    Returns the values joined with the separator given in the ``__init__`` method, which
-    defaults to ``u' '``. It doesn't accept Loader contexts.
-
-    When using the default separator, this processor is equivalent to the
-    function: ``u' '.join``
-
-    Examples:
-
-    >>> from scrapy.loader.processors import Join
-    >>> proc = Join()
-    >>> proc(['one', 'two', 'three'])
-    'one two three'
-    >>> proc = Join('<br>')
-    >>> proc(['one', 'two', 'three'])
-    'one<br>two<br>three'
-
-.. class:: Compose(*functions, **default_loader_context)
-
-    A processor which is constructed from the composition of the given
-    functions. This means that each input value of this processor is passed to
-    the first function, and the result of that function is passed to the second
-    function, and so on, until the last function returns the output value of
-    this processor.
-
-    By default, stop process on ``None`` value. This behaviour can be changed by
-    passing keyword argument ``stop_on_none=False``.
-
-    Example:
-
-    >>> from scrapy.loader.processors import Compose
-    >>> proc = Compose(lambda v: v[0], str.upper)
-    >>> proc(['hello', 'world'])
-    'HELLO'
-
-    Each function can optionally receive a ``loader_context`` parameter. For
-    those which do, this processor will pass the currently active :ref:`Loader
-    context <topics-loaders-context>` through that parameter.
-
-    The keyword arguments passed in the ``__init__`` method are used as the default
-    Loader context values passed to each function call. However, the final
-    Loader context values passed to functions are overridden with the currently
-    active Loader context accessible through the :meth:`ItemLoader.context`
-    attribute.
-
-.. class:: MapCompose(*functions, **default_loader_context)
-
-    A processor which is constructed from the composition of the given
-    functions, similar to the :class:`Compose` processor. The difference with
-    this processor is the way internal results are passed among functions,
-    which is as follows:
-
-    The input value of this processor is *iterated* and the first function is
-    applied to each element. The results of these function calls (one for each element)
-    are concatenated to construct a new iterable, which is then used to apply the
-    second function, and so on, until the last function is applied to each
-    value of the list of values collected so far. The output values of the last
-    function are concatenated together to produce the output of this processor.
-
-    Each particular function can return a value or a list of values, which is
-    flattened with the list of values returned by the same function applied to
-    the other input values. The functions can also return ``None`` in which
-    case the output of that function is ignored for further processing over the
-    chain.
-
-    This processor provides a convenient way to compose functions that only
-    work with single values (instead of iterables). For this reason the
-    :class:`MapCompose` processor is typically used as input processor, since
-    data is often extracted using the
-    :meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors
-    <topics-selectors>`, which returns a list of unicode strings.
-
-    The example below should clarify how it works:
-
-    >>> def filter_world(x):
-    ...     return None if x == 'world' else x
-    ...
-    >>> from scrapy.loader.processors import MapCompose
-    >>> proc = MapCompose(filter_world, str.upper)
-    >>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
-    ['HELLO, 'THIS', 'IS', 'SCRAPY']
-
-    As with the Compose processor, functions can receive Loader contexts, and
-    ``__init__`` method keyword arguments are used as default context values. See
-    :class:`Compose` processor for more info.
-
-.. class:: SelectJmes(json_path)
-
-    Queries the value using the json path provided to the ``__init__`` method and returns the output.
-    Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
-    This processor takes only one input at a time.
-
-    Example:
-
-    >>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
-    >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
-    >>> proc({'foo': 'bar'})
-    'bar'
-    >>> proc({'foo': {'bar': 'baz'}})
-    {'bar': 'baz'}
-
-    Working with Json:
-
-    >>> import json
-    >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
-    >>> proc_single_json_str('{"foo": "bar"}')
-    'bar'
-    >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
-    >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
-    ['bar']
+.. _itemloaders: https://itemloaders.readthedocs.io/en/latest/
+.. _processors: https://itemloaders.readthedocs.io/en/latest/built-in-processors.html
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 18f57945f..014951a8e 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -3,217 +3,86 @@ Item Loader
 
 See documentation in docs/topics/loaders.rst
 """
-from collections import defaultdict
-from contextlib import suppress
-
-from itemadapter import ItemAdapter
+import itemloaders
 
 from scrapy.item import Item
-from scrapy.loader.common import wrap_loader_context
-from scrapy.loader.processors import Identity
 from scrapy.selector import Selector
-from scrapy.utils.misc import arg_to_iter, extract_regex
-from scrapy.utils.python import flatten
 
 
-def unbound_method(method):
+class ItemLoader(itemloaders.ItemLoader):
     """
-    Allow to use single-argument functions as input or output processors
-    (no need to define an unused first 'self' argument)
+    A user-friendly abstraction to populate an :ref:`item <topics-items>` with data
+    by applying :ref:`field processors <topics-loaders-processors>` to scraped data.
+    When instantiated with a ``selector`` or a ``response`` it supports
+    data extraction from web pages using :ref:`selectors <topics-selectors>`.
+
+    :param item: The item instance to populate using subsequent calls to
+        :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
+        or :meth:`~ItemLoader.add_value`.
+    :type item: scrapy.item.Item
+
+    :param selector: The selector to extract data from, when using the
+        :meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or
+        :meth:`replace_css` method.
+    :type selector: :class:`~scrapy.selector.Selector` object
+
+    :param response: The response used to construct the selector using the
+        :attr:`default_selector_class`, unless the selector argument is given,
+        in which case this argument is ignored.
+    :type response: :class:`~scrapy.http.Response` object
+
+    If no item is given, one is instantiated automatically using the class in
+    :attr:`default_item_class`.
+
+    The item, selector, response and remaining keyword arguments are
+    assigned to the Loader context (accessible through the :attr:`context` attribute).
+
+    .. attribute:: item
+
+        The item object being parsed by this Item Loader.
+        This is mostly used as a property so, when attempting to override this
+        value, you may want to check out :attr:`default_item_class` first.
+
+    .. attribute:: context
+
+        The currently active :ref:`Context <loaders-context>` of this Item Loader.
+
+    .. attribute:: default_item_class
+
+        An :ref:`item <topics-items>` class (or factory), used to instantiate
+        items when not given in the ``__init__`` method.
+
+    .. attribute:: default_input_processor
+
+        The default input processor to use for those fields which don't specify
+        one.
+
+    .. attribute:: default_output_processor
+
+        The default output processor to use for those fields which don't specify
+        one.
+
+    .. attribute:: default_selector_class
+
+        The class used to construct the :attr:`selector` of this
+        :class:`ItemLoader`, if only a response is given in the ``__init__`` method.
+        If a selector is given in the ``__init__`` method this attribute is ignored.
+        This attribute is sometimes overridden in subclasses.
+
+    .. attribute:: selector
+
+        The :class:`~scrapy.selector.Selector` object to extract data from.
+        It's either the selector given in the ``__init__`` method or one created from
+        the response given in the ``__init__`` method using the
+        :attr:`default_selector_class`. This attribute is meant to be
+        read-only.
     """
-    with suppress(AttributeError):
-        if '.' not in method.__qualname__:
-            return method.__func__
-    return method
-
-
-class ItemLoader:
 
     default_item_class = Item
-    default_input_processor = Identity()
-    default_output_processor = Identity()
     default_selector_class = Selector
 
     def __init__(self, item=None, selector=None, response=None, parent=None, **context):
         if selector is None and response is not None:
             selector = self.default_selector_class(response)
-        self.selector = selector
-        context.update(selector=selector, response=response)
-        if item is None:
-            item = self.default_item_class()
-        self.context = context
-        self.parent = parent
-        self._local_item = context['item'] = item
-        self._local_values = defaultdict(list)
-        # values from initial item
-        for field_name, value in ItemAdapter(item).items():
-            self._values[field_name] += arg_to_iter(value)
-
-    @property
-    def _values(self):
-        if self.parent is not None:
-            return self.parent._values
-        else:
-            return self._local_values
-
-    @property
-    def item(self):
-        if self.parent is not None:
-            return self.parent.item
-        else:
-            return self._local_item
-
-    def nested_xpath(self, xpath, **context):
-        selector = self.selector.xpath(xpath)
-        context.update(selector=selector)
-        subloader = self.__class__(
-            item=self.item, parent=self, **context
-        )
-        return subloader
-
-    def nested_css(self, css, **context):
-        selector = self.selector.css(css)
-        context.update(selector=selector)
-        subloader = self.__class__(
-            item=self.item, parent=self, **context
-        )
-        return subloader
-
-    def add_value(self, field_name, value, *processors, **kw):
-        value = self.get_value(value, *processors, **kw)
-        if value is None:
-            return
-        if not field_name:
-            for k, v in value.items():
-                self._add_value(k, v)
-        else:
-            self._add_value(field_name, value)
-
-    def replace_value(self, field_name, value, *processors, **kw):
-        value = self.get_value(value, *processors, **kw)
-        if value is None:
-            return
-        if not field_name:
-            for k, v in value.items():
-                self._replace_value(k, v)
-        else:
-            self._replace_value(field_name, value)
-
-    def _add_value(self, field_name, value):
-        value = arg_to_iter(value)
-        processed_value = self._process_input_value(field_name, value)
-        if processed_value:
-            self._values[field_name] += arg_to_iter(processed_value)
-
-    def _replace_value(self, field_name, value):
-        self._values.pop(field_name, None)
-        self._add_value(field_name, value)
-
-    def get_value(self, value, *processors, **kw):
-        regex = kw.get('re', None)
-        if regex:
-            value = arg_to_iter(value)
-            value = flatten(extract_regex(regex, x) for x in value)
-
-        for proc in processors:
-            if value is None:
-                break
-            _proc = proc
-            proc = wrap_loader_context(proc, self.context)
-            try:
-                value = proc(value)
-            except Exception as e:
-                raise ValueError("Error with processor %s value=%r error='%s: %s'" %
-                                 (_proc.__class__.__name__, value,
-                                  type(e).__name__, str(e)))
-        return value
-
-    def load_item(self):
-        adapter = ItemAdapter(self.item)
-        for field_name in tuple(self._values):
-            value = self.get_output_value(field_name)
-            if value is not None:
-                adapter[field_name] = value
-        return adapter.item
-
-    def get_output_value(self, field_name):
-        proc = self.get_output_processor(field_name)
-        proc = wrap_loader_context(proc, self.context)
-        try:
-            return proc(self._values[field_name])
-        except Exception as e:
-            raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" %
-                             (field_name, self._values[field_name], type(e).__name__, str(e)))
-
-    def get_collected_values(self, field_name):
-        return self._values[field_name]
-
-    def get_input_processor(self, field_name):
-        proc = getattr(self, '%s_in' % field_name, None)
-        if not proc:
-            proc = self._get_item_field_attr(field_name, 'input_processor',
-                                             self.default_input_processor)
-        return unbound_method(proc)
-
-    def get_output_processor(self, field_name):
-        proc = getattr(self, '%s_out' % field_name, None)
-        if not proc:
-            proc = self._get_item_field_attr(field_name, 'output_processor',
-                                             self.default_output_processor)
-        return unbound_method(proc)
-
-    def _process_input_value(self, field_name, value):
-        proc = self.get_input_processor(field_name)
-        _proc = proc
-        proc = wrap_loader_context(proc, self.context)
-        try:
-            return proc(value)
-        except Exception as e:
-            raise ValueError(
-                "Error with input processor %s: field=%r value=%r "
-                "error='%s: %s'" % (_proc.__class__.__name__, field_name,
-                                    value, type(e).__name__, str(e)))
-
-    def _get_item_field_attr(self, field_name, key, default=None):
-        field_meta = ItemAdapter(self.item).get_field_meta(field_name)
-        return field_meta.get(key, default)
-
-    def _check_selector_method(self):
-        if self.selector is None:
-            raise RuntimeError("To use XPath or CSS selectors, "
-                               "%s must be instantiated with a selector "
-                               "or a response" % self.__class__.__name__)
-
-    def add_xpath(self, field_name, xpath, *processors, **kw):
-        values = self._get_xpathvalues(xpath, **kw)
-        self.add_value(field_name, values, *processors, **kw)
-
-    def replace_xpath(self, field_name, xpath, *processors, **kw):
-        values = self._get_xpathvalues(xpath, **kw)
-        self.replace_value(field_name, values, *processors, **kw)
-
-    def get_xpath(self, xpath, *processors, **kw):
-        values = self._get_xpathvalues(xpath, **kw)
-        return self.get_value(values, *processors, **kw)
-
-    def _get_xpathvalues(self, xpaths, **kw):
-        self._check_selector_method()
-        xpaths = arg_to_iter(xpaths)
-        return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths)
-
-    def add_css(self, field_name, css, *processors, **kw):
-        values = self._get_cssvalues(css, **kw)
-        self.add_value(field_name, values, *processors, **kw)
-
-    def replace_css(self, field_name, css, *processors, **kw):
-        values = self._get_cssvalues(css, **kw)
-        self.replace_value(field_name, values, *processors, **kw)
-
-    def get_css(self, css, *processors, **kw):
-        values = self._get_cssvalues(css, **kw)
-        return self.get_value(values, *processors, **kw)
-
-    def _get_cssvalues(self, csss, **kw):
-        self._check_selector_method()
-        csss = arg_to_iter(csss)
-        return flatten(self.selector.css(css).getall() for css in csss)
+        context.update(response=response)
+        super().__init__(item=item, selector=selector, parent=parent, **context)
diff --git a/scrapy/loader/common.py b/scrapy/loader/common.py
index 42f8de636..3b8a6ee94 100644
--- a/scrapy/loader/common.py
+++ b/scrapy/loader/common.py
@@ -1,14 +1,21 @@
 """Common functions used in Item Loaders code"""
 
-from functools import partial
-from scrapy.utils.python import get_func_args
+import warnings
+
+from itemloaders import common
+
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 
 
 def wrap_loader_context(function, context):
     """Wrap functions that receive loader_context to contain the context
     "pre-loaded" and expose a interface that receives only one argument
     """
-    if 'loader_context' in get_func_args(function):
-        return partial(function, loader_context=context)
-    else:
-        return function
+    warnings.warn(
+        "scrapy.loader.common.wrap_loader_context has moved to a new library."
+        "Please update your reference to itemloaders.common.wrap_loader_context",
+        ScrapyDeprecationWarning,
+        stacklevel=2
+    )
+
+    return common.wrap_loader_context(function, context)
diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index a7be65609..51fbd19eb 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -3,102 +3,19 @@ This module provides some commonly used processors for Item Loaders.
 
 See documentation in docs/topics/loaders.rst
 """
-from collections import ChainMap
+from itemloaders import processors
 
-from scrapy.utils.misc import arg_to_iter
-from scrapy.loader.common import wrap_loader_context
+from scrapy.utils.deprecate import create_deprecated_class
 
 
-class MapCompose:
+MapCompose = create_deprecated_class('MapCompose', processors.MapCompose)
 
-    def __init__(self, *functions, **default_loader_context):
-        self.functions = functions
-        self.default_loader_context = default_loader_context
+Compose = create_deprecated_class('Compose', processors.Compose)
 
-    def __call__(self, value, loader_context=None):
-        values = arg_to_iter(value)
-        if loader_context:
-            context = ChainMap(loader_context, self.default_loader_context)
-        else:
-            context = self.default_loader_context
-        wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
-        for func in wrapped_funcs:
-            next_values = []
-            for v in values:
-                try:
-                    next_values += arg_to_iter(func(v))
-                except Exception as e:
-                    raise ValueError("Error in MapCompose with "
-                                     "%s value=%r error='%s: %s'" %
-                                     (str(func), value, type(e).__name__,
-                                      str(e)))
-            values = next_values
-        return values
+TakeFirst = create_deprecated_class('TakeFirst', processors.TakeFirst)
 
+Identity = create_deprecated_class('Identity', processors.Identity)
 
-class Compose:
+SelectJmes = create_deprecated_class('SelectJmes', processors.SelectJmes)
 
-    def __init__(self, *functions, **default_loader_context):
-        self.functions = functions
-        self.stop_on_none = default_loader_context.get('stop_on_none', True)
-        self.default_loader_context = default_loader_context
-
-    def __call__(self, value, loader_context=None):
-        if loader_context:
-            context = ChainMap(loader_context, self.default_loader_context)
-        else:
-            context = self.default_loader_context
-        wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
-        for func in wrapped_funcs:
-            if value is None and self.stop_on_none:
-                break
-            try:
-                value = func(value)
-            except Exception as e:
-                raise ValueError("Error in Compose with "
-                                 "%s value=%r error='%s: %s'" %
-                                 (str(func), value, type(e).__name__, str(e)))
-        return value
-
-
-class TakeFirst:
-
-    def __call__(self, values):
-        for value in values:
-            if value is not None and value != '':
-                return value
-
-
-class Identity:
-
-    def __call__(self, values):
-        return values
-
-
-class SelectJmes:
-    """
-        Query the input string for the jmespath (given at instantiation),
-        and return the answer
-        Requires : jmespath(https://github.com/jmespath/jmespath)
-        Note: SelectJmes accepts only one input element at a time.
-    """
-    def __init__(self, json_path):
-        self.json_path = json_path
-        import jmespath
-        self.compiled_path = jmespath.compile(self.json_path)
-
-    def __call__(self, value):
-        """Query value for the jmespath query and return answer
-        :param value: a data structure (dict, list) to extract from
-        :return: Element extracted according to jmespath query
-        """
-        return self.compiled_path.search(value)
-
-
-class Join:
-
-    def __init__(self, separator=u' '):
-        self.separator = separator
-
-    def __call__(self, values):
-        return self.separator.join(values)
+Join = create_deprecated_class('Join', processors.Join)
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 8e5fde246..d6966be8e 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -15,6 +15,7 @@ from w3lib.html import replace_entities
 from scrapy.utils.datatypes import LocalWeakReferencedCache
 from scrapy.utils.python import flatten, to_unicode
 from scrapy.item import _BaseItem
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 
 
 _ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
@@ -86,6 +87,11 @@ def extract_regex(regex, text, encoding='utf-8'):
     * if the regex contains multiple numbered groups, all those will be returned (flattened)
     * if the regex doesn't contain any group the entire regex matching is returned
     """
+    warnings.warn(
+        "scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.",
+        ScrapyDeprecationWarning,
+        stacklevel=2
+    )
 
     if isinstance(regex, str):
         regex = re.compile(regex, re.UNICODE)
diff --git a/setup.cfg b/setup.cfg
index a9138c1c0..46a3d13fc 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -118,6 +118,9 @@ ignore_errors = True
 [mypy-tests.test_loader]
 ignore_errors = True
 
+[mypy-tests.test_loader_deprecated]
+ignore_errors = True
+
 [mypy-tests.test_pipeline_crawl]
 ignore_errors = True
 
diff --git a/setup.py b/setup.py
index 5a99fd1bf..f8d9b491b 100644
--- a/setup.py
+++ b/setup.py
@@ -71,6 +71,7 @@ setup(
         'Twisted>=17.9.0',
         'cryptography>=2.0',
         'cssselect>=0.9.1',
+        'itemloaders>=1.0.1',
         'lxml>=3.5.0',
         'parsel>=1.5.0',
         'PyDispatcher>=2.0.5',
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index dacb86e56..0551b1e95 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,7 +1,6 @@
 # Tests requirements
 attrs
 dataclasses; python_version == '3.6'
-jmespath
 mitmproxy; python_version >= '3.6'
 mitmproxy<4.0.0; python_version < '3.6'
 # https://github.com/pytest-dev/pytest-twisted/issues/93
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 8a9c6fca9..581183625 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -1,14 +1,12 @@
-from functools import partial
 import unittest
 
 import attr
 from itemadapter import ItemAdapter
+from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst
 
 from scrapy.http import HtmlResponse
 from scrapy.item import Item, Field
 from scrapy.loader import ItemLoader
-from scrapy.loader.processors import (Compose, Identity, Join,
-                                      MapCompose, SelectJmes, TakeFirst)
 from scrapy.selector import Selector
 
 
@@ -69,6 +67,10 @@ def processor_with_args(value, other=None, loader_context=None):
 
 class BasicItemLoaderTest(unittest.TestCase):
 
+    def test_add_value_on_unknown_field(self):
+        il = TestItemLoader()
+        self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo'])
+
     def test_load_item_using_default_loader(self):
         i = TestItem()
         i['summary'] = u'lala'
@@ -85,391 +87,6 @@ class BasicItemLoaderTest(unittest.TestCase):
         item = il.load_item()
         self.assertEqual(item['name'], [u'Marta'])
 
-    def test_load_item_ignore_none_field_values(self):
-        def validate_sku(value):
-            # Let's assume a SKU is only digits.
-            if value.isdigit():
-                return value
-
-        class MyLoader(ItemLoader):
-            name_out = Compose(lambda vs: vs[0])  # take first which allows empty values
-            price_out = Compose(TakeFirst(), float)
-            sku_out = Compose(TakeFirst(), validate_sku)
-
-        valid_fragment = u'SKU: 1234'
-        invalid_fragment = u'SKU: not available'
-        sku_re = 'SKU: (.+)'
-
-        il = MyLoader(item={})
-        # Should not return "sku: None".
-        il.add_value('sku', [invalid_fragment], re=sku_re)
-        # Should not ignore empty values.
-        il.add_value('name', u'')
-        il.add_value('price', [u'0'])
-        self.assertEqual(il.load_item(), {
-            'name': u'',
-            'price': 0.0,
-        })
-
-        il.replace_value('sku', [valid_fragment], re=sku_re)
-        self.assertEqual(il.load_item()['sku'], u'1234')
-
-    def test_self_referencing_loader(self):
-        class MyLoader(ItemLoader):
-            url_out = TakeFirst()
-
-            def img_url_out(self, values):
-                return (self.get_output_value('url') or '') + values[0]
-
-        il = MyLoader(item={})
-        il.add_value('url', 'http://example.com/')
-        il.add_value('img_url', '1234.png')
-        self.assertEqual(il.load_item(), {
-            'url': 'http://example.com/',
-            'img_url': 'http://example.com/1234.png',
-        })
-
-        il = MyLoader(item={})
-        il.add_value('img_url', '1234.png')
-        self.assertEqual(il.load_item(), {
-            'img_url': '1234.png',
-        })
-
-    def test_add_value(self):
-        il = TestItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-        il.add_value('name', u'pepe')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe'])
-
-        # test add object value
-        il.add_value('summary', {'key': 1})
-        self.assertEqual(il.get_collected_values('summary'), [{'key': 1}])
-
-        il.add_value(None, u'Jim', lambda x: {'name': x})
-        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim'])
-
-    def test_add_zero(self):
-        il = NameItemLoader()
-        il.add_value('name', 0)
-        self.assertEqual(il.get_collected_values('name'), [0])
-
-    def test_replace_value(self):
-        il = TestItemLoader()
-        il.replace_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-        il.replace_value('name', u'pepe')
-        self.assertEqual(il.get_collected_values('name'), [u'Pepe'])
-        self.assertEqual(il.get_output_value('name'), [u'Pepe'])
-
-        il.replace_value(None, u'Jim', lambda x: {'name': x})
-        self.assertEqual(il.get_collected_values('name'), [u'Jim'])
-
-    def test_get_value(self):
-        il = NameItemLoader()
-        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
-        self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
-        self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
-
-        il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')
-        self.assertEqual([u'foo'], il.get_collected_values('name'))
-        il.replace_value('name', u'name:bar', re=u'name:(.*)$')
-        self.assertEqual([u'bar'], il.get_collected_values('name'))
-
-    def test_iter_on_input_processor_input(self):
-        class NameFirstItemLoader(NameItemLoader):
-            name_in = TakeFirst()
-
-        il = NameFirstItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
-        il = NameFirstItemLoader()
-        il.add_value('name', [u'marta', u'jose'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
-
-        il = NameFirstItemLoader()
-        il.replace_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
-        il = NameFirstItemLoader()
-        il.replace_value('name', [u'marta', u'jose'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
-
-        il = NameFirstItemLoader()
-        il.add_value('name', u'marta')
-        il.add_value('name', [u'jose', u'pedro'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose'])
-
-    def test_map_compose_filter(self):
-        def filter_world(x):
-            return None if x == 'world' else x
-
-        proc = MapCompose(filter_world, str.upper)
-        self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
-                         ['HELLO', 'THIS', 'IS', 'SCRAPY'])
-
-    def test_map_compose_filter_multil(self):
-        class TestItemLoader(NameItemLoader):
-            name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])
-
-        il = TestItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Mart'])
-        item = il.load_item()
-        self.assertEqual(item['name'], [u'Mart'])
-
-    def test_default_input_processor(self):
-        il = DefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mart'])
-
-    def test_inherited_default_input_processor(self):
-        class InheritDefaultedItemLoader(DefaultedItemLoader):
-            pass
-
-        il = InheritDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mart'])
-
-    def test_input_processor_inheritance(self):
-        class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(lambda v: v.lower())
-
-        il = ChildItemLoader()
-        il.add_value('url', u'HTTP://scrapy.ORG')
-        self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org'])
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-
-        class ChildChildItemLoader(ChildItemLoader):
-            url_in = MapCompose(lambda v: v.upper())
-            summary_in = MapCompose(lambda v: v)
-
-        il = ChildChildItemLoader()
-        il.add_value('url', u'http://scrapy.org')
-        self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG'])
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-
-    def test_empty_map_compose(self):
-        class IdentityDefaultedItemLoader(DefaultedItemLoader):
-            name_in = MapCompose()
-
-        il = IdentityDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'marta'])
-
-    def test_identity_input_processor(self):
-        class IdentityDefaultedItemLoader(DefaultedItemLoader):
-            name_in = Identity()
-
-        il = IdentityDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'marta'])
-
-    def test_extend_custom_input_processors(self):
-        class ChildItemLoader(TestItemLoader):
-            name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
-
-        il = ChildItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mARTA'])
-
-    def test_extend_default_input_processors(self):
-        class ChildDefaultedItemLoader(DefaultedItemLoader):
-            name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
-
-        il = ChildDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'MART'])
-
-    def test_output_processor_using_function(self):
-        il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
-
-        class TakeFirstItemLoader(TestItemLoader):
-            name_out = u" ".join
-
-        il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
-
-    def test_output_processor_error(self):
-        class TestItemLoader(ItemLoader):
-            default_item_class = TestItem
-            name_out = MapCompose(float)
-
-        il = TestItemLoader()
-        il.add_value('name', [u'$10'])
-        try:
-            float(u'$10')
-        except Exception as e:
-            expected_exc_str = str(e)
-
-        exc = None
-        try:
-            il.load_item()
-        except Exception as e:
-            exc = e
-        assert isinstance(exc, ValueError)
-        s = str(exc)
-        assert 'name' in s, s
-        assert '$10' in s, s
-        assert 'ValueError' in s, s
-        assert expected_exc_str in s, s
-
-    def test_output_processor_using_classes(self):
-        il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
-
-        class TakeFirstItemLoader(TestItemLoader):
-            name_out = Join()
-
-        il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
-
-        class TakeFirstItemLoader(TestItemLoader):
-            name_out = Join("<br>")
-
-        il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta')
-
-    def test_default_output_processor(self):
-        il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
-
-        class LalaItemLoader(TestItemLoader):
-            default_output_processor = Identity()
-
-        il = LalaItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
-
-    def test_loader_context_on_declaration(self):
-        class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor_with_args, key=u'val')
-
-        il = ChildItemLoader()
-        il.add_value('url', u'text')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-
-    def test_loader_context_on_instantiation(self):
-        class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor_with_args)
-
-        il = ChildItemLoader(key=u'val')
-        il.add_value('url', u'text')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-
-    def test_loader_context_on_assign(self):
-        class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor_with_args)
-
-        il = ChildItemLoader()
-        il.context['key'] = u'val'
-        il.add_value('url', u'text')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-
-    def test_item_passed_to_input_processor_functions(self):
-        def processor(value, loader_context):
-            return loader_context['item']['name']
-
-        class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor)
-
-        it = TestItem(name='marta')
-        il = ChildItemLoader(item=it)
-        il.add_value('url', u'text')
-        self.assertEqual(il.get_output_value('url'), ['marta'])
-        il.replace_value('url', u'text2')
-        self.assertEqual(il.get_output_value('url'), ['marta'])
-
-    def test_add_value_on_unknown_field(self):
-        il = TestItemLoader()
-        self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo'])
-
-    def test_compose_processor(self):
-        class TestItemLoader(NameItemLoader):
-            name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])
-
-        il = TestItemLoader()
-        il.add_value('name', [u'marta', u'other'])
-        self.assertEqual(il.get_output_value('name'), u'Mart')
-        item = il.load_item()
-        self.assertEqual(item['name'], u'Mart')
-
-    def test_partial_processor(self):
-        def join(values, sep=None, loader_context=None, ignored=None):
-            if sep is not None:
-                return sep.join(values)
-            elif loader_context and 'sep' in loader_context:
-                return loader_context['sep'].join(values)
-            else:
-                return ''.join(values)
-
-        class TestItemLoader(NameItemLoader):
-            name_out = Compose(partial(join, sep='+'))
-            url_out = Compose(partial(join, loader_context={'sep': '.'}))
-            summary_out = Compose(partial(join, ignored='foo'))
-
-        il = TestItemLoader()
-        il.add_value('name', [u'rabbit', u'hole'])
-        il.add_value('url', [u'rabbit', u'hole'])
-        il.add_value('summary', [u'rabbit', u'hole'])
-        item = il.load_item()
-        self.assertEqual(item['name'], u'rabbit+hole')
-        self.assertEqual(item['url'], u'rabbit.hole')
-        self.assertEqual(item['summary'], u'rabbithole')
-
-    def test_error_input_processor(self):
-        class TestItem(Item):
-            name = Field()
-
-        class TestItemLoader(ItemLoader):
-            default_item_class = TestItem
-            name_in = MapCompose(float)
-
-        il = TestItemLoader()
-        self.assertRaises(ValueError, il.add_value, 'name',
-                          [u'marta', u'other'])
-
-    def test_error_output_processor(self):
-        class TestItem(Item):
-            name = Field()
-
-        class TestItemLoader(ItemLoader):
-            default_item_class = TestItem
-            name_out = Compose(Join(), float)
-
-        il = TestItemLoader()
-        il.add_value('name', u'marta')
-        with self.assertRaises(ValueError):
-            il.load_item()
-
-    def test_error_processor_as_argument(self):
-        class TestItem(Item):
-            name = Field()
-
-        class TestItemLoader(ItemLoader):
-            default_item_class = TestItem
-
-        il = TestItemLoader()
-        self.assertRaises(ValueError, il.add_value, 'name',
-                          [u'marta', u'other'], Compose(float))
-
 
 class InitializationTestMixin:
 
@@ -587,41 +204,6 @@ class BaseNoInputReprocessingLoader(ItemLoader):
     title_out = TakeFirst()
 
 
-class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader):
-    default_item_class = dict
-
-
-class NoInputReprocessingFromDictTest(unittest.TestCase):
-    """
-    Loaders initialized from loaded items must not reprocess fields (dict instances)
-    """
-    def test_avoid_reprocessing_with_initial_values_single(self):
-        il = NoInputReprocessingDictLoader(item=dict(title='foo'))
-        il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='foo'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
-
-    def test_avoid_reprocessing_with_initial_values_list(self):
-        il = NoInputReprocessingDictLoader(item=dict(title=['foo', 'bar']))
-        il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='foo'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
-
-    def test_avoid_reprocessing_without_initial_values_single(self):
-        il = NoInputReprocessingDictLoader()
-        il.add_value('title', 'foo')
-        il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='FOO'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
-
-    def test_avoid_reprocessing_without_initial_values_list(self):
-        il = NoInputReprocessingDictLoader()
-        il.add_value('title', ['foo', 'bar'])
-        il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='FOO'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
-
-
 class NoInputReprocessingItem(Item):
     title = Field()
 
@@ -661,25 +243,6 @@ class NoInputReprocessingFromItemTest(unittest.TestCase):
         self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'FOO'})
 
 
-class TestOutputProcessorDict(unittest.TestCase):
-    def test_output_processor(self):
-
-        class TempDict(dict):
-            def __init__(self, *args, **kwargs):
-                super(TempDict, self).__init__(self, *args, **kwargs)
-                self.setdefault('temp', 0.3)
-
-        class TempLoader(ItemLoader):
-            default_item_class = TempDict
-            default_input_processor = Identity()
-            default_output_processor = Compose(TakeFirst())
-
-        loader = TempLoader()
-        item = loader.load_item()
-        self.assertIsInstance(item, TempDict)
-        self.assertEqual(dict(item), {'temp': 0.3})
-
-
 class TestOutputProcessorItem(unittest.TestCase):
     def test_output_processor(self):
 
@@ -701,49 +264,6 @@ class TestOutputProcessorItem(unittest.TestCase):
         self.assertEqual(dict(item), {'temp': 0.3})
 
 
-class ProcessorsTest(unittest.TestCase):
-
-    def test_take_first(self):
-        proc = TakeFirst()
-        self.assertEqual(proc([None, '', 'hello', 'world']), 'hello')
-        self.assertEqual(proc([None, '', 0, 'hello', 'world']), 0)
-
-    def test_identity(self):
-        proc = Identity()
-        self.assertEqual(proc([None, '', 'hello', 'world']),
-                         [None, '', 'hello', 'world'])
-
-    def test_join(self):
-        proc = Join()
-        self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
-        self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
-        self.assertEqual(proc(['hello', 'world']), u'hello world')
-        self.assertIsInstance(proc(['hello', 'world']), str)
-
-    def test_compose(self):
-        proc = Compose(lambda v: v[0], str.upper)
-        self.assertEqual(proc(['hello', 'world']), 'HELLO')
-        proc = Compose(str.upper)
-        self.assertEqual(proc(None), None)
-        proc = Compose(str.upper, stop_on_none=False)
-        self.assertRaises(ValueError, proc, None)
-        proc = Compose(str.upper, lambda x: x + 1)
-        self.assertRaises(ValueError, proc, 'hello')
-
-    def test_mapcompose(self):
-        def filter_world(x):
-            return None if x == 'world' else x
-        proc = MapCompose(filter_world, str.upper)
-        self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
-                         [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
-        proc = MapCompose(filter_world, str.upper)
-        self.assertEqual(proc(None), [])
-        proc = MapCompose(filter_world, str.upper)
-        self.assertRaises(ValueError, proc, [1])
-        proc = MapCompose(filter_world, lambda x: x + 1)
-        self.assertRaises(ValueError, proc, 'hello')
-
-
 class SelectortemLoaderTest(unittest.TestCase):
     response = HtmlResponse(url="", encoding='utf-8', body=b"""
     <html>
@@ -921,6 +441,7 @@ class SubselectorLoaderTest(unittest.TestCase):
 
     def test_nested_xpath(self):
         l = NestedItemLoader(response=self.response)
+
         nl = l.nested_xpath("//header")
         nl.add_xpath('name', 'div/text()')
         nl.add_css('name_div', '#id')
@@ -998,31 +519,6 @@ class SubselectorLoaderTest(unittest.TestCase):
         self.assertEqual(item['image'], [u'/images/logo.png'])
 
 
-class SelectJmesTestCase(unittest.TestCase):
-    test_list_equals = {
-        'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
-        'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
-        'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
-        'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
-        'dict': (
-            'foo.bar[*].name',
-            {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
-            ['one', 'two']
-        ),
-        'list': ('[1]', [1, 2], 2)
-    }
-
-    def test_output(self):
-        for l in self.test_list_equals:
-            expr, test_list, expected = self.test_list_equals[l]
-            test = SelectJmes(expr)(test_list)
-            self.assertEqual(
-                test,
-                expected,
-                msg='test "{}" got {} expected {}'.format(l, test, expected)
-            )
-
-
 # Functions as processors
 
 def function_processor_strip(iterable):
@@ -1044,12 +540,6 @@ class FunctionProcessorItemLoader(ItemLoader):
     default_item_class = FunctionProcessorItem
 
 
-class FunctionProcessorDictLoader(ItemLoader):
-    default_item_class = dict
-    foo_in = function_processor_strip
-    foo_out = function_processor_upper
-
-
 class FunctionProcessorTestCase(unittest.TestCase):
 
     def test_processor_defined_in_item(self):
@@ -1061,15 +551,6 @@ class FunctionProcessorTestCase(unittest.TestCase):
             {'foo': ['BAR', 'ASDF', 'QWERTY']}
         )
 
-    def test_processor_defined_in_item_loader(self):
-        lo = FunctionProcessorDictLoader()
-        lo.add_value('foo', '  bar  ')
-        lo.add_value('foo', ['  asdf  ', '  qwerty  '])
-        self.assertEqual(
-            dict(lo.load_item()),
-            {'foo': ['BAR', 'ASDF', 'QWERTY']}
-        )
-
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
new file mode 100644
index 000000000..d0a59e8cd
--- /dev/null
+++ b/tests/test_loader_deprecated.py
@@ -0,0 +1,720 @@
+"""
+These tests are kept as references from the ones that were ported to a itemloaders library.
+Once we remove the references from scrapy, we can remove these tests.
+"""
+
+import unittest
+import warnings
+from functools import partial
+
+from itemloaders.processors import (Compose, Identity, Join,
+                                    MapCompose, SelectJmes, TakeFirst)
+
+from scrapy.item import Item, Field
+from scrapy.loader import ItemLoader
+from scrapy.loader.common import wrap_loader_context
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+from scrapy.utils.misc import extract_regex
+
+
+# test items
+class NameItem(Item):
+    name = Field()
+
+
+class TestItem(NameItem):
+    url = Field()
+    summary = Field()
+
+
+# test item loaders
+class NameItemLoader(ItemLoader):
+    default_item_class = TestItem
+
+
+class TestItemLoader(NameItemLoader):
+    name_in = MapCompose(lambda v: v.title())
+
+
+class DefaultedItemLoader(NameItemLoader):
+    default_input_processor = MapCompose(lambda v: v[:-1])
+
+
+# test processors
+def processor_with_args(value, other=None, loader_context=None):
+    if 'key' in loader_context:
+        return loader_context['key']
+    return value
+
+
+class BasicItemLoaderTest(unittest.TestCase):
+
+    def test_load_item_using_default_loader(self):
+        i = TestItem()
+        i['summary'] = u'lala'
+        il = ItemLoader(item=i)
+        il.add_value('name', u'marta')
+        item = il.load_item()
+        assert item is i
+        self.assertEqual(item['summary'], [u'lala'])
+        self.assertEqual(item['name'], [u'marta'])
+
+    def test_load_item_using_custom_loader(self):
+        il = TestItemLoader()
+        il.add_value('name', u'marta')
+        item = il.load_item()
+        self.assertEqual(item['name'], [u'Marta'])
+
+    def test_load_item_ignore_none_field_values(self):
+        def validate_sku(value):
+            # Let's assume a SKU is only digits.
+            if value.isdigit():
+                return value
+
+        class MyLoader(ItemLoader):
+            name_out = Compose(lambda vs: vs[0])  # take first which allows empty values
+            price_out = Compose(TakeFirst(), float)
+            sku_out = Compose(TakeFirst(), validate_sku)
+
+        valid_fragment = u'SKU: 1234'
+        invalid_fragment = u'SKU: not available'
+        sku_re = 'SKU: (.+)'
+
+        il = MyLoader(item={})
+        # Should not return "sku: None".
+        il.add_value('sku', [invalid_fragment], re=sku_re)
+        # Should not ignore empty values.
+        il.add_value('name', u'')
+        il.add_value('price', [u'0'])
+        self.assertEqual(il.load_item(), {
+            'name': u'',
+            'price': 0.0,
+        })
+
+        il.replace_value('sku', [valid_fragment], re=sku_re)
+        self.assertEqual(il.load_item()['sku'], u'1234')
+
+    def test_self_referencing_loader(self):
+        class MyLoader(ItemLoader):
+            url_out = TakeFirst()
+
+            def img_url_out(self, values):
+                return (self.get_output_value('url') or '') + values[0]
+
+        il = MyLoader(item={})
+        il.add_value('url', 'http://example.com/')
+        il.add_value('img_url', '1234.png')
+        self.assertEqual(il.load_item(), {
+            'url': 'http://example.com/',
+            'img_url': 'http://example.com/1234.png',
+        })
+
+        il = MyLoader(item={})
+        il.add_value('img_url', '1234.png')
+        self.assertEqual(il.load_item(), {
+            'img_url': '1234.png',
+        })
+
+    def test_add_value(self):
+        il = TestItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
+        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+        il.add_value('name', u'pepe')
+        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe'])
+        self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe'])
+
+        # test add object value
+        il.add_value('summary', {'key': 1})
+        self.assertEqual(il.get_collected_values('summary'), [{'key': 1}])
+
+        il.add_value(None, u'Jim', lambda x: {'name': x})
+        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim'])
+
+    def test_add_zero(self):
+        il = NameItemLoader()
+        il.add_value('name', 0)
+        self.assertEqual(il.get_collected_values('name'), [0])
+
+    def test_replace_value(self):
+        il = TestItemLoader()
+        il.replace_value('name', u'marta')
+        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
+        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+        il.replace_value('name', u'pepe')
+        self.assertEqual(il.get_collected_values('name'), [u'Pepe'])
+        self.assertEqual(il.get_output_value('name'), [u'Pepe'])
+
+        il.replace_value(None, u'Jim', lambda x: {'name': x})
+        self.assertEqual(il.get_collected_values('name'), [u'Jim'])
+
+    def test_get_value(self):
+        il = NameItemLoader()
+        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
+        self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
+        self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
+
+        il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')
+        self.assertEqual([u'foo'], il.get_collected_values('name'))
+        il.replace_value('name', u'name:bar', re=u'name:(.*)$')
+        self.assertEqual([u'bar'], il.get_collected_values('name'))
+
+    def test_iter_on_input_processor_input(self):
+        class NameFirstItemLoader(NameItemLoader):
+            name_in = TakeFirst()
+
+        il = NameFirstItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il = NameFirstItemLoader()
+        il.add_value('name', [u'marta', u'jose'])
+        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+
+        il = NameFirstItemLoader()
+        il.replace_value('name', u'marta')
+        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il = NameFirstItemLoader()
+        il.replace_value('name', [u'marta', u'jose'])
+        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+
+        il = NameFirstItemLoader()
+        il.add_value('name', u'marta')
+        il.add_value('name', [u'jose', u'pedro'])
+        self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose'])
+
+    def test_map_compose_filter(self):
+        def filter_world(x):
+            return None if x == 'world' else x
+
+        proc = MapCompose(filter_world, str.upper)
+        self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
+                         ['HELLO', 'THIS', 'IS', 'SCRAPY'])
+
+    def test_map_compose_filter_multil(self):
+        class TestItemLoader(NameItemLoader):
+            name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])
+
+        il = TestItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'Mart'])
+        item = il.load_item()
+        self.assertEqual(item['name'], [u'Mart'])
+
+    def test_default_input_processor(self):
+        il = DefaultedItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'mart'])
+
+    def test_inherited_default_input_processor(self):
+        class InheritDefaultedItemLoader(DefaultedItemLoader):
+            pass
+
+        il = InheritDefaultedItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'mart'])
+
+    def test_input_processor_inheritance(self):
+        class ChildItemLoader(TestItemLoader):
+            url_in = MapCompose(lambda v: v.lower())
+
+        il = ChildItemLoader()
+        il.add_value('url', u'HTTP://scrapy.ORG')
+        self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org'])
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+
+        class ChildChildItemLoader(ChildItemLoader):
+            url_in = MapCompose(lambda v: v.upper())
+            summary_in = MapCompose(lambda v: v)
+
+        il = ChildChildItemLoader()
+        il.add_value('url', u'http://scrapy.org')
+        self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG'])
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+
+    def test_empty_map_compose(self):
+        class IdentityDefaultedItemLoader(DefaultedItemLoader):
+            name_in = MapCompose()
+
+        il = IdentityDefaultedItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'marta'])
+
+    def test_identity_input_processor(self):
+        class IdentityDefaultedItemLoader(DefaultedItemLoader):
+            name_in = Identity()
+
+        il = IdentityDefaultedItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'marta'])
+
+    def test_extend_custom_input_processors(self):
+        class ChildItemLoader(TestItemLoader):
+            name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
+
+        il = ChildItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'mARTA'])
+
+    def test_extend_default_input_processors(self):
+        class ChildDefaultedItemLoader(DefaultedItemLoader):
+            name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
+
+        il = ChildDefaultedItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'MART'])
+
+    def test_output_processor_using_function(self):
+        il = TestItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+
+        class TakeFirstItemLoader(TestItemLoader):
+            name_out = u" ".join
+
+        il = TakeFirstItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
+
+    def test_output_processor_error(self):
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+            name_out = MapCompose(float)
+
+        il = TestItemLoader()
+        il.add_value('name', [u'$10'])
+        try:
+            float(u'$10')
+        except Exception as e:
+            expected_exc_str = str(e)
+
+        exc = None
+        try:
+            il.load_item()
+        except Exception as e:
+            exc = e
+        assert isinstance(exc, ValueError)
+        s = str(exc)
+        assert 'name' in s, s
+        assert '$10' in s, s
+        assert 'ValueError' in s, s
+        assert expected_exc_str in s, s
+
+    def test_output_processor_using_classes(self):
+        il = TestItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+
+        class TakeFirstItemLoader(TestItemLoader):
+            name_out = Join()
+
+        il = TakeFirstItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
+
+        class TakeFirstItemLoader(TestItemLoader):
+            name_out = Join("<br>")
+
+        il = TakeFirstItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta')
+
+    def test_default_output_processor(self):
+        il = TestItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+
+        class LalaItemLoader(TestItemLoader):
+            default_output_processor = Identity()
+
+        il = LalaItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+
+    def test_loader_context_on_declaration(self):
+        class ChildItemLoader(TestItemLoader):
+            url_in = MapCompose(processor_with_args, key=u'val')
+
+        il = ChildItemLoader()
+        il.add_value('url', u'text')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+        il.replace_value('url', u'text2')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+
+    def test_loader_context_on_instantiation(self):
+        class ChildItemLoader(TestItemLoader):
+            url_in = MapCompose(processor_with_args)
+
+        il = ChildItemLoader(key=u'val')
+        il.add_value('url', u'text')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+        il.replace_value('url', u'text2')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+
+    def test_loader_context_on_assign(self):
+        class ChildItemLoader(TestItemLoader):
+            url_in = MapCompose(processor_with_args)
+
+        il = ChildItemLoader()
+        il.context['key'] = u'val'
+        il.add_value('url', u'text')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+        il.replace_value('url', u'text2')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+
+    def test_item_passed_to_input_processor_functions(self):
+        def processor(value, loader_context):
+            return loader_context['item']['name']
+
+        class ChildItemLoader(TestItemLoader):
+            url_in = MapCompose(processor)
+
+        it = TestItem(name='marta')
+        il = ChildItemLoader(item=it)
+        il.add_value('url', u'text')
+        self.assertEqual(il.get_output_value('url'), ['marta'])
+        il.replace_value('url', u'text2')
+        self.assertEqual(il.get_output_value('url'), ['marta'])
+
+    def test_compose_processor(self):
+        class TestItemLoader(NameItemLoader):
+            name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])
+
+        il = TestItemLoader()
+        il.add_value('name', [u'marta', u'other'])
+        self.assertEqual(il.get_output_value('name'), u'Mart')
+        item = il.load_item()
+        self.assertEqual(item['name'], u'Mart')
+
+    def test_partial_processor(self):
+        def join(values, sep=None, loader_context=None, ignored=None):
+            if sep is not None:
+                return sep.join(values)
+            elif loader_context and 'sep' in loader_context:
+                return loader_context['sep'].join(values)
+            else:
+                return ''.join(values)
+
+        class TestItemLoader(NameItemLoader):
+            name_out = Compose(partial(join, sep='+'))
+            url_out = Compose(partial(join, loader_context={'sep': '.'}))
+            summary_out = Compose(partial(join, ignored='foo'))
+
+        il = TestItemLoader()
+        il.add_value('name', [u'rabbit', u'hole'])
+        il.add_value('url', [u'rabbit', u'hole'])
+        il.add_value('summary', [u'rabbit', u'hole'])
+        item = il.load_item()
+        self.assertEqual(item['name'], u'rabbit+hole')
+        self.assertEqual(item['url'], u'rabbit.hole')
+        self.assertEqual(item['summary'], u'rabbithole')
+
+    def test_error_input_processor(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+            name_in = MapCompose(float)
+
+        il = TestItemLoader()
+        self.assertRaises(ValueError, il.add_value, 'name',
+                          [u'marta', u'other'])
+
+    def test_error_output_processor(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+            name_out = Compose(Join(), float)
+
+        il = TestItemLoader()
+        il.add_value('name', u'marta')
+        with self.assertRaises(ValueError):
+            il.load_item()
+
+    def test_error_processor_as_argument(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+
+        il = TestItemLoader()
+        self.assertRaises(ValueError, il.add_value, 'name',
+                          [u'marta', u'other'], Compose(float))
+
+
+class InitializationFromDictTest(unittest.TestCase):
+
+    item_class = dict
+
+    def test_keep_single_value(self):
+        """Loaded item should contain values from the initial item"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo']})
+
+    def test_keep_list(self):
+        """Loaded item should contain values from the initial item"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+
+    def test_add_value_singlevalue_singlevalue(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        il.add_value('name', 'bar')
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+
+    def test_add_value_singlevalue_list(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        il.add_value('name', ['item', 'loader'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'item', 'loader']})
+
+    def test_add_value_list_singlevalue(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        il.add_value('name', 'qwerty')
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'qwerty']})
+
+    def test_add_value_list_list(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        il.add_value('name', ['item', 'loader'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'item', 'loader']})
+
+    def test_get_output_value_singlevalue(self):
+        """Getting output value must not remove value from item"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il.get_output_value('name'), ['foo'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(loaded_item, dict({'name': ['foo']}))
+
+    def test_get_output_value_list(self):
+        """Getting output value must not remove value from item"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il.get_output_value('name'), ['foo', 'bar'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(loaded_item, dict({'name': ['foo', 'bar']}))
+
+    def test_values_single(self):
+        """Values from initial item must be added to loader._values"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il._values.get('name'), ['foo'])
+
+    def test_values_list(self):
+        """Values from initial item must be added to loader._values"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il._values.get('name'), ['foo', 'bar'])
+
+
+class BaseNoInputReprocessingLoader(ItemLoader):
+    title_in = MapCompose(str.upper)
+    title_out = TakeFirst()
+
+
+class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader):
+    default_item_class = dict
+
+
+class NoInputReprocessingFromDictTest(unittest.TestCase):
+    """
+    Loaders initialized from loaded items must not reprocess fields (dict instances)
+    """
+    def test_avoid_reprocessing_with_initial_values_single(self):
+        il = NoInputReprocessingDictLoader(item=dict(title='foo'))
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='foo'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
+
+    def test_avoid_reprocessing_with_initial_values_list(self):
+        il = NoInputReprocessingDictLoader(item=dict(title=['foo', 'bar']))
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='foo'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
+
+    def test_avoid_reprocessing_without_initial_values_single(self):
+        il = NoInputReprocessingDictLoader()
+        il.add_value('title', 'foo')
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='FOO'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
+
+    def test_avoid_reprocessing_without_initial_values_list(self):
+        il = NoInputReprocessingDictLoader()
+        il.add_value('title', ['foo', 'bar'])
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='FOO'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
+
+
+class TestOutputProcessorDict(unittest.TestCase):
+    def test_output_processor(self):
+
+        class TempDict(dict):
+            def __init__(self, *args, **kwargs):
+                super(TempDict, self).__init__(self, *args, **kwargs)
+                self.setdefault('temp', 0.3)
+
+        class TempLoader(ItemLoader):
+            default_item_class = TempDict
+            default_input_processor = Identity()
+            default_output_processor = Compose(TakeFirst())
+
+        loader = TempLoader()
+        item = loader.load_item()
+        self.assertIsInstance(item, TempDict)
+        self.assertEqual(dict(item), {'temp': 0.3})
+
+
+class ProcessorsTest(unittest.TestCase):
+
+    def test_take_first(self):
+        proc = TakeFirst()
+        self.assertEqual(proc([None, '', 'hello', 'world']), 'hello')
+        self.assertEqual(proc([None, '', 0, 'hello', 'world']), 0)
+
+    def test_identity(self):
+        proc = Identity()
+        self.assertEqual(proc([None, '', 'hello', 'world']),
+                         [None, '', 'hello', 'world'])
+
+    def test_join(self):
+        proc = Join()
+        self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
+        self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
+        self.assertEqual(proc(['hello', 'world']), u'hello world')
+        self.assertIsInstance(proc(['hello', 'world']), str)
+
+    def test_compose(self):
+        proc = Compose(lambda v: v[0], str.upper)
+        self.assertEqual(proc(['hello', 'world']), 'HELLO')
+        proc = Compose(str.upper)
+        self.assertEqual(proc(None), None)
+        proc = Compose(str.upper, stop_on_none=False)
+        self.assertRaises(ValueError, proc, None)
+        proc = Compose(str.upper, lambda x: x + 1)
+        self.assertRaises(ValueError, proc, 'hello')
+
+    def test_mapcompose(self):
+        def filter_world(x):
+            return None if x == 'world' else x
+        proc = MapCompose(filter_world, str.upper)
+        self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
+                         [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
+        proc = MapCompose(filter_world, str.upper)
+        self.assertEqual(proc(None), [])
+        proc = MapCompose(filter_world, str.upper)
+        self.assertRaises(ValueError, proc, [1])
+        proc = MapCompose(filter_world, lambda x: x + 1)
+        self.assertRaises(ValueError, proc, 'hello')
+
+
+class SelectJmesTestCase(unittest.TestCase):
+    test_list_equals = {
+        'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+        'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
+        'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
+        'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+        'dict': (
+            'foo.bar[*].name',
+            {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
+            ['one', 'two']
+        ),
+        'list': ('[1]', [1, 2], 2)
+    }
+
+    def test_output(self):
+        for tl in self.test_list_equals:
+            expr, test_list, expected = self.test_list_equals[tl]
+            test = SelectJmes(expr)(test_list)
+            self.assertEqual(
+                test,
+                expected,
+                msg='test "{}" got {} expected {}'.format(tl, test, expected)
+            )
+
+
+# Functions as processors
+
+def function_processor_strip(iterable):
+    return [x.strip() for x in iterable]
+
+
+def function_processor_upper(iterable):
+    return [x.upper() for x in iterable]
+
+
+class FunctionProcessorItem(Item):
+    foo = Field(
+        input_processor=function_processor_strip,
+        output_processor=function_processor_upper,
+    )
+
+
+class FunctionProcessorDictLoader(ItemLoader):
+    default_item_class = dict
+    foo_in = function_processor_strip
+    foo_out = function_processor_upper
+
+
+class FunctionProcessorTestCase(unittest.TestCase):
+
+    def test_processor_defined_in_item_loader(self):
+        lo = FunctionProcessorDictLoader()
+        lo.add_value('foo', '  bar  ')
+        lo.add_value('foo', ['  asdf  ', '  qwerty  '])
+        self.assertEqual(
+            dict(lo.load_item()),
+            {'foo': ['BAR', 'ASDF', 'QWERTY']}
+        )
+
+
+class DeprecatedUtilityFunctionsTestCase(unittest.TestCase):
+
+    def test_deprecated_wrap_loader_context(self):
+        def function(*args):
+            return None
+
+        with warnings.catch_warnings(record=True) as w:
+            wrap_loader_context(function, context=dict())
+
+            assert len(w) == 1
+            assert issubclass(w[0].category, ScrapyDeprecationWarning)
+
+    def test_deprecated_extract_regex(self):
+        with warnings.catch_warnings(record=True) as w:
+            extract_regex(r'\w+', 'this is a test')
+
+            assert len(w) == 1
+            assert issubclass(w[0].category, ScrapyDeprecationWarning)
+
+
+if __name__ == "__main__":
+    unittest.main()

From 8bdcdb0a76e3780681dcfbbd4a0eee62e2bb05b1 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 16 Jul 2020 09:13:54 +0300
Subject: [PATCH 2927/2974] Add quotes to example in docs

---
 docs/topics/feed-exports.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 0bb5f1733..7e91b365d 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -461,7 +461,7 @@ For instance, if your settings include::
 
 And your :command:`crawl` command line is::
 
-    scrapy crawl spidername -o dirname/%(batch_id)d-filename%(batch_time)s.json
+    scrapy crawl spidername -o 'dirname/%(batch_id)d-filename%(batch_time)s.json'
 
 The command line above can generate a directory tree like::
 

From 0e0d1ad64323033017c4a01893b5337a980cb5ec Mon Sep 17 00:00:00 2001
From: Marc <noviluni@gmail.com>
Date: Thu, 16 Jul 2020 14:19:46 +0200
Subject: [PATCH 2928/2974] remove python 2 reminiscence in cookies

---
 scrapy/http/cookies.py     | 3 ---
 tests/test_http_cookies.py | 3 ---
 2 files changed, 6 deletions(-)

diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 3e810992c..0c97e6999 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -186,9 +186,6 @@ class WrappedResponse:
     def info(self):
         return self
 
-    # python3 cookiejars calls get_all
     def get_all(self, name, default=None):
         return [to_unicode(v, errors='replace')
                 for v in self.response.headers.getlist(name)]
-    # python2 cookiejars calls getheaders
-    getheaders = get_all
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index 45ddb42ba..540e27907 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -64,9 +64,6 @@ class WrappedResponseTest(TestCase):
     def test_info(self):
         self.assertIs(self.wrapped.info(), self.wrapped)
 
-    def test_getheaders(self):
-        self.assertEqual(self.wrapped.getheaders('content-type'), ['text/html'])
-
     def test_get_all(self):
         # get_all result must be native string
         self.assertEqual(self.wrapped.get_all('content-type'), ['text/html'])

From b97a39fda0b72d8a4dcd631599e5d9bf530a5bee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Thu, 16 Jul 2020 17:38:22 +0200
Subject: [PATCH 2929/2974] deprecate retry_on_eintr (#4683)

---
 scrapy/utils/python.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index afa8a8135..9204977cf 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -285,6 +285,7 @@ class WeakKeyCache:
         return self._weakdict[key]
 
 
+@deprecated
 def retry_on_eintr(function, *args, **kw):
     """Run a function and retry it while getting EINTR errors"""
     while True:

From 41263f61c6de8048023ba4c80e062f56b21e5a19 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Thu, 16 Jul 2020 18:41:45 +0300
Subject: [PATCH 2930/2974] Change single quotes to double in example in docs

---
 docs/topics/feed-exports.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 7e91b365d..fdc6e7cba 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -461,7 +461,7 @@ For instance, if your settings include::
 
 And your :command:`crawl` command line is::
 
-    scrapy crawl spidername -o 'dirname/%(batch_id)d-filename%(batch_time)s.json'
+    scrapy crawl spidername -o "dirname/%(batch_id)d-filename%(batch_time)s.json"
 
 The command line above can generate a directory tree like::
 

From d29bec60d795b13ecb6e5978cb9e4d8fbd298b08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 16 Jul 2020 23:19:24 +0200
Subject: [PATCH 2931/2974] Upgrade PyPy for CI, and test both 3.5 (oldest) and
 3.6 (newest) (#4504)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Upgrade PyPy for CI, and test both 3.5 (oldest) and 3.6 (newest)

* Log a detailed error message to discover why MockServer is not working

* Go for all lines!

* Disable tests based on mitmproxy while running on PyPy

* Fix test_get_func_args for PyPy 3.6+

* Make testPayloadDefaultCiphers work regardless of OpenSSL default ciphers

* Crossing fingers…

* Rename: testPayloadDefaultCiphers → testPayloadDisabledCipher

* Test the PyPy version currently documented as the minimum required version

* Fix the PYPY_VERSION tag

* Update the documentation about supported PyPy versions

* Also test the latest 3.5 Python version with PyPy

* Fix the PYPY_VERSION value for the latest 3.5 version

* Use pinned dependencies for asyncio and PyPy tests against oldest supported Python versions

* Fix PyPy installation for the pypy3-pinned Tox environment

* Try installing Cython

* Maybe PyPy requires lxml 3.6.0?

* install.rst: minor clarification

* lxml 4.0.0 is required on PyPy

* Require setuptools 18.5+

* Revert "Require setuptools 18.5+"

This reverts commit 017ec33ac2d237523cdd53be9be8169dd540759e.

* Maintain lxml as a dependency if setuptools < 18.5 is used
---
 .travis.yml                 | 16 +++++++++-----
 docs/faq.rst                | 14 ------------
 docs/intro/install.rst      | 12 +++++++---
 setup.py                    | 44 +++++++++++++++++++++++--------------
 tests/test_proxy_connect.py |  2 ++
 tests/test_utils_python.py  |  6 ++++-
 tests/test_webclient.py     |  6 +++--
 tox.ini                     | 31 +++++++++++++++++---------
 8 files changed, 80 insertions(+), 51 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b403ac54c..db720b918 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,19 +18,25 @@ matrix:
     - env: TOXENV=typing
       python: 3.8
 
-    - env: TOXENV=pypy3
     - env: TOXENV=pinned
       python: 3.5.2
-    - env: TOXENV=asyncio
+    - env: TOXENV=asyncio-pinned
       python: 3.5.2  # We use additional code to support 3.5.3 and earlier
+    - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0
+
     - env: TOXENV=py
       python: 3.5
     - env: TOXENV=asyncio
       python: 3.5  # We use specific code to support >= 3.5.4, < 3.6
+    - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0
+
     - env: TOXENV=py
       python: 3.6
+    - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1
+
     - env: TOXENV=py
       python: 3.7
+
     - env: TOXENV=py PYPI_RELEASE_JOB=true
       python: 3.8
       dist: bionic
@@ -42,9 +48,9 @@ matrix:
       dist: bionic
 install:
   - |
-      if [ "$TOXENV" = "pypy3" ]; then
-        export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
-        wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
+      if [[ ! -z "$PYPY_VERSION" ]]; then
+        export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
+        wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2"
         tar -jxf ${PYPY_VERSION}.tar.bz2
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
diff --git a/docs/faq.rst b/docs/faq.rst
index d5ea3cb87..ea2c8216f 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -64,20 +64,6 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
 
 .. _BeautifulSoup's official documentation: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#specifying-the-parser-to-use
 
-.. _faq-python-versions:
-
-What Python versions does Scrapy support?
------------------------------------------
-
-Scrapy is supported under Python 3.5.2+
-under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
-Python 3 support was added in Scrapy 1.1.
-PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
-Python 2 support was dropped in Scrapy 2.0.
-
-.. note::
-    For Python 3 support on Windows, it is recommended to use
-    Anaconda/Miniconda as :ref:`outlined in the installation guide <intro-install-windows>`.
 
 Did Scrapy "steal" X from Django?
 ---------------------------------
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index fb64d443c..6d65ae2ee 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -4,12 +4,18 @@
 Installation guide
 ==================
 
+.. _faq-python-versions:
+
+Supported Python versions
+=========================
+
+Scrapy requires Python 3.5.2+, either the CPython implementation (default) or
+the PyPy 5.9+ implementation (see :ref:`python:implementations`).
+
+
 Installing Scrapy
 =================
 
-Scrapy runs on Python 3.5.2 or above under CPython (default Python
-implementation) and PyPy (starting with PyPy 5.9).
-
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
 the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
 and macOS.
diff --git a/setup.py b/setup.py
index f8d9b491b..58090f7a2 100644
--- a/setup.py
+++ b/setup.py
@@ -18,12 +18,39 @@ def has_environment_marker_platform_impl_support():
     return parse_version(setuptools_version) >= parse_version('18.5')
 
 
+install_requires = [
+    'Twisted>=17.9.0',
+    'cryptography>=2.0',
+    'cssselect>=0.9.1',
+    'itemloaders>=1.0.1',
+    'lxml>=3.5.0',
+    'parsel>=1.5.0',
+    'PyDispatcher>=2.0.5',
+    'pyOpenSSL>=16.2.0',
+    'queuelib>=1.4.2',
+    'service_identity>=16.0.0',
+    'w3lib>=1.17.0',
+    'zope.interface>=4.1.3',
+    'protego>=0.1.15',
+    'itemadapter>=0.1.0',
+]
 extras_require = {}
 
 if has_environment_marker_platform_impl_support():
+    extras_require[':platform_python_implementation == "CPython"'] = [
+        'lxml>=3.5.0',
+    ]
     extras_require[':platform_python_implementation == "PyPy"'] = [
+        # Earlier lxml versions are affected by
+        # https://bitbucket.org/pypy/pypy/issues/2498/cython-on-pypy-3-dict-object-has-no,
+        # which was fixed in Cython 0.26, released on 2017-06-19, and used to
+        # generate the C headers of lxml release tarballs published since then, the
+        # first of which was:
+        'lxml>=4.0.0',
         'PyPyDispatcher>=2.1.0',
     ]
+else:
+    install_requires.append('lxml>=3.5.0')
 
 
 setup(
@@ -67,21 +94,6 @@ setup(
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
     python_requires='>=3.5.2',
-    install_requires=[
-        'Twisted>=17.9.0',
-        'cryptography>=2.0',
-        'cssselect>=0.9.1',
-        'itemloaders>=1.0.1',
-        'lxml>=3.5.0',
-        'parsel>=1.5.0',
-        'PyDispatcher>=2.0.5',
-        'pyOpenSSL>=16.2.0',
-        'queuelib>=1.4.2',
-        'service_identity>=16.0.0',
-        'w3lib>=1.17.0',
-        'zope.interface>=4.1.3',
-        'protego>=0.1.15',
-        'itemadapter>=0.1.0',
-    ],
+    install_requires=install_requires,
     extras_require=extras_require,
 )
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index fc5658ae7..a56e3c39a 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -60,6 +60,8 @@ def _wrong_credentials(proxy_url):
 
 @skipIf(sys.version_info < (3, 5, 4),
         "requires mitmproxy < 3.0.0, which these tests do not support")
+@skipIf("pypy" in sys.executable,
+        "mitmproxy does not support PyPy")
 @skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7),
         "mitmproxy does not support Windows when running Python < 3.7")
 class ProxyConnectTestCase(TestCase):
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 65e6ba876..ebce3c079 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -4,6 +4,7 @@ import operator
 import platform
 import unittest
 from itertools import count
+from sys import version_info
 from warnings import catch_warnings
 
 from scrapy.utils.python import (
@@ -214,9 +215,12 @@ class UtilsPythonTestCase(unittest.TestCase):
         else:
             self.assertEqual(
                 get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
-            self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
             self.assertEqual(
                 get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
+            if version_info < (3, 6):
+                self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
+            else:
+                self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
 
     def test_without_none_values(self):
         self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index c1c5945c2..ee64d455c 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -413,7 +413,9 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
             self.getURL("payload"), body=s, contextFactory=client_context_factory
         ).addCallback(self.assertEqual, to_bytes(s))
 
-    def testPayloadDefaultCiphers(self):
+    def testPayloadDisabledCipher(self):
         s = "0123456789" * 10
-        d = getPage(self.getURL("payload"), body=s, contextFactory=ScrapyClientContextFactory())
+        settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'})
+        client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)
+        d = getPage(self.getURL("payload"), body=s, contextFactory=client_context_factory)
         return self.assertFailure(d, OpenSSL.SSL.Error)
diff --git a/tox.ini b/tox.ini
index 5d79739bb..4557c63e3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -58,11 +58,6 @@ deps =
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
 
-[testenv:pypy3]
-basepython = pypy3
-commands =
-    py.test {posargs:--durations=10 docs scrapy tests}
-
 [pinned]
 deps =
     -ctests/constraints.txt
@@ -85,7 +80,6 @@ deps =
     Pillow==3.4.2
 
 [testenv:pinned]
-basepython = python3
 deps =
     {[pinned]deps}
     lxml==3.5.0
@@ -104,6 +98,27 @@ deps =
     reppy
     robotexclusionrulesparser
 
+[testenv:asyncio]
+commands =
+    {[testenv]commands} --reactor=asyncio
+
+[testenv:asyncio-pinned]
+commands = {[testenv:asyncio]commands}
+deps = {[testenv:pinned]deps}
+
+[testenv:pypy3]
+basepython = pypy3
+commands =
+    py.test {posargs:--durations=10 docs scrapy tests}
+
+[testenv:pypy3-pinned]
+basepython = {[testenv:pypy3]basepython}
+commands = {[testenv:pypy3]commands}
+deps =
+    {[pinned]deps}
+    lxml==4.0.0
+    PyPyDispatcher==2.1.0
+
 [docs]
 changedir = docs
 deps =
@@ -135,7 +150,3 @@ deps = {[docs]deps}
 setenv = {[docs]setenv}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
-
-[testenv:asyncio]
-commands =
-    {[testenv]commands} --reactor=asyncio

From 62a4ede5e995f83abd5a90f7dd6ac242f2f3870d Mon Sep 17 00:00:00 2001
From: Artur Shellunts <shellunts.artur@gmail.com>
Date: Fri, 17 Jul 2020 12:40:04 +0200
Subject: [PATCH 2932/2974] Remove deprecated classes BaseSgmlLinkExtractor,
 RegexLinkExtractor and SgmlLinkExtractor (#4356)

---
 scrapy/linkextractors/regex.py                |  41 -----
 scrapy/linkextractors/sgml.py                 | 151 ------------------
 tests/ignores.txt                             |   2 -
 .../link_extractor/linkextractor.html         |   2 +-
 .../link_extractor/linkextractor_latin1.html  |   2 +-
 5 files changed, 2 insertions(+), 196 deletions(-)
 delete mode 100644 scrapy/linkextractors/regex.py
 delete mode 100644 scrapy/linkextractors/sgml.py

diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py
deleted file mode 100644
index 3f2557248..000000000
--- a/scrapy/linkextractors/regex.py
+++ /dev/null
@@ -1,41 +0,0 @@
-import re
-from urllib.parse import urljoin
-
-from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
-
-from scrapy.link import Link
-from scrapy.linkextractors.sgml import SgmlLinkExtractor
-
-
-linkre = re.compile(
-        "<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
-        re.DOTALL | re.IGNORECASE)
-
-
-def clean_link(link_text):
-    """Remove leading and trailing whitespace and punctuation"""
-    return link_text.strip("\t\r\n '\"\x0c")
-
-
-class RegexLinkExtractor(SgmlLinkExtractor):
-    """High performant link extractor"""
-
-    def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
-        def clean_text(text):
-            return replace_escape_chars(remove_tags(text.decode(response_encoding))).strip()
-
-        def clean_url(url):
-            clean_url = ''
-            try:
-                clean_url = urljoin(base_url, replace_entities(clean_link(url.decode(response_encoding))))
-            except ValueError:
-                pass
-            return clean_url
-
-        if base_url is None:
-            base_url = get_base_url(response_text, response_url, response_encoding)
-
-        links_text = linkre.findall(response_text)
-        return [Link(clean_url(url).encode(response_encoding),
-                     clean_text(text))
-                for url, _, text in links_text]
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
deleted file mode 100644
index 2ba6bca45..000000000
--- a/scrapy/linkextractors/sgml.py
+++ /dev/null
@@ -1,151 +0,0 @@
-"""
-SGMLParser-based Link extractors
-"""
-import warnings
-from urllib.parse import urljoin
-from sgmllib import SGMLParser
-
-from w3lib.url import safe_url_string, canonicalize_url
-from w3lib.html import strip_html5_whitespace
-
-from scrapy.link import Link
-from scrapy.linkextractors import FilteringLinkExtractor
-from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
-from scrapy.utils.python import unique as unique_list, to_unicode
-from scrapy.utils.response import get_base_url
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-
-class BaseSgmlLinkExtractor(SGMLParser):
-
-    def __init__(self, tag="a", attr="href", unique=False, process_value=None,
-                 strip=True, canonicalized=False):
-        warnings.warn(
-            "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
-            "Please use scrapy.linkextractors.LinkExtractor",
-            ScrapyDeprecationWarning, stacklevel=2,
-        )
-        SGMLParser.__init__(self)
-        self.scan_tag = tag if callable(tag) else lambda t: t == tag
-        self.scan_attr = attr if callable(attr) else lambda a: a == attr
-        self.process_value = (lambda v: v) if process_value is None else process_value
-        self.current_link = None
-        self.unique = unique
-        self.strip = strip
-        if canonicalized:
-            self.link_key = lambda link: link.url
-        else:
-            self.link_key = lambda link: canonicalize_url(link.url,
-                                                          keep_fragments=True)
-
-    def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
-        """ Do the real extraction work """
-        self.reset()
-        self.feed(response_text)
-        self.close()
-
-        ret = []
-        if base_url is None:
-            base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
-        for link in self.links:
-            if isinstance(link.url, str):
-                link.url = link.url.encode(response_encoding)
-            try:
-                link.url = urljoin(base_url, link.url)
-            except ValueError:
-                continue
-            link.url = safe_url_string(link.url, response_encoding)
-            link.text = to_unicode(link.text, response_encoding, errors='replace').strip()
-            ret.append(link)
-
-        return ret
-
-    def _process_links(self, links):
-        """ Normalize and filter extracted links
-
-        The subclass should override it if necessary
-        """
-        return unique_list(links, key=self.link_key) if self.unique else links
-
-    def extract_links(self, response):
-        # wrapper needed to allow to work directly with text
-        links = self._extract_links(response.body, response.url, response.encoding)
-        links = self._process_links(links)
-        return links
-
-    def reset(self):
-        SGMLParser.reset(self)
-        self.links = []
-        self.base_url = None
-        self.current_link = None
-
-    def unknown_starttag(self, tag, attrs):
-        if tag == 'base':
-            self.base_url = dict(attrs).get('href')
-        if self.scan_tag(tag):
-            for attr, value in attrs:
-                if self.scan_attr(attr):
-                    if self.strip and value is not None:
-                        value = strip_html5_whitespace(value)
-                    url = self.process_value(value)
-                    if url is not None:
-                        link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
-                        self.links.append(link)
-                        self.current_link = link
-
-    def unknown_endtag(self, tag):
-        if self.scan_tag(tag):
-            self.current_link = None
-
-    def handle_data(self, data):
-        if self.current_link:
-            self.current_link.text = self.current_link.text + data
-
-    def matches(self, url):
-        """This extractor matches with any url, since
-        it doesn't contain any patterns"""
-        return True
-
-
-class SgmlLinkExtractor(FilteringLinkExtractor):
-
-    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
-                 tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True,
-                 process_value=None, deny_extensions=None, restrict_css=(),
-                 strip=True, restrict_text=()):
-        warnings.warn(
-            "SgmlLinkExtractor is deprecated and will be removed in future releases. "
-            "Please use scrapy.linkextractors.LinkExtractor",
-            ScrapyDeprecationWarning, stacklevel=2,
-        )
-
-        tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
-        tag_func = lambda x: x in tags
-        attr_func = lambda x: x in attrs
-
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
-            lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
-                                       unique=unique, process_value=process_value, strip=strip,
-                                       canonicalized=canonicalize)
-
-        super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
-                                                allow_domains=allow_domains, deny_domains=deny_domains,
-                                                restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
-                                                canonicalize=canonicalize, deny_extensions=deny_extensions,
-                                                restrict_text=restrict_text)
-
-    def extract_links(self, response):
-        base_url = None
-        if self.restrict_xpaths:
-            base_url = get_base_url(response)
-            body = u''.join(f
-                            for x in self.restrict_xpaths
-                            for f in response.xpath(x).getall()
-                            ).encode(response.encoding, errors='xmlcharrefreplace')
-        else:
-            body = response.body
-
-        links = self._extract_links(body, response.url, response.encoding, base_url)
-        links = self._process_links(links)
-        return links
diff --git a/tests/ignores.txt b/tests/ignores.txt
index f6e0d6fbe..222288841 100644
--- a/tests/ignores.txt
+++ b/tests/ignores.txt
@@ -1,5 +1,3 @@
-scrapy/linkextractors/sgml.py
-scrapy/linkextractors/regex.py
 scrapy/downloadermiddlewares/cookies.py
 scrapy/extensions/statsmailer.py
 scrapy/extensions/memusage.py
diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html
index 7d5db368a..2307ea865 100644
--- a/tests/sample_data/link_extractor/linkextractor.html
+++ b/tests/sample_data/link_extractor/linkextractor.html
@@ -1,7 +1,7 @@
 <html>
 <head>
 <base href='http://example.com' />
-<title>Sample page with links for testing RegexLinkExtractor</title>
+<title>Sample page with links for testing LinkExtractor</title>
 </head>
 <body>
 <div id='wrapper'>
diff --git a/tests/sample_data/link_extractor/linkextractor_latin1.html b/tests/sample_data/link_extractor/linkextractor_latin1.html
index fc31d7e5d..e7eee18de 100644
--- a/tests/sample_data/link_extractor/linkextractor_latin1.html
+++ b/tests/sample_data/link_extractor/linkextractor_latin1.html
@@ -2,7 +2,7 @@
   <head>
     <meta http-equiv="Content-Type" content="text/html; charset=latin-1">
     <base href='http://example.com' />
-    <title>Sample page with links for testing RegexLinkExtractor</title>
+    <title>Sample page with links for testing LinkExtractor</title>
   </head>
 <body>
   <div id='wrapper'>

From 86f7ac2f2b5d58e0b2588fa2aa4c777a8decf299 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Fri, 17 Jul 2020 17:48:25 +0300
Subject: [PATCH 2933/2974] Try to fix error at Windows

---
 tests/test_feedexport.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 129b7fc0b..cc124624d 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1204,6 +1204,8 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
             for path, feed in FEEDS.items():
                 dir_name = os.path.dirname(path)
+                if not os.path.exists(str(dir_name)):
+                    continue
                 for file in sorted(os.listdir(dir_name)):
                     with open(os.path.join(dir_name, file), 'rb') as f:
                         data = f.read()

From 3e0492741d93b05c464457b3b128a2b0d24c994b Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Sun, 19 Jul 2020 00:10:29 +0300
Subject: [PATCH 2934/2974] Another try to fix test errors on Windows

---
 tests/test_feedexport.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index cc124624d..c49b2e92f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1190,9 +1190,14 @@ class BatchDeliveriesTest(FeedExportTestBase):
     def run_and_export(self, spider_cls, settings):
         """ Run spider with specified settings; return exported data. """
 
+        def build_url(path):
+            if path[0] != '/':
+                path = '/' + path
+            return urljoin('file:', path)
+
         FEEDS = settings.get('FEEDS') or {}
         settings['FEEDS'] = {
-            urljoin('file:', file_path): feed
+            build_url(file_path): feed
             for file_path, feed in FEEDS.items()
         }
         content = defaultdict(list)
@@ -1204,8 +1209,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
             for path, feed in FEEDS.items():
                 dir_name = os.path.dirname(path)
-                if not os.path.exists(str(dir_name)):
-                    continue
                 for file in sorted(os.listdir(dir_name)):
                     with open(os.path.join(dir_name, file), 'rb') as f:
                         data = f.read()

From de297a3a167a6be3ac1ed94a891effefc12a6d00 Mon Sep 17 00:00:00 2001
From: Akshay Sharma <42249933+AKSHAYSHARMAJS@users.noreply.github.com>
Date: Mon, 20 Jul 2020 17:53:38 +0530
Subject: [PATCH 2935/2974] enable ANSI color (instead of ANSI color codes) in
 the Windows terminal #4393 (#4403)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* changed ie. -> i.e.(spelling error) on lines 667, 763 (issue scrapy#4332)

* updated all text files for issue #4332 (ie. -> i.e.)

* Apply ie. → i.e. in source comments

* ie → e.g.

* modified scrapy/utils/display.py to stop ANSI color sequences in the Windows terminal (issue #4393)

* modified scrapy/utils/display.py to stop ANSI color sequences in the Windows terminal (issue #4393)

* enabled virtual terminal processing (pr #4403)

* check for specific windows 10 version (pr #4403)

* fixing flake-8 test (pr #4403)

* added error handling for terminal info (pr #4403)

* corrected stderr (pr #4403)

* changed orientation, removed unwanted spaces (pr #4403)

* no need for style variable (pr #4403)

* fixing trailing whitespaces

* commenting windows check

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* small fixes

* Shifting _color_support_info() function

* enabled virtual terminal processing (pr #4403)

* check for specific windows 10 version (pr #4403)

* fixing flake-8 test (pr #4403)

* added error handling for terminal info (pr #4403)

* corrected stderr (pr #4403)

* changed orientation, removed unwanted spaces (pr #4403)

* no need for style variable (pr #4403)

* fixing trailing whitespaces

* commenting windows check

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* small fixes

* Shifting _color_support_info() function

* error handling

* error handlingy

* raise ValueError

* added in-built function for version comparison

* recommit changes

* changed check -> parse

* version comparison -> parse_version

* added scrapy/utils/display.py in pytest.ini

* Trigger

* Add simple test for scrapy.utils.display._colorize

* Flake8: E501 for tests/test_utils_display.py

* assertEquals -> assertEqual

* Normal formatter for all platforms

* separate test for windows

* all curses under try block

* added global TestStr

* more test added

* small fix

* covering exceptions

* windows test failing

* Refactor output color handling

* Fix pprint test

* fix flake8

Co-authored-by: Adrián Chaves <adrian@chaves.io>
Co-authored-by: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
---
 scrapy/utils/display.py     | 28 +++++++++++--
 tests/test_utils_display.py | 78 +++++++++++++++++++++++++++++++++++++
 2 files changed, 103 insertions(+), 3 deletions(-)
 create mode 100644 tests/test_utils_display.py

diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py
index 9735220ef..f4d17224b 100644
--- a/scrapy/utils/display.py
+++ b/scrapy/utils/display.py
@@ -2,20 +2,42 @@
 pprint and pformat wrappers with colorization support
 """
 
+import ctypes
+import platform
 import sys
+from distutils.version import LooseVersion as parse_version
 from pprint import pformat as pformat_
 
 
+def _enable_windows_terminal_processing():
+    # https://stackoverflow.com/a/36760881
+    kernel32 = ctypes.windll.kernel32
+    return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7))
+
+
+def _tty_supports_color():
+    if sys.platform != "win32":
+        return True
+
+    if parse_version(platform.version()) < parse_version("10.0.14393"):
+        return True
+
+    # Windows >= 10.0.14393 interprets ANSI escape sequences providing terminal
+    # processing is enabled.
+    return _enable_windows_terminal_processing()
+
+
 def _colorize(text, colorize=True):
-    if not colorize or not sys.stdout.isatty():
+    if not colorize or not sys.stdout.isatty() or not _tty_supports_color():
         return text
     try:
         from pygments import highlight
+    except ImportError:
+        return text
+    else:
         from pygments.formatters import TerminalFormatter
         from pygments.lexers import PythonLexer
         return highlight(text, PythonLexer(), TerminalFormatter())
-    except ImportError:
-        return text
 
 
 def pformat(obj, *args, **kwargs):
diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py
new file mode 100644
index 000000000..9ec8311d9
--- /dev/null
+++ b/tests/test_utils_display.py
@@ -0,0 +1,78 @@
+from io import StringIO
+
+from unittest import mock, TestCase
+
+from scrapy.utils.display import pformat, pprint
+
+
+class TestDisplay(TestCase):
+    object = {'a': 1}
+    colorized_string = (
+        "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
+        "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}\n"
+    )
+    plain_string = "{'a': 1}"
+
+    @mock.patch('sys.platform', 'linux')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat(self, isatty):
+        isatty.return_value = True
+        self.assertEqual(pformat(self.object), self.colorized_string)
+
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_dont_colorize(self, isatty):
+        isatty.return_value = True
+        self.assertEqual(pformat(self.object, colorize=False), self.plain_string)
+
+    def test_pformat_not_tty(self):
+        self.assertEqual(pformat(self.object), self.plain_string)
+
+    @mock.patch('sys.platform', 'win32')
+    @mock.patch('platform.version')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_old_windows(self, isatty, version):
+        isatty.return_value = True
+        version.return_value = '10.0.14392'
+        self.assertEqual(pformat(self.object), self.colorized_string)
+
+    @mock.patch('sys.platform', 'win32')
+    @mock.patch('scrapy.utils.display._enable_windows_terminal_processing')
+    @mock.patch('platform.version')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_windows_no_terminal_processing(self, isatty, version, terminal_processing):
+        isatty.return_value = True
+        version.return_value = '10.0.14393'
+        terminal_processing.return_value = False
+        self.assertEqual(pformat(self.object), self.plain_string)
+
+    @mock.patch('sys.platform', 'win32')
+    @mock.patch('scrapy.utils.display._enable_windows_terminal_processing')
+    @mock.patch('platform.version')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_windows(self, isatty, version, terminal_processing):
+        isatty.return_value = True
+        version.return_value = '10.0.14393'
+        terminal_processing.return_value = True
+        self.assertEqual(pformat(self.object), self.colorized_string)
+
+    @mock.patch('sys.platform', 'linux')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_no_pygments(self, isatty):
+        isatty.return_value = True
+
+        import builtins
+        real_import = builtins.__import__
+
+        def mock_import(name, globals, locals, fromlist, level):
+            if 'pygments' in name:
+                raise ImportError
+            return real_import(name, globals, locals, fromlist, level)
+
+        builtins.__import__ = mock_import
+        self.assertEqual(pformat(self.object), self.plain_string)
+        builtins.__import__ = real_import
+
+    def test_pprint(self):
+        with mock.patch('sys.stdout', new=StringIO()) as mock_out:
+            pprint(self.object)
+            self.assertEqual(mock_out.getvalue(), "{'a': 1}\n")

From ece4fa6c7cf7fdce9e5e77f56b209941ad401f53 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sat, 7 Mar 2020 19:21:45 +0000
Subject: [PATCH 2936/2974] Fix ignored testcase: boto is never installed

---
 tests/test_feedexport.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 37384081a..21da9fdcd 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -182,9 +182,9 @@ class S3FeedStorageTest(unittest.TestCase):
                 create=True)
     def test_parse_credentials(self):
         try:
-            import boto  # noqa: F401
+            import botocore  # noqa: F401
         except ImportError:
-            raise unittest.SkipTest("S3FeedStorage requires boto")
+            raise unittest.SkipTest("S3FeedStorage requires botocore")
         aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key',
                            'AWS_SECRET_ACCESS_KEY': 'settings_secret'}
         crawler = get_crawler(settings_dict=aws_credentials)

From 234c8b8c50a2db0d6d71455c382957099f4c0dcc Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sat, 7 Mar 2020 19:31:56 +0000
Subject: [PATCH 2937/2974] Removing deprecated S3FeedStorage without AWS keys
 instancing.

---
 scrapy/extensions/feedexport.py | 21 ++++-----------------
 1 file changed, 4 insertions(+), 17 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 72a34ae0d..e793c12dc 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -94,23 +94,10 @@ class FileFeedStorage:
 class S3FeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri, access_key=None, secret_key=None, acl=None):
-        # BEGIN Backward compatibility for initialising without keys (and
-        # without using from_crawler)
-        no_defaults = access_key is None and secret_key is None
-        if no_defaults:
-            from scrapy.utils.project import get_project_settings
-            settings = get_project_settings()
-            if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:
-                warnings.warn(
-                    "Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
-                    "without AWS keys is deprecated. Please supply credentials or "
-                    "use the `from_crawler()` constructor.",
-                    category=ScrapyDeprecationWarning,
-                    stacklevel=2
-                )
-                access_key = settings['AWS_ACCESS_KEY_ID']
-                secret_key = settings['AWS_SECRET_ACCESS_KEY']
-        # END Backward compatibility
+        no_keys = access_key is None and secret_key is None
+        if no_keys:
+            raise NotConfigured('%s is missing AWS credentials' %
+                                self.__class__.__name__)
         u = urlparse(uri)
         self.bucketname = u.hostname
         self.access_key = u.username or access_key

From 98e8086d1b3da8906d1395f453b1299b4e3b499e Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Sat, 7 Mar 2020 19:21:09 +0000
Subject: [PATCH 2938/2974] Adapt S3FeedStorage testcase

---
 tests/test_feedexport.py | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 21da9fdcd..34a67fff3 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -24,6 +24,7 @@ from zope.interface.verify import verifyObject
 
 import scrapy
 from scrapy.crawler import CrawlerRunner
+from scrapy.exceptions import NotConfigured
 from scrapy.exporters import CsvItemExporter
 from scrapy.extensions.feedexport import (
     BlockingFeedStorage,
@@ -176,10 +177,6 @@ class BlockingFeedStorageTest(unittest.TestCase):
 
 class S3FeedStorageTest(unittest.TestCase):
 
-    @mock.patch('scrapy.utils.project.get_project_settings',
-                new=mock.MagicMock(return_value={'AWS_ACCESS_KEY_ID': 'conf_key',
-                                                 'AWS_SECRET_ACCESS_KEY': 'conf_secret'}),
-                create=True)
     def test_parse_credentials(self):
         try:
             import botocore  # noqa: F401
@@ -205,12 +202,9 @@ class S3FeedStorageTest(unittest.TestCase):
                                 aws_credentials['AWS_SECRET_ACCESS_KEY'])
         self.assertEqual(storage.access_key, 'uri_key')
         self.assertEqual(storage.secret_key, 'uri_secret')
-        # Backward compatibility for initialising without settings
-        with warnings.catch_warnings(record=True) as w:
-            storage = S3FeedStorage('s3://mybucket/export.csv')
-            self.assertEqual(storage.access_key, 'conf_key')
-            self.assertEqual(storage.secret_key, 'conf_secret')
-            self.assertTrue('without AWS keys' in str(w[-1].message))
+        # Instantiate without credentials
+        with self.assertRaises(NotConfigured):
+            S3FeedStorage('s3://mybucket/export.csv')
 
     @defer.inlineCallbacks
     def test_store(self):

From 2829cd4268d22a328acd64a6816c479b1fe21f39 Mon Sep 17 00:00:00 2001
From: nyov <nyov@nexnode.net>
Date: Tue, 17 Mar 2020 10:19:13 +0000
Subject: [PATCH 2939/2974] Allow use without credentials

---
 scrapy/extensions/feedexport.py | 4 ----
 tests/test_feedexport.py        | 4 ----
 2 files changed, 8 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index e793c12dc..68d6533d3 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -94,10 +94,6 @@ class FileFeedStorage:
 class S3FeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri, access_key=None, secret_key=None, acl=None):
-        no_keys = access_key is None and secret_key is None
-        if no_keys:
-            raise NotConfigured('%s is missing AWS credentials' %
-                                self.__class__.__name__)
         u = urlparse(uri)
         self.bucketname = u.hostname
         self.access_key = u.username or access_key
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 34a67fff3..656bb515f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -24,7 +24,6 @@ from zope.interface.verify import verifyObject
 
 import scrapy
 from scrapy.crawler import CrawlerRunner
-from scrapy.exceptions import NotConfigured
 from scrapy.exporters import CsvItemExporter
 from scrapy.extensions.feedexport import (
     BlockingFeedStorage,
@@ -202,9 +201,6 @@ class S3FeedStorageTest(unittest.TestCase):
                                 aws_credentials['AWS_SECRET_ACCESS_KEY'])
         self.assertEqual(storage.access_key, 'uri_key')
         self.assertEqual(storage.secret_key, 'uri_secret')
-        # Instantiate without credentials
-        with self.assertRaises(NotConfigured):
-            S3FeedStorage('s3://mybucket/export.csv')
 
     @defer.inlineCallbacks
     def test_store(self):

From 430d22e46ebb67aba39005365e2e68d834d8fb86 Mon Sep 17 00:00:00 2001
From: Artur Shellunts <shellunts.artur@gmail.com>
Date: Tue, 21 Jul 2020 23:39:04 +0200
Subject: [PATCH 2940/2974] Remove not used import warnings

---
 tests/test_feedexport.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 656bb515f..e80e07554 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -5,7 +5,6 @@ import random
 import shutil
 import string
 import tempfile
-import warnings
 from io import BytesIO
 from logging import getLogger
 from pathlib import Path

From 8fae3d5bb768ebb3f9674cc38172226f404fb297 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 22 Jul 2020 16:08:35 -0300
Subject: [PATCH 2941/2974] Remove monkeypatches module from mypy section in
 setup.cfg

---
 setup.cfg | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 46a3d13fc..f8e7c0c91 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -13,9 +13,6 @@ follow_imports = skip
 [mypy-scrapy]
 ignore_errors = True
 
-[mypy-scrapy._monkeypatches]
-ignore_errors = True
-
 [mypy-scrapy.commands]
 ignore_errors = True
 

From a6c1d79b7cc3bc2c408eab356bbbf99a0536f110 Mon Sep 17 00:00:00 2001
From: BroodingKangaroo <johngolt33@gmail.com>
Date: Tue, 28 Jul 2020 11:53:05 +0300
Subject: [PATCH 2942/2974] pep8 tiny changes

---
 docs/topics/feed-exports.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 2c9774b55..dd4eb3c61 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -453,6 +453,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 
 FEED_EXPORT_BATCH_ITEM_COUNT
 -----------------------------
+
 Default: ``0``
 
 If assigned an integer number higher than ``0``, Scrapy generates multiple output files
@@ -474,7 +475,7 @@ generated:
 
 For instance, if your settings include::
 
-    FEED_EXPORT_BATCH_ITEM_COUNT=100
+    FEED_EXPORT_BATCH_ITEM_COUNT = 100
 
 And your :command:`crawl` command line is::
 

From 52658539370c442e63102a3208781335953cdf53 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Tue, 28 Jul 2020 06:15:14 -0300
Subject: [PATCH 2943/2974] Use ItemAdapter.field_names when writing header in
 CsvItemExporter (#4668)

---
 scrapy/exporters.py     |   8 +-
 tests/test_exporters.py | 203 ++++++++++++++++++++++++++++------------
 2 files changed, 146 insertions(+), 65 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 712572673..0aba1c904 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -243,12 +243,8 @@ class CsvItemExporter(BaseItemExporter):
     def _write_headers_and_set_fields_to_export(self, item):
         if self.include_headers_line:
             if not self.fields_to_export:
-                if isinstance(item, dict):
-                    # for dicts try using fields of the first item
-                    self.fields_to_export = list(item.keys())
-                else:
-                    # use fields declared in Item
-                    self.fields_to_export = list(item.fields.keys())
+                # use declared field names, or keys if the item is a dict
+                self.fields_to_export = ItemAdapter(item).field_names()
             row = list(self._build_row(self.fields_to_export))
             self.csv_writer.writerow(row)
 
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index b27380309..25da54a65 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -8,6 +8,7 @@ from io import BytesIO
 from datetime import datetime
 
 import lxml.etree
+from itemadapter import ItemAdapter
 
 from scrapy.item import Item, Field
 from scrapy.utils.python import to_unicode
@@ -23,10 +24,37 @@ class TestItem(Item):
     age = Field()
 
 
+def custom_serializer(value):
+    return str(int(value) + 2)
+
+
+class CustomFieldItem(Item):
+    name = Field()
+    age = Field(serializer=custom_serializer)
+
+
+try:
+    from dataclasses import make_dataclass, field
+except ImportError:
+    TestDataClass = None
+    CustomFieldDataclass = None
+else:
+    TestDataClass = make_dataclass("TestDataClass", [("name", str), ("age", int)])
+    CustomFieldDataclass = make_dataclass(
+        "CustomFieldDataclass",
+        [("name", str), ("age", int, field(metadata={"serializer": custom_serializer}))]
+    )
+
+
 class BaseItemExporterTest(unittest.TestCase):
 
+    item_class = TestItem
+    custom_field_item_class = CustomFieldItem
+
     def setUp(self):
-        self.i = TestItem(name=u'John\xa3', age=u'22')
+        if self.item_class is None:
+            raise unittest.SkipTest("item class is None")
+        self.i = self.item_class(name=u'John\xa3', age=u'22')
         self.output = BytesIO()
         self.ie = self._get_exporter()
 
@@ -39,7 +67,7 @@ class BaseItemExporterTest(unittest.TestCase):
     def _assert_expected_item(self, exported_dict):
         for k, v in exported_dict.items():
             exported_dict[k] = to_unicode(v)
-        self.assertEqual(self.i, exported_dict)
+        self.assertEqual(self.i, self.item_class(**exported_dict))
 
     def _get_nonstring_types_item(self):
         return {
@@ -63,13 +91,14 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertItemExportWorks(self.i)
 
     def test_export_dict_item(self):
-        self.assertItemExportWorks(dict(self.i))
+        self.assertItemExportWorks(ItemAdapter(self.i).asdict())
 
     def test_serialize_field(self):
-        res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name'])
+        a = ItemAdapter(self.i)
+        res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name'])
         self.assertEqual(res, u'John\xa3')
 
-        res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age'])
+        res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age'])
         self.assertEqual(res, u'22')
 
     def test_fields_to_export(self):
@@ -82,18 +111,16 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertEqual(name, u'John\xa3')
 
     def test_field_custom_serializer(self):
-        def custom_serializer(value):
-            return str(int(value) + 2)
-
-        class CustomFieldItem(Item):
-            name = Field()
-            age = Field(serializer=custom_serializer)
-
-        i = CustomFieldItem(name=u'John\xa3', age=u'22')
-
+        i = self.custom_field_item_class(name=u'John\xa3', age=u'22')
+        a = ItemAdapter(i)
         ie = self._get_exporter()
-        self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3')
-        self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
+        self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3')
+        self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24')
+
+
+class BaseItemExporterDataclassTest(BaseItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
 
 
 class PythonItemExporterTest(BaseItemExporterTest):
@@ -105,9 +132,9 @@ class PythonItemExporterTest(BaseItemExporterTest):
             PythonItemExporter(invalid_option='something')
 
     def test_nested_item(self):
-        i1 = TestItem(name=u'Joseph', age='22')
+        i1 = self.item_class(name=u'Joseph', age='22')
         i2 = dict(name=u'Maria', age=i1)
-        i3 = TestItem(name=u'Jesus', age=i2)
+        i3 = self.item_class(name=u'Jesus', age=i2)
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(type(exported), dict)
@@ -119,9 +146,9 @@ class PythonItemExporterTest(BaseItemExporterTest):
         self.assertEqual(type(exported['age']['age']), dict)
 
     def test_export_list(self):
-        i1 = TestItem(name=u'Joseph', age='22')
-        i2 = TestItem(name=u'Maria', age=[i1])
-        i3 = TestItem(name=u'Jesus', age=[i2])
+        i1 = self.item_class(name=u'Joseph', age='22')
+        i2 = self.item_class(name=u'Maria', age=[i1])
+        i3 = self.item_class(name=u'Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(
@@ -132,9 +159,9 @@ class PythonItemExporterTest(BaseItemExporterTest):
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
     def test_export_item_dict_list(self):
-        i1 = TestItem(name=u'Joseph', age='22')
+        i1 = self.item_class(name=u'Joseph', age='22')
         i2 = dict(name=u'Maria', age=[i1])
-        i3 = TestItem(name=u'Jesus', age=[i2])
+        i3 = self.item_class(name=u'Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(
@@ -146,7 +173,7 @@ class PythonItemExporterTest(BaseItemExporterTest):
 
     def test_export_binary(self):
         exporter = PythonItemExporter(binary=True)
-        value = TestItem(name=u'John\xa3', age=u'22')
+        value = self.item_class(name=u'John\xa3', age=u'22')
         expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
         self.assertEqual(expected, exporter.export_item(value))
 
@@ -157,6 +184,11 @@ class PythonItemExporterTest(BaseItemExporterTest):
         self.assertEqual(exported, item)
 
 
+class PythonItemExporterDataclassTest(PythonItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class PprintItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
@@ -166,6 +198,11 @@ class PprintItemExporterTest(BaseItemExporterTest):
         self._assert_expected_item(eval(self.output.getvalue()))
 
 
+class PprintItemExporterDataclassTest(PprintItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class PickleItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
@@ -175,8 +212,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
         self._assert_expected_item(pickle.loads(self.output.getvalue()))
 
     def test_export_multiple_items(self):
-        i1 = TestItem(name='hello', age='world')
-        i2 = TestItem(name='bye', age='world')
+        i1 = self.item_class(name='hello', age='world')
+        i2 = self.item_class(name='bye', age='world')
         f = BytesIO()
         ie = PickleItemExporter(f)
         ie.start_exporting()
@@ -184,8 +221,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
         ie.export_item(i2)
         ie.finish_exporting()
         f.seek(0)
-        self.assertEqual(pickle.load(f), i1)
-        self.assertEqual(pickle.load(f), i2)
+        self.assertEqual(self.item_class(**pickle.load(f)), i1)
+        self.assertEqual(self.item_class(**pickle.load(f)), i2)
 
     def test_nonstring_types_item(self):
         item = self._get_nonstring_types_item()
@@ -197,6 +234,11 @@ class PickleItemExporterTest(BaseItemExporterTest):
         self.assertEqual(pickle.loads(fp.getvalue()), item)
 
 
+class PickleItemExporterDataclassTest(PickleItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class MarshalItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
@@ -219,6 +261,11 @@ class MarshalItemExporterTest(BaseItemExporterTest):
         self.assertEqual(marshal.load(fp), item)
 
 
+class MarshalItemExporterDataclassTest(MarshalItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class CsvItemExporterTest(BaseItemExporterTest):
     def _get_exporter(self, **kwargs):
         return CsvItemExporter(self.output, **kwargs)
@@ -245,18 +292,18 @@ class CsvItemExporterTest(BaseItemExporterTest):
     def test_header_export_all(self):
         self.assertExportResult(
             item=self.i,
-            fields_to_export=self.i.fields.keys(),
+            fields_to_export=ItemAdapter(self.i).field_names(),
             expected=b'age,name\r\n22,John\xc2\xa3\r\n',
         )
 
     def test_header_export_all_dict(self):
         self.assertExportResult(
-            item=dict(self.i),
+            item=ItemAdapter(self.i).asdict(),
             expected=b'age,name\r\n22,John\xc2\xa3\r\n',
         )
 
     def test_header_export_single_field(self):
-        for item in [self.i, dict(self.i)]:
+        for item in [self.i, ItemAdapter(self.i).asdict()]:
             self.assertExportResult(
                 item=item,
                 fields_to_export=['age'],
@@ -264,7 +311,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
             )
 
     def test_header_export_two_items(self):
-        for item in [self.i, dict(self.i)]:
+        for item in [self.i, ItemAdapter(self.i).asdict()]:
             output = BytesIO()
             ie = CsvItemExporter(output)
             ie.start_exporting()
@@ -275,7 +322,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
                                 b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
 
     def test_header_no_header_line(self):
-        for item in [self.i, dict(self.i)]:
+        for item in [self.i, ItemAdapter(self.i).asdict()]:
             self.assertExportResult(
                 item=item,
                 include_headers_line=False,
@@ -309,6 +356,11 @@ class CsvItemExporterTest(BaseItemExporterTest):
         )
 
 
+class CsvItemExporterDataclassTest(CsvItemExporterTest):
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class XmlItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
@@ -318,8 +370,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
         def xmltuple(elem):
             children = list(elem.iterchildren())
             if children:
-                return [(child.tag, sorted(xmltuple(child)))
-                        for child in children]
+                return [(child.tag, sorted(xmltuple(child))) for child in children]
             else:
                 return [(elem.tag, [(elem.text, ())])]
 
@@ -345,17 +396,21 @@ class XmlItemExporterTest(BaseItemExporterTest):
 
     def test_multivalued_fields(self):
         self.assertExportResult(
-            TestItem(name=[u'John\xa3', u'Doe']),
-            (
-                b'<?xml version="1.0" encoding="utf-8"?>\n'
-                b'<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
-            )
+            self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]),
+            b"""<?xml version="1.0" encoding="utf-8"?>\n
+            <items>
+                <item>
+                    <name><value>John\xc2\xa3</value><value>Doe</value></name>
+                    <age><value>1</value><value>2</value><value>3</value></age>
+                </item>
+            </items>
+            """
         )
 
     def test_nested_item(self):
-        i1 = TestItem(name=u'foo\xa3hoo', age='22')
+        i1 = dict(name=u'foo\xa3hoo', age='22')
         i2 = dict(name=u'bar', age=i1)
-        i3 = TestItem(name=u'buz', age=i2)
+        i3 = self.item_class(name=u'buz', age=i2)
 
         self.assertExportResult(
             i3,
@@ -376,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
         )
 
     def test_nested_list_item(self):
-        i1 = TestItem(name=u'foo')
+        i1 = dict(name=u'foo')
         i2 = dict(name=u'bar', v2={"egg": ["spam"]})
-        i3 = TestItem(name=u'buz', age=[i1, i2])
+        i3 = self.item_class(name=u'buz', age=[i1, i2])
 
         self.assertExportResult(
             i3,
@@ -412,6 +467,12 @@ class XmlItemExporterTest(BaseItemExporterTest):
         )
 
 
+class XmlItemExporterDataclassTest(XmlItemExporterTest):
+
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class JsonLinesItemExporterTest(BaseItemExporterTest):
 
     _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
@@ -421,12 +482,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
 
     def _check_output(self):
         exported = json.loads(to_unicode(self.output.getvalue().strip()))
-        self.assertEqual(exported, dict(self.i))
+        self.assertEqual(exported, ItemAdapter(self.i).asdict())
 
     def test_nested_item(self):
-        i1 = TestItem(name=u'Joseph', age='22')
+        i1 = self.item_class(name=u'Joseph', age='22')
         i2 = dict(name=u'Maria', age=i1)
-        i3 = TestItem(name=u'Jesus', age=i2)
+        i3 = self.item_class(name=u'Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
@@ -449,6 +510,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         self.assertEqual(exported, item)
 
 
+class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest):
+
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
 class JsonItemExporterTest(JsonLinesItemExporterTest):
 
     _expected_nested = [JsonLinesItemExporterTest._expected_nested]
@@ -458,7 +525,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
 
     def _check_output(self):
         exported = json.loads(to_unicode(self.output.getvalue().strip()))
-        self.assertEqual(exported, [dict(self.i)])
+        self.assertEqual(exported, [ItemAdapter(self.i).asdict()])
 
     def assertTwoItemsExported(self, item):
         self.ie.start_exporting()
@@ -466,28 +533,28 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.export_item(item)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        self.assertEqual(exported, [dict(item), dict(item)])
+        self.assertEqual(exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()])
 
     def test_two_items(self):
         self.assertTwoItemsExported(self.i)
 
     def test_two_dict_items(self):
-        self.assertTwoItemsExported(dict(self.i))
+        self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
 
     def test_nested_item(self):
-        i1 = TestItem(name=u'Joseph\xa3', age='22')
-        i2 = TestItem(name=u'Maria', age=i1)
-        i3 = TestItem(name=u'Jesus', age=i2)
+        i1 = self.item_class(name=u'Joseph\xa3', age='22')
+        i2 = self.item_class(name=u'Maria', age=i1)
+        i3 = self.item_class(name=u'Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}}
+        expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
         self.assertEqual(exported, [expected])
 
     def test_nested_dict_item(self):
         i1 = dict(name=u'Joseph\xa3', age='22')
-        i2 = TestItem(name=u'Maria', age=i1)
+        i2 = self.item_class(name=u'Maria', age=i1)
         i3 = dict(name=u'Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
@@ -506,7 +573,19 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.assertEqual(exported, [item])
 
 
-class CustomItemExporterTest(unittest.TestCase):
+class JsonItemExporterDataclassTest(JsonItemExporterTest):
+
+    item_class = TestDataClass
+    custom_field_item_class = CustomFieldDataclass
+
+
+class CustomExporterItemTest(unittest.TestCase):
+
+    item_class = TestItem
+
+    def setUp(self):
+        if self.item_class is None:
+            raise unittest.SkipTest("item class is None")
 
     def test_exporter_custom_serializer(self):
         class CustomItemExporter(BaseItemExporter):
@@ -516,16 +595,22 @@ class CustomItemExporterTest(unittest.TestCase):
                 else:
                     return super(CustomItemExporter, self).serialize_field(field, name, value)
 
-        i = TestItem(name=u'John', age='22')
+        i = self.item_class(name=u'John', age='22')
+        a = ItemAdapter(i)
         ie = CustomItemExporter()
 
-        self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John')
-        self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '23')
+        self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John')
+        self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23')
 
         i2 = {'name': u'John', 'age': '22'}
         self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John')
         self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23')
 
 
+class CustomExporterDataclassTest(CustomExporterItemTest):
+
+    item_class = TestDataClass
+
+
 if __name__ == '__main__':
     unittest.main()

From e7a58fe1573176415a9ca054428c53c1ca29931a Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ksharma@atlassian.com>
Date: Wed, 29 Jul 2020 10:16:18 +0530
Subject: [PATCH 2944/2974] Code cleanup scrapy.utils.python.WeakKeyCache #4684

---
 scrapy/utils/python.py     | 12 ------------
 tests/test_utils_python.py | 18 +-----------------
 2 files changed, 1 insertion(+), 29 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 9204977cf..7a393925e 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -273,18 +273,6 @@ def equal_attributes(obj1, obj2, attributes):
     return True
 
 
-class WeakKeyCache:
-
-    def __init__(self, default_factory):
-        self.default_factory = default_factory
-        self._weakdict = weakref.WeakKeyDictionary()
-
-    def __getitem__(self, key):
-        if key not in self._weakdict:
-            self._weakdict[key] = self.default_factory(key)
-        return self._weakdict[key]
-
-
 @deprecated
 def retry_on_eintr(function, *args, **kw):
     """Run a function and retry it while getting EINTR errors"""
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index ebce3c079..b23ae2e52 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -9,7 +9,7 @@ from warnings import catch_warnings
 
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
-    WeakKeyCache, get_func_args, to_bytes, to_unicode,
+    get_func_args, to_bytes, to_unicode,
     without_none_values, MutableChain)
 
 
@@ -155,22 +155,6 @@ class UtilsPythonTestCase(unittest.TestCase):
         a.meta['z'] = 2
         self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
 
-    def test_weakkeycache(self):
-        class _Weakme:
-            pass
-
-        _values = count()
-        wk = WeakKeyCache(lambda k: next(_values))
-        k = _Weakme()
-        v = wk[k]
-        self.assertEqual(v, wk[k])
-        self.assertNotEqual(v, wk[_Weakme()])
-        self.assertEqual(v, wk[k])
-        del k
-        for _ in range(100):
-            if wk._weakdict:
-                gc.collect()
-        self.assertFalse(len(wk._weakdict))
 
     def test_get_func_args(self):
         def f1(a, b, c):

From 403bc7020a5e1ba2b59eced2cc5f4453c7650666 Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ksharma@atlassian.com>
Date: Wed, 29 Jul 2020 18:05:33 +0530
Subject: [PATCH 2945/2974] Code cleanup scrapy.utils.python.WeakKeyCache #4684
 and fixing ci alerts

---
 tests/test_utils_python.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index b23ae2e52..5a53d89e4 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -1,9 +1,7 @@
 import functools
-import gc
 import operator
 import platform
 import unittest
-from itertools import count
 from sys import version_info
 from warnings import catch_warnings
 
@@ -12,7 +10,6 @@ from scrapy.utils.python import (
     get_func_args, to_bytes, to_unicode,
     without_none_values, MutableChain)
 
-
 __doctests__ = ['scrapy.utils.python']
 
 
@@ -155,7 +152,6 @@ class UtilsPythonTestCase(unittest.TestCase):
         a.meta['z'] = 2
         self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
 
-
     def test_get_func_args(self):
         def f1(a, b, c):
             pass

From 49337bd2ae094d97d364948569f59b8211c8dbbe Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ksharma@atlassian.com>
Date: Thu, 30 Jul 2020 12:25:21 +0530
Subject: [PATCH 2947/2974] Code cleanup scrapy.utils.python.WeakKeyCache #4684
 and fixing ci alerts

---
 scrapy/utils/python.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 7a393925e..c8f921ff3 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -127,6 +127,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
     In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing
     the start position of the match, and the ending (regarding the entire text).
     """
+
     def _chunk_iter():
         offset = len(text)
         while True:
@@ -158,6 +159,7 @@ def memoizemethod_noargs(method):
         if self not in cache:
             cache[self] = method(self, *args, **kwargs)
         return cache[self]
+
     return new_method
 
 
@@ -273,6 +275,19 @@ def equal_attributes(obj1, obj2, attributes):
     return True
 
 
+@deprecated
+class WeakKeyCache:
+
+    def __init__(self, default_factory):
+        self.default_factory = default_factory
+        self._weakdict = weakref.WeakKeyDictionary()
+
+    def __getitem__(self, key):
+        if key not in self._weakdict:
+            self._weakdict[key] = self.default_factory(key)
+        return self._weakdict[key]
+
+
 @deprecated
 def retry_on_eintr(function, *args, **kw):
     """Run a function and retry it while getting EINTR errors"""

From 890b2138a605af2bfbf340a0d48d9d83c4cda53b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 30 Jul 2020 13:39:30 +0200
Subject: [PATCH 2948/2974] Remove the u prefix from strings

---
 docs/_ext/scrapydocs.py                      |   2 +-
 docs/topics/loaders.rst                      |   6 +-
 docs/topics/selectors.rst                    |   4 +-
 docs/utils/linkfix.py                        |   2 +-
 scrapy/http/request/form.py                  |   6 +-
 scrapy/linkextractors/lxmlhtml.py            |   2 +-
 scrapy/logformatter.py                       |   2 +-
 tests/test_cmdline/__init__.py               |   2 +-
 tests/test_downloader_handlers.py            |   6 +-
 tests/test_downloadermiddleware_cookies.py   |  16 +-
 tests/test_downloadermiddleware_httpproxy.py |   6 +-
 tests/test_downloadermiddleware_redirect.py  |   6 +-
 tests/test_downloadermiddleware_robotstxt.py |   4 +-
 tests/test_exporters.py                      |  84 +++----
 tests/test_feedexport.py                     |   8 +-
 tests/test_http_headers.py                   |   6 +-
 tests/test_http_request.py                   |  90 ++++----
 tests/test_http_response.py                  |  72 +++---
 tests/test_item.py                           |  42 ++--
 tests/test_linkextractors.py                 | 180 +++++++--------
 tests/test_loader.py                         | 124 +++++-----
 tests/test_loader_deprecated.py              | 226 +++++++++----------
 tests/test_logformatter.py                   |  12 +-
 tests/test_mail.py                           |   8 +-
 tests/test_responsetypes.py                  |  10 +-
 tests/test_robotstxt_interface.py            |   8 +-
 tests/test_selector.py                       |  32 +--
 tests/test_spider.py                         |  12 +-
 tests/test_utils_iterators.py                |  90 ++++----
 tests/test_utils_python.py                   |  16 +-
 tests/test_utils_reqser.py                   |   2 +-
 tests/test_utils_template.py                 |   4 +-
 32 files changed, 545 insertions(+), 545 deletions(-)

diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index 192123473..640660943 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -17,7 +17,7 @@ class SettingsListDirective(Directive):
 def is_setting_index(node):
     if node.tagname == 'index':
         # index entries for setting directives look like:
-        # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]
+        # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
         entry_type, info, refid = node['entries'][0][:3]
         return entry_type == 'pair' and info.endswith('; setting')
     return False
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index d0eeb4097..29d9c5805 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -237,10 +237,10 @@ metadata. Here is an example::
 
 >>> from scrapy.loader import ItemLoader
 >>> il = ItemLoader(item=Product())
->>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
->>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
+>>> il.add_value('name', ['Welcome to my', '<strong>website</strong>'])
+>>> il.add_value('price', ['&euro;', '<span>1000</span>'])
 >>> il.load_item()
-{'name': u'Welcome to my website', 'price': u'1000'}
+{'name': 'Welcome to my website', 'price': '1000'}
 
 The precedence order, for both input and output processors, is as follows:
 
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index bb46ea80f..5014df6ac 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -734,7 +734,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's
 Example selecting links in list item with a "class" attribute ending with a digit:
 
 >>> from scrapy import Selector
->>> doc = u"""
+>>> doc = """
 ... <div>
 ...     <ul>
 ...         <li class="item-0"><a href="link1.html">first item</a></li>
@@ -765,7 +765,7 @@ extracting text elements for example.
 Example extracting microdata (sample content taken from https://schema.org/Product)
 with groups of itemscopes and corresponding itemprops::
 
-    >>> doc = u"""
+    >>> doc = """
     ... <div itemscope itemtype="http://schema.org/Product">
     ...   <span itemprop="name">Kenmore White 17" Microwave</span>
     ...   <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
index 9acfc3b23..95a3f17d5 100755
--- a/docs/utils/linkfix.py
+++ b/docs/utils/linkfix.py
@@ -23,7 +23,7 @@ def main():
     _contents = None
 
     # A regex that matches standard linkcheck output lines
-    line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
+    line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
 
     # Read lines from the linkcheck output file
     try:
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 0e6ceef0b..a260798ac 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -133,7 +133,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
                         '  not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
                         namespaces={
                             "re": "http://exslt.org/regular-expressions"})
-    values = [(k, u'' if v is None else v)
+    values = [(k, '' if v is None else v)
               for k, v in (_value(e) for e in inputs)
               if k and k not in formdata_keys]
 
@@ -168,7 +168,7 @@ def _select_value(ele, n, v):
         # This is a workround to bug in lxml fixed 2.3.1
         # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
         selected_options = ele.xpath('.//option[@selected]')
-        v = [(o.get('value') or o.text or u'').strip() for o in selected_options]
+        v = [(o.get('value') or o.text or '').strip() for o in selected_options]
     return n, v
 
 
@@ -205,7 +205,7 @@ def _get_clickable(clickdata, form):
 
     # We didn't find it, so now we build an XPath expression out of the other
     # arguments, because they can be used as such
-    xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
+    xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items())
     el = form.xpath(xpath)
     if len(el) == 1:
         return (el[0].get('name'), el[0].get('value') or '')
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 1615d44d7..8b9f961ee 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -76,7 +76,7 @@ class LxmlParserLinkExtractor:
             url = safe_url_string(url, encoding=response_encoding)
             # to fix relative links after process_value
             url = urljoin(response_url, url)
-            link = Link(url, _collect_string_content(el) or u'',
+            link = Link(url, _collect_string_content(el) or '',
                         nofollow=rel_has_nofollow(el.get('rel')))
             links.append(link)
         return self._deduplicate_if_needed(links)
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 219145f13..0f9e6f1cb 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -44,7 +44,7 @@ class LogFormatter:
                 def dropped(self, item, exception, response, spider):
                     return {
                         'level': logging.INFO, # lowering the level from logging.WARNING
-                        'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s",
+                        'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",
                         'args': {
                             'exception': exception,
                             'item': item,
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index da99a6be8..591075a98 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -59,7 +59,7 @@ class CmdlineTest(unittest.TestCase):
                                     'EXTENSIONS=' + json.dumps(EXTENSIONS))
         # XXX: There's gotta be a smarter way to do this...
         self.assertNotIn("...", settingsstr)
-        for char in ("'", "<", ">", 'u"'):
+        for char in ("'", "<", ">"):
             settingsstr = settingsstr.replace(char, '"')
         settingsdict = json.loads(settingsstr)
         self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 51deb20f4..57d4cdd6b 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1110,7 +1110,7 @@ class DataURITestCase(unittest.TestCase):
 
     def test_default_mediatype(self):
         def _test(response):
-            self.assertEqual(response.text, u'\u038e\u03a3\u038e')
+            self.assertEqual(response.text, '\u038e\u03a3\u038e')
             self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "iso-8859-7")
 
@@ -1119,7 +1119,7 @@ class DataURITestCase(unittest.TestCase):
 
     def test_text_charset(self):
         def _test(response):
-            self.assertEqual(response.text, u'\u038e\u03a3\u038e')
+            self.assertEqual(response.text, '\u038e\u03a3\u038e')
             self.assertEqual(response.body, b'\xbe\xd3\xbe')
             self.assertEqual(response.encoding, "iso-8859-7")
 
@@ -1128,7 +1128,7 @@ class DataURITestCase(unittest.TestCase):
 
     def test_mediatype_parameters(self):
         def _test(response):
-            self.assertEqual(response.text, u'\u038e\u03a3\u038e')
+            self.assertEqual(response.text, '\u038e\u03a3\u038e')
             self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "utf-8")
 
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 9ccc2110b..010577415 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -277,33 +277,33 @@ class CookiesMiddlewareTest(TestCase):
 
     def test_request_cookies_encoding(self):
         # 1) UTF8-encoded bytes
-        req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')})
+        req1 = Request('http://example.org', cookies={'a': 'á'.encode('utf8')})
         assert self.mw.process_request(req1, self.spider) is None
         self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
 
         # 2) Non UTF8-encoded bytes
-        req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')})
+        req2 = Request('http://example.org', cookies={'a': 'á'.encode('latin1')})
         assert self.mw.process_request(req2, self.spider) is None
         self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
 
-        # 3) Unicode string
-        req3 = Request('http://example.org', cookies={'a': u'á'})
+        # 3) String
+        req3 = Request('http://example.org', cookies={'a': 'á'})
         assert self.mw.process_request(req3, self.spider) is None
         self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
 
     def test_request_headers_cookie_encoding(self):
         # 1) UTF8-encoded bytes
-        req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')})
+        req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')})
         assert self.mw.process_request(req1, self.spider) is None
         self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
 
         # 2) Non UTF8-encoded bytes
-        req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')})
+        req2 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('latin1')})
         assert self.mw.process_request(req2, self.spider) is None
         self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
 
-        # 3) Unicode string
-        req3 = Request('http://example.org', headers={'Cookie': u'a=á'})
+        # 3) String
+        req3 = Request('http://example.org', headers={'Cookie': 'a=á'})
         assert self.mw.process_request(req3, self.spider) is None
         self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
 
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 9841d7a76..351631eb8 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -88,7 +88,7 @@ class TestHttpProxyMiddleware(TestCase):
 
     def test_proxy_auth_encoding(self):
         # utf-8 encoding
-        os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128'
+        os.environ['http_proxy'] = 'https://m\u00E1n:pass@proxy:3128'
         mw = HttpProxyMiddleware(auth_encoding='utf-8')
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
@@ -96,7 +96,7 @@ class TestHttpProxyMiddleware(TestCase):
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
 
         # proxy from request.meta
-        req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
+        req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
         assert mw.process_request(req, spider) is None
         self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==')
@@ -109,7 +109,7 @@ class TestHttpProxyMiddleware(TestCase):
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
 
         # proxy from request.meta, latin-1 encoding
-        req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
+        req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
         assert mw.process_request(req, spider) is None
         self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz')
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 919dbed23..131332131 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -184,7 +184,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
 
     def test_latin1_location(self):
         req = Request('http://scrapytest.org/first')
-        latin1_location = u'/ação'.encode('latin1')  # HTTP historically supports latin1
+        latin1_location = '/ação'.encode('latin1')  # HTTP historically supports latin1
         resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302)
         req_result = self.mw.process_response(req, resp, self.spider)
         perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o'
@@ -192,7 +192,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
 
     def test_utf8_location(self):
         req = Request('http://scrapytest.org/first')
-        utf8_location = u'/ação'.encode('utf-8')  # header using UTF-8 encoding
+        utf8_location = '/ação'.encode('utf-8')  # header using UTF-8 encoding
         resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302)
         req_result = self.mw.process_response(req, resp, self.spider)
         perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
@@ -207,7 +207,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
         self.mw = MetaRefreshMiddleware.from_crawler(crawler)
 
     def _body(self, interval=5, url='http://example.org/newpage'):
-        html = u"""<html><head><meta http-equiv="refresh" content="{0};url={1}"/></head></html>"""
+        html = """<html><head><meta http-equiv="refresh" content="{0};url={1}"/></head></html>"""
         return html.format(interval, url).encode('utf-8')
 
     def test_priority_adjust(self):
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index b9452a0e7..f9936baba 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -30,7 +30,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
     def _get_successful_crawler(self):
         crawler = self.crawler
         crawler.settings.set('ROBOTSTXT_OBEY', True)
-        ROBOTS = u"""
+        ROBOTS = """
 User-Agent: *
 Disallow: /admin/
 Disallow: /static/
@@ -56,7 +56,7 @@ Disallow: /some/randome/page.html
             self.assertIgnored(Request('http://site.local/admin/main'), middleware),
             self.assertIgnored(Request('http://site.local/static/'), middleware),
             self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware),
-            self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware)
+            self.assertIgnored(Request('http://site.local/wiki/Käyttäjä:'), middleware)
         ], fireOnOneErrback=True)
 
     def test_robotstxt_ready_parser(self):
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 25da54a65..660c99ce1 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -54,7 +54,7 @@ class BaseItemExporterTest(unittest.TestCase):
     def setUp(self):
         if self.item_class is None:
             raise unittest.SkipTest("item class is None")
-        self.i = self.item_class(name=u'John\xa3', age=u'22')
+        self.i = self.item_class(name='John\xa3', age='22')
         self.output = BytesIO()
         self.ie = self._get_exporter()
 
@@ -96,25 +96,25 @@ class BaseItemExporterTest(unittest.TestCase):
     def test_serialize_field(self):
         a = ItemAdapter(self.i)
         res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name'])
-        self.assertEqual(res, u'John\xa3')
+        self.assertEqual(res, 'John\xa3')
 
         res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age'])
-        self.assertEqual(res, u'22')
+        self.assertEqual(res, '22')
 
     def test_fields_to_export(self):
         ie = self._get_exporter(fields_to_export=['name'])
-        self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')])
+        self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xa3')])
 
         ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
         _, name = list(ie._get_serialized_fields(self.i))[0]
         assert isinstance(name, str)
-        self.assertEqual(name, u'John\xa3')
+        self.assertEqual(name, 'John\xa3')
 
     def test_field_custom_serializer(self):
-        i = self.custom_field_item_class(name=u'John\xa3', age=u'22')
+        i = self.custom_field_item_class(name='John\xa3', age='22')
         a = ItemAdapter(i)
         ie = self._get_exporter()
-        self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3')
+        self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John\xa3')
         self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24')
 
 
@@ -132,48 +132,48 @@ class PythonItemExporterTest(BaseItemExporterTest):
             PythonItemExporter(invalid_option='something')
 
     def test_nested_item(self):
-        i1 = self.item_class(name=u'Joseph', age='22')
-        i2 = dict(name=u'Maria', age=i1)
-        i3 = self.item_class(name=u'Jesus', age=i2)
+        i1 = self.item_class(name='Joseph', age='22')
+        i2 = dict(name='Maria', age=i1)
+        i3 = self.item_class(name='Jesus', age=i2)
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(type(exported), dict)
         self.assertEqual(
             exported,
-            {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'}
+            {'age': {'age': {'age': '22', 'name': 'Joseph'}, 'name': 'Maria'}, 'name': 'Jesus'}
         )
         self.assertEqual(type(exported['age']), dict)
         self.assertEqual(type(exported['age']['age']), dict)
 
     def test_export_list(self):
-        i1 = self.item_class(name=u'Joseph', age='22')
-        i2 = self.item_class(name=u'Maria', age=[i1])
-        i3 = self.item_class(name=u'Jesus', age=[i2])
+        i1 = self.item_class(name='Joseph', age='22')
+        i2 = self.item_class(name='Maria', age=[i1])
+        i3 = self.item_class(name='Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(
             exported,
-            {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
+            {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'}
         )
         self.assertEqual(type(exported['age'][0]), dict)
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
     def test_export_item_dict_list(self):
-        i1 = self.item_class(name=u'Joseph', age='22')
-        i2 = dict(name=u'Maria', age=[i1])
-        i3 = self.item_class(name=u'Jesus', age=[i2])
+        i1 = self.item_class(name='Joseph', age='22')
+        i2 = dict(name='Maria', age=[i1])
+        i3 = self.item_class(name='Jesus', age=[i2])
         ie = self._get_exporter()
         exported = ie.export_item(i3)
         self.assertEqual(
             exported,
-            {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
+            {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'}
         )
         self.assertEqual(type(exported['age'][0]), dict)
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
     def test_export_binary(self):
         exporter = PythonItemExporter(binary=True)
-        value = self.item_class(name=u'John\xa3', age=u'22')
+        value = self.item_class(name='John\xa3', age='22')
         expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
         self.assertEqual(expected, exporter.export_item(value))
 
@@ -279,7 +279,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
         return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
 
     def _check_output(self):
-        self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')
+        self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n')
 
     def assertExportResult(self, item, expected, **kwargs):
         fp = BytesIO()
@@ -396,7 +396,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
 
     def test_multivalued_fields(self):
         self.assertExportResult(
-            self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]),
+            self.item_class(name=['John\xa3', 'Doe'], age=[1, 2, 3]),
             b"""<?xml version="1.0" encoding="utf-8"?>\n
             <items>
                 <item>
@@ -408,9 +408,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
         )
 
     def test_nested_item(self):
-        i1 = dict(name=u'foo\xa3hoo', age='22')
-        i2 = dict(name=u'bar', age=i1)
-        i3 = self.item_class(name=u'buz', age=i2)
+        i1 = dict(name='foo\xa3hoo', age='22')
+        i2 = dict(name='bar', age=i1)
+        i3 = self.item_class(name='buz', age=i2)
 
         self.assertExportResult(
             i3,
@@ -431,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
         )
 
     def test_nested_list_item(self):
-        i1 = dict(name=u'foo')
-        i2 = dict(name=u'bar', v2={"egg": ["spam"]})
-        i3 = self.item_class(name=u'buz', age=[i1, i2])
+        i1 = dict(name='foo')
+        i2 = dict(name='bar', v2={"egg": ["spam"]})
+        i3 = self.item_class(name='buz', age=[i1, i2])
 
         self.assertExportResult(
             i3,
@@ -475,7 +475,7 @@ class XmlItemExporterDataclassTest(XmlItemExporterTest):
 
 class JsonLinesItemExporterTest(BaseItemExporterTest):
 
-    _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
+    _expected_nested = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
 
     def _get_exporter(self, **kwargs):
         return JsonLinesItemExporter(self.output, **kwargs)
@@ -485,9 +485,9 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         self.assertEqual(exported, ItemAdapter(self.i).asdict())
 
     def test_nested_item(self):
-        i1 = self.item_class(name=u'Joseph', age='22')
-        i2 = dict(name=u'Maria', age=i1)
-        i3 = self.item_class(name=u'Jesus', age=i2)
+        i1 = self.item_class(name='Joseph', age='22')
+        i2 = dict(name='Maria', age=i1)
+        i3 = self.item_class(name='Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
@@ -542,25 +542,25 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
 
     def test_nested_item(self):
-        i1 = self.item_class(name=u'Joseph\xa3', age='22')
-        i2 = self.item_class(name=u'Maria', age=i1)
-        i3 = self.item_class(name=u'Jesus', age=i2)
+        i1 = self.item_class(name='Joseph\xa3', age='22')
+        i2 = self.item_class(name='Maria', age=i1)
+        i3 = self.item_class(name='Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
+        expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
         self.assertEqual(exported, [expected])
 
     def test_nested_dict_item(self):
-        i1 = dict(name=u'Joseph\xa3', age='22')
-        i2 = self.item_class(name=u'Maria', age=i1)
-        i3 = dict(name=u'Jesus', age=i2)
+        i1 = dict(name='Joseph\xa3', age='22')
+        i2 = self.item_class(name='Maria', age=i1)
+        i3 = dict(name='Jesus', age=i2)
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
         exported = json.loads(to_unicode(self.output.getvalue()))
-        expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}}
+        expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}}
         self.assertEqual(exported, [expected])
 
     def test_nonstring_types_item(self):
@@ -595,14 +595,14 @@ class CustomExporterItemTest(unittest.TestCase):
                 else:
                     return super(CustomItemExporter, self).serialize_field(field, name, value)
 
-        i = self.item_class(name=u'John', age='22')
+        i = self.item_class(name='John', age='22')
         a = ItemAdapter(i)
         ie = CustomItemExporter()
 
         self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John')
         self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23')
 
-        i2 = {'name': u'John', 'age': '22'}
+        i2 = {'name': 'John', 'age': '22'}
         self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John')
         self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23')
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c49b2e92f..b57349848 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -874,7 +874,7 @@ class FeedExportTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_encoding(self):
-        items = [dict({'foo': u'Test\xd6'})]
+        items = [dict({'foo': 'Test\xd6'})]
 
         formats = {
             'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
@@ -919,7 +919,7 @@ class FeedExportTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_multiple_configs(self):
-        items = [dict({'foo': u'FOO', 'bar': u'BAR'})]
+        items = [dict({'foo': 'FOO', 'bar': 'BAR'})]
 
         formats = {
             'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'),
@@ -1393,7 +1393,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_multiple_configs(self):
-        items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
+        items = [dict({'foo': 'FOO', 'bar': 'BAR'}), dict({'foo': 'FOO1', 'bar': 'BAR1'})]
 
         formats = {
             'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'),
@@ -1442,7 +1442,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_batch_item_count_feeds_setting(self):
-        items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})]
+        items = [dict({'foo': 'FOO'}), dict({'foo': 'FOO1'})]
         formats = {
             'json': ['[{"foo": "FOO"}]'.encode('utf-8'),
                      '[{"foo": "FOO1"}]'.encode('utf-8')],
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index cf3fc8496..64ff7a73d 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -39,19 +39,19 @@ class HeadersTest(unittest.TestCase):
         assert h.getlist('X-Forwarded-For') is not hlist
 
     def test_encode_utf8(self):
-        h = Headers({u'key': u'\xa3'}, encoding='utf-8')
+        h = Headers({'key': '\xa3'}, encoding='utf-8')
         key, val = dict(h).popitem()
         assert isinstance(key, bytes), key
         assert isinstance(val[0], bytes), val[0]
         self.assertEqual(val[0], b'\xc2\xa3')
 
     def test_encode_latin1(self):
-        h = Headers({u'key': u'\xa3'}, encoding='latin1')
+        h = Headers({'key': '\xa3'}, encoding='latin1')
         key, val = dict(h).popitem()
         self.assertEqual(val[0], b'\xa3')
 
     def test_encode_multiple(self):
-        h = Headers({u'key': [u'\xa3']}, encoding='utf-8')
+        h = Headers({'key': ['\xa3']}, encoding='utf-8')
         key, val = dict(h).popitem()
         self.assertEqual(val[0], b'\xc2\xa3')
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 63014b22d..f5cf4e798 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -60,8 +60,8 @@ class RequestTest(unittest.TestCase):
         self.assertFalse(p.headers is r.headers)
 
         # headers must not be unicode
-        h = Headers({'key1': u'val1', u'key2': 'val2'})
-        h[u'newkey'] = u'newval'
+        h = Headers({'key1': 'val1', 'key2': 'val2'})
+        h['newkey'] = 'newval'
         for k, v in h.items():
             self.assertIsInstance(k, bytes)
             for s in v:
@@ -89,30 +89,30 @@ class RequestTest(unittest.TestCase):
         self.assertEqual(r.url, "http://www.scrapy.org/blank%20space")
 
     def test_url_encoding(self):
-        r = self.request_class(url=u"http://www.scrapy.org/price/£")
+        r = self.request_class(url="http://www.scrapy.org/price/£")
         self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
 
     def test_url_encoding_other(self):
         # encoding affects only query part of URI, not path
         # path part should always be UTF-8 encoded before percent-escaping
-        r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="utf-8")
+        r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8")
         self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
 
-        r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="latin1")
+        r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1")
         self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
 
     def test_url_encoding_query(self):
-        r1 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ")
+        r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ")
         self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
 
         # should be same as above
-        r2 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")
+        r2 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")
         self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
 
     def test_url_encoding_query_latin1(self):
         # encoding is used for encoding query-string before percent-escaping;
         # path is still UTF-8 encoded before percent-escaping
-        r3 = self.request_class(url=u"http://www.scrapy.org/price/µ?currency=£", encoding="latin1")
+        r3 = self.request_class(url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1")
         self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3")
 
     def test_url_encoding_nonutf8_untouched(self):
@@ -131,16 +131,16 @@ class RequestTest(unittest.TestCase):
         # characters.  Otherwise, in the future the IRI will be mapped to
         # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different
         # URI from "http://www.example.org/r%E9sum%E9.html".
-        r1 = self.request_class(url=u"http://www.scrapy.org/price/%a3")
+        r1 = self.request_class(url="http://www.scrapy.org/price/%a3")
         self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3")
 
-        r2 = self.request_class(url=u"http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
+        r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
         self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
 
-        r3 = self.request_class(url=u"http://www.scrapy.org/résumé/%a3")
+        r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3")
         self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
 
-        r4 = self.request_class(url=u"http://www.example.org/r%E9sum%E9.html")
+        r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
         self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html")
 
     def test_body(self):
@@ -151,11 +151,11 @@ class RequestTest(unittest.TestCase):
         assert isinstance(r2.body, bytes)
         self.assertEqual(r2.encoding, 'utf-8')  # default encoding
 
-        r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8')
+        r3 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='utf-8')
         assert isinstance(r3.body, bytes)
         self.assertEqual(r3.body, b"Price: \xc2\xa3100")
 
-        r4 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='latin1')
+        r4 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='latin1')
         assert isinstance(r4.body, bytes)
         self.assertEqual(r4.body, b"Price: \xa3100")
 
@@ -164,7 +164,7 @@ class RequestTest(unittest.TestCase):
         r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
         self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")
         # unicode url
-        r = self.request_class(url=u"http://www.example.com/ajax.html#!key=value")
+        r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
         self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")
 
     def test_copy(self):
@@ -236,7 +236,7 @@ class RequestTest(unittest.TestCase):
         assert r4.dont_filter is False
 
     def test_method_always_str(self):
-        r = self.request_class("http://www.example.com", method=u"POST")
+        r = self.request_class("http://www.example.com", method="POST")
         assert isinstance(r.method, str)
 
     def test_immutable_attributes(self):
@@ -381,7 +381,7 @@ class FormRequestTest(RequestTest):
 
     def test_default_encoding_textual_data(self):
         # using default encoding (utf-8)
-        data = {u'µ one': u'two', u'price': u'£ 100'}
+        data = {'µ one': 'two', 'price': '£ 100'}
         r2 = self.request_class("http://www.example.com", formdata=data)
         self.assertEqual(r2.method, 'POST')
         self.assertEqual(r2.encoding, 'utf-8')
@@ -390,7 +390,7 @@ class FormRequestTest(RequestTest):
 
     def test_default_encoding_mixed_data(self):
         # using default encoding (utf-8)
-        data = {u'\u00b5one': b'two', b'price\xc2\xa3': u'\u00a3 100'}
+        data = {'\u00b5one': b'two', b'price\xc2\xa3': '\u00a3 100'}
         r2 = self.request_class("http://www.example.com", formdata=data)
         self.assertEqual(r2.method, 'POST')
         self.assertEqual(r2.encoding, 'utf-8')
@@ -406,14 +406,14 @@ class FormRequestTest(RequestTest):
         self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')
 
     def test_custom_encoding_textual_data(self):
-        data = {'price': u'£ 100'}
+        data = {'price': '£ 100'}
         r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
         self.assertEqual(r3.encoding, 'latin1')
         self.assertEqual(r3.body, b'price=%A3+100')
 
     def test_multi_key_values(self):
         # using multiples values for a single key
-        data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']}
+        data = {'price': '\xa3 100', 'colours': ['red', 'blue', 'green']}
         r3 = self.request_class("http://www.example.com", formdata=data)
         self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
 
@@ -450,10 +450,10 @@ class FormRequestTest(RequestTest):
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req, to_unicode=True)
-        self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
-        self.assertEqual(set(fs[u'one']), {u'two', u'three'})
-        self.assertEqual(fs[u'test2'], [u'xxx µ'])
-        self.assertEqual(fs[u'six'], [u'seven'])
+        self.assertEqual(set(fs['test £']), {'val1', 'val2'})
+        self.assertEqual(set(fs['one']), {'two', 'three'})
+        self.assertEqual(fs['test2'], ['xxx µ'])
+        self.assertEqual(fs['six'], ['seven'])
 
     def test_from_response_post_nonascii_bytes_latin1(self):
         response = _buildresponse(
@@ -471,14 +471,14 @@ class FormRequestTest(RequestTest):
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req, to_unicode=True, encoding='latin1')
-        self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
-        self.assertEqual(set(fs[u'one']), {u'two', u'three'})
-        self.assertEqual(fs[u'test2'], [u'xxx µ'])
-        self.assertEqual(fs[u'six'], [u'seven'])
+        self.assertEqual(set(fs['test £']), {'val1', 'val2'})
+        self.assertEqual(set(fs['one']), {'two', 'three'})
+        self.assertEqual(fs['test2'], ['xxx µ'])
+        self.assertEqual(fs['six'], ['seven'])
 
     def test_from_response_post_nonascii_unicode(self):
         response = _buildresponse(
-            u"""<form action="post.php" method="POST">
+            """<form action="post.php" method="POST">
             <input type="hidden" name="test £" value="val1">
             <input type="hidden" name="test £" value="val2">
             <input type="hidden" name="test2" value="xxx µ">
@@ -490,10 +490,10 @@ class FormRequestTest(RequestTest):
         self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
         self.assertEqual(req.url, "http://www.example.com/this/post.php")
         fs = _qs(req, to_unicode=True)
-        self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
-        self.assertEqual(set(fs[u'one']), {u'two', u'three'})
-        self.assertEqual(fs[u'test2'], [u'xxx µ'])
-        self.assertEqual(fs[u'six'], [u'seven'])
+        self.assertEqual(set(fs['test £']), {'val1', 'val2'})
+        self.assertEqual(set(fs['one']), {'two', 'three'})
+        self.assertEqual(fs['test2'], ['xxx µ'])
+        self.assertEqual(fs['six'], ['seven'])
 
     def test_from_response_duplicate_form_key(self):
         response = _buildresponse(
@@ -685,7 +685,7 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="two" value="clicked2">
             </form>""")
         req = self.request_class.from_response(
-            response, clickdata={u'name': u'clickable', u'value': u'clicked2'}
+            response, clickdata={'name': 'clickable', 'value': 'clicked2'}
         )
         fs = _qs(req)
         self.assertEqual(fs[b'clickable'], [b'clicked2'])
@@ -694,21 +694,21 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_unicode_clickdata(self):
         response = _buildresponse(
-            u"""<form action="get.php" method="GET">
+            """<form action="get.php" method="GET">
             <input type="submit" name="price in \u00a3" value="\u00a3 1000">
             <input type="submit" name="price in \u20ac" value="\u20ac 2000">
             <input type="hidden" name="poundsign" value="\u00a3">
             <input type="hidden" name="eurosign" value="\u20ac">
             </form>""")
         req = self.request_class.from_response(
-            response, clickdata={u'name': u'price in \u00a3'}
+            response, clickdata={'name': 'price in \u00a3'}
         )
         fs = _qs(req, to_unicode=True)
-        self.assertTrue(fs[u'price in \u00a3'])
+        self.assertTrue(fs['price in \u00a3'])
 
     def test_from_response_unicode_clickdata_latin1(self):
         response = _buildresponse(
-            u"""<form action="get.php" method="GET">
+            """<form action="get.php" method="GET">
             <input type="submit" name="price in \u00a3" value="\u00a3 1000">
             <input type="submit" name="price in \u00a5" value="\u00a5 2000">
             <input type="hidden" name="poundsign" value="\u00a3">
@@ -716,10 +716,10 @@ class FormRequestTest(RequestTest):
             </form>""",
             encoding='latin1')
         req = self.request_class.from_response(
-            response, clickdata={u'name': u'price in \u00a5'}
+            response, clickdata={'name': 'price in \u00a5'}
         )
         fs = _qs(req, to_unicode=True, encoding='latin1')
-        self.assertTrue(fs[u'price in \u00a5'])
+        self.assertTrue(fs['price in \u00a5'])
 
     def test_from_response_multiple_forms_clickdata(self):
         response = _buildresponse(
@@ -733,7 +733,7 @@ class FormRequestTest(RequestTest):
             </form>
             """)
         req = self.request_class.from_response(
-            response, formname='form2', clickdata={u'name': u'clickable'}
+            response, formname='form2', clickdata={'name': 'clickable'}
         )
         fs = _qs(req)
         self.assertEqual(fs[b'clickable'], [b'clicked2'])
@@ -1072,11 +1072,11 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_unicode_xpath(self):
         response = _buildresponse(b'<form name="\xd1\x8a"></form>')
-        r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']")
+        r = self.request_class.from_response(response, formxpath="//form[@name='\u044a']")
         fs = _qs(r)
         self.assertEqual(fs, {})
 
-        xpath = u"//form[@name='\u03b1']"
+        xpath = "//form[@name='\u03b1']"
         self.assertRaisesRegex(ValueError, re.escape(xpath),
                                self.request_class.from_response,
                                response, formxpath=xpath)
@@ -1246,13 +1246,13 @@ class XmlRpcRequestTest(RequestTest):
         self._test_request(params=('value',))
         self._test_request(params=('username', 'password'), methodname='login')
         self._test_request(params=('response', ), methodresponse='login')
-        self._test_request(params=(u'pas£',), encoding='utf-8')
+        self._test_request(params=('pas£',), encoding='utf-8')
         self._test_request(params=(None,), allow_none=1)
         self.assertRaises(TypeError, self._test_request)
         self.assertRaises(TypeError, self._test_request, params=(None,))
 
     def test_latin1(self):
-        self._test_request(params=(u'pas£',), encoding='latin1')
+        self._test_request(params=('pas£',), encoding='latin1')
 
 
 class JsonRequestTest(RequestTest):
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index e0ca3c0e6..56d017de6 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -318,28 +318,28 @@ class TextResponseTest(BaseResponseTest):
 
     def test_unicode_url(self):
         # instantiate with unicode url without encoding (should set default encoding)
-        resp = self.response_class(u"http://www.example.com/")
+        resp = self.response_class("http://www.example.com/")
         self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING)
 
         # make sure urls are converted to str
-        resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8')
+        resp = self.response_class(url="http://www.example.com/", encoding='utf-8')
         assert isinstance(resp.url, str)
 
-        resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')
+        resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='utf-8')
         self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
-        resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
+        resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='latin-1')
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
-        resp = self.response_class(u"http://www.example.com/price/\xa3",
+        resp = self.response_class("http://www.example.com/price/\xa3",
                                    headers={"Content-type": ["text/html; charset=utf-8"]})
         self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
-        resp = self.response_class(u"http://www.example.com/price/\xa3",
+        resp = self.response_class("http://www.example.com/price/\xa3",
                                    headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
 
     def test_unicode_body(self):
         unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 '
                           '\u0442\u0435\u043a\u0441\u0442')
-        self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body')
+        self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body='unicode body')
 
         original_string = unicode_string.encode('cp1251')
         r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
@@ -355,7 +355,7 @@ class TextResponseTest(BaseResponseTest):
     def test_encoding(self):
         r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3",
                                  headers={"Content-type": ["text/html; charset=utf-8"]})
-        r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
+        r2 = self.response_class("http://www.example.com", encoding='utf-8', body="\xa3")
         r3 = self.response_class("http://www.example.com", body=b"\xa3",
                                  headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3")
@@ -376,14 +376,14 @@ class TextResponseTest(BaseResponseTest):
         self.assertEqual(r5._headers_encoding(), None)
         self._assert_response_encoding(r5, "utf-8")
         assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii'
-        self._assert_response_values(r1, 'utf-8', u"\xa3")
-        self._assert_response_values(r2, 'utf-8', u"\xa3")
-        self._assert_response_values(r3, 'iso-8859-1', u"\xa3")
-        self._assert_response_values(r6, 'gb18030', u"\u2015")
-        self._assert_response_values(r7, 'gb18030', u"\u2015")
+        self._assert_response_values(r1, 'utf-8', "\xa3")
+        self._assert_response_values(r2, 'utf-8', "\xa3")
+        self._assert_response_values(r3, 'iso-8859-1', "\xa3")
+        self._assert_response_values(r6, 'gb18030', "\u2015")
+        self._assert_response_values(r7, 'gb18030', "\u2015")
 
         # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
-        self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3")
+        self.assertRaises(TypeError, self.response_class, "http://www.example.com", body="\xa3")
 
     def test_declared_encoding_invalid(self):
         """Check that unknown declared encodings are ignored"""
@@ -391,14 +391,14 @@ class TextResponseTest(BaseResponseTest):
                                 headers={"Content-type": ["text/html; charset=UKNOWN"]},
                                 body=b"\xc2\xa3")
         self.assertEqual(r._declared_encoding(), None)
-        self._assert_response_values(r, 'utf-8', u"\xa3")
+        self._assert_response_values(r, 'utf-8', "\xa3")
 
     def test_utf16(self):
         """Test utf-16 because UnicodeDammit is known to have problems with"""
         r = self.response_class("http://www.example.com",
                                 body=b'\xff\xfeh\x00i\x00',
                                 encoding='utf-16')
-        self._assert_response_values(r, 'utf-16', u"hi")
+        self._assert_response_values(r, 'utf-16', "hi")
 
     def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self):
         r6 = self.response_class("http://www.example.com",
@@ -406,8 +406,8 @@ class TextResponseTest(BaseResponseTest):
                                  body=b"\xef\xbb\xbfWORD\xe3\xab")
         self.assertEqual(r6.encoding, 'utf-8')
         self.assertIn(r6.text, {
-            u'WORD\ufffd\ufffd',  # w3lib < 1.19.0
-            u'WORD\ufffd',        # w3lib >= 1.19.0
+            'WORD\ufffd\ufffd',  # w3lib < 1.19.0
+            'WORD\ufffd',        # w3lib >= 1.19.0
         })
 
     def test_bom_is_removed_from_body(self):
@@ -422,9 +422,9 @@ class TextResponseTest(BaseResponseTest):
         # Test response without content-type and BOM encoding
         response = self.response_class(url, body=body)
         self.assertEqual(response.encoding, 'utf-8')
-        self.assertEqual(response.text, u'WORD')
+        self.assertEqual(response.text, 'WORD')
         response = self.response_class(url, body=body)
-        self.assertEqual(response.text, u'WORD')
+        self.assertEqual(response.text, 'WORD')
         self.assertEqual(response.encoding, 'utf-8')
 
         # Body caching sideeffect isn't triggered when encoding is declared in
@@ -432,28 +432,28 @@ class TextResponseTest(BaseResponseTest):
         # body
         response = self.response_class(url, headers=headers, body=body)
         self.assertEqual(response.encoding, 'utf-8')
-        self.assertEqual(response.text, u'WORD')
+        self.assertEqual(response.text, 'WORD')
         response = self.response_class(url, headers=headers, body=body)
-        self.assertEqual(response.text, u'WORD')
+        self.assertEqual(response.text, 'WORD')
         self.assertEqual(response.encoding, 'utf-8')
 
     def test_replace_wrong_encoding(self):
         """Test invalid chars are replaced properly"""
         r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX')
         # XXX: Policy for replacing invalid chars may suffer minor variations
-        # but it should always contain the unicode replacement char (u'\ufffd')
-        assert u'\ufffd' in r.text, repr(r.text)
-        assert u'PREFIX' in r.text, repr(r.text)
-        assert u'SUFFIX' in r.text, repr(r.text)
+        # but it should always contain the unicode replacement char ('\ufffd')
+        assert '\ufffd' in r.text, repr(r.text)
+        assert 'PREFIX' in r.text, repr(r.text)
+        assert 'SUFFIX' in r.text, repr(r.text)
 
         # Do not destroy html tags due to encoding bugs
         r = self.response_class("http://example.com", encoding='utf-8',
                                 body=b'\xf0<span>value</span>')
-        assert u'<span>value</span>' in r.text, repr(r.text)
+        assert '<span>value</span>' in r.text, repr(r.text)
 
         # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
         # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
-        # assert u'\ufffd' in r.text, repr(r.text)
+        # assert '\ufffd' in r.text, repr(r.text)
 
     def test_selector(self):
         body = b"<html><head><title>Some page</title><body></body></html>"
@@ -466,15 +466,15 @@ class TextResponseTest(BaseResponseTest):
 
         self.assertEqual(
             response.selector.xpath("//title/text()").getall(),
-            [u'Some page']
+            ['Some page']
         )
         self.assertEqual(
             response.selector.css("title::text").getall(),
-            [u'Some page']
+            ['Some page']
         )
         self.assertEqual(
             response.selector.re("Some (.*)</title>"),
-            [u'page']
+            ['page']
         )
 
     def test_selector_shortcuts(self):
@@ -595,7 +595,7 @@ class TextResponseTest(BaseResponseTest):
         resp1 = self.response_class(
             'http://example.com',
             encoding='utf8',
-            body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
+            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
         )
         req = self._assert_followed_url(
             resp1.css('a')[0],
@@ -607,7 +607,7 @@ class TextResponseTest(BaseResponseTest):
         resp2 = self.response_class(
             'http://example.com',
             encoding='cp1251',
-            body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
+            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
         )
         req = self._assert_followed_url(
             resp2.css('a')[0],
@@ -681,8 +681,8 @@ class TextResponseTest(BaseResponseTest):
 
     def test_body_as_unicode_deprecation_warning(self):
         with catch_warnings(record=True) as warnings:
-            r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8')
-            self.assertEqual(r1.body_as_unicode(), u'Hello')
+            r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8')
+            self.assertEqual(r1.body_as_unicode(), 'Hello')
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 
@@ -787,7 +787,7 @@ class XmlResponseTest(TextResponseTest):
 
         self.assertEqual(
             response.selector.xpath("//elem/text()").getall(),
-            [u'value']
+            ['value']
         )
 
     def test_selector_shortcuts(self):
diff --git a/tests/test_item.py b/tests/test_item.py
index 60468971c..0ce78f8c0 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -20,8 +20,8 @@ class ItemTest(unittest.TestCase):
             name = Field()
 
         i = TestItem()
-        i['name'] = u'name'
-        self.assertEqual(i['name'], u'name')
+        i['name'] = 'name'
+        self.assertEqual(i['name'], 'name')
 
     def test_init(self):
         class TestItem(Item):
@@ -30,17 +30,17 @@ class ItemTest(unittest.TestCase):
         i = TestItem()
         self.assertRaises(KeyError, i.__getitem__, 'name')
 
-        i2 = TestItem(name=u'john doe')
-        self.assertEqual(i2['name'], u'john doe')
+        i2 = TestItem(name='john doe')
+        self.assertEqual(i2['name'], 'john doe')
 
-        i3 = TestItem({'name': u'john doe'})
-        self.assertEqual(i3['name'], u'john doe')
+        i3 = TestItem({'name': 'john doe'})
+        self.assertEqual(i3['name'], 'john doe')
 
         i4 = TestItem(i3)
-        self.assertEqual(i4['name'], u'john doe')
+        self.assertEqual(i4['name'], 'john doe')
 
-        self.assertRaises(KeyError, TestItem, {'name': u'john doe',
-                                               'other': u'foo'})
+        self.assertRaises(KeyError, TestItem, {'name': 'john doe',
+                                               'other': 'foo'})
 
     def test_invalid_field(self):
         class TestItem(Item):
@@ -56,7 +56,7 @@ class ItemTest(unittest.TestCase):
             number = Field()
 
         i = TestItem()
-        i['name'] = u'John Doe'
+        i['name'] = 'John Doe'
         i['number'] = 123
         itemrepr = repr(i)
 
@@ -101,9 +101,9 @@ class ItemTest(unittest.TestCase):
 
         i = TestItem()
         self.assertRaises(KeyError, i.get_name)
-        i['name'] = u'lala'
-        self.assertEqual(i.get_name(), u'lala')
-        i.change_name(u'other')
+        i['name'] = 'lala'
+        self.assertEqual(i.get_name(), 'lala')
+        i.change_name('other')
         self.assertEqual(i.get_name(), 'other')
 
     def test_metaclass(self):
@@ -113,22 +113,22 @@ class ItemTest(unittest.TestCase):
             values = Field()
 
         i = TestItem()
-        i['name'] = u'John'
+        i['name'] = 'John'
         self.assertEqual(list(i.keys()), ['name'])
         self.assertEqual(list(i.values()), ['John'])
 
-        i['keys'] = u'Keys'
-        i['values'] = u'Values'
+        i['keys'] = 'Keys'
+        i['values'] = 'Values'
         self.assertSortedEqual(list(i.keys()), ['keys', 'values', 'name'])
-        self.assertSortedEqual(list(i.values()), [u'Keys', u'Values', u'John'])
+        self.assertSortedEqual(list(i.values()), ['Keys', 'Values', 'John'])
 
     def test_metaclass_with_fields_attribute(self):
         class TestItem(Item):
             fields = {'new': Field(default='X')}
 
-        item = TestItem(new=u'New')
+        item = TestItem(new='New')
         self.assertSortedEqual(list(item.keys()), ['new'])
-        self.assertSortedEqual(list(item.values()), [u'New'])
+        self.assertSortedEqual(list(item.values()), ['New'])
 
     def test_metaclass_inheritance(self):
         class ParentItem(Item):
@@ -238,8 +238,8 @@ class ItemTest(unittest.TestCase):
             name = Field()
 
         i = TestItem()
-        i['name'] = u'John'
-        self.assertEqual(dict(i), {'name': u'John'})
+        i['name'] = 'John'
+        self.assertEqual(dict(i), {'name': 'John'})
 
     def test_copy(self):
         class TestItem(Item):
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 8d4538eed..a0bafa5e5 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -31,31 +31,31 @@ class Base:
             page4_url = 'http://example.com/page%204.html'
 
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
-                Link(url='http://www.google.com/something', text=u''),
-                Link(url='http://example.com/innertag.html', text=u'inner tag'),
-                Link(url=page4_url, text=u'href with whitespaces'),
+                Link(url='http://www.google.com/something', text=''),
+                Link(url='http://example.com/innertag.html', text='inner tag'),
+                Link(url=page4_url, text='href with whitespaces'),
             ])
 
         def test_extract_filter_allow(self):
             lx = self.extractor_cls(allow=('sample', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
             ])
 
         def test_extract_filter_allow_with_duplicates(self):
             lx = self.extractor_cls(allow=('sample', ), unique=False)
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
+                Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
             ])
 
@@ -63,10 +63,10 @@ class Base:
             lx = self.extractor_cls(allow=('sample', ), unique=False,
                                     canonicalize=True)
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
+                Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
                 Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment')
             ])
 
@@ -74,22 +74,22 @@ class Base:
             lx = self.extractor_cls(allow=('sample',), unique=True,
                                     canonicalize=True)
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
             ])
 
         def test_extract_filter_allow_and_deny(self):
             lx = self.extractor_cls(allow=('sample', ), deny=('3', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
             ])
 
         def test_extract_filter_allowed_domains(self):
             lx = self.extractor_cls(allow_domains=('google.com', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://www.google.com/something', text=u''),
+                Link(url='http://www.google.com/something', text=''),
             ])
 
         def test_extraction_using_single_values(self):
@@ -97,27 +97,27 @@ class Base:
 
             lx = self.extractor_cls(allow='sample')
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html#foo',
                      text='sample 3 repetition with fragment')
             ])
 
             lx = self.extractor_cls(allow='sample', deny='3')
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
             ])
 
             lx = self.extractor_cls(allow_domains='google.com')
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://www.google.com/something', text=u''),
+                Link(url='http://www.google.com/something', text=''),
             ])
 
             lx = self.extractor_cls(deny_domains='example.com')
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://www.google.com/something', text=u''),
+                Link(url='http://www.google.com/something', text=''),
             ])
 
         def test_nofollow(self):
@@ -145,11 +145,11 @@ class Base:
 
             lx = self.extractor_cls()
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.org/about.html', text=u'About us'),
-                Link(url='http://example.org/follow.html', text=u'Follow this link'),
-                Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True),
-                Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'),
-                Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                Link(url='http://example.org/about.html', text='About us'),
+                Link(url='http://example.org/follow.html', text='Follow this link'),
+                Link(url='http://example.org/nofollow.html', text='Dont follow this one', nofollow=True),
+                Link(url='http://example.org/nofollow2.html', text='Choose to follow or not'),
+                Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
             ])
 
         def test_matches(self):
@@ -183,8 +183,8 @@ class Base:
         def test_restrict_xpaths(self):
             lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
             ])
 
         def test_restrict_xpaths_encoding(self):
@@ -202,14 +202,14 @@ class Base:
 
             lx = self.extractor_cls(restrict_xpaths="//div[@class='links']")
             self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.org/about.html', text=u'About us\xa3')])
+                             [Link(url='http://example.org/about.html', text='About us\xa3')])
 
         def test_restrict_xpaths_with_html_entities(self):
             html = b'<html><body><p><a href="/&hearts;/you?c=&euro;">text</a></p></body></html>'
             response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
             links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
             self.assertEqual(links,
-                             [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')])
+                             [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text='text')])
 
         def test_restrict_xpaths_concat_in_handle_data(self):
             """html entities cause SGMLParser to call handle_data hook twice"""
@@ -217,22 +217,22 @@ class Base:
             response = HtmlResponse("http://example.org", body=body, encoding='gb18030')
             lx = self.extractor_cls(restrict_xpaths="//div")
             self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
+                             [Link(url='http://example.org/foo', text='>\u4eac<\u4e1c',
                                    fragment='', nofollow=False)])
 
         def test_restrict_css(self):
             lx = self.extractor_cls(restrict_css=('#subwrapper a',))
             self.assertEqual(lx.extract_links(self.response), [
-                Link(url='http://example.com/sample2.html', text=u'sample 2')
+                Link(url='http://example.com/sample2.html', text='sample 2')
             ])
 
         def test_restrict_css_and_restrict_xpaths_together(self):
             lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ),
                                     restrict_css=('#subwrapper + a', ))
             self.assertEqual([link for link in lx.extract_links(self.response)], [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
             ])
 
         def test_area_tag_with_unicode_present(self):
@@ -243,7 +243,7 @@ class Base:
             lx.extract_links(response)
             lx.extract_links(response)
             self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.org/foo', text=u'',
+                             [Link(url='http://example.org/foo', text='',
                                    fragment='', nofollow=False)])
 
         def test_encoded_url(self):
@@ -251,7 +251,7 @@ class Base:
             response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
             lx = self.extractor_cls()
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
+                Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
             ])
 
         def test_encoded_url_in_restricted_xpath(self):
@@ -259,7 +259,7 @@ class Base:
             response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
             lx = self.extractor_cls(restrict_xpaths="//div")
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
+                Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
             ])
 
         def test_ignored_extensions(self):
@@ -268,7 +268,7 @@ class Base:
             response = HtmlResponse("http://example.org/", body=html)
             lx = self.extractor_cls()
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.org/page.html', text=u'asd'),
+                Link(url='http://example.org/page.html', text='asd'),
             ])
 
             # override denied extensions
@@ -308,25 +308,25 @@ class Base:
             page4_url = 'http://example.com/page%204.html'
 
             self.assertEqual(lx.extract_links(self.response), [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
-                Link(url='http://www.google.com/something', text=u''),
-                Link(url='http://example.com/innertag.html', text=u'inner tag'),
-                Link(url=page4_url, text=u'href with whitespaces'),
+                Link(url='http://www.google.com/something', text=''),
+                Link(url='http://example.com/innertag.html', text='inner tag'),
+                Link(url=page4_url, text='href with whitespaces'),
             ])
 
             lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=())
             self.assertEqual(lx.extract_links(self.response), [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample2.jpg', text=u''),
-                Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample2.jpg', text=''),
+                Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
-                Link(url='http://www.google.com/something', text=u''),
-                Link(url='http://example.com/innertag.html', text=u'inner tag'),
-                Link(url=page4_url, text=u'href with whitespaces'),
+                Link(url='http://www.google.com/something', text=''),
+                Link(url='http://example.com/innertag.html', text='inner tag'),
+                Link(url=page4_url, text='href with whitespaces'),
             ])
 
             lx = self.extractor_cls(attrs=None)
@@ -344,24 +344,24 @@ class Base:
 
             lx = self.extractor_cls()
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample1.html', text=u''),
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample1.html', text=''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
             ])
 
             lx = self.extractor_cls(tags="area")
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample1.html', text=u''),
+                Link(url='http://example.com/sample1.html', text=''),
             ])
 
             lx = self.extractor_cls(tags="a")
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
             ])
 
             lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=())
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                Link(url='http://example.com/sample2.jpg', text=u''),
+                Link(url='http://example.com/sample2.html', text='sample 2'),
+                Link(url='http://example.com/sample2.jpg', text=''),
             ])
 
         def test_tags_attrs(self):
@@ -375,14 +375,14 @@ class Base:
 
             lx = self.extractor_cls(tags='div', attrs='data-url')
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
-                Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
+                Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
+                Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
             ])
 
             lx = self.extractor_cls(tags=('div',), attrs=('data-url',))
             self.assertEqual(lx.extract_links(response), [
-                Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
-                Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
+                Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
+                Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
             ])
 
         def test_xhtml(self):
@@ -420,13 +420,13 @@ class Base:
             self.assertEqual(
                 lx.extract_links(response),
                 [
-                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
+                    Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text='Dont follow this one',
                          fragment='', nofollow=True),
-                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
+                    Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
                          fragment='', nofollow=False),
-                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                    Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
                 ]
             )
 
@@ -436,13 +436,13 @@ class Base:
             self.assertEqual(
                 lx.extract_links(response),
                 [
-                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
+                    Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text='Dont follow this one',
                          fragment='', nofollow=True),
-                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
+                    Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
                          fragment='', nofollow=False),
-                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                    Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
                 ]
             )
 
@@ -455,8 +455,8 @@ class Base:
             response = HtmlResponse("http://example.org/index.html", body=html)
             lx = self.extractor_cls()
             self.assertEqual([link for link in lx.extract_links(response)], [
-                Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-                Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
+                Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
+                Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
             ])
 
         def test_ftp_links(self):
@@ -467,7 +467,7 @@ class Base:
             response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8')
             lx = self.extractor_cls()
             self.assertEqual(lx.extract_links(response), [
-                Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False),
+                Link(url='ftp://www.external.com/', text='An Item', fragment='', nofollow=False),
             ])
 
         def test_pickle_extractor(self):
@@ -487,8 +487,8 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
         response = HtmlResponse("http://example.org/index.html", body=html)
         lx = self.extractor_cls()
         self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
+            Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
+            Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
         ])
 
     def test_link_restrict_text(self):
@@ -501,18 +501,18 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
         # Simple text inclusion test
         lx = self.extractor_cls(restrict_text='dog')
         self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+            Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
         ])
         # Unique regex test
         lx = self.extractor_cls(restrict_text=r'of.*dog')
         self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+            Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
         ])
         # Multiple regex test
         lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat'])
         self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False),
-            Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+            Link(url='http://example.org/item1.html', text='Pic of a cat', nofollow=False),
+            Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
         ])
 
     def test_restrict_xpaths_with_html_entities(self):
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 581183625..2ed6f365f 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -69,23 +69,23 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_add_value_on_unknown_field(self):
         il = TestItemLoader()
-        self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo'])
+        self.assertRaises(KeyError, il.add_value, 'wrong_field', ['lala', 'lolo'])
 
     def test_load_item_using_default_loader(self):
         i = TestItem()
-        i['summary'] = u'lala'
+        i['summary'] = 'lala'
         il = ItemLoader(item=i)
-        il.add_value('name', u'marta')
+        il.add_value('name', 'marta')
         item = il.load_item()
         assert item is i
-        self.assertEqual(item['summary'], [u'lala'])
-        self.assertEqual(item['name'], [u'marta'])
+        self.assertEqual(item['summary'], ['lala'])
+        self.assertEqual(item['name'], ['marta'])
 
     def test_load_item_using_custom_loader(self):
         il = TestItemLoader()
-        il.add_value('name', u'marta')
+        il.add_value('name', 'marta')
         item = il.load_item()
-        self.assertEqual(item['name'], [u'Marta'])
+        self.assertEqual(item['name'], ['Marta'])
 
 
 class InitializationTestMixin:
@@ -290,137 +290,137 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertRaises(RuntimeError, l.get_css, '#name::text')
 
     def test_init_method_with_selector(self):
-        sel = Selector(text=u"<html><body><div>marta</div></body></html>")
+        sel = Selector(text="<html><body><div>marta</div></body></html>")
         l = TestItemLoader(selector=sel)
         self.assertIs(l.selector, sel)
 
         l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
 
     def test_init_method_with_selector_css(self):
-        sel = Selector(text=u"<html><body><div>marta</div></body></html>")
+        sel = Selector(text="<html><body><div>marta</div></body></html>")
         l = TestItemLoader(selector=sel)
         self.assertIs(l.selector, sel)
 
         l.add_css('name', 'div::text')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
 
     def test_init_method_with_response(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
 
         l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
 
     def test_init_method_with_response_css(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
 
         l.add_css('name', 'div::text')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
 
         l.add_css('url', 'a::attr(href)')
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
 
         # combining/accumulating CSS selectors and XPath expressions
         l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Marta', u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta', 'Marta'])
 
         l.add_xpath('url', '//img/@src')
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org', u'/images/logo.png'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org', '/images/logo.png'])
 
     def test_add_xpath_re(self):
         l = TestItemLoader(response=self.response)
         l.add_xpath('name', '//div/text()', re='ma')
-        self.assertEqual(l.get_output_value('name'), [u'Ma'])
+        self.assertEqual(l.get_output_value('name'), ['Ma'])
 
     def test_replace_xpath(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
         l.replace_xpath('name', '//p/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
 
         l.replace_xpath('name', ['//p/text()', '//div/text()'])
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
 
     def test_get_xpath(self):
         l = TestItemLoader(response=self.response)
-        self.assertEqual(l.get_xpath('//p/text()'), [u'paragraph'])
-        self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), u'paragraph')
-        self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), u'pa')
+        self.assertEqual(l.get_xpath('//p/text()'), ['paragraph'])
+        self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), 'paragraph')
+        self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), 'pa')
 
-        self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), [u'paragraph', 'marta'])
+        self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), ['paragraph', 'marta'])
 
     def test_replace_xpath_multi_fields(self):
         l = TestItemLoader(response=self.response)
         l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
         l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
 
     def test_replace_xpath_re(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_xpath('name', '//div/text()')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
         l.replace_xpath('name', '//div/text()', re='ma')
-        self.assertEqual(l.get_output_value('name'), [u'Ma'])
+        self.assertEqual(l.get_output_value('name'), ['Ma'])
 
     def test_add_css_re(self):
         l = TestItemLoader(response=self.response)
         l.add_css('name', 'div::text', re='ma')
-        self.assertEqual(l.get_output_value('name'), [u'Ma'])
+        self.assertEqual(l.get_output_value('name'), ['Ma'])
 
         l.add_css('url', 'a::attr(href)', re='http://(.+)')
-        self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
 
     def test_replace_css(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_css('name', 'div::text')
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
         l.replace_css('name', 'p::text')
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
 
         l.replace_css('name', ['p::text', 'div::text'])
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
 
         l.add_css('url', 'a::attr(href)', re='http://(.+)')
-        self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
         l.replace_css('url', 'img::attr(src)')
-        self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
+        self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
 
     def test_get_css(self):
         l = TestItemLoader(response=self.response)
-        self.assertEqual(l.get_css('p::text'), [u'paragraph'])
-        self.assertEqual(l.get_css('p::text', TakeFirst()), u'paragraph')
-        self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), u'pa')
+        self.assertEqual(l.get_css('p::text'), ['paragraph'])
+        self.assertEqual(l.get_css('p::text', TakeFirst()), 'paragraph')
+        self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), 'pa')
 
-        self.assertEqual(l.get_css(['p::text', 'div::text']), [u'paragraph', 'marta'])
+        self.assertEqual(l.get_css(['p::text', 'div::text']), ['paragraph', 'marta'])
         self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']),
-                         [u'http://www.scrapy.org', u'/images/logo.png'])
+                         ['http://www.scrapy.org', '/images/logo.png'])
 
     def test_replace_css_multi_fields(self):
         l = TestItemLoader(response=self.response)
         l.add_css(None, 'div::text', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), [u'Marta'])
+        self.assertEqual(l.get_output_value('name'), ['Marta'])
         l.replace_css(None, 'p::text', TakeFirst(), lambda x: {'name': x})
-        self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+        self.assertEqual(l.get_output_value('name'), ['Paragraph'])
 
         l.add_css(None, 'a::attr(href)', TakeFirst(), lambda x: {'url': x})
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
         l.replace_css(None, 'img::attr(src)', TakeFirst(), lambda x: {'url': x})
-        self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
+        self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
 
     def test_replace_css_re(self):
         l = TestItemLoader(response=self.response)
         self.assertTrue(l.selector)
         l.add_css('url', 'a::attr(href)')
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
         l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)')
-        self.assertEqual(l.get_output_value('url'), [u'scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['scrapy.org'])
 
 
 class SubselectorLoaderTest(unittest.TestCase):
@@ -447,9 +447,9 @@ class SubselectorLoaderTest(unittest.TestCase):
         nl.add_css('name_div', '#id')
         nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
 
-        self.assertEqual(l.get_output_value('name'), [u'marta'])
-        self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
-        self.assertEqual(l.get_output_value('name_value'), [u'marta'])
+        self.assertEqual(l.get_output_value('name'), ['marta'])
+        self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])
+        self.assertEqual(l.get_output_value('name_value'), ['marta'])
 
         self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
         self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
@@ -462,9 +462,9 @@ class SubselectorLoaderTest(unittest.TestCase):
         nl.add_css('name_div', '#id')
         nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
 
-        self.assertEqual(l.get_output_value('name'), [u'marta'])
-        self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
-        self.assertEqual(l.get_output_value('name_value'), [u'marta'])
+        self.assertEqual(l.get_output_value('name'), ['marta'])
+        self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])
+        self.assertEqual(l.get_output_value('name_value'), ['marta'])
 
         self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
         self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
@@ -476,11 +476,11 @@ class SubselectorLoaderTest(unittest.TestCase):
         nl2 = nl1.nested_xpath('a')
 
         l.add_xpath('url', '//footer/a/@href')
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
         nl1.replace_xpath('url', 'img/@src')
-        self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
+        self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
         nl2.replace_xpath('url', '@href')
-        self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+        self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
 
     def test_nested_ordering(self):
         l = NestedItemLoader(response=self.response)
@@ -493,10 +493,10 @@ class SubselectorLoaderTest(unittest.TestCase):
         l.add_xpath('url', '//footer/a/@href')
 
         self.assertEqual(l.get_output_value('url'), [
-            u'/images/logo.png',
-            u'http://www.scrapy.org',
-            u'homepage',
-            u'http://www.scrapy.org',
+            '/images/logo.png',
+            'http://www.scrapy.org',
+            'homepage',
+            'http://www.scrapy.org',
         ])
 
     def test_nested_load_item(self):
@@ -514,9 +514,9 @@ class SubselectorLoaderTest(unittest.TestCase):
         assert item is nl1.item
         assert item is nl2.item
 
-        self.assertEqual(item['name'], [u'marta'])
-        self.assertEqual(item['url'], [u'http://www.scrapy.org'])
-        self.assertEqual(item['image'], [u'/images/logo.png'])
+        self.assertEqual(item['name'], ['marta'])
+        self.assertEqual(item['url'], ['http://www.scrapy.org'])
+        self.assertEqual(item['image'], ['/images/logo.png'])
 
 
 # Functions as processors
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index d0a59e8cd..eb14de14f 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -51,19 +51,19 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_load_item_using_default_loader(self):
         i = TestItem()
-        i['summary'] = u'lala'
+        i['summary'] = 'lala'
         il = ItemLoader(item=i)
-        il.add_value('name', u'marta')
+        il.add_value('name', 'marta')
         item = il.load_item()
         assert item is i
-        self.assertEqual(item['summary'], [u'lala'])
-        self.assertEqual(item['name'], [u'marta'])
+        self.assertEqual(item['summary'], ['lala'])
+        self.assertEqual(item['name'], ['marta'])
 
     def test_load_item_using_custom_loader(self):
         il = TestItemLoader()
-        il.add_value('name', u'marta')
+        il.add_value('name', 'marta')
         item = il.load_item()
-        self.assertEqual(item['name'], [u'Marta'])
+        self.assertEqual(item['name'], ['Marta'])
 
     def test_load_item_ignore_none_field_values(self):
         def validate_sku(value):
@@ -76,23 +76,23 @@ class BasicItemLoaderTest(unittest.TestCase):
             price_out = Compose(TakeFirst(), float)
             sku_out = Compose(TakeFirst(), validate_sku)
 
-        valid_fragment = u'SKU: 1234'
-        invalid_fragment = u'SKU: not available'
+        valid_fragment = 'SKU: 1234'
+        invalid_fragment = 'SKU: not available'
         sku_re = 'SKU: (.+)'
 
         il = MyLoader(item={})
         # Should not return "sku: None".
         il.add_value('sku', [invalid_fragment], re=sku_re)
         # Should not ignore empty values.
-        il.add_value('name', u'')
-        il.add_value('price', [u'0'])
+        il.add_value('name', '')
+        il.add_value('price', ['0'])
         self.assertEqual(il.load_item(), {
-            'name': u'',
+            'name': '',
             'price': 0.0,
         })
 
         il.replace_value('sku', [valid_fragment], re=sku_re)
-        self.assertEqual(il.load_item()['sku'], u'1234')
+        self.assertEqual(il.load_item()['sku'], '1234')
 
     def test_self_referencing_loader(self):
         class MyLoader(ItemLoader):
@@ -117,19 +117,19 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_add_value(self):
         il = TestItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-        il.add_value('name', u'pepe')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_collected_values('name'), ['Marta'])
+        self.assertEqual(il.get_output_value('name'), ['Marta'])
+        il.add_value('name', 'pepe')
+        self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe'])
+        self.assertEqual(il.get_output_value('name'), ['Marta', 'Pepe'])
 
         # test add object value
         il.add_value('summary', {'key': 1})
         self.assertEqual(il.get_collected_values('summary'), [{'key': 1}])
 
-        il.add_value(None, u'Jim', lambda x: {'name': x})
-        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim'])
+        il.add_value(None, 'Jim', lambda x: {'name': x})
+        self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe', 'Jim'])
 
     def test_add_zero(self):
         il = NameItemLoader()
@@ -138,49 +138,49 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_replace_value(self):
         il = TestItemLoader()
-        il.replace_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-        il.replace_value('name', u'pepe')
-        self.assertEqual(il.get_collected_values('name'), [u'Pepe'])
-        self.assertEqual(il.get_output_value('name'), [u'Pepe'])
+        il.replace_value('name', 'marta')
+        self.assertEqual(il.get_collected_values('name'), ['Marta'])
+        self.assertEqual(il.get_output_value('name'), ['Marta'])
+        il.replace_value('name', 'pepe')
+        self.assertEqual(il.get_collected_values('name'), ['Pepe'])
+        self.assertEqual(il.get_output_value('name'), ['Pepe'])
 
-        il.replace_value(None, u'Jim', lambda x: {'name': x})
-        self.assertEqual(il.get_collected_values('name'), [u'Jim'])
+        il.replace_value(None, 'Jim', lambda x: {'name': x})
+        self.assertEqual(il.get_collected_values('name'), ['Jim'])
 
     def test_get_value(self):
         il = NameItemLoader()
-        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
-        self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
-        self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
+        self.assertEqual('FOO', il.get_value(['foo', 'bar'], TakeFirst(), str.upper))
+        self.assertEqual(['foo', 'bar'], il.get_value(['name:foo', 'name:bar'], re='name:(.*)$'))
+        self.assertEqual('foo', il.get_value(['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$'))
 
-        il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')
-        self.assertEqual([u'foo'], il.get_collected_values('name'))
-        il.replace_value('name', u'name:bar', re=u'name:(.*)$')
-        self.assertEqual([u'bar'], il.get_collected_values('name'))
+        il.add_value('name', ['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$')
+        self.assertEqual(['foo'], il.get_collected_values('name'))
+        il.replace_value('name', 'name:bar', re='name:(.*)$')
+        self.assertEqual(['bar'], il.get_collected_values('name'))
 
     def test_iter_on_input_processor_input(self):
         class NameFirstItemLoader(NameItemLoader):
             name_in = TakeFirst()
 
         il = NameFirstItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_collected_values('name'), ['marta'])
         il = NameFirstItemLoader()
-        il.add_value('name', [u'marta', u'jose'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il.add_value('name', ['marta', 'jose'])
+        self.assertEqual(il.get_collected_values('name'), ['marta'])
 
         il = NameFirstItemLoader()
-        il.replace_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il.replace_value('name', 'marta')
+        self.assertEqual(il.get_collected_values('name'), ['marta'])
         il = NameFirstItemLoader()
-        il.replace_value('name', [u'marta', u'jose'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il.replace_value('name', ['marta', 'jose'])
+        self.assertEqual(il.get_collected_values('name'), ['marta'])
 
         il = NameFirstItemLoader()
-        il.add_value('name', u'marta')
-        il.add_value('name', [u'jose', u'pedro'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose'])
+        il.add_value('name', 'marta')
+        il.add_value('name', ['jose', 'pedro'])
+        self.assertEqual(il.get_collected_values('name'), ['marta', 'jose'])
 
     def test_map_compose_filter(self):
         def filter_world(x):
@@ -195,87 +195,87 @@ class BasicItemLoaderTest(unittest.TestCase):
             name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])
 
         il = TestItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Mart'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['Mart'])
         item = il.load_item()
-        self.assertEqual(item['name'], [u'Mart'])
+        self.assertEqual(item['name'], ['Mart'])
 
     def test_default_input_processor(self):
         il = DefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mart'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['mart'])
 
     def test_inherited_default_input_processor(self):
         class InheritDefaultedItemLoader(DefaultedItemLoader):
             pass
 
         il = InheritDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mart'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['mart'])
 
     def test_input_processor_inheritance(self):
         class ChildItemLoader(TestItemLoader):
             url_in = MapCompose(lambda v: v.lower())
 
         il = ChildItemLoader()
-        il.add_value('url', u'HTTP://scrapy.ORG')
-        self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org'])
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+        il.add_value('url', 'HTTP://scrapy.ORG')
+        self.assertEqual(il.get_output_value('url'), ['http://scrapy.org'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['Marta'])
 
         class ChildChildItemLoader(ChildItemLoader):
             url_in = MapCompose(lambda v: v.upper())
             summary_in = MapCompose(lambda v: v)
 
         il = ChildChildItemLoader()
-        il.add_value('url', u'http://scrapy.org')
-        self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG'])
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+        il.add_value('url', 'http://scrapy.org')
+        self.assertEqual(il.get_output_value('url'), ['HTTP://SCRAPY.ORG'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['Marta'])
 
     def test_empty_map_compose(self):
         class IdentityDefaultedItemLoader(DefaultedItemLoader):
             name_in = MapCompose()
 
         il = IdentityDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'marta'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['marta'])
 
     def test_identity_input_processor(self):
         class IdentityDefaultedItemLoader(DefaultedItemLoader):
             name_in = Identity()
 
         il = IdentityDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'marta'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['marta'])
 
     def test_extend_custom_input_processors(self):
         class ChildItemLoader(TestItemLoader):
             name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
 
         il = ChildItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mARTA'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['mARTA'])
 
     def test_extend_default_input_processors(self):
         class ChildDefaultedItemLoader(DefaultedItemLoader):
             name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
 
         il = ChildDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'MART'])
+        il.add_value('name', 'marta')
+        self.assertEqual(il.get_output_value('name'), ['MART'])
 
     def test_output_processor_using_function(self):
         il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
 
         class TakeFirstItemLoader(TestItemLoader):
-            name_out = u" ".join
+            name_out = " ".join
 
         il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), 'Mar Ta')
 
     def test_output_processor_error(self):
         class TestItemLoader(ItemLoader):
@@ -283,9 +283,9 @@ class BasicItemLoaderTest(unittest.TestCase):
             name_out = MapCompose(float)
 
         il = TestItemLoader()
-        il.add_value('name', [u'$10'])
+        il.add_value('name', ['$10'])
         try:
-            float(u'$10')
+            float('$10')
         except Exception as e:
             expected_exc_str = str(e)
 
@@ -303,53 +303,53 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_output_processor_using_classes(self):
         il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
 
         class TakeFirstItemLoader(TestItemLoader):
             name_out = Join()
 
         il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), 'Mar Ta')
 
         class TakeFirstItemLoader(TestItemLoader):
             name_out = Join("<br>")
 
         il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta')
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), 'Mar<br>Ta')
 
     def test_default_output_processor(self):
         il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
 
         class LalaItemLoader(TestItemLoader):
             default_output_processor = Identity()
 
         il = LalaItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+        il.add_value('name', ['mar', 'ta'])
+        self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
 
     def test_loader_context_on_declaration(self):
         class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor_with_args, key=u'val')
+            url_in = MapCompose(processor_with_args, key='val')
 
         il = ChildItemLoader()
-        il.add_value('url', u'text')
+        il.add_value('url', 'text')
         self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
+        il.replace_value('url', 'text2')
         self.assertEqual(il.get_output_value('url'), ['val'])
 
     def test_loader_context_on_instantiation(self):
         class ChildItemLoader(TestItemLoader):
             url_in = MapCompose(processor_with_args)
 
-        il = ChildItemLoader(key=u'val')
-        il.add_value('url', u'text')
+        il = ChildItemLoader(key='val')
+        il.add_value('url', 'text')
         self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
+        il.replace_value('url', 'text2')
         self.assertEqual(il.get_output_value('url'), ['val'])
 
     def test_loader_context_on_assign(self):
@@ -357,10 +357,10 @@ class BasicItemLoaderTest(unittest.TestCase):
             url_in = MapCompose(processor_with_args)
 
         il = ChildItemLoader()
-        il.context['key'] = u'val'
-        il.add_value('url', u'text')
+        il.context['key'] = 'val'
+        il.add_value('url', 'text')
         self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
+        il.replace_value('url', 'text2')
         self.assertEqual(il.get_output_value('url'), ['val'])
 
     def test_item_passed_to_input_processor_functions(self):
@@ -372,9 +372,9 @@ class BasicItemLoaderTest(unittest.TestCase):
 
         it = TestItem(name='marta')
         il = ChildItemLoader(item=it)
-        il.add_value('url', u'text')
+        il.add_value('url', 'text')
         self.assertEqual(il.get_output_value('url'), ['marta'])
-        il.replace_value('url', u'text2')
+        il.replace_value('url', 'text2')
         self.assertEqual(il.get_output_value('url'), ['marta'])
 
     def test_compose_processor(self):
@@ -382,10 +382,10 @@ class BasicItemLoaderTest(unittest.TestCase):
             name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])
 
         il = TestItemLoader()
-        il.add_value('name', [u'marta', u'other'])
-        self.assertEqual(il.get_output_value('name'), u'Mart')
+        il.add_value('name', ['marta', 'other'])
+        self.assertEqual(il.get_output_value('name'), 'Mart')
         item = il.load_item()
-        self.assertEqual(item['name'], u'Mart')
+        self.assertEqual(item['name'], 'Mart')
 
     def test_partial_processor(self):
         def join(values, sep=None, loader_context=None, ignored=None):
@@ -402,13 +402,13 @@ class BasicItemLoaderTest(unittest.TestCase):
             summary_out = Compose(partial(join, ignored='foo'))
 
         il = TestItemLoader()
-        il.add_value('name', [u'rabbit', u'hole'])
-        il.add_value('url', [u'rabbit', u'hole'])
-        il.add_value('summary', [u'rabbit', u'hole'])
+        il.add_value('name', ['rabbit', 'hole'])
+        il.add_value('url', ['rabbit', 'hole'])
+        il.add_value('summary', ['rabbit', 'hole'])
         item = il.load_item()
-        self.assertEqual(item['name'], u'rabbit+hole')
-        self.assertEqual(item['url'], u'rabbit.hole')
-        self.assertEqual(item['summary'], u'rabbithole')
+        self.assertEqual(item['name'], 'rabbit+hole')
+        self.assertEqual(item['url'], 'rabbit.hole')
+        self.assertEqual(item['summary'], 'rabbithole')
 
     def test_error_input_processor(self):
         class TestItem(Item):
@@ -420,7 +420,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
         il = TestItemLoader()
         self.assertRaises(ValueError, il.add_value, 'name',
-                          [u'marta', u'other'])
+                          ['marta', 'other'])
 
     def test_error_output_processor(self):
         class TestItem(Item):
@@ -431,7 +431,7 @@ class BasicItemLoaderTest(unittest.TestCase):
             name_out = Compose(Join(), float)
 
         il = TestItemLoader()
-        il.add_value('name', u'marta')
+        il.add_value('name', 'marta')
         with self.assertRaises(ValueError):
             il.load_item()
 
@@ -444,7 +444,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
         il = TestItemLoader()
         self.assertRaises(ValueError, il.add_value, 'name',
-                          [u'marta', u'other'], Compose(float))
+                          ['marta', 'other'], Compose(float))
 
 
 class InitializationFromDictTest(unittest.TestCase):
@@ -608,8 +608,8 @@ class ProcessorsTest(unittest.TestCase):
     def test_join(self):
         proc = Join()
         self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
-        self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
-        self.assertEqual(proc(['hello', 'world']), u'hello world')
+        self.assertEqual(proc(['', 'hello', 'world']), ' hello world')
+        self.assertEqual(proc(['hello', 'world']), 'hello world')
         self.assertIsInstance(proc(['hello', 'world']), str)
 
     def test_compose(self):
@@ -626,8 +626,8 @@ class ProcessorsTest(unittest.TestCase):
         def filter_world(x):
             return None if x == 'world' else x
         proc = MapCompose(filter_world, str.upper)
-        self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
-                         [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
+        self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
+                         ['HELLO', 'THIS', 'IS', 'SCRAPY'])
         proc = MapCompose(filter_world, str.upper)
         self.assertEqual(proc(None), [])
         proc = MapCompose(filter_world, str.upper)
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 7064337ad..b771e7d79 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -56,13 +56,13 @@ class LogFormatterTestCase(unittest.TestCase):
 
     def test_dropped(self):
         item = {}
-        exception = Exception(u"\u2018")
+        exception = Exception("\u2018")
         response = Response("http://www.example.com")
         logkws = self.formatter.dropped(item, exception, response, self.spider)
         logline = logkws['msg'] % logkws['args']
         lines = logline.splitlines()
         assert all(isinstance(x, str) for x in lines)
-        self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
+        self.assertEqual(lines, ["Dropped: \u2018", '{}'])
 
     def test_item_error(self):
         # In practice, the complete traceback is shown by passing the
@@ -72,7 +72,7 @@ class LogFormatterTestCase(unittest.TestCase):
         response = Response("http://www.example.com")
         logkws = self.formatter.item_error(item, exception, response, self.spider)
         logline = logkws['msg'] % logkws['args']
-        self.assertEqual(logline, u"Error processing {'key': 'value'}")
+        self.assertEqual(logline, "Error processing {'key': 'value'}")
 
     def test_spider_error(self):
         # In practice, the complete traceback is shown by passing the
@@ -107,20 +107,20 @@ class LogFormatterTestCase(unittest.TestCase):
 
     def test_scraped(self):
         item = CustomItem()
-        item['name'] = u'\xa3'
+        item['name'] = '\xa3'
         response = Response("http://www.example.com")
         logkws = self.formatter.scraped(item, response, self.spider)
         logline = logkws['msg'] % logkws['args']
         lines = logline.splitlines()
         assert all(isinstance(x, str) for x in lines)
-        self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])
+        self.assertEqual(lines, ["Scraped from <200 http://www.example.com>", 'name: \xa3'])
 
 
 class LogFormatterSubclass(LogFormatter):
     def crawled(self, request, response, spider):
         kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider)
         CRAWLEDMSG = (
-            u"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
+            "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
         )
         log_args = kwargs['args']
         log_args['flags'] = str(request.flags)
diff --git a/tests/test_mail.py b/tests/test_mail.py
index 53dbc0686..9b248fbfa 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -73,8 +73,8 @@ class MailSenderTest(unittest.TestCase):
         self.catched_msg = dict(**kwargs)
 
     def test_send_utf8(self):
-        subject = u'sübjèçt'
-        body = u'bödÿ-àéïöñß'
+        subject = 'sübjèçt'
+        body = 'bödÿ-àéïöñß'
         mailsender = MailSender(debug=True)
         mailsender.send(to=['test@scrapy.org'], subject=subject, body=body,
                         charset='utf-8', _callback=self._catch_mail_sent)
@@ -90,8 +90,8 @@ class MailSenderTest(unittest.TestCase):
         self.assertEqual(msg.get('Content-Type'), 'text/plain; charset="utf-8"')
 
     def test_send_attach_utf8(self):
-        subject = u'sübjèçt'
-        body = u'bödÿ-àéïöñß'
+        subject = 'sübjèçt'
+        body = 'bödÿ-àéïöñß'
         attach = BytesIO()
         attach.write(body.encode('utf-8'))
         attach.seek(0)
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index dd19a69d5..a175f88ca 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -23,11 +23,11 @@ class ResponseTypesTest(unittest.TestCase):
         mappings = [
             (b'attachment; filename="data.xml"', XmlResponse),
             (b'attachment; filename=data.xml', XmlResponse),
-            (u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
-            (u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
-            (u'attachment;filename=data高.doc'.encode('gbk'), Response),
-            (u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
-            (u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
+            ('attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
+            ('attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
+            ('attachment;filename=data高.doc'.encode('gbk'), Response),
+            ('attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
+            ('attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
 
         ]
         for source, cls in mappings:
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 24aaaf7ec..9d8c201dd 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -93,7 +93,7 @@ class BaseRobotParserTest:
         self.assertTrue(rp.allowed("https://site.local/disallowed", "*"))
 
     def test_unicode_url_and_useragent(self):
-        robotstxt_robotstxt_body = u"""
+        robotstxt_robotstxt_body = """
         User-Agent: *
         Disallow: /admin/
         Disallow: /static/
@@ -107,11 +107,11 @@ class BaseRobotParserTest:
         self.assertTrue(rp.allowed("https://site.local/", "*"))
         self.assertFalse(rp.allowed("https://site.local/admin/", "*"))
         self.assertFalse(rp.allowed("https://site.local/static/", "*"))
-        self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt"))
+        self.assertTrue(rp.allowed("https://site.local/admin/", "UnicödeBöt"))
         self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*"))
-        self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*"))
+        self.assertFalse(rp.allowed("https://site.local/wiki/Käyttäjä:", "*"))
         self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*"))
-        self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt"))
+        self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt"))
 
 
 class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 00e663c11..62036ad8c 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -25,19 +25,19 @@ class SelectorTestCase(unittest.TestCase):
         )
         self.assertEqual(
             [x.get() for x in sel.xpath("//input[@name='a']/@name")],
-            [u'a']
+            ['a']
         )
         self.assertEqual(
             [x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
-            [u'12.0']
+            ['12.0']
         )
         self.assertEqual(
             sel.xpath("concat('xpath', 'rules')").getall(),
-            [u'xpathrules']
+            ['xpathrules']
         )
         self.assertEqual(
             [x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
-            [u'12']
+            ['12']
         )
 
     def test_root_base_url(self):
@@ -52,30 +52,30 @@ class SelectorTestCase(unittest.TestCase):
         sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
         self.assertEqual(sel.type, 'xml')
         self.assertEqual(sel.xpath("//div").getall(),
-                         [u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
+                         ['<div><img src="a.jpg"><p>Hello</p></img></div>'])
 
         sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
         self.assertEqual(sel.type, 'html')
         self.assertEqual(sel.xpath("//div").getall(),
-                         [u'<div><img src="a.jpg"><p>Hello</p></div>'])
+                         ['<div><img src="a.jpg"><p>Hello</p></div>'])
 
     def test_http_header_encoding_precedence(self):
-        # u'\xa3'     = pound symbol in unicode
-        # u'\xc2\xa3' = pound symbol in utf-8
-        # u'\xa3'     = pound symbol in latin-1 (iso-8859-1)
+        # '\xa3'     = pound symbol in unicode
+        # '\xc2\xa3' = pound symbol in utf-8
+        # '\xa3'     = pound symbol in latin-1 (iso-8859-1)
 
-        meta = u'<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'
-        head = u'<head>' + meta + u'</head>'
-        body_content = u'<span id="blank">\xa3</span>'
-        body = u'<body>' + body_content + u'</body>'
-        html = u'<html>' + head + body + u'</html>'
+        meta = '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'
+        head = '<head>' + meta + '</head>'
+        body_content = '<span id="blank">\xa3</span>'
+        body = '<body>' + body_content + '</body>'
+        html = '<html>' + head + body + '</html>'
         encoding = 'utf-8'
         html_utf8 = html.encode(encoding)
 
         headers = {'Content-Type': ['text/html; charset=utf-8']}
         response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
         x = Selector(response)
-        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])
+        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ['\xa3'])
 
     def test_badly_encoded_body(self):
         # \xe9 alone isn't valid utf8 sequence
@@ -92,4 +92,4 @@ class SelectorTestCase(unittest.TestCase):
 
     def test_selector_bad_args(self):
         with self.assertRaisesRegex(ValueError, 'received both response and text'):
-            Selector(TextResponse(url='http://example.com', body=b''), text=u'')
+            Selector(TextResponse(url='http://example.com', body=b''), text='')
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 83c10a3c3..78157a9b9 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -153,13 +153,13 @@ class XMLFeedSpiderTest(SpiderTest):
             output = list(spider._parse(response))
             self.assertEqual(len(output), 2, iterator)
             self.assertEqual(output, [
-                {'loc': [u'http://www.example.com/Special-Offers.html'],
-                 'updated': [u'2009-08-16'],
-                 'custom': [u'fuu'],
-                 'other': [u'bar']},
+                {'loc': ['http://www.example.com/Special-Offers.html'],
+                 'updated': ['2009-08-16'],
+                 'custom': ['fuu'],
+                 'other': ['bar']},
                 {'loc': [],
-                 'updated': [u'2009-08-16'],
-                 'other': [u'foo'],
+                 'updated': ['2009-08-16'],
+                 'other': ['foo'],
                  'custom': []},
             ], iterator)
 
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index d17bb2cbc..298178f08 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -54,7 +54,7 @@ class XmliterTestCase(unittest.TestCase):
 
     def test_xmliter_unicode(self):
         # example taken from https://github.com/scrapy/scrapy/issues/1665
-        body = u"""<?xml version="1.0" encoding="UTF-8"?>
+        body = """<?xml version="1.0" encoding="UTF-8"?>
             <þingflokkar>
                <þingflokkur id="26">
                   <heiti />
@@ -97,15 +97,15 @@ class XmliterTestCase(unittest.TestCase):
             XmlResponse(url="http://example.com", body=body, encoding='utf-8'),
         ):
             attrs = []
-            for x in self.xmliter(r, u'þingflokkur'):
+            for x in self.xmliter(r, 'þingflokkur'):
                 attrs.append((x.attrib['id'],
-                              x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(),
-                              x.xpath(u'./tímabil/fyrstaþing/text()').getall()))
+                              x.xpath('./skammstafanir/stuttskammstöfun/text()').getall(),
+                              x.xpath('./tímabil/fyrstaþing/text()').getall()))
 
             self.assertEqual(attrs,
-                             [(u'26', [u'-'], [u'80']),
-                              (u'21', [u'Ab'], [u'76']),
-                              (u'27', [u'A'], [u'27'])])
+                             [('26', ['-'], ['80']),
+                              ('21', ['Ab'], ['76']),
+                              ('27', ['A'], ['27'])])
 
     def test_xmliter_text(self):
         body = (
@@ -114,7 +114,7 @@ class XmliterTestCase(unittest.TestCase):
         )
 
         self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
-                         [[u'one'], [u'two']])
+                         [['one'], ['two']])
 
     def test_xmliter_namespaces(self):
         body = b"""
@@ -179,7 +179,7 @@ class XmliterTestCase(unittest.TestCase):
         response = XmlResponse('http://www.example.com', body=body)
         self.assertEqual(
             next(self.xmliter(response, 'item')).get(),
-            u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
+            '<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
         )
 
 
@@ -265,10 +265,10 @@ class UtilsCsvTestCase(unittest.TestCase):
 
         result = [row for row in csv]
         self.assertEqual(result,
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
         # explicit type check cuz' we no like stinkin' autocasting! yarrr
         for result_row in result:
@@ -281,10 +281,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response, delimiter='\t')
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_quotechar(self):
         body1 = get_testdata('feeds', 'feed-sample6.csv')
@@ -294,19 +294,19 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv1 = csviter(response1, quotechar="'")
 
         self.assertEqual([row for row in csv1],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
         response2 = TextResponse(url="http://example.com/", body=body2)
         csv2 = csviter(response2, delimiter="|", quotechar="'")
 
         self.assertEqual([row for row in csv2],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_wrong_quotechar(self):
         body = get_testdata('feeds', 'feed-sample6.csv')
@@ -314,10 +314,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response)
 
         self.assertEqual([row for row in csv],
-                         [{u"'id'": u"1", u"'name'": u"'alpha'", u"'value'": u"'foobar'"},
-                          {u"'id'": u"2", u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"},
-                          {u"'id'": u"'3'", u"'name'": u"'multi'", u"'value'": u"'foo"},
-                          {u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}])
+                         [{"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"},
+                          {"'id'": "2", "'name'": "'unicode'", "'value'": "'\xfan\xedc\xf3d\xe9\u203d'"},
+                          {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"},
+                          {"'id'": "4", "'name'": "'empty'", "'value'": ""}])
 
     def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
         body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
@@ -325,10 +325,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response, delimiter='\t')
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_headers(self):
         sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()
@@ -338,10 +338,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response, headers=[h.decode('utf-8') for h in headers])
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': 'foo\nbar'},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_falserow(self):
         body = get_testdata('feeds', 'feed-sample3.csv')
@@ -351,10 +351,10 @@ class UtilsCsvTestCase(unittest.TestCase):
         csv = csviter(response)
 
         self.assertEqual([row for row in csv],
-                         [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
-                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
-                          {u'id': u'4', u'name': u'empty', u'value': u''}])
+                         [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+                          {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_exception(self):
         body = get_testdata('feeds', 'feed-sample3.csv')
@@ -377,8 +377,8 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual(
             list(csv),
             [
-                {u'id': u'1', u'name': u'latin1', u'value': u'test'},
-                {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'},
+                {'id': '1', 'name': 'latin1', 'value': 'test'},
+                {'id': '2', 'name': 'something', 'value': '\xf1\xe1\xe9\xf3'},
             ]
         )
 
@@ -387,8 +387,8 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual(
             list(csv),
             [
-                {u'id': u'1', u'name': u'cp852', u'value': u'test'},
-                {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
+                {'id': '1', 'name': 'cp852', 'value': 'test'},
+                {'id': '2', 'name': 'something', 'value': '\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
             ]
         )
 
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index ebce3c079..3f93f509e 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -34,13 +34,13 @@ class MutableChainTest(unittest.TestCase):
 
 class ToUnicodeTest(unittest.TestCase):
     def test_converting_an_utf8_encoded_string_to_unicode(self):
-        self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e')
+        self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e')
 
     def test_converting_a_latin_1_encoded_string_to_unicode(self):
-        self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e')
+        self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), 'lel\xf1e')
 
     def test_converting_a_unicode_to_unicode_should_return_the_same_object(self):
-        self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
+        self.assertEqual(to_unicode('\xf1e\xf1e\xf1e'), '\xf1e\xf1e\xf1e')
 
     def test_converting_a_strange_object_should_raise_TypeError(self):
         self.assertRaises(TypeError, to_unicode, 423)
@@ -48,16 +48,16 @@ class ToUnicodeTest(unittest.TestCase):
     def test_errors_argument(self):
         self.assertEqual(
             to_unicode(b'a\xedb', 'utf-8', errors='replace'),
-            u'a\ufffdb'
+            'a\ufffdb'
         )
 
 
 class ToBytesTest(unittest.TestCase):
     def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self):
-        self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49')
+        self.assertEqual(to_bytes('\xa3 49'), b'\xc2\xa3 49')
 
     def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):
-        self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49')
+        self.assertEqual(to_bytes('\xa3 49', 'latin-1'), b'\xa3 49')
 
     def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self):
         self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e')
@@ -67,7 +67,7 @@ class ToBytesTest(unittest.TestCase):
 
     def test_errors_argument(self):
         self.assertEqual(
-            to_bytes(u'a\ufffdb', 'latin-1', errors='replace'),
+            to_bytes('a\ufffdb', 'latin-1', errors='replace'),
             b'a?b'
         )
 
@@ -96,7 +96,7 @@ class BinaryIsTextTest(unittest.TestCase):
         assert binary_is_text(b"hello")
 
     def test_utf_16_strings_contain_null_bytes(self):
-        assert binary_is_text(u"hello".encode('utf-16'))
+        assert binary_is_text("hello".encode('utf-16'))
 
     def test_one_with_encoding(self):
         assert binary_is_text(b"<div>Price \xa3</div>")
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 450e4bdca..de94ec960 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -22,7 +22,7 @@ class RequestSerializationTest(unittest.TestCase):
             method="POST",
             body=b"some body",
             headers={'content-encoding': 'text/html; charset=latin-1'},
-            cookies={'currency': u'руб'},
+            cookies={'currency': 'руб'},
             encoding='latin-1',
             priority=20,
             meta={'a': 'b'},
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index 5a52dd695..5ff2e41ef 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -19,8 +19,8 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
     def test_simple_render(self):
 
         context = dict(project_name='proj', name='spi', classname='TheSpider')
-        template = u'from ${project_name}.spiders.${name} import ${classname}'
-        rendered = u'from proj.spiders.spi import TheSpider'
+        template = 'from ${project_name}.spiders.${name} import ${classname}'
+        rendered = 'from proj.spiders.spi import TheSpider'
 
         template_path = os.path.join(self.tmp_path, 'templ.py.tmpl')
         render_path = os.path.join(self.tmp_path, 'templ.py')

From 6f4ccec5675c00b0ec7877a0b1fcc234d5983490 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 30 Jul 2020 14:03:14 +0200
Subject: [PATCH 2949/2974] Cover our deprecation policy in the documentation

---
 docs/contributing.rst |  5 +++++
 docs/versioning.rst   | 22 ++++++++++++++++++++--
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 7b901dd00..525ad3497 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -108,6 +108,11 @@ Well-written patches should:
 
       tox -e docs-coverage
 
+* if you are removing deprecated code, first make sure that at least 1 year
+  (12 months) has passed since the release that introduced the deprecation.
+  See :ref:`deprecation-policy`.
+
+
 .. _submitting-patches:
 
 Submitting patches
diff --git a/docs/versioning.rst b/docs/versioning.rst
index 227085f02..57643ea9a 100644
--- a/docs/versioning.rst
+++ b/docs/versioning.rst
@@ -1,7 +1,7 @@
 .. _versioning:
 
 ============================
-Versioning and API Stability
+Versioning and API stability
 ============================
 
 Versioning
@@ -34,7 +34,7 @@ For example:
   production)
 
 
-API Stability
+API stability
 =============
 
 API stability was one of the major goals for the *1.0* release.
@@ -47,5 +47,23 @@ new methods or functionality but the existing methods should keep working the
 same way.
 
 
+.. _deprecation-policy:
+
+Deprecation policy
+==================
+
+We aim to maintain support for deprecated Scrapy features for at least 1 year.
+
+For example, if a feature is deprecated in a Scrapy version released on
+June 15th 2020, that feature should continue to work in versions released on
+June 14th 2021 or before that.
+
+Any new Scrapy release after a year *may* remove support for that deprecated
+feature.
+
+All deprecated features removed in a Scrapy release are explicitly mentioned in
+the :ref:`release notes <news>`.
+
+
 .. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases
 

From 3600582f56071dd9f40f31ed44e09294a78dc13f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 4 Aug 2020 20:05:56 +0200
Subject: [PATCH 2950/2974] Cover Scrapy 2.2.1 and 2.3 in the release notes
 (#4708)

---
 docs/news.rst                   | 133 ++++++++++++++++++++++++++++++++
 docs/topics/commands.rst        |   2 +
 docs/topics/developer-tools.rst |   6 +-
 docs/topics/feed-exports.rst    |  36 ++++++++-
 scrapy/utils/curl.py            |   3 +-
 5 files changed, 176 insertions(+), 4 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 80d130e4a..850b323ef 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,139 @@
 Release notes
 =============
 
+.. _release-2.3.0:
+
+Scrapy 2.3.0 (2020-08-04)
+-------------------------
+
+Highlights:
+
+*   :ref:`Feed exports <topics-feed-exports>` now support :ref:`Google Cloud
+    Storage <topics-feed-storage-gcs>` as a storage backend
+
+*   The new :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting allows to deliver
+    output items in batches of up to the specified number of items.
+
+    It also serves as a workaround for :ref:`delayed file delivery
+    <delayed-file-delivery>`, which causes Scrapy to only start item delivery
+    after the crawl has finished when using certain storage backends
+    (:ref:`S3 <topics-feed-storage-s3>`, :ref:`FTP <topics-feed-storage-ftp>`,
+    and now :ref:`GCS <topics-feed-storage-gcs>`).
+
+*   The base implementation of :ref:`item loaders <topics-loaders>` has been
+    moved into a separate library, :doc:`itemloaders <itemloaders:index>`,
+    allowing usage from outside Scrapy and a separate release schedule
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+*   Removed the following classes and their parent modules from
+    ``scrapy.linkextractors``:
+
+    *   ``htmlparser.HtmlParserLinkExtractor``
+    *   ``regex.RegexLinkExtractor``
+    *   ``sgml.BaseSgmlLinkExtractor``
+    *   ``sgml.SgmlLinkExtractor``
+
+    Use
+    :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
+    instead (:issue:`4356`, :issue:`4679`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+*   The ``scrapy.utils.python.retry_on_eintr`` function is now deprecated
+    (:issue:`4683`)
+
+
+New features
+~~~~~~~~~~~~
+
+*   :ref:`Feed exports <topics-feed-exports>` support :ref:`Google Cloud
+    Storage <topics-feed-storage-gcs>` (:issue:`685`, :issue:`3608`)
+
+*   New :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting for batch deliveries
+    (:issue:`4250`, :issue:`4434`)
+
+*   The :command:`parse` command now allows specifying an output file
+    (:issue:`4317`, :issue:`4377`)
+
+*   :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
+    :func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support
+    ``--data-raw`` (:issue:`4612`)
+
+*   A ``parse`` callback may now be used in built-in spider subclasses, such
+    as :class:`~scrapy.spiders.CrawlSpider` (:issue:`712`, :issue:`732`,
+    :issue:`781`, :issue:`4254` )
+
+
+Bug fixes
+~~~~~~~~~
+
+*   Fixed the :ref:`CSV exporting <topics-feed-format-csv>` of
+    :ref:`dataclass items <dataclass-items>` and :ref:`attr.s items
+    <attrs-items>` (:issue:`4667`, :issue:`4668`)
+
+*   :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
+    :func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request
+    method to ``POST`` when a request body is specified and no request method
+    is specified (:issue:`4612`)
+
+*   The processing of ANSI escape sequences in enabled in Windows 10.0.14393
+    and later, where it is required for colored output (:issue:`4393`,
+    :issue:`4403`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+*   Updated the `OpenSSL cipher list format`_ link in the documentation about
+    the :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting (:issue:`4653`)
+
+*   Simplified the code example in :ref:`topics-loaders-dataclass`
+    (:issue:`4652`)
+
+.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
+
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+*   The base implementation of :ref:`item loaders <topics-loaders>` has been
+    moved into :doc:`itemloaders <itemloaders:index>` (:issue:`4005`,
+    :issue:`4516`)
+
+*   Fixed a silenced error in some scheduler tests (:issue:`4644`,
+    :issue:`4645`)
+
+*   Renewed the localhost certificate used for SSL tests (:issue:`4650`)
+
+*   Removed cookie-handling code specific to Python 2 (:issue:`4682`)
+
+*   Stopped using Python 2 unicode literal syntax (:issue:`4704`)
+
+*   Stopped using a backlash for line continuation (:issue:`4673`)
+
+*   Removed unneeded entries from the MyPy exception list (:issue:`4690`)
+
+*   Automated tests now pass on Windows as part of our continuous integration
+    system (:issue:`4458`)
+
+*   Automated tests now pass on the latest PyPy version for supported Python
+    versions in our continuous integration system (:issue:`4504`)
+
+
+.. _release-2.2.1:
+
+Scrapy 2.2.1 (2020-07-17)
+-------------------------
+
+*   The :command:`startproject` command no longer makes unintended changes to
+    the permissions of files in the destination folder, such as removing
+    execution permissions (:issue:`4662`, :issue:`4666`)
+
+
 .. _release-2.2.0:
 
 Scrapy 2.2.0 (2020-06-24)
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 4fce51abc..9638a2322 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -493,6 +493,8 @@ Supported options:
 
 * ``--output`` or ``-o``: dump scraped items to a file
 
+  .. versionadded:: 2.3
+
 .. skip: start
 
 Usage example::
diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index 4e87a00f2..101aa159c 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -289,8 +289,10 @@ request::
         "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
 
 Alternatively, if you want to know the arguments needed to recreate that
-request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
-function to get a dictionary with the equivalent arguments.
+request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs`
+function to get a dictionary with the equivalent arguments:
+
+.. autofunction:: scrapy.utils.curl.curl_to_request_kwargs
 
 Note that to translate a cURL command into a Scrapy request,
 you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index dd4eb3c61..37b7096f6 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -100,6 +100,7 @@ The storages backends supported out of the box are:
  * :ref:`topics-feed-storage-fs`
  * :ref:`topics-feed-storage-ftp`
  * :ref:`topics-feed-storage-s3` (requires botocore_)
+ * :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
  * :ref:`topics-feed-storage-stdout`
 
 Some storage backends may be unavailable if the required external libraries are
@@ -169,6 +170,9 @@ FTP supports two different connection modes: `active or passive
 mode by default. To use the active connection mode instead, set the
 :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
 
+This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
+
+
 .. _topics-feed-storage-s3:
 
 S3
@@ -194,11 +198,16 @@ You can also define a custom ACL for exported feeds using this setting:
 
  * :setting:`FEED_STORAGE_S3_ACL`
 
+This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
+
+
 .. _topics-feed-storage-gcs:
 
 Google Cloud Storage (GCS)
 --------------------------
 
+.. versionadded:: 2.3
+
 The feeds are stored on `Google Cloud Storage`_.
 
  * URI scheme: ``gs``
@@ -206,7 +215,7 @@ The feeds are stored on `Google Cloud Storage`_.
 
    * ``gs://mybucket/path/to/export.csv``
 
- * Required external libraries: `google-cloud-storage <https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python>`_.
+ * Required external libraries: `google-cloud-storage`_.
 
 For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
 
@@ -215,6 +224,11 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following
  * :setting:`FEED_STORAGE_GCS_ACL`
  * :setting:`GCS_PROJECT_ID`
 
+This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
+
+.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
+
+
 .. _topics-feed-storage-stdout:
 
 Standard output
@@ -227,6 +241,26 @@ The feeds are written to the standard output of the Scrapy process.
  * Required external libraries: none
 
 
+.. _delayed-file-delivery:
+
+Delayed file delivery
+---------------------
+
+As indicated above, some of the described storage backends use delayed file
+delivery.
+
+These storage backends do not upload items to the feed URI as those items are
+scraped. Instead, Scrapy writes items into a temporary local file, and only
+once all the file contents have been written (i.e. at the end of the crawl) is
+that file uploaded to the feed URI.
+
+If you want item delivery to start earlier when using one of these storage
+backends, use :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` to split the output items
+in multiple files, with the specified maximum item count per file. That way, as
+soon as a file reaches the maximum item count, that file is delivered to the
+feed URI, allowing item delivery to start way before the end of the crawl.
+
+
 Settings
 ========
 
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index aa681522f..9c0efcec4 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -39,7 +39,8 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
 
     :param str curl_command: string containing the curl command
     :param bool ignore_unknown_options: If true, only a warning is emitted when
-    cURL options are unknown. Otherwise raises an error. (default: True)
+                                        cURL options are unknown. Otherwise
+                                        raises an error. (default: True)
     :return: dictionary of Request kwargs
     """
 

From 1278e76d9093b1c5c9ec810768d1066772a8a134 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 4 Aug 2020 20:07:02 +0200
Subject: [PATCH 2951/2974] =?UTF-8?q?Bump=20version:=202.2.0=20=E2=86=92?=
 =?UTF-8?q?=202.3.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 8d4d74bc5..3c1c8f891 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.2.0
+current_version = 2.3.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index ccbccc3dc..276cbf9e2 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.2.0
+2.3.0

From 4ee538e44b2650c49054b1f7f4c87ac70350471a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 4 Aug 2020 20:34:11 +0200
Subject: [PATCH 2952/2974] Update unicode references from Python 2 times in
 the documentation (#4703)

---
 docs/topics/exporters.rst        |  8 ++------
 docs/topics/loaders.rst          |  4 ++--
 docs/topics/request-response.rst | 30 ++++++++++++++++--------------
 docs/topics/selectors.rst        |  7 ++++---
 4 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index e5c99e5b1..8c84b85fc 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -166,8 +166,7 @@ BaseItemExporter
       By default, this method looks for a serializer :ref:`declared in the item
       field <topics-exporters-serializers>` and returns the result of applying
       that serializer to the value. If no serializer is found, it returns the
-      value unchanged except for ``unicode`` values which are encoded to
-      ``str`` using the encoding declared in the :attr:`encoding` attribute.
+      value unchanged.
 
       :param field: the field being serialized. If the source :ref:`item object
           <item-types>` does not define field metadata, *field* is an empty
@@ -217,10 +216,7 @@ BaseItemExporter
 
    .. attribute:: encoding
 
-      The encoding that will be used to encode unicode values. This only
-      affects unicode values (which are always serialized to str using this
-      encoding). Other value types are passed unchanged to the specific
-      serialization library.
+      The output character encoding.
 
    .. attribute:: indent
 
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 29d9c5805..c0f534493 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -193,10 +193,10 @@ Item Loaders are declared using a class definition syntax. Here is an example::
 
         default_output_processor = TakeFirst()
 
-        name_in = MapCompose(unicode.title)
+        name_in = MapCompose(str.title)
         name_out = Join()
 
-        price_in = MapCompose(unicode.strip)
+        price_in = MapCompose(str.strip)
 
         # ...
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index fbd8e4b73..1dffd1d55 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -51,12 +51,12 @@ Request objects
        given, the dict passed in this parameter will be shallow copied.
     :type meta: dict
 
-    :param body: the request body. If a ``unicode`` is passed, then it's encoded to
-      ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If
-      ``body`` is not given, an empty string is stored. Regardless of the
-      type of this argument, the final value stored will be a ``str`` (never
-      ``unicode`` or ``None``).
-    :type body: str or unicode
+    :param body: the request body. If a string is passed, then it's encoded as
+      bytes using the ``encoding`` passed (which defaults to ``utf-8``). If
+      ``body`` is not given, an empty bytes object is stored. Regardless of the
+      type of this argument, the final value stored will be a bytes object
+      (never a string or ``None``).
+    :type body: bytes or str
 
     :param headers: the headers of this request. The dict values can be strings
        (for single valued headers) or lists (for multi-valued headers). If
@@ -106,7 +106,7 @@ Request objects
 
     :param encoding: the encoding of this request (defaults to ``'utf-8'``).
        This encoding will be used to percent-encode the URL and to convert the
-       body to ``str`` (if given as ``unicode``).
+       body to bytes (if given as a string).
     :type encoding: string
 
     :param priority: the priority of this request (defaults to ``0``).
@@ -721,7 +721,7 @@ Response objects
     .. attribute:: Response.body
 
         The body of this Response. Keep in mind that Response.body
-        is always a bytes object. If you want the unicode version use
+        is always a bytes object. If you want the string version use
         :attr:`TextResponse.text` (only available in :class:`TextResponse`
         and subclasses).
 
@@ -842,9 +842,9 @@ TextResponse objects
     is the same as for the :class:`Response` class and is not documented here.
 
     :param encoding: is a string which contains the encoding to use for this
-       response. If you create a :class:`TextResponse` object with a unicode
+       response. If you create a :class:`TextResponse` object with a string as
        body, it will be encoded using this encoding (remember the body attribute
-       is always a string). If ``encoding`` is ``None`` (default value), the
+       is always a bytes object). If ``encoding`` is ``None`` (default value), the
        encoding will be looked up in the response headers and body instead.
     :type encoding: string
 
@@ -853,7 +853,7 @@ TextResponse objects
 
     .. attribute:: TextResponse.text
 
-       Response body, as unicode.
+       Response body, as a string.
 
        The same as ``response.body.decode(response.encoding)``, but the
        result is cached after the first call, so you can access
@@ -861,9 +861,11 @@ TextResponse objects
 
        .. note::
 
-            ``unicode(response.body)`` is not a correct way to convert response
-            body to unicode: you would be using the system default encoding
-            (typically ``ascii``) instead of the response encoding.
+            ``str(response.body)`` is not a correct way to convert the response
+            body into a string:
+
+            >>> str(b'body')
+            "b'body'"
 
 
     .. attribute:: TextResponse.encoding
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 5014df6ac..9e2c6ba42 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -64,7 +64,8 @@ more shortcuts: ``response.xpath()`` and ``response.css()``:
 
 Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
 constructed by passing either :class:`~scrapy.http.TextResponse` object or
-markup as an unicode string (in ``text`` argument).
+markup as a string (in ``text`` argument).
+
 Usually there is no need to construct Scrapy selectors manually:
 ``response`` object is available in Spider callbacks, so in most cases
 it is more convenient to use ``response.css()`` and ``response.xpath()``
@@ -383,7 +384,7 @@ Using selectors with regular expressions
 
 :class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting
 data using regular expressions. However, unlike using ``.xpath()`` or
-``.css()`` methods, ``.re()`` returns a list of unicode strings. So you
+``.css()`` methods, ``.re()`` returns a list of strings. So you
 can't construct nested ``.re()`` calls.
 
 Here's an example used to extract image names from the :ref:`HTML code
@@ -989,7 +990,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
       sel.xpath("//h1")
 
 2. Extract the text of all ``<h1>`` elements from an HTML response body,
-   returning a list of unicode strings::
+   returning a list of strings::
 
       sel.xpath("//h1").getall()         # this includes the h1 tag
       sel.xpath("//h1/text()").getall()  # this excludes the h1 tag

From 336f19f5cc6edd0392c77f38b857d3e40bf565da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Tue, 4 Aug 2020 20:42:01 +0200
Subject: [PATCH 2953/2974] Change super syntax (#4707)

---
 extras/qpsclient.py                          |  2 +-
 scrapy/commands/view.py                      |  2 +-
 scrapy/contracts/default.py                  |  2 +-
 scrapy/core/downloader/contextfactory.py     |  4 ++--
 scrapy/core/downloader/handlers/http11.py    | 10 +++++-----
 scrapy/core/downloader/tls.py                |  2 +-
 scrapy/core/spidermw.py                      |  2 +-
 scrapy/crawler.py                            |  2 +-
 scrapy/downloadermiddlewares/redirect.py     |  2 +-
 scrapy/exceptions.py                         |  4 ++--
 scrapy/exporters.py                          |  2 +-
 scrapy/http/headers.py                       |  8 ++++----
 scrapy/http/request/form.py                  |  2 +-
 scrapy/http/request/json_request.py          |  4 ++--
 scrapy/http/request/rpc.py                   |  2 +-
 scrapy/http/response/text.py                 | 10 +++++-----
 scrapy/item.py                               | 10 +++++-----
 scrapy/linkextractors/__init__.py            |  2 +-
 scrapy/linkextractors/lxmlhtml.py            |  2 +-
 scrapy/pipelines/files.py                    |  2 +-
 scrapy/pipelines/images.py                   |  3 +--
 scrapy/resolver.py                           | 10 +++++-----
 scrapy/selector/unified.py                   |  2 +-
 scrapy/settings/__init__.py                  |  2 +-
 scrapy/spidermiddlewares/httperror.py        |  2 +-
 scrapy/spiders/crawl.py                      |  4 ++--
 scrapy/spiders/init.py                       |  2 +-
 scrapy/spiders/sitemap.py                    |  2 +-
 scrapy/squeues.py                            | 12 ++++++------
 scrapy/statscollectors.py                    |  2 +-
 scrapy/utils/datatypes.py                    | 14 +++++++-------
 scrapy/utils/deprecate.py                    |  8 ++++----
 scrapy/utils/log.py                          |  2 +-
 scrapy/utils/serialize.py                    |  2 +-
 scrapy/utils/testsite.py                     |  4 ++--
 tests/spiders.py                             | 20 ++++++++++----------
 tests/test_command_parse.py                  |  2 +-
 tests/test_commands.py                       |  4 ++--
 tests/test_contracts.py                      |  2 +-
 tests/test_downloader_handlers.py            |  4 ++--
 tests/test_downloadermiddleware_httpcache.py |  4 ++--
 tests/test_downloadermiddleware_robotstxt.py |  4 ++--
 tests/test_exporters.py                      |  2 +-
 tests/test_http_request.py                   |  4 ++--
 tests/test_http_response.py                  |  2 +-
 tests/test_item.py                           |  2 +-
 tests/test_linkextractors.py                 |  2 +-
 tests/test_loader.py                         |  2 +-
 tests/test_loader_deprecated.py              |  2 +-
 tests/test_logformatter.py                   |  2 +-
 tests/test_middleware.py                     |  2 +-
 tests/test_pipeline_media.py                 | 12 ++++++------
 tests/test_request_left.py                   |  2 +-
 tests/test_robotstxt_interface.py            |  8 ++++----
 tests/test_scheduler.py                      |  4 ++--
 tests/test_spidermiddleware_httperror.py     |  2 +-
 56 files changed, 117 insertions(+), 118 deletions(-)

diff --git a/extras/qpsclient.py b/extras/qpsclient.py
index 7554f7eec..fe1f96cbb 100644
--- a/extras/qpsclient.py
+++ b/extras/qpsclient.py
@@ -27,7 +27,7 @@ class QPSSpider(Spider):
     slots = 1
 
     def __init__(self, *a, **kw):
-        super(QPSSpider, self).__init__(*a, **kw)
+        super().__init__(*a, **kw)
         if self.qps is not None:
             self.qps = float(self.qps)
             self.download_delay = 1 / self.qps
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index 908bee966..c8f873334 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -11,7 +11,7 @@ class Command(fetch.Command):
         return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
 
     def add_options(self, parser):
-        super(Command, self).add_options(parser)
+        super().add_options(parser)
         parser.remove_option("--headers")
 
     def _print_response(self, response, opts):
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 34f0d36d4..cfdcc7c25 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -56,7 +56,7 @@ class ReturnsContract(Contract):
     }
 
     def __init__(self, *args, **kwargs):
-        super(ReturnsContract, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
         if len(self.args) not in [1, 2, 3]:
             raise ValueError(
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 452242d47..8a7d656a1 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -20,7 +20,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
     """
 
     def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
-        super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._ssl_method = method
         self.tls_verbose_logging = tls_verbose_logging
         if tls_ciphers:
@@ -45,7 +45,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         #   (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
         #
         # * getattr() for `_ssl_method` attribute for context factories
-        #   not calling super(..., self).__init__
+        #   not calling super().__init__
         return CertificateOptions(
             verify=False,
             method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 22c9ac520..fb04d1fb7 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -126,7 +126,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
 
     def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
         proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
-        super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
+        super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
         self._tunnelReadyDeferred = defer.Deferred()
         self._tunneledHost = host
         self._tunneledPort = port
@@ -178,7 +178,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
 
     def connect(self, protocolFactory):
         self._protocolFactory = protocolFactory
-        connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory)
+        connectDeferred = super().connect(protocolFactory)
         connectDeferred.addCallback(self.requestTunnel)
         connectDeferred.addErrback(self.connectFailed)
         return self._tunnelReadyDeferred
@@ -215,7 +215,7 @@ class TunnelingAgent(Agent):
 
     def __init__(self, reactor, proxyConf, contextFactory=None,
                  connectTimeout=None, bindAddress=None, pool=None):
-        super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
+        super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
         self._proxyConf = proxyConf
         self._contextFactory = contextFactory
 
@@ -235,7 +235,7 @@ class TunnelingAgent(Agent):
         # otherwise, same remote host connection request could reuse
         # a cached tunneled connection to a different proxy
         key = key + self._proxyConf
-        return super(TunnelingAgent, self)._requestWithEndpoint(
+        return super()._requestWithEndpoint(
             key=key,
             endpoint=endpoint,
             method=method,
@@ -249,7 +249,7 @@ class TunnelingAgent(Agent):
 class ScrapyProxyAgent(Agent):
 
     def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
-        super(ScrapyProxyAgent, self).__init__(
+        super().__init__(
             reactor=reactor,
             connectTimeout=connectTimeout,
             bindAddress=bindAddress,
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index e43a3c83e..d9f3750d5 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -47,7 +47,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
     """
 
     def __init__(self, hostname, ctx, verbose_logging=False):
-        super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
+        super().__init__(hostname, ctx)
         self.verbose_logging = verbose_logging
 
     def _identityVerifyingInfoCallback(self, connection, where, ret):
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 35264a92b..5a99b96be 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -34,7 +34,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
 
     def _add_middleware(self, mw):
-        super(SpiderMiddlewareManager, self)._add_middleware(mw)
+        super()._add_middleware(mw)
         if hasattr(mw, 'process_spider_input'):
             self.methods['process_spider_input'].append(mw.process_spider_input)
         if hasattr(mw, 'process_start_requests'):
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 6f43771e2..48f19424c 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -277,7 +277,7 @@ class CrawlerProcess(CrawlerRunner):
     """
 
     def __init__(self, settings=None, install_root_handler=True):
-        super(CrawlerProcess, self).__init__(settings)
+        super().__init__(settings)
         install_shutdown_handlers(self._signal_shutdown)
         configure_logging(self.settings, install_root_handler)
         log_scrapy_info(self.settings)
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 366d60dcb..4053fecc5 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -92,7 +92,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
     enabled_setting = 'METAREFRESH_ENABLED'
 
     def __init__(self, settings):
-        super(MetaRefreshMiddleware, self).__init__(settings)
+        super().__init__(settings)
         self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
         self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')
 
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 45f152321..0c410f035 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -37,7 +37,7 @@ class CloseSpider(Exception):
     """Raise this from callbacks to request the spider to be closed"""
 
     def __init__(self, reason='cancelled'):
-        super(CloseSpider, self).__init__()
+        super().__init__()
         self.reason = reason
 
 
@@ -74,7 +74,7 @@ class UsageError(Exception):
 
     def __init__(self, *a, **kw):
         self.print_help = kw.pop('print_help', True)
-        super(UsageError, self).__init__(*a, **kw)
+        super().__init__(*a, **kw)
 
 
 class ScrapyDeprecationWarning(Warning):
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 0aba1c904..95518b3ac 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -301,7 +301,7 @@ class PythonItemExporter(BaseItemExporter):
 
     def _configure(self, options, dont_fail=False):
         self.binary = options.pop('binary', True)
-        super(PythonItemExporter, self)._configure(options, dont_fail)
+        super()._configure(options, dont_fail)
         if self.binary:
             warnings.warn(
                 "PythonItemExporter will drop support for binary export in the future",
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index dcaaeddfa..6bf9e5346 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -8,7 +8,7 @@ class Headers(CaselessDict):
 
     def __init__(self, seq=None, encoding='utf-8'):
         self.encoding = encoding
-        super(Headers, self).__init__(seq)
+        super().__init__(seq)
 
     def normkey(self, key):
         """Normalize key to bytes"""
@@ -37,19 +37,19 @@ class Headers(CaselessDict):
 
     def __getitem__(self, key):
         try:
-            return super(Headers, self).__getitem__(key)[-1]
+            return super().__getitem__(key)[-1]
         except IndexError:
             return None
 
     def get(self, key, def_val=None):
         try:
-            return super(Headers, self).get(key, def_val)[-1]
+            return super().get(key, def_val)[-1]
         except IndexError:
             return None
 
     def getlist(self, key, def_val=None):
         try:
-            return super(Headers, self).__getitem__(key)
+            return super().__getitem__(key)
         except KeyError:
             if def_val is not None:
                 return self.normvalue(def_val)
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index a260798ac..59af81321 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -24,7 +24,7 @@ class FormRequest(Request):
         if formdata and kwargs.get('method') is None:
             kwargs['method'] = 'POST'
 
-        super(FormRequest, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
         if formdata:
             items = formdata.items() if isinstance(formdata, dict) else formdata
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index f08b25280..eae3f9f6b 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -32,7 +32,7 @@ class JsonRequest(Request):
             if 'method' not in kwargs:
                 kwargs['method'] = 'POST'
 
-        super(JsonRequest, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.headers.setdefault('Content-Type', 'application/json')
         self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
 
@@ -47,7 +47,7 @@ class JsonRequest(Request):
         elif not body_passed and data_passed:
             kwargs['body'] = self._dumps(data)
 
-        return super(JsonRequest, self).replace(*args, **kwargs)
+        return super().replace(*args, **kwargs)
 
     def _dumps(self, data):
         """Convert to JSON """
diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py
index 811d3ad6b..c70912e49 100644
--- a/scrapy/http/request/rpc.py
+++ b/scrapy/http/request/rpc.py
@@ -31,5 +31,5 @@ class XmlRpcRequest(Request):
         if encoding is not None:
             kwargs['encoding'] = encoding
 
-        super(XmlRpcRequest, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.headers.setdefault('Content-Type', 'text/xml')
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 0f300c8da..a7bb34d48 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -35,13 +35,13 @@ class TextResponse(Response):
         self._cached_benc = None
         self._cached_ubody = None
         self._cached_selector = None
-        super(TextResponse, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
     def _set_url(self, url):
         if isinstance(url, str):
             self._url = to_unicode(url, self.encoding)
         else:
-            super(TextResponse, self)._set_url(url)
+            super()._set_url(url)
 
     def _set_body(self, body):
         self._body = b''  # used by encoding detection
@@ -51,7 +51,7 @@ class TextResponse(Response):
                                 type(self).__name__)
             self._body = body.encode(self._encoding)
         else:
-            super(TextResponse, self)._set_body(body)
+            super()._set_body(body)
 
     def replace(self, *args, **kwargs):
         kwargs.setdefault('encoding', self.encoding)
@@ -166,7 +166,7 @@ class TextResponse(Response):
         elif isinstance(url, parsel.SelectorList):
             raise ValueError("SelectorList is not supported")
         encoding = self.encoding if encoding is None else encoding
-        return super(TextResponse, self).follow(
+        return super().follow(
             url=url,
             callback=callback,
             method=method,
@@ -226,7 +226,7 @@ class TextResponse(Response):
             for sel in selectors:
                 with suppress(_InvalidSelector):
                     urls.append(_url_from_selector(sel))
-        return super(TextResponse, self).follow_all(
+        return super().follow_all(
             urls=urls,
             callback=callback,
             method=method,
diff --git a/scrapy/item.py b/scrapy/item.py
index 4ab83d1a0..c262a153c 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -39,7 +39,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
         if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
             warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
                  ScrapyDeprecationWarning, stacklevel=2)
-        return super(BaseItem, cls).__new__(cls, *args, **kwargs)
+        return super().__new__(cls, *args, **kwargs)
 
 
 class Field(dict):
@@ -55,7 +55,7 @@ class ItemMeta(_BaseItemMeta):
     def __new__(mcs, class_name, bases, attrs):
         classcell = attrs.pop('__classcell__', None)
         new_bases = tuple(base._class for base in bases if hasattr(base, '_class'))
-        _class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs)
+        _class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs)
 
         fields = getattr(_class, 'fields', {})
         new_attrs = {}
@@ -70,7 +70,7 @@ class ItemMeta(_BaseItemMeta):
         new_attrs['_class'] = _class
         if classcell is not None:
             new_attrs['__classcell__'] = classcell
-        return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs)
+        return super().__new__(mcs, class_name, bases, new_attrs)
 
 
 class DictItem(MutableMapping, BaseItem):
@@ -81,7 +81,7 @@ class DictItem(MutableMapping, BaseItem):
         if issubclass(cls, DictItem) and not issubclass(cls, Item):
             warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
                  ScrapyDeprecationWarning, stacklevel=2)
-        return super(DictItem, cls).__new__(cls, *args, **kwargs)
+        return super().__new__(cls, *args, **kwargs)
 
     def __init__(self, *args, **kwargs):
         self._values = {}
@@ -109,7 +109,7 @@ class DictItem(MutableMapping, BaseItem):
     def __setattr__(self, name, value):
         if not name.startswith('_'):
             raise AttributeError("Use item[%r] = %r to set field value" % (name, value))
-        super(DictItem, self).__setattr__(name, value)
+        super().__setattr__(name, value)
 
     def __len__(self):
         return len(self._values)
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 984a5c4e1..08a6ca1e8 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -65,7 +65,7 @@ class FilteringLinkExtractor:
             warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
                  'please use scrapy.linkextractors.LinkExtractor instead',
                  ScrapyDeprecationWarning, stacklevel=2)
-        return super(FilteringLinkExtractor, cls).__new__(cls)
+        return super().__new__(cls)
 
     def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
                  restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 8b9f961ee..e941c4321 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -126,7 +126,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
             strip=strip,
             canonicalized=canonicalize
         )
-        super(LxmlLinkExtractor, self).__init__(
+        super().__init__(
             link_extractor=lx,
             allow=allow,
             deny=deny,
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 487382a38..6bc5d46eb 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -376,7 +376,7 @@ class FilesPipeline(MediaPipeline):
             resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD
         )
 
-        super(FilesPipeline, self).__init__(download_func=download_func, settings=settings)
+        super().__init__(download_func=download_func, settings=settings)
 
     @classmethod
     def from_settings(cls, settings):
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 46f2bfb58..e2dd70215 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -45,8 +45,7 @@ class ImagesPipeline(FilesPipeline):
     DEFAULT_IMAGES_RESULT_FIELD = 'images'
 
     def __init__(self, store_uri, download_func=None, settings=None):
-        super(ImagesPipeline, self).__init__(store_uri, settings=settings,
-                                             download_func=download_func)
+        super().__init__(store_uri, settings=settings, download_func=download_func)
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index f69894b1e..f191deac6 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -17,7 +17,7 @@ class CachingThreadedResolver(ThreadedResolver):
     """
 
     def __init__(self, reactor, cache_size, timeout):
-        super(CachingThreadedResolver, self).__init__(reactor)
+        super().__init__(reactor)
         dnscache.limit = cache_size
         self.timeout = timeout
 
@@ -40,7 +40,7 @@ class CachingThreadedResolver(ThreadedResolver):
         # so the input argument above is simply overridden
         # to enforce Scrapy's DNS_TIMEOUT setting's value
         timeout = (self.timeout,)
-        d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
+        d = super().getHostByName(name, timeout)
         if dnscache.limit:
             d.addCallback(self._cache_result, name)
         return d
@@ -80,16 +80,16 @@ class CachingHostnameResolver:
         class CachingResolutionReceiver(resolutionReceiver):
 
             def resolutionBegan(self, resolution):
-                super(CachingResolutionReceiver, self).resolutionBegan(resolution)
+                super().resolutionBegan(resolution)
                 self.resolution = resolution
                 self.resolved = False
 
             def addressResolved(self, address):
-                super(CachingResolutionReceiver, self).addressResolved(address)
+                super().addressResolved(address)
                 self.resolved = True
 
             def resolutionComplete(self):
-                super(CachingResolutionReceiver, self).resolutionComplete()
+                super().resolutionComplete()
                 if self.resolved:
                     dnscache[hostName] = self.resolution
 
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 85a9bb526..f12c61081 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -79,4 +79,4 @@ class Selector(_ParselSelector, object_ref):
             kwargs.setdefault('base_url', response.url)
 
         self.response = response
-        super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
+        super().__init__(text=text, type=st, root=root, **kwargs)
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index ff8317cd1..b8ae32d7c 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -439,7 +439,7 @@ class Settings(BaseSettings):
         # Do not pass kwarg values here. We don't want to promote user-defined
         # dicts, and we want to update, not replace, default dicts with the
         # values given by the user
-        super(Settings, self).__init__()
+        super().__init__()
         self.setmodule(default_settings, 'default')
         # Promote default dictionaries to BaseSettings instances for per-key
         # priorities
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index 375042340..db9d0f2ae 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -15,7 +15,7 @@ class HttpError(IgnoreRequest):
 
     def __init__(self, response, *args, **kwargs):
         self.response = response
-        super(HttpError, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
 
 class HttpErrorMiddleware:
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index cb7260892..c9fbce08d 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -75,7 +75,7 @@ class CrawlSpider(Spider):
     rules = ()
 
     def __init__(self, *a, **kw):
-        super(CrawlSpider, self).__init__(*a, **kw)
+        super().__init__(*a, **kw)
         self._compile_rules()
 
     def _parse(self, response, **kwargs):
@@ -145,6 +145,6 @@ class CrawlSpider(Spider):
 
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):
-        spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)
+        spider = super().from_crawler(crawler, *args, **kwargs)
         spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
         return spider
diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py
index fd41133ea..fe8c94e78 100644
--- a/scrapy/spiders/init.py
+++ b/scrapy/spiders/init.py
@@ -6,7 +6,7 @@ class InitSpider(Spider):
     """Base Spider with initialization facilities"""
 
     def start_requests(self):
-        self._postinit_reqs = super(InitSpider, self).start_requests()
+        self._postinit_reqs = super().start_requests()
         return iterate_spider_output(self.init_request())
 
     def initialized(self, response=None):
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index c5360bfa7..1f72e76b7 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -18,7 +18,7 @@ class SitemapSpider(Spider):
     sitemap_alternate_links = False
 
     def __init__(self, *a, **kw):
-        super(SitemapSpider, self).__init__(*a, **kw)
+        super().__init__(*a, **kw)
         self._cbs = []
         for r, c in self.sitemap_rules:
             if isinstance(c, str):
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index c7ad4d53d..77ffda6f7 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -20,7 +20,7 @@ def _with_mkdir(queue_class):
             if not os.path.exists(dirname):
                 os.makedirs(dirname, exist_ok=True)
 
-            super(DirectoriesCreated, self).__init__(path, *args, **kwargs)
+            super().__init__(path, *args, **kwargs)
 
     return DirectoriesCreated
 
@@ -31,10 +31,10 @@ def _serializable_queue(queue_class, serialize, deserialize):
 
         def push(self, obj):
             s = serialize(obj)
-            super(SerializableQueue, self).push(s)
+            super().push(s)
 
         def pop(self):
-            s = super(SerializableQueue, self).pop()
+            s = super().pop()
             if s:
                 return deserialize(s)
 
@@ -47,7 +47,7 @@ def _scrapy_serialization_queue(queue_class):
 
         def __init__(self, crawler, key):
             self.spider = crawler.spider
-            super(ScrapyRequestQueue, self).__init__(key)
+            super().__init__(key)
 
         @classmethod
         def from_crawler(cls, crawler, key, *args, **kwargs):
@@ -55,10 +55,10 @@ def _scrapy_serialization_queue(queue_class):
 
         def push(self, request):
             request = request_to_dict(request, self.spider)
-            return super(ScrapyRequestQueue, self).push(request)
+            return super().push(request)
 
         def pop(self):
-            request = super(ScrapyRequestQueue, self).pop()
+            request = super().pop()
 
             if not request:
                 return None
diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py
index 579c60180..ba7d1a6bf 100644
--- a/scrapy/statscollectors.py
+++ b/scrapy/statscollectors.py
@@ -54,7 +54,7 @@ class StatsCollector:
 class MemoryStatsCollector(StatsCollector):
 
     def __init__(self, crawler):
-        super(MemoryStatsCollector, self).__init__(crawler)
+        super().__init__(crawler)
         self.spider_stats = {}
 
     def _persist_stats(self, stats, spider):
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 2a92d0588..e31284a7f 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -15,7 +15,7 @@ class CaselessDict(dict):
     __slots__ = ()
 
     def __init__(self, seq=None):
-        super(CaselessDict, self).__init__()
+        super().__init__()
         if seq:
             self.update(seq)
 
@@ -53,7 +53,7 @@ class CaselessDict(dict):
     def update(self, seq):
         seq = seq.items() if isinstance(seq, Mapping) else seq
         iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
-        super(CaselessDict, self).update(iseq)
+        super().update(iseq)
 
     @classmethod
     def fromkeys(cls, keys, value=None):
@@ -70,14 +70,14 @@ class LocalCache(collections.OrderedDict):
     """
 
     def __init__(self, limit=None):
-        super(LocalCache, self).__init__()
+        super().__init__()
         self.limit = limit
 
     def __setitem__(self, key, value):
         if self.limit:
             while len(self) >= self.limit:
                 self.popitem(last=False)
-        super(LocalCache, self).__setitem__(key, value)
+        super().__setitem__(key, value)
 
 
 class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
@@ -93,18 +93,18 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
     """
 
     def __init__(self, limit=None):
-        super(LocalWeakReferencedCache, self).__init__()
+        super().__init__()
         self.data = LocalCache(limit=limit)
 
     def __setitem__(self, key, value):
         try:
-            super(LocalWeakReferencedCache, self).__setitem__(key, value)
+            super().__setitem__(key, value)
         except TypeError:
             pass  # key is not weak-referenceable, skip caching
 
     def __getitem__(self, key):
         try:
-            return super(LocalWeakReferencedCache, self).__getitem__(key)
+            return super().__getitem__(key)
         except (TypeError, KeyError):
             return None  # key is either not weak-referenceable or not cached
 
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index 3dbea5fee..3c8e3c8b5 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -57,7 +57,7 @@ def create_deprecated_class(
         warned_on_subclass = False
 
         def __new__(metacls, name, bases, clsdict_):
-            cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_)
+            cls = super().__new__(metacls, name, bases, clsdict_)
             if metacls.deprecated_class is None:
                 metacls.deprecated_class = cls
             return cls
@@ -73,7 +73,7 @@ def create_deprecated_class(
                 if warn_once:
                     msg += ' (warning only on first subclass, there may be others)'
                 warnings.warn(msg, warn_category, stacklevel=2)
-            super(DeprecatedClass, cls).__init__(name, bases, clsdict_)
+            super().__init__(name, bases, clsdict_)
 
         # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass
         # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks
@@ -88,7 +88,7 @@ def create_deprecated_class(
                 # is the deprecated class itself - subclasses of the
                 # deprecated class should not use custom `__subclasscheck__`
                 # method.
-                return super(DeprecatedClass, cls).__subclasscheck__(sub)
+                return super().__subclasscheck__(sub)
 
             if not inspect.isclass(sub):
                 raise TypeError("issubclass() arg 1 must be a class")
@@ -102,7 +102,7 @@ def create_deprecated_class(
                 msg = instance_warn_message.format(cls=_clspath(cls, old_class_path),
                                                    new=_clspath(new_class, new_class_path))
                 warnings.warn(msg, warn_category, stacklevel=2)
-            return super(DeprecatedClass, cls).__call__(*args, **kwargs)
+            return super().__call__(*args, **kwargs)
 
     deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {})
 
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 51d276097..1d6a2c39d 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -176,7 +176,7 @@ class LogCounterHandler(logging.Handler):
     """Record log levels count into a crawler stats"""
 
     def __init__(self, crawler, *args, **kwargs):
-        super(LogCounterHandler, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.crawler = crawler
 
     def emit(self, record):
diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py
index dc9604578..cc3263602 100644
--- a/scrapy/utils/serialize.py
+++ b/scrapy/utils/serialize.py
@@ -33,7 +33,7 @@ class ScrapyJSONEncoder(json.JSONEncoder):
         elif isinstance(o, Response):
             return "<%s %s %s>" % (type(o).__name__, o.status, o.url)
         else:
-            return super(ScrapyJSONEncoder, self).default(o)
+            return super().default(o)
 
 
 class ScrapyJSONDecoder(json.JSONDecoder):
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index 66930ad2c..397e54703 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -7,12 +7,12 @@ class SiteTest:
 
     def setUp(self):
         from twisted.internet import reactor
-        super(SiteTest, self).setUp()
+        super().setUp()
         self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
         self.baseurl = "http://localhost:%d/" % self.site.getHost().port
 
     def tearDown(self):
-        super(SiteTest, self).tearDown()
+        super().tearDown()
         self.site.stopListening()
 
     def url(self, path):
diff --git a/tests/spiders.py b/tests/spiders.py
index 3eb681819..63bd726fb 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -19,7 +19,7 @@ from scrapy.utils.test import get_from_asyncio_queue
 
 class MockServerSpider(Spider):
     def __init__(self, mockserver=None, *args, **kwargs):
-        super(MockServerSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.mockserver = mockserver
 
 
@@ -28,7 +28,7 @@ class MetaSpider(MockServerSpider):
     name = 'meta'
 
     def __init__(self, *args, **kwargs):
-        super(MetaSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.meta = {}
 
     def closed(self, reason):
@@ -41,7 +41,7 @@ class FollowAllSpider(MetaSpider):
     link_extractor = LinkExtractor()
 
     def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs):
-        super(FollowAllSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.urls_visited = []
         self.times = []
         qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency}
@@ -60,7 +60,7 @@ class DelaySpider(MetaSpider):
     name = 'delay'
 
     def __init__(self, n=1, b=0, *args, **kwargs):
-        super(DelaySpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.n = n
         self.b = b
         self.t1 = self.t2 = self.t2_err = 0
@@ -82,7 +82,7 @@ class SimpleSpider(MetaSpider):
     name = 'simple'
 
     def __init__(self, url="http://localhost:8998", *args, **kwargs):
-        super(SimpleSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.start_urls = [url]
 
     def parse(self, response):
@@ -153,7 +153,7 @@ class ItemSpider(FollowAllSpider):
     name = 'item'
 
     def parse(self, response):
-        for request in super(ItemSpider, self).parse(response):
+        for request in super().parse(response):
             yield request
             yield Item()
             yield {}
@@ -172,7 +172,7 @@ class ErrorSpider(FollowAllSpider):
         raise self.exception_cls('Expected exception')
 
     def parse(self, response):
-        for request in super(ErrorSpider, self).parse(response):
+        for request in super().parse(response):
             yield request
             self.raise_exception()
 
@@ -183,7 +183,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
     fail_yielding = False
 
     def __init__(self, *a, **kw):
-        super(BrokenStartRequestsSpider, self).__init__(*a, **kw)
+        super().__init__(*a, **kw)
         self.seedsseen = []
 
     def start_requests(self):
@@ -201,7 +201,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
 
     def parse(self, response):
         self.seedsseen.append(response.meta.get('seed'))
-        for req in super(BrokenStartRequestsSpider, self).parse(response):
+        for req in super().parse(response):
             yield req
 
 
@@ -243,7 +243,7 @@ class DuplicateStartRequestsSpider(MockServerSpider):
                 yield Request(url, dont_filter=self.dont_filter)
 
     def __init__(self, url="http://localhost:8998", *args, **kwargs):
-        super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.visited = 0
 
     def parse(self, response):
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 5754a5478..e115f420f 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -17,7 +17,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
     command = 'parse'
 
     def setUp(self):
-        super(ParseCommandTest, self).setUp()
+        super().setUp()
         self.spider_name = 'parse_spider'
         fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
         with open(fname, 'w') as f:
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 42091ab00..8938156fc 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -151,7 +151,7 @@ def get_permissions_dict(path, renamings=None, ignore=None):
 class StartprojectTemplatesTest(ProjectTest):
 
     def setUp(self):
-        super(StartprojectTemplatesTest, self).setUp()
+        super().setUp()
         self.tmpl = join(self.temp_path, 'templates')
         self.tmpl_proj = join(self.tmpl, 'project')
 
@@ -315,7 +315,7 @@ class StartprojectTemplatesTest(ProjectTest):
 class CommandTest(ProjectTest):
 
     def setUp(self):
-        super(CommandTest, self).setUp()
+        super().setUp()
         self.call('startproject', self.project_name)
         self.cwd = join(self.temp_path, self.project_name)
         self.env['SCRAPY_SETTINGS_MODULE'] = '%s.settings' % self.project_name
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 99120b128..2e7e3ccc4 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -378,7 +378,7 @@ class ContractsManagerTest(unittest.TestCase):
             name = 'test_same_url'
 
             def __init__(self, *args, **kwargs):
-                super(TestSameUrlSpider, self).__init__(*args, **kwargs)
+                super().__init__(*args, **kwargs)
                 self.visited = 0
 
             def start_requests(s):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 57d4cdd6b..13063d106 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -530,7 +530,7 @@ class Https11InvalidDNSId(Https11TestCase):
     """Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
 
     def setUp(self):
-        super(Https11InvalidDNSId, self).setUp()
+        super().setUp()
         self.host = '127.0.0.1'
 
 
@@ -549,7 +549,7 @@ class Https11InvalidDNSPattern(Https11TestCase):
             'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
             'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
         )
-        super(Https11InvalidDNSPattern, self).setUp()
+        super().setUp()
 
 
 class Https11CustomCiphers(unittest.TestCase):
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 9b77c97a8..299fb0eb8 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -134,7 +134,7 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):
 
     def _get_settings(self, **new_settings):
         new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module)
-        return super(DbmStorageWithCustomDbmModuleTest, self)._get_settings(**new_settings)
+        return super()._get_settings(**new_settings)
 
     def test_custom_dbm_module_loaded(self):
         # make sure our dbm module has been loaded
@@ -151,7 +151,7 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
 
     def _get_settings(self, **new_settings):
         new_settings.setdefault('HTTPCACHE_GZIP', True)
-        return super(FilesystemStorageTest, self)._get_settings(**new_settings)
+        return super()._get_settings(**new_settings)
 
 
 class DummyPolicyTest(_BaseTest):
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index f9936baba..858138f81 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -189,7 +189,7 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
         skip = "Rerp parser is not installed"
 
     def setUp(self):
-        super(RobotsTxtMiddlewareWithRerpTest, self).setUp()
+        super().setUp()
         self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser')
 
 
@@ -198,5 +198,5 @@ class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):
         skip = "Reppy parser is not installed"
 
     def setUp(self):
-        super(RobotsTxtMiddlewareWithReppyTest, self).setUp()
+        super().setUp()
         self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser')
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 660c99ce1..6c25a0064 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -593,7 +593,7 @@ class CustomExporterItemTest(unittest.TestCase):
                 if name == 'age':
                     return str(int(value) + 1)
                 else:
-                    return super(CustomItemExporter, self).serialize_field(field, name, value)
+                    return super().serialize_field(field, name, value)
 
         i = self.item_class(name='John', age='22')
         a = ItemAdapter(i)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index f5cf4e798..0a303dbe2 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1265,7 +1265,7 @@ class JsonRequestTest(RequestTest):
 
     def setUp(self):
         warnings.simplefilter("always")
-        super(JsonRequestTest, self).setUp()
+        super().setUp()
 
     def test_data(self):
         r1 = self.request_class(url="http://www.example.com/")
@@ -1419,7 +1419,7 @@ class JsonRequestTest(RequestTest):
 
     def tearDown(self):
         warnings.resetwarnings()
-        super(JsonRequestTest, self).tearDown()
+        super().tearDown()
 
 
 if __name__ == "__main__":
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 56d017de6..f831ef5dc 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -305,7 +305,7 @@ class TextResponseTest(BaseResponseTest):
     response_class = TextResponse
 
     def test_replace(self):
-        super(TextResponseTest, self).test_replace()
+        super().test_replace()
         r1 = self.response_class("http://www.example.com", body="hello", encoding="cp852")
         r2 = r1.replace(url="http://www.example.com/other")
         r3 = r1.replace(url="http://www.example.com/other", encoding="latin1")
diff --git a/tests/test_item.py b/tests/test_item.py
index 0ce78f8c0..66fa761f0 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -312,7 +312,7 @@ class ItemMetaClassCellRegression(unittest.TestCase):
                 # requirement. When not done properly raises an error:
                 # TypeError: __class__ set to <class '__main__.MyItem'>
                 # defining 'MyItem' as <class '__main__.MyItem'>
-                super(MyItem, self).__init__(*args, **kwargs)
+                super().__init__(*args, **kwargs)
 
 
 class DictItemTest(unittest.TestCase):
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index a0bafa5e5..6f133d77a 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -516,7 +516,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
         ])
 
     def test_restrict_xpaths_with_html_entities(self):
-        super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()
+        super().test_restrict_xpaths_with_html_entities()
 
     def test_filteringlinkextractor_deprecation_warning(self):
         """Make sure the FilteringLinkExtractor deprecation warning is not
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 2ed6f365f..b0bc82f4e 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -250,7 +250,7 @@ class TestOutputProcessorItem(unittest.TestCase):
             temp = Field()
 
             def __init__(self, *args, **kwargs):
-                super(TempItem, self).__init__(self, *args, **kwargs)
+                super().__init__(self, *args, **kwargs)
                 self.setdefault('temp', 0.3)
 
         class TempLoader(ItemLoader):
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index eb14de14f..624dd9ab8 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -579,7 +579,7 @@ class TestOutputProcessorDict(unittest.TestCase):
 
         class TempDict(dict):
             def __init__(self, *args, **kwargs):
-                super(TempDict, self).__init__(self, *args, **kwargs)
+                super().__init__(self, *args, **kwargs)
                 self.setdefault('temp', 0.3)
 
         class TempLoader(ItemLoader):
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index b771e7d79..41ff3651d 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -118,7 +118,7 @@ class LogFormatterTestCase(unittest.TestCase):
 
 class LogFormatterSubclass(LogFormatter):
     def crawled(self, request, response, spider):
-        kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider)
+        kwargs = super().crawled(request, response, spider)
         CRAWLEDMSG = (
             "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
         )
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index 3364d2258..b2b75ef20 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -53,7 +53,7 @@ class TestMiddlewareManager(MiddlewareManager):
         return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']]
 
     def _add_middleware(self, mw):
-        super(TestMiddlewareManager, self)._add_middleware(mw)
+        super()._add_middleware(mw)
         if hasattr(mw, 'process'):
             self.methods['process'].append(mw.process)
 
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 19ff00350..4f130c0c9 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -162,18 +162,18 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 class MockedMediaPipeline(MediaPipeline):
 
     def __init__(self, *args, **kwargs):
-        super(MockedMediaPipeline, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._mockcalled = []
 
     def download(self, request, info):
         self._mockcalled.append('download')
-        return super(MockedMediaPipeline, self).download(request, info)
+        return super().download(request, info)
 
     def media_to_download(self, request, info):
         self._mockcalled.append('media_to_download')
         if 'result' in request.meta:
             return request.meta.get('result')
-        return super(MockedMediaPipeline, self).media_to_download(request, info)
+        return super().media_to_download(request, info)
 
     def get_media_requests(self, item, info):
         self._mockcalled.append('get_media_requests')
@@ -181,15 +181,15 @@ class MockedMediaPipeline(MediaPipeline):
 
     def media_downloaded(self, response, request, info):
         self._mockcalled.append('media_downloaded')
-        return super(MockedMediaPipeline, self).media_downloaded(response, request, info)
+        return super().media_downloaded(response, request, info)
 
     def media_failed(self, failure, request, info):
         self._mockcalled.append('media_failed')
-        return super(MockedMediaPipeline, self).media_failed(failure, request, info)
+        return super().media_failed(failure, request, info)
 
     def item_completed(self, results, item, info):
         self._mockcalled.append('item_completed')
-        item = super(MockedMediaPipeline, self).item_completed(results, item, info)
+        item = super().item_completed(results, item, info)
         item['results'] = results
         return item
 
diff --git a/tests/test_request_left.py b/tests/test_request_left.py
index 5cfef8e7d..373b2e49c 100644
--- a/tests/test_request_left.py
+++ b/tests/test_request_left.py
@@ -10,7 +10,7 @@ class SignalCatcherSpider(Spider):
     name = 'signal_catcher'
 
     def __init__(self, crawler, url, *args, **kwargs):
-        super(SignalCatcherSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         crawler.signals.connect(self.on_request_left,
                                 signal=request_left_downloader)
         self.caught_times = 0
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 9d8c201dd..4b15d0fab 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -117,7 +117,7 @@ class BaseRobotParserTest:
 class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
     def setUp(self):
         from scrapy.robotstxt import PythonRobotParser
-        super(PythonRobotParserTest, self)._setUp(PythonRobotParser)
+        super()._setUp(PythonRobotParser)
 
     def test_length_based_precedence(self):
         raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.")
@@ -132,7 +132,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
 
     def setUp(self):
         from scrapy.robotstxt import ReppyRobotParser
-        super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser)
+        super()._setUp(ReppyRobotParser)
 
     def test_order_based_precedence(self):
         raise unittest.SkipTest("Reppy does not support order based directives precedence.")
@@ -144,7 +144,7 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
 
     def setUp(self):
         from scrapy.robotstxt import RerpRobotParser
-        super(RerpRobotParserTest, self)._setUp(RerpRobotParser)
+        super()._setUp(RerpRobotParser)
 
     def test_length_based_precedence(self):
         raise unittest.SkipTest("Rerp does not support length based directives precedence.")
@@ -156,7 +156,7 @@ class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase):
 
     def setUp(self):
         from scrapy.robotstxt import ProtegoRobotParser
-        super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser)
+        super()._setUp(ProtegoRobotParser)
 
     def test_order_based_precedence(self):
         raise unittest.SkipTest("Protego does not support order based directives precedence.")
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 2b6cb0902..512a7460e 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -53,7 +53,7 @@ class MockCrawler(Crawler):
             JOBDIR=jobdir,
             DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
         )
-        super(MockCrawler, self).__init__(Spider, settings)
+        super().__init__(Spider, settings)
         self.engine = MockEngine(downloader=MockDownloader())
 
 
@@ -296,7 +296,7 @@ class StartUrlsSpider(Spider):
 
     def __init__(self, start_urls):
         self.start_urls = start_urls
-        super(StartUrlsSpider, self).__init__(name='StartUrlsSpider')
+        super().__init__(name='StartUrlsSpider')
 
     def parse(self, response):
         pass
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index e032b247c..e449cd706 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -19,7 +19,7 @@ class _HttpErrorSpider(MockServerSpider):
     bypass_status_codes = set()
 
     def __init__(self, *args, **kwargs):
-        super(_HttpErrorSpider, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.start_urls = [
             self.mockserver.url("/status?n=200"),
             self.mockserver.url("/status?n=404"),

From 9d84289109b2368d5929d8b60ce583529c19fe4c Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ikshitijsharma@gmail.com>
Date: Wed, 5 Aug 2020 09:11:59 +0530
Subject: [PATCH 2954/2974] deprecated weakkeycache by specifying in __init__

---
 scrapy/utils/python.py     |  4 +++-
 scrapy/utils/tester.py     |  3 +++
 tests/test_utils_python.py | 22 +++++++++++++++++++++-
 3 files changed, 27 insertions(+), 2 deletions(-)
 create mode 100644 scrapy/utils/tester.py

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index c8f921ff3..4756b07b6 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -7,10 +7,12 @@ import inspect
 import re
 import sys
 import weakref
+import warnings
 from functools import partial, wraps
 from itertools import chain
 
 from scrapy.utils.decorators import deprecated
+from scrapy.exceptions import ScrapyDeprecationWarning
 
 
 def flatten(x):
@@ -275,10 +277,10 @@ def equal_attributes(obj1, obj2, attributes):
     return True
 
 
-@deprecated
 class WeakKeyCache:
 
     def __init__(self, default_factory):
+        warnings.warn("Call to deprecated Class WeakKeyCache", category=ScrapyDeprecationWarning, stacklevel=2)
         self.default_factory = default_factory
         self._weakdict = weakref.WeakKeyDictionary()
 
diff --git a/scrapy/utils/tester.py b/scrapy/utils/tester.py
new file mode 100644
index 000000000..691e9bc1a
--- /dev/null
+++ b/scrapy/utils/tester.py
@@ -0,0 +1,3 @@
+from scrapy.utils.decorators import deprecated
+
+
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 5a53d89e4..ebce3c079 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -1,15 +1,18 @@
 import functools
+import gc
 import operator
 import platform
 import unittest
+from itertools import count
 from sys import version_info
 from warnings import catch_warnings
 
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
-    get_func_args, to_bytes, to_unicode,
+    WeakKeyCache, get_func_args, to_bytes, to_unicode,
     without_none_values, MutableChain)
 
+
 __doctests__ = ['scrapy.utils.python']
 
 
@@ -152,6 +155,23 @@ class UtilsPythonTestCase(unittest.TestCase):
         a.meta['z'] = 2
         self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
 
+    def test_weakkeycache(self):
+        class _Weakme:
+            pass
+
+        _values = count()
+        wk = WeakKeyCache(lambda k: next(_values))
+        k = _Weakme()
+        v = wk[k]
+        self.assertEqual(v, wk[k])
+        self.assertNotEqual(v, wk[_Weakme()])
+        self.assertEqual(v, wk[k])
+        del k
+        for _ in range(100):
+            if wk._weakdict:
+                gc.collect()
+        self.assertFalse(len(wk._weakdict))
+
     def test_get_func_args(self):
         def f1(a, b, c):
             pass

From b35d1f2b2c430f4d12cb8f9d408dfa0c0051746d Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ikshitijsharma@gmail.com>
Date: Wed, 5 Aug 2020 09:14:04 +0530
Subject: [PATCH 2955/2974] deleted tester.py

---
 scrapy/utils/tester.py | 3 ---
 1 file changed, 3 deletions(-)
 delete mode 100644 scrapy/utils/tester.py

diff --git a/scrapy/utils/tester.py b/scrapy/utils/tester.py
deleted file mode 100644
index 691e9bc1a..000000000
--- a/scrapy/utils/tester.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from scrapy.utils.decorators import deprecated
-
-

From 983b7ddf2e39c480efc6d104054f92f570714ac8 Mon Sep 17 00:00:00 2001
From: Kshitij Sharma <ikshitijsharma@gmail.com>
Date: Wed, 5 Aug 2020 16:13:52 +0530
Subject: [PATCH 2956/2974] aesthetic fixes

---
 scrapy/utils/python.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 4756b07b6..59f1b8371 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -6,13 +6,13 @@ import gc
 import inspect
 import re
 import sys
-import weakref
 import warnings
+import weakref
 from functools import partial, wraps
 from itertools import chain
 
-from scrapy.utils.decorators import deprecated
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.decorators import deprecated
 
 
 def flatten(x):
@@ -280,7 +280,7 @@ def equal_attributes(obj1, obj2, attributes):
 class WeakKeyCache:
 
     def __init__(self, default_factory):
-        warnings.warn("Call to deprecated Class WeakKeyCache", category=ScrapyDeprecationWarning, stacklevel=2)
+        warnings.warn("The WeakKeyCache class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2)
         self.default_factory = default_factory
         self._weakdict = weakref.WeakKeyDictionary()
 

From 4dc09f09aa9698b02f2cbf2e3001202388eba043 Mon Sep 17 00:00:00 2001
From: linchiwei123 <40888469+linchiwei123@users.noreply.github.com>
Date: Wed, 5 Aug 2020 22:23:19 +0800
Subject: [PATCH 2957/2974] Update setup.py

---
 setup.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/setup.py b/setup.py
index 58090f7a2..d0880051f 100644
--- a/setup.py
+++ b/setup.py
@@ -23,7 +23,6 @@ install_requires = [
     'cryptography>=2.0',
     'cssselect>=0.9.1',
     'itemloaders>=1.0.1',
-    'lxml>=3.5.0',
     'parsel>=1.5.0',
     'PyDispatcher>=2.0.5',
     'pyOpenSSL>=16.2.0',

From aefd43a6c6b8d2e32dfb9bd0c94529805ce5037e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 11 Aug 2020 12:52:54 +0200
Subject: [PATCH 2958/2974] Upgrade minimum dependencies for Python 3.6 support

---
 tests/requirements-py3.txt | 3 +--
 tox.ini                    | 4 ++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 00c56084d..4fa58d11b 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,8 +1,7 @@
 # Tests requirements
 attrs
 dataclasses; python_version == '3.6'
-mitmproxy; python_version >= '3.6'
-mitmproxy<4.0.0; python_version < '3.6'
+mitmproxy >=4, <5  # The latest version does not support some pinned dependencies
 # https://github.com/pytest-dev/pytest-twisted/issues/93
 pytest != 5.4, != 5.4.1
 pytest-azurepipelines
diff --git a/tox.ini b/tox.ini
index 11882c03f..4f5531aea 100644
--- a/tox.ini
+++ b/tox.ini
@@ -13,7 +13,7 @@ deps =
     -rtests/requirements-py3.txt
     # Extras
     boto3>=1.13.0
-    botocore>=1.3.23
+    botocore>=1.4.87
     Pillow>=3.4.2
 passenv =
     S3_TEST_FILE_URI
@@ -76,7 +76,7 @@ deps =
     zope.interface==4.1.3
     -rtests/requirements-py3.txt
     # Extras
-    botocore==1.3.23
+    botocore==1.4.87
     google-cloud-storage==1.29.0
     Pillow==3.4.2
 

From 1c4b4cc6b046e121be33a57031a0900ca1347e16 Mon Sep 17 00:00:00 2001
From: Ajay Mittur <ajay.cs18@bmsce.ac.in>
Date: Tue, 11 Aug 2020 17:42:44 +0530
Subject: [PATCH 2959/2974] Support defining file path based on item in media
 pipelines (#4686)

---
 docs/topics/media-pipeline.rst |  24 ++++---
 scrapy/pipelines/files.py      |  16 ++---
 scrapy/pipelines/images.py     |  14 ++--
 scrapy/pipelines/media.py      |  69 +++++++++++++++---
 tests/test_pipeline_files.py   |  13 ++++
 tests/test_pipeline_media.py   | 123 ++++++++++++++++++++++++++++++++-
 6 files changed, 225 insertions(+), 34 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 1f995ce14..487e26b8e 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -412,15 +412,16 @@ See here the methods that you can override in your custom Files Pipeline:
 
 .. class:: FilesPipeline
 
-   .. method:: file_path(self, request, response=None, info=None)
+   .. method:: file_path(self, request, response=None, info=None, *, item=None)
 
       This method is called once per downloaded item. It returns the
       download path of the file originating from the specified
       :class:`response <scrapy.http.Response>`.
 
       In addition to ``response``, this method receives the original
-      :class:`request <scrapy.Request>` and
-      :class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
+      :class:`request <scrapy.Request>`,
+      :class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and 
+      :class:`item <scrapy.item.Item>`
 
       You can override this method to customize the download path of each file.
 
@@ -436,9 +437,12 @@ See here the methods that you can override in your custom Files Pipeline:
 
         class MyFilesPipeline(FilesPipeline):
 
-            def file_path(self, request, response=None, info=None):
+            def file_path(self, request, response=None, info=None, *, item=None):
                 return 'files/' + os.path.basename(urlparse(request.url).path)
 
+      Similarly, you can use the ``item`` to determine the file path based on some item 
+      property.
+      
       By default the :meth:`file_path` method returns
       ``full/<request URL hash>.<extension>``.
 
@@ -544,15 +548,16 @@ See here the methods that you can override in your custom Images Pipeline:
     The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
     customizing the field names and adding custom behavior for images.
 
-   .. method:: file_path(self, request, response=None, info=None)
+   .. method:: file_path(self, request, response=None, info=None, *, item=None)
 
       This method is called once per downloaded item. It returns the
       download path of the file originating from the specified
       :class:`response <scrapy.http.Response>`.
 
       In addition to ``response``, this method receives the original
-      :class:`request <scrapy.Request>` and
-      :class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
+      :class:`request <scrapy.Request>`,
+      :class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and 
+      :class:`item <scrapy.item.Item>`
 
       You can override this method to customize the download path of each file.
 
@@ -568,9 +573,12 @@ See here the methods that you can override in your custom Images Pipeline:
 
         class MyImagesPipeline(ImagesPipeline):
 
-            def file_path(self, request, response=None, info=None):
+            def file_path(self, request, response=None, info=None, *, item=None):
                 return 'files/' + os.path.basename(urlparse(request.url).path)
 
+      Similarly, you can use the ``item`` to determine the file path based on some item 
+      property.
+      
       By default the :meth:`file_path` method returns
       ``full/<request URL hash>.<extension>``.
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 6bc5d46eb..5a2184681 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -409,7 +409,7 @@ class FilesPipeline(MediaPipeline):
         store_cls = self.STORE_SCHEMES[scheme]
         return store_cls(uri)
 
-    def media_to_download(self, request, info):
+    def media_to_download(self, request, info, *, item=None):
         def _onsuccess(result):
             if not result:
                 return  # returning None force download
@@ -436,7 +436,7 @@ class FilesPipeline(MediaPipeline):
             checksum = result.get('checksum', None)
             return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'}
 
-        path = self.file_path(request, info=info)
+        path = self.file_path(request, info=info, item=item)
         dfd = defer.maybeDeferred(self.store.stat_file, path, info)
         dfd.addCallbacks(_onsuccess, lambda _: None)
         dfd.addErrback(
@@ -460,7 +460,7 @@ class FilesPipeline(MediaPipeline):
 
         raise FileException
 
-    def media_downloaded(self, response, request, info):
+    def media_downloaded(self, response, request, info, *, item=None):
         referer = referer_str(request)
 
         if response.status != 200:
@@ -492,8 +492,8 @@ class FilesPipeline(MediaPipeline):
         self.inc_stats(info.spider, status)
 
         try:
-            path = self.file_path(request, response=response, info=info)
-            checksum = self.file_downloaded(response, request, info)
+            path = self.file_path(request, response=response, info=info, item=item)
+            checksum = self.file_downloaded(response, request, info, item=item)
         except FileException as exc:
             logger.warning(
                 'File (error): Error processing file from %(request)s '
@@ -522,8 +522,8 @@ class FilesPipeline(MediaPipeline):
         urls = ItemAdapter(item).get(self.files_urls_field, [])
         return [Request(u) for u in urls]
 
-    def file_downloaded(self, response, request, info):
-        path = self.file_path(request, response=response, info=info)
+    def file_downloaded(self, response, request, info, *, item=None):
+        path = self.file_path(request, response=response, info=info, item=item)
         buf = BytesIO(response.body)
         checksum = md5sum(buf)
         buf.seek(0)
@@ -535,7 +535,7 @@ class FilesPipeline(MediaPipeline):
             ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok]
         return item
 
-    def file_path(self, request, response=None, info=None):
+    def file_path(self, request, response=None, info=None, *, item=None):
         media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         media_ext = os.path.splitext(request.url)[1]
         # Handles empty and wild extensions by trying to guess the
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index e2dd70215..0a67a0b1d 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -103,12 +103,12 @@ class ImagesPipeline(FilesPipeline):
         store_uri = settings['IMAGES_STORE']
         return cls(store_uri, settings=settings)
 
-    def file_downloaded(self, response, request, info):
-        return self.image_downloaded(response, request, info)
+    def file_downloaded(self, response, request, info, *, item=None):
+        return self.image_downloaded(response, request, info, item=item)
 
-    def image_downloaded(self, response, request, info):
+    def image_downloaded(self, response, request, info, *, item=None):
         checksum = None
-        for path, image, buf in self.get_images(response, request, info):
+        for path, image, buf in self.get_images(response, request, info, item=item):
             if checksum is None:
                 buf.seek(0)
                 checksum = md5sum(buf)
@@ -119,8 +119,8 @@ class ImagesPipeline(FilesPipeline):
                 headers={'Content-Type': 'image/jpeg'})
         return checksum
 
-    def get_images(self, response, request, info):
-        path = self.file_path(request, response=response, info=info)
+    def get_images(self, response, request, info, *, item=None):
+        path = self.file_path(request, response=response, info=info, item=item)
         orig_image = Image.open(BytesIO(response.body))
 
         width, height = orig_image.size
@@ -166,7 +166,7 @@ class ImagesPipeline(FilesPipeline):
             ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok]
         return item
 
-    def file_path(self, request, response=None, info=None):
+    def file_path(self, request, response=None, info=None, *, item=None):
         image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         return 'full/%s.jpg' % (image_guid)
 
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index aa65f4f0e..2439de9a5 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -1,12 +1,16 @@
 import functools
 import logging
 from collections import defaultdict
+from inspect import signature
+from warnings import warn
+
 from twisted.internet.defer import Deferred, DeferredList
 from twisted.python.failure import Failure
 
 from scrapy.settings import Settings
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.defer import mustbe_deferred, defer_result
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.log import failure_to_exc_info
@@ -27,6 +31,7 @@ class MediaPipeline:
 
     def __init__(self, download_func=None, settings=None):
         self.download_func = download_func
+        self._expects_item = {}
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
@@ -38,6 +43,9 @@ class MediaPipeline:
         )
         self._handle_statuses(self.allow_redirects)
 
+        # Check if deprecated methods are being used and make them compatible
+        self._make_compatible()
+
     def _handle_statuses(self, allow_redirects):
         self.handle_httpstatus_list = None
         if allow_redirects:
@@ -77,11 +85,11 @@ class MediaPipeline:
     def process_item(self, item, spider):
         info = self.spiderinfo
         requests = arg_to_iter(self.get_media_requests(item, info))
-        dlist = [self._process_request(r, info) for r in requests]
+        dlist = [self._process_request(r, info, item) for r in requests]
         dfd = DeferredList(dlist, consumeErrors=1)
         return dfd.addCallback(self.item_completed, item, info)
 
-    def _process_request(self, request, info):
+    def _process_request(self, request, info, item):
         fp = request_fingerprint(request)
         cb = request.callback or (lambda _: _)
         eb = request.errback
@@ -102,34 +110,73 @@ class MediaPipeline:
 
         # Download request checking media_to_download hook output first
         info.downloading.add(fp)
-        dfd = mustbe_deferred(self.media_to_download, request, info)
-        dfd.addCallback(self._check_media_to_download, request, info)
+        dfd = mustbe_deferred(self.media_to_download, request, info, item=item)
+        dfd.addCallback(self._check_media_to_download, request, info, item=item)
         dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
         dfd.addErrback(lambda f: logger.error(
             f.value, exc_info=failure_to_exc_info(f), extra={'spider': info.spider})
         )
         return dfd.addBoth(lambda _: wad)  # it must return wad at last
 
+    def _make_compatible(self):
+        """Make overridable methods of MediaPipeline and subclasses backwards compatible"""
+        methods = [
+            "file_path", "media_to_download", "media_downloaded",
+            "file_downloaded", "image_downloaded", "get_images"
+        ]
+
+        for method_name in methods:
+            method = getattr(self, method_name, None)
+            if callable(method):
+                setattr(self, method_name, self._compatible(method))
+
+    def _compatible(self, func):
+        """Wrapper for overridable methods to allow backwards compatibility"""
+        self._check_signature(func)
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            if self._expects_item[func.__name__]:
+                return func(*args, **kwargs)
+
+            kwargs.pop('item', None)
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    def _check_signature(self, func):
+        sig = signature(func)
+        self._expects_item[func.__name__] = True
+
+        if 'item' not in sig.parameters:
+            old_params = str(sig)[1:-1]
+            new_params = old_params + ", *, item=None"
+            warn('%s(self, %s) is deprecated, '
+                 'please use %s(self, %s)'
+                 % (func.__name__, old_params, func.__name__, new_params),
+                 ScrapyDeprecationWarning, stacklevel=2)
+            self._expects_item[func.__name__] = False
+
     def _modify_media_request(self, request):
         if self.handle_httpstatus_list:
             request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list
         else:
             request.meta['handle_httpstatus_all'] = True
 
-    def _check_media_to_download(self, result, request, info):
+    def _check_media_to_download(self, result, request, info, item):
         if result is not None:
             return result
         if self.download_func:
             # this ugly code was left only to support tests. TODO: remove
             dfd = mustbe_deferred(self.download_func, request, info.spider)
             dfd.addCallbacks(
-                callback=self.media_downloaded, callbackArgs=(request, info),
+                callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item},
                 errback=self.media_failed, errbackArgs=(request, info))
         else:
             self._modify_media_request(request)
             dfd = self.crawler.engine.download(request, info.spider)
             dfd.addCallbacks(
-                callback=self.media_downloaded, callbackArgs=(request, info),
+                callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item},
                 errback=self.media_failed, errbackArgs=(request, info))
         return dfd
 
@@ -171,7 +218,7 @@ class MediaPipeline:
             defer_result(result).chainDeferred(wad)
 
     # Overridable Interface
-    def media_to_download(self, request, info):
+    def media_to_download(self, request, info, *, item=None):
         """Check request before starting download"""
         pass
 
@@ -179,7 +226,7 @@ class MediaPipeline:
         """Returns the media requests to download"""
         pass
 
-    def media_downloaded(self, response, request, info):
+    def media_downloaded(self, response, request, info, *, item=None):
         """Handler for success downloads"""
         return response
 
@@ -199,3 +246,7 @@ class MediaPipeline:
                         extra={'spider': info.spider}
                     )
         return item
+
+    def file_path(self, request, response=None, info=None, *, item=None):
+        """Returns the path where downloaded media should be stored"""
+        pass
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index a023dfcc8..b19b4ff2a 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -161,6 +161,19 @@ class FilesPipelineTestCase(unittest.TestCase):
         for p in patchers:
             p.stop()
 
+    def test_file_path_from_item(self):
+        """
+        Custom file path based on item data, overriding default implementation
+        """
+        class CustomFilesPipeline(FilesPipeline):
+            def file_path(self, request, response=None, info=None, item=None):
+                return 'full/%s' % item.get('path')
+
+        file_path = CustomFilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})).file_path
+        item = dict(path='path-to-store-file')
+        request = Request("http://example.com")
+        self.assertEqual(file_path(request, item=item), 'full/path-to-store-file')
+
 
 class FilesPipelineTestCaseFieldsMixin:
 
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 4f130c0c9..6afd47497 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -7,7 +7,9 @@ from twisted.internet.defer import Deferred, inlineCallbacks
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.request import request_fingerprint
+from scrapy.pipelines.images import ImagesPipeline
 from scrapy.pipelines.media import MediaPipeline
 from scrapy.pipelines.files import FileException
 from scrapy.utils.log import failure_to_exc_info
@@ -169,7 +171,7 @@ class MockedMediaPipeline(MediaPipeline):
         self._mockcalled.append('download')
         return super().download(request, info)
 
-    def media_to_download(self, request, info):
+    def media_to_download(self, request, info, *, item=None):
         self._mockcalled.append('media_to_download')
         if 'result' in request.meta:
             return request.meta.get('result')
@@ -179,7 +181,7 @@ class MockedMediaPipeline(MediaPipeline):
         self._mockcalled.append('get_media_requests')
         return item.get('requests')
 
-    def media_downloaded(self, response, request, info):
+    def media_downloaded(self, response, request, info, *, item=None):
         self._mockcalled.append('media_downloaded')
         return super().media_downloaded(response, request, info)
 
@@ -335,6 +337,123 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
             ['get_media_requests', 'media_to_download', 'item_completed'])
 
 
+class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
+
+    def __init__(self, *args, **kwargs):
+        super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs)
+        self._mockcalled = []
+
+    def get_media_requests(self, item, info):
+        item_url = item['image_urls'][0]
+        return Request(
+            item_url,
+            meta={'response': Response(item_url, status=200, body=b'data')}
+        )
+
+    def inc_stats(self, *args, **kwargs):
+        return True
+
+    def media_to_download(self, request, info):
+        self._mockcalled.append('media_to_download')
+        return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info)
+
+    def media_downloaded(self, response, request, info):
+        self._mockcalled.append('media_downloaded')
+        return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info)
+
+    def file_downloaded(self, response, request, info):
+        self._mockcalled.append('file_downloaded')
+        return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info)
+
+    def file_path(self, request, response=None, info=None):
+        self._mockcalled.append('file_path')
+        return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info)
+
+    def get_images(self, response, request, info):
+        self._mockcalled.append('get_images')
+        return []
+
+    def image_downloaded(self, response, request, info):
+        self._mockcalled.append('image_downloaded')
+        return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info)
+
+
+class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func)
+        self.pipe.open_spider(None)
+        self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[])
+
+    def _assert_method_called_with_warnings(self, method, message, warnings):
+        self.assertIn(method, self.pipe._mockcalled)
+        warningShown = False
+        for warning in warnings:
+            if warning['message'] == message and warning['category'] == ScrapyDeprecationWarning:
+                warningShown = True
+        self.assertTrue(warningShown)
+
+    @inlineCallbacks
+    def test_media_to_download_called(self):
+        yield self.pipe.process_item(self.item, None)
+        warnings = self.flushWarnings([MediaPipeline._compatible])
+        message = (
+            'media_to_download(self, request, info) is deprecated, '
+            'please use media_to_download(self, request, info, *, item=None)'
+        )
+        self._assert_method_called_with_warnings('media_to_download', message, warnings)
+
+    @inlineCallbacks
+    def test_media_downloaded_called(self):
+        yield self.pipe.process_item(self.item, None)
+        warnings = self.flushWarnings([MediaPipeline._compatible])
+        message = (
+            'media_downloaded(self, response, request, info) is deprecated, '
+            'please use media_downloaded(self, response, request, info, *, item=None)'
+        )
+        self._assert_method_called_with_warnings('media_downloaded', message, warnings)
+
+    @inlineCallbacks
+    def test_file_downloaded_called(self):
+        yield self.pipe.process_item(self.item, None)
+        warnings = self.flushWarnings([MediaPipeline._compatible])
+        message = (
+            'file_downloaded(self, response, request, info) is deprecated, '
+            'please use file_downloaded(self, response, request, info, *, item=None)'
+        )
+        self._assert_method_called_with_warnings('file_downloaded', message, warnings)
+
+    @inlineCallbacks
+    def test_file_path_called(self):
+        yield self.pipe.process_item(self.item, None)
+        warnings = self.flushWarnings([MediaPipeline._compatible])
+        message = (
+            'file_path(self, request, response=None, info=None) is deprecated, '
+            'please use file_path(self, request, response=None, info=None, *, item=None)'
+        )
+        self._assert_method_called_with_warnings('file_path', message, warnings)
+
+    @inlineCallbacks
+    def test_get_images_called(self):
+        yield self.pipe.process_item(self.item, None)
+        warnings = self.flushWarnings([MediaPipeline._compatible])
+        message = (
+            'get_images(self, response, request, info) is deprecated, '
+            'please use get_images(self, response, request, info, *, item=None)'
+        )
+        self._assert_method_called_with_warnings('get_images', message, warnings)
+
+    @inlineCallbacks
+    def test_image_downloaded_called(self):
+        yield self.pipe.process_item(self.item, None)
+        warnings = self.flushWarnings([MediaPipeline._compatible])
+        message = (
+            'image_downloaded(self, response, request, info) is deprecated, '
+            'please use image_downloaded(self, response, request, info, *, item=None)'
+        )
+        self._assert_method_called_with_warnings('image_downloaded', message, warnings)
+
+
 class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase):
 
     def _assert_request_no3xx(self, pipeline_class, settings):

From 394631fc0a731a0b8b0108edc1711c0c87e2ca15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 12 Aug 2020 12:08:09 +0200
Subject: [PATCH 2960/2974] Restore 3.5 support for mitmproxy-based tests

---
 tests/requirements-py3.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 4fa58d11b..b67c08403 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,7 +1,8 @@
 # Tests requirements
 attrs
 dataclasses; python_version == '3.6'
-mitmproxy >=4, <5  # The latest version does not support some pinned dependencies
+mitmproxy >=4, <5; python_version < '3.6'  # The latest version does not support some pinned dependencies
+mitmproxy <4; python_version < '3.6'
 # https://github.com/pytest-dev/pytest-twisted/issues/93
 pytest != 5.4, != 5.4.1
 pytest-azurepipelines

From 8e393a0b218b56cc8a70f7bda277969aa026790d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 12 Aug 2020 12:29:51 +0200
Subject: [PATCH 2961/2974] Do not change the mitmproxy version for no-3.6
 Python versions

---
 tests/requirements-py3.txt | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index b67c08403..b425ce771 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,8 +1,9 @@
 # Tests requirements
 attrs
-dataclasses; python_version == '3.6'
-mitmproxy >=4, <5; python_version < '3.6'  # The latest version does not support some pinned dependencies
-mitmproxy <4; python_version < '3.6'
+dataclasses; python_version ==3.6
+mitmproxy; python_version >=3.7
+mitmproxy >=4, <5; python_version >=3.6, <3.7
+mitmproxy <4; python_version <3.6
 # https://github.com/pytest-dev/pytest-twisted/issues/93
 pytest != 5.4, != 5.4.1
 pytest-azurepipelines

From b1de55d37d6a1b84e92df2d93d83e247e5d0e0d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 12 Aug 2020 12:34:40 +0200
Subject: [PATCH 2962/2974] Fix marker syntax

---
 tests/requirements-py3.txt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index b425ce771..b51177abb 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,9 +1,9 @@
 # Tests requirements
 attrs
-dataclasses; python_version ==3.6
-mitmproxy; python_version >=3.7
-mitmproxy >=4, <5; python_version >=3.6, <3.7
-mitmproxy <4; python_version <3.6
+dataclasses; python_version == '3.6'
+mitmproxy; python_version >= '3.7'
+mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7'
+mitmproxy < 4; python_version < '3.6'
 # https://github.com/pytest-dev/pytest-twisted/issues/93
 pytest != 5.4, != 5.4.1
 pytest-azurepipelines

From 5f4df622a17b1d8f9c3b9b693cb2f7119ffec27f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 13 Aug 2020 05:41:06 +0200
Subject: [PATCH 2963/2974] test_utils_iterators.py: support Windows the right
 way

---
 tests/__init__.py             |  2 +-
 tests/test_utils_iterators.py | 15 ++++++---------
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/tests/__init__.py b/tests/__init__.py
index 12ce79fa9..4253017fa 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -28,5 +28,5 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
 def get_testdata(*paths):
     """Return test data"""
     path = os.path.join(tests_datadir, *paths)
-    with open(path, 'rb') as f:
+    with open(path, 'rb', newline='') as f:
         return f.read()
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 298178f08..50190d4d1 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -7,9 +7,6 @@ from scrapy.http import XmlResponse, TextResponse, Response
 from tests import get_testdata
 
 
-FOOBAR_NL = "foo{}bar".format(os.linesep)
-
-
 class XmliterTestCase(unittest.TestCase):
 
     xmliter = staticmethod(xmliter)
@@ -267,7 +264,7 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual(result,
                          [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
                           {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
-                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
                           {'id': '4', 'name': 'empty', 'value': ''}])
 
         # explicit type check cuz' we no like stinkin' autocasting! yarrr
@@ -283,7 +280,7 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual([row for row in csv],
                          [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
                           {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
-                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
                           {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_quotechar(self):
@@ -296,7 +293,7 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual([row for row in csv1],
                          [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
                           {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
-                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
                           {'id': '4', 'name': 'empty', 'value': ''}])
 
         response2 = TextResponse(url="http://example.com/", body=body2)
@@ -305,7 +302,7 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual([row for row in csv2],
                          [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
                           {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
-                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
                           {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_wrong_quotechar(self):
@@ -327,7 +324,7 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual([row for row in csv],
                          [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
                           {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
-                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
                           {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_headers(self):
@@ -353,7 +350,7 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual([row for row in csv],
                          [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
                           {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
-                          {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+                          {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
                           {'id': '4', 'name': 'empty', 'value': ''}])
 
     def test_csviter_exception(self):

From 24ba5a71aca9b368eee21bd7d9043a7d26dba403 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 13 Aug 2020 06:35:09 +0200
Subject: [PATCH 2964/2974] Maybe the problem is not in the code after all

---
 .gitattributes    | 1 +
 tests/__init__.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 .gitattributes

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 000000000..dfbdf4208
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1 @@
+tests/sample_data/** binary
diff --git a/tests/__init__.py b/tests/__init__.py
index 4253017fa..12ce79fa9 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -28,5 +28,5 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
 def get_testdata(*paths):
     """Return test data"""
     path = os.path.join(tests_datadir, *paths)
-    with open(path, 'rb', newline='') as f:
+    with open(path, 'rb') as f:
         return f.read()

From 65e0abaea5eddb1dbb28b2bca9cd00e9b9409471 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 13 Aug 2020 09:05:51 +0200
Subject: [PATCH 2965/2974] Document FEED_URI_PARAMS

---
 docs/topics/feed-exports.rst | 75 ++++++++++++++++++++++++++++++++----
 1 file changed, 67 insertions(+), 8 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 37b7096f6..0f0f258dc 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -321,13 +321,14 @@ The following is a list of the accepted keys and the setting that is used
 as a fallback value if that key is not provided for a specific feed definition.
 
 * ``format``: the serialization format to be used for the feed.
-  See :ref:`topics-feed-format` for possible values. 
+  See :ref:`topics-feed-format` for possible values.
   Mandatory, no fallback setting
+* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
 * ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`
 * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
 * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
 * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
-* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
+* ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`
 
 .. setting:: FEED_EXPORT_ENCODING
 
@@ -500,7 +501,7 @@ generated:
 * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
   (e.g. ``2020-03-28T14-45-08.237134``)
 
-* ``%(batch_id)d`` - gets replaced by the sequence number of the batch.
+* ``%(batch_id)d`` - gets replaced by the 1-based sequence number of the batch.
 
   Use :ref:`printf-style string formatting <python:old-string-formatting>` to
   alter the number format. For example, to make the batch ID a 5-digit
@@ -517,16 +518,74 @@ And your :command:`crawl` command line is::
 
 The command line above can generate a directory tree like::
 
-->projectname
--->dirname
---->1-filename2020-03-28T14-45-08.237134.json
---->2-filename2020-03-28T14-45-09.148903.json
---->3-filename2020-03-28T14-45-10.046092.json
+    ->projectname
+    -->dirname
+    --->1-filename2020-03-28T14-45-08.237134.json
+    --->2-filename2020-03-28T14-45-09.148903.json
+    --->3-filename2020-03-28T14-45-10.046092.json
 
 Where the first and second files contain exactly 100 items. The last one contains
 100 items or fewer.
 
 
+.. setting:: FEED_URI_PARAMS
+
+FEED_URI_PARAMS
+---------------
+
+Default: ``None``
+
+A string with the import path of a function to set the parameters to apply with
+:ref:`printf-style string formatting <python:old-string-formatting>` to the
+feed URI.
+
+The function signature should be as follows:
+
+.. function:: uri_params(params, spider)
+
+   Return a :class:`dict` of key-value pairs to apply to the feed URI using
+   :ref:`printf-style string formatting <python:old-string-formatting>`.
+
+   :param params: default key-value pairs
+
+        Specifically:
+
+        -   ``batch_id``: ID of the file batch. See
+            :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
+
+            If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id``
+            is always ``1``.
+
+        -   ``batch_time``: UTC date and time, in ISO format with ``:``
+            replaced with ``-``.
+
+            See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
+
+        -   ``time``: ``batch_time``, with microseconds set to ``0``.
+   :type params: dict
+
+   :param spider: source spider of the feed items
+   :type spider: scrapy.spiders.Spider
+
+For example, to include the :attr:`name <scrapy.spiders.Spider.name>` of the
+source spider in the feed URI:
+
+#.  Define the following function somewhere in your project::
+
+        # myproject/utils.py
+        def uri_params(params, spider):
+            return {**params, 'spider_name': spider.name}
+
+#.  Point :setting:`FEED_URI_PARAMS` to that function in your settings::
+
+        # myproject/settings.py
+        FEED_URI_PARAMS = 'myproject.utils.uri_params'
+
+#.  Use ``%(spider_name)s`` in your feed URI::
+
+        scrapy crawl <spider_name> -o "%(spider_name)s.jl"
+
+
 .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
 .. _Amazon S3: https://aws.amazon.com/s3/
 .. _botocore: https://github.com/boto/botocore

From 756c368a6b0d2eef65f86f8418f9a7fbeff036c7 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 14 Aug 2020 22:09:24 +0500
Subject: [PATCH 2966/2974] Use a longer key in mitmproxy-ca.pem.

---
 tests/keys/mitmproxy-ca.pem | 78 +++++++++++++++++++++++--------------
 1 file changed, 48 insertions(+), 30 deletions(-)

diff --git a/tests/keys/mitmproxy-ca.pem b/tests/keys/mitmproxy-ca.pem
index 08004feca..cdef75f99 100644
--- a/tests/keys/mitmproxy-ca.pem
+++ b/tests/keys/mitmproxy-ca.pem
@@ -1,32 +1,50 @@
------BEGIN RSA PRIVATE KEY-----
-MIICWwIBAAKBgQDKLbznLxS7HSWvrmGcvVS6eQvjEWD705/csvnk/WtqAPfQMJKt
-auFBxzPt6RT60SHtj/2FKt2gqsiE6cNINxGN6fGYD7HtaM5HXRVPUKJaMipJwHha
-QivjIZoueraY/MtlyCkpp6dmMnHEpGY7OzwMyh1eCBHQ2JYx6VEzbks9ewIDAQAB
-AoGAMpS2ye/Rc+6a2xT5fskvRWe7PZe/d8E+IWz1cACmuuJ7HS7Jw3EV4esAZukF
-QqrHnjOD7akHwYZ4nCgPnyWH0lLx/4TIXE5QeLPFrhKOsSLCyhlCwNVJAdcOrDol
-Qh2694Dsd4gAy5o6TA02cBpqArnbAUERX46bHBZRA+ths8ECQQD6r1Ls+bTBR52w
-T3rPPhYj7EsXp40MJt0pLf1kjf+EH1bxsUqnxLawwo/lLE9omU73DFnfrAflk2Ll
-KUPCjjYpAkEAznchXk2ITeRcClrBNA+1Izpb5yG1qkfc79u/CEVDDOvt7RO/89Oj
-58R3pKTyffoo34fBdJz8GYDsmOeiyJEjAwJANpSHrJrtlQt/tMyJQ6gT7/xZmSvc
-1OF9U6L0wbj9AgpExtjAFWkKEdA6vj34iCChBb8FrmJpUb3WUWi7nReTiQJAFyIT
-9Av93LRcd7CJezrTUdolF/WX9DdPEvTtJ5ETHSyGIQ0Yccph0AMcYK82mFTiJYGB
-dH5uZLEkUVGK1KwmXwJAGWLdYiQyQitRWdoURcLb4OZ2gF3+7PASgFilI8YuoYhn
-Rl2Va3UtErPKJeMg2dTH18PuXykQMsQR1+rPxf1WSA==
------END RSA PRIVATE KEY-----
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCYp6U4G9YWITYB
+/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7ZDiT
+NUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMgz1BJ
+u6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniIggUH
+JrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE6HFB
+eIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/0zls
+90iyQ3E/AgMBAAECggEBAJA1dyAdM85uC04vKVNUJM1GDp0xS+0syBReJaKRI3nJ
+epoCj+RqxGag1pdaYLI0G84NTPqECz9LOyLdqpPgEfKRIxWlf9oWmSnfnXskArd8
+VfVcWYl6tEPv1TToTZIBmCbYLBFVbLxG/GrbK6uokdhUsqbdXwEKok2IEaSTRlDn
+v8BVXte00d9VEKKpmI6EY3f45uPQPHuJNcitP2HGW1mT/C6XoZR6wj+VvoRgUGQT
+I7PuktbYpQlLV+oX0uZz9frPGhjydUq0Jti5v3QAJEb+7D0cKrkZW+7fYDx4YkRU
+oDiuWEyO2kfpff52Qxs+xUXMiAyw6/8+TamKoAi1TIECgYEAyAzoztW6W4CjL2au
+/hN5VmbAvuBxq1m1G5KgXM1myX9V2CgH6OKwzJQNSCEfKMNOjqxB99T7C3tMCjgG
+gmbUzylTeciQFF+crrl2Rn/6qZS9dCo1hagb3K5eXMhLXoP425Y4sypNPPqULhPn
+YrUDFNAf89rRLqP1KMPLZ+uO7EECgYEAw1lWPxGV+X85iQxYN9xoX85htfJSBXTf
+dLirQ4bkykOxSA6ZzFuhDO/G373Q1rze4tmEO790uOCeaiXGgeWC1A+2PMO957i5
+9FqhDIkmerfdIttdEUMM9rQwuTcLnixGZkT5GHDzjtNinaIVB+pv7twRAESqN9dC
+QXh7IF7g/X8CgYBMhQOX+hCqZ24D95cAAJrs/ajEWj2geVPZFCDa3oZulJJVeBpu
+bieKWScra9/rS6mE0Ub6cTEFl0fisMNspcDI7NnNP3Y9FMVt3+rp1JIgw5AkGvEW
+CtN9egUGIGcT5A8Qj0lo3slkhcSgS2S6UNq431MZh51z5askyJ/JREULAQKBgFrR
+OatwfYzUfOcd+hVePpfr1rlDwqYOw6P8BoMKP2tZNR4Oy6maH7Fn98kk8eYjQGuu
+PC+avqUEqCEpFrRlAwGbnFl7ltoXozvatmyhhmYe/Iur+ASCa5B2DQDOenQ6mTAK
+eNPIDzMjSwGFzMk1UHx3it/ZDFmRlZfibzuJYIf5AoGBAIaPHk4qadK/XpcD4Wwx
+BOsDEIz27DGWdwWfd5r3EcV4zX/wNzH0G1Z8eydNjUqKzufMZgFwpcTu0Evesl1/
+B8kC8sLHxQoG5SvBu4dBxMwKIU9O9uFnX5SUYZUDpCtUYyZ+GtGom41Jwg5ENrwy
+HzPh2taMnCA0h1fNLFFBkw88
+-----END PRIVATE KEY-----
 -----BEGIN CERTIFICATE-----
-MIICnzCCAgigAwIBAgIGDI2K/EOjMA0GCSqGSIb3DQEBBQUAMCgxEjAQBgNVBAMT
-CW1pdG1wcm94eTESMBAGA1UEChMJbWl0bXByb3h5MB4XDTEzMDkyNjE0MzYxMVoX
-DTE1MDkxNjE0MzYxMVowKDESMBAGA1UEAxMJbWl0bXByb3h5MRIwEAYDVQQKEwlt
-aXRtcHJveHkwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMotvOcvFLsdJa+u
-YZy9VLp5C+MRYPvTn9yy+eT9a2oA99Awkq1q4UHHM+3pFPrRIe2P/YUq3aCqyITp
-w0g3EY3p8ZgPse1ozkddFU9QoloyKknAeFpCK+Mhmi56tpj8y2XIKSmnp2YyccSk
-Zjs7PAzKHV4IEdDYljHpUTNuSz17AgMBAAGjgdMwgdAwDwYDVR0TAQH/BAUwAwEB
-/zAUBglghkgBhvhCAQEBAf8EBAMCAgQwewYDVR0lAQH/BHEwbwYIKwYBBQUHAwEG
-CCsGAQUFBwMCBggrBgEFBQcDBAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQB
-gjcCARYGCisGAQQBgjcKAwEGCisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG
-+EIEATALBgNVHQ8EBAMCAQYwHQYDVR0OBBYEFJBEfawVwhEHHW6rS8nvZFlJ582n
-MA0GCSqGSIb3DQEBBQUAA4GBAHGl28Ip2CWS/MibCaFztLDxGiMBT4MW2yI2hf3D
-y9g1o7ra/fSEFdIc849xXyCsGWSkMsbDML272rCH4K73MUBxxkJm46AIyRVH1z2Z
-e96u4py1wNT8cznY15phr8pn36snlaHaYa+JcwGINMdSOk1VPHv6gqSC/vgUCgF1
-n95u
+MIIDoTCCAomgAwIBAgIGDodLQx9+MA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNVBAMM
+CW1pdG1wcm94eTESMBAGA1UECgwJbWl0bXByb3h5MB4XDTIwMDgxMjE3MDMyNloX
+DTIzMDgxNDE3MDMyNlowKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAlt
+aXRtcHJveHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCYp6U4G9YW
+ITYB/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7
+ZDiTNUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMg
+z1BJu6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniI
+ggUHJrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE
+6HFBeIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/
+0zls90iyQ3E/AgMBAAGjgdAwgc0wDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhC
+AQEEBAMCAgQweAYDVR0lBHEwbwYIKwYBBQUHAwEGCCsGAQUFBwMCBggrBgEFBQcD
+BAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQBgjcCARYGCisGAQQBgjcKAwEG
+CisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG+EIEATAOBgNVHQ8BAf8EBAMC
+AQYwHQYDVR0OBBYEFBCsLPpFz3l9rOOfGmfs+VRc3jhJMA0GCSqGSIb3DQEBCwUA
+A4IBAQADTpA15na6U5qqDCe0rr39fkS1/dY804Xnz7g/L3AsxPE1KOMijuJa8sKd
+kKwba1173FwMupfK39zY8jUxL8Qprdi92RO6CpoFUsL/icpA///lYhzUSqt32qwe
+gRNW3mtYBimOk6KH1NOfQnJolWpJh+g1OEsitQKEeKwIn5Hz+8/yS5tbwLgdnMlY
+1/it1H70JSdE7nfJueqN4cFfBsm6XaHZzacJJmN7WP88fd+zztnSQsBFbLlnjnqj
+envCDIwCrMywKNMqEBMwmBEGSAF47fVNYj6KzDAtMvBdDkYaHWpBf4tnFfk6v0wj
+wiKjdLjCmJgjGAQjRw5VYJ8JI0XO
 -----END CERTIFICATE-----

From 2aa4f3cbf96ad55aa4a1fa064758338daf16b110 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Mon, 17 Aug 2020 05:39:59 -0300
Subject: [PATCH 2967/2974] Conditional request attribute binding for responses
 (#4632)

---
 docs/topics/signals.rst                 |   5 +
 scrapy/core/engine.py                   |  13 +-
 scrapy/core/scraper.py                  |  62 ++++----
 setup.cfg                               |   3 +
 tests/test_request_attribute_binding.py | 202 ++++++++++++++++++++++++
 5 files changed, 250 insertions(+), 35 deletions(-)
 create mode 100644 tests/test_request_attribute_binding.py

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 255ba9d3f..1d99d8c28 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -423,6 +423,11 @@ response_received
     :param spider: the spider for which the response is intended
     :type spider: :class:`~scrapy.spiders.Spider` object
 
+.. note:: The ``request`` argument might not contain the original request that
+    reached the downloader, if a :ref:`topics-downloader-middleware` modifies
+    the :class:`~scrapy.http.Response` object and sets a specific ``request``
+    attribute.
+
 response_downloaded
 ~~~~~~~~~~~~~~~~~~~
 
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 86a6abb23..5e0dfe37c 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -243,12 +243,17 @@ class ExecutionEngine:
                     % (type(response), response)
                 )
             if isinstance(response, Response):
-                response.request = request  # tie request to response received
-                logkws = self.logformatter.crawled(request, response, spider)
+                if response.request is None:
+                    response.request = request
+                logkws = self.logformatter.crawled(response.request, response, spider)
                 if logkws is not None:
                     logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
-                self.signals.send_catch_log(signals.response_received,
-                                            response=response, request=request, spider=spider)
+                self.signals.send_catch_log(
+                    signal=signals.response_received,
+                    response=response,
+                    request=response.request,
+                    spider=spider,
+                )
             return response
 
         def _on_complete(_):
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 1ef0790a9..20bdb22a1 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -12,7 +12,7 @@ from scrapy import signals
 from scrapy.core.spidermw import SpiderMiddlewareManager
 from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy.http import Request, Response
-from scrapy.utils.defer import defer_result, defer_succeed, iter_errback, parallel
+from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel
 from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
 from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
 from scrapy.utils.spider import iterate_spider_output
@@ -120,40 +120,40 @@ class Scraper:
             response, request, deferred = slot.next_response_request_deferred()
             self._scrape(response, request, spider).chainDeferred(deferred)
 
-    def _scrape(self, response, request, spider):
-        """Handle the downloaded response or failure through the spider
-        callback/errback"""
-        if not isinstance(response, (Response, Failure)):
-            raise TypeError(
-                "Incorrect type: expected Response or Failure, got %s: %r"
-                % (type(response), response)
-            )
-
-        dfd = self._scrape2(response, request, spider)  # returns spider's processed output
-        dfd.addErrback(self.handle_spider_error, request, response, spider)
-        dfd.addCallback(self.handle_spider_output, request, response, spider)
+    def _scrape(self, result, request, spider):
+        """
+        Handle the downloaded response or failure through the spider callback/errback
+        """
+        if not isinstance(result, (Response, Failure)):
+            raise TypeError("Incorrect type: expected Response or Failure, got %s: %r" % (type(result), result))
+        dfd = self._scrape2(result, request, spider)  # returns spider's processed output
+        dfd.addErrback(self.handle_spider_error, request, result, spider)
+        dfd.addCallback(self.handle_spider_output, request, result, spider)
         return dfd
 
-    def _scrape2(self, request_result, request, spider):
-        """Handle the different cases of request's result been a Response or a
-        Failure"""
-        if not isinstance(request_result, Failure):
-            return self.spidermw.scrape_response(
-                self.call_spider, request_result, request, spider)
-        else:
-            dfd = self.call_spider(request_result, request, spider)
-            return dfd.addErrback(
-                self._log_download_errors, request_result, request, spider)
+    def _scrape2(self, result, request, spider):
+        """
+        Handle the different cases of request's result been a Response or a Failure
+        """
+        if isinstance(result, Response):
+            return self.spidermw.scrape_response(self.call_spider, result, request, spider)
+        else:  # result is a Failure
+            dfd = self.call_spider(result, request, spider)
+            return dfd.addErrback(self._log_download_errors, result, request, spider)
 
     def call_spider(self, result, request, spider):
-        result.request = request
-        dfd = defer_result(result)
-        callback = request.callback or spider._parse
-        warn_on_generator_with_return_value(spider, callback)
-        warn_on_generator_with_return_value(spider, request.errback)
-        dfd.addCallbacks(callback=callback,
-                         errback=request.errback,
-                         callbackKeywords=request.cb_kwargs)
+        if isinstance(result, Response):
+            if getattr(result, "request", None) is None:
+                result.request = request
+            callback = result.request.callback or spider._parse
+            warn_on_generator_with_return_value(spider, callback)
+            dfd = defer_succeed(result)
+            dfd.addCallback(callback, **result.request.cb_kwargs)
+        else:  # result is a Failure
+            result.request = request
+            warn_on_generator_with_return_value(spider, request.errback)
+            dfd = defer_fail(result)
+            dfd.addErrback(request.errback)
         return dfd.addCallback(iterate_spider_output)
 
     def handle_spider_error(self, _failure, request, response, spider):
diff --git a/setup.cfg b/setup.cfg
index f8e7c0c91..3a624ec94 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -130,6 +130,9 @@ ignore_errors = True
 [mypy-tests.test_pipelines]
 ignore_errors = True
 
+[mypy-tests.test_request_attribute_binding]
+ignore_errors = True
+
 [mypy-tests.test_request_cb_kwargs]
 ignore_errors = True
 
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
new file mode 100644
index 000000000..b60b7c579
--- /dev/null
+++ b/tests/test_request_attribute_binding.py
@@ -0,0 +1,202 @@
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+
+from scrapy import Request, signals
+from scrapy.crawler import CrawlerRunner
+from scrapy.http.response import Response
+
+from testfixtures import LogCapture
+
+from tests.mockserver import MockServer
+from tests.spiders import SingleRequestSpider
+
+
+OVERRIDEN_URL = "https://example.org"
+
+
+class ProcessResponseMiddleware:
+    def process_response(self, request, response, spider):
+        return response.replace(request=Request(OVERRIDEN_URL))
+
+
+class RaiseExceptionRequestMiddleware:
+    def process_request(self, request, spider):
+        1 / 0
+        return request
+
+
+class CatchExceptionOverrideRequestMiddleware:
+    def process_exception(self, request, exception, spider):
+        return Response(
+            url="http://localhost/",
+            body=b"Caught " + exception.__class__.__name__.encode("utf-8"),
+            request=Request(OVERRIDEN_URL),
+        )
+
+
+class CatchExceptionDoNotOverrideRequestMiddleware:
+    def process_exception(self, request, exception, spider):
+        return Response(
+            url="http://localhost/",
+            body=b"Caught " + exception.__class__.__name__.encode("utf-8"),
+        )
+
+
+class AlternativeCallbacksSpider(SingleRequestSpider):
+    name = "alternative_callbacks_spider"
+
+    def alt_callback(self, response, foo=None):
+        self.logger.info("alt_callback was invoked with foo=%s", foo)
+
+
+class AlternativeCallbacksMiddleware:
+    def process_response(self, request, response, spider):
+        new_request = request.replace(
+            url=OVERRIDEN_URL,
+            callback=spider.alt_callback,
+            cb_kwargs={"foo": "bar"},
+        )
+        return response.replace(request=new_request)
+
+
+class CrawlTestCase(TestCase):
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_response_200(self):
+        url = self.mockserver.url("/status?n=200")
+        crawler = CrawlerRunner().create_crawler(SingleRequestSpider)
+        yield crawler.crawl(seed=url, mockserver=self.mockserver)
+        response = crawler.spider.meta["responses"][0]
+        self.assertEqual(response.request.url, url)
+
+    @defer.inlineCallbacks
+    def test_response_error(self):
+        for status in ("404", "500"):
+            url = self.mockserver.url("/status?n={}".format(status))
+            crawler = CrawlerRunner().create_crawler(SingleRequestSpider)
+            yield crawler.crawl(seed=url, mockserver=self.mockserver)
+            failure = crawler.spider.meta["failure"]
+            response = failure.value.response
+            self.assertEqual(failure.request.url, url)
+            self.assertEqual(response.request.url, url)
+
+    @defer.inlineCallbacks
+    def test_downloader_middleware_raise_exception(self):
+        url = self.mockserver.url("/status?n=200")
+        runner = CrawlerRunner(settings={
+            "DOWNLOADER_MIDDLEWARES": {
+                __name__ + ".RaiseExceptionRequestMiddleware": 590,
+            },
+        })
+        crawler = runner.create_crawler(SingleRequestSpider)
+        yield crawler.crawl(seed=url, mockserver=self.mockserver)
+        failure = crawler.spider.meta["failure"]
+        self.assertEqual(failure.request.url, url)
+        self.assertIsInstance(failure.value, ZeroDivisionError)
+
+    @defer.inlineCallbacks
+    def test_downloader_middleware_override_request_in_process_response(self):
+        """
+        Downloader middleware which returns a response with an specific 'request' attribute.
+
+        * The spider callback should receive the overriden response.request
+        * Handlers listening to the response_received signal should receive the overriden response.request
+        * The "crawled" log message should show the overriden response.request
+        """
+        signal_params = {}
+
+        def signal_handler(response, request, spider):
+            signal_params["response"] = response
+            signal_params["request"] = request
+
+        url = self.mockserver.url("/status?n=200")
+        runner = CrawlerRunner(settings={
+            "DOWNLOADER_MIDDLEWARES": {
+                __name__ + ".ProcessResponseMiddleware": 595,
+            }
+        })
+        crawler = runner.create_crawler(SingleRequestSpider)
+        crawler.signals.connect(signal_handler, signal=signals.response_received)
+
+        with LogCapture() as log:
+            yield crawler.crawl(seed=url, mockserver=self.mockserver)
+
+        response = crawler.spider.meta["responses"][0]
+        self.assertEqual(response.request.url, OVERRIDEN_URL)
+
+        self.assertEqual(signal_params["response"].url, url)
+        self.assertEqual(signal_params["request"].url, OVERRIDEN_URL)
+
+        log.check_present(
+            ("scrapy.core.engine", "DEBUG", "Crawled (200) <GET {}> (referer: None)".format(OVERRIDEN_URL)),
+        )
+
+    @defer.inlineCallbacks
+    def test_downloader_middleware_override_in_process_exception(self):
+        """
+        An exception is raised but caught by the next middleware, which
+        returns a Response with a specific 'request' attribute.
+
+        The spider callback should receive the overriden response.request
+        """
+        url = self.mockserver.url("/status?n=200")
+        runner = CrawlerRunner(settings={
+            "DOWNLOADER_MIDDLEWARES": {
+                __name__ + ".RaiseExceptionRequestMiddleware": 590,
+                __name__ + ".CatchExceptionOverrideRequestMiddleware": 595,
+            },
+        })
+        crawler = runner.create_crawler(SingleRequestSpider)
+        yield crawler.crawl(seed=url, mockserver=self.mockserver)
+        response = crawler.spider.meta["responses"][0]
+        self.assertEqual(response.body, b"Caught ZeroDivisionError")
+        self.assertEqual(response.request.url, OVERRIDEN_URL)
+
+    @defer.inlineCallbacks
+    def test_downloader_middleware_do_not_override_in_process_exception(self):
+        """
+        An exception is raised but caught by the next middleware, which
+        returns a Response without a specific 'request' attribute.
+
+        The spider callback should receive the original response.request
+        """
+        url = self.mockserver.url("/status?n=200")
+        runner = CrawlerRunner(settings={
+            "DOWNLOADER_MIDDLEWARES": {
+                __name__ + ".RaiseExceptionRequestMiddleware": 590,
+                __name__ + ".CatchExceptionDoNotOverrideRequestMiddleware": 595,
+            },
+        })
+        crawler = runner.create_crawler(SingleRequestSpider)
+        yield crawler.crawl(seed=url, mockserver=self.mockserver)
+        response = crawler.spider.meta["responses"][0]
+        self.assertEqual(response.body, b"Caught ZeroDivisionError")
+        self.assertEqual(response.request.url, url)
+
+    @defer.inlineCallbacks
+    def test_downloader_middleware_alternative_callback(self):
+        """
+        Downloader middleware which returns a response with a
+        specific 'request' attribute, with an alternative callback
+        """
+        runner = CrawlerRunner(settings={
+            "DOWNLOADER_MIDDLEWARES": {
+                __name__ + ".AlternativeCallbacksMiddleware": 595,
+            }
+        })
+        crawler = runner.create_crawler(AlternativeCallbacksSpider)
+
+        with LogCapture() as log:
+            url = self.mockserver.url("/status?n=200")
+            yield crawler.crawl(seed=url, mockserver=self.mockserver)
+
+        log.check_present(
+            ("alternative_callbacks_spider", "INFO", "alt_callback was invoked with foo=bar"),
+        )

From a8e08d51cd50e8f0b58a60992d310e56d4f71641 Mon Sep 17 00:00:00 2001
From: Ajay Mittur <ajay.cs18@bmsce.ac.in>
Date: Mon, 17 Aug 2020 14:15:52 +0530
Subject: [PATCH 2968/2974] Check if file is already present on running `scrapy
 genspider` and terminate if so (#4623)

---
 scrapy/commands/genspider.py | 41 ++++++++++++++++------
 tests/test_commands.py       | 67 +++++++++++++++++++++++++++++++++++-
 2 files changed, 97 insertions(+), 11 deletions(-)

diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 4c7548e9c..74a077d1b 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -66,16 +66,9 @@ class Command(ScrapyCommand):
             print("Cannot create a spider with the same name as your project")
             return
 
-        try:
-            spidercls = self.crawler_process.spider_loader.load(name)
-        except KeyError:
-            pass
-        else:
-            # if spider already exists and not --force then halt
-            if not opts.force:
-                print("Spider %r already exists in module:" % name)
-                print("  %s" % spidercls.__module__)
-                return
+        if not opts.force and self._spider_exists(name):
+            return
+
         template_file = self._find_template(opts.template)
         if template_file:
             self._genspider(module, name, domain, opts.template, template_file)
@@ -119,6 +112,34 @@ class Command(ScrapyCommand):
             if filename.endswith('.tmpl'):
                 print("  %s" % splitext(filename)[0])
 
+    def _spider_exists(self, name):
+        if not self.settings.get('NEWSPIDER_MODULE'):
+            # if run as a standalone command and file with same filename already exists
+            if exists(name + ".py"):
+                print("%s already exists" % (abspath(name + ".py")))
+                return True
+            return False
+
+        try:
+            spidercls = self.crawler_process.spider_loader.load(name)
+        except KeyError:
+            pass
+        else:
+            # if spider with same name exists
+            print("Spider %r already exists in module:" % name)
+            print("  %s" % spidercls.__module__)
+            return True
+
+        # a file with the same name exists in the target directory
+        spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
+        spiders_dir = dirname(spiders_module.__file__)
+        spiders_dir_abs = abspath(spiders_dir)
+        if exists(join(spiders_dir_abs, name + ".py")):
+            print("%s already exists" % (join(spiders_dir_abs, (name + ".py"))))
+            return True
+
+        return False
+
     @property
     def templates_dir(self):
         return join(
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 8938156fc..109f006a8 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -8,7 +8,7 @@ import sys
 import tempfile
 from contextlib import contextmanager
 from itertools import chain
-from os.path import exists, join, abspath
+from os.path import exists, join, abspath, getmtime
 from pathlib import Path
 from shutil import rmtree, copytree
 from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
@@ -337,8 +337,11 @@ class GenspiderCommandTest(CommandTest):
         p, out, err = self.proc('genspider', spname, 'test.com', *args)
         self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out)
         self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
+        modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py'))
         p, out, err = self.proc('genspider', spname, 'test.com', *args)
         self.assertIn("Spider %r already exists in module" % spname, out)
+        modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py'))
+        self.assertEqual(modify_time_after, modify_time_before)
 
     def test_template_basic(self):
         self.test_template('basic')
@@ -360,6 +363,40 @@ class GenspiderCommandTest(CommandTest):
         self.assertEqual(2, self.call('genspider', self.project_name))
         assert not exists(join(self.proj_mod_path, 'spiders', '%s.py' % self.project_name))
 
+    def test_same_filename_as_existing_spider(self, force=False):
+        file_name = 'example'
+        file_path = join(self.proj_mod_path, 'spiders', '%s.py' % file_name)
+        self.assertEqual(0, self.call('genspider', file_name, 'example.com'))
+        assert exists(file_path)
+
+        # change name of spider but not its file name
+        with open(file_path, 'r+') as spider_file:
+            file_data = spider_file.read()
+            file_data = file_data.replace("name = \'example\'", "name = \'renamed\'")
+            spider_file.seek(0)
+            spider_file.write(file_data)
+            spider_file.truncate()
+        modify_time_before = getmtime(file_path)
+        file_contents_before = file_data
+
+        if force:
+            p, out, err = self.proc('genspider', '--force', file_name, 'example.com')
+            self.assertIn("Created spider %r using template \'basic\' in module" % file_name, out)
+            modify_time_after = getmtime(file_path)
+            self.assertNotEqual(modify_time_after, modify_time_before)
+            file_contents_after = open(file_path, 'r').read()
+            self.assertNotEqual(file_contents_after, file_contents_before)
+        else:
+            p, out, err = self.proc('genspider', file_name, 'example.com')
+            self.assertIn("%s already exists" % (file_path), out)
+            modify_time_after = getmtime(file_path)
+            self.assertEqual(modify_time_after, modify_time_before)
+            file_contents_after = open(file_path, 'r').read()
+            self.assertEqual(file_contents_after, file_contents_before)
+
+    def test_same_filename_as_existing_spider_force(self):
+        self.test_same_filename_as_existing_spider(force=True)
+
 
 class GenspiderStandaloneCommandTest(ProjectTest):
 
@@ -367,6 +404,34 @@ class GenspiderStandaloneCommandTest(ProjectTest):
         self.call('genspider', 'example', 'example.com')
         assert exists(join(self.temp_path, 'example.py'))
 
+    def test_same_name_as_existing_file(self, force=False):
+        file_name = 'example'
+        file_path = join(self.temp_path, file_name + '.py')
+        p, out, err = self.proc('genspider', file_name, 'example.com')
+        self.assertIn("Created spider %r using template \'basic\' " % file_name, out)
+        assert exists(file_path)
+        modify_time_before = getmtime(file_path)
+        file_contents_before = open(file_path, 'r').read()
+
+        if force:
+            # use different template to ensure contents were changed
+            p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com')
+            self.assertIn("Created spider %r using template \'crawl\' " % file_name, out)
+            modify_time_after = getmtime(file_path)
+            self.assertNotEqual(modify_time_after, modify_time_before)
+            file_contents_after = open(file_path, 'r').read()
+            self.assertNotEqual(file_contents_after, file_contents_before)
+        else:
+            p, out, err = self.proc('genspider', file_name, 'example.com')
+            self.assertIn("%s already exists" % join(self.temp_path, file_name + ".py"), out)
+            modify_time_after = getmtime(file_path)
+            self.assertEqual(modify_time_after, modify_time_before)
+            file_contents_after = open(file_path, 'r').read()
+            self.assertEqual(file_contents_after, file_contents_before)
+
+    def test_same_name_as_existing_file_force(self):
+        self.test_same_name_as_existing_file(force=True)
+
 
 class MiscCommandsTest(CommandTest):
 

From 55edf8d3b8885541cdbf9d1c62d9a6bbf634e2a0 Mon Sep 17 00:00:00 2001
From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com>
Date: Mon, 17 Aug 2020 18:50:52 +1000
Subject: [PATCH 2969/2974] Add typing hint to httpcache downloadermiddlewares
 (#4243)

---
 scrapy/downloadermiddlewares/httpcache.py | 41 ++++++++++++++++-------
 1 file changed, 29 insertions(+), 12 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 6db57bd8b..62f1c3a29 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -1,4 +1,5 @@
 from email.utils import formatdate
+from typing import Optional, Type, TypeVar
 
 from twisted.internet import defer
 from twisted.internet.error import (
@@ -13,10 +14,19 @@ from twisted.internet.error import (
 from twisted.web.client import ResponseFailed
 
 from scrapy import signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import IgnoreRequest, NotConfigured
+from scrapy.http.request import Request
+from scrapy.http.response import Response
+from scrapy.settings import Settings
+from scrapy.spiders import Spider
+from scrapy.statscollectors import StatsCollector
 from scrapy.utils.misc import load_object
 
 
+HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware")
+
+
 class HttpCacheMiddleware:
 
     DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
@@ -24,7 +34,7 @@ class HttpCacheMiddleware:
                            ConnectionLost, TCPTimedOutError, ResponseFailed,
                            IOError)
 
-    def __init__(self, settings, stats):
+    def __init__(self, settings: Settings, stats: StatsCollector) -> None:
         if not settings.getbool('HTTPCACHE_ENABLED'):
             raise NotConfigured
         self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings)
@@ -33,26 +43,26 @@ class HttpCacheMiddleware:
         self.stats = stats
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls: Type[HttpCacheMiddlewareTV], crawler: Crawler) -> HttpCacheMiddlewareTV:
         o = cls(crawler.settings, crawler.stats)
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
         return o
 
-    def spider_opened(self, spider):
+    def spider_opened(self, spider: Spider) -> None:
         self.storage.open_spider(spider)
 
-    def spider_closed(self, spider):
+    def spider_closed(self, spider: Spider) -> None:
         self.storage.close_spider(spider)
 
-    def process_request(self, request, spider):
+    def process_request(self, request: Request, spider: Spider) -> Optional[Response]:
         if request.meta.get('dont_cache', False):
-            return
+            return None
 
         # Skip uncacheable requests
         if not self.policy.should_cache_request(request):
             request.meta['_dont_cache'] = True  # flag as uncacheable
-            return
+            return None
 
         # Look for cached response and check if expired
         cachedresponse = self.storage.retrieve_response(spider, request)
@@ -61,7 +71,7 @@ class HttpCacheMiddleware:
             if self.ignore_missing:
                 self.stats.inc_value('httpcache/ignore', spider=spider)
                 raise IgnoreRequest("Ignored request not in cache: %s" % request)
-            return  # first time request
+            return None  # first time request
 
         # Return cached response only if not expired
         cachedresponse.flags.append('cached')
@@ -73,7 +83,9 @@ class HttpCacheMiddleware:
         # process_response hook
         request.meta['cached_response'] = cachedresponse
 
-    def process_response(self, request, response, spider):
+        return None
+
+    def process_response(self, request: Request, response: Response, spider: Spider) -> Response:
         if request.meta.get('dont_cache', False):
             return response
 
@@ -85,7 +97,7 @@ class HttpCacheMiddleware:
         # RFC2616 requires origin server to set Date header,
         # https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18
         if 'Date' not in response.headers:
-            response.headers['Date'] = formatdate(usegmt=1)
+            response.headers['Date'] = formatdate(usegmt=True)
 
         # Do not validate first-hand responses
         cachedresponse = request.meta.pop('cached_response', None)
@@ -102,13 +114,18 @@ class HttpCacheMiddleware:
         self._cache_response(spider, response, request, cachedresponse)
         return response
 
-    def process_exception(self, request, exception, spider):
+    def process_exception(
+        self, request: Request, exception: Exception, spider: Spider
+    ) -> Optional[Response]:
         cachedresponse = request.meta.pop('cached_response', None)
         if cachedresponse is not None and isinstance(exception, self.DOWNLOAD_EXCEPTIONS):
             self.stats.inc_value('httpcache/errorrecovery', spider=spider)
             return cachedresponse
+        return None
 
-    def _cache_response(self, spider, response, request, cachedresponse):
+    def _cache_response(
+        self, spider: Spider, response: Response, request: Request, cachedresponse: Optional[Response]
+    ) -> None:
         if self.policy.should_cache_response(response, request):
             self.stats.inc_value('httpcache/store', spider=spider)
             self.storage.store_response(spider, request, response)

From e70975f0bb4e8378bf0e00ae7a7014247cd59441 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 17 Aug 2020 15:10:08 +0200
Subject: [PATCH 2970/2974] Allow overwriting feeds (#4512)

Co-authored-by: Yuval Hager <yhager@yhager.com>
---
 docs/faq.rst                    |   6 +-
 docs/intro/overview.rst         |  26 +-
 docs/intro/tutorial.rst         |  13 +-
 docs/topics/feed-exports.rst    |  47 +++-
 scrapy/commands/__init__.py     |  15 +-
 scrapy/extensions/feedexport.py | 142 +++++++---
 scrapy/utils/conf.py            |  44 +++-
 scrapy/utils/ftp.py             |   6 +-
 scrapy/utils/python.py          |   3 +-
 tests/ftpserver.py              |  24 ++
 tests/mockserver.py             |  26 ++
 tests/requirements-py3.txt      |   1 +
 tests/test_commands.py          | 126 +++++++++
 tests/test_feedexport.py        | 442 ++++++++++++++++++++++++++++----
 tests/test_utils_conf.py        |  16 ++
 tests/test_utils_python.py      |   4 +
 16 files changed, 791 insertions(+), 150 deletions(-)
 create mode 100644 tests/ftpserver.py

diff --git a/docs/faq.rst b/docs/faq.rst
index ea2c8216f..9346ec358 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -236,15 +236,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file?
 
 To dump into a JSON file::
 
-    scrapy crawl myspider -o items.json
+    scrapy crawl myspider -O items.json
 
 To dump into a CSV file::
 
-    scrapy crawl myspider -o items.csv
+    scrapy crawl myspider -O items.csv
 
 To dump into a XML file::
 
-    scrapy crawl myspider -o items.xml
+    scrapy crawl myspider -O items.xml
 
 For more information see :ref:`topics-feed-exports`
 
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 01986b594..dd80c7bd0 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -42,30 +42,18 @@ http://quotes.toscrape.com, following the pagination::
             if next_page is not None:
                 yield response.follow(next_page, self.parse)
 
-
 Put this in a text file, name it to something like ``quotes_spider.py``
 and run the spider using the :command:`runspider` command::
 
-    scrapy runspider quotes_spider.py -o quotes.json
+    scrapy runspider quotes_spider.py -o quotes.jl
 
+When this finishes you will have in the ``quotes.jl`` file a list of the
+quotes in JSON Lines format, containing text and author, looking like this::
 
-When this finishes you will have in the ``quotes.json`` file a list of the
-quotes in JSON format, containing text and author, looking like this (reformatted
-here for better readability)::
-
-    [{
-        "author": "Jane Austen",
-        "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"
-    },
-    {
-        "author": "Groucho Marx",
-        "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d"
-    },
-    {
-        "author": "Steve Martin",
-        "text": "\u201cA day without sunshine is like, you know, night.\u201d"
-    },
-    ...]
+    {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
+    {"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"}
+    {"author": "Garrison Keillor", "text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d"}
+    ...
 
 
 What just happened?
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 5f35dc936..f96c78887 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -464,16 +464,15 @@ Storing the scraped data
 The simplest way to store the scraped data is by using :ref:`Feed exports
 <topics-feed-exports>`, with the following command::
 
-    scrapy crawl quotes -o quotes.json
+    scrapy crawl quotes -O quotes.json
 
 That will generate an ``quotes.json`` file containing all scraped items,
 serialized in `JSON`_.
 
-For historic reasons, Scrapy appends to a given file instead of overwriting
-its contents. If you run this command twice without removing the file
-before the second time, you'll end up with a broken JSON file.
-
-You can also use other formats, like `JSON Lines`_::
+The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead
+to append new content to any existing file. However, appending to a JSON file
+makes the file contents invalid JSON. When appending to a file, consider
+using a different serialization format, such as `JSON Lines`_::
 
     scrapy crawl quotes -o quotes.jl
 
@@ -704,7 +703,7 @@ Using spider arguments
 You can provide command line arguments to your spiders by using the ``-a``
 option when running them::
 
-    scrapy crawl quotes -o quotes-humor.json -a tag=humor
+    scrapy crawl quotes -O quotes-humor.json -a tag=humor
 
 These arguments are passed to the Spider's ``__init__`` method and become
 spider attributes by default.
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 0f0f258dc..cd4f7cf29 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -291,6 +291,7 @@ Default: ``{}``
 A dictionary in which every key is a feed URI (or a :class:`pathlib.Path`
 object) and each value is a nested dictionary containing configuration
 parameters for the specific feed.
+
 This setting is required for enabling the feed export feature.
 
 See :ref:`topics-feed-storage-backends` for supported URI schemes.
@@ -318,17 +319,43 @@ For instance::
     }
 
 The following is a list of the accepted keys and the setting that is used
-as a fallback value if that key is not provided for a specific feed definition.
+as a fallback value if that key is not provided for a specific feed definition:
+
+-   ``format``: the :ref:`serialization format <topics-feed-format>`.
+
+    This setting is mandatory, there is no fallback value.
+
+-   ``batch_item_count``: falls back to
+    :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
+
+-   ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`.
+
+-   ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
+
+-   ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
+
+-   ``overwrite``: whether to overwrite the file if it already exists
+    (``True``) or append to its content (``False``).
+
+    The default value depends on the :ref:`storage backend
+    <topics-feed-storage-backends>`:
+
+    -   :ref:`topics-feed-storage-fs`: ``False``
+
+    -   :ref:`topics-feed-storage-ftp`: ``True``
+
+        .. note:: Some FTP servers may not support appending to files (the
+                  ``APPE`` FTP command).
+
+    -   :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported
+        <https://forums.aws.amazon.com/message.jspa?messageID=540395>`_)
+
+    -   :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
+
+-   ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.
+
+-   ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
 
-* ``format``: the serialization format to be used for the feed.
-  See :ref:`topics-feed-format` for possible values.
-  Mandatory, no fallback setting
-* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
-* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`
-* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
-* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
-* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
-* ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`
 
 .. setting:: FEED_EXPORT_ENCODING
 
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 57ce4e522..cfd940fe7 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -115,9 +115,11 @@ class BaseRunSpiderCommand(ScrapyCommand):
         parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
                           help="set spider argument (may be repeated)")
         parser.add_option("-o", "--output", metavar="FILE", action="append",
-                          help="dump scraped items into FILE (use - for stdout)")
+                          help="append scraped items to the end of FILE (use - for stdout)")
+        parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
+                          help="dump scraped items into FILE, overwriting any existing file")
         parser.add_option("-t", "--output-format", metavar="FORMAT",
-                          help="format to use for dumping items with -o")
+                          help="format to use for dumping items")
 
     def process_options(self, args, opts):
         ScrapyCommand.process_options(self, args, opts)
@@ -125,6 +127,11 @@ class BaseRunSpiderCommand(ScrapyCommand):
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
-        if opts.output:
-            feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
+        if opts.output or opts.overwrite_output:
+            feeds = feed_process_params_from_cli(
+                self.settings,
+                opts.output,
+                opts.output_format,
+                opts.overwrite_output,
+            )
             self.settings.set('FEEDS', feeds, priority='cmdline')
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index b7a4e362e..980825499 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -24,17 +24,34 @@ from scrapy.utils.conf import feed_complete_default_values_from_settings
 from scrapy.utils.ftp import ftp_store_file
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
-from scrapy.utils.python import without_none_values
+from scrapy.utils.python import get_func_args, without_none_values
 
 
 logger = logging.getLogger(__name__)
 
 
+def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
+    argument_names = get_func_args(builder)
+    if 'feed_options' in argument_names:
+        kwargs['feed_options'] = feed_options
+    else:
+        warnings.warn(
+            "{} does not support the 'feed_options' keyword argument. Add a "
+            "'feed_options' parameter to its signature to remove this "
+            "warning. This parameter will become mandatory in a future "
+            "version of Scrapy."
+            .format(builder.__qualname__),
+            category=ScrapyDeprecationWarning
+        )
+    return builder(*preargs, uri, *args, **kwargs)
+
+
 class IFeedStorage(Interface):
     """Interface that all Feed Storages must implement"""
 
-    def __init__(uri):
-        """Initialize the storage with the parameters given in the URI"""
+    def __init__(uri, *, feed_options=None):
+        """Initialize the storage with the parameters given in the URI and the
+        feed-specific options (see :setting:`FEEDS`)"""
 
     def open(spider):
         """Open the storage for the given spider. It must return a file-like
@@ -64,10 +81,15 @@ class BlockingFeedStorage:
 @implementer(IFeedStorage)
 class StdoutFeedStorage:
 
-    def __init__(self, uri, _stdout=None):
+    def __init__(self, uri, _stdout=None, *, feed_options=None):
         if not _stdout:
             _stdout = sys.stdout.buffer
         self._stdout = _stdout
+        if feed_options and feed_options.get('overwrite', False) is True:
+            logger.warning('Standard output (stdout) storage does not support '
+                           'overwriting. To suppress this warning, remove the '
+                           'overwrite option from your FEEDS setting, or set '
+                           'it to False.')
 
     def open(self, spider):
         return self._stdout
@@ -79,14 +101,16 @@ class StdoutFeedStorage:
 @implementer(IFeedStorage)
 class FileFeedStorage:
 
-    def __init__(self, uri):
+    def __init__(self, uri, *, feed_options=None):
         self.path = file_uri_to_path(uri)
+        feed_options = feed_options or {}
+        self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab'
 
     def open(self, spider):
         dirname = os.path.dirname(self.path)
         if dirname and not os.path.exists(dirname):
             os.makedirs(dirname)
-        return open(self.path, 'ab')
+        return open(self.path, self.write_mode)
 
     def store(self, file):
         file.close()
@@ -94,7 +118,8 @@ class FileFeedStorage:
 
 class S3FeedStorage(BlockingFeedStorage):
 
-    def __init__(self, uri, access_key=None, secret_key=None, acl=None):
+    def __init__(self, uri, access_key=None, secret_key=None, acl=None, *,
+                 feed_options=None):
         u = urlparse(uri)
         self.bucketname = u.hostname
         self.access_key = u.username or access_key
@@ -111,14 +136,20 @@ class S3FeedStorage(BlockingFeedStorage):
         else:
             import boto
             self.connect_s3 = boto.connect_s3
+        if feed_options and feed_options.get('overwrite', True) is False:
+            logger.warning('S3 does not support appending to files. To '
+                           'suppress this warning, remove the overwrite '
+                           'option from your FEEDS setting or set it to True.')
 
     @classmethod
-    def from_crawler(cls, crawler, uri):
-        return cls(
-            uri=uri,
+    def from_crawler(cls, crawler, uri, *, feed_options=None):
+        return build_storage(
+            cls,
+            uri,
             access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
             secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
-            acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None
+            acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,
+            feed_options=feed_options,
         )
 
     def _store_in_thread(self, file):
@@ -135,6 +166,7 @@ class S3FeedStorage(BlockingFeedStorage):
             kwargs = {'policy': self.acl} if self.acl else {}
             key.set_contents_from_file(file, **kwargs)
             key.close()
+        file.close()
 
 
 class GCSFeedStorage(BlockingFeedStorage):
@@ -165,27 +197,31 @@ class GCSFeedStorage(BlockingFeedStorage):
 
 class FTPFeedStorage(BlockingFeedStorage):
 
-    def __init__(self, uri, use_active_mode=False):
+    def __init__(self, uri, use_active_mode=False, *, feed_options=None):
         u = urlparse(uri)
         self.host = u.hostname
         self.port = int(u.port or '21')
         self.username = u.username
-        self.password = unquote(u.password)
+        self.password = unquote(u.password or '')
         self.path = u.path
         self.use_active_mode = use_active_mode
+        self.overwrite = not feed_options or feed_options.get('overwrite', True)
 
     @classmethod
-    def from_crawler(cls, crawler, uri):
-        return cls(
-            uri=uri,
-            use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE')
+    def from_crawler(cls, crawler, uri, *, feed_options=None):
+        return build_storage(
+            cls,
+            uri,
+            crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE'),
+            feed_options=feed_options,
         )
 
     def _store_in_thread(self, file):
         ftp_store_file(
             path=self.path, file=file, host=self.host,
             port=self.port, username=self.username,
-            password=self.password, use_active_mode=self.use_active_mode
+            password=self.password, use_active_mode=self.use_active_mode,
+            overwrite=self.overwrite,
         )
 
 
@@ -242,32 +278,32 @@ class FeedExporter:
                 category=ScrapyDeprecationWarning, stacklevel=2,
             )
             uri = str(self.settings['FEED_URI'])  # handle pathlib.Path objects
-            feed = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
-            self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
+            feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
+            self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
         # End: Backward compatibility for FEED_URI and FEED_FORMAT settings
 
         # 'FEEDS' setting takes precedence over 'FEED_URI'
-        for uri, feed in self.settings.getdict('FEEDS').items():
+        for uri, feed_options in self.settings.getdict('FEEDS').items():
             uri = str(uri)  # handle pathlib.Path objects
-            self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
+            self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
-        for uri, feed in self.feeds.items():
-            if not self._storage_supported(uri):
+        for uri, feed_options in self.feeds.items():
+            if not self._storage_supported(uri, feed_options):
                 raise NotConfigured
             if not self._settings_are_valid():
                 raise NotConfigured
-            if not self._exporter_supported(feed['format']):
+            if not self._exporter_supported(feed_options['format']):
                 raise NotConfigured
 
     def open_spider(self, spider):
-        for uri, feed in self.feeds.items():
-            uri_params = self._get_uri_params(spider, feed['uri_params'])
+        for uri, feed_options in self.feeds.items():
+            uri_params = self._get_uri_params(spider, feed_options['uri_params'])
             self.slots.append(self._start_new_batch(
                 batch_id=1,
                 uri=uri % uri_params,
-                feed=feed,
+                feed_options=feed_options,
                 spider=spider,
                 uri_template=uri,
             ))
@@ -306,32 +342,32 @@ class FeedExporter:
         )
         return d
 
-    def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
+    def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template):
         """
         Redirect the output data stream to a new file.
         Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
         :param batch_id: sequence number of current batch
         :param uri: uri of the new batch to start
-        :param feed: dict with parameters of feed
+        :param feed_options: dict with parameters of feed
         :param spider: user spider
         :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri
         """
-        storage = self._get_storage(uri)
+        storage = self._get_storage(uri, feed_options)
         file = storage.open(spider)
         exporter = self._get_exporter(
             file=file,
-            format=feed['format'],
-            fields_to_export=feed['fields'],
-            encoding=feed['encoding'],
-            indent=feed['indent'],
+            format=feed_options['format'],
+            fields_to_export=feed_options['fields'],
+            encoding=feed_options['encoding'],
+            indent=feed_options['indent'],
         )
         slot = _FeedSlot(
             file=file,
             exporter=exporter,
             storage=storage,
             uri=uri,
-            format=feed['format'],
-            store_empty=feed['store_empty'],
+            format=feed_options['format'],
+            store_empty=feed_options['store_empty'],
             batch_id=batch_id,
             uri_template=uri_template,
         )
@@ -355,7 +391,7 @@ class FeedExporter:
                 slots.append(self._start_new_batch(
                     batch_id=slot.batch_id + 1,
                     uri=slot.uri_template % uri_params,
-                    feed=self.feeds[slot.uri_template],
+                    feed_options=self.feeds[slot.uri_template],
                     spider=spider,
                     uri_template=slot.uri_template,
                 ))
@@ -394,11 +430,11 @@ class FeedExporter:
                 return False
         return True
 
-    def _storage_supported(self, uri):
+    def _storage_supported(self, uri, feed_options):
         scheme = urlparse(uri).scheme
         if scheme in self.storages:
             try:
-                self._get_storage(uri)
+                self._get_storage(uri, feed_options)
                 return True
             except NotConfigured as e:
                 logger.error("Disabled feed storage scheme: %(scheme)s. "
@@ -416,8 +452,30 @@ class FeedExporter:
     def _get_exporter(self, file, format, *args, **kwargs):
         return self._get_instance(self.exporters[format], file, *args, **kwargs)
 
-    def _get_storage(self, uri):
-        return self._get_instance(self.storages[urlparse(uri).scheme], uri)
+    def _get_storage(self, uri, feed_options):
+        """Fork of create_instance specific to feed storage classes
+
+        It supports not passing the *feed_options* parameters to classes that
+        do not support it, and issuing a deprecation warning instead.
+        """
+        feedcls = self.storages[urlparse(uri).scheme]
+        crawler = getattr(self, 'crawler', None)
+
+        def build_instance(builder, *preargs):
+            return build_storage(builder, uri, preargs=preargs)
+
+        if crawler and hasattr(feedcls, 'from_crawler'):
+            instance = build_instance(feedcls.from_crawler, crawler)
+            method_name = 'from_crawler'
+        elif hasattr(feedcls, 'from_settings'):
+            instance = build_instance(feedcls.from_settings, self.settings)
+            method_name = 'from_settings'
+        else:
+            instance = build_instance(feedcls)
+            method_name = '__new__'
+        if instance is None:
+            raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name))
+        return instance
 
     def _get_uri_params(self, spider, uri_params, slot=None):
         params = {}
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index a83076c47..90a52b25b 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -127,7 +127,8 @@ def feed_complete_default_values_from_settings(feed, settings):
     return out
 
 
-def feed_process_params_from_cli(settings, output, output_format=None):
+def feed_process_params_from_cli(settings, output, output_format=None,
+                                 overwrite_output=None):
     """
     Receives feed export params (from the 'crawl' or 'runspider' commands),
     checks for inconsistencies in their quantities and returns a dictionary
@@ -139,22 +140,39 @@ def feed_process_params_from_cli(settings, output, output_format=None):
 
     def check_valid_format(output_format):
         if output_format not in valid_output_formats:
-            raise UsageError("Unrecognized output format '%s', set one after a"
-                             " colon using the -o option (i.e. -o <URI>:<FORMAT>)"
-                             " or as a file extension, from the supported list %s" %
-                             (output_format, tuple(valid_output_formats)))
+            raise UsageError(
+                "Unrecognized output format '%s'. Set a supported one (%s) "
+                "after a colon at the end of the output URI (i.e. -o/-O "
+                "<URI>:<FORMAT>) or as a file extension." % (
+                    output_format,
+                    tuple(valid_output_formats),
+                )
+            )
+
+    overwrite = False
+    if overwrite_output:
+        if output:
+            raise UsageError(
+                "Please use only one of -o/--output and -O/--overwrite-output"
+            )
+        output = overwrite_output
+        overwrite = True
 
     if output_format:
         if len(output) == 1:
             check_valid_format(output_format)
-            warnings.warn('The -t command line option is deprecated in favor'
-                          ' of specifying the output format within the -o'
-                          ' option, please check the -o option docs for more details',
-                          category=ScrapyDeprecationWarning, stacklevel=2)
+            message = (
+                'The -t command line option is deprecated in favor of '
+                'specifying the output format within the output URI. See the '
+                'documentation of the -o and -O options for more information.',
+            )
+            warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
             return {output[0]: {'format': output_format}}
         else:
-            raise UsageError('The -t command line option cannot be used if multiple'
-                             ' output files are specified with the -o option')
+            raise UsageError(
+                'The -t command-line option cannot be used if multiple output '
+                'URIs are specified'
+            )
 
     result = {}
     for element in output:
@@ -168,8 +186,10 @@ def feed_process_params_from_cli(settings, output, output_format=None):
                 feed_uri = 'stdout:'
         check_valid_format(feed_format)
         result[feed_uri] = {'format': feed_format}
+        if overwrite:
+            result[feed_uri]['overwrite'] = True
 
-    # FEEDS setting should take precedence over the -o and -t CLI options
+    # FEEDS setting should take precedence over the matching CLI options
     result.update(settings.getdict('FEEDS'))
 
     return result
diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index f07bdd748..19d56d6ec 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -20,7 +20,7 @@ def ftp_makedirs_cwd(ftp, path, first_call=True):
 
 def ftp_store_file(
         *, path, file, host, port,
-        username, password, use_active_mode=False):
+        username, password, use_active_mode=False, overwrite=True):
     """Opens a FTP connection with passed credentials,sets current directory
     to the directory extracted from given path, then uploads the file to server
     """
@@ -32,4 +32,6 @@ def ftp_store_file(
         file.seek(0)
         dirname, filename = posixpath.split(path)
         ftp_makedirs_cwd(ftp, dirname)
-        ftp.storbinary('STOR %s' % filename, file)
+        command = 'STOR' if overwrite else 'APPE'
+        ftp.storbinary('%s %s' % (command, filename), file)
+        file.close()
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 59f1b8371..1f2333264 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -198,7 +198,8 @@ def _getargspec_py23(func):
 def get_func_args(func, stripself=False):
     """Return the argument name list of a callable"""
     if inspect.isfunction(func):
-        func_args, _, _, _ = _getargspec_py23(func)
+        spec = inspect.getfullargspec(func)
+        func_args = spec.args + spec.kwonlyargs
     elif inspect.isclass(func):
         return get_func_args(func.__init__, True)
     elif inspect.ismethod(func):
diff --git a/tests/ftpserver.py b/tests/ftpserver.py
new file mode 100644
index 000000000..6f0289e08
--- /dev/null
+++ b/tests/ftpserver.py
@@ -0,0 +1,24 @@
+from argparse import ArgumentParser
+
+from pyftpdlib.authorizers import DummyAuthorizer
+from pyftpdlib.handlers import FTPHandler
+from pyftpdlib.servers import FTPServer
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument('-d', '--directory')
+    args = parser.parse_args()
+
+    authorizer = DummyAuthorizer()
+    full_permissions = 'elradfmwMT'
+    authorizer.add_anonymous(args.directory, perm=full_permissions)
+    handler = FTPHandler
+    handler.authorizer = authorizer
+    address = ('127.0.0.1', 2121)
+    server = FTPServer(address, handler)
+    server.serve_forever()
+
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 1f40473ba..48d7b8d37 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -3,7 +3,10 @@ import json
 import os
 import random
 import sys
+from pathlib import Path
+from shutil import rmtree
 from subprocess import Popen, PIPE
+from tempfile import mkdtemp
 from urllib.parse import urlencode
 
 from OpenSSL import SSL
@@ -256,6 +259,29 @@ class MockDNSServer:
         self.proc.communicate()
 
 
+class MockFTPServer:
+    """Creates an FTP server on port 2121 with a default passwordless user
+    (anonymous) and a temporary root path that you can read from the
+    :attr:`path` attribute."""
+
+    def __enter__(self):
+        self.path = Path(mkdtemp())
+        self.proc = Popen([sys.executable, '-u', '-m', 'tests.ftpserver', '-d', str(self.path)],
+                          stderr=PIPE, env=get_testenv())
+        for line in self.proc.stderr:
+            if b'starting FTP server' in line:
+                break
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        rmtree(str(self.path))
+        self.proc.kill()
+        self.proc.communicate()
+
+    def url(self, path):
+        return 'ftp://127.0.0.1:2121/' + path
+
+
 def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):
     factory = ssl.DefaultOpenSSLContextFactory(
         os.path.join(os.path.dirname(__file__), keyfile),
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index b51177abb..fe1cbc997 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -4,6 +4,7 @@ dataclasses; python_version == '3.6'
 mitmproxy; python_version >= '3.7'
 mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7'
 mitmproxy < 4; python_version < '3.6'
+pyftpdlib
 # https://github.com/pytest-dev/pytest-twisted/issues/93
 pytest != 5.4, != 5.4.1
 pytest-azurepipelines
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 109f006a8..f76f851e7 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -74,6 +74,7 @@ class ProjectTest(unittest.TestCase):
 
         def kill_proc():
             p.kill()
+            p.communicate()
             assert False, 'Command took too much time to complete'
 
         timer = Timer(15, kill_proc)
@@ -569,6 +570,55 @@ class BadSpider(scrapy.Spider):
         log = self.get_log(self.debug_log_spider, args=[])
         self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    def test_output(self):
+        spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+
+    def start_requests(self):
+        self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))
+        return []
+"""
+        args = ['-o', 'example.json']
+        log = self.get_log(spider_code, args=args)
+        self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log)
+
+    def test_overwrite_output(self):
+        spider_code = """
+import json
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+
+    def start_requests(self):
+        self.logger.debug(
+            'FEEDS: {}'.format(
+                json.dumps(self.settings.getdict('FEEDS'), sort_keys=True)
+            )
+        )
+        return []
+"""
+        args = ['-O', 'example.json']
+        log = self.get_log(spider_code, args=args)
+        self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
+
+    def test_output_and_overwrite_output(self):
+        spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+
+    def start_requests(self):
+        return []
+"""
+        args = ['-o', 'example1.json', '-O', 'example2.json']
+        log = self.get_log(spider_code, args=args)
+        self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
+
 
 class BenchCommandTest(CommandTest):
 
@@ -577,3 +627,79 @@ class BenchCommandTest(CommandTest):
                               '-s', 'CLOSESPIDER_TIMEOUT=0.01')
         self.assertIn('INFO: Crawled', log)
         self.assertNotIn('Unhandled Error', log)
+
+
+class CrawlCommandTest(CommandTest):
+
+    def crawl(self, code, args=()):
+        fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
+        with open(fname, 'w') as f:
+            f.write(code)
+        return self.proc('crawl', 'myspider', *args)
+
+    def get_log(self, code, args=()):
+        _, _, stderr = self.crawl(code, args=args)
+        return stderr
+
+    def test_no_output(self):
+        spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+
+    def start_requests(self):
+        self.logger.debug('It works!')
+        return []
+"""
+        log = self.get_log(spider_code)
+        self.assertIn("[myspider] DEBUG: It works!", log)
+
+    def test_output(self):
+        spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+
+    def start_requests(self):
+        self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))
+        return []
+"""
+        args = ['-o', 'example.json']
+        log = self.get_log(spider_code, args=args)
+        self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log)
+
+    def test_overwrite_output(self):
+        spider_code = """
+import json
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+
+    def start_requests(self):
+        self.logger.debug(
+            'FEEDS: {}'.format(
+                json.dumps(self.settings.getdict('FEEDS'), sort_keys=True)
+            )
+        )
+        return []
+"""
+        args = ['-O', 'example.json']
+        log = self.get_log(spider_code, args=args)
+        self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
+
+    def test_output_and_overwrite_output(self):
+        spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+
+    def start_requests(self):
+        return []
+"""
+        args = ['-o', 'example1.json', '-O', 'example2.json']
+        log = self.get_log(spider_code, args=args)
+        self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 2afc25a7a..850485b5e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -5,6 +5,7 @@ import random
 import shutil
 import string
 import tempfile
+import warnings
 from abc import ABC, abstractmethod
 from collections import defaultdict
 from io import BytesIO
@@ -25,7 +26,7 @@ from zope.interface.verify import verifyObject
 
 import scrapy
 from scrapy.crawler import CrawlerRunner
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.exporters import CsvItemExporter
 from scrapy.extensions.feedexport import (
     BlockingFeedStorage,
@@ -46,7 +47,7 @@ from scrapy.utils.test import (
     mock_google_cloud_storage,
 )
 
-from tests.mockserver import MockServer
+from tests.mockserver import MockFTPServer, MockServer
 
 
 class FileFeedStorageTest(unittest.TestCase):
@@ -75,8 +76,28 @@ class FileFeedStorageTest(unittest.TestCase):
         st = FileFeedStorage(path)
         verifyObject(IFeedStorage, st)
 
+    def _store(self, feed_options=None):
+        path = os.path.abspath(self.mktemp())
+        storage = FileFeedStorage(path, feed_options=feed_options)
+        spider = scrapy.Spider("default")
+        file = storage.open(spider)
+        file.write(b"content")
+        storage.store(file)
+        return path
+
+    def test_append(self):
+        path = self._store()
+        return self._assert_stores(FileFeedStorage(path), path, b"contentcontent")
+
+    def test_overwrite(self):
+        path = self._store({"overwrite": True})
+        return self._assert_stores(
+            FileFeedStorage(path, feed_options={"overwrite": True}),
+            path
+        )
+
     @defer.inlineCallbacks
-    def _assert_stores(self, storage, path):
+    def _assert_stores(self, storage, path, expected_content=b"content"):
         spider = scrapy.Spider("default")
         file = storage.open(spider)
         file.write(b"content")
@@ -84,7 +105,7 @@ class FileFeedStorageTest(unittest.TestCase):
         self.assertTrue(os.path.exists(path))
         try:
             with open(path, 'rb') as fp:
-                self.assertEqual(fp.read(), b"content")
+                self.assertEqual(fp.read(), expected_content)
         finally:
             os.unlink(path)
 
@@ -99,49 +120,74 @@ class FTPFeedStorageTest(unittest.TestCase):
         spider = TestSpider.from_crawler(crawler)
         return spider
 
-    def test_store(self):
-        uri = os.environ.get('FEEDTEST_FTP_URI')
-        path = os.environ.get('FEEDTEST_FTP_PATH')
-        if not (uri and path):
-            raise unittest.SkipTest("No FTP server available for testing")
-        st = FTPFeedStorage(uri)
-        verifyObject(IFeedStorage, st)
-        return self._assert_stores(st, path)
+    def _store(self, uri, content, feed_options=None, settings=None):
+        crawler = get_crawler(settings_dict=settings or {})
+        storage = FTPFeedStorage.from_crawler(
+            crawler,
+            uri,
+            feed_options=feed_options,
+        )
+        verifyObject(IFeedStorage, storage)
+        spider = self.get_test_spider()
+        file = storage.open(spider)
+        file.write(content)
+        return storage.store(file)
 
-    def test_store_active_mode(self):
-        uri = os.environ.get('FEEDTEST_FTP_URI')
-        path = os.environ.get('FEEDTEST_FTP_PATH')
-        if not (uri and path):
-            raise unittest.SkipTest("No FTP server available for testing")
-        use_active_mode = {'FEED_STORAGE_FTP_ACTIVE': True}
-        crawler = get_crawler(settings_dict=use_active_mode)
-        st = FTPFeedStorage.from_crawler(crawler, uri)
-        verifyObject(IFeedStorage, st)
-        return self._assert_stores(st, path)
+    def _assert_stored(self, path, content):
+        self.assertTrue(path.exists())
+        try:
+            with path.open('rb') as fp:
+                self.assertEqual(fp.read(), content)
+        finally:
+            os.unlink(str(path))
+
+    @defer.inlineCallbacks
+    def test_append(self):
+        with MockFTPServer() as ftp_server:
+            filename = 'file'
+            url = ftp_server.url(filename)
+            feed_options = {'overwrite': False}
+            yield self._store(url, b"foo", feed_options=feed_options)
+            yield self._store(url, b"bar", feed_options=feed_options)
+            self._assert_stored(ftp_server.path / filename, b"foobar")
+
+    @defer.inlineCallbacks
+    def test_overwrite(self):
+        with MockFTPServer() as ftp_server:
+            filename = 'file'
+            url = ftp_server.url(filename)
+            yield self._store(url, b"foo")
+            yield self._store(url, b"bar")
+            self._assert_stored(ftp_server.path / filename, b"bar")
+
+    @defer.inlineCallbacks
+    def test_append_active_mode(self):
+        with MockFTPServer() as ftp_server:
+            settings = {'FEED_STORAGE_FTP_ACTIVE': True}
+            filename = 'file'
+            url = ftp_server.url(filename)
+            feed_options = {'overwrite': False}
+            yield self._store(url, b"foo", feed_options=feed_options, settings=settings)
+            yield self._store(url, b"bar", feed_options=feed_options, settings=settings)
+            self._assert_stored(ftp_server.path / filename, b"foobar")
+
+    @defer.inlineCallbacks
+    def test_overwrite_active_mode(self):
+        with MockFTPServer() as ftp_server:
+            settings = {'FEED_STORAGE_FTP_ACTIVE': True}
+            filename = 'file'
+            url = ftp_server.url(filename)
+            yield self._store(url, b"foo", settings=settings)
+            yield self._store(url, b"bar", settings=settings)
+            self._assert_stored(ftp_server.path / filename, b"bar")
 
     def test_uri_auth_quote(self):
         # RFC3986: 3.2.1. User Information
         pw_quoted = quote(string.punctuation, safe='')
-        st = FTPFeedStorage('ftp://foo:%s@example.com/some_path' % pw_quoted)
+        st = FTPFeedStorage('ftp://foo:%s@example.com/some_path' % pw_quoted,
+                            {})
         self.assertEqual(st.password, string.punctuation)
 
-    @defer.inlineCallbacks
-    def _assert_stores(self, storage, path):
-        spider = self.get_test_spider()
-        file = storage.open(spider)
-        file.write(b"content")
-        yield storage.store(file)
-        self.assertTrue(os.path.exists(path))
-        try:
-            with open(path, 'rb') as fp:
-                self.assertEqual(fp.read(), b"content")
-            # again, to check s3 objects are overwritten
-            yield storage.store(BytesIO(b"new content"))
-            with open(path, 'rb') as fp:
-                self.assertEqual(fp.read(), b"new content")
-        finally:
-            os.unlink(path)
-
 
 class BlockingFeedStorageTest(unittest.TestCase):
 
@@ -190,8 +236,10 @@ class S3FeedStorageTest(unittest.TestCase):
                            'AWS_SECRET_ACCESS_KEY': 'settings_secret'}
         crawler = get_crawler(settings_dict=aws_credentials)
         # Instantiate with crawler
-        storage = S3FeedStorage.from_crawler(crawler,
-                                             's3://mybucket/export.csv')
+        storage = S3FeedStorage.from_crawler(
+            crawler,
+            's3://mybucket/export.csv',
+        )
         self.assertEqual(storage.access_key, 'settings_key')
         self.assertEqual(storage.secret_key, 'settings_secret')
         # Instantiate directly
@@ -254,7 +302,7 @@ class S3FeedStorageTest(unittest.TestCase):
         crawler = get_crawler(settings_dict=settings)
         storage = S3FeedStorage.from_crawler(
             crawler,
-            's3://mybucket/export.csv'
+            's3://mybucket/export.csv',
         )
         self.assertEqual(storage.access_key, 'access_key')
         self.assertEqual(storage.secret_key, 'secret_key')
@@ -269,7 +317,7 @@ class S3FeedStorageTest(unittest.TestCase):
         crawler = get_crawler(settings_dict=settings)
         storage = S3FeedStorage.from_crawler(
             crawler,
-            's3://mybucket/export.csv'
+            's3://mybucket/export.csv',
         )
         self.assertEqual(storage.access_key, 'access_key')
         self.assertEqual(storage.secret_key, 'secret_key')
@@ -370,6 +418,27 @@ class S3FeedStorageTest(unittest.TestCase):
             key.set_contents_from_file.call_args
         )
 
+    def test_overwrite_default(self):
+        with LogCapture() as log:
+            S3FeedStorage(
+                's3://mybucket/export.csv',
+                'access_key',
+                'secret_key',
+                'custom-acl'
+            )
+        self.assertNotIn('S3 does not support appending to files', str(log))
+
+    def test_overwrite_false(self):
+        with LogCapture() as log:
+            S3FeedStorage(
+                's3://mybucket/export.csv',
+                'access_key',
+                'secret_key',
+                'custom-acl',
+                feed_options={'overwrite': False},
+            )
+        self.assertIn('S3 does not support appending to files', str(log))
+
 
 class GCSFeedStorageTest(unittest.TestCase):
 
@@ -439,12 +508,22 @@ class StdoutFeedStorageTest(unittest.TestCase):
         yield storage.store(file)
         self.assertEqual(out.getvalue(), b"content")
 
+    def test_overwrite_default(self):
+        with LogCapture() as log:
+            StdoutFeedStorage('stdout:')
+        self.assertNotIn('Standard output (stdout) storage does not support overwriting', str(log))
+
+    def test_overwrite_true(self):
+        with LogCapture() as log:
+            StdoutFeedStorage('stdout:', feed_options={'overwrite': True})
+        self.assertIn('Standard output (stdout) storage does not support overwriting', str(log))
+
 
 class FromCrawlerMixin:
     init_with_crawler = False
 
     @classmethod
-    def from_crawler(cls, crawler, *args, **kwargs):
+    def from_crawler(cls, crawler, *args, feed_options=None, **kwargs):
         cls.init_with_crawler = True
         return cls(*args, **kwargs)
 
@@ -454,7 +533,11 @@ class FromCrawlerCsvItemExporter(CsvItemExporter, FromCrawlerMixin):
 
 
 class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
-    pass
+
+    @classmethod
+    def from_crawler(cls, crawler, *args, feed_options=None, **kwargs):
+        cls.init_with_crawler = True
+        return cls(*args, feed_options=feed_options, **kwargs)
 
 
 class DummyBlockingFeedStorage(BlockingFeedStorage):
@@ -588,8 +671,8 @@ class FeedExportTest(FeedExportTestBase):
 
         FEEDS = settings.get('FEEDS') or {}
         settings['FEEDS'] = {
-            printf_escape(path_to_url(file_path)): feed
-            for file_path, feed in FEEDS.items()
+            printf_escape(path_to_url(file_path)): feed_options
+            for file_path, feed_options in FEEDS.items()
         }
 
         content = {}
@@ -599,12 +682,12 @@ class FeedExportTest(FeedExportTestBase):
                 spider_cls.start_urls = [s.url('/')]
                 yield runner.crawl(spider_cls)
 
-            for file_path, feed in FEEDS.items():
+            for file_path, feed_options in FEEDS.items():
                 if not os.path.exists(str(file_path)):
                     continue
 
                 with open(str(file_path), 'rb') as f:
-                    content[feed['format']] = f.read()
+                    content[feed_options['format']] = f.read()
 
         finally:
             for file_path in FEEDS.keys():
@@ -1542,3 +1625,262 @@ class BatchDeliveriesTest(FeedExportTestBase):
             content = json.loads(content.decode('utf-8'))
             expected_batch, items = items[:batch_size], items[batch_size:]
             self.assertEqual(expected_batch, content)
+
+
+class FeedExportInitTest(unittest.TestCase):
+
+    def test_unsupported_storage(self):
+        settings = {
+            'FEEDS': {
+                'unsupported://uri': {},
+            },
+        }
+        crawler = get_crawler(settings_dict=settings)
+        with self.assertRaises(NotConfigured):
+            FeedExporter.from_crawler(crawler)
+
+    def test_unsupported_format(self):
+        settings = {
+            'FEEDS': {
+                'file://path': {
+                    'format': 'unsupported_format',
+                },
+            },
+        }
+        crawler = get_crawler(settings_dict=settings)
+        with self.assertRaises(NotConfigured):
+            FeedExporter.from_crawler(crawler)
+
+
+class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage):
+
+    def __init__(self, uri):
+        super().__init__(uri)
+
+
+class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase):
+    """Make sure that any feed exporter created by users before the
+    introduction of the ``feed_options`` parameter continues to work as
+    expected, and simply issues a warning."""
+
+    def test_init(self):
+        settings_dict = {
+            'FEED_URI': 'file:///tmp/foobar',
+            'FEED_STORAGES': {
+                'file': 'tests.test_feedexport.StdoutFeedStorageWithoutFeedOptions'
+            },
+        }
+        crawler = get_crawler(settings_dict=settings_dict)
+        feed_exporter = FeedExporter.from_crawler(crawler)
+        spider = scrapy.Spider("default")
+        with warnings.catch_warnings(record=True) as w:
+            feed_exporter.open_spider(spider)
+            messages = tuple(str(item.message) for item in w
+                             if item.category is ScrapyDeprecationWarning)
+            self.assertEqual(
+                messages,
+                (
+                    (
+                        "StdoutFeedStorageWithoutFeedOptions does not support "
+                        "the 'feed_options' keyword argument. Add a "
+                        "'feed_options' parameter to its signature to remove "
+                        "this warning. This parameter will become mandatory "
+                        "in a future version of Scrapy."
+                    ),
+                )
+            )
+
+
+class FileFeedStorageWithoutFeedOptions(FileFeedStorage):
+
+    def __init__(self, uri):
+        super().__init__(uri)
+
+
+class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
+    """Make sure that any feed exporter created by users before the
+    introduction of the ``feed_options`` parameter continues to work as
+    expected, and simply issues a warning."""
+
+    maxDiff = None
+
+    def test_init(self):
+        settings_dict = {
+            'FEED_URI': 'file:///tmp/foobar',
+            'FEED_STORAGES': {
+                'file': 'tests.test_feedexport.FileFeedStorageWithoutFeedOptions'
+            },
+        }
+        crawler = get_crawler(settings_dict=settings_dict)
+        feed_exporter = FeedExporter.from_crawler(crawler)
+        spider = scrapy.Spider("default")
+        with warnings.catch_warnings(record=True) as w:
+            feed_exporter.open_spider(spider)
+            messages = tuple(str(item.message) for item in w
+                             if item.category is ScrapyDeprecationWarning)
+            self.assertEqual(
+                messages,
+                (
+                    (
+                        "FileFeedStorageWithoutFeedOptions does not support "
+                        "the 'feed_options' keyword argument. Add a "
+                        "'feed_options' parameter to its signature to remove "
+                        "this warning. This parameter will become mandatory "
+                        "in a future version of Scrapy."
+                    ),
+                )
+            )
+
+
+class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
+
+    def __init__(self, uri, access_key, secret_key, acl):
+        super().__init__(uri, access_key, secret_key, acl)
+
+
+class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):
+
+    @classmethod
+    def from_crawler(cls, crawler, uri):
+        return super().from_crawler(crawler, uri)
+
+
+class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
+    """Make sure that any feed exporter created by users before the
+    introduction of the ``feed_options`` parameter continues to work as
+    expected, and simply issues a warning."""
+
+    maxDiff = None
+
+    def test_init(self):
+        settings_dict = {
+            'FEED_URI': 'file:///tmp/foobar',
+            'FEED_STORAGES': {
+                'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptions'
+            },
+        }
+        crawler = get_crawler(settings_dict=settings_dict)
+        feed_exporter = FeedExporter.from_crawler(crawler)
+        spider = scrapy.Spider("default")
+        spider.crawler = crawler
+        with warnings.catch_warnings(record=True) as w:
+            feed_exporter.open_spider(spider)
+            messages = tuple(str(item.message) for item in w
+                             if item.category is ScrapyDeprecationWarning)
+            self.assertEqual(
+                messages,
+                (
+                    (
+                        "S3FeedStorageWithoutFeedOptions does not support "
+                        "the 'feed_options' keyword argument. Add a "
+                        "'feed_options' parameter to its signature to remove "
+                        "this warning. This parameter will become mandatory "
+                        "in a future version of Scrapy."
+                    ),
+                )
+            )
+
+    def test_from_crawler(self):
+        settings_dict = {
+            'FEED_URI': 'file:///tmp/foobar',
+            'FEED_STORAGES': {
+                'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptionsWithFromCrawler'
+            },
+        }
+        crawler = get_crawler(settings_dict=settings_dict)
+        feed_exporter = FeedExporter.from_crawler(crawler)
+        spider = scrapy.Spider("default")
+        spider.crawler = crawler
+        with warnings.catch_warnings(record=True) as w:
+            feed_exporter.open_spider(spider)
+            messages = tuple(str(item.message) for item in w
+                             if item.category is ScrapyDeprecationWarning)
+            self.assertEqual(
+                messages,
+                (
+                    (
+                        "S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "
+                        "does not support the 'feed_options' keyword argument. Add a "
+                        "'feed_options' parameter to its signature to remove "
+                        "this warning. This parameter will become mandatory "
+                        "in a future version of Scrapy."
+                    ),
+                )
+            )
+
+
+class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage):
+
+    def __init__(self, uri, use_active_mode=False):
+        super().__init__(uri)
+
+
+class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage):
+
+    @classmethod
+    def from_crawler(cls, crawler, uri):
+        return super().from_crawler(crawler, uri)
+
+
+class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
+    """Make sure that any feed exporter created by users before the
+    introduction of the ``feed_options`` parameter continues to work as
+    expected, and simply issues a warning."""
+
+    maxDiff = None
+
+    def test_init(self):
+        settings_dict = {
+            'FEED_URI': 'file:///tmp/foobar',
+            'FEED_STORAGES': {
+                'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptions'
+            },
+        }
+        crawler = get_crawler(settings_dict=settings_dict)
+        feed_exporter = FeedExporter.from_crawler(crawler)
+        spider = scrapy.Spider("default")
+        spider.crawler = crawler
+        with warnings.catch_warnings(record=True) as w:
+            feed_exporter.open_spider(spider)
+            messages = tuple(str(item.message) for item in w
+                             if item.category is ScrapyDeprecationWarning)
+            self.assertEqual(
+                messages,
+                (
+                    (
+                        "FTPFeedStorageWithoutFeedOptions does not support "
+                        "the 'feed_options' keyword argument. Add a "
+                        "'feed_options' parameter to its signature to remove "
+                        "this warning. This parameter will become mandatory "
+                        "in a future version of Scrapy."
+                    ),
+                )
+            )
+
+    def test_from_crawler(self):
+        settings_dict = {
+            'FEED_URI': 'file:///tmp/foobar',
+            'FEED_STORAGES': {
+                'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptionsWithFromCrawler'
+            },
+        }
+        crawler = get_crawler(settings_dict=settings_dict)
+        feed_exporter = FeedExporter.from_crawler(crawler)
+        spider = scrapy.Spider("default")
+        spider.crawler = crawler
+        with warnings.catch_warnings(record=True) as w:
+            feed_exporter.open_spider(spider)
+            messages = tuple(str(item.message) for item in w
+                             if item.category is ScrapyDeprecationWarning)
+            self.assertEqual(
+                messages,
+                (
+                    (
+                        "FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "
+                        "does not support the 'feed_options' keyword argument. Add a "
+                        "'feed_options' parameter to its signature to remove "
+                        "this warning. This parameter will become mandatory "
+                        "in a future version of Scrapy."
+                    ),
+                )
+            )
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index f3ef36127..ccc65c4fd 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -141,6 +141,22 @@ class FeedExportConfigTestCase(unittest.TestCase):
             feed_process_params_from_cli(settings, ['-:pickle'])
         )
 
+    def test_feed_export_config_overwrite(self):
+        settings = Settings()
+        self.assertEqual(
+            {'output.json': {'format': 'json', 'overwrite': True}},
+            feed_process_params_from_cli(settings, [], None, ['output.json'])
+        )
+
+    def test_output_and_overwrite_output(self):
+        with self.assertRaises(UsageError):
+            feed_process_params_from_cli(
+                Settings(),
+                ['output1.json'],
+                None,
+                ['output2.json'],
+            )
+
     def test_feed_complete_default_values_from_settings_empty(self):
         feed = {}
         settings = Settings({
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3f93f509e..c298d0bd2 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -179,6 +179,9 @@ class UtilsPythonTestCase(unittest.TestCase):
         def f2(a, b=None, c=None):
             pass
 
+        def f3(a, b=None, *, c=None):
+            pass
+
         class A:
             def __init__(self, a, b, c):
                 pass
@@ -199,6 +202,7 @@ class UtilsPythonTestCase(unittest.TestCase):
 
         self.assertEqual(get_func_args(f1), ['a', 'b', 'c'])
         self.assertEqual(get_func_args(f2), ['a', 'b', 'c'])
+        self.assertEqual(get_func_args(f3), ['a', 'b', 'c'])
         self.assertEqual(get_func_args(A), ['a', 'b', 'c'])
         self.assertEqual(get_func_args(a.method), ['a', 'b', 'c'])
         self.assertEqual(get_func_args(partial_f1), ['b', 'c'])

From 42383cc267393c3c5fb89c9108759125939ab3e5 Mon Sep 17 00:00:00 2001
From: sakshamb2113 <44064539+sakshamb2113@users.noreply.github.com>
Date: Wed, 19 Aug 2020 12:48:14 +0530
Subject: [PATCH 2971/2974] Add a setting to customize the asyncio event loop
 (#4414)

---
 docs/topics/asyncio.rst                     | 12 +++++++++++
 docs/topics/settings.rst                    | 20 ++++++++++++++++++
 scrapy/crawler.py                           |  2 +-
 scrapy/settings/default_settings.py         |  2 ++
 scrapy/utils/log.py                         |  7 +++++++
 scrapy/utils/reactor.py                     | 12 ++++++++---
 tests/CrawlerProcess/asyncio_custom_loop.py | 17 +++++++++++++++
 tests/requirements-py3.txt                  |  1 +
 tests/test_commands.py                      | 23 +++++++++++++++++++++
 tests/test_crawler.py                       |  8 +++++++
 10 files changed, 100 insertions(+), 4 deletions(-)
 create mode 100644 tests/CrawlerProcess/asyncio_custom_loop.py

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 038a459fd..bfb430d52 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -26,3 +26,15 @@ reactor manually. You can do that using
 :func:`~scrapy.utils.reactor.install_reactor`::
 
     install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
+
+.. _using-custom-loops:
+
+Using custom asyncio loops
+==========================    
+
+You can also use custom asyncio event loops with the asyncio reactor. Set the
+:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to
+use it instead of the default asyncio event loop.
+
+
+
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 722ae4593..618b9989e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -216,6 +216,26 @@ Default: ``None``
 
 The name of the region associated with the AWS client.
 
+.. setting:: ASYNCIO_EVENT_LOOP
+
+ASYNCIO_EVENT_LOOP
+------------------
+
+Default: ``None``
+
+Import path of a given asyncio event loop class.
+
+If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the 
+asyncio event loop to be used with it. Set the setting to the import path of the 
+desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
+event loop will be used.
+
+If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
+function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop 
+class to be used.  
+
+Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
+
 .. setting:: BOT_NAME
 
 BOT_NAME
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index d028bea4d..4c6b0e496 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -340,5 +340,5 @@ class CrawlerProcess(CrawlerRunner):
 
     def _handle_twisted_reactor(self):
         if self.settings.get("TWISTED_REACTOR"):
-            install_reactor(self.settings["TWISTED_REACTOR"])
+            install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
         super()._handle_twisted_reactor()
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 896afa995..a0251394b 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -19,6 +19,8 @@ from os.path import join, abspath, dirname
 
 AJAXCRAWL_ENABLED = False
 
+ASYNCIO_EVENT_LOOP = None
+
 AUTOTHROTTLE_ENABLED = False
 AUTOTHROTTLE_DEBUG = False
 AUTOTHROTTLE_MAX_DELAY = 60.0
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 1d6a2c39d..e41315738 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -150,6 +150,13 @@ def log_scrapy_info(settings):
     logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
     from twisted.internet import reactor
     logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
+    from twisted.internet import asyncioreactor
+    if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):
+        logger.debug(
+            "Using asyncio event loop: %s.%s",
+            reactor._asyncioEventloop.__module__,
+            reactor._asyncioEventloop.__class__.__name__,
+        )
 
 
 class StreamLogger:
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 3c705f69b..879d27907 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -50,13 +50,19 @@ class CallLaterOnce:
         return self._func(*self._a, **self._kw)
 
 
-def install_reactor(reactor_path):
+def install_reactor(reactor_path, event_loop_path=None):
     """Installs the :mod:`~twisted.internet.reactor` with the specified
-    import path."""
+    import path. Also installs the asyncio event loop with the specified import
+    path if the asyncio reactor is enabled"""
     reactor_class = load_object(reactor_path)
     if reactor_class is asyncioreactor.AsyncioSelectorReactor:
         with suppress(error.ReactorAlreadyInstalledError):
-            asyncioreactor.install(asyncio.get_event_loop())
+            if event_loop_path is not None:
+                event_loop_class = load_object(event_loop_path)
+                event_loop = event_loop_class()
+            else:
+                event_loop = asyncio.new_event_loop()
+            asyncioreactor.install(eventloop=event_loop)
     else:
         *module, _ = reactor_path.split(".")
         installer_path = module + ["install"]
diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py
new file mode 100644
index 000000000..1e4ada722
--- /dev/null
+++ b/tests/CrawlerProcess/asyncio_custom_loop.py
@@ -0,0 +1,17 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    "ASYNCIO_EVENT_LOOP": "uvloop.Loop"
+})
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index fe1cbc997..44ddcded8 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -13,6 +13,7 @@ pytest-twisted >= 1.11
 pytest-xdist
 sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
 testfixtures
+uvloop; platform_system != "Windows"
 
 # optional for shell wrapper tests
 bpython
diff --git a/tests/test_commands.py b/tests/test_commands.py
index f76f851e7..ee8a92604 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -16,6 +16,7 @@ from tempfile import mkdtemp
 from threading import Timer
 from unittest import skipIf
 
+from pytest import mark
 from twisted.trial import unittest
 
 import scrapy
@@ -570,6 +571,28 @@ class BadSpider(scrapy.Spider):
         log = self.get_log(self.debug_log_spider, args=[])
         self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
+    @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+    def test_custom_asyncio_loop_enabled_true(self):
+        log = self.get_log(self.debug_log_spider, args=[
+            '-s',
+            'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor',
+            '-s',
+            'ASYNCIO_EVENT_LOOP=uvloop.Loop',
+        ])
+        self.assertIn("Using asyncio event loop: uvloop.Loop", log)
+
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
+    def test_custom_asyncio_loop_enabled_false(self):
+        log = self.get_log(self.debug_log_spider, args=[
+            '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'
+        ])
+        import asyncio
+        loop = asyncio.new_event_loop()
+        self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log)
+
     def test_output(self):
         spider_code = """
 import scrapy
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 1a4cfe813..7c2e251a9 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -345,6 +345,14 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
+    @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+    def test_custom_loop_asyncio(self):
+        log = self.run_script("asyncio_custom_loop.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertIn("Using asyncio event loop: uvloop.Loop", log)
+
 
 class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
     script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')

From a57db9e3024fe1426021b8b692a48f4c8db82a77 Mon Sep 17 00:00:00 2001
From: Hugo van Kemenade <hugovk@users.noreply.github.com>
Date: Wed, 19 Aug 2020 18:45:24 +0300
Subject: [PATCH 2972/2974] Bitbucket no longer supports Mercurial repositories
 (#4738)

---
 .travis.yml | 2 +-
 setup.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index db720b918..33a920bb6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -50,7 +50,7 @@ install:
   - |
       if [[ ! -z "$PYPY_VERSION" ]]; then
         export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
-        wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2"
+        wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
         tar -jxf ${PYPY_VERSION}.tar.bz2
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
diff --git a/setup.py b/setup.py
index d0880051f..52a27c368 100644
--- a/setup.py
+++ b/setup.py
@@ -41,7 +41,7 @@ if has_environment_marker_platform_impl_support():
     ]
     extras_require[':platform_python_implementation == "PyPy"'] = [
         # Earlier lxml versions are affected by
-        # https://bitbucket.org/pypy/pypy/issues/2498/cython-on-pypy-3-dict-object-has-no,
+        # https://foss.heptapod.net/pypy/pypy/-/issues/2498,
         # which was fixed in Cython 0.26, released on 2017-06-19, and used to
         # generate the C headers of lxml release tarballs published since then, the
         # first of which was:

From d68aab992e435aa1c88061e83d03e57e7d31533d Mon Sep 17 00:00:00 2001
From: Grisha Temchenko <soid.exe@gmail.com>
Date: Thu, 20 Aug 2020 09:22:07 -0400
Subject: [PATCH 2973/2974] Smarter generator check for combined return/yield
 statements (#4721)

---
 scrapy/utils/misc.py                          | 19 ++++++++++++++++++-
 ...t_return_with_argument_inside_generator.py | 19 +++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index d6966be8e..bd400bd30 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -5,6 +5,7 @@ import os
 import re
 import hashlib
 import warnings
+from collections import deque
 from contextlib import contextmanager
 from importlib import import_module
 from pkgutil import iter_modules
@@ -184,6 +185,22 @@ def set_environ(**kwargs):
                 os.environ[k] = v
 
 
+def walk_callable(node):
+    """Similar to ``ast.walk``, but walks only function body and skips nested
+    functions defined within the node.
+    """
+    todo = deque([node])
+    walked_func_def = False
+    while todo:
+        node = todo.popleft()
+        if isinstance(node, ast.FunctionDef):
+            if walked_func_def:
+                continue
+            walked_func_def = True
+        todo.extend(ast.iter_child_nodes(node))
+        yield node
+
+
 _generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
 
 
@@ -201,7 +218,7 @@ def is_generator_with_return_value(callable):
 
     if inspect.isgeneratorfunction(callable):
         tree = ast.parse(dedent(inspect.getsource(callable)))
-        for node in ast.walk(tree):
+        for node in walk_callable(tree):
             if isinstance(node, ast.Return) and not returns_none(node):
                 _generator_callbacks_cache[callable] = True
                 return _generator_callbacks_cache[callable]
diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
index bdbec1beb..2be38620c 100644
--- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py
+++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
@@ -29,9 +29,28 @@ class UtilsMiscPy3TestCase(unittest.TestCase):
             yield 1
             yield from g()
 
+        def m():
+            yield 1
+
+            def helper():
+                return 0
+
+            yield helper()
+
+        def n():
+            yield 1
+
+            def helper():
+                return 0
+
+            yield helper()
+            return 2
+
         assert is_generator_with_return_value(f)
         assert is_generator_with_return_value(g)
         assert not is_generator_with_return_value(h)
         assert not is_generator_with_return_value(i)
         assert not is_generator_with_return_value(j)
         assert not is_generator_with_return_value(k)  # not recursive
+        assert not is_generator_with_return_value(m)
+        assert is_generator_with_return_value(n)

From 2fbfe2c21411480c35a99ff5497ee59f2e5e0dbb Mon Sep 17 00:00:00 2001
From: yogendra0sharma <eh.yogendra@gmail.com>
Date: Fri, 21 Aug 2020 12:18:15 +0530
Subject: [PATCH 2974/2974] Removed appveyor.xml no longer needed

---
 appveyor.yml | 25 -------------------------
 1 file changed, 25 deletions(-)
 delete mode 100644 appveyor.yml

diff --git a/appveyor.yml b/appveyor.yml
deleted file mode 100644
index 7fd636864..000000000
--- a/appveyor.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-platform: x86
-version: '{branch}-{build}'
-environment:
-  matrix:
-    - PYTHON: "C:\\Python36"
-      TOX_ENV: py36
-
-branches:
-  only:
-    - master
-    - /d+\.\d+\.\d+[\w\-]*$/
-
-install:
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
-  - "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%"
-  - "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE"
-  - "pip install -U tox"
-
-build: false
-skip_tags: true
-test_script:
-  - "tox -e %TOX_ENV%"
-
-cache:
-  - '%LOCALAPPDATA%\pip\cache'